1 // SPDX-License-Identifier: BSD-3-Clause 2 /* 3 * Copyright 2018 Mellanox Technologies, Ltd 4 */ 5 #include <math.h> 6 7 #include <rte_tailq.h> 8 #include <rte_malloc.h> 9 #include <rte_mtr.h> 10 #include <rte_mtr_driver.h> 11 12 #include <mlx5_devx_cmds.h> 13 #include <mlx5_malloc.h> 14 15 #include "mlx5.h" 16 #include "mlx5_flow.h" 17 18 #ifdef HAVE_MLX5_HWS_SUPPORT 19 20 static void 21 mlx5_flow_meter_uninit_guest(struct rte_eth_dev *dev) 22 { 23 struct mlx5_priv *priv = dev->data->dev_private; 24 25 if (priv->hws_mpool) { 26 if (priv->hws_mpool->action) { 27 claim_zero(mlx5dr_action_destroy(priv->hws_mpool->action)); 28 priv->hws_mpool->action = NULL; 29 } 30 priv->hws_mpool->devx_obj = NULL; 31 priv->hws_mpool->idx_pool = NULL; 32 mlx5_free(priv->hws_mpool); 33 priv->hws_mpool = NULL; 34 } 35 } 36 37 void 38 mlx5_flow_meter_uninit(struct rte_eth_dev *dev) 39 { 40 struct mlx5_priv *priv = dev->data->dev_private; 41 42 if (priv->shared_host) { 43 mlx5_flow_meter_uninit_guest(dev); 44 return; 45 } 46 if (priv->mtr_policy_arr) { 47 mlx5_free(priv->mtr_policy_arr); 48 priv->mtr_policy_arr = NULL; 49 } 50 if (priv->mtr_profile_arr) { 51 mlx5_free(priv->mtr_profile_arr); 52 priv->mtr_profile_arr = NULL; 53 } 54 if (priv->hws_mpool) { 55 mlx5_aso_mtr_queue_uninit(priv->sh, priv->hws_mpool, NULL); 56 mlx5_ipool_destroy(priv->hws_mpool->idx_pool); 57 mlx5_free(priv->hws_mpool); 58 priv->hws_mpool = NULL; 59 } 60 if (priv->mtr_bulk.aso) { 61 mlx5_free(priv->mtr_bulk.aso); 62 priv->mtr_bulk.aso = NULL; 63 priv->mtr_bulk.size = 0; 64 mlx5_aso_queue_uninit(priv->sh, ASO_OPC_MOD_POLICER); 65 } 66 if (priv->mtr_bulk.action) { 67 mlx5dr_action_destroy(priv->mtr_bulk.action); 68 priv->mtr_bulk.action = NULL; 69 } 70 if (priv->mtr_bulk.devx_obj) { 71 claim_zero(mlx5_devx_cmd_destroy(priv->mtr_bulk.devx_obj)); 72 priv->mtr_bulk.devx_obj = NULL; 73 } 74 } 75 76 static int 77 mlx5_flow_meter_init_guest(struct rte_eth_dev *dev) 78 { 79 struct mlx5_priv *priv = dev->data->dev_private; 80 struct rte_eth_dev *host_dev = priv->shared_host; 81 struct mlx5_priv *host_priv = host_dev->data->dev_private; 82 int reg_id = 0; 83 uint32_t flags; 84 int ret = 0; 85 86 MLX5_ASSERT(priv->shared_host); 87 reg_id = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, NULL); 88 if (reg_id < 0) { 89 rte_errno = ENOMEM; 90 ret = -rte_errno; 91 DRV_LOG(ERR, "Meter register is not available."); 92 goto err; 93 } 94 priv->hws_mpool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct mlx5_aso_mtr_pool), 95 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 96 if (!priv->hws_mpool) { 97 rte_errno = ENOMEM; 98 ret = -rte_errno; 99 DRV_LOG(ERR, "Meter ipool allocation failed."); 100 goto err; 101 } 102 MLX5_ASSERT(host_priv->hws_mpool->idx_pool); 103 MLX5_ASSERT(host_priv->hws_mpool->devx_obj); 104 priv->hws_mpool->idx_pool = host_priv->hws_mpool->idx_pool; 105 priv->hws_mpool->devx_obj = host_priv->hws_mpool->devx_obj; 106 flags = MLX5DR_ACTION_FLAG_HWS_RX | MLX5DR_ACTION_FLAG_HWS_TX; 107 if (priv->sh->config.dv_esw_en && priv->master) 108 flags |= MLX5DR_ACTION_FLAG_HWS_FDB; 109 priv->hws_mpool->action = mlx5dr_action_create_aso_meter 110 (priv->dr_ctx, (struct mlx5dr_devx_obj *)priv->hws_mpool->devx_obj, 111 reg_id - REG_C_0, flags); 112 if (!priv->hws_mpool->action) { 113 rte_errno = ENOMEM; 114 ret = -rte_errno; 115 DRV_LOG(ERR, "Meter action creation failed."); 116 goto err; 117 } 118 return 0; 119 err: 120 mlx5_flow_meter_uninit(dev); 121 return ret; 122 } 123 124 int 125 mlx5_flow_meter_init(struct rte_eth_dev *dev, 126 uint32_t nb_meters, 127 uint32_t nb_meter_profiles, 128 uint32_t nb_meter_policies, 129 uint32_t nb_queues) 130 { 131 struct mlx5_priv *priv = dev->data->dev_private; 132 struct mlx5_devx_obj *dcs = NULL; 133 uint32_t log_obj_size; 134 int ret = 0; 135 int reg_id; 136 struct mlx5_aso_mtr *aso; 137 uint32_t i; 138 struct rte_flow_error error; 139 uint32_t flags; 140 uint32_t nb_mtrs = rte_align32pow2(nb_meters); 141 struct mlx5_indexed_pool_config cfg = { 142 .size = sizeof(struct mlx5_aso_mtr), 143 .trunk_size = 1 << 12, 144 .per_core_cache = 1 << 13, 145 .need_lock = 1, 146 .release_mem_en = !!priv->sh->config.reclaim_mode, 147 .malloc = mlx5_malloc, 148 .max_idx = nb_meters, 149 .free = mlx5_free, 150 .type = "mlx5_hw_mtr_mark_action", 151 }; 152 153 if (priv->shared_host) 154 return mlx5_flow_meter_init_guest(dev); 155 if (!nb_meters) { 156 ret = ENOTSUP; 157 rte_flow_error_set(&error, ENOMEM, 158 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 159 NULL, "Meter configuration is invalid."); 160 goto err; 161 } 162 if (!priv->mtr_en || !priv->sh->meter_aso_en) { 163 ret = ENOTSUP; 164 rte_flow_error_set(&error, ENOMEM, 165 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 166 NULL, "Meter ASO is not supported."); 167 goto err; 168 } 169 priv->mtr_config.nb_meters = nb_meters; 170 log_obj_size = rte_log2_u32(nb_meters >> 1); 171 dcs = mlx5_devx_cmd_create_flow_meter_aso_obj 172 (priv->sh->cdev->ctx, priv->sh->cdev->pdn, 173 log_obj_size); 174 if (!dcs) { 175 ret = ENOMEM; 176 rte_flow_error_set(&error, ENOMEM, 177 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 178 NULL, "Meter ASO object allocation failed."); 179 goto err; 180 } 181 priv->mtr_bulk.devx_obj = dcs; 182 reg_id = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, NULL); 183 if (reg_id < 0) { 184 ret = ENOTSUP; 185 rte_flow_error_set(&error, ENOMEM, 186 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 187 NULL, "Meter register is not available."); 188 goto err; 189 } 190 flags = MLX5DR_ACTION_FLAG_HWS_RX | MLX5DR_ACTION_FLAG_HWS_TX; 191 if (priv->sh->config.dv_esw_en && priv->master) 192 flags |= MLX5DR_ACTION_FLAG_HWS_FDB; 193 priv->mtr_bulk.action = mlx5dr_action_create_aso_meter 194 (priv->dr_ctx, (struct mlx5dr_devx_obj *)dcs, 195 reg_id - REG_C_0, flags); 196 if (!priv->mtr_bulk.action) { 197 ret = ENOMEM; 198 rte_flow_error_set(&error, ENOMEM, 199 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 200 NULL, "Meter action creation failed."); 201 goto err; 202 } 203 priv->mtr_bulk.aso = mlx5_malloc(MLX5_MEM_ZERO, 204 sizeof(struct mlx5_aso_mtr) * 205 nb_meters, 206 RTE_CACHE_LINE_SIZE, 207 SOCKET_ID_ANY); 208 if (!priv->mtr_bulk.aso) { 209 ret = ENOMEM; 210 rte_flow_error_set(&error, ENOMEM, 211 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 212 NULL, "Meter bulk ASO allocation failed."); 213 goto err; 214 } 215 priv->mtr_bulk.size = nb_meters; 216 aso = priv->mtr_bulk.aso; 217 for (i = 0; i < priv->mtr_bulk.size; i++) { 218 aso->type = ASO_METER_DIRECT; 219 aso->state = ASO_METER_WAIT; 220 aso->offset = i; 221 aso++; 222 } 223 priv->hws_mpool = mlx5_malloc(MLX5_MEM_ZERO, 224 sizeof(struct mlx5_aso_mtr_pool), 225 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 226 if (!priv->hws_mpool) { 227 ret = ENOMEM; 228 rte_flow_error_set(&error, ENOMEM, 229 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 230 NULL, "Meter ipool allocation failed."); 231 goto err; 232 } 233 priv->hws_mpool->devx_obj = priv->mtr_bulk.devx_obj; 234 priv->hws_mpool->action = priv->mtr_bulk.action; 235 priv->hws_mpool->nb_sq = nb_queues; 236 if (mlx5_aso_mtr_queue_init(priv->sh, priv->hws_mpool, 237 &priv->sh->mtrmng->pools_mng, nb_queues)) { 238 ret = ENOMEM; 239 rte_flow_error_set(&error, ENOMEM, 240 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 241 NULL, "Meter ASO queue allocation failed."); 242 goto err; 243 } 244 /* 245 * No need for local cache if Meter number is a small number. 246 * Since flow insertion rate will be very limited in that case. 247 * Here let's set the number to less than default trunk size 4K. 248 */ 249 if (nb_mtrs <= cfg.trunk_size) { 250 cfg.per_core_cache = 0; 251 cfg.trunk_size = nb_mtrs; 252 } else if (nb_mtrs <= MLX5_HW_IPOOL_SIZE_THRESHOLD) { 253 cfg.per_core_cache = MLX5_HW_IPOOL_CACHE_MIN; 254 } 255 priv->hws_mpool->idx_pool = mlx5_ipool_create(&cfg); 256 if (nb_meter_profiles) { 257 priv->mtr_config.nb_meter_profiles = nb_meter_profiles; 258 priv->mtr_profile_arr = 259 mlx5_malloc(MLX5_MEM_ZERO, 260 sizeof(struct mlx5_flow_meter_profile) * 261 nb_meter_profiles, 262 RTE_CACHE_LINE_SIZE, 263 SOCKET_ID_ANY); 264 if (!priv->mtr_profile_arr) { 265 ret = ENOMEM; 266 rte_flow_error_set(&error, ENOMEM, 267 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 268 NULL, "Meter profile allocation failed."); 269 goto err; 270 } 271 } 272 if (nb_meter_policies) { 273 priv->mtr_config.nb_meter_policies = nb_meter_policies; 274 priv->mtr_policy_arr = 275 mlx5_malloc(MLX5_MEM_ZERO, 276 sizeof(struct mlx5_flow_meter_policy) * 277 nb_meter_policies, 278 RTE_CACHE_LINE_SIZE, 279 SOCKET_ID_ANY); 280 if (!priv->mtr_policy_arr) { 281 ret = ENOMEM; 282 rte_flow_error_set(&error, ENOMEM, 283 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 284 NULL, "Meter policy allocation failed."); 285 goto err; 286 } 287 } 288 return 0; 289 err: 290 mlx5_flow_meter_uninit(dev); 291 return ret; 292 } 293 294 #endif /* HAVE_MLX5_HWS_SUPPORT */ 295 296 static int mlx5_flow_meter_disable(struct rte_eth_dev *dev, 297 uint32_t meter_id, struct rte_mtr_error *error); 298 299 /** 300 * Create the meter action. 301 * 302 * @param priv 303 * Pointer to mlx5_priv. 304 * @param[in] fm 305 * Pointer to flow meter to be converted. 306 * 307 * @return 308 * Pointer to the meter action on success, NULL otherwise. 309 */ 310 static void * 311 mlx5_flow_meter_action_create(struct mlx5_priv *priv, 312 struct mlx5_flow_meter_info *fm) 313 { 314 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER 315 struct mlx5dv_dr_flow_meter_attr mtr_init; 316 uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)]; 317 struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = 318 &fm->profile->srtcm_prm; 319 uint32_t cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir); 320 uint32_t ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir); 321 uint32_t val; 322 enum mlx5_meter_domain domain = 323 fm->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 324 fm->egress ? MLX5_MTR_DOMAIN_EGRESS : 325 MLX5_MTR_DOMAIN_INGRESS; 326 struct mlx5_flow_meter_def_policy *def_policy = 327 priv->sh->mtrmng->def_policy[domain]; 328 329 memset(fmp, 0, MLX5_ST_SZ_BYTES(flow_meter_parameters)); 330 MLX5_SET(flow_meter_parameters, fmp, valid, 1); 331 MLX5_SET(flow_meter_parameters, fmp, bucket_overflow, 1); 332 MLX5_SET(flow_meter_parameters, fmp, 333 start_color, MLX5_FLOW_COLOR_GREEN); 334 MLX5_SET(flow_meter_parameters, fmp, both_buckets_on_green, 0); 335 val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK; 336 MLX5_SET(flow_meter_parameters, fmp, cbs_exponent, val); 337 val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK; 338 MLX5_SET(flow_meter_parameters, fmp, cbs_mantissa, val); 339 val = (cbs_cir >> ASO_DSEG_XIR_EXP_OFFSET) & ASO_DSEG_EXP_MASK; 340 MLX5_SET(flow_meter_parameters, fmp, cir_exponent, val); 341 val = (cbs_cir & ASO_DSEG_MAN_MASK); 342 MLX5_SET(flow_meter_parameters, fmp, cir_mantissa, val); 343 val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK; 344 MLX5_SET(flow_meter_parameters, fmp, ebs_exponent, val); 345 val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK; 346 MLX5_SET(flow_meter_parameters, fmp, ebs_mantissa, val); 347 mtr_init.next_table = def_policy->sub_policy.tbl_rsc->obj; 348 mtr_init.reg_c_index = priv->sh->registers.aso_reg - REG_C_0; 349 mtr_init.flow_meter_parameter = fmp; 350 mtr_init.flow_meter_parameter_sz = 351 MLX5_ST_SZ_BYTES(flow_meter_parameters); 352 mtr_init.active = fm->active_state; 353 return mlx5_glue->dv_create_flow_action_meter(&mtr_init); 354 #else 355 (void)priv; 356 (void)fm; 357 return NULL; 358 #endif 359 } 360 361 /** 362 * Find meter profile by id. 363 * 364 * @param priv 365 * Pointer to mlx5_priv. 366 * @param meter_profile_id 367 * Meter profile id. 368 * 369 * @return 370 * Pointer to the profile found on success, NULL otherwise. 371 */ 372 static struct mlx5_flow_meter_profile * 373 mlx5_flow_meter_profile_find(struct mlx5_priv *priv, uint32_t meter_profile_id) 374 { 375 struct mlx5_flow_meter_profile *fmp; 376 union mlx5_l3t_data data; 377 int32_t ret; 378 379 if (priv->mtr_profile_arr) 380 return &priv->mtr_profile_arr[meter_profile_id]; 381 if (mlx5_l3t_get_entry(priv->mtr_profile_tbl, 382 meter_profile_id, &data) || !data.ptr) 383 return NULL; 384 fmp = data.ptr; 385 /* Remove reference taken by the mlx5_l3t_get_entry. */ 386 ret = mlx5_l3t_clear_entry(priv->mtr_profile_tbl, 387 meter_profile_id); 388 if (!ret || ret == -1) 389 return NULL; 390 return fmp; 391 } 392 393 /** 394 * Validate the MTR profile. 395 * 396 * @param[in] dev 397 * Pointer to Ethernet device. 398 * @param[in] meter_profile_id 399 * Meter profile id. 400 * @param[in] profile 401 * Pointer to meter profile detail. 402 * @param[out] error 403 * Pointer to the error structure. 404 * 405 * @return 406 * 0 on success, a negative errno value otherwise and rte_errno is set. 407 */ 408 static int 409 mlx5_flow_meter_profile_validate(struct rte_eth_dev *dev, 410 uint32_t meter_profile_id, 411 struct rte_mtr_meter_profile *profile, 412 struct rte_mtr_error *error) 413 { 414 struct mlx5_priv *priv = dev->data->dev_private; 415 struct mlx5_flow_meter_profile *fmp; 416 uint32_t ls_factor; 417 int ret; 418 uint64_t cir, cbs; 419 uint64_t eir, ebs; 420 uint64_t pir, pbs; 421 422 /* Profile must not be NULL. */ 423 if (profile == NULL) 424 return -rte_mtr_error_set(error, EINVAL, 425 RTE_MTR_ERROR_TYPE_METER_PROFILE, 426 NULL, "Meter profile is null."); 427 /* Meter profile ID must be valid. */ 428 if (priv->mtr_profile_arr) { 429 if (meter_profile_id >= priv->mtr_config.nb_meter_profiles) 430 return -rte_mtr_error_set(error, EINVAL, 431 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 432 NULL, "Meter profile id not valid."); 433 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 434 /* Meter profile must not exist. */ 435 if (fmp->initialized) 436 return -rte_mtr_error_set(error, EEXIST, 437 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 438 NULL, "Meter profile already exists."); 439 } else { 440 if (meter_profile_id == UINT32_MAX) 441 return -rte_mtr_error_set(error, EINVAL, 442 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 443 NULL, "Meter profile id not valid."); 444 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 445 /* Meter profile must not exist. */ 446 if (fmp) 447 return -rte_mtr_error_set(error, EEXIST, 448 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 449 NULL, "Meter profile already exists."); 450 } 451 if (!priv->sh->meter_aso_en) { 452 /* Old version is even not supported. */ 453 if (!priv->sh->cdev->config.hca_attr.qos.flow_meter_old) 454 return -rte_mtr_error_set(error, ENOTSUP, 455 RTE_MTR_ERROR_TYPE_METER_PROFILE, 456 NULL, "Metering is not supported."); 457 /* Old FW metering only supports srTCM. */ 458 if (profile->alg != RTE_MTR_SRTCM_RFC2697) { 459 return -rte_mtr_error_set(error, ENOTSUP, 460 RTE_MTR_ERROR_TYPE_METER_PROFILE, 461 NULL, "Metering algorithm is not supported."); 462 } else if (profile->srtcm_rfc2697.ebs) { 463 /* EBS is not supported for old metering. */ 464 return -rte_mtr_error_set(error, ENOTSUP, 465 RTE_MTR_ERROR_TYPE_METER_PROFILE, 466 NULL, "EBS is not supported."); 467 } 468 if (profile->packet_mode) 469 return -rte_mtr_error_set(error, ENOTSUP, 470 RTE_MTR_ERROR_TYPE_METER_PROFILE, NULL, 471 "Metering algorithm packet mode is not supported."); 472 } 473 ls_factor = profile->packet_mode ? MLX5_MTRS_PPS_MAP_BPS_SHIFT : 0; 474 switch (profile->alg) { 475 case RTE_MTR_SRTCM_RFC2697: 476 cir = profile->srtcm_rfc2697.cir << ls_factor; 477 cbs = profile->srtcm_rfc2697.cbs << ls_factor; 478 ebs = profile->srtcm_rfc2697.ebs << ls_factor; 479 /* EBS could be zero for old metering. */ 480 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX && 481 cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX && 482 ebs <= MLX5_SRTCM_XBS_MAX) { 483 ret = 0; 484 } else { 485 ret = -rte_mtr_error_set(error, ENOTSUP, 486 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 487 "Profile values out of range."); 488 } 489 break; 490 case RTE_MTR_TRTCM_RFC2698: 491 cir = profile->trtcm_rfc2698.cir << ls_factor; 492 cbs = profile->trtcm_rfc2698.cbs << ls_factor; 493 pir = profile->trtcm_rfc2698.pir << ls_factor; 494 pbs = profile->trtcm_rfc2698.pbs << ls_factor; 495 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX && 496 cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX && 497 pir >= cir && pir <= (MLX5_SRTCM_XIR_MAX * 2) && 498 pbs >= cbs && pbs <= (MLX5_SRTCM_XBS_MAX * 2)) { 499 ret = 0; 500 } else { 501 ret = -rte_mtr_error_set(error, ENOTSUP, 502 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 503 "Profile values out of range."); 504 } 505 break; 506 case RTE_MTR_TRTCM_RFC4115: 507 cir = profile->trtcm_rfc4115.cir << ls_factor; 508 cbs = profile->trtcm_rfc4115.cbs << ls_factor; 509 eir = profile->trtcm_rfc4115.eir << ls_factor; 510 ebs = profile->trtcm_rfc4115.ebs << ls_factor; 511 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX && 512 cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX && 513 eir <= MLX5_SRTCM_XIR_MAX && ebs <= MLX5_SRTCM_XBS_MAX) { 514 ret = 0; 515 } else { 516 ret = -rte_mtr_error_set(error, ENOTSUP, 517 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 518 "Profile values out of range."); 519 } 520 break; 521 default: 522 ret = -rte_mtr_error_set(error, ENOTSUP, 523 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 524 "Unknown metering algorithm."); 525 break; 526 } 527 return ret; 528 } 529 530 /* 531 * Calculate mantissa and exponent for cir / eir. 532 * 533 * @param[in] xir 534 * Value to be calculated. 535 * @param[out] man 536 * Pointer to the mantissa. 537 * @param[out] exp 538 * Pointer to the exp. 539 */ 540 static inline void 541 mlx5_flow_meter_xir_man_exp_calc(int64_t xir, uint8_t *man, uint8_t *exp) 542 { 543 int64_t _xir; 544 int64_t delta = INT64_MAX; 545 uint8_t _man = 0; 546 uint8_t _exp = 0; 547 uint64_t m, e; 548 549 /* Special case xir == 0 ? both exp and mantissa are 0. */ 550 if (xir == 0) { 551 *man = 0; 552 *exp = 0; 553 return; 554 } 555 for (m = 0; m <= 0xFF; m++) { /* man width 8 bit */ 556 for (e = 0; e <= 0x1F; e++) { /* exp width 5bit */ 557 _xir = (1000000000ULL * m) >> e; 558 if (llabs(xir - _xir) <= delta) { 559 delta = llabs(xir - _xir); 560 _man = m; 561 _exp = e; 562 } 563 } 564 } 565 *man = _man; 566 *exp = _exp; 567 } 568 569 /* 570 * Calculate mantissa and exponent for xbs. 571 * 572 * @param[in] xbs 573 * Value to be calculated. 574 * @param[out] man 575 * Pointer to the mantissa. 576 * @param[out] exp 577 * Pointer to the exp. 578 */ 579 static inline void 580 mlx5_flow_meter_xbs_man_exp_calc(uint64_t xbs, uint8_t *man, uint8_t *exp) 581 { 582 int _exp; 583 double _man; 584 585 /* Special case xbs == 0 ? both exp and mantissa are 0. */ 586 if (xbs == 0) { 587 *man = 0; 588 *exp = 0; 589 return; 590 } 591 /* xbs = xbs_mantissa * 2^xbs_exponent */ 592 _man = frexp(xbs, &_exp); 593 if (_exp >= MLX5_MAN_WIDTH) { 594 _man = _man * pow(2, MLX5_MAN_WIDTH); 595 _exp = _exp - MLX5_MAN_WIDTH; 596 } 597 *man = (uint8_t)ceil(_man); 598 *exp = _exp; 599 } 600 601 /** 602 * Fill the prm meter parameter. 603 * 604 * @param[in,out] fmp 605 * Pointer to meter profile to be converted. 606 * @param[out] error 607 * Pointer to the error structure. 608 * 609 * @return 610 * 0 on success, a negative errno value otherwise and rte_errno is set. 611 */ 612 static int 613 mlx5_flow_meter_param_fill(struct mlx5_flow_meter_profile *fmp, 614 struct rte_mtr_error *error) 615 { 616 struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = &fmp->srtcm_prm; 617 uint8_t man, exp; 618 uint32_t cbs_exp, cbs_man, cir_exp, cir_man; 619 uint32_t eir_exp, eir_man, ebs_exp, ebs_man; 620 uint64_t cir, cbs, eir, ebs; 621 622 switch (fmp->profile.alg) { 623 case RTE_MTR_SRTCM_RFC2697: 624 cir = fmp->profile.srtcm_rfc2697.cir; 625 cbs = fmp->profile.srtcm_rfc2697.cbs; 626 eir = 0; 627 ebs = fmp->profile.srtcm_rfc2697.ebs; 628 break; 629 case RTE_MTR_TRTCM_RFC2698: 630 MLX5_ASSERT(fmp->profile.trtcm_rfc2698.pir > 631 fmp->profile.trtcm_rfc2698.cir && 632 fmp->profile.trtcm_rfc2698.pbs > 633 fmp->profile.trtcm_rfc2698.cbs); 634 cir = fmp->profile.trtcm_rfc2698.cir; 635 cbs = fmp->profile.trtcm_rfc2698.cbs; 636 /* EIR / EBS are filled with PIR / PBS. */ 637 eir = fmp->profile.trtcm_rfc2698.pir; 638 ebs = fmp->profile.trtcm_rfc2698.pbs; 639 break; 640 case RTE_MTR_TRTCM_RFC4115: 641 cir = fmp->profile.trtcm_rfc4115.cir; 642 cbs = fmp->profile.trtcm_rfc4115.cbs; 643 eir = fmp->profile.trtcm_rfc4115.eir; 644 ebs = fmp->profile.trtcm_rfc4115.ebs; 645 break; 646 default: 647 return -rte_mtr_error_set(error, EINVAL, 648 RTE_MTR_ERROR_TYPE_METER_PROFILE, NULL, 649 "Metering algorithm mode is invalid"); 650 } 651 /* Adjust the values for PPS mode. */ 652 if (fmp->profile.packet_mode) { 653 cir <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT; 654 cbs <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT; 655 eir <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT; 656 ebs <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT; 657 } 658 /* cir = 8G * cir_mantissa * 1/(2^cir_exponent)) Bytes/Sec */ 659 mlx5_flow_meter_xir_man_exp_calc(cir, &man, &exp); 660 /* Check if cir mantissa is too large. */ 661 if (exp > ASO_DSEG_XIR_EXP_MASK) 662 return -rte_mtr_error_set(error, ENOTSUP, 663 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 664 "meter profile parameter cir is not supported."); 665 cir_man = man; 666 cir_exp = exp; 667 /* cbs = cbs_mantissa * 2^cbs_exponent */ 668 mlx5_flow_meter_xbs_man_exp_calc(cbs, &man, &exp); 669 /* Check if cbs mantissa is too large. */ 670 if (exp > ASO_DSEG_EXP_MASK) 671 return -rte_mtr_error_set(error, ENOTSUP, 672 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 673 "meter profile parameter cbs is not supported."); 674 cbs_man = man; 675 cbs_exp = exp; 676 srtcm->cbs_cir = rte_cpu_to_be_32(cbs_exp << ASO_DSEG_CBS_EXP_OFFSET | 677 cbs_man << ASO_DSEG_CBS_MAN_OFFSET | 678 cir_exp << ASO_DSEG_XIR_EXP_OFFSET | 679 cir_man); 680 mlx5_flow_meter_xir_man_exp_calc(eir, &man, &exp); 681 /* Check if eir mantissa is too large. */ 682 if (exp > ASO_DSEG_XIR_EXP_MASK) 683 return -rte_mtr_error_set(error, ENOTSUP, 684 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 685 "meter profile parameter eir is not supported."); 686 eir_man = man; 687 eir_exp = exp; 688 mlx5_flow_meter_xbs_man_exp_calc(ebs, &man, &exp); 689 /* Check if ebs mantissa is too large. */ 690 if (exp > ASO_DSEG_EXP_MASK) 691 return -rte_mtr_error_set(error, ENOTSUP, 692 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 693 "meter profile parameter ebs is not supported."); 694 ebs_man = man; 695 ebs_exp = exp; 696 srtcm->ebs_eir = rte_cpu_to_be_32(ebs_exp << ASO_DSEG_EBS_EXP_OFFSET | 697 ebs_man << ASO_DSEG_EBS_MAN_OFFSET | 698 eir_exp << ASO_DSEG_XIR_EXP_OFFSET | 699 eir_man); 700 if (srtcm->cbs_cir) 701 fmp->g_support = 1; 702 if (srtcm->ebs_eir) 703 fmp->y_support = 1; 704 return 0; 705 } 706 707 /** 708 * Callback to get MTR capabilities. 709 * 710 * @param[in] dev 711 * Pointer to Ethernet device. 712 * @param[out] cap 713 * Pointer to save MTR capabilities. 714 * @param[out] error 715 * Pointer to the error structure. 716 * 717 * @return 718 * 0 on success, a negative errno value otherwise and rte_errno is set. 719 */ 720 static int 721 mlx5_flow_mtr_cap_get(struct rte_eth_dev *dev, 722 struct rte_mtr_capabilities *cap, 723 struct rte_mtr_error *error __rte_unused) 724 { 725 struct mlx5_priv *priv = dev->data->dev_private; 726 struct mlx5_hca_qos_attr *qattr = &priv->sh->cdev->config.hca_attr.qos; 727 728 if (!priv->mtr_en) 729 return -rte_mtr_error_set(error, ENOTSUP, 730 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 731 "Meter is not supported"); 732 memset(cap, 0, sizeof(*cap)); 733 if (priv->sh->meter_aso_en) { 734 /* 2 meters per one ASO cache line. */ 735 cap->n_max = 1 << (qattr->log_max_num_meter_aso + 1); 736 cap->srtcm_rfc2697_packet_mode_supported = 1; 737 cap->trtcm_rfc2698_packet_mode_supported = 1; 738 cap->trtcm_rfc4115_packet_mode_supported = 1; 739 } else { 740 cap->n_max = 1 << qattr->log_max_flow_meter; 741 } 742 cap->srtcm_rfc2697_byte_mode_supported = 1; 743 cap->trtcm_rfc2698_byte_mode_supported = 1; 744 cap->trtcm_rfc4115_byte_mode_supported = 1; 745 cap->n_shared_max = cap->n_max; 746 cap->identical = 1; 747 cap->shared_identical = 1; 748 cap->shared_n_flows_per_mtr_max = 4 << 20; 749 /* 2M flows can share the same meter. */ 750 cap->chaining_n_mtrs_per_flow_max = 1; /* Chaining is not supported. */ 751 cap->meter_srtcm_rfc2697_n_max = qattr->flow_meter_old ? cap->n_max : 0; 752 cap->meter_trtcm_rfc2698_n_max = qattr->flow_meter_old ? cap->n_max : 0; 753 cap->meter_trtcm_rfc4115_n_max = qattr->flow_meter_old ? cap->n_max : 0; 754 cap->meter_rate_max = 1ULL << 40; /* 1 Tera tokens per sec. */ 755 cap->meter_policy_n_max = cap->n_max; 756 cap->stats_mask = RTE_MTR_STATS_N_BYTES_DROPPED | 757 RTE_MTR_STATS_N_PKTS_DROPPED; 758 return 0; 759 } 760 761 /** 762 * Callback to add MTR profile. 763 * 764 * @param[in] dev 765 * Pointer to Ethernet device. 766 * @param[in] meter_profile_id 767 * Meter profile id. 768 * @param[in] profile 769 * Pointer to meter profile detail. 770 * @param[out] error 771 * Pointer to the error structure. 772 * 773 * @return 774 * 0 on success, a negative errno value otherwise and rte_errno is set. 775 */ 776 static int 777 mlx5_flow_meter_profile_add(struct rte_eth_dev *dev, 778 uint32_t meter_profile_id, 779 struct rte_mtr_meter_profile *profile, 780 struct rte_mtr_error *error) 781 { 782 struct mlx5_priv *priv = dev->data->dev_private; 783 struct mlx5_flow_meter_profile *fmp; 784 union mlx5_l3t_data data; 785 int ret; 786 787 if (!priv->mtr_en) 788 return -rte_mtr_error_set(error, ENOTSUP, 789 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 790 "Meter is not supported"); 791 /* Check input params. */ 792 ret = mlx5_flow_meter_profile_validate(dev, meter_profile_id, 793 profile, error); 794 if (ret) 795 return ret; 796 /* Meter profile memory allocation. */ 797 fmp = mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct mlx5_flow_meter_profile), 798 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 799 if (fmp == NULL) 800 return -rte_mtr_error_set(error, ENOMEM, 801 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 802 NULL, "Meter profile memory " 803 "alloc failed."); 804 /* Fill profile info. */ 805 fmp->id = meter_profile_id; 806 fmp->profile = *profile; 807 /* Fill the flow meter parameters for the PRM. */ 808 ret = mlx5_flow_meter_param_fill(fmp, error); 809 if (ret) 810 goto error; 811 data.ptr = fmp; 812 ret = mlx5_l3t_set_entry(priv->mtr_profile_tbl, 813 meter_profile_id, &data); 814 if (ret) 815 return -rte_mtr_error_set(error, ENOTSUP, 816 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 817 NULL, "Meter profile insert fail."); 818 return 0; 819 error: 820 mlx5_free(fmp); 821 return ret; 822 } 823 824 /** 825 * Callback to delete MTR profile. 826 * 827 * @param[in] dev 828 * Pointer to Ethernet device. 829 * @param[in] meter_profile_id 830 * Meter profile id. 831 * @param[out] error 832 * Pointer to the error structure. 833 * 834 * @return 835 * 0 on success, a negative errno value otherwise and rte_errno is set. 836 */ 837 static int 838 mlx5_flow_meter_profile_delete(struct rte_eth_dev *dev, 839 uint32_t meter_profile_id, 840 struct rte_mtr_error *error) 841 { 842 struct mlx5_priv *priv = dev->data->dev_private; 843 struct mlx5_flow_meter_profile *fmp; 844 845 if (!priv->mtr_en) 846 return -rte_mtr_error_set(error, ENOTSUP, 847 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 848 "Meter is not supported"); 849 /* Meter profile must exist. */ 850 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 851 if (fmp == NULL) 852 return -rte_mtr_error_set(error, ENOENT, 853 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 854 &meter_profile_id, 855 "Meter profile id is invalid."); 856 /* Check profile is unused. */ 857 if (fmp->ref_cnt) 858 return -rte_mtr_error_set(error, EBUSY, 859 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 860 NULL, "Meter profile is in use."); 861 if (mlx5_l3t_clear_entry(priv->mtr_profile_tbl, meter_profile_id)) 862 return -rte_mtr_error_set(error, EBUSY, 863 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 864 NULL, "Meter profile remove fail."); 865 mlx5_free(fmp); 866 return 0; 867 } 868 869 /** 870 * Callback to get MTR profile. 871 * 872 * @param[in] dev 873 * Pointer to Ethernet device. 874 * @param[in] meter_profile_id 875 * Meter profile id. 876 * @param[out] error 877 * Pointer to the error structure. 878 * 879 * @return 880 * A valid handle in case of success, NULL otherwise. 881 */ 882 static struct rte_flow_meter_profile * 883 mlx5_flow_meter_profile_get(struct rte_eth_dev *dev, 884 uint32_t meter_profile_id, 885 struct rte_mtr_error *error) 886 { 887 struct mlx5_priv *priv = dev->data->dev_private; 888 889 if (!priv->mtr_en) { 890 rte_mtr_error_set(error, ENOTSUP, 891 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 892 "Meter is not supported"); 893 return NULL; 894 } 895 return (void *)(uintptr_t)mlx5_flow_meter_profile_find(priv, 896 meter_profile_id); 897 } 898 899 #if defined(HAVE_MLX5_HWS_SUPPORT) 900 /** 901 * Callback to add MTR profile with HWS. 902 * 903 * @param[in] dev 904 * Pointer to Ethernet device. 905 * @param[in] meter_profile_id 906 * Meter profile id. 907 * @param[in] profile 908 * Pointer to meter profile detail. 909 * @param[out] error 910 * Pointer to the error structure. 911 * 912 * @return 913 * 0 on success, a negative errno value otherwise and rte_errno is set. 914 */ 915 static int 916 mlx5_flow_meter_profile_hws_add(struct rte_eth_dev *dev, 917 uint32_t meter_profile_id, 918 struct rte_mtr_meter_profile *profile, 919 struct rte_mtr_error *error) 920 { 921 struct mlx5_priv *priv = dev->data->dev_private; 922 struct mlx5_flow_meter_profile *fmp; 923 int ret; 924 925 if (priv->shared_host) 926 return -rte_mtr_error_set(error, ENOTSUP, RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 927 "Meter profiles cannot be created on guest port"); 928 if (!priv->mtr_profile_arr) 929 return mlx5_flow_meter_profile_add(dev, meter_profile_id, profile, error); 930 /* Check input params. */ 931 ret = mlx5_flow_meter_profile_validate(dev, meter_profile_id, 932 profile, error); 933 if (ret) 934 return ret; 935 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 936 /* Fill profile info. */ 937 fmp->id = meter_profile_id; 938 fmp->profile = *profile; 939 fmp->initialized = 1; 940 /* Fill the flow meter parameters for the PRM. */ 941 return mlx5_flow_meter_param_fill(fmp, error); 942 } 943 944 /** 945 * Callback to delete MTR profile with HWS. 946 * 947 * @param[in] dev 948 * Pointer to Ethernet device. 949 * @param[in] meter_profile_id 950 * Meter profile id. 951 * @param[out] error 952 * Pointer to the error structure. 953 * 954 * @return 955 * 0 on success, a negative errno value otherwise and rte_errno is set. 956 */ 957 static int 958 mlx5_flow_meter_profile_hws_delete(struct rte_eth_dev *dev, 959 uint32_t meter_profile_id, 960 struct rte_mtr_error *error) 961 { 962 struct mlx5_priv *priv = dev->data->dev_private; 963 struct mlx5_flow_meter_profile *fmp; 964 965 if (priv->shared_host) 966 return -rte_mtr_error_set(error, ENOTSUP, RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 967 "Meter profiles cannot be destroyed through guest port"); 968 if (!priv->mtr_profile_arr) 969 return mlx5_flow_meter_profile_delete(dev, meter_profile_id, error); 970 /* Meter profile must exist. */ 971 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 972 if (!fmp->initialized) 973 return -rte_mtr_error_set(error, ENOENT, 974 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 975 &meter_profile_id, 976 "Meter profile id is invalid."); 977 /* Check profile is unused. */ 978 if (fmp->ref_cnt) 979 return -rte_mtr_error_set(error, EBUSY, 980 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 981 NULL, "Meter profile is in use."); 982 memset(fmp, 0, sizeof(struct mlx5_flow_meter_profile)); 983 return 0; 984 } 985 #endif 986 987 /** 988 * Find policy by id. 989 * 990 * @param[in] dev 991 * Pointer to Ethernet device. 992 * @param policy_id 993 * Policy id. 994 * 995 * @return 996 * Pointer to the policy found on success, NULL otherwise. 997 */ 998 struct mlx5_flow_meter_policy * 999 mlx5_flow_meter_policy_find(struct rte_eth_dev *dev, 1000 uint32_t policy_id, 1001 uint32_t *policy_idx) 1002 { 1003 struct mlx5_priv *priv = dev->data->dev_private; 1004 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 1005 union mlx5_l3t_data data; 1006 1007 if (priv->mtr_policy_arr) { 1008 if (policy_idx) 1009 *policy_idx = policy_id; 1010 return &priv->mtr_policy_arr[policy_id]; 1011 } 1012 if (policy_id > MLX5_MAX_SUB_POLICY_TBL_NUM || !priv->policy_idx_tbl) 1013 return NULL; 1014 if (mlx5_l3t_get_entry(priv->policy_idx_tbl, policy_id, &data) || 1015 !data.dword) 1016 return NULL; 1017 if (policy_idx) 1018 *policy_idx = data.dword; 1019 sub_policy = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 1020 data.dword); 1021 /* Remove reference taken by the mlx5_l3t_get_entry. */ 1022 mlx5_l3t_clear_entry(priv->policy_idx_tbl, policy_id); 1023 if (sub_policy) 1024 if (sub_policy->main_policy_id) 1025 return sub_policy->main_policy; 1026 return NULL; 1027 } 1028 1029 /** 1030 * Get the next meter from one meter's policy in hierarchy chain. 1031 * Lock free, mutex should be acquired by caller. 1032 * 1033 * @param[in] priv 1034 * Pointer to mlx5_priv. 1035 * @param[in] policy 1036 * Pointer to flow meter policy. 1037 * @param[out] mtr_idx 1038 * Pointer to Meter index. 1039 * 1040 * @return 1041 * Pointer to the next meter, or NULL when fail. 1042 */ 1043 struct mlx5_flow_meter_info * 1044 mlx5_flow_meter_hierarchy_next_meter(struct mlx5_priv *priv, 1045 struct mlx5_flow_meter_policy *policy, 1046 uint32_t *mtr_idx) 1047 { 1048 int i; 1049 1050 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 1051 if (policy->act_cnt[i].fate_action == MLX5_FLOW_FATE_MTR) 1052 return mlx5_flow_meter_find(priv, 1053 policy->act_cnt[i].next_mtr_id, 1054 mtr_idx); 1055 } 1056 return NULL; 1057 } 1058 1059 /** 1060 * Get the last meter's policy from one meter's policy in hierarchy. 1061 * 1062 * @param[in] dev 1063 * Pointer to Ethernet device. 1064 * @param[in] policy 1065 * Pointer to flow meter policy. 1066 * 1067 * @return 1068 * Pointer to the final meter's policy, or NULL when fail. 1069 */ 1070 struct mlx5_flow_meter_policy * 1071 mlx5_flow_meter_hierarchy_get_final_policy(struct rte_eth_dev *dev, 1072 struct mlx5_flow_meter_policy *policy) 1073 { 1074 struct mlx5_priv *priv = dev->data->dev_private; 1075 struct mlx5_flow_meter_info *next_fm; 1076 struct mlx5_flow_meter_policy *next_policy = policy; 1077 1078 while (next_policy->is_hierarchy) { 1079 rte_spinlock_lock(&next_policy->sl); 1080 next_fm = mlx5_flow_meter_hierarchy_next_meter(priv, next_policy, NULL); 1081 rte_spinlock_unlock(&next_policy->sl); 1082 if (!next_fm || next_fm->def_policy) 1083 return NULL; 1084 next_policy = mlx5_flow_meter_policy_find(dev, 1085 next_fm->policy_id, NULL); 1086 MLX5_ASSERT(next_policy); 1087 } 1088 return next_policy; 1089 } 1090 1091 /** 1092 * Callback to check MTR policy action validate 1093 * 1094 * @param[in] dev 1095 * Pointer to Ethernet device. 1096 * @param[in] actions 1097 * Pointer to meter policy action detail. 1098 * @param[out] error 1099 * Pointer to the error structure. 1100 * 1101 * @return 1102 * 0 on success, a negative errno value otherwise and rte_errno is set. 1103 */ 1104 static int 1105 mlx5_flow_meter_policy_validate(struct rte_eth_dev *dev, 1106 struct rte_mtr_meter_policy_params *policy, 1107 struct rte_mtr_error *error) 1108 { 1109 struct mlx5_priv *priv = dev->data->dev_private; 1110 struct rte_flow_attr attr = { .transfer = priv->sh->config.dv_esw_en ? 1111 1 : 0 }; 1112 bool is_rss = false; 1113 uint8_t policy_mode; 1114 uint8_t domain_bitmap; 1115 int ret; 1116 1117 if (!priv->mtr_en || !priv->sh->meter_aso_en) 1118 return -rte_mtr_error_set(error, ENOTSUP, 1119 RTE_MTR_ERROR_TYPE_METER_POLICY, 1120 NULL, "meter policy unsupported."); 1121 ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr, 1122 &is_rss, &domain_bitmap, &policy_mode, error); 1123 if (ret) 1124 return ret; 1125 return 0; 1126 } 1127 1128 #if defined(HAVE_MLX5_HWS_SUPPORT) 1129 /** 1130 * Callback to check MTR policy action validate for HWS 1131 * 1132 * @param[in] dev 1133 * Pointer to Ethernet device. 1134 * @param[in] actions 1135 * Pointer to meter policy action detail. 1136 * @param[out] error 1137 * Pointer to the error structure. 1138 * 1139 * @return 1140 * 0 on success, a negative errno value otherwise and rte_errno is set. 1141 */ 1142 static int 1143 mlx5_flow_meter_policy_hws_validate(struct rte_eth_dev *dev, 1144 struct rte_mtr_meter_policy_params *policy, 1145 struct rte_mtr_error *error) 1146 { 1147 struct mlx5_priv *priv = dev->data->dev_private; 1148 const struct rte_flow_actions_template_attr attr = { 1149 .transfer = priv->sh->config.dv_esw_en ? 1 : 0 }; 1150 int ret; 1151 int i; 1152 1153 if (!priv->mtr_en || !priv->sh->meter_aso_en) 1154 return -rte_mtr_error_set(error, ENOTSUP, 1155 RTE_MTR_ERROR_TYPE_METER_POLICY, 1156 NULL, "meter policy unsupported."); 1157 for (i = 0; i < RTE_COLORS; i++) { 1158 ret = mlx5_flow_actions_validate(dev, &attr, policy->actions[i], 1159 policy->actions[i], NULL); 1160 if (ret) 1161 return ret; 1162 } 1163 return 0; 1164 } 1165 #endif 1166 1167 static int 1168 __mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev, 1169 uint32_t policy_id, 1170 struct mlx5_flow_meter_policy *mtr_policy, 1171 struct rte_mtr_error *error, 1172 bool clear_l3t) 1173 { 1174 struct mlx5_priv *priv = dev->data->dev_private; 1175 struct mlx5_flow_meter_sub_policy *sub_policy; 1176 uint32_t i, j; 1177 uint16_t sub_policy_num; 1178 1179 rte_spinlock_lock(&mtr_policy->sl); 1180 if (mtr_policy->ref_cnt) { 1181 rte_spinlock_unlock(&mtr_policy->sl); 1182 return -rte_mtr_error_set(error, EBUSY, 1183 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1184 NULL, 1185 "Meter policy object is being used."); 1186 } 1187 mlx5_flow_destroy_policy_rules(dev, mtr_policy); 1188 mlx5_flow_destroy_mtr_acts(dev, mtr_policy); 1189 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 1190 sub_policy_num = (mtr_policy->sub_policy_num >> 1191 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) & 1192 MLX5_MTR_SUB_POLICY_NUM_MASK; 1193 if (sub_policy_num) { 1194 for (j = 0; j < sub_policy_num; j++) { 1195 sub_policy = mtr_policy->sub_policys[i][j]; 1196 if (sub_policy) 1197 mlx5_ipool_free 1198 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 1199 sub_policy->idx); 1200 } 1201 } 1202 } 1203 if (priv->policy_idx_tbl && clear_l3t) { 1204 if (mlx5_l3t_clear_entry(priv->policy_idx_tbl, policy_id)) { 1205 rte_spinlock_unlock(&mtr_policy->sl); 1206 return -rte_mtr_error_set(error, ENOTSUP, 1207 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL, 1208 "Fail to delete policy in index table."); 1209 } 1210 } 1211 rte_spinlock_unlock(&mtr_policy->sl); 1212 return 0; 1213 } 1214 1215 /** 1216 * Callback to add MTR policy. 1217 * 1218 * @param[in] dev 1219 * Pointer to Ethernet device. 1220 * @param[out] policy_id 1221 * Pointer to policy id 1222 * @param[in] actions 1223 * Pointer to meter policy action detail. 1224 * @param[out] error 1225 * Pointer to the error structure. 1226 * 1227 * @return 1228 * 0 on success, a negative errno value otherwise and rte_errno is set. 1229 */ 1230 static int 1231 mlx5_flow_meter_policy_add(struct rte_eth_dev *dev, 1232 uint32_t policy_id, 1233 struct rte_mtr_meter_policy_params *policy, 1234 struct rte_mtr_error *error) 1235 { 1236 struct mlx5_priv *priv = dev->data->dev_private; 1237 struct rte_flow_attr attr = { .transfer = priv->sh->config.dv_esw_en ? 1238 1 : 0 }; 1239 uint32_t sub_policy_idx = 0; 1240 uint32_t policy_idx = 0; 1241 struct mlx5_flow_meter_policy *mtr_policy = NULL; 1242 struct mlx5_flow_meter_sub_policy *sub_policy; 1243 bool is_rss = false; 1244 uint8_t policy_mode; 1245 uint32_t i; 1246 int ret; 1247 uint32_t policy_size = sizeof(struct mlx5_flow_meter_policy); 1248 uint16_t sub_policy_num; 1249 uint8_t domain_bitmap = 0; 1250 union mlx5_l3t_data data; 1251 bool skip_rule = false; 1252 1253 if (!priv->mtr_en) 1254 return -rte_mtr_error_set(error, ENOTSUP, 1255 RTE_MTR_ERROR_TYPE_METER_POLICY, 1256 NULL, "meter policy unsupported. "); 1257 if (policy_id == MLX5_INVALID_POLICY_ID) 1258 return -rte_mtr_error_set(error, ENOTSUP, 1259 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1260 NULL, "policy ID is invalid. "); 1261 if (policy_id == priv->sh->mtrmng->def_policy_id) 1262 return -rte_mtr_error_set(error, EEXIST, 1263 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1264 NULL, "default policy ID exists. "); 1265 mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx); 1266 if (mtr_policy) 1267 return -rte_mtr_error_set(error, EEXIST, 1268 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1269 NULL, "policy ID exists. "); 1270 ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr, 1271 &is_rss, &domain_bitmap, 1272 &policy_mode, error); 1273 if (ret) 1274 return ret; 1275 if (!domain_bitmap) 1276 return -rte_mtr_error_set(error, ENOTSUP, 1277 RTE_MTR_ERROR_TYPE_METER_POLICY, 1278 NULL, "fail to find policy domain."); 1279 if (policy_mode == MLX5_MTR_POLICY_MODE_DEF) { 1280 if (priv->sh->mtrmng->def_policy_id != MLX5_INVALID_POLICY_ID) 1281 return -rte_mtr_error_set(error, EEXIST, 1282 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1283 NULL, "a policy with similar actions " 1284 "is already configured"); 1285 if (mlx5_flow_create_def_policy(dev)) 1286 return -rte_mtr_error_set(error, ENOTSUP, 1287 RTE_MTR_ERROR_TYPE_METER_POLICY, 1288 NULL, 1289 "fail to create non-terminated policy."); 1290 priv->sh->mtrmng->def_policy_id = policy_id; 1291 return 0; 1292 } 1293 if (!priv->sh->meter_aso_en) 1294 return -rte_mtr_error_set(error, ENOTSUP, 1295 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL, 1296 "no ASO capability to support the policy "); 1297 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 1298 if (!(domain_bitmap & (1 << i))) 1299 continue; 1300 /* 1301 * If RSS is found, it means that only the ingress domain can 1302 * be supported. It is invalid to support RSS for one color 1303 * and egress / transfer domain actions for another. Drop and 1304 * jump action should have no impact. 1305 */ 1306 if (is_rss) { 1307 policy_size += 1308 sizeof(struct mlx5_flow_meter_sub_policy *) * 1309 MLX5_MTR_RSS_MAX_SUB_POLICY; 1310 break; 1311 } 1312 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *); 1313 } 1314 mtr_policy = mlx5_malloc(MLX5_MEM_ZERO, policy_size, 1315 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 1316 if (!mtr_policy) 1317 return -rte_mtr_error_set(error, ENOMEM, 1318 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL, 1319 "Memory alloc failed for meter policy."); 1320 if (policy_mode == MLX5_MTR_POLICY_MODE_OG) 1321 mtr_policy->skip_y = 1; 1322 else if (policy_mode == MLX5_MTR_POLICY_MODE_OY) 1323 mtr_policy->skip_g = 1; 1324 policy_size = sizeof(struct mlx5_flow_meter_policy); 1325 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 1326 if (!(domain_bitmap & (1 << i))) 1327 continue; 1328 if (i == MLX5_MTR_DOMAIN_INGRESS) 1329 mtr_policy->ingress = 1; 1330 if (i == MLX5_MTR_DOMAIN_EGRESS) 1331 mtr_policy->egress = 1; 1332 if (i == MLX5_MTR_DOMAIN_TRANSFER) 1333 mtr_policy->transfer = 1; 1334 sub_policy = mlx5_ipool_zmalloc 1335 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 1336 &sub_policy_idx); 1337 if (!sub_policy || 1338 sub_policy_idx > MLX5_MAX_SUB_POLICY_TBL_NUM) 1339 goto policy_add_err; 1340 sub_policy->idx = sub_policy_idx; 1341 sub_policy->main_policy = mtr_policy; 1342 if (!policy_idx) { 1343 policy_idx = sub_policy_idx; 1344 sub_policy->main_policy_id = 1; 1345 } 1346 mtr_policy->sub_policys[i] = 1347 (struct mlx5_flow_meter_sub_policy **) 1348 ((uint8_t *)mtr_policy + policy_size); 1349 mtr_policy->sub_policys[i][0] = sub_policy; 1350 sub_policy_num = (mtr_policy->sub_policy_num >> 1351 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) & 1352 MLX5_MTR_SUB_POLICY_NUM_MASK; 1353 sub_policy_num++; 1354 mtr_policy->sub_policy_num &= ~(MLX5_MTR_SUB_POLICY_NUM_MASK << 1355 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)); 1356 mtr_policy->sub_policy_num |= 1357 (sub_policy_num & MLX5_MTR_SUB_POLICY_NUM_MASK) << 1358 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i); 1359 /* 1360 * If RSS is found, it means that only the ingress domain can 1361 * be supported. It is invalid to support RSS for one color 1362 * and egress / transfer domain actions for another. Drop and 1363 * jump action should have no impact. 1364 */ 1365 if (is_rss) { 1366 mtr_policy->is_rss = 1; 1367 break; 1368 } 1369 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *); 1370 } 1371 rte_spinlock_init(&mtr_policy->sl); 1372 ret = mlx5_flow_create_mtr_acts(dev, mtr_policy, 1373 policy->actions, &attr, error); 1374 if (ret) 1375 goto policy_add_err; 1376 if (mtr_policy->is_hierarchy) { 1377 struct mlx5_flow_meter_policy *final_policy; 1378 1379 final_policy = 1380 mlx5_flow_meter_hierarchy_get_final_policy(dev, mtr_policy); 1381 if (!final_policy) 1382 goto policy_add_err; 1383 skip_rule = (final_policy->is_rss || final_policy->is_queue); 1384 } 1385 /* 1386 * If either Green or Yellow has queue / RSS action, all the policy 1387 * rules will be created later in the flow splitting stage. 1388 */ 1389 if (!is_rss && !mtr_policy->is_queue && !skip_rule) { 1390 /* Create policy rules in HW. */ 1391 ret = mlx5_flow_create_policy_rules(dev, mtr_policy); 1392 if (ret) 1393 goto policy_add_err; 1394 } 1395 data.dword = policy_idx; 1396 if (!priv->policy_idx_tbl) { 1397 priv->policy_idx_tbl = mlx5_l3t_create(MLX5_L3T_TYPE_DWORD); 1398 if (!priv->policy_idx_tbl) 1399 goto policy_add_err; 1400 } 1401 if (mlx5_l3t_set_entry(priv->policy_idx_tbl, policy_id, &data)) 1402 goto policy_add_err; 1403 return 0; 1404 policy_add_err: 1405 if (mtr_policy) { 1406 ret = __mlx5_flow_meter_policy_delete(dev, policy_id, 1407 mtr_policy, error, false); 1408 mlx5_free(mtr_policy); 1409 if (ret) 1410 return ret; 1411 } 1412 return -rte_mtr_error_set(error, ENOTSUP, 1413 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 1414 NULL, "Failed to create devx policy."); 1415 } 1416 1417 /** 1418 * Callback to delete MTR policy. 1419 * 1420 * @param[in] dev 1421 * Pointer to Ethernet device. 1422 * @param[in] policy_id 1423 * Meter policy id. 1424 * @param[out] error 1425 * Pointer to the error structure. 1426 * 1427 * @return 1428 * 0 on success, a negative errno value otherwise and rte_errno is set. 1429 */ 1430 static int 1431 mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev, 1432 uint32_t policy_id, 1433 struct rte_mtr_error *error) 1434 { 1435 struct mlx5_priv *priv = dev->data->dev_private; 1436 struct mlx5_flow_meter_policy *mtr_policy; 1437 uint32_t policy_idx; 1438 int ret; 1439 1440 if (policy_id == priv->sh->mtrmng->def_policy_id) { 1441 if (priv->sh->mtrmng->def_policy_ref_cnt > 0) 1442 return -rte_mtr_error_set(error, ENOTSUP, 1443 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL, 1444 "Meter policy object is being used."); 1445 priv->sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID; 1446 return 0; 1447 } 1448 mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx); 1449 if (!mtr_policy) 1450 return -rte_mtr_error_set(error, ENOTSUP, 1451 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL, 1452 "Meter policy id is invalid. "); 1453 ret = __mlx5_flow_meter_policy_delete(dev, policy_id, mtr_policy, 1454 error, true); 1455 if (ret) 1456 return ret; 1457 mlx5_free(mtr_policy); 1458 return 0; 1459 } 1460 1461 /** 1462 * Callback to get MTR policy. 1463 * 1464 * @param[in] dev 1465 * Pointer to Ethernet device. 1466 * @param[in] policy_id 1467 * Meter policy id. 1468 * @param[out] error 1469 * Pointer to the error structure. 1470 * 1471 * @return 1472 * A valid handle in case of success, NULL otherwise. 1473 */ 1474 static struct rte_flow_meter_policy * 1475 mlx5_flow_meter_policy_get(struct rte_eth_dev *dev, 1476 uint32_t policy_id, 1477 struct rte_mtr_error *error) 1478 { 1479 struct mlx5_priv *priv = dev->data->dev_private; 1480 uint32_t policy_idx; 1481 1482 if (!priv->mtr_en) { 1483 rte_mtr_error_set(error, ENOTSUP, 1484 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1485 "Meter is not supported"); 1486 return NULL; 1487 } 1488 return (void *)(uintptr_t)mlx5_flow_meter_policy_find(dev, policy_id, 1489 &policy_idx); 1490 } 1491 1492 #if defined(HAVE_MLX5_HWS_SUPPORT) 1493 /** 1494 * Callback to delete MTR policy for HWS. 1495 * 1496 * @param[in] dev 1497 * Pointer to Ethernet device. 1498 * @param[in] policy_id 1499 * Meter policy id. 1500 * @param[out] error 1501 * Pointer to the error structure. 1502 * 1503 * @return 1504 * 0 on success, a negative errno value otherwise and rte_errno is set. 1505 */ 1506 static int 1507 mlx5_flow_meter_policy_hws_delete(struct rte_eth_dev *dev, 1508 uint32_t policy_id, 1509 struct rte_mtr_error *error) 1510 { 1511 struct mlx5_priv *priv = dev->data->dev_private; 1512 struct mlx5_flow_meter_policy *mtr_policy; 1513 uint32_t i, j; 1514 uint32_t nb_flows = 0; 1515 int ret; 1516 struct rte_flow_op_attr op_attr = { .postpone = 1 }; 1517 struct rte_flow_op_result result[RTE_COLORS * MLX5_MTR_DOMAIN_MAX]; 1518 1519 if (!priv->mtr_policy_arr) 1520 return mlx5_flow_meter_policy_delete(dev, policy_id, error); 1521 /* Meter policy must exist. */ 1522 mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, NULL); 1523 if (!mtr_policy->initialized) 1524 return -rte_mtr_error_set(error, ENOENT, 1525 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL, 1526 "Meter policy does not exists."); 1527 /* Check policy is unused. */ 1528 if (mtr_policy->ref_cnt) 1529 return -rte_mtr_error_set(error, EBUSY, 1530 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1531 NULL, "Meter policy is in use."); 1532 rte_spinlock_lock(&priv->hw_ctrl_lock); 1533 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 1534 for (j = 0; j < RTE_COLORS; j++) { 1535 if (mtr_policy->hws_flow_rule[i][j]) { 1536 ret = rte_flow_async_destroy(dev->data->port_id, 1537 CTRL_QUEUE_ID(priv), &op_attr, 1538 mtr_policy->hws_flow_rule[i][j], 1539 NULL, NULL); 1540 if (ret < 0) 1541 continue; 1542 nb_flows++; 1543 } 1544 } 1545 } 1546 ret = rte_flow_push(dev->data->port_id, CTRL_QUEUE_ID(priv), NULL); 1547 while (nb_flows && (ret >= 0)) { 1548 ret = rte_flow_pull(dev->data->port_id, 1549 CTRL_QUEUE_ID(priv), result, 1550 nb_flows, NULL); 1551 nb_flows -= ret; 1552 } 1553 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 1554 if (mtr_policy->hws_flow_table[i]) 1555 rte_flow_template_table_destroy(dev->data->port_id, 1556 mtr_policy->hws_flow_table[i], NULL); 1557 } 1558 for (i = 0; i < RTE_COLORS; i++) { 1559 if (mtr_policy->hws_act_templ[i]) 1560 rte_flow_actions_template_destroy(dev->data->port_id, 1561 mtr_policy->hws_act_templ[i], NULL); 1562 } 1563 if (mtr_policy->hws_item_templ) 1564 rte_flow_pattern_template_destroy(dev->data->port_id, 1565 mtr_policy->hws_item_templ, NULL); 1566 rte_spinlock_unlock(&priv->hw_ctrl_lock); 1567 memset(mtr_policy, 0, sizeof(struct mlx5_flow_meter_policy)); 1568 return 0; 1569 } 1570 1571 /** 1572 * Callback to add MTR policy for HWS. 1573 * 1574 * @param[in] dev 1575 * Pointer to Ethernet device. 1576 * @param[out] policy_id 1577 * Pointer to policy id 1578 * @param[in] actions 1579 * Pointer to meter policy action detail. 1580 * @param[out] error 1581 * Pointer to the error structure. 1582 * 1583 * @return 1584 * 0 on success, a negative errno value otherwise and rte_errno is set. 1585 */ 1586 static int 1587 mlx5_flow_meter_policy_hws_add(struct rte_eth_dev *dev, 1588 uint32_t policy_id, 1589 struct rte_mtr_meter_policy_params *policy, 1590 struct rte_mtr_error *error) 1591 { 1592 struct mlx5_priv *priv = dev->data->dev_private; 1593 struct mlx5_flow_meter_policy *mtr_policy = NULL; 1594 const struct rte_flow_action *act; 1595 const struct rte_flow_action_meter *mtr; 1596 struct mlx5_flow_meter_info *fm; 1597 struct mlx5_flow_meter_policy *plc; 1598 uint8_t domain_color = MLX5_MTR_ALL_DOMAIN_BIT; 1599 bool is_rss = false; 1600 bool is_hierarchy = false; 1601 int i, j; 1602 uint32_t nb_colors = 0; 1603 uint32_t nb_flows = 0; 1604 int color; 1605 int ret; 1606 struct rte_flow_pattern_template_attr pta = {0}; 1607 struct rte_flow_actions_template_attr ata = {0}; 1608 struct rte_flow_template_table_attr ta = { {0}, 0 }; 1609 struct rte_flow_op_attr op_attr = { .postpone = 1 }; 1610 struct rte_flow_op_result result[RTE_COLORS * MLX5_MTR_DOMAIN_MAX]; 1611 const uint32_t color_mask = (UINT32_C(1) << MLX5_MTR_COLOR_BITS) - 1; 1612 int color_reg_c_idx = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 1613 0, NULL); 1614 struct rte_flow_item_tag tag_spec = { 1615 .data = 0, 1616 .index = color_reg_c_idx 1617 }; 1618 struct rte_flow_item_tag tag_mask = { 1619 .data = color_mask, 1620 .index = 0xff}; 1621 struct rte_flow_item pattern[] = { 1622 [0] = { 1623 .type = (enum rte_flow_item_type) 1624 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 1625 .spec = &tag_spec, 1626 .mask = &tag_mask, 1627 }, 1628 [1] = { .type = RTE_FLOW_ITEM_TYPE_END } 1629 }; 1630 1631 if (!priv->mtr_policy_arr) 1632 return mlx5_flow_meter_policy_add(dev, policy_id, policy, error); 1633 mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, NULL); 1634 if (mtr_policy->initialized) 1635 return -rte_mtr_error_set(error, EEXIST, 1636 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1637 NULL, "Meter policy already exists."); 1638 if (!policy || 1639 (!policy->actions[RTE_COLOR_RED] && 1640 !policy->actions[RTE_COLOR_YELLOW] && 1641 !policy->actions[RTE_COLOR_GREEN])) 1642 return -rte_mtr_error_set(error, EINVAL, 1643 RTE_MTR_ERROR_TYPE_METER_POLICY, 1644 NULL, "Meter policy actions are not valid."); 1645 if (policy->actions[RTE_COLOR_RED] == RTE_FLOW_ACTION_TYPE_END) 1646 mtr_policy->skip_r = 1; 1647 if (policy->actions[RTE_COLOR_YELLOW] == RTE_FLOW_ACTION_TYPE_END) 1648 mtr_policy->skip_y = 1; 1649 if (policy->actions[RTE_COLOR_GREEN] == RTE_FLOW_ACTION_TYPE_END) 1650 mtr_policy->skip_g = 1; 1651 if (mtr_policy->skip_r && mtr_policy->skip_y && mtr_policy->skip_g) 1652 return -rte_mtr_error_set(error, ENOTSUP, 1653 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1654 NULL, "Meter policy actions are empty."); 1655 for (i = 0; i < RTE_COLORS; i++) { 1656 act = policy->actions[i]; 1657 while (act && act->type != RTE_FLOW_ACTION_TYPE_END) { 1658 switch (act->type) { 1659 case RTE_FLOW_ACTION_TYPE_PORT_ID: 1660 /* fall-through. */ 1661 case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT: 1662 domain_color &= ~(MLX5_MTR_DOMAIN_INGRESS_BIT | 1663 MLX5_MTR_DOMAIN_EGRESS_BIT); 1664 break; 1665 case RTE_FLOW_ACTION_TYPE_RSS: 1666 is_rss = true; 1667 /* fall-through. */ 1668 case RTE_FLOW_ACTION_TYPE_QUEUE: 1669 domain_color &= ~(MLX5_MTR_DOMAIN_EGRESS_BIT | 1670 MLX5_MTR_DOMAIN_TRANSFER_BIT); 1671 break; 1672 case RTE_FLOW_ACTION_TYPE_METER: 1673 is_hierarchy = true; 1674 mtr = act->conf; 1675 fm = mlx5_flow_meter_find(priv, 1676 mtr->mtr_id, NULL); 1677 if (!fm) 1678 return -rte_mtr_error_set(error, EINVAL, 1679 RTE_MTR_ERROR_TYPE_MTR_ID, NULL, 1680 "Meter not found in meter hierarchy."); 1681 plc = mlx5_flow_meter_policy_find(dev, 1682 fm->policy_id, 1683 NULL); 1684 MLX5_ASSERT(plc); 1685 domain_color &= MLX5_MTR_ALL_DOMAIN_BIT & 1686 (plc->ingress << 1687 MLX5_MTR_DOMAIN_INGRESS); 1688 domain_color &= MLX5_MTR_ALL_DOMAIN_BIT & 1689 (plc->egress << 1690 MLX5_MTR_DOMAIN_EGRESS); 1691 domain_color &= MLX5_MTR_ALL_DOMAIN_BIT & 1692 (plc->transfer << 1693 MLX5_MTR_DOMAIN_TRANSFER); 1694 break; 1695 default: 1696 break; 1697 } 1698 act++; 1699 } 1700 } 1701 if (priv->sh->config.dv_esw_en) 1702 domain_color &= ~(MLX5_MTR_DOMAIN_EGRESS_BIT | 1703 MLX5_MTR_DOMAIN_TRANSFER_BIT); 1704 else 1705 domain_color &= ~MLX5_MTR_DOMAIN_TRANSFER_BIT; 1706 if (!domain_color) 1707 return -rte_mtr_error_set(error, ENOTSUP, 1708 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1709 NULL, "Meter policy domains are conflicting."); 1710 mtr_policy->is_rss = is_rss; 1711 mtr_policy->ingress = !!(domain_color & MLX5_MTR_DOMAIN_INGRESS_BIT); 1712 pta.ingress = mtr_policy->ingress; 1713 mtr_policy->egress = !!(domain_color & MLX5_MTR_DOMAIN_EGRESS_BIT); 1714 pta.egress = mtr_policy->egress; 1715 mtr_policy->transfer = !!(domain_color & MLX5_MTR_DOMAIN_TRANSFER_BIT); 1716 pta.transfer = mtr_policy->transfer; 1717 mtr_policy->group = MLX5_FLOW_TABLE_HWS_POLICY - policy_id; 1718 mtr_policy->is_hierarchy = is_hierarchy; 1719 mtr_policy->initialized = 1; 1720 rte_spinlock_lock(&priv->hw_ctrl_lock); 1721 mtr_policy->hws_item_templ = 1722 rte_flow_pattern_template_create(dev->data->port_id, 1723 &pta, pattern, NULL); 1724 if (!mtr_policy->hws_item_templ) 1725 goto policy_add_err; 1726 for (i = 0; i < RTE_COLORS; i++) { 1727 if (mtr_policy->skip_g && i == RTE_COLOR_GREEN) 1728 continue; 1729 if (mtr_policy->skip_y && i == RTE_COLOR_YELLOW) 1730 continue; 1731 if (mtr_policy->skip_r && i == RTE_COLOR_RED) 1732 continue; 1733 mtr_policy->hws_act_templ[nb_colors] = 1734 rte_flow_actions_template_create(dev->data->port_id, 1735 &ata, policy->actions[i], 1736 policy->actions[i], NULL); 1737 if (!mtr_policy->hws_act_templ[nb_colors]) 1738 goto policy_add_err; 1739 nb_colors++; 1740 } 1741 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 1742 memset(&ta, 0, sizeof(ta)); 1743 ta.nb_flows = RTE_COLORS; 1744 ta.flow_attr.group = mtr_policy->group; 1745 if (i == MLX5_MTR_DOMAIN_INGRESS) { 1746 if (!mtr_policy->ingress) 1747 continue; 1748 ta.flow_attr.ingress = 1; 1749 } else if (i == MLX5_MTR_DOMAIN_EGRESS) { 1750 if (!mtr_policy->egress) 1751 continue; 1752 ta.flow_attr.egress = 1; 1753 } else if (i == MLX5_MTR_DOMAIN_TRANSFER) { 1754 if (!mtr_policy->transfer) 1755 continue; 1756 ta.flow_attr.transfer = 1; 1757 } 1758 mtr_policy->hws_flow_table[i] = 1759 rte_flow_template_table_create(dev->data->port_id, 1760 &ta, &mtr_policy->hws_item_templ, 1, 1761 mtr_policy->hws_act_templ, nb_colors, 1762 NULL); 1763 if (!mtr_policy->hws_flow_table[i]) 1764 goto policy_add_err; 1765 nb_colors = 0; 1766 for (j = 0; j < RTE_COLORS; j++) { 1767 if (mtr_policy->skip_g && j == RTE_COLOR_GREEN) 1768 continue; 1769 if (mtr_policy->skip_y && j == RTE_COLOR_YELLOW) 1770 continue; 1771 if (mtr_policy->skip_r && j == RTE_COLOR_RED) 1772 continue; 1773 color = rte_col_2_mlx5_col((enum rte_color)j); 1774 tag_spec.data = color; 1775 mtr_policy->hws_flow_rule[i][j] = 1776 rte_flow_async_create(dev->data->port_id, 1777 CTRL_QUEUE_ID(priv), &op_attr, 1778 mtr_policy->hws_flow_table[i], 1779 pattern, 0, policy->actions[j], 1780 nb_colors, NULL, NULL); 1781 if (!mtr_policy->hws_flow_rule[i][j]) 1782 goto policy_add_err; 1783 nb_colors++; 1784 nb_flows++; 1785 } 1786 ret = rte_flow_push(dev->data->port_id, 1787 CTRL_QUEUE_ID(priv), NULL); 1788 if (ret < 0) 1789 goto policy_add_err; 1790 while (nb_flows) { 1791 ret = rte_flow_pull(dev->data->port_id, 1792 CTRL_QUEUE_ID(priv), result, 1793 nb_flows, NULL); 1794 if (ret < 0) 1795 goto policy_add_err; 1796 for (j = 0; j < ret; j++) { 1797 if (result[j].status == RTE_FLOW_OP_ERROR) 1798 goto policy_add_err; 1799 } 1800 nb_flows -= ret; 1801 } 1802 } 1803 rte_spinlock_unlock(&priv->hw_ctrl_lock); 1804 return 0; 1805 policy_add_err: 1806 rte_spinlock_unlock(&priv->hw_ctrl_lock); 1807 ret = mlx5_flow_meter_policy_hws_delete(dev, policy_id, error); 1808 memset(mtr_policy, 0, sizeof(struct mlx5_flow_meter_policy)); 1809 if (ret) 1810 return ret; 1811 return -rte_mtr_error_set(error, ENOTSUP, 1812 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 1813 NULL, "Failed to create meter policy."); 1814 } 1815 #endif 1816 /** 1817 * Check meter validation. 1818 * 1819 * @param[in] priv 1820 * Pointer to mlx5 private data structure. 1821 * @param[in] meter_id 1822 * Meter id. 1823 * @param[in] params 1824 * Pointer to rte meter parameters. 1825 * @param[out] error 1826 * Pointer to rte meter error structure. 1827 * 1828 * @return 1829 * 0 on success, a negative errno value otherwise and rte_errno is set. 1830 */ 1831 static int 1832 mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id, 1833 struct rte_mtr_params *params, 1834 struct rte_mtr_error *error) 1835 { 1836 /* Meter must use global drop action. */ 1837 if (!priv->sh->dr_drop_action) 1838 return -rte_mtr_error_set(error, ENOTSUP, 1839 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 1840 NULL, 1841 "No drop action ready for meter."); 1842 /* Meter params must not be NULL. */ 1843 if (params == NULL) 1844 return -rte_mtr_error_set(error, EINVAL, 1845 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 1846 NULL, "Meter object params null."); 1847 /* Previous meter color is not supported. */ 1848 if (params->use_prev_mtr_color && !priv->sh->meter_aso_en) 1849 return -rte_mtr_error_set(error, ENOTSUP, 1850 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 1851 NULL, 1852 "Previous meter color " 1853 "not supported."); 1854 if (params->meter_policy_id == MLX5_INVALID_POLICY_ID) 1855 return -rte_mtr_error_set(error, ENOENT, 1856 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1857 NULL, "Meter policy id not valid."); 1858 /* Validate meter id. */ 1859 if (mlx5_flow_meter_find(priv, meter_id, NULL)) 1860 return -rte_mtr_error_set(error, EEXIST, 1861 RTE_MTR_ERROR_TYPE_MTR_ID, NULL, 1862 "Meter object already exists."); 1863 return 0; 1864 } 1865 1866 /** 1867 * Modify the flow meter action. 1868 * 1869 * @param[in] priv 1870 * Pointer to mlx5 private data structure. 1871 * @param[in] fm 1872 * Pointer to flow meter to be modified. 1873 * @param[in] srtcm 1874 * Pointer to meter srtcm description parameter. 1875 * @param[in] modify_bits 1876 * The bit in srtcm to be updated. 1877 * @param[in] active_state 1878 * The state to be updated. 1879 * @return 1880 * 0 on success, o negative value otherwise. 1881 */ 1882 static int 1883 mlx5_flow_meter_action_modify(struct mlx5_priv *priv, 1884 struct mlx5_flow_meter_info *fm, 1885 const struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm, 1886 uint64_t modify_bits, uint32_t active_state, uint32_t is_enable) 1887 { 1888 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER 1889 struct mlx5_dev_ctx_shared *sh = priv->sh; 1890 uint32_t in[MLX5_ST_SZ_DW(flow_meter_parameters)] = { 0 }; 1891 uint32_t *attr; 1892 struct mlx5dv_dr_flow_meter_attr mod_attr = { 0 }; 1893 int ret; 1894 struct mlx5_aso_mtr *aso_mtr = NULL; 1895 uint32_t cbs_cir, ebs_eir, val; 1896 1897 if (sh->meter_aso_en) { 1898 fm->is_enable = !!is_enable; 1899 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm); 1900 ret = mlx5_aso_meter_update_by_wqe(priv, MLX5_HW_INV_QUEUE, 1901 aso_mtr, &priv->mtr_bulk, 1902 NULL, true); 1903 if (ret) 1904 return ret; 1905 ret = mlx5_aso_mtr_wait(priv, aso_mtr, false); 1906 if (ret) 1907 return ret; 1908 } else { 1909 /* Fill command parameters. */ 1910 mod_attr.reg_c_index = sh->registers.aso_reg - REG_C_0; 1911 mod_attr.flow_meter_parameter = in; 1912 mod_attr.flow_meter_parameter_sz = 1913 MLX5_ST_SZ_BYTES(flow_meter_parameters); 1914 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE) 1915 mod_attr.active = !!active_state; 1916 else 1917 mod_attr.active = 0; 1918 attr = in; 1919 cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir); 1920 ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir); 1921 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS) { 1922 val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & 1923 ASO_DSEG_EXP_MASK; 1924 MLX5_SET(flow_meter_parameters, attr, 1925 cbs_exponent, val); 1926 val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & 1927 ASO_DSEG_MAN_MASK; 1928 MLX5_SET(flow_meter_parameters, attr, 1929 cbs_mantissa, val); 1930 } 1931 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR) { 1932 val = (cbs_cir >> ASO_DSEG_XIR_EXP_OFFSET) & 1933 ASO_DSEG_EXP_MASK; 1934 MLX5_SET(flow_meter_parameters, attr, 1935 cir_exponent, val); 1936 val = cbs_cir & ASO_DSEG_MAN_MASK; 1937 MLX5_SET(flow_meter_parameters, attr, 1938 cir_mantissa, val); 1939 } 1940 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_EBS) { 1941 val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & 1942 ASO_DSEG_EXP_MASK; 1943 MLX5_SET(flow_meter_parameters, attr, 1944 ebs_exponent, val); 1945 val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & 1946 ASO_DSEG_MAN_MASK; 1947 MLX5_SET(flow_meter_parameters, attr, 1948 ebs_mantissa, val); 1949 } 1950 /* Apply modifications to meter only if it was created. */ 1951 if (fm->meter_action_g) { 1952 ret = mlx5_glue->dv_modify_flow_action_meter 1953 (fm->meter_action_g, &mod_attr, 1954 rte_cpu_to_be_64(modify_bits)); 1955 if (ret) 1956 return ret; 1957 } 1958 /* Update succeeded modify meter parameters. */ 1959 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE) 1960 fm->active_state = !!active_state; 1961 } 1962 return 0; 1963 #else 1964 (void)priv; 1965 (void)fm; 1966 (void)srtcm; 1967 (void)modify_bits; 1968 (void)active_state; 1969 (void)is_enable; 1970 return -ENOTSUP; 1971 #endif 1972 } 1973 1974 static int 1975 mlx5_flow_meter_stats_enable_update(struct rte_eth_dev *dev, 1976 struct mlx5_flow_meter_info *fm, 1977 uint64_t stats_mask) 1978 { 1979 fm->bytes_dropped = 1980 (stats_mask & RTE_MTR_STATS_N_BYTES_DROPPED) ? 1 : 0; 1981 fm->pkts_dropped = (stats_mask & RTE_MTR_STATS_N_PKTS_DROPPED) ? 1 : 0; 1982 if (fm->bytes_dropped || fm->pkts_dropped) { 1983 if (!fm->drop_cnt) { 1984 /* Alloc policer counters. */ 1985 fm->drop_cnt = mlx5_counter_alloc(dev); 1986 if (!fm->drop_cnt) 1987 return -1; 1988 } 1989 } else { 1990 if (fm->drop_cnt) { 1991 mlx5_counter_free(dev, fm->drop_cnt); 1992 fm->drop_cnt = 0; 1993 } 1994 } 1995 return 0; 1996 } 1997 1998 /** 1999 * Create meter rules. 2000 * 2001 * @param[in] dev 2002 * Pointer to Ethernet device. 2003 * @param[in] meter_id 2004 * Meter id. 2005 * @param[in] params 2006 * Pointer to rte meter parameters. 2007 * @param[in] shared 2008 * Meter shared with other flow or not. 2009 * @param[out] error 2010 * Pointer to rte meter error structure. 2011 * 2012 * @return 2013 * 0 on success, a negative errno value otherwise and rte_errno is set. 2014 */ 2015 static int 2016 mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, 2017 struct rte_mtr_params *params, int shared, 2018 struct rte_mtr_error *error) 2019 { 2020 struct mlx5_priv *priv = dev->data->dev_private; 2021 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; 2022 struct mlx5_flow_meter_profile *fmp; 2023 struct mlx5_flow_meter_info *fm; 2024 /* GCC fails to infer legacy_fm is set when !priv->sh->meter_aso_en. */ 2025 struct mlx5_legacy_flow_meter *legacy_fm = NULL; 2026 struct mlx5_flow_meter_policy *mtr_policy = NULL; 2027 struct mlx5_indexed_pool_config flow_ipool_cfg = { 2028 .size = 0, 2029 .trunk_size = 64, 2030 .need_lock = 1, 2031 .type = "mlx5_flow_mtr_flow_id_pool", 2032 }; 2033 struct mlx5_aso_mtr *aso_mtr; 2034 uint32_t mtr_idx, policy_idx; 2035 union mlx5_l3t_data data; 2036 int ret; 2037 uint8_t domain_bitmap; 2038 uint8_t mtr_id_bits; 2039 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 2040 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 2041 2042 if (!priv->mtr_en) 2043 return -rte_mtr_error_set(error, ENOTSUP, 2044 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 2045 "Meter is not supported"); 2046 /* Validate the parameters. */ 2047 ret = mlx5_flow_meter_validate(priv, meter_id, params, error); 2048 if (ret) 2049 return ret; 2050 /* Meter profile must exist. */ 2051 fmp = mlx5_flow_meter_profile_find(priv, params->meter_profile_id); 2052 if (fmp == NULL) 2053 return -rte_mtr_error_set(error, ENOENT, 2054 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 2055 NULL, "Meter profile id not valid."); 2056 /* Meter policy must exist. */ 2057 if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) { 2058 rte_atomic_fetch_add_explicit 2059 (&priv->sh->mtrmng->def_policy_ref_cnt, 2060 1, rte_memory_order_relaxed); 2061 domain_bitmap = MLX5_MTR_ALL_DOMAIN_BIT; 2062 if (!priv->sh->config.dv_esw_en) 2063 domain_bitmap &= ~MLX5_MTR_DOMAIN_TRANSFER_BIT; 2064 } else { 2065 if (!priv->sh->meter_aso_en) 2066 return -rte_mtr_error_set(error, ENOTSUP, 2067 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 2068 "Part of the policies cannot be " 2069 "supported without ASO "); 2070 mtr_policy = mlx5_flow_meter_policy_find(dev, 2071 params->meter_policy_id, &policy_idx); 2072 if (!mtr_policy) 2073 return -rte_mtr_error_set(error, ENOENT, 2074 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 2075 NULL, "Meter policy id not valid."); 2076 domain_bitmap = (mtr_policy->ingress ? 2077 MLX5_MTR_DOMAIN_INGRESS_BIT : 0) | 2078 (mtr_policy->egress ? 2079 MLX5_MTR_DOMAIN_EGRESS_BIT : 0) | 2080 (mtr_policy->transfer ? 2081 MLX5_MTR_DOMAIN_TRANSFER_BIT : 0); 2082 if (fmp->g_support && mtr_policy->skip_g) 2083 return -rte_mtr_error_set(error, ENOTSUP, 2084 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 2085 NULL, "Meter green policy is empty."); 2086 if (fmp->y_support && mtr_policy->skip_y) 2087 return -rte_mtr_error_set(error, ENOTSUP, 2088 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 2089 NULL, "Meter yellow policy is empty."); 2090 } 2091 /* Allocate the flow meter memory. */ 2092 if (priv->sh->meter_aso_en) { 2093 mtr_idx = mlx5_flow_mtr_alloc(dev); 2094 if (!mtr_idx) 2095 return -rte_mtr_error_set(error, ENOMEM, 2096 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 2097 "Memory alloc failed for meter."); 2098 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx); 2099 fm = &aso_mtr->fm; 2100 } else { 2101 if (fmp->y_support) 2102 return -rte_mtr_error_set(error, ENOMEM, 2103 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 2104 "Unsupported profile with yellow."); 2105 legacy_fm = mlx5_ipool_zmalloc 2106 (priv->sh->ipool[MLX5_IPOOL_MTR], &mtr_idx); 2107 if (legacy_fm == NULL) 2108 return -rte_mtr_error_set(error, ENOMEM, 2109 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 2110 "Memory alloc failed for meter."); 2111 legacy_fm->idx = mtr_idx; 2112 fm = &legacy_fm->fm; 2113 } 2114 mtr_id_bits = MLX5_REG_BITS - rte_clz32(mtr_idx); 2115 if ((mtr_id_bits + priv->sh->mtrmng->max_mtr_flow_bits) > 2116 mtr_reg_bits) { 2117 DRV_LOG(ERR, "Meter number exceeds max limit."); 2118 goto error; 2119 } 2120 if (mtr_id_bits > priv->sh->mtrmng->max_mtr_bits) 2121 priv->sh->mtrmng->max_mtr_bits = mtr_id_bits; 2122 /* Fill the flow meter parameters. */ 2123 fm->meter_id = meter_id; 2124 fm->policy_id = params->meter_policy_id; 2125 fm->profile = fmp; 2126 if (mlx5_flow_meter_stats_enable_update(dev, fm, params->stats_mask)) 2127 goto error; 2128 if (mlx5_flow_create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap)) 2129 goto error; 2130 /* Add to the flow meter list. */ 2131 if (!priv->sh->meter_aso_en) { 2132 MLX5_ASSERT(legacy_fm != NULL); 2133 TAILQ_INSERT_TAIL(fms, legacy_fm, next); 2134 } 2135 /* Add to the flow meter list. */ 2136 fm->active_state = 1; /* Config meter starts as active. */ 2137 fm->is_enable = params->meter_enable; 2138 fm->shared = !!shared; 2139 fm->color_aware = !!params->use_prev_mtr_color; 2140 rte_atomic_fetch_add_explicit(&fm->profile->ref_cnt, 1, rte_memory_order_relaxed); 2141 if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) { 2142 fm->def_policy = 1; 2143 fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg); 2144 if (!fm->flow_ipool) 2145 goto error; 2146 } 2147 rte_spinlock_init(&fm->sl); 2148 /* If ASO meter supported, update ASO flow meter by wqe. */ 2149 if (priv->sh->meter_aso_en) { 2150 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm); 2151 ret = mlx5_aso_meter_update_by_wqe(priv, MLX5_HW_INV_QUEUE, 2152 aso_mtr, &priv->mtr_bulk, NULL, true); 2153 if (ret) 2154 goto error; 2155 if (!priv->mtr_idx_tbl) { 2156 priv->mtr_idx_tbl = 2157 mlx5_l3t_create(MLX5_L3T_TYPE_DWORD); 2158 if (!priv->mtr_idx_tbl) 2159 goto error; 2160 } 2161 data.dword = mtr_idx; 2162 if (mlx5_l3t_set_entry(priv->mtr_idx_tbl, meter_id, &data)) 2163 goto error; 2164 } else if (!params->meter_enable && mlx5_flow_meter_disable(dev, meter_id, error)) { 2165 goto error; 2166 } 2167 fm->active_state = params->meter_enable; 2168 if (mtr_policy) 2169 rte_atomic_fetch_add_explicit(&mtr_policy->ref_cnt, 1, rte_memory_order_relaxed); 2170 return 0; 2171 error: 2172 mlx5_flow_destroy_mtr_tbls(dev, fm); 2173 /* Free policer counters. */ 2174 if (fm->drop_cnt) 2175 mlx5_counter_free(dev, fm->drop_cnt); 2176 if (priv->sh->meter_aso_en) 2177 mlx5_flow_mtr_free(dev, mtr_idx); 2178 else 2179 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], mtr_idx); 2180 return -rte_mtr_error_set(error, ENOTSUP, 2181 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 2182 NULL, "Failed to create devx meter."); 2183 } 2184 2185 #if defined(HAVE_MLX5_HWS_SUPPORT) 2186 /** 2187 * Create meter rules. 2188 * 2189 * @param[in] dev 2190 * Pointer to Ethernet device. 2191 * @param[in] meter_id 2192 * Meter id. 2193 * @param[in] params 2194 * Pointer to rte meter parameters. 2195 * @param[in] shared 2196 * Meter shared with other flow or not. 2197 * @param[out] error 2198 * Pointer to rte meter error structure. 2199 * 2200 * @return 2201 * 0 on success, a negative errno value otherwise and rte_errno is set. 2202 */ 2203 static int 2204 mlx5_flow_meter_hws_create(struct rte_eth_dev *dev, uint32_t meter_id, 2205 struct rte_mtr_params *params, int shared, 2206 struct rte_mtr_error *error) 2207 { 2208 struct mlx5_priv *priv = dev->data->dev_private; 2209 struct mlx5_flow_meter_profile *profile; 2210 struct mlx5_flow_meter_info *fm; 2211 struct mlx5_flow_meter_policy *policy = NULL; 2212 struct mlx5_aso_mtr *aso_mtr; 2213 struct mlx5_hw_q_job *job; 2214 int ret; 2215 2216 if (!priv->mtr_profile_arr || 2217 !priv->mtr_policy_arr || 2218 !priv->mtr_bulk.aso) 2219 return -rte_mtr_error_set(error, ENOTSUP, 2220 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 2221 "Meter bulk array is not allocated."); 2222 /* Meter profile must exist. */ 2223 profile = mlx5_flow_meter_profile_find(priv, params->meter_profile_id); 2224 if (!profile->initialized) 2225 return -rte_mtr_error_set(error, ENOENT, 2226 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 2227 NULL, "Meter profile id not valid."); 2228 /* Meter policy must exist. */ 2229 policy = mlx5_flow_meter_policy_find(dev, 2230 params->meter_policy_id, NULL); 2231 if (!policy->initialized) 2232 return -rte_mtr_error_set(error, ENOENT, 2233 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 2234 NULL, "Meter policy id not valid."); 2235 /* Meter ID must be valid. */ 2236 if (meter_id >= priv->mtr_config.nb_meters) 2237 return -rte_mtr_error_set(error, EINVAL, 2238 RTE_MTR_ERROR_TYPE_MTR_ID, 2239 NULL, "Meter id not valid."); 2240 /* Find ASO object. */ 2241 aso_mtr = mlx5_aso_meter_by_idx(priv, meter_id); 2242 fm = &aso_mtr->fm; 2243 if (fm->initialized) 2244 return -rte_mtr_error_set(error, ENOENT, 2245 RTE_MTR_ERROR_TYPE_MTR_ID, 2246 NULL, "Meter object already exists."); 2247 /* Fill the flow meter parameters. */ 2248 fm->meter_id = meter_id; 2249 fm->policy_id = params->meter_policy_id; 2250 fm->profile = profile; 2251 fm->meter_offset = meter_id; 2252 fm->group = policy->group; 2253 /* Add to the flow meter list. */ 2254 fm->active_state = 1; /* Config meter starts as active. */ 2255 fm->is_enable = params->meter_enable; 2256 fm->shared = !!shared; 2257 fm->initialized = 1; 2258 /* Update ASO flow meter by wqe. */ 2259 job = mlx5_flow_action_job_init(priv, MLX5_HW_INV_QUEUE, NULL, NULL, 2260 NULL, MLX5_HW_Q_JOB_TYPE_CREATE, NULL); 2261 if (!job) 2262 return -rte_mtr_error_set(error, ENOMEM, 2263 RTE_MTR_ERROR_TYPE_MTR_ID, 2264 NULL, "No job context."); 2265 ret = mlx5_aso_meter_update_by_wqe(priv, MLX5_HW_INV_QUEUE, aso_mtr, 2266 &priv->mtr_bulk, job, true); 2267 if (ret) { 2268 flow_hw_job_put(priv, job, CTRL_QUEUE_ID(priv)); 2269 return -rte_mtr_error_set(error, ENOTSUP, 2270 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 2271 NULL, "Failed to create devx meter."); 2272 } 2273 fm->active_state = params->meter_enable; 2274 rte_atomic_fetch_add_explicit(&fm->profile->ref_cnt, 1, rte_memory_order_relaxed); 2275 rte_atomic_fetch_add_explicit(&policy->ref_cnt, 1, rte_memory_order_relaxed); 2276 return 0; 2277 } 2278 #endif 2279 2280 static int 2281 mlx5_flow_meter_params_flush(struct rte_eth_dev *dev, 2282 struct mlx5_flow_meter_info *fm, 2283 uint32_t mtr_idx) 2284 { 2285 struct mlx5_priv *priv = dev->data->dev_private; 2286 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; 2287 struct mlx5_flow_meter_profile *fmp; 2288 struct mlx5_legacy_flow_meter *legacy_fm = NULL; 2289 struct mlx5_flow_meter_policy *mtr_policy; 2290 2291 /* Meter object must not have any owner. */ 2292 MLX5_ASSERT(!fm->ref_cnt); 2293 /* Get meter profile. */ 2294 fmp = fm->profile; 2295 if (fmp == NULL) 2296 return -1; 2297 /* Update dependencies. */ 2298 rte_atomic_fetch_sub_explicit(&fmp->ref_cnt, 1, rte_memory_order_relaxed); 2299 fm->profile = NULL; 2300 /* Remove from list. */ 2301 if (!priv->sh->meter_aso_en) { 2302 legacy_fm = container_of(fm, 2303 struct mlx5_legacy_flow_meter, fm); 2304 TAILQ_REMOVE(fms, legacy_fm, next); 2305 } 2306 /* Free drop counters. */ 2307 if (fm->drop_cnt) 2308 mlx5_counter_free(dev, fm->drop_cnt); 2309 /* Free meter flow table. */ 2310 if (fm->flow_ipool) { 2311 mlx5_ipool_destroy(fm->flow_ipool); 2312 fm->flow_ipool = 0; 2313 } 2314 mlx5_flow_destroy_mtr_tbls(dev, fm); 2315 if (fm->def_policy) 2316 rte_atomic_fetch_sub_explicit(&priv->sh->mtrmng->def_policy_ref_cnt, 2317 1, rte_memory_order_relaxed); 2318 if (priv->sh->meter_aso_en) { 2319 if (!fm->def_policy) { 2320 mtr_policy = mlx5_flow_meter_policy_find(dev, 2321 fm->policy_id, NULL); 2322 if (mtr_policy) 2323 rte_atomic_fetch_sub_explicit(&mtr_policy->ref_cnt, 2324 1, rte_memory_order_relaxed); 2325 fm->policy_id = 0; 2326 } 2327 fm->def_policy = 0; 2328 if (mlx5_l3t_clear_entry(priv->mtr_idx_tbl, fm->meter_id)) 2329 return -1; 2330 mlx5_flow_mtr_free(dev, mtr_idx); 2331 } else { 2332 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], 2333 legacy_fm->idx); 2334 } 2335 return 0; 2336 } 2337 2338 /** 2339 * Destroy meter rules. 2340 * 2341 * @param[in] dev 2342 * Pointer to Ethernet device. 2343 * @param[in] meter_id 2344 * Meter id. 2345 * @param[out] error 2346 * Pointer to rte meter error structure. 2347 * 2348 * @return 2349 * 0 on success, a negative errno value otherwise and rte_errno is set. 2350 */ 2351 static int 2352 mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id, 2353 struct rte_mtr_error *error) 2354 { 2355 struct mlx5_priv *priv = dev->data->dev_private; 2356 struct mlx5_flow_meter_info *fm; 2357 uint32_t mtr_idx = 0; 2358 2359 if (!priv->mtr_en) 2360 return -rte_mtr_error_set(error, ENOTSUP, 2361 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 2362 NULL, 2363 "Meter is not supported"); 2364 /* Meter object must exist. */ 2365 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 2366 if (fm == NULL) 2367 return -rte_mtr_error_set(error, ENOENT, 2368 RTE_MTR_ERROR_TYPE_MTR_ID, 2369 NULL, 2370 "Meter object id not valid."); 2371 /* Meter object must not have any owner. */ 2372 if (fm->ref_cnt > 0) 2373 return -rte_mtr_error_set(error, EBUSY, 2374 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 2375 NULL, 2376 "Meter object is being used."); 2377 /* Destroy the meter profile. */ 2378 if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx)) 2379 return -rte_mtr_error_set(error, EINVAL, 2380 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 2381 NULL, 2382 "MTR object meter profile invalid."); 2383 return 0; 2384 } 2385 2386 /** 2387 * Destroy meter rules. 2388 * 2389 * @param[in] dev 2390 * Pointer to Ethernet device. 2391 * @param[in] meter_id 2392 * Meter id. 2393 * @param[out] error 2394 * Pointer to rte meter error structure. 2395 * 2396 * @return 2397 * 0 on success, a negative errno value otherwise and rte_errno is set. 2398 */ 2399 static int 2400 mlx5_flow_meter_hws_destroy(struct rte_eth_dev *dev, uint32_t meter_id, 2401 struct rte_mtr_error *error) 2402 { 2403 struct mlx5_priv *priv = dev->data->dev_private; 2404 struct mlx5_aso_mtr *aso_mtr; 2405 struct mlx5_flow_meter_info *fm; 2406 struct mlx5_flow_meter_policy *policy; 2407 2408 if (!priv->mtr_profile_arr || 2409 !priv->mtr_policy_arr || 2410 !priv->mtr_bulk.aso) 2411 return -rte_mtr_error_set(error, ENOTSUP, 2412 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL, 2413 "Meter bulk array is not allocated."); 2414 /* Find ASO object. */ 2415 aso_mtr = mlx5_aso_meter_by_idx(priv, meter_id); 2416 fm = &aso_mtr->fm; 2417 if (!fm->initialized) 2418 return -rte_mtr_error_set(error, ENOENT, 2419 RTE_MTR_ERROR_TYPE_MTR_ID, 2420 NULL, "Meter object id not valid."); 2421 /* Meter object must not have any owner. */ 2422 if (fm->ref_cnt > 0) 2423 return -rte_mtr_error_set(error, EBUSY, 2424 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 2425 NULL, "Meter object is being used."); 2426 /* Destroy the meter profile. */ 2427 rte_atomic_fetch_sub_explicit(&fm->profile->ref_cnt, 2428 1, rte_memory_order_relaxed); 2429 /* Destroy the meter policy. */ 2430 policy = mlx5_flow_meter_policy_find(dev, 2431 fm->policy_id, NULL); 2432 rte_atomic_fetch_sub_explicit(&policy->ref_cnt, 2433 1, rte_memory_order_relaxed); 2434 memset(fm, 0, sizeof(struct mlx5_flow_meter_info)); 2435 return 0; 2436 } 2437 2438 /** 2439 * Modify meter state. 2440 * 2441 * @param[in] priv 2442 * Pointer to mlx5 private data structure. 2443 * @param[in] fm 2444 * Pointer to flow meter. 2445 * @param[in] new_state 2446 * New state to update. 2447 * @param[out] error 2448 * Pointer to rte meter error structure. 2449 * 2450 * @return 2451 * 0 on success, a negative errno value otherwise and rte_errno is set. 2452 */ 2453 static int 2454 mlx5_flow_meter_modify_state(struct mlx5_priv *priv, 2455 struct mlx5_flow_meter_info *fm, 2456 uint32_t new_state, 2457 struct rte_mtr_error *error) 2458 { 2459 static const struct mlx5_flow_meter_srtcm_rfc2697_prm srtcm = { 2460 .cbs_cir = RTE_BE32(MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL), 2461 .ebs_eir = 0, 2462 }; 2463 uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS | 2464 MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR; 2465 int ret; 2466 2467 if (new_state == MLX5_FLOW_METER_DISABLE) 2468 ret = mlx5_flow_meter_action_modify(priv, fm, 2469 &srtcm, modify_bits, 0, 0); 2470 else 2471 ret = mlx5_flow_meter_action_modify(priv, fm, 2472 &fm->profile->srtcm_prm, 2473 modify_bits, 0, 1); 2474 if (ret) 2475 return -rte_mtr_error_set(error, -ret, 2476 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 2477 NULL, 2478 new_state ? 2479 "Failed to enable meter." : 2480 "Failed to disable meter."); 2481 return 0; 2482 } 2483 2484 /** 2485 * Callback to enable flow meter. 2486 * 2487 * @param[in] dev 2488 * Pointer to Ethernet device. 2489 * @param[in] meter_id 2490 * Meter id. 2491 * @param[out] error 2492 * Pointer to rte meter error structure. 2493 * 2494 * @return 2495 * 0 on success, a negative errno value otherwise and rte_errno is set. 2496 */ 2497 static int 2498 mlx5_flow_meter_enable(struct rte_eth_dev *dev, 2499 uint32_t meter_id, 2500 struct rte_mtr_error *error) 2501 { 2502 struct mlx5_priv *priv = dev->data->dev_private; 2503 struct mlx5_flow_meter_info *fm; 2504 int ret; 2505 2506 if (!priv->mtr_en) 2507 return -rte_mtr_error_set(error, ENOTSUP, 2508 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 2509 "Meter is not supported"); 2510 /* Meter object must exist. */ 2511 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 2512 if (fm == NULL) 2513 return -rte_mtr_error_set(error, ENOENT, 2514 RTE_MTR_ERROR_TYPE_MTR_ID, 2515 NULL, "Meter not found."); 2516 if (fm->active_state == MLX5_FLOW_METER_ENABLE) 2517 return 0; 2518 ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_ENABLE, 2519 error); 2520 if (!ret) 2521 fm->active_state = MLX5_FLOW_METER_ENABLE; 2522 return ret; 2523 } 2524 2525 /** 2526 * Callback to disable flow meter. 2527 * 2528 * @param[in] dev 2529 * Pointer to Ethernet device. 2530 * @param[in] meter_id 2531 * Meter id. 2532 * @param[out] error 2533 * Pointer to rte meter error structure. 2534 * 2535 * @return 2536 * 0 on success, a negative errno value otherwise and rte_errno is set. 2537 */ 2538 static int 2539 mlx5_flow_meter_disable(struct rte_eth_dev *dev, 2540 uint32_t meter_id, 2541 struct rte_mtr_error *error) 2542 { 2543 struct mlx5_priv *priv = dev->data->dev_private; 2544 struct mlx5_flow_meter_info *fm; 2545 int ret; 2546 2547 if (!priv->mtr_en) 2548 return -rte_mtr_error_set(error, ENOTSUP, 2549 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 2550 "Meter is not supported"); 2551 /* Meter object must exist. */ 2552 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 2553 if (fm == NULL) 2554 return -rte_mtr_error_set(error, ENOENT, 2555 RTE_MTR_ERROR_TYPE_MTR_ID, 2556 NULL, "Meter not found."); 2557 if (fm->active_state == MLX5_FLOW_METER_DISABLE) 2558 return 0; 2559 ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_DISABLE, 2560 error); 2561 if (!ret) 2562 fm->active_state = MLX5_FLOW_METER_DISABLE; 2563 return ret; 2564 } 2565 2566 /** 2567 * Callback to update meter profile. 2568 * 2569 * @param[in] dev 2570 * Pointer to Ethernet device. 2571 * @param[in] meter_id 2572 * Meter id. 2573 * @param[in] meter_profile_id 2574 * To be updated meter profile id. 2575 * @param[out] error 2576 * Pointer to rte meter error structure. 2577 * 2578 * @return 2579 * 0 on success, a negative errno value otherwise and rte_errno is set. 2580 */ 2581 static int 2582 mlx5_flow_meter_profile_update(struct rte_eth_dev *dev, 2583 uint32_t meter_id, 2584 uint32_t meter_profile_id, 2585 struct rte_mtr_error *error) 2586 { 2587 struct mlx5_priv *priv = dev->data->dev_private; 2588 struct mlx5_flow_meter_profile *fmp; 2589 struct mlx5_flow_meter_profile *old_fmp; 2590 struct mlx5_flow_meter_info *fm; 2591 uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS | 2592 MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR; 2593 int ret; 2594 2595 if (!priv->mtr_en) 2596 return -rte_mtr_error_set(error, ENOTSUP, 2597 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 2598 "Meter is not supported"); 2599 /* Meter profile must exist. */ 2600 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 2601 if (fmp == NULL) 2602 return -rte_mtr_error_set(error, ENOENT, 2603 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 2604 NULL, "Meter profile not found."); 2605 /* Meter object must exist. */ 2606 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 2607 if (fm == NULL) 2608 return -rte_mtr_error_set(error, ENOENT, 2609 RTE_MTR_ERROR_TYPE_MTR_ID, 2610 NULL, "Meter not found."); 2611 /* MTR object already set to meter profile id. */ 2612 old_fmp = fm->profile; 2613 if (fmp == old_fmp) 2614 return 0; 2615 /* Update the profile. */ 2616 fm->profile = fmp; 2617 /* Update meter params in HW (if not disabled). */ 2618 if (fm->active_state == MLX5_FLOW_METER_DISABLE) 2619 goto dec_ref_cnt; 2620 ret = mlx5_flow_meter_action_modify(priv, fm, &fm->profile->srtcm_prm, 2621 modify_bits, fm->active_state, 1); 2622 if (ret) { 2623 fm->profile = old_fmp; 2624 return -rte_mtr_error_set(error, -ret, 2625 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 2626 NULL, "Failed to update meter" 2627 " parameters in hardware."); 2628 } 2629 dec_ref_cnt: 2630 old_fmp->ref_cnt--; 2631 fmp->ref_cnt++; 2632 return 0; 2633 } 2634 2635 /** 2636 * Callback to update meter stats mask. 2637 * 2638 * @param[in] dev 2639 * Pointer to Ethernet device. 2640 * @param[in] meter_id 2641 * Meter id. 2642 * @param[in] stats_mask 2643 * To be updated stats_mask. 2644 * @param[out] error 2645 * Pointer to rte meter error structure. 2646 * 2647 * @return 2648 * 0 on success, a negative errno value otherwise and rte_errno is set. 2649 */ 2650 static int 2651 mlx5_flow_meter_stats_update(struct rte_eth_dev *dev, 2652 uint32_t meter_id, 2653 uint64_t stats_mask, 2654 struct rte_mtr_error *error) 2655 { 2656 struct mlx5_priv *priv = dev->data->dev_private; 2657 struct mlx5_flow_meter_info *fm; 2658 2659 if (!priv->mtr_en) 2660 return -rte_mtr_error_set(error, ENOTSUP, 2661 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 2662 "Meter is not supported"); 2663 /* Meter object must exist. */ 2664 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 2665 if (fm == NULL) 2666 return -rte_mtr_error_set(error, ENOENT, 2667 RTE_MTR_ERROR_TYPE_MTR_ID, 2668 NULL, "Meter object id not valid."); 2669 if (mlx5_flow_meter_stats_enable_update(dev, fm, stats_mask)) 2670 return -rte_mtr_error_set(error, ENOENT, 2671 RTE_MTR_ERROR_TYPE_MTR_ID, 2672 NULL, "Fail to allocate " 2673 "counter for meter."); 2674 return 0; 2675 } 2676 2677 /** 2678 * Callback to read meter statistics. 2679 * 2680 * @param[in] dev 2681 * Pointer to Ethernet device. 2682 * @param[in] meter_id 2683 * Meter id. 2684 * @param[out] stats 2685 * Pointer to store the statistics. 2686 * @param[out] stats_mask 2687 * Pointer to store the stats_mask. 2688 * @param[in] clear 2689 * Statistic to be cleared after read or not. 2690 * @param[out] error 2691 * Pointer to rte meter error structure. 2692 * 2693 * @return 2694 * 0 on success, a negative errno value otherwise and rte_errno is set. 2695 */ 2696 static int 2697 mlx5_flow_meter_stats_read(struct rte_eth_dev *dev, 2698 uint32_t meter_id, 2699 struct rte_mtr_stats *stats, 2700 uint64_t *stats_mask, 2701 int clear, 2702 struct rte_mtr_error *error) 2703 { 2704 struct mlx5_priv *priv = dev->data->dev_private; 2705 struct mlx5_flow_meter_info *fm; 2706 uint64_t pkts; 2707 uint64_t bytes; 2708 int ret = 0; 2709 2710 if (!priv->mtr_en) 2711 return -rte_mtr_error_set(error, ENOTSUP, 2712 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 2713 "Meter is not supported"); 2714 /* Meter object must exist. */ 2715 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 2716 if (fm == NULL) 2717 return -rte_mtr_error_set(error, ENOENT, 2718 RTE_MTR_ERROR_TYPE_MTR_ID, 2719 NULL, "Meter object id not valid."); 2720 *stats_mask = 0; 2721 if (fm->bytes_dropped) 2722 *stats_mask |= RTE_MTR_STATS_N_BYTES_DROPPED; 2723 if (fm->pkts_dropped) 2724 *stats_mask |= RTE_MTR_STATS_N_PKTS_DROPPED; 2725 memset(stats, 0, sizeof(*stats)); 2726 if (fm->drop_cnt) { 2727 ret = mlx5_counter_query(dev, fm->drop_cnt, clear, &pkts, 2728 &bytes, NULL); 2729 if (ret) 2730 goto error; 2731 /* If need to read the packets, set it. */ 2732 if (fm->pkts_dropped) 2733 stats->n_pkts_dropped = pkts; 2734 /* If need to read the bytes, set it. */ 2735 if (fm->bytes_dropped) 2736 stats->n_bytes_dropped = bytes; 2737 } 2738 return 0; 2739 error: 2740 return -rte_mtr_error_set(error, ret, RTE_MTR_ERROR_TYPE_STATS, NULL, 2741 "Failed to read meter drop counters."); 2742 } 2743 2744 static const struct rte_mtr_ops mlx5_flow_mtr_ops = { 2745 .capabilities_get = mlx5_flow_mtr_cap_get, 2746 .meter_profile_add = mlx5_flow_meter_profile_add, 2747 .meter_profile_delete = mlx5_flow_meter_profile_delete, 2748 .meter_profile_get = mlx5_flow_meter_profile_get, 2749 .meter_policy_validate = mlx5_flow_meter_policy_validate, 2750 .meter_policy_add = mlx5_flow_meter_policy_add, 2751 .meter_policy_delete = mlx5_flow_meter_policy_delete, 2752 .meter_policy_get = mlx5_flow_meter_policy_get, 2753 .create = mlx5_flow_meter_create, 2754 .destroy = mlx5_flow_meter_destroy, 2755 .meter_enable = mlx5_flow_meter_enable, 2756 .meter_disable = mlx5_flow_meter_disable, 2757 .meter_profile_update = mlx5_flow_meter_profile_update, 2758 .meter_dscp_table_update = NULL, 2759 .stats_update = mlx5_flow_meter_stats_update, 2760 .stats_read = mlx5_flow_meter_stats_read, 2761 }; 2762 2763 #if defined(HAVE_MLX5_HWS_SUPPORT) 2764 static const struct rte_mtr_ops mlx5_flow_mtr_hws_ops = { 2765 .capabilities_get = mlx5_flow_mtr_cap_get, 2766 .meter_profile_add = mlx5_flow_meter_profile_hws_add, 2767 .meter_profile_delete = mlx5_flow_meter_profile_hws_delete, 2768 .meter_profile_get = mlx5_flow_meter_profile_get, 2769 .meter_policy_validate = mlx5_flow_meter_policy_hws_validate, 2770 .meter_policy_add = mlx5_flow_meter_policy_hws_add, 2771 .meter_policy_delete = mlx5_flow_meter_policy_hws_delete, 2772 .meter_policy_get = mlx5_flow_meter_policy_get, 2773 .create = mlx5_flow_meter_hws_create, 2774 .destroy = mlx5_flow_meter_hws_destroy, 2775 .meter_enable = mlx5_flow_meter_enable, 2776 .meter_disable = mlx5_flow_meter_disable, 2777 .meter_profile_update = mlx5_flow_meter_profile_update, 2778 .meter_dscp_table_update = NULL, 2779 .stats_update = NULL, 2780 .stats_read = NULL, 2781 }; 2782 #endif 2783 2784 /** 2785 * Get meter operations. 2786 * 2787 * @param dev 2788 * Pointer to Ethernet device structure. 2789 * @param arg 2790 * Pointer to set the mtr operations. 2791 * 2792 * @return 2793 * Always 0. 2794 */ 2795 int 2796 mlx5_flow_meter_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg) 2797 { 2798 #if defined(HAVE_MLX5_HWS_SUPPORT) 2799 struct mlx5_priv *priv = dev->data->dev_private; 2800 2801 if (priv->sh->config.dv_flow_en == 2) 2802 *(const struct rte_mtr_ops **)arg = &mlx5_flow_mtr_hws_ops; 2803 else 2804 *(const struct rte_mtr_ops **)arg = &mlx5_flow_mtr_ops; 2805 #else 2806 *(const struct rte_mtr_ops **)arg = &mlx5_flow_mtr_ops; 2807 #endif 2808 return 0; 2809 } 2810 2811 /** 2812 * Find meter by id. 2813 * 2814 * @param priv 2815 * Pointer to mlx5_priv. 2816 * @param meter_id 2817 * Meter id. 2818 * @param mtr_idx 2819 * Pointer to Meter index. 2820 * 2821 * @return 2822 * Pointer to the meter info found on success, NULL otherwise. 2823 */ 2824 struct mlx5_flow_meter_info * 2825 mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id, 2826 uint32_t *mtr_idx) 2827 { 2828 struct mlx5_legacy_flow_meter *legacy_fm; 2829 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; 2830 struct mlx5_aso_mtr *aso_mtr; 2831 struct mlx5_aso_mtr_pools_mng *pools_mng = 2832 &priv->sh->mtrmng->pools_mng; 2833 union mlx5_l3t_data data; 2834 uint16_t n_valid; 2835 2836 if (priv->mtr_bulk.aso) { 2837 if (mtr_idx) 2838 *mtr_idx = meter_id; 2839 aso_mtr = priv->mtr_bulk.aso + meter_id; 2840 return &aso_mtr->fm; 2841 } 2842 if (priv->sh->meter_aso_en) { 2843 rte_rwlock_read_lock(&pools_mng->resize_mtrwl); 2844 n_valid = pools_mng->n_valid; 2845 rte_rwlock_read_unlock(&pools_mng->resize_mtrwl); 2846 if (!n_valid || !priv->mtr_idx_tbl || 2847 (mlx5_l3t_get_entry(priv->mtr_idx_tbl, meter_id, &data) || 2848 !data.dword)) 2849 return NULL; 2850 if (mtr_idx) 2851 *mtr_idx = data.dword; 2852 aso_mtr = mlx5_aso_meter_by_idx(priv, data.dword); 2853 /* Remove reference taken by the mlx5_l3t_get_entry. */ 2854 mlx5_l3t_clear_entry(priv->mtr_idx_tbl, meter_id); 2855 if (!aso_mtr || aso_mtr->state == ASO_METER_FREE) 2856 return NULL; 2857 return &aso_mtr->fm; 2858 } 2859 TAILQ_FOREACH(legacy_fm, fms, next) 2860 if (meter_id == legacy_fm->fm.meter_id) { 2861 if (mtr_idx) 2862 *mtr_idx = legacy_fm->idx; 2863 return &legacy_fm->fm; 2864 } 2865 return NULL; 2866 } 2867 2868 /** 2869 * Find meter by index. 2870 * 2871 * @param priv 2872 * Pointer to mlx5_priv. 2873 * @param idx 2874 * Meter index. 2875 * 2876 * @return 2877 * Pointer to the meter info found on success, NULL otherwise. 2878 */ 2879 struct mlx5_flow_meter_info * 2880 flow_dv_meter_find_by_idx(struct mlx5_priv *priv, uint32_t idx) 2881 { 2882 struct mlx5_aso_mtr *aso_mtr; 2883 2884 if (priv->sh->meter_aso_en) { 2885 aso_mtr = mlx5_aso_meter_by_idx(priv, idx); 2886 if (!aso_mtr) 2887 return NULL; 2888 return &aso_mtr->fm; 2889 } else { 2890 return mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], idx); 2891 } 2892 } 2893 2894 /** 2895 * Attach meter to flow. 2896 * Unidirectional Meter creation can only be done 2897 * when flow direction is known, i.e. when calling meter_attach. 2898 * 2899 * @param [in] priv 2900 * Pointer to mlx5 private data. 2901 * @param[in] fm 2902 * Pointer to flow meter. 2903 * @param [in] attr 2904 * Pointer to flow attributes. 2905 * @param [out] error 2906 * Pointer to error structure. 2907 * 2908 * @return 2909 * 0 on success, a negative errno value otherwise and rte_errno is set. 2910 */ 2911 int 2912 mlx5_flow_meter_attach(struct mlx5_priv *priv, 2913 struct mlx5_flow_meter_info *fm, 2914 const struct rte_flow_attr *attr, 2915 struct rte_flow_error *error) 2916 { 2917 int ret = 0; 2918 2919 if (priv->sh->meter_aso_en) { 2920 struct mlx5_aso_mtr *aso_mtr; 2921 2922 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm); 2923 if (mlx5_aso_mtr_wait(priv, aso_mtr, false)) { 2924 return rte_flow_error_set(error, ENOENT, 2925 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2926 NULL, 2927 "Timeout in meter configuration"); 2928 } 2929 rte_spinlock_lock(&fm->sl); 2930 if (fm->shared || !fm->ref_cnt) { 2931 fm->ref_cnt++; 2932 } else { 2933 rte_flow_error_set(error, EINVAL, 2934 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 2935 "Meter cannot be shared"); 2936 ret = -1; 2937 } 2938 rte_spinlock_unlock(&fm->sl); 2939 } else { 2940 rte_spinlock_lock(&fm->sl); 2941 if (fm->meter_action_g) { 2942 if (fm->shared && 2943 attr->transfer == fm->transfer && 2944 attr->ingress == fm->ingress && 2945 attr->egress == fm->egress) { 2946 fm->ref_cnt++; 2947 } else { 2948 rte_flow_error_set(error, EINVAL, 2949 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 2950 fm->shared ? 2951 "Meter attr not match." : 2952 "Meter cannot be shared."); 2953 ret = -1; 2954 } 2955 } else { 2956 fm->ingress = attr->ingress; 2957 fm->egress = attr->egress; 2958 fm->transfer = attr->transfer; 2959 fm->ref_cnt = 1; 2960 /* This also creates the meter object. */ 2961 fm->meter_action_g = mlx5_flow_meter_action_create(priv, 2962 fm); 2963 if (!fm->meter_action_g) { 2964 fm->ref_cnt = 0; 2965 fm->ingress = 0; 2966 fm->egress = 0; 2967 fm->transfer = 0; 2968 rte_flow_error_set(error, EINVAL, 2969 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 2970 "Meter action create failed."); 2971 ret = -1; 2972 } 2973 } 2974 rte_spinlock_unlock(&fm->sl); 2975 } 2976 return ret ? -rte_errno : 0; 2977 } 2978 2979 /** 2980 * Detach meter from flow. 2981 * 2982 * @param [in] priv 2983 * Pointer to mlx5 private data. 2984 * @param [in] fm 2985 * Pointer to flow meter. 2986 */ 2987 void 2988 mlx5_flow_meter_detach(struct mlx5_priv *priv, 2989 struct mlx5_flow_meter_info *fm) 2990 { 2991 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER 2992 rte_spinlock_lock(&fm->sl); 2993 MLX5_ASSERT(fm->ref_cnt); 2994 if (--fm->ref_cnt == 0 && !priv->sh->meter_aso_en) { 2995 mlx5_glue->destroy_flow_action(fm->meter_action_g); 2996 fm->meter_action_g = NULL; 2997 fm->ingress = 0; 2998 fm->egress = 0; 2999 fm->transfer = 0; 3000 } 3001 rte_spinlock_unlock(&fm->sl); 3002 #else 3003 (void)priv; 3004 (void)fm; 3005 #endif 3006 } 3007 3008 /** 3009 * Flush meter with Rx queue configuration. 3010 * 3011 * @param[in] dev 3012 * Pointer to Ethernet device. 3013 */ 3014 void 3015 mlx5_flow_meter_rxq_flush(struct rte_eth_dev *dev) 3016 { 3017 struct mlx5_priv *priv = dev->data->dev_private; 3018 struct mlx5_flow_meter_sub_policy *sub_policy; 3019 struct mlx5_flow_meter_policy *mtr_policy; 3020 void *entry; 3021 uint32_t i, policy_idx; 3022 3023 if (!priv->mtr_en) 3024 return; 3025 if (priv->policy_idx_tbl) { 3026 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) { 3027 policy_idx = *(uint32_t *)entry; 3028 sub_policy = mlx5_ipool_get 3029 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 3030 policy_idx); 3031 if (!sub_policy || !sub_policy->main_policy) 3032 continue; 3033 mtr_policy = sub_policy->main_policy; 3034 if (mtr_policy->is_queue || mtr_policy->is_rss) 3035 mlx5_flow_destroy_sub_policy_with_rxq(dev, 3036 mtr_policy); 3037 } 3038 } 3039 } 3040 3041 /** 3042 * Iterate a meter hierarchy and flush all meters and policies if possible. 3043 * 3044 * @param[in] dev 3045 * Pointer to Ethernet device. 3046 * @param[in] fm 3047 * Pointer to flow meter. 3048 * @param[in] mtr_idx 3049 * .Meter's index 3050 * @param[out] error 3051 * Pointer to rte meter error structure. 3052 * 3053 * @return 3054 * 0 on success, a negative errno value otherwise and rte_errno is set. 3055 */ 3056 static int 3057 mlx5_flow_meter_flush_hierarchy(struct rte_eth_dev *dev, 3058 struct mlx5_flow_meter_info *fm, 3059 uint32_t mtr_idx, 3060 struct rte_mtr_error *error) 3061 { 3062 struct mlx5_priv *priv = dev->data->dev_private; 3063 struct mlx5_flow_meter_policy *policy; 3064 uint32_t policy_id; 3065 struct mlx5_flow_meter_info *next_fm; 3066 uint32_t next_mtr_idx; 3067 struct mlx5_flow_meter_policy *next_policy = NULL; 3068 3069 policy = mlx5_flow_meter_policy_find(dev, fm->policy_id, NULL); 3070 MLX5_ASSERT(policy); 3071 while (!fm->ref_cnt && policy->is_hierarchy) { 3072 policy_id = fm->policy_id; 3073 next_fm = mlx5_flow_meter_hierarchy_next_meter(priv, policy, &next_mtr_idx); 3074 if (next_fm) { 3075 next_policy = mlx5_flow_meter_policy_find(dev, 3076 next_fm->policy_id, 3077 NULL); 3078 MLX5_ASSERT(next_policy); 3079 } 3080 if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx)) 3081 return -rte_mtr_error_set(error, ENOTSUP, 3082 RTE_MTR_ERROR_TYPE_MTR_ID, 3083 NULL, 3084 "Failed to flush meter."); 3085 if (policy->ref_cnt) 3086 break; 3087 if (__mlx5_flow_meter_policy_delete(dev, policy_id, 3088 policy, error, true)) 3089 return -rte_errno; 3090 mlx5_free(policy); 3091 if (!next_fm || !next_policy) 3092 break; 3093 fm = next_fm; 3094 mtr_idx = next_mtr_idx; 3095 policy = next_policy; 3096 } 3097 return 0; 3098 } 3099 3100 /** 3101 * Flush all the hierarchy meters and their policies. 3102 * 3103 * @param[in] dev 3104 * Pointer to Ethernet device. 3105 * @param[out] error 3106 * Pointer to rte meter error structure. 3107 * 3108 * @return 3109 * 0 on success, a negative errno value otherwise and rte_errno is set. 3110 */ 3111 static int 3112 mlx5_flow_meter_flush_all_hierarchies(struct rte_eth_dev *dev, 3113 struct rte_mtr_error *error) 3114 { 3115 struct mlx5_priv *priv = dev->data->dev_private; 3116 struct mlx5_flow_meter_info *fm; 3117 struct mlx5_flow_meter_policy *policy; 3118 struct mlx5_flow_meter_sub_policy *sub_policy; 3119 struct mlx5_flow_meter_info *next_fm; 3120 struct mlx5_aso_mtr *aso_mtr; 3121 uint32_t mtr_idx = 0; 3122 uint32_t i, policy_idx; 3123 void *entry; 3124 3125 if (!priv->mtr_idx_tbl || !priv->policy_idx_tbl) 3126 return 0; 3127 MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) { 3128 mtr_idx = *(uint32_t *)entry; 3129 if (!mtr_idx) 3130 continue; 3131 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx); 3132 fm = &aso_mtr->fm; 3133 if (fm->ref_cnt || fm->def_policy) 3134 continue; 3135 if (mlx5_flow_meter_flush_hierarchy(dev, fm, mtr_idx, error)) 3136 return -rte_errno; 3137 } 3138 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) { 3139 policy_idx = *(uint32_t *)entry; 3140 sub_policy = mlx5_ipool_get 3141 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 3142 policy_idx); 3143 if (!sub_policy) 3144 return -rte_mtr_error_set(error, 3145 EINVAL, 3146 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 3147 NULL, "Meter policy invalid."); 3148 policy = sub_policy->main_policy; 3149 if (!policy || !policy->is_hierarchy || policy->ref_cnt) 3150 continue; 3151 next_fm = mlx5_flow_meter_hierarchy_next_meter(priv, policy, &mtr_idx); 3152 if (__mlx5_flow_meter_policy_delete(dev, i, policy, 3153 error, true)) 3154 return -rte_mtr_error_set(error, 3155 EINVAL, 3156 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 3157 NULL, "Meter policy invalid."); 3158 mlx5_free(policy); 3159 if (!next_fm || next_fm->ref_cnt || next_fm->def_policy) 3160 continue; 3161 if (mlx5_flow_meter_flush_hierarchy(dev, next_fm, 3162 mtr_idx, error)) 3163 return -rte_errno; 3164 } 3165 return 0; 3166 } 3167 /** 3168 * Flush meter configuration. 3169 * 3170 * @param[in] dev 3171 * Pointer to Ethernet device. 3172 * @param[out] error 3173 * Pointer to rte meter error structure. 3174 * 3175 * @return 3176 * 0 on success, a negative errno value otherwise and rte_errno is set. 3177 */ 3178 int 3179 mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error) 3180 { 3181 struct mlx5_priv *priv = dev->data->dev_private; 3182 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; 3183 struct mlx5_flow_meter_profile *fmp; 3184 struct mlx5_legacy_flow_meter *legacy_fm; 3185 struct mlx5_flow_meter_info *fm; 3186 struct mlx5_flow_meter_sub_policy *sub_policy; 3187 void *tmp; 3188 uint32_t i, mtr_idx, policy_idx; 3189 void *entry; 3190 struct mlx5_aso_mtr *aso_mtr; 3191 3192 if (!priv->mtr_en) 3193 return 0; 3194 if (priv->sh->meter_aso_en) { 3195 if (mlx5_flow_meter_flush_all_hierarchies(dev, error)) 3196 return -rte_errno; 3197 if (priv->mtr_idx_tbl) { 3198 MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) { 3199 mtr_idx = *(uint32_t *)entry; 3200 if (mtr_idx) { 3201 aso_mtr = 3202 mlx5_aso_meter_by_idx(priv, mtr_idx); 3203 fm = &aso_mtr->fm; 3204 (void)mlx5_flow_meter_params_flush(dev, 3205 fm, mtr_idx); 3206 } 3207 } 3208 mlx5_l3t_destroy(priv->mtr_idx_tbl); 3209 priv->mtr_idx_tbl = NULL; 3210 } 3211 } else { 3212 RTE_TAILQ_FOREACH_SAFE(legacy_fm, fms, next, tmp) { 3213 fm = &legacy_fm->fm; 3214 if (mlx5_flow_meter_params_flush(dev, fm, 0)) 3215 return -rte_mtr_error_set(error, EINVAL, 3216 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 3217 NULL, "MTR object meter profile invalid."); 3218 } 3219 } 3220 if (priv->mtr_bulk.aso) { 3221 for (i = 0; i < priv->mtr_config.nb_meters; i++) { 3222 aso_mtr = mlx5_aso_meter_by_idx(priv, i); 3223 fm = &aso_mtr->fm; 3224 if (fm->initialized) 3225 mlx5_flow_meter_hws_destroy(dev, i, error); 3226 } 3227 } 3228 if (priv->policy_idx_tbl) { 3229 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) { 3230 policy_idx = *(uint32_t *)entry; 3231 sub_policy = mlx5_ipool_get 3232 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 3233 policy_idx); 3234 if (!sub_policy) 3235 return -rte_mtr_error_set(error, 3236 EINVAL, 3237 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 3238 NULL, "MTR object " 3239 "meter policy invalid."); 3240 if (__mlx5_flow_meter_policy_delete(dev, i, 3241 sub_policy->main_policy, 3242 error, true)) 3243 return -rte_mtr_error_set(error, 3244 EINVAL, 3245 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 3246 NULL, "MTR object " 3247 "meter policy invalid."); 3248 mlx5_free(sub_policy->main_policy); 3249 } 3250 mlx5_l3t_destroy(priv->policy_idx_tbl); 3251 priv->policy_idx_tbl = NULL; 3252 } 3253 #if defined(HAVE_MLX5_HWS_SUPPORT) 3254 if (priv->mtr_policy_arr) { 3255 struct mlx5_flow_meter_policy *policy; 3256 3257 for (i = 0; i < priv->mtr_config.nb_meter_policies; i++) { 3258 policy = mlx5_flow_meter_policy_find(dev, i, 3259 &policy_idx); 3260 if (policy->initialized) { 3261 mlx5_flow_meter_policy_hws_delete(dev, i, 3262 error); 3263 } 3264 } 3265 } 3266 #endif 3267 if (priv->mtr_profile_tbl) { 3268 MLX5_L3T_FOREACH(priv->mtr_profile_tbl, i, entry) { 3269 fmp = entry; 3270 if (mlx5_flow_meter_profile_delete(dev, fmp->id, 3271 error)) 3272 return -rte_mtr_error_set(error, EINVAL, 3273 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 3274 NULL, "Fail to destroy " 3275 "meter profile."); 3276 } 3277 mlx5_l3t_destroy(priv->mtr_profile_tbl); 3278 priv->mtr_profile_tbl = NULL; 3279 } 3280 #if defined(HAVE_MLX5_HWS_SUPPORT) 3281 if (priv->mtr_profile_arr) { 3282 for (i = 0; i < priv->mtr_config.nb_meter_profiles; i++) { 3283 fmp = mlx5_flow_meter_profile_find(priv, i); 3284 if (fmp->initialized) { 3285 mlx5_flow_meter_profile_hws_delete(dev, i, 3286 error); 3287 } 3288 } 3289 } 3290 #endif 3291 /* Delete default policy table. */ 3292 mlx5_flow_destroy_def_policy(dev); 3293 if (priv->sh->refcnt == 1) 3294 mlx5_flow_destroy_mtr_drop_tbls(dev); 3295 #ifdef HAVE_MLX5_HWS_SUPPORT 3296 /* Destroy HWS configuration. */ 3297 mlx5_flow_meter_uninit(dev); 3298 #endif 3299 return 0; 3300 } 3301