1 // SPDX-License-Identifier: BSD-3-Clause 2 /* 3 * Copyright 2018 Mellanox Technologies, Ltd 4 */ 5 #include <math.h> 6 7 #include <rte_tailq.h> 8 #include <rte_malloc.h> 9 #include <rte_mtr.h> 10 #include <rte_mtr_driver.h> 11 12 #include <mlx5_devx_cmds.h> 13 #include <mlx5_malloc.h> 14 15 #include "mlx5.h" 16 #include "mlx5_flow.h" 17 18 #ifdef HAVE_MLX5_HWS_SUPPORT 19 20 static void 21 mlx5_flow_meter_uninit_guest(struct rte_eth_dev *dev) 22 { 23 struct mlx5_priv *priv = dev->data->dev_private; 24 25 if (priv->hws_mpool) { 26 if (priv->hws_mpool->action) { 27 claim_zero(mlx5dr_action_destroy(priv->hws_mpool->action)); 28 priv->hws_mpool->action = NULL; 29 } 30 priv->hws_mpool->devx_obj = NULL; 31 priv->hws_mpool->idx_pool = NULL; 32 mlx5_free(priv->hws_mpool); 33 priv->hws_mpool = NULL; 34 } 35 } 36 37 void 38 mlx5_flow_meter_uninit(struct rte_eth_dev *dev) 39 { 40 struct mlx5_priv *priv = dev->data->dev_private; 41 42 if (priv->shared_host) { 43 mlx5_flow_meter_uninit_guest(dev); 44 return; 45 } 46 if (priv->mtr_policy_arr) { 47 mlx5_free(priv->mtr_policy_arr); 48 priv->mtr_policy_arr = NULL; 49 } 50 if (priv->mtr_profile_arr) { 51 mlx5_free(priv->mtr_profile_arr); 52 priv->mtr_profile_arr = NULL; 53 } 54 if (priv->hws_mpool) { 55 mlx5_aso_mtr_queue_uninit(priv->sh, priv->hws_mpool, NULL); 56 mlx5_ipool_destroy(priv->hws_mpool->idx_pool); 57 mlx5_free(priv->hws_mpool); 58 priv->hws_mpool = NULL; 59 } 60 if (priv->mtr_bulk.aso) { 61 mlx5_free(priv->mtr_bulk.aso); 62 priv->mtr_bulk.aso = NULL; 63 priv->mtr_bulk.size = 0; 64 mlx5_aso_queue_uninit(priv->sh, ASO_OPC_MOD_POLICER); 65 } 66 if (priv->mtr_bulk.action) { 67 mlx5dr_action_destroy(priv->mtr_bulk.action); 68 priv->mtr_bulk.action = NULL; 69 } 70 if (priv->mtr_bulk.devx_obj) { 71 claim_zero(mlx5_devx_cmd_destroy(priv->mtr_bulk.devx_obj)); 72 priv->mtr_bulk.devx_obj = NULL; 73 } 74 } 75 76 static int 77 mlx5_flow_meter_init_guest(struct rte_eth_dev *dev) 78 { 79 struct mlx5_priv *priv = dev->data->dev_private; 80 struct rte_eth_dev *host_dev = priv->shared_host; 81 struct mlx5_priv *host_priv = host_dev->data->dev_private; 82 int reg_id = 0; 83 uint32_t flags; 84 int ret = 0; 85 86 MLX5_ASSERT(priv->shared_host); 87 reg_id = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, NULL); 88 if (reg_id < 0) { 89 rte_errno = ENOMEM; 90 ret = -rte_errno; 91 DRV_LOG(ERR, "Meter register is not available."); 92 goto err; 93 } 94 priv->hws_mpool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct mlx5_aso_mtr_pool), 95 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 96 if (!priv->hws_mpool) { 97 rte_errno = ENOMEM; 98 ret = -rte_errno; 99 DRV_LOG(ERR, "Meter ipool allocation failed."); 100 goto err; 101 } 102 MLX5_ASSERT(host_priv->hws_mpool->idx_pool); 103 MLX5_ASSERT(host_priv->hws_mpool->devx_obj); 104 priv->hws_mpool->idx_pool = host_priv->hws_mpool->idx_pool; 105 priv->hws_mpool->devx_obj = host_priv->hws_mpool->devx_obj; 106 flags = MLX5DR_ACTION_FLAG_HWS_RX | MLX5DR_ACTION_FLAG_HWS_TX; 107 if (priv->sh->config.dv_esw_en && priv->master) 108 flags |= MLX5DR_ACTION_FLAG_HWS_FDB; 109 priv->hws_mpool->action = mlx5dr_action_create_aso_meter 110 (priv->dr_ctx, (struct mlx5dr_devx_obj *)priv->hws_mpool->devx_obj, 111 reg_id - REG_C_0, flags); 112 if (!priv->hws_mpool->action) { 113 rte_errno = ENOMEM; 114 ret = -rte_errno; 115 DRV_LOG(ERR, "Meter action creation failed."); 116 goto err; 117 } 118 return 0; 119 err: 120 mlx5_flow_meter_uninit(dev); 121 return ret; 122 } 123 124 int 125 mlx5_flow_meter_init(struct rte_eth_dev *dev, 126 uint32_t nb_meters, 127 uint32_t nb_meter_profiles, 128 uint32_t nb_meter_policies, 129 uint32_t nb_queues) 130 { 131 struct mlx5_priv *priv = dev->data->dev_private; 132 struct mlx5_devx_obj *dcs = NULL; 133 uint32_t log_obj_size; 134 int ret = 0; 135 int reg_id; 136 struct mlx5_aso_mtr *aso; 137 uint32_t i; 138 struct rte_flow_error error; 139 uint32_t flags; 140 uint32_t nb_mtrs = rte_align32pow2(nb_meters); 141 struct mlx5_indexed_pool_config cfg = { 142 .size = sizeof(struct mlx5_aso_mtr), 143 .trunk_size = 1 << 12, 144 .per_core_cache = 1 << 13, 145 .need_lock = 1, 146 .release_mem_en = !!priv->sh->config.reclaim_mode, 147 .malloc = mlx5_malloc, 148 .max_idx = nb_meters, 149 .free = mlx5_free, 150 .type = "mlx5_hw_mtr_mark_action", 151 }; 152 153 if (priv->shared_host) 154 return mlx5_flow_meter_init_guest(dev); 155 if (!nb_meters) { 156 ret = ENOTSUP; 157 rte_flow_error_set(&error, ENOMEM, 158 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 159 NULL, "Meter configuration is invalid."); 160 goto err; 161 } 162 if (!priv->mtr_en || !priv->sh->meter_aso_en) { 163 ret = ENOTSUP; 164 rte_flow_error_set(&error, ENOMEM, 165 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 166 NULL, "Meter ASO is not supported."); 167 goto err; 168 } 169 priv->mtr_config.nb_meters = nb_meters; 170 log_obj_size = rte_log2_u32(nb_meters >> 1); 171 dcs = mlx5_devx_cmd_create_flow_meter_aso_obj 172 (priv->sh->cdev->ctx, priv->sh->cdev->pdn, 173 log_obj_size); 174 if (!dcs) { 175 ret = ENOMEM; 176 rte_flow_error_set(&error, ENOMEM, 177 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 178 NULL, "Meter ASO object allocation failed."); 179 goto err; 180 } 181 priv->mtr_bulk.devx_obj = dcs; 182 reg_id = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, NULL); 183 if (reg_id < 0) { 184 ret = ENOTSUP; 185 rte_flow_error_set(&error, ENOMEM, 186 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 187 NULL, "Meter register is not available."); 188 goto err; 189 } 190 flags = MLX5DR_ACTION_FLAG_HWS_RX | MLX5DR_ACTION_FLAG_HWS_TX; 191 if (priv->sh->config.dv_esw_en && priv->master) 192 flags |= MLX5DR_ACTION_FLAG_HWS_FDB; 193 priv->mtr_bulk.action = mlx5dr_action_create_aso_meter 194 (priv->dr_ctx, (struct mlx5dr_devx_obj *)dcs, 195 reg_id - REG_C_0, flags); 196 if (!priv->mtr_bulk.action) { 197 ret = ENOMEM; 198 rte_flow_error_set(&error, ENOMEM, 199 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 200 NULL, "Meter action creation failed."); 201 goto err; 202 } 203 priv->mtr_bulk.aso = mlx5_malloc(MLX5_MEM_ZERO, 204 sizeof(struct mlx5_aso_mtr) * 205 nb_meters, 206 RTE_CACHE_LINE_SIZE, 207 SOCKET_ID_ANY); 208 if (!priv->mtr_bulk.aso) { 209 ret = ENOMEM; 210 rte_flow_error_set(&error, ENOMEM, 211 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 212 NULL, "Meter bulk ASO allocation failed."); 213 goto err; 214 } 215 priv->mtr_bulk.size = nb_meters; 216 aso = priv->mtr_bulk.aso; 217 for (i = 0; i < priv->mtr_bulk.size; i++) { 218 aso->type = ASO_METER_DIRECT; 219 aso->state = ASO_METER_WAIT; 220 aso->offset = i; 221 aso++; 222 } 223 priv->hws_mpool = mlx5_malloc(MLX5_MEM_ZERO, 224 sizeof(struct mlx5_aso_mtr_pool), 225 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 226 if (!priv->hws_mpool) { 227 ret = ENOMEM; 228 rte_flow_error_set(&error, ENOMEM, 229 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 230 NULL, "Meter ipool allocation failed."); 231 goto err; 232 } 233 priv->hws_mpool->devx_obj = priv->mtr_bulk.devx_obj; 234 priv->hws_mpool->action = priv->mtr_bulk.action; 235 priv->hws_mpool->nb_sq = nb_queues; 236 if (mlx5_aso_mtr_queue_init(priv->sh, priv->hws_mpool, 237 &priv->sh->mtrmng->pools_mng, nb_queues)) { 238 ret = ENOMEM; 239 rte_flow_error_set(&error, ENOMEM, 240 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 241 NULL, "Meter ASO queue allocation failed."); 242 goto err; 243 } 244 /* 245 * No need for local cache if Meter number is a small number. 246 * Since flow insertion rate will be very limited in that case. 247 * Here let's set the number to less than default trunk size 4K. 248 */ 249 if (nb_mtrs <= cfg.trunk_size) { 250 cfg.per_core_cache = 0; 251 cfg.trunk_size = nb_mtrs; 252 } else if (nb_mtrs <= MLX5_HW_IPOOL_SIZE_THRESHOLD) { 253 cfg.per_core_cache = MLX5_HW_IPOOL_CACHE_MIN; 254 } 255 priv->hws_mpool->idx_pool = mlx5_ipool_create(&cfg); 256 if (nb_meter_profiles) { 257 priv->mtr_config.nb_meter_profiles = nb_meter_profiles; 258 priv->mtr_profile_arr = 259 mlx5_malloc(MLX5_MEM_ZERO, 260 sizeof(struct mlx5_flow_meter_profile) * 261 nb_meter_profiles, 262 RTE_CACHE_LINE_SIZE, 263 SOCKET_ID_ANY); 264 if (!priv->mtr_profile_arr) { 265 ret = ENOMEM; 266 rte_flow_error_set(&error, ENOMEM, 267 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 268 NULL, "Meter profile allocation failed."); 269 goto err; 270 } 271 } 272 if (nb_meter_policies) { 273 priv->mtr_config.nb_meter_policies = nb_meter_policies; 274 priv->mtr_policy_arr = 275 mlx5_malloc(MLX5_MEM_ZERO, 276 sizeof(struct mlx5_flow_meter_policy) * 277 nb_meter_policies, 278 RTE_CACHE_LINE_SIZE, 279 SOCKET_ID_ANY); 280 if (!priv->mtr_policy_arr) { 281 ret = ENOMEM; 282 rte_flow_error_set(&error, ENOMEM, 283 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 284 NULL, "Meter policy allocation failed."); 285 goto err; 286 } 287 } 288 return 0; 289 err: 290 mlx5_flow_meter_uninit(dev); 291 return ret; 292 } 293 294 #endif /* HAVE_MLX5_HWS_SUPPORT */ 295 296 static int mlx5_flow_meter_disable(struct rte_eth_dev *dev, 297 uint32_t meter_id, struct rte_mtr_error *error); 298 299 /** 300 * Create the meter action. 301 * 302 * @param priv 303 * Pointer to mlx5_priv. 304 * @param[in] fm 305 * Pointer to flow meter to be converted. 306 * 307 * @return 308 * Pointer to the meter action on success, NULL otherwise. 309 */ 310 static void * 311 mlx5_flow_meter_action_create(struct mlx5_priv *priv, 312 struct mlx5_flow_meter_info *fm) 313 { 314 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER 315 struct mlx5dv_dr_flow_meter_attr mtr_init; 316 uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)]; 317 struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = 318 &fm->profile->srtcm_prm; 319 uint32_t cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir); 320 uint32_t ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir); 321 uint32_t val; 322 enum mlx5_meter_domain domain = 323 fm->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 324 fm->egress ? MLX5_MTR_DOMAIN_EGRESS : 325 MLX5_MTR_DOMAIN_INGRESS; 326 struct mlx5_flow_meter_def_policy *def_policy = 327 priv->sh->mtrmng->def_policy[domain]; 328 329 memset(fmp, 0, MLX5_ST_SZ_BYTES(flow_meter_parameters)); 330 MLX5_SET(flow_meter_parameters, fmp, valid, 1); 331 MLX5_SET(flow_meter_parameters, fmp, bucket_overflow, 1); 332 MLX5_SET(flow_meter_parameters, fmp, 333 start_color, MLX5_FLOW_COLOR_GREEN); 334 MLX5_SET(flow_meter_parameters, fmp, both_buckets_on_green, 0); 335 val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK; 336 MLX5_SET(flow_meter_parameters, fmp, cbs_exponent, val); 337 val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK; 338 MLX5_SET(flow_meter_parameters, fmp, cbs_mantissa, val); 339 val = (cbs_cir >> ASO_DSEG_XIR_EXP_OFFSET) & ASO_DSEG_EXP_MASK; 340 MLX5_SET(flow_meter_parameters, fmp, cir_exponent, val); 341 val = (cbs_cir & ASO_DSEG_MAN_MASK); 342 MLX5_SET(flow_meter_parameters, fmp, cir_mantissa, val); 343 val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK; 344 MLX5_SET(flow_meter_parameters, fmp, ebs_exponent, val); 345 val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK; 346 MLX5_SET(flow_meter_parameters, fmp, ebs_mantissa, val); 347 mtr_init.next_table = def_policy->sub_policy.tbl_rsc->obj; 348 mtr_init.reg_c_index = priv->sh->registers.aso_reg - REG_C_0; 349 mtr_init.flow_meter_parameter = fmp; 350 mtr_init.flow_meter_parameter_sz = 351 MLX5_ST_SZ_BYTES(flow_meter_parameters); 352 mtr_init.active = fm->active_state; 353 return mlx5_glue->dv_create_flow_action_meter(&mtr_init); 354 #else 355 (void)priv; 356 (void)fm; 357 return NULL; 358 #endif 359 } 360 361 /** 362 * Find meter profile by id. 363 * 364 * @param priv 365 * Pointer to mlx5_priv. 366 * @param meter_profile_id 367 * Meter profile id. 368 * 369 * @return 370 * Pointer to the profile found on success, NULL otherwise. 371 */ 372 static struct mlx5_flow_meter_profile * 373 mlx5_flow_meter_profile_find(struct mlx5_priv *priv, uint32_t meter_profile_id) 374 { 375 struct mlx5_flow_meter_profile *fmp; 376 union mlx5_l3t_data data; 377 int32_t ret; 378 379 if (priv->mtr_profile_arr) 380 return &priv->mtr_profile_arr[meter_profile_id]; 381 if (mlx5_l3t_get_entry(priv->mtr_profile_tbl, 382 meter_profile_id, &data) || !data.ptr) 383 return NULL; 384 fmp = data.ptr; 385 /* Remove reference taken by the mlx5_l3t_get_entry. */ 386 ret = mlx5_l3t_clear_entry(priv->mtr_profile_tbl, 387 meter_profile_id); 388 if (!ret || ret == -1) 389 return NULL; 390 return fmp; 391 } 392 393 /** 394 * Validate the MTR profile. 395 * 396 * @param[in] dev 397 * Pointer to Ethernet device. 398 * @param[in] meter_profile_id 399 * Meter profile id. 400 * @param[in] profile 401 * Pointer to meter profile detail. 402 * @param[out] error 403 * Pointer to the error structure. 404 * 405 * @return 406 * 0 on success, a negative errno value otherwise and rte_errno is set. 407 */ 408 static int 409 mlx5_flow_meter_profile_validate(struct rte_eth_dev *dev, 410 uint32_t meter_profile_id, 411 struct rte_mtr_meter_profile *profile, 412 struct rte_mtr_error *error) 413 { 414 struct mlx5_priv *priv = dev->data->dev_private; 415 struct mlx5_flow_meter_profile *fmp; 416 uint32_t ls_factor; 417 int ret; 418 uint64_t cir, cbs; 419 uint64_t eir, ebs; 420 uint64_t pir, pbs; 421 422 /* Profile must not be NULL. */ 423 if (profile == NULL) 424 return -rte_mtr_error_set(error, EINVAL, 425 RTE_MTR_ERROR_TYPE_METER_PROFILE, 426 NULL, "Meter profile is null."); 427 /* Meter profile ID must be valid. */ 428 if (priv->mtr_profile_arr) { 429 if (meter_profile_id >= priv->mtr_config.nb_meter_profiles) 430 return -rte_mtr_error_set(error, EINVAL, 431 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 432 NULL, "Meter profile id not valid."); 433 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 434 /* Meter profile must not exist. */ 435 if (fmp->initialized) 436 return -rte_mtr_error_set(error, EEXIST, 437 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 438 NULL, "Meter profile already exists."); 439 } else { 440 if (meter_profile_id == UINT32_MAX) 441 return -rte_mtr_error_set(error, EINVAL, 442 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 443 NULL, "Meter profile id not valid."); 444 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 445 /* Meter profile must not exist. */ 446 if (fmp) 447 return -rte_mtr_error_set(error, EEXIST, 448 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 449 NULL, "Meter profile already exists."); 450 } 451 if (!priv->sh->meter_aso_en) { 452 /* Old version is even not supported. */ 453 if (!priv->sh->cdev->config.hca_attr.qos.flow_meter_old) 454 return -rte_mtr_error_set(error, ENOTSUP, 455 RTE_MTR_ERROR_TYPE_METER_PROFILE, 456 NULL, "Metering is not supported."); 457 /* Old FW metering only supports srTCM. */ 458 if (profile->alg != RTE_MTR_SRTCM_RFC2697) { 459 return -rte_mtr_error_set(error, ENOTSUP, 460 RTE_MTR_ERROR_TYPE_METER_PROFILE, 461 NULL, "Metering algorithm is not supported."); 462 } else if (profile->srtcm_rfc2697.ebs) { 463 /* EBS is not supported for old metering. */ 464 return -rte_mtr_error_set(error, ENOTSUP, 465 RTE_MTR_ERROR_TYPE_METER_PROFILE, 466 NULL, "EBS is not supported."); 467 } 468 if (profile->packet_mode) 469 return -rte_mtr_error_set(error, ENOTSUP, 470 RTE_MTR_ERROR_TYPE_METER_PROFILE, NULL, 471 "Metering algorithm packet mode is not supported."); 472 } 473 ls_factor = profile->packet_mode ? MLX5_MTRS_PPS_MAP_BPS_SHIFT : 0; 474 switch (profile->alg) { 475 case RTE_MTR_SRTCM_RFC2697: 476 cir = profile->srtcm_rfc2697.cir << ls_factor; 477 cbs = profile->srtcm_rfc2697.cbs << ls_factor; 478 ebs = profile->srtcm_rfc2697.ebs << ls_factor; 479 /* EBS could be zero for old metering. */ 480 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX && 481 cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX && 482 ebs <= MLX5_SRTCM_XBS_MAX) { 483 ret = 0; 484 } else { 485 ret = -rte_mtr_error_set(error, ENOTSUP, 486 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 487 "Profile values out of range."); 488 } 489 break; 490 case RTE_MTR_TRTCM_RFC2698: 491 cir = profile->trtcm_rfc2698.cir << ls_factor; 492 cbs = profile->trtcm_rfc2698.cbs << ls_factor; 493 pir = profile->trtcm_rfc2698.pir << ls_factor; 494 pbs = profile->trtcm_rfc2698.pbs << ls_factor; 495 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX && 496 cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX && 497 pir >= cir && pir <= (MLX5_SRTCM_XIR_MAX * 2) && 498 pbs >= cbs && pbs <= (MLX5_SRTCM_XBS_MAX * 2)) { 499 ret = 0; 500 } else { 501 ret = -rte_mtr_error_set(error, ENOTSUP, 502 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 503 "Profile values out of range."); 504 } 505 break; 506 case RTE_MTR_TRTCM_RFC4115: 507 cir = profile->trtcm_rfc4115.cir << ls_factor; 508 cbs = profile->trtcm_rfc4115.cbs << ls_factor; 509 eir = profile->trtcm_rfc4115.eir << ls_factor; 510 ebs = profile->trtcm_rfc4115.ebs << ls_factor; 511 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX && 512 cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX && 513 eir <= MLX5_SRTCM_XIR_MAX && ebs <= MLX5_SRTCM_XBS_MAX) { 514 ret = 0; 515 } else { 516 ret = -rte_mtr_error_set(error, ENOTSUP, 517 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 518 "Profile values out of range."); 519 } 520 break; 521 default: 522 ret = -rte_mtr_error_set(error, ENOTSUP, 523 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 524 "Unknown metering algorithm."); 525 break; 526 } 527 return ret; 528 } 529 530 /* 531 * Calculate mantissa and exponent for cir / eir. 532 * 533 * @param[in] xir 534 * Value to be calculated. 535 * @param[out] man 536 * Pointer to the mantissa. 537 * @param[out] exp 538 * Pointer to the exp. 539 */ 540 static inline void 541 mlx5_flow_meter_xir_man_exp_calc(int64_t xir, uint8_t *man, uint8_t *exp) 542 { 543 int64_t _xir; 544 int64_t delta = INT64_MAX; 545 uint8_t _man = 0; 546 uint8_t _exp = 0; 547 uint64_t m, e; 548 549 /* Special case xir == 0 ? both exp and mantissa are 0. */ 550 if (xir == 0) { 551 *man = 0; 552 *exp = 0; 553 return; 554 } 555 for (m = 0; m <= 0xFF; m++) { /* man width 8 bit */ 556 for (e = 0; e <= 0x1F; e++) { /* exp width 5bit */ 557 _xir = (1000000000ULL * m) >> e; 558 if (llabs(xir - _xir) <= delta) { 559 delta = llabs(xir - _xir); 560 _man = m; 561 _exp = e; 562 } 563 } 564 } 565 *man = _man; 566 *exp = _exp; 567 } 568 569 /* 570 * Calculate mantissa and exponent for xbs. 571 * 572 * @param[in] xbs 573 * Value to be calculated. 574 * @param[out] man 575 * Pointer to the mantissa. 576 * @param[out] exp 577 * Pointer to the exp. 578 */ 579 static inline void 580 mlx5_flow_meter_xbs_man_exp_calc(uint64_t xbs, uint8_t *man, uint8_t *exp) 581 { 582 int _exp; 583 double _man; 584 585 /* Special case xbs == 0 ? both exp and mantissa are 0. */ 586 if (xbs == 0) { 587 *man = 0; 588 *exp = 0; 589 return; 590 } 591 /* xbs = xbs_mantissa * 2^xbs_exponent */ 592 _man = frexp(xbs, &_exp); 593 if (_exp >= MLX5_MAN_WIDTH) { 594 _man = _man * pow(2, MLX5_MAN_WIDTH); 595 _exp = _exp - MLX5_MAN_WIDTH; 596 } 597 *man = (uint8_t)ceil(_man); 598 *exp = _exp; 599 } 600 601 /** 602 * Fill the prm meter parameter. 603 * 604 * @param[in,out] fmp 605 * Pointer to meter profile to be converted. 606 * @param[out] error 607 * Pointer to the error structure. 608 * 609 * @return 610 * 0 on success, a negative errno value otherwise and rte_errno is set. 611 */ 612 static int 613 mlx5_flow_meter_param_fill(struct mlx5_flow_meter_profile *fmp, 614 struct rte_mtr_error *error) 615 { 616 struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = &fmp->srtcm_prm; 617 uint8_t man, exp; 618 uint32_t cbs_exp, cbs_man, cir_exp, cir_man; 619 uint32_t eir_exp, eir_man, ebs_exp, ebs_man; 620 uint64_t cir, cbs, eir, ebs; 621 622 switch (fmp->profile.alg) { 623 case RTE_MTR_SRTCM_RFC2697: 624 cir = fmp->profile.srtcm_rfc2697.cir; 625 cbs = fmp->profile.srtcm_rfc2697.cbs; 626 eir = 0; 627 ebs = fmp->profile.srtcm_rfc2697.ebs; 628 break; 629 case RTE_MTR_TRTCM_RFC2698: 630 MLX5_ASSERT(fmp->profile.trtcm_rfc2698.pir > 631 fmp->profile.trtcm_rfc2698.cir && 632 fmp->profile.trtcm_rfc2698.pbs > 633 fmp->profile.trtcm_rfc2698.cbs); 634 cir = fmp->profile.trtcm_rfc2698.cir; 635 cbs = fmp->profile.trtcm_rfc2698.cbs; 636 /* EIR / EBS are filled with PIR / PBS. */ 637 eir = fmp->profile.trtcm_rfc2698.pir; 638 ebs = fmp->profile.trtcm_rfc2698.pbs; 639 break; 640 case RTE_MTR_TRTCM_RFC4115: 641 cir = fmp->profile.trtcm_rfc4115.cir; 642 cbs = fmp->profile.trtcm_rfc4115.cbs; 643 eir = fmp->profile.trtcm_rfc4115.eir; 644 ebs = fmp->profile.trtcm_rfc4115.ebs; 645 break; 646 default: 647 return -rte_mtr_error_set(error, EINVAL, 648 RTE_MTR_ERROR_TYPE_METER_PROFILE, NULL, 649 "Metering algorithm mode is invalid"); 650 } 651 /* Adjust the values for PPS mode. */ 652 if (fmp->profile.packet_mode) { 653 cir <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT; 654 cbs <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT; 655 eir <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT; 656 ebs <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT; 657 } 658 /* cir = 8G * cir_mantissa * 1/(2^cir_exponent)) Bytes/Sec */ 659 mlx5_flow_meter_xir_man_exp_calc(cir, &man, &exp); 660 /* Check if cir mantissa is too large. */ 661 if (exp > ASO_DSEG_XIR_EXP_MASK) 662 return -rte_mtr_error_set(error, ENOTSUP, 663 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 664 "meter profile parameter cir is not supported."); 665 cir_man = man; 666 cir_exp = exp; 667 /* cbs = cbs_mantissa * 2^cbs_exponent */ 668 mlx5_flow_meter_xbs_man_exp_calc(cbs, &man, &exp); 669 /* Check if cbs mantissa is too large. */ 670 if (exp > ASO_DSEG_EXP_MASK) 671 return -rte_mtr_error_set(error, ENOTSUP, 672 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 673 "meter profile parameter cbs is not supported."); 674 cbs_man = man; 675 cbs_exp = exp; 676 srtcm->cbs_cir = rte_cpu_to_be_32(cbs_exp << ASO_DSEG_CBS_EXP_OFFSET | 677 cbs_man << ASO_DSEG_CBS_MAN_OFFSET | 678 cir_exp << ASO_DSEG_XIR_EXP_OFFSET | 679 cir_man); 680 mlx5_flow_meter_xir_man_exp_calc(eir, &man, &exp); 681 /* Check if eir mantissa is too large. */ 682 if (exp > ASO_DSEG_XIR_EXP_MASK) 683 return -rte_mtr_error_set(error, ENOTSUP, 684 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 685 "meter profile parameter eir is not supported."); 686 eir_man = man; 687 eir_exp = exp; 688 mlx5_flow_meter_xbs_man_exp_calc(ebs, &man, &exp); 689 /* Check if ebs mantissa is too large. */ 690 if (exp > ASO_DSEG_EXP_MASK) 691 return -rte_mtr_error_set(error, ENOTSUP, 692 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 693 "meter profile parameter ebs is not supported."); 694 ebs_man = man; 695 ebs_exp = exp; 696 srtcm->ebs_eir = rte_cpu_to_be_32(ebs_exp << ASO_DSEG_EBS_EXP_OFFSET | 697 ebs_man << ASO_DSEG_EBS_MAN_OFFSET | 698 eir_exp << ASO_DSEG_XIR_EXP_OFFSET | 699 eir_man); 700 if (srtcm->cbs_cir) 701 fmp->g_support = 1; 702 if (srtcm->ebs_eir) 703 fmp->y_support = 1; 704 return 0; 705 } 706 707 /** 708 * Callback to get MTR maximum objects number. 709 * 710 * @param[in] priv 711 * Pointer to Ethernet device. 712 * 713 * @return 714 * Max number of meters. 715 */ 716 uint32_t 717 mlx5_flow_mtr_max_get(struct mlx5_priv *priv) 718 { 719 struct mlx5_hca_qos_attr *qattr = &priv->sh->cdev->config.hca_attr.qos; 720 721 /* Max number of meters. */ 722 return ((priv->sh->meter_aso_en) ? 723 1 << (qattr->log_max_num_meter_aso + 1) : 724 qattr->log_max_flow_meter); 725 } 726 727 /** 728 * Callback to get MTR capabilities. 729 * 730 * @param[in] dev 731 * Pointer to Ethernet device. 732 * @param[out] cap 733 * Pointer to save MTR capabilities. 734 * @param[out] error 735 * Pointer to the error structure. 736 * 737 * @return 738 * 0 on success, a negative errno value otherwise and rte_errno is set. 739 */ 740 static int 741 mlx5_flow_mtr_cap_get(struct rte_eth_dev *dev, 742 struct rte_mtr_capabilities *cap, 743 struct rte_mtr_error *error __rte_unused) 744 { 745 struct mlx5_priv *priv = dev->data->dev_private; 746 struct mlx5_hca_qos_attr *qattr = &priv->sh->cdev->config.hca_attr.qos; 747 748 if (!priv->mtr_en) 749 return -rte_mtr_error_set(error, ENOTSUP, 750 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 751 "Meter is not supported"); 752 memset(cap, 0, sizeof(*cap)); 753 cap->n_max = mlx5_flow_mtr_max_get(priv); 754 if (priv->sh->meter_aso_en) { 755 cap->srtcm_rfc2697_packet_mode_supported = 1; 756 cap->trtcm_rfc2698_packet_mode_supported = 1; 757 cap->trtcm_rfc4115_packet_mode_supported = 1; 758 } 759 cap->srtcm_rfc2697_byte_mode_supported = 1; 760 cap->trtcm_rfc2698_byte_mode_supported = 1; 761 cap->trtcm_rfc4115_byte_mode_supported = 1; 762 cap->n_shared_max = cap->n_max; 763 cap->identical = 1; 764 cap->shared_identical = 1; 765 cap->shared_n_flows_per_mtr_max = 4 << 20; 766 /* 2M flows can share the same meter. */ 767 cap->chaining_n_mtrs_per_flow_max = 1; /* Chaining is not supported. */ 768 cap->meter_srtcm_rfc2697_n_max = qattr->flow_meter_old ? cap->n_max : 0; 769 cap->meter_trtcm_rfc2698_n_max = qattr->flow_meter_old ? cap->n_max : 0; 770 cap->meter_trtcm_rfc4115_n_max = qattr->flow_meter_old ? cap->n_max : 0; 771 cap->meter_rate_max = 1ULL << 40; /* 1 Tera tokens per sec. */ 772 cap->meter_policy_n_max = cap->n_max; 773 cap->stats_mask = RTE_MTR_STATS_N_BYTES_DROPPED | 774 RTE_MTR_STATS_N_PKTS_DROPPED; 775 return 0; 776 } 777 778 /** 779 * Callback to add MTR profile. 780 * 781 * @param[in] dev 782 * Pointer to Ethernet device. 783 * @param[in] meter_profile_id 784 * Meter profile id. 785 * @param[in] profile 786 * Pointer to meter profile detail. 787 * @param[out] error 788 * Pointer to the error structure. 789 * 790 * @return 791 * 0 on success, a negative errno value otherwise and rte_errno is set. 792 */ 793 static int 794 mlx5_flow_meter_profile_add(struct rte_eth_dev *dev, 795 uint32_t meter_profile_id, 796 struct rte_mtr_meter_profile *profile, 797 struct rte_mtr_error *error) 798 { 799 struct mlx5_priv *priv = dev->data->dev_private; 800 struct mlx5_flow_meter_profile *fmp; 801 union mlx5_l3t_data data; 802 int ret; 803 804 if (!priv->mtr_en) 805 return -rte_mtr_error_set(error, ENOTSUP, 806 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 807 "Meter is not supported"); 808 /* Check input params. */ 809 ret = mlx5_flow_meter_profile_validate(dev, meter_profile_id, 810 profile, error); 811 if (ret) 812 return ret; 813 /* Meter profile memory allocation. */ 814 fmp = mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct mlx5_flow_meter_profile), 815 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 816 if (fmp == NULL) 817 return -rte_mtr_error_set(error, ENOMEM, 818 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 819 NULL, "Meter profile memory " 820 "alloc failed."); 821 /* Fill profile info. */ 822 fmp->id = meter_profile_id; 823 fmp->profile = *profile; 824 /* Fill the flow meter parameters for the PRM. */ 825 ret = mlx5_flow_meter_param_fill(fmp, error); 826 if (ret) 827 goto error; 828 data.ptr = fmp; 829 ret = mlx5_l3t_set_entry(priv->mtr_profile_tbl, 830 meter_profile_id, &data); 831 if (ret) 832 return -rte_mtr_error_set(error, ENOTSUP, 833 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 834 NULL, "Meter profile insert fail."); 835 return 0; 836 error: 837 mlx5_free(fmp); 838 return ret; 839 } 840 841 /** 842 * Callback to delete MTR profile. 843 * 844 * @param[in] dev 845 * Pointer to Ethernet device. 846 * @param[in] meter_profile_id 847 * Meter profile id. 848 * @param[out] error 849 * Pointer to the error structure. 850 * 851 * @return 852 * 0 on success, a negative errno value otherwise and rte_errno is set. 853 */ 854 static int 855 mlx5_flow_meter_profile_delete(struct rte_eth_dev *dev, 856 uint32_t meter_profile_id, 857 struct rte_mtr_error *error) 858 { 859 struct mlx5_priv *priv = dev->data->dev_private; 860 struct mlx5_flow_meter_profile *fmp; 861 862 if (!priv->mtr_en) 863 return -rte_mtr_error_set(error, ENOTSUP, 864 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 865 "Meter is not supported"); 866 /* Meter profile must exist. */ 867 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 868 if (fmp == NULL) 869 return -rte_mtr_error_set(error, ENOENT, 870 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 871 &meter_profile_id, 872 "Meter profile id is invalid."); 873 /* Check profile is unused. */ 874 if (fmp->ref_cnt) 875 return -rte_mtr_error_set(error, EBUSY, 876 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 877 NULL, "Meter profile is in use."); 878 if (mlx5_l3t_clear_entry(priv->mtr_profile_tbl, meter_profile_id)) 879 return -rte_mtr_error_set(error, EBUSY, 880 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 881 NULL, "Meter profile remove fail."); 882 mlx5_free(fmp); 883 return 0; 884 } 885 886 /** 887 * Callback to get MTR profile. 888 * 889 * @param[in] dev 890 * Pointer to Ethernet device. 891 * @param[in] meter_profile_id 892 * Meter profile id. 893 * @param[out] error 894 * Pointer to the error structure. 895 * 896 * @return 897 * A valid handle in case of success, NULL otherwise. 898 */ 899 static struct rte_flow_meter_profile * 900 mlx5_flow_meter_profile_get(struct rte_eth_dev *dev, 901 uint32_t meter_profile_id, 902 struct rte_mtr_error *error) 903 { 904 struct mlx5_priv *priv = dev->data->dev_private; 905 906 if (!priv->mtr_en) { 907 rte_mtr_error_set(error, ENOTSUP, 908 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 909 "Meter is not supported"); 910 return NULL; 911 } 912 return (void *)(uintptr_t)mlx5_flow_meter_profile_find(priv, 913 meter_profile_id); 914 } 915 916 #if defined(HAVE_MLX5_HWS_SUPPORT) 917 /** 918 * Callback to add MTR profile with HWS. 919 * 920 * @param[in] dev 921 * Pointer to Ethernet device. 922 * @param[in] meter_profile_id 923 * Meter profile id. 924 * @param[in] profile 925 * Pointer to meter profile detail. 926 * @param[out] error 927 * Pointer to the error structure. 928 * 929 * @return 930 * 0 on success, a negative errno value otherwise and rte_errno is set. 931 */ 932 static int 933 mlx5_flow_meter_profile_hws_add(struct rte_eth_dev *dev, 934 uint32_t meter_profile_id, 935 struct rte_mtr_meter_profile *profile, 936 struct rte_mtr_error *error) 937 { 938 struct mlx5_priv *priv = dev->data->dev_private; 939 struct mlx5_flow_meter_profile *fmp; 940 int ret; 941 942 if (priv->shared_host) 943 return -rte_mtr_error_set(error, ENOTSUP, RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 944 "Meter profiles cannot be created on guest port"); 945 if (!priv->mtr_profile_arr) 946 return mlx5_flow_meter_profile_add(dev, meter_profile_id, profile, error); 947 /* Check input params. */ 948 ret = mlx5_flow_meter_profile_validate(dev, meter_profile_id, 949 profile, error); 950 if (ret) 951 return ret; 952 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 953 /* Fill profile info. */ 954 fmp->id = meter_profile_id; 955 fmp->profile = *profile; 956 fmp->initialized = 1; 957 /* Fill the flow meter parameters for the PRM. */ 958 return mlx5_flow_meter_param_fill(fmp, error); 959 } 960 961 /** 962 * Callback to delete MTR profile with HWS. 963 * 964 * @param[in] dev 965 * Pointer to Ethernet device. 966 * @param[in] meter_profile_id 967 * Meter profile id. 968 * @param[out] error 969 * Pointer to the error structure. 970 * 971 * @return 972 * 0 on success, a negative errno value otherwise and rte_errno is set. 973 */ 974 static int 975 mlx5_flow_meter_profile_hws_delete(struct rte_eth_dev *dev, 976 uint32_t meter_profile_id, 977 struct rte_mtr_error *error) 978 { 979 struct mlx5_priv *priv = dev->data->dev_private; 980 struct mlx5_flow_meter_profile *fmp; 981 982 if (priv->shared_host) 983 return -rte_mtr_error_set(error, ENOTSUP, RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 984 "Meter profiles cannot be destroyed through guest port"); 985 if (!priv->mtr_profile_arr) 986 return mlx5_flow_meter_profile_delete(dev, meter_profile_id, error); 987 /* Meter profile must exist. */ 988 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 989 if (!fmp->initialized) 990 return -rte_mtr_error_set(error, ENOENT, 991 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 992 &meter_profile_id, 993 "Meter profile id is invalid."); 994 /* Check profile is unused. */ 995 if (fmp->ref_cnt) 996 return -rte_mtr_error_set(error, EBUSY, 997 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 998 NULL, "Meter profile is in use."); 999 memset(fmp, 0, sizeof(struct mlx5_flow_meter_profile)); 1000 return 0; 1001 } 1002 #endif 1003 1004 /** 1005 * Find policy by id. 1006 * 1007 * @param[in] dev 1008 * Pointer to Ethernet device. 1009 * @param policy_id 1010 * Policy id. 1011 * 1012 * @return 1013 * Pointer to the policy found on success, NULL otherwise. 1014 */ 1015 struct mlx5_flow_meter_policy * 1016 mlx5_flow_meter_policy_find(struct rte_eth_dev *dev, 1017 uint32_t policy_id, 1018 uint32_t *policy_idx) 1019 { 1020 struct mlx5_priv *priv = dev->data->dev_private; 1021 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 1022 union mlx5_l3t_data data; 1023 1024 if (priv->mtr_policy_arr) { 1025 if (policy_idx) 1026 *policy_idx = policy_id; 1027 return &priv->mtr_policy_arr[policy_id]; 1028 } 1029 if (policy_id > MLX5_MAX_SUB_POLICY_TBL_NUM || !priv->policy_idx_tbl) 1030 return NULL; 1031 if (mlx5_l3t_get_entry(priv->policy_idx_tbl, policy_id, &data) || 1032 !data.dword) 1033 return NULL; 1034 if (policy_idx) 1035 *policy_idx = data.dword; 1036 sub_policy = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 1037 data.dword); 1038 /* Remove reference taken by the mlx5_l3t_get_entry. */ 1039 mlx5_l3t_clear_entry(priv->policy_idx_tbl, policy_id); 1040 if (sub_policy) 1041 if (sub_policy->main_policy_id) 1042 return sub_policy->main_policy; 1043 return NULL; 1044 } 1045 1046 /** 1047 * Get the next meter from one meter's policy in hierarchy chain. 1048 * Lock free, mutex should be acquired by caller. 1049 * 1050 * @param[in] priv 1051 * Pointer to mlx5_priv. 1052 * @param[in] policy 1053 * Pointer to flow meter policy. 1054 * @param[out] mtr_idx 1055 * Pointer to Meter index. 1056 * 1057 * @return 1058 * Pointer to the next meter, or NULL when fail. 1059 */ 1060 struct mlx5_flow_meter_info * 1061 mlx5_flow_meter_hierarchy_next_meter(struct mlx5_priv *priv, 1062 struct mlx5_flow_meter_policy *policy, 1063 uint32_t *mtr_idx) 1064 { 1065 int i; 1066 1067 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 1068 if (policy->act_cnt[i].fate_action == MLX5_FLOW_FATE_MTR) 1069 return mlx5_flow_meter_find(priv, 1070 policy->act_cnt[i].next_mtr_id, 1071 mtr_idx); 1072 } 1073 return NULL; 1074 } 1075 1076 /** 1077 * Get the last meter's policy from one meter's policy in hierarchy. 1078 * 1079 * @param[in] dev 1080 * Pointer to Ethernet device. 1081 * @param[in] policy 1082 * Pointer to flow meter policy. 1083 * 1084 * @return 1085 * Pointer to the final meter's policy, or NULL when fail. 1086 */ 1087 struct mlx5_flow_meter_policy * 1088 mlx5_flow_meter_hierarchy_get_final_policy(struct rte_eth_dev *dev, 1089 struct mlx5_flow_meter_policy *policy) 1090 { 1091 struct mlx5_priv *priv = dev->data->dev_private; 1092 struct mlx5_flow_meter_info *next_fm; 1093 struct mlx5_flow_meter_policy *next_policy = policy; 1094 1095 while (next_policy->is_hierarchy) { 1096 rte_spinlock_lock(&next_policy->sl); 1097 next_fm = mlx5_flow_meter_hierarchy_next_meter(priv, next_policy, NULL); 1098 rte_spinlock_unlock(&next_policy->sl); 1099 if (!next_fm || next_fm->def_policy) 1100 return NULL; 1101 next_policy = mlx5_flow_meter_policy_find(dev, 1102 next_fm->policy_id, NULL); 1103 MLX5_ASSERT(next_policy); 1104 } 1105 return next_policy; 1106 } 1107 1108 /** 1109 * Callback to check MTR policy action validate 1110 * 1111 * @param[in] dev 1112 * Pointer to Ethernet device. 1113 * @param[in] actions 1114 * Pointer to meter policy action detail. 1115 * @param[out] error 1116 * Pointer to the error structure. 1117 * 1118 * @return 1119 * 0 on success, a negative errno value otherwise and rte_errno is set. 1120 */ 1121 static int 1122 mlx5_flow_meter_policy_validate(struct rte_eth_dev *dev, 1123 struct rte_mtr_meter_policy_params *policy, 1124 struct rte_mtr_error *error) 1125 { 1126 struct mlx5_priv *priv = dev->data->dev_private; 1127 struct rte_flow_attr attr = { .transfer = priv->sh->config.dv_esw_en ? 1128 1 : 0 }; 1129 bool is_rss = false; 1130 uint8_t policy_mode; 1131 uint8_t domain_bitmap; 1132 int ret; 1133 1134 if (!priv->mtr_en || !priv->sh->meter_aso_en) 1135 return -rte_mtr_error_set(error, ENOTSUP, 1136 RTE_MTR_ERROR_TYPE_METER_POLICY, 1137 NULL, "meter policy unsupported."); 1138 ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr, 1139 &is_rss, &domain_bitmap, &policy_mode, error); 1140 if (ret) 1141 return ret; 1142 return 0; 1143 } 1144 1145 #if defined(HAVE_MLX5_HWS_SUPPORT) 1146 /** 1147 * Callback to check MTR policy action validate for HWS 1148 * 1149 * @param[in] dev 1150 * Pointer to Ethernet device. 1151 * @param[in] actions 1152 * Pointer to meter policy action detail. 1153 * @param[out] error 1154 * Pointer to the error structure. 1155 * 1156 * @return 1157 * 0 on success, a negative errno value otherwise and rte_errno is set. 1158 */ 1159 static int 1160 mlx5_flow_meter_policy_hws_validate(struct rte_eth_dev *dev, 1161 struct rte_mtr_meter_policy_params *policy, 1162 struct rte_mtr_error *error) 1163 { 1164 struct mlx5_priv *priv = dev->data->dev_private; 1165 const struct rte_flow_actions_template_attr attr = { 1166 .transfer = priv->sh->config.dv_esw_en ? 1 : 0 }; 1167 int ret; 1168 int i; 1169 1170 if (!priv->mtr_en || !priv->sh->meter_aso_en) 1171 return -rte_mtr_error_set(error, ENOTSUP, 1172 RTE_MTR_ERROR_TYPE_METER_POLICY, 1173 NULL, "meter policy unsupported."); 1174 for (i = 0; i < RTE_COLORS; i++) { 1175 ret = mlx5_flow_actions_validate(dev, &attr, policy->actions[i], 1176 policy->actions[i], NULL); 1177 if (ret) 1178 return ret; 1179 } 1180 return 0; 1181 } 1182 #endif 1183 1184 static int 1185 __mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev, 1186 uint32_t policy_id, 1187 struct mlx5_flow_meter_policy *mtr_policy, 1188 struct rte_mtr_error *error, 1189 bool clear_l3t) 1190 { 1191 struct mlx5_priv *priv = dev->data->dev_private; 1192 struct mlx5_flow_meter_sub_policy *sub_policy; 1193 uint32_t i, j; 1194 uint16_t sub_policy_num; 1195 1196 rte_spinlock_lock(&mtr_policy->sl); 1197 if (mtr_policy->ref_cnt) { 1198 rte_spinlock_unlock(&mtr_policy->sl); 1199 return -rte_mtr_error_set(error, EBUSY, 1200 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1201 NULL, 1202 "Meter policy object is being used."); 1203 } 1204 mlx5_flow_destroy_policy_rules(dev, mtr_policy); 1205 mlx5_flow_destroy_mtr_acts(dev, mtr_policy); 1206 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 1207 sub_policy_num = (mtr_policy->sub_policy_num >> 1208 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) & 1209 MLX5_MTR_SUB_POLICY_NUM_MASK; 1210 if (sub_policy_num) { 1211 for (j = 0; j < sub_policy_num; j++) { 1212 sub_policy = mtr_policy->sub_policys[i][j]; 1213 if (sub_policy) 1214 mlx5_ipool_free 1215 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 1216 sub_policy->idx); 1217 } 1218 } 1219 } 1220 if (priv->policy_idx_tbl && clear_l3t) { 1221 if (mlx5_l3t_clear_entry(priv->policy_idx_tbl, policy_id)) { 1222 rte_spinlock_unlock(&mtr_policy->sl); 1223 return -rte_mtr_error_set(error, ENOTSUP, 1224 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL, 1225 "Fail to delete policy in index table."); 1226 } 1227 } 1228 rte_spinlock_unlock(&mtr_policy->sl); 1229 return 0; 1230 } 1231 1232 /** 1233 * Callback to add MTR policy. 1234 * 1235 * @param[in] dev 1236 * Pointer to Ethernet device. 1237 * @param[out] policy_id 1238 * Pointer to policy id 1239 * @param[in] actions 1240 * Pointer to meter policy action detail. 1241 * @param[out] error 1242 * Pointer to the error structure. 1243 * 1244 * @return 1245 * 0 on success, a negative errno value otherwise and rte_errno is set. 1246 */ 1247 static int 1248 mlx5_flow_meter_policy_add(struct rte_eth_dev *dev, 1249 uint32_t policy_id, 1250 struct rte_mtr_meter_policy_params *policy, 1251 struct rte_mtr_error *error) 1252 { 1253 struct mlx5_priv *priv = dev->data->dev_private; 1254 struct rte_flow_attr attr = { .transfer = priv->sh->config.dv_esw_en ? 1255 1 : 0 }; 1256 uint32_t sub_policy_idx = 0; 1257 uint32_t policy_idx = 0; 1258 struct mlx5_flow_meter_policy *mtr_policy = NULL; 1259 struct mlx5_flow_meter_sub_policy *sub_policy; 1260 bool is_rss = false; 1261 uint8_t policy_mode; 1262 uint32_t i; 1263 int ret; 1264 uint32_t policy_size = sizeof(struct mlx5_flow_meter_policy); 1265 uint16_t sub_policy_num; 1266 uint8_t domain_bitmap = 0; 1267 union mlx5_l3t_data data; 1268 bool skip_rule = false; 1269 1270 if (!priv->mtr_en) 1271 return -rte_mtr_error_set(error, ENOTSUP, 1272 RTE_MTR_ERROR_TYPE_METER_POLICY, 1273 NULL, "meter policy unsupported. "); 1274 if (policy_id == MLX5_INVALID_POLICY_ID) 1275 return -rte_mtr_error_set(error, ENOTSUP, 1276 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1277 NULL, "policy ID is invalid. "); 1278 if (policy_id == priv->sh->mtrmng->def_policy_id) 1279 return -rte_mtr_error_set(error, EEXIST, 1280 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1281 NULL, "default policy ID exists. "); 1282 mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx); 1283 if (mtr_policy) 1284 return -rte_mtr_error_set(error, EEXIST, 1285 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1286 NULL, "policy ID exists. "); 1287 ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr, 1288 &is_rss, &domain_bitmap, 1289 &policy_mode, error); 1290 if (ret) 1291 return ret; 1292 if (!domain_bitmap) 1293 return -rte_mtr_error_set(error, ENOTSUP, 1294 RTE_MTR_ERROR_TYPE_METER_POLICY, 1295 NULL, "fail to find policy domain."); 1296 if (policy_mode == MLX5_MTR_POLICY_MODE_DEF) { 1297 if (priv->sh->mtrmng->def_policy_id != MLX5_INVALID_POLICY_ID) 1298 return -rte_mtr_error_set(error, EEXIST, 1299 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1300 NULL, "a policy with similar actions " 1301 "is already configured"); 1302 if (mlx5_flow_create_def_policy(dev)) 1303 return -rte_mtr_error_set(error, ENOTSUP, 1304 RTE_MTR_ERROR_TYPE_METER_POLICY, 1305 NULL, 1306 "fail to create non-terminated policy."); 1307 priv->sh->mtrmng->def_policy_id = policy_id; 1308 return 0; 1309 } 1310 if (!priv->sh->meter_aso_en) 1311 return -rte_mtr_error_set(error, ENOTSUP, 1312 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL, 1313 "no ASO capability to support the policy "); 1314 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 1315 if (!(domain_bitmap & (1 << i))) 1316 continue; 1317 /* 1318 * If RSS is found, it means that only the ingress domain can 1319 * be supported. It is invalid to support RSS for one color 1320 * and egress / transfer domain actions for another. Drop and 1321 * jump action should have no impact. 1322 */ 1323 if (is_rss) { 1324 policy_size += 1325 sizeof(struct mlx5_flow_meter_sub_policy *) * 1326 MLX5_MTR_RSS_MAX_SUB_POLICY; 1327 break; 1328 } 1329 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *); 1330 } 1331 mtr_policy = mlx5_malloc(MLX5_MEM_ZERO, policy_size, 1332 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 1333 if (!mtr_policy) 1334 return -rte_mtr_error_set(error, ENOMEM, 1335 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL, 1336 "Memory alloc failed for meter policy."); 1337 if (policy_mode == MLX5_MTR_POLICY_MODE_OG) 1338 mtr_policy->skip_y = 1; 1339 else if (policy_mode == MLX5_MTR_POLICY_MODE_OY) 1340 mtr_policy->skip_g = 1; 1341 policy_size = sizeof(struct mlx5_flow_meter_policy); 1342 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 1343 if (!(domain_bitmap & (1 << i))) 1344 continue; 1345 if (i == MLX5_MTR_DOMAIN_INGRESS) 1346 mtr_policy->ingress = 1; 1347 if (i == MLX5_MTR_DOMAIN_EGRESS) 1348 mtr_policy->egress = 1; 1349 if (i == MLX5_MTR_DOMAIN_TRANSFER) 1350 mtr_policy->transfer = 1; 1351 sub_policy = mlx5_ipool_zmalloc 1352 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 1353 &sub_policy_idx); 1354 if (!sub_policy || 1355 sub_policy_idx > MLX5_MAX_SUB_POLICY_TBL_NUM) 1356 goto policy_add_err; 1357 sub_policy->idx = sub_policy_idx; 1358 sub_policy->main_policy = mtr_policy; 1359 if (!policy_idx) { 1360 policy_idx = sub_policy_idx; 1361 sub_policy->main_policy_id = 1; 1362 } 1363 mtr_policy->sub_policys[i] = 1364 (struct mlx5_flow_meter_sub_policy **) 1365 ((uint8_t *)mtr_policy + policy_size); 1366 mtr_policy->sub_policys[i][0] = sub_policy; 1367 sub_policy_num = (mtr_policy->sub_policy_num >> 1368 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) & 1369 MLX5_MTR_SUB_POLICY_NUM_MASK; 1370 sub_policy_num++; 1371 mtr_policy->sub_policy_num &= ~(MLX5_MTR_SUB_POLICY_NUM_MASK << 1372 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)); 1373 mtr_policy->sub_policy_num |= 1374 (sub_policy_num & MLX5_MTR_SUB_POLICY_NUM_MASK) << 1375 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i); 1376 /* 1377 * If RSS is found, it means that only the ingress domain can 1378 * be supported. It is invalid to support RSS for one color 1379 * and egress / transfer domain actions for another. Drop and 1380 * jump action should have no impact. 1381 */ 1382 if (is_rss) { 1383 mtr_policy->is_rss = 1; 1384 break; 1385 } 1386 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *); 1387 } 1388 rte_spinlock_init(&mtr_policy->sl); 1389 ret = mlx5_flow_create_mtr_acts(dev, mtr_policy, 1390 policy->actions, &attr, error); 1391 if (ret) 1392 goto policy_add_err; 1393 if (mtr_policy->is_hierarchy) { 1394 struct mlx5_flow_meter_policy *final_policy; 1395 1396 final_policy = 1397 mlx5_flow_meter_hierarchy_get_final_policy(dev, mtr_policy); 1398 if (!final_policy) 1399 goto policy_add_err; 1400 skip_rule = (final_policy->is_rss || final_policy->is_queue); 1401 } 1402 /* 1403 * If either Green or Yellow has queue / RSS action, all the policy 1404 * rules will be created later in the flow splitting stage. 1405 */ 1406 if (!is_rss && !mtr_policy->is_queue && !skip_rule) { 1407 /* Create policy rules in HW. */ 1408 ret = mlx5_flow_create_policy_rules(dev, mtr_policy); 1409 if (ret) 1410 goto policy_add_err; 1411 } 1412 data.dword = policy_idx; 1413 if (!priv->policy_idx_tbl) { 1414 priv->policy_idx_tbl = mlx5_l3t_create(MLX5_L3T_TYPE_DWORD); 1415 if (!priv->policy_idx_tbl) 1416 goto policy_add_err; 1417 } 1418 if (mlx5_l3t_set_entry(priv->policy_idx_tbl, policy_id, &data)) 1419 goto policy_add_err; 1420 return 0; 1421 policy_add_err: 1422 if (mtr_policy) { 1423 ret = __mlx5_flow_meter_policy_delete(dev, policy_id, 1424 mtr_policy, error, false); 1425 mlx5_free(mtr_policy); 1426 if (ret) 1427 return ret; 1428 } 1429 return -rte_mtr_error_set(error, ENOTSUP, 1430 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 1431 NULL, "Failed to create devx policy."); 1432 } 1433 1434 /** 1435 * Callback to delete MTR policy. 1436 * 1437 * @param[in] dev 1438 * Pointer to Ethernet device. 1439 * @param[in] policy_id 1440 * Meter policy id. 1441 * @param[out] error 1442 * Pointer to the error structure. 1443 * 1444 * @return 1445 * 0 on success, a negative errno value otherwise and rte_errno is set. 1446 */ 1447 static int 1448 mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev, 1449 uint32_t policy_id, 1450 struct rte_mtr_error *error) 1451 { 1452 struct mlx5_priv *priv = dev->data->dev_private; 1453 struct mlx5_flow_meter_policy *mtr_policy; 1454 uint32_t policy_idx; 1455 int ret; 1456 1457 if (policy_id == priv->sh->mtrmng->def_policy_id) { 1458 if (priv->sh->mtrmng->def_policy_ref_cnt > 0) 1459 return -rte_mtr_error_set(error, ENOTSUP, 1460 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL, 1461 "Meter policy object is being used."); 1462 priv->sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID; 1463 return 0; 1464 } 1465 mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx); 1466 if (!mtr_policy) 1467 return -rte_mtr_error_set(error, ENOTSUP, 1468 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL, 1469 "Meter policy id is invalid. "); 1470 ret = __mlx5_flow_meter_policy_delete(dev, policy_id, mtr_policy, 1471 error, true); 1472 if (ret) 1473 return ret; 1474 mlx5_free(mtr_policy); 1475 return 0; 1476 } 1477 1478 /** 1479 * Callback to get MTR policy. 1480 * 1481 * @param[in] dev 1482 * Pointer to Ethernet device. 1483 * @param[in] policy_id 1484 * Meter policy id. 1485 * @param[out] error 1486 * Pointer to the error structure. 1487 * 1488 * @return 1489 * A valid handle in case of success, NULL otherwise. 1490 */ 1491 static struct rte_flow_meter_policy * 1492 mlx5_flow_meter_policy_get(struct rte_eth_dev *dev, 1493 uint32_t policy_id, 1494 struct rte_mtr_error *error) 1495 { 1496 struct mlx5_priv *priv = dev->data->dev_private; 1497 uint32_t policy_idx; 1498 1499 if (!priv->mtr_en) { 1500 rte_mtr_error_set(error, ENOTSUP, 1501 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1502 "Meter is not supported"); 1503 return NULL; 1504 } 1505 return (void *)(uintptr_t)mlx5_flow_meter_policy_find(dev, policy_id, 1506 &policy_idx); 1507 } 1508 1509 #if defined(HAVE_MLX5_HWS_SUPPORT) 1510 /** 1511 * Callback to delete MTR policy for HWS. 1512 * 1513 * @param[in] dev 1514 * Pointer to Ethernet device. 1515 * @param[in] policy_id 1516 * Meter policy id. 1517 * @param[out] error 1518 * Pointer to the error structure. 1519 * 1520 * @return 1521 * 0 on success, a negative errno value otherwise and rte_errno is set. 1522 */ 1523 static int 1524 mlx5_flow_meter_policy_hws_delete(struct rte_eth_dev *dev, 1525 uint32_t policy_id, 1526 struct rte_mtr_error *error) 1527 { 1528 struct mlx5_priv *priv = dev->data->dev_private; 1529 struct mlx5_flow_meter_policy *mtr_policy; 1530 uint32_t i, j; 1531 uint32_t nb_flows = 0; 1532 int ret; 1533 struct rte_flow_op_attr op_attr = { .postpone = 1 }; 1534 struct rte_flow_op_result result[RTE_COLORS * MLX5_MTR_DOMAIN_MAX]; 1535 1536 if (!priv->mtr_policy_arr) 1537 return mlx5_flow_meter_policy_delete(dev, policy_id, error); 1538 /* Meter policy must exist. */ 1539 mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, NULL); 1540 if (!mtr_policy->initialized) 1541 return -rte_mtr_error_set(error, ENOENT, 1542 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL, 1543 "Meter policy does not exists."); 1544 /* Check policy is unused. */ 1545 if (mtr_policy->ref_cnt) 1546 return -rte_mtr_error_set(error, EBUSY, 1547 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1548 NULL, "Meter policy is in use."); 1549 rte_spinlock_lock(&priv->hw_ctrl_lock); 1550 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 1551 for (j = 0; j < RTE_COLORS; j++) { 1552 if (mtr_policy->hws_flow_rule[i][j]) { 1553 ret = rte_flow_async_destroy(dev->data->port_id, 1554 CTRL_QUEUE_ID(priv), &op_attr, 1555 mtr_policy->hws_flow_rule[i][j], 1556 NULL, NULL); 1557 if (ret < 0) 1558 continue; 1559 nb_flows++; 1560 } 1561 } 1562 } 1563 ret = rte_flow_push(dev->data->port_id, CTRL_QUEUE_ID(priv), NULL); 1564 while (nb_flows && (ret >= 0)) { 1565 ret = rte_flow_pull(dev->data->port_id, 1566 CTRL_QUEUE_ID(priv), result, 1567 nb_flows, NULL); 1568 nb_flows -= ret; 1569 } 1570 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 1571 if (mtr_policy->hws_flow_table[i]) 1572 rte_flow_template_table_destroy(dev->data->port_id, 1573 mtr_policy->hws_flow_table[i], NULL); 1574 } 1575 for (i = 0; i < RTE_COLORS; i++) { 1576 if (mtr_policy->hws_act_templ[i]) 1577 rte_flow_actions_template_destroy(dev->data->port_id, 1578 mtr_policy->hws_act_templ[i], NULL); 1579 } 1580 if (mtr_policy->hws_item_templ) 1581 rte_flow_pattern_template_destroy(dev->data->port_id, 1582 mtr_policy->hws_item_templ, NULL); 1583 rte_spinlock_unlock(&priv->hw_ctrl_lock); 1584 memset(mtr_policy, 0, sizeof(struct mlx5_flow_meter_policy)); 1585 return 0; 1586 } 1587 1588 /** 1589 * Callback to add MTR policy for HWS. 1590 * 1591 * @param[in] dev 1592 * Pointer to Ethernet device. 1593 * @param[out] policy_id 1594 * Pointer to policy id 1595 * @param[in] actions 1596 * Pointer to meter policy action detail. 1597 * @param[out] error 1598 * Pointer to the error structure. 1599 * 1600 * @return 1601 * 0 on success, a negative errno value otherwise and rte_errno is set. 1602 */ 1603 static int 1604 mlx5_flow_meter_policy_hws_add(struct rte_eth_dev *dev, 1605 uint32_t policy_id, 1606 struct rte_mtr_meter_policy_params *policy, 1607 struct rte_mtr_error *error) 1608 { 1609 struct mlx5_priv *priv = dev->data->dev_private; 1610 struct mlx5_flow_meter_policy *mtr_policy = NULL; 1611 const struct rte_flow_action *act; 1612 const struct rte_flow_action_meter *mtr; 1613 struct mlx5_flow_meter_info *fm; 1614 struct mlx5_flow_meter_policy *plc; 1615 uint8_t domain_color = MLX5_MTR_ALL_DOMAIN_BIT; 1616 bool is_rss = false; 1617 bool is_hierarchy = false; 1618 int i, j; 1619 uint32_t nb_colors = 0; 1620 uint32_t nb_flows = 0; 1621 int color; 1622 int ret; 1623 struct rte_flow_pattern_template_attr pta = {0}; 1624 struct rte_flow_actions_template_attr ata = {0}; 1625 struct rte_flow_template_table_attr ta = { {0}, 0 }; 1626 struct rte_flow_op_attr op_attr = { .postpone = 1 }; 1627 struct rte_flow_op_result result[RTE_COLORS * MLX5_MTR_DOMAIN_MAX]; 1628 const uint32_t color_mask = (UINT32_C(1) << MLX5_MTR_COLOR_BITS) - 1; 1629 int color_reg_c_idx = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 1630 0, NULL); 1631 struct rte_flow_item_tag tag_spec = { 1632 .data = 0, 1633 .index = color_reg_c_idx 1634 }; 1635 struct rte_flow_item_tag tag_mask = { 1636 .data = color_mask, 1637 .index = 0xff}; 1638 struct rte_flow_item pattern[] = { 1639 [0] = { 1640 .type = (enum rte_flow_item_type) 1641 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 1642 .spec = &tag_spec, 1643 .mask = &tag_mask, 1644 }, 1645 [1] = { .type = RTE_FLOW_ITEM_TYPE_END } 1646 }; 1647 1648 if (!priv->mtr_policy_arr) 1649 return mlx5_flow_meter_policy_add(dev, policy_id, policy, error); 1650 mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, NULL); 1651 if (mtr_policy->initialized) 1652 return -rte_mtr_error_set(error, EEXIST, 1653 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1654 NULL, "Meter policy already exists."); 1655 if (!policy || 1656 (!policy->actions[RTE_COLOR_RED] && 1657 !policy->actions[RTE_COLOR_YELLOW] && 1658 !policy->actions[RTE_COLOR_GREEN])) 1659 return -rte_mtr_error_set(error, EINVAL, 1660 RTE_MTR_ERROR_TYPE_METER_POLICY, 1661 NULL, "Meter policy actions are not valid."); 1662 if (policy->actions[RTE_COLOR_RED] == RTE_FLOW_ACTION_TYPE_END) 1663 mtr_policy->skip_r = 1; 1664 if (policy->actions[RTE_COLOR_YELLOW] == RTE_FLOW_ACTION_TYPE_END) 1665 mtr_policy->skip_y = 1; 1666 if (policy->actions[RTE_COLOR_GREEN] == RTE_FLOW_ACTION_TYPE_END) 1667 mtr_policy->skip_g = 1; 1668 if (mtr_policy->skip_r && mtr_policy->skip_y && mtr_policy->skip_g) 1669 return -rte_mtr_error_set(error, ENOTSUP, 1670 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1671 NULL, "Meter policy actions are empty."); 1672 for (i = 0; i < RTE_COLORS; i++) { 1673 act = policy->actions[i]; 1674 while (act && act->type != RTE_FLOW_ACTION_TYPE_END) { 1675 switch (act->type) { 1676 case RTE_FLOW_ACTION_TYPE_PORT_ID: 1677 /* fall-through. */ 1678 case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT: 1679 domain_color &= ~(MLX5_MTR_DOMAIN_INGRESS_BIT | 1680 MLX5_MTR_DOMAIN_EGRESS_BIT); 1681 break; 1682 case RTE_FLOW_ACTION_TYPE_RSS: 1683 is_rss = true; 1684 /* fall-through. */ 1685 case RTE_FLOW_ACTION_TYPE_QUEUE: 1686 domain_color &= ~(MLX5_MTR_DOMAIN_EGRESS_BIT | 1687 MLX5_MTR_DOMAIN_TRANSFER_BIT); 1688 break; 1689 case RTE_FLOW_ACTION_TYPE_METER: 1690 is_hierarchy = true; 1691 mtr = act->conf; 1692 fm = mlx5_flow_meter_find(priv, 1693 mtr->mtr_id, NULL); 1694 if (!fm) 1695 return -rte_mtr_error_set(error, EINVAL, 1696 RTE_MTR_ERROR_TYPE_MTR_ID, NULL, 1697 "Meter not found in meter hierarchy."); 1698 plc = mlx5_flow_meter_policy_find(dev, 1699 fm->policy_id, 1700 NULL); 1701 MLX5_ASSERT(plc); 1702 domain_color &= MLX5_MTR_ALL_DOMAIN_BIT & 1703 (plc->ingress << 1704 MLX5_MTR_DOMAIN_INGRESS); 1705 domain_color &= MLX5_MTR_ALL_DOMAIN_BIT & 1706 (plc->egress << 1707 MLX5_MTR_DOMAIN_EGRESS); 1708 domain_color &= MLX5_MTR_ALL_DOMAIN_BIT & 1709 (plc->transfer << 1710 MLX5_MTR_DOMAIN_TRANSFER); 1711 break; 1712 default: 1713 break; 1714 } 1715 act++; 1716 } 1717 } 1718 if (priv->sh->config.dv_esw_en) 1719 domain_color &= ~(MLX5_MTR_DOMAIN_EGRESS_BIT | 1720 MLX5_MTR_DOMAIN_TRANSFER_BIT); 1721 else 1722 domain_color &= ~MLX5_MTR_DOMAIN_TRANSFER_BIT; 1723 if (!domain_color) 1724 return -rte_mtr_error_set(error, ENOTSUP, 1725 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1726 NULL, "Meter policy domains are conflicting."); 1727 mtr_policy->is_rss = is_rss; 1728 mtr_policy->ingress = !!(domain_color & MLX5_MTR_DOMAIN_INGRESS_BIT); 1729 pta.ingress = mtr_policy->ingress; 1730 mtr_policy->egress = !!(domain_color & MLX5_MTR_DOMAIN_EGRESS_BIT); 1731 pta.egress = mtr_policy->egress; 1732 mtr_policy->transfer = !!(domain_color & MLX5_MTR_DOMAIN_TRANSFER_BIT); 1733 pta.transfer = mtr_policy->transfer; 1734 mtr_policy->group = MLX5_FLOW_TABLE_HWS_POLICY - policy_id; 1735 mtr_policy->is_hierarchy = is_hierarchy; 1736 mtr_policy->initialized = 1; 1737 rte_spinlock_lock(&priv->hw_ctrl_lock); 1738 mtr_policy->hws_item_templ = 1739 rte_flow_pattern_template_create(dev->data->port_id, 1740 &pta, pattern, NULL); 1741 if (!mtr_policy->hws_item_templ) 1742 goto policy_add_err; 1743 for (i = 0; i < RTE_COLORS; i++) { 1744 if (mtr_policy->skip_g && i == RTE_COLOR_GREEN) 1745 continue; 1746 if (mtr_policy->skip_y && i == RTE_COLOR_YELLOW) 1747 continue; 1748 if (mtr_policy->skip_r && i == RTE_COLOR_RED) 1749 continue; 1750 mtr_policy->hws_act_templ[nb_colors] = 1751 rte_flow_actions_template_create(dev->data->port_id, 1752 &ata, policy->actions[i], 1753 policy->actions[i], NULL); 1754 if (!mtr_policy->hws_act_templ[nb_colors]) 1755 goto policy_add_err; 1756 nb_colors++; 1757 } 1758 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 1759 memset(&ta, 0, sizeof(ta)); 1760 ta.nb_flows = RTE_COLORS; 1761 ta.flow_attr.group = mtr_policy->group; 1762 if (i == MLX5_MTR_DOMAIN_INGRESS) { 1763 if (!mtr_policy->ingress) 1764 continue; 1765 ta.flow_attr.ingress = 1; 1766 } else if (i == MLX5_MTR_DOMAIN_EGRESS) { 1767 if (!mtr_policy->egress) 1768 continue; 1769 ta.flow_attr.egress = 1; 1770 } else if (i == MLX5_MTR_DOMAIN_TRANSFER) { 1771 if (!mtr_policy->transfer) 1772 continue; 1773 ta.flow_attr.transfer = 1; 1774 } 1775 mtr_policy->hws_flow_table[i] = 1776 rte_flow_template_table_create(dev->data->port_id, 1777 &ta, &mtr_policy->hws_item_templ, 1, 1778 mtr_policy->hws_act_templ, nb_colors, 1779 NULL); 1780 if (!mtr_policy->hws_flow_table[i]) 1781 goto policy_add_err; 1782 nb_colors = 0; 1783 for (j = 0; j < RTE_COLORS; j++) { 1784 if (mtr_policy->skip_g && j == RTE_COLOR_GREEN) 1785 continue; 1786 if (mtr_policy->skip_y && j == RTE_COLOR_YELLOW) 1787 continue; 1788 if (mtr_policy->skip_r && j == RTE_COLOR_RED) 1789 continue; 1790 color = rte_col_2_mlx5_col((enum rte_color)j); 1791 tag_spec.data = color; 1792 mtr_policy->hws_flow_rule[i][j] = 1793 rte_flow_async_create(dev->data->port_id, 1794 CTRL_QUEUE_ID(priv), &op_attr, 1795 mtr_policy->hws_flow_table[i], 1796 pattern, 0, policy->actions[j], 1797 nb_colors, NULL, NULL); 1798 if (!mtr_policy->hws_flow_rule[i][j]) 1799 goto policy_add_err; 1800 nb_colors++; 1801 nb_flows++; 1802 } 1803 ret = rte_flow_push(dev->data->port_id, 1804 CTRL_QUEUE_ID(priv), NULL); 1805 if (ret < 0) 1806 goto policy_add_err; 1807 while (nb_flows) { 1808 ret = rte_flow_pull(dev->data->port_id, 1809 CTRL_QUEUE_ID(priv), result, 1810 nb_flows, NULL); 1811 if (ret < 0) 1812 goto policy_add_err; 1813 for (j = 0; j < ret; j++) { 1814 if (result[j].status == RTE_FLOW_OP_ERROR) 1815 goto policy_add_err; 1816 } 1817 nb_flows -= ret; 1818 } 1819 } 1820 rte_spinlock_unlock(&priv->hw_ctrl_lock); 1821 return 0; 1822 policy_add_err: 1823 rte_spinlock_unlock(&priv->hw_ctrl_lock); 1824 ret = mlx5_flow_meter_policy_hws_delete(dev, policy_id, error); 1825 memset(mtr_policy, 0, sizeof(struct mlx5_flow_meter_policy)); 1826 if (ret) 1827 return ret; 1828 return -rte_mtr_error_set(error, ENOTSUP, 1829 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 1830 NULL, "Failed to create meter policy."); 1831 } 1832 #endif 1833 /** 1834 * Check meter validation. 1835 * 1836 * @param[in] priv 1837 * Pointer to mlx5 private data structure. 1838 * @param[in] meter_id 1839 * Meter id. 1840 * @param[in] params 1841 * Pointer to rte meter parameters. 1842 * @param[out] error 1843 * Pointer to rte meter error structure. 1844 * 1845 * @return 1846 * 0 on success, a negative errno value otherwise and rte_errno is set. 1847 */ 1848 static int 1849 mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id, 1850 struct rte_mtr_params *params, 1851 struct rte_mtr_error *error) 1852 { 1853 /* Meter must use global drop action. */ 1854 if (!priv->sh->dr_drop_action) 1855 return -rte_mtr_error_set(error, ENOTSUP, 1856 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 1857 NULL, 1858 "No drop action ready for meter."); 1859 /* Meter params must not be NULL. */ 1860 if (params == NULL) 1861 return -rte_mtr_error_set(error, EINVAL, 1862 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 1863 NULL, "Meter object params null."); 1864 /* Previous meter color is not supported. */ 1865 if (params->use_prev_mtr_color && !priv->sh->meter_aso_en) 1866 return -rte_mtr_error_set(error, ENOTSUP, 1867 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 1868 NULL, 1869 "Previous meter color " 1870 "not supported."); 1871 if (params->meter_policy_id == MLX5_INVALID_POLICY_ID) 1872 return -rte_mtr_error_set(error, ENOENT, 1873 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1874 NULL, "Meter policy id not valid."); 1875 /* Validate meter id. */ 1876 if (mlx5_flow_meter_find(priv, meter_id, NULL)) 1877 return -rte_mtr_error_set(error, EEXIST, 1878 RTE_MTR_ERROR_TYPE_MTR_ID, NULL, 1879 "Meter object already exists."); 1880 return 0; 1881 } 1882 1883 /** 1884 * Modify the flow meter action. 1885 * 1886 * @param[in] priv 1887 * Pointer to mlx5 private data structure. 1888 * @param[in] fm 1889 * Pointer to flow meter to be modified. 1890 * @param[in] srtcm 1891 * Pointer to meter srtcm description parameter. 1892 * @param[in] modify_bits 1893 * The bit in srtcm to be updated. 1894 * @param[in] active_state 1895 * The state to be updated. 1896 * @return 1897 * 0 on success, o negative value otherwise. 1898 */ 1899 static int 1900 mlx5_flow_meter_action_modify(struct mlx5_priv *priv, 1901 struct mlx5_flow_meter_info *fm, 1902 const struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm, 1903 uint64_t modify_bits, uint32_t active_state, uint32_t is_enable) 1904 { 1905 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER 1906 struct mlx5_dev_ctx_shared *sh = priv->sh; 1907 uint32_t in[MLX5_ST_SZ_DW(flow_meter_parameters)] = { 0 }; 1908 uint32_t *attr; 1909 struct mlx5dv_dr_flow_meter_attr mod_attr = { 0 }; 1910 int ret; 1911 struct mlx5_aso_mtr *aso_mtr = NULL; 1912 uint32_t cbs_cir, ebs_eir, val; 1913 1914 if (sh->meter_aso_en) { 1915 fm->is_enable = !!is_enable; 1916 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm); 1917 ret = mlx5_aso_meter_update_by_wqe(priv, MLX5_HW_INV_QUEUE, 1918 aso_mtr, &priv->mtr_bulk, 1919 NULL, true); 1920 if (ret) 1921 return ret; 1922 ret = mlx5_aso_mtr_wait(priv, aso_mtr, false); 1923 if (ret) 1924 return ret; 1925 } else { 1926 /* Fill command parameters. */ 1927 mod_attr.reg_c_index = sh->registers.aso_reg - REG_C_0; 1928 mod_attr.flow_meter_parameter = in; 1929 mod_attr.flow_meter_parameter_sz = 1930 MLX5_ST_SZ_BYTES(flow_meter_parameters); 1931 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE) 1932 mod_attr.active = !!active_state; 1933 else 1934 mod_attr.active = 0; 1935 attr = in; 1936 cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir); 1937 ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir); 1938 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS) { 1939 val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & 1940 ASO_DSEG_EXP_MASK; 1941 MLX5_SET(flow_meter_parameters, attr, 1942 cbs_exponent, val); 1943 val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & 1944 ASO_DSEG_MAN_MASK; 1945 MLX5_SET(flow_meter_parameters, attr, 1946 cbs_mantissa, val); 1947 } 1948 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR) { 1949 val = (cbs_cir >> ASO_DSEG_XIR_EXP_OFFSET) & 1950 ASO_DSEG_EXP_MASK; 1951 MLX5_SET(flow_meter_parameters, attr, 1952 cir_exponent, val); 1953 val = cbs_cir & ASO_DSEG_MAN_MASK; 1954 MLX5_SET(flow_meter_parameters, attr, 1955 cir_mantissa, val); 1956 } 1957 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_EBS) { 1958 val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & 1959 ASO_DSEG_EXP_MASK; 1960 MLX5_SET(flow_meter_parameters, attr, 1961 ebs_exponent, val); 1962 val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & 1963 ASO_DSEG_MAN_MASK; 1964 MLX5_SET(flow_meter_parameters, attr, 1965 ebs_mantissa, val); 1966 } 1967 /* Apply modifications to meter only if it was created. */ 1968 if (fm->meter_action_g) { 1969 ret = mlx5_glue->dv_modify_flow_action_meter 1970 (fm->meter_action_g, &mod_attr, 1971 rte_cpu_to_be_64(modify_bits)); 1972 if (ret) 1973 return ret; 1974 } 1975 /* Update succeeded modify meter parameters. */ 1976 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE) 1977 fm->active_state = !!active_state; 1978 } 1979 return 0; 1980 #else 1981 (void)priv; 1982 (void)fm; 1983 (void)srtcm; 1984 (void)modify_bits; 1985 (void)active_state; 1986 (void)is_enable; 1987 return -ENOTSUP; 1988 #endif 1989 } 1990 1991 static int 1992 mlx5_flow_meter_stats_enable_update(struct rte_eth_dev *dev, 1993 struct mlx5_flow_meter_info *fm, 1994 uint64_t stats_mask) 1995 { 1996 fm->bytes_dropped = 1997 (stats_mask & RTE_MTR_STATS_N_BYTES_DROPPED) ? 1 : 0; 1998 fm->pkts_dropped = (stats_mask & RTE_MTR_STATS_N_PKTS_DROPPED) ? 1 : 0; 1999 if (fm->bytes_dropped || fm->pkts_dropped) { 2000 if (!fm->drop_cnt) { 2001 /* Alloc policer counters. */ 2002 fm->drop_cnt = mlx5_counter_alloc(dev); 2003 if (!fm->drop_cnt) 2004 return -1; 2005 } 2006 } else { 2007 if (fm->drop_cnt) { 2008 mlx5_counter_free(dev, fm->drop_cnt); 2009 fm->drop_cnt = 0; 2010 } 2011 } 2012 return 0; 2013 } 2014 2015 /** 2016 * Create meter rules. 2017 * 2018 * @param[in] dev 2019 * Pointer to Ethernet device. 2020 * @param[in] meter_id 2021 * Meter id. 2022 * @param[in] params 2023 * Pointer to rte meter parameters. 2024 * @param[in] shared 2025 * Meter shared with other flow or not. 2026 * @param[out] error 2027 * Pointer to rte meter error structure. 2028 * 2029 * @return 2030 * 0 on success, a negative errno value otherwise and rte_errno is set. 2031 */ 2032 static int 2033 mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, 2034 struct rte_mtr_params *params, int shared, 2035 struct rte_mtr_error *error) 2036 { 2037 struct mlx5_priv *priv = dev->data->dev_private; 2038 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; 2039 struct mlx5_flow_meter_profile *fmp; 2040 struct mlx5_flow_meter_info *fm; 2041 /* GCC fails to infer legacy_fm is set when !priv->sh->meter_aso_en. */ 2042 struct mlx5_legacy_flow_meter *legacy_fm = NULL; 2043 struct mlx5_flow_meter_policy *mtr_policy = NULL; 2044 struct mlx5_indexed_pool_config flow_ipool_cfg = { 2045 .size = 0, 2046 .trunk_size = 64, 2047 .need_lock = 1, 2048 .type = "mlx5_flow_mtr_flow_id_pool", 2049 }; 2050 struct mlx5_aso_mtr *aso_mtr; 2051 uint32_t mtr_idx, policy_idx; 2052 union mlx5_l3t_data data; 2053 int ret; 2054 uint8_t domain_bitmap; 2055 uint8_t mtr_id_bits; 2056 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 2057 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 2058 2059 if (!priv->mtr_en) 2060 return -rte_mtr_error_set(error, ENOTSUP, 2061 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 2062 "Meter is not supported"); 2063 /* Validate the parameters. */ 2064 ret = mlx5_flow_meter_validate(priv, meter_id, params, error); 2065 if (ret) 2066 return ret; 2067 /* Meter profile must exist. */ 2068 fmp = mlx5_flow_meter_profile_find(priv, params->meter_profile_id); 2069 if (fmp == NULL) 2070 return -rte_mtr_error_set(error, ENOENT, 2071 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 2072 NULL, "Meter profile id not valid."); 2073 /* Meter policy must exist. */ 2074 if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) { 2075 rte_atomic_fetch_add_explicit 2076 (&priv->sh->mtrmng->def_policy_ref_cnt, 2077 1, rte_memory_order_relaxed); 2078 domain_bitmap = MLX5_MTR_ALL_DOMAIN_BIT; 2079 if (!priv->sh->config.dv_esw_en) 2080 domain_bitmap &= ~MLX5_MTR_DOMAIN_TRANSFER_BIT; 2081 } else { 2082 if (!priv->sh->meter_aso_en) 2083 return -rte_mtr_error_set(error, ENOTSUP, 2084 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 2085 "Part of the policies cannot be " 2086 "supported without ASO "); 2087 mtr_policy = mlx5_flow_meter_policy_find(dev, 2088 params->meter_policy_id, &policy_idx); 2089 if (!mtr_policy) 2090 return -rte_mtr_error_set(error, ENOENT, 2091 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 2092 NULL, "Meter policy id not valid."); 2093 domain_bitmap = (mtr_policy->ingress ? 2094 MLX5_MTR_DOMAIN_INGRESS_BIT : 0) | 2095 (mtr_policy->egress ? 2096 MLX5_MTR_DOMAIN_EGRESS_BIT : 0) | 2097 (mtr_policy->transfer ? 2098 MLX5_MTR_DOMAIN_TRANSFER_BIT : 0); 2099 if (fmp->g_support && mtr_policy->skip_g) 2100 return -rte_mtr_error_set(error, ENOTSUP, 2101 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 2102 NULL, "Meter green policy is empty."); 2103 if (fmp->y_support && mtr_policy->skip_y) 2104 return -rte_mtr_error_set(error, ENOTSUP, 2105 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 2106 NULL, "Meter yellow policy is empty."); 2107 } 2108 /* Allocate the flow meter memory. */ 2109 if (priv->sh->meter_aso_en) { 2110 mtr_idx = mlx5_flow_mtr_alloc(dev); 2111 if (!mtr_idx) 2112 return -rte_mtr_error_set(error, ENOMEM, 2113 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 2114 "Memory alloc failed for meter."); 2115 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx); 2116 fm = &aso_mtr->fm; 2117 } else { 2118 if (fmp->y_support) 2119 return -rte_mtr_error_set(error, ENOMEM, 2120 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 2121 "Unsupported profile with yellow."); 2122 legacy_fm = mlx5_ipool_zmalloc 2123 (priv->sh->ipool[MLX5_IPOOL_MTR], &mtr_idx); 2124 if (legacy_fm == NULL) 2125 return -rte_mtr_error_set(error, ENOMEM, 2126 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 2127 "Memory alloc failed for meter."); 2128 legacy_fm->idx = mtr_idx; 2129 fm = &legacy_fm->fm; 2130 } 2131 mtr_id_bits = MLX5_REG_BITS - rte_clz32(mtr_idx); 2132 if ((mtr_id_bits + priv->sh->mtrmng->max_mtr_flow_bits) > 2133 mtr_reg_bits) { 2134 DRV_LOG(ERR, "Meter number exceeds max limit."); 2135 goto error; 2136 } 2137 if (mtr_id_bits > priv->sh->mtrmng->max_mtr_bits) 2138 priv->sh->mtrmng->max_mtr_bits = mtr_id_bits; 2139 /* Fill the flow meter parameters. */ 2140 fm->meter_id = meter_id; 2141 fm->policy_id = params->meter_policy_id; 2142 fm->profile = fmp; 2143 if (mlx5_flow_meter_stats_enable_update(dev, fm, params->stats_mask)) 2144 goto error; 2145 if (mlx5_flow_create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap)) 2146 goto error; 2147 /* Add to the flow meter list. */ 2148 if (!priv->sh->meter_aso_en) { 2149 MLX5_ASSERT(legacy_fm != NULL); 2150 TAILQ_INSERT_TAIL(fms, legacy_fm, next); 2151 } 2152 /* Add to the flow meter list. */ 2153 fm->active_state = 1; /* Config meter starts as active. */ 2154 fm->is_enable = params->meter_enable; 2155 fm->shared = !!shared; 2156 fm->color_aware = !!params->use_prev_mtr_color; 2157 rte_atomic_fetch_add_explicit(&fm->profile->ref_cnt, 1, rte_memory_order_relaxed); 2158 if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) { 2159 fm->def_policy = 1; 2160 fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg); 2161 if (!fm->flow_ipool) 2162 goto error; 2163 } 2164 rte_spinlock_init(&fm->sl); 2165 /* If ASO meter supported, update ASO flow meter by wqe. */ 2166 if (priv->sh->meter_aso_en) { 2167 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm); 2168 ret = mlx5_aso_meter_update_by_wqe(priv, MLX5_HW_INV_QUEUE, 2169 aso_mtr, &priv->mtr_bulk, NULL, true); 2170 if (ret) 2171 goto error; 2172 if (!priv->mtr_idx_tbl) { 2173 priv->mtr_idx_tbl = 2174 mlx5_l3t_create(MLX5_L3T_TYPE_DWORD); 2175 if (!priv->mtr_idx_tbl) 2176 goto error; 2177 } 2178 data.dword = mtr_idx; 2179 if (mlx5_l3t_set_entry(priv->mtr_idx_tbl, meter_id, &data)) 2180 goto error; 2181 } else if (!params->meter_enable && mlx5_flow_meter_disable(dev, meter_id, error)) { 2182 goto error; 2183 } 2184 fm->active_state = params->meter_enable; 2185 if (mtr_policy) 2186 rte_atomic_fetch_add_explicit(&mtr_policy->ref_cnt, 1, rte_memory_order_relaxed); 2187 return 0; 2188 error: 2189 mlx5_flow_destroy_mtr_tbls(dev, fm); 2190 /* Free policer counters. */ 2191 if (fm->drop_cnt) 2192 mlx5_counter_free(dev, fm->drop_cnt); 2193 if (priv->sh->meter_aso_en) 2194 mlx5_flow_mtr_free(dev, mtr_idx); 2195 else 2196 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], mtr_idx); 2197 return -rte_mtr_error_set(error, ENOTSUP, 2198 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 2199 NULL, "Failed to create devx meter."); 2200 } 2201 2202 #if defined(HAVE_MLX5_HWS_SUPPORT) 2203 /** 2204 * Create meter rules. 2205 * 2206 * @param[in] dev 2207 * Pointer to Ethernet device. 2208 * @param[in] meter_id 2209 * Meter id. 2210 * @param[in] params 2211 * Pointer to rte meter parameters. 2212 * @param[in] shared 2213 * Meter shared with other flow or not. 2214 * @param[out] error 2215 * Pointer to rte meter error structure. 2216 * 2217 * @return 2218 * 0 on success, a negative errno value otherwise and rte_errno is set. 2219 */ 2220 static int 2221 mlx5_flow_meter_hws_create(struct rte_eth_dev *dev, uint32_t meter_id, 2222 struct rte_mtr_params *params, int shared, 2223 struct rte_mtr_error *error) 2224 { 2225 struct mlx5_priv *priv = dev->data->dev_private; 2226 struct mlx5_flow_meter_profile *profile; 2227 struct mlx5_flow_meter_info *fm; 2228 struct mlx5_flow_meter_policy *policy = NULL; 2229 struct mlx5_aso_mtr *aso_mtr; 2230 struct mlx5_hw_q_job *job; 2231 int ret; 2232 2233 if (!priv->mtr_profile_arr || 2234 !priv->mtr_policy_arr || 2235 !priv->mtr_bulk.aso) 2236 return -rte_mtr_error_set(error, ENOTSUP, 2237 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 2238 "Meter bulk array is not allocated."); 2239 /* Meter profile must exist. */ 2240 profile = mlx5_flow_meter_profile_find(priv, params->meter_profile_id); 2241 if (!profile->initialized) 2242 return -rte_mtr_error_set(error, ENOENT, 2243 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 2244 NULL, "Meter profile id not valid."); 2245 /* Meter policy must exist. */ 2246 policy = mlx5_flow_meter_policy_find(dev, 2247 params->meter_policy_id, NULL); 2248 if (!policy->initialized) 2249 return -rte_mtr_error_set(error, ENOENT, 2250 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 2251 NULL, "Meter policy id not valid."); 2252 /* Meter ID must be valid. */ 2253 if (meter_id >= priv->mtr_config.nb_meters) 2254 return -rte_mtr_error_set(error, EINVAL, 2255 RTE_MTR_ERROR_TYPE_MTR_ID, 2256 NULL, "Meter id not valid."); 2257 /* Find ASO object. */ 2258 aso_mtr = mlx5_aso_meter_by_idx(priv, meter_id); 2259 fm = &aso_mtr->fm; 2260 if (fm->initialized) 2261 return -rte_mtr_error_set(error, ENOENT, 2262 RTE_MTR_ERROR_TYPE_MTR_ID, 2263 NULL, "Meter object already exists."); 2264 /* Fill the flow meter parameters. */ 2265 fm->meter_id = meter_id; 2266 fm->policy_id = params->meter_policy_id; 2267 fm->profile = profile; 2268 fm->meter_offset = meter_id; 2269 fm->group = policy->group; 2270 /* Add to the flow meter list. */ 2271 fm->active_state = 1; /* Config meter starts as active. */ 2272 fm->is_enable = params->meter_enable; 2273 fm->shared = !!shared; 2274 fm->initialized = 1; 2275 /* Update ASO flow meter by wqe. */ 2276 job = mlx5_flow_action_job_init(priv, MLX5_HW_INV_QUEUE, NULL, NULL, 2277 NULL, MLX5_HW_Q_JOB_TYPE_CREATE, NULL); 2278 if (!job) 2279 return -rte_mtr_error_set(error, ENOMEM, 2280 RTE_MTR_ERROR_TYPE_MTR_ID, 2281 NULL, "No job context."); 2282 ret = mlx5_aso_meter_update_by_wqe(priv, MLX5_HW_INV_QUEUE, aso_mtr, 2283 &priv->mtr_bulk, job, true); 2284 if (ret) { 2285 flow_hw_job_put(priv, job, CTRL_QUEUE_ID(priv)); 2286 return -rte_mtr_error_set(error, ENOTSUP, 2287 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 2288 NULL, "Failed to create devx meter."); 2289 } 2290 fm->active_state = params->meter_enable; 2291 rte_atomic_fetch_add_explicit(&fm->profile->ref_cnt, 1, rte_memory_order_relaxed); 2292 rte_atomic_fetch_add_explicit(&policy->ref_cnt, 1, rte_memory_order_relaxed); 2293 return 0; 2294 } 2295 #endif 2296 2297 static int 2298 mlx5_flow_meter_params_flush(struct rte_eth_dev *dev, 2299 struct mlx5_flow_meter_info *fm, 2300 uint32_t mtr_idx) 2301 { 2302 struct mlx5_priv *priv = dev->data->dev_private; 2303 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; 2304 struct mlx5_flow_meter_profile *fmp; 2305 struct mlx5_legacy_flow_meter *legacy_fm = NULL; 2306 struct mlx5_flow_meter_policy *mtr_policy; 2307 2308 /* Meter object must not have any owner. */ 2309 MLX5_ASSERT(!fm->ref_cnt); 2310 /* Get meter profile. */ 2311 fmp = fm->profile; 2312 if (fmp == NULL) 2313 return -1; 2314 /* Update dependencies. */ 2315 rte_atomic_fetch_sub_explicit(&fmp->ref_cnt, 1, rte_memory_order_relaxed); 2316 fm->profile = NULL; 2317 /* Remove from list. */ 2318 if (!priv->sh->meter_aso_en) { 2319 legacy_fm = container_of(fm, 2320 struct mlx5_legacy_flow_meter, fm); 2321 TAILQ_REMOVE(fms, legacy_fm, next); 2322 } 2323 /* Free drop counters. */ 2324 if (fm->drop_cnt) 2325 mlx5_counter_free(dev, fm->drop_cnt); 2326 /* Free meter flow table. */ 2327 if (fm->flow_ipool) { 2328 mlx5_ipool_destroy(fm->flow_ipool); 2329 fm->flow_ipool = 0; 2330 } 2331 mlx5_flow_destroy_mtr_tbls(dev, fm); 2332 if (fm->def_policy) 2333 rte_atomic_fetch_sub_explicit(&priv->sh->mtrmng->def_policy_ref_cnt, 2334 1, rte_memory_order_relaxed); 2335 if (priv->sh->meter_aso_en) { 2336 if (!fm->def_policy) { 2337 mtr_policy = mlx5_flow_meter_policy_find(dev, 2338 fm->policy_id, NULL); 2339 if (mtr_policy) 2340 rte_atomic_fetch_sub_explicit(&mtr_policy->ref_cnt, 2341 1, rte_memory_order_relaxed); 2342 fm->policy_id = 0; 2343 } 2344 fm->def_policy = 0; 2345 if (mlx5_l3t_clear_entry(priv->mtr_idx_tbl, fm->meter_id)) 2346 return -1; 2347 mlx5_flow_mtr_free(dev, mtr_idx); 2348 } else { 2349 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], 2350 legacy_fm->idx); 2351 } 2352 return 0; 2353 } 2354 2355 /** 2356 * Destroy meter rules. 2357 * 2358 * @param[in] dev 2359 * Pointer to Ethernet device. 2360 * @param[in] meter_id 2361 * Meter id. 2362 * @param[out] error 2363 * Pointer to rte meter error structure. 2364 * 2365 * @return 2366 * 0 on success, a negative errno value otherwise and rte_errno is set. 2367 */ 2368 static int 2369 mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id, 2370 struct rte_mtr_error *error) 2371 { 2372 struct mlx5_priv *priv = dev->data->dev_private; 2373 struct mlx5_flow_meter_info *fm; 2374 uint32_t mtr_idx = 0; 2375 2376 if (!priv->mtr_en) 2377 return -rte_mtr_error_set(error, ENOTSUP, 2378 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 2379 NULL, 2380 "Meter is not supported"); 2381 /* Meter object must exist. */ 2382 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 2383 if (fm == NULL) 2384 return -rte_mtr_error_set(error, ENOENT, 2385 RTE_MTR_ERROR_TYPE_MTR_ID, 2386 NULL, 2387 "Meter object id not valid."); 2388 /* Meter object must not have any owner. */ 2389 if (fm->ref_cnt > 0) 2390 return -rte_mtr_error_set(error, EBUSY, 2391 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 2392 NULL, 2393 "Meter object is being used."); 2394 /* Destroy the meter profile. */ 2395 if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx)) 2396 return -rte_mtr_error_set(error, EINVAL, 2397 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 2398 NULL, 2399 "MTR object meter profile invalid."); 2400 return 0; 2401 } 2402 2403 /** 2404 * Destroy meter rules. 2405 * 2406 * @param[in] dev 2407 * Pointer to Ethernet device. 2408 * @param[in] meter_id 2409 * Meter id. 2410 * @param[out] error 2411 * Pointer to rte meter error structure. 2412 * 2413 * @return 2414 * 0 on success, a negative errno value otherwise and rte_errno is set. 2415 */ 2416 static int 2417 mlx5_flow_meter_hws_destroy(struct rte_eth_dev *dev, uint32_t meter_id, 2418 struct rte_mtr_error *error) 2419 { 2420 struct mlx5_priv *priv = dev->data->dev_private; 2421 struct mlx5_aso_mtr *aso_mtr; 2422 struct mlx5_flow_meter_info *fm; 2423 struct mlx5_flow_meter_policy *policy; 2424 2425 if (!priv->mtr_profile_arr || 2426 !priv->mtr_policy_arr || 2427 !priv->mtr_bulk.aso) 2428 return -rte_mtr_error_set(error, ENOTSUP, 2429 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL, 2430 "Meter bulk array is not allocated."); 2431 /* Find ASO object. */ 2432 aso_mtr = mlx5_aso_meter_by_idx(priv, meter_id); 2433 fm = &aso_mtr->fm; 2434 if (!fm->initialized) 2435 return -rte_mtr_error_set(error, ENOENT, 2436 RTE_MTR_ERROR_TYPE_MTR_ID, 2437 NULL, "Meter object id not valid."); 2438 /* Meter object must not have any owner. */ 2439 if (fm->ref_cnt > 0) 2440 return -rte_mtr_error_set(error, EBUSY, 2441 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 2442 NULL, "Meter object is being used."); 2443 /* Destroy the meter profile. */ 2444 rte_atomic_fetch_sub_explicit(&fm->profile->ref_cnt, 2445 1, rte_memory_order_relaxed); 2446 /* Destroy the meter policy. */ 2447 policy = mlx5_flow_meter_policy_find(dev, 2448 fm->policy_id, NULL); 2449 rte_atomic_fetch_sub_explicit(&policy->ref_cnt, 2450 1, rte_memory_order_relaxed); 2451 memset(fm, 0, sizeof(struct mlx5_flow_meter_info)); 2452 return 0; 2453 } 2454 2455 /** 2456 * Modify meter state. 2457 * 2458 * @param[in] priv 2459 * Pointer to mlx5 private data structure. 2460 * @param[in] fm 2461 * Pointer to flow meter. 2462 * @param[in] new_state 2463 * New state to update. 2464 * @param[out] error 2465 * Pointer to rte meter error structure. 2466 * 2467 * @return 2468 * 0 on success, a negative errno value otherwise and rte_errno is set. 2469 */ 2470 static int 2471 mlx5_flow_meter_modify_state(struct mlx5_priv *priv, 2472 struct mlx5_flow_meter_info *fm, 2473 uint32_t new_state, 2474 struct rte_mtr_error *error) 2475 { 2476 static const struct mlx5_flow_meter_srtcm_rfc2697_prm srtcm = { 2477 .cbs_cir = RTE_BE32(MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL), 2478 .ebs_eir = 0, 2479 }; 2480 uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS | 2481 MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR; 2482 int ret; 2483 2484 if (new_state == MLX5_FLOW_METER_DISABLE) 2485 ret = mlx5_flow_meter_action_modify(priv, fm, 2486 &srtcm, modify_bits, 0, 0); 2487 else 2488 ret = mlx5_flow_meter_action_modify(priv, fm, 2489 &fm->profile->srtcm_prm, 2490 modify_bits, 0, 1); 2491 if (ret) 2492 return -rte_mtr_error_set(error, -ret, 2493 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 2494 NULL, 2495 new_state ? 2496 "Failed to enable meter." : 2497 "Failed to disable meter."); 2498 return 0; 2499 } 2500 2501 /** 2502 * Callback to enable flow meter. 2503 * 2504 * @param[in] dev 2505 * Pointer to Ethernet device. 2506 * @param[in] meter_id 2507 * Meter id. 2508 * @param[out] error 2509 * Pointer to rte meter error structure. 2510 * 2511 * @return 2512 * 0 on success, a negative errno value otherwise and rte_errno is set. 2513 */ 2514 static int 2515 mlx5_flow_meter_enable(struct rte_eth_dev *dev, 2516 uint32_t meter_id, 2517 struct rte_mtr_error *error) 2518 { 2519 struct mlx5_priv *priv = dev->data->dev_private; 2520 struct mlx5_flow_meter_info *fm; 2521 int ret; 2522 2523 if (!priv->mtr_en) 2524 return -rte_mtr_error_set(error, ENOTSUP, 2525 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 2526 "Meter is not supported"); 2527 /* Meter object must exist. */ 2528 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 2529 if (fm == NULL) 2530 return -rte_mtr_error_set(error, ENOENT, 2531 RTE_MTR_ERROR_TYPE_MTR_ID, 2532 NULL, "Meter not found."); 2533 if (fm->active_state == MLX5_FLOW_METER_ENABLE) 2534 return 0; 2535 ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_ENABLE, 2536 error); 2537 if (!ret) 2538 fm->active_state = MLX5_FLOW_METER_ENABLE; 2539 return ret; 2540 } 2541 2542 /** 2543 * Callback to disable flow meter. 2544 * 2545 * @param[in] dev 2546 * Pointer to Ethernet device. 2547 * @param[in] meter_id 2548 * Meter id. 2549 * @param[out] error 2550 * Pointer to rte meter error structure. 2551 * 2552 * @return 2553 * 0 on success, a negative errno value otherwise and rte_errno is set. 2554 */ 2555 static int 2556 mlx5_flow_meter_disable(struct rte_eth_dev *dev, 2557 uint32_t meter_id, 2558 struct rte_mtr_error *error) 2559 { 2560 struct mlx5_priv *priv = dev->data->dev_private; 2561 struct mlx5_flow_meter_info *fm; 2562 int ret; 2563 2564 if (!priv->mtr_en) 2565 return -rte_mtr_error_set(error, ENOTSUP, 2566 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 2567 "Meter is not supported"); 2568 /* Meter object must exist. */ 2569 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 2570 if (fm == NULL) 2571 return -rte_mtr_error_set(error, ENOENT, 2572 RTE_MTR_ERROR_TYPE_MTR_ID, 2573 NULL, "Meter not found."); 2574 if (fm->active_state == MLX5_FLOW_METER_DISABLE) 2575 return 0; 2576 ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_DISABLE, 2577 error); 2578 if (!ret) 2579 fm->active_state = MLX5_FLOW_METER_DISABLE; 2580 return ret; 2581 } 2582 2583 /** 2584 * Callback to update meter profile. 2585 * 2586 * @param[in] dev 2587 * Pointer to Ethernet device. 2588 * @param[in] meter_id 2589 * Meter id. 2590 * @param[in] meter_profile_id 2591 * To be updated meter profile id. 2592 * @param[out] error 2593 * Pointer to rte meter error structure. 2594 * 2595 * @return 2596 * 0 on success, a negative errno value otherwise and rte_errno is set. 2597 */ 2598 static int 2599 mlx5_flow_meter_profile_update(struct rte_eth_dev *dev, 2600 uint32_t meter_id, 2601 uint32_t meter_profile_id, 2602 struct rte_mtr_error *error) 2603 { 2604 struct mlx5_priv *priv = dev->data->dev_private; 2605 struct mlx5_flow_meter_profile *fmp; 2606 struct mlx5_flow_meter_profile *old_fmp; 2607 struct mlx5_flow_meter_info *fm; 2608 uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS | 2609 MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR; 2610 int ret; 2611 2612 if (!priv->mtr_en) 2613 return -rte_mtr_error_set(error, ENOTSUP, 2614 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 2615 "Meter is not supported"); 2616 /* Meter profile must exist. */ 2617 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 2618 if (fmp == NULL) 2619 return -rte_mtr_error_set(error, ENOENT, 2620 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 2621 NULL, "Meter profile not found."); 2622 /* Meter object must exist. */ 2623 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 2624 if (fm == NULL) 2625 return -rte_mtr_error_set(error, ENOENT, 2626 RTE_MTR_ERROR_TYPE_MTR_ID, 2627 NULL, "Meter not found."); 2628 /* MTR object already set to meter profile id. */ 2629 old_fmp = fm->profile; 2630 if (fmp == old_fmp) 2631 return 0; 2632 /* Update the profile. */ 2633 fm->profile = fmp; 2634 /* Update meter params in HW (if not disabled). */ 2635 if (fm->active_state == MLX5_FLOW_METER_DISABLE) 2636 goto dec_ref_cnt; 2637 ret = mlx5_flow_meter_action_modify(priv, fm, &fm->profile->srtcm_prm, 2638 modify_bits, fm->active_state, 1); 2639 if (ret) { 2640 fm->profile = old_fmp; 2641 return -rte_mtr_error_set(error, -ret, 2642 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 2643 NULL, "Failed to update meter" 2644 " parameters in hardware."); 2645 } 2646 dec_ref_cnt: 2647 old_fmp->ref_cnt--; 2648 fmp->ref_cnt++; 2649 return 0; 2650 } 2651 2652 /** 2653 * Callback to update meter stats mask. 2654 * 2655 * @param[in] dev 2656 * Pointer to Ethernet device. 2657 * @param[in] meter_id 2658 * Meter id. 2659 * @param[in] stats_mask 2660 * To be updated stats_mask. 2661 * @param[out] error 2662 * Pointer to rte meter error structure. 2663 * 2664 * @return 2665 * 0 on success, a negative errno value otherwise and rte_errno is set. 2666 */ 2667 static int 2668 mlx5_flow_meter_stats_update(struct rte_eth_dev *dev, 2669 uint32_t meter_id, 2670 uint64_t stats_mask, 2671 struct rte_mtr_error *error) 2672 { 2673 struct mlx5_priv *priv = dev->data->dev_private; 2674 struct mlx5_flow_meter_info *fm; 2675 2676 if (!priv->mtr_en) 2677 return -rte_mtr_error_set(error, ENOTSUP, 2678 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 2679 "Meter is not supported"); 2680 /* Meter object must exist. */ 2681 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 2682 if (fm == NULL) 2683 return -rte_mtr_error_set(error, ENOENT, 2684 RTE_MTR_ERROR_TYPE_MTR_ID, 2685 NULL, "Meter object id not valid."); 2686 if (mlx5_flow_meter_stats_enable_update(dev, fm, stats_mask)) 2687 return -rte_mtr_error_set(error, ENOENT, 2688 RTE_MTR_ERROR_TYPE_MTR_ID, 2689 NULL, "Fail to allocate " 2690 "counter for meter."); 2691 return 0; 2692 } 2693 2694 /** 2695 * Callback to read meter statistics. 2696 * 2697 * @param[in] dev 2698 * Pointer to Ethernet device. 2699 * @param[in] meter_id 2700 * Meter id. 2701 * @param[out] stats 2702 * Pointer to store the statistics. 2703 * @param[out] stats_mask 2704 * Pointer to store the stats_mask. 2705 * @param[in] clear 2706 * Statistic to be cleared after read or not. 2707 * @param[out] error 2708 * Pointer to rte meter error structure. 2709 * 2710 * @return 2711 * 0 on success, a negative errno value otherwise and rte_errno is set. 2712 */ 2713 static int 2714 mlx5_flow_meter_stats_read(struct rte_eth_dev *dev, 2715 uint32_t meter_id, 2716 struct rte_mtr_stats *stats, 2717 uint64_t *stats_mask, 2718 int clear, 2719 struct rte_mtr_error *error) 2720 { 2721 struct mlx5_priv *priv = dev->data->dev_private; 2722 struct mlx5_flow_meter_info *fm; 2723 uint64_t pkts; 2724 uint64_t bytes; 2725 int ret = 0; 2726 2727 if (!priv->mtr_en) 2728 return -rte_mtr_error_set(error, ENOTSUP, 2729 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 2730 "Meter is not supported"); 2731 /* Meter object must exist. */ 2732 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 2733 if (fm == NULL) 2734 return -rte_mtr_error_set(error, ENOENT, 2735 RTE_MTR_ERROR_TYPE_MTR_ID, 2736 NULL, "Meter object id not valid."); 2737 *stats_mask = 0; 2738 if (fm->bytes_dropped) 2739 *stats_mask |= RTE_MTR_STATS_N_BYTES_DROPPED; 2740 if (fm->pkts_dropped) 2741 *stats_mask |= RTE_MTR_STATS_N_PKTS_DROPPED; 2742 memset(stats, 0, sizeof(*stats)); 2743 if (fm->drop_cnt) { 2744 ret = mlx5_counter_query(dev, fm->drop_cnt, clear, &pkts, 2745 &bytes, NULL); 2746 if (ret) 2747 goto error; 2748 /* If need to read the packets, set it. */ 2749 if (fm->pkts_dropped) 2750 stats->n_pkts_dropped = pkts; 2751 /* If need to read the bytes, set it. */ 2752 if (fm->bytes_dropped) 2753 stats->n_bytes_dropped = bytes; 2754 } 2755 return 0; 2756 error: 2757 return -rte_mtr_error_set(error, ret, RTE_MTR_ERROR_TYPE_STATS, NULL, 2758 "Failed to read meter drop counters."); 2759 } 2760 2761 static const struct rte_mtr_ops mlx5_flow_mtr_ops = { 2762 .capabilities_get = mlx5_flow_mtr_cap_get, 2763 .meter_profile_add = mlx5_flow_meter_profile_add, 2764 .meter_profile_delete = mlx5_flow_meter_profile_delete, 2765 .meter_profile_get = mlx5_flow_meter_profile_get, 2766 .meter_policy_validate = mlx5_flow_meter_policy_validate, 2767 .meter_policy_add = mlx5_flow_meter_policy_add, 2768 .meter_policy_delete = mlx5_flow_meter_policy_delete, 2769 .meter_policy_get = mlx5_flow_meter_policy_get, 2770 .create = mlx5_flow_meter_create, 2771 .destroy = mlx5_flow_meter_destroy, 2772 .meter_enable = mlx5_flow_meter_enable, 2773 .meter_disable = mlx5_flow_meter_disable, 2774 .meter_profile_update = mlx5_flow_meter_profile_update, 2775 .meter_dscp_table_update = NULL, 2776 .stats_update = mlx5_flow_meter_stats_update, 2777 .stats_read = mlx5_flow_meter_stats_read, 2778 }; 2779 2780 #if defined(HAVE_MLX5_HWS_SUPPORT) 2781 static const struct rte_mtr_ops mlx5_flow_mtr_hws_ops = { 2782 .capabilities_get = mlx5_flow_mtr_cap_get, 2783 .meter_profile_add = mlx5_flow_meter_profile_hws_add, 2784 .meter_profile_delete = mlx5_flow_meter_profile_hws_delete, 2785 .meter_profile_get = mlx5_flow_meter_profile_get, 2786 .meter_policy_validate = mlx5_flow_meter_policy_hws_validate, 2787 .meter_policy_add = mlx5_flow_meter_policy_hws_add, 2788 .meter_policy_delete = mlx5_flow_meter_policy_hws_delete, 2789 .meter_policy_get = mlx5_flow_meter_policy_get, 2790 .create = mlx5_flow_meter_hws_create, 2791 .destroy = mlx5_flow_meter_hws_destroy, 2792 .meter_enable = mlx5_flow_meter_enable, 2793 .meter_disable = mlx5_flow_meter_disable, 2794 .meter_profile_update = mlx5_flow_meter_profile_update, 2795 .meter_dscp_table_update = NULL, 2796 .stats_update = NULL, 2797 .stats_read = NULL, 2798 }; 2799 #endif 2800 2801 /** 2802 * Get meter operations. 2803 * 2804 * @param dev 2805 * Pointer to Ethernet device structure. 2806 * @param arg 2807 * Pointer to set the mtr operations. 2808 * 2809 * @return 2810 * Always 0. 2811 */ 2812 int 2813 mlx5_flow_meter_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg) 2814 { 2815 #if defined(HAVE_MLX5_HWS_SUPPORT) 2816 struct mlx5_priv *priv = dev->data->dev_private; 2817 2818 if (priv->sh->config.dv_flow_en == 2) 2819 *(const struct rte_mtr_ops **)arg = &mlx5_flow_mtr_hws_ops; 2820 else 2821 *(const struct rte_mtr_ops **)arg = &mlx5_flow_mtr_ops; 2822 #else 2823 *(const struct rte_mtr_ops **)arg = &mlx5_flow_mtr_ops; 2824 #endif 2825 return 0; 2826 } 2827 2828 /** 2829 * Find meter by id. 2830 * 2831 * @param priv 2832 * Pointer to mlx5_priv. 2833 * @param meter_id 2834 * Meter id. 2835 * @param mtr_idx 2836 * Pointer to Meter index. 2837 * 2838 * @return 2839 * Pointer to the meter info found on success, NULL otherwise. 2840 */ 2841 struct mlx5_flow_meter_info * 2842 mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id, 2843 uint32_t *mtr_idx) 2844 { 2845 struct mlx5_legacy_flow_meter *legacy_fm; 2846 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; 2847 struct mlx5_aso_mtr *aso_mtr; 2848 struct mlx5_aso_mtr_pools_mng *pools_mng = 2849 &priv->sh->mtrmng->pools_mng; 2850 union mlx5_l3t_data data; 2851 uint16_t n_valid; 2852 2853 if (priv->mtr_bulk.aso) { 2854 if (mtr_idx) 2855 *mtr_idx = meter_id; 2856 aso_mtr = priv->mtr_bulk.aso + meter_id; 2857 return &aso_mtr->fm; 2858 } 2859 if (priv->sh->meter_aso_en) { 2860 rte_rwlock_read_lock(&pools_mng->resize_mtrwl); 2861 n_valid = pools_mng->n_valid; 2862 rte_rwlock_read_unlock(&pools_mng->resize_mtrwl); 2863 if (!n_valid || !priv->mtr_idx_tbl || 2864 (mlx5_l3t_get_entry(priv->mtr_idx_tbl, meter_id, &data) || 2865 !data.dword)) 2866 return NULL; 2867 if (mtr_idx) 2868 *mtr_idx = data.dword; 2869 aso_mtr = mlx5_aso_meter_by_idx(priv, data.dword); 2870 /* Remove reference taken by the mlx5_l3t_get_entry. */ 2871 mlx5_l3t_clear_entry(priv->mtr_idx_tbl, meter_id); 2872 if (!aso_mtr || aso_mtr->state == ASO_METER_FREE) 2873 return NULL; 2874 return &aso_mtr->fm; 2875 } 2876 TAILQ_FOREACH(legacy_fm, fms, next) 2877 if (meter_id == legacy_fm->fm.meter_id) { 2878 if (mtr_idx) 2879 *mtr_idx = legacy_fm->idx; 2880 return &legacy_fm->fm; 2881 } 2882 return NULL; 2883 } 2884 2885 /** 2886 * Find meter by index. 2887 * 2888 * @param priv 2889 * Pointer to mlx5_priv. 2890 * @param idx 2891 * Meter index. 2892 * 2893 * @return 2894 * Pointer to the meter info found on success, NULL otherwise. 2895 */ 2896 struct mlx5_flow_meter_info * 2897 flow_dv_meter_find_by_idx(struct mlx5_priv *priv, uint32_t idx) 2898 { 2899 struct mlx5_aso_mtr *aso_mtr; 2900 2901 if (priv->sh->meter_aso_en) { 2902 aso_mtr = mlx5_aso_meter_by_idx(priv, idx); 2903 if (!aso_mtr) 2904 return NULL; 2905 return &aso_mtr->fm; 2906 } else { 2907 return mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], idx); 2908 } 2909 } 2910 2911 /** 2912 * Attach meter to flow. 2913 * Unidirectional Meter creation can only be done 2914 * when flow direction is known, i.e. when calling meter_attach. 2915 * 2916 * @param [in] priv 2917 * Pointer to mlx5 private data. 2918 * @param[in] fm 2919 * Pointer to flow meter. 2920 * @param [in] attr 2921 * Pointer to flow attributes. 2922 * @param [out] error 2923 * Pointer to error structure. 2924 * 2925 * @return 2926 * 0 on success, a negative errno value otherwise and rte_errno is set. 2927 */ 2928 int 2929 mlx5_flow_meter_attach(struct mlx5_priv *priv, 2930 struct mlx5_flow_meter_info *fm, 2931 const struct rte_flow_attr *attr, 2932 struct rte_flow_error *error) 2933 { 2934 int ret = 0; 2935 2936 if (priv->sh->meter_aso_en) { 2937 struct mlx5_aso_mtr *aso_mtr; 2938 2939 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm); 2940 if (mlx5_aso_mtr_wait(priv, aso_mtr, false)) { 2941 return rte_flow_error_set(error, ENOENT, 2942 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2943 NULL, 2944 "Timeout in meter configuration"); 2945 } 2946 rte_spinlock_lock(&fm->sl); 2947 if (fm->shared || !fm->ref_cnt) { 2948 fm->ref_cnt++; 2949 } else { 2950 rte_flow_error_set(error, EINVAL, 2951 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 2952 "Meter cannot be shared"); 2953 ret = -1; 2954 } 2955 rte_spinlock_unlock(&fm->sl); 2956 } else { 2957 rte_spinlock_lock(&fm->sl); 2958 if (fm->meter_action_g) { 2959 if (fm->shared && 2960 attr->transfer == fm->transfer && 2961 attr->ingress == fm->ingress && 2962 attr->egress == fm->egress) { 2963 fm->ref_cnt++; 2964 } else { 2965 rte_flow_error_set(error, EINVAL, 2966 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 2967 fm->shared ? 2968 "Meter attr not match." : 2969 "Meter cannot be shared."); 2970 ret = -1; 2971 } 2972 } else { 2973 fm->ingress = attr->ingress; 2974 fm->egress = attr->egress; 2975 fm->transfer = attr->transfer; 2976 fm->ref_cnt = 1; 2977 /* This also creates the meter object. */ 2978 fm->meter_action_g = mlx5_flow_meter_action_create(priv, 2979 fm); 2980 if (!fm->meter_action_g) { 2981 fm->ref_cnt = 0; 2982 fm->ingress = 0; 2983 fm->egress = 0; 2984 fm->transfer = 0; 2985 rte_flow_error_set(error, EINVAL, 2986 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 2987 "Meter action create failed."); 2988 ret = -1; 2989 } 2990 } 2991 rte_spinlock_unlock(&fm->sl); 2992 } 2993 return ret ? -rte_errno : 0; 2994 } 2995 2996 /** 2997 * Detach meter from flow. 2998 * 2999 * @param [in] priv 3000 * Pointer to mlx5 private data. 3001 * @param [in] fm 3002 * Pointer to flow meter. 3003 */ 3004 void 3005 mlx5_flow_meter_detach(struct mlx5_priv *priv, 3006 struct mlx5_flow_meter_info *fm) 3007 { 3008 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER 3009 rte_spinlock_lock(&fm->sl); 3010 MLX5_ASSERT(fm->ref_cnt); 3011 if (--fm->ref_cnt == 0 && !priv->sh->meter_aso_en) { 3012 mlx5_glue->destroy_flow_action(fm->meter_action_g); 3013 fm->meter_action_g = NULL; 3014 fm->ingress = 0; 3015 fm->egress = 0; 3016 fm->transfer = 0; 3017 } 3018 rte_spinlock_unlock(&fm->sl); 3019 #else 3020 (void)priv; 3021 (void)fm; 3022 #endif 3023 } 3024 3025 /** 3026 * Flush meter with Rx queue configuration. 3027 * 3028 * @param[in] dev 3029 * Pointer to Ethernet device. 3030 */ 3031 void 3032 mlx5_flow_meter_rxq_flush(struct rte_eth_dev *dev) 3033 { 3034 struct mlx5_priv *priv = dev->data->dev_private; 3035 struct mlx5_flow_meter_sub_policy *sub_policy; 3036 struct mlx5_flow_meter_policy *mtr_policy; 3037 void *entry; 3038 uint32_t i, policy_idx; 3039 3040 if (!priv->mtr_en) 3041 return; 3042 if (priv->policy_idx_tbl) { 3043 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) { 3044 policy_idx = *(uint32_t *)entry; 3045 sub_policy = mlx5_ipool_get 3046 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 3047 policy_idx); 3048 if (!sub_policy || !sub_policy->main_policy) 3049 continue; 3050 mtr_policy = sub_policy->main_policy; 3051 if (mtr_policy->is_queue || mtr_policy->is_rss) 3052 mlx5_flow_destroy_sub_policy_with_rxq(dev, 3053 mtr_policy); 3054 } 3055 } 3056 } 3057 3058 /** 3059 * Iterate a meter hierarchy and flush all meters and policies if possible. 3060 * 3061 * @param[in] dev 3062 * Pointer to Ethernet device. 3063 * @param[in] fm 3064 * Pointer to flow meter. 3065 * @param[in] mtr_idx 3066 * .Meter's index 3067 * @param[out] error 3068 * Pointer to rte meter error structure. 3069 * 3070 * @return 3071 * 0 on success, a negative errno value otherwise and rte_errno is set. 3072 */ 3073 static int 3074 mlx5_flow_meter_flush_hierarchy(struct rte_eth_dev *dev, 3075 struct mlx5_flow_meter_info *fm, 3076 uint32_t mtr_idx, 3077 struct rte_mtr_error *error) 3078 { 3079 struct mlx5_priv *priv = dev->data->dev_private; 3080 struct mlx5_flow_meter_policy *policy; 3081 uint32_t policy_id; 3082 struct mlx5_flow_meter_info *next_fm; 3083 uint32_t next_mtr_idx; 3084 struct mlx5_flow_meter_policy *next_policy = NULL; 3085 3086 policy = mlx5_flow_meter_policy_find(dev, fm->policy_id, NULL); 3087 MLX5_ASSERT(policy); 3088 while (!fm->ref_cnt && policy->is_hierarchy) { 3089 policy_id = fm->policy_id; 3090 next_fm = mlx5_flow_meter_hierarchy_next_meter(priv, policy, &next_mtr_idx); 3091 if (next_fm) { 3092 next_policy = mlx5_flow_meter_policy_find(dev, 3093 next_fm->policy_id, 3094 NULL); 3095 MLX5_ASSERT(next_policy); 3096 } 3097 if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx)) 3098 return -rte_mtr_error_set(error, ENOTSUP, 3099 RTE_MTR_ERROR_TYPE_MTR_ID, 3100 NULL, 3101 "Failed to flush meter."); 3102 if (policy->ref_cnt) 3103 break; 3104 if (__mlx5_flow_meter_policy_delete(dev, policy_id, 3105 policy, error, true)) 3106 return -rte_errno; 3107 mlx5_free(policy); 3108 if (!next_fm || !next_policy) 3109 break; 3110 fm = next_fm; 3111 mtr_idx = next_mtr_idx; 3112 policy = next_policy; 3113 } 3114 return 0; 3115 } 3116 3117 /** 3118 * Flush all the hierarchy meters and their policies. 3119 * 3120 * @param[in] dev 3121 * Pointer to Ethernet device. 3122 * @param[out] error 3123 * Pointer to rte meter error structure. 3124 * 3125 * @return 3126 * 0 on success, a negative errno value otherwise and rte_errno is set. 3127 */ 3128 static int 3129 mlx5_flow_meter_flush_all_hierarchies(struct rte_eth_dev *dev, 3130 struct rte_mtr_error *error) 3131 { 3132 struct mlx5_priv *priv = dev->data->dev_private; 3133 struct mlx5_flow_meter_info *fm; 3134 struct mlx5_flow_meter_policy *policy; 3135 struct mlx5_flow_meter_sub_policy *sub_policy; 3136 struct mlx5_flow_meter_info *next_fm; 3137 struct mlx5_aso_mtr *aso_mtr; 3138 uint32_t mtr_idx = 0; 3139 uint32_t i, policy_idx; 3140 void *entry; 3141 3142 if (!priv->mtr_idx_tbl || !priv->policy_idx_tbl) 3143 return 0; 3144 MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) { 3145 mtr_idx = *(uint32_t *)entry; 3146 if (!mtr_idx) 3147 continue; 3148 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx); 3149 fm = &aso_mtr->fm; 3150 if (fm->ref_cnt || fm->def_policy) 3151 continue; 3152 if (mlx5_flow_meter_flush_hierarchy(dev, fm, mtr_idx, error)) 3153 return -rte_errno; 3154 } 3155 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) { 3156 policy_idx = *(uint32_t *)entry; 3157 sub_policy = mlx5_ipool_get 3158 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 3159 policy_idx); 3160 if (!sub_policy) 3161 return -rte_mtr_error_set(error, 3162 EINVAL, 3163 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 3164 NULL, "Meter policy invalid."); 3165 policy = sub_policy->main_policy; 3166 if (!policy || !policy->is_hierarchy || policy->ref_cnt) 3167 continue; 3168 next_fm = mlx5_flow_meter_hierarchy_next_meter(priv, policy, &mtr_idx); 3169 if (__mlx5_flow_meter_policy_delete(dev, i, policy, 3170 error, true)) 3171 return -rte_mtr_error_set(error, 3172 EINVAL, 3173 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 3174 NULL, "Meter policy invalid."); 3175 mlx5_free(policy); 3176 if (!next_fm || next_fm->ref_cnt || next_fm->def_policy) 3177 continue; 3178 if (mlx5_flow_meter_flush_hierarchy(dev, next_fm, 3179 mtr_idx, error)) 3180 return -rte_errno; 3181 } 3182 return 0; 3183 } 3184 /** 3185 * Flush meter configuration. 3186 * 3187 * @param[in] dev 3188 * Pointer to Ethernet device. 3189 * @param[out] error 3190 * Pointer to rte meter error structure. 3191 * 3192 * @return 3193 * 0 on success, a negative errno value otherwise and rte_errno is set. 3194 */ 3195 int 3196 mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error) 3197 { 3198 struct mlx5_priv *priv = dev->data->dev_private; 3199 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; 3200 struct mlx5_flow_meter_profile *fmp; 3201 struct mlx5_legacy_flow_meter *legacy_fm; 3202 struct mlx5_flow_meter_info *fm; 3203 struct mlx5_flow_meter_sub_policy *sub_policy; 3204 void *tmp; 3205 uint32_t i, mtr_idx, policy_idx; 3206 void *entry; 3207 struct mlx5_aso_mtr *aso_mtr; 3208 3209 if (!priv->mtr_en) 3210 return 0; 3211 if (priv->sh->meter_aso_en) { 3212 if (mlx5_flow_meter_flush_all_hierarchies(dev, error)) 3213 return -rte_errno; 3214 if (priv->mtr_idx_tbl) { 3215 MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) { 3216 mtr_idx = *(uint32_t *)entry; 3217 if (mtr_idx) { 3218 aso_mtr = 3219 mlx5_aso_meter_by_idx(priv, mtr_idx); 3220 fm = &aso_mtr->fm; 3221 (void)mlx5_flow_meter_params_flush(dev, 3222 fm, mtr_idx); 3223 } 3224 } 3225 mlx5_l3t_destroy(priv->mtr_idx_tbl); 3226 priv->mtr_idx_tbl = NULL; 3227 } 3228 } else { 3229 RTE_TAILQ_FOREACH_SAFE(legacy_fm, fms, next, tmp) { 3230 fm = &legacy_fm->fm; 3231 if (mlx5_flow_meter_params_flush(dev, fm, 0)) 3232 return -rte_mtr_error_set(error, EINVAL, 3233 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 3234 NULL, "MTR object meter profile invalid."); 3235 } 3236 } 3237 if (priv->mtr_bulk.aso) { 3238 for (i = 0; i < priv->mtr_config.nb_meters; i++) { 3239 aso_mtr = mlx5_aso_meter_by_idx(priv, i); 3240 fm = &aso_mtr->fm; 3241 if (fm->initialized) 3242 mlx5_flow_meter_hws_destroy(dev, i, error); 3243 } 3244 } 3245 if (priv->policy_idx_tbl) { 3246 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) { 3247 policy_idx = *(uint32_t *)entry; 3248 sub_policy = mlx5_ipool_get 3249 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 3250 policy_idx); 3251 if (!sub_policy) 3252 return -rte_mtr_error_set(error, 3253 EINVAL, 3254 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 3255 NULL, "MTR object " 3256 "meter policy invalid."); 3257 if (__mlx5_flow_meter_policy_delete(dev, i, 3258 sub_policy->main_policy, 3259 error, true)) 3260 return -rte_mtr_error_set(error, 3261 EINVAL, 3262 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 3263 NULL, "MTR object " 3264 "meter policy invalid."); 3265 mlx5_free(sub_policy->main_policy); 3266 } 3267 mlx5_l3t_destroy(priv->policy_idx_tbl); 3268 priv->policy_idx_tbl = NULL; 3269 } 3270 #if defined(HAVE_MLX5_HWS_SUPPORT) 3271 if (priv->mtr_policy_arr) { 3272 struct mlx5_flow_meter_policy *policy; 3273 3274 for (i = 0; i < priv->mtr_config.nb_meter_policies; i++) { 3275 policy = mlx5_flow_meter_policy_find(dev, i, 3276 &policy_idx); 3277 if (policy->initialized) { 3278 mlx5_flow_meter_policy_hws_delete(dev, i, 3279 error); 3280 } 3281 } 3282 } 3283 #endif 3284 if (priv->mtr_profile_tbl) { 3285 MLX5_L3T_FOREACH(priv->mtr_profile_tbl, i, entry) { 3286 fmp = entry; 3287 if (mlx5_flow_meter_profile_delete(dev, fmp->id, 3288 error)) 3289 return -rte_mtr_error_set(error, EINVAL, 3290 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 3291 NULL, "Fail to destroy " 3292 "meter profile."); 3293 } 3294 mlx5_l3t_destroy(priv->mtr_profile_tbl); 3295 priv->mtr_profile_tbl = NULL; 3296 } 3297 #if defined(HAVE_MLX5_HWS_SUPPORT) 3298 if (priv->mtr_profile_arr) { 3299 for (i = 0; i < priv->mtr_config.nb_meter_profiles; i++) { 3300 fmp = mlx5_flow_meter_profile_find(priv, i); 3301 if (fmp->initialized) { 3302 mlx5_flow_meter_profile_hws_delete(dev, i, 3303 error); 3304 } 3305 } 3306 } 3307 #endif 3308 /* Delete default policy table. */ 3309 mlx5_flow_destroy_def_policy(dev); 3310 if (priv->sh->refcnt == 1) 3311 mlx5_flow_destroy_mtr_drop_tbls(dev); 3312 #ifdef HAVE_MLX5_HWS_SUPPORT 3313 /* Destroy HWS configuration. */ 3314 mlx5_flow_meter_uninit(dev); 3315 #endif 3316 return 0; 3317 } 3318