1 // SPDX-License-Identifier: BSD-3-Clause 2 /* 3 * Copyright 2018 Mellanox Technologies, Ltd 4 */ 5 #include <math.h> 6 7 #include <rte_tailq.h> 8 #include <rte_malloc.h> 9 #include <rte_mtr.h> 10 #include <rte_mtr_driver.h> 11 12 #include <mlx5_devx_cmds.h> 13 #include <mlx5_malloc.h> 14 15 #include "mlx5.h" 16 #include "mlx5_flow.h" 17 18 static int mlx5_flow_meter_disable(struct rte_eth_dev *dev, 19 uint32_t meter_id, struct rte_mtr_error *error); 20 21 /** 22 * Create the meter action. 23 * 24 * @param priv 25 * Pointer to mlx5_priv. 26 * @param[in] fm 27 * Pointer to flow meter to be converted. 28 * 29 * @return 30 * Pointer to the meter action on success, NULL otherwise. 31 */ 32 static void * 33 mlx5_flow_meter_action_create(struct mlx5_priv *priv, 34 struct mlx5_flow_meter_info *fm) 35 { 36 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER 37 struct mlx5dv_dr_flow_meter_attr mtr_init; 38 uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)]; 39 struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = 40 &fm->profile->srtcm_prm; 41 uint32_t cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir); 42 uint32_t ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir); 43 uint32_t val; 44 enum mlx5_meter_domain domain = 45 fm->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 46 fm->egress ? MLX5_MTR_DOMAIN_EGRESS : 47 MLX5_MTR_DOMAIN_INGRESS; 48 struct mlx5_flow_meter_def_policy *def_policy = 49 priv->sh->mtrmng->def_policy[domain]; 50 51 memset(fmp, 0, MLX5_ST_SZ_BYTES(flow_meter_parameters)); 52 MLX5_SET(flow_meter_parameters, fmp, valid, 1); 53 MLX5_SET(flow_meter_parameters, fmp, bucket_overflow, 1); 54 MLX5_SET(flow_meter_parameters, fmp, 55 start_color, MLX5_FLOW_COLOR_GREEN); 56 MLX5_SET(flow_meter_parameters, fmp, both_buckets_on_green, 0); 57 val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK; 58 MLX5_SET(flow_meter_parameters, fmp, cbs_exponent, val); 59 val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK; 60 MLX5_SET(flow_meter_parameters, fmp, cbs_mantissa, val); 61 val = (cbs_cir >> ASO_DSEG_XIR_EXP_OFFSET) & ASO_DSEG_EXP_MASK; 62 MLX5_SET(flow_meter_parameters, fmp, cir_exponent, val); 63 val = (cbs_cir & ASO_DSEG_MAN_MASK); 64 MLX5_SET(flow_meter_parameters, fmp, cir_mantissa, val); 65 val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK; 66 MLX5_SET(flow_meter_parameters, fmp, ebs_exponent, val); 67 val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK; 68 MLX5_SET(flow_meter_parameters, fmp, ebs_mantissa, val); 69 mtr_init.next_table = def_policy->sub_policy.tbl_rsc->obj; 70 mtr_init.reg_c_index = priv->mtr_color_reg - REG_C_0; 71 mtr_init.flow_meter_parameter = fmp; 72 mtr_init.flow_meter_parameter_sz = 73 MLX5_ST_SZ_BYTES(flow_meter_parameters); 74 mtr_init.active = fm->active_state; 75 return mlx5_glue->dv_create_flow_action_meter(&mtr_init); 76 #else 77 (void)priv; 78 (void)fm; 79 return NULL; 80 #endif 81 } 82 83 /** 84 * Find meter profile by id. 85 * 86 * @param priv 87 * Pointer to mlx5_priv. 88 * @param meter_profile_id 89 * Meter profile id. 90 * 91 * @return 92 * Pointer to the profile found on success, NULL otherwise. 93 */ 94 static struct mlx5_flow_meter_profile * 95 mlx5_flow_meter_profile_find(struct mlx5_priv *priv, uint32_t meter_profile_id) 96 { 97 struct mlx5_flow_meter_profile *fmp; 98 union mlx5_l3t_data data; 99 int32_t ret; 100 101 if (priv->mtr_profile_arr) 102 return &priv->mtr_profile_arr[meter_profile_id]; 103 if (mlx5_l3t_get_entry(priv->mtr_profile_tbl, 104 meter_profile_id, &data) || !data.ptr) 105 return NULL; 106 fmp = data.ptr; 107 /* Remove reference taken by the mlx5_l3t_get_entry. */ 108 ret = mlx5_l3t_clear_entry(priv->mtr_profile_tbl, 109 meter_profile_id); 110 if (!ret || ret == -1) 111 return NULL; 112 return fmp; 113 } 114 115 /** 116 * Validate the MTR profile. 117 * 118 * @param[in] dev 119 * Pointer to Ethernet device. 120 * @param[in] meter_profile_id 121 * Meter profile id. 122 * @param[in] profile 123 * Pointer to meter profile detail. 124 * @param[out] error 125 * Pointer to the error structure. 126 * 127 * @return 128 * 0 on success, a negative errno value otherwise and rte_errno is set. 129 */ 130 static int 131 mlx5_flow_meter_profile_validate(struct rte_eth_dev *dev, 132 uint32_t meter_profile_id, 133 struct rte_mtr_meter_profile *profile, 134 struct rte_mtr_error *error) 135 { 136 struct mlx5_priv *priv = dev->data->dev_private; 137 struct mlx5_flow_meter_profile *fmp; 138 uint32_t ls_factor; 139 int ret; 140 uint64_t cir, cbs; 141 uint64_t eir, ebs; 142 uint64_t pir, pbs; 143 144 /* Profile must not be NULL. */ 145 if (profile == NULL) 146 return -rte_mtr_error_set(error, EINVAL, 147 RTE_MTR_ERROR_TYPE_METER_PROFILE, 148 NULL, "Meter profile is null."); 149 /* Meter profile ID must be valid. */ 150 if (priv->mtr_profile_arr) { 151 if (meter_profile_id >= priv->mtr_config.nb_meter_profiles) 152 return -rte_mtr_error_set(error, EINVAL, 153 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 154 NULL, "Meter profile id not valid."); 155 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 156 /* Meter profile must not exist. */ 157 if (fmp->initialized) 158 return -rte_mtr_error_set(error, EEXIST, 159 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 160 NULL, "Meter profile already exists."); 161 } else { 162 if (meter_profile_id == UINT32_MAX) 163 return -rte_mtr_error_set(error, EINVAL, 164 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 165 NULL, "Meter profile id not valid."); 166 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 167 /* Meter profile must not exist. */ 168 if (fmp) 169 return -rte_mtr_error_set(error, EEXIST, 170 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 171 NULL, "Meter profile already exists."); 172 } 173 if (!priv->sh->meter_aso_en) { 174 /* Old version is even not supported. */ 175 if (!priv->sh->cdev->config.hca_attr.qos.flow_meter_old) 176 return -rte_mtr_error_set(error, ENOTSUP, 177 RTE_MTR_ERROR_TYPE_METER_PROFILE, 178 NULL, "Metering is not supported."); 179 /* Old FW metering only supports srTCM. */ 180 if (profile->alg != RTE_MTR_SRTCM_RFC2697) { 181 return -rte_mtr_error_set(error, ENOTSUP, 182 RTE_MTR_ERROR_TYPE_METER_PROFILE, 183 NULL, "Metering algorithm is not supported."); 184 } else if (profile->srtcm_rfc2697.ebs) { 185 /* EBS is not supported for old metering. */ 186 return -rte_mtr_error_set(error, ENOTSUP, 187 RTE_MTR_ERROR_TYPE_METER_PROFILE, 188 NULL, "EBS is not supported."); 189 } 190 if (profile->packet_mode) 191 return -rte_mtr_error_set(error, ENOTSUP, 192 RTE_MTR_ERROR_TYPE_METER_PROFILE, NULL, 193 "Metering algorithm packet mode is not supported."); 194 } 195 ls_factor = profile->packet_mode ? MLX5_MTRS_PPS_MAP_BPS_SHIFT : 0; 196 switch (profile->alg) { 197 case RTE_MTR_SRTCM_RFC2697: 198 cir = profile->srtcm_rfc2697.cir << ls_factor; 199 cbs = profile->srtcm_rfc2697.cbs << ls_factor; 200 ebs = profile->srtcm_rfc2697.ebs << ls_factor; 201 /* EBS could be zero for old metering. */ 202 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX && 203 cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX && 204 ebs <= MLX5_SRTCM_XBS_MAX) { 205 ret = 0; 206 } else { 207 ret = -rte_mtr_error_set(error, ENOTSUP, 208 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 209 "Profile values out of range."); 210 } 211 break; 212 case RTE_MTR_TRTCM_RFC2698: 213 cir = profile->trtcm_rfc2698.cir << ls_factor; 214 cbs = profile->trtcm_rfc2698.cbs << ls_factor; 215 pir = profile->trtcm_rfc2698.pir << ls_factor; 216 pbs = profile->trtcm_rfc2698.pbs << ls_factor; 217 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX && 218 cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX && 219 pir >= cir && pir <= (MLX5_SRTCM_XIR_MAX * 2) && 220 pbs >= cbs && pbs <= (MLX5_SRTCM_XBS_MAX * 2)) { 221 ret = 0; 222 } else { 223 ret = -rte_mtr_error_set(error, ENOTSUP, 224 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 225 "Profile values out of range."); 226 } 227 break; 228 case RTE_MTR_TRTCM_RFC4115: 229 cir = profile->trtcm_rfc4115.cir << ls_factor; 230 cbs = profile->trtcm_rfc4115.cbs << ls_factor; 231 eir = profile->trtcm_rfc4115.eir << ls_factor; 232 ebs = profile->trtcm_rfc4115.ebs << ls_factor; 233 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX && 234 cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX && 235 eir <= MLX5_SRTCM_XIR_MAX && ebs <= MLX5_SRTCM_XBS_MAX) { 236 ret = 0; 237 } else { 238 ret = -rte_mtr_error_set(error, ENOTSUP, 239 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 240 "Profile values out of range."); 241 } 242 break; 243 default: 244 ret = -rte_mtr_error_set(error, ENOTSUP, 245 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 246 "Unknown metering algorithm."); 247 break; 248 } 249 return ret; 250 } 251 252 /* 253 * Calculate mantissa and exponent for cir / eir. 254 * 255 * @param[in] xir 256 * Value to be calculated. 257 * @param[out] man 258 * Pointer to the mantissa. 259 * @param[out] exp 260 * Pointer to the exp. 261 */ 262 static inline void 263 mlx5_flow_meter_xir_man_exp_calc(int64_t xir, uint8_t *man, uint8_t *exp) 264 { 265 int64_t _xir; 266 int64_t delta = INT64_MAX; 267 uint8_t _man = 0; 268 uint8_t _exp = 0; 269 uint64_t m, e; 270 271 /* Special case xir == 0 ? both exp and mantissa are 0. */ 272 if (xir == 0) { 273 *man = 0; 274 *exp = 0; 275 return; 276 } 277 for (m = 0; m <= 0xFF; m++) { /* man width 8 bit */ 278 for (e = 0; e <= 0x1F; e++) { /* exp width 5bit */ 279 _xir = (1000000000ULL * m) >> e; 280 if (llabs(xir - _xir) <= delta) { 281 delta = llabs(xir - _xir); 282 _man = m; 283 _exp = e; 284 } 285 } 286 } 287 *man = _man; 288 *exp = _exp; 289 } 290 291 /* 292 * Calculate mantissa and exponent for xbs. 293 * 294 * @param[in] xbs 295 * Value to be calculated. 296 * @param[out] man 297 * Pointer to the mantissa. 298 * @param[out] exp 299 * Pointer to the exp. 300 */ 301 static inline void 302 mlx5_flow_meter_xbs_man_exp_calc(uint64_t xbs, uint8_t *man, uint8_t *exp) 303 { 304 int _exp; 305 double _man; 306 307 /* Special case xbs == 0 ? both exp and mantissa are 0. */ 308 if (xbs == 0) { 309 *man = 0; 310 *exp = 0; 311 return; 312 } 313 /* xbs = xbs_mantissa * 2^xbs_exponent */ 314 _man = frexp(xbs, &_exp); 315 if (_exp >= MLX5_MAN_WIDTH) { 316 _man = _man * pow(2, MLX5_MAN_WIDTH); 317 _exp = _exp - MLX5_MAN_WIDTH; 318 } 319 *man = (uint8_t)ceil(_man); 320 *exp = _exp; 321 } 322 323 /** 324 * Fill the prm meter parameter. 325 * 326 * @param[in,out] fmp 327 * Pointer to meter profile to be converted. 328 * @param[out] error 329 * Pointer to the error structure. 330 * 331 * @return 332 * 0 on success, a negative errno value otherwise and rte_errno is set. 333 */ 334 static int 335 mlx5_flow_meter_param_fill(struct mlx5_flow_meter_profile *fmp, 336 struct rte_mtr_error *error) 337 { 338 struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = &fmp->srtcm_prm; 339 uint8_t man, exp; 340 uint32_t cbs_exp, cbs_man, cir_exp, cir_man; 341 uint32_t eir_exp, eir_man, ebs_exp, ebs_man; 342 uint64_t cir, cbs, eir, ebs; 343 344 switch (fmp->profile.alg) { 345 case RTE_MTR_SRTCM_RFC2697: 346 cir = fmp->profile.srtcm_rfc2697.cir; 347 cbs = fmp->profile.srtcm_rfc2697.cbs; 348 eir = 0; 349 ebs = fmp->profile.srtcm_rfc2697.ebs; 350 break; 351 case RTE_MTR_TRTCM_RFC2698: 352 MLX5_ASSERT(fmp->profile.trtcm_rfc2698.pir > 353 fmp->profile.trtcm_rfc2698.cir && 354 fmp->profile.trtcm_rfc2698.pbs > 355 fmp->profile.trtcm_rfc2698.cbs); 356 cir = fmp->profile.trtcm_rfc2698.cir; 357 cbs = fmp->profile.trtcm_rfc2698.cbs; 358 /* EIR / EBS are filled with PIR / PBS. */ 359 eir = fmp->profile.trtcm_rfc2698.pir; 360 ebs = fmp->profile.trtcm_rfc2698.pbs; 361 break; 362 case RTE_MTR_TRTCM_RFC4115: 363 cir = fmp->profile.trtcm_rfc4115.cir; 364 cbs = fmp->profile.trtcm_rfc4115.cbs; 365 eir = fmp->profile.trtcm_rfc4115.eir; 366 ebs = fmp->profile.trtcm_rfc4115.ebs; 367 break; 368 default: 369 return -rte_mtr_error_set(error, EINVAL, 370 RTE_MTR_ERROR_TYPE_METER_PROFILE, NULL, 371 "Metering algorithm mode is invalid"); 372 } 373 /* Adjust the values for PPS mode. */ 374 if (fmp->profile.packet_mode) { 375 cir <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT; 376 cbs <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT; 377 eir <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT; 378 ebs <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT; 379 } 380 /* cir = 8G * cir_mantissa * 1/(2^cir_exponent)) Bytes/Sec */ 381 mlx5_flow_meter_xir_man_exp_calc(cir, &man, &exp); 382 /* Check if cir mantissa is too large. */ 383 if (exp > ASO_DSEG_XIR_EXP_MASK) 384 return -rte_mtr_error_set(error, ENOTSUP, 385 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 386 "meter profile parameter cir is not supported."); 387 cir_man = man; 388 cir_exp = exp; 389 /* cbs = cbs_mantissa * 2^cbs_exponent */ 390 mlx5_flow_meter_xbs_man_exp_calc(cbs, &man, &exp); 391 /* Check if cbs mantissa is too large. */ 392 if (exp > ASO_DSEG_EXP_MASK) 393 return -rte_mtr_error_set(error, ENOTSUP, 394 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 395 "meter profile parameter cbs is not supported."); 396 cbs_man = man; 397 cbs_exp = exp; 398 srtcm->cbs_cir = rte_cpu_to_be_32(cbs_exp << ASO_DSEG_CBS_EXP_OFFSET | 399 cbs_man << ASO_DSEG_CBS_MAN_OFFSET | 400 cir_exp << ASO_DSEG_XIR_EXP_OFFSET | 401 cir_man); 402 mlx5_flow_meter_xir_man_exp_calc(eir, &man, &exp); 403 /* Check if eir mantissa is too large. */ 404 if (exp > ASO_DSEG_XIR_EXP_MASK) 405 return -rte_mtr_error_set(error, ENOTSUP, 406 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 407 "meter profile parameter eir is not supported."); 408 eir_man = man; 409 eir_exp = exp; 410 mlx5_flow_meter_xbs_man_exp_calc(ebs, &man, &exp); 411 /* Check if ebs mantissa is too large. */ 412 if (exp > ASO_DSEG_EXP_MASK) 413 return -rte_mtr_error_set(error, ENOTSUP, 414 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 415 "meter profile parameter ebs is not supported."); 416 ebs_man = man; 417 ebs_exp = exp; 418 srtcm->ebs_eir = rte_cpu_to_be_32(ebs_exp << ASO_DSEG_EBS_EXP_OFFSET | 419 ebs_man << ASO_DSEG_EBS_MAN_OFFSET | 420 eir_exp << ASO_DSEG_XIR_EXP_OFFSET | 421 eir_man); 422 if (srtcm->cbs_cir) 423 fmp->g_support = 1; 424 if (srtcm->ebs_eir) 425 fmp->y_support = 1; 426 return 0; 427 } 428 429 /** 430 * Callback to get MTR capabilities. 431 * 432 * @param[in] dev 433 * Pointer to Ethernet device. 434 * @param[out] cap 435 * Pointer to save MTR capabilities. 436 * @param[out] error 437 * Pointer to the error structure. 438 * 439 * @return 440 * 0 on success, a negative errno value otherwise and rte_errno is set. 441 */ 442 static int 443 mlx5_flow_mtr_cap_get(struct rte_eth_dev *dev, 444 struct rte_mtr_capabilities *cap, 445 struct rte_mtr_error *error __rte_unused) 446 { 447 struct mlx5_priv *priv = dev->data->dev_private; 448 struct mlx5_hca_qos_attr *qattr = &priv->sh->cdev->config.hca_attr.qos; 449 450 if (!priv->mtr_en) 451 return -rte_mtr_error_set(error, ENOTSUP, 452 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 453 "Meter is not supported"); 454 memset(cap, 0, sizeof(*cap)); 455 if (priv->sh->meter_aso_en) { 456 /* 2 meters per one ASO cache line. */ 457 cap->n_max = 1 << (qattr->log_max_num_meter_aso + 1); 458 cap->srtcm_rfc2697_packet_mode_supported = 1; 459 cap->trtcm_rfc2698_packet_mode_supported = 1; 460 cap->trtcm_rfc4115_packet_mode_supported = 1; 461 } else { 462 cap->n_max = 1 << qattr->log_max_flow_meter; 463 } 464 cap->srtcm_rfc2697_byte_mode_supported = 1; 465 cap->trtcm_rfc2698_byte_mode_supported = 1; 466 cap->trtcm_rfc4115_byte_mode_supported = 1; 467 cap->n_shared_max = cap->n_max; 468 cap->identical = 1; 469 cap->shared_identical = 1; 470 cap->shared_n_flows_per_mtr_max = 4 << 20; 471 /* 2M flows can share the same meter. */ 472 cap->chaining_n_mtrs_per_flow_max = 1; /* Chaining is not supported. */ 473 cap->meter_srtcm_rfc2697_n_max = qattr->flow_meter_old ? cap->n_max : 0; 474 cap->meter_trtcm_rfc2698_n_max = qattr->flow_meter_old ? cap->n_max : 0; 475 cap->meter_trtcm_rfc4115_n_max = qattr->flow_meter_old ? cap->n_max : 0; 476 cap->meter_rate_max = 1ULL << 40; /* 1 Tera tokens per sec. */ 477 cap->meter_policy_n_max = cap->n_max; 478 cap->stats_mask = RTE_MTR_STATS_N_BYTES_DROPPED | 479 RTE_MTR_STATS_N_PKTS_DROPPED; 480 return 0; 481 } 482 483 /** 484 * Callback to add MTR profile. 485 * 486 * @param[in] dev 487 * Pointer to Ethernet device. 488 * @param[in] meter_profile_id 489 * Meter profile id. 490 * @param[in] profile 491 * Pointer to meter profile detail. 492 * @param[out] error 493 * Pointer to the error structure. 494 * 495 * @return 496 * 0 on success, a negative errno value otherwise and rte_errno is set. 497 */ 498 static int 499 mlx5_flow_meter_profile_add(struct rte_eth_dev *dev, 500 uint32_t meter_profile_id, 501 struct rte_mtr_meter_profile *profile, 502 struct rte_mtr_error *error) 503 { 504 struct mlx5_priv *priv = dev->data->dev_private; 505 struct mlx5_flow_meter_profile *fmp; 506 union mlx5_l3t_data data; 507 int ret; 508 509 if (!priv->mtr_en) 510 return -rte_mtr_error_set(error, ENOTSUP, 511 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 512 "Meter is not supported"); 513 /* Check input params. */ 514 ret = mlx5_flow_meter_profile_validate(dev, meter_profile_id, 515 profile, error); 516 if (ret) 517 return ret; 518 /* Meter profile memory allocation. */ 519 fmp = mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct mlx5_flow_meter_profile), 520 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 521 if (fmp == NULL) 522 return -rte_mtr_error_set(error, ENOMEM, 523 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 524 NULL, "Meter profile memory " 525 "alloc failed."); 526 /* Fill profile info. */ 527 fmp->id = meter_profile_id; 528 fmp->profile = *profile; 529 /* Fill the flow meter parameters for the PRM. */ 530 ret = mlx5_flow_meter_param_fill(fmp, error); 531 if (ret) 532 goto error; 533 data.ptr = fmp; 534 ret = mlx5_l3t_set_entry(priv->mtr_profile_tbl, 535 meter_profile_id, &data); 536 if (ret) 537 return -rte_mtr_error_set(error, ENOTSUP, 538 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 539 NULL, "Meter profile insert fail."); 540 return 0; 541 error: 542 mlx5_free(fmp); 543 return ret; 544 } 545 546 /** 547 * Callback to delete MTR profile. 548 * 549 * @param[in] dev 550 * Pointer to Ethernet device. 551 * @param[in] meter_profile_id 552 * Meter profile id. 553 * @param[out] error 554 * Pointer to the error structure. 555 * 556 * @return 557 * 0 on success, a negative errno value otherwise and rte_errno is set. 558 */ 559 static int 560 mlx5_flow_meter_profile_delete(struct rte_eth_dev *dev, 561 uint32_t meter_profile_id, 562 struct rte_mtr_error *error) 563 { 564 struct mlx5_priv *priv = dev->data->dev_private; 565 struct mlx5_flow_meter_profile *fmp; 566 567 if (!priv->mtr_en) 568 return -rte_mtr_error_set(error, ENOTSUP, 569 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 570 "Meter is not supported"); 571 /* Meter profile must exist. */ 572 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 573 if (fmp == NULL) 574 return -rte_mtr_error_set(error, ENOENT, 575 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 576 &meter_profile_id, 577 "Meter profile id is invalid."); 578 /* Check profile is unused. */ 579 if (fmp->ref_cnt) 580 return -rte_mtr_error_set(error, EBUSY, 581 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 582 NULL, "Meter profile is in use."); 583 if (mlx5_l3t_clear_entry(priv->mtr_profile_tbl, meter_profile_id)) 584 return -rte_mtr_error_set(error, EBUSY, 585 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 586 NULL, "Meter profile remove fail."); 587 mlx5_free(fmp); 588 return 0; 589 } 590 591 /** 592 * Callback to get MTR profile. 593 * 594 * @param[in] dev 595 * Pointer to Ethernet device. 596 * @param[in] meter_profile_id 597 * Meter profile id. 598 * @param[out] error 599 * Pointer to the error structure. 600 * 601 * @return 602 * A valid handle in case of success, NULL otherwise. 603 */ 604 static struct rte_flow_meter_profile * 605 mlx5_flow_meter_profile_get(struct rte_eth_dev *dev, 606 uint32_t meter_profile_id, 607 struct rte_mtr_error *error) 608 { 609 struct mlx5_priv *priv = dev->data->dev_private; 610 611 if (!priv->mtr_en) { 612 rte_mtr_error_set(error, ENOTSUP, 613 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 614 "Meter is not supported"); 615 return NULL; 616 } 617 return (void *)(uintptr_t)mlx5_flow_meter_profile_find(priv, 618 meter_profile_id); 619 } 620 621 /** 622 * Callback to add MTR profile with HWS. 623 * 624 * @param[in] dev 625 * Pointer to Ethernet device. 626 * @param[in] meter_profile_id 627 * Meter profile id. 628 * @param[in] profile 629 * Pointer to meter profile detail. 630 * @param[out] error 631 * Pointer to the error structure. 632 * 633 * @return 634 * 0 on success, a negative errno value otherwise and rte_errno is set. 635 */ 636 static int 637 mlx5_flow_meter_profile_hws_add(struct rte_eth_dev *dev, 638 uint32_t meter_profile_id, 639 struct rte_mtr_meter_profile *profile, 640 struct rte_mtr_error *error) 641 { 642 struct mlx5_priv *priv = dev->data->dev_private; 643 struct mlx5_flow_meter_profile *fmp; 644 int ret; 645 646 if (!priv->mtr_profile_arr) 647 return -rte_mtr_error_set(error, ENOTSUP, 648 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 649 NULL, "Meter profile array is not allocated"); 650 /* Check input params. */ 651 ret = mlx5_flow_meter_profile_validate(dev, meter_profile_id, 652 profile, error); 653 if (ret) 654 return ret; 655 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 656 /* Fill profile info. */ 657 fmp->id = meter_profile_id; 658 fmp->profile = *profile; 659 fmp->initialized = 1; 660 /* Fill the flow meter parameters for the PRM. */ 661 return mlx5_flow_meter_param_fill(fmp, error); 662 } 663 664 /** 665 * Callback to delete MTR profile with HWS. 666 * 667 * @param[in] dev 668 * Pointer to Ethernet device. 669 * @param[in] meter_profile_id 670 * Meter profile id. 671 * @param[out] error 672 * Pointer to the error structure. 673 * 674 * @return 675 * 0 on success, a negative errno value otherwise and rte_errno is set. 676 */ 677 static int 678 mlx5_flow_meter_profile_hws_delete(struct rte_eth_dev *dev, 679 uint32_t meter_profile_id, 680 struct rte_mtr_error *error) 681 { 682 struct mlx5_priv *priv = dev->data->dev_private; 683 struct mlx5_flow_meter_profile *fmp; 684 685 if (!priv->mtr_profile_arr) 686 return -rte_mtr_error_set(error, ENOTSUP, 687 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 688 NULL, "Meter profile array is not allocated"); 689 /* Meter id must be valid. */ 690 if (meter_profile_id >= priv->mtr_config.nb_meter_profiles) 691 return -rte_mtr_error_set(error, EINVAL, 692 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 693 &meter_profile_id, 694 "Meter profile id not valid."); 695 /* Meter profile must exist. */ 696 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 697 if (!fmp->initialized) 698 return -rte_mtr_error_set(error, ENOENT, 699 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 700 &meter_profile_id, 701 "Meter profile id is invalid."); 702 /* Check profile is unused. */ 703 if (fmp->ref_cnt) 704 return -rte_mtr_error_set(error, EBUSY, 705 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 706 NULL, "Meter profile is in use."); 707 memset(fmp, 0, sizeof(struct mlx5_flow_meter_profile)); 708 return 0; 709 } 710 711 /** 712 * Find policy by id. 713 * 714 * @param[in] dev 715 * Pointer to Ethernet device. 716 * @param policy_id 717 * Policy id. 718 * 719 * @return 720 * Pointer to the policy found on success, NULL otherwise. 721 */ 722 struct mlx5_flow_meter_policy * 723 mlx5_flow_meter_policy_find(struct rte_eth_dev *dev, 724 uint32_t policy_id, 725 uint32_t *policy_idx) 726 { 727 struct mlx5_priv *priv = dev->data->dev_private; 728 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 729 union mlx5_l3t_data data; 730 731 if (priv->mtr_policy_arr) { 732 if (policy_idx) 733 *policy_idx = policy_id; 734 return &priv->mtr_policy_arr[policy_id]; 735 } 736 if (policy_id > MLX5_MAX_SUB_POLICY_TBL_NUM || !priv->policy_idx_tbl) 737 return NULL; 738 if (mlx5_l3t_get_entry(priv->policy_idx_tbl, policy_id, &data) || 739 !data.dword) 740 return NULL; 741 if (policy_idx) 742 *policy_idx = data.dword; 743 sub_policy = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 744 data.dword); 745 /* Remove reference taken by the mlx5_l3t_get_entry. */ 746 mlx5_l3t_clear_entry(priv->policy_idx_tbl, policy_id); 747 if (sub_policy) 748 if (sub_policy->main_policy_id) 749 return sub_policy->main_policy; 750 return NULL; 751 } 752 753 /** 754 * Get the next meter from one meter's policy in hierarchy chain. 755 * Lock free, mutex should be acquired by caller. 756 * 757 * @param[in] priv 758 * Pointer to mlx5_priv. 759 * @param[in] policy 760 * Pointer to flow meter policy. 761 * @param[out] mtr_idx 762 * Pointer to Meter index. 763 * 764 * @return 765 * Pointer to the next meter, or NULL when fail. 766 */ 767 struct mlx5_flow_meter_info * 768 mlx5_flow_meter_hierarchy_next_meter(struct mlx5_priv *priv, 769 struct mlx5_flow_meter_policy *policy, 770 uint32_t *mtr_idx) 771 { 772 int i; 773 774 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 775 if (policy->act_cnt[i].fate_action == MLX5_FLOW_FATE_MTR) 776 return mlx5_flow_meter_find(priv, 777 policy->act_cnt[i].next_mtr_id, 778 mtr_idx); 779 } 780 return NULL; 781 } 782 783 /** 784 * Get the last meter's policy from one meter's policy in hierarchy. 785 * 786 * @param[in] dev 787 * Pointer to Ethernet device. 788 * @param[in] policy 789 * Pointer to flow meter policy. 790 * 791 * @return 792 * Pointer to the final meter's policy, or NULL when fail. 793 */ 794 struct mlx5_flow_meter_policy * 795 mlx5_flow_meter_hierarchy_get_final_policy(struct rte_eth_dev *dev, 796 struct mlx5_flow_meter_policy *policy) 797 { 798 struct mlx5_priv *priv = dev->data->dev_private; 799 struct mlx5_flow_meter_info *next_fm; 800 struct mlx5_flow_meter_policy *next_policy = policy; 801 802 while (next_policy->is_hierarchy) { 803 rte_spinlock_lock(&next_policy->sl); 804 next_fm = mlx5_flow_meter_hierarchy_next_meter(priv, next_policy, NULL); 805 rte_spinlock_unlock(&next_policy->sl); 806 if (!next_fm || next_fm->def_policy) 807 return NULL; 808 next_policy = mlx5_flow_meter_policy_find(dev, 809 next_fm->policy_id, NULL); 810 MLX5_ASSERT(next_policy); 811 } 812 return next_policy; 813 } 814 815 /** 816 * Callback to check MTR policy action validate 817 * 818 * @param[in] dev 819 * Pointer to Ethernet device. 820 * @param[in] actions 821 * Pointer to meter policy action detail. 822 * @param[out] error 823 * Pointer to the error structure. 824 * 825 * @return 826 * 0 on success, a negative errno value otherwise and rte_errno is set. 827 */ 828 static int 829 mlx5_flow_meter_policy_validate(struct rte_eth_dev *dev, 830 struct rte_mtr_meter_policy_params *policy, 831 struct rte_mtr_error *error) 832 { 833 struct mlx5_priv *priv = dev->data->dev_private; 834 struct rte_flow_attr attr = { .transfer = priv->sh->config.dv_esw_en ? 835 1 : 0 }; 836 bool is_rss = false; 837 uint8_t policy_mode; 838 uint8_t domain_bitmap; 839 int ret; 840 841 if (!priv->mtr_en || !priv->sh->meter_aso_en) 842 return -rte_mtr_error_set(error, ENOTSUP, 843 RTE_MTR_ERROR_TYPE_METER_POLICY, 844 NULL, "meter policy unsupported."); 845 ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr, 846 &is_rss, &domain_bitmap, &policy_mode, error); 847 if (ret) 848 return ret; 849 return 0; 850 } 851 852 /** 853 * Callback to check MTR policy action validate for HWS 854 * 855 * @param[in] dev 856 * Pointer to Ethernet device. 857 * @param[in] actions 858 * Pointer to meter policy action detail. 859 * @param[out] error 860 * Pointer to the error structure. 861 * 862 * @return 863 * 0 on success, a negative errno value otherwise and rte_errno is set. 864 */ 865 static int 866 mlx5_flow_meter_policy_hws_validate(struct rte_eth_dev *dev, 867 struct rte_mtr_meter_policy_params *policy, 868 struct rte_mtr_error *error) 869 { 870 struct mlx5_priv *priv = dev->data->dev_private; 871 const struct rte_flow_actions_template_attr attr = { 872 .transfer = priv->sh->config.dv_esw_en ? 1 : 0 }; 873 int ret; 874 int i; 875 876 if (!priv->mtr_en || !priv->sh->meter_aso_en) 877 return -rte_mtr_error_set(error, ENOTSUP, 878 RTE_MTR_ERROR_TYPE_METER_POLICY, 879 NULL, "meter policy unsupported."); 880 for (i = 0; i < RTE_COLORS; i++) { 881 ret = mlx5_flow_actions_validate(dev, &attr, policy->actions[i], 882 policy->actions[i], NULL); 883 if (ret) 884 return ret; 885 } 886 return 0; 887 } 888 889 static int 890 __mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev, 891 uint32_t policy_id, 892 struct mlx5_flow_meter_policy *mtr_policy, 893 struct rte_mtr_error *error, 894 bool clear_l3t) 895 { 896 struct mlx5_priv *priv = dev->data->dev_private; 897 struct mlx5_flow_meter_sub_policy *sub_policy; 898 uint32_t i, j; 899 uint16_t sub_policy_num; 900 901 rte_spinlock_lock(&mtr_policy->sl); 902 if (mtr_policy->ref_cnt) { 903 rte_spinlock_unlock(&mtr_policy->sl); 904 return -rte_mtr_error_set(error, EBUSY, 905 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 906 NULL, 907 "Meter policy object is being used."); 908 } 909 mlx5_flow_destroy_policy_rules(dev, mtr_policy); 910 mlx5_flow_destroy_mtr_acts(dev, mtr_policy); 911 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 912 sub_policy_num = (mtr_policy->sub_policy_num >> 913 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) & 914 MLX5_MTR_SUB_POLICY_NUM_MASK; 915 if (sub_policy_num) { 916 for (j = 0; j < sub_policy_num; j++) { 917 sub_policy = mtr_policy->sub_policys[i][j]; 918 if (sub_policy) 919 mlx5_ipool_free 920 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 921 sub_policy->idx); 922 } 923 } 924 } 925 if (priv->policy_idx_tbl && clear_l3t) { 926 if (mlx5_l3t_clear_entry(priv->policy_idx_tbl, policy_id)) { 927 rte_spinlock_unlock(&mtr_policy->sl); 928 return -rte_mtr_error_set(error, ENOTSUP, 929 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL, 930 "Fail to delete policy in index table."); 931 } 932 } 933 rte_spinlock_unlock(&mtr_policy->sl); 934 return 0; 935 } 936 937 /** 938 * Callback to add MTR policy. 939 * 940 * @param[in] dev 941 * Pointer to Ethernet device. 942 * @param[out] policy_id 943 * Pointer to policy id 944 * @param[in] actions 945 * Pointer to meter policy action detail. 946 * @param[out] error 947 * Pointer to the error structure. 948 * 949 * @return 950 * 0 on success, a negative errno value otherwise and rte_errno is set. 951 */ 952 static int 953 mlx5_flow_meter_policy_add(struct rte_eth_dev *dev, 954 uint32_t policy_id, 955 struct rte_mtr_meter_policy_params *policy, 956 struct rte_mtr_error *error) 957 { 958 struct mlx5_priv *priv = dev->data->dev_private; 959 struct rte_flow_attr attr = { .transfer = priv->sh->config.dv_esw_en ? 960 1 : 0 }; 961 uint32_t sub_policy_idx = 0; 962 uint32_t policy_idx = 0; 963 struct mlx5_flow_meter_policy *mtr_policy = NULL; 964 struct mlx5_flow_meter_sub_policy *sub_policy; 965 bool is_rss = false; 966 uint8_t policy_mode; 967 uint32_t i; 968 int ret; 969 uint32_t policy_size = sizeof(struct mlx5_flow_meter_policy); 970 uint16_t sub_policy_num; 971 uint8_t domain_bitmap = 0; 972 union mlx5_l3t_data data; 973 bool skip_rule = false; 974 975 if (!priv->mtr_en) 976 return -rte_mtr_error_set(error, ENOTSUP, 977 RTE_MTR_ERROR_TYPE_METER_POLICY, 978 NULL, "meter policy unsupported. "); 979 if (policy_id == MLX5_INVALID_POLICY_ID) 980 return -rte_mtr_error_set(error, ENOTSUP, 981 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 982 NULL, "policy ID is invalid. "); 983 if (policy_id == priv->sh->mtrmng->def_policy_id) 984 return -rte_mtr_error_set(error, EEXIST, 985 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 986 NULL, "default policy ID exists. "); 987 mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx); 988 if (mtr_policy) 989 return -rte_mtr_error_set(error, EEXIST, 990 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 991 NULL, "policy ID exists. "); 992 ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr, 993 &is_rss, &domain_bitmap, 994 &policy_mode, error); 995 if (ret) 996 return ret; 997 if (!domain_bitmap) 998 return -rte_mtr_error_set(error, ENOTSUP, 999 RTE_MTR_ERROR_TYPE_METER_POLICY, 1000 NULL, "fail to find policy domain."); 1001 if (policy_mode == MLX5_MTR_POLICY_MODE_DEF) { 1002 if (priv->sh->mtrmng->def_policy_id != MLX5_INVALID_POLICY_ID) 1003 return -rte_mtr_error_set(error, EEXIST, 1004 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1005 NULL, "a policy with similar actions " 1006 "is already configured"); 1007 if (mlx5_flow_create_def_policy(dev)) 1008 return -rte_mtr_error_set(error, ENOTSUP, 1009 RTE_MTR_ERROR_TYPE_METER_POLICY, 1010 NULL, 1011 "fail to create non-terminated policy."); 1012 priv->sh->mtrmng->def_policy_id = policy_id; 1013 return 0; 1014 } 1015 if (!priv->sh->meter_aso_en) 1016 return -rte_mtr_error_set(error, ENOTSUP, 1017 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL, 1018 "no ASO capability to support the policy "); 1019 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 1020 if (!(domain_bitmap & (1 << i))) 1021 continue; 1022 /* 1023 * If RSS is found, it means that only the ingress domain can 1024 * be supported. It is invalid to support RSS for one color 1025 * and egress / transfer domain actions for another. Drop and 1026 * jump action should have no impact. 1027 */ 1028 if (is_rss) { 1029 policy_size += 1030 sizeof(struct mlx5_flow_meter_sub_policy *) * 1031 MLX5_MTR_RSS_MAX_SUB_POLICY; 1032 break; 1033 } 1034 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *); 1035 } 1036 mtr_policy = mlx5_malloc(MLX5_MEM_ZERO, policy_size, 1037 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 1038 if (!mtr_policy) 1039 return -rte_mtr_error_set(error, ENOMEM, 1040 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL, 1041 "Memory alloc failed for meter policy."); 1042 if (policy_mode == MLX5_MTR_POLICY_MODE_OG) 1043 mtr_policy->skip_y = 1; 1044 else if (policy_mode == MLX5_MTR_POLICY_MODE_OY) 1045 mtr_policy->skip_g = 1; 1046 policy_size = sizeof(struct mlx5_flow_meter_policy); 1047 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 1048 if (!(domain_bitmap & (1 << i))) 1049 continue; 1050 if (i == MLX5_MTR_DOMAIN_INGRESS) 1051 mtr_policy->ingress = 1; 1052 if (i == MLX5_MTR_DOMAIN_EGRESS) 1053 mtr_policy->egress = 1; 1054 if (i == MLX5_MTR_DOMAIN_TRANSFER) 1055 mtr_policy->transfer = 1; 1056 sub_policy = mlx5_ipool_zmalloc 1057 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 1058 &sub_policy_idx); 1059 if (!sub_policy || 1060 sub_policy_idx > MLX5_MAX_SUB_POLICY_TBL_NUM) 1061 goto policy_add_err; 1062 sub_policy->idx = sub_policy_idx; 1063 sub_policy->main_policy = mtr_policy; 1064 if (!policy_idx) { 1065 policy_idx = sub_policy_idx; 1066 sub_policy->main_policy_id = 1; 1067 } 1068 mtr_policy->sub_policys[i] = 1069 (struct mlx5_flow_meter_sub_policy **) 1070 ((uint8_t *)mtr_policy + policy_size); 1071 mtr_policy->sub_policys[i][0] = sub_policy; 1072 sub_policy_num = (mtr_policy->sub_policy_num >> 1073 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) & 1074 MLX5_MTR_SUB_POLICY_NUM_MASK; 1075 sub_policy_num++; 1076 mtr_policy->sub_policy_num &= ~(MLX5_MTR_SUB_POLICY_NUM_MASK << 1077 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)); 1078 mtr_policy->sub_policy_num |= 1079 (sub_policy_num & MLX5_MTR_SUB_POLICY_NUM_MASK) << 1080 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i); 1081 /* 1082 * If RSS is found, it means that only the ingress domain can 1083 * be supported. It is invalid to support RSS for one color 1084 * and egress / transfer domain actions for another. Drop and 1085 * jump action should have no impact. 1086 */ 1087 if (is_rss) { 1088 mtr_policy->is_rss = 1; 1089 break; 1090 } 1091 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *); 1092 } 1093 rte_spinlock_init(&mtr_policy->sl); 1094 ret = mlx5_flow_create_mtr_acts(dev, mtr_policy, 1095 policy->actions, &attr, error); 1096 if (ret) 1097 goto policy_add_err; 1098 if (mtr_policy->is_hierarchy) { 1099 struct mlx5_flow_meter_policy *final_policy; 1100 1101 final_policy = 1102 mlx5_flow_meter_hierarchy_get_final_policy(dev, mtr_policy); 1103 if (!final_policy) 1104 goto policy_add_err; 1105 skip_rule = (final_policy->is_rss || final_policy->is_queue); 1106 } 1107 /* 1108 * If either Green or Yellow has queue / RSS action, all the policy 1109 * rules will be created later in the flow splitting stage. 1110 */ 1111 if (!is_rss && !mtr_policy->is_queue && !skip_rule) { 1112 /* Create policy rules in HW. */ 1113 ret = mlx5_flow_create_policy_rules(dev, mtr_policy); 1114 if (ret) 1115 goto policy_add_err; 1116 } 1117 data.dword = policy_idx; 1118 if (!priv->policy_idx_tbl) { 1119 priv->policy_idx_tbl = mlx5_l3t_create(MLX5_L3T_TYPE_DWORD); 1120 if (!priv->policy_idx_tbl) 1121 goto policy_add_err; 1122 } 1123 if (mlx5_l3t_set_entry(priv->policy_idx_tbl, policy_id, &data)) 1124 goto policy_add_err; 1125 return 0; 1126 policy_add_err: 1127 if (mtr_policy) { 1128 ret = __mlx5_flow_meter_policy_delete(dev, policy_id, 1129 mtr_policy, error, false); 1130 mlx5_free(mtr_policy); 1131 if (ret) 1132 return ret; 1133 } 1134 return -rte_mtr_error_set(error, ENOTSUP, 1135 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 1136 NULL, "Failed to create devx policy."); 1137 } 1138 1139 /** 1140 * Callback to delete MTR policy. 1141 * 1142 * @param[in] dev 1143 * Pointer to Ethernet device. 1144 * @param[in] policy_id 1145 * Meter policy id. 1146 * @param[out] error 1147 * Pointer to the error structure. 1148 * 1149 * @return 1150 * 0 on success, a negative errno value otherwise and rte_errno is set. 1151 */ 1152 static int 1153 mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev, 1154 uint32_t policy_id, 1155 struct rte_mtr_error *error) 1156 { 1157 struct mlx5_priv *priv = dev->data->dev_private; 1158 struct mlx5_flow_meter_policy *mtr_policy; 1159 uint32_t policy_idx; 1160 int ret; 1161 1162 if (policy_id == priv->sh->mtrmng->def_policy_id) { 1163 if (priv->sh->mtrmng->def_policy_ref_cnt > 0) 1164 return -rte_mtr_error_set(error, ENOTSUP, 1165 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL, 1166 "Meter policy object is being used."); 1167 priv->sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID; 1168 return 0; 1169 } 1170 mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx); 1171 if (!mtr_policy) 1172 return -rte_mtr_error_set(error, ENOTSUP, 1173 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL, 1174 "Meter policy id is invalid. "); 1175 ret = __mlx5_flow_meter_policy_delete(dev, policy_id, mtr_policy, 1176 error, true); 1177 if (ret) 1178 return ret; 1179 mlx5_free(mtr_policy); 1180 return 0; 1181 } 1182 1183 /** 1184 * Callback to get MTR policy. 1185 * 1186 * @param[in] dev 1187 * Pointer to Ethernet device. 1188 * @param[in] policy_id 1189 * Meter policy id. 1190 * @param[out] error 1191 * Pointer to the error structure. 1192 * 1193 * @return 1194 * A valid handle in case of success, NULL otherwise. 1195 */ 1196 static struct rte_flow_meter_policy * 1197 mlx5_flow_meter_policy_get(struct rte_eth_dev *dev, 1198 uint32_t policy_id, 1199 struct rte_mtr_error *error) 1200 { 1201 struct mlx5_priv *priv = dev->data->dev_private; 1202 uint32_t policy_idx; 1203 1204 if (!priv->mtr_en) { 1205 rte_mtr_error_set(error, ENOTSUP, 1206 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1207 "Meter is not supported"); 1208 return NULL; 1209 } 1210 return (void *)(uintptr_t)mlx5_flow_meter_policy_find(dev, policy_id, 1211 &policy_idx); 1212 } 1213 1214 /** 1215 * Callback to delete MTR policy for HWS. 1216 * 1217 * @param[in] dev 1218 * Pointer to Ethernet device. 1219 * @param[in] policy_id 1220 * Meter policy id. 1221 * @param[out] error 1222 * Pointer to the error structure. 1223 * 1224 * @return 1225 * 0 on success, a negative errno value otherwise and rte_errno is set. 1226 */ 1227 static int 1228 mlx5_flow_meter_policy_hws_delete(struct rte_eth_dev *dev, 1229 uint32_t policy_id, 1230 struct rte_mtr_error *error) 1231 { 1232 struct mlx5_priv *priv = dev->data->dev_private; 1233 struct mlx5_flow_meter_policy *mtr_policy; 1234 uint32_t i, j; 1235 uint32_t nb_flows = 0; 1236 int ret; 1237 struct rte_flow_op_attr op_attr = { .postpone = 1 }; 1238 struct rte_flow_op_result result[RTE_COLORS * MLX5_MTR_DOMAIN_MAX]; 1239 1240 if (!priv->mtr_policy_arr) 1241 return -rte_mtr_error_set(error, ENOTSUP, 1242 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 1243 NULL, "Meter policy array is not allocated"); 1244 /* Meter id must be valid. */ 1245 if (policy_id >= priv->mtr_config.nb_meter_policies) 1246 return -rte_mtr_error_set(error, EINVAL, 1247 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1248 &policy_id, 1249 "Meter policy id not valid."); 1250 /* Meter policy must exist. */ 1251 mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, NULL); 1252 if (!mtr_policy->initialized) 1253 return -rte_mtr_error_set(error, ENOENT, 1254 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL, 1255 "Meter policy does not exists."); 1256 /* Check policy is unused. */ 1257 if (mtr_policy->ref_cnt) 1258 return -rte_mtr_error_set(error, EBUSY, 1259 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1260 NULL, "Meter policy is in use."); 1261 rte_spinlock_lock(&priv->hw_ctrl_lock); 1262 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 1263 for (j = 0; j < RTE_COLORS; j++) { 1264 if (mtr_policy->hws_flow_rule[i][j]) { 1265 ret = rte_flow_async_destroy(dev->data->port_id, 1266 CTRL_QUEUE_ID(priv), &op_attr, 1267 mtr_policy->hws_flow_rule[i][j], 1268 NULL, NULL); 1269 if (ret < 0) 1270 continue; 1271 nb_flows++; 1272 } 1273 } 1274 } 1275 ret = rte_flow_push(dev->data->port_id, CTRL_QUEUE_ID(priv), NULL); 1276 while (nb_flows && (ret >= 0)) { 1277 ret = rte_flow_pull(dev->data->port_id, 1278 CTRL_QUEUE_ID(priv), result, 1279 nb_flows, NULL); 1280 nb_flows -= ret; 1281 } 1282 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 1283 if (mtr_policy->hws_flow_table[i]) 1284 rte_flow_template_table_destroy(dev->data->port_id, 1285 mtr_policy->hws_flow_table[i], NULL); 1286 } 1287 for (i = 0; i < RTE_COLORS; i++) { 1288 if (mtr_policy->hws_act_templ[i]) 1289 rte_flow_actions_template_destroy(dev->data->port_id, 1290 mtr_policy->hws_act_templ[i], NULL); 1291 } 1292 if (mtr_policy->hws_item_templ) 1293 rte_flow_pattern_template_destroy(dev->data->port_id, 1294 mtr_policy->hws_item_templ, NULL); 1295 rte_spinlock_unlock(&priv->hw_ctrl_lock); 1296 memset(mtr_policy, 0, sizeof(struct mlx5_flow_meter_policy)); 1297 return 0; 1298 } 1299 1300 /** 1301 * Callback to add MTR policy for HWS. 1302 * 1303 * @param[in] dev 1304 * Pointer to Ethernet device. 1305 * @param[out] policy_id 1306 * Pointer to policy id 1307 * @param[in] actions 1308 * Pointer to meter policy action detail. 1309 * @param[out] error 1310 * Pointer to the error structure. 1311 * 1312 * @return 1313 * 0 on success, a negative errno value otherwise and rte_errno is set. 1314 */ 1315 static int 1316 mlx5_flow_meter_policy_hws_add(struct rte_eth_dev *dev, 1317 uint32_t policy_id, 1318 struct rte_mtr_meter_policy_params *policy, 1319 struct rte_mtr_error *error) 1320 { 1321 struct mlx5_priv *priv = dev->data->dev_private; 1322 struct mlx5_flow_meter_policy *mtr_policy = NULL; 1323 const struct rte_flow_action *act; 1324 const struct rte_flow_action_meter *mtr; 1325 struct mlx5_flow_meter_info *fm; 1326 struct mlx5_flow_meter_policy *plc; 1327 uint8_t domain_color = MLX5_MTR_ALL_DOMAIN_BIT; 1328 bool is_rss = false; 1329 bool is_hierarchy = false; 1330 int i, j; 1331 uint32_t nb_colors = 0; 1332 uint32_t nb_flows = 0; 1333 int color; 1334 int ret; 1335 struct rte_flow_pattern_template_attr pta = {0}; 1336 struct rte_flow_actions_template_attr ata = {0}; 1337 struct rte_flow_template_table_attr ta = { {0}, 0 }; 1338 struct rte_flow_op_attr op_attr = { .postpone = 1 }; 1339 struct rte_flow_op_result result[RTE_COLORS * MLX5_MTR_DOMAIN_MAX]; 1340 const uint32_t color_mask = (UINT32_C(1) << MLX5_MTR_COLOR_BITS) - 1; 1341 int color_reg_c_idx = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 1342 0, NULL); 1343 struct rte_flow_item_tag tag_spec = { 1344 .data = 0, 1345 .index = color_reg_c_idx 1346 }; 1347 struct rte_flow_item_tag tag_mask = { 1348 .data = color_mask, 1349 .index = 0xff}; 1350 struct rte_flow_item pattern[] = { 1351 [0] = { 1352 .type = (enum rte_flow_item_type) 1353 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 1354 .spec = &tag_spec, 1355 .mask = &tag_mask, 1356 }, 1357 [1] = { .type = RTE_FLOW_ITEM_TYPE_END } 1358 }; 1359 1360 if (!priv->mtr_policy_arr) 1361 return -rte_mtr_error_set(error, ENOTSUP, 1362 RTE_MTR_ERROR_TYPE_METER_POLICY, 1363 NULL, "Meter policy array is not allocated."); 1364 if (policy_id >= priv->mtr_config.nb_meter_policies) 1365 return -rte_mtr_error_set(error, ENOTSUP, 1366 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1367 NULL, "Meter policy id not valid."); 1368 mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, NULL); 1369 if (mtr_policy->initialized) 1370 return -rte_mtr_error_set(error, EEXIST, 1371 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1372 NULL, "Meter policy already exists."); 1373 if (!policy || 1374 (!policy->actions[RTE_COLOR_RED] && 1375 !policy->actions[RTE_COLOR_YELLOW] && 1376 !policy->actions[RTE_COLOR_GREEN])) 1377 return -rte_mtr_error_set(error, EINVAL, 1378 RTE_MTR_ERROR_TYPE_METER_POLICY, 1379 NULL, "Meter policy actions are not valid."); 1380 if (policy->actions[RTE_COLOR_RED] == RTE_FLOW_ACTION_TYPE_END) 1381 mtr_policy->skip_r = 1; 1382 if (policy->actions[RTE_COLOR_YELLOW] == RTE_FLOW_ACTION_TYPE_END) 1383 mtr_policy->skip_y = 1; 1384 if (policy->actions[RTE_COLOR_GREEN] == RTE_FLOW_ACTION_TYPE_END) 1385 mtr_policy->skip_g = 1; 1386 if (mtr_policy->skip_r && mtr_policy->skip_y && mtr_policy->skip_g) 1387 return -rte_mtr_error_set(error, ENOTSUP, 1388 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1389 NULL, "Meter policy actions are empty."); 1390 for (i = 0; i < RTE_COLORS; i++) { 1391 act = policy->actions[i]; 1392 while (act && act->type != RTE_FLOW_ACTION_TYPE_END) { 1393 switch (act->type) { 1394 case RTE_FLOW_ACTION_TYPE_PORT_ID: 1395 /* fall-through. */ 1396 case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT: 1397 domain_color &= ~(MLX5_MTR_DOMAIN_INGRESS_BIT | 1398 MLX5_MTR_DOMAIN_EGRESS_BIT); 1399 break; 1400 case RTE_FLOW_ACTION_TYPE_RSS: 1401 is_rss = true; 1402 /* fall-through. */ 1403 case RTE_FLOW_ACTION_TYPE_QUEUE: 1404 domain_color &= ~(MLX5_MTR_DOMAIN_EGRESS_BIT | 1405 MLX5_MTR_DOMAIN_TRANSFER_BIT); 1406 break; 1407 case RTE_FLOW_ACTION_TYPE_METER: 1408 is_hierarchy = true; 1409 mtr = act->conf; 1410 fm = mlx5_flow_meter_find(priv, 1411 mtr->mtr_id, NULL); 1412 if (!fm) 1413 return -rte_mtr_error_set(error, EINVAL, 1414 RTE_MTR_ERROR_TYPE_MTR_ID, NULL, 1415 "Meter not found in meter hierarchy."); 1416 plc = mlx5_flow_meter_policy_find(dev, 1417 fm->policy_id, 1418 NULL); 1419 MLX5_ASSERT(plc); 1420 domain_color &= MLX5_MTR_ALL_DOMAIN_BIT & 1421 (plc->ingress << 1422 MLX5_MTR_DOMAIN_INGRESS); 1423 domain_color &= MLX5_MTR_ALL_DOMAIN_BIT & 1424 (plc->egress << 1425 MLX5_MTR_DOMAIN_EGRESS); 1426 domain_color &= MLX5_MTR_ALL_DOMAIN_BIT & 1427 (plc->transfer << 1428 MLX5_MTR_DOMAIN_TRANSFER); 1429 break; 1430 default: 1431 break; 1432 } 1433 act++; 1434 } 1435 } 1436 if (priv->sh->config.dv_esw_en) 1437 domain_color &= ~(MLX5_MTR_DOMAIN_EGRESS_BIT | 1438 MLX5_MTR_DOMAIN_TRANSFER_BIT); 1439 else 1440 domain_color &= ~MLX5_MTR_DOMAIN_TRANSFER_BIT; 1441 if (!domain_color) 1442 return -rte_mtr_error_set(error, ENOTSUP, 1443 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1444 NULL, "Meter policy domains are conflicting."); 1445 mtr_policy->is_rss = is_rss; 1446 mtr_policy->ingress = !!(domain_color & MLX5_MTR_DOMAIN_INGRESS_BIT); 1447 pta.ingress = mtr_policy->ingress; 1448 mtr_policy->egress = !!(domain_color & MLX5_MTR_DOMAIN_EGRESS_BIT); 1449 pta.egress = mtr_policy->egress; 1450 mtr_policy->transfer = !!(domain_color & MLX5_MTR_DOMAIN_TRANSFER_BIT); 1451 pta.transfer = mtr_policy->transfer; 1452 mtr_policy->group = MLX5_FLOW_TABLE_HWS_POLICY - policy_id; 1453 mtr_policy->is_hierarchy = is_hierarchy; 1454 mtr_policy->initialized = 1; 1455 rte_spinlock_lock(&priv->hw_ctrl_lock); 1456 mtr_policy->hws_item_templ = 1457 rte_flow_pattern_template_create(dev->data->port_id, 1458 &pta, pattern, NULL); 1459 if (!mtr_policy->hws_item_templ) 1460 goto policy_add_err; 1461 for (i = 0; i < RTE_COLORS; i++) { 1462 if (mtr_policy->skip_g && i == RTE_COLOR_GREEN) 1463 continue; 1464 if (mtr_policy->skip_y && i == RTE_COLOR_YELLOW) 1465 continue; 1466 if (mtr_policy->skip_r && i == RTE_COLOR_RED) 1467 continue; 1468 mtr_policy->hws_act_templ[nb_colors] = 1469 rte_flow_actions_template_create(dev->data->port_id, 1470 &ata, policy->actions[i], 1471 policy->actions[i], NULL); 1472 if (!mtr_policy->hws_act_templ[nb_colors]) 1473 goto policy_add_err; 1474 nb_colors++; 1475 } 1476 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 1477 memset(&ta, 0, sizeof(ta)); 1478 ta.nb_flows = RTE_COLORS; 1479 ta.flow_attr.group = mtr_policy->group; 1480 if (i == MLX5_MTR_DOMAIN_INGRESS) { 1481 if (!mtr_policy->ingress) 1482 continue; 1483 ta.flow_attr.ingress = 1; 1484 } else if (i == MLX5_MTR_DOMAIN_EGRESS) { 1485 if (!mtr_policy->egress) 1486 continue; 1487 ta.flow_attr.egress = 1; 1488 } else if (i == MLX5_MTR_DOMAIN_TRANSFER) { 1489 if (!mtr_policy->transfer) 1490 continue; 1491 ta.flow_attr.transfer = 1; 1492 } 1493 mtr_policy->hws_flow_table[i] = 1494 rte_flow_template_table_create(dev->data->port_id, 1495 &ta, &mtr_policy->hws_item_templ, 1, 1496 mtr_policy->hws_act_templ, nb_colors, 1497 NULL); 1498 if (!mtr_policy->hws_flow_table[i]) 1499 goto policy_add_err; 1500 nb_colors = 0; 1501 for (j = 0; j < RTE_COLORS; j++) { 1502 if (mtr_policy->skip_g && j == RTE_COLOR_GREEN) 1503 continue; 1504 if (mtr_policy->skip_y && j == RTE_COLOR_YELLOW) 1505 continue; 1506 if (mtr_policy->skip_r && j == RTE_COLOR_RED) 1507 continue; 1508 color = rte_col_2_mlx5_col((enum rte_color)j); 1509 tag_spec.data = color; 1510 mtr_policy->hws_flow_rule[i][j] = 1511 rte_flow_async_create(dev->data->port_id, 1512 CTRL_QUEUE_ID(priv), &op_attr, 1513 mtr_policy->hws_flow_table[i], 1514 pattern, 0, policy->actions[j], 1515 nb_colors, NULL, NULL); 1516 if (!mtr_policy->hws_flow_rule[i][j]) 1517 goto policy_add_err; 1518 nb_colors++; 1519 nb_flows++; 1520 } 1521 ret = rte_flow_push(dev->data->port_id, 1522 CTRL_QUEUE_ID(priv), NULL); 1523 if (ret < 0) 1524 goto policy_add_err; 1525 while (nb_flows) { 1526 ret = rte_flow_pull(dev->data->port_id, 1527 CTRL_QUEUE_ID(priv), result, 1528 nb_flows, NULL); 1529 if (ret < 0) 1530 goto policy_add_err; 1531 for (j = 0; j < ret; j++) { 1532 if (result[j].status == RTE_FLOW_OP_ERROR) 1533 goto policy_add_err; 1534 } 1535 nb_flows -= ret; 1536 } 1537 } 1538 rte_spinlock_unlock(&priv->hw_ctrl_lock); 1539 return 0; 1540 policy_add_err: 1541 rte_spinlock_unlock(&priv->hw_ctrl_lock); 1542 ret = mlx5_flow_meter_policy_hws_delete(dev, policy_id, error); 1543 memset(mtr_policy, 0, sizeof(struct mlx5_flow_meter_policy)); 1544 if (ret) 1545 return ret; 1546 return -rte_mtr_error_set(error, ENOTSUP, 1547 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 1548 NULL, "Failed to create meter policy."); 1549 } 1550 1551 /** 1552 * Check meter validation. 1553 * 1554 * @param[in] priv 1555 * Pointer to mlx5 private data structure. 1556 * @param[in] meter_id 1557 * Meter id. 1558 * @param[in] params 1559 * Pointer to rte meter parameters. 1560 * @param[out] error 1561 * Pointer to rte meter error structure. 1562 * 1563 * @return 1564 * 0 on success, a negative errno value otherwise and rte_errno is set. 1565 */ 1566 static int 1567 mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id, 1568 struct rte_mtr_params *params, 1569 struct rte_mtr_error *error) 1570 { 1571 /* Meter must use global drop action. */ 1572 if (!priv->sh->dr_drop_action) 1573 return -rte_mtr_error_set(error, ENOTSUP, 1574 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 1575 NULL, 1576 "No drop action ready for meter."); 1577 /* Meter params must not be NULL. */ 1578 if (params == NULL) 1579 return -rte_mtr_error_set(error, EINVAL, 1580 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 1581 NULL, "Meter object params null."); 1582 /* Previous meter color is not supported. */ 1583 if (params->use_prev_mtr_color && !priv->sh->meter_aso_en) 1584 return -rte_mtr_error_set(error, ENOTSUP, 1585 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 1586 NULL, 1587 "Previous meter color " 1588 "not supported."); 1589 if (params->meter_policy_id == MLX5_INVALID_POLICY_ID) 1590 return -rte_mtr_error_set(error, ENOENT, 1591 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1592 NULL, "Meter policy id not valid."); 1593 /* Validate meter id. */ 1594 if (mlx5_flow_meter_find(priv, meter_id, NULL)) 1595 return -rte_mtr_error_set(error, EEXIST, 1596 RTE_MTR_ERROR_TYPE_MTR_ID, NULL, 1597 "Meter object already exists."); 1598 return 0; 1599 } 1600 1601 /** 1602 * Modify the flow meter action. 1603 * 1604 * @param[in] priv 1605 * Pointer to mlx5 private data structure. 1606 * @param[in] fm 1607 * Pointer to flow meter to be modified. 1608 * @param[in] srtcm 1609 * Pointer to meter srtcm description parameter. 1610 * @param[in] modify_bits 1611 * The bit in srtcm to be updated. 1612 * @param[in] active_state 1613 * The state to be updated. 1614 * @return 1615 * 0 on success, o negative value otherwise. 1616 */ 1617 static int 1618 mlx5_flow_meter_action_modify(struct mlx5_priv *priv, 1619 struct mlx5_flow_meter_info *fm, 1620 const struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm, 1621 uint64_t modify_bits, uint32_t active_state, uint32_t is_enable) 1622 { 1623 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER 1624 uint32_t in[MLX5_ST_SZ_DW(flow_meter_parameters)] = { 0 }; 1625 uint32_t *attr; 1626 struct mlx5dv_dr_flow_meter_attr mod_attr = { 0 }; 1627 int ret; 1628 struct mlx5_aso_mtr *aso_mtr = NULL; 1629 uint32_t cbs_cir, ebs_eir, val; 1630 1631 if (priv->sh->meter_aso_en) { 1632 fm->is_enable = !!is_enable; 1633 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm); 1634 ret = mlx5_aso_meter_update_by_wqe(priv->sh, MLX5_HW_INV_QUEUE, 1635 aso_mtr, &priv->mtr_bulk, NULL, true); 1636 if (ret) 1637 return ret; 1638 ret = mlx5_aso_mtr_wait(priv->sh, MLX5_HW_INV_QUEUE, aso_mtr); 1639 if (ret) 1640 return ret; 1641 } else { 1642 /* Fill command parameters. */ 1643 mod_attr.reg_c_index = priv->mtr_color_reg - REG_C_0; 1644 mod_attr.flow_meter_parameter = in; 1645 mod_attr.flow_meter_parameter_sz = 1646 MLX5_ST_SZ_BYTES(flow_meter_parameters); 1647 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE) 1648 mod_attr.active = !!active_state; 1649 else 1650 mod_attr.active = 0; 1651 attr = in; 1652 cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir); 1653 ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir); 1654 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS) { 1655 val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & 1656 ASO_DSEG_EXP_MASK; 1657 MLX5_SET(flow_meter_parameters, attr, 1658 cbs_exponent, val); 1659 val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & 1660 ASO_DSEG_MAN_MASK; 1661 MLX5_SET(flow_meter_parameters, attr, 1662 cbs_mantissa, val); 1663 } 1664 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR) { 1665 val = (cbs_cir >> ASO_DSEG_XIR_EXP_OFFSET) & 1666 ASO_DSEG_EXP_MASK; 1667 MLX5_SET(flow_meter_parameters, attr, 1668 cir_exponent, val); 1669 val = cbs_cir & ASO_DSEG_MAN_MASK; 1670 MLX5_SET(flow_meter_parameters, attr, 1671 cir_mantissa, val); 1672 } 1673 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_EBS) { 1674 val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & 1675 ASO_DSEG_EXP_MASK; 1676 MLX5_SET(flow_meter_parameters, attr, 1677 ebs_exponent, val); 1678 val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & 1679 ASO_DSEG_MAN_MASK; 1680 MLX5_SET(flow_meter_parameters, attr, 1681 ebs_mantissa, val); 1682 } 1683 /* Apply modifications to meter only if it was created. */ 1684 if (fm->meter_action_g) { 1685 ret = mlx5_glue->dv_modify_flow_action_meter 1686 (fm->meter_action_g, &mod_attr, 1687 rte_cpu_to_be_64(modify_bits)); 1688 if (ret) 1689 return ret; 1690 } 1691 /* Update succeeded modify meter parameters. */ 1692 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE) 1693 fm->active_state = !!active_state; 1694 } 1695 return 0; 1696 #else 1697 (void)priv; 1698 (void)fm; 1699 (void)srtcm; 1700 (void)modify_bits; 1701 (void)active_state; 1702 (void)is_enable; 1703 return -ENOTSUP; 1704 #endif 1705 } 1706 1707 static int 1708 mlx5_flow_meter_stats_enable_update(struct rte_eth_dev *dev, 1709 struct mlx5_flow_meter_info *fm, 1710 uint64_t stats_mask) 1711 { 1712 fm->bytes_dropped = 1713 (stats_mask & RTE_MTR_STATS_N_BYTES_DROPPED) ? 1 : 0; 1714 fm->pkts_dropped = (stats_mask & RTE_MTR_STATS_N_PKTS_DROPPED) ? 1 : 0; 1715 if (fm->bytes_dropped || fm->pkts_dropped) { 1716 if (!fm->drop_cnt) { 1717 /* Alloc policer counters. */ 1718 fm->drop_cnt = mlx5_counter_alloc(dev); 1719 if (!fm->drop_cnt) 1720 return -1; 1721 } 1722 } else { 1723 if (fm->drop_cnt) { 1724 mlx5_counter_free(dev, fm->drop_cnt); 1725 fm->drop_cnt = 0; 1726 } 1727 } 1728 return 0; 1729 } 1730 1731 /** 1732 * Create meter rules. 1733 * 1734 * @param[in] dev 1735 * Pointer to Ethernet device. 1736 * @param[in] meter_id 1737 * Meter id. 1738 * @param[in] params 1739 * Pointer to rte meter parameters. 1740 * @param[in] shared 1741 * Meter shared with other flow or not. 1742 * @param[out] error 1743 * Pointer to rte meter error structure. 1744 * 1745 * @return 1746 * 0 on success, a negative errno value otherwise and rte_errno is set. 1747 */ 1748 static int 1749 mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, 1750 struct rte_mtr_params *params, int shared, 1751 struct rte_mtr_error *error) 1752 { 1753 struct mlx5_priv *priv = dev->data->dev_private; 1754 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; 1755 struct mlx5_flow_meter_profile *fmp; 1756 struct mlx5_flow_meter_info *fm; 1757 /* GCC fails to infer legacy_fm is set when !priv->sh->meter_aso_en. */ 1758 struct mlx5_legacy_flow_meter *legacy_fm = NULL; 1759 struct mlx5_flow_meter_policy *mtr_policy = NULL; 1760 struct mlx5_indexed_pool_config flow_ipool_cfg = { 1761 .size = 0, 1762 .trunk_size = 64, 1763 .need_lock = 1, 1764 .type = "mlx5_flow_mtr_flow_id_pool", 1765 }; 1766 struct mlx5_aso_mtr *aso_mtr; 1767 uint32_t mtr_idx, policy_idx; 1768 union mlx5_l3t_data data; 1769 int ret; 1770 uint8_t domain_bitmap; 1771 uint8_t mtr_id_bits; 1772 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 1773 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 1774 1775 if (!priv->mtr_en) 1776 return -rte_mtr_error_set(error, ENOTSUP, 1777 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1778 "Meter is not supported"); 1779 /* Validate the parameters. */ 1780 ret = mlx5_flow_meter_validate(priv, meter_id, params, error); 1781 if (ret) 1782 return ret; 1783 /* Meter profile must exist. */ 1784 fmp = mlx5_flow_meter_profile_find(priv, params->meter_profile_id); 1785 if (fmp == NULL) 1786 return -rte_mtr_error_set(error, ENOENT, 1787 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 1788 NULL, "Meter profile id not valid."); 1789 /* Meter policy must exist. */ 1790 if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) { 1791 __atomic_add_fetch 1792 (&priv->sh->mtrmng->def_policy_ref_cnt, 1793 1, __ATOMIC_RELAXED); 1794 domain_bitmap = MLX5_MTR_ALL_DOMAIN_BIT; 1795 if (!priv->sh->config.dv_esw_en) 1796 domain_bitmap &= ~MLX5_MTR_DOMAIN_TRANSFER_BIT; 1797 } else { 1798 if (!priv->sh->meter_aso_en) 1799 return -rte_mtr_error_set(error, ENOTSUP, 1800 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1801 "Part of the policies cannot be " 1802 "supported without ASO "); 1803 mtr_policy = mlx5_flow_meter_policy_find(dev, 1804 params->meter_policy_id, &policy_idx); 1805 if (!mtr_policy) 1806 return -rte_mtr_error_set(error, ENOENT, 1807 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1808 NULL, "Meter policy id not valid."); 1809 domain_bitmap = (mtr_policy->ingress ? 1810 MLX5_MTR_DOMAIN_INGRESS_BIT : 0) | 1811 (mtr_policy->egress ? 1812 MLX5_MTR_DOMAIN_EGRESS_BIT : 0) | 1813 (mtr_policy->transfer ? 1814 MLX5_MTR_DOMAIN_TRANSFER_BIT : 0); 1815 if (fmp->g_support && mtr_policy->skip_g) 1816 return -rte_mtr_error_set(error, ENOTSUP, 1817 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1818 NULL, "Meter green policy is empty."); 1819 if (fmp->y_support && mtr_policy->skip_y) 1820 return -rte_mtr_error_set(error, ENOTSUP, 1821 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1822 NULL, "Meter yellow policy is empty."); 1823 } 1824 /* Allocate the flow meter memory. */ 1825 if (priv->sh->meter_aso_en) { 1826 mtr_idx = mlx5_flow_mtr_alloc(dev); 1827 if (!mtr_idx) 1828 return -rte_mtr_error_set(error, ENOMEM, 1829 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1830 "Memory alloc failed for meter."); 1831 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx); 1832 fm = &aso_mtr->fm; 1833 } else { 1834 if (fmp->y_support) 1835 return -rte_mtr_error_set(error, ENOMEM, 1836 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1837 "Unsupported profile with yellow."); 1838 legacy_fm = mlx5_ipool_zmalloc 1839 (priv->sh->ipool[MLX5_IPOOL_MTR], &mtr_idx); 1840 if (legacy_fm == NULL) 1841 return -rte_mtr_error_set(error, ENOMEM, 1842 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1843 "Memory alloc failed for meter."); 1844 legacy_fm->idx = mtr_idx; 1845 fm = &legacy_fm->fm; 1846 } 1847 mtr_id_bits = MLX5_REG_BITS - __builtin_clz(mtr_idx); 1848 if ((mtr_id_bits + priv->sh->mtrmng->max_mtr_flow_bits) > 1849 mtr_reg_bits) { 1850 DRV_LOG(ERR, "Meter number exceeds max limit."); 1851 goto error; 1852 } 1853 if (mtr_id_bits > priv->sh->mtrmng->max_mtr_bits) 1854 priv->sh->mtrmng->max_mtr_bits = mtr_id_bits; 1855 /* Fill the flow meter parameters. */ 1856 fm->meter_id = meter_id; 1857 fm->policy_id = params->meter_policy_id; 1858 fm->profile = fmp; 1859 if (mlx5_flow_meter_stats_enable_update(dev, fm, params->stats_mask)) 1860 goto error; 1861 if (mlx5_flow_create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap)) 1862 goto error; 1863 /* Add to the flow meter list. */ 1864 if (!priv->sh->meter_aso_en) { 1865 MLX5_ASSERT(legacy_fm != NULL); 1866 TAILQ_INSERT_TAIL(fms, legacy_fm, next); 1867 } 1868 /* Add to the flow meter list. */ 1869 fm->active_state = 1; /* Config meter starts as active. */ 1870 fm->is_enable = params->meter_enable; 1871 fm->shared = !!shared; 1872 fm->color_aware = !!params->use_prev_mtr_color; 1873 __atomic_add_fetch(&fm->profile->ref_cnt, 1, __ATOMIC_RELAXED); 1874 if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) { 1875 fm->def_policy = 1; 1876 fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg); 1877 if (!fm->flow_ipool) 1878 goto error; 1879 } 1880 rte_spinlock_init(&fm->sl); 1881 /* If ASO meter supported, update ASO flow meter by wqe. */ 1882 if (priv->sh->meter_aso_en) { 1883 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm); 1884 ret = mlx5_aso_meter_update_by_wqe(priv->sh, MLX5_HW_INV_QUEUE, 1885 aso_mtr, &priv->mtr_bulk, NULL, true); 1886 if (ret) 1887 goto error; 1888 if (!priv->mtr_idx_tbl) { 1889 priv->mtr_idx_tbl = 1890 mlx5_l3t_create(MLX5_L3T_TYPE_DWORD); 1891 if (!priv->mtr_idx_tbl) 1892 goto error; 1893 } 1894 data.dword = mtr_idx; 1895 if (mlx5_l3t_set_entry(priv->mtr_idx_tbl, meter_id, &data)) 1896 goto error; 1897 } else if (!params->meter_enable && mlx5_flow_meter_disable(dev, meter_id, error)) { 1898 goto error; 1899 } 1900 fm->active_state = params->meter_enable; 1901 if (mtr_policy) 1902 __atomic_add_fetch(&mtr_policy->ref_cnt, 1, __ATOMIC_RELAXED); 1903 return 0; 1904 error: 1905 mlx5_flow_destroy_mtr_tbls(dev, fm); 1906 /* Free policer counters. */ 1907 if (fm->drop_cnt) 1908 mlx5_counter_free(dev, fm->drop_cnt); 1909 if (priv->sh->meter_aso_en) 1910 mlx5_flow_mtr_free(dev, mtr_idx); 1911 else 1912 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], mtr_idx); 1913 return -rte_mtr_error_set(error, ENOTSUP, 1914 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 1915 NULL, "Failed to create devx meter."); 1916 } 1917 1918 /** 1919 * Create meter rules. 1920 * 1921 * @param[in] dev 1922 * Pointer to Ethernet device. 1923 * @param[in] meter_id 1924 * Meter id. 1925 * @param[in] params 1926 * Pointer to rte meter parameters. 1927 * @param[in] shared 1928 * Meter shared with other flow or not. 1929 * @param[out] error 1930 * Pointer to rte meter error structure. 1931 * 1932 * @return 1933 * 0 on success, a negative errno value otherwise and rte_errno is set. 1934 */ 1935 static int 1936 mlx5_flow_meter_hws_create(struct rte_eth_dev *dev, uint32_t meter_id, 1937 struct rte_mtr_params *params, int shared, 1938 struct rte_mtr_error *error) 1939 { 1940 struct mlx5_priv *priv = dev->data->dev_private; 1941 struct mlx5_flow_meter_profile *profile; 1942 struct mlx5_flow_meter_info *fm; 1943 struct mlx5_flow_meter_policy *policy = NULL; 1944 struct mlx5_aso_mtr *aso_mtr; 1945 int ret; 1946 1947 if (!priv->mtr_profile_arr || 1948 !priv->mtr_policy_arr || 1949 !priv->mtr_bulk.aso) 1950 return -rte_mtr_error_set(error, ENOTSUP, 1951 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1952 "Meter bulk array is not allocated."); 1953 /* Meter profile must exist. */ 1954 profile = mlx5_flow_meter_profile_find(priv, params->meter_profile_id); 1955 if (!profile->initialized) 1956 return -rte_mtr_error_set(error, ENOENT, 1957 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 1958 NULL, "Meter profile id not valid."); 1959 /* Meter policy must exist. */ 1960 policy = mlx5_flow_meter_policy_find(dev, 1961 params->meter_policy_id, NULL); 1962 if (!policy->initialized) 1963 return -rte_mtr_error_set(error, ENOENT, 1964 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1965 NULL, "Meter policy id not valid."); 1966 /* Meter ID must be valid. */ 1967 if (meter_id >= priv->mtr_config.nb_meters) 1968 return -rte_mtr_error_set(error, EINVAL, 1969 RTE_MTR_ERROR_TYPE_MTR_ID, 1970 NULL, "Meter id not valid."); 1971 /* Find ASO object. */ 1972 aso_mtr = mlx5_aso_meter_by_idx(priv, meter_id); 1973 fm = &aso_mtr->fm; 1974 if (fm->initialized) 1975 return -rte_mtr_error_set(error, ENOENT, 1976 RTE_MTR_ERROR_TYPE_MTR_ID, 1977 NULL, "Meter object already exists."); 1978 /* Fill the flow meter parameters. */ 1979 fm->meter_id = meter_id; 1980 fm->policy_id = params->meter_policy_id; 1981 fm->profile = profile; 1982 fm->meter_offset = meter_id; 1983 fm->group = policy->group; 1984 /* Add to the flow meter list. */ 1985 fm->active_state = 1; /* Config meter starts as active. */ 1986 fm->is_enable = params->meter_enable; 1987 fm->shared = !!shared; 1988 fm->initialized = 1; 1989 /* Update ASO flow meter by wqe. */ 1990 ret = mlx5_aso_meter_update_by_wqe(priv->sh, MLX5_HW_INV_QUEUE, aso_mtr, 1991 &priv->mtr_bulk, NULL, true); 1992 if (ret) 1993 return -rte_mtr_error_set(error, ENOTSUP, 1994 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 1995 NULL, "Failed to create devx meter."); 1996 fm->active_state = params->meter_enable; 1997 __atomic_add_fetch(&fm->profile->ref_cnt, 1, __ATOMIC_RELAXED); 1998 __atomic_add_fetch(&policy->ref_cnt, 1, __ATOMIC_RELAXED); 1999 return 0; 2000 } 2001 2002 static int 2003 mlx5_flow_meter_params_flush(struct rte_eth_dev *dev, 2004 struct mlx5_flow_meter_info *fm, 2005 uint32_t mtr_idx) 2006 { 2007 struct mlx5_priv *priv = dev->data->dev_private; 2008 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; 2009 struct mlx5_flow_meter_profile *fmp; 2010 struct mlx5_legacy_flow_meter *legacy_fm = NULL; 2011 struct mlx5_flow_meter_policy *mtr_policy; 2012 2013 /* Meter object must not have any owner. */ 2014 MLX5_ASSERT(!fm->ref_cnt); 2015 /* Get meter profile. */ 2016 fmp = fm->profile; 2017 if (fmp == NULL) 2018 return -1; 2019 /* Update dependencies. */ 2020 __atomic_sub_fetch(&fmp->ref_cnt, 1, __ATOMIC_RELAXED); 2021 fm->profile = NULL; 2022 /* Remove from list. */ 2023 if (!priv->sh->meter_aso_en) { 2024 legacy_fm = container_of(fm, 2025 struct mlx5_legacy_flow_meter, fm); 2026 TAILQ_REMOVE(fms, legacy_fm, next); 2027 } 2028 /* Free drop counters. */ 2029 if (fm->drop_cnt) 2030 mlx5_counter_free(dev, fm->drop_cnt); 2031 /* Free meter flow table. */ 2032 if (fm->flow_ipool) { 2033 mlx5_ipool_destroy(fm->flow_ipool); 2034 fm->flow_ipool = 0; 2035 } 2036 mlx5_flow_destroy_mtr_tbls(dev, fm); 2037 if (fm->def_policy) 2038 __atomic_sub_fetch(&priv->sh->mtrmng->def_policy_ref_cnt, 2039 1, __ATOMIC_RELAXED); 2040 if (priv->sh->meter_aso_en) { 2041 if (!fm->def_policy) { 2042 mtr_policy = mlx5_flow_meter_policy_find(dev, 2043 fm->policy_id, NULL); 2044 if (mtr_policy) 2045 __atomic_sub_fetch(&mtr_policy->ref_cnt, 2046 1, __ATOMIC_RELAXED); 2047 fm->policy_id = 0; 2048 } 2049 fm->def_policy = 0; 2050 if (mlx5_l3t_clear_entry(priv->mtr_idx_tbl, fm->meter_id)) 2051 return -1; 2052 mlx5_flow_mtr_free(dev, mtr_idx); 2053 } else { 2054 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], 2055 legacy_fm->idx); 2056 } 2057 return 0; 2058 } 2059 2060 /** 2061 * Destroy meter rules. 2062 * 2063 * @param[in] dev 2064 * Pointer to Ethernet device. 2065 * @param[in] meter_id 2066 * Meter id. 2067 * @param[out] error 2068 * Pointer to rte meter error structure. 2069 * 2070 * @return 2071 * 0 on success, a negative errno value otherwise and rte_errno is set. 2072 */ 2073 static int 2074 mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id, 2075 struct rte_mtr_error *error) 2076 { 2077 struct mlx5_priv *priv = dev->data->dev_private; 2078 struct mlx5_flow_meter_info *fm; 2079 uint32_t mtr_idx = 0; 2080 2081 if (!priv->mtr_en) 2082 return -rte_mtr_error_set(error, ENOTSUP, 2083 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 2084 NULL, 2085 "Meter is not supported"); 2086 /* Meter object must exist. */ 2087 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 2088 if (fm == NULL) 2089 return -rte_mtr_error_set(error, ENOENT, 2090 RTE_MTR_ERROR_TYPE_MTR_ID, 2091 NULL, 2092 "Meter object id not valid."); 2093 /* Meter object must not have any owner. */ 2094 if (fm->ref_cnt > 0) 2095 return -rte_mtr_error_set(error, EBUSY, 2096 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 2097 NULL, 2098 "Meter object is being used."); 2099 /* Destroy the meter profile. */ 2100 if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx)) 2101 return -rte_mtr_error_set(error, EINVAL, 2102 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 2103 NULL, 2104 "MTR object meter profile invalid."); 2105 return 0; 2106 } 2107 2108 /** 2109 * Destroy meter rules. 2110 * 2111 * @param[in] dev 2112 * Pointer to Ethernet device. 2113 * @param[in] meter_id 2114 * Meter id. 2115 * @param[out] error 2116 * Pointer to rte meter error structure. 2117 * 2118 * @return 2119 * 0 on success, a negative errno value otherwise and rte_errno is set. 2120 */ 2121 static int 2122 mlx5_flow_meter_hws_destroy(struct rte_eth_dev *dev, uint32_t meter_id, 2123 struct rte_mtr_error *error) 2124 { 2125 struct mlx5_priv *priv = dev->data->dev_private; 2126 struct mlx5_aso_mtr *aso_mtr; 2127 struct mlx5_flow_meter_info *fm; 2128 struct mlx5_flow_meter_policy *policy; 2129 2130 if (!priv->mtr_profile_arr || 2131 !priv->mtr_policy_arr || 2132 !priv->mtr_bulk.aso) 2133 return -rte_mtr_error_set(error, ENOTSUP, 2134 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL, 2135 "Meter bulk array is not allocated."); 2136 /* Find ASO object. */ 2137 aso_mtr = mlx5_aso_meter_by_idx(priv, meter_id); 2138 fm = &aso_mtr->fm; 2139 if (!fm->initialized) 2140 return -rte_mtr_error_set(error, ENOENT, 2141 RTE_MTR_ERROR_TYPE_MTR_ID, 2142 NULL, "Meter object id not valid."); 2143 /* Meter object must not have any owner. */ 2144 if (fm->ref_cnt > 0) 2145 return -rte_mtr_error_set(error, EBUSY, 2146 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 2147 NULL, "Meter object is being used."); 2148 /* Destroy the meter profile. */ 2149 __atomic_sub_fetch(&fm->profile->ref_cnt, 2150 1, __ATOMIC_RELAXED); 2151 /* Destroy the meter policy. */ 2152 policy = mlx5_flow_meter_policy_find(dev, 2153 fm->policy_id, NULL); 2154 __atomic_sub_fetch(&policy->ref_cnt, 2155 1, __ATOMIC_RELAXED); 2156 memset(fm, 0, sizeof(struct mlx5_flow_meter_info)); 2157 return 0; 2158 } 2159 2160 /** 2161 * Modify meter state. 2162 * 2163 * @param[in] priv 2164 * Pointer to mlx5 private data structure. 2165 * @param[in] fm 2166 * Pointer to flow meter. 2167 * @param[in] new_state 2168 * New state to update. 2169 * @param[out] error 2170 * Pointer to rte meter error structure. 2171 * 2172 * @return 2173 * 0 on success, a negative errno value otherwise and rte_errno is set. 2174 */ 2175 static int 2176 mlx5_flow_meter_modify_state(struct mlx5_priv *priv, 2177 struct mlx5_flow_meter_info *fm, 2178 uint32_t new_state, 2179 struct rte_mtr_error *error) 2180 { 2181 static const struct mlx5_flow_meter_srtcm_rfc2697_prm srtcm = { 2182 .cbs_cir = RTE_BE32(MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL), 2183 .ebs_eir = 0, 2184 }; 2185 uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS | 2186 MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR; 2187 int ret; 2188 2189 if (new_state == MLX5_FLOW_METER_DISABLE) 2190 ret = mlx5_flow_meter_action_modify(priv, fm, 2191 &srtcm, modify_bits, 0, 0); 2192 else 2193 ret = mlx5_flow_meter_action_modify(priv, fm, 2194 &fm->profile->srtcm_prm, 2195 modify_bits, 0, 1); 2196 if (ret) 2197 return -rte_mtr_error_set(error, -ret, 2198 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 2199 NULL, 2200 new_state ? 2201 "Failed to enable meter." : 2202 "Failed to disable meter."); 2203 return 0; 2204 } 2205 2206 /** 2207 * Callback to enable flow meter. 2208 * 2209 * @param[in] dev 2210 * Pointer to Ethernet device. 2211 * @param[in] meter_id 2212 * Meter id. 2213 * @param[out] error 2214 * Pointer to rte meter error structure. 2215 * 2216 * @return 2217 * 0 on success, a negative errno value otherwise and rte_errno is set. 2218 */ 2219 static int 2220 mlx5_flow_meter_enable(struct rte_eth_dev *dev, 2221 uint32_t meter_id, 2222 struct rte_mtr_error *error) 2223 { 2224 struct mlx5_priv *priv = dev->data->dev_private; 2225 struct mlx5_flow_meter_info *fm; 2226 int ret; 2227 2228 if (!priv->mtr_en) 2229 return -rte_mtr_error_set(error, ENOTSUP, 2230 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 2231 "Meter is not supported"); 2232 /* Meter object must exist. */ 2233 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 2234 if (fm == NULL) 2235 return -rte_mtr_error_set(error, ENOENT, 2236 RTE_MTR_ERROR_TYPE_MTR_ID, 2237 NULL, "Meter not found."); 2238 if (fm->active_state == MLX5_FLOW_METER_ENABLE) 2239 return 0; 2240 ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_ENABLE, 2241 error); 2242 if (!ret) 2243 fm->active_state = MLX5_FLOW_METER_ENABLE; 2244 return ret; 2245 } 2246 2247 /** 2248 * Callback to disable flow meter. 2249 * 2250 * @param[in] dev 2251 * Pointer to Ethernet device. 2252 * @param[in] meter_id 2253 * Meter id. 2254 * @param[out] error 2255 * Pointer to rte meter error structure. 2256 * 2257 * @return 2258 * 0 on success, a negative errno value otherwise and rte_errno is set. 2259 */ 2260 static int 2261 mlx5_flow_meter_disable(struct rte_eth_dev *dev, 2262 uint32_t meter_id, 2263 struct rte_mtr_error *error) 2264 { 2265 struct mlx5_priv *priv = dev->data->dev_private; 2266 struct mlx5_flow_meter_info *fm; 2267 int ret; 2268 2269 if (!priv->mtr_en) 2270 return -rte_mtr_error_set(error, ENOTSUP, 2271 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 2272 "Meter is not supported"); 2273 /* Meter object must exist. */ 2274 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 2275 if (fm == NULL) 2276 return -rte_mtr_error_set(error, ENOENT, 2277 RTE_MTR_ERROR_TYPE_MTR_ID, 2278 NULL, "Meter not found."); 2279 if (fm->active_state == MLX5_FLOW_METER_DISABLE) 2280 return 0; 2281 ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_DISABLE, 2282 error); 2283 if (!ret) 2284 fm->active_state = MLX5_FLOW_METER_DISABLE; 2285 return ret; 2286 } 2287 2288 /** 2289 * Callback to update meter profile. 2290 * 2291 * @param[in] dev 2292 * Pointer to Ethernet device. 2293 * @param[in] meter_id 2294 * Meter id. 2295 * @param[in] meter_profile_id 2296 * To be updated meter profile id. 2297 * @param[out] error 2298 * Pointer to rte meter error structure. 2299 * 2300 * @return 2301 * 0 on success, a negative errno value otherwise and rte_errno is set. 2302 */ 2303 static int 2304 mlx5_flow_meter_profile_update(struct rte_eth_dev *dev, 2305 uint32_t meter_id, 2306 uint32_t meter_profile_id, 2307 struct rte_mtr_error *error) 2308 { 2309 struct mlx5_priv *priv = dev->data->dev_private; 2310 struct mlx5_flow_meter_profile *fmp; 2311 struct mlx5_flow_meter_profile *old_fmp; 2312 struct mlx5_flow_meter_info *fm; 2313 uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS | 2314 MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR; 2315 int ret; 2316 2317 if (!priv->mtr_en) 2318 return -rte_mtr_error_set(error, ENOTSUP, 2319 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 2320 "Meter is not supported"); 2321 /* Meter profile must exist. */ 2322 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 2323 if (fmp == NULL) 2324 return -rte_mtr_error_set(error, ENOENT, 2325 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 2326 NULL, "Meter profile not found."); 2327 /* Meter object must exist. */ 2328 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 2329 if (fm == NULL) 2330 return -rte_mtr_error_set(error, ENOENT, 2331 RTE_MTR_ERROR_TYPE_MTR_ID, 2332 NULL, "Meter not found."); 2333 /* MTR object already set to meter profile id. */ 2334 old_fmp = fm->profile; 2335 if (fmp == old_fmp) 2336 return 0; 2337 /* Update the profile. */ 2338 fm->profile = fmp; 2339 /* Update meter params in HW (if not disabled). */ 2340 if (fm->active_state == MLX5_FLOW_METER_DISABLE) 2341 goto dec_ref_cnt; 2342 ret = mlx5_flow_meter_action_modify(priv, fm, &fm->profile->srtcm_prm, 2343 modify_bits, fm->active_state, 1); 2344 if (ret) { 2345 fm->profile = old_fmp; 2346 return -rte_mtr_error_set(error, -ret, 2347 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 2348 NULL, "Failed to update meter" 2349 " parameters in hardware."); 2350 } 2351 dec_ref_cnt: 2352 old_fmp->ref_cnt--; 2353 fmp->ref_cnt++; 2354 return 0; 2355 } 2356 2357 /** 2358 * Callback to update meter stats mask. 2359 * 2360 * @param[in] dev 2361 * Pointer to Ethernet device. 2362 * @param[in] meter_id 2363 * Meter id. 2364 * @param[in] stats_mask 2365 * To be updated stats_mask. 2366 * @param[out] error 2367 * Pointer to rte meter error structure. 2368 * 2369 * @return 2370 * 0 on success, a negative errno value otherwise and rte_errno is set. 2371 */ 2372 static int 2373 mlx5_flow_meter_stats_update(struct rte_eth_dev *dev, 2374 uint32_t meter_id, 2375 uint64_t stats_mask, 2376 struct rte_mtr_error *error) 2377 { 2378 struct mlx5_priv *priv = dev->data->dev_private; 2379 struct mlx5_flow_meter_info *fm; 2380 2381 if (!priv->mtr_en) 2382 return -rte_mtr_error_set(error, ENOTSUP, 2383 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 2384 "Meter is not supported"); 2385 /* Meter object must exist. */ 2386 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 2387 if (fm == NULL) 2388 return -rte_mtr_error_set(error, ENOENT, 2389 RTE_MTR_ERROR_TYPE_MTR_ID, 2390 NULL, "Meter object id not valid."); 2391 if (mlx5_flow_meter_stats_enable_update(dev, fm, stats_mask)) 2392 return -rte_mtr_error_set(error, ENOENT, 2393 RTE_MTR_ERROR_TYPE_MTR_ID, 2394 NULL, "Fail to allocate " 2395 "counter for meter."); 2396 return 0; 2397 } 2398 2399 /** 2400 * Callback to read meter statistics. 2401 * 2402 * @param[in] dev 2403 * Pointer to Ethernet device. 2404 * @param[in] meter_id 2405 * Meter id. 2406 * @param[out] stats 2407 * Pointer to store the statistics. 2408 * @param[out] stats_mask 2409 * Pointer to store the stats_mask. 2410 * @param[in] clear 2411 * Statistic to be cleared after read or not. 2412 * @param[out] error 2413 * Pointer to rte meter error structure. 2414 * 2415 * @return 2416 * 0 on success, a negative errno value otherwise and rte_errno is set. 2417 */ 2418 static int 2419 mlx5_flow_meter_stats_read(struct rte_eth_dev *dev, 2420 uint32_t meter_id, 2421 struct rte_mtr_stats *stats, 2422 uint64_t *stats_mask, 2423 int clear, 2424 struct rte_mtr_error *error) 2425 { 2426 struct mlx5_priv *priv = dev->data->dev_private; 2427 struct mlx5_flow_meter_info *fm; 2428 uint64_t pkts; 2429 uint64_t bytes; 2430 int ret = 0; 2431 2432 if (!priv->mtr_en) 2433 return -rte_mtr_error_set(error, ENOTSUP, 2434 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 2435 "Meter is not supported"); 2436 /* Meter object must exist. */ 2437 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 2438 if (fm == NULL) 2439 return -rte_mtr_error_set(error, ENOENT, 2440 RTE_MTR_ERROR_TYPE_MTR_ID, 2441 NULL, "Meter object id not valid."); 2442 *stats_mask = 0; 2443 if (fm->bytes_dropped) 2444 *stats_mask |= RTE_MTR_STATS_N_BYTES_DROPPED; 2445 if (fm->pkts_dropped) 2446 *stats_mask |= RTE_MTR_STATS_N_PKTS_DROPPED; 2447 memset(stats, 0, sizeof(*stats)); 2448 if (fm->drop_cnt) { 2449 ret = mlx5_counter_query(dev, fm->drop_cnt, clear, &pkts, 2450 &bytes, NULL); 2451 if (ret) 2452 goto error; 2453 /* If need to read the packets, set it. */ 2454 if (fm->pkts_dropped) 2455 stats->n_pkts_dropped = pkts; 2456 /* If need to read the bytes, set it. */ 2457 if (fm->bytes_dropped) 2458 stats->n_bytes_dropped = bytes; 2459 } 2460 return 0; 2461 error: 2462 return -rte_mtr_error_set(error, ret, RTE_MTR_ERROR_TYPE_STATS, NULL, 2463 "Failed to read meter drop counters."); 2464 } 2465 2466 static const struct rte_mtr_ops mlx5_flow_mtr_ops = { 2467 .capabilities_get = mlx5_flow_mtr_cap_get, 2468 .meter_profile_add = mlx5_flow_meter_profile_add, 2469 .meter_profile_delete = mlx5_flow_meter_profile_delete, 2470 .meter_profile_get = mlx5_flow_meter_profile_get, 2471 .meter_policy_validate = mlx5_flow_meter_policy_validate, 2472 .meter_policy_add = mlx5_flow_meter_policy_add, 2473 .meter_policy_delete = mlx5_flow_meter_policy_delete, 2474 .meter_policy_get = mlx5_flow_meter_policy_get, 2475 .create = mlx5_flow_meter_create, 2476 .destroy = mlx5_flow_meter_destroy, 2477 .meter_enable = mlx5_flow_meter_enable, 2478 .meter_disable = mlx5_flow_meter_disable, 2479 .meter_profile_update = mlx5_flow_meter_profile_update, 2480 .meter_dscp_table_update = NULL, 2481 .stats_update = mlx5_flow_meter_stats_update, 2482 .stats_read = mlx5_flow_meter_stats_read, 2483 }; 2484 2485 static const struct rte_mtr_ops mlx5_flow_mtr_hws_ops = { 2486 .capabilities_get = mlx5_flow_mtr_cap_get, 2487 .meter_profile_add = mlx5_flow_meter_profile_hws_add, 2488 .meter_profile_delete = mlx5_flow_meter_profile_hws_delete, 2489 .meter_profile_get = mlx5_flow_meter_profile_get, 2490 .meter_policy_validate = mlx5_flow_meter_policy_hws_validate, 2491 .meter_policy_add = mlx5_flow_meter_policy_hws_add, 2492 .meter_policy_delete = mlx5_flow_meter_policy_hws_delete, 2493 .meter_policy_get = mlx5_flow_meter_policy_get, 2494 .create = mlx5_flow_meter_hws_create, 2495 .destroy = mlx5_flow_meter_hws_destroy, 2496 .meter_enable = mlx5_flow_meter_enable, 2497 .meter_disable = mlx5_flow_meter_disable, 2498 .meter_profile_update = mlx5_flow_meter_profile_update, 2499 .meter_dscp_table_update = NULL, 2500 .stats_update = NULL, 2501 .stats_read = NULL, 2502 }; 2503 2504 /** 2505 * Get meter operations. 2506 * 2507 * @param dev 2508 * Pointer to Ethernet device structure. 2509 * @param arg 2510 * Pointer to set the mtr operations. 2511 * 2512 * @return 2513 * Always 0. 2514 */ 2515 int 2516 mlx5_flow_meter_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg) 2517 { 2518 struct mlx5_priv *priv = dev->data->dev_private; 2519 2520 if (priv->sh->config.dv_flow_en == 2) 2521 *(const struct rte_mtr_ops **)arg = &mlx5_flow_mtr_hws_ops; 2522 else 2523 *(const struct rte_mtr_ops **)arg = &mlx5_flow_mtr_ops; 2524 return 0; 2525 } 2526 2527 /** 2528 * Find meter by id. 2529 * 2530 * @param priv 2531 * Pointer to mlx5_priv. 2532 * @param meter_id 2533 * Meter id. 2534 * @param mtr_idx 2535 * Pointer to Meter index. 2536 * 2537 * @return 2538 * Pointer to the meter info found on success, NULL otherwise. 2539 */ 2540 struct mlx5_flow_meter_info * 2541 mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id, 2542 uint32_t *mtr_idx) 2543 { 2544 struct mlx5_legacy_flow_meter *legacy_fm; 2545 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; 2546 struct mlx5_aso_mtr *aso_mtr; 2547 struct mlx5_aso_mtr_pools_mng *pools_mng = 2548 &priv->sh->mtrmng->pools_mng; 2549 union mlx5_l3t_data data; 2550 uint16_t n_valid; 2551 2552 if (priv->mtr_bulk.aso) { 2553 if (mtr_idx) 2554 *mtr_idx = meter_id; 2555 aso_mtr = priv->mtr_bulk.aso + meter_id; 2556 return &aso_mtr->fm; 2557 } 2558 if (priv->sh->meter_aso_en) { 2559 rte_rwlock_read_lock(&pools_mng->resize_mtrwl); 2560 n_valid = pools_mng->n_valid; 2561 rte_rwlock_read_unlock(&pools_mng->resize_mtrwl); 2562 if (!n_valid || !priv->mtr_idx_tbl || 2563 (mlx5_l3t_get_entry(priv->mtr_idx_tbl, meter_id, &data) || 2564 !data.dword)) 2565 return NULL; 2566 if (mtr_idx) 2567 *mtr_idx = data.dword; 2568 aso_mtr = mlx5_aso_meter_by_idx(priv, data.dword); 2569 /* Remove reference taken by the mlx5_l3t_get_entry. */ 2570 mlx5_l3t_clear_entry(priv->mtr_idx_tbl, meter_id); 2571 if (!aso_mtr || aso_mtr->state == ASO_METER_FREE) 2572 return NULL; 2573 return &aso_mtr->fm; 2574 } 2575 TAILQ_FOREACH(legacy_fm, fms, next) 2576 if (meter_id == legacy_fm->fm.meter_id) { 2577 if (mtr_idx) 2578 *mtr_idx = legacy_fm->idx; 2579 return &legacy_fm->fm; 2580 } 2581 return NULL; 2582 } 2583 2584 /** 2585 * Find meter by index. 2586 * 2587 * @param priv 2588 * Pointer to mlx5_priv. 2589 * @param idx 2590 * Meter index. 2591 * 2592 * @return 2593 * Pointer to the meter info found on success, NULL otherwise. 2594 */ 2595 struct mlx5_flow_meter_info * 2596 flow_dv_meter_find_by_idx(struct mlx5_priv *priv, uint32_t idx) 2597 { 2598 struct mlx5_aso_mtr *aso_mtr; 2599 2600 if (priv->sh->meter_aso_en) { 2601 aso_mtr = mlx5_aso_meter_by_idx(priv, idx); 2602 if (!aso_mtr) 2603 return NULL; 2604 return &aso_mtr->fm; 2605 } else { 2606 return mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], idx); 2607 } 2608 } 2609 2610 /** 2611 * Attach meter to flow. 2612 * Unidirectional Meter creation can only be done 2613 * when flow direction is known, i.e. when calling meter_attach. 2614 * 2615 * @param [in] priv 2616 * Pointer to mlx5 private data. 2617 * @param[in] fm 2618 * Pointer to flow meter. 2619 * @param [in] attr 2620 * Pointer to flow attributes. 2621 * @param [out] error 2622 * Pointer to error structure. 2623 * 2624 * @return 2625 * 0 on success, a negative errno value otherwise and rte_errno is set. 2626 */ 2627 int 2628 mlx5_flow_meter_attach(struct mlx5_priv *priv, 2629 struct mlx5_flow_meter_info *fm, 2630 const struct rte_flow_attr *attr, 2631 struct rte_flow_error *error) 2632 { 2633 int ret = 0; 2634 2635 if (priv->sh->meter_aso_en) { 2636 struct mlx5_aso_mtr *aso_mtr; 2637 2638 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm); 2639 if (mlx5_aso_mtr_wait(priv->sh, MLX5_HW_INV_QUEUE, aso_mtr)) { 2640 return rte_flow_error_set(error, ENOENT, 2641 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2642 NULL, 2643 "Timeout in meter configuration"); 2644 } 2645 rte_spinlock_lock(&fm->sl); 2646 if (fm->shared || !fm->ref_cnt) { 2647 fm->ref_cnt++; 2648 } else { 2649 rte_flow_error_set(error, EINVAL, 2650 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 2651 "Meter cannot be shared"); 2652 ret = -1; 2653 } 2654 rte_spinlock_unlock(&fm->sl); 2655 } else { 2656 rte_spinlock_lock(&fm->sl); 2657 if (fm->meter_action_g) { 2658 if (fm->shared && 2659 attr->transfer == fm->transfer && 2660 attr->ingress == fm->ingress && 2661 attr->egress == fm->egress) { 2662 fm->ref_cnt++; 2663 } else { 2664 rte_flow_error_set(error, EINVAL, 2665 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 2666 fm->shared ? 2667 "Meter attr not match." : 2668 "Meter cannot be shared."); 2669 ret = -1; 2670 } 2671 } else { 2672 fm->ingress = attr->ingress; 2673 fm->egress = attr->egress; 2674 fm->transfer = attr->transfer; 2675 fm->ref_cnt = 1; 2676 /* This also creates the meter object. */ 2677 fm->meter_action_g = mlx5_flow_meter_action_create(priv, 2678 fm); 2679 if (!fm->meter_action_g) { 2680 fm->ref_cnt = 0; 2681 fm->ingress = 0; 2682 fm->egress = 0; 2683 fm->transfer = 0; 2684 rte_flow_error_set(error, EINVAL, 2685 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 2686 "Meter action create failed."); 2687 ret = -1; 2688 } 2689 } 2690 rte_spinlock_unlock(&fm->sl); 2691 } 2692 return ret ? -rte_errno : 0; 2693 } 2694 2695 /** 2696 * Detach meter from flow. 2697 * 2698 * @param [in] priv 2699 * Pointer to mlx5 private data. 2700 * @param [in] fm 2701 * Pointer to flow meter. 2702 */ 2703 void 2704 mlx5_flow_meter_detach(struct mlx5_priv *priv, 2705 struct mlx5_flow_meter_info *fm) 2706 { 2707 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER 2708 rte_spinlock_lock(&fm->sl); 2709 MLX5_ASSERT(fm->ref_cnt); 2710 if (--fm->ref_cnt == 0 && !priv->sh->meter_aso_en) { 2711 mlx5_glue->destroy_flow_action(fm->meter_action_g); 2712 fm->meter_action_g = NULL; 2713 fm->ingress = 0; 2714 fm->egress = 0; 2715 fm->transfer = 0; 2716 } 2717 rte_spinlock_unlock(&fm->sl); 2718 #else 2719 (void)priv; 2720 (void)fm; 2721 #endif 2722 } 2723 2724 /** 2725 * Flush meter with Rx queue configuration. 2726 * 2727 * @param[in] dev 2728 * Pointer to Ethernet device. 2729 */ 2730 void 2731 mlx5_flow_meter_rxq_flush(struct rte_eth_dev *dev) 2732 { 2733 struct mlx5_priv *priv = dev->data->dev_private; 2734 struct mlx5_flow_meter_sub_policy *sub_policy; 2735 struct mlx5_flow_meter_policy *mtr_policy; 2736 void *entry; 2737 uint32_t i, policy_idx; 2738 2739 if (!priv->mtr_en) 2740 return; 2741 if (priv->policy_idx_tbl) { 2742 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) { 2743 policy_idx = *(uint32_t *)entry; 2744 sub_policy = mlx5_ipool_get 2745 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 2746 policy_idx); 2747 if (!sub_policy || !sub_policy->main_policy) 2748 continue; 2749 mtr_policy = sub_policy->main_policy; 2750 if (mtr_policy->is_queue || mtr_policy->is_rss) 2751 mlx5_flow_destroy_sub_policy_with_rxq(dev, 2752 mtr_policy); 2753 } 2754 } 2755 } 2756 2757 /** 2758 * Iterate a meter hierarchy and flush all meters and policies if possible. 2759 * 2760 * @param[in] dev 2761 * Pointer to Ethernet device. 2762 * @param[in] fm 2763 * Pointer to flow meter. 2764 * @param[in] mtr_idx 2765 * .Meter's index 2766 * @param[out] error 2767 * Pointer to rte meter error structure. 2768 * 2769 * @return 2770 * 0 on success, a negative errno value otherwise and rte_errno is set. 2771 */ 2772 static int 2773 mlx5_flow_meter_flush_hierarchy(struct rte_eth_dev *dev, 2774 struct mlx5_flow_meter_info *fm, 2775 uint32_t mtr_idx, 2776 struct rte_mtr_error *error) 2777 { 2778 struct mlx5_priv *priv = dev->data->dev_private; 2779 struct mlx5_flow_meter_policy *policy; 2780 uint32_t policy_id; 2781 struct mlx5_flow_meter_info *next_fm; 2782 uint32_t next_mtr_idx; 2783 struct mlx5_flow_meter_policy *next_policy = NULL; 2784 2785 policy = mlx5_flow_meter_policy_find(dev, fm->policy_id, NULL); 2786 MLX5_ASSERT(policy); 2787 while (!fm->ref_cnt && policy->is_hierarchy) { 2788 policy_id = fm->policy_id; 2789 next_fm = mlx5_flow_meter_hierarchy_next_meter(priv, policy, &next_mtr_idx); 2790 if (next_fm) { 2791 next_policy = mlx5_flow_meter_policy_find(dev, 2792 next_fm->policy_id, 2793 NULL); 2794 MLX5_ASSERT(next_policy); 2795 } 2796 if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx)) 2797 return -rte_mtr_error_set(error, ENOTSUP, 2798 RTE_MTR_ERROR_TYPE_MTR_ID, 2799 NULL, 2800 "Failed to flush meter."); 2801 if (policy->ref_cnt) 2802 break; 2803 if (__mlx5_flow_meter_policy_delete(dev, policy_id, 2804 policy, error, true)) 2805 return -rte_errno; 2806 mlx5_free(policy); 2807 if (!next_fm || !next_policy) 2808 break; 2809 fm = next_fm; 2810 mtr_idx = next_mtr_idx; 2811 policy = next_policy; 2812 } 2813 return 0; 2814 } 2815 2816 /** 2817 * Flush all the hierarchy meters and their policies. 2818 * 2819 * @param[in] dev 2820 * Pointer to Ethernet device. 2821 * @param[out] error 2822 * Pointer to rte meter error structure. 2823 * 2824 * @return 2825 * 0 on success, a negative errno value otherwise and rte_errno is set. 2826 */ 2827 static int 2828 mlx5_flow_meter_flush_all_hierarchies(struct rte_eth_dev *dev, 2829 struct rte_mtr_error *error) 2830 { 2831 struct mlx5_priv *priv = dev->data->dev_private; 2832 struct mlx5_flow_meter_info *fm; 2833 struct mlx5_flow_meter_policy *policy; 2834 struct mlx5_flow_meter_sub_policy *sub_policy; 2835 struct mlx5_flow_meter_info *next_fm; 2836 struct mlx5_aso_mtr *aso_mtr; 2837 uint32_t mtr_idx = 0; 2838 uint32_t i, policy_idx; 2839 void *entry; 2840 2841 if (!priv->mtr_idx_tbl || !priv->policy_idx_tbl) 2842 return 0; 2843 MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) { 2844 mtr_idx = *(uint32_t *)entry; 2845 if (!mtr_idx) 2846 continue; 2847 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx); 2848 fm = &aso_mtr->fm; 2849 if (fm->ref_cnt || fm->def_policy) 2850 continue; 2851 if (mlx5_flow_meter_flush_hierarchy(dev, fm, mtr_idx, error)) 2852 return -rte_errno; 2853 } 2854 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) { 2855 policy_idx = *(uint32_t *)entry; 2856 sub_policy = mlx5_ipool_get 2857 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 2858 policy_idx); 2859 if (!sub_policy) 2860 return -rte_mtr_error_set(error, 2861 EINVAL, 2862 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 2863 NULL, "Meter policy invalid."); 2864 policy = sub_policy->main_policy; 2865 if (!policy || !policy->is_hierarchy || policy->ref_cnt) 2866 continue; 2867 next_fm = mlx5_flow_meter_hierarchy_next_meter(priv, policy, &mtr_idx); 2868 if (__mlx5_flow_meter_policy_delete(dev, i, policy, 2869 error, true)) 2870 return -rte_mtr_error_set(error, 2871 EINVAL, 2872 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 2873 NULL, "Meter policy invalid."); 2874 mlx5_free(policy); 2875 if (!next_fm || next_fm->ref_cnt || next_fm->def_policy) 2876 continue; 2877 if (mlx5_flow_meter_flush_hierarchy(dev, next_fm, 2878 mtr_idx, error)) 2879 return -rte_errno; 2880 } 2881 return 0; 2882 } 2883 /** 2884 * Flush meter configuration. 2885 * 2886 * @param[in] dev 2887 * Pointer to Ethernet device. 2888 * @param[out] error 2889 * Pointer to rte meter error structure. 2890 * 2891 * @return 2892 * 0 on success, a negative errno value otherwise and rte_errno is set. 2893 */ 2894 int 2895 mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error) 2896 { 2897 struct mlx5_priv *priv = dev->data->dev_private; 2898 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; 2899 struct mlx5_flow_meter_profile *fmp; 2900 struct mlx5_legacy_flow_meter *legacy_fm; 2901 struct mlx5_flow_meter_info *fm; 2902 struct mlx5_flow_meter_policy *policy; 2903 struct mlx5_flow_meter_sub_policy *sub_policy; 2904 void *tmp; 2905 uint32_t i, mtr_idx, policy_idx; 2906 void *entry; 2907 struct mlx5_aso_mtr *aso_mtr; 2908 2909 if (!priv->mtr_en) 2910 return 0; 2911 if (priv->sh->meter_aso_en) { 2912 if (mlx5_flow_meter_flush_all_hierarchies(dev, error)) 2913 return -rte_errno; 2914 if (priv->mtr_idx_tbl) { 2915 MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) { 2916 mtr_idx = *(uint32_t *)entry; 2917 if (mtr_idx) { 2918 aso_mtr = 2919 mlx5_aso_meter_by_idx(priv, mtr_idx); 2920 fm = &aso_mtr->fm; 2921 (void)mlx5_flow_meter_params_flush(dev, 2922 fm, mtr_idx); 2923 } 2924 } 2925 mlx5_l3t_destroy(priv->mtr_idx_tbl); 2926 priv->mtr_idx_tbl = NULL; 2927 } 2928 } else { 2929 RTE_TAILQ_FOREACH_SAFE(legacy_fm, fms, next, tmp) { 2930 fm = &legacy_fm->fm; 2931 if (mlx5_flow_meter_params_flush(dev, fm, 0)) 2932 return -rte_mtr_error_set(error, EINVAL, 2933 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 2934 NULL, "MTR object meter profile invalid."); 2935 } 2936 } 2937 if (priv->mtr_bulk.aso) { 2938 for (i = 0; i < priv->mtr_config.nb_meters; i++) { 2939 aso_mtr = mlx5_aso_meter_by_idx(priv, i); 2940 fm = &aso_mtr->fm; 2941 if (fm->initialized) 2942 mlx5_flow_meter_hws_destroy(dev, i, error); 2943 } 2944 } 2945 if (priv->policy_idx_tbl) { 2946 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) { 2947 policy_idx = *(uint32_t *)entry; 2948 sub_policy = mlx5_ipool_get 2949 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 2950 policy_idx); 2951 if (!sub_policy) 2952 return -rte_mtr_error_set(error, 2953 EINVAL, 2954 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 2955 NULL, "MTR object " 2956 "meter policy invalid."); 2957 if (__mlx5_flow_meter_policy_delete(dev, i, 2958 sub_policy->main_policy, 2959 error, true)) 2960 return -rte_mtr_error_set(error, 2961 EINVAL, 2962 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 2963 NULL, "MTR object " 2964 "meter policy invalid."); 2965 mlx5_free(sub_policy->main_policy); 2966 } 2967 mlx5_l3t_destroy(priv->policy_idx_tbl); 2968 priv->policy_idx_tbl = NULL; 2969 } 2970 if (priv->mtr_policy_arr) { 2971 for (i = 0; i < priv->mtr_config.nb_meter_policies; i++) { 2972 policy = mlx5_flow_meter_policy_find(dev, i, 2973 &policy_idx); 2974 if (policy->initialized) 2975 mlx5_flow_meter_policy_hws_delete(dev, i, 2976 error); 2977 } 2978 } 2979 if (priv->mtr_profile_tbl) { 2980 MLX5_L3T_FOREACH(priv->mtr_profile_tbl, i, entry) { 2981 fmp = entry; 2982 if (mlx5_flow_meter_profile_delete(dev, fmp->id, 2983 error)) 2984 return -rte_mtr_error_set(error, EINVAL, 2985 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 2986 NULL, "Fail to destroy " 2987 "meter profile."); 2988 } 2989 mlx5_l3t_destroy(priv->mtr_profile_tbl); 2990 priv->mtr_profile_tbl = NULL; 2991 } 2992 if (priv->mtr_profile_arr) { 2993 for (i = 0; i < priv->mtr_config.nb_meter_profiles; i++) { 2994 fmp = mlx5_flow_meter_profile_find(priv, i); 2995 if (fmp->initialized) 2996 mlx5_flow_meter_profile_hws_delete(dev, i, 2997 error); 2998 } 2999 } 3000 /* Delete default policy table. */ 3001 mlx5_flow_destroy_def_policy(dev); 3002 if (priv->sh->refcnt == 1) 3003 mlx5_flow_destroy_mtr_drop_tbls(dev); 3004 #ifdef HAVE_MLX5_HWS_SUPPORT 3005 /* Destroy HWS configuration. */ 3006 mlx5_flow_meter_uninit(dev); 3007 #endif 3008 return 0; 3009 } 3010