1 // SPDX-License-Identifier: BSD-3-Clause 2 /* 3 * Copyright 2018 Mellanox Technologies, Ltd 4 */ 5 #include <math.h> 6 7 #include <rte_tailq.h> 8 #include <rte_malloc.h> 9 #include <rte_mtr.h> 10 #include <rte_mtr_driver.h> 11 12 #include <mlx5_devx_cmds.h> 13 #include <mlx5_malloc.h> 14 15 #include "mlx5.h" 16 #include "mlx5_flow.h" 17 18 static int mlx5_flow_meter_disable(struct rte_eth_dev *dev, 19 uint32_t meter_id, struct rte_mtr_error *error); 20 21 /** 22 * Create the meter action. 23 * 24 * @param priv 25 * Pointer to mlx5_priv. 26 * @param[in] fm 27 * Pointer to flow meter to be converted. 28 * 29 * @return 30 * Pointer to the meter action on success, NULL otherwise. 31 */ 32 static void * 33 mlx5_flow_meter_action_create(struct mlx5_priv *priv, 34 struct mlx5_flow_meter_info *fm) 35 { 36 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER 37 struct mlx5dv_dr_flow_meter_attr mtr_init; 38 uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)]; 39 struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = 40 &fm->profile->srtcm_prm; 41 uint32_t cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir); 42 uint32_t ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir); 43 uint32_t val; 44 enum mlx5_meter_domain domain = 45 fm->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 46 fm->egress ? MLX5_MTR_DOMAIN_EGRESS : 47 MLX5_MTR_DOMAIN_INGRESS; 48 struct mlx5_flow_meter_def_policy *def_policy = 49 priv->sh->mtrmng->def_policy[domain]; 50 51 memset(fmp, 0, MLX5_ST_SZ_BYTES(flow_meter_parameters)); 52 MLX5_SET(flow_meter_parameters, fmp, valid, 1); 53 MLX5_SET(flow_meter_parameters, fmp, bucket_overflow, 1); 54 MLX5_SET(flow_meter_parameters, fmp, 55 start_color, MLX5_FLOW_COLOR_GREEN); 56 MLX5_SET(flow_meter_parameters, fmp, both_buckets_on_green, 0); 57 val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK; 58 MLX5_SET(flow_meter_parameters, fmp, cbs_exponent, val); 59 val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK; 60 MLX5_SET(flow_meter_parameters, fmp, cbs_mantissa, val); 61 val = (cbs_cir >> ASO_DSEG_XIR_EXP_OFFSET) & ASO_DSEG_EXP_MASK; 62 MLX5_SET(flow_meter_parameters, fmp, cir_exponent, val); 63 val = (cbs_cir & ASO_DSEG_MAN_MASK); 64 MLX5_SET(flow_meter_parameters, fmp, cir_mantissa, val); 65 val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK; 66 MLX5_SET(flow_meter_parameters, fmp, ebs_exponent, val); 67 val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK; 68 MLX5_SET(flow_meter_parameters, fmp, ebs_mantissa, val); 69 mtr_init.next_table = def_policy->sub_policy.tbl_rsc->obj; 70 mtr_init.reg_c_index = priv->sh->registers.aso_reg - REG_C_0; 71 mtr_init.flow_meter_parameter = fmp; 72 mtr_init.flow_meter_parameter_sz = 73 MLX5_ST_SZ_BYTES(flow_meter_parameters); 74 mtr_init.active = fm->active_state; 75 return mlx5_glue->dv_create_flow_action_meter(&mtr_init); 76 #else 77 (void)priv; 78 (void)fm; 79 return NULL; 80 #endif 81 } 82 83 /** 84 * Find meter profile by id. 85 * 86 * @param priv 87 * Pointer to mlx5_priv. 88 * @param meter_profile_id 89 * Meter profile id. 90 * 91 * @return 92 * Pointer to the profile found on success, NULL otherwise. 93 */ 94 static struct mlx5_flow_meter_profile * 95 mlx5_flow_meter_profile_find(struct mlx5_priv *priv, uint32_t meter_profile_id) 96 { 97 struct mlx5_flow_meter_profile *fmp; 98 union mlx5_l3t_data data; 99 int32_t ret; 100 101 if (priv->mtr_profile_arr) 102 return &priv->mtr_profile_arr[meter_profile_id]; 103 if (mlx5_l3t_get_entry(priv->mtr_profile_tbl, 104 meter_profile_id, &data) || !data.ptr) 105 return NULL; 106 fmp = data.ptr; 107 /* Remove reference taken by the mlx5_l3t_get_entry. */ 108 ret = mlx5_l3t_clear_entry(priv->mtr_profile_tbl, 109 meter_profile_id); 110 if (!ret || ret == -1) 111 return NULL; 112 return fmp; 113 } 114 115 /** 116 * Validate the MTR profile. 117 * 118 * @param[in] dev 119 * Pointer to Ethernet device. 120 * @param[in] meter_profile_id 121 * Meter profile id. 122 * @param[in] profile 123 * Pointer to meter profile detail. 124 * @param[out] error 125 * Pointer to the error structure. 126 * 127 * @return 128 * 0 on success, a negative errno value otherwise and rte_errno is set. 129 */ 130 static int 131 mlx5_flow_meter_profile_validate(struct rte_eth_dev *dev, 132 uint32_t meter_profile_id, 133 struct rte_mtr_meter_profile *profile, 134 struct rte_mtr_error *error) 135 { 136 struct mlx5_priv *priv = dev->data->dev_private; 137 struct mlx5_flow_meter_profile *fmp; 138 uint32_t ls_factor; 139 int ret; 140 uint64_t cir, cbs; 141 uint64_t eir, ebs; 142 uint64_t pir, pbs; 143 144 /* Profile must not be NULL. */ 145 if (profile == NULL) 146 return -rte_mtr_error_set(error, EINVAL, 147 RTE_MTR_ERROR_TYPE_METER_PROFILE, 148 NULL, "Meter profile is null."); 149 /* Meter profile ID must be valid. */ 150 if (priv->mtr_profile_arr) { 151 if (meter_profile_id >= priv->mtr_config.nb_meter_profiles) 152 return -rte_mtr_error_set(error, EINVAL, 153 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 154 NULL, "Meter profile id not valid."); 155 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 156 /* Meter profile must not exist. */ 157 if (fmp->initialized) 158 return -rte_mtr_error_set(error, EEXIST, 159 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 160 NULL, "Meter profile already exists."); 161 } else { 162 if (meter_profile_id == UINT32_MAX) 163 return -rte_mtr_error_set(error, EINVAL, 164 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 165 NULL, "Meter profile id not valid."); 166 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 167 /* Meter profile must not exist. */ 168 if (fmp) 169 return -rte_mtr_error_set(error, EEXIST, 170 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 171 NULL, "Meter profile already exists."); 172 } 173 if (!priv->sh->meter_aso_en) { 174 /* Old version is even not supported. */ 175 if (!priv->sh->cdev->config.hca_attr.qos.flow_meter_old) 176 return -rte_mtr_error_set(error, ENOTSUP, 177 RTE_MTR_ERROR_TYPE_METER_PROFILE, 178 NULL, "Metering is not supported."); 179 /* Old FW metering only supports srTCM. */ 180 if (profile->alg != RTE_MTR_SRTCM_RFC2697) { 181 return -rte_mtr_error_set(error, ENOTSUP, 182 RTE_MTR_ERROR_TYPE_METER_PROFILE, 183 NULL, "Metering algorithm is not supported."); 184 } else if (profile->srtcm_rfc2697.ebs) { 185 /* EBS is not supported for old metering. */ 186 return -rte_mtr_error_set(error, ENOTSUP, 187 RTE_MTR_ERROR_TYPE_METER_PROFILE, 188 NULL, "EBS is not supported."); 189 } 190 if (profile->packet_mode) 191 return -rte_mtr_error_set(error, ENOTSUP, 192 RTE_MTR_ERROR_TYPE_METER_PROFILE, NULL, 193 "Metering algorithm packet mode is not supported."); 194 } 195 ls_factor = profile->packet_mode ? MLX5_MTRS_PPS_MAP_BPS_SHIFT : 0; 196 switch (profile->alg) { 197 case RTE_MTR_SRTCM_RFC2697: 198 cir = profile->srtcm_rfc2697.cir << ls_factor; 199 cbs = profile->srtcm_rfc2697.cbs << ls_factor; 200 ebs = profile->srtcm_rfc2697.ebs << ls_factor; 201 /* EBS could be zero for old metering. */ 202 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX && 203 cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX && 204 ebs <= MLX5_SRTCM_XBS_MAX) { 205 ret = 0; 206 } else { 207 ret = -rte_mtr_error_set(error, ENOTSUP, 208 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 209 "Profile values out of range."); 210 } 211 break; 212 case RTE_MTR_TRTCM_RFC2698: 213 cir = profile->trtcm_rfc2698.cir << ls_factor; 214 cbs = profile->trtcm_rfc2698.cbs << ls_factor; 215 pir = profile->trtcm_rfc2698.pir << ls_factor; 216 pbs = profile->trtcm_rfc2698.pbs << ls_factor; 217 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX && 218 cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX && 219 pir >= cir && pir <= (MLX5_SRTCM_XIR_MAX * 2) && 220 pbs >= cbs && pbs <= (MLX5_SRTCM_XBS_MAX * 2)) { 221 ret = 0; 222 } else { 223 ret = -rte_mtr_error_set(error, ENOTSUP, 224 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 225 "Profile values out of range."); 226 } 227 break; 228 case RTE_MTR_TRTCM_RFC4115: 229 cir = profile->trtcm_rfc4115.cir << ls_factor; 230 cbs = profile->trtcm_rfc4115.cbs << ls_factor; 231 eir = profile->trtcm_rfc4115.eir << ls_factor; 232 ebs = profile->trtcm_rfc4115.ebs << ls_factor; 233 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX && 234 cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX && 235 eir <= MLX5_SRTCM_XIR_MAX && ebs <= MLX5_SRTCM_XBS_MAX) { 236 ret = 0; 237 } else { 238 ret = -rte_mtr_error_set(error, ENOTSUP, 239 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 240 "Profile values out of range."); 241 } 242 break; 243 default: 244 ret = -rte_mtr_error_set(error, ENOTSUP, 245 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 246 "Unknown metering algorithm."); 247 break; 248 } 249 return ret; 250 } 251 252 /* 253 * Calculate mantissa and exponent for cir / eir. 254 * 255 * @param[in] xir 256 * Value to be calculated. 257 * @param[out] man 258 * Pointer to the mantissa. 259 * @param[out] exp 260 * Pointer to the exp. 261 */ 262 static inline void 263 mlx5_flow_meter_xir_man_exp_calc(int64_t xir, uint8_t *man, uint8_t *exp) 264 { 265 int64_t _xir; 266 int64_t delta = INT64_MAX; 267 uint8_t _man = 0; 268 uint8_t _exp = 0; 269 uint64_t m, e; 270 271 /* Special case xir == 0 ? both exp and mantissa are 0. */ 272 if (xir == 0) { 273 *man = 0; 274 *exp = 0; 275 return; 276 } 277 for (m = 0; m <= 0xFF; m++) { /* man width 8 bit */ 278 for (e = 0; e <= 0x1F; e++) { /* exp width 5bit */ 279 _xir = (1000000000ULL * m) >> e; 280 if (llabs(xir - _xir) <= delta) { 281 delta = llabs(xir - _xir); 282 _man = m; 283 _exp = e; 284 } 285 } 286 } 287 *man = _man; 288 *exp = _exp; 289 } 290 291 /* 292 * Calculate mantissa and exponent for xbs. 293 * 294 * @param[in] xbs 295 * Value to be calculated. 296 * @param[out] man 297 * Pointer to the mantissa. 298 * @param[out] exp 299 * Pointer to the exp. 300 */ 301 static inline void 302 mlx5_flow_meter_xbs_man_exp_calc(uint64_t xbs, uint8_t *man, uint8_t *exp) 303 { 304 int _exp; 305 double _man; 306 307 /* Special case xbs == 0 ? both exp and mantissa are 0. */ 308 if (xbs == 0) { 309 *man = 0; 310 *exp = 0; 311 return; 312 } 313 /* xbs = xbs_mantissa * 2^xbs_exponent */ 314 _man = frexp(xbs, &_exp); 315 if (_exp >= MLX5_MAN_WIDTH) { 316 _man = _man * pow(2, MLX5_MAN_WIDTH); 317 _exp = _exp - MLX5_MAN_WIDTH; 318 } 319 *man = (uint8_t)ceil(_man); 320 *exp = _exp; 321 } 322 323 /** 324 * Fill the prm meter parameter. 325 * 326 * @param[in,out] fmp 327 * Pointer to meter profile to be converted. 328 * @param[out] error 329 * Pointer to the error structure. 330 * 331 * @return 332 * 0 on success, a negative errno value otherwise and rte_errno is set. 333 */ 334 static int 335 mlx5_flow_meter_param_fill(struct mlx5_flow_meter_profile *fmp, 336 struct rte_mtr_error *error) 337 { 338 struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = &fmp->srtcm_prm; 339 uint8_t man, exp; 340 uint32_t cbs_exp, cbs_man, cir_exp, cir_man; 341 uint32_t eir_exp, eir_man, ebs_exp, ebs_man; 342 uint64_t cir, cbs, eir, ebs; 343 344 switch (fmp->profile.alg) { 345 case RTE_MTR_SRTCM_RFC2697: 346 cir = fmp->profile.srtcm_rfc2697.cir; 347 cbs = fmp->profile.srtcm_rfc2697.cbs; 348 eir = 0; 349 ebs = fmp->profile.srtcm_rfc2697.ebs; 350 break; 351 case RTE_MTR_TRTCM_RFC2698: 352 MLX5_ASSERT(fmp->profile.trtcm_rfc2698.pir > 353 fmp->profile.trtcm_rfc2698.cir && 354 fmp->profile.trtcm_rfc2698.pbs > 355 fmp->profile.trtcm_rfc2698.cbs); 356 cir = fmp->profile.trtcm_rfc2698.cir; 357 cbs = fmp->profile.trtcm_rfc2698.cbs; 358 /* EIR / EBS are filled with PIR / PBS. */ 359 eir = fmp->profile.trtcm_rfc2698.pir; 360 ebs = fmp->profile.trtcm_rfc2698.pbs; 361 break; 362 case RTE_MTR_TRTCM_RFC4115: 363 cir = fmp->profile.trtcm_rfc4115.cir; 364 cbs = fmp->profile.trtcm_rfc4115.cbs; 365 eir = fmp->profile.trtcm_rfc4115.eir; 366 ebs = fmp->profile.trtcm_rfc4115.ebs; 367 break; 368 default: 369 return -rte_mtr_error_set(error, EINVAL, 370 RTE_MTR_ERROR_TYPE_METER_PROFILE, NULL, 371 "Metering algorithm mode is invalid"); 372 } 373 /* Adjust the values for PPS mode. */ 374 if (fmp->profile.packet_mode) { 375 cir <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT; 376 cbs <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT; 377 eir <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT; 378 ebs <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT; 379 } 380 /* cir = 8G * cir_mantissa * 1/(2^cir_exponent)) Bytes/Sec */ 381 mlx5_flow_meter_xir_man_exp_calc(cir, &man, &exp); 382 /* Check if cir mantissa is too large. */ 383 if (exp > ASO_DSEG_XIR_EXP_MASK) 384 return -rte_mtr_error_set(error, ENOTSUP, 385 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 386 "meter profile parameter cir is not supported."); 387 cir_man = man; 388 cir_exp = exp; 389 /* cbs = cbs_mantissa * 2^cbs_exponent */ 390 mlx5_flow_meter_xbs_man_exp_calc(cbs, &man, &exp); 391 /* Check if cbs mantissa is too large. */ 392 if (exp > ASO_DSEG_EXP_MASK) 393 return -rte_mtr_error_set(error, ENOTSUP, 394 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 395 "meter profile parameter cbs is not supported."); 396 cbs_man = man; 397 cbs_exp = exp; 398 srtcm->cbs_cir = rte_cpu_to_be_32(cbs_exp << ASO_DSEG_CBS_EXP_OFFSET | 399 cbs_man << ASO_DSEG_CBS_MAN_OFFSET | 400 cir_exp << ASO_DSEG_XIR_EXP_OFFSET | 401 cir_man); 402 mlx5_flow_meter_xir_man_exp_calc(eir, &man, &exp); 403 /* Check if eir mantissa is too large. */ 404 if (exp > ASO_DSEG_XIR_EXP_MASK) 405 return -rte_mtr_error_set(error, ENOTSUP, 406 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 407 "meter profile parameter eir is not supported."); 408 eir_man = man; 409 eir_exp = exp; 410 mlx5_flow_meter_xbs_man_exp_calc(ebs, &man, &exp); 411 /* Check if ebs mantissa is too large. */ 412 if (exp > ASO_DSEG_EXP_MASK) 413 return -rte_mtr_error_set(error, ENOTSUP, 414 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 415 "meter profile parameter ebs is not supported."); 416 ebs_man = man; 417 ebs_exp = exp; 418 srtcm->ebs_eir = rte_cpu_to_be_32(ebs_exp << ASO_DSEG_EBS_EXP_OFFSET | 419 ebs_man << ASO_DSEG_EBS_MAN_OFFSET | 420 eir_exp << ASO_DSEG_XIR_EXP_OFFSET | 421 eir_man); 422 if (srtcm->cbs_cir) 423 fmp->g_support = 1; 424 if (srtcm->ebs_eir) 425 fmp->y_support = 1; 426 return 0; 427 } 428 429 /** 430 * Callback to get MTR capabilities. 431 * 432 * @param[in] dev 433 * Pointer to Ethernet device. 434 * @param[out] cap 435 * Pointer to save MTR capabilities. 436 * @param[out] error 437 * Pointer to the error structure. 438 * 439 * @return 440 * 0 on success, a negative errno value otherwise and rte_errno is set. 441 */ 442 static int 443 mlx5_flow_mtr_cap_get(struct rte_eth_dev *dev, 444 struct rte_mtr_capabilities *cap, 445 struct rte_mtr_error *error __rte_unused) 446 { 447 struct mlx5_priv *priv = dev->data->dev_private; 448 struct mlx5_hca_qos_attr *qattr = &priv->sh->cdev->config.hca_attr.qos; 449 450 if (!priv->mtr_en) 451 return -rte_mtr_error_set(error, ENOTSUP, 452 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 453 "Meter is not supported"); 454 memset(cap, 0, sizeof(*cap)); 455 if (priv->sh->meter_aso_en) { 456 /* 2 meters per one ASO cache line. */ 457 cap->n_max = 1 << (qattr->log_max_num_meter_aso + 1); 458 cap->srtcm_rfc2697_packet_mode_supported = 1; 459 cap->trtcm_rfc2698_packet_mode_supported = 1; 460 cap->trtcm_rfc4115_packet_mode_supported = 1; 461 } else { 462 cap->n_max = 1 << qattr->log_max_flow_meter; 463 } 464 cap->srtcm_rfc2697_byte_mode_supported = 1; 465 cap->trtcm_rfc2698_byte_mode_supported = 1; 466 cap->trtcm_rfc4115_byte_mode_supported = 1; 467 cap->n_shared_max = cap->n_max; 468 cap->identical = 1; 469 cap->shared_identical = 1; 470 cap->shared_n_flows_per_mtr_max = 4 << 20; 471 /* 2M flows can share the same meter. */ 472 cap->chaining_n_mtrs_per_flow_max = 1; /* Chaining is not supported. */ 473 cap->meter_srtcm_rfc2697_n_max = qattr->flow_meter_old ? cap->n_max : 0; 474 cap->meter_trtcm_rfc2698_n_max = qattr->flow_meter_old ? cap->n_max : 0; 475 cap->meter_trtcm_rfc4115_n_max = qattr->flow_meter_old ? cap->n_max : 0; 476 cap->meter_rate_max = 1ULL << 40; /* 1 Tera tokens per sec. */ 477 cap->meter_policy_n_max = cap->n_max; 478 cap->stats_mask = RTE_MTR_STATS_N_BYTES_DROPPED | 479 RTE_MTR_STATS_N_PKTS_DROPPED; 480 return 0; 481 } 482 483 /** 484 * Callback to add MTR profile. 485 * 486 * @param[in] dev 487 * Pointer to Ethernet device. 488 * @param[in] meter_profile_id 489 * Meter profile id. 490 * @param[in] profile 491 * Pointer to meter profile detail. 492 * @param[out] error 493 * Pointer to the error structure. 494 * 495 * @return 496 * 0 on success, a negative errno value otherwise and rte_errno is set. 497 */ 498 static int 499 mlx5_flow_meter_profile_add(struct rte_eth_dev *dev, 500 uint32_t meter_profile_id, 501 struct rte_mtr_meter_profile *profile, 502 struct rte_mtr_error *error) 503 { 504 struct mlx5_priv *priv = dev->data->dev_private; 505 struct mlx5_flow_meter_profile *fmp; 506 union mlx5_l3t_data data; 507 int ret; 508 509 if (!priv->mtr_en) 510 return -rte_mtr_error_set(error, ENOTSUP, 511 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 512 "Meter is not supported"); 513 /* Check input params. */ 514 ret = mlx5_flow_meter_profile_validate(dev, meter_profile_id, 515 profile, error); 516 if (ret) 517 return ret; 518 /* Meter profile memory allocation. */ 519 fmp = mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct mlx5_flow_meter_profile), 520 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 521 if (fmp == NULL) 522 return -rte_mtr_error_set(error, ENOMEM, 523 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 524 NULL, "Meter profile memory " 525 "alloc failed."); 526 /* Fill profile info. */ 527 fmp->id = meter_profile_id; 528 fmp->profile = *profile; 529 /* Fill the flow meter parameters for the PRM. */ 530 ret = mlx5_flow_meter_param_fill(fmp, error); 531 if (ret) 532 goto error; 533 data.ptr = fmp; 534 ret = mlx5_l3t_set_entry(priv->mtr_profile_tbl, 535 meter_profile_id, &data); 536 if (ret) 537 return -rte_mtr_error_set(error, ENOTSUP, 538 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 539 NULL, "Meter profile insert fail."); 540 return 0; 541 error: 542 mlx5_free(fmp); 543 return ret; 544 } 545 546 /** 547 * Callback to delete MTR profile. 548 * 549 * @param[in] dev 550 * Pointer to Ethernet device. 551 * @param[in] meter_profile_id 552 * Meter profile id. 553 * @param[out] error 554 * Pointer to the error structure. 555 * 556 * @return 557 * 0 on success, a negative errno value otherwise and rte_errno is set. 558 */ 559 static int 560 mlx5_flow_meter_profile_delete(struct rte_eth_dev *dev, 561 uint32_t meter_profile_id, 562 struct rte_mtr_error *error) 563 { 564 struct mlx5_priv *priv = dev->data->dev_private; 565 struct mlx5_flow_meter_profile *fmp; 566 567 if (!priv->mtr_en) 568 return -rte_mtr_error_set(error, ENOTSUP, 569 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 570 "Meter is not supported"); 571 /* Meter profile must exist. */ 572 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 573 if (fmp == NULL) 574 return -rte_mtr_error_set(error, ENOENT, 575 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 576 &meter_profile_id, 577 "Meter profile id is invalid."); 578 /* Check profile is unused. */ 579 if (fmp->ref_cnt) 580 return -rte_mtr_error_set(error, EBUSY, 581 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 582 NULL, "Meter profile is in use."); 583 if (mlx5_l3t_clear_entry(priv->mtr_profile_tbl, meter_profile_id)) 584 return -rte_mtr_error_set(error, EBUSY, 585 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 586 NULL, "Meter profile remove fail."); 587 mlx5_free(fmp); 588 return 0; 589 } 590 591 /** 592 * Callback to get MTR profile. 593 * 594 * @param[in] dev 595 * Pointer to Ethernet device. 596 * @param[in] meter_profile_id 597 * Meter profile id. 598 * @param[out] error 599 * Pointer to the error structure. 600 * 601 * @return 602 * A valid handle in case of success, NULL otherwise. 603 */ 604 static struct rte_flow_meter_profile * 605 mlx5_flow_meter_profile_get(struct rte_eth_dev *dev, 606 uint32_t meter_profile_id, 607 struct rte_mtr_error *error) 608 { 609 struct mlx5_priv *priv = dev->data->dev_private; 610 611 if (!priv->mtr_en) { 612 rte_mtr_error_set(error, ENOTSUP, 613 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 614 "Meter is not supported"); 615 return NULL; 616 } 617 return (void *)(uintptr_t)mlx5_flow_meter_profile_find(priv, 618 meter_profile_id); 619 } 620 621 /** 622 * Callback to add MTR profile with HWS. 623 * 624 * @param[in] dev 625 * Pointer to Ethernet device. 626 * @param[in] meter_profile_id 627 * Meter profile id. 628 * @param[in] profile 629 * Pointer to meter profile detail. 630 * @param[out] error 631 * Pointer to the error structure. 632 * 633 * @return 634 * 0 on success, a negative errno value otherwise and rte_errno is set. 635 */ 636 static int 637 mlx5_flow_meter_profile_hws_add(struct rte_eth_dev *dev, 638 uint32_t meter_profile_id, 639 struct rte_mtr_meter_profile *profile, 640 struct rte_mtr_error *error) 641 { 642 struct mlx5_priv *priv = dev->data->dev_private; 643 struct mlx5_flow_meter_profile *fmp; 644 int ret; 645 646 if (!priv->mtr_profile_arr) 647 return mlx5_flow_meter_profile_add(dev, meter_profile_id, profile, error); 648 /* Check input params. */ 649 ret = mlx5_flow_meter_profile_validate(dev, meter_profile_id, 650 profile, error); 651 if (ret) 652 return ret; 653 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 654 /* Fill profile info. */ 655 fmp->id = meter_profile_id; 656 fmp->profile = *profile; 657 fmp->initialized = 1; 658 /* Fill the flow meter parameters for the PRM. */ 659 return mlx5_flow_meter_param_fill(fmp, error); 660 } 661 662 /** 663 * Callback to delete MTR profile with HWS. 664 * 665 * @param[in] dev 666 * Pointer to Ethernet device. 667 * @param[in] meter_profile_id 668 * Meter profile id. 669 * @param[out] error 670 * Pointer to the error structure. 671 * 672 * @return 673 * 0 on success, a negative errno value otherwise and rte_errno is set. 674 */ 675 static int 676 mlx5_flow_meter_profile_hws_delete(struct rte_eth_dev *dev, 677 uint32_t meter_profile_id, 678 struct rte_mtr_error *error) 679 { 680 struct mlx5_priv *priv = dev->data->dev_private; 681 struct mlx5_flow_meter_profile *fmp; 682 683 if (!priv->mtr_profile_arr) 684 return mlx5_flow_meter_profile_delete(dev, meter_profile_id, error); 685 /* Meter profile must exist. */ 686 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 687 if (!fmp->initialized) 688 return -rte_mtr_error_set(error, ENOENT, 689 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 690 &meter_profile_id, 691 "Meter profile id is invalid."); 692 /* Check profile is unused. */ 693 if (fmp->ref_cnt) 694 return -rte_mtr_error_set(error, EBUSY, 695 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 696 NULL, "Meter profile is in use."); 697 memset(fmp, 0, sizeof(struct mlx5_flow_meter_profile)); 698 return 0; 699 } 700 701 /** 702 * Find policy by id. 703 * 704 * @param[in] dev 705 * Pointer to Ethernet device. 706 * @param policy_id 707 * Policy id. 708 * 709 * @return 710 * Pointer to the policy found on success, NULL otherwise. 711 */ 712 struct mlx5_flow_meter_policy * 713 mlx5_flow_meter_policy_find(struct rte_eth_dev *dev, 714 uint32_t policy_id, 715 uint32_t *policy_idx) 716 { 717 struct mlx5_priv *priv = dev->data->dev_private; 718 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 719 union mlx5_l3t_data data; 720 721 if (priv->mtr_policy_arr) { 722 if (policy_idx) 723 *policy_idx = policy_id; 724 return &priv->mtr_policy_arr[policy_id]; 725 } 726 if (policy_id > MLX5_MAX_SUB_POLICY_TBL_NUM || !priv->policy_idx_tbl) 727 return NULL; 728 if (mlx5_l3t_get_entry(priv->policy_idx_tbl, policy_id, &data) || 729 !data.dword) 730 return NULL; 731 if (policy_idx) 732 *policy_idx = data.dword; 733 sub_policy = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 734 data.dword); 735 /* Remove reference taken by the mlx5_l3t_get_entry. */ 736 mlx5_l3t_clear_entry(priv->policy_idx_tbl, policy_id); 737 if (sub_policy) 738 if (sub_policy->main_policy_id) 739 return sub_policy->main_policy; 740 return NULL; 741 } 742 743 /** 744 * Get the next meter from one meter's policy in hierarchy chain. 745 * Lock free, mutex should be acquired by caller. 746 * 747 * @param[in] priv 748 * Pointer to mlx5_priv. 749 * @param[in] policy 750 * Pointer to flow meter policy. 751 * @param[out] mtr_idx 752 * Pointer to Meter index. 753 * 754 * @return 755 * Pointer to the next meter, or NULL when fail. 756 */ 757 struct mlx5_flow_meter_info * 758 mlx5_flow_meter_hierarchy_next_meter(struct mlx5_priv *priv, 759 struct mlx5_flow_meter_policy *policy, 760 uint32_t *mtr_idx) 761 { 762 int i; 763 764 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 765 if (policy->act_cnt[i].fate_action == MLX5_FLOW_FATE_MTR) 766 return mlx5_flow_meter_find(priv, 767 policy->act_cnt[i].next_mtr_id, 768 mtr_idx); 769 } 770 return NULL; 771 } 772 773 /** 774 * Get the last meter's policy from one meter's policy in hierarchy. 775 * 776 * @param[in] dev 777 * Pointer to Ethernet device. 778 * @param[in] policy 779 * Pointer to flow meter policy. 780 * 781 * @return 782 * Pointer to the final meter's policy, or NULL when fail. 783 */ 784 struct mlx5_flow_meter_policy * 785 mlx5_flow_meter_hierarchy_get_final_policy(struct rte_eth_dev *dev, 786 struct mlx5_flow_meter_policy *policy) 787 { 788 struct mlx5_priv *priv = dev->data->dev_private; 789 struct mlx5_flow_meter_info *next_fm; 790 struct mlx5_flow_meter_policy *next_policy = policy; 791 792 while (next_policy->is_hierarchy) { 793 rte_spinlock_lock(&next_policy->sl); 794 next_fm = mlx5_flow_meter_hierarchy_next_meter(priv, next_policy, NULL); 795 rte_spinlock_unlock(&next_policy->sl); 796 if (!next_fm || next_fm->def_policy) 797 return NULL; 798 next_policy = mlx5_flow_meter_policy_find(dev, 799 next_fm->policy_id, NULL); 800 MLX5_ASSERT(next_policy); 801 } 802 return next_policy; 803 } 804 805 /** 806 * Callback to check MTR policy action validate 807 * 808 * @param[in] dev 809 * Pointer to Ethernet device. 810 * @param[in] actions 811 * Pointer to meter policy action detail. 812 * @param[out] error 813 * Pointer to the error structure. 814 * 815 * @return 816 * 0 on success, a negative errno value otherwise and rte_errno is set. 817 */ 818 static int 819 mlx5_flow_meter_policy_validate(struct rte_eth_dev *dev, 820 struct rte_mtr_meter_policy_params *policy, 821 struct rte_mtr_error *error) 822 { 823 struct mlx5_priv *priv = dev->data->dev_private; 824 struct rte_flow_attr attr = { .transfer = priv->sh->config.dv_esw_en ? 825 1 : 0 }; 826 bool is_rss = false; 827 uint8_t policy_mode; 828 uint8_t domain_bitmap; 829 int ret; 830 831 if (!priv->mtr_en || !priv->sh->meter_aso_en) 832 return -rte_mtr_error_set(error, ENOTSUP, 833 RTE_MTR_ERROR_TYPE_METER_POLICY, 834 NULL, "meter policy unsupported."); 835 ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr, 836 &is_rss, &domain_bitmap, &policy_mode, error); 837 if (ret) 838 return ret; 839 return 0; 840 } 841 842 /** 843 * Callback to check MTR policy action validate for HWS 844 * 845 * @param[in] dev 846 * Pointer to Ethernet device. 847 * @param[in] actions 848 * Pointer to meter policy action detail. 849 * @param[out] error 850 * Pointer to the error structure. 851 * 852 * @return 853 * 0 on success, a negative errno value otherwise and rte_errno is set. 854 */ 855 static int 856 mlx5_flow_meter_policy_hws_validate(struct rte_eth_dev *dev, 857 struct rte_mtr_meter_policy_params *policy, 858 struct rte_mtr_error *error) 859 { 860 struct mlx5_priv *priv = dev->data->dev_private; 861 const struct rte_flow_actions_template_attr attr = { 862 .transfer = priv->sh->config.dv_esw_en ? 1 : 0 }; 863 int ret; 864 int i; 865 866 if (!priv->mtr_en || !priv->sh->meter_aso_en) 867 return -rte_mtr_error_set(error, ENOTSUP, 868 RTE_MTR_ERROR_TYPE_METER_POLICY, 869 NULL, "meter policy unsupported."); 870 for (i = 0; i < RTE_COLORS; i++) { 871 ret = mlx5_flow_actions_validate(dev, &attr, policy->actions[i], 872 policy->actions[i], NULL); 873 if (ret) 874 return ret; 875 } 876 return 0; 877 } 878 879 static int 880 __mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev, 881 uint32_t policy_id, 882 struct mlx5_flow_meter_policy *mtr_policy, 883 struct rte_mtr_error *error, 884 bool clear_l3t) 885 { 886 struct mlx5_priv *priv = dev->data->dev_private; 887 struct mlx5_flow_meter_sub_policy *sub_policy; 888 uint32_t i, j; 889 uint16_t sub_policy_num; 890 891 rte_spinlock_lock(&mtr_policy->sl); 892 if (mtr_policy->ref_cnt) { 893 rte_spinlock_unlock(&mtr_policy->sl); 894 return -rte_mtr_error_set(error, EBUSY, 895 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 896 NULL, 897 "Meter policy object is being used."); 898 } 899 mlx5_flow_destroy_policy_rules(dev, mtr_policy); 900 mlx5_flow_destroy_mtr_acts(dev, mtr_policy); 901 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 902 sub_policy_num = (mtr_policy->sub_policy_num >> 903 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) & 904 MLX5_MTR_SUB_POLICY_NUM_MASK; 905 if (sub_policy_num) { 906 for (j = 0; j < sub_policy_num; j++) { 907 sub_policy = mtr_policy->sub_policys[i][j]; 908 if (sub_policy) 909 mlx5_ipool_free 910 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 911 sub_policy->idx); 912 } 913 } 914 } 915 if (priv->policy_idx_tbl && clear_l3t) { 916 if (mlx5_l3t_clear_entry(priv->policy_idx_tbl, policy_id)) { 917 rte_spinlock_unlock(&mtr_policy->sl); 918 return -rte_mtr_error_set(error, ENOTSUP, 919 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL, 920 "Fail to delete policy in index table."); 921 } 922 } 923 rte_spinlock_unlock(&mtr_policy->sl); 924 return 0; 925 } 926 927 /** 928 * Callback to add MTR policy. 929 * 930 * @param[in] dev 931 * Pointer to Ethernet device. 932 * @param[out] policy_id 933 * Pointer to policy id 934 * @param[in] actions 935 * Pointer to meter policy action detail. 936 * @param[out] error 937 * Pointer to the error structure. 938 * 939 * @return 940 * 0 on success, a negative errno value otherwise and rte_errno is set. 941 */ 942 static int 943 mlx5_flow_meter_policy_add(struct rte_eth_dev *dev, 944 uint32_t policy_id, 945 struct rte_mtr_meter_policy_params *policy, 946 struct rte_mtr_error *error) 947 { 948 struct mlx5_priv *priv = dev->data->dev_private; 949 struct rte_flow_attr attr = { .transfer = priv->sh->config.dv_esw_en ? 950 1 : 0 }; 951 uint32_t sub_policy_idx = 0; 952 uint32_t policy_idx = 0; 953 struct mlx5_flow_meter_policy *mtr_policy = NULL; 954 struct mlx5_flow_meter_sub_policy *sub_policy; 955 bool is_rss = false; 956 uint8_t policy_mode; 957 uint32_t i; 958 int ret; 959 uint32_t policy_size = sizeof(struct mlx5_flow_meter_policy); 960 uint16_t sub_policy_num; 961 uint8_t domain_bitmap = 0; 962 union mlx5_l3t_data data; 963 bool skip_rule = false; 964 965 if (!priv->mtr_en) 966 return -rte_mtr_error_set(error, ENOTSUP, 967 RTE_MTR_ERROR_TYPE_METER_POLICY, 968 NULL, "meter policy unsupported. "); 969 if (policy_id == MLX5_INVALID_POLICY_ID) 970 return -rte_mtr_error_set(error, ENOTSUP, 971 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 972 NULL, "policy ID is invalid. "); 973 if (policy_id == priv->sh->mtrmng->def_policy_id) 974 return -rte_mtr_error_set(error, EEXIST, 975 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 976 NULL, "default policy ID exists. "); 977 mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx); 978 if (mtr_policy) 979 return -rte_mtr_error_set(error, EEXIST, 980 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 981 NULL, "policy ID exists. "); 982 ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr, 983 &is_rss, &domain_bitmap, 984 &policy_mode, error); 985 if (ret) 986 return ret; 987 if (!domain_bitmap) 988 return -rte_mtr_error_set(error, ENOTSUP, 989 RTE_MTR_ERROR_TYPE_METER_POLICY, 990 NULL, "fail to find policy domain."); 991 if (policy_mode == MLX5_MTR_POLICY_MODE_DEF) { 992 if (priv->sh->mtrmng->def_policy_id != MLX5_INVALID_POLICY_ID) 993 return -rte_mtr_error_set(error, EEXIST, 994 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 995 NULL, "a policy with similar actions " 996 "is already configured"); 997 if (mlx5_flow_create_def_policy(dev)) 998 return -rte_mtr_error_set(error, ENOTSUP, 999 RTE_MTR_ERROR_TYPE_METER_POLICY, 1000 NULL, 1001 "fail to create non-terminated policy."); 1002 priv->sh->mtrmng->def_policy_id = policy_id; 1003 return 0; 1004 } 1005 if (!priv->sh->meter_aso_en) 1006 return -rte_mtr_error_set(error, ENOTSUP, 1007 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL, 1008 "no ASO capability to support the policy "); 1009 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 1010 if (!(domain_bitmap & (1 << i))) 1011 continue; 1012 /* 1013 * If RSS is found, it means that only the ingress domain can 1014 * be supported. It is invalid to support RSS for one color 1015 * and egress / transfer domain actions for another. Drop and 1016 * jump action should have no impact. 1017 */ 1018 if (is_rss) { 1019 policy_size += 1020 sizeof(struct mlx5_flow_meter_sub_policy *) * 1021 MLX5_MTR_RSS_MAX_SUB_POLICY; 1022 break; 1023 } 1024 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *); 1025 } 1026 mtr_policy = mlx5_malloc(MLX5_MEM_ZERO, policy_size, 1027 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 1028 if (!mtr_policy) 1029 return -rte_mtr_error_set(error, ENOMEM, 1030 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL, 1031 "Memory alloc failed for meter policy."); 1032 if (policy_mode == MLX5_MTR_POLICY_MODE_OG) 1033 mtr_policy->skip_y = 1; 1034 else if (policy_mode == MLX5_MTR_POLICY_MODE_OY) 1035 mtr_policy->skip_g = 1; 1036 policy_size = sizeof(struct mlx5_flow_meter_policy); 1037 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 1038 if (!(domain_bitmap & (1 << i))) 1039 continue; 1040 if (i == MLX5_MTR_DOMAIN_INGRESS) 1041 mtr_policy->ingress = 1; 1042 if (i == MLX5_MTR_DOMAIN_EGRESS) 1043 mtr_policy->egress = 1; 1044 if (i == MLX5_MTR_DOMAIN_TRANSFER) 1045 mtr_policy->transfer = 1; 1046 sub_policy = mlx5_ipool_zmalloc 1047 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 1048 &sub_policy_idx); 1049 if (!sub_policy || 1050 sub_policy_idx > MLX5_MAX_SUB_POLICY_TBL_NUM) 1051 goto policy_add_err; 1052 sub_policy->idx = sub_policy_idx; 1053 sub_policy->main_policy = mtr_policy; 1054 if (!policy_idx) { 1055 policy_idx = sub_policy_idx; 1056 sub_policy->main_policy_id = 1; 1057 } 1058 mtr_policy->sub_policys[i] = 1059 (struct mlx5_flow_meter_sub_policy **) 1060 ((uint8_t *)mtr_policy + policy_size); 1061 mtr_policy->sub_policys[i][0] = sub_policy; 1062 sub_policy_num = (mtr_policy->sub_policy_num >> 1063 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) & 1064 MLX5_MTR_SUB_POLICY_NUM_MASK; 1065 sub_policy_num++; 1066 mtr_policy->sub_policy_num &= ~(MLX5_MTR_SUB_POLICY_NUM_MASK << 1067 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)); 1068 mtr_policy->sub_policy_num |= 1069 (sub_policy_num & MLX5_MTR_SUB_POLICY_NUM_MASK) << 1070 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i); 1071 /* 1072 * If RSS is found, it means that only the ingress domain can 1073 * be supported. It is invalid to support RSS for one color 1074 * and egress / transfer domain actions for another. Drop and 1075 * jump action should have no impact. 1076 */ 1077 if (is_rss) { 1078 mtr_policy->is_rss = 1; 1079 break; 1080 } 1081 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *); 1082 } 1083 rte_spinlock_init(&mtr_policy->sl); 1084 ret = mlx5_flow_create_mtr_acts(dev, mtr_policy, 1085 policy->actions, &attr, error); 1086 if (ret) 1087 goto policy_add_err; 1088 if (mtr_policy->is_hierarchy) { 1089 struct mlx5_flow_meter_policy *final_policy; 1090 1091 final_policy = 1092 mlx5_flow_meter_hierarchy_get_final_policy(dev, mtr_policy); 1093 if (!final_policy) 1094 goto policy_add_err; 1095 skip_rule = (final_policy->is_rss || final_policy->is_queue); 1096 } 1097 /* 1098 * If either Green or Yellow has queue / RSS action, all the policy 1099 * rules will be created later in the flow splitting stage. 1100 */ 1101 if (!is_rss && !mtr_policy->is_queue && !skip_rule) { 1102 /* Create policy rules in HW. */ 1103 ret = mlx5_flow_create_policy_rules(dev, mtr_policy); 1104 if (ret) 1105 goto policy_add_err; 1106 } 1107 data.dword = policy_idx; 1108 if (!priv->policy_idx_tbl) { 1109 priv->policy_idx_tbl = mlx5_l3t_create(MLX5_L3T_TYPE_DWORD); 1110 if (!priv->policy_idx_tbl) 1111 goto policy_add_err; 1112 } 1113 if (mlx5_l3t_set_entry(priv->policy_idx_tbl, policy_id, &data)) 1114 goto policy_add_err; 1115 return 0; 1116 policy_add_err: 1117 if (mtr_policy) { 1118 ret = __mlx5_flow_meter_policy_delete(dev, policy_id, 1119 mtr_policy, error, false); 1120 mlx5_free(mtr_policy); 1121 if (ret) 1122 return ret; 1123 } 1124 return -rte_mtr_error_set(error, ENOTSUP, 1125 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 1126 NULL, "Failed to create devx policy."); 1127 } 1128 1129 /** 1130 * Callback to delete MTR policy. 1131 * 1132 * @param[in] dev 1133 * Pointer to Ethernet device. 1134 * @param[in] policy_id 1135 * Meter policy id. 1136 * @param[out] error 1137 * Pointer to the error structure. 1138 * 1139 * @return 1140 * 0 on success, a negative errno value otherwise and rte_errno is set. 1141 */ 1142 static int 1143 mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev, 1144 uint32_t policy_id, 1145 struct rte_mtr_error *error) 1146 { 1147 struct mlx5_priv *priv = dev->data->dev_private; 1148 struct mlx5_flow_meter_policy *mtr_policy; 1149 uint32_t policy_idx; 1150 int ret; 1151 1152 if (policy_id == priv->sh->mtrmng->def_policy_id) { 1153 if (priv->sh->mtrmng->def_policy_ref_cnt > 0) 1154 return -rte_mtr_error_set(error, ENOTSUP, 1155 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL, 1156 "Meter policy object is being used."); 1157 priv->sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID; 1158 return 0; 1159 } 1160 mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx); 1161 if (!mtr_policy) 1162 return -rte_mtr_error_set(error, ENOTSUP, 1163 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL, 1164 "Meter policy id is invalid. "); 1165 ret = __mlx5_flow_meter_policy_delete(dev, policy_id, mtr_policy, 1166 error, true); 1167 if (ret) 1168 return ret; 1169 mlx5_free(mtr_policy); 1170 return 0; 1171 } 1172 1173 /** 1174 * Callback to get MTR policy. 1175 * 1176 * @param[in] dev 1177 * Pointer to Ethernet device. 1178 * @param[in] policy_id 1179 * Meter policy id. 1180 * @param[out] error 1181 * Pointer to the error structure. 1182 * 1183 * @return 1184 * A valid handle in case of success, NULL otherwise. 1185 */ 1186 static struct rte_flow_meter_policy * 1187 mlx5_flow_meter_policy_get(struct rte_eth_dev *dev, 1188 uint32_t policy_id, 1189 struct rte_mtr_error *error) 1190 { 1191 struct mlx5_priv *priv = dev->data->dev_private; 1192 uint32_t policy_idx; 1193 1194 if (!priv->mtr_en) { 1195 rte_mtr_error_set(error, ENOTSUP, 1196 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1197 "Meter is not supported"); 1198 return NULL; 1199 } 1200 return (void *)(uintptr_t)mlx5_flow_meter_policy_find(dev, policy_id, 1201 &policy_idx); 1202 } 1203 1204 /** 1205 * Callback to delete MTR policy for HWS. 1206 * 1207 * @param[in] dev 1208 * Pointer to Ethernet device. 1209 * @param[in] policy_id 1210 * Meter policy id. 1211 * @param[out] error 1212 * Pointer to the error structure. 1213 * 1214 * @return 1215 * 0 on success, a negative errno value otherwise and rte_errno is set. 1216 */ 1217 static int 1218 mlx5_flow_meter_policy_hws_delete(struct rte_eth_dev *dev, 1219 uint32_t policy_id, 1220 struct rte_mtr_error *error) 1221 { 1222 struct mlx5_priv *priv = dev->data->dev_private; 1223 struct mlx5_flow_meter_policy *mtr_policy; 1224 uint32_t i, j; 1225 uint32_t nb_flows = 0; 1226 int ret; 1227 struct rte_flow_op_attr op_attr = { .postpone = 1 }; 1228 struct rte_flow_op_result result[RTE_COLORS * MLX5_MTR_DOMAIN_MAX]; 1229 1230 if (!priv->mtr_policy_arr) 1231 return mlx5_flow_meter_policy_delete(dev, policy_id, error); 1232 /* Meter policy must exist. */ 1233 mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, NULL); 1234 if (!mtr_policy->initialized) 1235 return -rte_mtr_error_set(error, ENOENT, 1236 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL, 1237 "Meter policy does not exists."); 1238 /* Check policy is unused. */ 1239 if (mtr_policy->ref_cnt) 1240 return -rte_mtr_error_set(error, EBUSY, 1241 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1242 NULL, "Meter policy is in use."); 1243 rte_spinlock_lock(&priv->hw_ctrl_lock); 1244 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 1245 for (j = 0; j < RTE_COLORS; j++) { 1246 if (mtr_policy->hws_flow_rule[i][j]) { 1247 ret = rte_flow_async_destroy(dev->data->port_id, 1248 CTRL_QUEUE_ID(priv), &op_attr, 1249 mtr_policy->hws_flow_rule[i][j], 1250 NULL, NULL); 1251 if (ret < 0) 1252 continue; 1253 nb_flows++; 1254 } 1255 } 1256 } 1257 ret = rte_flow_push(dev->data->port_id, CTRL_QUEUE_ID(priv), NULL); 1258 while (nb_flows && (ret >= 0)) { 1259 ret = rte_flow_pull(dev->data->port_id, 1260 CTRL_QUEUE_ID(priv), result, 1261 nb_flows, NULL); 1262 nb_flows -= ret; 1263 } 1264 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 1265 if (mtr_policy->hws_flow_table[i]) 1266 rte_flow_template_table_destroy(dev->data->port_id, 1267 mtr_policy->hws_flow_table[i], NULL); 1268 } 1269 for (i = 0; i < RTE_COLORS; i++) { 1270 if (mtr_policy->hws_act_templ[i]) 1271 rte_flow_actions_template_destroy(dev->data->port_id, 1272 mtr_policy->hws_act_templ[i], NULL); 1273 } 1274 if (mtr_policy->hws_item_templ) 1275 rte_flow_pattern_template_destroy(dev->data->port_id, 1276 mtr_policy->hws_item_templ, NULL); 1277 rte_spinlock_unlock(&priv->hw_ctrl_lock); 1278 memset(mtr_policy, 0, sizeof(struct mlx5_flow_meter_policy)); 1279 return 0; 1280 } 1281 1282 /** 1283 * Callback to add MTR policy for HWS. 1284 * 1285 * @param[in] dev 1286 * Pointer to Ethernet device. 1287 * @param[out] policy_id 1288 * Pointer to policy id 1289 * @param[in] actions 1290 * Pointer to meter policy action detail. 1291 * @param[out] error 1292 * Pointer to the error structure. 1293 * 1294 * @return 1295 * 0 on success, a negative errno value otherwise and rte_errno is set. 1296 */ 1297 static int 1298 mlx5_flow_meter_policy_hws_add(struct rte_eth_dev *dev, 1299 uint32_t policy_id, 1300 struct rte_mtr_meter_policy_params *policy, 1301 struct rte_mtr_error *error) 1302 { 1303 struct mlx5_priv *priv = dev->data->dev_private; 1304 struct mlx5_flow_meter_policy *mtr_policy = NULL; 1305 const struct rte_flow_action *act; 1306 const struct rte_flow_action_meter *mtr; 1307 struct mlx5_flow_meter_info *fm; 1308 struct mlx5_flow_meter_policy *plc; 1309 uint8_t domain_color = MLX5_MTR_ALL_DOMAIN_BIT; 1310 bool is_rss = false; 1311 bool is_hierarchy = false; 1312 int i, j; 1313 uint32_t nb_colors = 0; 1314 uint32_t nb_flows = 0; 1315 int color; 1316 int ret; 1317 struct rte_flow_pattern_template_attr pta = {0}; 1318 struct rte_flow_actions_template_attr ata = {0}; 1319 struct rte_flow_template_table_attr ta = { {0}, 0 }; 1320 struct rte_flow_op_attr op_attr = { .postpone = 1 }; 1321 struct rte_flow_op_result result[RTE_COLORS * MLX5_MTR_DOMAIN_MAX]; 1322 const uint32_t color_mask = (UINT32_C(1) << MLX5_MTR_COLOR_BITS) - 1; 1323 int color_reg_c_idx = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 1324 0, NULL); 1325 struct rte_flow_item_tag tag_spec = { 1326 .data = 0, 1327 .index = color_reg_c_idx 1328 }; 1329 struct rte_flow_item_tag tag_mask = { 1330 .data = color_mask, 1331 .index = 0xff}; 1332 struct rte_flow_item pattern[] = { 1333 [0] = { 1334 .type = (enum rte_flow_item_type) 1335 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 1336 .spec = &tag_spec, 1337 .mask = &tag_mask, 1338 }, 1339 [1] = { .type = RTE_FLOW_ITEM_TYPE_END } 1340 }; 1341 1342 if (!priv->mtr_policy_arr) 1343 return mlx5_flow_meter_policy_add(dev, policy_id, policy, error); 1344 mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, NULL); 1345 if (mtr_policy->initialized) 1346 return -rte_mtr_error_set(error, EEXIST, 1347 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1348 NULL, "Meter policy already exists."); 1349 if (!policy || 1350 (!policy->actions[RTE_COLOR_RED] && 1351 !policy->actions[RTE_COLOR_YELLOW] && 1352 !policy->actions[RTE_COLOR_GREEN])) 1353 return -rte_mtr_error_set(error, EINVAL, 1354 RTE_MTR_ERROR_TYPE_METER_POLICY, 1355 NULL, "Meter policy actions are not valid."); 1356 if (policy->actions[RTE_COLOR_RED] == RTE_FLOW_ACTION_TYPE_END) 1357 mtr_policy->skip_r = 1; 1358 if (policy->actions[RTE_COLOR_YELLOW] == RTE_FLOW_ACTION_TYPE_END) 1359 mtr_policy->skip_y = 1; 1360 if (policy->actions[RTE_COLOR_GREEN] == RTE_FLOW_ACTION_TYPE_END) 1361 mtr_policy->skip_g = 1; 1362 if (mtr_policy->skip_r && mtr_policy->skip_y && mtr_policy->skip_g) 1363 return -rte_mtr_error_set(error, ENOTSUP, 1364 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1365 NULL, "Meter policy actions are empty."); 1366 for (i = 0; i < RTE_COLORS; i++) { 1367 act = policy->actions[i]; 1368 while (act && act->type != RTE_FLOW_ACTION_TYPE_END) { 1369 switch (act->type) { 1370 case RTE_FLOW_ACTION_TYPE_PORT_ID: 1371 /* fall-through. */ 1372 case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT: 1373 domain_color &= ~(MLX5_MTR_DOMAIN_INGRESS_BIT | 1374 MLX5_MTR_DOMAIN_EGRESS_BIT); 1375 break; 1376 case RTE_FLOW_ACTION_TYPE_RSS: 1377 is_rss = true; 1378 /* fall-through. */ 1379 case RTE_FLOW_ACTION_TYPE_QUEUE: 1380 domain_color &= ~(MLX5_MTR_DOMAIN_EGRESS_BIT | 1381 MLX5_MTR_DOMAIN_TRANSFER_BIT); 1382 break; 1383 case RTE_FLOW_ACTION_TYPE_METER: 1384 is_hierarchy = true; 1385 mtr = act->conf; 1386 fm = mlx5_flow_meter_find(priv, 1387 mtr->mtr_id, NULL); 1388 if (!fm) 1389 return -rte_mtr_error_set(error, EINVAL, 1390 RTE_MTR_ERROR_TYPE_MTR_ID, NULL, 1391 "Meter not found in meter hierarchy."); 1392 plc = mlx5_flow_meter_policy_find(dev, 1393 fm->policy_id, 1394 NULL); 1395 MLX5_ASSERT(plc); 1396 domain_color &= MLX5_MTR_ALL_DOMAIN_BIT & 1397 (plc->ingress << 1398 MLX5_MTR_DOMAIN_INGRESS); 1399 domain_color &= MLX5_MTR_ALL_DOMAIN_BIT & 1400 (plc->egress << 1401 MLX5_MTR_DOMAIN_EGRESS); 1402 domain_color &= MLX5_MTR_ALL_DOMAIN_BIT & 1403 (plc->transfer << 1404 MLX5_MTR_DOMAIN_TRANSFER); 1405 break; 1406 default: 1407 break; 1408 } 1409 act++; 1410 } 1411 } 1412 if (priv->sh->config.dv_esw_en) 1413 domain_color &= ~(MLX5_MTR_DOMAIN_EGRESS_BIT | 1414 MLX5_MTR_DOMAIN_TRANSFER_BIT); 1415 else 1416 domain_color &= ~MLX5_MTR_DOMAIN_TRANSFER_BIT; 1417 if (!domain_color) 1418 return -rte_mtr_error_set(error, ENOTSUP, 1419 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1420 NULL, "Meter policy domains are conflicting."); 1421 mtr_policy->is_rss = is_rss; 1422 mtr_policy->ingress = !!(domain_color & MLX5_MTR_DOMAIN_INGRESS_BIT); 1423 pta.ingress = mtr_policy->ingress; 1424 mtr_policy->egress = !!(domain_color & MLX5_MTR_DOMAIN_EGRESS_BIT); 1425 pta.egress = mtr_policy->egress; 1426 mtr_policy->transfer = !!(domain_color & MLX5_MTR_DOMAIN_TRANSFER_BIT); 1427 pta.transfer = mtr_policy->transfer; 1428 mtr_policy->group = MLX5_FLOW_TABLE_HWS_POLICY - policy_id; 1429 mtr_policy->is_hierarchy = is_hierarchy; 1430 mtr_policy->initialized = 1; 1431 rte_spinlock_lock(&priv->hw_ctrl_lock); 1432 mtr_policy->hws_item_templ = 1433 rte_flow_pattern_template_create(dev->data->port_id, 1434 &pta, pattern, NULL); 1435 if (!mtr_policy->hws_item_templ) 1436 goto policy_add_err; 1437 for (i = 0; i < RTE_COLORS; i++) { 1438 if (mtr_policy->skip_g && i == RTE_COLOR_GREEN) 1439 continue; 1440 if (mtr_policy->skip_y && i == RTE_COLOR_YELLOW) 1441 continue; 1442 if (mtr_policy->skip_r && i == RTE_COLOR_RED) 1443 continue; 1444 mtr_policy->hws_act_templ[nb_colors] = 1445 rte_flow_actions_template_create(dev->data->port_id, 1446 &ata, policy->actions[i], 1447 policy->actions[i], NULL); 1448 if (!mtr_policy->hws_act_templ[nb_colors]) 1449 goto policy_add_err; 1450 nb_colors++; 1451 } 1452 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 1453 memset(&ta, 0, sizeof(ta)); 1454 ta.nb_flows = RTE_COLORS; 1455 ta.flow_attr.group = mtr_policy->group; 1456 if (i == MLX5_MTR_DOMAIN_INGRESS) { 1457 if (!mtr_policy->ingress) 1458 continue; 1459 ta.flow_attr.ingress = 1; 1460 } else if (i == MLX5_MTR_DOMAIN_EGRESS) { 1461 if (!mtr_policy->egress) 1462 continue; 1463 ta.flow_attr.egress = 1; 1464 } else if (i == MLX5_MTR_DOMAIN_TRANSFER) { 1465 if (!mtr_policy->transfer) 1466 continue; 1467 ta.flow_attr.transfer = 1; 1468 } 1469 mtr_policy->hws_flow_table[i] = 1470 rte_flow_template_table_create(dev->data->port_id, 1471 &ta, &mtr_policy->hws_item_templ, 1, 1472 mtr_policy->hws_act_templ, nb_colors, 1473 NULL); 1474 if (!mtr_policy->hws_flow_table[i]) 1475 goto policy_add_err; 1476 nb_colors = 0; 1477 for (j = 0; j < RTE_COLORS; j++) { 1478 if (mtr_policy->skip_g && j == RTE_COLOR_GREEN) 1479 continue; 1480 if (mtr_policy->skip_y && j == RTE_COLOR_YELLOW) 1481 continue; 1482 if (mtr_policy->skip_r && j == RTE_COLOR_RED) 1483 continue; 1484 color = rte_col_2_mlx5_col((enum rte_color)j); 1485 tag_spec.data = color; 1486 mtr_policy->hws_flow_rule[i][j] = 1487 rte_flow_async_create(dev->data->port_id, 1488 CTRL_QUEUE_ID(priv), &op_attr, 1489 mtr_policy->hws_flow_table[i], 1490 pattern, 0, policy->actions[j], 1491 nb_colors, NULL, NULL); 1492 if (!mtr_policy->hws_flow_rule[i][j]) 1493 goto policy_add_err; 1494 nb_colors++; 1495 nb_flows++; 1496 } 1497 ret = rte_flow_push(dev->data->port_id, 1498 CTRL_QUEUE_ID(priv), NULL); 1499 if (ret < 0) 1500 goto policy_add_err; 1501 while (nb_flows) { 1502 ret = rte_flow_pull(dev->data->port_id, 1503 CTRL_QUEUE_ID(priv), result, 1504 nb_flows, NULL); 1505 if (ret < 0) 1506 goto policy_add_err; 1507 for (j = 0; j < ret; j++) { 1508 if (result[j].status == RTE_FLOW_OP_ERROR) 1509 goto policy_add_err; 1510 } 1511 nb_flows -= ret; 1512 } 1513 } 1514 rte_spinlock_unlock(&priv->hw_ctrl_lock); 1515 return 0; 1516 policy_add_err: 1517 rte_spinlock_unlock(&priv->hw_ctrl_lock); 1518 ret = mlx5_flow_meter_policy_hws_delete(dev, policy_id, error); 1519 memset(mtr_policy, 0, sizeof(struct mlx5_flow_meter_policy)); 1520 if (ret) 1521 return ret; 1522 return -rte_mtr_error_set(error, ENOTSUP, 1523 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 1524 NULL, "Failed to create meter policy."); 1525 } 1526 1527 /** 1528 * Check meter validation. 1529 * 1530 * @param[in] priv 1531 * Pointer to mlx5 private data structure. 1532 * @param[in] meter_id 1533 * Meter id. 1534 * @param[in] params 1535 * Pointer to rte meter parameters. 1536 * @param[out] error 1537 * Pointer to rte meter error structure. 1538 * 1539 * @return 1540 * 0 on success, a negative errno value otherwise and rte_errno is set. 1541 */ 1542 static int 1543 mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id, 1544 struct rte_mtr_params *params, 1545 struct rte_mtr_error *error) 1546 { 1547 /* Meter must use global drop action. */ 1548 if (!priv->sh->dr_drop_action) 1549 return -rte_mtr_error_set(error, ENOTSUP, 1550 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 1551 NULL, 1552 "No drop action ready for meter."); 1553 /* Meter params must not be NULL. */ 1554 if (params == NULL) 1555 return -rte_mtr_error_set(error, EINVAL, 1556 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 1557 NULL, "Meter object params null."); 1558 /* Previous meter color is not supported. */ 1559 if (params->use_prev_mtr_color && !priv->sh->meter_aso_en) 1560 return -rte_mtr_error_set(error, ENOTSUP, 1561 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 1562 NULL, 1563 "Previous meter color " 1564 "not supported."); 1565 if (params->meter_policy_id == MLX5_INVALID_POLICY_ID) 1566 return -rte_mtr_error_set(error, ENOENT, 1567 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1568 NULL, "Meter policy id not valid."); 1569 /* Validate meter id. */ 1570 if (mlx5_flow_meter_find(priv, meter_id, NULL)) 1571 return -rte_mtr_error_set(error, EEXIST, 1572 RTE_MTR_ERROR_TYPE_MTR_ID, NULL, 1573 "Meter object already exists."); 1574 return 0; 1575 } 1576 1577 /** 1578 * Modify the flow meter action. 1579 * 1580 * @param[in] priv 1581 * Pointer to mlx5 private data structure. 1582 * @param[in] fm 1583 * Pointer to flow meter to be modified. 1584 * @param[in] srtcm 1585 * Pointer to meter srtcm description parameter. 1586 * @param[in] modify_bits 1587 * The bit in srtcm to be updated. 1588 * @param[in] active_state 1589 * The state to be updated. 1590 * @return 1591 * 0 on success, o negative value otherwise. 1592 */ 1593 static int 1594 mlx5_flow_meter_action_modify(struct mlx5_priv *priv, 1595 struct mlx5_flow_meter_info *fm, 1596 const struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm, 1597 uint64_t modify_bits, uint32_t active_state, uint32_t is_enable) 1598 { 1599 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER 1600 struct mlx5_dev_ctx_shared *sh = priv->sh; 1601 uint32_t in[MLX5_ST_SZ_DW(flow_meter_parameters)] = { 0 }; 1602 uint32_t *attr; 1603 struct mlx5dv_dr_flow_meter_attr mod_attr = { 0 }; 1604 int ret; 1605 struct mlx5_aso_mtr *aso_mtr = NULL; 1606 uint32_t cbs_cir, ebs_eir, val; 1607 1608 if (sh->meter_aso_en) { 1609 fm->is_enable = !!is_enable; 1610 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm); 1611 ret = mlx5_aso_meter_update_by_wqe(sh, MLX5_HW_INV_QUEUE, 1612 aso_mtr, &priv->mtr_bulk, 1613 NULL, true); 1614 if (ret) 1615 return ret; 1616 ret = mlx5_aso_mtr_wait(sh, MLX5_HW_INV_QUEUE, aso_mtr); 1617 if (ret) 1618 return ret; 1619 } else { 1620 /* Fill command parameters. */ 1621 mod_attr.reg_c_index = sh->registers.aso_reg - REG_C_0; 1622 mod_attr.flow_meter_parameter = in; 1623 mod_attr.flow_meter_parameter_sz = 1624 MLX5_ST_SZ_BYTES(flow_meter_parameters); 1625 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE) 1626 mod_attr.active = !!active_state; 1627 else 1628 mod_attr.active = 0; 1629 attr = in; 1630 cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir); 1631 ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir); 1632 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS) { 1633 val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & 1634 ASO_DSEG_EXP_MASK; 1635 MLX5_SET(flow_meter_parameters, attr, 1636 cbs_exponent, val); 1637 val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & 1638 ASO_DSEG_MAN_MASK; 1639 MLX5_SET(flow_meter_parameters, attr, 1640 cbs_mantissa, val); 1641 } 1642 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR) { 1643 val = (cbs_cir >> ASO_DSEG_XIR_EXP_OFFSET) & 1644 ASO_DSEG_EXP_MASK; 1645 MLX5_SET(flow_meter_parameters, attr, 1646 cir_exponent, val); 1647 val = cbs_cir & ASO_DSEG_MAN_MASK; 1648 MLX5_SET(flow_meter_parameters, attr, 1649 cir_mantissa, val); 1650 } 1651 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_EBS) { 1652 val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & 1653 ASO_DSEG_EXP_MASK; 1654 MLX5_SET(flow_meter_parameters, attr, 1655 ebs_exponent, val); 1656 val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & 1657 ASO_DSEG_MAN_MASK; 1658 MLX5_SET(flow_meter_parameters, attr, 1659 ebs_mantissa, val); 1660 } 1661 /* Apply modifications to meter only if it was created. */ 1662 if (fm->meter_action_g) { 1663 ret = mlx5_glue->dv_modify_flow_action_meter 1664 (fm->meter_action_g, &mod_attr, 1665 rte_cpu_to_be_64(modify_bits)); 1666 if (ret) 1667 return ret; 1668 } 1669 /* Update succeeded modify meter parameters. */ 1670 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE) 1671 fm->active_state = !!active_state; 1672 } 1673 return 0; 1674 #else 1675 (void)priv; 1676 (void)fm; 1677 (void)srtcm; 1678 (void)modify_bits; 1679 (void)active_state; 1680 (void)is_enable; 1681 return -ENOTSUP; 1682 #endif 1683 } 1684 1685 static int 1686 mlx5_flow_meter_stats_enable_update(struct rte_eth_dev *dev, 1687 struct mlx5_flow_meter_info *fm, 1688 uint64_t stats_mask) 1689 { 1690 fm->bytes_dropped = 1691 (stats_mask & RTE_MTR_STATS_N_BYTES_DROPPED) ? 1 : 0; 1692 fm->pkts_dropped = (stats_mask & RTE_MTR_STATS_N_PKTS_DROPPED) ? 1 : 0; 1693 if (fm->bytes_dropped || fm->pkts_dropped) { 1694 if (!fm->drop_cnt) { 1695 /* Alloc policer counters. */ 1696 fm->drop_cnt = mlx5_counter_alloc(dev); 1697 if (!fm->drop_cnt) 1698 return -1; 1699 } 1700 } else { 1701 if (fm->drop_cnt) { 1702 mlx5_counter_free(dev, fm->drop_cnt); 1703 fm->drop_cnt = 0; 1704 } 1705 } 1706 return 0; 1707 } 1708 1709 /** 1710 * Create meter rules. 1711 * 1712 * @param[in] dev 1713 * Pointer to Ethernet device. 1714 * @param[in] meter_id 1715 * Meter id. 1716 * @param[in] params 1717 * Pointer to rte meter parameters. 1718 * @param[in] shared 1719 * Meter shared with other flow or not. 1720 * @param[out] error 1721 * Pointer to rte meter error structure. 1722 * 1723 * @return 1724 * 0 on success, a negative errno value otherwise and rte_errno is set. 1725 */ 1726 static int 1727 mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, 1728 struct rte_mtr_params *params, int shared, 1729 struct rte_mtr_error *error) 1730 { 1731 struct mlx5_priv *priv = dev->data->dev_private; 1732 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; 1733 struct mlx5_flow_meter_profile *fmp; 1734 struct mlx5_flow_meter_info *fm; 1735 /* GCC fails to infer legacy_fm is set when !priv->sh->meter_aso_en. */ 1736 struct mlx5_legacy_flow_meter *legacy_fm = NULL; 1737 struct mlx5_flow_meter_policy *mtr_policy = NULL; 1738 struct mlx5_indexed_pool_config flow_ipool_cfg = { 1739 .size = 0, 1740 .trunk_size = 64, 1741 .need_lock = 1, 1742 .type = "mlx5_flow_mtr_flow_id_pool", 1743 }; 1744 struct mlx5_aso_mtr *aso_mtr; 1745 uint32_t mtr_idx, policy_idx; 1746 union mlx5_l3t_data data; 1747 int ret; 1748 uint8_t domain_bitmap; 1749 uint8_t mtr_id_bits; 1750 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 1751 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 1752 1753 if (!priv->mtr_en) 1754 return -rte_mtr_error_set(error, ENOTSUP, 1755 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1756 "Meter is not supported"); 1757 /* Validate the parameters. */ 1758 ret = mlx5_flow_meter_validate(priv, meter_id, params, error); 1759 if (ret) 1760 return ret; 1761 /* Meter profile must exist. */ 1762 fmp = mlx5_flow_meter_profile_find(priv, params->meter_profile_id); 1763 if (fmp == NULL) 1764 return -rte_mtr_error_set(error, ENOENT, 1765 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 1766 NULL, "Meter profile id not valid."); 1767 /* Meter policy must exist. */ 1768 if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) { 1769 __atomic_fetch_add 1770 (&priv->sh->mtrmng->def_policy_ref_cnt, 1771 1, __ATOMIC_RELAXED); 1772 domain_bitmap = MLX5_MTR_ALL_DOMAIN_BIT; 1773 if (!priv->sh->config.dv_esw_en) 1774 domain_bitmap &= ~MLX5_MTR_DOMAIN_TRANSFER_BIT; 1775 } else { 1776 if (!priv->sh->meter_aso_en) 1777 return -rte_mtr_error_set(error, ENOTSUP, 1778 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1779 "Part of the policies cannot be " 1780 "supported without ASO "); 1781 mtr_policy = mlx5_flow_meter_policy_find(dev, 1782 params->meter_policy_id, &policy_idx); 1783 if (!mtr_policy) 1784 return -rte_mtr_error_set(error, ENOENT, 1785 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1786 NULL, "Meter policy id not valid."); 1787 domain_bitmap = (mtr_policy->ingress ? 1788 MLX5_MTR_DOMAIN_INGRESS_BIT : 0) | 1789 (mtr_policy->egress ? 1790 MLX5_MTR_DOMAIN_EGRESS_BIT : 0) | 1791 (mtr_policy->transfer ? 1792 MLX5_MTR_DOMAIN_TRANSFER_BIT : 0); 1793 if (fmp->g_support && mtr_policy->skip_g) 1794 return -rte_mtr_error_set(error, ENOTSUP, 1795 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1796 NULL, "Meter green policy is empty."); 1797 if (fmp->y_support && mtr_policy->skip_y) 1798 return -rte_mtr_error_set(error, ENOTSUP, 1799 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1800 NULL, "Meter yellow policy is empty."); 1801 } 1802 /* Allocate the flow meter memory. */ 1803 if (priv->sh->meter_aso_en) { 1804 mtr_idx = mlx5_flow_mtr_alloc(dev); 1805 if (!mtr_idx) 1806 return -rte_mtr_error_set(error, ENOMEM, 1807 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1808 "Memory alloc failed for meter."); 1809 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx); 1810 fm = &aso_mtr->fm; 1811 } else { 1812 if (fmp->y_support) 1813 return -rte_mtr_error_set(error, ENOMEM, 1814 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1815 "Unsupported profile with yellow."); 1816 legacy_fm = mlx5_ipool_zmalloc 1817 (priv->sh->ipool[MLX5_IPOOL_MTR], &mtr_idx); 1818 if (legacy_fm == NULL) 1819 return -rte_mtr_error_set(error, ENOMEM, 1820 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1821 "Memory alloc failed for meter."); 1822 legacy_fm->idx = mtr_idx; 1823 fm = &legacy_fm->fm; 1824 } 1825 mtr_id_bits = MLX5_REG_BITS - rte_clz32(mtr_idx); 1826 if ((mtr_id_bits + priv->sh->mtrmng->max_mtr_flow_bits) > 1827 mtr_reg_bits) { 1828 DRV_LOG(ERR, "Meter number exceeds max limit."); 1829 goto error; 1830 } 1831 if (mtr_id_bits > priv->sh->mtrmng->max_mtr_bits) 1832 priv->sh->mtrmng->max_mtr_bits = mtr_id_bits; 1833 /* Fill the flow meter parameters. */ 1834 fm->meter_id = meter_id; 1835 fm->policy_id = params->meter_policy_id; 1836 fm->profile = fmp; 1837 if (mlx5_flow_meter_stats_enable_update(dev, fm, params->stats_mask)) 1838 goto error; 1839 if (mlx5_flow_create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap)) 1840 goto error; 1841 /* Add to the flow meter list. */ 1842 if (!priv->sh->meter_aso_en) { 1843 MLX5_ASSERT(legacy_fm != NULL); 1844 TAILQ_INSERT_TAIL(fms, legacy_fm, next); 1845 } 1846 /* Add to the flow meter list. */ 1847 fm->active_state = 1; /* Config meter starts as active. */ 1848 fm->is_enable = params->meter_enable; 1849 fm->shared = !!shared; 1850 fm->color_aware = !!params->use_prev_mtr_color; 1851 __atomic_fetch_add(&fm->profile->ref_cnt, 1, __ATOMIC_RELAXED); 1852 if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) { 1853 fm->def_policy = 1; 1854 fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg); 1855 if (!fm->flow_ipool) 1856 goto error; 1857 } 1858 rte_spinlock_init(&fm->sl); 1859 /* If ASO meter supported, update ASO flow meter by wqe. */ 1860 if (priv->sh->meter_aso_en) { 1861 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm); 1862 ret = mlx5_aso_meter_update_by_wqe(priv->sh, MLX5_HW_INV_QUEUE, 1863 aso_mtr, &priv->mtr_bulk, NULL, true); 1864 if (ret) 1865 goto error; 1866 if (!priv->mtr_idx_tbl) { 1867 priv->mtr_idx_tbl = 1868 mlx5_l3t_create(MLX5_L3T_TYPE_DWORD); 1869 if (!priv->mtr_idx_tbl) 1870 goto error; 1871 } 1872 data.dword = mtr_idx; 1873 if (mlx5_l3t_set_entry(priv->mtr_idx_tbl, meter_id, &data)) 1874 goto error; 1875 } else if (!params->meter_enable && mlx5_flow_meter_disable(dev, meter_id, error)) { 1876 goto error; 1877 } 1878 fm->active_state = params->meter_enable; 1879 if (mtr_policy) 1880 __atomic_fetch_add(&mtr_policy->ref_cnt, 1, __ATOMIC_RELAXED); 1881 return 0; 1882 error: 1883 mlx5_flow_destroy_mtr_tbls(dev, fm); 1884 /* Free policer counters. */ 1885 if (fm->drop_cnt) 1886 mlx5_counter_free(dev, fm->drop_cnt); 1887 if (priv->sh->meter_aso_en) 1888 mlx5_flow_mtr_free(dev, mtr_idx); 1889 else 1890 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], mtr_idx); 1891 return -rte_mtr_error_set(error, ENOTSUP, 1892 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 1893 NULL, "Failed to create devx meter."); 1894 } 1895 1896 /** 1897 * Create meter rules. 1898 * 1899 * @param[in] dev 1900 * Pointer to Ethernet device. 1901 * @param[in] meter_id 1902 * Meter id. 1903 * @param[in] params 1904 * Pointer to rte meter parameters. 1905 * @param[in] shared 1906 * Meter shared with other flow or not. 1907 * @param[out] error 1908 * Pointer to rte meter error structure. 1909 * 1910 * @return 1911 * 0 on success, a negative errno value otherwise and rte_errno is set. 1912 */ 1913 static int 1914 mlx5_flow_meter_hws_create(struct rte_eth_dev *dev, uint32_t meter_id, 1915 struct rte_mtr_params *params, int shared, 1916 struct rte_mtr_error *error) 1917 { 1918 struct mlx5_priv *priv = dev->data->dev_private; 1919 struct mlx5_flow_meter_profile *profile; 1920 struct mlx5_flow_meter_info *fm; 1921 struct mlx5_flow_meter_policy *policy = NULL; 1922 struct mlx5_aso_mtr *aso_mtr; 1923 int ret; 1924 1925 if (!priv->mtr_profile_arr || 1926 !priv->mtr_policy_arr || 1927 !priv->mtr_bulk.aso) 1928 return -rte_mtr_error_set(error, ENOTSUP, 1929 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1930 "Meter bulk array is not allocated."); 1931 /* Meter profile must exist. */ 1932 profile = mlx5_flow_meter_profile_find(priv, params->meter_profile_id); 1933 if (!profile->initialized) 1934 return -rte_mtr_error_set(error, ENOENT, 1935 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 1936 NULL, "Meter profile id not valid."); 1937 /* Meter policy must exist. */ 1938 policy = mlx5_flow_meter_policy_find(dev, 1939 params->meter_policy_id, NULL); 1940 if (!policy->initialized) 1941 return -rte_mtr_error_set(error, ENOENT, 1942 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1943 NULL, "Meter policy id not valid."); 1944 /* Meter ID must be valid. */ 1945 if (meter_id >= priv->mtr_config.nb_meters) 1946 return -rte_mtr_error_set(error, EINVAL, 1947 RTE_MTR_ERROR_TYPE_MTR_ID, 1948 NULL, "Meter id not valid."); 1949 /* Find ASO object. */ 1950 aso_mtr = mlx5_aso_meter_by_idx(priv, meter_id); 1951 fm = &aso_mtr->fm; 1952 if (fm->initialized) 1953 return -rte_mtr_error_set(error, ENOENT, 1954 RTE_MTR_ERROR_TYPE_MTR_ID, 1955 NULL, "Meter object already exists."); 1956 /* Fill the flow meter parameters. */ 1957 fm->meter_id = meter_id; 1958 fm->policy_id = params->meter_policy_id; 1959 fm->profile = profile; 1960 fm->meter_offset = meter_id; 1961 fm->group = policy->group; 1962 /* Add to the flow meter list. */ 1963 fm->active_state = 1; /* Config meter starts as active. */ 1964 fm->is_enable = params->meter_enable; 1965 fm->shared = !!shared; 1966 fm->initialized = 1; 1967 /* Update ASO flow meter by wqe. */ 1968 ret = mlx5_aso_meter_update_by_wqe(priv->sh, MLX5_HW_INV_QUEUE, aso_mtr, 1969 &priv->mtr_bulk, NULL, true); 1970 if (ret) 1971 return -rte_mtr_error_set(error, ENOTSUP, 1972 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 1973 NULL, "Failed to create devx meter."); 1974 fm->active_state = params->meter_enable; 1975 __atomic_fetch_add(&fm->profile->ref_cnt, 1, __ATOMIC_RELAXED); 1976 __atomic_fetch_add(&policy->ref_cnt, 1, __ATOMIC_RELAXED); 1977 return 0; 1978 } 1979 1980 static int 1981 mlx5_flow_meter_params_flush(struct rte_eth_dev *dev, 1982 struct mlx5_flow_meter_info *fm, 1983 uint32_t mtr_idx) 1984 { 1985 struct mlx5_priv *priv = dev->data->dev_private; 1986 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; 1987 struct mlx5_flow_meter_profile *fmp; 1988 struct mlx5_legacy_flow_meter *legacy_fm = NULL; 1989 struct mlx5_flow_meter_policy *mtr_policy; 1990 1991 /* Meter object must not have any owner. */ 1992 MLX5_ASSERT(!fm->ref_cnt); 1993 /* Get meter profile. */ 1994 fmp = fm->profile; 1995 if (fmp == NULL) 1996 return -1; 1997 /* Update dependencies. */ 1998 __atomic_fetch_sub(&fmp->ref_cnt, 1, __ATOMIC_RELAXED); 1999 fm->profile = NULL; 2000 /* Remove from list. */ 2001 if (!priv->sh->meter_aso_en) { 2002 legacy_fm = container_of(fm, 2003 struct mlx5_legacy_flow_meter, fm); 2004 TAILQ_REMOVE(fms, legacy_fm, next); 2005 } 2006 /* Free drop counters. */ 2007 if (fm->drop_cnt) 2008 mlx5_counter_free(dev, fm->drop_cnt); 2009 /* Free meter flow table. */ 2010 if (fm->flow_ipool) { 2011 mlx5_ipool_destroy(fm->flow_ipool); 2012 fm->flow_ipool = 0; 2013 } 2014 mlx5_flow_destroy_mtr_tbls(dev, fm); 2015 if (fm->def_policy) 2016 __atomic_fetch_sub(&priv->sh->mtrmng->def_policy_ref_cnt, 2017 1, __ATOMIC_RELAXED); 2018 if (priv->sh->meter_aso_en) { 2019 if (!fm->def_policy) { 2020 mtr_policy = mlx5_flow_meter_policy_find(dev, 2021 fm->policy_id, NULL); 2022 if (mtr_policy) 2023 __atomic_fetch_sub(&mtr_policy->ref_cnt, 2024 1, __ATOMIC_RELAXED); 2025 fm->policy_id = 0; 2026 } 2027 fm->def_policy = 0; 2028 if (mlx5_l3t_clear_entry(priv->mtr_idx_tbl, fm->meter_id)) 2029 return -1; 2030 mlx5_flow_mtr_free(dev, mtr_idx); 2031 } else { 2032 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], 2033 legacy_fm->idx); 2034 } 2035 return 0; 2036 } 2037 2038 /** 2039 * Destroy meter rules. 2040 * 2041 * @param[in] dev 2042 * Pointer to Ethernet device. 2043 * @param[in] meter_id 2044 * Meter id. 2045 * @param[out] error 2046 * Pointer to rte meter error structure. 2047 * 2048 * @return 2049 * 0 on success, a negative errno value otherwise and rte_errno is set. 2050 */ 2051 static int 2052 mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id, 2053 struct rte_mtr_error *error) 2054 { 2055 struct mlx5_priv *priv = dev->data->dev_private; 2056 struct mlx5_flow_meter_info *fm; 2057 uint32_t mtr_idx = 0; 2058 2059 if (!priv->mtr_en) 2060 return -rte_mtr_error_set(error, ENOTSUP, 2061 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 2062 NULL, 2063 "Meter is not supported"); 2064 /* Meter object must exist. */ 2065 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 2066 if (fm == NULL) 2067 return -rte_mtr_error_set(error, ENOENT, 2068 RTE_MTR_ERROR_TYPE_MTR_ID, 2069 NULL, 2070 "Meter object id not valid."); 2071 /* Meter object must not have any owner. */ 2072 if (fm->ref_cnt > 0) 2073 return -rte_mtr_error_set(error, EBUSY, 2074 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 2075 NULL, 2076 "Meter object is being used."); 2077 /* Destroy the meter profile. */ 2078 if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx)) 2079 return -rte_mtr_error_set(error, EINVAL, 2080 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 2081 NULL, 2082 "MTR object meter profile invalid."); 2083 return 0; 2084 } 2085 2086 /** 2087 * Destroy meter rules. 2088 * 2089 * @param[in] dev 2090 * Pointer to Ethernet device. 2091 * @param[in] meter_id 2092 * Meter id. 2093 * @param[out] error 2094 * Pointer to rte meter error structure. 2095 * 2096 * @return 2097 * 0 on success, a negative errno value otherwise and rte_errno is set. 2098 */ 2099 static int 2100 mlx5_flow_meter_hws_destroy(struct rte_eth_dev *dev, uint32_t meter_id, 2101 struct rte_mtr_error *error) 2102 { 2103 struct mlx5_priv *priv = dev->data->dev_private; 2104 struct mlx5_aso_mtr *aso_mtr; 2105 struct mlx5_flow_meter_info *fm; 2106 struct mlx5_flow_meter_policy *policy; 2107 2108 if (!priv->mtr_profile_arr || 2109 !priv->mtr_policy_arr || 2110 !priv->mtr_bulk.aso) 2111 return -rte_mtr_error_set(error, ENOTSUP, 2112 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL, 2113 "Meter bulk array is not allocated."); 2114 /* Find ASO object. */ 2115 aso_mtr = mlx5_aso_meter_by_idx(priv, meter_id); 2116 fm = &aso_mtr->fm; 2117 if (!fm->initialized) 2118 return -rte_mtr_error_set(error, ENOENT, 2119 RTE_MTR_ERROR_TYPE_MTR_ID, 2120 NULL, "Meter object id not valid."); 2121 /* Meter object must not have any owner. */ 2122 if (fm->ref_cnt > 0) 2123 return -rte_mtr_error_set(error, EBUSY, 2124 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 2125 NULL, "Meter object is being used."); 2126 /* Destroy the meter profile. */ 2127 __atomic_fetch_sub(&fm->profile->ref_cnt, 2128 1, __ATOMIC_RELAXED); 2129 /* Destroy the meter policy. */ 2130 policy = mlx5_flow_meter_policy_find(dev, 2131 fm->policy_id, NULL); 2132 __atomic_fetch_sub(&policy->ref_cnt, 2133 1, __ATOMIC_RELAXED); 2134 memset(fm, 0, sizeof(struct mlx5_flow_meter_info)); 2135 return 0; 2136 } 2137 2138 /** 2139 * Modify meter state. 2140 * 2141 * @param[in] priv 2142 * Pointer to mlx5 private data structure. 2143 * @param[in] fm 2144 * Pointer to flow meter. 2145 * @param[in] new_state 2146 * New state to update. 2147 * @param[out] error 2148 * Pointer to rte meter error structure. 2149 * 2150 * @return 2151 * 0 on success, a negative errno value otherwise and rte_errno is set. 2152 */ 2153 static int 2154 mlx5_flow_meter_modify_state(struct mlx5_priv *priv, 2155 struct mlx5_flow_meter_info *fm, 2156 uint32_t new_state, 2157 struct rte_mtr_error *error) 2158 { 2159 static const struct mlx5_flow_meter_srtcm_rfc2697_prm srtcm = { 2160 .cbs_cir = RTE_BE32(MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL), 2161 .ebs_eir = 0, 2162 }; 2163 uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS | 2164 MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR; 2165 int ret; 2166 2167 if (new_state == MLX5_FLOW_METER_DISABLE) 2168 ret = mlx5_flow_meter_action_modify(priv, fm, 2169 &srtcm, modify_bits, 0, 0); 2170 else 2171 ret = mlx5_flow_meter_action_modify(priv, fm, 2172 &fm->profile->srtcm_prm, 2173 modify_bits, 0, 1); 2174 if (ret) 2175 return -rte_mtr_error_set(error, -ret, 2176 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 2177 NULL, 2178 new_state ? 2179 "Failed to enable meter." : 2180 "Failed to disable meter."); 2181 return 0; 2182 } 2183 2184 /** 2185 * Callback to enable flow meter. 2186 * 2187 * @param[in] dev 2188 * Pointer to Ethernet device. 2189 * @param[in] meter_id 2190 * Meter id. 2191 * @param[out] error 2192 * Pointer to rte meter error structure. 2193 * 2194 * @return 2195 * 0 on success, a negative errno value otherwise and rte_errno is set. 2196 */ 2197 static int 2198 mlx5_flow_meter_enable(struct rte_eth_dev *dev, 2199 uint32_t meter_id, 2200 struct rte_mtr_error *error) 2201 { 2202 struct mlx5_priv *priv = dev->data->dev_private; 2203 struct mlx5_flow_meter_info *fm; 2204 int ret; 2205 2206 if (!priv->mtr_en) 2207 return -rte_mtr_error_set(error, ENOTSUP, 2208 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 2209 "Meter is not supported"); 2210 /* Meter object must exist. */ 2211 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 2212 if (fm == NULL) 2213 return -rte_mtr_error_set(error, ENOENT, 2214 RTE_MTR_ERROR_TYPE_MTR_ID, 2215 NULL, "Meter not found."); 2216 if (fm->active_state == MLX5_FLOW_METER_ENABLE) 2217 return 0; 2218 ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_ENABLE, 2219 error); 2220 if (!ret) 2221 fm->active_state = MLX5_FLOW_METER_ENABLE; 2222 return ret; 2223 } 2224 2225 /** 2226 * Callback to disable flow meter. 2227 * 2228 * @param[in] dev 2229 * Pointer to Ethernet device. 2230 * @param[in] meter_id 2231 * Meter id. 2232 * @param[out] error 2233 * Pointer to rte meter error structure. 2234 * 2235 * @return 2236 * 0 on success, a negative errno value otherwise and rte_errno is set. 2237 */ 2238 static int 2239 mlx5_flow_meter_disable(struct rte_eth_dev *dev, 2240 uint32_t meter_id, 2241 struct rte_mtr_error *error) 2242 { 2243 struct mlx5_priv *priv = dev->data->dev_private; 2244 struct mlx5_flow_meter_info *fm; 2245 int ret; 2246 2247 if (!priv->mtr_en) 2248 return -rte_mtr_error_set(error, ENOTSUP, 2249 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 2250 "Meter is not supported"); 2251 /* Meter object must exist. */ 2252 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 2253 if (fm == NULL) 2254 return -rte_mtr_error_set(error, ENOENT, 2255 RTE_MTR_ERROR_TYPE_MTR_ID, 2256 NULL, "Meter not found."); 2257 if (fm->active_state == MLX5_FLOW_METER_DISABLE) 2258 return 0; 2259 ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_DISABLE, 2260 error); 2261 if (!ret) 2262 fm->active_state = MLX5_FLOW_METER_DISABLE; 2263 return ret; 2264 } 2265 2266 /** 2267 * Callback to update meter profile. 2268 * 2269 * @param[in] dev 2270 * Pointer to Ethernet device. 2271 * @param[in] meter_id 2272 * Meter id. 2273 * @param[in] meter_profile_id 2274 * To be updated meter profile id. 2275 * @param[out] error 2276 * Pointer to rte meter error structure. 2277 * 2278 * @return 2279 * 0 on success, a negative errno value otherwise and rte_errno is set. 2280 */ 2281 static int 2282 mlx5_flow_meter_profile_update(struct rte_eth_dev *dev, 2283 uint32_t meter_id, 2284 uint32_t meter_profile_id, 2285 struct rte_mtr_error *error) 2286 { 2287 struct mlx5_priv *priv = dev->data->dev_private; 2288 struct mlx5_flow_meter_profile *fmp; 2289 struct mlx5_flow_meter_profile *old_fmp; 2290 struct mlx5_flow_meter_info *fm; 2291 uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS | 2292 MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR; 2293 int ret; 2294 2295 if (!priv->mtr_en) 2296 return -rte_mtr_error_set(error, ENOTSUP, 2297 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 2298 "Meter is not supported"); 2299 /* Meter profile must exist. */ 2300 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 2301 if (fmp == NULL) 2302 return -rte_mtr_error_set(error, ENOENT, 2303 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 2304 NULL, "Meter profile not found."); 2305 /* Meter object must exist. */ 2306 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 2307 if (fm == NULL) 2308 return -rte_mtr_error_set(error, ENOENT, 2309 RTE_MTR_ERROR_TYPE_MTR_ID, 2310 NULL, "Meter not found."); 2311 /* MTR object already set to meter profile id. */ 2312 old_fmp = fm->profile; 2313 if (fmp == old_fmp) 2314 return 0; 2315 /* Update the profile. */ 2316 fm->profile = fmp; 2317 /* Update meter params in HW (if not disabled). */ 2318 if (fm->active_state == MLX5_FLOW_METER_DISABLE) 2319 goto dec_ref_cnt; 2320 ret = mlx5_flow_meter_action_modify(priv, fm, &fm->profile->srtcm_prm, 2321 modify_bits, fm->active_state, 1); 2322 if (ret) { 2323 fm->profile = old_fmp; 2324 return -rte_mtr_error_set(error, -ret, 2325 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 2326 NULL, "Failed to update meter" 2327 " parameters in hardware."); 2328 } 2329 dec_ref_cnt: 2330 old_fmp->ref_cnt--; 2331 fmp->ref_cnt++; 2332 return 0; 2333 } 2334 2335 /** 2336 * Callback to update meter stats mask. 2337 * 2338 * @param[in] dev 2339 * Pointer to Ethernet device. 2340 * @param[in] meter_id 2341 * Meter id. 2342 * @param[in] stats_mask 2343 * To be updated stats_mask. 2344 * @param[out] error 2345 * Pointer to rte meter error structure. 2346 * 2347 * @return 2348 * 0 on success, a negative errno value otherwise and rte_errno is set. 2349 */ 2350 static int 2351 mlx5_flow_meter_stats_update(struct rte_eth_dev *dev, 2352 uint32_t meter_id, 2353 uint64_t stats_mask, 2354 struct rte_mtr_error *error) 2355 { 2356 struct mlx5_priv *priv = dev->data->dev_private; 2357 struct mlx5_flow_meter_info *fm; 2358 2359 if (!priv->mtr_en) 2360 return -rte_mtr_error_set(error, ENOTSUP, 2361 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 2362 "Meter is not supported"); 2363 /* Meter object must exist. */ 2364 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 2365 if (fm == NULL) 2366 return -rte_mtr_error_set(error, ENOENT, 2367 RTE_MTR_ERROR_TYPE_MTR_ID, 2368 NULL, "Meter object id not valid."); 2369 if (mlx5_flow_meter_stats_enable_update(dev, fm, stats_mask)) 2370 return -rte_mtr_error_set(error, ENOENT, 2371 RTE_MTR_ERROR_TYPE_MTR_ID, 2372 NULL, "Fail to allocate " 2373 "counter for meter."); 2374 return 0; 2375 } 2376 2377 /** 2378 * Callback to read meter statistics. 2379 * 2380 * @param[in] dev 2381 * Pointer to Ethernet device. 2382 * @param[in] meter_id 2383 * Meter id. 2384 * @param[out] stats 2385 * Pointer to store the statistics. 2386 * @param[out] stats_mask 2387 * Pointer to store the stats_mask. 2388 * @param[in] clear 2389 * Statistic to be cleared after read or not. 2390 * @param[out] error 2391 * Pointer to rte meter error structure. 2392 * 2393 * @return 2394 * 0 on success, a negative errno value otherwise and rte_errno is set. 2395 */ 2396 static int 2397 mlx5_flow_meter_stats_read(struct rte_eth_dev *dev, 2398 uint32_t meter_id, 2399 struct rte_mtr_stats *stats, 2400 uint64_t *stats_mask, 2401 int clear, 2402 struct rte_mtr_error *error) 2403 { 2404 struct mlx5_priv *priv = dev->data->dev_private; 2405 struct mlx5_flow_meter_info *fm; 2406 uint64_t pkts; 2407 uint64_t bytes; 2408 int ret = 0; 2409 2410 if (!priv->mtr_en) 2411 return -rte_mtr_error_set(error, ENOTSUP, 2412 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 2413 "Meter is not supported"); 2414 /* Meter object must exist. */ 2415 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 2416 if (fm == NULL) 2417 return -rte_mtr_error_set(error, ENOENT, 2418 RTE_MTR_ERROR_TYPE_MTR_ID, 2419 NULL, "Meter object id not valid."); 2420 *stats_mask = 0; 2421 if (fm->bytes_dropped) 2422 *stats_mask |= RTE_MTR_STATS_N_BYTES_DROPPED; 2423 if (fm->pkts_dropped) 2424 *stats_mask |= RTE_MTR_STATS_N_PKTS_DROPPED; 2425 memset(stats, 0, sizeof(*stats)); 2426 if (fm->drop_cnt) { 2427 ret = mlx5_counter_query(dev, fm->drop_cnt, clear, &pkts, 2428 &bytes, NULL); 2429 if (ret) 2430 goto error; 2431 /* If need to read the packets, set it. */ 2432 if (fm->pkts_dropped) 2433 stats->n_pkts_dropped = pkts; 2434 /* If need to read the bytes, set it. */ 2435 if (fm->bytes_dropped) 2436 stats->n_bytes_dropped = bytes; 2437 } 2438 return 0; 2439 error: 2440 return -rte_mtr_error_set(error, ret, RTE_MTR_ERROR_TYPE_STATS, NULL, 2441 "Failed to read meter drop counters."); 2442 } 2443 2444 static const struct rte_mtr_ops mlx5_flow_mtr_ops = { 2445 .capabilities_get = mlx5_flow_mtr_cap_get, 2446 .meter_profile_add = mlx5_flow_meter_profile_add, 2447 .meter_profile_delete = mlx5_flow_meter_profile_delete, 2448 .meter_profile_get = mlx5_flow_meter_profile_get, 2449 .meter_policy_validate = mlx5_flow_meter_policy_validate, 2450 .meter_policy_add = mlx5_flow_meter_policy_add, 2451 .meter_policy_delete = mlx5_flow_meter_policy_delete, 2452 .meter_policy_get = mlx5_flow_meter_policy_get, 2453 .create = mlx5_flow_meter_create, 2454 .destroy = mlx5_flow_meter_destroy, 2455 .meter_enable = mlx5_flow_meter_enable, 2456 .meter_disable = mlx5_flow_meter_disable, 2457 .meter_profile_update = mlx5_flow_meter_profile_update, 2458 .meter_dscp_table_update = NULL, 2459 .stats_update = mlx5_flow_meter_stats_update, 2460 .stats_read = mlx5_flow_meter_stats_read, 2461 }; 2462 2463 static const struct rte_mtr_ops mlx5_flow_mtr_hws_ops = { 2464 .capabilities_get = mlx5_flow_mtr_cap_get, 2465 .meter_profile_add = mlx5_flow_meter_profile_hws_add, 2466 .meter_profile_delete = mlx5_flow_meter_profile_hws_delete, 2467 .meter_profile_get = mlx5_flow_meter_profile_get, 2468 .meter_policy_validate = mlx5_flow_meter_policy_hws_validate, 2469 .meter_policy_add = mlx5_flow_meter_policy_hws_add, 2470 .meter_policy_delete = mlx5_flow_meter_policy_hws_delete, 2471 .meter_policy_get = mlx5_flow_meter_policy_get, 2472 .create = mlx5_flow_meter_hws_create, 2473 .destroy = mlx5_flow_meter_hws_destroy, 2474 .meter_enable = mlx5_flow_meter_enable, 2475 .meter_disable = mlx5_flow_meter_disable, 2476 .meter_profile_update = mlx5_flow_meter_profile_update, 2477 .meter_dscp_table_update = NULL, 2478 .stats_update = NULL, 2479 .stats_read = NULL, 2480 }; 2481 2482 /** 2483 * Get meter operations. 2484 * 2485 * @param dev 2486 * Pointer to Ethernet device structure. 2487 * @param arg 2488 * Pointer to set the mtr operations. 2489 * 2490 * @return 2491 * Always 0. 2492 */ 2493 int 2494 mlx5_flow_meter_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg) 2495 { 2496 struct mlx5_priv *priv = dev->data->dev_private; 2497 2498 if (priv->sh->config.dv_flow_en == 2) 2499 *(const struct rte_mtr_ops **)arg = &mlx5_flow_mtr_hws_ops; 2500 else 2501 *(const struct rte_mtr_ops **)arg = &mlx5_flow_mtr_ops; 2502 return 0; 2503 } 2504 2505 /** 2506 * Find meter by id. 2507 * 2508 * @param priv 2509 * Pointer to mlx5_priv. 2510 * @param meter_id 2511 * Meter id. 2512 * @param mtr_idx 2513 * Pointer to Meter index. 2514 * 2515 * @return 2516 * Pointer to the meter info found on success, NULL otherwise. 2517 */ 2518 struct mlx5_flow_meter_info * 2519 mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id, 2520 uint32_t *mtr_idx) 2521 { 2522 struct mlx5_legacy_flow_meter *legacy_fm; 2523 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; 2524 struct mlx5_aso_mtr *aso_mtr; 2525 struct mlx5_aso_mtr_pools_mng *pools_mng = 2526 &priv->sh->mtrmng->pools_mng; 2527 union mlx5_l3t_data data; 2528 uint16_t n_valid; 2529 2530 if (priv->mtr_bulk.aso) { 2531 if (mtr_idx) 2532 *mtr_idx = meter_id; 2533 aso_mtr = priv->mtr_bulk.aso + meter_id; 2534 return &aso_mtr->fm; 2535 } 2536 if (priv->sh->meter_aso_en) { 2537 rte_rwlock_read_lock(&pools_mng->resize_mtrwl); 2538 n_valid = pools_mng->n_valid; 2539 rte_rwlock_read_unlock(&pools_mng->resize_mtrwl); 2540 if (!n_valid || !priv->mtr_idx_tbl || 2541 (mlx5_l3t_get_entry(priv->mtr_idx_tbl, meter_id, &data) || 2542 !data.dword)) 2543 return NULL; 2544 if (mtr_idx) 2545 *mtr_idx = data.dword; 2546 aso_mtr = mlx5_aso_meter_by_idx(priv, data.dword); 2547 /* Remove reference taken by the mlx5_l3t_get_entry. */ 2548 mlx5_l3t_clear_entry(priv->mtr_idx_tbl, meter_id); 2549 if (!aso_mtr || aso_mtr->state == ASO_METER_FREE) 2550 return NULL; 2551 return &aso_mtr->fm; 2552 } 2553 TAILQ_FOREACH(legacy_fm, fms, next) 2554 if (meter_id == legacy_fm->fm.meter_id) { 2555 if (mtr_idx) 2556 *mtr_idx = legacy_fm->idx; 2557 return &legacy_fm->fm; 2558 } 2559 return NULL; 2560 } 2561 2562 /** 2563 * Find meter by index. 2564 * 2565 * @param priv 2566 * Pointer to mlx5_priv. 2567 * @param idx 2568 * Meter index. 2569 * 2570 * @return 2571 * Pointer to the meter info found on success, NULL otherwise. 2572 */ 2573 struct mlx5_flow_meter_info * 2574 flow_dv_meter_find_by_idx(struct mlx5_priv *priv, uint32_t idx) 2575 { 2576 struct mlx5_aso_mtr *aso_mtr; 2577 2578 if (priv->sh->meter_aso_en) { 2579 aso_mtr = mlx5_aso_meter_by_idx(priv, idx); 2580 if (!aso_mtr) 2581 return NULL; 2582 return &aso_mtr->fm; 2583 } else { 2584 return mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], idx); 2585 } 2586 } 2587 2588 /** 2589 * Attach meter to flow. 2590 * Unidirectional Meter creation can only be done 2591 * when flow direction is known, i.e. when calling meter_attach. 2592 * 2593 * @param [in] priv 2594 * Pointer to mlx5 private data. 2595 * @param[in] fm 2596 * Pointer to flow meter. 2597 * @param [in] attr 2598 * Pointer to flow attributes. 2599 * @param [out] error 2600 * Pointer to error structure. 2601 * 2602 * @return 2603 * 0 on success, a negative errno value otherwise and rte_errno is set. 2604 */ 2605 int 2606 mlx5_flow_meter_attach(struct mlx5_priv *priv, 2607 struct mlx5_flow_meter_info *fm, 2608 const struct rte_flow_attr *attr, 2609 struct rte_flow_error *error) 2610 { 2611 int ret = 0; 2612 2613 if (priv->sh->meter_aso_en) { 2614 struct mlx5_aso_mtr *aso_mtr; 2615 2616 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm); 2617 if (mlx5_aso_mtr_wait(priv->sh, MLX5_HW_INV_QUEUE, aso_mtr)) { 2618 return rte_flow_error_set(error, ENOENT, 2619 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2620 NULL, 2621 "Timeout in meter configuration"); 2622 } 2623 rte_spinlock_lock(&fm->sl); 2624 if (fm->shared || !fm->ref_cnt) { 2625 fm->ref_cnt++; 2626 } else { 2627 rte_flow_error_set(error, EINVAL, 2628 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 2629 "Meter cannot be shared"); 2630 ret = -1; 2631 } 2632 rte_spinlock_unlock(&fm->sl); 2633 } else { 2634 rte_spinlock_lock(&fm->sl); 2635 if (fm->meter_action_g) { 2636 if (fm->shared && 2637 attr->transfer == fm->transfer && 2638 attr->ingress == fm->ingress && 2639 attr->egress == fm->egress) { 2640 fm->ref_cnt++; 2641 } else { 2642 rte_flow_error_set(error, EINVAL, 2643 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 2644 fm->shared ? 2645 "Meter attr not match." : 2646 "Meter cannot be shared."); 2647 ret = -1; 2648 } 2649 } else { 2650 fm->ingress = attr->ingress; 2651 fm->egress = attr->egress; 2652 fm->transfer = attr->transfer; 2653 fm->ref_cnt = 1; 2654 /* This also creates the meter object. */ 2655 fm->meter_action_g = mlx5_flow_meter_action_create(priv, 2656 fm); 2657 if (!fm->meter_action_g) { 2658 fm->ref_cnt = 0; 2659 fm->ingress = 0; 2660 fm->egress = 0; 2661 fm->transfer = 0; 2662 rte_flow_error_set(error, EINVAL, 2663 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 2664 "Meter action create failed."); 2665 ret = -1; 2666 } 2667 } 2668 rte_spinlock_unlock(&fm->sl); 2669 } 2670 return ret ? -rte_errno : 0; 2671 } 2672 2673 /** 2674 * Detach meter from flow. 2675 * 2676 * @param [in] priv 2677 * Pointer to mlx5 private data. 2678 * @param [in] fm 2679 * Pointer to flow meter. 2680 */ 2681 void 2682 mlx5_flow_meter_detach(struct mlx5_priv *priv, 2683 struct mlx5_flow_meter_info *fm) 2684 { 2685 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER 2686 rte_spinlock_lock(&fm->sl); 2687 MLX5_ASSERT(fm->ref_cnt); 2688 if (--fm->ref_cnt == 0 && !priv->sh->meter_aso_en) { 2689 mlx5_glue->destroy_flow_action(fm->meter_action_g); 2690 fm->meter_action_g = NULL; 2691 fm->ingress = 0; 2692 fm->egress = 0; 2693 fm->transfer = 0; 2694 } 2695 rte_spinlock_unlock(&fm->sl); 2696 #else 2697 (void)priv; 2698 (void)fm; 2699 #endif 2700 } 2701 2702 /** 2703 * Flush meter with Rx queue configuration. 2704 * 2705 * @param[in] dev 2706 * Pointer to Ethernet device. 2707 */ 2708 void 2709 mlx5_flow_meter_rxq_flush(struct rte_eth_dev *dev) 2710 { 2711 struct mlx5_priv *priv = dev->data->dev_private; 2712 struct mlx5_flow_meter_sub_policy *sub_policy; 2713 struct mlx5_flow_meter_policy *mtr_policy; 2714 void *entry; 2715 uint32_t i, policy_idx; 2716 2717 if (!priv->mtr_en) 2718 return; 2719 if (priv->policy_idx_tbl) { 2720 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) { 2721 policy_idx = *(uint32_t *)entry; 2722 sub_policy = mlx5_ipool_get 2723 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 2724 policy_idx); 2725 if (!sub_policy || !sub_policy->main_policy) 2726 continue; 2727 mtr_policy = sub_policy->main_policy; 2728 if (mtr_policy->is_queue || mtr_policy->is_rss) 2729 mlx5_flow_destroy_sub_policy_with_rxq(dev, 2730 mtr_policy); 2731 } 2732 } 2733 } 2734 2735 /** 2736 * Iterate a meter hierarchy and flush all meters and policies if possible. 2737 * 2738 * @param[in] dev 2739 * Pointer to Ethernet device. 2740 * @param[in] fm 2741 * Pointer to flow meter. 2742 * @param[in] mtr_idx 2743 * .Meter's index 2744 * @param[out] error 2745 * Pointer to rte meter error structure. 2746 * 2747 * @return 2748 * 0 on success, a negative errno value otherwise and rte_errno is set. 2749 */ 2750 static int 2751 mlx5_flow_meter_flush_hierarchy(struct rte_eth_dev *dev, 2752 struct mlx5_flow_meter_info *fm, 2753 uint32_t mtr_idx, 2754 struct rte_mtr_error *error) 2755 { 2756 struct mlx5_priv *priv = dev->data->dev_private; 2757 struct mlx5_flow_meter_policy *policy; 2758 uint32_t policy_id; 2759 struct mlx5_flow_meter_info *next_fm; 2760 uint32_t next_mtr_idx; 2761 struct mlx5_flow_meter_policy *next_policy = NULL; 2762 2763 policy = mlx5_flow_meter_policy_find(dev, fm->policy_id, NULL); 2764 MLX5_ASSERT(policy); 2765 while (!fm->ref_cnt && policy->is_hierarchy) { 2766 policy_id = fm->policy_id; 2767 next_fm = mlx5_flow_meter_hierarchy_next_meter(priv, policy, &next_mtr_idx); 2768 if (next_fm) { 2769 next_policy = mlx5_flow_meter_policy_find(dev, 2770 next_fm->policy_id, 2771 NULL); 2772 MLX5_ASSERT(next_policy); 2773 } 2774 if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx)) 2775 return -rte_mtr_error_set(error, ENOTSUP, 2776 RTE_MTR_ERROR_TYPE_MTR_ID, 2777 NULL, 2778 "Failed to flush meter."); 2779 if (policy->ref_cnt) 2780 break; 2781 if (__mlx5_flow_meter_policy_delete(dev, policy_id, 2782 policy, error, true)) 2783 return -rte_errno; 2784 mlx5_free(policy); 2785 if (!next_fm || !next_policy) 2786 break; 2787 fm = next_fm; 2788 mtr_idx = next_mtr_idx; 2789 policy = next_policy; 2790 } 2791 return 0; 2792 } 2793 2794 /** 2795 * Flush all the hierarchy meters and their policies. 2796 * 2797 * @param[in] dev 2798 * Pointer to Ethernet device. 2799 * @param[out] error 2800 * Pointer to rte meter error structure. 2801 * 2802 * @return 2803 * 0 on success, a negative errno value otherwise and rte_errno is set. 2804 */ 2805 static int 2806 mlx5_flow_meter_flush_all_hierarchies(struct rte_eth_dev *dev, 2807 struct rte_mtr_error *error) 2808 { 2809 struct mlx5_priv *priv = dev->data->dev_private; 2810 struct mlx5_flow_meter_info *fm; 2811 struct mlx5_flow_meter_policy *policy; 2812 struct mlx5_flow_meter_sub_policy *sub_policy; 2813 struct mlx5_flow_meter_info *next_fm; 2814 struct mlx5_aso_mtr *aso_mtr; 2815 uint32_t mtr_idx = 0; 2816 uint32_t i, policy_idx; 2817 void *entry; 2818 2819 if (!priv->mtr_idx_tbl || !priv->policy_idx_tbl) 2820 return 0; 2821 MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) { 2822 mtr_idx = *(uint32_t *)entry; 2823 if (!mtr_idx) 2824 continue; 2825 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx); 2826 fm = &aso_mtr->fm; 2827 if (fm->ref_cnt || fm->def_policy) 2828 continue; 2829 if (mlx5_flow_meter_flush_hierarchy(dev, fm, mtr_idx, error)) 2830 return -rte_errno; 2831 } 2832 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) { 2833 policy_idx = *(uint32_t *)entry; 2834 sub_policy = mlx5_ipool_get 2835 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 2836 policy_idx); 2837 if (!sub_policy) 2838 return -rte_mtr_error_set(error, 2839 EINVAL, 2840 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 2841 NULL, "Meter policy invalid."); 2842 policy = sub_policy->main_policy; 2843 if (!policy || !policy->is_hierarchy || policy->ref_cnt) 2844 continue; 2845 next_fm = mlx5_flow_meter_hierarchy_next_meter(priv, policy, &mtr_idx); 2846 if (__mlx5_flow_meter_policy_delete(dev, i, policy, 2847 error, true)) 2848 return -rte_mtr_error_set(error, 2849 EINVAL, 2850 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 2851 NULL, "Meter policy invalid."); 2852 mlx5_free(policy); 2853 if (!next_fm || next_fm->ref_cnt || next_fm->def_policy) 2854 continue; 2855 if (mlx5_flow_meter_flush_hierarchy(dev, next_fm, 2856 mtr_idx, error)) 2857 return -rte_errno; 2858 } 2859 return 0; 2860 } 2861 /** 2862 * Flush meter configuration. 2863 * 2864 * @param[in] dev 2865 * Pointer to Ethernet device. 2866 * @param[out] error 2867 * Pointer to rte meter error structure. 2868 * 2869 * @return 2870 * 0 on success, a negative errno value otherwise and rte_errno is set. 2871 */ 2872 int 2873 mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error) 2874 { 2875 struct mlx5_priv *priv = dev->data->dev_private; 2876 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; 2877 struct mlx5_flow_meter_profile *fmp; 2878 struct mlx5_legacy_flow_meter *legacy_fm; 2879 struct mlx5_flow_meter_info *fm; 2880 struct mlx5_flow_meter_policy *policy; 2881 struct mlx5_flow_meter_sub_policy *sub_policy; 2882 void *tmp; 2883 uint32_t i, mtr_idx, policy_idx; 2884 void *entry; 2885 struct mlx5_aso_mtr *aso_mtr; 2886 2887 if (!priv->mtr_en) 2888 return 0; 2889 if (priv->sh->meter_aso_en) { 2890 if (mlx5_flow_meter_flush_all_hierarchies(dev, error)) 2891 return -rte_errno; 2892 if (priv->mtr_idx_tbl) { 2893 MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) { 2894 mtr_idx = *(uint32_t *)entry; 2895 if (mtr_idx) { 2896 aso_mtr = 2897 mlx5_aso_meter_by_idx(priv, mtr_idx); 2898 fm = &aso_mtr->fm; 2899 (void)mlx5_flow_meter_params_flush(dev, 2900 fm, mtr_idx); 2901 } 2902 } 2903 mlx5_l3t_destroy(priv->mtr_idx_tbl); 2904 priv->mtr_idx_tbl = NULL; 2905 } 2906 } else { 2907 RTE_TAILQ_FOREACH_SAFE(legacy_fm, fms, next, tmp) { 2908 fm = &legacy_fm->fm; 2909 if (mlx5_flow_meter_params_flush(dev, fm, 0)) 2910 return -rte_mtr_error_set(error, EINVAL, 2911 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 2912 NULL, "MTR object meter profile invalid."); 2913 } 2914 } 2915 if (priv->mtr_bulk.aso) { 2916 for (i = 0; i < priv->mtr_config.nb_meters; i++) { 2917 aso_mtr = mlx5_aso_meter_by_idx(priv, i); 2918 fm = &aso_mtr->fm; 2919 if (fm->initialized) 2920 mlx5_flow_meter_hws_destroy(dev, i, error); 2921 } 2922 } 2923 if (priv->policy_idx_tbl) { 2924 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) { 2925 policy_idx = *(uint32_t *)entry; 2926 sub_policy = mlx5_ipool_get 2927 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 2928 policy_idx); 2929 if (!sub_policy) 2930 return -rte_mtr_error_set(error, 2931 EINVAL, 2932 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 2933 NULL, "MTR object " 2934 "meter policy invalid."); 2935 if (__mlx5_flow_meter_policy_delete(dev, i, 2936 sub_policy->main_policy, 2937 error, true)) 2938 return -rte_mtr_error_set(error, 2939 EINVAL, 2940 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 2941 NULL, "MTR object " 2942 "meter policy invalid."); 2943 mlx5_free(sub_policy->main_policy); 2944 } 2945 mlx5_l3t_destroy(priv->policy_idx_tbl); 2946 priv->policy_idx_tbl = NULL; 2947 } 2948 if (priv->mtr_policy_arr) { 2949 for (i = 0; i < priv->mtr_config.nb_meter_policies; i++) { 2950 policy = mlx5_flow_meter_policy_find(dev, i, 2951 &policy_idx); 2952 if (policy->initialized) 2953 mlx5_flow_meter_policy_hws_delete(dev, i, 2954 error); 2955 } 2956 } 2957 if (priv->mtr_profile_tbl) { 2958 MLX5_L3T_FOREACH(priv->mtr_profile_tbl, i, entry) { 2959 fmp = entry; 2960 if (mlx5_flow_meter_profile_delete(dev, fmp->id, 2961 error)) 2962 return -rte_mtr_error_set(error, EINVAL, 2963 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 2964 NULL, "Fail to destroy " 2965 "meter profile."); 2966 } 2967 mlx5_l3t_destroy(priv->mtr_profile_tbl); 2968 priv->mtr_profile_tbl = NULL; 2969 } 2970 if (priv->mtr_profile_arr) { 2971 for (i = 0; i < priv->mtr_config.nb_meter_profiles; i++) { 2972 fmp = mlx5_flow_meter_profile_find(priv, i); 2973 if (fmp->initialized) 2974 mlx5_flow_meter_profile_hws_delete(dev, i, 2975 error); 2976 } 2977 } 2978 /* Delete default policy table. */ 2979 mlx5_flow_destroy_def_policy(dev); 2980 if (priv->sh->refcnt == 1) 2981 mlx5_flow_destroy_mtr_drop_tbls(dev); 2982 #ifdef HAVE_MLX5_HWS_SUPPORT 2983 /* Destroy HWS configuration. */ 2984 mlx5_flow_meter_uninit(dev); 2985 #endif 2986 return 0; 2987 } 2988