1 // SPDX-License-Identifier: BSD-3-Clause 2 /* 3 * Copyright 2018 Mellanox Technologies, Ltd 4 */ 5 #include <math.h> 6 7 #include <rte_tailq.h> 8 #include <rte_malloc.h> 9 #include <rte_mtr.h> 10 #include <rte_mtr_driver.h> 11 12 #include <mlx5_devx_cmds.h> 13 #include <mlx5_malloc.h> 14 15 #include "mlx5.h" 16 #include "mlx5_flow.h" 17 18 static int mlx5_flow_meter_disable(struct rte_eth_dev *dev, 19 uint32_t meter_id, struct rte_mtr_error *error); 20 21 /** 22 * Create the meter action. 23 * 24 * @param priv 25 * Pointer to mlx5_priv. 26 * @param[in] fm 27 * Pointer to flow meter to be converted. 28 * 29 * @return 30 * Pointer to the meter action on success, NULL otherwise. 31 */ 32 static void * 33 mlx5_flow_meter_action_create(struct mlx5_priv *priv, 34 struct mlx5_flow_meter_info *fm) 35 { 36 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER 37 struct mlx5dv_dr_flow_meter_attr mtr_init; 38 uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)]; 39 struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = 40 &fm->profile->srtcm_prm; 41 uint32_t cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir); 42 uint32_t ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir); 43 uint32_t val; 44 enum mlx5_meter_domain domain = 45 fm->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 46 fm->egress ? MLX5_MTR_DOMAIN_EGRESS : 47 MLX5_MTR_DOMAIN_INGRESS; 48 struct mlx5_flow_meter_def_policy *def_policy = 49 priv->sh->mtrmng->def_policy[domain]; 50 51 memset(fmp, 0, MLX5_ST_SZ_BYTES(flow_meter_parameters)); 52 MLX5_SET(flow_meter_parameters, fmp, valid, 1); 53 MLX5_SET(flow_meter_parameters, fmp, bucket_overflow, 1); 54 MLX5_SET(flow_meter_parameters, fmp, 55 start_color, MLX5_FLOW_COLOR_GREEN); 56 MLX5_SET(flow_meter_parameters, fmp, both_buckets_on_green, 0); 57 val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK; 58 MLX5_SET(flow_meter_parameters, fmp, cbs_exponent, val); 59 val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK; 60 MLX5_SET(flow_meter_parameters, fmp, cbs_mantissa, val); 61 val = (cbs_cir >> ASO_DSEG_XIR_EXP_OFFSET) & ASO_DSEG_EXP_MASK; 62 MLX5_SET(flow_meter_parameters, fmp, cir_exponent, val); 63 val = (cbs_cir & ASO_DSEG_MAN_MASK); 64 MLX5_SET(flow_meter_parameters, fmp, cir_mantissa, val); 65 val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK; 66 MLX5_SET(flow_meter_parameters, fmp, ebs_exponent, val); 67 val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK; 68 MLX5_SET(flow_meter_parameters, fmp, ebs_mantissa, val); 69 mtr_init.next_table = def_policy->sub_policy.tbl_rsc->obj; 70 mtr_init.reg_c_index = priv->mtr_color_reg - REG_C_0; 71 mtr_init.flow_meter_parameter = fmp; 72 mtr_init.flow_meter_parameter_sz = 73 MLX5_ST_SZ_BYTES(flow_meter_parameters); 74 mtr_init.active = fm->active_state; 75 return mlx5_glue->dv_create_flow_action_meter(&mtr_init); 76 #else 77 (void)priv; 78 (void)fm; 79 return NULL; 80 #endif 81 } 82 83 /** 84 * Find meter profile by id. 85 * 86 * @param priv 87 * Pointer to mlx5_priv. 88 * @param meter_profile_id 89 * Meter profile id. 90 * 91 * @return 92 * Pointer to the profile found on success, NULL otherwise. 93 */ 94 static struct mlx5_flow_meter_profile * 95 mlx5_flow_meter_profile_find(struct mlx5_priv *priv, uint32_t meter_profile_id) 96 { 97 struct mlx5_flow_meter_profile *fmp; 98 union mlx5_l3t_data data; 99 int32_t ret; 100 101 if (mlx5_l3t_get_entry(priv->mtr_profile_tbl, 102 meter_profile_id, &data) || !data.ptr) 103 return NULL; 104 fmp = data.ptr; 105 /* Remove reference taken by the mlx5_l3t_get_entry. */ 106 ret = mlx5_l3t_clear_entry(priv->mtr_profile_tbl, 107 meter_profile_id); 108 if (!ret || ret == -1) 109 return NULL; 110 return fmp; 111 } 112 113 /** 114 * Validate the MTR profile. 115 * 116 * @param[in] dev 117 * Pointer to Ethernet device. 118 * @param[in] meter_profile_id 119 * Meter profile id. 120 * @param[in] profile 121 * Pointer to meter profile detail. 122 * @param[out] error 123 * Pointer to the error structure. 124 * 125 * @return 126 * 0 on success, a negative errno value otherwise and rte_errno is set. 127 */ 128 static int 129 mlx5_flow_meter_profile_validate(struct rte_eth_dev *dev, 130 uint32_t meter_profile_id, 131 struct rte_mtr_meter_profile *profile, 132 struct rte_mtr_error *error) 133 { 134 struct mlx5_priv *priv = dev->data->dev_private; 135 struct mlx5_flow_meter_profile *fmp; 136 uint32_t ls_factor; 137 int ret; 138 uint64_t cir, cbs; 139 uint64_t eir, ebs; 140 uint64_t pir, pbs; 141 142 /* Profile must not be NULL. */ 143 if (profile == NULL) 144 return -rte_mtr_error_set(error, EINVAL, 145 RTE_MTR_ERROR_TYPE_METER_PROFILE, 146 NULL, "Meter profile is null."); 147 /* Meter profile ID must be valid. */ 148 if (meter_profile_id == UINT32_MAX) 149 return -rte_mtr_error_set(error, EINVAL, 150 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 151 NULL, "Meter profile id not valid."); 152 /* Meter profile must not exist. */ 153 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 154 if (fmp) 155 return -rte_mtr_error_set(error, EEXIST, 156 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 157 NULL, 158 "Meter profile already exists."); 159 if (!priv->sh->meter_aso_en) { 160 /* Old version is even not supported. */ 161 if (!priv->sh->cdev->config.hca_attr.qos.flow_meter_old) 162 return -rte_mtr_error_set(error, ENOTSUP, 163 RTE_MTR_ERROR_TYPE_METER_PROFILE, 164 NULL, "Metering is not supported."); 165 /* Old FW metering only supports srTCM. */ 166 if (profile->alg != RTE_MTR_SRTCM_RFC2697) { 167 return -rte_mtr_error_set(error, ENOTSUP, 168 RTE_MTR_ERROR_TYPE_METER_PROFILE, 169 NULL, "Metering algorithm is not supported."); 170 } else if (profile->srtcm_rfc2697.ebs) { 171 /* EBS is not supported for old metering. */ 172 return -rte_mtr_error_set(error, ENOTSUP, 173 RTE_MTR_ERROR_TYPE_METER_PROFILE, 174 NULL, "EBS is not supported."); 175 } 176 if (profile->packet_mode) 177 return -rte_mtr_error_set(error, ENOTSUP, 178 RTE_MTR_ERROR_TYPE_METER_PROFILE, NULL, 179 "Metering algorithm packet mode is not supported."); 180 } 181 ls_factor = profile->packet_mode ? MLX5_MTRS_PPS_MAP_BPS_SHIFT : 0; 182 switch (profile->alg) { 183 case RTE_MTR_SRTCM_RFC2697: 184 cir = profile->srtcm_rfc2697.cir << ls_factor; 185 cbs = profile->srtcm_rfc2697.cbs << ls_factor; 186 ebs = profile->srtcm_rfc2697.ebs << ls_factor; 187 /* EBS could be zero for old metering. */ 188 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX && 189 cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX && 190 ebs <= MLX5_SRTCM_XBS_MAX) { 191 ret = 0; 192 } else { 193 ret = -rte_mtr_error_set(error, ENOTSUP, 194 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 195 "Profile values out of range."); 196 } 197 break; 198 case RTE_MTR_TRTCM_RFC2698: 199 cir = profile->trtcm_rfc2698.cir << ls_factor; 200 cbs = profile->trtcm_rfc2698.cbs << ls_factor; 201 pir = profile->trtcm_rfc2698.pir << ls_factor; 202 pbs = profile->trtcm_rfc2698.pbs << ls_factor; 203 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX && 204 cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX && 205 pir >= cir && pir <= (MLX5_SRTCM_XIR_MAX * 2) && 206 pbs >= cbs && pbs <= (MLX5_SRTCM_XBS_MAX * 2)) { 207 ret = 0; 208 } else { 209 ret = -rte_mtr_error_set(error, ENOTSUP, 210 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 211 "Profile values out of range."); 212 } 213 break; 214 case RTE_MTR_TRTCM_RFC4115: 215 cir = profile->trtcm_rfc4115.cir << ls_factor; 216 cbs = profile->trtcm_rfc4115.cbs << ls_factor; 217 eir = profile->trtcm_rfc4115.eir << ls_factor; 218 ebs = profile->trtcm_rfc4115.ebs << ls_factor; 219 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX && 220 cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX && 221 eir <= MLX5_SRTCM_XIR_MAX && ebs <= MLX5_SRTCM_XBS_MAX) { 222 ret = 0; 223 } else { 224 ret = -rte_mtr_error_set(error, ENOTSUP, 225 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 226 "Profile values out of range."); 227 } 228 break; 229 default: 230 ret = -rte_mtr_error_set(error, ENOTSUP, 231 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 232 "Unknown metering algorithm."); 233 break; 234 } 235 return ret; 236 } 237 238 /* 239 * Calculate mantissa and exponent for cir / eir. 240 * 241 * @param[in] xir 242 * Value to be calculated. 243 * @param[out] man 244 * Pointer to the mantissa. 245 * @param[out] exp 246 * Pointer to the exp. 247 */ 248 static inline void 249 mlx5_flow_meter_xir_man_exp_calc(int64_t xir, uint8_t *man, uint8_t *exp) 250 { 251 int64_t _xir; 252 int64_t delta = INT64_MAX; 253 uint8_t _man = 0; 254 uint8_t _exp = 0; 255 uint64_t m, e; 256 257 /* Special case xir == 0 ? both exp and mantissa are 0. */ 258 if (xir == 0) { 259 *man = 0; 260 *exp = 0; 261 return; 262 } 263 for (m = 0; m <= 0xFF; m++) { /* man width 8 bit */ 264 for (e = 0; e <= 0x1F; e++) { /* exp width 5bit */ 265 _xir = (1000000000ULL * m) >> e; 266 if (llabs(xir - _xir) <= delta) { 267 delta = llabs(xir - _xir); 268 _man = m; 269 _exp = e; 270 } 271 } 272 } 273 *man = _man; 274 *exp = _exp; 275 } 276 277 /* 278 * Calculate mantissa and exponent for xbs. 279 * 280 * @param[in] xbs 281 * Value to be calculated. 282 * @param[out] man 283 * Pointer to the mantissa. 284 * @param[out] exp 285 * Pointer to the exp. 286 */ 287 static inline void 288 mlx5_flow_meter_xbs_man_exp_calc(uint64_t xbs, uint8_t *man, uint8_t *exp) 289 { 290 int _exp; 291 double _man; 292 293 /* Special case xbs == 0 ? both exp and mantissa are 0. */ 294 if (xbs == 0) { 295 *man = 0; 296 *exp = 0; 297 return; 298 } 299 /* xbs = xbs_mantissa * 2^xbs_exponent */ 300 _man = frexp(xbs, &_exp); 301 if (_exp >= MLX5_MAN_WIDTH) { 302 _man = _man * pow(2, MLX5_MAN_WIDTH); 303 _exp = _exp - MLX5_MAN_WIDTH; 304 } 305 *man = (uint8_t)ceil(_man); 306 *exp = _exp; 307 } 308 309 /** 310 * Fill the prm meter parameter. 311 * 312 * @param[in,out] fmp 313 * Pointer to meter profile to be converted. 314 * @param[out] error 315 * Pointer to the error structure. 316 * 317 * @return 318 * 0 on success, a negative errno value otherwise and rte_errno is set. 319 */ 320 static int 321 mlx5_flow_meter_param_fill(struct mlx5_flow_meter_profile *fmp, 322 struct rte_mtr_error *error) 323 { 324 struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = &fmp->srtcm_prm; 325 uint8_t man, exp; 326 uint32_t cbs_exp, cbs_man, cir_exp, cir_man; 327 uint32_t eir_exp, eir_man, ebs_exp, ebs_man; 328 uint64_t cir, cbs, eir, ebs; 329 330 switch (fmp->profile.alg) { 331 case RTE_MTR_SRTCM_RFC2697: 332 cir = fmp->profile.srtcm_rfc2697.cir; 333 cbs = fmp->profile.srtcm_rfc2697.cbs; 334 eir = 0; 335 ebs = fmp->profile.srtcm_rfc2697.ebs; 336 break; 337 case RTE_MTR_TRTCM_RFC2698: 338 MLX5_ASSERT(fmp->profile.trtcm_rfc2698.pir > 339 fmp->profile.trtcm_rfc2698.cir && 340 fmp->profile.trtcm_rfc2698.pbs > 341 fmp->profile.trtcm_rfc2698.cbs); 342 cir = fmp->profile.trtcm_rfc2698.cir; 343 cbs = fmp->profile.trtcm_rfc2698.cbs; 344 /* EIR / EBS are filled with PIR / PBS. */ 345 eir = fmp->profile.trtcm_rfc2698.pir; 346 ebs = fmp->profile.trtcm_rfc2698.pbs; 347 break; 348 case RTE_MTR_TRTCM_RFC4115: 349 cir = fmp->profile.trtcm_rfc4115.cir; 350 cbs = fmp->profile.trtcm_rfc4115.cbs; 351 eir = fmp->profile.trtcm_rfc4115.eir; 352 ebs = fmp->profile.trtcm_rfc4115.ebs; 353 break; 354 default: 355 return -rte_mtr_error_set(error, EINVAL, 356 RTE_MTR_ERROR_TYPE_METER_PROFILE, NULL, 357 "Metering algorithm mode is invalid"); 358 } 359 /* Adjust the values for PPS mode. */ 360 if (fmp->profile.packet_mode) { 361 cir <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT; 362 cbs <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT; 363 eir <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT; 364 ebs <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT; 365 } 366 /* cir = 8G * cir_mantissa * 1/(2^cir_exponent)) Bytes/Sec */ 367 mlx5_flow_meter_xir_man_exp_calc(cir, &man, &exp); 368 /* Check if cir mantissa is too large. */ 369 if (exp > ASO_DSEG_XIR_EXP_MASK) 370 return -rte_mtr_error_set(error, ENOTSUP, 371 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 372 "meter profile parameter cir is not supported."); 373 cir_man = man; 374 cir_exp = exp; 375 /* cbs = cbs_mantissa * 2^cbs_exponent */ 376 mlx5_flow_meter_xbs_man_exp_calc(cbs, &man, &exp); 377 /* Check if cbs mantissa is too large. */ 378 if (exp > ASO_DSEG_EXP_MASK) 379 return -rte_mtr_error_set(error, ENOTSUP, 380 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 381 "meter profile parameter cbs is not supported."); 382 cbs_man = man; 383 cbs_exp = exp; 384 srtcm->cbs_cir = rte_cpu_to_be_32(cbs_exp << ASO_DSEG_CBS_EXP_OFFSET | 385 cbs_man << ASO_DSEG_CBS_MAN_OFFSET | 386 cir_exp << ASO_DSEG_XIR_EXP_OFFSET | 387 cir_man); 388 mlx5_flow_meter_xir_man_exp_calc(eir, &man, &exp); 389 /* Check if eir mantissa is too large. */ 390 if (exp > ASO_DSEG_XIR_EXP_MASK) 391 return -rte_mtr_error_set(error, ENOTSUP, 392 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 393 "meter profile parameter eir is not supported."); 394 eir_man = man; 395 eir_exp = exp; 396 mlx5_flow_meter_xbs_man_exp_calc(ebs, &man, &exp); 397 /* Check if ebs mantissa is too large. */ 398 if (exp > ASO_DSEG_EXP_MASK) 399 return -rte_mtr_error_set(error, ENOTSUP, 400 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 401 "meter profile parameter ebs is not supported."); 402 ebs_man = man; 403 ebs_exp = exp; 404 srtcm->ebs_eir = rte_cpu_to_be_32(ebs_exp << ASO_DSEG_EBS_EXP_OFFSET | 405 ebs_man << ASO_DSEG_EBS_MAN_OFFSET | 406 eir_exp << ASO_DSEG_XIR_EXP_OFFSET | 407 eir_man); 408 if (srtcm->cbs_cir) 409 fmp->g_support = 1; 410 if (srtcm->ebs_eir) 411 fmp->y_support = 1; 412 return 0; 413 } 414 415 /** 416 * Callback to get MTR capabilities. 417 * 418 * @param[in] dev 419 * Pointer to Ethernet device. 420 * @param[out] cap 421 * Pointer to save MTR capabilities. 422 * @param[out] error 423 * Pointer to the error structure. 424 * 425 * @return 426 * 0 on success, a negative errno value otherwise and rte_errno is set. 427 */ 428 static int 429 mlx5_flow_mtr_cap_get(struct rte_eth_dev *dev, 430 struct rte_mtr_capabilities *cap, 431 struct rte_mtr_error *error __rte_unused) 432 { 433 struct mlx5_priv *priv = dev->data->dev_private; 434 struct mlx5_hca_qos_attr *qattr = &priv->sh->cdev->config.hca_attr.qos; 435 436 if (!priv->mtr_en) 437 return -rte_mtr_error_set(error, ENOTSUP, 438 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 439 "Meter is not supported"); 440 memset(cap, 0, sizeof(*cap)); 441 if (priv->sh->meter_aso_en) { 442 /* 2 meters per one ASO cache line. */ 443 cap->n_max = 1 << (qattr->log_max_num_meter_aso + 1); 444 cap->srtcm_rfc2697_packet_mode_supported = 1; 445 cap->trtcm_rfc2698_packet_mode_supported = 1; 446 cap->trtcm_rfc4115_packet_mode_supported = 1; 447 } else { 448 cap->n_max = 1 << qattr->log_max_flow_meter; 449 } 450 cap->srtcm_rfc2697_byte_mode_supported = 1; 451 cap->trtcm_rfc2698_byte_mode_supported = 1; 452 cap->trtcm_rfc4115_byte_mode_supported = 1; 453 cap->n_shared_max = cap->n_max; 454 cap->identical = 1; 455 cap->shared_identical = 1; 456 cap->shared_n_flows_per_mtr_max = 4 << 20; 457 /* 2M flows can share the same meter. */ 458 cap->chaining_n_mtrs_per_flow_max = 1; /* Chaining is not supported. */ 459 cap->meter_srtcm_rfc2697_n_max = qattr->flow_meter_old ? cap->n_max : 0; 460 cap->meter_trtcm_rfc2698_n_max = qattr->flow_meter_old ? cap->n_max : 0; 461 cap->meter_trtcm_rfc4115_n_max = qattr->flow_meter_old ? cap->n_max : 0; 462 cap->meter_rate_max = 1ULL << 40; /* 1 Tera tokens per sec. */ 463 cap->meter_policy_n_max = cap->n_max; 464 cap->stats_mask = RTE_MTR_STATS_N_BYTES_DROPPED | 465 RTE_MTR_STATS_N_PKTS_DROPPED; 466 return 0; 467 } 468 469 /** 470 * Callback to add MTR profile. 471 * 472 * @param[in] dev 473 * Pointer to Ethernet device. 474 * @param[in] meter_profile_id 475 * Meter profile id. 476 * @param[in] profile 477 * Pointer to meter profile detail. 478 * @param[out] error 479 * Pointer to the error structure. 480 * 481 * @return 482 * 0 on success, a negative errno value otherwise and rte_errno is set. 483 */ 484 static int 485 mlx5_flow_meter_profile_add(struct rte_eth_dev *dev, 486 uint32_t meter_profile_id, 487 struct rte_mtr_meter_profile *profile, 488 struct rte_mtr_error *error) 489 { 490 struct mlx5_priv *priv = dev->data->dev_private; 491 struct mlx5_flow_meter_profile *fmp; 492 union mlx5_l3t_data data; 493 int ret; 494 495 if (!priv->mtr_en) 496 return -rte_mtr_error_set(error, ENOTSUP, 497 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 498 "Meter is not supported"); 499 /* Check input params. */ 500 ret = mlx5_flow_meter_profile_validate(dev, meter_profile_id, 501 profile, error); 502 if (ret) 503 return ret; 504 /* Meter profile memory allocation. */ 505 fmp = mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct mlx5_flow_meter_profile), 506 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 507 if (fmp == NULL) 508 return -rte_mtr_error_set(error, ENOMEM, 509 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 510 NULL, "Meter profile memory " 511 "alloc failed."); 512 /* Fill profile info. */ 513 fmp->id = meter_profile_id; 514 fmp->profile = *profile; 515 /* Fill the flow meter parameters for the PRM. */ 516 ret = mlx5_flow_meter_param_fill(fmp, error); 517 if (ret) 518 goto error; 519 data.ptr = fmp; 520 ret = mlx5_l3t_set_entry(priv->mtr_profile_tbl, 521 meter_profile_id, &data); 522 if (ret) 523 return -rte_mtr_error_set(error, ENOTSUP, 524 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 525 NULL, "Meter profile insert fail."); 526 return 0; 527 error: 528 mlx5_free(fmp); 529 return ret; 530 } 531 532 /** 533 * Callback to delete MTR profile. 534 * 535 * @param[in] dev 536 * Pointer to Ethernet device. 537 * @param[in] meter_profile_id 538 * Meter profile id. 539 * @param[out] error 540 * Pointer to the error structure. 541 * 542 * @return 543 * 0 on success, a negative errno value otherwise and rte_errno is set. 544 */ 545 static int 546 mlx5_flow_meter_profile_delete(struct rte_eth_dev *dev, 547 uint32_t meter_profile_id, 548 struct rte_mtr_error *error) 549 { 550 struct mlx5_priv *priv = dev->data->dev_private; 551 struct mlx5_flow_meter_profile *fmp; 552 553 if (!priv->mtr_en) 554 return -rte_mtr_error_set(error, ENOTSUP, 555 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 556 "Meter is not supported"); 557 /* Meter profile must exist. */ 558 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 559 if (fmp == NULL) 560 return -rte_mtr_error_set(error, ENOENT, 561 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 562 &meter_profile_id, 563 "Meter profile id is invalid."); 564 /* Check profile is unused. */ 565 if (fmp->ref_cnt) 566 return -rte_mtr_error_set(error, EBUSY, 567 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 568 NULL, "Meter profile is in use."); 569 if (mlx5_l3t_clear_entry(priv->mtr_profile_tbl, meter_profile_id)) 570 return -rte_mtr_error_set(error, EBUSY, 571 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 572 NULL, "Meter profile remove fail."); 573 mlx5_free(fmp); 574 return 0; 575 } 576 577 /** 578 * Find policy by id. 579 * 580 * @param[in] dev 581 * Pointer to Ethernet device. 582 * @param policy_id 583 * Policy id. 584 * 585 * @return 586 * Pointer to the policy found on success, NULL otherwise. 587 */ 588 struct mlx5_flow_meter_policy * 589 mlx5_flow_meter_policy_find(struct rte_eth_dev *dev, 590 uint32_t policy_id, 591 uint32_t *policy_idx) 592 { 593 struct mlx5_priv *priv = dev->data->dev_private; 594 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 595 union mlx5_l3t_data data; 596 597 if (policy_id > MLX5_MAX_SUB_POLICY_TBL_NUM || !priv->policy_idx_tbl) 598 return NULL; 599 if (mlx5_l3t_get_entry(priv->policy_idx_tbl, policy_id, &data) || 600 !data.dword) 601 return NULL; 602 if (policy_idx) 603 *policy_idx = data.dword; 604 sub_policy = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 605 data.dword); 606 /* Remove reference taken by the mlx5_l3t_get_entry. */ 607 mlx5_l3t_clear_entry(priv->policy_idx_tbl, policy_id); 608 if (sub_policy) 609 if (sub_policy->main_policy_id) 610 return sub_policy->main_policy; 611 return NULL; 612 } 613 614 /** 615 * Get the next meter from one meter's policy in hierarchy chain. 616 * Lock free, mutex should be acquired by caller. 617 * 618 * @param[in] priv 619 * Pointer to mlx5_priv. 620 * @param[in] policy 621 * Pointer to flow meter policy. 622 * @param[out] mtr_idx 623 * Pointer to Meter index. 624 * 625 * @return 626 * Pointer to the next meter, or NULL when fail. 627 */ 628 struct mlx5_flow_meter_info * 629 mlx5_flow_meter_hierarchy_next_meter(struct mlx5_priv *priv, 630 struct mlx5_flow_meter_policy *policy, 631 uint32_t *mtr_idx) 632 { 633 int i; 634 635 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 636 if (policy->act_cnt[i].fate_action == MLX5_FLOW_FATE_MTR) 637 return mlx5_flow_meter_find(priv, 638 policy->act_cnt[i].next_mtr_id, 639 mtr_idx); 640 } 641 return NULL; 642 } 643 644 /** 645 * Get the last meter's policy from one meter's policy in hierarchy. 646 * 647 * @param[in] dev 648 * Pointer to Ethernet device. 649 * @param[in] policy 650 * Pointer to flow meter policy. 651 * 652 * @return 653 * Pointer to the final meter's policy, or NULL when fail. 654 */ 655 struct mlx5_flow_meter_policy * 656 mlx5_flow_meter_hierarchy_get_final_policy(struct rte_eth_dev *dev, 657 struct mlx5_flow_meter_policy *policy) 658 { 659 struct mlx5_priv *priv = dev->data->dev_private; 660 struct mlx5_flow_meter_info *next_fm; 661 struct mlx5_flow_meter_policy *next_policy = policy; 662 663 while (next_policy->is_hierarchy) { 664 rte_spinlock_lock(&next_policy->sl); 665 next_fm = mlx5_flow_meter_hierarchy_next_meter(priv, next_policy, NULL); 666 rte_spinlock_unlock(&next_policy->sl); 667 if (!next_fm || next_fm->def_policy) 668 return NULL; 669 next_policy = mlx5_flow_meter_policy_find(dev, 670 next_fm->policy_id, NULL); 671 MLX5_ASSERT(next_policy); 672 } 673 return next_policy; 674 } 675 676 /** 677 * Callback to check MTR policy action validate 678 * 679 * @param[in] dev 680 * Pointer to Ethernet device. 681 * @param[in] actions 682 * Pointer to meter policy action detail. 683 * @param[out] error 684 * Pointer to the error structure. 685 * 686 * @return 687 * 0 on success, a negative errno value otherwise and rte_errno is set. 688 */ 689 static int 690 mlx5_flow_meter_policy_validate(struct rte_eth_dev *dev, 691 struct rte_mtr_meter_policy_params *policy, 692 struct rte_mtr_error *error) 693 { 694 struct mlx5_priv *priv = dev->data->dev_private; 695 struct rte_flow_attr attr = { .transfer = priv->sh->config.dv_esw_en ? 696 1 : 0 }; 697 bool is_rss = false; 698 uint8_t policy_mode; 699 uint8_t domain_bitmap; 700 int ret; 701 702 if (!priv->mtr_en || !priv->sh->meter_aso_en) 703 return -rte_mtr_error_set(error, ENOTSUP, 704 RTE_MTR_ERROR_TYPE_METER_POLICY, 705 NULL, "meter policy unsupported."); 706 ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr, 707 &is_rss, &domain_bitmap, &policy_mode, error); 708 if (ret) 709 return ret; 710 return 0; 711 } 712 713 static int 714 __mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev, 715 uint32_t policy_id, 716 struct mlx5_flow_meter_policy *mtr_policy, 717 struct rte_mtr_error *error, 718 bool clear_l3t) 719 { 720 struct mlx5_priv *priv = dev->data->dev_private; 721 struct mlx5_flow_meter_sub_policy *sub_policy; 722 uint32_t i, j; 723 uint16_t sub_policy_num; 724 725 rte_spinlock_lock(&mtr_policy->sl); 726 if (mtr_policy->ref_cnt) { 727 rte_spinlock_unlock(&mtr_policy->sl); 728 return -rte_mtr_error_set(error, EBUSY, 729 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 730 NULL, 731 "Meter policy object is being used."); 732 } 733 mlx5_flow_destroy_policy_rules(dev, mtr_policy); 734 mlx5_flow_destroy_mtr_acts(dev, mtr_policy); 735 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 736 sub_policy_num = (mtr_policy->sub_policy_num >> 737 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) & 738 MLX5_MTR_SUB_POLICY_NUM_MASK; 739 if (sub_policy_num) { 740 for (j = 0; j < sub_policy_num; j++) { 741 sub_policy = mtr_policy->sub_policys[i][j]; 742 if (sub_policy) 743 mlx5_ipool_free 744 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 745 sub_policy->idx); 746 } 747 } 748 } 749 if (priv->policy_idx_tbl && clear_l3t) { 750 if (mlx5_l3t_clear_entry(priv->policy_idx_tbl, policy_id)) { 751 rte_spinlock_unlock(&mtr_policy->sl); 752 return -rte_mtr_error_set(error, ENOTSUP, 753 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL, 754 "Fail to delete policy in index table."); 755 } 756 } 757 rte_spinlock_unlock(&mtr_policy->sl); 758 return 0; 759 } 760 761 /** 762 * Callback to add MTR policy. 763 * 764 * @param[in] dev 765 * Pointer to Ethernet device. 766 * @param[out] policy_id 767 * Pointer to policy id 768 * @param[in] actions 769 * Pointer to meter policy action detail. 770 * @param[out] error 771 * Pointer to the error structure. 772 * 773 * @return 774 * 0 on success, a negative errno value otherwise and rte_errno is set. 775 */ 776 static int 777 mlx5_flow_meter_policy_add(struct rte_eth_dev *dev, 778 uint32_t policy_id, 779 struct rte_mtr_meter_policy_params *policy, 780 struct rte_mtr_error *error) 781 { 782 struct mlx5_priv *priv = dev->data->dev_private; 783 struct rte_flow_attr attr = { .transfer = priv->sh->config.dv_esw_en ? 784 1 : 0 }; 785 uint32_t sub_policy_idx = 0; 786 uint32_t policy_idx = 0; 787 struct mlx5_flow_meter_policy *mtr_policy = NULL; 788 struct mlx5_flow_meter_sub_policy *sub_policy; 789 bool is_rss = false; 790 uint8_t policy_mode; 791 uint32_t i; 792 int ret; 793 uint32_t policy_size = sizeof(struct mlx5_flow_meter_policy); 794 uint16_t sub_policy_num; 795 uint8_t domain_bitmap = 0; 796 union mlx5_l3t_data data; 797 bool skip_rule = false; 798 799 if (!priv->mtr_en) 800 return -rte_mtr_error_set(error, ENOTSUP, 801 RTE_MTR_ERROR_TYPE_METER_POLICY, 802 NULL, "meter policy unsupported. "); 803 if (policy_id == MLX5_INVALID_POLICY_ID) 804 return -rte_mtr_error_set(error, ENOTSUP, 805 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 806 NULL, "policy ID is invalid. "); 807 if (policy_id == priv->sh->mtrmng->def_policy_id) 808 return -rte_mtr_error_set(error, EEXIST, 809 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 810 NULL, "default policy ID exists. "); 811 mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx); 812 if (mtr_policy) 813 return -rte_mtr_error_set(error, EEXIST, 814 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 815 NULL, "policy ID exists. "); 816 ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr, 817 &is_rss, &domain_bitmap, 818 &policy_mode, error); 819 if (ret) 820 return ret; 821 if (!domain_bitmap) 822 return -rte_mtr_error_set(error, ENOTSUP, 823 RTE_MTR_ERROR_TYPE_METER_POLICY, 824 NULL, "fail to find policy domain."); 825 if (policy_mode == MLX5_MTR_POLICY_MODE_DEF) { 826 if (priv->sh->mtrmng->def_policy_id != MLX5_INVALID_POLICY_ID) 827 return -rte_mtr_error_set(error, EEXIST, 828 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 829 NULL, "a policy with similar actions " 830 "is already configured"); 831 if (mlx5_flow_create_def_policy(dev)) 832 return -rte_mtr_error_set(error, ENOTSUP, 833 RTE_MTR_ERROR_TYPE_METER_POLICY, 834 NULL, 835 "fail to create non-terminated policy."); 836 priv->sh->mtrmng->def_policy_id = policy_id; 837 return 0; 838 } 839 if (!priv->sh->meter_aso_en) 840 return -rte_mtr_error_set(error, ENOTSUP, 841 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL, 842 "no ASO capability to support the policy "); 843 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 844 if (!(domain_bitmap & (1 << i))) 845 continue; 846 /* 847 * If RSS is found, it means that only the ingress domain can 848 * be supported. It is invalid to support RSS for one color 849 * and egress / transfer domain actions for another. Drop and 850 * jump action should have no impact. 851 */ 852 if (is_rss) { 853 policy_size += 854 sizeof(struct mlx5_flow_meter_sub_policy *) * 855 MLX5_MTR_RSS_MAX_SUB_POLICY; 856 break; 857 } 858 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *); 859 } 860 mtr_policy = mlx5_malloc(MLX5_MEM_ZERO, policy_size, 861 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 862 if (!mtr_policy) 863 return -rte_mtr_error_set(error, ENOMEM, 864 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL, 865 "Memory alloc failed for meter policy."); 866 if (policy_mode == MLX5_MTR_POLICY_MODE_OG) 867 mtr_policy->skip_y = 1; 868 else if (policy_mode == MLX5_MTR_POLICY_MODE_OY) 869 mtr_policy->skip_g = 1; 870 policy_size = sizeof(struct mlx5_flow_meter_policy); 871 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 872 if (!(domain_bitmap & (1 << i))) 873 continue; 874 if (i == MLX5_MTR_DOMAIN_INGRESS) 875 mtr_policy->ingress = 1; 876 if (i == MLX5_MTR_DOMAIN_EGRESS) 877 mtr_policy->egress = 1; 878 if (i == MLX5_MTR_DOMAIN_TRANSFER) 879 mtr_policy->transfer = 1; 880 sub_policy = mlx5_ipool_zmalloc 881 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 882 &sub_policy_idx); 883 if (!sub_policy || 884 sub_policy_idx > MLX5_MAX_SUB_POLICY_TBL_NUM) 885 goto policy_add_err; 886 sub_policy->idx = sub_policy_idx; 887 sub_policy->main_policy = mtr_policy; 888 if (!policy_idx) { 889 policy_idx = sub_policy_idx; 890 sub_policy->main_policy_id = 1; 891 } 892 mtr_policy->sub_policys[i] = 893 (struct mlx5_flow_meter_sub_policy **) 894 ((uint8_t *)mtr_policy + policy_size); 895 mtr_policy->sub_policys[i][0] = sub_policy; 896 sub_policy_num = (mtr_policy->sub_policy_num >> 897 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) & 898 MLX5_MTR_SUB_POLICY_NUM_MASK; 899 sub_policy_num++; 900 mtr_policy->sub_policy_num &= ~(MLX5_MTR_SUB_POLICY_NUM_MASK << 901 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)); 902 mtr_policy->sub_policy_num |= 903 (sub_policy_num & MLX5_MTR_SUB_POLICY_NUM_MASK) << 904 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i); 905 /* 906 * If RSS is found, it means that only the ingress domain can 907 * be supported. It is invalid to support RSS for one color 908 * and egress / transfer domain actions for another. Drop and 909 * jump action should have no impact. 910 */ 911 if (is_rss) { 912 mtr_policy->is_rss = 1; 913 break; 914 } 915 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *); 916 } 917 rte_spinlock_init(&mtr_policy->sl); 918 ret = mlx5_flow_create_mtr_acts(dev, mtr_policy, 919 policy->actions, &attr, error); 920 if (ret) 921 goto policy_add_err; 922 if (mtr_policy->is_hierarchy) { 923 struct mlx5_flow_meter_policy *final_policy; 924 925 final_policy = 926 mlx5_flow_meter_hierarchy_get_final_policy(dev, mtr_policy); 927 if (!final_policy) 928 goto policy_add_err; 929 skip_rule = (final_policy->is_rss || final_policy->is_queue); 930 } 931 /* 932 * If either Green or Yellow has queue / RSS action, all the policy 933 * rules will be created later in the flow splitting stage. 934 */ 935 if (!is_rss && !mtr_policy->is_queue && !skip_rule) { 936 /* Create policy rules in HW. */ 937 ret = mlx5_flow_create_policy_rules(dev, mtr_policy); 938 if (ret) 939 goto policy_add_err; 940 } 941 data.dword = policy_idx; 942 if (!priv->policy_idx_tbl) { 943 priv->policy_idx_tbl = mlx5_l3t_create(MLX5_L3T_TYPE_DWORD); 944 if (!priv->policy_idx_tbl) 945 goto policy_add_err; 946 } 947 if (mlx5_l3t_set_entry(priv->policy_idx_tbl, policy_id, &data)) 948 goto policy_add_err; 949 return 0; 950 policy_add_err: 951 if (mtr_policy) { 952 ret = __mlx5_flow_meter_policy_delete(dev, policy_id, 953 mtr_policy, error, false); 954 mlx5_free(mtr_policy); 955 if (ret) 956 return ret; 957 } 958 return -rte_mtr_error_set(error, ENOTSUP, 959 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 960 NULL, "Failed to create devx policy."); 961 } 962 963 /** 964 * Callback to delete MTR policy. 965 * 966 * @param[in] dev 967 * Pointer to Ethernet device. 968 * @param[in] policy_id 969 * Meter policy id. 970 * @param[out] error 971 * Pointer to the error structure. 972 * 973 * @return 974 * 0 on success, a negative errno value otherwise and rte_errno is set. 975 */ 976 static int 977 mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev, 978 uint32_t policy_id, 979 struct rte_mtr_error *error) 980 { 981 struct mlx5_priv *priv = dev->data->dev_private; 982 struct mlx5_flow_meter_policy *mtr_policy; 983 uint32_t policy_idx; 984 int ret; 985 986 if (policy_id == priv->sh->mtrmng->def_policy_id) { 987 if (priv->sh->mtrmng->def_policy_ref_cnt > 0) 988 return -rte_mtr_error_set(error, ENOTSUP, 989 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL, 990 "Meter policy object is being used."); 991 priv->sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID; 992 return 0; 993 } 994 mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx); 995 if (!mtr_policy) 996 return -rte_mtr_error_set(error, ENOTSUP, 997 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL, 998 "Meter policy id is invalid. "); 999 ret = __mlx5_flow_meter_policy_delete(dev, policy_id, mtr_policy, 1000 error, true); 1001 if (ret) 1002 return ret; 1003 mlx5_free(mtr_policy); 1004 return 0; 1005 } 1006 1007 /** 1008 * Check meter validation. 1009 * 1010 * @param[in] priv 1011 * Pointer to mlx5 private data structure. 1012 * @param[in] meter_id 1013 * Meter id. 1014 * @param[in] params 1015 * Pointer to rte meter parameters. 1016 * @param[out] error 1017 * Pointer to rte meter error structure. 1018 * 1019 * @return 1020 * 0 on success, a negative errno value otherwise and rte_errno is set. 1021 */ 1022 static int 1023 mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id, 1024 struct rte_mtr_params *params, 1025 struct rte_mtr_error *error) 1026 { 1027 /* Meter must use global drop action. */ 1028 if (!priv->sh->dr_drop_action) 1029 return -rte_mtr_error_set(error, ENOTSUP, 1030 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 1031 NULL, 1032 "No drop action ready for meter."); 1033 /* Meter params must not be NULL. */ 1034 if (params == NULL) 1035 return -rte_mtr_error_set(error, EINVAL, 1036 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 1037 NULL, "Meter object params null."); 1038 /* Previous meter color is not supported. */ 1039 if (params->use_prev_mtr_color && !priv->sh->meter_aso_en) 1040 return -rte_mtr_error_set(error, ENOTSUP, 1041 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 1042 NULL, 1043 "Previous meter color " 1044 "not supported."); 1045 if (params->meter_policy_id == MLX5_INVALID_POLICY_ID) 1046 return -rte_mtr_error_set(error, ENOENT, 1047 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1048 NULL, "Meter policy id not valid."); 1049 /* Validate meter id. */ 1050 if (mlx5_flow_meter_find(priv, meter_id, NULL)) 1051 return -rte_mtr_error_set(error, EEXIST, 1052 RTE_MTR_ERROR_TYPE_MTR_ID, NULL, 1053 "Meter object already exists."); 1054 return 0; 1055 } 1056 1057 /** 1058 * Modify the flow meter action. 1059 * 1060 * @param[in] priv 1061 * Pointer to mlx5 private data structure. 1062 * @param[in] fm 1063 * Pointer to flow meter to be modified. 1064 * @param[in] srtcm 1065 * Pointer to meter srtcm description parameter. 1066 * @param[in] modify_bits 1067 * The bit in srtcm to be updated. 1068 * @param[in] active_state 1069 * The state to be updated. 1070 * @return 1071 * 0 on success, o negative value otherwise. 1072 */ 1073 static int 1074 mlx5_flow_meter_action_modify(struct mlx5_priv *priv, 1075 struct mlx5_flow_meter_info *fm, 1076 const struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm, 1077 uint64_t modify_bits, uint32_t active_state, uint32_t is_enable) 1078 { 1079 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER 1080 uint32_t in[MLX5_ST_SZ_DW(flow_meter_parameters)] = { 0 }; 1081 uint32_t *attr; 1082 struct mlx5dv_dr_flow_meter_attr mod_attr = { 0 }; 1083 int ret; 1084 struct mlx5_aso_mtr *aso_mtr = NULL; 1085 uint32_t cbs_cir, ebs_eir, val; 1086 1087 if (priv->sh->meter_aso_en) { 1088 fm->is_enable = !!is_enable; 1089 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm); 1090 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr); 1091 if (ret) 1092 return ret; 1093 ret = mlx5_aso_mtr_wait(priv->sh, aso_mtr); 1094 if (ret) 1095 return ret; 1096 } else { 1097 /* Fill command parameters. */ 1098 mod_attr.reg_c_index = priv->mtr_color_reg - REG_C_0; 1099 mod_attr.flow_meter_parameter = in; 1100 mod_attr.flow_meter_parameter_sz = 1101 MLX5_ST_SZ_BYTES(flow_meter_parameters); 1102 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE) 1103 mod_attr.active = !!active_state; 1104 else 1105 mod_attr.active = 0; 1106 attr = in; 1107 cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir); 1108 ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir); 1109 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS) { 1110 val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & 1111 ASO_DSEG_EXP_MASK; 1112 MLX5_SET(flow_meter_parameters, attr, 1113 cbs_exponent, val); 1114 val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & 1115 ASO_DSEG_MAN_MASK; 1116 MLX5_SET(flow_meter_parameters, attr, 1117 cbs_mantissa, val); 1118 } 1119 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR) { 1120 val = (cbs_cir >> ASO_DSEG_XIR_EXP_OFFSET) & 1121 ASO_DSEG_EXP_MASK; 1122 MLX5_SET(flow_meter_parameters, attr, 1123 cir_exponent, val); 1124 val = cbs_cir & ASO_DSEG_MAN_MASK; 1125 MLX5_SET(flow_meter_parameters, attr, 1126 cir_mantissa, val); 1127 } 1128 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_EBS) { 1129 val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & 1130 ASO_DSEG_EXP_MASK; 1131 MLX5_SET(flow_meter_parameters, attr, 1132 ebs_exponent, val); 1133 val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & 1134 ASO_DSEG_MAN_MASK; 1135 MLX5_SET(flow_meter_parameters, attr, 1136 ebs_mantissa, val); 1137 } 1138 /* Apply modifications to meter only if it was created. */ 1139 if (fm->meter_action_g) { 1140 ret = mlx5_glue->dv_modify_flow_action_meter 1141 (fm->meter_action_g, &mod_attr, 1142 rte_cpu_to_be_64(modify_bits)); 1143 if (ret) 1144 return ret; 1145 } 1146 /* Update succeeded modify meter parameters. */ 1147 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE) 1148 fm->active_state = !!active_state; 1149 } 1150 return 0; 1151 #else 1152 (void)priv; 1153 (void)fm; 1154 (void)srtcm; 1155 (void)modify_bits; 1156 (void)active_state; 1157 (void)is_enable; 1158 return -ENOTSUP; 1159 #endif 1160 } 1161 1162 static int 1163 mlx5_flow_meter_stats_enable_update(struct rte_eth_dev *dev, 1164 struct mlx5_flow_meter_info *fm, 1165 uint64_t stats_mask) 1166 { 1167 fm->bytes_dropped = 1168 (stats_mask & RTE_MTR_STATS_N_BYTES_DROPPED) ? 1 : 0; 1169 fm->pkts_dropped = (stats_mask & RTE_MTR_STATS_N_PKTS_DROPPED) ? 1 : 0; 1170 if (fm->bytes_dropped || fm->pkts_dropped) { 1171 if (!fm->drop_cnt) { 1172 /* Alloc policer counters. */ 1173 fm->drop_cnt = mlx5_counter_alloc(dev); 1174 if (!fm->drop_cnt) 1175 return -1; 1176 } 1177 } else { 1178 if (fm->drop_cnt) { 1179 mlx5_counter_free(dev, fm->drop_cnt); 1180 fm->drop_cnt = 0; 1181 } 1182 } 1183 return 0; 1184 } 1185 1186 /** 1187 * Create meter rules. 1188 * 1189 * @param[in] dev 1190 * Pointer to Ethernet device. 1191 * @param[in] meter_id 1192 * Meter id. 1193 * @param[in] params 1194 * Pointer to rte meter parameters. 1195 * @param[in] shared 1196 * Meter shared with other flow or not. 1197 * @param[out] error 1198 * Pointer to rte meter error structure. 1199 * 1200 * @return 1201 * 0 on success, a negative errno value otherwise and rte_errno is set. 1202 */ 1203 static int 1204 mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, 1205 struct rte_mtr_params *params, int shared, 1206 struct rte_mtr_error *error) 1207 { 1208 struct mlx5_priv *priv = dev->data->dev_private; 1209 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; 1210 struct mlx5_flow_meter_profile *fmp; 1211 struct mlx5_flow_meter_info *fm; 1212 /* GCC fails to infer legacy_fm is set when !priv->sh->meter_aso_en. */ 1213 struct mlx5_legacy_flow_meter *legacy_fm = NULL; 1214 struct mlx5_flow_meter_policy *mtr_policy = NULL; 1215 struct mlx5_indexed_pool_config flow_ipool_cfg = { 1216 .size = 0, 1217 .trunk_size = 64, 1218 .need_lock = 1, 1219 .type = "mlx5_flow_mtr_flow_id_pool", 1220 }; 1221 struct mlx5_aso_mtr *aso_mtr; 1222 uint32_t mtr_idx, policy_idx; 1223 union mlx5_l3t_data data; 1224 int ret; 1225 uint8_t domain_bitmap; 1226 uint8_t mtr_id_bits; 1227 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 1228 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 1229 1230 if (!priv->mtr_en) 1231 return -rte_mtr_error_set(error, ENOTSUP, 1232 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1233 "Meter is not supported"); 1234 /* Validate the parameters. */ 1235 ret = mlx5_flow_meter_validate(priv, meter_id, params, error); 1236 if (ret) 1237 return ret; 1238 /* Meter profile must exist. */ 1239 fmp = mlx5_flow_meter_profile_find(priv, params->meter_profile_id); 1240 if (fmp == NULL) 1241 return -rte_mtr_error_set(error, ENOENT, 1242 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 1243 NULL, "Meter profile id not valid."); 1244 /* Meter policy must exist. */ 1245 if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) { 1246 __atomic_add_fetch 1247 (&priv->sh->mtrmng->def_policy_ref_cnt, 1248 1, __ATOMIC_RELAXED); 1249 domain_bitmap = MLX5_MTR_ALL_DOMAIN_BIT; 1250 if (!priv->sh->config.dv_esw_en) 1251 domain_bitmap &= ~MLX5_MTR_DOMAIN_TRANSFER_BIT; 1252 } else { 1253 if (!priv->sh->meter_aso_en) 1254 return -rte_mtr_error_set(error, ENOTSUP, 1255 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1256 "Part of the policies cannot be " 1257 "supported without ASO "); 1258 mtr_policy = mlx5_flow_meter_policy_find(dev, 1259 params->meter_policy_id, &policy_idx); 1260 if (!mtr_policy) 1261 return -rte_mtr_error_set(error, ENOENT, 1262 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1263 NULL, "Meter policy id not valid."); 1264 domain_bitmap = (mtr_policy->ingress ? 1265 MLX5_MTR_DOMAIN_INGRESS_BIT : 0) | 1266 (mtr_policy->egress ? 1267 MLX5_MTR_DOMAIN_EGRESS_BIT : 0) | 1268 (mtr_policy->transfer ? 1269 MLX5_MTR_DOMAIN_TRANSFER_BIT : 0); 1270 if (fmp->g_support && mtr_policy->skip_g) 1271 return -rte_mtr_error_set(error, ENOTSUP, 1272 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1273 NULL, "Meter green policy is empty."); 1274 if (fmp->y_support && mtr_policy->skip_y) 1275 return -rte_mtr_error_set(error, ENOTSUP, 1276 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1277 NULL, "Meter yellow policy is empty."); 1278 } 1279 /* Allocate the flow meter memory. */ 1280 if (priv->sh->meter_aso_en) { 1281 mtr_idx = mlx5_flow_mtr_alloc(dev); 1282 if (!mtr_idx) 1283 return -rte_mtr_error_set(error, ENOMEM, 1284 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1285 "Memory alloc failed for meter."); 1286 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx); 1287 fm = &aso_mtr->fm; 1288 } else { 1289 if (fmp->y_support) 1290 return -rte_mtr_error_set(error, ENOMEM, 1291 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1292 "Unsupported profile with yellow."); 1293 legacy_fm = mlx5_ipool_zmalloc 1294 (priv->sh->ipool[MLX5_IPOOL_MTR], &mtr_idx); 1295 if (legacy_fm == NULL) 1296 return -rte_mtr_error_set(error, ENOMEM, 1297 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1298 "Memory alloc failed for meter."); 1299 legacy_fm->idx = mtr_idx; 1300 fm = &legacy_fm->fm; 1301 } 1302 mtr_id_bits = MLX5_REG_BITS - __builtin_clz(mtr_idx); 1303 if ((mtr_id_bits + priv->sh->mtrmng->max_mtr_flow_bits) > 1304 mtr_reg_bits) { 1305 DRV_LOG(ERR, "Meter number exceeds max limit."); 1306 goto error; 1307 } 1308 if (mtr_id_bits > priv->sh->mtrmng->max_mtr_bits) 1309 priv->sh->mtrmng->max_mtr_bits = mtr_id_bits; 1310 /* Fill the flow meter parameters. */ 1311 fm->meter_id = meter_id; 1312 fm->policy_id = params->meter_policy_id; 1313 fm->profile = fmp; 1314 if (mlx5_flow_meter_stats_enable_update(dev, fm, params->stats_mask)) 1315 goto error; 1316 if (mlx5_flow_create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap)) 1317 goto error; 1318 /* Add to the flow meter list. */ 1319 if (!priv->sh->meter_aso_en) { 1320 MLX5_ASSERT(legacy_fm != NULL); 1321 TAILQ_INSERT_TAIL(fms, legacy_fm, next); 1322 } 1323 /* Add to the flow meter list. */ 1324 fm->active_state = 1; /* Config meter starts as active. */ 1325 fm->is_enable = params->meter_enable; 1326 fm->shared = !!shared; 1327 fm->color_aware = !!params->use_prev_mtr_color; 1328 __atomic_add_fetch(&fm->profile->ref_cnt, 1, __ATOMIC_RELAXED); 1329 if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) { 1330 fm->def_policy = 1; 1331 fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg); 1332 if (!fm->flow_ipool) 1333 goto error; 1334 } 1335 rte_spinlock_init(&fm->sl); 1336 /* If ASO meter supported, update ASO flow meter by wqe. */ 1337 if (priv->sh->meter_aso_en) { 1338 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm); 1339 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr); 1340 if (ret) 1341 goto error; 1342 if (!priv->mtr_idx_tbl) { 1343 priv->mtr_idx_tbl = 1344 mlx5_l3t_create(MLX5_L3T_TYPE_DWORD); 1345 if (!priv->mtr_idx_tbl) 1346 goto error; 1347 } 1348 data.dword = mtr_idx; 1349 if (mlx5_l3t_set_entry(priv->mtr_idx_tbl, meter_id, &data)) 1350 goto error; 1351 } else if (!params->meter_enable && mlx5_flow_meter_disable(dev, meter_id, error)) { 1352 goto error; 1353 } 1354 fm->active_state = params->meter_enable; 1355 if (mtr_policy) 1356 __atomic_add_fetch(&mtr_policy->ref_cnt, 1, __ATOMIC_RELAXED); 1357 return 0; 1358 error: 1359 mlx5_flow_destroy_mtr_tbls(dev, fm); 1360 /* Free policer counters. */ 1361 if (fm->drop_cnt) 1362 mlx5_counter_free(dev, fm->drop_cnt); 1363 if (priv->sh->meter_aso_en) 1364 mlx5_flow_mtr_free(dev, mtr_idx); 1365 else 1366 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], mtr_idx); 1367 return -rte_mtr_error_set(error, ENOTSUP, 1368 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 1369 NULL, "Failed to create devx meter."); 1370 } 1371 1372 static int 1373 mlx5_flow_meter_params_flush(struct rte_eth_dev *dev, 1374 struct mlx5_flow_meter_info *fm, 1375 uint32_t mtr_idx) 1376 { 1377 struct mlx5_priv *priv = dev->data->dev_private; 1378 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; 1379 struct mlx5_flow_meter_profile *fmp; 1380 struct mlx5_legacy_flow_meter *legacy_fm = NULL; 1381 struct mlx5_flow_meter_policy *mtr_policy; 1382 1383 /* Meter object must not have any owner. */ 1384 MLX5_ASSERT(!fm->ref_cnt); 1385 /* Get meter profile. */ 1386 fmp = fm->profile; 1387 if (fmp == NULL) 1388 return -1; 1389 /* Update dependencies. */ 1390 __atomic_sub_fetch(&fmp->ref_cnt, 1, __ATOMIC_RELAXED); 1391 fm->profile = NULL; 1392 /* Remove from list. */ 1393 if (!priv->sh->meter_aso_en) { 1394 legacy_fm = container_of(fm, 1395 struct mlx5_legacy_flow_meter, fm); 1396 TAILQ_REMOVE(fms, legacy_fm, next); 1397 } 1398 /* Free drop counters. */ 1399 if (fm->drop_cnt) 1400 mlx5_counter_free(dev, fm->drop_cnt); 1401 /* Free meter flow table. */ 1402 if (fm->flow_ipool) { 1403 mlx5_ipool_destroy(fm->flow_ipool); 1404 fm->flow_ipool = 0; 1405 } 1406 mlx5_flow_destroy_mtr_tbls(dev, fm); 1407 if (fm->def_policy) 1408 __atomic_sub_fetch(&priv->sh->mtrmng->def_policy_ref_cnt, 1409 1, __ATOMIC_RELAXED); 1410 if (priv->sh->meter_aso_en) { 1411 if (!fm->def_policy) { 1412 mtr_policy = mlx5_flow_meter_policy_find(dev, 1413 fm->policy_id, NULL); 1414 if (mtr_policy) 1415 __atomic_sub_fetch(&mtr_policy->ref_cnt, 1416 1, __ATOMIC_RELAXED); 1417 fm->policy_id = 0; 1418 } 1419 fm->def_policy = 0; 1420 if (mlx5_l3t_clear_entry(priv->mtr_idx_tbl, fm->meter_id)) 1421 return -1; 1422 mlx5_flow_mtr_free(dev, mtr_idx); 1423 } else { 1424 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], 1425 legacy_fm->idx); 1426 } 1427 return 0; 1428 } 1429 1430 /** 1431 * Destroy meter rules. 1432 * 1433 * @param[in] dev 1434 * Pointer to Ethernet device. 1435 * @param[in] meter_id 1436 * Meter id. 1437 * @param[out] error 1438 * Pointer to rte meter error structure. 1439 * 1440 * @return 1441 * 0 on success, a negative errno value otherwise and rte_errno is set. 1442 */ 1443 static int 1444 mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id, 1445 struct rte_mtr_error *error) 1446 { 1447 struct mlx5_priv *priv = dev->data->dev_private; 1448 struct mlx5_flow_meter_info *fm; 1449 uint32_t mtr_idx = 0; 1450 1451 if (!priv->mtr_en) 1452 return -rte_mtr_error_set(error, ENOTSUP, 1453 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 1454 NULL, 1455 "Meter is not supported"); 1456 /* Meter object must exist. */ 1457 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 1458 if (fm == NULL) 1459 return -rte_mtr_error_set(error, ENOENT, 1460 RTE_MTR_ERROR_TYPE_MTR_ID, 1461 NULL, 1462 "Meter object id not valid."); 1463 /* Meter object must not have any owner. */ 1464 if (fm->ref_cnt > 0) 1465 return -rte_mtr_error_set(error, EBUSY, 1466 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 1467 NULL, 1468 "Meter object is being used."); 1469 /* Destroy the meter profile. */ 1470 if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx)) 1471 return -rte_mtr_error_set(error, EINVAL, 1472 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 1473 NULL, 1474 "MTR object meter profile invalid."); 1475 return 0; 1476 } 1477 1478 /** 1479 * Modify meter state. 1480 * 1481 * @param[in] priv 1482 * Pointer to mlx5 private data structure. 1483 * @param[in] fm 1484 * Pointer to flow meter. 1485 * @param[in] new_state 1486 * New state to update. 1487 * @param[out] error 1488 * Pointer to rte meter error structure. 1489 * 1490 * @return 1491 * 0 on success, a negative errno value otherwise and rte_errno is set. 1492 */ 1493 static int 1494 mlx5_flow_meter_modify_state(struct mlx5_priv *priv, 1495 struct mlx5_flow_meter_info *fm, 1496 uint32_t new_state, 1497 struct rte_mtr_error *error) 1498 { 1499 static const struct mlx5_flow_meter_srtcm_rfc2697_prm srtcm = { 1500 .cbs_cir = RTE_BE32(MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL), 1501 .ebs_eir = 0, 1502 }; 1503 uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS | 1504 MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR; 1505 int ret; 1506 1507 if (new_state == MLX5_FLOW_METER_DISABLE) 1508 ret = mlx5_flow_meter_action_modify(priv, fm, 1509 &srtcm, modify_bits, 0, 0); 1510 else 1511 ret = mlx5_flow_meter_action_modify(priv, fm, 1512 &fm->profile->srtcm_prm, 1513 modify_bits, 0, 1); 1514 if (ret) 1515 return -rte_mtr_error_set(error, -ret, 1516 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 1517 NULL, 1518 new_state ? 1519 "Failed to enable meter." : 1520 "Failed to disable meter."); 1521 return 0; 1522 } 1523 1524 /** 1525 * Callback to enable flow meter. 1526 * 1527 * @param[in] dev 1528 * Pointer to Ethernet device. 1529 * @param[in] meter_id 1530 * Meter id. 1531 * @param[out] error 1532 * Pointer to rte meter error structure. 1533 * 1534 * @return 1535 * 0 on success, a negative errno value otherwise and rte_errno is set. 1536 */ 1537 static int 1538 mlx5_flow_meter_enable(struct rte_eth_dev *dev, 1539 uint32_t meter_id, 1540 struct rte_mtr_error *error) 1541 { 1542 struct mlx5_priv *priv = dev->data->dev_private; 1543 struct mlx5_flow_meter_info *fm; 1544 int ret; 1545 1546 if (!priv->mtr_en) 1547 return -rte_mtr_error_set(error, ENOTSUP, 1548 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1549 "Meter is not supported"); 1550 /* Meter object must exist. */ 1551 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 1552 if (fm == NULL) 1553 return -rte_mtr_error_set(error, ENOENT, 1554 RTE_MTR_ERROR_TYPE_MTR_ID, 1555 NULL, "Meter not found."); 1556 if (fm->active_state == MLX5_FLOW_METER_ENABLE) 1557 return 0; 1558 ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_ENABLE, 1559 error); 1560 if (!ret) 1561 fm->active_state = MLX5_FLOW_METER_ENABLE; 1562 return ret; 1563 } 1564 1565 /** 1566 * Callback to disable flow meter. 1567 * 1568 * @param[in] dev 1569 * Pointer to Ethernet device. 1570 * @param[in] meter_id 1571 * Meter id. 1572 * @param[out] error 1573 * Pointer to rte meter error structure. 1574 * 1575 * @return 1576 * 0 on success, a negative errno value otherwise and rte_errno is set. 1577 */ 1578 static int 1579 mlx5_flow_meter_disable(struct rte_eth_dev *dev, 1580 uint32_t meter_id, 1581 struct rte_mtr_error *error) 1582 { 1583 struct mlx5_priv *priv = dev->data->dev_private; 1584 struct mlx5_flow_meter_info *fm; 1585 int ret; 1586 1587 if (!priv->mtr_en) 1588 return -rte_mtr_error_set(error, ENOTSUP, 1589 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1590 "Meter is not supported"); 1591 /* Meter object must exist. */ 1592 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 1593 if (fm == NULL) 1594 return -rte_mtr_error_set(error, ENOENT, 1595 RTE_MTR_ERROR_TYPE_MTR_ID, 1596 NULL, "Meter not found."); 1597 if (fm->active_state == MLX5_FLOW_METER_DISABLE) 1598 return 0; 1599 ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_DISABLE, 1600 error); 1601 if (!ret) 1602 fm->active_state = MLX5_FLOW_METER_DISABLE; 1603 return ret; 1604 } 1605 1606 /** 1607 * Callback to update meter profile. 1608 * 1609 * @param[in] dev 1610 * Pointer to Ethernet device. 1611 * @param[in] meter_id 1612 * Meter id. 1613 * @param[in] meter_profile_id 1614 * To be updated meter profile id. 1615 * @param[out] error 1616 * Pointer to rte meter error structure. 1617 * 1618 * @return 1619 * 0 on success, a negative errno value otherwise and rte_errno is set. 1620 */ 1621 static int 1622 mlx5_flow_meter_profile_update(struct rte_eth_dev *dev, 1623 uint32_t meter_id, 1624 uint32_t meter_profile_id, 1625 struct rte_mtr_error *error) 1626 { 1627 struct mlx5_priv *priv = dev->data->dev_private; 1628 struct mlx5_flow_meter_profile *fmp; 1629 struct mlx5_flow_meter_profile *old_fmp; 1630 struct mlx5_flow_meter_info *fm; 1631 uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS | 1632 MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR; 1633 int ret; 1634 1635 if (!priv->mtr_en) 1636 return -rte_mtr_error_set(error, ENOTSUP, 1637 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1638 "Meter is not supported"); 1639 /* Meter profile must exist. */ 1640 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 1641 if (fmp == NULL) 1642 return -rte_mtr_error_set(error, ENOENT, 1643 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 1644 NULL, "Meter profile not found."); 1645 /* Meter object must exist. */ 1646 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 1647 if (fm == NULL) 1648 return -rte_mtr_error_set(error, ENOENT, 1649 RTE_MTR_ERROR_TYPE_MTR_ID, 1650 NULL, "Meter not found."); 1651 /* MTR object already set to meter profile id. */ 1652 old_fmp = fm->profile; 1653 if (fmp == old_fmp) 1654 return 0; 1655 /* Update the profile. */ 1656 fm->profile = fmp; 1657 /* Update meter params in HW (if not disabled). */ 1658 if (fm->active_state == MLX5_FLOW_METER_DISABLE) 1659 return 0; 1660 ret = mlx5_flow_meter_action_modify(priv, fm, &fm->profile->srtcm_prm, 1661 modify_bits, fm->active_state, 1); 1662 if (ret) { 1663 fm->profile = old_fmp; 1664 return -rte_mtr_error_set(error, -ret, 1665 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 1666 NULL, "Failed to update meter" 1667 " parameters in hardware."); 1668 } 1669 old_fmp->ref_cnt--; 1670 fmp->ref_cnt++; 1671 return 0; 1672 } 1673 1674 /** 1675 * Callback to update meter stats mask. 1676 * 1677 * @param[in] dev 1678 * Pointer to Ethernet device. 1679 * @param[in] meter_id 1680 * Meter id. 1681 * @param[in] stats_mask 1682 * To be updated stats_mask. 1683 * @param[out] error 1684 * Pointer to rte meter error structure. 1685 * 1686 * @return 1687 * 0 on success, a negative errno value otherwise and rte_errno is set. 1688 */ 1689 static int 1690 mlx5_flow_meter_stats_update(struct rte_eth_dev *dev, 1691 uint32_t meter_id, 1692 uint64_t stats_mask, 1693 struct rte_mtr_error *error) 1694 { 1695 struct mlx5_priv *priv = dev->data->dev_private; 1696 struct mlx5_flow_meter_info *fm; 1697 1698 if (!priv->mtr_en) 1699 return -rte_mtr_error_set(error, ENOTSUP, 1700 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1701 "Meter is not supported"); 1702 /* Meter object must exist. */ 1703 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 1704 if (fm == NULL) 1705 return -rte_mtr_error_set(error, ENOENT, 1706 RTE_MTR_ERROR_TYPE_MTR_ID, 1707 NULL, "Meter object id not valid."); 1708 if (mlx5_flow_meter_stats_enable_update(dev, fm, stats_mask)) 1709 return -rte_mtr_error_set(error, ENOENT, 1710 RTE_MTR_ERROR_TYPE_MTR_ID, 1711 NULL, "Fail to allocate " 1712 "counter for meter."); 1713 return 0; 1714 } 1715 1716 /** 1717 * Callback to read meter statistics. 1718 * 1719 * @param[in] dev 1720 * Pointer to Ethernet device. 1721 * @param[in] meter_id 1722 * Meter id. 1723 * @param[out] stats 1724 * Pointer to store the statistics. 1725 * @param[out] stats_mask 1726 * Pointer to store the stats_mask. 1727 * @param[in] clear 1728 * Statistic to be cleared after read or not. 1729 * @param[out] error 1730 * Pointer to rte meter error structure. 1731 * 1732 * @return 1733 * 0 on success, a negative errno value otherwise and rte_errno is set. 1734 */ 1735 static int 1736 mlx5_flow_meter_stats_read(struct rte_eth_dev *dev, 1737 uint32_t meter_id, 1738 struct rte_mtr_stats *stats, 1739 uint64_t *stats_mask, 1740 int clear, 1741 struct rte_mtr_error *error) 1742 { 1743 struct mlx5_priv *priv = dev->data->dev_private; 1744 struct mlx5_flow_meter_info *fm; 1745 uint64_t pkts; 1746 uint64_t bytes; 1747 int ret = 0; 1748 1749 if (!priv->mtr_en) 1750 return -rte_mtr_error_set(error, ENOTSUP, 1751 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1752 "Meter is not supported"); 1753 /* Meter object must exist. */ 1754 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 1755 if (fm == NULL) 1756 return -rte_mtr_error_set(error, ENOENT, 1757 RTE_MTR_ERROR_TYPE_MTR_ID, 1758 NULL, "Meter object id not valid."); 1759 *stats_mask = 0; 1760 if (fm->bytes_dropped) 1761 *stats_mask |= RTE_MTR_STATS_N_BYTES_DROPPED; 1762 if (fm->pkts_dropped) 1763 *stats_mask |= RTE_MTR_STATS_N_PKTS_DROPPED; 1764 memset(stats, 0, sizeof(*stats)); 1765 if (fm->drop_cnt) { 1766 ret = mlx5_counter_query(dev, fm->drop_cnt, clear, &pkts, 1767 &bytes, NULL); 1768 if (ret) 1769 goto error; 1770 /* If need to read the packets, set it. */ 1771 if (fm->pkts_dropped) 1772 stats->n_pkts_dropped = pkts; 1773 /* If need to read the bytes, set it. */ 1774 if (fm->bytes_dropped) 1775 stats->n_bytes_dropped = bytes; 1776 } 1777 return 0; 1778 error: 1779 return -rte_mtr_error_set(error, ret, RTE_MTR_ERROR_TYPE_STATS, NULL, 1780 "Failed to read meter drop counters."); 1781 } 1782 1783 static const struct rte_mtr_ops mlx5_flow_mtr_ops = { 1784 .capabilities_get = mlx5_flow_mtr_cap_get, 1785 .meter_profile_add = mlx5_flow_meter_profile_add, 1786 .meter_profile_delete = mlx5_flow_meter_profile_delete, 1787 .meter_policy_validate = mlx5_flow_meter_policy_validate, 1788 .meter_policy_add = mlx5_flow_meter_policy_add, 1789 .meter_policy_delete = mlx5_flow_meter_policy_delete, 1790 .create = mlx5_flow_meter_create, 1791 .destroy = mlx5_flow_meter_destroy, 1792 .meter_enable = mlx5_flow_meter_enable, 1793 .meter_disable = mlx5_flow_meter_disable, 1794 .meter_profile_update = mlx5_flow_meter_profile_update, 1795 .meter_dscp_table_update = NULL, 1796 .stats_update = mlx5_flow_meter_stats_update, 1797 .stats_read = mlx5_flow_meter_stats_read, 1798 }; 1799 1800 /** 1801 * Get meter operations. 1802 * 1803 * @param dev 1804 * Pointer to Ethernet device structure. 1805 * @param arg 1806 * Pointer to set the mtr operations. 1807 * 1808 * @return 1809 * Always 0. 1810 */ 1811 int 1812 mlx5_flow_meter_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg) 1813 { 1814 *(const struct rte_mtr_ops **)arg = &mlx5_flow_mtr_ops; 1815 return 0; 1816 } 1817 1818 /** 1819 * Find meter by id. 1820 * 1821 * @param priv 1822 * Pointer to mlx5_priv. 1823 * @param meter_id 1824 * Meter id. 1825 * @param mtr_idx 1826 * Pointer to Meter index. 1827 * 1828 * @return 1829 * Pointer to the meter info found on success, NULL otherwise. 1830 */ 1831 struct mlx5_flow_meter_info * 1832 mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id, 1833 uint32_t *mtr_idx) 1834 { 1835 struct mlx5_legacy_flow_meter *legacy_fm; 1836 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; 1837 struct mlx5_aso_mtr *aso_mtr; 1838 struct mlx5_aso_mtr_pools_mng *pools_mng = 1839 &priv->sh->mtrmng->pools_mng; 1840 union mlx5_l3t_data data; 1841 uint16_t n_valid; 1842 1843 if (priv->sh->meter_aso_en) { 1844 rte_rwlock_read_lock(&pools_mng->resize_mtrwl); 1845 n_valid = pools_mng->n_valid; 1846 rte_rwlock_read_unlock(&pools_mng->resize_mtrwl); 1847 if (!n_valid || !priv->mtr_idx_tbl || 1848 (mlx5_l3t_get_entry(priv->mtr_idx_tbl, meter_id, &data) || 1849 !data.dword)) 1850 return NULL; 1851 if (mtr_idx) 1852 *mtr_idx = data.dword; 1853 aso_mtr = mlx5_aso_meter_by_idx(priv, data.dword); 1854 /* Remove reference taken by the mlx5_l3t_get_entry. */ 1855 mlx5_l3t_clear_entry(priv->mtr_idx_tbl, meter_id); 1856 if (!aso_mtr || aso_mtr->state == ASO_METER_FREE) 1857 return NULL; 1858 return &aso_mtr->fm; 1859 } 1860 TAILQ_FOREACH(legacy_fm, fms, next) 1861 if (meter_id == legacy_fm->fm.meter_id) { 1862 if (mtr_idx) 1863 *mtr_idx = legacy_fm->idx; 1864 return &legacy_fm->fm; 1865 } 1866 return NULL; 1867 } 1868 1869 /** 1870 * Find meter by index. 1871 * 1872 * @param priv 1873 * Pointer to mlx5_priv. 1874 * @param idx 1875 * Meter index. 1876 * 1877 * @return 1878 * Pointer to the meter info found on success, NULL otherwise. 1879 */ 1880 struct mlx5_flow_meter_info * 1881 flow_dv_meter_find_by_idx(struct mlx5_priv *priv, uint32_t idx) 1882 { 1883 struct mlx5_aso_mtr *aso_mtr; 1884 1885 if (priv->sh->meter_aso_en) { 1886 aso_mtr = mlx5_aso_meter_by_idx(priv, idx); 1887 if (!aso_mtr) 1888 return NULL; 1889 return &aso_mtr->fm; 1890 } else { 1891 return mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], idx); 1892 } 1893 } 1894 1895 /** 1896 * Attach meter to flow. 1897 * Unidirectional Meter creation can only be done 1898 * when flow direction is known, i.e. when calling meter_attach. 1899 * 1900 * @param [in] priv 1901 * Pointer to mlx5 private data. 1902 * @param[in] fm 1903 * Pointer to flow meter. 1904 * @param [in] attr 1905 * Pointer to flow attributes. 1906 * @param [out] error 1907 * Pointer to error structure. 1908 * 1909 * @return 1910 * 0 on success, a negative errno value otherwise and rte_errno is set. 1911 */ 1912 int 1913 mlx5_flow_meter_attach(struct mlx5_priv *priv, 1914 struct mlx5_flow_meter_info *fm, 1915 const struct rte_flow_attr *attr, 1916 struct rte_flow_error *error) 1917 { 1918 int ret = 0; 1919 1920 if (priv->sh->meter_aso_en) { 1921 struct mlx5_aso_mtr *aso_mtr; 1922 1923 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm); 1924 if (mlx5_aso_mtr_wait(priv->sh, aso_mtr)) { 1925 return rte_flow_error_set(error, ENOENT, 1926 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1927 NULL, 1928 "Timeout in meter configuration"); 1929 } 1930 rte_spinlock_lock(&fm->sl); 1931 if (fm->shared || !fm->ref_cnt) { 1932 fm->ref_cnt++; 1933 } else { 1934 rte_flow_error_set(error, EINVAL, 1935 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 1936 "Meter cannot be shared"); 1937 ret = -1; 1938 } 1939 rte_spinlock_unlock(&fm->sl); 1940 } else { 1941 rte_spinlock_lock(&fm->sl); 1942 if (fm->meter_action_g) { 1943 if (fm->shared && 1944 attr->transfer == fm->transfer && 1945 attr->ingress == fm->ingress && 1946 attr->egress == fm->egress) { 1947 fm->ref_cnt++; 1948 } else { 1949 rte_flow_error_set(error, EINVAL, 1950 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 1951 fm->shared ? 1952 "Meter attr not match." : 1953 "Meter cannot be shared."); 1954 ret = -1; 1955 } 1956 } else { 1957 fm->ingress = attr->ingress; 1958 fm->egress = attr->egress; 1959 fm->transfer = attr->transfer; 1960 fm->ref_cnt = 1; 1961 /* This also creates the meter object. */ 1962 fm->meter_action_g = mlx5_flow_meter_action_create(priv, 1963 fm); 1964 if (!fm->meter_action_g) { 1965 fm->ref_cnt = 0; 1966 fm->ingress = 0; 1967 fm->egress = 0; 1968 fm->transfer = 0; 1969 rte_flow_error_set(error, EINVAL, 1970 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 1971 "Meter action create failed."); 1972 ret = -1; 1973 } 1974 } 1975 rte_spinlock_unlock(&fm->sl); 1976 } 1977 return ret ? -rte_errno : 0; 1978 } 1979 1980 /** 1981 * Detach meter from flow. 1982 * 1983 * @param [in] priv 1984 * Pointer to mlx5 private data. 1985 * @param [in] fm 1986 * Pointer to flow meter. 1987 */ 1988 void 1989 mlx5_flow_meter_detach(struct mlx5_priv *priv, 1990 struct mlx5_flow_meter_info *fm) 1991 { 1992 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER 1993 rte_spinlock_lock(&fm->sl); 1994 MLX5_ASSERT(fm->ref_cnt); 1995 if (--fm->ref_cnt == 0 && !priv->sh->meter_aso_en) { 1996 mlx5_glue->destroy_flow_action(fm->meter_action_g); 1997 fm->meter_action_g = NULL; 1998 fm->ingress = 0; 1999 fm->egress = 0; 2000 fm->transfer = 0; 2001 } 2002 rte_spinlock_unlock(&fm->sl); 2003 #else 2004 (void)priv; 2005 (void)fm; 2006 #endif 2007 } 2008 2009 /** 2010 * Flush meter with Rx queue configuration. 2011 * 2012 * @param[in] dev 2013 * Pointer to Ethernet device. 2014 */ 2015 void 2016 mlx5_flow_meter_rxq_flush(struct rte_eth_dev *dev) 2017 { 2018 struct mlx5_priv *priv = dev->data->dev_private; 2019 struct mlx5_flow_meter_sub_policy *sub_policy; 2020 struct mlx5_flow_meter_policy *mtr_policy; 2021 void *entry; 2022 uint32_t i, policy_idx; 2023 2024 if (!priv->mtr_en) 2025 return; 2026 if (priv->policy_idx_tbl) { 2027 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) { 2028 policy_idx = *(uint32_t *)entry; 2029 sub_policy = mlx5_ipool_get 2030 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 2031 policy_idx); 2032 if (!sub_policy || !sub_policy->main_policy) 2033 continue; 2034 mtr_policy = sub_policy->main_policy; 2035 if (mtr_policy->is_queue || mtr_policy->is_rss) 2036 mlx5_flow_destroy_sub_policy_with_rxq(dev, 2037 mtr_policy); 2038 } 2039 } 2040 } 2041 2042 /** 2043 * Iterate a meter hierarchy and flush all meters and policies if possible. 2044 * 2045 * @param[in] dev 2046 * Pointer to Ethernet device. 2047 * @param[in] fm 2048 * Pointer to flow meter. 2049 * @param[in] mtr_idx 2050 * .Meter's index 2051 * @param[out] error 2052 * Pointer to rte meter error structure. 2053 * 2054 * @return 2055 * 0 on success, a negative errno value otherwise and rte_errno is set. 2056 */ 2057 static int 2058 mlx5_flow_meter_flush_hierarchy(struct rte_eth_dev *dev, 2059 struct mlx5_flow_meter_info *fm, 2060 uint32_t mtr_idx, 2061 struct rte_mtr_error *error) 2062 { 2063 struct mlx5_priv *priv = dev->data->dev_private; 2064 struct mlx5_flow_meter_policy *policy; 2065 uint32_t policy_id; 2066 struct mlx5_flow_meter_info *next_fm; 2067 uint32_t next_mtr_idx; 2068 struct mlx5_flow_meter_policy *next_policy = NULL; 2069 2070 policy = mlx5_flow_meter_policy_find(dev, fm->policy_id, NULL); 2071 MLX5_ASSERT(policy); 2072 while (!fm->ref_cnt && policy->is_hierarchy) { 2073 policy_id = fm->policy_id; 2074 next_fm = mlx5_flow_meter_hierarchy_next_meter(priv, policy, &next_mtr_idx); 2075 if (next_fm) { 2076 next_policy = mlx5_flow_meter_policy_find(dev, 2077 next_fm->policy_id, 2078 NULL); 2079 MLX5_ASSERT(next_policy); 2080 } 2081 if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx)) 2082 return -rte_mtr_error_set(error, ENOTSUP, 2083 RTE_MTR_ERROR_TYPE_MTR_ID, 2084 NULL, 2085 "Failed to flush meter."); 2086 if (policy->ref_cnt) 2087 break; 2088 if (__mlx5_flow_meter_policy_delete(dev, policy_id, 2089 policy, error, true)) 2090 return -rte_errno; 2091 mlx5_free(policy); 2092 if (!next_fm || !next_policy) 2093 break; 2094 fm = next_fm; 2095 mtr_idx = next_mtr_idx; 2096 policy = next_policy; 2097 } 2098 return 0; 2099 } 2100 2101 /** 2102 * Flush all the hierarchy meters and their policies. 2103 * 2104 * @param[in] dev 2105 * Pointer to Ethernet device. 2106 * @param[out] error 2107 * Pointer to rte meter error structure. 2108 * 2109 * @return 2110 * 0 on success, a negative errno value otherwise and rte_errno is set. 2111 */ 2112 static int 2113 mlx5_flow_meter_flush_all_hierarchies(struct rte_eth_dev *dev, 2114 struct rte_mtr_error *error) 2115 { 2116 struct mlx5_priv *priv = dev->data->dev_private; 2117 struct mlx5_flow_meter_info *fm; 2118 struct mlx5_flow_meter_policy *policy; 2119 struct mlx5_flow_meter_sub_policy *sub_policy; 2120 struct mlx5_flow_meter_info *next_fm; 2121 struct mlx5_aso_mtr *aso_mtr; 2122 uint32_t mtr_idx = 0; 2123 uint32_t i, policy_idx; 2124 void *entry; 2125 2126 if (!priv->mtr_idx_tbl || !priv->policy_idx_tbl) 2127 return 0; 2128 MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) { 2129 mtr_idx = *(uint32_t *)entry; 2130 if (!mtr_idx) 2131 continue; 2132 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx); 2133 fm = &aso_mtr->fm; 2134 if (fm->ref_cnt || fm->def_policy) 2135 continue; 2136 if (mlx5_flow_meter_flush_hierarchy(dev, fm, mtr_idx, error)) 2137 return -rte_errno; 2138 } 2139 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) { 2140 policy_idx = *(uint32_t *)entry; 2141 sub_policy = mlx5_ipool_get 2142 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 2143 policy_idx); 2144 if (!sub_policy) 2145 return -rte_mtr_error_set(error, 2146 EINVAL, 2147 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 2148 NULL, "Meter policy invalid."); 2149 policy = sub_policy->main_policy; 2150 if (!policy || !policy->is_hierarchy || policy->ref_cnt) 2151 continue; 2152 next_fm = mlx5_flow_meter_hierarchy_next_meter(priv, policy, &mtr_idx); 2153 if (__mlx5_flow_meter_policy_delete(dev, i, policy, 2154 error, true)) 2155 return -rte_mtr_error_set(error, 2156 EINVAL, 2157 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 2158 NULL, "Meter policy invalid."); 2159 mlx5_free(policy); 2160 if (!next_fm || next_fm->ref_cnt || next_fm->def_policy) 2161 continue; 2162 if (mlx5_flow_meter_flush_hierarchy(dev, next_fm, 2163 mtr_idx, error)) 2164 return -rte_errno; 2165 } 2166 return 0; 2167 } 2168 /** 2169 * Flush meter configuration. 2170 * 2171 * @param[in] dev 2172 * Pointer to Ethernet device. 2173 * @param[out] error 2174 * Pointer to rte meter error structure. 2175 * 2176 * @return 2177 * 0 on success, a negative errno value otherwise and rte_errno is set. 2178 */ 2179 int 2180 mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error) 2181 { 2182 struct mlx5_priv *priv = dev->data->dev_private; 2183 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; 2184 struct mlx5_flow_meter_profile *fmp; 2185 struct mlx5_legacy_flow_meter *legacy_fm; 2186 struct mlx5_flow_meter_info *fm; 2187 struct mlx5_flow_meter_sub_policy *sub_policy; 2188 void *tmp; 2189 uint32_t i, mtr_idx, policy_idx; 2190 void *entry; 2191 struct mlx5_aso_mtr *aso_mtr; 2192 2193 if (!priv->mtr_en) 2194 return 0; 2195 if (priv->sh->meter_aso_en) { 2196 if (mlx5_flow_meter_flush_all_hierarchies(dev, error)) 2197 return -rte_errno; 2198 if (priv->mtr_idx_tbl) { 2199 MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) { 2200 mtr_idx = *(uint32_t *)entry; 2201 if (mtr_idx) { 2202 aso_mtr = 2203 mlx5_aso_meter_by_idx(priv, mtr_idx); 2204 fm = &aso_mtr->fm; 2205 (void)mlx5_flow_meter_params_flush(dev, 2206 fm, mtr_idx); 2207 } 2208 } 2209 mlx5_l3t_destroy(priv->mtr_idx_tbl); 2210 priv->mtr_idx_tbl = NULL; 2211 } 2212 } else { 2213 RTE_TAILQ_FOREACH_SAFE(legacy_fm, fms, next, tmp) { 2214 fm = &legacy_fm->fm; 2215 if (mlx5_flow_meter_params_flush(dev, fm, 0)) 2216 return -rte_mtr_error_set(error, EINVAL, 2217 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 2218 NULL, "MTR object meter profile invalid."); 2219 } 2220 } 2221 if (priv->policy_idx_tbl) { 2222 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) { 2223 policy_idx = *(uint32_t *)entry; 2224 sub_policy = mlx5_ipool_get 2225 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 2226 policy_idx); 2227 if (!sub_policy) 2228 return -rte_mtr_error_set(error, 2229 EINVAL, 2230 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 2231 NULL, "MTR object " 2232 "meter policy invalid."); 2233 if (__mlx5_flow_meter_policy_delete(dev, i, 2234 sub_policy->main_policy, 2235 error, true)) 2236 return -rte_mtr_error_set(error, 2237 EINVAL, 2238 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 2239 NULL, "MTR object " 2240 "meter policy invalid."); 2241 mlx5_free(sub_policy->main_policy); 2242 } 2243 mlx5_l3t_destroy(priv->policy_idx_tbl); 2244 priv->policy_idx_tbl = NULL; 2245 } 2246 if (priv->mtr_profile_tbl) { 2247 MLX5_L3T_FOREACH(priv->mtr_profile_tbl, i, entry) { 2248 fmp = entry; 2249 if (mlx5_flow_meter_profile_delete(dev, fmp->id, 2250 error)) 2251 return -rte_mtr_error_set(error, EINVAL, 2252 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 2253 NULL, "Fail to destroy " 2254 "meter profile."); 2255 } 2256 mlx5_l3t_destroy(priv->mtr_profile_tbl); 2257 priv->mtr_profile_tbl = NULL; 2258 } 2259 /* Delete default policy table. */ 2260 mlx5_flow_destroy_def_policy(dev); 2261 if (priv->sh->refcnt == 1) 2262 mlx5_flow_destroy_mtr_drop_tbls(dev); 2263 return 0; 2264 } 2265