1 // SPDX-License-Identifier: BSD-3-Clause 2 /* 3 * Copyright 2018 Mellanox Technologies, Ltd 4 */ 5 #include <math.h> 6 7 #include <rte_tailq.h> 8 #include <rte_malloc.h> 9 #include <rte_mtr.h> 10 #include <rte_mtr_driver.h> 11 12 #include <mlx5_devx_cmds.h> 13 #include <mlx5_malloc.h> 14 15 #include "mlx5.h" 16 #include "mlx5_flow.h" 17 18 static int mlx5_flow_meter_disable(struct rte_eth_dev *dev, 19 uint32_t meter_id, struct rte_mtr_error *error); 20 21 /** 22 * Create the meter action. 23 * 24 * @param priv 25 * Pointer to mlx5_priv. 26 * @param[in] fm 27 * Pointer to flow meter to be converted. 28 * 29 * @return 30 * Pointer to the meter action on success, NULL otherwise. 31 */ 32 static void * 33 mlx5_flow_meter_action_create(struct mlx5_priv *priv, 34 struct mlx5_flow_meter_info *fm) 35 { 36 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER 37 struct mlx5dv_dr_flow_meter_attr mtr_init; 38 uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)]; 39 struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = 40 &fm->profile->srtcm_prm; 41 uint32_t cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir); 42 uint32_t ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir); 43 uint32_t val; 44 enum mlx5_meter_domain domain = 45 fm->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 46 fm->egress ? MLX5_MTR_DOMAIN_EGRESS : 47 MLX5_MTR_DOMAIN_INGRESS; 48 struct mlx5_flow_meter_def_policy *def_policy = 49 priv->sh->mtrmng->def_policy[domain]; 50 51 memset(fmp, 0, MLX5_ST_SZ_BYTES(flow_meter_parameters)); 52 MLX5_SET(flow_meter_parameters, fmp, valid, 1); 53 MLX5_SET(flow_meter_parameters, fmp, bucket_overflow, 1); 54 MLX5_SET(flow_meter_parameters, fmp, 55 start_color, MLX5_FLOW_COLOR_GREEN); 56 MLX5_SET(flow_meter_parameters, fmp, both_buckets_on_green, 0); 57 val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK; 58 MLX5_SET(flow_meter_parameters, fmp, cbs_exponent, val); 59 val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK; 60 MLX5_SET(flow_meter_parameters, fmp, cbs_mantissa, val); 61 val = (cbs_cir >> ASO_DSEG_XIR_EXP_OFFSET) & ASO_DSEG_EXP_MASK; 62 MLX5_SET(flow_meter_parameters, fmp, cir_exponent, val); 63 val = (cbs_cir & ASO_DSEG_MAN_MASK); 64 MLX5_SET(flow_meter_parameters, fmp, cir_mantissa, val); 65 val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK; 66 MLX5_SET(flow_meter_parameters, fmp, ebs_exponent, val); 67 val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK; 68 MLX5_SET(flow_meter_parameters, fmp, ebs_mantissa, val); 69 mtr_init.next_table = def_policy->sub_policy.tbl_rsc->obj; 70 mtr_init.reg_c_index = priv->mtr_color_reg - REG_C_0; 71 mtr_init.flow_meter_parameter = fmp; 72 mtr_init.flow_meter_parameter_sz = 73 MLX5_ST_SZ_BYTES(flow_meter_parameters); 74 mtr_init.active = fm->active_state; 75 return mlx5_glue->dv_create_flow_action_meter(&mtr_init); 76 #else 77 (void)priv; 78 (void)fm; 79 return NULL; 80 #endif 81 } 82 83 /** 84 * Find meter profile by id. 85 * 86 * @param priv 87 * Pointer to mlx5_priv. 88 * @param meter_profile_id 89 * Meter profile id. 90 * 91 * @return 92 * Pointer to the profile found on success, NULL otherwise. 93 */ 94 static struct mlx5_flow_meter_profile * 95 mlx5_flow_meter_profile_find(struct mlx5_priv *priv, uint32_t meter_profile_id) 96 { 97 struct mlx5_flow_meter_profile *fmp; 98 union mlx5_l3t_data data; 99 int32_t ret; 100 101 if (mlx5_l3t_get_entry(priv->mtr_profile_tbl, 102 meter_profile_id, &data) || !data.ptr) 103 return NULL; 104 fmp = data.ptr; 105 /* Remove reference taken by the mlx5_l3t_get_entry. */ 106 ret = mlx5_l3t_clear_entry(priv->mtr_profile_tbl, 107 meter_profile_id); 108 if (!ret || ret == -1) 109 return NULL; 110 return fmp; 111 } 112 113 /** 114 * Validate the MTR profile. 115 * 116 * @param[in] dev 117 * Pointer to Ethernet device. 118 * @param[in] meter_profile_id 119 * Meter profile id. 120 * @param[in] profile 121 * Pointer to meter profile detail. 122 * @param[out] error 123 * Pointer to the error structure. 124 * 125 * @return 126 * 0 on success, a negative errno value otherwise and rte_errno is set. 127 */ 128 static int 129 mlx5_flow_meter_profile_validate(struct rte_eth_dev *dev, 130 uint32_t meter_profile_id, 131 struct rte_mtr_meter_profile *profile, 132 struct rte_mtr_error *error) 133 { 134 struct mlx5_priv *priv = dev->data->dev_private; 135 struct mlx5_flow_meter_profile *fmp; 136 uint32_t ls_factor; 137 int ret; 138 uint64_t cir, cbs; 139 uint64_t eir, ebs; 140 uint64_t pir, pbs; 141 142 /* Profile must not be NULL. */ 143 if (profile == NULL) 144 return -rte_mtr_error_set(error, EINVAL, 145 RTE_MTR_ERROR_TYPE_METER_PROFILE, 146 NULL, "Meter profile is null."); 147 /* Meter profile ID must be valid. */ 148 if (meter_profile_id == UINT32_MAX) 149 return -rte_mtr_error_set(error, EINVAL, 150 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 151 NULL, "Meter profile id not valid."); 152 /* Meter profile must not exist. */ 153 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 154 if (fmp) 155 return -rte_mtr_error_set(error, EEXIST, 156 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 157 NULL, 158 "Meter profile already exists."); 159 if (!priv->sh->meter_aso_en) { 160 /* Old version is even not supported. */ 161 if (!priv->sh->cdev->config.hca_attr.qos.flow_meter_old) 162 return -rte_mtr_error_set(error, ENOTSUP, 163 RTE_MTR_ERROR_TYPE_METER_PROFILE, 164 NULL, "Metering is not supported."); 165 /* Old FW metering only supports srTCM. */ 166 if (profile->alg != RTE_MTR_SRTCM_RFC2697) { 167 return -rte_mtr_error_set(error, ENOTSUP, 168 RTE_MTR_ERROR_TYPE_METER_PROFILE, 169 NULL, "Metering algorithm is not supported."); 170 } else if (profile->srtcm_rfc2697.ebs) { 171 /* EBS is not supported for old metering. */ 172 return -rte_mtr_error_set(error, ENOTSUP, 173 RTE_MTR_ERROR_TYPE_METER_PROFILE, 174 NULL, "EBS is not supported."); 175 } 176 if (profile->packet_mode) 177 return -rte_mtr_error_set(error, ENOTSUP, 178 RTE_MTR_ERROR_TYPE_METER_PROFILE, NULL, 179 "Metering algorithm packet mode is not supported."); 180 } 181 ls_factor = profile->packet_mode ? MLX5_MTRS_PPS_MAP_BPS_SHIFT : 0; 182 switch (profile->alg) { 183 case RTE_MTR_SRTCM_RFC2697: 184 cir = profile->srtcm_rfc2697.cir << ls_factor; 185 cbs = profile->srtcm_rfc2697.cbs << ls_factor; 186 ebs = profile->srtcm_rfc2697.ebs << ls_factor; 187 /* EBS could be zero for old metering. */ 188 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX && 189 cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX && 190 ebs <= MLX5_SRTCM_XBS_MAX) { 191 ret = 0; 192 } else { 193 ret = -rte_mtr_error_set(error, ENOTSUP, 194 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 195 "Profile values out of range."); 196 } 197 break; 198 case RTE_MTR_TRTCM_RFC2698: 199 cir = profile->trtcm_rfc2698.cir << ls_factor; 200 cbs = profile->trtcm_rfc2698.cbs << ls_factor; 201 pir = profile->trtcm_rfc2698.pir << ls_factor; 202 pbs = profile->trtcm_rfc2698.pbs << ls_factor; 203 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX && 204 cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX && 205 pir >= cir && pir <= (MLX5_SRTCM_XIR_MAX * 2) && 206 pbs >= cbs && pbs <= (MLX5_SRTCM_XBS_MAX * 2)) { 207 ret = 0; 208 } else { 209 ret = -rte_mtr_error_set(error, ENOTSUP, 210 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 211 "Profile values out of range."); 212 } 213 break; 214 case RTE_MTR_TRTCM_RFC4115: 215 cir = profile->trtcm_rfc4115.cir << ls_factor; 216 cbs = profile->trtcm_rfc4115.cbs << ls_factor; 217 eir = profile->trtcm_rfc4115.eir << ls_factor; 218 ebs = profile->trtcm_rfc4115.ebs << ls_factor; 219 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX && 220 cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX && 221 eir <= MLX5_SRTCM_XIR_MAX && ebs <= MLX5_SRTCM_XBS_MAX) { 222 ret = 0; 223 } else { 224 ret = -rte_mtr_error_set(error, ENOTSUP, 225 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 226 "Profile values out of range."); 227 } 228 break; 229 default: 230 ret = -rte_mtr_error_set(error, ENOTSUP, 231 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 232 "Unknown metering algorithm."); 233 break; 234 } 235 return ret; 236 } 237 238 /* 239 * Calculate mantissa and exponent for cir / eir. 240 * 241 * @param[in] xir 242 * Value to be calculated. 243 * @param[out] man 244 * Pointer to the mantissa. 245 * @param[out] exp 246 * Pointer to the exp. 247 */ 248 static inline void 249 mlx5_flow_meter_xir_man_exp_calc(int64_t xir, uint8_t *man, uint8_t *exp) 250 { 251 int64_t _xir; 252 int64_t delta = INT64_MAX; 253 uint8_t _man = 0; 254 uint8_t _exp = 0; 255 uint64_t m, e; 256 257 /* Special case xir == 0 ? both exp and mantissa are 0. */ 258 if (xir == 0) { 259 *man = 0; 260 *exp = 0; 261 return; 262 } 263 for (m = 0; m <= 0xFF; m++) { /* man width 8 bit */ 264 for (e = 0; e <= 0x1F; e++) { /* exp width 5bit */ 265 _xir = (1000000000ULL * m) >> e; 266 if (llabs(xir - _xir) <= delta) { 267 delta = llabs(xir - _xir); 268 _man = m; 269 _exp = e; 270 } 271 } 272 } 273 *man = _man; 274 *exp = _exp; 275 } 276 277 /* 278 * Calculate mantissa and exponent for xbs. 279 * 280 * @param[in] xbs 281 * Value to be calculated. 282 * @param[out] man 283 * Pointer to the mantissa. 284 * @param[out] exp 285 * Pointer to the exp. 286 */ 287 static inline void 288 mlx5_flow_meter_xbs_man_exp_calc(uint64_t xbs, uint8_t *man, uint8_t *exp) 289 { 290 int _exp; 291 double _man; 292 293 /* Special case xbs == 0 ? both exp and mantissa are 0. */ 294 if (xbs == 0) { 295 *man = 0; 296 *exp = 0; 297 return; 298 } 299 /* xbs = xbs_mantissa * 2^xbs_exponent */ 300 _man = frexp(xbs, &_exp); 301 if (_exp >= MLX5_MAN_WIDTH) { 302 _man = _man * pow(2, MLX5_MAN_WIDTH); 303 _exp = _exp - MLX5_MAN_WIDTH; 304 } 305 *man = (uint8_t)ceil(_man); 306 *exp = _exp; 307 } 308 309 /** 310 * Fill the prm meter parameter. 311 * 312 * @param[in,out] fmp 313 * Pointer to meter profile to be converted. 314 * @param[out] error 315 * Pointer to the error structure. 316 * 317 * @return 318 * 0 on success, a negative errno value otherwise and rte_errno is set. 319 */ 320 static int 321 mlx5_flow_meter_param_fill(struct mlx5_flow_meter_profile *fmp, 322 struct rte_mtr_error *error) 323 { 324 struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = &fmp->srtcm_prm; 325 uint8_t man, exp; 326 uint32_t cbs_exp, cbs_man, cir_exp, cir_man; 327 uint32_t eir_exp, eir_man, ebs_exp, ebs_man; 328 uint64_t cir, cbs, eir, ebs; 329 330 switch (fmp->profile.alg) { 331 case RTE_MTR_SRTCM_RFC2697: 332 cir = fmp->profile.srtcm_rfc2697.cir; 333 cbs = fmp->profile.srtcm_rfc2697.cbs; 334 eir = 0; 335 ebs = fmp->profile.srtcm_rfc2697.ebs; 336 break; 337 case RTE_MTR_TRTCM_RFC2698: 338 MLX5_ASSERT(fmp->profile.trtcm_rfc2698.pir > 339 fmp->profile.trtcm_rfc2698.cir && 340 fmp->profile.trtcm_rfc2698.pbs > 341 fmp->profile.trtcm_rfc2698.cbs); 342 cir = fmp->profile.trtcm_rfc2698.cir; 343 cbs = fmp->profile.trtcm_rfc2698.cbs; 344 /* EIR / EBS are filled with PIR / PBS. */ 345 eir = fmp->profile.trtcm_rfc2698.pir; 346 ebs = fmp->profile.trtcm_rfc2698.pbs; 347 break; 348 case RTE_MTR_TRTCM_RFC4115: 349 cir = fmp->profile.trtcm_rfc4115.cir; 350 cbs = fmp->profile.trtcm_rfc4115.cbs; 351 eir = fmp->profile.trtcm_rfc4115.eir; 352 ebs = fmp->profile.trtcm_rfc4115.ebs; 353 break; 354 default: 355 return -rte_mtr_error_set(error, EINVAL, 356 RTE_MTR_ERROR_TYPE_METER_PROFILE, NULL, 357 "Metering algorithm mode is invalid"); 358 } 359 /* Adjust the values for PPS mode. */ 360 if (fmp->profile.packet_mode) { 361 cir <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT; 362 cbs <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT; 363 eir <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT; 364 ebs <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT; 365 } 366 /* cir = 8G * cir_mantissa * 1/(2^cir_exponent)) Bytes/Sec */ 367 mlx5_flow_meter_xir_man_exp_calc(cir, &man, &exp); 368 /* Check if cir mantissa is too large. */ 369 if (exp > ASO_DSEG_XIR_EXP_MASK) 370 return -rte_mtr_error_set(error, ENOTSUP, 371 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 372 "meter profile parameter cir is not supported."); 373 cir_man = man; 374 cir_exp = exp; 375 /* cbs = cbs_mantissa * 2^cbs_exponent */ 376 mlx5_flow_meter_xbs_man_exp_calc(cbs, &man, &exp); 377 /* Check if cbs mantissa is too large. */ 378 if (exp > ASO_DSEG_EXP_MASK) 379 return -rte_mtr_error_set(error, ENOTSUP, 380 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 381 "meter profile parameter cbs is not supported."); 382 cbs_man = man; 383 cbs_exp = exp; 384 srtcm->cbs_cir = rte_cpu_to_be_32(cbs_exp << ASO_DSEG_CBS_EXP_OFFSET | 385 cbs_man << ASO_DSEG_CBS_MAN_OFFSET | 386 cir_exp << ASO_DSEG_XIR_EXP_OFFSET | 387 cir_man); 388 mlx5_flow_meter_xir_man_exp_calc(eir, &man, &exp); 389 /* Check if eir mantissa is too large. */ 390 if (exp > ASO_DSEG_XIR_EXP_MASK) 391 return -rte_mtr_error_set(error, ENOTSUP, 392 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 393 "meter profile parameter eir is not supported."); 394 eir_man = man; 395 eir_exp = exp; 396 mlx5_flow_meter_xbs_man_exp_calc(ebs, &man, &exp); 397 /* Check if ebs mantissa is too large. */ 398 if (exp > ASO_DSEG_EXP_MASK) 399 return -rte_mtr_error_set(error, ENOTSUP, 400 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 401 "meter profile parameter ebs is not supported."); 402 ebs_man = man; 403 ebs_exp = exp; 404 srtcm->ebs_eir = rte_cpu_to_be_32(ebs_exp << ASO_DSEG_EBS_EXP_OFFSET | 405 ebs_man << ASO_DSEG_EBS_MAN_OFFSET | 406 eir_exp << ASO_DSEG_XIR_EXP_OFFSET | 407 eir_man); 408 if (srtcm->cbs_cir) 409 fmp->g_support = 1; 410 if (srtcm->ebs_eir) 411 fmp->y_support = 1; 412 return 0; 413 } 414 415 /** 416 * Callback to get MTR capabilities. 417 * 418 * @param[in] dev 419 * Pointer to Ethernet device. 420 * @param[out] cap 421 * Pointer to save MTR capabilities. 422 * @param[out] error 423 * Pointer to the error structure. 424 * 425 * @return 426 * 0 on success, a negative errno value otherwise and rte_errno is set. 427 */ 428 static int 429 mlx5_flow_mtr_cap_get(struct rte_eth_dev *dev, 430 struct rte_mtr_capabilities *cap, 431 struct rte_mtr_error *error __rte_unused) 432 { 433 struct mlx5_priv *priv = dev->data->dev_private; 434 struct mlx5_hca_qos_attr *qattr = &priv->sh->cdev->config.hca_attr.qos; 435 436 if (!priv->mtr_en) 437 return -rte_mtr_error_set(error, ENOTSUP, 438 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 439 "Meter is not supported"); 440 memset(cap, 0, sizeof(*cap)); 441 if (priv->sh->meter_aso_en) { 442 /* 2 meters per one ASO cache line. */ 443 cap->n_max = 1 << (qattr->log_max_num_meter_aso + 1); 444 cap->srtcm_rfc2697_packet_mode_supported = 1; 445 cap->trtcm_rfc2698_packet_mode_supported = 1; 446 cap->trtcm_rfc4115_packet_mode_supported = 1; 447 } else { 448 cap->n_max = 1 << qattr->log_max_flow_meter; 449 } 450 cap->srtcm_rfc2697_byte_mode_supported = 1; 451 cap->trtcm_rfc2698_byte_mode_supported = 1; 452 cap->trtcm_rfc4115_byte_mode_supported = 1; 453 cap->n_shared_max = cap->n_max; 454 cap->identical = 1; 455 cap->shared_identical = 1; 456 cap->shared_n_flows_per_mtr_max = 4 << 20; 457 /* 2M flows can share the same meter. */ 458 cap->chaining_n_mtrs_per_flow_max = 1; /* Chaining is not supported. */ 459 cap->meter_srtcm_rfc2697_n_max = qattr->flow_meter_old ? cap->n_max : 0; 460 cap->meter_trtcm_rfc2698_n_max = qattr->flow_meter_old ? cap->n_max : 0; 461 cap->meter_trtcm_rfc4115_n_max = qattr->flow_meter_old ? cap->n_max : 0; 462 cap->meter_rate_max = 1ULL << 40; /* 1 Tera tokens per sec. */ 463 cap->meter_policy_n_max = cap->n_max; 464 cap->stats_mask = RTE_MTR_STATS_N_BYTES_DROPPED | 465 RTE_MTR_STATS_N_PKTS_DROPPED; 466 return 0; 467 } 468 469 /** 470 * Callback to add MTR profile. 471 * 472 * @param[in] dev 473 * Pointer to Ethernet device. 474 * @param[in] meter_profile_id 475 * Meter profile id. 476 * @param[in] profile 477 * Pointer to meter profile detail. 478 * @param[out] error 479 * Pointer to the error structure. 480 * 481 * @return 482 * 0 on success, a negative errno value otherwise and rte_errno is set. 483 */ 484 static int 485 mlx5_flow_meter_profile_add(struct rte_eth_dev *dev, 486 uint32_t meter_profile_id, 487 struct rte_mtr_meter_profile *profile, 488 struct rte_mtr_error *error) 489 { 490 struct mlx5_priv *priv = dev->data->dev_private; 491 struct mlx5_flow_meter_profile *fmp; 492 union mlx5_l3t_data data; 493 int ret; 494 495 if (!priv->mtr_en) 496 return -rte_mtr_error_set(error, ENOTSUP, 497 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 498 "Meter is not supported"); 499 /* Check input params. */ 500 ret = mlx5_flow_meter_profile_validate(dev, meter_profile_id, 501 profile, error); 502 if (ret) 503 return ret; 504 /* Meter profile memory allocation. */ 505 fmp = mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct mlx5_flow_meter_profile), 506 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 507 if (fmp == NULL) 508 return -rte_mtr_error_set(error, ENOMEM, 509 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 510 NULL, "Meter profile memory " 511 "alloc failed."); 512 /* Fill profile info. */ 513 fmp->id = meter_profile_id; 514 fmp->profile = *profile; 515 /* Fill the flow meter parameters for the PRM. */ 516 ret = mlx5_flow_meter_param_fill(fmp, error); 517 if (ret) 518 goto error; 519 data.ptr = fmp; 520 ret = mlx5_l3t_set_entry(priv->mtr_profile_tbl, 521 meter_profile_id, &data); 522 if (ret) 523 return -rte_mtr_error_set(error, ENOTSUP, 524 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 525 NULL, "Meter profile insert fail."); 526 return 0; 527 error: 528 mlx5_free(fmp); 529 return ret; 530 } 531 532 /** 533 * Callback to delete MTR profile. 534 * 535 * @param[in] dev 536 * Pointer to Ethernet device. 537 * @param[in] meter_profile_id 538 * Meter profile id. 539 * @param[out] error 540 * Pointer to the error structure. 541 * 542 * @return 543 * 0 on success, a negative errno value otherwise and rte_errno is set. 544 */ 545 static int 546 mlx5_flow_meter_profile_delete(struct rte_eth_dev *dev, 547 uint32_t meter_profile_id, 548 struct rte_mtr_error *error) 549 { 550 struct mlx5_priv *priv = dev->data->dev_private; 551 struct mlx5_flow_meter_profile *fmp; 552 553 if (!priv->mtr_en) 554 return -rte_mtr_error_set(error, ENOTSUP, 555 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 556 "Meter is not supported"); 557 /* Meter profile must exist. */ 558 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 559 if (fmp == NULL) 560 return -rte_mtr_error_set(error, ENOENT, 561 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 562 &meter_profile_id, 563 "Meter profile id is invalid."); 564 /* Check profile is unused. */ 565 if (fmp->ref_cnt) 566 return -rte_mtr_error_set(error, EBUSY, 567 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 568 NULL, "Meter profile is in use."); 569 if (mlx5_l3t_clear_entry(priv->mtr_profile_tbl, meter_profile_id)) 570 return -rte_mtr_error_set(error, EBUSY, 571 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 572 NULL, "Meter profile remove fail."); 573 mlx5_free(fmp); 574 return 0; 575 } 576 577 /** 578 * Find policy by id. 579 * 580 * @param[in] dev 581 * Pointer to Ethernet device. 582 * @param policy_id 583 * Policy id. 584 * 585 * @return 586 * Pointer to the policy found on success, NULL otherwise. 587 */ 588 struct mlx5_flow_meter_policy * 589 mlx5_flow_meter_policy_find(struct rte_eth_dev *dev, 590 uint32_t policy_id, 591 uint32_t *policy_idx) 592 { 593 struct mlx5_priv *priv = dev->data->dev_private; 594 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 595 union mlx5_l3t_data data; 596 597 if (policy_id > MLX5_MAX_SUB_POLICY_TBL_NUM || !priv->policy_idx_tbl) 598 return NULL; 599 if (mlx5_l3t_get_entry(priv->policy_idx_tbl, policy_id, &data) || 600 !data.dword) 601 return NULL; 602 if (policy_idx) 603 *policy_idx = data.dword; 604 sub_policy = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 605 data.dword); 606 /* Remove reference taken by the mlx5_l3t_get_entry. */ 607 mlx5_l3t_clear_entry(priv->policy_idx_tbl, policy_id); 608 if (sub_policy) 609 if (sub_policy->main_policy_id) 610 return sub_policy->main_policy; 611 return NULL; 612 } 613 614 /** 615 * Get the last meter's policy from one meter's policy in hierarchy. 616 * 617 * @param[in] dev 618 * Pointer to Ethernet device. 619 * @param[in] policy 620 * Pointer to flow meter policy. 621 * 622 * @return 623 * Pointer to the final meter's policy, or NULL when fail. 624 */ 625 struct mlx5_flow_meter_policy * 626 mlx5_flow_meter_hierarchy_get_final_policy(struct rte_eth_dev *dev, 627 struct mlx5_flow_meter_policy *policy) 628 { 629 struct mlx5_priv *priv = dev->data->dev_private; 630 struct mlx5_flow_meter_info *next_fm; 631 struct mlx5_flow_meter_policy *next_policy = policy; 632 633 while (next_policy->is_hierarchy) { 634 next_fm = mlx5_flow_meter_find(priv, 635 next_policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id, NULL); 636 if (!next_fm || next_fm->def_policy) 637 return NULL; 638 next_policy = mlx5_flow_meter_policy_find(dev, 639 next_fm->policy_id, NULL); 640 MLX5_ASSERT(next_policy); 641 } 642 return next_policy; 643 } 644 645 /** 646 * Callback to check MTR policy action validate 647 * 648 * @param[in] dev 649 * Pointer to Ethernet device. 650 * @param[in] actions 651 * Pointer to meter policy action detail. 652 * @param[out] error 653 * Pointer to the error structure. 654 * 655 * @return 656 * 0 on success, a negative errno value otherwise and rte_errno is set. 657 */ 658 static int 659 mlx5_flow_meter_policy_validate(struct rte_eth_dev *dev, 660 struct rte_mtr_meter_policy_params *policy, 661 struct rte_mtr_error *error) 662 { 663 struct mlx5_priv *priv = dev->data->dev_private; 664 struct rte_flow_attr attr = { .transfer = priv->sh->config.dv_esw_en ? 665 1 : 0 }; 666 bool is_rss = false; 667 uint8_t policy_mode; 668 uint8_t domain_bitmap; 669 int ret; 670 671 if (!priv->mtr_en || !priv->sh->meter_aso_en) 672 return -rte_mtr_error_set(error, ENOTSUP, 673 RTE_MTR_ERROR_TYPE_METER_POLICY, 674 NULL, "meter policy unsupported."); 675 ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr, 676 &is_rss, &domain_bitmap, &policy_mode, error); 677 if (ret) 678 return ret; 679 return 0; 680 } 681 682 static int 683 __mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev, 684 uint32_t policy_id, 685 struct mlx5_flow_meter_policy *mtr_policy, 686 struct rte_mtr_error *error, 687 bool clear_l3t) 688 { 689 struct mlx5_priv *priv = dev->data->dev_private; 690 struct mlx5_flow_meter_sub_policy *sub_policy; 691 uint32_t i, j; 692 uint16_t sub_policy_num; 693 694 rte_spinlock_lock(&mtr_policy->sl); 695 if (mtr_policy->ref_cnt) { 696 rte_spinlock_unlock(&mtr_policy->sl); 697 return -rte_mtr_error_set(error, EBUSY, 698 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 699 NULL, 700 "Meter policy object is being used."); 701 } 702 mlx5_flow_destroy_policy_rules(dev, mtr_policy); 703 mlx5_flow_destroy_mtr_acts(dev, mtr_policy); 704 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 705 sub_policy_num = (mtr_policy->sub_policy_num >> 706 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) & 707 MLX5_MTR_SUB_POLICY_NUM_MASK; 708 if (sub_policy_num) { 709 for (j = 0; j < sub_policy_num; j++) { 710 sub_policy = mtr_policy->sub_policys[i][j]; 711 if (sub_policy) 712 mlx5_ipool_free 713 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 714 sub_policy->idx); 715 } 716 } 717 } 718 if (priv->policy_idx_tbl && clear_l3t) { 719 if (mlx5_l3t_clear_entry(priv->policy_idx_tbl, policy_id)) { 720 rte_spinlock_unlock(&mtr_policy->sl); 721 return -rte_mtr_error_set(error, ENOTSUP, 722 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL, 723 "Fail to delete policy in index table."); 724 } 725 } 726 rte_spinlock_unlock(&mtr_policy->sl); 727 return 0; 728 } 729 730 /** 731 * Callback to add MTR policy. 732 * 733 * @param[in] dev 734 * Pointer to Ethernet device. 735 * @param[out] policy_id 736 * Pointer to policy id 737 * @param[in] actions 738 * Pointer to meter policy action detail. 739 * @param[out] error 740 * Pointer to the error structure. 741 * 742 * @return 743 * 0 on success, a negative errno value otherwise and rte_errno is set. 744 */ 745 static int 746 mlx5_flow_meter_policy_add(struct rte_eth_dev *dev, 747 uint32_t policy_id, 748 struct rte_mtr_meter_policy_params *policy, 749 struct rte_mtr_error *error) 750 { 751 struct mlx5_priv *priv = dev->data->dev_private; 752 struct rte_flow_attr attr = { .transfer = priv->sh->config.dv_esw_en ? 753 1 : 0 }; 754 uint32_t sub_policy_idx = 0; 755 uint32_t policy_idx = 0; 756 struct mlx5_flow_meter_policy *mtr_policy = NULL; 757 struct mlx5_flow_meter_sub_policy *sub_policy; 758 bool is_rss = false; 759 uint8_t policy_mode; 760 uint32_t i; 761 int ret; 762 uint32_t policy_size = sizeof(struct mlx5_flow_meter_policy); 763 uint16_t sub_policy_num; 764 uint8_t domain_bitmap = 0; 765 union mlx5_l3t_data data; 766 bool skip_rule = false; 767 768 if (!priv->mtr_en) 769 return -rte_mtr_error_set(error, ENOTSUP, 770 RTE_MTR_ERROR_TYPE_METER_POLICY, 771 NULL, "meter policy unsupported. "); 772 if (policy_id == MLX5_INVALID_POLICY_ID) 773 return -rte_mtr_error_set(error, ENOTSUP, 774 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 775 NULL, "policy ID is invalid. "); 776 if (policy_id == priv->sh->mtrmng->def_policy_id) 777 return -rte_mtr_error_set(error, EEXIST, 778 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 779 NULL, "default policy ID exists. "); 780 mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx); 781 if (mtr_policy) 782 return -rte_mtr_error_set(error, EEXIST, 783 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 784 NULL, "policy ID exists. "); 785 ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr, 786 &is_rss, &domain_bitmap, 787 &policy_mode, error); 788 if (ret) 789 return ret; 790 if (!domain_bitmap) 791 return -rte_mtr_error_set(error, ENOTSUP, 792 RTE_MTR_ERROR_TYPE_METER_POLICY, 793 NULL, "fail to find policy domain."); 794 if (policy_mode == MLX5_MTR_POLICY_MODE_DEF) { 795 if (priv->sh->mtrmng->def_policy_id != MLX5_INVALID_POLICY_ID) 796 return -rte_mtr_error_set(error, EEXIST, 797 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 798 NULL, "a policy with similar actions " 799 "is already configured"); 800 if (mlx5_flow_create_def_policy(dev)) 801 return -rte_mtr_error_set(error, ENOTSUP, 802 RTE_MTR_ERROR_TYPE_METER_POLICY, 803 NULL, 804 "fail to create non-terminated policy."); 805 priv->sh->mtrmng->def_policy_id = policy_id; 806 return 0; 807 } 808 if (!priv->sh->meter_aso_en) 809 return -rte_mtr_error_set(error, ENOTSUP, 810 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL, 811 "no ASO capability to support the policy "); 812 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 813 if (!(domain_bitmap & (1 << i))) 814 continue; 815 /* 816 * If RSS is found, it means that only the ingress domain can 817 * be supported. It is invalid to support RSS for one color 818 * and egress / transfer domain actions for another. Drop and 819 * jump action should have no impact. 820 */ 821 if (is_rss) { 822 policy_size += 823 sizeof(struct mlx5_flow_meter_sub_policy *) * 824 MLX5_MTR_RSS_MAX_SUB_POLICY; 825 break; 826 } 827 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *); 828 } 829 mtr_policy = mlx5_malloc(MLX5_MEM_ZERO, policy_size, 830 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 831 if (!mtr_policy) 832 return -rte_mtr_error_set(error, ENOMEM, 833 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL, 834 "Memory alloc failed for meter policy."); 835 if (policy_mode == MLX5_MTR_POLICY_MODE_OG) 836 mtr_policy->skip_y = 1; 837 else if (policy_mode == MLX5_MTR_POLICY_MODE_OY) 838 mtr_policy->skip_g = 1; 839 policy_size = sizeof(struct mlx5_flow_meter_policy); 840 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 841 if (!(domain_bitmap & (1 << i))) 842 continue; 843 if (i == MLX5_MTR_DOMAIN_INGRESS) 844 mtr_policy->ingress = 1; 845 if (i == MLX5_MTR_DOMAIN_EGRESS) 846 mtr_policy->egress = 1; 847 if (i == MLX5_MTR_DOMAIN_TRANSFER) 848 mtr_policy->transfer = 1; 849 sub_policy = mlx5_ipool_zmalloc 850 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 851 &sub_policy_idx); 852 if (!sub_policy || 853 sub_policy_idx > MLX5_MAX_SUB_POLICY_TBL_NUM) 854 goto policy_add_err; 855 sub_policy->idx = sub_policy_idx; 856 sub_policy->main_policy = mtr_policy; 857 if (!policy_idx) { 858 policy_idx = sub_policy_idx; 859 sub_policy->main_policy_id = 1; 860 } 861 mtr_policy->sub_policys[i] = 862 (struct mlx5_flow_meter_sub_policy **) 863 ((uint8_t *)mtr_policy + policy_size); 864 mtr_policy->sub_policys[i][0] = sub_policy; 865 sub_policy_num = (mtr_policy->sub_policy_num >> 866 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) & 867 MLX5_MTR_SUB_POLICY_NUM_MASK; 868 sub_policy_num++; 869 mtr_policy->sub_policy_num &= ~(MLX5_MTR_SUB_POLICY_NUM_MASK << 870 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)); 871 mtr_policy->sub_policy_num |= 872 (sub_policy_num & MLX5_MTR_SUB_POLICY_NUM_MASK) << 873 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i); 874 /* 875 * If RSS is found, it means that only the ingress domain can 876 * be supported. It is invalid to support RSS for one color 877 * and egress / transfer domain actions for another. Drop and 878 * jump action should have no impact. 879 */ 880 if (is_rss) { 881 mtr_policy->is_rss = 1; 882 break; 883 } 884 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *); 885 } 886 rte_spinlock_init(&mtr_policy->sl); 887 ret = mlx5_flow_create_mtr_acts(dev, mtr_policy, 888 policy->actions, error); 889 if (ret) 890 goto policy_add_err; 891 if (mtr_policy->is_hierarchy) { 892 struct mlx5_flow_meter_policy *final_policy; 893 894 final_policy = 895 mlx5_flow_meter_hierarchy_get_final_policy(dev, mtr_policy); 896 if (!final_policy) 897 goto policy_add_err; 898 skip_rule = (final_policy->is_rss || final_policy->is_queue); 899 } 900 /* 901 * If either Green or Yellow has queue / RSS action, all the policy 902 * rules will be created later in the flow splitting stage. 903 */ 904 if (!is_rss && !mtr_policy->is_queue && !skip_rule) { 905 /* Create policy rules in HW. */ 906 ret = mlx5_flow_create_policy_rules(dev, mtr_policy); 907 if (ret) 908 goto policy_add_err; 909 } 910 data.dword = policy_idx; 911 if (!priv->policy_idx_tbl) { 912 priv->policy_idx_tbl = mlx5_l3t_create(MLX5_L3T_TYPE_DWORD); 913 if (!priv->policy_idx_tbl) 914 goto policy_add_err; 915 } 916 if (mlx5_l3t_set_entry(priv->policy_idx_tbl, policy_id, &data)) 917 goto policy_add_err; 918 return 0; 919 policy_add_err: 920 if (mtr_policy) { 921 ret = __mlx5_flow_meter_policy_delete(dev, policy_id, 922 mtr_policy, error, false); 923 mlx5_free(mtr_policy); 924 if (ret) 925 return ret; 926 } 927 return -rte_mtr_error_set(error, ENOTSUP, 928 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 929 NULL, "Failed to create devx policy."); 930 } 931 932 /** 933 * Callback to delete MTR policy. 934 * 935 * @param[in] dev 936 * Pointer to Ethernet device. 937 * @param[in] policy_id 938 * Meter policy id. 939 * @param[out] error 940 * Pointer to the error structure. 941 * 942 * @return 943 * 0 on success, a negative errno value otherwise and rte_errno is set. 944 */ 945 static int 946 mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev, 947 uint32_t policy_id, 948 struct rte_mtr_error *error) 949 { 950 struct mlx5_priv *priv = dev->data->dev_private; 951 struct mlx5_flow_meter_policy *mtr_policy; 952 uint32_t policy_idx; 953 int ret; 954 955 if (policy_id == priv->sh->mtrmng->def_policy_id) { 956 if (priv->sh->mtrmng->def_policy_ref_cnt > 0) 957 return -rte_mtr_error_set(error, ENOTSUP, 958 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL, 959 "Meter policy object is being used."); 960 priv->sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID; 961 return 0; 962 } 963 mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx); 964 if (!mtr_policy) 965 return -rte_mtr_error_set(error, ENOTSUP, 966 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL, 967 "Meter policy id is invalid. "); 968 ret = __mlx5_flow_meter_policy_delete(dev, policy_id, mtr_policy, 969 error, true); 970 if (ret) 971 return ret; 972 mlx5_free(mtr_policy); 973 return 0; 974 } 975 976 /** 977 * Check meter validation. 978 * 979 * @param[in] priv 980 * Pointer to mlx5 private data structure. 981 * @param[in] meter_id 982 * Meter id. 983 * @param[in] params 984 * Pointer to rte meter parameters. 985 * @param[out] error 986 * Pointer to rte meter error structure. 987 * 988 * @return 989 * 0 on success, a negative errno value otherwise and rte_errno is set. 990 */ 991 static int 992 mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id, 993 struct rte_mtr_params *params, 994 struct rte_mtr_error *error) 995 { 996 /* Meter must use global drop action. */ 997 if (!priv->sh->dr_drop_action) 998 return -rte_mtr_error_set(error, ENOTSUP, 999 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 1000 NULL, 1001 "No drop action ready for meter."); 1002 /* Meter params must not be NULL. */ 1003 if (params == NULL) 1004 return -rte_mtr_error_set(error, EINVAL, 1005 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 1006 NULL, "Meter object params null."); 1007 /* Previous meter color is not supported. */ 1008 if (params->use_prev_mtr_color) 1009 return -rte_mtr_error_set(error, ENOTSUP, 1010 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 1011 NULL, 1012 "Previous meter color " 1013 "not supported."); 1014 if (params->meter_policy_id == MLX5_INVALID_POLICY_ID) 1015 return -rte_mtr_error_set(error, ENOENT, 1016 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1017 NULL, "Meter policy id not valid."); 1018 /* Validate meter id. */ 1019 if (mlx5_flow_meter_find(priv, meter_id, NULL)) 1020 return -rte_mtr_error_set(error, EEXIST, 1021 RTE_MTR_ERROR_TYPE_MTR_ID, NULL, 1022 "Meter object already exists."); 1023 return 0; 1024 } 1025 1026 /** 1027 * Modify the flow meter action. 1028 * 1029 * @param[in] priv 1030 * Pointer to mlx5 private data structure. 1031 * @param[in] fm 1032 * Pointer to flow meter to be modified. 1033 * @param[in] srtcm 1034 * Pointer to meter srtcm description parameter. 1035 * @param[in] modify_bits 1036 * The bit in srtcm to be updated. 1037 * @param[in] active_state 1038 * The state to be updated. 1039 * @return 1040 * 0 on success, o negative value otherwise. 1041 */ 1042 static int 1043 mlx5_flow_meter_action_modify(struct mlx5_priv *priv, 1044 struct mlx5_flow_meter_info *fm, 1045 const struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm, 1046 uint64_t modify_bits, uint32_t active_state, uint32_t is_enable) 1047 { 1048 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER 1049 uint32_t in[MLX5_ST_SZ_DW(flow_meter_parameters)] = { 0 }; 1050 uint32_t *attr; 1051 struct mlx5dv_dr_flow_meter_attr mod_attr = { 0 }; 1052 int ret; 1053 struct mlx5_aso_mtr *aso_mtr = NULL; 1054 uint32_t cbs_cir, ebs_eir, val; 1055 1056 if (priv->sh->meter_aso_en) { 1057 fm->is_enable = !!is_enable; 1058 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm); 1059 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr); 1060 if (ret) 1061 return ret; 1062 ret = mlx5_aso_mtr_wait(priv->sh, aso_mtr); 1063 if (ret) 1064 return ret; 1065 } else { 1066 /* Fill command parameters. */ 1067 mod_attr.reg_c_index = priv->mtr_color_reg - REG_C_0; 1068 mod_attr.flow_meter_parameter = in; 1069 mod_attr.flow_meter_parameter_sz = 1070 MLX5_ST_SZ_BYTES(flow_meter_parameters); 1071 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE) 1072 mod_attr.active = !!active_state; 1073 else 1074 mod_attr.active = 0; 1075 attr = in; 1076 cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir); 1077 ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir); 1078 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS) { 1079 val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & 1080 ASO_DSEG_EXP_MASK; 1081 MLX5_SET(flow_meter_parameters, attr, 1082 cbs_exponent, val); 1083 val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & 1084 ASO_DSEG_MAN_MASK; 1085 MLX5_SET(flow_meter_parameters, attr, 1086 cbs_mantissa, val); 1087 } 1088 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR) { 1089 val = (cbs_cir >> ASO_DSEG_XIR_EXP_OFFSET) & 1090 ASO_DSEG_EXP_MASK; 1091 MLX5_SET(flow_meter_parameters, attr, 1092 cir_exponent, val); 1093 val = cbs_cir & ASO_DSEG_MAN_MASK; 1094 MLX5_SET(flow_meter_parameters, attr, 1095 cir_mantissa, val); 1096 } 1097 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_EBS) { 1098 val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & 1099 ASO_DSEG_EXP_MASK; 1100 MLX5_SET(flow_meter_parameters, attr, 1101 ebs_exponent, val); 1102 val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & 1103 ASO_DSEG_MAN_MASK; 1104 MLX5_SET(flow_meter_parameters, attr, 1105 ebs_mantissa, val); 1106 } 1107 /* Apply modifications to meter only if it was created. */ 1108 if (fm->meter_action) { 1109 ret = mlx5_glue->dv_modify_flow_action_meter 1110 (fm->meter_action, &mod_attr, 1111 rte_cpu_to_be_64(modify_bits)); 1112 if (ret) 1113 return ret; 1114 } 1115 /* Update succeeded modify meter parameters. */ 1116 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE) 1117 fm->active_state = !!active_state; 1118 } 1119 return 0; 1120 #else 1121 (void)priv; 1122 (void)fm; 1123 (void)srtcm; 1124 (void)modify_bits; 1125 (void)active_state; 1126 (void)is_enable; 1127 return -ENOTSUP; 1128 #endif 1129 } 1130 1131 static int 1132 mlx5_flow_meter_stats_enable_update(struct rte_eth_dev *dev, 1133 struct mlx5_flow_meter_info *fm, 1134 uint64_t stats_mask) 1135 { 1136 fm->bytes_dropped = 1137 (stats_mask & RTE_MTR_STATS_N_BYTES_DROPPED) ? 1 : 0; 1138 fm->pkts_dropped = (stats_mask & RTE_MTR_STATS_N_PKTS_DROPPED) ? 1 : 0; 1139 if (fm->bytes_dropped || fm->pkts_dropped) { 1140 if (!fm->drop_cnt) { 1141 /* Alloc policer counters. */ 1142 fm->drop_cnt = mlx5_counter_alloc(dev); 1143 if (!fm->drop_cnt) 1144 return -1; 1145 } 1146 } else { 1147 if (fm->drop_cnt) { 1148 mlx5_counter_free(dev, fm->drop_cnt); 1149 fm->drop_cnt = 0; 1150 } 1151 } 1152 return 0; 1153 } 1154 1155 /** 1156 * Create meter rules. 1157 * 1158 * @param[in] dev 1159 * Pointer to Ethernet device. 1160 * @param[in] meter_id 1161 * Meter id. 1162 * @param[in] params 1163 * Pointer to rte meter parameters. 1164 * @param[in] shared 1165 * Meter shared with other flow or not. 1166 * @param[out] error 1167 * Pointer to rte meter error structure. 1168 * 1169 * @return 1170 * 0 on success, a negative errno value otherwise and rte_errno is set. 1171 */ 1172 static int 1173 mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, 1174 struct rte_mtr_params *params, int shared, 1175 struct rte_mtr_error *error) 1176 { 1177 struct mlx5_priv *priv = dev->data->dev_private; 1178 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; 1179 struct mlx5_flow_meter_profile *fmp; 1180 struct mlx5_flow_meter_info *fm; 1181 /* GCC fails to infer legacy_fm is set when !priv->sh->meter_aso_en. */ 1182 struct mlx5_legacy_flow_meter *legacy_fm = NULL; 1183 struct mlx5_flow_meter_policy *mtr_policy = NULL; 1184 struct mlx5_indexed_pool_config flow_ipool_cfg = { 1185 .size = 0, 1186 .trunk_size = 64, 1187 .need_lock = 1, 1188 .type = "mlx5_flow_mtr_flow_id_pool", 1189 }; 1190 struct mlx5_aso_mtr *aso_mtr; 1191 uint32_t mtr_idx, policy_idx; 1192 union mlx5_l3t_data data; 1193 int ret; 1194 uint8_t domain_bitmap; 1195 uint8_t mtr_id_bits; 1196 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 1197 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 1198 1199 if (!priv->mtr_en) 1200 return -rte_mtr_error_set(error, ENOTSUP, 1201 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1202 "Meter is not supported"); 1203 /* Validate the parameters. */ 1204 ret = mlx5_flow_meter_validate(priv, meter_id, params, error); 1205 if (ret) 1206 return ret; 1207 /* Meter profile must exist. */ 1208 fmp = mlx5_flow_meter_profile_find(priv, params->meter_profile_id); 1209 if (fmp == NULL) 1210 return -rte_mtr_error_set(error, ENOENT, 1211 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 1212 NULL, "Meter profile id not valid."); 1213 /* Meter policy must exist. */ 1214 if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) { 1215 __atomic_add_fetch 1216 (&priv->sh->mtrmng->def_policy_ref_cnt, 1217 1, __ATOMIC_RELAXED); 1218 domain_bitmap = MLX5_MTR_ALL_DOMAIN_BIT; 1219 if (!priv->sh->config.dv_esw_en) 1220 domain_bitmap &= ~MLX5_MTR_DOMAIN_TRANSFER_BIT; 1221 } else { 1222 if (!priv->sh->meter_aso_en) 1223 return -rte_mtr_error_set(error, ENOTSUP, 1224 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1225 "Part of the policies cannot be " 1226 "supported without ASO "); 1227 mtr_policy = mlx5_flow_meter_policy_find(dev, 1228 params->meter_policy_id, &policy_idx); 1229 if (!mtr_policy) 1230 return -rte_mtr_error_set(error, ENOENT, 1231 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1232 NULL, "Meter policy id not valid."); 1233 domain_bitmap = (mtr_policy->ingress ? 1234 MLX5_MTR_DOMAIN_INGRESS_BIT : 0) | 1235 (mtr_policy->egress ? 1236 MLX5_MTR_DOMAIN_EGRESS_BIT : 0) | 1237 (mtr_policy->transfer ? 1238 MLX5_MTR_DOMAIN_TRANSFER_BIT : 0); 1239 if (fmp->g_support && mtr_policy->skip_g) 1240 return -rte_mtr_error_set(error, ENOTSUP, 1241 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1242 NULL, "Meter green policy is empty."); 1243 if (fmp->y_support && mtr_policy->skip_y) 1244 return -rte_mtr_error_set(error, ENOTSUP, 1245 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1246 NULL, "Meter yellow policy is empty."); 1247 } 1248 /* Allocate the flow meter memory. */ 1249 if (priv->sh->meter_aso_en) { 1250 mtr_idx = mlx5_flow_mtr_alloc(dev); 1251 if (!mtr_idx) 1252 return -rte_mtr_error_set(error, ENOMEM, 1253 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1254 "Memory alloc failed for meter."); 1255 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx); 1256 fm = &aso_mtr->fm; 1257 } else { 1258 if (fmp->y_support) 1259 return -rte_mtr_error_set(error, ENOMEM, 1260 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1261 "Unsupported profile with yellow."); 1262 legacy_fm = mlx5_ipool_zmalloc 1263 (priv->sh->ipool[MLX5_IPOOL_MTR], &mtr_idx); 1264 if (legacy_fm == NULL) 1265 return -rte_mtr_error_set(error, ENOMEM, 1266 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1267 "Memory alloc failed for meter."); 1268 legacy_fm->idx = mtr_idx; 1269 fm = &legacy_fm->fm; 1270 } 1271 mtr_id_bits = MLX5_REG_BITS - __builtin_clz(mtr_idx); 1272 if ((mtr_id_bits + priv->sh->mtrmng->max_mtr_flow_bits) > 1273 mtr_reg_bits) { 1274 DRV_LOG(ERR, "Meter number exceeds max limit."); 1275 goto error; 1276 } 1277 if (mtr_id_bits > priv->sh->mtrmng->max_mtr_bits) 1278 priv->sh->mtrmng->max_mtr_bits = mtr_id_bits; 1279 /* Fill the flow meter parameters. */ 1280 fm->meter_id = meter_id; 1281 fm->policy_id = params->meter_policy_id; 1282 fm->profile = fmp; 1283 if (mlx5_flow_meter_stats_enable_update(dev, fm, params->stats_mask)) 1284 goto error; 1285 if (mlx5_flow_create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap)) 1286 goto error; 1287 /* Add to the flow meter list. */ 1288 if (!priv->sh->meter_aso_en) { 1289 MLX5_ASSERT(legacy_fm != NULL); 1290 TAILQ_INSERT_TAIL(fms, legacy_fm, next); 1291 } 1292 /* Add to the flow meter list. */ 1293 fm->active_state = 1; /* Config meter starts as active. */ 1294 fm->is_enable = params->meter_enable; 1295 fm->shared = !!shared; 1296 __atomic_add_fetch(&fm->profile->ref_cnt, 1, __ATOMIC_RELAXED); 1297 if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) { 1298 fm->def_policy = 1; 1299 fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg); 1300 if (!fm->flow_ipool) 1301 goto error; 1302 } 1303 rte_spinlock_init(&fm->sl); 1304 /* If ASO meter supported, update ASO flow meter by wqe. */ 1305 if (priv->sh->meter_aso_en) { 1306 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm); 1307 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr); 1308 if (ret) 1309 goto error; 1310 if (!priv->mtr_idx_tbl) { 1311 priv->mtr_idx_tbl = 1312 mlx5_l3t_create(MLX5_L3T_TYPE_DWORD); 1313 if (!priv->mtr_idx_tbl) 1314 goto error; 1315 } 1316 data.dword = mtr_idx; 1317 if (mlx5_l3t_set_entry(priv->mtr_idx_tbl, meter_id, &data)) 1318 goto error; 1319 } else if (!params->meter_enable && mlx5_flow_meter_disable(dev, meter_id, error)) { 1320 goto error; 1321 } 1322 fm->active_state = params->meter_enable; 1323 if (mtr_policy) 1324 __atomic_add_fetch(&mtr_policy->ref_cnt, 1, __ATOMIC_RELAXED); 1325 return 0; 1326 error: 1327 mlx5_flow_destroy_mtr_tbls(dev, fm); 1328 /* Free policer counters. */ 1329 if (fm->drop_cnt) 1330 mlx5_counter_free(dev, fm->drop_cnt); 1331 if (priv->sh->meter_aso_en) 1332 mlx5_flow_mtr_free(dev, mtr_idx); 1333 else 1334 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], mtr_idx); 1335 return -rte_mtr_error_set(error, ENOTSUP, 1336 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 1337 NULL, "Failed to create devx meter."); 1338 } 1339 1340 static int 1341 mlx5_flow_meter_params_flush(struct rte_eth_dev *dev, 1342 struct mlx5_flow_meter_info *fm, 1343 uint32_t mtr_idx) 1344 { 1345 struct mlx5_priv *priv = dev->data->dev_private; 1346 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; 1347 struct mlx5_flow_meter_profile *fmp; 1348 struct mlx5_legacy_flow_meter *legacy_fm = NULL; 1349 struct mlx5_flow_meter_policy *mtr_policy; 1350 1351 /* Meter object must not have any owner. */ 1352 MLX5_ASSERT(!fm->ref_cnt); 1353 /* Get meter profile. */ 1354 fmp = fm->profile; 1355 if (fmp == NULL) 1356 return -1; 1357 /* Update dependencies. */ 1358 __atomic_sub_fetch(&fmp->ref_cnt, 1, __ATOMIC_RELAXED); 1359 fm->profile = NULL; 1360 /* Remove from list. */ 1361 if (!priv->sh->meter_aso_en) { 1362 legacy_fm = container_of(fm, 1363 struct mlx5_legacy_flow_meter, fm); 1364 TAILQ_REMOVE(fms, legacy_fm, next); 1365 } 1366 /* Free drop counters. */ 1367 if (fm->drop_cnt) 1368 mlx5_counter_free(dev, fm->drop_cnt); 1369 /* Free meter flow table. */ 1370 if (fm->flow_ipool) { 1371 mlx5_ipool_destroy(fm->flow_ipool); 1372 fm->flow_ipool = 0; 1373 } 1374 mlx5_flow_destroy_mtr_tbls(dev, fm); 1375 if (fm->def_policy) 1376 __atomic_sub_fetch(&priv->sh->mtrmng->def_policy_ref_cnt, 1377 1, __ATOMIC_RELAXED); 1378 if (priv->sh->meter_aso_en) { 1379 if (!fm->def_policy) { 1380 mtr_policy = mlx5_flow_meter_policy_find(dev, 1381 fm->policy_id, NULL); 1382 if (mtr_policy) 1383 __atomic_sub_fetch(&mtr_policy->ref_cnt, 1384 1, __ATOMIC_RELAXED); 1385 fm->policy_id = 0; 1386 } 1387 fm->def_policy = 0; 1388 if (mlx5_l3t_clear_entry(priv->mtr_idx_tbl, fm->meter_id)) 1389 return -1; 1390 mlx5_flow_mtr_free(dev, mtr_idx); 1391 } else { 1392 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], 1393 legacy_fm->idx); 1394 } 1395 return 0; 1396 } 1397 1398 /** 1399 * Destroy meter rules. 1400 * 1401 * @param[in] dev 1402 * Pointer to Ethernet device. 1403 * @param[in] meter_id 1404 * Meter id. 1405 * @param[out] error 1406 * Pointer to rte meter error structure. 1407 * 1408 * @return 1409 * 0 on success, a negative errno value otherwise and rte_errno is set. 1410 */ 1411 static int 1412 mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id, 1413 struct rte_mtr_error *error) 1414 { 1415 struct mlx5_priv *priv = dev->data->dev_private; 1416 struct mlx5_flow_meter_info *fm; 1417 uint32_t mtr_idx = 0; 1418 1419 if (!priv->mtr_en) 1420 return -rte_mtr_error_set(error, ENOTSUP, 1421 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 1422 NULL, 1423 "Meter is not supported"); 1424 /* Meter object must exist. */ 1425 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 1426 if (fm == NULL) 1427 return -rte_mtr_error_set(error, ENOENT, 1428 RTE_MTR_ERROR_TYPE_MTR_ID, 1429 NULL, 1430 "Meter object id not valid."); 1431 /* Meter object must not have any owner. */ 1432 if (fm->ref_cnt > 0) 1433 return -rte_mtr_error_set(error, EBUSY, 1434 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 1435 NULL, 1436 "Meter object is being used."); 1437 /* Destroy the meter profile. */ 1438 if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx)) 1439 return -rte_mtr_error_set(error, EINVAL, 1440 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 1441 NULL, 1442 "MTR object meter profile invalid."); 1443 return 0; 1444 } 1445 1446 /** 1447 * Modify meter state. 1448 * 1449 * @param[in] priv 1450 * Pointer to mlx5 private data structure. 1451 * @param[in] fm 1452 * Pointer to flow meter. 1453 * @param[in] new_state 1454 * New state to update. 1455 * @param[out] error 1456 * Pointer to rte meter error structure. 1457 * 1458 * @return 1459 * 0 on success, a negative errno value otherwise and rte_errno is set. 1460 */ 1461 static int 1462 mlx5_flow_meter_modify_state(struct mlx5_priv *priv, 1463 struct mlx5_flow_meter_info *fm, 1464 uint32_t new_state, 1465 struct rte_mtr_error *error) 1466 { 1467 static const struct mlx5_flow_meter_srtcm_rfc2697_prm srtcm = { 1468 .cbs_cir = RTE_BE32(MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL), 1469 .ebs_eir = 0, 1470 }; 1471 uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS | 1472 MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR; 1473 int ret; 1474 1475 if (new_state == MLX5_FLOW_METER_DISABLE) 1476 ret = mlx5_flow_meter_action_modify(priv, fm, 1477 &srtcm, modify_bits, 0, 0); 1478 else 1479 ret = mlx5_flow_meter_action_modify(priv, fm, 1480 &fm->profile->srtcm_prm, 1481 modify_bits, 0, 1); 1482 if (ret) 1483 return -rte_mtr_error_set(error, -ret, 1484 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 1485 NULL, 1486 new_state ? 1487 "Failed to enable meter." : 1488 "Failed to disable meter."); 1489 return 0; 1490 } 1491 1492 /** 1493 * Callback to enable flow meter. 1494 * 1495 * @param[in] dev 1496 * Pointer to Ethernet device. 1497 * @param[in] meter_id 1498 * Meter id. 1499 * @param[out] error 1500 * Pointer to rte meter error structure. 1501 * 1502 * @return 1503 * 0 on success, a negative errno value otherwise and rte_errno is set. 1504 */ 1505 static int 1506 mlx5_flow_meter_enable(struct rte_eth_dev *dev, 1507 uint32_t meter_id, 1508 struct rte_mtr_error *error) 1509 { 1510 struct mlx5_priv *priv = dev->data->dev_private; 1511 struct mlx5_flow_meter_info *fm; 1512 int ret; 1513 1514 if (!priv->mtr_en) 1515 return -rte_mtr_error_set(error, ENOTSUP, 1516 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1517 "Meter is not supported"); 1518 /* Meter object must exist. */ 1519 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 1520 if (fm == NULL) 1521 return -rte_mtr_error_set(error, ENOENT, 1522 RTE_MTR_ERROR_TYPE_MTR_ID, 1523 NULL, "Meter not found."); 1524 if (fm->active_state == MLX5_FLOW_METER_ENABLE) 1525 return 0; 1526 ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_ENABLE, 1527 error); 1528 if (!ret) 1529 fm->active_state = MLX5_FLOW_METER_ENABLE; 1530 return ret; 1531 } 1532 1533 /** 1534 * Callback to disable flow meter. 1535 * 1536 * @param[in] dev 1537 * Pointer to Ethernet device. 1538 * @param[in] meter_id 1539 * Meter id. 1540 * @param[out] error 1541 * Pointer to rte meter error structure. 1542 * 1543 * @return 1544 * 0 on success, a negative errno value otherwise and rte_errno is set. 1545 */ 1546 static int 1547 mlx5_flow_meter_disable(struct rte_eth_dev *dev, 1548 uint32_t meter_id, 1549 struct rte_mtr_error *error) 1550 { 1551 struct mlx5_priv *priv = dev->data->dev_private; 1552 struct mlx5_flow_meter_info *fm; 1553 int ret; 1554 1555 if (!priv->mtr_en) 1556 return -rte_mtr_error_set(error, ENOTSUP, 1557 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1558 "Meter is not supported"); 1559 /* Meter object must exist. */ 1560 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 1561 if (fm == NULL) 1562 return -rte_mtr_error_set(error, ENOENT, 1563 RTE_MTR_ERROR_TYPE_MTR_ID, 1564 NULL, "Meter not found."); 1565 if (fm->active_state == MLX5_FLOW_METER_DISABLE) 1566 return 0; 1567 ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_DISABLE, 1568 error); 1569 if (!ret) 1570 fm->active_state = MLX5_FLOW_METER_DISABLE; 1571 return ret; 1572 } 1573 1574 /** 1575 * Callback to update meter profile. 1576 * 1577 * @param[in] dev 1578 * Pointer to Ethernet device. 1579 * @param[in] meter_id 1580 * Meter id. 1581 * @param[in] meter_profile_id 1582 * To be updated meter profile id. 1583 * @param[out] error 1584 * Pointer to rte meter error structure. 1585 * 1586 * @return 1587 * 0 on success, a negative errno value otherwise and rte_errno is set. 1588 */ 1589 static int 1590 mlx5_flow_meter_profile_update(struct rte_eth_dev *dev, 1591 uint32_t meter_id, 1592 uint32_t meter_profile_id, 1593 struct rte_mtr_error *error) 1594 { 1595 struct mlx5_priv *priv = dev->data->dev_private; 1596 struct mlx5_flow_meter_profile *fmp; 1597 struct mlx5_flow_meter_profile *old_fmp; 1598 struct mlx5_flow_meter_info *fm; 1599 uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS | 1600 MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR; 1601 int ret; 1602 1603 if (!priv->mtr_en) 1604 return -rte_mtr_error_set(error, ENOTSUP, 1605 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1606 "Meter is not supported"); 1607 /* Meter profile must exist. */ 1608 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 1609 if (fmp == NULL) 1610 return -rte_mtr_error_set(error, ENOENT, 1611 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 1612 NULL, "Meter profile not found."); 1613 /* Meter object must exist. */ 1614 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 1615 if (fm == NULL) 1616 return -rte_mtr_error_set(error, ENOENT, 1617 RTE_MTR_ERROR_TYPE_MTR_ID, 1618 NULL, "Meter not found."); 1619 /* MTR object already set to meter profile id. */ 1620 old_fmp = fm->profile; 1621 if (fmp == old_fmp) 1622 return 0; 1623 /* Update the profile. */ 1624 fm->profile = fmp; 1625 /* Update meter params in HW (if not disabled). */ 1626 if (fm->active_state == MLX5_FLOW_METER_DISABLE) 1627 return 0; 1628 ret = mlx5_flow_meter_action_modify(priv, fm, &fm->profile->srtcm_prm, 1629 modify_bits, fm->active_state, 1); 1630 if (ret) { 1631 fm->profile = old_fmp; 1632 return -rte_mtr_error_set(error, -ret, 1633 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 1634 NULL, "Failed to update meter" 1635 " parameters in hardware."); 1636 } 1637 old_fmp->ref_cnt--; 1638 fmp->ref_cnt++; 1639 return 0; 1640 } 1641 1642 /** 1643 * Callback to update meter stats mask. 1644 * 1645 * @param[in] dev 1646 * Pointer to Ethernet device. 1647 * @param[in] meter_id 1648 * Meter id. 1649 * @param[in] stats_mask 1650 * To be updated stats_mask. 1651 * @param[out] error 1652 * Pointer to rte meter error structure. 1653 * 1654 * @return 1655 * 0 on success, a negative errno value otherwise and rte_errno is set. 1656 */ 1657 static int 1658 mlx5_flow_meter_stats_update(struct rte_eth_dev *dev, 1659 uint32_t meter_id, 1660 uint64_t stats_mask, 1661 struct rte_mtr_error *error) 1662 { 1663 struct mlx5_priv *priv = dev->data->dev_private; 1664 struct mlx5_flow_meter_info *fm; 1665 1666 if (!priv->mtr_en) 1667 return -rte_mtr_error_set(error, ENOTSUP, 1668 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1669 "Meter is not supported"); 1670 /* Meter object must exist. */ 1671 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 1672 if (fm == NULL) 1673 return -rte_mtr_error_set(error, ENOENT, 1674 RTE_MTR_ERROR_TYPE_MTR_ID, 1675 NULL, "Meter object id not valid."); 1676 if (mlx5_flow_meter_stats_enable_update(dev, fm, stats_mask)) 1677 return -rte_mtr_error_set(error, ENOENT, 1678 RTE_MTR_ERROR_TYPE_MTR_ID, 1679 NULL, "Fail to allocate " 1680 "counter for meter."); 1681 return 0; 1682 } 1683 1684 /** 1685 * Callback to read meter statistics. 1686 * 1687 * @param[in] dev 1688 * Pointer to Ethernet device. 1689 * @param[in] meter_id 1690 * Meter id. 1691 * @param[out] stats 1692 * Pointer to store the statistics. 1693 * @param[out] stats_mask 1694 * Pointer to store the stats_mask. 1695 * @param[in] clear 1696 * Statistic to be cleared after read or not. 1697 * @param[out] error 1698 * Pointer to rte meter error structure. 1699 * 1700 * @return 1701 * 0 on success, a negative errno value otherwise and rte_errno is set. 1702 */ 1703 static int 1704 mlx5_flow_meter_stats_read(struct rte_eth_dev *dev, 1705 uint32_t meter_id, 1706 struct rte_mtr_stats *stats, 1707 uint64_t *stats_mask, 1708 int clear, 1709 struct rte_mtr_error *error) 1710 { 1711 struct mlx5_priv *priv = dev->data->dev_private; 1712 struct mlx5_flow_meter_info *fm; 1713 uint64_t pkts; 1714 uint64_t bytes; 1715 int ret = 0; 1716 1717 if (!priv->mtr_en) 1718 return -rte_mtr_error_set(error, ENOTSUP, 1719 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1720 "Meter is not supported"); 1721 /* Meter object must exist. */ 1722 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 1723 if (fm == NULL) 1724 return -rte_mtr_error_set(error, ENOENT, 1725 RTE_MTR_ERROR_TYPE_MTR_ID, 1726 NULL, "Meter object id not valid."); 1727 *stats_mask = 0; 1728 if (fm->bytes_dropped) 1729 *stats_mask |= RTE_MTR_STATS_N_BYTES_DROPPED; 1730 if (fm->pkts_dropped) 1731 *stats_mask |= RTE_MTR_STATS_N_PKTS_DROPPED; 1732 memset(stats, 0, sizeof(*stats)); 1733 if (fm->drop_cnt) { 1734 ret = mlx5_counter_query(dev, fm->drop_cnt, clear, &pkts, 1735 &bytes, NULL); 1736 if (ret) 1737 goto error; 1738 /* If need to read the packets, set it. */ 1739 if (fm->pkts_dropped) 1740 stats->n_pkts_dropped = pkts; 1741 /* If need to read the bytes, set it. */ 1742 if (fm->bytes_dropped) 1743 stats->n_bytes_dropped = bytes; 1744 } 1745 return 0; 1746 error: 1747 return -rte_mtr_error_set(error, ret, RTE_MTR_ERROR_TYPE_STATS, NULL, 1748 "Failed to read meter drop counters."); 1749 } 1750 1751 static const struct rte_mtr_ops mlx5_flow_mtr_ops = { 1752 .capabilities_get = mlx5_flow_mtr_cap_get, 1753 .meter_profile_add = mlx5_flow_meter_profile_add, 1754 .meter_profile_delete = mlx5_flow_meter_profile_delete, 1755 .meter_policy_validate = mlx5_flow_meter_policy_validate, 1756 .meter_policy_add = mlx5_flow_meter_policy_add, 1757 .meter_policy_delete = mlx5_flow_meter_policy_delete, 1758 .create = mlx5_flow_meter_create, 1759 .destroy = mlx5_flow_meter_destroy, 1760 .meter_enable = mlx5_flow_meter_enable, 1761 .meter_disable = mlx5_flow_meter_disable, 1762 .meter_profile_update = mlx5_flow_meter_profile_update, 1763 .meter_dscp_table_update = NULL, 1764 .stats_update = mlx5_flow_meter_stats_update, 1765 .stats_read = mlx5_flow_meter_stats_read, 1766 }; 1767 1768 /** 1769 * Get meter operations. 1770 * 1771 * @param dev 1772 * Pointer to Ethernet device structure. 1773 * @param arg 1774 * Pointer to set the mtr operations. 1775 * 1776 * @return 1777 * Always 0. 1778 */ 1779 int 1780 mlx5_flow_meter_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg) 1781 { 1782 *(const struct rte_mtr_ops **)arg = &mlx5_flow_mtr_ops; 1783 return 0; 1784 } 1785 1786 /** 1787 * Find meter by id. 1788 * 1789 * @param priv 1790 * Pointer to mlx5_priv. 1791 * @param meter_id 1792 * Meter id. 1793 * @param mtr_idx 1794 * Pointer to Meter index. 1795 * 1796 * @return 1797 * Pointer to the meter info found on success, NULL otherwise. 1798 */ 1799 struct mlx5_flow_meter_info * 1800 mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id, 1801 uint32_t *mtr_idx) 1802 { 1803 struct mlx5_legacy_flow_meter *legacy_fm; 1804 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; 1805 struct mlx5_aso_mtr *aso_mtr; 1806 struct mlx5_aso_mtr_pools_mng *pools_mng = 1807 &priv->sh->mtrmng->pools_mng; 1808 union mlx5_l3t_data data; 1809 uint16_t n_valid; 1810 1811 if (priv->sh->meter_aso_en) { 1812 rte_rwlock_read_lock(&pools_mng->resize_mtrwl); 1813 n_valid = pools_mng->n_valid; 1814 rte_rwlock_read_unlock(&pools_mng->resize_mtrwl); 1815 if (!n_valid || !priv->mtr_idx_tbl || 1816 (mlx5_l3t_get_entry(priv->mtr_idx_tbl, meter_id, &data) || 1817 !data.dword)) 1818 return NULL; 1819 if (mtr_idx) 1820 *mtr_idx = data.dword; 1821 aso_mtr = mlx5_aso_meter_by_idx(priv, data.dword); 1822 /* Remove reference taken by the mlx5_l3t_get_entry. */ 1823 mlx5_l3t_clear_entry(priv->mtr_idx_tbl, meter_id); 1824 if (!aso_mtr || aso_mtr->state == ASO_METER_FREE) 1825 return NULL; 1826 return &aso_mtr->fm; 1827 } 1828 TAILQ_FOREACH(legacy_fm, fms, next) 1829 if (meter_id == legacy_fm->fm.meter_id) { 1830 if (mtr_idx) 1831 *mtr_idx = legacy_fm->idx; 1832 return &legacy_fm->fm; 1833 } 1834 return NULL; 1835 } 1836 1837 /** 1838 * Find meter by index. 1839 * 1840 * @param priv 1841 * Pointer to mlx5_priv. 1842 * @param idx 1843 * Meter index. 1844 * 1845 * @return 1846 * Pointer to the meter info found on success, NULL otherwise. 1847 */ 1848 struct mlx5_flow_meter_info * 1849 flow_dv_meter_find_by_idx(struct mlx5_priv *priv, uint32_t idx) 1850 { 1851 struct mlx5_aso_mtr *aso_mtr; 1852 1853 if (priv->sh->meter_aso_en) { 1854 aso_mtr = mlx5_aso_meter_by_idx(priv, idx); 1855 if (!aso_mtr) 1856 return NULL; 1857 return &aso_mtr->fm; 1858 } else { 1859 return mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], idx); 1860 } 1861 } 1862 1863 /** 1864 * Attach meter to flow. 1865 * Unidirectional Meter creation can only be done 1866 * when flow direction is known, i.e. when calling meter_attach. 1867 * 1868 * @param [in] priv 1869 * Pointer to mlx5 private data. 1870 * @param[in] fm 1871 * Pointer to flow meter. 1872 * @param [in] attr 1873 * Pointer to flow attributes. 1874 * @param [out] error 1875 * Pointer to error structure. 1876 * 1877 * @return 1878 * 0 on success, a negative errno value otherwise and rte_errno is set. 1879 */ 1880 int 1881 mlx5_flow_meter_attach(struct mlx5_priv *priv, 1882 struct mlx5_flow_meter_info *fm, 1883 const struct rte_flow_attr *attr, 1884 struct rte_flow_error *error) 1885 { 1886 int ret = 0; 1887 1888 if (priv->sh->meter_aso_en) { 1889 struct mlx5_aso_mtr *aso_mtr; 1890 1891 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm); 1892 if (mlx5_aso_mtr_wait(priv->sh, aso_mtr)) { 1893 return rte_flow_error_set(error, ENOENT, 1894 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1895 NULL, 1896 "Timeout in meter configuration"); 1897 } 1898 rte_spinlock_lock(&fm->sl); 1899 if (fm->shared || !fm->ref_cnt) { 1900 fm->ref_cnt++; 1901 } else { 1902 rte_flow_error_set(error, EINVAL, 1903 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 1904 "Meter cannot be shared"); 1905 ret = -1; 1906 } 1907 rte_spinlock_unlock(&fm->sl); 1908 } else { 1909 rte_spinlock_lock(&fm->sl); 1910 if (fm->meter_action) { 1911 if (fm->shared && 1912 attr->transfer == fm->transfer && 1913 attr->ingress == fm->ingress && 1914 attr->egress == fm->egress) { 1915 fm->ref_cnt++; 1916 } else { 1917 rte_flow_error_set(error, EINVAL, 1918 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 1919 fm->shared ? 1920 "Meter attr not match." : 1921 "Meter cannot be shared."); 1922 ret = -1; 1923 } 1924 } else { 1925 fm->ingress = attr->ingress; 1926 fm->egress = attr->egress; 1927 fm->transfer = attr->transfer; 1928 fm->ref_cnt = 1; 1929 /* This also creates the meter object. */ 1930 fm->meter_action = mlx5_flow_meter_action_create(priv, 1931 fm); 1932 if (!fm->meter_action) { 1933 fm->ref_cnt = 0; 1934 fm->ingress = 0; 1935 fm->egress = 0; 1936 fm->transfer = 0; 1937 rte_flow_error_set(error, EINVAL, 1938 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 1939 "Meter action create failed."); 1940 ret = -1; 1941 } 1942 } 1943 rte_spinlock_unlock(&fm->sl); 1944 } 1945 return ret ? -rte_errno : 0; 1946 } 1947 1948 /** 1949 * Detach meter from flow. 1950 * 1951 * @param [in] priv 1952 * Pointer to mlx5 private data. 1953 * @param [in] fm 1954 * Pointer to flow meter. 1955 */ 1956 void 1957 mlx5_flow_meter_detach(struct mlx5_priv *priv, 1958 struct mlx5_flow_meter_info *fm) 1959 { 1960 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER 1961 rte_spinlock_lock(&fm->sl); 1962 MLX5_ASSERT(fm->ref_cnt); 1963 if (--fm->ref_cnt == 0 && !priv->sh->meter_aso_en) { 1964 mlx5_glue->destroy_flow_action(fm->meter_action); 1965 fm->meter_action = NULL; 1966 fm->ingress = 0; 1967 fm->egress = 0; 1968 fm->transfer = 0; 1969 } 1970 rte_spinlock_unlock(&fm->sl); 1971 #else 1972 (void)priv; 1973 (void)fm; 1974 #endif 1975 } 1976 1977 /** 1978 * Flush meter with Rx queue configuration. 1979 * 1980 * @param[in] dev 1981 * Pointer to Ethernet device. 1982 */ 1983 void 1984 mlx5_flow_meter_rxq_flush(struct rte_eth_dev *dev) 1985 { 1986 struct mlx5_priv *priv = dev->data->dev_private; 1987 struct mlx5_flow_meter_sub_policy *sub_policy; 1988 struct mlx5_flow_meter_policy *mtr_policy; 1989 void *entry; 1990 uint32_t i, policy_idx; 1991 1992 if (!priv->mtr_en) 1993 return; 1994 if (priv->policy_idx_tbl) { 1995 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) { 1996 policy_idx = *(uint32_t *)entry; 1997 sub_policy = mlx5_ipool_get 1998 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 1999 policy_idx); 2000 if (!sub_policy || !sub_policy->main_policy) 2001 continue; 2002 mtr_policy = sub_policy->main_policy; 2003 if (mtr_policy->is_queue || mtr_policy->is_rss) 2004 mlx5_flow_destroy_sub_policy_with_rxq(dev, 2005 mtr_policy); 2006 } 2007 } 2008 } 2009 2010 /** 2011 * Iterate a meter hierarchy and flush all meters and policies if possible. 2012 * 2013 * @param[in] dev 2014 * Pointer to Ethernet device. 2015 * @param[in] fm 2016 * Pointer to flow meter. 2017 * @param[in] mtr_idx 2018 * .Meter's index 2019 * @param[out] error 2020 * Pointer to rte meter error structure. 2021 * 2022 * @return 2023 * 0 on success, a negative errno value otherwise and rte_errno is set. 2024 */ 2025 static int 2026 mlx5_flow_meter_flush_hierarchy(struct rte_eth_dev *dev, 2027 struct mlx5_flow_meter_info *fm, 2028 uint32_t mtr_idx, 2029 struct rte_mtr_error *error) 2030 { 2031 struct mlx5_priv *priv = dev->data->dev_private; 2032 struct mlx5_flow_meter_policy *policy; 2033 uint32_t policy_id; 2034 struct mlx5_flow_meter_info *next_fm; 2035 uint32_t next_mtr_idx; 2036 struct mlx5_flow_meter_policy *next_policy = NULL; 2037 2038 policy = mlx5_flow_meter_policy_find(dev, fm->policy_id, NULL); 2039 MLX5_ASSERT(policy); 2040 while (!fm->ref_cnt && policy->is_hierarchy) { 2041 policy_id = fm->policy_id; 2042 next_fm = mlx5_flow_meter_find(priv, 2043 policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id, 2044 &next_mtr_idx); 2045 if (next_fm) { 2046 next_policy = mlx5_flow_meter_policy_find(dev, 2047 next_fm->policy_id, 2048 NULL); 2049 MLX5_ASSERT(next_policy); 2050 } 2051 if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx)) 2052 return -rte_mtr_error_set(error, ENOTSUP, 2053 RTE_MTR_ERROR_TYPE_MTR_ID, 2054 NULL, 2055 "Failed to flush meter."); 2056 if (policy->ref_cnt) 2057 break; 2058 if (__mlx5_flow_meter_policy_delete(dev, policy_id, 2059 policy, error, true)) 2060 return -rte_errno; 2061 mlx5_free(policy); 2062 if (!next_fm || !next_policy) 2063 break; 2064 fm = next_fm; 2065 mtr_idx = next_mtr_idx; 2066 policy = next_policy; 2067 } 2068 return 0; 2069 } 2070 2071 /** 2072 * Flush all the hierarchy meters and their policies. 2073 * 2074 * @param[in] dev 2075 * Pointer to Ethernet device. 2076 * @param[out] error 2077 * Pointer to rte meter error structure. 2078 * 2079 * @return 2080 * 0 on success, a negative errno value otherwise and rte_errno is set. 2081 */ 2082 static int 2083 mlx5_flow_meter_flush_all_hierarchies(struct rte_eth_dev *dev, 2084 struct rte_mtr_error *error) 2085 { 2086 struct mlx5_priv *priv = dev->data->dev_private; 2087 struct mlx5_flow_meter_info *fm; 2088 struct mlx5_flow_meter_policy *policy; 2089 struct mlx5_flow_meter_sub_policy *sub_policy; 2090 struct mlx5_flow_meter_info *next_fm; 2091 struct mlx5_aso_mtr *aso_mtr; 2092 uint32_t mtr_idx = 0; 2093 uint32_t i, policy_idx; 2094 void *entry; 2095 2096 if (!priv->mtr_idx_tbl || !priv->policy_idx_tbl) 2097 return 0; 2098 MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) { 2099 mtr_idx = *(uint32_t *)entry; 2100 if (!mtr_idx) 2101 continue; 2102 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx); 2103 fm = &aso_mtr->fm; 2104 if (fm->ref_cnt || fm->def_policy) 2105 continue; 2106 if (mlx5_flow_meter_flush_hierarchy(dev, fm, mtr_idx, error)) 2107 return -rte_errno; 2108 } 2109 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) { 2110 policy_idx = *(uint32_t *)entry; 2111 sub_policy = mlx5_ipool_get 2112 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 2113 policy_idx); 2114 if (!sub_policy) 2115 return -rte_mtr_error_set(error, 2116 EINVAL, 2117 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 2118 NULL, "Meter policy invalid."); 2119 policy = sub_policy->main_policy; 2120 if (!policy || !policy->is_hierarchy || policy->ref_cnt) 2121 continue; 2122 next_fm = mlx5_flow_meter_find(priv, 2123 policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id, 2124 &mtr_idx); 2125 if (__mlx5_flow_meter_policy_delete(dev, i, policy, 2126 error, true)) 2127 return -rte_mtr_error_set(error, 2128 EINVAL, 2129 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 2130 NULL, "Meter policy invalid."); 2131 mlx5_free(policy); 2132 if (!next_fm || next_fm->ref_cnt || next_fm->def_policy) 2133 continue; 2134 if (mlx5_flow_meter_flush_hierarchy(dev, next_fm, 2135 mtr_idx, error)) 2136 return -rte_errno; 2137 } 2138 return 0; 2139 } 2140 /** 2141 * Flush meter configuration. 2142 * 2143 * @param[in] dev 2144 * Pointer to Ethernet device. 2145 * @param[out] error 2146 * Pointer to rte meter error structure. 2147 * 2148 * @return 2149 * 0 on success, a negative errno value otherwise and rte_errno is set. 2150 */ 2151 int 2152 mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error) 2153 { 2154 struct mlx5_priv *priv = dev->data->dev_private; 2155 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; 2156 struct mlx5_flow_meter_profile *fmp; 2157 struct mlx5_legacy_flow_meter *legacy_fm; 2158 struct mlx5_flow_meter_info *fm; 2159 struct mlx5_flow_meter_sub_policy *sub_policy; 2160 void *tmp; 2161 uint32_t i, mtr_idx, policy_idx; 2162 void *entry; 2163 struct mlx5_aso_mtr *aso_mtr; 2164 2165 if (!priv->mtr_en) 2166 return 0; 2167 if (priv->sh->meter_aso_en) { 2168 if (mlx5_flow_meter_flush_all_hierarchies(dev, error)) 2169 return -rte_errno; 2170 if (priv->mtr_idx_tbl) { 2171 MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) { 2172 mtr_idx = *(uint32_t *)entry; 2173 if (mtr_idx) { 2174 aso_mtr = 2175 mlx5_aso_meter_by_idx(priv, mtr_idx); 2176 fm = &aso_mtr->fm; 2177 (void)mlx5_flow_meter_params_flush(dev, 2178 fm, mtr_idx); 2179 } 2180 } 2181 mlx5_l3t_destroy(priv->mtr_idx_tbl); 2182 priv->mtr_idx_tbl = NULL; 2183 } 2184 } else { 2185 RTE_TAILQ_FOREACH_SAFE(legacy_fm, fms, next, tmp) { 2186 fm = &legacy_fm->fm; 2187 if (mlx5_flow_meter_params_flush(dev, fm, 0)) 2188 return -rte_mtr_error_set(error, EINVAL, 2189 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 2190 NULL, "MTR object meter profile invalid."); 2191 } 2192 } 2193 if (priv->policy_idx_tbl) { 2194 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) { 2195 policy_idx = *(uint32_t *)entry; 2196 sub_policy = mlx5_ipool_get 2197 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 2198 policy_idx); 2199 if (!sub_policy) 2200 return -rte_mtr_error_set(error, 2201 EINVAL, 2202 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 2203 NULL, "MTR object " 2204 "meter policy invalid."); 2205 if (__mlx5_flow_meter_policy_delete(dev, i, 2206 sub_policy->main_policy, 2207 error, true)) 2208 return -rte_mtr_error_set(error, 2209 EINVAL, 2210 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 2211 NULL, "MTR object " 2212 "meter policy invalid."); 2213 mlx5_free(sub_policy->main_policy); 2214 } 2215 mlx5_l3t_destroy(priv->policy_idx_tbl); 2216 priv->policy_idx_tbl = NULL; 2217 } 2218 if (priv->mtr_profile_tbl) { 2219 MLX5_L3T_FOREACH(priv->mtr_profile_tbl, i, entry) { 2220 fmp = entry; 2221 if (mlx5_flow_meter_profile_delete(dev, fmp->id, 2222 error)) 2223 return -rte_mtr_error_set(error, EINVAL, 2224 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 2225 NULL, "Fail to destroy " 2226 "meter profile."); 2227 } 2228 mlx5_l3t_destroy(priv->mtr_profile_tbl); 2229 priv->mtr_profile_tbl = NULL; 2230 } 2231 /* Delete default policy table. */ 2232 mlx5_flow_destroy_def_policy(dev); 2233 if (priv->sh->refcnt == 1) 2234 mlx5_flow_destroy_mtr_drop_tbls(dev); 2235 return 0; 2236 } 2237