1 // SPDX-License-Identifier: BSD-3-Clause 2 /* 3 * Copyright 2018 Mellanox Technologies, Ltd 4 */ 5 #include <math.h> 6 7 #include <rte_tailq.h> 8 #include <rte_malloc.h> 9 #include <rte_mtr.h> 10 #include <rte_mtr_driver.h> 11 12 #include <mlx5_devx_cmds.h> 13 #include <mlx5_malloc.h> 14 15 #include "mlx5.h" 16 #include "mlx5_flow.h" 17 18 /** 19 * Create the meter action. 20 * 21 * @param priv 22 * Pointer to mlx5_priv. 23 * @param[in] fm 24 * Pointer to flow meter to be converted. 25 * 26 * @return 27 * Pointer to the meter action on success, NULL otherwise. 28 */ 29 static void * 30 mlx5_flow_meter_action_create(struct mlx5_priv *priv, 31 struct mlx5_flow_meter_info *fm) 32 { 33 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER 34 struct mlx5dv_dr_flow_meter_attr mtr_init; 35 uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)]; 36 struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = 37 &fm->profile->srtcm_prm; 38 uint32_t cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir); 39 uint32_t ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir); 40 uint32_t val; 41 enum mlx5_meter_domain domain = 42 fm->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 43 fm->egress ? MLX5_MTR_DOMAIN_EGRESS : 44 MLX5_MTR_DOMAIN_INGRESS; 45 struct mlx5_flow_meter_def_policy *def_policy = 46 priv->sh->mtrmng->def_policy[domain]; 47 48 memset(fmp, 0, MLX5_ST_SZ_BYTES(flow_meter_parameters)); 49 MLX5_SET(flow_meter_parameters, fmp, valid, 1); 50 MLX5_SET(flow_meter_parameters, fmp, bucket_overflow, 1); 51 MLX5_SET(flow_meter_parameters, fmp, 52 start_color, MLX5_FLOW_COLOR_GREEN); 53 MLX5_SET(flow_meter_parameters, fmp, both_buckets_on_green, 0); 54 val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK; 55 MLX5_SET(flow_meter_parameters, fmp, cbs_exponent, val); 56 val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK; 57 MLX5_SET(flow_meter_parameters, fmp, cbs_mantissa, val); 58 val = (cbs_cir >> ASO_DSEG_XIR_EXP_OFFSET) & ASO_DSEG_EXP_MASK; 59 MLX5_SET(flow_meter_parameters, fmp, cir_exponent, val); 60 val = (cbs_cir & ASO_DSEG_MAN_MASK); 61 MLX5_SET(flow_meter_parameters, fmp, cir_mantissa, val); 62 val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK; 63 MLX5_SET(flow_meter_parameters, fmp, ebs_exponent, val); 64 val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK; 65 MLX5_SET(flow_meter_parameters, fmp, ebs_mantissa, val); 66 mtr_init.next_table = def_policy->sub_policy.tbl_rsc->obj; 67 mtr_init.reg_c_index = priv->mtr_color_reg - REG_C_0; 68 mtr_init.flow_meter_parameter = fmp; 69 mtr_init.flow_meter_parameter_sz = 70 MLX5_ST_SZ_BYTES(flow_meter_parameters); 71 mtr_init.active = fm->active_state; 72 return mlx5_glue->dv_create_flow_action_meter(&mtr_init); 73 #else 74 (void)priv; 75 (void)fm; 76 return NULL; 77 #endif 78 } 79 80 /** 81 * Find meter profile by id. 82 * 83 * @param priv 84 * Pointer to mlx5_priv. 85 * @param meter_profile_id 86 * Meter profile id. 87 * 88 * @return 89 * Pointer to the profile found on success, NULL otherwise. 90 */ 91 static struct mlx5_flow_meter_profile * 92 mlx5_flow_meter_profile_find(struct mlx5_priv *priv, uint32_t meter_profile_id) 93 { 94 struct mlx5_flow_meter_profile *fmp; 95 union mlx5_l3t_data data; 96 int32_t ret; 97 98 if (mlx5_l3t_get_entry(priv->mtr_profile_tbl, 99 meter_profile_id, &data) || !data.ptr) 100 return NULL; 101 fmp = data.ptr; 102 /* Remove reference taken by the mlx5_l3t_get_entry. */ 103 ret = mlx5_l3t_clear_entry(priv->mtr_profile_tbl, 104 meter_profile_id); 105 if (!ret || ret == -1) 106 return NULL; 107 return fmp; 108 } 109 110 /** 111 * Validate the MTR profile. 112 * 113 * @param[in] dev 114 * Pointer to Ethernet device. 115 * @param[in] meter_profile_id 116 * Meter profile id. 117 * @param[in] profile 118 * Pointer to meter profile detail. 119 * @param[out] error 120 * Pointer to the error structure. 121 * 122 * @return 123 * 0 on success, a negative errno value otherwise and rte_errno is set. 124 */ 125 static int 126 mlx5_flow_meter_profile_validate(struct rte_eth_dev *dev, 127 uint32_t meter_profile_id, 128 struct rte_mtr_meter_profile *profile, 129 struct rte_mtr_error *error) 130 { 131 struct mlx5_priv *priv = dev->data->dev_private; 132 struct mlx5_flow_meter_profile *fmp; 133 uint32_t ls_factor; 134 int ret; 135 uint64_t cir, cbs; 136 uint64_t eir, ebs; 137 uint64_t pir, pbs; 138 139 /* Profile must not be NULL. */ 140 if (profile == NULL) 141 return -rte_mtr_error_set(error, EINVAL, 142 RTE_MTR_ERROR_TYPE_METER_PROFILE, 143 NULL, "Meter profile is null."); 144 /* Meter profile ID must be valid. */ 145 if (meter_profile_id == UINT32_MAX) 146 return -rte_mtr_error_set(error, EINVAL, 147 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 148 NULL, "Meter profile id not valid."); 149 /* Meter profile must not exist. */ 150 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 151 if (fmp) 152 return -rte_mtr_error_set(error, EEXIST, 153 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 154 NULL, 155 "Meter profile already exists."); 156 if (!priv->sh->meter_aso_en) { 157 /* Old version is even not supported. */ 158 if (!priv->config.hca_attr.qos.flow_meter_old) 159 return -rte_mtr_error_set(error, ENOTSUP, 160 RTE_MTR_ERROR_TYPE_METER_PROFILE, 161 NULL, "Metering is not supported."); 162 /* Old FW metering only supports srTCM. */ 163 if (profile->alg != RTE_MTR_SRTCM_RFC2697) { 164 return -rte_mtr_error_set(error, ENOTSUP, 165 RTE_MTR_ERROR_TYPE_METER_PROFILE, 166 NULL, "Metering algorithm is not supported."); 167 } else if (profile->srtcm_rfc2697.ebs) { 168 /* EBS is not supported for old metering. */ 169 return -rte_mtr_error_set(error, ENOTSUP, 170 RTE_MTR_ERROR_TYPE_METER_PROFILE, 171 NULL, "EBS is not supported."); 172 } 173 if (profile->packet_mode) 174 return -rte_mtr_error_set(error, ENOTSUP, 175 RTE_MTR_ERROR_TYPE_METER_PROFILE, NULL, 176 "Metering algorithm packet mode is not supported."); 177 } 178 ls_factor = profile->packet_mode ? MLX5_MTRS_PPS_MAP_BPS_SHIFT : 0; 179 switch (profile->alg) { 180 case RTE_MTR_SRTCM_RFC2697: 181 cir = profile->srtcm_rfc2697.cir << ls_factor; 182 cbs = profile->srtcm_rfc2697.cbs << ls_factor; 183 ebs = profile->srtcm_rfc2697.ebs << ls_factor; 184 /* EBS could be zero for old metering. */ 185 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX && 186 cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX && 187 ebs <= MLX5_SRTCM_XBS_MAX) { 188 ret = 0; 189 } else { 190 ret = -rte_mtr_error_set(error, ENOTSUP, 191 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 192 "Profile values out of range."); 193 } 194 break; 195 case RTE_MTR_TRTCM_RFC2698: 196 cir = profile->trtcm_rfc2698.cir << ls_factor; 197 cbs = profile->trtcm_rfc2698.cbs << ls_factor; 198 pir = profile->trtcm_rfc2698.pir << ls_factor; 199 pbs = profile->trtcm_rfc2698.pbs << ls_factor; 200 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX && 201 cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX && 202 pir >= cir && pir <= (MLX5_SRTCM_XIR_MAX * 2) && 203 pbs >= cbs && pbs <= (MLX5_SRTCM_XBS_MAX * 2)) { 204 ret = 0; 205 } else { 206 ret = -rte_mtr_error_set(error, ENOTSUP, 207 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 208 "Profile values out of range."); 209 } 210 break; 211 case RTE_MTR_TRTCM_RFC4115: 212 cir = profile->trtcm_rfc4115.cir << ls_factor; 213 cbs = profile->trtcm_rfc4115.cbs << ls_factor; 214 eir = profile->trtcm_rfc4115.eir << ls_factor; 215 ebs = profile->trtcm_rfc4115.ebs << ls_factor; 216 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX && 217 cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX && 218 eir <= MLX5_SRTCM_XIR_MAX && ebs <= MLX5_SRTCM_XBS_MAX) { 219 ret = 0; 220 } else { 221 ret = -rte_mtr_error_set(error, ENOTSUP, 222 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 223 "Profile values out of range."); 224 } 225 break; 226 default: 227 ret = -rte_mtr_error_set(error, ENOTSUP, 228 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 229 "Unknown metering algorithm."); 230 break; 231 } 232 return ret; 233 } 234 235 /* 236 * Calculate mantissa and exponent for cir / eir. 237 * 238 * @param[in] xir 239 * Value to be calculated. 240 * @param[out] man 241 * Pointer to the mantissa. 242 * @param[out] exp 243 * Pointer to the exp. 244 */ 245 static inline void 246 mlx5_flow_meter_xir_man_exp_calc(int64_t xir, uint8_t *man, uint8_t *exp) 247 { 248 int64_t _xir; 249 int64_t delta = INT64_MAX; 250 uint8_t _man = 0; 251 uint8_t _exp = 0; 252 uint64_t m, e; 253 254 /* Special case xir == 0 ? both exp and mantissa are 0. */ 255 if (xir == 0) { 256 *man = 0; 257 *exp = 0; 258 return; 259 } 260 for (m = 0; m <= 0xFF; m++) { /* man width 8 bit */ 261 for (e = 0; e <= 0x1F; e++) { /* exp width 5bit */ 262 _xir = (1000000000ULL * m) >> e; 263 if (llabs(xir - _xir) <= delta) { 264 delta = llabs(xir - _xir); 265 _man = m; 266 _exp = e; 267 } 268 } 269 } 270 *man = _man; 271 *exp = _exp; 272 } 273 274 /* 275 * Calculate mantissa and exponent for xbs. 276 * 277 * @param[in] xbs 278 * Value to be calculated. 279 * @param[out] man 280 * Pointer to the mantissa. 281 * @param[out] exp 282 * Pointer to the exp. 283 */ 284 static inline void 285 mlx5_flow_meter_xbs_man_exp_calc(uint64_t xbs, uint8_t *man, uint8_t *exp) 286 { 287 int _exp; 288 double _man; 289 290 /* Special case xbs == 0 ? both exp and mantissa are 0. */ 291 if (xbs == 0) { 292 *man = 0; 293 *exp = 0; 294 return; 295 } 296 /* xbs = xbs_mantissa * 2^xbs_exponent */ 297 _man = frexp(xbs, &_exp); 298 _man = _man * pow(2, MLX5_MAN_WIDTH); 299 _exp = _exp - MLX5_MAN_WIDTH; 300 *man = (uint8_t)ceil(_man); 301 *exp = _exp; 302 } 303 304 /** 305 * Fill the prm meter parameter. 306 * 307 * @param[in,out] fmp 308 * Pointer to meter profile to be converted. 309 * @param[out] error 310 * Pointer to the error structure. 311 * 312 * @return 313 * 0 on success, a negative errno value otherwise and rte_errno is set. 314 */ 315 static int 316 mlx5_flow_meter_param_fill(struct mlx5_flow_meter_profile *fmp, 317 struct rte_mtr_error *error) 318 { 319 struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = &fmp->srtcm_prm; 320 uint8_t man, exp; 321 uint32_t cbs_exp, cbs_man, cir_exp, cir_man; 322 uint32_t eir_exp, eir_man, ebs_exp, ebs_man; 323 uint64_t cir, cbs, eir, ebs; 324 325 switch (fmp->profile.alg) { 326 case RTE_MTR_SRTCM_RFC2697: 327 cir = fmp->profile.srtcm_rfc2697.cir; 328 cbs = fmp->profile.srtcm_rfc2697.cbs; 329 eir = 0; 330 ebs = fmp->profile.srtcm_rfc2697.ebs; 331 break; 332 case RTE_MTR_TRTCM_RFC2698: 333 MLX5_ASSERT(fmp->profile.trtcm_rfc2698.pir > 334 fmp->profile.trtcm_rfc2698.cir && 335 fmp->profile.trtcm_rfc2698.pbs > 336 fmp->profile.trtcm_rfc2698.cbs); 337 cir = fmp->profile.trtcm_rfc2698.cir; 338 cbs = fmp->profile.trtcm_rfc2698.cbs; 339 /* EIR / EBS are filled with PIR / PBS. */ 340 eir = fmp->profile.trtcm_rfc2698.pir; 341 ebs = fmp->profile.trtcm_rfc2698.pbs; 342 break; 343 case RTE_MTR_TRTCM_RFC4115: 344 cir = fmp->profile.trtcm_rfc4115.cir; 345 cbs = fmp->profile.trtcm_rfc4115.cbs; 346 eir = fmp->profile.trtcm_rfc4115.eir; 347 ebs = fmp->profile.trtcm_rfc4115.ebs; 348 break; 349 default: 350 return -rte_mtr_error_set(error, EINVAL, 351 RTE_MTR_ERROR_TYPE_METER_PROFILE, NULL, 352 "Metering algorithm mode is invalid"); 353 } 354 /* Adjust the values for PPS mode. */ 355 if (fmp->profile.packet_mode) { 356 cir <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT; 357 cbs <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT; 358 eir <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT; 359 ebs <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT; 360 } 361 /* cir = 8G * cir_mantissa * 1/(2^cir_exponent)) Bytes/Sec */ 362 mlx5_flow_meter_xir_man_exp_calc(cir, &man, &exp); 363 /* Check if cir mantissa is too large. */ 364 if (exp > ASO_DSEG_XIR_EXP_MASK) 365 return -rte_mtr_error_set(error, ENOTSUP, 366 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 367 "meter profile parameter cir is not supported."); 368 cir_man = man; 369 cir_exp = exp; 370 /* cbs = cbs_mantissa * 2^cbs_exponent */ 371 mlx5_flow_meter_xbs_man_exp_calc(cbs, &man, &exp); 372 /* Check if cbs mantissa is too large. */ 373 if (exp > ASO_DSEG_EXP_MASK) 374 return -rte_mtr_error_set(error, ENOTSUP, 375 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 376 "meter profile parameter cbs is not supported."); 377 cbs_man = man; 378 cbs_exp = exp; 379 srtcm->cbs_cir = rte_cpu_to_be_32(cbs_exp << ASO_DSEG_CBS_EXP_OFFSET | 380 cbs_man << ASO_DSEG_CBS_MAN_OFFSET | 381 cir_exp << ASO_DSEG_XIR_EXP_OFFSET | 382 cir_man); 383 mlx5_flow_meter_xir_man_exp_calc(eir, &man, &exp); 384 /* Check if eir mantissa is too large. */ 385 if (exp > ASO_DSEG_XIR_EXP_MASK) 386 return -rte_mtr_error_set(error, ENOTSUP, 387 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 388 "meter profile parameter eir is not supported."); 389 eir_man = man; 390 eir_exp = exp; 391 mlx5_flow_meter_xbs_man_exp_calc(ebs, &man, &exp); 392 /* Check if ebs mantissa is too large. */ 393 if (exp > ASO_DSEG_EXP_MASK) 394 return -rte_mtr_error_set(error, ENOTSUP, 395 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 396 "meter profile parameter ebs is not supported."); 397 ebs_man = man; 398 ebs_exp = exp; 399 srtcm->ebs_eir = rte_cpu_to_be_32(ebs_exp << ASO_DSEG_EBS_EXP_OFFSET | 400 ebs_man << ASO_DSEG_EBS_MAN_OFFSET | 401 eir_exp << ASO_DSEG_XIR_EXP_OFFSET | 402 eir_man); 403 if (srtcm->cbs_cir) 404 fmp->g_support = 1; 405 if (srtcm->ebs_eir) 406 fmp->y_support = 1; 407 return 0; 408 } 409 410 /** 411 * Callback to get MTR capabilities. 412 * 413 * @param[in] dev 414 * Pointer to Ethernet device. 415 * @param[out] cap 416 * Pointer to save MTR capabilities. 417 * @param[out] error 418 * Pointer to the error structure. 419 * 420 * @return 421 * 0 on success, a negative errno value otherwise and rte_errno is set. 422 */ 423 static int 424 mlx5_flow_mtr_cap_get(struct rte_eth_dev *dev, 425 struct rte_mtr_capabilities *cap, 426 struct rte_mtr_error *error __rte_unused) 427 { 428 struct mlx5_priv *priv = dev->data->dev_private; 429 struct mlx5_hca_qos_attr *qattr = &priv->config.hca_attr.qos; 430 431 if (!priv->mtr_en) 432 return -rte_mtr_error_set(error, ENOTSUP, 433 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 434 "Meter is not supported"); 435 memset(cap, 0, sizeof(*cap)); 436 if (priv->sh->meter_aso_en) { 437 /* 2 meters per one ASO cache line. */ 438 cap->n_max = 1 << (qattr->log_max_num_meter_aso + 1); 439 cap->srtcm_rfc2697_packet_mode_supported = 1; 440 } else { 441 cap->n_max = 1 << qattr->log_max_flow_meter; 442 cap->srtcm_rfc2697_packet_mode_supported = 0; 443 } 444 cap->srtcm_rfc2697_byte_mode_supported = 1; 445 cap->n_shared_max = cap->n_max; 446 cap->identical = 1; 447 cap->shared_identical = 1; 448 cap->shared_n_flows_per_mtr_max = 4 << 20; 449 /* 2M flows can share the same meter. */ 450 cap->chaining_n_mtrs_per_flow_max = 1; /* Chaining is not supported. */ 451 cap->meter_srtcm_rfc2697_n_max = qattr->flow_meter_old ? cap->n_max : 0; 452 cap->meter_rate_max = 1ULL << 40; /* 1 Tera tokens per sec. */ 453 cap->stats_mask = RTE_MTR_STATS_N_BYTES_DROPPED | 454 RTE_MTR_STATS_N_PKTS_DROPPED; 455 return 0; 456 } 457 458 /** 459 * Callback to add MTR profile. 460 * 461 * @param[in] dev 462 * Pointer to Ethernet device. 463 * @param[in] meter_profile_id 464 * Meter profile id. 465 * @param[in] profile 466 * Pointer to meter profile detail. 467 * @param[out] error 468 * Pointer to the error structure. 469 * 470 * @return 471 * 0 on success, a negative errno value otherwise and rte_errno is set. 472 */ 473 static int 474 mlx5_flow_meter_profile_add(struct rte_eth_dev *dev, 475 uint32_t meter_profile_id, 476 struct rte_mtr_meter_profile *profile, 477 struct rte_mtr_error *error) 478 { 479 struct mlx5_priv *priv = dev->data->dev_private; 480 struct mlx5_flow_meter_profile *fmp; 481 union mlx5_l3t_data data; 482 int ret; 483 484 if (!priv->mtr_en) 485 return -rte_mtr_error_set(error, ENOTSUP, 486 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 487 "Meter is not supported"); 488 /* Check input params. */ 489 ret = mlx5_flow_meter_profile_validate(dev, meter_profile_id, 490 profile, error); 491 if (ret) 492 return ret; 493 /* Meter profile memory allocation. */ 494 fmp = mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct mlx5_flow_meter_profile), 495 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 496 if (fmp == NULL) 497 return -rte_mtr_error_set(error, ENOMEM, 498 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 499 NULL, "Meter profile memory " 500 "alloc failed."); 501 /* Fill profile info. */ 502 fmp->id = meter_profile_id; 503 fmp->profile = *profile; 504 /* Fill the flow meter parameters for the PRM. */ 505 ret = mlx5_flow_meter_param_fill(fmp, error); 506 if (ret) 507 goto error; 508 data.ptr = fmp; 509 ret = mlx5_l3t_set_entry(priv->mtr_profile_tbl, 510 meter_profile_id, &data); 511 if (ret) 512 return -rte_mtr_error_set(error, ENOTSUP, 513 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 514 NULL, "Meter profile insert fail."); 515 return 0; 516 error: 517 mlx5_free(fmp); 518 return ret; 519 } 520 521 /** 522 * Callback to delete MTR profile. 523 * 524 * @param[in] dev 525 * Pointer to Ethernet device. 526 * @param[in] meter_profile_id 527 * Meter profile id. 528 * @param[out] error 529 * Pointer to the error structure. 530 * 531 * @return 532 * 0 on success, a negative errno value otherwise and rte_errno is set. 533 */ 534 static int 535 mlx5_flow_meter_profile_delete(struct rte_eth_dev *dev, 536 uint32_t meter_profile_id, 537 struct rte_mtr_error *error) 538 { 539 struct mlx5_priv *priv = dev->data->dev_private; 540 struct mlx5_flow_meter_profile *fmp; 541 542 if (!priv->mtr_en) 543 return -rte_mtr_error_set(error, ENOTSUP, 544 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 545 "Meter is not supported"); 546 /* Meter profile must exist. */ 547 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 548 if (fmp == NULL) 549 return -rte_mtr_error_set(error, ENOENT, 550 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 551 &meter_profile_id, 552 "Meter profile id is invalid."); 553 /* Check profile is unused. */ 554 if (fmp->ref_cnt) 555 return -rte_mtr_error_set(error, EBUSY, 556 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 557 NULL, "Meter profile is in use."); 558 if (mlx5_l3t_clear_entry(priv->mtr_profile_tbl, meter_profile_id)) 559 return -rte_mtr_error_set(error, EBUSY, 560 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 561 NULL, "Meter profile remove fail."); 562 mlx5_free(fmp); 563 return 0; 564 } 565 566 /** 567 * Find policy by id. 568 * 569 * @param[in] dev 570 * Pointer to Ethernet device. 571 * @param policy_id 572 * Policy id. 573 * 574 * @return 575 * Pointer to the policy found on success, NULL otherwise. 576 */ 577 struct mlx5_flow_meter_policy * 578 mlx5_flow_meter_policy_find(struct rte_eth_dev *dev, 579 uint32_t policy_id, 580 uint32_t *policy_idx) 581 { 582 struct mlx5_priv *priv = dev->data->dev_private; 583 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 584 union mlx5_l3t_data data; 585 586 if (policy_id > MLX5_MAX_SUB_POLICY_TBL_NUM || !priv->policy_idx_tbl) 587 return NULL; 588 if (mlx5_l3t_get_entry(priv->policy_idx_tbl, policy_id, &data) || 589 !data.dword) 590 return NULL; 591 if (policy_idx) 592 *policy_idx = data.dword; 593 sub_policy = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 594 data.dword); 595 /* Remove reference taken by the mlx5_l3t_get_entry. */ 596 mlx5_l3t_clear_entry(priv->policy_idx_tbl, policy_id); 597 if (sub_policy) 598 if (sub_policy->main_policy_id) 599 return sub_policy->main_policy; 600 return NULL; 601 } 602 603 /** 604 * Get the last meter's policy from one meter's policy in hierarchy. 605 * 606 * @param[in] dev 607 * Pointer to Ethernet device. 608 * @param[in] policy 609 * Pointer to flow meter policy. 610 * 611 * @return 612 * Pointer to the final meter's policy, or NULL when fail. 613 */ 614 struct mlx5_flow_meter_policy * 615 mlx5_flow_meter_hierarchy_get_final_policy(struct rte_eth_dev *dev, 616 struct mlx5_flow_meter_policy *policy) 617 { 618 struct mlx5_priv *priv = dev->data->dev_private; 619 struct mlx5_flow_meter_info *next_fm; 620 struct mlx5_flow_meter_policy *next_policy = policy; 621 622 while (next_policy->is_hierarchy) { 623 next_fm = mlx5_flow_meter_find(priv, 624 next_policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id, NULL); 625 if (!next_fm || next_fm->def_policy) 626 return NULL; 627 next_policy = mlx5_flow_meter_policy_find(dev, 628 next_fm->policy_id, NULL); 629 MLX5_ASSERT(next_policy); 630 } 631 return next_policy; 632 } 633 634 /** 635 * Callback to check MTR policy action validate 636 * 637 * @param[in] dev 638 * Pointer to Ethernet device. 639 * @param[in] actions 640 * Pointer to meter policy action detail. 641 * @param[out] error 642 * Pointer to the error structure. 643 * 644 * @return 645 * 0 on success, a negative errno value otherwise and rte_errno is set. 646 */ 647 static int 648 mlx5_flow_meter_policy_validate(struct rte_eth_dev *dev, 649 struct rte_mtr_meter_policy_params *policy, 650 struct rte_mtr_error *error) 651 { 652 struct mlx5_priv *priv = dev->data->dev_private; 653 struct rte_flow_attr attr = { .transfer = 654 priv->config.dv_esw_en ? 1 : 0}; 655 bool is_rss = false; 656 uint8_t policy_mode; 657 uint8_t domain_bitmap; 658 int ret; 659 660 if (!priv->mtr_en || !priv->sh->meter_aso_en) 661 return -rte_mtr_error_set(error, ENOTSUP, 662 RTE_MTR_ERROR_TYPE_METER_POLICY, 663 NULL, "meter policy unsupported."); 664 ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr, 665 &is_rss, &domain_bitmap, &policy_mode, error); 666 if (ret) 667 return ret; 668 return 0; 669 } 670 671 static int 672 __mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev, 673 uint32_t policy_id, 674 struct mlx5_flow_meter_policy *mtr_policy, 675 struct rte_mtr_error *error, 676 bool clear_l3t) 677 { 678 struct mlx5_priv *priv = dev->data->dev_private; 679 struct mlx5_flow_meter_sub_policy *sub_policy; 680 uint32_t i, j; 681 uint16_t sub_policy_num; 682 683 rte_spinlock_lock(&mtr_policy->sl); 684 if (mtr_policy->ref_cnt) { 685 rte_spinlock_unlock(&mtr_policy->sl); 686 return -rte_mtr_error_set(error, EBUSY, 687 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 688 NULL, 689 "Meter policy object is being used."); 690 } 691 mlx5_flow_destroy_policy_rules(dev, mtr_policy); 692 mlx5_flow_destroy_mtr_acts(dev, mtr_policy); 693 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 694 sub_policy_num = (mtr_policy->sub_policy_num >> 695 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) & 696 MLX5_MTR_SUB_POLICY_NUM_MASK; 697 if (sub_policy_num) { 698 for (j = 0; j < sub_policy_num; j++) { 699 sub_policy = mtr_policy->sub_policys[i][j]; 700 if (sub_policy) 701 mlx5_ipool_free 702 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 703 sub_policy->idx); 704 } 705 } 706 } 707 if (priv->policy_idx_tbl && clear_l3t) { 708 if (mlx5_l3t_clear_entry(priv->policy_idx_tbl, policy_id)) { 709 rte_spinlock_unlock(&mtr_policy->sl); 710 return -rte_mtr_error_set(error, ENOTSUP, 711 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL, 712 "Fail to delete policy in index table."); 713 } 714 } 715 rte_spinlock_unlock(&mtr_policy->sl); 716 return 0; 717 } 718 719 /** 720 * Callback to add MTR policy. 721 * 722 * @param[in] dev 723 * Pointer to Ethernet device. 724 * @param[out] policy_id 725 * Pointer to policy id 726 * @param[in] actions 727 * Pointer to meter policy action detail. 728 * @param[out] error 729 * Pointer to the error structure. 730 * 731 * @return 732 * 0 on success, a negative errno value otherwise and rte_errno is set. 733 */ 734 static int 735 mlx5_flow_meter_policy_add(struct rte_eth_dev *dev, 736 uint32_t policy_id, 737 struct rte_mtr_meter_policy_params *policy, 738 struct rte_mtr_error *error) 739 { 740 struct mlx5_priv *priv = dev->data->dev_private; 741 struct rte_flow_attr attr = { .transfer = 742 priv->config.dv_esw_en ? 1 : 0}; 743 uint32_t sub_policy_idx = 0; 744 uint32_t policy_idx = 0; 745 struct mlx5_flow_meter_policy *mtr_policy = NULL; 746 struct mlx5_flow_meter_sub_policy *sub_policy; 747 bool is_rss = false; 748 uint8_t policy_mode; 749 uint32_t i; 750 int ret; 751 uint32_t policy_size = sizeof(struct mlx5_flow_meter_policy); 752 uint16_t sub_policy_num; 753 uint8_t domain_bitmap = 0; 754 union mlx5_l3t_data data; 755 bool skip_rule = false; 756 757 if (!priv->mtr_en) 758 return -rte_mtr_error_set(error, ENOTSUP, 759 RTE_MTR_ERROR_TYPE_METER_POLICY, 760 NULL, "meter policy unsupported. "); 761 if (policy_id == MLX5_INVALID_POLICY_ID) 762 return -rte_mtr_error_set(error, ENOTSUP, 763 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 764 NULL, "policy ID is invalid. "); 765 if (policy_id == priv->sh->mtrmng->def_policy_id) 766 return -rte_mtr_error_set(error, EEXIST, 767 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 768 NULL, "default policy ID exists. "); 769 mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx); 770 if (mtr_policy) 771 return -rte_mtr_error_set(error, EEXIST, 772 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 773 NULL, "policy ID exists. "); 774 ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr, 775 &is_rss, &domain_bitmap, 776 &policy_mode, error); 777 if (ret) 778 return ret; 779 if (!domain_bitmap) 780 return -rte_mtr_error_set(error, ENOTSUP, 781 RTE_MTR_ERROR_TYPE_METER_POLICY, 782 NULL, "fail to find policy domain."); 783 if (policy_mode == MLX5_MTR_POLICY_MODE_DEF) { 784 if (priv->sh->mtrmng->def_policy_id != MLX5_INVALID_POLICY_ID) 785 return -rte_mtr_error_set(error, EEXIST, 786 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 787 NULL, "a policy with similar actions " 788 "is already configured"); 789 if (mlx5_flow_create_def_policy(dev)) 790 return -rte_mtr_error_set(error, ENOTSUP, 791 RTE_MTR_ERROR_TYPE_METER_POLICY, 792 NULL, 793 "fail to create non-terminated policy."); 794 priv->sh->mtrmng->def_policy_id = policy_id; 795 return 0; 796 } 797 if (!priv->sh->meter_aso_en) 798 return -rte_mtr_error_set(error, ENOTSUP, 799 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL, 800 "no ASO capability to support the policy "); 801 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 802 if (!(domain_bitmap & (1 << i))) 803 continue; 804 /* 805 * If RSS is found, it means that only the ingress domain can 806 * be supported. It is invalid to support RSS for one color 807 * and egress / transfer domain actions for another. Drop and 808 * jump action should have no impact. 809 */ 810 if (is_rss) { 811 policy_size += 812 sizeof(struct mlx5_flow_meter_sub_policy *) * 813 MLX5_MTR_RSS_MAX_SUB_POLICY; 814 break; 815 } 816 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *); 817 } 818 mtr_policy = mlx5_malloc(MLX5_MEM_ZERO, policy_size, 819 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 820 if (!mtr_policy) 821 return -rte_mtr_error_set(error, ENOMEM, 822 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL, 823 "Memory alloc failed for meter policy."); 824 if (policy_mode == MLX5_MTR_POLICY_MODE_OG) 825 mtr_policy->skip_y = 1; 826 else if (policy_mode == MLX5_MTR_POLICY_MODE_OY) 827 mtr_policy->skip_g = 1; 828 policy_size = sizeof(struct mlx5_flow_meter_policy); 829 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 830 if (!(domain_bitmap & (1 << i))) 831 continue; 832 if (i == MLX5_MTR_DOMAIN_INGRESS) 833 mtr_policy->ingress = 1; 834 if (i == MLX5_MTR_DOMAIN_EGRESS) 835 mtr_policy->egress = 1; 836 if (i == MLX5_MTR_DOMAIN_TRANSFER) 837 mtr_policy->transfer = 1; 838 sub_policy = mlx5_ipool_zmalloc 839 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 840 &sub_policy_idx); 841 if (!sub_policy || 842 sub_policy_idx > MLX5_MAX_SUB_POLICY_TBL_NUM) 843 goto policy_add_err; 844 sub_policy->idx = sub_policy_idx; 845 sub_policy->main_policy = mtr_policy; 846 if (!policy_idx) { 847 policy_idx = sub_policy_idx; 848 sub_policy->main_policy_id = 1; 849 } 850 mtr_policy->sub_policys[i] = 851 (struct mlx5_flow_meter_sub_policy **) 852 ((uint8_t *)mtr_policy + policy_size); 853 mtr_policy->sub_policys[i][0] = sub_policy; 854 sub_policy_num = (mtr_policy->sub_policy_num >> 855 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) & 856 MLX5_MTR_SUB_POLICY_NUM_MASK; 857 sub_policy_num++; 858 mtr_policy->sub_policy_num &= ~(MLX5_MTR_SUB_POLICY_NUM_MASK << 859 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)); 860 mtr_policy->sub_policy_num |= 861 (sub_policy_num & MLX5_MTR_SUB_POLICY_NUM_MASK) << 862 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i); 863 /* 864 * If RSS is found, it means that only the ingress domain can 865 * be supported. It is invalid to support RSS for one color 866 * and egress / transfer domain actions for another. Drop and 867 * jump action should have no impact. 868 */ 869 if (is_rss) { 870 mtr_policy->is_rss = 1; 871 break; 872 } 873 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *); 874 } 875 rte_spinlock_init(&mtr_policy->sl); 876 ret = mlx5_flow_create_mtr_acts(dev, mtr_policy, 877 policy->actions, error); 878 if (ret) 879 goto policy_add_err; 880 if (mtr_policy->is_hierarchy) { 881 struct mlx5_flow_meter_policy *final_policy; 882 883 final_policy = 884 mlx5_flow_meter_hierarchy_get_final_policy(dev, mtr_policy); 885 if (!final_policy) 886 goto policy_add_err; 887 skip_rule = (final_policy->is_rss || final_policy->is_queue); 888 } 889 /* 890 * If either Green or Yellow has queue / RSS action, all the policy 891 * rules will be created later in the flow splitting stage. 892 */ 893 if (!is_rss && !mtr_policy->is_queue && !skip_rule) { 894 /* Create policy rules in HW. */ 895 ret = mlx5_flow_create_policy_rules(dev, mtr_policy); 896 if (ret) 897 goto policy_add_err; 898 } 899 data.dword = policy_idx; 900 if (!priv->policy_idx_tbl) { 901 priv->policy_idx_tbl = mlx5_l3t_create(MLX5_L3T_TYPE_DWORD); 902 if (!priv->policy_idx_tbl) 903 goto policy_add_err; 904 } 905 if (mlx5_l3t_set_entry(priv->policy_idx_tbl, policy_id, &data)) 906 goto policy_add_err; 907 return 0; 908 policy_add_err: 909 if (mtr_policy) { 910 ret = __mlx5_flow_meter_policy_delete(dev, policy_id, 911 mtr_policy, error, false); 912 mlx5_free(mtr_policy); 913 if (ret) 914 return ret; 915 } 916 return -rte_mtr_error_set(error, ENOTSUP, 917 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 918 NULL, "Failed to create devx policy."); 919 } 920 921 /** 922 * Callback to delete MTR policy. 923 * 924 * @param[in] dev 925 * Pointer to Ethernet device. 926 * @param[in] policy_id 927 * Meter policy id. 928 * @param[out] error 929 * Pointer to the error structure. 930 * 931 * @return 932 * 0 on success, a negative errno value otherwise and rte_errno is set. 933 */ 934 static int 935 mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev, 936 uint32_t policy_id, 937 struct rte_mtr_error *error) 938 { 939 struct mlx5_priv *priv = dev->data->dev_private; 940 struct mlx5_flow_meter_policy *mtr_policy; 941 uint32_t policy_idx; 942 int ret; 943 944 if (policy_id == priv->sh->mtrmng->def_policy_id) { 945 if (priv->sh->mtrmng->def_policy_ref_cnt > 0) 946 return -rte_mtr_error_set(error, ENOTSUP, 947 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL, 948 "Meter policy object is being used."); 949 priv->sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID; 950 return 0; 951 } 952 mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx); 953 if (!mtr_policy) 954 return -rte_mtr_error_set(error, ENOTSUP, 955 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL, 956 "Meter policy id is invalid. "); 957 ret = __mlx5_flow_meter_policy_delete(dev, policy_id, mtr_policy, 958 error, true); 959 if (ret) 960 return ret; 961 mlx5_free(mtr_policy); 962 return 0; 963 } 964 965 /** 966 * Check meter validation. 967 * 968 * @param[in] priv 969 * Pointer to mlx5 private data structure. 970 * @param[in] meter_id 971 * Meter id. 972 * @param[in] params 973 * Pointer to rte meter parameters. 974 * @param[out] error 975 * Pointer to rte meter error structure. 976 * 977 * @return 978 * 0 on success, a negative errno value otherwise and rte_errno is set. 979 */ 980 static int 981 mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id, 982 struct rte_mtr_params *params, 983 struct rte_mtr_error *error) 984 { 985 /* Meter must use global drop action. */ 986 if (!priv->sh->dr_drop_action) 987 return -rte_mtr_error_set(error, ENOTSUP, 988 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 989 NULL, 990 "No drop action ready for meter."); 991 /* Meter params must not be NULL. */ 992 if (params == NULL) 993 return -rte_mtr_error_set(error, EINVAL, 994 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 995 NULL, "Meter object params null."); 996 /* Previous meter color is not supported. */ 997 if (params->use_prev_mtr_color) 998 return -rte_mtr_error_set(error, ENOTSUP, 999 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 1000 NULL, 1001 "Previous meter color " 1002 "not supported."); 1003 if (params->meter_policy_id == MLX5_INVALID_POLICY_ID) 1004 return -rte_mtr_error_set(error, ENOENT, 1005 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1006 NULL, "Meter policy id not valid."); 1007 /* Validate meter id. */ 1008 if (mlx5_flow_meter_find(priv, meter_id, NULL)) 1009 return -rte_mtr_error_set(error, EEXIST, 1010 RTE_MTR_ERROR_TYPE_MTR_ID, NULL, 1011 "Meter object already exists."); 1012 return 0; 1013 } 1014 1015 /** 1016 * Modify the flow meter action. 1017 * 1018 * @param[in] priv 1019 * Pointer to mlx5 private data structure. 1020 * @param[in] fm 1021 * Pointer to flow meter to be modified. 1022 * @param[in] srtcm 1023 * Pointer to meter srtcm description parameter. 1024 * @param[in] modify_bits 1025 * The bit in srtcm to be updated. 1026 * @param[in] active_state 1027 * The state to be updated. 1028 * @return 1029 * 0 on success, o negative value otherwise. 1030 */ 1031 static int 1032 mlx5_flow_meter_action_modify(struct mlx5_priv *priv, 1033 struct mlx5_flow_meter_info *fm, 1034 const struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm, 1035 uint64_t modify_bits, uint32_t active_state, uint32_t is_enable) 1036 { 1037 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER 1038 uint32_t in[MLX5_ST_SZ_DW(flow_meter_parameters)] = { 0 }; 1039 uint32_t *attr; 1040 struct mlx5dv_dr_flow_meter_attr mod_attr = { 0 }; 1041 int ret; 1042 struct mlx5_aso_mtr *aso_mtr = NULL; 1043 uint32_t cbs_cir, ebs_eir, val; 1044 1045 if (priv->sh->meter_aso_en) { 1046 fm->is_enable = !!is_enable; 1047 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm); 1048 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr); 1049 if (ret) 1050 return ret; 1051 ret = mlx5_aso_mtr_wait(priv->sh, aso_mtr); 1052 if (ret) 1053 return ret; 1054 } else { 1055 /* Fill command parameters. */ 1056 mod_attr.reg_c_index = priv->mtr_color_reg - REG_C_0; 1057 mod_attr.flow_meter_parameter = in; 1058 mod_attr.flow_meter_parameter_sz = 1059 MLX5_ST_SZ_BYTES(flow_meter_parameters); 1060 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE) 1061 mod_attr.active = !!active_state; 1062 else 1063 mod_attr.active = 0; 1064 attr = in; 1065 cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir); 1066 ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir); 1067 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS) { 1068 val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & 1069 ASO_DSEG_EXP_MASK; 1070 MLX5_SET(flow_meter_parameters, attr, 1071 cbs_exponent, val); 1072 val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & 1073 ASO_DSEG_MAN_MASK; 1074 MLX5_SET(flow_meter_parameters, attr, 1075 cbs_mantissa, val); 1076 } 1077 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR) { 1078 val = (cbs_cir >> ASO_DSEG_XIR_EXP_OFFSET) & 1079 ASO_DSEG_EXP_MASK; 1080 MLX5_SET(flow_meter_parameters, attr, 1081 cir_exponent, val); 1082 val = cbs_cir & ASO_DSEG_MAN_MASK; 1083 MLX5_SET(flow_meter_parameters, attr, 1084 cir_mantissa, val); 1085 } 1086 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_EBS) { 1087 val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & 1088 ASO_DSEG_EXP_MASK; 1089 MLX5_SET(flow_meter_parameters, attr, 1090 ebs_exponent, val); 1091 val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & 1092 ASO_DSEG_MAN_MASK; 1093 MLX5_SET(flow_meter_parameters, attr, 1094 ebs_mantissa, val); 1095 } 1096 /* Apply modifications to meter only if it was created. */ 1097 if (fm->meter_action) { 1098 ret = mlx5_glue->dv_modify_flow_action_meter 1099 (fm->meter_action, &mod_attr, 1100 rte_cpu_to_be_64(modify_bits)); 1101 if (ret) 1102 return ret; 1103 } 1104 /* Update succeeded modify meter parameters. */ 1105 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE) 1106 fm->active_state = !!active_state; 1107 } 1108 return 0; 1109 #else 1110 (void)priv; 1111 (void)fm; 1112 (void)srtcm; 1113 (void)modify_bits; 1114 (void)active_state; 1115 (void)is_enable; 1116 return -ENOTSUP; 1117 #endif 1118 } 1119 1120 static int 1121 mlx5_flow_meter_stats_enable_update(struct rte_eth_dev *dev, 1122 struct mlx5_flow_meter_info *fm, 1123 uint64_t stats_mask) 1124 { 1125 fm->bytes_dropped = 1126 (stats_mask & RTE_MTR_STATS_N_BYTES_DROPPED) ? 1 : 0; 1127 fm->pkts_dropped = (stats_mask & RTE_MTR_STATS_N_PKTS_DROPPED) ? 1 : 0; 1128 if (fm->bytes_dropped || fm->pkts_dropped) { 1129 if (!fm->drop_cnt) { 1130 /* Alloc policer counters. */ 1131 fm->drop_cnt = mlx5_counter_alloc(dev); 1132 if (!fm->drop_cnt) 1133 return -1; 1134 } 1135 } else { 1136 if (fm->drop_cnt) { 1137 mlx5_counter_free(dev, fm->drop_cnt); 1138 fm->drop_cnt = 0; 1139 } 1140 } 1141 return 0; 1142 } 1143 1144 /** 1145 * Create meter rules. 1146 * 1147 * @param[in] dev 1148 * Pointer to Ethernet device. 1149 * @param[in] meter_id 1150 * Meter id. 1151 * @param[in] params 1152 * Pointer to rte meter parameters. 1153 * @param[in] shared 1154 * Meter shared with other flow or not. 1155 * @param[out] error 1156 * Pointer to rte meter error structure. 1157 * 1158 * @return 1159 * 0 on success, a negative errno value otherwise and rte_errno is set. 1160 */ 1161 static int 1162 mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, 1163 struct rte_mtr_params *params, int shared, 1164 struct rte_mtr_error *error) 1165 { 1166 struct mlx5_priv *priv = dev->data->dev_private; 1167 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; 1168 struct mlx5_flow_meter_profile *fmp; 1169 struct mlx5_flow_meter_info *fm; 1170 /* GCC fails to infer legacy_fm is set when !priv->sh->meter_aso_en. */ 1171 struct mlx5_legacy_flow_meter *legacy_fm = NULL; 1172 struct mlx5_flow_meter_policy *mtr_policy = NULL; 1173 struct mlx5_indexed_pool_config flow_ipool_cfg = { 1174 .size = 0, 1175 .trunk_size = 64, 1176 .need_lock = 1, 1177 .type = "mlx5_flow_mtr_flow_id_pool", 1178 }; 1179 struct mlx5_aso_mtr *aso_mtr; 1180 uint32_t mtr_idx, policy_idx; 1181 union mlx5_l3t_data data; 1182 int ret; 1183 uint8_t domain_bitmap; 1184 uint8_t mtr_id_bits; 1185 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 1186 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 1187 1188 if (!priv->mtr_en) 1189 return -rte_mtr_error_set(error, ENOTSUP, 1190 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1191 "Meter is not supported"); 1192 /* Validate the parameters. */ 1193 ret = mlx5_flow_meter_validate(priv, meter_id, params, error); 1194 if (ret) 1195 return ret; 1196 /* Meter profile must exist. */ 1197 fmp = mlx5_flow_meter_profile_find(priv, params->meter_profile_id); 1198 if (fmp == NULL) 1199 return -rte_mtr_error_set(error, ENOENT, 1200 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 1201 NULL, "Meter profile id not valid."); 1202 /* Meter policy must exist. */ 1203 if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) { 1204 __atomic_add_fetch 1205 (&priv->sh->mtrmng->def_policy_ref_cnt, 1206 1, __ATOMIC_RELAXED); 1207 domain_bitmap = MLX5_MTR_ALL_DOMAIN_BIT; 1208 if (!priv->config.dv_esw_en) 1209 domain_bitmap &= ~MLX5_MTR_DOMAIN_TRANSFER_BIT; 1210 } else { 1211 if (!priv->sh->meter_aso_en) 1212 return -rte_mtr_error_set(error, ENOTSUP, 1213 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1214 "Part of the policies cannot be " 1215 "supported without ASO "); 1216 mtr_policy = mlx5_flow_meter_policy_find(dev, 1217 params->meter_policy_id, &policy_idx); 1218 if (!mtr_policy) 1219 return -rte_mtr_error_set(error, ENOENT, 1220 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1221 NULL, "Meter policy id not valid."); 1222 domain_bitmap = (mtr_policy->ingress ? 1223 MLX5_MTR_DOMAIN_INGRESS_BIT : 0) | 1224 (mtr_policy->egress ? 1225 MLX5_MTR_DOMAIN_EGRESS_BIT : 0) | 1226 (mtr_policy->transfer ? 1227 MLX5_MTR_DOMAIN_TRANSFER_BIT : 0); 1228 if (fmp->g_support && mtr_policy->skip_g) 1229 return -rte_mtr_error_set(error, ENOTSUP, 1230 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1231 NULL, "Meter green policy is empty."); 1232 if (fmp->y_support && mtr_policy->skip_y) 1233 return -rte_mtr_error_set(error, ENOTSUP, 1234 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1235 NULL, "Meter yellow policy is empty."); 1236 } 1237 /* Allocate the flow meter memory. */ 1238 if (priv->sh->meter_aso_en) { 1239 mtr_idx = mlx5_flow_mtr_alloc(dev); 1240 if (!mtr_idx) 1241 return -rte_mtr_error_set(error, ENOMEM, 1242 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1243 "Memory alloc failed for meter."); 1244 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx); 1245 fm = &aso_mtr->fm; 1246 } else { 1247 if (fmp->y_support) 1248 return -rte_mtr_error_set(error, ENOMEM, 1249 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1250 "Unsupported profile with yellow."); 1251 legacy_fm = mlx5_ipool_zmalloc 1252 (priv->sh->ipool[MLX5_IPOOL_MTR], &mtr_idx); 1253 if (legacy_fm == NULL) 1254 return -rte_mtr_error_set(error, ENOMEM, 1255 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1256 "Memory alloc failed for meter."); 1257 legacy_fm->idx = mtr_idx; 1258 fm = &legacy_fm->fm; 1259 } 1260 mtr_id_bits = MLX5_REG_BITS - __builtin_clz(mtr_idx); 1261 if ((mtr_id_bits + priv->sh->mtrmng->max_mtr_flow_bits) > 1262 mtr_reg_bits) { 1263 DRV_LOG(ERR, "Meter number exceeds max limit."); 1264 goto error; 1265 } 1266 if (mtr_id_bits > priv->sh->mtrmng->max_mtr_bits) 1267 priv->sh->mtrmng->max_mtr_bits = mtr_id_bits; 1268 /* Fill the flow meter parameters. */ 1269 fm->meter_id = meter_id; 1270 fm->policy_id = params->meter_policy_id; 1271 fm->profile = fmp; 1272 if (mlx5_flow_meter_stats_enable_update(dev, fm, params->stats_mask)) 1273 goto error; 1274 if (mlx5_flow_create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap)) 1275 goto error; 1276 /* Add to the flow meter list. */ 1277 if (!priv->sh->meter_aso_en) { 1278 MLX5_ASSERT(legacy_fm != NULL); 1279 TAILQ_INSERT_TAIL(fms, legacy_fm, next); 1280 } 1281 /* Add to the flow meter list. */ 1282 fm->active_state = 1; /* Config meter starts as active. */ 1283 fm->is_enable = 1; 1284 fm->shared = !!shared; 1285 __atomic_add_fetch(&fm->profile->ref_cnt, 1, __ATOMIC_RELAXED); 1286 if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) { 1287 fm->def_policy = 1; 1288 fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg); 1289 if (!fm->flow_ipool) 1290 goto error; 1291 } 1292 rte_spinlock_init(&fm->sl); 1293 /* If ASO meter supported, update ASO flow meter by wqe. */ 1294 if (priv->sh->meter_aso_en) { 1295 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm); 1296 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr); 1297 if (ret) 1298 goto error; 1299 if (!priv->mtr_idx_tbl) { 1300 priv->mtr_idx_tbl = 1301 mlx5_l3t_create(MLX5_L3T_TYPE_DWORD); 1302 if (!priv->mtr_idx_tbl) 1303 goto error; 1304 } 1305 data.dword = mtr_idx; 1306 if (mlx5_l3t_set_entry(priv->mtr_idx_tbl, meter_id, &data)) 1307 goto error; 1308 } 1309 if (mtr_policy) 1310 __atomic_add_fetch(&mtr_policy->ref_cnt, 1, __ATOMIC_RELAXED); 1311 return 0; 1312 error: 1313 mlx5_flow_destroy_mtr_tbls(dev, fm); 1314 /* Free policer counters. */ 1315 if (fm->drop_cnt) 1316 mlx5_counter_free(dev, fm->drop_cnt); 1317 if (priv->sh->meter_aso_en) 1318 mlx5_flow_mtr_free(dev, mtr_idx); 1319 else 1320 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], mtr_idx); 1321 return -rte_mtr_error_set(error, ENOTSUP, 1322 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 1323 NULL, "Failed to create devx meter."); 1324 } 1325 1326 static int 1327 mlx5_flow_meter_params_flush(struct rte_eth_dev *dev, 1328 struct mlx5_flow_meter_info *fm, 1329 uint32_t mtr_idx) 1330 { 1331 struct mlx5_priv *priv = dev->data->dev_private; 1332 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; 1333 struct mlx5_flow_meter_profile *fmp; 1334 struct mlx5_legacy_flow_meter *legacy_fm = NULL; 1335 struct mlx5_flow_meter_policy *mtr_policy; 1336 1337 /* Meter object must not have any owner. */ 1338 MLX5_ASSERT(!fm->ref_cnt); 1339 /* Get meter profile. */ 1340 fmp = fm->profile; 1341 if (fmp == NULL) 1342 return -1; 1343 /* Update dependencies. */ 1344 __atomic_sub_fetch(&fmp->ref_cnt, 1, __ATOMIC_RELAXED); 1345 fm->profile = NULL; 1346 /* Remove from list. */ 1347 if (!priv->sh->meter_aso_en) { 1348 legacy_fm = container_of(fm, 1349 struct mlx5_legacy_flow_meter, fm); 1350 TAILQ_REMOVE(fms, legacy_fm, next); 1351 } 1352 /* Free drop counters. */ 1353 if (fm->drop_cnt) 1354 mlx5_counter_free(dev, fm->drop_cnt); 1355 /* Free meter flow table. */ 1356 if (fm->flow_ipool) { 1357 mlx5_ipool_destroy(fm->flow_ipool); 1358 fm->flow_ipool = 0; 1359 } 1360 mlx5_flow_destroy_mtr_tbls(dev, fm); 1361 if (fm->def_policy) 1362 __atomic_sub_fetch(&priv->sh->mtrmng->def_policy_ref_cnt, 1363 1, __ATOMIC_RELAXED); 1364 if (priv->sh->meter_aso_en) { 1365 if (!fm->def_policy) { 1366 mtr_policy = mlx5_flow_meter_policy_find(dev, 1367 fm->policy_id, NULL); 1368 if (mtr_policy) 1369 __atomic_sub_fetch(&mtr_policy->ref_cnt, 1370 1, __ATOMIC_RELAXED); 1371 fm->policy_id = 0; 1372 } 1373 fm->def_policy = 0; 1374 if (mlx5_l3t_clear_entry(priv->mtr_idx_tbl, fm->meter_id)) 1375 return -1; 1376 mlx5_flow_mtr_free(dev, mtr_idx); 1377 } else { 1378 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], 1379 legacy_fm->idx); 1380 } 1381 return 0; 1382 } 1383 1384 /** 1385 * Destroy meter rules. 1386 * 1387 * @param[in] dev 1388 * Pointer to Ethernet device. 1389 * @param[in] meter_id 1390 * Meter id. 1391 * @param[out] error 1392 * Pointer to rte meter error structure. 1393 * 1394 * @return 1395 * 0 on success, a negative errno value otherwise and rte_errno is set. 1396 */ 1397 static int 1398 mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id, 1399 struct rte_mtr_error *error) 1400 { 1401 struct mlx5_priv *priv = dev->data->dev_private; 1402 struct mlx5_flow_meter_info *fm; 1403 uint32_t mtr_idx = 0; 1404 1405 if (!priv->mtr_en) 1406 return -rte_mtr_error_set(error, ENOTSUP, 1407 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 1408 NULL, 1409 "Meter is not supported"); 1410 /* Meter object must exist. */ 1411 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 1412 if (fm == NULL) 1413 return -rte_mtr_error_set(error, ENOENT, 1414 RTE_MTR_ERROR_TYPE_MTR_ID, 1415 NULL, 1416 "Meter object id not valid."); 1417 /* Meter object must not have any owner. */ 1418 if (fm->ref_cnt > 0) 1419 return -rte_mtr_error_set(error, EBUSY, 1420 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 1421 NULL, 1422 "Meter object is being used."); 1423 /* Destroy the meter profile. */ 1424 if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx)) 1425 return -rte_mtr_error_set(error, EINVAL, 1426 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 1427 NULL, 1428 "MTR object meter profile invalid."); 1429 return 0; 1430 } 1431 1432 /** 1433 * Modify meter state. 1434 * 1435 * @param[in] priv 1436 * Pointer to mlx5 private data structure. 1437 * @param[in] fm 1438 * Pointer to flow meter. 1439 * @param[in] new_state 1440 * New state to update. 1441 * @param[out] error 1442 * Pointer to rte meter error structure. 1443 * 1444 * @return 1445 * 0 on success, a negative errno value otherwise and rte_errno is set. 1446 */ 1447 static int 1448 mlx5_flow_meter_modify_state(struct mlx5_priv *priv, 1449 struct mlx5_flow_meter_info *fm, 1450 uint32_t new_state, 1451 struct rte_mtr_error *error) 1452 { 1453 static const struct mlx5_flow_meter_srtcm_rfc2697_prm srtcm = { 1454 .cbs_cir = RTE_BE32(MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL), 1455 .ebs_eir = 0, 1456 }; 1457 uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS | 1458 MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR; 1459 int ret; 1460 1461 if (new_state == MLX5_FLOW_METER_DISABLE) 1462 ret = mlx5_flow_meter_action_modify(priv, fm, 1463 &srtcm, modify_bits, 0, 0); 1464 else 1465 ret = mlx5_flow_meter_action_modify(priv, fm, 1466 &fm->profile->srtcm_prm, 1467 modify_bits, 0, 1); 1468 if (ret) 1469 return -rte_mtr_error_set(error, -ret, 1470 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 1471 NULL, 1472 new_state ? 1473 "Failed to enable meter." : 1474 "Failed to disable meter."); 1475 return 0; 1476 } 1477 1478 /** 1479 * Callback to enable flow meter. 1480 * 1481 * @param[in] dev 1482 * Pointer to Ethernet device. 1483 * @param[in] meter_id 1484 * Meter id. 1485 * @param[out] error 1486 * Pointer to rte meter error structure. 1487 * 1488 * @return 1489 * 0 on success, a negative errno value otherwise and rte_errno is set. 1490 */ 1491 static int 1492 mlx5_flow_meter_enable(struct rte_eth_dev *dev, 1493 uint32_t meter_id, 1494 struct rte_mtr_error *error) 1495 { 1496 struct mlx5_priv *priv = dev->data->dev_private; 1497 struct mlx5_flow_meter_info *fm; 1498 int ret; 1499 1500 if (!priv->mtr_en) 1501 return -rte_mtr_error_set(error, ENOTSUP, 1502 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1503 "Meter is not supported"); 1504 /* Meter object must exist. */ 1505 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 1506 if (fm == NULL) 1507 return -rte_mtr_error_set(error, ENOENT, 1508 RTE_MTR_ERROR_TYPE_MTR_ID, 1509 NULL, "Meter not found."); 1510 if (fm->active_state == MLX5_FLOW_METER_ENABLE) 1511 return 0; 1512 ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_ENABLE, 1513 error); 1514 if (!ret) 1515 fm->active_state = MLX5_FLOW_METER_ENABLE; 1516 return ret; 1517 } 1518 1519 /** 1520 * Callback to disable flow meter. 1521 * 1522 * @param[in] dev 1523 * Pointer to Ethernet device. 1524 * @param[in] meter_id 1525 * Meter id. 1526 * @param[out] error 1527 * Pointer to rte meter error structure. 1528 * 1529 * @return 1530 * 0 on success, a negative errno value otherwise and rte_errno is set. 1531 */ 1532 static int 1533 mlx5_flow_meter_disable(struct rte_eth_dev *dev, 1534 uint32_t meter_id, 1535 struct rte_mtr_error *error) 1536 { 1537 struct mlx5_priv *priv = dev->data->dev_private; 1538 struct mlx5_flow_meter_info *fm; 1539 int ret; 1540 1541 if (!priv->mtr_en) 1542 return -rte_mtr_error_set(error, ENOTSUP, 1543 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1544 "Meter is not supported"); 1545 /* Meter object must exist. */ 1546 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 1547 if (fm == NULL) 1548 return -rte_mtr_error_set(error, ENOENT, 1549 RTE_MTR_ERROR_TYPE_MTR_ID, 1550 NULL, "Meter not found."); 1551 if (fm->active_state == MLX5_FLOW_METER_DISABLE) 1552 return 0; 1553 ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_DISABLE, 1554 error); 1555 if (!ret) 1556 fm->active_state = MLX5_FLOW_METER_DISABLE; 1557 return ret; 1558 } 1559 1560 /** 1561 * Callback to update meter profile. 1562 * 1563 * @param[in] dev 1564 * Pointer to Ethernet device. 1565 * @param[in] meter_id 1566 * Meter id. 1567 * @param[in] meter_profile_id 1568 * To be updated meter profile id. 1569 * @param[out] error 1570 * Pointer to rte meter error structure. 1571 * 1572 * @return 1573 * 0 on success, a negative errno value otherwise and rte_errno is set. 1574 */ 1575 static int 1576 mlx5_flow_meter_profile_update(struct rte_eth_dev *dev, 1577 uint32_t meter_id, 1578 uint32_t meter_profile_id, 1579 struct rte_mtr_error *error) 1580 { 1581 struct mlx5_priv *priv = dev->data->dev_private; 1582 struct mlx5_flow_meter_profile *fmp; 1583 struct mlx5_flow_meter_profile *old_fmp; 1584 struct mlx5_flow_meter_info *fm; 1585 uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS | 1586 MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR; 1587 int ret; 1588 1589 if (!priv->mtr_en) 1590 return -rte_mtr_error_set(error, ENOTSUP, 1591 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1592 "Meter is not supported"); 1593 /* Meter profile must exist. */ 1594 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 1595 if (fmp == NULL) 1596 return -rte_mtr_error_set(error, ENOENT, 1597 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 1598 NULL, "Meter profile not found."); 1599 /* Meter object must exist. */ 1600 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 1601 if (fm == NULL) 1602 return -rte_mtr_error_set(error, ENOENT, 1603 RTE_MTR_ERROR_TYPE_MTR_ID, 1604 NULL, "Meter not found."); 1605 /* MTR object already set to meter profile id. */ 1606 old_fmp = fm->profile; 1607 if (fmp == old_fmp) 1608 return 0; 1609 /* Update the profile. */ 1610 fm->profile = fmp; 1611 /* Update meter params in HW (if not disabled). */ 1612 if (fm->active_state == MLX5_FLOW_METER_DISABLE) 1613 return 0; 1614 ret = mlx5_flow_meter_action_modify(priv, fm, &fm->profile->srtcm_prm, 1615 modify_bits, fm->active_state, 1); 1616 if (ret) { 1617 fm->profile = old_fmp; 1618 return -rte_mtr_error_set(error, -ret, 1619 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 1620 NULL, "Failed to update meter" 1621 " parameters in hardware."); 1622 } 1623 old_fmp->ref_cnt--; 1624 fmp->ref_cnt++; 1625 return 0; 1626 } 1627 1628 /** 1629 * Callback to update meter stats mask. 1630 * 1631 * @param[in] dev 1632 * Pointer to Ethernet device. 1633 * @param[in] meter_id 1634 * Meter id. 1635 * @param[in] stats_mask 1636 * To be updated stats_mask. 1637 * @param[out] error 1638 * Pointer to rte meter error structure. 1639 * 1640 * @return 1641 * 0 on success, a negative errno value otherwise and rte_errno is set. 1642 */ 1643 static int 1644 mlx5_flow_meter_stats_update(struct rte_eth_dev *dev, 1645 uint32_t meter_id, 1646 uint64_t stats_mask, 1647 struct rte_mtr_error *error) 1648 { 1649 struct mlx5_priv *priv = dev->data->dev_private; 1650 struct mlx5_flow_meter_info *fm; 1651 1652 if (!priv->mtr_en) 1653 return -rte_mtr_error_set(error, ENOTSUP, 1654 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1655 "Meter is not supported"); 1656 /* Meter object must exist. */ 1657 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 1658 if (fm == NULL) 1659 return -rte_mtr_error_set(error, ENOENT, 1660 RTE_MTR_ERROR_TYPE_MTR_ID, 1661 NULL, "Meter object id not valid."); 1662 if (mlx5_flow_meter_stats_enable_update(dev, fm, stats_mask)) 1663 return -rte_mtr_error_set(error, ENOENT, 1664 RTE_MTR_ERROR_TYPE_MTR_ID, 1665 NULL, "Fail to allocate " 1666 "counter for meter."); 1667 return 0; 1668 } 1669 1670 /** 1671 * Callback to read meter statistics. 1672 * 1673 * @param[in] dev 1674 * Pointer to Ethernet device. 1675 * @param[in] meter_id 1676 * Meter id. 1677 * @param[out] stats 1678 * Pointer to store the statistics. 1679 * @param[out] stats_mask 1680 * Pointer to store the stats_mask. 1681 * @param[in] clear 1682 * Statistic to be cleared after read or not. 1683 * @param[out] error 1684 * Pointer to rte meter error structure. 1685 * 1686 * @return 1687 * 0 on success, a negative errno value otherwise and rte_errno is set. 1688 */ 1689 static int 1690 mlx5_flow_meter_stats_read(struct rte_eth_dev *dev, 1691 uint32_t meter_id, 1692 struct rte_mtr_stats *stats, 1693 uint64_t *stats_mask, 1694 int clear, 1695 struct rte_mtr_error *error) 1696 { 1697 struct mlx5_priv *priv = dev->data->dev_private; 1698 struct mlx5_flow_meter_info *fm; 1699 uint64_t pkts; 1700 uint64_t bytes; 1701 int ret = 0; 1702 1703 if (!priv->mtr_en) 1704 return -rte_mtr_error_set(error, ENOTSUP, 1705 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1706 "Meter is not supported"); 1707 /* Meter object must exist. */ 1708 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 1709 if (fm == NULL) 1710 return -rte_mtr_error_set(error, ENOENT, 1711 RTE_MTR_ERROR_TYPE_MTR_ID, 1712 NULL, "Meter object id not valid."); 1713 *stats_mask = 0; 1714 if (fm->bytes_dropped) 1715 *stats_mask |= RTE_MTR_STATS_N_BYTES_DROPPED; 1716 if (fm->pkts_dropped) 1717 *stats_mask |= RTE_MTR_STATS_N_PKTS_DROPPED; 1718 memset(stats, 0, sizeof(*stats)); 1719 if (fm->drop_cnt) { 1720 ret = mlx5_counter_query(dev, fm->drop_cnt, clear, &pkts, 1721 &bytes); 1722 if (ret) 1723 goto error; 1724 /* If need to read the packets, set it. */ 1725 if (fm->pkts_dropped) 1726 stats->n_pkts_dropped = pkts; 1727 /* If need to read the bytes, set it. */ 1728 if (fm->bytes_dropped) 1729 stats->n_bytes_dropped = bytes; 1730 } 1731 return 0; 1732 error: 1733 return -rte_mtr_error_set(error, ret, RTE_MTR_ERROR_TYPE_STATS, NULL, 1734 "Failed to read meter drop counters."); 1735 } 1736 1737 static const struct rte_mtr_ops mlx5_flow_mtr_ops = { 1738 .capabilities_get = mlx5_flow_mtr_cap_get, 1739 .meter_profile_add = mlx5_flow_meter_profile_add, 1740 .meter_profile_delete = mlx5_flow_meter_profile_delete, 1741 .meter_policy_validate = mlx5_flow_meter_policy_validate, 1742 .meter_policy_add = mlx5_flow_meter_policy_add, 1743 .meter_policy_delete = mlx5_flow_meter_policy_delete, 1744 .create = mlx5_flow_meter_create, 1745 .destroy = mlx5_flow_meter_destroy, 1746 .meter_enable = mlx5_flow_meter_enable, 1747 .meter_disable = mlx5_flow_meter_disable, 1748 .meter_profile_update = mlx5_flow_meter_profile_update, 1749 .meter_dscp_table_update = NULL, 1750 .stats_update = mlx5_flow_meter_stats_update, 1751 .stats_read = mlx5_flow_meter_stats_read, 1752 }; 1753 1754 /** 1755 * Get meter operations. 1756 * 1757 * @param dev 1758 * Pointer to Ethernet device structure. 1759 * @param arg 1760 * Pointer to set the mtr operations. 1761 * 1762 * @return 1763 * Always 0. 1764 */ 1765 int 1766 mlx5_flow_meter_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg) 1767 { 1768 *(const struct rte_mtr_ops **)arg = &mlx5_flow_mtr_ops; 1769 return 0; 1770 } 1771 1772 /** 1773 * Find meter by id. 1774 * 1775 * @param priv 1776 * Pointer to mlx5_priv. 1777 * @param meter_id 1778 * Meter id. 1779 * @param mtr_idx 1780 * Pointer to Meter index. 1781 * 1782 * @return 1783 * Pointer to the meter info found on success, NULL otherwise. 1784 */ 1785 struct mlx5_flow_meter_info * 1786 mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id, 1787 uint32_t *mtr_idx) 1788 { 1789 struct mlx5_legacy_flow_meter *legacy_fm; 1790 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; 1791 struct mlx5_aso_mtr *aso_mtr; 1792 struct mlx5_aso_mtr_pools_mng *pools_mng = 1793 &priv->sh->mtrmng->pools_mng; 1794 union mlx5_l3t_data data; 1795 uint16_t n_valid; 1796 1797 if (priv->sh->meter_aso_en) { 1798 rte_rwlock_read_lock(&pools_mng->resize_mtrwl); 1799 n_valid = pools_mng->n_valid; 1800 rte_rwlock_read_unlock(&pools_mng->resize_mtrwl); 1801 if (!n_valid || !priv->mtr_idx_tbl || 1802 (mlx5_l3t_get_entry(priv->mtr_idx_tbl, meter_id, &data) || 1803 !data.dword)) 1804 return NULL; 1805 if (mtr_idx) 1806 *mtr_idx = data.dword; 1807 aso_mtr = mlx5_aso_meter_by_idx(priv, data.dword); 1808 /* Remove reference taken by the mlx5_l3t_get_entry. */ 1809 mlx5_l3t_clear_entry(priv->mtr_idx_tbl, meter_id); 1810 if (!aso_mtr || aso_mtr->state == ASO_METER_FREE) 1811 return NULL; 1812 return &aso_mtr->fm; 1813 } 1814 TAILQ_FOREACH(legacy_fm, fms, next) 1815 if (meter_id == legacy_fm->fm.meter_id) { 1816 if (mtr_idx) 1817 *mtr_idx = legacy_fm->idx; 1818 return &legacy_fm->fm; 1819 } 1820 return NULL; 1821 } 1822 1823 /** 1824 * Find meter by index. 1825 * 1826 * @param priv 1827 * Pointer to mlx5_priv. 1828 * @param idx 1829 * Meter index. 1830 * 1831 * @return 1832 * Pointer to the meter info found on success, NULL otherwise. 1833 */ 1834 struct mlx5_flow_meter_info * 1835 flow_dv_meter_find_by_idx(struct mlx5_priv *priv, uint32_t idx) 1836 { 1837 struct mlx5_aso_mtr *aso_mtr; 1838 1839 if (priv->sh->meter_aso_en) { 1840 aso_mtr = mlx5_aso_meter_by_idx(priv, idx); 1841 if (!aso_mtr) 1842 return NULL; 1843 return &aso_mtr->fm; 1844 } else { 1845 return mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], idx); 1846 } 1847 } 1848 1849 /** 1850 * Attach meter to flow. 1851 * Unidirectional Meter creation can only be done 1852 * when flow direction is known, i.e. when calling meter_attach. 1853 * 1854 * @param [in] priv 1855 * Pointer to mlx5 private data. 1856 * @param[in] fm 1857 * Pointer to flow meter. 1858 * @param [in] attr 1859 * Pointer to flow attributes. 1860 * @param [out] error 1861 * Pointer to error structure. 1862 * 1863 * @return 1864 * 0 on success, a negative errno value otherwise and rte_errno is set. 1865 */ 1866 int 1867 mlx5_flow_meter_attach(struct mlx5_priv *priv, 1868 struct mlx5_flow_meter_info *fm, 1869 const struct rte_flow_attr *attr, 1870 struct rte_flow_error *error) 1871 { 1872 int ret = 0; 1873 1874 if (priv->sh->meter_aso_en) { 1875 struct mlx5_aso_mtr *aso_mtr; 1876 1877 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm); 1878 if (mlx5_aso_mtr_wait(priv->sh, aso_mtr)) { 1879 return rte_flow_error_set(error, ENOENT, 1880 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1881 NULL, 1882 "Timeout in meter configuration"); 1883 } 1884 rte_spinlock_lock(&fm->sl); 1885 if (fm->shared || !fm->ref_cnt) { 1886 fm->ref_cnt++; 1887 } else { 1888 rte_flow_error_set(error, EINVAL, 1889 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 1890 "Meter cannot be shared"); 1891 ret = -1; 1892 } 1893 rte_spinlock_unlock(&fm->sl); 1894 } else { 1895 rte_spinlock_lock(&fm->sl); 1896 if (fm->meter_action) { 1897 if (fm->shared && 1898 attr->transfer == fm->transfer && 1899 attr->ingress == fm->ingress && 1900 attr->egress == fm->egress) { 1901 fm->ref_cnt++; 1902 } else { 1903 rte_flow_error_set(error, EINVAL, 1904 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 1905 fm->shared ? 1906 "Meter attr not match." : 1907 "Meter cannot be shared."); 1908 ret = -1; 1909 } 1910 } else { 1911 fm->ingress = attr->ingress; 1912 fm->egress = attr->egress; 1913 fm->transfer = attr->transfer; 1914 fm->ref_cnt = 1; 1915 /* This also creates the meter object. */ 1916 fm->meter_action = mlx5_flow_meter_action_create(priv, 1917 fm); 1918 if (!fm->meter_action) { 1919 fm->ref_cnt = 0; 1920 fm->ingress = 0; 1921 fm->egress = 0; 1922 fm->transfer = 0; 1923 rte_flow_error_set(error, EINVAL, 1924 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 1925 "Meter action create failed."); 1926 ret = -1; 1927 } 1928 } 1929 rte_spinlock_unlock(&fm->sl); 1930 } 1931 return ret ? -rte_errno : 0; 1932 } 1933 1934 /** 1935 * Detach meter from flow. 1936 * 1937 * @param [in] priv 1938 * Pointer to mlx5 private data. 1939 * @param [in] fm 1940 * Pointer to flow meter. 1941 */ 1942 void 1943 mlx5_flow_meter_detach(struct mlx5_priv *priv, 1944 struct mlx5_flow_meter_info *fm) 1945 { 1946 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER 1947 rte_spinlock_lock(&fm->sl); 1948 MLX5_ASSERT(fm->ref_cnt); 1949 if (--fm->ref_cnt == 0 && !priv->sh->meter_aso_en) { 1950 mlx5_glue->destroy_flow_action(fm->meter_action); 1951 fm->meter_action = NULL; 1952 fm->ingress = 0; 1953 fm->egress = 0; 1954 fm->transfer = 0; 1955 } 1956 rte_spinlock_unlock(&fm->sl); 1957 #else 1958 (void)priv; 1959 (void)fm; 1960 #endif 1961 } 1962 1963 /** 1964 * Flush meter with Rx queue configuration. 1965 * 1966 * @param[in] dev 1967 * Pointer to Ethernet device. 1968 */ 1969 void 1970 mlx5_flow_meter_rxq_flush(struct rte_eth_dev *dev) 1971 { 1972 struct mlx5_priv *priv = dev->data->dev_private; 1973 struct mlx5_flow_meter_sub_policy *sub_policy; 1974 struct mlx5_flow_meter_policy *mtr_policy; 1975 void *entry; 1976 uint32_t i, policy_idx; 1977 1978 if (!priv->mtr_en) 1979 return; 1980 if (priv->policy_idx_tbl) { 1981 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) { 1982 policy_idx = *(uint32_t *)entry; 1983 sub_policy = mlx5_ipool_get 1984 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 1985 policy_idx); 1986 if (!sub_policy || !sub_policy->main_policy) 1987 continue; 1988 mtr_policy = sub_policy->main_policy; 1989 if (mtr_policy->is_queue || mtr_policy->is_rss) 1990 mlx5_flow_destroy_sub_policy_with_rxq(dev, 1991 mtr_policy); 1992 } 1993 } 1994 } 1995 1996 /** 1997 * Iterate a meter hierarchy and flush all meters and policies if possible. 1998 * 1999 * @param[in] dev 2000 * Pointer to Ethernet device. 2001 * @param[in] fm 2002 * Pointer to flow meter. 2003 * @param[in] mtr_idx 2004 * .Meter's index 2005 * @param[out] error 2006 * Pointer to rte meter error structure. 2007 * 2008 * @return 2009 * 0 on success, a negative errno value otherwise and rte_errno is set. 2010 */ 2011 static int 2012 mlx5_flow_meter_flush_hierarchy(struct rte_eth_dev *dev, 2013 struct mlx5_flow_meter_info *fm, 2014 uint32_t mtr_idx, 2015 struct rte_mtr_error *error) 2016 { 2017 struct mlx5_priv *priv = dev->data->dev_private; 2018 struct mlx5_flow_meter_policy *policy; 2019 uint32_t policy_id; 2020 struct mlx5_flow_meter_info *next_fm; 2021 uint32_t next_mtr_idx; 2022 struct mlx5_flow_meter_policy *next_policy = NULL; 2023 2024 policy = mlx5_flow_meter_policy_find(dev, fm->policy_id, NULL); 2025 MLX5_ASSERT(policy); 2026 while (!fm->ref_cnt && policy->is_hierarchy) { 2027 policy_id = fm->policy_id; 2028 next_fm = mlx5_flow_meter_find(priv, 2029 policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id, 2030 &next_mtr_idx); 2031 if (next_fm) { 2032 next_policy = mlx5_flow_meter_policy_find(dev, 2033 next_fm->policy_id, 2034 NULL); 2035 MLX5_ASSERT(next_policy); 2036 } 2037 if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx)) 2038 return -rte_mtr_error_set(error, ENOTSUP, 2039 RTE_MTR_ERROR_TYPE_MTR_ID, 2040 NULL, 2041 "Failed to flush meter."); 2042 if (policy->ref_cnt) 2043 break; 2044 if (__mlx5_flow_meter_policy_delete(dev, policy_id, 2045 policy, error, true)) 2046 return -rte_errno; 2047 mlx5_free(policy); 2048 if (!next_fm || !next_policy) 2049 break; 2050 fm = next_fm; 2051 mtr_idx = next_mtr_idx; 2052 policy = next_policy; 2053 } 2054 return 0; 2055 } 2056 2057 /** 2058 * Flush all the hierarchy meters and their policies. 2059 * 2060 * @param[in] dev 2061 * Pointer to Ethernet device. 2062 * @param[out] error 2063 * Pointer to rte meter error structure. 2064 * 2065 * @return 2066 * 0 on success, a negative errno value otherwise and rte_errno is set. 2067 */ 2068 static int 2069 mlx5_flow_meter_flush_all_hierarchies(struct rte_eth_dev *dev, 2070 struct rte_mtr_error *error) 2071 { 2072 struct mlx5_priv *priv = dev->data->dev_private; 2073 struct mlx5_flow_meter_info *fm; 2074 struct mlx5_flow_meter_policy *policy; 2075 struct mlx5_flow_meter_sub_policy *sub_policy; 2076 struct mlx5_flow_meter_info *next_fm; 2077 struct mlx5_aso_mtr *aso_mtr; 2078 uint32_t mtr_idx = 0; 2079 uint32_t i, policy_idx; 2080 void *entry; 2081 2082 if (!priv->mtr_idx_tbl || !priv->policy_idx_tbl) 2083 return 0; 2084 MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) { 2085 mtr_idx = *(uint32_t *)entry; 2086 if (!mtr_idx) 2087 continue; 2088 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx); 2089 fm = &aso_mtr->fm; 2090 if (fm->ref_cnt || fm->def_policy) 2091 continue; 2092 if (mlx5_flow_meter_flush_hierarchy(dev, fm, mtr_idx, error)) 2093 return -rte_errno; 2094 } 2095 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) { 2096 policy_idx = *(uint32_t *)entry; 2097 sub_policy = mlx5_ipool_get 2098 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 2099 policy_idx); 2100 if (!sub_policy) 2101 return -rte_mtr_error_set(error, 2102 EINVAL, 2103 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 2104 NULL, "Meter policy invalid."); 2105 policy = sub_policy->main_policy; 2106 if (!policy || !policy->is_hierarchy || policy->ref_cnt) 2107 continue; 2108 next_fm = mlx5_flow_meter_find(priv, 2109 policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id, 2110 &mtr_idx); 2111 if (__mlx5_flow_meter_policy_delete(dev, i, policy, 2112 error, true)) 2113 return -rte_mtr_error_set(error, 2114 EINVAL, 2115 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 2116 NULL, "Meter policy invalid."); 2117 mlx5_free(policy); 2118 if (!next_fm || next_fm->ref_cnt || next_fm->def_policy) 2119 continue; 2120 if (mlx5_flow_meter_flush_hierarchy(dev, next_fm, 2121 mtr_idx, error)) 2122 return -rte_errno; 2123 } 2124 return 0; 2125 } 2126 /** 2127 * Flush meter configuration. 2128 * 2129 * @param[in] dev 2130 * Pointer to Ethernet device. 2131 * @param[out] error 2132 * Pointer to rte meter error structure. 2133 * 2134 * @return 2135 * 0 on success, a negative errno value otherwise and rte_errno is set. 2136 */ 2137 int 2138 mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error) 2139 { 2140 struct mlx5_priv *priv = dev->data->dev_private; 2141 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; 2142 struct mlx5_flow_meter_profile *fmp; 2143 struct mlx5_legacy_flow_meter *legacy_fm; 2144 struct mlx5_flow_meter_info *fm; 2145 struct mlx5_flow_meter_sub_policy *sub_policy; 2146 void *tmp; 2147 uint32_t i, mtr_idx, policy_idx; 2148 void *entry; 2149 struct mlx5_aso_mtr *aso_mtr; 2150 2151 if (!priv->mtr_en) 2152 return 0; 2153 if (priv->sh->meter_aso_en) { 2154 if (mlx5_flow_meter_flush_all_hierarchies(dev, error)) 2155 return -rte_errno; 2156 if (priv->mtr_idx_tbl) { 2157 MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) { 2158 mtr_idx = *(uint32_t *)entry; 2159 if (mtr_idx) { 2160 aso_mtr = 2161 mlx5_aso_meter_by_idx(priv, mtr_idx); 2162 fm = &aso_mtr->fm; 2163 (void)mlx5_flow_meter_params_flush(dev, 2164 fm, mtr_idx); 2165 } 2166 } 2167 mlx5_l3t_destroy(priv->mtr_idx_tbl); 2168 priv->mtr_idx_tbl = NULL; 2169 } 2170 } else { 2171 RTE_TAILQ_FOREACH_SAFE(legacy_fm, fms, next, tmp) { 2172 fm = &legacy_fm->fm; 2173 if (mlx5_flow_meter_params_flush(dev, fm, 0)) 2174 return -rte_mtr_error_set(error, EINVAL, 2175 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 2176 NULL, "MTR object meter profile invalid."); 2177 } 2178 } 2179 if (priv->policy_idx_tbl) { 2180 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) { 2181 policy_idx = *(uint32_t *)entry; 2182 sub_policy = mlx5_ipool_get 2183 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 2184 policy_idx); 2185 if (!sub_policy) 2186 return -rte_mtr_error_set(error, 2187 EINVAL, 2188 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 2189 NULL, "MTR object " 2190 "meter policy invalid."); 2191 if (__mlx5_flow_meter_policy_delete(dev, i, 2192 sub_policy->main_policy, 2193 error, true)) 2194 return -rte_mtr_error_set(error, 2195 EINVAL, 2196 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 2197 NULL, "MTR object " 2198 "meter policy invalid."); 2199 mlx5_free(sub_policy->main_policy); 2200 } 2201 mlx5_l3t_destroy(priv->policy_idx_tbl); 2202 priv->policy_idx_tbl = NULL; 2203 } 2204 if (priv->mtr_profile_tbl) { 2205 MLX5_L3T_FOREACH(priv->mtr_profile_tbl, i, entry) { 2206 fmp = entry; 2207 if (mlx5_flow_meter_profile_delete(dev, fmp->id, 2208 error)) 2209 return -rte_mtr_error_set(error, EINVAL, 2210 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 2211 NULL, "Fail to destroy " 2212 "meter profile."); 2213 } 2214 mlx5_l3t_destroy(priv->mtr_profile_tbl); 2215 priv->mtr_profile_tbl = NULL; 2216 } 2217 /* Delete default policy table. */ 2218 mlx5_flow_destroy_def_policy(dev); 2219 if (priv->sh->refcnt == 1) 2220 mlx5_flow_destroy_mtr_drop_tbls(dev); 2221 return 0; 2222 } 2223