1 // SPDX-License-Identifier: BSD-3-Clause 2 /* 3 * Copyright 2018 Mellanox Technologies, Ltd 4 */ 5 #include <math.h> 6 7 #include <rte_tailq.h> 8 #include <rte_malloc.h> 9 #include <rte_mtr.h> 10 #include <rte_mtr_driver.h> 11 12 #include <mlx5_devx_cmds.h> 13 #include <mlx5_malloc.h> 14 15 #include "mlx5.h" 16 #include "mlx5_flow.h" 17 18 /** 19 * Create the meter action. 20 * 21 * @param priv 22 * Pointer to mlx5_priv. 23 * @param[in] fm 24 * Pointer to flow meter to be converted. 25 * 26 * @return 27 * Pointer to the meter action on success, NULL otherwise. 28 */ 29 static void * 30 mlx5_flow_meter_action_create(struct mlx5_priv *priv, 31 struct mlx5_flow_meter_info *fm) 32 { 33 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER 34 struct mlx5dv_dr_flow_meter_attr mtr_init; 35 uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)]; 36 struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = 37 &fm->profile->srtcm_prm; 38 uint32_t cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir); 39 uint32_t ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir); 40 uint32_t val; 41 enum mlx5_meter_domain domain = 42 fm->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 43 fm->egress ? MLX5_MTR_DOMAIN_EGRESS : 44 MLX5_MTR_DOMAIN_INGRESS; 45 struct mlx5_flow_meter_def_policy *def_policy = 46 priv->sh->mtrmng->def_policy[domain]; 47 48 memset(fmp, 0, MLX5_ST_SZ_BYTES(flow_meter_parameters)); 49 MLX5_SET(flow_meter_parameters, fmp, valid, 1); 50 MLX5_SET(flow_meter_parameters, fmp, bucket_overflow, 1); 51 MLX5_SET(flow_meter_parameters, fmp, 52 start_color, MLX5_FLOW_COLOR_GREEN); 53 MLX5_SET(flow_meter_parameters, fmp, both_buckets_on_green, 0); 54 val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK; 55 MLX5_SET(flow_meter_parameters, fmp, cbs_exponent, val); 56 val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK; 57 MLX5_SET(flow_meter_parameters, fmp, cbs_mantissa, val); 58 val = (cbs_cir >> ASO_DSEG_CIR_EXP_OFFSET) & ASO_DSEG_EXP_MASK; 59 MLX5_SET(flow_meter_parameters, fmp, cir_exponent, val); 60 val = (cbs_cir & ASO_DSEG_MAN_MASK); 61 MLX5_SET(flow_meter_parameters, fmp, cir_mantissa, val); 62 val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK; 63 MLX5_SET(flow_meter_parameters, fmp, ebs_exponent, val); 64 val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK; 65 MLX5_SET(flow_meter_parameters, fmp, ebs_mantissa, val); 66 mtr_init.next_table = def_policy->sub_policy.tbl_rsc->obj; 67 mtr_init.reg_c_index = priv->mtr_color_reg - REG_C_0; 68 mtr_init.flow_meter_parameter = fmp; 69 mtr_init.flow_meter_parameter_sz = 70 MLX5_ST_SZ_BYTES(flow_meter_parameters); 71 mtr_init.active = fm->active_state; 72 return mlx5_glue->dv_create_flow_action_meter(&mtr_init); 73 #else 74 (void)priv; 75 (void)fm; 76 return NULL; 77 #endif 78 } 79 80 /** 81 * Find meter profile by id. 82 * 83 * @param priv 84 * Pointer to mlx5_priv. 85 * @param meter_profile_id 86 * Meter profile id. 87 * 88 * @return 89 * Pointer to the profile found on success, NULL otherwise. 90 */ 91 static struct mlx5_flow_meter_profile * 92 mlx5_flow_meter_profile_find(struct mlx5_priv *priv, uint32_t meter_profile_id) 93 { 94 struct mlx5_mtr_profiles *fmps = &priv->flow_meter_profiles; 95 struct mlx5_flow_meter_profile *fmp; 96 97 TAILQ_FOREACH(fmp, fmps, next) 98 if (meter_profile_id == fmp->id) 99 return fmp; 100 return NULL; 101 } 102 103 /** 104 * Validate the MTR profile. 105 * 106 * @param[in] dev 107 * Pointer to Ethernet device. 108 * @param[in] meter_profile_id 109 * Meter profile id. 110 * @param[in] profile 111 * Pointer to meter profile detail. 112 * @param[out] error 113 * Pointer to the error structure. 114 * 115 * @return 116 * 0 on success, a negative errno value otherwise and rte_errno is set. 117 */ 118 static int 119 mlx5_flow_meter_profile_validate(struct rte_eth_dev *dev, 120 uint32_t meter_profile_id, 121 struct rte_mtr_meter_profile *profile, 122 struct rte_mtr_error *error) 123 { 124 struct mlx5_priv *priv = dev->data->dev_private; 125 struct mlx5_flow_meter_profile *fmp; 126 127 /* Profile must not be NULL. */ 128 if (profile == NULL) 129 return -rte_mtr_error_set(error, EINVAL, 130 RTE_MTR_ERROR_TYPE_METER_PROFILE, 131 NULL, "Meter profile is null."); 132 /* Meter profile ID must be valid. */ 133 if (meter_profile_id == UINT32_MAX) 134 return -rte_mtr_error_set(error, EINVAL, 135 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 136 NULL, "Meter profile id not valid."); 137 /* Meter profile must not exist. */ 138 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 139 if (fmp) 140 return -rte_mtr_error_set(error, EEXIST, 141 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 142 NULL, 143 "Meter profile already exists."); 144 if (profile->alg == RTE_MTR_SRTCM_RFC2697) { 145 if (priv->config.hca_attr.qos.flow_meter_old) { 146 /* Verify support for flow meter parameters. */ 147 if (priv->sh->meter_aso_en && profile->packet_mode) { 148 if (profile->srtcm_rfc2697.cir > 0 && 149 (profile->srtcm_rfc2697.cir << 150 MLX5_MTRS_PPS_MAP_BPS_SHIFT) 151 <= MLX5_SRTCM_CIR_MAX && 152 profile->srtcm_rfc2697.cbs > 0 && 153 (profile->srtcm_rfc2697.cbs << 154 MLX5_MTRS_PPS_MAP_BPS_SHIFT) 155 <= MLX5_SRTCM_CBS_MAX && 156 (profile->srtcm_rfc2697.ebs << 157 MLX5_MTRS_PPS_MAP_BPS_SHIFT) 158 <= MLX5_SRTCM_EBS_MAX) 159 return 0; 160 return -rte_mtr_error_set 161 (error, ENOTSUP, 162 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 163 NULL, 164 profile->srtcm_rfc2697.ebs ? 165 "Metering value ebs must be 0." : 166 "Invalid metering parameters."); 167 } 168 if (profile->srtcm_rfc2697.cir > 0 && 169 profile->srtcm_rfc2697.cir <= 170 MLX5_SRTCM_CIR_MAX && 171 profile->srtcm_rfc2697.cbs > 0 && 172 profile->srtcm_rfc2697.cbs <= 173 MLX5_SRTCM_CBS_MAX && 174 profile->srtcm_rfc2697.ebs <= 175 MLX5_SRTCM_EBS_MAX) 176 return 0; 177 return -rte_mtr_error_set(error, ENOTSUP, 178 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 179 NULL, 180 profile->srtcm_rfc2697.ebs ? 181 "Metering value ebs must be 0." : 182 "Invalid metering parameters."); 183 } 184 } 185 return -rte_mtr_error_set(error, ENOTSUP, 186 RTE_MTR_ERROR_TYPE_METER_PROFILE, 187 NULL, "Metering algorithm not supported."); 188 } 189 190 /** 191 * Calculate mantissa and exponent for cir. 192 * 193 * @param[in] cir 194 * Value to be calculated. 195 * @param[out] man 196 * Pointer to the mantissa. 197 * @param[out] exp 198 * Pointer to the exp. 199 */ 200 static void 201 mlx5_flow_meter_cir_man_exp_calc(int64_t cir, uint8_t *man, uint8_t *exp) 202 { 203 int64_t _cir; 204 int64_t delta = INT64_MAX; 205 uint8_t _man = 0; 206 uint8_t _exp = 0; 207 uint64_t m, e; 208 209 for (m = 0; m <= 0xFF; m++) { /* man width 8 bit */ 210 for (e = 0; e <= 0x1F; e++) { /* exp width 5bit */ 211 _cir = (1000000000ULL * m) >> e; 212 if (llabs(cir - _cir) <= delta) { 213 delta = llabs(cir - _cir); 214 _man = m; 215 _exp = e; 216 } 217 } 218 } 219 *man = _man; 220 *exp = _exp; 221 } 222 223 /** 224 * Calculate mantissa and exponent for xbs. 225 * 226 * @param[in] xbs 227 * Value to be calculated. 228 * @param[out] man 229 * Pointer to the mantissa. 230 * @param[out] exp 231 * Pointer to the exp. 232 */ 233 static void 234 mlx5_flow_meter_xbs_man_exp_calc(uint64_t xbs, uint8_t *man, uint8_t *exp) 235 { 236 int _exp; 237 double _man; 238 239 /* Special case xbs == 0 ? both exp and matissa are 0. */ 240 if (xbs == 0) { 241 *man = 0; 242 *exp = 0; 243 return; 244 } 245 /* xbs = xbs_mantissa * 2^xbs_exponent */ 246 _man = frexp(xbs, &_exp); 247 _man = _man * pow(2, MLX5_MAN_WIDTH); 248 _exp = _exp - MLX5_MAN_WIDTH; 249 *man = (uint8_t)ceil(_man); 250 *exp = _exp; 251 } 252 253 /** 254 * Fill the prm meter parameter. 255 * 256 * @param[in,out] fmp 257 * Pointer to meter profie to be converted. 258 * @param[out] error 259 * Pointer to the error structure. 260 * 261 * @return 262 * 0 on success, a negative errno value otherwise and rte_errno is set. 263 */ 264 static int 265 mlx5_flow_meter_param_fill(struct mlx5_flow_meter_profile *fmp, 266 struct mlx5_priv *priv, struct rte_mtr_error *error) 267 { 268 struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = &fmp->srtcm_prm; 269 uint8_t man, exp; 270 uint32_t cbs_exp, cbs_man, cir_exp, cir_man; 271 uint32_t ebs_exp, ebs_man; 272 uint64_t cir, cbs, ebs; 273 274 if (fmp->profile.alg != RTE_MTR_SRTCM_RFC2697) 275 return -rte_mtr_error_set(error, ENOTSUP, 276 RTE_MTR_ERROR_TYPE_METER_PROFILE, 277 NULL, "Metering algorithm not supported."); 278 if (!priv->sh->meter_aso_en && fmp->profile.packet_mode) 279 return -rte_mtr_error_set(error, ENOTSUP, 280 RTE_MTR_ERROR_TYPE_METER_PROFILE, 281 NULL, "Metering algorithm packet mode not supported."); 282 if (priv->sh->meter_aso_en && fmp->profile.packet_mode) { 283 cir = fmp->profile.srtcm_rfc2697.cir << 284 MLX5_MTRS_PPS_MAP_BPS_SHIFT; 285 cbs = fmp->profile.srtcm_rfc2697.cbs << 286 MLX5_MTRS_PPS_MAP_BPS_SHIFT; 287 ebs = fmp->profile.srtcm_rfc2697.ebs << 288 MLX5_MTRS_PPS_MAP_BPS_SHIFT; 289 } else { 290 cir = fmp->profile.srtcm_rfc2697.cir; 291 cbs = fmp->profile.srtcm_rfc2697.cbs; 292 ebs = fmp->profile.srtcm_rfc2697.ebs; 293 } 294 /* cir = 8G * cir_mantissa * 1/(2^cir_exponent)) Bytes/Sec */ 295 mlx5_flow_meter_cir_man_exp_calc(cir, &man, &exp); 296 /* Check if cir mantissa is too large. */ 297 if (exp > ASO_DSEG_CIR_EXP_MASK) 298 return -rte_mtr_error_set(error, ENOTSUP, 299 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 300 "meter profile parameter cir is" 301 " not supported."); 302 cir_man = man; 303 cir_exp = exp; 304 /* cbs = cbs_mantissa * 2^cbs_exponent */ 305 mlx5_flow_meter_xbs_man_exp_calc(cbs, &man, &exp); 306 /* Check if cbs mantissa is too large. */ 307 if (exp > ASO_DSEG_EXP_MASK) 308 return -rte_mtr_error_set(error, ENOTSUP, 309 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 310 "meter profile parameter cbs is" 311 " not supported."); 312 cbs_man = man; 313 cbs_exp = exp; 314 srtcm->cbs_cir = rte_cpu_to_be_32(cbs_exp << ASO_DSEG_CBS_EXP_OFFSET | 315 cbs_man << ASO_DSEG_CBS_MAN_OFFSET | 316 cir_exp << ASO_DSEG_CIR_EXP_OFFSET | 317 cir_man); 318 mlx5_flow_meter_xbs_man_exp_calc(ebs, &man, &exp); 319 /* Check if ebs mantissa is too large. */ 320 if (exp > ASO_DSEG_EXP_MASK) 321 return -rte_mtr_error_set(error, ENOTSUP, 322 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 323 "meter profile parameter ebs is" 324 " not supported."); 325 ebs_man = man; 326 ebs_exp = exp; 327 srtcm->ebs_eir = rte_cpu_to_be_32(ebs_exp << ASO_DSEG_EBS_EXP_OFFSET | 328 ebs_man << ASO_DSEG_EBS_MAN_OFFSET); 329 return 0; 330 } 331 332 /** 333 * Callback to get MTR capabilities. 334 * 335 * @param[in] dev 336 * Pointer to Ethernet device. 337 * @param[out] cap 338 * Pointer to save MTR capabilities. 339 * @param[out] error 340 * Pointer to the error structure. 341 * 342 * @return 343 * 0 on success, a negative errno value otherwise and rte_errno is set. 344 */ 345 static int 346 mlx5_flow_mtr_cap_get(struct rte_eth_dev *dev, 347 struct rte_mtr_capabilities *cap, 348 struct rte_mtr_error *error __rte_unused) 349 { 350 struct mlx5_priv *priv = dev->data->dev_private; 351 struct mlx5_hca_qos_attr *qattr = &priv->config.hca_attr.qos; 352 353 if (!priv->mtr_en) 354 return -rte_mtr_error_set(error, ENOTSUP, 355 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 356 "Meter is not supported"); 357 memset(cap, 0, sizeof(*cap)); 358 if (priv->sh->meter_aso_en) { 359 /* 2 meters per one ASO cache line. */ 360 cap->n_max = 1 << (qattr->log_max_num_meter_aso + 1); 361 cap->srtcm_rfc2697_packet_mode_supported = 1; 362 } else { 363 cap->n_max = 1 << qattr->log_max_flow_meter; 364 cap->srtcm_rfc2697_packet_mode_supported = 0; 365 } 366 cap->srtcm_rfc2697_byte_mode_supported = 1; 367 cap->n_shared_max = cap->n_max; 368 cap->identical = 1; 369 cap->shared_identical = 1; 370 cap->shared_n_flows_per_mtr_max = 4 << 20; 371 /* 2M flows can share the same meter. */ 372 cap->chaining_n_mtrs_per_flow_max = 1; /* Chaining is not supported. */ 373 cap->meter_srtcm_rfc2697_n_max = qattr->flow_meter_old ? cap->n_max : 0; 374 cap->meter_rate_max = 1ULL << 40; /* 1 Tera tokens per sec. */ 375 cap->stats_mask = RTE_MTR_STATS_N_BYTES_DROPPED | 376 RTE_MTR_STATS_N_PKTS_DROPPED; 377 return 0; 378 } 379 380 /** 381 * Callback to add MTR profile. 382 * 383 * @param[in] dev 384 * Pointer to Ethernet device. 385 * @param[in] meter_profile_id 386 * Meter profile id. 387 * @param[in] profile 388 * Pointer to meter profile detail. 389 * @param[out] error 390 * Pointer to the error structure. 391 * 392 * @return 393 * 0 on success, a negative errno value otherwise and rte_errno is set. 394 */ 395 static int 396 mlx5_flow_meter_profile_add(struct rte_eth_dev *dev, 397 uint32_t meter_profile_id, 398 struct rte_mtr_meter_profile *profile, 399 struct rte_mtr_error *error) 400 { 401 struct mlx5_priv *priv = dev->data->dev_private; 402 struct mlx5_mtr_profiles *fmps = &priv->flow_meter_profiles; 403 struct mlx5_flow_meter_profile *fmp; 404 int ret; 405 406 if (!priv->mtr_en) 407 return -rte_mtr_error_set(error, ENOTSUP, 408 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 409 "Meter is not supported"); 410 /* Check input params. */ 411 ret = mlx5_flow_meter_profile_validate(dev, meter_profile_id, 412 profile, error); 413 if (ret) 414 return ret; 415 /* Meter profile memory allocation. */ 416 fmp = mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct mlx5_flow_meter_profile), 417 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 418 if (fmp == NULL) 419 return -rte_mtr_error_set(error, ENOMEM, 420 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 421 NULL, "Meter profile memory " 422 "alloc failed."); 423 /* Fill profile info. */ 424 fmp->id = meter_profile_id; 425 fmp->profile = *profile; 426 /* Fill the flow meter parameters for the PRM. */ 427 ret = mlx5_flow_meter_param_fill(fmp, priv, error); 428 if (ret) 429 goto error; 430 /* Add to list. */ 431 TAILQ_INSERT_TAIL(fmps, fmp, next); 432 return 0; 433 error: 434 mlx5_free(fmp); 435 return ret; 436 } 437 438 /** 439 * Callback to delete MTR profile. 440 * 441 * @param[in] dev 442 * Pointer to Ethernet device. 443 * @param[in] meter_profile_id 444 * Meter profile id. 445 * @param[out] error 446 * Pointer to the error structure. 447 * 448 * @return 449 * 0 on success, a negative errno value otherwise and rte_errno is set. 450 */ 451 static int 452 mlx5_flow_meter_profile_delete(struct rte_eth_dev *dev, 453 uint32_t meter_profile_id, 454 struct rte_mtr_error *error) 455 { 456 struct mlx5_priv *priv = dev->data->dev_private; 457 struct mlx5_flow_meter_profile *fmp; 458 459 if (!priv->mtr_en) 460 return -rte_mtr_error_set(error, ENOTSUP, 461 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 462 "Meter is not supported"); 463 /* Meter profile must exist. */ 464 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 465 if (fmp == NULL) 466 return -rte_mtr_error_set(error, ENOENT, 467 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 468 &meter_profile_id, 469 "Meter profile id is invalid."); 470 /* Check profile is unused. */ 471 if (fmp->ref_cnt) 472 return -rte_mtr_error_set(error, EBUSY, 473 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 474 NULL, "Meter profile is in use."); 475 /* Remove from list. */ 476 TAILQ_REMOVE(&priv->flow_meter_profiles, fmp, next); 477 mlx5_free(fmp); 478 return 0; 479 } 480 481 /** 482 * Find policy by id. 483 * 484 * @param[in] dev 485 * Pointer to Ethernet device. 486 * @param policy_id 487 * Policy id. 488 * 489 * @return 490 * Pointer to the policy found on success, NULL otherwise. 491 */ 492 struct mlx5_flow_meter_policy * 493 mlx5_flow_meter_policy_find(struct rte_eth_dev *dev, 494 uint32_t policy_id, 495 uint32_t *policy_idx) 496 { 497 struct mlx5_priv *priv = dev->data->dev_private; 498 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 499 union mlx5_l3t_data data; 500 501 if (policy_id > MLX5_MAX_SUB_POLICY_TBL_NUM || 502 !priv->sh->mtrmng->policy_idx_tbl) 503 return NULL; 504 if (mlx5_l3t_get_entry(priv->sh->mtrmng->policy_idx_tbl, 505 policy_id, &data) || 506 !data.dword) 507 return NULL; 508 if (policy_idx) 509 *policy_idx = data.dword; 510 sub_policy = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 511 data.dword); 512 /* Remove reference taken by the mlx5_l3t_get_entry. */ 513 mlx5_l3t_clear_entry(priv->sh->mtrmng->policy_idx_tbl, 514 policy_id); 515 if (sub_policy) 516 if (sub_policy->main_policy_id) 517 return sub_policy->main_policy; 518 return NULL; 519 } 520 521 /** 522 * Callback to check MTR policy action validate 523 * 524 * @param[in] dev 525 * Pointer to Ethernet device. 526 * @param[in] actions 527 * Pointer to meter policy action detail. 528 * @param[out] error 529 * Pointer to the error structure. 530 * 531 * @return 532 * 0 on success, a negative errno value otherwise and rte_errno is set. 533 */ 534 static int 535 mlx5_flow_meter_policy_validate(struct rte_eth_dev *dev, 536 struct rte_mtr_meter_policy_params *policy, 537 struct rte_mtr_error *error) 538 { 539 struct mlx5_priv *priv = dev->data->dev_private; 540 struct rte_flow_attr attr = { .transfer = 541 priv->config.dv_esw_en ? 1 : 0}; 542 bool is_rss = false; 543 bool is_def_policy = false; 544 uint8_t domain_bitmap; 545 int ret; 546 547 if (!priv->mtr_en || !priv->sh->meter_aso_en) 548 return -rte_mtr_error_set(error, ENOTSUP, 549 RTE_MTR_ERROR_TYPE_METER_POLICY, 550 NULL, "meter policy unsupported."); 551 ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr, 552 &is_rss, &domain_bitmap, &is_def_policy, error); 553 if (ret) 554 return ret; 555 return 0; 556 } 557 558 static int 559 __mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev, 560 uint32_t policy_id, 561 struct mlx5_flow_meter_policy *mtr_policy, 562 struct rte_mtr_error *error) 563 { 564 struct mlx5_priv *priv = dev->data->dev_private; 565 struct mlx5_flow_meter_sub_policy *sub_policy; 566 uint32_t i, j; 567 uint16_t sub_policy_num; 568 569 rte_spinlock_lock(&mtr_policy->sl); 570 if (mtr_policy->ref_cnt) { 571 rte_spinlock_unlock(&mtr_policy->sl); 572 return -rte_mtr_error_set(error, EBUSY, 573 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 574 NULL, 575 "Meter policy object is being used."); 576 } 577 mlx5_flow_destroy_policy_rules(dev, mtr_policy); 578 mlx5_flow_destroy_mtr_acts(dev, mtr_policy); 579 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 580 sub_policy_num = (mtr_policy->sub_policy_num >> 581 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) & 582 MLX5_MTR_SUB_POLICY_NUM_MASK; 583 if (sub_policy_num) { 584 for (j = 0; j < sub_policy_num; j++) { 585 sub_policy = mtr_policy->sub_policys[i][j]; 586 if (sub_policy) 587 mlx5_ipool_free 588 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 589 sub_policy->idx); 590 } 591 } 592 } 593 if (priv->sh->mtrmng->policy_idx_tbl) { 594 if (mlx5_l3t_clear_entry(priv->sh->mtrmng->policy_idx_tbl, 595 policy_id)) { 596 rte_spinlock_unlock(&mtr_policy->sl); 597 return -rte_mtr_error_set(error, ENOTSUP, 598 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL, 599 "Fail to delete policy in index table."); 600 } 601 } 602 rte_spinlock_unlock(&mtr_policy->sl); 603 return 0; 604 } 605 606 /** 607 * Callback to add MTR policy. 608 * 609 * @param[in] dev 610 * Pointer to Ethernet device. 611 * @param[out] policy_id 612 * Pointer to policy id 613 * @param[in] actions 614 * Pointer to meter policy action detail. 615 * @param[out] error 616 * Pointer to the error structure. 617 * 618 * @return 619 * 0 on success, a negative errno value otherwise and rte_errno is set. 620 */ 621 static int 622 mlx5_flow_meter_policy_add(struct rte_eth_dev *dev, 623 uint32_t policy_id, 624 struct rte_mtr_meter_policy_params *policy, 625 struct rte_mtr_error *error) 626 { 627 struct mlx5_priv *priv = dev->data->dev_private; 628 struct rte_flow_attr attr = { .transfer = 629 priv->config.dv_esw_en ? 1 : 0}; 630 uint32_t sub_policy_idx = 0; 631 uint32_t policy_idx = 0; 632 struct mlx5_flow_meter_policy *mtr_policy = NULL; 633 struct mlx5_flow_meter_sub_policy *sub_policy; 634 bool is_rss = false; 635 bool is_def_policy = false; 636 uint32_t i; 637 int ret; 638 uint32_t policy_size = sizeof(struct mlx5_flow_meter_policy); 639 uint16_t sub_policy_num; 640 uint8_t domain_bitmap = 0; 641 union mlx5_l3t_data data; 642 643 if (!priv->mtr_en) 644 return -rte_mtr_error_set(error, ENOTSUP, 645 RTE_MTR_ERROR_TYPE_METER_POLICY, 646 NULL, "meter policy unsupported."); 647 if (policy_id == MLX5_INVALID_POLICY_ID) 648 return -rte_mtr_error_set(error, ENOTSUP, 649 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL, 650 "policy ID is invalid. "); 651 if (policy_id == priv->sh->mtrmng->def_policy_id) 652 return -rte_mtr_error_set(error, EEXIST, 653 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL, 654 "policy ID exists. "); 655 mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, 656 &policy_idx); 657 if (mtr_policy) 658 return -rte_mtr_error_set(error, EEXIST, 659 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL, 660 "policy ID exists. "); 661 ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr, 662 &is_rss, &domain_bitmap, &is_def_policy, error); 663 if (ret) 664 return ret; 665 if (!domain_bitmap) 666 return -rte_mtr_error_set(error, ENOTSUP, 667 RTE_MTR_ERROR_TYPE_METER_POLICY, 668 NULL, "fail to find policy domain."); 669 if (is_def_policy) { 670 if (priv->sh->mtrmng->def_policy_id != MLX5_INVALID_POLICY_ID) 671 return -rte_mtr_error_set(error, EEXIST, 672 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 673 NULL, "a policy with similar actions " 674 "is already configured"); 675 if (mlx5_flow_create_def_policy(dev)) 676 return -rte_mtr_error_set(error, ENOTSUP, 677 RTE_MTR_ERROR_TYPE_METER_POLICY, 678 NULL, 679 "fail to create non-terminated policy."); 680 priv->sh->mtrmng->def_policy_id = policy_id; 681 return 0; 682 } 683 if (!priv->sh->meter_aso_en) 684 return -rte_mtr_error_set(error, ENOTSUP, 685 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL, 686 "no ASO capability to support the policy "); 687 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 688 if (!(domain_bitmap & (1 << i))) 689 continue; 690 if (is_rss) { 691 policy_size += 692 sizeof(struct mlx5_flow_meter_sub_policy *) * 693 MLX5_MTR_RSS_MAX_SUB_POLICY; 694 break; 695 } 696 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *); 697 } 698 mtr_policy = mlx5_malloc(MLX5_MEM_ZERO, policy_size, 699 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 700 if (!mtr_policy) 701 return -rte_mtr_error_set(error, ENOMEM, 702 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL, 703 "Memory alloc failed for meter policy."); 704 policy_size = sizeof(struct mlx5_flow_meter_policy); 705 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 706 if (!(domain_bitmap & (1 << i))) 707 continue; 708 if (i == MLX5_MTR_DOMAIN_INGRESS) 709 mtr_policy->ingress = 1; 710 if (i == MLX5_MTR_DOMAIN_EGRESS) 711 mtr_policy->egress = 1; 712 if (i == MLX5_MTR_DOMAIN_TRANSFER) 713 mtr_policy->transfer = 1; 714 sub_policy = mlx5_ipool_zmalloc 715 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 716 &sub_policy_idx); 717 if (!sub_policy) 718 goto policy_add_err; 719 if (sub_policy_idx > MLX5_MAX_SUB_POLICY_TBL_NUM) 720 goto policy_add_err; 721 sub_policy->idx = sub_policy_idx; 722 sub_policy->main_policy = mtr_policy; 723 if (!policy_idx) { 724 policy_idx = sub_policy_idx; 725 sub_policy->main_policy_id = 1; 726 } 727 mtr_policy->sub_policys[i] = 728 (struct mlx5_flow_meter_sub_policy **) 729 ((uint8_t *)mtr_policy + policy_size); 730 mtr_policy->sub_policys[i][0] = sub_policy; 731 sub_policy_num = (mtr_policy->sub_policy_num >> 732 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) & 733 MLX5_MTR_SUB_POLICY_NUM_MASK; 734 sub_policy_num++; 735 mtr_policy->sub_policy_num &= ~(MLX5_MTR_SUB_POLICY_NUM_MASK << 736 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)); 737 mtr_policy->sub_policy_num |= 738 (sub_policy_num & MLX5_MTR_SUB_POLICY_NUM_MASK) << 739 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i); 740 if (is_rss) { 741 mtr_policy->is_rss = 1; 742 break; 743 } 744 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *); 745 } 746 rte_spinlock_init(&mtr_policy->sl); 747 ret = mlx5_flow_create_mtr_acts(dev, mtr_policy, 748 policy->actions, error); 749 if (ret) 750 goto policy_add_err; 751 if (!is_rss) { 752 /* Create policy rules in HW. */ 753 ret = mlx5_flow_create_policy_rules(dev, mtr_policy); 754 if (ret) 755 goto policy_add_err; 756 } 757 data.dword = policy_idx; 758 if (!priv->sh->mtrmng->policy_idx_tbl) { 759 priv->sh->mtrmng->policy_idx_tbl = 760 mlx5_l3t_create(MLX5_L3T_TYPE_DWORD); 761 if (!priv->sh->mtrmng->policy_idx_tbl) 762 goto policy_add_err; 763 } 764 if (mlx5_l3t_set_entry(priv->sh->mtrmng->policy_idx_tbl, 765 policy_id, &data)) 766 goto policy_add_err; 767 return 0; 768 policy_add_err: 769 if (mtr_policy) { 770 ret = __mlx5_flow_meter_policy_delete(dev, policy_id, 771 mtr_policy, error); 772 mlx5_free(mtr_policy); 773 if (ret) 774 return ret; 775 } 776 return -rte_mtr_error_set(error, ENOTSUP, 777 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 778 NULL, "Failed to create devx policy."); 779 } 780 781 /** 782 * Callback to delete MTR policy. 783 * 784 * @param[in] dev 785 * Pointer to Ethernet device. 786 * @param[in] policy_id 787 * Meter policy id. 788 * @param[out] error 789 * Pointer to the error structure. 790 * 791 * @return 792 * 0 on success, a negative errno value otherwise and rte_errno is set. 793 */ 794 static int 795 mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev, 796 uint32_t policy_id, 797 struct rte_mtr_error *error) 798 { 799 struct mlx5_priv *priv = dev->data->dev_private; 800 struct mlx5_flow_meter_policy *mtr_policy; 801 uint32_t policy_idx; 802 int ret; 803 804 if (policy_id == priv->sh->mtrmng->def_policy_id) { 805 if (priv->sh->mtrmng->def_policy_ref_cnt > 0) 806 return -rte_mtr_error_set(error, ENOTSUP, 807 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL, 808 "Meter policy object is being used."); 809 priv->sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID; 810 return 0; 811 } 812 mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx); 813 if (!mtr_policy) 814 return -rte_mtr_error_set(error, ENOTSUP, 815 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL, 816 "Meter policy id is invalid. "); 817 ret = __mlx5_flow_meter_policy_delete(dev, policy_id, mtr_policy, 818 error); 819 if (ret) 820 return ret; 821 mlx5_free(mtr_policy); 822 return 0; 823 } 824 825 /** 826 * Check meter validation. 827 * 828 * @param[in] priv 829 * Pointer to mlx5 private data structure. 830 * @param[in] meter_id 831 * Meter id. 832 * @param[in] params 833 * Pointer to rte meter parameters. 834 * @param[out] error 835 * Pointer to rte meter error structure. 836 * 837 * @return 838 * 0 on success, a negative errno value otherwise and rte_errno is set. 839 */ 840 static int 841 mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id, 842 struct rte_mtr_params *params, 843 struct rte_mtr_error *error) 844 { 845 /* Meter must use global drop action. */ 846 if (!priv->sh->dr_drop_action) 847 return -rte_mtr_error_set(error, ENOTSUP, 848 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 849 NULL, 850 "No drop action ready for meter."); 851 /* Meter params must not be NULL. */ 852 if (params == NULL) 853 return -rte_mtr_error_set(error, EINVAL, 854 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 855 NULL, "Meter object params null."); 856 /* Previous meter color is not supported. */ 857 if (params->use_prev_mtr_color) 858 return -rte_mtr_error_set(error, ENOTSUP, 859 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 860 NULL, 861 "Previous meter color " 862 "not supported."); 863 if (params->meter_policy_id == MLX5_INVALID_POLICY_ID) 864 return -rte_mtr_error_set(error, ENOENT, 865 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 866 NULL, "Meter policy id not valid."); 867 /* Validate meter id. */ 868 if (mlx5_flow_meter_find(priv, meter_id, NULL)) 869 return -rte_mtr_error_set(error, EEXIST, 870 RTE_MTR_ERROR_TYPE_MTR_ID, NULL, 871 "Meter object already exists."); 872 return 0; 873 } 874 875 /** 876 * Modify the flow meter action. 877 * 878 * @param[in] priv 879 * Pointer to mlx5 private data structure. 880 * @param[in] fm 881 * Pointer to flow meter to be modified. 882 * @param[in] srtcm 883 * Pointer to meter srtcm description parameter. 884 * @param[in] modify_bits 885 * The bit in srtcm to be updated. 886 * @param[in] active_state 887 * The state to be updated. 888 * @return 889 * 0 on success, o negative value otherwise. 890 */ 891 static int 892 mlx5_flow_meter_action_modify(struct mlx5_priv *priv, 893 struct mlx5_flow_meter_info *fm, 894 const struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm, 895 uint64_t modify_bits, uint32_t active_state, uint32_t is_enable) 896 { 897 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER 898 uint32_t in[MLX5_ST_SZ_DW(flow_meter_parameters)] = { 0 }; 899 uint32_t *attr; 900 struct mlx5dv_dr_flow_meter_attr mod_attr = { 0 }; 901 int ret; 902 struct mlx5_aso_mtr *aso_mtr = NULL; 903 uint32_t cbs_cir, ebs_eir, val; 904 905 if (priv->sh->meter_aso_en) { 906 fm->is_enable = !!is_enable; 907 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm); 908 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr); 909 if (ret) 910 return ret; 911 ret = mlx5_aso_mtr_wait(priv->sh, aso_mtr); 912 if (ret) 913 return ret; 914 } else { 915 /* Fill command parameters. */ 916 mod_attr.reg_c_index = priv->mtr_color_reg - REG_C_0; 917 mod_attr.flow_meter_parameter = in; 918 mod_attr.flow_meter_parameter_sz = 919 MLX5_ST_SZ_BYTES(flow_meter_parameters); 920 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE) 921 mod_attr.active = !!active_state; 922 else 923 mod_attr.active = 0; 924 attr = in; 925 cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir); 926 ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir); 927 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS) { 928 val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & 929 ASO_DSEG_EXP_MASK; 930 MLX5_SET(flow_meter_parameters, attr, 931 cbs_exponent, val); 932 val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & 933 ASO_DSEG_MAN_MASK; 934 MLX5_SET(flow_meter_parameters, attr, 935 cbs_mantissa, val); 936 } 937 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR) { 938 val = (cbs_cir >> ASO_DSEG_CIR_EXP_OFFSET) & 939 ASO_DSEG_EXP_MASK; 940 MLX5_SET(flow_meter_parameters, attr, 941 cir_exponent, val); 942 val = cbs_cir & ASO_DSEG_MAN_MASK; 943 MLX5_SET(flow_meter_parameters, attr, 944 cir_mantissa, val); 945 } 946 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_EBS) { 947 val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & 948 ASO_DSEG_EXP_MASK; 949 MLX5_SET(flow_meter_parameters, attr, 950 ebs_exponent, val); 951 val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & 952 ASO_DSEG_MAN_MASK; 953 MLX5_SET(flow_meter_parameters, attr, 954 ebs_mantissa, val); 955 } 956 /* Apply modifications to meter only if it was created. */ 957 if (fm->meter_action) { 958 ret = mlx5_glue->dv_modify_flow_action_meter 959 (fm->meter_action, &mod_attr, 960 rte_cpu_to_be_64(modify_bits)); 961 if (ret) 962 return ret; 963 } 964 /* Update succeedded modify meter parameters. */ 965 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE) 966 fm->active_state = !!active_state; 967 } 968 return 0; 969 #else 970 (void)priv; 971 (void)fm; 972 (void)srtcm; 973 (void)modify_bits; 974 (void)active_state; 975 (void)is_enable; 976 return -ENOTSUP; 977 #endif 978 } 979 980 static int 981 mlx5_flow_meter_stats_enable_update(struct rte_eth_dev *dev, 982 struct mlx5_flow_meter_info *fm, 983 uint64_t stats_mask) 984 { 985 fm->bytes_dropped = 986 (stats_mask & RTE_MTR_STATS_N_BYTES_DROPPED) ? 1 : 0; 987 fm->pkts_dropped = (stats_mask & RTE_MTR_STATS_N_PKTS_DROPPED) ? 1 : 0; 988 if (fm->bytes_dropped || fm->pkts_dropped) { 989 if (!fm->drop_cnt) { 990 /* Alloc policer counters. */ 991 fm->drop_cnt = mlx5_counter_alloc(dev); 992 if (!fm->drop_cnt) 993 return -1; 994 } 995 } else { 996 if (fm->drop_cnt) { 997 mlx5_counter_free(dev, fm->drop_cnt); 998 fm->drop_cnt = 0; 999 } 1000 } 1001 return 0; 1002 } 1003 1004 /** 1005 * Create meter rules. 1006 * 1007 * @param[in] dev 1008 * Pointer to Ethernet device. 1009 * @param[in] meter_id 1010 * Meter id. 1011 * @param[in] params 1012 * Pointer to rte meter parameters. 1013 * @param[in] shared 1014 * Meter shared with other flow or not. 1015 * @param[out] error 1016 * Pointer to rte meter error structure. 1017 * 1018 * @return 1019 * 0 on success, a negative errno value otherwise and rte_errno is set. 1020 */ 1021 static int 1022 mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, 1023 struct rte_mtr_params *params, int shared, 1024 struct rte_mtr_error *error) 1025 { 1026 struct mlx5_priv *priv = dev->data->dev_private; 1027 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; 1028 struct mlx5_flow_meter_profile *fmp; 1029 struct mlx5_flow_meter_info *fm; 1030 struct mlx5_legacy_flow_meter *legacy_fm; 1031 struct mlx5_flow_meter_policy *mtr_policy = NULL; 1032 struct mlx5_indexed_pool_config flow_ipool_cfg = { 1033 .size = 0, 1034 .trunk_size = 64, 1035 .need_lock = 1, 1036 .type = "mlx5_flow_mtr_flow_id_pool", 1037 }; 1038 struct mlx5_aso_mtr *aso_mtr; 1039 uint32_t mtr_idx, policy_idx; 1040 union mlx5_l3t_data data; 1041 int ret; 1042 uint8_t domain_bitmap; 1043 uint8_t mtr_id_bits; 1044 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 1045 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 1046 1047 if (!priv->mtr_en) 1048 return -rte_mtr_error_set(error, ENOTSUP, 1049 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1050 "Meter is not supported"); 1051 /* Validate the parameters. */ 1052 ret = mlx5_flow_meter_validate(priv, meter_id, params, error); 1053 if (ret) 1054 return ret; 1055 /* Meter profile must exist. */ 1056 fmp = mlx5_flow_meter_profile_find(priv, params->meter_profile_id); 1057 if (fmp == NULL) 1058 return -rte_mtr_error_set(error, ENOENT, 1059 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 1060 NULL, "Meter profile id not valid."); 1061 /* Meter policy must exist. */ 1062 if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) { 1063 __atomic_add_fetch 1064 (&priv->sh->mtrmng->def_policy_ref_cnt, 1065 1, __ATOMIC_RELAXED); 1066 domain_bitmap = MLX5_MTR_ALL_DOMAIN_BIT; 1067 if (!priv->config.dv_esw_en) 1068 domain_bitmap &= ~MLX5_MTR_DOMAIN_TRANSFER_BIT; 1069 } else { 1070 mtr_policy = mlx5_flow_meter_policy_find(dev, 1071 params->meter_policy_id, &policy_idx); 1072 if (!priv->sh->meter_aso_en) 1073 return -rte_mtr_error_set(error, ENOTSUP, 1074 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1075 "Part of the policies cannot be " 1076 "supported without ASO "); 1077 if (!mtr_policy) 1078 return -rte_mtr_error_set(error, ENOENT, 1079 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1080 NULL, "Meter policy id not valid."); 1081 domain_bitmap = (mtr_policy->ingress ? 1082 MLX5_MTR_DOMAIN_INGRESS_BIT : 0) | 1083 (mtr_policy->egress ? 1084 MLX5_MTR_DOMAIN_EGRESS_BIT : 0) | 1085 (mtr_policy->transfer ? 1086 MLX5_MTR_DOMAIN_TRANSFER_BIT : 0); 1087 } 1088 /* Allocate the flow meter memory. */ 1089 if (priv->sh->meter_aso_en) { 1090 mtr_idx = mlx5_flow_mtr_alloc(dev); 1091 if (!mtr_idx) 1092 return -rte_mtr_error_set(error, ENOMEM, 1093 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1094 "Memory alloc failed for meter."); 1095 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx); 1096 fm = &aso_mtr->fm; 1097 } else { 1098 legacy_fm = mlx5_ipool_zmalloc 1099 (priv->sh->ipool[MLX5_IPOOL_MTR], &mtr_idx); 1100 if (legacy_fm == NULL) 1101 return -rte_mtr_error_set(error, ENOMEM, 1102 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1103 "Memory alloc failed for meter."); 1104 legacy_fm->idx = mtr_idx; 1105 fm = &legacy_fm->fm; 1106 } 1107 mtr_id_bits = MLX5_REG_BITS - __builtin_clz(mtr_idx); 1108 if ((mtr_id_bits + priv->sh->mtrmng->max_mtr_flow_bits) > 1109 mtr_reg_bits) { 1110 DRV_LOG(ERR, "Meter number exceeds max limit."); 1111 goto error; 1112 } 1113 if (mtr_id_bits > priv->sh->mtrmng->max_mtr_bits) 1114 priv->sh->mtrmng->max_mtr_bits = mtr_id_bits; 1115 /* Fill the flow meter parameters. */ 1116 fm->meter_id = meter_id; 1117 fm->policy_id = params->meter_policy_id; 1118 fm->profile = fmp; 1119 if (mlx5_flow_meter_stats_enable_update(dev, fm, params->stats_mask)) 1120 goto error; 1121 if (mlx5_flow_create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap)) 1122 goto error; 1123 /* Add to the flow meter list. */ 1124 if (!priv->sh->meter_aso_en) 1125 TAILQ_INSERT_TAIL(fms, legacy_fm, next); 1126 /* Add to the flow meter list. */ 1127 fm->active_state = 1; /* Config meter starts as active. */ 1128 fm->is_enable = 1; 1129 fm->shared = !!shared; 1130 __atomic_add_fetch(&fm->profile->ref_cnt, 1, __ATOMIC_RELAXED); 1131 if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) { 1132 fm->def_policy = 1; 1133 fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg); 1134 if (!fm->flow_ipool) 1135 goto error; 1136 } 1137 rte_spinlock_init(&fm->sl); 1138 /* If ASO meter supported, update ASO flow meter by wqe. */ 1139 if (priv->sh->meter_aso_en) { 1140 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm); 1141 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr); 1142 if (ret) 1143 goto error; 1144 if (!priv->mtr_idx_tbl) { 1145 priv->mtr_idx_tbl = 1146 mlx5_l3t_create(MLX5_L3T_TYPE_DWORD); 1147 if (!priv->mtr_idx_tbl) 1148 goto error; 1149 } 1150 data.dword = mtr_idx; 1151 if (mlx5_l3t_set_entry(priv->mtr_idx_tbl, meter_id, &data)) 1152 goto error; 1153 } 1154 if (mtr_policy) 1155 __atomic_add_fetch(&mtr_policy->ref_cnt, 1, __ATOMIC_RELAXED); 1156 return 0; 1157 error: 1158 mlx5_flow_destroy_mtr_tbls(dev, fm); 1159 /* Free policer counters. */ 1160 if (fm->drop_cnt) 1161 mlx5_counter_free(dev, fm->drop_cnt); 1162 if (priv->sh->meter_aso_en) 1163 mlx5_flow_mtr_free(dev, mtr_idx); 1164 else 1165 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], mtr_idx); 1166 return -rte_mtr_error_set(error, ENOTSUP, 1167 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 1168 NULL, "Failed to create devx meter."); 1169 } 1170 1171 static int 1172 mlx5_flow_meter_params_flush(struct rte_eth_dev *dev, 1173 struct mlx5_flow_meter_info *fm, 1174 uint32_t mtr_idx) 1175 { 1176 struct mlx5_priv *priv = dev->data->dev_private; 1177 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; 1178 struct mlx5_flow_meter_profile *fmp; 1179 struct mlx5_legacy_flow_meter *legacy_fm = NULL; 1180 struct mlx5_flow_meter_policy *mtr_policy; 1181 1182 /* Meter object must not have any owner. */ 1183 MLX5_ASSERT(!fm->ref_cnt); 1184 /* Get meter profile. */ 1185 fmp = fm->profile; 1186 if (fmp == NULL) 1187 return -1; 1188 /* Update dependencies. */ 1189 __atomic_sub_fetch(&fmp->ref_cnt, 1, __ATOMIC_RELAXED); 1190 fm->profile = NULL; 1191 /* Remove from list. */ 1192 if (!priv->sh->meter_aso_en) { 1193 legacy_fm = container_of(fm, 1194 struct mlx5_legacy_flow_meter, fm); 1195 TAILQ_REMOVE(fms, legacy_fm, next); 1196 } 1197 /* Free drop counters. */ 1198 if (fm->drop_cnt) 1199 mlx5_counter_free(dev, fm->drop_cnt); 1200 /* Free meter flow table. */ 1201 if (fm->flow_ipool) { 1202 mlx5_ipool_destroy(fm->flow_ipool); 1203 fm->flow_ipool = 0; 1204 } 1205 mlx5_flow_destroy_mtr_tbls(dev, fm); 1206 if (fm->def_policy) 1207 __atomic_sub_fetch(&priv->sh->mtrmng->def_policy_ref_cnt, 1208 1, __ATOMIC_RELAXED); 1209 if (priv->sh->meter_aso_en) { 1210 if (!fm->def_policy) { 1211 mtr_policy = mlx5_flow_meter_policy_find(dev, 1212 fm->policy_id, NULL); 1213 if (mtr_policy) 1214 __atomic_sub_fetch(&mtr_policy->ref_cnt, 1215 1, __ATOMIC_RELAXED); 1216 fm->policy_id = 0; 1217 } 1218 fm->def_policy = 0; 1219 if (mlx5_l3t_clear_entry(priv->mtr_idx_tbl, fm->meter_id)) 1220 return -1; 1221 mlx5_flow_mtr_free(dev, mtr_idx); 1222 } else { 1223 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], 1224 legacy_fm->idx); 1225 } 1226 return 0; 1227 } 1228 1229 /** 1230 * Destroy meter rules. 1231 * 1232 * @param[in] dev 1233 * Pointer to Ethernet device. 1234 * @param[in] meter_id 1235 * Meter id. 1236 * @param[out] error 1237 * Pointer to rte meter error structure. 1238 * 1239 * @return 1240 * 0 on success, a negative errno value otherwise and rte_errno is set. 1241 */ 1242 static int 1243 mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id, 1244 struct rte_mtr_error *error) 1245 { 1246 struct mlx5_priv *priv = dev->data->dev_private; 1247 struct mlx5_flow_meter_info *fm; 1248 uint32_t mtr_idx = 0; 1249 1250 if (!priv->mtr_en) 1251 return -rte_mtr_error_set(error, ENOTSUP, 1252 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 1253 NULL, 1254 "Meter is not supported"); 1255 /* Meter object must exist. */ 1256 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 1257 if (fm == NULL) 1258 return -rte_mtr_error_set(error, ENOENT, 1259 RTE_MTR_ERROR_TYPE_MTR_ID, 1260 NULL, 1261 "Meter object id not valid."); 1262 /* Meter object must not have any owner. */ 1263 if (fm->ref_cnt > 0) 1264 return -rte_mtr_error_set(error, EBUSY, 1265 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 1266 NULL, 1267 "Meter object is being used."); 1268 /* Destroy the meter profile. */ 1269 if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx)) 1270 return -rte_mtr_error_set(error, EINVAL, 1271 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 1272 NULL, 1273 "MTR object meter profile invalid."); 1274 return 0; 1275 } 1276 1277 /** 1278 * Modify meter state. 1279 * 1280 * @param[in] priv 1281 * Pointer to mlx5 private data structure. 1282 * @param[in] fm 1283 * Pointer to flow meter. 1284 * @param[in] new_state 1285 * New state to update. 1286 * @param[out] error 1287 * Pointer to rte meter error structure. 1288 * 1289 * @return 1290 * 0 on success, a negative errno value otherwise and rte_errno is set. 1291 */ 1292 static int 1293 mlx5_flow_meter_modify_state(struct mlx5_priv *priv, 1294 struct mlx5_flow_meter_info *fm, 1295 uint32_t new_state, 1296 struct rte_mtr_error *error) 1297 { 1298 static const struct mlx5_flow_meter_srtcm_rfc2697_prm srtcm = { 1299 .cbs_cir = RTE_BE32(MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL), 1300 .ebs_eir = 0, 1301 }; 1302 uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS | 1303 MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR; 1304 int ret; 1305 1306 if (new_state == MLX5_FLOW_METER_DISABLE) 1307 ret = mlx5_flow_meter_action_modify(priv, fm, 1308 &srtcm, modify_bits, 0, 0); 1309 else 1310 ret = mlx5_flow_meter_action_modify(priv, fm, 1311 &fm->profile->srtcm_prm, 1312 modify_bits, 0, 1); 1313 if (ret) 1314 return -rte_mtr_error_set(error, -ret, 1315 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 1316 NULL, 1317 new_state ? 1318 "Failed to enable meter." : 1319 "Failed to disable meter."); 1320 return 0; 1321 } 1322 1323 /** 1324 * Callback to enable flow meter. 1325 * 1326 * @param[in] dev 1327 * Pointer to Ethernet device. 1328 * @param[in] meter_id 1329 * Meter id. 1330 * @param[out] error 1331 * Pointer to rte meter error structure. 1332 * 1333 * @return 1334 * 0 on success, a negative errno value otherwise and rte_errno is set. 1335 */ 1336 static int 1337 mlx5_flow_meter_enable(struct rte_eth_dev *dev, 1338 uint32_t meter_id, 1339 struct rte_mtr_error *error) 1340 { 1341 struct mlx5_priv *priv = dev->data->dev_private; 1342 struct mlx5_flow_meter_info *fm; 1343 int ret; 1344 1345 if (!priv->mtr_en) 1346 return -rte_mtr_error_set(error, ENOTSUP, 1347 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1348 "Meter is not supported"); 1349 /* Meter object must exist. */ 1350 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 1351 if (fm == NULL) 1352 return -rte_mtr_error_set(error, ENOENT, 1353 RTE_MTR_ERROR_TYPE_MTR_ID, 1354 NULL, "Meter not found."); 1355 if (fm->active_state == MLX5_FLOW_METER_ENABLE) 1356 return 0; 1357 ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_ENABLE, 1358 error); 1359 if (!ret) 1360 fm->active_state = MLX5_FLOW_METER_ENABLE; 1361 return ret; 1362 } 1363 1364 /** 1365 * Callback to disable flow meter. 1366 * 1367 * @param[in] dev 1368 * Pointer to Ethernet device. 1369 * @param[in] meter_id 1370 * Meter id. 1371 * @param[out] error 1372 * Pointer to rte meter error structure. 1373 * 1374 * @return 1375 * 0 on success, a negative errno value otherwise and rte_errno is set. 1376 */ 1377 static int 1378 mlx5_flow_meter_disable(struct rte_eth_dev *dev, 1379 uint32_t meter_id, 1380 struct rte_mtr_error *error) 1381 { 1382 struct mlx5_priv *priv = dev->data->dev_private; 1383 struct mlx5_flow_meter_info *fm; 1384 int ret; 1385 1386 if (!priv->mtr_en) 1387 return -rte_mtr_error_set(error, ENOTSUP, 1388 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1389 "Meter is not supported"); 1390 /* Meter object must exist. */ 1391 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 1392 if (fm == NULL) 1393 return -rte_mtr_error_set(error, ENOENT, 1394 RTE_MTR_ERROR_TYPE_MTR_ID, 1395 NULL, "Meter not found."); 1396 if (fm->active_state == MLX5_FLOW_METER_DISABLE) 1397 return 0; 1398 ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_DISABLE, 1399 error); 1400 if (!ret) 1401 fm->active_state = MLX5_FLOW_METER_DISABLE; 1402 return ret; 1403 } 1404 1405 /** 1406 * Callback to update meter profile. 1407 * 1408 * @param[in] dev 1409 * Pointer to Ethernet device. 1410 * @param[in] meter_id 1411 * Meter id. 1412 * @param[in] meter_profile_id 1413 * To be updated meter profile id. 1414 * @param[out] error 1415 * Pointer to rte meter error structure. 1416 * 1417 * @return 1418 * 0 on success, a negative errno value otherwise and rte_errno is set. 1419 */ 1420 static int 1421 mlx5_flow_meter_profile_update(struct rte_eth_dev *dev, 1422 uint32_t meter_id, 1423 uint32_t meter_profile_id, 1424 struct rte_mtr_error *error) 1425 { 1426 struct mlx5_priv *priv = dev->data->dev_private; 1427 struct mlx5_flow_meter_profile *fmp; 1428 struct mlx5_flow_meter_profile *old_fmp; 1429 struct mlx5_flow_meter_info *fm; 1430 uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS | 1431 MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR; 1432 int ret; 1433 1434 if (!priv->mtr_en) 1435 return -rte_mtr_error_set(error, ENOTSUP, 1436 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1437 "Meter is not supported"); 1438 /* Meter profile must exist. */ 1439 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 1440 if (fmp == NULL) 1441 return -rte_mtr_error_set(error, ENOENT, 1442 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 1443 NULL, "Meter profile not found."); 1444 /* Meter object must exist. */ 1445 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 1446 if (fm == NULL) 1447 return -rte_mtr_error_set(error, ENOENT, 1448 RTE_MTR_ERROR_TYPE_MTR_ID, 1449 NULL, "Meter not found."); 1450 /* MTR object already set to meter profile id. */ 1451 old_fmp = fm->profile; 1452 if (fmp == old_fmp) 1453 return 0; 1454 /* Update the profile. */ 1455 fm->profile = fmp; 1456 /* Update meter params in HW (if not disabled). */ 1457 if (fm->active_state == MLX5_FLOW_METER_DISABLE) 1458 return 0; 1459 ret = mlx5_flow_meter_action_modify(priv, fm, &fm->profile->srtcm_prm, 1460 modify_bits, fm->active_state, 1); 1461 if (ret) { 1462 fm->profile = old_fmp; 1463 return -rte_mtr_error_set(error, -ret, 1464 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 1465 NULL, "Failed to update meter" 1466 " parmeters in hardware."); 1467 } 1468 old_fmp->ref_cnt--; 1469 fmp->ref_cnt++; 1470 return 0; 1471 } 1472 1473 /** 1474 * Callback to update meter stats mask. 1475 * 1476 * @param[in] dev 1477 * Pointer to Ethernet device. 1478 * @param[in] meter_id 1479 * Meter id. 1480 * @param[in] stats_mask 1481 * To be updated stats_mask. 1482 * @param[out] error 1483 * Pointer to rte meter error structure. 1484 * 1485 * @return 1486 * 0 on success, a negative errno value otherwise and rte_errno is set. 1487 */ 1488 static int 1489 mlx5_flow_meter_stats_update(struct rte_eth_dev *dev, 1490 uint32_t meter_id, 1491 uint64_t stats_mask, 1492 struct rte_mtr_error *error) 1493 { 1494 struct mlx5_priv *priv = dev->data->dev_private; 1495 struct mlx5_flow_meter_info *fm; 1496 1497 if (!priv->mtr_en) 1498 return -rte_mtr_error_set(error, ENOTSUP, 1499 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1500 "Meter is not supported"); 1501 /* Meter object must exist. */ 1502 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 1503 if (fm == NULL) 1504 return -rte_mtr_error_set(error, ENOENT, 1505 RTE_MTR_ERROR_TYPE_MTR_ID, 1506 NULL, "Meter object id not valid."); 1507 if (mlx5_flow_meter_stats_enable_update(dev, fm, stats_mask)) 1508 return -rte_mtr_error_set(error, ENOENT, 1509 RTE_MTR_ERROR_TYPE_MTR_ID, 1510 NULL, "Fail to allocate " 1511 "counter for meter."); 1512 return 0; 1513 } 1514 1515 /** 1516 * Callback to read meter statistics. 1517 * 1518 * @param[in] dev 1519 * Pointer to Ethernet device. 1520 * @param[in] meter_id 1521 * Meter id. 1522 * @param[out] stats 1523 * Pointer to store the statistics. 1524 * @param[out] stats_mask 1525 * Pointer to store the stats_mask. 1526 * @param[in] clear 1527 * Statistic to be cleared after read or not. 1528 * @param[out] error 1529 * Pointer to rte meter error structure. 1530 * 1531 * @return 1532 * 0 on success, a negative errno value otherwise and rte_errno is set. 1533 */ 1534 static int 1535 mlx5_flow_meter_stats_read(struct rte_eth_dev *dev, 1536 uint32_t meter_id, 1537 struct rte_mtr_stats *stats, 1538 uint64_t *stats_mask, 1539 int clear, 1540 struct rte_mtr_error *error) 1541 { 1542 struct mlx5_priv *priv = dev->data->dev_private; 1543 struct mlx5_flow_meter_info *fm; 1544 uint64_t pkts; 1545 uint64_t bytes; 1546 int ret = 0; 1547 1548 if (!priv->mtr_en) 1549 return -rte_mtr_error_set(error, ENOTSUP, 1550 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1551 "Meter is not supported"); 1552 /* Meter object must exist. */ 1553 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 1554 if (fm == NULL) 1555 return -rte_mtr_error_set(error, ENOENT, 1556 RTE_MTR_ERROR_TYPE_MTR_ID, 1557 NULL, "Meter object id not valid."); 1558 *stats_mask = 0; 1559 if (fm->bytes_dropped) 1560 *stats_mask |= RTE_MTR_STATS_N_BYTES_DROPPED; 1561 if (fm->pkts_dropped) 1562 *stats_mask |= RTE_MTR_STATS_N_PKTS_DROPPED; 1563 memset(stats, 0, sizeof(*stats)); 1564 if (fm->drop_cnt) { 1565 ret = mlx5_counter_query(dev, fm->drop_cnt, clear, &pkts, 1566 &bytes); 1567 if (ret) 1568 goto error; 1569 /* If need to read the packets, set it. */ 1570 if (fm->pkts_dropped) 1571 stats->n_pkts_dropped = pkts; 1572 /* If need to read the bytes, set it. */ 1573 if (fm->bytes_dropped) 1574 stats->n_bytes_dropped = bytes; 1575 } 1576 return 0; 1577 error: 1578 return -rte_mtr_error_set(error, ret, RTE_MTR_ERROR_TYPE_STATS, NULL, 1579 "Failed to read meter drop counters."); 1580 } 1581 1582 static const struct rte_mtr_ops mlx5_flow_mtr_ops = { 1583 .capabilities_get = mlx5_flow_mtr_cap_get, 1584 .meter_profile_add = mlx5_flow_meter_profile_add, 1585 .meter_profile_delete = mlx5_flow_meter_profile_delete, 1586 .meter_policy_validate = mlx5_flow_meter_policy_validate, 1587 .meter_policy_add = mlx5_flow_meter_policy_add, 1588 .meter_policy_delete = mlx5_flow_meter_policy_delete, 1589 .create = mlx5_flow_meter_create, 1590 .destroy = mlx5_flow_meter_destroy, 1591 .meter_enable = mlx5_flow_meter_enable, 1592 .meter_disable = mlx5_flow_meter_disable, 1593 .meter_profile_update = mlx5_flow_meter_profile_update, 1594 .meter_dscp_table_update = NULL, 1595 .stats_update = mlx5_flow_meter_stats_update, 1596 .stats_read = mlx5_flow_meter_stats_read, 1597 }; 1598 1599 /** 1600 * Get meter operations. 1601 * 1602 * @param dev 1603 * Pointer to Ethernet device structure. 1604 * @param arg 1605 * Pointer to set the mtr operations. 1606 * 1607 * @return 1608 * Always 0. 1609 */ 1610 int 1611 mlx5_flow_meter_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg) 1612 { 1613 *(const struct rte_mtr_ops **)arg = &mlx5_flow_mtr_ops; 1614 return 0; 1615 } 1616 1617 /** 1618 * Find meter by id. 1619 * 1620 * @param priv 1621 * Pointer to mlx5_priv. 1622 * @param meter_id 1623 * Meter id. 1624 * @param mtr_idx 1625 * Pointer to Meter index. 1626 * 1627 * @return 1628 * Pointer to the meter info found on success, NULL otherwise. 1629 */ 1630 struct mlx5_flow_meter_info * 1631 mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id, 1632 uint32_t *mtr_idx) 1633 { 1634 struct mlx5_legacy_flow_meter *legacy_fm; 1635 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; 1636 struct mlx5_aso_mtr *aso_mtr; 1637 struct mlx5_aso_mtr_pools_mng *pools_mng = 1638 &priv->sh->mtrmng->pools_mng; 1639 union mlx5_l3t_data data; 1640 1641 if (priv->sh->meter_aso_en) { 1642 rte_spinlock_lock(&pools_mng->mtrsl); 1643 if (!pools_mng->n_valid || !priv->mtr_idx_tbl) { 1644 rte_spinlock_unlock(&pools_mng->mtrsl); 1645 return NULL; 1646 } 1647 if (mlx5_l3t_get_entry(priv->mtr_idx_tbl, meter_id, &data) || 1648 !data.dword) { 1649 rte_spinlock_unlock(&pools_mng->mtrsl); 1650 return NULL; 1651 } 1652 if (mtr_idx) 1653 *mtr_idx = data.dword; 1654 aso_mtr = mlx5_aso_meter_by_idx(priv, data.dword); 1655 /* Remove reference taken by the mlx5_l3t_get_entry. */ 1656 mlx5_l3t_clear_entry(priv->mtr_idx_tbl, meter_id); 1657 rte_spinlock_unlock(&pools_mng->mtrsl); 1658 if (!aso_mtr || aso_mtr->state == ASO_METER_FREE) 1659 return NULL; 1660 return &aso_mtr->fm; 1661 } 1662 TAILQ_FOREACH(legacy_fm, fms, next) 1663 if (meter_id == legacy_fm->fm.meter_id) { 1664 if (mtr_idx) 1665 *mtr_idx = legacy_fm->idx; 1666 return &legacy_fm->fm; 1667 } 1668 return NULL; 1669 } 1670 1671 /** 1672 * Find meter by index. 1673 * 1674 * @param priv 1675 * Pointer to mlx5_priv. 1676 * @param idx 1677 * Meter index. 1678 * 1679 * @return 1680 * Pointer to the meter info found on success, NULL otherwise. 1681 */ 1682 struct mlx5_flow_meter_info * 1683 flow_dv_meter_find_by_idx(struct mlx5_priv *priv, uint32_t idx) 1684 { 1685 struct mlx5_aso_mtr *aso_mtr; 1686 1687 if (priv->sh->meter_aso_en) { 1688 aso_mtr = mlx5_aso_meter_by_idx(priv, idx); 1689 if (!aso_mtr) 1690 return NULL; 1691 return &aso_mtr->fm; 1692 } else { 1693 return mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], idx); 1694 } 1695 } 1696 1697 /** 1698 * Attach meter to flow. 1699 * Unidirectional Meter creation can only be done 1700 * when flow direction is known, i.e. when calling meter_attach. 1701 * 1702 * @param [in] priv 1703 * Pointer to mlx5 private data. 1704 * @param[in] fm 1705 * Pointer to flow meter. 1706 * @param [in] attr 1707 * Pointer to flow attributes. 1708 * @param [out] error 1709 * Pointer to error structure. 1710 * 1711 * @return 1712 * 0 on success, a negative errno value otherwise and rte_errno is set. 1713 */ 1714 int 1715 mlx5_flow_meter_attach(struct mlx5_priv *priv, 1716 struct mlx5_flow_meter_info *fm, 1717 const struct rte_flow_attr *attr, 1718 struct rte_flow_error *error) 1719 { 1720 int ret = 0; 1721 1722 if (priv->sh->meter_aso_en) { 1723 struct mlx5_aso_mtr *aso_mtr; 1724 1725 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm); 1726 if (mlx5_aso_mtr_wait(priv->sh, aso_mtr)) { 1727 return rte_flow_error_set(error, ENOENT, 1728 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1729 NULL, 1730 "Timeout in meter configuration"); 1731 } 1732 rte_spinlock_lock(&fm->sl); 1733 if (fm->shared || !fm->ref_cnt) { 1734 fm->ref_cnt++; 1735 } else { 1736 rte_flow_error_set(error, EINVAL, 1737 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 1738 "Meter cannot be shared"); 1739 ret = -1; 1740 } 1741 rte_spinlock_unlock(&fm->sl); 1742 } else { 1743 rte_spinlock_lock(&fm->sl); 1744 if (fm->meter_action) { 1745 if (fm->shared && 1746 attr->transfer == fm->transfer && 1747 attr->ingress == fm->ingress && 1748 attr->egress == fm->egress) { 1749 fm->ref_cnt++; 1750 } else { 1751 rte_flow_error_set(error, EINVAL, 1752 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 1753 fm->shared ? 1754 "Meter attr not match." : 1755 "Meter cannot be shared."); 1756 ret = -1; 1757 } 1758 } else { 1759 fm->ingress = attr->ingress; 1760 fm->egress = attr->egress; 1761 fm->transfer = attr->transfer; 1762 fm->ref_cnt = 1; 1763 /* This also creates the meter object. */ 1764 fm->meter_action = mlx5_flow_meter_action_create(priv, 1765 fm); 1766 if (!fm->meter_action) { 1767 fm->ref_cnt = 0; 1768 fm->ingress = 0; 1769 fm->egress = 0; 1770 fm->transfer = 0; 1771 rte_flow_error_set(error, EINVAL, 1772 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 1773 "Meter action create failed."); 1774 ret = -1; 1775 } 1776 } 1777 rte_spinlock_unlock(&fm->sl); 1778 } 1779 return ret ? -rte_errno : 0; 1780 } 1781 1782 /** 1783 * Detach meter from flow. 1784 * 1785 * @param [in] priv 1786 * Pointer to mlx5 private data. 1787 * @param [in] fm 1788 * Pointer to flow meter. 1789 */ 1790 void 1791 mlx5_flow_meter_detach(struct mlx5_priv *priv, 1792 struct mlx5_flow_meter_info *fm) 1793 { 1794 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER 1795 rte_spinlock_lock(&fm->sl); 1796 MLX5_ASSERT(fm->ref_cnt); 1797 if (--fm->ref_cnt == 0 && !priv->sh->meter_aso_en) { 1798 mlx5_glue->destroy_flow_action(fm->meter_action); 1799 fm->meter_action = NULL; 1800 fm->ingress = 0; 1801 fm->egress = 0; 1802 fm->transfer = 0; 1803 } 1804 rte_spinlock_unlock(&fm->sl); 1805 #else 1806 (void)priv; 1807 (void)fm; 1808 #endif 1809 } 1810 1811 /** 1812 * Flush meter configuration. 1813 * 1814 * @param[in] dev 1815 * Pointer to Ethernet device. 1816 * @param[out] error 1817 * Pointer to rte meter error structure. 1818 * 1819 * @return 1820 * 0 on success, a negative errno value otherwise and rte_errno is set. 1821 */ 1822 int 1823 mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error) 1824 { 1825 struct mlx5_priv *priv = dev->data->dev_private; 1826 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; 1827 struct mlx5_mtr_profiles *fmps = &priv->flow_meter_profiles; 1828 struct mlx5_flow_meter_profile *fmp; 1829 struct mlx5_legacy_flow_meter *legacy_fm; 1830 struct mlx5_flow_meter_info *fm; 1831 struct mlx5_flow_meter_sub_policy *sub_policy; 1832 void *tmp; 1833 uint32_t i, mtr_idx, policy_idx; 1834 void *entry; 1835 struct mlx5_aso_mtr *aso_mtr; 1836 1837 if (!priv->mtr_en) 1838 return 0; 1839 if (priv->sh->meter_aso_en) { 1840 if (priv->mtr_idx_tbl) { 1841 MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) { 1842 mtr_idx = *(uint32_t *)entry; 1843 if (mtr_idx) { 1844 aso_mtr = 1845 mlx5_aso_meter_by_idx(priv, mtr_idx); 1846 fm = &aso_mtr->fm; 1847 (void)mlx5_flow_meter_params_flush(dev, 1848 fm, mtr_idx); 1849 } 1850 } 1851 mlx5_l3t_destroy(priv->mtr_idx_tbl); 1852 priv->mtr_idx_tbl = NULL; 1853 } 1854 } else { 1855 TAILQ_FOREACH_SAFE(legacy_fm, fms, next, tmp) { 1856 fm = &legacy_fm->fm; 1857 if (mlx5_flow_meter_params_flush(dev, fm, 0)) 1858 return -rte_mtr_error_set(error, EINVAL, 1859 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 1860 NULL, "MTR object meter profile invalid."); 1861 } 1862 } 1863 if (priv->sh->mtrmng->policy_idx_tbl && priv->sh->refcnt == 1) { 1864 MLX5_L3T_FOREACH(priv->sh->mtrmng->policy_idx_tbl, 1865 i, entry) { 1866 policy_idx = *(uint32_t *)entry; 1867 sub_policy = mlx5_ipool_get 1868 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 1869 policy_idx); 1870 if (!sub_policy) 1871 return -rte_mtr_error_set(error, 1872 EINVAL, 1873 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1874 NULL, "MTR object " 1875 "meter policy invalid."); 1876 if (__mlx5_flow_meter_policy_delete(dev, i, 1877 sub_policy->main_policy, 1878 error)) 1879 return -rte_mtr_error_set(error, 1880 EINVAL, 1881 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1882 NULL, "MTR object " 1883 "meter policy invalid."); 1884 mlx5_free(sub_policy->main_policy); 1885 } 1886 mlx5_l3t_destroy(priv->sh->mtrmng->policy_idx_tbl); 1887 priv->sh->mtrmng->policy_idx_tbl = NULL; 1888 } 1889 TAILQ_FOREACH_SAFE(fmp, fmps, next, tmp) { 1890 /* Check unused. */ 1891 MLX5_ASSERT(!fmp->ref_cnt); 1892 /* Remove from list. */ 1893 TAILQ_REMOVE(&priv->flow_meter_profiles, fmp, next); 1894 mlx5_free(fmp); 1895 } 1896 /* Delete default policy table. */ 1897 mlx5_flow_destroy_def_policy(dev); 1898 if (priv->sh->refcnt == 1) 1899 mlx5_flow_destroy_mtr_drop_tbls(dev); 1900 return 0; 1901 } 1902