1 // SPDX-License-Identifier: BSD-3-Clause 2 /* 3 * Copyright 2018 Mellanox Technologies, Ltd 4 */ 5 #include <math.h> 6 7 #include <rte_tailq.h> 8 #include <rte_malloc.h> 9 #include <rte_mtr.h> 10 #include <rte_mtr_driver.h> 11 12 #include <mlx5_devx_cmds.h> 13 #include <mlx5_malloc.h> 14 15 #include "mlx5.h" 16 #include "mlx5_flow.h" 17 18 /** 19 * Create the meter action. 20 * 21 * @param priv 22 * Pointer to mlx5_priv. 23 * @param[in] fm 24 * Pointer to flow meter to be converted. 25 * 26 * @return 27 * Pointer to the meter action on success, NULL otherwise. 28 */ 29 static void * 30 mlx5_flow_meter_action_create(struct mlx5_priv *priv, 31 struct mlx5_flow_meter_info *fm) 32 { 33 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER 34 struct mlx5dv_dr_flow_meter_attr mtr_init; 35 uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)]; 36 struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = 37 &fm->profile->srtcm_prm; 38 uint32_t cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir); 39 uint32_t ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir); 40 uint32_t val; 41 enum mlx5_meter_domain domain = 42 fm->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 43 fm->egress ? MLX5_MTR_DOMAIN_EGRESS : 44 MLX5_MTR_DOMAIN_INGRESS; 45 struct mlx5_flow_meter_def_policy *def_policy = 46 priv->sh->mtrmng->def_policy[domain]; 47 48 memset(fmp, 0, MLX5_ST_SZ_BYTES(flow_meter_parameters)); 49 MLX5_SET(flow_meter_parameters, fmp, valid, 1); 50 MLX5_SET(flow_meter_parameters, fmp, bucket_overflow, 1); 51 MLX5_SET(flow_meter_parameters, fmp, 52 start_color, MLX5_FLOW_COLOR_GREEN); 53 MLX5_SET(flow_meter_parameters, fmp, both_buckets_on_green, 0); 54 val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK; 55 MLX5_SET(flow_meter_parameters, fmp, cbs_exponent, val); 56 val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK; 57 MLX5_SET(flow_meter_parameters, fmp, cbs_mantissa, val); 58 val = (cbs_cir >> ASO_DSEG_CIR_EXP_OFFSET) & ASO_DSEG_EXP_MASK; 59 MLX5_SET(flow_meter_parameters, fmp, cir_exponent, val); 60 val = (cbs_cir & ASO_DSEG_MAN_MASK); 61 MLX5_SET(flow_meter_parameters, fmp, cir_mantissa, val); 62 val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK; 63 MLX5_SET(flow_meter_parameters, fmp, ebs_exponent, val); 64 val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK; 65 MLX5_SET(flow_meter_parameters, fmp, ebs_mantissa, val); 66 mtr_init.next_table = def_policy->sub_policy.tbl_rsc->obj; 67 mtr_init.reg_c_index = priv->mtr_color_reg - REG_C_0; 68 mtr_init.flow_meter_parameter = fmp; 69 mtr_init.flow_meter_parameter_sz = 70 MLX5_ST_SZ_BYTES(flow_meter_parameters); 71 mtr_init.active = fm->active_state; 72 return mlx5_glue->dv_create_flow_action_meter(&mtr_init); 73 #else 74 (void)priv; 75 (void)fm; 76 return NULL; 77 #endif 78 } 79 80 /** 81 * Find meter profile by id. 82 * 83 * @param priv 84 * Pointer to mlx5_priv. 85 * @param meter_profile_id 86 * Meter profile id. 87 * 88 * @return 89 * Pointer to the profile found on success, NULL otherwise. 90 */ 91 static struct mlx5_flow_meter_profile * 92 mlx5_flow_meter_profile_find(struct mlx5_priv *priv, uint32_t meter_profile_id) 93 { 94 struct mlx5_mtr_profiles *fmps = &priv->flow_meter_profiles; 95 struct mlx5_flow_meter_profile *fmp; 96 97 TAILQ_FOREACH(fmp, fmps, next) 98 if (meter_profile_id == fmp->id) 99 return fmp; 100 return NULL; 101 } 102 103 /** 104 * Validate the MTR profile. 105 * 106 * @param[in] dev 107 * Pointer to Ethernet device. 108 * @param[in] meter_profile_id 109 * Meter profile id. 110 * @param[in] profile 111 * Pointer to meter profile detail. 112 * @param[out] error 113 * Pointer to the error structure. 114 * 115 * @return 116 * 0 on success, a negative errno value otherwise and rte_errno is set. 117 */ 118 static int 119 mlx5_flow_meter_profile_validate(struct rte_eth_dev *dev, 120 uint32_t meter_profile_id, 121 struct rte_mtr_meter_profile *profile, 122 struct rte_mtr_error *error) 123 { 124 struct mlx5_priv *priv = dev->data->dev_private; 125 struct mlx5_flow_meter_profile *fmp; 126 127 /* Profile must not be NULL. */ 128 if (profile == NULL) 129 return -rte_mtr_error_set(error, EINVAL, 130 RTE_MTR_ERROR_TYPE_METER_PROFILE, 131 NULL, "Meter profile is null."); 132 /* Meter profile ID must be valid. */ 133 if (meter_profile_id == UINT32_MAX) 134 return -rte_mtr_error_set(error, EINVAL, 135 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 136 NULL, "Meter profile id not valid."); 137 /* Meter profile must not exist. */ 138 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 139 if (fmp) 140 return -rte_mtr_error_set(error, EEXIST, 141 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 142 NULL, 143 "Meter profile already exists."); 144 if (profile->alg == RTE_MTR_SRTCM_RFC2697) { 145 if (priv->config.hca_attr.qos.flow_meter_old) { 146 /* Verify support for flow meter parameters. */ 147 if (priv->sh->meter_aso_en && profile->packet_mode) { 148 if (profile->srtcm_rfc2697.cir > 0 && 149 (profile->srtcm_rfc2697.cir << 150 MLX5_MTRS_PPS_MAP_BPS_SHIFT) 151 <= MLX5_SRTCM_CIR_MAX && 152 profile->srtcm_rfc2697.cbs > 0 && 153 (profile->srtcm_rfc2697.cbs << 154 MLX5_MTRS_PPS_MAP_BPS_SHIFT) 155 <= MLX5_SRTCM_CBS_MAX && 156 (profile->srtcm_rfc2697.ebs << 157 MLX5_MTRS_PPS_MAP_BPS_SHIFT) 158 <= MLX5_SRTCM_EBS_MAX) 159 return 0; 160 return -rte_mtr_error_set 161 (error, ENOTSUP, 162 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 163 NULL, 164 profile->srtcm_rfc2697.ebs ? 165 "Metering value ebs must be 0." : 166 "Invalid metering parameters."); 167 } 168 if (profile->srtcm_rfc2697.cir > 0 && 169 profile->srtcm_rfc2697.cir <= 170 MLX5_SRTCM_CIR_MAX && 171 profile->srtcm_rfc2697.cbs > 0 && 172 profile->srtcm_rfc2697.cbs <= 173 MLX5_SRTCM_CBS_MAX && 174 profile->srtcm_rfc2697.ebs <= 175 MLX5_SRTCM_EBS_MAX) 176 return 0; 177 return -rte_mtr_error_set(error, ENOTSUP, 178 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 179 NULL, 180 profile->srtcm_rfc2697.ebs ? 181 "Metering value ebs must be 0." : 182 "Invalid metering parameters."); 183 } 184 } 185 return -rte_mtr_error_set(error, ENOTSUP, 186 RTE_MTR_ERROR_TYPE_METER_PROFILE, 187 NULL, "Metering algorithm not supported."); 188 } 189 190 /** 191 * Calculate mantissa and exponent for cir. 192 * 193 * @param[in] cir 194 * Value to be calculated. 195 * @param[out] man 196 * Pointer to the mantissa. 197 * @param[out] exp 198 * Pointer to the exp. 199 */ 200 static void 201 mlx5_flow_meter_cir_man_exp_calc(int64_t cir, uint8_t *man, uint8_t *exp) 202 { 203 int64_t _cir; 204 int64_t delta = INT64_MAX; 205 uint8_t _man = 0; 206 uint8_t _exp = 0; 207 uint64_t m, e; 208 209 for (m = 0; m <= 0xFF; m++) { /* man width 8 bit */ 210 for (e = 0; e <= 0x1F; e++) { /* exp width 5bit */ 211 _cir = (1000000000ULL * m) >> e; 212 if (llabs(cir - _cir) <= delta) { 213 delta = llabs(cir - _cir); 214 _man = m; 215 _exp = e; 216 } 217 } 218 } 219 *man = _man; 220 *exp = _exp; 221 } 222 223 /** 224 * Calculate mantissa and exponent for xbs. 225 * 226 * @param[in] xbs 227 * Value to be calculated. 228 * @param[out] man 229 * Pointer to the mantissa. 230 * @param[out] exp 231 * Pointer to the exp. 232 */ 233 static void 234 mlx5_flow_meter_xbs_man_exp_calc(uint64_t xbs, uint8_t *man, uint8_t *exp) 235 { 236 int _exp; 237 double _man; 238 239 /* Special case xbs == 0 ? both exp and matissa are 0. */ 240 if (xbs == 0) { 241 *man = 0; 242 *exp = 0; 243 return; 244 } 245 /* xbs = xbs_mantissa * 2^xbs_exponent */ 246 _man = frexp(xbs, &_exp); 247 _man = _man * pow(2, MLX5_MAN_WIDTH); 248 _exp = _exp - MLX5_MAN_WIDTH; 249 *man = (uint8_t)ceil(_man); 250 *exp = _exp; 251 } 252 253 /** 254 * Fill the prm meter parameter. 255 * 256 * @param[in,out] fmp 257 * Pointer to meter profie to be converted. 258 * @param[out] error 259 * Pointer to the error structure. 260 * 261 * @return 262 * 0 on success, a negative errno value otherwise and rte_errno is set. 263 */ 264 static int 265 mlx5_flow_meter_param_fill(struct mlx5_flow_meter_profile *fmp, 266 struct mlx5_priv *priv, struct rte_mtr_error *error) 267 { 268 struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = &fmp->srtcm_prm; 269 uint8_t man, exp; 270 uint32_t cbs_exp, cbs_man, cir_exp, cir_man; 271 uint32_t ebs_exp, ebs_man; 272 uint64_t cir, cbs, ebs; 273 274 if (fmp->profile.alg != RTE_MTR_SRTCM_RFC2697) 275 return -rte_mtr_error_set(error, ENOTSUP, 276 RTE_MTR_ERROR_TYPE_METER_PROFILE, 277 NULL, "Metering algorithm not supported."); 278 if (!priv->sh->meter_aso_en && fmp->profile.packet_mode) 279 return -rte_mtr_error_set(error, ENOTSUP, 280 RTE_MTR_ERROR_TYPE_METER_PROFILE, 281 NULL, "Metering algorithm packet mode not supported."); 282 if (priv->sh->meter_aso_en && fmp->profile.packet_mode) { 283 cir = fmp->profile.srtcm_rfc2697.cir << 284 MLX5_MTRS_PPS_MAP_BPS_SHIFT; 285 cbs = fmp->profile.srtcm_rfc2697.cbs << 286 MLX5_MTRS_PPS_MAP_BPS_SHIFT; 287 ebs = fmp->profile.srtcm_rfc2697.ebs << 288 MLX5_MTRS_PPS_MAP_BPS_SHIFT; 289 } else { 290 cir = fmp->profile.srtcm_rfc2697.cir; 291 cbs = fmp->profile.srtcm_rfc2697.cbs; 292 ebs = fmp->profile.srtcm_rfc2697.ebs; 293 } 294 /* cir = 8G * cir_mantissa * 1/(2^cir_exponent)) Bytes/Sec */ 295 mlx5_flow_meter_cir_man_exp_calc(cir, &man, &exp); 296 /* Check if cir mantissa is too large. */ 297 if (exp > ASO_DSEG_CIR_EXP_MASK) 298 return -rte_mtr_error_set(error, ENOTSUP, 299 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 300 "meter profile parameter cir is" 301 " not supported."); 302 cir_man = man; 303 cir_exp = exp; 304 /* cbs = cbs_mantissa * 2^cbs_exponent */ 305 mlx5_flow_meter_xbs_man_exp_calc(cbs, &man, &exp); 306 /* Check if cbs mantissa is too large. */ 307 if (exp > ASO_DSEG_EXP_MASK) 308 return -rte_mtr_error_set(error, ENOTSUP, 309 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 310 "meter profile parameter cbs is" 311 " not supported."); 312 cbs_man = man; 313 cbs_exp = exp; 314 srtcm->cbs_cir = rte_cpu_to_be_32(cbs_exp << ASO_DSEG_CBS_EXP_OFFSET | 315 cbs_man << ASO_DSEG_CBS_MAN_OFFSET | 316 cir_exp << ASO_DSEG_CIR_EXP_OFFSET | 317 cir_man); 318 mlx5_flow_meter_xbs_man_exp_calc(ebs, &man, &exp); 319 /* Check if ebs mantissa is too large. */ 320 if (exp > ASO_DSEG_EXP_MASK) 321 return -rte_mtr_error_set(error, ENOTSUP, 322 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 323 "meter profile parameter ebs is" 324 " not supported."); 325 ebs_man = man; 326 ebs_exp = exp; 327 srtcm->ebs_eir = rte_cpu_to_be_32(ebs_exp << ASO_DSEG_EBS_EXP_OFFSET | 328 ebs_man << ASO_DSEG_EBS_MAN_OFFSET); 329 return 0; 330 } 331 332 /** 333 * Callback to get MTR capabilities. 334 * 335 * @param[in] dev 336 * Pointer to Ethernet device. 337 * @param[out] cap 338 * Pointer to save MTR capabilities. 339 * @param[out] error 340 * Pointer to the error structure. 341 * 342 * @return 343 * 0 on success, a negative errno value otherwise and rte_errno is set. 344 */ 345 static int 346 mlx5_flow_mtr_cap_get(struct rte_eth_dev *dev, 347 struct rte_mtr_capabilities *cap, 348 struct rte_mtr_error *error __rte_unused) 349 { 350 struct mlx5_priv *priv = dev->data->dev_private; 351 struct mlx5_hca_qos_attr *qattr = &priv->config.hca_attr.qos; 352 353 if (!priv->mtr_en) 354 return -rte_mtr_error_set(error, ENOTSUP, 355 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 356 "Meter is not supported"); 357 memset(cap, 0, sizeof(*cap)); 358 if (priv->sh->meter_aso_en) { 359 /* 2 meters per one ASO cache line. */ 360 cap->n_max = 1 << (qattr->log_max_num_meter_aso + 1); 361 cap->srtcm_rfc2697_packet_mode_supported = 1; 362 } else { 363 cap->n_max = 1 << qattr->log_max_flow_meter; 364 cap->srtcm_rfc2697_packet_mode_supported = 0; 365 } 366 cap->srtcm_rfc2697_byte_mode_supported = 1; 367 cap->n_shared_max = cap->n_max; 368 cap->identical = 1; 369 cap->shared_identical = 1; 370 cap->shared_n_flows_per_mtr_max = 4 << 20; 371 /* 2M flows can share the same meter. */ 372 cap->chaining_n_mtrs_per_flow_max = 1; /* Chaining is not supported. */ 373 cap->meter_srtcm_rfc2697_n_max = qattr->flow_meter_old ? cap->n_max : 0; 374 cap->meter_rate_max = 1ULL << 40; /* 1 Tera tokens per sec. */ 375 cap->stats_mask = RTE_MTR_STATS_N_BYTES_DROPPED | 376 RTE_MTR_STATS_N_PKTS_DROPPED; 377 return 0; 378 } 379 380 /** 381 * Callback to add MTR profile. 382 * 383 * @param[in] dev 384 * Pointer to Ethernet device. 385 * @param[in] meter_profile_id 386 * Meter profile id. 387 * @param[in] profile 388 * Pointer to meter profile detail. 389 * @param[out] error 390 * Pointer to the error structure. 391 * 392 * @return 393 * 0 on success, a negative errno value otherwise and rte_errno is set. 394 */ 395 static int 396 mlx5_flow_meter_profile_add(struct rte_eth_dev *dev, 397 uint32_t meter_profile_id, 398 struct rte_mtr_meter_profile *profile, 399 struct rte_mtr_error *error) 400 { 401 struct mlx5_priv *priv = dev->data->dev_private; 402 struct mlx5_mtr_profiles *fmps = &priv->flow_meter_profiles; 403 struct mlx5_flow_meter_profile *fmp; 404 int ret; 405 406 if (!priv->mtr_en) 407 return -rte_mtr_error_set(error, ENOTSUP, 408 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 409 "Meter is not supported"); 410 /* Check input params. */ 411 ret = mlx5_flow_meter_profile_validate(dev, meter_profile_id, 412 profile, error); 413 if (ret) 414 return ret; 415 /* Meter profile memory allocation. */ 416 fmp = mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct mlx5_flow_meter_profile), 417 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 418 if (fmp == NULL) 419 return -rte_mtr_error_set(error, ENOMEM, 420 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 421 NULL, "Meter profile memory " 422 "alloc failed."); 423 /* Fill profile info. */ 424 fmp->id = meter_profile_id; 425 fmp->profile = *profile; 426 /* Fill the flow meter parameters for the PRM. */ 427 ret = mlx5_flow_meter_param_fill(fmp, priv, error); 428 if (ret) 429 goto error; 430 /* Add to list. */ 431 TAILQ_INSERT_TAIL(fmps, fmp, next); 432 return 0; 433 error: 434 mlx5_free(fmp); 435 return ret; 436 } 437 438 /** 439 * Callback to delete MTR profile. 440 * 441 * @param[in] dev 442 * Pointer to Ethernet device. 443 * @param[in] meter_profile_id 444 * Meter profile id. 445 * @param[out] error 446 * Pointer to the error structure. 447 * 448 * @return 449 * 0 on success, a negative errno value otherwise and rte_errno is set. 450 */ 451 static int 452 mlx5_flow_meter_profile_delete(struct rte_eth_dev *dev, 453 uint32_t meter_profile_id, 454 struct rte_mtr_error *error) 455 { 456 struct mlx5_priv *priv = dev->data->dev_private; 457 struct mlx5_flow_meter_profile *fmp; 458 459 if (!priv->mtr_en) 460 return -rte_mtr_error_set(error, ENOTSUP, 461 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 462 "Meter is not supported"); 463 /* Meter profile must exist. */ 464 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 465 if (fmp == NULL) 466 return -rte_mtr_error_set(error, ENOENT, 467 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 468 &meter_profile_id, 469 "Meter profile id is invalid."); 470 /* Check profile is unused. */ 471 if (fmp->ref_cnt) 472 return -rte_mtr_error_set(error, EBUSY, 473 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 474 NULL, "Meter profile is in use."); 475 /* Remove from list. */ 476 TAILQ_REMOVE(&priv->flow_meter_profiles, fmp, next); 477 mlx5_free(fmp); 478 return 0; 479 } 480 481 /** 482 * Find policy by id. 483 * 484 * @param[in] dev 485 * Pointer to Ethernet device. 486 * @param policy_id 487 * Policy id. 488 * 489 * @return 490 * Pointer to the policy found on success, NULL otherwise. 491 */ 492 struct mlx5_flow_meter_policy * 493 mlx5_flow_meter_policy_find(struct rte_eth_dev *dev, 494 uint32_t policy_id, 495 uint32_t *policy_idx) 496 { 497 struct mlx5_priv *priv = dev->data->dev_private; 498 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 499 union mlx5_l3t_data data; 500 501 if (policy_id > MLX5_MAX_SUB_POLICY_TBL_NUM || 502 !priv->sh->mtrmng->policy_idx_tbl) 503 return NULL; 504 if (mlx5_l3t_get_entry(priv->sh->mtrmng->policy_idx_tbl, 505 policy_id, &data) || 506 !data.dword) 507 return NULL; 508 if (policy_idx) 509 *policy_idx = data.dword; 510 sub_policy = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 511 data.dword); 512 /* Remove reference taken by the mlx5_l3t_get_entry. */ 513 mlx5_l3t_clear_entry(priv->sh->mtrmng->policy_idx_tbl, 514 policy_id); 515 if (sub_policy) 516 if (sub_policy->main_policy_id) 517 return sub_policy->main_policy; 518 return NULL; 519 } 520 521 /** 522 * Callback to check MTR policy action validate 523 * 524 * @param[in] dev 525 * Pointer to Ethernet device. 526 * @param[in] actions 527 * Pointer to meter policy action detail. 528 * @param[out] error 529 * Pointer to the error structure. 530 * 531 * @return 532 * 0 on success, a negative errno value otherwise and rte_errno is set. 533 */ 534 static int 535 mlx5_flow_meter_policy_validate(struct rte_eth_dev *dev, 536 struct rte_mtr_meter_policy_params *policy, 537 struct rte_mtr_error *error) 538 { 539 struct mlx5_priv *priv = dev->data->dev_private; 540 struct rte_flow_attr attr = { .transfer = 541 priv->config.dv_esw_en ? 1 : 0}; 542 bool is_rss = false; 543 bool is_def_policy = false; 544 uint8_t domain_bitmap; 545 int ret; 546 547 if (!priv->mtr_en || !priv->sh->meter_aso_en) 548 return -rte_mtr_error_set(error, ENOTSUP, 549 RTE_MTR_ERROR_TYPE_METER_POLICY, 550 NULL, "meter policy unsupported."); 551 ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr, 552 &is_rss, &domain_bitmap, &is_def_policy, error); 553 if (ret) 554 return ret; 555 return 0; 556 } 557 558 static int 559 __mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev, 560 uint32_t policy_id, 561 struct mlx5_flow_meter_policy *mtr_policy, 562 struct rte_mtr_error *error, 563 bool clear_l3t) 564 { 565 struct mlx5_priv *priv = dev->data->dev_private; 566 struct mlx5_flow_meter_sub_policy *sub_policy; 567 uint32_t i, j; 568 uint16_t sub_policy_num; 569 570 rte_spinlock_lock(&mtr_policy->sl); 571 if (mtr_policy->ref_cnt) { 572 rte_spinlock_unlock(&mtr_policy->sl); 573 return -rte_mtr_error_set(error, EBUSY, 574 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 575 NULL, 576 "Meter policy object is being used."); 577 } 578 mlx5_flow_destroy_policy_rules(dev, mtr_policy); 579 mlx5_flow_destroy_mtr_acts(dev, mtr_policy); 580 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 581 sub_policy_num = (mtr_policy->sub_policy_num >> 582 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) & 583 MLX5_MTR_SUB_POLICY_NUM_MASK; 584 if (sub_policy_num) { 585 for (j = 0; j < sub_policy_num; j++) { 586 sub_policy = mtr_policy->sub_policys[i][j]; 587 if (sub_policy) 588 mlx5_ipool_free 589 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 590 sub_policy->idx); 591 } 592 } 593 } 594 if (priv->sh->mtrmng->policy_idx_tbl && clear_l3t) { 595 if (mlx5_l3t_clear_entry(priv->sh->mtrmng->policy_idx_tbl, 596 policy_id)) { 597 rte_spinlock_unlock(&mtr_policy->sl); 598 return -rte_mtr_error_set(error, ENOTSUP, 599 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL, 600 "Fail to delete policy in index table."); 601 } 602 } 603 rte_spinlock_unlock(&mtr_policy->sl); 604 return 0; 605 } 606 607 /** 608 * Callback to add MTR policy. 609 * 610 * @param[in] dev 611 * Pointer to Ethernet device. 612 * @param[out] policy_id 613 * Pointer to policy id 614 * @param[in] actions 615 * Pointer to meter policy action detail. 616 * @param[out] error 617 * Pointer to the error structure. 618 * 619 * @return 620 * 0 on success, a negative errno value otherwise and rte_errno is set. 621 */ 622 static int 623 mlx5_flow_meter_policy_add(struct rte_eth_dev *dev, 624 uint32_t policy_id, 625 struct rte_mtr_meter_policy_params *policy, 626 struct rte_mtr_error *error) 627 { 628 struct mlx5_priv *priv = dev->data->dev_private; 629 struct rte_flow_attr attr = { .transfer = 630 priv->config.dv_esw_en ? 1 : 0}; 631 uint32_t sub_policy_idx = 0; 632 uint32_t policy_idx = 0; 633 struct mlx5_flow_meter_policy *mtr_policy = NULL; 634 struct mlx5_flow_meter_sub_policy *sub_policy; 635 bool is_rss = false; 636 bool is_def_policy = false; 637 uint32_t i; 638 int ret; 639 uint32_t policy_size = sizeof(struct mlx5_flow_meter_policy); 640 uint16_t sub_policy_num; 641 uint8_t domain_bitmap = 0; 642 union mlx5_l3t_data data; 643 644 if (!priv->mtr_en) 645 return -rte_mtr_error_set(error, ENOTSUP, 646 RTE_MTR_ERROR_TYPE_METER_POLICY, 647 NULL, "meter policy unsupported."); 648 if (policy_id == MLX5_INVALID_POLICY_ID) 649 return -rte_mtr_error_set(error, ENOTSUP, 650 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL, 651 "policy ID is invalid. "); 652 if (policy_id == priv->sh->mtrmng->def_policy_id) 653 return -rte_mtr_error_set(error, EEXIST, 654 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL, 655 "policy ID exists. "); 656 mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, 657 &policy_idx); 658 if (mtr_policy) 659 return -rte_mtr_error_set(error, EEXIST, 660 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL, 661 "policy ID exists. "); 662 ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr, 663 &is_rss, &domain_bitmap, &is_def_policy, error); 664 if (ret) 665 return ret; 666 if (!domain_bitmap) 667 return -rte_mtr_error_set(error, ENOTSUP, 668 RTE_MTR_ERROR_TYPE_METER_POLICY, 669 NULL, "fail to find policy domain."); 670 if (is_def_policy) { 671 if (priv->sh->mtrmng->def_policy_id != MLX5_INVALID_POLICY_ID) 672 return -rte_mtr_error_set(error, EEXIST, 673 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 674 NULL, "a policy with similar actions " 675 "is already configured"); 676 if (mlx5_flow_create_def_policy(dev)) 677 return -rte_mtr_error_set(error, ENOTSUP, 678 RTE_MTR_ERROR_TYPE_METER_POLICY, 679 NULL, 680 "fail to create non-terminated policy."); 681 priv->sh->mtrmng->def_policy_id = policy_id; 682 return 0; 683 } 684 if (!priv->sh->meter_aso_en) 685 return -rte_mtr_error_set(error, ENOTSUP, 686 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL, 687 "no ASO capability to support the policy "); 688 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 689 if (!(domain_bitmap & (1 << i))) 690 continue; 691 if (is_rss) { 692 policy_size += 693 sizeof(struct mlx5_flow_meter_sub_policy *) * 694 MLX5_MTR_RSS_MAX_SUB_POLICY; 695 break; 696 } 697 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *); 698 } 699 mtr_policy = mlx5_malloc(MLX5_MEM_ZERO, policy_size, 700 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 701 if (!mtr_policy) 702 return -rte_mtr_error_set(error, ENOMEM, 703 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL, 704 "Memory alloc failed for meter policy."); 705 policy_size = sizeof(struct mlx5_flow_meter_policy); 706 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 707 if (!(domain_bitmap & (1 << i))) 708 continue; 709 if (i == MLX5_MTR_DOMAIN_INGRESS) 710 mtr_policy->ingress = 1; 711 if (i == MLX5_MTR_DOMAIN_EGRESS) 712 mtr_policy->egress = 1; 713 if (i == MLX5_MTR_DOMAIN_TRANSFER) 714 mtr_policy->transfer = 1; 715 sub_policy = mlx5_ipool_zmalloc 716 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 717 &sub_policy_idx); 718 if (!sub_policy) 719 goto policy_add_err; 720 if (sub_policy_idx > MLX5_MAX_SUB_POLICY_TBL_NUM) 721 goto policy_add_err; 722 sub_policy->idx = sub_policy_idx; 723 sub_policy->main_policy = mtr_policy; 724 if (!policy_idx) { 725 policy_idx = sub_policy_idx; 726 sub_policy->main_policy_id = 1; 727 } 728 mtr_policy->sub_policys[i] = 729 (struct mlx5_flow_meter_sub_policy **) 730 ((uint8_t *)mtr_policy + policy_size); 731 mtr_policy->sub_policys[i][0] = sub_policy; 732 sub_policy_num = (mtr_policy->sub_policy_num >> 733 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) & 734 MLX5_MTR_SUB_POLICY_NUM_MASK; 735 sub_policy_num++; 736 mtr_policy->sub_policy_num &= ~(MLX5_MTR_SUB_POLICY_NUM_MASK << 737 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)); 738 mtr_policy->sub_policy_num |= 739 (sub_policy_num & MLX5_MTR_SUB_POLICY_NUM_MASK) << 740 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i); 741 if (is_rss) { 742 mtr_policy->is_rss = 1; 743 break; 744 } 745 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *); 746 } 747 rte_spinlock_init(&mtr_policy->sl); 748 ret = mlx5_flow_create_mtr_acts(dev, mtr_policy, 749 policy->actions, error); 750 if (ret) 751 goto policy_add_err; 752 if (!is_rss && !mtr_policy->is_queue) { 753 /* Create policy rules in HW. */ 754 ret = mlx5_flow_create_policy_rules(dev, mtr_policy); 755 if (ret) 756 goto policy_add_err; 757 } 758 data.dword = policy_idx; 759 if (!priv->sh->mtrmng->policy_idx_tbl) { 760 priv->sh->mtrmng->policy_idx_tbl = 761 mlx5_l3t_create(MLX5_L3T_TYPE_DWORD); 762 if (!priv->sh->mtrmng->policy_idx_tbl) 763 goto policy_add_err; 764 } 765 if (mlx5_l3t_set_entry(priv->sh->mtrmng->policy_idx_tbl, 766 policy_id, &data)) 767 goto policy_add_err; 768 return 0; 769 policy_add_err: 770 if (mtr_policy) { 771 ret = __mlx5_flow_meter_policy_delete(dev, policy_id, 772 mtr_policy, error, false); 773 mlx5_free(mtr_policy); 774 if (ret) 775 return ret; 776 } 777 return -rte_mtr_error_set(error, ENOTSUP, 778 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 779 NULL, "Failed to create devx policy."); 780 } 781 782 /** 783 * Callback to delete MTR policy. 784 * 785 * @param[in] dev 786 * Pointer to Ethernet device. 787 * @param[in] policy_id 788 * Meter policy id. 789 * @param[out] error 790 * Pointer to the error structure. 791 * 792 * @return 793 * 0 on success, a negative errno value otherwise and rte_errno is set. 794 */ 795 static int 796 mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev, 797 uint32_t policy_id, 798 struct rte_mtr_error *error) 799 { 800 struct mlx5_priv *priv = dev->data->dev_private; 801 struct mlx5_flow_meter_policy *mtr_policy; 802 uint32_t policy_idx; 803 int ret; 804 805 if (policy_id == priv->sh->mtrmng->def_policy_id) { 806 if (priv->sh->mtrmng->def_policy_ref_cnt > 0) 807 return -rte_mtr_error_set(error, ENOTSUP, 808 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL, 809 "Meter policy object is being used."); 810 priv->sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID; 811 return 0; 812 } 813 mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx); 814 if (!mtr_policy) 815 return -rte_mtr_error_set(error, ENOTSUP, 816 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL, 817 "Meter policy id is invalid. "); 818 ret = __mlx5_flow_meter_policy_delete(dev, policy_id, mtr_policy, 819 error, true); 820 if (ret) 821 return ret; 822 mlx5_free(mtr_policy); 823 return 0; 824 } 825 826 /** 827 * Check meter validation. 828 * 829 * @param[in] priv 830 * Pointer to mlx5 private data structure. 831 * @param[in] meter_id 832 * Meter id. 833 * @param[in] params 834 * Pointer to rte meter parameters. 835 * @param[out] error 836 * Pointer to rte meter error structure. 837 * 838 * @return 839 * 0 on success, a negative errno value otherwise and rte_errno is set. 840 */ 841 static int 842 mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id, 843 struct rte_mtr_params *params, 844 struct rte_mtr_error *error) 845 { 846 /* Meter must use global drop action. */ 847 if (!priv->sh->dr_drop_action) 848 return -rte_mtr_error_set(error, ENOTSUP, 849 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 850 NULL, 851 "No drop action ready for meter."); 852 /* Meter params must not be NULL. */ 853 if (params == NULL) 854 return -rte_mtr_error_set(error, EINVAL, 855 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 856 NULL, "Meter object params null."); 857 /* Previous meter color is not supported. */ 858 if (params->use_prev_mtr_color) 859 return -rte_mtr_error_set(error, ENOTSUP, 860 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 861 NULL, 862 "Previous meter color " 863 "not supported."); 864 if (params->meter_policy_id == MLX5_INVALID_POLICY_ID) 865 return -rte_mtr_error_set(error, ENOENT, 866 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 867 NULL, "Meter policy id not valid."); 868 /* Validate meter id. */ 869 if (mlx5_flow_meter_find(priv, meter_id, NULL)) 870 return -rte_mtr_error_set(error, EEXIST, 871 RTE_MTR_ERROR_TYPE_MTR_ID, NULL, 872 "Meter object already exists."); 873 return 0; 874 } 875 876 /** 877 * Modify the flow meter action. 878 * 879 * @param[in] priv 880 * Pointer to mlx5 private data structure. 881 * @param[in] fm 882 * Pointer to flow meter to be modified. 883 * @param[in] srtcm 884 * Pointer to meter srtcm description parameter. 885 * @param[in] modify_bits 886 * The bit in srtcm to be updated. 887 * @param[in] active_state 888 * The state to be updated. 889 * @return 890 * 0 on success, o negative value otherwise. 891 */ 892 static int 893 mlx5_flow_meter_action_modify(struct mlx5_priv *priv, 894 struct mlx5_flow_meter_info *fm, 895 const struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm, 896 uint64_t modify_bits, uint32_t active_state, uint32_t is_enable) 897 { 898 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER 899 uint32_t in[MLX5_ST_SZ_DW(flow_meter_parameters)] = { 0 }; 900 uint32_t *attr; 901 struct mlx5dv_dr_flow_meter_attr mod_attr = { 0 }; 902 int ret; 903 struct mlx5_aso_mtr *aso_mtr = NULL; 904 uint32_t cbs_cir, ebs_eir, val; 905 906 if (priv->sh->meter_aso_en) { 907 fm->is_enable = !!is_enable; 908 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm); 909 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr); 910 if (ret) 911 return ret; 912 ret = mlx5_aso_mtr_wait(priv->sh, aso_mtr); 913 if (ret) 914 return ret; 915 } else { 916 /* Fill command parameters. */ 917 mod_attr.reg_c_index = priv->mtr_color_reg - REG_C_0; 918 mod_attr.flow_meter_parameter = in; 919 mod_attr.flow_meter_parameter_sz = 920 MLX5_ST_SZ_BYTES(flow_meter_parameters); 921 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE) 922 mod_attr.active = !!active_state; 923 else 924 mod_attr.active = 0; 925 attr = in; 926 cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir); 927 ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir); 928 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS) { 929 val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & 930 ASO_DSEG_EXP_MASK; 931 MLX5_SET(flow_meter_parameters, attr, 932 cbs_exponent, val); 933 val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & 934 ASO_DSEG_MAN_MASK; 935 MLX5_SET(flow_meter_parameters, attr, 936 cbs_mantissa, val); 937 } 938 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR) { 939 val = (cbs_cir >> ASO_DSEG_CIR_EXP_OFFSET) & 940 ASO_DSEG_EXP_MASK; 941 MLX5_SET(flow_meter_parameters, attr, 942 cir_exponent, val); 943 val = cbs_cir & ASO_DSEG_MAN_MASK; 944 MLX5_SET(flow_meter_parameters, attr, 945 cir_mantissa, val); 946 } 947 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_EBS) { 948 val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & 949 ASO_DSEG_EXP_MASK; 950 MLX5_SET(flow_meter_parameters, attr, 951 ebs_exponent, val); 952 val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & 953 ASO_DSEG_MAN_MASK; 954 MLX5_SET(flow_meter_parameters, attr, 955 ebs_mantissa, val); 956 } 957 /* Apply modifications to meter only if it was created. */ 958 if (fm->meter_action) { 959 ret = mlx5_glue->dv_modify_flow_action_meter 960 (fm->meter_action, &mod_attr, 961 rte_cpu_to_be_64(modify_bits)); 962 if (ret) 963 return ret; 964 } 965 /* Update succeedded modify meter parameters. */ 966 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE) 967 fm->active_state = !!active_state; 968 } 969 return 0; 970 #else 971 (void)priv; 972 (void)fm; 973 (void)srtcm; 974 (void)modify_bits; 975 (void)active_state; 976 (void)is_enable; 977 return -ENOTSUP; 978 #endif 979 } 980 981 static int 982 mlx5_flow_meter_stats_enable_update(struct rte_eth_dev *dev, 983 struct mlx5_flow_meter_info *fm, 984 uint64_t stats_mask) 985 { 986 fm->bytes_dropped = 987 (stats_mask & RTE_MTR_STATS_N_BYTES_DROPPED) ? 1 : 0; 988 fm->pkts_dropped = (stats_mask & RTE_MTR_STATS_N_PKTS_DROPPED) ? 1 : 0; 989 if (fm->bytes_dropped || fm->pkts_dropped) { 990 if (!fm->drop_cnt) { 991 /* Alloc policer counters. */ 992 fm->drop_cnt = mlx5_counter_alloc(dev); 993 if (!fm->drop_cnt) 994 return -1; 995 } 996 } else { 997 if (fm->drop_cnt) { 998 mlx5_counter_free(dev, fm->drop_cnt); 999 fm->drop_cnt = 0; 1000 } 1001 } 1002 return 0; 1003 } 1004 1005 /** 1006 * Create meter rules. 1007 * 1008 * @param[in] dev 1009 * Pointer to Ethernet device. 1010 * @param[in] meter_id 1011 * Meter id. 1012 * @param[in] params 1013 * Pointer to rte meter parameters. 1014 * @param[in] shared 1015 * Meter shared with other flow or not. 1016 * @param[out] error 1017 * Pointer to rte meter error structure. 1018 * 1019 * @return 1020 * 0 on success, a negative errno value otherwise and rte_errno is set. 1021 */ 1022 static int 1023 mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, 1024 struct rte_mtr_params *params, int shared, 1025 struct rte_mtr_error *error) 1026 { 1027 struct mlx5_priv *priv = dev->data->dev_private; 1028 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; 1029 struct mlx5_flow_meter_profile *fmp; 1030 struct mlx5_flow_meter_info *fm; 1031 struct mlx5_legacy_flow_meter *legacy_fm; 1032 struct mlx5_flow_meter_policy *mtr_policy = NULL; 1033 struct mlx5_indexed_pool_config flow_ipool_cfg = { 1034 .size = 0, 1035 .trunk_size = 64, 1036 .need_lock = 1, 1037 .type = "mlx5_flow_mtr_flow_id_pool", 1038 }; 1039 struct mlx5_aso_mtr *aso_mtr; 1040 uint32_t mtr_idx, policy_idx; 1041 union mlx5_l3t_data data; 1042 int ret; 1043 uint8_t domain_bitmap; 1044 uint8_t mtr_id_bits; 1045 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 1046 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 1047 1048 if (!priv->mtr_en) 1049 return -rte_mtr_error_set(error, ENOTSUP, 1050 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1051 "Meter is not supported"); 1052 /* Validate the parameters. */ 1053 ret = mlx5_flow_meter_validate(priv, meter_id, params, error); 1054 if (ret) 1055 return ret; 1056 /* Meter profile must exist. */ 1057 fmp = mlx5_flow_meter_profile_find(priv, params->meter_profile_id); 1058 if (fmp == NULL) 1059 return -rte_mtr_error_set(error, ENOENT, 1060 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 1061 NULL, "Meter profile id not valid."); 1062 /* Meter policy must exist. */ 1063 if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) { 1064 __atomic_add_fetch 1065 (&priv->sh->mtrmng->def_policy_ref_cnt, 1066 1, __ATOMIC_RELAXED); 1067 domain_bitmap = MLX5_MTR_ALL_DOMAIN_BIT; 1068 if (!priv->config.dv_esw_en) 1069 domain_bitmap &= ~MLX5_MTR_DOMAIN_TRANSFER_BIT; 1070 } else { 1071 mtr_policy = mlx5_flow_meter_policy_find(dev, 1072 params->meter_policy_id, &policy_idx); 1073 if (!priv->sh->meter_aso_en) 1074 return -rte_mtr_error_set(error, ENOTSUP, 1075 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1076 "Part of the policies cannot be " 1077 "supported without ASO "); 1078 if (!mtr_policy) 1079 return -rte_mtr_error_set(error, ENOENT, 1080 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1081 NULL, "Meter policy id not valid."); 1082 domain_bitmap = (mtr_policy->ingress ? 1083 MLX5_MTR_DOMAIN_INGRESS_BIT : 0) | 1084 (mtr_policy->egress ? 1085 MLX5_MTR_DOMAIN_EGRESS_BIT : 0) | 1086 (mtr_policy->transfer ? 1087 MLX5_MTR_DOMAIN_TRANSFER_BIT : 0); 1088 } 1089 /* Allocate the flow meter memory. */ 1090 if (priv->sh->meter_aso_en) { 1091 mtr_idx = mlx5_flow_mtr_alloc(dev); 1092 if (!mtr_idx) 1093 return -rte_mtr_error_set(error, ENOMEM, 1094 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1095 "Memory alloc failed for meter."); 1096 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx); 1097 fm = &aso_mtr->fm; 1098 } else { 1099 legacy_fm = mlx5_ipool_zmalloc 1100 (priv->sh->ipool[MLX5_IPOOL_MTR], &mtr_idx); 1101 if (legacy_fm == NULL) 1102 return -rte_mtr_error_set(error, ENOMEM, 1103 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1104 "Memory alloc failed for meter."); 1105 legacy_fm->idx = mtr_idx; 1106 fm = &legacy_fm->fm; 1107 } 1108 mtr_id_bits = MLX5_REG_BITS - __builtin_clz(mtr_idx); 1109 if ((mtr_id_bits + priv->sh->mtrmng->max_mtr_flow_bits) > 1110 mtr_reg_bits) { 1111 DRV_LOG(ERR, "Meter number exceeds max limit."); 1112 goto error; 1113 } 1114 if (mtr_id_bits > priv->sh->mtrmng->max_mtr_bits) 1115 priv->sh->mtrmng->max_mtr_bits = mtr_id_bits; 1116 /* Fill the flow meter parameters. */ 1117 fm->meter_id = meter_id; 1118 fm->policy_id = params->meter_policy_id; 1119 fm->profile = fmp; 1120 if (mlx5_flow_meter_stats_enable_update(dev, fm, params->stats_mask)) 1121 goto error; 1122 if (mlx5_flow_create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap)) 1123 goto error; 1124 /* Add to the flow meter list. */ 1125 if (!priv->sh->meter_aso_en) 1126 TAILQ_INSERT_TAIL(fms, legacy_fm, next); 1127 /* Add to the flow meter list. */ 1128 fm->active_state = 1; /* Config meter starts as active. */ 1129 fm->is_enable = 1; 1130 fm->shared = !!shared; 1131 __atomic_add_fetch(&fm->profile->ref_cnt, 1, __ATOMIC_RELAXED); 1132 if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) { 1133 fm->def_policy = 1; 1134 fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg); 1135 if (!fm->flow_ipool) 1136 goto error; 1137 } 1138 rte_spinlock_init(&fm->sl); 1139 /* If ASO meter supported, update ASO flow meter by wqe. */ 1140 if (priv->sh->meter_aso_en) { 1141 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm); 1142 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr); 1143 if (ret) 1144 goto error; 1145 if (!priv->mtr_idx_tbl) { 1146 priv->mtr_idx_tbl = 1147 mlx5_l3t_create(MLX5_L3T_TYPE_DWORD); 1148 if (!priv->mtr_idx_tbl) 1149 goto error; 1150 } 1151 data.dword = mtr_idx; 1152 if (mlx5_l3t_set_entry(priv->mtr_idx_tbl, meter_id, &data)) 1153 goto error; 1154 } 1155 if (mtr_policy) 1156 __atomic_add_fetch(&mtr_policy->ref_cnt, 1, __ATOMIC_RELAXED); 1157 return 0; 1158 error: 1159 mlx5_flow_destroy_mtr_tbls(dev, fm); 1160 /* Free policer counters. */ 1161 if (fm->drop_cnt) 1162 mlx5_counter_free(dev, fm->drop_cnt); 1163 if (priv->sh->meter_aso_en) 1164 mlx5_flow_mtr_free(dev, mtr_idx); 1165 else 1166 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], mtr_idx); 1167 return -rte_mtr_error_set(error, ENOTSUP, 1168 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 1169 NULL, "Failed to create devx meter."); 1170 } 1171 1172 static int 1173 mlx5_flow_meter_params_flush(struct rte_eth_dev *dev, 1174 struct mlx5_flow_meter_info *fm, 1175 uint32_t mtr_idx) 1176 { 1177 struct mlx5_priv *priv = dev->data->dev_private; 1178 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; 1179 struct mlx5_flow_meter_profile *fmp; 1180 struct mlx5_legacy_flow_meter *legacy_fm = NULL; 1181 struct mlx5_flow_meter_policy *mtr_policy; 1182 1183 /* Meter object must not have any owner. */ 1184 MLX5_ASSERT(!fm->ref_cnt); 1185 /* Get meter profile. */ 1186 fmp = fm->profile; 1187 if (fmp == NULL) 1188 return -1; 1189 /* Update dependencies. */ 1190 __atomic_sub_fetch(&fmp->ref_cnt, 1, __ATOMIC_RELAXED); 1191 fm->profile = NULL; 1192 /* Remove from list. */ 1193 if (!priv->sh->meter_aso_en) { 1194 legacy_fm = container_of(fm, 1195 struct mlx5_legacy_flow_meter, fm); 1196 TAILQ_REMOVE(fms, legacy_fm, next); 1197 } 1198 /* Free drop counters. */ 1199 if (fm->drop_cnt) 1200 mlx5_counter_free(dev, fm->drop_cnt); 1201 /* Free meter flow table. */ 1202 if (fm->flow_ipool) { 1203 mlx5_ipool_destroy(fm->flow_ipool); 1204 fm->flow_ipool = 0; 1205 } 1206 mlx5_flow_destroy_mtr_tbls(dev, fm); 1207 if (fm->def_policy) 1208 __atomic_sub_fetch(&priv->sh->mtrmng->def_policy_ref_cnt, 1209 1, __ATOMIC_RELAXED); 1210 if (priv->sh->meter_aso_en) { 1211 if (!fm->def_policy) { 1212 mtr_policy = mlx5_flow_meter_policy_find(dev, 1213 fm->policy_id, NULL); 1214 if (mtr_policy) 1215 __atomic_sub_fetch(&mtr_policy->ref_cnt, 1216 1, __ATOMIC_RELAXED); 1217 fm->policy_id = 0; 1218 } 1219 fm->def_policy = 0; 1220 if (mlx5_l3t_clear_entry(priv->mtr_idx_tbl, fm->meter_id)) 1221 return -1; 1222 mlx5_flow_mtr_free(dev, mtr_idx); 1223 } else { 1224 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], 1225 legacy_fm->idx); 1226 } 1227 return 0; 1228 } 1229 1230 /** 1231 * Destroy meter rules. 1232 * 1233 * @param[in] dev 1234 * Pointer to Ethernet device. 1235 * @param[in] meter_id 1236 * Meter id. 1237 * @param[out] error 1238 * Pointer to rte meter error structure. 1239 * 1240 * @return 1241 * 0 on success, a negative errno value otherwise and rte_errno is set. 1242 */ 1243 static int 1244 mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id, 1245 struct rte_mtr_error *error) 1246 { 1247 struct mlx5_priv *priv = dev->data->dev_private; 1248 struct mlx5_flow_meter_info *fm; 1249 uint32_t mtr_idx = 0; 1250 1251 if (!priv->mtr_en) 1252 return -rte_mtr_error_set(error, ENOTSUP, 1253 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 1254 NULL, 1255 "Meter is not supported"); 1256 /* Meter object must exist. */ 1257 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 1258 if (fm == NULL) 1259 return -rte_mtr_error_set(error, ENOENT, 1260 RTE_MTR_ERROR_TYPE_MTR_ID, 1261 NULL, 1262 "Meter object id not valid."); 1263 /* Meter object must not have any owner. */ 1264 if (fm->ref_cnt > 0) 1265 return -rte_mtr_error_set(error, EBUSY, 1266 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 1267 NULL, 1268 "Meter object is being used."); 1269 /* Destroy the meter profile. */ 1270 if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx)) 1271 return -rte_mtr_error_set(error, EINVAL, 1272 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 1273 NULL, 1274 "MTR object meter profile invalid."); 1275 return 0; 1276 } 1277 1278 /** 1279 * Modify meter state. 1280 * 1281 * @param[in] priv 1282 * Pointer to mlx5 private data structure. 1283 * @param[in] fm 1284 * Pointer to flow meter. 1285 * @param[in] new_state 1286 * New state to update. 1287 * @param[out] error 1288 * Pointer to rte meter error structure. 1289 * 1290 * @return 1291 * 0 on success, a negative errno value otherwise and rte_errno is set. 1292 */ 1293 static int 1294 mlx5_flow_meter_modify_state(struct mlx5_priv *priv, 1295 struct mlx5_flow_meter_info *fm, 1296 uint32_t new_state, 1297 struct rte_mtr_error *error) 1298 { 1299 static const struct mlx5_flow_meter_srtcm_rfc2697_prm srtcm = { 1300 .cbs_cir = RTE_BE32(MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL), 1301 .ebs_eir = 0, 1302 }; 1303 uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS | 1304 MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR; 1305 int ret; 1306 1307 if (new_state == MLX5_FLOW_METER_DISABLE) 1308 ret = mlx5_flow_meter_action_modify(priv, fm, 1309 &srtcm, modify_bits, 0, 0); 1310 else 1311 ret = mlx5_flow_meter_action_modify(priv, fm, 1312 &fm->profile->srtcm_prm, 1313 modify_bits, 0, 1); 1314 if (ret) 1315 return -rte_mtr_error_set(error, -ret, 1316 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 1317 NULL, 1318 new_state ? 1319 "Failed to enable meter." : 1320 "Failed to disable meter."); 1321 return 0; 1322 } 1323 1324 /** 1325 * Callback to enable flow meter. 1326 * 1327 * @param[in] dev 1328 * Pointer to Ethernet device. 1329 * @param[in] meter_id 1330 * Meter id. 1331 * @param[out] error 1332 * Pointer to rte meter error structure. 1333 * 1334 * @return 1335 * 0 on success, a negative errno value otherwise and rte_errno is set. 1336 */ 1337 static int 1338 mlx5_flow_meter_enable(struct rte_eth_dev *dev, 1339 uint32_t meter_id, 1340 struct rte_mtr_error *error) 1341 { 1342 struct mlx5_priv *priv = dev->data->dev_private; 1343 struct mlx5_flow_meter_info *fm; 1344 int ret; 1345 1346 if (!priv->mtr_en) 1347 return -rte_mtr_error_set(error, ENOTSUP, 1348 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1349 "Meter is not supported"); 1350 /* Meter object must exist. */ 1351 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 1352 if (fm == NULL) 1353 return -rte_mtr_error_set(error, ENOENT, 1354 RTE_MTR_ERROR_TYPE_MTR_ID, 1355 NULL, "Meter not found."); 1356 if (fm->active_state == MLX5_FLOW_METER_ENABLE) 1357 return 0; 1358 ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_ENABLE, 1359 error); 1360 if (!ret) 1361 fm->active_state = MLX5_FLOW_METER_ENABLE; 1362 return ret; 1363 } 1364 1365 /** 1366 * Callback to disable flow meter. 1367 * 1368 * @param[in] dev 1369 * Pointer to Ethernet device. 1370 * @param[in] meter_id 1371 * Meter id. 1372 * @param[out] error 1373 * Pointer to rte meter error structure. 1374 * 1375 * @return 1376 * 0 on success, a negative errno value otherwise and rte_errno is set. 1377 */ 1378 static int 1379 mlx5_flow_meter_disable(struct rte_eth_dev *dev, 1380 uint32_t meter_id, 1381 struct rte_mtr_error *error) 1382 { 1383 struct mlx5_priv *priv = dev->data->dev_private; 1384 struct mlx5_flow_meter_info *fm; 1385 int ret; 1386 1387 if (!priv->mtr_en) 1388 return -rte_mtr_error_set(error, ENOTSUP, 1389 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1390 "Meter is not supported"); 1391 /* Meter object must exist. */ 1392 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 1393 if (fm == NULL) 1394 return -rte_mtr_error_set(error, ENOENT, 1395 RTE_MTR_ERROR_TYPE_MTR_ID, 1396 NULL, "Meter not found."); 1397 if (fm->active_state == MLX5_FLOW_METER_DISABLE) 1398 return 0; 1399 ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_DISABLE, 1400 error); 1401 if (!ret) 1402 fm->active_state = MLX5_FLOW_METER_DISABLE; 1403 return ret; 1404 } 1405 1406 /** 1407 * Callback to update meter profile. 1408 * 1409 * @param[in] dev 1410 * Pointer to Ethernet device. 1411 * @param[in] meter_id 1412 * Meter id. 1413 * @param[in] meter_profile_id 1414 * To be updated meter profile id. 1415 * @param[out] error 1416 * Pointer to rte meter error structure. 1417 * 1418 * @return 1419 * 0 on success, a negative errno value otherwise and rte_errno is set. 1420 */ 1421 static int 1422 mlx5_flow_meter_profile_update(struct rte_eth_dev *dev, 1423 uint32_t meter_id, 1424 uint32_t meter_profile_id, 1425 struct rte_mtr_error *error) 1426 { 1427 struct mlx5_priv *priv = dev->data->dev_private; 1428 struct mlx5_flow_meter_profile *fmp; 1429 struct mlx5_flow_meter_profile *old_fmp; 1430 struct mlx5_flow_meter_info *fm; 1431 uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS | 1432 MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR; 1433 int ret; 1434 1435 if (!priv->mtr_en) 1436 return -rte_mtr_error_set(error, ENOTSUP, 1437 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1438 "Meter is not supported"); 1439 /* Meter profile must exist. */ 1440 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 1441 if (fmp == NULL) 1442 return -rte_mtr_error_set(error, ENOENT, 1443 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 1444 NULL, "Meter profile not found."); 1445 /* Meter object must exist. */ 1446 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 1447 if (fm == NULL) 1448 return -rte_mtr_error_set(error, ENOENT, 1449 RTE_MTR_ERROR_TYPE_MTR_ID, 1450 NULL, "Meter not found."); 1451 /* MTR object already set to meter profile id. */ 1452 old_fmp = fm->profile; 1453 if (fmp == old_fmp) 1454 return 0; 1455 /* Update the profile. */ 1456 fm->profile = fmp; 1457 /* Update meter params in HW (if not disabled). */ 1458 if (fm->active_state == MLX5_FLOW_METER_DISABLE) 1459 return 0; 1460 ret = mlx5_flow_meter_action_modify(priv, fm, &fm->profile->srtcm_prm, 1461 modify_bits, fm->active_state, 1); 1462 if (ret) { 1463 fm->profile = old_fmp; 1464 return -rte_mtr_error_set(error, -ret, 1465 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 1466 NULL, "Failed to update meter" 1467 " parmeters in hardware."); 1468 } 1469 old_fmp->ref_cnt--; 1470 fmp->ref_cnt++; 1471 return 0; 1472 } 1473 1474 /** 1475 * Callback to update meter stats mask. 1476 * 1477 * @param[in] dev 1478 * Pointer to Ethernet device. 1479 * @param[in] meter_id 1480 * Meter id. 1481 * @param[in] stats_mask 1482 * To be updated stats_mask. 1483 * @param[out] error 1484 * Pointer to rte meter error structure. 1485 * 1486 * @return 1487 * 0 on success, a negative errno value otherwise and rte_errno is set. 1488 */ 1489 static int 1490 mlx5_flow_meter_stats_update(struct rte_eth_dev *dev, 1491 uint32_t meter_id, 1492 uint64_t stats_mask, 1493 struct rte_mtr_error *error) 1494 { 1495 struct mlx5_priv *priv = dev->data->dev_private; 1496 struct mlx5_flow_meter_info *fm; 1497 1498 if (!priv->mtr_en) 1499 return -rte_mtr_error_set(error, ENOTSUP, 1500 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1501 "Meter is not supported"); 1502 /* Meter object must exist. */ 1503 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 1504 if (fm == NULL) 1505 return -rte_mtr_error_set(error, ENOENT, 1506 RTE_MTR_ERROR_TYPE_MTR_ID, 1507 NULL, "Meter object id not valid."); 1508 if (mlx5_flow_meter_stats_enable_update(dev, fm, stats_mask)) 1509 return -rte_mtr_error_set(error, ENOENT, 1510 RTE_MTR_ERROR_TYPE_MTR_ID, 1511 NULL, "Fail to allocate " 1512 "counter for meter."); 1513 return 0; 1514 } 1515 1516 /** 1517 * Callback to read meter statistics. 1518 * 1519 * @param[in] dev 1520 * Pointer to Ethernet device. 1521 * @param[in] meter_id 1522 * Meter id. 1523 * @param[out] stats 1524 * Pointer to store the statistics. 1525 * @param[out] stats_mask 1526 * Pointer to store the stats_mask. 1527 * @param[in] clear 1528 * Statistic to be cleared after read or not. 1529 * @param[out] error 1530 * Pointer to rte meter error structure. 1531 * 1532 * @return 1533 * 0 on success, a negative errno value otherwise and rte_errno is set. 1534 */ 1535 static int 1536 mlx5_flow_meter_stats_read(struct rte_eth_dev *dev, 1537 uint32_t meter_id, 1538 struct rte_mtr_stats *stats, 1539 uint64_t *stats_mask, 1540 int clear, 1541 struct rte_mtr_error *error) 1542 { 1543 struct mlx5_priv *priv = dev->data->dev_private; 1544 struct mlx5_flow_meter_info *fm; 1545 uint64_t pkts; 1546 uint64_t bytes; 1547 int ret = 0; 1548 1549 if (!priv->mtr_en) 1550 return -rte_mtr_error_set(error, ENOTSUP, 1551 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1552 "Meter is not supported"); 1553 /* Meter object must exist. */ 1554 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 1555 if (fm == NULL) 1556 return -rte_mtr_error_set(error, ENOENT, 1557 RTE_MTR_ERROR_TYPE_MTR_ID, 1558 NULL, "Meter object id not valid."); 1559 *stats_mask = 0; 1560 if (fm->bytes_dropped) 1561 *stats_mask |= RTE_MTR_STATS_N_BYTES_DROPPED; 1562 if (fm->pkts_dropped) 1563 *stats_mask |= RTE_MTR_STATS_N_PKTS_DROPPED; 1564 memset(stats, 0, sizeof(*stats)); 1565 if (fm->drop_cnt) { 1566 ret = mlx5_counter_query(dev, fm->drop_cnt, clear, &pkts, 1567 &bytes); 1568 if (ret) 1569 goto error; 1570 /* If need to read the packets, set it. */ 1571 if (fm->pkts_dropped) 1572 stats->n_pkts_dropped = pkts; 1573 /* If need to read the bytes, set it. */ 1574 if (fm->bytes_dropped) 1575 stats->n_bytes_dropped = bytes; 1576 } 1577 return 0; 1578 error: 1579 return -rte_mtr_error_set(error, ret, RTE_MTR_ERROR_TYPE_STATS, NULL, 1580 "Failed to read meter drop counters."); 1581 } 1582 1583 static const struct rte_mtr_ops mlx5_flow_mtr_ops = { 1584 .capabilities_get = mlx5_flow_mtr_cap_get, 1585 .meter_profile_add = mlx5_flow_meter_profile_add, 1586 .meter_profile_delete = mlx5_flow_meter_profile_delete, 1587 .meter_policy_validate = mlx5_flow_meter_policy_validate, 1588 .meter_policy_add = mlx5_flow_meter_policy_add, 1589 .meter_policy_delete = mlx5_flow_meter_policy_delete, 1590 .create = mlx5_flow_meter_create, 1591 .destroy = mlx5_flow_meter_destroy, 1592 .meter_enable = mlx5_flow_meter_enable, 1593 .meter_disable = mlx5_flow_meter_disable, 1594 .meter_profile_update = mlx5_flow_meter_profile_update, 1595 .meter_dscp_table_update = NULL, 1596 .stats_update = mlx5_flow_meter_stats_update, 1597 .stats_read = mlx5_flow_meter_stats_read, 1598 }; 1599 1600 /** 1601 * Get meter operations. 1602 * 1603 * @param dev 1604 * Pointer to Ethernet device structure. 1605 * @param arg 1606 * Pointer to set the mtr operations. 1607 * 1608 * @return 1609 * Always 0. 1610 */ 1611 int 1612 mlx5_flow_meter_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg) 1613 { 1614 *(const struct rte_mtr_ops **)arg = &mlx5_flow_mtr_ops; 1615 return 0; 1616 } 1617 1618 /** 1619 * Find meter by id. 1620 * 1621 * @param priv 1622 * Pointer to mlx5_priv. 1623 * @param meter_id 1624 * Meter id. 1625 * @param mtr_idx 1626 * Pointer to Meter index. 1627 * 1628 * @return 1629 * Pointer to the meter info found on success, NULL otherwise. 1630 */ 1631 struct mlx5_flow_meter_info * 1632 mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id, 1633 uint32_t *mtr_idx) 1634 { 1635 struct mlx5_legacy_flow_meter *legacy_fm; 1636 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; 1637 struct mlx5_aso_mtr *aso_mtr; 1638 struct mlx5_aso_mtr_pools_mng *pools_mng = 1639 &priv->sh->mtrmng->pools_mng; 1640 union mlx5_l3t_data data; 1641 1642 if (priv->sh->meter_aso_en) { 1643 rte_spinlock_lock(&pools_mng->mtrsl); 1644 if (!pools_mng->n_valid || !priv->mtr_idx_tbl) { 1645 rte_spinlock_unlock(&pools_mng->mtrsl); 1646 return NULL; 1647 } 1648 if (mlx5_l3t_get_entry(priv->mtr_idx_tbl, meter_id, &data) || 1649 !data.dword) { 1650 rte_spinlock_unlock(&pools_mng->mtrsl); 1651 return NULL; 1652 } 1653 if (mtr_idx) 1654 *mtr_idx = data.dword; 1655 aso_mtr = mlx5_aso_meter_by_idx(priv, data.dword); 1656 /* Remove reference taken by the mlx5_l3t_get_entry. */ 1657 mlx5_l3t_clear_entry(priv->mtr_idx_tbl, meter_id); 1658 rte_spinlock_unlock(&pools_mng->mtrsl); 1659 if (!aso_mtr || aso_mtr->state == ASO_METER_FREE) 1660 return NULL; 1661 return &aso_mtr->fm; 1662 } 1663 TAILQ_FOREACH(legacy_fm, fms, next) 1664 if (meter_id == legacy_fm->fm.meter_id) { 1665 if (mtr_idx) 1666 *mtr_idx = legacy_fm->idx; 1667 return &legacy_fm->fm; 1668 } 1669 return NULL; 1670 } 1671 1672 /** 1673 * Find meter by index. 1674 * 1675 * @param priv 1676 * Pointer to mlx5_priv. 1677 * @param idx 1678 * Meter index. 1679 * 1680 * @return 1681 * Pointer to the meter info found on success, NULL otherwise. 1682 */ 1683 struct mlx5_flow_meter_info * 1684 flow_dv_meter_find_by_idx(struct mlx5_priv *priv, uint32_t idx) 1685 { 1686 struct mlx5_aso_mtr *aso_mtr; 1687 1688 if (priv->sh->meter_aso_en) { 1689 aso_mtr = mlx5_aso_meter_by_idx(priv, idx); 1690 if (!aso_mtr) 1691 return NULL; 1692 return &aso_mtr->fm; 1693 } else { 1694 return mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], idx); 1695 } 1696 } 1697 1698 /** 1699 * Attach meter to flow. 1700 * Unidirectional Meter creation can only be done 1701 * when flow direction is known, i.e. when calling meter_attach. 1702 * 1703 * @param [in] priv 1704 * Pointer to mlx5 private data. 1705 * @param[in] fm 1706 * Pointer to flow meter. 1707 * @param [in] attr 1708 * Pointer to flow attributes. 1709 * @param [out] error 1710 * Pointer to error structure. 1711 * 1712 * @return 1713 * 0 on success, a negative errno value otherwise and rte_errno is set. 1714 */ 1715 int 1716 mlx5_flow_meter_attach(struct mlx5_priv *priv, 1717 struct mlx5_flow_meter_info *fm, 1718 const struct rte_flow_attr *attr, 1719 struct rte_flow_error *error) 1720 { 1721 int ret = 0; 1722 1723 if (priv->sh->meter_aso_en) { 1724 struct mlx5_aso_mtr *aso_mtr; 1725 1726 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm); 1727 if (mlx5_aso_mtr_wait(priv->sh, aso_mtr)) { 1728 return rte_flow_error_set(error, ENOENT, 1729 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1730 NULL, 1731 "Timeout in meter configuration"); 1732 } 1733 rte_spinlock_lock(&fm->sl); 1734 if (fm->shared || !fm->ref_cnt) { 1735 fm->ref_cnt++; 1736 } else { 1737 rte_flow_error_set(error, EINVAL, 1738 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 1739 "Meter cannot be shared"); 1740 ret = -1; 1741 } 1742 rte_spinlock_unlock(&fm->sl); 1743 } else { 1744 rte_spinlock_lock(&fm->sl); 1745 if (fm->meter_action) { 1746 if (fm->shared && 1747 attr->transfer == fm->transfer && 1748 attr->ingress == fm->ingress && 1749 attr->egress == fm->egress) { 1750 fm->ref_cnt++; 1751 } else { 1752 rte_flow_error_set(error, EINVAL, 1753 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 1754 fm->shared ? 1755 "Meter attr not match." : 1756 "Meter cannot be shared."); 1757 ret = -1; 1758 } 1759 } else { 1760 fm->ingress = attr->ingress; 1761 fm->egress = attr->egress; 1762 fm->transfer = attr->transfer; 1763 fm->ref_cnt = 1; 1764 /* This also creates the meter object. */ 1765 fm->meter_action = mlx5_flow_meter_action_create(priv, 1766 fm); 1767 if (!fm->meter_action) { 1768 fm->ref_cnt = 0; 1769 fm->ingress = 0; 1770 fm->egress = 0; 1771 fm->transfer = 0; 1772 rte_flow_error_set(error, EINVAL, 1773 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 1774 "Meter action create failed."); 1775 ret = -1; 1776 } 1777 } 1778 rte_spinlock_unlock(&fm->sl); 1779 } 1780 return ret ? -rte_errno : 0; 1781 } 1782 1783 /** 1784 * Detach meter from flow. 1785 * 1786 * @param [in] priv 1787 * Pointer to mlx5 private data. 1788 * @param [in] fm 1789 * Pointer to flow meter. 1790 */ 1791 void 1792 mlx5_flow_meter_detach(struct mlx5_priv *priv, 1793 struct mlx5_flow_meter_info *fm) 1794 { 1795 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER 1796 rte_spinlock_lock(&fm->sl); 1797 MLX5_ASSERT(fm->ref_cnt); 1798 if (--fm->ref_cnt == 0 && !priv->sh->meter_aso_en) { 1799 mlx5_glue->destroy_flow_action(fm->meter_action); 1800 fm->meter_action = NULL; 1801 fm->ingress = 0; 1802 fm->egress = 0; 1803 fm->transfer = 0; 1804 } 1805 rte_spinlock_unlock(&fm->sl); 1806 #else 1807 (void)priv; 1808 (void)fm; 1809 #endif 1810 } 1811 1812 /** 1813 * Flush meter with Rx queue configuration. 1814 * 1815 * @param[in] dev 1816 * Pointer to Ethernet device. 1817 */ 1818 void 1819 mlx5_flow_meter_rxq_flush(struct rte_eth_dev *dev) 1820 { 1821 struct mlx5_priv *priv = dev->data->dev_private; 1822 struct mlx5_flow_meter_sub_policy *sub_policy; 1823 struct mlx5_flow_meter_policy *mtr_policy; 1824 void *entry; 1825 uint32_t i, policy_idx; 1826 1827 if (!priv->mtr_en) 1828 return; 1829 if (priv->sh->mtrmng->policy_idx_tbl && priv->sh->refcnt == 1) { 1830 MLX5_L3T_FOREACH(priv->sh->mtrmng->policy_idx_tbl, 1831 i, entry) { 1832 policy_idx = *(uint32_t *)entry; 1833 sub_policy = mlx5_ipool_get 1834 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 1835 policy_idx); 1836 if (!sub_policy || !sub_policy->main_policy) 1837 continue; 1838 mtr_policy = sub_policy->main_policy; 1839 if (mtr_policy->is_queue || mtr_policy->is_rss) 1840 mlx5_flow_destroy_sub_policy_with_rxq(dev, 1841 mtr_policy); 1842 } 1843 } 1844 } 1845 1846 /** 1847 * Flush meter configuration. 1848 * 1849 * @param[in] dev 1850 * Pointer to Ethernet device. 1851 * @param[out] error 1852 * Pointer to rte meter error structure. 1853 * 1854 * @return 1855 * 0 on success, a negative errno value otherwise and rte_errno is set. 1856 */ 1857 int 1858 mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error) 1859 { 1860 struct mlx5_priv *priv = dev->data->dev_private; 1861 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; 1862 struct mlx5_mtr_profiles *fmps = &priv->flow_meter_profiles; 1863 struct mlx5_flow_meter_profile *fmp; 1864 struct mlx5_legacy_flow_meter *legacy_fm; 1865 struct mlx5_flow_meter_info *fm; 1866 struct mlx5_flow_meter_sub_policy *sub_policy; 1867 void *tmp; 1868 uint32_t i, mtr_idx, policy_idx; 1869 void *entry; 1870 struct mlx5_aso_mtr *aso_mtr; 1871 1872 if (!priv->mtr_en) 1873 return 0; 1874 if (priv->sh->meter_aso_en) { 1875 if (priv->mtr_idx_tbl) { 1876 MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) { 1877 mtr_idx = *(uint32_t *)entry; 1878 if (mtr_idx) { 1879 aso_mtr = 1880 mlx5_aso_meter_by_idx(priv, mtr_idx); 1881 fm = &aso_mtr->fm; 1882 (void)mlx5_flow_meter_params_flush(dev, 1883 fm, mtr_idx); 1884 } 1885 } 1886 mlx5_l3t_destroy(priv->mtr_idx_tbl); 1887 priv->mtr_idx_tbl = NULL; 1888 } 1889 } else { 1890 TAILQ_FOREACH_SAFE(legacy_fm, fms, next, tmp) { 1891 fm = &legacy_fm->fm; 1892 if (mlx5_flow_meter_params_flush(dev, fm, 0)) 1893 return -rte_mtr_error_set(error, EINVAL, 1894 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 1895 NULL, "MTR object meter profile invalid."); 1896 } 1897 } 1898 if (priv->sh->mtrmng->policy_idx_tbl && priv->sh->refcnt == 1) { 1899 MLX5_L3T_FOREACH(priv->sh->mtrmng->policy_idx_tbl, 1900 i, entry) { 1901 policy_idx = *(uint32_t *)entry; 1902 sub_policy = mlx5_ipool_get 1903 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 1904 policy_idx); 1905 if (!sub_policy) 1906 return -rte_mtr_error_set(error, 1907 EINVAL, 1908 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1909 NULL, "MTR object " 1910 "meter policy invalid."); 1911 if (__mlx5_flow_meter_policy_delete(dev, i, 1912 sub_policy->main_policy, 1913 error, true)) 1914 return -rte_mtr_error_set(error, 1915 EINVAL, 1916 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1917 NULL, "MTR object " 1918 "meter policy invalid."); 1919 mlx5_free(sub_policy->main_policy); 1920 } 1921 mlx5_l3t_destroy(priv->sh->mtrmng->policy_idx_tbl); 1922 priv->sh->mtrmng->policy_idx_tbl = NULL; 1923 } 1924 TAILQ_FOREACH_SAFE(fmp, fmps, next, tmp) { 1925 /* Check unused. */ 1926 MLX5_ASSERT(!fmp->ref_cnt); 1927 /* Remove from list. */ 1928 TAILQ_REMOVE(&priv->flow_meter_profiles, fmp, next); 1929 mlx5_free(fmp); 1930 } 1931 /* Delete default policy table. */ 1932 mlx5_flow_destroy_def_policy(dev); 1933 if (priv->sh->refcnt == 1) 1934 mlx5_flow_destroy_mtr_drop_tbls(dev); 1935 return 0; 1936 } 1937