1 // SPDX-License-Identifier: BSD-3-Clause 2 /* 3 * Copyright 2018 Mellanox Technologies, Ltd 4 */ 5 #include <math.h> 6 7 #include <rte_tailq.h> 8 #include <rte_malloc.h> 9 #include <rte_mtr.h> 10 #include <rte_mtr_driver.h> 11 12 #include <mlx5_devx_cmds.h> 13 #include <mlx5_malloc.h> 14 15 #include "mlx5.h" 16 #include "mlx5_flow.h" 17 18 /** 19 * Create the meter action. 20 * 21 * @param priv 22 * Pointer to mlx5_priv. 23 * @param[in] fm 24 * Pointer to flow meter to be converted. 25 * 26 * @return 27 * Pointer to the meter action on success, NULL otherwise. 28 */ 29 static void * 30 mlx5_flow_meter_action_create(struct mlx5_priv *priv, 31 struct mlx5_flow_meter_info *fm) 32 { 33 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER 34 struct mlx5dv_dr_flow_meter_attr mtr_init; 35 uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)]; 36 struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = 37 &fm->profile->srtcm_prm; 38 uint32_t cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir); 39 uint32_t ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir); 40 uint32_t val; 41 enum mlx5_meter_domain domain = 42 fm->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 43 fm->egress ? MLX5_MTR_DOMAIN_EGRESS : 44 MLX5_MTR_DOMAIN_INGRESS; 45 struct mlx5_flow_meter_def_policy *def_policy = 46 priv->sh->mtrmng->def_policy[domain]; 47 48 memset(fmp, 0, MLX5_ST_SZ_BYTES(flow_meter_parameters)); 49 MLX5_SET(flow_meter_parameters, fmp, valid, 1); 50 MLX5_SET(flow_meter_parameters, fmp, bucket_overflow, 1); 51 MLX5_SET(flow_meter_parameters, fmp, 52 start_color, MLX5_FLOW_COLOR_GREEN); 53 MLX5_SET(flow_meter_parameters, fmp, both_buckets_on_green, 0); 54 val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK; 55 MLX5_SET(flow_meter_parameters, fmp, cbs_exponent, val); 56 val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK; 57 MLX5_SET(flow_meter_parameters, fmp, cbs_mantissa, val); 58 val = (cbs_cir >> ASO_DSEG_CIR_EXP_OFFSET) & ASO_DSEG_EXP_MASK; 59 MLX5_SET(flow_meter_parameters, fmp, cir_exponent, val); 60 val = (cbs_cir & ASO_DSEG_MAN_MASK); 61 MLX5_SET(flow_meter_parameters, fmp, cir_mantissa, val); 62 val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK; 63 MLX5_SET(flow_meter_parameters, fmp, ebs_exponent, val); 64 val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK; 65 MLX5_SET(flow_meter_parameters, fmp, ebs_mantissa, val); 66 mtr_init.next_table = def_policy->sub_policy.tbl_rsc->obj; 67 mtr_init.reg_c_index = priv->mtr_color_reg - REG_C_0; 68 mtr_init.flow_meter_parameter = fmp; 69 mtr_init.flow_meter_parameter_sz = 70 MLX5_ST_SZ_BYTES(flow_meter_parameters); 71 mtr_init.active = fm->active_state; 72 return mlx5_glue->dv_create_flow_action_meter(&mtr_init); 73 #else 74 (void)priv; 75 (void)fm; 76 return NULL; 77 #endif 78 } 79 80 /** 81 * Find meter profile by id. 82 * 83 * @param priv 84 * Pointer to mlx5_priv. 85 * @param meter_profile_id 86 * Meter profile id. 87 * 88 * @return 89 * Pointer to the profile found on success, NULL otherwise. 90 */ 91 static struct mlx5_flow_meter_profile * 92 mlx5_flow_meter_profile_find(struct mlx5_priv *priv, uint32_t meter_profile_id) 93 { 94 struct mlx5_flow_meter_profile *fmp; 95 union mlx5_l3t_data data; 96 int32_t ret; 97 98 if (mlx5_l3t_get_entry(priv->mtr_profile_tbl, 99 meter_profile_id, &data) || !data.ptr) 100 return NULL; 101 fmp = data.ptr; 102 /* Remove reference taken by the mlx5_l3t_get_entry. */ 103 ret = mlx5_l3t_clear_entry(priv->mtr_profile_tbl, 104 meter_profile_id); 105 if (!ret || ret == -1) 106 return NULL; 107 return fmp; 108 } 109 110 /** 111 * Validate the MTR profile. 112 * 113 * @param[in] dev 114 * Pointer to Ethernet device. 115 * @param[in] meter_profile_id 116 * Meter profile id. 117 * @param[in] profile 118 * Pointer to meter profile detail. 119 * @param[out] error 120 * Pointer to the error structure. 121 * 122 * @return 123 * 0 on success, a negative errno value otherwise and rte_errno is set. 124 */ 125 static int 126 mlx5_flow_meter_profile_validate(struct rte_eth_dev *dev, 127 uint32_t meter_profile_id, 128 struct rte_mtr_meter_profile *profile, 129 struct rte_mtr_error *error) 130 { 131 struct mlx5_priv *priv = dev->data->dev_private; 132 struct mlx5_flow_meter_profile *fmp; 133 134 /* Profile must not be NULL. */ 135 if (profile == NULL) 136 return -rte_mtr_error_set(error, EINVAL, 137 RTE_MTR_ERROR_TYPE_METER_PROFILE, 138 NULL, "Meter profile is null."); 139 /* Meter profile ID must be valid. */ 140 if (meter_profile_id == UINT32_MAX) 141 return -rte_mtr_error_set(error, EINVAL, 142 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 143 NULL, "Meter profile id not valid."); 144 /* Meter profile must not exist. */ 145 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 146 if (fmp) 147 return -rte_mtr_error_set(error, EEXIST, 148 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 149 NULL, 150 "Meter profile already exists."); 151 if (profile->alg == RTE_MTR_SRTCM_RFC2697) { 152 if (priv->config.hca_attr.qos.flow_meter_old) { 153 /* Verify support for flow meter parameters. */ 154 if (priv->sh->meter_aso_en && profile->packet_mode) { 155 if (profile->srtcm_rfc2697.cir > 0 && 156 (profile->srtcm_rfc2697.cir << 157 MLX5_MTRS_PPS_MAP_BPS_SHIFT) 158 <= MLX5_SRTCM_CIR_MAX && 159 profile->srtcm_rfc2697.cbs > 0 && 160 (profile->srtcm_rfc2697.cbs << 161 MLX5_MTRS_PPS_MAP_BPS_SHIFT) 162 <= MLX5_SRTCM_CBS_MAX && 163 (profile->srtcm_rfc2697.ebs << 164 MLX5_MTRS_PPS_MAP_BPS_SHIFT) 165 <= MLX5_SRTCM_EBS_MAX) 166 return 0; 167 return -rte_mtr_error_set 168 (error, ENOTSUP, 169 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 170 NULL, 171 profile->srtcm_rfc2697.ebs ? 172 "Metering value ebs must be 0." : 173 "Invalid metering parameters."); 174 } 175 if (profile->srtcm_rfc2697.cir > 0 && 176 profile->srtcm_rfc2697.cir <= 177 MLX5_SRTCM_CIR_MAX && 178 profile->srtcm_rfc2697.cbs > 0 && 179 profile->srtcm_rfc2697.cbs <= 180 MLX5_SRTCM_CBS_MAX && 181 profile->srtcm_rfc2697.ebs <= 182 MLX5_SRTCM_EBS_MAX) 183 return 0; 184 return -rte_mtr_error_set(error, ENOTSUP, 185 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 186 NULL, 187 profile->srtcm_rfc2697.ebs ? 188 "Metering value ebs must be 0." : 189 "Invalid metering parameters."); 190 } 191 } 192 return -rte_mtr_error_set(error, ENOTSUP, 193 RTE_MTR_ERROR_TYPE_METER_PROFILE, 194 NULL, "Metering algorithm not supported."); 195 } 196 197 /** 198 * Calculate mantissa and exponent for cir. 199 * 200 * @param[in] cir 201 * Value to be calculated. 202 * @param[out] man 203 * Pointer to the mantissa. 204 * @param[out] exp 205 * Pointer to the exp. 206 */ 207 static void 208 mlx5_flow_meter_cir_man_exp_calc(int64_t cir, uint8_t *man, uint8_t *exp) 209 { 210 int64_t _cir; 211 int64_t delta = INT64_MAX; 212 uint8_t _man = 0; 213 uint8_t _exp = 0; 214 uint64_t m, e; 215 216 for (m = 0; m <= 0xFF; m++) { /* man width 8 bit */ 217 for (e = 0; e <= 0x1F; e++) { /* exp width 5bit */ 218 _cir = (1000000000ULL * m) >> e; 219 if (llabs(cir - _cir) <= delta) { 220 delta = llabs(cir - _cir); 221 _man = m; 222 _exp = e; 223 } 224 } 225 } 226 *man = _man; 227 *exp = _exp; 228 } 229 230 /** 231 * Calculate mantissa and exponent for xbs. 232 * 233 * @param[in] xbs 234 * Value to be calculated. 235 * @param[out] man 236 * Pointer to the mantissa. 237 * @param[out] exp 238 * Pointer to the exp. 239 */ 240 static void 241 mlx5_flow_meter_xbs_man_exp_calc(uint64_t xbs, uint8_t *man, uint8_t *exp) 242 { 243 int _exp; 244 double _man; 245 246 /* Special case xbs == 0 ? both exp and matissa are 0. */ 247 if (xbs == 0) { 248 *man = 0; 249 *exp = 0; 250 return; 251 } 252 /* xbs = xbs_mantissa * 2^xbs_exponent */ 253 _man = frexp(xbs, &_exp); 254 _man = _man * pow(2, MLX5_MAN_WIDTH); 255 _exp = _exp - MLX5_MAN_WIDTH; 256 *man = (uint8_t)ceil(_man); 257 *exp = _exp; 258 } 259 260 /** 261 * Fill the prm meter parameter. 262 * 263 * @param[in,out] fmp 264 * Pointer to meter profie to be converted. 265 * @param[out] error 266 * Pointer to the error structure. 267 * 268 * @return 269 * 0 on success, a negative errno value otherwise and rte_errno is set. 270 */ 271 static int 272 mlx5_flow_meter_param_fill(struct mlx5_flow_meter_profile *fmp, 273 struct mlx5_priv *priv, struct rte_mtr_error *error) 274 { 275 struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = &fmp->srtcm_prm; 276 uint8_t man, exp; 277 uint32_t cbs_exp, cbs_man, cir_exp, cir_man; 278 uint32_t ebs_exp, ebs_man; 279 uint64_t cir, cbs, ebs; 280 281 if (fmp->profile.alg != RTE_MTR_SRTCM_RFC2697) 282 return -rte_mtr_error_set(error, ENOTSUP, 283 RTE_MTR_ERROR_TYPE_METER_PROFILE, 284 NULL, "Metering algorithm not supported."); 285 if (!priv->sh->meter_aso_en && fmp->profile.packet_mode) 286 return -rte_mtr_error_set(error, ENOTSUP, 287 RTE_MTR_ERROR_TYPE_METER_PROFILE, 288 NULL, "Metering algorithm packet mode not supported."); 289 if (priv->sh->meter_aso_en && fmp->profile.packet_mode) { 290 cir = fmp->profile.srtcm_rfc2697.cir << 291 MLX5_MTRS_PPS_MAP_BPS_SHIFT; 292 cbs = fmp->profile.srtcm_rfc2697.cbs << 293 MLX5_MTRS_PPS_MAP_BPS_SHIFT; 294 ebs = fmp->profile.srtcm_rfc2697.ebs << 295 MLX5_MTRS_PPS_MAP_BPS_SHIFT; 296 } else { 297 cir = fmp->profile.srtcm_rfc2697.cir; 298 cbs = fmp->profile.srtcm_rfc2697.cbs; 299 ebs = fmp->profile.srtcm_rfc2697.ebs; 300 } 301 /* cir = 8G * cir_mantissa * 1/(2^cir_exponent)) Bytes/Sec */ 302 mlx5_flow_meter_cir_man_exp_calc(cir, &man, &exp); 303 /* Check if cir mantissa is too large. */ 304 if (exp > ASO_DSEG_CIR_EXP_MASK) 305 return -rte_mtr_error_set(error, ENOTSUP, 306 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 307 "meter profile parameter cir is" 308 " not supported."); 309 cir_man = man; 310 cir_exp = exp; 311 /* cbs = cbs_mantissa * 2^cbs_exponent */ 312 mlx5_flow_meter_xbs_man_exp_calc(cbs, &man, &exp); 313 /* Check if cbs mantissa is too large. */ 314 if (exp > ASO_DSEG_EXP_MASK) 315 return -rte_mtr_error_set(error, ENOTSUP, 316 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 317 "meter profile parameter cbs is" 318 " not supported."); 319 cbs_man = man; 320 cbs_exp = exp; 321 srtcm->cbs_cir = rte_cpu_to_be_32(cbs_exp << ASO_DSEG_CBS_EXP_OFFSET | 322 cbs_man << ASO_DSEG_CBS_MAN_OFFSET | 323 cir_exp << ASO_DSEG_CIR_EXP_OFFSET | 324 cir_man); 325 mlx5_flow_meter_xbs_man_exp_calc(ebs, &man, &exp); 326 /* Check if ebs mantissa is too large. */ 327 if (exp > ASO_DSEG_EXP_MASK) 328 return -rte_mtr_error_set(error, ENOTSUP, 329 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 330 "meter profile parameter ebs is" 331 " not supported."); 332 ebs_man = man; 333 ebs_exp = exp; 334 srtcm->ebs_eir = rte_cpu_to_be_32(ebs_exp << ASO_DSEG_EBS_EXP_OFFSET | 335 ebs_man << ASO_DSEG_EBS_MAN_OFFSET); 336 return 0; 337 } 338 339 /** 340 * Callback to get MTR capabilities. 341 * 342 * @param[in] dev 343 * Pointer to Ethernet device. 344 * @param[out] cap 345 * Pointer to save MTR capabilities. 346 * @param[out] error 347 * Pointer to the error structure. 348 * 349 * @return 350 * 0 on success, a negative errno value otherwise and rte_errno is set. 351 */ 352 static int 353 mlx5_flow_mtr_cap_get(struct rte_eth_dev *dev, 354 struct rte_mtr_capabilities *cap, 355 struct rte_mtr_error *error __rte_unused) 356 { 357 struct mlx5_priv *priv = dev->data->dev_private; 358 struct mlx5_hca_qos_attr *qattr = &priv->config.hca_attr.qos; 359 360 if (!priv->mtr_en) 361 return -rte_mtr_error_set(error, ENOTSUP, 362 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 363 "Meter is not supported"); 364 memset(cap, 0, sizeof(*cap)); 365 if (priv->sh->meter_aso_en) { 366 /* 2 meters per one ASO cache line. */ 367 cap->n_max = 1 << (qattr->log_max_num_meter_aso + 1); 368 cap->srtcm_rfc2697_packet_mode_supported = 1; 369 } else { 370 cap->n_max = 1 << qattr->log_max_flow_meter; 371 cap->srtcm_rfc2697_packet_mode_supported = 0; 372 } 373 cap->srtcm_rfc2697_byte_mode_supported = 1; 374 cap->n_shared_max = cap->n_max; 375 cap->identical = 1; 376 cap->shared_identical = 1; 377 cap->shared_n_flows_per_mtr_max = 4 << 20; 378 /* 2M flows can share the same meter. */ 379 cap->chaining_n_mtrs_per_flow_max = 1; /* Chaining is not supported. */ 380 cap->meter_srtcm_rfc2697_n_max = qattr->flow_meter_old ? cap->n_max : 0; 381 cap->meter_rate_max = 1ULL << 40; /* 1 Tera tokens per sec. */ 382 cap->stats_mask = RTE_MTR_STATS_N_BYTES_DROPPED | 383 RTE_MTR_STATS_N_PKTS_DROPPED; 384 return 0; 385 } 386 387 /** 388 * Callback to add MTR profile. 389 * 390 * @param[in] dev 391 * Pointer to Ethernet device. 392 * @param[in] meter_profile_id 393 * Meter profile id. 394 * @param[in] profile 395 * Pointer to meter profile detail. 396 * @param[out] error 397 * Pointer to the error structure. 398 * 399 * @return 400 * 0 on success, a negative errno value otherwise and rte_errno is set. 401 */ 402 static int 403 mlx5_flow_meter_profile_add(struct rte_eth_dev *dev, 404 uint32_t meter_profile_id, 405 struct rte_mtr_meter_profile *profile, 406 struct rte_mtr_error *error) 407 { 408 struct mlx5_priv *priv = dev->data->dev_private; 409 struct mlx5_flow_meter_profile *fmp; 410 union mlx5_l3t_data data; 411 int ret; 412 413 if (!priv->mtr_en) 414 return -rte_mtr_error_set(error, ENOTSUP, 415 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 416 "Meter is not supported"); 417 /* Check input params. */ 418 ret = mlx5_flow_meter_profile_validate(dev, meter_profile_id, 419 profile, error); 420 if (ret) 421 return ret; 422 /* Meter profile memory allocation. */ 423 fmp = mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct mlx5_flow_meter_profile), 424 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 425 if (fmp == NULL) 426 return -rte_mtr_error_set(error, ENOMEM, 427 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 428 NULL, "Meter profile memory " 429 "alloc failed."); 430 /* Fill profile info. */ 431 fmp->id = meter_profile_id; 432 fmp->profile = *profile; 433 /* Fill the flow meter parameters for the PRM. */ 434 ret = mlx5_flow_meter_param_fill(fmp, priv, error); 435 if (ret) 436 goto error; 437 data.ptr = fmp; 438 ret = mlx5_l3t_set_entry(priv->mtr_profile_tbl, 439 meter_profile_id, &data); 440 if (ret) 441 return -rte_mtr_error_set(error, ENOTSUP, 442 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 443 NULL, "Meter profile insert fail."); 444 return 0; 445 error: 446 mlx5_free(fmp); 447 return ret; 448 } 449 450 /** 451 * Callback to delete MTR profile. 452 * 453 * @param[in] dev 454 * Pointer to Ethernet device. 455 * @param[in] meter_profile_id 456 * Meter profile id. 457 * @param[out] error 458 * Pointer to the error structure. 459 * 460 * @return 461 * 0 on success, a negative errno value otherwise and rte_errno is set. 462 */ 463 static int 464 mlx5_flow_meter_profile_delete(struct rte_eth_dev *dev, 465 uint32_t meter_profile_id, 466 struct rte_mtr_error *error) 467 { 468 struct mlx5_priv *priv = dev->data->dev_private; 469 struct mlx5_flow_meter_profile *fmp; 470 471 if (!priv->mtr_en) 472 return -rte_mtr_error_set(error, ENOTSUP, 473 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 474 "Meter is not supported"); 475 /* Meter profile must exist. */ 476 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 477 if (fmp == NULL) 478 return -rte_mtr_error_set(error, ENOENT, 479 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 480 &meter_profile_id, 481 "Meter profile id is invalid."); 482 /* Check profile is unused. */ 483 if (fmp->ref_cnt) 484 return -rte_mtr_error_set(error, EBUSY, 485 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 486 NULL, "Meter profile is in use."); 487 if (mlx5_l3t_clear_entry(priv->mtr_profile_tbl, meter_profile_id)) 488 return -rte_mtr_error_set(error, EBUSY, 489 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 490 NULL, "Meter profile remove fail."); 491 mlx5_free(fmp); 492 return 0; 493 } 494 495 /** 496 * Find policy by id. 497 * 498 * @param[in] dev 499 * Pointer to Ethernet device. 500 * @param policy_id 501 * Policy id. 502 * 503 * @return 504 * Pointer to the policy found on success, NULL otherwise. 505 */ 506 struct mlx5_flow_meter_policy * 507 mlx5_flow_meter_policy_find(struct rte_eth_dev *dev, 508 uint32_t policy_id, 509 uint32_t *policy_idx) 510 { 511 struct mlx5_priv *priv = dev->data->dev_private; 512 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 513 union mlx5_l3t_data data; 514 515 if (policy_id > MLX5_MAX_SUB_POLICY_TBL_NUM || !priv->policy_idx_tbl) 516 return NULL; 517 if (mlx5_l3t_get_entry(priv->policy_idx_tbl, policy_id, &data) || 518 !data.dword) 519 return NULL; 520 if (policy_idx) 521 *policy_idx = data.dword; 522 sub_policy = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 523 data.dword); 524 /* Remove reference taken by the mlx5_l3t_get_entry. */ 525 mlx5_l3t_clear_entry(priv->policy_idx_tbl, policy_id); 526 if (sub_policy) 527 if (sub_policy->main_policy_id) 528 return sub_policy->main_policy; 529 return NULL; 530 } 531 532 /** 533 * Get the last meter's policy from one meter's policy in hierarchy. 534 * 535 * @param[in] dev 536 * Pointer to Ethernet device. 537 * @param[in] policy 538 * Pointer to flow meter policy. 539 * 540 * @return 541 * Pointer to the final meter's policy, or NULL when fail. 542 */ 543 struct mlx5_flow_meter_policy * 544 mlx5_flow_meter_hierarchy_get_final_policy(struct rte_eth_dev *dev, 545 struct mlx5_flow_meter_policy *policy) 546 { 547 struct mlx5_priv *priv = dev->data->dev_private; 548 struct mlx5_flow_meter_info *next_fm; 549 struct mlx5_flow_meter_policy *next_policy = policy; 550 551 while (next_policy->is_hierarchy) { 552 next_fm = mlx5_flow_meter_find(priv, 553 next_policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id, NULL); 554 if (!next_fm || next_fm->def_policy) 555 return NULL; 556 next_policy = mlx5_flow_meter_policy_find(dev, 557 next_fm->policy_id, NULL); 558 MLX5_ASSERT(next_policy); 559 } 560 return next_policy; 561 } 562 563 /** 564 * Callback to check MTR policy action validate 565 * 566 * @param[in] dev 567 * Pointer to Ethernet device. 568 * @param[in] actions 569 * Pointer to meter policy action detail. 570 * @param[out] error 571 * Pointer to the error structure. 572 * 573 * @return 574 * 0 on success, a negative errno value otherwise and rte_errno is set. 575 */ 576 static int 577 mlx5_flow_meter_policy_validate(struct rte_eth_dev *dev, 578 struct rte_mtr_meter_policy_params *policy, 579 struct rte_mtr_error *error) 580 { 581 struct mlx5_priv *priv = dev->data->dev_private; 582 struct rte_flow_attr attr = { .transfer = 583 priv->config.dv_esw_en ? 1 : 0}; 584 bool is_rss = false; 585 bool is_def_policy = false; 586 uint8_t domain_bitmap; 587 int ret; 588 589 if (!priv->mtr_en || !priv->sh->meter_aso_en) 590 return -rte_mtr_error_set(error, ENOTSUP, 591 RTE_MTR_ERROR_TYPE_METER_POLICY, 592 NULL, "meter policy unsupported."); 593 ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr, 594 &is_rss, &domain_bitmap, &is_def_policy, error); 595 if (ret) 596 return ret; 597 return 0; 598 } 599 600 static int 601 __mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev, 602 uint32_t policy_id, 603 struct mlx5_flow_meter_policy *mtr_policy, 604 struct rte_mtr_error *error, 605 bool clear_l3t) 606 { 607 struct mlx5_priv *priv = dev->data->dev_private; 608 struct mlx5_flow_meter_sub_policy *sub_policy; 609 uint32_t i, j; 610 uint16_t sub_policy_num; 611 612 rte_spinlock_lock(&mtr_policy->sl); 613 if (mtr_policy->ref_cnt) { 614 rte_spinlock_unlock(&mtr_policy->sl); 615 return -rte_mtr_error_set(error, EBUSY, 616 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 617 NULL, 618 "Meter policy object is being used."); 619 } 620 mlx5_flow_destroy_policy_rules(dev, mtr_policy); 621 mlx5_flow_destroy_mtr_acts(dev, mtr_policy); 622 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 623 sub_policy_num = (mtr_policy->sub_policy_num >> 624 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) & 625 MLX5_MTR_SUB_POLICY_NUM_MASK; 626 if (sub_policy_num) { 627 for (j = 0; j < sub_policy_num; j++) { 628 sub_policy = mtr_policy->sub_policys[i][j]; 629 if (sub_policy) 630 mlx5_ipool_free 631 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 632 sub_policy->idx); 633 } 634 } 635 } 636 if (priv->policy_idx_tbl && clear_l3t) { 637 if (mlx5_l3t_clear_entry(priv->policy_idx_tbl, policy_id)) { 638 rte_spinlock_unlock(&mtr_policy->sl); 639 return -rte_mtr_error_set(error, ENOTSUP, 640 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL, 641 "Fail to delete policy in index table."); 642 } 643 } 644 rte_spinlock_unlock(&mtr_policy->sl); 645 return 0; 646 } 647 648 /** 649 * Callback to add MTR policy. 650 * 651 * @param[in] dev 652 * Pointer to Ethernet device. 653 * @param[out] policy_id 654 * Pointer to policy id 655 * @param[in] actions 656 * Pointer to meter policy action detail. 657 * @param[out] error 658 * Pointer to the error structure. 659 * 660 * @return 661 * 0 on success, a negative errno value otherwise and rte_errno is set. 662 */ 663 static int 664 mlx5_flow_meter_policy_add(struct rte_eth_dev *dev, 665 uint32_t policy_id, 666 struct rte_mtr_meter_policy_params *policy, 667 struct rte_mtr_error *error) 668 { 669 struct mlx5_priv *priv = dev->data->dev_private; 670 struct rte_flow_attr attr = { .transfer = 671 priv->config.dv_esw_en ? 1 : 0}; 672 uint32_t sub_policy_idx = 0; 673 uint32_t policy_idx = 0; 674 struct mlx5_flow_meter_policy *mtr_policy = NULL; 675 struct mlx5_flow_meter_sub_policy *sub_policy; 676 bool is_rss = false; 677 bool is_def_policy = false; 678 uint32_t i; 679 int ret; 680 uint32_t policy_size = sizeof(struct mlx5_flow_meter_policy); 681 uint16_t sub_policy_num; 682 uint8_t domain_bitmap = 0; 683 union mlx5_l3t_data data; 684 bool skip_rule = false; 685 686 if (!priv->mtr_en) 687 return -rte_mtr_error_set(error, ENOTSUP, 688 RTE_MTR_ERROR_TYPE_METER_POLICY, 689 NULL, "meter policy unsupported."); 690 if (policy_id == MLX5_INVALID_POLICY_ID) 691 return -rte_mtr_error_set(error, ENOTSUP, 692 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL, 693 "policy ID is invalid. "); 694 if (policy_id == priv->sh->mtrmng->def_policy_id) 695 return -rte_mtr_error_set(error, EEXIST, 696 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL, 697 "policy ID exists. "); 698 mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, 699 &policy_idx); 700 if (mtr_policy) 701 return -rte_mtr_error_set(error, EEXIST, 702 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL, 703 "policy ID exists. "); 704 ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr, 705 &is_rss, &domain_bitmap, &is_def_policy, error); 706 if (ret) 707 return ret; 708 if (!domain_bitmap) 709 return -rte_mtr_error_set(error, ENOTSUP, 710 RTE_MTR_ERROR_TYPE_METER_POLICY, 711 NULL, "fail to find policy domain."); 712 if (is_def_policy) { 713 if (priv->sh->mtrmng->def_policy_id != MLX5_INVALID_POLICY_ID) 714 return -rte_mtr_error_set(error, EEXIST, 715 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 716 NULL, "a policy with similar actions " 717 "is already configured"); 718 if (mlx5_flow_create_def_policy(dev)) 719 return -rte_mtr_error_set(error, ENOTSUP, 720 RTE_MTR_ERROR_TYPE_METER_POLICY, 721 NULL, 722 "fail to create non-terminated policy."); 723 priv->sh->mtrmng->def_policy_id = policy_id; 724 return 0; 725 } 726 if (!priv->sh->meter_aso_en) 727 return -rte_mtr_error_set(error, ENOTSUP, 728 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL, 729 "no ASO capability to support the policy "); 730 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 731 if (!(domain_bitmap & (1 << i))) 732 continue; 733 if (is_rss) { 734 policy_size += 735 sizeof(struct mlx5_flow_meter_sub_policy *) * 736 MLX5_MTR_RSS_MAX_SUB_POLICY; 737 break; 738 } 739 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *); 740 } 741 mtr_policy = mlx5_malloc(MLX5_MEM_ZERO, policy_size, 742 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 743 if (!mtr_policy) 744 return -rte_mtr_error_set(error, ENOMEM, 745 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL, 746 "Memory alloc failed for meter policy."); 747 policy_size = sizeof(struct mlx5_flow_meter_policy); 748 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 749 if (!(domain_bitmap & (1 << i))) 750 continue; 751 if (i == MLX5_MTR_DOMAIN_INGRESS) 752 mtr_policy->ingress = 1; 753 if (i == MLX5_MTR_DOMAIN_EGRESS) 754 mtr_policy->egress = 1; 755 if (i == MLX5_MTR_DOMAIN_TRANSFER) 756 mtr_policy->transfer = 1; 757 sub_policy = mlx5_ipool_zmalloc 758 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 759 &sub_policy_idx); 760 if (!sub_policy) 761 goto policy_add_err; 762 if (sub_policy_idx > MLX5_MAX_SUB_POLICY_TBL_NUM) 763 goto policy_add_err; 764 sub_policy->idx = sub_policy_idx; 765 sub_policy->main_policy = mtr_policy; 766 if (!policy_idx) { 767 policy_idx = sub_policy_idx; 768 sub_policy->main_policy_id = 1; 769 } 770 mtr_policy->sub_policys[i] = 771 (struct mlx5_flow_meter_sub_policy **) 772 ((uint8_t *)mtr_policy + policy_size); 773 mtr_policy->sub_policys[i][0] = sub_policy; 774 sub_policy_num = (mtr_policy->sub_policy_num >> 775 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) & 776 MLX5_MTR_SUB_POLICY_NUM_MASK; 777 sub_policy_num++; 778 mtr_policy->sub_policy_num &= ~(MLX5_MTR_SUB_POLICY_NUM_MASK << 779 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)); 780 mtr_policy->sub_policy_num |= 781 (sub_policy_num & MLX5_MTR_SUB_POLICY_NUM_MASK) << 782 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i); 783 if (is_rss) { 784 mtr_policy->is_rss = 1; 785 break; 786 } 787 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *); 788 } 789 rte_spinlock_init(&mtr_policy->sl); 790 ret = mlx5_flow_create_mtr_acts(dev, mtr_policy, 791 policy->actions, error); 792 if (ret) 793 goto policy_add_err; 794 if (mtr_policy->is_hierarchy) { 795 struct mlx5_flow_meter_policy *final_policy; 796 797 final_policy = 798 mlx5_flow_meter_hierarchy_get_final_policy(dev, mtr_policy); 799 if (!final_policy) 800 goto policy_add_err; 801 skip_rule = (final_policy->is_rss || final_policy->is_queue); 802 } 803 if (!is_rss && !mtr_policy->is_queue && !skip_rule) { 804 /* Create policy rules in HW. */ 805 ret = mlx5_flow_create_policy_rules(dev, mtr_policy); 806 if (ret) 807 goto policy_add_err; 808 } 809 data.dword = policy_idx; 810 if (!priv->policy_idx_tbl) { 811 priv->policy_idx_tbl = mlx5_l3t_create(MLX5_L3T_TYPE_DWORD); 812 if (!priv->policy_idx_tbl) 813 goto policy_add_err; 814 } 815 if (mlx5_l3t_set_entry(priv->policy_idx_tbl, policy_id, &data)) 816 goto policy_add_err; 817 return 0; 818 policy_add_err: 819 if (mtr_policy) { 820 ret = __mlx5_flow_meter_policy_delete(dev, policy_id, 821 mtr_policy, error, false); 822 mlx5_free(mtr_policy); 823 if (ret) 824 return ret; 825 } 826 return -rte_mtr_error_set(error, ENOTSUP, 827 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 828 NULL, "Failed to create devx policy."); 829 } 830 831 /** 832 * Callback to delete MTR policy. 833 * 834 * @param[in] dev 835 * Pointer to Ethernet device. 836 * @param[in] policy_id 837 * Meter policy id. 838 * @param[out] error 839 * Pointer to the error structure. 840 * 841 * @return 842 * 0 on success, a negative errno value otherwise and rte_errno is set. 843 */ 844 static int 845 mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev, 846 uint32_t policy_id, 847 struct rte_mtr_error *error) 848 { 849 struct mlx5_priv *priv = dev->data->dev_private; 850 struct mlx5_flow_meter_policy *mtr_policy; 851 uint32_t policy_idx; 852 int ret; 853 854 if (policy_id == priv->sh->mtrmng->def_policy_id) { 855 if (priv->sh->mtrmng->def_policy_ref_cnt > 0) 856 return -rte_mtr_error_set(error, ENOTSUP, 857 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL, 858 "Meter policy object is being used."); 859 priv->sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID; 860 return 0; 861 } 862 mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx); 863 if (!mtr_policy) 864 return -rte_mtr_error_set(error, ENOTSUP, 865 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL, 866 "Meter policy id is invalid. "); 867 ret = __mlx5_flow_meter_policy_delete(dev, policy_id, mtr_policy, 868 error, true); 869 if (ret) 870 return ret; 871 mlx5_free(mtr_policy); 872 return 0; 873 } 874 875 /** 876 * Check meter validation. 877 * 878 * @param[in] priv 879 * Pointer to mlx5 private data structure. 880 * @param[in] meter_id 881 * Meter id. 882 * @param[in] params 883 * Pointer to rte meter parameters. 884 * @param[out] error 885 * Pointer to rte meter error structure. 886 * 887 * @return 888 * 0 on success, a negative errno value otherwise and rte_errno is set. 889 */ 890 static int 891 mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id, 892 struct rte_mtr_params *params, 893 struct rte_mtr_error *error) 894 { 895 /* Meter must use global drop action. */ 896 if (!priv->sh->dr_drop_action) 897 return -rte_mtr_error_set(error, ENOTSUP, 898 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 899 NULL, 900 "No drop action ready for meter."); 901 /* Meter params must not be NULL. */ 902 if (params == NULL) 903 return -rte_mtr_error_set(error, EINVAL, 904 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 905 NULL, "Meter object params null."); 906 /* Previous meter color is not supported. */ 907 if (params->use_prev_mtr_color) 908 return -rte_mtr_error_set(error, ENOTSUP, 909 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 910 NULL, 911 "Previous meter color " 912 "not supported."); 913 if (params->meter_policy_id == MLX5_INVALID_POLICY_ID) 914 return -rte_mtr_error_set(error, ENOENT, 915 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 916 NULL, "Meter policy id not valid."); 917 /* Validate meter id. */ 918 if (mlx5_flow_meter_find(priv, meter_id, NULL)) 919 return -rte_mtr_error_set(error, EEXIST, 920 RTE_MTR_ERROR_TYPE_MTR_ID, NULL, 921 "Meter object already exists."); 922 return 0; 923 } 924 925 /** 926 * Modify the flow meter action. 927 * 928 * @param[in] priv 929 * Pointer to mlx5 private data structure. 930 * @param[in] fm 931 * Pointer to flow meter to be modified. 932 * @param[in] srtcm 933 * Pointer to meter srtcm description parameter. 934 * @param[in] modify_bits 935 * The bit in srtcm to be updated. 936 * @param[in] active_state 937 * The state to be updated. 938 * @return 939 * 0 on success, o negative value otherwise. 940 */ 941 static int 942 mlx5_flow_meter_action_modify(struct mlx5_priv *priv, 943 struct mlx5_flow_meter_info *fm, 944 const struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm, 945 uint64_t modify_bits, uint32_t active_state, uint32_t is_enable) 946 { 947 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER 948 uint32_t in[MLX5_ST_SZ_DW(flow_meter_parameters)] = { 0 }; 949 uint32_t *attr; 950 struct mlx5dv_dr_flow_meter_attr mod_attr = { 0 }; 951 int ret; 952 struct mlx5_aso_mtr *aso_mtr = NULL; 953 uint32_t cbs_cir, ebs_eir, val; 954 955 if (priv->sh->meter_aso_en) { 956 fm->is_enable = !!is_enable; 957 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm); 958 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr); 959 if (ret) 960 return ret; 961 ret = mlx5_aso_mtr_wait(priv->sh, aso_mtr); 962 if (ret) 963 return ret; 964 } else { 965 /* Fill command parameters. */ 966 mod_attr.reg_c_index = priv->mtr_color_reg - REG_C_0; 967 mod_attr.flow_meter_parameter = in; 968 mod_attr.flow_meter_parameter_sz = 969 MLX5_ST_SZ_BYTES(flow_meter_parameters); 970 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE) 971 mod_attr.active = !!active_state; 972 else 973 mod_attr.active = 0; 974 attr = in; 975 cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir); 976 ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir); 977 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS) { 978 val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & 979 ASO_DSEG_EXP_MASK; 980 MLX5_SET(flow_meter_parameters, attr, 981 cbs_exponent, val); 982 val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & 983 ASO_DSEG_MAN_MASK; 984 MLX5_SET(flow_meter_parameters, attr, 985 cbs_mantissa, val); 986 } 987 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR) { 988 val = (cbs_cir >> ASO_DSEG_CIR_EXP_OFFSET) & 989 ASO_DSEG_EXP_MASK; 990 MLX5_SET(flow_meter_parameters, attr, 991 cir_exponent, val); 992 val = cbs_cir & ASO_DSEG_MAN_MASK; 993 MLX5_SET(flow_meter_parameters, attr, 994 cir_mantissa, val); 995 } 996 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_EBS) { 997 val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & 998 ASO_DSEG_EXP_MASK; 999 MLX5_SET(flow_meter_parameters, attr, 1000 ebs_exponent, val); 1001 val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & 1002 ASO_DSEG_MAN_MASK; 1003 MLX5_SET(flow_meter_parameters, attr, 1004 ebs_mantissa, val); 1005 } 1006 /* Apply modifications to meter only if it was created. */ 1007 if (fm->meter_action) { 1008 ret = mlx5_glue->dv_modify_flow_action_meter 1009 (fm->meter_action, &mod_attr, 1010 rte_cpu_to_be_64(modify_bits)); 1011 if (ret) 1012 return ret; 1013 } 1014 /* Update succeedded modify meter parameters. */ 1015 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE) 1016 fm->active_state = !!active_state; 1017 } 1018 return 0; 1019 #else 1020 (void)priv; 1021 (void)fm; 1022 (void)srtcm; 1023 (void)modify_bits; 1024 (void)active_state; 1025 (void)is_enable; 1026 return -ENOTSUP; 1027 #endif 1028 } 1029 1030 static int 1031 mlx5_flow_meter_stats_enable_update(struct rte_eth_dev *dev, 1032 struct mlx5_flow_meter_info *fm, 1033 uint64_t stats_mask) 1034 { 1035 fm->bytes_dropped = 1036 (stats_mask & RTE_MTR_STATS_N_BYTES_DROPPED) ? 1 : 0; 1037 fm->pkts_dropped = (stats_mask & RTE_MTR_STATS_N_PKTS_DROPPED) ? 1 : 0; 1038 if (fm->bytes_dropped || fm->pkts_dropped) { 1039 if (!fm->drop_cnt) { 1040 /* Alloc policer counters. */ 1041 fm->drop_cnt = mlx5_counter_alloc(dev); 1042 if (!fm->drop_cnt) 1043 return -1; 1044 } 1045 } else { 1046 if (fm->drop_cnt) { 1047 mlx5_counter_free(dev, fm->drop_cnt); 1048 fm->drop_cnt = 0; 1049 } 1050 } 1051 return 0; 1052 } 1053 1054 /** 1055 * Create meter rules. 1056 * 1057 * @param[in] dev 1058 * Pointer to Ethernet device. 1059 * @param[in] meter_id 1060 * Meter id. 1061 * @param[in] params 1062 * Pointer to rte meter parameters. 1063 * @param[in] shared 1064 * Meter shared with other flow or not. 1065 * @param[out] error 1066 * Pointer to rte meter error structure. 1067 * 1068 * @return 1069 * 0 on success, a negative errno value otherwise and rte_errno is set. 1070 */ 1071 static int 1072 mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, 1073 struct rte_mtr_params *params, int shared, 1074 struct rte_mtr_error *error) 1075 { 1076 struct mlx5_priv *priv = dev->data->dev_private; 1077 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; 1078 struct mlx5_flow_meter_profile *fmp; 1079 struct mlx5_flow_meter_info *fm; 1080 struct mlx5_legacy_flow_meter *legacy_fm; 1081 struct mlx5_flow_meter_policy *mtr_policy = NULL; 1082 struct mlx5_indexed_pool_config flow_ipool_cfg = { 1083 .size = 0, 1084 .trunk_size = 64, 1085 .need_lock = 1, 1086 .type = "mlx5_flow_mtr_flow_id_pool", 1087 }; 1088 struct mlx5_aso_mtr *aso_mtr; 1089 uint32_t mtr_idx, policy_idx; 1090 union mlx5_l3t_data data; 1091 int ret; 1092 uint8_t domain_bitmap; 1093 uint8_t mtr_id_bits; 1094 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 1095 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 1096 1097 if (!priv->mtr_en) 1098 return -rte_mtr_error_set(error, ENOTSUP, 1099 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1100 "Meter is not supported"); 1101 /* Validate the parameters. */ 1102 ret = mlx5_flow_meter_validate(priv, meter_id, params, error); 1103 if (ret) 1104 return ret; 1105 /* Meter profile must exist. */ 1106 fmp = mlx5_flow_meter_profile_find(priv, params->meter_profile_id); 1107 if (fmp == NULL) 1108 return -rte_mtr_error_set(error, ENOENT, 1109 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 1110 NULL, "Meter profile id not valid."); 1111 /* Meter policy must exist. */ 1112 if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) { 1113 __atomic_add_fetch 1114 (&priv->sh->mtrmng->def_policy_ref_cnt, 1115 1, __ATOMIC_RELAXED); 1116 domain_bitmap = MLX5_MTR_ALL_DOMAIN_BIT; 1117 if (!priv->config.dv_esw_en) 1118 domain_bitmap &= ~MLX5_MTR_DOMAIN_TRANSFER_BIT; 1119 } else { 1120 mtr_policy = mlx5_flow_meter_policy_find(dev, 1121 params->meter_policy_id, &policy_idx); 1122 if (!priv->sh->meter_aso_en) 1123 return -rte_mtr_error_set(error, ENOTSUP, 1124 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1125 "Part of the policies cannot be " 1126 "supported without ASO "); 1127 if (!mtr_policy) 1128 return -rte_mtr_error_set(error, ENOENT, 1129 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1130 NULL, "Meter policy id not valid."); 1131 domain_bitmap = (mtr_policy->ingress ? 1132 MLX5_MTR_DOMAIN_INGRESS_BIT : 0) | 1133 (mtr_policy->egress ? 1134 MLX5_MTR_DOMAIN_EGRESS_BIT : 0) | 1135 (mtr_policy->transfer ? 1136 MLX5_MTR_DOMAIN_TRANSFER_BIT : 0); 1137 } 1138 /* Allocate the flow meter memory. */ 1139 if (priv->sh->meter_aso_en) { 1140 mtr_idx = mlx5_flow_mtr_alloc(dev); 1141 if (!mtr_idx) 1142 return -rte_mtr_error_set(error, ENOMEM, 1143 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1144 "Memory alloc failed for meter."); 1145 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx); 1146 fm = &aso_mtr->fm; 1147 } else { 1148 legacy_fm = mlx5_ipool_zmalloc 1149 (priv->sh->ipool[MLX5_IPOOL_MTR], &mtr_idx); 1150 if (legacy_fm == NULL) 1151 return -rte_mtr_error_set(error, ENOMEM, 1152 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1153 "Memory alloc failed for meter."); 1154 legacy_fm->idx = mtr_idx; 1155 fm = &legacy_fm->fm; 1156 } 1157 mtr_id_bits = MLX5_REG_BITS - __builtin_clz(mtr_idx); 1158 if ((mtr_id_bits + priv->sh->mtrmng->max_mtr_flow_bits) > 1159 mtr_reg_bits) { 1160 DRV_LOG(ERR, "Meter number exceeds max limit."); 1161 goto error; 1162 } 1163 if (mtr_id_bits > priv->sh->mtrmng->max_mtr_bits) 1164 priv->sh->mtrmng->max_mtr_bits = mtr_id_bits; 1165 /* Fill the flow meter parameters. */ 1166 fm->meter_id = meter_id; 1167 fm->policy_id = params->meter_policy_id; 1168 fm->profile = fmp; 1169 if (mlx5_flow_meter_stats_enable_update(dev, fm, params->stats_mask)) 1170 goto error; 1171 if (mlx5_flow_create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap)) 1172 goto error; 1173 /* Add to the flow meter list. */ 1174 if (!priv->sh->meter_aso_en) 1175 TAILQ_INSERT_TAIL(fms, legacy_fm, next); 1176 /* Add to the flow meter list. */ 1177 fm->active_state = 1; /* Config meter starts as active. */ 1178 fm->is_enable = 1; 1179 fm->shared = !!shared; 1180 __atomic_add_fetch(&fm->profile->ref_cnt, 1, __ATOMIC_RELAXED); 1181 if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) { 1182 fm->def_policy = 1; 1183 fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg); 1184 if (!fm->flow_ipool) 1185 goto error; 1186 } 1187 rte_spinlock_init(&fm->sl); 1188 /* If ASO meter supported, update ASO flow meter by wqe. */ 1189 if (priv->sh->meter_aso_en) { 1190 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm); 1191 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr); 1192 if (ret) 1193 goto error; 1194 if (!priv->mtr_idx_tbl) { 1195 priv->mtr_idx_tbl = 1196 mlx5_l3t_create(MLX5_L3T_TYPE_DWORD); 1197 if (!priv->mtr_idx_tbl) 1198 goto error; 1199 } 1200 data.dword = mtr_idx; 1201 if (mlx5_l3t_set_entry(priv->mtr_idx_tbl, meter_id, &data)) 1202 goto error; 1203 } 1204 if (mtr_policy) 1205 __atomic_add_fetch(&mtr_policy->ref_cnt, 1, __ATOMIC_RELAXED); 1206 return 0; 1207 error: 1208 mlx5_flow_destroy_mtr_tbls(dev, fm); 1209 /* Free policer counters. */ 1210 if (fm->drop_cnt) 1211 mlx5_counter_free(dev, fm->drop_cnt); 1212 if (priv->sh->meter_aso_en) 1213 mlx5_flow_mtr_free(dev, mtr_idx); 1214 else 1215 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], mtr_idx); 1216 return -rte_mtr_error_set(error, ENOTSUP, 1217 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 1218 NULL, "Failed to create devx meter."); 1219 } 1220 1221 static int 1222 mlx5_flow_meter_params_flush(struct rte_eth_dev *dev, 1223 struct mlx5_flow_meter_info *fm, 1224 uint32_t mtr_idx) 1225 { 1226 struct mlx5_priv *priv = dev->data->dev_private; 1227 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; 1228 struct mlx5_flow_meter_profile *fmp; 1229 struct mlx5_legacy_flow_meter *legacy_fm = NULL; 1230 struct mlx5_flow_meter_policy *mtr_policy; 1231 1232 /* Meter object must not have any owner. */ 1233 MLX5_ASSERT(!fm->ref_cnt); 1234 /* Get meter profile. */ 1235 fmp = fm->profile; 1236 if (fmp == NULL) 1237 return -1; 1238 /* Update dependencies. */ 1239 __atomic_sub_fetch(&fmp->ref_cnt, 1, __ATOMIC_RELAXED); 1240 fm->profile = NULL; 1241 /* Remove from list. */ 1242 if (!priv->sh->meter_aso_en) { 1243 legacy_fm = container_of(fm, 1244 struct mlx5_legacy_flow_meter, fm); 1245 TAILQ_REMOVE(fms, legacy_fm, next); 1246 } 1247 /* Free drop counters. */ 1248 if (fm->drop_cnt) 1249 mlx5_counter_free(dev, fm->drop_cnt); 1250 /* Free meter flow table. */ 1251 if (fm->flow_ipool) { 1252 mlx5_ipool_destroy(fm->flow_ipool); 1253 fm->flow_ipool = 0; 1254 } 1255 mlx5_flow_destroy_mtr_tbls(dev, fm); 1256 if (fm->def_policy) 1257 __atomic_sub_fetch(&priv->sh->mtrmng->def_policy_ref_cnt, 1258 1, __ATOMIC_RELAXED); 1259 if (priv->sh->meter_aso_en) { 1260 if (!fm->def_policy) { 1261 mtr_policy = mlx5_flow_meter_policy_find(dev, 1262 fm->policy_id, NULL); 1263 if (mtr_policy) 1264 __atomic_sub_fetch(&mtr_policy->ref_cnt, 1265 1, __ATOMIC_RELAXED); 1266 fm->policy_id = 0; 1267 } 1268 fm->def_policy = 0; 1269 if (mlx5_l3t_clear_entry(priv->mtr_idx_tbl, fm->meter_id)) 1270 return -1; 1271 mlx5_flow_mtr_free(dev, mtr_idx); 1272 } else { 1273 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], 1274 legacy_fm->idx); 1275 } 1276 return 0; 1277 } 1278 1279 /** 1280 * Destroy meter rules. 1281 * 1282 * @param[in] dev 1283 * Pointer to Ethernet device. 1284 * @param[in] meter_id 1285 * Meter id. 1286 * @param[out] error 1287 * Pointer to rte meter error structure. 1288 * 1289 * @return 1290 * 0 on success, a negative errno value otherwise and rte_errno is set. 1291 */ 1292 static int 1293 mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id, 1294 struct rte_mtr_error *error) 1295 { 1296 struct mlx5_priv *priv = dev->data->dev_private; 1297 struct mlx5_flow_meter_info *fm; 1298 uint32_t mtr_idx = 0; 1299 1300 if (!priv->mtr_en) 1301 return -rte_mtr_error_set(error, ENOTSUP, 1302 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 1303 NULL, 1304 "Meter is not supported"); 1305 /* Meter object must exist. */ 1306 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 1307 if (fm == NULL) 1308 return -rte_mtr_error_set(error, ENOENT, 1309 RTE_MTR_ERROR_TYPE_MTR_ID, 1310 NULL, 1311 "Meter object id not valid."); 1312 /* Meter object must not have any owner. */ 1313 if (fm->ref_cnt > 0) 1314 return -rte_mtr_error_set(error, EBUSY, 1315 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 1316 NULL, 1317 "Meter object is being used."); 1318 /* Destroy the meter profile. */ 1319 if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx)) 1320 return -rte_mtr_error_set(error, EINVAL, 1321 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 1322 NULL, 1323 "MTR object meter profile invalid."); 1324 return 0; 1325 } 1326 1327 /** 1328 * Modify meter state. 1329 * 1330 * @param[in] priv 1331 * Pointer to mlx5 private data structure. 1332 * @param[in] fm 1333 * Pointer to flow meter. 1334 * @param[in] new_state 1335 * New state to update. 1336 * @param[out] error 1337 * Pointer to rte meter error structure. 1338 * 1339 * @return 1340 * 0 on success, a negative errno value otherwise and rte_errno is set. 1341 */ 1342 static int 1343 mlx5_flow_meter_modify_state(struct mlx5_priv *priv, 1344 struct mlx5_flow_meter_info *fm, 1345 uint32_t new_state, 1346 struct rte_mtr_error *error) 1347 { 1348 static const struct mlx5_flow_meter_srtcm_rfc2697_prm srtcm = { 1349 .cbs_cir = RTE_BE32(MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL), 1350 .ebs_eir = 0, 1351 }; 1352 uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS | 1353 MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR; 1354 int ret; 1355 1356 if (new_state == MLX5_FLOW_METER_DISABLE) 1357 ret = mlx5_flow_meter_action_modify(priv, fm, 1358 &srtcm, modify_bits, 0, 0); 1359 else 1360 ret = mlx5_flow_meter_action_modify(priv, fm, 1361 &fm->profile->srtcm_prm, 1362 modify_bits, 0, 1); 1363 if (ret) 1364 return -rte_mtr_error_set(error, -ret, 1365 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 1366 NULL, 1367 new_state ? 1368 "Failed to enable meter." : 1369 "Failed to disable meter."); 1370 return 0; 1371 } 1372 1373 /** 1374 * Callback to enable flow meter. 1375 * 1376 * @param[in] dev 1377 * Pointer to Ethernet device. 1378 * @param[in] meter_id 1379 * Meter id. 1380 * @param[out] error 1381 * Pointer to rte meter error structure. 1382 * 1383 * @return 1384 * 0 on success, a negative errno value otherwise and rte_errno is set. 1385 */ 1386 static int 1387 mlx5_flow_meter_enable(struct rte_eth_dev *dev, 1388 uint32_t meter_id, 1389 struct rte_mtr_error *error) 1390 { 1391 struct mlx5_priv *priv = dev->data->dev_private; 1392 struct mlx5_flow_meter_info *fm; 1393 int ret; 1394 1395 if (!priv->mtr_en) 1396 return -rte_mtr_error_set(error, ENOTSUP, 1397 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1398 "Meter is not supported"); 1399 /* Meter object must exist. */ 1400 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 1401 if (fm == NULL) 1402 return -rte_mtr_error_set(error, ENOENT, 1403 RTE_MTR_ERROR_TYPE_MTR_ID, 1404 NULL, "Meter not found."); 1405 if (fm->active_state == MLX5_FLOW_METER_ENABLE) 1406 return 0; 1407 ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_ENABLE, 1408 error); 1409 if (!ret) 1410 fm->active_state = MLX5_FLOW_METER_ENABLE; 1411 return ret; 1412 } 1413 1414 /** 1415 * Callback to disable flow meter. 1416 * 1417 * @param[in] dev 1418 * Pointer to Ethernet device. 1419 * @param[in] meter_id 1420 * Meter id. 1421 * @param[out] error 1422 * Pointer to rte meter error structure. 1423 * 1424 * @return 1425 * 0 on success, a negative errno value otherwise and rte_errno is set. 1426 */ 1427 static int 1428 mlx5_flow_meter_disable(struct rte_eth_dev *dev, 1429 uint32_t meter_id, 1430 struct rte_mtr_error *error) 1431 { 1432 struct mlx5_priv *priv = dev->data->dev_private; 1433 struct mlx5_flow_meter_info *fm; 1434 int ret; 1435 1436 if (!priv->mtr_en) 1437 return -rte_mtr_error_set(error, ENOTSUP, 1438 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1439 "Meter is not supported"); 1440 /* Meter object must exist. */ 1441 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 1442 if (fm == NULL) 1443 return -rte_mtr_error_set(error, ENOENT, 1444 RTE_MTR_ERROR_TYPE_MTR_ID, 1445 NULL, "Meter not found."); 1446 if (fm->active_state == MLX5_FLOW_METER_DISABLE) 1447 return 0; 1448 ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_DISABLE, 1449 error); 1450 if (!ret) 1451 fm->active_state = MLX5_FLOW_METER_DISABLE; 1452 return ret; 1453 } 1454 1455 /** 1456 * Callback to update meter profile. 1457 * 1458 * @param[in] dev 1459 * Pointer to Ethernet device. 1460 * @param[in] meter_id 1461 * Meter id. 1462 * @param[in] meter_profile_id 1463 * To be updated meter profile id. 1464 * @param[out] error 1465 * Pointer to rte meter error structure. 1466 * 1467 * @return 1468 * 0 on success, a negative errno value otherwise and rte_errno is set. 1469 */ 1470 static int 1471 mlx5_flow_meter_profile_update(struct rte_eth_dev *dev, 1472 uint32_t meter_id, 1473 uint32_t meter_profile_id, 1474 struct rte_mtr_error *error) 1475 { 1476 struct mlx5_priv *priv = dev->data->dev_private; 1477 struct mlx5_flow_meter_profile *fmp; 1478 struct mlx5_flow_meter_profile *old_fmp; 1479 struct mlx5_flow_meter_info *fm; 1480 uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS | 1481 MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR; 1482 int ret; 1483 1484 if (!priv->mtr_en) 1485 return -rte_mtr_error_set(error, ENOTSUP, 1486 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1487 "Meter is not supported"); 1488 /* Meter profile must exist. */ 1489 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 1490 if (fmp == NULL) 1491 return -rte_mtr_error_set(error, ENOENT, 1492 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 1493 NULL, "Meter profile not found."); 1494 /* Meter object must exist. */ 1495 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 1496 if (fm == NULL) 1497 return -rte_mtr_error_set(error, ENOENT, 1498 RTE_MTR_ERROR_TYPE_MTR_ID, 1499 NULL, "Meter not found."); 1500 /* MTR object already set to meter profile id. */ 1501 old_fmp = fm->profile; 1502 if (fmp == old_fmp) 1503 return 0; 1504 /* Update the profile. */ 1505 fm->profile = fmp; 1506 /* Update meter params in HW (if not disabled). */ 1507 if (fm->active_state == MLX5_FLOW_METER_DISABLE) 1508 return 0; 1509 ret = mlx5_flow_meter_action_modify(priv, fm, &fm->profile->srtcm_prm, 1510 modify_bits, fm->active_state, 1); 1511 if (ret) { 1512 fm->profile = old_fmp; 1513 return -rte_mtr_error_set(error, -ret, 1514 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 1515 NULL, "Failed to update meter" 1516 " parmeters in hardware."); 1517 } 1518 old_fmp->ref_cnt--; 1519 fmp->ref_cnt++; 1520 return 0; 1521 } 1522 1523 /** 1524 * Callback to update meter stats mask. 1525 * 1526 * @param[in] dev 1527 * Pointer to Ethernet device. 1528 * @param[in] meter_id 1529 * Meter id. 1530 * @param[in] stats_mask 1531 * To be updated stats_mask. 1532 * @param[out] error 1533 * Pointer to rte meter error structure. 1534 * 1535 * @return 1536 * 0 on success, a negative errno value otherwise and rte_errno is set. 1537 */ 1538 static int 1539 mlx5_flow_meter_stats_update(struct rte_eth_dev *dev, 1540 uint32_t meter_id, 1541 uint64_t stats_mask, 1542 struct rte_mtr_error *error) 1543 { 1544 struct mlx5_priv *priv = dev->data->dev_private; 1545 struct mlx5_flow_meter_info *fm; 1546 1547 if (!priv->mtr_en) 1548 return -rte_mtr_error_set(error, ENOTSUP, 1549 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1550 "Meter is not supported"); 1551 /* Meter object must exist. */ 1552 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 1553 if (fm == NULL) 1554 return -rte_mtr_error_set(error, ENOENT, 1555 RTE_MTR_ERROR_TYPE_MTR_ID, 1556 NULL, "Meter object id not valid."); 1557 if (mlx5_flow_meter_stats_enable_update(dev, fm, stats_mask)) 1558 return -rte_mtr_error_set(error, ENOENT, 1559 RTE_MTR_ERROR_TYPE_MTR_ID, 1560 NULL, "Fail to allocate " 1561 "counter for meter."); 1562 return 0; 1563 } 1564 1565 /** 1566 * Callback to read meter statistics. 1567 * 1568 * @param[in] dev 1569 * Pointer to Ethernet device. 1570 * @param[in] meter_id 1571 * Meter id. 1572 * @param[out] stats 1573 * Pointer to store the statistics. 1574 * @param[out] stats_mask 1575 * Pointer to store the stats_mask. 1576 * @param[in] clear 1577 * Statistic to be cleared after read or not. 1578 * @param[out] error 1579 * Pointer to rte meter error structure. 1580 * 1581 * @return 1582 * 0 on success, a negative errno value otherwise and rte_errno is set. 1583 */ 1584 static int 1585 mlx5_flow_meter_stats_read(struct rte_eth_dev *dev, 1586 uint32_t meter_id, 1587 struct rte_mtr_stats *stats, 1588 uint64_t *stats_mask, 1589 int clear, 1590 struct rte_mtr_error *error) 1591 { 1592 struct mlx5_priv *priv = dev->data->dev_private; 1593 struct mlx5_flow_meter_info *fm; 1594 uint64_t pkts; 1595 uint64_t bytes; 1596 int ret = 0; 1597 1598 if (!priv->mtr_en) 1599 return -rte_mtr_error_set(error, ENOTSUP, 1600 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1601 "Meter is not supported"); 1602 /* Meter object must exist. */ 1603 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 1604 if (fm == NULL) 1605 return -rte_mtr_error_set(error, ENOENT, 1606 RTE_MTR_ERROR_TYPE_MTR_ID, 1607 NULL, "Meter object id not valid."); 1608 *stats_mask = 0; 1609 if (fm->bytes_dropped) 1610 *stats_mask |= RTE_MTR_STATS_N_BYTES_DROPPED; 1611 if (fm->pkts_dropped) 1612 *stats_mask |= RTE_MTR_STATS_N_PKTS_DROPPED; 1613 memset(stats, 0, sizeof(*stats)); 1614 if (fm->drop_cnt) { 1615 ret = mlx5_counter_query(dev, fm->drop_cnt, clear, &pkts, 1616 &bytes); 1617 if (ret) 1618 goto error; 1619 /* If need to read the packets, set it. */ 1620 if (fm->pkts_dropped) 1621 stats->n_pkts_dropped = pkts; 1622 /* If need to read the bytes, set it. */ 1623 if (fm->bytes_dropped) 1624 stats->n_bytes_dropped = bytes; 1625 } 1626 return 0; 1627 error: 1628 return -rte_mtr_error_set(error, ret, RTE_MTR_ERROR_TYPE_STATS, NULL, 1629 "Failed to read meter drop counters."); 1630 } 1631 1632 static const struct rte_mtr_ops mlx5_flow_mtr_ops = { 1633 .capabilities_get = mlx5_flow_mtr_cap_get, 1634 .meter_profile_add = mlx5_flow_meter_profile_add, 1635 .meter_profile_delete = mlx5_flow_meter_profile_delete, 1636 .meter_policy_validate = mlx5_flow_meter_policy_validate, 1637 .meter_policy_add = mlx5_flow_meter_policy_add, 1638 .meter_policy_delete = mlx5_flow_meter_policy_delete, 1639 .create = mlx5_flow_meter_create, 1640 .destroy = mlx5_flow_meter_destroy, 1641 .meter_enable = mlx5_flow_meter_enable, 1642 .meter_disable = mlx5_flow_meter_disable, 1643 .meter_profile_update = mlx5_flow_meter_profile_update, 1644 .meter_dscp_table_update = NULL, 1645 .stats_update = mlx5_flow_meter_stats_update, 1646 .stats_read = mlx5_flow_meter_stats_read, 1647 }; 1648 1649 /** 1650 * Get meter operations. 1651 * 1652 * @param dev 1653 * Pointer to Ethernet device structure. 1654 * @param arg 1655 * Pointer to set the mtr operations. 1656 * 1657 * @return 1658 * Always 0. 1659 */ 1660 int 1661 mlx5_flow_meter_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg) 1662 { 1663 *(const struct rte_mtr_ops **)arg = &mlx5_flow_mtr_ops; 1664 return 0; 1665 } 1666 1667 /** 1668 * Find meter by id. 1669 * 1670 * @param priv 1671 * Pointer to mlx5_priv. 1672 * @param meter_id 1673 * Meter id. 1674 * @param mtr_idx 1675 * Pointer to Meter index. 1676 * 1677 * @return 1678 * Pointer to the meter info found on success, NULL otherwise. 1679 */ 1680 struct mlx5_flow_meter_info * 1681 mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id, 1682 uint32_t *mtr_idx) 1683 { 1684 struct mlx5_legacy_flow_meter *legacy_fm; 1685 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; 1686 struct mlx5_aso_mtr *aso_mtr; 1687 struct mlx5_aso_mtr_pools_mng *pools_mng = 1688 &priv->sh->mtrmng->pools_mng; 1689 union mlx5_l3t_data data; 1690 1691 if (priv->sh->meter_aso_en) { 1692 rte_spinlock_lock(&pools_mng->mtrsl); 1693 if (!pools_mng->n_valid || !priv->mtr_idx_tbl) { 1694 rte_spinlock_unlock(&pools_mng->mtrsl); 1695 return NULL; 1696 } 1697 if (mlx5_l3t_get_entry(priv->mtr_idx_tbl, meter_id, &data) || 1698 !data.dword) { 1699 rte_spinlock_unlock(&pools_mng->mtrsl); 1700 return NULL; 1701 } 1702 if (mtr_idx) 1703 *mtr_idx = data.dword; 1704 aso_mtr = mlx5_aso_meter_by_idx(priv, data.dword); 1705 /* Remove reference taken by the mlx5_l3t_get_entry. */ 1706 mlx5_l3t_clear_entry(priv->mtr_idx_tbl, meter_id); 1707 rte_spinlock_unlock(&pools_mng->mtrsl); 1708 if (!aso_mtr || aso_mtr->state == ASO_METER_FREE) 1709 return NULL; 1710 return &aso_mtr->fm; 1711 } 1712 TAILQ_FOREACH(legacy_fm, fms, next) 1713 if (meter_id == legacy_fm->fm.meter_id) { 1714 if (mtr_idx) 1715 *mtr_idx = legacy_fm->idx; 1716 return &legacy_fm->fm; 1717 } 1718 return NULL; 1719 } 1720 1721 /** 1722 * Find meter by index. 1723 * 1724 * @param priv 1725 * Pointer to mlx5_priv. 1726 * @param idx 1727 * Meter index. 1728 * 1729 * @return 1730 * Pointer to the meter info found on success, NULL otherwise. 1731 */ 1732 struct mlx5_flow_meter_info * 1733 flow_dv_meter_find_by_idx(struct mlx5_priv *priv, uint32_t idx) 1734 { 1735 struct mlx5_aso_mtr *aso_mtr; 1736 1737 if (priv->sh->meter_aso_en) { 1738 aso_mtr = mlx5_aso_meter_by_idx(priv, idx); 1739 if (!aso_mtr) 1740 return NULL; 1741 return &aso_mtr->fm; 1742 } else { 1743 return mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], idx); 1744 } 1745 } 1746 1747 /** 1748 * Attach meter to flow. 1749 * Unidirectional Meter creation can only be done 1750 * when flow direction is known, i.e. when calling meter_attach. 1751 * 1752 * @param [in] priv 1753 * Pointer to mlx5 private data. 1754 * @param[in] fm 1755 * Pointer to flow meter. 1756 * @param [in] attr 1757 * Pointer to flow attributes. 1758 * @param [out] error 1759 * Pointer to error structure. 1760 * 1761 * @return 1762 * 0 on success, a negative errno value otherwise and rte_errno is set. 1763 */ 1764 int 1765 mlx5_flow_meter_attach(struct mlx5_priv *priv, 1766 struct mlx5_flow_meter_info *fm, 1767 const struct rte_flow_attr *attr, 1768 struct rte_flow_error *error) 1769 { 1770 int ret = 0; 1771 1772 if (priv->sh->meter_aso_en) { 1773 struct mlx5_aso_mtr *aso_mtr; 1774 1775 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm); 1776 if (mlx5_aso_mtr_wait(priv->sh, aso_mtr)) { 1777 return rte_flow_error_set(error, ENOENT, 1778 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1779 NULL, 1780 "Timeout in meter configuration"); 1781 } 1782 rte_spinlock_lock(&fm->sl); 1783 if (fm->shared || !fm->ref_cnt) { 1784 fm->ref_cnt++; 1785 } else { 1786 rte_flow_error_set(error, EINVAL, 1787 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 1788 "Meter cannot be shared"); 1789 ret = -1; 1790 } 1791 rte_spinlock_unlock(&fm->sl); 1792 } else { 1793 rte_spinlock_lock(&fm->sl); 1794 if (fm->meter_action) { 1795 if (fm->shared && 1796 attr->transfer == fm->transfer && 1797 attr->ingress == fm->ingress && 1798 attr->egress == fm->egress) { 1799 fm->ref_cnt++; 1800 } else { 1801 rte_flow_error_set(error, EINVAL, 1802 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 1803 fm->shared ? 1804 "Meter attr not match." : 1805 "Meter cannot be shared."); 1806 ret = -1; 1807 } 1808 } else { 1809 fm->ingress = attr->ingress; 1810 fm->egress = attr->egress; 1811 fm->transfer = attr->transfer; 1812 fm->ref_cnt = 1; 1813 /* This also creates the meter object. */ 1814 fm->meter_action = mlx5_flow_meter_action_create(priv, 1815 fm); 1816 if (!fm->meter_action) { 1817 fm->ref_cnt = 0; 1818 fm->ingress = 0; 1819 fm->egress = 0; 1820 fm->transfer = 0; 1821 rte_flow_error_set(error, EINVAL, 1822 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 1823 "Meter action create failed."); 1824 ret = -1; 1825 } 1826 } 1827 rte_spinlock_unlock(&fm->sl); 1828 } 1829 return ret ? -rte_errno : 0; 1830 } 1831 1832 /** 1833 * Detach meter from flow. 1834 * 1835 * @param [in] priv 1836 * Pointer to mlx5 private data. 1837 * @param [in] fm 1838 * Pointer to flow meter. 1839 */ 1840 void 1841 mlx5_flow_meter_detach(struct mlx5_priv *priv, 1842 struct mlx5_flow_meter_info *fm) 1843 { 1844 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER 1845 rte_spinlock_lock(&fm->sl); 1846 MLX5_ASSERT(fm->ref_cnt); 1847 if (--fm->ref_cnt == 0 && !priv->sh->meter_aso_en) { 1848 mlx5_glue->destroy_flow_action(fm->meter_action); 1849 fm->meter_action = NULL; 1850 fm->ingress = 0; 1851 fm->egress = 0; 1852 fm->transfer = 0; 1853 } 1854 rte_spinlock_unlock(&fm->sl); 1855 #else 1856 (void)priv; 1857 (void)fm; 1858 #endif 1859 } 1860 1861 /** 1862 * Flush meter with Rx queue configuration. 1863 * 1864 * @param[in] dev 1865 * Pointer to Ethernet device. 1866 */ 1867 void 1868 mlx5_flow_meter_rxq_flush(struct rte_eth_dev *dev) 1869 { 1870 struct mlx5_priv *priv = dev->data->dev_private; 1871 struct mlx5_flow_meter_sub_policy *sub_policy; 1872 struct mlx5_flow_meter_policy *mtr_policy; 1873 void *entry; 1874 uint32_t i, policy_idx; 1875 1876 if (!priv->mtr_en) 1877 return; 1878 if (priv->policy_idx_tbl) { 1879 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) { 1880 policy_idx = *(uint32_t *)entry; 1881 sub_policy = mlx5_ipool_get 1882 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 1883 policy_idx); 1884 if (!sub_policy || !sub_policy->main_policy) 1885 continue; 1886 mtr_policy = sub_policy->main_policy; 1887 if (mtr_policy->is_queue || mtr_policy->is_rss) 1888 mlx5_flow_destroy_sub_policy_with_rxq(dev, 1889 mtr_policy); 1890 } 1891 } 1892 } 1893 1894 /** 1895 * Iterate a meter hierarchy and flush all meters and policies if possible. 1896 * 1897 * @param[in] dev 1898 * Pointer to Ethernet device. 1899 * @param[in] fm 1900 * Pointer to flow meter. 1901 * @param[in] mtr_idx 1902 * .Meter's index 1903 * @param[out] error 1904 * Pointer to rte meter error structure. 1905 * 1906 * @return 1907 * 0 on success, a negative errno value otherwise and rte_errno is set. 1908 */ 1909 static int 1910 mlx5_flow_meter_flush_hierarchy(struct rte_eth_dev *dev, 1911 struct mlx5_flow_meter_info *fm, 1912 uint32_t mtr_idx, 1913 struct rte_mtr_error *error) 1914 { 1915 struct mlx5_priv *priv = dev->data->dev_private; 1916 struct mlx5_flow_meter_policy *policy; 1917 uint32_t policy_id; 1918 struct mlx5_flow_meter_info *next_fm; 1919 uint32_t next_mtr_idx; 1920 struct mlx5_flow_meter_policy *next_policy = NULL; 1921 1922 policy = mlx5_flow_meter_policy_find(dev, fm->policy_id, NULL); 1923 MLX5_ASSERT(policy); 1924 while (!fm->ref_cnt && policy->is_hierarchy) { 1925 policy_id = fm->policy_id; 1926 next_fm = mlx5_flow_meter_find(priv, 1927 policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id, 1928 &next_mtr_idx); 1929 if (next_fm) { 1930 next_policy = mlx5_flow_meter_policy_find(dev, 1931 next_fm->policy_id, 1932 NULL); 1933 MLX5_ASSERT(next_policy); 1934 } 1935 if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx)) 1936 return -rte_mtr_error_set(error, ENOTSUP, 1937 RTE_MTR_ERROR_TYPE_MTR_ID, 1938 NULL, 1939 "Failed to flush meter."); 1940 if (policy->ref_cnt) 1941 break; 1942 if (__mlx5_flow_meter_policy_delete(dev, policy_id, 1943 policy, error, true)) 1944 return -rte_errno; 1945 mlx5_free(policy); 1946 if (!next_fm || !next_policy) 1947 break; 1948 fm = next_fm; 1949 mtr_idx = next_mtr_idx; 1950 policy = next_policy; 1951 } 1952 return 0; 1953 } 1954 1955 /** 1956 * Flush all the hierarchy meters and their policies. 1957 * 1958 * @param[in] dev 1959 * Pointer to Ethernet device. 1960 * @param[out] error 1961 * Pointer to rte meter error structure. 1962 * 1963 * @return 1964 * 0 on success, a negative errno value otherwise and rte_errno is set. 1965 */ 1966 static int 1967 mlx5_flow_meter_flush_all_hierarchies(struct rte_eth_dev *dev, 1968 struct rte_mtr_error *error) 1969 { 1970 struct mlx5_priv *priv = dev->data->dev_private; 1971 struct mlx5_flow_meter_info *fm; 1972 struct mlx5_flow_meter_policy *policy; 1973 struct mlx5_flow_meter_sub_policy *sub_policy; 1974 struct mlx5_flow_meter_info *next_fm; 1975 struct mlx5_aso_mtr *aso_mtr; 1976 uint32_t mtr_idx = 0; 1977 uint32_t i, policy_idx; 1978 void *entry; 1979 1980 if (!priv->mtr_idx_tbl || !priv->policy_idx_tbl) 1981 return 0; 1982 MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) { 1983 mtr_idx = *(uint32_t *)entry; 1984 if (!mtr_idx) 1985 continue; 1986 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx); 1987 fm = &aso_mtr->fm; 1988 if (fm->ref_cnt || fm->def_policy) 1989 continue; 1990 if (mlx5_flow_meter_flush_hierarchy(dev, fm, mtr_idx, error)) 1991 return -rte_errno; 1992 } 1993 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) { 1994 policy_idx = *(uint32_t *)entry; 1995 sub_policy = mlx5_ipool_get 1996 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 1997 policy_idx); 1998 if (!sub_policy) 1999 return -rte_mtr_error_set(error, 2000 EINVAL, 2001 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 2002 NULL, "Meter policy invalid."); 2003 policy = sub_policy->main_policy; 2004 if (!policy || !policy->is_hierarchy || policy->ref_cnt) 2005 continue; 2006 next_fm = mlx5_flow_meter_find(priv, 2007 policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id, 2008 &mtr_idx); 2009 if (__mlx5_flow_meter_policy_delete(dev, i, policy, 2010 error, true)) 2011 return -rte_mtr_error_set(error, 2012 EINVAL, 2013 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 2014 NULL, "Meter policy invalid."); 2015 mlx5_free(policy); 2016 if (!next_fm || next_fm->ref_cnt || next_fm->def_policy) 2017 continue; 2018 if (mlx5_flow_meter_flush_hierarchy(dev, next_fm, 2019 mtr_idx, error)) 2020 return -rte_errno; 2021 } 2022 return 0; 2023 } 2024 /** 2025 * Flush meter configuration. 2026 * 2027 * @param[in] dev 2028 * Pointer to Ethernet device. 2029 * @param[out] error 2030 * Pointer to rte meter error structure. 2031 * 2032 * @return 2033 * 0 on success, a negative errno value otherwise and rte_errno is set. 2034 */ 2035 int 2036 mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error) 2037 { 2038 struct mlx5_priv *priv = dev->data->dev_private; 2039 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; 2040 struct mlx5_flow_meter_profile *fmp; 2041 struct mlx5_legacy_flow_meter *legacy_fm; 2042 struct mlx5_flow_meter_info *fm; 2043 struct mlx5_flow_meter_sub_policy *sub_policy; 2044 void *tmp; 2045 uint32_t i, mtr_idx, policy_idx; 2046 void *entry; 2047 struct mlx5_aso_mtr *aso_mtr; 2048 2049 if (!priv->mtr_en) 2050 return 0; 2051 if (priv->sh->meter_aso_en) { 2052 if (mlx5_flow_meter_flush_all_hierarchies(dev, error)) 2053 return -rte_errno; 2054 if (priv->mtr_idx_tbl) { 2055 MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) { 2056 mtr_idx = *(uint32_t *)entry; 2057 if (mtr_idx) { 2058 aso_mtr = 2059 mlx5_aso_meter_by_idx(priv, mtr_idx); 2060 fm = &aso_mtr->fm; 2061 (void)mlx5_flow_meter_params_flush(dev, 2062 fm, mtr_idx); 2063 } 2064 } 2065 mlx5_l3t_destroy(priv->mtr_idx_tbl); 2066 priv->mtr_idx_tbl = NULL; 2067 } 2068 } else { 2069 TAILQ_FOREACH_SAFE(legacy_fm, fms, next, tmp) { 2070 fm = &legacy_fm->fm; 2071 if (mlx5_flow_meter_params_flush(dev, fm, 0)) 2072 return -rte_mtr_error_set(error, EINVAL, 2073 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 2074 NULL, "MTR object meter profile invalid."); 2075 } 2076 } 2077 if (priv->policy_idx_tbl) { 2078 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) { 2079 policy_idx = *(uint32_t *)entry; 2080 sub_policy = mlx5_ipool_get 2081 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 2082 policy_idx); 2083 if (!sub_policy) 2084 return -rte_mtr_error_set(error, 2085 EINVAL, 2086 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 2087 NULL, "MTR object " 2088 "meter policy invalid."); 2089 if (__mlx5_flow_meter_policy_delete(dev, i, 2090 sub_policy->main_policy, 2091 error, true)) 2092 return -rte_mtr_error_set(error, 2093 EINVAL, 2094 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 2095 NULL, "MTR object " 2096 "meter policy invalid."); 2097 mlx5_free(sub_policy->main_policy); 2098 } 2099 mlx5_l3t_destroy(priv->policy_idx_tbl); 2100 priv->policy_idx_tbl = NULL; 2101 } 2102 if (priv->mtr_profile_tbl) { 2103 MLX5_L3T_FOREACH(priv->mtr_profile_tbl, i, entry) { 2104 fmp = entry; 2105 if (mlx5_flow_meter_profile_delete(dev, fmp->id, 2106 error)) 2107 return -rte_mtr_error_set(error, EINVAL, 2108 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 2109 NULL, "Fail to destroy " 2110 "meter profile."); 2111 } 2112 mlx5_l3t_destroy(priv->mtr_profile_tbl); 2113 priv->mtr_profile_tbl = NULL; 2114 } 2115 /* Delete default policy table. */ 2116 mlx5_flow_destroy_def_policy(dev); 2117 if (priv->sh->refcnt == 1) 2118 mlx5_flow_destroy_mtr_drop_tbls(dev); 2119 return 0; 2120 } 2121