1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2021 NVIDIA Corporation & Affiliates 3 */ 4 #include <rte_malloc.h> 5 #include <mlx5_devx_cmds.h> 6 #include <mlx5_malloc.h> 7 #include "mlx5.h" 8 #include "mlx5_flow.h" 9 10 static_assert(sizeof(uint32_t) * CHAR_BIT >= MLX5_PORT_FLEX_ITEM_NUM, 11 "Flex item maximal number exceeds uint32_t bit width"); 12 13 /** 14 * Routine called once on port initialization to init flex item 15 * related infrastructure initialization 16 * 17 * @param dev 18 * Ethernet device to perform flex item initialization 19 * 20 * @return 21 * 0 on success, a negative errno value otherwise and rte_errno is set. 22 */ 23 int 24 mlx5_flex_item_port_init(struct rte_eth_dev *dev) 25 { 26 struct mlx5_priv *priv = dev->data->dev_private; 27 28 rte_spinlock_init(&priv->flex_item_sl); 29 MLX5_ASSERT(!priv->flex_item_map); 30 return 0; 31 } 32 33 /** 34 * Routine called once on port close to perform flex item 35 * related infrastructure cleanup. 36 * 37 * @param dev 38 * Ethernet device to perform cleanup 39 */ 40 void 41 mlx5_flex_item_port_cleanup(struct rte_eth_dev *dev) 42 { 43 struct mlx5_priv *priv = dev->data->dev_private; 44 uint32_t i; 45 46 for (i = 0; i < MLX5_PORT_FLEX_ITEM_NUM && priv->flex_item_map ; i++) { 47 if (priv->flex_item_map & (1 << i)) { 48 struct mlx5_flex_item *flex = &priv->flex_item[i]; 49 50 claim_zero(mlx5_list_unregister 51 (priv->sh->flex_parsers_dv, 52 &flex->devx_fp->entry)); 53 flex->devx_fp = NULL; 54 flex->refcnt = 0; 55 priv->flex_item_map &= ~(1 << i); 56 } 57 } 58 } 59 60 static int 61 mlx5_flex_index(struct mlx5_priv *priv, struct mlx5_flex_item *item) 62 { 63 uintptr_t start = (uintptr_t)&priv->flex_item[0]; 64 uintptr_t entry = (uintptr_t)item; 65 uintptr_t idx = (entry - start) / sizeof(struct mlx5_flex_item); 66 67 if (entry < start || 68 idx >= MLX5_PORT_FLEX_ITEM_NUM || 69 (entry - start) % sizeof(struct mlx5_flex_item) || 70 !(priv->flex_item_map & (1u << idx))) 71 return -1; 72 return (int)idx; 73 } 74 75 static struct mlx5_flex_item * 76 mlx5_flex_alloc(struct mlx5_priv *priv) 77 { 78 struct mlx5_flex_item *item = NULL; 79 80 rte_spinlock_lock(&priv->flex_item_sl); 81 if (~priv->flex_item_map) { 82 uint32_t idx = rte_bsf32(~priv->flex_item_map); 83 84 if (idx < MLX5_PORT_FLEX_ITEM_NUM) { 85 item = &priv->flex_item[idx]; 86 MLX5_ASSERT(!item->refcnt); 87 MLX5_ASSERT(!item->devx_fp); 88 item->devx_fp = NULL; 89 rte_atomic_store_explicit(&item->refcnt, 0, rte_memory_order_release); 90 priv->flex_item_map |= 1u << idx; 91 } 92 } 93 rte_spinlock_unlock(&priv->flex_item_sl); 94 return item; 95 } 96 97 static void 98 mlx5_flex_free(struct mlx5_priv *priv, struct mlx5_flex_item *item) 99 { 100 int idx = mlx5_flex_index(priv, item); 101 102 MLX5_ASSERT(idx >= 0 && 103 idx < MLX5_PORT_FLEX_ITEM_NUM && 104 (priv->flex_item_map & (1u << idx))); 105 if (idx >= 0) { 106 rte_spinlock_lock(&priv->flex_item_sl); 107 MLX5_ASSERT(!item->refcnt); 108 MLX5_ASSERT(!item->devx_fp); 109 item->devx_fp = NULL; 110 rte_atomic_store_explicit(&item->refcnt, 0, rte_memory_order_release); 111 priv->flex_item_map &= ~(1u << idx); 112 rte_spinlock_unlock(&priv->flex_item_sl); 113 } 114 } 115 116 static uint32_t 117 mlx5_flex_get_bitfield(const struct rte_flow_item_flex *item, 118 uint32_t pos, uint32_t width, uint32_t shift) 119 { 120 const uint8_t *ptr = item->pattern + pos / CHAR_BIT; 121 uint32_t val, vbits; 122 123 /* Proceed the bitfield start byte. */ 124 MLX5_ASSERT(width <= sizeof(uint32_t) * CHAR_BIT && width); 125 MLX5_ASSERT(width + shift <= sizeof(uint32_t) * CHAR_BIT); 126 if (item->length <= pos / CHAR_BIT) 127 return 0; 128 val = *ptr++ >> (pos % CHAR_BIT); 129 vbits = CHAR_BIT - pos % CHAR_BIT; 130 pos = (pos + vbits) / CHAR_BIT; 131 vbits = RTE_MIN(vbits, width); 132 val &= RTE_BIT32(vbits) - 1; 133 while (vbits < width && pos < item->length) { 134 uint32_t part = RTE_MIN(width - vbits, (uint32_t)CHAR_BIT); 135 uint32_t tmp = *ptr++; 136 137 pos++; 138 tmp &= RTE_BIT32(part) - 1; 139 val |= tmp << vbits; 140 vbits += part; 141 } 142 return rte_bswap32(val <<= shift); 143 } 144 145 #define SET_FP_MATCH_SAMPLE_ID(x, def, msk, val, sid) \ 146 do { \ 147 uint32_t tmp, out = (def); \ 148 tmp = MLX5_GET(fte_match_set_misc4, misc4_v, \ 149 prog_sample_field_value_##x); \ 150 tmp = (tmp & ~out) | (val); \ 151 MLX5_SET(fte_match_set_misc4, misc4_v, \ 152 prog_sample_field_value_##x, tmp); \ 153 tmp = MLX5_GET(fte_match_set_misc4, misc4_m, \ 154 prog_sample_field_value_##x); \ 155 tmp = (tmp & ~out) | (msk); \ 156 MLX5_SET(fte_match_set_misc4, misc4_m, \ 157 prog_sample_field_value_##x, tmp); \ 158 tmp = tmp ? (sid) : 0; \ 159 MLX5_SET(fte_match_set_misc4, misc4_v, \ 160 prog_sample_field_id_##x, tmp);\ 161 MLX5_SET(fte_match_set_misc4, misc4_m, \ 162 prog_sample_field_id_##x, tmp); \ 163 } while (0) 164 165 __rte_always_inline static void 166 mlx5_flex_set_match_sample(void *misc4_m, void *misc4_v, 167 uint32_t def, uint32_t mask, uint32_t value, 168 uint32_t sample_id, uint32_t id) 169 { 170 switch (id) { 171 case 0: 172 SET_FP_MATCH_SAMPLE_ID(0, def, mask, value, sample_id); 173 break; 174 case 1: 175 SET_FP_MATCH_SAMPLE_ID(1, def, mask, value, sample_id); 176 break; 177 case 2: 178 SET_FP_MATCH_SAMPLE_ID(2, def, mask, value, sample_id); 179 break; 180 case 3: 181 SET_FP_MATCH_SAMPLE_ID(3, def, mask, value, sample_id); 182 break; 183 case 4: 184 SET_FP_MATCH_SAMPLE_ID(4, def, mask, value, sample_id); 185 break; 186 case 5: 187 SET_FP_MATCH_SAMPLE_ID(5, def, mask, value, sample_id); 188 break; 189 case 6: 190 SET_FP_MATCH_SAMPLE_ID(6, def, mask, value, sample_id); 191 break; 192 case 7: 193 SET_FP_MATCH_SAMPLE_ID(7, def, mask, value, sample_id); 194 break; 195 default: 196 MLX5_ASSERT(false); 197 break; 198 } 199 #undef SET_FP_MATCH_SAMPLE_ID 200 } 201 202 /** 203 * Get the flex parser sample id and corresponding mask 204 * per shift and width information. 205 * 206 * @param[in] tp 207 * Mlx5 flex item sample mapping handle. 208 * @param[in] idx 209 * Mapping index. 210 * @param[in, out] pos 211 * Where to search the value and mask. 212 * @param[in] is_inner 213 * For inner matching or not. 214 * @param[in, def] def 215 * Mask generated by mapping shift and width. 216 * 217 * @return 218 * 0 on success, -1 to ignore. 219 */ 220 int 221 mlx5_flex_get_sample_id(const struct mlx5_flex_item *tp, 222 uint32_t idx, uint32_t *pos, 223 bool is_inner, uint32_t *def) 224 { 225 const struct mlx5_flex_pattern_field *map = tp->map + idx; 226 uint32_t id = map->reg_id; 227 228 *def = (RTE_BIT64(map->width) - 1) << map->shift; 229 /* Skip placeholders for DUMMY fields. */ 230 if (id == MLX5_INVALID_SAMPLE_REG_ID) { 231 *pos += map->width; 232 return -1; 233 } 234 MLX5_ASSERT(map->width); 235 MLX5_ASSERT(id < tp->devx_fp->num_samples); 236 if (tp->tunnel_mode == FLEX_TUNNEL_MODE_MULTI && is_inner) { 237 uint32_t num_samples = tp->devx_fp->num_samples / 2; 238 239 MLX5_ASSERT(tp->devx_fp->num_samples % 2 == 0); 240 MLX5_ASSERT(id < num_samples); 241 id += num_samples; 242 } 243 return id; 244 } 245 246 /** 247 * Get the flex parser mapping value per definer format_select_dw. 248 * 249 * @param[in] item 250 * Rte flex item pointer. 251 * @param[in] flex 252 * Mlx5 flex item sample mapping handle. 253 * @param[in] byte_off 254 * Mlx5 flex item format_select_dw. 255 * @param[in] is_mask 256 * Spec or mask. 257 * @param[in] tunnel 258 * Tunnel mode or not. 259 * @param[in, def] value 260 * Value calculated for this flex parser, either spec or mask. 261 * 262 * @return 263 * 0 on success, -1 for error. 264 */ 265 int 266 mlx5_flex_get_parser_value_per_byte_off(const struct rte_flow_item_flex *item, 267 void *flex, uint32_t byte_off, 268 bool is_mask, bool tunnel, uint32_t *value) 269 { 270 struct mlx5_flex_pattern_field *map; 271 struct mlx5_flex_item *tp = flex; 272 uint32_t def, i, pos, val; 273 int id; 274 275 *value = 0; 276 for (i = 0, pos = 0; i < tp->mapnum && pos < item->length * CHAR_BIT; i++) { 277 map = tp->map + i; 278 id = mlx5_flex_get_sample_id(tp, i, &pos, tunnel, &def); 279 if (id == -1) 280 continue; 281 if (id >= (int)tp->devx_fp->num_samples || id >= MLX5_GRAPH_NODE_SAMPLE_NUM) 282 return -1; 283 if (byte_off == tp->devx_fp->sample_info[id].sample_dw_data * sizeof(uint32_t)) { 284 val = mlx5_flex_get_bitfield(item, pos, map->width, map->shift); 285 if (is_mask) 286 val &= RTE_BE32(def); 287 *value |= val; 288 } 289 pos += map->width; 290 } 291 return 0; 292 } 293 294 /** 295 * Translate item pattern into matcher fields according to translation 296 * array. 297 * 298 * @param dev 299 * Ethernet device to translate flex item on. 300 * @param[in, out] matcher 301 * Flow matcher to configure 302 * @param[in, out] key 303 * Flow matcher value. 304 * @param[in] item 305 * Flow pattern to translate. 306 * @param[in] is_inner 307 * Inner Flex Item (follows after tunnel header). 308 * 309 * @return 310 * 0 on success, a negative errno value otherwise and rte_errno is set. 311 */ 312 void 313 mlx5_flex_flow_translate_item(struct rte_eth_dev *dev, 314 void *matcher, void *key, 315 const struct rte_flow_item *item, 316 bool is_inner) 317 { 318 const struct rte_flow_item_flex *spec, *mask; 319 void *misc4_m = MLX5_ADDR_OF(fte_match_param, matcher, 320 misc_parameters_4); 321 void *misc4_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_4); 322 struct mlx5_flex_item *tp; 323 uint32_t i, pos = 0; 324 uint32_t sample_id; 325 326 RTE_SET_USED(dev); 327 MLX5_ASSERT(item->spec && item->mask); 328 spec = item->spec; 329 mask = item->mask; 330 tp = (struct mlx5_flex_item *)spec->handle; 331 for (i = 0; i < tp->mapnum; i++) { 332 struct mlx5_flex_pattern_field *map = tp->map + i; 333 uint32_t val, msk, def; 334 int id = mlx5_flex_get_sample_id(tp, i, &pos, is_inner, &def); 335 336 if (id == -1) 337 continue; 338 MLX5_ASSERT(id < (int)tp->devx_fp->num_samples); 339 if (id >= (int)tp->devx_fp->num_samples || 340 id >= MLX5_GRAPH_NODE_SAMPLE_NUM) 341 return; 342 val = mlx5_flex_get_bitfield(spec, pos, map->width, map->shift); 343 msk = mlx5_flex_get_bitfield(mask, pos, map->width, map->shift); 344 sample_id = tp->devx_fp->sample_ids[id]; 345 mlx5_flex_set_match_sample(misc4_m, misc4_v, 346 def, msk & def, val & msk & def, 347 sample_id, id); 348 pos += map->width; 349 } 350 } 351 352 /** 353 * Convert flex item handle (from the RTE flow) to flex item index on port. 354 * Optionally can increment flex item object reference count. 355 * 356 * @param dev 357 * Ethernet device to acquire flex item on. 358 * @param[in] handle 359 * Flow item handle from item spec. 360 * @param[in] acquire 361 * If set - increment reference counter. 362 * 363 * @return 364 * >=0 - index on success, a negative errno value otherwise 365 * and rte_errno is set. 366 */ 367 int 368 mlx5_flex_acquire_index(struct rte_eth_dev *dev, 369 struct rte_flow_item_flex_handle *handle, 370 bool acquire) 371 { 372 struct mlx5_priv *priv = dev->data->dev_private; 373 struct mlx5_flex_item *flex = (struct mlx5_flex_item *)handle; 374 int ret = mlx5_flex_index(priv, flex); 375 376 if (ret < 0) { 377 errno = -EINVAL; 378 rte_errno = EINVAL; 379 return ret; 380 } 381 if (acquire) 382 rte_atomic_fetch_add_explicit(&flex->refcnt, 1, rte_memory_order_release); 383 return ret; 384 } 385 386 /** 387 * Release flex item index on port - decrements reference counter by index. 388 * 389 * @param dev 390 * Ethernet device to acquire flex item on. 391 * @param[in] index 392 * Flow item index. 393 * 394 * @return 395 * 0 - on success, a negative errno value otherwise and rte_errno is set. 396 */ 397 int 398 mlx5_flex_release_index(struct rte_eth_dev *dev, 399 int index) 400 { 401 struct mlx5_priv *priv = dev->data->dev_private; 402 struct mlx5_flex_item *flex; 403 404 if (index >= MLX5_PORT_FLEX_ITEM_NUM || 405 !(priv->flex_item_map & (1u << index))) { 406 errno = EINVAL; 407 rte_errno = -EINVAL; 408 return -EINVAL; 409 } 410 flex = priv->flex_item + index; 411 if (flex->refcnt <= 1) { 412 MLX5_ASSERT(false); 413 errno = EINVAL; 414 rte_errno = -EINVAL; 415 return -EINVAL; 416 } 417 rte_atomic_fetch_sub_explicit(&flex->refcnt, 1, rte_memory_order_release); 418 return 0; 419 } 420 421 /* 422 * Calculate largest mask value for a given shift. 423 * 424 * shift mask 425 * ------- --------------- 426 * 0 b111100 0x3C 427 * 1 b111110 0x3E 428 * 2 b111111 0x3F 429 * 3 b011111 0x1F 430 * 4 b001111 0x0F 431 * 5 b000111 0x07 432 */ 433 static uint8_t 434 mlx5_flex_hdr_len_mask(uint8_t shift, 435 const struct mlx5_hca_flex_attr *attr) 436 { 437 uint32_t base_mask; 438 int diff = shift - MLX5_PARSE_GRAPH_NODE_HDR_LEN_SHIFT_DWORD; 439 440 base_mask = mlx5_hca_parse_graph_node_base_hdr_len_mask(attr); 441 return diff == 0 ? base_mask : 442 diff < 0 ? (base_mask << -diff) & base_mask : base_mask >> diff; 443 } 444 445 static int 446 mlx5_flex_translate_length(struct mlx5_hca_flex_attr *attr, 447 const struct rte_flow_item_flex_conf *conf, 448 struct mlx5_flex_parser_devx *devx, 449 struct rte_flow_error *error) 450 { 451 const struct rte_flow_item_flex_field *field = &conf->next_header; 452 struct mlx5_devx_graph_node_attr *node = &devx->devx_conf; 453 uint32_t len_width, mask; 454 455 if (field->field_base % CHAR_BIT) 456 return rte_flow_error_set 457 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, 458 "not byte aligned header length field"); 459 switch (field->field_mode) { 460 case FIELD_MODE_DUMMY: 461 return rte_flow_error_set 462 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, 463 "invalid header length field mode (DUMMY)"); 464 case FIELD_MODE_FIXED: 465 if (!(attr->header_length_mode & 466 RTE_BIT32(MLX5_GRAPH_NODE_LEN_FIXED))) 467 return rte_flow_error_set 468 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, 469 "unsupported header length field mode (FIXED)"); 470 if (field->field_size || 471 field->offset_mask || field->offset_shift) 472 return rte_flow_error_set 473 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, 474 "invalid fields for fixed mode"); 475 if (field->field_base < 0) 476 return rte_flow_error_set 477 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, 478 "negative header length field base (FIXED)"); 479 node->header_length_mode = MLX5_GRAPH_NODE_LEN_FIXED; 480 break; 481 case FIELD_MODE_OFFSET: 482 if (!(attr->header_length_mode & 483 RTE_BIT32(MLX5_GRAPH_NODE_LEN_FIELD))) 484 return rte_flow_error_set 485 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, 486 "unsupported header length field mode (OFFSET)"); 487 if (!field->field_size) 488 return rte_flow_error_set 489 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, 490 "field size is a must for offset mode"); 491 if (field->field_size + field->offset_base < attr->header_length_mask_width) 492 return rte_flow_error_set 493 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, 494 "field size plus offset_base is too small"); 495 node->header_length_mode = MLX5_GRAPH_NODE_LEN_FIELD; 496 if (field->offset_mask == 0 || 497 !rte_is_power_of_2(field->offset_mask + 1)) 498 return rte_flow_error_set 499 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, 500 "invalid length field offset mask (OFFSET)"); 501 len_width = rte_fls_u32(field->offset_mask); 502 if (len_width > attr->header_length_mask_width) 503 return rte_flow_error_set 504 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, 505 "length field offset mask too wide (OFFSET)"); 506 mask = mlx5_flex_hdr_len_mask(field->offset_shift, attr); 507 if (mask < field->offset_mask) 508 return rte_flow_error_set 509 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, 510 "length field shift too big (OFFSET)"); 511 node->header_length_field_mask = RTE_MIN(mask, 512 field->offset_mask); 513 break; 514 case FIELD_MODE_BITMASK: 515 if (!(attr->header_length_mode & 516 RTE_BIT32(MLX5_GRAPH_NODE_LEN_BITMASK))) 517 return rte_flow_error_set 518 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, 519 "unsupported header length field mode (BITMASK)"); 520 if (attr->header_length_mask_width < field->field_size) 521 return rte_flow_error_set 522 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, 523 "header length field width exceeds limit"); 524 node->header_length_mode = MLX5_GRAPH_NODE_LEN_BITMASK; 525 mask = mlx5_flex_hdr_len_mask(field->offset_shift, attr); 526 if (mask < field->offset_mask) 527 return rte_flow_error_set 528 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, 529 "length field shift too big (BITMASK)"); 530 node->header_length_field_mask = RTE_MIN(mask, 531 field->offset_mask); 532 break; 533 default: 534 return rte_flow_error_set 535 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, 536 "unknown header length field mode"); 537 } 538 if (field->field_base / CHAR_BIT >= 0 && 539 field->field_base / CHAR_BIT > attr->max_base_header_length) 540 return rte_flow_error_set 541 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, 542 "header length field base exceeds limit"); 543 node->header_length_base_value = field->field_base / CHAR_BIT; 544 if (field->field_mode == FIELD_MODE_OFFSET || 545 field->field_mode == FIELD_MODE_BITMASK) { 546 if (field->offset_shift > 15 || field->offset_shift < 0) 547 return rte_flow_error_set 548 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, 549 "header length field shift exceeds limit"); 550 node->header_length_field_shift = field->offset_shift; 551 node->header_length_field_offset = field->offset_base; 552 } 553 if (field->field_mode == FIELD_MODE_OFFSET) { 554 if (field->field_size > attr->header_length_mask_width) { 555 node->header_length_field_offset += 556 field->field_size - attr->header_length_mask_width; 557 } else if (field->field_size < attr->header_length_mask_width) { 558 node->header_length_field_offset -= 559 attr->header_length_mask_width - field->field_size; 560 node->header_length_field_mask = 561 RTE_MIN(node->header_length_field_mask, 562 (1u << field->field_size) - 1); 563 } 564 } 565 return 0; 566 } 567 568 static int 569 mlx5_flex_translate_next(struct mlx5_hca_flex_attr *attr, 570 const struct rte_flow_item_flex_conf *conf, 571 struct mlx5_flex_parser_devx *devx, 572 struct rte_flow_error *error) 573 { 574 const struct rte_flow_item_flex_field *field = &conf->next_protocol; 575 struct mlx5_devx_graph_node_attr *node = &devx->devx_conf; 576 577 switch (field->field_mode) { 578 case FIELD_MODE_DUMMY: 579 if (conf->nb_outputs) 580 return rte_flow_error_set 581 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, 582 "next protocol field is required (DUMMY)"); 583 return 0; 584 case FIELD_MODE_FIXED: 585 break; 586 case FIELD_MODE_OFFSET: 587 return rte_flow_error_set 588 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, 589 "unsupported next protocol field mode (OFFSET)"); 590 break; 591 case FIELD_MODE_BITMASK: 592 return rte_flow_error_set 593 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, 594 "unsupported next protocol field mode (BITMASK)"); 595 default: 596 return rte_flow_error_set 597 (error, EINVAL, 598 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 599 "unknown next protocol field mode"); 600 } 601 MLX5_ASSERT(field->field_mode == FIELD_MODE_FIXED); 602 if (!conf->nb_outputs) 603 return rte_flow_error_set 604 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, 605 "out link(s) is required if next field present"); 606 if (attr->max_next_header_offset < field->field_base) 607 return rte_flow_error_set 608 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, 609 "next protocol field base exceeds limit"); 610 if (field->offset_shift) 611 return rte_flow_error_set 612 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, 613 "unsupported next protocol field shift"); 614 node->next_header_field_offset = field->field_base; 615 node->next_header_field_size = field->field_size; 616 return 0; 617 } 618 619 /* Helper structure to handle field bit intervals. */ 620 struct mlx5_flex_field_cover { 621 uint16_t num; 622 int32_t start[MLX5_FLEX_ITEM_MAPPING_NUM]; 623 int32_t end[MLX5_FLEX_ITEM_MAPPING_NUM]; 624 uint8_t mapped[MLX5_FLEX_ITEM_MAPPING_NUM / CHAR_BIT + 1]; 625 }; 626 627 static void 628 mlx5_flex_insert_field(struct mlx5_flex_field_cover *cover, 629 uint16_t num, int32_t start, int32_t end) 630 { 631 MLX5_ASSERT(num < MLX5_FLEX_ITEM_MAPPING_NUM); 632 MLX5_ASSERT(num <= cover->num); 633 if (num < cover->num) { 634 memmove(&cover->start[num + 1], &cover->start[num], 635 (cover->num - num) * sizeof(int32_t)); 636 memmove(&cover->end[num + 1], &cover->end[num], 637 (cover->num - num) * sizeof(int32_t)); 638 } 639 cover->start[num] = start; 640 cover->end[num] = end; 641 cover->num++; 642 } 643 644 static void 645 mlx5_flex_merge_field(struct mlx5_flex_field_cover *cover, uint16_t num) 646 { 647 uint32_t i, del = 0; 648 int32_t end; 649 650 MLX5_ASSERT(num < MLX5_FLEX_ITEM_MAPPING_NUM); 651 MLX5_ASSERT(num < (cover->num - 1)); 652 end = cover->end[num]; 653 for (i = num + 1; i < cover->num; i++) { 654 if (end < cover->start[i]) 655 break; 656 del++; 657 if (end <= cover->end[i]) { 658 cover->end[num] = cover->end[i]; 659 break; 660 } 661 } 662 if (del) { 663 MLX5_ASSERT(del < (cover->num - 1u - num)); 664 cover->num -= del; 665 MLX5_ASSERT(cover->num > num); 666 if ((cover->num - num) > 1) { 667 memmove(&cover->start[num + 1], 668 &cover->start[num + 1 + del], 669 (cover->num - num - 1) * sizeof(int32_t)); 670 memmove(&cover->end[num + 1], 671 &cover->end[num + 1 + del], 672 (cover->num - num - 1) * sizeof(int32_t)); 673 } 674 } 675 } 676 677 /* 678 * Validate the sample field and update interval array 679 * if parameters match with the 'match" field. 680 * Returns: 681 * < 0 - error 682 * == 0 - no match, interval array not updated 683 * > 0 - match, interval array updated 684 */ 685 static int 686 mlx5_flex_cover_sample(struct mlx5_flex_field_cover *cover, 687 struct rte_flow_item_flex_field *field, 688 struct rte_flow_item_flex_field *match, 689 struct mlx5_hca_flex_attr *attr, 690 struct rte_flow_error *error) 691 { 692 int32_t start, end; 693 uint32_t i; 694 695 switch (field->field_mode) { 696 case FIELD_MODE_DUMMY: 697 return 0; 698 case FIELD_MODE_FIXED: 699 if (!(attr->sample_offset_mode & 700 RTE_BIT32(MLX5_GRAPH_SAMPLE_OFFSET_FIXED))) 701 return rte_flow_error_set 702 (error, EINVAL, 703 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 704 "unsupported sample field mode (FIXED)"); 705 if (field->offset_shift) 706 return rte_flow_error_set 707 (error, EINVAL, 708 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 709 "invalid sample field shift (FIXED"); 710 if (field->field_base < 0) 711 return rte_flow_error_set 712 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, 713 "invalid sample field base (FIXED)"); 714 if (field->field_base / CHAR_BIT > attr->max_sample_base_offset) 715 return rte_flow_error_set 716 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, 717 "sample field base exceeds limit (FIXED)"); 718 break; 719 case FIELD_MODE_OFFSET: 720 if (!(attr->sample_offset_mode & 721 RTE_BIT32(MLX5_GRAPH_SAMPLE_OFFSET_FIELD))) 722 return rte_flow_error_set 723 (error, EINVAL, 724 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 725 "unsupported sample field mode (OFFSET)"); 726 if (field->field_base / CHAR_BIT >= 0 && 727 field->field_base / CHAR_BIT > attr->max_sample_base_offset) 728 return rte_flow_error_set 729 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, 730 "sample field base exceeds limit"); 731 break; 732 case FIELD_MODE_BITMASK: 733 if (!(attr->sample_offset_mode & 734 RTE_BIT32(MLX5_GRAPH_SAMPLE_OFFSET_BITMASK))) 735 return rte_flow_error_set 736 (error, EINVAL, 737 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 738 "unsupported sample field mode (BITMASK)"); 739 if (field->field_base / CHAR_BIT >= 0 && 740 field->field_base / CHAR_BIT > attr->max_sample_base_offset) 741 return rte_flow_error_set 742 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, 743 "sample field base exceeds limit"); 744 break; 745 default: 746 return rte_flow_error_set 747 (error, EINVAL, 748 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 749 "unknown data sample field mode"); 750 } 751 if (!match) { 752 if (!field->field_size) 753 return rte_flow_error_set 754 (error, EINVAL, 755 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 756 "zero sample field width"); 757 if (field->field_id) 758 DRV_LOG(DEBUG, "sample field id hint ignored"); 759 } else { 760 if (field->field_mode != match->field_mode || 761 field->offset_base | match->offset_base || 762 field->offset_mask | match->offset_mask || 763 field->offset_shift | match->offset_shift) 764 return 0; 765 } 766 start = field->field_base; 767 end = start + field->field_size; 768 /* Add the new or similar field to interval array. */ 769 if (!cover->num) { 770 cover->start[cover->num] = start; 771 cover->end[cover->num] = end; 772 cover->num = 1; 773 return 1; 774 } 775 for (i = 0; i < cover->num; i++) { 776 if (start > cover->end[i]) { 777 if (i >= (cover->num - 1u)) { 778 mlx5_flex_insert_field(cover, cover->num, 779 start, end); 780 break; 781 } 782 continue; 783 } 784 if (end < cover->start[i]) { 785 mlx5_flex_insert_field(cover, i, start, end); 786 break; 787 } 788 if (start < cover->start[i]) 789 cover->start[i] = start; 790 if (end > cover->end[i]) { 791 cover->end[i] = end; 792 if (i < (cover->num - 1u)) 793 mlx5_flex_merge_field(cover, i); 794 } 795 break; 796 } 797 return 1; 798 } 799 800 static void 801 mlx5_flex_config_sample(struct mlx5_devx_match_sample_attr *na, 802 struct rte_flow_item_flex_field *field, 803 enum rte_flow_item_flex_tunnel_mode tunnel_mode) 804 { 805 memset(na, 0, sizeof(struct mlx5_devx_match_sample_attr)); 806 na->flow_match_sample_en = 1; 807 switch (field->field_mode) { 808 case FIELD_MODE_FIXED: 809 na->flow_match_sample_offset_mode = 810 MLX5_GRAPH_SAMPLE_OFFSET_FIXED; 811 break; 812 case FIELD_MODE_OFFSET: 813 na->flow_match_sample_offset_mode = 814 MLX5_GRAPH_SAMPLE_OFFSET_FIELD; 815 na->flow_match_sample_field_offset = field->offset_base; 816 na->flow_match_sample_field_offset_mask = field->offset_mask; 817 na->flow_match_sample_field_offset_shift = field->offset_shift; 818 break; 819 case FIELD_MODE_BITMASK: 820 na->flow_match_sample_offset_mode = 821 MLX5_GRAPH_SAMPLE_OFFSET_BITMASK; 822 na->flow_match_sample_field_offset = field->offset_base; 823 na->flow_match_sample_field_offset_mask = field->offset_mask; 824 na->flow_match_sample_field_offset_shift = field->offset_shift; 825 break; 826 default: 827 MLX5_ASSERT(false); 828 break; 829 } 830 switch (tunnel_mode) { 831 case FLEX_TUNNEL_MODE_SINGLE: 832 /* Fallthrough */ 833 case FLEX_TUNNEL_MODE_TUNNEL: 834 na->flow_match_sample_tunnel_mode = 835 MLX5_GRAPH_SAMPLE_TUNNEL_FIRST; 836 break; 837 case FLEX_TUNNEL_MODE_MULTI: 838 /* Fallthrough */ 839 case FLEX_TUNNEL_MODE_OUTER: 840 na->flow_match_sample_tunnel_mode = 841 MLX5_GRAPH_SAMPLE_TUNNEL_OUTER; 842 break; 843 case FLEX_TUNNEL_MODE_INNER: 844 na->flow_match_sample_tunnel_mode = 845 MLX5_GRAPH_SAMPLE_TUNNEL_INNER; 846 break; 847 default: 848 MLX5_ASSERT(false); 849 break; 850 } 851 } 852 853 /* Map specified field to set/subset of allocated sample registers. */ 854 static int 855 mlx5_flex_map_sample(struct rte_flow_item_flex_field *field, 856 struct mlx5_flex_parser_devx *parser, 857 struct mlx5_flex_item *item, 858 struct rte_flow_error *error) 859 { 860 struct mlx5_devx_match_sample_attr node; 861 int32_t start = field->field_base; 862 int32_t end = start + field->field_size; 863 struct mlx5_flex_pattern_field *trans; 864 uint32_t i, done_bits = 0; 865 866 if (field->field_mode == FIELD_MODE_DUMMY) { 867 done_bits = field->field_size; 868 while (done_bits) { 869 uint32_t part = RTE_MIN(done_bits, 870 sizeof(uint32_t) * CHAR_BIT); 871 if (item->mapnum >= MLX5_FLEX_ITEM_MAPPING_NUM) 872 return rte_flow_error_set 873 (error, 874 EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, 875 "too many flex item pattern translations"); 876 trans = &item->map[item->mapnum]; 877 trans->reg_id = MLX5_INVALID_SAMPLE_REG_ID; 878 trans->shift = 0; 879 trans->width = part; 880 item->mapnum++; 881 done_bits -= part; 882 } 883 return 0; 884 } 885 mlx5_flex_config_sample(&node, field, item->tunnel_mode); 886 for (i = 0; i < parser->num_samples; i++) { 887 struct mlx5_devx_match_sample_attr *sample = 888 &parser->devx_conf.sample[i]; 889 int32_t reg_start, reg_end; 890 int32_t cov_start, cov_end; 891 892 MLX5_ASSERT(sample->flow_match_sample_en); 893 if (!sample->flow_match_sample_en) 894 break; 895 node.flow_match_sample_field_base_offset = 896 sample->flow_match_sample_field_base_offset; 897 if (memcmp(&node, sample, sizeof(node))) 898 continue; 899 reg_start = (int8_t)sample->flow_match_sample_field_base_offset; 900 reg_start *= CHAR_BIT; 901 reg_end = reg_start + 32; 902 if (end <= reg_start || start >= reg_end) 903 continue; 904 cov_start = RTE_MAX(reg_start, start); 905 cov_end = RTE_MIN(reg_end, end); 906 MLX5_ASSERT(cov_end > cov_start); 907 done_bits += cov_end - cov_start; 908 if (item->mapnum >= MLX5_FLEX_ITEM_MAPPING_NUM) 909 return rte_flow_error_set 910 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, 911 "too many flex item pattern translations"); 912 trans = &item->map[item->mapnum]; 913 item->mapnum++; 914 trans->reg_id = i; 915 trans->shift = cov_start - reg_start; 916 trans->width = cov_end - cov_start; 917 } 918 if (done_bits != field->field_size) { 919 MLX5_ASSERT(false); 920 return rte_flow_error_set 921 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, 922 "failed to map field to sample register"); 923 } 924 return 0; 925 } 926 927 /* Allocate sample registers for the specified field type and interval array. */ 928 static int 929 mlx5_flex_alloc_sample(struct mlx5_flex_field_cover *cover, 930 struct mlx5_flex_parser_devx *parser, 931 struct mlx5_flex_item *item, 932 struct rte_flow_item_flex_field *field, 933 struct mlx5_hca_flex_attr *attr, 934 struct rte_flow_error *error) 935 { 936 struct mlx5_devx_match_sample_attr node; 937 uint32_t idx = 0; 938 939 mlx5_flex_config_sample(&node, field, item->tunnel_mode); 940 while (idx < cover->num) { 941 int32_t start, end; 942 943 /* 944 * Sample base offsets are in bytes, should be aligned 945 * to 32-bit as required by firmware for samples. 946 */ 947 start = RTE_ALIGN_FLOOR(cover->start[idx], 948 sizeof(uint32_t) * CHAR_BIT); 949 node.flow_match_sample_field_base_offset = 950 (start / CHAR_BIT) & 0xFF; 951 /* Allocate sample register. */ 952 if (parser->num_samples >= MLX5_GRAPH_NODE_SAMPLE_NUM || 953 parser->num_samples >= attr->max_num_sample || 954 parser->num_samples >= attr->max_num_prog_sample) 955 return rte_flow_error_set 956 (error, EINVAL, 957 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 958 "no sample registers to handle all flex item fields"); 959 parser->devx_conf.sample[parser->num_samples] = node; 960 parser->num_samples++; 961 /* Remove or update covered intervals. */ 962 end = start + 32; 963 while (idx < cover->num) { 964 if (end >= cover->end[idx]) { 965 idx++; 966 continue; 967 } 968 if (end > cover->start[idx]) 969 cover->start[idx] = end; 970 break; 971 } 972 } 973 return 0; 974 } 975 976 static int 977 mlx5_flex_translate_sample(struct mlx5_hca_flex_attr *attr, 978 const struct rte_flow_item_flex_conf *conf, 979 struct mlx5_flex_parser_devx *parser, 980 struct mlx5_flex_item *item, 981 struct rte_flow_error *error) 982 { 983 struct mlx5_flex_field_cover cover; 984 uint32_t i, j; 985 int ret; 986 987 switch (conf->tunnel) { 988 case FLEX_TUNNEL_MODE_SINGLE: 989 /* Fallthrough */ 990 case FLEX_TUNNEL_MODE_OUTER: 991 /* Fallthrough */ 992 case FLEX_TUNNEL_MODE_INNER: 993 /* Fallthrough */ 994 case FLEX_TUNNEL_MODE_MULTI: 995 /* Fallthrough */ 996 case FLEX_TUNNEL_MODE_TUNNEL: 997 break; 998 default: 999 return rte_flow_error_set 1000 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, 1001 "unrecognized tunnel mode"); 1002 } 1003 item->tunnel_mode = conf->tunnel; 1004 if (conf->nb_samples > MLX5_FLEX_ITEM_MAPPING_NUM) 1005 return rte_flow_error_set 1006 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, 1007 "sample field number exceeds limit"); 1008 /* 1009 * The application can specify fields smaller or bigger than 32 bits 1010 * covered with single sample register and it can specify field 1011 * offsets in any order. 1012 * 1013 * Gather all similar fields together, build array of bit intervals 1014 * in ascending order and try to cover with the smallest set of sample 1015 * registers. 1016 */ 1017 memset(&cover, 0, sizeof(cover)); 1018 for (i = 0; i < conf->nb_samples; i++) { 1019 struct rte_flow_item_flex_field *fl = conf->sample_data + i; 1020 1021 /* Check whether field was covered in the previous iteration. */ 1022 if (cover.mapped[i / CHAR_BIT] & (1u << (i % CHAR_BIT))) 1023 continue; 1024 if (fl->field_mode == FIELD_MODE_DUMMY) 1025 continue; 1026 /* Build an interval array for the field and similar ones */ 1027 cover.num = 0; 1028 /* Add the first field to array unconditionally. */ 1029 ret = mlx5_flex_cover_sample(&cover, fl, NULL, attr, error); 1030 if (ret < 0) 1031 return ret; 1032 MLX5_ASSERT(ret > 0); 1033 cover.mapped[i / CHAR_BIT] |= 1u << (i % CHAR_BIT); 1034 for (j = i + 1; j < conf->nb_samples; j++) { 1035 struct rte_flow_item_flex_field *ft; 1036 1037 /* Add field to array if its type matches. */ 1038 ft = conf->sample_data + j; 1039 ret = mlx5_flex_cover_sample(&cover, ft, fl, 1040 attr, error); 1041 if (ret < 0) 1042 return ret; 1043 if (!ret) 1044 continue; 1045 cover.mapped[j / CHAR_BIT] |= 1u << (j % CHAR_BIT); 1046 } 1047 /* Allocate sample registers to cover array of intervals. */ 1048 ret = mlx5_flex_alloc_sample(&cover, parser, item, 1049 fl, attr, error); 1050 if (ret) 1051 return ret; 1052 } 1053 /* Build the item pattern translating data on flow creation. */ 1054 item->mapnum = 0; 1055 memset(&item->map, 0, sizeof(item->map)); 1056 for (i = 0; i < conf->nb_samples; i++) { 1057 struct rte_flow_item_flex_field *fl = conf->sample_data + i; 1058 1059 ret = mlx5_flex_map_sample(fl, parser, item, error); 1060 if (ret) { 1061 MLX5_ASSERT(false); 1062 return ret; 1063 } 1064 } 1065 if (conf->tunnel == FLEX_TUNNEL_MODE_MULTI) { 1066 /* 1067 * In FLEX_TUNNEL_MODE_MULTI tunnel mode PMD creates 2 sets 1068 * of samples. The first set is for outer and the second set 1069 * for inner flex flow item. Outer and inner samples differ 1070 * only in tunnel_mode. 1071 */ 1072 if (parser->num_samples > MLX5_GRAPH_NODE_SAMPLE_NUM / 2) 1073 return rte_flow_error_set 1074 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, 1075 "no sample registers for inner"); 1076 rte_memcpy(parser->devx_conf.sample + parser->num_samples, 1077 parser->devx_conf.sample, 1078 parser->num_samples * 1079 sizeof(parser->devx_conf.sample[0])); 1080 for (i = 0; i < parser->num_samples; i++) { 1081 struct mlx5_devx_match_sample_attr *sm = i + 1082 parser->devx_conf.sample + parser->num_samples; 1083 1084 sm->flow_match_sample_tunnel_mode = 1085 MLX5_GRAPH_SAMPLE_TUNNEL_INNER; 1086 } 1087 parser->num_samples *= 2; 1088 } 1089 return 0; 1090 } 1091 1092 static int 1093 mlx5_flex_arc_type(enum rte_flow_item_type type, int in) 1094 { 1095 switch (type) { 1096 case RTE_FLOW_ITEM_TYPE_ETH: 1097 return MLX5_GRAPH_ARC_NODE_MAC; 1098 case RTE_FLOW_ITEM_TYPE_IPV4: 1099 return in ? MLX5_GRAPH_ARC_NODE_IP : MLX5_GRAPH_ARC_NODE_IPV4; 1100 case RTE_FLOW_ITEM_TYPE_IPV6: 1101 return in ? MLX5_GRAPH_ARC_NODE_IP : MLX5_GRAPH_ARC_NODE_IPV6; 1102 case RTE_FLOW_ITEM_TYPE_UDP: 1103 return MLX5_GRAPH_ARC_NODE_UDP; 1104 case RTE_FLOW_ITEM_TYPE_TCP: 1105 return MLX5_GRAPH_ARC_NODE_TCP; 1106 case RTE_FLOW_ITEM_TYPE_MPLS: 1107 return MLX5_GRAPH_ARC_NODE_MPLS; 1108 case RTE_FLOW_ITEM_TYPE_GRE: 1109 return MLX5_GRAPH_ARC_NODE_GRE; 1110 case RTE_FLOW_ITEM_TYPE_GENEVE: 1111 return MLX5_GRAPH_ARC_NODE_GENEVE; 1112 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 1113 return MLX5_GRAPH_ARC_NODE_VXLAN_GPE; 1114 default: 1115 return -EINVAL; 1116 } 1117 } 1118 1119 static int 1120 mlx5_flex_arc_in_eth(const struct rte_flow_item *item, 1121 struct rte_flow_error *error) 1122 { 1123 const struct rte_flow_item_eth *spec = item->spec; 1124 const struct rte_flow_item_eth *mask = item->mask; 1125 struct rte_flow_item_eth eth = { .hdr.ether_type = RTE_BE16(0xFFFF) }; 1126 1127 if (memcmp(mask, ð, sizeof(struct rte_flow_item_eth))) { 1128 return rte_flow_error_set 1129 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item, 1130 "invalid eth item mask"); 1131 } 1132 return rte_be_to_cpu_16(spec->hdr.ether_type); 1133 } 1134 1135 static int 1136 mlx5_flex_arc_in_udp(const struct rte_flow_item *item, 1137 struct rte_flow_error *error) 1138 { 1139 const struct rte_flow_item_udp *spec = item->spec; 1140 const struct rte_flow_item_udp *mask = item->mask; 1141 struct rte_flow_item_udp udp = { .hdr.dst_port = RTE_BE16(0xFFFF) }; 1142 1143 if (memcmp(mask, &udp, sizeof(struct rte_flow_item_udp))) { 1144 return rte_flow_error_set 1145 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item, 1146 "invalid eth item mask"); 1147 } 1148 return rte_be_to_cpu_16(spec->hdr.dst_port); 1149 } 1150 1151 static int 1152 mlx5_flex_arc_in_ipv6(const struct rte_flow_item *item, 1153 struct rte_flow_error *error) 1154 { 1155 const struct rte_flow_item_ipv6 *spec = item->spec; 1156 const struct rte_flow_item_ipv6 *mask = item->mask; 1157 struct rte_flow_item_ipv6 ip = { .hdr.proto = 0xff }; 1158 1159 if (memcmp(mask, &ip, sizeof(struct rte_flow_item_ipv6))) { 1160 return rte_flow_error_set 1161 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item, 1162 "invalid ipv6 item mask, full mask is desired"); 1163 } 1164 return spec->hdr.proto; 1165 } 1166 1167 static int 1168 mlx5_flex_translate_arc_in(struct mlx5_hca_flex_attr *attr, 1169 const struct rte_flow_item_flex_conf *conf, 1170 struct mlx5_flex_parser_devx *devx, 1171 struct mlx5_flex_item *item, 1172 struct rte_flow_error *error) 1173 { 1174 struct mlx5_devx_graph_node_attr *node = &devx->devx_conf; 1175 uint32_t i; 1176 1177 RTE_SET_USED(item); 1178 if (conf->nb_inputs > attr->max_num_arc_in) 1179 return rte_flow_error_set 1180 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, 1181 "too many input links"); 1182 for (i = 0; i < conf->nb_inputs; i++) { 1183 struct mlx5_devx_graph_arc_attr *arc = node->in + i; 1184 struct rte_flow_item_flex_link *link = conf->input_link + i; 1185 const struct rte_flow_item *rte_item = &link->item; 1186 int arc_type; 1187 int ret; 1188 1189 if (!rte_item->spec || !rte_item->mask || rte_item->last) 1190 return rte_flow_error_set 1191 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, 1192 "invalid flex item IN arc format"); 1193 arc_type = mlx5_flex_arc_type(rte_item->type, true); 1194 if (arc_type < 0 || !(attr->node_in & RTE_BIT32(arc_type))) 1195 return rte_flow_error_set 1196 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, 1197 "unsupported flex item IN arc type"); 1198 arc->arc_parse_graph_node = arc_type; 1199 arc->start_inner_tunnel = 0; 1200 /* 1201 * Configure arc IN condition value. The value location depends 1202 * on protocol. Current FW version supports IP & UDP for IN 1203 * arcs only, and locations for these protocols are defined. 1204 * Add more protocols when available. 1205 */ 1206 switch (rte_item->type) { 1207 case RTE_FLOW_ITEM_TYPE_ETH: 1208 ret = mlx5_flex_arc_in_eth(rte_item, error); 1209 break; 1210 case RTE_FLOW_ITEM_TYPE_UDP: 1211 ret = mlx5_flex_arc_in_udp(rte_item, error); 1212 break; 1213 case RTE_FLOW_ITEM_TYPE_IPV6: 1214 ret = mlx5_flex_arc_in_ipv6(rte_item, error); 1215 break; 1216 default: 1217 MLX5_ASSERT(false); 1218 return rte_flow_error_set 1219 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, 1220 "unsupported flex item IN arc type"); 1221 } 1222 if (ret < 0) 1223 return ret; 1224 arc->compare_condition_value = (uint16_t)ret; 1225 } 1226 return 0; 1227 } 1228 1229 static int 1230 mlx5_flex_translate_arc_out(struct mlx5_hca_flex_attr *attr, 1231 const struct rte_flow_item_flex_conf *conf, 1232 struct mlx5_flex_parser_devx *devx, 1233 struct mlx5_flex_item *item, 1234 struct rte_flow_error *error) 1235 { 1236 struct mlx5_devx_graph_node_attr *node = &devx->devx_conf; 1237 bool is_tunnel = conf->tunnel == FLEX_TUNNEL_MODE_TUNNEL; 1238 uint32_t i; 1239 1240 RTE_SET_USED(item); 1241 if (conf->nb_outputs > attr->max_num_arc_out) 1242 return rte_flow_error_set 1243 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, 1244 "too many output links"); 1245 for (i = 0; i < conf->nb_outputs; i++) { 1246 struct mlx5_devx_graph_arc_attr *arc = node->out + i; 1247 struct rte_flow_item_flex_link *link = conf->output_link + i; 1248 const struct rte_flow_item *rte_item = &link->item; 1249 int arc_type; 1250 1251 if (rte_item->spec || rte_item->mask || rte_item->last) 1252 return rte_flow_error_set 1253 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, 1254 "flex node: invalid OUT arc format"); 1255 arc_type = mlx5_flex_arc_type(rte_item->type, false); 1256 if (arc_type < 0 || !(attr->node_out & RTE_BIT32(arc_type))) 1257 return rte_flow_error_set 1258 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, 1259 "unsupported flex item OUT arc type"); 1260 arc->arc_parse_graph_node = arc_type; 1261 arc->start_inner_tunnel = !!is_tunnel; 1262 arc->compare_condition_value = link->next; 1263 } 1264 return 0; 1265 } 1266 1267 /* Translate RTE flex item API configuration into flaex parser settings. */ 1268 static int 1269 mlx5_flex_translate_conf(struct rte_eth_dev *dev, 1270 const struct rte_flow_item_flex_conf *conf, 1271 struct mlx5_flex_parser_devx *devx, 1272 struct mlx5_flex_item *item, 1273 struct rte_flow_error *error) 1274 { 1275 struct mlx5_priv *priv = dev->data->dev_private; 1276 struct mlx5_hca_flex_attr *attr = &priv->sh->cdev->config.hca_attr.flex; 1277 int ret; 1278 1279 ret = mlx5_flex_translate_length(attr, conf, devx, error); 1280 if (ret) 1281 return ret; 1282 ret = mlx5_flex_translate_next(attr, conf, devx, error); 1283 if (ret) 1284 return ret; 1285 ret = mlx5_flex_translate_sample(attr, conf, devx, item, error); 1286 if (ret) 1287 return ret; 1288 ret = mlx5_flex_translate_arc_in(attr, conf, devx, item, error); 1289 if (ret) 1290 return ret; 1291 ret = mlx5_flex_translate_arc_out(attr, conf, devx, item, error); 1292 if (ret) 1293 return ret; 1294 return 0; 1295 } 1296 1297 /** 1298 * Create the flex item with specified configuration over the Ethernet device. 1299 * 1300 * @param dev 1301 * Ethernet device to create flex item on. 1302 * @param[in] conf 1303 * Flex item configuration. 1304 * @param[out] error 1305 * Perform verbose error reporting if not NULL. PMDs initialize this 1306 * structure in case of error only. 1307 * 1308 * @return 1309 * Non-NULL opaque pointer on success, NULL otherwise and rte_errno is set. 1310 */ 1311 struct rte_flow_item_flex_handle * 1312 flow_dv_item_create(struct rte_eth_dev *dev, 1313 const struct rte_flow_item_flex_conf *conf, 1314 struct rte_flow_error *error) 1315 { 1316 struct mlx5_priv *priv = dev->data->dev_private; 1317 struct mlx5_flex_parser_devx devx_config = { .devx_obj = NULL }; 1318 struct mlx5_flex_item *flex; 1319 struct mlx5_list_entry *ent; 1320 1321 MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY); 1322 flex = mlx5_flex_alloc(priv); 1323 if (!flex) { 1324 rte_flow_error_set(error, ENOMEM, 1325 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 1326 "too many flex items created on the port"); 1327 return NULL; 1328 } 1329 if (mlx5_flex_translate_conf(dev, conf, &devx_config, flex, error)) 1330 goto error; 1331 ent = mlx5_list_register(priv->sh->flex_parsers_dv, &devx_config); 1332 if (!ent) { 1333 rte_flow_error_set(error, ENOMEM, 1334 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 1335 "flex item creation failure"); 1336 goto error; 1337 } 1338 flex->devx_fp = container_of(ent, struct mlx5_flex_parser_devx, entry); 1339 /* Mark initialized flex item valid. */ 1340 rte_atomic_fetch_add_explicit(&flex->refcnt, 1, rte_memory_order_release); 1341 return (struct rte_flow_item_flex_handle *)flex; 1342 1343 error: 1344 mlx5_flex_free(priv, flex); 1345 return NULL; 1346 } 1347 1348 /** 1349 * Release the flex item on the specified Ethernet device. 1350 * 1351 * @param dev 1352 * Ethernet device to destroy flex item on. 1353 * @param[in] handle 1354 * Handle of the item existing on the specified device. 1355 * @param[out] error 1356 * Perform verbose error reporting if not NULL. PMDs initialize this 1357 * structure in case of error only. 1358 * 1359 * @return 1360 * 0 on success, a negative errno value otherwise and rte_errno is set. 1361 */ 1362 int 1363 flow_dv_item_release(struct rte_eth_dev *dev, 1364 const struct rte_flow_item_flex_handle *handle, 1365 struct rte_flow_error *error) 1366 { 1367 struct mlx5_priv *priv = dev->data->dev_private; 1368 struct mlx5_flex_item *flex = 1369 (struct mlx5_flex_item *)(uintptr_t)handle; 1370 uint32_t old_refcnt = 1; 1371 int rc; 1372 1373 MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY); 1374 rte_spinlock_lock(&priv->flex_item_sl); 1375 if (mlx5_flex_index(priv, flex) < 0) { 1376 rte_spinlock_unlock(&priv->flex_item_sl); 1377 return rte_flow_error_set(error, EINVAL, 1378 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 1379 "invalid flex item handle value"); 1380 } 1381 if (!rte_atomic_compare_exchange_strong_explicit(&flex->refcnt, &old_refcnt, 0, 1382 rte_memory_order_acquire, rte_memory_order_relaxed)) { 1383 rte_spinlock_unlock(&priv->flex_item_sl); 1384 return rte_flow_error_set(error, EBUSY, 1385 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 1386 "flex item has flow references"); 1387 } 1388 /* Flex item is marked as invalid, we can leave locked section. */ 1389 rte_spinlock_unlock(&priv->flex_item_sl); 1390 MLX5_ASSERT(flex->devx_fp); 1391 rc = mlx5_list_unregister(priv->sh->flex_parsers_dv, 1392 &flex->devx_fp->entry); 1393 flex->devx_fp = NULL; 1394 mlx5_flex_free(priv, flex); 1395 if (rc < 0) 1396 return rte_flow_error_set(error, EBUSY, 1397 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 1398 "flex item release failure"); 1399 return 0; 1400 } 1401 1402 /* DevX flex parser list callbacks. */ 1403 struct mlx5_list_entry * 1404 mlx5_flex_parser_create_cb(void *list_ctx, void *ctx) 1405 { 1406 struct mlx5_dev_ctx_shared *sh = list_ctx; 1407 struct mlx5_flex_parser_devx *fp, *conf = ctx; 1408 uint32_t i; 1409 uint8_t sample_info = sh->cdev->config.hca_attr.flex.query_match_sample_info; 1410 int ret; 1411 1412 fp = mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct mlx5_flex_parser_devx), 1413 0, SOCKET_ID_ANY); 1414 if (!fp) 1415 return NULL; 1416 /* Copy the requested configurations. */ 1417 fp->num_samples = conf->num_samples; 1418 memcpy(&fp->devx_conf, &conf->devx_conf, sizeof(fp->devx_conf)); 1419 /* Create DevX flex parser. */ 1420 fp->devx_obj = mlx5_devx_cmd_create_flex_parser(sh->cdev->ctx, 1421 &fp->devx_conf); 1422 if (!fp->devx_obj) 1423 goto error; 1424 /* Query the firmware assigned sample ids. */ 1425 ret = mlx5_devx_cmd_query_parse_samples(fp->devx_obj, 1426 fp->sample_ids, 1427 fp->num_samples, 1428 &fp->anchor_id); 1429 if (ret) 1430 goto error; 1431 /* Query sample information per ID. */ 1432 for (i = 0; i < fp->num_samples && sample_info; i++) { 1433 ret = mlx5_devx_cmd_match_sample_info_query(sh->cdev->ctx, fp->sample_ids[i], 1434 &fp->sample_info[i]); 1435 if (ret) 1436 goto error; 1437 } 1438 DRV_LOG(DEBUG, "DEVx flex parser %p created, samples num: %u", 1439 (const void *)fp, fp->num_samples); 1440 return &fp->entry; 1441 error: 1442 if (fp->devx_obj) 1443 mlx5_devx_cmd_destroy((void *)(uintptr_t)fp->devx_obj); 1444 if (fp) 1445 mlx5_free(fp); 1446 return NULL; 1447 } 1448 1449 int 1450 mlx5_flex_parser_match_cb(void *list_ctx, 1451 struct mlx5_list_entry *iter, void *ctx) 1452 { 1453 struct mlx5_flex_parser_devx *fp = 1454 container_of(iter, struct mlx5_flex_parser_devx, entry); 1455 struct mlx5_flex_parser_devx *org = 1456 container_of(ctx, struct mlx5_flex_parser_devx, entry); 1457 1458 RTE_SET_USED(list_ctx); 1459 return !iter || !ctx || memcmp(&fp->devx_conf, 1460 &org->devx_conf, 1461 sizeof(fp->devx_conf)); 1462 } 1463 1464 void 1465 mlx5_flex_parser_remove_cb(void *list_ctx, struct mlx5_list_entry *entry) 1466 { 1467 struct mlx5_flex_parser_devx *fp = 1468 container_of(entry, struct mlx5_flex_parser_devx, entry); 1469 1470 RTE_SET_USED(list_ctx); 1471 MLX5_ASSERT(fp->devx_obj); 1472 claim_zero(mlx5_devx_cmd_destroy(fp->devx_obj)); 1473 DRV_LOG(DEBUG, "DEVx flex parser %p destroyed", (const void *)fp); 1474 mlx5_free(entry); 1475 } 1476 1477 struct mlx5_list_entry * 1478 mlx5_flex_parser_clone_cb(void *list_ctx, 1479 struct mlx5_list_entry *entry, void *ctx) 1480 { 1481 struct mlx5_flex_parser_devx *fp; 1482 1483 RTE_SET_USED(list_ctx); 1484 RTE_SET_USED(entry); 1485 fp = mlx5_malloc(0, sizeof(struct mlx5_flex_parser_devx), 1486 0, SOCKET_ID_ANY); 1487 if (!fp) 1488 return NULL; 1489 memcpy(fp, ctx, sizeof(struct mlx5_flex_parser_devx)); 1490 return &fp->entry; 1491 } 1492 1493 void 1494 mlx5_flex_parser_clone_free_cb(void *list_ctx, struct mlx5_list_entry *entry) 1495 { 1496 struct mlx5_flex_parser_devx *fp = 1497 container_of(entry, struct mlx5_flex_parser_devx, entry); 1498 RTE_SET_USED(list_ctx); 1499 mlx5_free(fp); 1500 } 1501