1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2019 Mellanox Technologies, Ltd 3 */ 4 5 #include <unistd.h> 6 #include <string.h> 7 #include <stdio.h> 8 9 #include <rte_errno.h> 10 #include <rte_mempool.h> 11 #include <rte_class.h> 12 #include <rte_malloc.h> 13 14 #include "mlx5_common.h" 15 #include "mlx5_common_os.h" 16 #include "mlx5_common_log.h" 17 #include "mlx5_common_pci.h" 18 #include "mlx5_common_private.h" 19 20 uint8_t haswell_broadwell_cpu; 21 22 /* In case this is an x86_64 intel processor to check if 23 * we should use relaxed ordering. 24 */ 25 #ifdef RTE_ARCH_X86_64 26 /** 27 * This function returns processor identification and feature information 28 * into the registers. 29 * 30 * @param eax, ebx, ecx, edx 31 * Pointers to the registers that will hold cpu information. 32 * @param level 33 * The main category of information returned. 34 */ 35 static inline void mlx5_cpu_id(unsigned int level, 36 unsigned int *eax, unsigned int *ebx, 37 unsigned int *ecx, unsigned int *edx) 38 { 39 __asm__("cpuid\n\t" 40 : "=a" (*eax), "=b" (*ebx), "=c" (*ecx), "=d" (*edx) 41 : "0" (level)); 42 } 43 #endif 44 45 RTE_LOG_REGISTER_DEFAULT(mlx5_common_logtype, NOTICE) 46 47 /* Head of list of drivers. */ 48 static TAILQ_HEAD(mlx5_drivers, mlx5_class_driver) drivers_list = 49 TAILQ_HEAD_INITIALIZER(drivers_list); 50 51 /* Head of devices. */ 52 static TAILQ_HEAD(mlx5_devices, mlx5_common_device) devices_list = 53 TAILQ_HEAD_INITIALIZER(devices_list); 54 55 static const struct { 56 const char *name; 57 unsigned int drv_class; 58 } mlx5_classes[] = { 59 { .name = "vdpa", .drv_class = MLX5_CLASS_VDPA }, 60 { .name = "eth", .drv_class = MLX5_CLASS_ETH }, 61 /* Keep class "net" for backward compatibility. */ 62 { .name = "net", .drv_class = MLX5_CLASS_ETH }, 63 { .name = "regex", .drv_class = MLX5_CLASS_REGEX }, 64 { .name = "compress", .drv_class = MLX5_CLASS_COMPRESS }, 65 { .name = "crypto", .drv_class = MLX5_CLASS_CRYPTO }, 66 }; 67 68 static int 69 class_name_to_value(const char *class_name) 70 { 71 unsigned int i; 72 73 for (i = 0; i < RTE_DIM(mlx5_classes); i++) { 74 if (strcmp(class_name, mlx5_classes[i].name) == 0) 75 return mlx5_classes[i].drv_class; 76 } 77 return -EINVAL; 78 } 79 80 static struct mlx5_class_driver * 81 driver_get(uint32_t class) 82 { 83 struct mlx5_class_driver *driver; 84 85 TAILQ_FOREACH(driver, &drivers_list, next) { 86 if ((uint32_t)driver->drv_class == class) 87 return driver; 88 } 89 return NULL; 90 } 91 92 static int 93 devargs_class_handler(__rte_unused const char *key, 94 const char *class_names, void *opaque) 95 { 96 int *ret = opaque; 97 int class_val; 98 char *scratch; 99 char *found; 100 char *refstr = NULL; 101 102 *ret = 0; 103 scratch = strdup(class_names); 104 if (scratch == NULL) { 105 *ret = -ENOMEM; 106 return *ret; 107 } 108 found = strtok_r(scratch, ":", &refstr); 109 if (found == NULL) 110 /* Empty string. */ 111 goto err; 112 do { 113 /* Extract each individual class name. Multiple 114 * classes can be supplied as class=net:regex:foo:bar. 115 */ 116 class_val = class_name_to_value(found); 117 /* Check if its a valid class. */ 118 if (class_val < 0) { 119 *ret = -EINVAL; 120 goto err; 121 } 122 *ret |= class_val; 123 found = strtok_r(NULL, ":", &refstr); 124 } while (found != NULL); 125 err: 126 free(scratch); 127 if (*ret < 0) 128 DRV_LOG(ERR, "Invalid mlx5 class options: %s.\n", class_names); 129 return *ret; 130 } 131 132 static int 133 parse_class_options(const struct rte_devargs *devargs) 134 { 135 struct rte_kvargs *kvlist; 136 int ret = 0; 137 138 if (devargs == NULL) 139 return 0; 140 if (devargs->cls != NULL && devargs->cls->name != NULL) 141 /* Global syntax, only one class type. */ 142 return class_name_to_value(devargs->cls->name); 143 /* Legacy devargs support multiple classes. */ 144 kvlist = rte_kvargs_parse(devargs->args, NULL); 145 if (kvlist == NULL) 146 return 0; 147 rte_kvargs_process(kvlist, RTE_DEVARGS_KEY_CLASS, 148 devargs_class_handler, &ret); 149 rte_kvargs_free(kvlist); 150 return ret; 151 } 152 153 static const unsigned int mlx5_class_invalid_combinations[] = { 154 MLX5_CLASS_ETH | MLX5_CLASS_VDPA, 155 /* New class combination should be added here. */ 156 }; 157 158 static int 159 is_valid_class_combination(uint32_t user_classes) 160 { 161 unsigned int i; 162 163 /* Verify if user specified unsupported combination. */ 164 for (i = 0; i < RTE_DIM(mlx5_class_invalid_combinations); i++) { 165 if ((mlx5_class_invalid_combinations[i] & user_classes) == 166 mlx5_class_invalid_combinations[i]) 167 return -EINVAL; 168 } 169 /* Not found any invalid class combination. */ 170 return 0; 171 } 172 173 static bool 174 device_class_enabled(const struct mlx5_common_device *device, uint32_t class) 175 { 176 return (device->classes_loaded & class) > 0; 177 } 178 179 static bool 180 mlx5_bus_match(const struct mlx5_class_driver *drv, 181 const struct rte_device *dev) 182 { 183 if (mlx5_dev_is_pci(dev)) 184 return mlx5_dev_pci_match(drv, dev); 185 return true; 186 } 187 188 static struct mlx5_common_device * 189 to_mlx5_device(const struct rte_device *rte_dev) 190 { 191 struct mlx5_common_device *dev; 192 193 TAILQ_FOREACH(dev, &devices_list, next) { 194 if (rte_dev == dev->dev) 195 return dev; 196 } 197 return NULL; 198 } 199 200 int 201 mlx5_dev_to_pci_str(const struct rte_device *dev, char *addr, size_t size) 202 { 203 struct rte_pci_addr pci_addr = { 0 }; 204 int ret; 205 206 if (mlx5_dev_is_pci(dev)) { 207 /* Input might be <BDF>, format PCI address to <DBDF>. */ 208 ret = rte_pci_addr_parse(dev->name, &pci_addr); 209 if (ret != 0) 210 return -ENODEV; 211 rte_pci_device_name(&pci_addr, addr, size); 212 return 0; 213 } 214 #ifdef RTE_EXEC_ENV_LINUX 215 return mlx5_auxiliary_get_pci_str(RTE_DEV_TO_AUXILIARY_CONST(dev), 216 addr, size); 217 #else 218 rte_errno = ENODEV; 219 return -rte_errno; 220 #endif 221 } 222 223 static void 224 dev_release(struct mlx5_common_device *dev) 225 { 226 TAILQ_REMOVE(&devices_list, dev, next); 227 rte_free(dev); 228 } 229 230 static int 231 drivers_remove(struct mlx5_common_device *dev, uint32_t enabled_classes) 232 { 233 struct mlx5_class_driver *driver; 234 int local_ret = -ENODEV; 235 unsigned int i = 0; 236 int ret = 0; 237 238 enabled_classes &= dev->classes_loaded; 239 while (enabled_classes) { 240 driver = driver_get(RTE_BIT64(i)); 241 if (driver != NULL) { 242 local_ret = driver->remove(dev->dev); 243 if (local_ret == 0) 244 dev->classes_loaded &= ~RTE_BIT64(i); 245 else if (ret == 0) 246 ret = local_ret; 247 } 248 enabled_classes &= ~RTE_BIT64(i); 249 i++; 250 } 251 if (local_ret != 0 && ret == 0) 252 ret = local_ret; 253 return ret; 254 } 255 256 static int 257 drivers_probe(struct mlx5_common_device *dev, uint32_t user_classes) 258 { 259 struct mlx5_class_driver *driver; 260 uint32_t enabled_classes = 0; 261 bool already_loaded; 262 int ret; 263 264 TAILQ_FOREACH(driver, &drivers_list, next) { 265 if ((driver->drv_class & user_classes) == 0) 266 continue; 267 if (!mlx5_bus_match(driver, dev->dev)) 268 continue; 269 already_loaded = dev->classes_loaded & driver->drv_class; 270 if (already_loaded && driver->probe_again == 0) { 271 DRV_LOG(ERR, "Device %s is already probed", 272 dev->dev->name); 273 ret = -EEXIST; 274 goto probe_err; 275 } 276 ret = driver->probe(dev->dev); 277 if (ret < 0) { 278 DRV_LOG(ERR, "Failed to load driver %s", 279 driver->name); 280 goto probe_err; 281 } 282 enabled_classes |= driver->drv_class; 283 } 284 dev->classes_loaded |= enabled_classes; 285 return 0; 286 probe_err: 287 /* Only unload drivers which are enabled which were enabled 288 * in this probe instance. 289 */ 290 drivers_remove(dev, enabled_classes); 291 return ret; 292 } 293 294 int 295 mlx5_common_dev_probe(struct rte_device *eal_dev) 296 { 297 struct mlx5_common_device *dev; 298 uint32_t classes = 0; 299 bool new_device = false; 300 int ret; 301 302 DRV_LOG(INFO, "probe device \"%s\".", eal_dev->name); 303 ret = parse_class_options(eal_dev->devargs); 304 if (ret < 0) { 305 DRV_LOG(ERR, "Unsupported mlx5 class type: %s", 306 eal_dev->devargs->args); 307 return ret; 308 } 309 classes = ret; 310 if (classes == 0) 311 /* Default to net class. */ 312 classes = MLX5_CLASS_ETH; 313 dev = to_mlx5_device(eal_dev); 314 if (!dev) { 315 dev = rte_zmalloc("mlx5_common_device", sizeof(*dev), 0); 316 if (!dev) 317 return -ENOMEM; 318 dev->dev = eal_dev; 319 TAILQ_INSERT_HEAD(&devices_list, dev, next); 320 new_device = true; 321 } else { 322 /* Validate combination here. */ 323 ret = is_valid_class_combination(classes | 324 dev->classes_loaded); 325 if (ret != 0) { 326 DRV_LOG(ERR, "Unsupported mlx5 classes combination."); 327 return ret; 328 } 329 } 330 ret = drivers_probe(dev, classes); 331 if (ret) 332 goto class_err; 333 return 0; 334 class_err: 335 if (new_device) 336 dev_release(dev); 337 return ret; 338 } 339 340 int 341 mlx5_common_dev_remove(struct rte_device *eal_dev) 342 { 343 struct mlx5_common_device *dev; 344 int ret; 345 346 dev = to_mlx5_device(eal_dev); 347 if (!dev) 348 return -ENODEV; 349 /* Matching device found, cleanup and unload drivers. */ 350 ret = drivers_remove(dev, dev->classes_loaded); 351 if (ret != 0) 352 dev_release(dev); 353 return ret; 354 } 355 356 int 357 mlx5_common_dev_dma_map(struct rte_device *dev, void *addr, uint64_t iova, 358 size_t len) 359 { 360 struct mlx5_class_driver *driver = NULL; 361 struct mlx5_class_driver *temp; 362 struct mlx5_common_device *mdev; 363 int ret = -EINVAL; 364 365 mdev = to_mlx5_device(dev); 366 if (!mdev) 367 return -ENODEV; 368 TAILQ_FOREACH(driver, &drivers_list, next) { 369 if (!device_class_enabled(mdev, driver->drv_class) || 370 driver->dma_map == NULL) 371 continue; 372 ret = driver->dma_map(dev, addr, iova, len); 373 if (ret) 374 goto map_err; 375 } 376 return ret; 377 map_err: 378 TAILQ_FOREACH(temp, &drivers_list, next) { 379 if (temp == driver) 380 break; 381 if (device_class_enabled(mdev, temp->drv_class) && 382 temp->dma_map && temp->dma_unmap) 383 temp->dma_unmap(dev, addr, iova, len); 384 } 385 return ret; 386 } 387 388 int 389 mlx5_common_dev_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova, 390 size_t len) 391 { 392 struct mlx5_class_driver *driver; 393 struct mlx5_common_device *mdev; 394 int local_ret = -EINVAL; 395 int ret = 0; 396 397 mdev = to_mlx5_device(dev); 398 if (!mdev) 399 return -ENODEV; 400 /* There is no unmap error recovery in current implementation. */ 401 TAILQ_FOREACH_REVERSE(driver, &drivers_list, mlx5_drivers, next) { 402 if (!device_class_enabled(mdev, driver->drv_class) || 403 driver->dma_unmap == NULL) 404 continue; 405 local_ret = driver->dma_unmap(dev, addr, iova, len); 406 if (local_ret && (ret == 0)) 407 ret = local_ret; 408 } 409 if (local_ret) 410 ret = local_ret; 411 return ret; 412 } 413 414 void 415 mlx5_class_driver_register(struct mlx5_class_driver *driver) 416 { 417 mlx5_common_driver_on_register_pci(driver); 418 TAILQ_INSERT_TAIL(&drivers_list, driver, next); 419 } 420 421 static void mlx5_common_driver_init(void) 422 { 423 mlx5_common_pci_init(); 424 #ifdef RTE_EXEC_ENV_LINUX 425 mlx5_common_auxiliary_init(); 426 #endif 427 } 428 429 static bool mlx5_common_initialized; 430 431 /** 432 * One time innitialization routine for run-time dependency on glue library 433 * for multiple PMDs. Each mlx5 PMD that depends on mlx5_common module, 434 * must invoke in its constructor. 435 */ 436 void 437 mlx5_common_init(void) 438 { 439 if (mlx5_common_initialized) 440 return; 441 442 mlx5_glue_constructor(); 443 mlx5_common_driver_init(); 444 mlx5_common_initialized = true; 445 } 446 447 /** 448 * This function is responsible of initializing the variable 449 * haswell_broadwell_cpu by checking if the cpu is intel 450 * and reading the data returned from mlx5_cpu_id(). 451 * since haswell and broadwell cpus don't have improved performance 452 * when using relaxed ordering we want to check the cpu type before 453 * before deciding whether to enable RO or not. 454 * if the cpu is haswell or broadwell the variable will be set to 1 455 * otherwise it will be 0. 456 */ 457 RTE_INIT_PRIO(mlx5_is_haswell_broadwell_cpu, LOG) 458 { 459 #ifdef RTE_ARCH_X86_64 460 unsigned int broadwell_models[4] = {0x3d, 0x47, 0x4F, 0x56}; 461 unsigned int haswell_models[4] = {0x3c, 0x3f, 0x45, 0x46}; 462 unsigned int i, model, family, brand_id, vendor; 463 unsigned int signature_intel_ebx = 0x756e6547; 464 unsigned int extended_model; 465 unsigned int eax = 0; 466 unsigned int ebx = 0; 467 unsigned int ecx = 0; 468 unsigned int edx = 0; 469 int max_level; 470 471 mlx5_cpu_id(0, &eax, &ebx, &ecx, &edx); 472 vendor = ebx; 473 max_level = eax; 474 if (max_level < 1) { 475 haswell_broadwell_cpu = 0; 476 return; 477 } 478 mlx5_cpu_id(1, &eax, &ebx, &ecx, &edx); 479 model = (eax >> 4) & 0x0f; 480 family = (eax >> 8) & 0x0f; 481 brand_id = ebx & 0xff; 482 extended_model = (eax >> 12) & 0xf0; 483 /* Check if the processor is Haswell or Broadwell */ 484 if (vendor == signature_intel_ebx) { 485 if (family == 0x06) 486 model += extended_model; 487 if (brand_id == 0 && family == 0x6) { 488 for (i = 0; i < RTE_DIM(broadwell_models); i++) 489 if (model == broadwell_models[i]) { 490 haswell_broadwell_cpu = 1; 491 return; 492 } 493 for (i = 0; i < RTE_DIM(haswell_models); i++) 494 if (model == haswell_models[i]) { 495 haswell_broadwell_cpu = 1; 496 return; 497 } 498 } 499 } 500 #endif 501 haswell_broadwell_cpu = 0; 502 } 503 504 /** 505 * Allocate the User Access Region with DevX on specified device. 506 * 507 * @param [in] ctx 508 * Infiniband device context to perform allocation on. 509 * @param [in] mapping 510 * MLX5DV_UAR_ALLOC_TYPE_BF - allocate as cached memory with write-combining 511 * attributes (if supported by the host), the 512 * writes to the UAR registers must be followed 513 * by write memory barrier. 514 * MLX5DV_UAR_ALLOC_TYPE_NC - allocate as non-cached nenory, all writes are 515 * promoted to the registers immediately, no 516 * memory barriers needed. 517 * mapping < 0 - the first attempt is performed with MLX5DV_UAR_ALLOC_TYPE_BF, 518 * if this fails the next attempt with MLX5DV_UAR_ALLOC_TYPE_NC 519 * is performed. The drivers specifying negative values should 520 * always provide the write memory barrier operation after UAR 521 * register writings. 522 * If there is no definitions for the MLX5DV_UAR_ALLOC_TYPE_xx (older rdma 523 * library headers), the caller can specify 0. 524 * 525 * @return 526 * UAR object pointer on success, NULL otherwise and rte_errno is set. 527 */ 528 void * 529 mlx5_devx_alloc_uar(void *ctx, int mapping) 530 { 531 void *uar; 532 uint32_t retry, uar_mapping; 533 void *base_addr; 534 535 for (retry = 0; retry < MLX5_ALLOC_UAR_RETRY; ++retry) { 536 #ifdef MLX5DV_UAR_ALLOC_TYPE_NC 537 /* Control the mapping type according to the settings. */ 538 uar_mapping = (mapping < 0) ? 539 MLX5DV_UAR_ALLOC_TYPE_NC : mapping; 540 #else 541 /* 542 * It seems we have no way to control the memory mapping type 543 * for the UAR, the default "Write-Combining" type is supposed. 544 */ 545 uar_mapping = 0; 546 RTE_SET_USED(mapping); 547 #endif 548 uar = mlx5_glue->devx_alloc_uar(ctx, uar_mapping); 549 #ifdef MLX5DV_UAR_ALLOC_TYPE_NC 550 if (!uar && 551 mapping < 0 && 552 uar_mapping == MLX5DV_UAR_ALLOC_TYPE_BF) { 553 /* 554 * In some environments like virtual machine the 555 * Write Combining mapped might be not supported and 556 * UAR allocation fails. We tried "Non-Cached" mapping 557 * for the case. 558 */ 559 DRV_LOG(WARNING, "Failed to allocate DevX UAR (BF)"); 560 uar_mapping = MLX5DV_UAR_ALLOC_TYPE_NC; 561 uar = mlx5_glue->devx_alloc_uar(ctx, uar_mapping); 562 } else if (!uar && 563 mapping < 0 && 564 uar_mapping == MLX5DV_UAR_ALLOC_TYPE_NC) { 565 /* 566 * If Verbs/kernel does not support "Non-Cached" 567 * try the "Write-Combining". 568 */ 569 DRV_LOG(WARNING, "Failed to allocate DevX UAR (NC)"); 570 uar_mapping = MLX5DV_UAR_ALLOC_TYPE_BF; 571 uar = mlx5_glue->devx_alloc_uar(ctx, uar_mapping); 572 } 573 #endif 574 if (!uar) { 575 DRV_LOG(ERR, "Failed to allocate DevX UAR (BF/NC)"); 576 rte_errno = ENOMEM; 577 goto exit; 578 } 579 base_addr = mlx5_os_get_devx_uar_base_addr(uar); 580 if (base_addr) 581 break; 582 /* 583 * The UARs are allocated by rdma_core within the 584 * IB device context, on context closure all UARs 585 * will be freed, should be no memory/object leakage. 586 */ 587 DRV_LOG(WARNING, "Retrying to allocate DevX UAR"); 588 uar = NULL; 589 } 590 /* Check whether we finally succeeded with valid UAR allocation. */ 591 if (!uar) { 592 DRV_LOG(ERR, "Failed to allocate DevX UAR (NULL base)"); 593 rte_errno = ENOMEM; 594 } 595 /* 596 * Return void * instead of struct mlx5dv_devx_uar * 597 * is for compatibility with older rdma-core library headers. 598 */ 599 exit: 600 return uar; 601 } 602 603 RTE_PMD_EXPORT_NAME(mlx5_common_driver, __COUNTER__); 604