1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2019 Mellanox Technologies, Ltd 3 */ 4 5 #include <unistd.h> 6 #include <string.h> 7 #include <stdio.h> 8 9 #include <rte_errno.h> 10 #include <rte_mempool.h> 11 #include <rte_class.h> 12 #include <rte_malloc.h> 13 14 #include "mlx5_common.h" 15 #include "mlx5_common_os.h" 16 #include "mlx5_common_log.h" 17 #include "mlx5_common_private.h" 18 19 uint8_t haswell_broadwell_cpu; 20 21 /* In case this is an x86_64 intel processor to check if 22 * we should use relaxed ordering. 23 */ 24 #ifdef RTE_ARCH_X86_64 25 /** 26 * This function returns processor identification and feature information 27 * into the registers. 28 * 29 * @param eax, ebx, ecx, edx 30 * Pointers to the registers that will hold cpu information. 31 * @param level 32 * The main category of information returned. 33 */ 34 static inline void mlx5_cpu_id(unsigned int level, 35 unsigned int *eax, unsigned int *ebx, 36 unsigned int *ecx, unsigned int *edx) 37 { 38 __asm__("cpuid\n\t" 39 : "=a" (*eax), "=b" (*ebx), "=c" (*ecx), "=d" (*edx) 40 : "0" (level)); 41 } 42 #endif 43 44 RTE_LOG_REGISTER_DEFAULT(mlx5_common_logtype, NOTICE) 45 46 /* Head of list of drivers. */ 47 static TAILQ_HEAD(mlx5_drivers, mlx5_class_driver) drivers_list = 48 TAILQ_HEAD_INITIALIZER(drivers_list); 49 50 /* Head of devices. */ 51 static TAILQ_HEAD(mlx5_devices, mlx5_common_device) devices_list = 52 TAILQ_HEAD_INITIALIZER(devices_list); 53 54 static const struct { 55 const char *name; 56 unsigned int drv_class; 57 } mlx5_classes[] = { 58 { .name = "vdpa", .drv_class = MLX5_CLASS_VDPA }, 59 { .name = "eth", .drv_class = MLX5_CLASS_ETH }, 60 /* Keep class "net" for backward compatibility. */ 61 { .name = "net", .drv_class = MLX5_CLASS_ETH }, 62 { .name = "regex", .drv_class = MLX5_CLASS_REGEX }, 63 { .name = "compress", .drv_class = MLX5_CLASS_COMPRESS }, 64 { .name = "crypto", .drv_class = MLX5_CLASS_CRYPTO }, 65 }; 66 67 static int 68 class_name_to_value(const char *class_name) 69 { 70 unsigned int i; 71 72 for (i = 0; i < RTE_DIM(mlx5_classes); i++) { 73 if (strcmp(class_name, mlx5_classes[i].name) == 0) 74 return mlx5_classes[i].drv_class; 75 } 76 return -EINVAL; 77 } 78 79 static struct mlx5_class_driver * 80 driver_get(uint32_t class) 81 { 82 struct mlx5_class_driver *driver; 83 84 TAILQ_FOREACH(driver, &drivers_list, next) { 85 if ((uint32_t)driver->drv_class == class) 86 return driver; 87 } 88 return NULL; 89 } 90 91 static int 92 devargs_class_handler(__rte_unused const char *key, 93 const char *class_names, void *opaque) 94 { 95 int *ret = opaque; 96 int class_val; 97 char *scratch; 98 char *found; 99 char *refstr = NULL; 100 101 *ret = 0; 102 scratch = strdup(class_names); 103 if (scratch == NULL) { 104 *ret = -ENOMEM; 105 return *ret; 106 } 107 found = strtok_r(scratch, ":", &refstr); 108 if (found == NULL) 109 /* Empty string. */ 110 goto err; 111 do { 112 /* Extract each individual class name. Multiple 113 * classes can be supplied as class=net:regex:foo:bar. 114 */ 115 class_val = class_name_to_value(found); 116 /* Check if its a valid class. */ 117 if (class_val < 0) { 118 *ret = -EINVAL; 119 goto err; 120 } 121 *ret |= class_val; 122 found = strtok_r(NULL, ":", &refstr); 123 } while (found != NULL); 124 err: 125 free(scratch); 126 if (*ret < 0) 127 DRV_LOG(ERR, "Invalid mlx5 class options: %s.\n", class_names); 128 return *ret; 129 } 130 131 static int 132 parse_class_options(const struct rte_devargs *devargs) 133 { 134 struct rte_kvargs *kvlist; 135 int ret = 0; 136 137 if (devargs == NULL) 138 return 0; 139 if (devargs->cls != NULL && devargs->cls->name != NULL) 140 /* Global syntax, only one class type. */ 141 return class_name_to_value(devargs->cls->name); 142 /* Legacy devargs support multiple classes. */ 143 kvlist = rte_kvargs_parse(devargs->args, NULL); 144 if (kvlist == NULL) 145 return 0; 146 rte_kvargs_process(kvlist, RTE_DEVARGS_KEY_CLASS, 147 devargs_class_handler, &ret); 148 rte_kvargs_free(kvlist); 149 return ret; 150 } 151 152 static const unsigned int mlx5_class_invalid_combinations[] = { 153 MLX5_CLASS_ETH | MLX5_CLASS_VDPA, 154 /* New class combination should be added here. */ 155 }; 156 157 static int 158 is_valid_class_combination(uint32_t user_classes) 159 { 160 unsigned int i; 161 162 /* Verify if user specified unsupported combination. */ 163 for (i = 0; i < RTE_DIM(mlx5_class_invalid_combinations); i++) { 164 if ((mlx5_class_invalid_combinations[i] & user_classes) == 165 mlx5_class_invalid_combinations[i]) 166 return -EINVAL; 167 } 168 /* Not found any invalid class combination. */ 169 return 0; 170 } 171 172 static bool 173 device_class_enabled(const struct mlx5_common_device *device, uint32_t class) 174 { 175 return (device->classes_loaded & class) > 0; 176 } 177 178 static bool 179 mlx5_bus_match(const struct mlx5_class_driver *drv, 180 const struct rte_device *dev) 181 { 182 if (mlx5_dev_is_pci(dev)) 183 return mlx5_dev_pci_match(drv, dev); 184 return true; 185 } 186 187 static struct mlx5_common_device * 188 to_mlx5_device(const struct rte_device *rte_dev) 189 { 190 struct mlx5_common_device *dev; 191 192 TAILQ_FOREACH(dev, &devices_list, next) { 193 if (rte_dev == dev->dev) 194 return dev; 195 } 196 return NULL; 197 } 198 199 int 200 mlx5_dev_to_pci_str(const struct rte_device *dev, char *addr, size_t size) 201 { 202 struct rte_pci_addr pci_addr = { 0 }; 203 int ret; 204 205 if (mlx5_dev_is_pci(dev)) { 206 /* Input might be <BDF>, format PCI address to <DBDF>. */ 207 ret = rte_pci_addr_parse(dev->name, &pci_addr); 208 if (ret != 0) 209 return -ENODEV; 210 rte_pci_device_name(&pci_addr, addr, size); 211 return 0; 212 } 213 #ifdef RTE_EXEC_ENV_LINUX 214 return mlx5_auxiliary_get_pci_str(RTE_DEV_TO_AUXILIARY_CONST(dev), 215 addr, size); 216 #else 217 rte_errno = ENODEV; 218 return -rte_errno; 219 #endif 220 } 221 222 static void 223 dev_release(struct mlx5_common_device *dev) 224 { 225 TAILQ_REMOVE(&devices_list, dev, next); 226 rte_free(dev); 227 } 228 229 static int 230 drivers_remove(struct mlx5_common_device *dev, uint32_t enabled_classes) 231 { 232 struct mlx5_class_driver *driver; 233 int local_ret = -ENODEV; 234 unsigned int i = 0; 235 int ret = 0; 236 237 enabled_classes &= dev->classes_loaded; 238 while (enabled_classes) { 239 driver = driver_get(RTE_BIT64(i)); 240 if (driver != NULL) { 241 local_ret = driver->remove(dev->dev); 242 if (local_ret == 0) 243 dev->classes_loaded &= ~RTE_BIT64(i); 244 else if (ret == 0) 245 ret = local_ret; 246 } 247 enabled_classes &= ~RTE_BIT64(i); 248 i++; 249 } 250 if (local_ret != 0 && ret == 0) 251 ret = local_ret; 252 return ret; 253 } 254 255 static int 256 drivers_probe(struct mlx5_common_device *dev, uint32_t user_classes) 257 { 258 struct mlx5_class_driver *driver; 259 uint32_t enabled_classes = 0; 260 bool already_loaded; 261 int ret; 262 263 TAILQ_FOREACH(driver, &drivers_list, next) { 264 if ((driver->drv_class & user_classes) == 0) 265 continue; 266 if (!mlx5_bus_match(driver, dev->dev)) 267 continue; 268 already_loaded = dev->classes_loaded & driver->drv_class; 269 if (already_loaded && driver->probe_again == 0) { 270 DRV_LOG(ERR, "Device %s is already probed", 271 dev->dev->name); 272 ret = -EEXIST; 273 goto probe_err; 274 } 275 ret = driver->probe(dev->dev); 276 if (ret < 0) { 277 DRV_LOG(ERR, "Failed to load driver %s", 278 driver->name); 279 goto probe_err; 280 } 281 enabled_classes |= driver->drv_class; 282 } 283 dev->classes_loaded |= enabled_classes; 284 return 0; 285 probe_err: 286 /* Only unload drivers which are enabled which were enabled 287 * in this probe instance. 288 */ 289 drivers_remove(dev, enabled_classes); 290 return ret; 291 } 292 293 int 294 mlx5_common_dev_probe(struct rte_device *eal_dev) 295 { 296 struct mlx5_common_device *dev; 297 uint32_t classes = 0; 298 bool new_device = false; 299 int ret; 300 301 DRV_LOG(INFO, "probe device \"%s\".", eal_dev->name); 302 ret = parse_class_options(eal_dev->devargs); 303 if (ret < 0) { 304 DRV_LOG(ERR, "Unsupported mlx5 class type: %s", 305 eal_dev->devargs->args); 306 return ret; 307 } 308 classes = ret; 309 if (classes == 0) 310 /* Default to net class. */ 311 classes = MLX5_CLASS_ETH; 312 dev = to_mlx5_device(eal_dev); 313 if (!dev) { 314 dev = rte_zmalloc("mlx5_common_device", sizeof(*dev), 0); 315 if (!dev) 316 return -ENOMEM; 317 dev->dev = eal_dev; 318 TAILQ_INSERT_HEAD(&devices_list, dev, next); 319 new_device = true; 320 } else { 321 /* Validate combination here. */ 322 ret = is_valid_class_combination(classes | 323 dev->classes_loaded); 324 if (ret != 0) { 325 DRV_LOG(ERR, "Unsupported mlx5 classes combination."); 326 return ret; 327 } 328 } 329 ret = drivers_probe(dev, classes); 330 if (ret) 331 goto class_err; 332 return 0; 333 class_err: 334 if (new_device) 335 dev_release(dev); 336 return ret; 337 } 338 339 int 340 mlx5_common_dev_remove(struct rte_device *eal_dev) 341 { 342 struct mlx5_common_device *dev; 343 int ret; 344 345 dev = to_mlx5_device(eal_dev); 346 if (!dev) 347 return -ENODEV; 348 /* Matching device found, cleanup and unload drivers. */ 349 ret = drivers_remove(dev, dev->classes_loaded); 350 if (ret != 0) 351 dev_release(dev); 352 return ret; 353 } 354 355 int 356 mlx5_common_dev_dma_map(struct rte_device *dev, void *addr, uint64_t iova, 357 size_t len) 358 { 359 struct mlx5_class_driver *driver = NULL; 360 struct mlx5_class_driver *temp; 361 struct mlx5_common_device *mdev; 362 int ret = -EINVAL; 363 364 mdev = to_mlx5_device(dev); 365 if (!mdev) 366 return -ENODEV; 367 TAILQ_FOREACH(driver, &drivers_list, next) { 368 if (!device_class_enabled(mdev, driver->drv_class) || 369 driver->dma_map == NULL) 370 continue; 371 ret = driver->dma_map(dev, addr, iova, len); 372 if (ret) 373 goto map_err; 374 } 375 return ret; 376 map_err: 377 TAILQ_FOREACH(temp, &drivers_list, next) { 378 if (temp == driver) 379 break; 380 if (device_class_enabled(mdev, temp->drv_class) && 381 temp->dma_map && temp->dma_unmap) 382 temp->dma_unmap(dev, addr, iova, len); 383 } 384 return ret; 385 } 386 387 int 388 mlx5_common_dev_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova, 389 size_t len) 390 { 391 struct mlx5_class_driver *driver; 392 struct mlx5_common_device *mdev; 393 int local_ret = -EINVAL; 394 int ret = 0; 395 396 mdev = to_mlx5_device(dev); 397 if (!mdev) 398 return -ENODEV; 399 /* There is no unmap error recovery in current implementation. */ 400 TAILQ_FOREACH_REVERSE(driver, &drivers_list, mlx5_drivers, next) { 401 if (!device_class_enabled(mdev, driver->drv_class) || 402 driver->dma_unmap == NULL) 403 continue; 404 local_ret = driver->dma_unmap(dev, addr, iova, len); 405 if (local_ret && (ret == 0)) 406 ret = local_ret; 407 } 408 if (local_ret) 409 ret = local_ret; 410 return ret; 411 } 412 413 void 414 mlx5_class_driver_register(struct mlx5_class_driver *driver) 415 { 416 mlx5_common_driver_on_register_pci(driver); 417 TAILQ_INSERT_TAIL(&drivers_list, driver, next); 418 } 419 420 static void mlx5_common_driver_init(void) 421 { 422 mlx5_common_pci_init(); 423 #ifdef RTE_EXEC_ENV_LINUX 424 mlx5_common_auxiliary_init(); 425 #endif 426 } 427 428 static bool mlx5_common_initialized; 429 430 /** 431 * One time innitialization routine for run-time dependency on glue library 432 * for multiple PMDs. Each mlx5 PMD that depends on mlx5_common module, 433 * must invoke in its constructor. 434 */ 435 void 436 mlx5_common_init(void) 437 { 438 if (mlx5_common_initialized) 439 return; 440 441 mlx5_glue_constructor(); 442 mlx5_common_driver_init(); 443 mlx5_common_initialized = true; 444 } 445 446 /** 447 * This function is responsible of initializing the variable 448 * haswell_broadwell_cpu by checking if the cpu is intel 449 * and reading the data returned from mlx5_cpu_id(). 450 * since haswell and broadwell cpus don't have improved performance 451 * when using relaxed ordering we want to check the cpu type before 452 * before deciding whether to enable RO or not. 453 * if the cpu is haswell or broadwell the variable will be set to 1 454 * otherwise it will be 0. 455 */ 456 RTE_INIT_PRIO(mlx5_is_haswell_broadwell_cpu, LOG) 457 { 458 #ifdef RTE_ARCH_X86_64 459 unsigned int broadwell_models[4] = {0x3d, 0x47, 0x4F, 0x56}; 460 unsigned int haswell_models[4] = {0x3c, 0x3f, 0x45, 0x46}; 461 unsigned int i, model, family, brand_id, vendor; 462 unsigned int signature_intel_ebx = 0x756e6547; 463 unsigned int extended_model; 464 unsigned int eax = 0; 465 unsigned int ebx = 0; 466 unsigned int ecx = 0; 467 unsigned int edx = 0; 468 int max_level; 469 470 mlx5_cpu_id(0, &eax, &ebx, &ecx, &edx); 471 vendor = ebx; 472 max_level = eax; 473 if (max_level < 1) { 474 haswell_broadwell_cpu = 0; 475 return; 476 } 477 mlx5_cpu_id(1, &eax, &ebx, &ecx, &edx); 478 model = (eax >> 4) & 0x0f; 479 family = (eax >> 8) & 0x0f; 480 brand_id = ebx & 0xff; 481 extended_model = (eax >> 12) & 0xf0; 482 /* Check if the processor is Haswell or Broadwell */ 483 if (vendor == signature_intel_ebx) { 484 if (family == 0x06) 485 model += extended_model; 486 if (brand_id == 0 && family == 0x6) { 487 for (i = 0; i < RTE_DIM(broadwell_models); i++) 488 if (model == broadwell_models[i]) { 489 haswell_broadwell_cpu = 1; 490 return; 491 } 492 for (i = 0; i < RTE_DIM(haswell_models); i++) 493 if (model == haswell_models[i]) { 494 haswell_broadwell_cpu = 1; 495 return; 496 } 497 } 498 } 499 #endif 500 haswell_broadwell_cpu = 0; 501 } 502 503 /** 504 * Allocate the User Access Region with DevX on specified device. 505 * 506 * @param [in] ctx 507 * Infiniband device context to perform allocation on. 508 * @param [in] mapping 509 * MLX5DV_UAR_ALLOC_TYPE_BF - allocate as cached memory with write-combining 510 * attributes (if supported by the host), the 511 * writes to the UAR registers must be followed 512 * by write memory barrier. 513 * MLX5DV_UAR_ALLOC_TYPE_NC - allocate as non-cached nenory, all writes are 514 * promoted to the registers immediately, no 515 * memory barriers needed. 516 * mapping < 0 - the first attempt is performed with MLX5DV_UAR_ALLOC_TYPE_BF, 517 * if this fails the next attempt with MLX5DV_UAR_ALLOC_TYPE_NC 518 * is performed. The drivers specifying negative values should 519 * always provide the write memory barrier operation after UAR 520 * register writings. 521 * If there is no definitions for the MLX5DV_UAR_ALLOC_TYPE_xx (older rdma 522 * library headers), the caller can specify 0. 523 * 524 * @return 525 * UAR object pointer on success, NULL otherwise and rte_errno is set. 526 */ 527 void * 528 mlx5_devx_alloc_uar(void *ctx, int mapping) 529 { 530 void *uar; 531 uint32_t retry, uar_mapping; 532 void *base_addr; 533 534 for (retry = 0; retry < MLX5_ALLOC_UAR_RETRY; ++retry) { 535 #ifdef MLX5DV_UAR_ALLOC_TYPE_NC 536 /* Control the mapping type according to the settings. */ 537 uar_mapping = (mapping < 0) ? 538 MLX5DV_UAR_ALLOC_TYPE_NC : mapping; 539 #else 540 /* 541 * It seems we have no way to control the memory mapping type 542 * for the UAR, the default "Write-Combining" type is supposed. 543 */ 544 uar_mapping = 0; 545 RTE_SET_USED(mapping); 546 #endif 547 uar = mlx5_glue->devx_alloc_uar(ctx, uar_mapping); 548 #ifdef MLX5DV_UAR_ALLOC_TYPE_NC 549 if (!uar && 550 mapping < 0 && 551 uar_mapping == MLX5DV_UAR_ALLOC_TYPE_BF) { 552 /* 553 * In some environments like virtual machine the 554 * Write Combining mapped might be not supported and 555 * UAR allocation fails. We tried "Non-Cached" mapping 556 * for the case. 557 */ 558 DRV_LOG(WARNING, "Failed to allocate DevX UAR (BF)"); 559 uar_mapping = MLX5DV_UAR_ALLOC_TYPE_NC; 560 uar = mlx5_glue->devx_alloc_uar(ctx, uar_mapping); 561 } else if (!uar && 562 mapping < 0 && 563 uar_mapping == MLX5DV_UAR_ALLOC_TYPE_NC) { 564 /* 565 * If Verbs/kernel does not support "Non-Cached" 566 * try the "Write-Combining". 567 */ 568 DRV_LOG(WARNING, "Failed to allocate DevX UAR (NC)"); 569 uar_mapping = MLX5DV_UAR_ALLOC_TYPE_BF; 570 uar = mlx5_glue->devx_alloc_uar(ctx, uar_mapping); 571 } 572 #endif 573 if (!uar) { 574 DRV_LOG(ERR, "Failed to allocate DevX UAR (BF/NC)"); 575 rte_errno = ENOMEM; 576 goto exit; 577 } 578 base_addr = mlx5_os_get_devx_uar_base_addr(uar); 579 if (base_addr) 580 break; 581 /* 582 * The UARs are allocated by rdma_core within the 583 * IB device context, on context closure all UARs 584 * will be freed, should be no memory/object leakage. 585 */ 586 DRV_LOG(WARNING, "Retrying to allocate DevX UAR"); 587 uar = NULL; 588 } 589 /* Check whether we finally succeeded with valid UAR allocation. */ 590 if (!uar) { 591 DRV_LOG(ERR, "Failed to allocate DevX UAR (NULL base)"); 592 rte_errno = ENOMEM; 593 } 594 /* 595 * Return void * instead of struct mlx5dv_devx_uar * 596 * is for compatibility with older rdma-core library headers. 597 */ 598 exit: 599 return uar; 600 } 601 602 RTE_PMD_EXPORT_NAME(mlx5_common_driver, __COUNTER__); 603