1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2019 Mellanox Technologies, Ltd 3 */ 4 5 #include <unistd.h> 6 #include <string.h> 7 #include <stdio.h> 8 9 #include <rte_errno.h> 10 #include <rte_mempool.h> 11 #include <rte_class.h> 12 #include <rte_malloc.h> 13 14 #include "mlx5_common.h" 15 #include "mlx5_common_os.h" 16 #include "mlx5_common_log.h" 17 #include "mlx5_common_pci.h" 18 #include "mlx5_common_private.h" 19 20 uint8_t haswell_broadwell_cpu; 21 22 /* In case this is an x86_64 intel processor to check if 23 * we should use relaxed ordering. 24 */ 25 #ifdef RTE_ARCH_X86_64 26 /** 27 * This function returns processor identification and feature information 28 * into the registers. 29 * 30 * @param eax, ebx, ecx, edx 31 * Pointers to the registers that will hold cpu information. 32 * @param level 33 * The main category of information returned. 34 */ 35 static inline void mlx5_cpu_id(unsigned int level, 36 unsigned int *eax, unsigned int *ebx, 37 unsigned int *ecx, unsigned int *edx) 38 { 39 __asm__("cpuid\n\t" 40 : "=a" (*eax), "=b" (*ebx), "=c" (*ecx), "=d" (*edx) 41 : "0" (level)); 42 } 43 #endif 44 45 RTE_LOG_REGISTER_DEFAULT(mlx5_common_logtype, NOTICE) 46 47 /* Head of list of drivers. */ 48 static TAILQ_HEAD(mlx5_drivers, mlx5_class_driver) drivers_list = 49 TAILQ_HEAD_INITIALIZER(drivers_list); 50 51 /* Head of devices. */ 52 static TAILQ_HEAD(mlx5_devices, mlx5_common_device) devices_list = 53 TAILQ_HEAD_INITIALIZER(devices_list); 54 55 static const struct { 56 const char *name; 57 unsigned int drv_class; 58 } mlx5_classes[] = { 59 { .name = "vdpa", .drv_class = MLX5_CLASS_VDPA }, 60 { .name = "eth", .drv_class = MLX5_CLASS_ETH }, 61 /* Keep class "net" for backward compatibility. */ 62 { .name = "net", .drv_class = MLX5_CLASS_ETH }, 63 { .name = "regex", .drv_class = MLX5_CLASS_REGEX }, 64 { .name = "compress", .drv_class = MLX5_CLASS_COMPRESS }, 65 { .name = "crypto", .drv_class = MLX5_CLASS_CRYPTO }, 66 }; 67 68 static int 69 class_name_to_value(const char *class_name) 70 { 71 unsigned int i; 72 73 for (i = 0; i < RTE_DIM(mlx5_classes); i++) { 74 if (strcmp(class_name, mlx5_classes[i].name) == 0) 75 return mlx5_classes[i].drv_class; 76 } 77 return -EINVAL; 78 } 79 80 static struct mlx5_class_driver * 81 driver_get(uint32_t class) 82 { 83 struct mlx5_class_driver *driver; 84 85 TAILQ_FOREACH(driver, &drivers_list, next) { 86 if ((uint32_t)driver->drv_class == class) 87 return driver; 88 } 89 return NULL; 90 } 91 92 static int 93 devargs_class_handler(__rte_unused const char *key, 94 const char *class_names, void *opaque) 95 { 96 int *ret = opaque; 97 int class_val; 98 char *scratch; 99 char *found; 100 char *refstr = NULL; 101 102 *ret = 0; 103 scratch = strdup(class_names); 104 if (scratch == NULL) { 105 *ret = -ENOMEM; 106 return *ret; 107 } 108 found = strtok_r(scratch, ":", &refstr); 109 if (found == NULL) 110 /* Empty string. */ 111 goto err; 112 do { 113 /* Extract each individual class name. Multiple 114 * classes can be supplied as class=net:regex:foo:bar. 115 */ 116 class_val = class_name_to_value(found); 117 /* Check if its a valid class. */ 118 if (class_val < 0) { 119 *ret = -EINVAL; 120 goto err; 121 } 122 *ret |= class_val; 123 found = strtok_r(NULL, ":", &refstr); 124 } while (found != NULL); 125 err: 126 free(scratch); 127 if (*ret < 0) 128 DRV_LOG(ERR, "Invalid mlx5 class options: %s.\n", class_names); 129 return *ret; 130 } 131 132 static int 133 parse_class_options(const struct rte_devargs *devargs) 134 { 135 struct rte_kvargs *kvlist; 136 int ret = 0; 137 138 if (devargs == NULL) 139 return 0; 140 if (devargs->cls != NULL && devargs->cls->name != NULL) 141 /* Global syntax, only one class type. */ 142 return class_name_to_value(devargs->cls->name); 143 /* Legacy devargs support multiple classes. */ 144 kvlist = rte_kvargs_parse(devargs->args, NULL); 145 if (kvlist == NULL) 146 return 0; 147 rte_kvargs_process(kvlist, RTE_DEVARGS_KEY_CLASS, 148 devargs_class_handler, &ret); 149 rte_kvargs_free(kvlist); 150 return ret; 151 } 152 153 static const unsigned int mlx5_class_invalid_combinations[] = { 154 MLX5_CLASS_ETH | MLX5_CLASS_VDPA, 155 /* New class combination should be added here. */ 156 }; 157 158 static int 159 is_valid_class_combination(uint32_t user_classes) 160 { 161 unsigned int i; 162 163 /* Verify if user specified unsupported combination. */ 164 for (i = 0; i < RTE_DIM(mlx5_class_invalid_combinations); i++) { 165 if ((mlx5_class_invalid_combinations[i] & user_classes) == 166 mlx5_class_invalid_combinations[i]) 167 return -EINVAL; 168 } 169 /* Not found any invalid class combination. */ 170 return 0; 171 } 172 173 static bool 174 device_class_enabled(const struct mlx5_common_device *device, uint32_t class) 175 { 176 return (device->classes_loaded & class) > 0; 177 } 178 179 static bool 180 mlx5_bus_match(const struct mlx5_class_driver *drv, 181 const struct rte_device *dev) 182 { 183 if (mlx5_dev_is_pci(dev)) 184 return mlx5_dev_pci_match(drv, dev); 185 return true; 186 } 187 188 static struct mlx5_common_device * 189 to_mlx5_device(const struct rte_device *rte_dev) 190 { 191 struct mlx5_common_device *dev; 192 193 TAILQ_FOREACH(dev, &devices_list, next) { 194 if (rte_dev == dev->dev) 195 return dev; 196 } 197 return NULL; 198 } 199 200 static void 201 dev_release(struct mlx5_common_device *dev) 202 { 203 TAILQ_REMOVE(&devices_list, dev, next); 204 rte_free(dev); 205 } 206 207 static int 208 drivers_remove(struct mlx5_common_device *dev, uint32_t enabled_classes) 209 { 210 struct mlx5_class_driver *driver; 211 int local_ret = -ENODEV; 212 unsigned int i = 0; 213 int ret = 0; 214 215 enabled_classes &= dev->classes_loaded; 216 while (enabled_classes) { 217 driver = driver_get(RTE_BIT64(i)); 218 if (driver != NULL) { 219 local_ret = driver->remove(dev->dev); 220 if (local_ret == 0) 221 dev->classes_loaded &= ~RTE_BIT64(i); 222 else if (ret == 0) 223 ret = local_ret; 224 } 225 enabled_classes &= ~RTE_BIT64(i); 226 i++; 227 } 228 if (local_ret != 0 && ret == 0) 229 ret = local_ret; 230 return ret; 231 } 232 233 static int 234 drivers_probe(struct mlx5_common_device *dev, uint32_t user_classes) 235 { 236 struct mlx5_class_driver *driver; 237 uint32_t enabled_classes = 0; 238 bool already_loaded; 239 int ret; 240 241 TAILQ_FOREACH(driver, &drivers_list, next) { 242 if ((driver->drv_class & user_classes) == 0) 243 continue; 244 if (!mlx5_bus_match(driver, dev->dev)) 245 continue; 246 already_loaded = dev->classes_loaded & driver->drv_class; 247 if (already_loaded && driver->probe_again == 0) { 248 DRV_LOG(ERR, "Device %s is already probed", 249 dev->dev->name); 250 ret = -EEXIST; 251 goto probe_err; 252 } 253 ret = driver->probe(dev->dev); 254 if (ret < 0) { 255 DRV_LOG(ERR, "Failed to load driver %s", 256 driver->name); 257 goto probe_err; 258 } 259 enabled_classes |= driver->drv_class; 260 } 261 dev->classes_loaded |= enabled_classes; 262 return 0; 263 probe_err: 264 /* Only unload drivers which are enabled which were enabled 265 * in this probe instance. 266 */ 267 drivers_remove(dev, enabled_classes); 268 return ret; 269 } 270 271 int 272 mlx5_common_dev_probe(struct rte_device *eal_dev) 273 { 274 struct mlx5_common_device *dev; 275 uint32_t classes = 0; 276 bool new_device = false; 277 int ret; 278 279 DRV_LOG(INFO, "probe device \"%s\".", eal_dev->name); 280 ret = parse_class_options(eal_dev->devargs); 281 if (ret < 0) { 282 DRV_LOG(ERR, "Unsupported mlx5 class type: %s", 283 eal_dev->devargs->args); 284 return ret; 285 } 286 classes = ret; 287 if (classes == 0) 288 /* Default to net class. */ 289 classes = MLX5_CLASS_ETH; 290 dev = to_mlx5_device(eal_dev); 291 if (!dev) { 292 dev = rte_zmalloc("mlx5_common_device", sizeof(*dev), 0); 293 if (!dev) 294 return -ENOMEM; 295 dev->dev = eal_dev; 296 TAILQ_INSERT_HEAD(&devices_list, dev, next); 297 new_device = true; 298 } else { 299 /* Validate combination here. */ 300 ret = is_valid_class_combination(classes | 301 dev->classes_loaded); 302 if (ret != 0) { 303 DRV_LOG(ERR, "Unsupported mlx5 classes combination."); 304 return ret; 305 } 306 } 307 ret = drivers_probe(dev, classes); 308 if (ret) 309 goto class_err; 310 return 0; 311 class_err: 312 if (new_device) 313 dev_release(dev); 314 return ret; 315 } 316 317 int 318 mlx5_common_dev_remove(struct rte_device *eal_dev) 319 { 320 struct mlx5_common_device *dev; 321 int ret; 322 323 dev = to_mlx5_device(eal_dev); 324 if (!dev) 325 return -ENODEV; 326 /* Matching device found, cleanup and unload drivers. */ 327 ret = drivers_remove(dev, dev->classes_loaded); 328 if (ret != 0) 329 dev_release(dev); 330 return ret; 331 } 332 333 int 334 mlx5_common_dev_dma_map(struct rte_device *dev, void *addr, uint64_t iova, 335 size_t len) 336 { 337 struct mlx5_class_driver *driver = NULL; 338 struct mlx5_class_driver *temp; 339 struct mlx5_common_device *mdev; 340 int ret = -EINVAL; 341 342 mdev = to_mlx5_device(dev); 343 if (!mdev) 344 return -ENODEV; 345 TAILQ_FOREACH(driver, &drivers_list, next) { 346 if (!device_class_enabled(mdev, driver->drv_class) || 347 driver->dma_map == NULL) 348 continue; 349 ret = driver->dma_map(dev, addr, iova, len); 350 if (ret) 351 goto map_err; 352 } 353 return ret; 354 map_err: 355 TAILQ_FOREACH(temp, &drivers_list, next) { 356 if (temp == driver) 357 break; 358 if (device_class_enabled(mdev, temp->drv_class) && 359 temp->dma_map && temp->dma_unmap) 360 temp->dma_unmap(dev, addr, iova, len); 361 } 362 return ret; 363 } 364 365 int 366 mlx5_common_dev_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova, 367 size_t len) 368 { 369 struct mlx5_class_driver *driver; 370 struct mlx5_common_device *mdev; 371 int local_ret = -EINVAL; 372 int ret = 0; 373 374 mdev = to_mlx5_device(dev); 375 if (!mdev) 376 return -ENODEV; 377 /* There is no unmap error recovery in current implementation. */ 378 TAILQ_FOREACH_REVERSE(driver, &drivers_list, mlx5_drivers, next) { 379 if (!device_class_enabled(mdev, driver->drv_class) || 380 driver->dma_unmap == NULL) 381 continue; 382 local_ret = driver->dma_unmap(dev, addr, iova, len); 383 if (local_ret && (ret == 0)) 384 ret = local_ret; 385 } 386 if (local_ret) 387 ret = local_ret; 388 return ret; 389 } 390 391 void 392 mlx5_class_driver_register(struct mlx5_class_driver *driver) 393 { 394 mlx5_common_driver_on_register_pci(driver); 395 TAILQ_INSERT_TAIL(&drivers_list, driver, next); 396 } 397 398 static void mlx5_common_driver_init(void) 399 { 400 mlx5_common_pci_init(); 401 } 402 403 static bool mlx5_common_initialized; 404 405 /** 406 * One time innitialization routine for run-time dependency on glue library 407 * for multiple PMDs. Each mlx5 PMD that depends on mlx5_common module, 408 * must invoke in its constructor. 409 */ 410 void 411 mlx5_common_init(void) 412 { 413 if (mlx5_common_initialized) 414 return; 415 416 mlx5_glue_constructor(); 417 mlx5_common_driver_init(); 418 mlx5_common_initialized = true; 419 } 420 421 /** 422 * This function is responsible of initializing the variable 423 * haswell_broadwell_cpu by checking if the cpu is intel 424 * and reading the data returned from mlx5_cpu_id(). 425 * since haswell and broadwell cpus don't have improved performance 426 * when using relaxed ordering we want to check the cpu type before 427 * before deciding whether to enable RO or not. 428 * if the cpu is haswell or broadwell the variable will be set to 1 429 * otherwise it will be 0. 430 */ 431 RTE_INIT_PRIO(mlx5_is_haswell_broadwell_cpu, LOG) 432 { 433 #ifdef RTE_ARCH_X86_64 434 unsigned int broadwell_models[4] = {0x3d, 0x47, 0x4F, 0x56}; 435 unsigned int haswell_models[4] = {0x3c, 0x3f, 0x45, 0x46}; 436 unsigned int i, model, family, brand_id, vendor; 437 unsigned int signature_intel_ebx = 0x756e6547; 438 unsigned int extended_model; 439 unsigned int eax = 0; 440 unsigned int ebx = 0; 441 unsigned int ecx = 0; 442 unsigned int edx = 0; 443 int max_level; 444 445 mlx5_cpu_id(0, &eax, &ebx, &ecx, &edx); 446 vendor = ebx; 447 max_level = eax; 448 if (max_level < 1) { 449 haswell_broadwell_cpu = 0; 450 return; 451 } 452 mlx5_cpu_id(1, &eax, &ebx, &ecx, &edx); 453 model = (eax >> 4) & 0x0f; 454 family = (eax >> 8) & 0x0f; 455 brand_id = ebx & 0xff; 456 extended_model = (eax >> 12) & 0xf0; 457 /* Check if the processor is Haswell or Broadwell */ 458 if (vendor == signature_intel_ebx) { 459 if (family == 0x06) 460 model += extended_model; 461 if (brand_id == 0 && family == 0x6) { 462 for (i = 0; i < RTE_DIM(broadwell_models); i++) 463 if (model == broadwell_models[i]) { 464 haswell_broadwell_cpu = 1; 465 return; 466 } 467 for (i = 0; i < RTE_DIM(haswell_models); i++) 468 if (model == haswell_models[i]) { 469 haswell_broadwell_cpu = 1; 470 return; 471 } 472 } 473 } 474 #endif 475 haswell_broadwell_cpu = 0; 476 } 477 478 /** 479 * Allocate the User Access Region with DevX on specified device. 480 * 481 * @param [in] ctx 482 * Infiniband device context to perform allocation on. 483 * @param [in] mapping 484 * MLX5DV_UAR_ALLOC_TYPE_BF - allocate as cached memory with write-combining 485 * attributes (if supported by the host), the 486 * writes to the UAR registers must be followed 487 * by write memory barrier. 488 * MLX5DV_UAR_ALLOC_TYPE_NC - allocate as non-cached nenory, all writes are 489 * promoted to the registers immediately, no 490 * memory barriers needed. 491 * mapping < 0 - the first attempt is performed with MLX5DV_UAR_ALLOC_TYPE_BF, 492 * if this fails the next attempt with MLX5DV_UAR_ALLOC_TYPE_NC 493 * is performed. The drivers specifying negative values should 494 * always provide the write memory barrier operation after UAR 495 * register writings. 496 * If there is no definitions for the MLX5DV_UAR_ALLOC_TYPE_xx (older rdma 497 * library headers), the caller can specify 0. 498 * 499 * @return 500 * UAR object pointer on success, NULL otherwise and rte_errno is set. 501 */ 502 void * 503 mlx5_devx_alloc_uar(void *ctx, int mapping) 504 { 505 void *uar; 506 uint32_t retry, uar_mapping; 507 void *base_addr; 508 509 for (retry = 0; retry < MLX5_ALLOC_UAR_RETRY; ++retry) { 510 #ifdef MLX5DV_UAR_ALLOC_TYPE_NC 511 /* Control the mapping type according to the settings. */ 512 uar_mapping = (mapping < 0) ? 513 MLX5DV_UAR_ALLOC_TYPE_NC : mapping; 514 #else 515 /* 516 * It seems we have no way to control the memory mapping type 517 * for the UAR, the default "Write-Combining" type is supposed. 518 */ 519 uar_mapping = 0; 520 RTE_SET_USED(mapping); 521 #endif 522 uar = mlx5_glue->devx_alloc_uar(ctx, uar_mapping); 523 #ifdef MLX5DV_UAR_ALLOC_TYPE_NC 524 if (!uar && 525 mapping < 0 && 526 uar_mapping == MLX5DV_UAR_ALLOC_TYPE_BF) { 527 /* 528 * In some environments like virtual machine the 529 * Write Combining mapped might be not supported and 530 * UAR allocation fails. We tried "Non-Cached" mapping 531 * for the case. 532 */ 533 DRV_LOG(WARNING, "Failed to allocate DevX UAR (BF)"); 534 uar_mapping = MLX5DV_UAR_ALLOC_TYPE_NC; 535 uar = mlx5_glue->devx_alloc_uar(ctx, uar_mapping); 536 } else if (!uar && 537 mapping < 0 && 538 uar_mapping == MLX5DV_UAR_ALLOC_TYPE_NC) { 539 /* 540 * If Verbs/kernel does not support "Non-Cached" 541 * try the "Write-Combining". 542 */ 543 DRV_LOG(WARNING, "Failed to allocate DevX UAR (NC)"); 544 uar_mapping = MLX5DV_UAR_ALLOC_TYPE_BF; 545 uar = mlx5_glue->devx_alloc_uar(ctx, uar_mapping); 546 } 547 #endif 548 if (!uar) { 549 DRV_LOG(ERR, "Failed to allocate DevX UAR (BF/NC)"); 550 rte_errno = ENOMEM; 551 goto exit; 552 } 553 base_addr = mlx5_os_get_devx_uar_base_addr(uar); 554 if (base_addr) 555 break; 556 /* 557 * The UARs are allocated by rdma_core within the 558 * IB device context, on context closure all UARs 559 * will be freed, should be no memory/object leakage. 560 */ 561 DRV_LOG(WARNING, "Retrying to allocate DevX UAR"); 562 uar = NULL; 563 } 564 /* Check whether we finally succeeded with valid UAR allocation. */ 565 if (!uar) { 566 DRV_LOG(ERR, "Failed to allocate DevX UAR (NULL base)"); 567 rte_errno = ENOMEM; 568 } 569 /* 570 * Return void * instead of struct mlx5dv_devx_uar * 571 * is for compatibility with older rdma-core library headers. 572 */ 573 exit: 574 return uar; 575 } 576 577 RTE_PMD_EXPORT_NAME(mlx5_common_driver, __COUNTER__); 578