1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018-2022 Intel Corporation 3 */ 4 5 #include <rte_string_fns.h> 6 #include <rte_devargs.h> 7 #include <ctype.h> 8 9 #include "qat_device.h" 10 #include "adf_transport_access_macros.h" 11 #include "qat_sym.h" 12 #include "qat_comp_pmd.h" 13 #include "adf_pf2vf_msg.h" 14 #include "qat_pf2vf.h" 15 16 #define NOT_NULL(arg, func, msg, ...) \ 17 do { \ 18 if (arg == NULL) { \ 19 QAT_LOG(ERR, \ 20 msg, ##__VA_ARGS__); \ 21 func; \ 22 } \ 23 } while (0) 24 25 /* Hardware device information per generation */ 26 struct qat_gen_hw_data qat_gen_config[QAT_N_GENS]; 27 struct qat_dev_hw_spec_funcs *qat_dev_hw_spec[QAT_N_GENS]; 28 29 struct qat_service qat_service[QAT_MAX_SERVICES]; 30 31 /* per-process array of device data */ 32 struct qat_device_info qat_pci_devs[RTE_PMD_QAT_MAX_PCI_DEVICES]; 33 static int qat_nb_pci_devices; 34 int qat_legacy_capa; 35 36 /* 37 * The set of PCI devices this driver supports 38 */ 39 40 static const struct rte_pci_id pci_id_qat_map[] = { 41 { 42 RTE_PCI_DEVICE(0x8086, 0x0443), 43 }, 44 { 45 RTE_PCI_DEVICE(0x8086, 0x37c9), 46 }, 47 { 48 RTE_PCI_DEVICE(0x8086, 0x19e3), 49 }, 50 { 51 RTE_PCI_DEVICE(0x8086, 0x6f55), 52 }, 53 { 54 RTE_PCI_DEVICE(0x8086, 0x18ef), 55 }, 56 { 57 RTE_PCI_DEVICE(0x8086, 0x18a1), 58 }, 59 { 60 RTE_PCI_DEVICE(0x8086, 0x578b), 61 }, 62 { 63 RTE_PCI_DEVICE(0x8086, 0x4941), 64 }, 65 { 66 RTE_PCI_DEVICE(0x8086, 0x4943), 67 }, 68 { 69 RTE_PCI_DEVICE(0x8086, 0x4945), 70 }, 71 { 72 RTE_PCI_DEVICE(0x8086, 0x4947), 73 }, 74 { 75 RTE_PCI_DEVICE(0x8086, 0x1454), 76 }, 77 { 78 RTE_PCI_DEVICE(0x8086, 0x0da5), 79 }, 80 {.device_id = 0}, 81 }; 82 83 static int 84 qat_pci_get_extra_size(enum qat_device_gen qat_dev_gen) 85 { 86 struct qat_dev_hw_spec_funcs *ops_hw = 87 qat_dev_hw_spec[qat_dev_gen]; 88 if (ops_hw->qat_dev_get_extra_size == NULL) 89 return -ENOTSUP; 90 return ops_hw->qat_dev_get_extra_size(); 91 } 92 93 static struct qat_pci_device * 94 qat_pci_get_named_dev(const char *name) 95 { 96 unsigned int i; 97 98 if (name == NULL) 99 return NULL; 100 101 for (i = 0; i < RTE_PMD_QAT_MAX_PCI_DEVICES; i++) { 102 if (qat_pci_devs[i].mz && 103 (strcmp(((struct qat_pci_device *) 104 qat_pci_devs[i].mz->addr)->name, name) 105 == 0)) 106 return (struct qat_pci_device *) 107 qat_pci_devs[i].mz->addr; 108 } 109 110 return NULL; 111 } 112 113 static uint8_t 114 qat_pci_find_free_device_index(void) 115 { 116 uint8_t dev_id; 117 118 for (dev_id = 0; dev_id < RTE_PMD_QAT_MAX_PCI_DEVICES; 119 dev_id++) { 120 if (qat_pci_devs[dev_id].mz == NULL) 121 break; 122 } 123 return dev_id; 124 } 125 126 static struct qat_pci_device * 127 qat_get_qat_dev_from_pci_dev(struct rte_pci_device *pci_dev) 128 { 129 char name[QAT_DEV_NAME_MAX_LEN]; 130 131 rte_pci_device_name(&pci_dev->addr, name, sizeof(name)); 132 133 return qat_pci_get_named_dev(name); 134 } 135 136 static enum qat_device_gen 137 pick_gen(const struct rte_pci_device *pci_dev) 138 { 139 switch (pci_dev->id.device_id) { 140 case 0x0443: 141 return QAT_GEN1; 142 case 0x37c9: 143 case 0x19e3: 144 case 0x6f55: 145 case 0x18ef: 146 return QAT_GEN2; 147 case 0x18a1: 148 case 0x578b: 149 return QAT_GEN3; 150 case 0x4941: 151 case 0x4943: 152 case 0x4945: 153 return QAT_GEN4; 154 case 0x4947: 155 return QAT_GEN5; 156 case 0x1454: 157 return QAT_GEN_LCE; 158 case 0x0da5: 159 return QAT_VQAT; 160 default: 161 QAT_LOG(ERR, "Invalid dev_id, can't determine generation"); 162 return QAT_N_GENS; 163 } 164 } 165 166 static int 167 wireless_slice_support(uint16_t pci_dev_id) 168 { 169 return pci_dev_id == 0x578b || 170 pci_dev_id == 0x4947; 171 } 172 173 /* This function base on the atoi function peculiarity, non integral part 174 * other than the equals sign is ignored. It will not work with other conversion 175 * functions like strt*. 176 */ 177 char *qat_dev_cmdline_get_val(struct qat_pci_device *qat_dev, 178 const char *key) 179 { 180 if (qat_dev->command_line == NULL) 181 return NULL; 182 key = strstr(qat_dev->command_line, key); 183 /* At this point, a key should be validated */ 184 return key ? strchr(key, '=') + 1 : NULL; 185 } 186 187 static int cmdline_validate(const char *arg) 188 { 189 int i, len; 190 char *eq_sign = strchr(arg, '='); 191 /* Check for the equal sign */ 192 if (eq_sign == NULL) { 193 QAT_LOG(ERR, "malformed string, no equals sign, %s", arg); 194 return 0; 195 } 196 /* Check if an argument is not empty */ 197 len = strlen(eq_sign) - 1; 198 if (len == 0) { 199 QAT_LOG(ERR, "malformed string, empty argument, %s", arg); 200 return 0; 201 } 202 len = eq_sign - arg; 203 for (i = 0; i < QAT_MAX_SERVICES + 1; i++) { 204 int j = 0; 205 const char *def = NULL; 206 207 if (!qat_cmdline_defines[i]) 208 continue; 209 while ((def = qat_cmdline_defines[i][j++])) { 210 if (strncmp(def, arg, len)) 211 continue; 212 QAT_LOG(DEBUG, "Found %s command line argument", 213 def); 214 return 1; 215 } 216 } 217 return 0; 218 } 219 220 static int 221 qat_dev_parse_command_line(struct qat_pci_device *qat_dev, 222 struct rte_devargs *devargs) 223 { 224 int len = 0; 225 char *token = NULL; 226 227 if (!devargs) 228 return 0; 229 230 len = strlen(devargs->drv_str); 231 if (len == 0) 232 return 0; 233 /* Allocate per-device command line */ 234 qat_dev->command_line = rte_malloc(NULL, len, 0); 235 if (qat_dev->command_line == NULL) { 236 QAT_LOG(ERR, "Cannot allocate memory for command line"); 237 return -1; 238 } 239 strcpy(qat_dev->command_line, devargs->drv_str); 240 token = strtok(qat_dev->command_line, ","); 241 while (token != NULL) { 242 if (!cmdline_validate(token)) { 243 QAT_LOG(ERR, "Incorrect command line argument: %s", 244 token); 245 return -1; 246 } 247 token = strtok(NULL, ","); 248 } 249 /* Copy once againe the entire string, strtok already altered the contents */ 250 strcpy(qat_dev->command_line, devargs->drv_str); 251 return 0; 252 } 253 254 static struct qat_pci_device * 255 qat_pci_device_allocate(struct rte_pci_device *pci_dev) 256 { 257 struct qat_pci_device *qat_dev; 258 enum qat_device_gen qat_dev_gen; 259 uint8_t qat_dev_id = 0; 260 char name[QAT_DEV_NAME_MAX_LEN]; 261 struct rte_devargs *devargs = pci_dev->device.devargs; 262 struct qat_dev_hw_spec_funcs *ops_hw; 263 struct rte_mem_resource *mem_resource; 264 const struct rte_memzone *qat_dev_mz; 265 int qat_dev_size, extra_size; 266 char *cmdline = NULL; 267 268 rte_pci_device_name(&pci_dev->addr, name, sizeof(name)); 269 snprintf(name+strlen(name), QAT_DEV_NAME_MAX_LEN-strlen(name), "_qat"); 270 271 qat_dev_gen = pick_gen(pci_dev); 272 if (qat_dev_gen == QAT_N_GENS) { 273 QAT_LOG(ERR, "Invalid dev_id, can't determine generation"); 274 return NULL; 275 } 276 277 if (rte_eal_process_type() == RTE_PROC_SECONDARY) { 278 const struct rte_memzone *mz = rte_memzone_lookup(name); 279 280 if (mz == NULL) { 281 QAT_LOG(ERR, 282 "Secondary can't find %s mz, did primary create device?", 283 name); 284 return NULL; 285 } 286 qat_dev = mz->addr; 287 qat_pci_devs[qat_dev->qat_dev_id].mz = mz; 288 qat_pci_devs[qat_dev->qat_dev_id].pci_dev = pci_dev; 289 qat_nb_pci_devices++; 290 QAT_LOG(DEBUG, "QAT device %d found, name %s, total QATs %d", 291 qat_dev->qat_dev_id, qat_dev->name, qat_nb_pci_devices); 292 return qat_dev; 293 } 294 295 if (qat_pci_get_named_dev(name) != NULL) { 296 QAT_LOG(ERR, "QAT device with name %s already allocated!", 297 name); 298 return NULL; 299 } 300 301 qat_dev_id = qat_pci_find_free_device_index(); 302 if (qat_dev_id == RTE_PMD_QAT_MAX_PCI_DEVICES) { 303 QAT_LOG(ERR, "Reached maximum number of QAT devices"); 304 return NULL; 305 } 306 307 extra_size = qat_pci_get_extra_size(qat_dev_gen); 308 if (extra_size < 0) { 309 QAT_LOG(ERR, "QAT internal error: no pci pointer for gen %d", 310 qat_dev_gen); 311 return NULL; 312 } 313 314 qat_dev_size = sizeof(struct qat_pci_device) + sizeof(void *) * 315 QAT_MAX_SERVICES + extra_size; 316 qat_dev_mz = rte_memzone_reserve(name, qat_dev_size, 317 rte_socket_id(), 0); 318 319 if (qat_dev_mz == NULL) { 320 QAT_LOG(ERR, "Error when allocating memzone for QAT_%d", 321 qat_dev_id); 322 return NULL; 323 } 324 325 qat_dev = qat_dev_mz->addr; 326 memset(qat_dev, 0, qat_dev_size); 327 qat_dev->dev_private = qat_dev + 1; 328 strlcpy(qat_dev->name, name, QAT_DEV_NAME_MAX_LEN); 329 qat_dev->qat_dev_id = qat_dev_id; 330 qat_dev->qat_dev_gen = qat_dev_gen; 331 qat_pci_devs[qat_dev_id].pci_dev = pci_dev; 332 333 if (wireless_slice_support(pci_dev->id.device_id)) 334 qat_dev->has_wireless_slice = 1; 335 336 ops_hw = qat_dev_hw_spec[qat_dev->qat_dev_gen]; 337 NOT_NULL(ops_hw->qat_dev_get_misc_bar, goto error, 338 "QAT internal error! qat_dev_get_misc_bar function not set"); 339 if (ops_hw->qat_dev_get_misc_bar(&mem_resource, pci_dev) == 0) { 340 if (mem_resource->addr == NULL) { 341 QAT_LOG(ERR, "QAT cannot get access to VF misc bar"); 342 goto error; 343 } 344 qat_dev->misc_bar_io_addr = mem_resource->addr; 345 } else 346 qat_dev->misc_bar_io_addr = NULL; 347 348 /* Parse the command line */ 349 if (qat_dev_parse_command_line(qat_dev, devargs)) 350 goto error; 351 352 /* Parse the arguments */ 353 cmdline = qat_dev_cmdline_get_val(qat_dev, QAT_LEGACY_CAPA); 354 if (cmdline) 355 qat_legacy_capa = atoi(cmdline); 356 357 if (qat_read_qp_config(qat_dev)) { 358 QAT_LOG(ERR, 359 "Cannot acquire ring configuration for QAT_%d", 360 qat_dev_id); 361 goto error; 362 } 363 NOT_NULL(ops_hw->qat_dev_reset_ring_pairs, goto error, 364 "QAT internal error! Reset ring pairs function not set, gen : %d", 365 qat_dev_gen); 366 if (ops_hw->qat_dev_reset_ring_pairs(qat_dev)) { 367 QAT_LOG(ERR, 368 "Cannot reset ring pairs, does pf driver supports pf2vf comms?" 369 ); 370 goto error; 371 } 372 NOT_NULL(ops_hw->qat_dev_get_slice_map, goto error, 373 "QAT internal error! Read slice function not set, gen : %d", 374 qat_dev_gen); 375 if (ops_hw->qat_dev_get_slice_map(&qat_dev->slice_map, pci_dev) < 0) { 376 RTE_LOG(ERR, EAL, 377 "Cannot read slice configuration\n"); 378 goto error; 379 } 380 rte_spinlock_init(&qat_dev->arb_csr_lock); 381 382 /* No errors when allocating, attach memzone with 383 * qat_dev to list of devices 384 */ 385 qat_pci_devs[qat_dev_id].mz = qat_dev_mz; 386 qat_nb_pci_devices++; 387 388 QAT_LOG(DEBUG, "QAT device %d found, name %s, total QATs %d", 389 qat_dev->qat_dev_id, qat_dev->name, qat_nb_pci_devices); 390 391 return qat_dev; 392 error: 393 rte_free(qat_dev->command_line); 394 if (rte_memzone_free(qat_dev_mz)) { 395 QAT_LOG(DEBUG, 396 "QAT internal error! Trying to free already allocated memzone: %s", 397 qat_dev_mz->name); 398 } 399 return NULL; 400 } 401 402 static int 403 qat_pci_device_release(struct rte_pci_device *pci_dev) 404 { 405 struct qat_pci_device *qat_dev; 406 char name[QAT_DEV_NAME_MAX_LEN]; 407 int busy = 0, i; 408 409 if (pci_dev == NULL) 410 return -EINVAL; 411 412 rte_pci_device_name(&pci_dev->addr, name, sizeof(name)); 413 snprintf(name+strlen(name), QAT_DEV_NAME_MAX_LEN-strlen(name), "_qat"); 414 qat_dev = qat_pci_get_named_dev(name); 415 if (qat_dev != NULL) { 416 417 struct qat_device_info *inst = 418 &qat_pci_devs[qat_dev->qat_dev_id]; 419 /* Check that there are no service devs still on pci device */ 420 421 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 422 for (i = 0; i < QAT_MAX_SERVICES; i++) { 423 if (qat_dev->pmd[i] == NULL) 424 continue; 425 QAT_LOG(DEBUG, "QAT %s device %s is busy", 426 qat_service[i].name, name); 427 busy = 1; 428 if (busy) 429 return -EBUSY; 430 rte_memzone_free(inst->mz); 431 } 432 } 433 memset(inst, 0, sizeof(struct qat_device_info)); 434 qat_nb_pci_devices--; 435 QAT_LOG(DEBUG, "QAT device %s released, total QATs %d", 436 name, qat_nb_pci_devices); 437 } 438 return 0; 439 } 440 441 static int 442 qat_pci_dev_destroy(struct qat_pci_device *qat_pci_dev, 443 struct rte_pci_device *pci_dev) 444 { 445 int i; 446 447 for (i = 0; i < QAT_MAX_SERVICES; i++) { 448 if (!qat_service[i].dev_create) 449 continue; 450 qat_service[i].dev_destroy(qat_pci_dev); 451 } 452 return qat_pci_device_release(pci_dev); 453 } 454 455 static int qat_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 456 struct rte_pci_device *pci_dev) 457 { 458 int i, ret = 0, num_pmds_created = 0; 459 struct qat_pci_device *qat_pci_dev; 460 461 QAT_LOG(DEBUG, "Found QAT device at %02x:%02x.%x", 462 pci_dev->addr.bus, 463 pci_dev->addr.devid, 464 pci_dev->addr.function); 465 466 qat_pci_dev = qat_pci_device_allocate(pci_dev); 467 if (qat_pci_dev == NULL) 468 return -ENODEV; 469 470 for (i = 0; i < QAT_MAX_SERVICES; i++) { 471 if (!qat_service[i].dev_create) 472 continue; 473 ret = qat_service[i].dev_create(qat_pci_dev); 474 if (ret == 0) 475 num_pmds_created++; 476 else { 477 QAT_LOG(WARNING, "Failed to create %s PMD on device %s", 478 qat_service[i].name, 479 qat_pci_dev->name); 480 } 481 } 482 483 if (num_pmds_created == 0) 484 qat_pci_dev_destroy(qat_pci_dev, pci_dev); 485 486 return 0; 487 } 488 489 static int 490 qat_pci_remove(struct rte_pci_device *pci_dev) 491 { 492 struct qat_pci_device *qat_pci_dev; 493 494 if (pci_dev == NULL) 495 return -EINVAL; 496 497 qat_pci_dev = qat_get_qat_dev_from_pci_dev(pci_dev); 498 if (qat_pci_dev == NULL) 499 return 0; 500 501 return qat_pci_dev_destroy(qat_pci_dev, pci_dev); 502 } 503 504 static struct rte_pci_driver rte_qat_pmd = { 505 .id_table = pci_id_qat_map, 506 .drv_flags = RTE_PCI_DRV_NEED_MAPPING, 507 .probe = qat_pci_probe, 508 .remove = qat_pci_remove 509 }; 510 511 RTE_PMD_REGISTER_PCI(QAT_PCI_NAME, rte_qat_pmd); 512 RTE_PMD_REGISTER_PCI_TABLE(QAT_PCI_NAME, pci_id_qat_map); 513 RTE_PMD_REGISTER_KMOD_DEP(QAT_PCI_NAME, "* igb_uio | uio_pci_generic | vfio-pci"); 514