1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018-2022 Intel Corporation 3 */ 4 5 #include <rte_string_fns.h> 6 #include <rte_devargs.h> 7 #include <ctype.h> 8 9 #include "qat_device.h" 10 #include "adf_transport_access_macros.h" 11 #include "qat_sym.h" 12 #include "qat_comp_pmd.h" 13 #include "adf_pf2vf_msg.h" 14 #include "qat_pf2vf.h" 15 16 #define NOT_NULL(arg, func, msg, ...) \ 17 do { \ 18 if (arg == NULL) { \ 19 QAT_LOG(ERR, \ 20 msg, ##__VA_ARGS__); \ 21 func; \ 22 } \ 23 } while (0) 24 25 /* Hardware device information per generation */ 26 struct qat_gen_hw_data qat_gen_config[QAT_N_GENS]; 27 struct qat_dev_hw_spec_funcs *qat_dev_hw_spec[QAT_N_GENS]; 28 29 struct qat_service qat_service[QAT_MAX_SERVICES]; 30 31 /* per-process array of device data */ 32 struct qat_device_info qat_pci_devs[RTE_PMD_QAT_MAX_PCI_DEVICES]; 33 static int qat_nb_pci_devices; 34 35 /* 36 * The set of PCI devices this driver supports 37 */ 38 39 static const struct rte_pci_id pci_id_qat_map[] = { 40 { 41 RTE_PCI_DEVICE(0x8086, 0x0443), 42 }, 43 { 44 RTE_PCI_DEVICE(0x8086, 0x37c9), 45 }, 46 { 47 RTE_PCI_DEVICE(0x8086, 0x19e3), 48 }, 49 { 50 RTE_PCI_DEVICE(0x8086, 0x6f55), 51 }, 52 { 53 RTE_PCI_DEVICE(0x8086, 0x18ef), 54 }, 55 { 56 RTE_PCI_DEVICE(0x8086, 0x18a1), 57 }, 58 { 59 RTE_PCI_DEVICE(0x8086, 0x578b), 60 }, 61 { 62 RTE_PCI_DEVICE(0x8086, 0x4941), 63 }, 64 { 65 RTE_PCI_DEVICE(0x8086, 0x4943), 66 }, 67 { 68 RTE_PCI_DEVICE(0x8086, 0x4945), 69 }, 70 { 71 RTE_PCI_DEVICE(0x8086, 0x4947), 72 }, 73 { 74 RTE_PCI_DEVICE(0x8086, 0x1454), 75 }, 76 { 77 RTE_PCI_DEVICE(0x8086, 0x0da5), 78 }, 79 {.device_id = 0}, 80 }; 81 82 static int 83 qat_pci_get_extra_size(enum qat_device_gen qat_dev_gen) 84 { 85 struct qat_dev_hw_spec_funcs *ops_hw = 86 qat_dev_hw_spec[qat_dev_gen]; 87 if (ops_hw->qat_dev_get_extra_size == NULL) 88 return -ENOTSUP; 89 return ops_hw->qat_dev_get_extra_size(); 90 } 91 92 static struct qat_pci_device * 93 qat_pci_get_named_dev(const char *name) 94 { 95 unsigned int i; 96 97 if (name == NULL) 98 return NULL; 99 100 for (i = 0; i < RTE_PMD_QAT_MAX_PCI_DEVICES; i++) { 101 if (qat_pci_devs[i].mz && 102 (strcmp(((struct qat_pci_device *) 103 qat_pci_devs[i].mz->addr)->name, name) 104 == 0)) 105 return (struct qat_pci_device *) 106 qat_pci_devs[i].mz->addr; 107 } 108 109 return NULL; 110 } 111 112 static uint8_t 113 qat_pci_find_free_device_index(void) 114 { 115 uint8_t dev_id; 116 117 for (dev_id = 0; dev_id < RTE_PMD_QAT_MAX_PCI_DEVICES; 118 dev_id++) { 119 if (qat_pci_devs[dev_id].mz == NULL) 120 break; 121 } 122 return dev_id; 123 } 124 125 static struct qat_pci_device * 126 qat_get_qat_dev_from_pci_dev(struct rte_pci_device *pci_dev) 127 { 128 char name[QAT_DEV_NAME_MAX_LEN]; 129 130 rte_pci_device_name(&pci_dev->addr, name, sizeof(name)); 131 132 return qat_pci_get_named_dev(name); 133 } 134 135 static enum qat_device_gen 136 pick_gen(const struct rte_pci_device *pci_dev) 137 { 138 switch (pci_dev->id.device_id) { 139 case 0x0443: 140 return QAT_GEN1; 141 case 0x37c9: 142 case 0x19e3: 143 case 0x6f55: 144 case 0x18ef: 145 return QAT_GEN2; 146 case 0x18a1: 147 case 0x578b: 148 return QAT_GEN3; 149 case 0x4941: 150 case 0x4943: 151 case 0x4945: 152 return QAT_GEN4; 153 case 0x4947: 154 return QAT_GEN5; 155 case 0x1454: 156 return QAT_GEN_LCE; 157 case 0x0da5: 158 return QAT_VQAT; 159 default: 160 QAT_LOG(ERR, "Invalid dev_id, can't determine generation"); 161 return QAT_N_GENS; 162 } 163 } 164 165 static int 166 wireless_slice_support(uint16_t pci_dev_id) 167 { 168 return pci_dev_id == 0x578b || 169 pci_dev_id == 0x4947; 170 } 171 172 /* This function base on the atoi function peculiarity, non integral part 173 * other than the equals sign is ignored. It will not work with other conversion 174 * functions like strt*. 175 */ 176 char *qat_dev_cmdline_get_val(struct qat_pci_device *qat_dev, 177 const char *key) 178 { 179 if (qat_dev->command_line == NULL) 180 return NULL; 181 key = strstr(qat_dev->command_line, key); 182 /* At this point, a key should be validated */ 183 return key ? strchr(key, '=') + 1 : NULL; 184 } 185 186 static int cmdline_validate(const char *arg) 187 { 188 int i, len; 189 char *eq_sign = strchr(arg, '='); 190 /* Check for the equal sign */ 191 if (eq_sign == NULL) { 192 QAT_LOG(ERR, "malformed string, no equals sign, %s", arg); 193 return 0; 194 } 195 /* Check if an argument is not empty */ 196 len = strlen(eq_sign) - 1; 197 if (len == 0) { 198 QAT_LOG(ERR, "malformed string, empty argument, %s", arg); 199 return 0; 200 } 201 len = eq_sign - arg; 202 for (i = 0; i < QAT_MAX_SERVICES + 1; i++) { 203 int j = 0; 204 const char *def = NULL; 205 206 if (!qat_cmdline_defines[i]) 207 continue; 208 while ((def = qat_cmdline_defines[i][j++])) { 209 if (strncmp(def, arg, len)) 210 continue; 211 QAT_LOG(DEBUG, "Found %s command line argument", 212 def); 213 return 1; 214 } 215 } 216 return 0; 217 } 218 219 static int 220 qat_dev_parse_command_line(struct qat_pci_device *qat_dev, 221 struct rte_devargs *devargs) 222 { 223 int len = 0; 224 char *token = NULL; 225 226 if (!devargs) 227 return 0; 228 229 len = strlen(devargs->drv_str); 230 if (len == 0) 231 return 0; 232 /* Allocate per-device command line */ 233 qat_dev->command_line = rte_malloc(NULL, len, 0); 234 if (qat_dev->command_line == NULL) { 235 QAT_LOG(ERR, "Cannot allocate memory for command line"); 236 return -1; 237 } 238 strcpy(qat_dev->command_line, devargs->drv_str); 239 token = strtok(qat_dev->command_line, ","); 240 while (token != NULL) { 241 if (!cmdline_validate(token)) { 242 QAT_LOG(ERR, "Incorrect command line argument: %s", 243 token); 244 return -1; 245 } 246 token = strtok(NULL, ","); 247 } 248 /* Copy once againe the entire string, strtok already altered the contents */ 249 strcpy(qat_dev->command_line, devargs->drv_str); 250 return 0; 251 } 252 253 static struct qat_pci_device * 254 qat_pci_device_allocate(struct rte_pci_device *pci_dev) 255 { 256 struct qat_pci_device *qat_dev; 257 enum qat_device_gen qat_dev_gen; 258 uint8_t qat_dev_id = 0; 259 char name[QAT_DEV_NAME_MAX_LEN]; 260 struct rte_devargs *devargs = pci_dev->device.devargs; 261 struct qat_dev_hw_spec_funcs *ops_hw; 262 struct rte_mem_resource *mem_resource; 263 const struct rte_memzone *qat_dev_mz; 264 int qat_dev_size, extra_size; 265 char *cmdline = NULL; 266 267 rte_pci_device_name(&pci_dev->addr, name, sizeof(name)); 268 snprintf(name+strlen(name), QAT_DEV_NAME_MAX_LEN-strlen(name), "_qat"); 269 270 qat_dev_gen = pick_gen(pci_dev); 271 if (qat_dev_gen == QAT_N_GENS) { 272 QAT_LOG(ERR, "Invalid dev_id, can't determine generation"); 273 return NULL; 274 } 275 276 if (rte_eal_process_type() == RTE_PROC_SECONDARY) { 277 const struct rte_memzone *mz = rte_memzone_lookup(name); 278 279 if (mz == NULL) { 280 QAT_LOG(ERR, 281 "Secondary can't find %s mz, did primary create device?", 282 name); 283 return NULL; 284 } 285 qat_dev = mz->addr; 286 qat_pci_devs[qat_dev->qat_dev_id].mz = mz; 287 qat_pci_devs[qat_dev->qat_dev_id].pci_dev = pci_dev; 288 qat_nb_pci_devices++; 289 QAT_LOG(DEBUG, "QAT device %d found, name %s, total QATs %d", 290 qat_dev->qat_dev_id, qat_dev->name, qat_nb_pci_devices); 291 return qat_dev; 292 } 293 294 if (qat_pci_get_named_dev(name) != NULL) { 295 QAT_LOG(ERR, "QAT device with name %s already allocated!", 296 name); 297 return NULL; 298 } 299 300 qat_dev_id = qat_pci_find_free_device_index(); 301 if (qat_dev_id == RTE_PMD_QAT_MAX_PCI_DEVICES) { 302 QAT_LOG(ERR, "Reached maximum number of QAT devices"); 303 return NULL; 304 } 305 306 extra_size = qat_pci_get_extra_size(qat_dev_gen); 307 if (extra_size < 0) { 308 QAT_LOG(ERR, "QAT internal error: no pci pointer for gen %d", 309 qat_dev_gen); 310 return NULL; 311 } 312 313 qat_dev_size = sizeof(struct qat_pci_device) + sizeof(void *) * 314 QAT_MAX_SERVICES + extra_size; 315 qat_dev_mz = rte_memzone_reserve(name, qat_dev_size, 316 rte_socket_id(), 0); 317 318 if (qat_dev_mz == NULL) { 319 QAT_LOG(ERR, "Error when allocating memzone for QAT_%d", 320 qat_dev_id); 321 return NULL; 322 } 323 324 qat_dev = qat_dev_mz->addr; 325 memset(qat_dev, 0, qat_dev_size); 326 qat_dev->dev_private = qat_dev + 1; 327 strlcpy(qat_dev->name, name, QAT_DEV_NAME_MAX_LEN); 328 qat_dev->qat_dev_id = qat_dev_id; 329 qat_dev->qat_dev_gen = qat_dev_gen; 330 qat_pci_devs[qat_dev_id].pci_dev = pci_dev; 331 332 if (wireless_slice_support(pci_dev->id.device_id)) 333 qat_dev->options.has_wireless_slice = 1; 334 335 ops_hw = qat_dev_hw_spec[qat_dev->qat_dev_gen]; 336 NOT_NULL(ops_hw->qat_dev_get_misc_bar, goto error, 337 "QAT internal error! qat_dev_get_misc_bar function not set"); 338 if (ops_hw->qat_dev_get_misc_bar(&mem_resource, pci_dev) == 0) { 339 if (mem_resource->addr == NULL) { 340 QAT_LOG(ERR, "QAT cannot get access to VF misc bar"); 341 goto error; 342 } 343 qat_dev->misc_bar_io_addr = mem_resource->addr; 344 } else 345 qat_dev->misc_bar_io_addr = NULL; 346 347 /* Parse the command line */ 348 if (qat_dev_parse_command_line(qat_dev, devargs)) 349 goto error; 350 351 /* Parse the arguments */ 352 cmdline = qat_dev_cmdline_get_val(qat_dev, QAT_LEGACY_CAPA); 353 if (cmdline) 354 qat_dev->options.legacy_alg = atoi(cmdline); 355 356 if (qat_read_qp_config(qat_dev)) { 357 QAT_LOG(ERR, 358 "Cannot acquire ring configuration for QAT_%d", 359 qat_dev_id); 360 goto error; 361 } 362 NOT_NULL(ops_hw->qat_dev_reset_ring_pairs, goto error, 363 "QAT internal error! Reset ring pairs function not set, gen : %d", 364 qat_dev_gen); 365 if (ops_hw->qat_dev_reset_ring_pairs(qat_dev)) { 366 QAT_LOG(ERR, 367 "Cannot reset ring pairs, does pf driver supports pf2vf comms?" 368 ); 369 goto error; 370 } 371 NOT_NULL(ops_hw->qat_dev_get_slice_map, goto error, 372 "QAT internal error! Read slice function not set, gen : %d", 373 qat_dev_gen); 374 if (ops_hw->qat_dev_get_slice_map(&qat_dev->options.slice_map, pci_dev) < 0) { 375 QAT_LOG(ERR, 376 "Cannot read slice configuration"); 377 goto error; 378 } 379 rte_spinlock_init(&qat_dev->arb_csr_lock); 380 381 /* No errors when allocating, attach memzone with 382 * qat_dev to list of devices 383 */ 384 qat_pci_devs[qat_dev_id].mz = qat_dev_mz; 385 qat_nb_pci_devices++; 386 387 QAT_LOG(DEBUG, "QAT device %d found, name %s, total QATs %d", 388 qat_dev->qat_dev_id, qat_dev->name, qat_nb_pci_devices); 389 390 return qat_dev; 391 error: 392 rte_free(qat_dev->command_line); 393 rte_memzone_free(qat_dev_mz); 394 return NULL; 395 } 396 397 static int 398 qat_pci_device_release(struct rte_pci_device *pci_dev) 399 { 400 struct qat_pci_device *qat_dev; 401 char name[QAT_DEV_NAME_MAX_LEN]; 402 int busy = 0, i; 403 404 if (pci_dev == NULL) 405 return -EINVAL; 406 407 rte_pci_device_name(&pci_dev->addr, name, sizeof(name)); 408 snprintf(name+strlen(name), QAT_DEV_NAME_MAX_LEN-strlen(name), "_qat"); 409 qat_dev = qat_pci_get_named_dev(name); 410 if (qat_dev != NULL) { 411 412 struct qat_device_info *inst = 413 &qat_pci_devs[qat_dev->qat_dev_id]; 414 /* Check that there are no service devs still on pci device */ 415 416 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 417 for (i = 0; i < QAT_MAX_SERVICES; i++) { 418 if (qat_dev->pmd[i] != NULL) { 419 QAT_LOG(DEBUG, "QAT %s device %s is busy", 420 qat_service[i].name, name); 421 busy = 1; 422 } 423 } 424 if (busy) 425 return -EBUSY; 426 rte_memzone_free(inst->mz); 427 } 428 memset(inst, 0, sizeof(struct qat_device_info)); 429 qat_nb_pci_devices--; 430 QAT_LOG(DEBUG, "QAT device %s released, total QATs %d", 431 name, qat_nb_pci_devices); 432 } 433 return 0; 434 } 435 436 static int 437 qat_pci_dev_destroy(struct qat_pci_device *qat_pci_dev, 438 struct rte_pci_device *pci_dev) 439 { 440 int i; 441 442 for (i = 0; i < QAT_MAX_SERVICES; i++) { 443 if (!qat_service[i].dev_destroy) 444 continue; 445 qat_service[i].dev_destroy(qat_pci_dev); 446 } 447 return qat_pci_device_release(pci_dev); 448 } 449 450 static int qat_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 451 struct rte_pci_device *pci_dev) 452 { 453 int i, ret = 0, num_pmds_created = 0; 454 struct qat_pci_device *qat_pci_dev; 455 456 QAT_LOG(DEBUG, "Found QAT device at %02x:%02x.%x", 457 pci_dev->addr.bus, 458 pci_dev->addr.devid, 459 pci_dev->addr.function); 460 461 qat_pci_dev = qat_pci_device_allocate(pci_dev); 462 if (qat_pci_dev == NULL) 463 return -ENODEV; 464 465 for (i = 0; i < QAT_MAX_SERVICES; i++) { 466 if (!qat_service[i].dev_create) 467 continue; 468 ret = qat_service[i].dev_create(qat_pci_dev); 469 if (ret == 0) 470 num_pmds_created++; 471 else { 472 QAT_LOG(WARNING, "Failed to create %s PMD on device %s", 473 qat_service[i].name, 474 qat_pci_dev->name); 475 } 476 } 477 478 if (num_pmds_created == 0) 479 qat_pci_dev_destroy(qat_pci_dev, pci_dev); 480 481 return 0; 482 } 483 484 static int 485 qat_pci_remove(struct rte_pci_device *pci_dev) 486 { 487 struct qat_pci_device *qat_pci_dev; 488 489 if (pci_dev == NULL) 490 return -EINVAL; 491 492 qat_pci_dev = qat_get_qat_dev_from_pci_dev(pci_dev); 493 if (qat_pci_dev == NULL) 494 return 0; 495 496 return qat_pci_dev_destroy(qat_pci_dev, pci_dev); 497 } 498 499 static struct rte_pci_driver rte_qat_pmd = { 500 .id_table = pci_id_qat_map, 501 .drv_flags = RTE_PCI_DRV_NEED_MAPPING, 502 .probe = qat_pci_probe, 503 .remove = qat_pci_remove 504 }; 505 506 RTE_PMD_REGISTER_PCI(QAT_PCI_NAME, rte_qat_pmd); 507 RTE_PMD_REGISTER_PCI_TABLE(QAT_PCI_NAME, pci_id_qat_map); 508 RTE_PMD_REGISTER_KMOD_DEP(QAT_PCI_NAME, "* igb_uio | uio_pci_generic | vfio-pci"); 509