1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018-2020 Intel Corporation 3 */ 4 5 #include <rte_string_fns.h> 6 #include <rte_devargs.h> 7 #include <ctype.h> 8 9 #include "qat_device.h" 10 #include "adf_transport_access_macros.h" 11 #include "qat_sym_pmd.h" 12 #include "qat_comp_pmd.h" 13 #include "adf_pf2vf_msg.h" 14 #include "qat_pf2vf.h" 15 16 /* pv2vf data Gen 4*/ 17 struct qat_pf2vf_dev qat_pf2vf_gen4 = { 18 .pf2vf_offset = ADF_4XXXIOV_PF2VM_OFFSET, 19 .vf2pf_offset = ADF_4XXXIOV_VM2PF_OFFSET, 20 .pf2vf_type_shift = ADF_PFVF_2X_MSGTYPE_SHIFT, 21 .pf2vf_type_mask = ADF_PFVF_2X_MSGTYPE_MASK, 22 .pf2vf_data_shift = ADF_PFVF_2X_MSGDATA_SHIFT, 23 .pf2vf_data_mask = ADF_PFVF_2X_MSGDATA_MASK, 24 }; 25 26 /* Hardware device information per generation */ 27 __extension__ 28 struct qat_gen_hw_data qat_gen_config[] = { 29 [QAT_GEN1] = { 30 .dev_gen = QAT_GEN1, 31 .qp_hw_data = qat_gen1_qps, 32 .comp_num_im_bufs_required = QAT_NUM_INTERM_BUFS_GEN1 33 }, 34 [QAT_GEN2] = { 35 .dev_gen = QAT_GEN2, 36 .qp_hw_data = qat_gen1_qps, 37 /* gen2 has same ring layout as gen1 */ 38 .comp_num_im_bufs_required = QAT_NUM_INTERM_BUFS_GEN2 39 }, 40 [QAT_GEN3] = { 41 .dev_gen = QAT_GEN3, 42 .qp_hw_data = qat_gen3_qps, 43 .comp_num_im_bufs_required = QAT_NUM_INTERM_BUFS_GEN3 44 }, 45 [QAT_GEN4] = { 46 .dev_gen = QAT_GEN4, 47 .qp_hw_data = NULL, 48 .comp_num_im_bufs_required = QAT_NUM_INTERM_BUFS_GEN3, 49 .pf2vf_dev = &qat_pf2vf_gen4 50 }, 51 }; 52 53 /* per-process array of device data */ 54 struct qat_device_info qat_pci_devs[RTE_PMD_QAT_MAX_PCI_DEVICES]; 55 static int qat_nb_pci_devices; 56 57 /* 58 * The set of PCI devices this driver supports 59 */ 60 61 static const struct rte_pci_id pci_id_qat_map[] = { 62 { 63 RTE_PCI_DEVICE(0x8086, 0x0443), 64 }, 65 { 66 RTE_PCI_DEVICE(0x8086, 0x37c9), 67 }, 68 { 69 RTE_PCI_DEVICE(0x8086, 0x19e3), 70 }, 71 { 72 RTE_PCI_DEVICE(0x8086, 0x6f55), 73 }, 74 { 75 RTE_PCI_DEVICE(0x8086, 0x18ef), 76 }, 77 { 78 RTE_PCI_DEVICE(0x8086, 0x18a1), 79 }, 80 { 81 RTE_PCI_DEVICE(0x8086, 0x4941), 82 }, 83 {.device_id = 0}, 84 }; 85 86 static struct qat_pci_device * 87 qat_pci_get_named_dev(const char *name) 88 { 89 unsigned int i; 90 91 if (name == NULL) 92 return NULL; 93 94 for (i = 0; i < RTE_PMD_QAT_MAX_PCI_DEVICES; i++) { 95 if (qat_pci_devs[i].mz && 96 (strcmp(((struct qat_pci_device *) 97 qat_pci_devs[i].mz->addr)->name, name) 98 == 0)) 99 return (struct qat_pci_device *) 100 qat_pci_devs[i].mz->addr; 101 } 102 103 return NULL; 104 } 105 106 static uint8_t 107 qat_pci_find_free_device_index(void) 108 { 109 uint8_t dev_id; 110 111 for (dev_id = 0; dev_id < RTE_PMD_QAT_MAX_PCI_DEVICES; 112 dev_id++) { 113 if (qat_pci_devs[dev_id].mz == NULL) 114 break; 115 } 116 return dev_id; 117 } 118 119 struct qat_pci_device * 120 qat_get_qat_dev_from_pci_dev(struct rte_pci_device *pci_dev) 121 { 122 char name[QAT_DEV_NAME_MAX_LEN]; 123 124 rte_pci_device_name(&pci_dev->addr, name, sizeof(name)); 125 126 return qat_pci_get_named_dev(name); 127 } 128 129 static int 130 qat_gen4_reset_ring_pair(struct qat_pci_device *qat_pci_dev) 131 { 132 int ret = 0, i; 133 uint8_t data[4]; 134 struct qat_pf2vf_msg pf2vf_msg; 135 136 pf2vf_msg.msg_type = ADF_VF2PF_MSGTYPE_RP_RESET; 137 pf2vf_msg.block_hdr = -1; 138 for (i = 0; i < QAT_GEN4_BUNDLE_NUM; i++) { 139 pf2vf_msg.msg_data = i; 140 ret = qat_pf2vf_exch_msg(qat_pci_dev, pf2vf_msg, 1, data); 141 if (ret) { 142 QAT_LOG(ERR, "QAT error when reset bundle no %d", 143 i); 144 return ret; 145 } 146 } 147 148 return 0; 149 } 150 151 int qat_query_svc(struct qat_pci_device *qat_dev, uint8_t *val) 152 { 153 int ret = -(EINVAL); 154 struct qat_pf2vf_msg pf2vf_msg; 155 156 if (qat_dev->qat_dev_gen == QAT_GEN4) { 157 pf2vf_msg.msg_type = ADF_VF2PF_MSGTYPE_GET_SMALL_BLOCK_REQ; 158 pf2vf_msg.block_hdr = ADF_VF2PF_BLOCK_MSG_GET_RING_TO_SVC_REQ; 159 pf2vf_msg.msg_data = 2; 160 ret = qat_pf2vf_exch_msg(qat_dev, pf2vf_msg, 2, val); 161 } 162 163 return ret; 164 } 165 166 167 static void qat_dev_parse_cmd(const char *str, struct qat_dev_cmd_param 168 *qat_dev_cmd_param) 169 { 170 int i = 0; 171 const char *param; 172 173 while (1) { 174 char value_str[4] = { }; 175 176 param = qat_dev_cmd_param[i].name; 177 if (param == NULL) 178 return; 179 long value = 0; 180 const char *arg = strstr(str, param); 181 const char *arg2 = NULL; 182 183 if (arg) { 184 arg2 = arg + strlen(param); 185 if (*arg2 != '=') { 186 QAT_LOG(DEBUG, "parsing error '=' sign" 187 " should immediately follow %s", 188 param); 189 arg2 = NULL; 190 } else 191 arg2++; 192 } else { 193 QAT_LOG(DEBUG, "%s not provided", param); 194 } 195 if (arg2) { 196 int iter = 0; 197 while (iter < 2) { 198 if (!isdigit(*(arg2 + iter))) 199 break; 200 iter++; 201 } 202 if (!iter) { 203 QAT_LOG(DEBUG, "parsing error %s" 204 " no number provided", 205 param); 206 } else { 207 memcpy(value_str, arg2, iter); 208 value = strtol(value_str, NULL, 10); 209 if (value > MAX_QP_THRESHOLD_SIZE) { 210 QAT_LOG(DEBUG, "Exceeded max size of" 211 " threshold, setting to %d", 212 MAX_QP_THRESHOLD_SIZE); 213 value = MAX_QP_THRESHOLD_SIZE; 214 } 215 QAT_LOG(DEBUG, "parsing %s = %ld", 216 param, value); 217 } 218 } 219 qat_dev_cmd_param[i].val = value; 220 i++; 221 } 222 } 223 224 struct qat_pci_device * 225 qat_pci_device_allocate(struct rte_pci_device *pci_dev, 226 struct qat_dev_cmd_param *qat_dev_cmd_param) 227 { 228 struct qat_pci_device *qat_dev; 229 uint8_t qat_dev_id = 0; 230 char name[QAT_DEV_NAME_MAX_LEN]; 231 struct rte_devargs *devargs = pci_dev->device.devargs; 232 233 rte_pci_device_name(&pci_dev->addr, name, sizeof(name)); 234 snprintf(name+strlen(name), QAT_DEV_NAME_MAX_LEN-strlen(name), "_qat"); 235 236 if (rte_eal_process_type() == RTE_PROC_SECONDARY) { 237 const struct rte_memzone *mz = rte_memzone_lookup(name); 238 239 if (mz == NULL) { 240 QAT_LOG(ERR, 241 "Secondary can't find %s mz, did primary create device?", 242 name); 243 return NULL; 244 } 245 qat_dev = mz->addr; 246 qat_pci_devs[qat_dev->qat_dev_id].mz = mz; 247 qat_pci_devs[qat_dev->qat_dev_id].pci_dev = pci_dev; 248 qat_nb_pci_devices++; 249 QAT_LOG(DEBUG, "QAT device %d found, name %s, total QATs %d", 250 qat_dev->qat_dev_id, qat_dev->name, qat_nb_pci_devices); 251 return qat_dev; 252 } 253 254 if (qat_pci_get_named_dev(name) != NULL) { 255 QAT_LOG(ERR, "QAT device with name %s already allocated!", 256 name); 257 return NULL; 258 } 259 260 qat_dev_id = qat_pci_find_free_device_index(); 261 if (qat_dev_id == RTE_PMD_QAT_MAX_PCI_DEVICES) { 262 QAT_LOG(ERR, "Reached maximum number of QAT devices"); 263 return NULL; 264 } 265 266 qat_pci_devs[qat_dev_id].mz = rte_memzone_reserve(name, 267 sizeof(struct qat_pci_device), 268 rte_socket_id(), 0); 269 270 if (qat_pci_devs[qat_dev_id].mz == NULL) { 271 QAT_LOG(ERR, "Error when allocating memzone for QAT_%d", 272 qat_dev_id); 273 return NULL; 274 } 275 276 qat_dev = qat_pci_devs[qat_dev_id].mz->addr; 277 memset(qat_dev, 0, sizeof(*qat_dev)); 278 strlcpy(qat_dev->name, name, QAT_DEV_NAME_MAX_LEN); 279 qat_dev->qat_dev_id = qat_dev_id; 280 qat_pci_devs[qat_dev_id].pci_dev = pci_dev; 281 switch (pci_dev->id.device_id) { 282 case 0x0443: 283 qat_dev->qat_dev_gen = QAT_GEN1; 284 break; 285 case 0x37c9: 286 case 0x19e3: 287 case 0x6f55: 288 case 0x18ef: 289 qat_dev->qat_dev_gen = QAT_GEN2; 290 break; 291 case 0x18a1: 292 qat_dev->qat_dev_gen = QAT_GEN3; 293 break; 294 case 0x4941: 295 qat_dev->qat_dev_gen = QAT_GEN4; 296 break; 297 default: 298 QAT_LOG(ERR, "Invalid dev_id, can't determine generation"); 299 rte_memzone_free(qat_pci_devs[qat_dev->qat_dev_id].mz); 300 return NULL; 301 } 302 303 if (qat_dev->qat_dev_gen == QAT_GEN4) { 304 qat_dev->misc_bar_io_addr = pci_dev->mem_resource[2].addr; 305 if (qat_dev->misc_bar_io_addr == NULL) { 306 QAT_LOG(ERR, "QAT cannot get access to VF misc bar"); 307 return NULL; 308 } 309 } 310 311 if (devargs && devargs->drv_str) 312 qat_dev_parse_cmd(devargs->drv_str, qat_dev_cmd_param); 313 314 if (qat_dev->qat_dev_gen >= QAT_GEN4) { 315 if (qat_read_qp_config(qat_dev)) { 316 QAT_LOG(ERR, 317 "Cannot acquire ring configuration for QAT_%d", 318 qat_dev_id); 319 return NULL; 320 } 321 } 322 323 rte_spinlock_init(&qat_dev->arb_csr_lock); 324 qat_nb_pci_devices++; 325 326 QAT_LOG(DEBUG, "QAT device %d found, name %s, total QATs %d", 327 qat_dev->qat_dev_id, qat_dev->name, qat_nb_pci_devices); 328 329 return qat_dev; 330 } 331 332 static int 333 qat_pci_device_release(struct rte_pci_device *pci_dev) 334 { 335 struct qat_pci_device *qat_dev; 336 char name[QAT_DEV_NAME_MAX_LEN]; 337 int busy = 0; 338 339 if (pci_dev == NULL) 340 return -EINVAL; 341 342 rte_pci_device_name(&pci_dev->addr, name, sizeof(name)); 343 snprintf(name+strlen(name), QAT_DEV_NAME_MAX_LEN-strlen(name), "_qat"); 344 qat_dev = qat_pci_get_named_dev(name); 345 if (qat_dev != NULL) { 346 347 struct qat_device_info *inst = 348 &qat_pci_devs[qat_dev->qat_dev_id]; 349 /* Check that there are no service devs still on pci device */ 350 351 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 352 if (qat_dev->sym_dev != NULL) { 353 QAT_LOG(DEBUG, "QAT sym device %s is busy", 354 name); 355 busy = 1; 356 } 357 if (qat_dev->asym_dev != NULL) { 358 QAT_LOG(DEBUG, "QAT asym device %s is busy", 359 name); 360 busy = 1; 361 } 362 if (qat_dev->comp_dev != NULL) { 363 QAT_LOG(DEBUG, "QAT comp device %s is busy", 364 name); 365 busy = 1; 366 } 367 if (busy) 368 return -EBUSY; 369 rte_memzone_free(inst->mz); 370 } 371 memset(inst, 0, sizeof(struct qat_device_info)); 372 qat_nb_pci_devices--; 373 QAT_LOG(DEBUG, "QAT device %s released, total QATs %d", 374 name, qat_nb_pci_devices); 375 } 376 return 0; 377 } 378 379 static int 380 qat_pci_dev_destroy(struct qat_pci_device *qat_pci_dev, 381 struct rte_pci_device *pci_dev) 382 { 383 qat_sym_dev_destroy(qat_pci_dev); 384 qat_comp_dev_destroy(qat_pci_dev); 385 qat_asym_dev_destroy(qat_pci_dev); 386 return qat_pci_device_release(pci_dev); 387 } 388 389 static int qat_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 390 struct rte_pci_device *pci_dev) 391 { 392 int sym_ret = 0, asym_ret = 0, comp_ret = 0; 393 int num_pmds_created = 0; 394 struct qat_pci_device *qat_pci_dev; 395 struct qat_dev_cmd_param qat_dev_cmd_param[] = { 396 { SYM_ENQ_THRESHOLD_NAME, 0 }, 397 { ASYM_ENQ_THRESHOLD_NAME, 0 }, 398 { COMP_ENQ_THRESHOLD_NAME, 0 }, 399 { NULL, 0 }, 400 }; 401 402 QAT_LOG(DEBUG, "Found QAT device at %02x:%02x.%x", 403 pci_dev->addr.bus, 404 pci_dev->addr.devid, 405 pci_dev->addr.function); 406 407 qat_pci_dev = qat_pci_device_allocate(pci_dev, qat_dev_cmd_param); 408 if (qat_pci_dev == NULL) 409 return -ENODEV; 410 411 if (qat_pci_dev->qat_dev_gen == QAT_GEN4) { 412 if (qat_gen4_reset_ring_pair(qat_pci_dev)) { 413 QAT_LOG(ERR, 414 "Cannot reset ring pairs, does pf driver supports pf2vf comms?" 415 ); 416 return -ENODEV; 417 } 418 } 419 420 sym_ret = qat_sym_dev_create(qat_pci_dev, qat_dev_cmd_param); 421 if (sym_ret == 0) { 422 num_pmds_created++; 423 424 } 425 else 426 QAT_LOG(WARNING, 427 "Failed to create QAT SYM PMD on device %s", 428 qat_pci_dev->name); 429 430 comp_ret = qat_comp_dev_create(qat_pci_dev, qat_dev_cmd_param); 431 if (comp_ret == 0) 432 num_pmds_created++; 433 else 434 QAT_LOG(WARNING, 435 "Failed to create QAT COMP PMD on device %s", 436 qat_pci_dev->name); 437 438 asym_ret = qat_asym_dev_create(qat_pci_dev, qat_dev_cmd_param); 439 if (asym_ret == 0) 440 num_pmds_created++; 441 else 442 QAT_LOG(WARNING, 443 "Failed to create QAT ASYM PMD on device %s", 444 qat_pci_dev->name); 445 446 if (num_pmds_created == 0) 447 qat_pci_dev_destroy(qat_pci_dev, pci_dev); 448 449 return 0; 450 } 451 452 static int qat_pci_remove(struct rte_pci_device *pci_dev) 453 { 454 struct qat_pci_device *qat_pci_dev; 455 456 if (pci_dev == NULL) 457 return -EINVAL; 458 459 qat_pci_dev = qat_get_qat_dev_from_pci_dev(pci_dev); 460 if (qat_pci_dev == NULL) 461 return 0; 462 463 return qat_pci_dev_destroy(qat_pci_dev, pci_dev); 464 } 465 466 static struct rte_pci_driver rte_qat_pmd = { 467 .id_table = pci_id_qat_map, 468 .drv_flags = RTE_PCI_DRV_NEED_MAPPING, 469 .probe = qat_pci_probe, 470 .remove = qat_pci_remove 471 }; 472 473 __rte_weak int 474 qat_sym_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused, 475 struct qat_dev_cmd_param *qat_dev_cmd_param __rte_unused) 476 { 477 return 0; 478 } 479 480 __rte_weak int 481 qat_asym_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused, 482 struct qat_dev_cmd_param *qat_dev_cmd_param __rte_unused) 483 { 484 return 0; 485 } 486 487 __rte_weak int 488 qat_sym_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused) 489 { 490 return 0; 491 } 492 493 __rte_weak int 494 qat_asym_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused) 495 { 496 return 0; 497 } 498 499 __rte_weak int 500 qat_comp_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused, 501 struct qat_dev_cmd_param *qat_dev_cmd_param __rte_unused) 502 { 503 return 0; 504 } 505 506 __rte_weak int 507 qat_comp_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused) 508 { 509 return 0; 510 } 511 512 RTE_PMD_REGISTER_PCI(QAT_PCI_NAME, rte_qat_pmd); 513 RTE_PMD_REGISTER_PCI_TABLE(QAT_PCI_NAME, pci_id_qat_map); 514 RTE_PMD_REGISTER_KMOD_DEP(QAT_PCI_NAME, "* igb_uio | uio_pci_generic | vfio-pci"); 515