1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018-2020 Intel Corporation 3 */ 4 5 #include <rte_string_fns.h> 6 #include <rte_devargs.h> 7 #include <ctype.h> 8 9 #include "qat_device.h" 10 #include "adf_transport_access_macros.h" 11 #include "qat_sym_pmd.h" 12 #include "qat_comp_pmd.h" 13 14 /* Hardware device information per generation */ 15 __extension__ 16 struct qat_gen_hw_data qat_gen_config[] = { 17 [QAT_GEN1] = { 18 .dev_gen = QAT_GEN1, 19 .qp_hw_data = qat_gen1_qps, 20 .comp_num_im_bufs_required = QAT_NUM_INTERM_BUFS_GEN1 21 }, 22 [QAT_GEN2] = { 23 .dev_gen = QAT_GEN2, 24 .qp_hw_data = qat_gen1_qps, 25 /* gen2 has same ring layout as gen1 */ 26 .comp_num_im_bufs_required = QAT_NUM_INTERM_BUFS_GEN2 27 }, 28 [QAT_GEN3] = { 29 .dev_gen = QAT_GEN3, 30 .qp_hw_data = qat_gen3_qps, 31 .comp_num_im_bufs_required = QAT_NUM_INTERM_BUFS_GEN3 32 }, 33 }; 34 35 /* per-process array of device data */ 36 struct qat_device_info qat_pci_devs[RTE_PMD_QAT_MAX_PCI_DEVICES]; 37 static int qat_nb_pci_devices; 38 39 /* 40 * The set of PCI devices this driver supports 41 */ 42 43 static const struct rte_pci_id pci_id_qat_map[] = { 44 { 45 RTE_PCI_DEVICE(0x8086, 0x0443), 46 }, 47 { 48 RTE_PCI_DEVICE(0x8086, 0x37c9), 49 }, 50 { 51 RTE_PCI_DEVICE(0x8086, 0x19e3), 52 }, 53 { 54 RTE_PCI_DEVICE(0x8086, 0x6f55), 55 }, 56 { 57 RTE_PCI_DEVICE(0x8086, 0x18ef), 58 }, 59 { 60 RTE_PCI_DEVICE(0x8086, 0x18a1), 61 }, 62 {.device_id = 0}, 63 }; 64 65 static struct qat_pci_device * 66 qat_pci_get_named_dev(const char *name) 67 { 68 unsigned int i; 69 70 if (name == NULL) 71 return NULL; 72 73 for (i = 0; i < RTE_PMD_QAT_MAX_PCI_DEVICES; i++) { 74 if (qat_pci_devs[i].mz && 75 (strcmp(((struct qat_pci_device *) 76 qat_pci_devs[i].mz->addr)->name, name) 77 == 0)) 78 return (struct qat_pci_device *) 79 qat_pci_devs[i].mz->addr; 80 } 81 82 return NULL; 83 } 84 85 static uint8_t 86 qat_pci_find_free_device_index(void) 87 { 88 uint8_t dev_id; 89 90 for (dev_id = 0; dev_id < RTE_PMD_QAT_MAX_PCI_DEVICES; 91 dev_id++) { 92 if (qat_pci_devs[dev_id].mz == NULL) 93 break; 94 } 95 return dev_id; 96 } 97 98 struct qat_pci_device * 99 qat_get_qat_dev_from_pci_dev(struct rte_pci_device *pci_dev) 100 { 101 char name[QAT_DEV_NAME_MAX_LEN]; 102 103 rte_pci_device_name(&pci_dev->addr, name, sizeof(name)); 104 105 return qat_pci_get_named_dev(name); 106 } 107 108 static void qat_dev_parse_cmd(const char *str, struct qat_dev_cmd_param 109 *qat_dev_cmd_param) 110 { 111 int i = 0; 112 const char *param; 113 114 while (1) { 115 char value_str[4] = { }; 116 117 param = qat_dev_cmd_param[i].name; 118 if (param == NULL) 119 return; 120 long value = 0; 121 const char *arg = strstr(str, param); 122 const char *arg2 = NULL; 123 124 if (arg) { 125 arg2 = arg + strlen(param); 126 if (*arg2 != '=') { 127 QAT_LOG(DEBUG, "parsing error '=' sign" 128 " should immediately follow %s", 129 param); 130 arg2 = NULL; 131 } else 132 arg2++; 133 } else { 134 QAT_LOG(DEBUG, "%s not provided", param); 135 } 136 if (arg2) { 137 int iter = 0; 138 while (iter < 2) { 139 if (!isdigit(*(arg2 + iter))) 140 break; 141 iter++; 142 } 143 if (!iter) { 144 QAT_LOG(DEBUG, "parsing error %s" 145 " no number provided", 146 param); 147 } else { 148 memcpy(value_str, arg2, iter); 149 value = strtol(value_str, NULL, 10); 150 if (value > MAX_QP_THRESHOLD_SIZE) { 151 QAT_LOG(DEBUG, "Exceeded max size of" 152 " threshold, setting to %d", 153 MAX_QP_THRESHOLD_SIZE); 154 value = MAX_QP_THRESHOLD_SIZE; 155 } 156 QAT_LOG(DEBUG, "parsing %s = %ld", 157 param, value); 158 } 159 } 160 qat_dev_cmd_param[i].val = value; 161 i++; 162 } 163 } 164 165 struct qat_pci_device * 166 qat_pci_device_allocate(struct rte_pci_device *pci_dev, 167 struct qat_dev_cmd_param *qat_dev_cmd_param) 168 { 169 struct qat_pci_device *qat_dev; 170 uint8_t qat_dev_id = 0; 171 char name[QAT_DEV_NAME_MAX_LEN]; 172 struct rte_devargs *devargs = pci_dev->device.devargs; 173 174 rte_pci_device_name(&pci_dev->addr, name, sizeof(name)); 175 snprintf(name+strlen(name), QAT_DEV_NAME_MAX_LEN-strlen(name), "_qat"); 176 177 if (rte_eal_process_type() == RTE_PROC_SECONDARY) { 178 const struct rte_memzone *mz = rte_memzone_lookup(name); 179 180 if (mz == NULL) { 181 QAT_LOG(ERR, 182 "Secondary can't find %s mz, did primary create device?", 183 name); 184 return NULL; 185 } 186 qat_dev = mz->addr; 187 qat_pci_devs[qat_dev->qat_dev_id].mz = mz; 188 qat_pci_devs[qat_dev->qat_dev_id].pci_dev = pci_dev; 189 qat_nb_pci_devices++; 190 QAT_LOG(DEBUG, "QAT device %d found, name %s, total QATs %d", 191 qat_dev->qat_dev_id, qat_dev->name, qat_nb_pci_devices); 192 return qat_dev; 193 } 194 195 if (qat_pci_get_named_dev(name) != NULL) { 196 QAT_LOG(ERR, "QAT device with name %s already allocated!", 197 name); 198 return NULL; 199 } 200 201 qat_dev_id = qat_pci_find_free_device_index(); 202 if (qat_dev_id == RTE_PMD_QAT_MAX_PCI_DEVICES) { 203 QAT_LOG(ERR, "Reached maximum number of QAT devices"); 204 return NULL; 205 } 206 207 qat_pci_devs[qat_dev_id].mz = rte_memzone_reserve(name, 208 sizeof(struct qat_pci_device), 209 rte_socket_id(), 0); 210 211 if (qat_pci_devs[qat_dev_id].mz == NULL) { 212 QAT_LOG(ERR, "Error when allocating memzone for QAT_%d", 213 qat_dev_id); 214 return NULL; 215 } 216 217 qat_dev = qat_pci_devs[qat_dev_id].mz->addr; 218 memset(qat_dev, 0, sizeof(*qat_dev)); 219 strlcpy(qat_dev->name, name, QAT_DEV_NAME_MAX_LEN); 220 qat_dev->qat_dev_id = qat_dev_id; 221 qat_pci_devs[qat_dev_id].pci_dev = pci_dev; 222 switch (pci_dev->id.device_id) { 223 case 0x0443: 224 qat_dev->qat_dev_gen = QAT_GEN1; 225 break; 226 case 0x37c9: 227 case 0x19e3: 228 case 0x6f55: 229 case 0x18ef: 230 qat_dev->qat_dev_gen = QAT_GEN2; 231 break; 232 case 0x18a1: 233 qat_dev->qat_dev_gen = QAT_GEN3; 234 break; 235 default: 236 QAT_LOG(ERR, "Invalid dev_id, can't determine generation"); 237 rte_memzone_free(qat_pci_devs[qat_dev->qat_dev_id].mz); 238 return NULL; 239 } 240 241 if (devargs && devargs->drv_str) 242 qat_dev_parse_cmd(devargs->drv_str, qat_dev_cmd_param); 243 244 rte_spinlock_init(&qat_dev->arb_csr_lock); 245 qat_nb_pci_devices++; 246 247 QAT_LOG(DEBUG, "QAT device %d found, name %s, total QATs %d", 248 qat_dev->qat_dev_id, qat_dev->name, qat_nb_pci_devices); 249 250 return qat_dev; 251 } 252 253 static int 254 qat_pci_device_release(struct rte_pci_device *pci_dev) 255 { 256 struct qat_pci_device *qat_dev; 257 char name[QAT_DEV_NAME_MAX_LEN]; 258 int busy = 0; 259 260 if (pci_dev == NULL) 261 return -EINVAL; 262 263 rte_pci_device_name(&pci_dev->addr, name, sizeof(name)); 264 snprintf(name+strlen(name), QAT_DEV_NAME_MAX_LEN-strlen(name), "_qat"); 265 qat_dev = qat_pci_get_named_dev(name); 266 if (qat_dev != NULL) { 267 268 struct qat_device_info *inst = 269 &qat_pci_devs[qat_dev->qat_dev_id]; 270 /* Check that there are no service devs still on pci device */ 271 272 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 273 if (qat_dev->sym_dev != NULL) { 274 QAT_LOG(DEBUG, "QAT sym device %s is busy", 275 name); 276 busy = 1; 277 } 278 if (qat_dev->asym_dev != NULL) { 279 QAT_LOG(DEBUG, "QAT asym device %s is busy", 280 name); 281 busy = 1; 282 } 283 if (qat_dev->comp_dev != NULL) { 284 QAT_LOG(DEBUG, "QAT comp device %s is busy", 285 name); 286 busy = 1; 287 } 288 if (busy) 289 return -EBUSY; 290 rte_memzone_free(inst->mz); 291 } 292 memset(inst, 0, sizeof(struct qat_device_info)); 293 qat_nb_pci_devices--; 294 QAT_LOG(DEBUG, "QAT device %s released, total QATs %d", 295 name, qat_nb_pci_devices); 296 } 297 return 0; 298 } 299 300 static int 301 qat_pci_dev_destroy(struct qat_pci_device *qat_pci_dev, 302 struct rte_pci_device *pci_dev) 303 { 304 qat_sym_dev_destroy(qat_pci_dev); 305 qat_comp_dev_destroy(qat_pci_dev); 306 qat_asym_dev_destroy(qat_pci_dev); 307 return qat_pci_device_release(pci_dev); 308 } 309 310 static int qat_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 311 struct rte_pci_device *pci_dev) 312 { 313 int sym_ret = 0, asym_ret = 0, comp_ret = 0; 314 int num_pmds_created = 0; 315 struct qat_pci_device *qat_pci_dev; 316 struct qat_dev_cmd_param qat_dev_cmd_param[] = { 317 { SYM_ENQ_THRESHOLD_NAME, 0 }, 318 { ASYM_ENQ_THRESHOLD_NAME, 0 }, 319 { COMP_ENQ_THRESHOLD_NAME, 0 }, 320 { NULL, 0 }, 321 }; 322 323 QAT_LOG(DEBUG, "Found QAT device at %02x:%02x.%x", 324 pci_dev->addr.bus, 325 pci_dev->addr.devid, 326 pci_dev->addr.function); 327 328 qat_pci_dev = qat_pci_device_allocate(pci_dev, qat_dev_cmd_param); 329 if (qat_pci_dev == NULL) 330 return -ENODEV; 331 332 sym_ret = qat_sym_dev_create(qat_pci_dev, qat_dev_cmd_param); 333 if (sym_ret == 0) { 334 num_pmds_created++; 335 336 } 337 else 338 QAT_LOG(WARNING, 339 "Failed to create QAT SYM PMD on device %s", 340 qat_pci_dev->name); 341 342 comp_ret = qat_comp_dev_create(qat_pci_dev, qat_dev_cmd_param); 343 if (comp_ret == 0) 344 num_pmds_created++; 345 else 346 QAT_LOG(WARNING, 347 "Failed to create QAT COMP PMD on device %s", 348 qat_pci_dev->name); 349 350 asym_ret = qat_asym_dev_create(qat_pci_dev, qat_dev_cmd_param); 351 if (asym_ret == 0) 352 num_pmds_created++; 353 else 354 QAT_LOG(WARNING, 355 "Failed to create QAT ASYM PMD on device %s", 356 qat_pci_dev->name); 357 358 if (num_pmds_created == 0) 359 qat_pci_dev_destroy(qat_pci_dev, pci_dev); 360 361 return 0; 362 } 363 364 static int qat_pci_remove(struct rte_pci_device *pci_dev) 365 { 366 struct qat_pci_device *qat_pci_dev; 367 368 if (pci_dev == NULL) 369 return -EINVAL; 370 371 qat_pci_dev = qat_get_qat_dev_from_pci_dev(pci_dev); 372 if (qat_pci_dev == NULL) 373 return 0; 374 375 return qat_pci_dev_destroy(qat_pci_dev, pci_dev); 376 } 377 378 static struct rte_pci_driver rte_qat_pmd = { 379 .id_table = pci_id_qat_map, 380 .drv_flags = RTE_PCI_DRV_NEED_MAPPING, 381 .probe = qat_pci_probe, 382 .remove = qat_pci_remove 383 }; 384 385 __rte_weak int 386 qat_sym_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused, 387 struct qat_dev_cmd_param *qat_dev_cmd_param __rte_unused) 388 { 389 return 0; 390 } 391 392 __rte_weak int 393 qat_asym_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused, 394 struct qat_dev_cmd_param *qat_dev_cmd_param __rte_unused) 395 { 396 return 0; 397 } 398 399 __rte_weak int 400 qat_sym_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused) 401 { 402 return 0; 403 } 404 405 __rte_weak int 406 qat_asym_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused) 407 { 408 return 0; 409 } 410 411 __rte_weak int 412 qat_comp_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused, 413 struct qat_dev_cmd_param *qat_dev_cmd_param __rte_unused) 414 { 415 return 0; 416 } 417 418 __rte_weak int 419 qat_comp_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused) 420 { 421 return 0; 422 } 423 424 RTE_PMD_REGISTER_PCI(QAT_PCI_NAME, rte_qat_pmd); 425 RTE_PMD_REGISTER_PCI_TABLE(QAT_PCI_NAME, pci_id_qat_map); 426