1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2015-2018 Intel Corporation 3 */ 4 5 #include "qat_comp.h" 6 #include "qat_comp_pmd.h" 7 8 static const struct rte_compressdev_capabilities qat_comp_gen_capabilities[] = { 9 {/* COMPRESSION - deflate */ 10 .algo = RTE_COMP_ALGO_DEFLATE, 11 .comp_feature_flags = RTE_COMP_FF_MULTI_PKT_CHECKSUM | 12 RTE_COMP_FF_CRC32_CHECKSUM | 13 RTE_COMP_FF_ADLER32_CHECKSUM | 14 RTE_COMP_FF_CRC32_ADLER32_CHECKSUM | 15 RTE_COMP_FF_SHAREABLE_PRIV_XFORM | 16 RTE_COMP_FF_HUFFMAN_FIXED, 17 .window_size = {.min = 15, .max = 15, .increment = 0} }, 18 {RTE_COMP_ALGO_LIST_END, 0, {0, 0, 0} } }; 19 20 static void 21 qat_comp_stats_get(struct rte_compressdev *dev, 22 struct rte_compressdev_stats *stats) 23 { 24 struct qat_common_stats qat_stats = {0}; 25 struct qat_comp_dev_private *qat_priv; 26 27 if (stats == NULL || dev == NULL) { 28 QAT_LOG(ERR, "invalid ptr: stats %p, dev %p", stats, dev); 29 return; 30 } 31 qat_priv = dev->data->dev_private; 32 33 qat_stats_get(qat_priv->qat_dev, &qat_stats, QAT_SERVICE_COMPRESSION); 34 stats->enqueued_count = qat_stats.enqueued_count; 35 stats->dequeued_count = qat_stats.dequeued_count; 36 stats->enqueue_err_count = qat_stats.enqueue_err_count; 37 stats->dequeue_err_count = qat_stats.dequeue_err_count; 38 } 39 40 static void 41 qat_comp_stats_reset(struct rte_compressdev *dev) 42 { 43 struct qat_comp_dev_private *qat_priv; 44 45 if (dev == NULL) { 46 QAT_LOG(ERR, "invalid compressdev ptr %p", dev); 47 return; 48 } 49 qat_priv = dev->data->dev_private; 50 51 qat_stats_reset(qat_priv->qat_dev, QAT_SERVICE_COMPRESSION); 52 53 } 54 55 static int 56 qat_comp_qp_release(struct rte_compressdev *dev, uint16_t queue_pair_id) 57 { 58 struct qat_comp_dev_private *qat_private = dev->data->dev_private; 59 60 QAT_LOG(DEBUG, "Release comp qp %u on device %d", 61 queue_pair_id, dev->data->dev_id); 62 63 qat_private->qat_dev->qps_in_use[QAT_SERVICE_COMPRESSION][queue_pair_id] 64 = NULL; 65 66 return qat_qp_release((struct qat_qp **) 67 &(dev->data->queue_pairs[queue_pair_id])); 68 } 69 70 static int 71 qat_comp_qp_setup(struct rte_compressdev *dev, uint16_t qp_id, 72 uint32_t max_inflight_ops, int socket_id) 73 { 74 int ret = 0; 75 struct qat_qp_config qat_qp_conf; 76 77 struct qat_qp **qp_addr = 78 (struct qat_qp **)&(dev->data->queue_pairs[qp_id]); 79 struct qat_comp_dev_private *qat_private = dev->data->dev_private; 80 const struct qat_qp_hw_data *comp_hw_qps = 81 qat_gen_config[qat_private->qat_dev->qat_dev_gen] 82 .qp_hw_data[QAT_SERVICE_COMPRESSION]; 83 const struct qat_qp_hw_data *qp_hw_data = comp_hw_qps + qp_id; 84 85 /* If qp is already in use free ring memory and qp metadata. */ 86 if (*qp_addr != NULL) { 87 ret = qat_comp_qp_release(dev, qp_id); 88 if (ret < 0) 89 return ret; 90 } 91 if (qp_id >= qat_qps_per_service(comp_hw_qps, 92 QAT_SERVICE_COMPRESSION)) { 93 QAT_LOG(ERR, "qp_id %u invalid for this device", qp_id); 94 return -EINVAL; 95 } 96 97 qat_qp_conf.hw = qp_hw_data; 98 qat_qp_conf.build_request = qat_comp_build_request; 99 qat_qp_conf.cookie_size = sizeof(struct qat_comp_op_cookie); 100 qat_qp_conf.nb_descriptors = max_inflight_ops; 101 qat_qp_conf.socket_id = socket_id; 102 qat_qp_conf.service_str = "comp"; 103 104 ret = qat_qp_setup(qat_private->qat_dev, qp_addr, qp_id, &qat_qp_conf); 105 if (ret != 0) 106 return ret; 107 108 /* store a link to the qp in the qat_pci_device */ 109 qat_private->qat_dev->qps_in_use[QAT_SERVICE_COMPRESSION][qp_id] 110 = *qp_addr; 111 112 return ret; 113 } 114 115 static struct rte_mempool * 116 qat_comp_create_xform_pool(struct qat_comp_dev_private *comp_dev, 117 uint32_t num_elements) 118 { 119 char xform_pool_name[RTE_MEMPOOL_NAMESIZE]; 120 struct rte_mempool *mp; 121 122 snprintf(xform_pool_name, RTE_MEMPOOL_NAMESIZE, 123 "%s_xforms", comp_dev->qat_dev->name); 124 125 QAT_LOG(DEBUG, "xformpool: %s", xform_pool_name); 126 mp = rte_mempool_lookup(xform_pool_name); 127 128 if (mp != NULL) { 129 QAT_LOG(DEBUG, "xformpool already created"); 130 if (mp->size != num_elements) { 131 QAT_LOG(DEBUG, "xformpool wrong size - delete it"); 132 rte_mempool_free(mp); 133 mp = NULL; 134 comp_dev->xformpool = NULL; 135 } 136 } 137 138 if (mp == NULL) 139 mp = rte_mempool_create(xform_pool_name, 140 num_elements, 141 qat_comp_xform_size(), 0, 0, 142 NULL, NULL, NULL, NULL, rte_socket_id(), 143 0); 144 if (mp == NULL) { 145 QAT_LOG(ERR, "Err creating mempool %s w %d elements of size %d", 146 xform_pool_name, num_elements, qat_comp_xform_size()); 147 return NULL; 148 } 149 150 return mp; 151 } 152 153 static void 154 _qat_comp_dev_config_clear(struct qat_comp_dev_private *comp_dev) 155 { 156 /* Free private_xform pool */ 157 if (comp_dev->xformpool) { 158 /* Free internal mempool for private xforms */ 159 rte_mempool_free(comp_dev->xformpool); 160 comp_dev->xformpool = NULL; 161 } 162 } 163 164 static int 165 qat_comp_dev_config(struct rte_compressdev *dev, 166 struct rte_compressdev_config *config) 167 { 168 struct qat_comp_dev_private *comp_dev = dev->data->dev_private; 169 int ret = 0; 170 171 if (config->max_nb_streams != 0) { 172 QAT_LOG(ERR, 173 "QAT device does not support STATEFUL so max_nb_streams must be 0"); 174 return -EINVAL; 175 } 176 177 comp_dev->xformpool = qat_comp_create_xform_pool(comp_dev, 178 config->max_nb_priv_xforms); 179 if (comp_dev->xformpool == NULL) { 180 181 ret = -ENOMEM; 182 goto error_out; 183 } 184 return 0; 185 186 error_out: 187 _qat_comp_dev_config_clear(comp_dev); 188 return ret; 189 } 190 191 static int 192 qat_comp_dev_start(struct rte_compressdev *dev __rte_unused) 193 { 194 return 0; 195 } 196 197 static void 198 qat_comp_dev_stop(struct rte_compressdev *dev __rte_unused) 199 { 200 201 } 202 203 static int 204 qat_comp_dev_close(struct rte_compressdev *dev) 205 { 206 int i; 207 int ret = 0; 208 struct qat_comp_dev_private *comp_dev = dev->data->dev_private; 209 210 for (i = 0; i < dev->data->nb_queue_pairs; i++) { 211 ret = qat_comp_qp_release(dev, i); 212 if (ret < 0) 213 return ret; 214 } 215 216 _qat_comp_dev_config_clear(comp_dev); 217 218 return ret; 219 } 220 221 222 static void 223 qat_comp_dev_info_get(struct rte_compressdev *dev, 224 struct rte_compressdev_info *info) 225 { 226 struct qat_comp_dev_private *comp_dev = dev->data->dev_private; 227 const struct qat_qp_hw_data *comp_hw_qps = 228 qat_gen_config[comp_dev->qat_dev->qat_dev_gen] 229 .qp_hw_data[QAT_SERVICE_COMPRESSION]; 230 231 if (info != NULL) { 232 info->max_nb_queue_pairs = 233 qat_qps_per_service(comp_hw_qps, 234 QAT_SERVICE_COMPRESSION); 235 info->feature_flags = dev->feature_flags; 236 info->capabilities = comp_dev->qat_dev_capabilities; 237 } 238 } 239 240 static uint16_t 241 qat_comp_pmd_enqueue_op_burst(void *qp, struct rte_comp_op **ops, 242 uint16_t nb_ops) 243 { 244 return qat_enqueue_op_burst(qp, (void **)ops, nb_ops); 245 } 246 247 static uint16_t 248 qat_comp_pmd_dequeue_op_burst(void *qp, struct rte_comp_op **ops, 249 uint16_t nb_ops) 250 { 251 return qat_dequeue_op_burst(qp, (void **)ops, nb_ops); 252 } 253 254 static uint16_t 255 qat_comp_pmd_enq_deq_dummy_op_burst(void *qp __rte_unused, 256 struct rte_comp_op **ops __rte_unused, 257 uint16_t nb_ops __rte_unused) 258 { 259 QAT_DP_LOG(ERR, "QAT PMD detected wrong FW version !"); 260 return 0; 261 } 262 263 static struct rte_compressdev_ops compress_qat_dummy_ops = { 264 265 /* Device related operations */ 266 .dev_configure = NULL, 267 .dev_start = NULL, 268 .dev_stop = qat_comp_dev_stop, 269 .dev_close = qat_comp_dev_close, 270 .dev_infos_get = NULL, 271 272 .stats_get = NULL, 273 .stats_reset = qat_comp_stats_reset, 274 .queue_pair_setup = NULL, 275 .queue_pair_release = qat_comp_qp_release, 276 277 /* Compression related operations */ 278 .private_xform_create = NULL, 279 .private_xform_free = qat_comp_private_xform_free 280 }; 281 282 static uint16_t 283 qat_comp_pmd_dequeue_frst_op_burst(void *qp, struct rte_comp_op **ops, 284 uint16_t nb_ops) 285 { 286 uint16_t ret = qat_dequeue_op_burst(qp, (void **)ops, nb_ops); 287 struct qat_qp *tmp_qp = (struct qat_qp *)qp; 288 289 if (ret) { 290 if ((*ops)->debug_status == 291 (uint64_t)ERR_CODE_QAT_COMP_WRONG_FW) { 292 tmp_qp->qat_dev->comp_dev->compressdev->enqueue_burst = 293 qat_comp_pmd_enq_deq_dummy_op_burst; 294 tmp_qp->qat_dev->comp_dev->compressdev->dequeue_burst = 295 qat_comp_pmd_enq_deq_dummy_op_burst; 296 297 tmp_qp->qat_dev->comp_dev->compressdev->dev_ops = 298 &compress_qat_dummy_ops; 299 QAT_LOG(ERR, "QAT PMD detected wrong FW version !"); 300 301 } else { 302 tmp_qp->qat_dev->comp_dev->compressdev->dequeue_burst = 303 qat_comp_pmd_dequeue_op_burst; 304 } 305 } 306 return ret; 307 } 308 309 static struct rte_compressdev_ops compress_qat_ops = { 310 311 /* Device related operations */ 312 .dev_configure = qat_comp_dev_config, 313 .dev_start = qat_comp_dev_start, 314 .dev_stop = qat_comp_dev_stop, 315 .dev_close = qat_comp_dev_close, 316 .dev_infos_get = qat_comp_dev_info_get, 317 318 .stats_get = qat_comp_stats_get, 319 .stats_reset = qat_comp_stats_reset, 320 .queue_pair_setup = qat_comp_qp_setup, 321 .queue_pair_release = qat_comp_qp_release, 322 323 /* Compression related operations */ 324 .private_xform_create = qat_comp_private_xform_create, 325 .private_xform_free = qat_comp_private_xform_free 326 }; 327 328 int 329 qat_comp_dev_create(struct qat_pci_device *qat_pci_dev) 330 { 331 if (qat_pci_dev->qat_dev_gen == QAT_GEN1) { 332 QAT_LOG(ERR, "Compression PMD not supported on QAT dh895xcc"); 333 return 0; 334 } 335 336 struct rte_compressdev_pmd_init_params init_params = { 337 .name = "", 338 .socket_id = qat_pci_dev->pci_dev->device.numa_node, 339 }; 340 char name[RTE_COMPRESSDEV_NAME_MAX_LEN]; 341 struct rte_compressdev *compressdev; 342 struct qat_comp_dev_private *comp_dev; 343 344 snprintf(name, RTE_COMPRESSDEV_NAME_MAX_LEN, "%s_%s", 345 qat_pci_dev->name, "comp"); 346 QAT_LOG(DEBUG, "Creating QAT COMP device %s", name); 347 348 compressdev = rte_compressdev_pmd_create(name, 349 &qat_pci_dev->pci_dev->device, 350 sizeof(struct qat_comp_dev_private), 351 &init_params); 352 353 if (compressdev == NULL) 354 return -ENODEV; 355 356 compressdev->dev_ops = &compress_qat_ops; 357 358 compressdev->enqueue_burst = qat_comp_pmd_enqueue_op_burst; 359 compressdev->dequeue_burst = qat_comp_pmd_dequeue_frst_op_burst; 360 361 compressdev->feature_flags = RTE_COMPDEV_FF_HW_ACCELERATED; 362 363 comp_dev = compressdev->data->dev_private; 364 comp_dev->qat_dev = qat_pci_dev; 365 comp_dev->compressdev = compressdev; 366 qat_pci_dev->comp_dev = comp_dev; 367 368 switch (qat_pci_dev->qat_dev_gen) { 369 case QAT_GEN1: 370 case QAT_GEN2: 371 comp_dev->qat_dev_capabilities = qat_comp_gen_capabilities; 372 break; 373 default: 374 comp_dev->qat_dev_capabilities = qat_comp_gen_capabilities; 375 QAT_LOG(DEBUG, 376 "QAT gen %d capabilities unknown, default to GEN1", 377 qat_pci_dev->qat_dev_gen); 378 break; 379 } 380 381 QAT_LOG(DEBUG, 382 "Created QAT COMP device %s as compressdev instance %d", 383 name, compressdev->data->dev_id); 384 return 0; 385 } 386 387 int 388 qat_comp_dev_destroy(struct qat_pci_device *qat_pci_dev) 389 { 390 struct qat_comp_dev_private *comp_dev; 391 392 if (qat_pci_dev == NULL) 393 return -ENODEV; 394 395 comp_dev = qat_pci_dev->comp_dev; 396 if (comp_dev == NULL) 397 return 0; 398 399 /* clean up any resources used by the device */ 400 qat_comp_dev_close(comp_dev->compressdev); 401 402 rte_compressdev_pmd_destroy(comp_dev->compressdev); 403 qat_pci_dev->comp_dev = NULL; 404 405 return 0; 406 } 407