1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Advanced Micro Devices, Inc. All rights reserved. 3 */ 4 5 #include <rte_bus_pci.h> 6 #include <rte_bus_vdev.h> 7 #include <rte_common.h> 8 #include <rte_config.h> 9 #include <rte_cryptodev.h> 10 #include <rte_cryptodev_pmd.h> 11 #include <rte_pci.h> 12 #include <rte_dev.h> 13 #include <rte_malloc.h> 14 15 #include "ccp_crypto.h" 16 #include "ccp_dev.h" 17 #include "ccp_pmd_private.h" 18 19 /** 20 * Global static parameter used to find if CCP device is already initialized. 21 */ 22 static unsigned int ccp_pmd_init_done; 23 uint8_t ccp_cryptodev_driver_id; 24 25 struct ccp_pmd_init_params { 26 struct rte_cryptodev_pmd_init_params def_p; 27 bool auth_opt; 28 }; 29 30 #define CCP_CRYPTODEV_PARAM_NAME ("name") 31 #define CCP_CRYPTODEV_PARAM_SOCKET_ID ("socket_id") 32 #define CCP_CRYPTODEV_PARAM_MAX_NB_QP ("max_nb_queue_pairs") 33 #define CCP_CRYPTODEV_PARAM_AUTH_OPT ("ccp_auth_opt") 34 35 const char *ccp_pmd_valid_params[] = { 36 CCP_CRYPTODEV_PARAM_NAME, 37 CCP_CRYPTODEV_PARAM_SOCKET_ID, 38 CCP_CRYPTODEV_PARAM_MAX_NB_QP, 39 CCP_CRYPTODEV_PARAM_AUTH_OPT, 40 }; 41 42 /** ccp pmd auth option */ 43 enum ccp_pmd_auth_opt { 44 CCP_PMD_AUTH_OPT_CCP = 0, 45 CCP_PMD_AUTH_OPT_CPU, 46 }; 47 48 /** parse integer from integer argument */ 49 static int 50 parse_integer_arg(const char *key __rte_unused, 51 const char *value, void *extra_args) 52 { 53 int *i = (int *) extra_args; 54 55 *i = atoi(value); 56 if (*i < 0) { 57 CCP_LOG_ERR("Argument has to be positive.\n"); 58 return -EINVAL; 59 } 60 61 return 0; 62 } 63 64 /** parse name argument */ 65 static int 66 parse_name_arg(const char *key __rte_unused, 67 const char *value, void *extra_args) 68 { 69 struct rte_cryptodev_pmd_init_params *params = extra_args; 70 71 if (strlen(value) >= RTE_CRYPTODEV_NAME_MAX_LEN - 1) { 72 CCP_LOG_ERR("Invalid name %s, should be less than " 73 "%u bytes.\n", value, 74 RTE_CRYPTODEV_NAME_MAX_LEN - 1); 75 return -EINVAL; 76 } 77 78 strncpy(params->name, value, RTE_CRYPTODEV_NAME_MAX_LEN); 79 80 return 0; 81 } 82 83 /** parse authentication operation option */ 84 static int 85 parse_auth_opt_arg(const char *key __rte_unused, 86 const char *value, void *extra_args) 87 { 88 struct ccp_pmd_init_params *params = extra_args; 89 int i; 90 91 i = atoi(value); 92 if (i < CCP_PMD_AUTH_OPT_CCP || i > CCP_PMD_AUTH_OPT_CPU) { 93 CCP_LOG_ERR("Invalid ccp pmd auth option. " 94 "0->auth on CCP(default), " 95 "1->auth on CPU\n"); 96 return -EINVAL; 97 } 98 params->auth_opt = i; 99 return 0; 100 } 101 102 static int 103 ccp_pmd_parse_input_args(struct ccp_pmd_init_params *params, 104 const char *input_args) 105 { 106 struct rte_kvargs *kvlist = NULL; 107 int ret = 0; 108 109 if (params == NULL) 110 return -EINVAL; 111 112 if (input_args) { 113 kvlist = rte_kvargs_parse(input_args, 114 ccp_pmd_valid_params); 115 if (kvlist == NULL) 116 return -1; 117 118 ret = rte_kvargs_process(kvlist, 119 CCP_CRYPTODEV_PARAM_MAX_NB_QP, 120 &parse_integer_arg, 121 ¶ms->def_p.max_nb_queue_pairs); 122 if (ret < 0) 123 goto free_kvlist; 124 125 ret = rte_kvargs_process(kvlist, 126 CCP_CRYPTODEV_PARAM_SOCKET_ID, 127 &parse_integer_arg, 128 ¶ms->def_p.socket_id); 129 if (ret < 0) 130 goto free_kvlist; 131 132 ret = rte_kvargs_process(kvlist, 133 CCP_CRYPTODEV_PARAM_NAME, 134 &parse_name_arg, 135 ¶ms->def_p); 136 if (ret < 0) 137 goto free_kvlist; 138 139 ret = rte_kvargs_process(kvlist, 140 CCP_CRYPTODEV_PARAM_AUTH_OPT, 141 &parse_auth_opt_arg, 142 params); 143 if (ret < 0) 144 goto free_kvlist; 145 146 } 147 148 free_kvlist: 149 rte_kvargs_free(kvlist); 150 return ret; 151 } 152 153 static struct ccp_session * 154 get_ccp_session(struct ccp_qp *qp, struct rte_crypto_op *op) 155 { 156 struct ccp_session *sess = NULL; 157 158 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { 159 if (unlikely(op->sym->session == NULL)) 160 return NULL; 161 162 sess = (struct ccp_session *) 163 get_sym_session_private_data( 164 op->sym->session, 165 ccp_cryptodev_driver_id); 166 } else if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { 167 void *_sess; 168 void *_sess_private_data = NULL; 169 struct ccp_private *internals; 170 171 if (rte_mempool_get(qp->sess_mp, &_sess)) 172 return NULL; 173 if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) 174 return NULL; 175 176 sess = (struct ccp_session *)_sess_private_data; 177 178 internals = (struct ccp_private *)qp->dev->data->dev_private; 179 if (unlikely(ccp_set_session_parameters(sess, op->sym->xform, 180 internals) != 0)) { 181 rte_mempool_put(qp->sess_mp, _sess); 182 rte_mempool_put(qp->sess_mp, _sess_private_data); 183 sess = NULL; 184 } 185 op->sym->session = (struct rte_cryptodev_sym_session *)_sess; 186 set_sym_session_private_data(op->sym->session, 187 ccp_cryptodev_driver_id, 188 _sess_private_data); 189 } 190 191 return sess; 192 } 193 194 static uint16_t 195 ccp_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops, 196 uint16_t nb_ops) 197 { 198 struct ccp_session *sess = NULL; 199 struct ccp_qp *qp = queue_pair; 200 struct ccp_queue *cmd_q; 201 struct rte_cryptodev *dev = qp->dev; 202 uint16_t i, enq_cnt = 0, slots_req = 0; 203 204 if (nb_ops == 0) 205 return 0; 206 207 if (unlikely(rte_ring_full(qp->processed_pkts) != 0)) 208 return 0; 209 210 for (i = 0; i < nb_ops; i++) { 211 sess = get_ccp_session(qp, ops[i]); 212 if (unlikely(sess == NULL) && (i == 0)) { 213 qp->qp_stats.enqueue_err_count++; 214 return 0; 215 } else if (sess == NULL) { 216 nb_ops = i; 217 break; 218 } 219 slots_req += ccp_compute_slot_count(sess); 220 } 221 222 cmd_q = ccp_allot_queue(dev, slots_req); 223 if (unlikely(cmd_q == NULL)) 224 return 0; 225 226 enq_cnt = process_ops_to_enqueue(qp, ops, cmd_q, nb_ops, slots_req); 227 qp->qp_stats.enqueued_count += enq_cnt; 228 return enq_cnt; 229 } 230 231 static uint16_t 232 ccp_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, 233 uint16_t nb_ops) 234 { 235 struct ccp_qp *qp = queue_pair; 236 uint16_t nb_dequeued = 0, i; 237 238 nb_dequeued = process_ops_to_dequeue(qp, ops, nb_ops); 239 240 /* Free session if a session-less crypto op */ 241 for (i = 0; i < nb_dequeued; i++) 242 if (unlikely(ops[i]->sess_type == 243 RTE_CRYPTO_OP_SESSIONLESS)) { 244 rte_mempool_put(qp->sess_mp, 245 ops[i]->sym->session); 246 ops[i]->sym->session = NULL; 247 } 248 qp->qp_stats.dequeued_count += nb_dequeued; 249 250 return nb_dequeued; 251 } 252 253 /* 254 * The set of PCI devices this driver supports 255 */ 256 static struct rte_pci_id ccp_pci_id[] = { 257 { 258 RTE_PCI_DEVICE(0x1022, 0x1456), /* AMD CCP-5a */ 259 }, 260 { 261 RTE_PCI_DEVICE(0x1022, 0x1468), /* AMD CCP-5b */ 262 }, 263 {.device_id = 0}, 264 }; 265 266 /** Remove ccp pmd */ 267 static int 268 cryptodev_ccp_remove(struct rte_vdev_device *dev) 269 { 270 const char *name; 271 272 ccp_pmd_init_done = 0; 273 name = rte_vdev_device_name(dev); 274 if (name == NULL) 275 return -EINVAL; 276 277 RTE_LOG(INFO, PMD, "Closing ccp device %s on numa socket %u\n", 278 name, rte_socket_id()); 279 280 return 0; 281 } 282 283 /** Create crypto device */ 284 static int 285 cryptodev_ccp_create(const char *name, 286 struct rte_vdev_device *vdev, 287 struct ccp_pmd_init_params *init_params) 288 { 289 struct rte_cryptodev *dev; 290 struct ccp_private *internals; 291 uint8_t cryptodev_cnt = 0; 292 293 if (init_params->def_p.name[0] == '\0') 294 snprintf(init_params->def_p.name, 295 sizeof(init_params->def_p.name), 296 "%s", name); 297 298 dev = rte_cryptodev_pmd_create(init_params->def_p.name, 299 &vdev->device, 300 &init_params->def_p); 301 if (dev == NULL) { 302 CCP_LOG_ERR("failed to create cryptodev vdev"); 303 goto init_error; 304 } 305 306 cryptodev_cnt = ccp_probe_devices(ccp_pci_id); 307 308 if (cryptodev_cnt == 0) { 309 CCP_LOG_ERR("failed to detect CCP crypto device"); 310 goto init_error; 311 } 312 313 printf("CCP : Crypto device count = %d\n", cryptodev_cnt); 314 dev->driver_id = ccp_cryptodev_driver_id; 315 316 /* register rx/tx burst functions for data path */ 317 dev->dev_ops = ccp_pmd_ops; 318 dev->enqueue_burst = ccp_pmd_enqueue_burst; 319 dev->dequeue_burst = ccp_pmd_dequeue_burst; 320 321 dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | 322 RTE_CRYPTODEV_FF_HW_ACCELERATED | 323 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING; 324 325 internals = dev->data->dev_private; 326 327 internals->max_nb_qpairs = init_params->def_p.max_nb_queue_pairs; 328 internals->auth_opt = init_params->auth_opt; 329 internals->crypto_num_dev = cryptodev_cnt; 330 331 return 0; 332 333 init_error: 334 CCP_LOG_ERR("driver %s: %s() failed", 335 init_params->def_p.name, __func__); 336 cryptodev_ccp_remove(vdev); 337 338 return -EFAULT; 339 } 340 341 /** Probe ccp pmd */ 342 static int 343 cryptodev_ccp_probe(struct rte_vdev_device *vdev) 344 { 345 int rc = 0; 346 const char *name; 347 struct ccp_pmd_init_params init_params = { 348 .def_p = { 349 "", 350 sizeof(struct ccp_private), 351 rte_socket_id(), 352 CCP_PMD_MAX_QUEUE_PAIRS 353 }, 354 .auth_opt = CCP_PMD_AUTH_OPT_CCP, 355 }; 356 const char *input_args; 357 358 if (ccp_pmd_init_done) { 359 RTE_LOG(INFO, PMD, "CCP PMD already initialized\n"); 360 return -EFAULT; 361 } 362 name = rte_vdev_device_name(vdev); 363 if (name == NULL) 364 return -EINVAL; 365 366 input_args = rte_vdev_device_args(vdev); 367 ccp_pmd_parse_input_args(&init_params, input_args); 368 init_params.def_p.max_nb_queue_pairs = CCP_PMD_MAX_QUEUE_PAIRS; 369 370 RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name, 371 init_params.def_p.socket_id); 372 RTE_LOG(INFO, PMD, "Max number of queue pairs = %d\n", 373 init_params.def_p.max_nb_queue_pairs); 374 RTE_LOG(INFO, PMD, "Authentication offload to %s\n", 375 ((init_params.auth_opt == 0) ? "CCP" : "CPU")); 376 377 rc = cryptodev_ccp_create(name, vdev, &init_params); 378 if (rc) 379 return rc; 380 ccp_pmd_init_done = 1; 381 return 0; 382 } 383 384 static struct rte_vdev_driver cryptodev_ccp_pmd_drv = { 385 .probe = cryptodev_ccp_probe, 386 .remove = cryptodev_ccp_remove 387 }; 388 389 static struct cryptodev_driver ccp_crypto_drv; 390 391 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_CCP_PMD, cryptodev_ccp_pmd_drv); 392 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_CCP_PMD, 393 "max_nb_queue_pairs=<int> " 394 "socket_id=<int> " 395 "ccp_auth_opt=<int>"); 396 RTE_PMD_REGISTER_CRYPTO_DRIVER(ccp_crypto_drv, cryptodev_ccp_pmd_drv.driver, 397 ccp_cryptodev_driver_id); 398