1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2021 Intel Corporation 3 */ 4 5 #include <string.h> 6 7 #include <rte_common.h> 8 #include <rte_malloc.h> 9 10 #include "ipsec_mb_private.h" 11 12 #define IMB_MP_REQ_VER_STR "1.1.0" 13 14 /** Configure device */ 15 int 16 ipsec_mb_config(__rte_unused struct rte_cryptodev *dev, 17 __rte_unused struct rte_cryptodev_config *config) 18 { 19 return 0; 20 } 21 22 /** Start device */ 23 int 24 ipsec_mb_start(__rte_unused struct rte_cryptodev *dev) 25 { 26 return 0; 27 } 28 29 /** Stop device */ 30 void 31 ipsec_mb_stop(__rte_unused struct rte_cryptodev *dev) 32 { 33 } 34 35 /** Close device */ 36 int 37 ipsec_mb_close(__rte_unused struct rte_cryptodev *dev) 38 { 39 return 0; 40 } 41 42 /** Get device statistics */ 43 void 44 ipsec_mb_stats_get(struct rte_cryptodev *dev, 45 struct rte_cryptodev_stats *stats) 46 { 47 int qp_id; 48 49 for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) { 50 struct ipsec_mb_qp *qp = dev->data->queue_pairs[qp_id]; 51 if (qp == NULL) { 52 IPSEC_MB_LOG(DEBUG, "Uninitialised qp %d", qp_id); 53 continue; 54 } 55 56 stats->enqueued_count += qp->stats.enqueued_count; 57 stats->dequeued_count += qp->stats.dequeued_count; 58 59 stats->enqueue_err_count += qp->stats.enqueue_err_count; 60 stats->dequeue_err_count += qp->stats.dequeue_err_count; 61 } 62 } 63 64 /** Reset device statistics */ 65 void 66 ipsec_mb_stats_reset(struct rte_cryptodev *dev) 67 { 68 int qp_id; 69 70 for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) { 71 struct ipsec_mb_qp *qp = dev->data->queue_pairs[qp_id]; 72 73 memset(&qp->stats, 0, sizeof(qp->stats)); 74 } 75 } 76 77 /** Get device info */ 78 void 79 ipsec_mb_info_get(struct rte_cryptodev *dev, 80 struct rte_cryptodev_info *dev_info) 81 { 82 struct ipsec_mb_dev_private *internals = dev->data->dev_private; 83 struct ipsec_mb_internals *pmd_info = 84 &ipsec_mb_pmds[internals->pmd_type]; 85 86 if (dev_info != NULL) { 87 dev_info->driver_id = dev->driver_id; 88 dev_info->feature_flags = dev->feature_flags; 89 dev_info->capabilities = pmd_info->caps; 90 dev_info->max_nb_queue_pairs = internals->max_nb_queue_pairs; 91 /* No limit of number of sessions */ 92 dev_info->sym.max_nb_sessions = 0; 93 } 94 } 95 96 /** Release queue pair */ 97 int 98 ipsec_mb_qp_release(struct rte_cryptodev *dev, uint16_t qp_id) 99 { 100 struct ipsec_mb_qp *qp = dev->data->queue_pairs[qp_id]; 101 struct rte_ring *r = NULL; 102 103 if (qp != NULL && rte_eal_process_type() == RTE_PROC_PRIMARY) { 104 r = rte_ring_lookup(qp->name); 105 rte_ring_free(r); 106 107 #if IMB_VERSION(1, 1, 0) > IMB_VERSION_NUM 108 if (qp->mb_mgr) 109 free_mb_mgr(qp->mb_mgr); 110 #else 111 if (qp->mb_mgr_mz) { 112 rte_memzone_free(qp->mb_mgr_mz); 113 qp->mb_mgr = NULL; 114 } 115 #endif 116 rte_free(qp); 117 dev->data->queue_pairs[qp_id] = NULL; 118 } 119 return 0; 120 } 121 122 /** Set a unique name for the queue pair */ 123 int 124 ipsec_mb_qp_set_unique_name(struct rte_cryptodev *dev, 125 struct ipsec_mb_qp *qp) 126 { 127 uint32_t n = 128 snprintf(qp->name, sizeof(qp->name), "ipsec_mb_pmd_%u_qp_%u", 129 dev->data->dev_id, qp->id); 130 131 if (n >= sizeof(qp->name)) 132 return -1; 133 134 return 0; 135 } 136 137 /** Create a ring to place processed operations on */ 138 static struct rte_ring 139 *ipsec_mb_qp_create_processed_ops_ring( 140 struct ipsec_mb_qp *qp, unsigned int ring_size, int socket_id) 141 { 142 struct rte_ring *r; 143 char ring_name[RTE_CRYPTODEV_NAME_MAX_LEN]; 144 145 unsigned int n = rte_strlcpy(ring_name, qp->name, sizeof(ring_name)); 146 147 if (n >= sizeof(ring_name)) 148 return NULL; 149 150 r = rte_ring_lookup(ring_name); 151 if (r) { 152 if (rte_ring_get_size(r) >= ring_size) { 153 IPSEC_MB_LOG( 154 INFO, "Reusing existing ring %s for processed ops", 155 ring_name); 156 return r; 157 } 158 IPSEC_MB_LOG( 159 ERR, "Unable to reuse existing ring %s for processed ops", 160 ring_name); 161 return NULL; 162 } 163 164 return rte_ring_create(ring_name, ring_size, socket_id, 165 RING_F_SP_ENQ | RING_F_SC_DEQ); 166 } 167 168 #if IMB_VERSION(1, 1, 0) <= IMB_VERSION_NUM 169 static IMB_MGR * 170 ipsec_mb_alloc_mgr_from_memzone(const struct rte_memzone **mb_mgr_mz, 171 const char *mb_mgr_mz_name) 172 { 173 IMB_MGR *mb_mgr; 174 175 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 176 *mb_mgr_mz = rte_memzone_lookup(mb_mgr_mz_name); 177 if (*mb_mgr_mz == NULL) { 178 *mb_mgr_mz = rte_memzone_reserve(mb_mgr_mz_name, 179 imb_get_mb_mgr_size(), 180 rte_socket_id(), 0); 181 } 182 if (*mb_mgr_mz == NULL) { 183 IPSEC_MB_LOG(DEBUG, "Error allocating memzone for %s", 184 mb_mgr_mz_name); 185 return NULL; 186 } 187 mb_mgr = imb_set_pointers_mb_mgr((*mb_mgr_mz)->addr, 0, 1); 188 init_mb_mgr_auto(mb_mgr, NULL); 189 } else { 190 *mb_mgr_mz = rte_memzone_lookup(mb_mgr_mz_name); 191 if (*mb_mgr_mz == NULL) { 192 IPSEC_MB_LOG(ERR, 193 "Secondary can't find %s mz, did primary create it?", 194 mb_mgr_mz_name); 195 return NULL; 196 } 197 mb_mgr = imb_set_pointers_mb_mgr((*mb_mgr_mz)->addr, 0, 0); 198 } 199 return mb_mgr; 200 } 201 #endif 202 203 /** Setup a queue pair */ 204 int 205 ipsec_mb_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, 206 const struct rte_cryptodev_qp_conf *qp_conf, 207 int socket_id) 208 { 209 struct ipsec_mb_qp *qp = NULL; 210 struct ipsec_mb_dev_private *internals = dev->data->dev_private; 211 struct ipsec_mb_internals *pmd_data = 212 &ipsec_mb_pmds[internals->pmd_type]; 213 uint32_t qp_size; 214 int ret; 215 216 if (rte_eal_process_type() == RTE_PROC_SECONDARY) { 217 #if IMB_VERSION(1, 1, 0) > IMB_VERSION_NUM 218 IPSEC_MB_LOG(ERR, "The intel-ipsec-mb version (%s) does not support multiprocess," 219 "the minimum version required for this feature is %s.", 220 IMB_VERSION_STR, IMB_MP_REQ_VER_STR); 221 return -EINVAL; 222 #endif 223 qp = dev->data->queue_pairs[qp_id]; 224 if (qp == NULL) { 225 IPSEC_MB_LOG(ERR, "Primary process hasn't configured device qp."); 226 return -EINVAL; 227 } 228 } else { 229 /* Free memory prior to re-allocation if needed. */ 230 if (dev->data->queue_pairs[qp_id] != NULL) 231 ipsec_mb_qp_release(dev, qp_id); 232 233 qp_size = sizeof(*qp) + pmd_data->qp_priv_size; 234 /* Allocate the queue pair data structure. */ 235 qp = rte_zmalloc_socket("IPSEC PMD Queue Pair", qp_size, 236 RTE_CACHE_LINE_SIZE, socket_id); 237 if (qp == NULL) 238 return -ENOMEM; 239 } 240 241 #if IMB_VERSION(1, 1, 0) > IMB_VERSION_NUM 242 qp->mb_mgr = alloc_init_mb_mgr(); 243 #else 244 char mz_name[IPSEC_MB_MAX_MZ_NAME]; 245 snprintf(mz_name, sizeof(mz_name), "IMB_MGR_DEV_%d_QP_%d", 246 dev->data->dev_id, qp_id); 247 qp->mb_mgr = ipsec_mb_alloc_mgr_from_memzone(&(qp->mb_mgr_mz), 248 mz_name); 249 #endif 250 if (qp->mb_mgr == NULL) { 251 ret = -ENOMEM; 252 goto qp_setup_cleanup; 253 } 254 255 if (rte_eal_process_type() == RTE_PROC_SECONDARY) 256 return 0; 257 258 qp->id = qp_id; 259 dev->data->queue_pairs[qp_id] = qp; 260 if (ipsec_mb_qp_set_unique_name(dev, qp)) { 261 ret = -EINVAL; 262 goto qp_setup_cleanup; 263 } 264 265 qp->pmd_type = internals->pmd_type; 266 qp->sess_mp = qp_conf->mp_session; 267 qp->sess_mp_priv = qp_conf->mp_session_private; 268 269 qp->ingress_queue = ipsec_mb_qp_create_processed_ops_ring(qp, 270 qp_conf->nb_descriptors, socket_id); 271 if (qp->ingress_queue == NULL) { 272 ret = -EINVAL; 273 goto qp_setup_cleanup; 274 } 275 276 memset(&qp->stats, 0, sizeof(qp->stats)); 277 278 if (pmd_data->queue_pair_configure) { 279 ret = pmd_data->queue_pair_configure(qp); 280 if (ret < 0) 281 goto qp_setup_cleanup; 282 } 283 284 return 0; 285 286 qp_setup_cleanup: 287 #if IMB_VERSION(1, 1, 0) > IMB_VERSION_NUM 288 if (qp->mb_mgr) 289 free_mb_mgr(qp->mb_mgr); 290 #else 291 if (rte_eal_process_type() == RTE_PROC_SECONDARY) 292 return ret; 293 if (qp->mb_mgr_mz) 294 rte_memzone_free(qp->mb_mgr_mz); 295 #endif 296 rte_free(qp); 297 return ret; 298 } 299 300 /** Return the size of the specific pmd session structure */ 301 unsigned 302 ipsec_mb_sym_session_get_size(struct rte_cryptodev *dev) 303 { 304 struct ipsec_mb_dev_private *internals = dev->data->dev_private; 305 struct ipsec_mb_internals *pmd_data = 306 &ipsec_mb_pmds[internals->pmd_type]; 307 308 return pmd_data->session_priv_size; 309 } 310 311 /** Configure pmd specific multi-buffer session from a crypto xform chain */ 312 int 313 ipsec_mb_sym_session_configure( 314 struct rte_cryptodev *dev, struct rte_crypto_sym_xform *xform, 315 struct rte_cryptodev_sym_session *sess, struct rte_mempool *mempool) 316 { 317 void *sess_private_data; 318 struct ipsec_mb_dev_private *internals = dev->data->dev_private; 319 struct ipsec_mb_internals *pmd_data = 320 &ipsec_mb_pmds[internals->pmd_type]; 321 IMB_MGR *mb_mgr = alloc_init_mb_mgr(); 322 int ret = 0; 323 324 if (!mb_mgr) 325 return -ENOMEM; 326 327 if (unlikely(sess == NULL)) { 328 IPSEC_MB_LOG(ERR, "invalid session struct"); 329 free_mb_mgr(mb_mgr); 330 return -EINVAL; 331 } 332 333 if (rte_mempool_get(mempool, &sess_private_data)) { 334 IPSEC_MB_LOG(ERR, "Couldn't get object from session mempool"); 335 free_mb_mgr(mb_mgr); 336 return -ENOMEM; 337 } 338 339 ret = (*pmd_data->session_configure)(mb_mgr, sess_private_data, xform); 340 if (ret != 0) { 341 IPSEC_MB_LOG(ERR, "failed configure session parameters"); 342 343 /* Return session to mempool */ 344 rte_mempool_put(mempool, sess_private_data); 345 free_mb_mgr(mb_mgr); 346 return ret; 347 } 348 349 set_sym_session_private_data(sess, dev->driver_id, sess_private_data); 350 351 free_mb_mgr(mb_mgr); 352 return 0; 353 } 354 355 /** Clear the session memory */ 356 void 357 ipsec_mb_sym_session_clear(struct rte_cryptodev *dev, 358 struct rte_cryptodev_sym_session *sess) 359 { 360 uint8_t index = dev->driver_id; 361 void *sess_priv = get_sym_session_private_data(sess, index); 362 363 /* Zero out the whole structure */ 364 if (sess_priv) { 365 memset(sess_priv, 0, ipsec_mb_sym_session_get_size(dev)); 366 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); 367 368 set_sym_session_private_data(sess, index, NULL); 369 rte_mempool_put(sess_mp, sess_priv); 370 } 371 } 372