1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Intel Corporation 3 */ 4 5 #include <rte_malloc.h> 6 #include <rte_mbuf_pool_ops.h> 7 8 #include "cperf_test_common.h" 9 10 struct obj_params { 11 uint32_t src_buf_offset; 12 uint32_t dst_buf_offset; 13 uint16_t segment_sz; 14 uint16_t headroom_sz; 15 uint16_t data_len; 16 uint16_t segments_nb; 17 }; 18 19 static void 20 fill_single_seg_mbuf(struct rte_mbuf *m, struct rte_mempool *mp, 21 void *obj, uint32_t mbuf_offset, uint16_t segment_sz, 22 uint16_t headroom, uint16_t data_len) 23 { 24 uint32_t mbuf_hdr_size = sizeof(struct rte_mbuf); 25 26 /* start of buffer is after mbuf structure and priv data */ 27 m->priv_size = 0; 28 m->buf_addr = (char *)m + mbuf_hdr_size; 29 rte_mbuf_iova_set(m, rte_mempool_virt2iova(obj) + mbuf_offset + mbuf_hdr_size); 30 m->buf_len = segment_sz; 31 m->data_len = data_len; 32 m->pkt_len = data_len; 33 34 /* Use headroom specified for the buffer */ 35 m->data_off = headroom; 36 37 /* init some constant fields */ 38 m->pool = mp; 39 m->nb_segs = 1; 40 m->port = 0xff; 41 rte_mbuf_refcnt_set(m, 1); 42 m->next = NULL; 43 } 44 45 static void 46 fill_multi_seg_mbuf(struct rte_mbuf *m, struct rte_mempool *mp, 47 void *obj, uint32_t mbuf_offset, uint16_t segment_sz, 48 uint16_t headroom, uint16_t data_len, uint16_t segments_nb) 49 { 50 uint16_t mbuf_hdr_size = sizeof(struct rte_mbuf); 51 uint16_t remaining_segments = segments_nb; 52 rte_iova_t next_seg_phys_addr = rte_mempool_virt2iova(obj) + 53 mbuf_offset + mbuf_hdr_size; 54 55 do { 56 /* start of buffer is after mbuf structure and priv data */ 57 m->priv_size = 0; 58 m->buf_addr = (char *)m + mbuf_hdr_size; 59 rte_mbuf_iova_set(m, next_seg_phys_addr); 60 next_seg_phys_addr += mbuf_hdr_size + segment_sz; 61 m->buf_len = segment_sz; 62 m->data_len = data_len; 63 64 /* Use headroom specified for the buffer */ 65 m->data_off = headroom; 66 67 /* init some constant fields */ 68 m->pool = mp; 69 m->nb_segs = segments_nb; 70 m->port = 0xff; 71 rte_mbuf_refcnt_set(m, 1); 72 73 remaining_segments--; 74 if (remaining_segments > 0) { 75 m->next = (struct rte_mbuf *)((uint8_t *) m + mbuf_hdr_size + segment_sz); 76 m = m->next; 77 } else { 78 m->next = NULL; 79 } 80 } while (remaining_segments > 0); 81 } 82 83 static void 84 mempool_asym_obj_init(struct rte_mempool *mp, __rte_unused void *opaque_arg, 85 void *obj, __rte_unused unsigned int i) 86 { 87 struct rte_crypto_op *op = obj; 88 89 /* Set crypto operation */ 90 op->type = RTE_CRYPTO_OP_TYPE_ASYMMETRIC; 91 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 92 op->sess_type = RTE_CRYPTO_OP_WITH_SESSION; 93 op->phys_addr = rte_mem_virt2iova(obj); 94 op->mempool = mp; 95 } 96 97 static void 98 mempool_obj_init(struct rte_mempool *mp, 99 void *opaque_arg, 100 void *obj, 101 __rte_unused unsigned int i) 102 { 103 struct obj_params *params = opaque_arg; 104 struct rte_crypto_op *op = obj; 105 struct rte_mbuf *m = (struct rte_mbuf *) ((uint8_t *) obj + 106 params->src_buf_offset); 107 /* Set crypto operation */ 108 op->type = RTE_CRYPTO_OP_TYPE_SYMMETRIC; 109 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 110 op->sess_type = RTE_CRYPTO_OP_WITH_SESSION; 111 op->phys_addr = rte_mem_virt2iova(obj); 112 op->mempool = mp; 113 114 /* Set source buffer */ 115 op->sym->m_src = m; 116 if (params->segments_nb == 1) 117 fill_single_seg_mbuf(m, mp, obj, params->src_buf_offset, 118 params->segment_sz, params->headroom_sz, 119 params->data_len); 120 else 121 fill_multi_seg_mbuf(m, mp, obj, params->src_buf_offset, 122 params->segment_sz, params->headroom_sz, 123 params->data_len, params->segments_nb); 124 125 126 /* Set destination buffer */ 127 if (params->dst_buf_offset) { 128 m = (struct rte_mbuf *) ((uint8_t *) obj + 129 params->dst_buf_offset); 130 fill_single_seg_mbuf(m, mp, obj, params->dst_buf_offset, 131 params->segment_sz, params->headroom_sz, 132 params->data_len); 133 op->sym->m_dst = m; 134 } else 135 op->sym->m_dst = NULL; 136 } 137 138 int 139 cperf_alloc_common_memory(const struct cperf_options *options, 140 const struct cperf_test_vector *test_vector, 141 uint8_t dev_id, uint16_t qp_id, 142 size_t extra_op_priv_size, 143 uint32_t *src_buf_offset, 144 uint32_t *dst_buf_offset, 145 struct rte_mempool **pool) 146 { 147 const char *mp_ops_name; 148 char pool_name[32] = ""; 149 int ret; 150 151 /* Calculate the object size */ 152 uint16_t crypto_op_size = sizeof(struct rte_crypto_op); 153 uint16_t crypto_op_private_size; 154 155 if (cperf_is_asym_test(options)) { 156 crypto_op_size += sizeof(struct rte_crypto_asym_op); 157 snprintf(pool_name, RTE_MEMPOOL_NAMESIZE, "perf_asym_op_pool%u", 158 rte_socket_id()); 159 *pool = rte_crypto_op_pool_create( 160 pool_name, RTE_CRYPTO_OP_TYPE_ASYMMETRIC, 161 options->pool_sz, RTE_MEMPOOL_CACHE_MAX_SIZE, 0, 162 rte_socket_id()); 163 if (*pool == NULL) { 164 RTE_LOG(ERR, USER1, 165 "Cannot allocate mempool for device %u\n", 166 dev_id); 167 return -1; 168 } 169 rte_mempool_obj_iter(*pool, mempool_asym_obj_init, NULL); 170 return 0; 171 } 172 173 crypto_op_size += sizeof(struct rte_crypto_sym_op); 174 175 /* 176 * If doing AES-CCM, IV field needs to be 16 bytes long, 177 * and AAD field needs to be long enough to have 18 bytes, 178 * plus the length of the AAD, and all rounded to a 179 * multiple of 16 bytes. 180 */ 181 if (options->aead_algo == RTE_CRYPTO_AEAD_AES_CCM) { 182 crypto_op_private_size = extra_op_priv_size + 183 test_vector->cipher_iv.length + 184 test_vector->auth_iv.length + 185 RTE_ALIGN_CEIL(test_vector->aead_iv.length, 16) + 186 RTE_ALIGN_CEIL(options->aead_aad_sz + 18, 16); 187 } else { 188 crypto_op_private_size = extra_op_priv_size + 189 test_vector->cipher_iv.length + 190 test_vector->auth_iv.length + 191 test_vector->aead_iv.length + 192 options->aead_aad_sz; 193 } 194 195 uint16_t crypto_op_total_size = crypto_op_size + 196 crypto_op_private_size; 197 uint16_t crypto_op_total_size_padded = 198 RTE_CACHE_LINE_ROUNDUP(crypto_op_total_size); 199 uint32_t mbuf_size = sizeof(struct rte_mbuf) + options->segment_sz; 200 uint32_t max_size = options->max_buffer_size + options->digest_sz; 201 uint32_t segment_data_len = options->segment_sz - options->headroom_sz - 202 options->tailroom_sz; 203 uint16_t segments_nb = (max_size % segment_data_len) ? 204 (max_size / segment_data_len) + 1 : 205 (max_size / segment_data_len); 206 uint32_t obj_size = crypto_op_total_size_padded + 207 (mbuf_size * segments_nb); 208 209 snprintf(pool_name, sizeof(pool_name), "pool_cdev_%u_qp_%u", 210 dev_id, qp_id); 211 212 *src_buf_offset = crypto_op_total_size_padded; 213 214 struct obj_params params = { 215 .segment_sz = options->segment_sz, 216 .headroom_sz = options->headroom_sz, 217 /* Data len = segment size - (headroom + tailroom) */ 218 .data_len = options->segment_sz - 219 options->headroom_sz - 220 options->tailroom_sz, 221 .segments_nb = segments_nb, 222 .src_buf_offset = crypto_op_total_size_padded, 223 .dst_buf_offset = 0 224 }; 225 226 if (options->out_of_place) { 227 *dst_buf_offset = *src_buf_offset + 228 (mbuf_size * segments_nb); 229 params.dst_buf_offset = *dst_buf_offset; 230 /* Destination buffer will be one segment only */ 231 obj_size += max_size + sizeof(struct rte_mbuf) + 232 options->headroom_sz + options->tailroom_sz; 233 } 234 235 *pool = rte_mempool_create_empty(pool_name, 236 options->pool_sz, obj_size, 512, 0, 237 rte_socket_id(), 0); 238 if (*pool == NULL) { 239 RTE_LOG(ERR, USER1, 240 "Cannot allocate mempool for device %u\n", 241 dev_id); 242 return -1; 243 } 244 245 mp_ops_name = rte_mbuf_best_mempool_ops(); 246 247 ret = rte_mempool_set_ops_byname(*pool, 248 mp_ops_name, NULL); 249 if (ret != 0) { 250 RTE_LOG(ERR, USER1, 251 "Error setting mempool handler for device %u\n", 252 dev_id); 253 return -1; 254 } 255 256 ret = rte_mempool_populate_default(*pool); 257 if (ret < 0) { 258 RTE_LOG(ERR, USER1, 259 "Error populating mempool for device %u\n", 260 dev_id); 261 return -1; 262 } 263 264 rte_mempool_obj_iter(*pool, mempool_obj_init, (void *)¶ms); 265 266 return 0; 267 } 268 269 void 270 cperf_mbuf_set(struct rte_mbuf *mbuf, 271 const struct cperf_options *options, 272 const struct cperf_test_vector *test_vector) 273 { 274 uint32_t segment_sz = options->segment_sz - options->headroom_sz - options->tailroom_sz; 275 uint8_t *mbuf_data; 276 uint8_t *test_data; 277 uint32_t remaining_bytes = options->max_buffer_size; 278 279 if (options->op_type == CPERF_AEAD) { 280 test_data = (options->aead_op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ? 281 test_vector->plaintext.data : 282 test_vector->ciphertext.data; 283 } else { 284 test_data = 285 (options->cipher_op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ? 286 test_vector->plaintext.data : 287 test_vector->ciphertext.data; 288 } 289 290 while (remaining_bytes) { 291 mbuf_data = rte_pktmbuf_mtod(mbuf, uint8_t *); 292 293 if (remaining_bytes <= segment_sz) { 294 memcpy(mbuf_data, test_data, remaining_bytes); 295 return; 296 } 297 298 memcpy(mbuf_data, test_data, segment_sz); 299 remaining_bytes -= segment_sz; 300 test_data += segment_sz; 301 mbuf = mbuf->next; 302 } 303 } 304 305 bool 306 cperf_is_asym_test(const struct cperf_options *options) 307 { 308 if (options->op_type == CPERF_ASYM_MODEX || 309 options->op_type == CPERF_ASYM_SECP256R1 || 310 options->op_type == CPERF_ASYM_ED25519 || 311 options->op_type == CPERF_ASYM_SM2) 312 return true; 313 314 return false; 315 } 316