1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2020 Broadcom 3 * All rights reserved. 4 */ 5 6 #include <stdbool.h> 7 8 #include <rte_byteorder.h> 9 #include <rte_crypto_sym.h> 10 #include <rte_cryptodev.h> 11 #include <rte_mbuf.h> 12 #include <rte_mempool.h> 13 14 #include "bcmfs_sym_defs.h" 15 #include "bcmfs_sym_engine.h" 16 #include "bcmfs_sym_req.h" 17 #include "bcmfs_sym_session.h" 18 19 /** Process cipher operation */ 20 static int 21 process_crypto_cipher_op(struct rte_crypto_op *op, 22 struct rte_mbuf *mbuf_src, 23 struct rte_mbuf *mbuf_dst, 24 struct bcmfs_sym_session *sess, 25 struct bcmfs_sym_request *req) 26 { 27 int rc = 0; 28 struct fsattr src, dst, iv, key; 29 struct rte_crypto_sym_op *sym_op = op->sym; 30 31 fsattr_sz(&src) = sym_op->cipher.data.length; 32 fsattr_sz(&dst) = sym_op->cipher.data.length; 33 34 fsattr_va(&src) = rte_pktmbuf_mtod_offset 35 (mbuf_src, 36 uint8_t *, 37 op->sym->cipher.data.offset); 38 39 fsattr_va(&dst) = rte_pktmbuf_mtod_offset 40 (mbuf_dst, 41 uint8_t *, 42 op->sym->cipher.data.offset); 43 44 fsattr_pa(&src) = rte_pktmbuf_iova(mbuf_src); 45 fsattr_pa(&dst) = rte_pktmbuf_iova(mbuf_dst); 46 47 fsattr_va(&iv) = rte_crypto_op_ctod_offset(op, 48 uint8_t *, 49 sess->cipher.iv.offset); 50 51 fsattr_sz(&iv) = sess->cipher.iv.length; 52 53 fsattr_va(&key) = sess->cipher.key.data; 54 fsattr_pa(&key) = 0; 55 fsattr_sz(&key) = sess->cipher.key.length; 56 57 rc = bcmfs_crypto_build_cipher_req(req, sess->cipher.algo, 58 sess->cipher.op, &src, 59 &dst, &key, &iv); 60 if (rc) 61 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 62 63 return rc; 64 } 65 66 /** Process auth operation */ 67 static int 68 process_crypto_auth_op(struct rte_crypto_op *op, 69 struct rte_mbuf *mbuf_src, 70 struct bcmfs_sym_session *sess, 71 struct bcmfs_sym_request *req) 72 { 73 int rc = 0; 74 struct fsattr src, dst, mac, key, iv; 75 76 fsattr_sz(&src) = op->sym->auth.data.length; 77 fsattr_va(&src) = rte_pktmbuf_mtod_offset(mbuf_src, 78 uint8_t *, 79 op->sym->auth.data.offset); 80 fsattr_pa(&src) = rte_pktmbuf_iova(mbuf_src); 81 82 if (!sess->auth.op) { 83 fsattr_va(&mac) = op->sym->auth.digest.data; 84 fsattr_pa(&mac) = op->sym->auth.digest.phys_addr; 85 fsattr_sz(&mac) = sess->auth.digest_length; 86 } else { 87 fsattr_va(&dst) = op->sym->auth.digest.data; 88 fsattr_pa(&dst) = op->sym->auth.digest.phys_addr; 89 fsattr_sz(&dst) = sess->auth.digest_length; 90 } 91 92 fsattr_va(&key) = sess->auth.key.data; 93 fsattr_pa(&key) = 0; 94 fsattr_sz(&key) = sess->auth.key.length; 95 96 /* AES-GMAC uses AES-GCM-128 authenticator */ 97 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) { 98 fsattr_va(&iv) = rte_crypto_op_ctod_offset(op, 99 uint8_t *, 100 sess->auth.iv.offset); 101 fsattr_pa(&iv) = 0; 102 fsattr_sz(&iv) = sess->auth.iv.length; 103 } else { 104 fsattr_va(&iv) = NULL; 105 fsattr_sz(&iv) = 0; 106 } 107 108 rc = bcmfs_crypto_build_auth_req(req, sess->auth.algo, 109 sess->auth.op, 110 &src, 111 (sess->auth.op) ? (&dst) : NULL, 112 (sess->auth.op) ? NULL : (&mac), 113 &key, &iv); 114 115 if (rc) 116 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 117 118 return rc; 119 } 120 121 /** Process combined/chained mode operation */ 122 static int 123 process_crypto_combined_op(struct rte_crypto_op *op, 124 struct rte_mbuf *mbuf_src, 125 struct rte_mbuf *mbuf_dst, 126 struct bcmfs_sym_session *sess, 127 struct bcmfs_sym_request *req) 128 { 129 int rc = 0, aad_size = 0; 130 struct fsattr src, dst, iv; 131 struct rte_crypto_sym_op *sym_op = op->sym; 132 struct fsattr cipher_key, aad, mac, auth_key; 133 134 fsattr_sz(&src) = sym_op->cipher.data.length; 135 fsattr_sz(&dst) = sym_op->cipher.data.length; 136 137 fsattr_va(&src) = rte_pktmbuf_mtod_offset 138 (mbuf_src, 139 uint8_t *, 140 sym_op->cipher.data.offset); 141 142 fsattr_va(&dst) = rte_pktmbuf_mtod_offset 143 (mbuf_dst, 144 uint8_t *, 145 sym_op->cipher.data.offset); 146 147 fsattr_pa(&src) = rte_pktmbuf_iova_offset(mbuf_src, 148 sym_op->cipher.data.offset); 149 fsattr_pa(&dst) = rte_pktmbuf_iova_offset(mbuf_dst, 150 sym_op->cipher.data.offset); 151 152 fsattr_va(&iv) = rte_crypto_op_ctod_offset(op, 153 uint8_t *, 154 sess->cipher.iv.offset); 155 156 fsattr_pa(&iv) = 0; 157 fsattr_sz(&iv) = sess->cipher.iv.length; 158 159 fsattr_va(&cipher_key) = sess->cipher.key.data; 160 fsattr_pa(&cipher_key) = 0; 161 fsattr_sz(&cipher_key) = sess->cipher.key.length; 162 163 fsattr_va(&auth_key) = sess->auth.key.data; 164 fsattr_pa(&auth_key) = 0; 165 fsattr_sz(&auth_key) = sess->auth.key.length; 166 167 fsattr_va(&mac) = op->sym->auth.digest.data; 168 fsattr_pa(&mac) = op->sym->auth.digest.phys_addr; 169 fsattr_sz(&mac) = sess->auth.digest_length; 170 171 aad_size = sym_op->auth.data.length - sym_op->cipher.data.length; 172 173 if (aad_size > 0) { 174 fsattr_sz(&aad) = aad_size; 175 fsattr_va(&aad) = rte_pktmbuf_mtod_offset 176 (mbuf_src, 177 uint8_t *, 178 sym_op->auth.data.offset); 179 fsattr_pa(&aad) = rte_pktmbuf_iova_offset(mbuf_src, 180 sym_op->auth.data.offset); 181 } 182 183 rc = bcmfs_crypto_build_chain_request(req, sess->cipher.algo, 184 sess->cipher.op, 185 sess->auth.algo, 186 sess->auth.op, 187 &src, &dst, &cipher_key, 188 &auth_key, &iv, 189 (aad_size > 0) ? (&aad) : NULL, 190 &mac, sess->cipher_first); 191 192 if (rc) 193 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 194 195 return rc; 196 } 197 198 /** Process AEAD operation */ 199 static int 200 process_crypto_aead_op(struct rte_crypto_op *op, 201 struct rte_mbuf *mbuf_src, 202 struct rte_mbuf *mbuf_dst, 203 struct bcmfs_sym_session *sess, 204 struct bcmfs_sym_request *req) 205 { 206 int rc = 0; 207 struct fsattr src, dst, iv; 208 struct rte_crypto_sym_op *sym_op = op->sym; 209 struct fsattr key, aad, mac; 210 211 fsattr_sz(&src) = sym_op->aead.data.length; 212 fsattr_sz(&dst) = sym_op->aead.data.length; 213 214 fsattr_va(&src) = rte_pktmbuf_mtod_offset(mbuf_src, 215 uint8_t *, 216 sym_op->aead.data.offset); 217 218 fsattr_va(&dst) = rte_pktmbuf_mtod_offset(mbuf_dst, 219 uint8_t *, 220 sym_op->aead.data.offset); 221 222 fsattr_pa(&src) = rte_pktmbuf_iova_offset(mbuf_src, 223 sym_op->aead.data.offset); 224 fsattr_pa(&dst) = rte_pktmbuf_iova_offset(mbuf_dst, 225 sym_op->aead.data.offset); 226 227 fsattr_va(&iv) = rte_crypto_op_ctod_offset(op, 228 uint8_t *, 229 sess->aead.iv.offset); 230 231 fsattr_pa(&iv) = 0; 232 fsattr_sz(&iv) = sess->aead.iv.length; 233 234 fsattr_va(&key) = sess->aead.key.data; 235 fsattr_pa(&key) = 0; 236 fsattr_sz(&key) = sess->aead.key.length; 237 238 fsattr_va(&mac) = op->sym->aead.digest.data; 239 fsattr_pa(&mac) = op->sym->aead.digest.phys_addr; 240 fsattr_sz(&mac) = sess->aead.digest_length; 241 242 fsattr_va(&aad) = op->sym->aead.aad.data; 243 fsattr_pa(&aad) = op->sym->aead.aad.phys_addr; 244 fsattr_sz(&aad) = sess->aead.aad_length; 245 246 rc = bcmfs_crypto_build_aead_request(req, sess->aead.algo, 247 sess->aead.op, &src, &dst, 248 &key, &iv, &aad, &mac); 249 250 if (rc) 251 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 252 253 return rc; 254 } 255 256 /** Process crypto operation for mbuf */ 257 int 258 bcmfs_process_sym_crypto_op(struct rte_crypto_op *op, 259 struct bcmfs_sym_session *sess, 260 struct bcmfs_sym_request *req) 261 { 262 struct rte_mbuf *msrc, *mdst; 263 int rc = 0; 264 265 msrc = op->sym->m_src; 266 mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src; 267 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 268 269 switch (sess->chain_order) { 270 case BCMFS_SYM_CHAIN_ONLY_CIPHER: 271 rc = process_crypto_cipher_op(op, msrc, mdst, sess, req); 272 break; 273 case BCMFS_SYM_CHAIN_ONLY_AUTH: 274 rc = process_crypto_auth_op(op, msrc, sess, req); 275 break; 276 case BCMFS_SYM_CHAIN_CIPHER_AUTH: 277 case BCMFS_SYM_CHAIN_AUTH_CIPHER: 278 rc = process_crypto_combined_op(op, msrc, mdst, sess, req); 279 break; 280 case BCMFS_SYM_CHAIN_AEAD: 281 rc = process_crypto_aead_op(op, msrc, mdst, sess, req); 282 break; 283 default: 284 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 285 break; 286 } 287 288 return rc; 289 } 290