1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2021 Marvell. 3 */ 4 5 #include <cryptodev_pmd.h> 6 #include <rte_esp.h> 7 #include <rte_ip.h> 8 #include <rte_malloc.h> 9 #include <rte_security.h> 10 #include <rte_security_driver.h> 11 #include <rte_udp.h> 12 13 #include "cn10k_cryptodev_ops.h" 14 #include "cn10k_cryptodev_sec.h" 15 #include "cn10k_ipsec.h" 16 #include "cnxk_cryptodev.h" 17 #include "cnxk_cryptodev_ops.h" 18 #include "cnxk_ipsec.h" 19 #include "cnxk_security.h" 20 21 #include "roc_api.h" 22 23 static int 24 cn10k_ipsec_outb_sa_create(struct roc_cpt *roc_cpt, struct roc_cpt_lf *lf, 25 struct rte_security_ipsec_xform *ipsec_xfrm, 26 struct rte_crypto_sym_xform *crypto_xfrm, 27 struct cn10k_sec_session *sec_sess) 28 { 29 union roc_ot_ipsec_outb_param1 param1; 30 struct roc_ot_ipsec_outb_sa *sa_dptr; 31 struct cnxk_ipsec_outb_rlens rlens; 32 struct cn10k_ipsec_sa *sa; 33 union cpt_inst_w4 inst_w4; 34 void *out_sa; 35 int ret = 0; 36 37 sa = &sec_sess->sa; 38 out_sa = &sa->out_sa; 39 40 /* Allocate memory to be used as dptr for CPT ucode WRITE_SA op */ 41 sa_dptr = plt_zmalloc(sizeof(struct roc_ot_ipsec_outb_sa), 8); 42 if (sa_dptr == NULL) { 43 plt_err("Couldn't allocate memory for SA dptr"); 44 return -ENOMEM; 45 } 46 47 /* Translate security parameters to SA */ 48 ret = cnxk_ot_ipsec_outb_sa_fill(sa_dptr, ipsec_xfrm, crypto_xfrm); 49 if (ret) { 50 plt_err("Could not fill outbound session parameters"); 51 goto sa_dptr_free; 52 } 53 54 sec_sess->inst.w7 = cpt_inst_w7_get(roc_cpt, out_sa); 55 56 #ifdef LA_IPSEC_DEBUG 57 /* Use IV from application in debug mode */ 58 if (ipsec_xfrm->options.iv_gen_disable == 1) { 59 sa_dptr->w2.s.iv_src = ROC_IE_OT_SA_IV_SRC_FROM_SA; 60 if (crypto_xfrm->type == RTE_CRYPTO_SYM_XFORM_AEAD) { 61 sec_sess->iv_offset = crypto_xfrm->aead.iv.offset; 62 sec_sess->iv_length = crypto_xfrm->aead.iv.length; 63 } else if (crypto_xfrm->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { 64 sec_sess->iv_offset = crypto_xfrm->cipher.iv.offset; 65 sec_sess->iv_length = crypto_xfrm->cipher.iv.length; 66 } else { 67 sec_sess->iv_offset = crypto_xfrm->auth.iv.offset; 68 sec_sess->iv_length = crypto_xfrm->auth.iv.length; 69 } 70 } 71 #else 72 if (ipsec_xfrm->options.iv_gen_disable != 0) { 73 plt_err("Application provided IV not supported"); 74 ret = -ENOTSUP; 75 goto sa_dptr_free; 76 } 77 #endif 78 79 sec_sess->ipsec.is_outbound = true; 80 81 /* Get Rlen calculation data */ 82 ret = cnxk_ipsec_outb_rlens_get(&rlens, ipsec_xfrm, crypto_xfrm); 83 if (ret) 84 goto sa_dptr_free; 85 86 sec_sess->max_extended_len = rlens.max_extended_len; 87 88 /* pre-populate CPT INST word 4 */ 89 inst_w4.u64 = 0; 90 inst_w4.s.opcode_major = ROC_IE_OT_MAJOR_OP_PROCESS_OUTBOUND_IPSEC | ROC_IE_OT_INPLACE_BIT; 91 92 param1.u16 = 0; 93 94 param1.s.ttl_or_hop_limit = ipsec_xfrm->options.dec_ttl; 95 96 /* Disable IP checksum computation by default */ 97 param1.s.ip_csum_disable = ROC_IE_OT_SA_INNER_PKT_IP_CSUM_DISABLE; 98 99 if (ipsec_xfrm->options.ip_csum_enable) { 100 param1.s.ip_csum_disable = 101 ROC_IE_OT_SA_INNER_PKT_IP_CSUM_ENABLE; 102 } 103 104 /* Disable L4 checksum computation by default */ 105 param1.s.l4_csum_disable = ROC_IE_OT_SA_INNER_PKT_L4_CSUM_DISABLE; 106 107 if (ipsec_xfrm->options.l4_csum_enable) { 108 param1.s.l4_csum_disable = 109 ROC_IE_OT_SA_INNER_PKT_L4_CSUM_ENABLE; 110 } 111 112 inst_w4.s.param1 = param1.u16; 113 114 sec_sess->inst.w4 = inst_w4.u64; 115 116 if (ipsec_xfrm->options.stats == 1) { 117 /* Enable mib counters */ 118 sa_dptr->w0.s.count_mib_bytes = 1; 119 sa_dptr->w0.s.count_mib_pkts = 1; 120 } 121 122 memset(out_sa, 0, sizeof(struct roc_ot_ipsec_outb_sa)); 123 124 /* Copy word0 from sa_dptr to populate ctx_push_sz ctx_size fields */ 125 memcpy(out_sa, sa_dptr, 8); 126 127 plt_atomic_thread_fence(__ATOMIC_SEQ_CST); 128 129 /* Write session using microcode opcode */ 130 ret = roc_cpt_ctx_write(lf, sa_dptr, out_sa, 131 sizeof(struct roc_ot_ipsec_outb_sa)); 132 if (ret) { 133 plt_err("Could not write outbound session to hardware"); 134 goto sa_dptr_free; 135 } 136 137 /* Trigger CTX flush so that data is written back to DRAM */ 138 roc_cpt_lf_ctx_flush(lf, out_sa, false); 139 140 sec_sess->proto = RTE_SECURITY_PROTOCOL_IPSEC; 141 plt_atomic_thread_fence(__ATOMIC_SEQ_CST); 142 143 sa_dptr_free: 144 plt_free(sa_dptr); 145 146 return ret; 147 } 148 149 static int 150 cn10k_ipsec_inb_sa_create(struct roc_cpt *roc_cpt, struct roc_cpt_lf *lf, 151 struct rte_security_ipsec_xform *ipsec_xfrm, 152 struct rte_crypto_sym_xform *crypto_xfrm, 153 struct cn10k_sec_session *sec_sess) 154 { 155 union roc_ot_ipsec_inb_param1 param1; 156 struct roc_ot_ipsec_inb_sa *sa_dptr; 157 struct cn10k_ipsec_sa *sa; 158 union cpt_inst_w4 inst_w4; 159 void *in_sa; 160 int ret = 0; 161 162 sa = &sec_sess->sa; 163 in_sa = &sa->in_sa; 164 165 /* Allocate memory to be used as dptr for CPT ucode WRITE_SA op */ 166 sa_dptr = plt_zmalloc(sizeof(struct roc_ot_ipsec_inb_sa), 8); 167 if (sa_dptr == NULL) { 168 plt_err("Couldn't allocate memory for SA dptr"); 169 return -ENOMEM; 170 } 171 172 /* Translate security parameters to SA */ 173 ret = cnxk_ot_ipsec_inb_sa_fill(sa_dptr, ipsec_xfrm, crypto_xfrm, 174 false); 175 if (ret) { 176 plt_err("Could not fill inbound session parameters"); 177 goto sa_dptr_free; 178 } 179 180 sec_sess->ipsec.is_outbound = false; 181 sec_sess->inst.w7 = cpt_inst_w7_get(roc_cpt, in_sa); 182 183 /* Save index/SPI in cookie, specific required for Rx Inject */ 184 sa_dptr->w1.s.cookie = 0xFFFFFFFF; 185 186 /* pre-populate CPT INST word 4 */ 187 inst_w4.u64 = 0; 188 inst_w4.s.opcode_major = ROC_IE_OT_MAJOR_OP_PROCESS_INBOUND_IPSEC | ROC_IE_OT_INPLACE_BIT; 189 190 param1.u16 = 0; 191 192 /* Disable IP checksum verification by default */ 193 param1.s.ip_csum_disable = ROC_IE_OT_SA_INNER_PKT_IP_CSUM_DISABLE; 194 195 /* Set the ip chksum flag in mbuf before enqueue. 196 * Reset the flag in post process in case of errors 197 */ 198 if (ipsec_xfrm->options.ip_csum_enable) { 199 param1.s.ip_csum_disable = ROC_IE_OT_SA_INNER_PKT_IP_CSUM_ENABLE; 200 sec_sess->ipsec.ip_csum = RTE_MBUF_F_RX_IP_CKSUM_GOOD; 201 } 202 203 /* Disable L4 checksum verification by default */ 204 param1.s.l4_csum_disable = ROC_IE_OT_SA_INNER_PKT_L4_CSUM_DISABLE; 205 206 if (ipsec_xfrm->options.l4_csum_enable) { 207 param1.s.l4_csum_disable = 208 ROC_IE_OT_SA_INNER_PKT_L4_CSUM_ENABLE; 209 } 210 211 param1.s.esp_trailer_disable = 1; 212 213 inst_w4.s.param1 = param1.u16; 214 215 sec_sess->inst.w4 = inst_w4.u64; 216 217 if (ipsec_xfrm->options.stats == 1) { 218 /* Enable mib counters */ 219 sa_dptr->w0.s.count_mib_bytes = 1; 220 sa_dptr->w0.s.count_mib_pkts = 1; 221 } 222 223 memset(in_sa, 0, sizeof(struct roc_ot_ipsec_inb_sa)); 224 225 /* Copy word0 from sa_dptr to populate ctx_push_sz ctx_size fields */ 226 memcpy(in_sa, sa_dptr, 8); 227 228 plt_atomic_thread_fence(__ATOMIC_SEQ_CST); 229 230 /* Write session using microcode opcode */ 231 ret = roc_cpt_ctx_write(lf, sa_dptr, in_sa, 232 sizeof(struct roc_ot_ipsec_inb_sa)); 233 if (ret) { 234 plt_err("Could not write inbound session to hardware"); 235 goto sa_dptr_free; 236 } 237 238 /* Trigger CTX flush so that data is written back to DRAM */ 239 roc_cpt_lf_ctx_flush(lf, in_sa, true); 240 241 sec_sess->proto = RTE_SECURITY_PROTOCOL_IPSEC; 242 plt_atomic_thread_fence(__ATOMIC_SEQ_CST); 243 244 sa_dptr_free: 245 plt_free(sa_dptr); 246 247 return ret; 248 } 249 250 int 251 cn10k_ipsec_session_create(struct cnxk_cpt_vf *vf, struct cnxk_cpt_qp *qp, 252 struct rte_security_ipsec_xform *ipsec_xfrm, 253 struct rte_crypto_sym_xform *crypto_xfrm, 254 struct rte_security_session *sess) 255 { 256 struct roc_cpt *roc_cpt; 257 int ret; 258 259 ret = cnxk_ipsec_xform_verify(ipsec_xfrm, crypto_xfrm); 260 if (ret) 261 return ret; 262 263 roc_cpt = &vf->cpt; 264 265 if (ipsec_xfrm->direction == RTE_SECURITY_IPSEC_SA_DIR_INGRESS) 266 return cn10k_ipsec_inb_sa_create(roc_cpt, &qp->lf, ipsec_xfrm, crypto_xfrm, 267 (struct cn10k_sec_session *)sess); 268 else 269 return cn10k_ipsec_outb_sa_create(roc_cpt, &qp->lf, ipsec_xfrm, crypto_xfrm, 270 (struct cn10k_sec_session *)sess); 271 } 272 273 int 274 cn10k_sec_ipsec_session_destroy(struct cnxk_cpt_qp *qp, struct cn10k_sec_session *sess) 275 { 276 union roc_ot_ipsec_sa_word2 *w2; 277 struct cn10k_ipsec_sa *sa; 278 struct roc_cpt_lf *lf; 279 void *sa_dptr = NULL; 280 int ret; 281 282 lf = &qp->lf; 283 284 sa = &sess->sa; 285 286 /* Trigger CTX flush to write dirty data back to DRAM */ 287 roc_cpt_lf_ctx_flush(lf, &sa->in_sa, false); 288 289 ret = -1; 290 291 if (sess->ipsec.is_outbound) { 292 sa_dptr = plt_zmalloc(sizeof(struct roc_ot_ipsec_outb_sa), 8); 293 if (sa_dptr != NULL) { 294 roc_ot_ipsec_outb_sa_init(sa_dptr); 295 296 ret = roc_cpt_ctx_write( 297 lf, sa_dptr, &sa->out_sa, 298 sizeof(struct roc_ot_ipsec_outb_sa)); 299 } 300 } else { 301 sa_dptr = plt_zmalloc(sizeof(struct roc_ot_ipsec_inb_sa), 8); 302 if (sa_dptr != NULL) { 303 roc_ot_ipsec_inb_sa_init(sa_dptr, false); 304 305 ret = roc_cpt_ctx_write( 306 lf, sa_dptr, &sa->in_sa, 307 sizeof(struct roc_ot_ipsec_inb_sa)); 308 } 309 } 310 311 plt_free(sa_dptr); 312 313 if (ret) { 314 /* MC write_ctx failed. Attempt reload of CTX */ 315 316 /* Wait for 1 ms so that flush is complete */ 317 rte_delay_ms(1); 318 319 w2 = (union roc_ot_ipsec_sa_word2 *)&sa->in_sa.w2; 320 w2->s.valid = 0; 321 322 plt_atomic_thread_fence(__ATOMIC_SEQ_CST); 323 324 /* Trigger CTX reload to fetch new data from DRAM */ 325 roc_cpt_lf_ctx_reload(lf, &sa->in_sa); 326 } 327 328 return 0; 329 } 330 331 int 332 cn10k_ipsec_stats_get(struct cnxk_cpt_qp *qp, struct cn10k_sec_session *sess, 333 struct rte_security_stats *stats) 334 { 335 struct roc_ot_ipsec_outb_sa *out_sa; 336 struct roc_ot_ipsec_inb_sa *in_sa; 337 struct cn10k_ipsec_sa *sa; 338 339 stats->protocol = RTE_SECURITY_PROTOCOL_IPSEC; 340 sa = &sess->sa; 341 342 if (sess->ipsec.is_outbound) { 343 out_sa = &sa->out_sa; 344 roc_cpt_lf_ctx_flush(&qp->lf, out_sa, false); 345 rte_delay_ms(1); 346 stats->ipsec.opackets = out_sa->ctx.mib_pkts; 347 stats->ipsec.obytes = out_sa->ctx.mib_octs; 348 } else { 349 in_sa = &sa->in_sa; 350 roc_cpt_lf_ctx_flush(&qp->lf, in_sa, false); 351 rte_delay_ms(1); 352 stats->ipsec.ipackets = in_sa->ctx.mib_pkts; 353 stats->ipsec.ibytes = in_sa->ctx.mib_octs; 354 } 355 356 return 0; 357 } 358 359 int 360 cn10k_ipsec_session_update(struct cnxk_cpt_vf *vf, struct cnxk_cpt_qp *qp, 361 struct cn10k_sec_session *sess, struct rte_security_session_conf *conf) 362 { 363 struct roc_cpt *roc_cpt; 364 int ret; 365 366 if (conf->ipsec.direction == RTE_SECURITY_IPSEC_SA_DIR_INGRESS) 367 return -ENOTSUP; 368 369 ret = cnxk_ipsec_xform_verify(&conf->ipsec, conf->crypto_xform); 370 if (ret) 371 return ret; 372 373 roc_cpt = &vf->cpt; 374 375 return cn10k_ipsec_outb_sa_create(roc_cpt, &qp->lf, &conf->ipsec, conf->crypto_xform, 376 (struct cn10k_sec_session *)sess); 377 } 378