1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Cavium, Inc 3 */ 4 #ifndef _OTX_CRYPTODEV_HW_ACCESS_H_ 5 #define _OTX_CRYPTODEV_HW_ACCESS_H_ 6 7 #include <stdbool.h> 8 9 #include <rte_branch_prediction.h> 10 #include <rte_cryptodev.h> 11 #include <rte_cycles.h> 12 #include <rte_io.h> 13 #include <rte_memory.h> 14 #include <rte_prefetch.h> 15 16 #include "otx_cryptodev.h" 17 18 #include "cpt_common.h" 19 #include "cpt_hw_types.h" 20 #include "cpt_mcode_defines.h" 21 #include "cpt_pmd_logs.h" 22 23 #define CPT_INTR_POLL_INTERVAL_MS (50) 24 25 /* Default command queue length */ 26 #define DEFAULT_CMD_QLEN 2048 27 #define DEFAULT_CMD_QCHUNKS 2 28 29 /* Instruction memory benefits from being 1023, so introduce 30 * reserved entries so we can't overrun the instruction queue 31 */ 32 #define DEFAULT_CMD_QRSVD_SLOTS DEFAULT_CMD_QCHUNKS 33 #define DEFAULT_CMD_QCHUNK_SIZE \ 34 ((DEFAULT_CMD_QLEN - DEFAULT_CMD_QRSVD_SLOTS) / \ 35 DEFAULT_CMD_QCHUNKS) 36 37 #define CPT_CSR_REG_BASE(cpt) ((cpt)->reg_base) 38 39 /* Read hw register */ 40 #define CPT_READ_CSR(__hw_addr, __offset) \ 41 rte_read64_relaxed((uint8_t *)__hw_addr + __offset) 42 43 /* Write hw register */ 44 #define CPT_WRITE_CSR(__hw_addr, __offset, __val) \ 45 rte_write64_relaxed((__val), ((uint8_t *)__hw_addr + __offset)) 46 47 /* cpt instance */ 48 struct cpt_instance { 49 uint32_t queue_id; 50 uintptr_t rsvd; 51 struct rte_mempool *sess_mp; 52 struct rte_mempool *sess_mp_priv; 53 struct cpt_qp_meta_info meta_info; 54 uint8_t ca_enabled; 55 }; 56 57 struct command_chunk { 58 /** 128-byte aligned real_vaddr */ 59 uint8_t *head; 60 /** 128-byte aligned real_dma_addr */ 61 phys_addr_t dma_addr; 62 }; 63 64 /** 65 * Command queue structure 66 */ 67 struct command_queue { 68 /** Command queue host write idx */ 69 uint32_t idx; 70 /** Command queue chunk */ 71 uint32_t cchunk; 72 /** Command queue head; instructions are inserted here */ 73 uint8_t *qhead; 74 /** Command chunk list head */ 75 struct command_chunk chead[DEFAULT_CMD_QCHUNKS]; 76 }; 77 78 /** 79 * CPT VF device structure 80 */ 81 struct cpt_vf { 82 /** CPT instance */ 83 struct cpt_instance instance; 84 /** Register start address */ 85 uint8_t *reg_base; 86 /** Command queue information */ 87 struct command_queue cqueue; 88 /** Pending queue information */ 89 struct pending_queue pqueue; 90 91 /** Below fields are accessed only in control path */ 92 93 /** Env specific pdev representing the pci dev */ 94 void *pdev; 95 /** Calculated queue size */ 96 uint32_t qsize; 97 /** Device index (0...CPT_MAX_VQ_NUM)*/ 98 uint8_t vfid; 99 /** VF type of cpt_vf_type_t (SE_TYPE(2) or AE_TYPE(1) */ 100 uint8_t vftype; 101 /** VF group (0 - 8) */ 102 uint8_t vfgrp; 103 /** Operating node: Bits (46:44) in BAR0 address */ 104 uint8_t node; 105 106 /** VF-PF mailbox communication */ 107 108 /** Flag if acked */ 109 bool pf_acked; 110 /** Flag if not acked */ 111 bool pf_nacked; 112 113 /** Device name */ 114 char dev_name[32]; 115 } __rte_cache_aligned; 116 117 /* 118 * CPT Registers map for 81xx 119 */ 120 121 /* VF registers */ 122 #define CPTX_VQX_CTL(a, b) (0x0000100ll + 0x1000000000ll * \ 123 ((a) & 0x0) + 0x100000ll * (b)) 124 #define CPTX_VQX_SADDR(a, b) (0x0000200ll + 0x1000000000ll * \ 125 ((a) & 0x0) + 0x100000ll * (b)) 126 #define CPTX_VQX_DONE_WAIT(a, b) (0x0000400ll + 0x1000000000ll * \ 127 ((a) & 0x0) + 0x100000ll * (b)) 128 #define CPTX_VQX_INPROG(a, b) (0x0000410ll + 0x1000000000ll * \ 129 ((a) & 0x0) + 0x100000ll * (b)) 130 #define CPTX_VQX_DONE(a, b) (0x0000420ll + 0x1000000000ll * \ 131 ((a) & 0x1) + 0x100000ll * (b)) 132 #define CPTX_VQX_DONE_ACK(a, b) (0x0000440ll + 0x1000000000ll * \ 133 ((a) & 0x1) + 0x100000ll * (b)) 134 #define CPTX_VQX_DONE_INT_W1S(a, b) (0x0000460ll + 0x1000000000ll * \ 135 ((a) & 0x1) + 0x100000ll * (b)) 136 #define CPTX_VQX_DONE_INT_W1C(a, b) (0x0000468ll + 0x1000000000ll * \ 137 ((a) & 0x1) + 0x100000ll * (b)) 138 #define CPTX_VQX_DONE_ENA_W1S(a, b) (0x0000470ll + 0x1000000000ll * \ 139 ((a) & 0x1) + 0x100000ll * (b)) 140 #define CPTX_VQX_DONE_ENA_W1C(a, b) (0x0000478ll + 0x1000000000ll * \ 141 ((a) & 0x1) + 0x100000ll * (b)) 142 #define CPTX_VQX_MISC_INT(a, b) (0x0000500ll + 0x1000000000ll * \ 143 ((a) & 0x1) + 0x100000ll * (b)) 144 #define CPTX_VQX_MISC_INT_W1S(a, b) (0x0000508ll + 0x1000000000ll * \ 145 ((a) & 0x1) + 0x100000ll * (b)) 146 #define CPTX_VQX_MISC_ENA_W1S(a, b) (0x0000510ll + 0x1000000000ll * \ 147 ((a) & 0x1) + 0x100000ll * (b)) 148 #define CPTX_VQX_MISC_ENA_W1C(a, b) (0x0000518ll + 0x1000000000ll * \ 149 ((a) & 0x1) + 0x100000ll * (b)) 150 #define CPTX_VQX_DOORBELL(a, b) (0x0000600ll + 0x1000000000ll * \ 151 ((a) & 0x1) + 0x100000ll * (b)) 152 #define CPTX_VFX_PF_MBOXX(a, b, c) (0x0001000ll + 0x1000000000ll * \ 153 ((a) & 0x1) + 0x100000ll * (b) + \ 154 8ll * ((c) & 0x1)) 155 156 /* VF HAL functions */ 157 158 void 159 otx_cpt_poll_misc(struct cpt_vf *cptvf); 160 161 int 162 otx_cpt_hw_init(struct cpt_vf *cptvf, void *pdev, void *reg_base, char *name); 163 164 int 165 otx_cpt_deinit_device(void *dev); 166 167 int 168 otx_cpt_get_resource(const struct rte_cryptodev *dev, uint8_t group, 169 struct cpt_instance **instance, uint16_t qp_id); 170 171 int 172 otx_cpt_put_resource(struct cpt_instance *instance); 173 174 int 175 otx_cpt_start_device(void *cptvf); 176 177 void 178 otx_cpt_stop_device(void *cptvf); 179 180 /* Write to VQX_DOORBELL register 181 */ 182 static __rte_always_inline void 183 otx_cpt_write_vq_doorbell(struct cpt_vf *cptvf, uint32_t val) 184 { 185 cptx_vqx_doorbell_t vqx_dbell; 186 187 vqx_dbell.u = 0; 188 vqx_dbell.s.dbell_cnt = val * 8; /* Num of Instructions * 8 words */ 189 CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf), 190 CPTX_VQX_DOORBELL(0, 0), vqx_dbell.u); 191 } 192 193 static __rte_always_inline uint32_t 194 otx_cpt_read_vq_doorbell(struct cpt_vf *cptvf) 195 { 196 cptx_vqx_doorbell_t vqx_dbell; 197 198 vqx_dbell.u = CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf), 199 CPTX_VQX_DOORBELL(0, 0)); 200 return vqx_dbell.s.dbell_cnt; 201 } 202 203 static __rte_always_inline void 204 otx_cpt_ring_dbell(struct cpt_instance *instance, uint16_t count) 205 { 206 struct cpt_vf *cptvf = (struct cpt_vf *)instance; 207 /* Memory barrier to flush pending writes */ 208 rte_smp_wmb(); 209 otx_cpt_write_vq_doorbell(cptvf, count); 210 } 211 212 static __rte_always_inline void * 213 get_cpt_inst(struct command_queue *cqueue) 214 { 215 CPT_LOG_DP_DEBUG("CPT queue idx %u\n", cqueue->idx); 216 return &cqueue->qhead[cqueue->idx * CPT_INST_SIZE]; 217 } 218 219 static __rte_always_inline void 220 fill_cpt_inst(struct cpt_instance *instance, void *req, uint64_t ucmd_w3) 221 { 222 struct command_queue *cqueue; 223 cpt_inst_s_t *cpt_ist_p; 224 struct cpt_vf *cptvf = (struct cpt_vf *)instance; 225 struct cpt_request_info *user_req = (struct cpt_request_info *)req; 226 cqueue = &cptvf->cqueue; 227 cpt_ist_p = get_cpt_inst(cqueue); 228 rte_prefetch_non_temporal(cpt_ist_p); 229 230 /* EI0, EI1, EI2, EI3 are already prepared */ 231 /* HW W0 */ 232 cpt_ist_p->u[0] = 0; 233 /* HW W1 */ 234 cpt_ist_p->s8x.res_addr = user_req->comp_baddr; 235 /* HW W2 */ 236 cpt_ist_p->u[2] = 0; 237 /* HW W3 */ 238 cpt_ist_p->s8x.wq_ptr = 0; 239 240 /* MC EI0 */ 241 cpt_ist_p->s8x.ei0 = user_req->ist.ei0; 242 /* MC EI1 */ 243 cpt_ist_p->s8x.ei1 = user_req->ist.ei1; 244 /* MC EI2 */ 245 cpt_ist_p->s8x.ei2 = user_req->ist.ei2; 246 /* MC EI3 */ 247 cpt_ist_p->s8x.ei3 = ucmd_w3; 248 } 249 250 static __rte_always_inline void 251 mark_cpt_inst(struct cpt_instance *instance) 252 { 253 struct cpt_vf *cptvf = (struct cpt_vf *)instance; 254 struct command_queue *queue = &cptvf->cqueue; 255 if (unlikely(++queue->idx >= DEFAULT_CMD_QCHUNK_SIZE)) { 256 uint32_t cchunk = queue->cchunk; 257 MOD_INC(cchunk, DEFAULT_CMD_QCHUNKS); 258 queue->qhead = queue->chead[cchunk].head; 259 queue->idx = 0; 260 queue->cchunk = cchunk; 261 } 262 } 263 264 static __rte_always_inline uint8_t 265 check_nb_command_id(struct cpt_request_info *user_req, 266 struct cpt_instance *instance) 267 { 268 uint8_t ret = ERR_REQ_PENDING; 269 struct cpt_vf *cptvf = (struct cpt_vf *)instance; 270 volatile cpt_res_s_t *cptres; 271 272 cptres = (volatile cpt_res_s_t *)user_req->completion_addr; 273 274 if (unlikely(cptres->s8x.compcode == CPT_8X_COMP_E_NOTDONE)) { 275 /* 276 * Wait for some time for this command to get completed 277 * before timing out 278 */ 279 if (rte_get_timer_cycles() < user_req->time_out) 280 return ret; 281 /* 282 * TODO: See if alternate caddr can be used to not loop 283 * longer than needed. 284 */ 285 if ((cptres->s8x.compcode == CPT_8X_COMP_E_NOTDONE) && 286 (user_req->extra_time < TIME_IN_RESET_COUNT)) { 287 user_req->extra_time++; 288 return ret; 289 } 290 291 if (cptres->s8x.compcode != CPT_8X_COMP_E_NOTDONE) 292 goto complete; 293 294 ret = ERR_REQ_TIMEOUT; 295 CPT_LOG_DP_ERR("Request %p timedout", user_req); 296 otx_cpt_poll_misc(cptvf); 297 goto exit; 298 } 299 300 complete: 301 if (likely(cptres->s8x.compcode == CPT_8X_COMP_E_GOOD)) { 302 ret = 0; /* success */ 303 if (unlikely((uint8_t)*user_req->alternate_caddr)) { 304 ret = (uint8_t)*user_req->alternate_caddr; 305 CPT_LOG_DP_ERR("Request %p : failed with microcode" 306 " error, MC completion code : 0x%x", user_req, 307 ret); 308 } 309 CPT_LOG_DP_DEBUG("MC status %.8x\n", 310 *((volatile uint32_t *)user_req->alternate_caddr)); 311 CPT_LOG_DP_DEBUG("HW status %.8x\n", 312 *((volatile uint32_t *)user_req->completion_addr)); 313 } else if ((cptres->s8x.compcode == CPT_8X_COMP_E_SWERR) || 314 (cptres->s8x.compcode == CPT_8X_COMP_E_FAULT)) { 315 ret = (uint8_t)*user_req->alternate_caddr; 316 if (!ret) 317 ret = ERR_BAD_ALT_CCODE; 318 CPT_LOG_DP_DEBUG("Request %p : failed with %s : err code :%x", 319 user_req, 320 (cptres->s8x.compcode == CPT_8X_COMP_E_FAULT) ? 321 "DMA Fault" : "Software error", ret); 322 } else { 323 CPT_LOG_DP_ERR("Request %p : unexpected completion code %d", 324 user_req, cptres->s8x.compcode); 325 ret = (uint8_t)*user_req->alternate_caddr; 326 } 327 328 exit: 329 return ret; 330 } 331 332 #endif /* _OTX_CRYPTODEV_HW_ACCESS_H_ */ 333