1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Cavium, Inc 3 */ 4 #ifndef _OTX_CRYPTODEV_HW_ACCESS_H_ 5 #define _OTX_CRYPTODEV_HW_ACCESS_H_ 6 7 #include <stdbool.h> 8 9 #include <rte_branch_prediction.h> 10 #include <cryptodev_pmd.h> 11 #include <rte_cycles.h> 12 #include <rte_io.h> 13 #include <rte_memory.h> 14 #include <rte_prefetch.h> 15 16 #include "otx_cryptodev.h" 17 18 #include "cpt_common.h" 19 #include "cpt_hw_types.h" 20 #include "cpt_mcode_defines.h" 21 #include "cpt_pmd_logs.h" 22 23 #define CPT_INTR_POLL_INTERVAL_MS (50) 24 25 /* Default command queue length */ 26 #define DEFAULT_CMD_QLEN 2048 27 #define DEFAULT_CMD_QCHUNKS 2 28 29 /* Instruction memory benefits from being 1023, so introduce 30 * reserved entries so we can't overrun the instruction queue 31 */ 32 #define DEFAULT_CMD_QRSVD_SLOTS DEFAULT_CMD_QCHUNKS 33 #define DEFAULT_CMD_QCHUNK_SIZE \ 34 ((DEFAULT_CMD_QLEN - DEFAULT_CMD_QRSVD_SLOTS) / \ 35 DEFAULT_CMD_QCHUNKS) 36 37 #define CPT_CSR_REG_BASE(cpt) ((cpt)->reg_base) 38 39 /* Read hw register */ 40 #define CPT_READ_CSR(__hw_addr, __offset) \ 41 rte_read64_relaxed((uint8_t *)__hw_addr + __offset) 42 43 /* Write hw register */ 44 #define CPT_WRITE_CSR(__hw_addr, __offset, __val) \ 45 rte_write64_relaxed((__val), ((uint8_t *)__hw_addr + __offset)) 46 47 /* cpt instance */ 48 struct cpt_instance { 49 uint32_t queue_id; 50 uintptr_t rsvd; 51 struct rte_mempool *sess_mp; 52 struct cpt_qp_meta_info meta_info; 53 uint8_t ca_enabled; 54 }; 55 56 struct command_chunk { 57 /** 128-byte aligned real_vaddr */ 58 uint8_t *head; 59 /** 128-byte aligned real_dma_addr */ 60 phys_addr_t dma_addr; 61 }; 62 63 /** 64 * Command queue structure 65 */ 66 struct command_queue { 67 /** Command queue host write idx */ 68 uint32_t idx; 69 /** Command queue chunk */ 70 uint32_t cchunk; 71 /** Command queue head; instructions are inserted here */ 72 uint8_t *qhead; 73 /** Command chunk list head */ 74 struct command_chunk chead[DEFAULT_CMD_QCHUNKS]; 75 }; 76 77 /** 78 * CPT VF device structure 79 */ 80 struct __rte_cache_aligned cpt_vf { 81 /** CPT instance */ 82 struct cpt_instance instance; 83 /** Register start address */ 84 uint8_t *reg_base; 85 /** Command queue information */ 86 struct command_queue cqueue; 87 /** Pending queue information */ 88 struct pending_queue pqueue; 89 90 /** Below fields are accessed only in control path */ 91 92 /** Env specific pdev representing the pci dev */ 93 void *pdev; 94 /** Calculated queue size */ 95 uint32_t qsize; 96 /** Device index (0...CPT_MAX_VQ_NUM)*/ 97 uint8_t vfid; 98 /** VF type of cpt_vf_type_t (SE_TYPE(2) or AE_TYPE(1) */ 99 uint8_t vftype; 100 /** VF group (0 - 8) */ 101 uint8_t vfgrp; 102 /** Operating node: Bits (46:44) in BAR0 address */ 103 uint8_t node; 104 105 /** VF-PF mailbox communication */ 106 107 /** Flag if acked */ 108 bool pf_acked; 109 /** Flag if not acked */ 110 bool pf_nacked; 111 112 /** Device name */ 113 char dev_name[32]; 114 }; 115 116 /* 117 * CPT Registers map for 81xx 118 */ 119 120 /* VF registers */ 121 #define CPTX_VQX_CTL(a, b) (0x0000100ll + 0x1000000000ll * \ 122 ((a) & 0x0) + 0x100000ll * (b)) 123 #define CPTX_VQX_SADDR(a, b) (0x0000200ll + 0x1000000000ll * \ 124 ((a) & 0x0) + 0x100000ll * (b)) 125 #define CPTX_VQX_DONE_WAIT(a, b) (0x0000400ll + 0x1000000000ll * \ 126 ((a) & 0x0) + 0x100000ll * (b)) 127 #define CPTX_VQX_INPROG(a, b) (0x0000410ll + 0x1000000000ll * \ 128 ((a) & 0x0) + 0x100000ll * (b)) 129 #define CPTX_VQX_DONE(a, b) (0x0000420ll + 0x1000000000ll * \ 130 ((a) & 0x1) + 0x100000ll * (b)) 131 #define CPTX_VQX_DONE_ACK(a, b) (0x0000440ll + 0x1000000000ll * \ 132 ((a) & 0x1) + 0x100000ll * (b)) 133 #define CPTX_VQX_DONE_INT_W1S(a, b) (0x0000460ll + 0x1000000000ll * \ 134 ((a) & 0x1) + 0x100000ll * (b)) 135 #define CPTX_VQX_DONE_INT_W1C(a, b) (0x0000468ll + 0x1000000000ll * \ 136 ((a) & 0x1) + 0x100000ll * (b)) 137 #define CPTX_VQX_DONE_ENA_W1S(a, b) (0x0000470ll + 0x1000000000ll * \ 138 ((a) & 0x1) + 0x100000ll * (b)) 139 #define CPTX_VQX_DONE_ENA_W1C(a, b) (0x0000478ll + 0x1000000000ll * \ 140 ((a) & 0x1) + 0x100000ll * (b)) 141 #define CPTX_VQX_MISC_INT(a, b) (0x0000500ll + 0x1000000000ll * \ 142 ((a) & 0x1) + 0x100000ll * (b)) 143 #define CPTX_VQX_MISC_INT_W1S(a, b) (0x0000508ll + 0x1000000000ll * \ 144 ((a) & 0x1) + 0x100000ll * (b)) 145 #define CPTX_VQX_MISC_ENA_W1S(a, b) (0x0000510ll + 0x1000000000ll * \ 146 ((a) & 0x1) + 0x100000ll * (b)) 147 #define CPTX_VQX_MISC_ENA_W1C(a, b) (0x0000518ll + 0x1000000000ll * \ 148 ((a) & 0x1) + 0x100000ll * (b)) 149 #define CPTX_VQX_DOORBELL(a, b) (0x0000600ll + 0x1000000000ll * \ 150 ((a) & 0x1) + 0x100000ll * (b)) 151 #define CPTX_VFX_PF_MBOXX(a, b, c) (0x0001000ll + 0x1000000000ll * \ 152 ((a) & 0x1) + 0x100000ll * (b) + \ 153 8ll * ((c) & 0x1)) 154 155 /* VF HAL functions */ 156 157 void 158 otx_cpt_poll_misc(struct cpt_vf *cptvf); 159 160 int 161 otx_cpt_hw_init(struct cpt_vf *cptvf, void *pdev, void *reg_base, char *name); 162 163 int 164 otx_cpt_deinit_device(void *dev); 165 166 int 167 otx_cpt_get_resource(const struct rte_cryptodev *dev, uint8_t group, 168 struct cpt_instance **instance, uint16_t qp_id); 169 170 int 171 otx_cpt_put_resource(struct cpt_instance *instance); 172 173 int 174 otx_cpt_start_device(void *cptvf); 175 176 void 177 otx_cpt_stop_device(void *cptvf); 178 179 /* Write to VQX_DOORBELL register 180 */ 181 static __rte_always_inline void 182 otx_cpt_write_vq_doorbell(struct cpt_vf *cptvf, uint32_t val) 183 { 184 cptx_vqx_doorbell_t vqx_dbell; 185 186 vqx_dbell.u = 0; 187 vqx_dbell.s.dbell_cnt = val * 8; /* Num of Instructions * 8 words */ 188 CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf), 189 CPTX_VQX_DOORBELL(0, 0), vqx_dbell.u); 190 } 191 192 static __rte_always_inline uint32_t 193 otx_cpt_read_vq_doorbell(struct cpt_vf *cptvf) 194 { 195 cptx_vqx_doorbell_t vqx_dbell; 196 197 vqx_dbell.u = CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf), 198 CPTX_VQX_DOORBELL(0, 0)); 199 return vqx_dbell.s.dbell_cnt; 200 } 201 202 static __rte_always_inline void 203 otx_cpt_ring_dbell(struct cpt_instance *instance, uint16_t count) 204 { 205 struct cpt_vf *cptvf = (struct cpt_vf *)instance; 206 /* Memory barrier to flush pending writes */ 207 rte_smp_wmb(); 208 otx_cpt_write_vq_doorbell(cptvf, count); 209 } 210 211 static __rte_always_inline void * 212 get_cpt_inst(struct command_queue *cqueue) 213 { 214 CPT_LOG_DP_DEBUG("CPT queue idx %u", cqueue->idx); 215 return &cqueue->qhead[cqueue->idx * CPT_INST_SIZE]; 216 } 217 218 static __rte_always_inline void 219 fill_cpt_inst(struct cpt_instance *instance, void *req, uint64_t ucmd_w3) 220 { 221 struct command_queue *cqueue; 222 cpt_inst_s_t *cpt_ist_p; 223 struct cpt_vf *cptvf = (struct cpt_vf *)instance; 224 struct cpt_request_info *user_req = (struct cpt_request_info *)req; 225 cqueue = &cptvf->cqueue; 226 cpt_ist_p = get_cpt_inst(cqueue); 227 rte_prefetch_non_temporal(cpt_ist_p); 228 229 /* EI0, EI1, EI2, EI3 are already prepared */ 230 /* HW W0 */ 231 cpt_ist_p->u[0] = 0; 232 /* HW W1 */ 233 cpt_ist_p->s8x.res_addr = user_req->comp_baddr; 234 /* HW W2 */ 235 cpt_ist_p->u[2] = 0; 236 /* HW W3 */ 237 cpt_ist_p->s8x.wq_ptr = 0; 238 239 /* MC EI0 */ 240 cpt_ist_p->s8x.ei0 = user_req->ist.ei0; 241 /* MC EI1 */ 242 cpt_ist_p->s8x.ei1 = user_req->ist.ei1; 243 /* MC EI2 */ 244 cpt_ist_p->s8x.ei2 = user_req->ist.ei2; 245 /* MC EI3 */ 246 cpt_ist_p->s8x.ei3 = ucmd_w3; 247 } 248 249 static __rte_always_inline void 250 mark_cpt_inst(struct cpt_instance *instance) 251 { 252 struct cpt_vf *cptvf = (struct cpt_vf *)instance; 253 struct command_queue *queue = &cptvf->cqueue; 254 if (unlikely(++queue->idx >= DEFAULT_CMD_QCHUNK_SIZE)) { 255 uint32_t cchunk = queue->cchunk; 256 MOD_INC(cchunk, DEFAULT_CMD_QCHUNKS); 257 queue->qhead = queue->chead[cchunk].head; 258 queue->idx = 0; 259 queue->cchunk = cchunk; 260 } 261 } 262 263 static __rte_always_inline uint8_t 264 check_nb_command_id(struct cpt_request_info *user_req, 265 struct cpt_instance *instance) 266 { 267 uint8_t ret = ERR_REQ_PENDING; 268 struct cpt_vf *cptvf = (struct cpt_vf *)instance; 269 volatile cpt_res_s_t *cptres; 270 271 cptres = (volatile cpt_res_s_t *)user_req->completion_addr; 272 273 if (unlikely(cptres->s8x.compcode == CPT_8X_COMP_E_NOTDONE)) { 274 /* 275 * Wait for some time for this command to get completed 276 * before timing out 277 */ 278 if (rte_get_timer_cycles() < user_req->time_out) 279 return ret; 280 /* 281 * TODO: See if alternate caddr can be used to not loop 282 * longer than needed. 283 */ 284 if ((cptres->s8x.compcode == CPT_8X_COMP_E_NOTDONE) && 285 (user_req->extra_time < TIME_IN_RESET_COUNT)) { 286 user_req->extra_time++; 287 return ret; 288 } 289 290 if (cptres->s8x.compcode != CPT_8X_COMP_E_NOTDONE) 291 goto complete; 292 293 ret = ERR_REQ_TIMEOUT; 294 CPT_LOG_DP_ERR("Request %p timedout", user_req); 295 otx_cpt_poll_misc(cptvf); 296 goto exit; 297 } 298 299 complete: 300 if (likely(cptres->s8x.compcode == CPT_8X_COMP_E_GOOD)) { 301 ret = 0; /* success */ 302 if (unlikely((uint8_t)*user_req->alternate_caddr)) { 303 ret = (uint8_t)*user_req->alternate_caddr; 304 CPT_LOG_DP_ERR("Request %p : failed with microcode" 305 " error, MC completion code : 0x%x", user_req, 306 ret); 307 } 308 CPT_LOG_DP_DEBUG("MC status %.8x", 309 *((volatile uint32_t *)user_req->alternate_caddr)); 310 CPT_LOG_DP_DEBUG("HW status %.8x", 311 *((volatile uint32_t *)user_req->completion_addr)); 312 } else if ((cptres->s8x.compcode == CPT_8X_COMP_E_SWERR) || 313 (cptres->s8x.compcode == CPT_8X_COMP_E_FAULT)) { 314 ret = (uint8_t)*user_req->alternate_caddr; 315 if (!ret) 316 ret = ERR_BAD_ALT_CCODE; 317 CPT_LOG_DP_DEBUG("Request %p : failed with %s : err code :%x", 318 user_req, 319 (cptres->s8x.compcode == CPT_8X_COMP_E_FAULT) ? 320 "DMA Fault" : "Software error", ret); 321 } else { 322 CPT_LOG_DP_ERR("Request %p : unexpected completion code %d", 323 user_req, cptres->s8x.compcode); 324 ret = (uint8_t)*user_req->alternate_caddr; 325 } 326 327 exit: 328 return ret; 329 } 330 331 #endif /* _OTX_CRYPTODEV_HW_ACCESS_H_ */ 332