1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2021-2024 Advanced Micro Devices, Inc. 3 */ 4 5 #ifndef _IONIC_CRYPTO_H_ 6 #define _IONIC_CRYPTO_H_ 7 8 #include <stdint.h> 9 #include <stdbool.h> 10 #include <inttypes.h> 11 12 #include <rte_common.h> 13 #include <rte_dev.h> 14 #include <rte_cryptodev.h> 15 #include <cryptodev_pmd.h> 16 #include <rte_log.h> 17 #include <rte_bitmap.h> 18 19 #include "ionic_common.h" 20 #include "ionic_crypto_if.h" 21 #include "ionic_regs.h" 22 23 /* Devargs */ 24 /* NONE */ 25 26 #define IOCPT_MAX_RING_DESC 32768 27 #define IOCPT_MIN_RING_DESC 16 28 #define IOCPT_ADMINQ_LENGTH 16 /* must be a power of two */ 29 30 #define IOCPT_CRYPTOQ_WAIT 10 /* 1s */ 31 32 extern int iocpt_logtype; 33 #define RTE_LOGTYPE_IOCPT iocpt_logtype 34 35 #define IOCPT_PRINT(level, ...) \ 36 RTE_LOG_LINE_PREFIX(level, IOCPT, "%s(): ", __func__, __VA_ARGS__) 37 38 #define IOCPT_PRINT_CALL() IOCPT_PRINT(DEBUG, " >>") 39 40 const struct rte_cryptodev_capabilities *iocpt_get_caps(uint64_t flags); 41 42 static inline void iocpt_struct_size_checks(void) 43 { 44 RTE_BUILD_BUG_ON(sizeof(struct ionic_doorbell) != 8); 45 RTE_BUILD_BUG_ON(sizeof(struct ionic_intr) != 32); 46 RTE_BUILD_BUG_ON(sizeof(struct ionic_intr_status) != 8); 47 48 RTE_BUILD_BUG_ON(sizeof(union iocpt_dev_regs) != 4096); 49 RTE_BUILD_BUG_ON(sizeof(union iocpt_dev_info_regs) != 2048); 50 RTE_BUILD_BUG_ON(sizeof(union iocpt_dev_cmd_regs) != 2048); 51 52 RTE_BUILD_BUG_ON(sizeof(struct iocpt_admin_cmd) != 64); 53 RTE_BUILD_BUG_ON(sizeof(struct iocpt_admin_comp) != 16); 54 RTE_BUILD_BUG_ON(sizeof(struct iocpt_nop_cmd) != 64); 55 RTE_BUILD_BUG_ON(sizeof(struct iocpt_nop_comp) != 16); 56 57 /* Device commands */ 58 RTE_BUILD_BUG_ON(sizeof(struct iocpt_dev_identify_cmd) != 64); 59 RTE_BUILD_BUG_ON(sizeof(struct iocpt_dev_identify_comp) != 16); 60 RTE_BUILD_BUG_ON(sizeof(struct iocpt_dev_reset_cmd) != 64); 61 RTE_BUILD_BUG_ON(sizeof(struct iocpt_dev_reset_comp) != 16); 62 63 /* LIF commands */ 64 RTE_BUILD_BUG_ON(sizeof(struct iocpt_lif_identify_cmd) != 64); 65 RTE_BUILD_BUG_ON(sizeof(struct iocpt_lif_identify_comp) != 16); 66 RTE_BUILD_BUG_ON(sizeof(struct iocpt_lif_init_cmd) != 64); 67 RTE_BUILD_BUG_ON(sizeof(struct iocpt_lif_init_comp) != 16); 68 RTE_BUILD_BUG_ON(sizeof(struct iocpt_lif_reset_cmd) != 64); 69 RTE_BUILD_BUG_ON(sizeof(struct iocpt_lif_getattr_cmd) != 64); 70 RTE_BUILD_BUG_ON(sizeof(struct iocpt_lif_getattr_comp) != 16); 71 RTE_BUILD_BUG_ON(sizeof(struct iocpt_lif_setattr_cmd) != 64); 72 RTE_BUILD_BUG_ON(sizeof(struct iocpt_lif_setattr_comp) != 16); 73 74 /* Queue commands */ 75 RTE_BUILD_BUG_ON(sizeof(struct iocpt_q_identify_cmd) != 64); 76 RTE_BUILD_BUG_ON(sizeof(struct iocpt_q_identify_comp) != 16); 77 RTE_BUILD_BUG_ON(sizeof(struct iocpt_q_init_cmd) != 64); 78 RTE_BUILD_BUG_ON(sizeof(struct iocpt_q_init_comp) != 16); 79 RTE_BUILD_BUG_ON(sizeof(struct iocpt_q_control_cmd) != 64); 80 81 /* Crypto */ 82 RTE_BUILD_BUG_ON(sizeof(struct iocpt_crypto_desc) != 32); 83 RTE_BUILD_BUG_ON(sizeof(struct iocpt_crypto_sg_desc) != 256); 84 RTE_BUILD_BUG_ON(sizeof(struct iocpt_crypto_comp) != 16); 85 } 86 87 struct iocpt_dev_bars { 88 struct ionic_dev_bar bar[IONIC_BARS_MAX]; 89 uint32_t num_bars; 90 }; 91 92 /* Queue watchdog */ 93 #define IOCPT_Q_WDOG_SESS_IDX 0 94 #define IOCPT_Q_WDOG_KEY_LEN 16 95 #define IOCPT_Q_WDOG_IV_LEN 12 96 #define IOCPT_Q_WDOG_PLD_LEN 4 97 #define IOCPT_Q_WDOG_TAG_LEN 16 98 #define IOCPT_Q_WDOG_OP_TYPE RTE_CRYPTO_OP_TYPE_UNDEFINED 99 100 struct iocpt_qtype_info { 101 uint8_t version; 102 uint8_t supported; 103 uint64_t features; 104 uint16_t desc_sz; 105 uint16_t comp_sz; 106 uint16_t sg_desc_sz; 107 uint16_t max_sg_elems; 108 uint16_t sg_desc_stride; 109 }; 110 111 #define IOCPT_Q_F_INITED BIT(0) 112 #define IOCPT_Q_F_DEFERRED BIT(1) 113 #define IOCPT_Q_F_SG BIT(2) 114 115 #define Q_NEXT_TO_POST(_q, _n) (((_q)->head_idx + (_n)) & ((_q)->size_mask)) 116 #define Q_NEXT_TO_SRVC(_q, _n) (((_q)->tail_idx + (_n)) & ((_q)->size_mask)) 117 118 #define IOCPT_INFO_SZ(_q) ((_q)->num_segs * sizeof(void *)) 119 #define IOCPT_INFO_IDX(_q, _i) ((_i) * (_q)->num_segs) 120 #define IOCPT_INFO_PTR(_q, _i) (&(_q)->info[IOCPT_INFO_IDX((_q), _i)]) 121 122 struct iocpt_queue { 123 uint16_t num_descs; 124 uint16_t num_segs; 125 uint16_t head_idx; 126 uint16_t tail_idx; 127 uint16_t size_mask; 128 uint8_t type; 129 uint8_t hw_type; 130 void *base; 131 void *sg_base; 132 struct ionic_doorbell __iomem *db; 133 void **info; 134 135 uint32_t index; 136 uint32_t hw_index; 137 rte_iova_t base_pa; 138 rte_iova_t sg_base_pa; 139 }; 140 141 struct iocpt_cq { 142 uint16_t tail_idx; 143 uint16_t num_descs; 144 uint16_t size_mask; 145 bool done_color; 146 void *base; 147 rte_iova_t base_pa; 148 }; 149 150 #define IOCPT_COMMON_FIELDS \ 151 struct iocpt_queue q; \ 152 struct iocpt_cq cq; \ 153 struct iocpt_dev *dev; \ 154 const struct rte_memzone *base_z; \ 155 void *base; \ 156 rte_iova_t base_pa 157 158 struct iocpt_common_q { 159 IOCPT_COMMON_FIELDS; 160 }; 161 162 struct iocpt_admin_q { 163 IOCPT_COMMON_FIELDS; 164 165 uint16_t flags; 166 }; 167 168 struct iocpt_crypto_q { 169 /* cacheline0, cacheline1 */ 170 IOCPT_COMMON_FIELDS; 171 172 /* cacheline2 */ 173 uint64_t last_wdog_cycles; 174 uint16_t flags; 175 176 /* cacheline3 */ 177 struct rte_cryptodev_stats stats; 178 179 uint64_t enqueued_wdogs; 180 uint64_t dequeued_wdogs; 181 uint8_t wdog_iv[IOCPT_Q_WDOG_IV_LEN]; 182 uint8_t wdog_pld[IOCPT_Q_WDOG_PLD_LEN]; 183 uint8_t wdog_tag[IOCPT_Q_WDOG_TAG_LEN]; 184 }; 185 186 #define IOCPT_S_F_INITED BIT(0) 187 188 struct iocpt_session_priv { 189 struct iocpt_dev *dev; 190 191 uint32_t index; 192 193 uint16_t iv_offset; 194 uint16_t iv_length; 195 uint16_t digest_length; 196 uint16_t aad_length; 197 198 uint8_t flags; 199 uint8_t op; 200 uint8_t type; 201 202 uint16_t key_len; 203 uint8_t key[IOCPT_SESS_KEY_LEN_MAX_SYMM]; 204 }; 205 206 static inline uint32_t 207 iocpt_session_size(void) 208 { 209 return sizeof(struct iocpt_session_priv); 210 } 211 212 #define IOCPT_DEV_F_INITED BIT(0) 213 #define IOCPT_DEV_F_UP BIT(1) 214 #define IOCPT_DEV_F_FW_RESET BIT(2) 215 216 /* Combined dev / LIF object */ 217 struct iocpt_dev { 218 const char *name; 219 char fw_version[IOCPT_FWVERS_BUFLEN]; 220 struct iocpt_dev_bars bars; 221 struct iocpt_identity ident; 222 223 const struct iocpt_dev_intf *intf; 224 void *bus_dev; 225 struct rte_cryptodev *crypto_dev; 226 227 union iocpt_dev_info_regs __iomem *dev_info; 228 union iocpt_dev_cmd_regs __iomem *dev_cmd; 229 230 struct ionic_doorbell __iomem *db_pages; 231 struct ionic_intr __iomem *intr_ctrl; 232 233 uint32_t max_qps; 234 uint32_t max_sessions; 235 uint16_t state; 236 uint8_t driver_id; 237 uint8_t socket_id; 238 239 rte_spinlock_t adminq_lock; 240 rte_spinlock_t adminq_service_lock; 241 242 struct iocpt_admin_q *adminq; 243 struct iocpt_crypto_q **cryptoqs; 244 245 struct rte_bitmap *sess_bm; /* SET bit indicates index is free */ 246 247 uint64_t features; 248 uint32_t hw_features; 249 250 uint32_t info_sz; 251 struct iocpt_lif_info *info; 252 rte_iova_t info_pa; 253 const struct rte_memzone *info_z; 254 255 struct iocpt_qtype_info qtype_info[IOCPT_QTYPE_MAX]; 256 uint8_t qtype_ver[IOCPT_QTYPE_MAX]; 257 258 struct rte_cryptodev_stats stats_base; 259 }; 260 261 struct iocpt_dev_intf { 262 int (*setup_bars)(struct iocpt_dev *dev); 263 void (*unmap_bars)(struct iocpt_dev *dev); 264 }; 265 266 static inline int 267 iocpt_setup_bars(struct iocpt_dev *dev) 268 { 269 if (dev->intf->setup_bars == NULL) 270 return -EINVAL; 271 272 return (*dev->intf->setup_bars)(dev); 273 } 274 275 /** iocpt_admin_ctx - Admin command context. 276 * @pending_work: Flag that indicates a completion. 277 * @cmd: Admin command (64B) to be copied to the queue. 278 * @comp: Admin completion (16B) copied from the queue. 279 */ 280 struct iocpt_admin_ctx { 281 bool pending_work; 282 union iocpt_adminq_cmd cmd; 283 union iocpt_adminq_comp comp; 284 }; 285 286 int iocpt_probe(void *bus_dev, struct rte_device *rte_dev, 287 struct iocpt_dev_bars *bars, const struct iocpt_dev_intf *intf, 288 uint8_t driver_id, uint8_t socket_id); 289 int iocpt_remove(struct rte_device *rte_dev); 290 291 void iocpt_configure(struct iocpt_dev *dev); 292 int iocpt_assign_ops(struct rte_cryptodev *cdev); 293 int iocpt_start(struct iocpt_dev *dev); 294 void iocpt_stop(struct iocpt_dev *dev); 295 void iocpt_deinit(struct iocpt_dev *dev); 296 297 int iocpt_dev_identify(struct iocpt_dev *dev); 298 int iocpt_dev_init(struct iocpt_dev *dev, rte_iova_t info_pa); 299 int iocpt_dev_adminq_init(struct iocpt_dev *dev); 300 void iocpt_dev_reset(struct iocpt_dev *dev); 301 302 int iocpt_adminq_post_wait(struct iocpt_dev *dev, struct iocpt_admin_ctx *ctx); 303 304 int iocpt_cryptoq_alloc(struct iocpt_dev *dev, uint32_t socket_id, 305 uint32_t index, uint16_t ndescs); 306 void iocpt_cryptoq_free(struct iocpt_crypto_q *cptq); 307 308 int iocpt_session_init(struct iocpt_session_priv *priv); 309 int iocpt_session_update(struct iocpt_session_priv *priv); 310 void iocpt_session_deinit(struct iocpt_session_priv *priv); 311 312 struct ionic_doorbell __iomem *iocpt_db_map(struct iocpt_dev *dev, 313 struct iocpt_queue *q); 314 315 typedef bool (*iocpt_cq_cb)(struct iocpt_cq *cq, uint16_t cq_desc_index, 316 void *cb_arg); 317 uint32_t iocpt_cq_service(struct iocpt_cq *cq, uint32_t work_to_do, 318 iocpt_cq_cb cb, void *cb_arg); 319 320 void iocpt_get_stats(const struct iocpt_dev *dev, 321 struct rte_cryptodev_stats *stats); 322 void iocpt_reset_stats(struct iocpt_dev *dev); 323 324 static inline uint16_t 325 iocpt_q_space_avail(struct iocpt_queue *q) 326 { 327 uint16_t avail = q->tail_idx; 328 329 if (q->head_idx >= avail) 330 avail += q->num_descs - q->head_idx - 1; 331 else 332 avail -= q->head_idx + 1; 333 334 return avail; 335 } 336 337 static inline void 338 iocpt_q_flush(struct iocpt_queue *q) 339 { 340 uint64_t val = IONIC_DBELL_QID(q->hw_index) | q->head_idx; 341 342 #if defined(RTE_LIBRTE_IONIC_PMD_BARRIER_ERRATA) 343 /* On some devices the standard 'dmb' barrier is insufficient */ 344 asm volatile("dsb st" : : : "memory"); 345 rte_write64_relaxed(rte_cpu_to_le_64(val), q->db); 346 #else 347 rte_write64(rte_cpu_to_le_64(val), q->db); 348 #endif 349 } 350 351 static inline bool 352 iocpt_is_embedded(void) 353 { 354 #if defined(RTE_LIBRTE_IONIC_PMD_EMBEDDED) 355 return true; 356 #else 357 return false; 358 #endif 359 } 360 361 #endif /* _IONIC_CRYPTO_H_ */ 362