1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2021-2024 Advanced Micro Devices, Inc.
3 */
4
5 #ifndef _IONIC_CRYPTO_H_
6 #define _IONIC_CRYPTO_H_
7
8 #include <stdint.h>
9 #include <stdbool.h>
10 #include <inttypes.h>
11
12 #include <rte_common.h>
13 #include <rte_dev.h>
14 #include <rte_cryptodev.h>
15 #include <cryptodev_pmd.h>
16 #include <rte_log.h>
17 #include <rte_bitmap.h>
18
19 #include "ionic_common.h"
20 #include "ionic_crypto_if.h"
21 #include "ionic_regs.h"
22
23 /* Devargs */
24 /* NONE */
25
26 #define IOCPT_MAX_RING_DESC 32768
27 #define IOCPT_MIN_RING_DESC 16
28 #define IOCPT_ADMINQ_LENGTH 16 /* must be a power of two */
29
30 #define IOCPT_CRYPTOQ_WAIT 10 /* 1s */
31
32 extern int iocpt_logtype;
33 #define RTE_LOGTYPE_IOCPT iocpt_logtype
34
35 #define IOCPT_PRINT(level, ...) \
36 RTE_LOG_LINE_PREFIX(level, IOCPT, "%s(): ", __func__, __VA_ARGS__)
37
38 #define IOCPT_PRINT_CALL() IOCPT_PRINT(DEBUG, " >>")
39
40 const struct rte_cryptodev_capabilities *iocpt_get_caps(uint64_t flags);
41
iocpt_struct_size_checks(void)42 static inline void iocpt_struct_size_checks(void)
43 {
44 RTE_BUILD_BUG_ON(sizeof(struct ionic_doorbell) != 8);
45 RTE_BUILD_BUG_ON(sizeof(struct ionic_intr) != 32);
46 RTE_BUILD_BUG_ON(sizeof(struct ionic_intr_status) != 8);
47
48 RTE_BUILD_BUG_ON(sizeof(union iocpt_dev_regs) != 4096);
49 RTE_BUILD_BUG_ON(sizeof(union iocpt_dev_info_regs) != 2048);
50 RTE_BUILD_BUG_ON(sizeof(union iocpt_dev_cmd_regs) != 2048);
51
52 RTE_BUILD_BUG_ON(sizeof(struct iocpt_admin_cmd) != 64);
53 RTE_BUILD_BUG_ON(sizeof(struct iocpt_admin_comp) != 16);
54 RTE_BUILD_BUG_ON(sizeof(struct iocpt_nop_cmd) != 64);
55 RTE_BUILD_BUG_ON(sizeof(struct iocpt_nop_comp) != 16);
56
57 /* Device commands */
58 RTE_BUILD_BUG_ON(sizeof(struct iocpt_dev_identify_cmd) != 64);
59 RTE_BUILD_BUG_ON(sizeof(struct iocpt_dev_identify_comp) != 16);
60 RTE_BUILD_BUG_ON(sizeof(struct iocpt_dev_reset_cmd) != 64);
61 RTE_BUILD_BUG_ON(sizeof(struct iocpt_dev_reset_comp) != 16);
62
63 /* LIF commands */
64 RTE_BUILD_BUG_ON(sizeof(struct iocpt_lif_identify_cmd) != 64);
65 RTE_BUILD_BUG_ON(sizeof(struct iocpt_lif_identify_comp) != 16);
66 RTE_BUILD_BUG_ON(sizeof(struct iocpt_lif_init_cmd) != 64);
67 RTE_BUILD_BUG_ON(sizeof(struct iocpt_lif_init_comp) != 16);
68 RTE_BUILD_BUG_ON(sizeof(struct iocpt_lif_reset_cmd) != 64);
69 RTE_BUILD_BUG_ON(sizeof(struct iocpt_lif_getattr_cmd) != 64);
70 RTE_BUILD_BUG_ON(sizeof(struct iocpt_lif_getattr_comp) != 16);
71 RTE_BUILD_BUG_ON(sizeof(struct iocpt_lif_setattr_cmd) != 64);
72 RTE_BUILD_BUG_ON(sizeof(struct iocpt_lif_setattr_comp) != 16);
73
74 /* Queue commands */
75 RTE_BUILD_BUG_ON(sizeof(struct iocpt_q_identify_cmd) != 64);
76 RTE_BUILD_BUG_ON(sizeof(struct iocpt_q_identify_comp) != 16);
77 RTE_BUILD_BUG_ON(sizeof(struct iocpt_q_init_cmd) != 64);
78 RTE_BUILD_BUG_ON(sizeof(struct iocpt_q_init_comp) != 16);
79 RTE_BUILD_BUG_ON(sizeof(struct iocpt_q_control_cmd) != 64);
80
81 /* Crypto */
82 RTE_BUILD_BUG_ON(sizeof(struct iocpt_crypto_desc) != 32);
83 RTE_BUILD_BUG_ON(sizeof(struct iocpt_crypto_sg_desc) != 256);
84 RTE_BUILD_BUG_ON(sizeof(struct iocpt_crypto_comp) != 16);
85 }
86
87 struct iocpt_dev_bars {
88 struct ionic_dev_bar bar[IONIC_BARS_MAX];
89 uint32_t num_bars;
90 };
91
92 /* Queue watchdog */
93 #define IOCPT_Q_WDOG_SESS_IDX 0
94 #define IOCPT_Q_WDOG_KEY_LEN 16
95 #define IOCPT_Q_WDOG_IV_LEN 12
96 #define IOCPT_Q_WDOG_PLD_LEN 4
97 #define IOCPT_Q_WDOG_TAG_LEN 16
98 #define IOCPT_Q_WDOG_OP_TYPE RTE_CRYPTO_OP_TYPE_UNDEFINED
99
100 struct iocpt_qtype_info {
101 uint8_t version;
102 uint8_t supported;
103 uint64_t features;
104 uint16_t desc_sz;
105 uint16_t comp_sz;
106 uint16_t sg_desc_sz;
107 uint16_t max_sg_elems;
108 uint16_t sg_desc_stride;
109 };
110
111 #define IOCPT_Q_F_INITED BIT(0)
112 #define IOCPT_Q_F_DEFERRED BIT(1)
113 #define IOCPT_Q_F_SG BIT(2)
114
115 #define Q_NEXT_TO_POST(_q, _n) (((_q)->head_idx + (_n)) & ((_q)->size_mask))
116 #define Q_NEXT_TO_SRVC(_q, _n) (((_q)->tail_idx + (_n)) & ((_q)->size_mask))
117
118 #define IOCPT_INFO_SZ(_q) ((_q)->num_segs * sizeof(void *))
119 #define IOCPT_INFO_IDX(_q, _i) ((_i) * (_q)->num_segs)
120 #define IOCPT_INFO_PTR(_q, _i) (&(_q)->info[IOCPT_INFO_IDX((_q), _i)])
121
122 struct iocpt_queue {
123 uint16_t num_descs;
124 uint16_t num_segs;
125 uint16_t head_idx;
126 uint16_t tail_idx;
127 uint16_t size_mask;
128 uint8_t type;
129 uint8_t hw_type;
130 void *base;
131 void *sg_base;
132 struct ionic_doorbell __iomem *db;
133 void **info;
134
135 uint32_t index;
136 uint32_t hw_index;
137 rte_iova_t base_pa;
138 rte_iova_t sg_base_pa;
139 };
140
141 struct iocpt_cq {
142 uint16_t tail_idx;
143 uint16_t num_descs;
144 uint16_t size_mask;
145 bool done_color;
146 void *base;
147 rte_iova_t base_pa;
148 };
149
150 #define IOCPT_COMMON_FIELDS \
151 struct iocpt_queue q; \
152 struct iocpt_cq cq; \
153 struct iocpt_dev *dev; \
154 const struct rte_memzone *base_z; \
155 void *base; \
156 rte_iova_t base_pa
157
158 struct iocpt_common_q {
159 IOCPT_COMMON_FIELDS;
160 };
161
162 struct iocpt_admin_q {
163 IOCPT_COMMON_FIELDS;
164
165 uint16_t flags;
166 };
167
168 struct iocpt_crypto_q {
169 /* cacheline0, cacheline1 */
170 IOCPT_COMMON_FIELDS;
171
172 /* cacheline2 */
173 uint64_t last_wdog_cycles;
174 uint16_t flags;
175
176 /* cacheline3 */
177 struct rte_cryptodev_stats stats;
178
179 uint64_t enqueued_wdogs;
180 uint64_t dequeued_wdogs;
181 uint8_t wdog_iv[IOCPT_Q_WDOG_IV_LEN];
182 uint8_t wdog_pld[IOCPT_Q_WDOG_PLD_LEN];
183 uint8_t wdog_tag[IOCPT_Q_WDOG_TAG_LEN];
184 };
185
186 #define IOCPT_S_F_INITED BIT(0)
187
188 struct iocpt_session_priv {
189 struct iocpt_dev *dev;
190
191 uint32_t index;
192
193 uint16_t iv_offset;
194 uint16_t iv_length;
195 uint16_t digest_length;
196 uint16_t aad_length;
197
198 uint8_t flags;
199 uint8_t op;
200 uint8_t type;
201
202 uint16_t key_len;
203 uint8_t key[IOCPT_SESS_KEY_LEN_MAX_SYMM];
204 };
205
206 static inline uint32_t
iocpt_session_size(void)207 iocpt_session_size(void)
208 {
209 return sizeof(struct iocpt_session_priv);
210 }
211
212 #define IOCPT_DEV_F_INITED BIT(0)
213 #define IOCPT_DEV_F_UP BIT(1)
214 #define IOCPT_DEV_F_FW_RESET BIT(2)
215
216 /* Combined dev / LIF object */
217 struct iocpt_dev {
218 const char *name;
219 char fw_version[IOCPT_FWVERS_BUFLEN];
220 struct iocpt_dev_bars bars;
221 struct iocpt_identity ident;
222
223 const struct iocpt_dev_intf *intf;
224 void *bus_dev;
225 struct rte_cryptodev *crypto_dev;
226
227 union iocpt_dev_info_regs __iomem *dev_info;
228 union iocpt_dev_cmd_regs __iomem *dev_cmd;
229
230 struct ionic_doorbell __iomem *db_pages;
231 struct ionic_intr __iomem *intr_ctrl;
232
233 uint32_t max_qps;
234 uint32_t max_sessions;
235 uint16_t state;
236 uint8_t driver_id;
237 uint8_t socket_id;
238
239 rte_spinlock_t adminq_lock;
240 rte_spinlock_t adminq_service_lock;
241
242 struct iocpt_admin_q *adminq;
243 struct iocpt_crypto_q **cryptoqs;
244
245 struct rte_bitmap *sess_bm; /* SET bit indicates index is free */
246
247 uint64_t features;
248 uint32_t hw_features;
249
250 uint32_t info_sz;
251 struct iocpt_lif_info *info;
252 rte_iova_t info_pa;
253 const struct rte_memzone *info_z;
254
255 struct iocpt_qtype_info qtype_info[IOCPT_QTYPE_MAX];
256 uint8_t qtype_ver[IOCPT_QTYPE_MAX];
257
258 struct rte_cryptodev_stats stats_base;
259 };
260
261 struct iocpt_dev_intf {
262 int (*setup_bars)(struct iocpt_dev *dev);
263 void (*unmap_bars)(struct iocpt_dev *dev);
264 };
265
266 static inline int
iocpt_setup_bars(struct iocpt_dev * dev)267 iocpt_setup_bars(struct iocpt_dev *dev)
268 {
269 if (dev->intf->setup_bars == NULL)
270 return -EINVAL;
271
272 return (*dev->intf->setup_bars)(dev);
273 }
274
275 /** iocpt_admin_ctx - Admin command context.
276 * @pending_work: Flag that indicates a completion.
277 * @cmd: Admin command (64B) to be copied to the queue.
278 * @comp: Admin completion (16B) copied from the queue.
279 */
280 struct iocpt_admin_ctx {
281 bool pending_work;
282 union iocpt_adminq_cmd cmd;
283 union iocpt_adminq_comp comp;
284 };
285
286 int iocpt_probe(void *bus_dev, struct rte_device *rte_dev,
287 struct iocpt_dev_bars *bars, const struct iocpt_dev_intf *intf,
288 uint8_t driver_id, uint8_t socket_id);
289 int iocpt_remove(struct rte_device *rte_dev);
290
291 void iocpt_configure(struct iocpt_dev *dev);
292 int iocpt_assign_ops(struct rte_cryptodev *cdev);
293 int iocpt_start(struct iocpt_dev *dev);
294 void iocpt_stop(struct iocpt_dev *dev);
295 void iocpt_deinit(struct iocpt_dev *dev);
296
297 int iocpt_dev_identify(struct iocpt_dev *dev);
298 int iocpt_dev_init(struct iocpt_dev *dev, rte_iova_t info_pa);
299 int iocpt_dev_adminq_init(struct iocpt_dev *dev);
300 void iocpt_dev_reset(struct iocpt_dev *dev);
301
302 int iocpt_adminq_post_wait(struct iocpt_dev *dev, struct iocpt_admin_ctx *ctx);
303
304 int iocpt_cryptoq_alloc(struct iocpt_dev *dev, uint32_t socket_id,
305 uint32_t index, uint16_t ndescs);
306 void iocpt_cryptoq_free(struct iocpt_crypto_q *cptq);
307
308 int iocpt_session_init(struct iocpt_session_priv *priv);
309 int iocpt_session_update(struct iocpt_session_priv *priv);
310 void iocpt_session_deinit(struct iocpt_session_priv *priv);
311
312 struct ionic_doorbell __iomem *iocpt_db_map(struct iocpt_dev *dev,
313 struct iocpt_queue *q);
314
315 typedef bool (*iocpt_cq_cb)(struct iocpt_cq *cq, uint16_t cq_desc_index,
316 void *cb_arg);
317 uint32_t iocpt_cq_service(struct iocpt_cq *cq, uint32_t work_to_do,
318 iocpt_cq_cb cb, void *cb_arg);
319
320 void iocpt_get_stats(const struct iocpt_dev *dev,
321 struct rte_cryptodev_stats *stats);
322 void iocpt_reset_stats(struct iocpt_dev *dev);
323
324 static inline uint16_t
iocpt_q_space_avail(struct iocpt_queue * q)325 iocpt_q_space_avail(struct iocpt_queue *q)
326 {
327 uint16_t avail = q->tail_idx;
328
329 if (q->head_idx >= avail)
330 avail += q->num_descs - q->head_idx - 1;
331 else
332 avail -= q->head_idx + 1;
333
334 return avail;
335 }
336
337 static inline void
iocpt_q_flush(struct iocpt_queue * q)338 iocpt_q_flush(struct iocpt_queue *q)
339 {
340 uint64_t val = IONIC_DBELL_QID(q->hw_index) | q->head_idx;
341
342 #if defined(RTE_LIBRTE_IONIC_PMD_BARRIER_ERRATA)
343 /* On some devices the standard 'dmb' barrier is insufficient */
344 asm volatile("dsb st" : : : "memory");
345 rte_write64_relaxed(rte_cpu_to_le_64(val), q->db);
346 #else
347 rte_write64(rte_cpu_to_le_64(val), q->db);
348 #endif
349 }
350
351 static inline bool
iocpt_is_embedded(void)352 iocpt_is_embedded(void)
353 {
354 #if defined(RTE_LIBRTE_IONIC_PMD_EMBEDDED)
355 return true;
356 #else
357 return false;
358 #endif
359 }
360
361 #endif /* _IONIC_CRYPTO_H_ */
362