1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018 Advanced Micro Devices, Inc. All rights reserved.
3 */
4
5 #ifndef _CCP_DEV_H_
6 #define _CCP_DEV_H_
7
8 #include <limits.h>
9 #include <stdbool.h>
10 #include <stdint.h>
11 #include <string.h>
12
13 #include <bus_pci_driver.h>
14 #include <rte_atomic.h>
15 #include <rte_byteorder.h>
16 #include <rte_io.h>
17 #include <rte_pci.h>
18 #include <rte_spinlock.h>
19 #include <rte_crypto_sym.h>
20 #include <cryptodev_pmd.h>
21
22 /* CCP PCI device identifiers */
23 #define AMD_PCI_VENDOR_ID 0x1022
24 #define AMD_PCI_CCP_5A 0x1456
25 #define AMD_PCI_CCP_5B 0x1468
26 #define AMD_PCI_CCP_RV 0x15df
27
28 /**< CCP specific */
29 #define MAX_HW_QUEUES 5
30 #define CCP_MAX_TRNG_RETRIES 10
31 #define CCP_ALIGN(x, y) ((((x) + (y - 1)) / y) * y)
32
33 /**< CCP Register Mappings */
34 #define Q_MASK_REG 0x000
35 #define TRNG_OUT_REG 0x00c
36
37 /* CCP Version 5 Specifics */
38 #define CMD_QUEUE_MASK_OFFSET 0x00
39 #define CMD_QUEUE_PRIO_OFFSET 0x04
40 #define CMD_REQID_CONFIG_OFFSET 0x08
41 #define CMD_CMD_TIMEOUT_OFFSET 0x10
42 #define LSB_PUBLIC_MASK_LO_OFFSET 0x18
43 #define LSB_PUBLIC_MASK_HI_OFFSET 0x1C
44 #define LSB_PRIVATE_MASK_LO_OFFSET 0x20
45 #define LSB_PRIVATE_MASK_HI_OFFSET 0x24
46
47 #define CMD_Q_CONTROL_BASE 0x0000
48 #define CMD_Q_TAIL_LO_BASE 0x0004
49 #define CMD_Q_HEAD_LO_BASE 0x0008
50 #define CMD_Q_INT_ENABLE_BASE 0x000C
51 #define CMD_Q_INTERRUPT_STATUS_BASE 0x0010
52
53 #define CMD_Q_STATUS_BASE 0x0100
54 #define CMD_Q_INT_STATUS_BASE 0x0104
55
56 #define CMD_CONFIG_0_OFFSET 0x6000
57 #define CMD_TRNG_CTL_OFFSET 0x6008
58 #define CMD_AES_MASK_OFFSET 0x6010
59 #define CMD_CLK_GATE_CTL_OFFSET 0x603C
60
61 /* Address offset between two virtual queue registers */
62 #define CMD_Q_STATUS_INCR 0x1000
63
64 /* Bit masks */
65 #define CMD_Q_RUN 0x1
66 #define CMD_Q_SIZE 0x1F
67 #define CMD_Q_SHIFT 3
68 #define COMMANDS_PER_QUEUE 8192
69
70 #define QUEUE_SIZE_VAL ((ffs(COMMANDS_PER_QUEUE) - 2) & \
71 CMD_Q_SIZE)
72 #define Q_DESC_SIZE sizeof(struct ccp_desc)
73 #define Q_SIZE(n) (COMMANDS_PER_QUEUE*(n))
74
75 #define INT_COMPLETION 0x1
76 #define INT_ERROR 0x2
77 #define INT_QUEUE_STOPPED 0x4
78 #define ALL_INTERRUPTS (INT_COMPLETION| \
79 INT_ERROR| \
80 INT_QUEUE_STOPPED)
81
82 #define LSB_REGION_WIDTH 5
83 #define MAX_LSB_CNT 8
84
85 #define LSB_SIZE 16
86 #define LSB_ITEM_SIZE 32
87 #define SLSB_MAP_SIZE (MAX_LSB_CNT * LSB_SIZE)
88 #define LSB_ENTRY_NUMBER(LSB_ADDR) (LSB_ADDR / LSB_ITEM_SIZE)
89
90 /* General CCP Defines */
91
92 #define CCP_SB_BYTES 32
93 /* Word 0 */
94 #define CCP_CMD_DW0(p) ((p)->dw0)
95 #define CCP_CMD_SOC(p) (CCP_CMD_DW0(p).soc)
96 #define CCP_CMD_IOC(p) (CCP_CMD_DW0(p).ioc)
97 #define CCP_CMD_INIT(p) (CCP_CMD_DW0(p).init)
98 #define CCP_CMD_EOM(p) (CCP_CMD_DW0(p).eom)
99 #define CCP_CMD_FUNCTION(p) (CCP_CMD_DW0(p).function)
100 #define CCP_CMD_ENGINE(p) (CCP_CMD_DW0(p).engine)
101 #define CCP_CMD_PROT(p) (CCP_CMD_DW0(p).prot)
102
103 /* Word 1 */
104 #define CCP_CMD_DW1(p) ((p)->length)
105 #define CCP_CMD_LEN(p) (CCP_CMD_DW1(p))
106
107 /* Word 2 */
108 #define CCP_CMD_DW2(p) ((p)->src_lo)
109 #define CCP_CMD_SRC_LO(p) (CCP_CMD_DW2(p))
110
111 /* Word 3 */
112 #define CCP_CMD_DW3(p) ((p)->dw3)
113 #define CCP_CMD_SRC_MEM(p) ((p)->dw3.src_mem)
114 #define CCP_CMD_SRC_HI(p) ((p)->dw3.src_hi)
115 #define CCP_CMD_LSB_ID(p) ((p)->dw3.lsb_cxt_id)
116 #define CCP_CMD_FIX_SRC(p) ((p)->dw3.fixed)
117
118 /* Words 4/5 */
119 #define CCP_CMD_DW4(p) ((p)->dw4)
120 #define CCP_CMD_DST_LO(p) (CCP_CMD_DW4(p).dst_lo)
121 #define CCP_CMD_DW5(p) ((p)->dw5.fields.dst_hi)
122 #define CCP_CMD_DST_HI(p) (CCP_CMD_DW5(p))
123 #define CCP_CMD_DST_MEM(p) ((p)->dw5.fields.dst_mem)
124 #define CCP_CMD_FIX_DST(p) ((p)->dw5.fields.fixed)
125 #define CCP_CMD_SHA_LO(p) ((p)->dw4.sha_len_lo)
126 #define CCP_CMD_SHA_HI(p) ((p)->dw5.sha_len_hi)
127
128 /* Word 6/7 */
129 #define CCP_CMD_DW6(p) ((p)->key_lo)
130 #define CCP_CMD_KEY_LO(p) (CCP_CMD_DW6(p))
131 #define CCP_CMD_DW7(p) ((p)->dw7)
132 #define CCP_CMD_KEY_HI(p) ((p)->dw7.key_hi)
133 #define CCP_CMD_KEY_MEM(p) ((p)->dw7.key_mem)
134
135 /* bitmap */
136 enum {
137 BITS_PER_WORD = sizeof(unsigned long) * CHAR_BIT
138 };
139
140 #define WORD_OFFSET(b) ((b) / BITS_PER_WORD)
141 #define BIT_OFFSET(b) ((b) % BITS_PER_WORD)
142
143 #define CCP_DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
144 #define CCP_BITMAP_SIZE(nr) \
145 CCP_DIV_ROUND_UP(nr, CHAR_BIT * sizeof(unsigned long))
146
147 #define CCP_BITMAP_FIRST_WORD_MASK(start) \
148 (~0UL << ((start) & (BITS_PER_WORD - 1)))
149 #define CCP_BITMAP_LAST_WORD_MASK(nbits) \
150 (~0UL >> (-(nbits) & (BITS_PER_WORD - 1)))
151
152 #define __ccp_round_mask(x, y) ((typeof(x))((y)-1))
153 #define ccp_round_down(x, y) ((x) & ~__ccp_round_mask(x, y))
154
155 /** CCP registers Write/Read */
156
ccp_pci_reg_write(void * base,int offset,uint32_t value)157 static inline void ccp_pci_reg_write(void *base, int offset,
158 uint32_t value)
159 {
160 volatile void *reg_addr = ((uint8_t *)base + offset);
161
162 rte_write32((rte_cpu_to_le_32(value)), reg_addr);
163 }
164
ccp_pci_reg_read(void * base,int offset)165 static inline uint32_t ccp_pci_reg_read(void *base, int offset)
166 {
167 volatile void *reg_addr = ((uint8_t *)base + offset);
168
169 return rte_le_to_cpu_32(rte_read32(reg_addr));
170 }
171
172 #define CCP_READ_REG(hw_addr, reg_offset) \
173 ccp_pci_reg_read(hw_addr, reg_offset)
174
175 #define CCP_WRITE_REG(hw_addr, reg_offset, value) \
176 ccp_pci_reg_write(hw_addr, reg_offset, value)
177
178 /**
179 * A structure describing a CCP command queue.
180 */
181 struct __rte_cache_aligned ccp_queue {
182 struct ccp_device *dev;
183 char memz_name[RTE_MEMZONE_NAMESIZE];
184
185 rte_atomic64_t free_slots;
186 /**< available free slots updated from enq/deq calls */
187
188 /* Queue identifier */
189 uint64_t id; /**< queue id */
190 uint64_t qidx; /**< queue index */
191 uint64_t qsize; /**< queue size */
192
193 /* Queue address */
194 struct ccp_desc *qbase_desc;
195 void *qbase_addr;
196 phys_addr_t qbase_phys_addr;
197 /**< queue-page registers addr */
198 void *reg_base;
199
200 uint32_t qcontrol;
201 /**< queue ctrl reg */
202
203 int lsb;
204 /**< lsb region assigned to queue */
205 unsigned long lsbmask;
206 /**< lsb regions queue can access */
207 unsigned long lsbmap[CCP_BITMAP_SIZE(LSB_SIZE)];
208 /**< all lsb resources which queue is using */
209 uint32_t sb_key;
210 /**< lsb assigned for queue */
211 uint32_t sb_iv;
212 /**< lsb assigned for iv */
213 uint32_t sb_sha;
214 /**< lsb assigned for sha ctx */
215 uint32_t sb_hmac;
216 /**< lsb assigned for hmac ctx */
217 };
218
219 /**
220 * A structure describing a CCP device.
221 */
222 struct __rte_cache_aligned ccp_device {
223 TAILQ_ENTRY(ccp_device) next;
224 int id;
225 /**< ccp dev id on platform */
226 struct ccp_queue cmd_q[MAX_HW_QUEUES];
227 /**< ccp queue */
228 int cmd_q_count;
229 /**< no. of ccp Queues */
230 struct rte_pci_device *pci;
231 /**< ccp pci device */
232 unsigned long lsbmap[CCP_BITMAP_SIZE(SLSB_MAP_SIZE)];
233 /**< shared lsb mask of ccp */
234 rte_spinlock_t lsb_lock;
235 /**< protection for shared lsb region allocation */
236 int qidx;
237 /**< current queue index */
238 int hwrng_retries;
239 /**< retry counter for CCP TRNG */
240 };
241
242 /**< CCP H/W engine related */
243 /**
244 * ccp_engine - CCP operation identifiers
245 *
246 * @CCP_ENGINE_AES: AES operation
247 * @CCP_ENGINE_XTS_AES: 128-bit XTS AES operation
248 * @CCP_ENGINE_3DES: DES/3DES operation
249 * @CCP_ENGINE_SHA: SHA operation
250 * @CCP_ENGINE_RSA: RSA operation
251 * @CCP_ENGINE_PASSTHRU: pass-through operation
252 * @CCP_ENGINE_ZLIB_DECOMPRESS: unused
253 * @CCP_ENGINE_ECC: ECC operation
254 */
255 enum ccp_engine {
256 CCP_ENGINE_AES = 0,
257 CCP_ENGINE_XTS_AES_128,
258 CCP_ENGINE_3DES,
259 CCP_ENGINE_SHA,
260 CCP_ENGINE_RSA,
261 CCP_ENGINE_PASSTHRU,
262 CCP_ENGINE_ZLIB_DECOMPRESS,
263 CCP_ENGINE_ECC,
264 CCP_ENGINE__LAST,
265 };
266
267 /* Passthru engine */
268 /**
269 * ccp_passthru_bitwise - type of bitwise passthru operation
270 *
271 * @CCP_PASSTHRU_BITWISE_NOOP: no bitwise operation performed
272 * @CCP_PASSTHRU_BITWISE_AND: perform bitwise AND of src with mask
273 * @CCP_PASSTHRU_BITWISE_OR: perform bitwise OR of src with mask
274 * @CCP_PASSTHRU_BITWISE_XOR: perform bitwise XOR of src with mask
275 * @CCP_PASSTHRU_BITWISE_MASK: overwrite with mask
276 */
277 enum ccp_passthru_bitwise {
278 CCP_PASSTHRU_BITWISE_NOOP = 0,
279 CCP_PASSTHRU_BITWISE_AND,
280 CCP_PASSTHRU_BITWISE_OR,
281 CCP_PASSTHRU_BITWISE_XOR,
282 CCP_PASSTHRU_BITWISE_MASK,
283 CCP_PASSTHRU_BITWISE__LAST,
284 };
285
286 /**
287 * ccp_passthru_byteswap - type of byteswap passthru operation
288 *
289 * @CCP_PASSTHRU_BYTESWAP_NOOP: no byte swapping performed
290 * @CCP_PASSTHRU_BYTESWAP_32BIT: swap bytes within 32-bit words
291 * @CCP_PASSTHRU_BYTESWAP_256BIT: swap bytes within 256-bit words
292 */
293 enum ccp_passthru_byteswap {
294 CCP_PASSTHRU_BYTESWAP_NOOP = 0,
295 CCP_PASSTHRU_BYTESWAP_32BIT,
296 CCP_PASSTHRU_BYTESWAP_256BIT,
297 CCP_PASSTHRU_BYTESWAP__LAST,
298 };
299
300 /**
301 * CCP passthru
302 */
303 struct ccp_passthru {
304 phys_addr_t src_addr;
305 phys_addr_t dest_addr;
306 enum ccp_passthru_bitwise bit_mod;
307 enum ccp_passthru_byteswap byte_swap;
308 int len;
309 int dir;
310 };
311
312 /* CCP version 5: Union to define the function field (cmd_reg1/dword0) */
313 union ccp_function {
314 struct {
315 uint16_t size:7;
316 uint16_t encrypt:1;
317 uint16_t mode:5;
318 uint16_t type:2;
319 } aes;
320 struct {
321 uint16_t size:7;
322 uint16_t encrypt:1;
323 uint16_t mode:5;
324 uint16_t type:2;
325 } des;
326 struct {
327 uint16_t size:7;
328 uint16_t encrypt:1;
329 uint16_t rsvd:5;
330 uint16_t type:2;
331 } aes_xts;
332 struct {
333 uint16_t rsvd1:10;
334 uint16_t type:4;
335 uint16_t rsvd2:1;
336 } sha;
337 struct {
338 uint16_t mode:3;
339 uint16_t size:12;
340 } rsa;
341 struct {
342 uint16_t byteswap:2;
343 uint16_t bitwise:3;
344 uint16_t reflect:2;
345 uint16_t rsvd:8;
346 } pt;
347 struct {
348 uint16_t rsvd:13;
349 } zlib;
350 struct {
351 uint16_t size:10;
352 uint16_t type:2;
353 uint16_t mode:3;
354 } ecc;
355 uint16_t raw;
356 };
357
358
359 /**
360 * descriptor for version 5 CPP commands
361 * 8 32-bit words:
362 * word 0: function; engine; control bits
363 * word 1: length of source data
364 * word 2: low 32 bits of source pointer
365 * word 3: upper 16 bits of source pointer; source memory type
366 * word 4: low 32 bits of destination pointer
367 * word 5: upper 16 bits of destination pointer; destination memory
368 * type
369 * word 6: low 32 bits of key pointer
370 * word 7: upper 16 bits of key pointer; key memory type
371 */
372 struct dword0 {
373 uint32_t soc:1;
374 uint32_t ioc:1;
375 uint32_t rsvd1:1;
376 uint32_t init:1;
377 uint32_t eom:1;
378 uint32_t function:15;
379 uint32_t engine:4;
380 uint32_t prot:1;
381 uint32_t rsvd2:7;
382 };
383
384 struct dword3 {
385 uint32_t src_hi:16;
386 uint32_t src_mem:2;
387 uint32_t lsb_cxt_id:8;
388 uint32_t rsvd1:5;
389 uint32_t fixed:1;
390 };
391
392 union dword4 {
393 uint32_t dst_lo; /* NON-SHA */
394 uint32_t sha_len_lo; /* SHA */
395 };
396
397 union dword5 {
398 struct {
399 uint32_t dst_hi:16;
400 uint32_t dst_mem:2;
401 uint32_t rsvd1:13;
402 uint32_t fixed:1;
403 }
404 fields;
405 uint32_t sha_len_hi;
406 };
407
408 struct dword7 {
409 uint32_t key_hi:16;
410 uint32_t key_mem:2;
411 uint32_t rsvd1:14;
412 };
413
414 struct ccp_desc {
415 struct dword0 dw0;
416 uint32_t length;
417 uint32_t src_lo;
418 struct dword3 dw3;
419 union dword4 dw4;
420 union dword5 dw5;
421 uint32_t key_lo;
422 struct dword7 dw7;
423 };
424
425 /**
426 * ccp memory type
427 */
428 enum ccp_memtype {
429 CCP_MEMTYPE_SYSTEM = 0,
430 CCP_MEMTYPE_SB,
431 CCP_MEMTYPE_LOCAL,
432 CCP_MEMTYPE_LAST,
433 };
434
435 /**
436 * cmd id to follow order
437 */
438 enum ccp_cmd_order {
439 CCP_CMD_CIPHER = 0,
440 CCP_CMD_AUTH,
441 CCP_CMD_CIPHER_HASH,
442 CCP_CMD_HASH_CIPHER,
443 CCP_CMD_COMBINED,
444 CCP_CMD_NOT_SUPPORTED,
445 };
446
447 static inline uint32_t
low32_value(unsigned long addr)448 low32_value(unsigned long addr)
449 {
450 return ((uint64_t)addr) & 0x0ffffffff;
451 }
452
453 static inline uint32_t
high32_value(unsigned long addr)454 high32_value(unsigned long addr)
455 {
456 return ((uint64_t)addr >> 32) & 0x00000ffff;
457 }
458
459 /*
460 * Start CCP device
461 */
462 int ccp_dev_start(struct rte_cryptodev *dev);
463
464 /**
465 * Initialize one ccp device
466 *
467 * @dev rte pci device
468 * @return 0 on success otherwise -1
469 */
470 int ccp_probe_device(struct rte_pci_device *pci_dev);
471
472 /**
473 * allocate a ccp command queue
474 *
475 * @dev rte crypto device
476 * @param slot_req number of required
477 * @return allotted CCP queue on success otherwise NULL
478 */
479 struct ccp_queue *ccp_allot_queue(struct rte_cryptodev *dev, int slot_req);
480
481 /**
482 * read hwrng value
483 *
484 * @param trng_value data pointer to write RNG value
485 * @return 0 on success otherwise -1
486 */
487 int ccp_read_hwrng(uint32_t *trng_value);
488
489 #endif /* _CCP_DEV_H_ */
490