1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Advanced Micro Devices, Inc. All rights reserved. 3 */ 4 5 #ifndef _CCP_DEV_H_ 6 #define _CCP_DEV_H_ 7 8 #include <limits.h> 9 #include <stdbool.h> 10 #include <stdint.h> 11 #include <string.h> 12 13 #include <bus_pci_driver.h> 14 #include <rte_atomic.h> 15 #include <rte_byteorder.h> 16 #include <rte_io.h> 17 #include <rte_pci.h> 18 #include <rte_spinlock.h> 19 #include <rte_crypto_sym.h> 20 #include <cryptodev_pmd.h> 21 22 /* CCP PCI device identifiers */ 23 #define AMD_PCI_VENDOR_ID 0x1022 24 #define AMD_PCI_CCP_5A 0x1456 25 #define AMD_PCI_CCP_5B 0x1468 26 #define AMD_PCI_CCP_RV 0x15df 27 28 /**< CCP specific */ 29 #define MAX_HW_QUEUES 5 30 #define CCP_MAX_TRNG_RETRIES 10 31 #define CCP_ALIGN(x, y) ((((x) + (y - 1)) / y) * y) 32 33 /**< CCP Register Mappings */ 34 #define Q_MASK_REG 0x000 35 #define TRNG_OUT_REG 0x00c 36 37 /* CCP Version 5 Specifics */ 38 #define CMD_QUEUE_MASK_OFFSET 0x00 39 #define CMD_QUEUE_PRIO_OFFSET 0x04 40 #define CMD_REQID_CONFIG_OFFSET 0x08 41 #define CMD_CMD_TIMEOUT_OFFSET 0x10 42 #define LSB_PUBLIC_MASK_LO_OFFSET 0x18 43 #define LSB_PUBLIC_MASK_HI_OFFSET 0x1C 44 #define LSB_PRIVATE_MASK_LO_OFFSET 0x20 45 #define LSB_PRIVATE_MASK_HI_OFFSET 0x24 46 47 #define CMD_Q_CONTROL_BASE 0x0000 48 #define CMD_Q_TAIL_LO_BASE 0x0004 49 #define CMD_Q_HEAD_LO_BASE 0x0008 50 #define CMD_Q_INT_ENABLE_BASE 0x000C 51 #define CMD_Q_INTERRUPT_STATUS_BASE 0x0010 52 53 #define CMD_Q_STATUS_BASE 0x0100 54 #define CMD_Q_INT_STATUS_BASE 0x0104 55 56 #define CMD_CONFIG_0_OFFSET 0x6000 57 #define CMD_TRNG_CTL_OFFSET 0x6008 58 #define CMD_AES_MASK_OFFSET 0x6010 59 #define CMD_CLK_GATE_CTL_OFFSET 0x603C 60 61 /* Address offset between two virtual queue registers */ 62 #define CMD_Q_STATUS_INCR 0x1000 63 64 /* Bit masks */ 65 #define CMD_Q_RUN 0x1 66 #define CMD_Q_SIZE 0x1F 67 #define CMD_Q_SHIFT 3 68 #define COMMANDS_PER_QUEUE 8192 69 70 #define QUEUE_SIZE_VAL ((ffs(COMMANDS_PER_QUEUE) - 2) & \ 71 CMD_Q_SIZE) 72 #define Q_DESC_SIZE sizeof(struct ccp_desc) 73 #define Q_SIZE(n) (COMMANDS_PER_QUEUE*(n)) 74 75 #define INT_COMPLETION 0x1 76 #define INT_ERROR 0x2 77 #define INT_QUEUE_STOPPED 0x4 78 #define ALL_INTERRUPTS (INT_COMPLETION| \ 79 INT_ERROR| \ 80 INT_QUEUE_STOPPED) 81 82 #define LSB_REGION_WIDTH 5 83 #define MAX_LSB_CNT 8 84 85 #define LSB_SIZE 16 86 #define LSB_ITEM_SIZE 32 87 #define SLSB_MAP_SIZE (MAX_LSB_CNT * LSB_SIZE) 88 #define LSB_ENTRY_NUMBER(LSB_ADDR) (LSB_ADDR / LSB_ITEM_SIZE) 89 90 /* General CCP Defines */ 91 92 #define CCP_SB_BYTES 32 93 /* Word 0 */ 94 #define CCP_CMD_DW0(p) ((p)->dw0) 95 #define CCP_CMD_SOC(p) (CCP_CMD_DW0(p).soc) 96 #define CCP_CMD_IOC(p) (CCP_CMD_DW0(p).ioc) 97 #define CCP_CMD_INIT(p) (CCP_CMD_DW0(p).init) 98 #define CCP_CMD_EOM(p) (CCP_CMD_DW0(p).eom) 99 #define CCP_CMD_FUNCTION(p) (CCP_CMD_DW0(p).function) 100 #define CCP_CMD_ENGINE(p) (CCP_CMD_DW0(p).engine) 101 #define CCP_CMD_PROT(p) (CCP_CMD_DW0(p).prot) 102 103 /* Word 1 */ 104 #define CCP_CMD_DW1(p) ((p)->length) 105 #define CCP_CMD_LEN(p) (CCP_CMD_DW1(p)) 106 107 /* Word 2 */ 108 #define CCP_CMD_DW2(p) ((p)->src_lo) 109 #define CCP_CMD_SRC_LO(p) (CCP_CMD_DW2(p)) 110 111 /* Word 3 */ 112 #define CCP_CMD_DW3(p) ((p)->dw3) 113 #define CCP_CMD_SRC_MEM(p) ((p)->dw3.src_mem) 114 #define CCP_CMD_SRC_HI(p) ((p)->dw3.src_hi) 115 #define CCP_CMD_LSB_ID(p) ((p)->dw3.lsb_cxt_id) 116 #define CCP_CMD_FIX_SRC(p) ((p)->dw3.fixed) 117 118 /* Words 4/5 */ 119 #define CCP_CMD_DW4(p) ((p)->dw4) 120 #define CCP_CMD_DST_LO(p) (CCP_CMD_DW4(p).dst_lo) 121 #define CCP_CMD_DW5(p) ((p)->dw5.fields.dst_hi) 122 #define CCP_CMD_DST_HI(p) (CCP_CMD_DW5(p)) 123 #define CCP_CMD_DST_MEM(p) ((p)->dw5.fields.dst_mem) 124 #define CCP_CMD_FIX_DST(p) ((p)->dw5.fields.fixed) 125 #define CCP_CMD_SHA_LO(p) ((p)->dw4.sha_len_lo) 126 #define CCP_CMD_SHA_HI(p) ((p)->dw5.sha_len_hi) 127 128 /* Word 6/7 */ 129 #define CCP_CMD_DW6(p) ((p)->key_lo) 130 #define CCP_CMD_KEY_LO(p) (CCP_CMD_DW6(p)) 131 #define CCP_CMD_DW7(p) ((p)->dw7) 132 #define CCP_CMD_KEY_HI(p) ((p)->dw7.key_hi) 133 #define CCP_CMD_KEY_MEM(p) ((p)->dw7.key_mem) 134 135 /* bitmap */ 136 enum { 137 BITS_PER_WORD = sizeof(unsigned long) * CHAR_BIT 138 }; 139 140 #define WORD_OFFSET(b) ((b) / BITS_PER_WORD) 141 #define BIT_OFFSET(b) ((b) % BITS_PER_WORD) 142 143 #define CCP_DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d)) 144 #define CCP_BITMAP_SIZE(nr) \ 145 CCP_DIV_ROUND_UP(nr, CHAR_BIT * sizeof(unsigned long)) 146 147 #define CCP_BITMAP_FIRST_WORD_MASK(start) \ 148 (~0UL << ((start) & (BITS_PER_WORD - 1))) 149 #define CCP_BITMAP_LAST_WORD_MASK(nbits) \ 150 (~0UL >> (-(nbits) & (BITS_PER_WORD - 1))) 151 152 #define __ccp_round_mask(x, y) ((typeof(x))((y)-1)) 153 #define ccp_round_down(x, y) ((x) & ~__ccp_round_mask(x, y)) 154 155 /** CCP registers Write/Read */ 156 157 static inline void ccp_pci_reg_write(void *base, int offset, 158 uint32_t value) 159 { 160 volatile void *reg_addr = ((uint8_t *)base + offset); 161 162 rte_write32((rte_cpu_to_le_32(value)), reg_addr); 163 } 164 165 static inline uint32_t ccp_pci_reg_read(void *base, int offset) 166 { 167 volatile void *reg_addr = ((uint8_t *)base + offset); 168 169 return rte_le_to_cpu_32(rte_read32(reg_addr)); 170 } 171 172 #define CCP_READ_REG(hw_addr, reg_offset) \ 173 ccp_pci_reg_read(hw_addr, reg_offset) 174 175 #define CCP_WRITE_REG(hw_addr, reg_offset, value) \ 176 ccp_pci_reg_write(hw_addr, reg_offset, value) 177 178 /** 179 * A structure describing a CCP command queue. 180 */ 181 struct __rte_cache_aligned ccp_queue { 182 struct ccp_device *dev; 183 char memz_name[RTE_MEMZONE_NAMESIZE]; 184 185 rte_atomic64_t free_slots; 186 /**< available free slots updated from enq/deq calls */ 187 188 /* Queue identifier */ 189 uint64_t id; /**< queue id */ 190 uint64_t qidx; /**< queue index */ 191 uint64_t qsize; /**< queue size */ 192 193 /* Queue address */ 194 struct ccp_desc *qbase_desc; 195 void *qbase_addr; 196 phys_addr_t qbase_phys_addr; 197 /**< queue-page registers addr */ 198 void *reg_base; 199 200 uint32_t qcontrol; 201 /**< queue ctrl reg */ 202 203 int lsb; 204 /**< lsb region assigned to queue */ 205 unsigned long lsbmask; 206 /**< lsb regions queue can access */ 207 unsigned long lsbmap[CCP_BITMAP_SIZE(LSB_SIZE)]; 208 /**< all lsb resources which queue is using */ 209 uint32_t sb_key; 210 /**< lsb assigned for queue */ 211 uint32_t sb_iv; 212 /**< lsb assigned for iv */ 213 uint32_t sb_sha; 214 /**< lsb assigned for sha ctx */ 215 uint32_t sb_hmac; 216 /**< lsb assigned for hmac ctx */ 217 }; 218 219 /** 220 * A structure describing a CCP device. 221 */ 222 struct __rte_cache_aligned ccp_device { 223 TAILQ_ENTRY(ccp_device) next; 224 int id; 225 /**< ccp dev id on platform */ 226 struct ccp_queue cmd_q[MAX_HW_QUEUES]; 227 /**< ccp queue */ 228 int cmd_q_count; 229 /**< no. of ccp Queues */ 230 struct rte_pci_device *pci; 231 /**< ccp pci device */ 232 unsigned long lsbmap[CCP_BITMAP_SIZE(SLSB_MAP_SIZE)]; 233 /**< shared lsb mask of ccp */ 234 rte_spinlock_t lsb_lock; 235 /**< protection for shared lsb region allocation */ 236 int qidx; 237 /**< current queue index */ 238 int hwrng_retries; 239 /**< retry counter for CCP TRNG */ 240 }; 241 242 /**< CCP H/W engine related */ 243 /** 244 * ccp_engine - CCP operation identifiers 245 * 246 * @CCP_ENGINE_AES: AES operation 247 * @CCP_ENGINE_XTS_AES: 128-bit XTS AES operation 248 * @CCP_ENGINE_3DES: DES/3DES operation 249 * @CCP_ENGINE_SHA: SHA operation 250 * @CCP_ENGINE_RSA: RSA operation 251 * @CCP_ENGINE_PASSTHRU: pass-through operation 252 * @CCP_ENGINE_ZLIB_DECOMPRESS: unused 253 * @CCP_ENGINE_ECC: ECC operation 254 */ 255 enum ccp_engine { 256 CCP_ENGINE_AES = 0, 257 CCP_ENGINE_XTS_AES_128, 258 CCP_ENGINE_3DES, 259 CCP_ENGINE_SHA, 260 CCP_ENGINE_RSA, 261 CCP_ENGINE_PASSTHRU, 262 CCP_ENGINE_ZLIB_DECOMPRESS, 263 CCP_ENGINE_ECC, 264 CCP_ENGINE__LAST, 265 }; 266 267 /* Passthru engine */ 268 /** 269 * ccp_passthru_bitwise - type of bitwise passthru operation 270 * 271 * @CCP_PASSTHRU_BITWISE_NOOP: no bitwise operation performed 272 * @CCP_PASSTHRU_BITWISE_AND: perform bitwise AND of src with mask 273 * @CCP_PASSTHRU_BITWISE_OR: perform bitwise OR of src with mask 274 * @CCP_PASSTHRU_BITWISE_XOR: perform bitwise XOR of src with mask 275 * @CCP_PASSTHRU_BITWISE_MASK: overwrite with mask 276 */ 277 enum ccp_passthru_bitwise { 278 CCP_PASSTHRU_BITWISE_NOOP = 0, 279 CCP_PASSTHRU_BITWISE_AND, 280 CCP_PASSTHRU_BITWISE_OR, 281 CCP_PASSTHRU_BITWISE_XOR, 282 CCP_PASSTHRU_BITWISE_MASK, 283 CCP_PASSTHRU_BITWISE__LAST, 284 }; 285 286 /** 287 * ccp_passthru_byteswap - type of byteswap passthru operation 288 * 289 * @CCP_PASSTHRU_BYTESWAP_NOOP: no byte swapping performed 290 * @CCP_PASSTHRU_BYTESWAP_32BIT: swap bytes within 32-bit words 291 * @CCP_PASSTHRU_BYTESWAP_256BIT: swap bytes within 256-bit words 292 */ 293 enum ccp_passthru_byteswap { 294 CCP_PASSTHRU_BYTESWAP_NOOP = 0, 295 CCP_PASSTHRU_BYTESWAP_32BIT, 296 CCP_PASSTHRU_BYTESWAP_256BIT, 297 CCP_PASSTHRU_BYTESWAP__LAST, 298 }; 299 300 /** 301 * CCP passthru 302 */ 303 struct ccp_passthru { 304 phys_addr_t src_addr; 305 phys_addr_t dest_addr; 306 enum ccp_passthru_bitwise bit_mod; 307 enum ccp_passthru_byteswap byte_swap; 308 int len; 309 int dir; 310 }; 311 312 /* CCP version 5: Union to define the function field (cmd_reg1/dword0) */ 313 union ccp_function { 314 struct { 315 uint16_t size:7; 316 uint16_t encrypt:1; 317 uint16_t mode:5; 318 uint16_t type:2; 319 } aes; 320 struct { 321 uint16_t size:7; 322 uint16_t encrypt:1; 323 uint16_t mode:5; 324 uint16_t type:2; 325 } des; 326 struct { 327 uint16_t size:7; 328 uint16_t encrypt:1; 329 uint16_t rsvd:5; 330 uint16_t type:2; 331 } aes_xts; 332 struct { 333 uint16_t rsvd1:10; 334 uint16_t type:4; 335 uint16_t rsvd2:1; 336 } sha; 337 struct { 338 uint16_t mode:3; 339 uint16_t size:12; 340 } rsa; 341 struct { 342 uint16_t byteswap:2; 343 uint16_t bitwise:3; 344 uint16_t reflect:2; 345 uint16_t rsvd:8; 346 } pt; 347 struct { 348 uint16_t rsvd:13; 349 } zlib; 350 struct { 351 uint16_t size:10; 352 uint16_t type:2; 353 uint16_t mode:3; 354 } ecc; 355 uint16_t raw; 356 }; 357 358 359 /** 360 * descriptor for version 5 CPP commands 361 * 8 32-bit words: 362 * word 0: function; engine; control bits 363 * word 1: length of source data 364 * word 2: low 32 bits of source pointer 365 * word 3: upper 16 bits of source pointer; source memory type 366 * word 4: low 32 bits of destination pointer 367 * word 5: upper 16 bits of destination pointer; destination memory 368 * type 369 * word 6: low 32 bits of key pointer 370 * word 7: upper 16 bits of key pointer; key memory type 371 */ 372 struct dword0 { 373 uint32_t soc:1; 374 uint32_t ioc:1; 375 uint32_t rsvd1:1; 376 uint32_t init:1; 377 uint32_t eom:1; 378 uint32_t function:15; 379 uint32_t engine:4; 380 uint32_t prot:1; 381 uint32_t rsvd2:7; 382 }; 383 384 struct dword3 { 385 uint32_t src_hi:16; 386 uint32_t src_mem:2; 387 uint32_t lsb_cxt_id:8; 388 uint32_t rsvd1:5; 389 uint32_t fixed:1; 390 }; 391 392 union dword4 { 393 uint32_t dst_lo; /* NON-SHA */ 394 uint32_t sha_len_lo; /* SHA */ 395 }; 396 397 union dword5 { 398 struct { 399 uint32_t dst_hi:16; 400 uint32_t dst_mem:2; 401 uint32_t rsvd1:13; 402 uint32_t fixed:1; 403 } 404 fields; 405 uint32_t sha_len_hi; 406 }; 407 408 struct dword7 { 409 uint32_t key_hi:16; 410 uint32_t key_mem:2; 411 uint32_t rsvd1:14; 412 }; 413 414 struct ccp_desc { 415 struct dword0 dw0; 416 uint32_t length; 417 uint32_t src_lo; 418 struct dword3 dw3; 419 union dword4 dw4; 420 union dword5 dw5; 421 uint32_t key_lo; 422 struct dword7 dw7; 423 }; 424 425 /** 426 * ccp memory type 427 */ 428 enum ccp_memtype { 429 CCP_MEMTYPE_SYSTEM = 0, 430 CCP_MEMTYPE_SB, 431 CCP_MEMTYPE_LOCAL, 432 CCP_MEMTYPE_LAST, 433 }; 434 435 /** 436 * cmd id to follow order 437 */ 438 enum ccp_cmd_order { 439 CCP_CMD_CIPHER = 0, 440 CCP_CMD_AUTH, 441 CCP_CMD_CIPHER_HASH, 442 CCP_CMD_HASH_CIPHER, 443 CCP_CMD_COMBINED, 444 CCP_CMD_NOT_SUPPORTED, 445 }; 446 447 static inline uint32_t 448 low32_value(unsigned long addr) 449 { 450 return ((uint64_t)addr) & 0x0ffffffff; 451 } 452 453 static inline uint32_t 454 high32_value(unsigned long addr) 455 { 456 return ((uint64_t)addr >> 32) & 0x00000ffff; 457 } 458 459 /* 460 * Start CCP device 461 */ 462 int ccp_dev_start(struct rte_cryptodev *dev); 463 464 /** 465 * Initialize one ccp device 466 * 467 * @dev rte pci device 468 * @return 0 on success otherwise -1 469 */ 470 int ccp_probe_device(struct rte_pci_device *pci_dev); 471 472 /** 473 * allocate a ccp command queue 474 * 475 * @dev rte crypto device 476 * @param slot_req number of required 477 * @return allotted CCP queue on success otherwise NULL 478 */ 479 struct ccp_queue *ccp_allot_queue(struct rte_cryptodev *dev, int slot_req); 480 481 /** 482 * read hwrng value 483 * 484 * @param trng_value data pointer to write RNG value 485 * @return 0 on success otherwise -1 486 */ 487 int ccp_read_hwrng(uint32_t *trng_value); 488 489 #endif /* _CCP_DEV_H_ */ 490