1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Advanced Micro Devices, Inc. All rights reserved. 3 */ 4 5 #ifndef _CCP_DEV_H_ 6 #define _CCP_DEV_H_ 7 8 #include <limits.h> 9 #include <stdbool.h> 10 #include <stdint.h> 11 #include <string.h> 12 13 #include <rte_bus_pci.h> 14 #include <rte_atomic.h> 15 #include <rte_byteorder.h> 16 #include <rte_io.h> 17 #include <rte_pci.h> 18 #include <rte_spinlock.h> 19 #include <rte_crypto_sym.h> 20 #include <rte_cryptodev.h> 21 22 /**< CCP sspecific */ 23 #define MAX_HW_QUEUES 5 24 #define CCP_MAX_TRNG_RETRIES 10 25 #define CCP_ALIGN(x, y) ((((x) + (y - 1)) / y) * y) 26 27 /**< CCP Register Mappings */ 28 #define Q_MASK_REG 0x000 29 #define TRNG_OUT_REG 0x00c 30 31 /* CCP Version 5 Specifics */ 32 #define CMD_QUEUE_MASK_OFFSET 0x00 33 #define CMD_QUEUE_PRIO_OFFSET 0x04 34 #define CMD_REQID_CONFIG_OFFSET 0x08 35 #define CMD_CMD_TIMEOUT_OFFSET 0x10 36 #define LSB_PUBLIC_MASK_LO_OFFSET 0x18 37 #define LSB_PUBLIC_MASK_HI_OFFSET 0x1C 38 #define LSB_PRIVATE_MASK_LO_OFFSET 0x20 39 #define LSB_PRIVATE_MASK_HI_OFFSET 0x24 40 41 #define CMD_Q_CONTROL_BASE 0x0000 42 #define CMD_Q_TAIL_LO_BASE 0x0004 43 #define CMD_Q_HEAD_LO_BASE 0x0008 44 #define CMD_Q_INT_ENABLE_BASE 0x000C 45 #define CMD_Q_INTERRUPT_STATUS_BASE 0x0010 46 47 #define CMD_Q_STATUS_BASE 0x0100 48 #define CMD_Q_INT_STATUS_BASE 0x0104 49 50 #define CMD_CONFIG_0_OFFSET 0x6000 51 #define CMD_TRNG_CTL_OFFSET 0x6008 52 #define CMD_AES_MASK_OFFSET 0x6010 53 #define CMD_CLK_GATE_CTL_OFFSET 0x603C 54 55 /* Address offset between two virtual queue registers */ 56 #define CMD_Q_STATUS_INCR 0x1000 57 58 /* Bit masks */ 59 #define CMD_Q_RUN 0x1 60 #define CMD_Q_SIZE 0x1F 61 #define CMD_Q_SHIFT 3 62 #define COMMANDS_PER_QUEUE 8192 63 64 #define QUEUE_SIZE_VAL ((ffs(COMMANDS_PER_QUEUE) - 2) & \ 65 CMD_Q_SIZE) 66 #define Q_DESC_SIZE sizeof(struct ccp_desc) 67 #define Q_SIZE(n) (COMMANDS_PER_QUEUE*(n)) 68 69 #define INT_COMPLETION 0x1 70 #define INT_ERROR 0x2 71 #define INT_QUEUE_STOPPED 0x4 72 #define ALL_INTERRUPTS (INT_COMPLETION| \ 73 INT_ERROR| \ 74 INT_QUEUE_STOPPED) 75 76 #define LSB_REGION_WIDTH 5 77 #define MAX_LSB_CNT 8 78 79 #define LSB_SIZE 16 80 #define LSB_ITEM_SIZE 32 81 #define SLSB_MAP_SIZE (MAX_LSB_CNT * LSB_SIZE) 82 #define LSB_ENTRY_NUMBER(LSB_ADDR) (LSB_ADDR / LSB_ITEM_SIZE) 83 84 /* General CCP Defines */ 85 86 #define CCP_SB_BYTES 32 87 /* Word 0 */ 88 #define CCP_CMD_DW0(p) ((p)->dw0) 89 #define CCP_CMD_SOC(p) (CCP_CMD_DW0(p).soc) 90 #define CCP_CMD_IOC(p) (CCP_CMD_DW0(p).ioc) 91 #define CCP_CMD_INIT(p) (CCP_CMD_DW0(p).init) 92 #define CCP_CMD_EOM(p) (CCP_CMD_DW0(p).eom) 93 #define CCP_CMD_FUNCTION(p) (CCP_CMD_DW0(p).function) 94 #define CCP_CMD_ENGINE(p) (CCP_CMD_DW0(p).engine) 95 #define CCP_CMD_PROT(p) (CCP_CMD_DW0(p).prot) 96 97 /* Word 1 */ 98 #define CCP_CMD_DW1(p) ((p)->length) 99 #define CCP_CMD_LEN(p) (CCP_CMD_DW1(p)) 100 101 /* Word 2 */ 102 #define CCP_CMD_DW2(p) ((p)->src_lo) 103 #define CCP_CMD_SRC_LO(p) (CCP_CMD_DW2(p)) 104 105 /* Word 3 */ 106 #define CCP_CMD_DW3(p) ((p)->dw3) 107 #define CCP_CMD_SRC_MEM(p) ((p)->dw3.src_mem) 108 #define CCP_CMD_SRC_HI(p) ((p)->dw3.src_hi) 109 #define CCP_CMD_LSB_ID(p) ((p)->dw3.lsb_cxt_id) 110 #define CCP_CMD_FIX_SRC(p) ((p)->dw3.fixed) 111 112 /* Words 4/5 */ 113 #define CCP_CMD_DW4(p) ((p)->dw4) 114 #define CCP_CMD_DST_LO(p) (CCP_CMD_DW4(p).dst_lo) 115 #define CCP_CMD_DW5(p) ((p)->dw5.fields.dst_hi) 116 #define CCP_CMD_DST_HI(p) (CCP_CMD_DW5(p)) 117 #define CCP_CMD_DST_MEM(p) ((p)->dw5.fields.dst_mem) 118 #define CCP_CMD_FIX_DST(p) ((p)->dw5.fields.fixed) 119 #define CCP_CMD_SHA_LO(p) ((p)->dw4.sha_len_lo) 120 #define CCP_CMD_SHA_HI(p) ((p)->dw5.sha_len_hi) 121 122 /* Word 6/7 */ 123 #define CCP_CMD_DW6(p) ((p)->key_lo) 124 #define CCP_CMD_KEY_LO(p) (CCP_CMD_DW6(p)) 125 #define CCP_CMD_DW7(p) ((p)->dw7) 126 #define CCP_CMD_KEY_HI(p) ((p)->dw7.key_hi) 127 #define CCP_CMD_KEY_MEM(p) ((p)->dw7.key_mem) 128 129 /* bitmap */ 130 enum { 131 BITS_PER_WORD = sizeof(unsigned long) * CHAR_BIT 132 }; 133 134 #define WORD_OFFSET(b) ((b) / BITS_PER_WORD) 135 #define BIT_OFFSET(b) ((b) % BITS_PER_WORD) 136 137 #define CCP_DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d)) 138 #define CCP_BITMAP_SIZE(nr) \ 139 CCP_DIV_ROUND_UP(nr, CHAR_BIT * sizeof(unsigned long)) 140 141 #define CCP_BITMAP_FIRST_WORD_MASK(start) \ 142 (~0UL << ((start) & (BITS_PER_WORD - 1))) 143 #define CCP_BITMAP_LAST_WORD_MASK(nbits) \ 144 (~0UL >> (-(nbits) & (BITS_PER_WORD - 1))) 145 146 #define __ccp_round_mask(x, y) ((typeof(x))((y)-1)) 147 #define ccp_round_down(x, y) ((x) & ~__ccp_round_mask(x, y)) 148 149 /** CCP registers Write/Read */ 150 151 static inline void ccp_pci_reg_write(void *base, int offset, 152 uint32_t value) 153 { 154 volatile void *reg_addr = ((uint8_t *)base + offset); 155 156 rte_write32((rte_cpu_to_le_32(value)), reg_addr); 157 } 158 159 static inline uint32_t ccp_pci_reg_read(void *base, int offset) 160 { 161 volatile void *reg_addr = ((uint8_t *)base + offset); 162 163 return rte_le_to_cpu_32(rte_read32(reg_addr)); 164 } 165 166 #define CCP_READ_REG(hw_addr, reg_offset) \ 167 ccp_pci_reg_read(hw_addr, reg_offset) 168 169 #define CCP_WRITE_REG(hw_addr, reg_offset, value) \ 170 ccp_pci_reg_write(hw_addr, reg_offset, value) 171 172 TAILQ_HEAD(ccp_list, ccp_device); 173 174 extern struct ccp_list ccp_list; 175 176 /** 177 * CCP device version 178 */ 179 enum ccp_device_version { 180 CCP_VERSION_5A = 0, 181 CCP_VERSION_5B, 182 }; 183 184 /** 185 * A structure describing a CCP command queue. 186 */ 187 struct ccp_queue { 188 struct ccp_device *dev; 189 char memz_name[RTE_MEMZONE_NAMESIZE]; 190 191 rte_atomic64_t free_slots; 192 /**< available free slots updated from enq/deq calls */ 193 194 /* Queue identifier */ 195 uint64_t id; /**< queue id */ 196 uint64_t qidx; /**< queue index */ 197 uint64_t qsize; /**< queue size */ 198 199 /* Queue address */ 200 struct ccp_desc *qbase_desc; 201 void *qbase_addr; 202 phys_addr_t qbase_phys_addr; 203 /**< queue-page registers addr */ 204 void *reg_base; 205 206 uint32_t qcontrol; 207 /**< queue ctrl reg */ 208 209 int lsb; 210 /**< lsb region assigned to queue */ 211 unsigned long lsbmask; 212 /**< lsb regions queue can access */ 213 unsigned long lsbmap[CCP_BITMAP_SIZE(LSB_SIZE)]; 214 /**< all lsb resources which queue is using */ 215 uint32_t sb_key; 216 /**< lsb assigned for queue */ 217 uint32_t sb_iv; 218 /**< lsb assigned for iv */ 219 uint32_t sb_sha; 220 /**< lsb assigned for sha ctx */ 221 uint32_t sb_hmac; 222 /**< lsb assigned for hmac ctx */ 223 } __rte_cache_aligned; 224 225 /** 226 * A structure describing a CCP device. 227 */ 228 struct ccp_device { 229 TAILQ_ENTRY(ccp_device) next; 230 int id; 231 /**< ccp dev id on platform */ 232 struct ccp_queue cmd_q[MAX_HW_QUEUES]; 233 /**< ccp queue */ 234 int cmd_q_count; 235 /**< no. of ccp Queues */ 236 struct rte_pci_device pci; 237 /**< ccp pci identifier */ 238 unsigned long lsbmap[CCP_BITMAP_SIZE(SLSB_MAP_SIZE)]; 239 /**< shared lsb mask of ccp */ 240 rte_spinlock_t lsb_lock; 241 /**< protection for shared lsb region allocation */ 242 int qidx; 243 /**< current queue index */ 244 int hwrng_retries; 245 /**< retry counter for CCP TRNG */ 246 } __rte_cache_aligned; 247 248 /**< CCP H/W engine related */ 249 /** 250 * ccp_engine - CCP operation identifiers 251 * 252 * @CCP_ENGINE_AES: AES operation 253 * @CCP_ENGINE_XTS_AES: 128-bit XTS AES operation 254 * @CCP_ENGINE_3DES: DES/3DES operation 255 * @CCP_ENGINE_SHA: SHA operation 256 * @CCP_ENGINE_RSA: RSA operation 257 * @CCP_ENGINE_PASSTHRU: pass-through operation 258 * @CCP_ENGINE_ZLIB_DECOMPRESS: unused 259 * @CCP_ENGINE_ECC: ECC operation 260 */ 261 enum ccp_engine { 262 CCP_ENGINE_AES = 0, 263 CCP_ENGINE_XTS_AES_128, 264 CCP_ENGINE_3DES, 265 CCP_ENGINE_SHA, 266 CCP_ENGINE_RSA, 267 CCP_ENGINE_PASSTHRU, 268 CCP_ENGINE_ZLIB_DECOMPRESS, 269 CCP_ENGINE_ECC, 270 CCP_ENGINE__LAST, 271 }; 272 273 /* Passthru engine */ 274 /** 275 * ccp_passthru_bitwise - type of bitwise passthru operation 276 * 277 * @CCP_PASSTHRU_BITWISE_NOOP: no bitwise operation performed 278 * @CCP_PASSTHRU_BITWISE_AND: perform bitwise AND of src with mask 279 * @CCP_PASSTHRU_BITWISE_OR: perform bitwise OR of src with mask 280 * @CCP_PASSTHRU_BITWISE_XOR: perform bitwise XOR of src with mask 281 * @CCP_PASSTHRU_BITWISE_MASK: overwrite with mask 282 */ 283 enum ccp_passthru_bitwise { 284 CCP_PASSTHRU_BITWISE_NOOP = 0, 285 CCP_PASSTHRU_BITWISE_AND, 286 CCP_PASSTHRU_BITWISE_OR, 287 CCP_PASSTHRU_BITWISE_XOR, 288 CCP_PASSTHRU_BITWISE_MASK, 289 CCP_PASSTHRU_BITWISE__LAST, 290 }; 291 292 /** 293 * ccp_passthru_byteswap - type of byteswap passthru operation 294 * 295 * @CCP_PASSTHRU_BYTESWAP_NOOP: no byte swapping performed 296 * @CCP_PASSTHRU_BYTESWAP_32BIT: swap bytes within 32-bit words 297 * @CCP_PASSTHRU_BYTESWAP_256BIT: swap bytes within 256-bit words 298 */ 299 enum ccp_passthru_byteswap { 300 CCP_PASSTHRU_BYTESWAP_NOOP = 0, 301 CCP_PASSTHRU_BYTESWAP_32BIT, 302 CCP_PASSTHRU_BYTESWAP_256BIT, 303 CCP_PASSTHRU_BYTESWAP__LAST, 304 }; 305 306 /** 307 * CCP passthru 308 */ 309 struct ccp_passthru { 310 phys_addr_t src_addr; 311 phys_addr_t dest_addr; 312 enum ccp_passthru_bitwise bit_mod; 313 enum ccp_passthru_byteswap byte_swap; 314 int len; 315 int dir; 316 }; 317 318 /* CCP version 5: Union to define the function field (cmd_reg1/dword0) */ 319 union ccp_function { 320 struct { 321 uint16_t size:7; 322 uint16_t encrypt:1; 323 uint16_t mode:5; 324 uint16_t type:2; 325 } aes; 326 struct { 327 uint16_t size:7; 328 uint16_t encrypt:1; 329 uint16_t mode:5; 330 uint16_t type:2; 331 } des; 332 struct { 333 uint16_t size:7; 334 uint16_t encrypt:1; 335 uint16_t rsvd:5; 336 uint16_t type:2; 337 } aes_xts; 338 struct { 339 uint16_t rsvd1:10; 340 uint16_t type:4; 341 uint16_t rsvd2:1; 342 } sha; 343 struct { 344 uint16_t mode:3; 345 uint16_t size:12; 346 } rsa; 347 struct { 348 uint16_t byteswap:2; 349 uint16_t bitwise:3; 350 uint16_t reflect:2; 351 uint16_t rsvd:8; 352 } pt; 353 struct { 354 uint16_t rsvd:13; 355 } zlib; 356 struct { 357 uint16_t size:10; 358 uint16_t type:2; 359 uint16_t mode:3; 360 } ecc; 361 uint16_t raw; 362 }; 363 364 365 /** 366 * descriptor for version 5 CPP commands 367 * 8 32-bit words: 368 * word 0: function; engine; control bits 369 * word 1: length of source data 370 * word 2: low 32 bits of source pointer 371 * word 3: upper 16 bits of source pointer; source memory type 372 * word 4: low 32 bits of destination pointer 373 * word 5: upper 16 bits of destination pointer; destination memory 374 * type 375 * word 6: low 32 bits of key pointer 376 * word 7: upper 16 bits of key pointer; key memory type 377 */ 378 struct dword0 { 379 uint32_t soc:1; 380 uint32_t ioc:1; 381 uint32_t rsvd1:1; 382 uint32_t init:1; 383 uint32_t eom:1; 384 uint32_t function:15; 385 uint32_t engine:4; 386 uint32_t prot:1; 387 uint32_t rsvd2:7; 388 }; 389 390 struct dword3 { 391 uint32_t src_hi:16; 392 uint32_t src_mem:2; 393 uint32_t lsb_cxt_id:8; 394 uint32_t rsvd1:5; 395 uint32_t fixed:1; 396 }; 397 398 union dword4 { 399 uint32_t dst_lo; /* NON-SHA */ 400 uint32_t sha_len_lo; /* SHA */ 401 }; 402 403 union dword5 { 404 struct { 405 uint32_t dst_hi:16; 406 uint32_t dst_mem:2; 407 uint32_t rsvd1:13; 408 uint32_t fixed:1; 409 } 410 fields; 411 uint32_t sha_len_hi; 412 }; 413 414 struct dword7 { 415 uint32_t key_hi:16; 416 uint32_t key_mem:2; 417 uint32_t rsvd1:14; 418 }; 419 420 struct ccp_desc { 421 struct dword0 dw0; 422 uint32_t length; 423 uint32_t src_lo; 424 struct dword3 dw3; 425 union dword4 dw4; 426 union dword5 dw5; 427 uint32_t key_lo; 428 struct dword7 dw7; 429 }; 430 431 /** 432 * ccp memory type 433 */ 434 enum ccp_memtype { 435 CCP_MEMTYPE_SYSTEM = 0, 436 CCP_MEMTYPE_SB, 437 CCP_MEMTYPE_LOCAL, 438 CCP_MEMTYPE_LAST, 439 }; 440 441 /** 442 * cmd id to follow order 443 */ 444 enum ccp_cmd_order { 445 CCP_CMD_CIPHER = 0, 446 CCP_CMD_AUTH, 447 CCP_CMD_CIPHER_HASH, 448 CCP_CMD_HASH_CIPHER, 449 CCP_CMD_COMBINED, 450 CCP_CMD_NOT_SUPPORTED, 451 }; 452 453 static inline uint32_t 454 low32_value(unsigned long addr) 455 { 456 return ((uint64_t)addr) & 0x0ffffffff; 457 } 458 459 static inline uint32_t 460 high32_value(unsigned long addr) 461 { 462 return ((uint64_t)addr >> 32) & 0x00000ffff; 463 } 464 465 /* 466 * Start CCP device 467 */ 468 int ccp_dev_start(struct rte_cryptodev *dev); 469 470 /** 471 * Detect ccp platform and initialize all ccp devices 472 * 473 * @param ccp_id rte_pci_id list for supported CCP devices 474 * @return no. of successfully initialized CCP devices 475 */ 476 int ccp_probe_devices(const struct rte_pci_id *ccp_id); 477 478 /** 479 * allocate a ccp command queue 480 * 481 * @dev rte crypto device 482 * @param slot_req number of required 483 * @return allotted CCP queue on success otherwise NULL 484 */ 485 struct ccp_queue *ccp_allot_queue(struct rte_cryptodev *dev, int slot_req); 486 487 /** 488 * read hwrng value 489 * 490 * @param trng_value data pointer to write RNG value 491 * @return 0 on success otherwise -1 492 */ 493 int ccp_read_hwrng(uint32_t *trng_value); 494 495 #endif /* _CCP_DEV_H_ */ 496