1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2017 NXP 3 */ 4 5 #ifndef CAAM_JR_HW_SPECIFIC_H 6 #define CAAM_JR_HW_SPECIFIC_H 7 8 #include <caam_jr_config.h> 9 10 /* 11 * Offset to the registers of a job ring. 12 * Is different for each job ring. 13 */ 14 #define CHAN_BASE(jr) ((size_t)(jr)->register_base_addr) 15 16 #define SEC_JOB_RING_IS_FULL(pi, ci, ring_max_size, ring_threshold) \ 17 ((((pi) + 1 + ((ring_max_size) - (ring_threshold))) & \ 18 (ring_max_size - 1)) == ((ci))) 19 20 #define SEC_CIRCULAR_COUNTER(x, max) (((x) + 1) & (max - 1)) 21 22 /* 23 * Assert that cond is true. If !cond is true, display str and the vararg list 24 * in a printf-like syntax. also, if !cond is true, return altRet. 25 * 26 * \param cond A boolean expression to be asserted true 27 * \param altRet The value to be returned if cond doesn't hold true 28 * \param str A quoted char string 29 * 30 * E.g.: 31 * SEC_ASSERT(ret > 0, 0, "ERROR initializing app: code = %d\n", ret); 32 */ 33 #define SEC_ASSERT(cond, altRet, ...) do {\ 34 if (unlikely(!(cond))) {\ 35 CAAM_JR_ERR(__VA_ARGS__); \ 36 return altRet; \ 37 } \ 38 } while (0) 39 40 #define SEC_DP_ASSERT(cond, altRet, ...) do {\ 41 if (unlikely(!(cond))) {\ 42 CAAM_JR_DP_ERR(__VA_ARGS__); \ 43 return altRet; \ 44 } \ 45 } while (0) 46 47 #define ASSERT(x) 48 49 /* 50 * Constants representing various job ring registers 51 */ 52 #if CAAM_BYTE_ORDER == __BIG_ENDIAN 53 #define JR_REG_IRBA_OFFSET 0x0000 54 #define JR_REG_IRBA_OFFSET_LO 0x0004 55 #else 56 #define JR_REG_IRBA_OFFSET 0x0004 57 #define JR_REG_IRBA_OFFSET_LO 0x0000 58 #endif 59 60 #define JR_REG_IRSR_OFFSET 0x000C 61 #define JR_REG_IRSA_OFFSET 0x0014 62 #define JR_REG_IRJA_OFFSET 0x001C 63 64 #if CAAM_BYTE_ORDER == __BIG_ENDIAN 65 #define JR_REG_ORBA_OFFSET 0x0020 66 #define JR_REG_ORBA_OFFSET_LO 0x0024 67 #else 68 #define JR_REG_ORBA_OFFSET 0x0024 69 #define JR_REG_ORBA_OFFSET_LO 0x0020 70 #endif 71 72 #define JR_REG_ORSR_OFFSET 0x002C 73 #define JR_REG_ORJR_OFFSET 0x0034 74 #define JR_REG_ORSFR_OFFSET 0x003C 75 #define JR_REG_JROSR_OFFSET 0x0044 76 #define JR_REG_JRINT_OFFSET 0x004C 77 78 #define JR_REG_JRCFG_OFFSET 0x0050 79 #define JR_REG_JRCFG_OFFSET_LO 0x0054 80 81 #define JR_REG_IRRI_OFFSET 0x005C 82 #define JR_REG_ORWI_OFFSET 0x0064 83 #define JR_REG_JRCR_OFFSET 0x006C 84 85 /* 86 * Constants for error handling on job ring 87 */ 88 #define JR_REG_JRINT_ERR_TYPE_SHIFT 8 89 #define JR_REG_JRINT_ERR_ORWI_SHIFT 16 90 #define JR_REG_JRINIT_JRE_SHIFT 1 91 92 #define JRINT_JRE (1 << JR_REG_JRINIT_JRE_SHIFT) 93 #define JRINT_ERR_WRITE_STATUS (1 << JR_REG_JRINT_ERR_TYPE_SHIFT) 94 #define JRINT_ERR_BAD_INPUT_BASE (3 << JR_REG_JRINT_ERR_TYPE_SHIFT) 95 #define JRINT_ERR_BAD_OUTPUT_BASE (4 << JR_REG_JRINT_ERR_TYPE_SHIFT) 96 #define JRINT_ERR_WRITE_2_IRBA (5 << JR_REG_JRINT_ERR_TYPE_SHIFT) 97 #define JRINT_ERR_WRITE_2_ORBA (6 << JR_REG_JRINT_ERR_TYPE_SHIFT) 98 #define JRINT_ERR_RES_B4_HALT (7 << JR_REG_JRINT_ERR_TYPE_SHIFT) 99 #define JRINT_ERR_REM_TOO_MANY (8 << JR_REG_JRINT_ERR_TYPE_SHIFT) 100 #define JRINT_ERR_ADD_TOO_MANY (9 << JR_REG_JRINT_ERR_TYPE_SHIFT) 101 #define JRINT_ERR_HALT_MASK 0x0C 102 #define JRINT_ERR_HALT_INPROGRESS 0x04 103 #define JRINT_ERR_HALT_COMPLETE 0x08 104 105 #define JR_REG_JRCR_VAL_RESET 0x00000001 106 107 #define JR_REG_JRCFG_LO_ICTT_SHIFT 0x10 108 #define JR_REG_JRCFG_LO_ICDCT_SHIFT 0x08 109 #define JR_REG_JRCFG_LO_ICEN_EN 0x02 110 111 /* 112 * Constants for Descriptor Processing errors 113 */ 114 #define SEC_HW_ERR_SSRC_NO_SRC 0x00 115 #define SEC_HW_ERR_SSRC_CCB_ERR 0x02 116 #define SEC_HW_ERR_SSRC_JMP_HALT_U 0x03 117 #define SEC_HW_ERR_SSRC_DECO 0x04 118 #define SEC_HW_ERR_SSRC_JR 0x06 119 #define SEC_HW_ERR_SSRC_JMP_HALT_COND 0x07 120 121 #define SEC_HW_ERR_DECO_HFN_THRESHOLD 0xF1 122 #define SEC_HW_ERR_CCB_ICV_CHECK_FAIL 0x0A 123 124 /* 125 * Constants for descriptors 126 */ 127 /* Return higher 32 bits of physical address */ 128 #define PHYS_ADDR_HI(phys_addr) \ 129 (uint32_t)(((uint64_t)phys_addr) >> 32) 130 131 /* Return lower 32 bits of physical address */ 132 #define PHYS_ADDR_LO(phys_addr) \ 133 (uint32_t)(((uint64_t)phys_addr) & 0xFFFFFFFF) 134 135 /* 136 * Macros for extracting error codes for the job ring 137 */ 138 #define JR_REG_JRINT_ERR_TYPE_EXTRACT(value) ((value) & 0x00000F00) 139 #define JR_REG_JRINT_ERR_ORWI_EXTRACT(value) \ 140 (((value) & 0x3FFF0000) >> JR_REG_JRINT_ERR_ORWI_SHIFT) 141 #define JR_REG_JRINT_JRE_EXTRACT(value) ((value) & JRINT_JRE) 142 143 /* 144 * Macros for managing the job ring 145 */ 146 /* Read pointer to job ring input ring start address */ 147 #if defined(RTE_ARCH_ARM64) 148 #define hw_get_inp_queue_base(jr) ((((dma_addr_t)GET_JR_REG(IRBA, \ 149 (jr))) << 32) | \ 150 (GET_JR_REG_LO(IRBA, (jr)))) 151 152 /* Read pointer to job ring output ring start address */ 153 #define hw_get_out_queue_base(jr) (((dma_addr_t)(GET_JR_REG(ORBA, \ 154 (jr))) << 32) | \ 155 (GET_JR_REG_LO(ORBA, (jr)))) 156 #else 157 #define hw_get_inp_queue_base(jr) ((dma_addr_t)(GET_JR_REG_LO(IRBA, (jr)))) 158 159 #define hw_get_out_queue_base(jr) ((dma_addr_t)(GET_JR_REG_LO(ORBA, (jr)))) 160 #endif 161 162 /* 163 * IRJA - Input Ring Jobs Added Register shows 164 * how many new jobs were added to the Input Ring. 165 */ 166 #define hw_enqueue_desc_on_job_ring(job_ring) SET_JR_REG(IRJA, (job_ring), 1) 167 168 #define hw_set_input_ring_size(job_ring, size) SET_JR_REG(IRSR, job_ring, \ 169 (size)) 170 171 #define hw_set_output_ring_size(job_ring, size) SET_JR_REG(ORSR, job_ring, \ 172 (size)) 173 174 #if defined(RTE_ARCH_ARM64) 175 #define hw_set_input_ring_start_addr(job_ring, start_addr) \ 176 { \ 177 SET_JR_REG(IRBA, job_ring, PHYS_ADDR_HI(start_addr)); \ 178 SET_JR_REG_LO(IRBA, job_ring, PHYS_ADDR_LO(start_addr));\ 179 } 180 181 #define hw_set_output_ring_start_addr(job_ring, start_addr) \ 182 { \ 183 SET_JR_REG(ORBA, job_ring, PHYS_ADDR_HI(start_addr)); \ 184 SET_JR_REG_LO(ORBA, job_ring, PHYS_ADDR_LO(start_addr));\ 185 } 186 187 #else 188 #define hw_set_input_ring_start_addr(job_ring, start_addr) \ 189 { \ 190 SET_JR_REG(IRBA, job_ring, 0); \ 191 SET_JR_REG_LO(IRBA, job_ring, PHYS_ADDR_LO(start_addr));\ 192 } 193 194 #define hw_set_output_ring_start_addr(job_ring, start_addr) \ 195 { \ 196 SET_JR_REG(ORBA, job_ring, 0); \ 197 SET_JR_REG_LO(ORBA, job_ring, PHYS_ADDR_LO(start_addr));\ 198 } 199 #endif 200 201 /* ORJR - Output Ring Jobs Removed Register shows how many jobs were 202 * removed from the Output Ring for processing by software. This is done after 203 * the software has processed the entries. 204 */ 205 #define hw_remove_entries(jr, no_entries) SET_JR_REG(ORJR, (jr), (no_entries)) 206 207 /* IRSA - Input Ring Slots Available register holds the number of entries in 208 * the Job Ring's input ring. Once a job is enqueued, the value returned is 209 * decremented by the hardware by the number of jobs enqueued. 210 */ 211 #define hw_get_available_slots(jr) GET_JR_REG(IRSA, jr) 212 213 /* ORSFR - Output Ring Slots Full register holds the number of jobs which were 214 * processed by the SEC and can be retrieved by the software. Once a job has 215 * been processed by software, the user will call hw_remove_one_entry in order 216 * to notify the SEC that the entry was processed. 217 */ 218 #define hw_get_no_finished_jobs(jr) GET_JR_REG(ORSFR, jr) 219 220 /* 221 * Macros for manipulating JR registers 222 */ 223 #if CORE_BYTE_ORDER == CAAM_BYTE_ORDER 224 #define sec_read_32(addr) (*(volatile unsigned int *)(addr)) 225 #define sec_write_32(addr, val) (*(volatile unsigned int *)(addr) = (val)) 226 227 #else 228 #define sec_read_32(addr) rte_bswap32((*(volatile unsigned int *)(addr))) 229 #define sec_write_32(addr, val) \ 230 (*(volatile unsigned int *)(addr) = rte_bswap32(val)) 231 #endif 232 233 #if CAAM_BYTE_ORDER == __LITTLE_ENDIAN 234 #define sec_read_64(addr) (((u64)sec_read_32((u32 *)(addr) + 1) << 32) | \ 235 (sec_read_32((u32 *)(addr)))) 236 237 #define sec_write_64(addr, val) { \ 238 sec_write_32((u32 *)(addr) + 1, (u32)((val) >> 32)); \ 239 sec_write_32((u32 *)(addr), (u32)(val)); \ 240 } 241 #else /* CAAM_BYTE_ORDER == __BIG_ENDIAN */ 242 #define sec_read_64(addr) (((u64)sec_read_32((u32 *)(addr)) << 32) | \ 243 (sec_read_32((u32 *)(addr) + 1))) 244 245 #define sec_write_64(addr, val) { \ 246 sec_write_32((u32 *)(addr), (u32)((val) >> 32)); \ 247 sec_write_32((u32 *)(addr) + 1, (u32)(val)); \ 248 } 249 #endif 250 251 #if defined(RTE_ARCH_ARM64) 252 #define sec_read_addr(a) sec_read_64((a)) 253 #define sec_write_addr(a, v) sec_write_64((a), (v)) 254 #else 255 #define sec_read_addr(a) sec_read_32((a)) 256 #define sec_write_addr(a, v) sec_write_32((a), (v)) 257 #endif 258 259 #define JR_REG(name, jr) (CHAN_BASE(jr) + JR_REG_##name##_OFFSET) 260 #define JR_REG_LO(name, jr) (CHAN_BASE(jr) + JR_REG_##name##_OFFSET_LO) 261 262 #define GET_JR_REG(name, jr) (sec_read_32(JR_REG(name, (jr)))) 263 #define GET_JR_REG_LO(name, jr) (sec_read_32(JR_REG_LO(name, (jr)))) 264 265 #define SET_JR_REG(name, jr, value) \ 266 (sec_write_32(JR_REG(name, (jr)), value)) 267 #define SET_JR_REG_LO(name, jr, value) \ 268 (sec_write_32(JR_REG_LO(name, (jr)), value)) 269 270 /* Lists the possible states for a job ring. */ 271 typedef enum sec_job_ring_state_e { 272 SEC_JOB_RING_STATE_STARTED, /* Job ring is initialized */ 273 SEC_JOB_RING_STATE_RESET, /* Job ring reset is in progress */ 274 } sec_job_ring_state_t; 275 276 /* code or cmd block to caam */ 277 struct sec_cdb { 278 struct __rte_packed_begin { 279 union __rte_packed_begin { 280 uint32_t word; 281 struct { 282 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN 283 uint16_t rsvd63_48; 284 unsigned int rsvd47_39:9; 285 unsigned int idlen:7; 286 #else 287 unsigned int idlen:7; 288 unsigned int rsvd47_39:9; 289 uint16_t rsvd63_48; 290 #endif 291 } field; 292 } __rte_packed_end hi; 293 294 union __rte_packed_begin { 295 uint32_t word; 296 struct { 297 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN 298 unsigned int rsvd31_30:2; 299 unsigned int fsgt:1; 300 unsigned int lng:1; 301 unsigned int offset:2; 302 unsigned int abs:1; 303 unsigned int add_buf:1; 304 uint8_t pool_id; 305 uint16_t pool_buffer_size; 306 #else 307 uint16_t pool_buffer_size; 308 uint8_t pool_id; 309 unsigned int add_buf:1; 310 unsigned int abs:1; 311 unsigned int offset:2; 312 unsigned int lng:1; 313 unsigned int fsgt:1; 314 unsigned int rsvd31_30:2; 315 #endif 316 } field; 317 } __rte_packed_end lo; 318 } __rte_packed_end sh_hdr; 319 320 uint32_t sh_desc[SEC_JOB_DESCRIPTOR_SIZE]; 321 }; 322 323 struct caam_jr_qp { 324 struct sec_job_ring_t *ring; 325 uint64_t rx_pkts; 326 uint64_t rx_errs; 327 uint64_t rx_poll_err; 328 uint64_t tx_pkts; 329 uint64_t tx_errs; 330 uint64_t tx_ring_full; 331 }; 332 333 struct sec_job_ring_t { 334 /* TODO: Add wrapper macro to make it obvious this is the consumer index 335 * on the output ring 336 */ 337 uint32_t cidx; /* Consumer index for job ring (jobs array). 338 * @note: cidx and pidx are accessed from 339 * different threads. Place the cidx and pidx 340 * inside the structure so that they lay on 341 * different cachelines, to avoid false sharing 342 * between threads when the threads run on 343 * different cores! 344 */ 345 /* TODO: Add wrapper macro to make it obvious this is the producer index 346 * on the input ring 347 */ 348 uint32_t pidx; /* Producer index for job ring (jobs array) */ 349 350 phys_addr_t *input_ring;/* Ring of output descriptors received from SEC. 351 * Size of array is power of 2 to allow fast 352 * update of producer/consumer indexes with 353 * bitwise operations. 354 */ 355 356 struct sec_outring_entry *output_ring; 357 /* Ring of output descriptors received from SEC. 358 * Size of array is power of 2 to allow fast 359 * update of producer/consumer indexes with 360 * bitwise operations. 361 */ 362 363 int irq_fd; /* The file descriptor used for polling from 364 * user space for interrupts notifications 365 */ 366 uint32_t jr_mode; /* Model used by SEC Driver to receive 367 * notifications from SEC. Can be either 368 * of the three: #SEC_NOTIFICATION_TYPE_NAPI 369 * #SEC_NOTIFICATION_TYPE_IRQ or 370 * #SEC_NOTIFICATION_TYPE_POLL 371 */ 372 uint32_t napi_mode; /* Job ring mode if NAPI mode is chosen 373 * Used only when jr_mode is set to 374 * #SEC_NOTIFICATION_TYPE_NAPI 375 */ 376 void *register_base_addr; /* Base address for SEC's 377 * register memory for this job ring. 378 */ 379 uint8_t coalescing_en; /* notifies if coalescing is 380 * enabled for the job ring 381 */ 382 sec_job_ring_state_t jr_state; /* The state of this job ring */ 383 384 struct rte_mempool *ctx_pool; /* per dev mempool for caam_jr_op_ctx */ 385 unsigned int max_nb_queue_pairs; 386 unsigned int max_nb_sessions; 387 struct caam_jr_qp qps[RTE_CAAM_MAX_NB_SEC_QPS]; /* i/o queue for sec */ 388 }; 389 390 /* Union describing the possible error codes that 391 * can be set in the descriptor status word 392 */ 393 union __rte_packed_begin hw_error_code { 394 uint32_t error; 395 union __rte_packed_begin { 396 struct __rte_packed_begin { 397 uint32_t ssrc:4; 398 uint32_t ssed_val:28; 399 } __rte_packed_end value; 400 struct __rte_packed_begin { 401 uint32_t ssrc:4; 402 uint32_t res:28; 403 } __rte_packed_end no_status_src; 404 struct __rte_packed_begin { 405 uint32_t ssrc:4; 406 uint32_t jmp:1; 407 uint32_t res:11; 408 uint32_t desc_idx:8; 409 uint32_t cha_id:4; 410 uint32_t err_id:4; 411 } __rte_packed_end ccb_status_src; 412 struct __rte_packed_begin { 413 uint32_t ssrc:4; 414 uint32_t jmp:1; 415 uint32_t res:11; 416 uint32_t desc_idx:8; 417 uint32_t offset:8; 418 } __rte_packed_end jmp_halt_user_src; 419 struct __rte_packed_begin { 420 uint32_t ssrc:4; 421 uint32_t jmp:1; 422 uint32_t res:11; 423 uint32_t desc_idx:8; 424 uint32_t desc_err:8; 425 } __rte_packed_end deco_src; 426 struct __rte_packed_begin { 427 uint32_t ssrc:4; 428 uint32_t res:17; 429 uint32_t naddr:3; 430 uint32_t desc_err:8; 431 } __rte_packed_end jr_src; 432 struct __rte_packed_begin { 433 uint32_t ssrc:4; 434 uint32_t jmp:1; 435 uint32_t res:11; 436 uint32_t desc_idx:8; 437 uint32_t cond:8; 438 } __rte_packed_end jmp_halt_cond_src; 439 } __rte_packed_end error_desc; 440 } __rte_packed_end; 441 442 /* @brief Initialize a job ring/channel in SEC device. 443 * Write configuration register/s to properly initialize a job ring. 444 * 445 * @param [in] job_ring The job ring 446 * 447 * @retval 0 for success 448 * @retval other for error 449 */ 450 int hw_reset_job_ring(struct sec_job_ring_t *job_ring); 451 452 /* @brief Reset a job ring/channel in SEC device. 453 * Write configuration register/s to reset a job ring. 454 * 455 * @param [in] job_ring The job ring 456 * 457 * @retval 0 for success 458 * @retval -1 in case job ring reset failed 459 */ 460 int hw_shutdown_job_ring(struct sec_job_ring_t *job_ring); 461 462 /* @brief Handle a job ring/channel error in SEC device. 463 * Identify the error type and clear error bits if required. 464 * 465 * @param [in] job_ring The job ring 466 * @param [in] sec_error_code The job ring's error code 467 */ 468 void hw_handle_job_ring_error(struct sec_job_ring_t *job_ring, 469 uint32_t sec_error_code); 470 471 /* @brief Handle a job ring error in the device. 472 * Identify the error type and printout a explanatory 473 * messages. 474 * 475 * @param [in] job_ring The job ring 476 * 477 */ 478 void hw_job_ring_error_print(struct sec_job_ring_t *job_ring, int code); 479 480 /* @brief Set interrupt coalescing parameters on the Job Ring. 481 * @param [in] job_ring The job ring 482 * @param [in] irq_coalescing_timer Interrupt coalescing timer threshold. 483 * This value determines the maximum 484 * amount of time after processing a 485 * descriptor before raising an interrupt. 486 * @param [in] irq_coalescing_count Interrupt coalescing descriptor count 487 * threshold. 488 */ 489 int hw_job_ring_set_coalescing_param(struct sec_job_ring_t *job_ring, 490 uint16_t irq_coalescing_timer, 491 uint8_t irq_coalescing_count); 492 493 /* @brief Enable interrupt coalescing on a job ring 494 * @param [in] job_ring The job ring 495 */ 496 int hw_job_ring_enable_coalescing(struct sec_job_ring_t *job_ring); 497 498 /* @brief Disable interrupt coalescing on a job ring 499 * @param [in] job_ring The job ring 500 */ 501 int hw_job_ring_disable_coalescing(struct sec_job_ring_t *job_ring); 502 503 #endif /* CAAM_JR_HW_SPECIFIC_H */ 504