1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Cavium, Inc 3 */ 4 #include <assert.h> 5 #include <string.h> 6 #include <unistd.h> 7 8 #include <rte_branch_prediction.h> 9 #include <rte_common.h> 10 #include <rte_errno.h> 11 #include <rte_memzone.h> 12 13 #include "otx_cryptodev_hw_access.h" 14 #include "otx_cryptodev_mbox.h" 15 16 #include "cpt_pmd_logs.h" 17 #include "cpt_hw_types.h" 18 19 /* 20 * VF HAL functions 21 * Access its own BAR0/4 registers by passing VF number as 0. 22 * OS/PCI maps them accordingly. 23 */ 24 25 static int 26 otx_cpt_vf_init(struct cpt_vf *cptvf) 27 { 28 int ret = 0; 29 30 /* Check ready with PF */ 31 /* Gets chip ID / device Id from PF if ready */ 32 ret = otx_cpt_check_pf_ready(cptvf); 33 if (ret) { 34 CPT_LOG_ERR("%s: PF not responding to READY msg", 35 cptvf->dev_name); 36 ret = -EBUSY; 37 goto exit; 38 } 39 40 CPT_LOG_DP_DEBUG("%s: %s done", cptvf->dev_name, __func__); 41 42 exit: 43 return ret; 44 } 45 46 /* 47 * Read Interrupt status of the VF 48 * 49 * @param cptvf cptvf structure 50 */ 51 static uint64_t 52 otx_cpt_read_vf_misc_intr_status(struct cpt_vf *cptvf) 53 { 54 return CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf), CPTX_VQX_MISC_INT(0, 0)); 55 } 56 57 /* 58 * Clear mailbox interrupt of the VF 59 * 60 * @param cptvf cptvf structure 61 */ 62 static void 63 otx_cpt_clear_mbox_intr(struct cpt_vf *cptvf) 64 { 65 cptx_vqx_misc_int_t vqx_misc_int; 66 67 vqx_misc_int.u = CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf), 68 CPTX_VQX_MISC_INT(0, 0)); 69 /* W1C for the VF */ 70 vqx_misc_int.s.mbox = 1; 71 CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf), 72 CPTX_VQX_MISC_INT(0, 0), vqx_misc_int.u); 73 } 74 75 /* 76 * Clear instruction NCB read error interrupt of the VF 77 * 78 * @param cptvf cptvf structure 79 */ 80 static void 81 otx_cpt_clear_irde_intr(struct cpt_vf *cptvf) 82 { 83 cptx_vqx_misc_int_t vqx_misc_int; 84 85 vqx_misc_int.u = CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf), 86 CPTX_VQX_MISC_INT(0, 0)); 87 /* W1C for the VF */ 88 vqx_misc_int.s.irde = 1; 89 CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf), 90 CPTX_VQX_MISC_INT(0, 0), vqx_misc_int.u); 91 } 92 93 /* 94 * Clear NCB result write response error interrupt of the VF 95 * 96 * @param cptvf cptvf structure 97 */ 98 static void 99 otx_cpt_clear_nwrp_intr(struct cpt_vf *cptvf) 100 { 101 cptx_vqx_misc_int_t vqx_misc_int; 102 103 vqx_misc_int.u = CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf), 104 CPTX_VQX_MISC_INT(0, 0)); 105 /* W1C for the VF */ 106 vqx_misc_int.s.nwrp = 1; 107 CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf), 108 CPTX_VQX_MISC_INT(0, 0), vqx_misc_int.u); 109 } 110 111 /* 112 * Clear swerr interrupt of the VF 113 * 114 * @param cptvf cptvf structure 115 */ 116 static void 117 otx_cpt_clear_swerr_intr(struct cpt_vf *cptvf) 118 { 119 cptx_vqx_misc_int_t vqx_misc_int; 120 121 vqx_misc_int.u = CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf), 122 CPTX_VQX_MISC_INT(0, 0)); 123 /* W1C for the VF */ 124 vqx_misc_int.s.swerr = 1; 125 CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf), 126 CPTX_VQX_MISC_INT(0, 0), vqx_misc_int.u); 127 } 128 129 /* 130 * Clear hwerr interrupt of the VF 131 * 132 * @param cptvf cptvf structure 133 */ 134 static void 135 otx_cpt_clear_hwerr_intr(struct cpt_vf *cptvf) 136 { 137 cptx_vqx_misc_int_t vqx_misc_int; 138 139 vqx_misc_int.u = CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf), 140 CPTX_VQX_MISC_INT(0, 0)); 141 /* W1C for the VF */ 142 vqx_misc_int.s.hwerr = 1; 143 CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf), 144 CPTX_VQX_MISC_INT(0, 0), vqx_misc_int.u); 145 } 146 147 /* 148 * Clear translation fault interrupt of the VF 149 * 150 * @param cptvf cptvf structure 151 */ 152 static void 153 otx_cpt_clear_fault_intr(struct cpt_vf *cptvf) 154 { 155 cptx_vqx_misc_int_t vqx_misc_int; 156 157 vqx_misc_int.u = CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf), 158 CPTX_VQX_MISC_INT(0, 0)); 159 /* W1C for the VF */ 160 vqx_misc_int.s.fault = 1; 161 CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf), 162 CPTX_VQX_MISC_INT(0, 0), vqx_misc_int.u); 163 } 164 165 /* 166 * Clear doorbell overflow interrupt of the VF 167 * 168 * @param cptvf cptvf structure 169 */ 170 static void 171 otx_cpt_clear_dovf_intr(struct cpt_vf *cptvf) 172 { 173 cptx_vqx_misc_int_t vqx_misc_int; 174 175 vqx_misc_int.u = CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf), 176 CPTX_VQX_MISC_INT(0, 0)); 177 /* W1C for the VF */ 178 vqx_misc_int.s.dovf = 1; 179 CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf), 180 CPTX_VQX_MISC_INT(0, 0), vqx_misc_int.u); 181 } 182 183 /* Write to VQX_CTL register 184 */ 185 static void 186 otx_cpt_write_vq_ctl(struct cpt_vf *cptvf, bool val) 187 { 188 cptx_vqx_ctl_t vqx_ctl; 189 190 vqx_ctl.u = CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf), 191 CPTX_VQX_CTL(0, 0)); 192 vqx_ctl.s.ena = val; 193 CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf), 194 CPTX_VQX_CTL(0, 0), vqx_ctl.u); 195 } 196 197 /* Write to VQX_INPROG register 198 */ 199 static void 200 otx_cpt_write_vq_inprog(struct cpt_vf *cptvf, uint8_t val) 201 { 202 cptx_vqx_inprog_t vqx_inprg; 203 204 vqx_inprg.u = CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf), 205 CPTX_VQX_INPROG(0, 0)); 206 vqx_inprg.s.inflight = val; 207 CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf), 208 CPTX_VQX_INPROG(0, 0), vqx_inprg.u); 209 } 210 211 /* Write to VQX_DONE_WAIT NUMWAIT register 212 */ 213 static void 214 otx_cpt_write_vq_done_numwait(struct cpt_vf *cptvf, uint32_t val) 215 { 216 cptx_vqx_done_wait_t vqx_dwait; 217 218 vqx_dwait.u = CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf), 219 CPTX_VQX_DONE_WAIT(0, 0)); 220 vqx_dwait.s.num_wait = val; 221 CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf), 222 CPTX_VQX_DONE_WAIT(0, 0), vqx_dwait.u); 223 } 224 225 /* Write to VQX_DONE_WAIT NUM_WAIT register 226 */ 227 static void 228 otx_cpt_write_vq_done_timewait(struct cpt_vf *cptvf, uint16_t val) 229 { 230 cptx_vqx_done_wait_t vqx_dwait; 231 232 vqx_dwait.u = CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf), 233 CPTX_VQX_DONE_WAIT(0, 0)); 234 vqx_dwait.s.time_wait = val; 235 CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf), 236 CPTX_VQX_DONE_WAIT(0, 0), vqx_dwait.u); 237 } 238 239 /* Write to VQX_SADDR register 240 */ 241 static void 242 otx_cpt_write_vq_saddr(struct cpt_vf *cptvf, uint64_t val) 243 { 244 cptx_vqx_saddr_t vqx_saddr; 245 246 vqx_saddr.u = val; 247 CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf), 248 CPTX_VQX_SADDR(0, 0), vqx_saddr.u); 249 } 250 251 static void 252 otx_cpt_vfvq_init(struct cpt_vf *cptvf) 253 { 254 uint64_t base_addr = 0; 255 256 /* Disable the VQ */ 257 otx_cpt_write_vq_ctl(cptvf, 0); 258 259 /* Reset the doorbell */ 260 otx_cpt_write_vq_doorbell(cptvf, 0); 261 /* Clear inflight */ 262 otx_cpt_write_vq_inprog(cptvf, 0); 263 264 /* Write VQ SADDR */ 265 base_addr = (uint64_t)(cptvf->cqueue.chead[0].dma_addr); 266 otx_cpt_write_vq_saddr(cptvf, base_addr); 267 268 /* Configure timerhold / coalescence */ 269 otx_cpt_write_vq_done_timewait(cptvf, CPT_TIMER_THOLD); 270 otx_cpt_write_vq_done_numwait(cptvf, CPT_COUNT_THOLD); 271 272 /* Enable the VQ */ 273 otx_cpt_write_vq_ctl(cptvf, 1); 274 } 275 276 static int 277 cpt_vq_init(struct cpt_vf *cptvf, uint8_t group) 278 { 279 int err; 280 281 /* Convey VQ LEN to PF */ 282 err = otx_cpt_send_vq_size_msg(cptvf); 283 if (err) { 284 CPT_LOG_ERR("%s: PF not responding to QLEN msg", 285 cptvf->dev_name); 286 err = -EBUSY; 287 goto cleanup; 288 } 289 290 /* CPT VF device initialization */ 291 otx_cpt_vfvq_init(cptvf); 292 293 /* Send msg to PF to assign currnet Q to required group */ 294 cptvf->vfgrp = group; 295 err = otx_cpt_send_vf_grp_msg(cptvf, group); 296 if (err) { 297 CPT_LOG_ERR("%s: PF not responding to VF_GRP msg", 298 cptvf->dev_name); 299 err = -EBUSY; 300 goto cleanup; 301 } 302 303 CPT_LOG_DP_DEBUG("%s: %s done", cptvf->dev_name, __func__); 304 return 0; 305 306 cleanup: 307 return err; 308 } 309 310 void 311 otx_cpt_poll_misc(struct cpt_vf *cptvf) 312 { 313 uint64_t intr; 314 315 intr = otx_cpt_read_vf_misc_intr_status(cptvf); 316 317 if (!intr) 318 return; 319 320 /* Check for MISC interrupt types */ 321 if (likely(intr & CPT_VF_INTR_MBOX_MASK)) { 322 CPT_LOG_DP_DEBUG("%s: Mailbox interrupt 0x%lx on CPT VF %d", 323 cptvf->dev_name, (unsigned int long)intr, cptvf->vfid); 324 otx_cpt_handle_mbox_intr(cptvf); 325 otx_cpt_clear_mbox_intr(cptvf); 326 } else if (unlikely(intr & CPT_VF_INTR_IRDE_MASK)) { 327 otx_cpt_clear_irde_intr(cptvf); 328 CPT_LOG_DP_DEBUG("%s: Instruction NCB read error interrupt " 329 "0x%lx on CPT VF %d", cptvf->dev_name, 330 (unsigned int long)intr, cptvf->vfid); 331 } else if (unlikely(intr & CPT_VF_INTR_NWRP_MASK)) { 332 otx_cpt_clear_nwrp_intr(cptvf); 333 CPT_LOG_DP_DEBUG("%s: NCB response write error interrupt 0x%lx" 334 " on CPT VF %d", cptvf->dev_name, 335 (unsigned int long)intr, cptvf->vfid); 336 } else if (unlikely(intr & CPT_VF_INTR_SWERR_MASK)) { 337 otx_cpt_clear_swerr_intr(cptvf); 338 CPT_LOG_DP_DEBUG("%s: Software error interrupt 0x%lx on CPT VF " 339 "%d", cptvf->dev_name, (unsigned int long)intr, 340 cptvf->vfid); 341 } else if (unlikely(intr & CPT_VF_INTR_HWERR_MASK)) { 342 otx_cpt_clear_hwerr_intr(cptvf); 343 CPT_LOG_DP_DEBUG("%s: Hardware error interrupt 0x%lx on CPT VF " 344 "%d", cptvf->dev_name, (unsigned int long)intr, 345 cptvf->vfid); 346 } else if (unlikely(intr & CPT_VF_INTR_FAULT_MASK)) { 347 otx_cpt_clear_fault_intr(cptvf); 348 CPT_LOG_DP_DEBUG("%s: Translation fault interrupt 0x%lx on CPT VF " 349 "%d", cptvf->dev_name, (unsigned int long)intr, 350 cptvf->vfid); 351 } else if (unlikely(intr & CPT_VF_INTR_DOVF_MASK)) { 352 otx_cpt_clear_dovf_intr(cptvf); 353 CPT_LOG_DP_DEBUG("%s: Doorbell overflow interrupt 0x%lx on CPT VF " 354 "%d", cptvf->dev_name, (unsigned int long)intr, 355 cptvf->vfid); 356 } else 357 CPT_LOG_DP_ERR("%s: Unhandled interrupt 0x%lx in CPT VF %d", 358 cptvf->dev_name, (unsigned int long)intr, 359 cptvf->vfid); 360 } 361 362 int 363 otx_cpt_hw_init(struct cpt_vf *cptvf, void *pdev, void *reg_base, char *name) 364 { 365 memset(cptvf, 0, sizeof(struct cpt_vf)); 366 367 /* Bar0 base address */ 368 cptvf->reg_base = reg_base; 369 strncpy(cptvf->dev_name, name, 32); 370 371 cptvf->pdev = pdev; 372 373 /* To clear if there are any pending mbox msgs */ 374 otx_cpt_poll_misc(cptvf); 375 376 if (otx_cpt_vf_init(cptvf)) { 377 CPT_LOG_ERR("Failed to initialize CPT VF device"); 378 return -1; 379 } 380 381 return 0; 382 } 383 384 int 385 otx_cpt_deinit_device(void *dev) 386 { 387 struct cpt_vf *cptvf = (struct cpt_vf *)dev; 388 389 /* Do misc work one last time */ 390 otx_cpt_poll_misc(cptvf); 391 392 return 0; 393 } 394 395 int 396 otx_cpt_get_resource(void *dev, uint8_t group, struct cpt_instance **instance) 397 { 398 int ret = -ENOENT, len, qlen, i; 399 int chunk_len, chunks, chunk_size; 400 struct cpt_vf *cptvf = (struct cpt_vf *)dev; 401 struct cpt_instance *cpt_instance; 402 struct command_chunk *chunk_head = NULL, *chunk_prev = NULL; 403 struct command_chunk *chunk = NULL; 404 uint8_t *mem; 405 const struct rte_memzone *rz; 406 uint64_t dma_addr = 0, alloc_len, used_len; 407 uint64_t *next_ptr; 408 uint64_t pg_sz = sysconf(_SC_PAGESIZE); 409 410 CPT_LOG_DP_DEBUG("Initializing cpt resource %s", cptvf->dev_name); 411 412 cpt_instance = &cptvf->instance; 413 414 memset(&cptvf->cqueue, 0, sizeof(cptvf->cqueue)); 415 memset(&cptvf->pqueue, 0, sizeof(cptvf->pqueue)); 416 417 /* Chunks are of fixed size buffers */ 418 chunks = DEFAULT_CMD_QCHUNKS; 419 chunk_len = DEFAULT_CMD_QCHUNK_SIZE; 420 421 qlen = chunks * chunk_len; 422 /* Chunk size includes 8 bytes of next chunk ptr */ 423 chunk_size = chunk_len * CPT_INST_SIZE + CPT_NEXT_CHUNK_PTR_SIZE; 424 425 /* For command chunk structures */ 426 len = chunks * RTE_ALIGN(sizeof(struct command_chunk), 8); 427 428 /* For pending queue */ 429 len += qlen * RTE_ALIGN(sizeof(struct rid), 8); 430 431 /* So that instruction queues start as pg size aligned */ 432 len = RTE_ALIGN(len, pg_sz); 433 434 /* For Instruction queues */ 435 len += chunks * RTE_ALIGN(chunk_size, 128); 436 437 /* Wastage after instruction queues */ 438 len = RTE_ALIGN(len, pg_sz); 439 440 rz = rte_memzone_reserve_aligned(cptvf->dev_name, len, cptvf->node, 441 RTE_MEMZONE_SIZE_HINT_ONLY | 442 RTE_MEMZONE_256MB, 443 RTE_CACHE_LINE_SIZE); 444 if (!rz) { 445 ret = rte_errno; 446 goto cleanup; 447 } 448 449 mem = rz->addr; 450 dma_addr = rz->phys_addr; 451 alloc_len = len; 452 453 memset(mem, 0, len); 454 455 cpt_instance->rsvd = (uintptr_t)rz; 456 457 /* Pending queue setup */ 458 cptvf->pqueue.rid_queue = (struct rid *)mem; 459 cptvf->pqueue.enq_tail = 0; 460 cptvf->pqueue.deq_head = 0; 461 cptvf->pqueue.pending_count = 0; 462 463 mem += qlen * RTE_ALIGN(sizeof(struct rid), 8); 464 len -= qlen * RTE_ALIGN(sizeof(struct rid), 8); 465 dma_addr += qlen * RTE_ALIGN(sizeof(struct rid), 8); 466 467 /* Alignment wastage */ 468 used_len = alloc_len - len; 469 mem += RTE_ALIGN(used_len, pg_sz) - used_len; 470 len -= RTE_ALIGN(used_len, pg_sz) - used_len; 471 dma_addr += RTE_ALIGN(used_len, pg_sz) - used_len; 472 473 /* Init instruction queues */ 474 chunk_head = &cptvf->cqueue.chead[0]; 475 i = qlen; 476 477 chunk_prev = NULL; 478 for (i = 0; i < DEFAULT_CMD_QCHUNKS; i++) { 479 int csize; 480 481 chunk = &cptvf->cqueue.chead[i]; 482 chunk->head = mem; 483 chunk->dma_addr = dma_addr; 484 485 csize = RTE_ALIGN(chunk_size, 128); 486 mem += csize; 487 dma_addr += csize; 488 len -= csize; 489 490 if (chunk_prev) { 491 next_ptr = (uint64_t *)(chunk_prev->head + 492 chunk_size - 8); 493 *next_ptr = (uint64_t)chunk->dma_addr; 494 } 495 chunk_prev = chunk; 496 } 497 /* Circular loop */ 498 next_ptr = (uint64_t *)(chunk_prev->head + chunk_size - 8); 499 *next_ptr = (uint64_t)chunk_head->dma_addr; 500 501 assert(!len); 502 503 /* This is used for CPT(0)_PF_Q(0..15)_CTL.size config */ 504 cptvf->qsize = chunk_size / 8; 505 cptvf->cqueue.qhead = chunk_head->head; 506 cptvf->cqueue.idx = 0; 507 cptvf->cqueue.cchunk = 0; 508 509 if (cpt_vq_init(cptvf, group)) { 510 CPT_LOG_ERR("Failed to initialize CPT VQ of device %s", 511 cptvf->dev_name); 512 ret = -EBUSY; 513 goto cleanup; 514 } 515 516 *instance = cpt_instance; 517 518 CPT_LOG_DP_DEBUG("Crypto device (%s) initialized", cptvf->dev_name); 519 520 return 0; 521 cleanup: 522 rte_memzone_free(rz); 523 *instance = NULL; 524 return ret; 525 } 526 527 int 528 otx_cpt_put_resource(struct cpt_instance *instance) 529 { 530 struct cpt_vf *cptvf = (struct cpt_vf *)instance; 531 struct rte_memzone *rz; 532 533 if (!cptvf) { 534 CPT_LOG_ERR("Invalid CPTVF handle"); 535 return -EINVAL; 536 } 537 538 CPT_LOG_DP_DEBUG("Releasing cpt device %s", cptvf->dev_name); 539 540 rz = (struct rte_memzone *)instance->rsvd; 541 rte_memzone_free(rz); 542 return 0; 543 } 544 545 int 546 otx_cpt_start_device(void *dev) 547 { 548 int rc; 549 struct cpt_vf *cptvf = (struct cpt_vf *)dev; 550 551 rc = otx_cpt_send_vf_up(cptvf); 552 if (rc) { 553 CPT_LOG_ERR("Failed to mark CPT VF device %s UP, rc = %d", 554 cptvf->dev_name, rc); 555 return -EFAULT; 556 } 557 558 if ((cptvf->vftype != SE_TYPE) && (cptvf->vftype != AE_TYPE)) { 559 CPT_LOG_ERR("Fatal error, unexpected vf type %u, for CPT VF " 560 "device %s", cptvf->vftype, cptvf->dev_name); 561 return -ENOENT; 562 } 563 564 return 0; 565 } 566 567 void 568 otx_cpt_stop_device(void *dev) 569 { 570 int rc; 571 uint32_t pending, retries = 5; 572 struct cpt_vf *cptvf = (struct cpt_vf *)dev; 573 574 /* Wait for pending entries to complete */ 575 pending = otx_cpt_read_vq_doorbell(cptvf); 576 while (pending) { 577 CPT_LOG_DP_DEBUG("%s: Waiting for pending %u cmds to complete", 578 cptvf->dev_name, pending); 579 sleep(1); 580 pending = otx_cpt_read_vq_doorbell(cptvf); 581 retries--; 582 if (!retries) 583 break; 584 } 585 586 if (!retries && pending) { 587 CPT_LOG_ERR("%s: Timeout waiting for commands(%u)", 588 cptvf->dev_name, pending); 589 return; 590 } 591 592 rc = otx_cpt_send_vf_down(cptvf); 593 if (rc) { 594 CPT_LOG_ERR("Failed to bring down vf %s, rc %d", 595 cptvf->dev_name, rc); 596 return; 597 } 598 } 599