1 /* $OpenBSD: ufshci.c,v 1.46 2025/01/18 19:42:39 mglocker Exp $ */ 2 3 /* 4 * Copyright (c) 2022 Marcus Glocker <mglocker@openbsd.org> 5 * 6 * Permission to use, copy, modify, and distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 /* 20 * Universal Flash Storage Host Controller Interface (UFSHCI) 2.1 driver based 21 * on the JEDEC JESD223C.pdf and JESD220C-2_1.pdf specifications. 22 */ 23 24 #include "kstat.h" 25 26 #include <sys/param.h> 27 #include <sys/systm.h> 28 #include <sys/buf.h> 29 #include <sys/kernel.h> 30 #include <sys/malloc.h> 31 #include <sys/device.h> 32 #include <sys/queue.h> 33 #include <sys/mutex.h> 34 #include <sys/pool.h> 35 #include <sys/kstat.h> 36 37 #include <sys/atomic.h> 38 39 #include <machine/bus.h> 40 41 #include <scsi/scsi_all.h> 42 #include <scsi/scsi_disk.h> 43 #include <scsi/scsiconf.h> 44 45 #include <dev/ic/ufshcivar.h> 46 #include <dev/ic/ufshcireg.h> 47 48 #ifdef HIBERNATE 49 #include <uvm/uvm_extern.h> 50 #include <sys/hibernate.h> 51 #include <sys/disklabel.h> 52 #include <sys/disk.h> 53 #endif 54 55 #ifdef UFSHCI_DEBUG 56 int ufshci_debug = 1; 57 #endif 58 59 struct cfdriver ufshci_cd = { 60 NULL, "ufshci", DV_DULL 61 }; 62 63 int ufshci_reset(struct ufshci_softc *); 64 int ufshci_is_poll(struct ufshci_softc *, uint32_t); 65 struct ufshci_dmamem *ufshci_dmamem_alloc(struct ufshci_softc *, size_t); 66 void ufshci_dmamem_free(struct ufshci_softc *, 67 struct ufshci_dmamem *); 68 int ufshci_alloc(struct ufshci_softc *); 69 int ufshci_init(struct ufshci_softc *); 70 void ufshci_disable(struct ufshci_softc *); 71 int ufshci_doorbell_read(struct ufshci_softc *); 72 void ufshci_doorbell_write(struct ufshci_softc *, int); 73 int ufshci_doorbell_poll(struct ufshci_softc *, int, 74 uint32_t); 75 int ufshci_utr_cmd_nop(struct ufshci_softc *, 76 struct ufshci_ccb *, struct scsi_xfer *); 77 int ufshci_utr_cmd_lun(struct ufshci_softc *, 78 struct ufshci_ccb *, struct scsi_xfer *); 79 int ufshci_utr_cmd_inquiry(struct ufshci_softc *, 80 struct ufshci_ccb *, struct scsi_xfer *); 81 int ufshci_utr_cmd_capacity16(struct ufshci_softc *, 82 struct ufshci_ccb *, struct scsi_xfer *); 83 int ufshci_utr_cmd_capacity(struct ufshci_softc *, 84 struct ufshci_ccb *, struct scsi_xfer *); 85 int ufshci_utr_cmd_io(struct ufshci_softc *, 86 struct ufshci_ccb *, struct scsi_xfer *, int); 87 int ufshci_utr_cmd_sync(struct ufshci_softc *, 88 struct ufshci_ccb *, struct scsi_xfer *, 89 uint32_t, uint16_t); 90 void ufshci_xfer_complete(struct ufshci_softc *); 91 92 /* SCSI */ 93 int ufshci_ccb_alloc(struct ufshci_softc *, int); 94 void *ufshci_ccb_get(void *); 95 void ufshci_ccb_put(void *, void *); 96 void ufshci_ccb_free(struct ufshci_softc*, int); 97 98 void ufshci_scsi_cmd(struct scsi_xfer *); 99 100 void ufshci_scsi_inquiry(struct scsi_xfer *); 101 void ufshci_scsi_capacity16(struct scsi_xfer *); 102 void ufshci_scsi_capacity(struct scsi_xfer *); 103 void ufshci_scsi_sync(struct scsi_xfer *); 104 void ufshci_scsi_io(struct scsi_xfer *, int); 105 void ufshci_scsi_io_done(struct ufshci_softc *, 106 struct ufshci_ccb *); 107 void ufshci_scsi_done(struct ufshci_softc *, 108 struct ufshci_ccb *); 109 110 #ifdef HIBERNATE 111 int ufshci_hibernate_io(dev_t, daddr_t, vaddr_t, size_t, 112 int, void *); 113 #endif 114 115 #if NKSTAT > 0 116 void ufshci_kstat_attach(struct ufshci_softc *); 117 int ufshci_kstat_read_ccb(struct kstat *); 118 int ufshci_kstat_read_slot(struct kstat *); 119 #endif 120 121 const struct scsi_adapter ufshci_switch = { 122 ufshci_scsi_cmd, NULL, NULL, NULL, NULL 123 }; 124 125 int 126 ufshci_intr(void *arg) 127 { 128 struct ufshci_softc *sc = arg; 129 uint32_t status, hcs; 130 int handled = 0; 131 132 status = UFSHCI_READ_4(sc, UFSHCI_REG_IS); 133 134 if (status == 0) 135 return handled; 136 137 /* ACK interrupt */ 138 UFSHCI_WRITE_4(sc, UFSHCI_REG_IS, status); 139 140 if (status & UFSHCI_REG_IS_UCCS) { 141 handled = 1; 142 } 143 if (status & UFSHCI_REG_IS_UTRCS) { 144 ufshci_xfer_complete(sc); 145 146 handled = 1; 147 } 148 /* If Auto-Hibernate raises an interrupt, it's to yield an error. */ 149 if (status & UFSHCI_REG_IS_UHES) { 150 hcs = UFSHCI_READ_4(sc, UFSHCI_REG_HCS); 151 printf("%s: Auto-Hibernate enter error UPMCRS=0x%x\n", 152 __func__, UFSHCI_REG_HCS_UPMCRS(hcs)); 153 handled = 1; 154 } 155 if (status & UFSHCI_REG_IS_UHXS) { 156 hcs = UFSHCI_READ_4(sc, UFSHCI_REG_HCS); 157 printf("%s: Auto-Hibernate exit error UPMCRS=0x%x\n", 158 __func__, UFSHCI_REG_HCS_UPMCRS(hcs)); 159 handled = 1; 160 } 161 162 if (handled == 0) { 163 printf("%s: UNKNOWN interrupt, status=0x%08x\n", 164 sc->sc_dev.dv_xname, status); 165 } 166 167 return handled; 168 } 169 170 int 171 ufshci_attach(struct ufshci_softc *sc) 172 { 173 struct scsibus_attach_args saa; 174 175 mtx_init(&sc->sc_cmd_mtx, IPL_BIO); 176 mtx_init(&sc->sc_ccb_mtx, IPL_BIO); 177 SIMPLEQ_INIT(&sc->sc_ccb_list); 178 scsi_iopool_init(&sc->sc_iopool, sc, ufshci_ccb_get, ufshci_ccb_put); 179 180 if (ufshci_reset(sc)) 181 return 1; 182 183 sc->sc_ver = UFSHCI_READ_4(sc, UFSHCI_REG_VER); 184 printf(", UFSHCI %d.%d%d\n", 185 UFSHCI_REG_VER_MAJOR(sc->sc_ver), 186 UFSHCI_REG_VER_MINOR(sc->sc_ver), 187 UFSHCI_REG_VER_SUFFIX(sc->sc_ver)); 188 189 sc->sc_cap = UFSHCI_READ_4(sc, UFSHCI_REG_CAP); 190 sc->sc_hcpid = UFSHCI_READ_4(sc, UFSHCI_REG_HCPID); 191 sc->sc_hcmid = UFSHCI_READ_4(sc, UFSHCI_REG_HCMID); 192 sc->sc_nutmrs = UFSHCI_REG_CAP_NUTMRS(sc->sc_cap) + 1; 193 sc->sc_rtt = UFSHCI_REG_CAP_RTT(sc->sc_cap) + 1; 194 sc->sc_nutrs = UFSHCI_REG_CAP_NUTRS(sc->sc_cap) + 1; 195 196 #ifdef UFSHCI_DEBUG 197 printf("Capabilities (0x%08x):\n", sc->sc_cap); 198 printf(" CS=%d\n", sc->sc_cap & UFSHCI_REG_CAP_CS ? 1 : 0); 199 printf(" UICDMETMS=%d\n", sc->sc_cap & UFSHCI_REG_CAP_UICDMETMS ? 1 :0); 200 printf(" OODDS=%d\n", sc->sc_cap & UFSHCI_REG_CAP_OODDS ? 1 : 0); 201 printf(" 64AS=%d\n", sc->sc_cap & UFSHCI_REG_CAP_64AS ? 1 : 0); 202 printf(" AUTOH8=%d\n", sc->sc_cap & UFSHCI_REG_AUTOH8 ? 1 : 0); 203 printf(" NUTMRS=%d\n", sc->sc_nutmrs); 204 printf(" RTT=%d\n", sc->sc_rtt); 205 printf(" NUTRS=%d\n", sc->sc_nutrs); 206 printf(" HCPID=0x%08x\n", sc->sc_hcpid); 207 printf("HCMID (0x%08x):\n", sc->sc_hcmid); 208 printf(" BI=0x%04x\n", UFSHCI_REG_HCMID_BI(sc->sc_hcmid)); 209 printf(" MIC=0x%04x\n", UFSHCI_REG_HCMID_MIC(sc->sc_hcmid)); 210 #endif 211 if (sc->sc_nutrs < UFSHCI_SLOTS_MIN || 212 sc->sc_nutrs > UFSHCI_SLOTS_MAX) { 213 printf("%s: Invalid NUTRS value %d (must be %d-%d)!\n", 214 sc->sc_dev.dv_xname, sc->sc_nutrs, 215 UFSHCI_SLOTS_MIN, UFSHCI_SLOTS_MAX); 216 return 1; 217 } 218 if (sc->sc_nutrs == UFSHCI_SLOTS_MAX) 219 sc->sc_iacth = UFSHCI_INTR_AGGR_COUNT_MAX; 220 else 221 sc->sc_iacth = sc->sc_nutrs; 222 DPRINTF(1, "Intr. aggr. counter threshold:\nIACTH=%d\n", sc->sc_iacth); 223 224 /* 225 * XXX: 226 * At the moment normal interrupts work better for us than interrupt 227 * aggregation, because: 228 * 229 * 1. With interrupt aggregation enabled, the I/O performance 230 * isn't better, but even slightly worse depending on the 231 * UFS controller and architecture. 232 * 2. With interrupt aggregation enabled we currently see 233 * intermittent SCSI command stalling. Probably there is a 234 * race condition where new SCSI commands are getting 235 * scheduled, while we miss to reset the interrupt aggregation 236 * counter/timer, which leaves us with no more interrupts 237 * triggered. This needs to be fixed, but I couldn't figure 238 * out yet how. 239 */ 240 #if 0 241 sc->sc_flags |= UFSHCI_FLAGS_AGGR_INTR; /* Enable intr. aggregation */ 242 #endif 243 /* Allocate the DMA buffers and initialize the controller. */ 244 if (ufshci_alloc(sc)) 245 return 1; 246 if (ufshci_init(sc)) 247 return 1; 248 249 if (ufshci_ccb_alloc(sc, sc->sc_nutrs) != 0) { 250 printf("%s: %s: Can't allocate CCBs\n", 251 sc->sc_dev.dv_xname, __func__); 252 return 1; 253 } 254 255 /* Enable Auto-Hibernate Idle Timer (AHIT) and set it to 150ms. */ 256 if (sc->sc_cap & UFSHCI_REG_AUTOH8) { 257 UFSHCI_WRITE_4(sc, UFSHCI_REG_AHIT, 258 UFSHCI_REG_AHIT_TS(UFSHCI_REG_AHIT_TS_1MS) | 150); 259 } 260 261 /* Attach to SCSI layer */ 262 saa.saa_adapter = &ufshci_switch; 263 saa.saa_adapter_softc = sc; 264 saa.saa_adapter_buswidth = UFSHCI_TARGETS_MAX + 1; 265 saa.saa_luns = 1; 266 saa.saa_adapter_target = 0; 267 saa.saa_openings = sc->sc_nutrs; 268 saa.saa_pool = &sc->sc_iopool; 269 saa.saa_quirks = saa.saa_flags = 0; 270 saa.saa_wwpn = saa.saa_wwnn = 0; 271 #if NKSTAT > 0 272 ufshci_kstat_attach(sc); 273 #endif 274 config_found(&sc->sc_dev, &saa, scsiprint); 275 276 return 0; 277 } 278 279 int 280 ufshci_reset(struct ufshci_softc *sc) 281 { 282 int i; 283 int retry = 10; 284 uint32_t hce; 285 286 /* 287 * 7.1.1 Host Controller Initialization: 2) 288 * Reset and enable host controller 289 */ 290 UFSHCI_WRITE_4(sc, UFSHCI_REG_HCE, UFSHCI_REG_HCE_HCE); 291 292 /* 7.1.1 Host Controller Initialization: 3) */ 293 for (i = 0; i < retry; i++) { 294 hce = UFSHCI_READ_4(sc, UFSHCI_REG_HCE); 295 if (hce == 1) 296 break; 297 delay(1); 298 } 299 if (i == retry) { 300 printf("%s: Enabling Host Controller failed!\n", 301 sc->sc_dev.dv_xname); 302 return 1; 303 } 304 305 DPRINTF(2, "\n%s: Host Controller enabled (i=%d)\n", __func__, i); 306 307 return 0; 308 } 309 310 int 311 ufshci_is_poll(struct ufshci_softc *sc, uint32_t type) 312 { 313 uint32_t status; 314 int i, retry = 25; 315 316 for (i = 0; i < retry; i++) { 317 status = UFSHCI_READ_4(sc, UFSHCI_REG_IS); 318 if (status & type) 319 break; 320 delay(10); 321 } 322 if (i == retry) { 323 printf("%s: %s: timeout\n", sc->sc_dev.dv_xname, __func__); 324 return 1; 325 } 326 DPRINTF(3, "%s: completed after %d retries\n", __func__, i); 327 328 /* ACK interrupt */ 329 UFSHCI_WRITE_4(sc, UFSHCI_REG_IS, status); 330 331 return 0; 332 } 333 334 struct ufshci_dmamem * 335 ufshci_dmamem_alloc(struct ufshci_softc *sc, size_t size) 336 { 337 struct ufshci_dmamem *udm; 338 int nsegs; 339 340 udm = malloc(sizeof(*udm), M_DEVBUF, M_WAITOK | M_ZERO); 341 if (udm == NULL) 342 return NULL; 343 344 udm->udm_size = size; 345 346 if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0, 347 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW | 348 ((sc->sc_cap & UFSHCI_REG_CAP_64AS) ? BUS_DMA_64BIT : 0), 349 &udm->udm_map) != 0) 350 goto udmfree; 351 352 if (bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0, &udm->udm_seg, 353 1, &nsegs, BUS_DMA_WAITOK | BUS_DMA_ZERO) != 0) 354 goto destroy; 355 356 if (bus_dmamem_map(sc->sc_dmat, &udm->udm_seg, nsegs, size, 357 &udm->udm_kva, BUS_DMA_WAITOK) != 0) 358 goto free; 359 360 if (bus_dmamap_load(sc->sc_dmat, udm->udm_map, udm->udm_kva, size, 361 NULL, BUS_DMA_WAITOK) != 0) 362 goto unmap; 363 364 DPRINTF(2, "%s: size=%lu, page_size=%d, nsegs=%d\n", 365 __func__, size, PAGE_SIZE, nsegs); 366 367 return udm; 368 369 unmap: 370 bus_dmamem_unmap(sc->sc_dmat, udm->udm_kva, size); 371 free: 372 bus_dmamem_free(sc->sc_dmat, &udm->udm_seg, 1); 373 destroy: 374 bus_dmamap_destroy(sc->sc_dmat, udm->udm_map); 375 udmfree: 376 free(udm, M_DEVBUF, sizeof(*udm)); 377 378 return NULL; 379 } 380 381 void 382 ufshci_dmamem_free(struct ufshci_softc *sc, struct ufshci_dmamem *udm) 383 { 384 bus_dmamap_unload(sc->sc_dmat, udm->udm_map); 385 bus_dmamem_unmap(sc->sc_dmat, udm->udm_kva, udm->udm_size); 386 bus_dmamem_free(sc->sc_dmat, &udm->udm_seg, 1); 387 bus_dmamap_destroy(sc->sc_dmat, udm->udm_map); 388 free(udm, M_DEVBUF, sizeof(*udm)); 389 } 390 391 int 392 ufshci_alloc(struct ufshci_softc *sc) 393 { 394 /* 7.1.1 Host Controller Initialization: 13) */ 395 sc->sc_dmamem_utmrd = ufshci_dmamem_alloc(sc, 396 sizeof(struct ufshci_utmrd) * sc->sc_nutmrs); 397 if (sc->sc_dmamem_utmrd == NULL) { 398 printf("%s: Can't allocate DMA memory for UTMRD\n", 399 sc->sc_dev.dv_xname); 400 return 1; 401 } 402 403 /* 7.1.1 Host Controller Initialization: 15) */ 404 sc->sc_dmamem_utrd = ufshci_dmamem_alloc(sc, 405 sizeof(struct ufshci_utrd) * sc->sc_nutrs); 406 if (sc->sc_dmamem_utrd == NULL) { 407 printf("%s: Can't allocate DMA memory for UTRD\n", 408 sc->sc_dev.dv_xname); 409 return 1; 410 } 411 412 /* Allocate UCDs. */ 413 sc->sc_dmamem_ucd = ufshci_dmamem_alloc(sc, 414 sizeof(struct ufshci_ucd) * sc->sc_nutrs); 415 if (sc->sc_dmamem_ucd == NULL) { 416 printf("%s: Can't allocate DMA memory for UCD\n", 417 sc->sc_dev.dv_xname); 418 return 1; 419 } 420 421 return 0; 422 } 423 424 int 425 ufshci_init(struct ufshci_softc *sc) 426 { 427 uint32_t reg; 428 uint64_t dva; 429 430 /* 431 * 7.1.1 Host Controller Initialization: 4) 432 * TODO: Do we need to set DME_SET? 433 */ 434 435 /* 7.1.1 Host Controller Initialization: 5) */ 436 if (sc->sc_cap & UFSHCI_REG_AUTOH8) { 437 UFSHCI_WRITE_4(sc, UFSHCI_REG_IE, 438 UFSHCI_REG_IE_UTRCE | UFSHCI_REG_IE_UTMRCE | 439 UFSHCI_REG_IS_UHES | UFSHCI_REG_IS_UHXS); 440 } else { 441 UFSHCI_WRITE_4(sc, UFSHCI_REG_IE, 442 UFSHCI_REG_IE_UTRCE | UFSHCI_REG_IE_UTMRCE); 443 } 444 445 /* 7.1.1 Host Controller Initialization: 6) */ 446 UFSHCI_WRITE_4(sc, UFSHCI_REG_UICCMD, 447 UFSHCI_REG_UICCMD_CMDOP_DME_LINKSTARTUP); 448 if (ufshci_is_poll(sc, UFSHCI_REG_IS_UCCS)) 449 return 1; 450 451 /* 452 * 7.1.1 Host Controller Initialization: 7), 8), 9) 453 * TODO: Implement retry in case UFSHCI_REG_HCS returns 0 454 */ 455 reg = UFSHCI_READ_4(sc, UFSHCI_REG_HCS); 456 if (reg & UFSHCI_REG_HCS_DP) 457 DPRINTF(2, "%s: Device Presence SET\n", __func__); 458 else 459 DPRINTF(2, "%s: Device Presence NOT SET\n", __func__); 460 461 /* 462 * 7.1.1 Host Controller Initialization: 10) 463 * TODO: Enable additional interrupt on the IE register 464 */ 465 466 /* 7.1.1 Host Controller Initialization: 11) */ 467 if (sc->sc_flags & UFSHCI_FLAGS_AGGR_INTR) { 468 UFSHCI_WRITE_4(sc, UFSHCI_REG_UTRIACR, 469 UFSHCI_REG_UTRIACR_IAEN | 470 UFSHCI_REG_UTRIACR_IAPWEN | 471 UFSHCI_REG_UTRIACR_CTR | 472 UFSHCI_REG_UTRIACR_IACTH(sc->sc_iacth) | 473 UFSHCI_REG_UTRIACR_IATOVAL(UFSHCI_INTR_AGGR_TIMEOUT)); 474 } else { 475 UFSHCI_WRITE_4(sc, UFSHCI_REG_UTRIACR, 0); 476 } 477 478 /* 479 * 7.1.1 Host Controller Initialization: 12) 480 * TODO: More UIC commands to issue? 481 */ 482 483 /* 7.1.1 Host Controller Initialization: 14) */ 484 dva = UFSHCI_DMA_DVA(sc->sc_dmamem_utmrd); 485 DPRINTF(2, "%s: utmrd dva=%llu\n", __func__, dva); 486 UFSHCI_WRITE_4(sc, UFSHCI_REG_UTMRLBA, (uint32_t)dva); 487 UFSHCI_WRITE_4(sc, UFSHCI_REG_UTMRLBAU, (uint32_t)(dva >> 32)); 488 489 /* 7.1.1 Host Controller Initialization: 16) */ 490 dva = UFSHCI_DMA_DVA(sc->sc_dmamem_utrd); 491 DPRINTF(2, "%s: utrd dva=%llu\n", __func__, dva); 492 UFSHCI_WRITE_4(sc, UFSHCI_REG_UTRLBA, (uint32_t)dva); 493 UFSHCI_WRITE_4(sc, UFSHCI_REG_UTRLBAU, (uint32_t)(dva >> 32)); 494 495 /* 7.1.1 Host Controller Initialization: 17) */ 496 UFSHCI_WRITE_4(sc, UFSHCI_REG_UTMRLRSR, UFSHCI_REG_UTMRLRSR_START); 497 498 /* 7.1.1 Host Controller Initialization: 18) */ 499 UFSHCI_WRITE_4(sc, UFSHCI_REG_UTRLRSR, UFSHCI_REG_UTRLRSR_START); 500 501 /* 7.1.1 Host Controller Initialization: 19) */ 502 /* TODO: bMaxNumOfRTT will be set as the minimum value of 503 * bDeviceRTTCap and NORTT. ??? 504 */ 505 506 return 0; 507 } 508 509 void 510 ufshci_disable(struct ufshci_softc *sc) 511 { 512 /* Stop run queues. */ 513 UFSHCI_WRITE_4(sc, UFSHCI_REG_UTMRLRSR, UFSHCI_REG_UTMRLRSR_STOP); 514 UFSHCI_WRITE_4(sc, UFSHCI_REG_UTRLRSR, UFSHCI_REG_UTRLRSR_STOP); 515 516 /* Disable interrupts. */ 517 UFSHCI_WRITE_4(sc, UFSHCI_REG_IE, 0); 518 } 519 520 int 521 ufshci_doorbell_read(struct ufshci_softc *sc) 522 { 523 uint32_t reg; 524 525 reg = UFSHCI_READ_4(sc, UFSHCI_REG_UTRLDBR); 526 527 return reg; 528 } 529 530 void 531 ufshci_doorbell_write(struct ufshci_softc *sc, int slot) 532 { 533 uint32_t reg; 534 535 reg = (1U << slot); 536 537 UFSHCI_WRITE_4(sc, UFSHCI_REG_UTRLDBR, reg); 538 } 539 540 int 541 ufshci_doorbell_poll(struct ufshci_softc *sc, int slot, uint32_t timeout_ms) 542 { 543 uint32_t reg; 544 uint64_t timeout_us; 545 546 for (timeout_us = timeout_ms * 1000; timeout_us != 0; 547 timeout_us -= 1000) { 548 reg = UFSHCI_READ_4(sc, UFSHCI_REG_UTRLDBR); 549 if ((reg & (1U << slot)) == 0) 550 break; 551 delay(1000); 552 } 553 if (timeout_us == 0) { 554 printf("%s: %s: timeout\n", sc->sc_dev.dv_xname, __func__); 555 return 1; 556 } 557 558 return 0; 559 } 560 561 int 562 ufshci_utr_cmd_nop(struct ufshci_softc *sc, struct ufshci_ccb *ccb, 563 struct scsi_xfer *xs) 564 { 565 int slot, off, len; 566 uint64_t dva; 567 struct ufshci_utrd *utrd; 568 struct ufshci_ucd *ucd; 569 570 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 1) */ 571 slot = ccb->ccb_slot; 572 utrd = UFSHCI_DMA_KVA(sc->sc_dmamem_utrd); 573 utrd += slot; 574 memset(utrd, 0, sizeof(*utrd)); 575 576 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2a) */ 577 utrd->dw0 = UFSHCI_UTRD_DW0_CT_UFS; 578 579 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2b) */ 580 utrd->dw0 |= UFSHCI_UTRD_DW0_DD_NO; 581 582 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2c) */ 583 if (sc->sc_flags & UFSHCI_FLAGS_AGGR_INTR) 584 utrd->dw0 |= UFSHCI_UTRD_DW0_I_REG; 585 else 586 utrd->dw0 |= UFSHCI_UTRD_DW0_I_INT; 587 588 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2d) */ 589 utrd->dw2 = UFSHCI_UTRD_DW2_OCS_IOV; 590 591 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2e) */ 592 ucd = UFSHCI_DMA_KVA(sc->sc_dmamem_ucd); 593 ucd += slot; 594 memset(ucd, 0, sizeof(*ucd)); 595 596 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2f) */ 597 ucd->cmd.hdr.tc = UPIU_TC_I2T_NOP_OUT; 598 ucd->cmd.hdr.flags = 0; 599 ucd->cmd.hdr.lun = 0; 600 ucd->cmd.hdr.task_tag = slot; 601 ucd->cmd.hdr.cmd_set_type = 0; /* SCSI command */ 602 ucd->cmd.hdr.query = 0; 603 ucd->cmd.hdr.response = 0; 604 ucd->cmd.hdr.status = 0; 605 ucd->cmd.hdr.ehs_len = 0; 606 ucd->cmd.hdr.device_info = 0; 607 ucd->cmd.hdr.ds_len = 0; 608 609 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2g) */ 610 /* Already done with above memset */ 611 612 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 3) */ 613 dva = UFSHCI_DMA_DVA(sc->sc_dmamem_ucd); 614 utrd->dw4 = (uint32_t)dva; 615 utrd->dw5 = (uint32_t)(dva >> 32); 616 617 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 4) */ 618 off = sizeof(struct upiu_command) / 4; /* DWORD offset */ 619 utrd->dw6 = UFSHCI_UTRD_DW6_RUO(off); 620 621 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 5) */ 622 len = sizeof(struct upiu_response) / 4; /* DWORD length */ 623 utrd->dw6 |= UFSHCI_UTRD_DW6_RUL(len); 624 625 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 6) */ 626 off = (sizeof(struct upiu_command) + sizeof(struct upiu_response)) / 4; 627 utrd->dw7 = UFSHCI_UTRD_DW7_PRDTO(off); 628 629 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 7) */ 630 utrd->dw7 |= UFSHCI_UTRD_DW7_PRDTL(0); /* No data xfer */ 631 632 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 9) */ 633 if (UFSHCI_READ_4(sc, UFSHCI_REG_UTRLRSR) != 1) { 634 printf("%s: %s: UTRLRSR not set\n", 635 sc->sc_dev.dv_xname, __func__); 636 return 1; 637 } 638 639 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_utrd), 640 sizeof(*utrd) * slot, sizeof(*utrd), BUS_DMASYNC_PREWRITE); 641 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_ucd), 642 sizeof(*ucd) * slot, sizeof(*ucd), BUS_DMASYNC_PREWRITE); 643 644 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 14) */ 645 ccb->ccb_status = CCB_STATUS_INPROGRESS; 646 ufshci_doorbell_write(sc, slot); 647 648 return 0; 649 } 650 651 int 652 ufshci_utr_cmd_lun(struct ufshci_softc *sc, struct ufshci_ccb *ccb, 653 struct scsi_xfer *xs) 654 { 655 int slot, off, len, i; 656 uint64_t dva; 657 struct ufshci_utrd *utrd; 658 struct ufshci_ucd *ucd; 659 bus_dmamap_t dmap = ccb->ccb_dmamap; 660 661 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 1) */ 662 slot = ccb->ccb_slot; 663 utrd = UFSHCI_DMA_KVA(sc->sc_dmamem_utrd); 664 utrd += slot; 665 memset(utrd, 0, sizeof(*utrd)); 666 667 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2a) */ 668 utrd->dw0 = UFSHCI_UTRD_DW0_CT_UFS; 669 670 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2b) */ 671 utrd->dw0 |= UFSHCI_UTRD_DW0_DD_T2I; 672 673 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2c) */ 674 if (sc->sc_flags & UFSHCI_FLAGS_AGGR_INTR) 675 utrd->dw0 |= UFSHCI_UTRD_DW0_I_REG; 676 else 677 utrd->dw0 |= UFSHCI_UTRD_DW0_I_INT; 678 679 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2d) */ 680 utrd->dw2 = UFSHCI_UTRD_DW2_OCS_IOV; 681 682 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2e) */ 683 ucd = UFSHCI_DMA_KVA(sc->sc_dmamem_ucd); 684 ucd += slot; 685 memset(ucd, 0, sizeof(*ucd)); 686 687 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2f) */ 688 ucd->cmd.hdr.tc = UPIU_TC_I2T_COMMAND; 689 ucd->cmd.hdr.flags = (1 << 6); /* Bit-5 = Write, Bit-6 = Read */ 690 ucd->cmd.hdr.lun = 0; 691 ucd->cmd.hdr.task_tag = slot; 692 ucd->cmd.hdr.cmd_set_type = 0; /* SCSI command */ 693 ucd->cmd.hdr.query = 0; 694 ucd->cmd.hdr.response = 0; 695 ucd->cmd.hdr.status = 0; 696 ucd->cmd.hdr.ehs_len = 0; 697 ucd->cmd.hdr.device_info = 0; 698 ucd->cmd.hdr.ds_len = 0; 699 700 ucd->cmd.expected_xfer_len = htobe32(xs->datalen); 701 702 ucd->cmd.cdb[0] = REPORT_LUNS; 703 ucd->cmd.cdb[6] = 0; 704 ucd->cmd.cdb[7] = 0; 705 ucd->cmd.cdb[8] = 0; 706 ucd->cmd.cdb[9] = xs->datalen; 707 708 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2g) */ 709 /* Already done with above memset */ 710 711 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 3) */ 712 dva = UFSHCI_DMA_DVA(sc->sc_dmamem_ucd); 713 utrd->dw4 = (uint32_t)dva; 714 utrd->dw5 = (uint32_t)(dva >> 32); 715 716 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 4) */ 717 off = sizeof(struct upiu_command) / 4; /* DWORD offset */ 718 utrd->dw6 = UFSHCI_UTRD_DW6_RUO(off); 719 720 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 5) */ 721 len = sizeof(struct upiu_response) / 4; /* DWORD length */ 722 utrd->dw6 |= UFSHCI_UTRD_DW6_RUL(len); 723 724 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 6) */ 725 off = (sizeof(struct upiu_command) + sizeof(struct upiu_response)) / 4; 726 utrd->dw7 = UFSHCI_UTRD_DW7_PRDTO(off); 727 728 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 7) */ 729 utrd->dw7 |= UFSHCI_UTRD_DW7_PRDTL(dmap->dm_nsegs); 730 731 /* Build PRDT data segment. */ 732 for (i = 0; i < dmap->dm_nsegs; i++) { 733 dva = dmap->dm_segs[i].ds_addr; 734 ucd->prdt[i].dw0 = (uint32_t)dva; 735 ucd->prdt[i].dw1 = (uint32_t)(dva >> 32); 736 ucd->prdt[i].dw2 = 0; 737 ucd->prdt[i].dw3 = dmap->dm_segs[i].ds_len - 1; 738 } 739 740 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 9) */ 741 if (UFSHCI_READ_4(sc, UFSHCI_REG_UTRLRSR) != 1) { 742 printf("%s: %s: UTRLRSR not set\n", 743 sc->sc_dev.dv_xname, __func__); 744 return 1; 745 } 746 747 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_utrd), 748 sizeof(*utrd) * slot, sizeof(*utrd), BUS_DMASYNC_PREWRITE); 749 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_ucd), 750 sizeof(*ucd) * slot, sizeof(*ucd), BUS_DMASYNC_PREWRITE); 751 752 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 14) */ 753 ccb->ccb_status = CCB_STATUS_INPROGRESS; 754 ufshci_doorbell_write(sc, slot); 755 756 return 0; 757 } 758 759 int 760 ufshci_utr_cmd_inquiry(struct ufshci_softc *sc, struct ufshci_ccb *ccb, 761 struct scsi_xfer *xs) 762 { 763 int slot, off, len, i; 764 uint64_t dva; 765 struct ufshci_utrd *utrd; 766 struct ufshci_ucd *ucd; 767 bus_dmamap_t dmap = ccb->ccb_dmamap; 768 769 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 1) */ 770 slot = ccb->ccb_slot; 771 utrd = UFSHCI_DMA_KVA(sc->sc_dmamem_utrd); 772 utrd += slot; 773 memset(utrd, 0, sizeof(*utrd)); 774 775 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2a) */ 776 utrd->dw0 = UFSHCI_UTRD_DW0_CT_UFS; 777 778 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2b) */ 779 utrd->dw0 |= UFSHCI_UTRD_DW0_DD_T2I; 780 781 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2c) */ 782 if (sc->sc_flags & UFSHCI_FLAGS_AGGR_INTR) 783 utrd->dw0 |= UFSHCI_UTRD_DW0_I_REG; 784 else 785 utrd->dw0 |= UFSHCI_UTRD_DW0_I_INT; 786 787 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2d) */ 788 utrd->dw2 = UFSHCI_UTRD_DW2_OCS_IOV; 789 790 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2e) */ 791 ucd = UFSHCI_DMA_KVA(sc->sc_dmamem_ucd); 792 ucd += slot; 793 memset(ucd, 0, sizeof(*ucd)); 794 795 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2f) */ 796 ucd->cmd.hdr.tc = UPIU_TC_I2T_COMMAND; 797 ucd->cmd.hdr.flags = (1 << 6); /* Bit-5 = Write, Bit-6 = Read */ 798 ucd->cmd.hdr.lun = 0; 799 ucd->cmd.hdr.task_tag = slot; 800 ucd->cmd.hdr.cmd_set_type = 0; /* SCSI command */ 801 ucd->cmd.hdr.query = 0; 802 ucd->cmd.hdr.response = 0; 803 ucd->cmd.hdr.status = 0; 804 ucd->cmd.hdr.ehs_len = 0; 805 ucd->cmd.hdr.device_info = 0; 806 ucd->cmd.hdr.ds_len = 0; 807 808 ucd->cmd.expected_xfer_len = htobe32(xs->datalen); 809 810 ucd->cmd.cdb[0] = INQUIRY; /* 0x12 */ 811 ucd->cmd.cdb[3] = 0; 812 ucd->cmd.cdb[4] = xs->datalen; 813 814 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2g) */ 815 /* Already done with above memset */ 816 817 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 3) */ 818 dva = UFSHCI_DMA_DVA(sc->sc_dmamem_ucd) + (sizeof(*ucd) * slot); 819 utrd->dw4 = (uint32_t)dva; 820 utrd->dw5 = (uint32_t)(dva >> 32); 821 822 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 4) */ 823 off = sizeof(struct upiu_command) / 4; /* DWORD offset */ 824 utrd->dw6 = UFSHCI_UTRD_DW6_RUO(off); 825 826 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 5) */ 827 len = sizeof(struct upiu_response) / 4; /* DWORD length */ 828 utrd->dw6 |= UFSHCI_UTRD_DW6_RUL(len); 829 830 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 6) */ 831 off = (sizeof(struct upiu_command) + sizeof(struct upiu_response)) / 4; 832 utrd->dw7 = UFSHCI_UTRD_DW7_PRDTO(off); 833 834 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 7) */ 835 utrd->dw7 |= UFSHCI_UTRD_DW7_PRDTL(dmap->dm_nsegs); 836 837 /* Build PRDT data segment. */ 838 for (i = 0; i < dmap->dm_nsegs; i++) { 839 dva = dmap->dm_segs[i].ds_addr; 840 ucd->prdt[i].dw0 = (uint32_t)dva; 841 ucd->prdt[i].dw1 = (uint32_t)(dva >> 32); 842 ucd->prdt[i].dw2 = 0; 843 ucd->prdt[i].dw3 = dmap->dm_segs[i].ds_len - 1; 844 } 845 846 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 9) */ 847 if (UFSHCI_READ_4(sc, UFSHCI_REG_UTRLRSR) != 1) { 848 printf("%s: %s: UTRLRSR not set\n", 849 sc->sc_dev.dv_xname, __func__); 850 return 1; 851 } 852 853 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_utrd), 854 sizeof(*utrd) * slot, sizeof(*utrd), BUS_DMASYNC_PREWRITE); 855 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_ucd), 856 sizeof(*ucd) * slot, sizeof(*ucd), BUS_DMASYNC_PREWRITE); 857 858 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 14) */ 859 ccb->ccb_status = CCB_STATUS_INPROGRESS; 860 ufshci_doorbell_write(sc, slot); 861 862 return 0; 863 } 864 865 int 866 ufshci_utr_cmd_capacity16(struct ufshci_softc *sc, struct ufshci_ccb *ccb, 867 struct scsi_xfer *xs) 868 { 869 int slot, off, len, i; 870 uint64_t dva; 871 struct ufshci_utrd *utrd; 872 struct ufshci_ucd *ucd; 873 bus_dmamap_t dmap = ccb->ccb_dmamap; 874 875 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 1) */ 876 slot = ccb->ccb_slot; 877 utrd = UFSHCI_DMA_KVA(sc->sc_dmamem_utrd); 878 utrd += slot; 879 memset(utrd, 0, sizeof(*utrd)); 880 881 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2a) */ 882 utrd->dw0 = UFSHCI_UTRD_DW0_CT_UFS; 883 884 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2b) */ 885 utrd->dw0 |= UFSHCI_UTRD_DW0_DD_T2I; 886 887 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2c) */ 888 if (sc->sc_flags & UFSHCI_FLAGS_AGGR_INTR) 889 utrd->dw0 |= UFSHCI_UTRD_DW0_I_REG; 890 else 891 utrd->dw0 |= UFSHCI_UTRD_DW0_I_INT; 892 893 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2d) */ 894 utrd->dw2 = UFSHCI_UTRD_DW2_OCS_IOV; 895 896 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2e) */ 897 ucd = UFSHCI_DMA_KVA(sc->sc_dmamem_ucd); 898 ucd += slot; 899 memset(ucd, 0, sizeof(*ucd)); 900 901 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2f) */ 902 ucd->cmd.hdr.tc = UPIU_TC_I2T_COMMAND; 903 ucd->cmd.hdr.flags = (1 << 6); /* Bit-5 = Write, Bit-6 = Read */ 904 ucd->cmd.hdr.lun = 0; 905 ucd->cmd.hdr.task_tag = slot; 906 ucd->cmd.hdr.cmd_set_type = 0; /* SCSI command */ 907 ucd->cmd.hdr.query = 0; 908 ucd->cmd.hdr.response = 0; 909 ucd->cmd.hdr.status = 0; 910 ucd->cmd.hdr.ehs_len = 0; 911 ucd->cmd.hdr.device_info = 0; 912 ucd->cmd.hdr.ds_len = 0; 913 914 ucd->cmd.expected_xfer_len = htobe32(xs->datalen); 915 916 ucd->cmd.cdb[0] = READ_CAPACITY_16; /* 0x9e */ 917 ucd->cmd.cdb[1] = 0x10; /* Service Action */ 918 /* Logical Block Address = 0 for UFS */ 919 ucd->cmd.cdb[10] = 0; 920 ucd->cmd.cdb[11] = 0; 921 ucd->cmd.cdb[12] = 0; 922 ucd->cmd.cdb[13] = xs->datalen; 923 924 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2g) */ 925 /* Already done with above memset */ 926 927 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 3) */ 928 dva = UFSHCI_DMA_DVA(sc->sc_dmamem_ucd) + (sizeof(*ucd) * slot); 929 utrd->dw4 = (uint32_t)dva; 930 utrd->dw5 = (uint32_t)(dva >> 32); 931 932 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 4) */ 933 off = sizeof(struct upiu_command) / 4; /* DWORD offset */ 934 utrd->dw6 = UFSHCI_UTRD_DW6_RUO(off); 935 936 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 5) */ 937 len = sizeof(struct upiu_response) / 4; /* DWORD length */ 938 utrd->dw6 |= UFSHCI_UTRD_DW6_RUL(len); 939 940 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 6) */ 941 off = (sizeof(struct upiu_command) + sizeof(struct upiu_response)) / 4; 942 utrd->dw7 = UFSHCI_UTRD_DW7_PRDTO(off); 943 944 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 7) */ 945 utrd->dw7 |= UFSHCI_UTRD_DW7_PRDTL(dmap->dm_nsegs); 946 947 /* Build PRDT data segment. */ 948 for (i = 0; i < dmap->dm_nsegs; i++) { 949 dva = dmap->dm_segs[i].ds_addr; 950 ucd->prdt[i].dw0 = (uint32_t)dva; 951 ucd->prdt[i].dw1 = (uint32_t)(dva >> 32); 952 ucd->prdt[i].dw2 = 0; 953 ucd->prdt[i].dw3 = dmap->dm_segs[i].ds_len - 1; 954 } 955 956 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 9) */ 957 if (UFSHCI_READ_4(sc, UFSHCI_REG_UTRLRSR) != 1) { 958 printf("%s: %s: UTRLRSR not set\n", 959 sc->sc_dev.dv_xname, __func__); 960 return 1; 961 } 962 963 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_utrd), 964 sizeof(*utrd) * slot, sizeof(*utrd), BUS_DMASYNC_PREWRITE); 965 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_ucd), 966 sizeof(*ucd) * slot, sizeof(*ucd), BUS_DMASYNC_PREWRITE); 967 968 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 14) */ 969 ccb->ccb_status = CCB_STATUS_INPROGRESS; 970 ufshci_doorbell_write(sc, slot); 971 972 return 0; 973 } 974 975 int 976 ufshci_utr_cmd_capacity(struct ufshci_softc *sc, struct ufshci_ccb *ccb, 977 struct scsi_xfer *xs) 978 { 979 int slot, off, len, i; 980 uint64_t dva; 981 struct ufshci_utrd *utrd; 982 struct ufshci_ucd *ucd; 983 bus_dmamap_t dmap = ccb->ccb_dmamap; 984 985 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 1) */ 986 slot = ccb->ccb_slot; 987 utrd = UFSHCI_DMA_KVA(sc->sc_dmamem_utrd); 988 utrd += slot; 989 memset(utrd, 0, sizeof(*utrd)); 990 991 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2a) */ 992 utrd->dw0 = UFSHCI_UTRD_DW0_CT_UFS; 993 994 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2b) */ 995 utrd->dw0 |= UFSHCI_UTRD_DW0_DD_T2I; 996 997 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2c) */ 998 if (sc->sc_flags & UFSHCI_FLAGS_AGGR_INTR) 999 utrd->dw0 |= UFSHCI_UTRD_DW0_I_REG; 1000 else 1001 utrd->dw0 |= UFSHCI_UTRD_DW0_I_INT; 1002 1003 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2d) */ 1004 utrd->dw2 = UFSHCI_UTRD_DW2_OCS_IOV; 1005 1006 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2e) */ 1007 ucd = UFSHCI_DMA_KVA(sc->sc_dmamem_ucd); 1008 ucd += slot; 1009 memset(ucd, 0, sizeof(*ucd)); 1010 1011 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2f) */ 1012 ucd->cmd.hdr.tc = UPIU_TC_I2T_COMMAND; 1013 ucd->cmd.hdr.flags = (1 << 6); /* Bit-5 = Write, Bit-6 = Read */ 1014 ucd->cmd.hdr.lun = 0; 1015 ucd->cmd.hdr.task_tag = slot; 1016 ucd->cmd.hdr.cmd_set_type = 0; /* SCSI command */ 1017 ucd->cmd.hdr.query = 0; 1018 ucd->cmd.hdr.response = 0; 1019 ucd->cmd.hdr.status = 0; 1020 ucd->cmd.hdr.ehs_len = 0; 1021 ucd->cmd.hdr.device_info = 0; 1022 ucd->cmd.hdr.ds_len = 0; 1023 1024 ucd->cmd.expected_xfer_len = htobe32(xs->datalen); 1025 1026 ucd->cmd.cdb[0] = READ_CAPACITY; /* 0x25 */ 1027 /* Logical Block Address = 0 for UFS */ 1028 ucd->cmd.cdb[2] = 0; 1029 ucd->cmd.cdb[3] = 0; 1030 ucd->cmd.cdb[4] = 0; 1031 ucd->cmd.cdb[5] = 0; 1032 1033 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2g) */ 1034 /* Already done with above memset */ 1035 1036 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 3) */ 1037 dva = UFSHCI_DMA_DVA(sc->sc_dmamem_ucd) + (sizeof(*ucd) * slot); 1038 utrd->dw4 = (uint32_t)dva; 1039 utrd->dw5 = (uint32_t)(dva >> 32); 1040 1041 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 4) */ 1042 off = sizeof(struct upiu_command) / 4; /* DWORD offset */ 1043 utrd->dw6 = UFSHCI_UTRD_DW6_RUO(off); 1044 1045 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 5) */ 1046 len = sizeof(struct upiu_response) / 4; /* DWORD length */ 1047 utrd->dw6 |= UFSHCI_UTRD_DW6_RUL(len); 1048 1049 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 6) */ 1050 off = (sizeof(struct upiu_command) + sizeof(struct upiu_response)) / 4; 1051 utrd->dw7 = UFSHCI_UTRD_DW7_PRDTO(off); 1052 1053 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 7) */ 1054 utrd->dw7 |= UFSHCI_UTRD_DW7_PRDTL(dmap->dm_nsegs); 1055 1056 /* Build PRDT data segment. */ 1057 for (i = 0; i < dmap->dm_nsegs; i++) { 1058 dva = dmap->dm_segs[i].ds_addr; 1059 ucd->prdt[i].dw0 = (uint32_t)dva; 1060 ucd->prdt[i].dw1 = (uint32_t)(dva >> 32); 1061 ucd->prdt[i].dw2 = 0; 1062 ucd->prdt[i].dw3 = dmap->dm_segs[i].ds_len - 1; 1063 } 1064 1065 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 9) */ 1066 if (UFSHCI_READ_4(sc, UFSHCI_REG_UTRLRSR) != 1) { 1067 printf("%s: %s: UTRLRSR not set\n", 1068 sc->sc_dev.dv_xname, __func__); 1069 return 1; 1070 } 1071 1072 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_utrd), 1073 sizeof(*utrd) * slot, sizeof(*utrd), BUS_DMASYNC_PREWRITE); 1074 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_ucd), 1075 sizeof(*ucd) * slot, sizeof(*ucd), BUS_DMASYNC_PREWRITE); 1076 1077 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 14) */ 1078 ccb->ccb_status = CCB_STATUS_INPROGRESS; 1079 ufshci_doorbell_write(sc, slot); 1080 1081 return 0; 1082 } 1083 1084 int 1085 ufshci_utr_cmd_io(struct ufshci_softc *sc, struct ufshci_ccb *ccb, 1086 struct scsi_xfer *xs, int dir) 1087 { 1088 int slot, off, len, i; 1089 uint64_t dva; 1090 struct ufshci_utrd *utrd; 1091 struct ufshci_ucd *ucd; 1092 bus_dmamap_t dmap = ccb->ccb_dmamap; 1093 uint32_t blocks; 1094 uint64_t lba; 1095 1096 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 1) */ 1097 slot = ccb->ccb_slot; 1098 utrd = UFSHCI_DMA_KVA(sc->sc_dmamem_utrd); 1099 utrd += slot; 1100 memset(utrd, 0, sizeof(*utrd)); 1101 1102 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2a) */ 1103 utrd->dw0 = UFSHCI_UTRD_DW0_CT_UFS; 1104 1105 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2b) */ 1106 if (dir == SCSI_DATA_IN) 1107 utrd->dw0 |= UFSHCI_UTRD_DW0_DD_T2I; 1108 else 1109 utrd->dw0 |= UFSHCI_UTRD_DW0_DD_I2T; 1110 1111 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2c) */ 1112 if (sc->sc_flags & UFSHCI_FLAGS_AGGR_INTR) 1113 utrd->dw0 |= UFSHCI_UTRD_DW0_I_REG; 1114 else 1115 utrd->dw0 |= UFSHCI_UTRD_DW0_I_INT; 1116 1117 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2d) */ 1118 utrd->dw2 = UFSHCI_UTRD_DW2_OCS_IOV; 1119 1120 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2e) */ 1121 ucd = UFSHCI_DMA_KVA(sc->sc_dmamem_ucd); 1122 ucd += slot; 1123 memset(ucd, 0, sizeof(*ucd)); 1124 1125 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2f) */ 1126 ucd->cmd.hdr.tc = UPIU_TC_I2T_COMMAND; 1127 if (dir == SCSI_DATA_IN) 1128 ucd->cmd.hdr.flags = (1 << 6); /* Bit-6 = Read */ 1129 else 1130 ucd->cmd.hdr.flags = (1 << 5); /* Bit-5 = Write */ 1131 ucd->cmd.hdr.lun = 0; 1132 ucd->cmd.hdr.task_tag = slot; 1133 ucd->cmd.hdr.cmd_set_type = 0; /* SCSI command */ 1134 ucd->cmd.hdr.query = 0; 1135 ucd->cmd.hdr.response = 0; 1136 ucd->cmd.hdr.status = 0; 1137 ucd->cmd.hdr.ehs_len = 0; 1138 ucd->cmd.hdr.device_info = 0; 1139 ucd->cmd.hdr.ds_len = 0; 1140 1141 /* 1142 * JESD220C-2_1.pdf, page 88, d) Expected Data Transfer Length: 1143 * "When the COMMAND UPIU encodes a SCSI WRITE or SCSI READ command 1144 * (specifically WRITE (6), READ (6), WRITE (10), READ (10), 1145 * WRITE (16), or READ (16)), the value of this field shall be the 1146 * product of the Logical Block Size (bLogicalBlockSize) and the 1147 * TRANSFER LENGTH field of the CDB." 1148 */ 1149 scsi_cmd_rw_decode(&xs->cmd, &lba, &blocks); 1150 ucd->cmd.expected_xfer_len = htobe32(UFSHCI_LBS * blocks); 1151 1152 memcpy(ucd->cmd.cdb, &xs->cmd, sizeof(ucd->cmd.cdb)); 1153 1154 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2g) */ 1155 /* Already done with above memset */ 1156 1157 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 3) */ 1158 dva = UFSHCI_DMA_DVA(sc->sc_dmamem_ucd) + (sizeof(*ucd) * slot); 1159 utrd->dw4 = (uint32_t)dva; 1160 utrd->dw5 = (uint32_t)(dva >> 32); 1161 1162 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 4) */ 1163 off = sizeof(struct upiu_command) / 4; /* DWORD offset */ 1164 utrd->dw6 = UFSHCI_UTRD_DW6_RUO(off); 1165 1166 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 5) */ 1167 len = sizeof(struct upiu_response) / 4; /* DWORD length */ 1168 utrd->dw6 |= UFSHCI_UTRD_DW6_RUL(len); 1169 1170 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 6) */ 1171 off = (sizeof(struct upiu_command) + sizeof(struct upiu_response)) / 4; 1172 utrd->dw7 = UFSHCI_UTRD_DW7_PRDTO(off); 1173 1174 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 7) */ 1175 utrd->dw7 |= UFSHCI_UTRD_DW7_PRDTL(dmap->dm_nsegs); 1176 1177 /* Build PRDT data segment. */ 1178 for (i = 0; i < dmap->dm_nsegs; i++) { 1179 dva = dmap->dm_segs[i].ds_addr; 1180 ucd->prdt[i].dw0 = (uint32_t)dva; 1181 ucd->prdt[i].dw1 = (uint32_t)(dva >> 32); 1182 ucd->prdt[i].dw2 = 0; 1183 ucd->prdt[i].dw3 = dmap->dm_segs[i].ds_len - 1; 1184 } 1185 1186 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 9) */ 1187 if (UFSHCI_READ_4(sc, UFSHCI_REG_UTRLRSR) != 1) { 1188 printf("%s: %s: UTRLRSR not set\n", 1189 sc->sc_dev.dv_xname, __func__); 1190 return 1; 1191 } 1192 1193 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_utrd), 1194 sizeof(*utrd) * slot, sizeof(*utrd), BUS_DMASYNC_PREWRITE); 1195 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_ucd), 1196 sizeof(*ucd) * slot, sizeof(*ucd), BUS_DMASYNC_PREWRITE); 1197 #if NKSTAT > 0 1198 if (sc->sc_stats_slots) 1199 sc->sc_stats_slots[slot]++; 1200 #endif 1201 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 14) */ 1202 ccb->ccb_status = CCB_STATUS_INPROGRESS; 1203 ufshci_doorbell_write(sc, slot); 1204 1205 return 0; 1206 } 1207 1208 int 1209 ufshci_utr_cmd_sync(struct ufshci_softc *sc, struct ufshci_ccb *ccb, 1210 struct scsi_xfer *xs, uint32_t lba, uint16_t blocks) 1211 { 1212 int slot, off, len; 1213 uint64_t dva; 1214 struct ufshci_utrd *utrd; 1215 struct ufshci_ucd *ucd; 1216 1217 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 1) */ 1218 slot = ccb->ccb_slot; 1219 utrd = UFSHCI_DMA_KVA(sc->sc_dmamem_utrd); 1220 utrd += slot; 1221 memset(utrd, 0, sizeof(*utrd)); 1222 1223 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2a) */ 1224 utrd->dw0 = UFSHCI_UTRD_DW0_CT_UFS; 1225 1226 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2b) */ 1227 utrd->dw0 |= UFSHCI_UTRD_DW0_DD_I2T; 1228 1229 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2c) */ 1230 if (sc->sc_flags & UFSHCI_FLAGS_AGGR_INTR) 1231 utrd->dw0 |= UFSHCI_UTRD_DW0_I_REG; 1232 else 1233 utrd->dw0 |= UFSHCI_UTRD_DW0_I_INT; 1234 1235 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2d) */ 1236 utrd->dw2 = UFSHCI_UTRD_DW2_OCS_IOV; 1237 1238 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2e) */ 1239 ucd = UFSHCI_DMA_KVA(sc->sc_dmamem_ucd); 1240 ucd += slot; 1241 memset(ucd, 0, sizeof(*ucd)); 1242 1243 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2f) */ 1244 ucd->cmd.hdr.tc = UPIU_TC_I2T_COMMAND; 1245 ucd->cmd.hdr.flags = 0; /* No data transfer */ 1246 ucd->cmd.hdr.lun = 0; 1247 ucd->cmd.hdr.task_tag = slot; 1248 ucd->cmd.hdr.cmd_set_type = 0; /* SCSI command */ 1249 ucd->cmd.hdr.query = 0; 1250 ucd->cmd.hdr.response = 0; 1251 ucd->cmd.hdr.status = 0; 1252 ucd->cmd.hdr.ehs_len = 0; 1253 ucd->cmd.hdr.device_info = 0; 1254 ucd->cmd.hdr.ds_len = 0; 1255 1256 ucd->cmd.expected_xfer_len = htobe32(0); /* No data transfer */ 1257 1258 ucd->cmd.cdb[0] = SYNCHRONIZE_CACHE; /* 0x35 */ 1259 ucd->cmd.cdb[2] = (lba >> 24) & 0xff; 1260 ucd->cmd.cdb[3] = (lba >> 16) & 0xff; 1261 ucd->cmd.cdb[4] = (lba >> 8) & 0xff; 1262 ucd->cmd.cdb[5] = (lba >> 0) & 0xff; 1263 ucd->cmd.cdb[7] = (blocks >> 8) & 0xff; 1264 ucd->cmd.cdb[8] = (blocks >> 0) & 0xff; 1265 1266 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2g) */ 1267 /* Already done with above memset */ 1268 1269 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 3) */ 1270 dva = UFSHCI_DMA_DVA(sc->sc_dmamem_ucd) + (sizeof(*ucd) * slot); 1271 utrd->dw4 = (uint32_t)dva; 1272 utrd->dw5 = (uint32_t)(dva >> 32); 1273 1274 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 4) */ 1275 off = sizeof(struct upiu_command) / 4; /* DWORD offset */ 1276 utrd->dw6 = UFSHCI_UTRD_DW6_RUO(off); 1277 1278 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 5) */ 1279 len = sizeof(struct upiu_response) / 4; /* DWORD length */ 1280 utrd->dw6 |= UFSHCI_UTRD_DW6_RUL(len); 1281 1282 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 6) */ 1283 off = (sizeof(struct upiu_command) + sizeof(struct upiu_response)) / 4; 1284 utrd->dw7 = UFSHCI_UTRD_DW7_PRDTO(off); 1285 1286 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 7) */ 1287 utrd->dw7 |= UFSHCI_UTRD_DW7_PRDTL(0); /* No data xfer */ 1288 1289 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 9) */ 1290 if (UFSHCI_READ_4(sc, UFSHCI_REG_UTRLRSR) != 1) { 1291 printf("%s: %s: UTRLRSR not set\n", 1292 sc->sc_dev.dv_xname, __func__); 1293 return 1; 1294 } 1295 1296 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_utrd), 1297 sizeof(*utrd) * slot, sizeof(*utrd), BUS_DMASYNC_PREWRITE); 1298 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_ucd), 1299 sizeof(*ucd) * slot, sizeof(*ucd), BUS_DMASYNC_PREWRITE); 1300 1301 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 14) */ 1302 ccb->ccb_status = CCB_STATUS_INPROGRESS; 1303 ufshci_doorbell_write(sc, slot); 1304 1305 return 0; 1306 } 1307 1308 void 1309 ufshci_xfer_complete(struct ufshci_softc *sc) 1310 { 1311 struct ufshci_ccb *ccb; 1312 uint32_t reg; 1313 int i, timeout; 1314 1315 mtx_enter(&sc->sc_cmd_mtx); 1316 1317 /* Wait for all commands to complete. */ 1318 for (timeout = 5000; timeout != 0; timeout--) { 1319 reg = ufshci_doorbell_read(sc); 1320 if (reg == 0) 1321 break; 1322 delay(10); 1323 } 1324 if (timeout == 0) 1325 printf("%s: timeout (reg=0x%x)\n", __func__, reg); 1326 1327 for (i = 0; i < sc->sc_nutrs; i++) { 1328 ccb = &sc->sc_ccbs[i]; 1329 1330 /* Skip unused CCBs. */ 1331 if (ccb->ccb_status != CCB_STATUS_INPROGRESS) 1332 continue; 1333 1334 if (ccb->ccb_done == NULL) 1335 panic("ccb done wasn't defined"); 1336 1337 /* 7.2.3: Clear completion notification 3b) */ 1338 UFSHCI_WRITE_4(sc, UFSHCI_REG_UTRLCNR, (1U << i)); 1339 1340 /* 7.2.3: Mark software slot for reuse 3c) */ 1341 ccb->ccb_status = CCB_STATUS_READY2FREE; 1342 } 1343 1344 /* 7.2.3: Reset Interrupt Aggregation Counter and Timer 4) */ 1345 if (sc->sc_flags & UFSHCI_FLAGS_AGGR_INTR) { 1346 UFSHCI_WRITE_4(sc, UFSHCI_REG_UTRIACR, 1347 UFSHCI_REG_UTRIACR_IAEN | UFSHCI_REG_UTRIACR_CTR); 1348 } 1349 1350 mtx_leave(&sc->sc_cmd_mtx); 1351 1352 /* 1353 * Complete the CCB, which will re-schedule new transfers if any are 1354 * pending. 1355 */ 1356 for (i = 0; i < sc->sc_nutrs; i++) { 1357 ccb = &sc->sc_ccbs[i]; 1358 1359 /* 7.2.3: Process the transfer by higher OS layer 3a) */ 1360 if (ccb->ccb_status == CCB_STATUS_READY2FREE) 1361 ccb->ccb_done(sc, ccb); 1362 } 1363 } 1364 1365 int 1366 ufshci_activate(struct device *self, int act) 1367 { 1368 struct ufshci_softc *sc = (struct ufshci_softc *)self; 1369 int rv = 0; 1370 1371 switch (act) { 1372 case DVACT_POWERDOWN: 1373 DPRINTF(1, "%s: POWERDOWN\n", __func__); 1374 rv = config_activate_children(&sc->sc_dev, act); 1375 ufshci_disable(sc); 1376 break; 1377 case DVACT_RESUME: 1378 DPRINTF(1, "%s: RESUME\n", __func__); 1379 rv = ufshci_init(sc); 1380 if (rv == 0) 1381 rv = config_activate_children(&sc->sc_dev, act); 1382 break; 1383 default: 1384 rv = config_activate_children(&sc->sc_dev, act); 1385 break; 1386 } 1387 1388 return rv; 1389 } 1390 1391 /* SCSI */ 1392 1393 int 1394 ufshci_ccb_alloc(struct ufshci_softc *sc, int nccbs) 1395 { 1396 struct ufshci_ccb *ccb; 1397 int i; 1398 1399 DPRINTF(2, "%s: nccbs=%d, dma_size=%d, dma_nsegs=%d, " 1400 "dma_segmaxsize=%d\n", 1401 __func__, nccbs, UFSHCI_UCD_PRDT_MAX_XFER, UFSHCI_UCD_PRDT_MAX_SEGS, 1402 UFSHCI_UCD_PRDT_MAX_XFER); 1403 1404 sc->sc_ccbs = mallocarray(nccbs, sizeof(*ccb), M_DEVBUF, 1405 M_WAITOK | M_CANFAIL); 1406 if (sc->sc_ccbs == NULL) 1407 return 1; 1408 1409 for (i = 0; i < nccbs; i++) { 1410 ccb = &sc->sc_ccbs[i]; 1411 1412 if (bus_dmamap_create(sc->sc_dmat, UFSHCI_UCD_PRDT_MAX_XFER, 1413 UFSHCI_UCD_PRDT_MAX_SEGS, UFSHCI_UCD_PRDT_MAX_XFER, 0, 1414 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW | 1415 ((sc->sc_cap & UFSHCI_REG_CAP_64AS) ? BUS_DMA_64BIT : 0), 1416 &ccb->ccb_dmamap) != 0) 1417 goto free_maps; 1418 1419 ccb->ccb_cookie = NULL; 1420 ccb->ccb_status = CCB_STATUS_FREE; 1421 ccb->ccb_slot = i; 1422 1423 SIMPLEQ_INSERT_TAIL(&sc->sc_ccb_list, ccb, ccb_entry); 1424 } 1425 1426 return 0; 1427 1428 free_maps: 1429 ufshci_ccb_free(sc, nccbs); 1430 return 1; 1431 } 1432 1433 void * 1434 ufshci_ccb_get(void *cookie) 1435 { 1436 struct ufshci_softc *sc = cookie; 1437 struct ufshci_ccb *ccb; 1438 1439 mtx_enter(&sc->sc_ccb_mtx); 1440 ccb = SIMPLEQ_FIRST(&sc->sc_ccb_list); 1441 if (ccb != NULL) 1442 SIMPLEQ_REMOVE_HEAD(&sc->sc_ccb_list, ccb_entry); 1443 mtx_leave(&sc->sc_ccb_mtx); 1444 1445 return ccb; 1446 } 1447 1448 void 1449 ufshci_ccb_put(void *cookie, void *io) 1450 { 1451 struct ufshci_softc *sc = cookie; 1452 struct ufshci_ccb *ccb = io; 1453 1454 mtx_enter(&sc->sc_ccb_mtx); 1455 SIMPLEQ_INSERT_HEAD(&sc->sc_ccb_list, ccb, ccb_entry); 1456 mtx_leave(&sc->sc_ccb_mtx); 1457 } 1458 1459 void 1460 ufshci_ccb_free(struct ufshci_softc *sc, int nccbs) 1461 { 1462 struct ufshci_ccb *ccb; 1463 1464 while ((ccb = SIMPLEQ_FIRST(&sc->sc_ccb_list)) != NULL) { 1465 SIMPLEQ_REMOVE_HEAD(&sc->sc_ccb_list, ccb_entry); 1466 bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap); 1467 } 1468 1469 ufshci_dmamem_free(sc, sc->sc_dmamem_utrd); 1470 free(sc->sc_ccbs, M_DEVBUF, nccbs * sizeof(*ccb)); 1471 } 1472 1473 void 1474 ufshci_scsi_cmd(struct scsi_xfer *xs) 1475 { 1476 struct scsi_link *link = xs->sc_link; 1477 struct ufshci_softc *sc = link->bus->sb_adapter_softc; 1478 1479 mtx_enter(&sc->sc_cmd_mtx); 1480 1481 switch (xs->cmd.opcode) { 1482 1483 case READ_COMMAND: 1484 case READ_10: 1485 case READ_12: 1486 case READ_16: 1487 ufshci_scsi_io(xs, SCSI_DATA_IN); 1488 break; 1489 case WRITE_COMMAND: 1490 case WRITE_10: 1491 case WRITE_12: 1492 case WRITE_16: 1493 ufshci_scsi_io(xs, SCSI_DATA_OUT); 1494 break; 1495 case SYNCHRONIZE_CACHE: 1496 ufshci_scsi_sync(xs); 1497 break; 1498 case INQUIRY: 1499 ufshci_scsi_inquiry(xs); 1500 break; 1501 case READ_CAPACITY_16: 1502 ufshci_scsi_capacity16(xs); 1503 break; 1504 case READ_CAPACITY: 1505 ufshci_scsi_capacity(xs); 1506 break; 1507 case TEST_UNIT_READY: 1508 case PREVENT_ALLOW: 1509 case START_STOP: 1510 xs->error = XS_NOERROR; 1511 scsi_done(xs); 1512 break; 1513 default: 1514 DPRINTF(3, "%s: unhandled scsi command 0x%02x\n", 1515 __func__, xs->cmd.opcode); 1516 xs->error = XS_DRIVER_STUFFUP; 1517 scsi_done(xs); 1518 break; 1519 } 1520 1521 mtx_leave(&sc->sc_cmd_mtx); 1522 } 1523 1524 void 1525 ufshci_scsi_inquiry(struct scsi_xfer *xs) 1526 { 1527 struct scsi_link *link = xs->sc_link; 1528 struct ufshci_softc *sc = link->bus->sb_adapter_softc; 1529 struct ufshci_ccb *ccb = xs->io; 1530 bus_dmamap_t dmap = ccb->ccb_dmamap; 1531 int error; 1532 1533 DPRINTF(3, "%s: INQUIRY (%s)\n", 1534 __func__, ISSET(xs->flags, SCSI_POLL) ? "poll" : "no poll"); 1535 1536 if (xs->datalen > UPIU_SCSI_RSP_INQUIRY_SIZE) { 1537 DPRINTF(2, "%s: request len too large\n", __func__); 1538 goto error1; 1539 } 1540 1541 error = bus_dmamap_load(sc->sc_dmat, dmap, xs->data, xs->datalen, NULL, 1542 ISSET(xs->flags, SCSI_NOSLEEP) ? BUS_DMA_NOWAIT : BUS_DMA_WAITOK); 1543 if (error) { 1544 printf("%s: bus_dmamap_load error=%d\n", __func__, error); 1545 goto error1; 1546 } 1547 1548 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 1549 BUS_DMASYNC_PREREAD); 1550 1551 ccb->ccb_cookie = xs; 1552 ccb->ccb_done = ufshci_scsi_io_done; 1553 1554 /* Response length should be UPIU_SCSI_RSP_INQUIRY_SIZE. */ 1555 error = ufshci_utr_cmd_inquiry(sc, ccb, xs); 1556 if (error) 1557 goto error2; 1558 1559 if (ISSET(xs->flags, SCSI_POLL)) { 1560 if (ufshci_doorbell_poll(sc, ccb->ccb_slot, xs->timeout) == 0) { 1561 ccb->ccb_done(sc, ccb); 1562 return; 1563 } 1564 goto error2; 1565 } 1566 1567 return; 1568 1569 error2: 1570 bus_dmamap_unload(sc->sc_dmat, dmap); 1571 ccb->ccb_cookie = NULL; 1572 ccb->ccb_status = CCB_STATUS_FREE; 1573 ccb->ccb_done = NULL; 1574 error1: 1575 xs->error = XS_DRIVER_STUFFUP; 1576 scsi_done(xs); 1577 } 1578 1579 void 1580 ufshci_scsi_capacity16(struct scsi_xfer *xs) 1581 { 1582 struct scsi_link *link = xs->sc_link; 1583 struct ufshci_softc *sc = link->bus->sb_adapter_softc; 1584 struct ufshci_ccb *ccb = xs->io; 1585 bus_dmamap_t dmap = ccb->ccb_dmamap; 1586 int error; 1587 1588 DPRINTF(3, "%s: CAPACITY16 (%s)\n", 1589 __func__, ISSET(xs->flags, SCSI_POLL) ? "poll" : "no poll"); 1590 1591 if (xs->datalen > UPIU_SCSI_RSP_CAPACITY16_SIZE) { 1592 DPRINTF(2, "%s: request len too large\n", __func__); 1593 goto error1; 1594 } 1595 1596 error = bus_dmamap_load(sc->sc_dmat, dmap, xs->data, xs->datalen, NULL, 1597 ISSET(xs->flags, SCSI_NOSLEEP) ? BUS_DMA_NOWAIT : BUS_DMA_WAITOK); 1598 if (error) { 1599 printf("%s: bus_dmamap_load error=%d\n", __func__, error); 1600 goto error1; 1601 } 1602 1603 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 1604 BUS_DMASYNC_PREREAD); 1605 1606 ccb->ccb_cookie = xs; 1607 ccb->ccb_done = ufshci_scsi_io_done; 1608 1609 /* Response length should be UPIU_SCSI_RSP_CAPACITY16_SIZE. */ 1610 error = ufshci_utr_cmd_capacity16(sc, ccb, xs); 1611 if (error) 1612 goto error2; 1613 1614 if (ISSET(xs->flags, SCSI_POLL)) { 1615 if (ufshci_doorbell_poll(sc, ccb->ccb_slot, xs->timeout) == 0) { 1616 ccb->ccb_done(sc, ccb); 1617 return; 1618 } 1619 goto error2; 1620 } 1621 1622 return; 1623 1624 error2: 1625 bus_dmamap_unload(sc->sc_dmat, dmap); 1626 ccb->ccb_cookie = NULL; 1627 ccb->ccb_status = CCB_STATUS_FREE; 1628 ccb->ccb_done = NULL; 1629 error1: 1630 xs->error = XS_DRIVER_STUFFUP; 1631 scsi_done(xs); 1632 } 1633 1634 void 1635 ufshci_scsi_capacity(struct scsi_xfer *xs) 1636 { 1637 struct scsi_link *link = xs->sc_link; 1638 struct ufshci_softc *sc = link->bus->sb_adapter_softc; 1639 struct ufshci_ccb *ccb = xs->io; 1640 bus_dmamap_t dmap = ccb->ccb_dmamap; 1641 int error; 1642 1643 DPRINTF(3, "%s: CAPACITY (%s)\n", 1644 __func__, ISSET(xs->flags, SCSI_POLL) ? "poll" : "no poll"); 1645 1646 if (xs->datalen > UPIU_SCSI_RSP_CAPACITY_SIZE) { 1647 DPRINTF(2, "%s: request len too large\n", __func__); 1648 goto error1; 1649 } 1650 1651 error = bus_dmamap_load(sc->sc_dmat, dmap, xs->data, xs->datalen, NULL, 1652 ISSET(xs->flags, SCSI_NOSLEEP) ? BUS_DMA_NOWAIT : BUS_DMA_WAITOK); 1653 if (error) { 1654 printf("%s: bus_dmamap_load error=%d\n", __func__, error); 1655 goto error1; 1656 } 1657 1658 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 1659 BUS_DMASYNC_PREREAD); 1660 1661 ccb->ccb_cookie = xs; 1662 ccb->ccb_done = ufshci_scsi_io_done; 1663 1664 /* Response length should be UPIU_SCSI_RSP_CAPACITY_SIZE */ 1665 error = ufshci_utr_cmd_capacity(sc, ccb, xs); 1666 if (error) 1667 goto error2; 1668 1669 if (ISSET(xs->flags, SCSI_POLL)) { 1670 if (ufshci_doorbell_poll(sc, ccb->ccb_slot, xs->timeout) == 0) { 1671 ccb->ccb_done(sc, ccb); 1672 return; 1673 } 1674 goto error2; 1675 } 1676 1677 return; 1678 1679 error2: 1680 bus_dmamap_unload(sc->sc_dmat, dmap); 1681 ccb->ccb_cookie = NULL; 1682 ccb->ccb_status = CCB_STATUS_FREE; 1683 ccb->ccb_done = NULL; 1684 error1: 1685 xs->error = XS_DRIVER_STUFFUP; 1686 scsi_done(xs); 1687 } 1688 1689 void 1690 ufshci_scsi_sync(struct scsi_xfer *xs) 1691 { 1692 struct scsi_link *link = xs->sc_link; 1693 struct ufshci_softc *sc = link->bus->sb_adapter_softc; 1694 struct ufshci_ccb *ccb = xs->io; 1695 uint64_t lba; 1696 uint32_t blocks; 1697 int error; 1698 1699 /* lba = 0, blocks = 0: Synchronize all logical blocks. */ 1700 lba = 0; blocks = 0; 1701 1702 DPRINTF(3, "%s: SYNC, lba=%llu, blocks=%u (%s)\n", 1703 __func__, lba, blocks, 1704 ISSET(xs->flags, SCSI_POLL) ? "poll" : "no poll"); 1705 1706 ccb->ccb_cookie = xs; 1707 ccb->ccb_done = ufshci_scsi_done; 1708 1709 error = ufshci_utr_cmd_sync(sc, ccb, xs, (uint32_t)lba, 1710 (uint16_t)blocks); 1711 if (error) 1712 goto error; 1713 1714 if (ISSET(xs->flags, SCSI_POLL)) { 1715 if (ufshci_doorbell_poll(sc, ccb->ccb_slot, xs->timeout) == 0) { 1716 ccb->ccb_done(sc, ccb); 1717 return; 1718 } 1719 goto error; 1720 } 1721 1722 return; 1723 1724 error: 1725 ccb->ccb_cookie = NULL; 1726 ccb->ccb_status = CCB_STATUS_FREE; 1727 ccb->ccb_done = NULL; 1728 1729 xs->error = XS_DRIVER_STUFFUP; 1730 scsi_done(xs); 1731 } 1732 1733 void 1734 ufshci_scsi_io(struct scsi_xfer *xs, int dir) 1735 { 1736 struct scsi_link *link = xs->sc_link; 1737 struct ufshci_softc *sc = link->bus->sb_adapter_softc; 1738 struct ufshci_ccb *ccb = xs->io; 1739 bus_dmamap_t dmap = ccb->ccb_dmamap; 1740 int error; 1741 1742 if ((xs->flags & (SCSI_DATA_IN | SCSI_DATA_OUT)) != dir) 1743 goto error1; 1744 1745 error = bus_dmamap_load(sc->sc_dmat, dmap, xs->data, xs->datalen, NULL, 1746 ISSET(xs->flags, SCSI_NOSLEEP) ? BUS_DMA_NOWAIT : BUS_DMA_WAITOK); 1747 if (error) { 1748 printf("%s: bus_dmamap_load error=%d\n", __func__, error); 1749 goto error1; 1750 } 1751 1752 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 1753 ISSET(xs->flags, SCSI_DATA_IN) ? BUS_DMASYNC_PREREAD : 1754 BUS_DMASYNC_PREWRITE); 1755 1756 ccb->ccb_cookie = xs; 1757 ccb->ccb_done = ufshci_scsi_io_done; 1758 1759 if (dir == SCSI_DATA_IN) 1760 error = ufshci_utr_cmd_io(sc, ccb, xs, SCSI_DATA_IN); 1761 else 1762 error = ufshci_utr_cmd_io(sc, ccb, xs, SCSI_DATA_OUT); 1763 if (error) 1764 goto error2; 1765 1766 if (ISSET(xs->flags, SCSI_POLL)) { 1767 if (ufshci_doorbell_poll(sc, ccb->ccb_slot, xs->timeout) == 0) { 1768 ccb->ccb_done(sc, ccb); 1769 return; 1770 } 1771 goto error2; 1772 } 1773 1774 return; 1775 1776 error2: 1777 bus_dmamap_unload(sc->sc_dmat, dmap); 1778 ccb->ccb_cookie = NULL; 1779 ccb->ccb_status = CCB_STATUS_FREE; 1780 ccb->ccb_done = NULL; 1781 error1: 1782 xs->error = XS_DRIVER_STUFFUP; 1783 scsi_done(xs); 1784 } 1785 1786 void 1787 ufshci_scsi_io_done(struct ufshci_softc *sc, struct ufshci_ccb *ccb) 1788 { 1789 struct scsi_xfer *xs = ccb->ccb_cookie; 1790 bus_dmamap_t dmap = ccb->ccb_dmamap; 1791 struct ufshci_ucd *ucd; 1792 struct ufshci_utrd *utrd; 1793 1794 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 1795 ISSET(xs->flags, SCSI_DATA_IN) ? BUS_DMASYNC_POSTREAD : 1796 BUS_DMASYNC_POSTWRITE); 1797 1798 bus_dmamap_unload(sc->sc_dmat, dmap); 1799 1800 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_ucd), 1801 sizeof(*ucd) * ccb->ccb_slot, sizeof(*ucd), 1802 BUS_DMASYNC_POSTWRITE); 1803 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_utrd), 1804 sizeof(*utrd) * ccb->ccb_slot, sizeof(*utrd), 1805 BUS_DMASYNC_POSTWRITE); 1806 1807 /* TODO: Do more checks on the Response UPIU in case of errors? */ 1808 utrd = UFSHCI_DMA_KVA(sc->sc_dmamem_utrd); 1809 utrd += ccb->ccb_slot; 1810 ucd = UFSHCI_DMA_KVA(sc->sc_dmamem_ucd); 1811 ucd += ccb->ccb_slot; 1812 if (utrd->dw2 != UFSHCI_UTRD_DW2_OCS_SUCCESS) { 1813 printf("%s: error: slot=%d, ocs=0x%x, rsp-tc=0x%x\n", 1814 __func__, ccb->ccb_slot, utrd->dw2, ucd->rsp.hdr.tc); 1815 } 1816 1817 ccb->ccb_cookie = NULL; 1818 ccb->ccb_status = CCB_STATUS_FREE; 1819 ccb->ccb_done = NULL; 1820 1821 xs->error = (utrd->dw2 == UFSHCI_UTRD_DW2_OCS_SUCCESS) ? 1822 XS_NOERROR : XS_DRIVER_STUFFUP; 1823 xs->status = SCSI_OK; 1824 xs->resid = 0; 1825 scsi_done(xs); 1826 } 1827 1828 void 1829 ufshci_scsi_done(struct ufshci_softc *sc, struct ufshci_ccb *ccb) 1830 { 1831 struct scsi_xfer *xs = ccb->ccb_cookie; 1832 struct ufshci_ucd *ucd; 1833 struct ufshci_utrd *utrd; 1834 1835 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_ucd), 1836 sizeof(*ucd) * ccb->ccb_slot, sizeof(*ucd), 1837 BUS_DMASYNC_POSTWRITE); 1838 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_utrd), 1839 sizeof(*utrd) * ccb->ccb_slot, sizeof(*utrd), 1840 BUS_DMASYNC_POSTWRITE); 1841 1842 /* TODO: Do more checks on the Response UPIU in case of errors? */ 1843 utrd = UFSHCI_DMA_KVA(sc->sc_dmamem_utrd); 1844 utrd += ccb->ccb_slot; 1845 ucd = UFSHCI_DMA_KVA(sc->sc_dmamem_ucd); 1846 ucd += ccb->ccb_slot; 1847 if (utrd->dw2 != UFSHCI_UTRD_DW2_OCS_SUCCESS) { 1848 printf("%s: error: slot=%d, ocs=0x%x, rsp-tc=0x%x\n", 1849 __func__, ccb->ccb_slot, utrd->dw2, ucd->rsp.hdr.tc); 1850 } 1851 1852 ccb->ccb_cookie = NULL; 1853 ccb->ccb_status = CCB_STATUS_FREE; 1854 ccb->ccb_done = NULL; 1855 1856 xs->error = (utrd->dw2 == UFSHCI_UTRD_DW2_OCS_SUCCESS) ? 1857 XS_NOERROR : XS_DRIVER_STUFFUP; 1858 xs->status = SCSI_OK; 1859 xs->resid = 0; 1860 scsi_done(xs); 1861 } 1862 1863 #ifdef HIBERNATE 1864 int 1865 ufshci_hibernate_io(dev_t dev, daddr_t blkno, vaddr_t addr, size_t size, 1866 int op, void *page) 1867 { 1868 struct ufshci_hibernate_page { 1869 struct ufshci_utrd utrd; 1870 struct ufshci_ucd ucd; 1871 1872 struct ufshci_softc *sc; /* Copy of softc */ 1873 1874 daddr_t poffset; /* Start of SWAP partition */ 1875 size_t psize; /* Size of SWAP partition */ 1876 uint32_t secsize; /* Our sector size */ 1877 } *my = page; 1878 paddr_t data_phys, page_phys; 1879 uint64_t data_bus_phys, page_bus_phys; 1880 uint64_t timeout_us; 1881 int off, len, slot; 1882 uint32_t blocks, reg; 1883 uint64_t lba; 1884 1885 if (op == HIB_INIT) { 1886 struct device *disk; 1887 struct device *scsibus; 1888 extern struct cfdriver sd_cd; 1889 1890 /* Find ufshci softc. */ 1891 disk = disk_lookup(&sd_cd, DISKUNIT(dev)); 1892 if (disk == NULL) 1893 return ENOTTY; 1894 scsibus = disk->dv_parent; 1895 my->sc = (struct ufshci_softc *)disk->dv_parent->dv_parent; 1896 1897 /* Stop run queues and disable interrupts. */ 1898 ufshci_disable(my->sc); 1899 1900 /* Tell the controller the new hibernate UTRD address. */ 1901 pmap_extract(pmap_kernel(), (vaddr_t)page, &page_phys); 1902 page_bus_phys = page_phys + ((void *)&my->utrd - page); 1903 UFSHCI_WRITE_4(my->sc, UFSHCI_REG_UTRLBA, 1904 (uint32_t)page_bus_phys); 1905 UFSHCI_WRITE_4(my->sc, UFSHCI_REG_UTRLBAU, 1906 (uint32_t)(page_bus_phys >> 32)); 1907 1908 /* Start run queues. */ 1909 UFSHCI_WRITE_4(my->sc, UFSHCI_REG_UTMRLRSR, 1910 UFSHCI_REG_UTMRLRSR_START); 1911 UFSHCI_WRITE_4(my->sc, UFSHCI_REG_UTRLRSR, 1912 UFSHCI_REG_UTRLRSR_START); 1913 1914 my->poffset = blkno; 1915 my->psize = size; 1916 my->secsize = UFSHCI_LBS; 1917 1918 return 0; 1919 } 1920 1921 if (op != HIB_W) 1922 return 0; 1923 1924 if (blkno + (size / DEV_BSIZE) > my->psize) 1925 return E2BIG; 1926 blocks = size / my->secsize; 1927 lba = (blkno + my->poffset) / (my->secsize / DEV_BSIZE); 1928 1929 /* 1930 * The following code is a ripped down version of ufshci_utr_cmd_io() 1931 * adapted for hibernate. 1932 */ 1933 slot = 0; /* We only use the first slot for hibernate */ 1934 1935 memset(&my->utrd, 0, sizeof(struct ufshci_utrd)); 1936 1937 my->utrd.dw0 = UFSHCI_UTRD_DW0_CT_UFS; 1938 my->utrd.dw0 |= UFSHCI_UTRD_DW0_DD_I2T; 1939 my->utrd.dw0 |= UFSHCI_UTRD_DW0_I_REG; 1940 my->utrd.dw2 = UFSHCI_UTRD_DW2_OCS_IOV; 1941 1942 memset(&my->ucd, 0, sizeof(struct ufshci_ucd)); 1943 1944 my->ucd.cmd.hdr.tc = UPIU_TC_I2T_COMMAND; 1945 my->ucd.cmd.hdr.flags = (1 << 5); /* Bit-5 = Write */ 1946 1947 my->ucd.cmd.hdr.lun = 0; 1948 my->ucd.cmd.hdr.task_tag = slot; 1949 my->ucd.cmd.hdr.cmd_set_type = 0; /* SCSI command */ 1950 my->ucd.cmd.hdr.query = 0; 1951 my->ucd.cmd.hdr.response = 0; 1952 my->ucd.cmd.hdr.status = 0; 1953 my->ucd.cmd.hdr.ehs_len = 0; 1954 my->ucd.cmd.hdr.device_info = 0; 1955 my->ucd.cmd.hdr.ds_len = 0; 1956 1957 my->ucd.cmd.expected_xfer_len = htobe32(UFSHCI_LBS * blocks); 1958 my->ucd.cmd.cdb[0] = WRITE_10; /* 0x2a */ 1959 my->ucd.cmd.cdb[1] = (1 << 3); /* FUA: Force Unit Access */ 1960 my->ucd.cmd.cdb[2] = (lba >> 24) & 0xff; 1961 my->ucd.cmd.cdb[3] = (lba >> 16) & 0xff; 1962 my->ucd.cmd.cdb[4] = (lba >> 8) & 0xff; 1963 my->ucd.cmd.cdb[5] = (lba >> 0) & 0xff; 1964 my->ucd.cmd.cdb[7] = (blocks >> 8) & 0xff; 1965 my->ucd.cmd.cdb[8] = (blocks >> 0) & 0xff; 1966 1967 pmap_extract(pmap_kernel(), (vaddr_t)page, &page_phys); 1968 page_bus_phys = page_phys + ((void *)&my->ucd - page); 1969 my->utrd.dw4 = (uint32_t)page_bus_phys; 1970 my->utrd.dw5 = (uint32_t)(page_bus_phys >> 32); 1971 1972 off = sizeof(struct upiu_command) / 4; /* DWORD offset */ 1973 my->utrd.dw6 = UFSHCI_UTRD_DW6_RUO(off); 1974 1975 len = sizeof(struct upiu_response) / 4; /* DWORD length */ 1976 my->utrd.dw6 |= UFSHCI_UTRD_DW6_RUL(len); 1977 1978 off = (sizeof(struct upiu_command) + sizeof(struct upiu_response)) / 4; 1979 my->utrd.dw7 = UFSHCI_UTRD_DW7_PRDTO(off); 1980 1981 my->utrd.dw7 |= UFSHCI_UTRD_DW7_PRDTL(1); /* dm_nsegs */ 1982 1983 pmap_extract(pmap_kernel(), (vaddr_t)addr, &data_phys); 1984 data_bus_phys = data_phys; 1985 my->ucd.prdt[0].dw0 = (uint32_t)data_bus_phys; 1986 my->ucd.prdt[0].dw1 = (uint32_t)(data_bus_phys >> 32); 1987 my->ucd.prdt[0].dw2 = 0; 1988 my->ucd.prdt[0].dw3 = size - 1; /* ds_len */ 1989 1990 if (UFSHCI_READ_4(my->sc, UFSHCI_REG_UTRLRSR) != 1) 1991 return EIO; 1992 1993 ufshci_doorbell_write(my->sc, slot); 1994 1995 /* ufshci_doorbell_poll() adaption for hibernate. */ 1996 for (timeout_us = 1000000 * 1000; timeout_us != 0; 1997 timeout_us -= 1000) { 1998 reg = UFSHCI_READ_4(my->sc, UFSHCI_REG_UTRLDBR); 1999 if ((reg & (1U << slot)) == 0) 2000 break; 2001 delay(1000); 2002 } 2003 if (timeout_us == 0) 2004 return EIO; 2005 UFSHCI_WRITE_4(my->sc, UFSHCI_REG_UTRLCNR, (1U << slot)); 2006 2007 /* Check if the command was successfully executed. */ 2008 if (my->utrd.dw2 != UFSHCI_UTRD_DW2_OCS_SUCCESS) 2009 return EIO; 2010 2011 return 0; 2012 } 2013 #endif /* HIBERNATE */ 2014 2015 #if NKSTAT > 0 2016 struct kstat_kv ufshci_counters_slot[CCB_STATUS_COUNT] = { 2017 KSTAT_KV_UNIT_INITIALIZER("slots free", KSTAT_KV_T_COUNTER16, 2018 KSTAT_KV_U_NONE), 2019 KSTAT_KV_UNIT_INITIALIZER("slots inpr", KSTAT_KV_T_COUNTER16, 2020 KSTAT_KV_U_NONE), 2021 KSTAT_KV_UNIT_INITIALIZER("slots r2fr", KSTAT_KV_T_COUNTER16, 2022 KSTAT_KV_U_NONE), 2023 }; 2024 2025 void 2026 ufshci_kstat_attach(struct ufshci_softc *sc) 2027 { 2028 struct kstat *ks; 2029 struct kstat_kv *kvs; 2030 char name[KSTAT_KV_NAMELEN]; 2031 int i; 2032 2033 /* 2034 * Allocate array to count ccb slot utilization. 2035 */ 2036 sc->sc_stats_slots = mallocarray(sc->sc_nutrs, sizeof(uint64_t), 2037 M_DEVBUF, M_WAITOK | M_ZERO); 2038 if (sc->sc_stats_slots == NULL) { 2039 printf("%s: can't allocate stats_slots array\n", 2040 sc->sc_dev.dv_xname); 2041 return; 2042 } 2043 2044 /* 2045 * Setup 'ccbs' kstat. 2046 */ 2047 kvs = mallocarray(sc->sc_nutrs, sizeof(*kvs), M_DEVBUF, 2048 M_WAITOK | M_ZERO); 2049 if (kvs == NULL) { 2050 printf("%s: can't allocate kvs ccbs array\n", 2051 sc->sc_dev.dv_xname); 2052 return; 2053 } 2054 for (i = 0; i < sc->sc_nutrs; i++) { 2055 snprintf(name, sizeof(name), "slot %d ccbs", i); 2056 kstat_kv_unit_init(&kvs[i], name, KSTAT_KV_T_COUNTER64, 2057 KSTAT_KV_U_NONE); 2058 } 2059 2060 mtx_init(&sc->sc_kstat_mtx_ccb, IPL_SOFTCLOCK); 2061 2062 ks = kstat_create(sc->sc_dev.dv_xname, 0, "ccbs", 0, KSTAT_T_KV, 0); 2063 if (ks == NULL) { 2064 printf("%s: can't create ccbs kstats\n", sc->sc_dev.dv_xname); 2065 free(kvs, M_DEVBUF, sc->sc_nutrs * sizeof(*kvs)); 2066 return; 2067 } 2068 2069 kstat_set_mutex(ks, &sc->sc_kstat_mtx_ccb); 2070 ks->ks_softc = sc; 2071 ks->ks_data = kvs; 2072 ks->ks_datalen = sc->sc_nutrs * sizeof(*kvs); 2073 ks->ks_read = ufshci_kstat_read_ccb; 2074 2075 sc->sc_kstat_ccb = ks; 2076 kstat_install(ks); 2077 2078 /* 2079 * Setup 'slots' kstat. 2080 */ 2081 mtx_init(&sc->sc_kstat_mtx_slot, IPL_SOFTCLOCK); 2082 2083 ks = kstat_create(sc->sc_dev.dv_xname, 0, "slots", 0, KSTAT_T_KV, 0); 2084 if (ks == NULL) { 2085 printf("%s: can't create slots kstats\n", sc->sc_dev.dv_xname); 2086 return; 2087 } 2088 2089 kstat_set_mutex(ks, &sc->sc_kstat_mtx_slot); 2090 ks->ks_softc = sc; 2091 ks->ks_data = ufshci_counters_slot; 2092 ks->ks_datalen = CCB_STATUS_COUNT * sizeof(*kvs); 2093 ks->ks_read = ufshci_kstat_read_slot; 2094 2095 sc->sc_kstat_slot = ks; 2096 kstat_install(ks); 2097 } 2098 2099 int 2100 ufshci_kstat_read_ccb(struct kstat *ks) 2101 { 2102 struct ufshci_softc *sc = ks->ks_softc; 2103 struct kstat_kv *kvs = ks->ks_data; 2104 int i; 2105 2106 for (i = 0; i < sc->sc_nutrs; i++) 2107 kstat_kv_u64(&kvs[i]) = sc->sc_stats_slots[i]; 2108 2109 return 0; 2110 } 2111 2112 int 2113 ufshci_kstat_read_slot(struct kstat *ks) 2114 { 2115 struct ufshci_softc *sc = ks->ks_softc; 2116 struct kstat_kv *kvs = ks->ks_data; 2117 struct ufshci_ccb *ccb; 2118 uint16_t free, inprogress, ready2free; 2119 int i; 2120 2121 free = inprogress = ready2free = 0; 2122 2123 for (i = 0; i < sc->sc_nutrs; i++) { 2124 ccb = &sc->sc_ccbs[i]; 2125 2126 switch (ccb->ccb_status) { 2127 case CCB_STATUS_FREE: 2128 free++; 2129 break; 2130 case CCB_STATUS_INPROGRESS: 2131 inprogress++; 2132 break; 2133 case CCB_STATUS_READY2FREE: 2134 ready2free++; 2135 break; 2136 } 2137 } 2138 2139 kstat_kv_u16(&kvs[0]) = free; 2140 kstat_kv_u16(&kvs[1]) = inprogress; 2141 kstat_kv_u16(&kvs[2]) = ready2free; 2142 2143 return 0; 2144 } 2145 #endif /* NKSTAT > 0 */ 2146