1 /* $NetBSD: apple_dart.c,v 1.3 2021/09/06 14:03:17 jmcneill Exp $ */ 2 3 /*- 4 * Copyright (c) 2021 Mark Kettenis <kettenis@openbsd.org> 5 * Copyright (c) 2021 Jared McNeill <jmcneill@invisible.ca> 6 * 7 * Permission to use, copy, modify, and distribute this software for any 8 * purpose with or without fee is hereby granted, provided that the above 9 * copyright notice and this permission notice appear in all copies. 10 * 11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 14 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 16 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 17 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 */ 19 20 //#define APPLE_DART_DEBUG 21 22 #include <sys/cdefs.h> 23 __KERNEL_RCSID(0, "$NetBSD: apple_dart.c,v 1.3 2021/09/06 14:03:17 jmcneill Exp $"); 24 25 #include <sys/param.h> 26 #include <sys/bus.h> 27 #include <sys/device.h> 28 #include <sys/intr.h> 29 #include <sys/kernel.h> 30 #include <sys/systm.h> 31 #include <sys/kmem.h> 32 #include <sys/vmem.h> 33 34 #include <arm/cpufunc.h> 35 36 #include <dev/fdt/fdtvar.h> 37 38 /* 39 * DART registers 40 */ 41 #define DART_TLB_OP 0x0020 42 #define DART_TLB_OP_FLUSH __BIT(20) 43 #define DART_TLB_OP_BUSY __BIT(2) 44 #define DART_TLB_OP_SIDMASK 0x0034 45 #define DART_ERR_STATUS 0x0040 46 #define DART_ERR_ADDRL 0x0050 47 #define DART_ERR_ADDRH 0x0054 48 #define DART_CONFIG(sid) (0x0100 + (sid) * 0x4) 49 #define DART_CONFIG_TXEN __BIT(7) 50 #define DART_TTBR(sid, idx) (0x0200 + (sid) * 0x10 + (idx) * 0x4) 51 #define DART_TTBR_VALID __BIT(31) 52 #define DART_TTBR_SHIFT 12 53 54 #define DART_APERTURE_START 0x00100000 55 #define DART_APERTURE_SIZE 0x3fe00000 56 #define DART_PAGE_SIZE 16384 57 #define DART_PAGE_MASK (DART_PAGE_SIZE - 1) 58 59 #define DART_L1_TABLE 0xb 60 #define DART_L2_INVAL 0x0 61 #define DART_L2_PAGE 0x3 62 63 #define DART_ROUND_PAGE(pa) (((pa) + DART_PAGE_MASK) & ~DART_PAGE_MASK) 64 #define DART_TRUNC_PAGE(pa) ((pa) & ~DART_PAGE_MASK) 65 66 static const struct device_compatible_entry compat_data[] = { 67 { .compat = "apple,dart-m1", .value = 16 }, 68 DEVICE_COMPAT_EOL 69 }; 70 71 static struct arm32_dma_range apple_dart_dma_ranges[] = { 72 [0] = { 73 .dr_sysbase = 0, 74 .dr_busbase = 0, 75 .dr_len = UINTPTR_MAX, 76 .dr_flags = _BUS_DMAMAP_COHERENT, 77 } 78 }; 79 80 struct apple_dart_map_state { 81 bus_addr_t ams_dva; 82 bus_size_t ams_len; 83 }; 84 85 struct apple_dart_dma { 86 bus_dmamap_t dma_map; 87 bus_dma_segment_t dma_seg; 88 bus_size_t dma_size; 89 void *dma_kva; 90 }; 91 92 #define DART_DMA_MAP(_dma) ((_dma)->dma_map) 93 #define DART_DMA_LEN(_dma) ((_dma)->dma_size) 94 #define DART_DMA_DVA(_dma) ((_dma)->dma_map->dm_segs[0].ds_addr) 95 #define DART_DMA_KVA(_dma) ((_dma)->dma_kva) 96 97 struct apple_dart_softc { 98 device_t sc_dev; 99 int sc_phandle; 100 bus_space_tag_t sc_bst; 101 bus_space_handle_t sc_bsh; 102 bus_dma_tag_t sc_dmat; 103 104 uint64_t sc_sid_mask; 105 u_int sc_nsid; 106 107 vmem_t *sc_dvamap; 108 109 struct apple_dart_dma *sc_l1; 110 struct apple_dart_dma **sc_l2; 111 u_int sc_nl2; 112 113 struct arm32_bus_dma_tag sc_bus_dmat; 114 }; 115 116 #define DART_READ(sc, reg) \ 117 bus_space_read_4((sc)->sc_bst, (sc)->sc_bsh, (reg)) 118 #define DART_WRITE(sc, reg, val) \ 119 bus_space_write_4((sc)->sc_bst, (sc)->sc_bsh, (reg), (val)) 120 121 static void 122 apple_dart_flush_tlb(struct apple_dart_softc *sc) 123 { 124 dsb(sy); 125 isb(); 126 127 DART_WRITE(sc, DART_TLB_OP_SIDMASK, sc->sc_sid_mask); 128 DART_WRITE(sc, DART_TLB_OP, DART_TLB_OP_FLUSH); 129 while ((DART_READ(sc, DART_TLB_OP) & DART_TLB_OP_BUSY) != 0) { 130 __asm volatile ("yield" ::: "memory"); 131 } 132 } 133 134 static struct apple_dart_dma * 135 apple_dart_dma_alloc(bus_dma_tag_t dmat, bus_size_t size, bus_size_t align) 136 { 137 struct apple_dart_dma *dma; 138 int nsegs, error; 139 140 dma = kmem_zalloc(sizeof(*dma), KM_SLEEP); 141 dma->dma_size = size; 142 143 error = bus_dmamem_alloc(dmat, size, align, 0, &dma->dma_seg, 1, 144 &nsegs, BUS_DMA_WAITOK); 145 if (error != 0) { 146 goto destroy; 147 } 148 149 error = bus_dmamem_map(dmat, &dma->dma_seg, nsegs, size, 150 &dma->dma_kva, BUS_DMA_WAITOK | BUS_DMA_NOCACHE); 151 if (error != 0) { 152 goto free; 153 } 154 155 error = bus_dmamap_create(dmat, size, 1, size, 0, 156 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &dma->dma_map); 157 if (error != 0) { 158 goto dmafree; 159 } 160 161 error = bus_dmamap_load(dmat, dma->dma_map, dma->dma_kva, size, 162 NULL, BUS_DMA_WAITOK); 163 if (error != 0) { 164 goto unmap; 165 } 166 167 memset(dma->dma_kva, 0, size); 168 169 return dma; 170 171 destroy: 172 bus_dmamap_destroy(dmat, dma->dma_map); 173 unmap: 174 bus_dmamem_unmap(dmat, dma->dma_kva, size); 175 free: 176 bus_dmamem_free(dmat, &dma->dma_seg, 1); 177 dmafree: 178 kmem_free(dma, sizeof(*dma)); 179 return NULL; 180 } 181 182 static int 183 apple_dart_intr(void *priv) 184 { 185 struct apple_dart_softc * const sc = priv; 186 char fdt_path[128]; 187 uint64_t addr; 188 uint32_t status; 189 190 status = DART_READ(sc, DART_ERR_STATUS); 191 addr = DART_READ(sc, DART_ERR_ADDRL); 192 addr |= (uint64_t)DART_READ(sc, DART_ERR_ADDRH) << 32; 193 DART_WRITE(sc, DART_ERR_STATUS, status); 194 195 fdtbus_get_path(sc->sc_phandle, fdt_path, sizeof(fdt_path)); 196 197 printf("%s (%s): error addr 0x%016lx status 0x%08x\n", 198 device_xname(sc->sc_dev), fdt_path, addr, status); 199 200 return 1; 201 } 202 203 static volatile uint64_t * 204 apple_dart_lookup_tte(struct apple_dart_softc *sc, bus_addr_t dva) 205 { 206 int idx = dva / DART_PAGE_SIZE; 207 int l2_idx = idx / (DART_PAGE_SIZE / sizeof(uint64_t)); 208 int tte_idx = idx % (DART_PAGE_SIZE / sizeof(uint64_t)); 209 volatile uint64_t *l2; 210 211 l2 = DART_DMA_KVA(sc->sc_l2[l2_idx]); 212 return &l2[tte_idx]; 213 } 214 215 static void 216 apple_dart_unload_map(struct apple_dart_softc *sc, bus_dmamap_t map) 217 { 218 struct apple_dart_map_state *ams = map->_dm_iommu; 219 volatile uint64_t *tte; 220 int seg; 221 222 /* For each segment */ 223 for (seg = 0; seg < map->dm_nsegs; seg++) { 224 u_long len, dva; 225 226 if (ams[seg].ams_len == 0) { 227 continue; 228 } 229 230 dva = ams[seg].ams_dva; 231 len = ams[seg].ams_len; 232 233 while (len > 0) { 234 tte = apple_dart_lookup_tte(sc, dva); 235 *tte = DART_L2_INVAL; 236 237 dva += DART_PAGE_SIZE; 238 len -= DART_PAGE_SIZE; 239 } 240 241 vmem_xfree(sc->sc_dvamap, ams[seg].ams_dva, ams[seg].ams_len); 242 243 ams[seg].ams_dva = 0; 244 ams[seg].ams_len = 0; 245 } 246 247 apple_dart_flush_tlb(sc); 248 } 249 250 static int 251 apple_dart_load_map(struct apple_dart_softc *sc, bus_dmamap_t map) 252 { 253 struct apple_dart_map_state *ams = map->_dm_iommu; 254 volatile uint64_t *tte; 255 int seg, error; 256 257 /* For each segment */ 258 for (seg = 0; seg < map->dm_nsegs; seg++) { 259 paddr_t pa = map->dm_segs[seg]._ds_paddr; 260 psize_t off = pa - DART_TRUNC_PAGE(pa); 261 u_long len, dva; 262 263 len = DART_ROUND_PAGE(map->dm_segs[seg].ds_len + off); 264 265 #ifdef APPLE_DART_DEBUG 266 device_printf(sc->sc_dev, "load pa=%#lx off=%lu len=%lu ", 267 pa, off, len); 268 #endif 269 270 error = vmem_xalloc(sc->sc_dvamap, len, DART_PAGE_SIZE, 0, 271 0, VMEM_ADDR_MIN, VMEM_ADDR_MAX, VM_BESTFIT|VM_NOSLEEP, 272 &dva); 273 if (error != 0) { 274 apple_dart_unload_map(sc, map); 275 #ifdef APPLE_DART_DEBUG 276 printf("error=%d\n", error); 277 #endif 278 return error; 279 } 280 281 #ifdef APPLE_DART_DEBUG 282 printf("dva=%#lx\n", dva); 283 #endif 284 285 ams[seg].ams_dva = dva; 286 ams[seg].ams_len = len; 287 288 map->dm_segs[seg].ds_addr = dva + off; 289 290 pa = DART_TRUNC_PAGE(pa); 291 while (len > 0) { 292 tte = apple_dart_lookup_tte(sc, dva); 293 *tte = pa | DART_L2_PAGE; 294 295 pa += DART_PAGE_SIZE; 296 dva += DART_PAGE_SIZE; 297 len -= DART_PAGE_SIZE; 298 } 299 } 300 301 apple_dart_flush_tlb(sc); 302 303 return 0; 304 } 305 306 static int 307 apple_dart_dmamap_create(bus_dma_tag_t t, bus_size_t size, int nsegments, 308 bus_size_t maxsegsz, bus_size_t boundary, int flags, bus_dmamap_t *dmamap) 309 { 310 struct apple_dart_softc *sc = t->_cookie; 311 struct apple_dart_map_state *ams; 312 bus_dmamap_t map; 313 int error; 314 315 error = sc->sc_dmat->_dmamap_create(sc->sc_dmat, size, nsegments, 316 maxsegsz, boundary, flags, &map); 317 if (error != 0) { 318 return error; 319 } 320 321 ams = kmem_zalloc(map->_dm_segcnt * sizeof(*ams), 322 (flags & BUS_DMA_NOWAIT) != 0 ? KM_NOSLEEP : KM_SLEEP); 323 if (ams == NULL) { 324 sc->sc_dmat->_dmamap_destroy(sc->sc_dmat, map); 325 return ENOMEM; 326 } 327 328 map->_dm_iommu = ams; 329 *dmamap = map; 330 return 0; 331 } 332 333 static void 334 apple_dart_dmamap_destroy(bus_dma_tag_t t, bus_dmamap_t map) 335 { 336 struct apple_dart_softc *sc = t->_cookie; 337 struct apple_dart_map_state *ams = map->_dm_iommu; 338 339 kmem_free(ams, map->_dm_segcnt * sizeof(*ams)); 340 sc->sc_dmat->_dmamap_destroy(sc->sc_dmat, map); 341 } 342 343 static int 344 apple_dart_dmamap_load(bus_dma_tag_t t, bus_dmamap_t map, void *buf, 345 size_t buflen, struct proc *p, int flags) 346 { 347 struct apple_dart_softc *sc = t->_cookie; 348 int error; 349 350 error = sc->sc_dmat->_dmamap_load(sc->sc_dmat, map, 351 buf, buflen, p, flags); 352 if (error != 0) { 353 return error; 354 } 355 356 error = apple_dart_load_map(sc, map); 357 if (error != 0) { 358 sc->sc_dmat->_dmamap_unload(sc->sc_dmat, map); 359 } 360 361 return error; 362 } 363 364 static int 365 apple_dart_dmamap_load_mbuf(bus_dma_tag_t t, bus_dmamap_t map, 366 struct mbuf *m, int flags) 367 { 368 struct apple_dart_softc *sc = t->_cookie; 369 int error; 370 371 error = sc->sc_dmat->_dmamap_load_mbuf(sc->sc_dmat, map, 372 m, flags); 373 if (error != 0) { 374 return error; 375 } 376 377 error = apple_dart_load_map(sc, map); 378 if (error != 0) { 379 sc->sc_dmat->_dmamap_unload(sc->sc_dmat, map); 380 } 381 382 return error; 383 } 384 385 static int 386 apple_dart_dmamap_load_uio(bus_dma_tag_t t, bus_dmamap_t map, 387 struct uio *uio, int flags) 388 { 389 struct apple_dart_softc *sc = t->_cookie; 390 int error; 391 392 error = sc->sc_dmat->_dmamap_load_uio(sc->sc_dmat, map, 393 uio, flags); 394 if (error != 0) { 395 return error; 396 } 397 398 error = apple_dart_load_map(sc, map); 399 if (error != 0) { 400 sc->sc_dmat->_dmamap_unload(sc->sc_dmat, map); 401 } 402 403 return error; 404 } 405 406 static int 407 apple_dart_dmamap_load_raw(bus_dma_tag_t t, bus_dmamap_t map, 408 bus_dma_segment_t *segs, int nsegs, bus_size_t size, int flags) 409 { 410 struct apple_dart_softc *sc = t->_cookie; 411 int error; 412 413 error = sc->sc_dmat->_dmamap_load_raw(sc->sc_dmat, map, 414 segs, nsegs, size, flags); 415 if (error != 0) { 416 return error; 417 } 418 419 error = apple_dart_load_map(sc, map); 420 if (error != 0) { 421 sc->sc_dmat->_dmamap_unload(sc->sc_dmat, map); 422 } 423 424 return error; 425 } 426 427 static void 428 apple_dart_dmamap_unload(bus_dma_tag_t t, bus_dmamap_t map) 429 { 430 struct apple_dart_softc *sc = t->_cookie; 431 432 apple_dart_unload_map(sc, map); 433 sc->sc_dmat->_dmamap_unload(sc->sc_dmat, map); 434 } 435 436 static bus_dma_tag_t 437 apple_dart_iommu_map(device_t dev, const u_int *data, bus_dma_tag_t dmat) 438 { 439 struct apple_dart_softc * const sc = device_private(dev); 440 441 return &sc->sc_bus_dmat; 442 } 443 444 const struct fdtbus_iommu_func apple_dart_iommu_funcs = { 445 .map = apple_dart_iommu_map, 446 }; 447 448 static int 449 apple_dart_match(device_t parent, cfdata_t cf, void *aux) 450 { 451 struct fdt_attach_args * const faa = aux; 452 453 return of_compatible_match(faa->faa_phandle, compat_data); 454 } 455 456 static void 457 apple_dart_attach(device_t parent, device_t self, void *aux) 458 { 459 struct apple_dart_softc * const sc = device_private(self); 460 struct fdt_attach_args * const faa = aux; 461 const int phandle = faa->faa_phandle; 462 uint64_t sidmask64; 463 uint32_t sidmask32; 464 char intrstr[128]; 465 volatile uint64_t *l1; 466 bus_addr_t addr; 467 bus_size_t size; 468 u_int sid, idx; 469 paddr_t pa; 470 void *ih; 471 472 if (fdtbus_get_reg(phandle, 0, &addr, &size) != 0) { 473 aprint_error(": couldn't get registers\n"); 474 return; 475 } 476 if (!fdtbus_intr_str(phandle, 0, intrstr, sizeof(intrstr))) { 477 aprint_error(": couldn't decode interrupt\n"); 478 return; 479 } 480 481 sc->sc_dev = self; 482 sc->sc_phandle = phandle; 483 sc->sc_dmat = faa->faa_dmat; 484 sc->sc_bst = faa->faa_bst; 485 if (bus_space_map(sc->sc_bst, addr, size, 0, &sc->sc_bsh) != 0) { 486 aprint_error(": couldn't map registers\n"); 487 return; 488 } 489 sc->sc_nsid = of_compatible_lookup(phandle, compat_data)->value; 490 491 if (of_getprop_uint64(phandle, "sid-mask", &sidmask64) == 0) { 492 sc->sc_sid_mask = sidmask64; 493 } else if (of_getprop_uint32(phandle, "sid-mask", &sidmask32) == 0) { 494 sc->sc_sid_mask = sidmask32; 495 } else { 496 sc->sc_sid_mask = 0xffff; 497 } 498 499 aprint_naive("\n"); 500 aprint_normal(": Apple DART @ %#lx/%#lx, %u SIDs (mask 0x%lx)\n", 501 addr, size, sc->sc_nsid, sc->sc_sid_mask); 502 503 KASSERT(sc->sc_nsid == 16); 504 KASSERT(sc->sc_sid_mask == 0xffff); 505 506 sc->sc_dvamap = vmem_create(device_xname(self), 507 DART_APERTURE_START, DART_APERTURE_SIZE, DART_PAGE_SIZE, 508 NULL, NULL, NULL, 0, VM_SLEEP, IPL_HIGH); 509 if (sc->sc_dvamap == NULL) { 510 aprint_error_dev(self, "couldn't allocate DVA map\n"); 511 return; 512 } 513 514 /* Disable translations */ 515 for (sid = 0; sid < sc->sc_nsid; sid++) { 516 DART_WRITE(sc, DART_CONFIG(sid), 0); 517 } 518 519 /* Remove page tables */ 520 for (sid = 0; sid < sc->sc_nsid; sid++) { 521 for (idx = 0; idx < 4; idx++) { 522 DART_WRITE(sc, DART_TTBR(sid, idx), 0); 523 } 524 } 525 apple_dart_flush_tlb(sc); 526 527 /* 528 * Build translation tables. We pre-allocate the translation 529 * tables for the entire aperture such that we don't have to worry 530 * about growing them in an mpsafe manner later. 531 */ 532 533 const u_int ntte = howmany(DART_APERTURE_START + DART_APERTURE_SIZE - 1, 534 DART_PAGE_SIZE); 535 const u_int nl2 = howmany(ntte, DART_PAGE_SIZE / sizeof(uint64_t)); 536 const u_int nl1 = howmany(nl2, DART_PAGE_SIZE / sizeof(uint64_t)); 537 538 sc->sc_l1 = apple_dart_dma_alloc(sc->sc_dmat, 539 nl1 * DART_PAGE_SIZE, DART_PAGE_SIZE); 540 if (sc->sc_l1 == NULL) { 541 aprint_error_dev(self, "couldn't allocate L1 tables\n"); 542 return; 543 } 544 sc->sc_l2 = kmem_zalloc(nl2 * sizeof(*sc->sc_l2), KM_SLEEP); 545 sc->sc_nl2 = nl2; 546 547 l1 = DART_DMA_KVA(sc->sc_l1); 548 for (idx = 0; idx < nl2; idx++) { 549 sc->sc_l2[idx] = apple_dart_dma_alloc(sc->sc_dmat, 550 DART_PAGE_SIZE, DART_PAGE_SIZE); 551 if (sc->sc_l2[idx] == NULL) { 552 aprint_error_dev(self, 553 "couldn't allocate L2 tables\n"); 554 return; 555 } 556 l1[idx] = DART_DMA_DVA(sc->sc_l2[idx]) | DART_L1_TABLE; 557 } 558 559 /* Install page tables */ 560 for (sid = 0; sid < sc->sc_nsid; sid++) { 561 pa = DART_DMA_DVA(sc->sc_l1); 562 for (idx = 0; idx < nl1; idx++) { 563 DART_WRITE(sc, DART_TTBR(sid, idx), 564 (pa >> DART_TTBR_SHIFT) | DART_TTBR_VALID); 565 pa += DART_PAGE_SIZE; 566 } 567 } 568 apple_dart_flush_tlb(sc); 569 570 /* Enable translations */ 571 for (sid = 0; sid < sc->sc_nsid; sid++) { 572 DART_WRITE(sc, DART_CONFIG(sid), DART_CONFIG_TXEN); 573 } 574 575 ih = fdtbus_intr_establish_xname(phandle, 0, IPL_HIGH, FDT_INTR_MPSAFE, 576 apple_dart_intr, sc, device_xname(self)); 577 if (ih == NULL) { 578 aprint_error_dev(self, "couldn't establish interrupt on %s\n", 579 intrstr); 580 return; 581 } 582 aprint_normal_dev(self, "interrupting on %s\n", intrstr); 583 584 /* Setup bus DMA tag */ 585 sc->sc_bus_dmat = *sc->sc_dmat; 586 sc->sc_bus_dmat._ranges = apple_dart_dma_ranges; 587 sc->sc_bus_dmat._nranges = 1; 588 sc->sc_bus_dmat._cookie = sc; 589 sc->sc_bus_dmat._dmamap_create = apple_dart_dmamap_create; 590 sc->sc_bus_dmat._dmamap_destroy = apple_dart_dmamap_destroy; 591 sc->sc_bus_dmat._dmamap_load = apple_dart_dmamap_load; 592 sc->sc_bus_dmat._dmamap_load_mbuf = apple_dart_dmamap_load_mbuf; 593 sc->sc_bus_dmat._dmamap_load_uio = apple_dart_dmamap_load_uio; 594 sc->sc_bus_dmat._dmamap_load_raw = apple_dart_dmamap_load_raw; 595 sc->sc_bus_dmat._dmamap_unload = apple_dart_dmamap_unload; 596 597 fdtbus_register_iommu(self, phandle, &apple_dart_iommu_funcs); 598 } 599 600 CFATTACH_DECL_NEW(apple_dart, sizeof(struct apple_dart_softc), 601 apple_dart_match, apple_dart_attach, NULL, NULL); 602