Lines Matching +full:ats +full:- +full:supported
1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
4 * Copyright (c) 2019-2020 Ruslan Bukin <br@bsdpad.com>
8 * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
39 * In case of PCI-based devices, StreamID is a PCI rid.
42 * which contains per-device configuration.
44 * Stream table is a linear or 2-level walk table (this driver supports both).
67 * Register interface and Memory-based circular buffer queues are used
79 * in a producer-consumer fashion so that an output queue contains data
143 #define Q_WRP(q, p) ((p) & (1 << (q)->size_log2))
144 #define Q_IDX(q, p) ((p) & ((1 << (q)->size_log2) - 1))
182 "and a PCIe ATS Translation Request received."},
184 "The STE of a transaction marks non-substream transactions "
230 if (Q_IDX(q, q->lc.cons) != Q_IDX(q, q->lc.prod) ||
231 Q_WRP(q, q->lc.cons) == Q_WRP(q, q->lc.prod))
241 if (Q_IDX(q, q->lc.cons) == Q_IDX(q, q->lc.prod) &&
242 Q_WRP(q, q->lc.cons) == Q_WRP(q, q->lc.prod))
252 if ((Q_WRP(q, q->lc.cons) == Q_WRP(q, prod)) &&
253 (Q_IDX(q, q->lc.cons) >= Q_IDX(q, prod)))
256 if ((Q_WRP(q, q->lc.cons) != Q_WRP(q, prod)) &&
257 (Q_IDX(q, q->lc.cons) <= Q_IDX(q, prod)))
269 cons = (Q_WRP(q, q->lc.cons) | Q_IDX(q, q->lc.cons)) + 1;
270 val = (Q_OVF(q->lc.cons) | Q_WRP(q, cons) | Q_IDX(q, cons));
281 prod = (Q_WRP(q, q->lc.prod) | Q_IDX(q, q->lc.prod)) + 1;
282 val = (Q_OVF(q->lc.prod) | Q_WRP(q, prod) | Q_IDX(q, prod));
296 bus_write_4(sc->res[0], reg, val);
299 v = bus_read_4(sc->res[0], reg_ack);
302 } while (timeout--);
305 device_printf(sc->dev, "Failed to write reg.\n");
306 return (-1);
318 sz = (1 << q->size_log2) * dwords * 8;
321 q->vaddr = contigmalloc(sz, M_SMMU,
322 M_WAITOK | M_ZERO, 0, (1ul << 48) - 1, SMMU_Q_ALIGN, 0);
323 if (q->vaddr == NULL) {
324 device_printf(sc->dev, "failed to allocate %d bytes\n", sz);
325 return (-1);
328 q->prod_off = prod_off;
329 q->cons_off = cons_off;
330 q->paddr = vtophys(q->vaddr);
332 q->base = CMDQ_BASE_RA | EVENTQ_BASE_WA | PRIQ_BASE_WA;
333 q->base |= q->paddr & Q_BASE_ADDR_M;
334 q->base |= q->size_log2 << Q_LOG2SIZE_S;
345 err = smmu_init_queue(sc, &sc->cmdq,
351 err = smmu_init_queue(sc, &sc->evtq,
356 if (!(sc->features & SMMU_FEATURE_PRI))
360 err = smmu_init_queue(sc, &sc->priq,
379 strtab = &sc->strtab;
381 if (sc->features & SMMU_FEATURE_2_LVL_STREAM_TABLE) {
383 l1 = (void *)((uint64_t)strtab->vaddr +
385 device_printf(sc->dev, "L1 ste == %lx\n", l1[0]);
387 l1_desc = &strtab->l1[i];
388 ste = l1_desc->va;
392 ste = (void *)((uint64_t)strtab->vaddr +
398 device_printf(sc->dev, "ste[%d] == %lx\n", i, ste[i]);
407 device_printf(sc->dev, "%s\n", __func__);
409 vaddr = cd->vaddr;
411 device_printf(sc->dev, "cd[%d] == %lx\n", i, vaddr[i]);
420 evtq = &sc->evtq;
422 evtq->lc.val = bus_read_8(sc->res[0], evtq->prod_off);
423 entry_addr = (void *)((uint64_t)evtq->vaddr +
424 evtq->lc.cons * EVTQ_ENTRY_DWORDS * 8);
426 evtq->lc.cons = smmu_q_inc_cons(evtq);
427 bus_write_4(sc->res[0], evtq->cons_off, evtq->lc.cons);
440 dev = sc->dev;
462 device_printf(sc->dev,
463 "Event %s (%s) received.\n", ev->str, ev->msg);
465 device_printf(sc->dev, "Event 0x%x received\n", event_id);
467 device_printf(sc->dev, "SID %x, Input Address: %jx\n",
471 device_printf(sc->dev, "evt[%d] %x\n", i, evt[i]);
482 cmd[0] = entry->opcode << CMD_QUEUE_OPCODE_S;
484 switch (entry->opcode) {
486 cmd[0] |= (uint64_t)entry->tlbi.asid << TLBI_0_ASID_S;
487 cmd[1] = entry->tlbi.addr & TLBI_1_ADDR_M;
488 if (entry->tlbi.leaf) {
498 cmd[0] |= (uint64_t)entry->tlbi.asid << TLBI_0_ASID_S;
505 cmd[0] |= ((uint64_t)entry->cfgi.ssid << CFGI_0_SSID_S);
508 cmd[0] |= ((uint64_t)entry->cfgi.sid << CFGI_0_STE_SID_S);
509 cmd[1] |= ((uint64_t)entry->cfgi.leaf << CFGI_1_LEAF_S);
516 if (entry->sync.msiaddr) {
518 cmd[1] |= (entry->sync.msiaddr & SYNC_1_MSIADDRESS_M);
523 cmd[0] |= ((uint64_t)entry->prefetch.sid << PREFETCH_0_SID_S);
535 cmdq = &sc->cmdq;
543 cmdq->lc.cons = bus_read_4(sc->res[0], cmdq->cons_off);
547 entry_addr = (void *)((uint64_t)cmdq->vaddr +
548 Q_IDX(cmdq, cmdq->lc.prod) * CMDQ_ENTRY_DWORDS * 8);
552 cmdq->lc.prod = smmu_q_inc_prod(cmdq);
553 bus_write_4(sc->res[0], cmdq->prod_off, cmdq->lc.prod);
563 q->lc.val = bus_read_8(sc->res[0], q->prod_off);
579 q = &sc->cmdq;
580 prod = q->lc.prod;
584 cmd.sync.msiaddr = q->paddr + Q_IDX(q, prod) * CMDQ_ENTRY_DWORDS * 8;
588 base = (void *)((uint64_t)q->vaddr +
603 } while (timeout--);
606 device_printf(sc->dev, "Failed to sync\n");
746 if (sc->features & SMMU_FEATURE_STALL &&
747 ((sc->features & SMMU_FEATURE_STALL_FORCE) == 0))
751 val |= (cd->paddr & STE0_S1CONTEXTPTR_M);
777 strtab = &sc->strtab;
779 if (sc->features & SMMU_FEATURE_2_LVL_STREAM_TABLE) {
780 l1_desc = &strtab->l1[sid >> STRTAB_SPLIT];
781 addr = l1_desc->va;
782 addr += (sid & ((1 << STRTAB_SPLIT) - 1)) * STRTAB_STE_DWORDS;
784 addr = (void *)((uint64_t)strtab->vaddr +
835 p = &domain->p;
836 cd = domain->cd = malloc(sizeof(struct smmu_cd),
839 cd->vaddr = contigmalloc(size, M_SMMU,
842 (1ul << 40) - 1, /* high */
845 if (cd->vaddr == NULL) {
846 device_printf(sc->dev, "Failed to allocate CD\n");
850 cd->paddr = vtophys(cd->vaddr);
852 ptr = cd->vaddr;
859 val |= (uint64_t)domain->asid << CD0_ASID_S;
862 val |= ((64 - sc->ias) << CD0_T0SZ_S);
865 paddr = p->sp_l0_paddr & CD1_TTB0_M;
866 KASSERT(paddr == p->sp_l0_paddr, ("bad allocation 1"));
889 strtab = &sc->strtab;
890 strtab->num_l1_entries = (1 << sc->sid_bits);
892 size = strtab->num_l1_entries * (STRTAB_STE_DWORDS << 3);
895 device_printf(sc->dev,
897 __func__, size, strtab->num_l1_entries);
899 strtab->vaddr = contigmalloc(size, M_SMMU,
902 (1ul << 48) - 1, /* high */
905 if (strtab->vaddr == NULL) {
906 device_printf(sc->dev, "failed to allocate strtab\n");
911 reg |= sc->sid_bits << STRTAB_BASE_CFG_LOG2SIZE_S;
912 strtab->base_cfg = (uint32_t)reg;
914 base = vtophys(strtab->vaddr);
919 strtab->base = reg;
935 strtab = &sc->strtab;
937 size = STRTAB_L1_SZ_SHIFT - (ilog2(STRTAB_L1_DESC_DWORDS) + 3);
938 size = min(size, sc->sid_bits - STRTAB_SPLIT);
939 strtab->num_l1_entries = (1 << size);
942 l1size = strtab->num_l1_entries * (STRTAB_L1_DESC_DWORDS << 3);
945 device_printf(sc->dev,
947 __func__, size, strtab->num_l1_entries, l1size);
949 strtab->vaddr = contigmalloc(l1size, M_SMMU,
952 (1ul << 48) - 1, /* high */
955 if (strtab->vaddr == NULL) {
956 device_printf(sc->dev, "Failed to allocate 2lvl strtab.\n");
960 sz = strtab->num_l1_entries * sizeof(struct l1_desc);
962 strtab->l1 = malloc(sz, M_SMMU, M_WAITOK | M_ZERO);
967 strtab->base_cfg = (uint32_t)reg;
969 base = vtophys(strtab->vaddr);
974 strtab->base = reg_base;
984 if (sc->features & SMMU_FEATURE_2_LVL_STREAM_TABLE)
1002 strtab = &sc->strtab;
1003 l1_desc = &strtab->l1[sid >> STRTAB_SPLIT];
1004 if (l1_desc->va) {
1011 l1_desc->span = STRTAB_SPLIT + 1;
1012 l1_desc->va = contigmalloc(size, M_SMMU,
1015 (1ul << 48) - 1, /* high */
1018 if (l1_desc->va == NULL) {
1019 device_printf(sc->dev, "failed to allocate l2 entry\n");
1023 l1_desc->pa = vtophys(l1_desc->va);
1026 addr = (void *)((uint64_t)strtab->vaddr +
1030 val = l1_desc->pa & STRTAB_L1_DESC_L2PTR_M;
1031 KASSERT(val == l1_desc->pa, ("bad allocation 4"));
1032 val |= l1_desc->span;
1046 strtab = &sc->strtab;
1049 addr = (void *)((uint64_t)strtab->vaddr +
1053 l1_desc = &strtab->l1[sid >> STRTAB_SPLIT];
1054 free(l1_desc->va, M_SMMU);
1064 reg = bus_read_4(sc->res[0], SMMU_CR0);
1068 device_printf(sc->dev, "Could not disable SMMU.\n");
1084 } while (!smmu_q_empty(&sc->evtq));
1096 device_printf(sc->dev, "%s\n", __func__);
1108 device_printf(sc->dev, "SMMU Global Error\n");
1120 bus_write_8(sc->res[0], SMMU_GERROR_IRQ_CFG0, 0);
1121 bus_write_4(sc->res[0], SMMU_GERROR_IRQ_CFG1, 0);
1122 bus_write_4(sc->res[0], SMMU_GERROR_IRQ_CFG2, 0);
1124 bus_write_8(sc->res[0], SMMU_EVENTQ_IRQ_CFG0, 0);
1125 bus_write_4(sc->res[0], SMMU_EVENTQ_IRQ_CFG1, 0);
1126 bus_write_4(sc->res[0], SMMU_EVENTQ_IRQ_CFG2, 0);
1128 if (sc->features & CR0_PRIQEN) {
1129 bus_write_8(sc->res[0], SMMU_PRIQ_IRQ_CFG0, 0);
1130 bus_write_4(sc->res[0], SMMU_PRIQ_IRQ_CFG1, 0);
1131 bus_write_4(sc->res[0], SMMU_PRIQ_IRQ_CFG2, 0);
1137 device_printf(sc->dev, "Could not disable interrupts.\n");
1143 if (sc->features & SMMU_FEATURE_PRI)
1148 device_printf(sc->dev, "Could not enable interrupts.\n");
1165 if (data->type == INTR_MAP_DATA_ACPI) {
1167 ad->trig = INTR_TRIGGER_EDGE;
1168 ad->pol = INTR_POLARITY_HIGH;
1179 dev = sc->dev;
1186 smmu_configure_intr(sc, sc->res[1]);
1188 smmu_configure_intr(sc, sc->res[3]);
1189 smmu_configure_intr(sc, sc->res[4]);
1192 error = bus_setup_intr(dev, sc->res[1], INTR_TYPE_MISC,
1193 smmu_event_intr, NULL, sc, &sc->intr_cookie[0]);
1199 error = bus_setup_intr(dev, sc->res[4], INTR_TYPE_MISC,
1200 smmu_gerr_intr, NULL, sc, &sc->intr_cookie[2]);
1217 reg = bus_read_4(sc->res[0], SMMU_CR0);
1220 device_printf(sc->dev,
1225 device_printf(sc->dev,
1229 device_printf(sc->dev, "Could not enable interrupts.\n");
1239 bus_write_4(sc->res[0], SMMU_CR1, reg);
1242 bus_write_4(sc->res[0], SMMU_CR2, reg);
1245 strtab = &sc->strtab;
1246 bus_write_8(sc->res[0], SMMU_STRTAB_BASE, strtab->base);
1247 bus_write_4(sc->res[0], SMMU_STRTAB_BASE_CFG, strtab->base_cfg);
1250 bus_write_8(sc->res[0], SMMU_CMDQ_BASE, sc->cmdq.base);
1251 bus_write_4(sc->res[0], SMMU_CMDQ_PROD, sc->cmdq.lc.prod);
1252 bus_write_4(sc->res[0], SMMU_CMDQ_CONS, sc->cmdq.lc.cons);
1257 device_printf(sc->dev, "Could not enable command queue\n");
1264 if (sc->features & SMMU_FEATURE_HYP) {
1273 bus_write_8(sc->res[0], SMMU_EVENTQ_BASE, sc->evtq.base);
1274 bus_write_4(sc->res[0], SMMU_EVENTQ_PROD, sc->evtq.lc.prod);
1275 bus_write_4(sc->res[0], SMMU_EVENTQ_CONS, sc->evtq.lc.cons);
1280 device_printf(sc->dev, "Could not enable event queue\n");
1284 if (sc->features & SMMU_FEATURE_PRI) {
1286 bus_write_8(sc->res[0], SMMU_PRIQ_BASE, sc->priq.base);
1287 bus_write_4(sc->res[0], SMMU_PRIQ_PROD, sc->priq.lc.prod);
1288 bus_write_4(sc->res[0], SMMU_PRIQ_CONS, sc->priq.lc.cons);
1293 device_printf(sc->dev, "Could not enable PRI queue\n");
1298 if (sc->features & SMMU_FEATURE_ATS) {
1302 device_printf(sc->dev, "Could not enable ATS check.\n");
1310 device_printf(sc->dev, "Could not enable SMMU.\n");
1323 sc->features = 0;
1325 reg = bus_read_4(sc->res[0], SMMU_IDR0);
1329 device_printf(sc->dev,
1330 "2-level stream table supported.\n");
1331 sc->features |= SMMU_FEATURE_2_LVL_STREAM_TABLE;
1336 device_printf(sc->dev,
1337 "2-level CD table supported.\n");
1338 sc->features |= SMMU_FEATURE_2_LVL_CD;
1344 device_printf(sc->dev, "Mixed endianness supported.\n");
1345 sc->features |= SMMU_FEATURE_TT_LE;
1346 sc->features |= SMMU_FEATURE_TT_BE;
1350 device_printf(sc->dev,
1351 "Little endian supported only.\n");
1352 sc->features |= SMMU_FEATURE_TT_LE;
1356 device_printf(sc->dev, "Big endian supported only.\n");
1357 sc->features |= SMMU_FEATURE_TT_BE;
1360 device_printf(sc->dev, "Unsupported endianness.\n");
1365 sc->features |= SMMU_FEATURE_SEV;
1369 device_printf(sc->dev, "MSI feature present.\n");
1370 sc->features |= SMMU_FEATURE_MSI;
1375 device_printf(sc->dev, "HYP feature present.\n");
1376 sc->features |= SMMU_FEATURE_HYP;
1380 sc->features |= SMMU_FEATURE_ATS;
1383 sc->features |= SMMU_FEATURE_PRI;
1388 sc->features |= SMMU_FEATURE_STALL_FORCE;
1391 sc->features |= SMMU_FEATURE_STALL;
1395 /* Grab translation stages supported. */
1398 device_printf(sc->dev,
1399 "Stage 1 translation supported.\n");
1400 sc->features |= SMMU_FEATURE_S1P;
1404 device_printf(sc->dev,
1405 "Stage 2 translation supported.\n");
1406 sc->features |= SMMU_FEATURE_S2P;
1412 sc->ias = 40;
1415 device_printf(sc->dev, "No AArch64 table format support.\n");
1420 sc->asid_bits = 16;
1422 sc->asid_bits = 8;
1425 device_printf(sc->dev, "ASID bits %d\n", sc->asid_bits);
1428 sc->vmid_bits = 16;
1430 sc->vmid_bits = 8;
1432 reg = bus_read_4(sc->res[0], SMMU_IDR1);
1435 device_printf(sc->dev,
1436 "Embedded implementations not supported by this driver.\n");
1441 sc->cmdq.size_log2 = val;
1443 device_printf(sc->dev, "CMD queue bits %d\n", val);
1446 sc->evtq.size_log2 = val;
1448 device_printf(sc->dev, "EVENT queue bits %d\n", val);
1450 if (sc->features & SMMU_FEATURE_PRI) {
1452 sc->priq.size_log2 = val;
1454 device_printf(sc->dev, "PRI queue bits %d\n", val);
1457 sc->ssid_bits = (reg & IDR1_SSIDSIZE_M) >> IDR1_SSIDSIZE_S;
1458 sc->sid_bits = (reg & IDR1_SIDSIZE_M) >> IDR1_SIDSIZE_S;
1460 if (sc->sid_bits <= STRTAB_SPLIT)
1461 sc->features &= ~SMMU_FEATURE_2_LVL_STREAM_TABLE;
1464 device_printf(sc->dev, "SSID bits %d\n", sc->ssid_bits);
1465 device_printf(sc->dev, "SID bits %d\n", sc->sid_bits);
1469 reg = bus_read_4(sc->res[0], SMMU_IDR3);
1471 sc->features |= SMMU_FEATURE_RANGE_INV;
1474 reg = bus_read_4(sc->res[0], SMMU_IDR5);
1478 sc->oas = 32;
1481 sc->oas = 36;
1484 sc->oas = 40;
1487 sc->oas = 42;
1490 sc->oas = 44;
1493 sc->oas = 48;
1496 sc->oas = 52;
1500 sc->pgsizes = 0;
1502 sc->pgsizes |= 64 * 1024;
1504 sc->pgsizes |= 16 * 1024;
1506 sc->pgsizes |= 4 * 1024;
1509 sc->features |= SMMU_FEATURE_VAX;
1518 sc->asid_set_size = (1 << sc->asid_bits);
1519 sc->asid_set = bit_alloc(sc->asid_set_size, M_SMMU, M_WAITOK);
1520 mtx_init(&sc->asid_set_mutex, "asid set", NULL, MTX_SPIN);
1527 mtx_lock_spin(&sc->asid_set_mutex);
1528 bit_ffc(sc->asid_set, sc->asid_set_size, new_asid);
1529 if (*new_asid == -1) {
1530 mtx_unlock_spin(&sc->asid_set_mutex);
1533 bit_set(sc->asid_set, *new_asid);
1534 mtx_unlock_spin(&sc->asid_set_mutex);
1543 mtx_lock_spin(&sc->asid_set_mutex);
1544 bit_clear(sc->asid_set, asid);
1545 mtx_unlock_spin(&sc->asid_set_mutex);
1558 sc->dev = dev;
1560 mtx_init(&sc->sc_mtx, device_get_nameunit(sc->dev), "smmu", MTX_DEF);
1564 bus_release_resources(dev, smmu_spec, sc->res);
1571 "but not supported by hardware.\n");
1605 bus_release_resources(dev, smmu_spec, sc->res);
1617 device_printf(sc->dev, "%s\n", __func__);
1637 dprintf("%s: %lx, %ld, domain %d\n", __func__, va, size, domain->asid);
1640 if (smmu_pmap_remove(&domain->p, va) == 0) {
1642 smmu_tlbi_va(sc, va, domain->asid);
1670 dprintf("%s: %lx -> %lx, %ld, domain %d\n", __func__, va, pa, size,
1671 domain->asid);
1673 for (i = 0; size > 0; size -= PAGE_SIZE) {
1675 error = smmu_pmap_enter(&domain->p, va, pa, prot, 0);
1678 smmu_tlbi_va(sc, va, domain->asid);
1706 device_printf(sc->dev,
1711 domain->asid = (uint16_t)new_asid;
1713 smmu_pmap_pinit(&domain->p);
1718 device_printf(sc->dev, "Could not initialize CD\n");
1722 smmu_tlbi_asid(sc, domain->asid);
1724 LIST_INIT(&domain->ctx_list);
1727 LIST_INSERT_HEAD(&unit->domain_list, domain, next);
1730 iodom = &domain->iodom;
1733 * Use 48-bit address space regardless of VAX bit
1734 * as we need 64k IOMMU_PAGE_SIZE for 52-bit space.
1736 iodom->end = MAXADDR_48BIT;
1754 cd = domain->cd;
1756 smmu_pmap_remove_pages(&domain->p);
1757 smmu_pmap_release(&domain->p);
1759 smmu_tlbi_asid(sc, domain->asid);
1760 smmu_asid_free(sc, domain->asid);
1762 free(cd->vaddr, M_SMMU);
1778 smmu_init_ste(sc, domain->cd, (ctx->sid | i), ctx->bypass);
1810 ctx->dev = child;
1811 ctx->domain = domain;
1813 ctx->bypass = true;
1816 LIST_INSERT_HEAD(&domain->ctx_list, ctx, next);
1819 return (&ctx->ioctx);
1837 domain = ctx->domain;
1841 if (device_get_devclass(device_get_parent(ctx->dev)) == pci_class) {
1842 err = smmu_pci_get_sid(ctx->dev, NULL, &sid);
1846 ioctx->rid = pci_get_rid(dev);
1847 ctx->sid = sid;
1848 ctx->vendor = pci_get_vendor(ctx->dev);
1849 ctx->device = pci_get_device(ctx->dev);
1852 if (sc->features & SMMU_FEATURE_2_LVL_STREAM_TABLE) {
1853 err = smmu_init_l1_entry(sc, ctx->sid);
1865 smmu_init_ste(sc, domain->cd, ctx->sid, ctx->bypass);
1867 if (device_get_devclass(device_get_parent(ctx->dev)) == pci_class)
1868 if (iommu_is_buswide_ctx(iodom->iommu, pci_get_bus(ctx->dev)))
1880 IOMMU_ASSERT_LOCKED(ioctx->domain->iommu);
1885 smmu_deinit_ste(sc, ctx->sid);
1902 unit = &sc->unit;
1904 LIST_FOREACH(domain, &unit->domain_list, next) {
1905 LIST_FOREACH(ctx, &domain->ctx_list, next) {
1906 if (ctx->sid == sid)
1925 unit = &sc->unit;
1926 iommu = &unit->iommu;
1930 LIST_FOREACH(domain, &unit->domain_list, next) {
1931 IOMMU_DOMAIN_LOCK(&domain->iodom);
1932 LIST_FOREACH(ctx, &domain->ctx_list, next) {
1933 if (ctx->dev == child) {
1934 IOMMU_DOMAIN_UNLOCK(&domain->iodom);
1935 return (&ctx->ioctx);
1938 IOMMU_DOMAIN_UNLOCK(&domain->iodom);
1958 if (xref != sc->xref)
1974 return (-1);
1976 ctx->sid = cells[0];