Lines Matching +full:ctrl +full:- +full:len
1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
63 #define MOD_DEC(a, s, m) (((a) - (s)) % ((m) * (s)))
127 return (pci_cfgregread(softc->pci_seg, PCI_RID2BUS(softc->pci_rid),
128 PCI_RID2SLOT(softc->pci_rid), PCI_RID2FUNC(softc->pci_rid),
136 * If ATS is absent or disabled, return (-1), otherwise ATS
144 int qlen = -1;
150 return (-1);
158 printf("AMD-Vi: PCI device %d.%d.%d ATS %s qlen=%d\n",
162 qlen = (cap & PCIM_ATS_EN) ? qlen : -1;
183 cfg = softc->dev_cfg;
186 for (i = 0; i < softc->dev_cfg_cnt; i++) {
187 if ((cfg->start_id <= devid) && (cfg->end_id >= devid)) {
188 ivhd_ats = cfg->enable_ats;
196 device_printf(softc->dev,
210 softc->iotlb = false;
214 supported = (softc->ivhd_flag & IVHD_FLAG_IOTLB) ? true : false;
216 if (softc->pci_cap & AMDVI_PCI_CAP_IOTLB) {
218 device_printf(softc->dev, "IOTLB disabled by BIOS.\n");
221 device_printf(softc->dev, "IOTLB disabled by user.\n");
227 softc->iotlb = supported;
235 struct amdvi_ctrl *ctrl = softc->ctrl;
237 ctrl->cmd.len = 8; /* Use 256 command buffer entries. */
238 softc->cmd_max = 1 << ctrl->cmd.len;
240 softc->cmd = malloc(sizeof(struct amdvi_cmd) *
241 softc->cmd_max, M_AMDVI, M_WAITOK | M_ZERO);
243 if ((uintptr_t)softc->cmd & PAGE_MASK)
246 ctrl->cmd.base = vtophys(softc->cmd) / PAGE_SIZE;
251 ctrl->cmd_tail = 0;
252 ctrl->cmd_head = 0;
266 struct amdvi_ctrl *ctrl;
270 KASSERT(softc->cmd != NULL, ("cmd is NULL"));
272 ctrl = softc->ctrl;
273 KASSERT(ctrl != NULL, ("ctrl is NULL"));
275 tail = (struct amdvi_cmd *)((uint8_t *)softc->cmd +
276 ctrl->cmd_tail);
287 struct amdvi_ctrl *ctrl;
291 KASSERT(softc->cmd != NULL, ("cmd is NULL"));
293 ctrl = softc->ctrl;
294 KASSERT(ctrl != NULL, ("ctrl is NULL"));
296 ctrl->cmd_tail = MOD_INC(ctrl->cmd_tail, size, softc->cmd_max);
297 softc->total_cmd++;
300 device_printf(softc->dev, "cmd_tail: %s Tail:0x%x, Head:0x%x.\n",
301 ctrl->cmd_tail,
302 ctrl->cmd_head);
321 pa = vtophys(&softc->cmp_data);
322 cmd->opcode = AMDVI_CMP_WAIT_OPCODE;
323 cmd->word0 = (pa & 0xFFFFFFF8) | AMDVI_CMP_WAIT_STORE;
324 cmd->word1 = (pa >> 32) & 0xFFFFF;
325 cmd->addr = data;
338 cmd->opcode = AMDVI_INVD_DTE_OPCODE;
339 cmd->word0 = devid;
342 device_printf(softc->dev, "Invalidated DTE:0x%x\n", devid);
357 cmd->opcode = AMDVI_INVD_PAGE_OPCODE;
358 cmd->word1 = domain_id;
362 cmd->addr = addr;
363 cmd->addr |= pde ? AMDVI_INVD_PAGE_PDE : 0;
364 cmd->addr |= page ? AMDVI_INVD_PAGE_S : 0;
377 if (!softc->iotlb)
389 device_printf(softc->dev, "Invalidate IOTLB devID 0x%x"
392 cmd->opcode = AMDVI_INVD_IOTLB_OPCODE;
393 cmd->word0 = devid;
394 cmd->word1 = qlen;
395 cmd->addr = AMDVI_INVD_IOTLB_ALL_ADDR |
410 cmd->opcode = AMDVI_INVD_INTR_OPCODE;
411 cmd->word0 = devid;
414 device_printf(softc->dev, "Invalidate INTR map of devID 0x%x\n", devid);
436 device_printf(softc->dev, "Invalidate domain:0x%x\n", domain_id);
445 struct amdvi_ctrl *ctrl = softc->ctrl;
452 read = &softc->cmp_data;
459 status = (VERIFY == softc->cmp_data) ? true : false;
463 device_printf(softc->dev, "CMD completion DONE Tail:0x%x, "
464 "Head:0x%x, loop:%d.\n", ctrl->cmd_tail,
465 ctrl->cmd_head, loop);
473 struct amdvi_ctrl *ctrl;
478 ctrl = softc->ctrl;
479 KASSERT(ctrl != NULL, ("ctrl is NULL"));
481 if ((ctrl->control & AMDVI_CTRL_EN) == 0)
489 device_printf(softc->dev, "Error: completion failed"
491 ctrl->cmd_tail, ctrl->cmd_head);
499 struct amdvi_ctrl *ctrl;
503 ctrl = softc->ctrl;
504 device_printf(softc->dev, "Dump last %d command(s):\n", count);
509 off = MOD_DEC(ctrl->cmd_head, sizeof(struct amdvi_cmd),
510 softc->cmd_max);
511 for (i = 0; off != ctrl->cmd_tail && i < count; i++) {
512 cmd = (struct amdvi_cmd *)((uint8_t *)softc->cmd + off);
514 " 0x%x 0x%lx\n", i, off, cmd->opcode,
515 cmd->word0, cmd->word1, cmd->addr);
516 off = MOD_INC(off, sizeof(struct amdvi_cmd), softc->cmd_max);
523 struct amdvi_ctrl *ctrl;
525 ctrl = softc->ctrl;
526 ctrl->event.len = 8;
527 softc->event_max = 1 << ctrl->event.len;
528 softc->event = malloc(sizeof(struct amdvi_event) *
529 softc->event_max, M_AMDVI, M_WAITOK | M_ZERO);
530 if ((uintptr_t)softc->event & PAGE_MASK) {
531 device_printf(softc->dev, "Event buffer not aligned on page.");
534 ctrl->event.base = vtophys(softc->event) / PAGE_SIZE;
537 ctrl->evt_head = 0;
538 ctrl->evt_tail = 0;
634 switch (evt->opcode) {
636 amdvi_decode_inv_dte_evt(evt->devid, evt->pasid_domid,
637 evt->addr, evt->flag);
641 amdvi_decode_pf_evt(evt->devid, evt->pasid_domid,
642 evt->addr, evt->flag);
646 amdvi_decode_dte_hwerr_evt(evt->devid, evt->pasid_domid,
647 evt->addr, evt->flag);
651 amdvi_decode_page_hwerr_evt(evt->devid, evt->pasid_domid,
652 evt->addr, evt->flag);
658 printf("\t[%s EVT]\n", (evt->opcode == AMDVI_EVENT_ILLEGAL_CMD) ?
660 cmd = (struct amdvi_cmd *)PHYS_TO_DMAP(evt->addr);
662 cmd->opcode, cmd->word0, cmd->word1, cmd->addr);
667 evt->devid, evt->addr);
672 evt->devid, evt->addr, evt->flag >> 9,
673 (evt->flag >> 8) & 1);
678 printf("AMD-Vi: v2 events.\n");
682 printf("Unsupported AMD-Vi event:%d\n", evt->opcode);
689 struct amdvi_ctrl *ctrl;
693 ctrl = softc->ctrl;
695 for (i = 0; i < softc->event_max; i++) {
696 event = &softc->event[ctrl->evt_head / size];
697 if (!event->opcode)
699 device_printf(softc->dev, "\t[Event%d: Head:0x%x Tail:0x%x]\n",
700 i, ctrl->evt_head, ctrl->evt_tail);
702 ctrl->evt_head = MOD_INC(ctrl->evt_head, size,
703 softc->event_max);
710 struct amdvi_ctrl *ctrl;
712 ctrl = softc->ctrl;
713 ctrl->dte.base = vtophys(amdvi_dte) / PAGE_SIZE;
714 ctrl->dte.size = 0x1FF; /* 2MB device table. */
730 off = softc->cap_off;
742 softc->pci_cap = cap >> 24;
743 device_printf(softc->dev, "PCI cap 0x%x@0x%x feature:%b\n",
744 cap, off, softc->pci_cap,
754 struct amdvi_ctrl *ctrl;
757 ctrl = softc->ctrl;
758 device_printf(softc->dev, "EVT INTR %ld Status:0x%x"
759 " EVT Head:0x%x Tail:0x%x]\n", softc->event_intr_cnt++,
760 ctrl->status, ctrl->evt_head, ctrl->evt_tail);
762 softc->total_cmd, ctrl->cmd_tail, ctrl->cmd_head);
765 ctrl->status &= AMDVI_STATUS_EV_OF | AMDVI_STATUS_EV_INTR;
776 mmio_dev = softc->pci_dev;
784 struct amdvi_ctrl *ctrl;
788 dev = softc->dev;
789 mmio_dev = softc->pci_dev;
792 ctrl = softc->ctrl;
793 ctrl->status &= AMDVI_STATUS_EV_OF | AMDVI_STATUS_EV_INTR;
808 cfg = softc->dev_cfg;
809 for (i = 0; i < softc->dev_cfg_cnt; i++) {
810 device_printf(softc->dev, "device [0x%x - 0x%x] "
811 "config:%b%s\n", cfg->start_id, cfg->end_id,
812 cfg->data,
815 cfg->enable_ats ? "ATS enabled" : "");
831 result = softc->ctrl->cmd_head;
836 result = softc->ctrl->cmd_tail;
841 result = softc->ctrl->evt_head;
846 result = softc->ctrl->evt_tail;
852 device_printf(softc->dev, "Unknown sysctl:%d\n", type);
865 dev = softc->dev;
870 &softc->event_intr_cnt, "Event interrupt count");
872 &softc->total_cmd, "Command submitted count");
874 &softc->pci_rid, 0, "IOMMU RID");
895 dev = softc->dev;
929 dev = softc->dev;
936 if (softc->cmd)
937 free(softc->cmd, M_AMDVI);
939 if (softc->event)
940 free(softc->event, M_AMDVI);
953 printf("bhyve: Found %d AMD-Vi/IOMMU device(s), "
954 "use hw.vmm.amdvi.enable=1 to enable pass-through.\n",
996 if (create && ((softc->pci_cap & AMDVI_PCI_CAP_NPCACHE) == 0))
1010 dom->id = amdvi_domainId();
1011 //dom->maxaddr = maxaddr;
1013 printf("Created domain #%d\n", dom->id);
1018 if (dom->id || amdvi_host_ptp)
1019 dom->ptp = malloc(PAGE_SIZE, M_AMDVI, M_WAITOK | M_ZERO);
1021 dom->ptp_level = amdvi_ptp_level;
1023 amdvi_do_inv_domain(dom->id, true);
1040 /* XXX: Add super-page or PTE mapping > 4KB. */
1042 /* Super-page mapping. */
1048 & AMDVI_PT_MASK), level - 1);
1062 printf("Destroying domain %d\n", domain->id);
1064 if (domain->ptp)
1065 amdvi_free_ptp(domain->ptp, domain->ptp_level);
1067 amdvi_do_inv_domain(domain->id, false);
1079 const int PT_INDEX_MASK = (1 << PT_SHIFT) - 1; /* Based on PT_SHIFT */
1084 if (hpa & (pg_size - 1)) {
1088 if (gpa & (pg_size - 1)) {
1100 ((level - 1) << AMDVI_PD_LEVEL_SHIFT);
1110 shift -= PT_SHIFT;
1111 level--;
1134 uint64_t mapped, *ptp, len;
1138 level = domain->ptp_level;
1141 ptp = domain->ptp;
1145 len = amdvi_set_pt(ptp, level, gpa + mapped, hpa + mapped,
1147 if (!len) {
1152 mapped += len;
1160 uint64_t len)
1166 if (domain->id && !domain->ptp) {
1168 return (-1);
1173 * table set-up.
1175 if (domain->ptp)
1176 return (amdvi_update_mapping(domain, gpa, hpa, len, true));
1178 return (len);
1182 amdvi_remove_mapping(void *arg, vm_paddr_t gpa, uint64_t len)
1189 * table set-up.
1191 if (domain->ptp)
1192 return (amdvi_update_mapping(domain, gpa, 0, len, false));
1194 (len);
1205 for (j = 0; j < softc->dev_cfg_cnt; j++)
1206 if ((devid >= softc->dev_cfg[j].start_id) &&
1207 (devid <= softc->dev_cfg[j].end_id))
1215 * Set-up device table entry.
1232 if (amdvi_dev_support_iotlb(softc, devid) && softc->iotlb)
1233 temp->iotlb_enable = 1;
1237 temp->sup_second_io_fault = 1;
1238 temp->sup_all_io_fault = amdvi_disable_io_fault;
1240 temp->dt_valid = 1;
1241 temp->domain_id = domain->id;
1244 if (domain->ptp) {
1245 temp->pt_base = vtophys(domain->ptp) >> 12;
1246 temp->pt_level = amdvi_ptp_level;
1252 temp->pt_valid = 1;
1253 temp->read_allow = 1;
1254 temp->write_allow = 1;
1281 RID2PCI_STR(devid), domain->id);
1299 devid, domain->id);
1311 struct amdvi_ctrl *ctrl;
1319 ctrl = softc->ctrl;
1320 KASSERT(ctrl, ("ctrl is NULL\n"));
1328 if (softc->ivhd_flag & IVHD_FLAG_COH)
1330 if (softc->ivhd_flag & IVHD_FLAG_HTT)
1332 if (softc->ivhd_flag & IVHD_FLAG_RPPW)
1334 if (softc->ivhd_flag & IVHD_FLAG_PPW)
1336 if (softc->ivhd_flag & IVHD_FLAG_ISOC)
1339 ctrl->control = val;
1346 struct amdvi_ctrl *ctrl;
1353 ctrl = softc->ctrl;
1354 KASSERT(ctrl, ("ctrl is NULL\n"));
1356 ctrl->control = 0;
1367 amdvi_do_inv_domain(domain->id, false);