Lines Matching refs:ctrlr
111 struct nvme_ctrlr *ctrlr;
113 TAILQ_FOREACH(ctrlr, &g_nvme_ctrlrs, tailq) {
114 ctrlr_addr = spdk_pci_device_get_addr(ctrlr->pci_device);
116 return ctrlr;
124 get_pcie_reg_addr(struct nvme_ctrlr *ctrlr, uint32_t offset)
126 return (volatile void *)((uintptr_t)ctrlr->regs + offset);
130 get_pcie_reg_4(struct nvme_ctrlr *ctrlr, uint32_t offset, uint32_t *value)
133 *value = spdk_mmio_read_4(get_pcie_reg_addr(ctrlr, offset));
137 get_pcie_reg_8(struct nvme_ctrlr *ctrlr, uint32_t offset, uint64_t *value)
140 *value = spdk_mmio_read_8(get_pcie_reg_addr(ctrlr, offset));
144 set_pcie_reg_4(struct nvme_ctrlr *ctrlr, uint32_t offset, uint32_t value)
147 spdk_mmio_write_4(get_pcie_reg_addr(ctrlr, offset), value);
151 set_pcie_reg_8(struct nvme_ctrlr *ctrlr, uint32_t offset, uint64_t value)
154 spdk_mmio_write_8(get_pcie_reg_addr(ctrlr, offset), value);
158 nvme_ctrlr_get_cap(struct nvme_ctrlr *ctrlr, union spdk_nvme_cap_register *cap)
160 get_pcie_reg_8(ctrlr, offsetof(struct spdk_nvme_registers, cap), &cap->raw);
164 nvme_ctrlr_get_cc(struct nvme_ctrlr *ctrlr, union spdk_nvme_cc_register *cc)
166 get_pcie_reg_4(ctrlr, offsetof(struct spdk_nvme_registers, cc), &cc->raw);
170 nvme_ctrlr_get_csts(struct nvme_ctrlr *ctrlr, union spdk_nvme_csts_register *csts)
172 get_pcie_reg_4(ctrlr, offsetof(struct spdk_nvme_registers, csts), &csts->raw);
176 nvme_ctrlr_set_cc(struct nvme_ctrlr *ctrlr, const union spdk_nvme_cc_register *cc)
178 set_pcie_reg_4(ctrlr, offsetof(struct spdk_nvme_registers, cc.raw), cc->raw);
182 nvme_ctrlr_set_asq(struct nvme_ctrlr *ctrlr, uint64_t value)
184 set_pcie_reg_8(ctrlr, offsetof(struct spdk_nvme_registers, asq), value);
188 nvme_ctrlr_set_acq(struct nvme_ctrlr *ctrlr, uint64_t value)
190 set_pcie_reg_8(ctrlr, offsetof(struct spdk_nvme_registers, acq), value);
194 nvme_ctrlr_set_aqa(struct nvme_ctrlr *ctrlr, const union spdk_nvme_aqa_register *aqa)
196 set_pcie_reg_4(ctrlr, offsetof(struct spdk_nvme_registers, aqa.raw), aqa->raw);
209 init_qpair(struct nvme_ctrlr *ctrlr, uint16_t id, uint16_t num_entries)
266 doorbell_base = (volatile uint32_t *)&ctrlr->regs->doorbell[0];
267 qpair->sq_tdbl = doorbell_base + (2 * id + 0) * ctrlr->doorbell_stride_u32;
268 qpair->cq_hdbl = doorbell_base + (2 * id + 1) * ctrlr->doorbell_stride_u32;
276 struct nvme_ctrlr *ctrlr;
286 ctrlr = calloc(1, sizeof(*ctrlr));
287 if (!ctrlr) {
294 free(ctrlr);
301 free(ctrlr);
305 ctrlr->pci_device = pci_dev;
306 ctrlr->regs = (volatile struct spdk_nvme_registers *)reg_addr;
313 nvme_ctrlr_get_cap(ctrlr, &cap);
314 ctrlr->page_size = 1 << (12 + cap.bits.mpsmin);
315 ctrlr->doorbell_stride_u32 = 1 << cap.bits.dstrd;
317 ctrlr->cdata = spdk_zmalloc(sizeof(*ctrlr->cdata), ctrlr->page_size, NULL,
319 if (!ctrlr->cdata) {
322 free(ctrlr);
327 ctrlr->admin_qpair = init_qpair(ctrlr, 0, SPDK_NVME_ADMIN_QUEUE_MIN_ENTRIES);
328 if (!ctrlr->admin_qpair) {
331 spdk_free(ctrlr->cdata);
332 free(ctrlr);
336 TAILQ_INSERT_TAIL(ctrlrs, ctrlr, tailq);
374 struct nvme_ctrlr *ctrlr = ctx;
378 ctrlr->state = NVME_CTRLR_STATE_ERROR;
382 ctrlr->state = NVME_CTRLR_STATE_READY;
386 identify_ctrlr(struct nvme_ctrlr *ctrlr)
393 assert(((uintptr_t)ctrlr->cdata & (ctrlr->page_size - 1)) == 0);
394 prp1 = spdk_vtophys(ctrlr->cdata, NULL);
399 request = allocate_request(ctrlr->admin_qpair);
405 request->cb_arg = ctrlr;
416 submit_request(ctrlr->admin_qpair, request);
459 process_ctrlr_init(struct nvme_ctrlr *ctrlr)
466 if (ctrlr->state == NVME_CTRLR_STATE_READY) {
470 nvme_ctrlr_get_cc(ctrlr, &cc);
471 nvme_ctrlr_get_csts(ctrlr, &csts);
473 switch (ctrlr->state) {
477 ctrlr->state = NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1;
482 nvme_ctrlr_set_cc(ctrlr, &cc);
484 ctrlr->state = NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0;
489 nvme_ctrlr_set_cc(ctrlr, &cc);
490 ctrlr->state = NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0;
495 ctrlr->state = NVME_CTRLR_STATE_ENABLE;
499 nvme_ctrlr_set_asq(ctrlr, ctrlr->admin_qpair->sq_paddr);
500 nvme_ctrlr_set_acq(ctrlr, ctrlr->admin_qpair->cq_paddr);
503 aqa.bits.asqs = ctrlr->admin_qpair->num_entries - 1;
504 aqa.bits.acqs = ctrlr->admin_qpair->num_entries - 1;
505 nvme_ctrlr_set_aqa(ctrlr, &aqa);
510 nvme_ctrlr_set_cc(ctrlr, &cc);
511 ctrlr->state = NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1;
515 ctrlr->state = NVME_CTRLR_STATE_IDENTIFY;
519 ctrlr->state = NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY;
520 rc = identify_ctrlr(ctrlr);
523 process_completions(ctrlr->admin_qpair);
537 free_ctrlr(struct nvme_ctrlr *ctrlr)
539 spdk_pci_device_unmap_bar(ctrlr->pci_device, 0, (void *)ctrlr->regs);
540 spdk_pci_device_unclaim(ctrlr->pci_device);
541 spdk_pci_device_detach(ctrlr->pci_device);
542 free_qpair(ctrlr->admin_qpair);
543 spdk_free(ctrlr->cdata);
544 free(ctrlr);
550 struct nvme_ctrlr *ctrlr, *tmp;
565 ctrlr = TAILQ_FIRST(&ctrlrs);
566 TAILQ_REMOVE(&ctrlrs, ctrlr, tailq);
567 free_ctrlr(ctrlr);
574 TAILQ_FOREACH_SAFE(ctrlr, &ctrlrs, tailq, tmp) {
575 rc = process_ctrlr_init(ctrlr);
578 TAILQ_REMOVE(&ctrlrs, ctrlr, tailq);
579 free_ctrlr(ctrlr);
583 if (ctrlr->state == NVME_CTRLR_STATE_READY) {
584 TAILQ_REMOVE(&ctrlrs, ctrlr, tailq);
585 TAILQ_INSERT_TAIL(&g_nvme_ctrlrs, ctrlr, tailq);
588 attach_cb(cb_ctx, &ctrlr->pci_device->addr, ctrlr);
617 nvme_detach(struct nvme_ctrlr *ctrlr)
619 TAILQ_REMOVE(&g_nvme_ctrlrs, ctrlr, tailq);
620 free_ctrlr(ctrlr);
624 nvme_ctrlr_get_data(struct nvme_ctrlr *ctrlr)
626 return ctrlr->cdata;