128 	nvme_subqueue_t *queue = &sc->subqueues[qid];  in nvme_alloc_subqueue()  local
135 	lockinit(&queue->lk, "nvqlk", 0, 0);  in nvme_alloc_subqueue()
136 	queue->sc = sc;  in nvme_alloc_subqueue()
137 	queue->nqe = sc->maxqe;  in nvme_alloc_subqueue()
138 	queue->qid = qid;  in nvme_alloc_subqueue()
139 	queue->subq_doorbell_reg = NVME_REG_SUBQ_BELL(qid, sc->dstrd4);  in nvme_alloc_subqueue()
145 		error = bus_dmamem_alloc(sc->sque_tag, (void **)&queue->ksubq,  in nvme_alloc_subqueue()
146 					 BUS_DMA_ZERO, &queue->sque_map);  in nvme_alloc_subqueue()
149 		error = bus_dmamap_load(sc->sque_tag, queue->sque_map,  in nvme_alloc_subqueue()
150 					queue->ksubq,  in nvme_alloc_subqueue()
152 					nvme_dmamem_saveseg, &queue->psubq,  in nvme_alloc_subqueue()
163 		error = bus_dmamem_alloc(sc->prps_tag, (void **)&queue->kprps,  in nvme_alloc_subqueue()
164 					 BUS_DMA_ZERO, &queue->prps_map);  in nvme_alloc_subqueue()
167 		error = bus_dmamap_load(sc->prps_tag, queue->prps_map,  in nvme_alloc_subqueue()
168 					queue->kprps,  in nvme_alloc_subqueue()
170 					nvme_dmamem_saveseg, &queue->pprps,  in nvme_alloc_subqueue()
179 					 (void **)&queue->kdatapgs,  in nvme_alloc_subqueue()
180 					 BUS_DMA_ZERO, &queue->adm_map);  in nvme_alloc_subqueue()
183 		error = bus_dmamap_load(sc->adm_tag, queue->adm_map,  in nvme_alloc_subqueue()
184 					queue->kdatapgs,  in nvme_alloc_subqueue()
186 					nvme_dmamem_saveseg, &queue->pdatapgs,  in nvme_alloc_subqueue()
197 		queue->reqary = kmalloc(sizeof(nvme_request_t) * queue->nqe,  in nvme_alloc_subqueue()
199 		for (i = 0; i < queue->nqe; ++i) {  in nvme_alloc_subqueue()
200 			req = &queue->reqary[i];  in nvme_alloc_subqueue()
205 				queue->dump_req = req;  in nvme_alloc_subqueue()
210 				req->next_avail = queue->first_avail;  in nvme_alloc_subqueue()
211 				queue->first_avail = req;  in nvme_alloc_subqueue()
213 			req->subq = queue;  in nvme_alloc_subqueue()
214 			req->comq = &sc->comqueues[queue->comqid];  in nvme_alloc_subqueue()
217 				req->info = &queue->kdatapgs[i];  in nvme_alloc_subqueue()
218 				req->pinfo = queue->pdatapgs +  in nvme_alloc_subqueue()
235 	nvme_comqueue_t *queue = &sc->comqueues[qid];  in nvme_alloc_comqueue()  local
242 	lockinit(&queue->lk, "nvqlk", 0, 0);  in nvme_alloc_comqueue()
243 	queue->sc = sc;  in nvme_alloc_comqueue()
244 	queue->qid = qid;  in nvme_alloc_comqueue()
245 	queue->phase = NVME_COMQ_STATUS_PHASE;  in nvme_alloc_comqueue()
246 	queue->comq_doorbell_reg = NVME_REG_COMQ_BELL(qid, sc->dstrd4);  in nvme_alloc_comqueue()
249 		error = bus_dmamem_alloc(sc->cque_tag, (void **)&queue->kcomq,  in nvme_alloc_comqueue()
250 					 BUS_DMA_ZERO, &queue->cque_map);  in nvme_alloc_comqueue()
253 		error = bus_dmamap_load(sc->cque_tag, queue->cque_map,  in nvme_alloc_comqueue()
254 					queue->kcomq,  in nvme_alloc_comqueue()
256 					nvme_dmamem_saveseg, &queue->pcomq,  in nvme_alloc_comqueue()
266 		queue->nqe = sc->maxqe;  in nvme_alloc_comqueue()
276 	nvme_subqueue_t *queue = &sc->subqueues[qid];  in nvme_free_subqueue()  local
278 	queue->first_avail = NULL;  in nvme_free_subqueue()
279 	if (queue->reqary) {  in nvme_free_subqueue()
280 		kfree(queue->reqary, M_NVME);  in nvme_free_subqueue()
281 		queue->reqary = NULL;  in nvme_free_subqueue()
283 	if (queue->ksubq) {  in nvme_free_subqueue()
284 		bus_dmamem_free(sc->sque_tag, queue->ksubq, queue->sque_map);  in nvme_free_subqueue()
285 		bus_dmamap_unload(sc->sque_tag, queue->sque_map);  in nvme_free_subqueue()
286 		bus_dmamap_destroy(sc->sque_tag, queue->sque_map);  in nvme_free_subqueue()
288 	if (queue->kprps) {  in nvme_free_subqueue()
289 		bus_dmamem_free(sc->prps_tag, queue->kprps, queue->prps_map);  in nvme_free_subqueue()
290 		bus_dmamap_unload(sc->prps_tag, queue->prps_map);  in nvme_free_subqueue()
291 		bus_dmamap_destroy(sc->prps_tag, queue->prps_map);  in nvme_free_subqueue()
293 	if (queue->kdatapgs) {  in nvme_free_subqueue()
294 		bus_dmamem_free(sc->adm_tag, queue->kdatapgs, queue->adm_map);  in nvme_free_subqueue()
295 		bus_dmamap_unload(sc->adm_tag, queue->adm_map);  in nvme_free_subqueue()
296 		bus_dmamap_destroy(sc->adm_tag, queue->adm_map);  in nvme_free_subqueue()
298 	bzero(queue, sizeof(*queue));  in nvme_free_subqueue()
304 	nvme_comqueue_t *queue = &sc->comqueues[qid];  in nvme_free_comqueue()  local
309 	queue->nqe = 0;  in nvme_free_comqueue()
311 	if (queue->kcomq) {  in nvme_free_comqueue()
312 		bus_dmamem_free(sc->cque_tag, queue->kcomq, queue->cque_map);  in nvme_free_comqueue()
313 		bus_dmamap_unload(sc->cque_tag, queue->cque_map);  in nvme_free_comqueue()
314 		bus_dmamap_destroy(sc->cque_tag, queue->cque_map);  in nvme_free_comqueue()
316 	bzero(queue, sizeof(*queue));  in nvme_free_comqueue()
348 _nvme_fill_request(nvme_subqueue_t *queue, uint8_t opcode,  in _nvme_fill_request()  argument
383 		kprptab = queue->kprps +  in _nvme_fill_request()
385 		pprptab = queue->pprps +  in _nvme_fill_request()
426 nvme_get_request(nvme_subqueue_t *queue, uint8_t opcode,  in nvme_get_request()  argument
438 	lockmgr(&queue->lk, LK_EXCLUSIVE);  in nvme_get_request()
446 	if ((queue->subq_tail + queue->unsubmitted + 1) % queue->nqe ==  in nvme_get_request()
447 	    queue->subq_head) {  in nvme_get_request()
448 		lockmgr(&queue->lk, LK_RELEASE);  in nvme_get_request()
449 		KKASSERT(queue->qid != 0);  in nvme_get_request()
450 		atomic_swap_int(&queue->signal_requeue, 1);  in nvme_get_request()
461 		req = queue->first_avail;  in nvme_get_request()
464 			lockmgr(&queue->lk, LK_RELEASE);  in nvme_get_request()
465 			KKASSERT(queue->qid != 0);  in nvme_get_request()
466 			atomic_swap_int(&queue->signal_requeue, 1);  in nvme_get_request()
471 		if (atomic_cmpset_ptr(&queue->first_avail, req, next))  in nvme_get_request()
480 	++queue->unsubmitted;  in nvme_get_request()
481 	lockmgr(&queue->lk, LK_RELEASE);  in nvme_get_request()
483 	_nvme_fill_request(queue, opcode, kva, bytes, req);  in nvme_get_request()
493 nvme_get_dump_request(nvme_subqueue_t *queue, uint8_t opcode,  in nvme_get_dump_request()  argument
499 	error = lockmgr(&queue->lk, LK_EXCLUSIVE | LK_NOWAIT);  in nvme_get_dump_request()
500 	req = queue->dump_req;  in nvme_get_dump_request()
501 	++queue->unsubmitted;  in nvme_get_dump_request()
503 		lockmgr(&queue->lk, LK_RELEASE);  in nvme_get_dump_request()
504 	_nvme_fill_request(queue, opcode, kva, bytes, req);  in nvme_get_dump_request()
517 	nvme_subqueue_t *queue = req->subq;  in nvme_submit_request()  local
520 	cmd = &queue->ksubq[queue->subq_tail];  in nvme_submit_request()
521 	--queue->unsubmitted;  in nvme_submit_request()
522 	if (++queue->subq_tail == queue->nqe)  in nvme_submit_request()
523 		queue->subq_tail = 0;  in nvme_submit_request()
524 	KKASSERT(queue->subq_tail != queue->subq_head);  in nvme_submit_request()
528 	nvme_write(queue->sc, queue->subq_doorbell_reg, queue->subq_tail);  in nvme_submit_request()
609 	nvme_subqueue_t *queue = req->subq;  in nvme_put_request()  local
618 		next = queue->first_avail;  in nvme_put_request()
621 		if (atomic_cmpset_ptr(&queue->first_avail, next, req))  in nvme_put_request()
631 	if (atomic_swap_int(&queue->signal_requeue, 0)) {  in nvme_put_request()
632 		atomic_set_int(&queue->sc->admin_signal, ADMIN_SIG_REQUEUE);  in nvme_put_request()
633 		wakeup(&queue->sc->admin_signal);  in nvme_put_request()