xref: /openbsd-src/sys/dev/pci/vmwpvs.c (revision 99fd087599a8791921855f21bd7e36130f39aadc)
1 /*	$OpenBSD: vmwpvs.c,v 1.16 2020/02/05 16:29:30 krw Exp $ */
2 
3 /*
4  * Copyright (c) 2013 David Gwynne <dlg@openbsd.org>
5  *
6  * Permission to use, copy, modify, and distribute this software for any
7  * purpose with or without fee is hereby granted, provided that the above
8  * copyright notice and this permission notice appear in all copies.
9  *
10  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17  */
18 
19 #include <sys/param.h>
20 #include <sys/systm.h>
21 #include <sys/device.h>
22 #include <sys/ioctl.h>
23 #include <sys/malloc.h>
24 #include <sys/kernel.h>
25 #include <sys/rwlock.h>
26 #include <sys/dkio.h>
27 #include <sys/task.h>
28 
29 #include <machine/bus.h>
30 
31 #include <dev/pci/pcireg.h>
32 #include <dev/pci/pcivar.h>
33 #include <dev/pci/pcidevs.h>
34 
35 #include <scsi/scsi_all.h>
36 #include <scsi/scsi_message.h>
37 #include <scsi/scsiconf.h>
38 
39 /* pushbuttons */
40 #define VMWPVS_OPENINGS		64 /* according to the linux driver */
41 #define VMWPVS_RING_PAGES	2
42 #define VMWPVS_MAXSGL		(MAXPHYS / PAGE_SIZE)
43 #define VMWPVS_SENSELEN		roundup(sizeof(struct scsi_sense_data), 16)
44 
45 /* "chip" definitions */
46 
47 #define VMWPVS_R_COMMAND	0x0000
48 #define VMWPVS_R_COMMAND_DATA	0x0004
49 #define VMWPVS_R_COMMAND_STATUS	0x0008
50 #define VMWPVS_R_LAST_STS_0	0x0100
51 #define VMWPVS_R_LAST_STS_1	0x0104
52 #define VMWPVS_R_LAST_STS_2	0x0108
53 #define VMWPVS_R_LAST_STS_3	0x010c
54 #define VMWPVS_R_INTR_STATUS	0x100c
55 #define VMWPVS_R_INTR_MASK	0x2010
56 #define VMWPVS_R_KICK_NON_RW_IO	0x3014
57 #define VMWPVS_R_DEBUG		0x3018
58 #define VMWPVS_R_KICK_RW_IO	0x4018
59 
60 #define VMWPVS_INTR_CMPL_0	(1 << 0)
61 #define VMWPVS_INTR_CMPL_1	(1 << 1)
62 #define VMWPVS_INTR_CMPL_MASK	(VMWPVS_INTR_CMPL_0 | VMWPVS_INTR_CMPL_1)
63 #define VMWPVS_INTR_MSG_0	(1 << 2)
64 #define VMWPVS_INTR_MSG_1	(1 << 3)
65 #define VMWPVS_INTR_MSG_MASK	(VMWPVS_INTR_MSG_0 | VMWPVS_INTR_MSG_1)
66 #define VMWPVS_INTR_ALL_MASK	(VMWPVS_INTR_CMPL_MASK | VMWPVS_INTR_MSG_MASK)
67 
68 #define VMWPVS_PAGE_SHIFT	12
69 #define VMWPVS_PAGE_SIZE	(1 << VMWPVS_PAGE_SHIFT)
70 
71 #define VMWPVS_NPG_COMMAND	1
72 #define VMWPVS_NPG_INTR_STATUS	1
73 #define VMWPVS_NPG_MISC		2
74 #define VMWPVS_NPG_KICK_IO	2
75 #define VMWPVS_NPG_MSI_X	2
76 
77 #define VMWPVS_PG_COMMAND	0
78 #define VMWPVS_PG_INTR_STATUS	(VMWPVS_PG_COMMAND + \
79 				    VMWPVS_NPG_COMMAND * VMWPVS_PAGE_SIZE)
80 #define VMWPVS_PG_MISC		(VMWPVS_PG_INTR_STATUS + \
81 				    VMWPVS_NPG_INTR_STATUS * VMWPVS_PAGE_SIZE)
82 #define VMWPVS_PG_KICK_IO	(VMWPVS_PG_MISC + \
83 				    VMWPVS_NPG_MISC * VMWPVS_PAGE_SIZE)
84 #define VMWPVS_PG_MSI_X		(VMWPVS_PG_KICK_IO + \
85 				    VMWPVS_NPG_KICK_IO * VMWPVS_PAGE_SIZE)
86 #define VMMPVS_PG_LEN		(VMWPVS_PG_MSI_X + \
87 				    VMWPVS_NPG_MSI_X * VMWPVS_PAGE_SIZE)
88 
89 struct vmwpvw_ring_state {
90 	u_int32_t		req_prod;
91 	u_int32_t		req_cons;
92 	u_int32_t		req_entries; /* log 2 */
93 
94 	u_int32_t		cmp_prod;
95 	u_int32_t		cmp_cons;
96 	u_int32_t		cmp_entries; /* log 2 */
97 
98 	u_int32_t		__reserved[26];
99 
100 	u_int32_t		msg_prod;
101 	u_int32_t		msg_cons;
102 	u_int32_t		msg_entries; /* log 2 */
103 } __packed;
104 
105 struct vmwpvs_ring_req {
106 	u_int64_t		context;
107 
108 	u_int64_t		data_addr;
109 	u_int64_t		data_len;
110 
111 	u_int64_t		sense_addr;
112 	u_int32_t		sense_len;
113 
114 	u_int32_t		flags;
115 #define VMWPVS_REQ_SGL			(1 << 0)
116 #define VMWPVS_REQ_OOBCDB		(1 << 1)
117 #define VMWPVS_REQ_DIR_NONE		(1 << 2)
118 #define VMWPVS_REQ_DIR_IN		(1 << 3)
119 #define VMWPVS_REQ_DIR_OUT		(1 << 4)
120 
121 	u_int8_t		cdb[16];
122 	u_int8_t		cdblen;
123 	u_int8_t		lun[8];
124 	u_int8_t		tag;
125 	u_int8_t		bus;
126 	u_int8_t		target;
127 	u_int8_t		vcpu_hint;
128 
129 	u_int8_t		__reserved[59];
130 } __packed;
131 #define VMWPVS_REQ_COUNT	((VMWPVS_RING_PAGES * VMWPVS_PAGE_SIZE) / \
132 				    sizeof(struct vmwpvs_ring_req))
133 
134 struct vmwpvs_ring_cmp {
135 	u_int64_t		context;
136 	u_int64_t		data_len;
137 	u_int32_t		sense_len;
138 	u_int16_t		host_status;
139 	u_int16_t		scsi_status;
140 	u_int32_t		__reserved[2];
141 } __packed;
142 #define VMWPVS_CMP_COUNT	((VMWPVS_RING_PAGES * VMWPVS_PAGE_SIZE) / \
143 				    sizeof(struct vmwpvs_ring_cmp))
144 
145 struct vmwpvs_sge {
146 	u_int64_t		addr;
147 	u_int32_t		len;
148 	u_int32_t		flags;
149 } __packed;
150 
151 struct vmwpvs_ring_msg {
152 	u_int32_t		type;
153 	u_int32_t		__args[31];
154 } __packed;
155 #define VMWPVS_MSG_COUNT	((VMWPVS_RING_PAGES * VMWPVS_PAGE_SIZE) / \
156 				    sizeof(struct vmwpvs_ring_msg))
157 
158 #define VMWPVS_MSG_T_ADDED	0
159 #define VMWPVS_MSG_T_REMOVED	1
160 
161 struct vmwpvs_ring_msg_dev {
162 	u_int32_t		type;
163 	u_int32_t		bus;
164 	u_int32_t		target;
165 	u_int8_t		lun[8];
166 
167 	u_int32_t		__pad[27];
168 } __packed;
169 
170 struct vmwpvs_cfg_cmd {
171 	u_int64_t		cmp_addr;
172 	u_int32_t		pg_addr;
173 	u_int32_t		pg_addr_type;
174 	u_int32_t		pg_num;
175 	u_int32_t		__reserved;
176 } __packed;
177 
178 #define VMWPVS_MAX_RING_PAGES		32
179 struct vmwpvs_setup_rings_cmd {
180 	u_int32_t		req_pages;
181 	u_int32_t		cmp_pages;
182 	u_int64_t		state_ppn;
183 	u_int64_t		req_page_ppn[VMWPVS_MAX_RING_PAGES];
184 	u_int64_t		cmp_page_ppn[VMWPVS_MAX_RING_PAGES];
185 } __packed;
186 
187 #define VMWPVS_MAX_MSG_RING_PAGES	16
188 struct vmwpvs_setup_rings_msg {
189 	u_int32_t		msg_pages;
190 	u_int32_t		__reserved;
191 	u_int64_t		msg_page_ppn[VMWPVS_MAX_MSG_RING_PAGES];
192 } __packed;
193 
194 #define VMWPVS_CMD_FIRST		0
195 #define VMWPVS_CMD_ADAPTER_RESET	1
196 #define VMWPVS_CMD_ISSUE_SCSI		2
197 #define VMWPVS_CMD_SETUP_RINGS		3
198 #define VMWPVS_CMD_RESET_BUS		4
199 #define VMWPVS_CMD_RESET_DEVICE		5
200 #define VMWPVS_CMD_ABORT_CMD		6
201 #define VMWPVS_CMD_CONFIG		7
202 #define VMWPVS_CMD_SETUP_MSG_RING	8
203 #define VMWPVS_CMD_DEVICE_UNPLUG	9
204 #define VMWPVS_CMD_LAST			10
205 
206 #define VMWPVS_CFGPG_CONTROLLER		0x1958
207 #define VMWPVS_CFGPG_PHY		0x1959
208 #define VMWPVS_CFGPG_DEVICE		0x195a
209 
210 #define VMWPVS_CFGPGADDR_CONTROLLER	0x2120
211 #define VMWPVS_CFGPGADDR_TARGET		0x2121
212 #define VMWPVS_CFGPGADDR_PHY		0x2122
213 
214 struct vmwpvs_cfg_pg_header {
215 	u_int32_t		pg_num;
216 	u_int16_t		num_dwords;
217 	u_int16_t		host_status;
218 	u_int16_t		scsi_status;
219 	u_int16_t		__reserved[3];
220 } __packed;
221 
222 #define VMWPVS_HOST_STATUS_SUCCESS	0x00
223 #define VMWPVS_HOST_STATUS_LINKED_CMD_COMPLETED 0x0a
224 #define VMWPVS_HOST_STATUS_LINKED_CMD_COMPLETED_WITH_FLAG 0x0b
225 #define VMWPVS_HOST_STATUS_UNDERRUN	0x0c
226 #define VMWPVS_HOST_STATUS_SELTIMEOUT	0x11
227 #define VMWPVS_HOST_STATUS_DATARUN	0x12
228 #define VMWPVS_HOST_STATUS_BUSFREE	0x13
229 #define VMWPVS_HOST_STATUS_INVPHASE	0x14
230 #define VMWPVS_HOST_STATUS_LUNMISMATCH	0x17
231 #define VMWPVS_HOST_STATUS_INVPARAM	0x1a
232 #define VMWPVS_HOST_STATUS_SENSEFAILED	0x1b
233 #define VMWPVS_HOST_STATUS_TAGREJECT	0x1c
234 #define VMWPVS_HOST_STATUS_BADMSG	0x1d
235 #define VMWPVS_HOST_STATUS_HAHARDWARE	0x20
236 #define VMWPVS_HOST_STATUS_NORESPONSE	0x21
237 #define VMWPVS_HOST_STATUS_SENT_RST	0x22
238 #define VMWPVS_HOST_STATUS_RECV_RST	0x23
239 #define VMWPVS_HOST_STATUS_DISCONNECT	0x24
240 #define VMWPVS_HOST_STATUS_BUS_RESET	0x25
241 #define VMWPVS_HOST_STATUS_ABORT_QUEUE	0x26
242 #define VMWPVS_HOST_STATUS_HA_SOFTWARE	0x27
243 #define VMWPVS_HOST_STATUS_HA_TIMEOUT	0x30
244 #define VMWPVS_HOST_STATUS_SCSI_PARITY	0x34
245 
246 #define VMWPVS_SCSI_STATUS_OK		0x00
247 #define VMWPVS_SCSI_STATUS_CHECK	0x02
248 
249 struct vmwpvs_cfg_pg_controller {
250 	struct vmwpvs_cfg_pg_header header;
251 
252 	u_int64_t		wwnn;
253 	u_int16_t		manufacturer[64];
254 	u_int16_t		serial_number[64];
255 	u_int16_t		oprom_version[32];
256 	u_int16_t		hardware_version[32];
257 	u_int16_t		firmware_version[32];
258 	u_int32_t		num_phys;
259 	u_int8_t		use_consec_phy_wwns;
260 	u_int8_t		__reserved[3];
261 } __packed;
262 
263 /* driver stuff */
264 
265 struct vmwpvs_dmamem {
266 	bus_dmamap_t		dm_map;
267 	bus_dma_segment_t	dm_seg;
268 	size_t			dm_size;
269 	caddr_t			dm_kva;
270 };
271 #define VMWPVS_DMA_MAP(_dm)	(_dm)->dm_map
272 #define VMWPVS_DMA_DVA(_dm)	(_dm)->dm_map->dm_segs[0].ds_addr
273 #define VMWPVS_DMA_KVA(_dm)	(void *)(_dm)->dm_kva
274 
275 struct vmwpvs_sgl {
276 	struct vmwpvs_sge	list[VMWPVS_MAXSGL];
277 } __packed;
278 
279 struct vmwpvs_ccb {
280 	SIMPLEQ_ENTRY(vmwpvs_ccb)
281 				ccb_entry;
282 
283 	bus_dmamap_t		ccb_dmamap;
284 	struct scsi_xfer	*ccb_xs;
285 	u_int64_t		ccb_ctx;
286 
287 	struct vmwpvs_sgl	*ccb_sgl;
288 	bus_addr_t		ccb_sgl_offset;
289 
290 	void			*ccb_sense;
291 	bus_addr_t		ccb_sense_offset;
292 };
293 SIMPLEQ_HEAD(vmwpvs_ccb_list, vmwpvs_ccb);
294 
295 struct vmwpvs_softc {
296 	struct device		sc_dev;
297 
298 	pci_chipset_tag_t	sc_pc;
299 	pcitag_t		sc_tag;
300 
301 	bus_space_tag_t		sc_iot;
302 	bus_space_handle_t	sc_ioh;
303 	bus_size_t		sc_ios;
304 	bus_dma_tag_t		sc_dmat;
305 
306 	struct vmwpvs_dmamem	*sc_req_ring;
307 	struct vmwpvs_dmamem	*sc_cmp_ring;
308 	struct vmwpvs_dmamem	*sc_msg_ring;
309 	struct vmwpvs_dmamem	*sc_ring_state;
310 	struct mutex		sc_ring_mtx;
311 
312 	struct vmwpvs_dmamem	*sc_sgls;
313 	struct vmwpvs_dmamem	*sc_sense;
314 	struct vmwpvs_ccb	*sc_ccbs;
315 	struct vmwpvs_ccb_list	sc_ccb_list;
316 	struct mutex		sc_ccb_mtx;
317 
318 	void			*sc_ih;
319 
320 	struct task		sc_msg_task;
321 
322 	u_int			sc_bus_width;
323 
324 	struct scsi_link	sc_link;
325 	struct scsi_iopool	sc_iopool;
326 	struct scsibus_softc	*sc_scsibus;
327 };
328 #define DEVNAME(_s)		((_s)->sc_dev.dv_xname)
329 
330 int	vmwpvs_match(struct device *, void *, void *);
331 void	vmwpvs_attach(struct device *, struct device *, void *);
332 
333 int	vmwpvs_intx(void *);
334 int	vmwpvs_intr(void *);
335 
336 #define vmwpvs_read(_s, _r) \
337 	bus_space_read_4((_s)->sc_iot, (_s)->sc_ioh, (_r))
338 #define vmwpvs_write(_s, _r, _v) \
339 	bus_space_write_4((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v))
340 #define vmwpvs_barrier(_s, _r, _l, _d) \
341 	bus_space_barrier((_s)->sc_iot, (_s)->sc_ioh, (_r), (_l), (_d))
342 
343 struct cfattach vmwpvs_ca = {
344 	sizeof(struct vmwpvs_softc),
345 	vmwpvs_match,
346 	vmwpvs_attach,
347 	NULL
348 };
349 
350 struct cfdriver vmwpvs_cd = {
351 	NULL,
352 	"vmwpvs",
353 	DV_DULL
354 };
355 
356 void		vmwpvs_scsi_cmd(struct scsi_xfer *);
357 
358 struct scsi_adapter vmwpvs_switch = {
359 	vmwpvs_scsi_cmd, NULL, NULL, NULL, NULL
360 };
361 
362 #define dwordsof(s)		(sizeof(s) / sizeof(u_int32_t))
363 
364 void		vmwpvs_ccb_put(void *, void *);
365 void *		vmwpvs_ccb_get(void *);
366 
367 struct vmwpvs_dmamem *
368 		vmwpvs_dmamem_alloc(struct vmwpvs_softc *, size_t);
369 struct vmwpvs_dmamem *
370 		vmwpvs_dmamem_zalloc(struct vmwpvs_softc *, size_t);
371 void		vmwpvs_dmamem_free(struct vmwpvs_softc *,
372 		    struct vmwpvs_dmamem *);
373 
374 void		vmwpvs_cmd(struct vmwpvs_softc *, u_int32_t, void *, size_t);
375 int		vmwpvs_get_config(struct vmwpvs_softc *);
376 void		vmwpvs_setup_rings(struct vmwpvs_softc *);
377 void		vmwpvs_setup_msg_ring(struct vmwpvs_softc *);
378 void		vmwpvs_msg_task(void *);
379 
380 struct vmwpvs_ccb *
381 		vmwpvs_scsi_cmd_poll(struct vmwpvs_softc *);
382 struct vmwpvs_ccb *
383 		vmwpvs_scsi_cmd_done(struct vmwpvs_softc *,
384 		    struct vmwpvs_ring_cmp *);
385 
386 int
387 vmwpvs_match(struct device *parent, void *match, void *aux)
388 {
389 	struct pci_attach_args *pa = aux;
390 
391 	if (PCI_VENDOR(pa->pa_id) == PCI_VENDOR_VMWARE &&
392 	    PCI_PRODUCT(pa->pa_id) == PCI_PRODUCT_VMWARE_PVSCSI)
393 		return (1);
394 
395 	return (0);
396 }
397 
398 void
399 vmwpvs_attach(struct device *parent, struct device *self, void *aux)
400 {
401 	struct vmwpvs_softc *sc = (struct vmwpvs_softc *)self;
402 	struct pci_attach_args *pa = aux;
403 	struct scsibus_attach_args saa;
404 	pcireg_t memtype;
405 	u_int i, r, use_msg;
406 	int (*isr)(void *) = vmwpvs_intx;
407 	u_int32_t intmask;
408 	pci_intr_handle_t ih;
409 
410 	struct vmwpvs_ccb *ccb;
411 	struct vmwpvs_sgl *sgls;
412 	u_int8_t *sense;
413 
414 	sc->sc_pc = pa->pa_pc;
415 	sc->sc_tag = pa->pa_tag;
416 	sc->sc_dmat = pa->pa_dmat;
417 
418 	sc->sc_bus_width = 16;
419 	mtx_init(&sc->sc_ring_mtx, IPL_BIO);
420 	mtx_init(&sc->sc_ccb_mtx, IPL_BIO);
421 	task_set(&sc->sc_msg_task, vmwpvs_msg_task, sc);
422 	SIMPLEQ_INIT(&sc->sc_ccb_list);
423 
424 	for (r = PCI_MAPREG_START; r < PCI_MAPREG_END; r += sizeof(memtype)) {
425 		memtype = pci_mapreg_type(sc->sc_pc, sc->sc_tag, r);
426 		if ((memtype & PCI_MAPREG_TYPE_MASK) == PCI_MAPREG_TYPE_MEM)
427 			break;
428 	}
429 	if (r >= PCI_MAPREG_END) {
430 		printf(": unable to locate registers\n");
431 		return;
432 	}
433 
434 	if (pci_mapreg_map(pa, r, memtype, 0, &sc->sc_iot, &sc->sc_ioh,
435 	    NULL, &sc->sc_ios, VMMPVS_PG_LEN) != 0) {
436 		printf(": unable to map registers\n");
437 		return;
438 	}
439 
440 	/* hook up the interrupt */
441 	vmwpvs_write(sc, VMWPVS_R_INTR_MASK, 0);
442 
443 	if (pci_intr_map_msi(pa, &ih) == 0)
444 		isr = vmwpvs_intr;
445 	else if (pci_intr_map(pa, &ih) != 0) {
446 		printf(": unable to map interrupt\n");
447 		goto unmap;
448 	}
449 	printf(": %s\n", pci_intr_string(sc->sc_pc, ih));
450 
451 	/* do we have msg support? */
452 	vmwpvs_write(sc, VMWPVS_R_COMMAND, VMWPVS_CMD_SETUP_MSG_RING);
453 	use_msg = (vmwpvs_read(sc, VMWPVS_R_COMMAND_STATUS) != 0xffffffff);
454 
455 	if (vmwpvs_get_config(sc) != 0) {
456 		printf("%s: get configuration failed\n", DEVNAME(sc));
457 		goto unmap;
458 	}
459 
460 	sc->sc_ring_state = vmwpvs_dmamem_zalloc(sc, VMWPVS_PAGE_SIZE);
461 	if (sc->sc_ring_state == NULL) {
462 		printf("%s: unable to allocate ring state\n", DEVNAME(sc));
463 		goto unmap;
464 	}
465 
466 	sc->sc_req_ring = vmwpvs_dmamem_zalloc(sc,
467 	    VMWPVS_RING_PAGES * VMWPVS_PAGE_SIZE);
468 	if (sc->sc_req_ring == NULL) {
469 		printf("%s: unable to allocate req ring\n", DEVNAME(sc));
470 		goto free_ring_state;
471 	}
472 
473 	sc->sc_cmp_ring = vmwpvs_dmamem_zalloc(sc,
474 	    VMWPVS_RING_PAGES * VMWPVS_PAGE_SIZE);
475 	if (sc->sc_cmp_ring == NULL) {
476 		printf("%s: unable to allocate cmp ring\n", DEVNAME(sc));
477 		goto free_req_ring;
478 	}
479 
480 	if (use_msg) {
481 		sc->sc_msg_ring = vmwpvs_dmamem_zalloc(sc,
482 		    VMWPVS_RING_PAGES * VMWPVS_PAGE_SIZE);
483 		if (sc->sc_msg_ring == NULL) {
484 			printf("%s: unable to allocate msg ring\n",
485 			    DEVNAME(sc));
486 			goto free_cmp_ring;
487 		}
488 	}
489 
490 	r = (VMWPVS_RING_PAGES * VMWPVS_PAGE_SIZE) /
491 	    sizeof(struct vmwpvs_ring_req);
492 
493 	sc->sc_sgls = vmwpvs_dmamem_alloc(sc, r * sizeof(struct vmwpvs_sgl));
494 	if (sc->sc_sgls == NULL) {
495 		printf("%s: unable to allocate sgls\n", DEVNAME(sc));
496 		goto free_msg_ring;
497 	}
498 
499 	sc->sc_sense = vmwpvs_dmamem_alloc(sc, r * VMWPVS_SENSELEN);
500 	if (sc->sc_sense == NULL) {
501 		printf("%s: unable to allocate sense data\n", DEVNAME(sc));
502 		goto free_sgl;
503 	}
504 
505 	sc->sc_ccbs = mallocarray(r, sizeof(struct vmwpvs_ccb),
506 	    M_DEVBUF, M_WAITOK);
507 	/* cant fail */
508 
509 	sgls = VMWPVS_DMA_KVA(sc->sc_sgls);
510 	sense = VMWPVS_DMA_KVA(sc->sc_sense);
511 	for (i = 0; i < r; i++) {
512 		ccb = &sc->sc_ccbs[i];
513 
514 		if (bus_dmamap_create(sc->sc_dmat, MAXPHYS,
515 		    VMWPVS_MAXSGL, MAXPHYS, 0,
516 		    BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW,
517 		    &ccb->ccb_dmamap) != 0) {
518 			printf("%s: unable to create ccb map\n", DEVNAME(sc));
519 			goto free_ccbs;
520 		}
521 
522 		ccb->ccb_ctx = 0xdeadbeef00000000ULL | (u_int64_t)i;
523 
524 		ccb->ccb_sgl_offset = i * sizeof(*sgls);
525 		ccb->ccb_sgl = &sgls[i];
526 
527 		ccb->ccb_sense_offset = i * VMWPVS_SENSELEN;
528 		ccb->ccb_sense = sense + ccb->ccb_sense_offset;
529 
530 		vmwpvs_ccb_put(sc, ccb);
531 	}
532 
533 	sc->sc_ih = pci_intr_establish(sc->sc_pc, ih, IPL_BIO,
534 	    isr, sc, DEVNAME(sc));
535 	if (sc->sc_ih == NULL)
536 		goto free_msg_ring;
537 
538 	bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_cmp_ring), 0,
539 	    VMWPVS_RING_PAGES * VMWPVS_PAGE_SIZE, BUS_DMASYNC_PREREAD);
540 	bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_req_ring), 0,
541 	    VMWPVS_RING_PAGES * VMWPVS_PAGE_SIZE, BUS_DMASYNC_PREWRITE);
542 	if (use_msg) {
543 		bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_msg_ring), 0,
544 		    VMWPVS_RING_PAGES * VMWPVS_PAGE_SIZE, BUS_DMASYNC_PREREAD);
545 	}
546 	bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_ring_state), 0,
547 	    VMWPVS_PAGE_SIZE, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
548 
549 	intmask = VMWPVS_INTR_CMPL_MASK;
550 
551 	vmwpvs_setup_rings(sc);
552 	if (use_msg) {
553 		vmwpvs_setup_msg_ring(sc);
554 		intmask |= VMWPVS_INTR_MSG_MASK;
555 	}
556 
557 	vmwpvs_write(sc, VMWPVS_R_INTR_MASK, intmask);
558 
559 	/* controller init is done, lets plug the midlayer in */
560 
561 	scsi_iopool_init(&sc->sc_iopool, sc, vmwpvs_ccb_get, vmwpvs_ccb_put);
562 
563 	sc->sc_link.adapter = &vmwpvs_switch;
564 	sc->sc_link.adapter_softc = sc;
565 	sc->sc_link.adapter_target = -1;
566 	sc->sc_link.adapter_buswidth = sc->sc_bus_width;
567 	sc->sc_link.openings = VMWPVS_OPENINGS;
568 	sc->sc_link.pool = &sc->sc_iopool;
569 
570 	bzero(&saa, sizeof(saa));
571 	saa.saa_sc_link = &sc->sc_link;
572 
573 	sc->sc_scsibus = (struct scsibus_softc *)config_found(&sc->sc_dev,
574 	    &saa, scsiprint);
575 
576 	return;
577 free_ccbs:
578 	while ((ccb = vmwpvs_ccb_get(sc)) != NULL)
579 		bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap);
580 	free(sc->sc_ccbs, M_DEVBUF, r * sizeof(struct vmwpvs_ccb));
581 /* free_sense: */
582 	vmwpvs_dmamem_free(sc, sc->sc_sense);
583 free_sgl:
584 	vmwpvs_dmamem_free(sc, sc->sc_sgls);
585 free_msg_ring:
586 	if (use_msg)
587 		vmwpvs_dmamem_free(sc, sc->sc_msg_ring);
588 free_cmp_ring:
589 	vmwpvs_dmamem_free(sc, sc->sc_cmp_ring);
590 free_req_ring:
591 	vmwpvs_dmamem_free(sc, sc->sc_req_ring);
592 free_ring_state:
593 	vmwpvs_dmamem_free(sc, sc->sc_ring_state);
594 unmap:
595 	bus_space_unmap(sc->sc_iot, sc->sc_ioh, sc->sc_ios);
596 	sc->sc_ios = 0;
597 }
598 
599 void
600 vmwpvs_setup_rings(struct vmwpvs_softc *sc)
601 {
602 	struct vmwpvs_setup_rings_cmd cmd;
603 	u_int64_t ppn;
604 	u_int i;
605 
606 	memset(&cmd, 0, sizeof(cmd));
607 	cmd.req_pages = VMWPVS_RING_PAGES;
608 	cmd.cmp_pages = VMWPVS_RING_PAGES;
609 	cmd.state_ppn = VMWPVS_DMA_DVA(sc->sc_ring_state) >> VMWPVS_PAGE_SHIFT;
610 
611 	ppn = VMWPVS_DMA_DVA(sc->sc_req_ring) >> VMWPVS_PAGE_SHIFT;
612 	for (i = 0; i < VMWPVS_RING_PAGES; i++)
613 		cmd.req_page_ppn[i] = ppn + i;
614 
615 	ppn = VMWPVS_DMA_DVA(sc->sc_cmp_ring) >> VMWPVS_PAGE_SHIFT;
616 	for (i = 0; i < VMWPVS_RING_PAGES; i++)
617 		cmd.cmp_page_ppn[i] = ppn + i;
618 
619 	vmwpvs_cmd(sc, VMWPVS_CMD_SETUP_RINGS, &cmd, sizeof(cmd));
620 }
621 
622 void
623 vmwpvs_setup_msg_ring(struct vmwpvs_softc *sc)
624 {
625 	struct vmwpvs_setup_rings_msg cmd;
626 	u_int64_t ppn;
627 	u_int i;
628 
629 	memset(&cmd, 0, sizeof(cmd));
630 	cmd.msg_pages = VMWPVS_RING_PAGES;
631 
632 	ppn = VMWPVS_DMA_DVA(sc->sc_msg_ring) >> VMWPVS_PAGE_SHIFT;
633 	for (i = 0; i < VMWPVS_RING_PAGES; i++)
634 		cmd.msg_page_ppn[i] = ppn + i;
635 
636 	vmwpvs_cmd(sc, VMWPVS_CMD_SETUP_MSG_RING, &cmd, sizeof(cmd));
637 }
638 
639 int
640 vmwpvs_get_config(struct vmwpvs_softc *sc)
641 {
642 	struct vmwpvs_cfg_cmd cmd;
643 	struct vmwpvs_dmamem *dm;
644 	struct vmwpvs_cfg_pg_controller *pg;
645 	struct vmwpvs_cfg_pg_header *hdr;
646 	int rv = 0;
647 
648 	dm = vmwpvs_dmamem_alloc(sc, VMWPVS_PAGE_SIZE);
649 	if (dm == NULL)
650 		return (ENOMEM);
651 
652 	memset(&cmd, 0, sizeof(cmd));
653 	cmd.cmp_addr = VMWPVS_DMA_DVA(dm);
654 	cmd.pg_addr_type = VMWPVS_CFGPGADDR_CONTROLLER;
655 	cmd.pg_num = VMWPVS_CFGPG_CONTROLLER;
656 
657 	pg = VMWPVS_DMA_KVA(dm);
658 	memset(pg, 0, VMWPVS_PAGE_SIZE);
659 	hdr = &pg->header;
660 	hdr->host_status = VMWPVS_HOST_STATUS_INVPARAM;
661 	hdr->scsi_status = VMWPVS_SCSI_STATUS_CHECK;
662 
663 	bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(dm), 0, VMWPVS_PAGE_SIZE,
664 	    BUS_DMASYNC_PREREAD);
665 	vmwpvs_cmd(sc, VMWPVS_CMD_CONFIG, &cmd, sizeof(cmd));
666 	bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(dm), 0, VMWPVS_PAGE_SIZE,
667 	    BUS_DMASYNC_POSTREAD);
668 
669 	if (hdr->host_status != VMWPVS_HOST_STATUS_SUCCESS ||
670 	    hdr->scsi_status != VMWPVS_SCSI_STATUS_OK) {
671 		rv = EIO;
672 		goto done;
673 	}
674 
675 	sc->sc_bus_width = pg->num_phys;
676 
677 done:
678 	vmwpvs_dmamem_free(sc, dm);
679 
680 	return (rv);
681 
682 }
683 
684 void
685 vmwpvs_cmd(struct vmwpvs_softc *sc, u_int32_t cmd, void *buf, size_t len)
686 {
687 	u_int32_t *p = buf;
688 	u_int i;
689 
690 	len /= sizeof(*p);
691 
692 	vmwpvs_write(sc, VMWPVS_R_COMMAND, cmd);
693 	for (i = 0; i < len; i++)
694 		vmwpvs_write(sc, VMWPVS_R_COMMAND_DATA, p[i]);
695 }
696 
697 int
698 vmwpvs_intx(void *xsc)
699 {
700 	struct vmwpvs_softc *sc = xsc;
701 	u_int32_t status;
702 
703 	status = vmwpvs_read(sc, VMWPVS_R_INTR_STATUS);
704 	if ((status & VMWPVS_INTR_ALL_MASK) == 0)
705 		return (0);
706 
707 	vmwpvs_write(sc, VMWPVS_R_INTR_STATUS, status);
708 
709 	return (vmwpvs_intr(sc));
710 }
711 
712 int
713 vmwpvs_intr(void *xsc)
714 {
715 	struct vmwpvs_softc *sc = xsc;
716 	volatile struct vmwpvw_ring_state *s =
717 	    VMWPVS_DMA_KVA(sc->sc_ring_state);
718 	struct vmwpvs_ring_cmp *ring = VMWPVS_DMA_KVA(sc->sc_cmp_ring);
719 	struct vmwpvs_ccb_list list = SIMPLEQ_HEAD_INITIALIZER(list);
720 	struct vmwpvs_ccb *ccb;
721 	u_int32_t cons, prod;
722 	int msg;
723 
724 	mtx_enter(&sc->sc_ring_mtx);
725 
726 	bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_ring_state), 0,
727 	    VMWPVS_PAGE_SIZE, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
728 	cons = s->cmp_cons;
729 	prod = s->cmp_prod;
730 	s->cmp_cons = prod;
731 
732 	msg = (sc->sc_msg_ring != NULL && s->msg_cons != s->msg_prod);
733 
734 	bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_ring_state), 0,
735 	    VMWPVS_PAGE_SIZE, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
736 
737 	if (cons != prod) {
738 		bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_cmp_ring),
739 		    0, VMWPVS_PAGE_SIZE, BUS_DMASYNC_POSTREAD);
740 
741 		do {
742 			ccb = vmwpvs_scsi_cmd_done(sc,
743 			    &ring[cons++ % VMWPVS_CMP_COUNT]);
744 			SIMPLEQ_INSERT_TAIL(&list, ccb, ccb_entry);
745 		} while (cons != prod);
746 
747 		bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_cmp_ring),
748 		    0, VMWPVS_PAGE_SIZE, BUS_DMASYNC_PREREAD);
749 	}
750 
751 	mtx_leave(&sc->sc_ring_mtx);
752 
753 	while ((ccb = SIMPLEQ_FIRST(&list)) != NULL) {
754 		SIMPLEQ_REMOVE_HEAD(&list, ccb_entry);
755 		scsi_done(ccb->ccb_xs);
756 	}
757 
758 	if (msg)
759 		task_add(systq, &sc->sc_msg_task);
760 
761 	return (1);
762 }
763 
764 void
765 vmwpvs_msg_task(void *xsc)
766 {
767 	struct vmwpvs_softc *sc = xsc;
768 	volatile struct vmwpvw_ring_state *s =
769 	    VMWPVS_DMA_KVA(sc->sc_ring_state);
770 	struct vmwpvs_ring_msg *ring = VMWPVS_DMA_KVA(sc->sc_msg_ring);
771 	struct vmwpvs_ring_msg *msg;
772 	struct vmwpvs_ring_msg_dev *dvmsg;
773 	u_int32_t cons, prod;
774 
775 	mtx_enter(&sc->sc_ring_mtx);
776 	bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_ring_state), 0,
777 	    VMWPVS_PAGE_SIZE, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
778 	cons = s->msg_cons;
779 	prod = s->msg_prod;
780 	bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_ring_state), 0,
781 	    VMWPVS_PAGE_SIZE, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
782 	mtx_leave(&sc->sc_ring_mtx);
783 
784 	/*
785 	 * we dont have to lock around the msg ring cos the system taskq has
786 	 * only one thread.
787 	 */
788 
789 	bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_msg_ring), 0,
790 	    VMWPVS_RING_PAGES * VMWPVS_PAGE_SIZE, BUS_DMASYNC_POSTREAD);
791 	while (cons != prod) {
792 		msg = &ring[cons++ % VMWPVS_MSG_COUNT];
793 
794 		switch (letoh32(msg->type)) {
795 		case VMWPVS_MSG_T_ADDED:
796 			dvmsg = (struct vmwpvs_ring_msg_dev *)msg;
797 			if (letoh32(dvmsg->bus) != 0) {
798 				printf("%s: ignoring request to add device"
799 				    " on bus %d\n", DEVNAME(sc),
800 				    letoh32(msg->type));
801 				break;
802 			}
803 
804 			if (scsi_probe_lun(sc->sc_scsibus,
805 			    letoh32(dvmsg->target), dvmsg->lun[1]) != 0) {
806 				printf("%s: error probing target %d lun %d\n",
807 				    DEVNAME(sc), letoh32(dvmsg->target),
808 				    dvmsg->lun[1]);
809 			};
810 			break;
811 
812 		case VMWPVS_MSG_T_REMOVED:
813 			dvmsg = (struct vmwpvs_ring_msg_dev *)msg;
814 			if (letoh32(dvmsg->bus) != 0) {
815 				printf("%s: ignorint request to remove device"
816 				    " on bus %d\n", DEVNAME(sc),
817 				    letoh32(msg->type));
818 				break;
819 			}
820 
821 			if (scsi_detach_lun(sc->sc_scsibus,
822 			    letoh32(dvmsg->target), dvmsg->lun[1],
823 			    DETACH_FORCE) != 0) {
824 				printf("%s: error detaching target %d lun %d\n",
825 				    DEVNAME(sc), letoh32(dvmsg->target),
826 				    dvmsg->lun[1]);
827 			};
828 			break;
829 
830 		default:
831 			printf("%s: unknown msg type %u\n", DEVNAME(sc),
832 			    letoh32(msg->type));
833 			break;
834 		}
835 	}
836 	bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_msg_ring), 0,
837 	    VMWPVS_RING_PAGES * VMWPVS_PAGE_SIZE, BUS_DMASYNC_PREREAD);
838 
839 	mtx_enter(&sc->sc_ring_mtx);
840 	bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_ring_state), 0,
841 	    VMWPVS_PAGE_SIZE, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
842 	s->msg_cons = prod;
843 	bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_ring_state), 0,
844 	    VMWPVS_PAGE_SIZE, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
845 	mtx_leave(&sc->sc_ring_mtx);
846 }
847 
848 void
849 vmwpvs_scsi_cmd(struct scsi_xfer *xs)
850 {
851 	struct scsi_link *link = xs->sc_link;
852 	struct vmwpvs_softc *sc = link->adapter_softc;
853 	struct vmwpvs_ccb *ccb = xs->io;
854 	bus_dmamap_t dmap = ccb->ccb_dmamap;
855 	volatile struct vmwpvw_ring_state *s =
856 	    VMWPVS_DMA_KVA(sc->sc_ring_state);
857 	struct vmwpvs_ring_req *ring = VMWPVS_DMA_KVA(sc->sc_req_ring), *r;
858 	u_int32_t prod;
859 	struct vmwpvs_ccb_list list;
860 	int error;
861 	u_int i;
862 
863 	ccb->ccb_xs = xs;
864 
865 	if (xs->datalen > 0) {
866 		error = bus_dmamap_load(sc->sc_dmat, dmap,
867 		    xs->data, xs->datalen, NULL, (xs->flags & SCSI_NOSLEEP) ?
868 		    BUS_DMA_NOWAIT : BUS_DMA_WAITOK);
869 		if (error) {
870 			xs->error = XS_DRIVER_STUFFUP;
871 			scsi_done(xs);
872 			return;
873 		}
874 
875 		bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,
876 		    (xs->flags & SCSI_DATA_IN) ? BUS_DMASYNC_PREREAD :
877 		    BUS_DMASYNC_PREWRITE);
878 	}
879 
880 	mtx_enter(&sc->sc_ring_mtx);
881 
882 	bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_ring_state), 0,
883 	    VMWPVS_PAGE_SIZE, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
884 
885 	prod = s->req_prod;
886 	r = &ring[prod % VMWPVS_REQ_COUNT];
887 
888 	bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_req_ring),
889 	    prod * sizeof(*r), sizeof(*r), BUS_DMASYNC_POSTWRITE);
890 
891 	memset(r, 0, sizeof(*r));
892 	r->context = ccb->ccb_ctx;
893 
894 	if (xs->datalen > 0) {
895 		r->data_len = xs->datalen;
896 		if (dmap->dm_nsegs == 1) {
897 			r->data_addr = dmap->dm_segs[0].ds_addr;
898 		} else {
899 			struct vmwpvs_sge *sgl = ccb->ccb_sgl->list, *sge;
900 
901 			r->data_addr = VMWPVS_DMA_DVA(sc->sc_sgls) +
902 			    ccb->ccb_sgl_offset;
903 			r->flags = VMWPVS_REQ_SGL;
904 
905 			for (i = 0; i < dmap->dm_nsegs; i++) {
906 				sge = &sgl[i];
907 				sge->addr = dmap->dm_segs[i].ds_addr;
908 				sge->len = dmap->dm_segs[i].ds_len;
909 				sge->flags = 0;
910 			}
911 
912 			bus_dmamap_sync(sc->sc_dmat,
913 			    VMWPVS_DMA_MAP(sc->sc_sgls), ccb->ccb_sgl_offset,
914 			    sizeof(*sge) * dmap->dm_nsegs,
915 			    BUS_DMASYNC_PREWRITE);
916 		}
917 	}
918 	r->sense_addr = VMWPVS_DMA_DVA(sc->sc_sense) + ccb->ccb_sense_offset;
919 	r->sense_len = sizeof(xs->sense);
920 
921 	bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_req_ring), 0,
922 	    VMWPVS_RING_PAGES * VMWPVS_PAGE_SIZE, BUS_DMASYNC_POSTWRITE);
923 
924 	switch (xs->flags & (SCSI_DATA_IN | SCSI_DATA_OUT)) {
925 	case SCSI_DATA_IN:
926 		r->flags |= VMWPVS_REQ_DIR_IN;
927 		break;
928 	case SCSI_DATA_OUT:
929 		r->flags |= VMWPVS_REQ_DIR_OUT;
930 		break;
931 	default:
932 		r->flags |= VMWPVS_REQ_DIR_NONE;
933 		break;
934 	}
935 
936 	memcpy(r->cdb, xs->cmd, xs->cmdlen);
937 	r->cdblen = xs->cmdlen;
938 	r->lun[1] = link->lun; /* ugly :( */
939 	r->tag = MSG_SIMPLE_Q_TAG;
940 	r->bus = 0;
941 	r->target = link->target;
942 	r->vcpu_hint = 0;
943 
944 	bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_req_ring), 0,
945 	    VMWPVS_RING_PAGES * VMWPVS_PAGE_SIZE, BUS_DMASYNC_PREWRITE);
946 
947 	s->req_prod = prod + 1;
948 
949 	bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_ring_state), 0,
950 	    VMWPVS_PAGE_SIZE, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
951 
952 	vmwpvs_write(sc, xs->bp == NULL ?
953 	    VMWPVS_R_KICK_NON_RW_IO : VMWPVS_R_KICK_RW_IO, 0);
954 
955 	if (!ISSET(xs->flags, SCSI_POLL)) {
956 		mtx_leave(&sc->sc_ring_mtx);
957 		return;
958 	}
959 
960 	SIMPLEQ_INIT(&list);
961 	do {
962 		ccb = vmwpvs_scsi_cmd_poll(sc);
963 		SIMPLEQ_INSERT_TAIL(&list, ccb, ccb_entry);
964 	} while (xs->io != ccb);
965 
966 	mtx_leave(&sc->sc_ring_mtx);
967 
968 	while ((ccb = SIMPLEQ_FIRST(&list)) != NULL) {
969 		SIMPLEQ_REMOVE_HEAD(&list, ccb_entry);
970 		scsi_done(ccb->ccb_xs);
971 	}
972 }
973 
974 struct vmwpvs_ccb *
975 vmwpvs_scsi_cmd_poll(struct vmwpvs_softc *sc)
976 {
977 	volatile struct vmwpvw_ring_state *s =
978 	    VMWPVS_DMA_KVA(sc->sc_ring_state);
979 	struct vmwpvs_ring_cmp *ring = VMWPVS_DMA_KVA(sc->sc_cmp_ring);
980 	struct vmwpvs_ccb *ccb;
981 	u_int32_t prod, cons;
982 
983 	for (;;) {
984 		bus_dmamap_sync(sc->sc_dmat,
985 		    VMWPVS_DMA_MAP(sc->sc_ring_state), 0, VMWPVS_PAGE_SIZE,
986 		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
987 
988 		cons = s->cmp_cons;
989 		prod = s->cmp_prod;
990 
991 		if (cons != prod)
992 			s->cmp_cons = cons + 1;
993 
994 		bus_dmamap_sync(sc->sc_dmat,
995 		    VMWPVS_DMA_MAP(sc->sc_ring_state), 0, VMWPVS_PAGE_SIZE,
996 		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
997 
998 		if (cons != prod)
999 			break;
1000 		else
1001 			delay(1000);
1002 	}
1003 
1004 	bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_cmp_ring),
1005 	    0, VMWPVS_PAGE_SIZE * VMWPVS_RING_PAGES,
1006 	    BUS_DMASYNC_POSTREAD);
1007 	ccb = vmwpvs_scsi_cmd_done(sc, &ring[cons % VMWPVS_CMP_COUNT]);
1008 	bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_cmp_ring),
1009 	    0, VMWPVS_PAGE_SIZE * VMWPVS_RING_PAGES,
1010 	    BUS_DMASYNC_PREREAD);
1011 
1012 	return (ccb);
1013 }
1014 
1015 struct vmwpvs_ccb *
1016 vmwpvs_scsi_cmd_done(struct vmwpvs_softc *sc, struct vmwpvs_ring_cmp *c)
1017 {
1018 	u_int64_t ctx = c->context;
1019 	struct vmwpvs_ccb *ccb = &sc->sc_ccbs[ctx & 0xffffffff];
1020 	bus_dmamap_t dmap = ccb->ccb_dmamap;
1021 	struct scsi_xfer *xs = ccb->ccb_xs;
1022 
1023 	bus_dmamap_sync(sc->sc_dmat, VMWPVS_DMA_MAP(sc->sc_sense),
1024 	    ccb->ccb_sense_offset, sizeof(xs->sense), BUS_DMASYNC_POSTREAD);
1025 
1026 	if (xs->datalen > 0) {
1027 		if (dmap->dm_nsegs > 1) {
1028 			bus_dmamap_sync(sc->sc_dmat,
1029 			    VMWPVS_DMA_MAP(sc->sc_sgls), ccb->ccb_sgl_offset,
1030 			    sizeof(struct vmwpvs_sge) * dmap->dm_nsegs,
1031 			    BUS_DMASYNC_POSTWRITE);
1032 		}
1033 
1034 		bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,
1035 		    (xs->flags & SCSI_DATA_IN) ? BUS_DMASYNC_POSTREAD :
1036 		    BUS_DMASYNC_POSTWRITE);
1037 
1038 		bus_dmamap_unload(sc->sc_dmat, dmap);
1039 	}
1040 
1041 	xs->status = c->scsi_status;
1042 	switch (c->host_status) {
1043 	case VMWPVS_HOST_STATUS_SUCCESS:
1044 	case VMWPVS_HOST_STATUS_LINKED_CMD_COMPLETED:
1045 	case VMWPVS_HOST_STATUS_LINKED_CMD_COMPLETED_WITH_FLAG:
1046 		if (c->scsi_status == VMWPVS_SCSI_STATUS_CHECK) {
1047 			memcpy(&xs->sense, ccb->ccb_sense, sizeof(xs->sense));
1048 			xs->error = XS_SENSE;
1049 		} else
1050 			xs->error = XS_NOERROR;
1051 		xs->resid = 0;
1052 		break;
1053 
1054 	case VMWPVS_HOST_STATUS_UNDERRUN:
1055 	case VMWPVS_HOST_STATUS_DATARUN:
1056 		xs->resid = xs->datalen - c->data_len;
1057 		xs->error = XS_NOERROR;
1058 		break;
1059 
1060 	case VMWPVS_HOST_STATUS_SELTIMEOUT:
1061 		xs->error = XS_SELTIMEOUT;
1062 		break;
1063 
1064 	default:
1065 		printf("%s: %s:%d h:0x%x s:0x%x\n", DEVNAME(sc),
1066 		    __FUNCTION__, __LINE__, c->host_status, c->scsi_status);
1067 		xs->error = XS_DRIVER_STUFFUP;
1068 		break;
1069 	}
1070 
1071 	return (ccb);
1072 }
1073 
1074 void *
1075 vmwpvs_ccb_get(void *xsc)
1076 {
1077 	struct vmwpvs_softc *sc = xsc;
1078 	struct vmwpvs_ccb *ccb;
1079 
1080 	mtx_enter(&sc->sc_ccb_mtx);
1081 	ccb = SIMPLEQ_FIRST(&sc->sc_ccb_list);
1082 	if (ccb != NULL)
1083 		SIMPLEQ_REMOVE_HEAD(&sc->sc_ccb_list, ccb_entry);
1084 	mtx_leave(&sc->sc_ccb_mtx);
1085 
1086 	return (ccb);
1087 }
1088 
1089 void
1090 vmwpvs_ccb_put(void *xsc, void *io)
1091 {
1092 	struct vmwpvs_softc *sc = xsc;
1093 	struct vmwpvs_ccb *ccb = io;
1094 
1095 	mtx_enter(&sc->sc_ccb_mtx);
1096 	SIMPLEQ_INSERT_HEAD(&sc->sc_ccb_list, ccb, ccb_entry);
1097 	mtx_leave(&sc->sc_ccb_mtx);
1098 }
1099 
1100 struct vmwpvs_dmamem *
1101 vmwpvs_dmamem_alloc(struct vmwpvs_softc *sc, size_t size)
1102 {
1103 	struct vmwpvs_dmamem *dm;
1104 	int nsegs;
1105 
1106 	dm = malloc(sizeof(*dm), M_DEVBUF, M_NOWAIT | M_ZERO);
1107 	if (dm == NULL)
1108 		return (NULL);
1109 
1110 	dm->dm_size = size;
1111 
1112 	if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0,
1113 	    BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, &dm->dm_map) != 0)
1114 		goto dmfree;
1115 
1116 	if (bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0, &dm->dm_seg,
1117 	    1, &nsegs, BUS_DMA_NOWAIT | BUS_DMA_ZERO) != 0)
1118 		goto destroy;
1119 
1120 	if (bus_dmamem_map(sc->sc_dmat, &dm->dm_seg, nsegs, size,
1121 	    &dm->dm_kva, BUS_DMA_NOWAIT) != 0)
1122 		goto free;
1123 
1124 	if (bus_dmamap_load(sc->sc_dmat, dm->dm_map, dm->dm_kva, size,
1125 	    NULL, BUS_DMA_NOWAIT) != 0)
1126 		goto unmap;
1127 
1128 	return (dm);
1129 
1130 unmap:
1131 	bus_dmamem_unmap(sc->sc_dmat, dm->dm_kva, size);
1132 free:
1133 	bus_dmamem_free(sc->sc_dmat, &dm->dm_seg, 1);
1134 destroy:
1135 	bus_dmamap_destroy(sc->sc_dmat, dm->dm_map);
1136 dmfree:
1137 	free(dm, M_DEVBUF, sizeof *dm);
1138 
1139 	return (NULL);
1140 }
1141 
1142 struct vmwpvs_dmamem *
1143 vmwpvs_dmamem_zalloc(struct vmwpvs_softc *sc, size_t size)
1144 {
1145 	struct vmwpvs_dmamem *dm;
1146 
1147 	dm = vmwpvs_dmamem_alloc(sc, size);
1148 	if (dm == NULL)
1149 		return (NULL);
1150 
1151 	memset(VMWPVS_DMA_KVA(dm), 0, size);
1152 
1153 	return (dm);
1154 }
1155 
1156 void
1157 vmwpvs_dmamem_free(struct vmwpvs_softc *sc, struct vmwpvs_dmamem *dm)
1158 {
1159 	bus_dmamap_unload(sc->sc_dmat, dm->dm_map);
1160 	bus_dmamem_unmap(sc->sc_dmat, dm->dm_kva, dm->dm_size);
1161 	bus_dmamem_free(sc->sc_dmat, &dm->dm_seg, 1);
1162 	bus_dmamap_destroy(sc->sc_dmat, dm->dm_map);
1163 	free(dm, M_DEVBUF, sizeof *dm);
1164 }
1165