64 SDT_PROBE_DEFINE5(usb, device, pipe, open,
71 SDT_PROBE_DEFINE7(usb, device, pipe, open__intr,
80 SDT_PROBE_DEFINE2(usb, device, pipe, transfer__start,
83 SDT_PROBE_DEFINE3(usb, device, pipe, transfer__done,
87 SDT_PROBE_DEFINE2(usb, device, pipe, start,
91 SDT_PROBE_DEFINE1(usb, device, pipe, close,  "struct usbd_pipe *"/*pipe*/);
92 SDT_PROBE_DEFINE1(usb, device, pipe, abort__start,
94 SDT_PROBE_DEFINE1(usb, device, pipe, abort__done,
96 SDT_PROBE_DEFINE1(usb, device, pipe, clear__endpoint__stall,
98 SDT_PROBE_DEFINE1(usb, device, pipe, clear__endpoint__toggle,
194 usbd_dump_queue(struct usbd_pipe *pipe)  in usbd_dump_queue()  argument
199 	USBHIST_CALLARGS(usbdebug, "pipe = %#jx", (uintptr_t)pipe, 0, 0, 0);  in usbd_dump_queue()
201 	SIMPLEQ_FOREACH(xfer, &pipe->up_queue, ux_next) {  in usbd_dump_queue()
208 usbd_dump_pipe(struct usbd_pipe *pipe)  in usbd_dump_pipe()  argument
211 	USBHIST_CALLARGS(usbdebug, "pipe = %#jx", (uintptr_t)pipe, 0, 0, 0);  in usbd_dump_pipe()
213 	if (pipe == NULL)  in usbd_dump_pipe()
215 	usbd_dump_iface(pipe->up_iface);  in usbd_dump_pipe()
216 	usbd_dump_device(pipe->up_dev);  in usbd_dump_pipe()
217 	usbd_dump_endpoint(pipe->up_endpoint);  in usbd_dump_pipe()
220 	    pipe->up_running, pipe->up_aborting, 0, 0);  in usbd_dump_pipe()
222 	    "interval = %jd", (uintptr_t)pipe->up_intrxfer, pipe->up_repeat,  in usbd_dump_pipe()
223 	    pipe->up_interval, 0);  in usbd_dump_pipe()
229 	       uint8_t flags, struct usbd_pipe **pipe)  in usbd_open_pipe()  argument
231 	return (usbd_open_pipe_ival(iface, address, flags, pipe,  in usbd_open_pipe()
237 		    uint8_t flags, struct usbd_pipe **pipe, int ival)  in usbd_open_pipe_ival()  argument
280 	*pipe = p;  in usbd_open_pipe_ival()
283 	SDT_PROBE5(usb, device, pipe, open,  in usbd_open_pipe_ival()
296 		    uint8_t flags, struct usbd_pipe **pipe,  in usbd_open_pipe_intr()  argument
321 	*pipe = ipipe;  in usbd_open_pipe_intr()
324 	SDT_PROBE7(usb, device, pipe, open__intr,  in usbd_open_pipe_intr()
339 usbd_close_pipe(struct usbd_pipe *pipe)  in usbd_close_pipe()  argument
343 	KASSERT(pipe != NULL);  in usbd_close_pipe()
345 	usbd_lock_pipe(pipe);  in usbd_close_pipe()
346 	SDT_PROBE1(usb, device, pipe, close,  pipe);  in usbd_close_pipe()
347 	if (!SIMPLEQ_EMPTY(&pipe->up_queue)) {  in usbd_close_pipe()
349 		    pipe->up_dev->ud_addr);  in usbd_close_pipe()
350 		usbd_ar_pipe(pipe);  in usbd_close_pipe()
352 	KASSERT(SIMPLEQ_EMPTY(&pipe->up_queue));  in usbd_close_pipe()
353 	pipe->up_methods->upm_close(pipe);  in usbd_close_pipe()
354 	usbd_unlock_pipe(pipe);  in usbd_close_pipe()
356 	cv_destroy(&pipe->up_callingcv);  in usbd_close_pipe()
357 	if (pipe->up_intrxfer)  in usbd_close_pipe()
358 		usbd_destroy_xfer(pipe->up_intrxfer);  in usbd_close_pipe()
359 	usb_rem_task_wait(pipe->up_dev, &pipe->up_async_task, USB_TASKQ_DRIVER,  in usbd_close_pipe()
361 	usbd_endpoint_release(pipe->up_dev, pipe->up_endpoint);  in usbd_close_pipe()
362 	if (pipe->up_iface)  in usbd_close_pipe()
363 		usbd_iface_pipeunref(pipe->up_iface);  in usbd_close_pipe()
364 	kmem_free(pipe, pipe->up_dev->ud_bus->ub_pipesize);  in usbd_close_pipe()
370 	struct usbd_pipe *pipe = xfer->ux_pipe;  in usbd_transfer()  local
376 	    (uintptr_t)xfer, xfer->ux_flags, (uintptr_t)pipe, pipe->up_running);  in usbd_transfer()
382 		usbd_dump_queue(pipe);  in usbd_transfer()
413 	if (pipe->up_dev->ud_bus->ub_usepolling == 0)  in usbd_transfer()
414 		usbd_lock_pipe(pipe);  in usbd_transfer()
415 	if (pipe->up_aborting) {  in usbd_transfer()
421 		KASSERT(pipe->up_dev->ud_bus->ub_usepolling == 0);  in usbd_transfer()
422 		usbd_unlock_pipe(pipe);  in usbd_transfer()
430 	SDT_PROBE2(usb, device, pipe, transfer__start,  pipe, xfer);  in usbd_transfer()
435 		SIMPLEQ_INSERT_TAIL(&pipe->up_queue, xfer, ux_next);  in usbd_transfer()
436 		if (pipe->up_running && pipe->up_serialise) {  in usbd_transfer()
439 			pipe->up_running = 1;  in usbd_transfer()
444 		err = pipe->up_methods->upm_transfer(xfer);  in usbd_transfer()
446 	SDT_PROBE3(usb, device, pipe, transfer__done,  pipe, xfer, err);  in usbd_transfer()
448 	if (pipe->up_dev->ud_bus->ub_usepolling == 0)  in usbd_transfer()
449 		usbd_unlock_pipe(pipe);  in usbd_transfer()
459 		if (pipe->up_dev->ud_bus->ub_usepolling == 0)  in usbd_transfer()
460 			usbd_lock_pipe(pipe);  in usbd_transfer()
465 		SIMPLEQ_REMOVE_HEAD(&pipe->up_queue, ux_next);  in usbd_transfer()
466 		if (pipe->up_serialise)  in usbd_transfer()
467 			usbd_start_next(pipe);  in usbd_transfer()
468 		if (pipe->up_dev->ud_bus->ub_usepolling == 0)  in usbd_transfer()
469 			usbd_unlock_pipe(pipe);  in usbd_transfer()
488 	if (pipe->up_dev->ud_bus->ub_usepolling == 0)  in usbd_transfer()
489 		usbd_lock_pipe(pipe);  in usbd_transfer()
491 		if (pipe->up_dev->ud_bus->ub_usepolling)  in usbd_transfer()
498 			err = cv_wait_sig(&xfer->ux_cv, pipe->up_dev->ud_bus->ub_lock);  in usbd_transfer()
500 			cv_wait(&xfer->ux_cv, pipe->up_dev->ud_bus->ub_lock);  in usbd_transfer()
505 				pipe->up_methods->upm_abort(xfer);  in usbd_transfer()
512 	if (pipe->up_dev->ud_bus->ub_usepolling == 0)  in usbd_transfer()
513 		usbd_unlock_pipe(pipe);  in usbd_transfer()
651 usbd_create_xfer(struct usbd_pipe *pipe, size_t len, unsigned int flags,  in usbd_create_xfer()  argument
657 	struct usbd_xfer *xfer = usbd_alloc_xfer(pipe->up_dev, nframes);  in usbd_create_xfer()
661 	xfer->ux_pipe = pipe;  in usbd_create_xfer()
664 	xfer->ux_methods = pipe->up_methods;  in usbd_create_xfer()
684 	    xfer, pipe, len, flags, nframes);  in usbd_create_xfer()
813 usbd_abort_pipe(struct usbd_pipe *pipe)  in usbd_abort_pipe()  argument
816 	usbd_suspend_pipe(pipe);  in usbd_abort_pipe()
817 	usbd_resume_pipe(pipe);  in usbd_abort_pipe()
821 usbd_suspend_pipe(struct usbd_pipe *pipe)  in usbd_suspend_pipe()  argument
824 	usbd_lock_pipe(pipe);  in usbd_suspend_pipe()
825 	usbd_ar_pipe(pipe);  in usbd_suspend_pipe()
826 	usbd_unlock_pipe(pipe);  in usbd_suspend_pipe()
830 usbd_resume_pipe(struct usbd_pipe *pipe)  in usbd_resume_pipe()  argument
833 	usbd_lock_pipe(pipe);  in usbd_resume_pipe()
834 	KASSERT(SIMPLEQ_EMPTY(&pipe->up_queue));  in usbd_resume_pipe()
835 	pipe->up_aborting = 0;  in usbd_resume_pipe()
836 	usbd_unlock_pipe(pipe);  in usbd_resume_pipe()
840 usbd_clear_endpoint_stall(struct usbd_pipe *pipe)  in usbd_clear_endpoint_stall()  argument
842 	struct usbd_device *dev = pipe->up_dev;  in usbd_clear_endpoint_stall()
846 	SDT_PROBE1(usb, device, pipe, clear__endpoint__stall,  pipe);  in usbd_clear_endpoint_stall()
852 	SDT_PROBE1(usb, device, pipe, clear__endpoint__toggle,  pipe);  in usbd_clear_endpoint_stall()
853 	pipe->up_methods->upm_cleartoggle(pipe);  in usbd_clear_endpoint_stall()
856 	    pipe->up_endpoint->ue_edesc->bEndpointAddress, UF_ENDPOINT_HALT);  in usbd_clear_endpoint_stall()
860 		pipe->state = USBD_PIPE_ACTIVE;  in usbd_clear_endpoint_stall()
870 	struct usbd_pipe *pipe = arg;  in usbd_clear_endpoint_stall_task()  local
871 	struct usbd_device *dev = pipe->up_dev;  in usbd_clear_endpoint_stall_task()
873 	SDT_PROBE1(usb, device, pipe, clear__endpoint__stall,  pipe);  in usbd_clear_endpoint_stall_task()
874 	SDT_PROBE1(usb, device, pipe, clear__endpoint__toggle,  pipe);  in usbd_clear_endpoint_stall_task()
875 	pipe->up_methods->upm_cleartoggle(pipe);  in usbd_clear_endpoint_stall_task()
878 	    pipe->up_endpoint->ue_edesc->bEndpointAddress, UF_ENDPOINT_HALT);  in usbd_clear_endpoint_stall_task()
882 usbd_clear_endpoint_stall_async(struct usbd_pipe *pipe)  in usbd_clear_endpoint_stall_async()  argument
884 	usb_add_task(pipe->up_dev, &pipe->up_async_task, USB_TASKQ_DRIVER);  in usbd_clear_endpoint_stall_async()
888 usbd_clear_endpoint_toggle(struct usbd_pipe *pipe)  in usbd_clear_endpoint_toggle()  argument
891 	SDT_PROBE1(usb, device, pipe, clear__endpoint__toggle,  pipe);  in usbd_clear_endpoint_toggle()
892 	pipe->up_methods->upm_cleartoggle(pipe);  in usbd_clear_endpoint_toggle()
937 usbd_pipe2device_handle(struct usbd_pipe *pipe)  in usbd_pipe2device_handle()  argument
939 	KASSERT(pipe != NULL);  in usbd_pipe2device_handle()
941 	return pipe->up_dev;  in usbd_pipe2device_handle()
1027 usbd_ar_pipe(struct usbd_pipe *pipe)  in usbd_ar_pipe()  argument
1032 	USBHIST_CALLARGS(usbdebug, "pipe = %#jx", (uintptr_t)pipe, 0, 0, 0);  in usbd_ar_pipe()
1033 	SDT_PROBE1(usb, device, pipe, abort__start,  pipe);  in usbd_ar_pipe()
1036 	KASSERT(mutex_owned(pipe->up_dev->ud_bus->ub_lock));  in usbd_ar_pipe()
1037 	KASSERT(pipe->up_dev->ud_bus->ub_usepolling == 0);  in usbd_ar_pipe()
1045 	KASSERTMSG(pipe->up_abortlwp == NULL, "pipe->up_abortlwp=%p",  in usbd_ar_pipe()
1046 	    pipe->up_abortlwp);  in usbd_ar_pipe()
1047 	pipe->up_abortlwp = curlwp;  in usbd_ar_pipe()
1051 		usbd_dump_queue(pipe);  in usbd_ar_pipe()
1053 	pipe->up_repeat = 0;  in usbd_ar_pipe()
1054 	pipe->up_running = 0;  in usbd_ar_pipe()
1055 	pipe->up_aborting = 1;  in usbd_ar_pipe()
1056 	while ((xfer = SIMPLEQ_FIRST(&pipe->up_queue)) != NULL) {  in usbd_ar_pipe()
1058 		    "(methods = %#jx)", (uintptr_t)pipe, (uintptr_t)xfer,  in usbd_ar_pipe()
1059 		    (uintptr_t)pipe->up_methods, 0);  in usbd_ar_pipe()
1065 			SIMPLEQ_REMOVE_HEAD(&pipe->up_queue, ux_next);  in usbd_ar_pipe()
1069 			pipe->up_methods->upm_abort(xfer);  in usbd_ar_pipe()
1070 			while (pipe->up_callingxfer == xfer) {  in usbd_ar_pipe()
1073 				    (uintptr_t)pipe, (uintptr_t)xfer, 0, 0);  in usbd_ar_pipe()
1074 				cv_wait(&pipe->up_callingcv,  in usbd_ar_pipe()
1075 				    pipe->up_dev->ud_bus->ub_lock);  in usbd_ar_pipe()
1087 	while (pipe->up_callingxfer) {  in usbd_ar_pipe()
1090 		    (uintptr_t)pipe, (uintptr_t)pipe->up_callingxfer, 0, 0);  in usbd_ar_pipe()
1091 		cv_wait(&pipe->up_callingcv, pipe->up_dev->ud_bus->ub_lock);  in usbd_ar_pipe()
1094 	KASSERT(mutex_owned(pipe->up_dev->ud_bus->ub_lock));  in usbd_ar_pipe()
1095 	KASSERTMSG(pipe->up_abortlwp == curlwp, "pipe->up_abortlwp=%p",  in usbd_ar_pipe()
1096 	    pipe->up_abortlwp);  in usbd_ar_pipe()
1097 	pipe->up_abortlwp = NULL;  in usbd_ar_pipe()
1099 	SDT_PROBE1(usb, device, pipe, abort__done,  pipe);  in usbd_ar_pipe()
1106 	struct usbd_pipe *pipe = xfer->ux_pipe;  in usb_transfer_complete()  local
1107 	struct usbd_bus *bus = pipe->up_dev->ud_bus;  in usb_transfer_complete()
1111 	int repeat = pipe->up_repeat;  in usb_transfer_complete()
1115 	    "actlen = %jd", (uintptr_t)pipe, (uintptr_t)xfer, xfer->ux_status,  in usb_transfer_complete()
1118 	KASSERT(polling || mutex_owned(pipe->up_dev->ud_bus->ub_lock));  in usb_transfer_complete()
1121 	KASSERT(pipe != NULL);  in usb_transfer_complete()
1128 	if (pipe->up_dev->ud_quirks &&  in usb_transfer_complete()
1129 	    pipe->up_dev->ud_quirks->uq_flags & UQ_MISS_OUT_ACK &&  in usb_transfer_complete()
1147 		KASSERTMSG(!SIMPLEQ_EMPTY(&pipe->up_queue),  in usb_transfer_complete()
1148 		    "pipe %p is empty, but xfer %p wants to complete", pipe,  in usb_transfer_complete()
1150 		KASSERTMSG(xfer == SIMPLEQ_FIRST(&pipe->up_queue),  in usb_transfer_complete()
1152 		   SIMPLEQ_FIRST(&pipe->up_queue));  in usb_transfer_complete()
1157 		SIMPLEQ_REMOVE_HEAD(&pipe->up_queue, ux_next);  in usb_transfer_complete()
1160 	    (uintptr_t)xfer, repeat, (uintptr_t)SIMPLEQ_FIRST(&pipe->up_queue),  in usb_transfer_complete()
1164 	++pipe->up_dev->ud_bus->ub_stats.uds_requests  in usb_transfer_complete()
1165 		[pipe->up_endpoint->ue_edesc->bmAttributes & UE_XFERTYPE];  in usb_transfer_complete()
1176 	    (uintptr_t)pipe->up_methods->upm_done, 0, 0);  in usb_transfer_complete()
1178 	pipe->up_methods->upm_done(xfer);  in usb_transfer_complete()
1195 			KASSERT(pipe->up_callingxfer == NULL);  in usb_transfer_complete()
1196 			pipe->up_callingxfer = xfer;  in usb_transfer_complete()
1197 			mutex_exit(pipe->up_dev->ud_bus->ub_lock);  in usb_transfer_complete()
1198 			if (!(pipe->up_flags & USBD_MPSAFE))  in usb_transfer_complete()
1205 			if (!(pipe->up_flags & USBD_MPSAFE))  in usb_transfer_complete()
1207 			mutex_enter(pipe->up_dev->ud_bus->ub_lock);  in usb_transfer_complete()
1208 			KASSERT(pipe->up_callingxfer == xfer);  in usb_transfer_complete()
1209 			pipe->up_callingxfer = NULL;  in usb_transfer_complete()
1210 			cv_broadcast(&pipe->up_callingcv);  in usb_transfer_complete()
1225 		if (erred && pipe->up_iface != NULL)	/* not control pipe */  in usb_transfer_complete()
1226 			pipe->up_running = 0;  in usb_transfer_complete()
1228 	if (pipe->up_running && pipe->up_serialise)  in usb_transfer_complete()
1229 		usbd_start_next(pipe);  in usb_transfer_complete()
1234 usbd_start_next(struct usbd_pipe *pipe)  in usbd_start_next()  argument
1241 	KASSERT(pipe != NULL);  in usbd_start_next()
1242 	KASSERT(pipe->up_methods != NULL);  in usbd_start_next()
1243 	KASSERT(pipe->up_methods->upm_start != NULL);  in usbd_start_next()
1244 	KASSERT(pipe->up_serialise == true);  in usbd_start_next()
1246 	int polling = pipe->up_dev->ud_bus->ub_usepolling;  in usbd_start_next()
1247 	KASSERT(polling || mutex_owned(pipe->up_dev->ud_bus->ub_lock));  in usbd_start_next()
1250 	xfer = SIMPLEQ_FIRST(&pipe->up_queue);  in usbd_start_next()
1251 	USBHIST_CALLARGS(usbdebug, "pipe = %#jx, xfer = %#jx", (uintptr_t)pipe,  in usbd_start_next()
1254 		pipe->up_running = 0;  in usbd_start_next()
1256 		SDT_PROBE2(usb, device, pipe, start,  pipe, xfer);  in usbd_start_next()
1257 		err = pipe->up_methods->upm_start(xfer);  in usbd_start_next()
1261 			pipe->up_running = 0;  in usbd_start_next()
1266 	KASSERT(polling || mutex_owned(pipe->up_dev->ud_bus->ub_lock));  in usbd_start_next()