xref: /netbsd-src/sys/external/bsd/dwc2/dist/dwc2_hcdintr.c (revision 40ee8ddd3195603c135fd14e5ed18a28c58490d1)
1 /*	$NetBSD: dwc2_hcdintr.c,v 1.15 2018/08/12 09:59:30 skrll Exp $	*/
2 
3 /*
4  * hcd_intr.c - DesignWare HS OTG Controller host-mode interrupt handling
5  *
6  * Copyright (C) 2004-2013 Synopsys, Inc.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions, and the following disclaimer,
13  *    without modification.
14  * 2. Redistributions in binary form must reproduce the above copyright
15  *    notice, this list of conditions and the following disclaimer in the
16  *    documentation and/or other materials provided with the distribution.
17  * 3. The names of the above-listed copyright holders may not be used
18  *    to endorse or promote products derived from this software without
19  *    specific prior written permission.
20  *
21  * ALTERNATIVELY, this software may be distributed under the terms of the
22  * GNU General Public License ("GPL") as published by the Free Software
23  * Foundation; either version 2 of the License, or (at your option) any
24  * later version.
25  *
26  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
27  * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
28  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
29  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
30  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
31  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
32  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
33  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
34  * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
35  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
36  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37  */
38 
39 /*
40  * This file contains the interrupt handlers for Host mode
41  */
42 #include <sys/cdefs.h>
43 __KERNEL_RCSID(0, "$NetBSD: dwc2_hcdintr.c,v 1.15 2018/08/12 09:59:30 skrll Exp $");
44 
45 #include <sys/types.h>
46 #include <sys/pool.h>
47 
48 #include <dev/usb/usb.h>
49 #include <dev/usb/usbdi.h>
50 #include <dev/usb/usbdivar.h>
51 #include <dev/usb/usb_mem.h>
52 
53 #include <machine/param.h>
54 
55 #include <linux/kernel.h>
56 
57 #include <dwc2/dwc2.h>
58 #include <dwc2/dwc2var.h>
59 
60 #include "dwc2_core.h"
61 #include "dwc2_hcd.h"
62 
63 /*
64  * If we get this many NAKs on a split transaction we'll slow down
65  * retransmission.  A 1 here means delay after the first NAK.
66  */
67 #define DWC2_NAKS_BEFORE_DELAY		3
68 int dwc2_naks_before_delay = DWC2_NAKS_BEFORE_DELAY;
69 
70 #define DWC2_OUT_NAKS_BEFORE_DELAY	1
71 int dwc2_out_naks_before_delay = DWC2_OUT_NAKS_BEFORE_DELAY;
72 
73 /* This function is for debug only */
dwc2_track_missed_sofs(struct dwc2_hsotg * hsotg)74 static void dwc2_track_missed_sofs(struct dwc2_hsotg *hsotg)
75 {
76 #ifdef CONFIG_USB_DWC2_TRACK_MISSED_SOFS
77 	u16 curr_frame_number = hsotg->frame_number;
78 
79 	if (hsotg->frame_num_idx < FRAME_NUM_ARRAY_SIZE) {
80 		if (((hsotg->last_frame_num + 1) & HFNUM_MAX_FRNUM) !=
81 		    curr_frame_number) {
82 			hsotg->frame_num_array[hsotg->frame_num_idx] =
83 					curr_frame_number;
84 			hsotg->last_frame_num_array[hsotg->frame_num_idx] =
85 					hsotg->last_frame_num;
86 			hsotg->frame_num_idx++;
87 		}
88 	} else if (!hsotg->dumped_frame_num_array) {
89 		int i;
90 
91 		dev_info(hsotg->dev, "Frame     Last Frame\n");
92 		dev_info(hsotg->dev, "-----     ----------\n");
93 		for (i = 0; i < FRAME_NUM_ARRAY_SIZE; i++) {
94 			dev_info(hsotg->dev, "0x%04x    0x%04x\n",
95 				 hsotg->frame_num_array[i],
96 				 hsotg->last_frame_num_array[i]);
97 		}
98 		hsotg->dumped_frame_num_array = 1;
99 	}
100 	hsotg->last_frame_num = curr_frame_number;
101 #endif
102 }
103 
dwc2_hc_handle_tt_clear(struct dwc2_hsotg * hsotg,struct dwc2_host_chan * chan,struct dwc2_qtd * qtd)104 static void dwc2_hc_handle_tt_clear(struct dwc2_hsotg *hsotg,
105 				    struct dwc2_host_chan *chan,
106 				    struct dwc2_qtd *qtd)
107 {
108 // 	struct urb *usb_urb;
109 
110 	if (!chan->qh)
111 		return;
112 
113 	if (chan->qh->dev_speed == USB_SPEED_HIGH)
114 		return;
115 
116 	if (!qtd->urb)
117 		return;
118 
119 
120 	if (qtd->urb->status != -EPIPE && qtd->urb->status != -EREMOTEIO) {
121 		chan->qh->tt_buffer_dirty = 1;
122 			chan->qh->tt_buffer_dirty = 0;
123 	}
124 }
125 
126 /*
127  * Handles the start-of-frame interrupt in host mode. Non-periodic
128  * transactions may be queued to the DWC_otg controller for the current
129  * (micro)frame. Periodic transactions may be queued to the controller
130  * for the next (micro)frame.
131  */
dwc2_sof_intr(struct dwc2_hsotg * hsotg)132 static void dwc2_sof_intr(struct dwc2_hsotg *hsotg)
133 {
134 	struct list_head *qh_entry;
135 	struct dwc2_qh *qh;
136 	enum dwc2_transaction_type tr_type;
137 
138 	/* Clear interrupt */
139 	DWC2_WRITE_4(hsotg, GINTSTS, GINTSTS_SOF);
140 
141 #ifdef DEBUG_SOF
142 	dev_vdbg(hsotg->dev, "--Start of Frame Interrupt--\n");
143 #endif
144 
145 	hsotg->frame_number = dwc2_hcd_get_frame_number(hsotg);
146 
147 	dwc2_track_missed_sofs(hsotg);
148 
149 	/* Determine whether any periodic QHs should be executed */
150 	qh_entry = hsotg->periodic_sched_inactive.next;
151 	while (qh_entry != &hsotg->periodic_sched_inactive) {
152 		qh = list_entry(qh_entry, struct dwc2_qh, qh_list_entry);
153 		qh_entry = qh_entry->next;
154 		if (dwc2_frame_num_le(qh->sched_frame, hsotg->frame_number))
155 			/*
156 			 * Move QH to the ready list to be executed next
157 			 * (micro)frame
158 			 */
159 			list_move(&qh->qh_list_entry,
160 				  &hsotg->periodic_sched_ready);
161 	}
162 	tr_type = dwc2_hcd_select_transactions(hsotg);
163 	if (tr_type != DWC2_TRANSACTION_NONE)
164 		dwc2_hcd_queue_transactions(hsotg, tr_type);
165 }
166 
167 /*
168  * Handles the Rx FIFO Level Interrupt, which indicates that there is
169  * at least one packet in the Rx FIFO. The packets are moved from the FIFO to
170  * memory if the DWC_otg controller is operating in Slave mode.
171  */
dwc2_rx_fifo_level_intr(struct dwc2_hsotg * hsotg)172 static void dwc2_rx_fifo_level_intr(struct dwc2_hsotg *hsotg)
173 {
174 	u32 grxsts, chnum, bcnt, pktsts;
175 	struct dwc2_host_chan *chan;
176 
177 	if (dbg_perio())
178 		dev_vdbg(hsotg->dev, "--RxFIFO Level Interrupt--\n");
179 
180 	grxsts = DWC2_READ_4(hsotg, GRXSTSP);
181 	chnum = (grxsts & GRXSTS_HCHNUM_MASK) >> GRXSTS_HCHNUM_SHIFT;
182 	chan = hsotg->hc_ptr_array[chnum];
183 	if (!chan) {
184 		dev_err(hsotg->dev, "Unable to get corresponding channel\n");
185 		return;
186 	}
187 
188 	bcnt = (grxsts & GRXSTS_BYTECNT_MASK) >> GRXSTS_BYTECNT_SHIFT;
189 	pktsts = (grxsts & GRXSTS_PKTSTS_MASK) >> GRXSTS_PKTSTS_SHIFT;
190 
191 	/* Packet Status */
192 	if (dbg_perio()) {
193 		dev_vdbg(hsotg->dev, "    Ch num = %d\n", chnum);
194 		dev_vdbg(hsotg->dev, "    Count = %d\n", bcnt);
195 		dev_vdbg(hsotg->dev, "    DPID = %d, chan.dpid = %d\n",
196 			 (grxsts & GRXSTS_DPID_MASK) >> GRXSTS_DPID_SHIFT,
197 			 chan->data_pid_start);
198 		dev_vdbg(hsotg->dev, "    PStatus = %d\n", pktsts);
199 	}
200 
201 	switch (pktsts) {
202 	case GRXSTS_PKTSTS_HCHIN:
203 		/* Read the data into the host buffer */
204 		if (bcnt > 0) {
205 			dwc2_read_packet(hsotg, chan->xfer_buf, bcnt);
206 
207 			/* Update the HC fields for the next packet received */
208 			chan->xfer_count += bcnt;
209 			chan->xfer_buf += bcnt;
210 		}
211 		break;
212 	case GRXSTS_PKTSTS_HCHIN_XFER_COMP:
213 	case GRXSTS_PKTSTS_DATATOGGLEERR:
214 	case GRXSTS_PKTSTS_HCHHALTED:
215 		/* Handled in interrupt, just ignore data */
216 		break;
217 	default:
218 		dev_err(hsotg->dev,
219 			"RxFIFO Level Interrupt: Unknown status %d\n", pktsts);
220 		break;
221 	}
222 }
223 
224 /*
225  * This interrupt occurs when the non-periodic Tx FIFO is half-empty. More
226  * data packets may be written to the FIFO for OUT transfers. More requests
227  * may be written to the non-periodic request queue for IN transfers. This
228  * interrupt is enabled only in Slave mode.
229  */
dwc2_np_tx_fifo_empty_intr(struct dwc2_hsotg * hsotg)230 static void dwc2_np_tx_fifo_empty_intr(struct dwc2_hsotg *hsotg)
231 {
232 	dev_vdbg(hsotg->dev, "--Non-Periodic TxFIFO Empty Interrupt--\n");
233 	dwc2_hcd_queue_transactions(hsotg, DWC2_TRANSACTION_NON_PERIODIC);
234 }
235 
236 /*
237  * This interrupt occurs when the periodic Tx FIFO is half-empty. More data
238  * packets may be written to the FIFO for OUT transfers. More requests may be
239  * written to the periodic request queue for IN transfers. This interrupt is
240  * enabled only in Slave mode.
241  */
dwc2_perio_tx_fifo_empty_intr(struct dwc2_hsotg * hsotg)242 static void dwc2_perio_tx_fifo_empty_intr(struct dwc2_hsotg *hsotg)
243 {
244 	if (dbg_perio())
245 		dev_vdbg(hsotg->dev, "--Periodic TxFIFO Empty Interrupt--\n");
246 	dwc2_hcd_queue_transactions(hsotg, DWC2_TRANSACTION_PERIODIC);
247 }
248 
dwc2_hprt0_enable(struct dwc2_hsotg * hsotg,u32 hprt0,u32 * hprt0_modify)249 static void dwc2_hprt0_enable(struct dwc2_hsotg *hsotg, u32 hprt0,
250 			      u32 *hprt0_modify)
251 {
252 	struct dwc2_core_params *params = hsotg->core_params;
253 	int do_reset = 0;
254 	u32 usbcfg;
255 	u32 prtspd;
256 	u32 hcfg;
257 	u32 fslspclksel;
258 	u32 hfir;
259 
260 	dev_vdbg(hsotg->dev, "%s(%p)\n", __func__, hsotg);
261 
262 	/* Every time when port enables calculate HFIR.FrInterval */
263 	hfir = DWC2_READ_4(hsotg, HFIR);
264 	hfir &= ~HFIR_FRINT_MASK;
265 	hfir |= dwc2_calc_frame_interval(hsotg) << HFIR_FRINT_SHIFT &
266 		HFIR_FRINT_MASK;
267 	DWC2_WRITE_4(hsotg, HFIR, hfir);
268 
269 	/* Check if we need to adjust the PHY clock speed for low power */
270 	if (!params->host_support_fs_ls_low_power) {
271 		/* Port has been enabled, set the reset change flag */
272 		hsotg->flags.b.port_reset_change = 1;
273 
274 		dwc2_root_intr(hsotg->hsotg_sc);
275 		return;
276 	}
277 
278 	usbcfg = DWC2_READ_4(hsotg, GUSBCFG);
279 	prtspd = (hprt0 & HPRT0_SPD_MASK) >> HPRT0_SPD_SHIFT;
280 
281 	if (prtspd == HPRT0_SPD_LOW_SPEED || prtspd == HPRT0_SPD_FULL_SPEED) {
282 		/* Low power */
283 		if (!(usbcfg & GUSBCFG_PHY_LP_CLK_SEL)) {
284 			/* Set PHY low power clock select for FS/LS devices */
285 			usbcfg |= GUSBCFG_PHY_LP_CLK_SEL;
286 			DWC2_WRITE_4(hsotg, GUSBCFG, usbcfg);
287 			do_reset = 1;
288 		}
289 
290 		hcfg = DWC2_READ_4(hsotg, HCFG);
291 		fslspclksel = (hcfg & HCFG_FSLSPCLKSEL_MASK) >>
292 			      HCFG_FSLSPCLKSEL_SHIFT;
293 
294 		if (prtspd == HPRT0_SPD_LOW_SPEED &&
295 		    params->host_ls_low_power_phy_clk ==
296 		    DWC2_HOST_LS_LOW_POWER_PHY_CLK_PARAM_6MHZ) {
297 			/* 6 MHZ */
298 			dev_vdbg(hsotg->dev,
299 				 "FS_PHY programming HCFG to 6 MHz\n");
300 			if (fslspclksel != HCFG_FSLSPCLKSEL_6_MHZ) {
301 				fslspclksel = HCFG_FSLSPCLKSEL_6_MHZ;
302 				hcfg &= ~HCFG_FSLSPCLKSEL_MASK;
303 				hcfg |= fslspclksel << HCFG_FSLSPCLKSEL_SHIFT;
304 				DWC2_WRITE_4(hsotg, HCFG, hcfg);
305 				do_reset = 1;
306 			}
307 		} else {
308 			/* 48 MHZ */
309 			dev_vdbg(hsotg->dev,
310 				 "FS_PHY programming HCFG to 48 MHz\n");
311 			if (fslspclksel != HCFG_FSLSPCLKSEL_48_MHZ) {
312 				fslspclksel = HCFG_FSLSPCLKSEL_48_MHZ;
313 				hcfg &= ~HCFG_FSLSPCLKSEL_MASK;
314 				hcfg |= fslspclksel << HCFG_FSLSPCLKSEL_SHIFT;
315 				DWC2_WRITE_4(hsotg, HCFG, hcfg);
316 				do_reset = 1;
317 			}
318 		}
319 	} else {
320 		/* Not low power */
321 		if (usbcfg & GUSBCFG_PHY_LP_CLK_SEL) {
322 			usbcfg &= ~GUSBCFG_PHY_LP_CLK_SEL;
323 			DWC2_WRITE_4(hsotg, GUSBCFG, usbcfg);
324 			do_reset = 1;
325 		}
326 	}
327 
328 	if (do_reset) {
329 		*hprt0_modify |= HPRT0_RST;
330 		DWC2_WRITE_4(hsotg, HPRT0, *hprt0_modify);
331 		queue_delayed_work(hsotg->wq_otg, &hsotg->reset_work,
332 				   msecs_to_jiffies(60));
333 	} else {
334 		/* Port has been enabled, set the reset change flag */
335 		hsotg->flags.b.port_reset_change = 1;
336 		dwc2_root_intr(hsotg->hsotg_sc);
337 
338 	}
339 }
340 
341 /*
342  * There are multiple conditions that can cause a port interrupt. This function
343  * determines which interrupt conditions have occurred and handles them
344  * appropriately.
345  */
dwc2_port_intr(struct dwc2_hsotg * hsotg)346 static void dwc2_port_intr(struct dwc2_hsotg *hsotg)
347 {
348 	u32 hprt0;
349 	u32 hprt0_modify;
350 
351 	dev_vdbg(hsotg->dev, "--Port Interrupt--\n");
352 
353 	hprt0 = DWC2_READ_4(hsotg, HPRT0);
354 	hprt0_modify = hprt0;
355 
356 	/*
357 	 * Clear appropriate bits in HPRT0 to clear the interrupt bit in
358 	 * GINTSTS
359 	 */
360 	hprt0_modify &= ~(HPRT0_ENA | HPRT0_CONNDET | HPRT0_ENACHG |
361 			  HPRT0_OVRCURRCHG);
362 
363 	/*
364 	 * Port Connect Detected
365 	 * Set flag and clear if detected
366 	 */
367 	if (hprt0 & HPRT0_CONNDET) {
368 		DWC2_WRITE_4(hsotg, HPRT0, hprt0_modify | HPRT0_CONNDET);
369 
370 		dev_vdbg(hsotg->dev,
371 			 "--Port Interrupt HPRT0=0x%08x Port Connect Detected--\n",
372 			 hprt0);
373 		dwc2_hcd_connect(hsotg);
374 
375 		/*
376 		 * The Hub driver asserts a reset when it sees port connect
377 		 * status change flag
378 		 */
379 	}
380 
381 	/*
382 	 * Port Enable Changed
383 	 * Clear if detected - Set internal flag if disabled
384 	 */
385 	if (hprt0 & HPRT0_ENACHG) {
386 		DWC2_WRITE_4(hsotg, HPRT0, hprt0_modify | HPRT0_ENACHG);
387 		dev_vdbg(hsotg->dev,
388 			 "  --Port Interrupt HPRT0=0x%08x Port Enable Changed (now %d)--\n",
389 			 hprt0, !!(hprt0 & HPRT0_ENA));
390 		if (hprt0 & HPRT0_ENA) {
391 			hsotg->new_connection = true;
392 			dwc2_hprt0_enable(hsotg, hprt0, &hprt0_modify);
393 		} else {
394 			hsotg->flags.b.port_enable_change = 1;
395 			if (hsotg->core_params->dma_desc_fs_enable) {
396 				u32 hcfg;
397 
398 				hsotg->core_params->dma_desc_enable = 0;
399 				hsotg->new_connection = false;
400 				hcfg = DWC2_READ_4(hsotg, HCFG);
401 				hcfg &= ~HCFG_DESCDMA;
402 				DWC2_WRITE_4(hsotg, HCFG, hcfg);
403 			}
404 		}
405 	}
406 
407 	/* Overcurrent Change Interrupt */
408 	if (hprt0 & HPRT0_OVRCURRCHG) {
409 		DWC2_WRITE_4(hsotg, HPRT0, hprt0_modify | HPRT0_OVRCURRCHG);
410 		dev_vdbg(hsotg->dev,
411 			 "  --Port Interrupt HPRT0=0x%08x Port Overcurrent Changed--\n",
412 			 hprt0);
413 		hsotg->flags.b.port_over_current_change = 1;
414 	}
415 
416 	if (hsotg->flags.b.port_connect_status_change ||
417 	    hsotg->flags.b.port_enable_change ||
418 	    hsotg->flags.b.port_over_current_change)
419 		dwc2_root_intr(hsotg->hsotg_sc);
420 }
421 
422 /*
423  * Gets the actual length of a transfer after the transfer halts. halt_status
424  * holds the reason for the halt.
425  *
426  * For IN transfers where halt_status is DWC2_HC_XFER_COMPLETE, *short_read
427  * is set to 1 upon return if less than the requested number of bytes were
428  * transferred. short_read may also be NULL on entry, in which case it remains
429  * unchanged.
430  */
dwc2_get_actual_xfer_length(struct dwc2_hsotg * hsotg,struct dwc2_host_chan * chan,int chnum,struct dwc2_qtd * qtd,enum dwc2_halt_status halt_status,int * short_read)431 static u32 dwc2_get_actual_xfer_length(struct dwc2_hsotg *hsotg,
432 				       struct dwc2_host_chan *chan, int chnum,
433 				       struct dwc2_qtd *qtd,
434 				       enum dwc2_halt_status halt_status,
435 				       int *short_read)
436 {
437 	u32 hctsiz, count, length;
438 
439 	hctsiz = DWC2_READ_4(hsotg, HCTSIZ(chnum));
440 
441 	if (halt_status == DWC2_HC_XFER_COMPLETE) {
442 		if (chan->ep_is_in) {
443 			count = (hctsiz & TSIZ_XFERSIZE_MASK) >>
444 				TSIZ_XFERSIZE_SHIFT;
445 			length = chan->xfer_len - count;
446 			if (short_read != NULL)
447 				*short_read = (count != 0);
448 		} else if (chan->qh->do_split) {
449 			length = qtd->ssplit_out_xfer_count;
450 		} else {
451 			length = chan->xfer_len;
452 		}
453 	} else {
454 		/*
455 		 * Must use the hctsiz.pktcnt field to determine how much data
456 		 * has been transferred. This field reflects the number of
457 		 * packets that have been transferred via the USB. This is
458 		 * always an integral number of packets if the transfer was
459 		 * halted before its normal completion. (Can't use the
460 		 * hctsiz.xfersize field because that reflects the number of
461 		 * bytes transferred via the AHB, not the USB).
462 		 */
463 		count = (hctsiz & TSIZ_PKTCNT_MASK) >> TSIZ_PKTCNT_SHIFT;
464 		length = (chan->start_pkt_count - count) * chan->max_packet;
465 	}
466 
467 	return length;
468 }
469 
470 /**
471  * dwc2_update_urb_state() - Updates the state of the URB after a Transfer
472  * Complete interrupt on the host channel. Updates the actual_length field
473  * of the URB based on the number of bytes transferred via the host channel.
474  * Sets the URB status if the data transfer is finished.
475  *
476  * Return: 1 if the data transfer specified by the URB is completely finished,
477  * 0 otherwise
478  */
dwc2_update_urb_state(struct dwc2_hsotg * hsotg,struct dwc2_host_chan * chan,int chnum,struct dwc2_hcd_urb * urb,struct dwc2_qtd * qtd)479 static int dwc2_update_urb_state(struct dwc2_hsotg *hsotg,
480 				 struct dwc2_host_chan *chan, int chnum,
481 				 struct dwc2_hcd_urb *urb,
482 				 struct dwc2_qtd *qtd)
483 {
484 	int xfer_done = 0;
485 	int short_read = 0;
486 	int xfer_length = dwc2_get_actual_xfer_length(hsotg, chan, chnum, qtd,
487 						      DWC2_HC_XFER_COMPLETE,
488 						      &short_read);
489 
490 	if (urb->actual_length + xfer_length > urb->length) {
491 		dev_warn(hsotg->dev, "%s(): trimming xfer length\n", __func__);
492 		xfer_length = urb->length - urb->actual_length;
493 	}
494 
495 	/* Non DWORD-aligned buffer case handling */
496 	if (chan->align_buf && xfer_length) {
497 		dev_vdbg(hsotg->dev, "%s(): non-aligned buffer\n", __func__);
498 		usb_syncmem(urb->usbdma, 0, chan->qh->dw_align_buf_size,
499 		    chan->ep_is_in ?
500 		    BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
501 		if (chan->ep_is_in)
502 			memcpy(urb->buf + urb->actual_length,
503 					chan->qh->dw_align_buf, xfer_length);
504 		usb_syncmem(urb->usbdma, 0, chan->qh->dw_align_buf_size,
505 		    chan->ep_is_in ?
506 		    BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE);
507 	}
508 
509 	dev_vdbg(hsotg->dev, "urb->actual_length=%d xfer_length=%d\n",
510 		 urb->actual_length, xfer_length);
511 	urb->actual_length += xfer_length;
512 
513 	if (xfer_length && chan->ep_type == USB_ENDPOINT_XFER_BULK &&
514 	    (urb->flags & URB_SEND_ZERO_PACKET) &&
515 	    urb->actual_length >= urb->length &&
516 	    !(urb->length % chan->max_packet)) {
517 		xfer_done = 0;
518 	} else if (short_read || urb->actual_length >= urb->length) {
519 		xfer_done = 1;
520 		urb->status = 0;
521 	}
522 
523 	dev_vdbg(hsotg->dev, "DWC_otg: %s: %s, channel %d\n",
524 		 __func__, (chan->ep_is_in ? "IN" : "OUT"), chnum);
525 	dev_vdbg(hsotg->dev, "  chan->xfer_len %d\n", chan->xfer_len);
526 	dev_vdbg(hsotg->dev, "  hctsiz.xfersize %d\n",
527 		 (DWC2_READ_4(hsotg, HCTSIZ(chnum)) & TSIZ_XFERSIZE_MASK) >> TSIZ_XFERSIZE_SHIFT);
528 	dev_vdbg(hsotg->dev, "  urb->transfer_buffer_length %d\n", urb->length);
529 	dev_vdbg(hsotg->dev, "  urb->actual_length %d\n", urb->actual_length);
530 	dev_vdbg(hsotg->dev, "  short_read %d, xfer_done %d\n", short_read,
531 		 xfer_done);
532 
533 	return xfer_done;
534 }
535 
536 /*
537  * Save the starting data toggle for the next transfer. The data toggle is
538  * saved in the QH for non-control transfers and it's saved in the QTD for
539  * control transfers.
540  */
dwc2_hcd_save_data_toggle(struct dwc2_hsotg * hsotg,struct dwc2_host_chan * chan,int chnum,struct dwc2_qtd * qtd)541 void dwc2_hcd_save_data_toggle(struct dwc2_hsotg *hsotg,
542 			       struct dwc2_host_chan *chan, int chnum,
543 			       struct dwc2_qtd *qtd)
544 {
545 	u32 hctsiz = DWC2_READ_4(hsotg, HCTSIZ(chnum));
546 	u32 pid = (hctsiz & TSIZ_SC_MC_PID_MASK) >> TSIZ_SC_MC_PID_SHIFT;
547 
548 	if (chan->ep_type != USB_ENDPOINT_XFER_CONTROL) {
549 		if (pid == TSIZ_SC_MC_PID_DATA0)
550 			chan->qh->data_toggle = DWC2_HC_PID_DATA0;
551 		else
552 			chan->qh->data_toggle = DWC2_HC_PID_DATA1;
553 	} else {
554 		if (pid == TSIZ_SC_MC_PID_DATA0)
555 			qtd->data_toggle = DWC2_HC_PID_DATA0;
556 		else
557 			qtd->data_toggle = DWC2_HC_PID_DATA1;
558 	}
559 }
560 
561 /**
562  * dwc2_update_isoc_urb_state() - Updates the state of an Isochronous URB when
563  * the transfer is stopped for any reason. The fields of the current entry in
564  * the frame descriptor array are set based on the transfer state and the input
565  * halt_status. Completes the Isochronous URB if all the URB frames have been
566  * completed.
567  *
568  * Return: DWC2_HC_XFER_COMPLETE if there are more frames remaining to be
569  * transferred in the URB. Otherwise return DWC2_HC_XFER_URB_COMPLETE.
570  */
dwc2_update_isoc_urb_state(struct dwc2_hsotg * hsotg,struct dwc2_host_chan * chan,int chnum,struct dwc2_qtd * qtd,enum dwc2_halt_status halt_status)571 static enum dwc2_halt_status dwc2_update_isoc_urb_state(
572 		struct dwc2_hsotg *hsotg, struct dwc2_host_chan *chan,
573 		int chnum, struct dwc2_qtd *qtd,
574 		enum dwc2_halt_status halt_status)
575 {
576 	struct dwc2_hcd_iso_packet_desc *frame_desc;
577 	struct dwc2_hcd_urb *urb = qtd->urb;
578 
579 	if (!urb)
580 		return DWC2_HC_XFER_NO_HALT_STATUS;
581 
582 	frame_desc = &urb->iso_descs[qtd->isoc_frame_index];
583 
584 	switch (halt_status) {
585 	case DWC2_HC_XFER_COMPLETE:
586 		frame_desc->status = 0;
587 		frame_desc->actual_length = dwc2_get_actual_xfer_length(hsotg,
588 					chan, chnum, qtd, halt_status, NULL);
589 
590 		/* Non DWORD-aligned buffer case handling */
591 		if (chan->align_buf && frame_desc->actual_length) {
592 			dev_vdbg(hsotg->dev, "%s(): non-aligned buffer\n",
593 				 __func__);
594 			usb_dma_t *ud = &chan->qh->dw_align_buf_usbdma;
595 
596 			usb_syncmem(ud, 0, chan->qh->dw_align_buf_size,
597 			    chan->ep_is_in ?
598 			    BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
599 			if (chan->ep_is_in)
600 				memcpy(urb->buf + frame_desc->offset +
601 					qtd->isoc_split_offset,
602 					chan->qh->dw_align_buf,
603 					frame_desc->actual_length);
604 			usb_syncmem(ud, 0, chan->qh->dw_align_buf_size,
605 			    chan->ep_is_in ?
606 			    BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE);
607 		}
608 		break;
609 	case DWC2_HC_XFER_FRAME_OVERRUN:
610 		urb->error_count++;
611 		if (chan->ep_is_in)
612 			frame_desc->status = -ENOSR;
613 		else
614 			frame_desc->status = -ECOMM;
615 		frame_desc->actual_length = 0;
616 		break;
617 	case DWC2_HC_XFER_BABBLE_ERR:
618 		urb->error_count++;
619 		frame_desc->status = -EOVERFLOW;
620 		/* Don't need to update actual_length in this case */
621 		break;
622 	case DWC2_HC_XFER_XACT_ERR:
623 		urb->error_count++;
624 		frame_desc->status = -EPROTO;
625 		frame_desc->actual_length = dwc2_get_actual_xfer_length(hsotg,
626 					chan, chnum, qtd, halt_status, NULL);
627 
628 		/* Non DWORD-aligned buffer case handling */
629 		if (chan->align_buf && frame_desc->actual_length) {
630 			dev_vdbg(hsotg->dev, "%s(): non-aligned buffer\n",
631 				 __func__);
632 			usb_dma_t *ud = &chan->qh->dw_align_buf_usbdma;
633 
634 			usb_syncmem(ud, 0, chan->qh->dw_align_buf_size,
635 			    chan->ep_is_in ?
636 			    BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
637 			if (chan->ep_is_in)
638 				memcpy(urb->buf + frame_desc->offset +
639 					qtd->isoc_split_offset,
640 					chan->qh->dw_align_buf,
641 					frame_desc->actual_length);
642 			usb_syncmem(ud, 0, chan->qh->dw_align_buf_size,
643 			    chan->ep_is_in ?
644 			    BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE);
645 		}
646 
647 		/* Skip whole frame */
648 		if (chan->qh->do_split &&
649 		    chan->ep_type == USB_ENDPOINT_XFER_ISOC && chan->ep_is_in &&
650 		    hsotg->core_params->dma_enable > 0) {
651 			qtd->complete_split = 0;
652 			qtd->isoc_split_offset = 0;
653 		}
654 
655 		break;
656 	default:
657 		dev_err(hsotg->dev, "Unhandled halt_status (%d)\n",
658 			halt_status);
659 		break;
660 	}
661 
662 	if (++qtd->isoc_frame_index == urb->packet_count) {
663 		/*
664 		 * urb->status is not used for isoc transfers. The individual
665 		 * frame_desc statuses are used instead.
666 		 */
667 		dwc2_host_complete(hsotg, qtd, 0);
668 		halt_status = DWC2_HC_XFER_URB_COMPLETE;
669 	} else {
670 		halt_status = DWC2_HC_XFER_COMPLETE;
671 	}
672 
673 	return halt_status;
674 }
675 
676 /*
677  * Frees the first QTD in the QH's list if free_qtd is 1. For non-periodic
678  * QHs, removes the QH from the active non-periodic schedule. If any QTDs are
679  * still linked to the QH, the QH is added to the end of the inactive
680  * non-periodic schedule. For periodic QHs, removes the QH from the periodic
681  * schedule if no more QTDs are linked to the QH.
682  */
dwc2_deactivate_qh(struct dwc2_hsotg * hsotg,struct dwc2_qh * qh,int free_qtd)683 static void dwc2_deactivate_qh(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh,
684 			       int free_qtd)
685 {
686 	int continue_split = 0;
687 	struct dwc2_qtd *qtd;
688 
689 	if (dbg_qh(qh))
690 		dev_vdbg(hsotg->dev, "  %s(%p,%p,%d)\n", __func__,
691 			 hsotg, qh, free_qtd);
692 
693 	if (list_empty(&qh->qtd_list)) {
694 		dev_dbg(hsotg->dev, "## QTD list empty ##\n");
695 		goto no_qtd;
696 	}
697 
698 	qtd = list_first_entry(&qh->qtd_list, struct dwc2_qtd, qtd_list_entry);
699 
700 	if (qtd->complete_split)
701 		continue_split = 1;
702 	else if (qtd->isoc_split_pos == DWC2_HCSPLT_XACTPOS_MID ||
703 		 qtd->isoc_split_pos == DWC2_HCSPLT_XACTPOS_END)
704 		continue_split = 1;
705 
706 	if (free_qtd) {
707 		dwc2_hcd_qtd_unlink_and_free(hsotg, qtd, qh);
708 		continue_split = 0;
709 	}
710 
711 no_qtd:
712 	if (qh->channel)
713 		qh->channel->align_buf = 0;
714 	qh->channel = NULL;
715 	dwc2_hcd_qh_deactivate(hsotg, qh, continue_split);
716 }
717 
718 /**
719  * dwc2_release_channel() - Releases a host channel for use by other transfers
720  *
721  * @hsotg:       The HCD state structure
722  * @chan:        The host channel to release
723  * @qtd:         The QTD associated with the host channel. This QTD may be
724  *               freed if the transfer is complete or an error has occurred.
725  * @halt_status: Reason the channel is being released. This status
726  *               determines the actions taken by this function.
727  *
728  * Also attempts to select and queue more transactions since at least one host
729  * channel is available.
730  */
dwc2_release_channel(struct dwc2_hsotg * hsotg,struct dwc2_host_chan * chan,struct dwc2_qtd * qtd,enum dwc2_halt_status halt_status)731 static void dwc2_release_channel(struct dwc2_hsotg *hsotg,
732 				 struct dwc2_host_chan *chan,
733 				 struct dwc2_qtd *qtd,
734 				 enum dwc2_halt_status halt_status)
735 {
736 	enum dwc2_transaction_type tr_type;
737 	u32 haintmsk;
738 	int free_qtd = 0;
739 
740 	if (dbg_hc(chan))
741 		dev_vdbg(hsotg->dev, "  %s: channel %d, halt_status %d\n",
742 			 __func__, chan->hc_num, halt_status);
743 
744 	switch (halt_status) {
745 	case DWC2_HC_XFER_URB_COMPLETE:
746 		free_qtd = 1;
747 		break;
748 	case DWC2_HC_XFER_AHB_ERR:
749 	case DWC2_HC_XFER_STALL:
750 	case DWC2_HC_XFER_BABBLE_ERR:
751 		free_qtd = 1;
752 		break;
753 	case DWC2_HC_XFER_XACT_ERR:
754 		if (qtd && qtd->error_count >= 3) {
755 			dev_vdbg(hsotg->dev,
756 				 "  Complete URB with transaction error\n");
757 			free_qtd = 1;
758 			dwc2_host_complete(hsotg, qtd, -EPROTO);
759 		}
760 		break;
761 	case DWC2_HC_XFER_URB_DEQUEUE:
762 		/*
763 		 * The QTD has already been removed and the QH has been
764 		 * deactivated. Don't want to do anything except release the
765 		 * host channel and try to queue more transfers.
766 		 */
767 		goto cleanup;
768 	case DWC2_HC_XFER_PERIODIC_INCOMPLETE:
769 		dev_vdbg(hsotg->dev, "  Complete URB with I/O error\n");
770 		free_qtd = 1;
771 		dwc2_host_complete(hsotg, qtd, -EIO);
772 		break;
773 	case DWC2_HC_XFER_NO_HALT_STATUS:
774 	default:
775 		break;
776 	}
777 
778 	dwc2_deactivate_qh(hsotg, chan->qh, free_qtd);
779 
780 cleanup:
781 	/*
782 	 * Release the host channel for use by other transfers. The cleanup
783 	 * function clears the channel interrupt enables and conditions, so
784 	 * there's no need to clear the Channel Halted interrupt separately.
785 	 */
786 	if (!list_empty(&chan->hc_list_entry))
787 		list_del(&chan->hc_list_entry);
788 	dwc2_hc_cleanup(hsotg, chan);
789 	list_add_tail(&chan->hc_list_entry, &hsotg->free_hc_list);
790 
791 	if (hsotg->core_params->uframe_sched > 0) {
792 		hsotg->available_host_channels++;
793 	} else {
794 		switch (chan->ep_type) {
795 		case USB_ENDPOINT_XFER_CONTROL:
796 		case USB_ENDPOINT_XFER_BULK:
797 			hsotg->non_periodic_channels--;
798 			break;
799 		default:
800 			/*
801 			 * Don't release reservations for periodic channels
802 			 * here. That's done when a periodic transfer is
803 			 * descheduled (i.e. when the QH is removed from the
804 			 * periodic schedule).
805 			 */
806 			break;
807 		}
808 	}
809 
810 	haintmsk = DWC2_READ_4(hsotg, HAINTMSK);
811 	haintmsk &= ~(1 << chan->hc_num);
812 	DWC2_WRITE_4(hsotg, HAINTMSK, haintmsk);
813 
814 	/* Try to queue more transfers now that there's a free channel */
815 	tr_type = dwc2_hcd_select_transactions(hsotg);
816 	if (tr_type != DWC2_TRANSACTION_NONE)
817 		dwc2_hcd_queue_transactions(hsotg, tr_type);
818 }
819 
820 /*
821  * Halts a host channel. If the channel cannot be halted immediately because
822  * the request queue is full, this function ensures that the FIFO empty
823  * interrupt for the appropriate queue is enabled so that the halt request can
824  * be queued when there is space in the request queue.
825  *
826  * This function may also be called in DMA mode. In that case, the channel is
827  * simply released since the core always halts the channel automatically in
828  * DMA mode.
829  */
dwc2_halt_channel(struct dwc2_hsotg * hsotg,struct dwc2_host_chan * chan,struct dwc2_qtd * qtd,enum dwc2_halt_status halt_status)830 static void dwc2_halt_channel(struct dwc2_hsotg *hsotg,
831 			      struct dwc2_host_chan *chan, struct dwc2_qtd *qtd,
832 			      enum dwc2_halt_status halt_status)
833 {
834 	if (dbg_hc(chan))
835 		dev_vdbg(hsotg->dev, "%s()\n", __func__);
836 
837 	if (hsotg->core_params->dma_enable > 0) {
838 		if (dbg_hc(chan))
839 			dev_vdbg(hsotg->dev, "DMA enabled\n");
840 		dwc2_release_channel(hsotg, chan, qtd, halt_status);
841 		return;
842 	}
843 
844 	/* Slave mode processing */
845 	dwc2_hc_halt(hsotg, chan, halt_status);
846 
847 	if (chan->halt_on_queue) {
848 		u32 gintmsk;
849 
850 		dev_vdbg(hsotg->dev, "Halt on queue\n");
851 		if (chan->ep_type == USB_ENDPOINT_XFER_CONTROL ||
852 		    chan->ep_type == USB_ENDPOINT_XFER_BULK) {
853 			dev_vdbg(hsotg->dev, "control/bulk\n");
854 			/*
855 			 * Make sure the Non-periodic Tx FIFO empty interrupt
856 			 * is enabled so that the non-periodic schedule will
857 			 * be processed
858 			 */
859 			gintmsk = DWC2_READ_4(hsotg, GINTMSK);
860 			gintmsk |= GINTSTS_NPTXFEMP;
861 			DWC2_WRITE_4(hsotg, GINTMSK, gintmsk);
862 		} else {
863 			dev_vdbg(hsotg->dev, "isoc/intr\n");
864 			/*
865 			 * Move the QH from the periodic queued schedule to
866 			 * the periodic assigned schedule. This allows the
867 			 * halt to be queued when the periodic schedule is
868 			 * processed.
869 			 */
870 			list_move(&chan->qh->qh_list_entry,
871 				  &hsotg->periodic_sched_assigned);
872 
873 			/*
874 			 * Make sure the Periodic Tx FIFO Empty interrupt is
875 			 * enabled so that the periodic schedule will be
876 			 * processed
877 			 */
878 			gintmsk = DWC2_READ_4(hsotg, GINTMSK);
879 			gintmsk |= GINTSTS_PTXFEMP;
880 			DWC2_WRITE_4(hsotg, GINTMSK, gintmsk);
881 		}
882 	}
883 }
884 
885 /*
886  * Performs common cleanup for non-periodic transfers after a Transfer
887  * Complete interrupt. This function should be called after any endpoint type
888  * specific handling is finished to release the host channel.
889  */
dwc2_complete_non_periodic_xfer(struct dwc2_hsotg * hsotg,struct dwc2_host_chan * chan,int chnum,struct dwc2_qtd * qtd,enum dwc2_halt_status halt_status)890 static void dwc2_complete_non_periodic_xfer(struct dwc2_hsotg *hsotg,
891 					    struct dwc2_host_chan *chan,
892 					    int chnum, struct dwc2_qtd *qtd,
893 					    enum dwc2_halt_status halt_status)
894 {
895 	dev_vdbg(hsotg->dev, "%s()\n", __func__);
896 
897 	qtd->error_count = 0;
898 
899 	if (chan->hcint & HCINTMSK_NYET) {
900 		/*
901 		 * Got a NYET on the last transaction of the transfer. This
902 		 * means that the endpoint should be in the PING state at the
903 		 * beginning of the next transfer.
904 		 */
905 		dev_vdbg(hsotg->dev, "got NYET\n");
906 		chan->qh->ping_state = 1;
907 	}
908 
909 	/*
910 	 * Always halt and release the host channel to make it available for
911 	 * more transfers. There may still be more phases for a control
912 	 * transfer or more data packets for a bulk transfer at this point,
913 	 * but the host channel is still halted. A channel will be reassigned
914 	 * to the transfer when the non-periodic schedule is processed after
915 	 * the channel is released. This allows transactions to be queued
916 	 * properly via dwc2_hcd_queue_transactions, which also enables the
917 	 * Tx FIFO Empty interrupt if necessary.
918 	 */
919 	if (chan->ep_is_in) {
920 		/*
921 		 * IN transfers in Slave mode require an explicit disable to
922 		 * halt the channel. (In DMA mode, this call simply releases
923 		 * the channel.)
924 		 */
925 		dwc2_halt_channel(hsotg, chan, qtd, halt_status);
926 	} else {
927 		/*
928 		 * The channel is automatically disabled by the core for OUT
929 		 * transfers in Slave mode
930 		 */
931 		dwc2_release_channel(hsotg, chan, qtd, halt_status);
932 	}
933 }
934 
935 /*
936  * Performs common cleanup for periodic transfers after a Transfer Complete
937  * interrupt. This function should be called after any endpoint type specific
938  * handling is finished to release the host channel.
939  */
dwc2_complete_periodic_xfer(struct dwc2_hsotg * hsotg,struct dwc2_host_chan * chan,int chnum,struct dwc2_qtd * qtd,enum dwc2_halt_status halt_status)940 static void dwc2_complete_periodic_xfer(struct dwc2_hsotg *hsotg,
941 					struct dwc2_host_chan *chan, int chnum,
942 					struct dwc2_qtd *qtd,
943 					enum dwc2_halt_status halt_status)
944 {
945 	u32 hctsiz = DWC2_READ_4(hsotg, HCTSIZ(chnum));
946 
947 	qtd->error_count = 0;
948 
949 	if (!chan->ep_is_in || (hctsiz & TSIZ_PKTCNT_MASK) == 0)
950 		/* Core halts channel in these cases */
951 		dwc2_release_channel(hsotg, chan, qtd, halt_status);
952 	else
953 		/* Flush any outstanding requests from the Tx queue */
954 		dwc2_halt_channel(hsotg, chan, qtd, halt_status);
955 }
956 
dwc2_xfercomp_isoc_split_in(struct dwc2_hsotg * hsotg,struct dwc2_host_chan * chan,int chnum,struct dwc2_qtd * qtd)957 static int dwc2_xfercomp_isoc_split_in(struct dwc2_hsotg *hsotg,
958 				       struct dwc2_host_chan *chan, int chnum,
959 				       struct dwc2_qtd *qtd)
960 {
961 	struct dwc2_hcd_iso_packet_desc *frame_desc;
962 	u32 len;
963 
964 	if (!qtd->urb)
965 		return 0;
966 
967 	frame_desc = &qtd->urb->iso_descs[qtd->isoc_frame_index];
968 	len = dwc2_get_actual_xfer_length(hsotg, chan, chnum, qtd,
969 					  DWC2_HC_XFER_COMPLETE, NULL);
970 	if (!len) {
971 		qtd->complete_split = 0;
972 		qtd->isoc_split_offset = 0;
973 		return 0;
974 	}
975 
976 	frame_desc->actual_length += len;
977 
978 	if (chan->align_buf) {
979 		dev_vdbg(hsotg->dev, "%s(): non-aligned buffer\n", __func__);
980 		usb_syncmem(qtd->urb->usbdma, chan->qh->dw_align_buf_dma,
981 		    chan->qh->dw_align_buf_size, BUS_DMASYNC_POSTREAD);
982 		memcpy(qtd->urb->buf + frame_desc->offset +
983 		       qtd->isoc_split_offset, chan->qh->dw_align_buf, len);
984 		usb_syncmem(qtd->urb->usbdma, chan->qh->dw_align_buf_dma,
985 		    chan->qh->dw_align_buf_size, BUS_DMASYNC_PREREAD);
986 	}
987 
988 	qtd->isoc_split_offset += len;
989 
990 	if (frame_desc->actual_length >= frame_desc->length) {
991 		frame_desc->status = 0;
992 		qtd->isoc_frame_index++;
993 		qtd->complete_split = 0;
994 		qtd->isoc_split_offset = 0;
995 	}
996 
997 	if (qtd->isoc_frame_index == qtd->urb->packet_count) {
998 		dwc2_host_complete(hsotg, qtd, 0);
999 		dwc2_release_channel(hsotg, chan, qtd,
1000 				     DWC2_HC_XFER_URB_COMPLETE);
1001 	} else {
1002 		dwc2_release_channel(hsotg, chan, qtd,
1003 				     DWC2_HC_XFER_NO_HALT_STATUS);
1004 	}
1005 
1006 	return 1;	/* Indicates that channel released */
1007 }
1008 
1009 /*
1010  * Handles a host channel Transfer Complete interrupt. This handler may be
1011  * called in either DMA mode or Slave mode.
1012  */
dwc2_hc_xfercomp_intr(struct dwc2_hsotg * hsotg,struct dwc2_host_chan * chan,int chnum,struct dwc2_qtd * qtd)1013 static void dwc2_hc_xfercomp_intr(struct dwc2_hsotg *hsotg,
1014 				  struct dwc2_host_chan *chan, int chnum,
1015 				  struct dwc2_qtd *qtd)
1016 {
1017 	struct dwc2_hcd_urb *urb = qtd->urb;
1018 	enum dwc2_halt_status halt_status = DWC2_HC_XFER_COMPLETE;
1019 	int pipe_type;
1020 	int urb_xfer_done;
1021 
1022 	if (dbg_hc(chan))
1023 		dev_vdbg(hsotg->dev,
1024 			 "--Host Channel %d Interrupt: Transfer Complete--\n",
1025 			 chnum);
1026 
1027 	if (!urb)
1028 		goto handle_xfercomp_done;
1029 
1030 	pipe_type = dwc2_hcd_get_pipe_type(&urb->pipe_info);
1031 
1032 	if (hsotg->core_params->dma_desc_enable > 0) {
1033 		dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum, halt_status);
1034 		if (pipe_type == USB_ENDPOINT_XFER_ISOC)
1035 			/* Do not disable the interrupt, just clear it */
1036 			return;
1037 		goto handle_xfercomp_done;
1038 	}
1039 
1040 	/* Handle xfer complete on CSPLIT */
1041 	if (chan->qh->do_split) {
1042 		if (chan->ep_type == USB_ENDPOINT_XFER_ISOC && chan->ep_is_in &&
1043 		    hsotg->core_params->dma_enable > 0) {
1044 			if (qtd->complete_split &&
1045 			    dwc2_xfercomp_isoc_split_in(hsotg, chan, chnum,
1046 							qtd))
1047 				goto handle_xfercomp_done;
1048 		} else {
1049 			qtd->complete_split = 0;
1050 		}
1051 	}
1052 
1053 	/* Update the QTD and URB states */
1054 	switch (pipe_type) {
1055 	case USB_ENDPOINT_XFER_CONTROL:
1056 		switch (qtd->control_phase) {
1057 		case DWC2_CONTROL_SETUP:
1058 			if (urb->length > 0)
1059 				qtd->control_phase = DWC2_CONTROL_DATA;
1060 			else
1061 				qtd->control_phase = DWC2_CONTROL_STATUS;
1062 			dev_vdbg(hsotg->dev,
1063 				 "  Control setup transaction done\n");
1064 			halt_status = DWC2_HC_XFER_COMPLETE;
1065 			break;
1066 		case DWC2_CONTROL_DATA:
1067 			urb_xfer_done = dwc2_update_urb_state(hsotg, chan,
1068 							      chnum, urb, qtd);
1069 			if (urb_xfer_done) {
1070 				qtd->control_phase = DWC2_CONTROL_STATUS;
1071 				dev_vdbg(hsotg->dev,
1072 					 "  Control data transfer done\n");
1073 			} else {
1074 				dwc2_hcd_save_data_toggle(hsotg, chan, chnum,
1075 							  qtd);
1076 			}
1077 			halt_status = DWC2_HC_XFER_COMPLETE;
1078 			break;
1079 		case DWC2_CONTROL_STATUS:
1080 			dev_vdbg(hsotg->dev, "  Control transfer complete\n");
1081 			if (urb->status == -EINPROGRESS)
1082 				urb->status = 0;
1083 			dwc2_host_complete(hsotg, qtd, urb->status);
1084 			halt_status = DWC2_HC_XFER_URB_COMPLETE;
1085 			break;
1086 		}
1087 
1088 		dwc2_complete_non_periodic_xfer(hsotg, chan, chnum, qtd,
1089 						halt_status);
1090 		break;
1091 	case USB_ENDPOINT_XFER_BULK:
1092 		dev_vdbg(hsotg->dev, "  Bulk transfer complete\n");
1093 		urb_xfer_done = dwc2_update_urb_state(hsotg, chan, chnum, urb,
1094 						      qtd);
1095 		if (urb_xfer_done) {
1096 			dwc2_host_complete(hsotg, qtd, urb->status);
1097 			halt_status = DWC2_HC_XFER_URB_COMPLETE;
1098 		} else {
1099 			halt_status = DWC2_HC_XFER_COMPLETE;
1100 		}
1101 
1102 		dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1103 		dwc2_complete_non_periodic_xfer(hsotg, chan, chnum, qtd,
1104 						halt_status);
1105 		break;
1106 	case USB_ENDPOINT_XFER_INT:
1107 		dev_vdbg(hsotg->dev, "  Interrupt transfer complete\n");
1108 		urb_xfer_done = dwc2_update_urb_state(hsotg, chan, chnum, urb,
1109 						      qtd);
1110 
1111 		/*
1112 		 * Interrupt URB is done on the first transfer complete
1113 		 * interrupt
1114 		 */
1115 		if (urb_xfer_done) {
1116 			dwc2_host_complete(hsotg, qtd, urb->status);
1117 			halt_status = DWC2_HC_XFER_URB_COMPLETE;
1118 		} else {
1119 			halt_status = DWC2_HC_XFER_COMPLETE;
1120 		}
1121 
1122 		dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1123 		dwc2_complete_periodic_xfer(hsotg, chan, chnum, qtd,
1124 					    halt_status);
1125 		break;
1126 	case USB_ENDPOINT_XFER_ISOC:
1127 		if (dbg_perio())
1128 			dev_vdbg(hsotg->dev, "  Isochronous transfer complete\n");
1129 		if (qtd->isoc_split_pos == DWC2_HCSPLT_XACTPOS_ALL)
1130 			halt_status = dwc2_update_isoc_urb_state(hsotg, chan,
1131 					chnum, qtd, DWC2_HC_XFER_COMPLETE);
1132 		dwc2_complete_periodic_xfer(hsotg, chan, chnum, qtd,
1133 					    halt_status);
1134 		break;
1135 	}
1136 
1137 handle_xfercomp_done:
1138 	disable_hc_int(hsotg, chnum, HCINTMSK_XFERCOMPL);
1139 }
1140 
1141 /*
1142  * Handles a host channel STALL interrupt. This handler may be called in
1143  * either DMA mode or Slave mode.
1144  */
dwc2_hc_stall_intr(struct dwc2_hsotg * hsotg,struct dwc2_host_chan * chan,int chnum,struct dwc2_qtd * qtd)1145 static void dwc2_hc_stall_intr(struct dwc2_hsotg *hsotg,
1146 			       struct dwc2_host_chan *chan, int chnum,
1147 			       struct dwc2_qtd *qtd)
1148 {
1149 	struct dwc2_hcd_urb *urb = qtd->urb;
1150 	int pipe_type;
1151 
1152 	dev_dbg(hsotg->dev, "--Host Channel %d Interrupt: STALL Received--\n",
1153 		chnum);
1154 
1155 	if (hsotg->core_params->dma_desc_enable > 0) {
1156 		dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
1157 					    DWC2_HC_XFER_STALL);
1158 		goto handle_stall_done;
1159 	}
1160 
1161 	if (!urb)
1162 		goto handle_stall_halt;
1163 
1164 	pipe_type = dwc2_hcd_get_pipe_type(&urb->pipe_info);
1165 
1166 	if (pipe_type == USB_ENDPOINT_XFER_CONTROL)
1167 		dwc2_host_complete(hsotg, qtd, -EPIPE);
1168 
1169 	if (pipe_type == USB_ENDPOINT_XFER_BULK ||
1170 	    pipe_type == USB_ENDPOINT_XFER_INT) {
1171 		dwc2_host_complete(hsotg, qtd, -EPIPE);
1172 		/*
1173 		 * USB protocol requires resetting the data toggle for bulk
1174 		 * and interrupt endpoints when a CLEAR_FEATURE(ENDPOINT_HALT)
1175 		 * setup command is issued to the endpoint. Anticipate the
1176 		 * CLEAR_FEATURE command since a STALL has occurred and reset
1177 		 * the data toggle now.
1178 		 */
1179 		chan->qh->data_toggle = 0;
1180 	}
1181 
1182 handle_stall_halt:
1183 	dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_STALL);
1184 
1185 handle_stall_done:
1186 	disable_hc_int(hsotg, chnum, HCINTMSK_STALL);
1187 }
1188 
1189 /*
1190  * Updates the state of the URB when a transfer has been stopped due to an
1191  * abnormal condition before the transfer completes. Modifies the
1192  * actual_length field of the URB to reflect the number of bytes that have
1193  * actually been transferred via the host channel.
1194  */
dwc2_update_urb_state_abn(struct dwc2_hsotg * hsotg,struct dwc2_host_chan * chan,int chnum,struct dwc2_hcd_urb * urb,struct dwc2_qtd * qtd,enum dwc2_halt_status halt_status)1195 static void dwc2_update_urb_state_abn(struct dwc2_hsotg *hsotg,
1196 				      struct dwc2_host_chan *chan, int chnum,
1197 				      struct dwc2_hcd_urb *urb,
1198 				      struct dwc2_qtd *qtd,
1199 				      enum dwc2_halt_status halt_status)
1200 {
1201 	u32 xfer_length = dwc2_get_actual_xfer_length(hsotg, chan, chnum,
1202 						      qtd, halt_status, NULL);
1203 
1204 	if (urb->actual_length + xfer_length > urb->length) {
1205 		dev_warn(hsotg->dev, "%s(): trimming xfer length\n", __func__);
1206 		xfer_length = urb->length - urb->actual_length;
1207 	}
1208 
1209 	/* Non DWORD-aligned buffer case handling */
1210 	if (chan->align_buf && xfer_length && chan->ep_is_in) {
1211 		dev_vdbg(hsotg->dev, "%s(): non-aligned buffer\n", __func__);
1212 
1213 		usb_dma_t *ud = &chan->qh->dw_align_buf_usbdma;
1214 
1215 		usb_syncmem(ud, 0, chan->qh->dw_align_buf_size,
1216 		    chan->ep_is_in ?
1217 		    BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
1218 		if (chan->ep_is_in)
1219 			memcpy(urb->buf + urb->actual_length,
1220 					chan->qh->dw_align_buf,
1221 					xfer_length);
1222 		usb_syncmem(ud, 0, chan->qh->dw_align_buf_size,
1223 		    chan->ep_is_in ?
1224 		    BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE);
1225 	}
1226 
1227 	urb->actual_length += xfer_length;
1228 
1229 	dev_vdbg(hsotg->dev, "DWC_otg: %s: %s, channel %d\n",
1230 		 __func__, (chan->ep_is_in ? "IN" : "OUT"), chnum);
1231 	dev_vdbg(hsotg->dev, "  chan->start_pkt_count %d\n",
1232 		 chan->start_pkt_count);
1233 	dev_vdbg(hsotg->dev, "  hctsiz.pktcnt %d\n",
1234 		 (DWC2_READ_4(hsotg, HCTSIZ(chnum)) & TSIZ_PKTCNT_MASK) >> TSIZ_PKTCNT_SHIFT);
1235 	dev_vdbg(hsotg->dev, "  chan->max_packet %d\n", chan->max_packet);
1236 	dev_vdbg(hsotg->dev, "  bytes_transferred %d\n",
1237 		 xfer_length);
1238 	dev_vdbg(hsotg->dev, "  urb->actual_length %d\n",
1239 		 urb->actual_length);
1240 	dev_vdbg(hsotg->dev, "  urb->transfer_buffer_length %d\n",
1241 		 urb->length);
1242 }
1243 
1244 /*
1245  * Handles a host channel NAK interrupt. This handler may be called in either
1246  * DMA mode or Slave mode.
1247  */
dwc2_hc_nak_intr(struct dwc2_hsotg * hsotg,struct dwc2_host_chan * chan,int chnum,struct dwc2_qtd * qtd)1248 static void dwc2_hc_nak_intr(struct dwc2_hsotg *hsotg,
1249 			     struct dwc2_host_chan *chan, int chnum,
1250 			     struct dwc2_qtd *qtd)
1251 {
1252 	if (!qtd) {
1253 		dev_dbg(hsotg->dev, "%s: qtd is NULL\n", __func__);
1254 		return;
1255 	}
1256 
1257 	if (!qtd->urb) {
1258 		dev_dbg(hsotg->dev, "%s: qtd->urb is NULL\n", __func__);
1259 		return;
1260 	}
1261 
1262 	if (dbg_hc(chan))
1263 		dev_vdbg(hsotg->dev, "--Host Channel %d Interrupt: NAK Received--\n",
1264 			 chnum);
1265 
1266 	/*
1267 	 * Handle NAK for IN/OUT SSPLIT/CSPLIT transfers, bulk, control, and
1268 	 * interrupt. Re-start the SSPLIT transfer.
1269 	 *
1270 	 * Normally for non-periodic transfers we'll retry right away, but to
1271 	 * avoid interrupt storms we'll wait before retrying if we've got
1272 	 * several NAKs. If we didn't do this we'd retry directly from the
1273 	 * interrupt handler and could end up quickly getting another
1274 	 * interrupt (another NAK), which we'd retry.
1275 	 *
1276 	 * Note that in DMA mode software only gets involved to re-send NAKed
1277 	 * transfers for split transactions unless the core is missing OUT NAK
1278 	 * enhancement.
1279 	 */
1280 	if (chan->do_split) {
1281 		/*
1282 		 * When we get control/bulk NAKs then remember this so we holdoff on
1283 		 * this qh until the beginning of the next frame
1284 		 */
1285 		switch (dwc2_hcd_get_pipe_type(&qtd->urb->pipe_info)) {
1286 		case USB_ENDPOINT_XFER_CONTROL:
1287 		case USB_ENDPOINT_XFER_BULK:
1288 			chan->qh->nak_frame = dwc2_hcd_get_frame_number(hsotg);
1289 			break;
1290 		}
1291 
1292 		if (chan->complete_split)
1293 			qtd->error_count = 0;
1294 		qtd->complete_split = 0;
1295 		qtd->num_naks++;
1296 		qtd->qh->want_wait = qtd->num_naks >= dwc2_naks_before_delay;
1297 		dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_NAK);
1298 		goto handle_nak_done;
1299 	}
1300 
1301 	switch (dwc2_hcd_get_pipe_type(&qtd->urb->pipe_info)) {
1302 	case USB_ENDPOINT_XFER_CONTROL:
1303 	case USB_ENDPOINT_XFER_BULK:
1304 		if (hsotg->core_params->dma_enable > 0 && chan->ep_is_in) {
1305 			/*
1306 			 * NAK interrupts are enabled on bulk/control IN
1307 			 * transfers in DMA mode for the sole purpose of
1308 			 * resetting the error count after a transaction error
1309 			 * occurs. The core will continue transferring data.
1310 			 */
1311 			qtd->error_count = 0;
1312 			break;
1313 		}
1314 
1315 		/*
1316 		 * NAK interrupts normally occur during OUT transfers in DMA
1317 		 * or Slave mode. For IN transfers, more requests will be
1318 		 * queued as request queue space is available.
1319 		 */
1320 		qtd->error_count = 0;
1321 
1322 		if (hsotg->core_params->dma_enable > 0 && !chan->ep_is_in) {
1323 			/*
1324 			 * Avoid interrupt storms.
1325 			 */
1326 			qtd->num_naks++;
1327 			qtd->qh->want_wait = qtd->num_naks >= dwc2_out_naks_before_delay;
1328 		}
1329 		if (!chan->qh->ping_state) {
1330 			dwc2_update_urb_state_abn(hsotg, chan, chnum, qtd->urb,
1331 						  qtd, DWC2_HC_XFER_NAK);
1332 			dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1333 
1334 			if (chan->speed == USB_SPEED_HIGH)
1335 				chan->qh->ping_state = 1;
1336 		}
1337 
1338 		/*
1339 		 * Halt the channel so the transfer can be re-started from
1340 		 * the appropriate point or the PING protocol will
1341 		 * start/continue
1342 		 */
1343 		dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_NAK);
1344 		break;
1345 	case USB_ENDPOINT_XFER_INT:
1346 		qtd->error_count = 0;
1347 		dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_NAK);
1348 		break;
1349 	case USB_ENDPOINT_XFER_ISOC:
1350 		/* Should never get called for isochronous transfers */
1351 		dev_err(hsotg->dev, "NACK interrupt for ISOC transfer\n");
1352 		break;
1353 	}
1354 
1355 handle_nak_done:
1356 	disable_hc_int(hsotg, chnum, HCINTMSK_NAK);
1357 }
1358 
1359 /*
1360  * Handles a host channel ACK interrupt. This interrupt is enabled when
1361  * performing the PING protocol in Slave mode, when errors occur during
1362  * either Slave mode or DMA mode, and during Start Split transactions.
1363  */
dwc2_hc_ack_intr(struct dwc2_hsotg * hsotg,struct dwc2_host_chan * chan,int chnum,struct dwc2_qtd * qtd)1364 static void dwc2_hc_ack_intr(struct dwc2_hsotg *hsotg,
1365 			     struct dwc2_host_chan *chan, int chnum,
1366 			     struct dwc2_qtd *qtd)
1367 {
1368 	struct dwc2_hcd_iso_packet_desc *frame_desc;
1369 
1370 	if (dbg_hc(chan))
1371 		dev_vdbg(hsotg->dev, "--Host Channel %d Interrupt: ACK Received--\n",
1372 			 chnum);
1373 
1374 	if (chan->do_split) {
1375 		/* Handle ACK on SSPLIT. ACK should not occur in CSPLIT. */
1376 		if (!chan->ep_is_in &&
1377 		    chan->data_pid_start != DWC2_HC_PID_SETUP)
1378 			qtd->ssplit_out_xfer_count = chan->xfer_len;
1379 
1380 		if (chan->ep_type != USB_ENDPOINT_XFER_ISOC || chan->ep_is_in) {
1381 			qtd->complete_split = 1;
1382 			dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_ACK);
1383 		} else {
1384 			/* ISOC OUT */
1385 			switch (chan->xact_pos) {
1386 			case DWC2_HCSPLT_XACTPOS_ALL:
1387 				break;
1388 			case DWC2_HCSPLT_XACTPOS_END:
1389 				qtd->isoc_split_pos = DWC2_HCSPLT_XACTPOS_ALL;
1390 				qtd->isoc_split_offset = 0;
1391 				break;
1392 			case DWC2_HCSPLT_XACTPOS_BEGIN:
1393 			case DWC2_HCSPLT_XACTPOS_MID:
1394 				/*
1395 				 * For BEGIN or MID, calculate the length for
1396 				 * the next microframe to determine the correct
1397 				 * SSPLIT token, either MID or END
1398 				 */
1399 				frame_desc = &qtd->urb->iso_descs[
1400 						qtd->isoc_frame_index];
1401 				qtd->isoc_split_offset += 188;
1402 
1403 				if (frame_desc->length - qtd->isoc_split_offset
1404 							<= 188)
1405 					qtd->isoc_split_pos =
1406 							DWC2_HCSPLT_XACTPOS_END;
1407 				else
1408 					qtd->isoc_split_pos =
1409 							DWC2_HCSPLT_XACTPOS_MID;
1410 				break;
1411 			}
1412 		}
1413 	} else {
1414 		qtd->error_count = 0;
1415 
1416 		if (chan->qh->ping_state) {
1417 			chan->qh->ping_state = 0;
1418 			/*
1419 			 * Halt the channel so the transfer can be re-started
1420 			 * from the appropriate point. This only happens in
1421 			 * Slave mode. In DMA mode, the ping_state is cleared
1422 			 * when the transfer is started because the core
1423 			 * automatically executes the PING, then the transfer.
1424 			 */
1425 			dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_ACK);
1426 		}
1427 	}
1428 
1429 	/*
1430 	 * If the ACK occurred when _not_ in the PING state, let the channel
1431 	 * continue transferring data after clearing the error count
1432 	 */
1433 	disable_hc_int(hsotg, chnum, HCINTMSK_ACK);
1434 }
1435 
1436 /*
1437  * Handles a host channel NYET interrupt. This interrupt should only occur on
1438  * Bulk and Control OUT endpoints and for complete split transactions. If a
1439  * NYET occurs at the same time as a Transfer Complete interrupt, it is
1440  * handled in the xfercomp interrupt handler, not here. This handler may be
1441  * called in either DMA mode or Slave mode.
1442  */
dwc2_hc_nyet_intr(struct dwc2_hsotg * hsotg,struct dwc2_host_chan * chan,int chnum,struct dwc2_qtd * qtd)1443 static void dwc2_hc_nyet_intr(struct dwc2_hsotg *hsotg,
1444 			      struct dwc2_host_chan *chan, int chnum,
1445 			      struct dwc2_qtd *qtd)
1446 {
1447 	if (dbg_hc(chan))
1448 		dev_vdbg(hsotg->dev, "--Host Channel %d Interrupt: NYET Received--\n",
1449 			 chnum);
1450 
1451 	/*
1452 	 * NYET on CSPLIT
1453 	 * re-do the CSPLIT immediately on non-periodic
1454 	 */
1455 	if (chan->do_split && chan->complete_split) {
1456 		if (chan->ep_is_in && chan->ep_type == USB_ENDPOINT_XFER_ISOC &&
1457 		    hsotg->core_params->dma_enable > 0) {
1458 			qtd->complete_split = 0;
1459 			qtd->isoc_split_offset = 0;
1460 			qtd->isoc_frame_index++;
1461 			if (qtd->urb &&
1462 			    qtd->isoc_frame_index == qtd->urb->packet_count) {
1463 				dwc2_host_complete(hsotg, qtd, 0);
1464 				dwc2_release_channel(hsotg, chan, qtd,
1465 						     DWC2_HC_XFER_URB_COMPLETE);
1466 			} else {
1467 				dwc2_release_channel(hsotg, chan, qtd,
1468 						DWC2_HC_XFER_NO_HALT_STATUS);
1469 			}
1470 			goto handle_nyet_done;
1471 		}
1472 
1473 		if (chan->ep_type == USB_ENDPOINT_XFER_INT ||
1474 		    chan->ep_type == USB_ENDPOINT_XFER_ISOC) {
1475 			int frnum = dwc2_hcd_get_frame_number(hsotg);
1476 
1477 			if (dwc2_full_frame_num(frnum) !=
1478 			    dwc2_full_frame_num(chan->qh->sched_frame)) {
1479 				/*
1480 				 * No longer in the same full speed frame.
1481 				 * Treat this as a transaction error.
1482 				 */
1483 #if 0
1484 				/*
1485 				 * Todo: Fix system performance so this can
1486 				 * be treated as an error. Right now complete
1487 				 * splits cannot be scheduled precisely enough
1488 				 * due to other system activity, so this error
1489 				 * occurs regularly in Slave mode.
1490 				 */
1491 				qtd->error_count++;
1492 #endif
1493 				qtd->complete_split = 0;
1494 				dwc2_halt_channel(hsotg, chan, qtd,
1495 						  DWC2_HC_XFER_XACT_ERR);
1496 				/* Todo: add support for isoc release */
1497 				goto handle_nyet_done;
1498 			}
1499 		}
1500 
1501 		dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_NYET);
1502 		goto handle_nyet_done;
1503 	}
1504 
1505 	chan->qh->ping_state = 1;
1506 	qtd->error_count = 0;
1507 
1508 	dwc2_update_urb_state_abn(hsotg, chan, chnum, qtd->urb, qtd,
1509 				  DWC2_HC_XFER_NYET);
1510 	dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1511 
1512 	/*
1513 	 * Halt the channel and re-start the transfer so the PING protocol
1514 	 * will start
1515 	 */
1516 	dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_NYET);
1517 
1518 handle_nyet_done:
1519 	disable_hc_int(hsotg, chnum, HCINTMSK_NYET);
1520 }
1521 
1522 /*
1523  * Handles a host channel babble interrupt. This handler may be called in
1524  * either DMA mode or Slave mode.
1525  */
dwc2_hc_babble_intr(struct dwc2_hsotg * hsotg,struct dwc2_host_chan * chan,int chnum,struct dwc2_qtd * qtd)1526 static void dwc2_hc_babble_intr(struct dwc2_hsotg *hsotg,
1527 				struct dwc2_host_chan *chan, int chnum,
1528 				struct dwc2_qtd *qtd)
1529 {
1530 	dev_dbg(hsotg->dev, "--Host Channel %d Interrupt: Babble Error--\n",
1531 		chnum);
1532 
1533 // 	dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
1534 
1535 	if (hsotg->core_params->dma_desc_enable > 0) {
1536 		dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
1537 					    DWC2_HC_XFER_BABBLE_ERR);
1538 		goto disable_int;
1539 	}
1540 
1541 	if (chan->ep_type != USB_ENDPOINT_XFER_ISOC) {
1542 		dwc2_host_complete(hsotg, qtd, -EOVERFLOW);
1543 		dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_BABBLE_ERR);
1544 	} else {
1545 		enum dwc2_halt_status halt_status;
1546 
1547 		halt_status = dwc2_update_isoc_urb_state(hsotg, chan, chnum,
1548 						qtd, DWC2_HC_XFER_BABBLE_ERR);
1549 		dwc2_halt_channel(hsotg, chan, qtd, halt_status);
1550 	}
1551 
1552 disable_int:
1553 	disable_hc_int(hsotg, chnum, HCINTMSK_BBLERR);
1554 }
1555 
1556 /*
1557  * Handles a host channel AHB error interrupt. This handler is only called in
1558  * DMA mode.
1559  */
dwc2_hc_ahberr_intr(struct dwc2_hsotg * hsotg,struct dwc2_host_chan * chan,int chnum,struct dwc2_qtd * qtd)1560 static void dwc2_hc_ahberr_intr(struct dwc2_hsotg *hsotg,
1561 				struct dwc2_host_chan *chan, int chnum,
1562 				struct dwc2_qtd *qtd)
1563 {
1564 	struct dwc2_hcd_urb *urb = qtd->urb;
1565 
1566 	dev_dbg(hsotg->dev, "--Host Channel %d Interrupt: AHB Error--\n",
1567 		chnum);
1568 
1569 	if (!urb)
1570 		goto handle_ahberr_halt;
1571 
1572 // 	dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
1573 
1574 #ifdef DWC2_DEBUG
1575 	const char *pipetype, *speed;
1576 
1577 	u32 hcchar = DWC2_READ_4(hsotg, HCCHAR(chnum));
1578 	u32 hcsplt = DWC2_READ_4(hsotg, HCSPLT(chnum));
1579 	u32 hctsiz = DWC2_READ_4(hsotg, HCTSIZ(chnum));
1580 	u32 hc_dma = DWC2_READ_4(hsotg, HCDMA(chnum));
1581 
1582 	dev_err(hsotg->dev, "AHB ERROR, Channel %d\n", chnum);
1583 	dev_err(hsotg->dev, "  hcchar 0x%08x, hcsplt 0x%08x\n", hcchar, hcsplt);
1584 	dev_err(hsotg->dev, "  hctsiz 0x%08x, hc_dma 0x%08x\n", hctsiz, hc_dma);
1585 	dev_err(hsotg->dev, "  Device address: %d\n",
1586 		dwc2_hcd_get_dev_addr(&urb->pipe_info));
1587 	dev_err(hsotg->dev, "  Endpoint: %d, %s\n",
1588 		dwc2_hcd_get_ep_num(&urb->pipe_info),
1589 		dwc2_hcd_is_pipe_in(&urb->pipe_info) ? "IN" : "OUT");
1590 
1591 	switch (dwc2_hcd_get_pipe_type(&urb->pipe_info)) {
1592 	case USB_ENDPOINT_XFER_CONTROL:
1593 		pipetype = "CONTROL";
1594 		break;
1595 	case USB_ENDPOINT_XFER_BULK:
1596 		pipetype = "BULK";
1597 		break;
1598 	case USB_ENDPOINT_XFER_INT:
1599 		pipetype = "INTERRUPT";
1600 		break;
1601 	case USB_ENDPOINT_XFER_ISOC:
1602 		pipetype = "ISOCHRONOUS";
1603 		break;
1604 	default:
1605 		pipetype = "UNKNOWN";
1606 		break;
1607 	}
1608 
1609 	dev_err(hsotg->dev, "  Endpoint type: %s\n", pipetype);
1610 
1611 	switch (chan->speed) {
1612 	case USB_SPEED_HIGH:
1613 		speed = "HIGH";
1614 		break;
1615 	case USB_SPEED_FULL:
1616 		speed = "FULL";
1617 		break;
1618 	case USB_SPEED_LOW:
1619 		speed = "LOW";
1620 		break;
1621 	default:
1622 		speed = "UNKNOWN";
1623 		break;
1624 	}
1625 
1626 	dev_err(hsotg->dev, "  Speed: %s\n", speed);
1627 
1628 	dev_err(hsotg->dev, "  Max packet size: %d\n",
1629 		dwc2_hcd_get_mps(&urb->pipe_info));
1630 	dev_err(hsotg->dev, "  Data buffer length: %d\n", urb->length);
1631 	dev_err(hsotg->dev, "  Transfer buffer: %p, Transfer DMA: %08lx\n",
1632 		urb->buf, (unsigned long)urb->dma);
1633 	dev_err(hsotg->dev, "  Setup buffer: %p, Setup DMA: %08lx\n",
1634 		urb->setup_packet, (unsigned long)urb->setup_dma);
1635 	dev_err(hsotg->dev, "  Interval: %d\n", urb->interval);
1636 #endif
1637 
1638 	/* Core halts the channel for Descriptor DMA mode */
1639 	if (hsotg->core_params->dma_desc_enable > 0) {
1640 		dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
1641 					    DWC2_HC_XFER_AHB_ERR);
1642 		goto handle_ahberr_done;
1643 	}
1644 
1645 	dwc2_host_complete(hsotg, qtd, -EIO);
1646 
1647 handle_ahberr_halt:
1648 	/*
1649 	 * Force a channel halt. Don't call dwc2_halt_channel because that won't
1650 	 * write to the HCCHARn register in DMA mode to force the halt.
1651 	 */
1652 	dwc2_hc_halt(hsotg, chan, DWC2_HC_XFER_AHB_ERR);
1653 
1654 handle_ahberr_done:
1655 	disable_hc_int(hsotg, chnum, HCINTMSK_AHBERR);
1656 }
1657 
1658 /*
1659  * Handles a host channel transaction error interrupt. This handler may be
1660  * called in either DMA mode or Slave mode.
1661  */
dwc2_hc_xacterr_intr(struct dwc2_hsotg * hsotg,struct dwc2_host_chan * chan,int chnum,struct dwc2_qtd * qtd)1662 static void dwc2_hc_xacterr_intr(struct dwc2_hsotg *hsotg,
1663 				 struct dwc2_host_chan *chan, int chnum,
1664 				 struct dwc2_qtd *qtd)
1665 {
1666 	dev_dbg(hsotg->dev,
1667 		"--Host Channel %d Interrupt: Transaction Error--\n", chnum);
1668 
1669 // 	dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
1670 
1671 	if (hsotg->core_params->dma_desc_enable > 0) {
1672 		dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
1673 					    DWC2_HC_XFER_XACT_ERR);
1674 		goto handle_xacterr_done;
1675 	}
1676 
1677 	switch (dwc2_hcd_get_pipe_type(&qtd->urb->pipe_info)) {
1678 	case USB_ENDPOINT_XFER_CONTROL:
1679 	case USB_ENDPOINT_XFER_BULK:
1680 		qtd->error_count++;
1681 		if (!chan->qh->ping_state) {
1682 
1683 			dwc2_update_urb_state_abn(hsotg, chan, chnum, qtd->urb,
1684 						  qtd, DWC2_HC_XFER_XACT_ERR);
1685 			dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1686 			if (!chan->ep_is_in && chan->speed == USB_SPEED_HIGH)
1687 				chan->qh->ping_state = 1;
1688 		}
1689 
1690 		/*
1691 		 * Halt the channel so the transfer can be re-started from
1692 		 * the appropriate point or the PING protocol will start
1693 		 */
1694 		dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_XACT_ERR);
1695 		break;
1696 	case USB_ENDPOINT_XFER_INT:
1697 		qtd->error_count++;
1698 		if (chan->do_split && chan->complete_split)
1699 			qtd->complete_split = 0;
1700 		dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_XACT_ERR);
1701 		break;
1702 	case USB_ENDPOINT_XFER_ISOC:
1703 		{
1704 			enum dwc2_halt_status halt_status;
1705 
1706 			halt_status = dwc2_update_isoc_urb_state(hsotg, chan,
1707 					chnum, qtd, DWC2_HC_XFER_XACT_ERR);
1708 			dwc2_halt_channel(hsotg, chan, qtd, halt_status);
1709 		}
1710 		break;
1711 	}
1712 
1713 handle_xacterr_done:
1714 	disable_hc_int(hsotg, chnum, HCINTMSK_XACTERR);
1715 }
1716 
1717 /*
1718  * Handles a host channel frame overrun interrupt. This handler may be called
1719  * in either DMA mode or Slave mode.
1720  */
dwc2_hc_frmovrun_intr(struct dwc2_hsotg * hsotg,struct dwc2_host_chan * chan,int chnum,struct dwc2_qtd * qtd)1721 static void dwc2_hc_frmovrun_intr(struct dwc2_hsotg *hsotg,
1722 				  struct dwc2_host_chan *chan, int chnum,
1723 				  struct dwc2_qtd *qtd)
1724 {
1725 	enum dwc2_halt_status halt_status;
1726 
1727 	if (dbg_hc(chan))
1728 		dev_dbg(hsotg->dev, "--Host Channel %d Interrupt: Frame Overrun--\n",
1729 			chnum);
1730 
1731 	dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
1732 
1733 	switch (dwc2_hcd_get_pipe_type(&qtd->urb->pipe_info)) {
1734 	case USB_ENDPOINT_XFER_CONTROL:
1735 	case USB_ENDPOINT_XFER_BULK:
1736 		break;
1737 	case USB_ENDPOINT_XFER_INT:
1738 		dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_FRAME_OVERRUN);
1739 		break;
1740 	case USB_ENDPOINT_XFER_ISOC:
1741 		halt_status = dwc2_update_isoc_urb_state(hsotg, chan, chnum,
1742 					qtd, DWC2_HC_XFER_FRAME_OVERRUN);
1743 		dwc2_halt_channel(hsotg, chan, qtd, halt_status);
1744 		break;
1745 	}
1746 
1747 	disable_hc_int(hsotg, chnum, HCINTMSK_FRMOVRUN);
1748 }
1749 
1750 /*
1751  * Handles a host channel data toggle error interrupt. This handler may be
1752  * called in either DMA mode or Slave mode.
1753  */
dwc2_hc_datatglerr_intr(struct dwc2_hsotg * hsotg,struct dwc2_host_chan * chan,int chnum,struct dwc2_qtd * qtd)1754 static void dwc2_hc_datatglerr_intr(struct dwc2_hsotg *hsotg,
1755 				    struct dwc2_host_chan *chan, int chnum,
1756 				    struct dwc2_qtd *qtd)
1757 {
1758 	dev_dbg(hsotg->dev,
1759 		"--Host Channel %d Interrupt: Data Toggle Error--\n", chnum);
1760 
1761 	if (chan->ep_is_in)
1762 		qtd->error_count = 0;
1763 	else
1764 		dev_err(hsotg->dev,
1765 			"Data Toggle Error on OUT transfer, channel %d\n",
1766 			chnum);
1767 
1768 // 	dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
1769 	disable_hc_int(hsotg, chnum, HCINTMSK_DATATGLERR);
1770 }
1771 
1772 /*
1773  * For debug only. It checks that a valid halt status is set and that
1774  * HCCHARn.chdis is clear. If there's a problem, corrective action is
1775  * taken and a warning is issued.
1776  *
1777  * Return: true if halt status is ok, false otherwise
1778  */
dwc2_halt_status_ok(struct dwc2_hsotg * hsotg,struct dwc2_host_chan * chan,int chnum,struct dwc2_qtd * qtd)1779 static bool dwc2_halt_status_ok(struct dwc2_hsotg *hsotg,
1780 				struct dwc2_host_chan *chan, int chnum,
1781 				struct dwc2_qtd *qtd)
1782 {
1783 #ifdef DWC2_DEBUG
1784 	u32 hcchar;
1785 	u32 hctsiz;
1786 	u32 hcintmsk;
1787 	u32 hcsplt;
1788 
1789 	if (chan->halt_status == DWC2_HC_XFER_NO_HALT_STATUS) {
1790 		/*
1791 		 * This code is here only as a check. This condition should
1792 		 * never happen. Ignore the halt if it does occur.
1793 		 */
1794 		hcchar = DWC2_READ_4(hsotg, HCCHAR(chnum));
1795 		hctsiz = DWC2_READ_4(hsotg, HCTSIZ(chnum));
1796 		hcintmsk = DWC2_READ_4(hsotg, HCINTMSK(chnum));
1797 		hcsplt = DWC2_READ_4(hsotg, HCSPLT(chnum));
1798 		dev_dbg(hsotg->dev,
1799 			"%s: chan->halt_status DWC2_HC_XFER_NO_HALT_STATUS,\n",
1800 			 __func__);
1801 		dev_dbg(hsotg->dev,
1802 			"channel %d, hcchar 0x%08x, hctsiz 0x%08x,\n",
1803 			chnum, hcchar, hctsiz);
1804 		dev_dbg(hsotg->dev,
1805 			"hcint 0x%08x, hcintmsk 0x%08x, hcsplt 0x%08x,\n",
1806 			chan->hcint, hcintmsk, hcsplt);
1807 		if (qtd)
1808 			dev_dbg(hsotg->dev, "qtd->complete_split %d\n",
1809 				qtd->complete_split);
1810 		dev_warn(hsotg->dev,
1811 			 "%s: no halt status, channel %d, ignoring interrupt\n",
1812 			 __func__, chnum);
1813 		return false;
1814 	}
1815 
1816 	/*
1817 	 * This code is here only as a check. hcchar.chdis should never be set
1818 	 * when the halt interrupt occurs. Halt the channel again if it does
1819 	 * occur.
1820 	 */
1821 	hcchar = DWC2_READ_4(hsotg, HCCHAR(chnum));
1822 	if (hcchar & HCCHAR_CHDIS) {
1823 		dev_warn(hsotg->dev,
1824 			 "%s: hcchar.chdis set unexpectedly, hcchar 0x%08x, trying to halt again\n",
1825 			 __func__, hcchar);
1826 		chan->halt_pending = 0;
1827 		dwc2_halt_channel(hsotg, chan, qtd, chan->halt_status);
1828 		return false;
1829 	}
1830 #endif
1831 
1832 	return true;
1833 }
1834 
1835 /*
1836  * Handles a host Channel Halted interrupt in DMA mode. This handler
1837  * determines the reason the channel halted and proceeds accordingly.
1838  */
dwc2_hc_chhltd_intr_dma(struct dwc2_hsotg * hsotg,struct dwc2_host_chan * chan,int chnum,struct dwc2_qtd * qtd)1839 static void dwc2_hc_chhltd_intr_dma(struct dwc2_hsotg *hsotg,
1840 				    struct dwc2_host_chan *chan, int chnum,
1841 				    struct dwc2_qtd *qtd)
1842 {
1843 	u32 hcintmsk;
1844 	int out_nak_enh = 0;
1845 
1846 	if (dbg_hc(chan))
1847 		dev_vdbg(hsotg->dev,
1848 			 "--Host Channel %d Interrupt: DMA Channel Halted--\n",
1849 			 chnum);
1850 
1851 	/*
1852 	 * For core with OUT NAK enhancement, the flow for high-speed
1853 	 * CONTROL/BULK OUT is handled a little differently
1854 	 */
1855 	if (hsotg->hw_params.snpsid >= DWC2_CORE_REV_2_71a) {
1856 		if (chan->speed == USB_SPEED_HIGH && !chan->ep_is_in &&
1857 		    (chan->ep_type == USB_ENDPOINT_XFER_CONTROL ||
1858 		     chan->ep_type == USB_ENDPOINT_XFER_BULK)) {
1859 			out_nak_enh = 1;
1860 		}
1861 	}
1862 
1863 	if (chan->halt_status == DWC2_HC_XFER_URB_DEQUEUE ||
1864 	    (chan->halt_status == DWC2_HC_XFER_AHB_ERR &&
1865 	     hsotg->core_params->dma_desc_enable <= 0)) {
1866 		if (hsotg->core_params->dma_desc_enable > 0)
1867 			dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
1868 						    chan->halt_status);
1869 		else
1870 			/*
1871 			 * Just release the channel. A dequeue can happen on a
1872 			 * transfer timeout. In the case of an AHB Error, the
1873 			 * channel was forced to halt because there's no way to
1874 			 * gracefully recover.
1875 			 */
1876 			dwc2_release_channel(hsotg, chan, qtd,
1877 					     chan->halt_status);
1878 		return;
1879 	}
1880 
1881 	hcintmsk = DWC2_READ_4(hsotg, HCINTMSK(chnum));
1882 
1883 	if (chan->hcint & HCINTMSK_XFERCOMPL) {
1884 		/*
1885 		 * Todo: This is here because of a possible hardware bug. Spec
1886 		 * says that on SPLIT-ISOC OUT transfers in DMA mode that a HALT
1887 		 * interrupt w/ACK bit set should occur, but I only see the
1888 		 * XFERCOMP bit, even with it masked out. This is a workaround
1889 		 * for that behavior. Should fix this when hardware is fixed.
1890 		 */
1891 		if (chan->ep_type == USB_ENDPOINT_XFER_ISOC && !chan->ep_is_in)
1892 			dwc2_hc_ack_intr(hsotg, chan, chnum, qtd);
1893 		dwc2_hc_xfercomp_intr(hsotg, chan, chnum, qtd);
1894 	} else if (chan->hcint & HCINTMSK_STALL) {
1895 		dwc2_hc_stall_intr(hsotg, chan, chnum, qtd);
1896 	} else if ((chan->hcint & HCINTMSK_XACTERR) &&
1897 		   hsotg->core_params->dma_desc_enable <= 0) {
1898 		if (out_nak_enh) {
1899 			if (chan->hcint &
1900 			    (HCINTMSK_NYET | HCINTMSK_NAK | HCINTMSK_ACK)) {
1901 				dev_vdbg(hsotg->dev,
1902 					 "XactErr with NYET/NAK/ACK\n");
1903 				qtd->error_count = 0;
1904 			} else {
1905 				dev_vdbg(hsotg->dev,
1906 					 "XactErr without NYET/NAK/ACK\n");
1907 			}
1908 		}
1909 
1910 		/*
1911 		 * Must handle xacterr before nak or ack. Could get a xacterr
1912 		 * at the same time as either of these on a BULK/CONTROL OUT
1913 		 * that started with a PING. The xacterr takes precedence.
1914 		 */
1915 		dwc2_hc_xacterr_intr(hsotg, chan, chnum, qtd);
1916 	} else if ((chan->hcint & HCINTMSK_XCS_XACT) &&
1917 		   hsotg->core_params->dma_desc_enable > 0) {
1918 		dwc2_hc_xacterr_intr(hsotg, chan, chnum, qtd);
1919 	} else if ((chan->hcint & HCINTMSK_AHBERR) &&
1920 		   hsotg->core_params->dma_desc_enable > 0) {
1921 		dwc2_hc_ahberr_intr(hsotg, chan, chnum, qtd);
1922 	} else if (chan->hcint & HCINTMSK_BBLERR) {
1923 		dwc2_hc_babble_intr(hsotg, chan, chnum, qtd);
1924 	} else if (chan->hcint & HCINTMSK_FRMOVRUN) {
1925 		dwc2_hc_frmovrun_intr(hsotg, chan, chnum, qtd);
1926 	} else if (!out_nak_enh) {
1927 		if (chan->hcint & HCINTMSK_NYET) {
1928 			/*
1929 			 * Must handle nyet before nak or ack. Could get a nyet
1930 			 * at the same time as either of those on a BULK/CONTROL
1931 			 * OUT that started with a PING. The nyet takes
1932 			 * precedence.
1933 			 */
1934 			dwc2_hc_nyet_intr(hsotg, chan, chnum, qtd);
1935 		} else if ((chan->hcint & HCINTMSK_NAK) &&
1936 			   !(hcintmsk & HCINTMSK_NAK)) {
1937 			/*
1938 			 * If nak is not masked, it's because a non-split IN
1939 			 * transfer is in an error state. In that case, the nak
1940 			 * is handled by the nak interrupt handler, not here.
1941 			 * Handle nak here for BULK/CONTROL OUT transfers, which
1942 			 * halt on a NAK to allow rewinding the buffer pointer.
1943 			 */
1944 			dwc2_hc_nak_intr(hsotg, chan, chnum, qtd);
1945 		} else if ((chan->hcint & HCINTMSK_ACK) &&
1946 			   !(hcintmsk & HCINTMSK_ACK)) {
1947 			/*
1948 			 * If ack is not masked, it's because a non-split IN
1949 			 * transfer is in an error state. In that case, the ack
1950 			 * is handled by the ack interrupt handler, not here.
1951 			 * Handle ack here for split transfers. Start splits
1952 			 * halt on ACK.
1953 			 */
1954 			dwc2_hc_ack_intr(hsotg, chan, chnum, qtd);
1955 		} else {
1956 			if (chan->ep_type == USB_ENDPOINT_XFER_INT ||
1957 			    chan->ep_type == USB_ENDPOINT_XFER_ISOC) {
1958 				/*
1959 				 * A periodic transfer halted with no other
1960 				 * channel interrupts set. Assume it was halted
1961 				 * by the core because it could not be completed
1962 				 * in its scheduled (micro)frame.
1963 				 */
1964 				dev_dbg(hsotg->dev,
1965 					"%s: Halt channel %d (assume incomplete periodic transfer)\n",
1966 					__func__, chnum);
1967 				dwc2_halt_channel(hsotg, chan, qtd,
1968 					DWC2_HC_XFER_PERIODIC_INCOMPLETE);
1969 			} else {
1970 				dev_err(hsotg->dev,
1971 					"%s: Channel %d - ChHltd set, but reason is unknown\n",
1972 					__func__, chnum);
1973 				dev_err(hsotg->dev,
1974 					"hcint 0x%08x, intsts 0x%08x\n",
1975 					chan->hcint,
1976 					DWC2_READ_4(hsotg, GINTSTS));
1977 				goto error;
1978 			}
1979 		}
1980 	} else {
1981 		dev_info(hsotg->dev,
1982 			 "NYET/NAK/ACK/other in non-error case, 0x%08x\n",
1983 			 chan->hcint);
1984 error:
1985 		/* Failthrough: use 3-strikes rule */
1986 		qtd->error_count++;
1987 		dwc2_update_urb_state_abn(hsotg, chan, chnum, qtd->urb,
1988 					  qtd, DWC2_HC_XFER_XACT_ERR);
1989 		dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1990 		dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_XACT_ERR);
1991 	}
1992 }
1993 
1994 /*
1995  * Handles a host channel Channel Halted interrupt
1996  *
1997  * In slave mode, this handler is called only when the driver specifically
1998  * requests a halt. This occurs during handling other host channel interrupts
1999  * (e.g. nak, xacterr, stall, nyet, etc.).
2000  *
2001  * In DMA mode, this is the interrupt that occurs when the core has finished
2002  * processing a transfer on a channel. Other host channel interrupts (except
2003  * ahberr) are disabled in DMA mode.
2004  */
dwc2_hc_chhltd_intr(struct dwc2_hsotg * hsotg,struct dwc2_host_chan * chan,int chnum,struct dwc2_qtd * qtd)2005 static void dwc2_hc_chhltd_intr(struct dwc2_hsotg *hsotg,
2006 				struct dwc2_host_chan *chan, int chnum,
2007 				struct dwc2_qtd *qtd)
2008 {
2009 	if (dbg_hc(chan))
2010 		dev_vdbg(hsotg->dev, "--Host Channel %d Interrupt: Channel Halted--\n",
2011 			 chnum);
2012 
2013 	if (hsotg->core_params->dma_enable > 0) {
2014 		dwc2_hc_chhltd_intr_dma(hsotg, chan, chnum, qtd);
2015 	} else {
2016 		if (!dwc2_halt_status_ok(hsotg, chan, chnum, qtd))
2017 			return;
2018 		dwc2_release_channel(hsotg, chan, qtd, chan->halt_status);
2019 	}
2020 }
2021 
2022 /*
2023  * Check if the given qtd is still the top of the list (and thus valid).
2024  *
2025  * If dwc2_hcd_qtd_unlink_and_free() has been called since we grabbed
2026  * the qtd from the top of the list, this will return false (otherwise true).
2027  */
dwc2_check_qtd_still_ok(struct dwc2_qtd * qtd,struct dwc2_qh * qh)2028 static bool dwc2_check_qtd_still_ok(struct dwc2_qtd *qtd, struct dwc2_qh *qh)
2029 {
2030 	struct dwc2_qtd *cur_head;
2031 
2032 	if (qh == NULL)
2033 		return false;
2034 
2035 	cur_head = list_first_entry(&qh->qtd_list, struct dwc2_qtd,
2036 				    qtd_list_entry);
2037 	return (cur_head == qtd);
2038 }
2039 
2040 /* Handles interrupt for a specific Host Channel */
dwc2_hc_n_intr(struct dwc2_hsotg * hsotg,int chnum)2041 static void dwc2_hc_n_intr(struct dwc2_hsotg *hsotg, int chnum)
2042 {
2043 	struct dwc2_qtd *qtd;
2044 	struct dwc2_host_chan *chan;
2045 	u32 hcint, hcintmsk;
2046 
2047 	chan = hsotg->hc_ptr_array[chnum];
2048 
2049 	hcint = DWC2_READ_4(hsotg, HCINT(chnum));
2050 	hcintmsk = DWC2_READ_4(hsotg, HCINTMSK(chnum));
2051 	if (!chan) {
2052 		dev_err(hsotg->dev, "## hc_ptr_array for channel is NULL ##\n");
2053 		DWC2_WRITE_4(hsotg, HCINT(chnum), hcint);
2054 		return;
2055 	}
2056 
2057 	if (dbg_hc(chan)) {
2058 		dev_vdbg(hsotg->dev, "--Host Channel Interrupt--, Channel %d\n",
2059 			 chnum);
2060 		dev_vdbg(hsotg->dev,
2061 			 "  hcint 0x%08x, hcintmsk 0x%08x, hcint&hcintmsk 0x%08x\n",
2062 			 hcint, hcintmsk, hcint & hcintmsk);
2063 	}
2064 
2065 	DWC2_WRITE_4(hsotg, HCINT(chnum), hcint);
2066 	chan->hcint = hcint;
2067 	hcint &= hcintmsk;
2068 
2069 	/*
2070 	 * If the channel was halted due to a dequeue, the qtd list might
2071 	 * be empty or at least the first entry will not be the active qtd.
2072 	 * In this case, take a shortcut and just release the channel.
2073 	 */
2074 	if (chan->halt_status == DWC2_HC_XFER_URB_DEQUEUE) {
2075 		/*
2076 		 * If the channel was halted, this should be the only
2077 		 * interrupt unmasked
2078 		 */
2079 		WARN_ON(hcint != HCINTMSK_CHHLTD);
2080 		if (hsotg->core_params->dma_desc_enable > 0)
2081 			dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
2082 						    chan->halt_status);
2083 		else
2084 			dwc2_release_channel(hsotg, chan, NULL,
2085 					     chan->halt_status);
2086 		return;
2087 	}
2088 
2089 	if (list_empty(&chan->qh->qtd_list)) {
2090 		/*
2091 		 * TODO: Will this ever happen with the
2092 		 * DWC2_HC_XFER_URB_DEQUEUE handling above?
2093 		 */
2094 		dev_dbg(hsotg->dev, "## no QTD queued for channel %d ##\n",
2095 			chnum);
2096 		dev_dbg(hsotg->dev,
2097 			"  hcint 0x%08x, hcintmsk 0x%08x, hcint&hcintmsk 0x%08x\n",
2098 			chan->hcint, hcintmsk, hcint);
2099 		chan->halt_status = DWC2_HC_XFER_NO_HALT_STATUS;
2100 		disable_hc_int(hsotg, chnum, HCINTMSK_CHHLTD);
2101 		chan->hcint = 0;
2102 		return;
2103 	}
2104 
2105 	qtd = list_first_entry(&chan->qh->qtd_list, struct dwc2_qtd,
2106 			       qtd_list_entry);
2107 
2108 	if (hsotg->core_params->dma_enable <= 0) {
2109 		if ((hcint & HCINTMSK_CHHLTD) && hcint != HCINTMSK_CHHLTD)
2110 			hcint &= ~HCINTMSK_CHHLTD;
2111 	}
2112 
2113 	if (hcint & HCINTMSK_XFERCOMPL) {
2114 		dwc2_hc_xfercomp_intr(hsotg, chan, chnum, qtd);
2115 		/*
2116 		 * If NYET occurred at same time as Xfer Complete, the NYET is
2117 		 * handled by the Xfer Complete interrupt handler. Don't want
2118 		 * to call the NYET interrupt handler in this case.
2119 		 */
2120 		hcint &= ~HCINTMSK_NYET;
2121 	}
2122 
2123 	if (hcint & HCINTMSK_CHHLTD) {
2124 		dwc2_hc_chhltd_intr(hsotg, chan, chnum, qtd);
2125 		if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2126 			goto exit;
2127 	}
2128 	if (hcint & HCINTMSK_AHBERR) {
2129 		dwc2_hc_ahberr_intr(hsotg, chan, chnum, qtd);
2130 		if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2131 			goto exit;
2132 	}
2133 	if (hcint & HCINTMSK_STALL) {
2134 		dwc2_hc_stall_intr(hsotg, chan, chnum, qtd);
2135 		if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2136 			goto exit;
2137 	}
2138 	if (hcint & HCINTMSK_NAK) {
2139 		dwc2_hc_nak_intr(hsotg, chan, chnum, qtd);
2140 		if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2141 			goto exit;
2142 	}
2143 	if (hcint & HCINTMSK_ACK) {
2144 		dwc2_hc_ack_intr(hsotg, chan, chnum, qtd);
2145 		if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2146 			goto exit;
2147 	}
2148 	if (hcint & HCINTMSK_NYET) {
2149 		dwc2_hc_nyet_intr(hsotg, chan, chnum, qtd);
2150 		if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2151 			goto exit;
2152 	}
2153 	if (hcint & HCINTMSK_XACTERR) {
2154 		dwc2_hc_xacterr_intr(hsotg, chan, chnum, qtd);
2155 		if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2156 			goto exit;
2157 	}
2158 	if (hcint & HCINTMSK_BBLERR) {
2159 		dwc2_hc_babble_intr(hsotg, chan, chnum, qtd);
2160 		if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2161 			goto exit;
2162 	}
2163 	if (hcint & HCINTMSK_FRMOVRUN) {
2164 		dwc2_hc_frmovrun_intr(hsotg, chan, chnum, qtd);
2165 		if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2166 			goto exit;
2167 	}
2168 	if (hcint & HCINTMSK_DATATGLERR) {
2169 		dwc2_hc_datatglerr_intr(hsotg, chan, chnum, qtd);
2170 		if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2171 			goto exit;
2172 	}
2173 
2174 exit:
2175 	chan->hcint = 0;
2176 }
2177 
2178 /*
2179  * This interrupt indicates that one or more host channels has a pending
2180  * interrupt. There are multiple conditions that can cause each host channel
2181  * interrupt. This function determines which conditions have occurred for each
2182  * host channel interrupt and handles them appropriately.
2183  */
dwc2_hc_intr(struct dwc2_hsotg * hsotg)2184 static void dwc2_hc_intr(struct dwc2_hsotg *hsotg)
2185 {
2186 	u32 haint;
2187 	int i;
2188 
2189 	haint = DWC2_READ_4(hsotg, HAINT);
2190 	if (dbg_perio()) {
2191 		dev_vdbg(hsotg->dev, "%s()\n", __func__);
2192 
2193 		dev_vdbg(hsotg->dev, "HAINT=%08x\n", haint);
2194 	}
2195 
2196 	for (i = 0; i < hsotg->core_params->host_channels; i++) {
2197 		if (haint & (1 << i))
2198 			dwc2_hc_n_intr(hsotg, i);
2199 	}
2200 }
2201 
2202 /* This function handles interrupts for the HCD */
dwc2_handle_hcd_intr(struct dwc2_hsotg * hsotg)2203 irqreturn_t dwc2_handle_hcd_intr(struct dwc2_hsotg *hsotg)
2204 {
2205 	u32 gintsts, dbg_gintsts;
2206 	irqreturn_t retval = IRQ_NONE;
2207 
2208 	if (!dwc2_is_controller_alive(hsotg)) {
2209 		dev_warn(hsotg->dev, "Controller is dead\n");
2210 		return retval;
2211 	}
2212 
2213 	KASSERT(mutex_owned(&hsotg->lock));
2214 
2215 	/* Check if HOST Mode */
2216 	if (dwc2_is_host_mode(hsotg)) {
2217 		gintsts = dwc2_read_core_intr(hsotg);
2218 		if (!gintsts) {
2219 			return retval;
2220 		}
2221 
2222 		retval = IRQ_HANDLED;
2223 
2224 		dbg_gintsts = gintsts;
2225 #ifndef DEBUG_SOF
2226 		dbg_gintsts &= ~GINTSTS_SOF;
2227 #endif
2228 		if (!dbg_perio())
2229 			dbg_gintsts &= ~(GINTSTS_HCHINT | GINTSTS_RXFLVL |
2230 					 GINTSTS_PTXFEMP);
2231 
2232 		/* Only print if there are any non-suppressed interrupts left */
2233 		if (dbg_gintsts)
2234 			dev_vdbg(hsotg->dev,
2235 				 "DWC OTG HCD Interrupt Detected gintsts&gintmsk=0x%08x\n",
2236 				 gintsts);
2237 
2238 		if (gintsts & GINTSTS_SOF)
2239 			dwc2_sof_intr(hsotg);
2240 		if (gintsts & GINTSTS_RXFLVL)
2241 			dwc2_rx_fifo_level_intr(hsotg);
2242 		if (gintsts & GINTSTS_NPTXFEMP)
2243 			dwc2_np_tx_fifo_empty_intr(hsotg);
2244 		if (gintsts & GINTSTS_PRTINT)
2245 			dwc2_port_intr(hsotg);
2246 		if (gintsts & GINTSTS_HCHINT)
2247 			dwc2_hc_intr(hsotg);
2248 		if (gintsts & GINTSTS_PTXFEMP)
2249 			dwc2_perio_tx_fifo_empty_intr(hsotg);
2250 
2251 		if (dbg_gintsts) {
2252 			dev_vdbg(hsotg->dev,
2253 				 "DWC OTG HCD Finished Servicing Interrupts\n");
2254 			dev_vdbg(hsotg->dev,
2255 				 "DWC OTG HCD gintsts=0x%08x gintmsk=0x%08x\n",
2256 				 DWC2_READ_4(hsotg, GINTSTS),
2257 				 DWC2_READ_4(hsotg, GINTMSK));
2258 		}
2259 	}
2260 
2261 	return retval;
2262 }
2263