xref: /netbsd-src/sys/dev/pci/if_aq.c (revision 481d3881954fd794ca5f2d880b68c53a5db8620e)
1 /*	$NetBSD: if_aq.c,v 1.49 2024/07/05 04:31:51 rin Exp $	*/
2 
3 /**
4  * aQuantia Corporation Network Driver
5  * Copyright (C) 2014-2017 aQuantia Corporation. All rights reserved
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  *
11  *   (1) Redistributions of source code must retain the above
12  *   copyright notice, this list of conditions and the following
13  *   disclaimer.
14  *
15  *   (2) Redistributions in binary form must reproduce the above
16  *   copyright notice, this list of conditions and the following
17  *   disclaimer in the documentation and/or other materials provided
18  *   with the distribution.
19  *
20  *   (3) The name of the author may not be used to endorse or promote
21  *   products derived from this software without specific prior
22  *   written permission.
23  *
24  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
25  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
26  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
28  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
29  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
30  * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
31  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
33  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
34  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35  *
36  */
37 
38 /*-
39  * Copyright (c) 2020 Ryo Shimizu
40  * All rights reserved.
41  *
42  * Redistribution and use in source and binary forms, with or without
43  * modification, are permitted provided that the following conditions
44  * are met:
45  * 1. Redistributions of source code must retain the above copyright
46  *    notice, this list of conditions and the following disclaimer.
47  * 2. Redistributions in binary form must reproduce the above copyright
48  *    notice, this list of conditions and the following disclaimer in the
49  *    documentation and/or other materials provided with the distribution.
50  *
51  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
52  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
53  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
54  * DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
55  * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
56  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
57  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
58  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
59  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
60  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
61  * POSSIBILITY OF SUCH DAMAGE.
62  */
63 
64 #include <sys/cdefs.h>
65 __KERNEL_RCSID(0, "$NetBSD: if_aq.c,v 1.49 2024/07/05 04:31:51 rin Exp $");
66 
67 #ifdef _KERNEL_OPT
68 #include "opt_if_aq.h"
69 #include "sysmon_envsys.h"
70 #endif
71 
72 #include <sys/param.h>
73 #include <sys/types.h>
74 #include <sys/bitops.h>
75 #include <sys/cprng.h>
76 #include <sys/cpu.h>
77 #include <sys/interrupt.h>
78 #include <sys/module.h>
79 #include <sys/pcq.h>
80 
81 #include <net/bpf.h>
82 #include <net/if.h>
83 #include <net/if_dl.h>
84 #include <net/if_media.h>
85 #include <net/if_ether.h>
86 #include <net/rss_config.h>
87 
88 #include <dev/pci/pcivar.h>
89 #include <dev/pci/pcireg.h>
90 #include <dev/pci/pcidevs.h>
91 #include <dev/sysmon/sysmonvar.h>
92 
93 /* driver configuration */
94 #define CONFIG_INTR_MODERATION_ENABLE	true	/* delayed interrupt */
95 #undef CONFIG_LRO_SUPPORT			/* no LRO not supported */
96 #undef CONFIG_NO_TXRX_INDEPENDENT		/* share TX/RX interrupts */
97 
98 #define AQ_NINTR_MAX			(AQ_RSSQUEUE_MAX + AQ_RSSQUEUE_MAX + 1)
99 					/* TX + RX + LINK. must be <= 32 */
100 #define AQ_LINKSTAT_IRQ			31	/* for legacy mode */
101 
102 #define AQ_TXD_NUM			2048	/* per ring. 8*n && 32~8184 */
103 #define AQ_RXD_NUM			2048	/* per ring. 8*n && 32~8184 */
104 /* minimum required to send a packet (vlan needs additional TX descriptor) */
105 #define AQ_TXD_MIN			(1 + 1)
106 
107 
108 /* hardware specification */
109 #define AQ_RINGS_NUM			32
110 #define AQ_RSSQUEUE_MAX			8
111 #define AQ_RX_DESCRIPTOR_MIN		32
112 #define AQ_TX_DESCRIPTOR_MIN		32
113 #define AQ_RX_DESCRIPTOR_MAX		8184
114 #define AQ_TX_DESCRIPTOR_MAX		8184
115 #define AQ_TRAFFICCLASS_NUM		8
116 #define AQ_RSS_HASHKEY_SIZE		40
117 #define AQ_RSS_INDIRECTION_TABLE_MAX	64
118 
119 #define AQ1_JUMBO_MTU_REV_A		9000
120 #define AQ1_JUMBO_MTU_REV_B		16338
121 #define AQ2_JUMBO_MTU			16338
122 
123 /*
124  * TERMINOLOGY
125  *	ATL  (AQ1) = Atlantic.  AQC100,107-109,111,112.
126  *	ATL2 (AQ2) = Atlantic2. AQC113-116.
127  *	MPI = MAC PHY INTERFACE?
128  *	RPO = RX Protocol Offloading
129  *	TPO = TX Protocol Offloading
130  *	RPF = RX Packet Filter
131  *	TPB = TX Packet buffer
132  *	RPB = RX Packet buffer
133  *	ART = Action Resolver Table
134  *	TC  = Traffic Class
135  */
136 
137 enum aq_hwtype {
138 	HWTYPE_AQ1,
139 	HWTYPE_AQ2
140 };
141 
142 /* registers */
143 #define AQ_FW_SOFTRESET_REG			0x0000
144 #define  AQ_FW_SOFTRESET_RESET			__BIT(15) /* soft reset bit */
145 #define  AQ_FW_SOFTRESET_DIS			__BIT(14) /* reset disable */
146 
147 #define AQ_FW_VERSION_REG			0x0018
148 #define AQ_HW_REVISION_REG			0x001c
149 #define AQ2_HW_FPGA_VERSION_REG			0x00f4	/* AQ2 */
150 #define AQ_GLB_NVR_INTERFACE1_REG		0x0100
151 
152 #define AQ_FW_MBOX_CMD_REG			0x0200
153 #define  AQ_FW_MBOX_CMD_EXECUTE			0x00008000
154 #define  AQ_FW_MBOX_CMD_BUSY			0x00000100
155 #define AQ_FW_MBOX_ADDR_REG			0x0208
156 #define AQ_FW_MBOX_VAL_REG			0x020c
157 
158 #define FW2X_LED_MIN_VERSION			0x03010026	/* >= 3.1.38 */
159 #define FW2X_LED_REG				0x031c
160 #define  FW2X_LED_DEFAULT			0x00000000
161 #define  FW2X_LED_NONE				0x0000003f
162 #define  FW2X_LINKLED				__BITS(0,1)
163 #define   FW2X_LINKLED_ACTIVE			0
164 #define   FW2X_LINKLED_ON			1
165 #define   FW2X_LINKLED_BLINK			2
166 #define   FW2X_LINKLED_OFF			3
167 #define  FW2X_STATUSLED				__BITS(2,5)
168 #define   FW2X_STATUSLED_ORANGE			0
169 #define   FW2X_STATUSLED_ORANGE_BLINK		2
170 #define   FW2X_STATUSLED_OFF			3
171 #define   FW2X_STATUSLED_GREEN			4
172 #define   FW2X_STATUSLED_ORANGE_GREEN_BLINK	8
173 #define   FW2X_STATUSLED_GREEN_BLINK		10
174 
175 #define FW_MPI_MBOX_ADDR_REG			0x0360
176 #define FW1X_MPI_INIT1_REG			0x0364
177 #define FW1X_MPI_CONTROL_REG			0x0368
178 #define FW1X_MPI_STATE_REG			0x036c
179 #define  FW1X_MPI_STATE_MODE			__BITS(7,0)
180 #define  FW1X_MPI_STATE_SPEED			__BITS(32,16)
181 #define  FW1X_MPI_STATE_DISABLE_DIRTYWAKE	__BITS(25)
182 #define  FW1X_MPI_STATE_DOWNSHIFT		__BITS(31,28)
183 #define FW1X_MPI_INIT2_REG			0x0370
184 #define FW1X_MPI_EFUSEADDR_REG			0x0374
185 
186 #define FW2X_MPI_EFUSEADDR_REG			0x0364
187 #define FW2X_MPI_CONTROL_REG			0x0368	/* 64bit */
188 #define FW2X_MPI_STATE_REG			0x0370	/* 64bit */
189 #define FW_BOOT_EXIT_CODE_REG			0x0388
190 #define  RBL_STATUS_DEAD			0x0000dead
191 #define  RBL_STATUS_SUCCESS			0x0000abba
192 #define  RBL_STATUS_FAILURE			0x00000bad
193 #define  RBL_STATUS_HOST_BOOT			0x0000f1a7
194 
195 #define AQ_FW_GLB_CPU_SEM_REG(i)		(0x03a0 + (i) * 4)
196 #define AQ1_FW_SEM_RAM_REG			AQ_FW_GLB_CPU_SEM_REG(2)
197 #define AQ2_ART_SEM_REG				AQ_FW_GLB_CPU_SEM_REG(3)
198 
199 #define AQ_FW_GLB_CTL2_REG			0x0404
200 #define  AQ_FW_GLB_CTL2_MCP_UP_FORCE_INTERRUPT	__BIT(1)
201 
202 #define AQ_GLB_GENERAL_PROVISIONING9_REG	0x0520
203 #define AQ_GLB_NVR_PROVISIONING2_REG		0x0534
204 
205 #define FW_MPI_DAISY_CHAIN_STATUS_REG		0x0704
206 
207 #define AQ_PCI_REG_CONTROL_6_REG		0x1014
208 
209 /* msix bitmap */
210 #define AQ_INTR_STATUS_REG			0x2000	/* intr status */
211 #define AQ_INTR_STATUS_CLR_REG			0x2050	/* intr status clear */
212 #define AQ_INTR_MASK_REG			0x2060	/* intr mask set */
213 #define AQ_INTR_MASK_CLR_REG			0x2070	/* intr mask clear */
214 #define AQ_INTR_AUTOMASK_REG			0x2090
215 
216 /* AQ_INTR_IRQ_MAP_TXRX_REG[AQ_RINGS_NUM] 0x2100-0x2140 */
217 #define AQ_INTR_IRQ_MAP_TXRX_REG(i)		(0x2100 + ((i) / 2) * 4)
218 #define AQ_INTR_IRQ_MAP_TX_REG(i)		AQ_INTR_IRQ_MAP_TXRX_REG(i)
219 #define  AQ_INTR_IRQ_MAP_TX_IRQMAP(i)		(__BITS(28,24) >> (((i) & 1)*8))
220 #define  AQ_INTR_IRQ_MAP_TX_EN(i)		(__BIT(31)     >> (((i) & 1)*8))
221 #define AQ_INTR_IRQ_MAP_RX_REG(i)		AQ_INTR_IRQ_MAP_TXRX_REG(i)
222 #define  AQ_INTR_IRQ_MAP_RX_IRQMAP(i)		(__BITS(12,8)  >> (((i) & 1)*8))
223 #define  AQ_INTR_IRQ_MAP_RX_EN(i)		(__BIT(15)     >> (((i) & 1)*8))
224 
225 /* AQ_GEN_INTR_MAP_REG[AQ_RINGS_NUM] 0x2180-0x2200 */
226 #define AQ_GEN_INTR_MAP_REG(i)			(0x2180 + (i) * 4)
227 #define  AQ_B0_ERR_INT				8
228 
229 #define AQ_INTR_CTRL_REG			0x2300
230 #define  AQ_INTR_CTRL_IRQMODE			__BITS(1,0)
231 #define  AQ_INTR_CTRL_IRQMODE_LEGACY		0
232 #define  AQ_INTR_CTRL_IRQMODE_MSI		1
233 #define  AQ_INTR_CTRL_IRQMODE_MSIX		2
234 #define  AQ_INTR_CTRL_MULTIVEC			__BIT(2)
235 #define  AQ_INTR_CTRL_AUTO_MASK			__BIT(5)
236 #define  AQ_INTR_CTRL_CLR_ON_READ		__BIT(7)
237 #define  AQ_INTR_CTRL_RESET_DIS			__BIT(29)
238 #define  AQ_INTR_CTRL_RESET_IRQ			__BIT(31)
239 
240 #define AQ_MBOXIF_POWER_GATING_CONTROL_REG	0x32a8
241 
242 #define FW_MPI_RESETCTRL_REG			0x4000
243 #define  FW_MPI_RESETCTRL_RESET_DIS		__BIT(29)
244 
245 #define RX_SYSCONTROL_REG			0x5000
246 #define  RX_SYSCONTROL_RPF_TPO_SYS_LOOPBACK	__BIT(8)
247 #define  RX_SYSCONTROL_RPB_DMA_SYS_LOOPBACK	__BIT(6)
248 #define  RX_SYSCONTROL_RPB_DMA_NET_LOOPBACK	__BIT(4)
249 #define  RX_SYSCONTROL_RESET_DIS		__BIT(29)
250 
251 #define RX_TCP_RSS_HASH_REG			0x5040
252 #define  RX_TCP_RSS_HASH_RPF2			__BITS(19,16)
253 #define  RX_TCP_RSS_HASH_TYPE			__BITS(15,0)
254 
255 /* for RPF_*_REG.ACTION */
256 #define RPF_ACTION_DISCARD			0
257 #define RPF_ACTION_HOST				1
258 #define RPF_ACTION_MANAGEMENT			2
259 #define RPF_ACTION_HOST_MANAGEMENT		3
260 #define RPF_ACTION_WOL				4
261 
262 #define RPF_L2BC_REG				0x5100
263 #define  RPF_L2BC_EN				__BIT(0)
264 #define  RPF_L2BC_PROMISC			__BIT(3)
265 #define  RPF_L2BC_ACTION			__BITS(12,14)
266 #define  RPF_L2BC_THRESHOLD			__BITS(31,16)
267 
268 /* RPF_L2UC_*_REG[34] (AQ2 has [38]) */
269 #define RPF_L2UC_LSW_REG(i)			(0x5110 + (i) * 8)
270 #define RPF_L2UC_MSW_REG(i)			(0x5114 + (i) * 8)
271 #define  RPF_L2UC_MSW_MACADDR_HI		__BITS(15,0)
272 #define  RPF_L2UC_MSW_ACTION			__BITS(18,16)
273 #define  RPF_L2UC_MSW_TAG			__BITS(27,22)	/* AQ2 */
274 #define  RPF_L2UC_MSW_EN			__BIT(31)
275 
276 #define AQ_HW_MAC_OWN				0 /* index of own address */
277 #define AQ1_HW_MAC_NUM				34
278 #define AQ2_HW_MAC_NUM				38
279 #define AQ_HW_MAC_NUM(sc)			\
280 	(HWTYPE_AQ2_P((sc)) ? AQ2_HW_MAC_NUM : AQ1_HW_MAC_NUM)
281 
282 /* RPF_MCAST_FILTER_REG[8] 0x5250-0x5270 */
283 #define RPF_MCAST_FILTER_REG(i)			(0x5250 + (i) * 4)
284 #define  RPF_MCAST_FILTER_EN			__BIT(31)
285 #define RPF_MCAST_FILTER_MASK_REG		0x5270
286 #define  RPF_MCAST_FILTER_MASK_ALLMULTI		__BIT(14)
287 
288 #define RPF_VLAN_MODE_REG			0x5280
289 #define  RPF_VLAN_MODE_PROMISC			__BIT(1)
290 #define  RPF_VLAN_MODE_ACCEPT_UNTAGGED		__BIT(2)
291 #define  RPF_VLAN_MODE_UNTAGGED_ACTION		__BITS(5,3)
292 
293 #define RPF_VLAN_TPID_REG			0x5284
294 #define  RPF_VLAN_TPID_OUTER			__BITS(31,16)
295 #define  RPF_VLAN_TPID_INNER			__BITS(15,0)
296 
297 /* RPF_VLAN_FILTER_REG[RPF_VLAN_MAX_FILTERS] 0x5290-0x52d0 */
298 #define RPF_VLAN_MAX_FILTERS			16
299 #define RPF_VLAN_FILTER_REG(i)			(0x5290 + (i) * 4)
300 #define  RPF_VLAN_FILTER_EN			__BIT(31)
301 #define  RPF_VLAN_FILTER_RXQ_EN			__BIT(28)
302 #define  RPF_VLAN_FILTER_RXQ			__BITS(24,20)
303 #define  RPF_VLAN_FILTER_ACTION			__BITS(18,16)
304 #define  RPF_VLAN_FILTER_TAG			__BITS(15,12)	/* AQ2 */
305 #define  RPF_VLAN_FILTER_ID			__BITS(11,0)
306 
307 /* RPF_ETHERTYPE_FILTER_REG[AQ_RINGS_NUM] 0x5300-0x5380 */
308 #define RPF_ETHERTYPE_FILTER_REG(i)		(0x5300 + (i) * 4)
309 #define  RPF_ETHERTYPE_FILTER_EN		__BIT(31)
310 #define  RPF_ETHERTYPE_FILTER_PRIO_EN		__BIT(30)
311 #define  RPF_ETHERTYPE_FILTER_RXQF_EN		__BIT(29)
312 #define  RPF_ETHERTYPE_FILTER_PRIO		__BITS(28,26)
313 #define  RPF_ETHERTYPE_FILTER_RXQF		__BITS(24,20)
314 #define  RPF_ETHERTYPE_FILTER_MNG_RXQF		__BIT(19)
315 #define  RPF_ETHERTYPE_FILTER_ACTION		__BITS(18,16)
316 #define  RPF_ETHERTYPE_FILTER_VAL		__BITS(15,0)
317 
318 /* RPF_L3_FILTER_REG[8] 0x5380-0x53a0 */
319 #define RPF_L3_FILTER_REG(i)			(0x5380 + (i) * 4)
320 #define  RPF_L3_FILTER_L4_EN			__BIT(31)
321 #define  RPF_L3_FILTER_IPV6_EN			__BIT(30)
322 #define  RPF_L3_FILTER_SRCADDR_EN		__BIT(29)
323 #define  RPF_L3_FILTER_DSTADDR_EN		__BIT(28)
324 #define  RPF_L3_FILTER_L4_SRCPORT_EN		__BIT(27)
325 #define  RPF_L3_FILTER_L4_DSTPORT_EN		__BIT(26)
326 #define  RPF_L3_FILTER_L4_PROTO_EN		__BIT(25)
327 #define  RPF_L3_FILTER_ARP_EN			__BIT(24)
328 #define  RPF_L3_FILTER_L4_RXQUEUE_EN		__BIT(23)
329 #define  RPF_L3_FILTER_L4_RXQUEUE_MANAGEMENT_EN	__BIT(22)
330 #define  RPF_L3_FILTER_L4_ACTION		__BITS(16,18)
331 #define  RPF_L3_FILTER_L4_RXQUEUE		__BITS(12,8)
332 #define  RPF_L3_FILTER_L4_PROTO			__BITS(2,0)
333 #define   RPF_L3_FILTER_L4_PROTO_TCP		0
334 #define   RPF_L3_FILTER_L4_PROTO_UDP		1
335 #define   RPF_L3_FILTER_L4_PROTO_SCTP		2
336 #define   RPF_L3_FILTER_L4_PROTO_ICMP		3
337 /* parameters of RPF_L3_FILTER_REG[8] */
338 #define RPF_L3_FILTER_SRCADDR_REG(i)		(0x53b0 + (i) * 4)
339 #define RPF_L3_FILTER_DSTADDR_REG(i)		(0x53d0 + (i) * 4)
340 #define RPF_L3_FILTER_L4_SRCPORT_REG(i)		(0x5400 + (i) * 4)
341 #define RPF_L3_FILTER_L4_DSTPORT_REG(i)		(0x5420 + (i) * 4)
342 
343 #define RX_FLR_RSS_CONTROL1_REG			0x54c0
344 #define  RX_FLR_RSS_CONTROL1_EN			__BIT(31)
345 
346 #define RPF_RPB_RX_TC_UPT_REG			0x54c4
347 #define  RPF_RPB_RX_TC_UPT_MASK(i)		(0x00000007 << ((i) * 4))
348 
349 #define RPF_RSS_KEY_ADDR_REG			0x54d0
350 #define  RPF_RSS_KEY_ADDR			__BITS(4,0)
351 #define  RPF_RSS_KEY_WR_EN			__BIT(5)
352 #define RPF_RSS_KEY_WR_DATA_REG			0x54d4
353 #define RPF_RSS_KEY_RD_DATA_REG			0x54d8
354 
355 #define RPF_RSS_REDIR_ADDR_REG			0x54e0
356 #define  RPF_RSS_REDIR_ADDR			__BITS(3,0)
357 #define  RPF_RSS_REDIR_WR_EN			__BIT(4)
358 
359 #define RPF_RSS_REDIR_WR_DATA_REG		0x54e4
360 #define  RPF_RSS_REDIR_WR_DATA			__BITS(15,0)
361 
362 #define RPO_HWCSUM_REG				0x5580
363 #define  RPO_HWCSUM_IP4CSUM_EN			__BIT(1)
364 #define  RPO_HWCSUM_L4CSUM_EN			__BIT(0) /* TCP/UDP/SCTP */
365 
366 #define RPO_LRO_ENABLE_REG			0x5590
367 
368 #define RPO_LRO_CONF_REG			0x5594
369 #define  RPO_LRO_CONF_QSESSION_LIMIT		__BITS(13,12)
370 #define  RPO_LRO_CONF_TOTAL_DESC_LIMIT		__BITS(6,5)
371 #define  RPO_LRO_CONF_PATCHOPTIMIZATION_EN	__BIT(15)
372 #define  RPO_LRO_CONF_MIN_PAYLOAD_OF_FIRST_PKT	__BITS(4,0)
373 #define RPO_LRO_RSC_MAX_REG			0x5598
374 
375 /* RPO_LRO_LDES_MAX_REG[32/8] 0x55a0-0x55b0 */
376 #define RPO_LRO_LDES_MAX_REG(i)			(0x55a0 + (i / 8) * 4)
377 #define  RPO_LRO_LDES_MAX_MASK(i)		(0x00000003 << ((i & 7) * 4))
378 #define RPO_LRO_TB_DIV_REG			0x5620
379 #define  RPO_LRO_TB_DIV				__BITS(20,31)
380 #define RPO_LRO_INACTIVE_IVAL_REG		0x5620
381 #define  RPO_LRO_INACTIVE_IVAL			__BITS(10,19)
382 #define RPO_LRO_MAX_COALESCING_IVAL_REG		0x5620
383 #define  RPO_LRO_MAX_COALESCING_IVAL		__BITS(9,0)
384 
385 #define RPB_RPF_RX_REG				0x5700
386 #define  RPB_RPF_RX_TC_MODE			__BIT(8)
387 #define  RPB_RPF_RX_FC_MODE			__BITS(5,4)
388 #define  RPB_RPF_RX_BUF_EN			__BIT(0)
389 
390 /* RPB_RXB_BUFSIZE_REG[AQ_TRAFFICCLASS_NUM] 0x5710-0x5790 */
391 #define RPB_RXB_BUFSIZE_REG(i)			(0x5710 + (i) * 0x10)
392 #define  RPB_RXB_BUFSIZE			__BITS(8,0)
393 #define RPB_RXB_XOFF_REG(i)			(0x5714 + (i) * 0x10)
394 #define  RPB_RXB_XOFF_EN			__BIT(31)
395 #define  RPB_RXB_XOFF_THRESH_HI			__BITS(29,16)
396 #define  RPB_RXB_XOFF_THRESH_LO			__BITS(13,0)
397 
398 #define RX_DMA_DESC_CACHE_INIT_REG		0x5a00
399 #define  RX_DMA_DESC_CACHE_INIT			__BIT(0)
400 
401 #define RX_DMA_INT_DESC_WRWB_EN_REG		0x05a30
402 #define  RX_DMA_INT_DESC_WRWB_EN		__BIT(2)
403 #define  RX_DMA_INT_DESC_MODERATE_EN		__BIT(3)
404 
405 /* RX_INTR_MODERATION_CTL_REG[AQ_RINGS_NUM] 0x5a40-0x5ac0 */
406 #define RX_INTR_MODERATION_CTL_REG(i)		(0x5a40 + (i) * 4)
407 #define  RX_INTR_MODERATION_CTL_EN		__BIT(1)
408 #define  RX_INTR_MODERATION_CTL_MIN		__BITS(15,8)
409 #define  RX_INTR_MODERATION_CTL_MAX		__BITS(24,16)
410 
411 /* RX_DMA_DESC_*[AQ_RINGS_NUM] 0x5b00-0x5f00 */
412 #define RX_DMA_DESC_BASE_ADDRLSW_REG(i)		(0x5b00 + (i) * 0x20)
413 #define RX_DMA_DESC_BASE_ADDRMSW_REG(i)		(0x5b04 + (i) * 0x20)
414 #define RX_DMA_DESC_REG(i)			(0x5b08 + (i) * 0x20)
415 #define  RX_DMA_DESC_LEN			__BITS(12,3)	/* RXD_NUM/8 */
416 #define  RX_DMA_DESC_RESET			__BIT(25)
417 #define  RX_DMA_DESC_HEADER_SPLIT		__BIT(28)
418 #define  RX_DMA_DESC_VLAN_STRIP			__BIT(29)
419 #define  RX_DMA_DESC_EN				__BIT(31)
420 #define RX_DMA_DESC_HEAD_PTR_REG(i)		(0x5b0c + (i) * 0x20)
421 #define  RX_DMA_DESC_HEAD_PTR			__BITS(12,0)
422 #define RX_DMA_DESC_TAIL_PTR_REG(i)		(0x5b10 + (i) * 0x20)
423 #define RX_DMA_DESC_BUFSIZE_REG(i)		(0x5b18 + (i) * 0x20)
424 #define  RX_DMA_DESC_BUFSIZE_DATA		__BITS(4,0)
425 #define  RX_DMA_DESC_BUFSIZE_HDR		__BITS(12,8)
426 
427 /* RX_DMA_DCAD_REG[AQ_RINGS_NUM] 0x6100-0x6180 */
428 #define RX_DMA_DCAD_REG(i)			(0x6100 + (i) * 4)
429 #define  RX_DMA_DCAD_CPUID			__BITS(7,0)
430 #define  RX_DMA_DCAD_PAYLOAD_EN			__BIT(29)
431 #define  RX_DMA_DCAD_HEADER_EN			__BIT(30)
432 #define  RX_DMA_DCAD_DESC_EN			__BIT(31)
433 
434 #define RX_DMA_DCA_REG				0x6180
435 #define  RX_DMA_DCA_EN				__BIT(31)
436 #define  RX_DMA_DCA_MODE			__BITS(3,0)
437 
438 /* counters */
439 #define RX_DMA_GOOD_PKT_COUNTERLSW		0x6800
440 #define RX_DMA_GOOD_OCTET_COUNTERLSW		0x6808
441 #define RX_DMA_DROP_PKT_CNT_REG			0x6818
442 #define RX_DMA_COALESCED_PKT_CNT_REG		0x6820
443 
444 #define TX_SYSCONTROL_REG			0x7000
445 #define  TX_SYSCONTROL_TPO_PKT_SYS_LOOPBACK	__BIT(7)
446 #define  TX_SYSCONTROL_TPB_DMA_SYS_LOOPBACK	__BIT(6)
447 #define  TX_SYSCONTROL_TPB_DMA_NET_LOOPBACK	__BIT(4)
448 #define  TX_SYSCONTROL_RESET_DIS		__BIT(29)
449 
450 #define TX_TPO2_REG				0x7040
451 #define  TX_TPO2_EN				__BIT(16)
452 
453 #define TPS_DESC_VM_ARB_MODE_REG		0x7300
454 #define  TPS_DESC_VM_ARB_MODE			__BIT(0)
455 #define TPS_DESC_RATE_REG			0x7310
456 #define  TPS_DESC_RATE_TA_RST			__BIT(31)
457 #define  TPS_DESC_RATE_LIM			__BITS(10,0)
458 #define TPS_DESC_TC_ARB_MODE_REG		0x7200
459 #define  TPS_DESC_TC_ARB_MODE			__BITS(1,0)
460 #define TPS_DATA_TC_ARB_MODE_REG		0x7100
461 #define  TPS_DATA_TC_ARB_MODE			__BIT(0)
462 
463 /* TPS_DATA_TCT_REG[AQ_TRAFFICCLASS_NUM] 0x7110-0x7130 */
464 #define TPS_DATA_TCT_REG(i)			(0x7110 + (i) * 4)
465 #define  TPS_DATA_TCT_CREDIT_MAX		__BITS(16,27)
466 #define  TPS_DATA_TCT_WEIGHT			__BITS(8,0)
467 /* TPS_DATA_TCT_REG[AQ_TRAFFICCLASS_NUM] 0x7210-0x7230 */
468 #define TPS_DESC_TCT_REG(i)			(0x7210 + (i) * 4)
469 #define  TPS_DESC_TCT_CREDIT_MAX		__BITS(16,27)
470 #define  TPS_DESC_TCT_WEIGHT			__BITS(8,0)
471 
472 #define AQ1_HW_TXBUF_MAX			160
473 #define AQ1_HW_RXBUF_MAX			320
474 #define AQ2_HW_TXBUF_MAX			128
475 #define AQ2_HW_RXBUF_MAX			192
476 
477 #define TPO_HWCSUM_REG				0x7800
478 #define  TPO_HWCSUM_IP4CSUM_EN			__BIT(1)
479 #define  TPO_HWCSUM_L4CSUM_EN			__BIT(0) /* TCP/UDP/SCTP */
480 
481 #define TDM_LSO_EN_REG				0x7810
482 
483 #define THM_LSO_TCP_FLAG1_REG			0x7820
484 #define  THM_LSO_TCP_FLAG1_FIRST		__BITS(11,0)
485 #define  THM_LSO_TCP_FLAG1_MID			__BITS(27,16)
486 #define THM_LSO_TCP_FLAG2_REG			0x7824
487 #define  THM_LSO_TCP_FLAG2_LAST			__BITS(11,0)
488 
489 #define TPB_TX_BUF_REG				0x7900
490 #define  TPB_TX_BUF_EN				__BIT(0)
491 #define  TPB_TX_BUF_SCP_INS_EN			__BIT(2)
492 #define  TPB_TX_BUF_CLK_GATE_EN			__BIT(5)
493 #define  TPB_TX_BUF_TC_MODE			__BIT(8)
494 #define  TPB_TX_BUF_TC_Q_RAND_MAP_EN		__BIT(9)	/* AQ2 */
495 
496 /* TPB_TXB_BUFSIZE_REG[AQ_TRAFFICCLASS_NUM] 0x7910-7990 */
497 #define TPB_TXB_BUFSIZE_REG(i)			(0x7910 + (i) * 0x10)
498 #define  TPB_TXB_BUFSIZE			__BITS(7,0)
499 #define TPB_TXB_THRESH_REG(i)			(0x7914 + (i) * 0x10)
500 #define  TPB_TXB_THRESH_HI			__BITS(16,28)
501 #define  TPB_TXB_THRESH_LO			__BITS(12,0)
502 
503 #define AQ_HW_TX_DMA_TOTAL_REQ_LIMIT_REG	0x7b20
504 #define TX_DMA_INT_DESC_WRWB_EN_REG		0x7b40
505 #define  TX_DMA_INT_DESC_WRWB_EN		__BIT(1)
506 #define  TX_DMA_INT_DESC_MODERATE_EN		__BIT(4)
507 
508 /* TX_DMA_DESC_*[AQ_RINGS_NUM] 0x7c00-0x8400 */
509 #define TX_DMA_DESC_BASE_ADDRLSW_REG(i)		(0x7c00 + (i) * 0x40)
510 #define TX_DMA_DESC_BASE_ADDRMSW_REG(i)		(0x7c04 + (i) * 0x40)
511 #define TX_DMA_DESC_REG(i)			(0x7c08 + (i) * 0x40)
512 #define  TX_DMA_DESC_LEN			__BITS(12, 3)	/* TXD_NUM/8 */
513 #define  TX_DMA_DESC_EN				__BIT(31)
514 #define TX_DMA_DESC_HEAD_PTR_REG(i)		(0x7c0c + (i) * 0x40)
515 #define  TX_DMA_DESC_HEAD_PTR			__BITS(12,0)
516 #define TX_DMA_DESC_TAIL_PTR_REG(i)		(0x7c10 + (i) * 0x40)
517 #define TX_DMA_DESC_WRWB_THRESH_REG(i)		(0x7c18 + (i) * 0x40)
518 #define  TX_DMA_DESC_WRWB_THRESH		__BITS(14,8)
519 
520 /* TDM_DCAD_REG[AQ_RINGS_NUM] 0x8400-0x8480 */
521 #define TDM_DCAD_REG(i)				(0x8400 + (i) * 4)
522 #define  TDM_DCAD_CPUID				__BITS(7,0)
523 #define  TDM_DCAD_CPUID_EN			__BIT(31)
524 
525 #define TDM_DCA_REG				0x8480
526 #define  TDM_DCA_EN				__BIT(31)
527 #define  TDM_DCA_MODE				__BITS(3,0)
528 
529 /* TX_INTR_MODERATION_CTL_REG[AQ_RINGS_NUM] 0x8980-0x8a00 */
530 #define TX_INTR_MODERATION_CTL_REG(i)		(0x8980 + (i) * 4)
531 #define  TX_INTR_MODERATION_CTL_EN		__BIT(1)
532 #define  TX_INTR_MODERATION_CTL_MIN		__BITS(15,8)
533 #define  TX_INTR_MODERATION_CTL_MAX		__BITS(24,16)
534 
535 /* AQ2 (ATL2) registers */
536 #define AQ2_QUEUE_MODE				0x0c9c
537 
538 #define AQ2_MIF_HOST_FINISHED_STATUS_WRITE_REG	0x0e00
539 #define AQ2_MIF_HOST_FINISHED_STATUS_READ_REG	0x0e04
540 #define  AQ2_MIF_HOST_FINISHED_STATUS_ACK	__BIT(0)
541 
542 #define AQ2_MCP_HOST_REQ_INT_REG		0x0f00
543 #define  AQ2_MCP_HOST_REQ_INT_READY		__BIT(0)
544 #define AQ2_MCP_HOST_REQ_INT_SET_REG		0x0f04
545 #define AQ2_MCP_HOST_REQ_INT_CLR_REG		0x0f08
546 
547 #define AQ2_PHI_EXT_TAG_REG			0x1000
548 #define  AQ2_PHI_EXT_TAG_ENABLE			__BIT(5)
549 
550 #define AQ2_MIF_BOOT_REG			0x3040
551 #define  AQ2_MIF_BOOT_HOST_DATA_LOADED		__BIT(16)
552 #define  AQ2_MIF_BOOT_BOOT_STARTED		__BIT(24)
553 #define  AQ2_MIF_BOOT_CRASH_INIT		__BIT(27)
554 #define  AQ2_MIF_BOOT_BOOT_CODE_FAILED		__BIT(28)
555 #define  AQ2_MIF_BOOT_FW_INIT_FAILED		__BIT(29)
556 #define  AQ2_MIF_BOOT_FW_INIT_COMP_SUCCESS	__BIT(31)
557 
558 /* ART(Action Resolver Table) */
559 #define AQ2_ART_ACTION_ACT_MASK			__BITS(9,8)
560 #define AQ2_ART_ACTION_RSS_MASK			__BIT(7)
561 #define AQ2_ART_ACTION_INDEX_MASK		__BITS(6,2)
562 #define AQ2_ART_ACTION_ENABLE_MASK		__BIT(0)
563 #define AQ2_ART_ACTION(act, rss, idx, en)		\
564 	(__SHIFTIN((act), AQ2_ART_ACTION_ACT_MASK) |	\
565 	__SHIFTIN((rss), AQ2_ART_ACTION_RSS_MASK) |	\
566 	__SHIFTIN((idx), AQ2_ART_ACTION_INDEX_MASK) |	\
567 	__SHIFTIN((en), AQ2_ART_ACTION_ENABLE_MASK))
568 #define AQ2_ART_ACTION_DROP			AQ2_ART_ACTION(0, 0, 0, 1)
569 #define AQ2_ART_ACTION_DISABLE			AQ2_ART_ACTION(0, 0, 0, 0)
570 #define AQ2_ART_ACTION_ASSIGN_QUEUE(q)		AQ2_ART_ACTION(1, 0, (q), 1)
571 #define AQ2_ART_ACTION_ASSIGN_TC(tc)		AQ2_ART_ACTION(1, 1, (tc), 1)
572 
573 #define AQ2_RPF_TAG_PCP_MASK			__BITS(31,29)
574 #define AQ2_RPF_TAG_FLEX_MASK			__BITS(28,27)
575 #define AQ2_RPF_TAG_UNKNOWN_MASK		__BITS(26,24)
576 #define AQ2_RPF_TAG_L4_MASK			__BITS(23,21)
577 #define AQ2_RPF_TAG_L3_V6_MASK			__BITS(20,18)
578 #define AQ2_RPF_TAG_L3_V4_MASK			__BITS(17,15)
579 #define AQ2_RPF_TAG_UNTAG_MASK			__BIT(14)
580 #define AQ2_RPF_TAG_VLAN_MASK			__BITS(13,10)
581 #define AQ2_RPF_TAG_ET_MASK			__BITS(9,7)
582 #define AQ2_RPF_TAG_ALLMC_MASK			__BIT(6)
583 #define AQ2_RPF_TAG_UC_MASK			__BITS(5,0)
584 
585 /* index of aq2_filter_art_set() */
586 #define AQ2_RPF_INDEX_L2_PROMISC_OFF		0
587 #define AQ2_RPF_INDEX_VLAN_PROMISC_OFF		1
588 #define AQ2_RPF_INDEX_L3L4_USER			8
589 #define AQ2_RPF_INDEX_ET_PCP_USER		24
590 #define AQ2_RPF_INDEX_VLAN_USER			40
591 #define AQ2_RPF_INDEX_PCP_TO_TC			56
592 
593 #define AQ2_RPF_L2BC_TAG_REG			0x50f0
594 #define  AQ2_RPF_L2BC_TAG_MASK			__BITS(5,0)
595 
596 #define AQ2_RPF_NEW_CTRL_REG			0x5104
597 #define  AQ2_RPF_NEW_CTRL_ENABLE		__BIT(11)
598 
599 #define AQ2_RPF_L2UC_TAG_REG(i)			(0x5110 + (i) * 8)
600 #define  AQ2_RPF_L2UC_TAG_MASK			__BITS(27,22)
601 
602 #define AQ2_RPF_REDIR2_REG			0x54c8
603 #define  AQ2_RPF_REDIR2_INDEX			__BIT(12)
604 #define  AQ2_RPF_REDIR2_HASHTYPE		__BITS(8,0)
605 #define  AQ2_RPF_REDIR2_HASHTYPE_NONE		0
606 #define  AQ2_RPF_REDIR2_HASHTYPE_IP		__BIT(0)
607 #define  AQ2_RPF_REDIR2_HASHTYPE_TCP4		__BIT(1)
608 #define  AQ2_RPF_REDIR2_HASHTYPE_UDP4		__BIT(2)
609 #define  AQ2_RPF_REDIR2_HASHTYPE_IP6		__BIT(3)
610 #define  AQ2_RPF_REDIR2_HASHTYPE_TCP6		__BIT(4)
611 #define  AQ2_RPF_REDIR2_HASHTYPE_UDP6		__BIT(5)
612 #define  AQ2_RPF_REDIR2_HASHTYPE_IP6EX		__BIT(6)
613 #define  AQ2_RPF_REDIR2_HASHTYPE_TCP6EX		__BIT(7)
614 #define  AQ2_RPF_REDIR2_HASHTYPE_UDP6EX		__BIT(8)
615 #define  AQ2_RPF_REDIR2_HASHTYPE_ALL		__BITS(8,0)
616 
617 #define AQ2_RX_Q_TC_MAP_REG(i)			(0x5900 + (i) * 4)
618 
619 #define AQ2_RDM_RX_DESC_RD_REQ_LIMIT_REG	0x5a04
620 
621 #define AQ2_RPF_RSS_REDIR_REG(tc, i)		\
622 	(0x6200 + (0x100 * ((tc) >> 2)) + (i) * 4)
623 #define AQ2_RPF_RSS_REDIR_TC_MASK(tc)		\
624 	(__BITS(4,0) << (5 * ((tc) & 3)))
625 
626 #define AQ2_RPF_L3_V6_V4_SELECT_REG		0x6500
627 #define  AQ2_RPF_L3_V6_V4_SELECT_EN		__BIT(23)
628 
629 #define AQ2_RPF_REC_TAB_ENABLE_REG		0x6ff0
630 #define  AQ2_RPF_REC_TAB_ENABLE_MASK		__BITS(15,0)
631 
632 #define AQ2_TX_Q_TC_MAP_REG(i)			(0x799c + (i) * 4)
633 
634 #define AQ2_LAUNCHTIME_CTRL_REG			0x7a1c
635 #define  AQ2_LAUNCHTIME_CTRL_RATIO		__BITS(15,8)
636 #define  AQ2_LAUNCHTIME_CTRL_RATIO_SPEED_QUATER	4
637 #define  AQ2_LAUNCHTIME_CTRL_RATIO_SPEED_HALF	2
638 #define  AQ2_LAUNCHTIME_CTRL_RATIO_SPEED_FULL	1
639 
640 /* AT2_TX_INTR_MODERATION_CTL_REG[AQ_RINGS_NUM] 0x7c28-0x8428 */
641 #define AQ2_TX_INTR_MODERATION_CTL_REG(i)	(0x7c28 + (i) * 0x40)
642 #define  AQ2_TX_INTR_MODERATION_CTL_EN		__BIT(1)
643 #define  AQ2_TX_INTR_MODERATION_CTL_MIN		__BITS(15,8)
644 #define  AQ2_TX_INTR_MODERATION_CTL_MAX		__BITS(24,16)
645 
646 /* RW shared buffer */
647 #define AQ2_FW_INTERFACE_IN_MTU_REG				0x12000
648 #define AQ2_FW_INTERFACE_IN_MAC_ADDRESS_REG			0x12008
649 #define AQ2_FW_INTERFACE_IN_LINK_CONTROL_REG			0x12010
650 #define  AQ2_FW_INTERFACE_IN_LINK_CONTROL_PROMISCUOUS_MODE	__BIT(13)
651 #define  AQ2_FW_INTERFACE_IN_LINK_CONTROL_FRAME_PADDING_REMOVAL_RX __BIT(12)
652 #define  AQ2_FW_INTERFACE_IN_LINK_CONTROL_CRC_FORWARDING	__BIT(11)
653 #define  AQ2_FW_INTERFACE_IN_LINK_CONTROL_TX_PADDING		__BIT(10)
654 #define  AQ2_FW_INTERFACE_IN_LINK_CONTROL_CONTROL_FRAME		__BIT(9)
655 #define  AQ2_FW_INTERFACE_IN_LINK_CONTROL_DISCARD_ERROR_FRAME	__BIT(8)
656 #define  AQ2_FW_INTERFACE_IN_LINK_CONTROL_DISABLE_LENGTH_CHECK	__BIT(7)
657 #define  AQ2_FW_INTERFACE_IN_LINK_CONTROL_FLOW_CONTROL_MODE	__BIT(6)
658 #define  AQ2_FW_INTERFACE_IN_LINK_CONTROL_DISCARD_SHORT_FRAMES	__BIT(5)
659 #define  AQ2_FW_INTERFACE_IN_LINK_CONTROL_DISABLE_CRC_CORRUPTION __BIT(4)
660 #define  AQ2_FW_INTERFACE_IN_LINK_CONTROL_MODE			__BITS(3,0)
661 #define  AQ2_FW_INTERFACE_IN_LINK_CONTROL_MODE_INVALID		0
662 #define  AQ2_FW_INTERFACE_IN_LINK_CONTROL_MODE_ACTIVE		1
663 #define  AQ2_FW_INTERFACE_IN_LINK_CONTROL_MODE_SLEEP_PROXY	2
664 #define  AQ2_FW_INTERFACE_IN_LINK_CONTROL_MODE_LOWPOWER		3
665 #define  AQ2_FW_INTERFACE_IN_LINK_CONTROL_MODE_SHUTDOWN		4
666 
667 #define AQ2_FW_INTERFACE_IN_LINK_OPTIONS_REG			0x12018
668 #define  AQ2_FW_INTERFACE_IN_LINK_OPTIONS_DOWNSHIFT_RETRY	__BITS(31,28)
669 #define  AQ2_FW_INTERFACE_IN_LINK_OPTIONS_DOWNSHIFT		__BIT(27)
670 #define  AQ2_FW_INTERFACE_IN_LINK_OPTIONS_PAUSE_TX		__BIT(25)
671 #define  AQ2_FW_INTERFACE_IN_LINK_OPTIONS_PAUSE_RX		__BIT(24)
672 #define  AQ2_FW_INTERFACE_IN_LINK_OPTIONS_EEE_10G		__BIT(20)
673 #define  AQ2_FW_INTERFACE_IN_LINK_OPTIONS_EEE_5G		__BIT(19)
674 #define  AQ2_FW_INTERFACE_IN_LINK_OPTIONS_EEE_2G5		__BIT(18)
675 #define  AQ2_FW_INTERFACE_IN_LINK_OPTIONS_EEE_1G		__BIT(17)
676 #define  AQ2_FW_INTERFACE_IN_LINK_OPTIONS_EEE_100M		__BIT(16)
677 #define  AQ2_FW_INTERFACE_IN_LINK_OPTIONS_RATE_10G		__BIT(15)
678 #define  AQ2_FW_INTERFACE_IN_LINK_OPTIONS_RATE_N5G		__BIT(14)
679 #define  AQ2_FW_INTERFACE_IN_LINK_OPTIONS_RATE_5G		__BIT(13)
680 #define  AQ2_FW_INTERFACE_IN_LINK_OPTIONS_RATE_N2G5		__BIT(12)
681 #define  AQ2_FW_INTERFACE_IN_LINK_OPTIONS_RATE_2G5		__BIT(11)
682 #define  AQ2_FW_INTERFACE_IN_LINK_OPTIONS_RATE_1G		__BIT(10)
683 #define  AQ2_FW_INTERFACE_IN_LINK_OPTIONS_RATE_100M		__BIT(9)
684 #define  AQ2_FW_INTERFACE_IN_LINK_OPTIONS_RATE_10M		__BIT(8)
685 #define  AQ2_FW_INTERFACE_IN_LINK_OPTIONS_RATE_1G_HD		__BIT(7)
686 #define  AQ2_FW_INTERFACE_IN_LINK_OPTIONS_RATE_100M_HD		__BIT(6)
687 #define  AQ2_FW_INTERFACE_IN_LINK_OPTIONS_RATE_10M_HD		__BIT(5)
688 #define  AQ2_FW_INTERFACE_IN_LINK_OPTIONS_EXTERNAL_LOOPBACK	__BIT(4)
689 #define  AQ2_FW_INTERFACE_IN_LINK_OPTIONS_INTERNAL_LOOPBACK	__BIT(3)
690 #define  AQ2_FW_INTERFACE_IN_LINK_OPTIONS_MINIMAL_LINK_SPEED	__BIT(2)
691 #define  AQ2_FW_INTERFACE_IN_LINK_OPTIONS_LINK_RENEGOTIATE	__BIT(1)
692 #define  AQ2_FW_INTERFACE_IN_LINK_OPTIONS_LINK_UP		__BIT(0)
693 
694 #define AQ2_FW_INTERFACE_IN_THERMAL_SHUTDOWN_REG		0x12020
695 #define  AQ2_FW_INTERFACE_IN_THERMAL_SHUTDOWN_WARN_TEMP		__BITS(24,31)
696 #define  AQ2_FW_INTERFACE_IN_THERMAL_SHUTDOWN_COLD_TEMP		__BITS(16,23)
697 #define  AQ2_FW_INTERFACE_IN_THERMAL_SHUTDOWN_SHUTDOWN_TEMP	__BITS(15,8)
698 #define  AQ2_FW_INTERFACE_IN_THERMAL_SHUTDOWN_WARNING_ENABLE	__BIT(1)
699 #define  AQ2_FW_INTERFACE_IN_THERMAL_SHUTDOWN_ENABLE		__BIT(0)
700 
701 #define AQ2_FW_INTERFACE_IN_SLEEP_PROXY				0x12028
702 #define AQ2_FW_INTERFACE_IN_PAUSE_QUANTA			0x12984
703 
704 #define AQ2_FW_INTERFACE_IN_CABLE_DIAG_CONTROL_REG		0x12a44
705 #define AQ2_FW_INTERFACE_IN_DATA_BUFFER_STATUS_OFF_REG		0x12a4c
706 #define AQ2_FW_INTERFACE_IN_DATA_BUFFER_STATUS_LEN_REG		0x12a50
707 #define AQ2_FW_INTERFACE_IN_REQUEST_POLICY_REG			0x12a58
708 #define  AQ2_FW_INTERFACE_IN_REQUEST_POLICY_MCAST_QUEUE_OR_TC		__BIT(23)
709 #define  AQ2_FW_INTERFACE_IN_REQUEST_POLICY_MCAST_RX_QUEUE_TC_INDEX	__BITS(22,18)
710 #define  AQ2_FW_INTERFACE_IN_REQUEST_POLICY_MCAST_ACCEPT		__BIT(16)
711 #define  AQ2_FW_INTERFACE_IN_REQUEST_POLICY_BCAST_QUEUE_OR_TC		__BIT(15)
712 #define  AQ2_FW_INTERFACE_IN_REQUEST_POLICY_BCAST_RX_QUEUE_TC_INDEX	__BITS(14,10)
713 #define  AQ2_FW_INTERFACE_IN_REQUEST_POLICY_BCAST_ACCEPT		__BIT(8)
714 #define  AQ2_FW_INTERFACE_IN_REQUEST_POLICY_PROMISC_QUEUE_OR_TC		__BIT(7)
715 #define  AQ2_FW_INTERFACE_IN_REQUEST_POLICY_PROMISC_RX_QUEUE_TX_INDEX	__BITS(6,2)
716 #define  AQ2_FW_INTERFACE_IN_REQUEST_POLICY_PROMISC_MCAST		__BIT(1)
717 #define  AQ2_FW_INTERFACE_IN_REQUEST_POLICY_PROMISC_ALL			__BIT(0)
718 
719 /* RO shared buffer */
720 #define AQ2_FW_INTERFACE_OUT_TRANSACTION_ID_REG		0x13000
721 #define  AQ2_FW_INTERFACE_OUT_TRANSACTION_ID_B		__BITS(31,16)
722 #define  AQ2_FW_INTERFACE_OUT_TRANSACTION_ID_A		__BITS(15,0)
723 #define AQ2_FW_INTERFACE_OUT_VERSION_BUNDLE_REG		0x13004
724 #define AQ2_FW_INTERFACE_OUT_VERSION_MAC_REG		0x13008
725 #define AQ2_FW_INTERFACE_OUT_VERSION_PHY_REG		0x1300c
726 #define  AQ2_FW_INTERFACE_OUT_VERSION_BUILD		__BITS(31,16)
727 #define  AQ2_FW_INTERFACE_OUT_VERSION_MINOR		__BITS(8,15)
728 #define  AQ2_FW_INTERFACE_OUT_VERSION_MAJOR		__BITS(7,0)
729 #define AQ2_FW_INTERFACE_OUT_VERSION_IFACE_REG		0x13010
730 #define  AQ2_FW_INTERFACE_OUT_VERSION_IFACE_VER		__BITS(3,0)
731 #define  AQ2_FW_INTERFACE_OUT_VERSION_IFACE_VER_A0	0
732 #define  AQ2_FW_INTERFACE_OUT_VERSION_IFACE_VER_B0	1
733 #define AQ2_FW_INTERFACE_OUT_LINK_STATUS_REG		0x13014
734 #define  AQ2_FW_INTERFACE_OUT_LINK_STATUS_DUPLEX	__BIT(11)
735 #define  AQ2_FW_INTERFACE_OUT_LINK_STATUS_EEE		__BIT(10)
736 #define  AQ2_FW_INTERFACE_OUT_LINK_STATUS_PAUSE_RX	__BIT(9)
737 #define  AQ2_FW_INTERFACE_OUT_LINK_STATUS_PAUSE_TX	__BIT(8)
738 #define  AQ2_FW_INTERFACE_OUT_LINK_STATUS_RATE		__BITS(7,4)
739 #define  AQ2_FW_INTERFACE_OUT_LINK_STATUS_RATE_10G	6
740 #define  AQ2_FW_INTERFACE_OUT_LINK_STATUS_RATE_5G	5
741 #define  AQ2_FW_INTERFACE_OUT_LINK_STATUS_RATE_2G5	4
742 #define  AQ2_FW_INTERFACE_OUT_LINK_STATUS_RATE_1G	3
743 #define  AQ2_FW_INTERFACE_OUT_LINK_STATUS_RATE_100M	2
744 #define  AQ2_FW_INTERFACE_OUT_LINK_STATUS_RATE_10M	1
745 #define  AQ2_FW_INTERFACE_OUT_LINK_STATUS_RATE_INVALID	0
746 #define  AQ2_FW_INTERFACE_OUT_LINK_STATUS_STATE		__BITS(3,0)
747 #define AQ2_FW_INTERFACE_OUT_WOL_STATUS_REG		0x13018
748 
749 #define AQ2_FW_INTERFACE_OUT_MAC_HEALTH_MONITOR		0x13610
750 #define AQ2_FW_INTERFACE_OUT_PHY_HEALTH_MONITOR		0x13620
751 typedef struct aq2_health_monitor {
752 	uint32_t data1;
753 #define HEALTH_MONITOR_DATA1_READY			__BIT(0)
754 #define HEALTH_MONITOR_DATA1_FAULT			__BIT(1)
755 #define HEALTH_MONITOR_DATA1_FLASHLESS_FINISHED		__BIT(2)
756 #define HEALTH_MONITOR_DATA1_HOT_WARNING		__BIT(2)
757 #define HEALTH_MONITOR_DATA1_TEMPERATURE		__BITS(15,8)
758 #define HEALTH_MONITOR_DATA1_HEARTBEAT			__BITS(31,16)
759 	uint32_t data2;
760 #define HEALTH_MONITOR_DATA2_FAULTCODE			__BITS(15,0)
761 } aq2_health_monitor_t;
762 
763 #define AQ2_FW_INTERFACE_OUT_CABLE_DIAG_STATUS_LANE_REG(i) (0x13630 + (i) * 4)
764 #define AQ2_FW_INTERFACE_OUT_CABLE_DIAG_STATUS_IDSTAT_REG	0x13640
765 #define  AQ2_FW_INTERFACE_OUT_CABLE_DIAG_STATUS_IDSTAT_ID	__BITS(7,0)
766 #define  AQ2_FW_INTERFACE_OUT_CABLE_DIAG_STATUS_IDSTAT_STATUS	__BITS(8,11)
767 
768 #define AQ2_FW_INTERFACE_OUT_DEVICE_LINK_CAPS_REG	0x13648
769 #define AQ2_FW_INTERFACE_OUT_SLEEP_PROXY_CAPS_REG	0x13650
770 #define AQ2_FW_INTERFACE_OUT_LKP_LINK_CAPS_REG		0x13660
771 #define  AQ2_FW_INTERFACE_OUT_LKP_LINK_CAPS_PAUSE_TX	__BIT(25)
772 #define  AQ2_FW_INTERFACE_OUT_LKP_LINK_CAPS_PAUSE_RX	__BIT(24)
773 #define  AQ2_FW_INTERFACE_OUT_LKP_LINK_CAPS_EEE_10G	__BIT(23)
774 #define  AQ2_FW_INTERFACE_OUT_LKP_LINK_CAPS_EEE_5G	__BIT(21)
775 #define  AQ2_FW_INTERFACE_OUT_LKP_LINK_CAPS_EEE_2G5	__BIT(19)
776 #define  AQ2_FW_INTERFACE_OUT_LKP_LINK_CAPS_EEE_1G	__BIT(18)
777 #define  AQ2_FW_INTERFACE_OUT_LKP_LINK_CAPS_EEE_100M	__BIT(17)
778 #define  AQ2_FW_INTERFACE_OUT_LKP_LINK_CAPS_RATE_10G	__BIT(15)
779 #define  AQ2_FW_INTERFACE_OUT_LKP_LINK_CAPS_RATE_N5G	__BIT(14)
780 #define  AQ2_FW_INTERFACE_OUT_LKP_LINK_CAPS_RATE_5G	__BIT(13)
781 #define  AQ2_FW_INTERFACE_OUT_LKP_LINK_CAPS_RATE_N2G5	__BIT(12)
782 #define  AQ2_FW_INTERFACE_OUT_LKP_LINK_CAPS_RATE_2G5	__BIT(11)
783 #define  AQ2_FW_INTERFACE_OUT_LKP_LINK_CAPS_RATE_1G	__BIT(10)
784 #define  AQ2_FW_INTERFACE_OUT_LKP_LINK_CAPS_RATE_100M	__BIT(9)
785 #define  AQ2_FW_INTERFACE_OUT_LKP_LINK_CAPS_RATE_10M	__BIT(8)
786 #define  AQ2_FW_INTERFACE_OUT_LKP_LINK_CAPS_RATE_1G_HD	__BIT(7)
787 #define  AQ2_FW_INTERFACE_OUT_LKP_LINK_CAPS_RATE_100M_HD __BIT(6)
788 #define  AQ2_FW_INTERFACE_OUT_LKP_LINK_CAPS_RATE_10M_HD	__BIT(5)
789 
790 #define AQ2_FW_INTERFACE_OUT_CORE_DUMP_REG		0x13668
791 #define AQ2_FW_INTERFACE_OUT_STATS_REG			0x13700
792 struct aq2_statistics_a0 {
793 	uint32_t link_up;
794 	uint32_t link_down;
795 	uint64_t tx_unicast_octets;
796 	uint64_t tx_multicast_octets;
797 	uint64_t tx_broadcast_octets;
798 	uint64_t rx_unicast_octets;
799 	uint64_t rx_multicast_octets;
800 	uint64_t rx_broadcast_octets;
801 	uint32_t tx_unicast_frames;
802 	uint32_t tx_multicast_frames;
803 	uint32_t tx_broadcast_frames;
804 	uint32_t tx_errors;
805 	uint32_t rx_unicast_frames;
806 	uint32_t rx_multicast_frames;
807 	uint32_t rx_broadcast_frames;
808 	uint32_t rx_dropped_frames;
809 	uint32_t rx_errors;
810 	uint32_t tx_good_frames;
811 	uint32_t rx_good_frames;
812 	uint32_t reserved1;
813 	uint32_t main_loop_cycles;
814 	uint32_t reserved2;
815 };
816 
817 struct aq2_statistics_b0 {
818 	uint64_t rx_good_octets;
819 	uint64_t rx_pause_frames;
820 	uint64_t rx_good_frames;
821 	uint64_t rx_errors;
822 	uint64_t rx_unicast_frames;
823 	uint64_t rx_multicast_frames;
824 	uint64_t rx_broadcast_frames;
825 	uint64_t tx_good_octets;
826 	uint64_t tx_pause_frames;
827 	uint64_t tx_good_frames;
828 	uint64_t tx_errors;
829 	uint64_t tx_unicast_frames;
830 	uint64_t tx_multicast_frames;
831 	uint64_t tx_broadcast_frames;
832 	uint32_t main_loop_cycles;
833 } __packed;
834 
835 typedef struct aq2_statistics {
836 	union {
837 		struct aq2_statistics_a0 a0;
838 		struct aq2_statistics_b0 b0;
839 	};
840 } aq2_statistics_t;
841 
842 #define AQ2_FW_INTERFACE_OUT_FILTER_CAPS_REG		0x13774
843 typedef struct aq2_filter_caps {
844 	uint32_t caps1;
845 #define  AQ2_FW_INTERFACE_OUT_FILTER_CAPS1_ETHTYPE_FILTER_COUNT		__BITS(24,31)
846 #define  AQ2_FW_INTERFACE_OUT_FILTER_CAPS1_ETHTYPE_FILTER_BASE_INDEX	__BITS(16,23)
847 #define  AQ2_FW_INTERFACE_OUT_FILTER_CAPS1_L2_FILTER_COUNT		__BITS(8,15)
848 #define  AQ2_FW_INTERFACE_OUT_FILTER_CAPS1_FLEXIBLE_FILTER_MASK		__BITS(6,7)
849 #define  AQ2_FW_INTERFACE_OUT_FILTER_CAPS1_L2_FILTER_BASE_INDEX		__BITS(0,5)
850 	uint32_t caps2;
851 #define  AQ2_FW_INTERFACE_OUT_FILTER_CAPS2_L3_IP6_FILTER_COUNT		__BITS(28,31)
852 #define  AQ2_FW_INTERFACE_OUT_FILTER_CAPS2_L3_IP6_FILTER_BASE_INDEX	__BITS(24,27)
853 #define  AQ2_FW_INTERFACE_OUT_FILTER_CAPS2_L3_IP4_FILTER_COUNT		__BITS(20,23)
854 #define  AQ2_FW_INTERFACE_OUT_FILTER_CAPS2_L3_IP4_FILTER_BASE_INDEX	__BITS(16,19)
855 #define  AQ2_FW_INTERFACE_OUT_FILTER_CAPS2_VLAN_FILTER_COUNT		__BITS(8,15)
856 #define  AQ2_FW_INTERFACE_OUT_FILTER_CAPS2_VLAN_FILTER_BASE_INDEX	__BITS(0,7)
857 	uint32_t caps3;
858 #define  AQ2_FW_INTERFACE_OUT_FILTER_CAPS3_RESOLVER_TABLE_COUNT		__BITS(24,31)
859 #define  AQ2_FW_INTERFACE_OUT_FILTER_CAPS3_RESOLVER_BASE_INDEX		__BITS(16,23)
860 #define  AQ2_FW_INTERFACE_OUT_FILTER_CAPS3_L4_FLEX_FILTER_COUNT		__BITS(12,15)
861 #define  AQ2_FW_INTERFACE_OUT_FILTER_CAPS3_L4_FLEX_FILTER_BASE_INDEX	__BITS(8,11)
862 #define  AQ2_FW_INTERFACE_OUT_FILTER_CAPS3_L4_FILTER_COUNT		__BITS(4,7)
863 #define  AQ2_FW_INTERFACE_OUT_FILTER_CAPS3_L4_FILTER_BASE_INDEX		__BITS(0,3)
864 } aq2_filter_caps_t;
865 
866 #define AQ2_FW_INTERFACE_OUT_DEVICE_CAPS_REG		0x13780
867 #define AQ2_FW_INTERFACE_OUT_MANAGEMENT_STATUS_REG	0x1378c
868 #define AQ2_FW_INTERFACE_OUT_TRACE_REG			0x13800
869 
870 #define AQ2_RPF_ACT_ART_REQ_TAG_REG(i)			(0x14000 + (i) * 0x10)
871 #define AQ2_RPF_ACT_ART_REQ_MASK_REG(i)			(0x14004 + (i) * 0x10)
872 #define AQ2_RPF_ACT_ART_REQ_ACTION_REG(i)		(0x14008 + (i) * 0x10)
873 
874 #define FW1X_CTRL_10G				__BIT(0)
875 #define FW1X_CTRL_5G				__BIT(1)
876 #define FW1X_CTRL_5GSR				__BIT(2)
877 #define FW1X_CTRL_2G5				__BIT(3)
878 #define FW1X_CTRL_1G				__BIT(4)
879 #define FW1X_CTRL_100M				__BIT(5)
880 
881 #define FW2X_CTRL_10BASET_HD			__BIT(0)
882 #define FW2X_CTRL_10BASET_FD			__BIT(1)
883 #define FW2X_CTRL_100BASETX_HD			__BIT(2)
884 #define FW2X_CTRL_100BASET4_HD			__BIT(3)
885 #define FW2X_CTRL_100BASET2_HD			__BIT(4)
886 #define FW2X_CTRL_100BASETX_FD			__BIT(5)
887 #define FW2X_CTRL_100BASET2_FD			__BIT(6)
888 #define FW2X_CTRL_1000BASET_HD			__BIT(7)
889 #define FW2X_CTRL_1000BASET_FD			__BIT(8)
890 #define FW2X_CTRL_2P5GBASET_FD			__BIT(9)
891 #define FW2X_CTRL_5GBASET_FD			__BIT(10)
892 #define FW2X_CTRL_10GBASET_FD			__BIT(11)
893 #define FW2X_CTRL_RESERVED1			__BIT(32)
894 #define FW2X_CTRL_10BASET_EEE			__BIT(33)
895 #define FW2X_CTRL_RESERVED2			__BIT(34)
896 #define FW2X_CTRL_PAUSE				__BIT(35)
897 #define FW2X_CTRL_ASYMMETRIC_PAUSE		__BIT(36)
898 #define FW2X_CTRL_100BASETX_EEE			__BIT(37)
899 #define FW2X_CTRL_RESERVED3			__BIT(38)
900 #define FW2X_CTRL_RESERVED4			__BIT(39)
901 #define FW2X_CTRL_1000BASET_FD_EEE		__BIT(40)
902 #define FW2X_CTRL_2P5GBASET_FD_EEE		__BIT(41)
903 #define FW2X_CTRL_5GBASET_FD_EEE		__BIT(42)
904 #define FW2X_CTRL_10GBASET_FD_EEE		__BIT(43)
905 #define FW2X_CTRL_RESERVED5			__BIT(44)
906 #define FW2X_CTRL_RESERVED6			__BIT(45)
907 #define FW2X_CTRL_RESERVED7			__BIT(46)
908 #define FW2X_CTRL_RESERVED8			__BIT(47)
909 #define FW2X_CTRL_RESERVED9			__BIT(48)
910 #define FW2X_CTRL_CABLE_DIAG			__BIT(49)
911 #define FW2X_CTRL_TEMPERATURE			__BIT(50)
912 #define FW2X_CTRL_DOWNSHIFT			__BIT(51)
913 #define FW2X_CTRL_PTP_AVB_EN			__BIT(52)
914 #define FW2X_CTRL_MEDIA_DETECT			__BIT(53)
915 #define FW2X_CTRL_LINK_DROP			__BIT(54)
916 #define FW2X_CTRL_SLEEP_PROXY			__BIT(55)
917 #define FW2X_CTRL_WOL				__BIT(56)
918 #define FW2X_CTRL_MAC_STOP			__BIT(57)
919 #define FW2X_CTRL_EXT_LOOPBACK			__BIT(58)
920 #define FW2X_CTRL_INT_LOOPBACK			__BIT(59)
921 #define FW2X_CTRL_EFUSE_AGENT			__BIT(60)
922 #define FW2X_CTRL_WOL_TIMER			__BIT(61)
923 #define FW2X_CTRL_STATISTICS			__BIT(62)
924 #define FW2X_CTRL_TRANSACTION_ID		__BIT(63)
925 
926 #define FW2X_SNPRINTB			\
927 	"\177\020"			\
928 	"b\x23" "PAUSE\0"		\
929 	"b\x24" "ASYMMETRIC-PAUSE\0"	\
930 	"b\x31" "CABLE-DIAG\0"		\
931 	"b\x32" "TEMPERATURE\0"		\
932 	"b\x33" "DOWNSHIFT\0"		\
933 	"b\x34" "PTP-AVB\0"		\
934 	"b\x35" "MEDIA-DETECT\0"	\
935 	"b\x36" "LINK-DROP\0"		\
936 	"b\x37" "SLEEP-PROXY\0"		\
937 	"b\x38" "WOL\0"			\
938 	"b\x39" "MAC-STOP\0"		\
939 	"b\x3a" "EXT-LOOPBACK\0"	\
940 	"b\x3b" "INT-LOOPBACK\0"	\
941 	"b\x3c" "EFUSE-AGENT\0"		\
942 	"b\x3d" "WOL-TIMER\0"		\
943 	"b\x3e" "STATISTICS\0"		\
944 	"b\x3f" "TRANSACTION-ID\0"	\
945 	"\0"
946 
947 #define FW2X_CTRL_RATE_100M			FW2X_CTRL_100BASETX_FD
948 #define FW2X_CTRL_RATE_1G			FW2X_CTRL_1000BASET_FD
949 #define FW2X_CTRL_RATE_2G5			FW2X_CTRL_2P5GBASET_FD
950 #define FW2X_CTRL_RATE_5G			FW2X_CTRL_5GBASET_FD
951 #define FW2X_CTRL_RATE_10G			FW2X_CTRL_10GBASET_FD
952 #define FW2X_CTRL_RATE_MASK		\
953 	(FW2X_CTRL_RATE_100M |		\
954 	 FW2X_CTRL_RATE_1G |		\
955 	 FW2X_CTRL_RATE_2G5 |		\
956 	 FW2X_CTRL_RATE_5G |		\
957 	 FW2X_CTRL_RATE_10G)
958 #define FW2X_CTRL_EEE_MASK		\
959 	(FW2X_CTRL_10BASET_EEE |	\
960 	 FW2X_CTRL_100BASETX_EEE |	\
961 	 FW2X_CTRL_1000BASET_FD_EEE |	\
962 	 FW2X_CTRL_2P5GBASET_FD_EEE |	\
963 	 FW2X_CTRL_5GBASET_FD_EEE |	\
964 	 FW2X_CTRL_10GBASET_FD_EEE)
965 
966 typedef enum aq_fw_bootloader_mode {
967 	FW_BOOT_MODE_UNKNOWN = 0,
968 	FW_BOOT_MODE_FLB,
969 	FW_BOOT_MODE_RBL_FLASH,
970 	FW_BOOT_MODE_RBL_HOST_BOOTLOAD
971 } aq_fw_bootloader_mode_t;
972 
973 #define AQ_WRITE_REG(sc, reg, val)				\
974 	bus_space_write_4((sc)->sc_iot, (sc)->sc_ioh, (reg), (val))
975 
976 #define AQ_READ_REG(sc, reg)					\
977 	bus_space_read_4((sc)->sc_iot, (sc)->sc_ioh, (reg))
978 
979 #define AQ_READ_REGS(sc, reg, p, cnt)	\
980 	bus_space_read_region_4((sc)->sc_iot, (sc)->sc_ioh, (reg), (p), (cnt))
981 
982 #define AQ_READ64_REG(sc, reg)					\
983 	((uint64_t)AQ_READ_REG(sc, reg) |			\
984 	(((uint64_t)AQ_READ_REG(sc, (reg) + 4)) << 32))
985 
986 #define AQ_WRITE64_REG(sc, reg, val)				\
987 	do {							\
988 		AQ_WRITE_REG(sc, reg, (uint32_t)val);		\
989 		AQ_WRITE_REG(sc, reg + 4, (uint32_t)(val >> 32)); \
990 	} while (/* CONSTCOND */0)
991 
992 #define AQ_READ_REG_BIT(sc, reg, mask)				\
993 	__SHIFTOUT(AQ_READ_REG(sc, reg), mask)
994 
995 #define AQ_WRITE_REG_BIT(sc, reg, mask, val)			\
996 	do {							\
997 		uint32_t _v;					\
998 		_v = AQ_READ_REG((sc), (reg));			\
999 		_v &= ~(mask);					\
1000 		if ((val) != 0)					\
1001 			_v |= __SHIFTIN((val), (mask));		\
1002 		AQ_WRITE_REG((sc), (reg), _v);			\
1003 	} while (/* CONSTCOND */ 0)
1004 
1005 #define WAIT_FOR(expr, us, n, errp)				\
1006 	do {							\
1007 		unsigned int _n;				\
1008 		for (_n = n; (!(expr)) && _n != 0; --_n) {	\
1009 			delay((us));				\
1010 		}						\
1011 		if ((errp != NULL)) {				\
1012 			if (_n == 0)				\
1013 				*(errp) = ETIMEDOUT;		\
1014 			else					\
1015 				*(errp) = 0;			\
1016 		}						\
1017 	} while (/* CONSTCOND */ 0)
1018 
1019 #define msec_delay(x)	DELAY(1000 * (x))
1020 
1021 typedef struct aq_mailbox_header {
1022 	uint32_t version;
1023 	uint32_t transaction_id;
1024 	int32_t error;
1025 } __packed __aligned(4) aq_mailbox_header_t;
1026 
1027 typedef struct aq_hw_stats_s {
1028 	uint32_t uprc;
1029 	uint32_t mprc;
1030 	uint32_t bprc;
1031 	uint32_t erpt;
1032 	uint32_t uptc;
1033 	uint32_t mptc;
1034 	uint32_t bptc;
1035 	uint32_t erpr;
1036 	uint32_t mbtc;
1037 	uint32_t bbtc;
1038 	uint32_t mbrc;
1039 	uint32_t bbrc;
1040 	uint32_t ubrc;
1041 	uint32_t ubtc;
1042 	uint32_t ptc;
1043 	uint32_t prc;
1044 	uint32_t dpc;	/* not exists in fw2x_msm_statistics */
1045 	uint32_t cprc;	/* not exists in fw2x_msm_statistics */
1046 } __packed __aligned(4) aq_hw_stats_s_t;
1047 
1048 typedef struct fw1x_mailbox {
1049 	aq_mailbox_header_t header;
1050 	aq_hw_stats_s_t msm;
1051 } __packed __aligned(4) fw1x_mailbox_t;
1052 
1053 typedef struct fw2x_msm_statistics {
1054 	uint32_t uprc;
1055 	uint32_t mprc;
1056 	uint32_t bprc;
1057 	uint32_t erpt;
1058 	uint32_t uptc;
1059 	uint32_t mptc;
1060 	uint32_t bptc;
1061 	uint32_t erpr;
1062 	uint32_t mbtc;
1063 	uint32_t bbtc;
1064 	uint32_t mbrc;
1065 	uint32_t bbrc;
1066 	uint32_t ubrc;
1067 	uint32_t ubtc;
1068 	uint32_t ptc;
1069 	uint32_t prc;
1070 } __packed __aligned(4) fw2x_msm_statistics_t;
1071 
1072 typedef struct fw2x_phy_cable_diag_data {
1073 	uint32_t lane_data[4];
1074 } __packed __aligned(4) fw2x_phy_cable_diag_data_t;
1075 
1076 typedef struct fw2x_capabilities {
1077 	uint32_t caps_lo;
1078 	uint32_t caps_hi;
1079 } __packed __aligned(4) fw2x_capabilities_t;
1080 
1081 typedef struct fw2x_mailbox {		/* struct fwHostInterface */
1082 	aq_mailbox_header_t header;
1083 	fw2x_msm_statistics_t msm;	/* msmStatistics_t msm; */
1084 
1085 	uint32_t phy_info1;
1086 #define PHYINFO1_FAULT_CODE	__BITS(31,16)
1087 #define PHYINFO1_PHY_H_BIT	__BITS(0,15)
1088 	uint32_t phy_info2;
1089 #define PHYINFO2_TEMPERATURE	__BITS(15,0)
1090 #define PHYINFO2_CABLE_LEN	__BITS(23,16)
1091 
1092 	fw2x_phy_cable_diag_data_t diag_data;
1093 	uint32_t reserved[8];
1094 
1095 	fw2x_capabilities_t caps;
1096 
1097 	/* ... */
1098 } __packed __aligned(4) fw2x_mailbox_t;
1099 
1100 typedef enum aq_link_speed {
1101 	AQ_LINK_NONE	= 0,
1102 	AQ_LINK_10G	= __BIT(0),
1103 	AQ_LINK_5G	= __BIT(1),
1104 	AQ_LINK_2G5	= __BIT(2),
1105 	AQ_LINK_1G	= __BIT(3),
1106 	AQ_LINK_100M	= __BIT(4),
1107 	AQ_LINK_10M	= __BIT(5)
1108 } aq_link_speed_t;
1109 #define AQ_LINK_ALL	(AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5 | \
1110 			 AQ_LINK_5G | AQ_LINK_10G )
1111 #define AQ_LINK_AUTO	__BITS(31, 0)
1112 
1113 typedef enum aq_link_fc {
1114 	AQ_FC_NONE = 0,
1115 	AQ_FC_RX = __BIT(0),
1116 	AQ_FC_TX = __BIT(1),
1117 	AQ_FC_ALL = (AQ_FC_RX | AQ_FC_TX)
1118 } aq_link_fc_t;
1119 
1120 typedef enum aq_link_eee {
1121 	AQ_EEE_DISABLE = 0,
1122 	AQ_EEE_ENABLE = 1
1123 } aq_link_eee_t;
1124 
1125 typedef enum aq_hw_fw_mpi_state {
1126 	MPI_DEINIT	= 0,
1127 	MPI_RESET	= 1,
1128 	MPI_INIT	= 2,
1129 	MPI_POWER	= 4
1130 } aq_hw_fw_mpi_state_t;
1131 
1132 enum aq_media_type {
1133 	AQ_MEDIA_TYPE_UNKNOWN = 0,
1134 	AQ_MEDIA_TYPE_FIBRE,
1135 	AQ_MEDIA_TYPE_TP
1136 };
1137 
1138 struct aq_rx_desc_read {
1139 	uint64_t buf_addr;
1140 	uint64_t hdr_addr;
1141 } __packed __aligned(8);
1142 
1143 struct aq_rx_desc_wb {
1144 	uint32_t type;
1145 #define RXDESC_TYPE_RSSTYPE		__BITS(3,0)
1146 #define  RXDESC_TYPE_RSSTYPE_NONE		0
1147 #define  RXDESC_TYPE_RSSTYPE_IPV4		2
1148 #define  RXDESC_TYPE_RSSTYPE_IPV6		3
1149 #define  RXDESC_TYPE_RSSTYPE_IPV4_TCP		4
1150 #define  RXDESC_TYPE_RSSTYPE_IPV6_TCP		5
1151 #define  RXDESC_TYPE_RSSTYPE_IPV4_UDP		6
1152 #define  RXDESC_TYPE_RSSTYPE_IPV6_UDP		7
1153 #define RXDESC_TYPE_PKTTYPE_ETHER	__BITS(5,4)
1154 #define  RXDESC_TYPE_PKTTYPE_ETHER_IPV4		0
1155 #define  RXDESC_TYPE_PKTTYPE_ETHER_IPV6		1
1156 #define  RXDESC_TYPE_PKTTYPE_ETHER_OTHERS	2
1157 #define  RXDESC_TYPE_PKTTYPE_ETHER_ARP		3
1158 #define RXDESC_TYPE_PKTTYPE_PROTO	__BITS(8,6)
1159 #define  RXDESC_TYPE_PKTTYPE_PROTO_TCP		0
1160 #define  RXDESC_TYPE_PKTTYPE_PROTO_UDP		1
1161 #define  RXDESC_TYPE_PKTTYPE_PROTO_SCTP		2
1162 #define  RXDESC_TYPE_PKTTYPE_PROTO_ICMP		3
1163 #define  RXDESC_TYPE_PKTTYPE_PROTO_OTHERS	4
1164 #define RXDESC_TYPE_PKTTYPE_VLAN	__BIT(9)
1165 #define RXDESC_TYPE_PKTTYPE_VLAN_DOUBLE	__BIT(10)
1166 #define RXDESC_TYPE_MAC_DMA_ERR		__BIT(12)
1167 #define RXDESC_TYPE_RESERVED		__BITS(18,13)
1168 #define RXDESC_TYPE_IPV4_CSUM_CHECKED	__BIT(19)	/* PKTTYPE_ETHER_IPV4 */
1169 #define RXDESC_TYPE_TCPUDP_CSUM_CHECKED	__BIT(20)
1170 #define RXDESC_TYPE_SPH			__BIT(21)
1171 #define RXDESC_TYPE_HDR_LEN		__BITS(31,22)
1172 	uint32_t rss_hash;
1173 	uint16_t status;
1174 #define RXDESC_STATUS_DD		__BIT(0)
1175 #define RXDESC_STATUS_EOP		__BIT(1)
1176 #define RXDESC_STATUS_MACERR		__BIT(2)
1177 #define RXDESC_STATUS_IPV4_CSUM_NG	__BIT(3)
1178 #define RXDESC_STATUS_TCPUDP_CSUM_ERROR	__BIT(4)
1179 #define RXDESC_STATUS_TCPUDP_CSUM_OK	__BIT(5)
1180 
1181 #define RXDESC_STATUS_STAT		__BITS(2,5)
1182 #define RXDESC_STATUS_ESTAT		__BITS(6,11)
1183 #define RXDESC_STATUS_RSC_CNT		__BITS(12,15)
1184 	uint16_t pkt_len;
1185 	uint16_t next_desc_ptr;
1186 	uint16_t vlan;
1187 } __packed __aligned(4);
1188 
1189 typedef union aq_rx_desc {
1190 	struct aq_rx_desc_read read;
1191 	struct aq_rx_desc_wb wb;
1192 } __packed __aligned(8) aq_rx_desc_t;
1193 
1194 typedef struct aq_tx_desc {
1195 	uint64_t buf_addr;
1196 	uint32_t ctl1;
1197 #define AQ_TXDESC_CTL1_TYPE_MASK	0x00000003
1198 #define AQ_TXDESC_CTL1_TYPE_TXD		0x00000001
1199 #define AQ_TXDESC_CTL1_TYPE_TXC		0x00000002
1200 #define AQ_TXDESC_CTL1_BLEN		__BITS(19,4)	/* TXD */
1201 #define AQ_TXDESC_CTL1_DD		__BIT(20)	/* TXD */
1202 #define AQ_TXDESC_CTL1_EOP		__BIT(21)	/* TXD */
1203 #define AQ_TXDESC_CTL1_CMD_VLAN		__BIT(22)	/* TXD */
1204 #define AQ_TXDESC_CTL1_CMD_FCS		__BIT(23)	/* TXD */
1205 #define AQ_TXDESC_CTL1_CMD_IP4CSUM	__BIT(24)	/* TXD */
1206 #define AQ_TXDESC_CTL1_CMD_L4CSUM	__BIT(25)	/* TXD */
1207 #define AQ_TXDESC_CTL1_CMD_LSO		__BIT(26)	/* TXD */
1208 #define AQ_TXDESC_CTL1_CMD_WB		__BIT(27)	/* TXD */
1209 #define AQ_TXDESC_CTL1_CMD_VXLAN	__BIT(28)	/* TXD */
1210 #define AQ_TXDESC_CTL1_VID		__BITS(15,4)	/* TXC */
1211 #define AQ_TXDESC_CTL1_LSO_IPV6		__BIT(21)	/* TXC */
1212 #define AQ_TXDESC_CTL1_LSO_TCP		__BIT(22)	/* TXC */
1213 	uint32_t ctl2;
1214 #define AQ_TXDESC_CTL2_LEN		__BITS(31,14)
1215 #define AQ_TXDESC_CTL2_CTX_EN		__BIT(13)
1216 #define AQ_TXDESC_CTL2_CTX_IDX		__BIT(12)
1217 } __packed __aligned(8) aq_tx_desc_t;
1218 
1219 struct aq_txring {
1220 	struct aq_softc *txr_sc;
1221 	int txr_index;
1222 	kmutex_t txr_mutex;
1223 	bool txr_active;
1224 	bool txr_stopping;
1225 	bool txr_sending;
1226 	time_t txr_lastsent;
1227 
1228 	pcq_t *txr_pcq;
1229 	void *txr_softint;
1230 
1231 	aq_tx_desc_t *txr_txdesc;	/* aq_tx_desc_t[AQ_TXD_NUM] */
1232 	bus_dmamap_t txr_txdesc_dmamap;
1233 	bus_dma_segment_t txr_txdesc_seg[1];
1234 	bus_size_t txr_txdesc_size;
1235 
1236 	struct {
1237 		struct mbuf *m;
1238 		bus_dmamap_t dmamap;
1239 	} txr_mbufs[AQ_TXD_NUM];
1240 	unsigned int txr_prodidx;
1241 	unsigned int txr_considx;
1242 	int txr_nfree;
1243 };
1244 
1245 struct aq_rxring {
1246 	struct aq_softc *rxr_sc;
1247 	int rxr_index;
1248 	kmutex_t rxr_mutex;
1249 	bool rxr_active;
1250 	bool rxr_discarding;
1251 	bool rxr_stopping;
1252 	struct mbuf *rxr_receiving_m;		/* receiving jumboframe */
1253 	struct mbuf *rxr_receiving_m_last;	/* last mbuf of jumboframe */
1254 
1255 	aq_rx_desc_t *rxr_rxdesc;	/* aq_rx_desc_t[AQ_RXD_NUM] */
1256 	bus_dmamap_t rxr_rxdesc_dmamap;
1257 	bus_dma_segment_t rxr_rxdesc_seg[1];
1258 	bus_size_t rxr_rxdesc_size;
1259 	struct {
1260 		struct mbuf *m;
1261 		bus_dmamap_t dmamap;
1262 	} rxr_mbufs[AQ_RXD_NUM];
1263 	unsigned int rxr_readidx;
1264 };
1265 
1266 struct aq_queue {
1267 	struct aq_softc *sc;
1268 	struct aq_txring txring;
1269 	struct aq_rxring rxring;
1270 };
1271 
1272 struct aq_softc;
1273 struct aq_firmware_ops {
1274 	int (*reset)(struct aq_softc *);
1275 	int (*get_mac_addr)(struct aq_softc *);
1276 	int (*set_mode)(struct aq_softc *, aq_hw_fw_mpi_state_t,
1277 	    aq_link_speed_t, aq_link_fc_t, aq_link_eee_t);
1278 	int (*get_mode)(struct aq_softc *, aq_hw_fw_mpi_state_t *,
1279 	    aq_link_speed_t *, aq_link_fc_t *, aq_link_eee_t *);
1280 	int (*get_stats)(struct aq_softc *, aq_hw_stats_s_t *);
1281 #if NSYSMON_ENVSYS > 0
1282 	int (*get_temperature)(struct aq_softc *, uint32_t *);
1283 #endif
1284 };
1285 
1286 #ifdef AQ_EVENT_COUNTERS
1287 #define AQ_EVCNT_DECL(name)						\
1288 	char sc_evcount_##name##_name[32];				\
1289 	struct evcnt sc_evcount_##name##_ev;
1290 #define AQ_EVCNT_ATTACH(sc, name, desc, evtype)				\
1291 	do {								\
1292 		snprintf((sc)->sc_evcount_##name##_name,		\
1293 		    sizeof((sc)->sc_evcount_##name##_name),		\
1294 		    "%s", desc);					\
1295 		evcnt_attach_dynamic(&(sc)->sc_evcount_##name##_ev,	\
1296 		    (evtype), NULL, device_xname((sc)->sc_dev),		\
1297 		    (sc)->sc_evcount_##name##_name);			\
1298 	} while (/*CONSTCOND*/0)
1299 #define AQ_EVCNT_ATTACH_MISC(sc, name, desc)				\
1300 	AQ_EVCNT_ATTACH(sc, name, desc, EVCNT_TYPE_MISC)
1301 #define AQ_EVCNT_DETACH(sc, name)					\
1302 	if ((sc)->sc_evcount_##name##_name[0] != '\0')			\
1303 		evcnt_detach(&(sc)->sc_evcount_##name##_ev)
1304 #define AQ_EVCNT_ADD(sc, name, val)					\
1305 	((sc)->sc_evcount_##name##_ev.ev_count += (val))
1306 #endif /* AQ_EVENT_COUNTERS */
1307 
1308 #define AQ_LOCK(sc)		mutex_enter(&(sc)->sc_mutex);
1309 #define AQ_UNLOCK(sc)		mutex_exit(&(sc)->sc_mutex);
1310 #define AQ_LOCKED(sc)		KASSERT(mutex_owned(&(sc)->sc_mutex));
1311 
1312 /* lock for firmware interface */
1313 #define AQ_MPI_LOCK(sc)		mutex_enter(&(sc)->sc_mpi_mutex);
1314 #define AQ_MPI_UNLOCK(sc)	mutex_exit(&(sc)->sc_mpi_mutex);
1315 #define AQ_MPI_LOCKED(sc)	KASSERT(mutex_owned(&(sc)->sc_mpi_mutex));
1316 
1317 
1318 struct aq_softc {
1319 	device_t sc_dev;
1320 
1321 	bus_space_tag_t sc_iot;
1322 	bus_space_handle_t sc_ioh;
1323 	bus_size_t sc_iosize;
1324 	bus_dma_tag_t sc_dmat;
1325 
1326 	void *sc_ihs[AQ_NINTR_MAX];
1327 	pci_intr_handle_t *sc_intrs;
1328 
1329 	int sc_tx_irq[AQ_RSSQUEUE_MAX];
1330 	int sc_rx_irq[AQ_RSSQUEUE_MAX];
1331 	int sc_linkstat_irq;
1332 	bool sc_use_txrx_independent_intr;
1333 	bool sc_no_link_intr;
1334 
1335 #if NSYSMON_ENVSYS > 0
1336 	struct sysmon_envsys *sc_sme;
1337 	envsys_data_t sc_sensor_temp;
1338 #endif
1339 
1340 	callout_t sc_tick_ch;
1341 
1342 	int sc_nintrs;
1343 	bool sc_msix;
1344 
1345 	struct aq_queue sc_queue[AQ_RSSQUEUE_MAX];
1346 	int sc_nqueues;
1347 	uint32_t sc_tc_mode;	/* traffic class mode (4 or 8) */
1348 	uint32_t sc_tcs;	/* traffic class num */
1349 
1350 	pci_chipset_tag_t sc_pc;
1351 	pcitag_t sc_pcitag;
1352 	uint16_t sc_product;
1353 	uint16_t sc_revision;
1354 
1355 	kmutex_t sc_mutex;
1356 	kmutex_t sc_mpi_mutex;
1357 
1358 	const struct aq_firmware_ops *sc_fw_ops;
1359 	uint64_t sc_fw_caps;			/* AQ1 */
1360 	aq2_filter_caps_t sc_filter_caps;	/* AQ2 */
1361 	uint32_t sc_filter_art_base_index;	/* AQ2 */
1362 	enum aq_media_type sc_media_type;
1363 	aq_link_speed_t sc_available_rates;
1364 
1365 	aq_link_speed_t sc_link_rate;
1366 	aq_link_fc_t sc_link_fc;
1367 	aq_link_eee_t sc_link_eee;
1368 
1369 	uint32_t sc_fw_version;
1370 #define FW_VERSION_MAJOR(sc)	(((sc)->sc_fw_version >> 24) & 0xff)
1371 #define FW_VERSION_MINOR(sc)	(((sc)->sc_fw_version >> 16) & 0xff)
1372 #define FW_VERSION_BUILD(sc)	((sc)->sc_fw_version & 0xffff)
1373 	uint32_t sc_features;
1374 #define FEATURES_MIPS		0x00000001
1375 #define FEATURES_TPO2		0x00000002
1376 #define FEATURES_RPF2		0x00000004
1377 #define FEATURES_MPI_AQ		0x00000008
1378 #define FEATURES_AQ1_REV_A0	0x01000000
1379 #define FEATURES_AQ1_REV_A	(FEATURES_AQ1_REV_A0)
1380 #define FEATURES_AQ1_REV_B0	0x02000000
1381 #define FEATURES_AQ1_REV_B1	0x04000000
1382 #define FEATURES_AQ1_REV_B	(FEATURES_AQ1_REV_B0 | FEATURES_AQ1_REV_B1)
1383 #define FEATURES_AQ1		(FEATURES_AQ1_REV_A | FEATURES_AQ1_REV_B)
1384 #define FEATURES_AQ2		0x10000000
1385 #define FEATURES_AQ2_IFACE_A0	0x20000000
1386 #define FEATURES_AQ2_IFACE_B0	0x40000000
1387 #define HWTYPE_AQ1_P(sc)	(((sc)->sc_features & FEATURES_AQ1) != 0)
1388 #define HWTYPE_AQ2_P(sc)	(((sc)->sc_features & FEATURES_AQ2) != 0)
1389 
1390 	int sc_max_mtu;
1391 	uint32_t sc_mbox_addr;
1392 
1393 	bool sc_rbl_enabled;
1394 	bool sc_fast_start_enabled;
1395 	bool sc_flash_present;
1396 
1397 	bool sc_intr_moderation_enable;
1398 	bool sc_rss_enable;
1399 
1400 	struct ethercom sc_ethercom;
1401 	struct ether_addr sc_enaddr;
1402 	struct ifmedia sc_media;
1403 	int sc_ec_capenable;		/* last ec_capenable */
1404 	unsigned short sc_if_flags;	/* last if_flags */
1405 
1406 	bool sc_tx_sending;
1407 	bool sc_stopping;
1408 
1409 	struct workqueue *sc_reset_wq;
1410 	struct work sc_reset_work;
1411 	volatile unsigned sc_reset_pending;
1412 
1413 	bool sc_trigger_reset;
1414 
1415 #ifdef AQ_EVENT_COUNTERS
1416 	aq_hw_stats_s_t sc_statistics[2];
1417 	int sc_statistics_idx;
1418 	bool sc_poll_statistics;
1419 
1420 	AQ_EVCNT_DECL(uprc);
1421 	AQ_EVCNT_DECL(mprc);
1422 	AQ_EVCNT_DECL(bprc);
1423 	AQ_EVCNT_DECL(erpt);
1424 	AQ_EVCNT_DECL(uptc);
1425 	AQ_EVCNT_DECL(mptc);
1426 	AQ_EVCNT_DECL(bptc);
1427 	AQ_EVCNT_DECL(erpr);
1428 	AQ_EVCNT_DECL(mbtc);
1429 	AQ_EVCNT_DECL(bbtc);
1430 	AQ_EVCNT_DECL(mbrc);
1431 	AQ_EVCNT_DECL(bbrc);
1432 	AQ_EVCNT_DECL(ubrc);
1433 	AQ_EVCNT_DECL(ubtc);
1434 	AQ_EVCNT_DECL(ptc);
1435 	AQ_EVCNT_DECL(prc);
1436 	AQ_EVCNT_DECL(dpc);
1437 	AQ_EVCNT_DECL(cprc);
1438 #endif
1439 };
1440 
1441 static int aq_match(device_t, cfdata_t, void *);
1442 static void aq_attach(device_t, device_t, void *);
1443 static int aq_detach(device_t, int);
1444 
1445 static int aq_setup_msix(struct aq_softc *, struct pci_attach_args *);
1446 static int aq_setup_legacy(struct aq_softc *, struct pci_attach_args *,
1447     pci_intr_type_t);
1448 
1449 static int aq_ifmedia_change(struct ifnet * const);
1450 static void aq_ifmedia_status(struct ifnet * const, struct ifmediareq *);
1451 static int aq_vlan_cb(struct ethercom *ec, uint16_t vid, bool set);
1452 static int aq_ifflags_cb(struct ethercom *);
1453 static int aq_init(struct ifnet *);
1454 static int aq_init_locked(struct ifnet *);
1455 static void aq_send_common_locked(struct ifnet *, struct aq_softc *,
1456     struct aq_txring *, bool);
1457 static int aq_transmit(struct ifnet *, struct mbuf *);
1458 static void aq_deferred_transmit(void *);
1459 static void aq_start(struct ifnet *);
1460 static void aq_stop(struct ifnet *, int);
1461 static void aq_stop_locked(struct ifnet *, bool);
1462 static int aq_ioctl(struct ifnet *, unsigned long, void *);
1463 
1464 static int aq_txrx_rings_alloc(struct aq_softc *);
1465 static void aq_txrx_rings_free(struct aq_softc *);
1466 static int aq_tx_pcq_alloc(struct aq_softc *, struct aq_txring *);
1467 static void aq_tx_pcq_free(struct aq_softc *, struct aq_txring *);
1468 
1469 static void aq_initmedia(struct aq_softc *);
1470 static void aq_enable_intr(struct aq_softc *, bool, bool);
1471 
1472 static void aq_handle_reset_work(struct work *, void *);
1473 static void aq_unset_stopping_flags(struct aq_softc *);
1474 static void aq_set_stopping_flags(struct aq_softc *);
1475 
1476 #if NSYSMON_ENVSYS > 0
1477 static void aq_temp_refresh(struct sysmon_envsys *, envsys_data_t *);
1478 #endif
1479 static void aq_tick(void *);
1480 static int aq_legacy_intr(void *);
1481 static int aq_link_intr(void *);
1482 static int aq_txrx_intr(void *);
1483 static int aq_tx_intr(void *);
1484 static int aq_rx_intr(void *);
1485 
1486 static int aq_set_linkmode(struct aq_softc *, aq_link_speed_t, aq_link_fc_t,
1487     aq_link_eee_t);
1488 static int aq_get_linkmode(struct aq_softc *, aq_link_speed_t *, aq_link_fc_t *,
1489     aq_link_eee_t *);
1490 
1491 static int aq1_fw_reboot(struct aq_softc *);
1492 static int aq1_fw_version_init(struct aq_softc *);
1493 static int aq_hw_init(struct aq_softc *);
1494 static int aq1_hw_init_ucp(struct aq_softc *);
1495 static int aq_hw_reset(struct aq_softc *);
1496 static int aq1_fw_downld_dwords(struct aq_softc *, uint32_t, uint32_t *,
1497     uint32_t);
1498 static int aq1_get_mac_addr(struct aq_softc *);
1499 static int aq_init_rss(struct aq_softc *);
1500 static int aq_set_capability(struct aq_softc *);
1501 
1502 static int fw1x_reset(struct aq_softc *);
1503 static int fw1x_set_mode(struct aq_softc *, aq_hw_fw_mpi_state_t,
1504     aq_link_speed_t, aq_link_fc_t, aq_link_eee_t);
1505 static int fw1x_get_mode(struct aq_softc *, aq_hw_fw_mpi_state_t *,
1506     aq_link_speed_t *, aq_link_fc_t *, aq_link_eee_t *);
1507 static int fw1x_get_stats(struct aq_softc *, aq_hw_stats_s_t *);
1508 
1509 static int fw2x_reset(struct aq_softc *);
1510 static int fw2x_set_mode(struct aq_softc *, aq_hw_fw_mpi_state_t,
1511     aq_link_speed_t, aq_link_fc_t, aq_link_eee_t);
1512 static int fw2x_get_mode(struct aq_softc *, aq_hw_fw_mpi_state_t *,
1513     aq_link_speed_t *, aq_link_fc_t *, aq_link_eee_t *);
1514 static int fw2x_get_stats(struct aq_softc *, aq_hw_stats_s_t *);
1515 #if NSYSMON_ENVSYS > 0
1516 static int fw2x_get_temperature(struct aq_softc *, uint32_t *);
1517 #endif
1518 
1519 #ifndef AQ_WATCHDOG_TIMEOUT
1520 #define AQ_WATCHDOG_TIMEOUT 5
1521 #endif
1522 static int aq_watchdog_timeout = AQ_WATCHDOG_TIMEOUT;
1523 
1524 static int aq2_fw_reboot(struct aq_softc *);
1525 static int aq2_fw_reset(struct aq_softc *);
1526 static int aq2_get_mac_addr(struct aq_softc *);
1527 static int aq2_fw_set_mode(struct aq_softc *, aq_hw_fw_mpi_state_t,
1528     aq_link_speed_t, aq_link_fc_t, aq_link_eee_t);
1529 static int aq2_fw_get_mode(struct aq_softc *, aq_hw_fw_mpi_state_t *,
1530     aq_link_speed_t *, aq_link_fc_t *, aq_link_eee_t *);
1531 static int aq2_init_filter(struct aq_softc *);
1532 static int aq2_filter_art_set(struct aq_softc *, uint32_t, uint32_t, uint32_t,
1533     uint32_t);
1534 static int aq2_fw_get_stats(struct aq_softc *, aq_hw_stats_s_t *);
1535 #if NSYSMON_ENVSYS > 0
1536 static int aq2_fw_get_temperature(struct aq_softc *, uint32_t *);
1537 #endif
1538 
1539 static const struct aq_firmware_ops aq_fw1x_ops = {
1540 	.reset = fw1x_reset,
1541 	.get_mac_addr = aq1_get_mac_addr,
1542 	.set_mode = fw1x_set_mode,
1543 	.get_mode = fw1x_get_mode,
1544 	.get_stats = fw1x_get_stats,
1545 #if NSYSMON_ENVSYS > 0
1546 	.get_temperature = NULL
1547 #endif
1548 };
1549 
1550 static const struct aq_firmware_ops aq_fw2x_ops = {
1551 	.reset = fw2x_reset,
1552 	.get_mac_addr = aq1_get_mac_addr,
1553 	.set_mode = fw2x_set_mode,
1554 	.get_mode = fw2x_get_mode,
1555 	.get_stats = fw2x_get_stats,
1556 #if NSYSMON_ENVSYS > 0
1557 	.get_temperature = fw2x_get_temperature
1558 #endif
1559 };
1560 
1561 static const struct aq_firmware_ops aq2_fw_ops = {
1562 	.reset = aq2_fw_reset,
1563 	.get_mac_addr = aq2_get_mac_addr,
1564 	.set_mode = aq2_fw_set_mode,
1565 	.get_mode = aq2_fw_get_mode,
1566 	.get_stats = aq2_fw_get_stats,
1567 #if NSYSMON_ENVSYS > 0
1568 	.get_temperature = aq2_fw_get_temperature
1569 #endif
1570 };
1571 
1572 CFATTACH_DECL3_NEW(aq, sizeof(struct aq_softc),
1573     aq_match, aq_attach, aq_detach, NULL, NULL, NULL, DVF_DETACH_SHUTDOWN);
1574 
1575 static const struct aq_product {
1576 	pci_vendor_id_t aq_vendor;
1577 	pci_product_id_t aq_product;
1578 	const char *aq_name;
1579 	enum aq_hwtype aq_hwtype;
1580 	enum aq_media_type aq_media_type;
1581 	aq_link_speed_t aq_available_rates;
1582 } aq_products[] = {
1583 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC100,
1584 	  "Aquantia AQC100 10 Gigabit Network Adapter", HWTYPE_AQ1,
1585 	  AQ_MEDIA_TYPE_FIBRE, AQ_LINK_ALL
1586 	},
1587 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC107,
1588 	  "Aquantia AQC107 10 Gigabit Network Adapter", HWTYPE_AQ1,
1589 	  AQ_MEDIA_TYPE_TP, AQ_LINK_ALL
1590 	},
1591 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC108,
1592 	  "Aquantia AQC108 5 Gigabit Network Adapter", HWTYPE_AQ1,
1593 	  AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5 | AQ_LINK_5G
1594 	},
1595 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC109,
1596 	  "Aquantia AQC109 2.5 Gigabit Network Adapter", HWTYPE_AQ1,
1597 	  AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5
1598 	},
1599 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC111,
1600 	  "Aquantia AQC111 5 Gigabit Network Adapter", HWTYPE_AQ1,
1601 	  AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5 | AQ_LINK_5G
1602 	},
1603 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC112,
1604 	  "Aquantia AQC112 2.5 Gigabit Network Adapter", HWTYPE_AQ1,
1605 	  AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5
1606 	},
1607 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC100S,
1608 	  "Aquantia AQC100S 10 Gigabit Network Adapter", HWTYPE_AQ1,
1609 	  AQ_MEDIA_TYPE_FIBRE, AQ_LINK_ALL
1610 	},
1611 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC107S,
1612 	  "Aquantia AQC107S 10 Gigabit Network Adapter", HWTYPE_AQ1,
1613 	  AQ_MEDIA_TYPE_TP, AQ_LINK_ALL
1614 	},
1615 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC108S,
1616 	  "Aquantia AQC108S 5 Gigabit Network Adapter", HWTYPE_AQ1,
1617 	  AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5 | AQ_LINK_5G
1618 	},
1619 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC109S,
1620 	  "Aquantia AQC109S 2.5 Gigabit Network Adapter", HWTYPE_AQ1,
1621 	  AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5
1622 	},
1623 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC111S,
1624 	  "Aquantia AQC111S 5 Gigabit Network Adapter", HWTYPE_AQ1,
1625 	  AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5 | AQ_LINK_5G
1626 	},
1627 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC112S,
1628 	  "Aquantia AQC112S 2.5 Gigabit Network Adapter", HWTYPE_AQ1,
1629 	  AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5
1630 	},
1631 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_D100,
1632 	  "Aquantia D100 10 Gigabit Network Adapter", HWTYPE_AQ1,
1633 	  AQ_MEDIA_TYPE_FIBRE, AQ_LINK_ALL
1634 	},
1635 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_D107,
1636 	  "Aquantia D107 10 Gigabit Network Adapter", HWTYPE_AQ1,
1637 	  AQ_MEDIA_TYPE_TP, AQ_LINK_ALL
1638 	},
1639 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_D108,
1640 	  "Aquantia D108 5 Gigabit Network Adapter", HWTYPE_AQ1,
1641 	  AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5 | AQ_LINK_5G
1642 	},
1643 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_D109,
1644 	  "Aquantia D109 2.5 Gigabit Network Adapter", HWTYPE_AQ1,
1645 	  AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5
1646 	},
1647 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC113DEV,
1648 	  "Aquantia AQC113DEV 10 Gigabit Network Adapter", HWTYPE_AQ2,
1649 	  AQ_MEDIA_TYPE_TP, AQ_LINK_ALL | AQ_LINK_10M
1650 	},
1651 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC113,
1652 	  "Aquantia AQC113 10 Gigabit Network Adapter", HWTYPE_AQ2,
1653 	  AQ_MEDIA_TYPE_TP, AQ_LINK_ALL | AQ_LINK_10M
1654 	},
1655 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC113C,
1656 	  "Aquantia AQC113C 10 Gigabit Network Adapter", HWTYPE_AQ2,
1657 	  AQ_MEDIA_TYPE_TP, AQ_LINK_ALL | AQ_LINK_10M
1658 	},
1659 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC113CA,
1660 	  "Aquantia AQC113CA 10 Gigabit Network Adapter", HWTYPE_AQ2,
1661 	  AQ_MEDIA_TYPE_TP, AQ_LINK_ALL | AQ_LINK_10M
1662 	},
1663 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC113CS,
1664 	  "Aquantia AQC113CS 10 Gigabit Network Adapter", HWTYPE_AQ2,
1665 	  AQ_MEDIA_TYPE_TP, AQ_LINK_ALL | AQ_LINK_10M
1666 	},
1667 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC114CS,
1668 	  "Aquantia AQC114CS 5 Gigabit Network Adapter", HWTYPE_AQ2,
1669 	  AQ_MEDIA_TYPE_TP,
1670 	  AQ_LINK_10M | AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5 | AQ_LINK_5G
1671 	},
1672 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC115C,
1673 	  "Aquantia AQC115C 2.5 Gigabit Network Adapter", HWTYPE_AQ2,
1674 	  AQ_MEDIA_TYPE_TP,
1675 	  AQ_LINK_10M | AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5
1676 	},
1677 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC116C,
1678 	  "Aquantia AQC116C Gigabit Network Adapter", HWTYPE_AQ2,
1679 	  AQ_MEDIA_TYPE_TP, AQ_LINK_10M | AQ_LINK_100M | AQ_LINK_1G
1680 	}
1681 };
1682 
1683 static const struct aq_product *
aq_lookup(const struct pci_attach_args * pa)1684 aq_lookup(const struct pci_attach_args *pa)
1685 {
1686 	unsigned int i;
1687 
1688 	for (i = 0; i < __arraycount(aq_products); i++) {
1689 		if (PCI_VENDOR(pa->pa_id)  == aq_products[i].aq_vendor &&
1690 		    PCI_PRODUCT(pa->pa_id) == aq_products[i].aq_product)
1691 			return &aq_products[i];
1692 	}
1693 	return NULL;
1694 }
1695 
1696 static int
aq_match(device_t parent,cfdata_t cf,void * aux)1697 aq_match(device_t parent, cfdata_t cf, void *aux)
1698 {
1699 	struct pci_attach_args * const pa = aux;
1700 
1701 	if (aq_lookup(pa) != NULL)
1702 		return 1;
1703 
1704 	return 0;
1705 }
1706 
1707 static void
aq_attach(device_t parent,device_t self,void * aux)1708 aq_attach(device_t parent, device_t self, void *aux)
1709 {
1710 	struct aq_softc * const sc = device_private(self);
1711 	struct pci_attach_args * const pa = aux;
1712 	struct ifnet * const ifp = &sc->sc_ethercom.ec_if;
1713 	pci_chipset_tag_t pc;
1714 	pcitag_t tag;
1715 	pcireg_t command, memtype, bar;
1716 	const struct aq_product *aqp;
1717 	int error;
1718 
1719 	sc->sc_dev = self;
1720 	mutex_init(&sc->sc_mutex, MUTEX_DEFAULT, IPL_NET);
1721 	mutex_init(&sc->sc_mpi_mutex, MUTEX_DEFAULT, IPL_NET);
1722 
1723 	sc->sc_pc = pc = pa->pa_pc;
1724 	sc->sc_pcitag = tag = pa->pa_tag;
1725 	sc->sc_dmat = pci_dma64_available(pa) ? pa->pa_dmat64 : pa->pa_dmat;
1726 
1727 	command = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG);
1728 	command |= PCI_COMMAND_MASTER_ENABLE;
1729 	pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, command);
1730 
1731 	sc->sc_product = PCI_PRODUCT(pa->pa_id);
1732 	sc->sc_revision = PCI_REVISION(pa->pa_class);
1733 
1734 	aqp = aq_lookup(pa);
1735 	KASSERT(aqp != NULL);
1736 
1737 	pci_aprint_devinfo_fancy(pa, "Ethernet controller", aqp->aq_name, 1);
1738 
1739 	bar = pci_conf_read(pc, tag, PCI_BAR(0));
1740 	if ((PCI_MAPREG_MEM_ADDR(bar) == 0) ||
1741 	    (PCI_MAPREG_TYPE(bar) != PCI_MAPREG_TYPE_MEM)) {
1742 		aprint_error_dev(sc->sc_dev, "wrong BAR type\n");
1743 		return;
1744 	}
1745 	memtype = pci_mapreg_type(pc, tag, PCI_BAR(0));
1746 	if (pci_mapreg_map(pa, PCI_BAR(0), memtype, 0, &sc->sc_iot, &sc->sc_ioh,
1747 	    NULL, &sc->sc_iosize) != 0) {
1748 		aprint_error_dev(sc->sc_dev, "unable to map register\n");
1749 		return;
1750 	}
1751 
1752 	switch (aqp->aq_hwtype) {
1753 	case HWTYPE_AQ1:
1754 		error = aq1_fw_reboot(sc);
1755 		break;
1756 	case HWTYPE_AQ2:
1757 		error = aq2_fw_reboot(sc);
1758 		break;
1759 	default:
1760 		error = ENOTSUP;
1761 		break;
1762 	}
1763 	if (error != 0)
1764 		goto attach_failure;
1765 
1766 	/* max queue num is 8, and must be 2^n */
1767 	if (ncpu >= 8)
1768 		sc->sc_nqueues = 8;
1769 	else if (ncpu >= 4)
1770 		sc->sc_nqueues = 4;
1771 	else if (ncpu >= 2)
1772 		sc->sc_nqueues = 2;
1773 	else
1774 		sc->sc_nqueues = 1;
1775 
1776 	sc->sc_tc_mode = (sc->sc_nqueues <= 4) ? 8 : 4;
1777 	sc->sc_tcs = 1;
1778 
1779 	int msixcount = pci_msix_count(pa->pa_pc, pa->pa_tag);
1780 #ifndef CONFIG_NO_TXRX_INDEPENDENT
1781 	if (msixcount >= (sc->sc_nqueues * 2 + 1)) {
1782 		/* TX intrs + RX intrs + LINKSTAT intrs */
1783 		sc->sc_use_txrx_independent_intr = true;
1784 		sc->sc_msix = true;
1785 	} else if (msixcount >= (sc->sc_nqueues * 2)) {
1786 		/* TX intrs + RX intrs */
1787 		sc->sc_use_txrx_independent_intr = true;
1788 		sc->sc_msix = true;
1789 	} else
1790 #endif
1791 	if (msixcount >= (sc->sc_nqueues + 1)) {
1792 		/* TX/RX intrs LINKSTAT intrs */
1793 		sc->sc_use_txrx_independent_intr = false;
1794 		sc->sc_msix = true;
1795 	} else if (msixcount >= sc->sc_nqueues) {
1796 		/* TX/RX intrs */
1797 		sc->sc_use_txrx_independent_intr = false;
1798 		sc->sc_no_link_intr = true;
1799 		sc->sc_msix = true;
1800 	} else {
1801 		/* giving up using MSI-X */
1802 		sc->sc_msix = false;
1803 	}
1804 
1805 	aprint_debug_dev(sc->sc_dev,
1806 	    "ncpu=%d, pci_msix_count=%d."
1807 	    " allocate %d interrupts for %d%s queues%s\n",
1808 	    ncpu, msixcount,
1809 	    (sc->sc_use_txrx_independent_intr ?
1810 	    (sc->sc_nqueues * 2) : sc->sc_nqueues) +
1811 	    (sc->sc_no_link_intr ? 0 : 1),
1812 	    sc->sc_nqueues,
1813 	    sc->sc_use_txrx_independent_intr ? "*2" : "",
1814 	    (sc->sc_no_link_intr) ? "" : ", and link status");
1815 
1816 	if (sc->sc_msix)
1817 		error = aq_setup_msix(sc, pa);
1818 	else
1819 		error = ENODEV;
1820 
1821 	if (error != 0) {
1822 		/* if MSI-X failed, fallback to MSI with single queue */
1823 		sc->sc_use_txrx_independent_intr = false;
1824 		sc->sc_msix = false;
1825 		sc->sc_nqueues = 1;
1826 		sc->sc_no_link_intr = false;
1827 		aprint_debug_dev(sc->sc_dev, "MSI-X failed: %d, trying MSI",
1828 		    error);
1829 		error = aq_setup_legacy(sc, pa, PCI_INTR_TYPE_MSI);
1830 	}
1831 	if (error != 0) {
1832 		/* if MSI failed, fallback to INTx */
1833 		aprint_debug_dev(sc->sc_dev, "MSI failed: %d, trying legacy",
1834 		    error);
1835 		error = aq_setup_legacy(sc, pa, PCI_INTR_TYPE_INTX);
1836 	}
1837 	if (error != 0)
1838 		goto attach_failure;
1839 
1840 	callout_init(&sc->sc_tick_ch, CALLOUT_MPSAFE);
1841 	callout_setfunc(&sc->sc_tick_ch, aq_tick, sc);
1842 
1843 	char wqname[MAXCOMLEN];
1844 	snprintf(wqname, sizeof(wqname), "%sReset", device_xname(sc->sc_dev));
1845 	error = workqueue_create(&sc->sc_reset_wq, wqname,
1846 	    aq_handle_reset_work, sc, PRI_SOFTNET, IPL_SOFTCLOCK,
1847 	    WQ_MPSAFE);
1848 	if (error != 0) {
1849 		aprint_error_dev(sc->sc_dev,
1850 		    "unable to create reset workqueue\n");
1851 		goto attach_failure;
1852 	}
1853 
1854 	sc->sc_intr_moderation_enable = CONFIG_INTR_MODERATION_ENABLE;
1855 
1856 	if (sc->sc_msix && (sc->sc_nqueues > 1))
1857 		sc->sc_rss_enable = true;
1858 	else
1859 		sc->sc_rss_enable = false;
1860 
1861 	error = aq_txrx_rings_alloc(sc);
1862 	if (error != 0)
1863 		goto attach_failure;
1864 
1865 	error = aq_hw_reset(sc);
1866 	if (error != 0)
1867 		goto attach_failure;
1868 
1869 	error = sc->sc_fw_ops->get_mac_addr(sc);
1870 	if (error != 0)
1871 		goto attach_failure;
1872 
1873 	aq_init_rss(sc);
1874 
1875 	error = aq_hw_init(sc);	/* initialize and interrupts */
1876 	if (error != 0)
1877 		goto attach_failure;
1878 
1879 	sc->sc_media_type = aqp->aq_media_type;
1880 	sc->sc_available_rates = aqp->aq_available_rates;
1881 
1882 	sc->sc_ethercom.ec_ifmedia = &sc->sc_media;
1883 	ifmedia_init(&sc->sc_media, IFM_IMASK,
1884 	    aq_ifmedia_change, aq_ifmedia_status);
1885 	aq_initmedia(sc);
1886 
1887 	strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ);
1888 	ifp->if_softc = sc;
1889 	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1890 	ifp->if_extflags = IFEF_MPSAFE;
1891 	ifp->if_baudrate = IF_Gbps(10);
1892 	ifp->if_init = aq_init;
1893 	ifp->if_ioctl = aq_ioctl;
1894 	if (sc->sc_msix && (sc->sc_nqueues > 1))
1895 		ifp->if_transmit = aq_transmit;
1896 	ifp->if_start = aq_start;
1897 	ifp->if_stop = aq_stop;
1898 	ifp->if_watchdog = NULL;
1899 	IFQ_SET_READY(&ifp->if_snd);
1900 
1901 	/* initialize capabilities */
1902 	sc->sc_ethercom.ec_capabilities = 0;
1903 	sc->sc_ethercom.ec_capenable = 0;
1904 #if notyet
1905 	/* TODO */
1906 	sc->sc_ethercom.ec_capabilities |= ETHERCAP_EEE;
1907 #endif
1908 	sc->sc_ethercom.ec_capabilities |=
1909 	    ETHERCAP_JUMBO_MTU |
1910 	    ETHERCAP_VLAN_MTU |
1911 	    ETHERCAP_VLAN_HWTAGGING |
1912 	    ETHERCAP_VLAN_HWFILTER;
1913 	sc->sc_ethercom.ec_capenable |=
1914 	    ETHERCAP_VLAN_HWTAGGING |
1915 	    ETHERCAP_VLAN_HWFILTER;
1916 
1917 	ifp->if_capabilities = 0;
1918 	ifp->if_capenable = 0;
1919 #ifdef CONFIG_LRO_SUPPORT
1920 	ifp->if_capabilities |= IFCAP_LRO;
1921 	ifp->if_capenable |= IFCAP_LRO;
1922 #endif
1923 #if notyet
1924 	/* TSO */
1925 	ifp->if_capabilities |= IFCAP_TSOv4 | IFCAP_TSOv6;
1926 #endif
1927 
1928 	/* TX hardware checksum offloading */
1929 	ifp->if_capabilities |= IFCAP_CSUM_IPv4_Tx;
1930 	ifp->if_capabilities |= IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_TCPv6_Tx;
1931 	ifp->if_capabilities |= IFCAP_CSUM_UDPv4_Tx | IFCAP_CSUM_UDPv6_Tx;
1932 	/* RX hardware checksum offloading */
1933 	ifp->if_capabilities |= IFCAP_CSUM_IPv4_Rx;
1934 	ifp->if_capabilities |= IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_TCPv6_Rx;
1935 	ifp->if_capabilities |= IFCAP_CSUM_UDPv4_Rx | IFCAP_CSUM_UDPv6_Rx;
1936 
1937 	if_initialize(ifp);
1938 	ifp->if_percpuq = if_percpuq_create(ifp);
1939 	if_deferred_start_init(ifp, NULL);
1940 	ether_ifattach(ifp, sc->sc_enaddr.ether_addr_octet);
1941 	ether_set_vlan_cb(&sc->sc_ethercom, aq_vlan_cb);
1942 	ether_set_ifflags_cb(&sc->sc_ethercom, aq_ifflags_cb);
1943 	if_register(ifp);
1944 
1945 	/* only intr about link */
1946 	aq_enable_intr(sc, /*link*/true, /*txrx*/false);
1947 
1948 	/* update media */
1949 	aq_ifmedia_change(ifp);
1950 
1951 #if NSYSMON_ENVSYS > 0
1952 	/* temperature monitoring */
1953 	if (sc->sc_fw_ops != NULL && sc->sc_fw_ops->get_temperature != NULL &&
1954 	    (((sc->sc_fw_caps & FW2X_CTRL_TEMPERATURE) != 0) ||
1955 	    HWTYPE_AQ2_P(sc))) {
1956 		sc->sc_sme = sysmon_envsys_create();
1957 		sc->sc_sme->sme_name = device_xname(self);
1958 		sc->sc_sme->sme_cookie = sc;
1959 		sc->sc_sme->sme_flags = 0;
1960 		sc->sc_sme->sme_refresh = aq_temp_refresh;
1961 		sc->sc_sensor_temp.units = ENVSYS_STEMP;
1962 		sc->sc_sensor_temp.state = ENVSYS_SINVALID;
1963 		snprintf(sc->sc_sensor_temp.desc, ENVSYS_DESCLEN, "PHY");
1964 
1965 		sysmon_envsys_sensor_attach(sc->sc_sme, &sc->sc_sensor_temp);
1966 		if (sysmon_envsys_register(sc->sc_sme)) {
1967 			sysmon_envsys_destroy(sc->sc_sme);
1968 			sc->sc_sme = NULL;
1969 			aprint_debug_dev(sc->sc_dev, "failed to create envsys");
1970 			error = EINVAL;
1971 			goto attach_failure;
1972 		}
1973 
1974 		/*
1975 		 * for unknown reasons, the first call of fw2x_get_temperature()
1976 		 * will always fail (firmware matter?), so run once now.
1977 		 */
1978 		aq_temp_refresh(sc->sc_sme, &sc->sc_sensor_temp);
1979 	}
1980 #endif
1981 
1982 #ifdef AQ_EVENT_COUNTERS
1983 	/* get starting statistics values */
1984 	if (sc->sc_fw_ops != NULL && sc->sc_fw_ops->get_stats != NULL &&
1985 	    sc->sc_fw_ops->get_stats(sc, &sc->sc_statistics[0]) == 0) {
1986 		sc->sc_poll_statistics = true;
1987 	}
1988 
1989 	AQ_EVCNT_ATTACH_MISC(sc, uprc, "RX unicast packet");
1990 	AQ_EVCNT_ATTACH_MISC(sc, bprc, "RX broadcast packet");
1991 	AQ_EVCNT_ATTACH_MISC(sc, mprc, "RX multicast packet");
1992 	AQ_EVCNT_ATTACH_MISC(sc, erpr, "RX error packet");
1993 	AQ_EVCNT_ATTACH_MISC(sc, ubrc, "RX unicast bytes");
1994 	AQ_EVCNT_ATTACH_MISC(sc, bbrc, "RX broadcast bytes");
1995 	AQ_EVCNT_ATTACH_MISC(sc, mbrc, "RX multicast bytes");
1996 	AQ_EVCNT_ATTACH_MISC(sc, prc, "RX good packet");
1997 	AQ_EVCNT_ATTACH_MISC(sc, uptc, "TX unicast packet");
1998 	AQ_EVCNT_ATTACH_MISC(sc, bptc, "TX broadcast packet");
1999 	AQ_EVCNT_ATTACH_MISC(sc, mptc, "TX multicast packet");
2000 	AQ_EVCNT_ATTACH_MISC(sc, erpt, "TX error packet");
2001 	AQ_EVCNT_ATTACH_MISC(sc, ubtc, "TX unicast bytes");
2002 	AQ_EVCNT_ATTACH_MISC(sc, bbtc, "TX broadcast bytes");
2003 	AQ_EVCNT_ATTACH_MISC(sc, mbtc, "TX multicast bytes");
2004 	AQ_EVCNT_ATTACH_MISC(sc, ptc, "TX good packet");
2005 	AQ_EVCNT_ATTACH_MISC(sc, dpc, "DMA drop packet");
2006 	AQ_EVCNT_ATTACH_MISC(sc, cprc, "RX coalesced packet");
2007 #endif
2008 
2009 	if (pmf_device_register(self, NULL, NULL))
2010 		pmf_class_network_register(self, ifp);
2011 	else
2012 		aprint_error_dev(self, "couldn't establish power handler\n");
2013 
2014 	return;
2015 
2016  attach_failure:
2017 	aprint_debug_dev(sc->sc_dev, "attach failed: %d", error);
2018 	aq_detach(self, 0);
2019 }
2020 
2021 static int
aq_detach(device_t self,int flags __unused)2022 aq_detach(device_t self, int flags __unused)
2023 {
2024 	struct aq_softc * const sc = device_private(self);
2025 	struct ifnet * const ifp = &sc->sc_ethercom.ec_if;
2026 	int i;
2027 
2028 	if (sc->sc_dev == NULL)
2029 		return 0;
2030 
2031 	if (sc->sc_iosize != 0) {
2032 		if (ifp->if_softc != NULL) {
2033 			IFNET_LOCK(ifp);
2034 			aq_stop(ifp, 1);
2035 			IFNET_UNLOCK(ifp);
2036 		}
2037 
2038 		for (i = 0; i < AQ_NINTR_MAX; i++) {
2039 			if (sc->sc_ihs[i] != NULL) {
2040 				pci_intr_disestablish(sc->sc_pc, sc->sc_ihs[i]);
2041 				sc->sc_ihs[i] = NULL;
2042 			}
2043 		}
2044 		if (sc->sc_nintrs > 0) {
2045 			callout_stop(&sc->sc_tick_ch);
2046 
2047 			pci_intr_release(sc->sc_pc, sc->sc_intrs,
2048 			    sc->sc_nintrs);
2049 			sc->sc_intrs = NULL;
2050 			sc->sc_nintrs = 0;
2051 		}
2052 
2053 		if (sc->sc_reset_wq != NULL) {
2054 			workqueue_destroy(sc->sc_reset_wq);
2055 			sc->sc_reset_wq = NULL;
2056 		}
2057 
2058 		aq_txrx_rings_free(sc);
2059 
2060 		if (ifp->if_softc != NULL) {
2061 			ether_ifdetach(ifp);
2062 			if_detach(ifp);
2063 		}
2064 
2065 		bus_space_unmap(sc->sc_iot, sc->sc_ioh, sc->sc_iosize);
2066 		sc->sc_iosize = 0;
2067 	}
2068 
2069 #if NSYSMON_ENVSYS > 0
2070 	if (sc->sc_sme != NULL) {
2071 		/* all sensors associated with this will also be detached */
2072 		sysmon_envsys_unregister(sc->sc_sme);
2073 	}
2074 #endif
2075 
2076 #ifdef AQ_EVENT_COUNTERS
2077 	AQ_EVCNT_DETACH(sc, uprc);
2078 	AQ_EVCNT_DETACH(sc, mprc);
2079 	AQ_EVCNT_DETACH(sc, bprc);
2080 	AQ_EVCNT_DETACH(sc, erpt);
2081 	AQ_EVCNT_DETACH(sc, uptc);
2082 	AQ_EVCNT_DETACH(sc, mptc);
2083 	AQ_EVCNT_DETACH(sc, bptc);
2084 	AQ_EVCNT_DETACH(sc, erpr);
2085 	AQ_EVCNT_DETACH(sc, mbtc);
2086 	AQ_EVCNT_DETACH(sc, bbtc);
2087 	AQ_EVCNT_DETACH(sc, mbrc);
2088 	AQ_EVCNT_DETACH(sc, bbrc);
2089 	AQ_EVCNT_DETACH(sc, ubrc);
2090 	AQ_EVCNT_DETACH(sc, ubtc);
2091 	AQ_EVCNT_DETACH(sc, ptc);
2092 	AQ_EVCNT_DETACH(sc, prc);
2093 	AQ_EVCNT_DETACH(sc, dpc);
2094 	AQ_EVCNT_DETACH(sc, cprc);
2095 #endif
2096 
2097 	if (sc->sc_ethercom.ec_ifmedia != NULL) {
2098 		ifmedia_fini(&sc->sc_media);
2099 		sc->sc_ethercom.ec_ifmedia = NULL;
2100 	}
2101 
2102 	mutex_destroy(&sc->sc_mpi_mutex);
2103 	mutex_destroy(&sc->sc_mutex);
2104 	sc->sc_dev = NULL;
2105 
2106 	return 0;
2107 }
2108 
2109 static int
aq_establish_intr(struct aq_softc * sc,int intno,kcpuset_t * affinity,int (* func)(void *),void * arg,const char * xname)2110 aq_establish_intr(struct aq_softc *sc, int intno, kcpuset_t *affinity,
2111     int (*func)(void *), void *arg, const char *xname)
2112 {
2113 	char intrbuf[PCI_INTRSTR_LEN];
2114 	pci_chipset_tag_t pc = sc->sc_pc;
2115 	void *vih;
2116 	const char *intrstr = NULL;
2117 
2118 	intrstr = pci_intr_string(pc, sc->sc_intrs[intno], intrbuf,
2119 	    sizeof(intrbuf));
2120 
2121 	pci_intr_setattr(pc, &sc->sc_intrs[intno], PCI_INTR_MPSAFE, true);
2122 
2123 	vih = pci_intr_establish_xname(pc, sc->sc_intrs[intno],
2124 	    IPL_NET, func, arg, xname);
2125 	if (vih == NULL) {
2126 		aprint_error_dev(sc->sc_dev,
2127 		    "unable to establish MSI-X%s%s for %s\n",
2128 		    intrstr ? " at " : "",
2129 		    intrstr ? intrstr : "", xname);
2130 		return EIO;
2131 	}
2132 	sc->sc_ihs[intno] = vih;
2133 
2134 	if (affinity != NULL) {
2135 		/* Round-robin affinity */
2136 		kcpuset_zero(affinity);
2137 		kcpuset_set(affinity, intno % ncpu);
2138 		interrupt_distribute(vih, affinity, NULL);
2139 	}
2140 
2141 	return 0;
2142 }
2143 
2144 static int
aq_establish_msix_intr(struct aq_softc * sc)2145 aq_establish_msix_intr(struct aq_softc *sc)
2146 {
2147 	kcpuset_t *affinity;
2148 	int error, intno, i;
2149 	char intr_xname[INTRDEVNAMEBUF];
2150 
2151 	kcpuset_create(&affinity, false);
2152 
2153 	intno = 0;
2154 
2155 	if (sc->sc_use_txrx_independent_intr) {
2156 		for (i = 0; i < sc->sc_nqueues; i++) {
2157 			snprintf(intr_xname, sizeof(intr_xname), "%s RX%d",
2158 			    device_xname(sc->sc_dev), i);
2159 			sc->sc_rx_irq[i] = intno;
2160 			error = aq_establish_intr(sc, intno++, affinity,
2161 			   aq_rx_intr, &sc->sc_queue[i].rxring, intr_xname);
2162 			if (error != 0)
2163 				goto fail;
2164 		}
2165 		for (i = 0; i < sc->sc_nqueues; i++) {
2166 			snprintf(intr_xname, sizeof(intr_xname), "%s TX%d",
2167 			    device_xname(sc->sc_dev), i);
2168 			sc->sc_tx_irq[i] = intno;
2169 			error = aq_establish_intr(sc, intno++, affinity,
2170 			    aq_tx_intr, &sc->sc_queue[i].txring, intr_xname);
2171 			if (error != 0)
2172 				goto fail;
2173 		}
2174 	} else {
2175 		for (i = 0; i < sc->sc_nqueues; i++) {
2176 			snprintf(intr_xname, sizeof(intr_xname), "%s TXRX%d",
2177 			    device_xname(sc->sc_dev), i);
2178 			sc->sc_rx_irq[i] = intno;
2179 			sc->sc_tx_irq[i] = intno;
2180 			error = aq_establish_intr(sc, intno++, affinity,
2181 			    aq_txrx_intr, &sc->sc_queue[i], intr_xname);
2182 			if (error != 0)
2183 				goto fail;
2184 		}
2185 	}
2186 
2187 	if (!sc->sc_no_link_intr) {
2188 		snprintf(intr_xname, sizeof(intr_xname), "%s LINK",
2189 		    device_xname(sc->sc_dev));
2190 		sc->sc_linkstat_irq = intno;
2191 		error = aq_establish_intr(sc, intno++, affinity,
2192 		    aq_link_intr, sc, intr_xname);
2193 		if (error != 0)
2194 			goto fail;
2195 	}
2196 
2197 	kcpuset_destroy(affinity);
2198 	return 0;
2199 
2200  fail:
2201 	for (i = 0; i < AQ_NINTR_MAX; i++) {
2202 		if (sc->sc_ihs[i] != NULL) {
2203 			pci_intr_disestablish(sc->sc_pc, sc->sc_ihs[i]);
2204 			sc->sc_ihs[i] = NULL;
2205 		}
2206 	}
2207 
2208 	kcpuset_destroy(affinity);
2209 	return ENOMEM;
2210 }
2211 
2212 static int
aq_setup_msix(struct aq_softc * sc,struct pci_attach_args * pa)2213 aq_setup_msix(struct aq_softc *sc, struct pci_attach_args *pa)
2214 {
2215 	int nqueue = sc->sc_nqueues;
2216 	bool txrx_independent = sc->sc_use_txrx_independent_intr;
2217 	bool linkintr = !sc->sc_no_link_intr;
2218 	int error, nintr;
2219 
2220 	if (txrx_independent)
2221 		nintr = nqueue * 2;
2222 	else
2223 		nintr = nqueue;
2224 
2225 	if (linkintr)
2226 		nintr++;
2227 
2228 	error = pci_msix_alloc_exact(pa, &sc->sc_intrs, nintr);
2229 	if (error != 0) {
2230 		aprint_error_dev(sc->sc_dev,
2231 		    "failed to allocate MSI-X interrupts\n");
2232 		goto fail;
2233 	}
2234 
2235 	error = aq_establish_msix_intr(sc);
2236 	if (error == 0) {
2237 		sc->sc_nintrs = nintr;
2238 	} else {
2239 		pci_intr_release(sc->sc_pc, sc->sc_intrs, nintr);
2240 		sc->sc_nintrs = 0;
2241 	}
2242  fail:
2243 	return error;
2244 
2245 }
2246 
2247 static int
aq_setup_legacy(struct aq_softc * sc,struct pci_attach_args * pa,pci_intr_type_t inttype)2248 aq_setup_legacy(struct aq_softc *sc, struct pci_attach_args *pa,
2249     pci_intr_type_t inttype)
2250 {
2251 	int counts[PCI_INTR_TYPE_SIZE];
2252 	int error, nintr;
2253 
2254 	nintr = 1;
2255 
2256 	memset(counts, 0, sizeof(counts));
2257 	counts[inttype] = nintr;
2258 
2259 	error = pci_intr_alloc(pa, &sc->sc_intrs, counts, inttype);
2260 	if (error != 0) {
2261 		aprint_error_dev(sc->sc_dev,
2262 		    "failed to allocate%s interrupts\n",
2263 		    (inttype == PCI_INTR_TYPE_MSI) ? " MSI" : "");
2264 		return error;
2265 	}
2266 	error = aq_establish_intr(sc, 0, NULL, aq_legacy_intr, sc,
2267 	    device_xname(sc->sc_dev));
2268 	if (error == 0) {
2269 		sc->sc_nintrs = nintr;
2270 	} else {
2271 		pci_intr_release(sc->sc_pc, sc->sc_intrs, nintr);
2272 		sc->sc_nintrs = 0;
2273 	}
2274 	return error;
2275 }
2276 
2277 static void
aq1_global_software_reset(struct aq_softc * sc)2278 aq1_global_software_reset(struct aq_softc *sc)
2279 {
2280 	uint32_t v;
2281 
2282 	AQ_WRITE_REG_BIT(sc, RX_SYSCONTROL_REG, RX_SYSCONTROL_RESET_DIS, 0);
2283 	AQ_WRITE_REG_BIT(sc, TX_SYSCONTROL_REG, TX_SYSCONTROL_RESET_DIS, 0);
2284 	AQ_WRITE_REG_BIT(sc, FW_MPI_RESETCTRL_REG,
2285 	    FW_MPI_RESETCTRL_RESET_DIS, 0);
2286 
2287 	v = AQ_READ_REG(sc, AQ_FW_SOFTRESET_REG);
2288 	v &= ~AQ_FW_SOFTRESET_DIS;
2289 	v |= AQ_FW_SOFTRESET_RESET;
2290 	AQ_WRITE_REG(sc, AQ_FW_SOFTRESET_REG, v);
2291 }
2292 
2293 static int
aq1_mac_soft_reset_rbl(struct aq_softc * sc,aq_fw_bootloader_mode_t * mode)2294 aq1_mac_soft_reset_rbl(struct aq_softc *sc, aq_fw_bootloader_mode_t *mode)
2295 {
2296 	int timo;
2297 
2298 	aprint_debug_dev(sc->sc_dev, "RBL> MAC reset STARTED!\n");
2299 
2300 	AQ_WRITE_REG(sc, AQ_FW_GLB_CTL2_REG, 0x40e1);
2301 	AQ_WRITE_REG(sc, AQ_FW_GLB_CPU_SEM_REG(0), 1);
2302 	AQ_WRITE_REG(sc, AQ_MBOXIF_POWER_GATING_CONTROL_REG, 0);
2303 
2304 	/* MAC FW will reload PHY FW if 1E.1000.3 was cleaned - #undone */
2305 	AQ_WRITE_REG(sc, FW_BOOT_EXIT_CODE_REG, RBL_STATUS_DEAD);
2306 
2307 	aq1_global_software_reset(sc);
2308 
2309 	AQ_WRITE_REG(sc, AQ_FW_GLB_CTL2_REG, 0x40e0);
2310 
2311 	/* Wait for RBL to finish boot process. */
2312 #define RBL_TIMEOUT_MS	10000
2313 	uint16_t rbl_status;
2314 	for (timo = RBL_TIMEOUT_MS; timo > 0; timo--) {
2315 		rbl_status = AQ_READ_REG(sc, FW_BOOT_EXIT_CODE_REG) & 0xffff;
2316 		if (rbl_status != 0 && rbl_status != RBL_STATUS_DEAD)
2317 			break;
2318 		msec_delay(1);
2319 	}
2320 	if (timo <= 0) {
2321 		aprint_error_dev(sc->sc_dev,
2322 		    "RBL> RBL restart failed: timeout\n");
2323 		return EBUSY;
2324 	}
2325 	switch (rbl_status) {
2326 	case RBL_STATUS_SUCCESS:
2327 		if (mode != NULL)
2328 			*mode = FW_BOOT_MODE_RBL_FLASH;
2329 		aprint_debug_dev(sc->sc_dev, "RBL> reset complete! [Flash]\n");
2330 		break;
2331 	case RBL_STATUS_HOST_BOOT:
2332 		if (mode != NULL)
2333 			*mode = FW_BOOT_MODE_RBL_HOST_BOOTLOAD;
2334 		aprint_debug_dev(sc->sc_dev,
2335 		    "RBL> reset complete! [Host Bootload]\n");
2336 		break;
2337 	case RBL_STATUS_FAILURE:
2338 	default:
2339 		aprint_error_dev(sc->sc_dev,
2340 		    "unknown RBL status 0x%x\n", rbl_status);
2341 		return EBUSY;
2342 	}
2343 
2344 	return 0;
2345 }
2346 
2347 static int
aq1_mac_soft_reset_flb(struct aq_softc * sc)2348 aq1_mac_soft_reset_flb(struct aq_softc *sc)
2349 {
2350 	uint32_t v;
2351 	int timo;
2352 
2353 	AQ_WRITE_REG(sc, AQ_FW_GLB_CTL2_REG, 0x40e1);
2354 	/*
2355 	 * Let Felicity hardware to complete SMBUS transaction before
2356 	 * Global software reset.
2357 	 */
2358 	msec_delay(50);
2359 
2360 	/*
2361 	 * If SPI burst transaction was interrupted(before running the script),
2362 	 * global software reset may not clear SPI interface.
2363 	 * Clean it up manually before global reset.
2364 	 */
2365 	AQ_WRITE_REG(sc, AQ_GLB_NVR_PROVISIONING2_REG, 0x00a0);
2366 	AQ_WRITE_REG(sc, AQ_GLB_NVR_INTERFACE1_REG, 0x009f);
2367 	AQ_WRITE_REG(sc, AQ_GLB_NVR_INTERFACE1_REG, 0x809f);
2368 	msec_delay(50);
2369 
2370 	v = AQ_READ_REG(sc, AQ_FW_SOFTRESET_REG);
2371 	v &= ~AQ_FW_SOFTRESET_DIS;
2372 	v |= AQ_FW_SOFTRESET_RESET;
2373 	AQ_WRITE_REG(sc, AQ_FW_SOFTRESET_REG, v);
2374 
2375 	/* Kickstart. */
2376 	AQ_WRITE_REG(sc, AQ_FW_GLB_CTL2_REG, 0x80e0);
2377 	AQ_WRITE_REG(sc, AQ_MBOXIF_POWER_GATING_CONTROL_REG, 0);
2378 	if (!sc->sc_fast_start_enabled)
2379 		AQ_WRITE_REG(sc, AQ_GLB_GENERAL_PROVISIONING9_REG, 1);
2380 
2381 	/*
2382 	 * For the case SPI burst transaction was interrupted (by MCP reset
2383 	 * above), wait until it is completed by hardware.
2384 	 */
2385 	msec_delay(50);
2386 
2387 	/* MAC Kickstart */
2388 	if (!sc->sc_fast_start_enabled) {
2389 		AQ_WRITE_REG(sc, AQ_FW_GLB_CTL2_REG, 0x180e0);
2390 
2391 		uint32_t flb_status;
2392 		for (timo = 0; timo < 1000; timo++) {
2393 			flb_status = AQ_READ_REG(sc,
2394 			    FW_MPI_DAISY_CHAIN_STATUS_REG) & 0x10;
2395 			if (flb_status != 0)
2396 				break;
2397 			msec_delay(1);
2398 		}
2399 		if (flb_status == 0) {
2400 			aprint_error_dev(sc->sc_dev,
2401 			    "FLB> MAC kickstart failed: timed out\n");
2402 			return ETIMEDOUT;
2403 		}
2404 		aprint_debug_dev(sc->sc_dev,
2405 		    "FLB> MAC kickstart done, %d ms\n", timo);
2406 		/* FW reset */
2407 		AQ_WRITE_REG(sc, AQ_FW_GLB_CTL2_REG, 0x80e0);
2408 		/*
2409 		 * Let Felicity hardware complete SMBUS transaction before
2410 		 * Global software reset.
2411 		 */
2412 		msec_delay(50);
2413 		sc->sc_fast_start_enabled = true;
2414 	}
2415 	AQ_WRITE_REG(sc, AQ_FW_GLB_CPU_SEM_REG(0), 1);
2416 
2417 	/* PHY Kickstart: #undone */
2418 	aq1_global_software_reset(sc);
2419 
2420 	for (timo = 0; timo < 1000; timo++) {
2421 		if (AQ_READ_REG(sc, AQ_FW_VERSION_REG) != 0)
2422 			break;
2423 		msec_delay(10);
2424 	}
2425 	if (timo >= 1000) {
2426 		aprint_error_dev(sc->sc_dev, "FLB> Global Soft Reset failed\n");
2427 		return ETIMEDOUT;
2428 	}
2429 	aprint_debug_dev(sc->sc_dev, "FLB> F/W restart: %d ms\n", timo * 10);
2430 	return 0;
2431 
2432 }
2433 
2434 static int
aq1_mac_soft_reset(struct aq_softc * sc,aq_fw_bootloader_mode_t * mode)2435 aq1_mac_soft_reset(struct aq_softc *sc, aq_fw_bootloader_mode_t *mode)
2436 {
2437 	if (sc->sc_rbl_enabled)
2438 		return aq1_mac_soft_reset_rbl(sc, mode);
2439 
2440 	if (mode != NULL)
2441 		*mode = FW_BOOT_MODE_FLB;
2442 	return aq1_mac_soft_reset_flb(sc);
2443 }
2444 
2445 static int
aq1_fw_read_version(struct aq_softc * sc)2446 aq1_fw_read_version(struct aq_softc *sc)
2447 {
2448 	int i, error = EBUSY;
2449 #define MAC_FW_START_TIMEOUT_MS	10000
2450 	for (i = 0; i < MAC_FW_START_TIMEOUT_MS; i++) {
2451 		sc->sc_fw_version = AQ_READ_REG(sc, AQ_FW_VERSION_REG);
2452 		if (sc->sc_fw_version != 0) {
2453 			error = 0;
2454 			break;
2455 		}
2456 		delay(1000);
2457 	}
2458 	return error;
2459 }
2460 
2461 static int
aq1_fw_reboot(struct aq_softc * sc)2462 aq1_fw_reboot(struct aq_softc *sc)
2463 {
2464 	uint32_t ver, v, bootExitCode;
2465 	int i, error;
2466 
2467 	ver = AQ_READ_REG(sc, AQ_FW_VERSION_REG);
2468 
2469 	for (i = 1000; i > 0; i--) {
2470 		v = AQ_READ_REG(sc, FW_MPI_DAISY_CHAIN_STATUS_REG);
2471 		bootExitCode = AQ_READ_REG(sc, FW_BOOT_EXIT_CODE_REG);
2472 		if (v != 0x06000000 || bootExitCode != 0)
2473 			break;
2474 	}
2475 	if (i <= 0) {
2476 		aprint_error_dev(sc->sc_dev,
2477 		    "F/W reset failed. Neither RBL nor FLB started\n");
2478 		return ETIMEDOUT;
2479 	}
2480 	sc->sc_rbl_enabled = (bootExitCode != 0);
2481 
2482 	/*
2483 	 * Having FW version 0 is an indicator that cold start
2484 	 * is in progress. This means two things:
2485 	 * 1) Driver have to wait for FW/HW to finish boot (500ms giveup)
2486 	 * 2) Driver may skip reset sequence and save time.
2487 	 */
2488 	if (sc->sc_fast_start_enabled && (ver != 0)) {
2489 		error = aq1_fw_read_version(sc);
2490 		/* Skip reset as it just completed */
2491 		if (error == 0)
2492 			return 0;
2493 	}
2494 
2495 	aq_fw_bootloader_mode_t mode = FW_BOOT_MODE_UNKNOWN;
2496 	error = aq1_mac_soft_reset(sc, &mode);
2497 	if (error != 0) {
2498 		aprint_error_dev(sc->sc_dev, "MAC reset failed: %d\n", error);
2499 		return ENXIO;
2500 	}
2501 
2502 	switch (mode) {
2503 	case FW_BOOT_MODE_FLB:
2504 		aprint_debug_dev(sc->sc_dev,
2505 		    "FLB> F/W successfully loaded from flash.\n");
2506 		sc->sc_flash_present = true;
2507 		break;
2508 	case FW_BOOT_MODE_RBL_FLASH:
2509 		aprint_debug_dev(sc->sc_dev,
2510 		    "RBL> F/W loaded from flash. Host Bootload disabled.\n");
2511 		sc->sc_flash_present = true;
2512 		break;
2513 	case FW_BOOT_MODE_UNKNOWN:
2514 		aprint_error_dev(sc->sc_dev,
2515 		    "F/W bootload error: unknown bootloader type\n");
2516 		return ENOTSUP;
2517 	case FW_BOOT_MODE_RBL_HOST_BOOTLOAD:
2518 		aprint_debug_dev(sc->sc_dev, "RBL> Host Bootload mode\n");
2519 		aprint_error_dev(sc->sc_dev,
2520 		    "RBL> F/W Host Bootload not implemented\n");
2521 		return ENOTSUP;
2522 	}
2523 
2524 	error = aq1_fw_read_version(sc);
2525 	if (error != 0)
2526 		return error;
2527 
2528 	error = aq1_fw_version_init(sc);
2529 	if (error != 0)
2530 		return error;
2531 
2532 	error = aq1_hw_init_ucp(sc);
2533 	if (error < 0)
2534 		return error;
2535 
2536 	KASSERT(sc->sc_mbox_addr != 0);
2537 	return 0;
2538 }
2539 
2540 static int
aq_hw_reset(struct aq_softc * sc)2541 aq_hw_reset(struct aq_softc *sc)
2542 {
2543 	int error;
2544 
2545 	/* disable irq */
2546 	AQ_WRITE_REG_BIT(sc, AQ_INTR_CTRL_REG, AQ_INTR_CTRL_RESET_DIS, 0);
2547 
2548 	/* apply */
2549 	AQ_WRITE_REG_BIT(sc, AQ_INTR_CTRL_REG, AQ_INTR_CTRL_RESET_IRQ, 1);
2550 
2551 	/* wait ack 10 times by 1ms */
2552 	WAIT_FOR(
2553 	    (AQ_READ_REG(sc, AQ_INTR_CTRL_REG) & AQ_INTR_CTRL_RESET_IRQ) == 0,
2554 	    1000, 10, &error);
2555 	if (error != 0) {
2556 		aprint_error_dev(sc->sc_dev,
2557 		    "atlantic: IRQ reset failed: %d\n", error);
2558 		return error;
2559 	}
2560 
2561 	return sc->sc_fw_ops->reset(sc);
2562 }
2563 
2564 static int
aq1_hw_init_ucp(struct aq_softc * sc)2565 aq1_hw_init_ucp(struct aq_softc *sc)
2566 {
2567 	int timo;
2568 
2569 	if (FW_VERSION_MAJOR(sc) == 1) {
2570 		if (AQ_READ_REG(sc, FW1X_MPI_INIT2_REG) == 0)
2571 			AQ_WRITE_REG(sc, FW1X_MPI_INIT2_REG, 0xfefefefe);
2572 		AQ_WRITE_REG(sc, FW1X_MPI_INIT1_REG, 0);
2573 	}
2574 
2575 	/* Wait a maximum of 10sec. It usually takes about 5sec. */
2576 	for (timo = 10000; timo > 0; timo--) {
2577 		sc->sc_mbox_addr = AQ_READ_REG(sc, FW_MPI_MBOX_ADDR_REG);
2578 		if (sc->sc_mbox_addr != 0)
2579 			break;
2580 		delay(1000);
2581 	}
2582 	if (sc->sc_mbox_addr == 0) {
2583 		aprint_error_dev(sc->sc_dev, "cannot get mbox addr\n");
2584 		return ETIMEDOUT;
2585 	}
2586 
2587 #define AQ_FW_MIN_VERSION	0x01050006
2588 #define AQ_FW_MIN_VERSION_STR	"1.5.6"
2589 	if (sc->sc_fw_version < AQ_FW_MIN_VERSION) {
2590 		aprint_error_dev(sc->sc_dev,
2591 		    "atlantic: wrong FW version: " AQ_FW_MIN_VERSION_STR
2592 		    " or later required, this is %d.%d.%d\n",
2593 		    FW_VERSION_MAJOR(sc),
2594 		    FW_VERSION_MINOR(sc),
2595 		    FW_VERSION_BUILD(sc));
2596 		return ENOTSUP;
2597 	}
2598 
2599 	return 0;
2600 }
2601 
2602 static int
aq1_fw_version_init(struct aq_softc * sc)2603 aq1_fw_version_init(struct aq_softc *sc)
2604 {
2605 	int error = 0;
2606 	char fw_vers[sizeof("F/W version xxxxx.xxxxx.xxxxx")];
2607 
2608 	if (FW_VERSION_MAJOR(sc) == 1) {
2609 		sc->sc_fw_ops = &aq_fw1x_ops;
2610 	} else if ((FW_VERSION_MAJOR(sc) == 2) || (FW_VERSION_MAJOR(sc) == 3)) {
2611 		sc->sc_fw_ops = &aq_fw2x_ops;
2612 	} else {
2613 		aprint_error_dev(sc->sc_dev,
2614 		    "Unsupported F/W version %d.%d.%d\n",
2615 		    FW_VERSION_MAJOR(sc), FW_VERSION_MINOR(sc),
2616 		    FW_VERSION_BUILD(sc));
2617 		return ENOTSUP;
2618 	}
2619 	snprintf(fw_vers, sizeof(fw_vers), "F/W version %d.%d.%d",
2620 	    FW_VERSION_MAJOR(sc), FW_VERSION_MINOR(sc), FW_VERSION_BUILD(sc));
2621 
2622 	/* detect revision */
2623 	uint32_t hwrev = AQ_READ_REG(sc, AQ_HW_REVISION_REG);
2624 	switch (hwrev & 0x0000000f) {
2625 	case 0x01:
2626 		aprint_normal_dev(sc->sc_dev, "Atlantic revision A0, %s\n",
2627 		    fw_vers);
2628 		sc->sc_features |= FEATURES_AQ1_REV_A0 |
2629 		    FEATURES_MPI_AQ | FEATURES_MIPS;
2630 		sc->sc_max_mtu = AQ1_JUMBO_MTU_REV_A;
2631 		break;
2632 	case 0x02:
2633 		aprint_normal_dev(sc->sc_dev, "Atlantic revision B0, %s\n",
2634 		    fw_vers);
2635 		sc->sc_features |= FEATURES_AQ1_REV_B0 |
2636 		    FEATURES_MPI_AQ | FEATURES_MIPS |
2637 		    FEATURES_TPO2 | FEATURES_RPF2;
2638 		sc->sc_max_mtu = AQ1_JUMBO_MTU_REV_B;
2639 		break;
2640 	case 0x0A:
2641 		aprint_normal_dev(sc->sc_dev, "Atlantic revision B1, %s\n",
2642 		    fw_vers);
2643 		sc->sc_features |= FEATURES_AQ1_REV_B1 |
2644 		    FEATURES_MPI_AQ | FEATURES_MIPS |
2645 		    FEATURES_TPO2 | FEATURES_RPF2;
2646 		sc->sc_max_mtu = AQ1_JUMBO_MTU_REV_B;
2647 		break;
2648 	default:
2649 		aprint_error_dev(sc->sc_dev,
2650 		    "Unknown revision (0x%08x)\n", hwrev);
2651 		sc->sc_features = 0;
2652 		sc->sc_max_mtu = ETHERMTU;
2653 		error = ENOTSUP;
2654 		break;
2655 	}
2656 	return error;
2657 }
2658 
2659 static int
fw1x_reset(struct aq_softc * sc)2660 fw1x_reset(struct aq_softc *sc)
2661 {
2662 	struct aq_mailbox_header mbox;
2663 	const int retryCount = 1000;
2664 	uint32_t tid0;
2665 	int i;
2666 
2667 	tid0 = ~0;	/*< Initial value of MBOX transactionId. */
2668 
2669 	for (i = 0; i < retryCount; ++i) {
2670 		/*
2671 		 * Read the beginning of Statistics structure to capture
2672 		 * the Transaction ID.
2673 		 */
2674 		aq1_fw_downld_dwords(sc, sc->sc_mbox_addr,
2675 		    (uint32_t *)&mbox, sizeof(mbox) / sizeof(uint32_t));
2676 
2677 		/* Successfully read the stats. */
2678 		if (tid0 == ~0U) {
2679 			/* We have read the initial value. */
2680 			tid0 = mbox.transaction_id;
2681 			continue;
2682 		} else if (mbox.transaction_id != tid0) {
2683 			/*
2684 			 * Compare transaction ID to initial value.
2685 			 * If it's different means f/w is alive.
2686 			 * We're done.
2687 			 */
2688 			return 0;
2689 		}
2690 
2691 		/*
2692 		 * Transaction ID value haven't changed since last time.
2693 		 * Try reading the stats again.
2694 		 */
2695 		delay(10);
2696 	}
2697 	aprint_error_dev(sc->sc_dev, "F/W 1.x reset finalize timeout\n");
2698 	return EBUSY;
2699 }
2700 
2701 static int
fw1x_set_mode(struct aq_softc * sc,aq_hw_fw_mpi_state_t mode,aq_link_speed_t speed,aq_link_fc_t fc,aq_link_eee_t eee)2702 fw1x_set_mode(struct aq_softc *sc, aq_hw_fw_mpi_state_t mode,
2703     aq_link_speed_t speed, aq_link_fc_t fc, aq_link_eee_t eee)
2704 {
2705 	uint32_t mpictrl = 0;
2706 	uint32_t mpispeed = 0;
2707 
2708 	if (speed & AQ_LINK_10G)
2709 		mpispeed |= FW1X_CTRL_10G;
2710 	if (speed & AQ_LINK_5G)
2711 		mpispeed |= (FW1X_CTRL_5G | FW1X_CTRL_5GSR);
2712 	if (speed & AQ_LINK_2G5)
2713 		mpispeed |= FW1X_CTRL_2G5;
2714 	if (speed & AQ_LINK_1G)
2715 		mpispeed |= FW1X_CTRL_1G;
2716 	if (speed & AQ_LINK_100M)
2717 		mpispeed |= FW1X_CTRL_100M;
2718 
2719 	mpictrl |= __SHIFTIN(mode, FW1X_MPI_STATE_MODE);
2720 	mpictrl |= __SHIFTIN(mpispeed, FW1X_MPI_STATE_SPEED);
2721 	AQ_WRITE_REG(sc, FW1X_MPI_CONTROL_REG, mpictrl);
2722 	return 0;
2723 }
2724 
2725 static int
fw1x_get_mode(struct aq_softc * sc,aq_hw_fw_mpi_state_t * modep,aq_link_speed_t * speedp,aq_link_fc_t * fcp,aq_link_eee_t * eeep)2726 fw1x_get_mode(struct aq_softc *sc, aq_hw_fw_mpi_state_t *modep,
2727     aq_link_speed_t *speedp, aq_link_fc_t *fcp, aq_link_eee_t *eeep)
2728 {
2729 	uint32_t mpistate, mpi_speed;
2730 	aq_link_speed_t speed = AQ_LINK_NONE;
2731 
2732 	mpistate = AQ_READ_REG(sc, FW1X_MPI_STATE_REG);
2733 
2734 	if (modep != NULL)
2735 		*modep = __SHIFTOUT(mpistate, FW1X_MPI_STATE_MODE);
2736 
2737 	mpi_speed = __SHIFTOUT(mpistate, FW1X_MPI_STATE_SPEED);
2738 	if (mpi_speed & FW1X_CTRL_10G)
2739 		speed = AQ_LINK_10G;
2740 	else if (mpi_speed & (FW1X_CTRL_5G|FW1X_CTRL_5GSR))
2741 		speed = AQ_LINK_5G;
2742 	else if (mpi_speed & FW1X_CTRL_2G5)
2743 		speed = AQ_LINK_2G5;
2744 	else if (mpi_speed & FW1X_CTRL_1G)
2745 		speed = AQ_LINK_1G;
2746 	else if (mpi_speed & FW1X_CTRL_100M)
2747 		speed = AQ_LINK_100M;
2748 
2749 	if (speedp != NULL)
2750 		*speedp = speed;
2751 
2752 	if (fcp != NULL)
2753 		*fcp = AQ_FC_NONE;
2754 
2755 	if (eeep != NULL)
2756 		*eeep = AQ_EEE_DISABLE;
2757 
2758 	return 0;
2759 }
2760 
2761 static int
fw1x_get_stats(struct aq_softc * sc,aq_hw_stats_s_t * stats)2762 fw1x_get_stats(struct aq_softc *sc, aq_hw_stats_s_t *stats)
2763 {
2764 	int error;
2765 
2766 	error = aq1_fw_downld_dwords(sc,
2767 	    sc->sc_mbox_addr + offsetof(fw1x_mailbox_t, msm), (uint32_t *)stats,
2768 	    sizeof(aq_hw_stats_s_t) / sizeof(uint32_t));
2769 	if (error < 0) {
2770 		device_printf(sc->sc_dev,
2771 		    "fw1x> download statistics data FAILED, error %d", error);
2772 		return error;
2773 	}
2774 
2775 	stats->dpc = AQ_READ_REG(sc, RX_DMA_DROP_PKT_CNT_REG);
2776 	stats->cprc = AQ_READ_REG(sc, RX_DMA_COALESCED_PKT_CNT_REG);
2777 	return 0;
2778 }
2779 
2780 static int
fw2x_reset(struct aq_softc * sc)2781 fw2x_reset(struct aq_softc *sc)
2782 {
2783 	fw2x_capabilities_t caps = { 0 };
2784 	int error;
2785 
2786 	error = aq1_fw_downld_dwords(sc,
2787 	    sc->sc_mbox_addr + offsetof(fw2x_mailbox_t, caps),
2788 	    (uint32_t *)&caps, sizeof caps / sizeof(uint32_t));
2789 	if (error != 0) {
2790 		aprint_error_dev(sc->sc_dev,
2791 		    "fw2x> can't get F/W capabilities mask, error %d\n",
2792 		    error);
2793 		return error;
2794 	}
2795 	sc->sc_fw_caps = caps.caps_lo | ((uint64_t)caps.caps_hi << 32);
2796 
2797 	char buf[256];
2798 	snprintb(buf, sizeof(buf), FW2X_SNPRINTB, sc->sc_fw_caps);
2799 	aprint_verbose_dev(sc->sc_dev, "fw2x> F/W capabilities=%s\n", buf);
2800 
2801 	return 0;
2802 }
2803 
2804 static int
fw2x_set_mode(struct aq_softc * sc,aq_hw_fw_mpi_state_t mode,aq_link_speed_t speed,aq_link_fc_t fc,aq_link_eee_t eee)2805 fw2x_set_mode(struct aq_softc *sc, aq_hw_fw_mpi_state_t mode,
2806     aq_link_speed_t speed, aq_link_fc_t fc, aq_link_eee_t eee)
2807 {
2808 	uint64_t mpi_ctrl;
2809 	int error = 0;
2810 
2811 	AQ_MPI_LOCK(sc);
2812 
2813 	mpi_ctrl = AQ_READ64_REG(sc, FW2X_MPI_CONTROL_REG);
2814 
2815 	switch (mode) {
2816 	case MPI_INIT:
2817 		mpi_ctrl &= ~FW2X_CTRL_RATE_MASK;
2818 		if (speed & AQ_LINK_10G)
2819 			mpi_ctrl |= FW2X_CTRL_RATE_10G;
2820 		if (speed & AQ_LINK_5G)
2821 			mpi_ctrl |= FW2X_CTRL_RATE_5G;
2822 		if (speed & AQ_LINK_2G5)
2823 			mpi_ctrl |= FW2X_CTRL_RATE_2G5;
2824 		if (speed & AQ_LINK_1G)
2825 			mpi_ctrl |= FW2X_CTRL_RATE_1G;
2826 		if (speed & AQ_LINK_100M)
2827 			mpi_ctrl |= FW2X_CTRL_RATE_100M;
2828 
2829 		mpi_ctrl &= ~FW2X_CTRL_LINK_DROP;
2830 
2831 		mpi_ctrl &= ~FW2X_CTRL_EEE_MASK;
2832 		if (eee == AQ_EEE_ENABLE)
2833 			mpi_ctrl |= FW2X_CTRL_EEE_MASK;
2834 
2835 		mpi_ctrl &= ~(FW2X_CTRL_PAUSE | FW2X_CTRL_ASYMMETRIC_PAUSE);
2836 		if (fc & AQ_FC_RX)
2837 			mpi_ctrl |= FW2X_CTRL_PAUSE;
2838 		if (fc & AQ_FC_TX)
2839 			mpi_ctrl |= FW2X_CTRL_ASYMMETRIC_PAUSE;
2840 		break;
2841 	case MPI_DEINIT:
2842 		mpi_ctrl &= ~(FW2X_CTRL_RATE_MASK | FW2X_CTRL_EEE_MASK);
2843 		mpi_ctrl &= ~(FW2X_CTRL_PAUSE | FW2X_CTRL_ASYMMETRIC_PAUSE);
2844 		break;
2845 	default:
2846 		device_printf(sc->sc_dev, "fw2x> unknown MPI state %d\n", mode);
2847 		error =  EINVAL;
2848 		goto failure;
2849 	}
2850 	AQ_WRITE64_REG(sc, FW2X_MPI_CONTROL_REG, mpi_ctrl);
2851 
2852  failure:
2853 	AQ_MPI_UNLOCK(sc);
2854 	return error;
2855 }
2856 
2857 static int
fw2x_get_mode(struct aq_softc * sc,aq_hw_fw_mpi_state_t * modep,aq_link_speed_t * speedp,aq_link_fc_t * fcp,aq_link_eee_t * eeep)2858 fw2x_get_mode(struct aq_softc *sc, aq_hw_fw_mpi_state_t *modep,
2859     aq_link_speed_t *speedp, aq_link_fc_t *fcp, aq_link_eee_t *eeep)
2860 {
2861 	uint64_t mpi_state = AQ_READ64_REG(sc, FW2X_MPI_STATE_REG);
2862 
2863 	if (modep != NULL) {
2864 		uint64_t mpi_ctrl = AQ_READ64_REG(sc, FW2X_MPI_CONTROL_REG);
2865 		if (mpi_ctrl & FW2X_CTRL_RATE_MASK)
2866 			*modep = MPI_INIT;
2867 		else
2868 			*modep = MPI_DEINIT;
2869 	}
2870 
2871 	aq_link_speed_t speed = AQ_LINK_NONE;
2872 	if (mpi_state & FW2X_CTRL_RATE_10G)
2873 		speed = AQ_LINK_10G;
2874 	else if (mpi_state & FW2X_CTRL_RATE_5G)
2875 		speed = AQ_LINK_5G;
2876 	else if (mpi_state & FW2X_CTRL_RATE_2G5)
2877 		speed = AQ_LINK_2G5;
2878 	else if (mpi_state & FW2X_CTRL_RATE_1G)
2879 		speed = AQ_LINK_1G;
2880 	else if (mpi_state & FW2X_CTRL_RATE_100M)
2881 		speed = AQ_LINK_100M;
2882 
2883 	if (speedp != NULL)
2884 		*speedp = speed;
2885 
2886 	aq_link_fc_t fc = AQ_FC_NONE;
2887 	if (mpi_state & FW2X_CTRL_PAUSE)
2888 		fc |= AQ_FC_RX;
2889 	if (mpi_state & FW2X_CTRL_ASYMMETRIC_PAUSE)
2890 		fc |= AQ_FC_TX;
2891 	if (fcp != NULL)
2892 		*fcp = fc;
2893 
2894 	/* XXX: TODO: EEE */
2895 	if (eeep != NULL)
2896 		*eeep = AQ_EEE_DISABLE;
2897 
2898 	return 0;
2899 }
2900 
2901 static int
toggle_mpi_ctrl_and_wait(struct aq_softc * sc,uint64_t mask,uint32_t timeout_ms,uint32_t try_count)2902 toggle_mpi_ctrl_and_wait(struct aq_softc *sc, uint64_t mask,
2903     uint32_t timeout_ms, uint32_t try_count)
2904 {
2905 	uint64_t mpi_ctrl = AQ_READ64_REG(sc, FW2X_MPI_CONTROL_REG);
2906 	uint64_t mpi_state = AQ_READ64_REG(sc, FW2X_MPI_STATE_REG);
2907 	int error;
2908 
2909 	/* First, check that control and state values are consistent */
2910 	if ((mpi_ctrl & mask) != (mpi_state & mask)) {
2911 		device_printf(sc->sc_dev,
2912 		    "fw2x> MPI control (%#llx) and state (%#llx)"
2913 		    " are not consistent for mask %#llx!\n",
2914 		    (unsigned long long)mpi_ctrl, (unsigned long long)mpi_state,
2915 		    (unsigned long long)mask);
2916 		return EINVAL;
2917 	}
2918 
2919 	/* Invert bits (toggle) in control register */
2920 	mpi_ctrl ^= mask;
2921 	AQ_WRITE64_REG(sc, FW2X_MPI_CONTROL_REG, mpi_ctrl);
2922 
2923 	/* Clear all bits except masked */
2924 	mpi_ctrl &= mask;
2925 
2926 	/* Wait for FW reflecting change in state register */
2927 	WAIT_FOR((AQ_READ64_REG(sc, FW2X_MPI_CONTROL_REG) & mask) == mpi_ctrl,
2928 	    1000 * timeout_ms, try_count, &error);
2929 	if (error != 0) {
2930 		device_printf(sc->sc_dev,
2931 		    "f/w2x> timeout while waiting for response"
2932 		    " in state register for bit %#llx!",
2933 		    (unsigned long long)mask);
2934 		return error;
2935 	}
2936 	return 0;
2937 }
2938 
2939 static int
fw2x_get_stats(struct aq_softc * sc,aq_hw_stats_s_t * stats)2940 fw2x_get_stats(struct aq_softc *sc, aq_hw_stats_s_t *stats)
2941 {
2942 	int error;
2943 
2944 	AQ_MPI_LOCK(sc);
2945 	/* Say to F/W to update the statistics */
2946 	error = toggle_mpi_ctrl_and_wait(sc, FW2X_CTRL_STATISTICS, 1, 25);
2947 	if (error != 0) {
2948 		device_printf(sc->sc_dev,
2949 		    "fw2x> statistics update error %d\n", error);
2950 		goto failure;
2951 	}
2952 
2953 	CTASSERT(sizeof(fw2x_msm_statistics_t) <= sizeof(struct aq_hw_stats_s));
2954 	error = aq1_fw_downld_dwords(sc,
2955 	    sc->sc_mbox_addr + offsetof(fw2x_mailbox_t, msm), (uint32_t *)stats,
2956 	    sizeof(fw2x_msm_statistics_t) / sizeof(uint32_t));
2957 	if (error != 0) {
2958 		device_printf(sc->sc_dev,
2959 		    "fw2x> download statistics data FAILED, error %d", error);
2960 		goto failure;
2961 	}
2962 	stats->dpc = AQ_READ_REG(sc, RX_DMA_DROP_PKT_CNT_REG);
2963 	stats->cprc = AQ_READ_REG(sc, RX_DMA_COALESCED_PKT_CNT_REG);
2964 
2965  failure:
2966 	AQ_MPI_UNLOCK(sc);
2967 	return error;
2968 }
2969 
2970 #if NSYSMON_ENVSYS > 0
2971 static int
fw2x_get_temperature(struct aq_softc * sc,uint32_t * temp)2972 fw2x_get_temperature(struct aq_softc *sc, uint32_t *temp)
2973 {
2974 	int error;
2975 	uint32_t value, celsius;
2976 
2977 	AQ_MPI_LOCK(sc);
2978 
2979 	/* Say to F/W to update the temperature */
2980 	error = toggle_mpi_ctrl_and_wait(sc, FW2X_CTRL_TEMPERATURE, 1, 25);
2981 	if (error != 0)
2982 		goto failure;
2983 
2984 	error = aq1_fw_downld_dwords(sc,
2985 	    sc->sc_mbox_addr + offsetof(fw2x_mailbox_t, phy_info2),
2986 	    &value, sizeof(value) / sizeof(uint32_t));
2987 	if (error != 0)
2988 		goto failure;
2989 
2990 	/* 1/256 decrees C to microkelvin */
2991 	celsius = __SHIFTOUT(value, PHYINFO2_TEMPERATURE);
2992 	if (celsius == 0) {
2993 		error = EIO;
2994 		goto failure;
2995 	}
2996 	*temp = celsius * (1000000 / 256) + 273150000;
2997 
2998  failure:
2999 	AQ_MPI_UNLOCK(sc);
3000 	return 0;
3001 }
3002 #endif
3003 
3004 static int
aq1_fw_downld_dwords(struct aq_softc * sc,uint32_t addr,uint32_t * p,uint32_t cnt)3005 aq1_fw_downld_dwords(struct aq_softc *sc, uint32_t addr, uint32_t *p,
3006     uint32_t cnt)
3007 {
3008 	uint32_t v;
3009 	int error = 0;
3010 
3011 	WAIT_FOR(AQ_READ_REG(sc, AQ1_FW_SEM_RAM_REG) == 1, 1, 10000, &error);
3012 	if (error != 0) {
3013 		AQ_WRITE_REG(sc, AQ1_FW_SEM_RAM_REG, 1);
3014 		v = AQ_READ_REG(sc, AQ1_FW_SEM_RAM_REG);
3015 		if (v == 0) {
3016 			device_printf(sc->sc_dev,
3017 			    "%s:%d: timeout\n", __func__, __LINE__);
3018 			return ETIMEDOUT;
3019 		}
3020 	}
3021 
3022 	AQ_WRITE_REG(sc, AQ_FW_MBOX_ADDR_REG, addr);
3023 
3024 	error = 0;
3025 	for (; cnt > 0 && error == 0; cnt--) {
3026 		/* execute mailbox interface */
3027 		AQ_WRITE_REG_BIT(sc, AQ_FW_MBOX_CMD_REG,
3028 		    AQ_FW_MBOX_CMD_EXECUTE, 1);
3029 		if (sc->sc_features & FEATURES_AQ1_REV_B1) {
3030 			WAIT_FOR(AQ_READ_REG(sc, AQ_FW_MBOX_ADDR_REG) != addr,
3031 			    1, 1000, &error);
3032 		} else {
3033 			WAIT_FOR((AQ_READ_REG(sc, AQ_FW_MBOX_CMD_REG) &
3034 			    AQ_FW_MBOX_CMD_BUSY) == 0,
3035 			    1, 1000, &error);
3036 		}
3037 		*p++ = AQ_READ_REG(sc, AQ_FW_MBOX_VAL_REG);
3038 		addr += sizeof(uint32_t);
3039 	}
3040 	AQ_WRITE_REG(sc, AQ1_FW_SEM_RAM_REG, 1);
3041 
3042 	if (error != 0)
3043 		device_printf(sc->sc_dev,
3044 		    "%s:%d: timeout\n", __func__, __LINE__);
3045 
3046 	return error;
3047 }
3048 
3049 /* read my mac address */
3050 static int
aq1_get_mac_addr(struct aq_softc * sc)3051 aq1_get_mac_addr(struct aq_softc *sc)
3052 {
3053 	uint32_t mac_addr[2];
3054 	uint32_t efuse_shadow_addr;
3055 	int err;
3056 
3057 	efuse_shadow_addr = 0;
3058 	if (FW_VERSION_MAJOR(sc) >= 2)
3059 		efuse_shadow_addr = AQ_READ_REG(sc, FW2X_MPI_EFUSEADDR_REG);
3060 	else
3061 		efuse_shadow_addr = AQ_READ_REG(sc, FW1X_MPI_EFUSEADDR_REG);
3062 
3063 	if (efuse_shadow_addr == 0) {
3064 		aprint_error_dev(sc->sc_dev, "cannot get efuse addr\n");
3065 		return ENXIO;
3066 	}
3067 
3068 	memset(mac_addr, 0, sizeof(mac_addr));
3069 	err = aq1_fw_downld_dwords(sc, efuse_shadow_addr + (40 * 4),
3070 	    mac_addr, __arraycount(mac_addr));
3071 	if (err < 0)
3072 		return err;
3073 
3074 	if (mac_addr[0] == 0 && mac_addr[1] == 0) {
3075 		aprint_error_dev(sc->sc_dev, "mac address not found\n");
3076 		return ENXIO;
3077 	}
3078 
3079 	mac_addr[0] = htobe32(mac_addr[0]);
3080 	mac_addr[1] = htobe32(mac_addr[1]);
3081 
3082 	memcpy(sc->sc_enaddr.ether_addr_octet,
3083 	    (uint8_t *)mac_addr, ETHER_ADDR_LEN);
3084 	aprint_normal_dev(sc->sc_dev, "Etheraddr: %s\n",
3085 	    ether_sprintf(sc->sc_enaddr.ether_addr_octet));
3086 
3087 	return 0;
3088 }
3089 
3090 /* set multicast filter. index 0 for own address */
3091 static int
aq_set_mac_addr(struct aq_softc * sc,int index,uint8_t * enaddr)3092 aq_set_mac_addr(struct aq_softc *sc, int index, uint8_t *enaddr)
3093 {
3094 	uint32_t h, l;
3095 
3096 	if (index >= AQ_HW_MAC_NUM(sc))
3097 		return EINVAL;
3098 
3099 	if (enaddr == NULL) {
3100 		/* disable */
3101 		AQ_WRITE_REG_BIT(sc,
3102 		    RPF_L2UC_MSW_REG(index), RPF_L2UC_MSW_EN, 0);
3103 		return 0;
3104 	}
3105 
3106 	h = (enaddr[0] << 8) | (enaddr[1]);
3107 	l = ((uint32_t)enaddr[2] << 24) | (enaddr[3] << 16) |
3108 	    (enaddr[4] << 8) | (enaddr[5]);
3109 
3110 	/* disable, set, and enable */
3111 	AQ_WRITE_REG_BIT(sc, RPF_L2UC_MSW_REG(index), RPF_L2UC_MSW_EN, 0);
3112 	AQ_WRITE_REG(sc, RPF_L2UC_LSW_REG(index), l);
3113 	AQ_WRITE_REG_BIT(sc, RPF_L2UC_MSW_REG(index),
3114 	    RPF_L2UC_MSW_MACADDR_HI, h);
3115 	AQ_WRITE_REG_BIT(sc, RPF_L2UC_MSW_REG(index),
3116 	    RPF_L2UC_MSW_ACTION, RPF_ACTION_HOST);
3117 	if (HWTYPE_AQ2_P(sc)) {
3118 		AQ_WRITE_REG_BIT(sc, RPF_L2UC_MSW_REG(index),
3119 		    RPF_L2UC_MSW_TAG, __SHIFTIN(1, AQ2_RPF_TAG_UC_MASK));
3120 	}
3121 	AQ_WRITE_REG_BIT(sc, RPF_L2UC_MSW_REG(index), RPF_L2UC_MSW_EN, 1);
3122 
3123 	return 0;
3124 }
3125 
3126 static int
aq_set_capability(struct aq_softc * sc)3127 aq_set_capability(struct aq_softc *sc)
3128 {
3129 	struct ifnet * const ifp = &sc->sc_ethercom.ec_if;
3130 	int ip4csum_tx =
3131 	    ((ifp->if_capenable & IFCAP_CSUM_IPv4_Tx) == 0) ? 0 : 1;
3132 	int ip4csum_rx =
3133 	    ((ifp->if_capenable & IFCAP_CSUM_IPv4_Rx) == 0) ? 0 : 1;
3134 	int l4csum_tx = ((ifp->if_capenable &
3135 	   (IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_UDPv4_Tx |
3136 	   IFCAP_CSUM_TCPv6_Tx | IFCAP_CSUM_UDPv6_Tx)) == 0) ? 0 : 1;
3137 	int l4csum_rx =
3138 	   ((ifp->if_capenable & (IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx |
3139 	   IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx)) == 0) ? 0 : 1;
3140 	uint32_t lso =
3141 	   ((ifp->if_capenable & (IFCAP_TSOv4 | IFCAP_TSOv6)) == 0) ?
3142 	   0 : 0xffffffff;
3143 	uint32_t lro = ((ifp->if_capenable & IFCAP_LRO) == 0) ?
3144 	    0 : 0xffffffff;
3145 	uint32_t i, v;
3146 
3147 	/* TX checksums offloads*/
3148 	AQ_WRITE_REG_BIT(sc, TPO_HWCSUM_REG, TPO_HWCSUM_IP4CSUM_EN, ip4csum_tx);
3149 	AQ_WRITE_REG_BIT(sc, TPO_HWCSUM_REG, TPO_HWCSUM_L4CSUM_EN, l4csum_tx);
3150 
3151 	/* RX checksums offloads*/
3152 	AQ_WRITE_REG_BIT(sc, RPO_HWCSUM_REG, RPO_HWCSUM_IP4CSUM_EN, ip4csum_rx);
3153 	AQ_WRITE_REG_BIT(sc, RPO_HWCSUM_REG, RPO_HWCSUM_L4CSUM_EN, l4csum_rx);
3154 
3155 	/* LSO offloads*/
3156 	AQ_WRITE_REG(sc, TDM_LSO_EN_REG, lso);
3157 
3158 #define AQ_B0_LRO_RXD_MAX	16
3159 	v = (8 < AQ_B0_LRO_RXD_MAX) ? 3 :
3160 	    (4 < AQ_B0_LRO_RXD_MAX) ? 2 :
3161 	    (2 < AQ_B0_LRO_RXD_MAX) ? 1 : 0;
3162 	for (i = 0; i < AQ_RINGS_NUM; i++) {
3163 		AQ_WRITE_REG_BIT(sc, RPO_LRO_LDES_MAX_REG(i),
3164 		    RPO_LRO_LDES_MAX_MASK(i), v);
3165 	}
3166 
3167 	AQ_WRITE_REG_BIT(sc, RPO_LRO_TB_DIV_REG, RPO_LRO_TB_DIV, 0x61a);
3168 	AQ_WRITE_REG_BIT(sc, RPO_LRO_INACTIVE_IVAL_REG,
3169 	    RPO_LRO_INACTIVE_IVAL, 0);
3170 	/*
3171 	 * the LRO timebase divider is 5 uS (0x61a),
3172 	 * to get a maximum coalescing interval of 250 uS,
3173 	 * we need to multiply by 50(0x32) to get
3174 	 * the default value 250 uS
3175 	 */
3176 	AQ_WRITE_REG_BIT(sc, RPO_LRO_MAX_COALESCING_IVAL_REG,
3177 	    RPO_LRO_MAX_COALESCING_IVAL, 50);
3178 	AQ_WRITE_REG_BIT(sc, RPO_LRO_CONF_REG,
3179 	    RPO_LRO_CONF_QSESSION_LIMIT, 1);
3180 	AQ_WRITE_REG_BIT(sc, RPO_LRO_CONF_REG,
3181 	    RPO_LRO_CONF_TOTAL_DESC_LIMIT, 2);
3182 	AQ_WRITE_REG_BIT(sc, RPO_LRO_CONF_REG,
3183 	    RPO_LRO_CONF_PATCHOPTIMIZATION_EN, 0);
3184 	AQ_WRITE_REG_BIT(sc, RPO_LRO_CONF_REG,
3185 	    RPO_LRO_CONF_MIN_PAYLOAD_OF_FIRST_PKT, 10);
3186 	AQ_WRITE_REG(sc, RPO_LRO_RSC_MAX_REG, 1);
3187 	AQ_WRITE_REG(sc, RPO_LRO_ENABLE_REG, lro);
3188 
3189 	return 0;
3190 }
3191 
3192 static int
aq_set_filter(struct aq_softc * sc)3193 aq_set_filter(struct aq_softc *sc)
3194 {
3195 	struct ifnet * const ifp = &sc->sc_ethercom.ec_if;
3196 	struct ethercom * const ec = &sc->sc_ethercom;
3197 	struct ether_multi *enm;
3198 	struct ether_multistep step;
3199 	int idx, error = 0;
3200 
3201 	if (HWTYPE_AQ2_P(sc)) {
3202 		uint32_t action = (ifp->if_flags & IFF_PROMISC) ?
3203 		    AQ2_ART_ACTION_DISABLE : AQ2_ART_ACTION_DROP;
3204 		aq2_filter_art_set(sc, AQ2_RPF_INDEX_L2_PROMISC_OFF, 0,
3205 		    AQ2_RPF_TAG_UC_MASK | AQ2_RPF_TAG_ALLMC_MASK, action);
3206 		aq2_filter_art_set(sc, AQ2_RPF_INDEX_VLAN_PROMISC_OFF, 0,
3207 		    AQ2_RPF_TAG_VLAN_MASK | AQ2_RPF_TAG_UNTAG_MASK, action);
3208 	}
3209 
3210 	if (ifp->if_flags & IFF_PROMISC) {
3211 		AQ_WRITE_REG_BIT(sc, RPF_L2BC_REG, RPF_L2BC_PROMISC,
3212 		    (ifp->if_flags & IFF_PROMISC) ? 1 : 0);
3213 		ec->ec_flags |= ETHER_F_ALLMULTI;
3214 		goto done;
3215 	}
3216 
3217 	/* clear all table */
3218 	for (idx = 0; idx < AQ_HW_MAC_NUM(sc); idx++) {
3219 		if (idx == AQ_HW_MAC_OWN)	/* already used for own */
3220 			continue;
3221 		aq_set_mac_addr(sc, idx, NULL);
3222 	}
3223 
3224 	/* don't accept all multicast */
3225 	AQ_WRITE_REG_BIT(sc, RPF_MCAST_FILTER_MASK_REG,
3226 	    RPF_MCAST_FILTER_MASK_ALLMULTI, 0);
3227 	AQ_WRITE_REG_BIT(sc, RPF_MCAST_FILTER_REG(0),
3228 	    RPF_MCAST_FILTER_EN, 0);
3229 
3230 	idx = 0;
3231 	ETHER_LOCK(ec);
3232 	ETHER_FIRST_MULTI(step, ec, enm);
3233 	while (enm != NULL) {
3234 		if (idx == AQ_HW_MAC_OWN)
3235 			idx++;
3236 
3237 		if ((idx >= AQ_HW_MAC_NUM(sc)) ||
3238 		    memcmp(enm->enm_addrlo, enm->enm_addrhi, ETHER_ADDR_LEN)) {
3239 			/*
3240 			 * too many filters.
3241 			 * fallback to accept all multicast addresses.
3242 			 */
3243 			AQ_WRITE_REG_BIT(sc, RPF_MCAST_FILTER_MASK_REG,
3244 			    RPF_MCAST_FILTER_MASK_ALLMULTI, 1);
3245 			AQ_WRITE_REG_BIT(sc, RPF_MCAST_FILTER_REG(0),
3246 			    RPF_MCAST_FILTER_EN, 1);
3247 			ec->ec_flags |= ETHER_F_ALLMULTI;
3248 			ETHER_UNLOCK(ec);
3249 			goto done;
3250 		}
3251 
3252 		/* add a filter */
3253 		aq_set_mac_addr(sc, idx++, enm->enm_addrlo);
3254 
3255 		ETHER_NEXT_MULTI(step, enm);
3256 	}
3257 	ec->ec_flags &= ~ETHER_F_ALLMULTI;
3258 	ETHER_UNLOCK(ec);
3259 
3260  done:
3261 	return error;
3262 }
3263 
3264 static int
aq2_filter_art_set(struct aq_softc * sc,uint32_t idx,uint32_t tag,uint32_t mask,uint32_t action)3265 aq2_filter_art_set(struct aq_softc *sc, uint32_t idx,
3266     uint32_t tag, uint32_t mask, uint32_t action)
3267 {
3268 	int error;
3269 
3270 	AQ_MPI_LOCK(sc);
3271 
3272 	WAIT_FOR(AQ_READ_REG(sc, AQ2_ART_SEM_REG) == 1, 10, 10000, &error);
3273 	if (error != 0) {
3274 		device_printf(sc->sc_dev, "%s: timeout\n", __func__);
3275 		goto done;
3276 	}
3277 
3278 	idx += sc->sc_filter_art_base_index;
3279 	AQ_WRITE_REG(sc, AQ2_RPF_ACT_ART_REQ_TAG_REG(idx), tag);
3280 	AQ_WRITE_REG(sc, AQ2_RPF_ACT_ART_REQ_MASK_REG(idx), mask);
3281 	AQ_WRITE_REG(sc, AQ2_RPF_ACT_ART_REQ_ACTION_REG(idx), action);
3282 
3283 	AQ_WRITE_REG(sc, AQ2_ART_SEM_REG, 1);
3284 
3285  done:
3286 	AQ_MPI_UNLOCK(sc);
3287 	return 0;
3288 }
3289 
3290 static int
aq2_init_filter(struct aq_softc * sc)3291 aq2_init_filter(struct aq_softc *sc)
3292 {
3293 	AQ_WRITE_REG_BIT(sc, AQ2_RPF_REC_TAB_ENABLE_REG,
3294 	    AQ2_RPF_REC_TAB_ENABLE_MASK, 0xffff);
3295 	AQ_WRITE_REG_BIT(sc, RPF_L2UC_MSW_REG(0),
3296 	    RPF_L2UC_MSW_TAG, __SHIFTIN(1, AQ2_RPF_TAG_UC_MASK));
3297 	AQ_WRITE_REG_BIT(sc, AQ2_RPF_L2BC_TAG_REG,
3298 	    AQ2_RPF_L2BC_TAG_MASK, __SHIFTIN(1, AQ2_RPF_TAG_UC_MASK));
3299 
3300 	aq2_filter_art_set(sc, AQ2_RPF_INDEX_L2_PROMISC_OFF,
3301 	    0, AQ2_RPF_TAG_UC_MASK | AQ2_RPF_TAG_ALLMC_MASK,
3302 	    AQ2_ART_ACTION_DROP);
3303 	aq2_filter_art_set(sc, AQ2_RPF_INDEX_VLAN_PROMISC_OFF,
3304 	    0, AQ2_RPF_TAG_VLAN_MASK | AQ2_RPF_TAG_UNTAG_MASK,
3305 	    AQ2_ART_ACTION_DROP);
3306 
3307 	for (int i = 0; i < 8; i++) {
3308 		aq2_filter_art_set(sc, AQ2_RPF_INDEX_PCP_TO_TC + i,
3309 		    __SHIFTIN(i, AQ2_RPF_TAG_PCP_MASK), AQ2_RPF_TAG_PCP_MASK,
3310 		    AQ2_ART_ACTION_ASSIGN_TC(i % sc->sc_nqueues));
3311 	}
3312 
3313 	return 0;
3314 }
3315 
3316 static int
aq2_interface_buffer_read(struct aq_softc * sc,uint32_t reg0,uint32_t * data0,uint32_t size0)3317 aq2_interface_buffer_read(struct aq_softc *sc, uint32_t reg0, uint32_t *data0,
3318     uint32_t size0)
3319 {
3320 	uint32_t tid0, tid1, reg, *data, size;
3321 	int timo;
3322 
3323 	for (timo = 10000; timo > 0; timo--) {
3324 		tid0 = AQ_READ_REG(sc, AQ2_FW_INTERFACE_OUT_TRANSACTION_ID_REG);
3325 		if (__SHIFTOUT(tid0, AQ2_FW_INTERFACE_OUT_TRANSACTION_ID_A) !=
3326 		    __SHIFTOUT(tid0, AQ2_FW_INTERFACE_OUT_TRANSACTION_ID_B)) {
3327 			delay(10);
3328 			continue;
3329 		}
3330 
3331 		for (reg = reg0, data = data0, size = size0;
3332 		    size >= 4; reg += 4, data++, size -= 4) {
3333 			*data = AQ_READ_REG(sc, reg);
3334 		}
3335 
3336 		tid1 = AQ_READ_REG(sc, AQ2_FW_INTERFACE_OUT_TRANSACTION_ID_REG);
3337 		if (tid0 == tid1)
3338 			break;
3339 	}
3340 	if (timo == 0) {
3341 		device_printf(sc->sc_dev, "%s: timeout\n", __func__);
3342 		return ETIMEDOUT;
3343 	}
3344 	return 0;
3345 }
3346 
3347 static int
aq2_fw_reboot(struct aq_softc * sc)3348 aq2_fw_reboot(struct aq_softc *sc)
3349 {
3350 	uint32_t v;
3351 	int timo;
3352 	char buf[32];
3353 
3354 	/* It seems that there is still only one type of firmware ABI in aq2 */
3355 	sc->sc_fw_ops = &aq2_fw_ops;
3356 	sc->sc_features |= FEATURES_AQ2;
3357 	sc->sc_max_mtu = AQ2_JUMBO_MTU;
3358 
3359 	AQ_WRITE_REG(sc, AQ2_MCP_HOST_REQ_INT_CLR_REG, 1);
3360 	AQ_WRITE_REG(sc, AQ2_MIF_BOOT_REG, 1);	/* reboot request */
3361 	for (timo = 200000; timo > 0; timo--) {
3362 		v = AQ_READ_REG(sc, AQ2_MIF_BOOT_REG);
3363 		if ((v & AQ2_MIF_BOOT_BOOT_STARTED) && v != 0xffffffff)
3364 			break;
3365 		delay(10);
3366 	}
3367 	if (timo <= 0) {
3368 		aprint_error_dev(sc->sc_dev, "FW reboot timeout\n");
3369 		return ETIMEDOUT;
3370 	}
3371 
3372 	for (timo = 2000000; timo > 0; timo--) {
3373 		v = AQ_READ_REG(sc, AQ2_MIF_BOOT_REG);
3374 		if ((v & AQ2_MIF_BOOT_FW_INIT_FAILED) ||
3375 		    (v & AQ2_MIF_BOOT_FW_INIT_COMP_SUCCESS))
3376 			break;
3377 		v = AQ_READ_REG(sc, AQ2_MCP_HOST_REQ_INT_REG);
3378 		if (v & AQ2_MCP_HOST_REQ_INT_READY)
3379 			break;
3380 		delay(10);
3381 	}
3382 	if (timo <= 0) {
3383 		aprint_error_dev(sc->sc_dev, "FW restart timeout\n");
3384 		return ETIMEDOUT;
3385 	}
3386 
3387 	v = AQ_READ_REG(sc, AQ2_MIF_BOOT_REG);
3388 	if (v & AQ2_MIF_BOOT_FW_INIT_FAILED) {
3389 		aprint_error_dev(sc->sc_dev, "FW restart failed\n");
3390 		return ETIMEDOUT;
3391 	}
3392 
3393 	v = AQ_READ_REG(sc, AQ2_MCP_HOST_REQ_INT_REG);
3394 	if (v & AQ2_MCP_HOST_REQ_INT_READY) {
3395 		aprint_error_dev(sc->sc_dev, "firmware required\n");
3396 		return ENXIO;
3397 	}
3398 
3399 	/*
3400 	 * Get aq2 firmware version.
3401 	 * Note that the bit layout and its meaning are different from aq1.
3402 	 */
3403 	aq2_interface_buffer_read(sc, AQ2_FW_INTERFACE_OUT_VERSION_BUNDLE_REG,
3404 	    (uint32_t *)&v, sizeof(v));
3405 	sc->sc_fw_version =
3406 	    __SHIFTOUT(v, AQ2_FW_INTERFACE_OUT_VERSION_MAJOR) << 24 |
3407 	    __SHIFTOUT(v, AQ2_FW_INTERFACE_OUT_VERSION_MINOR) << 16 |
3408 	    __SHIFTOUT(v, AQ2_FW_INTERFACE_OUT_VERSION_BUILD);
3409 
3410 	aq2_interface_buffer_read(sc, AQ2_FW_INTERFACE_OUT_VERSION_IFACE_REG,
3411 	    (uint32_t *)&v, sizeof(v));
3412 	switch (__SHIFTOUT(v, AQ2_FW_INTERFACE_OUT_VERSION_IFACE_VER)) {
3413 	case AQ2_FW_INTERFACE_OUT_VERSION_IFACE_VER_A0:
3414 		sc->sc_features |= FEATURES_AQ2_IFACE_A0;
3415 		strncpy(buf, "A0", sizeof(buf));
3416 		break;
3417 	case AQ2_FW_INTERFACE_OUT_VERSION_IFACE_VER_B0:
3418 		sc->sc_features |= FEATURES_AQ2_IFACE_B0;
3419 		strncpy(buf, "B0", sizeof(buf));
3420 		break;
3421 	default:
3422 		snprintf(buf, sizeof(buf), "(unknown 0x%08x)", v);
3423 		break;
3424 	}
3425 	aprint_normal_dev(sc->sc_dev,
3426 	    "Atlantic2 %s, F/W version %d.%d.%d\n", buf,
3427 	    FW_VERSION_MAJOR(sc), FW_VERSION_MINOR(sc), FW_VERSION_BUILD(sc));
3428 
3429 	aq2_interface_buffer_read(sc, AQ2_FW_INTERFACE_OUT_FILTER_CAPS_REG,
3430 	    (uint32_t *)&sc->sc_filter_caps, sizeof(sc->sc_filter_caps));
3431 	sc->sc_filter_art_base_index = __SHIFTOUT(sc->sc_filter_caps.caps3,
3432 	    AQ2_FW_INTERFACE_OUT_FILTER_CAPS3_RESOLVER_BASE_INDEX) * 8;
3433 
3434 	/* debug info */
3435 	v = AQ_READ_REG(sc, AQ_HW_REVISION_REG);
3436 	aprint_debug_dev(sc->sc_dev, "HW Rev: 0x%08x\n", v);
3437 
3438 	aq2_interface_buffer_read(sc, AQ2_FW_INTERFACE_OUT_VERSION_MAC_REG,
3439 	    (uint32_t *)&v, sizeof(v));
3440 	aprint_debug_dev(sc->sc_dev, "MAC Version %d.%d.%d\n",
3441 	    (int)__SHIFTOUT(v, AQ2_FW_INTERFACE_OUT_VERSION_MAJOR),
3442 	    (int)__SHIFTOUT(v, AQ2_FW_INTERFACE_OUT_VERSION_MINOR),
3443 	    (int)__SHIFTOUT(v, AQ2_FW_INTERFACE_OUT_VERSION_BUILD));
3444 
3445 	aq2_interface_buffer_read(sc, AQ2_FW_INTERFACE_OUT_VERSION_PHY_REG,
3446 	    (uint32_t *)&v, sizeof(v));
3447 	aprint_debug_dev(sc->sc_dev, "PHY Version %d.%d.%d\n",
3448 	    (int)__SHIFTOUT(v, AQ2_FW_INTERFACE_OUT_VERSION_MAJOR),
3449 	    (int)__SHIFTOUT(v, AQ2_FW_INTERFACE_OUT_VERSION_MINOR),
3450 	    (int)__SHIFTOUT(v, AQ2_FW_INTERFACE_OUT_VERSION_BUILD));
3451 
3452 	v = AQ_READ_REG(sc, AQ2_HW_FPGA_VERSION_REG);
3453 	aprint_debug_dev(sc->sc_dev, "AQ2 FPGA Version: %d.%d.%d.%d\n",
3454 	    (int)__SHIFTOUT(v, __BITS(31, 24)),
3455 	    (int)__SHIFTOUT(v, __BITS(23, 16)),
3456 	    (int)__SHIFTOUT(v, __BITS(15, 8)),
3457 	    (int)__SHIFTOUT(v, __BITS(7, 0)));
3458 
3459 	aprint_debug_dev(sc->sc_dev, "FILTER CAPS: 0x%08x,0x%08x,0x%08x\n",
3460 	    sc->sc_filter_caps.caps1, sc->sc_filter_caps.caps2,
3461 	    sc->sc_filter_caps.caps3);
3462 
3463 	return 0;
3464 }
3465 
3466 static int
aq2_fw_wait_shared_ack(struct aq_softc * sc)3467 aq2_fw_wait_shared_ack(struct aq_softc *sc)
3468 {
3469 	int error;
3470 
3471 	AQ_WRITE_REG(sc, AQ2_MIF_HOST_FINISHED_STATUS_WRITE_REG,
3472 	    AQ2_MIF_HOST_FINISHED_STATUS_ACK);
3473 	WAIT_FOR((AQ_READ_REG(sc, AQ2_MIF_HOST_FINISHED_STATUS_READ_REG) &
3474 	    AQ2_MIF_HOST_FINISHED_STATUS_ACK) == 0, 100, 100000, &error);
3475 
3476 	return error;
3477 }
3478 
3479 static int
aq2_fw_reset(struct aq_softc * sc)3480 aq2_fw_reset(struct aq_softc *sc)
3481 {
3482 	AQ_WRITE_REG_BIT(sc, AQ2_FW_INTERFACE_IN_LINK_CONTROL_REG,
3483 	    AQ2_FW_INTERFACE_IN_LINK_CONTROL_MODE,
3484 	    AQ2_FW_INTERFACE_IN_LINK_CONTROL_MODE_ACTIVE);
3485 
3486 	AQ_WRITE_REG(sc, AQ2_FW_INTERFACE_IN_MTU_REG,
3487 	    AQ2_JUMBO_MTU + sizeof(struct ether_header));
3488 
3489 	uint32_t v = AQ_READ_REG(sc, AQ2_FW_INTERFACE_IN_REQUEST_POLICY_REG);
3490 	v |= AQ2_FW_INTERFACE_IN_REQUEST_POLICY_MCAST_QUEUE_OR_TC;
3491 	v &= ~AQ2_FW_INTERFACE_IN_REQUEST_POLICY_MCAST_RX_QUEUE_TC_INDEX;
3492 	v |= AQ2_FW_INTERFACE_IN_REQUEST_POLICY_MCAST_ACCEPT;
3493 	v |= AQ2_FW_INTERFACE_IN_REQUEST_POLICY_BCAST_QUEUE_OR_TC;
3494 	v &= AQ2_FW_INTERFACE_IN_REQUEST_POLICY_BCAST_RX_QUEUE_TC_INDEX;
3495 	v |= AQ2_FW_INTERFACE_IN_REQUEST_POLICY_BCAST_ACCEPT;
3496 	v |= AQ2_FW_INTERFACE_IN_REQUEST_POLICY_PROMISC_QUEUE_OR_TC;
3497 	v &= ~AQ2_FW_INTERFACE_IN_REQUEST_POLICY_PROMISC_RX_QUEUE_TX_INDEX;
3498 	AQ_WRITE_REG(sc, AQ2_FW_INTERFACE_IN_REQUEST_POLICY_REG, v);
3499 
3500 	return aq2_fw_wait_shared_ack(sc);
3501 }
3502 
3503 static int
aq2_fw_set_mode(struct aq_softc * sc,aq_hw_fw_mpi_state_t mode,aq_link_speed_t speed,aq_link_fc_t fc,aq_link_eee_t eee)3504 aq2_fw_set_mode(struct aq_softc *sc, aq_hw_fw_mpi_state_t mode,
3505     aq_link_speed_t speed, aq_link_fc_t fc, aq_link_eee_t eee)
3506 {
3507 	uint32_t v;
3508 	int error;
3509 
3510 	AQ_MPI_LOCK(sc);
3511 
3512 	v = AQ_READ_REG(sc, AQ2_FW_INTERFACE_IN_LINK_OPTIONS_REG);
3513 	v &= ~(
3514 	    AQ2_FW_INTERFACE_IN_LINK_OPTIONS_RATE_10G |
3515 	    AQ2_FW_INTERFACE_IN_LINK_OPTIONS_RATE_N5G |
3516 	    AQ2_FW_INTERFACE_IN_LINK_OPTIONS_RATE_5G |
3517 	    AQ2_FW_INTERFACE_IN_LINK_OPTIONS_RATE_N2G5 |
3518 	    AQ2_FW_INTERFACE_IN_LINK_OPTIONS_RATE_2G5 |
3519 	    AQ2_FW_INTERFACE_IN_LINK_OPTIONS_RATE_1G |
3520 	    AQ2_FW_INTERFACE_IN_LINK_OPTIONS_RATE_100M |
3521 	    AQ2_FW_INTERFACE_IN_LINK_OPTIONS_RATE_10M |
3522 	    AQ2_FW_INTERFACE_IN_LINK_OPTIONS_RATE_1G_HD |
3523 	    AQ2_FW_INTERFACE_IN_LINK_OPTIONS_RATE_100M_HD |
3524 	    AQ2_FW_INTERFACE_IN_LINK_OPTIONS_RATE_10M_HD);
3525 
3526 	v &= ~AQ2_FW_INTERFACE_IN_LINK_OPTIONS_LINK_UP;
3527 
3528 	if (speed & AQ_LINK_10G)
3529 		v |= AQ2_FW_INTERFACE_IN_LINK_OPTIONS_RATE_10G;
3530 	if (speed & AQ_LINK_5G)
3531 		v |= AQ2_FW_INTERFACE_IN_LINK_OPTIONS_RATE_N5G |
3532 		    AQ2_FW_INTERFACE_IN_LINK_OPTIONS_RATE_5G;
3533 	if (speed & AQ_LINK_2G5)
3534 		v |= AQ2_FW_INTERFACE_IN_LINK_OPTIONS_RATE_N2G5 |
3535 		    AQ2_FW_INTERFACE_IN_LINK_OPTIONS_RATE_2G5;
3536 	if (speed & AQ_LINK_1G)
3537 		v |= AQ2_FW_INTERFACE_IN_LINK_OPTIONS_RATE_1G |
3538 		    AQ2_FW_INTERFACE_IN_LINK_OPTIONS_RATE_1G_HD;
3539 	if (speed & AQ_LINK_100M)
3540 		v |= AQ2_FW_INTERFACE_IN_LINK_OPTIONS_RATE_100M |
3541 		    AQ2_FW_INTERFACE_IN_LINK_OPTIONS_RATE_100M_HD;
3542 	if (speed & AQ_LINK_10M) {
3543 		v |= AQ2_FW_INTERFACE_IN_LINK_OPTIONS_RATE_10M |
3544 		    AQ2_FW_INTERFACE_IN_LINK_OPTIONS_RATE_10M_HD;
3545 	}
3546 
3547 	/* flow control */
3548 	v &= ~(AQ2_FW_INTERFACE_IN_LINK_OPTIONS_PAUSE_TX |
3549 	    AQ2_FW_INTERFACE_IN_LINK_OPTIONS_PAUSE_RX);
3550 	if (fc & AQ_FC_TX)
3551 		v |= AQ2_FW_INTERFACE_IN_LINK_OPTIONS_PAUSE_TX;
3552 	if (fc & AQ_FC_RX)
3553 		v |= AQ2_FW_INTERFACE_IN_LINK_OPTIONS_PAUSE_RX;
3554 
3555 	if (speed == AQ_LINK_NONE) {
3556 		AQ_WRITE_REG_BIT(sc, AQ2_FW_INTERFACE_IN_LINK_CONTROL_REG,
3557 		    AQ2_FW_INTERFACE_IN_LINK_CONTROL_MODE,
3558 		    AQ2_FW_INTERFACE_IN_LINK_CONTROL_MODE_SHUTDOWN);
3559 	} else {
3560 		AQ_WRITE_REG_BIT(sc, AQ2_FW_INTERFACE_IN_LINK_CONTROL_REG,
3561 		    AQ2_FW_INTERFACE_IN_LINK_CONTROL_MODE,
3562 		    AQ2_FW_INTERFACE_IN_LINK_CONTROL_MODE_ACTIVE);
3563 		v |= AQ2_FW_INTERFACE_IN_LINK_OPTIONS_LINK_UP;
3564 	}
3565 
3566 	AQ_WRITE_REG(sc, AQ2_FW_INTERFACE_IN_LINK_OPTIONS_REG, v);
3567 	error = aq2_fw_wait_shared_ack(sc);
3568 
3569 	AQ_MPI_UNLOCK(sc);
3570 	return error;
3571 }
3572 
3573 static int
aq2_fw_get_mode(struct aq_softc * sc,aq_hw_fw_mpi_state_t * modep,aq_link_speed_t * speedp,aq_link_fc_t * fcp,aq_link_eee_t * eeep)3574 aq2_fw_get_mode(struct aq_softc *sc, aq_hw_fw_mpi_state_t *modep,
3575     aq_link_speed_t *speedp, aq_link_fc_t *fcp, aq_link_eee_t *eeep)
3576 {
3577 	aq_link_speed_t speed;
3578 	uint32_t v;
3579 
3580 	v = AQ_READ_REG(sc, AQ2_FW_INTERFACE_OUT_LINK_STATUS_REG);
3581 	switch (__SHIFTOUT(v, AQ2_FW_INTERFACE_OUT_LINK_STATUS_RATE)) {
3582 	case AQ2_FW_INTERFACE_OUT_LINK_STATUS_RATE_10G:
3583 		speed = AQ_LINK_10G;
3584 		break;
3585 	case AQ2_FW_INTERFACE_OUT_LINK_STATUS_RATE_5G:
3586 		speed = AQ_LINK_5G;
3587 		break;
3588 	case AQ2_FW_INTERFACE_OUT_LINK_STATUS_RATE_2G5:
3589 		speed = AQ_LINK_2G5;
3590 		break;
3591 	case AQ2_FW_INTERFACE_OUT_LINK_STATUS_RATE_1G:
3592 		speed = AQ_LINK_1G;
3593 		break;
3594 	case AQ2_FW_INTERFACE_OUT_LINK_STATUS_RATE_100M:
3595 		speed = AQ_LINK_100M;
3596 		break;
3597 	case AQ2_FW_INTERFACE_OUT_LINK_STATUS_RATE_10M:
3598 		speed = AQ_LINK_10M;
3599 		break;
3600 	case AQ2_FW_INTERFACE_OUT_LINK_STATUS_RATE_INVALID:
3601 	default:
3602 		speed = AQ_LINK_NONE;
3603 		break;
3604 	}
3605 	if (speedp != NULL)
3606 		*speedp = speed;
3607 
3608 	aq_link_fc_t fc = 0;
3609 	if (v & AQ2_FW_INTERFACE_OUT_LINK_STATUS_PAUSE_TX)
3610 		fc |= AQ_FC_TX;
3611 	if (v & AQ2_FW_INTERFACE_OUT_LINK_STATUS_PAUSE_RX)
3612 		fc |= AQ_FC_RX;
3613 	if (fcp != NULL)
3614 		*fcp = fc;
3615 
3616 	aq_link_eee_t eee;
3617 	eee = (v & AQ2_FW_INTERFACE_OUT_LINK_STATUS_EEE) ?
3618 	    AQ_EEE_ENABLE : AQ_EEE_DISABLE;
3619 	if (eeep != NULL)
3620 		*eeep = eee;
3621 
3622 	return -1;
3623 }
3624 
3625 static int
aq2_fw_get_stats(struct aq_softc * sc,aq_hw_stats_s_t * stats)3626 aq2_fw_get_stats(struct aq_softc *sc, aq_hw_stats_s_t *stats)
3627 {
3628 	aq2_statistics_t aq2stat;
3629 	int error;
3630 
3631 	AQ_MPI_LOCK(sc);
3632 	error = aq2_interface_buffer_read(sc, AQ2_FW_INTERFACE_OUT_STATS_REG,
3633 	    (uint32_t *)&aq2stat, sizeof(aq2stat));
3634 	AQ_MPI_UNLOCK(sc);
3635 	if (error != 0)
3636 		return error;
3637 
3638 	if (sc->sc_features & FEATURES_AQ2_IFACE_A0) {
3639 		/* RX */
3640 		stats->uprc = aq2stat.a0.rx_unicast_frames;
3641 		stats->mprc = aq2stat.a0.rx_multicast_frames;
3642 		stats->bprc = aq2stat.a0.rx_broadcast_frames;
3643 		stats->erpr = aq2stat.a0.rx_errors;
3644 		stats->ubrc = aq2stat.a0.rx_unicast_octets;
3645 		stats->bbrc = aq2stat.a0.rx_broadcast_octets;
3646 		stats->mbrc = aq2stat.a0.rx_multicast_octets;
3647 		stats->prc = aq2stat.a0.rx_good_frames;
3648 		/* TX */
3649 		stats->uptc = aq2stat.a0.tx_unicast_frames;
3650 		stats->bptc = aq2stat.a0.tx_broadcast_frames;
3651 		stats->mptc = aq2stat.a0.tx_multicast_frames;
3652 		stats->erpt = aq2stat.a0.tx_errors;
3653 		stats->ubtc = aq2stat.a0.tx_unicast_octets;
3654 		stats->bbtc = aq2stat.a0.tx_broadcast_octets;
3655 		stats->mbtc = aq2stat.a0.tx_multicast_octets;
3656 		stats->ptc = aq2stat.a0.tx_good_frames;
3657 	} else if (sc->sc_features & FEATURES_AQ2_IFACE_B0) {
3658 		/* RX */
3659 		stats->uprc = aq2stat.b0.rx_unicast_frames;
3660 		stats->mprc = aq2stat.b0.rx_multicast_frames;
3661 		stats->bprc = aq2stat.b0.rx_broadcast_frames;
3662 		stats->erpr = aq2stat.b0.rx_errors;
3663 		stats->ubrc = 0;
3664 		stats->bbrc = 0;
3665 		stats->mbrc = 0;
3666 		stats->prc = aq2stat.b0.rx_good_frames;
3667 		/* TX */
3668 		stats->uptc = aq2stat.b0.tx_unicast_frames;
3669 		stats->bptc = aq2stat.b0.tx_multicast_frames;
3670 		stats->mptc = aq2stat.b0.tx_broadcast_frames;
3671 		stats->erpt = aq2stat.b0.tx_errors;
3672 		stats->ubtc = 0;
3673 		stats->bbtc = 0;
3674 		stats->mbtc = 0;
3675 		stats->ptc = aq2stat.b0.tx_good_frames;
3676 	} else {
3677 		return ENOTSUP;
3678 	}
3679 	stats->dpc = AQ_READ64_REG(sc, RX_DMA_DROP_PKT_CNT_REG);
3680 	stats->cprc = AQ_READ64_REG(sc, RX_DMA_COALESCED_PKT_CNT_REG);
3681 
3682 	return error;
3683 }
3684 
3685 #if NSYSMON_ENVSYS > 0
3686 static int
aq2_fw_get_temperature(struct aq_softc * sc,uint32_t * temp)3687 aq2_fw_get_temperature(struct aq_softc *sc, uint32_t *temp)
3688 {
3689 	aq2_health_monitor_t health;
3690 	uint32_t data;
3691 
3692 	AQ_MPI_LOCK(sc);
3693 
3694 	aq2_interface_buffer_read(sc, AQ2_FW_INTERFACE_OUT_PHY_HEALTH_MONITOR,
3695 	    (uint32_t *)&health, sizeof(health));
3696 
3697 	AQ_MPI_UNLOCK(sc);
3698 
3699 	data = __SHIFTOUT(health.data1, HEALTH_MONITOR_DATA1_TEMPERATURE);
3700 	if (data == 0)
3701 		return EIO;
3702 
3703 	*temp = data * 1000000 + 273150000;
3704 	return 0;
3705 }
3706 #endif
3707 
3708 static int
aq2_get_mac_addr(struct aq_softc * sc)3709 aq2_get_mac_addr(struct aq_softc *sc)
3710 {
3711 	uint32_t mac_addr[2];
3712 
3713 	memset(mac_addr, 0, sizeof(mac_addr));
3714 	AQ_READ_REGS(sc, AQ2_FW_INTERFACE_IN_MAC_ADDRESS_REG,
3715 	    mac_addr, __arraycount(mac_addr));
3716 
3717 	if (mac_addr[0] == 0 && mac_addr[1] == 0) {
3718 		aprint_error_dev(sc->sc_dev, "mac address not found\n");
3719 		return ENXIO;
3720 	}
3721 
3722 	HTOLE32(mac_addr[0]);
3723 	HTOLE32(mac_addr[1]);
3724 
3725 	memcpy(sc->sc_enaddr.ether_addr_octet,
3726 	    (uint8_t *)mac_addr, ETHER_ADDR_LEN);
3727 	aprint_normal_dev(sc->sc_dev, "Etheraddr: %s\n",
3728 	    ether_sprintf(sc->sc_enaddr.ether_addr_octet));
3729 
3730 	return 0;
3731 }
3732 
3733 static int
aq_ifmedia_change(struct ifnet * const ifp)3734 aq_ifmedia_change(struct ifnet * const ifp)
3735 {
3736 	struct aq_softc * const sc = ifp->if_softc;
3737 
3738 	aq_link_speed_t rate = AQ_LINK_NONE;
3739 	aq_link_fc_t fc = AQ_FC_NONE;
3740 	aq_link_eee_t eee = AQ_EEE_DISABLE;
3741 
3742 	if (IFM_TYPE(sc->sc_media.ifm_media) != IFM_ETHER)
3743 		return EINVAL;
3744 
3745 	switch (IFM_SUBTYPE(sc->sc_media.ifm_media)) {
3746 	case IFM_AUTO:
3747 		rate = AQ_LINK_AUTO;
3748 		break;
3749 	case IFM_NONE:
3750 		rate = AQ_LINK_NONE;
3751 		break;
3752 	case IFM_10_T:
3753 		rate = AQ_LINK_10M;
3754 		break;
3755 	case IFM_100_TX:
3756 		rate = AQ_LINK_100M;
3757 		break;
3758 	case IFM_1000_T:
3759 		rate = AQ_LINK_1G;
3760 		break;
3761 	case IFM_2500_T:
3762 		rate = AQ_LINK_2G5;
3763 		break;
3764 	case IFM_5000_T:
3765 		rate = AQ_LINK_5G;
3766 		break;
3767 	case IFM_10G_T:
3768 		rate = AQ_LINK_10G;
3769 		break;
3770 	default:
3771 		device_printf(sc->sc_dev, "unknown media: 0x%X\n",
3772 		    IFM_SUBTYPE(sc->sc_media.ifm_media));
3773 		return ENODEV;
3774 	}
3775 
3776 	if (sc->sc_media.ifm_media & IFM_FLOW)
3777 		fc = AQ_FC_ALL;
3778 
3779 	/* XXX: todo EEE */
3780 
3781 	/* re-initialize hardware with new parameters */
3782 	aq_set_linkmode(sc, rate, fc, eee);
3783 
3784 	return 0;
3785 }
3786 
3787 static void
aq_ifmedia_status(struct ifnet * const ifp,struct ifmediareq * ifmr)3788 aq_ifmedia_status(struct ifnet * const ifp, struct ifmediareq *ifmr)
3789 {
3790 	struct aq_softc * const sc = ifp->if_softc;
3791 
3792 	/* update ifm_active */
3793 	ifmr->ifm_active = IFM_ETHER;
3794 	if (sc->sc_link_fc & AQ_FC_RX)
3795 		ifmr->ifm_active |= IFM_ETH_RXPAUSE;
3796 	if (sc->sc_link_fc & AQ_FC_TX)
3797 		ifmr->ifm_active |= IFM_ETH_TXPAUSE;
3798 
3799 	/* XXX: need to detect fulldup or halfdup */
3800 	switch (sc->sc_link_rate) {
3801 	case AQ_LINK_10M:
3802 		ifmr->ifm_active |= IFM_10_T | IFM_FDX;
3803 		break;
3804 	case AQ_LINK_100M:
3805 		ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
3806 		break;
3807 	case AQ_LINK_1G:
3808 		ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
3809 		break;
3810 	case AQ_LINK_2G5:
3811 		ifmr->ifm_active |= IFM_2500_T | IFM_FDX;
3812 		break;
3813 	case AQ_LINK_5G:
3814 		ifmr->ifm_active |= IFM_5000_T | IFM_FDX;
3815 		break;
3816 	case AQ_LINK_10G:
3817 		ifmr->ifm_active |= IFM_10G_T | IFM_FDX;
3818 		break;
3819 	default:
3820 		ifmr->ifm_active |= IFM_NONE;
3821 		break;
3822 	}
3823 
3824 	/* update ifm_status */
3825 	ifmr->ifm_status = IFM_AVALID;
3826 	if (sc->sc_link_rate != AQ_LINK_NONE)
3827 		ifmr->ifm_status |= IFM_ACTIVE;
3828 }
3829 
3830 static void
aq_initmedia(struct aq_softc * sc)3831 aq_initmedia(struct aq_softc *sc)
3832 {
3833 #define IFMEDIA_ETHER_ADD(sc, media)	\
3834 	ifmedia_add(&(sc)->sc_media, IFM_ETHER | media, 0, NULL);
3835 
3836 	IFMEDIA_ETHER_ADD(sc, IFM_NONE);
3837 
3838 	if (sc->sc_available_rates & AQ_LINK_10M) {
3839 		IFMEDIA_ETHER_ADD(sc, IFM_10_T);
3840 		IFMEDIA_ETHER_ADD(sc, IFM_10_T | IFM_FDX);
3841 	}
3842 	if (sc->sc_available_rates & AQ_LINK_100M) {
3843 		IFMEDIA_ETHER_ADD(sc, IFM_100_TX);
3844 		IFMEDIA_ETHER_ADD(sc, IFM_100_TX | IFM_FLOW);
3845 		IFMEDIA_ETHER_ADD(sc, IFM_100_TX | IFM_FDX | IFM_FLOW);
3846 	}
3847 	if (sc->sc_available_rates & AQ_LINK_1G) {
3848 		IFMEDIA_ETHER_ADD(sc, IFM_1000_T | IFM_FDX);
3849 		IFMEDIA_ETHER_ADD(sc, IFM_1000_T | IFM_FDX | IFM_FLOW);
3850 	}
3851 	if (sc->sc_available_rates & AQ_LINK_2G5) {
3852 		IFMEDIA_ETHER_ADD(sc, IFM_2500_T | IFM_FDX);
3853 		IFMEDIA_ETHER_ADD(sc, IFM_2500_T | IFM_FDX | IFM_FLOW);
3854 	}
3855 	if (sc->sc_available_rates & AQ_LINK_5G) {
3856 		IFMEDIA_ETHER_ADD(sc, IFM_5000_T | IFM_FDX);
3857 		IFMEDIA_ETHER_ADD(sc, IFM_5000_T | IFM_FDX | IFM_FLOW);
3858 	}
3859 	if (sc->sc_available_rates & AQ_LINK_10G) {
3860 		IFMEDIA_ETHER_ADD(sc, IFM_10G_T | IFM_FDX);
3861 		IFMEDIA_ETHER_ADD(sc, IFM_10G_T | IFM_FDX | IFM_FLOW);
3862 	}
3863 	IFMEDIA_ETHER_ADD(sc, IFM_AUTO);
3864 	IFMEDIA_ETHER_ADD(sc, IFM_AUTO | IFM_FLOW);
3865 
3866 	/* default: auto without flowcontrol */
3867 	ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO);
3868 	aq_set_linkmode(sc, AQ_LINK_AUTO, AQ_FC_NONE, AQ_EEE_DISABLE);
3869 }
3870 
3871 static int
aq_set_linkmode(struct aq_softc * sc,aq_link_speed_t speed,aq_link_fc_t fc,aq_link_eee_t eee)3872 aq_set_linkmode(struct aq_softc *sc, aq_link_speed_t speed, aq_link_fc_t fc,
3873     aq_link_eee_t eee)
3874 {
3875 	return sc->sc_fw_ops->set_mode(sc, MPI_INIT, speed, fc, eee);
3876 }
3877 
3878 static int
aq_get_linkmode(struct aq_softc * sc,aq_link_speed_t * speed,aq_link_fc_t * fc,aq_link_eee_t * eee)3879 aq_get_linkmode(struct aq_softc *sc, aq_link_speed_t *speed, aq_link_fc_t *fc,
3880    aq_link_eee_t *eee)
3881 {
3882 	aq_hw_fw_mpi_state_t mode;
3883 	int error;
3884 
3885 	error = sc->sc_fw_ops->get_mode(sc, &mode, speed, fc, eee);
3886 	if (error != 0)
3887 		return error;
3888 	if (mode != MPI_INIT)
3889 		return ENXIO;
3890 
3891 	return 0;
3892 }
3893 
3894 static void
aq_hw_init_tx_path(struct aq_softc * sc)3895 aq_hw_init_tx_path(struct aq_softc *sc)
3896 {
3897 	/* Tx TC/RSS number config */
3898 	AQ_WRITE_REG_BIT(sc, TPB_TX_BUF_REG, TPB_TX_BUF_TC_MODE,
3899 	    (sc->sc_tc_mode == 4) ? 1 : 0);
3900 
3901 	AQ_WRITE_REG_BIT(sc, THM_LSO_TCP_FLAG1_REG,
3902 	    THM_LSO_TCP_FLAG1_FIRST, 0x0ff6);
3903 	AQ_WRITE_REG_BIT(sc, THM_LSO_TCP_FLAG1_REG,
3904 	    THM_LSO_TCP_FLAG1_MID,   0x0ff6);
3905 	AQ_WRITE_REG_BIT(sc, THM_LSO_TCP_FLAG2_REG,
3906 	   THM_LSO_TCP_FLAG2_LAST,  0x0f7f);
3907 
3908 	/* misc */
3909 	AQ_WRITE_REG(sc, TX_TPO2_REG,
3910 	   (sc->sc_features & FEATURES_TPO2) ? TX_TPO2_EN : 0);
3911 	AQ_WRITE_REG_BIT(sc, TDM_DCA_REG, TDM_DCA_EN, 0);
3912 	AQ_WRITE_REG_BIT(sc, TDM_DCA_REG, TDM_DCA_MODE, 0);
3913 
3914 	AQ_WRITE_REG_BIT(sc, TPB_TX_BUF_REG, TPB_TX_BUF_SCP_INS_EN, 1);
3915 
3916 	if ((sc->sc_features & FEATURES_AQ1_REV_B) || HWTYPE_AQ2_P(sc)) {
3917 		AQ_WRITE_REG_BIT(sc, TPB_TX_BUF_REG, TPB_TX_BUF_CLK_GATE_EN, 0);
3918 	}
3919 }
3920 
3921 static void
aq_hw_init_rx_path(struct aq_softc * sc)3922 aq_hw_init_rx_path(struct aq_softc *sc)
3923 {
3924 	int i;
3925 
3926 	/* Rx TC/RSS number config */
3927 	AQ_WRITE_REG_BIT(sc, RPB_RPF_RX_REG, RPB_RPF_RX_TC_MODE,
3928 	    (sc->sc_tc_mode == 4) ? 1 : 0);
3929 
3930 	/* Rx flow control */
3931 	AQ_WRITE_REG_BIT(sc, RPB_RPF_RX_REG, RPB_RPF_RX_FC_MODE, 0);
3932 
3933 	if (HWTYPE_AQ2_P(sc)) {
3934 		AQ_WRITE_REG_BIT(sc, AQ2_RPF_REDIR2_REG,
3935 		    AQ2_RPF_REDIR2_HASHTYPE, AQ2_RPF_REDIR2_HASHTYPE_ALL);
3936 	}
3937 
3938 	if (sc->sc_rss_enable) {
3939 		/* RSS Ring selection */
3940 		switch (sc->sc_nqueues) {
3941 		case 2:
3942 			AQ_WRITE_REG(sc, RX_FLR_RSS_CONTROL1_REG,
3943 			    RX_FLR_RSS_CONTROL1_EN | 0x11111111);
3944 			break;
3945 		case 4:
3946 			AQ_WRITE_REG(sc, RX_FLR_RSS_CONTROL1_REG,
3947 			    RX_FLR_RSS_CONTROL1_EN | 0x22222222);
3948 			break;
3949 		case 8:
3950 			AQ_WRITE_REG(sc, RX_FLR_RSS_CONTROL1_REG,
3951 			    RX_FLR_RSS_CONTROL1_EN | 0x33333333);
3952 			break;
3953 		}
3954 	} else {
3955 		/* disable RSS */
3956 		AQ_WRITE_REG(sc, RX_FLR_RSS_CONTROL1_REG, 0);
3957 	}
3958 
3959 	if (HWTYPE_AQ1_P(sc)) {
3960 		/* multicast filter */
3961 		for (i = 0; i < 32; i++) {
3962 			AQ_WRITE_REG_BIT(sc, RPF_ETHERTYPE_FILTER_REG(i),
3963 			   RPF_ETHERTYPE_FILTER_EN, 0);
3964 		}
3965 	}
3966 
3967 	/* L2 and Multicast filters */
3968 	for (i = 0; i < AQ_HW_MAC_NUM(sc); i++) {
3969 		AQ_WRITE_REG_BIT(sc, RPF_L2UC_MSW_REG(i), RPF_L2UC_MSW_EN, 0);
3970 		AQ_WRITE_REG_BIT(sc, RPF_L2UC_MSW_REG(i), RPF_L2UC_MSW_ACTION,
3971 		    RPF_ACTION_HOST);
3972 	}
3973 	AQ_WRITE_REG(sc, RPF_MCAST_FILTER_MASK_REG, 0);
3974 	AQ_WRITE_REG(sc, RPF_MCAST_FILTER_REG(0), 0x00010fff);
3975 
3976 	/* Vlan filters */
3977 	AQ_WRITE_REG_BIT(sc, RPF_VLAN_TPID_REG, RPF_VLAN_TPID_OUTER,
3978 	    ETHERTYPE_QINQ);
3979 	AQ_WRITE_REG_BIT(sc, RPF_VLAN_TPID_REG, RPF_VLAN_TPID_INNER,
3980 	    ETHERTYPE_VLAN);
3981 	AQ_WRITE_REG_BIT(sc, RPF_VLAN_MODE_REG, RPF_VLAN_MODE_PROMISC, 0);
3982 
3983 	if ((sc->sc_features & FEATURES_AQ1_REV_B) || HWTYPE_AQ2_P(sc)) {
3984 		AQ_WRITE_REG_BIT(sc, RPF_VLAN_MODE_REG,
3985 		    RPF_VLAN_MODE_ACCEPT_UNTAGGED, 1);
3986 		AQ_WRITE_REG_BIT(sc, RPF_VLAN_MODE_REG,
3987 		    RPF_VLAN_MODE_UNTAGGED_ACTION, RPF_ACTION_HOST);
3988 	}
3989 
3990 	if (HWTYPE_AQ2_P(sc)) {
3991 		aq2_init_filter(sc);
3992 	}
3993 
3994 	AQ_WRITE_REG_BIT(sc, RX_DMA_INT_DESC_WRWB_EN_REG,
3995 	    RX_DMA_INT_DESC_WRWB_EN, 1);
3996 
3997 	if (HWTYPE_AQ1_P(sc)) {
3998 		if (sc->sc_features & FEATURES_RPF2) {
3999 			AQ_WRITE_REG(sc, RX_TCP_RSS_HASH_REG,
4000 			    RX_TCP_RSS_HASH_RPF2);
4001 		} else {
4002 			AQ_WRITE_REG(sc, RX_TCP_RSS_HASH_REG, 0);
4003 		}
4004 		/*
4005 		 * XXX: RX_TCP_RSS_HASH_REG:
4006 		 *  linux   set 0x000f0000
4007 		 *  freebsd set 0x000f001e
4008 		 */
4009 		/* RSS hash type set for IP/TCP */
4010 		AQ_WRITE_REG_BIT(sc, RX_TCP_RSS_HASH_REG,
4011 		    RX_TCP_RSS_HASH_TYPE, 0x001e);
4012 	}
4013 
4014 	AQ_WRITE_REG_BIT(sc, RPF_L2BC_REG, RPF_L2BC_EN, 1);
4015 	AQ_WRITE_REG_BIT(sc, RPF_L2BC_REG, RPF_L2BC_ACTION, RPF_ACTION_HOST);
4016 	AQ_WRITE_REG_BIT(sc, RPF_L2BC_REG, RPF_L2BC_THRESHOLD, 0xffff);
4017 
4018 	AQ_WRITE_REG_BIT(sc, RX_DMA_DCA_REG, RX_DMA_DCA_EN, 0);
4019 	AQ_WRITE_REG_BIT(sc, RX_DMA_DCA_REG, RX_DMA_DCA_MODE, 0);
4020 }
4021 
4022 static void
aq_hw_interrupt_moderation_set(struct aq_softc * sc)4023 aq_hw_interrupt_moderation_set(struct aq_softc *sc)
4024 {
4025 	uint32_t v;
4026 	int i;
4027 
4028 	if (sc->sc_intr_moderation_enable) {
4029 		unsigned int tx_min, rx_min;	/* 0-255 */
4030 		unsigned int tx_max, rx_max;	/* 0-511? */
4031 
4032 		switch (sc->sc_link_rate) {
4033 		case AQ_LINK_10M:
4034 		case AQ_LINK_100M:
4035 			tx_min = 0x4f;
4036 			tx_max = 0xff;
4037 			rx_min = 0x04;
4038 			rx_max = 0x50;
4039 			break;
4040 		case AQ_LINK_1G:
4041 		default:
4042 			tx_min = 0x4f;
4043 			tx_max = 0xff;
4044 			rx_min = 0x30;
4045 			rx_max = 0x80;
4046 			break;
4047 		case AQ_LINK_2G5:
4048 			tx_min = 0x4f;
4049 			tx_max = 0xff;
4050 			rx_min = 0x18;
4051 			rx_max = 0xe0;
4052 			break;
4053 		case AQ_LINK_5G:
4054 			tx_min = 0x4f;
4055 			tx_max = 0xff;
4056 			rx_min = 0x0c;
4057 			rx_max = 0x70;
4058 			break;
4059 		case AQ_LINK_10G:
4060 			tx_min = 0x4f;
4061 			tx_max = 0x1ff;
4062 			rx_min = 0x06;	/* freebsd use 80 */
4063 			rx_max = 0x38;	/* freebsd use 120 */
4064 			break;
4065 		}
4066 
4067 		AQ_WRITE_REG_BIT(sc, TX_DMA_INT_DESC_WRWB_EN_REG,
4068 		    TX_DMA_INT_DESC_WRWB_EN, 0);
4069 		AQ_WRITE_REG_BIT(sc, TX_DMA_INT_DESC_WRWB_EN_REG,
4070 		    TX_DMA_INT_DESC_MODERATE_EN, 1);
4071 		AQ_WRITE_REG_BIT(sc, RX_DMA_INT_DESC_WRWB_EN_REG,
4072 		    RX_DMA_INT_DESC_WRWB_EN, 0);
4073 		AQ_WRITE_REG_BIT(sc, RX_DMA_INT_DESC_WRWB_EN_REG,
4074 		    RX_DMA_INT_DESC_MODERATE_EN, 1);
4075 
4076 		if (HWTYPE_AQ2_P(sc)) {
4077 			v = __SHIFTIN(tx_min, AQ2_TX_INTR_MODERATION_CTL_MIN) |
4078 			    __SHIFTIN(tx_max, AQ2_TX_INTR_MODERATION_CTL_MAX) |
4079 			    AQ2_TX_INTR_MODERATION_CTL_EN;
4080 			for (i = 0; i < AQ_RINGS_NUM; i++) {
4081 				AQ_WRITE_REG(sc,
4082 				    AQ2_TX_INTR_MODERATION_CTL_REG(i), v);
4083 			}
4084 		} else {
4085 			v = __SHIFTIN(tx_min, TX_INTR_MODERATION_CTL_MIN) |
4086 			    __SHIFTIN(tx_max, TX_INTR_MODERATION_CTL_MAX) |
4087 			    TX_INTR_MODERATION_CTL_EN;
4088 			for (i = 0; i < AQ_RINGS_NUM; i++) {
4089 				AQ_WRITE_REG(sc,
4090 				    TX_INTR_MODERATION_CTL_REG(i), v);
4091 			}
4092 		}
4093 
4094 		for (i = 0; i < AQ_RINGS_NUM; i++) {
4095 			AQ_WRITE_REG(sc, RX_INTR_MODERATION_CTL_REG(i),
4096 			    __SHIFTIN(rx_min, RX_INTR_MODERATION_CTL_MIN) |
4097 			    __SHIFTIN(rx_max, RX_INTR_MODERATION_CTL_MAX) |
4098 			    RX_INTR_MODERATION_CTL_EN);
4099 		}
4100 
4101 	} else {
4102 		AQ_WRITE_REG_BIT(sc, TX_DMA_INT_DESC_WRWB_EN_REG,
4103 		    TX_DMA_INT_DESC_WRWB_EN, 1);
4104 		AQ_WRITE_REG_BIT(sc, TX_DMA_INT_DESC_WRWB_EN_REG,
4105 		    TX_DMA_INT_DESC_MODERATE_EN, 0);
4106 		AQ_WRITE_REG_BIT(sc, RX_DMA_INT_DESC_WRWB_EN_REG,
4107 		    RX_DMA_INT_DESC_WRWB_EN, 1);
4108 		AQ_WRITE_REG_BIT(sc, RX_DMA_INT_DESC_WRWB_EN_REG,
4109 		    RX_DMA_INT_DESC_MODERATE_EN, 0);
4110 
4111 		if (HWTYPE_AQ2_P(sc)) {
4112 			for (i = 0; i < AQ_RINGS_NUM; i++) {
4113 				AQ_WRITE_REG(sc,
4114 				    AQ2_TX_INTR_MODERATION_CTL_REG(i), 0);
4115 			}
4116 		} else {
4117 			for (i = 0; i < AQ_RINGS_NUM; i++) {
4118 				AQ_WRITE_REG(sc,
4119 				    TX_INTR_MODERATION_CTL_REG(i), 0);
4120 			}
4121 		}
4122 
4123 		for (i = 0; i < AQ_RINGS_NUM; i++) {
4124 			AQ_WRITE_REG(sc, RX_INTR_MODERATION_CTL_REG(i), 0);
4125 		}
4126 	}
4127 }
4128 
4129 static void
aq_hw_qos_set(struct aq_softc * sc)4130 aq_hw_qos_set(struct aq_softc *sc)
4131 {
4132 	uint32_t tc, tx_bufsize, rx_bufsize;
4133 
4134 	/* TPS Descriptor rate init */
4135 	AQ_WRITE_REG_BIT(sc, TPS_DESC_RATE_REG, TPS_DESC_RATE_TA_RST, 0);
4136 	AQ_WRITE_REG_BIT(sc, TPS_DESC_RATE_REG, TPS_DESC_RATE_LIM, 0xa);
4137 
4138 	/* TPS VM init */
4139 	AQ_WRITE_REG_BIT(sc, TPS_DESC_VM_ARB_MODE_REG, TPS_DESC_VM_ARB_MODE, 0);
4140 
4141 	/* TPS TC credits init */
4142 	AQ_WRITE_REG_BIT(sc, TPS_DESC_TC_ARB_MODE_REG, TPS_DESC_TC_ARB_MODE, 0);
4143 	AQ_WRITE_REG_BIT(sc, TPS_DATA_TC_ARB_MODE_REG, TPS_DATA_TC_ARB_MODE, 0);
4144 
4145 	if (HWTYPE_AQ1_P(sc)) {
4146 		tx_bufsize = AQ1_HW_TXBUF_MAX / sc->sc_tcs;
4147 		rx_bufsize = AQ1_HW_RXBUF_MAX / sc->sc_tcs;
4148 	} else {
4149 		tx_bufsize = AQ2_HW_TXBUF_MAX / sc->sc_tcs;
4150 		rx_bufsize = AQ2_HW_RXBUF_MAX / sc->sc_tcs;
4151 	}
4152 
4153 	for (tc = 0; tc < sc->sc_tcs; tc++) {
4154 		AQ_WRITE_REG_BIT(sc, TPS_DATA_TCT_REG(tc),
4155 		    TPS_DATA_TCT_CREDIT_MAX, 0xfff);
4156 		AQ_WRITE_REG_BIT(sc, TPS_DATA_TCT_REG(tc),
4157 		    TPS_DATA_TCT_WEIGHT, 0x64);
4158 		AQ_WRITE_REG_BIT(sc, TPS_DESC_TCT_REG(tc),
4159 		    TPS_DESC_TCT_CREDIT_MAX, 0x50);
4160 		AQ_WRITE_REG_BIT(sc, TPS_DESC_TCT_REG(tc),
4161 		    TPS_DESC_TCT_WEIGHT, 0x1e);
4162 
4163 		/* Tx buf size */
4164 		AQ_WRITE_REG_BIT(sc, TPB_TXB_BUFSIZE_REG(tc), TPB_TXB_BUFSIZE,
4165 		    tx_bufsize);
4166 		AQ_WRITE_REG_BIT(sc, TPB_TXB_THRESH_REG(tc), TPB_TXB_THRESH_HI,
4167 		    (tx_bufsize * (1024 / 32) * 66) / 100);
4168 		AQ_WRITE_REG_BIT(sc, TPB_TXB_THRESH_REG(tc), TPB_TXB_THRESH_LO,
4169 		    (tx_bufsize * (1024 / 32) * 50) / 100);
4170 
4171 		/* QoS Rx buf size per TC */
4172 		AQ_WRITE_REG_BIT(sc, RPB_RXB_XOFF_REG(tc), RPB_RXB_XOFF_EN, 0);
4173 		AQ_WRITE_REG_BIT(sc, RPB_RXB_BUFSIZE_REG(tc), RPB_RXB_BUFSIZE,
4174 		    rx_bufsize);
4175 		AQ_WRITE_REG_BIT(sc, RPB_RXB_XOFF_REG(tc),
4176 		    RPB_RXB_XOFF_THRESH_HI,
4177 		    (rx_bufsize * (1024 / 32) * 66) / 100);
4178 		AQ_WRITE_REG_BIT(sc, RPB_RXB_XOFF_REG(tc),
4179 		    RPB_RXB_XOFF_THRESH_LO,
4180 		    (rx_bufsize * (1024 / 32) * 50) / 100);
4181 	}
4182 
4183 	/* QoS 802.1p priority -> TC mapping */
4184 	for (int pri = 0; pri < 8; pri++) {
4185 		AQ_WRITE_REG_BIT(sc, RPF_RPB_RX_TC_UPT_REG,
4186 		    RPF_RPB_RX_TC_UPT_MASK(pri), sc->sc_tcs * pri / 8);
4187 	}
4188 
4189 	/* ring to TC mapping */
4190 	if (HWTYPE_AQ2_P(sc)) {
4191 		AQ_WRITE_REG_BIT(sc, TPB_TX_BUF_REG,
4192 		    TPB_TX_BUF_TC_Q_RAND_MAP_EN, 1);
4193 		switch (sc->sc_tc_mode) {
4194 		case 4:
4195 			AQ_WRITE_REG(sc, AQ2_TX_Q_TC_MAP_REG(0), 0x00000000);
4196 			AQ_WRITE_REG(sc, AQ2_TX_Q_TC_MAP_REG(1), 0x00000000);
4197 			AQ_WRITE_REG(sc, AQ2_TX_Q_TC_MAP_REG(2), 0x01010101);
4198 			AQ_WRITE_REG(sc, AQ2_TX_Q_TC_MAP_REG(3), 0x01010101);
4199 			AQ_WRITE_REG(sc, AQ2_TX_Q_TC_MAP_REG(4), 0x02020202);
4200 			AQ_WRITE_REG(sc, AQ2_TX_Q_TC_MAP_REG(5), 0x02020202);
4201 			AQ_WRITE_REG(sc, AQ2_TX_Q_TC_MAP_REG(6), 0x03030303);
4202 			AQ_WRITE_REG(sc, AQ2_TX_Q_TC_MAP_REG(7), 0x03030303);
4203 
4204 			AQ_WRITE_REG(sc, AQ2_RX_Q_TC_MAP_REG(0), 0x00000000);
4205 			AQ_WRITE_REG(sc, AQ2_RX_Q_TC_MAP_REG(1), 0x11111111);
4206 			AQ_WRITE_REG(sc, AQ2_RX_Q_TC_MAP_REG(2), 0x22222222);
4207 			AQ_WRITE_REG(sc, AQ2_RX_Q_TC_MAP_REG(3), 0x33333333);
4208 			break;
4209 		case 8:
4210 			AQ_WRITE_REG(sc, AQ2_TX_Q_TC_MAP_REG(0), 0x00000000);
4211 			AQ_WRITE_REG(sc, AQ2_TX_Q_TC_MAP_REG(1), 0x01010101);
4212 			AQ_WRITE_REG(sc, AQ2_TX_Q_TC_MAP_REG(2), 0x02020202);
4213 			AQ_WRITE_REG(sc, AQ2_TX_Q_TC_MAP_REG(3), 0x03030303);
4214 			AQ_WRITE_REG(sc, AQ2_TX_Q_TC_MAP_REG(4), 0x04040404);
4215 			AQ_WRITE_REG(sc, AQ2_TX_Q_TC_MAP_REG(5), 0x05050505);
4216 			AQ_WRITE_REG(sc, AQ2_TX_Q_TC_MAP_REG(6), 0x06060606);
4217 			AQ_WRITE_REG(sc, AQ2_TX_Q_TC_MAP_REG(7), 0x07070707);
4218 
4219 			AQ_WRITE_REG(sc, AQ2_RX_Q_TC_MAP_REG(0), 0x11110000);
4220 			AQ_WRITE_REG(sc, AQ2_RX_Q_TC_MAP_REG(1), 0x33332222);
4221 			AQ_WRITE_REG(sc, AQ2_RX_Q_TC_MAP_REG(2), 0x55554444);
4222 			AQ_WRITE_REG(sc, AQ2_RX_Q_TC_MAP_REG(3), 0x77776666);
4223 			break;
4224 		}
4225 	}
4226 }
4227 
4228 static int
aq_init_rss(struct aq_softc * sc)4229 aq_init_rss(struct aq_softc *sc)
4230 {
4231 	CTASSERT(AQ_RSS_HASHKEY_SIZE == RSS_KEYSIZE);
4232 	uint32_t rss_key[RSS_KEYSIZE / sizeof(uint32_t)];
4233 	uint8_t rss_table[AQ_RSS_INDIRECTION_TABLE_MAX];
4234 	unsigned int i;
4235 	int error;
4236 
4237 	if (HWTYPE_AQ2_P(sc)) {
4238 		uint32_t q_per_tc = (sc->sc_tc_mode == 8) ? 4 : 8;
4239 		uint32_t tc;
4240 
4241 		AQ_WRITE_REG_BIT(sc, AQ2_RPF_REDIR2_REG,
4242 		    AQ2_RPF_REDIR2_INDEX, (sc->sc_tc_mode == 8) ? 1 : 0);
4243 		for (i = 0; i < AQ_RSS_INDIRECTION_TABLE_MAX; i++) {
4244 			for (tc = 0; tc < sc->sc_tc_mode; tc++) {
4245 				uint32_t q = tc * q_per_tc + (i % sc->sc_nqueues);
4246 				AQ_WRITE_REG_BIT(sc, AQ2_RPF_RSS_REDIR_REG(tc, i),
4247 				    AQ2_RPF_RSS_REDIR_TC_MASK(tc), q);
4248 			}
4249 		}
4250 	}
4251 
4252 	/* initialize rss key */
4253 	rss_getkey((uint8_t *)rss_key);
4254 
4255 	/* hash to ring table */
4256 	for (i = 0; i < AQ_RSS_INDIRECTION_TABLE_MAX; i++) {
4257 		rss_table[i] = i % sc->sc_nqueues;
4258 	}
4259 
4260 	/*
4261 	 * set rss key
4262 	 */
4263 	for (i = 0; i < __arraycount(rss_key); i++) {
4264 		uint32_t key_data = sc->sc_rss_enable ? ntohl(rss_key[i]) : 0;
4265 		AQ_WRITE_REG(sc, RPF_RSS_KEY_WR_DATA_REG, key_data);
4266 		AQ_WRITE_REG_BIT(sc, RPF_RSS_KEY_ADDR_REG,
4267 		    RPF_RSS_KEY_ADDR, __arraycount(rss_key) - 1 - i);
4268 		AQ_WRITE_REG_BIT(sc, RPF_RSS_KEY_ADDR_REG,
4269 		    RPF_RSS_KEY_WR_EN, 1);
4270 		WAIT_FOR(AQ_READ_REG_BIT(sc, RPF_RSS_KEY_ADDR_REG,
4271 		    RPF_RSS_KEY_WR_EN) == 0, 1000, 10, &error);
4272 		if (error != 0) {
4273 			device_printf(sc->sc_dev, "%s: rss key write timeout\n",
4274 			    __func__);
4275 			goto rss_set_timeout;
4276 		}
4277 	}
4278 
4279 	/*
4280 	 * set rss indirection table
4281 	 *
4282 	 * AQ's rss redirect table is consist of 3bit*64 (192bit) packed array.
4283 	 * we'll make it by __BITMAP(3) macros.
4284 	 */
4285 	__BITMAP_TYPE(, uint16_t, 3 * AQ_RSS_INDIRECTION_TABLE_MAX) bit3x64;
4286 	__BITMAP_ZERO(&bit3x64);
4287 
4288 #define AQ_3BIT_PACKED_ARRAY_SET(bitmap, idx, val)		\
4289 	do {							\
4290 		if (val & 1) {					\
4291 			__BITMAP_SET((idx) * 3, (bitmap));	\
4292 		} else {					\
4293 			__BITMAP_CLR((idx) * 3, (bitmap));	\
4294 		}						\
4295 		if (val & 2) {					\
4296 			__BITMAP_SET((idx) * 3 + 1, (bitmap));	\
4297 		} else {					\
4298 			__BITMAP_CLR((idx) * 3 + 1, (bitmap));	\
4299 		}						\
4300 		if (val & 4) {					\
4301 			__BITMAP_SET((idx) * 3 + 2, (bitmap));	\
4302 		} else {					\
4303 			__BITMAP_CLR((idx) * 3 + 2, (bitmap));	\
4304 		}						\
4305 	} while (0 /* CONSTCOND */)
4306 
4307 	for (i = 0; i < AQ_RSS_INDIRECTION_TABLE_MAX; i++) {
4308 		AQ_3BIT_PACKED_ARRAY_SET(&bit3x64, i, rss_table[i]);
4309 	}
4310 
4311 	/* write 192bit data in steps of 16bit */
4312 	for (i = 0; i < (int)__arraycount(bit3x64._b); i++) {
4313 		AQ_WRITE_REG_BIT(sc, RPF_RSS_REDIR_WR_DATA_REG,
4314 		    RPF_RSS_REDIR_WR_DATA, bit3x64._b[i]);
4315 		AQ_WRITE_REG_BIT(sc, RPF_RSS_REDIR_ADDR_REG,
4316 		    RPF_RSS_REDIR_ADDR, i);
4317 		AQ_WRITE_REG_BIT(sc, RPF_RSS_REDIR_ADDR_REG,
4318 		    RPF_RSS_REDIR_WR_EN, 1);
4319 
4320 		WAIT_FOR(AQ_READ_REG_BIT(sc, RPF_RSS_REDIR_ADDR_REG,
4321 		    RPF_RSS_REDIR_WR_EN) == 0, 1000, 10, &error);
4322 		if (error != 0)
4323 			break;
4324 	}
4325 
4326  rss_set_timeout:
4327 	return error;
4328 }
4329 
4330 static void
aq1_hw_l3_filter_set(struct aq_softc * sc)4331 aq1_hw_l3_filter_set(struct aq_softc *sc)
4332 {
4333 	int i;
4334 
4335 	/* clear all filter */
4336 	for (i = 0; i < 8; i++) {
4337 		AQ_WRITE_REG_BIT(sc, RPF_L3_FILTER_REG(i),
4338 		    RPF_L3_FILTER_L4_EN, 0);
4339 	}
4340 }
4341 
4342 static void
aq_set_vlan_filters(struct aq_softc * sc)4343 aq_set_vlan_filters(struct aq_softc *sc)
4344 {
4345 	struct ethercom * const ec = &sc->sc_ethercom;
4346 	struct ifnet * const ifp = &sc->sc_ethercom.ec_if;
4347 	struct vlanid_list *vlanidp;
4348 	int i;
4349 
4350 	ETHER_LOCK(ec);
4351 
4352 	/* disable all vlan filters */
4353 	for (i = 0; i < RPF_VLAN_MAX_FILTERS; i++) {
4354 		AQ_WRITE_REG(sc, RPF_VLAN_FILTER_REG(i), 0);
4355 		if (HWTYPE_AQ2_P(sc)) {
4356 			aq2_filter_art_set(sc, AQ2_RPF_INDEX_VLAN_USER + i,
4357 			    0, 0, AQ2_ART_ACTION_DISABLE);
4358 			AQ_WRITE_REG_BIT(sc, RPF_VLAN_FILTER_REG(i),
4359 			    RPF_VLAN_FILTER_TAG, 1);
4360 		}
4361 	}
4362 
4363 	/* count VID */
4364 	i = 0;
4365 	SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list)
4366 		i++;
4367 
4368 	if (((sc->sc_ethercom.ec_capenable & ETHERCAP_VLAN_HWFILTER) == 0) ||
4369 	    (ifp->if_flags & IFF_PROMISC) ||
4370 	    (i > RPF_VLAN_MAX_FILTERS)) {
4371 		/*
4372 		 * no vlan hwfilter, in promiscuous mode, or too many VID?
4373 		 * must receive all VID
4374 		 */
4375 		AQ_WRITE_REG_BIT(sc, RPF_VLAN_MODE_REG,
4376 		    RPF_VLAN_MODE_PROMISC, 1);
4377 		goto done;
4378 	}
4379 
4380 	/* receive only selected VID */
4381 	AQ_WRITE_REG_BIT(sc, RPF_VLAN_MODE_REG, RPF_VLAN_MODE_PROMISC, 0);
4382 	i = 0;
4383 	SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) {
4384 		AQ_WRITE_REG_BIT(sc, RPF_VLAN_FILTER_REG(i),
4385 		    RPF_VLAN_FILTER_EN, 1);
4386 		AQ_WRITE_REG_BIT(sc, RPF_VLAN_FILTER_REG(i),
4387 		    RPF_VLAN_FILTER_RXQ_EN, 0);
4388 		AQ_WRITE_REG_BIT(sc, RPF_VLAN_FILTER_REG(i),
4389 		    RPF_VLAN_FILTER_RXQ, 0);
4390 		AQ_WRITE_REG_BIT(sc, RPF_VLAN_FILTER_REG(i),
4391 		    RPF_VLAN_FILTER_ACTION, RPF_ACTION_HOST);
4392 		AQ_WRITE_REG_BIT(sc, RPF_VLAN_FILTER_REG(i),
4393 		    RPF_VLAN_FILTER_ID, vlanidp->vid);
4394 
4395 		if (HWTYPE_AQ2_P(sc)) {
4396 			/*
4397 			 * If you want to fix the ring (CPU) for each VLAN ID,
4398 			 * Use AQ2_ART_ACTION_ASSIGN_QUEUE(i % sc->sc_nqueues)
4399 			 * instead of AQ2_ART_ACTION_ASSIGN_TC().
4400 			 */
4401 			uint32_t action =
4402 			    AQ2_ART_ACTION_ASSIGN_TC(i % sc->sc_nqueues);
4403 
4404 			AQ_WRITE_REG_BIT(sc, RPF_VLAN_FILTER_REG(i),
4405 			    RPF_VLAN_FILTER_TAG, i + 2);
4406 			aq2_filter_art_set(sc, AQ2_RPF_INDEX_VLAN_USER + i,
4407 			    __SHIFTIN(i + 2, AQ2_RPF_TAG_VLAN_MASK),
4408 			    AQ2_RPF_TAG_VLAN_MASK, action);
4409 		}
4410 		i++;
4411 	}
4412 
4413  done:
4414 	ETHER_UNLOCK(ec);
4415 }
4416 
4417 static int
aq_hw_init(struct aq_softc * sc)4418 aq_hw_init(struct aq_softc *sc)
4419 {
4420 	uint32_t v;
4421 
4422 	if (HWTYPE_AQ1_P(sc)) {
4423 		/* Force limit MRRS on RDM/TDM to 2K */
4424 		v = AQ_READ_REG(sc, AQ_PCI_REG_CONTROL_6_REG);
4425 		AQ_WRITE_REG(sc, AQ_PCI_REG_CONTROL_6_REG,
4426 		    (v & ~0x0707) | 0x0404);
4427 
4428 		/*
4429 		 * TX DMA total request limit. B0 hardware is not capable to
4430 		 * handle more than (8K-MRRS) incoming DMA data.
4431 		 * Value 24 in 256byte units
4432 		 */
4433 		AQ_WRITE_REG(sc, AQ_HW_TX_DMA_TOTAL_REQ_LIMIT_REG, 24);
4434 	}
4435 
4436 	if (HWTYPE_AQ2_P(sc)) {
4437 		uint32_t fpgaver, speed;
4438 		fpgaver = AQ_READ_REG(sc, AQ2_HW_FPGA_VERSION_REG);
4439 		if (fpgaver < 0x01000000)
4440 			speed = AQ2_LAUNCHTIME_CTRL_RATIO_SPEED_FULL;
4441 		else if (fpgaver >= 0x01008502)
4442 			speed = AQ2_LAUNCHTIME_CTRL_RATIO_SPEED_HALF;
4443 		else
4444 			speed = AQ2_LAUNCHTIME_CTRL_RATIO_SPEED_QUATER;
4445 		AQ_WRITE_REG_BIT(sc, AQ2_LAUNCHTIME_CTRL_REG,
4446 		    AQ2_LAUNCHTIME_CTRL_RATIO, speed);
4447 	}
4448 
4449 	aq_hw_init_tx_path(sc);
4450 	aq_hw_init_rx_path(sc);
4451 
4452 	aq_hw_interrupt_moderation_set(sc);
4453 
4454 	aq_set_mac_addr(sc, AQ_HW_MAC_OWN, sc->sc_enaddr.ether_addr_octet);
4455 	aq_set_linkmode(sc, AQ_LINK_NONE, AQ_FC_NONE, AQ_EEE_DISABLE);
4456 
4457 	aq_hw_qos_set(sc);
4458 
4459 	if (HWTYPE_AQ2_P(sc)) {
4460 		AQ_WRITE_REG_BIT(sc, AQ2_RPF_NEW_CTRL_REG,
4461 		    AQ2_RPF_NEW_CTRL_ENABLE, 1);
4462 	}
4463 
4464 	/* Enable interrupt */
4465 	int irqmode;
4466 	if (sc->sc_msix)
4467 		irqmode =  AQ_INTR_CTRL_IRQMODE_MSIX;
4468 	else
4469 		irqmode =  AQ_INTR_CTRL_IRQMODE_MSI;
4470 
4471 	AQ_WRITE_REG(sc, AQ_INTR_CTRL_REG, AQ_INTR_CTRL_RESET_DIS);
4472 	AQ_WRITE_REG_BIT(sc, AQ_INTR_CTRL_REG, AQ_INTR_CTRL_MULTIVEC,
4473 	    sc->sc_msix ? 1 : 0);
4474 	AQ_WRITE_REG_BIT(sc, AQ_INTR_CTRL_REG, AQ_INTR_CTRL_IRQMODE, irqmode);
4475 
4476 	AQ_WRITE_REG(sc, AQ_INTR_AUTOMASK_REG, 0xffffffff);
4477 
4478 	AQ_WRITE_REG(sc, AQ_GEN_INTR_MAP_REG(0),
4479 	    ((AQ_B0_ERR_INT << 24) | (1 << 31)) |
4480 	    ((AQ_B0_ERR_INT << 16) | (1 << 23))
4481 	);
4482 
4483 	/* link interrupt */
4484 	if (!sc->sc_msix)
4485 		sc->sc_linkstat_irq = AQ_LINKSTAT_IRQ;
4486 	AQ_WRITE_REG(sc, AQ_GEN_INTR_MAP_REG(3),
4487 	    __BIT(7) | sc->sc_linkstat_irq);
4488 
4489 	return 0;
4490 }
4491 
4492 static int
aq_update_link_status(struct aq_softc * sc)4493 aq_update_link_status(struct aq_softc *sc)
4494 {
4495 	struct ifnet * const ifp = &sc->sc_ethercom.ec_if;
4496 	aq_link_speed_t rate = AQ_LINK_NONE;
4497 	aq_link_fc_t fc = AQ_FC_NONE;
4498 	aq_link_eee_t eee = AQ_EEE_DISABLE;
4499 	unsigned int speed;
4500 	int changed = 0;
4501 
4502 	aq_get_linkmode(sc, &rate, &fc, &eee);
4503 
4504 	if (sc->sc_link_rate != rate)
4505 		changed = 1;
4506 	if (sc->sc_link_fc != fc)
4507 		changed = 1;
4508 	if (sc->sc_link_eee != eee)
4509 		changed = 1;
4510 
4511 	if (changed) {
4512 		switch (rate) {
4513 		case AQ_LINK_10M:
4514 			speed = 10;
4515 			break;
4516 		case AQ_LINK_100M:
4517 			speed = 100;
4518 			break;
4519 		case AQ_LINK_1G:
4520 			speed = 1000;
4521 			break;
4522 		case AQ_LINK_2G5:
4523 			speed = 2500;
4524 			break;
4525 		case AQ_LINK_5G:
4526 			speed = 5000;
4527 			break;
4528 		case AQ_LINK_10G:
4529 			speed = 10000;
4530 			break;
4531 		case AQ_LINK_NONE:
4532 		default:
4533 			speed = 0;
4534 			break;
4535 		}
4536 
4537 		if (sc->sc_link_rate == AQ_LINK_NONE && rate != AQ_LINK_NONE) {
4538 			/* link DOWN -> UP */
4539 			device_printf(sc->sc_dev, "link is UP: speed=%u\n",
4540 			    speed);
4541 			if_link_state_change(ifp, LINK_STATE_UP);
4542 		} else if (rate == AQ_LINK_NONE) {
4543 			/* link UP -> DOWN */
4544 			device_printf(sc->sc_dev, "link is DOWN\n");
4545 			if_link_state_change(ifp, LINK_STATE_DOWN);
4546 		} else {
4547 			device_printf(sc->sc_dev,
4548 			    "link mode changed: speed=%u, fc=0x%x, eee=%x\n",
4549 			    speed, fc, eee);
4550 		}
4551 
4552 		sc->sc_link_rate = rate;
4553 		sc->sc_link_fc = fc;
4554 		sc->sc_link_eee = eee;
4555 
4556 		/* update interrupt timing according to new link speed */
4557 		aq_hw_interrupt_moderation_set(sc);
4558 	}
4559 
4560 	return changed;
4561 }
4562 
4563 #ifdef AQ_EVENT_COUNTERS
4564 static void
aq_update_statistics(struct aq_softc * sc)4565 aq_update_statistics(struct aq_softc *sc)
4566 {
4567 	int prev = sc->sc_statistics_idx;
4568 	int cur = prev ^ 1;
4569 
4570 	if (sc->sc_fw_ops->get_stats(sc, &sc->sc_statistics[cur]) != 0)
4571 		return;
4572 
4573 	/*
4574 	 * some aq's internal statistics counters are 32bit.
4575 	 * cauculate delta, and add to evcount
4576 	 */
4577 #define ADD_DELTA(cur, prev, name)				\
4578 	do {							\
4579 		uint32_t n;					\
4580 		n = (uint32_t)(sc->sc_statistics[cur].name -	\
4581 		    sc->sc_statistics[prev].name);		\
4582 		if (n != 0) {					\
4583 			AQ_EVCNT_ADD(sc, name, n);		\
4584 		}						\
4585 	} while (/*CONSTCOND*/0);
4586 
4587 	ADD_DELTA(cur, prev, uprc);
4588 	ADD_DELTA(cur, prev, mprc);
4589 	ADD_DELTA(cur, prev, bprc);
4590 	ADD_DELTA(cur, prev, prc);
4591 	ADD_DELTA(cur, prev, erpr);
4592 	ADD_DELTA(cur, prev, uptc);
4593 	ADD_DELTA(cur, prev, mptc);
4594 	ADD_DELTA(cur, prev, bptc);
4595 	ADD_DELTA(cur, prev, ptc);
4596 	ADD_DELTA(cur, prev, erpt);
4597 	ADD_DELTA(cur, prev, mbtc);
4598 	ADD_DELTA(cur, prev, bbtc);
4599 	ADD_DELTA(cur, prev, mbrc);
4600 	ADD_DELTA(cur, prev, bbrc);
4601 	ADD_DELTA(cur, prev, ubrc);
4602 	ADD_DELTA(cur, prev, ubtc);
4603 	ADD_DELTA(cur, prev, dpc);
4604 	ADD_DELTA(cur, prev, cprc);
4605 
4606 	sc->sc_statistics_idx = cur;
4607 }
4608 #endif /* AQ_EVENT_COUNTERS */
4609 
4610 /* allocate and map one DMA block */
4611 static int
_alloc_dma(struct aq_softc * sc,bus_size_t size,bus_size_t * sizep,void ** addrp,bus_dmamap_t * mapp,bus_dma_segment_t * seg)4612 _alloc_dma(struct aq_softc *sc, bus_size_t size, bus_size_t *sizep,
4613     void **addrp, bus_dmamap_t *mapp, bus_dma_segment_t *seg)
4614 {
4615 	int nsegs, error;
4616 
4617 	if ((error = bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0, seg,
4618 	    1, &nsegs, 0)) != 0) {
4619 		aprint_error_dev(sc->sc_dev,
4620 		    "unable to allocate DMA buffer, error=%d\n", error);
4621 		goto fail_alloc;
4622 	}
4623 
4624 	if ((error = bus_dmamem_map(sc->sc_dmat, seg, 1, size, addrp,
4625 	    BUS_DMA_COHERENT)) != 0) {
4626 		aprint_error_dev(sc->sc_dev,
4627 		    "unable to map DMA buffer, error=%d\n", error);
4628 		goto fail_map;
4629 	}
4630 
4631 	if ((error = bus_dmamap_create(sc->sc_dmat, size, 1, size, 0,
4632 	    0, mapp)) != 0) {
4633 		aprint_error_dev(sc->sc_dev,
4634 		    "unable to create DMA map, error=%d\n", error);
4635 		goto fail_create;
4636 	}
4637 
4638 	if ((error = bus_dmamap_load(sc->sc_dmat, *mapp, *addrp, size, NULL,
4639 	    0)) != 0) {
4640 		aprint_error_dev(sc->sc_dev,
4641 		    "unable to load DMA map, error=%d\n", error);
4642 		goto fail_load;
4643 	}
4644 
4645 	*sizep = size;
4646 	return 0;
4647 
4648  fail_load:
4649 	bus_dmamap_destroy(sc->sc_dmat, *mapp);
4650 	*mapp = NULL;
4651  fail_create:
4652 	bus_dmamem_unmap(sc->sc_dmat, *addrp, size);
4653 	*addrp = NULL;
4654  fail_map:
4655 	bus_dmamem_free(sc->sc_dmat, seg, 1);
4656 	memset(seg, 0, sizeof(*seg));
4657  fail_alloc:
4658 	*sizep = 0;
4659 	return error;
4660 }
4661 
4662 static void
_free_dma(struct aq_softc * sc,bus_size_t * sizep,void ** addrp,bus_dmamap_t * mapp,bus_dma_segment_t * seg)4663 _free_dma(struct aq_softc *sc, bus_size_t *sizep, void **addrp,
4664     bus_dmamap_t *mapp, bus_dma_segment_t *seg)
4665 {
4666 	if (*mapp != NULL) {
4667 		bus_dmamap_destroy(sc->sc_dmat, *mapp);
4668 		*mapp = NULL;
4669 	}
4670 	if (*addrp != NULL) {
4671 		bus_dmamem_unmap(sc->sc_dmat, *addrp, *sizep);
4672 		*addrp = NULL;
4673 	}
4674 	if (*sizep != 0) {
4675 		bus_dmamem_free(sc->sc_dmat, seg, 1);
4676 		memset(seg, 0, sizeof(*seg));
4677 		*sizep = 0;
4678 	}
4679 }
4680 
4681 static int
aq_txring_alloc(struct aq_softc * sc,struct aq_txring * txring)4682 aq_txring_alloc(struct aq_softc *sc, struct aq_txring *txring)
4683 {
4684 	int i, error;
4685 
4686 	/* allocate tx descriptors */
4687 	error = _alloc_dma(sc, sizeof(aq_tx_desc_t) * AQ_TXD_NUM,
4688 	    &txring->txr_txdesc_size, (void **)&txring->txr_txdesc,
4689 	    &txring->txr_txdesc_dmamap, txring->txr_txdesc_seg);
4690 	if (error != 0)
4691 		return error;
4692 
4693 	memset(txring->txr_txdesc, 0, sizeof(aq_tx_desc_t) * AQ_TXD_NUM);
4694 
4695 	/* fill tx ring with dmamap */
4696 	for (i = 0; i < AQ_TXD_NUM; i++) {
4697 #define AQ_MAXDMASIZE	(16 * 1024)
4698 #define AQ_NTXSEGS	32
4699 		/* XXX: TODO: error check */
4700 		bus_dmamap_create(sc->sc_dmat, AQ_MAXDMASIZE, AQ_NTXSEGS,
4701 		    AQ_MAXDMASIZE, 0, 0, &txring->txr_mbufs[i].dmamap);
4702 	}
4703 	return 0;
4704 }
4705 
4706 static void
aq_txring_free(struct aq_softc * sc,struct aq_txring * txring)4707 aq_txring_free(struct aq_softc *sc, struct aq_txring *txring)
4708 {
4709 	int i;
4710 
4711 	_free_dma(sc, &txring->txr_txdesc_size, (void **)&txring->txr_txdesc,
4712 	    &txring->txr_txdesc_dmamap, txring->txr_txdesc_seg);
4713 
4714 	for (i = 0; i < AQ_TXD_NUM; i++) {
4715 		if (txring->txr_mbufs[i].dmamap != NULL) {
4716 			if (txring->txr_mbufs[i].m != NULL) {
4717 				bus_dmamap_unload(sc->sc_dmat,
4718 				    txring->txr_mbufs[i].dmamap);
4719 				m_freem(txring->txr_mbufs[i].m);
4720 				txring->txr_mbufs[i].m = NULL;
4721 			}
4722 			bus_dmamap_destroy(sc->sc_dmat,
4723 			    txring->txr_mbufs[i].dmamap);
4724 			txring->txr_mbufs[i].dmamap = NULL;
4725 		}
4726 	}
4727 }
4728 
4729 static int
aq_rxring_alloc(struct aq_softc * sc,struct aq_rxring * rxring)4730 aq_rxring_alloc(struct aq_softc *sc, struct aq_rxring *rxring)
4731 {
4732 	int i, error;
4733 
4734 	/* allocate rx descriptors */
4735 	error = _alloc_dma(sc, sizeof(aq_rx_desc_t) * AQ_RXD_NUM,
4736 	    &rxring->rxr_rxdesc_size, (void **)&rxring->rxr_rxdesc,
4737 	    &rxring->rxr_rxdesc_dmamap, rxring->rxr_rxdesc_seg);
4738 	if (error != 0)
4739 		return error;
4740 
4741 	memset(rxring->rxr_rxdesc, 0, sizeof(aq_rx_desc_t) * AQ_RXD_NUM);
4742 
4743 	/* fill rxring with dmamaps */
4744 	for (i = 0; i < AQ_RXD_NUM; i++) {
4745 		rxring->rxr_mbufs[i].m = NULL;
4746 		/* XXX: TODO: error check */
4747 		bus_dmamap_create(sc->sc_dmat, MCLBYTES, 1, MCLBYTES, 0, 0,
4748 		    &rxring->rxr_mbufs[i].dmamap);
4749 	}
4750 	return 0;
4751 }
4752 
4753 static void
aq_rxdrain(struct aq_softc * sc,struct aq_rxring * rxring)4754 aq_rxdrain(struct aq_softc *sc, struct aq_rxring *rxring)
4755 {
4756 	int i;
4757 
4758 	/* free all mbufs allocated for RX */
4759 	for (i = 0; i < AQ_RXD_NUM; i++) {
4760 		if (rxring->rxr_mbufs[i].m != NULL) {
4761 			bus_dmamap_unload(sc->sc_dmat,
4762 			    rxring->rxr_mbufs[i].dmamap);
4763 			m_freem(rxring->rxr_mbufs[i].m);
4764 			rxring->rxr_mbufs[i].m = NULL;
4765 		}
4766 	}
4767 }
4768 
4769 static void
aq_rxring_free(struct aq_softc * sc,struct aq_rxring * rxring)4770 aq_rxring_free(struct aq_softc *sc, struct aq_rxring *rxring)
4771 {
4772 	int i;
4773 
4774 	/* free all mbufs and dmamaps */
4775 	aq_rxdrain(sc, rxring);
4776 	for (i = 0; i < AQ_RXD_NUM; i++) {
4777 		if (rxring->rxr_mbufs[i].dmamap != NULL) {
4778 			bus_dmamap_destroy(sc->sc_dmat,
4779 			    rxring->rxr_mbufs[i].dmamap);
4780 			rxring->rxr_mbufs[i].dmamap = NULL;
4781 		}
4782 	}
4783 
4784 	/* free RX descriptor */
4785 	_free_dma(sc, &rxring->rxr_rxdesc_size, (void **)&rxring->rxr_rxdesc,
4786 	    &rxring->rxr_rxdesc_dmamap, rxring->rxr_rxdesc_seg);
4787 }
4788 
4789 static void
aq_rxring_setmbuf(struct aq_softc * sc,struct aq_rxring * rxring,int idx,struct mbuf * m)4790 aq_rxring_setmbuf(struct aq_softc *sc, struct aq_rxring *rxring, int idx,
4791     struct mbuf *m)
4792 {
4793 	int error;
4794 
4795 	/* if mbuf already exists, unload and free */
4796 	if (rxring->rxr_mbufs[idx].m != NULL) {
4797 		bus_dmamap_unload(sc->sc_dmat, rxring->rxr_mbufs[idx].dmamap);
4798 		m_freem(rxring->rxr_mbufs[idx].m);
4799 		rxring->rxr_mbufs[idx].m = NULL;
4800 	}
4801 
4802 	rxring->rxr_mbufs[idx].m = m;
4803 
4804 	m->m_len = m->m_pkthdr.len = m->m_ext.ext_size;
4805 	error = bus_dmamap_load_mbuf(sc->sc_dmat, rxring->rxr_mbufs[idx].dmamap,
4806 	    m, BUS_DMA_READ | BUS_DMA_NOWAIT);
4807 	if (error) {
4808 		device_printf(sc->sc_dev,
4809 		    "unable to load rx DMA map %d, error = %d\n", idx, error);
4810 		panic("%s: unable to load rx DMA map. error=%d",
4811 		    __func__, error);
4812 	}
4813 	bus_dmamap_sync(sc->sc_dmat, rxring->rxr_mbufs[idx].dmamap, 0,
4814 	    rxring->rxr_mbufs[idx].dmamap->dm_mapsize, BUS_DMASYNC_PREREAD);
4815 }
4816 
4817 static inline void
aq_rxring_reset_desc(struct aq_softc * sc,struct aq_rxring * rxring,int idx)4818 aq_rxring_reset_desc(struct aq_softc *sc, struct aq_rxring *rxring, int idx)
4819 {
4820 	/* refill rxdesc, and sync */
4821 	rxring->rxr_rxdesc[idx].read.buf_addr =
4822 	   htole64(rxring->rxr_mbufs[idx].dmamap->dm_segs[0].ds_addr);
4823 	rxring->rxr_rxdesc[idx].read.hdr_addr = 0;
4824 	bus_dmamap_sync(sc->sc_dmat, rxring->rxr_rxdesc_dmamap,
4825 	    sizeof(aq_rx_desc_t) * idx, sizeof(aq_rx_desc_t),
4826 	    BUS_DMASYNC_PREWRITE);
4827 }
4828 
4829 static struct mbuf *
aq_alloc_mbuf(void)4830 aq_alloc_mbuf(void)
4831 {
4832 	struct mbuf *m;
4833 
4834 	MGETHDR(m, M_DONTWAIT, MT_DATA);
4835 	if (m == NULL)
4836 		return NULL;
4837 
4838 	MCLGET(m, M_DONTWAIT);
4839 	if ((m->m_flags & M_EXT) == 0) {
4840 		m_freem(m);
4841 		return NULL;
4842 	}
4843 
4844 	return m;
4845 }
4846 
4847 /* allocate mbuf and unload dmamap */
4848 static int
aq_rxring_add(struct aq_softc * sc,struct aq_rxring * rxring,int idx)4849 aq_rxring_add(struct aq_softc *sc, struct aq_rxring *rxring, int idx)
4850 {
4851 	struct mbuf *m;
4852 
4853 	m = aq_alloc_mbuf();
4854 	if (m == NULL)
4855 		return ENOBUFS;
4856 
4857 	aq_rxring_setmbuf(sc, rxring, idx, m);
4858 	return 0;
4859 }
4860 
4861 static int
aq_txrx_rings_alloc(struct aq_softc * sc)4862 aq_txrx_rings_alloc(struct aq_softc *sc)
4863 {
4864 	int n, error;
4865 
4866 	for (n = 0; n < sc->sc_nqueues; n++) {
4867 		sc->sc_queue[n].sc = sc;
4868 		sc->sc_queue[n].txring.txr_sc = sc;
4869 		sc->sc_queue[n].txring.txr_index = n;
4870 		mutex_init(&sc->sc_queue[n].txring.txr_mutex, MUTEX_DEFAULT,
4871 		    IPL_NET);
4872 		error = aq_txring_alloc(sc, &sc->sc_queue[n].txring);
4873 		if (error != 0)
4874 			goto failure;
4875 
4876 		error = aq_tx_pcq_alloc(sc, &sc->sc_queue[n].txring);
4877 		if (error != 0)
4878 			goto failure;
4879 
4880 		sc->sc_queue[n].rxring.rxr_sc = sc;
4881 		sc->sc_queue[n].rxring.rxr_index = n;
4882 		mutex_init(&sc->sc_queue[n].rxring.rxr_mutex, MUTEX_DEFAULT,
4883 		   IPL_NET);
4884 		error = aq_rxring_alloc(sc, &sc->sc_queue[n].rxring);
4885 		if (error != 0)
4886 			break;
4887 	}
4888 
4889  failure:
4890 	return error;
4891 }
4892 
4893 static void
aq_txrx_rings_free(struct aq_softc * sc)4894 aq_txrx_rings_free(struct aq_softc *sc)
4895 {
4896 	int n;
4897 
4898 	for (n = 0; n < sc->sc_nqueues; n++) {
4899 		aq_txring_free(sc, &sc->sc_queue[n].txring);
4900 		mutex_destroy(&sc->sc_queue[n].txring.txr_mutex);
4901 
4902 		aq_tx_pcq_free(sc, &sc->sc_queue[n].txring);
4903 
4904 		aq_rxring_free(sc, &sc->sc_queue[n].rxring);
4905 		mutex_destroy(&sc->sc_queue[n].rxring.rxr_mutex);
4906 	}
4907 }
4908 
4909 static int
aq_tx_pcq_alloc(struct aq_softc * sc,struct aq_txring * txring)4910 aq_tx_pcq_alloc(struct aq_softc *sc, struct aq_txring *txring)
4911 {
4912 	int error = 0;
4913 	txring->txr_softint = NULL;
4914 
4915 	txring->txr_pcq = pcq_create(AQ_TXD_NUM, KM_NOSLEEP);
4916 	if (txring->txr_pcq == NULL) {
4917 		aprint_error_dev(sc->sc_dev,
4918 		    "unable to allocate pcq for TXring[%d]\n",
4919 		    txring->txr_index);
4920 		error = ENOMEM;
4921 		goto done;
4922 	}
4923 
4924 	txring->txr_softint = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE,
4925 	    aq_deferred_transmit, txring);
4926 	if (txring->txr_softint == NULL) {
4927 		aprint_error_dev(sc->sc_dev,
4928 		    "unable to establish softint for TXring[%d]\n",
4929 		    txring->txr_index);
4930 		error = ENOENT;
4931 	}
4932 
4933  done:
4934 	return error;
4935 }
4936 
4937 static void
aq_tx_pcq_free(struct aq_softc * sc,struct aq_txring * txring)4938 aq_tx_pcq_free(struct aq_softc *sc, struct aq_txring *txring)
4939 {
4940 	struct mbuf *m;
4941 
4942 	if (txring->txr_softint != NULL) {
4943 		softint_disestablish(txring->txr_softint);
4944 		txring->txr_softint = NULL;
4945 	}
4946 
4947 	if (txring->txr_pcq != NULL) {
4948 		while ((m = pcq_get(txring->txr_pcq)) != NULL)
4949 			m_freem(m);
4950 		pcq_destroy(txring->txr_pcq);
4951 		txring->txr_pcq = NULL;
4952 	}
4953 }
4954 
4955 #if NSYSMON_ENVSYS > 0
4956 static void
aq_temp_refresh(struct sysmon_envsys * sme,envsys_data_t * edata)4957 aq_temp_refresh(struct sysmon_envsys *sme, envsys_data_t *edata)
4958 {
4959 	struct aq_softc *sc;
4960 	uint32_t temp;
4961 	int error;
4962 
4963 	sc = sme->sme_cookie;
4964 
4965 	error = sc->sc_fw_ops->get_temperature(sc, &temp);
4966 	if (error == 0) {
4967 		edata->value_cur = temp;
4968 		edata->state = ENVSYS_SVALID;
4969 	} else {
4970 		edata->state = ENVSYS_SINVALID;
4971 	}
4972 }
4973 #endif
4974 
4975 
4976 
4977 static bool
aq_watchdog_check(struct aq_softc * const sc)4978 aq_watchdog_check(struct aq_softc * const sc)
4979 {
4980 
4981 	AQ_LOCKED(sc);
4982 
4983 	bool ok = true;
4984 	for (int n = 0; n < sc->sc_nqueues; n++) {
4985 		struct aq_txring *txring = &sc->sc_queue[n].txring;
4986 
4987 		mutex_enter(&txring->txr_mutex);
4988 		if (txring->txr_sending &&
4989 		    time_uptime - txring->txr_lastsent > aq_watchdog_timeout)
4990 			ok = false;
4991 
4992 		mutex_exit(&txring->txr_mutex);
4993 
4994 		if (!ok)
4995 			return false;
4996 	}
4997 
4998 	if (sc->sc_trigger_reset) {
4999 		/* debug operation, no need for atomicity or reliability */
5000 		sc->sc_trigger_reset = 0;
5001 		return false;
5002 	}
5003 
5004 	return true;
5005 }
5006 
5007 
5008 
5009 static bool
aq_watchdog_tick(struct ifnet * ifp)5010 aq_watchdog_tick(struct ifnet *ifp)
5011 {
5012 	struct aq_softc * const sc = ifp->if_softc;
5013 
5014 	AQ_LOCKED(sc);
5015 
5016 	if (!sc->sc_trigger_reset && aq_watchdog_check(sc))
5017 		return true;
5018 
5019 	if (atomic_swap_uint(&sc->sc_reset_pending, 1) == 0) {
5020 		workqueue_enqueue(sc->sc_reset_wq, &sc->sc_reset_work, NULL);
5021 	}
5022 
5023 	return false;
5024 }
5025 
5026 static void
aq_tick(void * arg)5027 aq_tick(void *arg)
5028 {
5029 	struct aq_softc * const sc = arg;
5030 
5031 	AQ_LOCK(sc);
5032 	if (sc->sc_stopping) {
5033 		AQ_UNLOCK(sc);
5034 		return;
5035 	}
5036 
5037 	aq_update_link_status(sc);
5038 
5039 #ifdef AQ_EVENT_COUNTERS
5040 	if (sc->sc_poll_statistics)
5041 		aq_update_statistics(sc);
5042 #endif
5043 
5044 	struct ifnet * const ifp = &sc->sc_ethercom.ec_if;
5045 	const bool ok = aq_watchdog_tick(ifp);
5046 	if (ok)
5047 		callout_schedule(&sc->sc_tick_ch, hz);
5048 
5049 	AQ_UNLOCK(sc);
5050 }
5051 
5052 /* interrupt enable/disable */
5053 static void
aq_enable_intr(struct aq_softc * sc,bool link,bool txrx)5054 aq_enable_intr(struct aq_softc *sc, bool link, bool txrx)
5055 {
5056 	uint32_t imask = 0;
5057 	int i;
5058 
5059 	if (txrx) {
5060 		for (i = 0; i < sc->sc_nqueues; i++) {
5061 			imask |= __BIT(sc->sc_tx_irq[i]);
5062 			imask |= __BIT(sc->sc_rx_irq[i]);
5063 		}
5064 	}
5065 
5066 	if (link)
5067 		imask |= __BIT(sc->sc_linkstat_irq);
5068 
5069 	AQ_WRITE_REG(sc, AQ_INTR_MASK_REG, imask);
5070 	AQ_WRITE_REG(sc, AQ_INTR_STATUS_CLR_REG, 0xffffffff);
5071 }
5072 
5073 static int
aq_legacy_intr(void * arg)5074 aq_legacy_intr(void *arg)
5075 {
5076 	struct aq_softc *sc = arg;
5077 	uint32_t status;
5078 	int nintr = 0;
5079 
5080 	status = AQ_READ_REG(sc, AQ_INTR_STATUS_REG);
5081 	AQ_WRITE_REG(sc, AQ_INTR_STATUS_CLR_REG, 0xffffffff);
5082 
5083 	if (status & __BIT(sc->sc_linkstat_irq)) {
5084 		AQ_LOCK(sc);
5085 		if (!sc->sc_stopping)
5086 			callout_schedule(&sc->sc_tick_ch, 0);
5087 		AQ_UNLOCK(sc);
5088 		nintr++;
5089 	}
5090 
5091 	if (status & __BIT(sc->sc_rx_irq[0])) {
5092 		nintr += aq_rx_intr(&sc->sc_queue[0].rxring);
5093 	}
5094 
5095 	if (status & __BIT(sc->sc_tx_irq[0])) {
5096 		nintr += aq_tx_intr(&sc->sc_queue[0].txring);
5097 	}
5098 
5099 	return nintr;
5100 }
5101 
5102 static int
aq_txrx_intr(void * arg)5103 aq_txrx_intr(void *arg)
5104 {
5105 	struct aq_queue *queue = arg;
5106 	struct aq_softc *sc = queue->sc;
5107 	struct aq_txring *txring = &queue->txring;
5108 	struct aq_rxring *rxring = &queue->rxring;
5109 	uint32_t status;
5110 	int nintr = 0;
5111 	int txringidx, rxringidx, txirq, rxirq;
5112 
5113 	txringidx = txring->txr_index;
5114 	rxringidx = rxring->rxr_index;
5115 	txirq = sc->sc_tx_irq[txringidx];
5116 	rxirq = sc->sc_rx_irq[rxringidx];
5117 
5118 	status = AQ_READ_REG(sc, AQ_INTR_STATUS_REG);
5119 	if ((status & (__BIT(txirq) | __BIT(rxirq))) == 0) {
5120 		/* stray interrupt? */
5121 		return 0;
5122 	}
5123 
5124 	nintr += aq_rx_intr(rxring);
5125 	nintr += aq_tx_intr(txring);
5126 
5127 	return nintr;
5128 }
5129 
5130 static int
aq_link_intr(void * arg)5131 aq_link_intr(void *arg)
5132 {
5133 	struct aq_softc * const sc = arg;
5134 	uint32_t status;
5135 	int nintr = 0;
5136 
5137 	status = AQ_READ_REG(sc, AQ_INTR_STATUS_REG);
5138 	if (status & __BIT(sc->sc_linkstat_irq)) {
5139 		AQ_LOCK(sc);
5140 		if (!sc->sc_stopping)
5141 			callout_schedule(&sc->sc_tick_ch, 0);
5142 		AQ_UNLOCK(sc);
5143 		AQ_WRITE_REG(sc, AQ_INTR_STATUS_CLR_REG,
5144 		    __BIT(sc->sc_linkstat_irq));
5145 		nintr++;
5146 	}
5147 
5148 	return nintr;
5149 }
5150 
5151 static void
aq_txring_reset(struct aq_softc * sc,struct aq_txring * txring,bool start)5152 aq_txring_reset(struct aq_softc *sc, struct aq_txring *txring, bool start)
5153 {
5154 	const int ringidx = txring->txr_index;
5155 	int i;
5156 
5157 	mutex_enter(&txring->txr_mutex);
5158 
5159 	txring->txr_prodidx = 0;
5160 	txring->txr_considx = 0;
5161 	txring->txr_nfree = AQ_TXD_NUM;
5162 	txring->txr_active = false;
5163 
5164 	/* free mbufs untransmitted */
5165 	for (i = 0; i < AQ_TXD_NUM; i++) {
5166 		m_freem(txring->txr_mbufs[i].m);
5167 		txring->txr_mbufs[i].m = NULL;
5168 	}
5169 
5170 	/* disable DMA */
5171 	AQ_WRITE_REG_BIT(sc, TX_DMA_DESC_REG(ringidx), TX_DMA_DESC_EN, 0);
5172 
5173 	if (start) {
5174 		/* TX descriptor physical address */
5175 		paddr_t paddr = txring->txr_txdesc_dmamap->dm_segs[0].ds_addr;
5176 		AQ_WRITE_REG(sc, TX_DMA_DESC_BASE_ADDRLSW_REG(ringidx), paddr);
5177 		AQ_WRITE_REG(sc, TX_DMA_DESC_BASE_ADDRMSW_REG(ringidx),
5178 		    (uint32_t)((uint64_t)paddr >> 32));
5179 
5180 		/* TX descriptor size */
5181 		AQ_WRITE_REG_BIT(sc, TX_DMA_DESC_REG(ringidx), TX_DMA_DESC_LEN,
5182 		    AQ_TXD_NUM / 8);
5183 
5184 		/* reload TAIL pointer */
5185 		txring->txr_prodidx = txring->txr_considx =
5186 		    AQ_READ_REG(sc, TX_DMA_DESC_TAIL_PTR_REG(ringidx));
5187 		AQ_WRITE_REG(sc, TX_DMA_DESC_WRWB_THRESH_REG(ringidx), 0);
5188 
5189 		/* Mapping interrupt vector */
5190 		AQ_WRITE_REG_BIT(sc, AQ_INTR_IRQ_MAP_TX_REG(ringidx),
5191 		    AQ_INTR_IRQ_MAP_TX_IRQMAP(ringidx), sc->sc_tx_irq[ringidx]);
5192 		AQ_WRITE_REG_BIT(sc, AQ_INTR_IRQ_MAP_TX_REG(ringidx),
5193 		    AQ_INTR_IRQ_MAP_TX_EN(ringidx), true);
5194 
5195 		/* enable DMA */
5196 		AQ_WRITE_REG_BIT(sc, TX_DMA_DESC_REG(ringidx),
5197 		    TX_DMA_DESC_EN, 1);
5198 
5199 		const int cpuid = 0;	/* XXX? */
5200 		AQ_WRITE_REG_BIT(sc, TDM_DCAD_REG(ringidx),
5201 		    TDM_DCAD_CPUID, cpuid);
5202 		AQ_WRITE_REG_BIT(sc, TDM_DCAD_REG(ringidx),
5203 		    TDM_DCAD_CPUID_EN, 0);
5204 
5205 		txring->txr_active = true;
5206 	}
5207 
5208 	mutex_exit(&txring->txr_mutex);
5209 }
5210 
5211 static int
aq_rxring_reset(struct aq_softc * sc,struct aq_rxring * rxring,bool start)5212 aq_rxring_reset(struct aq_softc *sc, struct aq_rxring *rxring, bool start)
5213 {
5214 	const int ringidx = rxring->rxr_index;
5215 	int i;
5216 	int error = 0;
5217 
5218 	mutex_enter(&rxring->rxr_mutex);
5219 	rxring->rxr_active = false;
5220 	rxring->rxr_discarding = false;
5221 	if (rxring->rxr_receiving_m != NULL) {
5222 		m_freem(rxring->rxr_receiving_m);
5223 		rxring->rxr_receiving_m = NULL;
5224 		rxring->rxr_receiving_m_last = NULL;
5225 	}
5226 
5227 	/* disable DMA */
5228 	AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_REG(ringidx), RX_DMA_DESC_EN, 0);
5229 
5230 	/* free all RX mbufs */
5231 	aq_rxdrain(sc, rxring);
5232 
5233 	if (start) {
5234 		for (i = 0; i < AQ_RXD_NUM; i++) {
5235 			error = aq_rxring_add(sc, rxring, i);
5236 			if (error != 0) {
5237 				aq_rxdrain(sc, rxring);
5238 				return error;
5239 			}
5240 			aq_rxring_reset_desc(sc, rxring, i);
5241 		}
5242 
5243 		/* RX descriptor physical address */
5244 		paddr_t paddr = rxring->rxr_rxdesc_dmamap->dm_segs[0].ds_addr;
5245 		AQ_WRITE_REG(sc, RX_DMA_DESC_BASE_ADDRLSW_REG(ringidx), paddr);
5246 		AQ_WRITE_REG(sc, RX_DMA_DESC_BASE_ADDRMSW_REG(ringidx),
5247 		    (uint32_t)((uint64_t)paddr >> 32));
5248 
5249 		/* RX descriptor size */
5250 		AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_REG(ringidx), RX_DMA_DESC_LEN,
5251 		    AQ_RXD_NUM / 8);
5252 
5253 		/* maximum receive frame size */
5254 		AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_BUFSIZE_REG(ringidx),
5255 		    RX_DMA_DESC_BUFSIZE_DATA, MCLBYTES / 1024);
5256 		AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_BUFSIZE_REG(ringidx),
5257 		    RX_DMA_DESC_BUFSIZE_HDR, 0 / 1024);
5258 
5259 		AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_REG(ringidx),
5260 		    RX_DMA_DESC_HEADER_SPLIT, 0);
5261 		AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_REG(ringidx),
5262 		    RX_DMA_DESC_VLAN_STRIP,
5263 		    (sc->sc_ethercom.ec_capenable & ETHERCAP_VLAN_HWTAGGING) ?
5264 		    1 : 0);
5265 
5266 		/*
5267 		 * reload TAIL pointer, and update readidx
5268 		 * (HEAD pointer cannot write)
5269 		 */
5270 		rxring->rxr_readidx = AQ_READ_REG_BIT(sc,
5271 		    RX_DMA_DESC_HEAD_PTR_REG(ringidx), RX_DMA_DESC_HEAD_PTR);
5272 		AQ_WRITE_REG(sc, RX_DMA_DESC_TAIL_PTR_REG(ringidx),
5273 		    (rxring->rxr_readidx + AQ_RXD_NUM - 1) % AQ_RXD_NUM);
5274 
5275 		/* Rx ring set mode */
5276 
5277 		/* Mapping interrupt vector */
5278 		AQ_WRITE_REG_BIT(sc, AQ_INTR_IRQ_MAP_RX_REG(ringidx),
5279 		    AQ_INTR_IRQ_MAP_RX_IRQMAP(ringidx), sc->sc_rx_irq[ringidx]);
5280 		AQ_WRITE_REG_BIT(sc, AQ_INTR_IRQ_MAP_RX_REG(ringidx),
5281 		    AQ_INTR_IRQ_MAP_RX_EN(ringidx), 1);
5282 
5283 		const int cpuid = 0;	/* XXX? */
5284 		AQ_WRITE_REG_BIT(sc, RX_DMA_DCAD_REG(ringidx),
5285 		    RX_DMA_DCAD_CPUID, cpuid);
5286 		AQ_WRITE_REG_BIT(sc, RX_DMA_DCAD_REG(ringidx),
5287 		    RX_DMA_DCAD_DESC_EN, 0);
5288 		AQ_WRITE_REG_BIT(sc, RX_DMA_DCAD_REG(ringidx),
5289 		    RX_DMA_DCAD_HEADER_EN, 0);
5290 		AQ_WRITE_REG_BIT(sc, RX_DMA_DCAD_REG(ringidx),
5291 		    RX_DMA_DCAD_PAYLOAD_EN, 0);
5292 
5293 		/* enable DMA. start receiving */
5294 		AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_REG(ringidx),
5295 		    RX_DMA_DESC_EN, 1);
5296 
5297 		rxring->rxr_active = true;
5298 	}
5299 
5300 	mutex_exit(&rxring->rxr_mutex);
5301 	return error;
5302 }
5303 
5304 #define TXRING_NEXTIDX(idx)	\
5305 	(((idx) >= (AQ_TXD_NUM - 1)) ? 0 : ((idx) + 1))
5306 #define RXRING_NEXTIDX(idx)	\
5307 	(((idx) >= (AQ_RXD_NUM - 1)) ? 0 : ((idx) + 1))
5308 
5309 static int
aq_encap_txring(struct aq_softc * sc,struct aq_txring * txring,struct mbuf * m)5310 aq_encap_txring(struct aq_softc *sc, struct aq_txring *txring, struct mbuf *m)
5311 {
5312 	bus_dmamap_t map;
5313 	uint32_t ctl1, ctl1_ctx, ctl2;
5314 	int idx, i, error;
5315 
5316 	idx = txring->txr_prodidx;
5317 	map = txring->txr_mbufs[idx].dmamap;
5318 
5319 	error = bus_dmamap_load_mbuf(sc->sc_dmat, map, m,
5320 	    BUS_DMA_WRITE | BUS_DMA_NOWAIT);
5321 	if (error == EFBIG) {
5322 		struct mbuf *n;
5323 		n = m_defrag(m, M_DONTWAIT);
5324 		if (n == NULL)
5325 			return EFBIG;
5326 		/* m_defrag() preserve m */
5327 		KASSERT(n == m);
5328 		error = bus_dmamap_load_mbuf(sc->sc_dmat, map, m,
5329 		    BUS_DMA_WRITE | BUS_DMA_NOWAIT);
5330 	}
5331 	if (error != 0)
5332 		return error;
5333 
5334 	/*
5335 	 * check spaces of free descriptors.
5336 	 * +1 is additional descriptor for context (vlan, etc,.)
5337 	 */
5338 	if ((map->dm_nsegs + 1) > txring->txr_nfree) {
5339 		bus_dmamap_unload(sc->sc_dmat, map);
5340 		return EAGAIN;
5341 	}
5342 
5343 	/* sync dma for mbuf */
5344 	bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize,
5345 	    BUS_DMASYNC_PREWRITE);
5346 
5347 	ctl1_ctx = 0;
5348 	ctl2 = __SHIFTIN(m->m_pkthdr.len, AQ_TXDESC_CTL2_LEN);
5349 
5350 	if (vlan_has_tag(m)) {
5351 		ctl1 = AQ_TXDESC_CTL1_TYPE_TXC;
5352 		ctl1 |= __SHIFTIN(vlan_get_tag(m), AQ_TXDESC_CTL1_VID);
5353 
5354 		ctl1_ctx |= AQ_TXDESC_CTL1_CMD_VLAN;
5355 		ctl2 |= AQ_TXDESC_CTL2_CTX_EN;
5356 
5357 		/* fill context descriptor and forward index */
5358 		txring->txr_txdesc[idx].buf_addr = 0;
5359 		txring->txr_txdesc[idx].ctl1 = htole32(ctl1);
5360 		txring->txr_txdesc[idx].ctl2 = 0;
5361 
5362 		idx = TXRING_NEXTIDX(idx);
5363 		txring->txr_nfree--;
5364 	}
5365 
5366 	if (m->m_pkthdr.csum_flags & M_CSUM_IPv4)
5367 		ctl1_ctx |= AQ_TXDESC_CTL1_CMD_IP4CSUM;
5368 	if (m->m_pkthdr.csum_flags &
5369 	    (M_CSUM_TCPv4 | M_CSUM_UDPv4 | M_CSUM_TCPv6 | M_CSUM_UDPv6)) {
5370 		ctl1_ctx |= AQ_TXDESC_CTL1_CMD_L4CSUM;
5371 	}
5372 
5373 	/* fill descriptor(s) */
5374 	for (i = 0; i < map->dm_nsegs; i++) {
5375 		ctl1 = ctl1_ctx | AQ_TXDESC_CTL1_TYPE_TXD |
5376 		    __SHIFTIN(map->dm_segs[i].ds_len, AQ_TXDESC_CTL1_BLEN);
5377 		ctl1 |= AQ_TXDESC_CTL1_CMD_FCS;
5378 
5379 		if (i == 0) {
5380 			/* remember mbuf of these descriptors */
5381 			txring->txr_mbufs[idx].m = m;
5382 		} else {
5383 			txring->txr_mbufs[idx].m = NULL;
5384 		}
5385 
5386 		if (i == map->dm_nsegs - 1) {
5387 			/* last segment, mark an EndOfPacket, and cause intr */
5388 			ctl1 |= AQ_TXDESC_CTL1_EOP | AQ_TXDESC_CTL1_CMD_WB;
5389 		}
5390 
5391 		txring->txr_txdesc[idx].buf_addr =
5392 		    htole64(map->dm_segs[i].ds_addr);
5393 		txring->txr_txdesc[idx].ctl1 = htole32(ctl1);
5394 		txring->txr_txdesc[idx].ctl2 = htole32(ctl2);
5395 
5396 		bus_dmamap_sync(sc->sc_dmat, txring->txr_txdesc_dmamap,
5397 		    sizeof(aq_tx_desc_t) * idx, sizeof(aq_tx_desc_t),
5398 		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
5399 
5400 		idx = TXRING_NEXTIDX(idx);
5401 		txring->txr_nfree--;
5402 	}
5403 
5404 	txring->txr_prodidx = idx;
5405 
5406 	return 0;
5407 }
5408 
5409 static int
aq_tx_intr(void * arg)5410 aq_tx_intr(void *arg)
5411 {
5412 	struct aq_txring * const txring = arg;
5413 	struct aq_softc * const sc = txring->txr_sc;
5414 	struct ifnet * const ifp = &sc->sc_ethercom.ec_if;
5415 	struct mbuf *m;
5416 	const int ringidx = txring->txr_index;
5417 	unsigned int idx, hw_head, n = 0;
5418 
5419 	mutex_enter(&txring->txr_mutex);
5420 
5421 	if (!txring->txr_active)
5422 		goto tx_intr_done;
5423 
5424 	hw_head = AQ_READ_REG_BIT(sc, TX_DMA_DESC_HEAD_PTR_REG(ringidx),
5425 	    TX_DMA_DESC_HEAD_PTR);
5426 	if (hw_head == txring->txr_considx) {
5427 		txring->txr_sending = false;
5428 		goto tx_intr_done;
5429 	}
5430 
5431 	net_stat_ref_t nsr = IF_STAT_GETREF(ifp);
5432 
5433 	for (idx = txring->txr_considx; idx != hw_head;
5434 	    idx = TXRING_NEXTIDX(idx), n++) {
5435 
5436 		if ((m = txring->txr_mbufs[idx].m) != NULL) {
5437 			bus_dmamap_unload(sc->sc_dmat,
5438 			    txring->txr_mbufs[idx].dmamap);
5439 
5440 			if_statinc_ref(ifp, nsr, if_opackets);
5441 			if_statadd_ref(ifp, nsr, if_obytes, m->m_pkthdr.len);
5442 			if (m->m_flags & M_MCAST)
5443 				if_statinc_ref(ifp, nsr, if_omcasts);
5444 
5445 			m_freem(m);
5446 			txring->txr_mbufs[idx].m = NULL;
5447 		}
5448 
5449 		txring->txr_nfree++;
5450 	}
5451 	txring->txr_considx = idx;
5452 
5453 	IF_STAT_PUTREF(ifp);
5454 
5455 	/* no more pending TX packet, cancel watchdog */
5456 	if (txring->txr_nfree >= AQ_TXD_NUM)
5457 		txring->txr_sending = false;
5458 
5459  tx_intr_done:
5460 	mutex_exit(&txring->txr_mutex);
5461 
5462 	AQ_WRITE_REG(sc, AQ_INTR_STATUS_CLR_REG, __BIT(sc->sc_tx_irq[ringidx]));
5463 	return n;
5464 }
5465 
5466 static int
aq_rx_intr(void * arg)5467 aq_rx_intr(void *arg)
5468 {
5469 	struct aq_rxring * const rxring = arg;
5470 	struct aq_softc * const sc = rxring->rxr_sc;
5471 	struct ifnet * const ifp = &sc->sc_ethercom.ec_if;
5472 	const int ringidx = rxring->rxr_index;
5473 	aq_rx_desc_t *rxd;
5474 	struct mbuf *m, *m0, *mprev, *new_m;
5475 	uint32_t rxd_type, rxd_hash __unused;
5476 	uint16_t rxd_status, rxd_pktlen;
5477 	uint16_t rxd_nextdescptr __unused, rxd_vlan __unused;
5478 	unsigned int idx, n = 0;
5479 	bool discarding;
5480 
5481 	mutex_enter(&rxring->rxr_mutex);
5482 
5483 	if (!rxring->rxr_active)
5484 		goto rx_intr_done;
5485 
5486 	if (rxring->rxr_readidx == AQ_READ_REG_BIT(sc,
5487 	    RX_DMA_DESC_HEAD_PTR_REG(ringidx), RX_DMA_DESC_HEAD_PTR)) {
5488 		goto rx_intr_done;
5489 	}
5490 
5491 	net_stat_ref_t nsr = IF_STAT_GETREF(ifp);
5492 
5493 	/* restore ring context */
5494 	discarding = rxring->rxr_discarding;
5495 	m0 = rxring->rxr_receiving_m;
5496 	mprev = rxring->rxr_receiving_m_last;
5497 
5498 	for (idx = rxring->rxr_readidx;
5499 	    idx != AQ_READ_REG_BIT(sc, RX_DMA_DESC_HEAD_PTR_REG(ringidx),
5500 	    RX_DMA_DESC_HEAD_PTR); idx = RXRING_NEXTIDX(idx), n++) {
5501 
5502 		bus_dmamap_sync(sc->sc_dmat, rxring->rxr_rxdesc_dmamap,
5503 		    sizeof(aq_rx_desc_t) * idx, sizeof(aq_rx_desc_t),
5504 		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
5505 
5506 		rxd = &rxring->rxr_rxdesc[idx];
5507 		rxd_status = le16toh(rxd->wb.status);
5508 
5509 		if ((rxd_status & RXDESC_STATUS_DD) == 0)
5510 			break;	/* not yet done */
5511 
5512 		rxd_type = le32toh(rxd->wb.type);
5513 		rxd_pktlen = le16toh(rxd->wb.pkt_len);
5514 		rxd_nextdescptr = le16toh(rxd->wb.next_desc_ptr);
5515 		rxd_hash = le32toh(rxd->wb.rss_hash);
5516 		rxd_vlan = le16toh(rxd->wb.vlan);
5517 
5518 		/*
5519 		 * Some segments are being dropped while receiving jumboframe.
5520 		 * Discard until EOP.
5521 		 */
5522 		if (discarding)
5523 			goto rx_next;
5524 
5525 		if ((rxd_status & RXDESC_STATUS_MACERR) ||
5526 		    (rxd_type & RXDESC_TYPE_MAC_DMA_ERR)) {
5527 			if_statinc_ref(ifp, nsr, if_ierrors);
5528 			if (m0 != NULL) {
5529 				m_freem(m0);
5530 				m0 = mprev = NULL;
5531 			}
5532 			discarding = true;
5533 			goto rx_next;
5534 		}
5535 
5536 		bus_dmamap_sync(sc->sc_dmat, rxring->rxr_mbufs[idx].dmamap, 0,
5537 		    rxring->rxr_mbufs[idx].dmamap->dm_mapsize,
5538 		    BUS_DMASYNC_POSTREAD);
5539 		m = rxring->rxr_mbufs[idx].m;
5540 
5541 		new_m = aq_alloc_mbuf();
5542 		if (new_m == NULL) {
5543 			/*
5544 			 * cannot allocate new mbuf.
5545 			 * discard this packet, and reuse mbuf for next.
5546 			 */
5547 			if_statinc_ref(ifp, nsr, if_iqdrops);
5548 			if (m0 != NULL) {
5549 				m_freem(m0);
5550 				m0 = mprev = NULL;
5551 			}
5552 			discarding = true;
5553 			goto rx_next;
5554 		}
5555 		rxring->rxr_mbufs[idx].m = NULL;
5556 		aq_rxring_setmbuf(sc, rxring, idx, new_m);
5557 
5558 		if (m0 == NULL) {
5559 			m0 = m;
5560 		} else {
5561 			if (m->m_flags & M_PKTHDR)
5562 				m_remove_pkthdr(m);
5563 			mprev->m_next = m;
5564 		}
5565 		mprev = m;
5566 
5567 		if ((rxd_status & RXDESC_STATUS_EOP) == 0) {
5568 			/* to be continued in the next segment */
5569 			m->m_len = MCLBYTES;
5570 		} else {
5571 			/* the last segment */
5572 			int mlen = rxd_pktlen % MCLBYTES;
5573 			if (mlen == 0)
5574 				mlen = MCLBYTES;
5575 			m->m_len = mlen;
5576 			m0->m_pkthdr.len = rxd_pktlen;
5577 			/* VLAN offloading */
5578 			if ((sc->sc_ethercom.ec_capenable &
5579 			    ETHERCAP_VLAN_HWTAGGING) &&
5580 			    (__SHIFTOUT(rxd_type, RXDESC_TYPE_PKTTYPE_VLAN) ||
5581 			    __SHIFTOUT(rxd_type,
5582 			    RXDESC_TYPE_PKTTYPE_VLAN_DOUBLE))) {
5583 				vlan_set_tag(m0, rxd_vlan);
5584 			}
5585 
5586 			/* Checksum offloading */
5587 			unsigned int pkttype_eth =
5588 			    __SHIFTOUT(rxd_type, RXDESC_TYPE_PKTTYPE_ETHER);
5589 			if ((ifp->if_capabilities & IFCAP_CSUM_IPv4_Rx) &&
5590 			    (pkttype_eth == RXDESC_TYPE_PKTTYPE_ETHER_IPV4) &&
5591 			    __SHIFTOUT(rxd_type,
5592 			    RXDESC_TYPE_IPV4_CSUM_CHECKED)) {
5593 				m0->m_pkthdr.csum_flags |= M_CSUM_IPv4;
5594 				if (__SHIFTOUT(rxd_status,
5595 				    RXDESC_STATUS_IPV4_CSUM_NG))
5596 					m0->m_pkthdr.csum_flags |=
5597 					    M_CSUM_IPv4_BAD;
5598 			}
5599 
5600 			/*
5601 			 * aq will always mark BAD for fragment packets,
5602 			 * but this is not a problem because the IP stack
5603 			 * ignores the CSUM flag in fragment packets.
5604 			 */
5605 			if (__SHIFTOUT(rxd_type,
5606 			    RXDESC_TYPE_TCPUDP_CSUM_CHECKED)) {
5607 				bool checked = false;
5608 				unsigned int pkttype_proto =
5609 				    __SHIFTOUT(rxd_type,
5610 				    RXDESC_TYPE_PKTTYPE_PROTO);
5611 
5612 				if (pkttype_proto ==
5613 				    RXDESC_TYPE_PKTTYPE_PROTO_TCP) {
5614 					if ((pkttype_eth ==
5615 					    RXDESC_TYPE_PKTTYPE_ETHER_IPV4) &&
5616 					    (ifp->if_capabilities &
5617 					    IFCAP_CSUM_TCPv4_Rx)) {
5618 						m0->m_pkthdr.csum_flags |=
5619 						    M_CSUM_TCPv4;
5620 						checked = true;
5621 					} else if ((pkttype_eth ==
5622 					    RXDESC_TYPE_PKTTYPE_ETHER_IPV6) &&
5623 					    (ifp->if_capabilities &
5624 					    IFCAP_CSUM_TCPv6_Rx)) {
5625 						m0->m_pkthdr.csum_flags |=
5626 						    M_CSUM_TCPv6;
5627 						checked = true;
5628 					}
5629 				} else if (pkttype_proto ==
5630 				    RXDESC_TYPE_PKTTYPE_PROTO_UDP) {
5631 					if ((pkttype_eth ==
5632 					    RXDESC_TYPE_PKTTYPE_ETHER_IPV4) &&
5633 					    (ifp->if_capabilities &
5634 					    IFCAP_CSUM_UDPv4_Rx)) {
5635 						m0->m_pkthdr.csum_flags |=
5636 						    M_CSUM_UDPv4;
5637 						checked = true;
5638 					} else if ((pkttype_eth ==
5639 					    RXDESC_TYPE_PKTTYPE_ETHER_IPV6) &&
5640 					    (ifp->if_capabilities &
5641 					    IFCAP_CSUM_UDPv6_Rx)) {
5642 						m0->m_pkthdr.csum_flags |=
5643 						    M_CSUM_UDPv6;
5644 						checked = true;
5645 					}
5646 				}
5647 				if (checked &&
5648 				    (__SHIFTOUT(rxd_status,
5649 				    RXDESC_STATUS_TCPUDP_CSUM_ERROR) ||
5650 				    !__SHIFTOUT(rxd_status,
5651 				    RXDESC_STATUS_TCPUDP_CSUM_OK))) {
5652 					m0->m_pkthdr.csum_flags |=
5653 					    M_CSUM_TCP_UDP_BAD;
5654 				}
5655 			}
5656 
5657 			m_set_rcvif(m0, ifp);
5658 			if_statinc_ref(ifp, nsr, if_ipackets);
5659 			if_statadd_ref(ifp, nsr, if_ibytes, m0->m_pkthdr.len);
5660 			if_percpuq_enqueue(ifp->if_percpuq, m0);
5661 			m0 = mprev = NULL;
5662 		}
5663 
5664  rx_next:
5665 		if (discarding && (rxd_status & RXDESC_STATUS_EOP) != 0)
5666 			discarding = false;
5667 
5668 		aq_rxring_reset_desc(sc, rxring, idx);
5669 		AQ_WRITE_REG(sc, RX_DMA_DESC_TAIL_PTR_REG(ringidx), idx);
5670 	}
5671 	/* save ring context */
5672 	rxring->rxr_readidx = idx;
5673 	rxring->rxr_discarding = discarding;
5674 	rxring->rxr_receiving_m = m0;
5675 	rxring->rxr_receiving_m_last = mprev;
5676 
5677 	IF_STAT_PUTREF(ifp);
5678 
5679  rx_intr_done:
5680 	mutex_exit(&rxring->rxr_mutex);
5681 
5682 	AQ_WRITE_REG(sc, AQ_INTR_STATUS_CLR_REG, __BIT(sc->sc_rx_irq[ringidx]));
5683 	return n;
5684 }
5685 
5686 static int
aq_vlan_cb(struct ethercom * ec,uint16_t vid,bool set)5687 aq_vlan_cb(struct ethercom *ec, uint16_t vid, bool set)
5688 {
5689 	struct ifnet * const ifp = &ec->ec_if;
5690 	struct aq_softc * const sc = ifp->if_softc;
5691 
5692 	aq_set_vlan_filters(sc);
5693 	return 0;
5694 }
5695 
5696 static int
aq_ifflags_cb(struct ethercom * ec)5697 aq_ifflags_cb(struct ethercom *ec)
5698 {
5699 	struct ifnet * const ifp = &ec->ec_if;
5700 	struct aq_softc * const sc = ifp->if_softc;
5701 	int i, ecchange, error = 0;
5702 	unsigned short iffchange;
5703 
5704 	AQ_LOCK(sc);
5705 
5706 	iffchange = ifp->if_flags ^ sc->sc_if_flags;
5707 	if ((iffchange & IFF_PROMISC) != 0)
5708 		error = aq_set_filter(sc);
5709 
5710 	ecchange = ec->ec_capenable ^ sc->sc_ec_capenable;
5711 	if (ecchange & ETHERCAP_VLAN_HWTAGGING) {
5712 		for (i = 0; i < AQ_RINGS_NUM; i++) {
5713 			AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_REG(i),
5714 			    RX_DMA_DESC_VLAN_STRIP,
5715 			    (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) ?
5716 			    1 : 0);
5717 		}
5718 	}
5719 
5720 	/* vlan configuration depends on also interface promiscuous mode */
5721 	if ((ecchange & ETHERCAP_VLAN_HWFILTER) || (iffchange & IFF_PROMISC))
5722 		aq_set_vlan_filters(sc);
5723 
5724 	sc->sc_ec_capenable = ec->ec_capenable;
5725 	sc->sc_if_flags = ifp->if_flags;
5726 
5727 	AQ_UNLOCK(sc);
5728 
5729 	return error;
5730 }
5731 
5732 
5733 static int
aq_init(struct ifnet * ifp)5734 aq_init(struct ifnet *ifp)
5735 {
5736 	struct aq_softc * const sc = ifp->if_softc;
5737 
5738 	AQ_LOCK(sc);
5739 
5740 	int ret = aq_init_locked(ifp);
5741 
5742 	AQ_UNLOCK(sc);
5743 
5744 	return ret;
5745 }
5746 
5747 static int
aq_init_locked(struct ifnet * ifp)5748 aq_init_locked(struct ifnet *ifp)
5749 {
5750 	struct aq_softc * const sc = ifp->if_softc;
5751 	int i, error = 0;
5752 
5753 	KASSERT(IFNET_LOCKED(ifp));
5754 	AQ_LOCKED(sc);
5755 
5756 	aq_stop_locked(ifp, false);
5757 
5758 	aq_set_vlan_filters(sc);
5759 	aq_set_capability(sc);
5760 
5761 	for (i = 0; i < sc->sc_nqueues; i++) {
5762 		aq_txring_reset(sc, &sc->sc_queue[i].txring, true);
5763 	}
5764 
5765 	/* invalidate RX descriptor cache */
5766 	AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_CACHE_INIT_REG, RX_DMA_DESC_CACHE_INIT,
5767 	    AQ_READ_REG_BIT(sc,
5768 	    RX_DMA_DESC_CACHE_INIT_REG, RX_DMA_DESC_CACHE_INIT) ^ 1);
5769 
5770 	/* start RX */
5771 	for (i = 0; i < sc->sc_nqueues; i++) {
5772 		error = aq_rxring_reset(sc, &sc->sc_queue[i].rxring, true);
5773 		if (error != 0) {
5774 			device_printf(sc->sc_dev, "%s: cannot allocate rxbuf\n",
5775 			    __func__);
5776 			goto aq_init_failure;
5777 		}
5778 	}
5779 	aq_init_rss(sc);
5780 	if (HWTYPE_AQ1_P(sc))
5781 		aq1_hw_l3_filter_set(sc);
5782 
5783 	/* ring reset? */
5784 	aq_unset_stopping_flags(sc);
5785 
5786 	callout_schedule(&sc->sc_tick_ch, hz);
5787 
5788 	/* ready */
5789 	ifp->if_flags |= IFF_RUNNING;
5790 
5791 	/* start TX and RX */
5792 	aq_enable_intr(sc, /*link*/true, /*txrx*/true);
5793 	AQ_WRITE_REG_BIT(sc, TPB_TX_BUF_REG, TPB_TX_BUF_EN, 1);
5794 	AQ_WRITE_REG_BIT(sc, RPB_RPF_RX_REG, RPB_RPF_RX_BUF_EN, 1);
5795 
5796  aq_init_failure:
5797 	sc->sc_if_flags = ifp->if_flags;
5798 
5799 	return error;
5800 }
5801 
5802 static void
aq_send_common_locked(struct ifnet * ifp,struct aq_softc * sc,struct aq_txring * txring,bool is_transmit)5803 aq_send_common_locked(struct ifnet *ifp, struct aq_softc *sc,
5804     struct aq_txring *txring, bool is_transmit)
5805 {
5806 	struct mbuf *m, *n;
5807 	int npkt, error;
5808 
5809 	if (txring->txr_nfree < AQ_TXD_MIN)
5810 		return;
5811 
5812 	for (npkt = 0; ; npkt++) {
5813 		if (is_transmit)
5814 			m = pcq_peek(txring->txr_pcq);
5815 		else
5816 			IFQ_POLL(&ifp->if_snd, m);
5817 		if (m == NULL)
5818 			break;
5819 
5820 		error = aq_encap_txring(sc, txring, m);
5821 		if (error == EAGAIN) {
5822 			/* Not enough descriptors available. try again later */
5823 			break;
5824 		}
5825 
5826 		if (is_transmit)
5827 			pcq_get(txring->txr_pcq);
5828 		else
5829 			IFQ_DEQUEUE(&ifp->if_snd, n);
5830 
5831 		if (error != 0) {
5832 			/* too many mbuf chains? or other errors. */
5833 			m_freem(m);
5834 			if_statinc(ifp, if_oerrors);
5835 			break;
5836 		}
5837 
5838 		/* update tail ptr */
5839 		AQ_WRITE_REG(sc, TX_DMA_DESC_TAIL_PTR_REG(txring->txr_index),
5840 		    txring->txr_prodidx);
5841 
5842 		/* Pass the packet to any BPF listeners */
5843 		bpf_mtap(ifp, m, BPF_D_OUT);
5844 	}
5845 
5846 	if (npkt) {
5847 		/* Set a watchdog timer in case the chip flakes out. */
5848 		txring->txr_lastsent = time_uptime;
5849 		txring->txr_sending = true;
5850 	}
5851 }
5852 
5853 static void
aq_start(struct ifnet * ifp)5854 aq_start(struct ifnet *ifp)
5855 {
5856 	struct aq_softc * const sc = ifp->if_softc;
5857 	/* aq_start() always use TX ring[0] */
5858 	struct aq_txring * const txring = &sc->sc_queue[0].txring;
5859 
5860 	mutex_enter(&txring->txr_mutex);
5861 	if (txring->txr_active && !txring->txr_stopping)
5862 		aq_send_common_locked(ifp, sc, txring, false);
5863 	mutex_exit(&txring->txr_mutex);
5864 }
5865 
5866 static inline unsigned int
aq_select_txqueue(struct aq_softc * sc,struct mbuf * m)5867 aq_select_txqueue(struct aq_softc *sc, struct mbuf *m)
5868 {
5869 	return (cpu_index(curcpu()) % sc->sc_nqueues);
5870 }
5871 
5872 static int
aq_transmit(struct ifnet * ifp,struct mbuf * m)5873 aq_transmit(struct ifnet *ifp, struct mbuf *m)
5874 {
5875 	struct aq_softc * const sc = ifp->if_softc;
5876 	const int ringidx = aq_select_txqueue(sc, m);
5877 	struct aq_txring * const txring = &sc->sc_queue[ringidx].txring;
5878 
5879 	if (__predict_false(!pcq_put(txring->txr_pcq, m))) {
5880 		m_freem(m);
5881 		return ENOBUFS;
5882 	}
5883 
5884 	if (mutex_tryenter(&txring->txr_mutex)) {
5885 		aq_send_common_locked(ifp, sc, txring, true);
5886 		mutex_exit(&txring->txr_mutex);
5887 	} else {
5888 		kpreempt_disable();
5889 		softint_schedule(txring->txr_softint);
5890 		kpreempt_enable();
5891 	}
5892 	return 0;
5893 }
5894 
5895 static void
aq_deferred_transmit(void * arg)5896 aq_deferred_transmit(void *arg)
5897 {
5898 	struct aq_txring * const txring = arg;
5899 	struct aq_softc * const sc = txring->txr_sc;
5900 	struct ifnet * const ifp = &sc->sc_ethercom.ec_if;
5901 
5902 	mutex_enter(&txring->txr_mutex);
5903 	if (pcq_peek(txring->txr_pcq) != NULL)
5904 		aq_send_common_locked(ifp, sc, txring, true);
5905 	mutex_exit(&txring->txr_mutex);
5906 }
5907 
5908 
5909 static void
aq_unset_stopping_flags(struct aq_softc * sc)5910 aq_unset_stopping_flags(struct aq_softc *sc)
5911 {
5912 
5913 	AQ_LOCKED(sc);
5914 
5915 	/* Must unset stopping flags in ascending order. */
5916 	for (int i = 0; i < sc->sc_nqueues; i++) {
5917 		struct aq_txring *txr = &sc->sc_queue[i].txring;
5918 		struct aq_rxring *rxr = &sc->sc_queue[i].rxring;
5919 
5920 		mutex_enter(&txr->txr_mutex);
5921 		txr->txr_stopping = false;
5922 		mutex_exit(&txr->txr_mutex);
5923 
5924 		mutex_enter(&rxr->rxr_mutex);
5925 		rxr->rxr_stopping = false;
5926 		mutex_exit(&rxr->rxr_mutex);
5927 	}
5928 
5929 	sc->sc_stopping = false;
5930 }
5931 
5932 static void
aq_set_stopping_flags(struct aq_softc * sc)5933 aq_set_stopping_flags(struct aq_softc *sc)
5934 {
5935 
5936 	AQ_LOCKED(sc);
5937 
5938 	/* Must unset stopping flags in ascending order. */
5939 	for (int i = 0; i < sc->sc_nqueues; i++) {
5940 		struct aq_txring *txr = &sc->sc_queue[i].txring;
5941 		struct aq_rxring *rxr = &sc->sc_queue[i].rxring;
5942 
5943 		mutex_enter(&txr->txr_mutex);
5944 		txr->txr_stopping = true;
5945 		mutex_exit(&txr->txr_mutex);
5946 
5947 		mutex_enter(&rxr->rxr_mutex);
5948 		rxr->rxr_stopping = true;
5949 		mutex_exit(&rxr->rxr_mutex);
5950 	}
5951 
5952 	sc->sc_stopping = true;
5953 }
5954 
5955 
5956 static void
aq_stop(struct ifnet * ifp,int disable)5957 aq_stop(struct ifnet *ifp, int disable)
5958 {
5959 	struct aq_softc * const sc = ifp->if_softc;
5960 
5961 	ASSERT_SLEEPABLE();
5962 	KASSERT(IFNET_LOCKED(ifp));
5963 
5964 	AQ_LOCK(sc);
5965 	aq_stop_locked(ifp, disable ? true : false);
5966 	AQ_UNLOCK(sc);
5967 }
5968 
5969 
5970 
5971 static void
aq_stop_locked(struct ifnet * ifp,bool disable)5972 aq_stop_locked(struct ifnet *ifp, bool disable)
5973 {
5974 	struct aq_softc * const sc = ifp->if_softc;
5975 	int i;
5976 
5977 	KASSERT(IFNET_LOCKED(ifp));
5978 	AQ_LOCKED(sc);
5979 
5980 	aq_set_stopping_flags(sc);
5981 
5982 	if ((ifp->if_flags & IFF_RUNNING) == 0)
5983 		goto already_stopped;
5984 
5985 	/* disable tx/rx interrupts */
5986 	aq_enable_intr(sc, /*link*/true, /*txrx*/false);
5987 
5988 	AQ_WRITE_REG_BIT(sc, TPB_TX_BUF_REG, TPB_TX_BUF_EN, 0);
5989 	for (i = 0; i < sc->sc_nqueues; i++) {
5990 		aq_txring_reset(sc, &sc->sc_queue[i].txring, false);
5991 	}
5992 
5993 	AQ_WRITE_REG_BIT(sc, RPB_RPF_RX_REG, RPB_RPF_RX_BUF_EN, 0);
5994 	for (i = 0; i < sc->sc_nqueues; i++) {
5995 		aq_rxring_reset(sc, &sc->sc_queue[i].rxring, false);
5996 	}
5997 
5998 	/* invalidate RX descriptor cache */
5999 	AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_CACHE_INIT_REG, RX_DMA_DESC_CACHE_INIT,
6000 	    AQ_READ_REG_BIT(sc,
6001 	    RX_DMA_DESC_CACHE_INIT_REG, RX_DMA_DESC_CACHE_INIT) ^ 1);
6002 
6003  already_stopped:
6004 	aq_enable_intr(sc, /*link*/false, /*txrx*/false);
6005 	callout_halt(&sc->sc_tick_ch, &sc->sc_mutex);
6006 
6007 	ifp->if_flags &= ~IFF_RUNNING;
6008 	sc->sc_if_flags = ifp->if_flags;
6009 }
6010 
6011 
6012 static void
aq_handle_reset_work(struct work * work,void * arg)6013 aq_handle_reset_work(struct work *work, void *arg)
6014 {
6015 	struct aq_softc * const sc = arg;
6016 	struct ifnet * const ifp = &sc->sc_ethercom.ec_if;
6017 
6018 	printf("%s: watchdog timeout -- resetting\n", ifp->if_xname);
6019 
6020 	AQ_LOCK(sc);
6021 
6022 	device_printf(sc->sc_dev, "%s: INTR_MASK/STATUS = %08x/%08x\n",
6023 	    __func__, AQ_READ_REG(sc, AQ_INTR_MASK_REG),
6024 	    AQ_READ_REG(sc, AQ_INTR_STATUS_REG));
6025 
6026 	for (int n = 0; n < sc->sc_nqueues; n++) {
6027 		struct aq_txring *txring = &sc->sc_queue[n].txring;
6028 		u_int head = AQ_READ_REG_BIT(sc,
6029 		    TX_DMA_DESC_HEAD_PTR_REG(txring->txr_index),
6030 		    TX_DMA_DESC_HEAD_PTR);
6031 		u_int tail = AQ_READ_REG(sc,
6032 		    TX_DMA_DESC_TAIL_PTR_REG(txring->txr_index));
6033 
6034 		device_printf(sc->sc_dev, "%s: TXring[%u] HEAD/TAIL=%u/%u\n",
6035 		    __func__, txring->txr_index, head, tail);
6036 
6037 		aq_tx_intr(txring);
6038 	}
6039 
6040 	AQ_UNLOCK(sc);
6041 
6042 	/* Don't want ioctl operations to happen */
6043 	IFNET_LOCK(ifp);
6044 
6045 	/* reset the interface. */
6046 	aq_init(ifp);
6047 
6048 	IFNET_UNLOCK(ifp);
6049 
6050 	atomic_store_relaxed(&sc->sc_reset_pending, 0);
6051 }
6052 
6053 static int
aq_ioctl(struct ifnet * ifp,unsigned long cmd,void * data)6054 aq_ioctl(struct ifnet *ifp, unsigned long cmd, void *data)
6055 {
6056 	struct aq_softc * const sc = ifp->if_softc;
6057 	struct ifreq * const ifr = data;
6058 	int error = 0;
6059 
6060 	switch (cmd) {
6061 	case SIOCADDMULTI:
6062 	case SIOCDELMULTI:
6063 		break;
6064 	default:
6065 		KASSERT(IFNET_LOCKED(ifp));
6066 	}
6067 
6068 	const int s = splnet();
6069 	switch (cmd) {
6070 	case SIOCSIFMTU:
6071 		if (ifr->ifr_mtu < ETHERMIN || ifr->ifr_mtu > sc->sc_max_mtu) {
6072 			error = EINVAL;
6073 		} else {
6074 			ifp->if_mtu = ifr->ifr_mtu;
6075 			error = 0;	/* no need to reset (no ENETRESET) */
6076 		}
6077 		break;
6078 	default:
6079 		error = ether_ioctl(ifp, cmd, data);
6080 		break;
6081 	}
6082 	splx(s);
6083 
6084 	if (error != ENETRESET)
6085 		return error;
6086 
6087 	switch (cmd) {
6088 	case SIOCSIFCAP:
6089 		error = aq_set_capability(sc);
6090 		break;
6091 	case SIOCADDMULTI:
6092 	case SIOCDELMULTI:
6093 		AQ_LOCK(sc);
6094 		if ((sc->sc_if_flags & IFF_RUNNING) != 0) {
6095 			/*
6096 			 * Multicast list has changed; set the hardware filter
6097 			 * accordingly.
6098 			 */
6099 			error = aq_set_filter(sc);
6100 		}
6101 		AQ_UNLOCK(sc);
6102 		break;
6103 	}
6104 
6105 	return error;
6106 }
6107 
6108 
6109 MODULE(MODULE_CLASS_DRIVER, if_aq, "pci");
6110 
6111 #ifdef _MODULE
6112 #include "ioconf.c"
6113 #endif
6114 
6115 static int
if_aq_modcmd(modcmd_t cmd,void * opaque)6116 if_aq_modcmd(modcmd_t cmd, void *opaque)
6117 {
6118 	int error = 0;
6119 
6120 	switch (cmd) {
6121 	case MODULE_CMD_INIT:
6122 #ifdef _MODULE
6123 		error = config_init_component(cfdriver_ioconf_if_aq,
6124 		    cfattach_ioconf_if_aq, cfdata_ioconf_if_aq);
6125 #endif
6126 		return error;
6127 	case MODULE_CMD_FINI:
6128 #ifdef _MODULE
6129 		error = config_fini_component(cfdriver_ioconf_if_aq,
6130 		    cfattach_ioconf_if_aq, cfdata_ioconf_if_aq);
6131 #endif
6132 		return error;
6133 	default:
6134 		return ENOTTY;
6135 	}
6136 }
6137