1 /* $NetBSD: if_cnmac.c,v 1.29 2022/09/29 07:00:46 skrll Exp $ */
2
3 /*
4 * Copyright (c) 2007 Internet Initiative Japan, Inc.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29 #include <sys/cdefs.h>
30 __KERNEL_RCSID(0, "$NetBSD: if_cnmac.c,v 1.29 2022/09/29 07:00:46 skrll Exp $");
31
32 /*
33 * If no free send buffer is available, free all the sent buffers and bail out.
34 */
35 #define CNMAC_SEND_QUEUE_CHECK
36
37 /* XXX XXX XXX XXX XXX XXX */
38
39 #include <sys/param.h>
40 #include <sys/systm.h>
41 #include <sys/pool.h>
42 #include <sys/mbuf.h>
43 #include <sys/kernel.h>
44 #include <sys/socket.h>
45 #include <sys/ioctl.h>
46 #include <sys/errno.h>
47 #include <sys/device.h>
48 #include <sys/queue.h>
49 #include <sys/conf.h>
50 #include <sys/sysctl.h>
51 #include <sys/syslog.h>
52
53 #include <net/if.h>
54 #include <net/if_media.h>
55 #include <net/if_ether.h>
56 #include <net/route.h>
57 #include <net/bpf.h>
58
59 #include <netinet/in.h>
60 #include <netinet/in_systm.h>
61 #include <netinet/in_var.h>
62 #include <netinet/ip.h>
63
64 #include <sys/bus.h>
65 #include <machine/intr.h>
66 #include <machine/endian.h>
67 #include <machine/locore.h>
68
69 #include <dev/mii/mii.h>
70 #include <dev/mii/miivar.h>
71
72 #include <mips/cpuregs.h>
73
74 #include <mips/cavium/octeonreg.h>
75 #include <mips/cavium/octeonvar.h>
76 #include <mips/cavium/include/iobusvar.h>
77
78 #include <mips/cavium/dev/octeon_ciureg.h>
79 #include <mips/cavium/dev/octeon_faureg.h>
80 #include <mips/cavium/dev/octeon_fpareg.h>
81 #include <mips/cavium/dev/octeon_gmxreg.h>
82 #include <mips/cavium/dev/octeon_pipreg.h>
83 #include <mips/cavium/dev/octeon_powreg.h>
84 #include <mips/cavium/dev/octeon_fauvar.h>
85 #include <mips/cavium/dev/octeon_fpavar.h>
86 #include <mips/cavium/dev/octeon_gmxvar.h>
87 #include <mips/cavium/dev/octeon_ipdvar.h>
88 #include <mips/cavium/dev/octeon_pipvar.h>
89 #include <mips/cavium/dev/octeon_pkovar.h>
90 #include <mips/cavium/dev/octeon_powvar.h>
91 #include <mips/cavium/dev/octeon_smivar.h>
92
93 #include <mips/cavium/dev/if_cnmacvar.h>
94
95 /*
96 * Set the PKO to think command buffers are an odd length. This makes it so we
97 * never have to divide a comamnd across two buffers.
98 */
99 #define OCTEON_POOL_NWORDS_CMD \
100 (((uint32_t)OCTEON_POOL_SIZE_CMD / sizeof(uint64_t)) - 1)
101 #define FPA_COMMAND_BUFFER_POOL_NWORDS OCTEON_POOL_NWORDS_CMD /* XXX */
102
103 static void cnmac_buf_init(struct cnmac_softc *);
104
105 static int cnmac_match(device_t, struct cfdata *, void *);
106 static void cnmac_attach(device_t, device_t, void *);
107 static void cnmac_pip_init(struct cnmac_softc *);
108 static void cnmac_ipd_init(struct cnmac_softc *);
109 static void cnmac_pko_init(struct cnmac_softc *);
110
111 static void cnmac_board_mac_addr(uint8_t *, size_t, struct cnmac_softc *);
112
113 static int cnmac_mii_readreg(device_t, int, int, uint16_t *);
114 static int cnmac_mii_writereg(device_t, int, int, uint16_t);
115 static void cnmac_mii_statchg(struct ifnet *);
116
117 static int cnmac_mediainit(struct cnmac_softc *);
118 static void cnmac_mediastatus(struct ifnet *, struct ifmediareq *);
119
120 static inline void cnmac_send_queue_flush_prefetch(struct cnmac_softc *);
121 static inline void cnmac_send_queue_flush_fetch(struct cnmac_softc *);
122 static inline void cnmac_send_queue_flush(struct cnmac_softc *);
123 static inline void cnmac_send_queue_flush_sync(struct cnmac_softc *);
124 static void cnmac_send_queue_check_and_flush(struct cnmac_softc *);
125 static inline int cnmac_send_queue_is_full(struct cnmac_softc *);
126 static inline void cnmac_send_queue_add(struct cnmac_softc *, struct mbuf *,
127 uint64_t *);
128 static inline void cnmac_send_queue_del(struct cnmac_softc *, struct mbuf **,
129 uint64_t **);
130 static inline int cnmac_buf_free_work(struct cnmac_softc *, uint64_t *);
131 static inline void cnmac_buf_ext_free(struct mbuf *, void *, size_t, void *);
132
133 static int cnmac_ioctl(struct ifnet *, u_long, void *);
134 static void cnmac_watchdog(struct ifnet *);
135 static int cnmac_init(struct ifnet *);
136 static void cnmac_stop(struct ifnet *, int);
137 static void cnmac_start(struct ifnet *);
138
139 static inline int cnmac_send_cmd(struct cnmac_softc *, uint64_t, uint64_t,
140 int *);
141 static inline uint64_t cnmac_send_makecmd_w1(int, paddr_t);
142 static inline uint64_t cnmac_send_makecmd_w0(uint64_t, uint64_t, size_t, int,
143 int);
144 static inline int cnmac_send_makecmd_gbuf(struct cnmac_softc *, struct mbuf *,
145 uint64_t *, int *);
146 static inline int cnmac_send_makecmd(struct cnmac_softc *, struct mbuf *,
147 uint64_t *, uint64_t *, uint64_t *);
148 static inline int cnmac_send_buf(struct cnmac_softc *, struct mbuf *,
149 uint64_t *, int *);
150 static inline int cnmac_send(struct cnmac_softc *, struct mbuf *, int *);
151
152 static int cnmac_reset(struct cnmac_softc *);
153 static int cnmac_configure(struct cnmac_softc *);
154 static int cnmac_configure_common(struct cnmac_softc *);
155
156 static void cnmac_tick_free(void *);
157 static void cnmac_tick_misc(void *);
158
159 static inline int cnmac_recv_mbuf(struct cnmac_softc *, uint64_t *,
160 struct mbuf **);
161 static inline int cnmac_recv_check(struct cnmac_softc *, uint64_t);
162 static inline int cnmac_recv(struct cnmac_softc *, uint64_t *);
163 static int cnmac_intr(void *);
164
165 /* device parameters */
166 int cnmac_param_pko_cmd_w0_n2 = 1;
167
168 CFATTACH_DECL_NEW(cnmac, sizeof(struct cnmac_softc),
169 cnmac_match, cnmac_attach, NULL, NULL);
170
171 /* ---- buffer management */
172
173 static const struct cnmac_pool_param {
174 int poolno;
175 size_t size;
176 size_t nelems;
177 } cnmac_pool_params[] = {
178 #define _ENTRY(x) { OCTEON_POOL_NO_##x, OCTEON_POOL_SIZE_##x, OCTEON_POOL_NELEMS_##x }
179 _ENTRY(PKT),
180 _ENTRY(WQE),
181 _ENTRY(CMD),
182 _ENTRY(SG)
183 #undef _ENTRY
184 };
185 struct octfpa_buf *cnmac_pools[FPA_NPOOLS];
186 #define cnmac_fb_pkt cnmac_pools[OCTEON_POOL_NO_PKT]
187 #define cnmac_fb_wqe cnmac_pools[OCTEON_POOL_NO_WQE]
188 #define cnmac_fb_cmd cnmac_pools[OCTEON_POOL_NO_CMD]
189 #define cnmac_fb_sg cnmac_pools[OCTEON_POOL_NO_SG]
190
191 static int cnmac_npowgroups = 0;
192
193 static void
cnmac_buf_init(struct cnmac_softc * sc)194 cnmac_buf_init(struct cnmac_softc *sc)
195 {
196 static int once;
197 int i;
198 const struct cnmac_pool_param *pp;
199 struct octfpa_buf *fb;
200
201 if (once == 1)
202 return;
203 once = 1;
204
205 for (i = 0; i < (int)__arraycount(cnmac_pool_params); i++) {
206 pp = &cnmac_pool_params[i];
207 octfpa_buf_init(pp->poolno, pp->size, pp->nelems, &fb);
208 cnmac_pools[i] = fb;
209 }
210 }
211
212 /* ---- autoconf */
213
214 static int
cnmac_match(device_t parent,struct cfdata * match,void * aux)215 cnmac_match(device_t parent, struct cfdata *match, void *aux)
216 {
217 struct octgmx_attach_args *ga = aux;
218
219 if (strcmp(match->cf_name, ga->ga_name) != 0) {
220 return 0;
221 }
222 return 1;
223 }
224
225 static void
cnmac_attach(device_t parent,device_t self,void * aux)226 cnmac_attach(device_t parent, device_t self, void *aux)
227 {
228 struct cnmac_softc *sc = device_private(self);
229 struct octgmx_attach_args *ga = aux;
230 struct ifnet *ifp = &sc->sc_ethercom.ec_if;
231 prop_dictionary_t dict;
232 prop_object_t clk;
233 uint8_t enaddr[ETHER_ADDR_LEN];
234
235 if (cnmac_npowgroups >= OCTEON_POW_GROUP_MAX) {
236 printf(": out of POW groups\n");
237 }
238
239 sc->sc_dev = self;
240 sc->sc_regt = ga->ga_regt;
241 sc->sc_port = ga->ga_portno;
242 sc->sc_port_type = ga->ga_port_type;
243 sc->sc_gmx = ga->ga_gmx;
244 sc->sc_gmx_port = ga->ga_gmx_port;
245 sc->sc_smi = ga->ga_smi;
246 sc->sc_powgroup = cnmac_npowgroups++;
247
248 if (sc->sc_port >= CVMSEG_LM_ETHER_COUNT) {
249 /*
250 * If we got here, increase CVMSEG_LM_ETHER_COUNT
251 * in octeonvar.h .
252 */
253 printf("%s: ERROR out of CVMSEG LM buffers\n",
254 device_xname(self));
255 return;
256 }
257
258 sc->sc_init_flag = 0;
259 /*
260 * XXXUEBAYASI
261 * Setting PIP_IP_OFFSET[OFFSET] to 8 causes panic ... why???
262 */
263 sc->sc_ip_offset = 0/* XXX */;
264
265 if (MIPS_PRID_IMPL(mips_options.mips_cpu_id) <= MIPS_CN30XX) {
266 SET(sc->sc_quirks, CNMAC_QUIRKS_NO_PRE_ALIGN);
267 SET(sc->sc_quirks, CNMAC_QUIRKS_NO_RX_INBND);
268 }
269
270 cnmac_board_mac_addr(enaddr, sizeof(enaddr), sc);
271 printf("%s: Ethernet address %s\n", device_xname(self),
272 ether_sprintf(enaddr));
273
274 SIMPLEQ_INIT(&sc->sc_sendq);
275 sc->sc_soft_req_thresh = 15/* XXX */;
276 sc->sc_ext_callback_cnt = 0;
277
278 octgmx_stats_init(sc->sc_gmx_port);
279
280 callout_init(&sc->sc_tick_misc_ch, 0);
281 callout_setfunc(&sc->sc_tick_misc_ch, cnmac_tick_misc, sc);
282
283 callout_init(&sc->sc_tick_free_ch, 0);
284 callout_setfunc(&sc->sc_tick_free_ch, cnmac_tick_free, sc);
285
286 const int dv_unit = device_unit(self);
287 octfau_op_init(&sc->sc_fau_done,
288 OCTEON_CVMSEG_ETHER_OFFSET(dv_unit, csm_ether_fau_done),
289 OCT_FAU_REG_ADDR_END - (8 * (dv_unit + 1))/* XXX */);
290 octfau_op_set_8(&sc->sc_fau_done, 0);
291
292 cnmac_pip_init(sc);
293 cnmac_ipd_init(sc);
294 cnmac_pko_init(sc);
295
296 cnmac_configure_common(sc);
297
298 sc->sc_gmx_port->sc_ipd = sc->sc_ipd;
299 sc->sc_gmx_port->sc_port_mii = &sc->sc_mii;
300 sc->sc_gmx_port->sc_port_ec = &sc->sc_ethercom;
301 /* XXX */
302 sc->sc_gmx_port->sc_quirks = sc->sc_quirks;
303
304 /* XXX */
305 sc->sc_pow = &octpow_softc;
306
307 cnmac_mediainit(sc);
308
309 strncpy(ifp->if_xname, device_xname(self), sizeof(ifp->if_xname));
310 ifp->if_softc = sc;
311 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
312 ifp->if_ioctl = cnmac_ioctl;
313 ifp->if_start = cnmac_start;
314 ifp->if_watchdog = cnmac_watchdog;
315 ifp->if_init = cnmac_init;
316 ifp->if_stop = cnmac_stop;
317 IFQ_SET_MAXLEN(&ifp->if_snd, uimax(GATHER_QUEUE_SIZE, IFQ_MAXLEN));
318 IFQ_SET_READY(&ifp->if_snd);
319
320
321 ifp->if_capabilities =
322 #if 0 /* XXX: no tx checksum yet */
323 IFCAP_CSUM_IPv4_Tx | IFCAP_CSUM_IPv4_Rx |
324 IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_TCPv4_Rx |
325 IFCAP_CSUM_UDPv4_Tx | IFCAP_CSUM_UDPv4_Rx |
326 IFCAP_CSUM_TCPv6_Tx | IFCAP_CSUM_TCPv6_Rx |
327 IFCAP_CSUM_UDPv6_Tx | IFCAP_CSUM_UDPv6_Rx;
328 #else
329 IFCAP_CSUM_IPv4_Rx | IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx |
330 IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx;
331 #endif
332
333 /* 802.1Q VLAN-sized frames are supported */
334 sc->sc_ethercom.ec_capabilities |= ETHERCAP_VLAN_MTU;
335
336 octgmx_set_mac_addr(sc->sc_gmx_port, enaddr);
337
338 if_attach(ifp);
339 ether_ifattach(ifp, enaddr);
340 octgmx_set_filter(sc->sc_gmx_port);
341
342 #if 1
343 cnmac_buf_init(sc);
344 #endif
345
346 sc->sc_ih = octeon_intr_establish(POW_WORKQ_IRQ(sc->sc_powgroup),
347 IPL_NET, cnmac_intr, sc);
348 if (sc->sc_ih == NULL)
349 panic("%s: could not set up interrupt", device_xname(self));
350
351 dict = device_properties(sc->sc_gmx->sc_dev);
352
353 clk = prop_dictionary_get(dict, "rgmii-tx");
354 if (clk)
355 sc->sc_gmx_port->sc_clk_tx_setting =
356 prop_number_signed_value(clk);
357 clk = prop_dictionary_get(dict, "rgmii-rx");
358 if (clk)
359 sc->sc_gmx_port->sc_clk_rx_setting =
360 prop_number_signed_value(clk);
361 }
362
363 /* ---- submodules */
364
365 /* XXX */
366 static void
cnmac_pip_init(struct cnmac_softc * sc)367 cnmac_pip_init(struct cnmac_softc *sc)
368 {
369 struct octpip_attach_args pip_aa;
370
371 pip_aa.aa_port = sc->sc_port;
372 pip_aa.aa_regt = sc->sc_regt;
373 pip_aa.aa_tag_type = POW_TAG_TYPE_ORDERED/* XXX */;
374 pip_aa.aa_receive_group = sc->sc_powgroup;
375 pip_aa.aa_ip_offset = sc->sc_ip_offset;
376 octpip_init(&pip_aa, &sc->sc_pip);
377 octpip_port_config(sc->sc_pip);
378 }
379
380 /* XXX */
381 static void
cnmac_ipd_init(struct cnmac_softc * sc)382 cnmac_ipd_init(struct cnmac_softc *sc)
383 {
384 struct octipd_attach_args ipd_aa;
385
386 ipd_aa.aa_port = sc->sc_port;
387 ipd_aa.aa_regt = sc->sc_regt;
388 ipd_aa.aa_first_mbuff_skip = 184/* XXX */;
389 ipd_aa.aa_not_first_mbuff_skip = 0/* XXX */;
390 octipd_init(&ipd_aa, &sc->sc_ipd);
391 }
392
393 /* XXX */
394 static void
cnmac_pko_init(struct cnmac_softc * sc)395 cnmac_pko_init(struct cnmac_softc *sc)
396 {
397 struct octpko_attach_args pko_aa;
398
399 pko_aa.aa_port = sc->sc_port;
400 pko_aa.aa_regt = sc->sc_regt;
401 pko_aa.aa_cmdptr = &sc->sc_cmdptr;
402 pko_aa.aa_cmd_buf_pool = OCTEON_POOL_NO_CMD;
403 pko_aa.aa_cmd_buf_size = OCTEON_POOL_NWORDS_CMD;
404 octpko_init(&pko_aa, &sc->sc_pko);
405 }
406
407 /* ---- XXX */
408
409 #define ADDR2UINT64(u, a) \
410 do { \
411 u = \
412 (((uint64_t)a[0] << 40) | ((uint64_t)a[1] << 32) | \
413 ((uint64_t)a[2] << 24) | ((uint64_t)a[3] << 16) | \
414 ((uint64_t)a[4] << 8) | ((uint64_t)a[5] << 0)); \
415 } while (0)
416 #define UINT642ADDR(a, u) \
417 do { \
418 a[0] = (uint8_t)((u) >> 40); a[1] = (uint8_t)((u) >> 32); \
419 a[2] = (uint8_t)((u) >> 24); a[3] = (uint8_t)((u) >> 16); \
420 a[4] = (uint8_t)((u) >> 8); a[5] = (uint8_t)((u) >> 0); \
421 } while (0)
422
423 static void
cnmac_board_mac_addr(uint8_t * enaddr,size_t size,struct cnmac_softc * sc)424 cnmac_board_mac_addr(uint8_t *enaddr, size_t size, struct cnmac_softc *sc)
425 {
426 prop_dictionary_t dict;
427 prop_data_t ea;
428
429 dict = device_properties(sc->sc_dev);
430 KASSERT(dict != NULL);
431 ea = prop_dictionary_get(dict, "mac-address");
432 KASSERT(ea != NULL);
433 memcpy(enaddr, prop_data_value(ea), size);
434 }
435
436 /* ---- media */
437
438 static int
cnmac_mii_readreg(device_t self,int phy_addr,int reg,uint16_t * val)439 cnmac_mii_readreg(device_t self, int phy_addr, int reg, uint16_t *val)
440 {
441 struct cnmac_softc *sc = device_private(self);
442
443 return octsmi_read(sc->sc_smi, phy_addr, reg, val);
444 }
445
446 static int
cnmac_mii_writereg(device_t self,int phy_addr,int reg,uint16_t val)447 cnmac_mii_writereg(device_t self, int phy_addr, int reg, uint16_t val)
448 {
449 struct cnmac_softc *sc = device_private(self);
450
451 return octsmi_write(sc->sc_smi, phy_addr, reg, val);
452 }
453
454 static void
cnmac_mii_statchg(struct ifnet * ifp)455 cnmac_mii_statchg(struct ifnet *ifp)
456 {
457 struct cnmac_softc *sc = ifp->if_softc;
458
459 octpko_port_enable(sc->sc_pko, 0);
460 octgmx_port_enable(sc->sc_gmx_port, 0);
461
462 cnmac_reset(sc);
463
464 if (ISSET(ifp->if_flags, IFF_RUNNING))
465 octgmx_set_filter(sc->sc_gmx_port);
466
467 octpko_port_enable(sc->sc_pko, 1);
468 octgmx_port_enable(sc->sc_gmx_port, 1);
469 }
470
471 static int
cnmac_mediainit(struct cnmac_softc * sc)472 cnmac_mediainit(struct cnmac_softc *sc)
473 {
474 struct ifnet *ifp = &sc->sc_ethercom.ec_if;
475 struct mii_data *mii = &sc->sc_mii;
476 prop_object_t phy;
477
478 mii->mii_ifp = ifp;
479 mii->mii_readreg = cnmac_mii_readreg;
480 mii->mii_writereg = cnmac_mii_writereg;
481 mii->mii_statchg = cnmac_mii_statchg;
482 sc->sc_ethercom.ec_mii = mii;
483
484 /* Initialize ifmedia structures. */
485 ifmedia_init(&mii->mii_media, 0, ether_mediachange, cnmac_mediastatus);
486
487 phy = prop_dictionary_get(device_properties(sc->sc_dev), "phy-addr");
488 KASSERT(phy != NULL);
489
490 mii_attach(sc->sc_dev, mii, 0xffffffff, prop_number_signed_value(phy),
491 MII_OFFSET_ANY, MIIF_DOPAUSE);
492
493 /* XXX XXX XXX */
494 if (LIST_FIRST(&mii->mii_phys) != NULL) {
495 /* XXX XXX XXX */
496 ifmedia_set(&mii->mii_media, IFM_ETHER | IFM_AUTO);
497 /* XXX XXX XXX */
498 } else {
499 /* XXX XXX XXX */
500 ifmedia_add(&mii->mii_media, IFM_ETHER | IFM_NONE,
501 MII_MEDIA_NONE, NULL);
502 /* XXX XXX XXX */
503 ifmedia_set(&mii->mii_media, IFM_ETHER | IFM_NONE);
504 /* XXX XXX XXX */
505 }
506 /* XXX XXX XXX */
507
508 return 0;
509 }
510
511 static void
cnmac_mediastatus(struct ifnet * ifp,struct ifmediareq * ifmr)512 cnmac_mediastatus(struct ifnet *ifp, struct ifmediareq *ifmr)
513 {
514 struct cnmac_softc *sc = ifp->if_softc;
515
516 mii_pollstat(&sc->sc_mii);
517
518 ifmr->ifm_status = sc->sc_mii.mii_media_status;
519 ifmr->ifm_active = sc->sc_mii.mii_media_active;
520 ifmr->ifm_active = (sc->sc_mii.mii_media_active & ~IFM_ETH_FMASK) |
521 sc->sc_gmx_port->sc_port_flowflags;
522 }
523
524 /* ---- send buffer garbage collection */
525
526 static inline void
cnmac_send_queue_flush_prefetch(struct cnmac_softc * sc)527 cnmac_send_queue_flush_prefetch(struct cnmac_softc *sc)
528 {
529
530 KASSERT(sc->sc_prefetch == 0);
531 octfau_op_inc_fetch_8(&sc->sc_fau_done, 0);
532 sc->sc_prefetch = 1;
533 }
534
535 static inline void
cnmac_send_queue_flush_fetch(struct cnmac_softc * sc)536 cnmac_send_queue_flush_fetch(struct cnmac_softc *sc)
537 {
538
539 KASSERT(sc->sc_prefetch == 1);
540 sc->sc_hard_done_cnt = octfau_op_inc_read_8(&sc->sc_fau_done);
541 KASSERT(sc->sc_hard_done_cnt <= 0);
542 sc->sc_prefetch = 0;
543 }
544
545 static inline void
cnmac_send_queue_flush(struct cnmac_softc * sc)546 cnmac_send_queue_flush(struct cnmac_softc *sc)
547 {
548 const int64_t sent_count = sc->sc_hard_done_cnt;
549 int i;
550
551 KASSERT(sc->sc_flush == 0);
552 KASSERT(sent_count <= 0);
553
554 for (i = 0; i < 0 - sent_count; i++) {
555 struct mbuf *m;
556 uint64_t *gbuf;
557
558 cnmac_send_queue_del(sc, &m, &gbuf);
559
560 octfpa_buf_put(cnmac_fb_sg, gbuf);
561
562 m_freem(m);
563
564 sc->sc_txbusy = false;
565 }
566
567 octfau_op_inc_fetch_8(&sc->sc_fau_done, i);
568 sc->sc_flush = i;
569 }
570
571 static inline void
cnmac_send_queue_flush_sync(struct cnmac_softc * sc)572 cnmac_send_queue_flush_sync(struct cnmac_softc *sc)
573 {
574 if (sc->sc_flush == 0)
575 return;
576
577 KASSERT(sc->sc_flush > 0);
578
579 /* XXX XXX XXX */
580 octfau_op_inc_read_8(&sc->sc_fau_done);
581 sc->sc_soft_req_cnt -= sc->sc_flush;
582 KASSERT(sc->sc_soft_req_cnt >= 0);
583 /* XXX XXX XXX */
584
585 sc->sc_flush = 0;
586 }
587
588 static inline int
cnmac_send_queue_is_full(struct cnmac_softc * sc)589 cnmac_send_queue_is_full(struct cnmac_softc *sc)
590 {
591 #ifdef CNMAC_SEND_QUEUE_CHECK
592 int64_t nofree_cnt;
593
594 nofree_cnt = sc->sc_soft_req_cnt + sc->sc_hard_done_cnt;
595
596 if (__predict_false(nofree_cnt == GATHER_QUEUE_SIZE - 1)) {
597 cnmac_send_queue_flush(sc);
598 cnmac_send_queue_flush_sync(sc);
599 return 1;
600 }
601
602 #endif
603 return 0;
604 }
605
606 static void
cnmac_send_queue_check_and_flush(struct cnmac_softc * sc)607 cnmac_send_queue_check_and_flush(struct cnmac_softc *sc)
608 {
609 int s;
610
611 /* XXX XXX XXX */
612 s = splnet();
613 if (sc->sc_soft_req_cnt > 0) {
614 cnmac_send_queue_flush_prefetch(sc);
615 cnmac_send_queue_flush_fetch(sc);
616 cnmac_send_queue_flush(sc);
617 cnmac_send_queue_flush_sync(sc);
618 }
619 splx(s);
620 /* XXX XXX XXX */
621 }
622
623 /*
624 * (Ab)use m_nextpkt and m_paddr to maintain mbuf chain and pointer to gather
625 * buffer. Other mbuf members may be used by m_freem(), so don't touch them!
626 */
627
628 struct _send_queue_entry {
629 union {
630 struct mbuf _sqe_s_mbuf;
631 struct {
632 char _sqe_s_entry_pad[offsetof(struct mbuf, m_nextpkt)];
633 SIMPLEQ_ENTRY(_send_queue_entry) _sqe_s_entry_entry;
634 } _sqe_s_entry;
635 struct {
636 char _sqe_s_gbuf_pad[offsetof(struct mbuf, m_paddr)];
637 uint64_t *_sqe_s_gbuf_gbuf;
638 } _sqe_s_gbuf;
639 } _sqe_u;
640 #define _sqe_entry _sqe_u._sqe_s_entry._sqe_s_entry_entry
641 #define _sqe_gbuf _sqe_u._sqe_s_gbuf._sqe_s_gbuf_gbuf
642 };
643
644 static inline void
cnmac_send_queue_add(struct cnmac_softc * sc,struct mbuf * m,uint64_t * gbuf)645 cnmac_send_queue_add(struct cnmac_softc *sc, struct mbuf *m,
646 uint64_t *gbuf)
647 {
648 struct _send_queue_entry *sqe = (struct _send_queue_entry *)m;
649
650 sqe->_sqe_gbuf = gbuf;
651 SIMPLEQ_INSERT_TAIL(&sc->sc_sendq, sqe, _sqe_entry);
652
653 if ((m->m_flags & M_EXT) && m->m_ext.ext_free != NULL)
654 sc->sc_ext_callback_cnt++;
655 }
656
657 static inline void
cnmac_send_queue_del(struct cnmac_softc * sc,struct mbuf ** rm,uint64_t ** rgbuf)658 cnmac_send_queue_del(struct cnmac_softc *sc, struct mbuf **rm, uint64_t **rgbuf)
659 {
660 struct _send_queue_entry *sqe;
661
662 sqe = SIMPLEQ_FIRST(&sc->sc_sendq);
663 KASSERT(sqe != NULL);
664 SIMPLEQ_REMOVE_HEAD(&sc->sc_sendq, _sqe_entry);
665
666 *rm = (void *)sqe;
667 *rgbuf = sqe->_sqe_gbuf;
668
669 if (((*rm)->m_flags & M_EXT) && (*rm)->m_ext.ext_free != NULL) {
670 sc->sc_ext_callback_cnt--;
671 KASSERT(sc->sc_ext_callback_cnt >= 0);
672 }
673 }
674
675 static inline int
cnmac_buf_free_work(struct cnmac_softc * sc,uint64_t * work)676 cnmac_buf_free_work(struct cnmac_softc *sc, uint64_t *work)
677 {
678
679 /* XXX when jumbo frame */
680 if (ISSET(work[2], PIP_WQE_WORD2_IP_BUFS)) {
681 paddr_t addr;
682 paddr_t start_buffer;
683
684 addr = work[3] & PIP_WQE_WORD3_ADDR;
685 start_buffer = addr & ~(2048 - 1);
686
687 octfpa_buf_put_paddr(cnmac_fb_pkt, start_buffer);
688 }
689
690 octfpa_buf_put(cnmac_fb_wqe, work);
691
692 return 0;
693 }
694
695 static inline void
cnmac_buf_ext_free(struct mbuf * m,void * buf,size_t size,void * arg)696 cnmac_buf_ext_free(struct mbuf *m, void *buf, size_t size, void *arg)
697 {
698 octfpa_buf_put(cnmac_fb_pkt, buf);
699
700 KASSERT(m != NULL);
701
702 pool_cache_put(mb_cache, m);
703 }
704
705 /* ---- ifnet interfaces */
706
707 static int
cnmac_ioctl(struct ifnet * ifp,u_long cmd,void * data)708 cnmac_ioctl(struct ifnet *ifp, u_long cmd, void *data)
709 {
710 struct cnmac_softc *sc = ifp->if_softc;
711 struct ifreq *ifr = (struct ifreq *)data;
712 int s, error;
713
714 s = splnet();
715 switch (cmd) {
716 case SIOCSIFMEDIA:
717 /* Flow control requires full-duplex mode. */
718 if (IFM_SUBTYPE(ifr->ifr_media) == IFM_AUTO ||
719 (ifr->ifr_media & IFM_FDX) == 0) {
720 ifr->ifr_media &= ~IFM_ETH_FMASK;
721 }
722 if (IFM_SUBTYPE(ifr->ifr_media) != IFM_AUTO) {
723 if ((ifr->ifr_media & IFM_ETH_FMASK) == IFM_FLOW) {
724 ifr->ifr_media |=
725 IFM_ETH_TXPAUSE | IFM_ETH_RXPAUSE;
726 }
727 sc->sc_gmx_port->sc_port_flowflags =
728 ifr->ifr_media & IFM_ETH_FMASK;
729 }
730 error = ifmedia_ioctl(ifp, ifr, &sc->sc_mii.mii_media, cmd);
731 break;
732 default:
733 error = ether_ioctl(ifp, cmd, data);
734 break;
735 }
736
737 if (error == ENETRESET) {
738 if (ISSET(ifp->if_flags, IFF_RUNNING))
739 octgmx_set_filter(sc->sc_gmx_port);
740 error = 0;
741 }
742
743 cnmac_start(ifp);
744
745 splx(s);
746
747 return error;
748 }
749
750 /* ---- send (output) */
751
752 static inline uint64_t
cnmac_send_makecmd_w0(uint64_t fau0,uint64_t fau1,size_t len,int segs,int ipoffp1)753 cnmac_send_makecmd_w0(uint64_t fau0, uint64_t fau1, size_t len, int segs,
754 int ipoffp1)
755 {
756
757 return octpko_cmd_word0(
758 OCT_FAU_OP_SIZE_64, /* sz1 */
759 OCT_FAU_OP_SIZE_64, /* sz0 */
760 1, fau1, 1, fau0, /* s1, reg1, s0, reg0 */
761 0, /* le */
762 cnmac_param_pko_cmd_w0_n2, /* n2 */
763 1, 0, /* q, r */
764 (segs == 1) ? 0 : 1, /* g */
765 0, 0, 1, /* ipoffp1, ii, df */
766 segs, (int)len); /* segs, totalbytes */
767 }
768
769 static inline uint64_t
cnmac_send_makecmd_w1(int size,paddr_t addr)770 cnmac_send_makecmd_w1(int size, paddr_t addr)
771 {
772
773 return octpko_cmd_word1(
774 0, 0, /* i, back */
775 OCTEON_POOL_NO_SG, /* pool */
776 size, addr); /* size, addr */
777 }
778
779 static inline int
cnmac_send_makecmd_gbuf(struct cnmac_softc * sc,struct mbuf * m0,uint64_t * gbuf,int * rsegs)780 cnmac_send_makecmd_gbuf(struct cnmac_softc *sc, struct mbuf *m0, uint64_t *gbuf,
781 int *rsegs)
782 {
783 struct mbuf *m;
784 int segs = 0;
785 uintptr_t laddr, rlen, nlen;
786
787 for (m = m0; m != NULL; m = m->m_next) {
788
789 if (__predict_false(m->m_len == 0))
790 continue;
791
792 /* Aligned 4k */
793 laddr = (uintptr_t)m->m_data & (PAGE_SIZE - 1);
794
795 if (laddr + m->m_len > PAGE_SIZE) {
796 /* XXX XXX XXX */
797 rlen = PAGE_SIZE - laddr;
798 nlen = m->m_len - rlen;
799 *(gbuf + segs) = cnmac_send_makecmd_w1(rlen,
800 kvtophys((vaddr_t)m->m_data));
801 segs++;
802 if (segs > 63) {
803 return 1;
804 }
805 /* XXX XXX XXX */
806 } else {
807 rlen = 0;
808 nlen = m->m_len;
809 }
810
811 *(gbuf + segs) = cnmac_send_makecmd_w1(nlen,
812 kvtophys((vaddr_t)(m->m_data + rlen)));
813 segs++;
814 if (segs > 63) {
815 return 1;
816 }
817 }
818
819 KASSERT(m == NULL);
820
821 *rsegs = segs;
822
823 return 0;
824 }
825
826 static inline int
cnmac_send_makecmd(struct cnmac_softc * sc,struct mbuf * m,uint64_t * gbuf,uint64_t * rpko_cmd_w0,uint64_t * rpko_cmd_w1)827 cnmac_send_makecmd(struct cnmac_softc *sc, struct mbuf *m,
828 uint64_t *gbuf, uint64_t *rpko_cmd_w0, uint64_t *rpko_cmd_w1)
829 {
830 uint64_t pko_cmd_w0, pko_cmd_w1;
831 int ipoffp1;
832 int segs;
833 int result = 0;
834
835 if (cnmac_send_makecmd_gbuf(sc, m, gbuf, &segs)) {
836 log(LOG_WARNING, "%s: there are a lot of number of segments"
837 " of transmission data", device_xname(sc->sc_dev));
838 result = 1;
839 goto done;
840 }
841
842 /* Get the IP packet offset for TCP/UDP checksum offloading. */
843 ipoffp1 = (m->m_pkthdr.csum_flags & (M_CSUM_TCPv4 | M_CSUM_UDPv4))
844 ? (ETHER_HDR_LEN + 1) : 0;
845
846 /*
847 * segs == 1 -> link mode (single continuous buffer)
848 * WORD1[size] is number of bytes pointed by segment
849 *
850 * segs > 1 -> gather mode (scatter-gather buffer)
851 * WORD1[size] is number of segments
852 */
853 pko_cmd_w0 = cnmac_send_makecmd_w0(sc->sc_fau_done.fd_regno,
854 0, m->m_pkthdr.len, segs, ipoffp1);
855 if (segs == 1) {
856 pko_cmd_w1 = cnmac_send_makecmd_w1(
857 m->m_pkthdr.len, kvtophys((vaddr_t)m->m_data));
858 } else {
859 #ifdef __mips_n32
860 KASSERT(MIPS_KSEG0_P(gbuf));
861 pko_cmd_w1 = cnmac_send_makecmd_w1(segs,
862 MIPS_KSEG0_TO_PHYS(gbuf));
863 #else
864 pko_cmd_w1 = cnmac_send_makecmd_w1(segs,
865 MIPS_XKPHYS_TO_PHYS(gbuf));
866 #endif
867 }
868
869 *rpko_cmd_w0 = pko_cmd_w0;
870 *rpko_cmd_w1 = pko_cmd_w1;
871
872 done:
873 return result;
874 }
875
876 static inline int
cnmac_send_cmd(struct cnmac_softc * sc,uint64_t pko_cmd_w0,uint64_t pko_cmd_w1,int * pwdc)877 cnmac_send_cmd(struct cnmac_softc *sc, uint64_t pko_cmd_w0,
878 uint64_t pko_cmd_w1, int *pwdc)
879 {
880 uint64_t *cmdptr;
881 int result = 0;
882
883 #ifdef __mips_n32
884 KASSERT((sc->sc_cmdptr.cmdptr & ~MIPS_PHYS_MASK) == 0);
885 cmdptr = (uint64_t *)MIPS_PHYS_TO_KSEG0(sc->sc_cmdptr.cmdptr);
886 #else
887 cmdptr = (uint64_t *)MIPS_PHYS_TO_XKPHYS_CACHED(sc->sc_cmdptr.cmdptr);
888 #endif
889 cmdptr += sc->sc_cmdptr.cmdptr_idx;
890
891 KASSERT(cmdptr != NULL);
892
893 *cmdptr++ = pko_cmd_w0;
894 *cmdptr++ = pko_cmd_w1;
895
896 KASSERT(sc->sc_cmdptr.cmdptr_idx + 2 <= FPA_COMMAND_BUFFER_POOL_NWORDS - 1);
897
898 if (sc->sc_cmdptr.cmdptr_idx + 2 == FPA_COMMAND_BUFFER_POOL_NWORDS - 1) {
899 paddr_t buf;
900
901 buf = octfpa_buf_get_paddr(cnmac_fb_cmd);
902 if (buf == 0) {
903 log(LOG_WARNING,
904 "%s: can not allocate command buffer from free pool allocator\n",
905 device_xname(sc->sc_dev));
906 result = 1;
907 goto done;
908 }
909 *cmdptr++ = buf;
910 sc->sc_cmdptr.cmdptr = (uint64_t)buf;
911 sc->sc_cmdptr.cmdptr_idx = 0;
912 } else {
913 sc->sc_cmdptr.cmdptr_idx += 2;
914 }
915
916 *pwdc += 2;
917
918 done:
919 return result;
920 }
921
922 static inline int
cnmac_send_buf(struct cnmac_softc * sc,struct mbuf * m,uint64_t * gbuf,int * pwdc)923 cnmac_send_buf(struct cnmac_softc *sc, struct mbuf *m, uint64_t *gbuf,
924 int *pwdc)
925 {
926 int result = 0, error;
927 uint64_t pko_cmd_w0, pko_cmd_w1;
928
929 error = cnmac_send_makecmd(sc, m, gbuf, &pko_cmd_w0, &pko_cmd_w1);
930 if (error != 0) {
931 /* Already logging */
932 result = error;
933 goto done;
934 }
935
936 error = cnmac_send_cmd(sc, pko_cmd_w0, pko_cmd_w1, pwdc);
937 if (error != 0) {
938 /* Already logging */
939 result = error;
940 }
941
942 done:
943 return result;
944 }
945
946 static inline int
cnmac_send(struct cnmac_softc * sc,struct mbuf * m,int * pwdc)947 cnmac_send(struct cnmac_softc *sc, struct mbuf *m, int *pwdc)
948 {
949 paddr_t gaddr = 0;
950 uint64_t *gbuf = NULL;
951 int result = 0, error;
952
953 gaddr = octfpa_buf_get_paddr(cnmac_fb_sg);
954 if (gaddr == 0) {
955 log(LOG_WARNING, "%s: can not allocate gather buffer from "
956 "free pool allocator\n", device_xname(sc->sc_dev));
957 result = 1;
958 goto done;
959 }
960
961 #ifdef __mips_n32
962 KASSERT((gaddr & ~MIPS_PHYS_MASK) == 0);
963 gbuf = (uint64_t *)(uintptr_t)MIPS_PHYS_TO_KSEG0(gaddr);
964 #else
965 gbuf = (uint64_t *)(uintptr_t)MIPS_PHYS_TO_XKPHYS_CACHED(gaddr);
966 #endif
967
968 KASSERT(gbuf != NULL);
969
970 error = cnmac_send_buf(sc, m, gbuf, pwdc);
971 if (error != 0) {
972 /* Already logging */
973 octfpa_buf_put_paddr(cnmac_fb_sg, gaddr);
974 result = error;
975 goto done;
976 }
977
978 cnmac_send_queue_add(sc, m, gbuf);
979
980 done:
981 return result;
982 }
983
984 static void
cnmac_start(struct ifnet * ifp)985 cnmac_start(struct ifnet *ifp)
986 {
987 struct cnmac_softc *sc = ifp->if_softc;
988 struct mbuf *m;
989 int wdc = 0;
990
991 /*
992 * Performance tuning
993 * pre-send iobdma request
994 */
995 cnmac_send_queue_flush_prefetch(sc);
996
997 if ((ifp->if_flags & IFF_RUNNING) == 0)
998 goto last;
999
1000 if (sc->sc_txbusy)
1001 goto last;
1002
1003 if (__predict_false(!octgmx_link_status(sc->sc_gmx_port)))
1004 goto last;
1005
1006 for (;;) {
1007 IFQ_POLL(&ifp->if_snd, m);
1008 if (__predict_false(m == NULL))
1009 break;
1010
1011 /* XXX XXX XXX */
1012 cnmac_send_queue_flush_fetch(sc);
1013
1014 /*
1015 * If no free send buffer is available, free all the sent
1016 * buffers and bail out.
1017 */
1018 if (cnmac_send_queue_is_full(sc)) {
1019 sc->sc_txbusy = true;
1020 if (wdc > 0)
1021 octpko_op_doorbell_write(sc->sc_port,
1022 sc->sc_port, wdc);
1023 callout_schedule(&sc->sc_tick_free_ch, 1);
1024 return;
1025 }
1026 /* XXX XXX XXX */
1027
1028 IFQ_DEQUEUE(&ifp->if_snd, m);
1029
1030 bpf_mtap(ifp, m, BPF_D_OUT);
1031
1032 /* XXX XXX XXX */
1033 if (sc->sc_soft_req_cnt > sc->sc_soft_req_thresh)
1034 cnmac_send_queue_flush(sc);
1035 if (cnmac_send(sc, m, &wdc)) {
1036 IF_DROP(&ifp->if_snd);
1037 m_freem(m);
1038 log(LOG_WARNING,
1039 "%s: failed in the transmission of the packet\n",
1040 device_xname(sc->sc_dev));
1041 } else
1042 sc->sc_soft_req_cnt++;
1043
1044 if (sc->sc_flush)
1045 cnmac_send_queue_flush_sync(sc);
1046 /* XXX XXX XXX */
1047
1048 /* Send next iobdma request */
1049 cnmac_send_queue_flush_prefetch(sc);
1050 }
1051
1052 if (wdc > 0)
1053 octpko_op_doorbell_write(sc->sc_port, sc->sc_port, wdc);
1054
1055 last:
1056 cnmac_send_queue_flush_fetch(sc);
1057 callout_schedule(&sc->sc_tick_free_ch, 1);
1058 }
1059
1060 static void
cnmac_watchdog(struct ifnet * ifp)1061 cnmac_watchdog(struct ifnet *ifp)
1062 {
1063 struct cnmac_softc *sc = ifp->if_softc;
1064
1065 printf("%s: device timeout\n", device_xname(sc->sc_dev));
1066
1067 cnmac_configure(sc);
1068
1069 SET(ifp->if_flags, IFF_RUNNING);
1070 sc->sc_txbusy = false;
1071 ifp->if_timer = 0;
1072
1073 cnmac_start(ifp);
1074 }
1075
1076 static int
cnmac_init(struct ifnet * ifp)1077 cnmac_init(struct ifnet *ifp)
1078 {
1079 struct cnmac_softc *sc = ifp->if_softc;
1080
1081 /* XXX don't disable commonly used parts!!! XXX */
1082 if (sc->sc_init_flag == 0) {
1083 /* Cancel any pending I/O. */
1084 cnmac_stop(ifp, 0);
1085
1086 /* Initialize the device */
1087 cnmac_configure(sc);
1088
1089 octpko_enable(sc->sc_pko);
1090 octipd_enable(sc->sc_ipd);
1091
1092 sc->sc_init_flag = 1;
1093 } else {
1094 octgmx_port_enable(sc->sc_gmx_port, 1);
1095 }
1096 mii_ifmedia_change(&sc->sc_mii);
1097
1098 octgmx_set_filter(sc->sc_gmx_port);
1099
1100 callout_schedule(&sc->sc_tick_misc_ch, hz);
1101 callout_schedule(&sc->sc_tick_free_ch, hz);
1102
1103 SET(ifp->if_flags, IFF_RUNNING);
1104 sc->sc_txbusy = false;
1105
1106 return 0;
1107 }
1108
1109 static void
cnmac_stop(struct ifnet * ifp,int disable)1110 cnmac_stop(struct ifnet *ifp, int disable)
1111 {
1112 struct cnmac_softc *sc = ifp->if_softc;
1113
1114 callout_stop(&sc->sc_tick_misc_ch);
1115 callout_stop(&sc->sc_tick_free_ch);
1116
1117 mii_down(&sc->sc_mii);
1118
1119 octgmx_port_enable(sc->sc_gmx_port, 0);
1120
1121 /* Mark the interface as down and cancel the watchdog timer. */
1122 CLR(ifp->if_flags, IFF_RUNNING);
1123 sc->sc_txbusy = false;
1124 ifp->if_timer = 0;
1125 }
1126
1127 /* ---- misc */
1128
1129 static int
cnmac_reset(struct cnmac_softc * sc)1130 cnmac_reset(struct cnmac_softc *sc)
1131 {
1132 octgmx_reset_speed(sc->sc_gmx_port);
1133 octgmx_reset_flowctl(sc->sc_gmx_port);
1134 octgmx_reset_timing(sc->sc_gmx_port);
1135
1136 return 0;
1137 }
1138
1139 static int
cnmac_configure(struct cnmac_softc * sc)1140 cnmac_configure(struct cnmac_softc *sc)
1141 {
1142 octgmx_port_enable(sc->sc_gmx_port, 0);
1143
1144 cnmac_reset(sc);
1145
1146 cnmac_configure_common(sc);
1147
1148 octpko_port_config(sc->sc_pko);
1149 octpko_port_enable(sc->sc_pko, 1);
1150 octpow_config(sc->sc_pow, sc->sc_powgroup);
1151
1152 octgmx_tx_stats_rd_clr(sc->sc_gmx_port, 1);
1153 octgmx_rx_stats_rd_clr(sc->sc_gmx_port, 1);
1154
1155 octgmx_port_enable(sc->sc_gmx_port, 1);
1156
1157 return 0;
1158 }
1159
1160 static int
cnmac_configure_common(struct cnmac_softc * sc)1161 cnmac_configure_common(struct cnmac_softc *sc)
1162 {
1163 static int once;
1164
1165 if (once == 1)
1166 return 0;
1167 once = 1;
1168
1169 octipd_config(sc->sc_ipd);
1170 octpko_config(sc->sc_pko);
1171
1172 return 0;
1173 }
1174
1175 /* ---- receive (input) */
1176
1177 static inline int
cnmac_recv_mbuf(struct cnmac_softc * sc,uint64_t * work,struct mbuf ** rm)1178 cnmac_recv_mbuf(struct cnmac_softc *sc, uint64_t *work, struct mbuf **rm)
1179 {
1180 struct mbuf *m;
1181 vaddr_t addr;
1182 vaddr_t ext_buf;
1183 size_t ext_size;
1184 uint64_t word1 = work[1];
1185 uint64_t word2 = work[2];
1186 uint64_t word3 = work[3];
1187
1188 MGETHDR(m, M_NOWAIT, MT_DATA);
1189 if (m == NULL)
1190 return 1;
1191
1192 octfpa_buf_put(cnmac_fb_wqe, work);
1193
1194 if (__SHIFTOUT(word2, PIP_WQE_WORD2_IP_BUFS) != 1)
1195 panic("%s: expected one buffer, got %" PRId64, __func__,
1196 __SHIFTOUT(word2, PIP_WQE_WORD2_IP_BUFS));
1197
1198
1199 #ifdef __mips_n32
1200 KASSERT((word3 & ~MIPS_PHYS_MASK) == 0);
1201 addr = MIPS_PHYS_TO_KSEG0(word3 & PIP_WQE_WORD3_ADDR);
1202 #else
1203 addr = MIPS_PHYS_TO_XKPHYS_CACHED(word3 & PIP_WQE_WORD3_ADDR);
1204 #endif
1205
1206 ext_size = OCTEON_POOL_SIZE_PKT;
1207 ext_buf = addr & ~(ext_size - 1);
1208 MEXTADD(m, ext_buf, ext_size, 0, cnmac_buf_ext_free, NULL);
1209
1210 m->m_data = (void *)addr;
1211 m->m_len = m->m_pkthdr.len = (word1 & PIP_WQE_WORD1_LEN) >> 48;
1212 m_set_rcvif(m, &sc->sc_ethercom.ec_if);
1213
1214 /* Not readonly buffer */
1215 m->m_flags |= M_EXT_RW;
1216
1217 *rm = m;
1218
1219 KASSERT(*rm != NULL);
1220
1221 return 0;
1222 }
1223
1224 static inline int
cnmac_recv_check(struct cnmac_softc * sc,uint64_t word2)1225 cnmac_recv_check(struct cnmac_softc *sc, uint64_t word2)
1226 {
1227 static struct timeval rxerr_log_interval = { 0, 2500000 };
1228 uint64_t opecode;
1229
1230 if (__predict_true(!ISSET(word2, PIP_WQE_WORD2_NOIP_RE)))
1231 return 0;
1232
1233 opecode = word2 & PIP_WQE_WORD2_NOIP_OPECODE;
1234 if ((sc->sc_ethercom.ec_if.if_flags & IFF_DEBUG) &&
1235 ratecheck(&sc->sc_rxerr_log_last, &rxerr_log_interval))
1236 log(LOG_DEBUG, "%s: rx error (%"PRId64")\n",
1237 device_xname(sc->sc_dev), opecode);
1238
1239 /* This error is harmless */
1240 if (opecode == PIP_WQE_WORD2_RE_OPCODE_OVRRUN)
1241 return 0;
1242
1243 return 1;
1244 }
1245
1246 static inline int
cnmac_recv(struct cnmac_softc * sc,uint64_t * work)1247 cnmac_recv(struct cnmac_softc *sc, uint64_t *work)
1248 {
1249 struct ifnet *ifp;
1250 struct mbuf *m;
1251 uint64_t word2;
1252
1253 KASSERT(sc != NULL);
1254 KASSERT(work != NULL);
1255
1256 word2 = work[2];
1257 ifp = &sc->sc_ethercom.ec_if;
1258
1259 KASSERT(ifp != NULL);
1260
1261 if (!ISSET(ifp->if_flags, IFF_RUNNING))
1262 goto drop;
1263
1264 if (__predict_false(cnmac_recv_check(sc, word2) != 0)) {
1265 if_statinc(ifp, if_ierrors);
1266 goto drop;
1267 }
1268
1269 if (__predict_false(cnmac_recv_mbuf(sc, work, &m) != 0)) {
1270 if_statinc(ifp, if_ierrors);
1271 goto drop;
1272 }
1273
1274 /* work[0] .. work[3] may not be valid any more */
1275
1276 KASSERT(m != NULL);
1277
1278 octipd_offload(word2, m->m_data, &m->m_pkthdr.csum_flags);
1279
1280 if_percpuq_enqueue(ifp->if_percpuq, m);
1281
1282 return 0;
1283
1284 drop:
1285 cnmac_buf_free_work(sc, work);
1286 return 1;
1287 }
1288
1289 static int
cnmac_intr(void * arg)1290 cnmac_intr(void *arg)
1291 {
1292 struct cnmac_softc *sc = arg;
1293 uint64_t *work;
1294 uint64_t wqmask = __BIT(sc->sc_powgroup);
1295 uint32_t coreid = 0; /* XXX octeon_get_coreid() */
1296 uint32_t port;
1297
1298 _POW_WR8(sc->sc_pow, POW_PP_GRP_MSK_OFFSET(coreid), wqmask);
1299
1300 octpow_tag_sw_wait();
1301 octpow_work_request_async(OCTEON_CVMSEG_OFFSET(csm_pow_intr),
1302 POW_NO_WAIT);
1303
1304 for (;;) {
1305 work = (uint64_t *)octpow_work_response_async(
1306 OCTEON_CVMSEG_OFFSET(csm_pow_intr));
1307 if (work == NULL)
1308 break;
1309
1310 octpow_tag_sw_wait();
1311 octpow_work_request_async(OCTEON_CVMSEG_OFFSET(csm_pow_intr),
1312 POW_NO_WAIT);
1313
1314 port = __SHIFTOUT(work[1], PIP_WQE_WORD1_IPRT);
1315 if (port != sc->sc_port) {
1316 printf("%s: unexpected wqe port %u, should be %u\n",
1317 device_xname(sc->sc_dev), port, sc->sc_port);
1318 goto wqe_error;
1319 }
1320
1321 (void)cnmac_recv(sc, work);
1322
1323 cnmac_send_queue_check_and_flush(sc);
1324 }
1325
1326 _POW_WR8(sc->sc_pow, POW_WQ_INT_OFFSET, wqmask);
1327
1328 return 1;
1329
1330 wqe_error:
1331 printf("word0: 0x%016" PRIx64 "\n", work[0]);
1332 printf("word1: 0x%016" PRIx64 "\n", work[1]);
1333 printf("word2: 0x%016" PRIx64 "\n", work[2]);
1334 printf("word3: 0x%016" PRIx64 "\n", work[3]);
1335 panic("wqe_error");
1336 }
1337
1338 /* ---- tick */
1339
1340 /*
1341 * cnmac_tick_free
1342 *
1343 * => garbage collect send gather buffer / mbuf
1344 * => called at softclock
1345 */
1346 static void
cnmac_tick_free(void * arg)1347 cnmac_tick_free(void *arg)
1348 {
1349 struct cnmac_softc *sc = arg;
1350 int timo;
1351
1352 cnmac_send_queue_check_and_flush(sc);
1353
1354 timo = (sc->sc_ext_callback_cnt > 0) ? 1 : hz;
1355 callout_schedule(&sc->sc_tick_free_ch, timo);
1356 }
1357
1358 /*
1359 * cnmac_tick_misc
1360 *
1361 * => collect statistics
1362 * => check link status
1363 * => called at softclock
1364 */
1365 static void
cnmac_tick_misc(void * arg)1366 cnmac_tick_misc(void *arg)
1367 {
1368 struct cnmac_softc *sc = arg;
1369 struct ifnet *ifp;
1370 int s;
1371
1372 s = splnet();
1373
1374 ifp = &sc->sc_ethercom.ec_if;
1375
1376 octgmx_stats(sc->sc_gmx_port);
1377 octpip_stats(sc->sc_pip, ifp, sc->sc_port);
1378 mii_tick(&sc->sc_mii);
1379
1380 splx(s);
1381
1382 callout_schedule(&sc->sc_tick_misc_ch, hz);
1383 }
1384