1 /*
2 * This file is provided under a CDDLv1 license. When using or
3 * redistributing this file, you may do so under this license.
4 * In redistributing this file this license must be included
5 * and no other modification of this header file is permitted.
6 *
7 * CDDL LICENSE SUMMARY
8 *
9 * Copyright(c) 1999 - 2009 Intel Corporation. All rights reserved.
10 *
11 * The contents of this file are subject to the terms of Version
12 * 1.0 of the Common Development and Distribution License (the "License").
13 *
14 * You should have received a copy of the License with this software.
15 * You can obtain a copy of the License at
16 * http://www.opensolaris.org/os/licensing.
17 * See the License for the specific language governing permissions
18 * and limitations under the License.
19 */
20
21 /*
22 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24
25 /*
26 * **********************************************************************
27 * *
28 * Module Name: *
29 * e1000g_rx.c *
30 * *
31 * Abstract: *
32 * This file contains some routines that take care of Receive *
33 * interrupt and also for the received packets it sends up to *
34 * upper layer. *
35 * It tries to do a zero copy if free buffers are available in *
36 * the pool. *
37 * *
38 * **********************************************************************
39 */
40
41 #include "e1000g_sw.h"
42 #include "e1000g_debug.h"
43
44 static p_rx_sw_packet_t e1000g_get_buf(e1000g_rx_data_t *rx_data);
45 #pragma inline(e1000g_get_buf)
46
47 /*
48 * e1000g_rxfree_func - the call-back function to reclaim rx buffer
49 *
50 * This function is called when an mp is freed by the user thru
51 * freeb call (Only for mp constructed through desballoc call)
52 * It returns back the freed buffer to the freelist
53 */
54 void
e1000g_rxfree_func(p_rx_sw_packet_t packet)55 e1000g_rxfree_func(p_rx_sw_packet_t packet)
56 {
57 e1000g_rx_data_t *rx_data;
58 private_devi_list_t *devi_node;
59 struct e1000g *Adapter;
60 uint32_t ring_cnt;
61 uint32_t ref_cnt;
62 unsigned char *address;
63
64 if (packet->ref_cnt == 0) {
65 /*
66 * This case only happens when rx buffers are being freed
67 * in e1000g_stop() and freemsg() is called.
68 */
69 return;
70 }
71
72 rx_data = (e1000g_rx_data_t *)(uintptr_t)packet->rx_data;
73
74 if (packet->mp == NULL) {
75 /*
76 * Allocate a mblk that binds to the data buffer
77 */
78 address = (unsigned char *)packet->rx_buf->address;
79 if (address != NULL) {
80 packet->mp = desballoc((unsigned char *)
81 address, packet->rx_buf->size,
82 BPRI_MED, &packet->free_rtn);
83 }
84 }
85
86 /*
87 * Enqueue the recycled packets in a recycle queue. When freelist
88 * dries up, move the entire chain of packets from recycle queue
89 * to freelist. This helps in avoiding per packet mutex contention
90 * around freelist.
91 */
92 mutex_enter(&rx_data->recycle_lock);
93 QUEUE_PUSH_TAIL(&rx_data->recycle_list, &packet->Link);
94 rx_data->recycle_freepkt++;
95 mutex_exit(&rx_data->recycle_lock);
96
97 ref_cnt = atomic_dec_32_nv(&packet->ref_cnt);
98 if (ref_cnt == 0) {
99 mutex_enter(&e1000g_rx_detach_lock);
100 e1000g_free_rx_sw_packet(packet, B_FALSE);
101
102 atomic_dec_32(&rx_data->pending_count);
103 atomic_dec_32(&e1000g_mblks_pending);
104
105 if ((rx_data->pending_count == 0) &&
106 (rx_data->flag & E1000G_RX_STOPPED)) {
107 devi_node = rx_data->priv_devi_node;
108
109 if (devi_node != NULL) {
110 ring_cnt = atomic_dec_32_nv(
111 &devi_node->pending_rx_count);
112 if ((ring_cnt == 0) &&
113 (devi_node->flag &
114 E1000G_PRIV_DEVI_DETACH)) {
115 e1000g_free_priv_devi_node(
116 devi_node);
117 }
118 } else {
119 Adapter = rx_data->rx_ring->adapter;
120 atomic_dec_32(
121 &Adapter->pending_rx_count);
122 }
123
124 e1000g_free_rx_pending_buffers(rx_data);
125 e1000g_free_rx_data(rx_data);
126 }
127 mutex_exit(&e1000g_rx_detach_lock);
128 }
129 }
130
131 /*
132 * e1000g_rx_setup - setup rx data structures
133 *
134 * This routine initializes all of the receive related
135 * structures. This includes the receive descriptors, the
136 * actual receive buffers, and the rx_sw_packet software
137 * structures.
138 */
139 void
e1000g_rx_setup(struct e1000g * Adapter)140 e1000g_rx_setup(struct e1000g *Adapter)
141 {
142 struct e1000_hw *hw;
143 p_rx_sw_packet_t packet;
144 struct e1000_rx_desc *descriptor;
145 uint32_t buf_low;
146 uint32_t buf_high;
147 uint32_t reg_val;
148 uint32_t rctl;
149 uint32_t rxdctl;
150 uint32_t ert;
151 uint16_t phy_data;
152 int i;
153 int size;
154 e1000g_rx_data_t *rx_data;
155
156 hw = &Adapter->shared;
157 rx_data = Adapter->rx_ring->rx_data;
158
159 /*
160 * zero out all of the receive buffer descriptor memory
161 * assures any previous data or status is erased
162 */
163 bzero(rx_data->rbd_area,
164 sizeof (struct e1000_rx_desc) * Adapter->rx_desc_num);
165
166 if (!Adapter->rx_buffer_setup) {
167 /* Init the list of "Receive Buffer" */
168 QUEUE_INIT_LIST(&rx_data->recv_list);
169
170 /* Init the list of "Free Receive Buffer" */
171 QUEUE_INIT_LIST(&rx_data->free_list);
172
173 /* Init the list of "Free Receive Buffer" */
174 QUEUE_INIT_LIST(&rx_data->recycle_list);
175 /*
176 * Setup Receive list and the Free list. Note that
177 * the both were allocated in one packet area.
178 */
179 packet = rx_data->packet_area;
180 descriptor = rx_data->rbd_first;
181
182 for (i = 0; i < Adapter->rx_desc_num;
183 i++, packet = packet->next, descriptor++) {
184 ASSERT(packet != NULL);
185 ASSERT(descriptor != NULL);
186 descriptor->buffer_addr =
187 packet->rx_buf->dma_address;
188
189 /* Add this rx_sw_packet to the receive list */
190 QUEUE_PUSH_TAIL(&rx_data->recv_list,
191 &packet->Link);
192 }
193
194 for (i = 0; i < Adapter->rx_freelist_num;
195 i++, packet = packet->next) {
196 ASSERT(packet != NULL);
197 /* Add this rx_sw_packet to the free list */
198 QUEUE_PUSH_TAIL(&rx_data->free_list,
199 &packet->Link);
200 }
201 rx_data->avail_freepkt = Adapter->rx_freelist_num;
202 rx_data->recycle_freepkt = 0;
203
204 Adapter->rx_buffer_setup = B_TRUE;
205 } else {
206 /* Setup the initial pointer to the first rx descriptor */
207 packet = (p_rx_sw_packet_t)
208 QUEUE_GET_HEAD(&rx_data->recv_list);
209 descriptor = rx_data->rbd_first;
210
211 for (i = 0; i < Adapter->rx_desc_num; i++) {
212 ASSERT(packet != NULL);
213 ASSERT(descriptor != NULL);
214 descriptor->buffer_addr =
215 packet->rx_buf->dma_address;
216
217 /* Get next rx_sw_packet */
218 packet = (p_rx_sw_packet_t)
219 QUEUE_GET_NEXT(&rx_data->recv_list, &packet->Link);
220 descriptor++;
221 }
222 }
223
224 E1000_WRITE_REG(&Adapter->shared, E1000_RDTR, Adapter->rx_intr_delay);
225 E1000G_DEBUGLOG_1(Adapter, E1000G_INFO_LEVEL,
226 "E1000_RDTR: 0x%x\n", Adapter->rx_intr_delay);
227 if (hw->mac.type >= e1000_82540) {
228 E1000_WRITE_REG(&Adapter->shared, E1000_RADV,
229 Adapter->rx_intr_abs_delay);
230 E1000G_DEBUGLOG_1(Adapter, E1000G_INFO_LEVEL,
231 "E1000_RADV: 0x%x\n", Adapter->rx_intr_abs_delay);
232 }
233
234 /*
235 * Setup our descriptor pointers
236 */
237 rx_data->rbd_next = rx_data->rbd_first;
238
239 size = Adapter->rx_desc_num * sizeof (struct e1000_rx_desc);
240 E1000_WRITE_REG(hw, E1000_RDLEN(0), size);
241 size = E1000_READ_REG(hw, E1000_RDLEN(0));
242
243 /* To get lower order bits */
244 buf_low = (uint32_t)rx_data->rbd_dma_addr;
245 /* To get the higher order bits */
246 buf_high = (uint32_t)(rx_data->rbd_dma_addr >> 32);
247
248 E1000_WRITE_REG(hw, E1000_RDBAH(0), buf_high);
249 E1000_WRITE_REG(hw, E1000_RDBAL(0), buf_low);
250
251 /*
252 * Setup our HW Rx Head & Tail descriptor pointers
253 */
254 E1000_WRITE_REG(hw, E1000_RDT(0),
255 (uint32_t)(rx_data->rbd_last - rx_data->rbd_first));
256 E1000_WRITE_REG(hw, E1000_RDH(0), 0);
257
258 /*
259 * Setup the Receive Control Register (RCTL), and ENABLE the
260 * receiver. The initial configuration is to: Enable the receiver,
261 * accept broadcasts, discard bad packets (and long packets),
262 * disable VLAN filter checking, set the receive descriptor
263 * minimum threshold size to 1/2, and the receive buffer size to
264 * 2k.
265 */
266 rctl = E1000_RCTL_EN | /* Enable Receive Unit */
267 E1000_RCTL_BAM | /* Accept Broadcast Packets */
268 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT) |
269 E1000_RCTL_RDMTS_HALF |
270 E1000_RCTL_LBM_NO; /* Loopback Mode = none */
271
272 if (Adapter->default_mtu > ETHERMTU)
273 rctl |= E1000_RCTL_LPE; /* Large Packet Enable bit */
274
275 if (Adapter->strip_crc)
276 rctl |= E1000_RCTL_SECRC; /* Strip Ethernet CRC */
277
278 if (Adapter->mem_workaround_82546 &&
279 ((hw->mac.type == e1000_82545) ||
280 (hw->mac.type == e1000_82546) ||
281 (hw->mac.type == e1000_82546_rev_3))) {
282 rctl |= E1000_RCTL_SZ_2048;
283 } else {
284 if ((Adapter->max_frame_size > FRAME_SIZE_UPTO_2K) &&
285 (Adapter->max_frame_size <= FRAME_SIZE_UPTO_4K))
286 rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX;
287 else if ((Adapter->max_frame_size > FRAME_SIZE_UPTO_4K) &&
288 (Adapter->max_frame_size <= FRAME_SIZE_UPTO_8K))
289 rctl |= E1000_RCTL_SZ_8192 | E1000_RCTL_BSEX;
290 else if ((Adapter->max_frame_size > FRAME_SIZE_UPTO_8K) &&
291 (Adapter->max_frame_size <= FRAME_SIZE_UPTO_16K))
292 rctl |= E1000_RCTL_SZ_16384 | E1000_RCTL_BSEX;
293 else
294 rctl |= E1000_RCTL_SZ_2048;
295 }
296
297 if (e1000_tbi_sbp_enabled_82543(hw))
298 rctl |= E1000_RCTL_SBP;
299
300 /*
301 * Enable Early Receive Threshold (ERT) on supported devices.
302 * Only takes effect when packet size is equal or larger than the
303 * specified value (in 8 byte units), e.g. using jumbo frames.
304 */
305 if ((hw->mac.type == e1000_82573) ||
306 (hw->mac.type == e1000_82574) ||
307 (hw->mac.type == e1000_ich9lan) ||
308 (hw->mac.type == e1000_ich10lan)) {
309
310 ert = E1000_ERT_2048;
311
312 /*
313 * Special modification when ERT and
314 * jumbo frames are enabled
315 */
316 if (Adapter->default_mtu > ETHERMTU) {
317 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(0));
318 E1000_WRITE_REG(hw, E1000_RXDCTL(0), rxdctl | 0x3);
319 ert |= (1 << 13);
320 }
321
322 E1000_WRITE_REG(hw, E1000_ERT, ert);
323 }
324
325 /* Workaround errata on 82577/8 adapters with large frames */
326 if ((hw->mac.type == e1000_pchlan) &&
327 (Adapter->default_mtu > ETHERMTU)) {
328
329 (void) e1000_read_phy_reg(hw, PHY_REG(770, 26), &phy_data);
330 phy_data &= 0xfff8;
331 phy_data |= (1 << 2);
332 (void) e1000_write_phy_reg(hw, PHY_REG(770, 26), phy_data);
333
334 if (hw->phy.type == e1000_phy_82577) {
335 (void) e1000_read_phy_reg(hw, 22, &phy_data);
336 phy_data &= 0x0fff;
337 phy_data |= (1 << 14);
338 (void) e1000_write_phy_reg(hw, 0x10, 0x2823);
339 (void) e1000_write_phy_reg(hw, 0x11, 0x0003);
340 (void) e1000_write_phy_reg(hw, 22, phy_data);
341 }
342 }
343
344 reg_val =
345 E1000_RXCSUM_TUOFL | /* TCP/UDP checksum offload Enable */
346 E1000_RXCSUM_IPOFL; /* IP checksum offload Enable */
347
348 E1000_WRITE_REG(hw, E1000_RXCSUM, reg_val);
349
350 /*
351 * Workaround: Set bit 16 (IPv6_ExDIS) to disable the
352 * processing of received IPV6 extension headers
353 */
354 if ((hw->mac.type == e1000_82571) || (hw->mac.type == e1000_82572)) {
355 reg_val = E1000_READ_REG(hw, E1000_RFCTL);
356 reg_val |= (E1000_RFCTL_IPV6_EX_DIS |
357 E1000_RFCTL_NEW_IPV6_EXT_DIS);
358 E1000_WRITE_REG(hw, E1000_RFCTL, reg_val);
359 }
360
361 /* Write to enable the receive unit */
362 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
363 }
364
365 /*
366 * e1000g_get_buf - get an rx sw packet from the free_list
367 */
368 static p_rx_sw_packet_t
e1000g_get_buf(e1000g_rx_data_t * rx_data)369 e1000g_get_buf(e1000g_rx_data_t *rx_data)
370 {
371 p_rx_sw_packet_t packet;
372 struct e1000g *Adapter;
373
374 Adapter = rx_data->rx_ring->adapter;
375
376 mutex_enter(&rx_data->freelist_lock);
377 packet = (p_rx_sw_packet_t)
378 QUEUE_POP_HEAD(&rx_data->free_list);
379 if (packet != NULL) {
380 rx_data->avail_freepkt--;
381 goto end;
382 }
383
384 /*
385 * If the freelist has no packets, check the recycle list
386 * to see if there are any available descriptor there.
387 */
388 mutex_enter(&rx_data->recycle_lock);
389 QUEUE_SWITCH(&rx_data->free_list, &rx_data->recycle_list);
390 rx_data->avail_freepkt = rx_data->recycle_freepkt;
391 rx_data->recycle_freepkt = 0;
392 mutex_exit(&rx_data->recycle_lock);
393 packet = (p_rx_sw_packet_t)QUEUE_POP_HEAD(&rx_data->free_list);
394 if (packet != NULL) {
395 rx_data->avail_freepkt--;
396 goto end;
397 }
398
399 if (Adapter->rx_freelist_num < Adapter->rx_freelist_limit) {
400 (void) e1000g_increase_rx_packets(rx_data);
401 packet = (p_rx_sw_packet_t)
402 QUEUE_POP_HEAD(&rx_data->free_list);
403 if (packet != NULL) {
404 rx_data->avail_freepkt--;
405 }
406 }
407
408 end:
409 mutex_exit(&rx_data->freelist_lock);
410 return (packet);
411 }
412
413 /*
414 * e1000g_receive - main receive routine
415 *
416 * This routine will process packets received in an interrupt
417 */
418 mblk_t *
e1000g_receive(e1000g_rx_ring_t * rx_ring,mblk_t ** tail,uint_t sz)419 e1000g_receive(e1000g_rx_ring_t *rx_ring, mblk_t **tail, uint_t sz)
420 {
421 struct e1000_hw *hw;
422 mblk_t *nmp;
423 mblk_t *ret_mp;
424 mblk_t *ret_nmp;
425 struct e1000_rx_desc *current_desc;
426 struct e1000_rx_desc *last_desc;
427 p_rx_sw_packet_t packet;
428 p_rx_sw_packet_t newpkt;
429 uint16_t length;
430 uint32_t pkt_count;
431 uint32_t desc_count;
432 boolean_t accept_frame;
433 boolean_t end_of_packet;
434 boolean_t need_copy;
435 struct e1000g *Adapter;
436 dma_buffer_t *rx_buf;
437 uint16_t cksumflags;
438 uint_t chain_sz = 0;
439 e1000g_rx_data_t *rx_data;
440 uint32_t max_size;
441 uint32_t min_size;
442
443 ret_mp = NULL;
444 ret_nmp = NULL;
445 pkt_count = 0;
446 desc_count = 0;
447 cksumflags = 0;
448
449 Adapter = rx_ring->adapter;
450 rx_data = rx_ring->rx_data;
451 hw = &Adapter->shared;
452
453 /* Sync the Rx descriptor DMA buffers */
454 (void) ddi_dma_sync(rx_data->rbd_dma_handle,
455 0, 0, DDI_DMA_SYNC_FORKERNEL);
456
457 if (e1000g_check_dma_handle(rx_data->rbd_dma_handle) != DDI_FM_OK) {
458 ddi_fm_service_impact(Adapter->dip, DDI_SERVICE_DEGRADED);
459 Adapter->e1000g_state |= E1000G_ERROR;
460 return (NULL);
461 }
462
463 current_desc = rx_data->rbd_next;
464 if (!(current_desc->status & E1000_RXD_STAT_DD)) {
465 /*
466 * don't send anything up. just clear the RFD
467 */
468 E1000G_DEBUG_STAT(rx_ring->stat_none);
469 return (NULL);
470 }
471
472 max_size = Adapter->max_frame_size - ETHERFCSL - VLAN_TAGSZ;
473 min_size = ETHERMIN;
474
475 /*
476 * Loop through the receive descriptors starting at the last known
477 * descriptor owned by the hardware that begins a packet.
478 */
479 while ((current_desc->status & E1000_RXD_STAT_DD) &&
480 (pkt_count < Adapter->rx_limit_onintr) &&
481 ((sz == E1000G_CHAIN_NO_LIMIT) || (chain_sz <= sz))) {
482
483 desc_count++;
484 /*
485 * Now this can happen in Jumbo frame situation.
486 */
487 if (current_desc->status & E1000_RXD_STAT_EOP) {
488 /* packet has EOP set */
489 end_of_packet = B_TRUE;
490 } else {
491 /*
492 * If this received buffer does not have the
493 * End-Of-Packet bit set, the received packet
494 * will consume multiple buffers. We won't send this
495 * packet upstack till we get all the related buffers.
496 */
497 end_of_packet = B_FALSE;
498 }
499
500 /*
501 * Get a pointer to the actual receive buffer
502 * The mp->b_rptr is mapped to The CurrentDescriptor
503 * Buffer Address.
504 */
505 packet =
506 (p_rx_sw_packet_t)QUEUE_POP_HEAD(&rx_data->recv_list);
507 ASSERT(packet != NULL);
508
509 rx_buf = packet->rx_buf;
510
511 length = current_desc->length;
512
513 #ifdef __sparc
514 if (packet->dma_type == USE_DVMA)
515 dvma_sync(rx_buf->dma_handle, 0,
516 DDI_DMA_SYNC_FORKERNEL);
517 else
518 (void) ddi_dma_sync(rx_buf->dma_handle,
519 E1000G_IPALIGNROOM, length,
520 DDI_DMA_SYNC_FORKERNEL);
521 #else
522 (void) ddi_dma_sync(rx_buf->dma_handle,
523 E1000G_IPALIGNROOM, length,
524 DDI_DMA_SYNC_FORKERNEL);
525 #endif
526
527 if (e1000g_check_dma_handle(
528 rx_buf->dma_handle) != DDI_FM_OK) {
529 ddi_fm_service_impact(Adapter->dip,
530 DDI_SERVICE_DEGRADED);
531 Adapter->e1000g_state |= E1000G_ERROR;
532
533 goto rx_drop;
534 }
535
536 accept_frame = (current_desc->errors == 0) ||
537 ((current_desc->errors &
538 (E1000_RXD_ERR_TCPE | E1000_RXD_ERR_IPE)) != 0);
539
540 if (hw->mac.type == e1000_82543) {
541 unsigned char last_byte;
542
543 last_byte =
544 *((unsigned char *)rx_buf->address + length - 1);
545
546 if (TBI_ACCEPT(hw,
547 current_desc->status, current_desc->errors,
548 current_desc->length, last_byte,
549 Adapter->min_frame_size, Adapter->max_frame_size)) {
550
551 e1000_tbi_adjust_stats(Adapter,
552 length, hw->mac.addr);
553
554 length--;
555 accept_frame = B_TRUE;
556 } else if (e1000_tbi_sbp_enabled_82543(hw) &&
557 (current_desc->errors == E1000_RXD_ERR_CE)) {
558 accept_frame = B_TRUE;
559 }
560 }
561
562 /*
563 * Indicate the packet to the NOS if it was good.
564 * Normally, hardware will discard bad packets for us.
565 * Check for the packet to be a valid Ethernet packet
566 */
567 if (!accept_frame) {
568 /*
569 * error in incoming packet, either the packet is not a
570 * ethernet size packet, or the packet has an error. In
571 * either case, the packet will simply be discarded.
572 */
573 E1000G_DEBUGLOG_0(Adapter, E1000G_INFO_LEVEL,
574 "Process Receive Interrupts: Error in Packet\n");
575
576 E1000G_STAT(rx_ring->stat_error);
577 /*
578 * Returning here as we are done here. There is
579 * no point in waiting for while loop to elapse
580 * and the things which were done. More efficient
581 * and less error prone...
582 */
583 goto rx_drop;
584 }
585
586 /*
587 * If the Ethernet CRC is not stripped by the hardware,
588 * we need to strip it before sending it up to the stack.
589 */
590 if (end_of_packet && !Adapter->strip_crc) {
591 if (length > ETHERFCSL) {
592 length -= ETHERFCSL;
593 } else {
594 /*
595 * If the fragment is smaller than the CRC,
596 * drop this fragment, do the processing of
597 * the end of the packet.
598 */
599 if (rx_data->rx_mblk_tail == NULL) {
600 E1000G_STAT(rx_ring->stat_crc_only_pkt);
601 goto rx_next_desc;
602 }
603
604 rx_data->rx_mblk_tail->b_wptr -=
605 ETHERFCSL - length;
606 rx_data->rx_mblk_len -=
607 ETHERFCSL - length;
608 goto rx_end_of_packet;
609 }
610 }
611
612 need_copy = B_TRUE;
613
614 if (length <= Adapter->rx_bcopy_thresh)
615 goto rx_copy;
616
617 /*
618 * Get the pre-constructed mblk that was associated
619 * to the receive data buffer.
620 */
621 if (packet->mp == NULL) {
622 packet->mp = desballoc((unsigned char *)
623 rx_buf->address, length,
624 BPRI_MED, &packet->free_rtn);
625 }
626
627 if (packet->mp != NULL) {
628 /*
629 * We have two sets of buffer pool. One associated with
630 * the Rxdescriptors and other a freelist buffer pool.
631 * Each time we get a good packet, Try to get a buffer
632 * from the freelist pool using e1000g_get_buf. If we
633 * get free buffer, then replace the descriptor buffer
634 * address with the free buffer we just got, and pass
635 * the pre-constructed mblk upstack. (note no copying)
636 *
637 * If we failed to get a free buffer, then try to
638 * allocate a new buffer(mp) and copy the recv buffer
639 * content to our newly allocated buffer(mp). Don't
640 * disturb the desriptor buffer address. (note copying)
641 */
642 newpkt = e1000g_get_buf(rx_data);
643
644 if (newpkt != NULL) {
645 /*
646 * Get the mblk associated to the data,
647 * and strip it off the sw packet.
648 */
649 nmp = packet->mp;
650 packet->mp = NULL;
651 atomic_inc_32(&packet->ref_cnt);
652
653 /*
654 * Now replace old buffer with the new
655 * one we got from free list
656 * Both the RxSwPacket as well as the
657 * Receive Buffer Descriptor will now
658 * point to this new packet.
659 */
660 packet = newpkt;
661
662 current_desc->buffer_addr =
663 newpkt->rx_buf->dma_address;
664
665 need_copy = B_FALSE;
666 } else {
667 /* EMPTY */
668 E1000G_DEBUG_STAT(rx_ring->stat_no_freepkt);
669 }
670 }
671
672 rx_copy:
673 if (need_copy) {
674 /*
675 * No buffers available on free list,
676 * bcopy the data from the buffer and
677 * keep the original buffer. Dont want to
678 * do this.. Yack but no other way
679 */
680 if ((nmp = allocb(length + E1000G_IPALIGNROOM,
681 BPRI_MED)) == NULL) {
682 /*
683 * The system has no buffers available
684 * to send up the incoming packet, hence
685 * the packet will have to be processed
686 * when there're more buffers available.
687 */
688 E1000G_STAT(rx_ring->stat_allocb_fail);
689 goto rx_drop;
690 }
691 nmp->b_rptr += E1000G_IPALIGNROOM;
692 nmp->b_wptr += E1000G_IPALIGNROOM;
693 /*
694 * The free list did not have any buffers
695 * available, so, the received packet will
696 * have to be copied into a mp and the original
697 * buffer will have to be retained for future
698 * packet reception.
699 */
700 bcopy(rx_buf->address, nmp->b_wptr, length);
701 }
702
703 ASSERT(nmp != NULL);
704 nmp->b_wptr += length;
705
706 if (rx_data->rx_mblk == NULL) {
707 /*
708 * TCP/UDP checksum offload and
709 * IP checksum offload
710 */
711 if (!(current_desc->status & E1000_RXD_STAT_IXSM)) {
712 /*
713 * Check TCP/UDP checksum
714 */
715 if ((current_desc->status &
716 E1000_RXD_STAT_TCPCS) &&
717 !(current_desc->errors &
718 E1000_RXD_ERR_TCPE))
719 cksumflags |= HCK_FULLCKSUM_OK;
720 /*
721 * Check IP Checksum
722 */
723 if ((current_desc->status &
724 E1000_RXD_STAT_IPCS) &&
725 !(current_desc->errors &
726 E1000_RXD_ERR_IPE))
727 cksumflags |= HCK_IPV4_HDRCKSUM_OK;
728 }
729 }
730
731 /*
732 * We need to maintain our packet chain in the global
733 * Adapter structure, for the Rx processing can end
734 * with a fragment that has no EOP set.
735 */
736 if (rx_data->rx_mblk == NULL) {
737 /* Get the head of the message chain */
738 rx_data->rx_mblk = nmp;
739 rx_data->rx_mblk_tail = nmp;
740 rx_data->rx_mblk_len = length;
741 } else { /* Not the first packet */
742 /* Continue adding buffers */
743 rx_data->rx_mblk_tail->b_cont = nmp;
744 rx_data->rx_mblk_tail = nmp;
745 rx_data->rx_mblk_len += length;
746 }
747 ASSERT(rx_data->rx_mblk != NULL);
748 ASSERT(rx_data->rx_mblk_tail != NULL);
749 ASSERT(rx_data->rx_mblk_tail->b_cont == NULL);
750
751 /*
752 * Now this MP is ready to travel upwards but some more
753 * fragments are coming.
754 * We will send packet upwards as soon as we get EOP
755 * set on the packet.
756 */
757 if (!end_of_packet) {
758 /*
759 * continue to get the next descriptor,
760 * Tail would be advanced at the end
761 */
762 goto rx_next_desc;
763 }
764
765 rx_end_of_packet:
766 if (E1000G_IS_VLAN_PACKET(rx_data->rx_mblk->b_rptr))
767 max_size = Adapter->max_frame_size - ETHERFCSL;
768
769 if ((rx_data->rx_mblk_len > max_size) ||
770 (rx_data->rx_mblk_len < min_size)) {
771 E1000G_STAT(rx_ring->stat_size_error);
772 goto rx_drop;
773 }
774
775 /*
776 * Found packet with EOP
777 * Process the last fragment.
778 */
779 if (cksumflags != 0) {
780 mac_hcksum_set(rx_data->rx_mblk,
781 0, 0, 0, 0, cksumflags);
782 cksumflags = 0;
783 }
784
785 /*
786 * Count packets that span multi-descriptors
787 */
788 E1000G_DEBUG_STAT_COND(rx_ring->stat_multi_desc,
789 (rx_data->rx_mblk->b_cont != NULL));
790
791 /*
792 * Append to list to send upstream
793 */
794 if (ret_mp == NULL) {
795 ret_mp = ret_nmp = rx_data->rx_mblk;
796 } else {
797 ret_nmp->b_next = rx_data->rx_mblk;
798 ret_nmp = rx_data->rx_mblk;
799 }
800 ret_nmp->b_next = NULL;
801 *tail = ret_nmp;
802 chain_sz += length;
803
804 rx_data->rx_mblk = NULL;
805 rx_data->rx_mblk_tail = NULL;
806 rx_data->rx_mblk_len = 0;
807
808 pkt_count++;
809
810 rx_next_desc:
811 /*
812 * Zero out the receive descriptors status
813 */
814 current_desc->status = 0;
815
816 if (current_desc == rx_data->rbd_last)
817 rx_data->rbd_next = rx_data->rbd_first;
818 else
819 rx_data->rbd_next++;
820
821 last_desc = current_desc;
822 current_desc = rx_data->rbd_next;
823
824 /*
825 * Put the buffer that we just indicated back
826 * at the end of our list
827 */
828 QUEUE_PUSH_TAIL(&rx_data->recv_list,
829 &packet->Link);
830 } /* while loop */
831
832 /* Sync the Rx descriptor DMA buffers */
833 (void) ddi_dma_sync(rx_data->rbd_dma_handle,
834 0, 0, DDI_DMA_SYNC_FORDEV);
835
836 /*
837 * Advance the E1000's Receive Queue #0 "Tail Pointer".
838 */
839 E1000_WRITE_REG(hw, E1000_RDT(0),
840 (uint32_t)(last_desc - rx_data->rbd_first));
841
842 if (e1000g_check_acc_handle(Adapter->osdep.reg_handle) != DDI_FM_OK) {
843 ddi_fm_service_impact(Adapter->dip, DDI_SERVICE_DEGRADED);
844 Adapter->e1000g_state |= E1000G_ERROR;
845 }
846
847 Adapter->rx_pkt_cnt = pkt_count;
848
849 return (ret_mp);
850
851 rx_drop:
852 /*
853 * Zero out the receive descriptors status
854 */
855 current_desc->status = 0;
856
857 /* Sync the Rx descriptor DMA buffers */
858 (void) ddi_dma_sync(rx_data->rbd_dma_handle,
859 0, 0, DDI_DMA_SYNC_FORDEV);
860
861 if (current_desc == rx_data->rbd_last)
862 rx_data->rbd_next = rx_data->rbd_first;
863 else
864 rx_data->rbd_next++;
865
866 last_desc = current_desc;
867
868 QUEUE_PUSH_TAIL(&rx_data->recv_list, &packet->Link);
869 /*
870 * Reclaim all old buffers already allocated during
871 * Jumbo receives.....for incomplete reception
872 */
873 if (rx_data->rx_mblk != NULL) {
874 freemsg(rx_data->rx_mblk);
875 rx_data->rx_mblk = NULL;
876 rx_data->rx_mblk_tail = NULL;
877 rx_data->rx_mblk_len = 0;
878 }
879 /*
880 * Advance the E1000's Receive Queue #0 "Tail Pointer".
881 */
882 E1000_WRITE_REG(hw, E1000_RDT(0),
883 (uint32_t)(last_desc - rx_data->rbd_first));
884
885 if (e1000g_check_acc_handle(Adapter->osdep.reg_handle) != DDI_FM_OK) {
886 ddi_fm_service_impact(Adapter->dip, DDI_SERVICE_DEGRADED);
887 Adapter->e1000g_state |= E1000G_ERROR;
888 }
889
890 return (ret_mp);
891 }
892