Home | History | Annotate | Download | only in nge
      1  5578  mx205022 /*
      2  5578  mx205022  * CDDL HEADER START
      3  5578  mx205022  *
      4  5578  mx205022  * The contents of this file are subject to the terms of the
      5  5578  mx205022  * Common Development and Distribution License (the "License").
      6  5578  mx205022  * You may not use this file except in compliance with the License.
      7  5578  mx205022  *
      8  5578  mx205022  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
      9  5578  mx205022  * or http://www.opensolaris.org/os/licensing.
     10  5578  mx205022  * See the License for the specific language governing permissions
     11  5578  mx205022  * and limitations under the License.
     12  5578  mx205022  *
     13  5578  mx205022  * When distributing Covered Code, include this CDDL HEADER in each
     14  5578  mx205022  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
     15  5578  mx205022  * If applicable, add the following below this CDDL HEADER, with the
     16  5578  mx205022  * fields enclosed by brackets "[]" replaced with your own identifying
     17  5578  mx205022  * information: Portions Copyright [yyyy] [name of copyright owner]
     18  5578  mx205022  *
     19  5578  mx205022  * CDDL HEADER END
     20  5578  mx205022  */
     21  5578  mx205022 
     22  5574  mx205022 /*
     23  8848      Zhen  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
     24  5574  mx205022  * Use is subject to license terms.
     25  5574  mx205022  */
     26  5574  mx205022 
     27  5574  mx205022 #include "nge.h"
     28  5574  mx205022 
     29  5574  mx205022 #undef	NGE_DBG
     30  5574  mx205022 #define	NGE_DBG		NGE_DBG_RECV
     31  5574  mx205022 
     32  5574  mx205022 #define	RXD_END		0x20000000
     33  5574  mx205022 #define	RXD_ERR		0x40000000
     34  5574  mx205022 #define	RXD_OWN		0x80000000
     35  5574  mx205022 #define	RXD_CSUM_MSK	0x1C000000
     36  5574  mx205022 #define	RXD_BCNT_MSK	0x00003FFF
     37  5574  mx205022 
     38  5574  mx205022 #define	RXD_CK8G_NO_HSUM	0x0
     39  5574  mx205022 #define	RXD_CK8G_TCP_SUM_ERR	0x04000000
     40  5574  mx205022 #define	RXD_CK8G_UDP_SUM_ERR	0x08000000
     41  5574  mx205022 #define	RXD_CK8G_IP_HSUM_ERR	0x0C000000
     42  5574  mx205022 #define	RXD_CK8G_IP_HSUM	0x10000000
     43  5574  mx205022 #define	RXD_CK8G_TCP_SUM	0x14000000
     44  5574  mx205022 #define	RXD_CK8G_UDP_SUM	0x18000000
     45  5574  mx205022 #define	RXD_CK8G_RESV		0x1C000000
     46  5574  mx205022 
     47  5574  mx205022 extern ddi_device_acc_attr_t nge_data_accattr;
     48  5574  mx205022 
     49  5574  mx205022 /*
     50  5988  vb160487  * Callback code invoked from STREAMs when the recv data buffer is free for
     51  5988  vb160487  * recycling.
     52  5988  vb160487  *
     53  5988  vb160487  * The following table describes function behaviour:
     54  5988  vb160487  *
     55  5988  vb160487  *                      | mac stopped | mac running
     56  5988  vb160487  * ---------------------------------------------------
     57  5988  vb160487  * buffer delivered     | free buffer | recycle buffer
     58  5988  vb160487  * buffer not delivered | do nothing  | recycle buffer (*)
     59  5988  vb160487  *
     60  5988  vb160487  * Note (*):
     61  5988  vb160487  *   Recycle buffer only if mac state did not change during execution of
     62  5988  vb160487  *   function. Otherwise if mac state changed, set buffer delivered & re-enter
     63  5988  vb160487  *   function by calling freemsg().
     64  5574  mx205022  */
     65  5574  mx205022 
     66  5574  mx205022 void
     67  5574  mx205022 nge_recv_recycle(caddr_t arg)
     68  5574  mx205022 {
     69  5574  mx205022 	boolean_t val;
     70  5574  mx205022 	boolean_t valid;
     71  5574  mx205022 	nge_t *ngep;
     72  5574  mx205022 	dma_area_t *bufp;
     73  5574  mx205022 	buff_ring_t *brp;
     74  5574  mx205022 	nge_sw_statistics_t *sw_stp;
     75  5574  mx205022 
     76  5574  mx205022 	bufp = (dma_area_t *)arg;
     77  5574  mx205022 	ngep = (nge_t *)bufp->private;
     78  5574  mx205022 	brp = ngep->buff;
     79  5574  mx205022 	sw_stp = &ngep->statistics.sw_statistics;
     80  5574  mx205022 
     81  5574  mx205022 	/*
     82  5574  mx205022 	 * Free the buffer directly if the buffer was allocated
     83  5574  mx205022 	 * previously or mac was stopped.
     84  5574  mx205022 	 */
     85  5574  mx205022 	if (bufp->signature != brp->buf_sign) {
     86  5574  mx205022 		if (bufp->rx_delivered == B_TRUE) {
     87  5574  mx205022 			nge_free_dma_mem(bufp);
     88  5574  mx205022 			kmem_free(bufp, sizeof (dma_area_t));
     89  5574  mx205022 			val = nge_atomic_decrease(&brp->rx_hold, 1);
     90  5574  mx205022 			ASSERT(val == B_TRUE);
     91  5574  mx205022 		}
     92  5574  mx205022 		return;
     93  5574  mx205022 	}
     94  5574  mx205022 
     95  5574  mx205022 	/*
     96  5574  mx205022 	 * recycle the data buffer again and fill them in free ring
     97  5574  mx205022 	 */
     98  5574  mx205022 	bufp->rx_recycle.free_func = nge_recv_recycle;
     99  5574  mx205022 	bufp->rx_recycle.free_arg = (caddr_t)bufp;
    100  5574  mx205022 
    101  5574  mx205022 	bufp->mp = desballoc(DMA_VPTR(*bufp),
    102  5574  mx205022 	    ngep->buf_size + NGE_HEADROOM, 0, &bufp->rx_recycle);
    103  5574  mx205022 
    104  5574  mx205022 	if (bufp->mp == NULL) {
    105  5574  mx205022 		sw_stp->mp_alloc_err++;
    106  5574  mx205022 		sw_stp->recy_free++;
    107  5574  mx205022 		nge_free_dma_mem(bufp);
    108  5574  mx205022 		kmem_free(bufp, sizeof (dma_area_t));
    109  5574  mx205022 		val = nge_atomic_decrease(&brp->rx_hold, 1);
    110  5574  mx205022 		ASSERT(val == B_TRUE);
    111  5574  mx205022 	} else {
    112  5574  mx205022 
    113  5574  mx205022 		mutex_enter(brp->recycle_lock);
    114  5574  mx205022 		if (bufp->signature != brp->buf_sign)
    115  5574  mx205022 			valid = B_TRUE;
    116  5574  mx205022 		else
    117  5574  mx205022 			valid = B_FALSE;
    118  5574  mx205022 		bufp->rx_delivered = valid;
    119  5574  mx205022 		if (bufp->rx_delivered == B_FALSE)  {
    120  5574  mx205022 			bufp->next = brp->recycle_list;
    121  5574  mx205022 			brp->recycle_list = bufp;
    122  5574  mx205022 		}
    123  5574  mx205022 		mutex_exit(brp->recycle_lock);
    124  5574  mx205022 		if (valid == B_TRUE)
    125  5574  mx205022 			/* call nge_rx_recycle again to free it */
    126  5574  mx205022 			freemsg(bufp->mp);
    127  5574  mx205022 		else {
    128  5574  mx205022 			val = nge_atomic_decrease(&brp->rx_hold, 1);
    129  5574  mx205022 			ASSERT(val == B_TRUE);
    130  5574  mx205022 		}
    131  5574  mx205022 	}
    132  5574  mx205022 }
    133  5574  mx205022 
    134  5574  mx205022 /*
    135  5574  mx205022  * Checking the rx's BDs (one or more) to receive
    136  5574  mx205022  * one complete packet.
    137  5574  mx205022  * start_index: the start indexer of BDs for one packet.
    138  5574  mx205022  * end_index: the end indexer of BDs for one packet.
    139  5574  mx205022  */
    140  5574  mx205022 static mblk_t *nge_recv_packet(nge_t *ngep, uint32_t start_index, size_t len);
    141  5574  mx205022 #pragma	inline(nge_recv_packet)
    142  5574  mx205022 
    143  5574  mx205022 static mblk_t *
    144  5574  mx205022 nge_recv_packet(nge_t *ngep, uint32_t start_index, size_t len)
    145  5574  mx205022 {
    146  5574  mx205022 	uint8_t *rptr;
    147  5574  mx205022 	uint32_t minsize;
    148  5574  mx205022 	uint32_t maxsize;
    149  5574  mx205022 	mblk_t *mp;
    150  5574  mx205022 	buff_ring_t *brp;
    151  5574  mx205022 	sw_rx_sbd_t *srbdp;
    152  5574  mx205022 	dma_area_t *bufp;
    153  5574  mx205022 	nge_sw_statistics_t *sw_stp;
    154  5574  mx205022 	void *hw_bd_p;
    155  5574  mx205022 
    156  5574  mx205022 	brp = ngep->buff;
    157  5574  mx205022 	minsize = ETHERMIN;
    158  5574  mx205022 	maxsize = ngep->max_sdu;
    159  5574  mx205022 	sw_stp = &ngep->statistics.sw_statistics;
    160  5574  mx205022 	mp = NULL;
    161  5574  mx205022 
    162  5574  mx205022 	srbdp = &brp->sw_rbds[start_index];
    163  5574  mx205022 	DMA_SYNC(*srbdp->bufp, DDI_DMA_SYNC_FORKERNEL);
    164  5574  mx205022 	hw_bd_p = DMA_VPTR(srbdp->desc);
    165  5574  mx205022 
    166  5574  mx205022 	/*
    167  5574  mx205022 	 * First check the free_list, if it is NULL,
    168  5574  mx205022 	 * make the recycle_list be free_list.
    169  5574  mx205022 	 */
    170  5574  mx205022 	if (brp->free_list == NULL) {
    171  5574  mx205022 		mutex_enter(brp->recycle_lock);
    172  5574  mx205022 		brp->free_list = brp->recycle_list;
    173  5574  mx205022 		brp->recycle_list = NULL;
    174  5574  mx205022 		mutex_exit(brp->recycle_lock);
    175  5574  mx205022 	}
    176  5574  mx205022 	bufp = brp->free_list;
    177  5574  mx205022 	/* If it's not a qualified packet, delete it */
    178  5574  mx205022 	if (len > maxsize || len < minsize) {
    179  5574  mx205022 		ngep->desc_attr.rxd_fill(hw_bd_p, &srbdp->bufp->cookie,
    180  5574  mx205022 		    srbdp->bufp->alength);
    181  5574  mx205022 		srbdp->flags = CONTROLER_OWN;
    182  5574  mx205022 		return (NULL);
    183  5574  mx205022 	}
    184  5574  mx205022 
    185  5574  mx205022 	/*
    186  5574  mx205022 	 * If receive packet size is smaller than RX bcopy threshold,
    187  5574  mx205022 	 * or there is no available buffer in free_list or recycle list,
    188  5574  mx205022 	 * we use bcopy directly.
    189  5574  mx205022 	 */
    190  5574  mx205022 	if (len <= ngep->param_rxbcopy_threshold || bufp == NULL)
    191  5574  mx205022 		brp->rx_bcopy = B_TRUE;
    192  5574  mx205022 	else
    193  5574  mx205022 		brp->rx_bcopy = B_FALSE;
    194  5574  mx205022 
    195  5574  mx205022 	if (brp->rx_bcopy) {
    196  5574  mx205022 		mp = allocb(len + NGE_HEADROOM, 0);
    197  5574  mx205022 		if (mp == NULL) {
    198  5574  mx205022 			sw_stp->mp_alloc_err++;
    199  5574  mx205022 			ngep->desc_attr.rxd_fill(hw_bd_p, &srbdp->bufp->cookie,
    200  5574  mx205022 			    srbdp->bufp->alength);
    201  5574  mx205022 			srbdp->flags = CONTROLER_OWN;
    202  5574  mx205022 			return (NULL);
    203  5574  mx205022 		}
    204  5574  mx205022 		rptr = DMA_VPTR(*srbdp->bufp);
    205  5574  mx205022 		mp->b_rptr = mp->b_rptr + NGE_HEADROOM;
    206  5574  mx205022 		bcopy(rptr + NGE_HEADROOM, mp->b_rptr, len);
    207  5574  mx205022 		mp->b_wptr = mp->b_rptr + len;
    208  5574  mx205022 	} else {
    209  5574  mx205022 		mp = srbdp->bufp->mp;
    210  5574  mx205022 		/*
    211  5574  mx205022 		 * Make sure the packet *contents* 4-byte aligned
    212  5574  mx205022 		 */
    213  5574  mx205022 		mp->b_rptr += NGE_HEADROOM;
    214  5574  mx205022 		mp->b_wptr = mp->b_rptr + len;
    215  5574  mx205022 		mp->b_next = mp->b_cont = NULL;
    216  5574  mx205022 		srbdp->bufp->rx_delivered = B_TRUE;
    217  5574  mx205022 		srbdp->bufp = NULL;
    218  5574  mx205022 		nge_atomic_increase(&brp->rx_hold, 1);
    219  5574  mx205022 
    220  5574  mx205022 		/* Fill the buffer from free_list */
    221  5574  mx205022 		srbdp->bufp = bufp;
    222  5574  mx205022 		brp->free_list = bufp->next;
    223  5574  mx205022 		bufp->next = NULL;
    224  5574  mx205022 	}
    225  5574  mx205022 
    226  5574  mx205022 	/* replenish the buffer for hardware descriptor */
    227  5574  mx205022 	ngep->desc_attr.rxd_fill(hw_bd_p, &srbdp->bufp->cookie,
    228  5574  mx205022 	    srbdp->bufp->alength);
    229  5574  mx205022 	srbdp->flags = CONTROLER_OWN;
    230  5574  mx205022 	sw_stp->rbytes += len;
    231  5574  mx205022 	sw_stp->recv_count++;
    232  5574  mx205022 
    233  5574  mx205022 	return (mp);
    234  5574  mx205022 }
    235  5574  mx205022 
    236  5574  mx205022 
    237  5574  mx205022 #define	RX_HW_ERR	0x01
    238  5574  mx205022 #define	RX_SUM_NO	0x02
    239  5574  mx205022 #define	RX_SUM_ERR	0x04
    240  5574  mx205022 
    241  5574  mx205022 /*
    242  5574  mx205022  * Statistic the rx's error
    243  5574  mx205022  * and generate a log msg for these.
    244  5574  mx205022  * Note:
    245  5574  mx205022  * RXE, Parity Error, Symbo error, CRC error
    246  5574  mx205022  * have been recored by nvidia's  hardware
    247  5574  mx205022  * statistics part (nge_statistics). So it is uncessary to record them by
    248  5574  mx205022  * driver in this place.
    249  5574  mx205022  */
    250  5574  mx205022 static uint32_t
    251  5574  mx205022 nge_rxsta_handle(nge_t *ngep, uint32_t stflag, uint32_t *pflags);
    252  5574  mx205022 #pragma	inline(nge_rxsta_handle)
    253  5574  mx205022 
    254  5574  mx205022 static uint32_t
    255  5574  mx205022 nge_rxsta_handle(nge_t *ngep,  uint32_t stflag, uint32_t *pflags)
    256  5574  mx205022 {
    257  5574  mx205022 	uint32_t errors;
    258  5574  mx205022 	uint32_t err_flag;
    259  5574  mx205022 	nge_sw_statistics_t *sw_stp;
    260  5574  mx205022 
    261  5574  mx205022 	err_flag = 0;
    262  5574  mx205022 	sw_stp = &ngep->statistics.sw_statistics;
    263  5574  mx205022 
    264  5574  mx205022 	if ((RXD_END & stflag) == 0)
    265  5574  mx205022 		return (RX_HW_ERR);
    266  5574  mx205022 
    267  5574  mx205022 	errors = stflag & RXD_CSUM_MSK;
    268  5574  mx205022 	switch (errors) {
    269  5574  mx205022 	default:
    270  5574  mx205022 	break;
    271  5574  mx205022 
    272  5574  mx205022 	case RXD_CK8G_TCP_SUM:
    273  5574  mx205022 	case RXD_CK8G_UDP_SUM:
    274  5574  mx205022 		*pflags |= HCK_FULLCKSUM;
    275  5574  mx205022 		*pflags |= HCK_IPV4_HDRCKSUM;
    276  5574  mx205022 		*pflags |= HCK_FULLCKSUM_OK;
    277  5574  mx205022 		break;
    278  5574  mx205022 
    279  5574  mx205022 	case RXD_CK8G_TCP_SUM_ERR:
    280  5574  mx205022 	case RXD_CK8G_UDP_SUM_ERR:
    281  5574  mx205022 		sw_stp->tcp_hwsum_err++;
    282  5574  mx205022 		*pflags |= HCK_IPV4_HDRCKSUM;
    283  5574  mx205022 		break;
    284  5574  mx205022 
    285  5574  mx205022 	case RXD_CK8G_IP_HSUM:
    286  5574  mx205022 		*pflags |= HCK_IPV4_HDRCKSUM;
    287  5574  mx205022 		break;
    288  5574  mx205022 
    289  5574  mx205022 	case RXD_CK8G_NO_HSUM:
    290  5574  mx205022 		err_flag |= RX_SUM_NO;
    291  5574  mx205022 		break;
    292  5574  mx205022 
    293  5574  mx205022 	case RXD_CK8G_IP_HSUM_ERR:
    294  5574  mx205022 		sw_stp->ip_hwsum_err++;
    295  5574  mx205022 		err_flag |=  RX_SUM_ERR;
    296  5574  mx205022 		break;
    297  5574  mx205022 	}
    298  5574  mx205022 
    299  5574  mx205022 	if ((stflag & RXD_ERR) != 0)	{
    300  5574  mx205022 
    301  5574  mx205022 		err_flag |= RX_HW_ERR;
    302  5574  mx205022 		NGE_DEBUG(("Receive desc error, status: 0x%x", stflag));
    303  5574  mx205022 	}
    304  5574  mx205022 
    305  5574  mx205022 	return (err_flag);
    306  5574  mx205022 }
    307  5574  mx205022 
    308  5574  mx205022 static mblk_t *
    309  5574  mx205022 nge_recv_ring(nge_t *ngep)
    310  5574  mx205022 {
    311  5574  mx205022 	uint32_t stflag;
    312  5574  mx205022 	uint32_t flag_err;
    313  5574  mx205022 	uint32_t sum_flags;
    314  5574  mx205022 	size_t len;
    315  5574  mx205022 	uint64_t end_index;
    316  5574  mx205022 	uint64_t sync_start;
    317  5574  mx205022 	mblk_t *mp;
    318  5574  mx205022 	mblk_t **tail;
    319  5574  mx205022 	mblk_t *head;
    320  5574  mx205022 	recv_ring_t *rrp;
    321  5574  mx205022 	buff_ring_t *brp;
    322  5574  mx205022 	sw_rx_sbd_t *srbdp;
    323  5574  mx205022 	void * hw_bd_p;
    324  5574  mx205022 	nge_mode_cntl mode_cntl;
    325  5574  mx205022 
    326  5574  mx205022 	mp = NULL;
    327  5574  mx205022 	head = NULL;
    328  5574  mx205022 	tail = &head;
    329  5574  mx205022 	rrp = ngep->recv;
    330  5574  mx205022 	brp = ngep->buff;
    331  5574  mx205022 
    332  5574  mx205022 	end_index = sync_start = rrp->prod_index;
    333  5574  mx205022 	/* Sync the descriptor for kernel */
    334  5574  mx205022 	if (sync_start + ngep->param_recv_max_packet <= ngep->rx_desc) {
    335  5574  mx205022 		(void) ddi_dma_sync(rrp->desc.dma_hdl,
    336  5574  mx205022 		    sync_start * ngep->desc_attr.rxd_size,
    337  5574  mx205022 		    ngep->param_recv_max_packet * ngep->desc_attr.rxd_size,
    338  5574  mx205022 		    DDI_DMA_SYNC_FORKERNEL);
    339  5574  mx205022 	} else {
    340  5574  mx205022 		(void) ddi_dma_sync(rrp->desc.dma_hdl,
    341  5574  mx205022 		    sync_start * ngep->desc_attr.rxd_size,
    342  5574  mx205022 		    0,
    343  5574  mx205022 		    DDI_DMA_SYNC_FORKERNEL);
    344  5574  mx205022 		(void) ddi_dma_sync(rrp->desc.dma_hdl,
    345  5574  mx205022 		    0,
    346  5574  mx205022 		    (ngep->param_recv_max_packet + sync_start - ngep->rx_desc) *
    347  5574  mx205022 		    ngep->desc_attr.rxd_size,
    348  5574  mx205022 		    DDI_DMA_SYNC_FORKERNEL);
    349  5574  mx205022 	}
    350  5574  mx205022 
    351  5574  mx205022 	/*
    352  5574  mx205022 	 * Looking through the rx's ring to find the good packets
    353  5574  mx205022 	 * and try to receive more and more packets in rx's ring
    354  5574  mx205022 	 */
    355  5574  mx205022 	for (;;) {
    356  5574  mx205022 		sum_flags = 0;
    357  5574  mx205022 		flag_err = 0;
    358  5574  mx205022 		end_index = rrp->prod_index;
    359  5574  mx205022 		srbdp = &brp->sw_rbds[end_index];
    360  5574  mx205022 		hw_bd_p = DMA_VPTR(srbdp->desc);
    361  5574  mx205022 		stflag = ngep->desc_attr.rxd_check(hw_bd_p, &len);
    362  5574  mx205022 		/*
    363  5574  mx205022 		 * If there is no packet in receving ring
    364  5574  mx205022 		 * break the loop
    365  5574  mx205022 		 */
    366  5574  mx205022 		if ((stflag & RXD_OWN) != 0 || HOST_OWN == srbdp->flags)
    367  5574  mx205022 			break;
    368  5574  mx205022 
    369  5574  mx205022 		ngep->recv_count++;
    370  5574  mx205022 		flag_err = nge_rxsta_handle(ngep, stflag, &sum_flags);
    371  5574  mx205022 		if ((flag_err & RX_HW_ERR) == 0) {
    372  5574  mx205022 			srbdp->flags = NGE_END_PACKET;
    373  5574  mx205022 			mp = nge_recv_packet(ngep, end_index, len);
    374  5574  mx205022 		} else {
    375  5574  mx205022 			/* Hardware error, re-use the buffer */
    376  5574  mx205022 			ngep->desc_attr.rxd_fill(hw_bd_p, &srbdp->bufp->cookie,
    377  5574  mx205022 			    srbdp->bufp->alength);
    378  5574  mx205022 			srbdp->flags = CONTROLER_OWN;
    379  5574  mx205022 		}
    380  5574  mx205022 		if (mp != NULL) {
    381  5574  mx205022 			if (!(flag_err & (RX_SUM_NO | RX_SUM_ERR))) {
    382  5574  mx205022 				(void) hcksum_assoc(mp, NULL, NULL,
    383  5574  mx205022 				    0, 0, 0, 0, sum_flags, 0);
    384  5574  mx205022 			}
    385  5574  mx205022 			*tail = mp;
    386  5574  mx205022 			tail = &mp->b_next;
    387  5574  mx205022 			mp = NULL;
    388  5574  mx205022 		}
    389  5574  mx205022 		rrp->prod_index = NEXT(end_index, rrp->desc.nslots);
    390  8848      Zhen 		if (ngep->recv_count >= ngep->param_recv_max_packet)
    391  5574  mx205022 			break;
    392  5574  mx205022 	}
    393  5574  mx205022 
    394  5574  mx205022 	/* Sync the descriptors for device */
    395  5659  jj146644 	if (sync_start + ngep->recv_count <= ngep->rx_desc) {
    396  5574  mx205022 		(void) ddi_dma_sync(rrp->desc.dma_hdl,
    397  5574  mx205022 		    sync_start * ngep->desc_attr.rxd_size,
    398  5659  jj146644 		    ngep->recv_count * ngep->desc_attr.rxd_size,
    399  5574  mx205022 		    DDI_DMA_SYNC_FORDEV);
    400  5574  mx205022 	} else {
    401  5574  mx205022 		(void) ddi_dma_sync(rrp->desc.dma_hdl,
    402  5574  mx205022 		    sync_start * ngep->desc_attr.rxd_size,
    403  5574  mx205022 		    0,
    404  5574  mx205022 		    DDI_DMA_SYNC_FORDEV);
    405  5574  mx205022 		(void) ddi_dma_sync(rrp->desc.dma_hdl,
    406  5574  mx205022 		    0,
    407  5659  jj146644 		    (ngep->recv_count + sync_start - ngep->rx_desc) *
    408  5574  mx205022 		    ngep->desc_attr.rxd_size,
    409  5574  mx205022 		    DDI_DMA_SYNC_FORDEV);
    410  5574  mx205022 	}
    411  5574  mx205022 	mode_cntl.mode_val = nge_reg_get32(ngep, NGE_MODE_CNTL);
    412  5574  mx205022 	mode_cntl.mode_bits.rxdm = NGE_SET;
    413  5574  mx205022 	mode_cntl.mode_bits.tx_rcom_en = NGE_SET;
    414  5574  mx205022 	nge_reg_put32(ngep, NGE_MODE_CNTL, mode_cntl.mode_val);
    415  5574  mx205022 
    416  5574  mx205022 	return (head);
    417  5574  mx205022 }
    418  5574  mx205022 
    419  5574  mx205022 void
    420  5574  mx205022 nge_receive(nge_t *ngep)
    421  5574  mx205022 {
    422  5574  mx205022 	mblk_t *mp;
    423  5574  mx205022 	recv_ring_t *rrp;
    424  5574  mx205022 	rrp = ngep->recv;
    425  5574  mx205022 
    426  5574  mx205022 	mp = nge_recv_ring(ngep);
    427  5574  mx205022 	mutex_exit(ngep->genlock);
    428  5574  mx205022 	if (mp != NULL)
    429  5574  mx205022 		mac_rx(ngep->mh, rrp->handle, mp);
    430  5574  mx205022 	mutex_enter(ngep->genlock);
    431  5574  mx205022 }
    432  5574  mx205022 
    433  5574  mx205022 void
    434  5574  mx205022 nge_hot_rxd_fill(void *hwd, const ddi_dma_cookie_t *cookie, size_t len)
    435  5574  mx205022 {
    436  5574  mx205022 	uint64_t dmac_addr;
    437  5574  mx205022 	hot_rx_bd * hw_bd_p;
    438  5574  mx205022 
    439  5574  mx205022 	hw_bd_p = (hot_rx_bd *)hwd;
    440  5574  mx205022 	dmac_addr = cookie->dmac_laddress + NGE_HEADROOM;
    441  5574  mx205022 
    442  5574  mx205022 	hw_bd_p->cntl_status.cntl_val = 0;
    443  5574  mx205022 
    444  5574  mx205022 	hw_bd_p->host_buf_addr_hi = dmac_addr >> 32;
    445  7781       Min 	hw_bd_p->host_buf_addr_lo = (uint32_t)dmac_addr;
    446  5574  mx205022 	hw_bd_p->cntl_status.control_bits.bcnt = len - 1;
    447  5574  mx205022 
    448  5574  mx205022 	membar_producer();
    449  5574  mx205022 	hw_bd_p->cntl_status.control_bits.own = NGE_SET;
    450  5574  mx205022 }
    451  5574  mx205022 
    452  5574  mx205022 void
    453  5574  mx205022 nge_sum_rxd_fill(void *hwd, const ddi_dma_cookie_t *cookie, size_t len)
    454  5574  mx205022 {
    455  5574  mx205022 	sum_rx_bd * hw_bd_p;
    456  5574  mx205022 
    457  5574  mx205022 	hw_bd_p = hwd;
    458  5574  mx205022 
    459  5574  mx205022 	hw_bd_p->cntl_status.cntl_val = 0;
    460  5574  mx205022 
    461  7781       Min 	hw_bd_p->host_buf_addr =
    462  7781       Min 	    (uint32_t)(cookie->dmac_address + NGE_HEADROOM);
    463  5574  mx205022 	hw_bd_p->cntl_status.control_bits.bcnt = len - 1;
    464  5574  mx205022 
    465  5574  mx205022 	membar_producer();
    466  5574  mx205022 	hw_bd_p->cntl_status.control_bits.own = NGE_SET;
    467  5574  mx205022 }
    468  5574  mx205022 
    469  5574  mx205022 uint32_t
    470  5574  mx205022 nge_hot_rxd_check(const void *hwd, size_t *len)
    471  5574  mx205022 {
    472  5574  mx205022 	uint32_t err_flag;
    473  5574  mx205022 	const hot_rx_bd * hrbdp;
    474  5574  mx205022 
    475  5574  mx205022 	hrbdp = hwd;
    476  9906      Zhen 	err_flag = hrbdp->cntl_status.cntl_val;
    477  9906      Zhen 	*len = err_flag & RXD_BCNT_MSK;
    478  5574  mx205022 	return (err_flag);
    479  5574  mx205022 }
    480  5574  mx205022 
    481  5574  mx205022 uint32_t
    482  5574  mx205022 nge_sum_rxd_check(const void *hwd, size_t *len)
    483  5574  mx205022 {
    484  5574  mx205022 	uint32_t err_flag;
    485  5574  mx205022 	const sum_rx_bd * hrbdp;
    486  5574  mx205022 
    487  5574  mx205022 	hrbdp = hwd;
    488  5574  mx205022 
    489  9906      Zhen 	err_flag = hrbdp->cntl_status.cntl_val;
    490  9906      Zhen 	*len = err_flag & RXD_BCNT_MSK;
    491  5574  mx205022 	return (err_flag);
    492  5574  mx205022 }
    493