Home | History | Annotate | Download | only in nge
      1 /*
      2  * CDDL HEADER START
      3  *
      4  * The contents of this file are subject to the terms of the
      5  * Common Development and Distribution License (the "License").
      6  * You may not use this file except in compliance with the License.
      7  *
      8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
      9  * or http://www.opensolaris.org/os/licensing.
     10  * See the License for the specific language governing permissions
     11  * and limitations under the License.
     12  *
     13  * When distributing Covered Code, include this CDDL HEADER in each
     14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
     15  * If applicable, add the following below this CDDL HEADER, with the
     16  * fields enclosed by brackets "[]" replaced with your own identifying
     17  * information: Portions Copyright [yyyy] [name of copyright owner]
     18  *
     19  * CDDL HEADER END
     20  */
     21 
     22 /*
     23  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
     24  * Use is subject to license terms.
     25  */
     26 
     27 #include "nge.h"
     28 
     29 #undef	NGE_DBG
     30 #define	NGE_DBG		NGE_DBG_RECV
     31 
     32 #define	RXD_END		0x20000000
     33 #define	RXD_ERR		0x40000000
     34 #define	RXD_OWN		0x80000000
     35 #define	RXD_CSUM_MSK	0x1C000000
     36 #define	RXD_BCNT_MSK	0x00003FFF
     37 
     38 #define	RXD_CK8G_NO_HSUM	0x0
     39 #define	RXD_CK8G_TCP_SUM_ERR	0x04000000
     40 #define	RXD_CK8G_UDP_SUM_ERR	0x08000000
     41 #define	RXD_CK8G_IP_HSUM_ERR	0x0C000000
     42 #define	RXD_CK8G_IP_HSUM	0x10000000
     43 #define	RXD_CK8G_TCP_SUM	0x14000000
     44 #define	RXD_CK8G_UDP_SUM	0x18000000
     45 #define	RXD_CK8G_RESV		0x1C000000
     46 
     47 extern ddi_device_acc_attr_t nge_data_accattr;
     48 
     49 /*
     50  * Callback code invoked from STREAMs when the recv data buffer is free for
     51  * recycling.
     52  *
     53  * The following table describes function behaviour:
     54  *
     55  *                      | mac stopped | mac running
     56  * ---------------------------------------------------
     57  * buffer delivered     | free buffer | recycle buffer
     58  * buffer not delivered | do nothing  | recycle buffer (*)
     59  *
     60  * Note (*):
     61  *   Recycle buffer only if mac state did not change during execution of
     62  *   function. Otherwise if mac state changed, set buffer delivered & re-enter
     63  *   function by calling freemsg().
     64  */
     65 
     66 void
     67 nge_recv_recycle(caddr_t arg)
     68 {
     69 	boolean_t val;
     70 	boolean_t valid;
     71 	nge_t *ngep;
     72 	dma_area_t *bufp;
     73 	buff_ring_t *brp;
     74 	nge_sw_statistics_t *sw_stp;
     75 
     76 	bufp = (dma_area_t *)arg;
     77 	ngep = (nge_t *)bufp->private;
     78 	brp = ngep->buff;
     79 	sw_stp = &ngep->statistics.sw_statistics;
     80 
     81 	/*
     82 	 * Free the buffer directly if the buffer was allocated
     83 	 * previously or mac was stopped.
     84 	 */
     85 	if (bufp->signature != brp->buf_sign) {
     86 		if (bufp->rx_delivered == B_TRUE) {
     87 			nge_free_dma_mem(bufp);
     88 			kmem_free(bufp, sizeof (dma_area_t));
     89 			val = nge_atomic_decrease(&brp->rx_hold, 1);
     90 			ASSERT(val == B_TRUE);
     91 		}
     92 		return;
     93 	}
     94 
     95 	/*
     96 	 * recycle the data buffer again and fill them in free ring
     97 	 */
     98 	bufp->rx_recycle.free_func = nge_recv_recycle;
     99 	bufp->rx_recycle.free_arg = (caddr_t)bufp;
    100 
    101 	bufp->mp = desballoc(DMA_VPTR(*bufp),
    102 	    ngep->buf_size + NGE_HEADROOM, 0, &bufp->rx_recycle);
    103 
    104 	if (bufp->mp == NULL) {
    105 		sw_stp->mp_alloc_err++;
    106 		sw_stp->recy_free++;
    107 		nge_free_dma_mem(bufp);
    108 		kmem_free(bufp, sizeof (dma_area_t));
    109 		val = nge_atomic_decrease(&brp->rx_hold, 1);
    110 		ASSERT(val == B_TRUE);
    111 	} else {
    112 
    113 		mutex_enter(brp->recycle_lock);
    114 		if (bufp->signature != brp->buf_sign)
    115 			valid = B_TRUE;
    116 		else
    117 			valid = B_FALSE;
    118 		bufp->rx_delivered = valid;
    119 		if (bufp->rx_delivered == B_FALSE)  {
    120 			bufp->next = brp->recycle_list;
    121 			brp->recycle_list = bufp;
    122 		}
    123 		mutex_exit(brp->recycle_lock);
    124 		if (valid == B_TRUE)
    125 			/* call nge_rx_recycle again to free it */
    126 			freemsg(bufp->mp);
    127 		else {
    128 			val = nge_atomic_decrease(&brp->rx_hold, 1);
    129 			ASSERT(val == B_TRUE);
    130 		}
    131 	}
    132 }
    133 
    134 /*
    135  * Checking the rx's BDs (one or more) to receive
    136  * one complete packet.
    137  * start_index: the start indexer of BDs for one packet.
    138  * end_index: the end indexer of BDs for one packet.
    139  */
    140 static mblk_t *nge_recv_packet(nge_t *ngep, uint32_t start_index, size_t len);
    141 #pragma	inline(nge_recv_packet)
    142 
    143 static mblk_t *
    144 nge_recv_packet(nge_t *ngep, uint32_t start_index, size_t len)
    145 {
    146 	uint8_t *rptr;
    147 	uint32_t minsize;
    148 	uint32_t maxsize;
    149 	mblk_t *mp;
    150 	buff_ring_t *brp;
    151 	sw_rx_sbd_t *srbdp;
    152 	dma_area_t *bufp;
    153 	nge_sw_statistics_t *sw_stp;
    154 	void *hw_bd_p;
    155 
    156 	brp = ngep->buff;
    157 	minsize = ETHERMIN;
    158 	maxsize = ngep->max_sdu;
    159 	sw_stp = &ngep->statistics.sw_statistics;
    160 	mp = NULL;
    161 
    162 	srbdp = &brp->sw_rbds[start_index];
    163 	DMA_SYNC(*srbdp->bufp, DDI_DMA_SYNC_FORKERNEL);
    164 	hw_bd_p = DMA_VPTR(srbdp->desc);
    165 
    166 	/*
    167 	 * First check the free_list, if it is NULL,
    168 	 * make the recycle_list be free_list.
    169 	 */
    170 	if (brp->free_list == NULL) {
    171 		mutex_enter(brp->recycle_lock);
    172 		brp->free_list = brp->recycle_list;
    173 		brp->recycle_list = NULL;
    174 		mutex_exit(brp->recycle_lock);
    175 	}
    176 	bufp = brp->free_list;
    177 	/* If it's not a qualified packet, delete it */
    178 	if (len > maxsize || len < minsize) {
    179 		ngep->desc_attr.rxd_fill(hw_bd_p, &srbdp->bufp->cookie,
    180 		    srbdp->bufp->alength);
    181 		srbdp->flags = CONTROLER_OWN;
    182 		return (NULL);
    183 	}
    184 
    185 	/*
    186 	 * If receive packet size is smaller than RX bcopy threshold,
    187 	 * or there is no available buffer in free_list or recycle list,
    188 	 * we use bcopy directly.
    189 	 */
    190 	if (len <= ngep->param_rxbcopy_threshold || bufp == NULL)
    191 		brp->rx_bcopy = B_TRUE;
    192 	else
    193 		brp->rx_bcopy = B_FALSE;
    194 
    195 	if (brp->rx_bcopy) {
    196 		mp = allocb(len + NGE_HEADROOM, 0);
    197 		if (mp == NULL) {
    198 			sw_stp->mp_alloc_err++;
    199 			ngep->desc_attr.rxd_fill(hw_bd_p, &srbdp->bufp->cookie,
    200 			    srbdp->bufp->alength);
    201 			srbdp->flags = CONTROLER_OWN;
    202 			return (NULL);
    203 		}
    204 		rptr = DMA_VPTR(*srbdp->bufp);
    205 		mp->b_rptr = mp->b_rptr + NGE_HEADROOM;
    206 		bcopy(rptr + NGE_HEADROOM, mp->b_rptr, len);
    207 		mp->b_wptr = mp->b_rptr + len;
    208 	} else {
    209 		mp = srbdp->bufp->mp;
    210 		/*
    211 		 * Make sure the packet *contents* 4-byte aligned
    212 		 */
    213 		mp->b_rptr += NGE_HEADROOM;
    214 		mp->b_wptr = mp->b_rptr + len;
    215 		mp->b_next = mp->b_cont = NULL;
    216 		srbdp->bufp->rx_delivered = B_TRUE;
    217 		srbdp->bufp = NULL;
    218 		nge_atomic_increase(&brp->rx_hold, 1);
    219 
    220 		/* Fill the buffer from free_list */
    221 		srbdp->bufp = bufp;
    222 		brp->free_list = bufp->next;
    223 		bufp->next = NULL;
    224 	}
    225 
    226 	/* replenish the buffer for hardware descriptor */
    227 	ngep->desc_attr.rxd_fill(hw_bd_p, &srbdp->bufp->cookie,
    228 	    srbdp->bufp->alength);
    229 	srbdp->flags = CONTROLER_OWN;
    230 	sw_stp->rbytes += len;
    231 	sw_stp->recv_count++;
    232 
    233 	return (mp);
    234 }
    235 
    236 
    237 #define	RX_HW_ERR	0x01
    238 #define	RX_SUM_NO	0x02
    239 #define	RX_SUM_ERR	0x04
    240 
    241 /*
    242  * Statistic the rx's error
    243  * and generate a log msg for these.
    244  * Note:
    245  * RXE, Parity Error, Symbo error, CRC error
    246  * have been recored by nvidia's  hardware
    247  * statistics part (nge_statistics). So it is uncessary to record them by
    248  * driver in this place.
    249  */
    250 static uint32_t
    251 nge_rxsta_handle(nge_t *ngep, uint32_t stflag, uint32_t *pflags);
    252 #pragma	inline(nge_rxsta_handle)
    253 
    254 static uint32_t
    255 nge_rxsta_handle(nge_t *ngep,  uint32_t stflag, uint32_t *pflags)
    256 {
    257 	uint32_t errors;
    258 	uint32_t err_flag;
    259 	nge_sw_statistics_t *sw_stp;
    260 
    261 	err_flag = 0;
    262 	sw_stp = &ngep->statistics.sw_statistics;
    263 
    264 	if ((RXD_END & stflag) == 0)
    265 		return (RX_HW_ERR);
    266 
    267 	errors = stflag & RXD_CSUM_MSK;
    268 	switch (errors) {
    269 	default:
    270 	break;
    271 
    272 	case RXD_CK8G_TCP_SUM:
    273 	case RXD_CK8G_UDP_SUM:
    274 		*pflags |= HCK_FULLCKSUM;
    275 		*pflags |= HCK_IPV4_HDRCKSUM;
    276 		*pflags |= HCK_FULLCKSUM_OK;
    277 		break;
    278 
    279 	case RXD_CK8G_TCP_SUM_ERR:
    280 	case RXD_CK8G_UDP_SUM_ERR:
    281 		sw_stp->tcp_hwsum_err++;
    282 		*pflags |= HCK_IPV4_HDRCKSUM;
    283 		break;
    284 
    285 	case RXD_CK8G_IP_HSUM:
    286 		*pflags |= HCK_IPV4_HDRCKSUM;
    287 		break;
    288 
    289 	case RXD_CK8G_NO_HSUM:
    290 		err_flag |= RX_SUM_NO;
    291 		break;
    292 
    293 	case RXD_CK8G_IP_HSUM_ERR:
    294 		sw_stp->ip_hwsum_err++;
    295 		err_flag |=  RX_SUM_ERR;
    296 		break;
    297 	}
    298 
    299 	if ((stflag & RXD_ERR) != 0)	{
    300 
    301 		err_flag |= RX_HW_ERR;
    302 		NGE_DEBUG(("Receive desc error, status: 0x%x", stflag));
    303 	}
    304 
    305 	return (err_flag);
    306 }
    307 
    308 static mblk_t *
    309 nge_recv_ring(nge_t *ngep)
    310 {
    311 	uint32_t stflag;
    312 	uint32_t flag_err;
    313 	uint32_t sum_flags;
    314 	size_t len;
    315 	uint64_t end_index;
    316 	uint64_t sync_start;
    317 	mblk_t *mp;
    318 	mblk_t **tail;
    319 	mblk_t *head;
    320 	recv_ring_t *rrp;
    321 	buff_ring_t *brp;
    322 	sw_rx_sbd_t *srbdp;
    323 	void * hw_bd_p;
    324 	nge_mode_cntl mode_cntl;
    325 
    326 	mp = NULL;
    327 	head = NULL;
    328 	tail = &head;
    329 	rrp = ngep->recv;
    330 	brp = ngep->buff;
    331 
    332 	end_index = sync_start = rrp->prod_index;
    333 	/* Sync the descriptor for kernel */
    334 	if (sync_start + ngep->param_recv_max_packet <= ngep->rx_desc) {
    335 		(void) ddi_dma_sync(rrp->desc.dma_hdl,
    336 		    sync_start * ngep->desc_attr.rxd_size,
    337 		    ngep->param_recv_max_packet * ngep->desc_attr.rxd_size,
    338 		    DDI_DMA_SYNC_FORKERNEL);
    339 	} else {
    340 		(void) ddi_dma_sync(rrp->desc.dma_hdl,
    341 		    sync_start * ngep->desc_attr.rxd_size,
    342 		    0,
    343 		    DDI_DMA_SYNC_FORKERNEL);
    344 		(void) ddi_dma_sync(rrp->desc.dma_hdl,
    345 		    0,
    346 		    (ngep->param_recv_max_packet + sync_start - ngep->rx_desc) *
    347 		    ngep->desc_attr.rxd_size,
    348 		    DDI_DMA_SYNC_FORKERNEL);
    349 	}
    350 
    351 	/*
    352 	 * Looking through the rx's ring to find the good packets
    353 	 * and try to receive more and more packets in rx's ring
    354 	 */
    355 	for (;;) {
    356 		sum_flags = 0;
    357 		flag_err = 0;
    358 		end_index = rrp->prod_index;
    359 		srbdp = &brp->sw_rbds[end_index];
    360 		hw_bd_p = DMA_VPTR(srbdp->desc);
    361 		stflag = ngep->desc_attr.rxd_check(hw_bd_p, &len);
    362 		/*
    363 		 * If there is no packet in receving ring
    364 		 * break the loop
    365 		 */
    366 		if ((stflag & RXD_OWN) != 0 || HOST_OWN == srbdp->flags)
    367 			break;
    368 
    369 		ngep->recv_count++;
    370 		flag_err = nge_rxsta_handle(ngep, stflag, &sum_flags);
    371 		if ((flag_err & RX_HW_ERR) == 0) {
    372 			srbdp->flags = NGE_END_PACKET;
    373 			mp = nge_recv_packet(ngep, end_index, len);
    374 		} else {
    375 			/* Hardware error, re-use the buffer */
    376 			ngep->desc_attr.rxd_fill(hw_bd_p, &srbdp->bufp->cookie,
    377 			    srbdp->bufp->alength);
    378 			srbdp->flags = CONTROLER_OWN;
    379 		}
    380 		if (mp != NULL) {
    381 			if (!(flag_err & (RX_SUM_NO | RX_SUM_ERR))) {
    382 				(void) hcksum_assoc(mp, NULL, NULL,
    383 				    0, 0, 0, 0, sum_flags, 0);
    384 			}
    385 			*tail = mp;
    386 			tail = &mp->b_next;
    387 			mp = NULL;
    388 		}
    389 		rrp->prod_index = NEXT(end_index, rrp->desc.nslots);
    390 		if (ngep->recv_count >= ngep->param_recv_max_packet)
    391 			break;
    392 	}
    393 
    394 	/* Sync the descriptors for device */
    395 	if (sync_start + ngep->recv_count <= ngep->rx_desc) {
    396 		(void) ddi_dma_sync(rrp->desc.dma_hdl,
    397 		    sync_start * ngep->desc_attr.rxd_size,
    398 		    ngep->recv_count * ngep->desc_attr.rxd_size,
    399 		    DDI_DMA_SYNC_FORDEV);
    400 	} else {
    401 		(void) ddi_dma_sync(rrp->desc.dma_hdl,
    402 		    sync_start * ngep->desc_attr.rxd_size,
    403 		    0,
    404 		    DDI_DMA_SYNC_FORDEV);
    405 		(void) ddi_dma_sync(rrp->desc.dma_hdl,
    406 		    0,
    407 		    (ngep->recv_count + sync_start - ngep->rx_desc) *
    408 		    ngep->desc_attr.rxd_size,
    409 		    DDI_DMA_SYNC_FORDEV);
    410 	}
    411 	mode_cntl.mode_val = nge_reg_get32(ngep, NGE_MODE_CNTL);
    412 	mode_cntl.mode_bits.rxdm = NGE_SET;
    413 	mode_cntl.mode_bits.tx_rcom_en = NGE_SET;
    414 	nge_reg_put32(ngep, NGE_MODE_CNTL, mode_cntl.mode_val);
    415 
    416 	return (head);
    417 }
    418 
    419 void
    420 nge_receive(nge_t *ngep)
    421 {
    422 	mblk_t *mp;
    423 	recv_ring_t *rrp;
    424 	rrp = ngep->recv;
    425 
    426 	mp = nge_recv_ring(ngep);
    427 	mutex_exit(ngep->genlock);
    428 	if (mp != NULL)
    429 		mac_rx(ngep->mh, rrp->handle, mp);
    430 	mutex_enter(ngep->genlock);
    431 }
    432 
    433 void
    434 nge_hot_rxd_fill(void *hwd, const ddi_dma_cookie_t *cookie, size_t len)
    435 {
    436 	uint64_t dmac_addr;
    437 	hot_rx_bd * hw_bd_p;
    438 
    439 	hw_bd_p = (hot_rx_bd *)hwd;
    440 	dmac_addr = cookie->dmac_laddress + NGE_HEADROOM;
    441 
    442 	hw_bd_p->cntl_status.cntl_val = 0;
    443 
    444 	hw_bd_p->host_buf_addr_hi = dmac_addr >> 32;
    445 	hw_bd_p->host_buf_addr_lo = (uint32_t)dmac_addr;
    446 	hw_bd_p->cntl_status.control_bits.bcnt = len - 1;
    447 
    448 	membar_producer();
    449 	hw_bd_p->cntl_status.control_bits.own = NGE_SET;
    450 }
    451 
    452 void
    453 nge_sum_rxd_fill(void *hwd, const ddi_dma_cookie_t *cookie, size_t len)
    454 {
    455 	sum_rx_bd * hw_bd_p;
    456 
    457 	hw_bd_p = hwd;
    458 
    459 	hw_bd_p->cntl_status.cntl_val = 0;
    460 
    461 	hw_bd_p->host_buf_addr =
    462 	    (uint32_t)(cookie->dmac_address + NGE_HEADROOM);
    463 	hw_bd_p->cntl_status.control_bits.bcnt = len - 1;
    464 
    465 	membar_producer();
    466 	hw_bd_p->cntl_status.control_bits.own = NGE_SET;
    467 }
    468 
    469 uint32_t
    470 nge_hot_rxd_check(const void *hwd, size_t *len)
    471 {
    472 	uint32_t err_flag;
    473 	const hot_rx_bd * hrbdp;
    474 
    475 	hrbdp = hwd;
    476 	err_flag = hrbdp->cntl_status.cntl_val;
    477 	*len = err_flag & RXD_BCNT_MSK;
    478 	return (err_flag);
    479 }
    480 
    481 uint32_t
    482 nge_sum_rxd_check(const void *hwd, size_t *len)
    483 {
    484 	uint32_t err_flag;
    485 	const sum_rx_bd * hrbdp;
    486 
    487 	hrbdp = hwd;
    488 
    489 	err_flag = hrbdp->cntl_status.cntl_val;
    490 	*len = err_flag & RXD_BCNT_MSK;
    491 	return (err_flag);
    492 }
    493