common/os/ip_cksum.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */
/* Copyright (c) 1990 Mentat Inc. */

#include <sys/types.h>
#include <sys/inttypes.h>
#include <sys/systm.h>
#include <sys/stream.h>
#include <sys/strsun.h>
#include <sys/debug.h>
#include <sys/ddi.h>
#include <sys/vtrace.h>
#include <inet/sctp_crc32.h>
#include <inet/ip.h>

#include <sys/multidata.h>
#include <sys/multidata_impl.h>

extern unsigned int 	ip_ocsum(ushort_t *address, int halfword_count,
    unsigned int sum);

/*
 * Checksum routine for Internet Protocol family headers.
 * This routine is very heavily used in the network
 * code and should be modified for each CPU to be as fast as possible.
 */

#define	mp_len(mp) ((mp)->b_wptr - (mp)->b_rptr)

/*
 * Even/Odd checks. Usually it is performed on pointers but may be
 * used on integers as well. uintptr_t is long enough to hold both
 * integer and pointer.
 */
#define	is_odd(p) (((uintptr_t)(p) & 0x1) != 0)
#define	is_even(p) (!is_odd(p))


#ifdef ZC_TEST
/*
 * Disable the TCP s/w cksum.
 * XXX - This is just a hack for testing purpose. Don't use it for
 * anything else!
 */
int noswcksum = 0;
#endif
/*
 * Note: this does not ones-complement the result since it is used
 * when computing partial checksums.
 * For nonSTRUIO_IP mblks, assumes mp->b_rptr+offset is 16 bit aligned.
 * For STRUIO_IP mblks, assumes mp->b_datap->db_struiobase is 16 bit aligned.
 *
 * Note: for STRUIO_IP special mblks some data may have been previously
 *	 checksumed, this routine will handle additional data prefixed within
 *	 an mblk or b_cont (chained) mblk(s). This routine will also handle
 *	 suffixed b_cont mblk(s) and data suffixed within an mblk.
 */
unsigned int
ip_cksum(mblk_t *mp, int offset, uint_t sum)
{
	ushort_t *w;
	ssize_t	mlen;
	int pmlen;
	mblk_t *pmp;
	dblk_t *dp = mp->b_datap;
	ushort_t psum = 0;

#ifdef ZC_TEST
	if (noswcksum)
		return (0xffff);
#endif
	ASSERT(dp);

	if (mp->b_cont == NULL) {
		/*
		 * May be fast-path, only one mblk.
		 */
		w = (ushort_t *)(mp->b_rptr + offset);
		if (dp->db_struioflag & STRUIO_IP) {
			/*
			 * Checksum any data not already done by
			 * the caller and add in any partial checksum.
			 */
			if ((offset > dp->db_cksumstart) ||
			    mp->b_wptr != (uchar_t *)(mp->b_rptr +
			    dp->db_cksumend)) {
				/*
				 * Mblk data pointers aren't inclusive
				 * of uio data, so disregard checksum.
				 *
				 * not using all of data in dblk make sure
				 * not use to use the precalculated checksum
				 * in this case.
				 */
				dp->db_struioflag &= ~STRUIO_IP;
				goto norm;
			}
			ASSERT(mp->b_wptr == (mp->b_rptr + dp->db_cksumend));
			psum = *(ushort_t *)dp->db_struioun.data;
			if ((mlen = dp->db_cksumstart - offset) < 0)
				mlen = 0;
			if (is_odd(mlen))
				goto slow;
			if (mlen && dp->db_cksumstart != dp->db_cksumstuff &&
			    dp->db_cksumend != dp->db_cksumstuff) {
				/*
				 * There is prefix data to do and some uio
				 * data has already been checksumed and there
				 * is more uio data to do, so do the prefix
				 * data first, then do the remainder of the
				 * uio data.
				 */
				sum = ip_ocsum(w, mlen >> 1, sum);
				w = (ushort_t *)(mp->b_rptr +
				    dp->db_cksumstuff);
				if (is_odd(w)) {
					pmp = mp;
					goto slow1;
				}
				mlen = dp->db_cksumend - dp->db_cksumstuff;
			} else if (dp->db_cksumend != dp->db_cksumstuff) {
				/*
				 * There may be uio data to do, if there is
				 * prefix data to do then add in all of the
				 * uio data (if any) to do, else just do any
				 * uio data.
				 */
				if (mlen)
					mlen += dp->db_cksumend
					    - dp->db_cksumstuff;
				else {
					w = (ushort_t *)(mp->b_rptr +
					    dp->db_cksumstuff);
					if (is_odd(w))
						goto slow;
					mlen = dp->db_cksumend
					    - dp->db_cksumstuff;
				}
			} else if (mlen == 0)
				return (psum);

			if (is_odd(mlen))
				goto slow;
			sum += psum;
		} else {
			/*
			 * Checksum all data not already done by the caller.
			 */
		norm:
			mlen = mp->b_wptr - (uchar_t *)w;
			if (is_odd(mlen))
				goto slow;
		}
		ASSERT(is_even(w));
		ASSERT(is_even(mlen));
		return (ip_ocsum(w, mlen >> 1, sum));
	}
	if (dp->db_struioflag & STRUIO_IP)
		psum = *(ushort_t *)dp->db_struioun.data;
slow:
	pmp = 0;
slow1:
	mlen = 0;
	pmlen = 0;
	for (; ; ) {
		/*
		 * Each trip around loop adds in word(s) from one mbuf segment
		 * (except for when pmp == mp, then its two partial trips).
		 */
		w = (ushort_t *)(mp->b_rptr + offset);
		if (pmp) {
			/*
			 * This is the second trip around for this mblk.
			 */
			pmp = 0;
			mlen = 0;
			goto douio;
		} else if (dp->db_struioflag & STRUIO_IP) {
			/*
			 * Checksum any data not already done by the
			 * caller and add in any partial checksum.
			 */
			if ((offset > dp->db_cksumstart) ||
			    mp->b_wptr != (uchar_t *)(mp->b_rptr +
			    dp->db_cksumend)) {
				/*
				 * Mblk data pointers aren't inclusive
				 * of uio data, so disregard checksum.
				 *
				 * not using all of data in dblk make sure
				 * not use to use the precalculated checksum
				 * in this case.
				 */
				dp->db_struioflag &= ~STRUIO_IP;
				goto snorm;
			}
			ASSERT(mp->b_wptr == (mp->b_rptr + dp->db_cksumend));
			if ((mlen = dp->db_cksumstart - offset) < 0)
				mlen = 0;
			if (mlen && dp->db_cksumstart != dp->db_cksumstuff) {
				/*
				 * There is prefix data too do and some
				 * uio data has already been checksumed,
				 * so do the prefix data only this trip.
				 */
				pmp = mp;
			} else {
				/*
				 * Add in any partial cksum (if any) and
				 * do the remainder of the uio data.
				 */
				int odd;
			douio:
				odd = is_odd(dp->db_cksumstuff -
				    dp->db_cksumstart);
				if (pmlen == -1) {
					/*
					 * Previous mlen was odd, so swap
					 * the partial checksum bytes.
					 */
					sum += ((psum << 8) & 0xffff)
					    | (psum >> 8);
					if (odd)
						pmlen = 0;
				} else {
					sum += psum;
					if (odd)
						pmlen = -1;
				}
				if (dp->db_cksumend != dp->db_cksumstuff) {
					/*
					 * If prefix data to do and then all
					 * the uio data nees to be checksumed,
					 * else just do any uio data.
					 */
					if (mlen)
						mlen += dp->db_cksumend
						    - dp->db_cksumstuff;
					else {
						w = (ushort_t *)(mp->b_rptr +
						    dp->db_cksumstuff);
						mlen = dp->db_cksumend -
						    dp->db_cksumstuff;
					}
				}
			}
		} else {
			/*
			 * Checksum all of the mblk data.
			 */
		snorm:
			mlen = mp->b_wptr - (uchar_t *)w;
		}

		mp = mp->b_cont;
		if (mlen > 0 && pmlen == -1) {
			/*
			 * There is a byte left from the last
			 * segment; add it into the checksum.
			 * Don't have to worry about a carry-
			 * out here because we make sure that
			 * high part of (32 bit) sum is small
			 * below.
			 */
#ifdef _LITTLE_ENDIAN
			sum += *(uchar_t *)w << 8;
#else
			sum += *(uchar_t *)w;
#endif
			w = (ushort_t *)((char *)w + 1);
			mlen--;
			pmlen = 0;
		}
		if (mlen > 0) {
			if (is_even(w)) {
				sum = ip_ocsum(w, mlen>>1, sum);
				w += mlen>>1;
				/*
				 * If we had an odd number of bytes,
				 * then the last byte goes in the high
				 * part of the sum, and we take the
				 * first byte to the low part of the sum
				 * the next time around the loop.
				 */
				if (is_odd(mlen)) {
#ifdef _LITTLE_ENDIAN
					sum += *(uchar_t *)w;
#else
					sum += *(uchar_t *)w << 8;
#endif
					pmlen = -1;
				}
			} else {
				ushort_t swsum;
#ifdef _LITTLE_ENDIAN
				sum += *(uchar_t *)w;
#else
				sum += *(uchar_t *)w << 8;
#endif
				mlen--;
				w = (ushort_t *)(1 + (uintptr_t)w);

				/* Do a separate checksum and copy operation */
				swsum = ip_ocsum(w, mlen>>1, 0);
				sum += ((swsum << 8) & 0xffff) | (swsum >> 8);
				w += mlen>>1;
				/*
				 * If we had an even number of bytes,
				 * then the last byte goes in the low
				 * part of the sum.  Otherwise we had an
				 * odd number of bytes and we take the first
				 * byte to the low part of the sum the
				 * next time around the loop.
				 */
				if (is_odd(mlen)) {
#ifdef _LITTLE_ENDIAN
					sum += *(uchar_t *)w << 8;
#else
					sum += *(uchar_t *)w;
#endif
				}
				else
					pmlen = -1;
			}
		}
		/*
		 * Locate the next block with some data.
		 * If there is a word split across a boundary we
		 * will wrap to the top with mlen == -1 and
		 * then add it in shifted appropriately.
		 */
		offset = 0;
		if (! pmp) {
			for (; ; ) {
				if (mp == 0) {
					goto done;
				}
				if (mp_len(mp))
					break;
				mp = mp->b_cont;
			}
			dp = mp->b_datap;
			if (dp->db_struioflag & STRUIO_IP)
				psum = *(ushort_t *)dp->db_struioun.data;
		} else
			mp = pmp;
	}
done:
	/*
	 * Add together high and low parts of sum
	 * and carry to get cksum.
	 * Have to be careful to not drop the last
	 * carry here.
	 */
	sum = (sum & 0xFFFF) + (sum >> 16);
	sum = (sum & 0xFFFF) + (sum >> 16);
	TRACE_3(TR_FAC_IP, TR_IP_CKSUM_END,
	    "ip_cksum_end:(%S) type %d (%X)", "ip_cksum", 1, sum);
	return (sum);
}

uint32_t
sctp_cksum(mblk_t *mp, int offset)
{
	uint32_t crc32;
	uchar_t *p = NULL;

	crc32 = 0xFFFFFFFF;
	p = mp->b_rptr + offset;
	crc32 = sctp_crc32(crc32, p, mp->b_wptr - p);
	for (mp = mp->b_cont; mp != NULL; mp = mp->b_cont) {
		crc32 = sctp_crc32(crc32, mp->b_rptr, MBLKL(mp));
	}

	/* Complement the result */
	crc32 = ~crc32;

	return (crc32);
}

/*
 * Routine to compute Internet checksum (16-bit 1's complement) of a given
 * Multidata packet descriptor.  As in the non-Multidata routine, this doesn't
 * 1's complement the result, such that it may be used to compute partial
 * checksums.  Since it works on buffer spans rather than mblks, this routine
 * does not handle existing partial checksum value as in the STRUIO_IP special
 * mblk case (supporting this is rather trivial, but is perhaps of no use at
 * the moment unless synchronous streams and delayed checksum calculation are
 * revived.)
 *
 * Note also here that the given Multidata packet descriptor must refer to
 * a header buffer, i.e. it must have a header fragment.  In addition, the
 * offset must lie within the boundary of the header fragment.  For the
 * outbound tcp (MDT) case, this will not be an issue because the stack
 * ensures that such conditions are met, and that there is no need whatsoever
 * to compute partial checksums on an arbitrary offset that is not part of
 * the header fragment.  We may need to revisit this routine to handle all
 * cases of the inbound (MDR) case, especially when we need to perform partial
 * checksum calculation due to padded bytes (non-zeroes) in the frame.
 */
uint_t
ip_md_cksum(pdesc_t *pd, int offset, uint_t sum)
{
	pdescinfo_t	*pdi = &pd->pd_pdi;
	uchar_t		*reg_start, *reg_end;
	ssize_t		mlen, i;
	ushort_t	*w;
	boolean_t	byteleft = B_FALSE;

	ASSERT((pdi->flags & PDESC_HAS_REF) != 0);
	ASSERT(pdi->hdr_rptr != NULL && pdi->hdr_wptr != NULL);
	ASSERT(offset <= PDESC_HDRL(pdi));

	for (i = 0; i < pdi->pld_cnt + 1; i++) {
		if (i == 0) {
			reg_start = pdi->hdr_rptr;
			reg_end = pdi->hdr_wptr;
		} else {
			reg_start = pdi->pld_ary[i - 1].pld_rptr;
			reg_end = pdi->pld_ary[i - 1].pld_wptr;
			offset = 0;
		}

		w = (ushort_t *)(reg_start + offset);
		mlen = reg_end - (uchar_t *)w;

		if (mlen > 0 && byteleft) {
			/*
			 * There is a byte left from the last
			 * segment; add it into the checksum.
			 * Don't have to worry about a carry-
			 * out here because we make sure that
			 * high part of (32 bit) sum is small
			 * below.
			 */
#ifdef _LITTLE_ENDIAN
			sum += *(uchar_t *)w << 8;
#else
			sum += *(uchar_t *)w;
#endif
			w = (ushort_t *)((char *)w + 1);
			mlen--;
			byteleft = B_FALSE;
		}

		if (mlen == 0)
			continue;

		if (is_even(w)) {
			sum = ip_ocsum(w, mlen >> 1, sum);
			w += mlen >> 1;
			/*
			 * If we had an odd number of bytes,
			 * then the last byte goes in the high
			 * part of the sum, and we take the
			 * first byte to the low part of the sum
			 * the next time around the loop.
			 */
			if (is_odd(mlen)) {
#ifdef _LITTLE_ENDIAN
				sum += *(uchar_t *)w;
#else
				sum += *(uchar_t *)w << 8;
#endif
				byteleft = B_TRUE;
			}
		} else {
			ushort_t swsum;
#ifdef _LITTLE_ENDIAN
			sum += *(uchar_t *)w;
#else
			sum += *(uchar_t *)w << 8;
#endif
			mlen--;
			w = (ushort_t *)(1 + (uintptr_t)w);

			/* Do a separate checksum and copy operation */
			swsum = ip_ocsum(w, mlen >> 1, 0);
			sum += ((swsum << 8) & 0xffff) | (swsum >> 8);
			w += mlen >> 1;
			/*
			 * If we had an even number of bytes,
			 * then the last byte goes in the low
			 * part of the sum.  Otherwise we had an
			 * odd number of bytes and we take the first
			 * byte to the low part of the sum the
			 * next time around the loop.
			 */
			if (is_odd(mlen)) {
#ifdef _LITTLE_ENDIAN
				sum += *(uchar_t *)w << 8;
#else
				sum += *(uchar_t *)w;
#endif
			} else {
				byteleft = B_TRUE;
			}
		}
	}

	/*
	 * Add together high and low parts of sum and carry to get cksum.
	 * Have to be careful to not drop the last carry here.
	 */
	sum = (sum & 0xffff) + (sum >> 16);
	sum = (sum & 0xffff) + (sum >> 16);

	return (sum);
}

/* Return the IP checksum for the IP header at "iph". */
uint16_t
ip_csum_hdr(ipha_t *ipha)
{
	uint16_t	*uph;
	uint32_t	sum;
	int		opt_len;

	opt_len = (ipha->ipha_version_and_hdr_length & 0xF) -
	    IP_SIMPLE_HDR_LENGTH_IN_WORDS;
	uph = (uint16_t *)ipha;
	sum = uph[0] + uph[1] + uph[2] + uph[3] + uph[4] +
	    uph[5] + uph[6] + uph[7] + uph[8] + uph[9];
	if (opt_len > 0) {
		do {
			sum += uph[10];
			sum += uph[11];
			uph += 2;
		} while (--opt_len);
	}
	sum = (sum & 0xFFFF) + (sum >> 16);
	sum = ~(sum + (sum >> 16)) & 0xFFFF;
	if (sum == 0xffff)
		sum = 0;
	return ((uint16_t)sum);
}