Home | History | Annotate | Download | only in fs
      1  0  stevel /*
      2  0  stevel  * CDDL HEADER START
      3  0  stevel  *
      4  0  stevel  * The contents of this file are subject to the terms of the
      5  0  stevel  * Common Development and Distribution License, Version 1.0 only
      6  0  stevel  * (the "License").  You may not use this file except in compliance
      7  0  stevel  * with the License.
      8  0  stevel  *
      9  0  stevel  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
     10  0  stevel  * or http://www.opensolaris.org/os/licensing.
     11  0  stevel  * See the License for the specific language governing permissions
     12  0  stevel  * and limitations under the License.
     13  0  stevel  *
     14  0  stevel  * When distributing Covered Code, include this CDDL HEADER in each
     15  0  stevel  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
     16  0  stevel  * If applicable, add the following below this CDDL HEADER, with the
     17  0  stevel  * fields enclosed by brackets "[]" replaced with your own identifying
     18  0  stevel  * information: Portions Copyright [yyyy] [name of copyright owner]
     19  0  stevel  *
     20  0  stevel  * CDDL HEADER END
     21  0  stevel  */
     22  0  stevel /*
     23  0  stevel  * Copyright (c) 1998,2001 by Sun Microsystems, Inc.
     24  0  stevel  * All rights reserved.
     25  0  stevel  *
     26  0  stevel  */
     27  0  stevel 
     28  0  stevel #pragma ident	"%Z%%M%	%I%	%E% SMI"
     29  0  stevel 
     30  0  stevel #include <sys/types.h>
     31  0  stevel #include <sys/cmn_err.h>
     32  0  stevel #include <sys/kmem.h>
     33  0  stevel #include <sys/systm.h>
     34  0  stevel #include <sys/debug.h>
     35  0  stevel #include <sys/ddi.h>
     36  0  stevel 
     37  0  stevel #include <sys/fdbuffer.h>
     38  0  stevel 
     39  0  stevel #ifdef DEBUG
     40  0  stevel static int fdb_debug;
     41  0  stevel #define	FDB_D_CREATE	001
     42  0  stevel #define	FDB_D_ALLOC	002
     43  0  stevel #define	FDB_D_IO	004
     44  0  stevel #define	FDB_D_ASYNC	010
     45  0  stevel #define	DEBUGF(lvl, args)	{ if ((lvl) & fdb_debug) cmn_err args; }
     46  0  stevel #else
     47  0  stevel #define	DEBUGF(level, args)
     48  0  stevel #endif
     49  0  stevel static struct kmem_cache *fdb_cache;
     50  0  stevel static void fdb_zero_holes(fdbuffer_t *fdb);
     51  0  stevel 
     52  0  stevel /* ARGSUSED */
     53  0  stevel static int
     54  0  stevel fdb_cache_constructor(void *buf, void *cdrarg, int kmflags)
     55  0  stevel {
     56  0  stevel 	fdbuffer_t *fdb = buf;
     57  0  stevel 
     58  0  stevel 	mutex_init(&fdb->fd_mutex, NULL, MUTEX_DEFAULT, NULL);
     59  0  stevel 
     60  0  stevel 	return (0);
     61  0  stevel }
     62  0  stevel 
     63  0  stevel /* ARGSUSED */
     64  0  stevel static void
     65  0  stevel fdb_cache_destructor(void *buf, void *cdrarg)
     66  0  stevel {
     67  0  stevel 	fdbuffer_t *fdb = buf;
     68  0  stevel 
     69  0  stevel 	mutex_destroy(&fdb->fd_mutex);
     70  0  stevel }
     71  0  stevel 
     72  0  stevel void
     73  0  stevel fdb_init()
     74  0  stevel {
     75  0  stevel 	fdb_cache = kmem_cache_create("fdb_cache", sizeof (fdbuffer_t),
     76  0  stevel 	    0, fdb_cache_constructor, fdb_cache_destructor,
     77  0  stevel 	    NULL, NULL, NULL, 0);
     78  0  stevel }
     79  0  stevel 
     80  0  stevel static void
     81  0  stevel fdb_prepare(fdbuffer_t *fdb)
     82  0  stevel {
     83  0  stevel 	fdb->fd_holes = NULL;
     84  0  stevel 	fdb->fd_iofunc = NULL;
     85  0  stevel 	fdb->fd_iargp = NULL;
     86  0  stevel 	fdb->fd_parentbp = NULL;
     87  0  stevel 	fdb->fd_resid = 0;
     88  0  stevel 	fdb->fd_iocount = 0;
     89  0  stevel 	fdb->fd_iodispatch = 0;
     90  0  stevel 	fdb->fd_err = 0;
     91  0  stevel }
     92  0  stevel 
     93  0  stevel fdbuffer_t *
     94  0  stevel fdb_page_create(page_t *pp, size_t len, int flags)
     95  0  stevel {
     96  0  stevel 	fdbuffer_t *fdb;
     97  0  stevel 
     98  0  stevel 	DEBUGF(FDB_D_CREATE, (CE_NOTE,
     99  0  stevel 	    "?fdb_page_create: pp: %p len: %lux flags: %x",
    100  0  stevel 	    (void *)pp, len, flags));
    101  0  stevel 
    102  0  stevel 	ASSERT(flags & (FDB_READ|FDB_WRITE));
    103  0  stevel 
    104  0  stevel 	fdb = kmem_cache_alloc(fdb_cache, KM_SLEEP);
    105  0  stevel 
    106  0  stevel 	fdb_prepare(fdb);
    107  0  stevel 
    108  0  stevel 	fdb->fd_type = FDB_PAGEIO;
    109  0  stevel 	fdb->fd_len = len;
    110  0  stevel 	fdb->fd_state = flags;
    111  0  stevel 	fdb->fd_pages = pp;
    112  0  stevel 
    113  0  stevel 	return (fdb);
    114  0  stevel }
    115  0  stevel 
    116  0  stevel fdbuffer_t *
    117  0  stevel fdb_addr_create(
    118  0  stevel 	caddr_t addr,
    119  0  stevel 	size_t len,
    120  0  stevel 	int flags,
    121  0  stevel 	page_t **pplist,
    122  0  stevel 	struct proc *procp)
    123  0  stevel {
    124  0  stevel 	fdbuffer_t *fdb;
    125  0  stevel 
    126  0  stevel 	DEBUGF(FDB_D_CREATE, (CE_NOTE,
    127  0  stevel 	    "?fdb_addr_create: addr: %p len: %lux flags: %x",
    128  0  stevel 	    (void *)addr, len, flags));
    129  0  stevel 
    130  0  stevel 	ASSERT(flags & (FDB_READ|FDB_WRITE));
    131  0  stevel 
    132  0  stevel 	fdb = kmem_cache_alloc(fdb_cache, KM_SLEEP);
    133  0  stevel 
    134  0  stevel 	fdb_prepare(fdb);
    135  0  stevel 
    136  0  stevel 	fdb->fd_type = FDB_VADDR;
    137  0  stevel 	fdb->fd_len = len;
    138  0  stevel 	fdb->fd_state = flags;
    139  0  stevel 	fdb->fd_addr = addr;
    140  0  stevel 	fdb->fd_shadow = pplist;
    141  0  stevel 	fdb->fd_procp = procp;
    142  0  stevel 
    143  0  stevel 	return (fdb);
    144  0  stevel }
    145  0  stevel 
    146  0  stevel void
    147  0  stevel fdb_set_iofunc(fdbuffer_t *fdb, fdb_iodone_t iofunc, void *ioargp, int flag)
    148  0  stevel {
    149  0  stevel 	ASSERT(fdb);
    150  0  stevel 	ASSERT(iofunc);
    151  0  stevel 	ASSERT((flag & ~FDB_ICALLBACK) == 0);
    152  0  stevel 
    153  0  stevel 	fdb->fd_iofunc = iofunc;
    154  0  stevel 	fdb->fd_iargp = ioargp;
    155  0  stevel 
    156  0  stevel 	mutex_enter(&fdb->fd_mutex);
    157  0  stevel 
    158  0  stevel 	if (flag & FDB_ICALLBACK)
    159  0  stevel 		fdb->fd_state |= FDB_ICALLBACK;
    160  0  stevel 
    161  0  stevel 	fdb->fd_state |= FDB_ASYNC;
    162  0  stevel 
    163  0  stevel 	mutex_exit(&fdb->fd_mutex);
    164  0  stevel }
    165  0  stevel 
    166  0  stevel int
    167  0  stevel fdb_get_error(fdbuffer_t *fdb)
    168  0  stevel {
    169  0  stevel 	return (fdb->fd_err);
    170  0  stevel }
    171  0  stevel 
    172  0  stevel void
    173  0  stevel fdb_free(fdbuffer_t *fdb)
    174  0  stevel {
    175  0  stevel 	fdb_holes_t *fdh, *fdhp;
    176  0  stevel 
    177  0  stevel 	DEBUGF(FDB_D_CREATE, (CE_NOTE, "?fdb_free: addr: %p flags: %x",
    178  0  stevel 	    (void *)fdb, fdb->fd_state));
    179  0  stevel 
    180  0  stevel 	ASSERT(fdb);
    181  0  stevel 	ASSERT(fdb->fd_iodispatch == 0);
    182  0  stevel 
    183  0  stevel 	if (fdb->fd_state & FDB_ZEROHOLE) {
    184  0  stevel 		fdb_zero_holes(fdb);
    185  0  stevel 	}
    186  0  stevel 
    187  0  stevel 	for (fdh = fdb->fd_holes; fdh; ) {
    188  0  stevel 		fdhp = fdh;
    189  0  stevel 		fdh = fdh->next_hole;
    190  0  stevel 		kmem_free(fdhp, sizeof (fdb_holes_t));
    191  0  stevel 	}
    192  0  stevel 
    193  0  stevel 	if (fdb->fd_parentbp != NULL) {
    194  0  stevel 		switch (fdb->fd_type) {
    195  0  stevel 		case FDB_PAGEIO:
    196  0  stevel 			pageio_done(fdb->fd_parentbp);
    197  0  stevel 			break;
    198  0  stevel 		case FDB_VADDR:
    199  0  stevel 			kmem_free(fdb->fd_parentbp, sizeof (struct buf));
    200  0  stevel 			break;
    201  0  stevel 		default:
    202  0  stevel 			cmn_err(CE_CONT, "?fdb_free: Unknown fdb type.");
    203  0  stevel 			break;
    204  0  stevel 		}
    205  0  stevel 	}
    206  0  stevel 
    207  0  stevel 	kmem_cache_free(fdb_cache, fdb);
    208  0  stevel 
    209  0  stevel }
    210  0  stevel 
    211  0  stevel /*
    212  0  stevel  * The offset should be from the begining of the buffer
    213  0  stevel  * it has nothing to do with file offset. This fact should be
    214  0  stevel  * reflected in the caller of this routine.
    215  0  stevel  */
    216  0  stevel 
    217  0  stevel void
    218  0  stevel fdb_add_hole(fdbuffer_t *fdb, u_offset_t off, size_t len)
    219  0  stevel {
    220  0  stevel 	fdb_holes_t *this_hole;
    221  0  stevel 
    222  0  stevel 	ASSERT(fdb);
    223  0  stevel 	ASSERT(off < fdb->fd_len);
    224  0  stevel 
    225  0  stevel 	DEBUGF(FDB_D_IO, (CE_NOTE, "?fdb_add_hole: off %llx len %lx",
    226  0  stevel 	    off, len));
    227  0  stevel 
    228  0  stevel 	this_hole = kmem_alloc(sizeof (fdb_holes_t), KM_SLEEP);
    229  0  stevel 	this_hole->off = off;
    230  0  stevel 	this_hole->len = len;
    231  0  stevel 
    232  0  stevel 	if (fdb->fd_holes == NULL || off < fdb->fd_holes->off) {
    233  0  stevel 		this_hole->next_hole = fdb->fd_holes;
    234  0  stevel 		fdb->fd_holes = this_hole;
    235  0  stevel 	} else {
    236  0  stevel 		fdb_holes_t *fdhp = fdb->fd_holes;
    237  0  stevel 
    238  0  stevel 		while (fdhp->next_hole && off > fdhp->next_hole->off)
    239  0  stevel 			fdhp = fdhp->next_hole;
    240  0  stevel 
    241  0  stevel 		this_hole->next_hole = fdhp->next_hole;
    242  0  stevel 		fdhp->next_hole = this_hole;
    243  0  stevel 	}
    244  0  stevel 
    245  0  stevel 	mutex_enter(&fdb->fd_mutex);
    246  0  stevel 
    247  0  stevel 	fdb->fd_iocount += len;
    248  0  stevel 
    249  0  stevel 	mutex_exit(&fdb->fd_mutex);
    250  0  stevel }
    251  0  stevel 
    252  0  stevel fdb_holes_t *
    253  0  stevel fdb_get_holes(fdbuffer_t *fdb)
    254  0  stevel {
    255  0  stevel 	ASSERT(fdb);
    256  0  stevel 
    257  0  stevel 	if (fdb->fd_state & FDB_ZEROHOLE) {
    258  0  stevel 		fdb_zero_holes(fdb);
    259  0  stevel 	}
    260  0  stevel 
    261  0  stevel 	return (fdb->fd_holes);
    262  0  stevel }
    263  0  stevel 
    264  0  stevel /*
    265  0  stevel  * Note that offsets refer to offsets from the begining of the buffer
    266  0  stevel  * and as such the memory should be cleared accordingly.
    267  0  stevel  */
    268  0  stevel 
    269  0  stevel static void
    270  0  stevel fdb_zero_holes(fdbuffer_t *fdb)
    271  0  stevel {
    272  0  stevel 	fdb_holes_t *fdh = fdb->fd_holes;
    273  0  stevel 	page_t *pp;
    274  0  stevel 
    275  0  stevel 	ASSERT(fdb);
    276  0  stevel 
    277  0  stevel 	if (!fdh)
    278  0  stevel 		return;
    279  0  stevel 
    280  0  stevel 	switch (fdb->fd_type) {
    281  0  stevel 	case FDB_PAGEIO:
    282  0  stevel 		pp = fdb->fd_pages;
    283  0  stevel 		while (fdh) {
    284  0  stevel 			fdb_holes_t *pfdh = fdh;
    285  0  stevel 			size_t l = fdh->len;
    286  0  stevel 			u_offset_t o = fdh->off;
    287  0  stevel 			ASSERT(pp);
    288  0  stevel 
    289  0  stevel 			do {
    290  0  stevel 				int  zerolen;
    291  0  stevel 				ASSERT(o >= pp->p_offset);
    292  0  stevel 
    293  0  stevel 				/*
    294  0  stevel 				 * This offset is wrong since
    295  0  stevel 				 * the offset passed from the pages
    296  0  stevel 				 * perspective starts at some virtual
    297  0  stevel 				 * address but the hole is relative
    298  0  stevel 				 * to the beginning of the fdbuffer.
    299  0  stevel 				 */
    300  0  stevel 				if (o >= pp->p_offset + PAGESIZE)
    301  0  stevel 					continue;
    302  0  stevel 
    303  0  stevel 				zerolen = min(PAGESIZE, l);
    304  0  stevel 
    305  0  stevel 				ASSERT(zerolen > 0);
    306  0  stevel 				ASSERT(zerolen <= PAGESIZE);
    307  0  stevel 
    308  0  stevel 				pagezero(pp, ((uintptr_t)o & PAGEOFFSET),
    309  0  stevel 				    zerolen);
    310  0  stevel 
    311  0  stevel 				l -= zerolen;
    312  0  stevel 				o += zerolen;
    313  0  stevel 
    314  0  stevel 				if (l == 0)
    315  0  stevel 					break;
    316  0  stevel 
    317  0  stevel 			} while (pp = page_list_next(pp));
    318  0  stevel 
    319  0  stevel 			if (!pp)
    320  0  stevel 				break;
    321  0  stevel 
    322  0  stevel 			fdh = fdh->next_hole;
    323  0  stevel 			kmem_free(pfdh, sizeof (fdb_holes_t));
    324  0  stevel 		}
    325  0  stevel 		break;
    326  0  stevel 	case FDB_VADDR:
    327  0  stevel 		while (fdh) {
    328  0  stevel 			fdb_holes_t *pfdh = fdh;
    329  0  stevel 
    330  0  stevel 			bzero(fdb->fd_addr + fdh->off, fdh->len);
    331  0  stevel 
    332  0  stevel 			fdh = fdh->next_hole;
    333  0  stevel 			kmem_free(pfdh, sizeof (fdb_holes_t));
    334  0  stevel 		}
    335  0  stevel 	default:
    336  0  stevel 		panic("fdb_zero_holes: Unknown fdb type.");
    337  0  stevel 		break;
    338  0  stevel 	}
    339  0  stevel }
    340  0  stevel 
    341  0  stevel 
    342  0  stevel buf_t *
    343  0  stevel fdb_iosetup(fdbuffer_t *fdb, u_offset_t off, size_t len, struct vnode *vp,
    344  0  stevel     int b_flags)
    345  0  stevel {
    346  0  stevel 	buf_t *bp;
    347  0  stevel 
    348  0  stevel 	DEBUGF(FDB_D_IO, (CE_NOTE,
    349  0  stevel 	    "?fdb_iosetup: off: %llx len: %lux fdb: len: %lux flags: %x",
    350  0  stevel 	    off, len, fdb->fd_len, fdb->fd_state));
    351  0  stevel 
    352  0  stevel 	ASSERT(fdb);
    353  0  stevel 
    354  0  stevel 	mutex_enter(&fdb->fd_mutex);
    355  0  stevel 
    356  0  stevel 	ASSERT(((b_flags & B_READ) && (fdb->fd_state & FDB_READ)) ||
    357  0  stevel 	    ((b_flags & B_WRITE) && (fdb->fd_state & FDB_WRITE)));
    358  0  stevel 	/*
    359  0  stevel 	 * The fdb can be used either in sync or async mode, if the
    360  0  stevel 	 * buffer has not been used it may be used in either mode, but
    361  0  stevel 	 * once you have started to use the buf in either mode all
    362  0  stevel 	 * subsequent i/o requests must take place the same way.
    363  0  stevel 	 */
    364  0  stevel 
    365  0  stevel 	ASSERT(((b_flags & B_ASYNC) &&
    366  0  stevel 	    ((fdb->fd_state & FDB_ASYNC) || !(fdb->fd_state & FDB_SYNC))) ||
    367  0  stevel 	    (!(b_flags & B_ASYNC) &&
    368  0  stevel 	    ((fdb->fd_state & FDB_SYNC) || !(fdb->fd_state & FDB_ASYNC))));
    369  0  stevel 
    370  0  stevel 
    371  0  stevel 	fdb->fd_state |= b_flags & B_ASYNC ? FDB_ASYNC : FDB_SYNC;
    372  0  stevel 
    373  0  stevel 	fdb->fd_iodispatch++;
    374  0  stevel 
    375  0  stevel 	ASSERT((fdb->fd_state & FDB_ASYNC && fdb->fd_iofunc != NULL) ||
    376  0  stevel 	    fdb->fd_state & FDB_SYNC);
    377  0  stevel 
    378  0  stevel 	mutex_exit(&fdb->fd_mutex);
    379  0  stevel 
    380  0  stevel 	ASSERT((len & (DEV_BSIZE - 1)) == 0);
    381  0  stevel 	ASSERT(off+len <= fdb->fd_len);
    382  0  stevel 
    383  0  stevel 	switch (fdb->fd_type) {
    384  0  stevel 	case FDB_PAGEIO:
    385  0  stevel 		if (fdb->fd_parentbp == NULL) {
    386  0  stevel 			bp = pageio_setup(fdb->fd_pages, len, vp, b_flags);
    387  0  stevel 			fdb->fd_parentbp = bp;
    388  0  stevel 		}
    389  0  stevel 		break;
    390  0  stevel 	case FDB_VADDR:
    391  0  stevel 		if (fdb->fd_parentbp == NULL) {
    392  0  stevel 
    393  0  stevel 			bp = kmem_alloc(sizeof (buf_t), KM_SLEEP);
    394  0  stevel 			bioinit(bp);
    395  0  stevel 			bp->b_error = 0;
    396  0  stevel 			bp->b_proc = fdb->fd_procp;
    397  0  stevel 			bp->b_flags = b_flags | B_BUSY | B_PHYS;
    398  0  stevel 			bp->b_bcount = len;
    399  0  stevel 			bp->b_un.b_addr = fdb->fd_addr;
    400  0  stevel 			bp->b_shadow = fdb->fd_shadow;
    401  0  stevel 			if (fdb->fd_shadow != NULL)
    402  0  stevel 				bp->b_flags |= B_SHADOW;
    403  0  stevel 			fdb->fd_parentbp = bp;
    404  0  stevel 		}
    405  0  stevel 		break;
    406  0  stevel 	default:
    407  0  stevel 		panic("fdb_iosetup: Unsupported fdb type.");
    408  0  stevel 		break;
    409  0  stevel 	};
    410  0  stevel 
    411  0  stevel 	bp = bioclone(fdb->fd_parentbp, off, len, 0, 0,
    412  0  stevel 	    (b_flags & B_ASYNC) ? (int (*)())fdb_iodone : NULL,
    413  0  stevel 	    NULL, KM_SLEEP);
    414  0  stevel 
    415  0  stevel 	bp->b_forw = (struct buf *)fdb;
    416  0  stevel 
    417  0  stevel 	if (b_flags & B_ASYNC)
    418  0  stevel 		bp->b_flags |= B_ASYNC;
    419  0  stevel 
    420  0  stevel 	return (bp);
    421  0  stevel }
    422  0  stevel 
    423  0  stevel size_t
    424  0  stevel fdb_get_iolen(fdbuffer_t *fdb)
    425  0  stevel {
    426  0  stevel 	ASSERT(fdb);
    427  0  stevel 	ASSERT(fdb->fd_iodispatch == 0);
    428  0  stevel 
    429  0  stevel 	return (fdb->fd_iocount - fdb->fd_resid);
    430  0  stevel }
    431  0  stevel 
    432  0  stevel void
    433  0  stevel fdb_ioerrdone(fdbuffer_t *fdb, int error)
    434  0  stevel {
    435  0  stevel 	ASSERT(fdb);
    436  0  stevel 	ASSERT(fdb->fd_state & FDB_ASYNC);
    437  0  stevel 
    438  0  stevel 	DEBUGF(FDB_D_IO, (CE_NOTE,
    439  0  stevel 	    "?fdb_ioerrdone: fdb: len: %lux flags: %x error: %d",
    440  0  stevel 	    fdb->fd_len, fdb->fd_state, error));
    441  0  stevel 
    442  0  stevel 	mutex_enter(&fdb->fd_mutex);
    443  0  stevel 
    444  0  stevel 	fdb->fd_err = error;
    445  0  stevel 
    446  0  stevel 	if (error)
    447  0  stevel 		fdb->fd_state |= FDB_ERROR;
    448  0  stevel 	else
    449  0  stevel 		fdb->fd_state |= FDB_DONE;
    450  0  stevel 
    451  0  stevel 	/*
    452  0  stevel 	 * If there is outstanding i/o return wainting for i/o's to complete.
    453  0  stevel 	 */
    454  0  stevel 	if (fdb->fd_iodispatch > 0) {
    455  0  stevel 		mutex_exit(&fdb->fd_mutex);
    456  0  stevel 		return;
    457  0  stevel 	}
    458  0  stevel 
    459  0  stevel 	mutex_exit(&fdb->fd_mutex);
    460  0  stevel 	fdb->fd_iofunc(fdb, fdb->fd_iargp, NULL);
    461  0  stevel }
    462  0  stevel 
    463  0  stevel void
    464  0  stevel fdb_iodone(buf_t *bp)
    465  0  stevel {
    466  0  stevel 	fdbuffer_t *fdb = (fdbuffer_t *)bp->b_forw;
    467  0  stevel 	int	error, isasync;
    468  0  stevel 	int	icallback;
    469  0  stevel 
    470  0  stevel 	ASSERT(fdb);
    471  0  stevel 
    472  0  stevel 	DEBUGF(FDB_D_IO, (CE_NOTE,
    473  0  stevel 	    "?fdb_iodone: fdb: len: %lux flags: %x error: %d",
    474  0  stevel 	    fdb->fd_len, fdb->fd_state, geterror(bp)));
    475  0  stevel 
    476  0  stevel 	if (bp->b_flags & B_REMAPPED)
    477  0  stevel 		bp_mapout(bp);
    478  0  stevel 
    479  0  stevel 	mutex_enter(&fdb->fd_mutex);
    480  0  stevel 
    481  0  stevel 	icallback = fdb->fd_state & FDB_ICALLBACK;
    482  0  stevel 	isasync = fdb->fd_state & FDB_ASYNC;
    483  0  stevel 
    484  0  stevel 	ASSERT(fdb->fd_iodispatch > 0);
    485  0  stevel 	fdb->fd_iodispatch--;
    486  0  stevel 
    487  0  stevel 	if (error = geterror(bp)) {
    488  0  stevel 		fdb->fd_err = error;
    489  0  stevel 		if (bp->b_resid)
    490  0  stevel 			fdb->fd_resid += bp->b_resid;
    491  0  stevel 		else
    492  0  stevel 			fdb->fd_resid += bp->b_bcount;
    493  0  stevel 	}
    494  0  stevel 
    495  0  stevel 	fdb->fd_iocount += bp->b_bcount;
    496  0  stevel 
    497  0  stevel 	/*
    498  0  stevel 	 * ioack collects the total amount of i/o accounted for
    499  0  stevel 	 * this includes:
    500  0  stevel 	 *
    501  0  stevel 	 *	- i/o completed
    502  0  stevel 	 *	- i/o attempted but not completed,
    503  0  stevel 	 *	- i/o not done due to holes.
    504  0  stevel 	 *
    505  0  stevel 	 * Once the entire i/o ranges has been accounted for we'll
    506  0  stevel 	 * call the async function associated with the fdb.
    507  0  stevel 	 *
    508  0  stevel 	 */
    509  0  stevel 
    510  0  stevel 	if ((fdb->fd_iodispatch == 0) &&
    511  0  stevel 	    (fdb->fd_state & (FDB_ERROR|FDB_DONE))) {
    512  0  stevel 
    513  0  stevel 		mutex_exit(&fdb->fd_mutex);
    514  0  stevel 
    515  0  stevel 		if (isasync || icallback) {
    516  0  stevel 			fdb->fd_iofunc(fdb, fdb->fd_iargp, bp);
    517  0  stevel 		}
    518  0  stevel 
    519  0  stevel 	} else {
    520  0  stevel 
    521  0  stevel 		mutex_exit(&fdb->fd_mutex);
    522  0  stevel 
    523  0  stevel 		if (icallback) {
    524  0  stevel 			fdb->fd_iofunc(fdb, fdb->fd_iargp, bp);
    525  0  stevel 		}
    526  0  stevel 	}
    527  0  stevel 
    528  0  stevel 	freerbuf(bp);
    529  0  stevel }
    530