Home | History | Annotate | Download | only in zfs
      1 /*
      2  * CDDL HEADER START
      3  *
      4  * The contents of this file are subject to the terms of the
      5  * Common Development and Distribution License (the "License").
      6  * You may not use this file except in compliance with the License.
      7  *
      8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
      9  * or http://www.opensolaris.org/os/licensing.
     10  * See the License for the specific language governing permissions
     11  * and limitations under the License.
     12  *
     13  * When distributing Covered Code, include this CDDL HEADER in each
     14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
     15  * If applicable, add the following below this CDDL HEADER, with the
     16  * fields enclosed by brackets "[]" replaced with your own identifying
     17  * information: Portions Copyright [yyyy] [name of copyright owner]
     18  *
     19  * CDDL HEADER END
     20  */
     21 /*
     22  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
     23  * Use is subject to license terms.
     24  */
     25 
     26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
     27 
     28 #include <sys/zfs_context.h>
     29 #include <sys/dbuf.h>
     30 #include <sys/dnode.h>
     31 #include <sys/dmu.h>
     32 #include <sys/dmu_impl.h>
     33 #include <sys/dmu_tx.h>
     34 #include <sys/dmu_objset.h>
     35 #include <sys/dsl_dir.h>
     36 #include <sys/dsl_dataset.h>
     37 #include <sys/spa.h>
     38 #include <sys/zio.h>
     39 #include <sys/dmu_zfetch.h>
     40 
     41 static int free_range_compar(const void *node1, const void *node2);
     42 
     43 static kmem_cache_t *dnode_cache;
     44 
     45 static dnode_phys_t dnode_phys_zero;
     46 
     47 int zfs_default_bs = SPA_MINBLOCKSHIFT;
     48 int zfs_default_ibs = DN_MAX_INDBLKSHIFT;
     49 
     50 /* ARGSUSED */
     51 static int
     52 dnode_cons(void *arg, void *unused, int kmflag)
     53 {
     54 	int i;
     55 	dnode_t *dn = arg;
     56 	bzero(dn, sizeof (dnode_t));
     57 
     58 	rw_init(&dn->dn_struct_rwlock, NULL, RW_DEFAULT, NULL);
     59 	mutex_init(&dn->dn_mtx, NULL, MUTEX_DEFAULT, NULL);
     60 	mutex_init(&dn->dn_dbufs_mtx, NULL, MUTEX_DEFAULT, NULL);
     61 	refcount_create(&dn->dn_holds);
     62 	refcount_create(&dn->dn_tx_holds);
     63 
     64 	for (i = 0; i < TXG_SIZE; i++) {
     65 		avl_create(&dn->dn_ranges[i], free_range_compar,
     66 		    sizeof (free_range_t),
     67 		    offsetof(struct free_range, fr_node));
     68 		list_create(&dn->dn_dirty_records[i],
     69 		    sizeof (dbuf_dirty_record_t),
     70 		    offsetof(dbuf_dirty_record_t, dr_dirty_node));
     71 	}
     72 
     73 	list_create(&dn->dn_dbufs, sizeof (dmu_buf_impl_t),
     74 	    offsetof(dmu_buf_impl_t, db_link));
     75 
     76 	return (0);
     77 }
     78 
     79 /* ARGSUSED */
     80 static void
     81 dnode_dest(void *arg, void *unused)
     82 {
     83 	int i;
     84 	dnode_t *dn = arg;
     85 
     86 	rw_destroy(&dn->dn_struct_rwlock);
     87 	mutex_destroy(&dn->dn_mtx);
     88 	mutex_destroy(&dn->dn_dbufs_mtx);
     89 	refcount_destroy(&dn->dn_holds);
     90 	refcount_destroy(&dn->dn_tx_holds);
     91 
     92 	for (i = 0; i < TXG_SIZE; i++) {
     93 		avl_destroy(&dn->dn_ranges[i]);
     94 		list_destroy(&dn->dn_dirty_records[i]);
     95 	}
     96 
     97 	list_destroy(&dn->dn_dbufs);
     98 }
     99 
    100 void
    101 dnode_init(void)
    102 {
    103 	dnode_cache = kmem_cache_create("dnode_t",
    104 	    sizeof (dnode_t),
    105 	    0, dnode_cons, dnode_dest, NULL, NULL, NULL, 0);
    106 }
    107 
    108 void
    109 dnode_fini(void)
    110 {
    111 	kmem_cache_destroy(dnode_cache);
    112 }
    113 
    114 
    115 #ifdef ZFS_DEBUG
    116 void
    117 dnode_verify(dnode_t *dn)
    118 {
    119 	int drop_struct_lock = FALSE;
    120 
    121 	ASSERT(dn->dn_phys);
    122 	ASSERT(dn->dn_objset);
    123 
    124 	ASSERT(dn->dn_phys->dn_type < DMU_OT_NUMTYPES);
    125 
    126 	if (!(zfs_flags & ZFS_DEBUG_DNODE_VERIFY))
    127 		return;
    128 
    129 	if (!RW_WRITE_HELD(&dn->dn_struct_rwlock)) {
    130 		rw_enter(&dn->dn_struct_rwlock, RW_READER);
    131 		drop_struct_lock = TRUE;
    132 	}
    133 	if (dn->dn_phys->dn_type != DMU_OT_NONE || dn->dn_allocated_txg != 0) {
    134 		int i;
    135 		ASSERT3U(dn->dn_indblkshift, >=, 0);
    136 		ASSERT3U(dn->dn_indblkshift, <=, SPA_MAXBLOCKSHIFT);
    137 		if (dn->dn_datablkshift) {
    138 			ASSERT3U(dn->dn_datablkshift, >=, SPA_MINBLOCKSHIFT);
    139 			ASSERT3U(dn->dn_datablkshift, <=, SPA_MAXBLOCKSHIFT);
    140 			ASSERT3U(1<<dn->dn_datablkshift, ==, dn->dn_datablksz);
    141 		}
    142 		ASSERT3U(dn->dn_nlevels, <=, 30);
    143 		ASSERT3U(dn->dn_type, <=, DMU_OT_NUMTYPES);
    144 		ASSERT3U(dn->dn_nblkptr, >=, 1);
    145 		ASSERT3U(dn->dn_nblkptr, <=, DN_MAX_NBLKPTR);
    146 		ASSERT3U(dn->dn_bonuslen, <=, DN_MAX_BONUSLEN);
    147 		ASSERT3U(dn->dn_datablksz, ==,
    148 		    dn->dn_datablkszsec << SPA_MINBLOCKSHIFT);
    149 		ASSERT3U(ISP2(dn->dn_datablksz), ==, dn->dn_datablkshift != 0);
    150 		ASSERT3U((dn->dn_nblkptr - 1) * sizeof (blkptr_t) +
    151 		    dn->dn_bonuslen, <=, DN_MAX_BONUSLEN);
    152 		for (i = 0; i < TXG_SIZE; i++) {
    153 			ASSERT3U(dn->dn_next_nlevels[i], <=, dn->dn_nlevels);
    154 		}
    155 	}
    156 	if (dn->dn_phys->dn_type != DMU_OT_NONE)
    157 		ASSERT3U(dn->dn_phys->dn_nlevels, <=, dn->dn_nlevels);
    158 	ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || dn->dn_dbuf != NULL);
    159 	if (dn->dn_dbuf != NULL) {
    160 		ASSERT3P(dn->dn_phys, ==,
    161 		    (dnode_phys_t *)dn->dn_dbuf->db.db_data +
    162 		    (dn->dn_object % (dn->dn_dbuf->db.db_size >> DNODE_SHIFT)));
    163 	}
    164 	if (drop_struct_lock)
    165 		rw_exit(&dn->dn_struct_rwlock);
    166 }
    167 #endif
    168 
    169 void
    170 dnode_byteswap(dnode_phys_t *dnp)
    171 {
    172 	uint64_t *buf64 = (void*)&dnp->dn_blkptr;
    173 	int i;
    174 
    175 	if (dnp->dn_type == DMU_OT_NONE) {
    176 		bzero(dnp, sizeof (dnode_phys_t));
    177 		return;
    178 	}
    179 
    180 	dnp->dn_datablkszsec = BSWAP_16(dnp->dn_datablkszsec);
    181 	dnp->dn_bonuslen = BSWAP_16(dnp->dn_bonuslen);
    182 	dnp->dn_maxblkid = BSWAP_64(dnp->dn_maxblkid);
    183 	dnp->dn_used = BSWAP_64(dnp->dn_used);
    184 
    185 	/*
    186 	 * dn_nblkptr is only one byte, so it's OK to read it in either
    187 	 * byte order.  We can't read dn_bouslen.
    188 	 */
    189 	ASSERT(dnp->dn_indblkshift <= SPA_MAXBLOCKSHIFT);
    190 	ASSERT(dnp->dn_nblkptr <= DN_MAX_NBLKPTR);
    191 	for (i = 0; i < dnp->dn_nblkptr * sizeof (blkptr_t)/8; i++)
    192 		buf64[i] = BSWAP_64(buf64[i]);
    193 
    194 	/*
    195 	 * OK to check dn_bonuslen for zero, because it won't matter if
    196 	 * we have the wrong byte order.  This is necessary because the
    197 	 * dnode dnode is smaller than a regular dnode.
    198 	 */
    199 	if (dnp->dn_bonuslen != 0) {
    200 		/*
    201 		 * Note that the bonus length calculated here may be
    202 		 * longer than the actual bonus buffer.  This is because
    203 		 * we always put the bonus buffer after the last block
    204 		 * pointer (instead of packing it against the end of the
    205 		 * dnode buffer).
    206 		 */
    207 		int off = (dnp->dn_nblkptr-1) * sizeof (blkptr_t);
    208 		size_t len = DN_MAX_BONUSLEN - off;
    209 		ASSERT3U(dnp->dn_bonustype, <, DMU_OT_NUMTYPES);
    210 		dmu_ot[dnp->dn_bonustype].ot_byteswap(dnp->dn_bonus + off, len);
    211 	}
    212 }
    213 
    214 void
    215 dnode_buf_byteswap(void *vbuf, size_t size)
    216 {
    217 	dnode_phys_t *buf = vbuf;
    218 	int i;
    219 
    220 	ASSERT3U(sizeof (dnode_phys_t), ==, (1<<DNODE_SHIFT));
    221 	ASSERT((size & (sizeof (dnode_phys_t)-1)) == 0);
    222 
    223 	size >>= DNODE_SHIFT;
    224 	for (i = 0; i < size; i++) {
    225 		dnode_byteswap(buf);
    226 		buf++;
    227 	}
    228 }
    229 
    230 static int
    231 free_range_compar(const void *node1, const void *node2)
    232 {
    233 	const free_range_t *rp1 = node1;
    234 	const free_range_t *rp2 = node2;
    235 
    236 	if (rp1->fr_blkid < rp2->fr_blkid)
    237 		return (-1);
    238 	else if (rp1->fr_blkid > rp2->fr_blkid)
    239 		return (1);
    240 	else return (0);
    241 }
    242 
    243 void
    244 dnode_setbonuslen(dnode_t *dn, int newsize, dmu_tx_t *tx)
    245 {
    246 	ASSERT3U(refcount_count(&dn->dn_holds), >=, 1);
    247 
    248 	dnode_setdirty(dn, tx);
    249 	rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
    250 	ASSERT3U(newsize, <=, DN_MAX_BONUSLEN -
    251 	    (dn->dn_nblkptr-1) * sizeof (blkptr_t));
    252 	dn->dn_bonuslen = newsize;
    253 	if (newsize == 0)
    254 		dn->dn_next_bonuslen[tx->tx_txg & TXG_MASK] = DN_ZERO_BONUSLEN;
    255 	else
    256 		dn->dn_next_bonuslen[tx->tx_txg & TXG_MASK] = dn->dn_bonuslen;
    257 	rw_exit(&dn->dn_struct_rwlock);
    258 }
    259 
    260 static void
    261 dnode_setdblksz(dnode_t *dn, int size)
    262 {
    263 	ASSERT3U(P2PHASE(size, SPA_MINBLOCKSIZE), ==, 0);
    264 	ASSERT3U(size, <=, SPA_MAXBLOCKSIZE);
    265 	ASSERT3U(size, >=, SPA_MINBLOCKSIZE);
    266 	ASSERT3U(size >> SPA_MINBLOCKSHIFT, <,
    267 	    1<<(sizeof (dn->dn_phys->dn_datablkszsec) * 8));
    268 	dn->dn_datablksz = size;
    269 	dn->dn_datablkszsec = size >> SPA_MINBLOCKSHIFT;
    270 	dn->dn_datablkshift = ISP2(size) ? highbit(size - 1) : 0;
    271 }
    272 
    273 static dnode_t *
    274 dnode_create(objset_impl_t *os, dnode_phys_t *dnp, dmu_buf_impl_t *db,
    275     uint64_t object)
    276 {
    277 	dnode_t *dn = kmem_cache_alloc(dnode_cache, KM_SLEEP);
    278 	(void) dnode_cons(dn, NULL, 0); /* XXX */
    279 
    280 	dn->dn_objset = os;
    281 	dn->dn_object = object;
    282 	dn->dn_dbuf = db;
    283 	dn->dn_phys = dnp;
    284 
    285 	if (dnp->dn_datablkszsec)
    286 		dnode_setdblksz(dn, dnp->dn_datablkszsec << SPA_MINBLOCKSHIFT);
    287 	dn->dn_indblkshift = dnp->dn_indblkshift;
    288 	dn->dn_nlevels = dnp->dn_nlevels;
    289 	dn->dn_type = dnp->dn_type;
    290 	dn->dn_nblkptr = dnp->dn_nblkptr;
    291 	dn->dn_checksum = dnp->dn_checksum;
    292 	dn->dn_compress = dnp->dn_compress;
    293 	dn->dn_bonustype = dnp->dn_bonustype;
    294 	dn->dn_bonuslen = dnp->dn_bonuslen;
    295 	dn->dn_maxblkid = dnp->dn_maxblkid;
    296 
    297 	dmu_zfetch_init(&dn->dn_zfetch, dn);
    298 
    299 	ASSERT(dn->dn_phys->dn_type < DMU_OT_NUMTYPES);
    300 	mutex_enter(&os->os_lock);
    301 	list_insert_head(&os->os_dnodes, dn);
    302 	mutex_exit(&os->os_lock);
    303 
    304 	arc_space_consume(sizeof (dnode_t));
    305 	return (dn);
    306 }
    307 
    308 static void
    309 dnode_destroy(dnode_t *dn)
    310 {
    311 	objset_impl_t *os = dn->dn_objset;
    312 
    313 #ifdef ZFS_DEBUG
    314 	int i;
    315 
    316 	for (i = 0; i < TXG_SIZE; i++) {
    317 		ASSERT(!list_link_active(&dn->dn_dirty_link[i]));
    318 		ASSERT(NULL == list_head(&dn->dn_dirty_records[i]));
    319 		ASSERT(0 == avl_numnodes(&dn->dn_ranges[i]));
    320 	}
    321 	ASSERT(NULL == list_head(&dn->dn_dbufs));
    322 #endif
    323 
    324 	mutex_enter(&os->os_lock);
    325 	list_remove(&os->os_dnodes, dn);
    326 	mutex_exit(&os->os_lock);
    327 
    328 	if (dn->dn_dirtyctx_firstset) {
    329 		kmem_free(dn->dn_dirtyctx_firstset, 1);
    330 		dn->dn_dirtyctx_firstset = NULL;
    331 	}
    332 	dmu_zfetch_rele(&dn->dn_zfetch);
    333 	if (dn->dn_bonus) {
    334 		mutex_enter(&dn->dn_bonus->db_mtx);
    335 		dbuf_evict(dn->dn_bonus);
    336 		dn->dn_bonus = NULL;
    337 	}
    338 	kmem_cache_free(dnode_cache, dn);
    339 	arc_space_return(sizeof (dnode_t));
    340 }
    341 
    342 void
    343 dnode_allocate(dnode_t *dn, dmu_object_type_t ot, int blocksize, int ibs,
    344     dmu_object_type_t bonustype, int bonuslen, dmu_tx_t *tx)
    345 {
    346 	int i;
    347 
    348 	if (blocksize == 0)
    349 		blocksize = 1 << zfs_default_bs;
    350 	else if (blocksize > SPA_MAXBLOCKSIZE)
    351 		blocksize = SPA_MAXBLOCKSIZE;
    352 	else
    353 		blocksize = P2ROUNDUP(blocksize, SPA_MINBLOCKSIZE);
    354 
    355 	if (ibs == 0)
    356 		ibs = zfs_default_ibs;
    357 
    358 	ibs = MIN(MAX(ibs, DN_MIN_INDBLKSHIFT), DN_MAX_INDBLKSHIFT);
    359 
    360 	dprintf("os=%p obj=%llu txg=%llu blocksize=%d ibs=%d\n", dn->dn_objset,
    361 	    dn->dn_object, tx->tx_txg, blocksize, ibs);
    362 
    363 	ASSERT(dn->dn_type == DMU_OT_NONE);
    364 	ASSERT(bcmp(dn->dn_phys, &dnode_phys_zero, sizeof (dnode_phys_t)) == 0);
    365 	ASSERT(dn->dn_phys->dn_type == DMU_OT_NONE);
    366 	ASSERT(ot != DMU_OT_NONE);
    367 	ASSERT3U(ot, <, DMU_OT_NUMTYPES);
    368 	ASSERT((bonustype == DMU_OT_NONE && bonuslen == 0) ||
    369 	    (bonustype != DMU_OT_NONE && bonuslen != 0));
    370 	ASSERT3U(bonustype, <, DMU_OT_NUMTYPES);
    371 	ASSERT3U(bonuslen, <=, DN_MAX_BONUSLEN);
    372 	ASSERT(dn->dn_type == DMU_OT_NONE);
    373 	ASSERT3U(dn->dn_maxblkid, ==, 0);
    374 	ASSERT3U(dn->dn_allocated_txg, ==, 0);
    375 	ASSERT3U(dn->dn_assigned_txg, ==, 0);
    376 	ASSERT(refcount_is_zero(&dn->dn_tx_holds));
    377 	ASSERT3U(refcount_count(&dn->dn_holds), <=, 1);
    378 	ASSERT3P(list_head(&dn->dn_dbufs), ==, NULL);
    379 
    380 	for (i = 0; i < TXG_SIZE; i++) {
    381 		ASSERT3U(dn->dn_next_nlevels[i], ==, 0);
    382 		ASSERT3U(dn->dn_next_indblkshift[i], ==, 0);
    383 		ASSERT3U(dn->dn_next_bonuslen[i], ==, 0);
    384 		ASSERT3U(dn->dn_next_blksz[i], ==, 0);
    385 		ASSERT(!list_link_active(&dn->dn_dirty_link[i]));
    386 		ASSERT3P(list_head(&dn->dn_dirty_records[i]), ==, NULL);
    387 		ASSERT3U(avl_numnodes(&dn->dn_ranges[i]), ==, 0);
    388 	}
    389 
    390 	dn->dn_type = ot;
    391 	dnode_setdblksz(dn, blocksize);
    392 	dn->dn_indblkshift = ibs;
    393 	dn->dn_nlevels = 1;
    394 	dn->dn_nblkptr = 1 + ((DN_MAX_BONUSLEN - bonuslen) >> SPA_BLKPTRSHIFT);
    395 	dn->dn_bonustype = bonustype;
    396 	dn->dn_bonuslen = bonuslen;
    397 	dn->dn_checksum = ZIO_CHECKSUM_INHERIT;
    398 	dn->dn_compress = ZIO_COMPRESS_INHERIT;
    399 	dn->dn_dirtyctx = 0;
    400 
    401 	dn->dn_free_txg = 0;
    402 	if (dn->dn_dirtyctx_firstset) {
    403 		kmem_free(dn->dn_dirtyctx_firstset, 1);
    404 		dn->dn_dirtyctx_firstset = NULL;
    405 	}
    406 
    407 	dn->dn_allocated_txg = tx->tx_txg;
    408 
    409 	dnode_setdirty(dn, tx);
    410 	dn->dn_next_indblkshift[tx->tx_txg & TXG_MASK] = ibs;
    411 	dn->dn_next_bonuslen[tx->tx_txg & TXG_MASK] = dn->dn_bonuslen;
    412 	dn->dn_next_blksz[tx->tx_txg & TXG_MASK] = dn->dn_datablksz;
    413 }
    414 
    415 void
    416 dnode_reallocate(dnode_t *dn, dmu_object_type_t ot, int blocksize,
    417     dmu_object_type_t bonustype, int bonuslen, dmu_tx_t *tx)
    418 {
    419 	int i, old_nblkptr;
    420 	dmu_buf_impl_t *db = NULL;
    421 
    422 	ASSERT3U(blocksize, >=, SPA_MINBLOCKSIZE);
    423 	ASSERT3U(blocksize, <=, SPA_MAXBLOCKSIZE);
    424 	ASSERT3U(blocksize % SPA_MINBLOCKSIZE, ==, 0);
    425 	ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT || dmu_tx_private_ok(tx));
    426 	ASSERT(tx->tx_txg != 0);
    427 	ASSERT((bonustype == DMU_OT_NONE && bonuslen == 0) ||
    428 	    (bonustype != DMU_OT_NONE && bonuslen != 0));
    429 	ASSERT3U(bonustype, <, DMU_OT_NUMTYPES);
    430 	ASSERT3U(bonuslen, <=, DN_MAX_BONUSLEN);
    431 
    432 	for (i = 0; i < TXG_SIZE; i++)
    433 		ASSERT(!list_link_active(&dn->dn_dirty_link[i]));
    434 
    435 	/* clean up any unreferenced dbufs */
    436 	dnode_evict_dbufs(dn);
    437 	ASSERT3P(list_head(&dn->dn_dbufs), ==, NULL);
    438 
    439 	/*
    440 	 * XXX I should really have a generation number to tell if we
    441 	 * need to do this...
    442 	 */
    443 	if (blocksize != dn->dn_datablksz ||
    444 	    dn->dn_bonustype != bonustype || dn->dn_bonuslen != bonuslen) {
    445 		/* free all old data */
    446 		dnode_free_range(dn, 0, -1ULL, tx);
    447 	}
    448 
    449 	/* change blocksize */
    450 	rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
    451 	if (blocksize != dn->dn_datablksz &&
    452 	    (!BP_IS_HOLE(&dn->dn_phys->dn_blkptr[0]) ||
    453 	    list_head(&dn->dn_dbufs) != NULL)) {
    454 		db = dbuf_hold(dn, 0, FTAG);
    455 		dbuf_new_size(db, blocksize, tx);
    456 	}
    457 	dnode_setdblksz(dn, blocksize);
    458 	dnode_setdirty(dn, tx);
    459 	dn->dn_next_bonuslen[tx->tx_txg&TXG_MASK] = bonuslen;
    460 	dn->dn_next_blksz[tx->tx_txg&TXG_MASK] = blocksize;
    461 	rw_exit(&dn->dn_struct_rwlock);
    462 	if (db)
    463 		dbuf_rele(db, FTAG);
    464 
    465 	/* change type */
    466 	dn->dn_type = ot;
    467 
    468 	/* change bonus size and type */
    469 	mutex_enter(&dn->dn_mtx);
    470 	old_nblkptr = dn->dn_nblkptr;
    471 	dn->dn_bonustype = bonustype;
    472 	dn->dn_bonuslen = bonuslen;
    473 	dn->dn_nblkptr = 1 + ((DN_MAX_BONUSLEN - bonuslen) >> SPA_BLKPTRSHIFT);
    474 	dn->dn_checksum = ZIO_CHECKSUM_INHERIT;
    475 	dn->dn_compress = ZIO_COMPRESS_INHERIT;
    476 	ASSERT3U(dn->dn_nblkptr, <=, DN_MAX_NBLKPTR);
    477 
    478 	/* XXX - for now, we can't make nblkptr smaller */
    479 	ASSERT3U(dn->dn_nblkptr, >=, old_nblkptr);
    480 
    481 	/* fix up the bonus db_size if dn_nblkptr has changed */
    482 	if (dn->dn_bonus && dn->dn_bonuslen != old_nblkptr) {
    483 		dn->dn_bonus->db.db_size =
    484 		    DN_MAX_BONUSLEN - (dn->dn_nblkptr-1) * sizeof (blkptr_t);
    485 		ASSERT(dn->dn_bonuslen <= dn->dn_bonus->db.db_size);
    486 	}
    487 
    488 	dn->dn_allocated_txg = tx->tx_txg;
    489 	mutex_exit(&dn->dn_mtx);
    490 }
    491 
    492 void
    493 dnode_special_close(dnode_t *dn)
    494 {
    495 	/*
    496 	 * Wait for final references to the dnode to clear.  This can
    497 	 * only happen if the arc is asyncronously evicting state that
    498 	 * has a hold on this dnode while we are trying to evict this
    499 	 * dnode.
    500 	 */
    501 	while (refcount_count(&dn->dn_holds) > 0)
    502 		delay(1);
    503 	dnode_destroy(dn);
    504 }
    505 
    506 dnode_t *
    507 dnode_special_open(objset_impl_t *os, dnode_phys_t *dnp, uint64_t object)
    508 {
    509 	dnode_t *dn = dnode_create(os, dnp, NULL, object);
    510 	DNODE_VERIFY(dn);
    511 	return (dn);
    512 }
    513 
    514 static void
    515 dnode_buf_pageout(dmu_buf_t *db, void *arg)
    516 {
    517 	dnode_t **children_dnodes = arg;
    518 	int i;
    519 	int epb = db->db_size >> DNODE_SHIFT;
    520 
    521 	for (i = 0; i < epb; i++) {
    522 		dnode_t *dn = children_dnodes[i];
    523 		int n;
    524 
    525 		if (dn == NULL)
    526 			continue;
    527 #ifdef ZFS_DEBUG
    528 		/*
    529 		 * If there are holds on this dnode, then there should
    530 		 * be holds on the dnode's containing dbuf as well; thus
    531 		 * it wouldn't be eligable for eviction and this function
    532 		 * would not have been called.
    533 		 */
    534 		ASSERT(refcount_is_zero(&dn->dn_holds));
    535 		ASSERT(list_head(&dn->dn_dbufs) == NULL);
    536 		ASSERT(refcount_is_zero(&dn->dn_tx_holds));
    537 
    538 		for (n = 0; n < TXG_SIZE; n++)
    539 			ASSERT(!list_link_active(&dn->dn_dirty_link[n]));
    540 #endif
    541 		children_dnodes[i] = NULL;
    542 		dnode_destroy(dn);
    543 	}
    544 	kmem_free(children_dnodes, epb * sizeof (dnode_t *));
    545 }
    546 
    547 /*
    548  * errors:
    549  * EINVAL - invalid object number.
    550  * EIO - i/o error.
    551  * succeeds even for free dnodes.
    552  */
    553 int
    554 dnode_hold_impl(objset_impl_t *os, uint64_t object, int flag,
    555     void *tag, dnode_t **dnp)
    556 {
    557 	int epb, idx, err;
    558 	int drop_struct_lock = FALSE;
    559 	int type;
    560 	uint64_t blk;
    561 	dnode_t *mdn, *dn;
    562 	dmu_buf_impl_t *db;
    563 	dnode_t **children_dnodes;
    564 
    565 	if (object == 0 || object >= DN_MAX_OBJECT)
    566 		return (EINVAL);
    567 
    568 	mdn = os->os_meta_dnode;
    569 
    570 	DNODE_VERIFY(mdn);
    571 
    572 	if (!RW_WRITE_HELD(&mdn->dn_struct_rwlock)) {
    573 		rw_enter(&mdn->dn_struct_rwlock, RW_READER);
    574 		drop_struct_lock = TRUE;
    575 	}
    576 
    577 	blk = dbuf_whichblock(mdn, object * sizeof (dnode_phys_t));
    578 
    579 	db = dbuf_hold(mdn, blk, FTAG);
    580 	if (drop_struct_lock)
    581 		rw_exit(&mdn->dn_struct_rwlock);
    582 	if (db == NULL)
    583 		return (EIO);
    584 	err = dbuf_read(db, NULL, DB_RF_CANFAIL);
    585 	if (err) {
    586 		dbuf_rele(db, FTAG);
    587 		return (err);
    588 	}
    589 
    590 	ASSERT3U(db->db.db_size, >=, 1<<DNODE_SHIFT);
    591 	epb = db->db.db_size >> DNODE_SHIFT;
    592 
    593 	idx = object & (epb-1);
    594 
    595 	children_dnodes = dmu_buf_get_user(&db->db);
    596 	if (children_dnodes == NULL) {
    597 		dnode_t **winner;
    598 		children_dnodes = kmem_zalloc(epb * sizeof (dnode_t *),
    599 		    KM_SLEEP);
    600 		if (winner = dmu_buf_set_user(&db->db, children_dnodes, NULL,
    601 		    dnode_buf_pageout)) {
    602 			kmem_free(children_dnodes, epb * sizeof (dnode_t *));
    603 			children_dnodes = winner;
    604 		}
    605 	}
    606 
    607 	if ((dn = children_dnodes[idx]) == NULL) {
    608 		dnode_phys_t *dnp = (dnode_phys_t *)db->db.db_data+idx;
    609 		dnode_t *winner;
    610 
    611 		dn = dnode_create(os, dnp, db, object);
    612 		winner = atomic_cas_ptr(&children_dnodes[idx], NULL, dn);
    613 		if (winner != NULL) {
    614 			dnode_destroy(dn);
    615 			dn = winner;
    616 		}
    617 	}
    618 
    619 	mutex_enter(&dn->dn_mtx);
    620 	type = dn->dn_type;
    621 	if (dn->dn_free_txg ||
    622 	    ((flag & DNODE_MUST_BE_ALLOCATED) && type == DMU_OT_NONE) ||
    623 	    ((flag & DNODE_MUST_BE_FREE) && type != DMU_OT_NONE)) {
    624 		mutex_exit(&dn->dn_mtx);
    625 		dbuf_rele(db, FTAG);
    626 		return (type == DMU_OT_NONE ? ENOENT : EEXIST);
    627 	}
    628 	mutex_exit(&dn->dn_mtx);
    629 
    630 	if (refcount_add(&dn->dn_holds, tag) == 1)
    631 		dbuf_add_ref(db, dn);
    632 
    633 	DNODE_VERIFY(dn);
    634 	ASSERT3P(dn->dn_dbuf, ==, db);
    635 	ASSERT3U(dn->dn_object, ==, object);
    636 	dbuf_rele(db, FTAG);
    637 
    638 	*dnp = dn;
    639 	return (0);
    640 }
    641 
    642 /*
    643  * Return held dnode if the object is allocated, NULL if not.
    644  */
    645 int
    646 dnode_hold(objset_impl_t *os, uint64_t object, void *tag, dnode_t **dnp)
    647 {
    648 	return (dnode_hold_impl(os, object, DNODE_MUST_BE_ALLOCATED, tag, dnp));
    649 }
    650 
    651 /*
    652  * Can only add a reference if there is already at least one
    653  * reference on the dnode.  Returns FALSE if unable to add a
    654  * new reference.
    655  */
    656 boolean_t
    657 dnode_add_ref(dnode_t *dn, void *tag)
    658 {
    659 	mutex_enter(&dn->dn_mtx);
    660 	if (refcount_is_zero(&dn->dn_holds)) {
    661 		mutex_exit(&dn->dn_mtx);
    662 		return (FALSE);
    663 	}
    664 	VERIFY(1 < refcount_add(&dn->dn_holds, tag));
    665 	mutex_exit(&dn->dn_mtx);
    666 	return (TRUE);
    667 }
    668 
    669 void
    670 dnode_rele(dnode_t *dn, void *tag)
    671 {
    672 	uint64_t refs;
    673 
    674 	mutex_enter(&dn->dn_mtx);
    675 	refs = refcount_remove(&dn->dn_holds, tag);
    676 	mutex_exit(&dn->dn_mtx);
    677 	/* NOTE: the DNODE_DNODE does not have a dn_dbuf */
    678 	if (refs == 0 && dn->dn_dbuf)
    679 		dbuf_rele(dn->dn_dbuf, dn);
    680 }
    681 
    682 void
    683 dnode_setdirty(dnode_t *dn, dmu_tx_t *tx)
    684 {
    685 	objset_impl_t *os = dn->dn_objset;
    686 	uint64_t txg = tx->tx_txg;
    687 
    688 	if (dn->dn_object == DMU_META_DNODE_OBJECT)
    689 		return;
    690 
    691 	DNODE_VERIFY(dn);
    692 
    693 #ifdef ZFS_DEBUG
    694 	mutex_enter(&dn->dn_mtx);
    695 	ASSERT(dn->dn_phys->dn_type || dn->dn_allocated_txg);
    696 	/* ASSERT(dn->dn_free_txg == 0 || dn->dn_free_txg >= txg); */
    697 	mutex_exit(&dn->dn_mtx);
    698 #endif
    699 
    700 	mutex_enter(&os->os_lock);
    701 
    702 	/*
    703 	 * If we are already marked dirty, we're done.
    704 	 */
    705 	if (list_link_active(&dn->dn_dirty_link[txg & TXG_MASK])) {
    706 		mutex_exit(&os->os_lock);
    707 		return;
    708 	}
    709 
    710 	ASSERT(!refcount_is_zero(&dn->dn_holds) || list_head(&dn->dn_dbufs));
    711 	ASSERT(dn->dn_datablksz != 0);
    712 	ASSERT3U(dn->dn_next_bonuslen[txg&TXG_MASK], ==, 0);
    713 	ASSERT3U(dn->dn_next_blksz[txg&TXG_MASK], ==, 0);
    714 
    715 	dprintf_ds(os->os_dsl_dataset, "obj=%llu txg=%llu\n",
    716 	    dn->dn_object, txg);
    717 
    718 	if (dn->dn_free_txg > 0 && dn->dn_free_txg <= txg) {
    719 		list_insert_tail(&os->os_free_dnodes[txg&TXG_MASK], dn);
    720 	} else {
    721 		list_insert_tail(&os->os_dirty_dnodes[txg&TXG_MASK], dn);
    722 	}
    723 
    724 	mutex_exit(&os->os_lock);
    725 
    726 	/*
    727 	 * The dnode maintains a hold on its containing dbuf as
    728 	 * long as there are holds on it.  Each instantiated child
    729 	 * dbuf maintaines a hold on the dnode.  When the last child
    730 	 * drops its hold, the dnode will drop its hold on the
    731 	 * containing dbuf. We add a "dirty hold" here so that the
    732 	 * dnode will hang around after we finish processing its
    733 	 * children.
    734 	 */
    735 	VERIFY(dnode_add_ref(dn, (void *)(uintptr_t)tx->tx_txg));
    736 
    737 	(void) dbuf_dirty(dn->dn_dbuf, tx);
    738 
    739 	dsl_dataset_dirty(os->os_dsl_dataset, tx);
    740 }
    741 
    742 void
    743 dnode_free(dnode_t *dn, dmu_tx_t *tx)
    744 {
    745 	int txgoff = tx->tx_txg & TXG_MASK;
    746 
    747 	dprintf("dn=%p txg=%llu\n", dn, tx->tx_txg);
    748 
    749 	/* we should be the only holder... hopefully */
    750 	/* ASSERT3U(refcount_count(&dn->dn_holds), ==, 1); */
    751 
    752 	mutex_enter(&dn->dn_mtx);
    753 	if (dn->dn_type == DMU_OT_NONE || dn->dn_free_txg) {
    754 		mutex_exit(&dn->dn_mtx);
    755 		return;
    756 	}
    757 	dn->dn_free_txg = tx->tx_txg;
    758 	mutex_exit(&dn->dn_mtx);
    759 
    760 	/*
    761 	 * If the dnode is already dirty, it needs to be moved from
    762 	 * the dirty list to the free list.
    763 	 */
    764 	mutex_enter(&dn->dn_objset->os_lock);
    765 	if (list_link_active(&dn->dn_dirty_link[txgoff])) {
    766 		list_remove(&dn->dn_objset->os_dirty_dnodes[txgoff], dn);
    767 		list_insert_tail(&dn->dn_objset->os_free_dnodes[txgoff], dn);
    768 		mutex_exit(&dn->dn_objset->os_lock);
    769 	} else {
    770 		mutex_exit(&dn->dn_objset->os_lock);
    771 		dnode_setdirty(dn, tx);
    772 	}
    773 }
    774 
    775 /*
    776  * Try to change the block size for the indicated dnode.  This can only
    777  * succeed if there are no blocks allocated or dirty beyond first block
    778  */
    779 int
    780 dnode_set_blksz(dnode_t *dn, uint64_t size, int ibs, dmu_tx_t *tx)
    781 {
    782 	dmu_buf_impl_t *db, *db_next;
    783 	int have_db0 = FALSE;
    784 
    785 	if (size == 0)
    786 		size = SPA_MINBLOCKSIZE;
    787 	if (size > SPA_MAXBLOCKSIZE)
    788 		size = SPA_MAXBLOCKSIZE;
    789 	else
    790 		size = P2ROUNDUP(size, SPA_MINBLOCKSIZE);
    791 
    792 	if (ibs == dn->dn_indblkshift)
    793 		ibs = 0;
    794 
    795 	if (size >> SPA_MINBLOCKSHIFT == dn->dn_datablkszsec && ibs == 0)
    796 		return (0);
    797 
    798 	rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
    799 
    800 	/* Check for any allocated blocks beyond the first */
    801 	if (dn->dn_phys->dn_maxblkid != 0)
    802 		goto fail;
    803 
    804 	mutex_enter(&dn->dn_dbufs_mtx);
    805 	for (db = list_head(&dn->dn_dbufs); db; db = db_next) {
    806 		db_next = list_next(&dn->dn_dbufs, db);
    807 
    808 		if (db->db_blkid == 0) {
    809 			have_db0 = TRUE;
    810 		} else if (db->db_blkid != DB_BONUS_BLKID) {
    811 			mutex_exit(&dn->dn_dbufs_mtx);
    812 			goto fail;
    813 		}
    814 	}
    815 	mutex_exit(&dn->dn_dbufs_mtx);
    816 
    817 	if (ibs && dn->dn_nlevels != 1)
    818 		goto fail;
    819 
    820 	db = NULL;
    821 	if (!BP_IS_HOLE(&dn->dn_phys->dn_blkptr[0]) || have_db0) {
    822 		/* obtain the old block */
    823 		db = dbuf_hold(dn, 0, FTAG);
    824 		dbuf_new_size(db, size, tx);
    825 	}
    826 
    827 	dnode_setdblksz(dn, size);
    828 	dnode_setdirty(dn, tx);
    829 	dn->dn_next_blksz[tx->tx_txg&TXG_MASK] = size;
    830 	if (ibs) {
    831 		dn->dn_indblkshift = ibs;
    832 		dn->dn_next_indblkshift[tx->tx_txg&TXG_MASK] = ibs;
    833 	}
    834 
    835 	if (db)
    836 		dbuf_rele(db, FTAG);
    837 
    838 	rw_exit(&dn->dn_struct_rwlock);
    839 	return (0);
    840 
    841 fail:
    842 	rw_exit(&dn->dn_struct_rwlock);
    843 	return (ENOTSUP);
    844 }
    845 
    846 void
    847 dnode_new_blkid(dnode_t *dn, uint64_t blkid, dmu_tx_t *tx)
    848 {
    849 	uint64_t txgoff = tx->tx_txg & TXG_MASK;
    850 	int drop_struct_lock = FALSE;
    851 	int epbs, new_nlevels;
    852 	uint64_t sz;
    853 
    854 	ASSERT(blkid != DB_BONUS_BLKID);
    855 
    856 	if (!RW_WRITE_HELD(&dn->dn_struct_rwlock)) {
    857 		rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
    858 		drop_struct_lock = TRUE;
    859 	}
    860 
    861 	if (blkid <= dn->dn_maxblkid)
    862 		goto out;
    863 
    864 	dn->dn_maxblkid = blkid;
    865 
    866 	/*
    867 	 * Compute the number of levels necessary to support the new maxblkid.
    868 	 */
    869 	new_nlevels = 1;
    870 	epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT;
    871 	for (sz = dn->dn_nblkptr;
    872 	    sz <= blkid && sz >= dn->dn_nblkptr; sz <<= epbs)
    873 		new_nlevels++;
    874 
    875 	if (new_nlevels > dn->dn_nlevels) {
    876 		int old_nlevels = dn->dn_nlevels;
    877 		dmu_buf_impl_t *db;
    878 		list_t *list;
    879 		dbuf_dirty_record_t *new, *dr, *dr_next;
    880 
    881 		dn->dn_nlevels = new_nlevels;
    882 
    883 		ASSERT3U(new_nlevels, >, dn->dn_next_nlevels[txgoff]);
    884 		dn->dn_next_nlevels[txgoff] = new_nlevels;
    885 
    886 		/* dirty the left indirects */
    887 		db = dbuf_hold_level(dn, old_nlevels, 0, FTAG);
    888 		new = dbuf_dirty(db, tx);
    889 		dbuf_rele(db, FTAG);
    890 
    891 		/* transfer the dirty records to the new indirect */
    892 		mutex_enter(&dn->dn_mtx);
    893 		mutex_enter(&new->dt.di.dr_mtx);
    894 		list = &dn->dn_dirty_records[txgoff];
    895 		for (dr = list_head(list); dr; dr = dr_next) {
    896 			dr_next = list_next(&dn->dn_dirty_records[txgoff], dr);
    897 			if (dr->dr_dbuf->db_level != new_nlevels-1 &&
    898 			    dr->dr_dbuf->db_blkid != DB_BONUS_BLKID) {
    899 				ASSERT(dr->dr_dbuf->db_level == old_nlevels-1);
    900 				list_remove(&dn->dn_dirty_records[txgoff], dr);
    901 				list_insert_tail(&new->dt.di.dr_children, dr);
    902 				dr->dr_parent = new;
    903 			}
    904 		}
    905 		mutex_exit(&new->dt.di.dr_mtx);
    906 		mutex_exit(&dn->dn_mtx);
    907 	}
    908 
    909 o