Home | History | Annotate | Download | only in zfs
      1 /*
      2  * CDDL HEADER START
      3  *
      4  * The contents of this file are subject to the terms of the
      5  * Common Development and Distribution License (the "License").
      6  * You may not use this file except in compliance with the License.
      7  *
      8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
      9  * or http://www.opensolaris.org/os/licensing.
     10  * See the License for the specific language governing permissions
     11  * and limitations under the License.
     12  *
     13  * When distributing Covered Code, include this CDDL HEADER in each
     14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
     15  * If applicable, add the following below this CDDL HEADER, with the
     16  * fields enclosed by brackets "[]" replaced with your own identifying
     17  * information: Portions Copyright [yyyy] [name of copyright owner]
     18  *
     19  * CDDL HEADER END
     20  */
     21 /*
     22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
     23  * Use is subject to license terms.
     24  */
     25 
     26 #include <sys/dsl_pool.h>
     27 #include <sys/dsl_dataset.h>
     28 #include <sys/dsl_dir.h>
     29 #include <sys/dsl_synctask.h>
     30 #include <sys/dmu_tx.h>
     31 #include <sys/dmu_objset.h>
     32 #include <sys/arc.h>
     33 #include <sys/zap.h>
     34 #include <sys/zio.h>
     35 #include <sys/zfs_context.h>
     36 #include <sys/fs/zfs.h>
     37 #include <sys/zfs_znode.h>
     38 #include <sys/spa_impl.h>
     39 
     40 int zfs_no_write_throttle = 0;
     41 int zfs_write_limit_shift = 3;			/* 1/8th of physical memory */
     42 int zfs_txg_synctime = 5000;		/* target millisecs to sync a txg */
     43 
     44 uint64_t zfs_write_limit_min = 32 << 20;	/* min write limit is 32MB */
     45 uint64_t zfs_write_limit_max = 0;		/* max data payload per txg */
     46 uint64_t zfs_write_limit_inflated = 0;
     47 uint64_t zfs_write_limit_override = 0;
     48 
     49 kmutex_t zfs_write_limit_lock;
     50 
     51 static pgcnt_t old_physmem = 0;
     52 
     53 static int
     54 dsl_pool_open_special_dir(dsl_pool_t *dp, const char *name, dsl_dir_t **ddp)
     55 {
     56 	uint64_t obj;
     57 	int err;
     58 
     59 	err = zap_lookup(dp->dp_meta_objset,
     60 	    dp->dp_root_dir->dd_phys->dd_child_dir_zapobj,
     61 	    name, sizeof (obj), 1, &obj);
     62 	if (err)
     63 		return (err);
     64 
     65 	return (dsl_dir_open_obj(dp, obj, name, dp, ddp));
     66 }
     67 
     68 static dsl_pool_t *
     69 dsl_pool_open_impl(spa_t *spa, uint64_t txg)
     70 {
     71 	dsl_pool_t *dp;
     72 	blkptr_t *bp = spa_get_rootblkptr(spa);
     73 
     74 	dp = kmem_zalloc(sizeof (dsl_pool_t), KM_SLEEP);
     75 	dp->dp_spa = spa;
     76 	dp->dp_meta_rootbp = *bp;
     77 	rw_init(&dp->dp_config_rwlock, NULL, RW_DEFAULT, NULL);
     78 	dp->dp_write_limit = zfs_write_limit_min;
     79 	txg_init(dp, txg);
     80 
     81 	txg_list_create(&dp->dp_dirty_datasets,
     82 	    offsetof(dsl_dataset_t, ds_dirty_link));
     83 	txg_list_create(&dp->dp_dirty_dirs,
     84 	    offsetof(dsl_dir_t, dd_dirty_link));
     85 	txg_list_create(&dp->dp_sync_tasks,
     86 	    offsetof(dsl_sync_task_group_t, dstg_node));
     87 	list_create(&dp->dp_synced_datasets, sizeof (dsl_dataset_t),
     88 	    offsetof(dsl_dataset_t, ds_synced_link));
     89 
     90 	mutex_init(&dp->dp_lock, NULL, MUTEX_DEFAULT, NULL);
     91 	mutex_init(&dp->dp_scrub_cancel_lock, NULL, MUTEX_DEFAULT, NULL);
     92 
     93 	dp->dp_vnrele_taskq = taskq_create("zfs_vn_rele_taskq", 1, minclsyspri,
     94 	    1, 4, 0);
     95 
     96 	return (dp);
     97 }
     98 
     99 int
    100 dsl_pool_open(spa_t *spa, uint64_t txg, dsl_pool_t **dpp)
    101 {
    102 	int err;
    103 	dsl_pool_t *dp = dsl_pool_open_impl(spa, txg);
    104 	dsl_dir_t *dd;
    105 	dsl_dataset_t *ds;
    106 
    107 	rw_enter(&dp->dp_config_rwlock, RW_WRITER);
    108 	err = dmu_objset_open_impl(spa, NULL, &dp->dp_meta_rootbp,
    109 	    &dp->dp_meta_objset);
    110 	if (err)
    111 		goto out;
    112 
    113 	err = zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
    114 	    DMU_POOL_ROOT_DATASET, sizeof (uint64_t), 1,
    115 	    &dp->dp_root_dir_obj);
    116 	if (err)
    117 		goto out;
    118 
    119 	err = dsl_dir_open_obj(dp, dp->dp_root_dir_obj,
    120 	    NULL, dp, &dp->dp_root_dir);
    121 	if (err)
    122 		goto out;
    123 
    124 	err = dsl_pool_open_special_dir(dp, MOS_DIR_NAME, &dp->dp_mos_dir);
    125 	if (err)
    126 		goto out;
    127 
    128 	if (spa_version(spa) >= SPA_VERSION_ORIGIN) {
    129 		err = dsl_pool_open_special_dir(dp, ORIGIN_DIR_NAME, &dd);
    130 		if (err)
    131 			goto out;
    132 		err = dsl_dataset_hold_obj(dp, dd->dd_phys->dd_head_dataset_obj,
    133 		    FTAG, &ds);
    134 		if (err == 0) {
    135 			err = dsl_dataset_hold_obj(dp,
    136 			    ds->ds_phys->ds_prev_snap_obj, dp,
    137 			    &dp->dp_origin_snap);
    138 			dsl_dataset_rele(ds, FTAG);
    139 		}
    140 		dsl_dir_close(dd, dp);
    141 		if (err)
    142 			goto out;
    143 	}
    144 
    145 	err = zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
    146 	    DMU_POOL_TMP_USERREFS, sizeof (uint64_t), 1,
    147 	    &dp->dp_tmp_userrefs_obj);
    148 	if (err == ENOENT)
    149 		err = 0;
    150 	if (err)
    151 		goto out;
    152 
    153 	/* get scrub status */
    154 	err = zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
    155 	    DMU_POOL_SCRUB_FUNC, sizeof (uint32_t), 1,
    156 	    &dp->dp_scrub_func);
    157 	if (err == 0) {
    158 		err = zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
    159 		    DMU_POOL_SCRUB_QUEUE, sizeof (uint64_t), 1,
    160 		    &dp->dp_scrub_queue_obj);
    161 		if (err)
    162 			goto out;
    163 		err = zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
    164 		    DMU_POOL_SCRUB_MIN_TXG, sizeof (uint64_t), 1,
    165 		    &dp->dp_scrub_min_txg);
    166 		if (err)
    167 			goto out;
    168 		err = zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
    169 		    DMU_POOL_SCRUB_MAX_TXG, sizeof (uint64_t), 1,
    170 		    &dp->dp_scrub_max_txg);
    171 		if (err)
    172 			goto out;
    173 		err = zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
    174 		    DMU_POOL_SCRUB_BOOKMARK, sizeof (uint64_t),
    175 		    sizeof (dp->dp_scrub_bookmark) / sizeof (uint64_t),
    176 		    &dp->dp_scrub_bookmark);
    177 		if (err)
    178 			goto out;
    179 		err = zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
    180 		    DMU_POOL_SCRUB_DDT_BOOKMARK, sizeof (uint64_t),
    181 		    sizeof (dp->dp_scrub_ddt_bookmark) / sizeof (uint64_t),
    182 		    &dp->dp_scrub_ddt_bookmark);
    183 		if (err && err != ENOENT)
    184 			goto out;
    185 		err = zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
    186 		    DMU_POOL_SCRUB_DDT_CLASS_MAX, sizeof (uint64_t), 1,
    187 		    &dp->dp_scrub_ddt_class_max);
    188 		if (err && err != ENOENT)
    189 			goto out;
    190 		err = zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
    191 		    DMU_POOL_SCRUB_ERRORS, sizeof (uint64_t), 1,
    192 		    &spa->spa_scrub_errors);
    193 		if (err)
    194 			goto out;
    195 		if (spa_version(spa) < SPA_VERSION_DSL_SCRUB) {
    196 			/*
    197 			 * A new-type scrub was in progress on an old
    198 			 * pool.  Restart from the beginning, since the
    199 			 * old software may have changed the pool in the
    200 			 * meantime.
    201 			 */
    202 			dsl_pool_scrub_restart(dp);
    203 		}
    204 	} else {
    205 		/*
    206 		 * It's OK if there is no scrub in progress (and if
    207 		 * there was an I/O error, ignore it).
    208 		 */
    209 		err = 0;
    210 	}
    211 
    212 out:
    213 	rw_exit(&dp->dp_config_rwlock);
    214 	if (err)
    215 		dsl_pool_close(dp);
    216 	else
    217 		*dpp = dp;
    218 
    219 	return (err);
    220 }
    221 
    222 void
    223 dsl_pool_close(dsl_pool_t *dp)
    224 {
    225 	/* drop our references from dsl_pool_open() */
    226 
    227 	/*
    228 	 * Since we held the origin_snap from "syncing" context (which
    229 	 * includes pool-opening context), it actually only got a "ref"
    230 	 * and not a hold, so just drop that here.
    231 	 */
    232 	if (dp->dp_origin_snap)
    233 		dsl_dataset_drop_ref(dp->dp_origin_snap, dp);
    234 	if (dp->dp_mos_dir)
    235 		dsl_dir_close(dp->dp_mos_dir, dp);
    236 	if (dp->dp_root_dir)
    237 		dsl_dir_close(dp->dp_root_dir, dp);
    238 
    239 	/* undo the dmu_objset_open_impl(mos) from dsl_pool_open() */
    240 	if (dp->dp_meta_objset)
    241 		dmu_objset_evict(dp->dp_meta_objset);
    242 
    243 	txg_list_destroy(&dp->dp_dirty_datasets);
    244 	txg_list_destroy(&dp->dp_dirty_dirs);
    245 	list_destroy(&dp->dp_synced_datasets);
    246 
    247 	arc_flush(dp->dp_spa);
    248 	txg_fini(dp);
    249 	rw_destroy(&dp->dp_config_rwlock);
    250 	mutex_destroy(&dp->dp_lock);
    251 	mutex_destroy(&dp->dp_scrub_cancel_lock);
    252 	taskq_destroy(dp->dp_vnrele_taskq);
    253 	if (dp->dp_blkstats)
    254 		kmem_free(dp->dp_blkstats, sizeof (zfs_all_blkstats_t));
    255 	kmem_free(dp, sizeof (dsl_pool_t));
    256 }
    257 
    258 dsl_pool_t *
    259 dsl_pool_create(spa_t *spa, nvlist_t *zplprops, uint64_t txg)
    260 {
    261 	int err;
    262 	dsl_pool_t *dp = dsl_pool_open_impl(spa, txg);
    263 	dmu_tx_t *tx = dmu_tx_create_assigned(dp, txg);
    264 	objset_t *os;
    265 	dsl_dataset_t *ds;
    266 	uint64_t dsobj;
    267 
    268 	/* create and open the MOS (meta-objset) */
    269 	dp->dp_meta_objset = dmu_objset_create_impl(spa,
    270 	    NULL, &dp->dp_meta_rootbp, DMU_OST_META, tx);
    271 
    272 	/* create the pool directory */
    273 	err = zap_create_claim(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
    274 	    DMU_OT_OBJECT_DIRECTORY, DMU_OT_NONE, 0, tx);
    275 	ASSERT3U(err, ==, 0);
    276 
    277 	/* create and open the root dir */
    278 	dp->dp_root_dir_obj = dsl_dir_create_sync(dp, NULL, NULL, tx);
    279 	VERIFY(0 == dsl_dir_open_obj(dp, dp->dp_root_dir_obj,
    280 	    NULL, dp, &dp->dp_root_dir));
    281 
    282 	/* create and open the meta-objset dir */
    283 	(void) dsl_dir_create_sync(dp, dp->dp_root_dir, MOS_DIR_NAME, tx);
    284 	VERIFY(0 == dsl_pool_open_special_dir(dp,
    285 	    MOS_DIR_NAME, &dp->dp_mos_dir));
    286 
    287 	if (spa_version(spa) >= SPA_VERSION_DSL_SCRUB)
    288 		dsl_pool_create_origin(dp, tx);
    289 
    290 	/* create the root dataset */
    291 	dsobj = dsl_dataset_create_sync_dd(dp->dp_root_dir, NULL, 0, tx);
    292 
    293 	/* create the root objset */
    294 	VERIFY(0 == dsl_dataset_hold_obj(dp, dsobj, FTAG, &ds));
    295 	os = dmu_objset_create_impl(dp->dp_spa, ds,
    296 	    dsl_dataset_get_blkptr(ds), DMU_OST_ZFS, tx);
    297 #ifdef _KERNEL
    298 	zfs_create_fs(os, kcred, zplprops, tx);
    299 #endif
    300 	dsl_dataset_rele(ds, FTAG);
    301 
    302 	dmu_tx_commit(tx);
    303 
    304 	return (dp);
    305 }
    306 
    307 void
    308 dsl_pool_sync(dsl_pool_t *dp, uint64_t txg)
    309 {
    310 	zio_t *zio;
    311 	dmu_tx_t *tx;
    312 	dsl_dir_t *dd;
    313 	dsl_dataset_t *ds;
    314 	dsl_sync_task_group_t *dstg;
    315 	objset_t *mos = dp->dp_meta_objset;
    316 	hrtime_t start, write_time;
    317 	uint64_t data_written;
    318 	int err;
    319 
    320 	tx = dmu_tx_create_assigned(dp, txg);
    321 
    322 	dp->dp_read_overhead = 0;
    323 	start = gethrtime();
    324 
    325 	zio = zio_root(dp->dp_spa, NULL, NULL, ZIO_FLAG_MUSTSUCCEED);
    326 	while (ds = txg_list_remove(&dp->dp_dirty_datasets, txg)) {
    327 		/*
    328 		 * We must not sync any non-MOS datasets twice, because
    329 		 * we may have taken a snapshot of them.  However, we
    330 		 * may sync newly-created datasets on pass 2.
    331 		 */
    332 		ASSERT(!list_link_active(&ds->ds_synced_link));
    333 		list_insert_tail(&dp->dp_synced_datasets, ds);
    334 		dsl_dataset_sync(ds, zio, tx);
    335 	}
    336 	DTRACE_PROBE(pool_sync__1setup);
    337 	err = zio_wait(zio);
    338 
    339 	write_time = gethrtime() - start;
    340 	ASSERT(err == 0);
    341 	DTRACE_PROBE(pool_sync__2rootzio);
    342 
    343 	for (ds = list_head(&dp->dp_synced_datasets); ds;
    344 	    ds = list_next(&dp->dp_synced_datasets, ds))
    345 		dmu_objset_do_userquota_callbacks(ds->ds_objset, tx);
    346 
    347 	/*
    348 	 * Sync the datasets again to push out the changes due to
    349 	 * userquota updates.  This must be done before we process the
    350 	 * sync tasks, because that could cause a snapshot of a dataset
    351 	 * whose ds_bp will be rewritten when we do this 2nd sync.
    352 	 */
    353 	zio = zio_root(dp->dp_spa, NULL, NULL, ZIO_FLAG_MUSTSUCCEED);
    354 	while (ds = txg_list_remove(&dp->dp_dirty_datasets, txg)) {
    355 		ASSERT(list_link_active(&ds->ds_synced_link));
    356 		dmu_buf_rele(ds->ds_dbuf, ds);
    357 		dsl_dataset_sync(ds, zio, tx);
    358 	}
    359 	err = zio_wait(zio);
    360 
    361 	/*
    362 	 * If anything was added to a deadlist during a zio done callback,
    363 	 * it had to be put on the deferred queue.  Enqueue it for real now.
    364 	 */
    365 	for (ds = list_head(&dp->dp_synced_datasets); ds;
    366 	    ds = list_next(&dp->dp_synced_datasets, ds))
    367 		bplist_sync(&ds->ds_deadlist,
    368 		    bplist_enqueue_cb, &ds->ds_deadlist, tx);
    369 
    370 	while (dstg = txg_list_remove(&dp->dp_sync_tasks, txg)) {
    371 		/*
    372 		 * No more sync tasks should have been added while we
    373 		 * were syncing.
    374 		 */
    375 		ASSERT(spa_sync_pass(dp->dp_spa) == 1);
    376 		dsl_sync_task_group_sync(dstg, tx);
    377 	}
    378 	DTRACE_PROBE(pool_sync__3task);
    379 
    380 	start = gethrtime();
    381 	while (dd = txg_list_remove(&dp->dp_dirty_dirs, txg))
    382 		dsl_dir_sync(dd, tx);
    383 	write_time += gethrtime() - start;
    384 
    385 	if (spa_sync_pass(dp->dp_spa) == 1)
    386 		dsl_pool_scrub_sync(dp, tx);
    387 
    388 	start = gethrtime();
    389 	if (list_head(&mos->os_dirty_dnodes[txg & TXG_MASK]) != NULL ||
    390 	    list_head(&mos->os_free_dnodes[txg & TXG_MASK]) != NULL) {
    391 		zio = zio_root(dp->dp_spa, NULL, NULL, ZIO_FLAG_MUSTSUCCEED);
    392 		dmu_objset_sync(mos, zio, tx);
    393 		err = zio_wait(zio);
    394 		ASSERT(err == 0);
    395 		dprintf_bp(&dp->dp_meta_rootbp, "meta objset rootbp is %s", "");
    396 		spa_set_rootblkptr(dp->dp_spa, &dp->dp_meta_rootbp);
    397 	}
    398 	write_time += gethrtime() - start;
    399 	DTRACE_PROBE2(pool_sync__4io, hrtime_t, write_time,
    400 	    hrtime_t, dp->dp_read_overhead);
    401 	write_time -= dp->dp_read_overhead;
    402 
    403 	dmu_tx_commit(tx);
    404 
    405 	data_written = dp->dp_space_towrite[txg & TXG_MASK];
    406 	dp->dp_space_towrite[txg & TXG_MASK] = 0;
    407 	ASSERT(dp->dp_tempreserved[txg & TXG_MASK] == 0);
    408 
    409 	/*
    410 	 * If the write limit max has not been explicitly set, set it
    411 	 * to a fraction of available physical memory (default 1/8th).
    412 	 * Note that we must inflate the limit because the spa
    413 	 * inflates write sizes to account for data replication.
    414 	 * Check this each sync phase to catch changing memory size.
    415 	 */
    416 	if (physmem != old_physmem && zfs_write_limit_shift) {
    417 		mutex_enter(&zfs_write_limit_lock);
    418 		old_physmem = physmem;
    419 		zfs_write_limit_max = ptob(physmem) >> zfs_write_limit_shift;
    420 		zfs_write_limit_inflated = MAX(zfs_write_limit_min,
    421 		    spa_get_asize(dp->dp_spa, zfs_write_limit_max));
    422 		mutex_exit(&zfs_write_limit_lock);
    423 	}
    424 
    425 	/*
    426 	 * Attempt to keep the sync time consistent by adjusting the
    427 	 * amount of write traffic allowed into each transaction group.
    428 	 * Weight the throughput calculation towards the current value:
    429 	 * 	thru = 3/4 old_thru + 1/4 new_thru
    430 	 */
    431 	ASSERT(zfs_write_limit_min > 0);
    432 	if (data_written > zfs_write_limit_min / 8 && write_time > 0) {
    433 		uint64_t throughput = (data_written * NANOSEC) / write_time;
    434 		if (dp->dp_throughput)
    435 			dp->dp_throughput = throughput / 4 +
    436 			    3 * dp->dp_throughput / 4;
    437 		else
    438 			dp->dp_throughput = throughput;
    439 		dp->dp_write_limit = MIN(zfs_write_limit_inflated,
    440 		    MAX(zfs_write_limit_min,
    441 		    dp->dp_throughput * zfs_txg_synctime / MILLISEC));
    442 	}
    443 }
    444 
    445 void
    446 dsl_pool_sync_done(dsl_pool_t *dp, uint64_t txg)
    447 {
    448 	dsl_dataset_t *ds;
    449 	objset_t *os;
    450 
    451 	while (ds = list_head(&dp->dp_synced_datasets)) {
    452 		list_remove(&dp->dp_synced_datasets, ds);
    453 		os = ds->ds_objset;
    454 		zil_clean(os->os_zil);
    455 		ASSERT(!dmu_objset_is_dirty(os, txg));
    456 		dmu_buf_rele(ds->ds_dbuf, ds);
    457 	}
    458 	ASSERT(!dmu_objset_is_dirty(dp->dp_meta_objset, txg));
    459 }
    460 
    461 /*
    462  * TRUE if the current thread is the tx_sync_thread or if we
    463  * are being called from SPA context during pool initialization.
    464  */
    465 int
    466 dsl_pool_sync_context(dsl_pool_t *dp)
    467 {
    468 	return (curthread == dp->dp_tx.tx_sync_thread ||
    469 	    spa_get_dsl(dp->dp_spa) == NULL);
    470 }
    471 
    472 uint64_t
    473 dsl_pool_adjustedsize(dsl_pool_t *dp, boolean_t netfree)
    474 {
    475 	uint64_t space, resv;
    476 
    477 	/*
    478 	 * Reserve about 1.6% (1/64), or at least 32MB, for allocation
    479 	 * efficiency.
    480 	 * XXX The intent log is not accounted for, so it must fit
    481 	 * within this slop.
    482 	 *
    483 	 * If we're trying to assess whether it's OK to do a free,
    484 	 * cut the reservation in half to allow forward progress
    485 	 * (e.g. make it possible to rm(1) files from a full pool).
    486 	 */
    487 	space = spa_get_dspace(dp->dp_spa);
    488 	resv = MAX(space >> 6, SPA_MINDEVSIZE >> 1);
    489 	if (netfree)
    490 		resv >>= 1;
    491 
    492 	return (space - resv);
    493 }
    494 
    495 int
    496 dsl_pool_tempreserve_space(dsl_pool_t *dp, uint64_t space, dmu_tx_t *tx)
    497 {
    498 	uint64_t reserved = 0;
    499 	uint64_t write_limit = (zfs_write_limit_override ?
    500 	    zfs_write_limit_override : dp->dp_write_limit);
    501 
    502 	if (zfs_no_write_throttle) {
    503 		atomic_add_64(&dp->dp_tempreserved[tx->tx_txg & TXG_MASK],
    504 		    space);
    505 		return (0);
    506 	}
    507 
    508 	/*
    509 	 * Check to see if we have exceeded the maximum allowed IO for
    510 	 * this transaction group.  We can do this without locks since
    511 	 * a little slop here is ok.  Note that we do the reserved check
    512 	 * with only half the requested reserve: this is because the
    513 	 * reserve requests are worst-case, and we really don't want to
    514 	 * throttle based off of worst-case estimates.
    515 	 */
    516 	if (write_limit > 0) {
    517 		reserved = dp->dp_space_towrite[tx->tx_txg & TXG_MASK]
    518 		    + dp->dp_tempreserved[tx->tx_txg & TXG_MASK] / 2;
    519 
    520 		if (reserved && reserved > write_limit)
    521 			return (ERESTART);
    522 	}
    523 
    524 	atomic_add_64(&dp->dp_tempreserved[tx->tx_txg & TXG_MASK], space);
    525 
    526 	/*
    527 	 * If this transaction group is over 7/8ths capacity, delay
    528 	 * the caller 1 clock tick.  This will slow down the "fill"
    529 	 * rate until the sync process can catch up with us.
    530 	 */
    531 	if (reserved && reserved > (write_limit - (write_limit >> 3)))
    532 		txg_delay(dp, tx->tx_txg, 1);
    533 
    534 	return (0);
    535 }
    536 
    537 void
    538 dsl_pool_tempreserve_clear(dsl_pool_t *dp, int64_t space, dmu_tx_t *tx)
    539 {
    540 	ASSERT(dp->dp_tempreserved[tx->tx_txg & TXG_MASK] >= space);
    541 	atomic_add_64(&dp->dp_tempreserved[tx->tx_txg & TXG_MASK], -space);
    542 }
    543 
    544 void
    545 dsl_pool_memory_pressure(dsl_pool_t *dp)
    546 {
    547 	uint64_t space_inuse = 0;
    548 	int i;
    549 
    550 	if (dp->dp_write_limit == zfs_write_limit_min)
    551 		return;
    552 
    553 	for (i = 0; i < TXG_SIZE; i++) {
    554 		space_inuse += dp->dp_space_towrite[i];
    555 		space_inuse += dp->dp_tempreserved[i];
    556 	}
    557 	dp->dp_write_limit = MAX(zfs_write_limit_min,
    558 	    MIN(dp->dp_write_limit, space_inuse / 4));
    559 }
    560 
    561 void
    562 dsl_pool_willuse_space(dsl_pool_t *dp, int64_t space, dmu_tx_t *tx)
    563 {
    564 	if (space > 0) {
    565 		mutex_enter(&dp->dp_lock);
    566 		dp->dp_space_towrite[tx->tx_txg & TXG_MASK] += space;
    567 		mutex_exit(&dp->dp_lock);
    568 	}
    569 }
    570 
    571 /* ARGSUSED */
    572 static int
    573 upgrade_clones_cb(spa_t *spa, uint64_t dsobj, const char *dsname, void *arg)
    574 {
    575 	dmu_tx_t *tx = arg;
    576 	dsl_dataset_t *ds, *prev = NULL;
    577 	int err;
    578 	dsl_pool_t *dp = spa_get_dsl(spa);
    579 
    580 	err = dsl_dataset_hold_obj(dp, dsobj, FTAG, &ds);
    581 	if (err)
    582 		return (err);
    583 
    584 	while (ds->ds_phys->ds_prev_snap_obj != 0) {
    585 		err = dsl_dataset_hold_obj(dp, ds->ds_phys->ds_prev_snap_obj,
    586 		    FTAG, &prev);
    587 		if (err) {
    588 			dsl_dataset_rele(ds, FTAG);
    589 			return (err);
    590 		}
    591 
    592 		if (prev->ds_phys->ds_next_snap_obj != ds->ds_object)
    593 			break;
    594 		dsl_dataset_rele(ds, FTAG);
    595 		ds = prev;
    596 		prev = NULL;
    597 	}
    598 
    599 	if (prev == NULL) {
    600 		prev = dp->dp_origin_snap;
    601 
    602 		/*
    603 		 * The $ORIGIN can't have any data, or the accounting
    604 		 * will be wrong.
    605 		 */
    606 		ASSERT(prev->ds_phys->ds_bp.blk_birth == 0);
    607 
    608 		/* The origin doesn't get attached to itself */
    609 		if (ds->ds_object == prev->ds_object) {
    610 			dsl_dataset_rele(ds, FTAG);
    611 			return (0);
    612 		}
    613 
    614 		dmu_buf_will_dirty(ds->ds_dbuf, tx);
    615 		ds->ds_phys->ds_prev_snap_obj = prev->ds_object;
    616 		ds->ds_phys->ds_prev_snap_txg = prev->ds_phys->ds_creation_txg;
    617 
    618 		dmu_buf_will_dirty(ds->ds_dir->dd_dbuf, tx);
    619 		ds->ds_dir->dd_phys->dd_origin_obj = prev->ds_object;
    620 
    621 		dmu_buf_will_dirty(prev->ds_dbuf, tx);
    622 		prev->ds_phys->ds_num_children++;
    623 
    624 		if (ds->ds_phys->ds_next_snap_obj == 0) {
    625 			ASSERT(ds->ds_prev == NULL);
    626 			VERIFY(0 == dsl_dataset_hold_obj(dp,
    627 			    ds->ds_phys->ds_prev_snap_obj, ds, &ds->ds_prev));
    628 		}
    629 	}
    630 
    631 	ASSERT(ds->ds_dir->dd_phys->dd_origin_obj == prev->ds_object);
    632 	ASSERT(ds->ds_phys->ds_prev_snap_obj == prev->ds_object);
    633 
    634 	if (prev->ds_phys->ds_next_clones_obj == 0) {
    635 		dmu_buf_will_dirty(prev->ds_dbuf, tx);
    636 		prev->ds_phys->ds_next_clones_obj =
    637 		    zap_create(dp->dp_meta_objset,
    638 		    DMU_OT_NEXT_CLONES, DMU_OT_NONE, 0, tx);
    639 	}
    640 	VERIFY(0 == zap_add_int(dp->dp_meta_objset,
    641 	    prev->ds_phys->ds_next_clones_obj, ds->ds_object, tx));
    642 
    643 	dsl_dataset_rele(ds, FTAG);
    644 	if (prev != dp->dp_origin_snap)
    645 		dsl_dataset_rele(prev, FTAG);
    646 	return (0);
    647 }
    648 
    649 void
    650 dsl_pool_upgrade_clones(dsl_pool_t *dp, dmu_tx_t *tx)
    651 {
    652 	ASSERT(dmu_tx_is_syncing(tx));
    653 	ASSERT(dp->dp_origin_snap != NULL);
    654 
    655 	VERIFY3U(0, ==, dmu_objset_find_spa(dp->dp_spa, NULL, upgrade_clones_cb,
    656 	    tx, DS_FIND_CHILDREN));
    657 }
    658 
    659 void
    660 dsl_pool_create_origin(dsl_pool_t *dp, dmu_tx_t *tx)
    661 {
    662 	uint64_t dsobj;
    663 	dsl_dataset_t *ds;
    664 
    665 	ASSERT(dmu_tx_is_syncing(tx));
    666 	ASSERT(dp->dp_origin_snap == NULL);
    667 
    668 	/* create the origin dir, ds, & snap-ds */
    669 	rw_enter(&dp->dp_config_rwlock, RW_WRITER);
    670 	dsobj = dsl_dataset_create_sync(dp->dp_root_dir, ORIGIN_DIR_NAME,
    671 	    NULL, 0, kcred, tx);
    672 	VERIFY(0 == dsl_dataset_hold_obj(dp, dsobj, FTAG, &ds));
    673 	dsl_dataset_snapshot_sync(ds, ORIGIN_DIR_NAME, kcred, tx);
    674 	VERIFY(0 == dsl_dataset_hold_obj(dp, ds->ds_phys->ds_prev_snap_obj,
    675 	    dp, &dp->dp_origin_snap));
    676 	dsl_dataset_rele(ds, FTAG);
    677 	rw_exit(&dp->dp_config_rwlock);
    678 }
    679 
    680 taskq_t *
    681 dsl_pool_vnrele_taskq(dsl_pool_t *dp)
    682 {
    683 	return (dp->dp_vnrele_taskq);
    684 }
    685 
    686 /*
    687  * Walk through the pool-wide zap object of temporary snapshot user holds
    688  * and release them.
    689  */
    690 void
    691 dsl_pool_clean_tmp_userrefs(dsl_pool_t *dp)
    692 {
    693 	zap_attribute_t za;
    694 	zap_cursor_t zc;
    695 	objset_t *mos = dp->dp_meta_objset;
    696 	uint64_t zapobj = dp->dp_tmp_userrefs_obj;
    697 
    698 	if (zapobj == 0)
    699 		return;
    700 	ASSERT(spa_version(dp->dp_spa) >= SPA_VERSION_USERREFS);
    701 
    702 	for (zap_cursor_init(&zc, mos, zapobj);
    703 	    zap_cursor_retrieve(&zc, &za) == 0;
    704 	    zap_cursor_advance(&zc)) {
    705 		char *htag;
    706 		uint64_t dsobj;
    707 
    708 		htag = strchr(za.za_name, '-');
    709 		*htag = '\0';
    710 		++htag;
    711 		dsobj = strtonum(za.za_name, NULL);
    712 		(void) dsl_dataset_user_release_tmp(dp, dsobj, htag);
    713 	}
    714 	zap_cursor_fini(&zc);
    715 }
    716 
    717 /*
    718  * Create the pool-wide zap object for storing temporary snapshot holds.
    719  */
    720 void
    721 dsl_pool_user_hold_create_obj(dsl_pool_t *dp, dmu_tx_t *tx)
    722 {
    723 	objset_t *mos = dp->dp_meta_objset;
    724 
    725 	ASSERT(dp->dp_tmp_userrefs_obj == 0);
    726 	ASSERT(dmu_tx_is_syncing(tx));
    727 
    728 	dp->dp_tmp_userrefs_obj = zap_create(mos, DMU_OT_USERREFS,
    729 	    DMU_OT_NONE, 0, tx);
    730 
    731 	VERIFY(zap_add(mos, DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_TMP_USERREFS,
    732 	    sizeof (uint64_t), 1, &dp->dp_tmp_userrefs_obj, tx) == 0);
    733 }
    734 
    735 static int
    736 dsl_pool_user_hold_rele_impl(dsl_pool_t *dp, uint64_t dsobj,
    737     const char *tag, uint64_t *now, dmu_tx_t *tx, boolean_t holding)
    738 {
    739 	objset_t *mos = dp->dp_meta_objset;
    740 	uint64_t zapobj = dp->dp_tmp_userrefs_obj;
    741 	char *name;
    742 	int error;
    743 
    744 	ASSERT(spa_version(dp->dp_spa) >= SPA_VERSION_USERREFS);
    745 	ASSERT(dmu_tx_is_syncing(tx));
    746 
    747 	/*
    748 	 * If the pool was created prior to SPA_VERSION_USERREFS, the
    749 	 * zap object for temporary holds might not exist yet.
    750 	 */
    751 	if (zapobj == 0) {
    752 		if (holding) {
    753 			dsl_pool_user_hold_create_obj(dp, tx);
    754 			zapobj = dp->dp_tmp_userrefs_obj;
    755 		} else {
    756 			return (ENOENT);
    757 		}
    758 	}
    759 
    760 	name = kmem_asprintf("%llx-%s", (u_longlong_t)dsobj, tag);
    761 	if (holding)
    762 		error = zap_add(mos, zapobj, name, 8, 1, now, tx);
    763 	else
    764 		error = zap_remove(mos, zapobj, name, tx);
    765 	strfree(name);
    766 
    767 	return (error);
    768 }
    769 
    770 /*
    771  * Add a temporary hold for the given dataset object and tag.
    772  */
    773 int
    774 dsl_pool_user_hold(dsl_pool_t *dp, uint64_t dsobj, const char *tag,
    775     uint64_t *now, dmu_tx_t *tx)
    776 {
    777 	return (dsl_pool_user_hold_rele_impl(dp, dsobj, tag, now, tx, B_TRUE));
    778 }
    779 
    780 /*
    781  * Release a temporary hold for the given dataset object and tag.
    782  */
    783 int
    784 dsl_pool_user_release(dsl_pool_t *dp, uint64_t dsobj, const char *tag,
    785     dmu_tx_t *tx)
    786 {
    787 	return (dsl_pool_user_hold_rele_impl(dp, dsobj, tag, NULL,
    788 	    tx, B_FALSE));
    789 }
    790