Home | History | Annotate | Download | only in zfs
      1 /*
      2  * CDDL HEADER START
      3  *
      4  * The contents of this file are subject to the terms of the
      5  * Common Development and Distribution License (the "License").
      6  * You may not use this file except in compliance with the License.
      7  *
      8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
      9  * or http://www.opensolaris.org/os/licensing.
     10  * See the License for the specific language governing permissions
     11  * and limitations under the License.
     12  *
     13  * When distributing Covered Code, include this CDDL HEADER in each
     14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
     15  * If applicable, add the following below this CDDL HEADER, with the
     16  * fields enclosed by brackets "[]" replaced with your own identifying
     17  * information: Portions Copyright [yyyy] [name of copyright owner]
     18  *
     19  * CDDL HEADER END
     20  */
     21 /*
     22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
     23  * Use is subject to license terms.
     24  */
     25 
     26 #include <sys/dsl_pool.h>
     27 #include <sys/dsl_dataset.h>
     28 #include <sys/dsl_dir.h>
     29 #include <sys/dsl_synctask.h>
     30 #include <sys/dmu_tx.h>
     31 #include <sys/dmu_objset.h>
     32 #include <sys/arc.h>
     33 #include <sys/zap.h>
     34 #include <sys/zio.h>
     35 #include <sys/zfs_context.h>
     36 #include <sys/fs/zfs.h>
     37 #include <sys/zfs_znode.h>
     38 #include <sys/spa_impl.h>
     39 
     40 int zfs_no_write_throttle = 0;
     41 int zfs_write_limit_shift = 3;			/* 1/8th of physical memory */
     42 int zfs_txg_synctime_ms = 5000;		/* target millisecs to sync a txg */
     43 
     44 uint64_t zfs_write_limit_min = 32 << 20;	/* min write limit is 32MB */
     45 uint64_t zfs_write_limit_max = 0;		/* max data payload per txg */
     46 uint64_t zfs_write_limit_inflated = 0;
     47 uint64_t zfs_write_limit_override = 0;
     48 
     49 kmutex_t zfs_write_limit_lock;
     50 
     51 static pgcnt_t old_physmem = 0;
     52 
     53 static int
     54 dsl_pool_open_special_dir(dsl_pool_t *dp, const char *name, dsl_dir_t **ddp)
     55 {
     56 	uint64_t obj;
     57 	int err;
     58 
     59 	err = zap_lookup(dp->dp_meta_objset,
     60 	    dp->dp_root_dir->dd_phys->dd_child_dir_zapobj,
     61 	    name, sizeof (obj), 1, &obj);
     62 	if (err)
     63 		return (err);
     64 
     65 	return (dsl_dir_open_obj(dp, obj, name, dp, ddp));
     66 }
     67 
     68 static dsl_pool_t *
     69 dsl_pool_open_impl(spa_t *spa, uint64_t txg)
     70 {
     71 	dsl_pool_t *dp;
     72 	blkptr_t *bp = spa_get_rootblkptr(spa);
     73 
     74 	dp = kmem_zalloc(sizeof (dsl_pool_t), KM_SLEEP);
     75 	dp->dp_spa = spa;
     76 	dp->dp_meta_rootbp = *bp;
     77 	rw_init(&dp->dp_config_rwlock, NULL, RW_DEFAULT, NULL);
     78 	dp->dp_write_limit = zfs_write_limit_min;
     79 	txg_init(dp, txg);
     80 
     81 	txg_list_create(&dp->dp_dirty_datasets,
     82 	    offsetof(dsl_dataset_t, ds_dirty_link));
     83 	txg_list_create(&dp->dp_dirty_dirs,
     84 	    offsetof(dsl_dir_t, dd_dirty_link));
     85 	txg_list_create(&dp->dp_sync_tasks,
     86 	    offsetof(dsl_sync_task_group_t, dstg_node));
     87 	list_create(&dp->dp_synced_datasets, sizeof (dsl_dataset_t),
     88 	    offsetof(dsl_dataset_t, ds_synced_link));
     89 
     90 	mutex_init(&dp->dp_lock, NULL, MUTEX_DEFAULT, NULL);
     91 	mutex_init(&dp->dp_scrub_cancel_lock, NULL, MUTEX_DEFAULT, NULL);
     92 
     93 	dp->dp_vnrele_taskq = taskq_create("zfs_vn_rele_taskq", 1, minclsyspri,
     94 	    1, 4, 0);
     95 
     96 	return (dp);
     97 }
     98 
     99 int
    100 dsl_pool_open(spa_t *spa, uint64_t txg, dsl_pool_t **dpp)
    101 {
    102 	int err;
    103 	dsl_pool_t *dp = dsl_pool_open_impl(spa, txg);
    104 	dsl_dir_t *dd;
    105 	dsl_dataset_t *ds;
    106 
    107 	rw_enter(&dp->dp_config_rwlock, RW_WRITER);
    108 	err = dmu_objset_open_impl(spa, NULL, &dp->dp_meta_rootbp,
    109 	    &dp->dp_meta_objset);
    110 	if (err)
    111 		goto out;
    112 
    113 	err = zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
    114 	    DMU_POOL_ROOT_DATASET, sizeof (uint64_t), 1,
    115 	    &dp->dp_root_dir_obj);
    116 	if (err)
    117 		goto out;
    118 
    119 	err = dsl_dir_open_obj(dp, dp->dp_root_dir_obj,
    120 	    NULL, dp, &dp->dp_root_dir);
    121 	if (err)
    122 		goto out;
    123 
    124 	err = dsl_pool_open_special_dir(dp, MOS_DIR_NAME, &dp->dp_mos_dir);
    125 	if (err)
    126 		goto out;
    127 
    128 	if (spa_version(spa) >= SPA_VERSION_ORIGIN) {
    129 		err = dsl_pool_open_special_dir(dp, ORIGIN_DIR_NAME, &dd);
    130 		if (err)
    131 			goto out;
    132 		err = dsl_dataset_hold_obj(dp, dd->dd_phys->dd_head_dataset_obj,
    133 		    FTAG, &ds);
    134 		if (err == 0) {
    135 			err = dsl_dataset_hold_obj(dp,
    136 			    ds->ds_phys->ds_prev_snap_obj, dp,
    137 			    &dp->dp_origin_snap);
    138 			dsl_dataset_rele(ds, FTAG);
    139 		}
    140 		dsl_dir_close(dd, dp);
    141 		if (err)
    142 			goto out;
    143 	}
    144 
    145 	err = zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
    146 	    DMU_POOL_TMP_USERREFS, sizeof (uint64_t), 1,
    147 	    &dp->dp_tmp_userrefs_obj);
    148 	if (err == ENOENT)
    149 		err = 0;
    150 	if (err)
    151 		goto out;
    152 
    153 	/* get scrub status */
    154 	err = zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
    155 	    DMU_POOL_SCRUB_FUNC, sizeof (uint32_t), 1,
    156 	    &dp->dp_scrub_func);
    157 	if (err == 0) {
    158 		err = zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
    159 		    DMU_POOL_SCRUB_QUEUE, sizeof (uint64_t), 1,
    160 		    &dp->dp_scrub_queue_obj);
    161 		if (err)
    162 			goto out;
    163 		err = zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
    164 		    DMU_POOL_SCRUB_MIN_TXG, sizeof (uint64_t), 1,
    165 		    &dp->dp_scrub_min_txg);
    166 		if (err)
    167 			goto out;
    168 		err = zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
    169 		    DMU_POOL_SCRUB_MAX_TXG, sizeof (uint64_t), 1,
    170 		    &dp->dp_scrub_max_txg);
    171 		if (err)
    172 			goto out;
    173 		err = zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
    174 		    DMU_POOL_SCRUB_BOOKMARK, sizeof (uint64_t),
    175 		    sizeof (dp->dp_scrub_bookmark) / sizeof (uint64_t),
    176 		    &dp->dp_scrub_bookmark);
    177 		if (err)
    178 			goto out;
    179 		err = zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
    180 		    DMU_POOL_SCRUB_DDT_BOOKMARK, sizeof (uint64_t),
    181 		    sizeof (dp->dp_scrub_ddt_bookmark) / sizeof (uint64_t),
    182 		    &dp->dp_scrub_ddt_bookmark);
    183 		if (err && err != ENOENT)
    184 			goto out;
    185 		err = zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
    186 		    DMU_POOL_SCRUB_DDT_CLASS_MAX, sizeof (uint64_t), 1,
    187 		    &dp->dp_scrub_ddt_class_max);
    188 		if (err && err != ENOENT)
    189 			goto out;
    190 		err = zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
    191 		    DMU_POOL_SCRUB_ERRORS, sizeof (uint64_t), 1,
    192 		    &spa->spa_scrub_errors);
    193 		if (err)
    194 			goto out;
    195 		if (spa_version(spa) < SPA_VERSION_DSL_SCRUB) {
    196 			/*
    197 			 * A new-type scrub was in progress on an old
    198 			 * pool.  Restart from the beginning, since the
    199 			 * old software may have changed the pool in the
    200 			 * meantime.
    201 			 */
    202 			dsl_pool_scrub_restart(dp);
    203 		}
    204 	} else {
    205 		/*
    206 		 * It's OK if there is no scrub in progress (and if
    207 		 * there was an I/O error, ignore it).
    208 		 */
    209 		err = 0;
    210 	}
    211 
    212 out:
    213 	rw_exit(&dp->dp_config_rwlock);
    214 	if (err)
    215 		dsl_pool_close(dp);
    216 	else
    217 		*dpp = dp;
    218 
    219 	return (err);
    220 }
    221 
    222 void
    223 dsl_pool_close(dsl_pool_t *dp)
    224 {
    225 	/* drop our references from dsl_pool_open() */
    226 
    227 	/*
    228 	 * Since we held the origin_snap from "syncing" context (which
    229 	 * includes pool-opening context), it actually only got a "ref"
    230 	 * and not a hold, so just drop that here.
    231 	 */
    232 	if (dp->dp_origin_snap)
    233 		dsl_dataset_drop_ref(dp->dp_origin_snap, dp);
    234 	if (dp->dp_mos_dir)
    235 		dsl_dir_close(dp->dp_mos_dir, dp);
    236 	if (dp->dp_root_dir)
    237 		dsl_dir_close(dp->dp_root_dir, dp);
    238 
    239 	/* undo the dmu_objset_open_impl(mos) from dsl_pool_open() */
    240 	if (dp->dp_meta_objset)
    241 		dmu_objset_evict(dp->dp_meta_objset);
    242 
    243 	txg_list_destroy(&dp->dp_dirty_datasets);
    244 	txg_list_destroy(&dp->dp_dirty_dirs);
    245 	list_destroy(&dp->dp_synced_datasets);
    246 
    247 	arc_flush(dp->dp_spa);
    248 	txg_fini(dp);
    249 	rw_destroy(&dp->dp_config_rwlock);
    250 	mutex_destroy(&dp->dp_lock);
    251 	mutex_destroy(&dp->dp_scrub_cancel_lock);
    252 	taskq_destroy(dp->dp_vnrele_taskq);
    253 	if (dp->dp_blkstats)
    254 		kmem_free(dp->dp_blkstats, sizeof (zfs_all_blkstats_t));
    255 	kmem_free(dp, sizeof (dsl_pool_t));
    256 }
    257 
    258 dsl_pool_t *
    259 dsl_pool_create(spa_t *spa, nvlist_t *zplprops, uint64_t txg)
    260 {
    261 	int err;
    262 	dsl_pool_t *dp = dsl_pool_open_impl(spa, txg);
    263 	dmu_tx_t *tx = dmu_tx_create_assigned(dp, txg);
    264 	objset_t *os;
    265 	dsl_dataset_t *ds;
    266 	uint64_t dsobj;
    267 
    268 	/* create and open the MOS (meta-objset) */
    269 	dp->dp_meta_objset = dmu_objset_create_impl(spa,
    270 	    NULL, &dp->dp_meta_rootbp, DMU_OST_META, tx);
    271 
    272 	/* create the pool directory */
    273 	err = zap_create_claim(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
    274 	    DMU_OT_OBJECT_DIRECTORY, DMU_OT_NONE, 0, tx);
    275 	ASSERT3U(err, ==, 0);
    276 
    277 	/* create and open the root dir */
    278 	dp->dp_root_dir_obj = dsl_dir_create_sync(dp, NULL, NULL, tx);
    279 	VERIFY(0 == dsl_dir_open_obj(dp, dp->dp_root_dir_obj,
    280 	    NULL, dp, &dp->dp_root_dir));
    281 
    282 	/* create and open the meta-objset dir */
    283 	(void) dsl_dir_create_sync(dp, dp->dp_root_dir, MOS_DIR_NAME, tx);
    284 	VERIFY(0 == dsl_pool_open_special_dir(dp,
    285 	    MOS_DIR_NAME, &dp->dp_mos_dir));
    286 
    287 	if (spa_version(spa) >= SPA_VERSION_DSL_SCRUB)
    288 		dsl_pool_create_origin(dp, tx);
    289 
    290 	/* create the root dataset */
    291 	dsobj = dsl_dataset_create_sync_dd(dp->dp_root_dir, NULL, 0, tx);
    292 
    293 	/* create the root objset */
    294 	VERIFY(0 == dsl_dataset_hold_obj(dp, dsobj, FTAG, &ds));
    295 	os = dmu_objset_create_impl(dp->dp_spa, ds,
    296 	    dsl_dataset_get_blkptr(ds), DMU_OST_ZFS, tx);
    297 #ifdef _KERNEL
    298 	zfs_create_fs(os, kcred, zplprops, tx);
    299 #endif
    300 	dsl_dataset_rele(ds, FTAG);
    301 
    302 	dmu_tx_commit(tx);
    303 
    304 	return (dp);
    305 }
    306 
    307 void
    308 dsl_pool_sync(dsl_pool_t *dp, uint64_t txg)
    309 {
    310 	zio_t *zio;
    311 	dmu_tx_t *tx;
    312 	dsl_dir_t *dd;
    313 	dsl_dataset_t *ds;
    314 	dsl_sync_task_group_t *dstg;
    315 	objset_t *mos = dp->dp_meta_objset;
    316 	hrtime_t start, write_time;
    317 	uint64_t data_written;
    318 	int err;
    319 
    320 	tx = dmu_tx_create_assigned(dp, txg);
    321 
    322 	dp->dp_read_overhead = 0;
    323 	start = gethrtime();
    324 
    325 	zio = zio_root(dp->dp_spa, NULL, NULL, ZIO_FLAG_MUSTSUCCEED);
    326 	while (ds = txg_list_remove(&dp->dp_dirty_datasets, txg)) {
    327 		/*
    328 		 * We must not sync any non-MOS datasets twice, because
    329 		 * we may have taken a snapshot of them.  However, we
    330 		 * may sync newly-created datasets on pass 2.
    331 		 */
    332 		ASSERT(!list_link_active(&ds->ds_synced_link));
    333 		list_insert_tail(&dp->dp_synced_datasets, ds);
    334 		dsl_dataset_sync(ds, zio, tx);
    335 	}
    336 	DTRACE_PROBE(pool_sync__1setup);
    337 	err = zio_wait(zio);
    338 
    339 	write_time = gethrtime() - start;
    340 	ASSERT(err == 0);
    341 	DTRACE_PROBE(pool_sync__2rootzio);
    342 
    343 	for (ds = list_head(&dp->dp_synced_datasets); ds;
    344 	    ds = list_next(&dp->dp_synced_datasets, ds))
    345 		dmu_objset_do_userquota_callbacks(ds->ds_objset, tx);
    346 
    347 	/*
    348 	 * Sync the datasets again to push out the changes due to
    349 	 * userquota updates.  This must be done before we process the
    350 	 * sync tasks, because that could cause a snapshot of a dataset
    351 	 * whose ds_bp will be rewritten when we do this 2nd sync.
    352 	 */
    353 	zio = zio_root(dp->dp_spa, NULL, NULL, ZIO_FLAG_MUSTSUCCEED);
    354 	while (ds = txg_list_remove(&dp->dp_dirty_datasets, txg)) {
    355 		ASSERT(list_link_active(&ds->ds_synced_link));
    356 		dmu_buf_rele(ds->ds_dbuf, ds);
    357 		dsl_dataset_sync(ds, zio, tx);
    358 	}
    359 	err = zio_wait(zio);
    360 
    361 	/*
    362 	 * If anything was added to a deadlist during a zio done callback,
    363 	 * it had to be put on the deferred queue.  Enqueue it for real now.
    364 	 */
    365 	for (ds = list_head(&dp->dp_synced_datasets); ds;
    366 	    ds = list_next(&dp->dp_synced_datasets, ds))
    367 		bplist_sync(&ds->ds_deadlist,
    368 		    bplist_enqueue_cb, &ds->ds_deadlist, tx);
    369 
    370 	while (dstg = txg_list_remove(&dp->dp_sync_tasks, txg)) {
    371 		/*
    372 		 * No more sync tasks should have been added while we
    373 		 * were syncing.
    374 		 */
    375 		ASSERT(spa_sync_pass(dp->dp_spa) == 1);
    376 		dsl_sync_task_group_sync(dstg, tx);
    377 	}
    378 	DTRACE_PROBE(pool_sync__3task);
    379 
    380 	start = gethrtime();
    381 	while (dd = txg_list_remove(&dp->dp_dirty_dirs, txg))
    382 		dsl_dir_sync(dd, tx);
    383 	write_time += gethrtime() - start;
    384 
    385 	if (spa_sync_pass(dp->dp_spa) == 1) {
    386 		dp->dp_scrub_prefetch_zio_root = zio_root(dp->dp_spa, NULL,
    387 		    NULL, ZIO_FLAG_CANFAIL);
    388 		dsl_pool_scrub_sync(dp, tx);
    389 		(void) zio_wait(dp->dp_scrub_prefetch_zio_root);
    390 	}
    391 
    392 	start = gethrtime();
    393 	if (list_head(&mos->os_dirty_dnodes[txg & TXG_MASK]) != NULL ||
    394 	    list_head(&mos->os_free_dnodes[txg & TXG_MASK]) != NULL) {
    395 		zio = zio_root(dp->dp_spa, NULL, NULL, ZIO_FLAG_MUSTSUCCEED);
    396 		dmu_objset_sync(mos, zio, tx);
    397 		err = zio_wait(zio);
    398 		ASSERT(err == 0);
    399 		dprintf_bp(&dp->dp_meta_rootbp, "meta objset rootbp is %s", "");
    400 		spa_set_rootblkptr(dp->dp_spa, &dp->dp_meta_rootbp);
    401 	}
    402 	write_time += gethrtime() - start;
    403 	DTRACE_PROBE2(pool_sync__4io, hrtime_t, write_time,
    404 	    hrtime_t, dp->dp_read_overhead);
    405 	write_time -= dp->dp_read_overhead;
    406 
    407 	dmu_tx_commit(tx);
    408 
    409 	data_written = dp->dp_space_towrite[txg & TXG_MASK];
    410 	dp->dp_space_towrite[txg & TXG_MASK] = 0;
    411 	ASSERT(dp->dp_tempreserved[txg & TXG_MASK] == 0);
    412 
    413 	/*
    414 	 * If the write limit max has not been explicitly set, set it
    415 	 * to a fraction of available physical memory (default 1/8th).
    416 	 * Note that we must inflate the limit because the spa
    417 	 * inflates write sizes to account for data replication.
    418 	 * Check this each sync phase to catch changing memory size.
    419 	 */
    420 	if (physmem != old_physmem && zfs_write_limit_shift) {
    421 		mutex_enter(&zfs_write_limit_lock);
    422 		old_physmem = physmem;
    423 		zfs_write_limit_max = ptob(physmem) >> zfs_write_limit_shift;
    424 		zfs_write_limit_inflated = MAX(zfs_write_limit_min,
    425 		    spa_get_asize(dp->dp_spa, zfs_write_limit_max));
    426 		mutex_exit(&zfs_write_limit_lock);
    427 	}
    428 
    429 	/*
    430 	 * Attempt to keep the sync time consistent by adjusting the
    431 	 * amount of write traffic allowed into each transaction group.
    432 	 * Weight the throughput calculation towards the current value:
    433 	 * 	thru = 3/4 old_thru + 1/4 new_thru
    434 	 */
    435 	ASSERT(zfs_write_limit_min > 0);
    436 	if (data_written > zfs_write_limit_min / 8 && write_time > 0) {
    437 		uint64_t throughput = (data_written * NANOSEC) / write_time;
    438 		if (dp->dp_throughput)
    439 			dp->dp_throughput = throughput / 4 +
    440 			    3 * dp->dp_throughput / 4;
    441 		else
    442 			dp->dp_throughput = throughput;
    443 		dp->dp_write_limit = MIN(zfs_write_limit_inflated,
    444 		    MAX(zfs_write_limit_min,
    445 		    dp->dp_throughput * zfs_txg_synctime_ms / MILLISEC));
    446 	}
    447 }
    448 
    449 void
    450 dsl_pool_sync_done(dsl_pool_t *dp, uint64_t txg)
    451 {
    452 	dsl_dataset_t *ds;
    453 	objset_t *os;
    454 
    455 	while (ds = list_head(&dp->dp_synced_datasets)) {
    456 		list_remove(&dp->dp_synced_datasets, ds);
    457 		os = ds->ds_objset;
    458 		zil_clean(os->os_zil);
    459 		ASSERT(!dmu_objset_is_dirty(os, txg));
    460 		dmu_buf_rele(ds->ds_dbuf, ds);
    461 	}
    462 	ASSERT(!dmu_objset_is_dirty(dp->dp_meta_objset, txg));
    463 }
    464 
    465 /*
    466  * TRUE if the current thread is the tx_sync_thread or if we
    467  * are being called from SPA context during pool initialization.
    468  */
    469 int
    470 dsl_pool_sync_context(dsl_pool_t *dp)
    471 {
    472 	return (curthread == dp->dp_tx.tx_sync_thread ||
    473 	    spa_get_dsl(dp->dp_spa) == NULL);
    474 }
    475 
    476 uint64_t
    477 dsl_pool_adjustedsize(dsl_pool_t *dp, boolean_t netfree)
    478 {
    479 	uint64_t space, resv;
    480 
    481 	/*
    482 	 * Reserve about 1.6% (1/64), or at least 32MB, for allocation
    483 	 * efficiency.
    484 	 * XXX The intent log is not accounted for, so it must fit
    485 	 * within this slop.
    486 	 *
    487 	 * If we're trying to assess whether it's OK to do a free,
    488 	 * cut the reservation in half to allow forward progress
    489 	 * (e.g. make it possible to rm(1) files from a full pool).
    490 	 */
    491 	space = spa_get_dspace(dp->dp_spa);
    492 	resv = MAX(space >> 6, SPA_MINDEVSIZE >> 1);
    493 	if (netfree)
    494 		resv >>= 1;
    495 
    496 	return (space - resv);
    497 }
    498 
    499 int
    500 dsl_pool_tempreserve_space(dsl_pool_t *dp, uint64_t space, dmu_tx_t *tx)
    501 {
    502 	uint64_t reserved = 0;
    503 	uint64_t write_limit = (zfs_write_limit_override ?
    504 	    zfs_write_limit_override : dp->dp_write_limit);
    505 
    506 	if (zfs_no_write_throttle) {
    507 		atomic_add_64(&dp->dp_tempreserved[tx->tx_txg & TXG_MASK],
    508 		    space);
    509 		return (0);
    510 	}
    511 
    512 	/*
    513 	 * Check to see if we have exceeded the maximum allowed IO for
    514 	 * this transaction group.  We can do this without locks since
    515 	 * a little slop here is ok.  Note that we do the reserved check
    516 	 * with only half the requested reserve: this is because the
    517 	 * reserve requests are worst-case, and we really don't want to
    518 	 * throttle based off of worst-case estimates.
    519 	 */
    520 	if (write_limit > 0) {
    521 		reserved = dp->dp_space_towrite[tx->tx_txg & TXG_MASK]
    522 		    + dp->dp_tempreserved[tx->tx_txg & TXG_MASK] / 2;
    523 
    524 		if (reserved && reserved > write_limit)
    525 			return (ERESTART);
    526 	}
    527 
    528 	atomic_add_64(&dp->dp_tempreserved[tx->tx_txg & TXG_MASK], space);
    529 
    530 	/*
    531 	 * If this transaction group is over 7/8ths capacity, delay
    532 	 * the caller 1 clock tick.  This will slow down the "fill"
    533 	 * rate until the sync process can catch up with us.
    534 	 */
    535 	if (reserved && reserved > (write_limit - (write_limit >> 3)))
    536 		txg_delay(dp, tx->tx_txg, 1);
    537 
    538 	return (0);
    539 }
    540 
    541 void
    542 dsl_pool_tempreserve_clear(dsl_pool_t *dp, int64_t space, dmu_tx_t *tx)
    543 {
    544 	ASSERT(dp->dp_tempreserved[tx->tx_txg & TXG_MASK] >= space);
    545 	atomic_add_64(&dp->dp_tempreserved[tx->tx_txg & TXG_MASK], -space);
    546 }
    547 
    548 void
    549 dsl_pool_memory_pressure(dsl_pool_t *dp)
    550 {
    551 	uint64_t space_inuse = 0;
    552 	int i;
    553 
    554 	if (dp->dp_write_limit == zfs_write_limit_min)
    555 		return;
    556 
    557 	for (i = 0; i < TXG_SIZE; i++) {
    558 		space_inuse += dp->dp_space_towrite[i];
    559 		space_inuse += dp->dp_tempreserved[i];
    560 	}
    561 	dp->dp_write_limit = MAX(zfs_write_limit_min,
    562 	    MIN(dp->dp_write_limit, space_inuse / 4));
    563 }
    564 
    565 void
    566 dsl_pool_willuse_space(dsl_pool_t *dp, int64_t space, dmu_tx_t *tx)
    567 {
    568 	if (space > 0) {
    569 		mutex_enter(&dp->dp_lock);
    570 		dp->dp_space_towrite[tx->tx_txg & TXG_MASK] += space;
    571 		mutex_exit(&dp->dp_lock);
    572 	}
    573 }
    574 
    575 /* ARGSUSED */
    576 static int
    577 upgrade_clones_cb(spa_t *spa, uint64_t dsobj, const char *dsname, void *arg)
    578 {
    579 	dmu_tx_t *tx = arg;
    580 	dsl_dataset_t *ds, *prev = NULL;
    581 	int err;
    582 	dsl_pool_t *dp = spa_get_dsl(spa);
    583 
    584 	err = dsl_dataset_hold_obj(dp, dsobj, FTAG, &ds);
    585 	if (err)
    586 		return (err);
    587 
    588 	while (ds->ds_phys->ds_prev_snap_obj != 0) {
    589 		err = dsl_dataset_hold_obj(dp, ds->ds_phys->ds_prev_snap_obj,
    590 		    FTAG, &prev);
    591 		if (err) {
    592 			dsl_dataset_rele(ds, FTAG);
    593 			return (err);
    594 		}
    595 
    596 		if (prev->ds_phys->ds_next_snap_obj != ds->ds_object)
    597 			break;
    598 		dsl_dataset_rele(ds, FTAG);
    599 		ds = prev;
    600 		prev = NULL;
    601 	}
    602 
    603 	if (prev == NULL) {
    604 		prev = dp->dp_origin_snap;
    605 
    606 		/*
    607 		 * The $ORIGIN can't have any data, or the accounting
    608 		 * will be wrong.
    609 		 */
    610 		ASSERT(prev->ds_phys->ds_bp.blk_birth == 0);
    611 
    612 		/* The origin doesn't get attached to itself */
    613 		if (ds->ds_object == prev->ds_object) {
    614 			dsl_dataset_rele(ds, FTAG);
    615 			return (0);
    616 		}
    617 
    618 		dmu_buf_will_dirty(ds->ds_dbuf, tx);
    619 		ds->ds_phys->ds_prev_snap_obj = prev->ds_object;
    620 		ds->ds_phys->ds_prev_snap_txg = prev->ds_phys->ds_creation_txg;
    621 
    622 		dmu_buf_will_dirty(ds->ds_dir->dd_dbuf, tx);
    623 		ds->ds_dir->dd_phys->dd_origin_obj = prev->ds_object;
    624 
    625 		dmu_buf_will_dirty(prev->ds_dbuf, tx);
    626 		prev->ds_phys->ds_num_children++;
    627 
    628 		if (ds->ds_phys->ds_next_snap_obj == 0) {
    629 			ASSERT(ds->ds_prev == NULL);
    630 			VERIFY(0 == dsl_dataset_hold_obj(dp,
    631 			    ds->ds_phys->ds_prev_snap_obj, ds, &ds->ds_prev));
    632 		}
    633 	}
    634 
    635 	ASSERT(ds->ds_dir->dd_phys->dd_origin_obj == prev->ds_object);
    636 	ASSERT(ds->ds_phys->ds_prev_snap_obj == prev->ds_object);
    637 
    638 	if (prev->ds_phys->ds_next_clones_obj == 0) {
    639 		dmu_buf_will_dirty(prev->ds_dbuf, tx);
    640 		prev->ds_phys->ds_next_clones_obj =
    641 		    zap_create(dp->dp_meta_objset,
    642 		    DMU_OT_NEXT_CLONES, DMU_OT_NONE, 0, tx);
    643 	}
    644 	VERIFY(0 == zap_add_int(dp->dp_meta_objset,
    645 	    prev->ds_phys->ds_next_clones_obj, ds->ds_object, tx));
    646 
    647 	dsl_dataset_rele(ds, FTAG);
    648 	if (prev != dp->dp_origin_snap)
    649 		dsl_dataset_rele(prev, FTAG);
    650 	return (0);
    651 }
    652 
    653 void
    654 dsl_pool_upgrade_clones(dsl_pool_t *dp, dmu_tx_t *tx)
    655 {
    656 	ASSERT(dmu_tx_is_syncing(tx));
    657 	ASSERT(dp->dp_origin_snap != NULL);
    658 
    659 	VERIFY3U(0, ==, dmu_objset_find_spa(dp->dp_spa, NULL, upgrade_clones_cb,
    660 	    tx, DS_FIND_CHILDREN));
    661 }
    662 
    663 void
    664 dsl_pool_create_origin(dsl_pool_t *dp, dmu_tx_t *tx)
    665 {
    666 	uint64_t dsobj;
    667 	dsl_dataset_t *ds;
    668 
    669 	ASSERT(dmu_tx_is_syncing(tx));
    670 	ASSERT(dp->dp_origin_snap == NULL);
    671 
    672 	/* create the origin dir, ds, & snap-ds */
    673 	rw_enter(&dp->dp_config_rwlock, RW_WRITER);
    674 	dsobj = dsl_dataset_create_sync(dp->dp_root_dir, ORIGIN_DIR_NAME,
    675 	    NULL, 0, kcred, tx);
    676 	VERIFY(0 == dsl_dataset_hold_obj(dp, dsobj, FTAG, &ds));
    677 	dsl_dataset_snapshot_sync(ds, ORIGIN_DIR_NAME, kcred, tx);
    678 	VERIFY(0 == dsl_dataset_hold_obj(dp, ds->ds_phys->ds_prev_snap_obj,
    679 	    dp, &dp->dp_origin_snap));
    680 	dsl_dataset_rele(ds, FTAG);
    681 	rw_exit(&dp->dp_config_rwlock);
    682 }
    683 
    684 taskq_t *
    685 dsl_pool_vnrele_taskq(dsl_pool_t *dp)
    686 {
    687 	return (dp->dp_vnrele_taskq);
    688 }
    689 
    690 /*
    691  * Walk through the pool-wide zap object of temporary snapshot user holds
    692  * and release them.
    693  */
    694 void
    695 dsl_pool_clean_tmp_userrefs(dsl_pool_t *dp)
    696 {
    697 	zap_attribute_t za;
    698 	zap_cursor_t zc;
    699 	objset_t *mos = dp->dp_meta_objset;
    700 	uint64_t zapobj = dp->dp_tmp_userrefs_obj;
    701 
    702 	if (zapobj == 0)
    703 		return;
    704 	ASSERT(spa_version(dp->dp_spa) >= SPA_VERSION_USERREFS);
    705 
    706 	for (zap_cursor_init(&zc, mos, zapobj);
    707 	    zap_cursor_retrieve(&zc, &za) == 0;
    708 	    zap_cursor_advance(&zc)) {
    709 		char *htag;
    710 		uint64_t dsobj;
    711 
    712 		htag = strchr(za.za_name, '-');
    713 		*htag = '\0';
    714 		++htag;
    715 		dsobj = strtonum(za.za_name, NULL);
    716 		(void) dsl_dataset_user_release_tmp(dp, dsobj, htag);
    717 	}
    718 	zap_cursor_fini(&zc);
    719 }
    720 
    721 /*
    722  * Create the pool-wide zap object for storing temporary snapshot holds.
    723  */
    724 void
    725 dsl_pool_user_hold_create_obj(dsl_pool_t *dp, dmu_tx_t *tx)
    726 {
    727 	objset_t *mos = dp->dp_meta_objset;
    728 
    729 	ASSERT(dp->dp_tmp_userrefs_obj == 0);
    730 	ASSERT(dmu_tx_is_syncing(tx));
    731 
    732 	dp->dp_tmp_userrefs_obj = zap_create(mos, DMU_OT_USERREFS,
    733 	    DMU_OT_NONE, 0, tx);
    734 
    735 	VERIFY(zap_add(mos, DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_TMP_USERREFS,
    736 	    sizeof (uint64_t), 1, &dp->dp_tmp_userrefs_obj, tx) == 0);
    737 }
    738 
    739 static int
    740 dsl_pool_user_hold_rele_impl(dsl_pool_t *dp, uint64_t dsobj,
    741     const char *tag, uint64_t *now, dmu_tx_t *tx, boolean_t holding)
    742 {
    743 	objset_t *mos = dp->dp_meta_objset;
    744 	uint64_t zapobj = dp->dp_tmp_userrefs_obj;
    745 	char *name;
    746 	int error;
    747 
    748 	ASSERT(spa_version(dp->dp_spa) >= SPA_VERSION_USERREFS);
    749 	ASSERT(dmu_tx_is_syncing(tx));
    750 
    751 	/*
    752 	 * If the pool was created prior to SPA_VERSION_USERREFS, the
    753 	 * zap object for temporary holds might not exist yet.
    754 	 */
    755 	if (zapobj == 0) {
    756 		if (holding) {
    757 			dsl_pool_user_hold_create_obj(dp, tx);
    758 			zapobj = dp->dp_tmp_userrefs_obj;
    759 		} else {
    760 			return (ENOENT);
    761 		}
    762 	}
    763 
    764 	name = kmem_asprintf("%llx-%s", (u_longlong_t)dsobj, tag);
    765 	if (holding)
    766 		error = zap_add(mos, zapobj, name, 8, 1, now, tx);
    767 	else
    768 		error = zap_remove(mos, zapobj, name, tx);
    769 	strfree(name);
    770 
    771 	return (error);
    772 }
    773 
    774 /*
    775  * Add a temporary hold for the given dataset object and tag.
    776  */
    777 int
    778 dsl_pool_user_hold(dsl_pool_t *dp, uint64_t dsobj, const char *tag,
    779     uint64_t *now, dmu_tx_t *tx)
    780 {
    781 	return (dsl_pool_user_hold_rele_impl(dp, dsobj, tag, now, tx, B_TRUE));
    782 }
    783 
    784 /*
    785  * Release a temporary hold for the given dataset object and tag.
    786  */
    787 int
    788 dsl_pool_user_release(dsl_pool_t *dp, uint64_t dsobj, const char *tag,
    789     dmu_tx_t *tx)
    790 {
    791 	return (dsl_pool_user_hold_rele_impl(dp, dsobj, tag, NULL,
    792 	    tx, B_FALSE));
    793 }
    794