1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #include <sys/zfs_context.h> 27 #include <sys/spa.h> 28 #include <sys/dmu.h> 29 #include <sys/zap.h> 30 #include <sys/arc.h> 31 #include <sys/stat.h> 32 #include <sys/resource.h> 33 #include <sys/zil.h> 34 #include <sys/zil_impl.h> 35 #include <sys/dsl_dataset.h> 36 #include <sys/vdev.h> 37 #include <sys/dmu_tx.h> 38 39 /* 40 * The zfs intent log (ZIL) saves transaction records of system calls 41 * that change the file system in memory with enough information 42 * to be able to replay them. These are stored in memory until 43 * either the DMU transaction group (txg) commits them to the stable pool 44 * and they can be discarded, or they are flushed to the stable log 45 * (also in the pool) due to a fsync, O_DSYNC or other synchronous 46 * requirement. In the event of a panic or power fail then those log 47 * records (transactions) are replayed. 48 * 49 * There is one ZIL per file system. Its on-disk (pool) format consists 50 * of 3 parts: 51 * 52 * - ZIL header 53 * - ZIL blocks 54 * - ZIL records 55 * 56 * A log record holds a system call transaction. Log blocks can 57 * hold many log records and the blocks are chained together. 58 * Each ZIL block contains a block pointer (blkptr_t) to the next 59 * ZIL block in the chain. The ZIL header points to the first 60 * block in the chain. Note there is not a fixed place in the pool 61 * to hold blocks. They are dynamically allocated and freed as 62 * needed from the blocks available. Figure X shows the ZIL structure: 63 */ 64 65 /* 66 * This global ZIL switch affects all pools 67 */ 68 int zil_disable = 0; /* disable intent logging */ 69 70 /* 71 * Tunable parameter for debugging or performance analysis. Setting 72 * zfs_nocacheflush will cause corruption on power loss if a volatile 73 * out-of-order write cache is enabled. 74 */ 75 boolean_t zfs_nocacheflush = B_FALSE; 76 77 static kmem_cache_t *zil_lwb_cache; 78 79 static int 80 zil_dva_compare(const void *x1, const void *x2) 81 { 82 const dva_t *dva1 = x1; 83 const dva_t *dva2 = x2; 84 85 if (DVA_GET_VDEV(dva1) < DVA_GET_VDEV(dva2)) 86 return (-1); 87 if (DVA_GET_VDEV(dva1) > DVA_GET_VDEV(dva2)) 88 return (1); 89 90 if (DVA_GET_OFFSET(dva1) < DVA_GET_OFFSET(dva2)) 91 return (-1); 92 if (DVA_GET_OFFSET(dva1) > DVA_GET_OFFSET(dva2)) 93 return (1); 94 95 return (0); 96 } 97 98 static void 99 zil_dva_tree_init(avl_tree_t *t) 100 { 101 avl_create(t, zil_dva_compare, sizeof (zil_dva_node_t), 102 offsetof(zil_dva_node_t, zn_node)); 103 } 104 105 static void 106 zil_dva_tree_fini(avl_tree_t *t) 107 { 108 zil_dva_node_t *zn; 109 void *cookie = NULL; 110 111 while ((zn = avl_destroy_nodes(t, &cookie)) != NULL) 112 kmem_free(zn, sizeof (zil_dva_node_t)); 113 114 avl_destroy(t); 115 } 116 117 static int 118 zil_dva_tree_add(avl_tree_t *t, dva_t *dva) 119 { 120 zil_dva_node_t *zn; 121 avl_index_t where; 122 123 if (avl_find(t, dva, &where) != NULL) 124 return (EEXIST); 125 126 zn = kmem_alloc(sizeof (zil_dva_node_t), KM_SLEEP); 127 zn->zn_dva = *dva; 128 avl_insert(t, zn, where); 129 130 return (0); 131 } 132 133 static zil_header_t * 134 zil_header_in_syncing_context(zilog_t *zilog) 135 { 136 return ((zil_header_t *)zilog->zl_header); 137 } 138 139 static void 140 zil_init_log_chain(zilog_t *zilog, blkptr_t *bp) 141 { 142 zio_cksum_t *zc = &bp->blk_cksum; 143 144 zc->zc_word[ZIL_ZC_GUID_0] = spa_get_random(-1ULL); 145 zc->zc_word[ZIL_ZC_GUID_1] = spa_get_random(-1ULL); 146 zc->zc_word[ZIL_ZC_OBJSET] = dmu_objset_id(zilog->zl_os); 147 zc->zc_word[ZIL_ZC_SEQ] = 1ULL; 148 } 149 150 /* 151 * Read a log block, make sure it's valid, and byteswap it if necessary. 152 */ 153 static int 154 zil_read_log_block(zilog_t *zilog, const blkptr_t *bp, arc_buf_t **abufpp) 155 { 156 blkptr_t blk = *bp; 157 zbookmark_t zb; 158 uint32_t aflags = ARC_WAIT; 159 int error; 160 161 zb.zb_objset = bp->blk_cksum.zc_word[ZIL_ZC_OBJSET]; 162 zb.zb_object = 0; 163 zb.zb_level = -1; 164 zb.zb_blkid = bp->blk_cksum.zc_word[ZIL_ZC_SEQ]; 165 166 *abufpp = NULL; 167 168 /* 169 * We shouldn't be doing any scrubbing while we're doing log 170 * replay, it's OK to not lock. 171 */ 172 error = arc_read_nolock(NULL, zilog->zl_spa, &blk, 173 arc_getbuf_func, abufpp, ZIO_PRIORITY_SYNC_READ, ZIO_FLAG_CANFAIL | 174 ZIO_FLAG_SPECULATIVE | ZIO_FLAG_SCRUB, &aflags, &zb); 175 176 if (error == 0) { 177 char *data = (*abufpp)->b_data; 178 uint64_t blksz = BP_GET_LSIZE(bp); 179 zil_trailer_t *ztp = (zil_trailer_t *)(data + blksz) - 1; 180 zio_cksum_t cksum = bp->blk_cksum; 181 182 /* 183 * Validate the checksummed log block. 184 * 185 * Sequence numbers should be... sequential. The checksum 186 * verifier for the next block should be bp's checksum plus 1. 187 * 188 * Also check the log chain linkage and size used. 189 */ 190 cksum.zc_word[ZIL_ZC_SEQ]++; 191 192 if (bcmp(&cksum, &ztp->zit_next_blk.blk_cksum, 193 sizeof (cksum)) || BP_IS_HOLE(&ztp->zit_next_blk) || 194 (ztp->zit_nused > (blksz - sizeof (zil_trailer_t)))) { 195 error = ECKSUM; 196 } 197 198 if (error) { 199 VERIFY(arc_buf_remove_ref(*abufpp, abufpp) == 1); 200 *abufpp = NULL; 201 } 202 } 203 204 dprintf("error %d on %llu:%llu\n", error, zb.zb_objset, zb.zb_blkid); 205 206 return (error); 207 } 208 209 /* 210 * Parse the intent log, and call parse_func for each valid record within. 211 * Return the highest sequence number. 212 */ 213 uint64_t 214 zil_parse(zilog_t *zilog, zil_parse_blk_func_t *parse_blk_func, 215 zil_parse_lr_func_t *parse_lr_func, void *arg, uint64_t txg) 216 { 217 const zil_header_t *zh = zilog->zl_header; 218 uint64_t claim_seq = zh->zh_claim_seq; 219 uint64_t seq = 0; 220 uint64_t max_seq = 0; 221 blkptr_t blk = zh->zh_log; 222 arc_buf_t *abuf; 223 char *lrbuf, *lrp; 224 zil_trailer_t *ztp; 225 int reclen, error; 226 227 if (BP_IS_HOLE(&blk)) 228 return (max_seq); 229 230 /* 231 * Starting at the block pointed to by zh_log we read the log chain. 232 * For each block in the chain we strongly check that block to 233 * ensure its validity. We stop when an invalid block is found. 234 * For each block pointer in the chain we call parse_blk_func(). 235 * For each record in each valid block we call parse_lr_func(). 236 * If the log has been claimed, stop if we encounter a sequence 237 * number greater than the highest claimed sequence number. 238 */ 239 zil_dva_tree_init(&zilog->zl_dva_tree); 240 for (;;) { 241 seq = blk.blk_cksum.zc_word[ZIL_ZC_SEQ]; 242 243 if (claim_seq != 0 && seq > claim_seq) 244 break; 245 246 ASSERT(max_seq < seq); 247 max_seq = seq; 248 249 error = zil_read_log_block(zilog, &blk, &abuf); 250 251 if (parse_blk_func != NULL) 252 parse_blk_func(zilog, &blk, arg, txg); 253 254 if (error) 255 break; 256 257 lrbuf = abuf->b_data; 258 ztp = (zil_trailer_t *)(lrbuf + BP_GET_LSIZE(&blk)) - 1; 259 blk = ztp->zit_next_blk; 260 261 if (parse_lr_func == NULL) { 262 VERIFY(arc_buf_remove_ref(abuf, &abuf) == 1); 263 continue; 264 } 265 266 for (lrp = lrbuf; lrp < lrbuf + ztp->zit_nused; lrp += reclen) { 267 lr_t *lr = (lr_t *)lrp; 268 reclen = lr->lrc_reclen; 269 ASSERT3U(reclen, >=, sizeof (lr_t)); 270 parse_lr_func(zilog, lr, arg, txg); 271 } 272 VERIFY(arc_buf_remove_ref(abuf, &abuf) == 1); 273 } 274 zil_dva_tree_fini(&zilog->zl_dva_tree); 275 276 return (max_seq); 277 } 278 279 /* ARGSUSED */ 280 static void 281 zil_claim_log_block(zilog_t *zilog, blkptr_t *bp, void *tx, uint64_t first_txg) 282 { 283 spa_t *spa = zilog->zl_spa; 284 int err; 285 286 /* 287 * Claim log block if not already committed and not already claimed. 288 */ 289 if (bp->blk_birth >= first_txg && 290 zil_dva_tree_add(&zilog->zl_dva_tree, BP_IDENTITY(bp)) == 0) { 291 err = zio_wait(zio_claim(NULL, spa, first_txg, bp, NULL, NULL, 292 ZIO_FLAG_MUSTSUCCEED)); 293 ASSERT(err == 0); 294 } 295 } 296 297 static void 298 zil_claim_log_record(zilog_t *zilog, lr_t *lrc, void *tx, uint64_t first_txg) 299 { 300 if (lrc->lrc_txtype == TX_WRITE) { 301 lr_write_t *lr = (lr_write_t *)lrc; 302 zil_claim_log_block(zilog, &lr->lr_blkptr, tx, first_txg); 303 } 304 } 305 306 /* ARGSUSED */ 307 static void 308 zil_free_log_block(zilog_t *zilog, blkptr_t *bp, void *tx, uint64_t claim_txg) 309 { 310 zio_free_blk(zilog->zl_spa, bp, dmu_tx_get_txg(tx)); 311 } 312 313 static void 314 zil_free_log_record(zilog_t *zilog, lr_t *lrc, void *tx, uint64_t claim_txg) 315 { 316 /* 317 * If we previously claimed it, we need to free it. 318 */ 319 if (claim_txg != 0 && lrc->lrc_txtype == TX_WRITE) { 320 lr_write_t *lr = (lr_write_t *)lrc; 321 blkptr_t *bp = &lr->lr_blkptr; 322 if (bp->blk_birth >= claim_txg && 323 !zil_dva_tree_add(&zilog->zl_dva_tree, BP_IDENTITY(bp))) { 324 (void) arc_free(NULL, zilog->zl_spa, 325 dmu_tx_get_txg(tx), bp, NULL, NULL, ARC_WAIT); 326 } 327 } 328 } 329 330 /* 331 * Create an on-disk intent log. 332 */ 333 static void 334 zil_create(zilog_t *zilog) 335 { 336 const zil_header_t *zh = zilog->zl_header; 337 lwb_t *lwb; 338 uint64_t txg = 0; 339 dmu_tx_t *tx = NULL; 340 blkptr_t blk; 341 int error = 0; 342 343 /* 344 * Wait for any previous destroy to complete. 345 */ 346 txg_wait_synced(zilog->zl_dmu_pool, zilog->zl_destroy_txg); 347 348 ASSERT(zh->zh_claim_txg == 0); 349 ASSERT(zh->zh_replay_seq == 0); 350 351 blk = zh->zh_log; 352 353 /* 354 * If we don't already have an initial log block, allocate one now. 355 */ 356 if (BP_IS_HOLE(&blk)) { 357 tx = dmu_tx_create(zilog->zl_os); 358 (void) dmu_tx_assign(tx, TXG_WAIT); 359 dsl_dataset_dirty(dmu_objset_ds(zilog->zl_os), tx); 360 txg = dmu_tx_get_txg(tx); 361 362 error = zio_alloc_blk(zilog->zl_spa, ZIL_MIN_BLKSZ, &blk, 363 NULL, txg); 364 365 if (error == 0) 366 zil_init_log_chain(zilog, &blk); 367 } 368 369 /* 370 * Allocate a log write buffer (lwb) for the first log block. 371 */ 372 if (error == 0) { 373 lwb = kmem_cache_alloc(zil_lwb_cache, KM_SLEEP); 374 lwb->lwb_zilog = zilog; 375 lwb->lwb_blk = blk; 376 lwb->lwb_nused = 0; 377 lwb->lwb_sz = BP_GET_LSIZE(&lwb->lwb_blk); 378 lwb->lwb_buf = zio_buf_alloc(lwb->lwb_sz); 379 lwb->lwb_max_txg = txg; 380 lwb->lwb_zio = NULL; 381 382 mutex_enter(&zilog->zl_lock); 383 list_insert_tail(&zilog->zl_lwb_list, lwb); 384 mutex_exit(&zilog->zl_lock); 385 } 386 387 /* 388 * If we just allocated the first log block, commit our transaction 389 * and wait for zil_sync() to stuff the block poiner into zh_log. 390 * (zh is part of the MOS, so we cannot modify it in open context.) 391 */ 392 if (tx != NULL) { 393 dmu_tx_commit(tx); 394 txg_wait_synced(zilog->zl_dmu_pool, txg); 395 } 396 397 ASSERT(bcmp(&blk, &zh->zh_log, sizeof (blk)) == 0); 398 } 399 400 /* 401 * In one tx, free all log blocks and clear the log header. 402 * If keep_first is set, then we're replaying a log with no content. 403 * We want to keep the first block, however, so that the first 404 * synchronous transaction doesn't require a txg_wait_synced() 405 * in zil_create(). We don't need to txg_wait_synced() here either 406 * when keep_first is set, because both zil_create() and zil_destroy() 407 * will wait for any in-progress destroys to complete. 408 */ 409 void 410 zil_destroy(zilog_t *zilog, boolean_t keep_first) 411 { 412 const zil_header_t *zh = zilog->zl_header; 413 lwb_t *lwb; 414 dmu_tx_t *tx; 415 uint64_t txg; 416 417 /* 418 * Wait for any previous destroy to complete. 419 */ 420 txg_wait_synced(zilog->zl_dmu_pool, zilog->zl_destroy_txg); 421 422 if (BP_IS_HOLE(&zh->zh_log)) 423 return; 424 425 tx = dmu_tx_create(zilog->zl_os); 426 (void) dmu_tx_assign(tx, TXG_WAIT); 427 dsl_dataset_dirty(dmu_objset_ds(zilog->zl_os), tx); 428 txg = dmu_tx_get_txg(tx); 429 430 mutex_enter(&zilog->zl_lock); 431 432 /* 433 * It is possible for the ZIL to get the previously mounted zilog 434 * structure of the same dataset if quickly remounted and the dbuf 435 * eviction has not completed. In this case we can see a non 436 * empty lwb list and keep_first will be set. We fix this by 437 * clearing the keep_first. This will be slower but it's very rare. 438 */ 439 if (!list_is_empty(&zilog->zl_lwb_list) && keep_first) 440 keep_first = B_FALSE; 441 442 ASSERT3U(zilog->zl_destroy_txg, <, txg); 443 zilog->zl_destroy_txg = txg; 444 zilog->zl_keep_first = keep_first; 445 446 if (!list_is_empty(&zilog->zl_lwb_list)) { 447 ASSERT(zh->zh_claim_txg == 0); 448 ASSERT(!keep_first); 449 while ((lwb = list_head(&zilog->zl_lwb_list)) != NULL) { 450 list_remove(&zilog->zl_lwb_list, lwb); 451 if (lwb->lwb_buf != NULL) 452 zio_buf_free(lwb->lwb_buf, lwb->lwb_sz); 453 zio_free_blk(zilog->zl_spa, &lwb->lwb_blk, txg); 454 kmem_cache_free(zil_lwb_cache, lwb); 455 } 456 } else { 457 if (!keep_first) { 458 (void) zil_parse(zilog, zil_free_log_block, 459 zil_free_log_record, tx, zh->zh_claim_txg); 460 } 461 } 462 mutex_exit(&zilog->zl_lock); 463 464 dmu_tx_commit(tx); 465 } 466 467 /* 468 * zil_rollback_destroy() is only called by the rollback code. 469 * We already have a syncing tx. Rollback has exclusive access to the 470 * dataset, so we don't have to worry about concurrent zil access. 471 * The actual freeing of any log blocks occurs in zil_sync() later in 472 * this txg syncing phase. 473 */ 474 void 475 zil_rollback_destroy(zilog_t *zilog, dmu_tx_t *tx) 476 { 477 const zil_header_t *zh = zilog->zl_header; 478 uint64_t txg; 479 480 if (BP_IS_HOLE(&zh->zh_log)) 481 return; 482 483 txg = dmu_tx_get_txg(tx); 484 ASSERT3U(zilog->zl_destroy_txg, <, txg); 485 zilog->zl_destroy_txg = txg; 486 zilog->zl_keep_first = B_FALSE; 487 488 /* 489 * Ensure there's no outstanding ZIL IO. No lwbs or just the 490 * unused one that allocated in advance is ok. 491 */ 492 ASSERT(zilog->zl_lwb_list.list_head.list_next == 493 zilog->zl_lwb_list.list_head.list_prev); 494 (void) zil_parse(zilog, zil_free_log_block, zil_free_log_record, 495 tx, zh->zh_claim_txg); 496 } 497 498 int 499 zil_claim(char *osname, void *txarg) 500 { 501 dmu_tx_t *tx = txarg; 502 uint64_t first_txg = dmu_tx_get_txg(tx); 503 zilog_t *zilog; 504 zil_header_t *zh; 505 objset_t *os; 506 int error; 507 508 error = dmu_objset_open(osname, DMU_OST_ANY, DS_MODE_USER, &os); 509 if (error) { 510 cmn_err(CE_WARN, "can't open objset for %s", osname); 511 return (0); 512 } 513 514 zilog = dmu_objset_zil(os); 515 zh = zil_header_in_syncing_context(zilog); 516 517 /* 518 * Claim all log blocks if we haven't already done so, and remember 519 * the highest claimed sequence number. This ensures that if we can 520 * read only part of the log now (e.g. due to a missing device), 521 * but we can read the entire log later, we will not try to replay 522 * or destroy beyond the last block we successfully claimed. 523 */ 524 ASSERT3U(zh->zh_claim_txg, <=, first_txg); 525 if (zh->zh_claim_txg == 0 && !BP_IS_HOLE(&zh->zh_log)) { 526 zh->zh_claim_txg = first_txg; 527 zh->zh_claim_seq = zil_parse(zilog, zil_claim_log_block, 528 zil_claim_log_record, tx, first_txg); 529 dsl_dataset_dirty(dmu_objset_ds(os), tx); 530 } 531 532 ASSERT3U(first_txg, ==, (spa_last_synced_txg(zilog->zl_spa) + 1)); 533 dmu_objset_close(os); 534 return (0); 535 } 536 537 /* 538 * Check the log by walking the log chain. 539 * Checksum errors are ok as they indicate the end of the chain. 540 * Any other error (no device or read failure) returns an error. 541 */ 542 /* ARGSUSED */ 543 int 544 zil_check_log_chain(char *osname, void *txarg) 545 { 546 zilog_t *zilog; 547 zil_header_t *zh; 548 blkptr_t blk; 549 arc_buf_t *abuf; 550 objset_t *os; 551 char *lrbuf; 552 zil_trailer_t *ztp; 553 int error; 554 555 error = dmu_objset_open(osname, DMU_OST_ANY, DS_MODE_USER, &os); 556 if (error) { 557 cmn_err(CE_WARN, "can't open objset for %s", osname); 558 return (0); 559 } 560 561 zilog = dmu_objset_zil(os); 562 zh = zil_header_in_syncing_context(zilog); 563 blk = zh->zh_log; 564 if (BP_IS_HOLE(&blk)) { 565 dmu_objset_close(os); 566 return (0); /* no chain */ 567 } 568 569 for (;;) { 570 error = zil_read_log_block(zilog, &blk, &abuf); 571 if (error) 572 break; 573 lrbuf = abuf->b_data; 574 ztp = (zil_trailer_t *)(lrbuf + BP_GET_LSIZE(&blk)) - 1; 575 blk = ztp->zit_next_blk; 576 VERIFY(arc_buf_remove_ref(abuf, &abuf) == 1); 577 } 578 dmu_objset_close(os); 579 if (error == ECKSUM) 580 return (0); /* normal end of chain */ 581 return (error); 582 } 583 584 /* 585 * Clear a log chain 586 */ 587 /* ARGSUSED */ 588 int 589 zil_clear_log_chain(char *osname, void *txarg) 590 { 591 zilog_t *zilog; 592 zil_header_t *zh; 593 objset_t *os; 594 dmu_tx_t *tx; 595 int error; 596 597 error = dmu_objset_open(osname, DMU_OST_ANY, DS_MODE_USER, &os); 598 if (error) { 599 cmn_err(CE_WARN, "can't open objset for %s", osname); 600 return (0); 601 } 602 603 zilog = dmu_objset_zil(os); 604 tx = dmu_tx_create(zilog->zl_os); 605 (void) dmu_tx_assign(tx, TXG_WAIT); 606 zh = zil_header_in_syncing_context(zilog); 607 BP_ZERO(&zh->zh_log); 608 dsl_dataset_dirty(dmu_objset_ds(os), tx); 609 dmu_tx_commit(tx); 610 dmu_objset_close(os); 611 return (0); 612 } 613 614 static int 615 zil_vdev_compare(const void *x1, const void *x2) 616 { 617 uint64_t v1 = ((zil_vdev_node_t *)x1)->zv_vdev; 618 uint64_t v2 = ((zil_vdev_node_t *)x2)->zv_vdev; 619 620 if (v1 < v2) 621 return (-1); 622 if (v1 > v2) 623 return (1); 624 625 return (0); 626 } 627 628 void 629 zil_add_block(zilog_t *zilog, blkptr_t *bp) 630 { 631 avl_tree_t *t = &zilog->zl_vdev_tree; 632 avl_index_t where; 633 zil_vdev_node_t *zv, zvsearch; 634 int ndvas = BP_GET_NDVAS(bp); 635 int i; 636 637 if (zfs_nocacheflush) 638 return; 639 640 ASSERT(zilog->zl_writer); 641 642 /* 643 * Even though we're zl_writer, we still need a lock because the 644 * zl_get_data() callbacks may have dmu_sync() done callbacks 645 * that will run concurrently. 646 */ 647 mutex_enter(&zilog->zl_vdev_lock); 648 for (i = 0; i < ndvas; i++) { 649 zvsearch.zv_vdev = DVA_GET_VDEV(&bp->blk_dva[i]); 650 if (avl_find(t, &zvsearch, &where) == NULL) { 651 zv = kmem_alloc(sizeof (*zv), KM_SLEEP); 652 zv->zv_vdev = zvsearch.zv_vdev; 653 avl_insert(t, zv, where); 654 } 655 } 656 mutex_exit(&zilog->zl_vdev_lock); 657 } 658 659 void 660 zil_flush_vdevs(zilog_t *zilog) 661 { 662 spa_t *spa = zilog->zl_spa; 663 avl_tree_t *t = &zilog->zl_vdev_tree; 664 void *cookie = NULL; 665 zil_vdev_node_t *zv; 666 zio_t *zio; 667 668 ASSERT(zilog->zl_writer); 669 670 /* 671 * We don't need zl_vdev_lock here because we're the zl_writer, 672 * and all zl_get_data() callbacks are done. 673 */ 674 if (avl_numnodes(t) == 0) 675 return; 676 677 spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); 678 679 zio = zio_root(spa, NULL, NULL, ZIO_FLAG_CANFAIL); 680 681 while ((zv = avl_destroy_nodes(t, &cookie)) != NULL) { 682 vdev_t *vd = vdev_lookup_top(spa, zv->zv_vdev); 683 if (vd != NULL) 684 zio_flush(zio, vd); 685 kmem_free(zv, sizeof (*zv)); 686 } 687 688 /* 689 * Wait for all the flushes to complete. Not all devices actually 690 * support the DKIOCFLUSHWRITECACHE ioctl, so it's OK if it fails. 691 */ 692 (void) zio_wait(zio); 693 694 spa_config_exit(spa, SCL_STATE, FTAG); 695 } 696 697 /* 698 * Function called when a log block write completes 699 */ 700 static void 701 zil_lwb_write_done(zio_t *zio) 702 { 703 lwb_t *lwb = zio->io_private; 704 zilog_t *zilog = lwb->lwb_zilog; 705 706 ASSERT(BP_GET_COMPRESS(zio->io_bp) == ZIO_COMPRESS_OFF); 707 ASSERT(BP_GET_CHECKSUM(zio->io_bp) == ZIO_CHECKSUM_ZILOG); 708 ASSERT(BP_GET_TYPE(zio->io_bp) == DMU_OT_INTENT_LOG); 709 ASSERT(BP_GET_LEVEL(zio->io_bp) == 0); 710 ASSERT(BP_GET_BYTEORDER(zio->io_bp) == ZFS_HOST_BYTEORDER); 711 ASSERT(!BP_IS_GANG(zio->io_bp)); 712 ASSERT(!BP_IS_HOLE(zio->io_bp)); 713 ASSERT(zio->io_bp->blk_fill == 0); 714 715 /* 716 * Now that we've written this log block, we have a stable pointer 717 * to the next block in the chain, so it's OK to let the txg in 718 * which we allocated the next block sync. 719 */ 720 txg_rele_to_sync(&lwb->lwb_txgh); 721 722 zio_buf_free(lwb->lwb_buf, lwb->lwb_sz); 723 mutex_enter(&zilog->zl_lock); 724 lwb->lwb_buf = NULL; 725 if (zio->io_error) 726 zilog->zl_log_error = B_TRUE; 727 mutex_exit(&zilog->zl_lock); 728 } 729 730 /* 731 * Initialize the io for a log block. 732 */ 733 static void 734 zil_lwb_write_init(zilog_t *zilog, lwb_t *lwb) 735 { 736 zbookmark_t zb; 737 738 zb.zb_objset = lwb->lwb_blk.blk_cksum.zc_word[ZIL_ZC_OBJSET]; 739 zb.zb_object = 0; 740 zb.zb_level = -1; 741 zb.zb_blkid = lwb->lwb_blk.blk_cksum.zc_word[ZIL_ZC_SEQ]; 742 743 if (zilog->zl_root_zio == NULL) { 744 zilog->zl_root_zio = zio_root(zilog->zl_spa, NULL, NULL, 745 ZIO_FLAG_CANFAIL); 746 } 747 if (lwb->lwb_zio == NULL) { 748 lwb->lwb_zio = zio_rewrite(zilog->zl_root_zio, zilog->zl_spa, 749 0, &lwb->lwb_blk, lwb->lwb_buf, 750 lwb->lwb_sz, zil_lwb_write_done, lwb, 751 ZIO_PRIORITY_LOG_WRITE, ZIO_FLAG_CANFAIL, &zb); 752 } 753 } 754 755 /* 756 * Start a log block write and advance to the next log block. 757 * Calls are serialized. 758 */ 759 static lwb_t * 760 zil_lwb_write_start(zilog_t *zilog, lwb_t *lwb) 761 { 762 lwb_t *nlwb; 763 zil_trailer_t *ztp = (zil_trailer_t *)(lwb->lwb_buf + lwb->lwb_sz) - 1; 764 spa_t *spa = zilog->zl_spa; 765 blkptr_t *bp = &ztp->zit_next_blk; 766 uint64_t txg; 767 uint64_t zil_blksz; 768 int error; 769 770 ASSERT(lwb->lwb_nused <= ZIL_BLK_DATA_SZ(lwb)); 771 772 /* 773 * Allocate the next block and save its address in this block 774 * before writing it in order to establish the log chain. 775 * Note that if the allocation of nlwb synced before we wrote 776 * the block that points at it (lwb), we'd leak it if we crashed. 777 * Therefore, we don't do txg_rele_to_sync() until zil_lwb_write_done(). 778 */ 779 txg = txg_hold_open(zilog->zl_dmu_pool, &lwb->lwb_txgh); 780 txg_rele_to_quiesce(&lwb->lwb_txgh); 781 782 /* 783 * Pick a ZIL blocksize. We request a size that is the 784 * maximum of the previous used size, the current used size and 785 * the amount waiting in the queue. 786 */ 787 zil_blksz = MAX(zilog->zl_prev_used, 788 zilog->zl_cur_used + sizeof (*ztp)); 789 zil_blksz = MAX(zil_blksz, zilog->zl_itx_list_sz + sizeof (*ztp)); 790 zil_blksz = P2ROUNDUP_TYPED(zil_blksz, ZIL_MIN_BLKSZ, uint64_t); 791 if (zil_blksz > ZIL_MAX_BLKSZ) 792 zil_blksz = ZIL_MAX_BLKSZ; 793 794 BP_ZERO(bp); 795 /* pass the old blkptr in order to spread log blocks across devs */ 796 error = zio_alloc_blk(spa, zil_blksz, bp, &lwb->lwb_blk, txg); 797 if (error) { 798 dmu_tx_t *tx = dmu_tx_create_assigned(zilog->zl_dmu_pool, txg); 799 800 /* 801 * We dirty the dataset to ensure that zil_sync() will 802 * be called to remove this lwb from our zl_lwb_list. 803 * Failing to do so, may leave an lwb with a NULL lwb_buf 804 * hanging around on the zl_lwb_list. 805 */ 806 dsl_dataset_dirty(dmu_objset_ds(zilog->zl_os), tx); 807 dmu_tx_commit(tx); 808 809 /* 810 * Since we've just experienced an allocation failure so we 811 * terminate the current lwb and send it on its way. 812 */ 813 ztp->zit_pad = 0; 814 ztp->zit_nused = lwb->lwb_nused; 815 ztp->zit_bt.zbt_cksum = lwb->lwb_blk.blk_cksum; 816 zio_nowait(lwb->lwb_zio); 817 818 /* 819 * By returning NULL the caller will call tx_wait_synced() 820 */ 821 return (NULL); 822 } 823 824 ASSERT3U(bp->blk_birth, ==, txg); 825 ztp->zit_pad = 0; 826 ztp->zit_nused = lwb->lwb_nused; 827 ztp->zit_bt.zbt_cksum = lwb->lwb_blk.blk_cksum; 828 bp->blk_cksum = lwb->lwb_blk.blk_cksum; 829 bp->blk_cksum.zc_word[ZIL_ZC_SEQ]++; 830 831 /* 832 * Allocate a new log write buffer (lwb). 833 */ 834 nlwb = kmem_cache_alloc(zil_lwb_cache, KM_SLEEP); 835 836 nlwb->lwb_zilog = zilog; 837 nlwb->lwb_blk = *bp; 838 nlwb->lwb_nused = 0; 839 nlwb->lwb_sz = BP_GET_LSIZE(&nlwb->lwb_blk); 840 nlwb->lwb_buf = zio_buf_alloc(nlwb->lwb_sz); 841 nlwb->lwb_max_txg = txg; 842 nlwb->lwb_zio = NULL; 843 844 /* 845 * Put new lwb at the end of the log chain 846 */ 847 mutex_enter(&zilog->zl_lock); 848 list_insert_tail(&zilog->zl_lwb_list, nlwb); 849 mutex_exit(&zilog->zl_lock); 850 851 /* Record the block for later vdev flushing */ 852 zil_add_block(zilog, &lwb->lwb_blk); 853 854 /* 855 * kick off the write for the old log block 856 */ 857 dprintf_bp(&lwb->lwb_blk, "lwb %p txg %llu: ", lwb, txg); 858 ASSERT(lwb->lwb_zio); 859 zio_nowait(lwb->lwb_zio); 860 861 return (nlwb); 862 } 863 864 static lwb_t * 865 zil_lwb_commit(zilog_t *zilog, itx_t *itx, lwb_t *lwb) 866 { 867 lr_t *lrc = &itx->itx_lr; /* common log record */ 868 lr_write_t *lr = (lr_write_t *)lrc; 869 uint64_t txg = lrc->lrc_txg; 870 uint64_t reclen = lrc->lrc_reclen; 871 uint64_t dlen; 872 873 if (lwb == NULL) 874 return (NULL); 875 ASSERT(lwb->lwb_buf != NULL); 876 877 if (lrc->lrc_txtype == TX_WRITE && itx->itx_wr_state == WR_NEED_COPY) 878 dlen = P2ROUNDUP_TYPED( 879 lr->lr_length, sizeof (uint64_t), uint64_t); 880 else 881 dlen = 0; 882 883 zilog->zl_cur_used += (reclen + dlen); 884 885 zil_lwb_write_init(zilog, lwb); 886 887 /* 888 * If this record won't fit in the current log block, start a new one. 889 */ 890 if (lwb->lwb_nused + reclen + dlen > ZIL_BLK_DATA_SZ(lwb)) { 891 lwb = zil_lwb_write_start(zilog, lwb); 892 if (lwb == NULL) 893 return (NULL); 894 zil_lwb_write_init(zilog, lwb); 895 ASSERT(lwb->lwb_nused == 0); 896 if (reclen + dlen > ZIL_BLK_DATA_SZ(lwb)) { 897 txg_wait_synced(zilog->zl_dmu_pool, txg); 898 return (lwb); 899 } 900 } 901 902 /* 903 * Update the lrc_seq, to be log record sequence number. See zil.h 904 * Then copy the record to the log buffer. 905 */ 906 lrc->lrc_seq = ++zilog->zl_lr_seq; /* we are single threaded */ 907 bcopy(lrc, lwb->lwb_buf + lwb->lwb_nused, reclen); 908 909 /* 910 * If it's a write, fetch the data or get its blkptr as appropriate. 911 */ 912 if (lrc->lrc_txtype == TX_WRITE) { 913 if (txg > spa_freeze_txg(zilog->zl_spa)) 914 txg_wait_synced(zilog->zl_dmu_pool, txg); 915 if (itx->itx_wr_state != WR_COPIED) { 916 char *dbuf; 917 int error; 918 919 /* alignment is guaranteed */ 920 lr = (lr_write_t *)(lwb->lwb_buf + lwb->lwb_nused); 921 if (dlen) { 922 ASSERT(itx->itx_wr_state == WR_NEED_COPY); 923 dbuf = lwb->lwb_buf + lwb->lwb_nused + reclen; 924 lr->lr_common.lrc_reclen += dlen; 925 } else { 926 ASSERT(itx->itx_wr_state == WR_INDIRECT); 927 dbuf = NULL; 928 } 929 error = zilog->zl_get_data( 930 itx->itx_private, lr, dbuf, lwb->lwb_zio); 931 if (error) { 932 ASSERT(error == ENOENT || error == EEXIST || 933 error == EALREADY); 934 return (lwb); 935 } 936 } 937 } 938 939 lwb->lwb_nused += reclen + dlen; 940 lwb->lwb_max_txg = MAX(lwb->lwb_max_txg, txg); 941 ASSERT3U(lwb->lwb_nused, <=, ZIL_BLK_DATA_SZ(lwb)); 942 ASSERT3U(P2PHASE(lwb->lwb_nused, sizeof (uint64_t)), ==, 0); 943 944 return (lwb); 945 } 946 947 itx_t * 948 zil_itx_create(uint64_t txtype, size_t lrsize) 949 { 950 itx_t *itx; 951 952 lrsize = P2ROUNDUP_TYPED(lrsize, sizeof (uint64_t), size_t); 953 954 itx = kmem_alloc(offsetof(itx_t, itx_lr) + lrsize, KM_SLEEP); 955 itx->itx_lr.lrc_txtype = txtype; 956 itx->itx_lr.lrc_reclen = lrsize; 957 itx->itx_sod = lrsize; /* if write & WR_NEED_COPY will be increased */ 958 itx->itx_lr.lrc_seq = 0; /* defensive */ 959 960 return (itx); 961 } 962 963 uint64_t 964 zil_itx_assign(zilog_t *zilog, itx_t *itx, dmu_tx_t *tx) 965 { 966 uint64_t seq; 967 968 ASSERT(itx->itx_lr.lrc_seq == 0); 969 970 mutex_enter(&zilog->zl_lock); 971 list_insert_tail(&zilog->zl_itx_list, itx); 972 zilog->zl_itx_list_sz += itx->itx_sod; 973 itx->itx_lr.lrc_txg = dmu_tx_get_txg(tx); 974 itx->itx_lr.lrc_seq = seq = ++zilog->zl_itx_seq; 975 mutex_exit(&zilog->zl_lock); 976 977 return (seq); 978 } 979 980 /* 981 * Free up all in-memory intent log transactions that have now been synced. 982 */ 983 static void 984 zil_itx_clean(zilog_t *zilog) 985 { 986 uint64_t synced_txg = spa_last_synced_txg(zilog->zl_spa); 987 uint64_t freeze_txg = spa_freeze_txg(zilog->zl_spa); 988 list_t clean_list; 989 itx_t *itx; 990 991 list_create(&clean_list, sizeof (itx_t), offsetof(itx_t, itx_node)); 992 993 mutex_enter(&zilog->zl_lock); 994 /* wait for a log writer to finish walking list */ 995 while (zilog->zl_writer) { 996 cv_wait(&zilog->zl_cv_writer, &zilog->zl_lock); 997 } 998 999 /* 1000 * Move the sync'd log transactions to a separate list so we can call 1001 * kmem_free without holding the zl_lock. 10