1 1544 eschrock /* 2 1544 eschrock * CDDL HEADER START 3 1544 eschrock * 4 1544 eschrock * The contents of this file are subject to the terms of the 5 1544 eschrock * Common Development and Distribution License (the "License"). 6 1544 eschrock * You may not use this file except in compliance with the License. 7 1544 eschrock * 8 1544 eschrock * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 1544 eschrock * or http://www.opensolaris.org/os/licensing. 10 1544 eschrock * See the License for the specific language governing permissions 11 1544 eschrock * and limitations under the License. 12 1544 eschrock * 13 1544 eschrock * When distributing Covered Code, include this CDDL HEADER in each 14 1544 eschrock * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 1544 eschrock * If applicable, add the following below this CDDL HEADER, with the 16 1544 eschrock * fields enclosed by brackets "[]" replaced with your own identifying 17 1544 eschrock * information: Portions Copyright [yyyy] [name of copyright owner] 18 1544 eschrock * 19 1544 eschrock * CDDL HEADER END 20 1544 eschrock */ 21 1544 eschrock /* 22 9725 Eric * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 1544 eschrock * Use is subject to license terms. 24 1544 eschrock */ 25 1544 eschrock 26 1544 eschrock /* 27 1544 eschrock * ZFS fault injection 28 1544 eschrock * 29 1544 eschrock * To handle fault injection, we keep track of a series of zinject_record_t 30 1544 eschrock * structures which describe which logical block(s) should be injected with a 31 1544 eschrock * fault. These are kept in a global list. Each record corresponds to a given 32 1544 eschrock * spa_t and maintains a special hold on the spa_t so that it cannot be deleted 33 1544 eschrock * or exported while the injection record exists. 34 1544 eschrock * 35 1544 eschrock * Device level injection is done using the 'zi_guid' field. If this is set, it 36 1544 eschrock * means that the error is destined for a particular device, not a piece of 37 1544 eschrock * data. 38 1544 eschrock * 39 1544 eschrock * This is a rather poor data structure and algorithm, but we don't expect more 40 1544 eschrock * than a few faults at any one time, so it should be sufficient for our needs. 41 1544 eschrock */ 42 1544 eschrock 43 1544 eschrock #include <sys/arc.h> 44 1544 eschrock #include <sys/zio_impl.h> 45 1544 eschrock #include <sys/zfs_ioctl.h> 46 1544 eschrock #include <sys/vdev_impl.h> 47 10922 Jeff #include <sys/dmu_objset.h> 48 6615 gw25295 #include <sys/fs/zfs.h> 49 1544 eschrock 50 1544 eschrock uint32_t zio_injection_enabled; 51 1544 eschrock 52 1544 eschrock typedef struct inject_handler { 53 1544 eschrock int zi_id; 54 1544 eschrock spa_t *zi_spa; 55 1544 eschrock zinject_record_t zi_record; 56 1544 eschrock list_node_t zi_link; 57 1544 eschrock } inject_handler_t; 58 1544 eschrock 59 1544 eschrock static list_t inject_handlers; 60 1544 eschrock static krwlock_t inject_lock; 61 1544 eschrock static int inject_next_id = 1; 62 1544 eschrock 63 1544 eschrock /* 64 1544 eschrock * Returns true if the given record matches the I/O in progress. 65 1544 eschrock */ 66 1544 eschrock static boolean_t 67 1544 eschrock zio_match_handler(zbookmark_t *zb, uint64_t type, 68 1544 eschrock zinject_record_t *record, int error) 69 1544 eschrock { 70 1544 eschrock /* 71 1544 eschrock * Check for a match against the MOS, which is based on type 72 1544 eschrock */ 73 10922 Jeff if (zb->zb_objset == DMU_META_OBJSET && 74 10922 Jeff record->zi_objset == DMU_META_OBJSET && 75 10922 Jeff record->zi_object == DMU_META_DNODE_OBJECT) { 76 1544 eschrock if (record->zi_type == DMU_OT_NONE || 77 1544 eschrock type == record->zi_type) 78 1544 eschrock return (record->zi_freq == 0 || 79 1544 eschrock spa_get_random(100) < record->zi_freq); 80 1544 eschrock else 81 1544 eschrock return (B_FALSE); 82 1544 eschrock } 83 1544 eschrock 84 1544 eschrock /* 85 1544 eschrock * Check for an exact match. 86 1544 eschrock */ 87 1544 eschrock if (zb->zb_objset == record->zi_objset && 88 1544 eschrock zb->zb_object == record->zi_object && 89 1544 eschrock zb->zb_level == record->zi_level && 90 1544 eschrock zb->zb_blkid >= record->zi_start && 91 1544 eschrock zb->zb_blkid <= record->zi_end && 92 1544 eschrock error == record->zi_error) 93 1544 eschrock return (record->zi_freq == 0 || 94 1544 eschrock spa_get_random(100) < record->zi_freq); 95 1544 eschrock 96 1544 eschrock return (B_FALSE); 97 1544 eschrock } 98 1544 eschrock 99 1544 eschrock /* 100 10594 George * Panic the system when a config change happens in the function 101 10594 George * specified by tag. 102 10594 George */ 103 10594 George void 104 10594 George zio_handle_panic_injection(spa_t *spa, char *tag) 105 10594 George { 106 10594 George inject_handler_t *handler; 107 10594 George 108 10594 George rw_enter(&inject_lock, RW_READER); 109 10594 George 110 10594 George for (handler = list_head(&inject_handlers); handler != NULL; 111 10594 George handler = list_next(&inject_handlers, handler)) { 112 10594 George 113 10594 George if (spa != handler->zi_spa) 114 10594 George continue; 115 10594 George 116 10594 George if (strcmp(tag, handler->zi_record.zi_func) == 0) 117 10594 George panic("Panic requested in function %s\n", tag); 118 10594 George } 119 10594 George 120 10594 George rw_exit(&inject_lock); 121 10594 George } 122 10594 George 123 10594 George /* 124 1544 eschrock * Determine if the I/O in question should return failure. Returns the errno 125 1544 eschrock * to be returned to the caller. 126 1544 eschrock */ 127 1544 eschrock int 128 1544 eschrock zio_handle_fault_injection(zio_t *zio, int error) 129 1544 eschrock { 130 1544 eschrock int ret = 0; 131 1544 eschrock inject_handler_t *handler; 132 1544 eschrock 133 1544 eschrock /* 134 1544 eschrock * Ignore I/O not associated with any logical data. 135 1544 eschrock */ 136 1544 eschrock if (zio->io_logical == NULL) 137 1544 eschrock return (0); 138 1544 eschrock 139 1544 eschrock /* 140 1544 eschrock * Currently, we only support fault injection on reads. 141 1544 eschrock */ 142 1544 eschrock if (zio->io_type != ZIO_TYPE_READ) 143 1544 eschrock return (0); 144 1544 eschrock 145 1544 eschrock rw_enter(&inject_lock, RW_READER); 146 1544 eschrock 147 1544 eschrock for (handler = list_head(&inject_handlers); handler != NULL; 148 1544 eschrock handler = list_next(&inject_handlers, handler)) { 149 1544 eschrock 150 1544 eschrock /* Ignore errors not destined for this pool */ 151 1544 eschrock if (zio->io_spa != handler->zi_spa) 152 1544 eschrock continue; 153 1544 eschrock 154 10594 George /* Ignore device errors and panic injection */ 155 10594 George if (handler->zi_record.zi_guid != 0 || 156 10921 Tim handler->zi_record.zi_func[0] != '\0' || 157 10921 Tim handler->zi_record.zi_duration != 0) 158 1544 eschrock continue; 159 1544 eschrock 160 1544 eschrock /* If this handler matches, return EIO */ 161 1544 eschrock if (zio_match_handler(&zio->io_logical->io_bookmark, 162 1544 eschrock zio->io_bp ? BP_GET_TYPE(zio->io_bp) : DMU_OT_NONE, 163 1544 eschrock &handler->zi_record, error)) { 164 1544 eschrock ret = error; 165 1544 eschrock break; 166 1544 eschrock } 167 1544 eschrock } 168 1544 eschrock 169 1544 eschrock rw_exit(&inject_lock); 170 1544 eschrock 171 1544 eschrock return (ret); 172 1544 eschrock } 173 1544 eschrock 174 6615 gw25295 /* 175 6615 gw25295 * Determine if the zio is part of a label update and has an injection 176 6615 gw25295 * handler associated with that portion of the label. Currently, we 177 6615 gw25295 * allow error injection in either the nvlist or the uberblock region of 178 6615 gw25295 * of the vdev label. 179 6615 gw25295 */ 180 6615 gw25295 int 181 6615 gw25295 zio_handle_label_injection(zio_t *zio, int error) 182 6615 gw25295 { 183 6615 gw25295 inject_handler_t *handler; 184 6615 gw25295 vdev_t *vd = zio->io_vd; 185 6615 gw25295 uint64_t offset = zio->io_offset; 186 6615 gw25295 int label; 187 6615 gw25295 int ret = 0; 188 6615 gw25295 189 10685 George if (offset >= VDEV_LABEL_START_SIZE && 190 6615 gw25295 offset < vd->vdev_psize - VDEV_LABEL_END_SIZE) 191 6615 gw25295 return (0); 192 6615 gw25295 193 6615 gw25295 rw_enter(&inject_lock, RW_READER); 194 6615 gw25295 195 6615 gw25295 for (handler = list_head(&inject_handlers); handler != NULL; 196 6615 gw25295 handler = list_next(&inject_handlers, handler)) { 197 6615 gw25295 uint64_t start = handler->zi_record.zi_start; 198 6615 gw25295 uint64_t end = handler->zi_record.zi_end; 199 6615 gw25295 200 10594 George /* Ignore device only faults or panic injection */ 201 10594 George if (handler->zi_record.zi_start == 0 || 202 10921 Tim handler->zi_record.zi_func[0] != '\0' || 203 10921 Tim handler->zi_record.zi_duration != 0) 204 6615 gw25295 continue; 205 6615 gw25295 206 6615 gw25295 /* 207 6615 gw25295 * The injection region is the relative offsets within a 208 6615 gw25295 * vdev label. We must determine the label which is being 209 6615 gw25295 * updated and adjust our region accordingly. 210 6615 gw25295 */ 211 6615 gw25295 label = vdev_label_number(vd->vdev_psize, offset); 212 6615 gw25295 start = vdev_label_offset(vd->vdev_psize, label, start); 213 6615 gw25295 end = vdev_label_offset(vd->vdev_psize, label, end); 214 6615 gw25295 215 6615 gw25295 if (zio->io_vd->vdev_guid == handler->zi_record.zi_guid && 216 6615 gw25295 (offset >= start && offset <= end)) { 217 6615 gw25295 ret = error; 218 6615 gw25295 break; 219 6615 gw25295 } 220 6615 gw25295 } 221 6615 gw25295 rw_exit(&inject_lock); 222 6615 gw25295 return (ret); 223 6615 gw25295 } 224 6615 gw25295 225 6615 gw25295 226 1544 eschrock int 227 9725 Eric zio_handle_device_injection(vdev_t *vd, zio_t *zio, int error) 228 1544 eschrock { 229 1544 eschrock inject_handler_t *handler; 230 1544 eschrock int ret = 0; 231 1544 eschrock 232 10685 George /* 233 10685 George * We skip over faults in the labels unless it's during 234 10685 George * device open (i.e. zio == NULL). 235 10685 George */ 236 10685 George if (zio != NULL) { 237 10685 George uint64_t offset = zio->io_offset; 238 10685 George 239 10685 George if (offset < VDEV_LABEL_START_SIZE || 240 10685 George offset >= vd->vdev_psize - VDEV_LABEL_END_SIZE) 241 10685 George return (0); 242 10685 George } 243 10685 George 244 1544 eschrock rw_enter(&inject_lock, RW_READER); 245 1544 eschrock 246 1544 eschrock for (handler = list_head(&inject_handlers); handler != NULL; 247 1544 eschrock handler = list_next(&inject_handlers, handler)) { 248 6615 gw25295 249 10921 Tim /* 250 10921 Tim * Ignore label specific faults, panic injection 251 10921 Tim * or fake writes 252 10921 Tim */ 253 10594 George if (handler->zi_record.zi_start != 0 || 254 10921 Tim handler->zi_record.zi_func[0] != '\0' || 255 10921 Tim handler->zi_record.zi_duration != 0) 256 6615 gw25295 continue; 257 1544 eschrock 258 1544 eschrock if (vd->vdev_guid == handler->zi_record.zi_guid) { 259 9725 Eric if (handler->zi_record.zi_failfast && 260 9725 Eric (zio == NULL || (zio->io_flags & 261 9725 Eric (ZIO_FLAG_IO_RETRY | ZIO_FLAG_TRYHARD)))) { 262 9725 Eric continue; 263 9725 Eric } 264 10685 George 265 10685 George /* Handle type specific I/O failures */ 266 10685 George if (zio != NULL && 267 10685 George handler->zi_record.zi_iotype != ZIO_TYPES && 268 10685 George handler->zi_record.zi_iotype != zio->io_type) 269 10685 George continue; 270 9725 Eric 271 1544 eschrock if (handler->zi_record.zi_error == error) { 272 1544 eschrock /* 273 1544 eschrock * For a failed open, pretend like the device 274 1544 eschrock * has gone away. 275 1544 eschrock */ 276 1544 eschrock if (error == ENXIO) 277 1544 eschrock vd->vdev_stat.vs_aux = 278 1544 eschrock VDEV_AUX_OPEN_FAILED; 279 1544 eschrock ret = error; 280 1544 eschrock break; 281 1544 eschrock } 282 1544 eschrock if (handler->zi_record.zi_error == ENXIO) { 283 1544 eschrock ret = EIO; 284 1544 eschrock break; 285 1544 eschrock } 286 1544 eschrock } 287 1544 eschrock } 288 1544 eschrock 289 1544 eschrock rw_exit(&inject_lock); 290 1544 eschrock 291 1544 eschrock return (ret); 292 10921 Tim } 293 10921 Tim 294 10921 Tim /* 295 10921 Tim * Simulate hardware that ignores cache flushes. For requested number 296 10921 Tim * of seconds nix the actual writing to disk. 297 10921 Tim */ 298 10921 Tim void 299 10921 Tim zio_handle_ignored_writes(zio_t *zio) 300 10921 Tim { 301 10921 Tim inject_handler_t *handler; 302 10921 Tim 303 10921 Tim rw_enter(&inject_lock, RW_READER); 304 10921 Tim 305 10921 Tim for (handler = list_head(&inject_handlers); handler != NULL; 306 10921 Tim handler = list_next(&inject_handlers, handler)) { 307 10921 Tim 308 10921 Tim /* Ignore errors not destined for this pool */ 309 10921 Tim if (zio->io_spa != handler->zi_spa) 310 10921 Tim continue; 311 10921 Tim 312 10921 Tim if (handler->zi_record.zi_duration == 0) 313 10921 Tim continue; 314 10921 Tim 315 10921 Tim /* 316 10921 Tim * Positive duration implies # of seconds, negative 317 10921 Tim * a number of txgs 318 10921 Tim */ 319 10921 Tim if (handler->zi_record.zi_timer == 0) { 320 10921 Tim if (handler->zi_record.zi_duration > 0) 321 11066 rafael handler->zi_record.zi_timer = ddi_get_lbolt64(); 322 10921 Tim else 323 10921 Tim handler->zi_record.zi_timer = zio->io_txg; 324 10921 Tim } 325 11026 Tim 326 11026 Tim /* Have a "problem" writing 60% of the time */ 327 11026 Tim if (spa_get_random(100) < 60) 328 11026 Tim zio->io_pipeline &= ~ZIO_VDEV_IO_STAGES; 329 10921 Tim break; 330 10921 Tim } 331 10921 Tim 332 10921 Tim rw_exit(&inject_lock); 333 10921 Tim } 334 10921 Tim 335 10921 Tim void 336 10921 Tim spa_handle_ignored_writes(spa_t *spa) 337 10921 Tim { 338 10921 Tim inject_handler_t *handler; 339 10921 Tim 340 10921 Tim if (zio_injection_enabled == 0) 341 10921 Tim return; 342 10921 Tim 343 10921 Tim rw_enter(&inject_lock, RW_READER); 344 10921 Tim 345 10921 Tim for (handler = list_head(&inject_handlers); handler != NULL; 346 10921 Tim handler = list_next(&inject_handlers, handler)) { 347 10921 Tim 348 10921 Tim /* Ignore errors not destined for this pool */ 349 10921 Tim if (spa != handler->zi_spa) 350 10921 Tim continue; 351 10921 Tim 352 10921 Tim if (handler->zi_record.zi_duration == 0) 353 10921 Tim continue; 354 10921 Tim 355 10921 Tim if (handler->zi_record.zi_duration > 0) { 356 10921 Tim VERIFY(handler->zi_record.zi_timer == 0 || 357 10921 Tim handler->zi_record.zi_timer + 358 11066 rafael handler->zi_record.zi_duration * hz > 359 11066 rafael ddi_get_lbolt64()); 360 10921 Tim } else { 361 10921 Tim /* duration is negative so the subtraction here adds */ 362 10921 Tim VERIFY(handler->zi_record.zi_timer == 0 || 363 10921 Tim handler->zi_record.zi_timer - 364 10921 Tim handler->zi_record.zi_duration >= 365 10922 Jeff spa_syncing_txg(spa)); 366 10921 Tim } 367 10921 Tim } 368 10921 Tim 369 10921 Tim rw_exit(&inject_lock); 370 1544 eschrock } 371 1544 eschrock 372 1544 eschrock /* 373 1544 eschrock * Create a new handler for the given record. We add it to the list, adding 374 1544 eschrock * a reference to the spa_t in the process. We increment zio_injection_enabled, 375 1544 eschrock * which is the switch to trigger all fault injection. 376 1544 eschrock */ 377 1544 eschrock int 378 1544 eschrock zio_inject_fault(char *name, int flags, int *id, zinject_record_t *record) 379 1544 eschrock { 380 1544 eschrock inject_handler_t *handler; 381 1544 eschrock int error; 382 1544 eschrock spa_t *spa; 383 1544 eschrock 384 1544 eschrock /* 385 1544 eschrock * If this is pool-wide metadata, make sure we unload the corresponding 386 1544 eschrock * spa_t, so that the next attempt to load it will trigger the fault. 387 1544 eschrock * We call spa_reset() to unload the pool appropriately. 388 1544 eschrock */ 389 1544 eschrock if (flags & ZINJECT_UNLOAD_SPA) 390 1544 eschrock if ((error = spa_reset(name)) != 0) 391 1544 eschrock return (error); 392 1544 eschrock 393 1544 eschrock if (!(flags & ZINJECT_NULL)) { 394 1544 eschrock /* 395 1544 eschrock * spa_inject_ref() will add an injection reference, which will 396 1544 eschrock * prevent the pool from being removed from the namespace while 397 1544 eschrock * still allowing it to be unloaded. 398 1544 eschrock */ 399 1544 eschrock if ((spa = spa_inject_addref(name)) == NULL) 400 1544 eschrock return (ENOENT); 401 1544 eschrock 402 1544 eschrock handler = kmem_alloc(sizeof (inject_handler_t), KM_SLEEP); 403 1544 eschrock 404 1544 eschrock rw_enter(&inject_lock, RW_WRITER); 405 1544 eschrock 406 1544 eschrock *id = handler->zi_id = inject_next_id++; 407 1544 eschrock handler->zi_spa = spa; 408 1544 eschrock handler->zi_record = *record; 409 1544 eschrock list_insert_tail(&inject_handlers, handler); 410 1544 eschrock atomic_add_32(&zio_injection_enabled, 1); 411 1544 eschrock 412 1544 eschrock rw_exit(&inject_lock); 413 1544 eschrock } 414 1544 eschrock 415 1544 eschrock /* 416 1544 eschrock * Flush the ARC, so that any attempts to read this data will end up 417 1544 eschrock * going to the ZIO layer. Note that this is a little overkill, but 418 1544 eschrock * we don't have the necessary ARC interfaces to do anything else, and 419 1544 eschrock * fault injection isn't a performance critical path. 420 1544 eschrock */ 421 1544 eschrock if (flags & ZINJECT_FLUSH_ARC) 422 5642 maybee arc_flush(NULL); 423 1544 eschrock 424 1544 eschrock return (0); 425 1544 eschrock } 426 1544 eschrock 427 1544 eschrock /* 428 1544 eschrock * Returns the next record with an ID greater than that supplied to the 429 1544 eschrock * function. Used to iterate over all handlers in the system. 430 1544 eschrock */ 431 1544 eschrock int 432 1544 eschrock zio_inject_list_next(int *id, char *name, size_t buflen, 433 1544 eschrock zinject_record_t *record) 434 1544 eschrock { 435 1544 eschrock inject_handler_t *handler; 436 1544 eschrock int ret; 437 1544 eschrock 438 1544 eschrock mutex_enter(&spa_namespace_lock); 439 1544 eschrock rw_enter(&inject_lock, RW_READER); 440 1544 eschrock 441 1544 eschrock for (handler = list_head(&inject_handlers); handler != NULL; 442 1544 eschrock handler = list_next(&inject_handlers, handler)) 443 1544 eschrock if (handler->zi_id > *id) 444 1544 eschrock break; 445 1544 eschrock 446 1544 eschrock if (handler) { 447 1544 eschrock *record = handler->zi_record; 448 1544 eschrock *id = handler->zi_id; 449 1544 eschrock (void) strncpy(name, spa_name(handler->zi_spa), buflen); 450 1544 eschrock ret = 0; 451 1544 eschrock } else { 452 1544 eschrock ret = ENOENT; 453 1544 eschrock } 454 1544 eschrock 455 1544 eschrock rw_exit(&inject_lock); 456 1544 eschrock mutex_exit(&spa_namespace_lock); 457 1544 eschrock 458 1544 eschrock return (ret); 459 1544 eschrock } 460 1544 eschrock 461 1544 eschrock /* 462 1544 eschrock * Clear the fault handler with the given identifier, or return ENOENT if none 463 1544 eschrock * exists. 464 1544 eschrock */ 465 1544 eschrock int 466 1544 eschrock zio_clear_fault(int id) 467 1544 eschrock { 468 1544 eschrock inject_handler_t *handler; 469 1544 eschrock int ret; 470 1544 eschrock 471 1544 eschrock rw_enter(&inject_lock, RW_WRITER); 472 1544 eschrock 473 1544 eschrock for (handler = list_head(&inject_handlers); handler != NULL; 474 1544 eschrock handler = list_next(&inject_handlers, handler)) 475 1544 eschrock if (handler->zi_id == id) 476 1544 eschrock break; 477 1544 eschrock 478 1544 eschrock if (handler == NULL) { 479 1544 eschrock ret = ENOENT; 480 1544 eschrock } else { 481 1544 eschrock list_remove(&inject_handlers, handler); 482 1544 eschrock spa_inject_delref(handler->zi_spa); 483 1544 eschrock kmem_free(handler, sizeof (inject_handler_t)); 484 1544 eschrock atomic_add_32(&zio_injection_enabled, -1); 485 1544 eschrock ret = 0; 486 1544 eschrock } 487 1544 eschrock 488 1544 eschrock rw_exit(&inject_lock); 489 1544 eschrock 490 1544 eschrock return (ret); 491 1544 eschrock } 492 1544 eschrock 493 1544 eschrock void 494 1544 eschrock zio_inject_init(void) 495 1544 eschrock { 496 7313 Eric rw_init(&inject_lock, NULL, RW_DEFAULT, NULL); 497 1544 eschrock list_create(&inject_handlers, sizeof (inject_handler_t), 498 1544 eschrock offsetof(inject_handler_t, zi_link)); 499 1544 eschrock } 500 1544 eschrock 501 1544 eschrock void 502 1544 eschrock zio_inject_fini(void) 503 1544 eschrock { 504 1544 eschrock list_destroy(&inject_handlers); 505 7313 Eric rw_destroy(&inject_lock); 506 1544 eschrock } 507
Indexes created Sat Nov 28 12:39:51 UTC 2009
Terms of Use |
Privacy |
Trademarks |
Copyright Policy |
Site Guidelines |
Help
Your use of this web site or any of its content or software indicates your agreement to be bound by these Terms of Use.
Copyright © 1995-2008 Sun Microsystems, Inc.