Lines Matching +full:fault +full:- +full:inject
9 * or https://opensource.org/licenses/CDDL-1.0.
25 * Copyright (c) 2023-2025, Klara, Inc.
29 * ZFS Fault Injector
32 * from a user-visible object type and name to an internal representation.
38 * Errors can be injected into a particular vdev using the '-d' option. This
41 * ECHILD, and EILSEQ. These can be controlled through the '-e' option and the
46 * only apply to read operations (-T read) and will flip a bit after the device
49 * For label faults, the -L option must be specified. This allows faults
55 * zinject -d device [-e errno] [-L <uber | nvlist | pad1 | pad2>] pool
72 * errors on a particular object or blkid, we inject errors across
98 * These types should be self-explanatory. This tuple is then passed to the
99 * kernel via a special ioctl() to initiate fault injection for the given
100 * object. Note that 'type' is not strictly necessary for fault injection, but
101 * is used when translating existing faults into a human-readable string.
107 * zinject <-a | -u pool>
108 * zinject -c <id|all>
109 * zinject [-q] <-t type> [-f freq] [-u] [-a] [-m] [-e errno] [-l level]
110 * [-r range] <object>
111 * zinject [-f freq] [-a] [-m] [-u] -b objset:object:level:start:end pool
116 * The '-c' option will clear the given handler, or all handlers if 'all' is
119 * The '-e' option takes a string describing the errno to simulate. This must
121 * will result in the same behavior, but RAID-Z will produce a different set of
124 * The '-a', '-u', and '-m' flags toggle internal flush behavior. If '-a' is
125 * specified, then the ARC cache is flushed appropriately. If '-u' is
127 * specified independently of any other handlers. The '-m' flag automatically
131 * The '-f' flag controls the frequency of errors injected, expressed as a
140 * human-readable interface has been designed. It allows developers to specify
208 return ("-");
234 return (-1);
288 "\tzinject -c <id|all>\n"
293 "\tzinject -p <function name> pool\n"
294 "\t\tInject a panic fault at the specified function. Only \n"
298 "\tzinject -d device [-e errno] [-L <nvlist|uber|pad1|pad2>] [-F]\n"
299 "\t\t[-T <read|write|free|claim|flush|all>] [-f frequency] pool\n\n"
300 "\t\tInject a fault into a particular device or the device's\n"
308 "\tzinject -d device -A <degrade|fault> -D <delay secs> pool\n"
311 "\tzinject -d device -D latency:lanes pool\n"
319 "\t\tFor example, with a single lane delay of 10 ms (-D 10:1),\n"
328 "\t\tlanes (-D 10:2), then the device will be able to service\n"
336 "\t\tof '-D 10:1', is roughly equivalent to a single invocation\n"
337 "\t\tof '-D 10:2'. This also means, one can specify multiple\n"
339 "\t\tinvocation of '-D 10:1' followed by '-D 25:2' will\n"
343 "\tzinject -P import|export -s <seconds> pool\n"
348 "\tzinject -I [-s <seconds> | -g <txgs>] pool\n"
355 "\tzinject -b objset:object:level:blkid pool\n"
361 "\tzinject [-q] <-t type> [-C dvas] [-e errno] [-l level]\n"
362 "\t\t[-r range] [-a] [-m] [-u] [-f freq] <object>\n"
364 "\t\tInject an error into the object specified by the '-t' option\n"
366 "\t\tinterpreted depending on the '-t' option.\n"
368 "\t\t-q\tQuiet mode. Only print out the handler number added.\n"
369 "\t\t-e\tInject a specific error. Must be one of 'io',\n"
371 "\t\t-C\tInject the given error only into specific DVAs. The\n"
372 "\t\t\tDVAs should be specified as a list of 0-indexed DVAs\n"
374 "\t\t-l\tInject error at a particular block level. Default is "
376 "\t\t-m\tAutomatically remount underlying filesystem.\n"
377 "\t\t-r\tInject error over a particular logical range of an\n"
380 "\t\t-a\tFlush the ARC cache. Can be specified without any\n"
382 "\t\t-u\tUnload the associated pool. Can be specified with only\n"
384 "\t\t-f\tOnly inject errors a fraction of the time. Expressed as\n"
387 "\t-t data\t\tInject an error into the plain file contents of a\n"
391 "\t-t dnode\tInject an error into the metadnode in the block\n"
393 "\t\t\t'-r' option is incompatible with this mode. The object\n"
397 "\t-t <mos>\tInject errors into the MOS for objects of the given\n"
418 return (-1);
430 if (record->zi_guid != 0 || record->zi_func[0] != '\0' ||
431 record->zi_duration != 0) {
436 (void) printf("%3s %-15s %-6s %-6s %-8s %3s %-4s "
437 "%-15s %-6s %-15s\n", "ID", "POOL", "OBJSET", "OBJECT",
438 "TYPE", "LVL", "DVAs", "RANGE", "MATCH", "INJECT");
439 (void) printf("--- --------------- ------ "
440 "------ -------- --- ---- --------------- "
441 "------ ------\n");
447 if (record->zi_start == 0 && record->zi_end == -1ULL)
451 (u_longlong_t)record->zi_start,
452 (u_longlong_t)record->zi_end);
455 (void) printf("%3d %-15s %-6llu %-6llu %-8s %-3d 0x%02x %-15s "
456 "%6llu %6llu\n", id, pool, (u_longlong_t)record->zi_objset,
457 (u_longlong_t)record->zi_object, type_to_name(record->zi_type),
458 record->zi_level, record->zi_dvas, rangebuf,
459 (u_longlong_t)record->zi_match_count,
460 (u_longlong_t)record->zi_inject_count);
471 if (record->zi_guid == 0 || record->zi_func[0] != '\0')
474 if (record->zi_cmd == ZINJECT_DELAY_IO)
478 (void) printf("%3s %-15s %-16s %-5s %-10s %-9s "
479 "%-6s %-6s\n",
481 "MATCH", "INJECT");
483 "--- --------------- ---------------- "
484 "----- ---------- --------- "
485 "------ ------\n");
490 double freq = record->zi_freq == 0 ? 100.0f :
491 (((double)record->zi_freq) / ZI_PERCENTAGE_MAX) * 100.0f;
493 (void) printf("%3d %-15s %llx %-5s %-10s %8.4f%% "
494 "%6llu %6llu\n", id, pool, (u_longlong_t)record->zi_guid,
495 iotype_to_str(record->zi_iotype), err_to_str(record->zi_error),
496 freq, (u_longlong_t)record->zi_match_count,
497 (u_longlong_t)record->zi_inject_count);
508 if (record->zi_guid == 0 || record->zi_func[0] != '\0')
511 if (record->zi_cmd != ZINJECT_DELAY_IO)
515 (void) printf("%3s %-15s %-16s %-10s %-5s %-9s "
516 "%-6s %-6s\n",
518 "MATCH", "INJECT");
519 (void) printf("--- --------------- ---------------- "
520 "---------- ----- --------- "
521 "------ ------\n");
526 double freq = record->zi_freq == 0 ? 100.0f :
527 (((double)record->zi_freq) / ZI_PERCENTAGE_MAX) * 100.0f;
529 (void) printf("%3d %-15s %llx %10llu %5llu %8.4f%% "
530 "%6llu %6llu\n", id, pool, (u_longlong_t)record->zi_guid,
531 (u_longlong_t)NSEC2MSEC(record->zi_timer),
532 (u_longlong_t)record->zi_nlanes, freq,
533 (u_longlong_t)record->zi_match_count,
534 (u_longlong_t)record->zi_inject_count);
545 if (record->zi_func[0] == '\0')
549 (void) printf("%3s %-15s %s\n", "ID", "POOL", "FUNCTION");
550 (void) printf("--- --------------- ----------------\n");
555 (void) printf("%3d %-15s %s\n", id, pool, record->zi_func);
566 if (record->zi_cmd != ZINJECT_DELAY_IMPORT &&
567 record->zi_cmd != ZINJECT_DELAY_EXPORT) {
572 (void) printf("%3s %-19s %-11s %s\n",
574 (void) printf("--- ------------------- -----------"
575 " -------\n");
580 (void) printf("%3d %-19s %-11llu %s\n",
581 id, pool, (u_longlong_t)record->zi_duration,
582 record->zi_cmd == ZINJECT_DELAY_IMPORT ? "import": "export");
648 * Remove all fault injection handlers.
662 * Remove a specific fault injection handler.
683 * Register a new fault injection handler.
703 if (record->zi_cmd == ZINJECT_DELAY_IMPORT)
705 if (record->zi_cmd == ZINJECT_DELAY_EXPORT)
710 if (record->zi_cmd == ZINJECT_DELAY_IMPORT)
729 if (record->zi_guid) {
731 (u_longlong_t)record->zi_guid);
732 } else if (record->zi_func[0] != '\0') {
734 record->zi_func);
735 } else if (record->zi_duration > 0) {
737 (u_longlong_t)record->zi_duration);
738 } else if (record->zi_duration < 0) {
740 (u_longlong_t)-record->zi_duration);
741 } else if (record->zi_timer > 0) {
743 (u_longlong_t)NSEC2MSEC(record->zi_timer));
746 (u_longlong_t)record->zi_objset);
748 (u_longlong_t)record->zi_object);
750 (u_longlong_t)record->zi_type);
751 (void) printf(" level: %d\n", record->zi_level);
752 if (record->zi_start == 0 &&
753 record->zi_end == -1ULL)
757 (u_longlong_t)record->zi_start,
758 (u_longlong_t)record->zi_end);
759 (void) printf(" dvas: 0x%x\n", record->zi_dvas);
773 zc.zc_guid = record->zi_guid;
793 * off this value being non-zero in translate_device(), to
794 * determine if the fault is a ZINJECT_DELAY_IO fault or not.
836 * "1" -> 0b0010 (0x2)
837 * "0,1" -> 0b0011 (0x3)
838 * "0,1,2" -> 0b0111 (0x7)
861 if (mask & (1 << ((*c) - '0')))
864 mask |= (1 << ((*c) - '0'));
929 * available handlers, direct the user to '-h' for help
934 (void) printf("Run 'zinject -h' for usage "
942 ":aA:b:C:d:D:f:Fg:qhIc:t:T:l:mr:s:e:uL:p:P:")) != -1) {
950 } else if (strcasecmp(optarg, "fault") == 0) {
954 "must be 'degrade' or 'fault'\n", optarg);
1030 record.zi_duration *= -1;
1126 (void) fprintf(stderr, "option -%c requires an "
1140 argc -= optind;
1148 * '-c' is invalid with any other options.
1153 (void) fprintf(stderr, "cancel (-c) incompatible with "
1160 (void) fprintf(stderr, "extraneous argument to '-c'\n");
1183 * Device (-d) injection uses a completely different mechanism
1189 (void) fprintf(stderr, "device (-d) incompatible with "
1197 (void) fprintf(stderr, "device (-d) injection requires "
1253 (void) fprintf(stderr, "raw (-b) format with "
1261 (void) fprintf(stderr, "raw (-b) format expects a "
1290 "options\n", "import|export delay (-P)");
1297 (void) fprintf(stderr, "panic (-p) injection requires "
1314 "options\n", "import|export delay (-P)");
1321 (void) fprintf(stderr, "import|export delay (-P) "
1322 "injection requires a duration (-s) and a single "
1333 (void) fprintf(stderr, "hardware failure (-I) "
1341 (void) fprintf(stderr, "-s or -g meaningless "
1342 "without -I (ignore writes)\n");
1348 "in seconds (-s) or a number of txgs (-g) "
1354 (void) fprintf(stderr, "ignore writes (-I) "
1365 (void) fprintf(stderr, "at least one of '-b', '-d', "
1366 "'-t', '-a', '-p', '-I' or '-u' "
1378 "'-f'\n");
1402 (void) fprintf(stderr, "the '-C' option may "
1441 * If this is pool-wide metadata, unmount everything. The ioctl() will
1442 * unload the pool, so that we trigger spa-wide reopen of metadata next