1import argparse 2import enum 3import os 4import shlex 5import sys 6 7import lit.reports 8import lit.util 9 10 11class TestOrder(enum.Enum): 12 DEFAULT = enum.auto() 13 RANDOM = enum.auto() 14 15 16def parse_args(): 17 parser = argparse.ArgumentParser(prog='lit') 18 parser.add_argument('test_paths', 19 nargs='+', 20 metavar="TEST_PATH", 21 help='File or path to include in the test suite') 22 23 parser.add_argument('--version', 24 action='version', 25 version='%(prog)s ' + lit.__version__) 26 27 parser.add_argument("-j", "--threads", "--workers", 28 dest="workers", 29 metavar="N", 30 help="Number of workers used for testing", 31 type=_positive_int, 32 default=lit.util.usable_core_count()) 33 parser.add_argument("--config-prefix", 34 dest="configPrefix", 35 metavar="NAME", 36 help="Prefix for 'lit' config files") 37 parser.add_argument("-D", "--param", 38 dest="user_params", 39 metavar="NAME=VAL", 40 help="Add 'NAME' = 'VAL' to the user defined parameters", 41 action="append", 42 default=[]) 43 44 format_group = parser.add_argument_group("Output Format") 45 # FIXME: I find these names very confusing, although I like the 46 # functionality. 47 format_group.add_argument("-q", "--quiet", 48 help="Suppress no error output", 49 action="store_true") 50 format_group.add_argument("-s", "--succinct", 51 help="Reduce amount of output." 52 " Additionally, show a progress bar," 53 " unless --no-progress-bar is specified.", 54 action="store_true") 55 format_group.add_argument("-v", "--verbose", 56 dest="showOutput", 57 help="Show test output for failures", 58 action="store_true") 59 format_group.add_argument("-vv", "--echo-all-commands", 60 dest="echoAllCommands", 61 action="store_true", 62 help="Echo all commands as they are executed to stdout. In case of " 63 "failure, last command shown will be the failing one.") 64 format_group.add_argument("-a", "--show-all", 65 dest="showAllOutput", 66 help="Display all commandlines and output", 67 action="store_true") 68 format_group.add_argument("-o", "--output", 69 type=lit.reports.JsonReport, 70 help="Write test results to the provided path", 71 metavar="PATH") 72 format_group.add_argument("--no-progress-bar", 73 dest="useProgressBar", 74 help="Do not use curses based progress bar", 75 action="store_false") 76 77 # Note: this does not generate flags for user-defined result codes. 78 success_codes = [c for c in lit.Test.ResultCode.all_codes() 79 if not c.isFailure] 80 for code in success_codes: 81 format_group.add_argument( 82 "--show-{}".format(code.name.lower()), 83 dest="shown_codes", 84 help="Show {} tests ({})".format(code.label.lower(), code.name), 85 action="append_const", 86 const=code, 87 default=[]) 88 89 execution_group = parser.add_argument_group("Test Execution") 90 execution_group.add_argument("--path", 91 help="Additional paths to add to testing environment", 92 action="append", 93 default=[]) 94 execution_group.add_argument("--vg", 95 dest="useValgrind", 96 help="Run tests under valgrind", 97 action="store_true") 98 execution_group.add_argument("--vg-leak", 99 dest="valgrindLeakCheck", 100 help="Check for memory leaks under valgrind", 101 action="store_true") 102 execution_group.add_argument("--vg-arg", 103 dest="valgrindArgs", 104 metavar="ARG", 105 help="Specify an extra argument for valgrind", 106 action="append", 107 default=[]) 108 execution_group.add_argument("--time-tests", 109 help="Track elapsed wall time for each test", 110 action="store_true") 111 execution_group.add_argument("--no-execute", 112 dest="noExecute", 113 help="Don't execute any tests (assume PASS)", 114 action="store_true") 115 execution_group.add_argument("--xunit-xml-output", 116 type=lit.reports.XunitReport, 117 help="Write XUnit-compatible XML test reports to the specified file") 118 execution_group.add_argument("--time-trace-output", 119 type=lit.reports.TimeTraceReport, 120 help="Write Chrome tracing compatible JSON to the specified file") 121 execution_group.add_argument("--timeout", 122 dest="maxIndividualTestTime", 123 help="Maximum time to spend running a single test (in seconds). " 124 "0 means no time limit. [Default: 0]", 125 type=_non_negative_int) 126 execution_group.add_argument("--max-failures", 127 help="Stop execution after the given number of failures.", 128 type=_positive_int) 129 execution_group.add_argument("--allow-empty-runs", 130 help="Do not fail the run if all tests are filtered out", 131 action="store_true") 132 execution_group.add_argument("--ignore-fail", 133 dest="ignoreFail", 134 action="store_true", 135 help="Exit with status zero even if some tests fail") 136 execution_group.add_argument("--no-indirectly-run-check", 137 dest="indirectlyRunCheck", 138 help="Do not error if a test would not be run if the user had " 139 "specified the containing directory instead of naming the " 140 "test directly.", 141 action="store_false") 142 143 selection_group = parser.add_argument_group("Test Selection") 144 selection_group.add_argument("--max-tests", 145 metavar="N", 146 help="Maximum number of tests to run", 147 type=_positive_int) 148 selection_group.add_argument("--max-time", 149 dest="timeout", 150 metavar="N", 151 help="Maximum time to spend testing (in seconds)", 152 type=_positive_int) 153 selection_group.add_argument("--shuffle", 154 help="Run tests in random order", 155 action="store_true") 156 selection_group.add_argument("-i", "--incremental", 157 help="Run failed tests first (DEPRECATED: now always enabled)", 158 action="store_true") 159 selection_group.add_argument("--filter", 160 metavar="REGEX", 161 type=_case_insensitive_regex, 162 help="Only run tests with paths matching the given regular expression", 163 default=os.environ.get("LIT_FILTER", ".*")) 164 selection_group.add_argument("--filter-out", 165 metavar="REGEX", 166 type=_case_insensitive_regex, 167 help="Filter out tests with paths matching the given regular expression", 168 default=os.environ.get("LIT_FILTER_OUT", "^$")) 169 selection_group.add_argument("--xfail", 170 metavar="LIST", 171 type=_semicolon_list, 172 help="XFAIL tests with paths in the semicolon separated list", 173 default=os.environ.get("LIT_XFAIL", "")) 174 selection_group.add_argument("--num-shards", 175 dest="numShards", 176 metavar="M", 177 help="Split testsuite into M pieces and only run one", 178 type=_positive_int, 179 default=os.environ.get("LIT_NUM_SHARDS")) 180 selection_group.add_argument("--run-shard", 181 dest="runShard", 182 metavar="N", 183 help="Run shard #N of the testsuite", 184 type=_positive_int, 185 default=os.environ.get("LIT_RUN_SHARD")) 186 187 debug_group = parser.add_argument_group("Debug and Experimental Options") 188 debug_group.add_argument("--debug", 189 help="Enable debugging (for 'lit' development)", 190 action="store_true") 191 debug_group.add_argument("--show-suites", 192 help="Show discovered test suites and exit", 193 action="store_true") 194 debug_group.add_argument("--show-tests", 195 help="Show all discovered tests and exit", 196 action="store_true") 197 debug_group.add_argument("--show-used-features", 198 help="Show all features used in the test suite (in XFAIL, UNSUPPORTED and REQUIRES) and exit", 199 action="store_true") 200 201 # LIT is special: environment variables override command line arguments. 202 env_args = shlex.split(os.environ.get("LIT_OPTS", "")) 203 args = sys.argv[1:] + env_args 204 opts = parser.parse_args(args) 205 206 # Validate command line options 207 if opts.echoAllCommands: 208 opts.showOutput = True 209 210 if opts.incremental: 211 print('WARNING: --incremental is deprecated. Failing tests now always run first.') 212 213 if opts.shuffle: 214 opts.order = TestOrder.RANDOM 215 else: 216 opts.order = TestOrder.DEFAULT 217 218 if opts.numShards or opts.runShard: 219 if not opts.numShards or not opts.runShard: 220 parser.error("--num-shards and --run-shard must be used together") 221 if opts.runShard > opts.numShards: 222 parser.error("--run-shard must be between 1 and --num-shards (inclusive)") 223 opts.shard = (opts.runShard, opts.numShards) 224 else: 225 opts.shard = None 226 227 opts.reports = filter(None, [opts.output, opts.xunit_xml_output, opts.time_trace_output]) 228 229 return opts 230 231 232def _positive_int(arg): 233 return _int(arg, 'positive', lambda i: i > 0) 234 235 236def _non_negative_int(arg): 237 return _int(arg, 'non-negative', lambda i: i >= 0) 238 239 240def _int(arg, kind, pred): 241 desc = "requires {} integer, but found '{}'" 242 try: 243 i = int(arg) 244 except ValueError: 245 raise _error(desc, kind, arg) 246 if not pred(i): 247 raise _error(desc, kind, arg) 248 return i 249 250 251def _case_insensitive_regex(arg): 252 import re 253 try: 254 return re.compile(arg, re.IGNORECASE) 255 except re.error as reason: 256 raise _error("invalid regular expression: '{}', {}", arg, reason) 257 258 259def _semicolon_list(arg): 260 return arg.split(';') 261 262 263def _error(desc, *args): 264 msg = desc.format(*args) 265 return argparse.ArgumentTypeError(msg) 266