xref: /llvm-project/llvm/utils/lit/tests/unit/TestRunner.py (revision 2ee8763fecad644c1b25bfdb06a884d291633009) 
1# RUN: %{python} %s
2#
3# END.
4
5
6import os.path
7import platform
8import unittest
9
10import lit.discovery
11import lit.LitConfig
12import lit.Test as Test
13from lit.TestRunner import (
14    ParserKind,
15    IntegratedTestKeywordParser,
16    parseIntegratedTestScript,
17)
18
19
20class TestIntegratedTestKeywordParser(unittest.TestCase):
21    inputTestCase = None
22
23    @staticmethod
24    def load_keyword_parser_lit_tests():
25        """
26        Create and load the LIT test suite and test objects used by
27        TestIntegratedTestKeywordParser
28        """
29        # Create the global config object.
30        lit_config = lit.LitConfig.LitConfig(
31            progname="lit",
32            path=[],
33            quiet=False,
34            useValgrind=False,
35            valgrindLeakCheck=False,
36            valgrindArgs=[],
37            noExecute=False,
38            debug=False,
39            isWindows=(platform.system() == "Windows"),
40            order="smart",
41            params={},
42        )
43        TestIntegratedTestKeywordParser.litConfig = lit_config
44        # Perform test discovery.
45        test_path = os.path.dirname(os.path.dirname(__file__))
46        inputs = [os.path.join(test_path, "Inputs/testrunner-custom-parsers/")]
47        assert os.path.isdir(inputs[0])
48        tests = lit.discovery.find_tests_for_inputs(lit_config, inputs)
49        assert len(tests) == 1 and "there should only be one test"
50        TestIntegratedTestKeywordParser.inputTestCase = tests[0]
51
52    @staticmethod
53    def make_parsers():
54        def custom_parse(line_number, line, output):
55            if output is None:
56                output = []
57            output += [part for part in line.split(" ") if part.strip()]
58            return output
59
60        return [
61            IntegratedTestKeywordParser("MY_TAG.", ParserKind.TAG),
62            IntegratedTestKeywordParser("MY_DNE_TAG.", ParserKind.TAG),
63            IntegratedTestKeywordParser("MY_LIST:", ParserKind.LIST),
64            IntegratedTestKeywordParser("MY_SPACE_LIST:", ParserKind.SPACE_LIST),
65            IntegratedTestKeywordParser("MY_BOOL:", ParserKind.BOOLEAN_EXPR),
66            IntegratedTestKeywordParser("MY_INT:", ParserKind.INTEGER),
67            IntegratedTestKeywordParser("MY_RUN:", ParserKind.COMMAND),
68            IntegratedTestKeywordParser("MY_CUSTOM:", ParserKind.CUSTOM, custom_parse),
69            IntegratedTestKeywordParser("MY_DEFINE:", ParserKind.DEFINE),
70            IntegratedTestKeywordParser("MY_REDEFINE:", ParserKind.REDEFINE),
71        ]
72
73    @staticmethod
74    def get_parser(parser_list, keyword):
75        for p in parser_list:
76            if p.keyword == keyword:
77                return p
78        assert False and "parser not found"
79
80    @staticmethod
81    def parse_test(parser_list, allow_result=False):
82        script = parseIntegratedTestScript(
83            TestIntegratedTestKeywordParser.inputTestCase,
84            additional_parsers=parser_list,
85            require_script=False,
86        )
87        if isinstance(script, lit.Test.Result):
88            assert allow_result
89        else:
90            assert isinstance(script, list)
91            assert len(script) == 0
92        return script
93
94    def test_tags(self):
95        parsers = self.make_parsers()
96        self.parse_test(parsers)
97        tag_parser = self.get_parser(parsers, "MY_TAG.")
98        dne_tag_parser = self.get_parser(parsers, "MY_DNE_TAG.")
99        self.assertTrue(tag_parser.getValue())
100        self.assertFalse(dne_tag_parser.getValue())
101
102    def test_lists(self):
103        parsers = self.make_parsers()
104        self.parse_test(parsers)
105        list_parser = self.get_parser(parsers, "MY_LIST:")
106        self.assertEqual(list_parser.getValue(), ["one", "two", "three", "four"])
107
108    def test_space_lists(self):
109        parsers = self.make_parsers()
110        self.parse_test(parsers)
111        space_list_parser = self.get_parser(parsers, "MY_SPACE_LIST:")
112        self.assertEqual(
113            space_list_parser.getValue(),
114            [
115                "orange",
116                "tabby",
117                "tortie",
118                "tuxedo",
119                "void",
120                "multiple",
121                "spaces",
122                "cute,",
123                "fluffy,",
124                "kittens",
125            ],
126        )
127
128    def test_commands(self):
129        parsers = self.make_parsers()
130        self.parse_test(parsers)
131        cmd_parser = self.get_parser(parsers, "MY_RUN:")
132        value = cmd_parser.getValue()
133        self.assertEqual(len(value), 2)  # there are only two run lines
134        self.assertEqual(value[0].command.strip(), "%dbg(MY_RUN: at line 4)  baz")
135        self.assertEqual(value[1].command.strip(), "%dbg(MY_RUN: at line 12)  foo  bar")
136
137    def test_boolean(self):
138        parsers = self.make_parsers()
139        self.parse_test(parsers)
140        bool_parser = self.get_parser(parsers, "MY_BOOL:")
141        value = bool_parser.getValue()
142        self.assertEqual(len(value), 2)  # there are only two run lines
143        self.assertEqual(value[0].strip(), "a && (b)")
144        self.assertEqual(value[1].strip(), "d")
145
146    def test_integer(self):
147        parsers = self.make_parsers()
148        self.parse_test(parsers)
149        int_parser = self.get_parser(parsers, "MY_INT:")
150        value = int_parser.getValue()
151        self.assertEqual(len(value), 2)  # there are only two MY_INT: lines
152        self.assertEqual(type(value[0]), int)
153        self.assertEqual(value[0], 4)
154        self.assertEqual(type(value[1]), int)
155        self.assertEqual(value[1], 6)
156
157    def test_bad_parser_type(self):
158        parsers = self.make_parsers() + ["BAD_PARSER_TYPE"]
159        script = self.parse_test(parsers, allow_result=True)
160        self.assertTrue(isinstance(script, lit.Test.Result))
161        self.assertEqual(script.code, lit.Test.UNRESOLVED)
162        self.assertEqual(
163            "Additional parser must be an instance of " "IntegratedTestKeywordParser",
164            script.output,
165        )
166
167    def test_duplicate_keyword(self):
168        parsers = self.make_parsers() + [
169            IntegratedTestKeywordParser("KEY:", ParserKind.BOOLEAN_EXPR),
170            IntegratedTestKeywordParser("KEY:", ParserKind.BOOLEAN_EXPR),
171        ]
172        script = self.parse_test(parsers, allow_result=True)
173        self.assertTrue(isinstance(script, lit.Test.Result))
174        self.assertEqual(script.code, lit.Test.UNRESOLVED)
175        self.assertEqual("Parser for keyword 'KEY:' already exists", script.output)
176
177    def test_boolean_unterminated(self):
178        parsers = self.make_parsers() + [
179            IntegratedTestKeywordParser(
180                "MY_BOOL_UNTERMINATED:", ParserKind.BOOLEAN_EXPR
181            )
182        ]
183        script = self.parse_test(parsers, allow_result=True)
184        self.assertTrue(isinstance(script, lit.Test.Result))
185        self.assertEqual(script.code, lit.Test.UNRESOLVED)
186        self.assertEqual(
187            "Test has unterminated 'MY_BOOL_UNTERMINATED:' lines " "(with '\\')",
188            script.output,
189        )
190
191    def test_custom(self):
192        parsers = self.make_parsers()
193        self.parse_test(parsers)
194        custom_parser = self.get_parser(parsers, "MY_CUSTOM:")
195        value = custom_parser.getValue()
196        self.assertEqual(value, ["a", "b", "c"])
197
198    def test_defines(self):
199        parsers = self.make_parsers()
200        self.parse_test(parsers)
201        cmd_parser = self.get_parser(parsers, "MY_DEFINE:")
202        value = cmd_parser.getValue()
203        self.assertEqual(len(value), 1)  # there's only one MY_DEFINE directive
204        self.assertEqual(value[0].new_subst, True)
205        self.assertEqual(value[0].name, "%{name}")
206        self.assertEqual(value[0].value, "value one")
207
208    def test_redefines(self):
209        parsers = self.make_parsers()
210        self.parse_test(parsers)
211        cmd_parser = self.get_parser(parsers, "MY_REDEFINE:")
212        value = cmd_parser.getValue()
213        self.assertEqual(len(value), 1)  # there's only one MY_REDEFINE directive
214        self.assertEqual(value[0].new_subst, False)
215        self.assertEqual(value[0].name, "%{name}")
216        self.assertEqual(value[0].value, "value two")
217
218    def test_bad_keywords(self):
219        def custom_parse(line_number, line, output):
220            return output
221
222        try:
223            IntegratedTestKeywordParser("TAG_NO_SUFFIX", ParserKind.TAG),
224            self.fail("TAG_NO_SUFFIX failed to raise an exception")
225        except ValueError as e:
226            pass
227        except BaseException as e:
228            self.fail("TAG_NO_SUFFIX raised the wrong exception: %r" % e)
229
230        try:
231            IntegratedTestKeywordParser("TAG_WITH_COLON:", ParserKind.TAG),
232            self.fail("TAG_WITH_COLON: failed to raise an exception")
233        except ValueError as e:
234            pass
235        except BaseException as e:
236            self.fail("TAG_WITH_COLON: raised the wrong exception: %r" % e)
237
238        try:
239            IntegratedTestKeywordParser("LIST_WITH_DOT.", ParserKind.LIST),
240            self.fail("LIST_WITH_DOT. failed to raise an exception")
241        except ValueError as e:
242            pass
243        except BaseException as e:
244            self.fail("LIST_WITH_DOT. raised the wrong exception: %r" % e)
245
246        try:
247            IntegratedTestKeywordParser("SPACE_LIST_WITH_DOT.", ParserKind.SPACE_LIST),
248            self.fail("SPACE_LIST_WITH_DOT. failed to raise an exception")
249        except ValueError as e:
250            pass
251        except BaseException as e:
252            self.fail("SPACE_LIST_WITH_DOT. raised the wrong exception: %r" % e)
253
254        try:
255            IntegratedTestKeywordParser(
256                "CUSTOM_NO_SUFFIX", ParserKind.CUSTOM, custom_parse
257            ),
258            self.fail("CUSTOM_NO_SUFFIX failed to raise an exception")
259        except ValueError as e:
260            pass
261        except BaseException as e:
262            self.fail("CUSTOM_NO_SUFFIX raised the wrong exception: %r" % e)
263
264        # Both '.' and ':' are allowed for CUSTOM keywords.
265        try:
266            IntegratedTestKeywordParser(
267                "CUSTOM_WITH_DOT.", ParserKind.CUSTOM, custom_parse
268            ),
269        except BaseException as e:
270            self.fail("CUSTOM_WITH_DOT. raised an exception: %r" % e)
271        try:
272            IntegratedTestKeywordParser(
273                "CUSTOM_WITH_COLON:", ParserKind.CUSTOM, custom_parse
274            ),
275        except BaseException as e:
276            self.fail("CUSTOM_WITH_COLON: raised an exception: %r" % e)
277
278        try:
279            IntegratedTestKeywordParser("CUSTOM_NO_PARSER:", ParserKind.CUSTOM),
280            self.fail("CUSTOM_NO_PARSER: failed to raise an exception")
281        except ValueError as e:
282            pass
283        except BaseException as e:
284            self.fail("CUSTOM_NO_PARSER: raised the wrong exception: %r" % e)
285
286
287class TestApplySubtitutions(unittest.TestCase):
288    def test_simple(self):
289        script = ["echo %bar"]
290        substitutions = [("%bar", "hello")]
291        result = lit.TestRunner.applySubstitutions(script, substitutions)
292        self.assertEqual(result, ["echo hello"])
293
294    def test_multiple_substitutions(self):
295        script = ["echo %bar %baz"]
296        substitutions = [
297            ("%bar", "hello"),
298            ("%baz", "world"),
299            ("%useless", "shouldnt expand"),
300        ]
301        result = lit.TestRunner.applySubstitutions(script, substitutions)
302        self.assertEqual(result, ["echo hello world"])
303
304    def test_multiple_script_lines(self):
305        script = ["%cxx %compile_flags -c -o %t.o", "%cxx %link_flags %t.o -o %t.exe"]
306        substitutions = [
307            ("%cxx", "clang++"),
308            ("%compile_flags", "-std=c++11 -O3"),
309            ("%link_flags", "-lc++"),
310        ]
311        result = lit.TestRunner.applySubstitutions(script, substitutions)
312        self.assertEqual(
313            result,
314            ["clang++ -std=c++11 -O3 -c -o %t.o", "clang++ -lc++ %t.o -o %t.exe"],
315        )
316
317    def test_recursive_substitution_real(self):
318        script = ["%build %s"]
319        substitutions = [
320            ("%cxx", "clang++"),
321            ("%compile_flags", "-std=c++11 -O3"),
322            ("%link_flags", "-lc++"),
323            ("%build", "%cxx %compile_flags %link_flags %s -o %t.exe"),
324        ]
325        result = lit.TestRunner.applySubstitutions(
326            script, substitutions, recursion_limit=3
327        )
328        self.assertEqual(result, ["clang++ -std=c++11 -O3 -lc++ %s -o %t.exe %s"])
329
330    def test_recursive_substitution_limit(self):
331        script = ["%rec5"]
332        # Make sure the substitutions are not in an order where the global
333        # substitution would appear to be recursive just because they are
334        # processed in the right order.
335        substitutions = [
336            ("%rec1", "STOP"),
337            ("%rec2", "%rec1"),
338            ("%rec3", "%rec2"),
339            ("%rec4", "%rec3"),
340            ("%rec5", "%rec4"),
341        ]
342        for limit in [5, 6, 7]:
343            result = lit.TestRunner.applySubstitutions(
344                script, substitutions, recursion_limit=limit
345            )
346            self.assertEqual(result, ["STOP"])
347
348    def test_recursive_substitution_limit_exceeded(self):
349        script = ["%rec5"]
350        substitutions = [
351            ("%rec1", "STOP"),
352            ("%rec2", "%rec1"),
353            ("%rec3", "%rec2"),
354            ("%rec4", "%rec3"),
355            ("%rec5", "%rec4"),
356        ]
357        for limit in [0, 1, 2, 3, 4]:
358            try:
359                lit.TestRunner.applySubstitutions(
360                    script, substitutions, recursion_limit=limit
361                )
362                self.fail("applySubstitutions should have raised an exception")
363            except ValueError:
364                pass
365
366    def test_recursive_substitution_invalid_value(self):
367        script = ["%rec5"]
368        substitutions = [
369            ("%rec1", "STOP"),
370            ("%rec2", "%rec1"),
371            ("%rec3", "%rec2"),
372            ("%rec4", "%rec3"),
373            ("%rec5", "%rec4"),
374        ]
375        for limit in [-1, -2, -3, "foo"]:
376            try:
377                lit.TestRunner.applySubstitutions(
378                    script, substitutions, recursion_limit=limit
379                )
380                self.fail("applySubstitutions should have raised an exception")
381            except AssertionError:
382                pass
383
384
385if __name__ == "__main__":
386    TestIntegratedTestKeywordParser.load_keyword_parser_lit_tests()
387    unittest.main(verbosity=2)
388