1 // Copyright 2012 Google Inc.
2 // All rights reserved.
3 //
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are
6 // met:
7 //
8 // * Redistributions of source code must retain the above copyright
9 // notice, this list of conditions and the following disclaimer.
10 // * Redistributions in binary form must reproduce the above copyright
11 // notice, this list of conditions and the following disclaimer in the
12 // documentation and/or other materials provided with the distribution.
13 // * Neither the name of Google Inc. nor the names of its contributors
14 // may be used to endorse or promote products derived from this software
15 // without specific prior written permission.
16 //
17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 #include <stdlib.h>
30
31 #include <atf-c.h>
32
33 #define INTERFACE "atf"
34 #include "common_inttest.h"
35
36
37 ATF_TC(list__ok);
ATF_TC_HEAD(list__ok,tc)38 ATF_TC_HEAD(list__ok, tc) { setup(tc, true); }
ATF_TC_BODY(list__ok,tc)39 ATF_TC_BODY(list__ok, tc)
40 {
41 char* helpers = helpers_path(tc);
42 check(EXIT_SUCCESS,
43 "test_case{name='fail'}\n"
44 "test_case{name='pass'}\n"
45 "test_case{name='signal'}\n"
46 "test_case{name='sleep'}\n"
47 "test_case{name='cleanup_check_work_directory', has_cleanup='true'}\n"
48 "test_case{name='cleanup_fail', has_cleanup='true'}\n"
49 "test_case{name='cleanup_signal', has_cleanup='true'}\n"
50 "test_case{name='cleanup_sleep', has_cleanup='true'}\n"
51 "test_case{name='body_and_cleanup_fail', has_cleanup='true'}\n"
52 "test_case{name='print_config', has_cleanup='true'}\n",
53 "",
54 "list", helpers, NULL);
55 free(helpers);
56 }
57
58
59 ATF_TC(test__pass);
ATF_TC_HEAD(test__pass,tc)60 ATF_TC_HEAD(test__pass, tc) { setup(tc, true); }
ATF_TC_BODY(test__pass,tc)61 ATF_TC_BODY(test__pass, tc)
62 {
63 char* helpers = helpers_path(tc);
64 check(EXIT_SUCCESS,
65 "First line to stdout\nSecond line to stdout\n",
66 "First line to stderr\nSecond line to stderr\n",
67 "test", helpers, "pass", "test-result", NULL);
68 free(helpers);
69
70 ATF_REQUIRE(atf_utils_compare_file("test-result", "passed\n"));
71 }
72
73
74 ATF_TC(test__fail);
ATF_TC_HEAD(test__fail,tc)75 ATF_TC_HEAD(test__fail, tc) { setup(tc, true); }
ATF_TC_BODY(test__fail,tc)76 ATF_TC_BODY(test__fail, tc)
77 {
78 char* helpers = helpers_path(tc);
79 check(EXIT_FAILURE, "First line to stdout\n", "First line to stderr\n",
80 "test", helpers, "fail", "test-result", NULL);
81 free(helpers);
82
83 ATF_REQUIRE(atf_utils_compare_file("test-result",
84 "failed: This is the failure message\n"));
85 }
86
87
88 ATF_TC(test__crash);
ATF_TC_HEAD(test__crash,tc)89 ATF_TC_HEAD(test__crash, tc) { setup(tc, true); }
ATF_TC_BODY(test__crash,tc)90 ATF_TC_BODY(test__crash, tc)
91 {
92 char* helpers = helpers_path(tc);
93 check(EXIT_FAILURE, "", "save:crash.err",
94 "test", helpers, "signal", "test-result", NULL);
95 free(helpers);
96
97 ATF_REQUIRE(atf_utils_compare_file("test-result",
98 "broken: Premature exit; test case received signal 6 (core dumped)\n"));
99
100 ATF_REQUIRE(atf_utils_grep_file("About to die due to SIGABRT!",
101 "crash.err"));
102 ATF_REQUIRE(atf_utils_grep_file("attempting to gather stack trace",
103 "crash.err"));
104 }
105
106
107 ATF_TC(test__timeout);
ATF_TC_HEAD(test__timeout,tc)108 ATF_TC_HEAD(test__timeout, tc) { setup(tc, true); }
ATF_TC_BODY(test__timeout,tc)109 ATF_TC_BODY(test__timeout, tc)
110 {
111 char* helpers = helpers_path(tc);
112 check(EXIT_FAILURE, "", "Subprocess timed out; sending KILL signal...\n",
113 "-t1", "test", helpers, "sleep", "test-result", NULL);
114 free(helpers);
115
116 ATF_REQUIRE(atf_utils_compare_file("test-result",
117 "broken: Test case body timed out\n"));
118 }
119
120
121 ATF_TC(test__result_priority);
ATF_TC_HEAD(test__result_priority,tc)122 ATF_TC_HEAD(test__result_priority, tc) { setup(tc, true); }
ATF_TC_BODY(test__result_priority,tc)123 ATF_TC_BODY(test__result_priority, tc)
124 {
125 char* helpers = helpers_path(tc);
126 check(EXIT_FAILURE, "Killing cleanup\n", "",
127 "test", "-vhas.cleanup=true", helpers, "body_and_cleanup_fail",
128 "test-result", NULL);
129 free(helpers);
130
131 ATF_REQUIRE(atf_utils_compare_file("test-result", "failed: Body fails\n"));
132 }
133
134
135 ATF_TC(test__cleanup__ok);
ATF_TC_HEAD(test__cleanup__ok,tc)136 ATF_TC_HEAD(test__cleanup__ok, tc) { setup(tc, true); }
ATF_TC_BODY(test__cleanup__ok,tc)137 ATF_TC_BODY(test__cleanup__ok, tc)
138 {
139 char* helpers = helpers_path(tc);
140 check(EXIT_SUCCESS,
141 "Body stdout\nCleanup stdout\n"
142 "Cleanup properly ran in the same directory as the body\n",
143 "Body stderr\nCleanup stderr\n",
144 "test", "-vhas.cleanup=true", helpers, "cleanup_check_work_directory",
145 "test-result", NULL);
146 free(helpers);
147
148 ATF_REQUIRE(atf_utils_compare_file("test-result", "passed\n"));
149 }
150
151
152 ATF_TC(test__cleanup__fail);
ATF_TC_HEAD(test__cleanup__fail,tc)153 ATF_TC_HEAD(test__cleanup__fail, tc) { setup(tc, true); }
ATF_TC_BODY(test__cleanup__fail,tc)154 ATF_TC_BODY(test__cleanup__fail, tc)
155 {
156 char* helpers = helpers_path(tc);
157 check(EXIT_FAILURE, "", "",
158 "test", "-vhas.cleanup=true", helpers, "cleanup_fail", "test-result",
159 NULL);
160 free(helpers);
161
162 ATF_REQUIRE(atf_utils_compare_file("test-result", "broken: Test case "
163 "cleanup exited with code 1\n"));
164 }
165
166
167 ATF_TC(test__cleanup__crash);
ATF_TC_HEAD(test__cleanup__crash,tc)168 ATF_TC_HEAD(test__cleanup__crash, tc) { setup(tc, true); }
ATF_TC_BODY(test__cleanup__crash,tc)169 ATF_TC_BODY(test__cleanup__crash, tc)
170 {
171 char* helpers = helpers_path(tc);
172 check(EXIT_FAILURE, "", "save:crash.err",
173 "test", "-vhas.cleanup=true", helpers, "cleanup_signal",
174 "test-result", NULL);
175 free(helpers);
176
177 ATF_REQUIRE(atf_utils_compare_file("test-result",
178 "broken: Test case cleanup received signal 6 (core dumped)\n"));
179
180 ATF_REQUIRE(atf_utils_grep_file("About to die due to SIGABRT!",
181 "crash.err"));
182 ATF_REQUIRE(atf_utils_grep_file("attempting to gather stack trace",
183 "crash.err"));
184 }
185
186
187 ATF_TC(test__cleanup__timeout);
ATF_TC_HEAD(test__cleanup__timeout,tc)188 ATF_TC_HEAD(test__cleanup__timeout, tc) { setup(tc, true); }
ATF_TC_BODY(test__cleanup__timeout,tc)189 ATF_TC_BODY(test__cleanup__timeout, tc)
190 {
191 char* helpers = helpers_path(tc);
192 check(EXIT_FAILURE, "", "Subprocess timed out; sending KILL signal...\n",
193 "-t1", "test", "-vhas.cleanup=true", helpers, "cleanup_sleep",
194 "test-result", NULL);
195 free(helpers);
196
197 ATF_REQUIRE(atf_utils_compare_file("test-result", "broken: Test case "
198 "cleanup timed out\n"));
199 }
200
201
202 ATF_TC(test__config__builtin);
ATF_TC_HEAD(test__config__builtin,tc)203 ATF_TC_HEAD(test__config__builtin, tc) { setup(tc, true); }
ATF_TC_BODY(test__config__builtin,tc)204 ATF_TC_BODY(test__config__builtin, tc)
205 {
206 char* helpers = helpers_path(tc);
207 check(EXIT_SUCCESS, "", "",
208 "test", helpers, "print_config", "test-result", NULL);
209 free(helpers);
210
211 ATF_REQUIRE(atf_utils_compare_file("test-result", "passed\n"));
212 }
213
214
215 ATF_TC(test__config__custom);
ATF_TC_HEAD(test__config__custom,tc)216 ATF_TC_HEAD(test__config__custom, tc) { setup(tc, true); }
ATF_TC_BODY(test__config__custom,tc)217 ATF_TC_BODY(test__config__custom, tc)
218 {
219 char* helpers = helpers_path(tc);
220 check(EXIT_SUCCESS,
221 "body my-var1 value1\n"
222 "body v2 a b c foo\n"
223 "cleanup my-var1 value1\n"
224 "cleanup v2 a b c foo\n",
225 "",
226 "test", "-vmy-var1=value1", "-vv2=a b c foo", helpers,
227 "print_config", "test-result", NULL);
228 free(helpers);
229
230 ATF_REQUIRE(atf_utils_compare_file("test-result", "passed\n"));
231 }
232
233
234 ATF_TC(test__invalid_test_case_name);
ATF_TC_HEAD(test__invalid_test_case_name,tc)235 ATF_TC_HEAD(test__invalid_test_case_name, tc) { setup(tc, false); }
ATF_TC_BODY(test__invalid_test_case_name,tc)236 ATF_TC_BODY(test__invalid_test_case_name, tc)
237 {
238 char* helpers = helpers_path(tc);
239 check(EXIT_FAILURE, "", // TODO(jmmv): Should be EXIT_INTERNAL_ERROR.
240 "atf_helpers: ERROR: Unknown test case `foo'\n"
241 "atf_helpers: See atf-test-program(1) for usage details.\n",
242 "test", "-vhas.cleanup=false", helpers, "foo", "test-result", NULL);
243 free(helpers);
244
245 ATF_REQUIRE(atf_utils_compare_file("test-result",
246 "broken: Premature exit; test case exited with code 1\n"));
247 }
248
249
250 ATF_TC(test__missing_test_program);
ATF_TC_HEAD(test__missing_test_program,tc)251 ATF_TC_HEAD(test__missing_test_program, tc) { setup(tc, false); }
ATF_TC_BODY(test__missing_test_program,tc)252 ATF_TC_BODY(test__missing_test_program, tc)
253 {
254 check(EXIT_INTERNAL_ERROR, "",
255 "kyua-atf-tester: execvp failed: No such file or directory\n",
256 "test", "./non-existent", "pass", "test-result", NULL);
257
258 ATF_REQUIRE(!atf_utils_file_exists("test-result"));
259 }
260
261
ATF_TP_ADD_TCS(tp)262 ATF_TP_ADD_TCS(tp)
263 {
264 ATF_TP_ADD_TC(tp, top__missing_command);
265 ATF_TP_ADD_TC(tp, top__unknown_command);
266
267 ATF_TP_ADD_TC(tp, list__ok);
268
269 ATF_TP_ADD_TC(tp, test__pass);
270 ATF_TP_ADD_TC(tp, test__fail);
271 ATF_TP_ADD_TC(tp, test__crash);
272 ATF_TP_ADD_TC(tp, test__timeout);
273 ATF_TP_ADD_TC(tp, test__result_priority);
274 ATF_TP_ADD_TC(tp, test__cleanup__ok);
275 ATF_TP_ADD_TC(tp, test__cleanup__fail);
276 ATF_TP_ADD_TC(tp, test__cleanup__crash);
277 ATF_TP_ADD_TC(tp, test__cleanup__timeout);
278 ATF_TP_ADD_TC(tp, test__config__builtin);
279 ATF_TP_ADD_TC(tp, test__config__custom);
280 ATF_TP_ADD_TC(tp, test__missing_test_program);
281 ATF_TP_ADD_TC(tp, test__invalid_test_case_name);
282
283 return atf_no_error();
284 }
285