1 // Copyright 2012 Google Inc.
2 // All rights reserved.
3 //
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are
6 // met:
7 //
8 // * Redistributions of source code must retain the above copyright
9 // notice, this list of conditions and the following disclaimer.
10 // * Redistributions in binary form must reproduce the above copyright
11 // notice, this list of conditions and the following disclaimer in the
12 // documentation and/or other materials provided with the distribution.
13 // * Neither the name of Google Inc. nor the names of its contributors
14 // may be used to endorse or promote products derived from this software
15 // without specific prior written permission.
16 //
17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 #include "atf_result.h"
30
31 #include <sys/resource.h>
32 #include <sys/wait.h>
33
34 #include <signal.h>
35 #include <stdlib.h>
36 #include <unistd.h>
37
38 #include <atf-c.h>
39
40 #include "error.h"
41
42
43 /// Evalutes an expression and ensures it does not return an error.
44 ///
45 /// \param expr A expression that must evaluate to kyua_error_t.
46 #define RE(expr) ATF_REQUIRE(!kyua_error_is_set(expr))
47
48
49 /// Generates a wait(2) status for a successful exit code.
50 ///
51 /// \param exitstatus The exit status to encode in the status.
52 ///
53 /// \return The wait(2) status.
54 static int
generate_wait_exitstatus(const int exitstatus)55 generate_wait_exitstatus(const int exitstatus)
56 {
57 const pid_t pid = fork();
58 ATF_REQUIRE(pid != -1);
59 if (pid == 0) {
60 exit(exitstatus);
61 } else {
62 int status;
63 ATF_REQUIRE(waitpid(pid, &status, 0) != -1);
64 return status;
65 }
66 }
67
68
69 /// Generates a wait(2) status for a termination due to a signal.
70 ///
71 /// \param signo The signal number to encode in the status.
72 ///
73 /// \return The wait(2) status.
74 static int
generate_wait_termsig(const int signo)75 generate_wait_termsig(const int signo)
76 {
77 // Explicitly disable core files to avoid inconsistent behaviors across
78 // operating systems. Some of the signal numbers passed to this function
79 // may have a meaning or not depending on the system, and this might mean
80 // that a core gets generated arbitrarily. As a result of this, our string
81 // comparisons below fail.
82 struct rlimit rl;
83 rl.rlim_cur = 0;
84 rl.rlim_max = RLIM_INFINITY;
85 if (setrlimit(RLIMIT_CORE, &rl) == -1)
86 atf_tc_skip("Failed to lower the core size limit");
87
88 const pid_t pid = fork();
89 ATF_REQUIRE(pid != -1);
90 if (pid == 0) {
91 kill(getpid(), signo);
92 abort();
93 } else {
94 int status;
95 ATF_REQUIRE(waitpid(pid, &status, 0) != -1);
96 return status;
97 }
98 }
99
100
101 /// Validates an execution of kyua_atf_result_rewrite().
102 ///
103 /// The code of this check is all within the macro instead of in a separate
104 /// function so that we get meaningful line numbers in test failure messages.
105 ///
106 /// \param name Name of the test case.
107 /// \param input Text of the ATF result.
108 /// \param wait_status Exit status of the ATF test case, as returned by wait(2).
109 /// \param timed_out Whether the test case timed out or not.
110 /// \param exp_output Text of the expected generic result.
111 /// \param exp_success Expected value of the success result.
112 #define CHECK(name, input, wait_status, timed_out, exp_output, exp_success) \
113 ATF_TC_WITHOUT_HEAD(name); \
114 ATF_TC_BODY(name, tc) \
115 { \
116 bool success; \
117 atf_utils_create_file("in.txt", "%s", input); \
118 RE(kyua_atf_result_rewrite("in.txt", "out.txt", wait_status, \
119 timed_out, &success)); \
120 atf_utils_cat_file("out.txt", "OUTPUT: "); \
121 ATF_REQUIRE(atf_utils_compare_file("out.txt", exp_output)); \
122 ATF_REQUIRE_EQ(exp_success, success); \
123 }
124
125
126 CHECK(rewrite__expected_death__exit_failure,
127 "expected_death: Some text\n",
128 generate_wait_exitstatus(EXIT_FAILURE), false,
129 "expected_failure: Some text\n",
130 true);
131 CHECK(rewrite__expected_death__signaled,
132 "expected_death: Foo bar\n",
133 generate_wait_termsig(1), false,
134 "expected_failure: Foo bar\n",
135 true);
136 CHECK(rewrite__expected_death__exit_success,
137 "expected_death: Some text\n",
138 generate_wait_exitstatus(EXIT_SUCCESS), false,
139 "failed: Test case expected to die but exited successfully\n",
140 false);
141 CHECK(rewrite__expected_death__no_reason,
142 "expected_death\n",
143 generate_wait_exitstatus(EXIT_FAILURE), false,
144 "broken: Test case should have reported a failure reason but didn't; "
145 "test case exited with code 1\n",
146 false);
147 CHECK(rewrite__expected_death__unexpected_arg,
148 "expected_death(23): Some text\n",
149 generate_wait_exitstatus(EXIT_FAILURE), false,
150 "broken: Unknown test case result status expected_death(23); "
151 "test case exited with code 1\n",
152 false);
153
154
155 CHECK(rewrite__expected_exit__exit_success,
156 "expected_exit: Some text\n",
157 generate_wait_exitstatus(EXIT_SUCCESS), false,
158 "expected_failure: Some text\n",
159 true);
160 CHECK(rewrite__expected_exit__exit_failure,
161 "expected_exit: Some other text\n",
162 generate_wait_exitstatus(EXIT_FAILURE), false,
163 "expected_failure: Some other text\n",
164 true);
165 CHECK(rewrite__expected_exit__arg_match,
166 "expected_exit(15): Some text\n",
167 generate_wait_exitstatus(15), false,
168 "expected_failure: Some text\n",
169 true);
170 CHECK(rewrite__expected_exit__arg_not_match,
171 "expected_exit(18): Some text\n",
172 generate_wait_exitstatus(15), false,
173 "failed: Test case expected to exit with code 18 but got code 15\n",
174 false);
175 CHECK(rewrite__expected_exit__signaled,
176 "expected_exit: Foo bar\n",
177 generate_wait_termsig(1), false,
178 "failed: Test case expected to exit normally but received signal 1\n",
179 false);
180 CHECK(rewrite__expected_exit__no_reason,
181 "expected_exit\n",
182 generate_wait_exitstatus(EXIT_FAILURE), false,
183 "broken: Test case should have reported a failure reason but didn't; "
184 "test case exited with code 1\n",
185 false);
186 CHECK(rewrite__expected_exit__bad_arg,
187 "expected_exit(abc): Some text\n",
188 generate_wait_exitstatus(25), false,
189 "broken: Invalid status argument (abc): not a number; test case exited "
190 "with code 25\n",
191 false);
192
193
194 CHECK(rewrite__expected_failure__ok,
195 "expected_failure: Some text\n",
196 generate_wait_exitstatus(EXIT_SUCCESS), false,
197 "expected_failure: Some text\n",
198 true);
199 CHECK(rewrite__expected_failure__failed,
200 "expected_failure: Some text\n",
201 generate_wait_exitstatus(EXIT_FAILURE), false,
202 "failed: Test case expected a failure but exited with error code 1\n",
203 false);
204 CHECK(rewrite__expected_failure__signaled,
205 "expected_failure: Foo bar\n",
206 generate_wait_termsig(1), false,
207 "failed: Test case expected a failure but received signal 1\n",
208 false);
209 CHECK(rewrite__expected_failure__no_reason,
210 "expected_failure\n",
211 generate_wait_exitstatus(EXIT_FAILURE), false,
212 "broken: Test case should have reported a failure reason but didn't; "
213 "test case exited with code 1\n",
214 false);
215 CHECK(rewrite__expected_failure__unexpected_arg,
216 "expected_failure(23): Some text\n",
217 generate_wait_exitstatus(EXIT_FAILURE), false,
218 "broken: Unknown test case result status expected_failure(23); "
219 "test case exited with code 1\n",
220 false);
221
222
223 CHECK(rewrite__expected_signal__ok,
224 "expected_signal: Some text\n",
225 generate_wait_termsig(6), false,
226 "expected_failure: Some text\n",
227 true);
228 CHECK(rewrite__expected_signal__arg_match,
229 "expected_signal(15): Some text\n",
230 generate_wait_termsig(15), false,
231 "expected_failure: Some text\n",
232 true);
233 CHECK(rewrite__expected_signal__arg_not_match,
234 "expected_signal(18): Some text\n",
235 generate_wait_termsig(15), false,
236 "failed: Test case expected to receive signal 18 but got 15\n",
237 false);
238 CHECK(rewrite__expected_signal__exited,
239 "expected_signal: Foo bar\n",
240 generate_wait_exitstatus(12), false,
241 "failed: Test case expected to receive a signal but exited with code 12\n",
242 false);
243 CHECK(rewrite__expected_signal__no_reason,
244 "expected_signal\n",
245 generate_wait_termsig(15), false,
246 "broken: Test case should have reported a failure reason but didn't; "
247 "test case received signal 15\n",
248 false);
249 CHECK(rewrite__expected_signal__bad_arg,
250 "expected_signal(abc): Some text\n",
251 generate_wait_termsig(25), false,
252 "broken: Invalid status argument (abc): not a number; test case received "
253 "signal 25\n",
254 false);
255
256
257 CHECK(rewrite__expected_timeout__ok,
258 "expected_timeout: Some text\n",
259 generate_wait_exitstatus(EXIT_SUCCESS), true,
260 "expected_failure: Some text\n",
261 true);
262 CHECK(rewrite__expected_timeout__exited,
263 "expected_timeout: Foo bar\n",
264 generate_wait_exitstatus(12), false,
265 "failed: Test case expected to time out but exited with code 12\n",
266 false);
267 CHECK(rewrite__expected_timeout__signaled,
268 "expected_timeout: Foo bar\n",
269 generate_wait_termsig(15), false,
270 "failed: Test case expected to time out but received signal 15\n",
271 false);
272 CHECK(rewrite__expected_timeout__no_reason,
273 "expected_timeout\n",
274 generate_wait_exitstatus(EXIT_FAILURE), false,
275 "broken: Test case should have reported a failure reason but didn't; "
276 "test case exited with code 1\n",
277 false);
278 CHECK(rewrite__expected_timeout__unexpected_arg,
279 "expected_timeout(23): Some text\n",
280 generate_wait_exitstatus(EXIT_FAILURE), false,
281 "broken: Unknown test case result status expected_timeout(23); "
282 "test case exited with code 1\n",
283 false);
284
285
286 CHECK(rewrite__failed__ok,
287 "failed: Some text\n",
288 generate_wait_exitstatus(EXIT_FAILURE), false,
289 "failed: Some text\n",
290 false);
291 CHECK(rewrite__failed__exit_success,
292 "failed: Some text\n",
293 generate_wait_exitstatus(EXIT_SUCCESS), false,
294 "broken: Test case reported a failed result but exited with a successful "
295 "exit code\n",
296 false);
297 CHECK(rewrite__failed__signaled,
298 "failed: Not used\n",
299 generate_wait_termsig(1), false,
300 "broken: Test case reported a failed result but received signal 1\n",
301 false);
302 CHECK(rewrite__failed__no_reason,
303 "failed\n",
304 generate_wait_exitstatus(EXIT_FAILURE), false,
305 "broken: Test case should have reported a failure reason but didn't; "
306 "test case exited with code 1\n",
307 false);
308 CHECK(rewrite__failed__unexpected_arg,
309 "failed(23): Some text\n",
310 generate_wait_exitstatus(EXIT_FAILURE), false,
311 "broken: Unknown test case result status failed(23); "
312 "test case exited with code 1\n",
313 false);
314
315
316 CHECK(rewrite__passed__ok,
317 "passed\n",
318 generate_wait_exitstatus(EXIT_SUCCESS), false,
319 "passed\n",
320 true);
321 CHECK(rewrite__passed__exit_failure,
322 "passed\n",
323 generate_wait_exitstatus(EXIT_FAILURE), false,
324 "broken: Test case reported a passed result but returned a non-zero "
325 "exit code 1\n",
326 false);
327 CHECK(rewrite__passed__signaled,
328 "passed\n",
329 generate_wait_termsig(1), false,
330 "broken: Test case reported a passed result but received signal 1\n",
331 false);
332 CHECK(rewrite__passed__unexpected_reason,
333 "passed: This should not be here\n",
334 generate_wait_exitstatus(EXIT_SUCCESS), false,
335 "broken: Found unexpected reason in passed test result; test case exited "
336 "with code 0\n",
337 false);
338 CHECK(rewrite__passed__unexpected_arg,
339 "passed(0)\n",
340 generate_wait_exitstatus(EXIT_SUCCESS), false,
341 "broken: Unknown test case result status passed(0); "
342 "test case exited with code 0\n",
343 false);
344
345
346 CHECK(rewrite__skipped__ok,
347 "skipped: Does not apply\n",
348 generate_wait_exitstatus(EXIT_SUCCESS), false,
349 "skipped: Does not apply\n",
350 true);
351 CHECK(rewrite__skipped__exit_failure,
352 "skipped: Some reason\n",
353 generate_wait_exitstatus(EXIT_FAILURE), false,
354 "broken: Test case reported a skipped result but returned a non-zero "
355 "exit code 1\n",
356 false);
357 CHECK(rewrite__skipped__signaled,
358 "skipped: Not used\n",
359 generate_wait_termsig(1), false,
360 "broken: Test case reported a skipped result but received signal 1\n",
361 false);
362 CHECK(rewrite__skipped__no_reason,
363 "skipped\n",
364 generate_wait_exitstatus(EXIT_SUCCESS), false,
365 "broken: Test case should have reported a failure reason but didn't; "
366 "test case exited with code 0\n",
367 false);
368 CHECK(rewrite__skipped__unexpected_arg,
369 "skipped(1): Some text\n",
370 generate_wait_exitstatus(EXIT_SUCCESS), false,
371 "broken: Unknown test case result status skipped(1); "
372 "test case exited with code 0\n",
373 false);
374
375
376 CHECK(rewrite__timed_out,
377 "this invalid result file should not be parsed\n",
378 generate_wait_exitstatus(EXIT_SUCCESS), true,
379 "broken: Test case body timed out\n",
380 false);
381
382
383 ATF_TC_WITHOUT_HEAD(rewrite__missing_file);
ATF_TC_BODY(rewrite__missing_file,tc)384 ATF_TC_BODY(rewrite__missing_file, tc)
385 {
386 bool success;
387 RE(kyua_atf_result_rewrite("in.txt", "out.txt",
388 generate_wait_exitstatus(EXIT_SUCCESS),
389 false, &success));
390 atf_utils_cat_file("out.txt", "OUTPUT: ");
391 ATF_REQUIRE(atf_utils_compare_file("out.txt",
392 "broken: Premature exit; test case exited with code 0\n"));
393 ATF_REQUIRE(!success);
394 }
395 CHECK(rewrite__empty,
396 "",
397 generate_wait_exitstatus(EXIT_FAILURE), false,
398 "broken: Empty result file in.txt; test case exited with code 1\n",
399 false);
400 CHECK(rewrite__unknown_status,
401 "foo\n",
402 generate_wait_exitstatus(123), false,
403 "broken: Unknown test case result status foo; test case exited with "
404 "code 123\n",
405 false);
406 CHECK(rewrite__status_without_newline,
407 "failed",
408 generate_wait_termsig(1), false,
409 "broken: Missing newline in result file; test case received signal 1\n",
410 false);
411 CHECK(rewrite__status_multiline,
412 "failed: First line\nSecond line\nThird line",
413 generate_wait_exitstatus(EXIT_FAILURE), false,
414 "failed: First line<<NEWLINE>>Second line<<NEWLINE>>Third line\n",
415 false);
416 CHECK(rewrite__status_multiline_2,
417 "failed: First line\nSecond line\nThird line\n",
418 generate_wait_exitstatus(EXIT_FAILURE), false,
419 "failed: First line<<NEWLINE>>Second line<<NEWLINE>>Third line\n",
420 false);
421
422
423 /// Validates an execution of kyua_atf_result_cleanup_rewrite().
424 ///
425 /// The code of this check is all within the macro instead of in a separate
426 /// function so that we get meaningful line numbers in test failure messages.
427 ///
428 /// \param name Name of the test case.
429 /// \param wait_status Exit status of the ATF test case, as returned by wait(2).
430 /// \param timed_out Whether the test case timed out or not.
431 /// \param exp_output Text of the expected generic result.
432 /// \param exp_success Expected value of the success result.
433 #define CHECK_CLEANUP(name, wait_status, timed_out, exp_output, exp_success) \
434 ATF_TC_WITHOUT_HEAD(name); \
435 ATF_TC_BODY(name, tc) \
436 { \
437 bool success; \
438 atf_utils_create_file("out.txt", "skipped: Preexistent file\n"); \
439 RE(kyua_atf_result_cleanup_rewrite("out.txt", wait_status, \
440 timed_out, &success)); \
441 atf_utils_cat_file("out.txt", "OUTPUT: "); \
442 ATF_REQUIRE(atf_utils_compare_file("out.txt", exp_output)); \
443 ATF_REQUIRE_EQ(exp_success, success); \
444 }
445
446
447 CHECK_CLEANUP(cleanup_rewrite__ok,
448 generate_wait_exitstatus(EXIT_SUCCESS), false,
449 "skipped: Preexistent file\n", // Previous file not overwritten.
450 true);
451 CHECK_CLEANUP(cleanup_rewrite__exit_failure,
452 generate_wait_exitstatus(EXIT_FAILURE), false,
453 "broken: Test case cleanup exited with code 1\n",
454 false);
455 CHECK_CLEANUP(cleanup_rewrite__signaled,
456 generate_wait_termsig(1), false,
457 "broken: Test case cleanup received signal 1\n",
458 false);
459 CHECK_CLEANUP(cleanup_rewrite__timed_out,
460 generate_wait_exitstatus(EXIT_SUCCESS), true,
461 "broken: Test case cleanup timed out\n",
462 false);
463
464
ATF_TP_ADD_TCS(tp)465 ATF_TP_ADD_TCS(tp)
466 {
467 ATF_TP_ADD_TC(tp, rewrite__expected_death__exit_failure);
468 ATF_TP_ADD_TC(tp, rewrite__expected_death__signaled);
469 ATF_TP_ADD_TC(tp, rewrite__expected_death__exit_success);
470 ATF_TP_ADD_TC(tp, rewrite__expected_death__no_reason);
471 ATF_TP_ADD_TC(tp, rewrite__expected_death__unexpected_arg);
472
473 ATF_TP_ADD_TC(tp, rewrite__expected_exit__exit_success);
474 ATF_TP_ADD_TC(tp, rewrite__expected_exit__exit_failure);
475 ATF_TP_ADD_TC(tp, rewrite__expected_exit__arg_match);
476 ATF_TP_ADD_TC(tp, rewrite__expected_exit__arg_not_match);
477 ATF_TP_ADD_TC(tp, rewrite__expected_exit__signaled);
478 ATF_TP_ADD_TC(tp, rewrite__expected_exit__no_reason);
479 ATF_TP_ADD_TC(tp, rewrite__expected_exit__bad_arg);
480
481 ATF_TP_ADD_TC(tp, rewrite__expected_failure__ok);
482 ATF_TP_ADD_TC(tp, rewrite__expected_failure__failed);
483 ATF_TP_ADD_TC(tp, rewrite__expected_failure__signaled);
484 ATF_TP_ADD_TC(tp, rewrite__expected_failure__no_reason);
485 ATF_TP_ADD_TC(tp, rewrite__expected_failure__unexpected_arg);
486
487 ATF_TP_ADD_TC(tp, rewrite__expected_signal__ok);
488 ATF_TP_ADD_TC(tp, rewrite__expected_signal__arg_match);
489 ATF_TP_ADD_TC(tp, rewrite__expected_signal__arg_not_match);
490 ATF_TP_ADD_TC(tp, rewrite__expected_signal__exited);
491 ATF_TP_ADD_TC(tp, rewrite__expected_signal__no_reason);
492 ATF_TP_ADD_TC(tp, rewrite__expected_signal__bad_arg);
493
494 ATF_TP_ADD_TC(tp, rewrite__expected_timeout__ok);
495 ATF_TP_ADD_TC(tp, rewrite__expected_timeout__exited);
496 ATF_TP_ADD_TC(tp, rewrite__expected_timeout__signaled);
497 ATF_TP_ADD_TC(tp, rewrite__expected_timeout__no_reason);
498 ATF_TP_ADD_TC(tp, rewrite__expected_timeout__unexpected_arg);
499
500 ATF_TP_ADD_TC(tp, rewrite__failed__ok);
501 ATF_TP_ADD_TC(tp, rewrite__failed__exit_success);
502 ATF_TP_ADD_TC(tp, rewrite__failed__signaled);
503 ATF_TP_ADD_TC(tp, rewrite__failed__no_reason);
504 ATF_TP_ADD_TC(tp, rewrite__failed__unexpected_arg);
505
506 ATF_TP_ADD_TC(tp, rewrite__passed__ok);
507 ATF_TP_ADD_TC(tp, rewrite__passed__exit_failure);
508 ATF_TP_ADD_TC(tp, rewrite__passed__signaled);
509 ATF_TP_ADD_TC(tp, rewrite__passed__unexpected_reason);
510 ATF_TP_ADD_TC(tp, rewrite__passed__unexpected_arg);
511
512 ATF_TP_ADD_TC(tp, rewrite__skipped__ok);
513 ATF_TP_ADD_TC(tp, rewrite__skipped__exit_failure);
514 ATF_TP_ADD_TC(tp, rewrite__skipped__signaled);
515 ATF_TP_ADD_TC(tp, rewrite__skipped__no_reason);
516 ATF_TP_ADD_TC(tp, rewrite__skipped__unexpected_arg);
517
518 ATF_TP_ADD_TC(tp, rewrite__timed_out);
519
520 ATF_TP_ADD_TC(tp, rewrite__missing_file);
521 ATF_TP_ADD_TC(tp, rewrite__empty);
522 ATF_TP_ADD_TC(tp, rewrite__unknown_status);
523 ATF_TP_ADD_TC(tp, rewrite__status_without_newline);
524 ATF_TP_ADD_TC(tp, rewrite__status_multiline);
525 ATF_TP_ADD_TC(tp, rewrite__status_multiline_2);
526
527 ATF_TP_ADD_TC(tp, cleanup_rewrite__ok);
528 ATF_TP_ADD_TC(tp, cleanup_rewrite__exit_failure);
529 ATF_TP_ADD_TC(tp, cleanup_rewrite__signaled);
530 ATF_TP_ADD_TC(tp, cleanup_rewrite__timed_out);
531
532 return atf_no_error();
533 }
534