1 // Copyright 2012 Google Inc. 2 // All rights reserved. 3 // 4 // Redistribution and use in source and binary forms, with or without 5 // modification, are permitted provided that the following conditions are 6 // met: 7 // 8 // * Redistributions of source code must retain the above copyright 9 // notice, this list of conditions and the following disclaimer. 10 // * Redistributions in binary form must reproduce the above copyright 11 // notice, this list of conditions and the following disclaimer in the 12 // documentation and/or other materials provided with the distribution. 13 // * Neither the name of Google Inc. nor the names of its contributors 14 // may be used to endorse or promote products derived from this software 15 // without specific prior written permission. 16 // 17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 #include "atf_result.h" 30 31 #include <sys/resource.h> 32 #include <sys/wait.h> 33 34 #include <signal.h> 35 #include <stdlib.h> 36 #include <unistd.h> 37 38 #include <atf-c.h> 39 40 #include "error.h" 41 42 43 /// Evalutes an expression and ensures it does not return an error. 44 /// 45 /// \param expr A expression that must evaluate to kyua_error_t. 46 #define RE(expr) ATF_REQUIRE(!kyua_error_is_set(expr)) 47 48 49 /// Generates a wait(2) status for a successful exit code. 50 /// 51 /// \param exitstatus The exit status to encode in the status. 52 /// 53 /// \return The wait(2) status. 54 static int 55 generate_wait_exitstatus(const int exitstatus) 56 { 57 const pid_t pid = fork(); 58 ATF_REQUIRE(pid != -1); 59 if (pid == 0) { 60 exit(exitstatus); 61 } else { 62 int status; 63 ATF_REQUIRE(waitpid(pid, &status, 0) != -1); 64 return status; 65 } 66 } 67 68 69 /// Generates a wait(2) status for a termination due to a signal. 70 /// 71 /// \param signo The signal number to encode in the status. 72 /// 73 /// \return The wait(2) status. 74 static int 75 generate_wait_termsig(const int signo) 76 { 77 // Explicitly disable core files to avoid inconsistent behaviors across 78 // operating systems. Some of the signal numbers passed to this function 79 // may have a meaning or not depending on the system, and this might mean 80 // that a core gets generated arbitrarily. As a result of this, our string 81 // comparisons below fail. 82 struct rlimit rl; 83 rl.rlim_cur = 0; 84 rl.rlim_max = RLIM_INFINITY; 85 if (setrlimit(RLIMIT_CORE, &rl) == -1) 86 atf_tc_skip("Failed to lower the core size limit"); 87 88 const pid_t pid = fork(); 89 ATF_REQUIRE(pid != -1); 90 if (pid == 0) { 91 kill(getpid(), signo); 92 abort(); 93 } else { 94 int status; 95 ATF_REQUIRE(waitpid(pid, &status, 0) != -1); 96 return status; 97 } 98 } 99 100 101 /// Validates an execution of kyua_atf_result_rewrite(). 102 /// 103 /// The code of this check is all within the macro instead of in a separate 104 /// function so that we get meaningful line numbers in test failure messages. 105 /// 106 /// \param name Name of the test case. 107 /// \param input Text of the ATF result. 108 /// \param wait_status Exit status of the ATF test case, as returned by wait(2). 109 /// \param timed_out Whether the test case timed out or not. 110 /// \param exp_output Text of the expected generic result. 111 /// \param exp_success Expected value of the success result. 112 #define CHECK(name, input, wait_status, timed_out, exp_output, exp_success) \ 113 ATF_TC_WITHOUT_HEAD(name); \ 114 ATF_TC_BODY(name, tc) \ 115 { \ 116 bool success; \ 117 atf_utils_create_file("in.txt", "%s", input); \ 118 RE(kyua_atf_result_rewrite("in.txt", "out.txt", wait_status, \ 119 timed_out, &success)); \ 120 atf_utils_cat_file("out.txt", "OUTPUT: "); \ 121 ATF_REQUIRE(atf_utils_compare_file("out.txt", exp_output)); \ 122 ATF_REQUIRE_EQ(exp_success, success); \ 123 } 124 125 126 CHECK(rewrite__expected_death__exit_failure, 127 "expected_death: Some text\n", 128 generate_wait_exitstatus(EXIT_FAILURE), false, 129 "expected_failure: Some text\n", 130 true); 131 CHECK(rewrite__expected_death__signaled, 132 "expected_death: Foo bar\n", 133 generate_wait_termsig(1), false, 134 "expected_failure: Foo bar\n", 135 true); 136 CHECK(rewrite__expected_death__exit_success, 137 "expected_death: Some text\n", 138 generate_wait_exitstatus(EXIT_SUCCESS), false, 139 "failed: Test case expected to die but exited successfully\n", 140 false); 141 CHECK(rewrite__expected_death__no_reason, 142 "expected_death\n", 143 generate_wait_exitstatus(EXIT_FAILURE), false, 144 "broken: Test case should have reported a failure reason but didn't; " 145 "test case exited with code 1\n", 146 false); 147 CHECK(rewrite__expected_death__unexpected_arg, 148 "expected_death(23): Some text\n", 149 generate_wait_exitstatus(EXIT_FAILURE), false, 150 "broken: Unknown test case result status expected_death(23); " 151 "test case exited with code 1\n", 152 false); 153 154 155 CHECK(rewrite__expected_exit__exit_success, 156 "expected_exit: Some text\n", 157 generate_wait_exitstatus(EXIT_SUCCESS), false, 158 "expected_failure: Some text\n", 159 true); 160 CHECK(rewrite__expected_exit__exit_failure, 161 "expected_exit: Some other text\n", 162 generate_wait_exitstatus(EXIT_FAILURE), false, 163 "expected_failure: Some other text\n", 164 true); 165 CHECK(rewrite__expected_exit__arg_match, 166 "expected_exit(15): Some text\n", 167 generate_wait_exitstatus(15), false, 168 "expected_failure: Some text\n", 169 true); 170 CHECK(rewrite__expected_exit__arg_not_match, 171 "expected_exit(18): Some text\n", 172 generate_wait_exitstatus(15), false, 173 "failed: Test case expected to exit with code 18 but got code 15\n", 174 false); 175 CHECK(rewrite__expected_exit__signaled, 176 "expected_exit: Foo bar\n", 177 generate_wait_termsig(1), false, 178 "failed: Test case expected to exit normally but received signal 1\n", 179 false); 180 CHECK(rewrite__expected_exit__no_reason, 181 "expected_exit\n", 182 generate_wait_exitstatus(EXIT_FAILURE), false, 183 "broken: Test case should have reported a failure reason but didn't; " 184 "test case exited with code 1\n", 185 false); 186 CHECK(rewrite__expected_exit__bad_arg, 187 "expected_exit(abc): Some text\n", 188 generate_wait_exitstatus(25), false, 189 "broken: Invalid status argument (abc): not a number; test case exited " 190 "with code 25\n", 191 false); 192 193 194 CHECK(rewrite__expected_failure__ok, 195 "expected_failure: Some text\n", 196 generate_wait_exitstatus(EXIT_SUCCESS), false, 197 "expected_failure: Some text\n", 198 true); 199 CHECK(rewrite__expected_failure__failed, 200 "expected_failure: Some text\n", 201 generate_wait_exitstatus(EXIT_FAILURE), false, 202 "failed: Test case expected a failure but exited with error code 1\n", 203 false); 204 CHECK(rewrite__expected_failure__signaled, 205 "expected_failure: Foo bar\n", 206 generate_wait_termsig(1), false, 207 "failed: Test case expected a failure but received signal 1\n", 208 false); 209 CHECK(rewrite__expected_failure__no_reason, 210 "expected_failure\n", 211 generate_wait_exitstatus(EXIT_FAILURE), false, 212 "broken: Test case should have reported a failure reason but didn't; " 213 "test case exited with code 1\n", 214 false); 215 CHECK(rewrite__expected_failure__unexpected_arg, 216 "expected_failure(23): Some text\n", 217 generate_wait_exitstatus(EXIT_FAILURE), false, 218 "broken: Unknown test case result status expected_failure(23); " 219 "test case exited with code 1\n", 220 false); 221 222 223 CHECK(rewrite__expected_signal__ok, 224 "expected_signal: Some text\n", 225 generate_wait_termsig(6), false, 226 "expected_failure: Some text\n", 227 true); 228 CHECK(rewrite__expected_signal__arg_match, 229 "expected_signal(15): Some text\n", 230 generate_wait_termsig(15), false, 231 "expected_failure: Some text\n", 232 true); 233 CHECK(rewrite__expected_signal__arg_not_match, 234 "expected_signal(18): Some text\n", 235 generate_wait_termsig(15), false, 236 "failed: Test case expected to receive signal 18 but got 15\n", 237 false); 238 CHECK(rewrite__expected_signal__exited, 239 "expected_signal: Foo bar\n", 240 generate_wait_exitstatus(12), false, 241 "failed: Test case expected to receive a signal but exited with code 12\n", 242 false); 243 CHECK(rewrite__expected_signal__no_reason, 244 "expected_signal\n", 245 generate_wait_termsig(15), false, 246 "broken: Test case should have reported a failure reason but didn't; " 247 "test case received signal 15\n", 248 false); 249 CHECK(rewrite__expected_signal__bad_arg, 250 "expected_signal(abc): Some text\n", 251 generate_wait_termsig(25), false, 252 "broken: Invalid status argument (abc): not a number; test case received " 253 "signal 25\n", 254 false); 255 256 257 CHECK(rewrite__expected_timeout__ok, 258 "expected_timeout: Some text\n", 259 generate_wait_exitstatus(EXIT_SUCCESS), true, 260 "expected_failure: Some text\n", 261 true); 262 CHECK(rewrite__expected_timeout__exited, 263 "expected_timeout: Foo bar\n", 264 generate_wait_exitstatus(12), false, 265 "failed: Test case expected to time out but exited with code 12\n", 266 false); 267 CHECK(rewrite__expected_timeout__signaled, 268 "expected_timeout: Foo bar\n", 269 generate_wait_termsig(15), false, 270 "failed: Test case expected to time out but received signal 15\n", 271 false); 272 CHECK(rewrite__expected_timeout__no_reason, 273 "expected_timeout\n", 274 generate_wait_exitstatus(EXIT_FAILURE), false, 275 "broken: Test case should have reported a failure reason but didn't; " 276 "test case exited with code 1\n", 277 false); 278 CHECK(rewrite__expected_timeout__unexpected_arg, 279 "expected_timeout(23): Some text\n", 280 generate_wait_exitstatus(EXIT_FAILURE), false, 281 "broken: Unknown test case result status expected_timeout(23); " 282 "test case exited with code 1\n", 283 false); 284 285 286 CHECK(rewrite__failed__ok, 287 "failed: Some text\n", 288 generate_wait_exitstatus(EXIT_FAILURE), false, 289 "failed: Some text\n", 290 false); 291 CHECK(rewrite__failed__exit_success, 292 "failed: Some text\n", 293 generate_wait_exitstatus(EXIT_SUCCESS), false, 294 "broken: Test case reported a failed result but exited with a successful " 295 "exit code\n", 296 false); 297 CHECK(rewrite__failed__signaled, 298 "failed: Not used\n", 299 generate_wait_termsig(1), false, 300 "broken: Test case reported a failed result but received signal 1\n", 301 false); 302 CHECK(rewrite__failed__no_reason, 303 "failed\n", 304 generate_wait_exitstatus(EXIT_FAILURE), false, 305 "broken: Test case should have reported a failure reason but didn't; " 306 "test case exited with code 1\n", 307 false); 308 CHECK(rewrite__failed__unexpected_arg, 309 "failed(23): Some text\n", 310 generate_wait_exitstatus(EXIT_FAILURE), false, 311 "broken: Unknown test case result status failed(23); " 312 "test case exited with code 1\n", 313 false); 314 315 316 CHECK(rewrite__passed__ok, 317 "passed\n", 318 generate_wait_exitstatus(EXIT_SUCCESS), false, 319 "passed\n", 320 true); 321 CHECK(rewrite__passed__exit_failure, 322 "passed\n", 323 generate_wait_exitstatus(EXIT_FAILURE), false, 324 "broken: Test case reported a passed result but returned a non-zero " 325 "exit code 1\n", 326 false); 327 CHECK(rewrite__passed__signaled, 328 "passed\n", 329 generate_wait_termsig(1), false, 330 "broken: Test case reported a passed result but received signal 1\n", 331 false); 332 CHECK(rewrite__passed__unexpected_reason, 333 "passed: This should not be here\n", 334 generate_wait_exitstatus(EXIT_SUCCESS), false, 335 "broken: Found unexpected reason in passed test result; test case exited " 336 "with code 0\n", 337 false); 338 CHECK(rewrite__passed__unexpected_arg, 339 "passed(0)\n", 340 generate_wait_exitstatus(EXIT_SUCCESS), false, 341 "broken: Unknown test case result status passed(0); " 342 "test case exited with code 0\n", 343 false); 344 345 346 CHECK(rewrite__skipped__ok, 347 "skipped: Does not apply\n", 348 generate_wait_exitstatus(EXIT_SUCCESS), false, 349 "skipped: Does not apply\n", 350 true); 351 CHECK(rewrite__skipped__exit_failure, 352 "skipped: Some reason\n", 353 generate_wait_exitstatus(EXIT_FAILURE), false, 354 "broken: Test case reported a skipped result but returned a non-zero " 355 "exit code 1\n", 356 false); 357 CHECK(rewrite__skipped__signaled, 358 "skipped: Not used\n", 359 generate_wait_termsig(1), false, 360 "broken: Test case reported a skipped result but received signal 1\n", 361 false); 362 CHECK(rewrite__skipped__no_reason, 363 "skipped\n", 364 generate_wait_exitstatus(EXIT_SUCCESS), false, 365 "broken: Test case should have reported a failure reason but didn't; " 366 "test case exited with code 0\n", 367 false); 368 CHECK(rewrite__skipped__unexpected_arg, 369 "skipped(1): Some text\n", 370 generate_wait_exitstatus(EXIT_SUCCESS), false, 371 "broken: Unknown test case result status skipped(1); " 372 "test case exited with code 0\n", 373 false); 374 375 376 CHECK(rewrite__timed_out, 377 "this invalid result file should not be parsed\n", 378 generate_wait_exitstatus(EXIT_SUCCESS), true, 379 "broken: Test case body timed out\n", 380 false); 381 382 383 ATF_TC_WITHOUT_HEAD(rewrite__missing_file); 384 ATF_TC_BODY(rewrite__missing_file, tc) 385 { 386 bool success; 387 RE(kyua_atf_result_rewrite("in.txt", "out.txt", 388 generate_wait_exitstatus(EXIT_SUCCESS), 389 false, &success)); 390 atf_utils_cat_file("out.txt", "OUTPUT: "); 391 ATF_REQUIRE(atf_utils_compare_file("out.txt", 392 "broken: Premature exit; test case exited with code 0\n")); 393 ATF_REQUIRE(!success); 394 } 395 CHECK(rewrite__empty, 396 "", 397 generate_wait_exitstatus(EXIT_FAILURE), false, 398 "broken: Empty result file in.txt; test case exited with code 1\n", 399 false); 400 CHECK(rewrite__unknown_status, 401 "foo\n", 402 generate_wait_exitstatus(123), false, 403 "broken: Unknown test case result status foo; test case exited with " 404 "code 123\n", 405 false); 406 CHECK(rewrite__status_without_newline, 407 "failed", 408 generate_wait_termsig(1), false, 409 "broken: Missing newline in result file; test case received signal 1\n", 410 false); 411 CHECK(rewrite__status_multiline, 412 "failed: First line\nSecond line\nThird line", 413 generate_wait_exitstatus(EXIT_FAILURE), false, 414 "failed: First line<<NEWLINE>>Second line<<NEWLINE>>Third line\n", 415 false); 416 CHECK(rewrite__status_multiline_2, 417 "failed: First line\nSecond line\nThird line\n", 418 generate_wait_exitstatus(EXIT_FAILURE), false, 419 "failed: First line<<NEWLINE>>Second line<<NEWLINE>>Third line\n", 420 false); 421 422 423 /// Validates an execution of kyua_atf_result_cleanup_rewrite(). 424 /// 425 /// The code of this check is all within the macro instead of in a separate 426 /// function so that we get meaningful line numbers in test failure messages. 427 /// 428 /// \param name Name of the test case. 429 /// \param wait_status Exit status of the ATF test case, as returned by wait(2). 430 /// \param timed_out Whether the test case timed out or not. 431 /// \param exp_output Text of the expected generic result. 432 /// \param exp_success Expected value of the success result. 433 #define CHECK_CLEANUP(name, wait_status, timed_out, exp_output, exp_success) \ 434 ATF_TC_WITHOUT_HEAD(name); \ 435 ATF_TC_BODY(name, tc) \ 436 { \ 437 bool success; \ 438 atf_utils_create_file("out.txt", "skipped: Preexistent file\n"); \ 439 RE(kyua_atf_result_cleanup_rewrite("out.txt", wait_status, \ 440 timed_out, &success)); \ 441 atf_utils_cat_file("out.txt", "OUTPUT: "); \ 442 ATF_REQUIRE(atf_utils_compare_file("out.txt", exp_output)); \ 443 ATF_REQUIRE_EQ(exp_success, success); \ 444 } 445 446 447 CHECK_CLEANUP(cleanup_rewrite__ok, 448 generate_wait_exitstatus(EXIT_SUCCESS), false, 449 "skipped: Preexistent file\n", // Previous file not overwritten. 450 true); 451 CHECK_CLEANUP(cleanup_rewrite__exit_failure, 452 generate_wait_exitstatus(EXIT_FAILURE), false, 453 "broken: Test case cleanup exited with code 1\n", 454 false); 455 CHECK_CLEANUP(cleanup_rewrite__signaled, 456 generate_wait_termsig(1), false, 457 "broken: Test case cleanup received signal 1\n", 458 false); 459 CHECK_CLEANUP(cleanup_rewrite__timed_out, 460 generate_wait_exitstatus(EXIT_SUCCESS), true, 461 "broken: Test case cleanup timed out\n", 462 false); 463 464 465 ATF_TP_ADD_TCS(tp) 466 { 467 ATF_TP_ADD_TC(tp, rewrite__expected_death__exit_failure); 468 ATF_TP_ADD_TC(tp, rewrite__expected_death__signaled); 469 ATF_TP_ADD_TC(tp, rewrite__expected_death__exit_success); 470 ATF_TP_ADD_TC(tp, rewrite__expected_death__no_reason); 471 ATF_TP_ADD_TC(tp, rewrite__expected_death__unexpected_arg); 472 473 ATF_TP_ADD_TC(tp, rewrite__expected_exit__exit_success); 474 ATF_TP_ADD_TC(tp, rewrite__expected_exit__exit_failure); 475 ATF_TP_ADD_TC(tp, rewrite__expected_exit__arg_match); 476 ATF_TP_ADD_TC(tp, rewrite__expected_exit__arg_not_match); 477 ATF_TP_ADD_TC(tp, rewrite__expected_exit__signaled); 478 ATF_TP_ADD_TC(tp, rewrite__expected_exit__no_reason); 479 ATF_TP_ADD_TC(tp, rewrite__expected_exit__bad_arg); 480 481 ATF_TP_ADD_TC(tp, rewrite__expected_failure__ok); 482 ATF_TP_ADD_TC(tp, rewrite__expected_failure__failed); 483 ATF_TP_ADD_TC(tp, rewrite__expected_failure__signaled); 484 ATF_TP_ADD_TC(tp, rewrite__expected_failure__no_reason); 485 ATF_TP_ADD_TC(tp, rewrite__expected_failure__unexpected_arg); 486 487 ATF_TP_ADD_TC(tp, rewrite__expected_signal__ok); 488 ATF_TP_ADD_TC(tp, rewrite__expected_signal__arg_match); 489 ATF_TP_ADD_TC(tp, rewrite__expected_signal__arg_not_match); 490 ATF_TP_ADD_TC(tp, rewrite__expected_signal__exited); 491 ATF_TP_ADD_TC(tp, rewrite__expected_signal__no_reason); 492 ATF_TP_ADD_TC(tp, rewrite__expected_signal__bad_arg); 493 494 ATF_TP_ADD_TC(tp, rewrite__expected_timeout__ok); 495 ATF_TP_ADD_TC(tp, rewrite__expected_timeout__exited); 496 ATF_TP_ADD_TC(tp, rewrite__expected_timeout__signaled); 497 ATF_TP_ADD_TC(tp, rewrite__expected_timeout__no_reason); 498 ATF_TP_ADD_TC(tp, rewrite__expected_timeout__unexpected_arg); 499 500 ATF_TP_ADD_TC(tp, rewrite__failed__ok); 501 ATF_TP_ADD_TC(tp, rewrite__failed__exit_success); 502 ATF_TP_ADD_TC(tp, rewrite__failed__signaled); 503 ATF_TP_ADD_TC(tp, rewrite__failed__no_reason); 504 ATF_TP_ADD_TC(tp, rewrite__failed__unexpected_arg); 505 506 ATF_TP_ADD_TC(tp, rewrite__passed__ok); 507 ATF_TP_ADD_TC(tp, rewrite__passed__exit_failure); 508 ATF_TP_ADD_TC(tp, rewrite__passed__signaled); 509 ATF_TP_ADD_TC(tp, rewrite__passed__unexpected_reason); 510 ATF_TP_ADD_TC(tp, rewrite__passed__unexpected_arg); 511 512 ATF_TP_ADD_TC(tp, rewrite__skipped__ok); 513 ATF_TP_ADD_TC(tp, rewrite__skipped__exit_failure); 514 ATF_TP_ADD_TC(tp, rewrite__skipped__signaled); 515 ATF_TP_ADD_TC(tp, rewrite__skipped__no_reason); 516 ATF_TP_ADD_TC(tp, rewrite__skipped__unexpected_arg); 517 518 ATF_TP_ADD_TC(tp, rewrite__timed_out); 519 520 ATF_TP_ADD_TC(tp, rewrite__missing_file); 521 ATF_TP_ADD_TC(tp, rewrite__empty); 522 ATF_TP_ADD_TC(tp, rewrite__unknown_status); 523 ATF_TP_ADD_TC(tp, rewrite__status_without_newline); 524 ATF_TP_ADD_TC(tp, rewrite__status_multiline); 525 ATF_TP_ADD_TC(tp, rewrite__status_multiline_2); 526 527 ATF_TP_ADD_TC(tp, cleanup_rewrite__ok); 528 ATF_TP_ADD_TC(tp, cleanup_rewrite__exit_failure); 529 ATF_TP_ADD_TC(tp, cleanup_rewrite__signaled); 530 ATF_TP_ADD_TC(tp, cleanup_rewrite__timed_out); 531 532 return atf_no_error(); 533 } 534