1 // 2 // Automated Testing Framework (atf) 3 // 4 // Copyright (c) 2007 The NetBSD Foundation, Inc. 5 // All rights reserved. 6 // 7 // Redistribution and use in source and binary forms, with or without 8 // modification, are permitted provided that the following conditions 9 // are met: 10 // 1. Redistributions of source code must retain the above copyright 11 // notice, this list of conditions and the following disclaimer. 12 // 2. Redistributions in binary form must reproduce the above copyright 13 // notice, this list of conditions and the following disclaimer in the 14 // documentation and/or other materials provided with the distribution. 15 // 16 // THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND 17 // CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 // INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 19 // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 // IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY 21 // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 23 // GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER 25 // IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 26 // OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 27 // IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 // 29 30 extern "C" { 31 #include <sys/types.h> 32 #include <sys/param.h> 33 #include <sys/stat.h> 34 #include <sys/wait.h> 35 #include <unistd.h> 36 } 37 38 #include <algorithm> 39 #include <cassert> 40 #include <cerrno> 41 #include <cstdlib> 42 #include <cstring> 43 #include <fstream> 44 #include <iostream> 45 #include <map> 46 #include <string> 47 48 #include "application.hpp" 49 #include "atffile.hpp" 50 #include "config.hpp" 51 #include "config_file.hpp" 52 #include "env.hpp" 53 #include "exceptions.hpp" 54 #include "fs.hpp" 55 #include "parser.hpp" 56 #include "process.hpp" 57 #include "requirements.hpp" 58 #include "test-program.hpp" 59 #include "text.hpp" 60 61 namespace { 62 63 typedef std::map< std::string, std::string > vars_map; 64 65 } // anonymous namespace 66 67 class atf_run : public tools::application::app { 68 static const char* m_description; 69 70 vars_map m_cmdline_vars; 71 72 static vars_map::value_type parse_var(const std::string&); 73 74 void process_option(int, const char*); 75 std::string specific_args(void) const; 76 options_set specific_options(void) const; 77 78 void parse_vflag(const std::string&); 79 80 std::vector< std::string > conf_args(void) const; 81 82 size_t count_tps(std::vector< std::string >) const; 83 84 int run_test(const tools::fs::path&, const std::string &, 85 tools::test_program::atf_tps_writer&, 86 const vars_map&); 87 int run_test_directory(const tools::fs::path&, 88 tools::test_program::atf_tps_writer&); 89 int run_test_program(const tools::fs::path&, 90 const std::string tc, 91 tools::test_program::atf_tps_writer&, 92 const vars_map&); 93 94 tools::test_program::test_case_result get_test_case_result( 95 const std::string&, const tools::process::status&, 96 const tools::fs::path&) const; 97 98 public: 99 atf_run(void); 100 101 int main(void); 102 }; 103 104 static void 105 sanitize_gdb_env(void) 106 { 107 try { 108 tools::env::unset("TERM"); 109 } catch (...) { 110 // Just swallow exceptions here; they cannot propagate into C, which 111 // is where this function is called from, and even if these exceptions 112 // appear they are benign. 113 } 114 } 115 116 static void 117 dump_stacktrace(const tools::fs::path& tp, const tools::process::status& s, 118 const tools::fs::path& workdir, 119 tools::test_program::atf_tps_writer& w) 120 { 121 assert(s.signaled() && s.coredump()); 122 123 w.stderr_tc("Test program crashed; attempting to get stack trace"); 124 125 const tools::fs::path corename = workdir / 126 (tp.leaf_name().substr(0, MAXCOMLEN) + ".core"); 127 if (!tools::fs::exists(corename)) { 128 w.stderr_tc("Expected file " + corename.str() + " not found"); 129 return; 130 } 131 132 const tools::fs::path gdb(GDB); 133 const tools::fs::path gdbout = workdir / "gdb.out"; 134 const tools::process::argv_array args(gdb.leaf_name().c_str(), "-batch", 135 "-q", "-ex", "bt", tp.c_str(), 136 corename.c_str(), NULL); 137 tools::process::status status = tools::process::exec( 138 gdb, args, 139 tools::process::stream_redirect_path(gdbout), 140 tools::process::stream_redirect_path(tools::fs::path("/dev/null")), 141 sanitize_gdb_env); 142 if (!status.exited() || status.exitstatus() != EXIT_SUCCESS) { 143 w.stderr_tc("Execution of " GDB " failed"); 144 return; 145 } 146 147 std::ifstream input(gdbout.c_str()); 148 if (input) { 149 std::string line; 150 while (std::getline(input, line).good()) 151 w.stderr_tc(line); 152 input.close(); 153 } 154 155 w.stderr_tc("Stack trace complete"); 156 } 157 158 const char* atf_run::m_description = 159 "atf-run is a tool that runs tests programs and collects their " 160 "results."; 161 162 atf_run::atf_run(void) : 163 app(m_description, "atf-run(1)", "atf(7)") 164 { 165 } 166 167 void 168 atf_run::process_option(int ch, const char* arg) 169 { 170 switch (ch) { 171 case 'v': 172 parse_vflag(arg); 173 break; 174 175 default: 176 std::abort(); 177 } 178 } 179 180 std::string 181 atf_run::specific_args(void) 182 const 183 { 184 return "[test1 .. testN]"; 185 } 186 187 atf_run::options_set 188 atf_run::specific_options(void) 189 const 190 { 191 using tools::application::option; 192 options_set opts; 193 opts.insert(option('v', "var=value", "Sets the configuration variable " 194 "`var' to `value'; overrides " 195 "values in configuration files")); 196 return opts; 197 } 198 199 void 200 atf_run::parse_vflag(const std::string& str) 201 { 202 if (str.empty()) 203 throw std::runtime_error("-v requires a non-empty argument"); 204 205 std::vector< std::string > ws = tools::text::split(str, "="); 206 if (ws.size() == 1 && str[str.length() - 1] == '=') { 207 m_cmdline_vars[ws[0]] = ""; 208 } else { 209 if (ws.size() != 2) 210 throw std::runtime_error("-v requires an argument of the form " 211 "var=value"); 212 213 m_cmdline_vars[ws[0]] = ws[1]; 214 } 215 } 216 217 int 218 atf_run::run_test(const tools::fs::path& tp, 219 const std::string &tc, 220 tools::test_program::atf_tps_writer& w, 221 const vars_map& config) 222 { 223 tools::fs::file_info fi(tp); 224 225 int errcode; 226 if (fi.get_type() == tools::fs::file_info::dir_type) 227 errcode = run_test_directory(tp, w); 228 else { 229 const vars_map effective_config = 230 tools::config_file::merge_configs(config, m_cmdline_vars); 231 232 errcode = run_test_program(tp, tc, w, effective_config); 233 } 234 return errcode; 235 } 236 237 int 238 atf_run::run_test_directory(const tools::fs::path& tp, 239 tools::test_program::atf_tps_writer& w) 240 { 241 tools::atffile af = tools::read_atffile(tp / "Atffile"); 242 243 vars_map test_suite_vars; 244 { 245 vars_map::const_iterator iter = af.props().find("test-suite"); 246 assert(iter != af.props().end()); 247 test_suite_vars = tools::config_file::read_config_files((*iter).second); 248 } 249 250 bool ok = true; 251 for (std::vector< std::string >::const_iterator iter = af.tps().begin(); 252 iter != af.tps().end(); iter++) { 253 const bool result = run_test(tp / *iter, "", w, 254 tools::config_file::merge_configs(af.conf(), test_suite_vars)); 255 ok &= (result == EXIT_SUCCESS); 256 } 257 258 return ok ? EXIT_SUCCESS : EXIT_FAILURE; 259 } 260 261 tools::test_program::test_case_result 262 atf_run::get_test_case_result(const std::string& broken_reason, 263 const tools::process::status& s, 264 const tools::fs::path& resfile) 265 const 266 { 267 using tools::text::to_string; 268 using tools::test_program::read_test_case_result; 269 using tools::test_program::test_case_result; 270 271 if (!broken_reason.empty()) { 272 test_case_result tcr; 273 274 try { 275 tcr = read_test_case_result(resfile); 276 277 if (tcr.state() == "expected_timeout") { 278 return tcr; 279 } else { 280 return test_case_result("failed", -1, broken_reason); 281 } 282 } catch (const std::runtime_error&) { 283 return test_case_result("failed", -1, broken_reason); 284 } 285 } 286 287 if (s.exited()) { 288 test_case_result tcr; 289 290 try { 291 tcr = read_test_case_result(resfile); 292 } catch (const std::runtime_error& e) { 293 return test_case_result("failed", -1, "Test case exited " 294 "normally but failed to create the results file: " + 295 std::string(e.what())); 296 } 297 298 if (tcr.state() == "expected_death") { 299 return tcr; 300 } else if (tcr.state() == "expected_exit") { 301 if (tcr.value() == -1 || s.exitstatus() == tcr.value()) 302 return tcr; 303 else 304 return test_case_result("failed", -1, "Test case was " 305 "expected to exit with a " + to_string(tcr.value()) + 306 " error code but returned " + to_string(s.exitstatus())); 307 } else if (tcr.state() == "expected_failure") { 308 if (s.exitstatus() == EXIT_SUCCESS) 309 return tcr; 310 else 311 return test_case_result("failed", -1, "Test case returned an " 312 "error in expected_failure mode but it should not have"); 313 } else if (tcr.state() == "expected_signal") { 314 return test_case_result("failed", -1, "Test case exited cleanly " 315 "but was expected to receive a signal"); 316 } else if (tcr.state() == "failed") { 317 if (s.exitstatus() == EXIT_SUCCESS) 318 return test_case_result("failed", -1, "Test case " 319 "exited successfully but reported failure"); 320 else 321 return tcr; 322 } else if (tcr.state() == "passed") { 323 if (s.exitstatus() == EXIT_SUCCESS) 324 return tcr; 325 else 326 return test_case_result("failed", -1, "Test case exited as " 327 "passed but reported an error"); 328 } else if (tcr.state() == "skipped") { 329 if (s.exitstatus() == EXIT_SUCCESS) 330 return tcr; 331 else 332 return test_case_result("failed", -1, "Test case exited as " 333 "skipped but reported an error"); 334 } 335 } else if (s.signaled()) { 336 test_case_result tcr; 337 338 try { 339 tcr = read_test_case_result(resfile); 340 } catch (const std::runtime_error&) { 341 return test_case_result("failed", -1, "Test program received " 342 "signal " + tools::text::to_string(s.termsig()) + 343 (s.coredump() ? " (core dumped)" : "")); 344 } 345 346 if (tcr.state() == "expected_death") { 347 return tcr; 348 } else if (tcr.state() == "expected_signal") { 349 if (tcr.value() == -1 || s.termsig() == tcr.value()) 350 return tcr; 351 else 352 return test_case_result("failed", -1, "Test case was " 353 "expected to exit due to a " + to_string(tcr.value()) + 354 " signal but got " + to_string(s.termsig())); 355 } else { 356 return test_case_result("failed", -1, "Test program received " 357 "signal " + tools::text::to_string(s.termsig()) + 358 (s.coredump() ? " (core dumped)" : "") + " and created a " 359 "bogus results file"); 360 } 361 } 362 std::abort(); 363 return test_case_result(); 364 } 365 366 int 367 atf_run::run_test_program(const tools::fs::path& tp, 368 const std::string tc, 369 tools::test_program::atf_tps_writer& w, 370 const vars_map& config) 371 { 372 int errcode = EXIT_SUCCESS; 373 374 tools::test_program::metadata md; 375 try { 376 md = tools::test_program::get_metadata(tp, config); 377 } catch (const tools::parser::format_error& e) { 378 w.start_tp(tp.str(), 0); 379 w.end_tp("Invalid format for test case list: " + std::string(e.what())); 380 return EXIT_FAILURE; 381 } catch (const tools::parser::parse_errors& e) { 382 const std::string reason = tools::text::join(e, "; "); 383 w.start_tp(tp.str(), 0); 384 w.end_tp("Invalid format for test case list: " + reason); 385 return EXIT_FAILURE; 386 } 387 388 tools::fs::temp_dir resdir( 389 tools::fs::path(tools::config::get("atf_workdir")) / "atf-run.XXXXXX"); 390 391 w.start_tp(tp.str(), md.test_cases.size()); 392 if (md.test_cases.empty()) { 393 w.end_tp("Bogus test program: reported 0 test cases"); 394 errcode = EXIT_FAILURE; 395 } else { 396 for (std::map< std::string, vars_map >::const_iterator iter 397 = md.test_cases.begin(); iter != md.test_cases.end(); iter++) { 398 const std::string& tcname = (*iter).first; 399 const vars_map& tcmd = (*iter).second; 400 401 if (! tc.empty() && tcname != tc) 402 continue; 403 404 w.start_tc(tcname); 405 406 try { 407 const std::string& reqfail = tools::check_requirements( 408 tcmd, config); 409 if (!reqfail.empty()) { 410 w.end_tc("skipped", reqfail); 411 continue; 412 } 413 } catch (const std::runtime_error& e) { 414 w.end_tc("failed", e.what()); 415 errcode = EXIT_FAILURE; 416 continue; 417 } 418 419 const std::pair< int, int > user = tools::get_required_user( 420 tcmd, config); 421 422 tools::fs::path resfile = resdir.get_path() / "tcr"; 423 assert(!tools::fs::exists(resfile)); 424 try { 425 const bool has_cleanup = tools::text::to_bool( 426 (*tcmd.find("has.cleanup")).second); 427 428 tools::fs::temp_dir workdir(tools::fs::path(tools::config::get( 429 "atf_workdir")) / "atf-run.XXXXXX"); 430 if (user.first != -1 && user.second != -1) { 431 if (::chown(workdir.get_path().c_str(), user.first, 432 user.second) == -1) { 433 throw tools::system_error("chown(" + 434 workdir.get_path().str() + ")", "chown(2) failed", 435 errno); 436 } 437 resfile = workdir.get_path() / "tcr"; 438 } 439 440 std::pair< std::string, const tools::process::status > s = 441 tools::test_program::run_test_case( 442 tp, tcname, "body", tcmd, config, 443 resfile, workdir.get_path(), w); 444 if (s.second.signaled() && s.second.coredump()) 445 dump_stacktrace(tp, s.second, workdir.get_path(), w); 446 if (has_cleanup) 447 (void)tools::test_program::run_test_case( 448 tp, tcname, "cleanup", tcmd, 449 config, resfile, workdir.get_path(), w); 450 451 // TODO: Force deletion of workdir. 452 453 tools::test_program::test_case_result tcr = 454 get_test_case_result(s.first, s.second, resfile); 455 456 w.end_tc(tcr.state(), tcr.reason()); 457 if (tcr.state() == "failed") 458 errcode = EXIT_FAILURE; 459 } catch (...) { 460 if (tools::fs::exists(resfile)) 461 tools::fs::remove(resfile); 462 throw; 463 } 464 if (tools::fs::exists(resfile)) 465 tools::fs::remove(resfile); 466 467 } 468 w.end_tp(""); 469 } 470 471 return errcode; 472 } 473 474 static void 475 colon_split(const std::string &s, std::string &tp, std::string &tc) 476 { 477 size_t colon_pos = s.rfind(':'); 478 if (colon_pos != std::string::npos && colon_pos < s.size() - 1) { 479 tp = s.substr(0, colon_pos); 480 tc = s.substr(colon_pos + 1); 481 } else { 482 tp = s; 483 tc = ""; 484 } 485 } 486 487 size_t 488 atf_run::count_tps(std::vector< std::string > tps) 489 const 490 { 491 size_t ntps = 0; 492 493 for (std::vector< std::string >::const_iterator iter = tps.begin(); 494 iter != tps.end(); iter++) { 495 std::string tpname, tcname; 496 colon_split(*iter, tpname, tcname); 497 tools::fs::path tp(tpname); 498 tools::fs::file_info fi(tp); 499 500 if (fi.get_type() == tools::fs::file_info::dir_type) { 501 tools::atffile af = tools::read_atffile(tp / "Atffile"); 502 std::vector< std::string > aux = af.tps(); 503 for (std::vector< std::string >::iterator i2 = aux.begin(); 504 i2 != aux.end(); i2++) 505 *i2 = (tp / *i2).str(); 506 ntps += count_tps(aux); 507 } else 508 ntps++; 509 } 510 511 return ntps; 512 } 513 514 static 515 void 516 call_hook(const std::string& tool, const std::string& hook) 517 { 518 const tools::fs::path sh(tools::config::get("atf_shell")); 519 const tools::fs::path hooks = 520 tools::fs::path(tools::config::get("atf_pkgdatadir")) / (tool + ".hooks"); 521 522 const tools::process::status s = 523 tools::process::exec(sh, 524 tools::process::argv_array(sh.c_str(), hooks.c_str(), 525 hook.c_str(), NULL), 526 tools::process::stream_inherit(), 527 tools::process::stream_inherit()); 528 529 530 if (!s.exited() || s.exitstatus() != EXIT_SUCCESS) 531 throw std::runtime_error("Failed to run the '" + hook + "' hook " 532 "for '" + tool + "'"); 533 } 534 535 int 536 atf_run::main(void) 537 { 538 tools::atffile af = tools::read_atffile(tools::fs::path("Atffile")); 539 540 std::vector< std::string > tps; 541 tps = af.tps(); 542 if (m_argc >= 1) { 543 // TODO: Ensure that the given test names are listed in the 544 // Atffile. Take into account that the file can be using globs. 545 tps.clear(); 546 for (int i = 0; i < m_argc; i++) 547 tps.push_back(m_argv[i]); 548 } 549 550 // Read configuration data for this test suite. 551 vars_map test_suite_vars; 552 { 553 vars_map::const_iterator iter = af.props().find("test-suite"); 554 assert(iter != af.props().end()); 555 test_suite_vars = tools::config_file::read_config_files((*iter).second); 556 } 557 558 tools::test_program::atf_tps_writer w(std::cout); 559 call_hook("atf-run", "info_start_hook"); 560 w.ntps(count_tps(tps)); 561 562 bool ok = true; 563 for (std::vector< std::string >::const_iterator iter = tps.begin(); 564 iter != tps.end(); iter++) { 565 std::string tp, tc; 566 colon_split(*iter, tp, tc); 567 const bool result = run_test(tools::fs::path(tp), tc, w, 568 tools::config_file::merge_configs(af.conf(), test_suite_vars)); 569 ok &= (result == EXIT_SUCCESS); 570 } 571 572 call_hook("atf-run", "info_end_hook"); 573 574 return ok ? EXIT_SUCCESS : EXIT_FAILURE; 575 } 576 577 int 578 main(int argc, char* const* argv) 579 { 580 return atf_run().run(argc, argv); 581 } 582