1 // 2 // Automated Testing Framework (atf) 3 // 4 // Copyright (c) 2007, 2008, 2009 The NetBSD Foundation, Inc. 5 // All rights reserved. 6 // 7 // Redistribution and use in source and binary forms, with or without 8 // modification, are permitted provided that the following conditions 9 // are met: 10 // 1. Redistributions of source code must retain the above copyright 11 // notice, this list of conditions and the following disclaimer. 12 // 2. Redistributions in binary form must reproduce the above copyright 13 // notice, this list of conditions and the following disclaimer in the 14 // documentation and/or other materials provided with the distribution. 15 // 16 // THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND 17 // CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 // INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 19 // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 // IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY 21 // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 23 // GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER 25 // IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 26 // OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 27 // IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 // 29 30 extern "C" { 31 #include <sys/types.h> 32 #include <sys/stat.h> 33 #include <sys/time.h> 34 #include <sys/wait.h> 35 #include <signal.h> 36 #include <unistd.h> 37 } 38 39 #include <algorithm> 40 #include <cctype> 41 #include <cerrno> 42 #include <cstdlib> 43 #include <cstring> 44 #include <fstream> 45 #include <iostream> 46 #include <map> 47 #include <memory> 48 #include <sstream> 49 #include <stdexcept> 50 #include <vector> 51 52 extern "C" { 53 #include "atf-c/error.h" 54 #include "atf-c/object.h" 55 } 56 57 #include "atf-c++/application.hpp" 58 #include "atf-c++/config.hpp" 59 #include "atf-c++/env.hpp" 60 #include "atf-c++/exceptions.hpp" 61 #include "atf-c++/expand.hpp" 62 #include "atf-c++/formats.hpp" 63 #include "atf-c++/fs.hpp" 64 #include "atf-c++/io.hpp" 65 #include "atf-c++/sanity.hpp" 66 #include "atf-c++/signals.hpp" 67 #include "atf-c++/tests.hpp" 68 #include "atf-c++/text.hpp" 69 #include "atf-c++/ui.hpp" 70 #include "atf-c++/user.hpp" 71 72 namespace impl = atf::tests; 73 #define IMPL_NAME "atf::tests" 74 75 // ------------------------------------------------------------------------ 76 // Auxiliary stuff for the timeout implementation. 77 // ------------------------------------------------------------------------ 78 79 namespace timeout { 80 static pid_t current_body = 0; 81 static bool killed = false; 82 83 void 84 sigalrm_handler(int signo) 85 { 86 PRE(signo == SIGALRM); 87 88 if (current_body != 0) { 89 ::killpg(current_body, SIGTERM); 90 killed = true; 91 } 92 } 93 } // namespace timeout 94 95 // ------------------------------------------------------------------------ 96 // The "tcr" class. 97 // ------------------------------------------------------------------------ 98 99 const impl::tcr::state impl::tcr::passed_state = atf_tcr_passed_state; 100 const impl::tcr::state impl::tcr::failed_state = atf_tcr_failed_state; 101 const impl::tcr::state impl::tcr::skipped_state = atf_tcr_skipped_state; 102 103 impl::tcr::tcr(state s) 104 { 105 PRE(s == passed_state); 106 107 atf_error_t err = atf_tcr_init(&m_tcr, s); 108 if (atf_is_error(err)) 109 throw_atf_error(err); 110 } 111 112 impl::tcr::tcr(state s, const std::string& r) 113 { 114 PRE(s == failed_state || s == skipped_state); 115 PRE(!r.empty()); 116 117 atf_error_t err = atf_tcr_init_reason_fmt(&m_tcr, s, "%s", r.c_str()); 118 if (atf_is_error(err)) 119 throw_atf_error(err); 120 } 121 122 impl::tcr::tcr(const tcr& o) 123 { 124 if (o.get_state() == passed_state) 125 atf_tcr_init(&m_tcr, o.get_state()); 126 else 127 atf_tcr_init_reason_fmt(&m_tcr, o.get_state(), "%s", 128 o.get_reason().c_str()); 129 } 130 131 impl::tcr::~tcr(void) 132 { 133 atf_tcr_fini(&m_tcr); 134 } 135 136 impl::tcr::state 137 impl::tcr::get_state(void) 138 const 139 { 140 return atf_tcr_get_state(&m_tcr); 141 } 142 143 const std::string 144 impl::tcr::get_reason(void) 145 const 146 { 147 const atf_dynstr_t* r = atf_tcr_get_reason(&m_tcr); 148 return atf_dynstr_cstring(r); 149 } 150 151 impl::tcr& 152 impl::tcr::operator=(const tcr& o) 153 { 154 if (this != &o) { 155 atf_tcr_fini(&m_tcr); 156 157 if (o.get_state() == passed_state) 158 atf_tcr_init(&m_tcr, o.get_state()); 159 else 160 atf_tcr_init_reason_fmt(&m_tcr, o.get_state(), "%s", 161 o.get_reason().c_str()); 162 } 163 return *this; 164 } 165 166 // ------------------------------------------------------------------------ 167 // The "tc" class. 168 // ------------------------------------------------------------------------ 169 170 static std::map< atf_tc_t*, impl::tc* > wraps; 171 static std::map< const atf_tc_t*, const impl::tc* > cwraps; 172 173 void 174 impl::tc::wrap_head(atf_tc_t *tc) 175 { 176 std::map< atf_tc_t*, impl::tc* >::iterator iter = wraps.find(tc); 177 INV(iter != wraps.end()); 178 (*iter).second->head(); 179 } 180 181 void 182 impl::tc::wrap_body(const atf_tc_t *tc) 183 { 184 std::map< const atf_tc_t*, const impl::tc* >::const_iterator iter = 185 cwraps.find(tc); 186 INV(iter != cwraps.end()); 187 (*iter).second->body(); 188 } 189 190 void 191 impl::tc::wrap_cleanup(const atf_tc_t *tc) 192 { 193 std::map< const atf_tc_t*, const impl::tc* >::const_iterator iter = 194 cwraps.find(tc); 195 INV(iter != cwraps.end()); 196 (*iter).second->cleanup(); 197 } 198 199 impl::tc::tc(const std::string& ident) : 200 m_ident(ident) 201 { 202 } 203 204 impl::tc::~tc(void) 205 { 206 cwraps.erase(&m_tc); 207 wraps.erase(&m_tc); 208 209 atf_tc_fini(&m_tc); 210 atf_map_fini(&m_config); 211 } 212 213 void 214 impl::tc::init(const vars_map& config) 215 { 216 atf_error_t err; 217 218 err = atf_map_init(&m_config); 219 if (atf_is_error(err)) 220 throw_atf_error(err); 221 222 for (vars_map::const_iterator iter = config.begin(); 223 iter != config.end(); iter++) { 224 const char *var = (*iter).first.c_str(); 225 const char *val = (*iter).second.c_str(); 226 227 err = atf_map_insert(&m_config, var, ::strdup(val), true); 228 if (atf_is_error(err)) { 229 atf_map_fini(&m_config); 230 throw_atf_error(err); 231 } 232 } 233 234 wraps[&m_tc] = this; 235 cwraps[&m_tc] = this; 236 237 err = atf_tc_init(&m_tc, m_ident.c_str(), wrap_head, wrap_body, 238 wrap_cleanup, &m_config); 239 if (atf_is_error(err)) { 240 atf_map_fini(&m_config); 241 throw_atf_error(err); 242 } 243 } 244 245 bool 246 impl::tc::has_config_var(const std::string& var) 247 const 248 { 249 return atf_tc_has_config_var(&m_tc, var.c_str()); 250 } 251 252 bool 253 impl::tc::has_md_var(const std::string& var) 254 const 255 { 256 return atf_tc_has_md_var(&m_tc, var.c_str()); 257 } 258 259 const std::string 260 impl::tc::get_config_var(const std::string& var) 261 const 262 { 263 return atf_tc_get_config_var(&m_tc, var.c_str()); 264 } 265 266 const std::string 267 impl::tc::get_config_var(const std::string& var, const std::string& defval) 268 const 269 { 270 return atf_tc_get_config_var_wd(&m_tc, var.c_str(), defval.c_str()); 271 } 272 273 const std::string 274 impl::tc::get_md_var(const std::string& var) 275 const 276 { 277 return atf_tc_get_md_var(&m_tc, var.c_str()); 278 } 279 280 void 281 impl::tc::set_md_var(const std::string& var, const std::string& val) 282 { 283 atf_error_t err = atf_tc_set_md_var(&m_tc, var.c_str(), val.c_str()); 284 if (atf_is_error(err)) 285 throw_atf_error(err); 286 } 287 288 impl::tcr 289 impl::tc::run(int fdout, int fderr, const fs::path& workdirbase) 290 const 291 { 292 atf_tcr_t tcrc; 293 tcr tcrr(tcr::failed_state, "UNINITIALIZED"); 294 295 atf_error_t err = atf_tc_run(&m_tc, &tcrc, fdout, fderr, 296 workdirbase.c_path()); 297 if (atf_is_error(err)) 298 throw_atf_error(err); 299 300 if (atf_tcr_has_reason(&tcrc)) { 301 const atf_dynstr_t* r = atf_tcr_get_reason(&tcrc); 302 tcrr = tcr(atf_tcr_get_state(&tcrc), atf_dynstr_cstring(r)); 303 } else { 304 tcrr = tcr(atf_tcr_get_state(&tcrc)); 305 } 306 307 atf_tcr_fini(&tcrc); 308 return tcrr; 309 } 310 311 void 312 impl::tc::cleanup(void) 313 const 314 { 315 } 316 317 void 318 impl::tc::require_prog(const std::string& prog) 319 const 320 { 321 atf_tc_require_prog(prog.c_str()); 322 } 323 324 void 325 impl::tc::pass(void) 326 { 327 atf_tc_pass(); 328 } 329 330 void 331 impl::tc::fail(const std::string& reason) 332 { 333 atf_tc_fail("%s", reason.c_str()); 334 } 335 336 void 337 impl::tc::skip(const std::string& reason) 338 { 339 atf_tc_skip("%s", reason.c_str()); 340 } 341 342 // ------------------------------------------------------------------------ 343 // The "tp" class. 344 // ------------------------------------------------------------------------ 345 346 class tp : public atf::application::app { 347 public: 348 typedef std::vector< impl::tc * > tc_vector; 349 350 private: 351 static const char* m_description; 352 353 bool m_lflag; 354 int m_results_fd; 355 std::auto_ptr< std::ostream > m_results_os; 356 atf::fs::path m_srcdir; 357 atf::fs::path m_workdir; 358 std::vector< std::string > m_tcnames; 359 360 atf::tests::vars_map m_vars; 361 362 std::string specific_args(void) const; 363 options_set specific_options(void) const; 364 void process_option(int, const char*); 365 366 void (*m_add_tcs)(tc_vector&); 367 tc_vector m_tcs; 368 369 void parse_vflag(const std::string&); 370 void handle_srcdir(void); 371 372 tc_vector init_tcs(void); 373 static tc_vector filter_tcs(tc_vector, 374 const std::vector< std::string >&); 375 376 std::ostream& results_stream(void); 377 378 int list_tcs(void); 379 int run_tcs(void); 380 381 public: 382 tp(void (*)(tc_vector&)); 383 ~tp(void); 384 385 int main(void); 386 }; 387 388 const char* tp::m_description = 389 "This is an independent atf test program."; 390 391 tp::tp(void (*add_tcs)(tc_vector&)) : 392 app(m_description, "atf-test-program(1)", "atf(7)"), 393 m_lflag(false), 394 m_results_fd(STDOUT_FILENO), 395 m_srcdir("."), 396 m_workdir(atf::config::get("atf_workdir")), 397 m_add_tcs(add_tcs) 398 { 399 } 400 401 tp::~tp(void) 402 { 403 for (tc_vector::iterator iter = m_tcs.begin(); 404 iter != m_tcs.end(); iter++) { 405 impl::tc* tc = *iter; 406 407 delete tc; 408 } 409 } 410 411 std::string 412 tp::specific_args(void) 413 const 414 { 415 return "[test_case1 [.. test_caseN]]"; 416 } 417 418 tp::options_set 419 tp::specific_options(void) 420 const 421 { 422 using atf::application::option; 423 options_set opts; 424 opts.insert(option('l', "", "List test cases and their purpose")); 425 opts.insert(option('r', "fd", "The file descriptor to which the test " 426 "program will send the results of the " 427 "test cases")); 428 opts.insert(option('s', "srcdir", "Directory where the test's data " 429 "files are located")); 430 opts.insert(option('v', "var=value", "Sets the configuration variable " 431 "`var' to `value'")); 432 opts.insert(option('w', "workdir", "Directory where the test's " 433 "temporary files are located")); 434 return opts; 435 } 436 437 void 438 tp::process_option(int ch, const char* arg) 439 { 440 switch (ch) { 441 case 'l': 442 m_lflag = true; 443 break; 444 445 case 'r': 446 { 447 std::istringstream ss(arg); 448 ss >> m_results_fd; 449 } 450 break; 451 452 case 's': 453 m_srcdir = atf::fs::path(arg); 454 break; 455 456 case 'v': 457 parse_vflag(arg); 458 break; 459 460 case 'w': 461 m_workdir = atf::fs::path(arg); 462 break; 463 464 default: 465 UNREACHABLE; 466 } 467 } 468 469 void 470 tp::parse_vflag(const std::string& str) 471 { 472 if (str.empty()) 473 throw std::runtime_error("-v requires a non-empty argument"); 474 475 std::vector< std::string > ws = atf::text::split(str, "="); 476 if (ws.size() == 1 && str[str.length() - 1] == '=') { 477 m_vars[ws[0]] = ""; 478 } else { 479 if (ws.size() != 2) 480 throw std::runtime_error("-v requires an argument of the form " 481 "var=value"); 482 483 m_vars[ws[0]] = ws[1]; 484 } 485 } 486 487 void 488 tp::handle_srcdir(void) 489 { 490 if (!atf::fs::exists(m_srcdir / m_prog_name)) 491 throw std::runtime_error("Cannot find the test program in the " 492 "source directory `" + m_srcdir.str() + "'"); 493 494 if (!m_srcdir.is_absolute()) 495 m_srcdir = m_srcdir.to_absolute(); 496 497 m_vars["srcdir"] = m_srcdir.str(); 498 } 499 500 tp::tc_vector 501 tp::init_tcs(void) 502 { 503 m_add_tcs(m_tcs); 504 for (tc_vector::iterator iter = m_tcs.begin(); 505 iter != m_tcs.end(); iter++) { 506 impl::tc* tc = *iter; 507 508 tc->init(m_vars); 509 } 510 return m_tcs; 511 } 512 513 // 514 // An auxiliary unary predicate that compares the given test case's 515 // identifier to the identifier stored in it. 516 // 517 class tc_equal_to_ident { 518 const std::string& m_ident; 519 520 public: 521 tc_equal_to_ident(const std::string& i) : 522 m_ident(i) 523 { 524 } 525 526 bool operator()(const impl::tc* tc) 527 { 528 return tc->get_md_var("ident") == m_ident; 529 } 530 }; 531 532 tp::tc_vector 533 tp::filter_tcs(tc_vector tcs, const std::vector< std::string >& tcnames) 534 { 535 tc_vector tcso; 536 537 if (tcnames.empty()) { 538 // Special case: added for efficiency because this is the most 539 // typical situation. 540 tcso = tcs; 541 } else { 542 // Collect all the test cases' identifiers. 543 std::vector< std::string > ids; 544 for (tc_vector::iterator iter = tcs.begin(); 545 iter != tcs.end(); iter++) { 546 impl::tc* tc = *iter; 547 548 ids.push_back(tc->get_md_var("ident")); 549 } 550 551 // Iterate over all names provided by the user and, for each one, 552 // expand it as if it were a glob pattern. Collect all expansions. 553 std::vector< std::string > exps; 554 for (std::vector< std::string >::const_iterator iter = tcnames.begin(); 555 iter != tcnames.end(); iter++) { 556 const std::string& glob = *iter; 557 558 std::vector< std::string > ms = 559 atf::expand::expand_glob(glob, ids); 560 if (ms.empty()) 561 throw std::runtime_error("Unknown test case `" + glob + "'"); 562 exps.insert(exps.end(), ms.begin(), ms.end()); 563 } 564 565 // For each expansion, locate its corresponding test case and add 566 // it to the output set. 567 for (std::vector< std::string >::const_iterator iter = exps.begin(); 568 iter != exps.end(); iter++) { 569 const std::string& name = *iter; 570 571 tc_vector::iterator tciter = 572 std::find_if(tcs.begin(), tcs.end(), tc_equal_to_ident(name)); 573 INV(tciter != tcs.end()); 574 tcso.push_back(*tciter); 575 } 576 } 577 578 return tcso; 579 } 580 581 int 582 tp::list_tcs(void) 583 { 584 tc_vector tcs = filter_tcs(init_tcs(), m_tcnames); 585 586 std::string::size_type maxlen = 0; 587 for (tc_vector::const_iterator iter = tcs.begin(); 588 iter != tcs.end(); iter++) { 589 const impl::tc* tc = *iter; 590 591 if (maxlen < tc->get_md_var("ident").length()) 592 maxlen = tc->get_md_var("ident").length(); 593 } 594 595 for (tc_vector::const_iterator iter = tcs.begin(); 596 iter != tcs.end(); iter++) { 597 const impl::tc* tc = *iter; 598 599 std::cout << atf::ui::format_text_with_tag(tc->get_md_var("descr"), 600 tc->get_md_var("ident"), 601 false, maxlen + 4) 602 << std::endl; 603 } 604 605 return EXIT_SUCCESS; 606 } 607 608 std::ostream& 609 tp::results_stream(void) 610 { 611 if (m_results_fd == STDOUT_FILENO) 612 return std::cout; 613 else if (m_results_fd == STDERR_FILENO) 614 return std::cerr; 615 else 616 return *m_results_os; 617 } 618 619 int 620 tp::run_tcs(void) 621 { 622 tc_vector tcs = filter_tcs(init_tcs(), m_tcnames); 623 624 if (!atf::fs::exists(m_workdir)) 625 throw std::runtime_error("Cannot find the work directory `" + 626 m_workdir.str() + "'"); 627 628 int errcode = EXIT_SUCCESS; 629 630 atf::signals::signal_holder sighup(SIGHUP); 631 atf::signals::signal_holder sigint(SIGINT); 632 atf::signals::signal_holder sigterm(SIGTERM); 633 634 atf::formats::atf_tcs_writer w(results_stream(), std::cout, std::cerr, 635 tcs.size()); 636 for (tc_vector::iterator iter = tcs.begin(); 637 iter != tcs.end(); iter++) { 638 impl::tc* tc = *iter; 639 640 w.start_tc(tc->get_md_var("ident")); 641 impl::tcr tcr = tc->run(STDOUT_FILENO, STDERR_FILENO, m_workdir); 642 w.end_tc(tcr); 643 644 sighup.process(); 645 sigint.process(); 646 sigterm.process(); 647 648 if (tcr.get_state() == impl::tcr::failed_state) 649 errcode = EXIT_FAILURE; 650 } 651 652 return errcode; 653 } 654 655 int 656 tp::main(void) 657 { 658 int errcode; 659 660 handle_srcdir(); 661 662 for (int i = 0; i < m_argc; i++) 663 m_tcnames.push_back(m_argv[i]); 664 665 if (m_lflag) 666 errcode = list_tcs(); 667 else { 668 if (m_results_fd != STDOUT_FILENO && m_results_fd != STDERR_FILENO) { 669 atf::io::file_handle fh(m_results_fd); 670 m_results_os = 671 std::auto_ptr< std::ostream >(new atf::io::postream(fh)); 672 } 673 errcode = run_tcs(); 674 } 675 676 return errcode; 677 } 678 679 namespace atf { 680 namespace tests { 681 int run_tp(int, char* const*, void (*)(tp::tc_vector&)); 682 } 683 } 684 685 int 686 impl::run_tp(int argc, char* const* argv, void (*add_tcs)(tp::tc_vector&)) 687 { 688 return tp(add_tcs).run(argc, argv); 689 } 690