1 // Copyright 2011 Google Inc. 2 // All rights reserved. 3 // 4 // Redistribution and use in source and binary forms, with or without 5 // modification, are permitted provided that the following conditions are 6 // met: 7 // 8 // * Redistributions of source code must retain the above copyright 9 // notice, this list of conditions and the following disclaimer. 10 // * Redistributions in binary form must reproduce the above copyright 11 // notice, this list of conditions and the following disclaimer in the 12 // documentation and/or other materials provided with the distribution. 13 // * Neither the name of Google Inc. nor the names of its contributors 14 // may be used to endorse or promote products derived from this software 15 // without specific prior written permission. 16 // 17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 #include "cli/common.hpp" 30 31 #include <fstream> 32 33 #include <atf-c++.hpp> 34 35 #include "engine/exceptions.hpp" 36 #include "engine/filters.hpp" 37 #include "engine/test_case.hpp" 38 #include "engine/test_program.hpp" 39 #include "engine/test_result.hpp" 40 #include "utils/cmdline/exceptions.hpp" 41 #include "utils/cmdline/globals.hpp" 42 #include "utils/cmdline/parser.ipp" 43 #include "utils/cmdline/ui_mock.hpp" 44 #include "utils/datetime.hpp" 45 #include "utils/env.hpp" 46 #include "utils/fs/exceptions.hpp" 47 #include "utils/fs/operations.hpp" 48 #include "utils/fs/path.hpp" 49 #include "utils/optional.ipp" 50 #include "utils/sanity.hpp" 51 52 namespace cmdline = utils::cmdline; 53 namespace config = utils::config; 54 namespace datetime = utils::datetime; 55 namespace fs = utils::fs; 56 57 using utils::optional; 58 59 60 namespace { 61 62 63 /// Syntactic sugar to instantiate engine::test_filter objects. 64 inline engine::test_filter 65 mkfilter(const char* test_program, const char* test_case) 66 { 67 return engine::test_filter(fs::path(test_program), test_case); 68 } 69 70 71 } // anonymous namespace 72 73 74 ATF_TEST_CASE_WITHOUT_HEAD(build_root_path__default); 75 ATF_TEST_CASE_BODY(build_root_path__default) 76 { 77 std::map< std::string, std::vector< std::string > > options; 78 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 79 80 ATF_REQUIRE(!cli::build_root_path(mock_cmdline)); 81 } 82 83 84 ATF_TEST_CASE_WITHOUT_HEAD(build_root_path__explicit); 85 ATF_TEST_CASE_BODY(build_root_path__explicit) 86 { 87 std::map< std::string, std::vector< std::string > > options; 88 options["build-root"].push_back("/my//path"); 89 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 90 91 ATF_REQUIRE(cli::build_root_path(mock_cmdline)); 92 ATF_REQUIRE_EQ("/my/path", cli::build_root_path(mock_cmdline).get().str()); 93 } 94 95 96 ATF_TEST_CASE_WITHOUT_HEAD(get_home__ok); 97 ATF_TEST_CASE_BODY(get_home__ok) 98 { 99 const fs::path home("/foo/bar"); 100 utils::setenv("HOME", home.str()); 101 const optional< fs::path > computed = cli::get_home(); 102 ATF_REQUIRE(computed); 103 ATF_REQUIRE_EQ(home, computed.get()); 104 } 105 106 107 ATF_TEST_CASE_WITHOUT_HEAD(get_home__missing); 108 ATF_TEST_CASE_BODY(get_home__missing) 109 { 110 utils::unsetenv("HOME"); 111 ATF_REQUIRE(!cli::get_home()); 112 } 113 114 115 ATF_TEST_CASE_WITHOUT_HEAD(get_home__invalid); 116 ATF_TEST_CASE_BODY(get_home__invalid) 117 { 118 utils::setenv("HOME", ""); 119 ATF_REQUIRE(!cli::get_home()); 120 } 121 122 123 ATF_TEST_CASE_WITHOUT_HEAD(kyuafile_path__default); 124 ATF_TEST_CASE_BODY(kyuafile_path__default) 125 { 126 std::map< std::string, std::vector< std::string > > options; 127 options["kyuafile"].push_back(cli::kyuafile_option.default_value()); 128 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 129 130 ATF_REQUIRE_EQ(cli::kyuafile_option.default_value(), 131 cli::kyuafile_path(mock_cmdline).str()); 132 } 133 134 135 ATF_TEST_CASE_WITHOUT_HEAD(kyuafile_path__explicit); 136 ATF_TEST_CASE_BODY(kyuafile_path__explicit) 137 { 138 std::map< std::string, std::vector< std::string > > options; 139 options["kyuafile"].push_back("/my//path"); 140 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 141 142 ATF_REQUIRE_EQ("/my/path", cli::kyuafile_path(mock_cmdline).str()); 143 } 144 145 146 ATF_TEST_CASE_WITHOUT_HEAD(result_types__default); 147 ATF_TEST_CASE_BODY(result_types__default) 148 { 149 std::map< std::string, std::vector< std::string > > options; 150 options["results-filter"].push_back( 151 cli::results_filter_option.default_value()); 152 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 153 154 cli::result_types exp_types; 155 exp_types.push_back(engine::test_result::skipped); 156 exp_types.push_back(engine::test_result::expected_failure); 157 exp_types.push_back(engine::test_result::broken); 158 exp_types.push_back(engine::test_result::failed); 159 ATF_REQUIRE(exp_types == cli::get_result_types(mock_cmdline)); 160 } 161 162 163 ATF_TEST_CASE_WITHOUT_HEAD(result_types__empty); 164 ATF_TEST_CASE_BODY(result_types__empty) 165 { 166 std::map< std::string, std::vector< std::string > > options; 167 options["results-filter"].push_back(""); 168 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 169 170 cli::result_types exp_types; 171 exp_types.push_back(engine::test_result::passed); 172 exp_types.push_back(engine::test_result::skipped); 173 exp_types.push_back(engine::test_result::expected_failure); 174 exp_types.push_back(engine::test_result::broken); 175 exp_types.push_back(engine::test_result::failed); 176 ATF_REQUIRE(exp_types == cli::get_result_types(mock_cmdline)); 177 } 178 179 180 ATF_TEST_CASE_WITHOUT_HEAD(result_types__explicit__all); 181 ATF_TEST_CASE_BODY(result_types__explicit__all) 182 { 183 std::map< std::string, std::vector< std::string > > options; 184 options["results-filter"].push_back("passed,skipped,xfail,broken,failed"); 185 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 186 187 cli::result_types exp_types; 188 exp_types.push_back(engine::test_result::passed); 189 exp_types.push_back(engine::test_result::skipped); 190 exp_types.push_back(engine::test_result::expected_failure); 191 exp_types.push_back(engine::test_result::broken); 192 exp_types.push_back(engine::test_result::failed); 193 ATF_REQUIRE(exp_types == cli::get_result_types(mock_cmdline)); 194 } 195 196 197 ATF_TEST_CASE_WITHOUT_HEAD(result_types__explicit__some); 198 ATF_TEST_CASE_BODY(result_types__explicit__some) 199 { 200 std::map< std::string, std::vector< std::string > > options; 201 options["results-filter"].push_back("skipped,broken"); 202 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 203 204 cli::result_types exp_types; 205 exp_types.push_back(engine::test_result::skipped); 206 exp_types.push_back(engine::test_result::broken); 207 ATF_REQUIRE(exp_types == cli::get_result_types(mock_cmdline)); 208 } 209 210 211 ATF_TEST_CASE_WITHOUT_HEAD(result_types__explicit__invalid); 212 ATF_TEST_CASE_BODY(result_types__explicit__invalid) 213 { 214 std::map< std::string, std::vector< std::string > > options; 215 options["results-filter"].push_back("skipped,foo,broken"); 216 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 217 218 ATF_REQUIRE_THROW_RE(std::runtime_error, "Unknown result type 'foo'", 219 cli::get_result_types(mock_cmdline)); 220 } 221 222 223 ATF_TEST_CASE_WITHOUT_HEAD(store_path__default__create_directory__ok); 224 ATF_TEST_CASE_BODY(store_path__default__create_directory__ok) 225 { 226 std::map< std::string, std::vector< std::string > > options; 227 options["store"].push_back(cli::store_option.default_value()); 228 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 229 230 const fs::path home("homedir"); 231 utils::setenv("HOME", home.str()); 232 233 ATF_REQUIRE(!fs::exists(home / ".kyua")); 234 ATF_REQUIRE_EQ(home / ".kyua/store.db", cli::store_path(mock_cmdline)); 235 ATF_REQUIRE(fs::exists(home / ".kyua")); 236 } 237 238 239 ATF_TEST_CASE(store_path__default__create_directory__fail); 240 ATF_TEST_CASE_HEAD(store_path__default__create_directory__fail) 241 { 242 set_md_var("require.user", "unprivileged"); 243 } 244 ATF_TEST_CASE_BODY(store_path__default__create_directory__fail) 245 { 246 std::map< std::string, std::vector< std::string > > options; 247 options["store"].push_back(cli::store_option.default_value()); 248 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 249 250 const fs::path home("homedir"); 251 utils::setenv("HOME", home.str()); 252 fs::mkdir(home, 0555); 253 254 ATF_REQUIRE_THROW(fs::error, cli::store_path(mock_cmdline).str()); 255 ATF_REQUIRE(!fs::exists(home / ".kyua")); 256 } 257 258 259 ATF_TEST_CASE_WITHOUT_HEAD(store_path__default__no_home); 260 ATF_TEST_CASE_BODY(store_path__default__no_home) 261 { 262 std::map< std::string, std::vector< std::string > > options; 263 options["store"].push_back(cli::store_option.default_value()); 264 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 265 266 utils::unsetenv("HOME"); 267 268 ATF_REQUIRE_EQ("kyua-store.db", cli::store_path(mock_cmdline).str()); 269 } 270 271 272 ATF_TEST_CASE_WITHOUT_HEAD(store_path__explicit); 273 ATF_TEST_CASE_BODY(store_path__explicit) 274 { 275 std::map< std::string, std::vector< std::string > > options; 276 options["store"].push_back("/my//path"); 277 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 278 279 const fs::path home("homedir"); 280 ATF_REQUIRE_EQ("/my/path", cli::store_path(mock_cmdline).str()); 281 ATF_REQUIRE(!fs::exists(home / ".kyua")); 282 } 283 284 285 ATF_TEST_CASE_WITHOUT_HEAD(parse_filters__none); 286 ATF_TEST_CASE_BODY(parse_filters__none) 287 { 288 const cmdline::args_vector args; 289 const std::set< engine::test_filter > filters = cli::parse_filters(args); 290 ATF_REQUIRE(filters.empty()); 291 } 292 293 294 ATF_TEST_CASE_WITHOUT_HEAD(parse_filters__ok); 295 ATF_TEST_CASE_BODY(parse_filters__ok) 296 { 297 cmdline::args_vector args; 298 args.push_back("foo"); 299 args.push_back("bar/baz"); 300 args.push_back("other:abc"); 301 args.push_back("other:bcd"); 302 const std::set< engine::test_filter > filters = cli::parse_filters(args); 303 304 std::set< engine::test_filter > exp_filters; 305 exp_filters.insert(mkfilter("foo", "")); 306 exp_filters.insert(mkfilter("bar/baz", "")); 307 exp_filters.insert(mkfilter("other", "abc")); 308 exp_filters.insert(mkfilter("other", "bcd")); 309 310 ATF_REQUIRE(exp_filters == filters); 311 } 312 313 314 ATF_TEST_CASE_WITHOUT_HEAD(parse_filters__duplicate); 315 ATF_TEST_CASE_BODY(parse_filters__duplicate) 316 { 317 cmdline::args_vector args; 318 args.push_back("foo/bar//baz"); 319 args.push_back("hello/world:yes"); 320 args.push_back("foo//bar/baz"); 321 ATF_REQUIRE_THROW_RE(cmdline::error, "Duplicate.*'foo/bar/baz'", 322 cli::parse_filters(args)); 323 } 324 325 326 ATF_TEST_CASE_WITHOUT_HEAD(parse_filters__nondisjoint); 327 ATF_TEST_CASE_BODY(parse_filters__nondisjoint) 328 { 329 cmdline::args_vector args; 330 args.push_back("foo/bar"); 331 args.push_back("hello/world:yes"); 332 args.push_back("foo/bar:baz"); 333 ATF_REQUIRE_THROW_RE(cmdline::error, "'foo/bar'.*'foo/bar:baz'.*disjoint", 334 cli::parse_filters(args)); 335 } 336 337 338 ATF_TEST_CASE_WITHOUT_HEAD(report_unused_filters__none); 339 ATF_TEST_CASE_BODY(report_unused_filters__none) 340 { 341 std::set< engine::test_filter > unused; 342 343 cmdline::ui_mock ui; 344 ATF_REQUIRE(!cli::report_unused_filters(unused, &ui)); 345 ATF_REQUIRE(ui.out_log().empty()); 346 ATF_REQUIRE(ui.err_log().empty()); 347 } 348 349 350 ATF_TEST_CASE_WITHOUT_HEAD(report_unused_filters__some); 351 ATF_TEST_CASE_BODY(report_unused_filters__some) 352 { 353 std::set< engine::test_filter > unused; 354 unused.insert(mkfilter("a/b", "")); 355 unused.insert(mkfilter("hey/d", "yes")); 356 357 cmdline::ui_mock ui; 358 cmdline::init("progname"); 359 ATF_REQUIRE(cli::report_unused_filters(unused, &ui)); 360 ATF_REQUIRE(ui.out_log().empty()); 361 ATF_REQUIRE_EQ(2, ui.err_log().size()); 362 ATF_REQUIRE( atf::utils::grep_collection("No.*matched.*'a/b'", 363 ui.err_log())); 364 ATF_REQUIRE( atf::utils::grep_collection("No.*matched.*'hey/d:yes'", 365 ui.err_log())); 366 } 367 368 369 ATF_TEST_CASE_WITHOUT_HEAD(format_delta); 370 ATF_TEST_CASE_BODY(format_delta) 371 { 372 ATF_REQUIRE_EQ("0.000s", cli::format_delta(datetime::delta())); 373 ATF_REQUIRE_EQ("0.012s", cli::format_delta(datetime::delta(0, 12300))); 374 ATF_REQUIRE_EQ("0.999s", cli::format_delta(datetime::delta(0, 999000))); 375 ATF_REQUIRE_EQ("51.321s", cli::format_delta(datetime::delta(51, 321000))); 376 } 377 378 379 ATF_TEST_CASE_WITHOUT_HEAD(format_result__no_reason); 380 ATF_TEST_CASE_BODY(format_result__no_reason) 381 { 382 ATF_REQUIRE_EQ("passed", cli::format_result( 383 engine::test_result(engine::test_result::passed))); 384 ATF_REQUIRE_EQ("failed", cli::format_result( 385 engine::test_result(engine::test_result::failed))); 386 } 387 388 389 ATF_TEST_CASE_WITHOUT_HEAD(format_result__with_reason); 390 ATF_TEST_CASE_BODY(format_result__with_reason) 391 { 392 ATF_REQUIRE_EQ("broken: Something", cli::format_result( 393 engine::test_result(engine::test_result::broken, "Something"))); 394 ATF_REQUIRE_EQ("expected_failure: A B C", cli::format_result( 395 engine::test_result(engine::test_result::expected_failure, "A B C"))); 396 ATF_REQUIRE_EQ("failed: More text", cli::format_result( 397 engine::test_result(engine::test_result::failed, "More text"))); 398 ATF_REQUIRE_EQ("skipped: Bye", cli::format_result( 399 engine::test_result(engine::test_result::skipped, "Bye"))); 400 } 401 402 403 ATF_TEST_CASE_WITHOUT_HEAD(format_test_case_id__test_case); 404 ATF_TEST_CASE_BODY(format_test_case_id__test_case) 405 { 406 const engine::test_program test_program( 407 "mock", fs::path("foo/bar/baz"), fs::path("unused-root"), 408 "unused-suite-name", engine::metadata_builder().build()); 409 const engine::test_case test_case("mock", test_program, "abc", 410 engine::metadata_builder().build()); 411 ATF_REQUIRE_EQ("foo/bar/baz:abc", cli::format_test_case_id(test_case)); 412 } 413 414 415 ATF_TEST_CASE_WITHOUT_HEAD(format_test_case_id__test_filter); 416 ATF_TEST_CASE_BODY(format_test_case_id__test_filter) 417 { 418 const engine::test_filter filter(fs::path("foo/bar"), "baz"); 419 ATF_REQUIRE_EQ("foo/bar:baz", cli::format_test_case_id(filter)); 420 } 421 422 423 ATF_INIT_TEST_CASES(tcs) 424 { 425 ATF_ADD_TEST_CASE(tcs, build_root_path__default); 426 ATF_ADD_TEST_CASE(tcs, build_root_path__explicit); 427 428 ATF_ADD_TEST_CASE(tcs, get_home__ok); 429 ATF_ADD_TEST_CASE(tcs, get_home__missing); 430 ATF_ADD_TEST_CASE(tcs, get_home__invalid); 431 432 ATF_ADD_TEST_CASE(tcs, kyuafile_path__default); 433 ATF_ADD_TEST_CASE(tcs, kyuafile_path__explicit); 434 435 ATF_ADD_TEST_CASE(tcs, result_types__default); 436 ATF_ADD_TEST_CASE(tcs, result_types__empty); 437 ATF_ADD_TEST_CASE(tcs, result_types__explicit__all); 438 ATF_ADD_TEST_CASE(tcs, result_types__explicit__some); 439 ATF_ADD_TEST_CASE(tcs, result_types__explicit__invalid); 440 441 ATF_ADD_TEST_CASE(tcs, store_path__default__create_directory__ok); 442 ATF_ADD_TEST_CASE(tcs, store_path__default__create_directory__fail); 443 ATF_ADD_TEST_CASE(tcs, store_path__default__no_home); 444 ATF_ADD_TEST_CASE(tcs, store_path__explicit); 445 446 ATF_ADD_TEST_CASE(tcs, parse_filters__none); 447 ATF_ADD_TEST_CASE(tcs, parse_filters__ok); 448 ATF_ADD_TEST_CASE(tcs, parse_filters__duplicate); 449 ATF_ADD_TEST_CASE(tcs, parse_filters__nondisjoint); 450 451 ATF_ADD_TEST_CASE(tcs, report_unused_filters__none); 452 ATF_ADD_TEST_CASE(tcs, report_unused_filters__some); 453 454 ATF_ADD_TEST_CASE(tcs, format_delta); 455 456 ATF_ADD_TEST_CASE(tcs, format_result__no_reason); 457 ATF_ADD_TEST_CASE(tcs, format_result__with_reason); 458 459 ATF_ADD_TEST_CASE(tcs, format_test_case_id__test_case); 460 ATF_ADD_TEST_CASE(tcs, format_test_case_id__test_filter); 461 } 462