1 // Copyright 2011 Google Inc. 2 // All rights reserved. 3 // 4 // Redistribution and use in source and binary forms, with or without 5 // modification, are permitted provided that the following conditions are 6 // met: 7 // 8 // * Redistributions of source code must retain the above copyright 9 // notice, this list of conditions and the following disclaimer. 10 // * Redistributions in binary form must reproduce the above copyright 11 // notice, this list of conditions and the following disclaimer in the 12 // documentation and/or other materials provided with the distribution. 13 // * Neither the name of Google Inc. nor the names of its contributors 14 // may be used to endorse or promote products derived from this software 15 // without specific prior written permission. 16 // 17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 #include "cli/common.hpp" 30 31 #include <fstream> 32 33 #include <atf-c++.hpp> 34 35 #include "engine/exceptions.hpp" 36 #include "engine/filters.hpp" 37 #include "engine/test_case.hpp" 38 #include "engine/test_program.hpp" 39 #include "engine/test_result.hpp" 40 #include "utils/cmdline/exceptions.hpp" 41 #include "utils/cmdline/globals.hpp" 42 #include "utils/cmdline/parser.ipp" 43 #include "utils/cmdline/ui_mock.hpp" 44 #include "utils/datetime.hpp" 45 #include "utils/env.hpp" 46 #include "utils/fs/exceptions.hpp" 47 #include "utils/fs/operations.hpp" 48 #include "utils/fs/path.hpp" 49 #include "utils/optional.ipp" 50 #include "utils/sanity.hpp" 51 52 namespace cmdline = utils::cmdline; 53 namespace config = utils::config; 54 namespace datetime = utils::datetime; 55 namespace fs = utils::fs; 56 57 using utils::optional; 58 59 60 namespace { 61 62 63 /// Syntactic sugar to instantiate engine::test_filter objects. 64 inline engine::test_filter 65 mkfilter(const char* test_program, const char* test_case) 66 { 67 return engine::test_filter(fs::path(test_program), test_case); 68 } 69 70 71 } // anonymous namespace 72 73 74 ATF_TEST_CASE_WITHOUT_HEAD(build_root_path__default); 75 ATF_TEST_CASE_BODY(build_root_path__default) 76 { 77 std::map< std::string, std::vector< std::string > > options; 78 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 79 80 ATF_REQUIRE(!cli::build_root_path(mock_cmdline)); 81 } 82 83 84 ATF_TEST_CASE_WITHOUT_HEAD(build_root_path__explicit); 85 ATF_TEST_CASE_BODY(build_root_path__explicit) 86 { 87 std::map< std::string, std::vector< std::string > > options; 88 options["build-root"].push_back("/my//path"); 89 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 90 91 ATF_REQUIRE(cli::build_root_path(mock_cmdline)); 92 ATF_REQUIRE_EQ("/my/path", cli::build_root_path(mock_cmdline).get().str()); 93 } 94 95 96 ATF_TEST_CASE_WITHOUT_HEAD(get_home__ok); 97 ATF_TEST_CASE_BODY(get_home__ok) 98 { 99 const fs::path home("/foo/bar"); 100 utils::setenv("HOME", home.str()); 101 const optional< fs::path > computed = cli::get_home(); 102 ATF_REQUIRE(computed); 103 ATF_REQUIRE_EQ(home, computed.get()); 104 } 105 106 107 ATF_TEST_CASE_WITHOUT_HEAD(get_home__missing); 108 ATF_TEST_CASE_BODY(get_home__missing) 109 { 110 utils::unsetenv("HOME"); 111 ATF_REQUIRE(!cli::get_home()); 112 } 113 114 115 ATF_TEST_CASE_WITHOUT_HEAD(get_home__invalid); 116 ATF_TEST_CASE_BODY(get_home__invalid) 117 { 118 utils::setenv("HOME", ""); 119 ATF_REQUIRE(!cli::get_home()); 120 } 121 122 123 ATF_TEST_CASE_WITHOUT_HEAD(kyuafile_path__default); 124 ATF_TEST_CASE_BODY(kyuafile_path__default) 125 { 126 std::map< std::string, std::vector< std::string > > options; 127 options["kyuafile"].push_back(cli::kyuafile_option.default_value()); 128 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 129 130 ATF_REQUIRE_EQ(cli::kyuafile_option.default_value(), 131 cli::kyuafile_path(mock_cmdline).str()); 132 } 133 134 135 ATF_TEST_CASE_WITHOUT_HEAD(kyuafile_path__explicit); 136 ATF_TEST_CASE_BODY(kyuafile_path__explicit) 137 { 138 std::map< std::string, std::vector< std::string > > options; 139 options["kyuafile"].push_back("/my//path"); 140 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 141 142 ATF_REQUIRE_EQ("/my/path", cli::kyuafile_path(mock_cmdline).str()); 143 } 144 145 146 ATF_TEST_CASE_WITHOUT_HEAD(store_path__default__create_directory__ok); 147 ATF_TEST_CASE_BODY(store_path__default__create_directory__ok) 148 { 149 std::map< std::string, std::vector< std::string > > options; 150 options["store"].push_back(cli::store_option.default_value()); 151 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 152 153 const fs::path home("homedir"); 154 utils::setenv("HOME", home.str()); 155 156 ATF_REQUIRE(!fs::exists(home / ".kyua")); 157 ATF_REQUIRE_EQ(home / ".kyua/store.db", cli::store_path(mock_cmdline)); 158 ATF_REQUIRE(fs::exists(home / ".kyua")); 159 } 160 161 162 ATF_TEST_CASE(store_path__default__create_directory__fail); 163 ATF_TEST_CASE_HEAD(store_path__default__create_directory__fail) 164 { 165 set_md_var("require.user", "unprivileged"); 166 } 167 ATF_TEST_CASE_BODY(store_path__default__create_directory__fail) 168 { 169 std::map< std::string, std::vector< std::string > > options; 170 options["store"].push_back(cli::store_option.default_value()); 171 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 172 173 const fs::path home("homedir"); 174 utils::setenv("HOME", home.str()); 175 fs::mkdir(home, 0555); 176 177 ATF_REQUIRE_THROW(fs::error, cli::store_path(mock_cmdline).str()); 178 ATF_REQUIRE(!fs::exists(home / ".kyua")); 179 } 180 181 182 ATF_TEST_CASE_WITHOUT_HEAD(store_path__default__no_home); 183 ATF_TEST_CASE_BODY(store_path__default__no_home) 184 { 185 std::map< std::string, std::vector< std::string > > options; 186 options["store"].push_back(cli::store_option.default_value()); 187 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 188 189 utils::unsetenv("HOME"); 190 191 ATF_REQUIRE_EQ("kyua-store.db", cli::store_path(mock_cmdline).str()); 192 } 193 194 195 ATF_TEST_CASE_WITHOUT_HEAD(store_path__explicit); 196 ATF_TEST_CASE_BODY(store_path__explicit) 197 { 198 std::map< std::string, std::vector< std::string > > options; 199 options["store"].push_back("/my//path"); 200 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 201 202 const fs::path home("homedir"); 203 ATF_REQUIRE_EQ("/my/path", cli::store_path(mock_cmdline).str()); 204 ATF_REQUIRE(!fs::exists(home / ".kyua")); 205 } 206 207 208 ATF_TEST_CASE_WITHOUT_HEAD(parse_filters__none); 209 ATF_TEST_CASE_BODY(parse_filters__none) 210 { 211 const cmdline::args_vector args; 212 const std::set< engine::test_filter > filters = cli::parse_filters(args); 213 ATF_REQUIRE(filters.empty()); 214 } 215 216 217 ATF_TEST_CASE_WITHOUT_HEAD(parse_filters__ok); 218 ATF_TEST_CASE_BODY(parse_filters__ok) 219 { 220 cmdline::args_vector args; 221 args.push_back("foo"); 222 args.push_back("bar/baz"); 223 args.push_back("other:abc"); 224 args.push_back("other:bcd"); 225 const std::set< engine::test_filter > filters = cli::parse_filters(args); 226 227 std::set< engine::test_filter > exp_filters; 228 exp_filters.insert(mkfilter("foo", "")); 229 exp_filters.insert(mkfilter("bar/baz", "")); 230 exp_filters.insert(mkfilter("other", "abc")); 231 exp_filters.insert(mkfilter("other", "bcd")); 232 233 ATF_REQUIRE(exp_filters == filters); 234 } 235 236 237 ATF_TEST_CASE_WITHOUT_HEAD(parse_filters__duplicate); 238 ATF_TEST_CASE_BODY(parse_filters__duplicate) 239 { 240 cmdline::args_vector args; 241 args.push_back("foo/bar//baz"); 242 args.push_back("hello/world:yes"); 243 args.push_back("foo//bar/baz"); 244 ATF_REQUIRE_THROW_RE(cmdline::error, "Duplicate.*'foo/bar/baz'", 245 cli::parse_filters(args)); 246 } 247 248 249 ATF_TEST_CASE_WITHOUT_HEAD(parse_filters__nondisjoint); 250 ATF_TEST_CASE_BODY(parse_filters__nondisjoint) 251 { 252 cmdline::args_vector args; 253 args.push_back("foo/bar"); 254 args.push_back("hello/world:yes"); 255 args.push_back("foo/bar:baz"); 256 ATF_REQUIRE_THROW_RE(cmdline::error, "'foo/bar'.*'foo/bar:baz'.*disjoint", 257 cli::parse_filters(args)); 258 } 259 260 261 ATF_TEST_CASE_WITHOUT_HEAD(report_unused_filters__none); 262 ATF_TEST_CASE_BODY(report_unused_filters__none) 263 { 264 std::set< engine::test_filter > unused; 265 266 cmdline::ui_mock ui; 267 ATF_REQUIRE(!cli::report_unused_filters(unused, &ui)); 268 ATF_REQUIRE(ui.out_log().empty()); 269 ATF_REQUIRE(ui.err_log().empty()); 270 } 271 272 273 ATF_TEST_CASE_WITHOUT_HEAD(report_unused_filters__some); 274 ATF_TEST_CASE_BODY(report_unused_filters__some) 275 { 276 std::set< engine::test_filter > unused; 277 unused.insert(mkfilter("a/b", "")); 278 unused.insert(mkfilter("hey/d", "yes")); 279 280 cmdline::ui_mock ui; 281 cmdline::init("progname"); 282 ATF_REQUIRE(cli::report_unused_filters(unused, &ui)); 283 ATF_REQUIRE(ui.out_log().empty()); 284 ATF_REQUIRE_EQ(2, ui.err_log().size()); 285 ATF_REQUIRE( atf::utils::grep_collection("No.*matched.*'a/b'", 286 ui.err_log())); 287 ATF_REQUIRE( atf::utils::grep_collection("No.*matched.*'hey/d:yes'", 288 ui.err_log())); 289 } 290 291 292 ATF_TEST_CASE_WITHOUT_HEAD(format_delta); 293 ATF_TEST_CASE_BODY(format_delta) 294 { 295 ATF_REQUIRE_EQ("0.000s", cli::format_delta(datetime::delta())); 296 ATF_REQUIRE_EQ("0.012s", cli::format_delta(datetime::delta(0, 12300))); 297 ATF_REQUIRE_EQ("0.999s", cli::format_delta(datetime::delta(0, 999000))); 298 ATF_REQUIRE_EQ("51.321s", cli::format_delta(datetime::delta(51, 321000))); 299 } 300 301 302 ATF_TEST_CASE_WITHOUT_HEAD(format_result__no_reason); 303 ATF_TEST_CASE_BODY(format_result__no_reason) 304 { 305 ATF_REQUIRE_EQ("passed", cli::format_result( 306 engine::test_result(engine::test_result::passed))); 307 ATF_REQUIRE_EQ("failed", cli::format_result( 308 engine::test_result(engine::test_result::failed))); 309 } 310 311 312 ATF_TEST_CASE_WITHOUT_HEAD(format_result__with_reason); 313 ATF_TEST_CASE_BODY(format_result__with_reason) 314 { 315 ATF_REQUIRE_EQ("broken: Something", cli::format_result( 316 engine::test_result(engine::test_result::broken, "Something"))); 317 ATF_REQUIRE_EQ("expected_failure: A B C", cli::format_result( 318 engine::test_result(engine::test_result::expected_failure, "A B C"))); 319 ATF_REQUIRE_EQ("failed: More text", cli::format_result( 320 engine::test_result(engine::test_result::failed, "More text"))); 321 ATF_REQUIRE_EQ("skipped: Bye", cli::format_result( 322 engine::test_result(engine::test_result::skipped, "Bye"))); 323 } 324 325 326 ATF_TEST_CASE_WITHOUT_HEAD(format_test_case_id__test_case); 327 ATF_TEST_CASE_BODY(format_test_case_id__test_case) 328 { 329 const engine::test_program test_program( 330 "mock", fs::path("foo/bar/baz"), fs::path("unused-root"), 331 "unused-suite-name", engine::metadata_builder().build()); 332 const engine::test_case test_case("mock", test_program, "abc", 333 engine::metadata_builder().build()); 334 ATF_REQUIRE_EQ("foo/bar/baz:abc", cli::format_test_case_id(test_case)); 335 } 336 337 338 ATF_TEST_CASE_WITHOUT_HEAD(format_test_case_id__test_filter); 339 ATF_TEST_CASE_BODY(format_test_case_id__test_filter) 340 { 341 const engine::test_filter filter(fs::path("foo/bar"), "baz"); 342 ATF_REQUIRE_EQ("foo/bar:baz", cli::format_test_case_id(filter)); 343 } 344 345 346 ATF_INIT_TEST_CASES(tcs) 347 { 348 ATF_ADD_TEST_CASE(tcs, build_root_path__default); 349 ATF_ADD_TEST_CASE(tcs, build_root_path__explicit); 350 351 ATF_ADD_TEST_CASE(tcs, get_home__ok); 352 ATF_ADD_TEST_CASE(tcs, get_home__missing); 353 ATF_ADD_TEST_CASE(tcs, get_home__invalid); 354 355 ATF_ADD_TEST_CASE(tcs, kyuafile_path__default); 356 ATF_ADD_TEST_CASE(tcs, kyuafile_path__explicit); 357 358 ATF_ADD_TEST_CASE(tcs, store_path__default__create_directory__ok); 359 ATF_ADD_TEST_CASE(tcs, store_path__default__create_directory__fail); 360 ATF_ADD_TEST_CASE(tcs, store_path__default__no_home); 361 ATF_ADD_TEST_CASE(tcs, store_path__explicit); 362 363 ATF_ADD_TEST_CASE(tcs, parse_filters__none); 364 ATF_ADD_TEST_CASE(tcs, parse_filters__ok); 365 ATF_ADD_TEST_CASE(tcs, parse_filters__duplicate); 366 ATF_ADD_TEST_CASE(tcs, parse_filters__nondisjoint); 367 368 ATF_ADD_TEST_CASE(tcs, report_unused_filters__none); 369 ATF_ADD_TEST_CASE(tcs, report_unused_filters__some); 370 371 ATF_ADD_TEST_CASE(tcs, format_delta); 372 373 ATF_ADD_TEST_CASE(tcs, format_result__no_reason); 374 ATF_ADD_TEST_CASE(tcs, format_result__with_reason); 375 376 ATF_ADD_TEST_CASE(tcs, format_test_case_id__test_case); 377 ATF_ADD_TEST_CASE(tcs, format_test_case_id__test_filter); 378 } 379