1 // Copyright 2011 Google Inc.
2 // All rights reserved.
3 //
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are
6 // met:
7 //
8 // * Redistributions of source code must retain the above copyright
9 // notice, this list of conditions and the following disclaimer.
10 // * Redistributions in binary form must reproduce the above copyright
11 // notice, this list of conditions and the following disclaimer in the
12 // documentation and/or other materials provided with the distribution.
13 // * Neither the name of Google Inc. nor the names of its contributors
14 // may be used to endorse or promote products derived from this software
15 // without specific prior written permission.
16 //
17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 #include "cli/common.hpp"
30
31 #include <fstream>
32
33 #include <atf-c++.hpp>
34
35 #include "engine/exceptions.hpp"
36 #include "engine/filters.hpp"
37 #include "engine/test_case.hpp"
38 #include "engine/test_program.hpp"
39 #include "engine/test_result.hpp"
40 #include "utils/cmdline/exceptions.hpp"
41 #include "utils/cmdline/globals.hpp"
42 #include "utils/cmdline/parser.ipp"
43 #include "utils/cmdline/ui_mock.hpp"
44 #include "utils/datetime.hpp"
45 #include "utils/env.hpp"
46 #include "utils/fs/exceptions.hpp"
47 #include "utils/fs/operations.hpp"
48 #include "utils/fs/path.hpp"
49 #include "utils/optional.ipp"
50 #include "utils/sanity.hpp"
51
52 namespace cmdline = utils::cmdline;
53 namespace config = utils::config;
54 namespace datetime = utils::datetime;
55 namespace fs = utils::fs;
56
57 using utils::optional;
58
59
60 namespace {
61
62
63 /// Syntactic sugar to instantiate engine::test_filter objects.
64 inline engine::test_filter
mkfilter(const char * test_program,const char * test_case)65 mkfilter(const char* test_program, const char* test_case)
66 {
67 return engine::test_filter(fs::path(test_program), test_case);
68 }
69
70
71 } // anonymous namespace
72
73
74 ATF_TEST_CASE_WITHOUT_HEAD(build_root_path__default);
ATF_TEST_CASE_BODY(build_root_path__default)75 ATF_TEST_CASE_BODY(build_root_path__default)
76 {
77 std::map< std::string, std::vector< std::string > > options;
78 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector());
79
80 ATF_REQUIRE(!cli::build_root_path(mock_cmdline));
81 }
82
83
84 ATF_TEST_CASE_WITHOUT_HEAD(build_root_path__explicit);
ATF_TEST_CASE_BODY(build_root_path__explicit)85 ATF_TEST_CASE_BODY(build_root_path__explicit)
86 {
87 std::map< std::string, std::vector< std::string > > options;
88 options["build-root"].push_back("/my//path");
89 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector());
90
91 ATF_REQUIRE(cli::build_root_path(mock_cmdline));
92 ATF_REQUIRE_EQ("/my/path", cli::build_root_path(mock_cmdline).get().str());
93 }
94
95
96 ATF_TEST_CASE_WITHOUT_HEAD(get_home__ok);
ATF_TEST_CASE_BODY(get_home__ok)97 ATF_TEST_CASE_BODY(get_home__ok)
98 {
99 const fs::path home("/foo/bar");
100 utils::setenv("HOME", home.str());
101 const optional< fs::path > computed = cli::get_home();
102 ATF_REQUIRE(computed);
103 ATF_REQUIRE_EQ(home, computed.get());
104 }
105
106
107 ATF_TEST_CASE_WITHOUT_HEAD(get_home__missing);
ATF_TEST_CASE_BODY(get_home__missing)108 ATF_TEST_CASE_BODY(get_home__missing)
109 {
110 utils::unsetenv("HOME");
111 ATF_REQUIRE(!cli::get_home());
112 }
113
114
115 ATF_TEST_CASE_WITHOUT_HEAD(get_home__invalid);
ATF_TEST_CASE_BODY(get_home__invalid)116 ATF_TEST_CASE_BODY(get_home__invalid)
117 {
118 utils::setenv("HOME", "");
119 ATF_REQUIRE(!cli::get_home());
120 }
121
122
123 ATF_TEST_CASE_WITHOUT_HEAD(kyuafile_path__default);
ATF_TEST_CASE_BODY(kyuafile_path__default)124 ATF_TEST_CASE_BODY(kyuafile_path__default)
125 {
126 std::map< std::string, std::vector< std::string > > options;
127 options["kyuafile"].push_back(cli::kyuafile_option.default_value());
128 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector());
129
130 ATF_REQUIRE_EQ(cli::kyuafile_option.default_value(),
131 cli::kyuafile_path(mock_cmdline).str());
132 }
133
134
135 ATF_TEST_CASE_WITHOUT_HEAD(kyuafile_path__explicit);
ATF_TEST_CASE_BODY(kyuafile_path__explicit)136 ATF_TEST_CASE_BODY(kyuafile_path__explicit)
137 {
138 std::map< std::string, std::vector< std::string > > options;
139 options["kyuafile"].push_back("/my//path");
140 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector());
141
142 ATF_REQUIRE_EQ("/my/path", cli::kyuafile_path(mock_cmdline).str());
143 }
144
145
146 ATF_TEST_CASE_WITHOUT_HEAD(result_types__default);
ATF_TEST_CASE_BODY(result_types__default)147 ATF_TEST_CASE_BODY(result_types__default)
148 {
149 std::map< std::string, std::vector< std::string > > options;
150 options["results-filter"].push_back(
151 cli::results_filter_option.default_value());
152 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector());
153
154 cli::result_types exp_types;
155 exp_types.push_back(engine::test_result::skipped);
156 exp_types.push_back(engine::test_result::expected_failure);
157 exp_types.push_back(engine::test_result::broken);
158 exp_types.push_back(engine::test_result::failed);
159 ATF_REQUIRE(exp_types == cli::get_result_types(mock_cmdline));
160 }
161
162
163 ATF_TEST_CASE_WITHOUT_HEAD(result_types__empty);
ATF_TEST_CASE_BODY(result_types__empty)164 ATF_TEST_CASE_BODY(result_types__empty)
165 {
166 std::map< std::string, std::vector< std::string > > options;
167 options["results-filter"].push_back("");
168 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector());
169
170 cli::result_types exp_types;
171 exp_types.push_back(engine::test_result::passed);
172 exp_types.push_back(engine::test_result::skipped);
173 exp_types.push_back(engine::test_result::expected_failure);
174 exp_types.push_back(engine::test_result::broken);
175 exp_types.push_back(engine::test_result::failed);
176 ATF_REQUIRE(exp_types == cli::get_result_types(mock_cmdline));
177 }
178
179
180 ATF_TEST_CASE_WITHOUT_HEAD(result_types__explicit__all);
ATF_TEST_CASE_BODY(result_types__explicit__all)181 ATF_TEST_CASE_BODY(result_types__explicit__all)
182 {
183 std::map< std::string, std::vector< std::string > > options;
184 options["results-filter"].push_back("passed,skipped,xfail,broken,failed");
185 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector());
186
187 cli::result_types exp_types;
188 exp_types.push_back(engine::test_result::passed);
189 exp_types.push_back(engine::test_result::skipped);
190 exp_types.push_back(engine::test_result::expected_failure);
191 exp_types.push_back(engine::test_result::broken);
192 exp_types.push_back(engine::test_result::failed);
193 ATF_REQUIRE(exp_types == cli::get_result_types(mock_cmdline));
194 }
195
196
197 ATF_TEST_CASE_WITHOUT_HEAD(result_types__explicit__some);
ATF_TEST_CASE_BODY(result_types__explicit__some)198 ATF_TEST_CASE_BODY(result_types__explicit__some)
199 {
200 std::map< std::string, std::vector< std::string > > options;
201 options["results-filter"].push_back("skipped,broken");
202 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector());
203
204 cli::result_types exp_types;
205 exp_types.push_back(engine::test_result::skipped);
206 exp_types.push_back(engine::test_result::broken);
207 ATF_REQUIRE(exp_types == cli::get_result_types(mock_cmdline));
208 }
209
210
211 ATF_TEST_CASE_WITHOUT_HEAD(result_types__explicit__invalid);
ATF_TEST_CASE_BODY(result_types__explicit__invalid)212 ATF_TEST_CASE_BODY(result_types__explicit__invalid)
213 {
214 std::map< std::string, std::vector< std::string > > options;
215 options["results-filter"].push_back("skipped,foo,broken");
216 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector());
217
218 ATF_REQUIRE_THROW_RE(std::runtime_error, "Unknown result type 'foo'",
219 cli::get_result_types(mock_cmdline));
220 }
221
222
223 ATF_TEST_CASE_WITHOUT_HEAD(store_path__default__create_directory__ok);
ATF_TEST_CASE_BODY(store_path__default__create_directory__ok)224 ATF_TEST_CASE_BODY(store_path__default__create_directory__ok)
225 {
226 std::map< std::string, std::vector< std::string > > options;
227 options["store"].push_back(cli::store_option.default_value());
228 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector());
229
230 const fs::path home("homedir");
231 utils::setenv("HOME", home.str());
232
233 ATF_REQUIRE(!fs::exists(home / ".kyua"));
234 ATF_REQUIRE_EQ(home / ".kyua/store.db", cli::store_path(mock_cmdline));
235 ATF_REQUIRE(fs::exists(home / ".kyua"));
236 }
237
238
239 ATF_TEST_CASE(store_path__default__create_directory__fail);
ATF_TEST_CASE_HEAD(store_path__default__create_directory__fail)240 ATF_TEST_CASE_HEAD(store_path__default__create_directory__fail)
241 {
242 set_md_var("require.user", "unprivileged");
243 }
ATF_TEST_CASE_BODY(store_path__default__create_directory__fail)244 ATF_TEST_CASE_BODY(store_path__default__create_directory__fail)
245 {
246 std::map< std::string, std::vector< std::string > > options;
247 options["store"].push_back(cli::store_option.default_value());
248 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector());
249
250 const fs::path home("homedir");
251 utils::setenv("HOME", home.str());
252 fs::mkdir(home, 0555);
253
254 ATF_REQUIRE_THROW(fs::error, cli::store_path(mock_cmdline).str());
255 ATF_REQUIRE(!fs::exists(home / ".kyua"));
256 }
257
258
259 ATF_TEST_CASE_WITHOUT_HEAD(store_path__default__no_home);
ATF_TEST_CASE_BODY(store_path__default__no_home)260 ATF_TEST_CASE_BODY(store_path__default__no_home)
261 {
262 std::map< std::string, std::vector< std::string > > options;
263 options["store"].push_back(cli::store_option.default_value());
264 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector());
265
266 utils::unsetenv("HOME");
267
268 ATF_REQUIRE_EQ("kyua-store.db", cli::store_path(mock_cmdline).str());
269 }
270
271
272 ATF_TEST_CASE_WITHOUT_HEAD(store_path__explicit);
ATF_TEST_CASE_BODY(store_path__explicit)273 ATF_TEST_CASE_BODY(store_path__explicit)
274 {
275 std::map< std::string, std::vector< std::string > > options;
276 options["store"].push_back("/my//path");
277 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector());
278
279 const fs::path home("homedir");
280 ATF_REQUIRE_EQ("/my/path", cli::store_path(mock_cmdline).str());
281 ATF_REQUIRE(!fs::exists(home / ".kyua"));
282 }
283
284
285 ATF_TEST_CASE_WITHOUT_HEAD(parse_filters__none);
ATF_TEST_CASE_BODY(parse_filters__none)286 ATF_TEST_CASE_BODY(parse_filters__none)
287 {
288 const cmdline::args_vector args;
289 const std::set< engine::test_filter > filters = cli::parse_filters(args);
290 ATF_REQUIRE(filters.empty());
291 }
292
293
294 ATF_TEST_CASE_WITHOUT_HEAD(parse_filters__ok);
ATF_TEST_CASE_BODY(parse_filters__ok)295 ATF_TEST_CASE_BODY(parse_filters__ok)
296 {
297 cmdline::args_vector args;
298 args.push_back("foo");
299 args.push_back("bar/baz");
300 args.push_back("other:abc");
301 args.push_back("other:bcd");
302 const std::set< engine::test_filter > filters = cli::parse_filters(args);
303
304 std::set< engine::test_filter > exp_filters;
305 exp_filters.insert(mkfilter("foo", ""));
306 exp_filters.insert(mkfilter("bar/baz", ""));
307 exp_filters.insert(mkfilter("other", "abc"));
308 exp_filters.insert(mkfilter("other", "bcd"));
309
310 ATF_REQUIRE(exp_filters == filters);
311 }
312
313
314 ATF_TEST_CASE_WITHOUT_HEAD(parse_filters__duplicate);
ATF_TEST_CASE_BODY(parse_filters__duplicate)315 ATF_TEST_CASE_BODY(parse_filters__duplicate)
316 {
317 cmdline::args_vector args;
318 args.push_back("foo/bar//baz");
319 args.push_back("hello/world:yes");
320 args.push_back("foo//bar/baz");
321 ATF_REQUIRE_THROW_RE(cmdline::error, "Duplicate.*'foo/bar/baz'",
322 cli::parse_filters(args));
323 }
324
325
326 ATF_TEST_CASE_WITHOUT_HEAD(parse_filters__nondisjoint);
ATF_TEST_CASE_BODY(parse_filters__nondisjoint)327 ATF_TEST_CASE_BODY(parse_filters__nondisjoint)
328 {
329 cmdline::args_vector args;
330 args.push_back("foo/bar");
331 args.push_back("hello/world:yes");
332 args.push_back("foo/bar:baz");
333 ATF_REQUIRE_THROW_RE(cmdline::error, "'foo/bar'.*'foo/bar:baz'.*disjoint",
334 cli::parse_filters(args));
335 }
336
337
338 ATF_TEST_CASE_WITHOUT_HEAD(report_unused_filters__none);
ATF_TEST_CASE_BODY(report_unused_filters__none)339 ATF_TEST_CASE_BODY(report_unused_filters__none)
340 {
341 std::set< engine::test_filter > unused;
342
343 cmdline::ui_mock ui;
344 ATF_REQUIRE(!cli::report_unused_filters(unused, &ui));
345 ATF_REQUIRE(ui.out_log().empty());
346 ATF_REQUIRE(ui.err_log().empty());
347 }
348
349
350 ATF_TEST_CASE_WITHOUT_HEAD(report_unused_filters__some);
ATF_TEST_CASE_BODY(report_unused_filters__some)351 ATF_TEST_CASE_BODY(report_unused_filters__some)
352 {
353 std::set< engine::test_filter > unused;
354 unused.insert(mkfilter("a/b", ""));
355 unused.insert(mkfilter("hey/d", "yes"));
356
357 cmdline::ui_mock ui;
358 cmdline::init("progname");
359 ATF_REQUIRE(cli::report_unused_filters(unused, &ui));
360 ATF_REQUIRE(ui.out_log().empty());
361 ATF_REQUIRE_EQ(2, ui.err_log().size());
362 ATF_REQUIRE( atf::utils::grep_collection("No.*matched.*'a/b'",
363 ui.err_log()));
364 ATF_REQUIRE( atf::utils::grep_collection("No.*matched.*'hey/d:yes'",
365 ui.err_log()));
366 }
367
368
369 ATF_TEST_CASE_WITHOUT_HEAD(format_delta);
ATF_TEST_CASE_BODY(format_delta)370 ATF_TEST_CASE_BODY(format_delta)
371 {
372 ATF_REQUIRE_EQ("0.000s", cli::format_delta(datetime::delta()));
373 ATF_REQUIRE_EQ("0.012s", cli::format_delta(datetime::delta(0, 12300)));
374 ATF_REQUIRE_EQ("0.999s", cli::format_delta(datetime::delta(0, 999000)));
375 ATF_REQUIRE_EQ("51.321s", cli::format_delta(datetime::delta(51, 321000)));
376 }
377
378
379 ATF_TEST_CASE_WITHOUT_HEAD(format_result__no_reason);
ATF_TEST_CASE_BODY(format_result__no_reason)380 ATF_TEST_CASE_BODY(format_result__no_reason)
381 {
382 ATF_REQUIRE_EQ("passed", cli::format_result(
383 engine::test_result(engine::test_result::passed)));
384 ATF_REQUIRE_EQ("failed", cli::format_result(
385 engine::test_result(engine::test_result::failed)));
386 }
387
388
389 ATF_TEST_CASE_WITHOUT_HEAD(format_result__with_reason);
ATF_TEST_CASE_BODY(format_result__with_reason)390 ATF_TEST_CASE_BODY(format_result__with_reason)
391 {
392 ATF_REQUIRE_EQ("broken: Something", cli::format_result(
393 engine::test_result(engine::test_result::broken, "Something")));
394 ATF_REQUIRE_EQ("expected_failure: A B C", cli::format_result(
395 engine::test_result(engine::test_result::expected_failure, "A B C")));
396 ATF_REQUIRE_EQ("failed: More text", cli::format_result(
397 engine::test_result(engine::test_result::failed, "More text")));
398 ATF_REQUIRE_EQ("skipped: Bye", cli::format_result(
399 engine::test_result(engine::test_result::skipped, "Bye")));
400 }
401
402
403 ATF_TEST_CASE_WITHOUT_HEAD(format_test_case_id__test_case);
ATF_TEST_CASE_BODY(format_test_case_id__test_case)404 ATF_TEST_CASE_BODY(format_test_case_id__test_case)
405 {
406 const engine::test_program test_program(
407 "mock", fs::path("foo/bar/baz"), fs::path("unused-root"),
408 "unused-suite-name", engine::metadata_builder().build());
409 const engine::test_case test_case("mock", test_program, "abc",
410 engine::metadata_builder().build());
411 ATF_REQUIRE_EQ("foo/bar/baz:abc", cli::format_test_case_id(test_case));
412 }
413
414
415 ATF_TEST_CASE_WITHOUT_HEAD(format_test_case_id__test_filter);
ATF_TEST_CASE_BODY(format_test_case_id__test_filter)416 ATF_TEST_CASE_BODY(format_test_case_id__test_filter)
417 {
418 const engine::test_filter filter(fs::path("foo/bar"), "baz");
419 ATF_REQUIRE_EQ("foo/bar:baz", cli::format_test_case_id(filter));
420 }
421
422
ATF_INIT_TEST_CASES(tcs)423 ATF_INIT_TEST_CASES(tcs)
424 {
425 ATF_ADD_TEST_CASE(tcs, build_root_path__default);
426 ATF_ADD_TEST_CASE(tcs, build_root_path__explicit);
427
428 ATF_ADD_TEST_CASE(tcs, get_home__ok);
429 ATF_ADD_TEST_CASE(tcs, get_home__missing);
430 ATF_ADD_TEST_CASE(tcs, get_home__invalid);
431
432 ATF_ADD_TEST_CASE(tcs, kyuafile_path__default);
433 ATF_ADD_TEST_CASE(tcs, kyuafile_path__explicit);
434
435 ATF_ADD_TEST_CASE(tcs, result_types__default);
436 ATF_ADD_TEST_CASE(tcs, result_types__empty);
437 ATF_ADD_TEST_CASE(tcs, result_types__explicit__all);
438 ATF_ADD_TEST_CASE(tcs, result_types__explicit__some);
439 ATF_ADD_TEST_CASE(tcs, result_types__explicit__invalid);
440
441 ATF_ADD_TEST_CASE(tcs, store_path__default__create_directory__ok);
442 ATF_ADD_TEST_CASE(tcs, store_path__default__create_directory__fail);
443 ATF_ADD_TEST_CASE(tcs, store_path__default__no_home);
444 ATF_ADD_TEST_CASE(tcs, store_path__explicit);
445
446 ATF_ADD_TEST_CASE(tcs, parse_filters__none);
447 ATF_ADD_TEST_CASE(tcs, parse_filters__ok);
448 ATF_ADD_TEST_CASE(tcs, parse_filters__duplicate);
449 ATF_ADD_TEST_CASE(tcs, parse_filters__nondisjoint);
450
451 ATF_ADD_TEST_CASE(tcs, report_unused_filters__none);
452 ATF_ADD_TEST_CASE(tcs, report_unused_filters__some);
453
454 ATF_ADD_TEST_CASE(tcs, format_delta);
455
456 ATF_ADD_TEST_CASE(tcs, format_result__no_reason);
457 ATF_ADD_TEST_CASE(tcs, format_result__with_reason);
458
459 ATF_ADD_TEST_CASE(tcs, format_test_case_id__test_case);
460 ATF_ADD_TEST_CASE(tcs, format_test_case_id__test_filter);
461 }
462