1 // Copyright 2012 Google Inc.
2 // All rights reserved.
3 //
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are
6 // met:
7 //
8 // * Redistributions of source code must retain the above copyright
9 // notice, this list of conditions and the following disclaimer.
10 // * Redistributions in binary form must reproduce the above copyright
11 // notice, this list of conditions and the following disclaimer in the
12 // documentation and/or other materials provided with the distribution.
13 // * Neither the name of Google Inc. nor the names of its contributors
14 // may be used to endorse or promote products derived from this software
15 // without specific prior written permission.
16 //
17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 #include "engine/testers.hpp"
30
31 extern "C" {
32 #include <sys/stat.h>
33
34 #include <signal.h>
35 }
36
37 #include <cstdlib>
38
39 #include <atf-c++.hpp>
40
41 #include "engine/exceptions.hpp"
42 #include "utils/datetime.hpp"
43 #include "utils/env.hpp"
44 #include "utils/format/macros.hpp"
45 #include "utils/fs/path.hpp"
46 #include "utils/fs/operations.hpp"
47 #include "utils/optional.ipp"
48 #include "utils/passwd.hpp"
49
50 namespace datetime = utils::datetime;
51 namespace fs = utils::fs;
52 namespace passwd = utils::passwd;
53
54 using utils::none;
55 using utils::optional;
56
57
58 namespace {
59
60
61 /// Creates a mock tester that exits cleanly.
62 ///
63 /// The interface accepted by the tester is 'mock'. This tester outputs the
64 /// arguments passed to it and then prints a message to both the stdout and the
65 /// stderr.
66 ///
67 /// \param exit_status Code to exit with.
68 static void
create_mock_tester_exit(const int exit_status)69 create_mock_tester_exit(const int exit_status)
70 {
71 atf::utils::create_file(
72 "kyua-mock-tester",
73 F("#! /bin/sh\n"
74 "while [ ${#} -gt 0 ]; do\n"
75 " echo \"Arg: ${1}\"\n"
76 " shift\n"
77 "done\n"
78 "echo 'tester output'\n"
79 "echo 'tester error' 1>&2\n"
80 "exit %s\n") % exit_status);
81 ATF_REQUIRE(::chmod("kyua-mock-tester", 0755) != -1);
82
83 utils::setenv("KYUA_TESTERSDIR", fs::current_path().str());
84 }
85
86
87 /// Creates a mock tester that receives a signal.
88 ///
89 /// The interface accepted by the tester is 'mock'. This tester outputs the
90 /// arguments passed to it and then prints a message to both the stdout and the
91 /// stderr.
92 ///
93 /// \param term_sig Signal to deliver to the tester. If the tester does not
94 /// exit due to this reason, it exits with an arbitrary non-zero code.
95 static void
create_mock_tester_signal(const int term_sig)96 create_mock_tester_signal(const int term_sig)
97 {
98 atf::utils::create_file(
99 "kyua-mock-tester",
100 F("#! /bin/sh\n"
101 "while [ ${#} -gt 0 ]; do\n"
102 " echo \"Arg: ${1}\"\n"
103 " shift\n"
104 "done\n"
105 "echo 'tester output'\n"
106 "echo 'tester error' 1>&2\n"
107 "kill -%s $$\n"
108 "echo 'signal did not terminate the process\n"
109 "exit 0\n") % term_sig);
110 ATF_REQUIRE(::chmod("kyua-mock-tester", 0755) != -1);
111
112 utils::setenv("KYUA_TESTERSDIR", fs::current_path().str());
113 }
114
115
116 } // anonymous namespace
117
118
119 ATF_TEST_CASE_WITHOUT_HEAD(tester__list__defaults);
ATF_TEST_CASE_BODY(tester__list__defaults)120 ATF_TEST_CASE_BODY(tester__list__defaults)
121 {
122 create_mock_tester_exit(EXIT_SUCCESS);
123 engine::tester tester("mock", none, none);
124 const std::string output = tester.list(fs::path("/foo/bar"));
125
126 const std::string exp_output =
127 "Arg: list\n"
128 "Arg: /foo/bar\n"
129 "tester output\n"
130 "tester error\n";
131 ATF_REQUIRE_EQ(exp_output, output);
132 }
133
134
135 ATF_TEST_CASE_WITHOUT_HEAD(tester__list__explicit_common_args);
ATF_TEST_CASE_BODY(tester__list__explicit_common_args)136 ATF_TEST_CASE_BODY(tester__list__explicit_common_args)
137 {
138 const passwd::user user("fake", 123, 456);
139 const datetime::delta timeout(15, 0);
140
141 create_mock_tester_exit(EXIT_SUCCESS);
142 engine::tester tester("mock", utils::make_optional(user),
143 utils::make_optional(timeout));
144 const std::string output = tester.list(fs::path("/another/program/1"));
145
146 const std::string exp_output =
147 "Arg: -u123\n"
148 "Arg: -g456\n"
149 "Arg: -t15\n"
150 "Arg: list\n"
151 "Arg: /another/program/1\n"
152 "tester output\n"
153 "tester error\n";
154 ATF_REQUIRE_EQ(exp_output, output);
155 }
156
157
158 ATF_TEST_CASE_WITHOUT_HEAD(tester__list__unknown_interface);
ATF_TEST_CASE_BODY(tester__list__unknown_interface)159 ATF_TEST_CASE_BODY(tester__list__unknown_interface)
160 {
161 utils::setenv("KYUA_TESTERSDIR", ".");
162 engine::tester tester("non-existent", none, none);
163 ATF_REQUIRE_THROW_RE(engine::error, "Unknown interface non-existent",
164 tester.list(fs::path("does-not-matter")));
165 }
166
167
168 ATF_TEST_CASE_WITHOUT_HEAD(tester__list__tester_fails);
ATF_TEST_CASE_BODY(tester__list__tester_fails)169 ATF_TEST_CASE_BODY(tester__list__tester_fails)
170 {
171 create_mock_tester_exit(EXIT_FAILURE);
172 engine::tester tester("mock", none, none);
173 ATF_REQUIRE_THROW_RE(
174 engine::error,
175 "Tester did not exit cleanly:.*tester output.*tester error",
176 tester.list(fs::path("does-not-matter")));
177 }
178
179
180 ATF_TEST_CASE_WITHOUT_HEAD(tester__list__tester_crashes);
ATF_TEST_CASE_BODY(tester__list__tester_crashes)181 ATF_TEST_CASE_BODY(tester__list__tester_crashes)
182 {
183 create_mock_tester_signal(SIGKILL);
184 engine::tester tester("mock", none, none);
185 ATF_REQUIRE_THROW_RE(
186 engine::error,
187 "Tester did not exit cleanly:.*tester output.*tester error",
188 tester.list(fs::path("does-not-matter")));
189 }
190
191
192 ATF_TEST_CASE_WITHOUT_HEAD(tester__test__defaults);
ATF_TEST_CASE_BODY(tester__test__defaults)193 ATF_TEST_CASE_BODY(tester__test__defaults)
194 {
195 std::map< std::string, std::string > vars;
196
197 create_mock_tester_exit(EXIT_FAILURE);
198 engine::tester tester("mock", none, none);
199 tester.test(fs::path("/foo/bar"), "test-case", fs::path("/the/result/file"),
200 fs::path("tester.out"), fs::path("tester.err"), vars);
201
202 const std::string exp_output =
203 "Arg: test\n"
204 "Arg: /foo/bar\n"
205 "Arg: test-case\n"
206 "Arg: /the/result/file\n"
207 "tester output\n";
208 const std::string exp_error =
209 "tester error\n";
210 ATF_REQUIRE(atf::utils::compare_file("tester.out", exp_output));
211 ATF_REQUIRE(atf::utils::compare_file("tester.err", exp_error));
212 }
213
214
215 ATF_TEST_CASE_WITHOUT_HEAD(tester__test__explicit_common_args_and_vars);
ATF_TEST_CASE_BODY(tester__test__explicit_common_args_and_vars)216 ATF_TEST_CASE_BODY(tester__test__explicit_common_args_and_vars)
217 {
218 const passwd::user user("fake", 123, 456);
219 const datetime::delta timeout(15, 0);
220
221 std::map< std::string, std::string > vars;
222 vars["var1"] = "value1";
223 vars["variable-2"] = "value with spaces";
224
225 create_mock_tester_exit(EXIT_SUCCESS);
226 engine::tester tester("mock", utils::make_optional(user),
227 utils::make_optional(timeout));
228 tester.test(fs::path("/foo/bar"), "test-case", fs::path("/the/result/file"),
229 fs::path("tester.out"), fs::path("tester.err"), vars);
230
231 const std::string exp_output =
232 "Arg: -u123\n"
233 "Arg: -g456\n"
234 "Arg: -t15\n"
235 "Arg: test\n"
236 "Arg: -vvar1=value1\n"
237 "Arg: -vvariable-2=value with spaces\n"
238 "Arg: /foo/bar\n"
239 "Arg: test-case\n"
240 "Arg: /the/result/file\n"
241 "tester output\n";
242 const std::string exp_error =
243 "tester error\n";
244 ATF_REQUIRE(atf::utils::compare_file("tester.out", exp_output));
245 ATF_REQUIRE(atf::utils::compare_file("tester.err", exp_error));
246 }
247
248
249 ATF_TEST_CASE_WITHOUT_HEAD(tester__test__unknown_interface);
ATF_TEST_CASE_BODY(tester__test__unknown_interface)250 ATF_TEST_CASE_BODY(tester__test__unknown_interface)
251 {
252 const std::map< std::string, std::string > vars;
253
254 utils::setenv("KYUA_TESTERSDIR", ".");
255 engine::tester tester("non-existent", none, none);
256 ATF_REQUIRE_THROW_RE(engine::error, "Unknown interface non-existent",
257 tester.test(fs::path("foo"), "bar", fs::path("baz"),
258 fs::path("out"), fs::path("err"), vars));
259 }
260
261
262 ATF_TEST_CASE_WITHOUT_HEAD(tester__test__tester_fails);
ATF_TEST_CASE_BODY(tester__test__tester_fails)263 ATF_TEST_CASE_BODY(tester__test__tester_fails)
264 {
265 const std::map< std::string, std::string > vars;
266
267 create_mock_tester_exit(2);
268 engine::tester tester("mock", none, none);
269 ATF_REQUIRE_THROW_RE(
270 engine::error,
271 "Tester failed with code 2; this is a bug",
272 tester.test(fs::path("foo"), "bar", fs::path("baz"),
273 fs::path("out"), fs::path("err"), vars));
274 }
275
276
277 ATF_TEST_CASE_WITHOUT_HEAD(tester__test__tester_crashes);
ATF_TEST_CASE_BODY(tester__test__tester_crashes)278 ATF_TEST_CASE_BODY(tester__test__tester_crashes)
279 {
280 const std::map< std::string, std::string > vars;
281
282 create_mock_tester_signal(SIGKILL);
283 engine::tester tester("mock", none, none);
284 ATF_REQUIRE_THROW_RE(
285 engine::error,
286 F("Tester received signal %s; this is a bug") % SIGKILL,
287 tester.test(fs::path("foo"), "bar", fs::path("baz"),
288 fs::path("out"), fs::path("err"), vars));
289 }
290
291
292 ATF_TEST_CASE_WITHOUT_HEAD(tester_path__default);
ATF_TEST_CASE_BODY(tester_path__default)293 ATF_TEST_CASE_BODY(tester_path__default)
294 {
295 ATF_REQUIRE(atf::utils::file_exists(engine::tester_path("atf").str()));
296 ATF_REQUIRE(atf::utils::file_exists(engine::tester_path("plain").str()));
297 }
298
299
300 ATF_TEST_CASE_WITHOUT_HEAD(tester_path__custom);
ATF_TEST_CASE_BODY(tester_path__custom)301 ATF_TEST_CASE_BODY(tester_path__custom)
302 {
303 fs::mkdir(fs::path("testers"), 0755);
304 atf::utils::create_file("testers/kyua-mock-1-tester", "Not a binary");
305 atf::utils::create_file("testers/kyua-mock-2-tester", "Not a binary");
306 utils::setenv("KYUA_TESTERSDIR", (fs::current_path() / "testers").str());
307
308 const fs::path mock1 = engine::tester_path("mock-1");
309 ATF_REQUIRE(mock1.is_absolute());
310 ATF_REQUIRE(atf::utils::file_exists(mock1.str()));
311
312 const fs::path mock2 = engine::tester_path("mock-2");
313 ATF_REQUIRE(mock2.is_absolute());
314 ATF_REQUIRE(atf::utils::file_exists(mock2.str()));
315
316 ATF_REQUIRE_THROW_RE(engine::error, "Unknown interface mock-3",
317 engine::tester_path("mock-3"));
318 }
319
320
321 ATF_TEST_CASE_WITHOUT_HEAD(tester_path__cached);
ATF_TEST_CASE_BODY(tester_path__cached)322 ATF_TEST_CASE_BODY(tester_path__cached)
323 {
324 fs::mkdir(fs::path("testers"), 0755);
325 atf::utils::create_file("testers/kyua-mock-tester", "Not a binary");
326 utils::setenv("KYUA_TESTERSDIR", (fs::current_path() / "testers").str());
327
328 const fs::path mock = engine::tester_path("mock");
329 ATF_REQUIRE(atf::utils::file_exists(mock.str()));
330 ATF_REQUIRE(::unlink(mock.c_str()) != -1);
331 ATF_REQUIRE(!atf::utils::file_exists(mock.str()));
332 ATF_REQUIRE_EQ(mock, engine::tester_path("mock"));
333 }
334
335
336 ATF_TEST_CASE_WITHOUT_HEAD(tester_path__empty);
ATF_TEST_CASE_BODY(tester_path__empty)337 ATF_TEST_CASE_BODY(tester_path__empty)
338 {
339 fs::mkdir(fs::path("testers"), 0755);
340 atf::utils::create_file("testers/kyua--tester", "Not a binary");
341 utils::setenv("KYUA_TESTERSDIR", (fs::current_path() / "testers").str());
342
343 ATF_REQUIRE_THROW_RE(engine::error, "Unknown interface ",
344 engine::tester_path(""));
345 }
346
347
348 ATF_TEST_CASE_WITHOUT_HEAD(tester_path__missing);
ATF_TEST_CASE_BODY(tester_path__missing)349 ATF_TEST_CASE_BODY(tester_path__missing)
350 {
351 utils::setenv("KYUA_TESTERSDIR", fs::current_path().str());
352 ATF_REQUIRE_THROW_RE(engine::error, "Unknown interface plain",
353 engine::tester_path("plain"));
354 }
355
356
ATF_INIT_TEST_CASES(tcs)357 ATF_INIT_TEST_CASES(tcs)
358 {
359 ATF_ADD_TEST_CASE(tcs, tester__list__defaults);
360 ATF_ADD_TEST_CASE(tcs, tester__list__explicit_common_args);
361 ATF_ADD_TEST_CASE(tcs, tester__list__unknown_interface);
362 ATF_ADD_TEST_CASE(tcs, tester__list__tester_fails);
363 ATF_ADD_TEST_CASE(tcs, tester__list__tester_crashes);
364
365 ATF_ADD_TEST_CASE(tcs, tester__test__defaults);
366 ATF_ADD_TEST_CASE(tcs, tester__test__explicit_common_args_and_vars);
367 ATF_ADD_TEST_CASE(tcs, tester__test__unknown_interface);
368 ATF_ADD_TEST_CASE(tcs, tester__test__tester_fails);
369 ATF_ADD_TEST_CASE(tcs, tester__test__tester_crashes);
370
371 ATF_ADD_TEST_CASE(tcs, tester_path__default);
372 ATF_ADD_TEST_CASE(tcs, tester_path__custom);
373 ATF_ADD_TEST_CASE(tcs, tester_path__cached);
374 ATF_ADD_TEST_CASE(tcs, tester_path__empty);
375 ATF_ADD_TEST_CASE(tcs, tester_path__missing);
376 }
377