1 // Copyright 2012 Google Inc. 2 // All rights reserved. 3 // 4 // Redistribution and use in source and binary forms, with or without 5 // modification, are permitted provided that the following conditions are 6 // met: 7 // 8 // * Redistributions of source code must retain the above copyright 9 // notice, this list of conditions and the following disclaimer. 10 // * Redistributions in binary form must reproduce the above copyright 11 // notice, this list of conditions and the following disclaimer in the 12 // documentation and/or other materials provided with the distribution. 13 // * Neither the name of Google Inc. nor the names of its contributors 14 // may be used to endorse or promote products derived from this software 15 // without specific prior written permission. 16 // 17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 #include "engine/testers.hpp" 30 31 extern "C" { 32 #include <sys/stat.h> 33 34 #include <signal.h> 35 } 36 37 #include <cstdlib> 38 39 #include <atf-c++.hpp> 40 41 #include "engine/exceptions.hpp" 42 #include "utils/datetime.hpp" 43 #include "utils/env.hpp" 44 #include "utils/format/macros.hpp" 45 #include "utils/fs/path.hpp" 46 #include "utils/fs/operations.hpp" 47 #include "utils/optional.ipp" 48 #include "utils/passwd.hpp" 49 50 namespace datetime = utils::datetime; 51 namespace fs = utils::fs; 52 namespace passwd = utils::passwd; 53 54 using utils::none; 55 using utils::optional; 56 57 58 namespace { 59 60 61 /// Creates a mock tester that exits cleanly. 62 /// 63 /// The interface accepted by the tester is 'mock'. This tester outputs the 64 /// arguments passed to it and then prints a message to both the stdout and the 65 /// stderr. 66 /// 67 /// \param exit_status Code to exit with. 68 static void 69 create_mock_tester_exit(const int exit_status) 70 { 71 atf::utils::create_file( 72 "kyua-mock-tester", 73 F("#! /bin/sh\n" 74 "while [ ${#} -gt 0 ]; do\n" 75 " echo \"Arg: ${1}\"\n" 76 " shift\n" 77 "done\n" 78 "echo 'tester output'\n" 79 "echo 'tester error' 1>&2\n" 80 "exit %s\n") % exit_status); 81 ATF_REQUIRE(::chmod("kyua-mock-tester", 0755) != -1); 82 83 utils::setenv("KYUA_TESTERSDIR", fs::current_path().str()); 84 } 85 86 87 /// Creates a mock tester that receives a signal. 88 /// 89 /// The interface accepted by the tester is 'mock'. This tester outputs the 90 /// arguments passed to it and then prints a message to both the stdout and the 91 /// stderr. 92 /// 93 /// \param term_sig Signal to deliver to the tester. If the tester does not 94 /// exit due to this reason, it exits with an arbitrary non-zero code. 95 static void 96 create_mock_tester_signal(const int term_sig) 97 { 98 atf::utils::create_file( 99 "kyua-mock-tester", 100 F("#! /bin/sh\n" 101 "while [ ${#} -gt 0 ]; do\n" 102 " echo \"Arg: ${1}\"\n" 103 " shift\n" 104 "done\n" 105 "echo 'tester output'\n" 106 "echo 'tester error' 1>&2\n" 107 "kill -%s $$\n" 108 "echo 'signal did not terminate the process\n" 109 "exit 0\n") % term_sig); 110 ATF_REQUIRE(::chmod("kyua-mock-tester", 0755) != -1); 111 112 utils::setenv("KYUA_TESTERSDIR", fs::current_path().str()); 113 } 114 115 116 } // anonymous namespace 117 118 119 ATF_TEST_CASE_WITHOUT_HEAD(tester__list__defaults); 120 ATF_TEST_CASE_BODY(tester__list__defaults) 121 { 122 create_mock_tester_exit(EXIT_SUCCESS); 123 engine::tester tester("mock", none, none); 124 const std::string output = tester.list(fs::path("/foo/bar")); 125 126 const std::string exp_output = 127 "Arg: list\n" 128 "Arg: /foo/bar\n" 129 "tester output\n" 130 "tester error\n"; 131 ATF_REQUIRE_EQ(exp_output, output); 132 } 133 134 135 ATF_TEST_CASE_WITHOUT_HEAD(tester__list__explicit_common_args); 136 ATF_TEST_CASE_BODY(tester__list__explicit_common_args) 137 { 138 const passwd::user user("fake", 123, 456); 139 const datetime::delta timeout(15, 0); 140 141 create_mock_tester_exit(EXIT_SUCCESS); 142 engine::tester tester("mock", utils::make_optional(user), 143 utils::make_optional(timeout)); 144 const std::string output = tester.list(fs::path("/another/program/1")); 145 146 const std::string exp_output = 147 "Arg: -u123\n" 148 "Arg: -g456\n" 149 "Arg: -t15\n" 150 "Arg: list\n" 151 "Arg: /another/program/1\n" 152 "tester output\n" 153 "tester error\n"; 154 ATF_REQUIRE_EQ(exp_output, output); 155 } 156 157 158 ATF_TEST_CASE_WITHOUT_HEAD(tester__list__unknown_interface); 159 ATF_TEST_CASE_BODY(tester__list__unknown_interface) 160 { 161 utils::setenv("KYUA_TESTERSDIR", "."); 162 engine::tester tester("non-existent", none, none); 163 ATF_REQUIRE_THROW_RE(engine::error, "Unknown interface non-existent", 164 tester.list(fs::path("does-not-matter"))); 165 } 166 167 168 ATF_TEST_CASE_WITHOUT_HEAD(tester__list__tester_fails); 169 ATF_TEST_CASE_BODY(tester__list__tester_fails) 170 { 171 create_mock_tester_exit(EXIT_FAILURE); 172 engine::tester tester("mock", none, none); 173 ATF_REQUIRE_THROW_RE( 174 engine::error, 175 "Tester did not exit cleanly:.*tester output.*tester error", 176 tester.list(fs::path("does-not-matter"))); 177 } 178 179 180 ATF_TEST_CASE_WITHOUT_HEAD(tester__list__tester_crashes); 181 ATF_TEST_CASE_BODY(tester__list__tester_crashes) 182 { 183 create_mock_tester_signal(SIGKILL); 184 engine::tester tester("mock", none, none); 185 ATF_REQUIRE_THROW_RE( 186 engine::error, 187 "Tester did not exit cleanly:.*tester output.*tester error", 188 tester.list(fs::path("does-not-matter"))); 189 } 190 191 192 ATF_TEST_CASE_WITHOUT_HEAD(tester__test__defaults); 193 ATF_TEST_CASE_BODY(tester__test__defaults) 194 { 195 std::map< std::string, std::string > vars; 196 197 create_mock_tester_exit(EXIT_FAILURE); 198 engine::tester tester("mock", none, none); 199 tester.test(fs::path("/foo/bar"), "test-case", fs::path("/the/result/file"), 200 fs::path("tester.out"), fs::path("tester.err"), vars); 201 202 const std::string exp_output = 203 "Arg: test\n" 204 "Arg: /foo/bar\n" 205 "Arg: test-case\n" 206 "Arg: /the/result/file\n" 207 "tester output\n"; 208 const std::string exp_error = 209 "tester error\n"; 210 ATF_REQUIRE(atf::utils::compare_file("tester.out", exp_output)); 211 ATF_REQUIRE(atf::utils::compare_file("tester.err", exp_error)); 212 } 213 214 215 ATF_TEST_CASE_WITHOUT_HEAD(tester__test__explicit_common_args_and_vars); 216 ATF_TEST_CASE_BODY(tester__test__explicit_common_args_and_vars) 217 { 218 const passwd::user user("fake", 123, 456); 219 const datetime::delta timeout(15, 0); 220 221 std::map< std::string, std::string > vars; 222 vars["var1"] = "value1"; 223 vars["variable-2"] = "value with spaces"; 224 225 create_mock_tester_exit(EXIT_SUCCESS); 226 engine::tester tester("mock", utils::make_optional(user), 227 utils::make_optional(timeout)); 228 tester.test(fs::path("/foo/bar"), "test-case", fs::path("/the/result/file"), 229 fs::path("tester.out"), fs::path("tester.err"), vars); 230 231 const std::string exp_output = 232 "Arg: -u123\n" 233 "Arg: -g456\n" 234 "Arg: -t15\n" 235 "Arg: test\n" 236 "Arg: -vvar1=value1\n" 237 "Arg: -vvariable-2=value with spaces\n" 238 "Arg: /foo/bar\n" 239 "Arg: test-case\n" 240 "Arg: /the/result/file\n" 241 "tester output\n"; 242 const std::string exp_error = 243 "tester error\n"; 244 ATF_REQUIRE(atf::utils::compare_file("tester.out", exp_output)); 245 ATF_REQUIRE(atf::utils::compare_file("tester.err", exp_error)); 246 } 247 248 249 ATF_TEST_CASE_WITHOUT_HEAD(tester__test__unknown_interface); 250 ATF_TEST_CASE_BODY(tester__test__unknown_interface) 251 { 252 const std::map< std::string, std::string > vars; 253 254 utils::setenv("KYUA_TESTERSDIR", "."); 255 engine::tester tester("non-existent", none, none); 256 ATF_REQUIRE_THROW_RE(engine::error, "Unknown interface non-existent", 257 tester.test(fs::path("foo"), "bar", fs::path("baz"), 258 fs::path("out"), fs::path("err"), vars)); 259 } 260 261 262 ATF_TEST_CASE_WITHOUT_HEAD(tester__test__tester_fails); 263 ATF_TEST_CASE_BODY(tester__test__tester_fails) 264 { 265 const std::map< std::string, std::string > vars; 266 267 create_mock_tester_exit(2); 268 engine::tester tester("mock", none, none); 269 ATF_REQUIRE_THROW_RE( 270 engine::error, 271 "Tester failed with code 2; this is a bug", 272 tester.test(fs::path("foo"), "bar", fs::path("baz"), 273 fs::path("out"), fs::path("err"), vars)); 274 } 275 276 277 ATF_TEST_CASE_WITHOUT_HEAD(tester__test__tester_crashes); 278 ATF_TEST_CASE_BODY(tester__test__tester_crashes) 279 { 280 const std::map< std::string, std::string > vars; 281 282 create_mock_tester_signal(SIGKILL); 283 engine::tester tester("mock", none, none); 284 ATF_REQUIRE_THROW_RE( 285 engine::error, 286 F("Tester received signal %s; this is a bug") % SIGKILL, 287 tester.test(fs::path("foo"), "bar", fs::path("baz"), 288 fs::path("out"), fs::path("err"), vars)); 289 } 290 291 292 ATF_TEST_CASE_WITHOUT_HEAD(tester_path__default); 293 ATF_TEST_CASE_BODY(tester_path__default) 294 { 295 ATF_REQUIRE(atf::utils::file_exists(engine::tester_path("atf").str())); 296 ATF_REQUIRE(atf::utils::file_exists(engine::tester_path("plain").str())); 297 } 298 299 300 ATF_TEST_CASE_WITHOUT_HEAD(tester_path__custom); 301 ATF_TEST_CASE_BODY(tester_path__custom) 302 { 303 fs::mkdir(fs::path("testers"), 0755); 304 atf::utils::create_file("testers/kyua-mock-1-tester", "Not a binary"); 305 atf::utils::create_file("testers/kyua-mock-2-tester", "Not a binary"); 306 utils::setenv("KYUA_TESTERSDIR", (fs::current_path() / "testers").str()); 307 308 const fs::path mock1 = engine::tester_path("mock-1"); 309 ATF_REQUIRE(mock1.is_absolute()); 310 ATF_REQUIRE(atf::utils::file_exists(mock1.str())); 311 312 const fs::path mock2 = engine::tester_path("mock-2"); 313 ATF_REQUIRE(mock2.is_absolute()); 314 ATF_REQUIRE(atf::utils::file_exists(mock2.str())); 315 316 ATF_REQUIRE_THROW_RE(engine::error, "Unknown interface mock-3", 317 engine::tester_path("mock-3")); 318 } 319 320 321 ATF_TEST_CASE_WITHOUT_HEAD(tester_path__cached); 322 ATF_TEST_CASE_BODY(tester_path__cached) 323 { 324 fs::mkdir(fs::path("testers"), 0755); 325 atf::utils::create_file("testers/kyua-mock-tester", "Not a binary"); 326 utils::setenv("KYUA_TESTERSDIR", (fs::current_path() / "testers").str()); 327 328 const fs::path mock = engine::tester_path("mock"); 329 ATF_REQUIRE(atf::utils::file_exists(mock.str())); 330 ATF_REQUIRE(::unlink(mock.c_str()) != -1); 331 ATF_REQUIRE(!atf::utils::file_exists(mock.str())); 332 ATF_REQUIRE_EQ(mock, engine::tester_path("mock")); 333 } 334 335 336 ATF_TEST_CASE_WITHOUT_HEAD(tester_path__empty); 337 ATF_TEST_CASE_BODY(tester_path__empty) 338 { 339 fs::mkdir(fs::path("testers"), 0755); 340 atf::utils::create_file("testers/kyua--tester", "Not a binary"); 341 utils::setenv("KYUA_TESTERSDIR", (fs::current_path() / "testers").str()); 342 343 ATF_REQUIRE_THROW_RE(engine::error, "Unknown interface ", 344 engine::tester_path("")); 345 } 346 347 348 ATF_TEST_CASE_WITHOUT_HEAD(tester_path__missing); 349 ATF_TEST_CASE_BODY(tester_path__missing) 350 { 351 utils::setenv("KYUA_TESTERSDIR", fs::current_path().str()); 352 ATF_REQUIRE_THROW_RE(engine::error, "Unknown interface plain", 353 engine::tester_path("plain")); 354 } 355 356 357 ATF_INIT_TEST_CASES(tcs) 358 { 359 ATF_ADD_TEST_CASE(tcs, tester__list__defaults); 360 ATF_ADD_TEST_CASE(tcs, tester__list__explicit_common_args); 361 ATF_ADD_TEST_CASE(tcs, tester__list__unknown_interface); 362 ATF_ADD_TEST_CASE(tcs, tester__list__tester_fails); 363 ATF_ADD_TEST_CASE(tcs, tester__list__tester_crashes); 364 365 ATF_ADD_TEST_CASE(tcs, tester__test__defaults); 366 ATF_ADD_TEST_CASE(tcs, tester__test__explicit_common_args_and_vars); 367 ATF_ADD_TEST_CASE(tcs, tester__test__unknown_interface); 368 ATF_ADD_TEST_CASE(tcs, tester__test__tester_fails); 369 ATF_ADD_TEST_CASE(tcs, tester__test__tester_crashes); 370 371 ATF_ADD_TEST_CASE(tcs, tester_path__default); 372 ATF_ADD_TEST_CASE(tcs, tester_path__custom); 373 ATF_ADD_TEST_CASE(tcs, tester_path__cached); 374 ATF_ADD_TEST_CASE(tcs, tester_path__empty); 375 ATF_ADD_TEST_CASE(tcs, tester_path__missing); 376 } 377