1 // Copyright 2012 Google Inc. 2 // All rights reserved. 3 // 4 // Redistribution and use in source and binary forms, with or without 5 // modification, are permitted provided that the following conditions are 6 // met: 7 // 8 // * Redistributions of source code must retain the above copyright 9 // notice, this list of conditions and the following disclaimer. 10 // * Redistributions in binary form must reproduce the above copyright 11 // notice, this list of conditions and the following disclaimer in the 12 // documentation and/or other materials provided with the distribution. 13 // * Neither the name of Google Inc. nor the names of its contributors 14 // may be used to endorse or promote products derived from this software 15 // without specific prior written permission. 16 // 17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 #include "cli.h" 30 31 #include <assert.h> 32 #include <stdlib.h> 33 #include <stdio.h> 34 #include <string.h> 35 36 #include <atf-c.h> 37 38 #include "defs.h" 39 #include "error.h" 40 #include "run.h" 41 42 43 /// Dumps the contents of a run_params object to stdout. 44 /// 45 /// We only print the settings that are relevant for testing purposes. 46 /// 47 /// \param run_params The run parameters to be printed. 48 static void 49 dump_run_params(const kyua_run_params_t* run_params) 50 { 51 printf("timeout_seconds: %lu\n", run_params->timeout_seconds); 52 53 if (run_params->unprivileged_user == getuid()) 54 printf("unprivileged_user: self\n"); 55 else 56 printf("unprivileged_user: %ld\n", (long)run_params->unprivileged_user); 57 58 if (run_params->unprivileged_group == getgid()) 59 printf("unprivileged_group: self\n"); 60 else 61 printf("unprivileged_group: %ld\n", 62 (long)run_params->unprivileged_group); 63 } 64 65 66 /// Helper to validate argument passing to the list_test_cases method. 67 /// 68 /// This prints the value of all arguments to stdout so that the caller can 69 /// compare the printed output to the expected values. 70 /// 71 /// \param test_program Test program path. 72 /// \param run_params Execution parameters to configure the test process. 73 /// 74 /// \return An error if the test_program is set to the magic keyword 'error'; OK 75 /// otherwise. 76 static kyua_error_t 77 list_test_cases(const char* test_program, const kyua_run_params_t* run_params) 78 { 79 if (strcmp(test_program, "error") == 0) 80 return kyua_oom_error_new(); 81 else { 82 printf("test_program: %s\n", test_program); 83 dump_run_params(run_params); 84 return kyua_error_ok(); 85 } 86 } 87 88 89 /// Helper to validate argument passing to the run_test_cases method. 90 /// 91 /// This prints the value of all arguments to stdout so that the caller can 92 /// compare the printed output to the expected values. 93 /// 94 /// \param test_program Test program path. 95 /// \param test_case Test case name. 96 /// \param result_file Path to the result file. 97 /// \param user_variables User configuration variables. 98 /// \param run_params Execution parameters to configure the test process. 99 /// \param [out] success Whether the test case returned with a successful result 100 /// or not. Set to true if result_file is the magic word 'pass'. 101 /// 102 /// \return An error if the test_program is set to the magic keyword 'error'; OK 103 /// otherwise. 104 static kyua_error_t 105 run_test_case(const char* test_program, const char* test_case, 106 const char* result_file, const char* const user_variables[], 107 const kyua_run_params_t* run_params, bool* success) 108 { 109 if (strcmp(test_program, "error") == 0) 110 return kyua_oom_error_new(); 111 else { 112 printf("test_program: %s\n", test_program); 113 printf("test_case: %s\n", test_case); 114 printf("result_file: %s\n", result_file); 115 const char* const* iter; 116 for (iter = user_variables; *iter != NULL; ++iter) 117 printf("variable: %s\n", *iter); 118 dump_run_params(run_params); 119 *success = strcmp(result_file, "pass") == 0; 120 return kyua_error_ok(); 121 } 122 } 123 124 125 /// Definition of a mock tester. 126 static kyua_cli_tester_t mock_tester = { 127 .list_test_cases = list_test_cases, 128 .run_test_case = run_test_case, 129 }; 130 131 132 /// Definition of a tester with invalid values. 133 /// 134 /// This is to be used when the called code is not supposed to invoke any of the 135 /// tester methods. 136 static kyua_cli_tester_t unused_tester = { 137 .list_test_cases = NULL, 138 .run_test_case = NULL, 139 }; 140 141 142 /// Counts the number of arguments in an argv vector. 143 /// 144 /// \param argv The NULL-terminated arguments vector to be passed to the 145 /// kyua_cli_main function. 146 /// 147 /// \return The number of arguments in argv. 148 static int 149 count_argv(char* const* const argv) 150 { 151 int argc = 0; 152 char* const* arg; 153 for (arg = argv; *arg != NULL; arg++) 154 argc++; 155 return argc; 156 } 157 158 159 ATF_TC_WITHOUT_HEAD(main__unknown_option); 160 ATF_TC_BODY(main__unknown_option, tc) 161 { 162 const pid_t pid = atf_utils_fork(); 163 if (pid == 0) { 164 char arg0[] = "unused-progname"; 165 char arg1[] = "-Z"; 166 char* const argv[] = {arg0, arg1, NULL}; 167 exit(kyua_cli_main(count_argv(argv), argv, &unused_tester)); 168 } 169 atf_utils_wait(pid, EXIT_USAGE_ERROR, "", "cli_test: Unknown option -Z\n"); 170 } 171 172 173 ATF_TC_WITHOUT_HEAD(main__missing_option_argument); 174 ATF_TC_BODY(main__missing_option_argument, tc) 175 { 176 const pid_t pid = atf_utils_fork(); 177 if (pid == 0) { 178 char arg0[] = "unused-progname"; 179 char arg1[] = "-t"; 180 char* const argv[] = {arg0, arg1, NULL}; 181 exit(kyua_cli_main(count_argv(argv), argv, &unused_tester)); 182 } 183 atf_utils_wait(pid, EXIT_USAGE_ERROR, "", "cli_test: -t requires an " 184 "argument\n"); 185 } 186 187 188 ATF_TC_WITHOUT_HEAD(main__unknown_command); 189 ATF_TC_BODY(main__unknown_command, tc) 190 { 191 const pid_t pid = atf_utils_fork(); 192 if (pid == 0) { 193 char arg0[] = "unused-progname"; 194 char arg1[] = "foobar"; 195 char* const argv[] = {arg0, arg1, NULL}; 196 exit(kyua_cli_main(count_argv(argv), argv, &unused_tester)); 197 } 198 atf_utils_wait(pid, EXIT_USAGE_ERROR, "", "cli_test: Unknown command " 199 "'foobar'\n"); 200 } 201 202 203 ATF_TC_WITHOUT_HEAD(main__missing_command); 204 ATF_TC_BODY(main__missing_command, tc) 205 { 206 const pid_t pid = atf_utils_fork(); 207 if (pid == 0) { 208 char arg0[] = "unused-progname"; 209 char* const argv[] = {arg0, NULL}; 210 exit(kyua_cli_main(count_argv(argv), argv, &unused_tester)); 211 } 212 atf_utils_wait(pid, EXIT_USAGE_ERROR, "", "cli_test: Must provide a " 213 "command\n"); 214 } 215 216 217 /// Checks that a textual argument to a numerical flag raises an error. 218 /// 219 /// \param flag The generic flag to test. 220 /// \param error_message The expected error message when the flag is invalid. 221 static void 222 check_flag_not_a_number(const char flag, const char *error_message) 223 { 224 const pid_t pid = atf_utils_fork(); 225 if (pid == 0) { 226 char arg0[] = "unused-progname"; 227 char arg1[] = "-?foo"; 228 arg1[1] = flag; 229 char* const argv[] = {arg0, arg1, NULL}; 230 exit(kyua_cli_main(count_argv(argv), argv, &unused_tester)); 231 } 232 char experr[256]; 233 snprintf(experr, sizeof(experr), "cli_test: %s 'foo' (not a number)\n", 234 error_message); 235 atf_utils_wait(pid, EXIT_USAGE_ERROR, "", experr); 236 } 237 238 239 /// Checks that an out of range value to a numerical flag raises an error. 240 /// 241 /// \param flag The generic flag to test. 242 /// \param error_message The expected error message when the flag is invalid. 243 static void 244 check_flag_out_of_range(const char flag, const char *error_message) 245 { 246 const pid_t pid = atf_utils_fork(); 247 if (pid == 0) { 248 char arg0[] = "unused-progname"; 249 char arg1[] = "-?99999999999999999999"; 250 arg1[1] = flag; 251 char* const argv[] = {arg0, arg1, NULL}; 252 exit(kyua_cli_main(count_argv(argv), argv, &unused_tester)); 253 } 254 char experr[256]; 255 snprintf(experr, sizeof(experr), "cli_test: %s '99999999999999999999' " 256 "(out of range)\n", error_message); 257 atf_utils_wait(pid, EXIT_USAGE_ERROR, "", experr); 258 } 259 260 261 ATF_TC_WITHOUT_HEAD(main__gflag__not_a_number); 262 ATF_TC_BODY(main__gflag__not_a_number, tc) 263 { 264 check_flag_not_a_number('g', "Invalid GID"); 265 } 266 267 268 ATF_TC_WITHOUT_HEAD(main__gflag__out_of_range); 269 ATF_TC_BODY(main__gflag__out_of_range, tc) 270 { 271 check_flag_out_of_range('g', "Invalid GID"); 272 } 273 274 275 ATF_TC_WITHOUT_HEAD(main__tflag__not_a_number); 276 ATF_TC_BODY(main__tflag__not_a_number, tc) 277 { 278 check_flag_not_a_number('t', "Invalid timeout value"); 279 } 280 281 282 ATF_TC_WITHOUT_HEAD(main__tflag__out_of_range); 283 ATF_TC_BODY(main__tflag__out_of_range, tc) 284 { 285 check_flag_out_of_range('t', "Invalid timeout value"); 286 } 287 288 289 ATF_TC_WITHOUT_HEAD(main__uflag__not_a_number); 290 ATF_TC_BODY(main__uflag__not_a_number, tc) 291 { 292 check_flag_not_a_number('u', "Invalid UID"); 293 } 294 295 296 ATF_TC_WITHOUT_HEAD(main__uflag__out_of_range); 297 ATF_TC_BODY(main__uflag__out_of_range, tc) 298 { 299 check_flag_out_of_range('u', "Invalid UID"); 300 } 301 302 303 ATF_TC_WITHOUT_HEAD(list__ok); 304 ATF_TC_BODY(list__ok, tc) 305 { 306 const pid_t pid = atf_utils_fork(); 307 if (pid == 0) { 308 char arg0[] = "unused-progname"; 309 char arg1[] = "list"; 310 char arg2[] = "the-program"; 311 char* const argv[] = {arg0, arg1, arg2, NULL}; 312 exit(kyua_cli_main(count_argv(argv), argv, &mock_tester)); 313 } 314 atf_utils_wait(pid, EXIT_SUCCESS, 315 "test_program: the-program\n" 316 "timeout_seconds: 60\n" 317 "unprivileged_user: self\n" 318 "unprivileged_group: self\n", 319 ""); 320 } 321 322 323 ATF_TC_WITHOUT_HEAD(list__custom_run_params); 324 ATF_TC_BODY(list__custom_run_params, tc) 325 { 326 const pid_t pid = atf_utils_fork(); 327 if (pid == 0) { 328 char arg0[] = "unused-progname"; 329 char arg1[] = "-g987"; 330 char arg2[] = "-t123"; 331 char arg3[] = "-u45"; 332 char arg4[] = "list"; 333 char arg5[] = "the-program"; 334 char* const argv[] = {arg0, arg1, arg2, arg3, arg4, arg5, NULL}; 335 exit(kyua_cli_main(count_argv(argv), argv, &mock_tester)); 336 } 337 atf_utils_wait(pid, EXIT_SUCCESS, 338 "test_program: the-program\n" 339 "timeout_seconds: 123\n" 340 "unprivileged_user: 45\n" 341 "unprivileged_group: 987\n", 342 ""); 343 } 344 345 346 ATF_TC_WITHOUT_HEAD(list__error); 347 ATF_TC_BODY(list__error, tc) 348 { 349 const pid_t pid = atf_utils_fork(); 350 if (pid == 0) { 351 char arg0[] = "unused-progname"; 352 char arg1[] = "list"; 353 char arg2[] = "error"; 354 char* const argv[] = {arg0, arg1, arg2, NULL}; 355 exit(kyua_cli_main(count_argv(argv), argv, &mock_tester)); 356 } 357 atf_utils_wait(pid, EXIT_INTERNAL_ERROR, "", "cli_test: Not enough " 358 "memory\n"); 359 } 360 361 362 ATF_TC_WITHOUT_HEAD(list__missing_arguments); 363 ATF_TC_BODY(list__missing_arguments, tc) 364 { 365 const pid_t pid = atf_utils_fork(); 366 if (pid == 0) { 367 char arg0[] = "unused-progname"; 368 char arg1[] = "list"; 369 char* const argv[] = {arg0, arg1, NULL}; 370 exit(kyua_cli_main(count_argv(argv), argv, &unused_tester)); 371 } 372 atf_utils_wait(pid, EXIT_USAGE_ERROR, "", "cli_test: No test program " 373 "provided\n"); 374 } 375 376 377 ATF_TC_WITHOUT_HEAD(list__too_many_arguments); 378 ATF_TC_BODY(list__too_many_arguments, tc) 379 { 380 const pid_t pid = atf_utils_fork(); 381 if (pid == 0) { 382 char arg0[] = "unused-progname"; 383 char arg1[] = "list"; 384 char arg2[] = "first"; 385 char arg3[] = "second"; 386 char* const argv[] = {arg0, arg1, arg2, arg3, NULL}; 387 exit(kyua_cli_main(count_argv(argv), argv, &unused_tester)); 388 } 389 atf_utils_wait(pid, EXIT_USAGE_ERROR, "", "cli_test: Only one test program " 390 "allowed\n"); 391 } 392 393 394 ATF_TC_WITHOUT_HEAD(test__ok__pass); 395 ATF_TC_BODY(test__ok__pass, tc) 396 { 397 const pid_t pid = atf_utils_fork(); 398 if (pid == 0) { 399 char arg0[] = "unused-progname"; 400 char arg1[] = "test"; 401 char arg2[] = "the-program"; 402 char arg3[] = "the-test-case"; 403 char arg4[] = "pass"; 404 char* const argv[] = {arg0, arg1, arg2, arg3, arg4, NULL}; 405 exit(kyua_cli_main(count_argv(argv), argv, &mock_tester)); 406 } 407 atf_utils_wait(pid, EXIT_SUCCESS, 408 "test_program: the-program\n" 409 "test_case: the-test-case\n" 410 "result_file: pass\n" 411 "timeout_seconds: 60\n" 412 "unprivileged_user: self\n" 413 "unprivileged_group: self\n", 414 ""); 415 } 416 417 418 ATF_TC_WITHOUT_HEAD(test__ok__fail); 419 ATF_TC_BODY(test__ok__fail, tc) 420 { 421 const pid_t pid = atf_utils_fork(); 422 if (pid == 0) { 423 char arg0[] = "unused-progname"; 424 char arg1[] = "test"; 425 char arg2[] = "the-program"; 426 char arg3[] = "the-test-case"; 427 char arg4[] = "fail"; 428 char* const argv[] = {arg0, arg1, arg2, arg3, arg4, NULL}; 429 exit(kyua_cli_main(count_argv(argv), argv, &mock_tester)); 430 } 431 atf_utils_wait(pid, EXIT_FAILURE, 432 "test_program: the-program\n" 433 "test_case: the-test-case\n" 434 "result_file: fail\n" 435 "timeout_seconds: 60\n" 436 "unprivileged_user: self\n" 437 "unprivileged_group: self\n", 438 ""); 439 } 440 441 442 ATF_TC_WITHOUT_HEAD(test__custom_run_params); 443 ATF_TC_BODY(test__custom_run_params, tc) 444 { 445 const pid_t pid = atf_utils_fork(); 446 if (pid == 0) { 447 char arg0[] = "unused-progname"; 448 char arg1[] = "-g987"; 449 char arg2[] = "-t123"; 450 char arg3[] = "-u45"; 451 char arg4[] = "test"; 452 char arg5[] = "the-program"; 453 char arg6[] = "the-test-case"; 454 char arg7[] = "pass"; 455 char* const argv[] = {arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, 456 NULL}; 457 exit(kyua_cli_main(count_argv(argv), argv, &mock_tester)); 458 } 459 atf_utils_wait(pid, EXIT_SUCCESS, 460 "test_program: the-program\n" 461 "test_case: the-test-case\n" 462 "result_file: pass\n" 463 "timeout_seconds: 123\n" 464 "unprivileged_user: 45\n" 465 "unprivileged_group: 987\n", 466 ""); 467 } 468 469 470 ATF_TC_WITHOUT_HEAD(test__config_variables); 471 ATF_TC_BODY(test__config_variables, tc) 472 { 473 const pid_t pid = atf_utils_fork(); 474 if (pid == 0) { 475 char arg0[] = "unused-progname"; 476 char arg1[] = "test"; 477 char arg2[] = "-vfoo=bar"; 478 char arg3[] = "-va=c"; 479 char arg4[] = "the-program"; 480 char arg5[] = "the-test-case"; 481 char arg6[] = "pass"; 482 char* const argv[] = {arg0, arg1, arg2, arg3, arg4, arg5, arg6, NULL}; 483 exit(kyua_cli_main(count_argv(argv), argv, &mock_tester)); 484 } 485 atf_utils_wait(pid, EXIT_SUCCESS, 486 "test_program: the-program\n" 487 "test_case: the-test-case\n" 488 "result_file: pass\n" 489 "variable: foo=bar\n" 490 "variable: a=c\n" 491 "timeout_seconds: 60\n" 492 "unprivileged_user: self\n" 493 "unprivileged_group: self\n", 494 ""); 495 } 496 497 498 ATF_TC_WITHOUT_HEAD(test__error); 499 ATF_TC_BODY(test__error, tc) 500 { 501 const pid_t pid = atf_utils_fork(); 502 if (pid == 0) { 503 char arg0[] = "unused-progname"; 504 char arg1[] = "test"; 505 char arg2[] = "error"; 506 char* const argv[] = {arg0, arg1, arg2, arg2, arg2, NULL}; 507 exit(kyua_cli_main(count_argv(argv), argv, &mock_tester)); 508 } 509 atf_utils_wait(pid, EXIT_INTERNAL_ERROR, "", "cli_test: Not enough " 510 "memory\n"); 511 } 512 513 514 /// Checks that the test command validates the right number of arguments. 515 /// 516 /// \param count Number of arguments to pass to the test command. 517 static void 518 check_test_invalid_arguments(const unsigned int count) 519 { 520 printf("Checking with %d arguments\n", count); 521 const pid_t pid = atf_utils_fork(); 522 if (pid == 0) { 523 char arg0[] = "unused-progname"; 524 char arg1[] = "test"; 525 char argX[] = "arg"; 526 assert(count <= 4); 527 char* argv[] = {arg0, arg1, argX, argX, argX, argX, NULL}; 528 argv[2 + count] = NULL; 529 exit(kyua_cli_main(2 + count, argv, &unused_tester)); 530 } 531 atf_utils_wait(pid, EXIT_USAGE_ERROR, "", "cli_test: Must provide a test " 532 "program, a test case name and a result file\n"); 533 } 534 535 536 ATF_TC_WITHOUT_HEAD(test__invalid_arguments); 537 ATF_TC_BODY(test__invalid_arguments, tc) 538 { 539 check_test_invalid_arguments(0); 540 check_test_invalid_arguments(1); 541 check_test_invalid_arguments(2); 542 check_test_invalid_arguments(4); 543 } 544 545 546 ATF_TC_WITHOUT_HEAD(test__unknown_option); 547 ATF_TC_BODY(test__unknown_option, tc) 548 { 549 const pid_t pid = atf_utils_fork(); 550 if (pid == 0) { 551 char arg0[] = "unused-progname"; 552 char arg1[] = "test"; 553 char arg2[] = "-Z"; 554 char* const argv[] = {arg0, arg1, arg2, NULL}; 555 exit(kyua_cli_main(count_argv(argv), argv, &unused_tester)); 556 } 557 atf_utils_wait(pid, EXIT_USAGE_ERROR, "", "cli_test: Unknown test option " 558 "-Z\n"); 559 } 560 561 562 ATF_TC_WITHOUT_HEAD(test__missing_option_argument); 563 ATF_TC_BODY(test__missing_option_argument, tc) 564 { 565 const pid_t pid = atf_utils_fork(); 566 if (pid == 0) { 567 char arg0[] = "unused-progname"; 568 char arg1[] = "test"; 569 char arg2[] = "-v"; 570 char* const argv[] = {arg0, arg1, arg2, NULL}; 571 exit(kyua_cli_main(count_argv(argv), argv, &unused_tester)); 572 } 573 atf_utils_wait(pid, EXIT_USAGE_ERROR, "", "cli_test: test's -v requires an " 574 "argument\n"); 575 } 576 577 578 ATF_TP_ADD_TCS(tp) 579 { 580 ATF_TP_ADD_TC(tp, main__unknown_option); 581 ATF_TP_ADD_TC(tp, main__missing_option_argument); 582 ATF_TP_ADD_TC(tp, main__unknown_command); 583 ATF_TP_ADD_TC(tp, main__missing_command); 584 ATF_TP_ADD_TC(tp, main__gflag__not_a_number); 585 ATF_TP_ADD_TC(tp, main__gflag__out_of_range); 586 ATF_TP_ADD_TC(tp, main__tflag__not_a_number); 587 ATF_TP_ADD_TC(tp, main__tflag__out_of_range); 588 ATF_TP_ADD_TC(tp, main__uflag__not_a_number); 589 ATF_TP_ADD_TC(tp, main__uflag__out_of_range); 590 591 ATF_TP_ADD_TC(tp, list__ok); 592 ATF_TP_ADD_TC(tp, list__custom_run_params); 593 ATF_TP_ADD_TC(tp, list__error); 594 ATF_TP_ADD_TC(tp, list__missing_arguments); 595 ATF_TP_ADD_TC(tp, list__too_many_arguments); 596 597 ATF_TP_ADD_TC(tp, test__ok__pass); 598 ATF_TP_ADD_TC(tp, test__ok__fail); 599 ATF_TP_ADD_TC(tp, test__custom_run_params); 600 ATF_TP_ADD_TC(tp, test__config_variables); 601 ATF_TP_ADD_TC(tp, test__error); 602 ATF_TP_ADD_TC(tp, test__invalid_arguments); 603 ATF_TP_ADD_TC(tp, test__unknown_option); 604 ATF_TP_ADD_TC(tp, test__missing_option_argument); 605 606 return atf_no_error(); 607 } 608