1 // Copyright 2011 Google Inc. 2 // All rights reserved. 3 // 4 // Redistribution and use in source and binary forms, with or without 5 // modification, are permitted provided that the following conditions are 6 // met: 7 // 8 // * Redistributions of source code must retain the above copyright 9 // notice, this list of conditions and the following disclaimer. 10 // * Redistributions in binary form must reproduce the above copyright 11 // notice, this list of conditions and the following disclaimer in the 12 // documentation and/or other materials provided with the distribution. 13 // * Neither the name of Google Inc. nor the names of its contributors 14 // may be used to endorse or promote products derived from this software 15 // without specific prior written permission. 16 // 17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 #include "cli/cmd_report.hpp" 30 31 #include <cstddef> 32 #include <cstdlib> 33 #include <fstream> 34 #include <map> 35 #include <vector> 36 37 #include "cli/common.ipp" 38 #include "engine/action.hpp" 39 #include "engine/context.hpp" 40 #include "engine/drivers/scan_action.hpp" 41 #include "engine/test_result.hpp" 42 #include "utils/cmdline/exceptions.hpp" 43 #include "utils/cmdline/parser.ipp" 44 #include "utils/defs.hpp" 45 #include "utils/format/macros.hpp" 46 #include "utils/optional.ipp" 47 48 namespace cmdline = utils::cmdline; 49 namespace config = utils::config; 50 namespace datetime = utils::datetime; 51 namespace fs = utils::fs; 52 namespace scan_action = engine::drivers::scan_action; 53 54 using cli::cmd_report; 55 using utils::optional; 56 57 58 namespace { 59 60 61 /// Collection of result types. 62 /// 63 /// This is a vector rather than a set because we want to respect the order in 64 /// which the user provided the types. 65 typedef std::vector< engine::test_result::result_type > result_types; 66 67 68 /// Converts a set of result type names to identifiers. 69 /// 70 /// \param names The collection of names to process; may be empty. 71 /// 72 /// \return The result type identifiers corresponding to the input names. 73 /// 74 /// \throw std::runtime_error If any name in the input names is invalid. 75 static result_types 76 parse_types(const std::vector< std::string >& names) 77 { 78 using engine::test_result; 79 typedef std::map< std::string, test_result::result_type > types_map; 80 types_map valid_types; 81 valid_types["broken"] = test_result::broken; 82 valid_types["failed"] = test_result::failed; 83 valid_types["passed"] = test_result::passed; 84 valid_types["skipped"] = test_result::skipped; 85 valid_types["xfail"] = test_result::expected_failure; 86 87 result_types types; 88 for (std::vector< std::string >::const_iterator iter = names.begin(); 89 iter != names.end(); ++iter) { 90 const types_map::const_iterator match = valid_types.find(*iter); 91 if (match == valid_types.end()) 92 throw std::runtime_error(F("Unknown result type '%s'") % *iter); 93 else 94 types.push_back((*match).second); 95 } 96 return types; 97 } 98 99 100 /// Generates a plain-text report intended to be printed to the console. 101 class console_hooks : public scan_action::base_hooks { 102 /// Indirection to print the output to the correct file stream. 103 cli::file_writer _writer; 104 105 /// Whether to include the runtime context in the output or not. 106 const bool _show_context; 107 108 /// Collection of result types to include in the report. 109 const result_types& _results_filters; 110 111 /// The action ID loaded. 112 int64_t _action_id; 113 114 /// The total run time of the tests. 115 datetime::delta _runtime; 116 117 /// Representation of a single result. 118 struct result_data { 119 /// The relative path to the test program. 120 fs::path binary_path; 121 122 /// The name of the test case. 123 std::string test_case_name; 124 125 /// The result of the test case. 126 engine::test_result result; 127 128 /// The duration of the test case execution. 129 datetime::delta duration; 130 131 /// Constructs a new results data. 132 /// 133 /// \param binary_path_ The relative path to the test program. 134 /// \param test_case_name_ The name of the test case. 135 /// \param result_ The result of the test case. 136 /// \param duration_ The duration of the test case execution. 137 result_data(const fs::path& binary_path_, 138 const std::string& test_case_name_, 139 const engine::test_result& result_, 140 const datetime::delta& duration_) : 141 binary_path(binary_path_), test_case_name(test_case_name_), 142 result(result_), duration(duration_) 143 { 144 } 145 }; 146 147 /// Results received, broken down by their type. 148 /// 149 /// Note that this may not include all results, as keeping the whole list in 150 /// memory may be too much. 151 std::map< engine::test_result::result_type, 152 std::vector< result_data > > _results; 153 154 /// Prints the execution context to the output. 155 /// 156 /// \param context The context to dump. 157 void 158 print_context(const engine::context& context) 159 { 160 _writer("===> Execution context"); 161 162 _writer(F("Current directory: %s") % context.cwd()); 163 const std::map< std::string, std::string >& env = context.env(); 164 if (env.empty()) 165 _writer("No environment variables recorded"); 166 else { 167 _writer("Environment variables:"); 168 for (std::map< std::string, std::string >::const_iterator 169 iter = env.begin(); iter != env.end(); iter++) { 170 _writer(F(" %s=%s") % (*iter).first % (*iter).second); 171 } 172 } 173 } 174 175 /// Counts how many results of a given type have been received. 176 std::size_t 177 count_results(const engine::test_result::result_type type) 178 { 179 const std::map< engine::test_result::result_type, 180 std::vector< result_data > >::const_iterator iter = 181 _results.find(type); 182 if (iter == _results.end()) 183 return 0; 184 else 185 return (*iter).second.size(); 186 } 187 188 /// Prints a set of results. 189 void 190 print_results(const engine::test_result::result_type type, 191 const char* title) 192 { 193 const std::map< engine::test_result::result_type, 194 std::vector< result_data > >::const_iterator iter2 = 195 _results.find(type); 196 if (iter2 == _results.end()) 197 return; 198 const std::vector< result_data >& all = (*iter2).second; 199 200 _writer(F("===> %s") % title); 201 for (std::vector< result_data >::const_iterator iter = all.begin(); 202 iter != all.end(); iter++) { 203 _writer(F("%s:%s -> %s [%s]") % (*iter).binary_path % 204 (*iter).test_case_name % 205 cli::format_result((*iter).result) % 206 cli::format_delta((*iter).duration)); 207 } 208 } 209 210 public: 211 /// Constructor for the hooks. 212 /// 213 /// \param ui_ The user interface object of the caller command. 214 /// \param outfile_ The file to which to send the output. 215 /// \param show_context_ Whether to include the runtime context in 216 /// the output or not. 217 /// \param results_filters_ The result types to include in the report. 218 /// Cannot be empty. 219 console_hooks(cmdline::ui* ui_, const fs::path& outfile_, 220 const bool show_context_, 221 const result_types& results_filters_) : 222 _writer(ui_, outfile_), 223 _show_context(show_context_), 224 _results_filters(results_filters_) 225 { 226 PRE(!results_filters_.empty()); 227 } 228 229 /// Callback executed when an action is found. 230 /// 231 /// \param action_id The identifier of the loaded action. 232 /// \param action The action loaded from the database. 233 void 234 got_action(const int64_t action_id, const engine::action& action) 235 { 236 _action_id = action_id; 237 if (_show_context) 238 print_context(action.runtime_context()); 239 } 240 241 /// Callback executed when a test results is found. 242 /// 243 /// \param iter Container for the test result's data. 244 void 245 got_result(store::results_iterator& iter) 246 { 247 _runtime += iter.duration(); 248 const engine::test_result result = iter.result(); 249 _results[result.type()].push_back( 250 result_data(iter.test_program()->relative_path(), 251 iter.test_case_name(), iter.result(), iter.duration())); 252 } 253 254 /// Prints the tests summary. 255 void 256 print_tests(void) 257 { 258 using engine::test_result; 259 typedef std::map< test_result::result_type, const char* > types_map; 260 261 types_map titles; 262 titles[engine::test_result::broken] = "Broken tests"; 263 titles[engine::test_result::expected_failure] = "Expected failures"; 264 titles[engine::test_result::failed] = "Failed tests"; 265 titles[engine::test_result::passed] = "Passed tests"; 266 titles[engine::test_result::skipped] = "Skipped tests"; 267 268 for (result_types::const_iterator iter = _results_filters.begin(); 269 iter != _results_filters.end(); ++iter) { 270 const types_map::const_iterator match = titles.find(*iter); 271 INV_MSG(match != titles.end(), "Conditional does not match user " 272 "input validation in parse_types()"); 273 print_results((*match).first, (*match).second); 274 } 275 276 const std::size_t broken = count_results(test_result::broken); 277 const std::size_t failed = count_results(test_result::failed); 278 const std::size_t passed = count_results(test_result::passed); 279 const std::size_t skipped = count_results(test_result::skipped); 280 const std::size_t xfail = count_results(test_result::expected_failure); 281 const std::size_t total = broken + failed + passed + skipped + xfail; 282 283 _writer("===> Summary"); 284 _writer(F("Action: %s") % _action_id); 285 _writer(F("Test cases: %s total, %s skipped, %s expected failures, " 286 "%s broken, %s failed") % 287 total % skipped % xfail % broken % failed); 288 _writer(F("Total time: %s") % cli::format_delta(_runtime)); 289 } 290 }; 291 292 293 } // anonymous namespace 294 295 296 const fs::path cli::file_writer::_stdout_path("/dev/stdout"); 297 const fs::path cli::file_writer::_stderr_path("/dev/stderr"); 298 299 300 /// Constructs a new file_writer wrapper. 301 /// 302 /// \param ui_ The UI object of the caller command. 303 /// \param path_ The path to the output file. 304 cli::file_writer::file_writer(cmdline::ui* const ui_, const fs::path& path_) : 305 _ui(ui_), _output_path(path_) 306 { 307 if (path_ != _stdout_path && path_ != _stderr_path) { 308 _output_file.reset(new std::ofstream(path_.c_str())); 309 if (!*(_output_file)) { 310 throw std::runtime_error(F("Cannot open output file %s") % path_); 311 } 312 } 313 } 314 315 /// Destructor. 316 cli::file_writer::~file_writer(void) 317 { 318 } 319 320 /// Writes a message to the selected output. 321 /// 322 /// \param message The message to write; should not include a termination 323 /// new line. 324 void 325 cli::file_writer::operator()(const std::string& message) 326 { 327 if (_output_path == _stdout_path) 328 _ui->out(message); 329 else if (_output_path == _stderr_path) 330 _ui->err(message); 331 else { 332 INV(_output_file.get() != NULL); 333 (*_output_file) << message << '\n'; 334 } 335 } 336 337 338 /// Default constructor for cmd_report. 339 cmd_report::cmd_report(void) : cli_command( 340 "report", "", 0, 0, 341 "Generates a user-friendly, plain-text report with the result of a " 342 "previous action") 343 { 344 add_option(store_option); 345 add_option(cmdline::bool_option( 346 "show-context", "Include the execution context in the report")); 347 add_option(cmdline::int_option( 348 "action", "The action to report; if not specified, defaults to the " 349 "latest action in the database", "id")); 350 add_option(cmdline::path_option( 351 "output", "The file to which to write the report", 352 "path", "/dev/stdout")); 353 add_option(cmdline::list_option( 354 "results-filter", "Comma-separated list of result types to include in " 355 "the report", "types", "skipped,xfail,broken,failed")); 356 } 357 358 359 /// Entry point for the "report" subcommand. 360 /// 361 /// \param ui Object to interact with the I/O of the program. 362 /// \param cmdline Representation of the command line to the subcommand. 363 /// \param unused_user_config The runtime configuration of the program. 364 /// 365 /// \return 0 if everything is OK, 1 if the statement is invalid or if there is 366 /// any other problem. 367 int 368 cmd_report::run(cmdline::ui* ui, const cmdline::parsed_cmdline& cmdline, 369 const config::tree& UTILS_UNUSED_PARAM(user_config)) 370 { 371 optional< int64_t > action_id; 372 if (cmdline.has_option("action")) 373 action_id = cmdline.get_option< cmdline::int_option >("action"); 374 375 result_types types = parse_types( 376 cmdline.get_option< cmdline::list_option >("results-filter")); 377 if (types.empty()) { 378 types.push_back(engine::test_result::passed); 379 types.push_back(engine::test_result::skipped); 380 types.push_back(engine::test_result::expected_failure); 381 types.push_back(engine::test_result::broken); 382 types.push_back(engine::test_result::failed); 383 } 384 385 console_hooks hooks( 386 ui, cmdline.get_option< cmdline::path_option >("output"), 387 cmdline.has_option("show-context"), types); 388 scan_action::drive(store_path(cmdline), action_id, hooks); 389 hooks.print_tests(); 390 391 return EXIT_SUCCESS; 392 } 393