xref: /netbsd-src/external/bsd/kyua-cli/dist/cli/cmd_report.cpp (revision f39f9c9b2b3d39fa4e71f38ebea4c5d12192a641)
1 // Copyright 2011 Google Inc.
2 // All rights reserved.
3 //
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are
6 // met:
7 //
8 // * Redistributions of source code must retain the above copyright
9 //   notice, this list of conditions and the following disclaimer.
10 // * Redistributions in binary form must reproduce the above copyright
11 //   notice, this list of conditions and the following disclaimer in the
12 //   documentation and/or other materials provided with the distribution.
13 // * Neither the name of Google Inc. nor the names of its contributors
14 //   may be used to endorse or promote products derived from this software
15 //   without specific prior written permission.
16 //
17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 
29 #include "cli/cmd_report.hpp"
30 
31 #include <cstddef>
32 #include <cstdlib>
33 #include <fstream>
34 #include <map>
35 #include <vector>
36 
37 #include "cli/common.ipp"
38 #include "engine/action.hpp"
39 #include "engine/context.hpp"
40 #include "engine/drivers/scan_action.hpp"
41 #include "engine/test_result.hpp"
42 #include "utils/cmdline/exceptions.hpp"
43 #include "utils/cmdline/parser.ipp"
44 #include "utils/defs.hpp"
45 #include "utils/format/macros.hpp"
46 #include "utils/optional.ipp"
47 
48 namespace cmdline = utils::cmdline;
49 namespace config = utils::config;
50 namespace datetime = utils::datetime;
51 namespace fs = utils::fs;
52 namespace scan_action = engine::drivers::scan_action;
53 
54 using cli::cmd_report;
55 using utils::optional;
56 
57 
58 namespace {
59 
60 
61 /// Generates a plain-text report intended to be printed to the console.
62 class console_hooks : public scan_action::base_hooks {
63     /// Indirection to print the output to the correct file stream.
64     cli::file_writer _writer;
65 
66     /// Whether to include the runtime context in the output or not.
67     const bool _show_context;
68 
69     /// Collection of result types to include in the report.
70     const cli::result_types& _results_filters;
71 
72     /// The action ID loaded.
73     int64_t _action_id;
74 
75     /// The total run time of the tests.
76     datetime::delta _runtime;
77 
78     /// Representation of a single result.
79     struct result_data {
80         /// The relative path to the test program.
81         fs::path binary_path;
82 
83         /// The name of the test case.
84         std::string test_case_name;
85 
86         /// The result of the test case.
87         engine::test_result result;
88 
89         /// The duration of the test case execution.
90         datetime::delta duration;
91 
92         /// Constructs a new results data.
93         ///
94         /// \param binary_path_ The relative path to the test program.
95         /// \param test_case_name_ The name of the test case.
96         /// \param result_ The result of the test case.
97         /// \param duration_ The duration of the test case execution.
result_data__anon3c141dea0111::console_hooks::result_data98         result_data(const fs::path& binary_path_,
99                     const std::string& test_case_name_,
100                     const engine::test_result& result_,
101                     const datetime::delta& duration_) :
102             binary_path(binary_path_), test_case_name(test_case_name_),
103             result(result_), duration(duration_)
104         {
105         }
106     };
107 
108     /// Results received, broken down by their type.
109     ///
110     /// Note that this may not include all results, as keeping the whole list in
111     /// memory may be too much.
112     std::map< engine::test_result::result_type,
113               std::vector< result_data > > _results;
114 
115     /// Prints the execution context to the output.
116     ///
117     /// \param context The context to dump.
118     void
print_context(const engine::context & context)119     print_context(const engine::context& context)
120     {
121         _writer("===> Execution context");
122 
123         _writer(F("Current directory: %s") % context.cwd());
124         const std::map< std::string, std::string >& env = context.env();
125         if (env.empty())
126             _writer("No environment variables recorded");
127         else {
128             _writer("Environment variables:");
129             for (std::map< std::string, std::string >::const_iterator
130                      iter = env.begin(); iter != env.end(); iter++) {
131                 _writer(F("    %s=%s") % (*iter).first % (*iter).second);
132             }
133         }
134     }
135 
136     /// Counts how many results of a given type have been received.
137     std::size_t
count_results(const engine::test_result::result_type type)138     count_results(const engine::test_result::result_type type)
139     {
140         const std::map< engine::test_result::result_type,
141                         std::vector< result_data > >::const_iterator iter =
142             _results.find(type);
143         if (iter == _results.end())
144             return 0;
145         else
146             return (*iter).second.size();
147     }
148 
149     /// Prints a set of results.
150     void
print_results(const engine::test_result::result_type type,const char * title)151     print_results(const engine::test_result::result_type type,
152                   const char* title)
153     {
154         const std::map< engine::test_result::result_type,
155                         std::vector< result_data > >::const_iterator iter2 =
156             _results.find(type);
157         if (iter2 == _results.end())
158             return;
159         const std::vector< result_data >& all = (*iter2).second;
160 
161         _writer(F("===> %s") % title);
162         for (std::vector< result_data >::const_iterator iter = all.begin();
163              iter != all.end(); iter++) {
164             _writer(F("%s:%s  ->  %s  [%s]") % (*iter).binary_path %
165                     (*iter).test_case_name %
166                     cli::format_result((*iter).result) %
167                     cli::format_delta((*iter).duration));
168         }
169     }
170 
171 public:
172     /// Constructor for the hooks.
173     ///
174     /// \param ui_ The user interface object of the caller command.
175     /// \param outfile_ The file to which to send the output.
176     /// \param show_context_ Whether to include the runtime context in
177     ///     the output or not.
178     /// \param results_filters_ The result types to include in the report.
179     ///     Cannot be empty.
console_hooks(cmdline::ui * ui_,const fs::path & outfile_,const bool show_context_,const cli::result_types & results_filters_)180     console_hooks(cmdline::ui* ui_, const fs::path& outfile_,
181                   const bool show_context_,
182                   const cli::result_types& results_filters_) :
183         _writer(ui_, outfile_),
184         _show_context(show_context_),
185         _results_filters(results_filters_)
186     {
187         PRE(!results_filters_.empty());
188     }
189 
190     /// Callback executed when an action is found.
191     ///
192     /// \param action_id The identifier of the loaded action.
193     /// \param action The action loaded from the database.
194     void
got_action(const int64_t action_id,const engine::action & action)195     got_action(const int64_t action_id, const engine::action& action)
196     {
197         _action_id = action_id;
198         if (_show_context)
199             print_context(action.runtime_context());
200     }
201 
202     /// Callback executed when a test results is found.
203     ///
204     /// \param iter Container for the test result's data.
205     void
got_result(store::results_iterator & iter)206     got_result(store::results_iterator& iter)
207     {
208         _runtime += iter.duration();
209         const engine::test_result result = iter.result();
210         _results[result.type()].push_back(
211             result_data(iter.test_program()->relative_path(),
212                         iter.test_case_name(), iter.result(), iter.duration()));
213     }
214 
215     /// Prints the tests summary.
216     void
print_tests(void)217     print_tests(void)
218     {
219         using engine::test_result;
220         typedef std::map< test_result::result_type, const char* > types_map;
221 
222         types_map titles;
223         titles[engine::test_result::broken] = "Broken tests";
224         titles[engine::test_result::expected_failure] = "Expected failures";
225         titles[engine::test_result::failed] = "Failed tests";
226         titles[engine::test_result::passed] = "Passed tests";
227         titles[engine::test_result::skipped] = "Skipped tests";
228 
229         for (cli::result_types::const_iterator iter = _results_filters.begin();
230              iter != _results_filters.end(); ++iter) {
231             const types_map::const_iterator match = titles.find(*iter);
232             INV_MSG(match != titles.end(), "Conditional does not match user "
233                     "input validation in parse_types()");
234             print_results((*match).first, (*match).second);
235         }
236 
237         const std::size_t broken = count_results(test_result::broken);
238         const std::size_t failed = count_results(test_result::failed);
239         const std::size_t passed = count_results(test_result::passed);
240         const std::size_t skipped = count_results(test_result::skipped);
241         const std::size_t xfail = count_results(test_result::expected_failure);
242         const std::size_t total = broken + failed + passed + skipped + xfail;
243 
244         _writer("===> Summary");
245         _writer(F("Action: %s") % _action_id);
246         _writer(F("Test cases: %s total, %s skipped, %s expected failures, "
247                   "%s broken, %s failed") %
248                 total % skipped % xfail % broken % failed);
249         _writer(F("Total time: %s") % cli::format_delta(_runtime));
250     }
251 };
252 
253 
254 }  // anonymous namespace
255 
256 
257 const fs::path cli::file_writer::_stdout_path("/dev/stdout");
258 const fs::path cli::file_writer::_stderr_path("/dev/stderr");
259 
260 
261 /// Constructs a new file_writer wrapper.
262 ///
263 /// \param ui_ The UI object of the caller command.
264 /// \param path_ The path to the output file.
file_writer(cmdline::ui * const ui_,const fs::path & path_)265 cli::file_writer::file_writer(cmdline::ui* const ui_, const fs::path& path_) :
266     _ui(ui_), _output_path(path_)
267 {
268     if (path_ != _stdout_path && path_ != _stderr_path) {
269         _output_file.reset(new std::ofstream(path_.c_str()));
270         if (!*(_output_file)) {
271             throw std::runtime_error(F("Cannot open output file %s") % path_);
272         }
273     }
274 }
275 
276 /// Destructor.
~file_writer(void)277 cli::file_writer::~file_writer(void)
278 {
279 }
280 
281 /// Writes a message to the selected output.
282 ///
283 /// \param message The message to write; should not include a termination
284 ///     new line.
285 void
operator ()(const std::string & message)286 cli::file_writer::operator()(const std::string& message)
287 {
288     if (_output_path == _stdout_path)
289         _ui->out(message);
290     else if (_output_path == _stderr_path)
291         _ui->err(message);
292     else {
293         INV(_output_file.get() != NULL);
294         (*_output_file) << message << '\n';
295     }
296 }
297 
298 
299 /// Default constructor for cmd_report.
cmd_report(void)300 cmd_report::cmd_report(void) : cli_command(
301     "report", "", 0, 0,
302     "Generates a user-friendly, plain-text report with the result of a "
303     "previous action")
304 {
305     add_option(store_option);
306     add_option(cmdline::bool_option(
307         "show-context", "Include the execution context in the report"));
308     add_option(cmdline::int_option(
309         "action", "The action to report; if not specified, defaults to the "
310         "latest action in the database", "id"));
311     add_option(cmdline::path_option(
312         "output", "The file to which to write the report",
313         "path", "/dev/stdout"));
314     add_option(results_filter_option);
315 }
316 
317 
318 /// Entry point for the "report" subcommand.
319 ///
320 /// \param ui Object to interact with the I/O of the program.
321 /// \param cmdline Representation of the command line to the subcommand.
322 /// \param unused_user_config The runtime configuration of the program.
323 ///
324 /// \return 0 if everything is OK, 1 if the statement is invalid or if there is
325 /// any other problem.
326 int
run(cmdline::ui * ui,const cmdline::parsed_cmdline & cmdline,const config::tree & UTILS_UNUSED_PARAM (user_config))327 cmd_report::run(cmdline::ui* ui, const cmdline::parsed_cmdline& cmdline,
328                 const config::tree& UTILS_UNUSED_PARAM(user_config))
329 {
330     optional< int64_t > action_id;
331     if (cmdline.has_option("action"))
332         action_id = cmdline.get_option< cmdline::int_option >("action");
333 
334     const result_types types = get_result_types(cmdline);
335     console_hooks hooks(
336         ui, cmdline.get_option< cmdline::path_option >("output"),
337         cmdline.has_option("show-context"), types);
338     scan_action::drive(store_path(cmdline), action_id, hooks);
339     hooks.print_tests();
340 
341     return EXIT_SUCCESS;
342 }
343