1 // Copyright 2011 Google Inc. 2 // All rights reserved. 3 // 4 // Redistribution and use in source and binary forms, with or without 5 // modification, are permitted provided that the following conditions are 6 // met: 7 // 8 // * Redistributions of source code must retain the above copyright 9 // notice, this list of conditions and the following disclaimer. 10 // * Redistributions in binary form must reproduce the above copyright 11 // notice, this list of conditions and the following disclaimer in the 12 // documentation and/or other materials provided with the distribution. 13 // * Neither the name of Google Inc. nor the names of its contributors 14 // may be used to endorse or promote products derived from this software 15 // without specific prior written permission. 16 // 17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 #include "cli/common.hpp" 30 31 #include <algorithm> 32 33 #include "engine/filters.hpp" 34 #include "engine/test_case.hpp" 35 #include "engine/test_program.hpp" 36 #include "engine/test_result.hpp" 37 #include "utils/cmdline/exceptions.hpp" 38 #include "utils/cmdline/parser.ipp" 39 #include "utils/datetime.hpp" 40 #include "utils/env.hpp" 41 #include "utils/format/macros.hpp" 42 #include "utils/logging/macros.hpp" 43 #include "utils/fs/exceptions.hpp" 44 #include "utils/fs/operations.hpp" 45 #include "utils/fs/path.hpp" 46 #include "utils/optional.ipp" 47 48 namespace cmdline = utils::cmdline; 49 namespace datetime = utils::datetime; 50 namespace fs = utils::fs; 51 52 using utils::none; 53 using utils::optional; 54 55 56 /// Standard definition of the option to specify the build root. 57 const cmdline::path_option cli::build_root_option( 58 "build-root", 59 "Path to the built test programs, if different from the location of the " 60 "Kyuafile scripts", 61 "path"); 62 63 64 /// Standard definition of the option to specify a Kyuafile. 65 const cmdline::path_option cli::kyuafile_option( 66 'k', "kyuafile", 67 "Path to the test suite definition", 68 "file", "Kyuafile"); 69 70 71 /// Standard definition of the option to specify filters on test results. 72 const cmdline::list_option cli::results_filter_option( 73 "results-filter", "Comma-separated list of result types to include in " 74 "the report", "types", "skipped,xfail,broken,failed"); 75 76 77 /// Standard definition of the option to specify the store. 78 const cmdline::path_option cli::store_option( 79 's', "store", 80 "Path to the store database", 81 "file", "~/.kyua/store.db"); 82 83 84 namespace { 85 86 87 /// Converts a set of result type names to identifiers. 88 /// 89 /// \param names The collection of names to process; may be empty. 90 /// 91 /// \return The result type identifiers corresponding to the input names. 92 /// 93 /// \throw std::runtime_error If any name in the input names is invalid. 94 static cli::result_types 95 parse_types(const std::vector< std::string >& names) 96 { 97 using engine::test_result; 98 typedef std::map< std::string, test_result::result_type > types_map; 99 types_map valid_types; 100 valid_types["broken"] = test_result::broken; 101 valid_types["failed"] = test_result::failed; 102 valid_types["passed"] = test_result::passed; 103 valid_types["skipped"] = test_result::skipped; 104 valid_types["xfail"] = test_result::expected_failure; 105 106 cli::result_types types; 107 for (std::vector< std::string >::const_iterator iter = names.begin(); 108 iter != names.end(); ++iter) { 109 const types_map::const_iterator match = valid_types.find(*iter); 110 if (match == valid_types.end()) 111 throw std::runtime_error(F("Unknown result type '%s'") % *iter); 112 else 113 types.push_back((*match).second); 114 } 115 return types; 116 } 117 118 119 } // anonymous namespace 120 121 122 /// Gets the path to the build root, if any. 123 /// 124 /// This is just syntactic sugar to simplify quierying the 'build_root_option'. 125 /// 126 /// \param cmdline The parsed command line. 127 /// 128 /// \return The path to the build root, if specified; none otherwise. 129 optional< fs::path > 130 cli::build_root_path(const cmdline::parsed_cmdline& cmdline) 131 { 132 optional< fs::path > build_root; 133 if (cmdline.has_option(build_root_option.long_name())) 134 build_root = cmdline.get_option< cmdline::path_option >( 135 build_root_option.long_name()); 136 return build_root; 137 } 138 139 140 /// Gets the value of the HOME environment variable with path validation. 141 /// 142 /// \return The value of the HOME environment variable if it is a valid path; 143 /// none if it is not defined or if it contains an invalid path. 144 optional< fs::path > 145 cli::get_home(void) 146 { 147 const optional< std::string > home = utils::getenv("HOME"); 148 if (home) { 149 try { 150 return utils::make_optional(fs::path(home.get())); 151 } catch (const fs::error& e) { 152 LW(F("Invalid value '%s' in HOME environment variable: %s") % 153 home.get() % e.what()); 154 return none; 155 } 156 } else 157 return none; 158 } 159 160 161 /// Gets the path to the Kyuafile to be loaded. 162 /// 163 /// This is just syntactic sugar to simplify quierying the 'kyuafile_option'. 164 /// 165 /// \param cmdline The parsed command line. 166 /// 167 /// \return The path to the Kyuafile to be loaded. 168 fs::path 169 cli::kyuafile_path(const cmdline::parsed_cmdline& cmdline) 170 { 171 return cmdline.get_option< cmdline::path_option >( 172 kyuafile_option.long_name()); 173 } 174 175 176 /// Gets the filters for the result types. 177 /// 178 /// \param cmdline The parsed command line. 179 /// 180 /// \return A collection of result types to be used for filtering. 181 /// 182 /// \throw std::runtime_error If any of the user-provided filters is invalid. 183 cli::result_types 184 cli::get_result_types(const utils::cmdline::parsed_cmdline& cmdline) 185 { 186 result_types types = parse_types( 187 cmdline.get_option< cmdline::list_option >("results-filter")); 188 if (types.empty()) { 189 types.push_back(engine::test_result::passed); 190 types.push_back(engine::test_result::skipped); 191 types.push_back(engine::test_result::expected_failure); 192 types.push_back(engine::test_result::broken); 193 types.push_back(engine::test_result::failed); 194 } 195 return types; 196 } 197 198 199 /// Gets the path to the store to be used. 200 /// 201 /// This has the side-effect of creating the directory in which to store the 202 /// database if and only if the path to the database matches the default value. 203 /// When the user does not specify an override for the location of the database, 204 /// he should not care about the directory existing. Any of this is not a big 205 /// deal though, because logs are also stored within ~/.kyua and thus we will 206 /// most likely end up creating the directory anyway. 207 /// 208 /// \param cmdline The parsed command line. 209 /// 210 /// \return The path to the store to be used. 211 /// 212 /// \throw fs::error If the creation of the directory fails. 213 fs::path 214 cli::store_path(const cmdline::parsed_cmdline& cmdline) 215 { 216 fs::path store = cmdline.get_option< cmdline::path_option >( 217 store_option.long_name()); 218 if (store == fs::path(store_option.default_value())) { 219 const optional< fs::path > home = cli::get_home(); 220 if (home) { 221 store = home.get() / ".kyua/store.db"; 222 fs::mkdir_p(store.branch_path(), 0777); 223 } else { 224 store = fs::path("kyua-store.db"); 225 LW("HOME not defined; creating store database in current " 226 "directory"); 227 } 228 } 229 LI(F("Store database set to: %s") % store); 230 return store; 231 } 232 233 234 /// Parses a set of command-line arguments to construct test filters. 235 /// 236 /// \param args The command-line arguments representing test filters. 237 /// 238 /// \throw cmdline:error If any of the arguments is invalid, or if they 239 /// represent a non-disjoint collection of filters. 240 std::set< engine::test_filter > 241 cli::parse_filters(const cmdline::args_vector& args) 242 { 243 std::set< engine::test_filter > filters; 244 245 try { 246 for (cmdline::args_vector::const_iterator iter = args.begin(); 247 iter != args.end(); iter++) { 248 const engine::test_filter filter(engine::test_filter::parse(*iter)); 249 if (filters.find(filter) != filters.end()) 250 throw cmdline::error(F("Duplicate filter '%s'") % filter.str()); 251 filters.insert(filter); 252 } 253 check_disjoint_filters(filters); 254 } catch (const std::runtime_error& e) { 255 throw cmdline::error(e.what()); 256 } 257 258 return filters; 259 } 260 261 262 /// Reports the filters that have not matched any tests as errors. 263 /// 264 /// \param unused The collection of unused filters to report. 265 /// \param ui The user interface object through which errors are to be reported. 266 /// 267 /// \return True if there are any unused filters. The caller should report this 268 /// as an error to the user by means of a non-successful exit code. 269 bool 270 cli::report_unused_filters(const std::set< engine::test_filter >& unused, 271 cmdline::ui* ui) 272 { 273 for (std::set< engine::test_filter >::const_iterator iter = unused.begin(); 274 iter != unused.end(); iter++) { 275 cmdline::print_warning(ui, F("No test cases matched by the filter '%s'") 276 % (*iter).str()); 277 } 278 279 return !unused.empty(); 280 } 281 282 283 /// Formats a time delta for user presentation. 284 /// 285 /// \param delta The time delta to format. 286 /// 287 /// \return A user-friendly representation of the time delta. 288 std::string 289 cli::format_delta(const datetime::delta& delta) 290 { 291 return F("%.3ss") % (delta.seconds + (delta.useconds / 1000000.0)); 292 } 293 294 295 /// Formats a test case result for user presentation. 296 /// 297 /// \param result The result to format. 298 /// 299 /// \return A user-friendly representation of the result. 300 std::string 301 cli::format_result(const engine::test_result& result) 302 { 303 std::string text; 304 305 using engine::test_result; 306 switch (result.type()) { 307 case test_result::broken: text = "broken"; break; 308 case test_result::expected_failure: text = "expected_failure"; break; 309 case test_result::failed: text = "failed"; break; 310 case test_result::passed: text = "passed"; break; 311 case test_result::skipped: text = "skipped"; break; 312 } 313 INV(!text.empty()); 314 315 if (!result.reason().empty()) 316 text += ": " + result.reason(); 317 318 return text; 319 } 320 321 322 /// Formats the identifier of a test case for user presentation. 323 /// 324 /// \param test_case The test case whose identifier to format. 325 /// 326 /// \return A string representing the test case uniquely within a test suite. 327 std::string 328 cli::format_test_case_id(const engine::test_case& test_case) 329 { 330 return F("%s:%s") % test_case.container_test_program().relative_path() % 331 test_case.name(); 332 } 333 334 335 /// Formats a filter using the same syntax of a test case. 336 /// 337 /// \param test_filter The filter to format. 338 /// 339 /// \return A string representing the test filter. 340 std::string 341 cli::format_test_case_id(const engine::test_filter& test_filter) 342 { 343 return F("%s:%s") % test_filter.test_program % test_filter.test_case; 344 } 345