dist/engine/test_program.cpp

// Copyright 2010 Google Inc.
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
//   notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above copyright
//   notice, this list of conditions and the following disclaimer in the
//   documentation and/or other materials provided with the distribution.
// * Neither the name of Google Inc. nor the names of its contributors
//   may be used to endorse or promote products derived from this software
//   without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include "engine/test_program.hpp"

#include <algorithm>
#include <map>
#include <sstream>
#include <stdexcept>

#include <lutok/operations.hpp>
#include <lutok/state.ipp>

#include "engine/exceptions.hpp"
#include "engine/test_result.hpp"
#include "engine/testers.hpp"
#include "utils/format/macros.hpp"
#include "utils/logging/macros.hpp"
#include "utils/logging/operations.hpp"
#include "utils/optional.ipp"
#include "utils/sanity.hpp"
#include "utils/text/operations.ipp"

namespace fs = utils::fs;
namespace logging = utils::logging;
namespace text = utils::text;

using utils::none;
using utils::optional;


namespace {


/// Lua hook for the test_case function.
///
/// \pre state(-1) contains the arguments to the function.
///
/// \param state The Lua state in which we are running.
///
/// \return The number of return values, which is always 0.
static int
lua_test_case(lutok::state& state)
{
    if (!state.is_table())
        throw std::runtime_error("Oh noes"); // XXX

    state.get_global("_test_cases");
    engine::test_cases_vector* test_cases =
        *state.to_userdata< engine::test_cases_vector* >();
    state.pop(1);

    state.get_global("_test_program");
    const engine::test_program* test_program =
        *state.to_userdata< engine::test_program* >();
    state.pop(1);

    state.push_string("name");
    state.get_table(-2);
    const std::string name = state.to_string();
    state.pop(1);

    engine::metadata_builder mdbuilder(test_program->get_metadata());

    state.push_nil();
    while (state.next(-2)) {
        if (!state.is_string(-2))
            throw std::runtime_error("Oh oh");  // XXX
        const std::string property = state.to_string(-2);

        if (!state.is_string(-1))
            throw std::runtime_error("Oh oh");  // XXX
        const std::string value = state.to_string(-1);

        if (property != "name")
            mdbuilder.set_string(property, value);

        state.pop(1);
    }
    state.pop(1);

    engine::test_case_ptr test_case(
        new engine::test_case(test_program->interface_name(), *test_program,
                              name, mdbuilder.build()));
    test_cases->push_back(test_case);

    return 0;
}


/// Sets up the Lua state to process the output of a test case list.
///
/// \param [in,out] state The Lua state to configure.
/// \param test_program Pointer to the test program being loaded.
/// \param [out] test_cases Vector that will contain the list of test cases.
static void
setup_lua_state(lutok::state& state, const engine::test_program* test_program,
                engine::test_cases_vector* test_cases)
{
    *state.new_userdata< engine::test_cases_vector* >() = test_cases;
    state.set_global("_test_cases");

    *state.new_userdata< const engine::test_program* >() = test_program;
    state.set_global("_test_program");

    state.push_cxx_function(lua_test_case);
    state.set_global("test_case");
}


/// Loads the list of test cases from a test program.
///
/// \param test_program Representation of the test program to load.
///
/// \return A list of test cases.
static engine::test_cases_vector
load_test_cases(const engine::test_program& test_program)
{
    const engine::tester tester(test_program.interface_name(), none, none);
    const std::string output = tester.list(test_program.absolute_path());

    engine::test_cases_vector test_cases;
    lutok::state state;
    setup_lua_state(state, &test_program, &test_cases);
    lutok::do_string(state, output, 0);
    return test_cases;
}


/// Predicate to compare two test cases via pointers to them.
///
/// \param tc1 Entry in a map of test case names to test case pointers.
/// \param tc2 Entry in a map of test case names to test case pointers.
///
/// \return True if the test case in tc1 is the same as in tc2.  Note that the
/// container test programs are NOT compared.
static bool
compare_test_case(const std::pair< std::string, engine::test_case_ptr >& tc1,
                  const std::pair< std::string, engine::test_case_ptr >& tc2)
{
    return tc1.first == tc2.first && *tc1.second == *tc2.second;
}


/// Compares if two sets of test cases hold the same values.
///
/// \param tests1 First collection of test cases.
/// \param tests2 Second collection of test cases.
///
/// \return True if both collections hold the same test cases (value-wise, not
/// pointer-wise); false otherwise.
static bool
compare_test_cases(const optional< engine::test_cases_vector >& tests1,
                   const optional< engine::test_cases_vector >& tests2)
{
    if (!tests1 && !tests2)
        return true;
    else if ((tests1 && !tests2) || (!tests1 && tests2))
        return false;
    INV(tests1 && tests2);

    // This is very inefficient, but because it should only be used in our own
    // tests, it doesn't matter.
    std::map< std::string, engine::test_case_ptr > map1, map2;
    for (engine::test_cases_vector::const_iterator iter = tests1.get().begin();
         iter != tests1.get().end(); ++iter)
        map1.insert(make_pair((*iter)->name(), *iter));
    for (engine::test_cases_vector::const_iterator iter = tests2.get().begin();
         iter != tests2.get().end(); ++iter)
        map2.insert(make_pair((*iter)->name(), *iter));
    return std::equal(map1.begin(), map1.end(), map2.begin(),
                      compare_test_case);
}


}  // anonymous namespace


/// Internal implementation of a test_program.
struct engine::test_program::impl {
    /// Name of the test program interface.
    std::string interface_name;

    /// Name of the test program binary relative to root.
    fs::path binary;

    /// Root of the test suite containing the test program.
    fs::path root;

    /// Name of the test suite this program belongs to.
    std::string test_suite_name;

    /// Metadata of the test program.
    metadata md;

    /// List of test cases in the test program; lazily initialized.
    optional< test_cases_vector > test_cases;

    /// Constructor.
    ///
    /// \param interface_name_ Name of the test program interface.
    /// \param binary_ The name of the test program binary relative to root_.
    /// \param root_ The root of the test suite containing the test program.
    /// \param test_suite_name_ The name of the test suite this program
    ///     belongs to.
    /// \param md_ Metadata of the test program.
    impl(const std::string& interface_name_, const fs::path& binary_,
         const fs::path& root_, const std::string& test_suite_name_,
         const metadata& md_) :
        interface_name(interface_name_),
        binary(binary_),
        root(root_),
        test_suite_name(test_suite_name_),
        md(md_)
    {
        PRE_MSG(!binary.is_absolute(),
                F("The program '%s' must be relative to the root of the test "
                  "suite '%s'") % binary % root);
    }

    /// Equality comparator.
    ///
    /// \param other The other object to compare this one to.
    ///
    /// \return True if this object and other are equal; false otherwise.
    bool
    operator==(const impl& other) const
    {
        return (interface_name == other.interface_name &&
                binary == other.binary &&
                root == other.root &&
                test_suite_name == other.test_suite_name &&
                md == other.md &&
                compare_test_cases(test_cases, other.test_cases));
    }
};


/// Constructs a new test program.
///
/// \param interface_name_ Name of the test program interface.
/// \param binary_ The name of the test program binary relative to root_.
/// \param root_ The root of the test suite containing the test program.
/// \param test_suite_name_ The name of the test suite this program belongs to.
/// \param md_ Metadata of the test program.
engine::test_program::test_program(const std::string& interface_name_,
                                   const fs::path& binary_,
                                   const fs::path& root_,
                                   const std::string& test_suite_name_,
                                   const metadata& md_) :
    _pimpl(new impl(interface_name_, binary_, root_, test_suite_name_, md_))
{
}


/// Destroys a test program.
engine::test_program::~test_program(void)
{
}


/// Gets the name of the test program interface.
///
/// \return An interface name.
const std::string&
engine::test_program::interface_name(void) const
{
    return _pimpl->interface_name;
}


/// Gets the path to the test program relative to the root of the test suite.
///
/// \return The relative path to the test program binary.
const fs::path&
engine::test_program::relative_path(void) const
{
    return _pimpl->binary;
}


/// Gets the absolute path to the test program.
///
/// \return The absolute path to the test program binary.
const fs::path
engine::test_program::absolute_path(void) const
{
    const fs::path full_path = _pimpl->root / _pimpl->binary;
    return full_path.is_absolute() ? full_path : full_path.to_absolute();
}


/// Gets the root of the test suite containing this test program.
///
/// \return The path to the root of the test suite.
const fs::path&
engine::test_program::root(void) const
{
    return _pimpl->root;
}


/// Gets the name of the test suite containing this test program.
///
/// \return The name of the test suite.
const std::string&
engine::test_program::test_suite_name(void) const
{
    return _pimpl->test_suite_name;
}


/// Gets the metadata of the test program.
///
/// \return The metadata.
const engine::metadata&
engine::test_program::get_metadata(void) const
{
    return _pimpl->md;
}


/// Gets a test case by its name.
///
/// \param name The name of the test case to locate.
///
/// \return The requested test case.
///
/// \throw not_found_error If the specified test case is not in the test
///     program.
const engine::test_case_ptr&
engine::test_program::find(const std::string& name) const
{
    // TODO(jmmv): Should use a test_cases_map instead of a vector to optimize
    // lookups.
    const test_cases_vector& tcs = test_cases();
    for (test_cases_vector::const_iterator iter = tcs.begin();
         iter != tcs.end(); iter++) {
        if ((*iter)->name() == name)
            return *iter;
    }
    throw not_found_error(F("Unknown test case %s in test program %s") % name %
                          relative_path());
}


/// Gets the list of test cases from the test program.
///
/// Note that this operation may be expensive because it may lazily load the
/// test cases list from the test program.  Errors during the processing of the
/// test case list are represented as a single test case describing the failure.
///
/// \return The list of test cases provided by the test program.
const engine::test_cases_vector&
engine::test_program::test_cases(void) const
{
    if (!_pimpl->test_cases) {
        try {
            _pimpl->test_cases = load_test_cases(*this);
        } catch (const std::runtime_error& e) {
            // TODO(jmmv): This is a very ugly workaround for the fact that we
            // cannot report failures at the test-program level.  We should
            // either address this, or move this reporting to the testers
            // themselves.
            LW(F("Failed to load test cases list: %s") % e.what());
            engine::test_cases_vector fake_test_cases;
            fake_test_cases.push_back(test_case_ptr(new test_case(
                _pimpl->interface_name, *this, "__test_cases_list__",
                "Represents the correct processing of the test cases list",
                test_result(engine::test_result::broken, e.what()))));
            _pimpl->test_cases = fake_test_cases;
        }
    }
    return _pimpl->test_cases.get();
}


/// Sets the collection of test cases included in this test program.
///
/// This function is provided so that when we load test programs from the
/// database we can populate them with the test cases they include.  We don't
/// want such test programs to be executed to gather this information.
///
/// We cannot provide this collection of tests in the constructor of the test
/// program because the test cases have to point to their test programs.
///
/// \pre The test program must not have attempted to load its test cases yet.
///     I.e. test_cases() has not been called.
///
/// \param test_cases_ The test cases to add to this test program.
void
engine::test_program::set_test_cases(const test_cases_vector& test_cases_)
{
    PRE(!_pimpl->test_cases);
    _pimpl->test_cases = test_cases_;
}


/// Equality comparator.
///
/// \param other The other object to compare this one to.
///
/// \return True if this object and other are equal; false otherwise.
bool
engine::test_program::operator==(const test_program& other) const
{
    return _pimpl == other._pimpl || *_pimpl == *other._pimpl;
}


/// Inequality comparator.
///
/// \param other The other object to compare this one to.
///
/// \return True if this object and other are different; false otherwise.
bool
engine::test_program::operator!=(const test_program& other) const
{
    return !(*this == other);
}


/// Injects the object into a stream.
///
/// \param output The stream into which to inject the object.
/// \param object The object to format.
///
/// \return The output stream.
std::ostream&
engine::operator<<(std::ostream& output, const test_cases_vector& object)
{
    output << "[";
    for (test_cases_vector::size_type i = 0; i < object.size(); ++i) {
        if (i != 0)
            output << ", ";
        output << *object[i];
    }
    output << "]";
    return output;
}


/// Injects the object into a stream.
///
/// \param output The stream into which to inject the object.
/// \param object The object to format.
///
/// \return The output stream.
std::ostream&
engine::operator<<(std::ostream& output, const test_program& object)
{
    output << F("test_program{interface=%s, binary=%s, root=%s, test_suite=%s, "
                "metadata=%s, test_cases=%s}")
        % text::quote(object.interface_name(), '\'')
        % text::quote(object.relative_path().str(), '\'')
        % text::quote(object.root().str(), '\'')
        % text::quote(object.test_suite_name(), '\'')
        % object.get_metadata()
        % object.test_cases();
    return output;
}