From 49eaf7c3fd98a2983ac7c9f0994fbca0b73a33c1 Mon Sep 17 00:00:00 2001 From: Daniel Kirchner Date: Tue, 6 Mar 2018 20:45:34 +0100 Subject: [PATCH 1/4] Infrastructure for extracting syntax tests in separate test files. --- appveyor.yml | 2 +- scripts/isolate_tests.py | 7 +- scripts/tests.sh | 2 +- test/TestHelper.cpp | 9 ++ test/TestHelper.h | 1 + test/boostTest.cpp | 2 + .../SolidityNameAndTypeResolution.cpp | 52 ------- test/libsolidity/SyntaxTestParser.cpp | 97 +++++++++++++ test/libsolidity/SyntaxTestParser.h | 57 ++++++++ test/libsolidity/SyntaxTester.cpp | 134 ++++++++++++++++++ test/libsolidity/SyntaxTester.h | 48 +++++++ .../double_stateVariable_declaration.sol | 6 + .../double_variable_declaration.sol | 8 ++ .../double_variable_declaration_050.sol | 11 ++ test/libsolidity/syntaxTests/smoke_test.sol | 6 + 15 files changed, 387 insertions(+), 55 deletions(-) create mode 100644 test/libsolidity/SyntaxTestParser.cpp create mode 100644 test/libsolidity/SyntaxTestParser.h create mode 100644 test/libsolidity/SyntaxTester.cpp create mode 100644 test/libsolidity/SyntaxTester.h create mode 100644 test/libsolidity/syntaxTests/double_stateVariable_declaration.sol create mode 100644 test/libsolidity/syntaxTests/double_variable_declaration.sol create mode 100644 test/libsolidity/syntaxTests/double_variable_declaration_050.sol create mode 100644 test/libsolidity/syntaxTests/smoke_test.sol diff --git a/appveyor.yml b/appveyor.yml index ef5f69074..5fd85482f 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -71,7 +71,7 @@ build_script: test_script: - cd %APPVEYOR_BUILD_FOLDER%\build\test\%CONFIGURATION% - - soltest.exe --show-progress -- --no-ipc --no-smt + - soltest.exe --show-progress -- --testpath %APPVEYOR_BUILD_FOLDER%\test --no-ipc --no-smt # Skip bytecode compare if private key is not available - cd %APPVEYOR_BUILD_FOLDER% - ps: if ($env:priv_key) { diff --git a/scripts/isolate_tests.py b/scripts/isolate_tests.py index cfaef2106..5bf577d3b 100755 --- a/scripts/isolate_tests.py +++ b/scripts/isolate_tests.py @@ -86,10 +86,15 @@ if __name__ == '__main__': for root, subdirs, files in os.walk(path): if '_build' in subdirs: subdirs.remove('_build') + if 'compilationTests' in subdirs: + subdirs.remove('compilationTests') for f in files: path = join(root, f) if docs: cases = extract_docs_cases(path) else: - cases = extract_test_cases(path) + if f.endswith(".sol"): + cases = [open(path, "r").read()] + else: + cases = extract_test_cases(path) write_cases(cases) diff --git a/scripts/tests.sh b/scripts/tests.sh index bf4ee3d9d..37ffafc28 100755 --- a/scripts/tests.sh +++ b/scripts/tests.sh @@ -116,7 +116,7 @@ do log=--logger=JUNIT,test_suite,$log_directory/noopt_$vm.xml $testargs_no_opt fi fi - "$REPO_ROOT"/build/test/soltest $progress $log -- "$optimize" --evm-version "$vm" --ipcpath /tmp/test/geth.ipc + "$REPO_ROOT"/build/test/soltest $progress $log -- --testpath "$REPO_ROOT"/test "$optimize" --evm-version "$vm" --ipcpath /tmp/test/geth.ipc done done diff --git a/test/TestHelper.cpp b/test/TestHelper.cpp index e0d4423dc..77fa204fb 100644 --- a/test/TestHelper.cpp +++ b/test/TestHelper.cpp @@ -43,6 +43,11 @@ Options::Options() ipcPath = suite.argv[i + 1]; i++; } + else if (string(suite.argv[i]) == "--testpath" && i + 1 < suite.argc) + { + testPath = suite.argv[i + 1]; + i++; + } else if (string(suite.argv[i]) == "--optimize") optimize = true; else if (string(suite.argv[i]) == "--evm-version") @@ -60,6 +65,10 @@ Options::Options() if (!disableIPC && ipcPath.empty()) if (auto path = getenv("ETH_TEST_IPC")) ipcPath = path; + + if (testPath.empty()) + if (auto path = getenv("ETH_TEST_PATH")) + testPath = path; } dev::solidity::EVMVersion Options::evmVersion() const diff --git a/test/TestHelper.h b/test/TestHelper.h index 8c2eec362..9c61be3b9 100644 --- a/test/TestHelper.h +++ b/test/TestHelper.h @@ -35,6 +35,7 @@ namespace test struct Options: boost::noncopyable { std::string ipcPath; + std::string testPath; bool showMessages = false; bool optimize = false; bool disableIPC = false; diff --git a/test/boostTest.cpp b/test/boostTest.cpp index a3cc51c5b..fa9df3fd6 100644 --- a/test/boostTest.cpp +++ b/test/boostTest.cpp @@ -36,6 +36,7 @@ #pragma GCC diagnostic pop #include +#include using namespace boost::unit_test; @@ -54,6 +55,7 @@ test_suite* init_unit_test_suite( int /*argc*/, char* /*argv*/[] ) { master_test_suite_t& master = framework::master_test_suite(); master.p_name.value = "SolidityTests"; + dev::solidity::test::SyntaxTester::registerTests(); if (dev::test::Options::get().disableIPC) { for (auto suite: { diff --git a/test/libsolidity/SolidityNameAndTypeResolution.cpp b/test/libsolidity/SolidityNameAndTypeResolution.cpp index 997b610ea..1f76c01b5 100644 --- a/test/libsolidity/SolidityNameAndTypeResolution.cpp +++ b/test/libsolidity/SolidityNameAndTypeResolution.cpp @@ -43,27 +43,6 @@ namespace test BOOST_FIXTURE_TEST_SUITE(SolidityNameAndTypeResolution, AnalysisFramework) -BOOST_AUTO_TEST_CASE(smoke_test) -{ - char const* text = R"( - contract test { - uint256 stateVariable1; - function fun(uint256 arg1) public { uint256 y; y = arg1; } - } - )"; - CHECK_SUCCESS(text); -} - -BOOST_AUTO_TEST_CASE(double_stateVariable_declaration) -{ - char const* text = R"( - contract test { - uint256 variable; - uint128 variable; - } - )"; - CHECK_ERROR(text, DeclarationError, "Identifier already declared."); -} BOOST_AUTO_TEST_CASE(double_function_declaration) { @@ -76,37 +55,6 @@ BOOST_AUTO_TEST_CASE(double_function_declaration) CHECK_ERROR(text, DeclarationError, "Function with same name and arguments defined twice."); } -BOOST_AUTO_TEST_CASE(double_variable_declaration) -{ - string text = R"( - contract test { - function f() pure public { - uint256 x; - if (true) { uint256 x; } - } - } - )"; - CHECK_ERROR(text, DeclarationError, "Identifier already declared"); -} - -BOOST_AUTO_TEST_CASE(double_variable_declaration_050) -{ - string text = R"( - pragma experimental "v0.5.0"; - contract test { - function f() pure public { - uint256 x; - if (true) { uint256 x; } - } - } - )"; - CHECK_WARNING_ALLOW_MULTI(text, (vector{ - "This declaration shadows an existing declaration.", - "Unused local variable", - "Unused local variable" - })); -} - BOOST_AUTO_TEST_CASE(double_variable_declaration_disjoint_scope) { string text = R"( diff --git a/test/libsolidity/SyntaxTestParser.cpp b/test/libsolidity/SyntaxTestParser.cpp new file mode 100644 index 000000000..c37caca53 --- /dev/null +++ b/test/libsolidity/SyntaxTestParser.cpp @@ -0,0 +1,97 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#include +#include +#include +#include +#include +#include + +using namespace dev; +using namespace solidity; +using namespace dev::solidity::test; +using namespace std; + +template +void skipWhitespace(IteratorType& it, IteratorType end) +{ + while (it != end && isspace(*it)) + ++it; +} + +template +void skipSlashes(IteratorType& it, IteratorType end) +{ + while (it != end && *it == '/') + ++it; +} + +std::string SyntaxTestParser::parseSource(std::istream& _stream) +{ + std::string source; + string line; + string const delimiter("// ----"); + while (getline(_stream, line)) + if (boost::algorithm::starts_with(line, delimiter)) + break; + else + source += line + "\n"; + return source; +} + +std::vector SyntaxTestParser::parseExpectations(std::istream& _stream) +{ + std::vector expectations; + std::string line; + while (getline(_stream, line)) + { + auto it = line.begin(); + + skipSlashes(it, line.end()); + skipWhitespace(it, line.end()); + + if (it == line.end()) continue; + + auto typeBegin = it; + while (it != line.end() && *it != ':') + ++it; + string errorType(typeBegin, it); + + // skip colon + if (it != line.end()) it++; + + skipWhitespace(it, line.end()); + + string errorMessage(it, line.end()); + expectations.emplace_back(SyntaxTestExpectation{move(errorType), move(errorMessage)}); + } + return expectations; +} + +SyntaxTest SyntaxTestParser::parse(string const& _filename) +{ + ifstream file(_filename); + if (!file) + BOOST_THROW_EXCEPTION(runtime_error("cannot open test contract: \"" + _filename + "\"")); + file.exceptions(ios::badbit); + + SyntaxTest result; + result.source = parseSource(file); + result.expectations = parseExpectations(file); + return result; +} diff --git a/test/libsolidity/SyntaxTestParser.h b/test/libsolidity/SyntaxTestParser.h new file mode 100644 index 000000000..9e295a0b1 --- /dev/null +++ b/test/libsolidity/SyntaxTestParser.h @@ -0,0 +1,57 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#pragma once + +#include +#include +#include +#include + +namespace dev +{ +namespace solidity +{ +namespace test +{ + +struct SyntaxTestExpectation +{ + std::string type; + std::string message; +}; + +struct SyntaxTest +{ + std::string source; + std::vector expectations; +}; + +class SyntaxTestParser +{ +public: + SyntaxTestParser() = default; + + SyntaxTest parse(std::string const& _filename); +private: + std::string parseSource(std::istream& _stream); + std::vector parseExpectations(std::istream& _stream); +}; + +} +} +} diff --git a/test/libsolidity/SyntaxTester.cpp b/test/libsolidity/SyntaxTester.cpp new file mode 100644 index 000000000..4e5457604 --- /dev/null +++ b/test/libsolidity/SyntaxTester.cpp @@ -0,0 +1,134 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#include +#include +#include +#include + +using namespace dev; +using namespace solidity; +using namespace dev::solidity::test; +using namespace std; +using namespace boost::unit_test; +namespace fs = boost::filesystem; + +void SyntaxTester::runTest(SyntaxTest const& _test) +{ + vector unexpectedErrors; + auto expectations = _test.expectations; + auto errorList = parseAnalyseAndReturnError(_test.source, true, true, true).second; + + bool errorsMatch = true; + + if (errorList.size() != expectations.size()) + errorsMatch = false; + else + { + for (size_t i = 0; i < errorList.size(); i++) + { + if ( + !(errorList[i]->typeName() == expectations[i].type) || + !(errorMessage(*errorList[i]) == expectations[i].message) + ) + { + errorsMatch = false; + break; + } + } + } + + if (!errorsMatch) + { + string msg = "Test expectation mismatch.\nExpected result:\n"; + if (expectations.empty()) + msg += "\tSuccess\n"; + else + for (auto const& expectation: expectations) + msg += "\t" + expectation.type + ": " + expectation.message + "\n"; + msg += "Obtained result:\n"; + if (errorList.empty()) + msg += "\tSuccess\n"; + else + for (auto const& error: errorList) + msg += "\t" + error->typeName() + ": " + errorMessage(*error) + "\n"; + BOOST_ERROR(msg); + } +} + +std::string SyntaxTester::errorMessage(Error const& _e) +{ + if (_e.comment()) + return boost::replace_all_copy(*_e.comment(), "\n", "\\n"); + else + return "NONE"; +} + +int SyntaxTester::registerTests( + test_suite& _suite, + fs::path const& _basepath, + fs::path const& _path +) +{ + + int numTestsAdded = 0; + fs::path fullpath = _basepath / _path; + if (fs::is_directory(fullpath)) + { + test_suite* sub_suite = BOOST_TEST_SUITE(_path.filename().string()); + for (auto const& entry: boost::iterator_range( + fs::directory_iterator(fullpath), + fs::directory_iterator() + )) + numTestsAdded += registerTests(*sub_suite, _basepath, _path / entry.path().filename()); + _suite.add(sub_suite); + } + else + { + _suite.add(make_test_case( + [fullpath] { SyntaxTester().runTest(SyntaxTestParser().parse(fullpath.string())); }, + _path.stem().string(), + _path.string(), + 0 + )); + numTestsAdded = 1; + } + return numTestsAdded; +} + +void SyntaxTester::registerTests() +{ + if(dev::test::Options::get().testPath.empty()) + throw runtime_error( + "No path to the test files was specified. " + "Use the --testpath command line option or " + "the ETH_TEST_PATH environment variable." + ); + auto testPath = fs::path(dev::test::Options::get().testPath); + + if (fs::exists(testPath) && fs::is_directory(testPath)) + { + int numTestsAdded = registerTests( + framework::master_test_suite(), + testPath / "libsolidity", + "syntaxTests" + ); + solAssert(numTestsAdded > 0, "no syntax tests found in libsolidity/syntaxTests"); + } + else + solAssert(false, "libsolidity/syntaxTests directory not found"); +} diff --git a/test/libsolidity/SyntaxTester.h b/test/libsolidity/SyntaxTester.h new file mode 100644 index 000000000..d61668aa2 --- /dev/null +++ b/test/libsolidity/SyntaxTester.h @@ -0,0 +1,48 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#pragma once + +#include +#include +#include +#include + +namespace dev +{ +namespace solidity +{ +namespace test +{ + +class SyntaxTester: public AnalysisFramework +{ +public: + static void registerTests(); +private: + static int registerTests( + boost::unit_test::test_suite& _suite, + boost::filesystem::path const& _basepath, + boost::filesystem::path const& _path + ); + static std::string errorMessage(Error const& _e); + void runTest(SyntaxTest const& _test); +}; + +} +} +} diff --git a/test/libsolidity/syntaxTests/double_stateVariable_declaration.sol b/test/libsolidity/syntaxTests/double_stateVariable_declaration.sol new file mode 100644 index 000000000..c5507b649 --- /dev/null +++ b/test/libsolidity/syntaxTests/double_stateVariable_declaration.sol @@ -0,0 +1,6 @@ +contract test { + uint256 variable; + uint128 variable; +} +// ---- +// DeclarationError: Identifier already declared. diff --git a/test/libsolidity/syntaxTests/double_variable_declaration.sol b/test/libsolidity/syntaxTests/double_variable_declaration.sol new file mode 100644 index 000000000..3349cfece --- /dev/null +++ b/test/libsolidity/syntaxTests/double_variable_declaration.sol @@ -0,0 +1,8 @@ +contract test { + function f() pure public { + uint256 x; + if (true) { uint256 x; } + } +} +// ---- +// DeclarationError: Identifier already declared. diff --git a/test/libsolidity/syntaxTests/double_variable_declaration_050.sol b/test/libsolidity/syntaxTests/double_variable_declaration_050.sol new file mode 100644 index 000000000..9c2d40d52 --- /dev/null +++ b/test/libsolidity/syntaxTests/double_variable_declaration_050.sol @@ -0,0 +1,11 @@ +pragma experimental "v0.5.0"; +contract test { + function f() pure public { + uint256 x; + if (true) { uint256 x; } + } +} +// ---- +// Warning: This declaration shadows an existing declaration. +// Warning: Unused local variable. +// Warning: Unused local variable. diff --git a/test/libsolidity/syntaxTests/smoke_test.sol b/test/libsolidity/syntaxTests/smoke_test.sol new file mode 100644 index 000000000..2d48098a1 --- /dev/null +++ b/test/libsolidity/syntaxTests/smoke_test.sol @@ -0,0 +1,6 @@ +contract test { + uint256 stateVariable1; + function fun(uint256 arg1) public { uint256 y; y = arg1; } +} +// ---- +// Warning: Function state mutability can be restricted to pure From 317c1f7fa36a24259a3c678fad255406df9da64d Mon Sep 17 00:00:00 2001 From: Daniel Kirchner Date: Fri, 9 Mar 2018 11:03:16 +0100 Subject: [PATCH 2/4] Workaround for boost < 1.59.0 --- test/libsolidity/SyntaxTester.cpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/libsolidity/SyntaxTester.cpp b/test/libsolidity/SyntaxTester.cpp index 4e5457604..886bc3bfe 100644 --- a/test/libsolidity/SyntaxTester.cpp +++ b/test/libsolidity/SyntaxTester.cpp @@ -27,6 +27,19 @@ using namespace std; using namespace boost::unit_test; namespace fs = boost::filesystem; +#if BOOST_VERSION < 105900 +test_case *make_test_case( + function const& _fn, + string const& _name, + string const&, // _filename + size_t // _line +) +{ + return make_test_case(_fn, _name); +} +#endif + + void SyntaxTester::runTest(SyntaxTest const& _test) { vector unexpectedErrors; From 3232561d979954f0625102d33cf042fc5eda7211 Mon Sep 17 00:00:00 2001 From: Daniel Kirchner Date: Mon, 12 Mar 2018 13:33:37 +0100 Subject: [PATCH 3/4] Refactoring; fuse SyntaxTestParser and SyntaxTester to SyntaxTest. --- docs/contributing.rst | 15 +- test/TestHelper.h | 2 +- test/boostTest.cpp | 8 +- test/libsolidity/SyntaxTest.cpp | 205 ++++++++++++++++++ .../{SyntaxTester.h => SyntaxTest.h} | 41 +++- test/libsolidity/SyntaxTestParser.cpp | 97 --------- test/libsolidity/SyntaxTestParser.h | 57 ----- test/libsolidity/SyntaxTester.cpp | 147 ------------- 8 files changed, 258 insertions(+), 314 deletions(-) create mode 100644 test/libsolidity/SyntaxTest.cpp rename test/libsolidity/{SyntaxTester.h => SyntaxTest.h} (54%) delete mode 100644 test/libsolidity/SyntaxTestParser.cpp delete mode 100644 test/libsolidity/SyntaxTestParser.h delete mode 100644 test/libsolidity/SyntaxTester.cpp diff --git a/docs/contributing.rst b/docs/contributing.rst index a5efba8bf..8b4695e40 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -69,15 +69,22 @@ Solidity includes different types of tests. They are included in the application called ``soltest``. Some of them require the ``cpp-ethereum`` client in testing mode, some others require ``libz3`` to be installed. -To disable the z3 tests, use ``./build/test/soltest -- --no-smt`` and -to run a subset of the tests that do not require ``cpp-ethereum``, use ``./build/test/soltest -- --no-ipc``. +``soltest`` reads test contracts that are annotated with expected results +stored in ``./test/libsolidity/syntaxTests``. In order for soltest to find these +tests the root test directory has to be specified using the ``--testpath`` command +line option, e.g. ``./build/test/soltest -- --testpath ./test``. + +To disable the z3 tests, use ``./build/test/soltest -- --no-smt --testpath ./test`` and +to run a subset of the tests that do not require ``cpp-ethereum``, use +``./build/test/soltest -- --no-ipc --testpath ./test``. For all other tests, you need to install `cpp-ethereum `_ and run it in testing mode: ``eth --test -d /tmp/testeth``. -Then you run the actual tests: ``./build/test/soltest -- --ipcpath /tmp/testeth/geth.ipc``. +Then you run the actual tests: ``./build/test/soltest -- --ipcpath /tmp/testeth/geth.ipc --testpath ./test``. To run a subset of tests, filters can be used: -``soltest -t TestSuite/TestName -- --ipcpath /tmp/testeth/geth.ipc``, where ``TestName`` can be a wildcard ``*``. +``soltest -t TestSuite/TestName -- --ipcpath /tmp/testeth/geth.ipc --testpath ./test``, +where ``TestName`` can be a wildcard ``*``. Alternatively, there is a testing script at ``scripts/test.sh`` which executes all tests and runs ``cpp-ethereum`` automatically if it is in the path (but does not download it). diff --git a/test/TestHelper.h b/test/TestHelper.h index 9c61be3b9..f7b1d94c8 100644 --- a/test/TestHelper.h +++ b/test/TestHelper.h @@ -35,7 +35,7 @@ namespace test struct Options: boost::noncopyable { std::string ipcPath; - std::string testPath; + boost::filesystem::path testPath; bool showMessages = false; bool optimize = false; bool disableIPC = false; diff --git a/test/boostTest.cpp b/test/boostTest.cpp index fa9df3fd6..e557ff95a 100644 --- a/test/boostTest.cpp +++ b/test/boostTest.cpp @@ -36,7 +36,7 @@ #pragma GCC diagnostic pop #include -#include +#include using namespace boost::unit_test; @@ -55,7 +55,11 @@ test_suite* init_unit_test_suite( int /*argc*/, char* /*argv*/[] ) { master_test_suite_t& master = framework::master_test_suite(); master.p_name.value = "SolidityTests"; - dev::solidity::test::SyntaxTester::registerTests(); + solAssert(dev::solidity::test::SyntaxTest::registerTests( + master, + dev::test::Options::get().testPath / "libsolidity", + "syntaxTests" + ) > 0, "no syntax tests found"); if (dev::test::Options::get().disableIPC) { for (auto suite: { diff --git a/test/libsolidity/SyntaxTest.cpp b/test/libsolidity/SyntaxTest.cpp new file mode 100644 index 000000000..8cbe2f36c --- /dev/null +++ b/test/libsolidity/SyntaxTest.cpp @@ -0,0 +1,205 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#include +#include +#include +#include +#include +#include +#include + +using namespace dev; +using namespace solidity; +using namespace dev::solidity::test; +using namespace std; +namespace fs = boost::filesystem; +using namespace boost::unit_test; + +template +void skipWhitespace(IteratorType& it, IteratorType end) +{ + while (it != end && isspace(*it)) + ++it; +} + +template +void skipSlashes(IteratorType& it, IteratorType end) +{ + while (it != end && *it == '/') + ++it; +} + +SyntaxTest::SyntaxTest(string const& _filename) +{ + ifstream file(_filename); + if (!file) + BOOST_THROW_EXCEPTION(runtime_error("cannot open test contract: \"" + _filename + "\"")); + file.exceptions(ios::badbit); + + m_source = parseSource(file); + m_expectations = parseExpectations(file); +} + +bool SyntaxTest::run(ostream& _stream, string const& _indent) +{ + m_errorList = parseAnalyseAndReturnError(m_source, true, true, true).second; + if (!matchesExpectations(m_errorList)) + { + std::string nextIndentLevel = _indent.empty() ? "\t" : _indent + _indent; + _stream << _indent << "Expected result:" << endl; + printExpected(_stream, nextIndentLevel); + _stream << _indent << "Obtained result:\n"; + printErrorList(_stream, m_errorList, nextIndentLevel); + return false; + } + return true; +} + +void SyntaxTest::printExpected(ostream& _stream, string const& _indent) const +{ + if (m_expectations.empty()) + _stream << _indent << "Success" << endl; + else + for (auto const& expectation: m_expectations) + _stream << _indent << expectation.type << ": " << expectation.message << endl; +} + +void SyntaxTest::printErrorList( + ostream& _stream, + ErrorList const& _errorList, + string const& _indent +) const +{ + if (_errorList.empty()) + _stream << _indent << "Success" << endl; + else + for (auto const& error: _errorList) + _stream << _indent << error->typeName() << ": " << errorMessage(*error) << endl; +} + +bool SyntaxTest::matchesExpectations(ErrorList const& _errorList) const +{ + if (_errorList.size() != m_expectations.size()) + return false; + else + for (size_t i = 0; i < _errorList.size(); i++) + if ( + !(_errorList[i]->typeName() == m_expectations[i].type) || + !(errorMessage(*_errorList[i]) == m_expectations[i].message) + ) + return false; + return true; +} + +string SyntaxTest::errorMessage(Error const& _e) +{ + if (_e.comment()) + return boost::replace_all_copy(*_e.comment(), "\n", "\\n"); + else + return "NONE"; +} + +string SyntaxTest::parseSource(istream& _stream) +{ + string source; + string line; + string const delimiter("// ----"); + while (getline(_stream, line)) + if (boost::algorithm::starts_with(line, delimiter)) + break; + else + source += line + "\n"; + return source; +} + +vector SyntaxTest::parseExpectations(istream& _stream) +{ + vector expectations; + string line; + while (getline(_stream, line)) + { + auto it = line.begin(); + + skipSlashes(it, line.end()); + skipWhitespace(it, line.end()); + + if (it == line.end()) continue; + + auto typeBegin = it; + while (it != line.end() && *it != ':') + ++it; + string errorType(typeBegin, it); + + // skip colon + if (it != line.end()) it++; + + skipWhitespace(it, line.end()); + + string errorMessage(it, line.end()); + expectations.emplace_back(SyntaxTestExpectation{move(errorType), move(errorMessage)}); + } + return expectations; +} + +#if BOOST_VERSION < 105900 +test_case *make_test_case( + function const& _fn, + string const& _name, + string const& /* _filename */, + size_t /* _line */ +) +{ + return make_test_case(_fn, _name); +} +#endif + +int SyntaxTest::registerTests( + boost::unit_test::test_suite& _suite, + boost::filesystem::path const& _basepath, + boost::filesystem::path const& _path +) +{ + int numTestsAdded = 0; + fs::path fullpath = _basepath / _path; + if (fs::is_directory(fullpath)) + { + test_suite* sub_suite = BOOST_TEST_SUITE(_path.filename().string()); + for (auto const& entry: boost::iterator_range( + fs::directory_iterator(fullpath), + fs::directory_iterator() + )) + numTestsAdded += registerTests(*sub_suite, _basepath, _path / entry.path().filename()); + _suite.add(sub_suite); + } + else + { + _suite.add(make_test_case( + [fullpath] + { + std::stringstream errorStream; + if (!SyntaxTest(fullpath.string()).run(errorStream, "")) + BOOST_ERROR("Test expectation mismatch.\n" + errorStream.str()); + }, + _path.stem().string(), + _path.string(), + 0 + )); + numTestsAdded = 1; + } + return numTestsAdded; +} diff --git a/test/libsolidity/SyntaxTester.h b/test/libsolidity/SyntaxTest.h similarity index 54% rename from test/libsolidity/SyntaxTester.h rename to test/libsolidity/SyntaxTest.h index d61668aa2..4379c77bc 100644 --- a/test/libsolidity/SyntaxTester.h +++ b/test/libsolidity/SyntaxTest.h @@ -18,10 +18,16 @@ #pragma once #include -#include -#include +#include + +#include #include +#include +#include +#include +#include + namespace dev { namespace solidity @@ -29,18 +35,41 @@ namespace solidity namespace test { -class SyntaxTester: public AnalysisFramework +struct SyntaxTestExpectation +{ + std::string type; + std::string message; +}; + + +class SyntaxTest: AnalysisFramework { public: - static void registerTests(); -private: + SyntaxTest(std::string const& _filename); + + bool run(std::ostream& _stream, std::string const& _indent); + + void printExpected(std::ostream& _stream, std::string const& _indent) const; + void printErrorList( + std::ostream& _stream, + ErrorList const& _errors, + std::string const& _indent + ) const; + static int registerTests( boost::unit_test::test_suite& _suite, boost::filesystem::path const& _basepath, boost::filesystem::path const& _path ); +private: + bool matchesExpectations(ErrorList const& _errors) const; static std::string errorMessage(Error const& _e); - void runTest(SyntaxTest const& _test); + static std::string parseSource(std::istream& _stream); + static std::vector parseExpectations(std::istream& _stream); + + std::string m_source; + std::vector m_expectations; + ErrorList m_errorList; }; } diff --git a/test/libsolidity/SyntaxTestParser.cpp b/test/libsolidity/SyntaxTestParser.cpp deleted file mode 100644 index c37caca53..000000000 --- a/test/libsolidity/SyntaxTestParser.cpp +++ /dev/null @@ -1,97 +0,0 @@ -/* - This file is part of solidity. - - solidity is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - solidity is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with solidity. If not, see . -*/ - -#include -#include -#include -#include -#include -#include - -using namespace dev; -using namespace solidity; -using namespace dev::solidity::test; -using namespace std; - -template -void skipWhitespace(IteratorType& it, IteratorType end) -{ - while (it != end && isspace(*it)) - ++it; -} - -template -void skipSlashes(IteratorType& it, IteratorType end) -{ - while (it != end && *it == '/') - ++it; -} - -std::string SyntaxTestParser::parseSource(std::istream& _stream) -{ - std::string source; - string line; - string const delimiter("// ----"); - while (getline(_stream, line)) - if (boost::algorithm::starts_with(line, delimiter)) - break; - else - source += line + "\n"; - return source; -} - -std::vector SyntaxTestParser::parseExpectations(std::istream& _stream) -{ - std::vector expectations; - std::string line; - while (getline(_stream, line)) - { - auto it = line.begin(); - - skipSlashes(it, line.end()); - skipWhitespace(it, line.end()); - - if (it == line.end()) continue; - - auto typeBegin = it; - while (it != line.end() && *it != ':') - ++it; - string errorType(typeBegin, it); - - // skip colon - if (it != line.end()) it++; - - skipWhitespace(it, line.end()); - - string errorMessage(it, line.end()); - expectations.emplace_back(SyntaxTestExpectation{move(errorType), move(errorMessage)}); - } - return expectations; -} - -SyntaxTest SyntaxTestParser::parse(string const& _filename) -{ - ifstream file(_filename); - if (!file) - BOOST_THROW_EXCEPTION(runtime_error("cannot open test contract: \"" + _filename + "\"")); - file.exceptions(ios::badbit); - - SyntaxTest result; - result.source = parseSource(file); - result.expectations = parseExpectations(file); - return result; -} diff --git a/test/libsolidity/SyntaxTestParser.h b/test/libsolidity/SyntaxTestParser.h deleted file mode 100644 index 9e295a0b1..000000000 --- a/test/libsolidity/SyntaxTestParser.h +++ /dev/null @@ -1,57 +0,0 @@ -/* - This file is part of solidity. - - solidity is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - solidity is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with solidity. If not, see . -*/ - -#pragma once - -#include -#include -#include -#include - -namespace dev -{ -namespace solidity -{ -namespace test -{ - -struct SyntaxTestExpectation -{ - std::string type; - std::string message; -}; - -struct SyntaxTest -{ - std::string source; - std::vector expectations; -}; - -class SyntaxTestParser -{ -public: - SyntaxTestParser() = default; - - SyntaxTest parse(std::string const& _filename); -private: - std::string parseSource(std::istream& _stream); - std::vector parseExpectations(std::istream& _stream); -}; - -} -} -} diff --git a/test/libsolidity/SyntaxTester.cpp b/test/libsolidity/SyntaxTester.cpp deleted file mode 100644 index 886bc3bfe..000000000 --- a/test/libsolidity/SyntaxTester.cpp +++ /dev/null @@ -1,147 +0,0 @@ -/* - This file is part of solidity. - - solidity is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - solidity is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with solidity. If not, see . -*/ - -#include -#include -#include -#include - -using namespace dev; -using namespace solidity; -using namespace dev::solidity::test; -using namespace std; -using namespace boost::unit_test; -namespace fs = boost::filesystem; - -#if BOOST_VERSION < 105900 -test_case *make_test_case( - function const& _fn, - string const& _name, - string const&, // _filename - size_t // _line -) -{ - return make_test_case(_fn, _name); -} -#endif - - -void SyntaxTester::runTest(SyntaxTest const& _test) -{ - vector unexpectedErrors; - auto expectations = _test.expectations; - auto errorList = parseAnalyseAndReturnError(_test.source, true, true, true).second; - - bool errorsMatch = true; - - if (errorList.size() != expectations.size()) - errorsMatch = false; - else - { - for (size_t i = 0; i < errorList.size(); i++) - { - if ( - !(errorList[i]->typeName() == expectations[i].type) || - !(errorMessage(*errorList[i]) == expectations[i].message) - ) - { - errorsMatch = false; - break; - } - } - } - - if (!errorsMatch) - { - string msg = "Test expectation mismatch.\nExpected result:\n"; - if (expectations.empty()) - msg += "\tSuccess\n"; - else - for (auto const& expectation: expectations) - msg += "\t" + expectation.type + ": " + expectation.message + "\n"; - msg += "Obtained result:\n"; - if (errorList.empty()) - msg += "\tSuccess\n"; - else - for (auto const& error: errorList) - msg += "\t" + error->typeName() + ": " + errorMessage(*error) + "\n"; - BOOST_ERROR(msg); - } -} - -std::string SyntaxTester::errorMessage(Error const& _e) -{ - if (_e.comment()) - return boost::replace_all_copy(*_e.comment(), "\n", "\\n"); - else - return "NONE"; -} - -int SyntaxTester::registerTests( - test_suite& _suite, - fs::path const& _basepath, - fs::path const& _path -) -{ - - int numTestsAdded = 0; - fs::path fullpath = _basepath / _path; - if (fs::is_directory(fullpath)) - { - test_suite* sub_suite = BOOST_TEST_SUITE(_path.filename().string()); - for (auto const& entry: boost::iterator_range( - fs::directory_iterator(fullpath), - fs::directory_iterator() - )) - numTestsAdded += registerTests(*sub_suite, _basepath, _path / entry.path().filename()); - _suite.add(sub_suite); - } - else - { - _suite.add(make_test_case( - [fullpath] { SyntaxTester().runTest(SyntaxTestParser().parse(fullpath.string())); }, - _path.stem().string(), - _path.string(), - 0 - )); - numTestsAdded = 1; - } - return numTestsAdded; -} - -void SyntaxTester::registerTests() -{ - if(dev::test::Options::get().testPath.empty()) - throw runtime_error( - "No path to the test files was specified. " - "Use the --testpath command line option or " - "the ETH_TEST_PATH environment variable." - ); - auto testPath = fs::path(dev::test::Options::get().testPath); - - if (fs::exists(testPath) && fs::is_directory(testPath)) - { - int numTestsAdded = registerTests( - framework::master_test_suite(), - testPath / "libsolidity", - "syntaxTests" - ); - solAssert(numTestsAdded > 0, "no syntax tests found in libsolidity/syntaxTests"); - } - else - solAssert(false, "libsolidity/syntaxTests directory not found"); -} From 7091b6c8b5a91a13ff02255cc0bca08266527e4f Mon Sep 17 00:00:00 2001 From: Daniel Kirchner Date: Tue, 13 Mar 2018 12:30:56 +0100 Subject: [PATCH 4/4] Minor adjustments. --- test/libsolidity/SyntaxTest.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/libsolidity/SyntaxTest.cpp b/test/libsolidity/SyntaxTest.cpp index 8cbe2f36c..f1c60458a 100644 --- a/test/libsolidity/SyntaxTest.cpp +++ b/test/libsolidity/SyntaxTest.cpp @@ -48,7 +48,7 @@ SyntaxTest::SyntaxTest(string const& _filename) { ifstream file(_filename); if (!file) - BOOST_THROW_EXCEPTION(runtime_error("cannot open test contract: \"" + _filename + "\"")); + BOOST_THROW_EXCEPTION(runtime_error("Cannot open test contract: \"" + _filename + "\".")); file.exceptions(ios::badbit); m_source = parseSource(file); @@ -60,7 +60,7 @@ bool SyntaxTest::run(ostream& _stream, string const& _indent) m_errorList = parseAnalyseAndReturnError(m_source, true, true, true).second; if (!matchesExpectations(m_errorList)) { - std::string nextIndentLevel = _indent.empty() ? "\t" : _indent + _indent; + std::string nextIndentLevel = _indent + "\t"; _stream << _indent << "Expected result:" << endl; printExpected(_stream, nextIndentLevel); _stream << _indent << "Obtained result:\n"; @@ -99,8 +99,8 @@ bool SyntaxTest::matchesExpectations(ErrorList const& _errorList) const else for (size_t i = 0; i < _errorList.size(); i++) if ( - !(_errorList[i]->typeName() == m_expectations[i].type) || - !(errorMessage(*_errorList[i]) == m_expectations[i].message) + (_errorList[i]->typeName() != m_expectations[i].type) || + (errorMessage(*_errorList[i]) != m_expectations[i].message) ) return false; return true;