Merge pull request #6681 from rocky/recoveringParser1

Add Steve Johnson-style parser recovery rules:

test/InteractiveTests.h

@@ -56,6 +56,7 @@ Testsuite const g_interactiveTestsuites[] = {
 	{"Yul Interpreter",     "libyul",      "yulInterpreterTests", false, false, &yul::test::YulInterpreterTest::create},
 	{"Yul Object Compiler", "libyul",      "objectCompiler",      false, false, &yul::test::ObjectCompilerTest::create},
 	{"Syntax",              "libsolidity", "syntaxTests",         false, false, &SyntaxTest::create},
+	{"ErrorRecovery",       "libsolidity", "errorRecoveryTests",  false, false, &SyntaxTest::createErrorRecovery},
 	{"Semantic",            "libsolidity", "semanticTests",       false, true,  &SemanticTest::create},
 	{"JSON AST",            "libsolidity", "ASTJSON",             false, false, &ASTJSONTest::create},
 	{"SMT Checker",         "libsolidity", "smtCheckerTests",     true,  false, &SyntaxTest::create},
@@ -66,4 +67,3 @@ Testsuite const g_interactiveTestsuites[] = {
 }
 }
 }
-

test/liblangutil/CharStream.cpp

@@ -0,0 +1,53 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/**
+ * @author Rocky Bernstein <rocky.bernstein@consensys.net>
+ * @date 2019
+ * Unit tests for the CharStream class.
+ */
+
+#include <liblangutil/CharStream.h>
+#include <liblangutil/Exceptions.h>
+
+#include <test/Options.h>
+
+namespace langutil
+{
+namespace test
+{
+
+BOOST_AUTO_TEST_SUITE(CharStreamtest)
+
+BOOST_AUTO_TEST_CASE(test_fail)
+{
+	auto const source = std::make_shared<CharStream>("now is the time for testing", "source");
+
+	BOOST_CHECK('n' == source->get());
+	BOOST_CHECK('n' == source->get());
+	BOOST_CHECK('o' == source->advanceAndGet());
+	BOOST_CHECK('n' == source->rollback(1));
+	BOOST_CHECK('w' == source->setPosition(2));
+	BOOST_REQUIRE_THROW(
+		source->setPosition(200),
+		::langutil::InternalCompilerError
+	);
+}
+
+BOOST_AUTO_TEST_SUITE_END()
+
+}
+} // end namespaces

test/libsolidity/AnalysisFramework.cpp

@@ -44,12 +44,15 @@ AnalysisFramework::parseAnalyseAndReturnError(
 	string const& _source,
 	bool _reportWarnings,
 	bool _insertVersionPragma,
-	bool _allowMultipleErrors
+	bool _allowMultipleErrors,
+	bool _allowRecoveryErrors
 )
 {
 	compiler().reset();
 	compiler().setSources({{"", _insertVersionPragma ? "pragma solidity >=0.0;\n" + _source : _source}});
 	compiler().setEVMVersion(dev::test::Options::get().evmVersion());
+	compiler().setParserErrorRecovery(_allowRecoveryErrors);
+	_allowMultipleErrors = _allowMultipleErrors || _allowRecoveryErrors;
 	if (!compiler().parse())
 	{
 		BOOST_FAIL("Parsing contract failed in analysis test suite:" + formatErrors());

test/libsolidity/AnalysisFramework.h

@@ -50,7 +50,8 @@ protected:
 		std::string const& _source,
 		bool _reportWarnings = false,
 		bool _insertVersionPragma = true,
-		bool _allowMultipleErrors = false
+		bool _allowMultipleErrors = false,
+		bool _allowRecoveryErrors = false
 	);
 	virtual ~AnalysisFramework() = default;
 

test/libsolidity/SMTChecker.cpp

@@ -42,14 +42,16 @@ protected:
 		std::string const& _source,
 		bool _reportWarnings = false,
 		bool _insertVersionPragma = true,
-		bool _allowMultipleErrors = false
+		bool _allowMultipleErrors = false,
+		bool _allowRecoveryErrors = false
 	)
 	{
 		return AnalysisFramework::parseAnalyseAndReturnError(
 			"pragma experimental SMTChecker;\n" + _source,
 			_reportWarnings,
 			_insertVersionPragma,
-			_allowMultipleErrors
+			_allowMultipleErrors,
+			_allowRecoveryErrors
 		);
 	}
 };

test/libsolidity/SyntaxTest.cpp

@@ -52,7 +52,7 @@ int parseUnsignedInteger(string::iterator& _it, string::iterator _end)
 
 }
 
-SyntaxTest::SyntaxTest(string const& _filename, langutil::EVMVersion _evmVersion): m_evmVersion(_evmVersion)
+SyntaxTest::SyntaxTest(string const& _filename, langutil::EVMVersion _evmVersion, bool _errorRecovery): m_evmVersion(_evmVersion)
 {
 	ifstream file(_filename);
 	if (!file)
@@ -67,6 +67,7 @@ SyntaxTest::SyntaxTest(string const& _filename, langutil::EVMVersion _evmVersion
 		m_settings.erase("optimize-yul");
 	}
 	m_expectations = parseExpectations(file);
+	m_errorRecovery = _errorRecovery;
 }
 
 TestCase::TestResult SyntaxTest::run(ostream& _stream, string const& _linePrefix, bool _formatted)
@@ -75,6 +76,7 @@ TestCase::TestResult SyntaxTest::run(ostream& _stream, string const& _linePrefix
 	compiler().reset();
 	compiler().setSources({{"", versionPragma + m_source}});
 	compiler().setEVMVersion(m_evmVersion);
+	compiler().setParserErrorRecovery(m_errorRecovery);
 	compiler().setOptimiserSettings(
 		m_optimiseYul ?
 		OptimiserSettings::full() :

test/libsolidity/SyntaxTest.h

@@ -54,8 +54,14 @@ class SyntaxTest: AnalysisFramework, public EVMVersionRestrictedTestCase
 {
 public:
 	static std::unique_ptr<TestCase> create(Config const& _config)
-	{ return std::make_unique<SyntaxTest>(_config.filename, _config.evmVersion); }
-	SyntaxTest(std::string const& _filename, langutil::EVMVersion _evmVersion);
+	{
+		return std::make_unique<SyntaxTest>(_config.filename, _config.evmVersion, false);
+	}
+	static std::unique_ptr<TestCase> createErrorRecovery(Config const& _config)
+	{
+		return std::make_unique<SyntaxTest>(_config.filename, _config.evmVersion, true);
+	}
+	SyntaxTest(std::string const& _filename, langutil::EVMVersion _evmVersion, bool _errorRecovery = false);
 
 	TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override;
 
@@ -84,6 +90,7 @@ protected:
 	std::vector<SyntaxTestError> m_errorList;
 	bool m_optimiseYul = false;
 	langutil::EVMVersion const m_evmVersion;
+	bool m_errorRecovery = false;
 };
 
 }

test/libsolidity/errorRecoveryTests/constructor_recovers.sol

@@ -0,0 +1,20 @@
+pragma solidity >=0.0.0;
+
+contract Error1 {
+  constructor() public {
+    balances[tx.origin] = ; // missing RHS.
+  }
+
+  // Without error recovery we stop due to the above error.
+  // Error recovery however recovers at the above ';'
+  // There should be an AST for the above, albeit with error
+  // nodes.
+
+  // This function parses properly and should give AST info.
+  function five() public view returns(uint) {
+    return 5;
+  }
+}
+// ----
+// ParserError: (95-96): Expected primary expression.
+// Warning: (95-96): Recovered in Statement at ';'.

test/libsolidity/errorRecoveryTests/contract_recovery.sol

@@ -0,0 +1,7 @@
+contract Errort6 {
+  using foo for  ; // missing type name
+}
+
+// ----
+// ParserError: (36-37): Expected type name
+// Warning: (59-60): Recovered in ContractDefinition at '}'.

test/libsolidity/errorRecoveryTests/do_not_delete_at_error.sol

@@ -0,0 +1,13 @@
+pragma solidity >=0.0.0;
+
+// Example to show why deleting the token at the
+// is bad when error recovery is in effect. Here, ")" is missing
+// and there is a ";" instead. That causes us to
+// not be able to synchronize to ';'. Advance again and
+// '}' is deleted and then we can't synchronize the contract.
+// There should be an an AST created this contract (with errors).
+contract Error2 {
+	mapping (address => uint balances; // missing ) before "balances"
+}
+// ----
+// ParserError: (417-425): Expected ')' but got identifier

test/libsolidity/errorRecoveryTests/error_to_eos.sol

@@ -0,0 +1,23 @@
+// Example which where scanning hits EOS, so we reset.
+// Here we recover in the contractDefinition.
+// There should be an an AST created this contract (with errors).
+contract Error2 {
+  mapping (address => uint balances) // missing ;
+}
+
+// There is no error in this contract
+contract SendCoin {
+  function sendCoin(address receiver, uint amount) public returns(bool sufficient) {
+    if (balances[msg.sender] < amount) return false;
+    balances[msg.sender] -= amount;
+    balances[receiver] += amount;
+    emit Transfer(msg.sender, receiver, amount);
+    return true;
+  }
+}
+
+// ----
+// ParserError: (212-220): Expected ')' but got identifier
+// ParserError: (220-221): Expected ';' but got ')'
+// ParserError: (220-221): Function, variable, struct or modifier declaration expected.
+// Warning: (235-236): Recovered in ContractDefinition at '}'.

test/libsolidity/errorRecoveryTests/missing_rhs.sol

@@ -0,0 +1,11 @@
+pragma solidity >=0.0.0;
+
+contract Error3 {
+	constructor() public {
+	    balances[tx.origin] = ; // missing RHS.
+	}
+
+}
+// ----
+// ParserError: (95-96): Expected primary expression.
+// Warning: (95-96): Recovered in Statement at ';'.

test/libsolidity/errorRecoveryTests/multiple_errors.sol

@@ -0,0 +1,29 @@
+// An example with multiple errors.
+// Most are caught by inserting an expected token.
+// However some us S C Johnson recovery to
+// skip over tokens.
+
+pragma solidity >=0.0.0;
+
+contract Error4 {
+  constructor() public {
+    balances[tx.origin] = 1 2; // missing operator
+  }
+
+  function sendCoin(address receiver, uint amount) public returns(bool sufficient) {
+    if (balances[msg.sender] < amount) return false;
+    balances[msg.sender] -= amount   // Missing ";"
+    balances[receiver] += amount   // Another missing ";"
+    emit Transfer(msg.sender  // truncated line
+    return true;
+  }
+
+
+}
+// ----
+// ParserError: (249-250): Expected ';' but got 'Number'
+// ParserError: (471-479): Expected ';' but got identifier
+// ParserError: (529-533): Expected ';' but got 'emit'
+// ParserError: (577-583): Expected ',' but got 'return'
+// ParserError: (577-583): Expected primary expression.
+// Warning: (588-589): Recovered in Statement at ';'.