Merge pull request #9252 from ethereum/develop

Merge develop into breaking.

Changelog.md

@@ -34,6 +34,8 @@ Compiler Features:
 Bugfixes:
  * NatSpec: Do not consider ``////`` and ``/***`` as NatSpec comments.
  * Type Checker: Fix internal error related to ``using for`` applied to non-libraries.
+ * Type Checker: Do not disallow assigning to calldata variables.
+ * Wasm backend: Fix code generation for for-loops with pre statements.
  * Yul: Fix source location of variable multi-assignment.
 
 

docs/Solidity.g4

@@ -11,23 +11,11 @@ sourceUnit
   : (pragmaDirective | importDirective | structDefinition | enumDefinition | contractDefinition)* EOF ;
 
 pragmaDirective
-  : 'pragma' pragmaName pragmaValue ';' ;
+  : 'pragma' pragmaName ( ~';' )* ';' ;
 
 pragmaName
   : identifier ;
 
-pragmaValue
-  : version | expression ;
-
-version
-  : versionConstraint versionConstraint? ;
-
-versionConstraint
-  : versionOperator? VersionLiteral ;
-
-versionOperator
-  : '^' | '~' | '>=' | '>' | '<' | '<=' | '=' ;
-
 importDirective
   : 'import' StringLiteralFragment ('as' identifier)? ';'
   | 'import' ('*' | identifier) ('as' identifier)? 'from' StringLiteralFragment ';'
@@ -473,9 +461,6 @@ fragment
 SingleQuotedStringCharacter
   : ~['\r\n\\] | ('\\' .) ;
 
-VersionLiteral
-  : [0-9]+ ( '.' [0-9]+ ('.' [0-9]+)? )? ;
-
 WS
   : [ \t\r\n\u000C]+ -> skip ;
 

libsolidity/ast/AST.cpp

@@ -492,12 +492,7 @@ DeclarationAnnotation& Declaration::annotation() const
 bool VariableDeclaration::isLValue() const
 {
 	// Constant declared variables are Read-Only
-	if (isConstant())
-		return false;
-	// External function arguments of reference type are Read-Only
-	if (isExternalCallableParameter() && dynamic_cast<ReferenceType const*>(type()))
-		return false;
-	return true;
+	return !isConstant();
 }
 
 bool VariableDeclaration::isLocalVariable() const

libyul/backends/wasm/WasmCodeTransform.cpp

@@ -257,9 +257,10 @@ wasm::Expression WasmCodeTransform::operator()(ForLoop const& _for)
 	YulString eqz_instruction = YulString(conditionType.str() + ".eqz");
 	yulAssert(WasmDialect::instance().builtin(eqz_instruction), "");
 
+	std::vector<wasm::Expression> statements = visit(_for.pre.statements);
+
 	wasm::Loop loop;
 	loop.labelName = newLabel();
-	loop.statements = visit(_for.pre.statements);
 	loop.statements.emplace_back(wasm::BranchIf{wasm::Label{breakLabel}, make_unique<wasm::Expression>(
 		wasm::BuiltinCall{eqz_instruction.str(), make_vector<wasm::Expression>(
 			visitReturnByValue(*_for.condition)
@@ -269,7 +270,8 @@ wasm::Expression WasmCodeTransform::operator()(ForLoop const& _for)
 	loop.statements += visit(_for.post.statements);
 	loop.statements.emplace_back(wasm::Branch{wasm::Label{loop.labelName}});
 
-	return { wasm::Block{breakLabel, make_vector<wasm::Expression>(move(loop))} };
+	statements += make_vector<wasm::Expression>(move(loop));
+	return wasm::Block{breakLabel, move(statements)};
 }
 
 wasm::Expression WasmCodeTransform::operator()(Break const&)

test/CommonSyntaxTest.cpp

@@ -58,10 +58,10 @@ int parseUnsignedInteger(string::iterator& _it, string::iterator _end)
 
 CommonSyntaxTest::CommonSyntaxTest(string const& _filename, langutil::EVMVersion _evmVersion):
 	EVMVersionRestrictedTestCase(_filename),
+	m_sources(m_reader.sources().sources),
+	m_expectations(parseExpectations(m_reader.stream())),
 	m_evmVersion(_evmVersion)
 {
-	m_sources = m_reader.sources();
-	m_expectations = parseExpectations(m_reader.stream());
 }
 
 TestCase::TestResult CommonSyntaxTest::run(ostream& _stream, string const& _linePrefix, bool _formatted)
@@ -94,12 +94,10 @@ void CommonSyntaxTest::printSource(ostream& _stream, string const& _linePrefix,
 	if (m_sources.empty())
 		return;
 
-	bool outputSourceNames = true;
-	if (m_sources.size() == 1 && m_sources.begin()->first.empty())
-		outputSourceNames = false;
+	bool outputSourceNames = (m_sources.size() != 1 || !m_sources.begin()->first.empty());
 
-	if (_formatted)
-		for (auto const& [name, source]: m_sources)
+	for (auto const& [name, source]: m_sources)
+		if (_formatted)
 		{
 			if (source.empty())
 				continue;
@@ -139,8 +137,7 @@ void CommonSyntaxTest::printSource(ostream& _stream, string const& _linePrefix,
 			}
 			_stream << formatting::RESET;
 		}
-	else
-		for (auto const& [name, source]: m_sources)
+		else
 		{
 			if (outputSourceNames)
 				_stream << _linePrefix << "==== Source: " + name << " ====" << endl;

test/ExecutionFramework.h

@@ -61,22 +61,28 @@ public:
 	virtual ~ExecutionFramework() = default;
 
 	virtual bytes const& compileAndRunWithoutCheck(
-		std::string const& _sourceCode,
+		std::map<std::string, std::string> const& _sourceCode,
 		u256 const& _value = 0,
 		std::string const& _contractName = "",
-		bytes const& _arguments = bytes(),
-		std::map<std::string, Address> const& _libraryAddresses = std::map<std::string, Address>()
+		bytes const& _arguments = {},
+		std::map<std::string, Address> const& _libraryAddresses = {}
 	) = 0;
 
 	bytes const& compileAndRun(
 		std::string const& _sourceCode,
 		u256 const& _value = 0,
 		std::string const& _contractName = "",
-		bytes const& _arguments = bytes(),
-		std::map<std::string, Address> const& _libraryAddresses = std::map<std::string, Address>()
+		bytes const& _arguments = {},
+		std::map<std::string, Address> const& _libraryAddresses = {}
 	)
 	{
-		compileAndRunWithoutCheck(_sourceCode, _value, _contractName, _arguments, _libraryAddresses);
+		compileAndRunWithoutCheck(
+			{{"", _sourceCode}},
+			_value,
+			_contractName,
+			_arguments,
+			_libraryAddresses
+		);
 		BOOST_REQUIRE(m_transactionSuccessful);
 		BOOST_REQUIRE(!m_output.empty());
 		return m_output;
@@ -293,4 +299,3 @@ protected:
 
 
 } // end namespaces
-

test/TestCaseReader.cpp

@@ -37,11 +37,11 @@ TestCaseReader::TestCaseReader(string const& _filename):
 	m_unreadSettings = m_settings;
 }
 
-string const& TestCaseReader::source()
+string const& TestCaseReader::source() const
 {
-	if (m_sources.size() != 1)
+	if (m_sources.sources.size() != 1)
 		BOOST_THROW_EXCEPTION(runtime_error("Expected single source definition, but got multiple sources."));
-	return m_sources.begin()->second;
+	return m_sources.sources.at(m_sources.mainSourceFile);
 }
 
 string TestCaseReader::simpleExpectations()
@@ -93,7 +93,7 @@ void TestCaseReader::ensureAllSettingsRead() const
 		);
 }
 
-pair<map<string, string>, size_t> TestCaseReader::parseSourcesAndSettingsWithLineNumber(istream& _stream)
+pair<SourceMap, size_t> TestCaseReader::parseSourcesAndSettingsWithLineNumber(istream& _stream)
 {
 	map<string, string> sources;
 	string currentSourceName;
@@ -145,8 +145,9 @@ pair<map<string, string>, size_t> TestCaseReader::parseSourcesAndSettingsWithLin
 		else
 			throw runtime_error(string("Expected \"//\" or \"// ---\" to terminate settings and source."));
 	}
+	// Register the last source as the main one
 	sources[currentSourceName] = currentSource;
-	return { sources, lineNumber };
+	return {{move(sources), move(currentSourceName)}, lineNumber};
 }
 
 string TestCaseReader::parseSimpleExpectations(istream& _file)

test/TestCaseReader.h

@@ -23,6 +23,16 @@
 
 namespace solidity::frontend::test
 {
+
+/**
+ * A map for registering source names that also contains the main source name in a test case.
+ */
+struct SourceMap
+{
+	std::map<std::string, std::string> sources;
+	std::string mainSourceFile;
+};
+
 /**
  * A reader for test case data file, which parses source, settings and (optionally) simple expectations.
  */
@@ -32,10 +42,10 @@ public:
 	TestCaseReader() = default;
 	explicit TestCaseReader(std::string const& _filename);
 
-	std::map<std::string, std::string> const& sources() { return m_sources; }
-	std::string const& source();
-	std::size_t lineNumber() { return m_lineNumber; }
-	std::map<std::string, std::string> const& settings() { return m_settings; }
+	SourceMap const& sources() const { return m_sources; }
+	std::string const& source() const;
+	std::size_t lineNumber() const { return m_lineNumber; }
+	std::map<std::string, std::string> const& settings() const { return m_settings; }
 	std::ifstream& stream() { return m_file; }
 
 	std::string simpleExpectations();
@@ -47,11 +57,11 @@ public:
 	void ensureAllSettingsRead() const;
 
 private:
-	std::pair<std::map<std::string, std::string>, std::size_t> parseSourcesAndSettingsWithLineNumber(std::istream& _file);
+	std::pair<SourceMap, std::size_t> parseSourcesAndSettingsWithLineNumber(std::istream& _file);
 	static std::string parseSimpleExpectations(std::istream& _file);
 
 	std::ifstream m_file;
-	std::map<std::string, std::string> m_sources;
+	SourceMap m_sources;
 	std::size_t m_lineNumber = 0;
 	std::map<std::string, std::string> m_settings;
 	std::map<std::string, std::string> m_unreadSettings; ///< tracks which settings are left unread

test/libsolidity/ABIEncoderTests.cpp

@@ -777,7 +777,7 @@ BOOST_AUTO_TEST_CASE(struct_in_constructor_data_short)
 
 	NEW_ENCODER(
 		BOOST_CHECK(
-			compileAndRunWithoutCheck(sourceCode, 0, "C", encodeArgs(0x20, 0x60, 0x03, 0x80, 0x00)).empty()
+			compileAndRunWithoutCheck({{"", sourceCode}}, 0, "C", encodeArgs(0x20, 0x60, 0x03, 0x80, 0x00)).empty()
 		);
 	)
 }

test/libsolidity/SemanticTest.cpp

@@ -42,11 +42,10 @@ namespace fs = boost::filesystem;
 SemanticTest::SemanticTest(string const& _filename, langutil::EVMVersion _evmVersion, bool enforceViaYul):
 	SolidityExecutionFramework(_evmVersion),
 	EVMVersionRestrictedTestCase(_filename),
+	m_sources(m_reader.sources()),
+	m_lineOffset(m_reader.lineNumber()),
 	m_enforceViaYul(enforceViaYul)
 {
-	m_source = m_reader.source();
-	m_lineOffset = m_reader.lineNumber();
-
 	string choice = m_reader.stringSetting("compileViaYul", "false");
 	if (choice == "also")
 	{
@@ -239,12 +238,48 @@ TestCase::TestResult SemanticTest::run(ostream& _stream, string const& _linePref
 	return TestResult::Success;
 }
 
-void SemanticTest::printSource(ostream& _stream, string const& _linePrefix, bool) const
+void SemanticTest::printSource(ostream& _stream, string const& _linePrefix, bool _formatted) const
 {
-	stringstream stream(m_source);
-	string line;
-	while (getline(stream, line))
-		_stream << _linePrefix << line << endl;
+	if (m_sources.sources.empty())
+		return;
+
+	bool outputNames = (m_sources.sources.size() != 1 || !m_sources.sources.begin()->first.empty());
+
+	for (auto const& [name, source]: m_sources.sources)
+		if (_formatted)
+		{
+			if (source.empty())
+				continue;
+
+			if (outputNames)
+				_stream << _linePrefix << formatting::CYAN << "==== Source: " << name << " ====" << formatting::RESET << endl;
+			vector<char const*> sourceFormatting(source.length(), formatting::RESET);
+
+			_stream << _linePrefix << sourceFormatting.front() << source.front();
+			for (size_t i = 1; i < source.length(); i++)
+			{
+				if (sourceFormatting[i] != sourceFormatting[i - 1])
+					_stream << sourceFormatting[i];
+				if (source[i] != '\n')
+					_stream << source[i];
+				else
+				{
+					_stream << formatting::RESET << endl;
+					if (i + 1 < source.length())
+						_stream << _linePrefix << sourceFormatting[i];
+				}
+			}
+			_stream << formatting::RESET;
+		}
+		else
+		{
+			if (outputNames)
+				_stream << _linePrefix << "==== Source: " + name << " ====" << endl;
+			stringstream stream(source);
+			string line;
+			while (getline(stream, line))
+				_stream << _linePrefix << line << endl;
+		}
 }
 
 void SemanticTest::printUpdatedExpectations(ostream& _stream, string const&) const
@@ -276,6 +311,6 @@ void SemanticTest::parseExpectations(istream& _stream)
 
 bool SemanticTest::deploy(string const& _contractName, u256 const& _value, bytes const& _arguments, map<string, solidity::test::Address> const& _libraries)
 {
-	auto output = compileAndRunWithoutCheck(m_source, _value, _contractName, _arguments, _libraries);
+	auto output = compileAndRunWithoutCheck(m_sources.sources, _value, _contractName, _arguments, _libraries);
 	return !output.empty() && m_transactionSuccessful;
 }

test/libsolidity/SemanticTest.h

@@ -58,9 +58,8 @@ public:
 	/// Compiles and deploys currently held source.
 	/// Returns true if deployment was successful, false otherwise.
 	bool deploy(std::string const& _contractName, u256 const& _value, bytes const& _arguments, std::map<std::string, solidity::test::Address> const& _libraries = {});
-
 private:
-	std::string m_source;
+	SourceMap m_sources;
 	std::size_t m_lineOffset;
 	std::vector<TestFunctionCall> m_tests;
 	bool m_runWithYul = false;

test/libsolidity/SolidityEndToEndTest.cpp

@@ -1723,7 +1723,7 @@ BOOST_AUTO_TEST_CASE(internal_constructor)
 			constructor() internal {}
 		}
 	)";
-	BOOST_CHECK(compileAndRunWithoutCheck(sourceCode, 0, "C").empty());
+	BOOST_CHECK(compileAndRunWithoutCheck({{"", sourceCode}}, 0, "C").empty());
 }
 
 BOOST_AUTO_TEST_CASE(default_fallback_throws)
@@ -3618,7 +3618,7 @@ BOOST_AUTO_TEST_CASE(evm_exceptions_in_constructor_out_of_baund)
 			}
 		}
 	)";
-	ABI_CHECK(compileAndRunWithoutCheck(sourceCode, 0, "A"), encodeArgs());
+	ABI_CHECK(compileAndRunWithoutCheck({{"", sourceCode}}, 0, "A"), encodeArgs());
 	BOOST_CHECK(!m_transactionSuccessful);
 }
 

test/libsolidity/SolidityExecutionFramework.cpp

@@ -31,22 +31,18 @@ using namespace solidity::frontend;
 using namespace solidity::frontend::test;
 using namespace std;
 
-bytes SolidityExecutionFramework::compileContract(
-	string const& _sourceCode,
+bytes SolidityExecutionFramework::multiSourceCompileContract(
+	map<string, string> const& _sourceCode,
 	string const& _contractName,
 	map<string, Address> const& _libraryAddresses
 )
 {
-	// Silence compiler version warning
-	std::string sourceCode = "pragma solidity >=0.0;\n";
-	if (
-		solidity::test::CommonOptions::get().useABIEncoderV2 &&
-		_sourceCode.find("pragma experimental ABIEncoderV2;") == std::string::npos
-	)
-		sourceCode += "pragma experimental ABIEncoderV2;\n";
-	sourceCode += _sourceCode;
+	map<string, string> sourcesWithPreamble = _sourceCode;
+	for (auto& entry: sourcesWithPreamble)
+		entry.second = addPreamble(entry.second);
+
 	m_compiler.reset();
-	m_compiler.setSources({{"", sourceCode}});
+	m_compiler.setSources(sourcesWithPreamble);
 	m_compiler.setLibraries(_libraryAddresses);
 	m_compiler.setRevertStringBehaviour(m_revertStrings);
 	m_compiler.setEVMVersion(m_evmVersion);
@@ -85,3 +81,28 @@ bytes SolidityExecutionFramework::compileContract(
 		cout << "metadata: " << m_compiler.metadata(contractName) << endl;
 	return obj.bytecode;
 }
+
+bytes SolidityExecutionFramework::compileContract(
+	string const& _sourceCode,
+	string const& _contractName,
+	map<string, Address> const& _libraryAddresses
+)
+{
+	return multiSourceCompileContract(
+		{{"", _sourceCode}},
+		_contractName,
+		_libraryAddresses
+	);
+}
+
+string SolidityExecutionFramework::addPreamble(string const& _sourceCode)
+{
+	// Silence compiler version warning
+	string preamble = "pragma solidity >=0.0;\n";
+	if (
+		solidity::test::CommonOptions::get().useABIEncoderV2 &&
+		_sourceCode.find("pragma experimental ABIEncoderV2;") == string::npos
+	)
+		preamble += "pragma experimental ABIEncoderV2;\n";
+	return preamble + _sourceCode;
+}

test/libsolidity/SolidityExecutionFramework.h

@@ -47,14 +47,14 @@ public:
 	{}
 
 	bytes const& compileAndRunWithoutCheck(
-		std::string const& _sourceCode,
+		std::map<std::string, std::string> const& _sourceCode,
 		u256 const& _value = 0,
 		std::string const& _contractName = "",
-		bytes const& _arguments = bytes(),
-		std::map<std::string, solidity::test::Address> const& _libraryAddresses = std::map<std::string, solidity::test::Address>()
+		bytes const& _arguments = {},
+		std::map<std::string, solidity::test::Address> const& _libraryAddresses = {}
 	) override
 	{
-		bytes bytecode = compileContract(_sourceCode, _contractName, _libraryAddresses);
+		bytes bytecode = multiSourceCompileContract(_sourceCode, _contractName, _libraryAddresses);
 		sendMessage(bytecode + _arguments, true, _value);
 		return m_output;
 	}
@@ -62,16 +62,23 @@ public:
 	bytes compileContract(
 		std::string const& _sourceCode,
 		std::string const& _contractName = "",
-		std::map<std::string, solidity::test::Address> const& _libraryAddresses = std::map<std::string, solidity::test::Address>()
+		std::map<std::string, solidity::test::Address> const& _libraryAddresses = {}
 	);
 
+	bytes multiSourceCompileContract(
+		std::map<std::string, std::string> const& _sources,
+		std::string const& _contractName = "",
+		std::map<std::string, solidity::test::Address> const& _libraryAddresses = {}
+	);
+
+	/// Returns @param _sourceCode prefixed with the version pragma and the ABIEncoderV2 pragma,
+	/// the latter only if it is required.
+	static std::string addPreamble(std::string const& _sourceCode);
 protected:
 	solidity::frontend::CompilerStack m_compiler;
 	bool m_compileViaYul = false;
 	bool m_showMetadata = false;
 	RevertStrings m_revertStrings = RevertStrings::Default;
-
 };
 
 } // end namespaces
-

test/libsolidity/semanticTests/multiSource/import.sol

@@ -0,0 +1,12 @@
+==== Source: A ====
+contract A {
+	function g(uint256 x) public view returns(uint256) { return x + 1; }
+}
+==== Source: B ====
+import "A";
+contract B is A {
+	function f(uint256 x) public view returns(uint256) { return x; }
+}
+// ----
+// f(uint256): 1337 -> 1337
+// g(uint256): 1337 -> 1338

test/libsolidity/syntaxTests/lvalues/external_reference_argument.sol

@@ -4,4 +4,3 @@ contract C {
     }
 }
 // ----
-// TypeError 7128: (96-97): External function arguments of reference type are read-only.

test/libsolidity/syntaxTests/pragma/version_check.sol

@@ -0,0 +1,7 @@
+// SPDX-License-Identifier: GPL-3.0
+pragma solidity < 142857;
+pragma solidity >= 0.0;
+pragma solidity >= 0.0.0;
+
+contract C {
+}

test/libsolidity/syntaxTests/structs/memory_to_calldata.sol

@@ -5,7 +5,5 @@ contract Test {
     function g(S calldata s) external { S memory m; s = m; }
 }
 // ----
-// TypeError 7128: (114-115): External function arguments of reference type are read-only.
 // TypeError 7407: (118-122): Type struct Test.S memory is not implicitly convertible to expected type struct Test.S calldata.
-// TypeError 7128: (178-179): External function arguments of reference type are read-only.
 // TypeError 7407: (182-183): Type struct Test.S memory is not implicitly convertible to expected type struct Test.S calldata.

test/tools/isoltest.cpp

@@ -21,17 +21,13 @@
 #include <memory>
 #include <test/Common.h>
 #include <test/tools/IsolTestOptions.h>
-#include <test/libsolidity/AnalysisFramework.h>
 #include <test/InteractiveTests.h>
 #include <test/EVMHost.h>
 
-#include <boost/algorithm/string.hpp>
 #include <boost/algorithm/string/replace.hpp>
 #include <boost/filesystem.hpp>
-#include <boost/program_options.hpp>
 
 #include <cstdlib>
-#include <fstream>
 #include <iostream>
 #include <queue>
 #include <regex>