Merge remote-tracking branch 'origin/develop' into breaking

Changelog.md

@@ -28,8 +28,11 @@ AST Changes:
 
 Language Features:
  * Ability to select the abi coder using ``pragma abicoder v1`` and ``pragma abicoder v2``.
+ * Immutable variables with literal number values are considered pure.
 
 Compiler Features:
+ * Command Line Interface: Report error if file could not be read in ``--standard-json`` mode.
+ * Command Line interface: Report proper error for each output file which could not be written. Previously an exception was thrown, and execution aborted, on the first error.
  * SMTChecker: Add division by zero checks in the CHC engine.
  * SMTChecker: Support ``selector`` for expressions with value known at compile-time.
  * Command Line Interface: New option ``--model-checker-timeout`` sets a timeout in milliseconds for each individual query performed by the SMTChecker.

libsolidity/analysis/ViewPureChecker.cpp

@@ -186,7 +186,13 @@ void ViewPureChecker::endVisit(Identifier const& _identifier)
 	bool writes = _identifier.annotation().willBeWrittenTo;
 	if (VariableDeclaration const* varDecl = dynamic_cast<VariableDeclaration const*>(declaration))
 	{
-		if (varDecl->isStateVariable() && !varDecl->isConstant())
+		if (varDecl->immutable())
+		{
+			// Immutables that are assigned literals are pure.
+			if (!(varDecl->value() && varDecl->value()->annotation().type->category() == Type::Category::RationalNumber))
+				mutability = StateMutability::View;
+		}
+		else if (varDecl->isStateVariable() && !varDecl->isConstant())
 			mutability = writes ? StateMutability::NonPayable : StateMutability::View;
 	}
 	else if (MagicVariableDeclaration const* magicVar = dynamic_cast<MagicVariableDeclaration const*>(declaration))
@@ -291,7 +297,7 @@ void ViewPureChecker::reportMutability(
 		m_currentFunction->stateMutability() == StateMutability::Pure ||
 		m_currentFunction->stateMutability() == StateMutability::NonPayable,
 		""
-				);
+	);
 }
 
 ViewPureChecker::MutabilityAndLocation const& ViewPureChecker::modifierMutability(

libsolidity/codegen/YulUtilFunctions.cpp

@@ -2808,8 +2808,7 @@ string YulUtilFunctions::conversionFunction(Type const& _from, Type const& _to)
 			.render();
 		});
 	}
-
+	else if (_from.category() == Type::Category::ArraySlice)
-	if (_from.category() == Type::Category::ArraySlice)
 	{
 		solAssert(_from.isDynamicallySized(), "");
 		solAssert(_from.dataStoredIn(DataLocation::CallData), "");
@@ -2840,6 +2839,14 @@ string YulUtilFunctions::conversionFunction(Type const& _from, Type const& _to)
 			.render();
 		});
 	}
+	else if (_from.category() == Type::Category::Array)
+	{
+		solAssert(_to.category() == Type::Category::Array, "");
+		return arrayConversionFunction(
+			dynamic_cast<ArrayType const&>(_from),
+			dynamic_cast<ArrayType const&>(_to)
+		);
+	}
 
 	if (_from.sizeOnStack() != 1 || _to.sizeOnStack() != 1)
 		return conversionFunctionSpecial(_from, _to);
@@ -2946,42 +2953,6 @@ string YulUtilFunctions::conversionFunction(Type const& _from, Type const& _to)
 		case Type::Category::FixedPoint:
 			solUnimplemented("Fixed point types not implemented.");
 			break;
-		case Type::Category::Array:
-		{
-			if (_from == _to)
-				body = "converted := value";
-			else
-			{
-				ArrayType const& from = dynamic_cast<decltype(from)>(_from);
-				ArrayType const& to = dynamic_cast<decltype(to)>(_to);
-
-				switch (to.location())
-				{
-				case DataLocation::Storage:
-					// Other cases are done explicitly in LValue::storeValue, and only possible by assignment.
-					solAssert(
-						(to.isPointer() || (from.isByteArray() && to.isByteArray())) &&
-						from.location() == DataLocation::Storage,
-						"Invalid conversion to storage type."
-					);
-					body = "converted := value";
-					break;
-				case DataLocation::Memory:
-					// Copy the array to a free position in memory, unless it is already in memory.
-					if (from.location() == DataLocation::Memory)
-						body = "converted := value";
-					else if (from.location() == DataLocation::CallData)
-						solUnimplemented("Conversion of calldata types not yet implemented.");
-					else
-						body = "converted := " + copyArrayFromStorageToMemoryFunction(from, to) + "(value)";
-					break;
-				case DataLocation::CallData:
-					solUnimplemented("Conversion of calldata types not yet implemented.");
-					break;
-				}
-			}
-			break;
-		}
 		case Type::Category::Struct:
 		{
 			solAssert(toCategory == Type::Category::Struct, "");
@@ -3074,6 +3045,71 @@ string YulUtilFunctions::conversionFunction(Type const& _from, Type const& _to)
 	});
 }
 
+string YulUtilFunctions::arrayConversionFunction(ArrayType const& _from, ArrayType const& _to)
+{
+	solUnimplementedAssert(_to.location() != DataLocation::CallData, "Conversion of calldata types not yet implemented.");
+	// Other cases are done explicitly in LValue::storeValue, and only possible by assignment.
+	if (_to.location() == DataLocation::Storage)
+		solAssert(
+			(_to.isPointer() || (_from.isByteArray() && _to.isByteArray())) &&
+			_from.location() == DataLocation::Storage,
+			"Invalid conversion to storage type."
+		);
+	if (_to.location() == DataLocation::Memory && _from.location() == DataLocation::CallData)
+	{
+		solUnimplementedAssert(_from.isDynamicallySized(), "");
+		solUnimplementedAssert(!_from.baseType()->isDynamicallyEncoded(), "");
+		solUnimplementedAssert(_from.isByteArray() && _to.isByteArray() && _to.isDynamicallySized(), "");
+	}
+
+	string functionName =
+		"convert_array_" +
+		_from.identifier() +
+		"_to_" +
+		_to.identifier();
+
+	return m_functionCollector.createFunction(functionName, [&]() {
+		Whiskers templ(R"(
+			function <functionName>(value<?fromCalldataDynamic>, length</fromCalldataDynamic>) -> converted {
+				<body>
+			}
+		)");
+		templ("functionName", functionName);
+		templ("fromCalldataDynamic", _from.dataStoredIn(DataLocation::CallData) && _from.isDynamicallySized());
+
+		if (
+			_from == _to ||
+			(_from.dataStoredIn(DataLocation::Memory) && _to.dataStoredIn(DataLocation::Memory)) ||
+			_to.dataStoredIn(DataLocation::Storage)
+		)
+			templ("body", "converted := value");
+		else if (_to.dataStoredIn(DataLocation::Memory))
+			templ(
+				"body",
+				Whiskers(R"(
+					// Copy the array to a free position in memory
+					<?fromStorage>
+						converted := <arrayStorageToMem>(value)
+					</fromStorage>
+					<?fromCalldata>
+						converted := <allocateMemoryArray>(length)
+						<copyToMemory>(value, add(converted, 0x20), length)
+					</fromCalldata>
+				)")
+				("fromStorage", _from.dataStoredIn(DataLocation::Storage))
+				("fromCalldata", _from.dataStoredIn(DataLocation::CallData))
+				("allocateMemoryArray", _from.dataStoredIn(DataLocation::CallData) ? allocateMemoryArrayFunction(_to) : "")
+				("copyToMemory", _from.dataStoredIn(DataLocation::CallData) ? copyToMemoryFunction(true) : "")
+				("arrayStorageToMem", _from.dataStoredIn(DataLocation::Storage) ? copyArrayFromStorageToMemoryFunction(_from, _to) : "")
+				.render()
+			);
+		else
+			solAssert(false, "");
+
+		return templ.render();
+	});
+}
+
 string YulUtilFunctions::cleanupFunction(Type const& _type)
 {
 	string functionName = string("cleanup_") + _type.identifier();
@@ -3564,29 +3600,6 @@ string YulUtilFunctions::conversionFunctionSpecial(Type const& _from, Type const
 			.render();
 		}
 
-		if (_from.category() == Type::Category::Array && _to.category() == Type::Category::Array)
-		{
-			auto const& fromArrayType = dynamic_cast<ArrayType const&>(_from);
-			auto const& toArrayType = dynamic_cast<ArrayType const&>(_to);
-
-			solAssert(!fromArrayType.baseType()->isDynamicallyEncoded(), "");
-			solUnimplementedAssert(fromArrayType.isByteArray() && toArrayType.isByteArray(), "");
-			solUnimplementedAssert(toArrayType.location() == DataLocation::Memory, "");
-			solUnimplementedAssert(fromArrayType.location() == DataLocation::CallData, "");
-			solUnimplementedAssert(toArrayType.isDynamicallySized(), "");
-
-			Whiskers templ(R"(
-				function <functionName>(offset, length) -> converted {
-					converted := <allocateMemoryArray>(length)
-					<copyToMemory>(offset, add(converted, 0x20), length)
-				}
-			)");
-			templ("functionName", functionName);
-			templ("allocateMemoryArray", allocateMemoryArrayFunction(toArrayType));
-			templ("copyToMemory", copyToMemoryFunction(fromArrayType.location() == DataLocation::CallData));
-			return templ.render();
-		}
-
 		solUnimplementedAssert(
 			_from.category() == Type::Category::StringLiteral,
 			"Type conversion " + _from.toString() + " -> " + _to.toString() + " not yet implemented."

libsolidity/codegen/YulUtilFunctions.h

@@ -440,6 +440,9 @@ public:
 	);
 
 private:
+	/// Special case of conversion functions - handles all array conversions.
+	std::string arrayConversionFunction(ArrayType const& _from, ArrayType const& _to);
+
 	/// Special case of conversionFunction - handles everything that does not
 	/// use exactly one variable to hold the value.
 	std::string conversionFunctionSpecial(Type const& _from, Type const& _to);

libsolutil/CommonIO.cpp

@@ -47,8 +47,7 @@ inline T readFile(std::string const& _file)
 	T ret;
 	size_t const c_elementSize = sizeof(typename T::value_type);
 	std::ifstream is(_file, std::ifstream::binary);
-	if (!is)
+	assertThrow(is, FileNotFound, _file);
-		return ret;
 
 	// get length of file:
 	is.seekg(0, is.end);

libsolutil/CommonIO.h

@@ -32,7 +32,8 @@ namespace solidity::util
 {
 
 /// Retrieve and returns the contents of the given file as a std::string.
-/// If the file doesn't exist or isn't readable, returns an empty container / bytes.
+/// If the file doesn't exist, it will throw a FileNotFound exception.
+/// If the file is empty, returns an empty string.
 std::string readFileAsString(std::string const& _file);
 
 /// Retrieve and returns the contents of standard input (until EOF).

libsolutil/Exceptions.h

@@ -48,7 +48,7 @@ private:
 DEV_SIMPLE_EXCEPTION(InvalidAddress);
 DEV_SIMPLE_EXCEPTION(BadHexCharacter);
 DEV_SIMPLE_EXCEPTION(BadHexCase);
-DEV_SIMPLE_EXCEPTION(FileError);
+DEV_SIMPLE_EXCEPTION(FileNotFound);
 DEV_SIMPLE_EXCEPTION(DataTooLong);
 DEV_SIMPLE_EXCEPTION(StringTooLong);
 

solc/CommandLineInterface.cpp

@@ -614,6 +614,7 @@ bool CommandLineInterface::readInputFilesAndConfigureRemappings()
 					continue;
 				}
 
+				// NOTE: we ignore the FileNotFound exception as we manually check above
 				m_sourceCodes[infile.generic_string()] = readFileAsString(infile.string());
 				path = boost::filesystem::canonical(infile).string();
 			}
@@ -643,6 +644,10 @@ bool CommandLineInterface::parseLibraryOption(string const& _input)
 	{
 		// Thrown e.g. if path is too long.
 	}
+	catch (FileNotFound const&)
+	{
+		// Should not happen if `fs::is_regular_file` is correct.
+	}
 
 	vector<string> libraries;
 	boost::split(libraries, data, boost::is_space() || boost::is_any_of(","), boost::token_compress_on);
@@ -738,7 +743,11 @@ void CommandLineInterface::createFile(string const& _fileName, string const& _da
 	ofstream outFile(pathName);
 	outFile << _data;
 	if (!outFile)
-		BOOST_THROW_EXCEPTION(FileError() << errinfo_comment("Could not write to file: " + pathName));
+	{
+		serr() << "Could not write to file \"" << pathName << "\"." << endl;
+		m_error = true;
+		return;
+	}
 }
 
 void CommandLineInterface::createJson(string const& _fileName, string const& _json)
@@ -1146,6 +1155,7 @@ bool CommandLineInterface::processInput()
 			if (!boost::filesystem::is_regular_file(canonicalPath))
 				return ReadCallback::Result{false, "Not a valid file."};
 
+			// NOTE: we ignore the FileNotFound exception as we manually check above
 			auto contents = readFileAsString(canonicalPath.string());
 			m_sourceCodes[path.generic_string()] = contents;
 			return ReadCallback::Result{true, contents};
@@ -1232,7 +1242,17 @@ bool CommandLineInterface::processInput()
 		if (jsonFile.empty())
 			input = readStandardInput();
 		else
-			input = readFileAsString(jsonFile);
+		{
+			try
+			{
+				input = readFileAsString(jsonFile);
+			}
+			catch (FileNotFound const&)
+			{
+				serr() << "File not found: " << jsonFile << endl;
+				return false;
+			}
+		}
 		StandardCompiler compiler(fileReader);
 		sout() << compiler.compile(std::move(input)) << endl;
 		return true;

test/cmdlineTests.sh

@@ -254,26 +254,37 @@ printTask "Running general commandline tests..."
     cd "$REPO_ROOT"/test/cmdlineTests/
     for tdir in */
     do
-        if [ -e "${tdir}/input.json" ]
+        printTask " - ${tdir}"
+
+        # Strip trailing slash from $tdir. `find` on MacOS X won't strip it and will produce double slashes.
+        tdir=$(basename "${tdir}")
+
+        inputFiles="$(find "${tdir}" -name 'input.*' -type f -exec printf "%s\n" "{}" \;)"
+        inputCount="$(echo "${inputFiles}" | wc -l)"
+        if (( ${inputCount} == 0 ))
         then
+            printError "No input files found."
+            exit 1
+        fi
+
+        if (( ${inputCount} > 1 ))
+        then
+            printError "Ambiguous input. Found input files in multiple formats:"
+            echo -e "${inputFiles}"
+            exit 1
+        fi
+
+        # Use printf to get rid of the trailing newline
+        inputFile=$(printf "%s" "${inputFiles}")
+
+        if [ "${inputFile}" = "${tdir}/input.json" ]
+        then
+            stdin="${inputFile}"
             inputFile=""
-            stdin="${tdir}/input.json"
             stdout="$(cat ${tdir}/output.json 2>/dev/null || true)"
             stdoutExpectationFile="${tdir}/output.json"
             args="--standard-json "$(cat ${tdir}/args 2>/dev/null || true)
         else
-            if [[ -e "${tdir}input.yul" && -e "${tdir}input.sol" ]]
-            then
-                printError "Ambiguous input. Found both input.sol and input.yul."
-                exit 1
-            fi
-
-            if [ -e "${tdir}input.yul" ]
-            then
-                inputFile="${tdir}input.yul"
-            else
-                inputFile="${tdir}input.sol"
-            fi
             stdin=""
             stdout="$(cat ${tdir}/output 2>/dev/null || true)"
             stdoutExpectationFile="${tdir}/output"
@@ -282,7 +293,6 @@ printTask "Running general commandline tests..."
         exitCode=$(cat ${tdir}/exit 2>/dev/null || true)
         err="$(cat ${tdir}/err 2>/dev/null || true)"
         stderrExpectationFile="${tdir}/err"
-        printTask " - ${tdir}"
         test_solc_behaviour "$inputFile" \
                             "$args" \
                             "$stdin" \

test/cmdlineTests/standard_file_not_found/args

@@ -0,0 +1 @@
+--standard-json

test/cmdlineTests/standard_file_not_found/err

@@ -0,0 +1 @@
+File not found: standard_file_not_found/input.sol

test/cmdlineTests/standard_file_not_found/exit

@@ -0,0 +1 @@
+1

test/cmdlineTests/standard_immutable_references/input.json

@@ -2,7 +2,7 @@
  "language": "Solidity",
  "sources": {
   "a.sol": {
-   "content": "// SPDX-License-Identifier: GPL-3.0\ncontract A { uint256 immutable x = 1; function f() public view returns (uint256) { return x; } }"
+   "content": "// SPDX-License-Identifier: GPL-3.0\ncontract A { uint256 immutable x = 1 + 3; function f() public pure returns (uint256) { return x; } }"
   }
  },
  "settings": {

test/cmdlineTests/standard_immutable_references/output.json

@@ -1,2 +1,2 @@
-{"contracts":{"a.sol":{"A":{"evm":{"deployedBytecode":{"generatedSources":[],"immutableReferences":{"3":[{"length":32,"start":77}]},"linkReferences":{},"object":"<BYTECODE REMOVED>","opcodes":"<OPCODES REMOVED>","sourceMap":"36:96:0:-:0;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;74:56;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;108:7;126:1;119:8;;74:56;:::o"}}}}},"errors":[{"component":"general","errorCode":"3420","formattedMessage":"a.sol: Warning: Source file does not specify required compiler version!
+{"contracts":{"a.sol":{"A":{"evm":{"deployedBytecode":{"generatedSources":[],"immutableReferences":{"5":[{"length":32,"start":77}]},"linkReferences":{},"object":"<BYTECODE REMOVED>","opcodes":"<OPCODES REMOVED>","sourceMap":"36:100:0:-:0;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;78:56;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;112:7;130:1;123:8;;78:56;:::o"}}}}},"errors":[{"component":"general","errorCode":"3420","formattedMessage":"a.sol: Warning: Source file does not specify required compiler version!
 ","message":"Source file does not specify required compiler version!","severity":"warning","sourceLocation":{"end":-1,"file":"a.sol","start":-1},"type":"Warning"}],"sources":{"a.sol":{"id":0}}}

test/libsolidity/semanticTests/structs/calldata/calldata_struct_with_nested_array_to_storage.sol

@@ -15,6 +15,6 @@ contract C {
     }
 }
 // ====
-// compileViaYul: true
+// compileViaYul: also
 // ----
 // f(uint32, (uint128, uint256[][2], uint32)): 55, 0x40, 77, 0x60, 88, 0x40, 0x40, 2, 1, 2 -> 55, 77, 1, 2, 88

test/libsolidity/syntaxTests/immutable/unrelated_reading.sol

@@ -1,7 +1,7 @@
 contract C {
     uint immutable x = 1;
 
-    function readX() internal view returns(uint) {
+    function readX() internal pure returns(uint) {
         return x + 3;
     }
 }

test/libsolidity/syntaxTests/viewPureChecker/immutable.sol

@@ -5,4 +5,3 @@ contract B {
     }
 }
 // ----
-// TypeError 2527: (100-101): Function declared as pure, but this expression (potentially) reads from the environment or state and thus requires "view".

test/tools/yulopti.cpp

@@ -20,6 +20,7 @@
  */
 
 #include <libsolutil/CommonIO.h>
+#include <libsolutil/Exceptions.h>
 #include <liblangutil/ErrorReporter.h>
 #include <liblangutil/Scanner.h>
 #include <libyul/AsmAnalysis.h>
@@ -243,8 +244,18 @@ Allowed options)",
 	}
 
 	string input;
+	try
+	{
+		input = readFileAsString(arguments["input-file"].as<string>());
+	}
+	catch (FileNotFound const& _exception)
+	{
+		cerr << "File not found:" << _exception.comment() << endl;
+		return 1;
+	}
+
 	if (arguments.count("input-file"))
-		YulOpti{}.runInteractive(readFileAsString(arguments["input-file"].as<string>()));
+		YulOpti{}.runInteractive(input);
 	else
 		cout << options;
 

test/tools/yulrun.cpp

@@ -35,6 +35,7 @@
 
 #include <libsolutil/CommonIO.h>
 #include <libsolutil/CommonData.h>
+#include <libsolutil/Exceptions.h>
 
 #include <boost/program_options.hpp>
 
@@ -137,10 +138,19 @@ Allowed options)",
 	else
 	{
 		string input;
-
 		if (arguments.count("input-file"))
 			for (string path: arguments["input-file"].as<vector<string>>())
-				input += readFileAsString(path);
+			{
+				try
+				{
+					input += readFileAsString(path);
+				}
+				catch (FileNotFound const&)
+				{
+					cerr << "File not found: " << path << endl;
+					return 1;
+				}
+			}
 		else
 			input = readStandardInput();