Merge pull request #10147 from ethereum/viair

Add viaIR option to CompilerStack (and solc)

Changelog.md

@@ -5,6 +5,9 @@ Language Features:
  * Immutable variables with literal number values are considered pure.
 
 Compiler Features:
+ * Command Line Interface: New option ``--experimental-via-ir`` allows switching compilation process to go through
+   the Yul intermediate representation. This is highly experimental and is used for development purposes.
+ * Standard JSON: New option ``settings.viaIR`` allows the same switch as ``--experimental-via-ir`` on the commandline.
  * Command Line Interface: Report error if file could not be read in ``--standard-json`` mode.
  * Command Line interface: Report proper error for each output file which could not be written. Previously an exception was thrown, and execution aborted, on the first error.
  * SMTChecker: Add division by zero checks in the CHC engine.

docs/using-the-compiler.rst

@@ -252,6 +252,9 @@ Input Description
         // Affects type checking and code generation. Can be homestead,
         // tangerineWhistle, spuriousDragon, byzantium, constantinople, petersburg, istanbul or berlin
         "evmVersion": "byzantium",
+        // Optional: Change compilation pipeline to go through the Yul intermediate representation.
+        // This is a highly EXPERIMENTAL feature, not to be used for production. This is false by default.
+        "viaIR": true,
         // Optional: Debugging settings
         "debug": {
           // How to treat revert (and require) reason strings. Settings are

libsolidity/interface/CompilerStack.cpp

@@ -131,6 +131,13 @@ void CompilerStack::setRemappings(vector<Remapping> const& _remappings)
 	m_remappings = _remappings;
 }
 
+void CompilerStack::setViaIR(bool _viaIR)
+{
+	if (m_stackState >= ParsedAndImported)
+		BOOST_THROW_EXCEPTION(CompilerError() << errinfo_comment("Must set viaIR before parsing."));
+	m_viaIR = _viaIR;
+}
+
 void CompilerStack::setEVMVersion(langutil::EVMVersion _version)
 {
 	if (m_stackState >= ParsedAndImported)
@@ -213,6 +220,7 @@ void CompilerStack::reset(bool _keepSettings)
 	{
 		m_remappings.clear();
 		m_libraries.clear();
+		m_viaIR = false;
 		m_evmVersion = langutil::EVMVersion();
 		m_modelCheckerSettings = ModelCheckerSettings{};
 		m_enabledSMTSolvers = smtutil::SMTSolverChoice::All();
@@ -532,10 +540,15 @@ bool CompilerStack::compile(State _stopAfter)
 				{
 					try
 					{
-						if (m_generateEvmBytecode)
+						if (m_viaIR || m_generateIR || m_generateEwasm)
-							compileContract(*contract, otherCompilers);
-						if (m_generateIR || m_generateEwasm)
 							generateIR(*contract);
+						if (m_generateEvmBytecode)
+						{
+							if (m_viaIR)
+								generateEVMFromIR(*contract);
+							else
+								compileContract(*contract, otherCompilers);
+						}
 						if (m_generateEwasm)
 							generateEwasm(*contract);
 					}
@@ -1250,6 +1263,36 @@ void CompilerStack::generateIR(ContractDefinition const& _contract)
 	tie(compiledContract.yulIR, compiledContract.yulIROptimized) = generator.run(_contract, otherYulSources);
 }
 
+void CompilerStack::generateEVMFromIR(ContractDefinition const& _contract)
+{
+	solAssert(m_stackState >= AnalysisPerformed, "");
+	if (m_hasError)
+		BOOST_THROW_EXCEPTION(CompilerError() << errinfo_comment("Called generateEVMFromIR with errors."));
+
+	if (!_contract.canBeDeployed())
+		return;
+
+	Contract& compiledContract = m_contracts.at(_contract.fullyQualifiedName());
+	solAssert(!compiledContract.yulIROptimized.empty(), "");
+	if (!compiledContract.object.bytecode.empty())
+		return;
+
+	// Re-parse the Yul IR in EVM dialect
+	yul::AssemblyStack stack(m_evmVersion, yul::AssemblyStack::Language::StrictAssembly, m_optimiserSettings);
+	stack.parseAndAnalyze("", compiledContract.yulIROptimized);
+	stack.optimize();
+
+	//cout << yul::AsmPrinter{}(*stack.parserResult()->code) << endl;
+
+	// TODO: support passing metadata
+	auto result = stack.assemble(yul::AssemblyStack::Machine::EVM);
+	compiledContract.object = std::move(*result.bytecode);
+	// TODO: support runtimeObject
+	// TODO: add EIP-170 size check for runtimeObject
+	// TODO: refactor assemblyItems, runtimeAssemblyItems, generatedSources,
+	//       assemblyString, assemblyJSON, and functionEntryPoints to work with this code path
+}
+
 void CompilerStack::generateEwasm(ContractDefinition const& _contract)
 {
 	solAssert(m_stackState >= AnalysisPerformed, "");
@@ -1395,6 +1438,8 @@ string CompilerStack::createMetadata(Contract const& _contract) const
 	static vector<string> hashes{"ipfs", "bzzr1", "none"};
 	meta["settings"]["metadata"]["bytecodeHash"] = hashes.at(unsigned(m_metadataHash));
 
+	if (m_viaIR)
+		meta["settings"]["viaIR"] = m_viaIR;
 	meta["settings"]["evmVersion"] = m_evmVersion.name();
 	meta["settings"]["compilationTarget"][_contract.contract->sourceUnitName()] =
 		*_contract.contract->annotation().canonicalName;
@@ -1517,7 +1562,7 @@ bytes CompilerStack::createCBORMetadata(Contract const& _contract) const
 	else
 		solAssert(m_metadataHash == MetadataHash::None, "Invalid metadata hash");
 
-	if (experimentalMode)
+	if (experimentalMode || m_viaIR)
 		encoder.pushBool("experimental", true);
 	if (m_release)
 		encoder.pushBytes("solc", VersionCompactBytes);

libsolidity/interface/CompilerStack.h

@@ -162,6 +162,10 @@ public:
 		m_parserErrorRecovery = _wantErrorRecovery;
 	}
 
+	/// Sets the pipeline to go through the Yul IR or not.
+	/// Must be set before parsing.
+	void setViaIR(bool _viaIR);
+
 	/// Set the EVM version used before running compile.
 	/// When called without an argument it will revert to the default version.
 	/// Must be set before parsing.
@@ -399,7 +403,12 @@ private:
 	/// The IR is stored but otherwise unused.
 	void generateIR(ContractDefinition const& _contract);
 
+	/// Generate EVM representation for a single contract.
+	/// Depends on output generated by generateIR.
+	void generateEVMFromIR(ContractDefinition const& _contract);
+
 	/// Generate Ewasm representation for a single contract.
+	/// Depends on output generated by generateIR.
 	void generateEwasm(ContractDefinition const& _contract);
 
 	/// Links all the known library addresses in the available objects. Any unknown
@@ -455,6 +464,7 @@ private:
 	OptimiserSettings m_optimiserSettings;
 	RevertStrings m_revertStrings = RevertStrings::Default;
 	State m_stopAfter = State::CompilationSuccessful;
+	bool m_viaIR = false;
 	langutil::EVMVersion m_evmVersion;
 	ModelCheckerSettings m_modelCheckerSettings;
 	smtutil::SMTSolverChoice m_enabledSMTSolvers;

libsolidity/interface/StandardCompiler.cpp

@@ -414,7 +414,7 @@ std::optional<Json::Value> checkAuxiliaryInputKeys(Json::Value const& _input)
 
 std::optional<Json::Value> checkSettingsKeys(Json::Value const& _input)
 {
-	static set<string> keys{"parserErrorRecovery", "debug", "evmVersion", "libraries", "metadata", "optimizer", "outputSelection", "remappings", "stopAfter"};
+	static set<string> keys{"parserErrorRecovery", "debug", "evmVersion", "libraries", "metadata", "optimizer", "outputSelection", "remappings", "stopAfter", "viaIR"};
 	return checkKeys(_input, keys, "settings");
 }
 
@@ -749,6 +749,13 @@ std::variant<StandardCompiler::InputsAndSettings, Json::Value> StandardCompiler:
 		ret.parserErrorRecovery = settings["parserErrorRecovery"].asBool();
 	}
 
+	if (settings.isMember("viaIR"))
+	{
+		if (!settings["viaIR"].isBool())
+			return formatFatalError("JSONError", "\"settings.viaIR\" must be a Boolean.");
+		ret.viaIR = settings["viaIR"].asBool();
+	}
+
 	if (settings.isMember("evmVersion"))
 	{
 		if (!settings["evmVersion"].isString())
@@ -906,6 +913,7 @@ Json::Value StandardCompiler::compileSolidity(StandardCompiler::InputsAndSetting
 	compilerStack.setSources(sourceList);
 	for (auto const& smtLib2Response: _inputsAndSettings.smtLib2Responses)
 		compilerStack.addSMTLib2Response(smtLib2Response.first, smtLib2Response.second);
+	compilerStack.setViaIR(_inputsAndSettings.viaIR);
 	compilerStack.setEVMVersion(_inputsAndSettings.evmVersion);
 	compilerStack.setParserErrorRecovery(_inputsAndSettings.parserErrorRecovery);
 	compilerStack.setRemappings(_inputsAndSettings.remappings);

libsolidity/interface/StandardCompiler.h

@@ -72,6 +72,7 @@ private:
 		CompilerStack::MetadataHash metadataHash = CompilerStack::MetadataHash::IPFS;
 		Json::Value outputSelection;
 		ModelCheckerSettings modelCheckerSettings = ModelCheckerSettings{};
+		bool viaIR = false;
 	};
 
 	/// Parses the input json (and potentially invokes the read callback) and either returns

solc/CommandLineInterface.cpp

@@ -127,6 +127,7 @@ static string const g_strEVM = "evm";
 static string const g_strEVM15 = "evm15";
 static string const g_strEVMVersion = "evm-version";
 static string const g_strEwasm = "ewasm";
+static string const g_strExperimentalViaIR = "experimental-via-ir";
 static string const g_strGeneratedSources = "generated-sources";
 static string const g_strGeneratedSourcesRuntime = "generated-sources-runtime";
 static string const g_strGas = "gas";
@@ -211,6 +212,7 @@ static string const g_argYul = g_strYul;
 static string const g_argIR = g_strIR;
 static string const g_argIROptimized = g_strIROptimized;
 static string const g_argEwasm = g_strEwasm;
+static string const g_argExperimentalViaIR = g_strExperimentalViaIR;
 static string const g_argLibraries = g_strLibraries;
 static string const g_argLink = g_strLink;
 static string const g_argMachine = g_strMachine;
@@ -825,6 +827,10 @@ General Information)").c_str(),
 			"Select desired EVM version. Either homestead, tangerineWhistle, spuriousDragon, "
 			"byzantium, constantinople, petersburg, istanbul (default) or berlin."
 		)
+		(
+			g_strExperimentalViaIR.c_str(),
+			"Turn on experimental compilation mode via the IR (EXPERIMENTAL)."
+		)
 		(
 			g_strRevertStrings.c_str(),
 			po::value<string>()->value_name(boost::join(g_revertStringsArgs, ",")),
@@ -1456,6 +1462,8 @@ bool CommandLineInterface::processInput()
 
 		if (m_args.count(g_argLibraries))
 			m_compiler->setLibraries(m_libraries);
+		if (m_args.count(g_argExperimentalViaIR))
+			m_compiler->setViaIR(true);
 		m_compiler->setEVMVersion(m_evmVersion);
 		m_compiler->setRevertStringBehaviour(m_revertStrings);
 		// TODO: Perhaps we should not compile unless requested

test/cmdlineTests/standard_viair_requested/input.json

@@ -0,0 +1,21 @@
+{
+	"language": "Solidity",
+	"sources":
+	{
+		"A":
+		{
+			"content": "// SPDX-License-Identifier: GPL-3.0\npragma solidity >=0.0; contract C {} contract D { function f() public { C c = new C(); } }"
+		}
+	},
+	"settings":
+	{
+		"optimizer": {
+			"enabled": true
+		},
+		"outputSelection":
+		{
+			"*": { "*": ["ir", "evm.bytecode.object", "evm.bytecode.generatedSources", "evm.deployedBytecode.object"] }
+		},
+		"viaIR": true
+	}
+}

test/cmdlineTests/standard_viair_requested/output.json

@@ -0,0 +1,210 @@
+{"contracts":{"A":{"C":{"evm":{"bytecode":{"generatedSources":[],"linkReferences":{},"object":"<BYTECODE REMOVED>","opcodes":"<OPCODES REMOVED>","sourceMap":""},"deployedBytecode":{"generatedSources":[],"immutableReferences":{},"linkReferences":{},"object":"","opcodes":"","sourceMap":""}},"ir":"/*******************************************************
+ *                       WARNING                       *
+ *  Solidity to Yul compilation is still EXPERIMENTAL  *
+ *       It can result in LOSS OF FUNDS or worse       *
+ *                !USE AT YOUR OWN RISK!               *
+ *******************************************************/
+
+
+object \"C_2\" {
+    code {
+        mstore(64, 128)
+        if callvalue() { revert(0, 0) }
+
+        constructor_C_2()
+
+        codecopy(0, dataoffset(\"C_2_deployed\"), datasize(\"C_2_deployed\"))
+
+        return(0, datasize(\"C_2_deployed\"))
+
+        function constructor_C_2() {
+
+        }
+
+    }
+    object \"C_2_deployed\" {
+        code {
+            mstore(64, 128)
+
+            if iszero(lt(calldatasize(), 4))
+            {
+                let selector := shift_right_224_unsigned(calldataload(0))
+                switch selector
+
+                default {}
+            }
+            if iszero(calldatasize()) {  }
+            revert(0, 0)
+
+            function shift_right_224_unsigned(value) -> newValue {
+                newValue :=
+
+                shr(224, value)
+
+            }
+
+        }
+
+    }
+
+}
+
+"},"D":{"evm":{"bytecode":{"generatedSources":[],"linkReferences":{},"object":"<BYTECODE REMOVED>","opcodes":"<OPCODES REMOVED>","sourceMap":""},"deployedBytecode":{"generatedSources":[],"immutableReferences":{},"linkReferences":{},"object":"","opcodes":"","sourceMap":""}},"ir":"/*******************************************************
+ *                       WARNING                       *
+ *  Solidity to Yul compilation is still EXPERIMENTAL  *
+ *       It can result in LOSS OF FUNDS or worse       *
+ *                !USE AT YOUR OWN RISK!               *
+ *******************************************************/
+
+
+object \"D_13\" {
+    code {
+        mstore(64, 128)
+        if callvalue() { revert(0, 0) }
+
+        constructor_D_13()
+
+        codecopy(0, dataoffset(\"D_13_deployed\"), datasize(\"D_13_deployed\"))
+
+        return(0, datasize(\"D_13_deployed\"))
+
+        function constructor_D_13() {
+
+        }
+
+    }
+    object \"D_13_deployed\" {
+        code {
+            mstore(64, 128)
+
+            if iszero(lt(calldatasize(), 4))
+            {
+                let selector := shift_right_224_unsigned(calldataload(0))
+                switch selector
+
+                case 0x26121ff0
+                {
+                    // f()
+                    if callvalue() { revert(0, 0) }
+                    abi_decode_tuple_(4, calldatasize())
+                    fun_f_12()
+                    let memPos := allocateMemory(0)
+                    let memEnd := abi_encode_tuple__to__fromStack(memPos  )
+                    return(memPos, sub(memEnd, memPos))
+                }
+
+                default {}
+            }
+            if iszero(calldatasize()) {  }
+            revert(0, 0)
+
+            function abi_decode_tuple_(headStart, dataEnd)   {
+                if slt(sub(dataEnd, headStart), 0) { revert(0, 0) }
+
+            }
+
+            function abi_encode_tuple__to__fromStack(headStart ) -> tail {
+                tail := add(headStart, 0)
+
+            }
+
+            function allocateMemory(size) -> memPtr {
+                memPtr := mload(64)
+                let newFreePtr := add(memPtr, size)
+                // protect against overflow
+                if or(gt(newFreePtr, 0xffffffffffffffff), lt(newFreePtr, memPtr)) { panic_error() }
+                mstore(64, newFreePtr)
+            }
+
+            function allocateTemporaryMemory() -> memPtr {
+                memPtr := mload(64)
+            }
+
+            function fun_f_12() {
+
+                let _1 := allocateTemporaryMemory()
+                let _2 := add(_1, datasize(\"C_2\"))
+                if or(gt(_2, 0xffffffffffffffff), lt(_2, _1)) { panic_error() }
+                datacopy(_1, dataoffset(\"C_2\"), datasize(\"C_2\"))
+                _2 := abi_encode_tuple__to__fromStack(_2)
+
+                let expr_9_address := create(0, _1, sub(_2, _1))
+
+                releaseTemporaryMemory()
+                let vloc_c_6_address := expr_9_address
+
+            }
+
+            function panic_error() {
+                invalid()
+            }
+
+            function releaseTemporaryMemory() {
+            }
+
+            function shift_right_224_unsigned(value) -> newValue {
+                newValue :=
+
+                shr(224, value)
+
+            }
+
+        }
+        /*******************************************************
+        *                       WARNING                       *
+        *  Solidity to Yul compilation is still EXPERIMENTAL  *
+        *       It can result in LOSS OF FUNDS or worse       *
+        *                !USE AT YOUR OWN RISK!               *
+        *******************************************************/
+
+        object \"C_2\" {
+            code {
+                mstore(64, 128)
+                if callvalue() { revert(0, 0) }
+
+                constructor_C_2()
+
+                codecopy(0, dataoffset(\"C_2_deployed\"), datasize(\"C_2_deployed\"))
+
+                return(0, datasize(\"C_2_deployed\"))
+
+                function constructor_C_2() {
+
+                }
+
+            }
+            object \"C_2_deployed\" {
+                code {
+                    mstore(64, 128)
+
+                    if iszero(lt(calldatasize(), 4))
+                    {
+                        let selector := shift_right_224_unsigned(calldataload(0))
+                        switch selector
+
+                        default {}
+                    }
+                    if iszero(calldatasize()) {  }
+                    revert(0, 0)
+
+                    function shift_right_224_unsigned(value) -> newValue {
+                        newValue :=
+
+                        shr(224, value)
+
+                    }
+
+                }
+
+            }
+
+        }
+
+    }
+
+}
+
+"}}},"errors":[{"component":"general","errorCode":"2072","formattedMessage":"A:2:73: Warning: Unused local variable.
+pragma solidity >=0.0; contract C {} contract D { function f() public { C c = new C(); } }
+                                                                        ^-^
+","message":"Unused local variable.","severity":"warning","sourceLocation":{"end":111,"file":"A","start":108},"type":"Warning"}],"sources":{"A":{"id":0}}}

test/cmdlineTests/viair_subobjects/args

@@ -0,0 +1 @@
+--ir-optimized --experimental-via-ir --optimize --bin --bin-runtime

test/cmdlineTests/viair_subobjects/err

@@ -0,0 +1,5 @@
+Warning: Unused local variable.
+ --> viair_subobjects/input.sol:7:9:
+  |
+7 |         C c = new C();
+  |         ^^^

test/cmdlineTests/viair_subobjects/input.sol

@@ -0,0 +1,9 @@
+// SPDX-License-Identifier: GPL-3.0
+pragma solidity >=0.6.0;
+
+contract C {}
+contract D {
+    function f() public {
+        C c = new C();
+    }
+}

test/cmdlineTests/viair_subobjects/output

@@ -0,0 +1,109 @@
+
+======= viair_subobjects/input.sol:C =======
+Binary:
+60806040523415600f5760006000fd5b600a80601e600039806000f350fe608060405260006000fd
+Binary of the runtime part:
+
+Optimized IR:
+/*******************************************************
+ *                       WARNING                       *
+ *  Solidity to Yul compilation is still EXPERIMENTAL  *
+ *       It can result in LOSS OF FUNDS or worse       *
+ *                !USE AT YOUR OWN RISK!               *
+ *******************************************************/
+
+object "C_2" {
+    code {
+        {
+            mstore(64, 128)
+            if callvalue() { revert(0, 0) }
+            let _1 := datasize("C_2_deployed")
+            codecopy(0, dataoffset("C_2_deployed"), _1)
+            return(0, _1)
+        }
+    }
+    object "C_2_deployed" {
+        code {
+            {
+                mstore(64, 128)
+                revert(0, 0)
+            }
+        }
+    }
+}
+
+
+======= viair_subobjects/input.sol:D =======
+Binary:
+608060405234156100105760006000fd5b60d380610020600039806000f350fe6080604052600436101515610074576000803560e01c6326121ff0141561007257341561002a578081fd5b806003193601121561003a578081fd5b6028806080016080811067ffffffffffffffff8211171561005757fe5b50806100ab60803980608083f05050806100708261007e565bf35b505b60006000fd6100a9565b6000604051905081810181811067ffffffffffffffff8211171561009e57fe5b80604052505b919050565bfe60806040523415600f5760006000fd5b600a80601e600039806000f350fe608060405260006000fd
+Binary of the runtime part:
+
+Optimized IR:
+/*******************************************************
+ *                       WARNING                       *
+ *  Solidity to Yul compilation is still EXPERIMENTAL  *
+ *       It can result in LOSS OF FUNDS or worse       *
+ *                !USE AT YOUR OWN RISK!               *
+ *******************************************************/
+
+object "D_13" {
+    code {
+        {
+            mstore(64, 128)
+            if callvalue() { revert(0, 0) }
+            let _1 := datasize("D_13_deployed")
+            codecopy(0, dataoffset("D_13_deployed"), _1)
+            return(0, _1)
+        }
+    }
+    object "D_13_deployed" {
+        code {
+            {
+                mstore(64, 128)
+                if iszero(lt(calldatasize(), 4))
+                {
+                    let _1 := 0
+                    if eq(0x26121ff0, shr(224, calldataload(_1)))
+                    {
+                        if callvalue() { revert(_1, _1) }
+                        if slt(add(calldatasize(), not(3)), _1) { revert(_1, _1) }
+                        let _2 := datasize("C_2")
+                        let _3 := add(128, _2)
+                        if or(gt(_3, 0xffffffffffffffff), lt(_3, 128)) { invalid() }
+                        datacopy(128, dataoffset("C_2"), _2)
+                        pop(create(_1, 128, _2))
+                        return(allocateMemory(_1), _1)
+                    }
+                }
+                revert(0, 0)
+            }
+            function allocateMemory(size) -> memPtr
+            {
+                memPtr := mload(64)
+                let newFreePtr := add(memPtr, size)
+                if or(gt(newFreePtr, 0xffffffffffffffff), lt(newFreePtr, memPtr)) { invalid() }
+                mstore(64, newFreePtr)
+            }
+        }
+        object "C_2" {
+            code {
+                {
+                    mstore(64, 128)
+                    if callvalue() { revert(0, 0) }
+                    let _1 := datasize("C_2_deployed")
+                    codecopy(0, dataoffset("C_2_deployed"), _1)
+                    return(0, _1)
+                }
+            }
+            object "C_2_deployed" {
+                code {
+                    {
+                        mstore(64, 128)
+                        revert(0, 0)
+                    }
+                }
+            }
+        }
+    }
+}
+

test/libsolidity/Metadata.cpp

@@ -282,6 +282,38 @@ BOOST_AUTO_TEST_CASE(metadata_useLiteralContent)
 	check(sourceCode, false);
 }
 
+BOOST_AUTO_TEST_CASE(metadata_viair)
+{
+	char const* sourceCode = R"(
+		pragma solidity >=0.0;
+		contract test {
+		}
+	)";
+
+	auto check = [](char const* _src, bool _viair)
+	{
+		CompilerStack compilerStack;
+		compilerStack.setSources({{"", std::string(_src)}});
+		compilerStack.setEVMVersion(solidity::test::CommonOptions::get().evmVersion());
+		compilerStack.setOptimiserSettings(solidity::test::CommonOptions::get().optimize);
+		compilerStack.setViaIR(_viair);
+		BOOST_REQUIRE_MESSAGE(compilerStack.compile(), "Compiling contract failed");
+		string metadata_str = compilerStack.metadata("test");
+		Json::Value metadata;
+		util::jsonParseStrict(metadata_str, metadata);
+		BOOST_CHECK(solidity::test::isValidMetadata(metadata_str));
+		BOOST_CHECK(metadata.isMember("settings"));
+		if (_viair)
+		{
+			BOOST_CHECK(metadata["settings"].isMember("viaIR"));
+			BOOST_CHECK(metadata["settings"]["viaIR"].asBool());
+		}
+	};
+
+	check(sourceCode, true);
+	check(sourceCode, false);
+}
+
 BOOST_AUTO_TEST_CASE(metadata_revert_strings)
 {
 	CompilerStack compilerStack;