Adding source location support to AssemblyStack and thus debugging Yul sources

Changelog.md

@@ -4,6 +4,7 @@ Language Features:
 
 
 Compiler Features:
+ * AssemblyStack: Support for source locations (source mappings) and thus debugging Yul sources.
 
 
 Bugfixes:

libevmasm/AssemblyItem.cpp

@@ -19,12 +19,14 @@
 
 #include <libsolutil/CommonData.h>
 #include <libsolutil/FixedHash.h>
+#include <liblangutil/SourceLocation.h>
 
 #include <fstream>
 
 using namespace std;
 using namespace solidity;
 using namespace solidity::evmasm;
+using namespace solidity::langutil;
 
 static_assert(sizeof(size_t) <= 8, "size_t must be at most 64-bits wide");
 
@@ -281,3 +283,92 @@ ostream& solidity::evmasm::operator<<(ostream& _out, AssemblyItem const& _item)
 	}
 	return _out;
 }
+
+std::string AssemblyItem::computeSourceMapping(
+	AssemblyItems const& _items,
+	map<string, unsigned> const& _sourceIndicesMap
+)
+{
+	string ret;
+
+	int prevStart = -1;
+	int prevLength = -1;
+	int prevSourceIndex = -1;
+	size_t prevModifierDepth = -1;
+	char prevJump = 0;
+	for (auto const& item: _items)
+	{
+		if (!ret.empty())
+			ret += ";";
+
+		SourceLocation const& location = item.location();
+		int length = location.start != -1 && location.end != -1 ? location.end - location.start : -1;
+		int sourceIndex =
+			location.source && _sourceIndicesMap.count(location.source->name()) ?
+			_sourceIndicesMap.at(location.source->name()) :
+			-1;
+		char jump = '-';
+		if (item.getJumpType() == evmasm::AssemblyItem::JumpType::IntoFunction)
+			jump = 'i';
+		else if (item.getJumpType() == evmasm::AssemblyItem::JumpType::OutOfFunction)
+			jump = 'o';
+		size_t modifierDepth = item.m_modifierDepth;
+
+		unsigned components = 5;
+		if (modifierDepth == prevModifierDepth)
+		{
+			components--;
+			if (jump == prevJump)
+			{
+				components--;
+				if (sourceIndex == prevSourceIndex)
+				{
+					components--;
+					if (length == prevLength)
+					{
+						components--;
+						if (location.start == prevStart)
+							components--;
+					}
+				}
+			}
+		}
+
+		if (components-- > 0)
+		{
+			if (location.start != prevStart)
+				ret += to_string(location.start);
+			if (components-- > 0)
+			{
+				ret += ':';
+				if (length != prevLength)
+					ret += to_string(length);
+				if (components-- > 0)
+				{
+					ret += ':';
+					if (sourceIndex != prevSourceIndex)
+						ret += to_string(sourceIndex);
+					if (components-- > 0)
+					{
+						ret += ':';
+						if (jump != prevJump)
+							ret += jump;
+						if (components-- > 0)
+						{
+							ret += ':';
+							if (modifierDepth != prevModifierDepth)
+								ret += to_string(modifierDepth);
+						}
+					}
+				}
+			}
+		}
+
+		prevStart = location.start;
+		prevLength = length;
+		prevSourceIndex = sourceIndex;
+		prevJump = jump;
+		prevModifierDepth = modifierDepth;
+	}
+	return ret;
+}

libevmasm/AssemblyItem.h

@@ -48,6 +48,8 @@ enum AssemblyItemType {
 };
 
 class Assembly;
+class AssemblyItem;
+using AssemblyItems = std::vector<AssemblyItem>;
 
 class AssemblyItem
 {
@@ -122,6 +124,11 @@ public:
 	}
 	bool operator!=(Instruction _instr) const { return !operator==(_instr); }
 
+	static std::string computeSourceMapping(
+		AssemblyItems const& _items,
+		std::map<std::string, unsigned> const& _sourceIndicesMap
+	);
+
 	/// @returns an upper bound for the number of bytes required by this item, assuming that
 	/// the value of a jump tag takes @a _addressLength bytes.
 	unsigned bytesRequired(unsigned _addressLength) const;
@@ -157,8 +164,6 @@ private:
 	mutable std::shared_ptr<u256> m_pushedValue;
 };
 
-using AssemblyItems = std::vector<AssemblyItem>;
-
 inline size_t bytesRequired(AssemblyItems const& _items, size_t _addressLength)
 {
 	size_t size = 0;

liblangutil/Scanner.h

@@ -98,6 +98,7 @@ public:
 	std::string const& source() const noexcept { return m_source->source(); }
 
 	std::shared_ptr<CharStream> charStream() noexcept { return m_source; }
+	std::shared_ptr<CharStream const> charStream() const noexcept { return m_source; }
 
 	/// Resets the scanner as if newly constructed with _source as input.
 	void reset(CharStream _source);

libsolidity/interface/CompilerStack.cpp

@@ -565,7 +565,7 @@ string const* CompilerStack::sourceMapping(string const& _contractName) const
 	if (!c.sourceMapping)
 	{
 		if (auto items = assemblyItems(_contractName))
-			c.sourceMapping = make_unique<string>(computeSourceMapping(*items));
+			c.sourceMapping = make_unique<string>(evmasm::AssemblyItem::computeSourceMapping(*items, sourceIndices()));
 	}
 	return c.sourceMapping.get();
 }
@@ -579,7 +579,9 @@ string const* CompilerStack::runtimeSourceMapping(string const& _contractName) c
 	if (!c.runtimeSourceMapping)
 	{
 		if (auto items = runtimeAssemblyItems(_contractName))
-			c.runtimeSourceMapping = make_unique<string>(computeSourceMapping(*items));
+			c.runtimeSourceMapping = make_unique<string>(
+				evmasm::AssemblyItem::computeSourceMapping(*items, sourceIndices())
+			);
 	}
 	return c.runtimeSourceMapping.get();
 }
@@ -1390,95 +1392,6 @@ bytes CompilerStack::createCBORMetadata(string const& _metadata, bool _experimen
 	return encoder.serialise();
 }
 
-string CompilerStack::computeSourceMapping(evmasm::AssemblyItems const& _items) const
-{
-	if (m_stackState != CompilationSuccessful)
-		BOOST_THROW_EXCEPTION(CompilerError() << errinfo_comment("Compilation was not successful."));
-
-	string ret;
-	map<string, unsigned> sourceIndicesMap = sourceIndices();
-	int prevStart = -1;
-	int prevLength = -1;
-	int prevSourceIndex = -1;
-	size_t prevModifierDepth = -1;
-	char prevJump = 0;
-	for (auto const& item: _items)
-	{
-		if (!ret.empty())
-			ret += ";";
-
-		SourceLocation const& location = item.location();
-		int length = location.start != -1 && location.end != -1 ? location.end - location.start : -1;
-		int sourceIndex =
-			location.source && sourceIndicesMap.count(location.source->name()) ?
-			sourceIndicesMap.at(location.source->name()) :
-			-1;
-		char jump = '-';
-		if (item.getJumpType() == evmasm::AssemblyItem::JumpType::IntoFunction)
-			jump = 'i';
-		else if (item.getJumpType() == evmasm::AssemblyItem::JumpType::OutOfFunction)
-			jump = 'o';
-		size_t modifierDepth = item.m_modifierDepth;
-
-		unsigned components = 5;
-		if (modifierDepth == prevModifierDepth)
-		{
-			components--;
-			if (jump == prevJump)
-			{
-				components--;
-				if (sourceIndex == prevSourceIndex)
-				{
-					components--;
-					if (length == prevLength)
-					{
-						components--;
-						if (location.start == prevStart)
-							components--;
-					}
-				}
-			}
-		}
-
-		if (components-- > 0)
-		{
-			if (location.start != prevStart)
-				ret += to_string(location.start);
-			if (components-- > 0)
-			{
-				ret += ':';
-				if (length != prevLength)
-					ret += to_string(length);
-				if (components-- > 0)
-				{
-					ret += ':';
-					if (sourceIndex != prevSourceIndex)
-						ret += to_string(sourceIndex);
-					if (components-- > 0)
-					{
-						ret += ':';
-						if (jump != prevJump)
-							ret += jump;
-						if (components-- > 0)
-						{
-							ret += ':';
-							if (modifierDepth != prevModifierDepth)
-								ret += to_string(modifierDepth);
-						}
-					}
-				}
-			}
-		}
-
-		prevStart = location.start;
-		prevLength = length;
-		prevSourceIndex = sourceIndex;
-		prevJump = jump;
-		prevModifierDepth = modifierDepth;
-	}
-	return ret;
-}
-
 namespace
 {
 

libsolidity/interface/CompilerStack.h

@@ -401,9 +401,6 @@ private:
 	/// @returns the metadata CBOR for the given serialised metadata JSON.
 	bytes createCBORMetadata(std::string const& _metadata, bool _experimentalMode);
 
-	/// @returns the computer source mapping string.
-	std::string computeSourceMapping(evmasm::AssemblyItems const& _items) const;
-
 	/// @returns the contract ABI as a JSON object.
 	/// This will generate the JSON object and store it in the Contract object if it is not present yet.
 	Json::Value const& contractABI(Contract const&) const;

libsolidity/interface/StandardCompiler.cpp

@@ -1081,7 +1081,7 @@ Json::Value StandardCompiler::compileYul(InputsAndSettings _inputsAndSettings)
 		{ "evm.bytecode", "evm.bytecode.object", "evm.bytecode.opcodes", "evm.bytecode.sourceMap", "evm.bytecode.linkReferences" },
 		wildcardMatchesExperimental
 	))
-		output["contracts"][sourceName][contractName]["evm"]["bytecode"] = collectEVMObject(*object.bytecode, nullptr);
+		output["contracts"][sourceName][contractName]["evm"]["bytecode"] = collectEVMObject(*object.bytecode, object.sourceMappings.get());
 
 	if (isArtifactRequested(_inputsAndSettings.outputSelection, sourceName, contractName, "irOptimized", wildcardMatchesExperimental))
 		output["contracts"][sourceName][contractName]["irOptimized"] = stack.print();

libyul/AssemblyStack.cpp

@@ -204,6 +204,12 @@ MachineAssemblyObject AssemblyStack::assemble(Machine _machine) const
 		compileEVM(adapter, false, m_optimiserSettings.optimizeStackAllocation);
 		object.bytecode = make_shared<evmasm::LinkerObject>(assembly.assemble());
 		object.assembly = assembly.assemblyString();
+		object.sourceMappings = make_unique<string>(
+			evmasm::AssemblyItem::computeSourceMapping(
+				assembly.items(),
+				{{scanner().charStream() ? scanner().charStream()->name() : "", 0}}
+			)
+		);
 		return object;
 	}
 	case Machine::EVM15:

libyul/AssemblyStack.h

@@ -48,6 +48,7 @@ struct MachineAssemblyObject
 {
 	std::shared_ptr<evmasm::LinkerObject> bytecode;
 	std::string assembly;
+	std::unique_ptr<std::string> sourceMappings;
 };
 
 /*
@@ -114,6 +115,8 @@ private:
 	std::shared_ptr<yul::Object> m_parserResult;
 	langutil::ErrorList m_errors;
 	langutil::ErrorReporter m_errorReporter;
+
+	std::unique_ptr<std::string> m_sourceMappings;
 };
 
 }

test/cmdlineTests/standard_yul/output.json

@@ -14,7 +14,7 @@
   sstore
     /* \"A\":0:42   */
   pop
-","bytecode":{"linkReferences":{},"object":"bytecode removed","opcodes":"opcodes removed","sourceMap":""}},"ir":"object \"object\" {
+","bytecode":{"linkReferences":{},"object":"bytecode removed","opcodes":"opcodes removed","sourceMap":"sourceMap removed"}},"ir":"object \"object\" {
     code {
         let x := mload(0)
         sstore(add(x, 0), 0)

test/cmdlineTests/standard_yul_object/output.json

@@ -13,7 +13,7 @@
   pop
 stop
 data_4e03657aea45a94fc7d47ba826c8d667c0d1e6e33a64a036ec44f58fa12d6c45 616263
-","bytecode":{"linkReferences":{},"object":"bytecode removed","opcodes":"opcodes removed","sourceMap":""}},"ir":"object \"NamedObject\" {
+","bytecode":{"linkReferences":{},"object":"bytecode removed","opcodes":"opcodes removed","sourceMap":"sourceMap removed"}},"ir":"object \"NamedObject\" {
     code {
         let x := dataoffset(\"DataName\")
         sstore(add(x, 0), 0)

test/cmdlineTests/standard_yul_object_name/output.json

@@ -22,7 +22,7 @@ sub_0: assembly {
         /* \"A\":137:149   */
       revert
 }
-","bytecode":{"linkReferences":{},"object":"bytecode removed","opcodes":"opcodes removed","sourceMap":""}},"ir":"object \"NamedObject\" {
+","bytecode":{"linkReferences":{},"object":"bytecode removed","opcodes":"opcodes removed","sourceMap":"sourceMap removed"}},"ir":"object \"NamedObject\" {
     code {
         let x := dataoffset(\"DataName\")
         sstore(add(x, 0), 0)

test/cmdlineTests/standard_yul_optimized/output.json

@@ -5,7 +5,7 @@
   mload
     /* \"A\":20:40   */
   sstore
-","bytecode":{"linkReferences":{},"object":"bytecode removed","opcodes":"opcodes removed","sourceMap":""}},"ir":"object \"object\" {
+","bytecode":{"linkReferences":{},"object":"bytecode removed","opcodes":"opcodes removed","sourceMap":"sourceMap removed"}},"ir":"object \"object\" {
     code {
         let x := mload(0)
         sstore(add(x, 0), 0)

test/libyul/ObjectCompilerTest.cpp

@@ -74,6 +74,7 @@ TestCase::TestResult ObjectCompilerTest::run(ostream& _stream, string const& _li
 
 	MachineAssemblyObject obj = stack.assemble(AssemblyStack::Machine::EVM);
 	solAssert(obj.bytecode, "");
+	solAssert(obj.sourceMappings, "");
 
 	m_obtainedResult = "Assembly:\n" + obj.assembly;
 	if (obj.bytecode->bytecode.empty())
@@ -84,6 +85,8 @@ TestCase::TestResult ObjectCompilerTest::run(ostream& _stream, string const& _li
 			toHex(obj.bytecode->bytecode) +
 			"\nOpcodes: " +
 			boost::trim_copy(evmasm::disassemble(obj.bytecode->bytecode)) +
+			"\nSourceMappings:" +
+			(obj.sourceMappings->empty() ? "" : " " + *obj.sourceMappings) +
 			"\n";
 
 	if (m_expectation != m_obtainedResult)

test/libyul/objectCompiler/data.yul

@@ -9,3 +9,4 @@ object "a" {
 // data_acaf3289d7b601cbd114fb36c4d29c85bbfd5e133f14cb355c3fd8d99367964f 48656c6c6f2c20576f726c6421
 // Bytecode: fe
 // Opcodes: INVALID
+// SourceMappings:

test/libyul/objectCompiler/datacopy.yul

@@ -46,3 +46,4 @@ object "a" {
 // }
 // Bytecode: 600b600d600039600b6000f3fe6000600055600d600052fe
 // Opcodes: PUSH1 0xB PUSH1 0xD PUSH1 0x0 CODECOPY PUSH1 0xB PUSH1 0x0 RETURN INVALID PUSH1 0x0 PUSH1 0x0 SSTORE PUSH1 0xD PUSH1 0x0 MSTORE INVALID
+// SourceMappings: 26:47:0:-:0;;35:1;26:47;78:26;85:1;78:26

test/libyul/objectCompiler/dataoffset_code.yul

@@ -27,3 +27,4 @@ object "a" {
 // }
 // Bytecode: 6006600055fe6008600055fe
 // Opcodes: PUSH1 0x6 PUSH1 0x0 SSTORE INVALID PUSH1 0x8 PUSH1 0x0 SSTORE INVALID
+// SourceMappings: 22:28:0:-:0;29:1;22:28

test/libyul/objectCompiler/dataoffset_data.yul

@@ -14,3 +14,4 @@ object "a" {
 // data_acaf3289d7b601cbd114fb36c4d29c85bbfd5e133f14cb355c3fd8d99367964f 48656c6c6f2c20576f726c6421
 // Bytecode: 6006600055fe48656c6c6f2c20576f726c6421
 // Opcodes: PUSH1 0x6 PUSH1 0x0 SSTORE INVALID 0x48 PUSH6 0x6C6C6F2C2057 PUSH16 0x726C6421000000000000000000000000
+// SourceMappings: 22:30:0:-:0;29:1;22:30

test/libyul/objectCompiler/dataoffset_self.yul

@@ -14,3 +14,4 @@ object "a" {
 // data_acaf3289d7b601cbd114fb36c4d29c85bbfd5e133f14cb355c3fd8d99367964f 48656c6c6f2c20576f726c6421
 // Bytecode: 6000600055fe
 // Opcodes: PUSH1 0x0 PUSH1 0x0 SSTORE INVALID
+// SourceMappings: 22:26:0:-:0;29:1;22:26

test/libyul/objectCompiler/datasize_code.yul

@@ -27,3 +27,4 @@ object "a" {
 // }
 // Bytecode: 6006600055fe
 // Opcodes: PUSH1 0x6 PUSH1 0x0 SSTORE INVALID
+// SourceMappings: 22:26:0:-:0;29:1;22:26

test/libyul/objectCompiler/datasize_data.yul

@@ -14,3 +14,4 @@ object "a" {
 // data_acaf3289d7b601cbd114fb36c4d29c85bbfd5e133f14cb355c3fd8d99367964f 48656c6c6f2c20576f726c6421
 // Bytecode: 600d600055fe
 // Opcodes: PUSH1 0xD PUSH1 0x0 SSTORE INVALID
+// SourceMappings: 22:28:0:-:0;29:1;22:28

test/libyul/objectCompiler/datasize_self.yul

@@ -14,3 +14,4 @@ object "a" {
 // data_acaf3289d7b601cbd114fb36c4d29c85bbfd5e133f14cb355c3fd8d99367964f 48656c6c6f2c20576f726c6421
 // Bytecode: 6006600055fe
 // Opcodes: PUSH1 0x6 PUSH1 0x0 SSTORE INVALID
+// SourceMappings: 22:24:0:-:0;29:1;22:24

test/libyul/objectCompiler/function_series.yul

@@ -28,3 +28,4 @@ object "Contract" {
 //   sstore
 // Bytecode: 6009565b5b565b5b565b6001600055
 // Opcodes: PUSH1 0x9 JUMP JUMPDEST JUMPDEST JUMP JUMPDEST JUMPDEST JUMP JUMPDEST PUSH1 0x1 PUSH1 0x0 SSTORE
+// SourceMappings: 33:15:0:-:0;;;46:2;;53:15;66:2;;;83:1;80;73:12

test/libyul/objectCompiler/namedObjectCode.yul

@@ -11,3 +11,4 @@ object "a" {
 //   sstore
 // Bytecode: 6001600055
 // Opcodes: PUSH1 0x1 PUSH1 0x0 SSTORE
+// SourceMappings: 32:1:0:-:0;29;22:12

test/libyul/objectCompiler/nested_optimizer.yul

@@ -38,3 +38,4 @@ object "a" {
 // }
 // Bytecode: 600060003555fe
 // Opcodes: PUSH1 0x0 PUSH1 0x0 CALLDATALOAD SSTORE INVALID
+// SourceMappings: 48:1:0:-:0;;35:15;107:20

test/libyul/objectCompiler/simple.yul

@@ -11,3 +11,4 @@
 //   sstore
 // Bytecode: 6001600055
 // Opcodes: PUSH1 0x1 PUSH1 0x0 SSTORE
+// SourceMappings: 14:1:0:-:0;11;4:12

test/libyul/objectCompiler/simple_optimizer.yul

@@ -17,3 +17,4 @@
 //   sstore
 // Bytecode: 600060003555
 // Opcodes: PUSH1 0x0 PUSH1 0x0 CALLDATALOAD SSTORE
+// SourceMappings: 26:1:0:-:0;;13:15;79:20

test/libyul/objectCompiler/subObject.yul

@@ -19,3 +19,4 @@ object "a" {
 // }
 // Bytecode: fe
 // Opcodes: INVALID
+// SourceMappings:

test/libyul/objectCompiler/subSubObject.yul

@@ -37,3 +37,4 @@ object "a" {
 // }
 // Bytecode: fe
 // Opcodes: INVALID
+// SourceMappings: