Merge pull request #11594 from ethereum/ObjectCompiler-use-src

Yul ObjectCompiler: parse `@use-src ...`

libsolutil/StringUtils.h

@@ -177,4 +177,22 @@ inline std::string formatNumberReadable(
 	return str;
 }
 
+/// Safely converts an usigned integer as string into an unsigned int type.
+///
+/// @return the converted number or nullopt in case of an failure (including if it would not fit).
+inline std::optional<unsigned> toUnsignedInt(std::string const& _value)
+{
+	try
+	{
+		auto const ulong = stoul(_value);
+		if (ulong > std::numeric_limits<unsigned>::max())
+			return std::nullopt;
+		return static_cast<unsigned>(ulong);
+	}
+	catch (...)
+	{
+		return std::nullopt;
+	}
+}
+
 }

libyul/AsmParser.h

@@ -72,13 +72,17 @@ public:
 	explicit Parser(
 		langutil::ErrorReporter& _errorReporter,
 		Dialect const& _dialect,
-		std::map<unsigned, std::shared_ptr<std::string const>> _sourceNames
+		std::optional<std::map<unsigned, std::shared_ptr<std::string const>>> _sourceNames
 	):
 		ParserBase(_errorReporter),
 		m_dialect(_dialect),
 		m_sourceNames{std::move(_sourceNames)},
 		m_debugDataOverride{DebugData::create()},
-		m_useSourceLocationFrom{UseSourceLocationFrom::Comments}
+		m_useSourceLocationFrom{
+			m_sourceNames.has_value() ?
+			UseSourceLocationFrom::Comments :
+			UseSourceLocationFrom::Scanner
+		}
 	{}
 
 	/// Parses an inline assembly block starting with `{` and ending with `}`.

libyul/ObjectParser.cpp

@@ -26,6 +26,11 @@
 #include <libyul/Exceptions.h>
 
 #include <liblangutil/Token.h>
+#include <liblangutil/Scanner.h>
+
+#include <libsolutil/StringUtils.h>
+
+#include <regex>
 
 using namespace std;
 using namespace solidity;
@@ -40,6 +45,8 @@ shared_ptr<Object> ObjectParser::parse(shared_ptr<Scanner> const& _scanner, bool
 	{
 		shared_ptr<Object> object;
 		m_scanner = _scanner;
+		m_sourceNameMapping = tryParseSourceNameMapping();
+
 		if (currentToken() == Token::LBrace)
 		{
 			// Special case: Code-only form.
@@ -104,9 +111,62 @@ shared_ptr<Block> ObjectParser::parseCode()
 	return parseBlock();
 }
 
+optional<ObjectParser::SourceNameMap> ObjectParser::tryParseSourceNameMapping() const
+{
+	// @use-src 0:"abc.sol", 1:"foo.sol", 2:"bar.sol"
+	//
+	// UseSrcList := UseSrc (',' UseSrc)*
+	// UseSrc     := [0-9]+ ':' FileName
+	// FileName   := "(([^\"]|\.)*)"
+
+	// Matches some "@use-src TEXT".
+	static std::regex const lineRE = std::regex(
+		"(^|\\s)@use-src\\b",
+		std::regex_constants::ECMAScript | std::regex_constants::optimize
+	);
+	std::smatch sm;
+	if (!std::regex_search(m_scanner->currentCommentLiteral(), sm, lineRE))
+		return nullopt;
+
+	solAssert(sm.size() == 2, "");
+	auto text = m_scanner->currentCommentLiteral().substr(static_cast<size_t>(sm.position() + sm.length()));
+	Scanner scanner(make_shared<CharStream>(text, ""));
+	if (scanner.currentToken() == Token::EOS)
+		return SourceNameMap{};
+	SourceNameMap sourceNames;
+
+	while (scanner.currentToken() != Token::EOS)
+	{
+		if (scanner.currentToken() != Token::Number)
+			break;
+		auto sourceIndex = toUnsignedInt(scanner.currentLiteral());
+		if (!sourceIndex)
+			break;
+		if (scanner.next() != Token::Colon)
+			break;
+		if (scanner.next() != Token::StringLiteral)
+			break;
+		sourceNames[*sourceIndex] = make_shared<string const>(scanner.currentLiteral());
+
+		Token const next = scanner.next();
+		if (next == Token::EOS)
+			return {move(sourceNames)};
+		if (next != Token::Comma)
+			break;
+		scanner.next();
+	}
+
+	m_errorReporter.syntaxError(
+		9804_error,
+		m_scanner->currentCommentLocation(),
+		"Error parsing arguments to @use-src. Expected: <number> \":\" \"<filename>\", ..."
+	);
+	return nullopt;
+}
+
 shared_ptr<Block> ObjectParser::parseBlock()
 {
-	Parser parser(m_errorReporter, m_dialect);
+	Parser parser(m_errorReporter, m_dialect, m_sourceNameMapping);
 	shared_ptr<Block> block = parser.parse(m_scanner, true);
 	yulAssert(block || m_errorReporter.hasErrors(), "Invalid block but no error!");
 	return block;

libyul/ObjectParser.h

@@ -55,7 +55,11 @@ public:
 	/// @returns an empty shared pointer on error.
 	std::shared_ptr<Object> parse(std::shared_ptr<langutil::Scanner> const& _scanner, bool _reuseScanner);
 
+	using SourceNameMap = std::map<unsigned, std::shared_ptr<std::string const>>;
+	std::optional<SourceNameMap> const& sourceNameMapping() const noexcept { return m_sourceNameMapping; }
+
 private:
+	std::optional<SourceNameMap> tryParseSourceNameMapping() const;
 	std::shared_ptr<Object> parseObject(Object* _containingObject = nullptr);
 	std::shared_ptr<Block> parseCode();
 	std::shared_ptr<Block> parseBlock();
@@ -66,6 +70,8 @@ private:
 	void addNamedSubObject(Object& _container, YulString _name, std::shared_ptr<ObjectNode> _subObject);
 
 	Dialect const& m_dialect;
+
+	std::optional<SourceNameMap> m_sourceNameMapping;
 };
 
 }

scripts/error_codes.py

@@ -191,9 +191,9 @@ def examine_id_coverage(top_dir, source_id_to_file_names, new_ids_only=False):
 
     # white list of ids which are not covered by tests
     white_ids = {
-        "6367", # these three are temporarily whitelisted until both PRs are merged.
-        "5798",
+        "9804", # Tested in test/libyul/ObjectParser.cpp.
         "2674",
+        "6367",
         "3805", # "This is a pre-release compiler version, please do not use it in production."
                 # The warning may or may not exist in a compiler build.
         "4591", # "There are more than 256 warnings. Ignoring the rest."

test/libyul/ObjectParser.cpp

@@ -24,16 +24,23 @@
 #include <test/libsolidity/ErrorCheck.h>
 
 #include <libyul/AssemblyStack.h>
+#include <libyul/backends/evm/EVMDialect.h>
 
 #include <libsolidity/interface/OptimiserSettings.h>
 
+#include <boost/algorithm/string.hpp>
 #include <boost/algorithm/string/replace.hpp>
+#include <boost/algorithm/string/split.hpp>
 #include <boost/test/unit_test.hpp>
 
+#include <range/v3/view/iota.hpp>
+
 #include <memory>
 #include <optional>
 #include <string>
+#include <sstream>
 
+using namespace ranges;
 using namespace std;
 using namespace solidity::frontend;
 using namespace solidity::langutil;
@@ -44,7 +51,7 @@ namespace solidity::yul::test
 namespace
 {
 
-std::pair<bool, ErrorList> parse(string const& _source)
+pair<bool, ErrorList> parse(string const& _source)
 {
 	try
 	{
@@ -63,7 +70,7 @@ std::pair<bool, ErrorList> parse(string const& _source)
 	return {false, {}};
 }
 
-std::optional<Error> parseAndReturnFirstError(string const& _source, bool _allowWarnings = true)
+optional<Error> parseAndReturnFirstError(string const& _source, bool _allowWarnings = true)
 {
 	bool success;
 	ErrorList errors;
@@ -88,12 +95,12 @@ std::optional<Error> parseAndReturnFirstError(string const& _source, bool _allow
 	return {};
 }
 
-bool successParse(std::string const& _source, bool _allowWarnings = true)
+bool successParse(string const& _source, bool _allowWarnings = true)
 {
 	return !parseAndReturnFirstError(_source, _allowWarnings);
 }
 
-Error expectError(std::string const& _source, bool _allowWarnings = false)
+Error expectError(string const& _source, bool _allowWarnings = false)
 {
 
 	auto error = parseAndReturnFirstError(_source, _allowWarnings);
@@ -101,6 +108,20 @@ Error expectError(std::string const& _source, bool _allowWarnings = false)
 	return *error;
 }
 
+tuple<optional<ObjectParser::SourceNameMap>, ErrorList> tryGetSourceLocationMapping(string _source)
+{
+	vector<string> lines;
+	boost::split(lines, _source, boost::is_any_of("\n"));
+	string source = util::joinHumanReadablePrefixed(lines, "\n///") + "\n{}\n";
+
+	ErrorList errors;
+	ErrorReporter reporter(errors);
+	Dialect const& dialect = yul::EVMDialect::strictAssemblyForEVM(EVMVersion::berlin());
+	ObjectParser objectParser{reporter, dialect};
+	objectParser.parse(make_shared<Scanner>(CharStream(move(source), "")), false);
+	return {objectParser.sourceNameMapping(), std::move(errors)};
+}
+
 }
 
 #define CHECK_ERROR(text, typ, substring) \
@@ -162,6 +183,98 @@ BOOST_AUTO_TEST_CASE(to_string)
 	BOOST_CHECK_EQUAL(asmStack.print(), expectation);
 }
 
+BOOST_AUTO_TEST_CASE(use_src_empty)
+{
+	auto const [mapping, _] = tryGetSourceLocationMapping("");
+	BOOST_REQUIRE(!mapping);
+}
+
+BOOST_AUTO_TEST_CASE(use_src_simple)
+{
+	auto const [mapping, _] = tryGetSourceLocationMapping(R"(@use-src 0:"contract.sol")");
+	BOOST_REQUIRE(mapping.has_value());
+	BOOST_REQUIRE_EQUAL(mapping->size(), 1);
+	BOOST_REQUIRE_EQUAL(*mapping->at(0), "contract.sol");
+}
+
+BOOST_AUTO_TEST_CASE(use_src_multiple)
+{
+	auto const [mapping, _] = tryGetSourceLocationMapping(R"(@use-src 0:"contract.sol", 1:"misc.yul")");
+	BOOST_REQUIRE(mapping);
+	BOOST_REQUIRE_EQUAL(mapping->size(), 2);
+	BOOST_REQUIRE_EQUAL(*mapping->at(0), "contract.sol");
+	BOOST_REQUIRE_EQUAL(*mapping->at(1), "misc.yul");
+}
+
+BOOST_AUTO_TEST_CASE(use_src_escaped_filenames)
+{
+	auto const [mapping, _] = tryGetSourceLocationMapping(
+		R"(@use-src 42:"con\"tract@\".sol")"
+	);
+	BOOST_REQUIRE(mapping);
+	BOOST_REQUIRE_EQUAL(mapping->size(), 1);
+	BOOST_REQUIRE(mapping->count(42));
+	BOOST_REQUIRE_EQUAL(*mapping->at(42), "con\"tract@\".sol");
+}
+
+BOOST_AUTO_TEST_CASE(use_src_invalid_syntax_malformed_param_1)
+{
+	// open quote arg, missing closing quote
+	auto const [mapping, errors] = tryGetSourceLocationMapping(R"(@use-src 42_"con")");
+
+	BOOST_REQUIRE_EQUAL(errors.size(), 1);
+	BOOST_CHECK_EQUAL(errors.front()->errorId().error, 9804);
+}
+
+BOOST_AUTO_TEST_CASE(use_src_invalid_syntax_malformed_param_2)
+{
+	// open quote arg, missing closing quote
+	auto const [mapping, errors] = tryGetSourceLocationMapping(R"(@use-src 42:"con)");
+
+	BOOST_REQUIRE_EQUAL(errors.size(), 1);
+	BOOST_CHECK_EQUAL(errors.front()->errorId().error, 9804);
+}
+
+BOOST_AUTO_TEST_CASE(use_src_error_unexpected_trailing_tokens)
+{
+	auto const [mapping, errors] = tryGetSourceLocationMapping(
+		R"(@use-src 1:"file.sol" @use-src 2:"foo.sol")"
+	);
+
+	BOOST_REQUIRE_EQUAL(errors.size(), 1);
+	BOOST_CHECK_EQUAL(errors.front()->errorId().error, 9804);
+}
+
+BOOST_AUTO_TEST_CASE(use_src_multiline)
+{
+	auto const [mapping, _] = tryGetSourceLocationMapping(
+		" @use-src \n  0:\"contract.sol\" \n , \n 1:\"misc.yul\""
+	);
+	BOOST_REQUIRE(mapping);
+	BOOST_REQUIRE_EQUAL(mapping->size(), 2);
+	BOOST_REQUIRE_EQUAL(*mapping->at(0), "contract.sol");
+	BOOST_REQUIRE_EQUAL(*mapping->at(1), "misc.yul");
+}
+
+BOOST_AUTO_TEST_CASE(use_src_empty_body)
+{
+	auto const [mapping, _] = tryGetSourceLocationMapping("@use-src");
+	BOOST_REQUIRE(mapping);
+	BOOST_REQUIRE_EQUAL(mapping->size(), 0);
+}
+
+BOOST_AUTO_TEST_CASE(use_src_leading_text)
+{
+	auto const [mapping, _] = tryGetSourceLocationMapping(
+		"@something else @use-src 0:\"contract.sol\", 1:\"misc.sol\""s
+	);
+	BOOST_REQUIRE(mapping);
+	BOOST_REQUIRE_EQUAL(mapping->size(), 2);
+	BOOST_REQUIRE(mapping->find(0) != mapping->end());
+	BOOST_REQUIRE_EQUAL(*mapping->at(0), "contract.sol");
+	BOOST_REQUIRE_EQUAL(*mapping->at(1), "misc.sol");
+}
+
 BOOST_AUTO_TEST_SUITE_END()
 
 }

test/libyul/objectCompiler/sourceLocations.yul

@@ -0,0 +1,53 @@
+// something else
+/*-- another unrelated comment --*/
+/// @use-src 3: "abc.sol" , 2: "def.sol"
+object "a" {
+  code {
+    /// @src 3:0:2
+    datacopy(0, dataoffset("sub"), datasize("sub"))
+    return(0,
+      /** @src 2:5:6 */
+      datasize("sub")
+    )
+  }
+  object "sub" {
+    code {
+      /// @src 2:70:72
+      sstore(0, dataoffset("sub"))
+      /**
+       * @something else
+       * @src 3:2:5
+       */
+      mstore(
+        0,
+        datasize("data1")
+        /// @src 3:90:2
+      )
+    }
+    data "data1" "Hello, World!"
+  }
+}
+// ----
+// Assembly:
+//     /* "abc.sol":0:2   */
+//   codecopy(0x00, dataOffset(sub_0), dataSize(sub_0))
+//     /* "def.sol":5:6   */
+//   dataSize(sub_0)
+//     /* "abc.sol":0:2   */
+//   0x00
+//   return
+// stop
+//
+// sub_0: assembly {
+//         /* "def.sol":70:72   */
+//       0x00
+//       dup1
+//       sstore
+//         /* "abc.sol":2:5   */
+//       mstore(0x00, 0x0d)
+//     stop
+//     data_acaf3289d7b601cbd114fb36c4d29c85bbfd5e133f14cb355c3fd8d99367964f 48656c6c6f2c20576f726c6421
+// }
+// Bytecode: 600a600d600039600a6000f3fe60008055600d600052fe
+// Opcodes: PUSH1 0xA PUSH1 0xD PUSH1 0x0 CODECOPY PUSH1 0xA PUSH1 0x0 RETURN INVALID PUSH1 0x0 DUP1 SSTORE PUSH1 0xD PUSH1 0x0 MSTORE INVALID
+// SourceMappings: 0:2::-:0;;;;5:1;0:2;