Merge pull request #5358 from ethereum/yulObjects

[Yul] Yul objects parser
This commit is contained in:
chriseth 2018-11-28 11:33:29 +01:00 committed by GitHub
commit 7cbf046864
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 635 additions and 34 deletions

View File

@ -11,6 +11,7 @@ Compiler Features:
* SMTChecker: Support ``msg``, ``tx`` and ``block`` member variables.
* SMTChecker: Support ``gasleft()`` and ``blockhash()`` functions.
* SMTChecker: Support internal bound function calls.
* Yul: Support Yul objects in ``--assemble`` and ``--yul`` commandline options.
Bugfixes:

View File

@ -30,6 +30,7 @@
#include <libyul/AsmCodeGen.h>
#include <libyul/backends/evm/EVMCodeTransform.h>
#include <libyul/backends/evm/EVMAssembly.h>
#include <libyul/ObjectParser.h>
#include <libevmasm/Assembly.h>
@ -69,30 +70,22 @@ bool AssemblyStack::parseAndAnalyze(std::string const& _sourceName, std::string
m_errors.clear();
m_analysisSuccessful = false;
m_scanner = make_shared<Scanner>(CharStream(_source), _sourceName);
m_parserResult = yul::Parser(m_errorReporter, languageToAsmFlavour(m_language)).parse(m_scanner, false);
m_parserResult = yul::ObjectParser(m_errorReporter, languageToAsmFlavour(m_language)).parse(m_scanner, false);
if (!m_errorReporter.errors().empty())
return false;
solAssert(m_parserResult, "");
return analyzeParsed();
}
bool AssemblyStack::analyze(yul::Block const& _block, Scanner const* _scanner)
{
m_errors.clear();
m_analysisSuccessful = false;
if (_scanner)
m_scanner = make_shared<Scanner>(*_scanner);
m_parserResult = make_shared<yul::Block>(_block);
solAssert(m_parserResult->code, "");
return analyzeParsed();
}
bool AssemblyStack::analyzeParsed()
{
m_analysisInfo = make_shared<yul::AsmAnalysisInfo>();
yul::AsmAnalyzer analyzer(*m_analysisInfo, m_errorReporter, m_evmVersion, boost::none, languageToAsmFlavour(m_language));
m_analysisSuccessful = analyzer.analyze(*m_parserResult);
solAssert(m_parserResult, "");
solAssert(m_parserResult->code, "");
m_parserResult->analysisInfo = make_shared<yul::AsmAnalysisInfo>();
yul::AsmAnalyzer analyzer(*m_parserResult->analysisInfo, m_errorReporter, m_evmVersion, boost::none, languageToAsmFlavour(m_language));
m_analysisSuccessful = analyzer.analyze(*m_parserResult->code);
return m_analysisSuccessful;
}
@ -100,7 +93,8 @@ MachineAssemblyObject AssemblyStack::assemble(Machine _machine) const
{
solAssert(m_analysisSuccessful, "");
solAssert(m_parserResult, "");
solAssert(m_analysisInfo, "");
solAssert(m_parserResult->code, "");
solAssert(m_parserResult->analysisInfo, "");
switch (_machine)
{
@ -108,7 +102,7 @@ MachineAssemblyObject AssemblyStack::assemble(Machine _machine) const
{
MachineAssemblyObject object;
eth::Assembly assembly;
yul::CodeGenerator::assemble(*m_parserResult, *m_analysisInfo, assembly);
yul::CodeGenerator::assemble(*m_parserResult->code, *m_parserResult->analysisInfo, assembly);
object.bytecode = make_shared<eth::LinkerObject>(assembly.assemble());
object.assembly = assembly.assemblyString();
return object;
@ -117,7 +111,7 @@ MachineAssemblyObject AssemblyStack::assemble(Machine _machine) const
{
MachineAssemblyObject object;
yul::EVMAssembly assembly(true);
yul::CodeTransform(assembly, *m_analysisInfo, m_language == Language::Yul, true)(*m_parserResult);
yul::CodeTransform(assembly, *m_parserResult->analysisInfo, m_language == Language::Yul, true)(*m_parserResult->code);
object.bytecode = make_shared<eth::LinkerObject>(assembly.finalize());
/// TODO: fill out text representation
return object;
@ -132,5 +126,6 @@ MachineAssemblyObject AssemblyStack::assemble(Machine _machine) const
string AssemblyStack::print() const
{
solAssert(m_parserResult, "");
return yul::AsmPrinter(m_language == Language::Yul)(*m_parserResult);
solAssert(m_parserResult->code, "");
return m_parserResult->toString(m_language == Language::Yul) + "\n";
}

View File

@ -24,6 +24,9 @@
#include <liblangutil/ErrorReporter.h>
#include <liblangutil/EVMVersion.h>
#include <libyul/Object.h>
#include <libyul/ObjectParser.h>
#include <libevmasm/LinkerObject.h>
#include <string>
@ -34,12 +37,6 @@ namespace langutil
class Scanner;
}
namespace yul
{
struct AsmAnalysisInfo;
struct Block;
}
namespace dev
{
namespace solidity
@ -72,10 +69,6 @@ public:
/// Multiple calls overwrite the previous state.
bool parseAndAnalyze(std::string const& _sourceName, std::string const& _source);
/// Runs analysis step on the supplied block, returns false if input cannot be assembled.
/// Multiple calls overwrite the previous state.
bool analyze(yul::Block const& _block, langutil::Scanner const* _scanner = nullptr);
/// Run the assembly step (should only be called after parseAndAnalyze).
MachineAssemblyObject assemble(Machine _machine) const;
@ -94,8 +87,7 @@ private:
std::shared_ptr<langutil::Scanner> m_scanner;
bool m_analysisSuccessful = false;
std::shared_ptr<yul::Block> m_parserResult;
std::shared_ptr<yul::AsmAnalysisInfo> m_analysisInfo;
std::shared_ptr<yul::Object> m_parserResult;
langutil::ErrorList m_errors;
langutil::ErrorReporter m_errorReporter;
};

View File

@ -6,6 +6,8 @@ add_library(yul
AsmPrinter.cpp
AsmScope.cpp
AsmScopeFiller.cpp
Object.cpp
ObjectParser.cpp
backends/evm/EVMAssembly.cpp
backends/evm/EVMCodeTransform.cpp
optimiser/ASTCopier.cpp

61
libyul/Object.cpp Normal file
View File

@ -0,0 +1,61 @@
/*
This file is part of solidity.
solidity is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
solidity is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with solidity. If not, see <http://www.gnu.org/licenses/>.
*/
/**
* Yul code and data object container.
*/
#include <libyul/Object.h>
#include <libyul/AsmPrinter.h>
#include <libyul/Exceptions.h>
#include <libdevcore/Visitor.h>
#include <libdevcore/CommonData.h>
#include <boost/algorithm/string/replace.hpp>
using namespace dev;
using namespace yul;
using namespace std;
namespace
{
string indent(std::string const& _input)
{
if (_input.empty())
return _input;
return boost::replace_all_copy(" " + _input, "\n", "\n ");
}
}
string Data::toString(bool) const
{
return "data \"" + name.str() + "\" hex\"" + dev::toHex(data) + "\"";
}
string Object::toString(bool _yul) const
{
yulAssert(code, "No code");
string inner = "code " + AsmPrinter{_yul}(*code);
for (auto const& obj: subObjects)
inner += "\n" + obj->toString(_yul);
return "object \"" + name.str() + "\" {\n" + indent(inner) + "\n}";
}

72
libyul/Object.h Normal file
View File

@ -0,0 +1,72 @@
/*
This file is part of solidity.
solidity is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
solidity is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with solidity. If not, see <http://www.gnu.org/licenses/>.
*/
/**
* Yul code and data object container.
*/
#pragma once
#include <libyul/AsmDataForward.h>
#include <libyul/YulString.h>
#include <libdevcore/Common.h>
#include <memory>
namespace yul
{
struct AsmAnalysisInfo;
/**
* Generic base class for both Yul objects and Yul data.
*/
struct ObjectNode
{
virtual ~ObjectNode() {}
virtual std::string toString(bool _yul) const = 0;
YulString name;
};
/**
* Named data in Yul objects.
*/
struct Data: ObjectNode
{
Data(YulString _name, dev::bytes _data): data(std::move(_data)) { name = _name; }
std::string toString(bool _yul) const override;
dev::bytes data;
};
/**
* Yul code and data object container.
*/
struct Object: ObjectNode
{
public:
/// @returns a (parseable) string representation. Includes types if @a _yul is set.
std::string toString(bool _yul) const override;
std::shared_ptr<Block> code;
std::vector<std::shared_ptr<ObjectNode>> subObjects;
std::map<YulString, size_t> subIndexByName;
std::shared_ptr<yul::AsmAnalysisInfo> analysisInfo;
};
}

146
libyul/ObjectParser.cpp Normal file
View File

@ -0,0 +1,146 @@
/*
This file is part of solidity.
solidity is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
solidity is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with solidity. If not, see <http://www.gnu.org/licenses/>.
*/
/**
* Parser for Yul code and data object container.
*/
#include <libyul/ObjectParser.h>
#include <libyul/AsmParser.h>
#include <libyul/Exceptions.h>
#include <liblangutil/Token.h>
using namespace dev;
using namespace langutil;
using namespace yul;
using namespace std;
shared_ptr<Object> ObjectParser::parse(shared_ptr<Scanner> const& _scanner, bool _reuseScanner)
{
m_recursionDepth = 0;
try
{
shared_ptr<Object> object;
m_scanner = _scanner;
if (currentToken() == Token::LBrace)
{
// Special case: Code-only form.
object = make_shared<Object>();
object->name = YulString{"object"};
object->code = parseBlock();
if (!object->code)
return nullptr;
}
else
object = parseObject();
if (object && !_reuseScanner)
expectToken(Token::EOS);
return object;
}
catch (FatalError const&)
{
if (m_errorReporter.errors().empty())
throw; // Something is weird here, rather throw again.
}
return nullptr;
}
shared_ptr<Object> ObjectParser::parseObject(Object* _containingObject)
{
RecursionGuard guard(*this);
if (currentToken() != Token::Identifier || currentLiteral() != "object")
fatalParserError("Expected keyword \"object\".");
advance();
shared_ptr<Object> ret = make_shared<Object>();
ret->name = parseUniqueName(_containingObject);
expectToken(Token::LBrace);
ret->code = parseCode();
while (currentToken() != Token::RBrace)
{
if (currentToken() == Token::Identifier && currentLiteral() == "object")
parseObject(ret.get());
else if (currentToken() == Token::Identifier && currentLiteral() == "data")
parseData(*ret);
else
fatalParserError("Expected keyword \"data\" or \"object\" or \"}\".");
}
if (_containingObject)
addNamedSubObject(*_containingObject, ret->name, ret);
expectToken(Token::RBrace);
return ret;
}
shared_ptr<Block> ObjectParser::parseCode()
{
if (currentToken() != Token::Identifier || currentLiteral() != "code")
fatalParserError("Expected keyword \"code\".");
advance();
return parseBlock();
}
shared_ptr<Block> ObjectParser::parseBlock()
{
Parser parser(m_errorReporter, m_flavour);
shared_ptr<Block> block = parser.parse(m_scanner, true);
yulAssert(block || m_errorReporter.hasErrors(), "Invalid block but no error!");
return block;
}
void ObjectParser::parseData(Object& _containingObject)
{
solAssert(
currentToken() == Token::Identifier && currentLiteral() == "data",
"parseData called on wrong input."
);
advance();
YulString name = parseUniqueName(&_containingObject);
expectToken(Token::StringLiteral, false);
addNamedSubObject(_containingObject, name, make_shared<Data>(name, asBytes(currentLiteral())));
advance();
}
YulString ObjectParser::parseUniqueName(Object const* _containingObject)
{
expectToken(Token::StringLiteral, false);
YulString name{currentLiteral()};
if (name.empty())
parserError("Object name cannot be empty.");
else if (_containingObject && _containingObject->name == name)
parserError("Object name cannot be the same as the name of the containing object.");
else if (_containingObject && _containingObject->subIndexByName.count(name))
parserError("Object name \"" + name.str() + "\" already exists inside the containing object.");
advance();
return name;
}
void ObjectParser::addNamedSubObject(Object& _container, YulString _name, shared_ptr<ObjectNode> _subObject)
{
_container.subIndexByName[_name] = _container.subObjects.size();
_container.subObjects.emplace_back(std::move(_subObject));
}

72
libyul/ObjectParser.h Normal file
View File

@ -0,0 +1,72 @@
/*
This file is part of solidity.
solidity is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
solidity is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with solidity. If not, see <http://www.gnu.org/licenses/>.
*/
/**
* Parser for Yul code and data object container.
*/
#pragma once
#include <libyul/YulString.h>
#include <libyul/Object.h>
#include <liblangutil/ErrorReporter.h>
#include <liblangutil/ParserBase.h>
#include <libdevcore/Common.h>
#include <memory>
namespace langutil
{
class Scanner;
}
namespace yul
{
/**
* Yul object parser. Invokes the inline assembly parser.
*/
class ObjectParser: public langutil::ParserBase
{
public:
explicit ObjectParser(
langutil::ErrorReporter& _errorReporter,
yul::AsmFlavour _flavour = yul::AsmFlavour::Loose
):
ParserBase(_errorReporter), m_flavour(_flavour) {}
/// Parses a Yul object.
/// Falls back to code-only parsing if the source starts with `{`.
/// @param _reuseScanner if true, do check for end of input after the last `}`.
/// @returns an empty shared pointer on error.
std::shared_ptr<Object> parse(std::shared_ptr<langutil::Scanner> const& _scanner, bool _reuseScanner);
private:
std::shared_ptr<Object> parseObject(Object* _containingObject = nullptr);
std::shared_ptr<Block> parseCode();
std::shared_ptr<Block> parseBlock();
void parseData(Object& _containingObject);
/// Tries to parse a name that is non-empty and unique inside the containing object.
YulString parseUniqueName(Object const* _containingObject);
void addNamedSubObject(Object& _container, YulString _name, std::shared_ptr<ObjectNode> _subObject);
yul::AsmFlavour m_flavour;
};
}

View File

@ -124,7 +124,8 @@ void parsePrintCompare(string const& _source, bool _canWarn = false)
BOOST_REQUIRE(Error::containsOnlyWarnings(stack.errors()));
else
BOOST_REQUIRE(stack.errors().empty());
BOOST_CHECK_EQUAL(stack.print(), _source);
string expectation = "object \"object\" {\n code " + boost::replace_all_copy(_source, "\n", "\n ") + "\n}\n";
BOOST_CHECK_EQUAL(stack.print(), expectation);
}
}
@ -567,12 +568,14 @@ BOOST_AUTO_TEST_CASE(print_string_literals)
BOOST_AUTO_TEST_CASE(print_string_literal_unicode)
{
string source = "{ let x := \"\\u1bac\" }";
string parsed = "{\n let x := \"\\xe1\\xae\\xac\"\n}";
string parsed = "object \"object\" {\n code {\n let x := \"\\xe1\\xae\\xac\"\n }\n}\n";
AssemblyStack stack(dev::test::Options::get().evmVersion());
BOOST_REQUIRE(stack.parseAndAnalyze("", source));
BOOST_REQUIRE(stack.errors().empty());
BOOST_CHECK_EQUAL(stack.print(), parsed);
parsePrintCompare(parsed);
string parsedInner = "{\n let x := \"\\xe1\\xae\\xac\"\n}";
parsePrintCompare(parsedInner);
}
BOOST_AUTO_TEST_CASE(print_if)

View File

@ -0,0 +1,257 @@
/*
This file is part of solidity.
solidity is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
solidity is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with solidity. If not, see <http://www.gnu.org/licenses/>.
*/
/**
* @date 2018
* Unit tests for the Yul object parser.
*/
#include <test/Options.h>
#include <test/libsolidity/ErrorCheck.h>
#include <libsolidity/interface/AssemblyStack.h>
#include <boost/optional.hpp>
#include <boost/algorithm/string/replace.hpp>
#include <string>
#include <memory>
using namespace std;
using namespace langutil;
namespace dev
{
namespace yul
{
namespace test
{
namespace
{
std::pair<bool, ErrorList> parse(string const& _source)
{
try
{
solidity::AssemblyStack asmStack(
dev::test::Options::get().evmVersion(),
solidity::AssemblyStack::Language::StrictAssembly
);
bool success = asmStack.parseAndAnalyze("source", _source);
return {success, asmStack.errors()};
}
catch (FatalError const&)
{
BOOST_FAIL("Fatal error leaked.");
}
return {false, {}};
}
boost::optional<Error> parseAndReturnFirstError(string const& _source, bool _allowWarnings = true)
{
bool success;
ErrorList errors;
tie(success, errors) = parse(_source);
if (!success)
{
BOOST_REQUIRE_EQUAL(errors.size(), 1);
return *errors.front();
}
else
{
// If success is true, there might still be an error in the assembly stage.
if (_allowWarnings && Error::containsOnlyWarnings(errors))
return {};
else if (!errors.empty())
{
if (!_allowWarnings)
BOOST_CHECK_EQUAL(errors.size(), 1);
return *errors.front();
}
}
return {};
}
bool successParse(std::string const& _source, bool _allowWarnings = true)
{
return !parseAndReturnFirstError(_source, _allowWarnings);
}
Error expectError(std::string const& _source, bool _allowWarnings = false)
{
auto error = parseAndReturnFirstError(_source, _allowWarnings);
BOOST_REQUIRE(error);
return *error;
}
}
#define CHECK_ERROR(text, typ, substring) \
do \
{ \
Error err = expectError((text), false); \
BOOST_CHECK(err.type() == (Error::Type::typ)); \
BOOST_CHECK(solidity::searchErrorMessage(err, (substring))); \
} while(0)
BOOST_AUTO_TEST_SUITE(YulObjectParser)
BOOST_AUTO_TEST_CASE(empty_code)
{
BOOST_CHECK(successParse("{ }"));
}
BOOST_AUTO_TEST_CASE(object_with_empty_code)
{
BOOST_CHECK(successParse("object \"a\" { code { } }"));
}
BOOST_AUTO_TEST_CASE(non_object)
{
CHECK_ERROR("code {}", ParserError, "Expected keyword \"object\"");
}
BOOST_AUTO_TEST_CASE(empty_name)
{
CHECK_ERROR("object \"\" { code {} }", ParserError, "Object name cannot be empty");
}
BOOST_AUTO_TEST_CASE(recursion_depth)
{
string input;
for (size_t i = 0; i < 20000; i++)
input += "object \"a" + to_string(i) + "\" { code {} ";
for (size_t i = 0; i < 20000; i++)
input += "}";
CHECK_ERROR(input, ParserError, "recursion");
}
BOOST_AUTO_TEST_CASE(object_with_code)
{
BOOST_CHECK(successParse("object \"a\" { code { let x := mload(0) sstore(0, x) } }"));
}
BOOST_AUTO_TEST_CASE(object_with_code_and_data)
{
BOOST_CHECK(successParse("object \"a\" { code { let x := mload(0) sstore(0, x) } data \"b\" hex\"01010202\" }"));
}
BOOST_AUTO_TEST_CASE(object_with_non_code_at_start)
{
CHECK_ERROR("object \"a\" { data \"d\" hex\"0102\" code { } }", ParserError, "Expected keyword \"code\"");
}
BOOST_AUTO_TEST_CASE(nested_object)
{
string code = R"(
object "outer" {
code { let x := mload(0) }
data "x" "stringdata"
object "inner" {
code { mstore(0, 1) }
object "inner inner" { code {} }
data "innerx" "abc"
data "innery" "def"
}
}
)";
BOOST_CHECK(successParse(code));
}
BOOST_AUTO_TEST_CASE(incomplete)
{
CHECK_ERROR("object \"abc\" { code {} } object", ParserError, "Expected end of source");
}
BOOST_AUTO_TEST_CASE(reuse_object_name)
{
string code = R"(
object "outer" {
code { }
data "outer" "stringdata"
}
)";
CHECK_ERROR(code, ParserError, "Object name cannot be the same as the name of the containing object");
}
BOOST_AUTO_TEST_CASE(reuse_object_in_subobject)
{
string code = R"(
object "outer" {
code { }
object "outer" { code {} }
}
)";
CHECK_ERROR(code, ParserError, "Object name cannot be the same as the name of the containing object");
}
BOOST_AUTO_TEST_CASE(reuse_object_of_sibling)
{
string code = R"(
object "O" {
code { }
object "i" { code {} }
data "i" "abc"
}
)";
CHECK_ERROR(code, ParserError, "already exists inside the");
}
BOOST_AUTO_TEST_CASE(to_string)
{
string code = R"(
object "O" {
code { let x := mload(0) if x { sstore(0, 1) } }
object "i" { code {} data "j" "def" }
data "j" "abc"
data "k" hex"010203"
}
)";
string expectation = R"(object "O" {
code {
let x := mload(0)
if x
{
sstore(0, 1)
}
}
object "i" {
code {
}
data "j" hex"646566"
}
data "j" hex"616263"
data "k" hex"010203"
}
)";
expectation = boost::replace_all_copy(expectation, "\t", " ");
solidity::AssemblyStack asmStack(
dev::test::Options::get().evmVersion(),
solidity::AssemblyStack::Language::StrictAssembly
);
BOOST_REQUIRE(asmStack.parseAndAnalyze("source", code));
BOOST_CHECK_EQUAL(asmStack.print(), expectation);
}
BOOST_AUTO_TEST_SUITE_END()
}
}
} // end namespaces