Parse everything up to function bodies and report parser errors with location.

AST.h

@@ -32,6 +32,9 @@
 namespace dev {
 namespace solidity {
 
+// Used as pointers to AST nodes, to be replaced by more clever pointers, e.g. pointers which do
+// not do reference counting but point to a special memory area that is completely released
+// explicitly.
 template <class T>
 using ptr = std::shared_ptr<T>;
 template <class T>
@@ -47,9 +50,11 @@ class Expression;
 class ASTNode
 {
 public:
-    explicit ASTNode(const Location& _location)
+    explicit ASTNode(Location const& _location)
         : m_location(_location)
     {}
+
+    Location getLocation() const { return m_location; }
 private:
     Location m_location;
 };
@@ -57,13 +62,12 @@ private:
 class ContractDefinition : public ASTNode
 {
 public:
-    ContractDefinition(const Location& _location,
+    ContractDefinition(Location const& _location,
-                       const std::string& _name,
+                       std::string const& _name,
-                       const vecptr<StructDefinition>& _definedStructs,
+                       vecptr<StructDefinition> const& _definedStructs,
-                       const vecptr<VariableDeclaration>& _stateVariables,
+                       vecptr<VariableDeclaration> const& _stateVariables,
-                       const vecptr<FunctionDefinition>& _definedFunctions)
+                       vecptr<FunctionDefinition> const& _definedFunctions)
-        : ASTNode(_location),
+        : ASTNode(_location), m_name(_name),
-          m_name(_name),
           m_definedStructs(_definedStructs),
           m_stateVariables(_stateVariables),
           m_definedFunctions(_definedFunctions)
@@ -78,33 +82,61 @@ private:
 
 class StructDefinition : public ASTNode
 {
+public:
+    StructDefinition(Location const& _location,
+                     std::string const& _name,
+                     vecptr<VariableDeclaration> const& _members)
+        : ASTNode(_location), m_name(_name), m_members(_members)
+    {}
 private:
     std::string m_name;
     vecptr<VariableDeclaration> m_members;
 };
 
+/// Used as function parameter list and return list
+/// None of the parameters is allowed to contain mappings (not even recursively
+/// inside structs)
+class ParameterList : public ASTNode
+{
+public:
+    ParameterList(Location const& _location, vecptr<VariableDeclaration> const& _parameters)
+        : ASTNode(_location), m_parameters(_parameters)
+    {}
+private:
+    vecptr<VariableDeclaration> m_parameters;
+};
+
 class FunctionDefinition : public ASTNode
 {
+public:
+    FunctionDefinition(Location const& _location, std::string const& _name, bool _isPublic,
+                       ptr<ParameterList> const& _parameters,
+                       bool _isDeclaredConst,
+                       ptr<ParameterList> const& _returnParameters,
+                       ptr<Block> const& _body)
+        : ASTNode(_location), m_name(_name), m_isPublic(_isPublic), m_parameters(_parameters),
+          m_isDeclaredConst(_isDeclaredConst), m_returnParameters(_returnParameters),
+          m_body(_body)
+    {}
 private:
     std::string m_name;
-    vecptr<VariableDeclaration> m_arguments;
+    bool m_isPublic;
+    ptr<ParameterList> m_parameters;
     bool m_isDeclaredConst;
-    vecptr<VariableDeclaration> m_returns;
+    ptr<ParameterList> m_returnParameters;
     ptr<Block> m_body;
 };
 
 class VariableDeclaration : public ASTNode
 {
 public:
-    VariableDeclaration(const Location& _location,
+    VariableDeclaration(Location const& _location,
-                        const ptr<TypeName>& _type,
+                        ptr<TypeName> const& _type,
-                        const std::string& _name)
+                        std::string const& _name)
-        : ASTNode(_location),
+        : ASTNode(_location), m_type(_type), m_name(_name)
-          m_type(_type),
-          m_name(_name)
     {}
 private:
-    ptr<TypeName> m_type; ///<s can be empty ("var")
+    ptr<TypeName> m_type; ///< can be empty ("var")
     std::string m_name;
 };
 
@@ -114,7 +146,7 @@ private:
 class TypeName : public ASTNode
 {
 public:
-    explicit TypeName(const Location& _location)
+    explicit TypeName(Location const& _location)
         : ASTNode(_location)
     {}
 };
@@ -123,7 +155,7 @@ public:
 class ElementaryTypeName : public TypeName
 {
 public:
-    explicit ElementaryTypeName(const Location& _location, Token::Value _type)
+    explicit ElementaryTypeName(Location const& _location, Token::Value _type)
         : TypeName(_location), m_type(_type)
     {}
 private:
@@ -133,18 +165,19 @@ private:
 class UserDefinedTypeName : public TypeName
 {
 public:
-    UserDefinedTypeName(const Location& _location, const std::string& _name)
+    UserDefinedTypeName(Location const& _location, std::string const& _name)
         : TypeName(_location), m_name(_name)
     {}
 private:
     std::string m_name;
 };
 
-class MappingTypeName : public TypeName
+class Mapping : public TypeName
 {
 public:
-    explicit MappingTypeName(const Location& _location)
+    Mapping(Location const& _location, ptr<ElementaryTypeName> const& _keyType,
-        : TypeName(_location)
+            ptr<TypeName> const& _valueType)
+        : TypeName(_location), m_keyType(_keyType), m_valueType(_valueType)
     {}
 private:
     ptr<ElementaryTypeName> m_keyType;
@@ -158,10 +191,18 @@ private:
 
 class Statement : public ASTNode
 {
+public:
+    explicit Statement(Location const& _location)
+        : ASTNode(_location)
+    {}
 };
 
 class Block : public Statement
 {
+public:
+    explicit Block(Location const& _location)
+        : Statement(_location)
+    {}
 private:
     vecptr<Statement> m_statements;
 };
@@ -245,10 +286,12 @@ private:
     Token::Value m_operator;
 };
 
+/// Can be ordinary function call, type cast or struct construction.
 class FunctionCall : public Expression
 {
 private:
-    std::string m_functionName; // TODO only calls to fixed, named functions for now
+    // if m_functionName is the name of a type, store the token directly
+    std::string m_functionName; // "in place" calls of return values are not possible for now
     vecptr<Expression> m_arguments;
 };
 

BaseTypes.h

@@ -4,19 +4,16 @@
 namespace dev {
 namespace solidity {
 
-// Representation of an interval of source positions.
+/// Representation of an interval of source positions.
+/// The interval includes start and excludes end.
 struct Location {
-    Location(int b, int e) : beg_pos(b), end_pos(e) { }
+    Location(int _start, int _end) : start(_start), end(_end) { }
-    Location() : beg_pos(0), end_pos(0) { }
+    Location() : start(-1), end(-1) { }
 
-    bool IsValid() const {
+    bool IsValid() const { return start >= 0 && end >= start; }
-        return beg_pos >= 0 && end_pos >= beg_pos;
-    }
 
-    static Location invalid() { return Location(-1, -1); }
+    int start;
-
+    int end;
-    int beg_pos;
-    int end_pos;
 };
 
 } }

Parser.cpp

@@ -20,6 +20,7 @@
  * Solidity parser.
  */
 
+#include "libdevcore/Log.h"
 #include "libsolidity/BaseTypes.h"
 #include "libsolidity/Parser.h"
 #include "libsolidity/Scanner.h"
@@ -27,9 +28,9 @@
 namespace dev {
 namespace solidity {
 
-ptr<ASTNode> Parser::parse(Scanner& _scanner)
+ptr<ASTNode> Parser::parse(std::shared_ptr<Scanner> const& _scanner)
 {
-    m_scanner = &_scanner;
+    m_scanner = _scanner;
 
     return parseContractDefinition();
 }
@@ -41,20 +42,22 @@ class Parser::ASTNodeFactory
 {
 public:
     ASTNodeFactory(const Parser& _parser)
-        : m_parser(_parser),
+        : m_parser(_parser), m_location(_parser.getPosition(), -1)
-          m_location(_parser.getPosition(), -1)
     {}
 
-    void markEndPosition()
+    void markEndPosition() { m_location.end = m_parser.getEndPosition(); }
+
+    /// Set the end position to the one of the given node.
+    void setEndPositionFromNode(const ptr<ASTNode>& _node)
     {
-        m_location.end_pos = m_parser.getEndPosition();
+        m_location.end = _node->getLocation().end;
     }
 
     /// @todo: check that this actually uses perfect forwarding
     template <class NodeType, typename... Args>
     ptr<NodeType> createNode(Args&&... _args)
     {
-        if (m_location.end_pos < 0) markEndPosition();
+        if (m_location.end < 0) markEndPosition();
         return std::make_shared<NodeType>(m_location, std::forward<Args>(_args)...);
     }
 
@@ -65,12 +68,12 @@ private:
 
 int Parser::getPosition() const
 {
-    return m_scanner->getCurrentLocation().beg_pos;
+    return m_scanner->getCurrentLocation().start;
 }
 
 int Parser::getEndPosition() const
 {
-    return m_scanner->getCurrentLocation().end_pos;
+    return m_scanner->getCurrentLocation().end;
 }
 
 
@@ -98,7 +101,6 @@ ptr<ContractDefinition> Parser::parseContractDefinition()
             functions.push_back(parseFunctionDefinition(visibilityIsPublic));
         } else if (currentToken == Token::STRUCT) {
             structs.push_back(parseStructDefinition());
-            expectToken(Token::SEMICOLON);
         } else if (currentToken == Token::IDENTIFIER || currentToken == Token::MAPPING ||
                    Token::IsElementaryTypeName(currentToken)) {
             stateVariables.push_back(parseVariableDeclaration());
@@ -117,19 +119,57 @@ ptr<ContractDefinition> Parser::parseContractDefinition()
 
 ptr<FunctionDefinition> Parser::parseFunctionDefinition(bool _isPublic)
 {
-    (void) _isPublic;
+    ASTNodeFactory nodeFactory(*this);
-    throwExpectationError("Function parsing is not yet implemented.");
+
+    expectToken(Token::FUNCTION);
+    std::string name(expectIdentifier());
+    ptr<ParameterList> parameters(parseParameterList());
+    bool isDeclaredConst = false;
+    if (m_scanner->getCurrentToken() == Token::CONST) {
+        isDeclaredConst = true;
+        m_scanner->next();
+    }
+    ptr<ParameterList> returnParameters;
+    if (m_scanner->getCurrentToken() == Token::RETURNS) {
+        m_scanner->next();
+        returnParameters = parseParameterList();
+    }
+    ptr<Block> block = parseBlock();
+    nodeFactory.setEndPositionFromNode(block);
+    return nodeFactory.createNode<FunctionDefinition>(name, _isPublic, parameters,
+                                                      isDeclaredConst, returnParameters, block);
 }
 
 ptr<StructDefinition> Parser::parseStructDefinition()
 {
-    throwExpectationError("Struct definition parsing is not yet implemented.");
+    ASTNodeFactory nodeFactory(*this);
+
+    expectToken(Token::STRUCT);
+    std::string name = expectIdentifier();
+    vecptr<VariableDeclaration> members;
+    expectToken(Token::LBRACE);
+    while (m_scanner->getCurrentToken() != Token::RBRACE) {
+        members.push_back(parseVariableDeclaration());
+        expectToken(Token::SEMICOLON);
+    }
+    nodeFactory.markEndPosition();
+    expectToken(Token::RBRACE);
+
+    return nodeFactory.createNode<StructDefinition>(name, members);
 }
 
 ptr<VariableDeclaration> Parser::parseVariableDeclaration()
 {
     ASTNodeFactory nodeFactory(*this);
 
+    ptr<TypeName> type = parseTypeName();
+    nodeFactory.markEndPosition();
+    std::string name = expectIdentifier();
+    return nodeFactory.createNode<VariableDeclaration>(type, name);
+}
+
+ptr<TypeName> Parser::parseTypeName()
+{
     ptr<TypeName> type;
     Token::Value token = m_scanner->getCurrentToken();
     if (Token::IsElementaryTypeName(token)) {
@@ -139,17 +179,67 @@ ptr<VariableDeclaration> Parser::parseVariableDeclaration()
         type = ASTNodeFactory(*this).createNode<TypeName>();
         m_scanner->next();
     } else if (token == Token::MAPPING) {
-        // TODO
+        type = parseMapping();
-        throwExpectationError("mappings are not yet implemented");
     } else if (token == Token::IDENTIFIER) {
         type = ASTNodeFactory(*this).createNode<UserDefinedTypeName>(m_scanner->getCurrentLiteral());
         m_scanner->next();
     } else {
-        throwExpectationError("Expected variable declaration");
+        throwExpectationError("Expected type name");
+    }
+
+    return type;
+}
+
+ptr<Mapping> Parser::parseMapping()
+{
+    ASTNodeFactory nodeFactory(*this);
+
+    expectToken(Token::MAPPING);
+    expectToken(Token::LPAREN);
+
+    if (!Token::IsElementaryTypeName(m_scanner->getCurrentToken()))
+        throwExpectationError("Expected elementary type name for mapping key type");
+    ptr<ElementaryTypeName> keyType;
+    keyType = ASTNodeFactory(*this).createNode<ElementaryTypeName>(m_scanner->getCurrentToken());
+    m_scanner->next();
+
+    expectToken(Token::ARROW);
+    ptr<TypeName> valueType = parseTypeName();
+    nodeFactory.markEndPosition();
+    expectToken(Token::RPAREN);
+
+    return nodeFactory.createNode<Mapping>(keyType, valueType);
+}
+
+ptr<ParameterList> Parser::parseParameterList()
+{
+    ASTNodeFactory nodeFactory(*this);
+
+    vecptr<VariableDeclaration> parameters;
+    expectToken(Token::LPAREN);
+    if (m_scanner->getCurrentToken() != Token::RPAREN) {
+        parameters.push_back(parseVariableDeclaration());
+        while (m_scanner->getCurrentToken() != Token::RPAREN) {
+            expectToken(Token::COMMA);
+            parameters.push_back(parseVariableDeclaration());
+        }
     }
     nodeFactory.markEndPosition();
-    std::string name = expectIdentifier();
+    m_scanner->next();
-    return nodeFactory.createNode<VariableDeclaration>(type, name);
+    return nodeFactory.createNode<ParameterList>(parameters);
+}
+
+ptr<Block> Parser::parseBlock()
+{
+    ASTNodeFactory nodeFactory(*this);
+    expectToken(Token::LBRACE);
+    while (m_scanner->getCurrentToken() != Token::RBRACE) {
+        m_scanner->next();
+        // @todo
+    }
+    nodeFactory.markEndPosition();
+    expectToken(Token::RBRACE);
+    return nodeFactory.createNode<Block>();
 }
 
 void Parser::expectToken(Token::Value _value)
@@ -171,9 +261,16 @@ std::string Parser::expectIdentifier()
 
 void Parser::throwExpectationError(const std::string& _description)
 {
-    (void) _description;
+    int line, column;
+    std::tie(line, column) = m_scanner->translatePositionToLineColumn(getPosition());
+    cwarn << "Solidity parser error: " << _description
+          << "at line " << (line + 1)
+          << ", column " << (column + 1);
+    cwarn << m_scanner->getLineAtPosition(getPosition());
+    cwarn << std::string(column, ' ') << "^";
+
     /// @todo make a proper exception hierarchy
-    throw std::exception();//_description);
+    throw std::exception();
 }
 
 

Parser.h

@@ -32,7 +32,7 @@ class Scanner;
 class Parser
 {
 public:
-    ptr<ASTNode> parse(Scanner& _scanner);
+    ptr<ASTNode> parse(std::shared_ptr<Scanner> const& _scanner);
 
 private:
     class ASTNodeFactory;
@@ -48,6 +48,10 @@ private:
     ptr<FunctionDefinition> parseFunctionDefinition(bool _isPublic);
     ptr<StructDefinition> parseStructDefinition();
     ptr<VariableDeclaration> parseVariableDeclaration();
+    ptr<TypeName> parseTypeName();
+    ptr<Mapping> parseMapping();
+    ptr<ParameterList> parseParameterList();
+    ptr<Block> parseBlock();
     /// @}
 
     /// Helper functions
@@ -58,7 +62,7 @@ private:
     void throwExpectationError(const std::string& _description);
     /// @}
 
-    Scanner* m_scanner;
+    std::shared_ptr<Scanner> m_scanner;
 };
 
 } }

Scanner.cpp

@@ -40,6 +40,9 @@
 // You should have received a copy of the GNU General Public License
 // along with cpp-ethereum.  If not, see <http://www.gnu.org/licenses/>.
 
+#include <algorithm>
+#include <tuple>
+
 #include <libsolidity/Scanner.h>
 
 namespace dev {
@@ -121,6 +124,7 @@ Token::Value Scanner::next()
     return m_current_token.token;
 }
 
+
 bool Scanner::skipWhitespace()
 {
     const int start_position = getSourcePos();
@@ -182,7 +186,7 @@ void Scanner::scanToken()
   Token::Value token;
   do {
     // Remember the position of the next token
-    m_next_token.location.beg_pos = getSourcePos();
+    m_next_token.location.start = getSourcePos();
 
     switch (m_char) {
       case '\n':
@@ -401,7 +405,7 @@ void Scanner::scanToken()
     // whitespace.
   } while (token == Token::WHITESPACE);
 
-  m_next_token.location.end_pos = getSourcePos();
+  m_next_token.location.end = getSourcePos();
   m_next_token.token = token;
 }
 
@@ -546,7 +550,7 @@ Token::Value Scanner::scanNumber(bool _periodSeen)
 
 #define KEYWORDS(KEYWORD_GROUP, KEYWORD)                                     \
   KEYWORD_GROUP('a')                                                         \
-  KEYWORD("address", Token::BREAK)                                           \
+  KEYWORD("address", Token::ADDRESS)                                           \
   KEYWORD_GROUP('b')                                                         \
   KEYWORD("break", Token::BREAK)                                             \
   KEYWORD("bool", Token::BOOL)                                               \
@@ -588,8 +592,8 @@ Token::Value Scanner::scanNumber(bool _periodSeen)
   KEYWORD("interface", Token::FUTURE_STRICT_RESERVED_WORD)                   \
   KEYWORD_GROUP('l')                                                         \
   KEYWORD_GROUP('m')                                                         \
-  KEYWORD_GROUP('n')                                                         \
   KEYWORD("mapping", Token::MAPPING)                                         \
+  KEYWORD_GROUP('n')                                                         \
   KEYWORD("new", Token::NEW)                                                 \
   KEYWORD("null", Token::NULL_LITERAL)                                       \
   KEYWORD_GROUP('p')                                                         \
@@ -600,8 +604,9 @@ Token::Value Scanner::scanNumber(bool _periodSeen)
   KEYWORD_GROUP('r')                                                         \
   KEYWORD("real", Token::REAL)                                               \
   KEYWORD("return", Token::RETURN)                                           \
+  KEYWORD("returns", Token::RETURNS)                                         \
   KEYWORD_GROUP('s')                                                         \
-  KEYWORD("string", Token::STRING_TYPE)                                           \
+  KEYWORD("string", Token::STRING_TYPE)                                      \
   KEYWORD("struct", Token::STRUCT)                                           \
   KEYWORD("switch", Token::SWITCH)                                           \
   KEYWORD_GROUP('t')                                                         \
@@ -671,5 +676,38 @@ Token::Value Scanner::scanIdentifierOrKeyword()
     return KeywordOrIdentifierToken(m_next_token.literal);
 }
 
+std::string CharStream::getLineAtPosition(int _position) const
+{
+    // if _position points to \n, it returns the line before the \n
+    using size_type = std::string::size_type;
+    size_type searchStart = std::min<size_type>(m_source.size(), _position);
+    if (searchStart > 0) searchStart--;
+    size_type lineStart = m_source.rfind('\n', searchStart);
+    if (lineStart == std::string::npos)
+        lineStart = 0;
+    else
+        lineStart++;
+    return m_source.substr(lineStart,
+                           std::min(m_source.find('\n', lineStart),
+                                    m_source.size()) - lineStart);
+}
+
+std::tuple<int, int> CharStream::translatePositionToLineColumn(int _position) const
+{
+    using size_type = std::string::size_type;
+    size_type searchPosition = std::min<size_type>(m_source.size(), _position);
+    int lineNumber = std::count(m_source.begin(), m_source.begin() + searchPosition, '\n');
+
+    size_type lineStart;
+    if (searchPosition == 0) {
+        lineStart = 0;
+    } else {
+        lineStart = m_source.rfind('\n', searchPosition - 1);
+        lineStart = lineStart == std::string::npos ? 0 : lineStart + 1;
+    }
+
+    return std::tuple<int, int>(lineNumber, searchPosition - lineStart);
+}
+
 
 } }

Scanner.h

@@ -82,6 +82,12 @@ public:
         return get();
     }
 
+    /// Functions that help pretty-printing parse errors
+    /// Do only use in error cases, they are quite expensive.
+    /// @{
+    std::string getLineAtPosition(int _position) const;
+    std::tuple<int, int> translatePositionToLineColumn(int _position) const;
+    /// @}
 private:
     std::string m_source;
     size_t m_pos;
@@ -134,6 +140,16 @@ public:
     Location peekLocation() const { return m_next_token.location; }
     const std::string& peekLiteral() const { return m_next_token.literal; }
 
+    /// Functions that help pretty-printing parse errors.
+    /// Do only use in error cases, they are quite expensive.
+    /// @{
+    std::string getLineAtPosition(int _position) const { return m_source.getLineAtPosition(_position); }
+    std::tuple<int, int> translatePositionToLineColumn(int _position) const
+    {
+        return m_source.translatePositionToLineColumn(_position);
+    }
+    /// @}
+
     // Returns true if there was a line terminator before the peek'ed token,
     // possibly inside a multi-line comment.
     bool hasAnyLineTerminatorBeforeNext() const {

Token.h

@@ -169,6 +169,7 @@ namespace solidity {
   K(PUBLIC, "public", 0)                                             \
   K(PRIVATE, "private", 0)                                           \
   K(RETURN, "return", 0)                                             \
+  K(RETURNS, "returns", 0)                                           \
   K(STRUCT, "struct", 0)                                             \
   K(SWITCH, "switch", 0)                                             \
   K(THIS, "this", 0)                                                 \

grammar.txt

@@ -1,18 +1,18 @@
 ContractDefinition = 'contract' Identifier '{' ContractPart* '}'
-ContractPart = VariableDeclaration ';' | StructDefinition ';' |
+ContractPart = VariableDeclaration ';' | StructDefinition |
-               FunctionDefinition ';' | 'public:' | 'private:'
+               FunctionDefinition | 'public:' | 'private:'
 
 StructDefinition = 'struct' Identifier '{'
                      ( VariableDeclaration (';' VariableDeclaration)* )? '}
 
-FunctionDefinition = 'function' Identifier ArgumentList 'const'?
+FunctionDefinition = 'function' Identifier ParameterList 'const'?
-                     'returns' ArgumentList Block
+                     ( 'returns' ParameterList )? Block
-ArgumentList = '(' ( VariableDeclaration (',' VariableDeclaration)* )? ')'
+ParameterList = '(' ( VariableDeclaration (',' VariableDeclaration)* )? ')'
 // semantic restriction: mappings and structs (recursively) containing mappings
 // are not allowed in argument lists
 VariableDeclaration = TypeName Identifier
-TypeName = PredefinedType | Identifier | MappingType
+TypeName = ElementaryTypeName | Identifier | Mapping
-MappingType = 'mapping' '(' SimplePredefinedType '=>' TypeName ')'
+Mapping = 'mapping' '(' ElementaryTypeName '=>' TypeName ')'
 
 Block = '{' Statement* '}'
 Statement = IfStatement | WhileStatement | Continue | Break | Return | VariableAssignment | Expression ';' | Block