From 6ddfebafae2ec419ea40cd1907a700ba9812593c Mon Sep 17 00:00:00 2001 From: Lefteris Karapetsas Date: Thu, 27 Nov 2014 15:21:22 +0100 Subject: [PATCH 1/8] Solidity function AST nodes get documentation attribute --- AST.h | 20 +++++++++++++------- Parser.cpp | 5 +++-- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/AST.h b/AST.h index 3fb251d95..dc34e3695 100644 --- a/AST.h +++ b/AST.h @@ -174,13 +174,17 @@ class FunctionDefinition: public Declaration { public: FunctionDefinition(Location const& _location, ASTPointer const& _name, bool _isPublic, - ASTPointer const& _parameters, - bool _isDeclaredConst, - ASTPointer const& _returnParameters, - ASTPointer const& _body): - Declaration(_location, _name), m_isPublic(_isPublic), m_parameters(_parameters), - m_isDeclaredConst(_isDeclaredConst), m_returnParameters(_returnParameters), - m_body(_body) {} + std::string const& _documentation, + ASTPointer const& _parameters, + bool _isDeclaredConst, + ASTPointer const& _returnParameters, + ASTPointer const& _body): + Declaration(_location, _name), m_isPublic(_isPublic), + m_parameters(_parameters), + m_isDeclaredConst(_isDeclaredConst), + m_returnParameters(_returnParameters), + m_body(_body), + m_documentation(_documentation) {} virtual void accept(ASTVisitor& _visitor) override; bool isPublic() const { return m_isPublic; } @@ -190,6 +194,7 @@ public: std::vector> const& getReturnParameters() const { return m_returnParameters->getParameters(); } ASTPointer const& getReturnParameterList() const { return m_returnParameters; } Block& getBody() { return *m_body; } + std::string& getDocumentation() { return m_documentation; } void addLocalVariable(VariableDeclaration const& _localVariable) { m_localVariables.push_back(&_localVariable); } std::vector const& getLocalVariables() const { return m_localVariables; } @@ -203,6 +208,7 @@ private: bool m_isDeclaredConst; ASTPointer m_returnParameters; ASTPointer m_body; + std::string m_documentation; std::vector m_localVariables; }; diff --git a/Parser.cpp b/Parser.cpp index 276da0728..3cf44014f 100644 --- a/Parser.cpp +++ b/Parser.cpp @@ -142,8 +142,9 @@ ASTPointer Parser::parseFunctionDefinition(bool _isPublic) } ASTPointer block = parseBlock(); nodeFactory.setEndPositionFromNode(block); - return nodeFactory.createNode(name, _isPublic, parameters, - isDeclaredConst, returnParameters, block); + return nodeFactory.createNode(name, _isPublic, m_scanner->getCurrentCommentLiteral(), + parameters, + isDeclaredConst, returnParameters, block); } ASTPointer Parser::parseStructDefinition() From e11e651929aaffe23d03b706c6770f92a57af6a6 Mon Sep 17 00:00:00 2001 From: Lefteris Karapetsas Date: Thu, 27 Nov 2014 18:57:50 +0100 Subject: [PATCH 2/8] Solidity work for documentation strings - Still a work in progress - Parser now properly gets each function's doc comment - Small changes in the scanner - Multiline comments are considered --- Parser.cpp | 4 +++- Scanner.cpp | 34 +++++++++++++++++++++++++++++----- Scanner.h | 41 +++++++++++++++++++++++++++++++++++++---- 3 files changed, 69 insertions(+), 10 deletions(-) diff --git a/Parser.cpp b/Parser.cpp index 3cf44014f..17fd2d78c 100644 --- a/Parser.cpp +++ b/Parser.cpp @@ -118,6 +118,8 @@ ASTPointer Parser::parseFunctionDefinition(bool _isPublic) { ASTNodeFactory nodeFactory(*this); expectToken(Token::FUNCTION); + std::string docstring = m_scanner->getCurrentCommentLiteral(); + m_scanner->clearCurrentCommentLiteral(); ASTPointer name(expectIdentifierToken()); ASTPointer parameters(parseParameterList()); bool isDeclaredConst = false; @@ -142,7 +144,7 @@ ASTPointer Parser::parseFunctionDefinition(bool _isPublic) } ASTPointer block = parseBlock(); nodeFactory.setEndPositionFromNode(block); - return nodeFactory.createNode(name, _isPublic, m_scanner->getCurrentCommentLiteral(), + return nodeFactory.createNode(name, _isPublic, docstring, parameters, isDeclaredConst, returnParameters, block); } diff --git a/Scanner.cpp b/Scanner.cpp index dd18a320f..c40d98af5 100644 --- a/Scanner.cpp +++ b/Scanner.cpp @@ -180,10 +180,26 @@ Token::Value Scanner::skipSingleLineComment() /// For the moment this function simply consumes a single line triple slash doc comment Token::Value Scanner::scanDocumentationComment() { - LiteralScope literal(this); + LiteralScope literal(this, LITERAL_TYPE_COMMENT); advance(); //consume the last '/' - while (!isSourcePastEndOfInput() && !isLineTerminator(m_char)) + while (!isSourcePastEndOfInput()) { + if (isLineTerminator(m_char)) + { + // check if next line is also a documentation comment + skipWhitespace(); + if (m_source.get(0) == '/' && + m_source.get(1) == '/' && + m_source.get(2) == '/' && + !m_source.isPastEndOfInput(3)) + { + m_source.advanceBy(3); + addCommentLiteralChar('\n'); + } + else + break; // next line is not a documentation comment, we are done + + } addCommentLiteralChar(m_char); advance(); } @@ -474,7 +490,7 @@ Token::Value Scanner::scanString() { char const quote = m_char; advance(); // consume quote - LiteralScope literal(this); + LiteralScope literal(this, LITERAL_TYPE_STRING); while (m_char != quote && !isSourcePastEndOfInput() && !isLineTerminator(m_char)) { char c = m_char; @@ -505,7 +521,7 @@ void Scanner::scanDecimalDigits() Token::Value Scanner::scanNumber(char _charSeen) { enum { DECIMAL, HEX, BINARY } kind = DECIMAL; - LiteralScope literal(this); + LiteralScope literal(this, LITERAL_TYPE_NUMBER); if (_charSeen == '.') { // we have already seen a decimal point of the float @@ -758,7 +774,7 @@ Token::Value Scanner::scanIdentifierOrKeyword() { if (asserts(isIdentifierStart(m_char))) BOOST_THROW_EXCEPTION(InternalCompilerError()); - LiteralScope literal(this); + LiteralScope literal(this, LITERAL_TYPE_STRING); addLiteralCharAndAdvance(); // Scan the rest of the identifier characters. while (isIdentifierPart(m_char)) @@ -777,6 +793,14 @@ char CharStream::advanceAndGet() return get(); } +void CharStream::advanceBy(size_t _chars) +{ + if (asserts(!isPastEndOfInput(_chars))) + BOOST_THROW_EXCEPTION(InternalCompilerError()); + + m_pos += _chars; +} + char CharStream::rollback(size_t _amount) { if (asserts(m_pos >= _amount)) diff --git a/Scanner.h b/Scanner.h index 957f02b1f..5123ccccc 100644 --- a/Scanner.h +++ b/Scanner.h @@ -74,9 +74,10 @@ public: CharStream(): m_pos(0) {} explicit CharStream(std::string const& _source): m_source(_source), m_pos(0) {} int getPos() const { return m_pos; } - bool isPastEndOfInput() const { return m_pos >= m_source.size(); } - char get() const { return m_source[m_pos]; } + bool isPastEndOfInput(size_t _charsForward = 0) const { return (m_pos + _charsForward) >= m_source.size(); } + char get(size_t _charsForward = 0) const { return m_source[m_pos + _charsForward]; } char advanceAndGet(); + void advanceBy(size_t _chars); char rollback(size_t _amount); ///@{ @@ -93,19 +94,45 @@ private: }; + class Scanner { public: + + enum LiteralType { + LITERAL_TYPE_STRING, + LITERAL_TYPE_NUMBER, // not really different from string type in behaviour + LITERAL_TYPE_COMMENT + }; /// Scoped helper for literal recording. Automatically drops the literal /// if aborting the scanning before it's complete. class LiteralScope { public: - explicit LiteralScope(Scanner* self): m_scanner(self), m_complete(false) { m_scanner->startNewLiteral(); } - ~LiteralScope() { if (!m_complete) m_scanner->dropLiteral(); } + explicit LiteralScope(Scanner* _self, enum LiteralType _type) + : m_type(_type) + , m_scanner(_self) + , m_complete(false) + { + if (_type == LITERAL_TYPE_COMMENT) + m_scanner->startNewCommentLiteral(); + else + m_scanner->startNewLiteral(); + } + ~LiteralScope() + { + if (!m_complete) + { + if (m_type == LITERAL_TYPE_COMMENT) + m_scanner->dropCommentLiteral(); + else + m_scanner->dropLiteral(); + } + } void complete() { m_complete = true; } private: + enum LiteralType m_type; Scanner* m_scanner; bool m_complete; }; @@ -133,8 +160,12 @@ public: ///@{ ///@name Information about the current comment token + Location getCurrentCommentLocation() const { return m_skippedComment.location; } std::string const& getCurrentCommentLiteral() const { return m_skippedComment.literal; } + /// Called by the parser during FunctionDefinition parsing to clear the current comment + void clearCurrentCommentLiteral() { m_skippedComment.literal.clear(); } + ///@} ///@{ @@ -166,9 +197,11 @@ private: ///@{ ///@name Literal buffer support inline void startNewLiteral() { m_nextToken.literal.clear(); } + inline void startNewCommentLiteral() { m_nextSkippedComment.literal.clear(); } inline void addLiteralChar(char c) { m_nextToken.literal.push_back(c); } inline void addCommentLiteralChar(char c) { m_nextSkippedComment.literal.push_back(c); } inline void dropLiteral() { m_nextToken.literal.clear(); } + inline void dropCommentLiteral() { m_nextSkippedComment.literal.clear(); } inline void addLiteralCharAndAdvance() { addLiteralChar(m_char); advance(); } ///@} From 034870b4c0e7c2ea6b786e4a023dc8e64ae7314c Mon Sep 17 00:00:00 2001 From: Lefteris Karapetsas Date: Fri, 28 Nov 2014 00:06:38 +0100 Subject: [PATCH 3/8] Fixing multiline comment parsing in solidity --- Scanner.cpp | 6 ++++-- Scanner.h | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Scanner.cpp b/Scanner.cpp index c40d98af5..4da9874a2 100644 --- a/Scanner.cpp +++ b/Scanner.cpp @@ -193,8 +193,8 @@ Token::Value Scanner::scanDocumentationComment() m_source.get(2) == '/' && !m_source.isPastEndOfInput(3)) { - m_source.advanceBy(3); addCommentLiteralChar('\n'); + m_char = m_source.advanceBy(3); } else break; // next line is not a documentation comment, we are done @@ -793,12 +793,14 @@ char CharStream::advanceAndGet() return get(); } -void CharStream::advanceBy(size_t _chars) +char CharStream::advanceBy(size_t _chars) { if (asserts(!isPastEndOfInput(_chars))) BOOST_THROW_EXCEPTION(InternalCompilerError()); m_pos += _chars; + + return m_source[m_pos]; } char CharStream::rollback(size_t _amount) diff --git a/Scanner.h b/Scanner.h index 5123ccccc..edec344ac 100644 --- a/Scanner.h +++ b/Scanner.h @@ -77,7 +77,7 @@ public: bool isPastEndOfInput(size_t _charsForward = 0) const { return (m_pos + _charsForward) >= m_source.size(); } char get(size_t _charsForward = 0) const { return m_source[m_pos + _charsForward]; } char advanceAndGet(); - void advanceBy(size_t _chars); + char advanceBy(size_t _chars); char rollback(size_t _amount); ///@{ From e3e90c4e6bf3a5cb6399d6cfb8677d79a4a3f166 Mon Sep 17 00:00:00 2001 From: Lefteris Karapetsas Date: Fri, 28 Nov 2014 01:26:37 +0100 Subject: [PATCH 4/8] Solidity FunctionDefinition used ASTString shared ptr for docstrings --- AST.h | 15 ++++++++++----- Parser.cpp | 8 ++++++-- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/AST.h b/AST.h index dc34e3695..81a12ad1a 100644 --- a/AST.h +++ b/AST.h @@ -173,8 +173,9 @@ private: class FunctionDefinition: public Declaration { public: - FunctionDefinition(Location const& _location, ASTPointer const& _name, bool _isPublic, - std::string const& _documentation, + FunctionDefinition(Location const& _location, ASTPointer const& _name, + bool _isPublic, + ASTPointer const& _documentation, ASTPointer const& _parameters, bool _isDeclaredConst, ASTPointer const& _returnParameters, @@ -184,7 +185,9 @@ public: m_isDeclaredConst(_isDeclaredConst), m_returnParameters(_returnParameters), m_body(_body), - m_documentation(_documentation) {} + m_documentation(_documentation) + {} + virtual void accept(ASTVisitor& _visitor) override; bool isPublic() const { return m_isPublic; } @@ -194,7 +197,9 @@ public: std::vector> const& getReturnParameters() const { return m_returnParameters->getParameters(); } ASTPointer const& getReturnParameterList() const { return m_returnParameters; } Block& getBody() { return *m_body; } - std::string& getDocumentation() { return m_documentation; } + /// @return A shared pointer of an ASTString. + /// Can contain a nullptr in which case indicates absence of documentation + ASTPointer const& getDocumentation() { return m_documentation; } void addLocalVariable(VariableDeclaration const& _localVariable) { m_localVariables.push_back(&_localVariable); } std::vector const& getLocalVariables() const { return m_localVariables; } @@ -208,7 +213,7 @@ private: bool m_isDeclaredConst; ASTPointer m_returnParameters; ASTPointer m_body; - std::string m_documentation; + ASTPointer m_documentation; std::vector m_localVariables; }; diff --git a/Parser.cpp b/Parser.cpp index 17fd2d78c..72921623a 100644 --- a/Parser.cpp +++ b/Parser.cpp @@ -117,9 +117,13 @@ ASTPointer Parser::parseContractDefinition() ASTPointer Parser::parseFunctionDefinition(bool _isPublic) { ASTNodeFactory nodeFactory(*this); + ASTPointer docstring; expectToken(Token::FUNCTION); - std::string docstring = m_scanner->getCurrentCommentLiteral(); - m_scanner->clearCurrentCommentLiteral(); + if (m_scanner->getCurrentCommentLiteral() != "") + { + docstring = std::make_shared(m_scanner->getCurrentCommentLiteral()); + m_scanner->clearCurrentCommentLiteral(); + } ASTPointer name(expectIdentifierToken()); ASTPointer parameters(parseParameterList()); bool isDeclaredConst = false; From db7b118ece60ae20f20c871bd257a1af5366423f Mon Sep 17 00:00:00 2001 From: Lefteris Karapetsas Date: Fri, 28 Nov 2014 11:17:18 +0100 Subject: [PATCH 5/8] Solidity natspec docstring test improvements - Adding a test for docstring being between function signature and function body - Properly checking for exceptions in parsing - Small parser fix --- Parser.cpp | 2 +- Scanner.h | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Parser.cpp b/Parser.cpp index 72921623a..9ed081cce 100644 --- a/Parser.cpp +++ b/Parser.cpp @@ -118,12 +118,12 @@ ASTPointer Parser::parseFunctionDefinition(bool _isPublic) { ASTNodeFactory nodeFactory(*this); ASTPointer docstring; - expectToken(Token::FUNCTION); if (m_scanner->getCurrentCommentLiteral() != "") { docstring = std::make_shared(m_scanner->getCurrentCommentLiteral()); m_scanner->clearCurrentCommentLiteral(); } + expectToken(Token::FUNCTION); ASTPointer name(expectIdentifierToken()); ASTPointer parameters(parseParameterList()); bool isDeclaredConst = false; diff --git a/Scanner.h b/Scanner.h index edec344ac..702310102 100644 --- a/Scanner.h +++ b/Scanner.h @@ -109,8 +109,7 @@ public: class LiteralScope { public: - explicit LiteralScope(Scanner* _self, enum LiteralType _type) - : m_type(_type) + explicit LiteralScope(Scanner* _self, enum LiteralType _type): m_type(_type) , m_scanner(_self) , m_complete(false) { From a59546473916fcccb2ddc444e7264a94c551c617 Mon Sep 17 00:00:00 2001 From: Lefteris Karapetsas Date: Sun, 30 Nov 2014 22:43:40 +0100 Subject: [PATCH 6/8] Simplifying scanning for natspec documentation - Scanner no longer remembers the last natspect comment until a new one is encountered. It remembers it only until the next scan() --- Parser.cpp | 4 +--- Scanner.cpp | 44 +++++++++++++++----------------------------- Scanner.h | 8 +++----- 3 files changed, 19 insertions(+), 37 deletions(-) diff --git a/Parser.cpp b/Parser.cpp index 9ed081cce..0506bc3e3 100644 --- a/Parser.cpp +++ b/Parser.cpp @@ -119,10 +119,8 @@ ASTPointer Parser::parseFunctionDefinition(bool _isPublic) ASTNodeFactory nodeFactory(*this); ASTPointer docstring; if (m_scanner->getCurrentCommentLiteral() != "") - { docstring = std::make_shared(m_scanner->getCurrentCommentLiteral()); - m_scanner->clearCurrentCommentLiteral(); - } + expectToken(Token::FUNCTION); ASTPointer name(expectIdentifierToken()); ASTPointer parameters(parseParameterList()); diff --git a/Scanner.cpp b/Scanner.cpp index 4da9874a2..3335e9df4 100644 --- a/Scanner.cpp +++ b/Scanner.cpp @@ -104,16 +104,14 @@ int hexValue(char c) void Scanner::reset(CharStream const& _source) { - bool foundDocComment; m_source = _source; m_char = m_source.get(); skipWhitespace(); - foundDocComment = scanToken(); + scanToken(); - // special version of Scanner:next() taking the previous scanToken() result into account m_currentToken = m_nextToken; - if (scanToken() || foundDocComment) - m_skippedComment = m_nextSkippedComment; + m_skippedComment = m_nextSkippedComment; + scanToken(); } @@ -142,8 +140,9 @@ BOOST_STATIC_ASSERT(Token::NUM_TOKENS <= 0x100); Token::Value Scanner::next() { m_currentToken = m_nextToken; - if (scanToken()) - m_skippedComment = m_nextSkippedComment; + m_skippedComment = m_nextSkippedComment; + scanToken(); + return m_currentToken.token; } @@ -188,13 +187,13 @@ Token::Value Scanner::scanDocumentationComment() { // check if next line is also a documentation comment skipWhitespace(); - if (m_source.get(0) == '/' && + if (!m_source.isPastEndOfInput(3) && + m_source.get(0) == '/' && m_source.get(1) == '/' && - m_source.get(2) == '/' && - !m_source.isPastEndOfInput(3)) + m_source.get(2) == '/') { addCommentLiteralChar('\n'); - m_char = m_source.advanceBy(3); + m_char = m_source.advanceAndGet(3); } else break; // next line is not a documentation comment, we are done @@ -230,10 +229,10 @@ Token::Value Scanner::skipMultiLineComment() return Token::ILLEGAL; } -bool Scanner::scanToken() +void Scanner::scanToken() { - bool foundDocComment = false; m_nextToken.literal.clear(); + m_nextSkippedComment.literal.clear(); Token::Value token; do { @@ -345,7 +344,6 @@ bool Scanner::scanToken() m_nextSkippedComment.location.end = getSourcePos(); m_nextSkippedComment.token = comment; token = Token::WHITESPACE; - foundDocComment = true; } else token = skipSingleLineComment(); @@ -441,8 +439,6 @@ bool Scanner::scanToken() while (token == Token::WHITESPACE); m_nextToken.location.end = getSourcePos(); m_nextToken.token = token; - - return foundDocComment; } bool Scanner::scanEscape() @@ -783,23 +779,13 @@ Token::Value Scanner::scanIdentifierOrKeyword() return KeywordOrIdentifierToken(m_nextToken.literal); } -char CharStream::advanceAndGet() +char CharStream::advanceAndGet(size_t _chars) { if (isPastEndOfInput()) return 0; - ++m_pos; - if (isPastEndOfInput()) - return 0; - return get(); -} - -char CharStream::advanceBy(size_t _chars) -{ - if (asserts(!isPastEndOfInput(_chars))) - BOOST_THROW_EXCEPTION(InternalCompilerError()); - m_pos += _chars; - + if (isPastEndOfInput()) + return 0; return m_source[m_pos]; } diff --git a/Scanner.h b/Scanner.h index 702310102..e9262ba39 100644 --- a/Scanner.h +++ b/Scanner.h @@ -76,8 +76,7 @@ public: int getPos() const { return m_pos; } bool isPastEndOfInput(size_t _charsForward = 0) const { return (m_pos + _charsForward) >= m_source.size(); } char get(size_t _charsForward = 0) const { return m_source[m_pos + _charsForward]; } - char advanceAndGet(); - char advanceBy(size_t _chars); + char advanceAndGet(size_t _chars=1); char rollback(size_t _amount); ///@{ @@ -213,9 +212,8 @@ private: bool scanHexByte(char& o_scannedByte); - /// Scans a single Solidity token. Returns true if the scanned token was - /// a skipped documentation comment. False in all other cases. - bool scanToken(); + /// Scans a single Solidity token. + void scanToken(); /// Skips all whitespace and @returns true if something was skipped. bool skipWhitespace(); From 21fdd84c09edd837fde1299c8e4f0b9545a8c1fa Mon Sep 17 00:00:00 2001 From: Lefteris Karapetsas Date: Sun, 30 Nov 2014 23:25:42 +0100 Subject: [PATCH 7/8] Moving LiteralScope to Scanner.cpp --- Scanner.cpp | 41 +++++++++++++++++++++++++++++++++++++++++ Scanner.h | 42 +----------------------------------------- 2 files changed, 42 insertions(+), 41 deletions(-) diff --git a/Scanner.cpp b/Scanner.cpp index 3335e9df4..6ef8a6c79 100644 --- a/Scanner.cpp +++ b/Scanner.cpp @@ -102,6 +102,47 @@ int hexValue(char c) } } // end anonymous namespace + + +/// Scoped helper for literal recording. Automatically drops the literal +/// if aborting the scanning before it's complete. +enum LiteralType { + LITERAL_TYPE_STRING, + LITERAL_TYPE_NUMBER, // not really different from string type in behaviour + LITERAL_TYPE_COMMENT +}; + +class LiteralScope +{ +public: + explicit LiteralScope(Scanner* _self, enum LiteralType _type): m_type(_type) + , m_scanner(_self) + , m_complete(false) + { + if (_type == LITERAL_TYPE_COMMENT) + m_scanner->m_nextSkippedComment.literal.clear(); + else + m_scanner->m_nextToken.literal.clear(); + } + ~LiteralScope() + { + if (!m_complete) + { + if (m_type == LITERAL_TYPE_COMMENT) + m_scanner->m_nextSkippedComment.literal.clear(); + else + m_scanner->m_nextToken.literal.clear(); + } + } + void complete() { m_complete = true; } + +private: + enum LiteralType m_type; + Scanner* m_scanner; + bool m_complete; +}; // end of LiteralScope class + + void Scanner::reset(CharStream const& _source) { m_source = _source; diff --git a/Scanner.h b/Scanner.h index e9262ba39..49ac3651c 100644 --- a/Scanner.h +++ b/Scanner.h @@ -96,45 +96,9 @@ private: class Scanner { + friend class LiteralScope; public: - enum LiteralType { - LITERAL_TYPE_STRING, - LITERAL_TYPE_NUMBER, // not really different from string type in behaviour - LITERAL_TYPE_COMMENT - }; - /// Scoped helper for literal recording. Automatically drops the literal - /// if aborting the scanning before it's complete. - class LiteralScope - { - public: - explicit LiteralScope(Scanner* _self, enum LiteralType _type): m_type(_type) - , m_scanner(_self) - , m_complete(false) - { - if (_type == LITERAL_TYPE_COMMENT) - m_scanner->startNewCommentLiteral(); - else - m_scanner->startNewLiteral(); - } - ~LiteralScope() - { - if (!m_complete) - { - if (m_type == LITERAL_TYPE_COMMENT) - m_scanner->dropCommentLiteral(); - else - m_scanner->dropLiteral(); - } - } - void complete() { m_complete = true; } - - private: - enum LiteralType m_type; - Scanner* m_scanner; - bool m_complete; - }; - Scanner() { reset(CharStream()); } explicit Scanner(CharStream const& _source) { reset(_source); } @@ -194,12 +158,8 @@ private: ///@{ ///@name Literal buffer support - inline void startNewLiteral() { m_nextToken.literal.clear(); } - inline void startNewCommentLiteral() { m_nextSkippedComment.literal.clear(); } inline void addLiteralChar(char c) { m_nextToken.literal.push_back(c); } inline void addCommentLiteralChar(char c) { m_nextSkippedComment.literal.push_back(c); } - inline void dropLiteral() { m_nextToken.literal.clear(); } - inline void dropCommentLiteral() { m_nextSkippedComment.literal.clear(); } inline void addLiteralCharAndAdvance() { addLiteralChar(m_char); advance(); } ///@} From 4b845073105bc76793a72a092630fc3bc5077882 Mon Sep 17 00:00:00 2001 From: Lefteris Karapetsas Date: Mon, 1 Dec 2014 01:05:55 +0100 Subject: [PATCH 8/8] simplifying Scanner::reset() --- Scanner.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Scanner.cpp b/Scanner.cpp index 6ef8a6c79..4ffc2223f 100644 --- a/Scanner.cpp +++ b/Scanner.cpp @@ -150,9 +150,7 @@ void Scanner::reset(CharStream const& _source) skipWhitespace(); scanToken(); - m_currentToken = m_nextToken; - m_skippedComment = m_nextSkippedComment; - scanToken(); + next(); }