documentation comments are now always skipped but saved as special tokens at the Scanner

Scanner.cpp

@@ -102,13 +102,14 @@ int HexValue(char c)
 }
 } // end anonymous namespace
 
-void Scanner::reset(CharStream const& _source, bool _skipDocumentationComments)
+void Scanner::reset(CharStream const& _source)
 {
+	bool found_doc_comment;
 	m_source = _source;
 	m_char = m_source.get();
 	skipWhitespace();
-	scanToken(_skipDocumentationComments);
+	found_doc_comment = scanToken();
-	next(_skipDocumentationComments);
+	next(found_doc_comment);
 }
 
 
@@ -134,10 +135,11 @@ bool Scanner::scanHexByte(char& o_scannedByte)
 // Ensure that tokens can be stored in a byte.
 BOOST_STATIC_ASSERT(Token::NUM_TOKENS <= 0x100);
 
-Token::Value Scanner::next(bool _skipDocumentationComments)
+Token::Value Scanner::next(bool _change_skipped_comment)
 {
 	m_current_token = m_next_token;
-	scanToken(_skipDocumentationComments);
+	if (scanToken() || _change_skipped_comment)
+		m_skipped_comment = m_next_skipped_comment;
 	return m_current_token.token;
 }
 
@@ -180,7 +182,7 @@ Token::Value Scanner::scanDocumentationComment()
 	{
 		char c = m_char;
 		advance();
-		addLiteralChar(c);
+		addCommentLiteralChar(c);
 	}
 	literal.Complete();
 	return Token::COMMENT_LITERAL;
@@ -209,8 +211,9 @@ Token::Value Scanner::skipMultiLineComment()
 	return Token::ILLEGAL;
 }
 
-void Scanner::scanToken(bool _skipDocumentationComments)
+bool Scanner::scanToken()
 {
+	bool found_doc_comment = false;
 	m_next_token.literal.clear();
 	Token::Value token;
 	do
@@ -315,8 +318,16 @@ void Scanner::scanToken(bool _skipDocumentationComments)
 			{
 				if (!advance()) /* double slash comment directly before EOS */
 					token = Token::WHITESPACE;
-				else if (!_skipDocumentationComments)
+				else if (m_char == '/')
-					token = scanDocumentationComment();
+				{
+					Token::Value comment;
+					m_next_skipped_comment.location.start = getSourcePos();
+					comment = scanDocumentationComment();
+					m_next_skipped_comment.location.end = getSourcePos();
+					m_next_skipped_comment.token = comment;
+					token = Token::WHITESPACE;
+					found_doc_comment = true;
+				}
 				else
 					token = skipSingleLineComment();
 			}
@@ -411,6 +422,8 @@ void Scanner::scanToken(bool _skipDocumentationComments)
 	while (token == Token::WHITESPACE);
 	m_next_token.location.end = getSourcePos();
 	m_next_token.token = token;
+
+	return found_doc_comment;
 }
 
 bool Scanner::scanEscape()

Scanner.h

@@ -111,31 +111,34 @@ public:
 	};
 
 	Scanner() { reset(CharStream()); }
-	explicit Scanner(CharStream const& _source, bool _skipDocumentationComments = true)
+	explicit Scanner(CharStream const& _source) { reset(_source); }
-	{
-		reset(_source, _skipDocumentationComments);
-	}
 
 	/// Resets the scanner as if newly constructed with _input as input.
-	void reset(CharStream const& _source, bool _skipDocumentationComments = true);
+	void reset(CharStream const& _source);
 
-	/// Returns the next token and advances input.
+	/// Returns the next token and advances input. If called from reset()
-	Token::Value next(bool _skipDocumentationComments = true);
+	/// and ScanToken() found a documentation token then next should be called
+	/// with _change_skipped_comment=true
+	Token::Value next(bool _change_skipped_comment = false);
 
 	///@{
 	///@name Information about the current token
 
 	/// Returns the current token
-	Token::Value getCurrentToken(bool _skipDocumentationComments = true)
+	Token::Value getCurrentToken()
 	{
-		if (!_skipDocumentationComments)
-			next(_skipDocumentationComments);
 		return m_current_token.token;
 	}
 	Location getCurrentLocation() const { return m_current_token.location; }
 	std::string const& getCurrentLiteral() const { return m_current_token.literal; }
 	///@}
 
+	///@{
+	///@name Information about the current comment token
+	Location getCurrentCommentLocation() const { return m_skipped_comment.location; }
+	std::string const& getCurrentCommentLiteral() const { return m_skipped_comment.literal; }
+	///@}
+
 	///@{
 	///@name Information about the next token
 
@@ -154,7 +157,7 @@ public:
 	///@}
 
 private:
-	// Used for the current and look-ahead token.
+	// Used for the current and look-ahead token and comments
 	struct TokenDesc
 	{
 		Token::Value token;
@@ -166,6 +169,7 @@ private:
 	///@name Literal buffer support
 	inline void startNewLiteral() { m_next_token.literal.clear(); }
 	inline void addLiteralChar(char c) { m_next_token.literal.push_back(c); }
+	inline void addCommentLiteralChar(char c) { m_next_skipped_comment.literal.push_back(c); }
 	inline void dropLiteral() { m_next_token.literal.clear(); }
 	inline void addLiteralCharAndAdvance() { addLiteralChar(m_char); advance(); }
 	///@}
@@ -179,8 +183,9 @@ private:
 
 	bool scanHexByte(char& o_scannedByte);
 
-	/// Scans a single Solidity token.
+	/// Scans a single Solidity token. Returns true if the scanned token was
-	void scanToken(bool _skipDocumentationComments = true);
+	/// a skipped documentation comment. False in all other cases.
+	bool scanToken();
 
 	/// Skips all whitespace and @returns true if something was skipped.
 	bool skipWhitespace();
@@ -203,6 +208,9 @@ private:
 	int getSourcePos() { return m_source.getPos(); }
 	bool isSourcePastEndOfInput() { return m_source.isPastEndOfInput(); }
 
+	TokenDesc m_skipped_comment;  // desc for current skipped comment
+	TokenDesc m_next_skipped_comment; // desc for next skiped comment
+
 	TokenDesc m_current_token;  // desc for current token (as returned by Next())
 	TokenDesc m_next_token;     // desc for next token (one token look-ahead)