Solidity work for documentation strings

- Still a work in progress

- Parser now properly gets each function's doc comment

- Small changes in the scanner

- Multiline comments are considered

Parser.cpp

@@ -118,6 +118,8 @@ ASTPointer<FunctionDefinition> Parser::parseFunctionDefinition(bool _isPublic)
 {
 	ASTNodeFactory nodeFactory(*this);
 	expectToken(Token::FUNCTION);
+	std::string docstring = m_scanner->getCurrentCommentLiteral();
+	m_scanner->clearCurrentCommentLiteral();
 	ASTPointer<ASTString> name(expectIdentifierToken());
 	ASTPointer<ParameterList> parameters(parseParameterList());
 	bool isDeclaredConst = false;
@@ -142,7 +144,7 @@ ASTPointer<FunctionDefinition> Parser::parseFunctionDefinition(bool _isPublic)
 	}
 	ASTPointer<Block> block = parseBlock();
 	nodeFactory.setEndPositionFromNode(block);
-	return nodeFactory.createNode<FunctionDefinition>(name, _isPublic, m_scanner->getCurrentCommentLiteral(), 
+	return nodeFactory.createNode<FunctionDefinition>(name, _isPublic, docstring,
 													  parameters,
 													  isDeclaredConst, returnParameters, block);
 }

Scanner.cpp

@@ -180,10 +180,26 @@ Token::Value Scanner::skipSingleLineComment()
 /// For the moment this function simply consumes a single line triple slash doc comment
 Token::Value Scanner::scanDocumentationComment()
 {
-	LiteralScope literal(this);
+	LiteralScope literal(this, LITERAL_TYPE_COMMENT);
 	advance(); //consume the last '/'
-	while (!isSourcePastEndOfInput() && !isLineTerminator(m_char))
+	while (!isSourcePastEndOfInput())
 	{
+		if (isLineTerminator(m_char))
+		{
+			// check if next line is also a documentation comment
+			skipWhitespace();
+			if (m_source.get(0) == '/' &&
+				m_source.get(1) == '/' &&
+				m_source.get(2) == '/' &&
+				!m_source.isPastEndOfInput(3))
+			{
+				m_source.advanceBy(3);
+				addCommentLiteralChar('\n');
+			}
+			else
+				break; // next line is not a documentation comment, we are done
+
+		}
 		addCommentLiteralChar(m_char);
 		advance();
 	}
@@ -474,7 +490,7 @@ Token::Value Scanner::scanString()
 {
 	char const quote = m_char;
 	advance();  // consume quote
-	LiteralScope literal(this);
+	LiteralScope literal(this, LITERAL_TYPE_STRING);
 	while (m_char != quote && !isSourcePastEndOfInput() && !isLineTerminator(m_char))
 	{
 		char c = m_char;
@@ -505,7 +521,7 @@ void Scanner::scanDecimalDigits()
 Token::Value Scanner::scanNumber(char _charSeen)
 {
 	enum { DECIMAL, HEX, BINARY } kind = DECIMAL;
-	LiteralScope literal(this);
+	LiteralScope literal(this, LITERAL_TYPE_NUMBER);
 	if (_charSeen == '.')
 	{
 		// we have already seen a decimal point of the float
@@ -758,7 +774,7 @@ Token::Value Scanner::scanIdentifierOrKeyword()
 {
 	if (asserts(isIdentifierStart(m_char)))
 		BOOST_THROW_EXCEPTION(InternalCompilerError());
-	LiteralScope literal(this);
+	LiteralScope literal(this, LITERAL_TYPE_STRING);
 	addLiteralCharAndAdvance();
 	// Scan the rest of the identifier characters.
 	while (isIdentifierPart(m_char))
@@ -777,6 +793,14 @@ char CharStream::advanceAndGet()
 	return get();
 }
 
+void CharStream::advanceBy(size_t _chars)
+{
+	if (asserts(!isPastEndOfInput(_chars)))
+		BOOST_THROW_EXCEPTION(InternalCompilerError());
+
+	m_pos += _chars;
+}
+
 char CharStream::rollback(size_t _amount)
 {
 	if (asserts(m_pos >= _amount))

Scanner.h

@@ -74,9 +74,10 @@ public:
 	CharStream(): m_pos(0) {}
 	explicit CharStream(std::string const& _source): m_source(_source), m_pos(0) {}
 	int getPos() const { return m_pos; }
-	bool isPastEndOfInput() const { return m_pos >= m_source.size(); }
-	char get() const { return m_source[m_pos]; }
+	bool isPastEndOfInput(size_t _charsForward = 0) const { return (m_pos + _charsForward) >= m_source.size(); }
+	char get(size_t _charsForward = 0) const { return m_source[m_pos + _charsForward]; }
 	char advanceAndGet();
+	void advanceBy(size_t _chars);
 	char rollback(size_t _amount);
 
 	///@{
@@ -93,19 +94,45 @@ private:
 };
 
 
+
 class Scanner
 {
 public:
+
+	enum LiteralType {
+		LITERAL_TYPE_STRING,
+		LITERAL_TYPE_NUMBER, // not really different from string type in behaviour
+		LITERAL_TYPE_COMMENT
+	};
 	/// Scoped helper for literal recording. Automatically drops the literal
 	/// if aborting the scanning before it's complete.
 	class LiteralScope
 	{
 	public:
-		explicit LiteralScope(Scanner* self): m_scanner(self), m_complete(false) { m_scanner->startNewLiteral(); }
-		~LiteralScope() { if (!m_complete) m_scanner->dropLiteral(); }
+		explicit LiteralScope(Scanner* _self, enum LiteralType _type)
+		: m_type(_type)
+		, m_scanner(_self)
+		, m_complete(false)
+		{
+			if (_type == LITERAL_TYPE_COMMENT)
+				m_scanner->startNewCommentLiteral();
+			else
+				m_scanner->startNewLiteral();
+		}
+		~LiteralScope()
+		{
+			if (!m_complete)
+			{
+				if (m_type == LITERAL_TYPE_COMMENT)
+					m_scanner->dropCommentLiteral();
+				else
+					m_scanner->dropLiteral();
+			}
+		}
 		void complete() { m_complete = true; }
 
 	private:
+		enum LiteralType m_type;
 		Scanner* m_scanner;
 		bool m_complete;
 	};
@@ -133,8 +160,12 @@ public:
 
 	///@{
 	///@name Information about the current comment token
+
 	Location getCurrentCommentLocation() const { return m_skippedComment.location; }
 	std::string const& getCurrentCommentLiteral() const { return m_skippedComment.literal; }
+	/// Called by the parser during FunctionDefinition parsing to clear the current comment
+	void clearCurrentCommentLiteral() { m_skippedComment.literal.clear(); }
+
 	///@}
 
 	///@{
@@ -166,9 +197,11 @@ private:
 	///@{
 	///@name Literal buffer support
 	inline void startNewLiteral() { m_nextToken.literal.clear(); }
+	inline void startNewCommentLiteral() { m_nextSkippedComment.literal.clear(); }
 	inline void addLiteralChar(char c) { m_nextToken.literal.push_back(c); }
 	inline void addCommentLiteralChar(char c) { m_nextSkippedComment.literal.push_back(c); }
 	inline void dropLiteral() { m_nextToken.literal.clear(); }
+	inline void dropCommentLiteral() { m_nextSkippedComment.literal.clear(); }
 	inline void addLiteralCharAndAdvance() { addLiteralChar(m_char); advance(); }
 	///@}