solidity/liblangutil/Scanner.h

274 lines
9.0 KiB
C
Raw Normal View History

2014-10-20 14:37:04 +00:00
/*
2019-02-13 15:56:46 +00:00
* This file is part of solidity.
*
* solidity is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* solidity is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with solidity. If not, see <http://www.gnu.org/licenses/>.
*
* This file is derived from the file "scanner.h", which was part of the
* V8 project. The original copyright header follows:
*
* Copyright 2006-2012, the V8 project authors. All rights reserved.
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
* * Neither the name of Google Inc. nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
2014-10-20 14:37:04 +00:00
*/
/**
* @author Christian <c@ethdev.com>
* @date 2014
* Solidity scanner.
*/
#pragma once
#include <liblangutil/Token.h>
2018-11-14 14:18:55 +00:00
#include <liblangutil/CharStream.h>
#include <liblangutil/SourceLocation.h>
#include <libsolutil/Common.h>
#include <libsolutil/CommonData.h>
#include <optional>
#include <iosfwd>
2019-12-11 16:31:36 +00:00
namespace solidity::langutil
2014-10-16 12:08:54 +00:00
{
class AstRawString;
class AstValueFactory;
class ParserRecorder;
enum class ScannerKind
{
Solidity,
Yul
};
enum class ScannerError
{
NoError,
IllegalToken,
IllegalHexString,
IllegalHexDigit,
IllegalCommentTerminator,
IllegalEscapeSequence,
2020-07-15 14:06:43 +00:00
IllegalCharacterInString,
IllegalStringEndQuote,
IllegalNumberSeparator,
IllegalExponent,
IllegalNumberEnd,
DirectionalOverrideUnderflow,
DirectionalOverrideMismatch,
OctalNotAllowed,
};
std::string to_string(ScannerError _errorCode);
std::ostream& operator<<(std::ostream& os, ScannerError _errorCode);
2014-10-16 12:08:54 +00:00
class Scanner
{
2014-11-30 22:25:42 +00:00
friend class LiteralScope;
public:
explicit Scanner(std::shared_ptr<CharStream> _source) { reset(std::move(_source)); }
explicit Scanner(CharStream _source = CharStream()) { reset(std::move(_source)); }
2014-10-09 10:28:37 +00:00
std::string const& source() const noexcept { return m_source->source(); }
2016-11-14 10:46:43 +00:00
std::shared_ptr<CharStream> charStream() noexcept { return m_source; }
std::shared_ptr<CharStream const> charStream() const noexcept { return m_source; }
/// Resets the scanner as if newly constructed with _source as input.
void reset(CharStream _source);
void reset(std::shared_ptr<CharStream> _source);
/// Resets scanner to the start of input.
void reset();
2014-10-09 10:28:37 +00:00
/// Changes the scanner mode.
void setScannerMode(ScannerKind _kind)
{
m_kind = _kind;
// Invalidate lookahead buffer.
rescan();
}
/// @returns the next token and advances input
Token next();
2014-10-16 21:49:45 +00:00
/// Set scanner to a specific offset. This is used in error recovery.
void setPosition(size_t _offset);
///@{
///@name Information about the current token
2014-10-16 21:49:45 +00:00
/// @returns the current token
Token currentToken() const
{
2020-01-22 19:10:56 +00:00
return m_tokens[Current].token;
}
ElementaryTypeNameToken currentElementaryTypeNameToken() const
{
unsigned firstSize;
unsigned secondSize;
2020-01-22 19:10:56 +00:00
std::tie(firstSize, secondSize) = m_tokens[Current].extendedTokenInfo;
return ElementaryTypeNameToken(m_tokens[Current].token, firstSize, secondSize);
}
2020-01-22 19:10:56 +00:00
SourceLocation currentLocation() const { return m_tokens[Current].location; }
std::string const& currentLiteral() const { return m_tokens[Current].literal; }
std::tuple<unsigned, unsigned> const& currentTokenInfo() const { return m_tokens[Current].extendedTokenInfo; }
/// Retrieves the last error that occurred during lexical analysis.
/// @note If no error occurred, the value is undefined.
2020-01-22 19:10:56 +00:00
ScannerError currentError() const noexcept { return m_tokens[Current].error; }
///@}
///@{
///@name Information about the current comment token
2020-01-22 19:10:56 +00:00
SourceLocation currentCommentLocation() const { return m_skippedComments[Current].location; }
std::string const& currentCommentLiteral() const { return m_skippedComments[Current].literal; }
/// Called by the parser during FunctionDefinition parsing to clear the current comment
2020-01-22 19:10:56 +00:00
void clearCurrentCommentLiteral() { m_skippedComments[Current].literal.clear(); }
///@}
///@{
///@name Information about the next token
2014-10-09 10:28:37 +00:00
/// @returns the next token without advancing input.
2020-01-22 19:10:56 +00:00
Token peekNextToken() const { return m_tokens[Next].token; }
SourceLocation peekLocation() const { return m_tokens[Next].location; }
std::string const& peekLiteral() const { return m_tokens[Next].literal; }
Token peekNextNextToken() const { return m_tokens[NextNext].token; }
///@}
2014-10-09 10:28:37 +00:00
private:
inline Token setError(ScannerError _error) noexcept
{
2020-01-22 19:10:56 +00:00
m_tokens[NextNext].error = _error;
return Token::Illegal;
}
2014-11-20 21:08:16 +00:00
/// Used for the current and look-ahead token and comments
2014-10-16 12:08:54 +00:00
struct TokenDesc
{
Token token;
SourceLocation location;
2014-10-09 10:28:37 +00:00
std::string literal;
ScannerError error = ScannerError::NoError;
std::tuple<unsigned, unsigned> extendedTokenInfo;
2014-10-09 10:28:37 +00:00
};
///@{
///@name Literal buffer support
2020-01-22 19:10:56 +00:00
inline void addLiteralChar(char c) { m_tokens[NextNext].literal.push_back(c); }
inline void addCommentLiteralChar(char c) { m_skippedComments[NextNext].literal.push_back(c); }
2014-10-16 12:08:54 +00:00
inline void addLiteralCharAndAdvance() { addLiteralChar(m_char); advance(); }
void addUnicodeAsUTF8(unsigned codepoint);
///@}
2014-10-09 10:28:37 +00:00
bool advance() { m_char = m_source->advanceAndGet(); return !m_source->isPastEndOfInput(); }
void rollback(size_t _amount) { m_char = m_source->rollback(_amount); }
/// Rolls back to the start of the current token and re-runs the scanner.
void rescan();
2014-10-09 10:28:37 +00:00
inline Token selectErrorToken(ScannerError _err) { advance(); return setError(_err); }
inline Token selectToken(Token _tok) { advance(); return _tok; }
2014-10-16 21:49:45 +00:00
/// If the next character is _next, advance and return _then, otherwise return _else.
inline Token selectToken(char _next, Token _then, Token _else);
2014-10-09 10:28:37 +00:00
2014-11-05 13:20:56 +00:00
bool scanHexByte(char& o_scannedByte);
std::optional<unsigned> scanUnicode();
2014-10-09 10:28:37 +00:00
/// Scans a single Solidity token.
void scanToken();
2014-10-09 10:28:37 +00:00
/// Skips all whitespace and @returns true if something was skipped.
2014-10-09 10:28:37 +00:00
bool skipWhitespace();
/// Skips all whitespace that are neither '\r' nor '\n'.
bool skipWhitespaceExceptUnicodeLinebreak();
Token skipSingleLineComment();
Token skipMultiLineComment();
2014-10-09 10:28:37 +00:00
/// Tests if current source position is CR, LF or CRLF.
bool atEndOfLine() const;
/// Tries to consume CR, LF or CRLF line terminators and returns success or failure.
bool tryScanEndOfLine();
2014-10-09 10:28:37 +00:00
void scanDecimalDigits();
Token scanNumber(char _charSeen = 0);
std::tuple<Token, unsigned, unsigned> scanIdentifierOrKeyword();
2014-10-09 10:28:37 +00:00
Token scanString(bool const _isUnicode);
Token scanHexString();
/// Scans a single line comment and returns its corrected end position.
size_t scanSingleLineDocComment();
Token scanMultiLineDocComment();
/// Scans a slash '/' and depending on the characters returns the appropriate token
Token scanSlash();
2014-10-09 10:28:37 +00:00
2014-10-16 21:49:45 +00:00
/// Scans an escape-sequence which is part of a string and adds the
/// decoded character to the current literal. Returns true if a pattern
/// is scanned.
2014-10-09 10:28:37 +00:00
bool scanEscape();
/// @returns true iff we are currently positioned at a unicode line break.
bool isUnicodeLinebreak();
2014-10-16 21:49:45 +00:00
/// Return the current source position.
size_t sourcePos() const { return m_source->position(); }
bool isSourcePastEndOfInput() const { return m_source->isPastEndOfInput(); }
2014-10-09 10:28:37 +00:00
2020-01-22 19:10:56 +00:00
enum TokenIndex { Current, Next, NextNext };
2020-01-22 19:10:56 +00:00
TokenDesc m_skippedComments[3] = {}; // desc for the current, next and nextnext skipped comment
TokenDesc m_tokens[3] = {}; // desc for the current, next and nextnext token
2014-10-09 10:28:37 +00:00
std::shared_ptr<CharStream> m_source;
2021-06-29 12:38:59 +00:00
std::shared_ptr<std::string const> m_sourceName;
2014-10-09 10:28:37 +00:00
ScannerKind m_kind = ScannerKind::Solidity;
2014-10-16 21:49:45 +00:00
/// one character look-ahead, equals 0 at end of input
2014-10-09 10:28:37 +00:00
char m_char;
};
2014-10-16 12:08:54 +00:00
}