cerc-io/solidity
11
0
Fork 0
mirror of https://github.com/ethereum/solidity synced 2023-10-03 13:03:40 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

added two functions in Token to handle long identifiers, redid fromIdentifierOrKeyword, and made complementary changes in scanner and parser

Browse Source
This commit is contained in:
RJ Catalano
2016-02-18 11:22:58 -06:00
parent f4da126018
commit 84f2eb461b
7 changed files with 253 additions and 157 deletions
Download Patch File Download Diff File Expand all files Collapse all files
libsolidity/ast/Types.cpp
+4 -3
View File
@@ -156,10 +156,11 @@ TypePointer Type::fromElementaryTypeName(ElementaryTypeNameToken const& _type)
TypePointer Type::fromElementaryTypeName(string const& _name)
{
string details;
unsigned short firstNum;
unsigned short secondNum;
Token::Value token;
tie(token, details) = Token::fromIdentifierOrKeyword(_name);
return fromElementaryTypeName(ElementaryTypeNameToken(token, details));
tie(token, firstNum, secondNum) = Token::fromIdentifierOrKeyword(_name);
return fromElementaryTypeName(ElementaryTypeNameToken(token, firstNum, secondNum));
}
TypePointer Type::forLiteral(Literal const& _literal)
libsolidity/parsing/Parser.cpp
+19 -7
View File
@@ -590,7 +590,10 @@ ASTPointer<TypeName> Parser::parseTypeName(bool _allowVar)
Token::Value token = m_scanner->currentToken();
if (Token::isElementaryTypeName(token))
{
ElementaryTypeNameToken elemTypeName(token, m_scanner->currentTokenInfo());
unsigned firstSize;
unsigned secondSize;
tie(firstSize, secondSize) = m_scanner->currentTokenInfo();
ElementaryTypeNameToken elemTypeName(token, firstSize, secondSize);
type = ASTNodeFactory(*this).createNode<ElementaryTypeName>(elemTypeName);
m_scanner->next();
}
@@ -628,11 +631,14 @@ ASTPointer<Mapping> Parser::parseMapping()
expectToken(Token::Mapping);
expectToken(Token::LParen);
ASTPointer<ElementaryTypeName> keyType;
if (Token::isElementaryTypeName(m_scanner->currentToken()))
keyType = ASTNodeFactory(*this).createNode<ElementaryTypeName>
(ElementaryTypeNameToken(m_scanner->currentToken(), m_scanner->currentTokenInfo()));
else
Token::Value token = m_scanner->currentToken();
if (!Token::isElementaryTypeName(token))
fatalParserError(string("Expected elementary type name for mapping key type"));
unsigned firstSize;
unsigned secondSize;
tie(firstSize, secondSize) = m_scanner->currentTokenInfo();
ElementaryTypeNameToken elemTypeName(token, firstSize, secondSize);
keyType = ASTNodeFactory(*this).createNode<ElementaryTypeName>(elemTypeName);
m_scanner->next();
expectToken(Token::Arrow);
bool const allowVar = false;
@@ -829,7 +835,10 @@ ASTPointer<Statement> Parser::parseSimpleStatement(ASTPointer<ASTString> const&
else
{
startedWithElementary = true;
ElementaryTypeNameToken elemToken(m_scanner->currentToken(), m_scanner->currentTokenInfo());
unsigned firstNum;
unsigned secondNum;
tie(firstNum, secondNum) = m_scanner->currentTokenInfo();
ElementaryTypeNameToken elemToken(m_scanner->currentToken(), firstNum, secondNum);
path.push_back(ASTNodeFactory(*this).createNode<ElementaryTypeNameExpression>(elemToken));
m_scanner->next();
}
@@ -1138,7 +1147,10 @@ ASTPointer<Expression> Parser::parsePrimaryExpression()
if (Token::isElementaryTypeName(token))
{
//used for casts
ElementaryTypeNameToken elementaryExpression(m_scanner->currentToken(), m_scanner->currentTokenInfo());
unsigned firstSize;
unsigned secondSize;
tie(firstSize, secondSize) = m_scanner->currentTokenInfo();
ElementaryTypeNameToken elementaryExpression(m_scanner->currentToken(), firstSize, secondSize);
expression = nodeFactory.createNode<ElementaryTypeNameExpression>(elementaryExpression);
m_scanner->next();
}
libsolidity/parsing/Scanner.cpp
+8 -6
View File
@@ -381,12 +381,14 @@ Token::Value Scanner::scanSlash()
void Scanner::scanToken()
{
m_nextToken.literal.clear();
m_nextToken.extendedTokenInfo.clear();
m_nextToken.extendedTokenInfo = make_tuple(0, 0);
m_nextSkippedComment.literal.clear();
m_nextSkippedComment.extendedTokenInfo.clear();
m_nextSkippedComment.extendedTokenInfo = make_tuple(0, 0);
Token::Value token;
string tokenExtension;
// M and N are for the purposes of grabbing different type sizes
unsigned M;
unsigned N;
do
{
// Remember the position of the next token
@@ -554,7 +556,7 @@ void Scanner::scanToken()
break;
default:
if (isIdentifierStart(m_char))
tie(token, tokenExtension) = scanIdentifierOrKeyword();
tie(token, M, N) = scanIdentifierOrKeyword();
else if (isDecimalDigit(m_char))
token = scanNumber();
else if (skipWhitespace())
@@ -571,7 +573,7 @@ void Scanner::scanToken()
while (token == Token::Whitespace);
m_nextToken.location.end = sourcePos();
m_nextToken.token = token;
m_nextToken.extendedTokenInfo = tokenExtension;
m_nextToken.extendedTokenInfo = make_tuple(M,N);
}
bool Scanner::scanEscape()
@@ -709,7 +711,7 @@ Token::Value Scanner::scanNumber(char _charSeen)
return Token::Number;
}
tuple<Token::Value, string> Scanner::scanIdentifierOrKeyword()
tuple<Token::Value, unsigned, unsigned> Scanner::scanIdentifierOrKeyword()
{
solAssert(isIdentifierStart(m_char), "");
LiteralScope literal(this, LITERAL_TYPE_STRING);
libsolidity/parsing/Scanner.h
+3 -3
View File
@@ -122,7 +122,7 @@ public:
SourceLocation currentLocation() const { return m_currentToken.location; }
std::string const& currentLiteral() const { return m_currentToken.literal; }
std::string const& currentTokenInfo() const { return m_currentToken.extendedTokenInfo; }
std::tuple<unsigned, unsigned> const& currentTokenInfo() const { return m_currentToken.extendedTokenInfo; }
///@}
///@{
@@ -161,7 +161,7 @@ private:
Token::Value token;
SourceLocation location;
std::string literal;
std::string extendedTokenInfo;
std::tuple<unsigned, unsigned> extendedTokenInfo;
};
///@{
@@ -192,7 +192,7 @@ private:
void scanDecimalDigits();
Token::Value scanNumber(char _charSeen = 0);
std::tuple<Token::Value, std::string> scanIdentifierOrKeyword();
std::tuple<Token::Value, unsigned, unsigned> scanIdentifierOrKeyword();
Token::Value scanString();
Token::Value scanSingleLineDocComment();
libsolidity/parsing/Token.cpp
+93 -29
View File
@@ -50,36 +50,65 @@ namespace dev
namespace solidity
{
tuple<string, unsigned int, unsigned int> ElementaryTypeNameToken::parseDetails(Token::Value _baseType, string const& _details)
void ElementaryTypeNameToken::parseDetails(Token::Value _baseType, unsigned const& _first, unsigned const& _second)
{
solAssert(Token::isElementaryTypeName(_baseType), "");
string baseType = Token::toString(_baseType);
if (_details.length() == 0)
return make_tuple(baseType, 0, 0);
if (_first == 0 && _second == 0)
{
m_name = baseType;
m_firstNumber = _first;
m_secondNumber = _second;
}
if (baseType == "bytesM")
{
solAssert(_second == 0, "There should not be a second size argument to type bytesM.");
for (unsigned m = 1; m <= 32; m++)
if (to_string(m) == _details)
return make_tuple(baseType.substr(0, baseType.size()-1) + to_string(m), m, 0);
if (m == _first)
{
m_name = baseType.substr(0, baseType.size()-1) + to_string(_first);
m_firstNumber = _first;
m_secondNumber = _second;
}
}
else if (baseType == "uintM" || baseType == "intM")
{
solAssert(_second == 0, "There should not be a second size argument to type " + baseType + ".");
for (unsigned m = 8; m <= 256; m+=8)
if (to_string(m) == _details)
return make_tuple(baseType.substr(0, baseType.size()-1) + to_string(m), m, 0);
if (m == _first)
{
m_name = baseType.substr(0, baseType.size()-1) + to_string(_first);
m_firstNumber = _first;
m_secondNumber = _second;
}
}
else if (baseType == "ufixedMxN" || baseType == "fixedMxN")
{
for (unsigned m = 0; m <= 256; m+=8)
for (unsigned n = 8; m + n <= 256; n+=8)
if ((to_string(m) + "x" + to_string(n)) == _details)
return make_tuple(baseType.substr(0, baseType.size()-3) + to_string(m) + "x" + to_string(n), m, n);
if (m == _first && n == _second)
{
m_name = baseType.substr(0, baseType.size()-3) +
to_string(_first) +
"x" +
to_string(_second);
m_firstNumber = _first;
m_secondNumber = _second;
}
}
BOOST_THROW_EXCEPTION(Error(Error::Type::TypeError) <<
errinfo_comment("Cannot create elementary type name token out of type " + baseType + " and size " + _details)
);
if (m_name.empty())
BOOST_THROW_EXCEPTION(Error(Error::Type::TypeError) <<
errinfo_comment(
"Cannot create elementary type name token out of type " +
baseType +
" and size(s) " +
to_string(_first) +
" and " +
to_string(_second)
)
);
}
#define T(name, string, precedence) #name,
@@ -112,33 +141,68 @@ char const Token::m_tokenType[] =
{
TOKEN_LIST(KT, KK)
};
tuple<Token::Value, string> Token::fromIdentifierOrKeyword(const string& _literal)
unsigned Token::extractM(string _literal)
{
try
{
unsigned short m = stoi(_literal.substr(_literal.find_first_of("0123456789")));
return m;
}
catch(out_of_range& e)
{
return 0;
}
}
pair<unsigned, unsigned> Token::extractMxN(string _literal)
{
try
{
unsigned short m = stoi(_literal.substr(0, _literal.find_last_of("x") - 1));
unsigned short n = stoi(_literal.substr(_literal.find_last_of("x") + 1));
return make_pair(m, n);
}
catch (out_of_range& e)
{
return make_pair(0, 0);
}
}
tuple<Token::Value, unsigned short, unsigned short> Token::fromIdentifierOrKeyword(string const& _literal)
{
string token = _literal;
string details;
if (_literal == "uintM" || _literal == "intM" || _literal == "fixedMxN" || _literal == "ufixedMxN" || _literal == "bytesM")
return make_pair(Token::Identifier, details);
if (_literal.find_first_of("0123456789") != string::npos)
{
string baseType = _literal.substr(0, _literal.find_first_of("0123456789"));
short m = stoi(_literal.substr(_literal.find_first_of("0123456789")));
if (baseType == "bytes")
{
details = (0 < m && m <= 32) ? to_string(m) : "";
token = details.empty() ? _literal : baseType + "M";
unsigned short m = extractM(_literal);
if (0 < m && m <= 32)
return make_tuple(Token::BytesM, m, 0);
return make_tuple(Token::Identifier, 0, 0);
}
else if (baseType == "uint" || baseType == "int")
{
details = (0 < m && m <= 256 && m % 8 == 0) ? to_string(m) : "";
token = details.empty() ? _literal : baseType + "M";
unsigned short m = extractM(_literal);
if (0 < m && m <= 256 && m % 8 == 0)
{
if (baseType == "uint")
return make_tuple(Token::UIntM, m, 0);
else
return make_tuple(Token::IntM, m, 0);
}
return make_tuple(Token::Identifier, 0, 0);
}
else if (baseType == "ufixed" || baseType == "fixed")
{
m = stoi(to_string(m).substr(0, to_string(m).find_first_of("x") - 1));
short n = stoi(_literal.substr(_literal.find_last_of("x") + 1));
details = (0 < n + m && n + m <= 256 && ((n % 8 == 0) && (m % 8 == 0))) ?
to_string(m) + "x" + to_string(n) : "";
token = details.empty() ? _literal : baseType + "MxN" ;
unsigned short m;
unsigned short n;
tie(m, n) = extractMxN(_literal);
if (0 < n + m && n + m <= 256 && ((n % 8 == 0) && (m % 8 == 0)))
{
if (baseType == "ufixed")
return make_tuple(Token::UFixedMxN, m, n);
else
return make_tuple(Token::FixedMxN, m, n);
}
return make_tuple(Token::Identifier, 0, 0);
}
}
// The following macros are used inside TOKEN_LIST and cause non-keyword tokens to be ignored
@@ -148,8 +212,8 @@ tuple<Token::Value, string> Token::fromIdentifierOrKeyword(const string& _litera
static const map<string, Token::Value> keywords({TOKEN_LIST(TOKEN, KEYWORD)});
#undef KEYWORD
#undef TOKEN
auto it = keywords.find(token);
return it == keywords.end() ? make_pair(Token::Identifier, details) : make_pair(it->second, details);
auto it = keywords.find(_literal);
return it == keywords.end() ? make_tuple(Token::Identifier, 0, 0) : make_tuple(it->second, 0, 0);
}
#undef KT
libsolidity/parsing/Token.h
+15 -10
View File
@@ -189,19 +189,19 @@ namespace solidity
K(After, "after", 0) \
/* type keywords*/ \
K(Int, "int", 0) \
K(IntM, "intM", 0) \
T(IntM, "intM", 0) \
K(UInt, "uint", 0) \
K(UIntM, "uintM", 0) \
T(UIntM, "uintM", 0) \
K(Bytes, "bytes", 0) \
K(BytesM, "bytesM", 0) \
T(BytesM, "bytesM", 0) \
K(Byte, "byte", 0) \
K(String, "string", 0) \
K(Address, "address", 0) \
K(Bool, "bool", 0) \
K(Fixed, "fixed", 0) \
K(FixedMxN, "fixedMxN", 0) \
T(FixedMxN, "fixedMxN", 0) \
K(UFixed, "ufixed", 0) \
K(UFixedMxN, "ufixedMxN", 0) \
T(UFixedMxN, "ufixedMxN", 0) \
T(TypesEnd, NULL, 0) /* used as type enum end marker */ \
\
/* Literals */ \
@@ -302,9 +302,14 @@ public:
return m_precedence[tok];
}
static std::tuple<Token::Value, std::string> fromIdentifierOrKeyword(std::string const& _name);
static std::tuple<Token::Value, unsigned short, unsigned short> fromIdentifierOrKeyword(std::string const& _literal);
private:
// extractM and extractMxN provide a safe way to extract numbers,
// if out_of_range error is thrown, they returns 0s, therefore securing
// the variable's identity as an identifier.
static unsigned extractM(std::string _literal);
static std::pair<unsigned, unsigned> extractMxN(std::string _literal);
static char const* const m_name[NUM_TOKENS];
static char const* const m_string[NUM_TOKENS];
static int8_t const m_precedence[NUM_TOKENS];
@@ -314,9 +319,9 @@ private:
class ElementaryTypeNameToken
{
public:
ElementaryTypeNameToken(Token::Value _token, std::string const& _detail)
ElementaryTypeNameToken(Token::Value _token, unsigned const& _firstNumber, unsigned const& _secondNumber)
{
std::tie(m_name, m_firstNumber, m_secondNumber) = parseDetails(_token, _detail);
parseDetails(_token, _firstNumber, _secondNumber);
m_token = _token;
}
@@ -331,8 +336,8 @@ private:
std::string m_name;
unsigned int m_firstNumber;
unsigned int m_secondNumber;
/// throws if _details is malformed
std::tuple<std::string, unsigned int, unsigned int> parseDetails(Token::Value _baseType, std::string const& _details);
/// throws if type is not properly sized
void parseDetails(Token::Value _baseType, unsigned const& _first, unsigned const& _second);
};
}