LSP: Implement semantic tokens

This commit is contained in:
Marenz 2022-05-03 16:32:48 +02:00
parent f629f9eff0
commit ca3af4b2a2
10 changed files with 698 additions and 0 deletions

View File

@ -4,6 +4,7 @@ Language Features:
Compiler Features:
* LSP: Add rudimentary support for semantic highlighting.
Bugfixes:

View File

@ -163,6 +163,8 @@ set(sources
lsp/HandlerBase.h
lsp/LanguageServer.cpp
lsp/LanguageServer.h
lsp/SemanticTokensBuilder.cpp
lsp/SemanticTokensBuilder.h
lsp/Transport.cpp
lsp/Transport.h
lsp/Utils.cpp

View File

@ -26,6 +26,7 @@
// LSP feature implementations
#include <libsolidity/lsp/GotoDefinition.h>
#include <libsolidity/lsp/SemanticTokensBuilder.h>
#include <liblangutil/SourceReferenceExtractor.h>
#include <liblangutil/CharStream.h>
@ -64,6 +65,49 @@ int toDiagnosticSeverity(Error::Type _errorType)
return -1;
}
Json::Value semanticTokensLegend()
{
Json::Value legend = Json::objectValue;
// NOTE! The (alphabetical) order and items must match exactly the items of
// their respective enum class members.
Json::Value tokenTypes = Json::arrayValue;
tokenTypes.append("class");
tokenTypes.append("comment");
tokenTypes.append("enum");
tokenTypes.append("enumMember");
tokenTypes.append("event");
tokenTypes.append("function");
tokenTypes.append("interface");
tokenTypes.append("keyword");
tokenTypes.append("macro");
tokenTypes.append("method");
tokenTypes.append("modifier");
tokenTypes.append("number");
tokenTypes.append("operator");
tokenTypes.append("parameter");
tokenTypes.append("property");
tokenTypes.append("string");
tokenTypes.append("struct");
tokenTypes.append("type");
tokenTypes.append("typeParameter");
tokenTypes.append("variable");
legend["tokenTypes"] = tokenTypes;
Json::Value tokenModifiers = Json::arrayValue;
tokenModifiers.append("abstract");
tokenModifiers.append("declaration");
tokenModifiers.append("definition");
tokenModifiers.append("deprecated");
tokenModifiers.append("documentation");
tokenModifiers.append("modification");
tokenModifiers.append("readonly");
legend["tokenModifiers"] = tokenModifiers;
return legend;
}
}
LanguageServer::LanguageServer(Transport& _transport):
@ -81,6 +125,7 @@ LanguageServer::LanguageServer(Transport& _transport):
{"textDocument/didChange", bind(&LanguageServer::handleTextDocumentDidChange, this, _2)},
{"textDocument/didClose", bind(&LanguageServer::handleTextDocumentDidClose, this, _2)},
{"textDocument/implementation", GotoDefinition(*this) },
{"textDocument/semanticTokens/full", bind(&LanguageServer::semanticTokensFull, this, _1, _2)},
{"workspace/didChangeConfiguration", bind(&LanguageServer::handleWorkspaceDidChangeConfiguration, this, _2)},
},
m_fileRepository("/" /* basePath */),
@ -266,10 +311,30 @@ void LanguageServer::handleInitialize(MessageID _id, Json::Value const& _args)
replyArgs["capabilities"]["implementationProvider"] = true;
replyArgs["capabilities"]["textDocumentSync"]["change"] = 2; // 0=none, 1=full, 2=incremental
replyArgs["capabilities"]["textDocumentSync"]["openClose"] = true;
replyArgs["capabilities"]["semanticTokensProvider"]["legend"] = semanticTokensLegend();
replyArgs["capabilities"]["semanticTokensProvider"]["range"] = false;
replyArgs["capabilities"]["semanticTokensProvider"]["full"] = true; // XOR requests.full.delta = true
m_client.reply(_id, move(replyArgs));
}
void LanguageServer::semanticTokensFull(MessageID _id, Json::Value const& _args)
{
auto uri = _args["textDocument"]["uri"];
compile();
auto const sourceName = m_fileRepository.uriToSourceUnitName(uri.as<string>());
SourceUnit const& ast = m_compilerStack.ast(sourceName);
m_compilerStack.charStream(sourceName);
Json::Value data = SemanticTokensBuilder().build(ast, m_compilerStack.charStream(sourceName));
Json::Value reply = Json::objectValue;
reply["data"] = data;
m_client.reply(_id, std::move(reply));
}
void LanguageServer::handleWorkspaceDidChangeConfiguration(Json::Value const& _args)
{
requireServerInitialized();

View File

@ -73,6 +73,7 @@ private:
void handleTextDocumentDidChange(Json::Value const& _args);
void handleTextDocumentDidClose(Json::Value const& _args);
void handleGotoDefinition(MessageID _id, Json::Value const& _args);
void semanticTokensFull(MessageID _id, Json::Value const& _args);
/// Invoked when the server user-supplied configuration changes (initiated by the client).
void changeConfiguration(Json::Value const&);

View File

@ -0,0 +1,283 @@
/*
This file is part of solidity.
solidity is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
solidity is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with solidity. If not, see <http://www.gnu.org/licenses/>.
*/
// SPDX-License-Identifier: GPL-3.0
#include <libsolidity/lsp/SemanticTokensBuilder.h>
#include <libsolidity/lsp/Utils.h>
#include <liblangutil/CharStream.h>
#include <liblangutil/SourceLocation.h>
#include <fmt/format.h>
using namespace std;
using namespace solidity::langutil;
using namespace solidity::frontend;
namespace solidity::lsp
{
namespace
{
optional<SemanticTokenType> semanticTokenTypeForType(frontend::Type const* _type)
{
if (!_type)
return nullopt;
switch (_type->category())
{
case frontend::Type::Category::Address: return SemanticTokenType::Class;
case frontend::Type::Category::Bool: return SemanticTokenType::Number;
case frontend::Type::Category::Enum: return SemanticTokenType::Enum;
case frontend::Type::Category::Function: return SemanticTokenType::Function;
case frontend::Type::Category::Integer: return SemanticTokenType::Number;
case frontend::Type::Category::RationalNumber: return SemanticTokenType::Number;
case frontend::Type::Category::StringLiteral: return SemanticTokenType::String;
case frontend::Type::Category::Struct: return SemanticTokenType::Struct;
case frontend::Type::Category::Contract: return SemanticTokenType::Class;
default:
lspDebug(fmt::format("semanticTokenTypeForType: unknown category: {}", static_cast<unsigned>(_type->category())));
return SemanticTokenType::Type;
}
}
SemanticTokenType semanticTokenTypeForExpression(frontend::Type const* _type)
{
if (!_type)
return SemanticTokenType::Variable;
switch (_type->category())
{
case frontend::Type::Category::Enum:
return SemanticTokenType::Enum;
default:
return SemanticTokenType::Variable;
}
}
} // end namespace
Json::Value SemanticTokensBuilder::build(SourceUnit const& _sourceUnit, CharStream const& _charStream)
{
reset(&_charStream);
_sourceUnit.accept(*this);
return m_encodedTokens;
}
void SemanticTokensBuilder::reset(CharStream const* _charStream)
{
m_encodedTokens = Json::arrayValue;
m_charStream = _charStream;
m_lastLine = 0;
m_lastStartChar = 0;
}
void SemanticTokensBuilder::encode(
SourceLocation const& _sourceLocation,
SemanticTokenType _tokenType,
SemanticTokenModifiers _modifiers
)
{
/*
https://microsoft.github.io/language-server-protocol/specifications/specification-3-17/#textDocument_semanticTokens
// Step-1: Absolute positions
{ line: 2, startChar: 5, length: 3, tokenType: 0, tokenModifiers: 3 },
{ line: 2, startChar: 10, length: 4, tokenType: 1, tokenModifiers: 0 },
{ line: 5, startChar: 2, length: 7, tokenType: 2, tokenModifiers: 0 }
// Step-2: Relative positions as intermediate step
{ deltaLine: 2, deltaStartChar: 5, length: 3, tokenType: 0, tokenModifiers: 3 },
{ deltaLine: 0, deltaStartChar: 5, length: 4, tokenType: 1, tokenModifiers: 0 },
{ deltaLine: 3, deltaStartChar: 2, length: 7, tokenType: 2, tokenModifiers: 0 }
// Step-3: final array result
// 1st token, 2nd token, 3rd token
[ 2,5,3,0,3, 0,5,4,1,0, 3,2,7,2,0 ]
So traverse through the AST and assign each leaf a token 5-tuple.
*/
// solAssert(_sourceLocation.isValid());
if (!_sourceLocation.isValid())
return;
auto const [line, startChar] = m_charStream->translatePositionToLineColumn(_sourceLocation.start);
auto const length = _sourceLocation.end - _sourceLocation.start;
lspDebug(fmt::format("encode [{}:{}..{}] {}", line, startChar, length, _tokenType));
m_encodedTokens.append(line - m_lastLine);
if (line == m_lastLine)
m_encodedTokens.append(startChar - m_lastStartChar);
else
m_encodedTokens.append(startChar);
m_encodedTokens.append(length);
m_encodedTokens.append(static_cast<int>(_tokenType));
m_encodedTokens.append(static_cast<int>(_modifiers));
m_lastLine = line;
m_lastStartChar = startChar;
}
bool SemanticTokensBuilder::visit(frontend::ContractDefinition const& _node)
{
encode(_node.nameLocation(), SemanticTokenType::Class);
return true;
}
bool SemanticTokensBuilder::visit(frontend::ElementaryTypeName const& _node)
{
encode(_node.location(), SemanticTokenType::Type);
return true;
}
bool SemanticTokensBuilder::visit(frontend::ElementaryTypeNameExpression const& _node)
{
if (auto const tokenType = semanticTokenTypeForType(_node.annotation().type); tokenType.has_value())
encode(_node.location(), tokenType.value());
return true;
}
bool SemanticTokensBuilder::visit(frontend::EnumDefinition const& _node)
{
encode(_node.nameLocation(), SemanticTokenType::Enum);
return true;
}
bool SemanticTokensBuilder::visit(frontend::EnumValue const& _node)
{
encode(_node.nameLocation(), SemanticTokenType::EnumMember);
return true;
}
bool SemanticTokensBuilder::visit(frontend::ErrorDefinition const& _node)
{
encode(_node.nameLocation(), SemanticTokenType::Event);
return true;
}
bool SemanticTokensBuilder::visit(frontend::FunctionDefinition const& _node)
{
encode(_node.nameLocation(), SemanticTokenType::Function);
return true;
}
bool SemanticTokensBuilder::visit(frontend::ModifierDefinition const& _node)
{
encode(_node.nameLocation(), SemanticTokenType::Modifier);
return true;
}
void SemanticTokensBuilder::endVisit(frontend::Literal const& _literal)
{
encode(_literal.location(), SemanticTokenType::Number);
}
void SemanticTokensBuilder::endVisit(frontend::StructuredDocumentation const& _documentation)
{
encode(_documentation.location(), SemanticTokenType::Comment);
}
void SemanticTokensBuilder::endVisit(frontend::Identifier const& _identifier)
{
lspDebug(fmt::format("Identifier: {}, {}..{} cat={}", _identifier.name(), _identifier.location().start, _identifier.location().end, _identifier.annotation().type->category()));
SemanticTokenModifiers modifiers = SemanticTokenModifiers::None;
if (_identifier.annotation().isConstant.set() && *_identifier.annotation().isConstant)
{
lspDebug("OMG We've found a const!");
modifiers = modifiers | SemanticTokenModifiers::Readonly;
}
encode(_identifier.location(), semanticTokenTypeForExpression(_identifier.annotation().type), modifiers);
}
void SemanticTokensBuilder::endVisit(frontend::IdentifierPath const& _node)
{
lspDebug(fmt::format("IdentifierPath: identifier path [{}..{}]", _node.location().start, _node.location().end));
for (size_t i = 0; i < _node.path().size(); ++i)
lspDebug(fmt::format(" [{}]: {}", i, _node.path().at(i)));
if (dynamic_cast<EnumDefinition const*>(_node.annotation().referencedDeclaration))
encode(_node.location(), SemanticTokenType::EnumMember);
else
encode(_node.location(), SemanticTokenType::Variable);
}
bool SemanticTokensBuilder::visit(frontend::MemberAccess const& _node)
{
lspDebug(fmt::format("[{}..{}] MemberAccess({}): {}", _node.location().start, _node.location().end, _node.annotation().referencedDeclaration ? _node.annotation().referencedDeclaration->name() : "?", _node.memberName()));
auto const memberNameLength = static_cast<int>(_node.memberName().size());
auto const memberTokenType = semanticTokenTypeForExpression(_node.annotation().type);
auto lhsLocation = _node.location();
lhsLocation.end -= (memberNameLength + 1 /*exclude the dot*/);
auto rhsLocation = _node.location();
rhsLocation.start = rhsLocation.end - static_cast<int>(memberNameLength);
if (memberTokenType == SemanticTokenType::Enum)
{
// Special handling for enumeration symbols.
encode(lhsLocation, SemanticTokenType::Enum);
encode(rhsLocation, SemanticTokenType::EnumMember);
}
else if (memberTokenType == SemanticTokenType::Function)
{
// Special handling for function symbols.
encode(lhsLocation, SemanticTokenType::Variable);
encode(rhsLocation, memberTokenType);
}
else
{
encode(rhsLocation, memberTokenType);
}
return false; // we handle LHS and RHS explicitly above.
}
void SemanticTokensBuilder::endVisit(PragmaDirective const& _pragma)
{
encode(_pragma.location(), SemanticTokenType::Macro);
// NOTE: It would be nice if we could highlight based on the symbols,
// such as (version) numerics be different than identifiers.
}
bool SemanticTokensBuilder::visit(frontend::UserDefinedTypeName const& _node)
{
if (auto const token = semanticTokenTypeForType(_node.annotation().type); token.has_value())
encode(_node.location(), *token);
return false;
}
bool SemanticTokensBuilder::visit(frontend::VariableDeclaration const& _node)
{
lspDebug(fmt::format("VariableDeclaration: {}", _node.name()));
if (auto const token = semanticTokenTypeForType(_node.typeName().annotation().type); token.has_value())
encode(_node.typeName().location(), *token);
encode(_node.nameLocation(), SemanticTokenType::Variable);
if (_node.overrides())
_node.overrides()->accept(*this);
if (_node.value())
_node.value()->accept(*this);
return false;
}
} // end namespace

View File

@ -0,0 +1,123 @@
/*
This file is part of solidity.
solidity is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
solidity is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with solidity. If not, see <http://www.gnu.org/licenses/>.
*/
// SPDX-License-Identifier: GPL-3.0
#include <libsolidity/ast/AST.h>
#include <libsolidity/ast/ASTVisitor.h>
#include <json/json.h>
#include <fmt/format.h>
namespace solidity::langutil
{
class CharStream;
struct SourceLocation;
}
namespace solidity::lsp
{
// See: https://microsoft.github.io/language-server-protocol/specifications/specification-3-17/#semanticTokenTypes
enum class SemanticTokenType
{
Class,
Comment,
Enum,
EnumMember,
Event,
Function,
Interface,
Keyword,
Macro,
Method,
Modifier,
Number,
Operator,
Parameter,
Property,
String,
Struct,
Type,
TypeParameter,
Variable,
// Unused below:
// Namespace,
// Regexp,
};
enum class SemanticTokenModifiers
{
None = 0,
// Member integer values must be bit-values as
// they can be OR'd together.
Abstract = 0x0001,
Declaration = 0x0002,
Definition = 0x0004,
Deprecated = 0x0008,
Documentation = 0x0010,
Modification = 0x0020,
Readonly = 0x0040,
// Unused below:
// Static,
// Async,
// DefaultLibrary,
};
constexpr SemanticTokenModifiers operator|(SemanticTokenModifiers a, SemanticTokenModifiers b) noexcept
{
return static_cast<SemanticTokenModifiers>(static_cast<int>(a) | static_cast<int>(b));
}
class SemanticTokensBuilder: public frontend::ASTConstVisitor
{
public:
Json::Value build(frontend::SourceUnit const& _sourceUnit, langutil::CharStream const& _charStream);
void reset(langutil::CharStream const* _charStream);
void encode(
langutil::SourceLocation const& _sourceLocation,
SemanticTokenType _tokenType,
SemanticTokenModifiers _modifiers = SemanticTokenModifiers::None
);
bool visit(frontend::ContractDefinition const&) override;
bool visit(frontend::ElementaryTypeName const&) override;
bool visit(frontend::ElementaryTypeNameExpression const&) override;
bool visit(frontend::EnumDefinition const&) override;
bool visit(frontend::EnumValue const&) override;
bool visit(frontend::ErrorDefinition const&) override;
bool visit(frontend::FunctionDefinition const&) override;
bool visit(frontend::ModifierDefinition const&) override;
void endVisit(frontend::Literal const&) override;
void endVisit(frontend::StructuredDocumentation const&) override;
void endVisit(frontend::Identifier const&) override;
void endVisit(frontend::IdentifierPath const&) override;
bool visit(frontend::MemberAccess const&) override;
void endVisit(frontend::PragmaDirective const&) override;
bool visit(frontend::UserDefinedTypeName const&) override;
bool visit(frontend::VariableDeclaration const&) override;
private:
Json::Value m_encodedTokens;
langutil::CharStream const* m_charStream;
int m_lastLine;
int m_lastStartChar;
};
} // end namespace

View File

@ -0,0 +1,42 @@
// SPDX-License-Identifier: UNLICENSED
pragma solidity >=0.8.0;
enum Weather {
Sunny,
Cloudy,
Rainy
}
enum Color {
Red,
Green,
Blue
}
function getColorEnum() pure returns (Color result)
{
result = Color.Red;
}
// ----
// -> textDocument/semanticTokens/full {
// }
// <- {
// "data": [
// 1, 0, 24, 8, 0,
// 2, 5, 7, 2, 0,
// 1, 4, 5, 3, 0,
// 1, 4, 6, 3, 0,
// 1, 4, 5, 3, 0,
// 3, 5, 5, 2, 0,
// 1, 4, 3, 3, 0,
// 1, 4, 5, 3, 0,
// 1, 4, 4, 3, 0,
// 3, 9, 12, 5, 0,
// 0, 29, 5, 2, 0,
// 0, 6, 6, 19, 0,
// 2, 4, 6, 2, 0,
// 0, 9, 5, 2, 0,
// 0, 6, 3, 3, 0
// ]
// }

View File

@ -0,0 +1,52 @@
// SPDX-License-Identifier: UNLICENSED
pragma solidity >=0.8.0;
library Lib
{
function add(uint a, uint b) public pure returns (uint result)
{
result = a + b;
}
function warningWithUnused() public pure
{
uint unused;
// ^^^^^^^^^^^ @unusedVariable
}
}
contract Contract
{
function doNothing() pure public returns (bool)
{
return true;
}
}
// ----
// functions: @unusedVariable 2072
// -> textDocument/semanticTokens/full {
// }
// <- {
// "data": [
// 1, 0, 24, 8, 0,
// 2, 8, 3, 0, 0,
// 2, 13, 3, 5, 0,
// 0, 4, 4, 11, 0,
// 0, 5, 1, 19, 0,
// 0, 3, 4, 11, 0,
// 0, 5, 1, 19, 0,
// 0, 24, 4, 11, 0,
// 0, 5, 6, 19, 0,
// 2, 8, 6, 19, 0,
// 0, 9, 1, 19, 0,
// 0, 4, 1, 19, 0,
// 3, 13, 17, 5, 0,
// 2, 8, 4, 11, 0,
// 0, 5, 6, 19, 0,
// 5, 9, 8, 0, 0,
// 2, 13, 9, 5, 0,
// 0, 33, 4, 11, 0,
// 2, 15, 4, 11, 0
// ]
// }

View File

@ -0,0 +1,81 @@
// SPDX-License-Identifier: UNLICENSED
pragma solidity >=0.8.0;
contract C
{
bool public locked = false;
int public calls = 0;
int public totalSum = 0;
function add(uint a, uint b) lock() monitor(a, b) public returns (uint result)
{
result = a + b;
}
modifier lock()
{
require(!locked);
locked = true;
_;
locked = false;
}
modifier monitor(uint a, uint b)
{
calls++;
totalSum = totalSum + a + b;
// ^^^^^^^^^^^^^^^^ @totalSumWarning
// ^^^^^^^^^^^^ @totalSumWarningSub
_;
}
}
// ----
// modifiers: @totalSumWarningSub 2271 @totalSumWarning 2271
// -> textDocument/semanticTokens/full {
// }
// <- {
// "data": [
// 1, 0, 24, 8, 0,
// 2, 9, 1, 0, 0,
// 2, 4, 4, 11, 0,
// 0, 12, 6, 19, 0,
// 0, 9, 5, 11, 0,
// 1, 4, 3, 11, 0,
// 0, 11, 5, 19, 0,
// 0, 8, 1, 11, 0,
// 1, 4, 3, 11, 0,
// 0, 11, 8, 19, 0,
// 0, 11, 1, 11, 0,
// 2, 13, 3, 5, 0,
// 0, 4, 4, 11, 0,
// 0, 5, 1, 19, 0,
// 0, 3, 4, 11, 0,
// 0, 5, 1, 19, 0,
// 0, 40, 4, 11, 0,
// 0, 5, 6, 19, 0,
// 0, -42, 4, 19, 0,
// 0, 7, 7, 19, 0,
// 0, 8, 1, 19, 0,
// 0, 3, 1, 19, 0,
// 2, 8, 6, 19, 0,
// 0, 9, 1, 19, 0,
// 0, 4, 1, 19, 0,
// 3, 13, 4, 10, 0,
// 2, 8, 7, 19, 0,
// 0, 9, 6, 19, 0,
// 1, 8, 6, 19, 0,
// 0, 9, 4, 11, 0,
// 2, 8, 6, 19, 0,
// 0, 9, 5, 11, 0,
// 3, 13, 7, 10, 0,
// 0, 8, 4, 11, 0,
// 0, 5, 1, 19, 0,
// 0, 3, 4, 11, 0,
// 0, 5, 1, 19, 0,
// 2, 8, 5, 19, 0,
// 1, 8, 8, 19, 0,
// 0, 11, 8, 19, 0,
// 0, 11, 1, 19, 0,
// 0, 4, 1, 19, 0
// ]
// }

View File

@ -0,0 +1,48 @@
// SPDX-License-Identifier: UNLICENSED
pragma solidity >=0.8.0;
struct Tag
{
uint id;
string name;
}
struct RGBColor
{
uint8 red;
uint8 green;
uint8 blue;
Tag tag;
}
function memberAccess(RGBColor memory color) pure returns(uint)
{
return color.red + color.green + color.blue;
}
// ----
// -> textDocument/semanticTokens/full {
// }
// <- {
// "data": [
// 1, 0, 24, 8, 0,
// 4, 4, 4, 11, 0,
// 0, 5, 2, 19, 0,
// 1, 4, 6, 17, 0,
// 0, 7, 4, 19, 0,
// 5, 4, 5, 11, 0,
// 0, 6, 3, 19, 0,
// 1, 4, 5, 11, 0,
// 0, 6, 5, 19, 0,
// 1, 4, 5, 11, 0,
// 0, 6, 4, 19, 0,
// 1, 4, 3, 16, 0,
// 0, 4, 3, 19, 0,
// 3, 9, 12, 5, 0,
// 0, 13, 8, 16, 0,
// 0, 16, 5, 19, 0,
// 0, 20, 4, 11, 0,
// 2, 17, 3, 19, 0,
// 0, 12, 5, 19, 0,
// 0, 14, 4, 19, 0
// ]
// }