From 4837d4236110660871f0a5d8ac1a93ad9e7b6a96 Mon Sep 17 00:00:00 2001 From: Alexander Arlt Date: Fri, 28 Apr 2023 11:29:59 -0500 Subject: [PATCH] Add experimental support to import AST via Standard JSON. --- Changelog.md | 1 + docs/using-the-compiler.rst | 10 +- libsolidity/interface/StandardCompiler.cpp | 194 +++++++++++------- libsolidity/interface/StandardCompiler.h | 1 + scripts/ASTImportTest.sh | 23 ++- solc/CommandLineInterface.cpp | 7 +- .../combined_json_with_base_path/output | 6 +- .../no_contract_combined_json/output | 3 +- .../standard_import_ast/input.json | 94 +++++++++ .../standard_import_ast/output.json | 97 +++++++++ .../input.json | 50 +++++ .../output.json | 30 +++ test/libsolidity/StandardCompiler.cpp | 2 +- 13 files changed, 436 insertions(+), 82 deletions(-) create mode 100644 test/cmdlineTests/standard_import_ast/input.json create mode 100644 test/cmdlineTests/standard_import_ast/output.json create mode 100644 test/cmdlineTests/standard_import_ast_select_bytecode/input.json create mode 100644 test/cmdlineTests/standard_import_ast_select_bytecode/output.json diff --git a/Changelog.md b/Changelog.md index f00234d8b..aed64a0c0 100644 --- a/Changelog.md +++ b/Changelog.md @@ -14,6 +14,7 @@ Compiler Features: * Parser: Unary plus is no longer recognized as a unary operator in the AST and triggers an error at the parsing stage (rather than later during the analysis). * SMTChecker: Properties that are proved safe are now reported explicitly at the end of the analysis. By default, only the number of safe properties is shown. The CLI option ``--model-checker-show-proved-safe`` and the JSON option ``settings.modelChecker.showProvedSafe`` can be enabled to show the full list of safe properties. * SMTChecker: Group all messages about unsupported language features in a single warning. The CLI option ``--model-checker-show-unsupported`` and the JSON option ``settings.modelChecker.showUnsupported`` can be enabled to show the full list. + * Standard JSON Interface: Add experimental support for importing ASTs via Standard JSON. * Yul EVM Code Transform: If available, use ``push0`` instead of ``codesize`` to produce an arbitrary value on stack in order to create equal stack heights between branches. diff --git a/docs/using-the-compiler.rst b/docs/using-the-compiler.rst index 3931728da..fa1a4ec32 100644 --- a/docs/using-the-compiler.rst +++ b/docs/using-the-compiler.rst @@ -203,7 +203,7 @@ Input Description .. code-block:: javascript { - // Required: Source code language. Currently supported are "Solidity" and "Yul". + // Required: Source code language. Currently supported are "Solidity", "Yul" and "SolidityAST" (experimental). "language": "Solidity", // Required "sources": @@ -230,6 +230,14 @@ Input Description // If files are used, their directories should be added to the command line via // `--allow-paths `. ] + // If language is set to "SolidityAST", an AST needs to be supplied under the "ast" key. + // Note that importing ASTs is experimental and in particular that: + // - importing invalid ASTs can produce undefined results and + // - no proper error reporting is available on invalid ASTs. + // Furthermore, note that the AST import only consumes the fields of the AST as + // produced by the compiler in "stopAfter": "parsing" mode and then re-performs + // analysis, so any analysis-based annotations of the AST are ignored upon import. + "ast": { ... } // formatted as the json ast requested with the ``ast`` output selection. }, "destructible": { diff --git a/libsolidity/interface/StandardCompiler.cpp b/libsolidity/interface/StandardCompiler.cpp index 1511b2179..dbcbe798a 100644 --- a/libsolidity/interface/StandardCompiler.cpp +++ b/libsolidity/interface/StandardCompiler.cpp @@ -657,72 +657,84 @@ std::variant StandardCompiler: ret.errors = Json::arrayValue; - for (auto const& sourceName: sources.getMemberNames()) + if (ret.language == "Solidity" || ret.language == "Yul") { - string hash; - - if (auto result = checkSourceKeys(sources[sourceName], sourceName)) - return *result; - - if (sources[sourceName]["keccak256"].isString()) - hash = sources[sourceName]["keccak256"].asString(); - - if (sources[sourceName]["content"].isString()) + for (auto const& sourceName: sources.getMemberNames()) { - string content = sources[sourceName]["content"].asString(); - if (!hash.empty() && !hashMatchesContent(hash, content)) - ret.errors.append(formatError( - Error::Type::IOError, - "general", - "Mismatch between content and supplied hash for \"" + sourceName + "\"" - )); - else - ret.sources[sourceName] = content; - } - else if (sources[sourceName]["urls"].isArray()) - { - if (!m_readFile) - return formatFatalError(Error::Type::JSONError, "No import callback supplied, but URL is requested."); + string hash; - vector failures; - bool found = false; + if (auto result = checkSourceKeys(sources[sourceName], sourceName)) + return *result; - for (auto const& url: sources[sourceName]["urls"]) + if (sources[sourceName]["keccak256"].isString()) + hash = sources[sourceName]["keccak256"].asString(); + + if (sources[sourceName]["content"].isString()) { - if (!url.isString()) - return formatFatalError(Error::Type::JSONError, "URL must be a string."); - ReadCallback::Result result = m_readFile(ReadCallback::kindString(ReadCallback::Kind::ReadFile), url.asString()); - if (result.success) - { - if (!hash.empty() && !hashMatchesContent(hash, result.responseOrErrorMessage)) - ret.errors.append(formatError( - Error::Type::IOError, - "general", - "Mismatch between content and supplied hash for \"" + sourceName + "\" at \"" + url.asString() + "\"" - )); - else - { - ret.sources[sourceName] = result.responseOrErrorMessage; - found = true; - break; - } - } + string content = sources[sourceName]["content"].asString(); + if (!hash.empty() && !hashMatchesContent(hash, content)) + ret.errors.append(formatError( + Error::Type::IOError, + "general", + "Mismatch between content and supplied hash for \"" + sourceName + "\"" + )); else - failures.push_back("Cannot import url (\"" + url.asString() + "\"): " + result.responseOrErrorMessage); + ret.sources[sourceName] = content; } - - for (auto const& failure: failures) + else if (sources[sourceName]["urls"].isArray()) { - /// If the import succeeded, let mark all the others as warnings, otherwise all of them are errors. - ret.errors.append(formatError( - found ? Error::Type::Warning : Error::Type::IOError, - "general", - failure - )); + if (!m_readFile) + return formatFatalError( + Error::Type::JSONError, "No import callback supplied, but URL is requested." + ); + + vector failures; + bool found = false; + + for (auto const& url: sources[sourceName]["urls"]) + { + if (!url.isString()) + return formatFatalError(Error::Type::JSONError, "URL must be a string."); + ReadCallback::Result result = m_readFile(ReadCallback::kindString(ReadCallback::Kind::ReadFile), url.asString()); + if (result.success) + { + if (!hash.empty() && !hashMatchesContent(hash, result.responseOrErrorMessage)) + ret.errors.append(formatError( + Error::Type::IOError, + "general", + "Mismatch between content and supplied hash for \"" + sourceName + "\" at \"" + url.asString() + "\"" + )); + else + { + ret.sources[sourceName] = result.responseOrErrorMessage; + found = true; + break; + } + } + else + failures.push_back( + "Cannot import url (\"" + url.asString() + "\"): " + result.responseOrErrorMessage + ); + } + + for (auto const& failure: failures) + { + /// If the import succeeded, let mark all the others as warnings, otherwise all of them are errors. + ret.errors.append(formatError( + found ? Error::Type::Warning : Error::Type::IOError, + "general", + failure + )); + } } + else + return formatFatalError(Error::Type::JSONError, "Invalid input source specified."); } - else - return formatFatalError(Error::Type::JSONError, "Invalid input source specified."); + } + else if (ret.language == "SolidityAST") + { + for (auto const& sourceName: sources.getMemberNames()) + ret.sources[sourceName] = util::jsonCompactPrint(sources[sourceName]); } Json::Value const& auxInputs = _input["auxiliaryInput"]; @@ -1117,7 +1129,24 @@ std::variant StandardCompiler: ret.modelCheckerSettings.timeout = modelCheckerSettings["timeout"].asUInt(); } - return { std::move(ret) }; + return {std::move(ret)}; +} + +map StandardCompiler::parseAstFromInput(StringMap const& _sources) +{ + map sourceJsons; + for (auto const& [sourceName, sourceCode]: _sources) + { + Json::Value ast; + astAssert(util::jsonParseStrict(sourceCode, ast), "Input file could not be parsed to JSON"); + string astKey = ast.isMember("ast") ? "ast" : "AST"; + + astAssert(ast.isMember(astKey), "astkey is not member"); + astAssert(ast[astKey]["nodeType"].asString() == "SourceUnit", "Top-level node should be a 'SourceUnit'"); + astAssert(sourceJsons.count(sourceName) == 0, "All sources must have unique names"); + sourceJsons.emplace(sourceName, std::move(ast[astKey])); + } + return sourceJsons; } Json::Value StandardCompiler::compileSolidity(StandardCompiler::InputsAndSettings _inputsAndSettings) @@ -1125,7 +1154,8 @@ Json::Value StandardCompiler::compileSolidity(StandardCompiler::InputsAndSetting CompilerStack compilerStack(m_readFile); StringMap sourceList = std::move(_inputsAndSettings.sources); - compilerStack.setSources(sourceList); + if (_inputsAndSettings.language == "Solidity") + compilerStack.setSources(sourceList); for (auto const& smtLib2Response: _inputsAndSettings.smtLib2Responses) compilerStack.addSMTLib2Response(smtLib2Response.first, smtLib2Response.second); compilerStack.setViaIR(_inputsAndSettings.viaIR); @@ -1153,23 +1183,37 @@ Json::Value StandardCompiler::compileSolidity(StandardCompiler::InputsAndSetting try { - if (binariesRequested) - compilerStack.compile(); - else - compilerStack.parseAndAnalyze(_inputsAndSettings.stopAfter); - - for (auto const& error: compilerStack.errors()) + if (_inputsAndSettings.language == "SolidityAST") { - Error const& err = dynamic_cast(*error); + try + { + compilerStack.importASTs(parseAstFromInput(sourceList)); + if (!compilerStack.analyze()) + errors.append(formatError(Error::Type::FatalError, "general", "Analysis of the AST failed.")); + if (binariesRequested) + compilerStack.compile(); + } + catch (util::Exception const& _exc) + { + solThrow(util::Exception, "Failed to import AST: "s + _exc.what()); + } + } + else + { + if (binariesRequested) + compilerStack.compile(); + else + compilerStack.parseAndAnalyze(_inputsAndSettings.stopAfter); - errors.append(formatErrorWithException( - compilerStack, - *error, - err.type(), - "general", - "", - err.errorId() - )); + for (auto const& error: compilerStack.errors()) + errors.append(formatErrorWithException( + compilerStack, + *error, + error->type(), + "general", + "", + error->errorId() + )); } } /// This is only thrown in a very few locations. @@ -1558,8 +1602,10 @@ Json::Value StandardCompiler::compile(Json::Value const& _input) noexcept return compileSolidity(std::move(settings)); else if (settings.language == "Yul") return compileYul(std::move(settings)); + else if (settings.language == "SolidityAST") + return compileSolidity(std::move(settings)); else - return formatFatalError(Error::Type::JSONError, "Only \"Solidity\" or \"Yul\" is supported as a language."); + return formatFatalError(Error::Type::JSONError, "Only \"Solidity\", \"Yul\" or \"SolidityAST\" is supported as a language."); } catch (Json::LogicError const& _exception) { diff --git a/libsolidity/interface/StandardCompiler.h b/libsolidity/interface/StandardCompiler.h index f06dd0c5a..588813d76 100644 --- a/libsolidity/interface/StandardCompiler.h +++ b/libsolidity/interface/StandardCompiler.h @@ -95,6 +95,7 @@ private: /// it in condensed form or an error as a json object. std::variant parseInput(Json::Value const& _input); + std::map parseAstFromInput(StringMap const& _sources); Json::Value compileSolidity(InputsAndSettings _inputsAndSettings); Json::Value compileYul(InputsAndSettings _inputsAndSettings); diff --git a/scripts/ASTImportTest.sh b/scripts/ASTImportTest.sh index 4d6bc56d5..faf98c192 100755 --- a/scripts/ASTImportTest.sh +++ b/scripts/ASTImportTest.sh @@ -101,6 +101,7 @@ function test_ast_import_export_equivalence local export_command=("$SOLC" --combined-json ast --pretty-json --json-indent 4 "${input_files[@]}") local import_command=("$SOLC" --import-ast --combined-json ast --pretty-json --json-indent 4 expected.json) + local import_via_standard_json_command=("$SOLC" --combined-json ast --pretty-json --json-indent 4 --standard-json standard_json_input.json) # export ast - save ast json as expected result (silently) if ! "${export_command[@]}" > expected.json 2> stderr_export.txt @@ -118,8 +119,28 @@ function test_ast_import_export_equivalence return 1 fi + echo ". += {\"sources\":" > _ast_json.json + jq .sources expected.json >> _ast_json.json + echo "}" >> _ast_json.json + echo "{\"language\": \"SolidityAST\", \"settings\": {\"outputSelection\": {\"*\": {\"\": [\"ast\"]}}}}" > standard_json.json + jq --from-file _ast_json.json standard_json.json > standard_json_input.json + + # (re)import ast via standard json - and export it again as obtained result (silently) + if ! "${import_via_standard_json_command[@]}" > obtained_standard_json.json 2> stderr_import.txt + then + print_stderr_stdout "ERROR: AST reimport failed (import) for input file ${sol_file}." ./stderr_import.txt ./obtained_standard_json.json + print_used_commands "$(pwd)" "${export_command[*]} > expected.json" "${import_command[*]}" + return 1 + fi + + jq .sources expected.json > expected_standard_json.json + jq .sources obtained_standard_json.json > obtained_standard_json_.json + jq 'walk(if type == "object" and has("ast") then .AST = .ast | del(.ast) else . end)' < obtained_standard_json_.json > obtained_standard_json.json + jq --sort-keys . < obtained_standard_json.json > obtained_standard_json_.json + mv obtained_standard_json_.json obtained_standard_json.json + # compare expected and obtained ASTs - if ! diff_files expected.json obtained.json + if ! diff_files expected.json obtained.json || ! diff_files expected_standard_json.json obtained_standard_json.json then printError "ERROR: AST reimport failed for ${sol_file}" if (( EXIT_ON_ERROR == 1 )) diff --git a/solc/CommandLineInterface.cpp b/solc/CommandLineInterface.cpp index 5037552c3..7da0d3bf1 100644 --- a/solc/CommandLineInterface.cpp +++ b/solc/CommandLineInterface.cpp @@ -879,9 +879,12 @@ void CommandLineInterface::handleCombinedJSON() output[g_strSources] = Json::Value(Json::objectValue); for (auto const& sourceCode: m_fileReader.sourceUnits()) { - ASTJsonExporter converter(m_compiler->state(), m_compiler->sourceIndices()); output[g_strSources][sourceCode.first] = Json::Value(Json::objectValue); - output[g_strSources][sourceCode.first]["AST"] = converter.toJson(m_compiler->ast(sourceCode.first)); + output[g_strSources][sourceCode.first]["AST"] = ASTJsonExporter( + m_compiler->state(), + m_compiler->sourceIndices() + ).toJson(m_compiler->ast(sourceCode.first)); + output[g_strSources][sourceCode.first]["id"] = m_compiler->sourceIndices().at(sourceCode.first); } } diff --git a/test/cmdlineTests/combined_json_with_base_path/output b/test/cmdlineTests/combined_json_with_base_path/output index 127773320..ec9bbcbcb 100644 --- a/test/cmdlineTests/combined_json_with_base_path/output +++ b/test/cmdlineTests/combined_json_with_base_path/output @@ -61,7 +61,8 @@ } ], "src": "36:38:0" - } + }, + "id": 0 }, "combined_json_with_base_path/input.sol": { @@ -105,7 +106,8 @@ } ], "src": "36:42:1" - } + }, + "id": 1 } }, "version": "" diff --git a/test/cmdlineTests/no_contract_combined_json/output b/test/cmdlineTests/no_contract_combined_json/output index af7300896..e0caa6fa1 100644 --- a/test/cmdlineTests/no_contract_combined_json/output +++ b/test/cmdlineTests/no_contract_combined_json/output @@ -29,7 +29,8 @@ } ], "src": "36:22:0" - } + }, + "id": 0 } }, "version": "" diff --git a/test/cmdlineTests/standard_import_ast/input.json b/test/cmdlineTests/standard_import_ast/input.json new file mode 100644 index 000000000..5ae7a5297 --- /dev/null +++ b/test/cmdlineTests/standard_import_ast/input.json @@ -0,0 +1,94 @@ +{ + "language": "SolidityAST", + "sources": { + "A": { + "ast": { + "absolutePath": "A", + "exportedSymbols": { + "C": [ + 6 + ] + }, + "id": 7, + "license": "GPL-3.0", + "nodeType": "SourceUnit", + "nodes": [ + { + "id": 1, + "literals": [ + "solidity", + ">=", + "0.0" + ], + "nodeType": "PragmaDirective", + "src": "36:22:0" + }, + { + "abstract": false, + "baseContracts": [], + "canonicalName": "C", + "contractDependencies": [], + "contractKind": "contract", + "fullyImplemented": true, + "id": 6, + "linearizedBaseContracts": [ + 6 + ], + "name": "C", + "nameLocation": "68:1:0", + "nodeType": "ContractDefinition", + "nodes": [ + { + "body": { + "id": 4, + "nodeType": "Block", + "src": "97:2:0", + "statements": [] + }, + "functionSelector": "26121ff0", + "id": 5, + "implemented": true, + "kind": "function", + "modifiers": [], + "name": "f", + "nameLocation": "81:1:0", + "nodeType": "FunctionDefinition", + "parameters": { + "id": 2, + "nodeType": "ParameterList", + "parameters": [], + "src": "82:2:0" + }, + "returnParameters": { + "id": 3, + "nodeType": "ParameterList", + "parameters": [], + "src": "97:0:0" + }, + "scope": 6, + "src": "72:27:0", + "stateMutability": "pure", + "virtual": false, + "visibility": "public" + } + ], + "scope": 7, + "src": "59:42:0", + "usedErrors": [] + } + ], + "src": "36:65:0" + }, + "id": 0 + } + }, + "settings": { + "outputSelection": { + "*": { + "": [ + "ast" + ] + } + } + } +} diff --git a/test/cmdlineTests/standard_import_ast/output.json b/test/cmdlineTests/standard_import_ast/output.json new file mode 100644 index 000000000..6ecac5ca3 --- /dev/null +++ b/test/cmdlineTests/standard_import_ast/output.json @@ -0,0 +1,97 @@ +{ + "sources": + { + "A": + { + "ast": + { + "absolutePath": "A", + "exportedSymbols": + { + "C": + [ + 6 + ] + }, + "id": 7, + "license": "GPL-3.0", + "nodeType": "SourceUnit", + "nodes": + [ + { + "id": 1, + "literals": + [ + "solidity", + ">=", + "0.0" + ], + "nodeType": "PragmaDirective", + "src": "36:22:0" + }, + { + "abstract": false, + "baseContracts": [], + "canonicalName": "C", + "contractDependencies": [], + "contractKind": "contract", + "fullyImplemented": true, + "id": 6, + "linearizedBaseContracts": + [ + 6 + ], + "name": "C", + "nameLocation": "68:1:0", + "nodeType": "ContractDefinition", + "nodes": + [ + { + "body": + { + "id": 4, + "nodeType": "Block", + "src": "97:2:0", + "statements": [] + }, + "functionSelector": "26121ff0", + "id": 5, + "implemented": true, + "kind": "function", + "modifiers": [], + "name": "f", + "nameLocation": "81:1:0", + "nodeType": "FunctionDefinition", + "parameters": + { + "id": 2, + "nodeType": "ParameterList", + "parameters": [], + "src": "82:2:0" + }, + "returnParameters": + { + "id": 3, + "nodeType": "ParameterList", + "parameters": [], + "src": "97:0:0" + }, + "scope": 6, + "src": "72:27:0", + "stateMutability": "pure", + "virtual": false, + "visibility": "public" + } + ], + "scope": 7, + "src": "59:42:0", + "usedErrors": [], + "usedEvents": [] + } + ], + "src": "36:65:0" + }, + "id": 0 + } + } +} diff --git a/test/cmdlineTests/standard_import_ast_select_bytecode/input.json b/test/cmdlineTests/standard_import_ast_select_bytecode/input.json new file mode 100644 index 000000000..6458cb7e1 --- /dev/null +++ b/test/cmdlineTests/standard_import_ast_select_bytecode/input.json @@ -0,0 +1,50 @@ +{ + "language": "SolidityAST", + "sources": { + "A": { + "ast": { + "absolutePath": "empty_contract.sol", + "exportedSymbols": { + "test": [ + 1 + ] + }, + "id": 2, + "nodeType": "SourceUnit", + "nodes": [ + { + "abstract": false, + "baseContracts": [], + "canonicalName": "test", + "contractDependencies": [], + "contractKind": "contract", + "fullyImplemented": true, + "id": 1, + "linearizedBaseContracts": [ + 1 + ], + "name": "test", + "nameLocation": "9:4:0", + "nodeType": "ContractDefinition", + "nodes": [], + "scope": 2, + "src": "0:17:0", + "usedErrors": [] + } + ], + "src": "0:124:0" + }, + "id": 0 + } + }, + "settings": { + "outputSelection": { + "*": { + "*": [ + "evm.bytecode", + "evm.bytecode.sourceMap" + ] + } + } + } +} diff --git a/test/cmdlineTests/standard_import_ast_select_bytecode/output.json b/test/cmdlineTests/standard_import_ast_select_bytecode/output.json new file mode 100644 index 000000000..0fca3f271 --- /dev/null +++ b/test/cmdlineTests/standard_import_ast_select_bytecode/output.json @@ -0,0 +1,30 @@ +{ + "contracts": + { + "A": + { + "test": + { + "evm": + { + "bytecode": + { + "functionDebugData": {}, + "generatedSources": [], + "linkReferences": {}, + "object": "", + "opcodes":"", + "sourceMap":"" + } + } + } + } + }, + "sources": + { + "A": + { + "id": 0 + } + } +} diff --git a/test/libsolidity/StandardCompiler.cpp b/test/libsolidity/StandardCompiler.cpp index 8bc0be7a2..785f14284 100644 --- a/test/libsolidity/StandardCompiler.cpp +++ b/test/libsolidity/StandardCompiler.cpp @@ -187,7 +187,7 @@ BOOST_AUTO_TEST_CASE(invalid_language) } )"; Json::Value result = compile(input); - BOOST_CHECK(containsError(result, "JSONError", "Only \"Solidity\" or \"Yul\" is supported as a language.")); + BOOST_CHECK(containsError(result, "JSONError", "Only \"Solidity\", \"Yul\" or \"SolidityAST\" is supported as a language.")); } BOOST_AUTO_TEST_CASE(valid_language)