From 257672e56f5ba9d88e443100f95da879ad7b9dbb Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Tue, 6 Apr 2021 16:12:06 +0200 Subject: [PATCH] Extract file reading logic from CommandLineInterface into FileReader class. --- libsolidity/CMakeLists.txt | 2 + libsolidity/interface/FileReader.cpp | 98 +++++++++++++++++++++++++ libsolidity/interface/FileReader.h | 90 +++++++++++++++++++++++ solc/CommandLineInterface.cpp | 106 ++++++++------------------- solc/CommandLineInterface.h | 9 +-- 5 files changed, 224 insertions(+), 81 deletions(-) create mode 100644 libsolidity/interface/FileReader.cpp create mode 100644 libsolidity/interface/FileReader.h diff --git a/libsolidity/CMakeLists.txt b/libsolidity/CMakeLists.txt index 0707c9d43..6b8426a1e 100644 --- a/libsolidity/CMakeLists.txt +++ b/libsolidity/CMakeLists.txt @@ -133,6 +133,8 @@ set(sources interface/CompilerStack.cpp interface/CompilerStack.h interface/DebugSettings.h + interface/FileReader.cpp + interface/FileReader.h interface/ImportRemapper.cpp interface/ImportRemapper.h interface/GasEstimator.cpp diff --git a/libsolidity/interface/FileReader.cpp b/libsolidity/interface/FileReader.cpp new file mode 100644 index 000000000..8390c568c --- /dev/null +++ b/libsolidity/interface/FileReader.cpp @@ -0,0 +1,98 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ +// SPDX-License-Identifier: GPL-3.0 +#include + +#include + +#include +#include +#include + +using solidity::frontend::ReadCallback; +using solidity::langutil::InternalCompilerError; +using solidity::util::errinfo_comment; +using solidity::util::readFileAsString; +using std::string; + +namespace solidity::frontend +{ + +void FileReader::setSource(boost::filesystem::path const& _path, SourceCode _source) +{ + m_sourceCodes[_path.generic_string()] = std::move(_source); +} + +void FileReader::setSources(StringMap _sources) +{ + m_sourceCodes = std::move(_sources); +} + +ReadCallback::Result FileReader::readFile(string const& _kind, string const& _path) +{ + try + { + if (_kind != ReadCallback::kindString(ReadCallback::Kind::ReadFile)) + BOOST_THROW_EXCEPTION(InternalCompilerError() << errinfo_comment( + "ReadFile callback used as callback kind " + + _kind + )); + string validPath = _path; + if (validPath.find("file://") == 0) + validPath.erase(0, 7); + + auto const path = m_basePath / validPath; + auto canonicalPath = boost::filesystem::weakly_canonical(path); + bool isAllowed = false; + for (auto const& allowedDir: m_allowedDirectories) + { + // If dir is a prefix of boostPath, we are fine. + if ( + std::distance(allowedDir.begin(), allowedDir.end()) <= std::distance(canonicalPath.begin(), canonicalPath.end()) && + std::equal(allowedDir.begin(), allowedDir.end(), canonicalPath.begin()) + ) + { + isAllowed = true; + break; + } + } + if (!isAllowed) + return ReadCallback::Result{false, "File outside of allowed directories."}; + + if (!boost::filesystem::exists(canonicalPath)) + return ReadCallback::Result{false, "File not found."}; + + if (!boost::filesystem::is_regular_file(canonicalPath)) + return ReadCallback::Result{false, "Not a valid file."}; + + // NOTE: we ignore the FileNotFound exception as we manually check above + auto contents = readFileAsString(canonicalPath.string()); + m_sourceCodes[path.generic_string()] = contents; + return ReadCallback::Result{true, contents}; + } + catch (util::Exception const& _exception) + { + return ReadCallback::Result{false, "Exception in read callback: " + boost::diagnostic_information(_exception)}; + } + catch (...) + { + return ReadCallback::Result{false, "Unknown exception in read callback."}; + } +} + +} + diff --git a/libsolidity/interface/FileReader.h b/libsolidity/interface/FileReader.h new file mode 100644 index 000000000..c5ea4d71d --- /dev/null +++ b/libsolidity/interface/FileReader.h @@ -0,0 +1,90 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ +// SPDX-License-Identifier: GPL-3.0 +#pragma once + +#include +#include + +#include + +#include +#include + +namespace solidity::frontend +{ + +/// FileReader - used for progressively loading source code. +/// +/// It is used in solc to load files from CLI parameters, stdin, or from JSON and +/// also used in the solc language server where solc is a long running process. +class FileReader +{ +public: + using StringMap = std::map; + using PathMap = std::map; + using FileSystemPathSet = std::set; + + /// Constructs a FileReader with a base path and a set of allowed directories that + /// will be used when requesting files from this file reader instance. + explicit FileReader( + boost::filesystem::path _basePath = {}, + FileSystemPathSet _allowedDirectories = {} + ): + m_basePath(std::move(_basePath)), + m_allowedDirectories(std::move(_allowedDirectories)), + m_sourceCodes() + {} + + void setBasePath(boost::filesystem::path _path) { m_basePath = std::move(_path); } + boost::filesystem::path const& basePath() const noexcept { return m_basePath; } + + void allowDirectory(boost::filesystem::path _path) { m_allowedDirectories.insert(std::move(_path)); } + FileSystemPathSet const& allowedDirectories() const noexcept { return m_allowedDirectories; } + + StringMap const& sourceCodes() const noexcept { return m_sourceCodes; } + + /// Retrieves the source code for a given source unit ID. + SourceCode const& sourceCode(SourceUnitName const& _sourceUnitName) const { return m_sourceCodes.at(_sourceUnitName); } + + /// Resets all sources to the given map of source unit ID to source codes. + void setSources(StringMap _sources); + + /// Adds the source code for a given source unit ID. + void setSource(boost::filesystem::path const& _path, SourceCode _source); + + /// Reads a given file at @p _path of kind @p _kind from the local filesystem and returns the result. + /// @p _kind must always be passed as "source". + frontend::ReadCallback::Result readFile(std::string const& _kind, std::string const& _path); + + frontend::ReadCallback::Callback reader() + { + return [this](std::string const& _kind, std::string const& _path) { return readFile(_kind, _path); }; + } + +private: + /// Base path, used for resolving relative paths in imports. + boost::filesystem::path m_basePath; + + /// list of allowed directories to read files from + FileSystemPathSet m_allowedDirectories; + + /// map of input files to source code strings + StringMap m_sourceCodes; +}; + +} diff --git a/solc/CommandLineInterface.cpp b/solc/CommandLineInterface.cpp index f561870cd..2de9c1350 100644 --- a/solc/CommandLineInterface.cpp +++ b/solc/CommandLineInterface.cpp @@ -26,6 +26,7 @@ #include "solidity/BuildInfo.h" #include "license.h" +#include #include #include #include @@ -58,6 +59,8 @@ #include #include +#include + #include #include #include @@ -623,14 +626,16 @@ bool CommandLineInterface::readInputFilesAndConfigureRemappings() } // NOTE: we ignore the FileNotFound exception as we manually check above - m_sourceCodes[infile.generic_string()] = readFileAsString(infile.string()); + m_fileReader.setSource(infile, readFileAsString(infile.string())); path = boost::filesystem::canonical(infile).string(); } - m_allowedDirectories.push_back(boost::filesystem::path(path).remove_filename()); + m_fileReader.allowDirectory(boost::filesystem::path(path).remove_filename()); } + if (addStdin) - m_sourceCodes[g_stdinFileName] = readStandardInput(); - if (m_sourceCodes.size() == 0) + m_fileReader.setSource(g_stdinFileName, readStandardInput()); + + if (m_fileReader.sourceCodes().size() == 0) { serr() << "No input files given. If you wish to use the standard input please specify \"-\" explicitly." << endl; return false; @@ -739,10 +744,10 @@ map CommandLineInterface::parseAstFromInput() map sourceJsons; map tmpSources; - for (auto const& srcPair: m_sourceCodes) + for (SourceCode const& sourceCode: m_fileReader.sourceCodes() | ranges::views::values) { Json::Value ast; - astAssert(jsonParseStrict(srcPair.second, ast), "Input file could not be parsed to JSON"); + astAssert(jsonParseStrict(sourceCode, ast), "Input file could not be parsed to JSON"); astAssert(ast.isMember("sources"), "Invalid Format for import-JSON: Must have 'sources'-object"); for (auto& src: ast["sources"].getMemberNames()) @@ -757,7 +762,8 @@ map CommandLineInterface::parseAstFromInput() } } - m_sourceCodes = std::move(tmpSources); + m_fileReader.setSources(tmpSources); + return sourceJsons; } @@ -1163,58 +1169,6 @@ General Information)").c_str(), bool CommandLineInterface::processInput() { - ReadCallback::Callback fileReader = [this](string const& _kind, string const& _path) - { - try - { - if (_kind != ReadCallback::kindString(ReadCallback::Kind::ReadFile)) - BOOST_THROW_EXCEPTION(InternalCompilerError() << errinfo_comment( - "ReadFile callback used as callback kind " + - _kind - )); - string validPath = _path; - if (validPath.find("file://") == 0) - validPath.erase(0, 7); - - auto const path = m_basePath / validPath; - auto canonicalPath = boost::filesystem::weakly_canonical(path); - bool isAllowed = false; - for (auto const& allowedDir: m_allowedDirectories) - { - // If dir is a prefix of boostPath, we are fine. - if ( - std::distance(allowedDir.begin(), allowedDir.end()) <= std::distance(canonicalPath.begin(), canonicalPath.end()) && - std::equal(allowedDir.begin(), allowedDir.end(), canonicalPath.begin()) - ) - { - isAllowed = true; - break; - } - } - if (!isAllowed) - return ReadCallback::Result{false, "File outside of allowed directories."}; - - if (!boost::filesystem::exists(canonicalPath)) - return ReadCallback::Result{false, "File not found."}; - - if (!boost::filesystem::is_regular_file(canonicalPath)) - return ReadCallback::Result{false, "Not a valid file."}; - - // NOTE: we ignore the FileNotFound exception as we manually check above - auto contents = readFileAsString(canonicalPath.string()); - m_sourceCodes[path.generic_string()] = contents; - return ReadCallback::Result{true, contents}; - } - catch (Exception const& _exception) - { - return ReadCallback::Result{false, "Exception in read callback: " + boost::diagnostic_information(_exception)}; - } - catch (...) - { - return ReadCallback::Result{false, "Unknown exception in read callback."}; - } - }; - if (m_args.count(g_argBasePath)) { boost::filesystem::path const fspath{m_args[g_argBasePath].as()}; @@ -1223,9 +1177,7 @@ bool CommandLineInterface::processInput() serr() << "Base path must be a directory: \"" << fspath << "\"\n"; return false; } - m_basePath = fspath; - if (!contains(m_allowedDirectories, fspath)) - m_allowedDirectories.push_back(fspath); + m_fileReader.setBasePath(fspath); } if (m_args.count(g_argAllowPaths)) @@ -1240,7 +1192,7 @@ bool CommandLineInterface::processInput() // it. if (filesystem_path.filename() == ".") filesystem_path.remove_filename(); - m_allowedDirectories.push_back(filesystem_path); + m_fileReader.allowDirectory(filesystem_path); } } @@ -1298,7 +1250,7 @@ bool CommandLineInterface::processInput() return false; } } - StandardCompiler compiler(fileReader); + StandardCompiler compiler(m_fileReader.reader()); sout() << compiler.compile(std::move(input)) << endl; return true; } @@ -1490,7 +1442,7 @@ bool CommandLineInterface::processInput() if (m_args.count(g_argModelCheckerTimeout)) m_modelCheckerSettings.timeout = m_args[g_argModelCheckerTimeout].as(); - m_compiler = make_unique(fileReader); + m_compiler = make_unique(m_fileReader.reader()); SourceReferenceFormatter formatter(serr(false), m_coloredOutput, m_withErrorIds); @@ -1564,7 +1516,7 @@ bool CommandLineInterface::processInput() } else { - m_compiler->setSources(m_sourceCodes); + m_compiler->setSources(m_fileReader.sourceCodes()); if (m_args.count(g_argErrorRecovery)) m_compiler->setParserErrorRecovery(true); } @@ -1714,7 +1666,7 @@ void CommandLineInterface::handleCombinedJSON() if (requests.count(g_strAst)) { output[g_strSources] = Json::Value(Json::objectValue); - for (auto const& sourceCode: m_sourceCodes) + for (auto const& sourceCode: m_fileReader.sourceCodes()) { ASTJsonConverter converter(m_compiler->state(), m_compiler->sourceIndices()); output[g_strSources][sourceCode.first] = Json::Value(Json::objectValue); @@ -1737,12 +1689,12 @@ void CommandLineInterface::handleAst() return; vector asts; - for (auto const& sourceCode: m_sourceCodes) + for (auto const& sourceCode: m_fileReader.sourceCodes()) asts.push_back(&m_compiler->ast(sourceCode.first)); if (m_args.count(g_argOutputDir)) { - for (auto const& sourceCode: m_sourceCodes) + for (auto const& sourceCode: m_fileReader.sourceCodes()) { stringstream data; string postfix = ""; @@ -1755,7 +1707,7 @@ void CommandLineInterface::handleAst() else { sout() << "JSON AST (compact format):" << endl << endl; - for (auto const& sourceCode: m_sourceCodes) + for (auto const& sourceCode: m_fileReader.sourceCodes()) { sout() << endl << "======= " << sourceCode.first << " =======" << endl; ASTJsonConverter(m_compiler->state(), m_compiler->sourceIndices()).print(sout(), m_compiler->ast(sourceCode.first)); @@ -1796,7 +1748,9 @@ bool CommandLineInterface::link() replacement += "__"; librariesReplacements[replacement] = library.second; } - for (auto& src: m_sourceCodes) + + FileReader::StringMap sourceCodes = m_fileReader.sourceCodes(); + for (auto& src: sourceCodes) { auto end = src.second.end(); for (auto it = src.second.begin(); it != end;) @@ -1831,12 +1785,14 @@ bool CommandLineInterface::link() while (!src.second.empty() && *prev(src.second.end()) == '\n') src.second.resize(src.second.size() - 1); } + m_fileReader.setSources(move(sourceCodes)); + return true; } void CommandLineInterface::writeLinkedFiles() { - for (auto const& src: m_sourceCodes) + for (auto const& src: m_fileReader.sourceCodes()) if (src.first == g_stdinFileName) sout() << src.second << endl; else @@ -1880,7 +1836,7 @@ bool CommandLineInterface::assemble( bool successful = true; map assemblyStacks; - for (auto const& src: m_sourceCodes) + for (auto const& src: m_fileReader.sourceCodes()) { OptimiserSettings settings = _optimize ? OptimiserSettings::full() : OptimiserSettings::minimal(); if (_yulOptimiserSteps.has_value()) @@ -1931,7 +1887,7 @@ bool CommandLineInterface::assemble( if (!successful) return false; - for (auto const& src: m_sourceCodes) + for (auto const& src: m_fileReader.sourceCodes()) { string machine = _targetMachine == yul::AssemblyStack::Machine::EVM ? "EVM" : @@ -2043,7 +1999,7 @@ void CommandLineInterface::outputCompilationResults() if (m_args.count(g_argAsmJson)) ret = jsonPrettyPrint(removeNullMembers(m_compiler->assemblyJSON(contract))); else - ret = m_compiler->assemblyString(contract, m_sourceCodes); + ret = m_compiler->assemblyString(contract, m_fileReader.sourceCodes()); if (m_args.count(g_argOutputDir)) { diff --git a/solc/CommandLineInterface.h b/solc/CommandLineInterface.h index 1e6e740c1..73c220e37 100644 --- a/solc/CommandLineInterface.h +++ b/solc/CommandLineInterface.h @@ -24,6 +24,7 @@ #include #include +#include #include #include #include @@ -113,16 +114,12 @@ private: bool m_onlyLink = false; + FileReader m_fileReader; + /// Compiler arguments variable map boost::program_options::variables_map m_args; - /// map of input files to source code strings - std::map m_sourceCodes; /// list of remappings std::vector m_remappings; - /// list of allowed directories to read files from - std::vector m_allowedDirectories; - /// Base path, used for resolving relative paths in imports. - boost::filesystem::path m_basePath; /// map of library names to addresses std::map m_libraries; /// Solidity compiler stack