Extract file reading logic from CommandLineInterface into FileReader class.

libsolidity/CMakeLists.txt

@@ -133,6 +133,8 @@ set(sources
 	interface/CompilerStack.cpp
 	interface/CompilerStack.h
 	interface/DebugSettings.h
+	interface/FileReader.cpp
+	interface/FileReader.h
 	interface/ImportRemapper.cpp
 	interface/ImportRemapper.h
 	interface/GasEstimator.cpp

libsolidity/interface/FileReader.cpp

@@ -0,0 +1,98 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+// SPDX-License-Identifier: GPL-3.0
+#include <libsolidity/interface/FileReader.h>
+
+#include <liblangutil/Exceptions.h>
+
+#include <libsolutil/CommonData.h>
+#include <libsolutil/CommonIO.h>
+#include <libsolutil/Exceptions.h>
+
+using solidity::frontend::ReadCallback;
+using solidity::langutil::InternalCompilerError;
+using solidity::util::errinfo_comment;
+using solidity::util::readFileAsString;
+using std::string;
+
+namespace solidity::frontend
+{
+
+void FileReader::setSource(boost::filesystem::path const& _path, SourceCode _source)
+{
+	m_sourceCodes[_path.generic_string()] = std::move(_source);
+}
+
+void FileReader::setSources(StringMap _sources)
+{
+	m_sourceCodes = std::move(_sources);
+}
+
+ReadCallback::Result FileReader::readFile(string const& _kind, string const& _path)
+{
+	try
+	{
+		if (_kind != ReadCallback::kindString(ReadCallback::Kind::ReadFile))
+			BOOST_THROW_EXCEPTION(InternalCompilerError() << errinfo_comment(
+				"ReadFile callback used as callback kind " +
+				_kind
+			));
+		string validPath = _path;
+		if (validPath.find("file://") == 0)
+			validPath.erase(0, 7);
+
+		auto const path = m_basePath / validPath;
+		auto canonicalPath = boost::filesystem::weakly_canonical(path);
+		bool isAllowed = false;
+		for (auto const& allowedDir: m_allowedDirectories)
+		{
+			// If dir is a prefix of boostPath, we are fine.
+			if (
+				std::distance(allowedDir.begin(), allowedDir.end()) <= std::distance(canonicalPath.begin(), canonicalPath.end()) &&
+				std::equal(allowedDir.begin(), allowedDir.end(), canonicalPath.begin())
+			)
+			{
+				isAllowed = true;
+				break;
+			}
+		}
+		if (!isAllowed)
+			return ReadCallback::Result{false, "File outside of allowed directories."};
+
+		if (!boost::filesystem::exists(canonicalPath))
+			return ReadCallback::Result{false, "File not found."};
+
+		if (!boost::filesystem::is_regular_file(canonicalPath))
+			return ReadCallback::Result{false, "Not a valid file."};
+
+		// NOTE: we ignore the FileNotFound exception as we manually check above
+		auto contents = readFileAsString(canonicalPath.string());
+		m_sourceCodes[path.generic_string()] = contents;
+		return ReadCallback::Result{true, contents};
+	}
+	catch (util::Exception const& _exception)
+	{
+		return ReadCallback::Result{false, "Exception in read callback: " + boost::diagnostic_information(_exception)};
+	}
+	catch (...)
+	{
+		return ReadCallback::Result{false, "Unknown exception in read callback."};
+	}
+}
+
+}
+

libsolidity/interface/FileReader.h

@@ -0,0 +1,90 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+// SPDX-License-Identifier: GPL-3.0
+#pragma once
+
+#include <libsolidity/interface/ImportRemapper.h>
+#include <libsolidity/interface/ReadFile.h>
+
+#include <boost/filesystem.hpp>
+
+#include <map>
+#include <set>
+
+namespace solidity::frontend
+{
+
+/// FileReader - used for progressively loading source code.
+///
+/// It is used in solc to load files from CLI parameters, stdin, or from JSON and
+/// also used in the solc language server where solc is a long running process.
+class FileReader
+{
+public:
+	using StringMap = std::map<SourceUnitName, SourceCode>;
+	using PathMap = std::map<SourceUnitName, boost::filesystem::path>;
+	using FileSystemPathSet = std::set<boost::filesystem::path>;
+
+	/// Constructs a FileReader with a base path and a set of allowed directories that
+	/// will be used when requesting files from this file reader instance.
+	explicit FileReader(
+		boost::filesystem::path _basePath = {},
+		FileSystemPathSet _allowedDirectories = {}
+	):
+		m_basePath(std::move(_basePath)),
+		m_allowedDirectories(std::move(_allowedDirectories)),
+		m_sourceCodes()
+	{}
+
+	void setBasePath(boost::filesystem::path _path) { m_basePath = std::move(_path); }
+	boost::filesystem::path const& basePath() const noexcept { return m_basePath; }
+
+	void allowDirectory(boost::filesystem::path _path) { m_allowedDirectories.insert(std::move(_path)); }
+	FileSystemPathSet const& allowedDirectories() const noexcept { return m_allowedDirectories; }
+
+	StringMap const& sourceCodes() const noexcept { return m_sourceCodes; }
+
+	/// Retrieves the source code for a given source unit ID.
+	SourceCode const& sourceCode(SourceUnitName const& _sourceUnitName) const { return m_sourceCodes.at(_sourceUnitName); }
+
+	/// Resets all sources to the given map of source unit ID to source codes.
+	void setSources(StringMap _sources);
+
+	/// Adds the source code for a given source unit ID.
+	void setSource(boost::filesystem::path const& _path, SourceCode _source);
+
+	/// Reads a given file at @p _path of kind @p _kind from the local filesystem and returns the result.
+	/// @p _kind must always be passed as "source".
+	frontend::ReadCallback::Result readFile(std::string const& _kind, std::string const& _path);
+
+	frontend::ReadCallback::Callback reader()
+	{
+		return [this](std::string const& _kind, std::string const& _path) { return readFile(_kind, _path); };
+	}
+
+private:
+	/// Base path, used for resolving relative paths in imports.
+	boost::filesystem::path m_basePath;
+
+	/// list of allowed directories to read files from
+	FileSystemPathSet m_allowedDirectories;
+
+	/// map of input files to source code strings
+	StringMap m_sourceCodes;
+};
+
+}

solc/CommandLineInterface.cpp

@@ -26,6 +26,7 @@
 #include "solidity/BuildInfo.h"
 #include "license.h"
 
+#include <libsolidity/interface/FileReader.h>
 #include <libsolidity/interface/Version.h>
 #include <libsolidity/parsing/Parser.h>
 #include <libsolidity/ast/ASTJsonConverter.h>
@@ -58,6 +59,8 @@
 #include <algorithm>
 #include <memory>
 
+#include <range/v3/view/map.hpp>
+
 #include <boost/filesystem.hpp>
 #include <boost/filesystem/operations.hpp>
 #include <boost/range/adaptor/transformed.hpp>
@@ -623,14 +626,16 @@ bool CommandLineInterface::readInputFilesAndConfigureRemappings()
 				}
 
 				// NOTE: we ignore the FileNotFound exception as we manually check above
-				m_sourceCodes[infile.generic_string()] = readFileAsString(infile.string());
+				m_fileReader.setSource(infile, readFileAsString(infile.string()));
 				path = boost::filesystem::canonical(infile).string();
 			}
-			m_allowedDirectories.push_back(boost::filesystem::path(path).remove_filename());
+			m_fileReader.allowDirectory(boost::filesystem::path(path).remove_filename());
 		}
+
 	if (addStdin)
-		m_sourceCodes[g_stdinFileName] = readStandardInput();
-	if (m_sourceCodes.size() == 0)
+		m_fileReader.setSource(g_stdinFileName, readStandardInput());
+
+	if (m_fileReader.sourceCodes().size() == 0)
 	{
 		serr() << "No input files given. If you wish to use the standard input please specify \"-\" explicitly." << endl;
 		return false;
@@ -739,10 +744,10 @@ map<string, Json::Value> CommandLineInterface::parseAstFromInput()
 	map<string, Json::Value> sourceJsons;
 	map<string, string> tmpSources;
 
-	for (auto const& srcPair: m_sourceCodes)
+	for (SourceCode const& sourceCode: m_fileReader.sourceCodes() | ranges::views::values)
 	{
 		Json::Value ast;
-		astAssert(jsonParseStrict(srcPair.second, ast), "Input file could not be parsed to JSON");
+		astAssert(jsonParseStrict(sourceCode, ast), "Input file could not be parsed to JSON");
 		astAssert(ast.isMember("sources"), "Invalid Format for import-JSON: Must have 'sources'-object");
 
 		for (auto& src: ast["sources"].getMemberNames())
@@ -757,7 +762,8 @@ map<string, Json::Value> CommandLineInterface::parseAstFromInput()
 		}
 	}
 
-	m_sourceCodes = std::move(tmpSources);
+	m_fileReader.setSources(tmpSources);
+
 	return sourceJsons;
 }
 
@@ -1163,58 +1169,6 @@ General Information)").c_str(),
 
 bool CommandLineInterface::processInput()
 {
-	ReadCallback::Callback fileReader = [this](string const& _kind, string const& _path)
-	{
-		try
-		{
-			if (_kind != ReadCallback::kindString(ReadCallback::Kind::ReadFile))
-				BOOST_THROW_EXCEPTION(InternalCompilerError() << errinfo_comment(
-					"ReadFile callback used as callback kind " +
-					_kind
-				));
-			string validPath = _path;
-			if (validPath.find("file://") == 0)
-				validPath.erase(0, 7);
-
-			auto const path = m_basePath / validPath;
-			auto canonicalPath = boost::filesystem::weakly_canonical(path);
-			bool isAllowed = false;
-			for (auto const& allowedDir: m_allowedDirectories)
-			{
-				// If dir is a prefix of boostPath, we are fine.
-				if (
-					std::distance(allowedDir.begin(), allowedDir.end()) <= std::distance(canonicalPath.begin(), canonicalPath.end()) &&
-					std::equal(allowedDir.begin(), allowedDir.end(), canonicalPath.begin())
-				)
-				{
-					isAllowed = true;
-					break;
-				}
-			}
-			if (!isAllowed)
-				return ReadCallback::Result{false, "File outside of allowed directories."};
-
-			if (!boost::filesystem::exists(canonicalPath))
-				return ReadCallback::Result{false, "File not found."};
-
-			if (!boost::filesystem::is_regular_file(canonicalPath))
-				return ReadCallback::Result{false, "Not a valid file."};
-
-			// NOTE: we ignore the FileNotFound exception as we manually check above
-			auto contents = readFileAsString(canonicalPath.string());
-			m_sourceCodes[path.generic_string()] = contents;
-			return ReadCallback::Result{true, contents};
-		}
-		catch (Exception const& _exception)
-		{
-			return ReadCallback::Result{false, "Exception in read callback: " + boost::diagnostic_information(_exception)};
-		}
-		catch (...)
-		{
-			return ReadCallback::Result{false, "Unknown exception in read callback."};
-		}
-	};
-
 	if (m_args.count(g_argBasePath))
 	{
 		boost::filesystem::path const fspath{m_args[g_argBasePath].as<string>()};
@@ -1223,9 +1177,7 @@ bool CommandLineInterface::processInput()
 			serr() << "Base path must be a directory: \"" << fspath << "\"\n";
 			return false;
 		}
-		m_basePath = fspath;
-		if (!contains(m_allowedDirectories, fspath))
-			m_allowedDirectories.push_back(fspath);
+		m_fileReader.setBasePath(fspath);
 	}
 
 	if (m_args.count(g_argAllowPaths))
@@ -1240,7 +1192,7 @@ bool CommandLineInterface::processInput()
 			// it.
 			if (filesystem_path.filename() == ".")
 				filesystem_path.remove_filename();
-			m_allowedDirectories.push_back(filesystem_path);
+			m_fileReader.allowDirectory(filesystem_path);
 		}
 	}
 
@@ -1298,7 +1250,7 @@ bool CommandLineInterface::processInput()
 				return false;
 			}
 		}
-		StandardCompiler compiler(fileReader);
+		StandardCompiler compiler(m_fileReader.reader());
 		sout() << compiler.compile(std::move(input)) << endl;
 		return true;
 	}
@@ -1490,7 +1442,7 @@ bool CommandLineInterface::processInput()
 	if (m_args.count(g_argModelCheckerTimeout))
 		m_modelCheckerSettings.timeout = m_args[g_argModelCheckerTimeout].as<unsigned>();
 
-	m_compiler = make_unique<CompilerStack>(fileReader);
+	m_compiler = make_unique<CompilerStack>(m_fileReader.reader());
 
 	SourceReferenceFormatter formatter(serr(false), m_coloredOutput, m_withErrorIds);
 
@@ -1564,7 +1516,7 @@ bool CommandLineInterface::processInput()
 		}
 		else
 		{
-			m_compiler->setSources(m_sourceCodes);
+			m_compiler->setSources(m_fileReader.sourceCodes());
 			if (m_args.count(g_argErrorRecovery))
 				m_compiler->setParserErrorRecovery(true);
 		}
@@ -1714,7 +1666,7 @@ void CommandLineInterface::handleCombinedJSON()
 	if (requests.count(g_strAst))
 	{
 		output[g_strSources] = Json::Value(Json::objectValue);
-		for (auto const& sourceCode: m_sourceCodes)
+		for (auto const& sourceCode: m_fileReader.sourceCodes())
 		{
 			ASTJsonConverter converter(m_compiler->state(), m_compiler->sourceIndices());
 			output[g_strSources][sourceCode.first] = Json::Value(Json::objectValue);
@@ -1737,12 +1689,12 @@ void CommandLineInterface::handleAst()
 		return;
 
 	vector<ASTNode const*> asts;
-	for (auto const& sourceCode: m_sourceCodes)
+	for (auto const& sourceCode: m_fileReader.sourceCodes())
 		asts.push_back(&m_compiler->ast(sourceCode.first));
 
 	if (m_args.count(g_argOutputDir))
 	{
-		for (auto const& sourceCode: m_sourceCodes)
+		for (auto const& sourceCode: m_fileReader.sourceCodes())
 		{
 			stringstream data;
 			string postfix = "";
@@ -1755,7 +1707,7 @@ void CommandLineInterface::handleAst()
 	else
 	{
 		sout() << "JSON AST (compact format):" << endl << endl;
-		for (auto const& sourceCode: m_sourceCodes)
+		for (auto const& sourceCode: m_fileReader.sourceCodes())
 		{
 			sout() << endl << "======= " << sourceCode.first << " =======" << endl;
 			ASTJsonConverter(m_compiler->state(), m_compiler->sourceIndices()).print(sout(), m_compiler->ast(sourceCode.first));
@@ -1796,7 +1748,9 @@ bool CommandLineInterface::link()
 		replacement += "__";
 		librariesReplacements[replacement] = library.second;
 	}
-	for (auto& src: m_sourceCodes)
+
+	FileReader::StringMap sourceCodes = m_fileReader.sourceCodes();
+	for (auto& src: sourceCodes)
 	{
 		auto end = src.second.end();
 		for (auto it = src.second.begin(); it != end;)
@@ -1831,12 +1785,14 @@ bool CommandLineInterface::link()
 		while (!src.second.empty() && *prev(src.second.end()) == '\n')
 			src.second.resize(src.second.size() - 1);
 	}
+	m_fileReader.setSources(move(sourceCodes));
+
 	return true;
 }
 
 void CommandLineInterface::writeLinkedFiles()
 {
-	for (auto const& src: m_sourceCodes)
+	for (auto const& src: m_fileReader.sourceCodes())
 		if (src.first == g_stdinFileName)
 			sout() << src.second << endl;
 		else
@@ -1880,7 +1836,7 @@ bool CommandLineInterface::assemble(
 
 	bool successful = true;
 	map<string, yul::AssemblyStack> assemblyStacks;
-	for (auto const& src: m_sourceCodes)
+	for (auto const& src: m_fileReader.sourceCodes())
 	{
 		OptimiserSettings settings = _optimize ? OptimiserSettings::full() : OptimiserSettings::minimal();
 		if (_yulOptimiserSteps.has_value())
@@ -1931,7 +1887,7 @@ bool CommandLineInterface::assemble(
 	if (!successful)
 		return false;
 
-	for (auto const& src: m_sourceCodes)
+	for (auto const& src: m_fileReader.sourceCodes())
 	{
 		string machine =
 			_targetMachine == yul::AssemblyStack::Machine::EVM ? "EVM" :
@@ -2043,7 +1999,7 @@ void CommandLineInterface::outputCompilationResults()
 			if (m_args.count(g_argAsmJson))
 				ret = jsonPrettyPrint(removeNullMembers(m_compiler->assemblyJSON(contract)));
 			else
-				ret = m_compiler->assemblyString(contract, m_sourceCodes);
+				ret = m_compiler->assemblyString(contract, m_fileReader.sourceCodes());
 
 			if (m_args.count(g_argOutputDir))
 			{

solc/CommandLineInterface.h

@@ -24,6 +24,7 @@
 
 #include <libsolidity/interface/CompilerStack.h>
 #include <libsolidity/interface/DebugSettings.h>
+#include <libsolidity/interface/FileReader.h>
 #include <libsolidity/interface/ImportRemapper.h>
 #include <libyul/AssemblyStack.h>
 #include <liblangutil/EVMVersion.h>
@@ -113,16 +114,12 @@ private:
 
 	bool m_onlyLink = false;
 
+	FileReader m_fileReader;
+
 	/// Compiler arguments variable map
 	boost::program_options::variables_map m_args;
-	/// map of input files to source code strings
-	std::map<std::string, std::string> m_sourceCodes;
 	/// list of remappings
 	std::vector<ImportRemapper::Remapping> m_remappings;
-	/// list of allowed directories to read files from
-	std::vector<boost::filesystem::path> m_allowedDirectories;
-	/// Base path, used for resolving relative paths in imports.
-	boost::filesystem::path m_basePath;
 	/// map of library names to addresses
 	std::map<std::string, util::h160> m_libraries;
 	/// Solidity compiler stack