Multi-source and multi-contract compiler.

BaseTypes.h

@@ -22,6 +22,8 @@
 
 #pragma once
 
+#include <memory>
+#include <string>
 #include <ostream>
 
 namespace dev
@@ -38,8 +40,9 @@ struct Location
 	Location(int _start, int _end): start(_start), end(_end) { }
 	Location(): start(-1), end(-1) { }
 
-	bool IsValid() const { return start >= 0 && end >= start; }
+	bool IsValid() const { return !!sourceName && start >= 0 && end >= start; }
 
+	std::shared_ptr<std::string> sourceName;
 	int start;
 	int end;
 };
@@ -47,7 +50,7 @@ struct Location
 /// Stream output for Location (used e.g. in boost exceptions).
 inline std::ostream& operator<<(std::ostream& _out, Location const& _location)
 {
-	return _out << "[" << _location.start << "," << _location.end << ")";
+	return _out << *_location.sourceName << "[" << _location.start << "," << _location.end << ")";
 }
 
 }

CompilerStack.cpp

@@ -35,24 +35,43 @@ namespace dev
 namespace solidity
 {
 
+void CompilerStack::addSource(string const& _name, string const& _content)
+{
+	if (m_sources.count(_name))
+		BOOST_THROW_EXCEPTION(CompilerError() << errinfo_comment("Source by given name already exists."));
+
+	reset(true);
+	m_sources[_name].scanner = make_shared<Scanner>(CharStream(_content), _name);
+}
+
 void CompilerStack::setSource(string const& _sourceCode)
 {
 	reset();
-	m_scanner = make_shared<Scanner>(CharStream(_sourceCode));
+	addSource("", _sourceCode);
 }
 
 void CompilerStack::parse()
 {
-	if (!m_scanner)
-		BOOST_THROW_EXCEPTION(CompilerError() << errinfo_comment("Source not available."));
-	m_sourceUnitASTNode = Parser().parse(m_scanner);
+	for (auto& sourcePair: m_sources)
+	{
+		sourcePair.second.scanner->reset();
+		sourcePair.second.ast = Parser().parse(sourcePair.second.scanner);
+	}
+	resolveImports();
+
 	m_globalContext = make_shared<GlobalContext>();
-	for (ASTPointer<ASTNode> const& node: m_sourceUnitASTNode->getNodes())
-		if (ContractDefinition* contract = dynamic_cast<ContractDefinition*>(node.get()))
-		{
-			m_globalContext->setCurrentContract(*contract);
-			NameAndTypeResolver(m_globalContext->getDeclarations()).resolveNamesAndTypes(*contract);
-		}
+	NameAndTypeResolver resolver(m_globalContext->getDeclarations());
+	for (Source const* source: m_sourceOrder)
+		resolver.registerDeclarations(*source->ast);
+	for (Source const* source: m_sourceOrder)
+		for (ASTPointer<ASTNode> const& node: source->ast->getNodes())
+			if (ContractDefinition* contract = dynamic_cast<ContractDefinition*>(node.get()))
+			{
+				m_globalContext->setCurrentContract(*contract);
+				resolver.updateDeclaration(*m_globalContext->getCurrentThis());
+				resolver.resolveNamesAndTypes(*contract);
+				m_contracts[contract->getName()].contract = contract;
+			}
 	m_parseSuccessful = true;
 }
 
@@ -62,82 +81,91 @@ void CompilerStack::parse(string const& _sourceCode)
 	parse();
 }
 
-bytes const& CompilerStack::compile(bool _optimize)
+void CompilerStack::compile(bool _optimize)
 {
 	if (!m_parseSuccessful)
 		BOOST_THROW_EXCEPTION(CompilerError() << errinfo_comment("Parsing was not successful."));
-	//@todo returns only the last contract for now
-	for (ASTPointer<ASTNode> const& node: m_sourceUnitASTNode->getNodes())
-		if (ContractDefinition* contract = dynamic_cast<ContractDefinition*>(node.get()))
-		{
-			m_bytecode.clear();
-			m_compiler = make_shared<Compiler>();
-			m_compiler->compileContract(*contract, m_globalContext->getMagicVariables());
-			m_bytecode = m_compiler->getAssembledBytecode(_optimize);
-		}
-	return m_bytecode;
+	for (Source const* source: m_sourceOrder)
+		for (ASTPointer<ASTNode> const& node: source->ast->getNodes())
+			if (ContractDefinition* contract = dynamic_cast<ContractDefinition*>(node.get()))
+			{
+				m_globalContext->setCurrentContract(*contract);
+				shared_ptr<Compiler> compiler = make_shared<Compiler>();
+				compiler->compileContract(*contract, m_globalContext->getMagicVariables());
+				Contract& compiledContract = m_contracts[contract->getName()];
+				compiledContract.bytecode = compiler->getAssembledBytecode(_optimize);
+				compiledContract.compiler = move(compiler);
+			}
 }
 
 bytes const& CompilerStack::compile(string const& _sourceCode, bool _optimize)
 {
 	parse(_sourceCode);
-	return compile(_optimize);
+	compile(_optimize);
+	return getBytecode();
 }
 
-void CompilerStack::streamAssembly(ostream& _outStream)
+bytes const& CompilerStack::getBytecode(string const& _contractName)
 {
-	if (!m_compiler || m_bytecode.empty())
-		BOOST_THROW_EXCEPTION(CompilerError() << errinfo_comment("Compilation was not successful."));
-	m_compiler->streamAssembly(_outStream);
+	return getContract(_contractName).bytecode;
 }
 
-string const& CompilerStack::getInterface()
+void CompilerStack::streamAssembly(ostream& _outStream, string const& _contractName)
 {
-	if (!m_parseSuccessful)
-		BOOST_THROW_EXCEPTION(CompilerError() << errinfo_comment("Parsing was not successful."));
-	if (m_interface.empty())
+	getContract(_contractName).compiler->streamAssembly(_outStream);
+}
+
+string const& CompilerStack::getInterface(std::string const& _contractName)
+{
+	Contract& contract = getContract(_contractName);
+	if (contract.interface.empty())
 	{
-		//@todo returns only the last contract for now
-		for (ASTPointer<ASTNode> const& node: m_sourceUnitASTNode->getNodes())
-			if (ContractDefinition const* contract = dynamic_cast<ContractDefinition*>(node.get()))
+		stringstream interface;
+		interface << '[';
+		vector<FunctionDefinition const*> exportedFunctions = contract.contract->getInterfaceFunctions();
+		unsigned functionsCount = exportedFunctions.size();
+		for (FunctionDefinition const* f: exportedFunctions)
+		{
+			auto streamVariables = [&](vector<ASTPointer<VariableDeclaration>> const& _vars)
 			{
-				stringstream interface;
-				interface << '[';
-				vector<FunctionDefinition const*> exportedFunctions = contract->getInterfaceFunctions();
-				unsigned functionsCount = exportedFunctions.size();
-				for (FunctionDefinition const* f: exportedFunctions)
+				unsigned varCount = _vars.size();
+				for (ASTPointer<VariableDeclaration> const& var: _vars)
 				{
-					auto streamVariables = [&](vector<ASTPointer<VariableDeclaration>> const& _vars)
-					{
-						unsigned varCount = _vars.size();
-						for (ASTPointer<VariableDeclaration> const& var: _vars)
-						{
-							interface << "{"
-									  << "\"name\":" << escaped(var->getName(), false) << ","
-									  << "\"type\":" << escaped(var->getType()->toString(), false)
-									  << "}";
-							if (--varCount > 0)
-								interface << ",";
-						}
-					};
-
-					interface << '{'
-							  << "\"name\":" << escaped(f->getName(), false) << ","
-							  << "\"inputs\":[";
-					streamVariables(f->getParameters());
-					interface << "],"
-							  << "\"outputs\":[";
-					streamVariables(f->getReturnParameters());
-					interface << "]"
+					interface << "{"
+							  << "\"name\":" << escaped(var->getName(), false) << ","
+							  << "\"type\":" << escaped(var->getType()->toString(), false)
 							  << "}";
-					if (--functionsCount > 0)
+					if (--varCount > 0)
 						interface << ",";
 				}
-				interface << ']';
-				m_interface = interface.str();
-			}
+			};
+
+			interface << '{'
+					  << "\"name\":" << escaped(f->getName(), false) << ","
+					  << "\"inputs\":[";
+			streamVariables(f->getParameters());
+			interface << "],"
+					  << "\"outputs\":[";
+			streamVariables(f->getReturnParameters());
+			interface << "]"
+					  << "}";
+			if (--functionsCount > 0)
+				interface << ",";
+		}
+		interface << ']';
+		contract.interface = interface.str();
 	}
-	return m_interface;
+	return contract.interface;
+}
+
+Scanner const& CompilerStack::getScanner(string const& _sourceName)
+{
+	return *getSource(_sourceName).scanner;
+}
+
+SourceUnit& CompilerStack::getAST(string const& _sourceName)
+{
+	return *getSource(_sourceName).ast;
 }
 
 bytes CompilerStack::staticCompile(std::string const& _sourceCode, bool _optimize)
@@ -146,7 +174,68 @@ bytes CompilerStack::staticCompile(std::string const& _sourceCode, bool _optimiz
 	return stack.compile(_sourceCode, _optimize);
 }
 
+void CompilerStack::reset(bool _keepSources)
+{
+	m_parseSuccessful = false;
+	if (_keepSources)
+		for (auto sourcePair: m_sources)
+			sourcePair.second.reset();
+	else
+		m_sources.clear();
+	m_globalContext.reset();
+	m_compiler.reset();
+	m_sourceOrder.clear();
+	m_contracts.clear();
+}
 
+void CompilerStack::resolveImports()
+{
+	// topological sorting (depth first search) of the import graph, cutting potential cycles
+	vector<Source const*> sourceOrder;
+	set<Source const*> sourcesSeen;
 
+	function<void(Source const*)> toposort = [&](Source const* _source)
+	{
+		if (sourcesSeen.count(_source))
+			return;
+		sourcesSeen.insert(_source);
+		for (ASTPointer<ASTNode> const& node: _source->ast->getNodes())
+			if (ImportDirective const* import = dynamic_cast<ImportDirective*>(node.get()))
+			{
+				string const& url = import->getURL();
+				if (!m_sources.count(url))
+					BOOST_THROW_EXCEPTION(ParserError()
+										  << errinfo_sourceLocation(import->getLocation())
+										  << errinfo_comment("Source not found."));
+				toposort(&m_sources[url]);
+			}
+		sourceOrder.push_back(_source);
+	};
+
+	for (auto const& sourcePair: m_sources)
+		toposort(&sourcePair.second);
+
+	swap(m_sourceOrder, sourceOrder);
+}
+
+CompilerStack::Contract& CompilerStack::getContract(string const& _contractName)
+{
+	if (m_contracts.empty())
+		BOOST_THROW_EXCEPTION(CompilerError() << errinfo_comment("No compiled contracts found."));
+	if (_contractName.empty())
+		return m_contracts.begin()->second;
+	auto it = m_contracts.find(_contractName);
+	if (it == m_contracts.end())
+		BOOST_THROW_EXCEPTION(CompilerError() << errinfo_comment("Contract " + _contractName + " not found."));
+	return it->second;
+}
+
+CompilerStack::Source& CompilerStack::getSource(string const& _sourceName)
+{
+	auto it = m_sources.find(_sourceName);
+	if (it == m_sources.end())
+		BOOST_THROW_EXCEPTION(CompilerError() << errinfo_comment("Given source file not found."));
+	return it->second;
+}
 }
 }

CompilerStack.h

@@ -25,6 +25,7 @@
 #include <ostream>
 #include <string>
 #include <memory>
+#include <boost/noncopyable.hpp>
 #include <libdevcore/Common.h>
 
 namespace dev {
@@ -33,53 +34,81 @@ namespace solidity {
 // forward declarations
 class Scanner;
 class SourceUnit;
-class ContractDefinition;
 class Compiler;
 class GlobalContext;
+class ContractDefinition;
 
 /**
  * Easy to use and self-contained Solidity compiler with as few header dependencies as possible.
  * It holds state and can be used to either step through the compilation stages (and abort e.g.
  * before compilation to bytecode) or run the whole compilation in one call.
  */
-class CompilerStack
+class CompilerStack: boost::noncopyable
 {
 public:
-	CompilerStack() {}
-	void reset() {  *this = CompilerStack(); }
+	CompilerStack(): m_parseSuccessful(false) {}
+	/// Adds a source object (e.g. file) to the parser. After this, parse has to be called again.
+	void addSource(std::string const& _name, std::string const& _content);
 	void setSource(std::string const& _sourceCode);
+	/// Parses all source units that were added
 	void parse();
+	/// Sets the given source code as the only source unit and parses it.
 	void parse(std::string const& _sourceCode);
-	/// Compiles the contract that was previously parsed.
-	bytes const& compile(bool _optimize = false);
+	/// Compiles the source units that were prevously added and parsed.
+	void compile(bool _optimize = false);
 	/// Parses and compiles the given source code.
+	/// @returns the compiled bytecode
 	bytes const& compile(std::string const& _sourceCode, bool _optimize = false);
 
-	bytes const& getBytecode() const { return m_bytecode; }
+	bytes const& getBytecode(std::string const& _contractName = "");
 	/// Streams a verbose version of the assembly to @a _outStream.
 	/// Prerequisite: Successful compilation.
-	void streamAssembly(std::ostream& _outStream);
+	void streamAssembly(std::ostream& _outStream, std::string const& _contractName = "");
 
 	/// Returns a string representing the contract interface in JSON.
 	/// Prerequisite: Successful call to parse or compile.
-	std::string const& getInterface();
+	std::string const& getInterface(std::string const& _contractName = "");
 
 	/// Returns the previously used scanner, useful for counting lines during error reporting.
-	Scanner const& getScanner() const { return *m_scanner; }
-	SourceUnit& getAST() const { return *m_sourceUnitASTNode; }
+	Scanner const& getScanner(std::string const& _sourceName = "");
+	SourceUnit& getAST(std::string const& _sourceName = "");
 
 	/// Compile the given @a _sourceCode to bytecode. If a scanner is provided, it is used for
 	/// scanning the source code - this is useful for printing exception information.
 	static bytes staticCompile(std::string const& _sourceCode, bool _optimize = false);
 
 private:
-	std::shared_ptr<Scanner> m_scanner;
-	std::shared_ptr<GlobalContext> m_globalContext;
-	std::shared_ptr<SourceUnit> m_sourceUnitASTNode;
+	/**
+	 * Information pertaining to one source unit, filled gradually during parsing and compilation.
+	 */
+	struct Source
+	{
+		std::shared_ptr<Scanner> scanner;
+		std::shared_ptr<SourceUnit> ast;
+		std::string interface;
+		void reset() { scanner.reset(); ast.reset(); interface.clear(); }
+	};
+
+	struct Contract
+	{
+		ContractDefinition const* contract;
+		std::string interface;
+		std::shared_ptr<Compiler> compiler;
+		bytes bytecode;
+	};
+
+	void reset(bool _keepSources = false);
+	void resolveImports();
+
+	Contract& getContract(std::string const& _contractName = "");
+	Source& getSource(std::string const& _sourceName = "");
+
 	bool m_parseSuccessful;
-	std::string m_interface;
+	std::map<std::string, Source> m_sources;
+	std::shared_ptr<GlobalContext> m_globalContext;
 	std::shared_ptr<Compiler> m_compiler;
-	bytes m_bytecode;
+	std::vector<Source const*> m_sourceOrder;
+	std::map<std::string, Contract> m_contracts;
 };
 
 }

DeclarationContainer.cpp

@@ -28,9 +28,9 @@ namespace dev
 namespace solidity
 {
 
-bool DeclarationContainer::registerDeclaration(Declaration& _declaration)
+bool DeclarationContainer::registerDeclaration(Declaration& _declaration, bool _update)
 {
-	if (m_declarations.find(_declaration.getName()) != m_declarations.end())
+	if (!_update && m_declarations.find(_declaration.getName()) != m_declarations.end())
 		return false;
 	m_declarations[_declaration.getName()] = &_declaration;
 	return true;

DeclarationContainer.h

@@ -43,7 +43,7 @@ public:
 		m_enclosingDeclaration(_enclosingDeclaration), m_enclosingContainer(_enclosingContainer) {}
 	/// Registers the declaration in the scope unless its name is already declared. Returns true iff
 	/// it was not yet declared.
-	bool registerDeclaration(Declaration& _declaration);
+	bool registerDeclaration(Declaration& _declaration, bool _update = false);
 	Declaration* resolveName(ASTString const& _name, bool _recursive = false) const;
 	Declaration* getEnclosingDeclaration() const { return m_enclosingDeclaration; }
 

GlobalContext.cpp

@@ -74,11 +74,10 @@ vector<Declaration*> GlobalContext::getDeclarations() const
 	declarations.reserve(m_magicVariables.size() + 1);
 	for (ASTPointer<Declaration> const& variable: m_magicVariables)
 		declarations.push_back(variable.get());
-	declarations.push_back(getCurrentThis());
 	return declarations;
 }
 
-MagicVariableDeclaration*GlobalContext::getCurrentThis() const
+MagicVariableDeclaration* GlobalContext::getCurrentThis() const
 {
 	if (!m_thisPointer[m_currentContract])
 		m_thisPointer[m_currentContract] = make_shared<MagicVariableDeclaration>(

GlobalContext.h

@@ -47,12 +47,13 @@ class GlobalContext: private boost::noncopyable
 public:
 	GlobalContext();
 	void setCurrentContract(ContractDefinition const& _contract);
+	MagicVariableDeclaration* getCurrentThis() const;
 
 	std::vector<MagicVariableDeclaration const*> getMagicVariables() const;
+	/// Returns a vector of all magic variables, excluding "this".
 	std::vector<Declaration*> getDeclarations() const;
 
 private:
-	MagicVariableDeclaration* getCurrentThis() const;
 	std::vector<std::shared_ptr<MagicVariableDeclaration>> m_magicVariables;
 	ContractDefinition const* m_currentContract;
 	std::map<ContractDefinition const*, std::shared_ptr<MagicVariableDeclaration>> mutable m_thisPointer;

NameAndTypeResolver.cpp

@@ -38,9 +38,13 @@ NameAndTypeResolver::NameAndTypeResolver(std::vector<Declaration*> const& _globa
 		m_scopes[nullptr].registerDeclaration(*declaration);
 }
 
+void NameAndTypeResolver::registerDeclarations(SourceUnit& _sourceUnit)
+{
+	DeclarationRegistrationHelper registrar(m_scopes, _sourceUnit);
+}
+
 void NameAndTypeResolver::resolveNamesAndTypes(ContractDefinition& _contract)
 {
-	DeclarationRegistrationHelper registrar(m_scopes, _contract);
 	m_currentScope = &m_scopes[&_contract];
 	for (ASTPointer<StructDefinition> const& structDef: _contract.getDefinedStructs())
 		ReferencesResolver resolver(*structDef, *this, nullptr);
@@ -65,6 +69,12 @@ void NameAndTypeResolver::resolveNamesAndTypes(ContractDefinition& _contract)
 	m_currentScope = &m_scopes[nullptr];
 }
 
+void NameAndTypeResolver::updateDeclaration(Declaration& _declaration)
+{
+	m_scopes[nullptr].registerDeclaration(_declaration, true);
+	_declaration.setScope(nullptr);
+}
+
 Declaration* NameAndTypeResolver::resolveName(ASTString const& _name, Declaration const* _scope) const
 {
 	auto iterator = m_scopes.find(_scope);

NameAndTypeResolver.h

@@ -42,7 +42,13 @@ class NameAndTypeResolver: private boost::noncopyable
 {
 public:
 	explicit NameAndTypeResolver(std::vector<Declaration*> const& _globals);
+	/// Registers all declarations found in the source unit.
+	void registerDeclarations(SourceUnit& _sourceUnit);
+	/// Resolves all names and types referenced from the given contract.
 	void resolveNamesAndTypes(ContractDefinition& _contract);
+	/// Updates the given global declaration (used for "this"). Not to be used with declarations
+	/// that create their own scope.
+	void updateDeclaration(Declaration& _declaration);
 
 	/// Resolves the given @a _name inside the scope @a _scope. If @a _scope is omitted,
 	/// the global scope is used (i.e. the one containing only the contract).

Scanner.cpp

@@ -143,17 +143,28 @@ private:
 }; // end of LiteralScope class
 
 
-void Scanner::reset(CharStream const& _source)
+void Scanner::reset(CharStream const& _source, std::string const& _sourceName)
 {
 	m_source = _source;
+
+	shared_ptr<string> sourceName = make_shared<string>(_sourceName);
+	m_currentToken.location.sourceName = sourceName;
+	m_nextToken.location.sourceName = sourceName;
+	m_skippedComment.location.sourceName = sourceName;
+	m_nextSkippedComment.location.sourceName = sourceName;
+
+	reset();
+}
+
+void Scanner::reset()
+{
+	m_source.reset();
 	m_char = m_source.get();
 	skipWhitespace();
 	scanToken();
-
 	next();
 }
 
-
 bool Scanner::scanHexByte(char& o_scannedByte)
 {
 	char x = 0;

Scanner.h

@@ -79,6 +79,8 @@ public:
 	char advanceAndGet(size_t _chars=1);
 	char rollback(size_t _amount);
 
+	void reset() { m_pos = 0; }
+
 	///@{
 	///@name Error printing helper functions
 	/// Functions that help pretty-printing parse errors
@@ -99,11 +101,12 @@ class Scanner
 	friend class LiteralScope;
 public:
 
-	Scanner() { reset(CharStream()); }
-	explicit Scanner(CharStream const& _source) { reset(_source); }
+	explicit Scanner(CharStream const& _source = CharStream(), std::string const& _sourceName = "") { reset(_source, _sourceName); }
 
-	/// Resets the scanner as if newly constructed with _input as input.
-	void reset(CharStream const& _source);
+	/// Resets the scanner as if newly constructed with _source and _sourceName as input.
+	void reset(CharStream const& _source, std::string const& _sourceName);
+	/// Resets scanner to the start of input.
+	void reset();
 
 	/// Returns the next token and advances input
 	Token::Value next();