Merge pull request #9862 from ethereum/develop

Merge develop into breaking

libsolidity/ast/AST.cpp

@@ -492,7 +492,7 @@ CallableDeclaration const* Scopable::functionOrModifierDefinition() const
 
 string Scopable::sourceUnitName() const
 {
-	return sourceUnit().annotation().path;
+	return *sourceUnit().annotation().path;
 }
 
 DeclarationAnnotation& Declaration::annotation() const

libsolidity/ast/ASTAnnotations.h

@@ -47,6 +47,7 @@ namespace solidity::frontend
 
 class Type;
 using TypePointer = Type const*;
+using namespace util;
 
 struct ASTAnnotation
 {
@@ -88,9 +89,9 @@ struct StructurallyDocumentedAnnotation
 struct SourceUnitAnnotation: ASTAnnotation
 {
 	/// The "absolute" (in the compiler sense) path of this source unit.
-	std::string path;
+	SetOnce<std::string> path;
 	/// The exported symbols (all global symbols).
-	std::map<ASTString, std::vector<Declaration const*>> exportedSymbols;
+	SetOnce<std::map<ASTString, std::vector<Declaration const*>>> exportedSymbols;
 	/// Experimental features.
 	std::set<ExperimentalFeature> experimentalFeatures;
 };
@@ -122,7 +123,7 @@ struct DeclarationAnnotation: ASTAnnotation, ScopableAnnotation
 struct ImportAnnotation: DeclarationAnnotation
 {
 	/// The absolute path of the source unit to import.
-	std::string absolutePath;
+	SetOnce<std::string> absolutePath;
 	/// The actual source unit.
 	SourceUnit const* sourceUnit = nullptr;
 };
@@ -130,7 +131,7 @@ struct ImportAnnotation: DeclarationAnnotation
 struct TypeDeclarationAnnotation: DeclarationAnnotation
 {
 	/// The name of this type, prefixed by proper namespaces if globally accessible.
-	std::string canonicalName;
+	SetOnce<std::string> canonicalName;
 };
 
 struct StructDeclarationAnnotation: TypeDeclarationAnnotation
@@ -149,7 +150,7 @@ struct StructDeclarationAnnotation: TypeDeclarationAnnotation
 struct ContractDefinitionAnnotation: TypeDeclarationAnnotation, StructurallyDocumentedAnnotation
 {
 	/// List of functions and modifiers without a body. Can also contain functions from base classes.
-	std::vector<Declaration const*> unimplementedDeclarations;
+	std::optional<std::vector<Declaration const*>> unimplementedDeclarations;
 	/// List of all (direct and indirect) base contracts in order from derived to
 	/// base, including the contract itself.
 	std::vector<ContractDefinition const*> linearizedBaseContracts;
@@ -243,16 +244,16 @@ struct ExpressionAnnotation: ASTAnnotation
 	/// Inferred type of the expression.
 	TypePointer type = nullptr;
 	/// Whether the expression is a constant variable
-	bool isConstant = false;
+	SetOnce<bool> isConstant;
 	/// Whether the expression is pure, i.e. compile-time constant.
-	bool isPure = false;
+	SetOnce<bool> isPure;
 	/// Whether it is an LValue (i.e. something that can be assigned to).
-	bool isLValue = false;
+	SetOnce<bool> isLValue;
 	/// Whether the expression is used in a context where the LValue is actually required.
 	bool willBeWrittenTo = false;
 	/// Whether the expression is an lvalue that is only assigned.
 	/// Would be false for --, ++, delete, +=, -=, ....
-	bool lValueOfOrdinaryAssignment = false;
+	SetOnce<bool> lValueOfOrdinaryAssignment;
 
 	/// Types and - if given - names of arguments if the expr. is a function
 	/// that is called, used for overload resolution

libsolidity/ast/ASTJsonConverter.cpp

@@ -39,10 +39,33 @@
 #include <vector>
 #include <algorithm>
 #include <limits>
+#include <type_traits>
 
 using namespace std;
 using namespace solidity::langutil;
 
+namespace
+{
+
+template<typename V, template<typename> typename C>
+void addIfSet(std::vector<pair<string, Json::Value>>& _attributes, string const& _name, C<V> const& _value)
+{
+	if constexpr (std::is_same_v<C<V>, solidity::util::SetOnce<V>>)
+	{
+		if (!_value.set())
+			return;
+	}
+	else if constexpr (std::is_same_v<C<V>, optional<V>>)
+	{
+		if (!_value.has_value())
+			return;
+	}
+
+	_attributes.emplace_back(_name, *_value);
+}
+
+}
+
 namespace solidity::frontend
 {
 
@@ -181,12 +204,14 @@ void ASTJsonConverter::appendExpressionAttributes(
 {
 	std::vector<pair<string, Json::Value>> exprAttributes = {
 		make_pair("typeDescriptions", typePointerToJson(_annotation.type)),
-		make_pair("isConstant", _annotation.isConstant),
-		make_pair("isPure", _annotation.isPure),
-		make_pair("isLValue", _annotation.isLValue),
 		make_pair("lValueRequested", _annotation.willBeWrittenTo),
 		make_pair("argumentTypes", typePointerToJson(_annotation.arguments))
 	};
+
+	addIfSet(exprAttributes, "isLValue", _annotation.isLValue);
+	addIfSet(exprAttributes, "isPure", _annotation.isPure);
+	addIfSet(exprAttributes, "isConstant", _annotation.isConstant);
+
 	_attributes += exprAttributes;
 }
 
@@ -214,23 +239,27 @@ Json::Value ASTJsonConverter::toJson(ASTNode const& _node)
 
 bool ASTJsonConverter::visit(SourceUnit const& _node)
 {
-	Json::Value exportedSymbols = Json::objectValue;
-	for (auto const& sym: _node.annotation().exportedSymbols)
+	std::vector<pair<string, Json::Value>> attributes = {
+		make_pair("license", _node.licenseString() ? Json::Value(*_node.licenseString()) : Json::nullValue),
+		make_pair("nodes", toJson(_node.nodes()))
+	};
+
+	if (_node.annotation().exportedSymbols.set())
 	{
-		exportedSymbols[sym.first] = Json::arrayValue;
-		for (Declaration const* overload: sym.second)
-			exportedSymbols[sym.first].append(nodeId(*overload));
-	}
-	setJsonNode(
-		_node,
-		"SourceUnit",
+		Json::Value exportedSymbols = Json::objectValue;
+		for (auto const& sym: *_node.annotation().exportedSymbols)
 		{
-			make_pair("absolutePath", _node.annotation().path),
-			make_pair("exportedSymbols", move(exportedSymbols)),
-			make_pair("license", _node.licenseString() ? Json::Value(*_node.licenseString()) : Json::nullValue),
-			make_pair("nodes", toJson(_node.nodes()))
+			exportedSymbols[sym.first] = Json::arrayValue;
+			for (Declaration const* overload: sym.second)
+				exportedSymbols[sym.first].append(nodeId(*overload));
 		}
-	);
+
+		attributes.emplace_back("exportedSymbols", exportedSymbols);
+	};
+
+	addIfSet(attributes, "absolutePath", _node.annotation().path);
+
+	setJsonNode(_node, "SourceUnit", std::move(attributes));
 	return false;
 }
 
@@ -249,10 +278,12 @@ bool ASTJsonConverter::visit(ImportDirective const& _node)
 {
 	std::vector<pair<string, Json::Value>> attributes = {
 		make_pair("file", _node.path()),
-		make_pair("absolutePath", _node.annotation().absolutePath),
 		make_pair(m_legacy ? "SourceUnit" : "sourceUnit", nodeId(*_node.annotation().sourceUnit)),
 		make_pair("scope", idOrNull(_node.scope()))
 	};
+
+	addIfSet(attributes, "absolutePath", _node.annotation().absolutePath);
+
 	attributes.emplace_back("unitAlias", _node.name());
 	Json::Value symbolAliases(Json::arrayValue);
 	for (auto const& symbolAlias: _node.symbolAliases())
@@ -270,18 +301,23 @@ bool ASTJsonConverter::visit(ImportDirective const& _node)
 
 bool ASTJsonConverter::visit(ContractDefinition const& _node)
 {
-	setJsonNode(_node, "ContractDefinition", {
+	std::vector<pair<string, Json::Value>> attributes = {
 		make_pair("name", _node.name()),
 		make_pair("documentation", _node.documentation() ? toJson(*_node.documentation()) : Json::nullValue),
 		make_pair("contractKind", contractKind(_node.contractKind())),
 		make_pair("abstract", _node.abstract()),
-		make_pair("fullyImplemented", _node.annotation().unimplementedDeclarations.empty()),
-		make_pair("linearizedBaseContracts", getContainerIds(_node.annotation().linearizedBaseContracts)),
 		make_pair("baseContracts", toJson(_node.baseContracts())),
 		make_pair("contractDependencies", getContainerIds(_node.annotation().contractDependencies, true)),
 		make_pair("nodes", toJson(_node.subNodes())),
 		make_pair("scope", idOrNull(_node.scope()))
-	});
+	};
+
+	if (_node.annotation().unimplementedDeclarations.has_value())
+		attributes.emplace_back("fullyImplemented", _node.annotation().unimplementedDeclarations->empty());
+	if (!_node.annotation().linearizedBaseContracts.empty())
+		attributes.emplace_back("linearizedBaseContracts", getContainerIds(_node.annotation().linearizedBaseContracts));
+
+	setJsonNode(_node, "ContractDefinition", std::move(attributes));
 	return false;
 }
 
@@ -305,23 +341,31 @@ bool ASTJsonConverter::visit(UsingForDirective const& _node)
 
 bool ASTJsonConverter::visit(StructDefinition const& _node)
 {
-	setJsonNode(_node, "StructDefinition", {
+	std::vector<pair<string, Json::Value>> attributes = {
 		make_pair("name", _node.name()),
 		make_pair("visibility", Declaration::visibilityToString(_node.visibility())),
-		make_pair("canonicalName", _node.annotation().canonicalName),
 		make_pair("members", toJson(_node.members())),
 		make_pair("scope", idOrNull(_node.scope()))
-	});
+	};
+
+	addIfSet(attributes,"canonicalName", _node.annotation().canonicalName);
+
+	setJsonNode(_node, "StructDefinition", std::move(attributes));
+
 	return false;
 }
 
 bool ASTJsonConverter::visit(EnumDefinition const& _node)
 {
-	setJsonNode(_node, "EnumDefinition", {
+	std::vector<pair<string, Json::Value>> attributes = {
 		make_pair("name", _node.name()),
-		make_pair("canonicalName", _node.annotation().canonicalName),
 		make_pair("members", toJson(_node.members()))
-	});
+	};
+
+	addIfSet(attributes,"canonicalName", _node.annotation().canonicalName);
+
+	setJsonNode(_node, "EnumDefinition", std::move(attributes));
+
 	return false;
 }
 

libsolidity/ast/Types.cpp

@@ -44,6 +44,7 @@
 #include <boost/range/algorithm/copy.hpp>
 
 #include <limits>
+#include <unordered_set>
 #include <utility>
 
 using namespace std;
@@ -54,57 +55,6 @@ using namespace solidity::frontend;
 namespace
 {
 
-struct TypeComp
-{
-	bool operator()(Type const* lhs, Type const* rhs) const
-	{
-		solAssert(lhs && rhs, "");
-		return lhs->richIdentifier() < rhs->richIdentifier();
-	}
-};
-using TypeSet = std::set<Type const*, TypeComp>;
-
-void oversizedSubtypesInner(
-	Type const& _type,
-	bool _includeType,
-	set<StructDefinition const*>& _structsSeen,
-	TypeSet& _oversizedSubtypes
-)
-{
-	switch (_type.category())
-	{
-	case Type::Category::Array:
-	{
-		auto const& t = dynamic_cast<ArrayType const&>(_type);
-		if (_includeType && t.storageSizeUpperBound() >= bigint(1) << 64)
-			_oversizedSubtypes.insert(&t);
-		oversizedSubtypesInner(*t.baseType(), t.isDynamicallySized(), _structsSeen, _oversizedSubtypes);
-		break;
-	}
-	case Type::Category::Struct:
-	{
-		auto const& t = dynamic_cast<StructType const&>(_type);
-		if (_structsSeen.count(&t.structDefinition()))
-			return;
-		if (_includeType && t.storageSizeUpperBound() >= bigint(1) << 64)
-			_oversizedSubtypes.insert(&t);
-		_structsSeen.insert(&t.structDefinition());
-		for (auto const& m: t.members(nullptr))
-			oversizedSubtypesInner(*m.type, false, _structsSeen, _oversizedSubtypes);
-		_structsSeen.erase(&t.structDefinition());
-		break;
-	}
-	case Type::Category::Mapping:
-	{
-		auto const* valueType = dynamic_cast<MappingType const&>(_type).valueType();
-		oversizedSubtypesInner(*valueType, true, _structsSeen, _oversizedSubtypes);
-		break;
-	}
-	default:
-		break;
-	}
-}
-
 /// Check whether (_base ** _exp) fits into 4096 bits.
 bool fitsPrecisionExp(bigint const& _base, bigint const& _exp)
 {
@@ -201,16 +151,6 @@ util::Result<TypePointers> transformParametersToExternal(TypePointers const& _pa
 
 }
 
-vector<frontend::Type const*> solidity::frontend::oversizedSubtypes(frontend::Type const& _type)
-{
-	set<StructDefinition const*> structsSeen;
-	TypeSet oversized;
-	oversizedSubtypesInner(_type, true, structsSeen, oversized);
-	vector<frontend::Type const*> res;
-	copy(oversized.cbegin(), oversized.cend(), back_inserter(res));
-	return res;
-}
-
 void Type::clearCache() const
 {
 	m_members.clear();
@@ -404,10 +344,16 @@ TypePointer Type::fullEncodingType(bool _inLibraryCall, bool _encoderV2, bool) c
 		return encodingType;
 	TypePointer baseType = encodingType;
 	while (auto const* arrayType = dynamic_cast<ArrayType const*>(baseType))
+	{
 		baseType = arrayType->baseType();
-	if (dynamic_cast<StructType const*>(baseType))
-		if (!_encoderV2)
+
+		auto const* baseArrayType = dynamic_cast<ArrayType const*>(baseType);
+		if (!_encoderV2 && baseArrayType && baseArrayType->isDynamicallySized())
 			return nullptr;
+	}
+	if (!_encoderV2 && dynamic_cast<StructType const*>(baseType))
+		return nullptr;
+
 	return encodingType;
 }
 
@@ -1613,6 +1559,21 @@ TypeResult ContractType::unaryOperatorResult(Token _operator) const
 		return nullptr;
 }
 
+vector<Type const*> CompositeType::fullDecomposition() const
+{
+	vector<Type const*> res = {this};
+	unordered_set<string> seen = {richIdentifier()};
+	for (size_t k = 0; k < res.size(); ++k)
+		if (auto composite = dynamic_cast<CompositeType const*>(res[k]))
+			for (Type const* next: composite->decomposition())
+				if (seen.count(next->richIdentifier()) == 0)
+				{
+					seen.insert(next->richIdentifier());
+					res.push_back(next);
+				}
+	return res;
+}
+
 Type const* ReferenceType::withLocation(DataLocation _location, bool _isPointer) const
 {
 	return TypeProvider::withLocation(this, _location, _isPointer);
@@ -2155,7 +2116,7 @@ string ContractType::toString(bool) const
 
 string ContractType::canonicalName() const
 {
-	return m_contract.annotation().canonicalName;
+	return *m_contract.annotation().canonicalName;
 }
 
 MemberList::MemberMap ContractType::nativeMembers(ASTNode const*) const
@@ -2406,7 +2367,7 @@ bool StructType::containsNestedMapping() const
 
 string StructType::toString(bool _short) const
 {
-	string ret = "struct " + m_struct.annotation().canonicalName;
+	string ret = "struct " + *m_struct.annotation().canonicalName;
 	if (!_short)
 		ret += " " + stringForReferencePart();
 	return ret;
@@ -2585,7 +2546,7 @@ string StructType::signatureInExternalFunction(bool _structsByName) const
 
 string StructType::canonicalName() const
 {
-	return m_struct.annotation().canonicalName;
+	return *m_struct.annotation().canonicalName;
 }
 
 FunctionTypePointer StructType::constructorType() const
@@ -2650,6 +2611,13 @@ vector<tuple<string, TypePointer>> StructType::makeStackItems() const
 	solAssert(false, "");
 }
 
+vector<Type const*> StructType::decomposition() const
+{
+	vector<Type const*> res;
+	for (MemberList::Member const& member: members(nullptr))
+		res.push_back(member.type);
+	return res;
+}
 
 TypePointer EnumType::encodingType() const
 {
@@ -2685,12 +2653,12 @@ unsigned EnumType::storageBytes() const
 
 string EnumType::toString(bool) const
 {
-	return string("enum ") + m_enum.annotation().canonicalName;
+	return string("enum ") + *m_enum.annotation().canonicalName;
 }
 
 string EnumType::canonicalName() const
 {
-	return m_enum.annotation().canonicalName;
+	return *m_enum.annotation().canonicalName;
 }
 
 size_t EnumType::numberOfMembers() const
@@ -3163,7 +3131,7 @@ string FunctionType::toString(bool _short) const
 		auto const* functionDefinition = dynamic_cast<FunctionDefinition const*>(m_declaration);
 		solAssert(functionDefinition, "");
 		if (auto const* contract = dynamic_cast<ContractDefinition const*>(functionDefinition->scope()))
-			name += contract->annotation().canonicalName + ".";
+			name += *contract->annotation().canonicalName + ".";
 		name += functionDefinition->name();
 	}
 	name += '(';
@@ -3954,7 +3922,7 @@ bool ModuleType::operator==(Type const& _other) const
 MemberList::MemberMap ModuleType::nativeMembers(ASTNode const*) const
 {
 	MemberList::MemberMap symbols;
-	for (auto const& symbolName: m_sourceUnit.annotation().exportedSymbols)
+	for (auto const& symbolName: *m_sourceUnit.annotation().exportedSymbols)
 		for (Declaration const* symbol: symbolName.second)
 			symbols.emplace_back(symbolName.first, symbol->type(), symbol);
 	return symbols;
@@ -3962,7 +3930,7 @@ MemberList::MemberMap ModuleType::nativeMembers(ASTNode const*) const
 
 string ModuleType::toString(bool) const
 {
-	return string("module \"") + m_sourceUnit.annotation().path + string("\"");
+	return string("module \"") + *m_sourceUnit.annotation().path + string("\"");
 }
 
 string MagicType::richIdentifier() const

libsolidity/ast/Types.h

@@ -60,8 +60,6 @@ using BoolResult = util::Result<bool>;
 namespace solidity::frontend
 {
 
-std::vector<frontend::Type const*> oversizedSubtypes(frontend::Type const& _type);
-
 inline rational makeRational(bigint const& _numerator, bigint const& _denominator)
 {
 	solAssert(_denominator != 0, "division by zero");
@@ -694,11 +692,37 @@ public:
 	TypeResult interfaceType(bool) const override { return this; }
 };
 
+/**
+ * Base class for types which can be thought of as several elements of other types put together.
+ * For example a struct is composed of its members, an array is composed of multiple copies of its
+ * base element and a mapping is composed of its value type elements (note that keys are not
+ * stored anywhere).
+ */
+class CompositeType: public Type
+{
+protected:
+	CompositeType() = default;
+
+public:
+	/// @returns a list containing the type itself, elements of its decomposition,
+	/// elements of decomposition of these elements and so on, up to non-composite types.
+	/// Each type is included only once.
+	std::vector<Type const*> fullDecomposition() const;
+
+protected:
+	/// @returns a list of types that together make up the data part of this type.
+	/// Contains all types that will have to be implicitly stored, whenever an object of this type is stored.
+	/// In particular, it returns the base type for arrays and array slices, the member types for structs,
+	/// the component types for tuples and the value type for mappings
+	/// (note that the key type of a mapping is *not* part of the list).
+	virtual std::vector<Type const*> decomposition() const = 0;
+};
+
 /**
  * Base class used by types which are not value types and can be stored either in storage, memory
  * or calldata. This is currently used by arrays and structs.
  */
-class ReferenceType: public Type
+class ReferenceType: public CompositeType
 {
 protected:
 	explicit ReferenceType(DataLocation _location): m_location(_location) {}
@@ -829,6 +853,8 @@ public:
 
 protected:
 	std::vector<std::tuple<std::string, TypePointer>> makeStackItems() const override;
+	std::vector<Type const*> decomposition() const override { return {m_baseType}; }
+
 private:
 	/// String is interpreted as a subtype of Bytes.
 	enum class ArrayKind { Ordinary, Bytes, String };
@@ -869,6 +895,8 @@ public:
 
 protected:
 	std::vector<std::tuple<std::string, TypePointer>> makeStackItems() const override;
+	std::vector<Type const*> decomposition() const override { return {m_arrayType.baseType()}; }
+
 private:
 	ArrayType const& m_arrayType;
 };
@@ -994,6 +1022,8 @@ public:
 
 protected:
 	std::vector<std::tuple<std::string, TypePointer>> makeStackItems() const override;
+	std::vector<Type const*> decomposition() const override;
+
 private:
 	StructDefinition const& m_struct;
 	// Caches for interfaceType(bool)
@@ -1044,7 +1074,7 @@ private:
  * Type that can hold a finite sequence of values of different types.
  * In some cases, the components are empty pointers (when used as placeholders).
  */
-class TupleType: public Type
+class TupleType: public CompositeType
 {
 public:
 	explicit TupleType(std::vector<TypePointer> _types = {}): m_components(std::move(_types)) {}
@@ -1067,6 +1097,16 @@ public:
 
 protected:
 	std::vector<std::tuple<std::string, TypePointer>> makeStackItems() const override;
+	std::vector<Type const*> decomposition() const override
+	{
+		// Currently calling TupleType::decomposition() is not expected, because we cannot declare a variable of a tuple type.
+		// If that changes, before removing the solAssert, make sure the function does the right thing and is used properly.
+		// Note that different tuple members can have different data locations, so using decomposition() to check
+		// the tuple validity for a data location might require special care.
+		solUnimplemented("Tuple decomposition is not expected.");
+		return m_components;
+	}
+
 private:
 	std::vector<TypePointer> const m_components;
 };
@@ -1349,7 +1389,7 @@ private:
  * The type of a mapping, there is one distinct type per key/value type pair.
  * Mappings always occupy their own storage slot, but do not actually use it.
  */
-class MappingType: public Type
+class MappingType: public CompositeType
 {
 public:
 	MappingType(Type const* _keyType, Type const* _valueType):
@@ -1373,6 +1413,9 @@ public:
 	Type const* keyType() const { return m_keyType; }
 	Type const* valueType() const { return m_valueType; }
 
+protected:
+	std::vector<Type const*> decomposition() const override { return {m_valueType}; }
+
 private:
 	TypePointer m_keyType;
 	TypePointer m_valueType;