Merge pull request #13265 from ethereum/no-append-metadata

Add `--no-append-metadata` in CLI and `metadata.append` in JSON

Changelog.md

@@ -4,6 +4,8 @@ Language Features:
 
 
 Compiler Features:
+* Commandline Interface: Add `--no-cbor-metadata` that skips CBOR metadata from getting appended at the end of the bytecode.
+* Standard JSON: Add a boolean field `settings.metadata.appendCBOR` that skips CBOR metadata from getting appended at the end of the bytecode.
 * Yul Optimizer: Allow replacing the previously hard-coded cleanup sequence by specifying custom steps after a colon delimiter (``:``) in the sequence string.
 
 

docs/metadata.rst

@@ -220,6 +220,10 @@ Whereas release builds of solc use a 3 byte encoding of the version as shown
 above (one byte each for major, minor and patch version number), prerelease builds
 will instead use a complete version string including commit hash and build date.
 
+The commandline flag ``--no-cbor-metadata`` can be used to skip metadata
+from getting appended at the end of the deployed bytecode. Equivalently, the
+boolean field ``settings.metadata.appendCBOR`` in Standard JSON input can be set to false.
+
 .. note::
   The CBOR mapping can also contain other keys, so it is better to fully
   decode the data instead of relying on it starting with ``0xa264``.

docs/using-the-compiler.rst

@@ -323,6 +323,9 @@ Input Description
         },
         // Metadata settings (optional)
         "metadata": {
+          // The CBOR metadata is appended at the end of the bytecode by default.
+          // Setting this to false omits the metadata from the runtime and deploy time code.
+          "appendCBOR": true,
           // Use only literal content and not URLs (false by default)
           "useLiteralContent": true,
           // Use the given hash method for the metadata hash that is appended to the bytecode.

libsolidity/interface/CompilerStack.cpp

@@ -308,6 +308,7 @@ void CompilerStack::reset(bool _keepSettings)
 		m_revertStrings = RevertStrings::Default;
 		m_optimiserSettings = OptimiserSettings::minimal();
 		m_metadataLiteralSources = false;
+		m_metadataFormat = defaultMetadataFormat();
 		m_metadataHash = MetadataHash::IPFS;
 		m_stopAfter = State::CompilationSuccessful;
 	}
@@ -1548,6 +1549,9 @@ string CompilerStack::createMetadata(Contract const& _contract, bool _forIR) con
 	if (m_revertStrings != RevertStrings::Default)
 		meta["settings"]["debug"]["revertStrings"] = revertStringsToString(m_revertStrings);
 
+	if (m_metadataFormat == MetadataFormat::NoMetadata)
+		meta["settings"]["metadata"]["appendCBOR"] = false;
+
 	if (m_metadataLiteralSources)
 		meta["settings"]["metadata"]["useLiteralContent"] = true;
 

libsolidity/interface/CompilerStack.h

@@ -345,10 +345,13 @@ public:
 	Json::Value gasEstimates(std::string const& _contractName) const;
 
 	/// Changes the format of the metadata appended at the end of the bytecode.
-	/// This is mostly a workaround to avoid bytecode and gas differences between compiler builds
-	/// caused by differences in metadata. Should only be used for testing.
 	void setMetadataFormat(MetadataFormat _metadataFormat) { m_metadataFormat = _metadataFormat; }
 
+	static MetadataFormat defaultMetadataFormat()
+	{
+		return VersionIsRelease ? MetadataFormat::WithReleaseVersionTag : MetadataFormat::WithPrereleaseVersionTag;
+	}
+
 private:
 	/// The state per source unit. Filled gradually during parsing.
 	struct Source
@@ -515,7 +518,7 @@ private:
 	/// Whether or not there has been an error during processing.
 	/// If this is true, the stack will refuse to generate code.
 	bool m_hasError = false;
-	MetadataFormat m_metadataFormat = VersionIsRelease ? MetadataFormat::WithReleaseVersionTag : MetadataFormat::WithPrereleaseVersionTag;
+	MetadataFormat m_metadataFormat = defaultMetadataFormat();
 };
 
 }

libsolidity/interface/StandardCompiler.cpp

@@ -509,6 +509,8 @@ std::optional<Json::Value> checkMetadataKeys(Json::Value const& _input)
 {
 	if (_input.isObject())
 	{
+		if (_input.isMember("appendCBOR") && !_input["appendCBOR"].isBool())
+			return formatFatalError(Error::Type::JSONError, "\"settings.metadata.appendCBOR\" must be Boolean");
 		if (_input.isMember("useLiteralContent") && !_input["useLiteralContent"].isBool())
 			return formatFatalError(Error::Type::JSONError, "\"settings.metadata.useLiteralContent\" must be Boolean");
 
@@ -516,7 +518,7 @@ std::optional<Json::Value> checkMetadataKeys(Json::Value const& _input)
 		if (_input.isMember("bytecodeHash") && !hashes.count(_input["bytecodeHash"].asString()))
 			return formatFatalError(Error::Type::JSONError, "\"settings.metadata.bytecodeHash\" must be \"ipfs\", \"bzzr1\" or \"none\"");
 	}
-	static set<string> keys{"useLiteralContent", "bytecodeHash"};
+	static set<string> keys{"appendCBOR", "useLiteralContent", "bytecodeHash"};
 	return checkKeys(_input, keys, "settings.metadata");
 }
 
@@ -911,6 +913,12 @@ std::variant<StandardCompiler::InputsAndSettings, Json::Value> StandardCompiler:
 	if (auto result = checkMetadataKeys(metadataSettings))
 		return *result;
 
+	solAssert(CompilerStack::defaultMetadataFormat() != CompilerStack::MetadataFormat::NoMetadata, "");
+	ret.metadataFormat =
+		metadataSettings.get("appendCBOR", Json::Value(true)).asBool() ?
+		CompilerStack::defaultMetadataFormat() :
+		CompilerStack::MetadataFormat::NoMetadata;
+
 	ret.metadataLiteralSources = metadataSettings.get("useLiteralContent", Json::Value(false)).asBool();
 	if (metadataSettings.isMember("bytecodeHash"))
 	{
@@ -1086,6 +1094,7 @@ Json::Value StandardCompiler::compileSolidity(StandardCompiler::InputsAndSetting
 		compilerStack.selectDebugInfo(_inputsAndSettings.debugInfoSelection.value());
 	compilerStack.setLibraries(_inputsAndSettings.libraries);
 	compilerStack.useMetadataLiteralSources(_inputsAndSettings.metadataLiteralSources);
+	compilerStack.setMetadataFormat(_inputsAndSettings.metadataFormat);
 	compilerStack.setMetadataHash(_inputsAndSettings.metadataHash);
 	compilerStack.setRequestedContractNames(requestedContractNames(_inputsAndSettings.outputSelection));
 	compilerStack.setModelCheckerSettings(_inputsAndSettings.modelCheckerSettings);

libsolidity/interface/StandardCompiler.h

@@ -83,6 +83,7 @@ private:
 		std::optional<langutil::DebugInfoSelection> debugInfoSelection;
 		std::map<std::string, util::h160> libraries;
 		bool metadataLiteralSources = false;
+		CompilerStack::MetadataFormat metadataFormat = CompilerStack::defaultMetadataFormat();
 		CompilerStack::MetadataHash metadataHash = CompilerStack::MetadataHash::IPFS;
 		Json::Value outputSelection;
 		ModelCheckerSettings modelCheckerSettings = ModelCheckerSettings{};

solc/CommandLineInterface.cpp

@@ -688,6 +688,7 @@ void CommandLineInterface::compile()
 	{
 		if (m_options.metadata.literalSources)
 			m_compiler->useMetadataLiteralSources(true);
+		m_compiler->setMetadataFormat(m_options.metadata.format);
 		m_compiler->setMetadataHash(m_options.metadata.hash);
 		if (m_options.modelChecker.initialize)
 			m_compiler->setModelCheckerSettings(m_options.modelChecker.settings);

solc/CommandLineParser.cpp

@@ -62,6 +62,7 @@ static string const g_strLibraries = "libraries";
 static string const g_strLink = "link";
 static string const g_strLSP = "lsp";
 static string const g_strMachine = "machine";
+static string const g_strNoCBORMetadata = "no-cbor-metadata";
 static string const g_strMetadataHash = "metadata-hash";
 static string const g_strMetadataLiteral = "metadata-literal";
 static string const g_strModelCheckerContracts = "model-checker-contracts";
@@ -240,6 +241,7 @@ bool CommandLineOptions::operator==(CommandLineOptions const& _other) const noex
 		compiler.outputs == _other.compiler.outputs &&
 		compiler.estimateGas == _other.compiler.estimateGas &&
 		compiler.combinedJsonRequests == _other.compiler.combinedJsonRequests &&
+		metadata.format == _other.metadata.format &&
 		metadata.hash == _other.metadata.hash &&
 		metadata.literalSources == _other.metadata.literalSources &&
 		optimizer.enabled == _other.optimizer.enabled &&
@@ -748,6 +750,10 @@ General Information)").c_str(),
 
 	po::options_description metadataOptions("Metadata Options");
 	metadataOptions.add_options()
+		(
+			g_strNoCBORMetadata.c_str(),
+			"Do not append CBOR metadata to the end of the bytecode."
+		)
 		(
 			g_strMetadataHash.c_str(),
 			po::value<string>()->value_name(util::joinHumanReadable(g_metadataHashArgs, ",")),
@@ -922,6 +928,7 @@ void CommandLineParser::processArgs()
 		{g_strExperimentalViaIR, {InputMode::Compiler, InputMode::CompilerWithASTImport}},
 		{g_strViaIR, {InputMode::Compiler, InputMode::CompilerWithASTImport}},
 		{g_strMetadataLiteral, {InputMode::Compiler, InputMode::CompilerWithASTImport}},
+		{g_strNoCBORMetadata, {InputMode::Compiler, InputMode::CompilerWithASTImport}},
 		{g_strMetadataHash, {InputMode::Compiler, InputMode::CompilerWithASTImport}},
 		{g_strModelCheckerShowUnproved, {InputMode::Compiler, InputMode::CompilerWithASTImport}},
 		{g_strModelCheckerDivModNoSlacks, {InputMode::Compiler, InputMode::CompilerWithASTImport}},
@@ -1221,6 +1228,21 @@ void CommandLineParser::processArgs()
 			solThrow(CommandLineValidationError, "Invalid option for --" + g_strMetadataHash + ": " + hashStr);
 	}
 
+	if (m_args.count(g_strNoCBORMetadata))
+	{
+		if (
+			m_args.count(g_strMetadataHash) &&
+			m_options.metadata.hash != CompilerStack::MetadataHash::None
+		)
+			solThrow(
+				CommandLineValidationError,
+				"Cannot specify a metadata hashing method when --" +
+				g_strNoCBORMetadata + " is set."
+			);
+
+		m_options.metadata.format = CompilerStack::MetadataFormat::NoMetadata;
+	}
+
 	if (m_args.count(g_strModelCheckerContracts))
 	{
 		string contractsStr = m_args[g_strModelCheckerContracts].as<string>();

solc/CommandLineParser.h

@@ -215,6 +215,7 @@ struct CommandLineOptions
 
 	struct
 	{
+		CompilerStack::MetadataFormat format = CompilerStack::defaultMetadataFormat();
 		CompilerStack::MetadataHash hash = CompilerStack::MetadataHash::IPFS;
 		bool literalSources = false;
 	} metadata;

test/cmdlineTests/metadata_append_standard_json/args

@@ -0,0 +1 @@
+--pretty-json --json-indent 4

test/cmdlineTests/metadata_append_standard_json/input.json

@@ -0,0 +1,28 @@
+{
+	"language": "Solidity",
+	"sources":
+	{
+		"A":
+		{
+			"content": "// SPDX-License-Identifier: GPL-3.0\npragma solidity >=0.0;\n\ncontract test {}"
+		}
+	},
+	"settings":
+	{
+		"viaIR": true,
+		"optimizer": {
+			"enabled": true
+		},
+		"metadata":
+		{
+			"appendCBOR": false
+		},
+		"outputSelection":
+		{
+			"A":
+			{
+				"test": ["evm.bytecode"]
+			}
+		}
+	}
+}

test/cmdlineTests/metadata_append_standard_json/output.json

@@ -0,0 +1,30 @@
+{
+    "contracts":
+    {
+        "A":
+        {
+            "test":
+            {
+                "evm":
+                {
+                    "bytecode":
+                    {
+                        "functionDebugData": {},
+                        "generatedSources": [],
+                        "linkReferences": {},
+                        "object": "608080604052346013576004908160198239f35b600080fdfe600080fd",
+                        "opcodes": "PUSH1 0x80 DUP1 PUSH1 0x40 MSTORE CALLVALUE PUSH1 0x13 JUMPI PUSH1 0x4 SWAP1 DUP2 PUSH1 0x19 DUP3 CODECOPY RETURN JUMPDEST PUSH1 0x0 DUP1 REVERT INVALID PUSH1 0x0 DUP1 REVERT ",
+                        "sourceMap": "60:16:0:-:0;;;;;;;;;;;;;;;;;"
+                    }
+                }
+            }
+        }
+    },
+    "sources":
+    {
+        "A":
+        {
+            "id": 0
+        }
+    }
+}

test/cmdlineTests/no_append_incorrect_combination/args

@@ -0,0 +1 @@
+--no-cbor-metadata --metadata-hash ipfs

test/cmdlineTests/no_append_incorrect_combination/err

@@ -0,0 +1 @@
+Cannot specify a metadata hashing method when --no-cbor-metadata is set.

test/cmdlineTests/no_append_incorrect_combination/exit

@@ -0,0 +1 @@
+1

test/cmdlineTests/no_append_incorrect_combination/input.sol

@@ -0,0 +1,10 @@
+// SPDX-License-Identifier: GPL-3.0
+pragma solidity >= 0.0.0;
+
+contract Empty {}
+
+contract C {
+    function f() external returns (bytes memory, bytes memory){
+        return (type(Empty).creationCode, type(Empty).runtimeCode);
+    }
+}

test/cmdlineTests/no_append_metadata/args

@@ -0,0 +1 @@
+--no-cbor-metadata --bin --via-ir --optimize

test/cmdlineTests/no_append_metadata/input.sol

@@ -0,0 +1,10 @@
+// SPDX-License-Identifier: GPL-3.0
+pragma solidity >= 0.0.0;
+
+// NOTE: The expected output would be the binary with no metadata.
+// The commandline tests would replace bytecode with metadata in it with
+// <BYTECODE REMOVED>
+// Therefore, not having that means success!
+
+contract C {
+}

test/cmdlineTests/no_append_metadata/output

@@ -0,0 +1,4 @@
+
+======= no_append_metadata/input.sol:C =======
+Binary:
+608080604052346013576004908160198239f35b600080fdfe600080fd

test/libsolidity/GasCosts.cpp

@@ -92,6 +92,7 @@ BOOST_AUTO_TEST_CASE(string_storage)
 		}
 	)";
 	m_compiler.setMetadataFormat(CompilerStack::MetadataFormat::NoMetadata);
+	m_appendCBORMetadata = false;
 	compileAndRun(sourceCode);
 
 	auto evmVersion = solidity::test::CommonOptions::get().evmVersion();

test/libsolidity/SemanticTest.cpp

@@ -53,7 +53,7 @@ SemanticTest::SemanticTest(
 	bool _enforceGasCost,
 	u256 _enforceGasCostMinValue
 ):
-	SolidityExecutionFramework(_evmVersion, _vmPaths),
+	SolidityExecutionFramework(_evmVersion, _vmPaths, false),
 	EVMVersionRestrictedTestCase(_filename),
 	m_sources(m_reader.sources()),
 	m_lineOffset(m_reader.lineNumber()),

test/libsolidity/SolidityExecutionFramework.cpp

@@ -62,6 +62,9 @@ bytes SolidityExecutionFramework::multiSourceCompileContract(
 	m_compiler.enableEvmBytecodeGeneration(!m_compileViaYul);
 	m_compiler.enableIRGeneration(m_compileViaYul);
 	m_compiler.setRevertStringBehaviour(m_revertStrings);
+	if (!m_appendCBORMetadata) {
+		m_compiler.setMetadataFormat(CompilerStack::MetadataFormat::NoMetadata);
+	}
 	m_compiler.setMetadataHash(m_metadataHash);
 	if (!m_compiler.compile())
 	{

test/libsolidity/SolidityExecutionFramework.h

@@ -40,8 +40,14 @@ class SolidityExecutionFramework: public solidity::test::ExecutionFramework
 
 public:
 	SolidityExecutionFramework(): m_showMetadata(solidity::test::CommonOptions::get().showMetadata) {}
-	explicit SolidityExecutionFramework(langutil::EVMVersion _evmVersion, std::vector<boost::filesystem::path> const& _vmPaths):
-		ExecutionFramework(_evmVersion, _vmPaths), m_showMetadata(solidity::test::CommonOptions::get().showMetadata)
+	explicit SolidityExecutionFramework(
+		langutil::EVMVersion _evmVersion,
+		std::vector<boost::filesystem::path> const& _vmPaths,
+		bool _appendCBORMetadata = true
+	):
+		ExecutionFramework(_evmVersion, _vmPaths),
+		m_showMetadata(solidity::test::CommonOptions::get().showMetadata),
+		m_appendCBORMetadata(_appendCBORMetadata)
 	{}
 
 	bytes const& compileAndRunWithoutCheck(
@@ -80,6 +86,7 @@ protected:
 	bool m_compileViaYul = false;
 	bool m_compileToEwasm = false;
 	bool m_showMetadata = false;
+	bool m_appendCBORMetadata = true;
 	CompilerStack::MetadataHash m_metadataHash = CompilerStack::MetadataHash::IPFS;
 	RevertStrings m_revertStrings = RevertStrings::Default;
 };

test/solc/CommandLineParser.cpp

@@ -224,6 +224,15 @@ BOOST_AUTO_TEST_CASE(cli_mode_options)
 	}
 }
 
+BOOST_AUTO_TEST_CASE(no_cbor_metadata)
+{
+	vector<string> commandLine = {"solc", "--no-cbor-metadata", "contract.sol"};
+	CommandLineOptions parsedOptions = parseCommandLine(commandLine);
+	bool assert = parsedOptions.metadata.format == CompilerStack::MetadataFormat::NoMetadata;
+
+	BOOST_TEST(assert);
+}
+
 BOOST_AUTO_TEST_CASE(via_ir_options)
 {
 	BOOST_TEST(!parseCommandLine({"solc", "contract.sol"}).output.viaIR);