From d3abbd5610824ad82337d9926655da02d45063ea Mon Sep 17 00:00:00 2001 From: chriseth Date: Wed, 8 Jul 2020 20:13:11 +0200 Subject: [PATCH 01/22] Remove unneccessary masking of tags. --- Changelog.md | 1 + libevmasm/PeepholeOptimiser.cpp | 13 ++++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/Changelog.md b/Changelog.md index 9795ecb51..ca81e04d8 100644 --- a/Changelog.md +++ b/Changelog.md @@ -2,6 +2,7 @@ Compiler Features: * Code Generator: Evaluate ``keccak256`` of string literals at compile-time. + * Peephole Optimizer: Remove unnecessary masking of tags. Bugfixes: * Type Checker: Fix overload resolution in combination with ``{value: ...}``. diff --git a/libevmasm/PeepholeOptimiser.cpp b/libevmasm/PeepholeOptimiser.cpp index 861cae0f2..dd5f9399c 100644 --- a/libevmasm/PeepholeOptimiser.cpp +++ b/libevmasm/PeepholeOptimiser.cpp @@ -266,9 +266,10 @@ struct TagConjunctions: SimplePeepholeOptimizerMethod std::back_insert_iterator _out ) { + if (_and != Instruction::AND) + return false; if ( _pushTag.type() == PushTag && - _and == Instruction::AND && _pushConstant.type() == Push && (_pushConstant.data() & u256(0xFFFFFFFF)) == u256(0xFFFFFFFF) ) @@ -276,6 +277,16 @@ struct TagConjunctions: SimplePeepholeOptimizerMethod *_out = _pushTag; return true; } + else if ( + // tag and constant are swapped + _pushConstant.type() == PushTag && + _pushTag.type() == Push && + (_pushTag.data() & u256(0xFFFFFFFF)) == u256(0xFFFFFFFF) + ) + { + *_out = _pushConstant; + return true; + } else return false; } From ec2393d3b656291636f827f81b067caac17bad89 Mon Sep 17 00:00:00 2001 From: chriseth Date: Thu, 9 Jul 2020 14:24:49 +0200 Subject: [PATCH 02/22] Refactor interpreter. --- test/libyul/EwasmTranslationTest.cpp | 4 +- test/libyul/YulInterpreterTest.cpp | 3 +- test/tools/ossfuzz/yulFuzzerCommon.cpp | 3 +- test/tools/yulInterpreter/Interpreter.cpp | 81 +++++++++++------------ test/tools/yulInterpreter/Interpreter.h | 48 +++++++------- test/tools/yulrun.cpp | 5 +- 6 files changed, 69 insertions(+), 75 deletions(-) diff --git a/test/libyul/EwasmTranslationTest.cpp b/test/libyul/EwasmTranslationTest.cpp index 8ef5ba0a0..9ef203dd0 100644 --- a/test/libyul/EwasmTranslationTest.cpp +++ b/test/libyul/EwasmTranslationTest.cpp @@ -99,11 +99,9 @@ string EwasmTranslationTest::interpret() InterpreterState state; state.maxTraceSize = 10000; state.maxSteps = 100000; - WasmDialect dialect; - Interpreter interpreter(state, dialect); try { - interpreter(*m_object->code); + Interpreter::run(state, WasmDialect{}, *m_object->code); } catch (InterpreterTerminatedGeneric const&) { diff --git a/test/libyul/YulInterpreterTest.cpp b/test/libyul/YulInterpreterTest.cpp index 9925b9546..fa84c0869 100644 --- a/test/libyul/YulInterpreterTest.cpp +++ b/test/libyul/YulInterpreterTest.cpp @@ -88,10 +88,9 @@ string YulInterpreterTest::interpret() InterpreterState state; state.maxTraceSize = 10000; state.maxSteps = 10000; - Interpreter interpreter(state, EVMDialect::strictAssemblyForEVMObjects(langutil::EVMVersion{})); try { - interpreter(*m_ast); + Interpreter::run(state, EVMDialect::strictAssemblyForEVMObjects(langutil::EVMVersion{}), *m_ast); } catch (InterpreterTerminatedGeneric const&) { diff --git a/test/tools/ossfuzz/yulFuzzerCommon.cpp b/test/tools/ossfuzz/yulFuzzerCommon.cpp index 214caf3f8..3682eef48 100644 --- a/test/tools/ossfuzz/yulFuzzerCommon.cpp +++ b/test/tools/ossfuzz/yulFuzzerCommon.cpp @@ -44,12 +44,11 @@ yulFuzzerUtil::TerminationReason yulFuzzerUtil::interpret( 0xc7, 0x60, 0x5f, 0x7c, 0xcd, 0xfb, 0x92, 0xcd, 0x8e, 0xf3, 0x9b, 0xe4, 0x4f, 0x6c, 0x14, 0xde }; - Interpreter interpreter(state, _dialect); TerminationReason reason = TerminationReason::None; try { - interpreter(*_ast); + Interpreter::run(state, _dialect, *_ast); } catch (StepLimitReached const&) { diff --git a/test/tools/yulInterpreter/Interpreter.cpp b/test/tools/yulInterpreter/Interpreter.cpp index 5f0861d14..cd4fa5d69 100644 --- a/test/tools/yulInterpreter/Interpreter.cpp +++ b/test/tools/yulInterpreter/Interpreter.cpp @@ -64,6 +64,12 @@ void InterpreterState::dumpTraceAndState(ostream& _out) const _out << " " << slot.first.hex() << ": " << slot.second.hex() << endl; } +void Interpreter::run(InterpreterState& _state, Dialect const& _dialect, Block const& _ast) +{ + Scope scope; + Interpreter{_state, _dialect, scope}(_ast); +} + void Interpreter::operator()(ExpressionStatement const& _expressionStatement) { evaluateMulti(_expressionStatement.expression); @@ -94,8 +100,7 @@ void Interpreter::operator()(VariableDeclaration const& _declaration) YulString varName = _declaration.variables.at(i).name; solAssert(!m_variables.count(varName), ""); m_variables[varName] = values.at(i); - solAssert(!m_scopes.back().count(varName), ""); - m_scopes.back().emplace(varName, nullptr); + m_scope->names.emplace(varName, nullptr); } } @@ -128,8 +133,8 @@ void Interpreter::operator()(ForLoop const& _forLoop) { solAssert(_forLoop.condition, ""); - openScope(); - ScopeGuard g([this]{ closeScope(); }); + enterScope(_forLoop.pre); + ScopeGuard g([this]{ leaveScope(); }); for (auto const& statement: _forLoop.pre.statements) { @@ -176,14 +181,13 @@ void Interpreter::operator()(Block const& _block) m_state.trace.emplace_back("Interpreter execution step limit reached."); throw StepLimitReached(); } - openScope(); + enterScope(_block); // Register functions. for (auto const& statement: _block.statements) if (holds_alternative(statement)) { FunctionDefinition const& funDef = std::get(statement); - solAssert(!m_scopes.back().count(funDef.name), ""); - m_scopes.back().emplace(funDef.name, &funDef); + m_scope->names.emplace(funDef.name, &funDef); } for (auto const& statement: _block.statements) @@ -193,29 +197,41 @@ void Interpreter::operator()(Block const& _block) break; } - closeScope(); + leaveScope(); } u256 Interpreter::evaluate(Expression const& _expression) { - ExpressionEvaluator ev(m_state, m_dialect, m_variables, m_scopes); + ExpressionEvaluator ev(m_state, m_dialect, *m_scope, m_variables); ev.visit(_expression); return ev.value(); } vector Interpreter::evaluateMulti(Expression const& _expression) { - ExpressionEvaluator ev(m_state, m_dialect, m_variables, m_scopes); + ExpressionEvaluator ev(m_state, m_dialect, *m_scope, m_variables); ev.visit(_expression); return ev.values(); } -void Interpreter::closeScope() +void Interpreter::enterScope(Block const& _block) { - for (auto const& [var, funDeclaration]: m_scopes.back()) + if (!m_scope->subScopes.count(&_block)) + m_scope->subScopes[&_block] = make_unique(Scope{ + {}, + {}, + m_scope + }); + m_scope = m_scope->subScopes[&_block].get(); +} + +void Interpreter::leaveScope() +{ + for (auto const& [var, funDeclaration]: m_scope->names) if (!funDeclaration) - solAssert(m_variables.erase(var) == 1, ""); - m_scopes.pop_back(); + m_variables.erase(var); + m_scope = m_scope->parent; + yulAssert(m_scope, ""); } void ExpressionEvaluator::operator()(Literal const& _literal) @@ -253,10 +269,15 @@ void ExpressionEvaluator::operator()(FunctionCall const& _funCall) return; } - auto [functionScopes, fun] = findFunctionAndScope(_funCall.functionName.name); + Scope* scope = &m_scope; + for (; scope; scope = scope->parent) + if (scope->names.count(_funCall.functionName.name)) + break; + yulAssert(scope, ""); - solAssert(fun, "Function not found."); - solAssert(m_values.size() == fun->parameters.size(), ""); + FunctionDefinition const* fun = scope->names.at(_funCall.functionName.name); + yulAssert(fun, "Function not found."); + yulAssert(m_values.size() == fun->parameters.size(), ""); map variables; for (size_t i = 0; i < fun->parameters.size(); ++i) variables[fun->parameters.at(i).name] = m_values.at(i); @@ -264,7 +285,7 @@ void ExpressionEvaluator::operator()(FunctionCall const& _funCall) variables[fun->returnVariables.at(i).name] = 0; m_state.controlFlowState = ControlFlowState::Default; - Interpreter interpreter(m_state, m_dialect, variables, functionScopes); + Interpreter interpreter(m_state, m_dialect, *scope, std::move(variables)); interpreter(fun->body); m_state.controlFlowState = ControlFlowState::Default; @@ -297,27 +318,3 @@ void ExpressionEvaluator::evaluateArgs(vector const& _expr) m_values = std::move(values); std::reverse(m_values.begin(), m_values.end()); } - -pair< - vector>, - FunctionDefinition const* -> ExpressionEvaluator::findFunctionAndScope(YulString _functionName) const -{ - FunctionDefinition const* fun = nullptr; - std::vector> newScopes; - for (auto const& scope: m_scopes) - { - // Copy over all functions. - newScopes.emplace_back(); - for (auto const& [name, funDef]: scope) - if (funDef) - newScopes.back().emplace(name, funDef); - // Stop at the called function. - if (scope.count(_functionName)) - { - fun = scope.at(_functionName); - break; - } - } - return {move(newScopes), fun}; -} diff --git a/test/tools/yulInterpreter/Interpreter.h b/test/tools/yulInterpreter/Interpreter.h index f50cbe6c4..3c9619d21 100644 --- a/test/tools/yulInterpreter/Interpreter.h +++ b/test/tools/yulInterpreter/Interpreter.h @@ -96,23 +96,37 @@ struct InterpreterState void dumpTraceAndState(std::ostream& _out) const; }; +/** + * Scope structure built and maintained during execution. + */ +struct Scope +{ + /// Used for variables and functions. Value is nullptr for variables. + std::map names; + std::map> subScopes; + Scope* parent = nullptr; +}; + /** * Yul interpreter. */ class Interpreter: public ASTWalker { public: + static void run(InterpreterState& _state, Dialect const& _dialect, Block const& _ast); + Interpreter( InterpreterState& _state, Dialect const& _dialect, - std::map _variables = {}, - std::vector> _scopes = {} + Scope& _scope, + std::map _variables = {} ): m_dialect(_dialect), m_state(_state), m_variables(std::move(_variables)), - m_scopes(std::move(_scopes)) - {} + m_scope(&_scope) + { + } void operator()(ExpressionStatement const& _statement) override; void operator()(Assignment const& _assignment) override; @@ -136,18 +150,14 @@ private: /// Evaluates the expression and returns its value. std::vector evaluateMulti(Expression const& _expression); - void openScope() { m_scopes.emplace_back(); } - /// Unregisters variables and functions. - void closeScope(); + void enterScope(Block const& _block); + void leaveScope(); Dialect const& m_dialect; InterpreterState& m_state; /// Values of variables. std::map m_variables; - /// Scopes of variables and functions. Used for lookup, clearing at end of blocks - /// and passing over the visible functions across function calls. - /// The pointer is nullptr if and only if the key is a variable. - std::vector> m_scopes; + Scope* m_scope; }; /** @@ -159,13 +169,13 @@ public: ExpressionEvaluator( InterpreterState& _state, Dialect const& _dialect, - std::map const& _variables, - std::vector> const& _scopes + Scope& _scope, + std::map const& _variables ): m_state(_state), m_dialect(_dialect), m_variables(_variables), - m_scopes(_scopes) + m_scope(_scope) {} void operator()(Literal const&) override; @@ -184,19 +194,11 @@ private: /// stores it in m_value. void evaluateArgs(std::vector const& _expr); - /// Finds the function called @a _functionName in the current scope stack and returns - /// the function's scope stack (with variables removed) and definition. - std::pair< - std::vector>, - FunctionDefinition const* - > findFunctionAndScope(YulString _functionName) const; - InterpreterState& m_state; Dialect const& m_dialect; /// Values of variables. std::map const& m_variables; - /// Stack of scopes in the current context. - std::vector> const& m_scopes; + Scope& m_scope; /// Current value of the expression std::vector m_values; }; diff --git a/test/tools/yulrun.cpp b/test/tools/yulrun.cpp index 1e82fba7f..494115540 100644 --- a/test/tools/yulrun.cpp +++ b/test/tools/yulrun.cpp @@ -88,11 +88,10 @@ void interpret(string const& _source) InterpreterState state; state.maxTraceSize = 10000; - Dialect const& dialect(EVMDialect::strictAssemblyForEVMObjects(langutil::EVMVersion{})); - Interpreter interpreter(state, dialect); try { - interpreter(*ast); + Dialect const& dialect(EVMDialect::strictAssemblyForEVMObjects(langutil::EVMVersion{})); + Interpreter::run(state, dialect, *ast); } catch (InterpreterTerminatedGeneric const&) { From 3cf5ed9514211d18d5171f9ddbf849c850fcd1e4 Mon Sep 17 00:00:00 2001 From: chriseth Date: Thu, 9 Jul 2020 14:56:34 +0200 Subject: [PATCH 03/22] Use plain strings for comparison. --- .../EwasmBuiltinInterpreter.cpp | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/test/tools/yulInterpreter/EwasmBuiltinInterpreter.cpp b/test/tools/yulInterpreter/EwasmBuiltinInterpreter.cpp index b09fb5a87..1d8e29e91 100644 --- a/test/tools/yulInterpreter/EwasmBuiltinInterpreter.cpp +++ b/test/tools/yulInterpreter/EwasmBuiltinInterpreter.cpp @@ -76,11 +76,12 @@ u256 EwasmBuiltinInterpreter::evalBuiltin(YulString _fun, vector const& _a for (u256 const& a: _arguments) arg.emplace_back(uint64_t(a & uint64_t(-1))); - if (_fun == "datasize"_yulstring) + string fun = _fun.str(); + if (fun == "datasize") return u256(keccak256(h256(_arguments.at(0)))) & 0xfff; - else if (_fun == "dataoffset"_yulstring) + else if (fun == "dataoffset") return u256(keccak256(h256(_arguments.at(0) + 2))) & 0xfff; - else if (_fun == "datacopy"_yulstring) + else if (fun == "datacopy") { // This is identical to codecopy. if (accessMemory(_arguments.at(0), _arguments.at(2))) @@ -93,49 +94,49 @@ u256 EwasmBuiltinInterpreter::evalBuiltin(YulString _fun, vector const& _a ); return 0; } - else if (_fun == "i32.drop"_yulstring || _fun == "i64.drop"_yulstring || _fun == "nop"_yulstring) + else if (fun == "i32.drop" || fun == "i64.drop" || fun == "nop") return {}; - else if (_fun == "i32.wrap_i64"_yulstring) + else if (fun == "i32.wrap_i64") return arg.at(0) & uint32_t(-1); - else if (_fun == "i64.extend_i32_u"_yulstring) + else if (fun == "i64.extend_i32_u") // Return the same as above because everything is u256 anyway. return arg.at(0) & uint32_t(-1); - else if (_fun == "unreachable"_yulstring) + else if (fun == "unreachable") { logTrace(evmasm::Instruction::INVALID, {}); throw ExplicitlyTerminated(); } - else if (_fun == "i64.store"_yulstring) + else if (fun == "i64.store") { accessMemory(arg[0], 8); writeMemoryWord(arg[0], arg[1]); return 0; } - else if (_fun == "i64.store8"_yulstring || _fun == "i32.store8"_yulstring) + else if (fun == "i64.store8" || fun == "i32.store8") { accessMemory(arg[0], 1); writeMemoryByte(arg[0], static_cast(arg[1] & 0xff)); return 0; } - else if (_fun == "i64.load"_yulstring) + else if (fun == "i64.load") { accessMemory(arg[0], 8); return readMemoryWord(arg[0]); } - else if (_fun == "i32.store"_yulstring) + else if (fun == "i32.store") { accessMemory(arg[0], 4); writeMemoryHalfWord(arg[0], arg[1]); return 0; } - else if (_fun == "i32.load"_yulstring) + else if (fun == "i32.load") { accessMemory(arg[0], 4); return readMemoryHalfWord(arg[0]); } - string prefix = _fun.str(); + string prefix = fun; string suffix; auto dot = prefix.find("."); if (dot != string::npos) From 87557896ed8bf88f5860629f73c332cfdb5bd4ed Mon Sep 17 00:00:00 2001 From: Alex Beregszaszi Date: Wed, 11 Dec 2019 13:23:43 +0000 Subject: [PATCH 04/22] Add ctz and popcnt to WasmDialect --- Changelog.md | 3 +++ libyul/backends/wasm/WasmDialect.cpp | 20 ++++++++++++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/Changelog.md b/Changelog.md index e4310c339..a63b80697 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,8 @@ ### 0.6.12 (unreleased) +Language Features: + * Wasm backend: Add ``i32.ctz``, ``i64.ctz``, ``i32.popcnt``, and ``i64.popcnt``. + Compiler Features: * Code Generator: Evaluate ``keccak256`` of string literals at compile-time. diff --git a/libyul/backends/wasm/WasmDialect.cpp b/libyul/backends/wasm/WasmDialect.cpp index 39138dfb2..9dddddd69 100644 --- a/libyul/backends/wasm/WasmDialect.cpp +++ b/libyul/backends/wasm/WasmDialect.cpp @@ -38,13 +38,18 @@ WasmDialect::WasmDialect() "add", "sub", "mul", + // TODO: div_s "div_u", + // TODO: rem_s "rem_u", "and", "or", "xor", "shl", + // TODO: shr_s "shr_u", + // TODO: rotl + // TODO: rotr }) addFunction(t.str() + "." + name, {t, t}, {t}); @@ -52,9 +57,13 @@ WasmDialect::WasmDialect() for (auto const& name: { "eq", "ne", + // TODO: lt_s "lt_u", + // TODO: gt_s "gt_u", + // TODO: le_s "le_u", + // TODO: ge_s "ge_u" }) addFunction(t.str() + "." + name, {t, t}, {i32}); @@ -62,8 +71,13 @@ WasmDialect::WasmDialect() addFunction("i32.eqz", {i32}, {i32}); addFunction("i64.eqz", {i64}, {i32}); - addFunction("i32.clz", {i32}, {i32}); - addFunction("i64.clz", {i64}, {i64}); + for (auto t: types) + for (auto const& name: { + "clz", + "ctz", + "popcnt", + }) + addFunction(t.str() + "." + name, {t}, {t}); addFunction("i32.wrap_i64", {i64}, {i32}); @@ -73,6 +87,7 @@ WasmDialect::WasmDialect() m_functions["i32.store"_yulstring].sideEffects.invalidatesStorage = false; addFunction("i64.store", {i32, i64}, {}, false); m_functions["i64.store"_yulstring].sideEffects.invalidatesStorage = false; + // TODO: add i32.store16, i64.store8, i64.store16, i64.store32 addFunction("i32.store8", {i32, i32}, {}, false); m_functions["i32.store8"_yulstring].sideEffects.invalidatesStorage = false; @@ -89,6 +104,7 @@ WasmDialect::WasmDialect() m_functions["i64.load"_yulstring].sideEffects.invalidatesMemory = false; m_functions["i64.load"_yulstring].sideEffects.sideEffectFree = true; m_functions["i64.load"_yulstring].sideEffects.sideEffectFreeIfNoMSize = true; + // TODO: add i32.load8, i32.load16, i64.load8, i64.load16, i64.load32 // Drop is actually overloaded for all types, but Yul does not support that. // Because of that, we introduce "i32.drop" and "i64.drop". From 8df4baac4292d72cc1fffd4851cb3c5e4c248d28 Mon Sep 17 00:00:00 2001 From: Alex Beregszaszi Date: Wed, 8 Jul 2020 23:16:43 +0100 Subject: [PATCH 05/22] Implement ctz/popcnt in yulInterpreter --- .../EwasmBuiltinInterpreter.cpp | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/test/tools/yulInterpreter/EwasmBuiltinInterpreter.cpp b/test/tools/yulInterpreter/EwasmBuiltinInterpreter.cpp index 440ffcd7a..5ad07c5bb 100644 --- a/test/tools/yulInterpreter/EwasmBuiltinInterpreter.cpp +++ b/test/tools/yulInterpreter/EwasmBuiltinInterpreter.cpp @@ -67,6 +67,50 @@ uint64_t clz64(uint64_t _v) return r; } +/// Count trailing zeros for uint32. Following WebAssembly rules, it returns 32 for @a _v being zero. +/// NOTE: the ctz builtin of the compiler may or may not do this +uint32_t ctz32(uint32_t _v) +{ + if (_v == 0) + return 32; + + uint32_t r = 0; + while (!(_v & 1)) + { + r++; + _v >>= 1; + } + return r; +} + +/// Count trailing zeros for uint64. Following WebAssembly rules, it returns 64 for @a _v being zero. +/// NOTE: the ctz builtin of the compiler may or may not do this +uint64_t ctz64(uint64_t _v) +{ + if (_v == 0) + return 64; + + uint64_t r = 0; + while (!(_v & 1)) + { + r++; + _v >>= 1; + } + return r; +} + +/// Count number of bits set for uint64 +uint64_t popcnt(uint64_t _v) +{ + uint64_t r = 0; + while (_v) + { + r += (_v & 1); + _v >>= 1; + } + return r; +} + } using u512 = boost::multiprecision::number>; @@ -139,6 +183,10 @@ u256 EwasmBuiltinInterpreter::evalBuiltin(YulString _fun, vector const& _a return clz64(arg[0] & uint32_t(-1)) - 32; else if (_fun == "i64.clz"_yulstring) return clz64(arg[0]); + else if (_fun == "i32.ctz"_yulstring) + return ctz32(uint32_t(arg[0] & uint32_t(-1))); + else if (_fun == "i64.ctz"_yulstring) + return ctz64(arg[0]); string prefix = _fun.str(); string suffix; @@ -207,6 +255,8 @@ u256 EwasmBuiltinInterpreter::evalWasmBuiltin(string const& _fun, vector c return arg[0] != arg[1] ? 1 : 0; else if (_fun == "eqz") return arg[0] == 0 ? 1 : 0; + else if (_fun == "popcnt") + return popcnt(arg[0]); else if (_fun == "lt_u") return arg[0] < arg[1] ? 1 : 0; else if (_fun == "gt_u") From 080f33a72c31b07e185e04088873285fcf2db1db Mon Sep 17 00:00:00 2001 From: chriseth Date: Mon, 13 Jul 2020 11:16:30 +0200 Subject: [PATCH 06/22] Remove swap after dup. --- Changelog.md | 1 + libevmasm/PeepholeOptimiser.cpp | 26 +++++++++- test/cmdlineTests/dup_opt_peephole/args | 1 + test/cmdlineTests/dup_opt_peephole/err | 5 ++ test/cmdlineTests/dup_opt_peephole/input.sol | 9 ++++ test/cmdlineTests/dup_opt_peephole/output | 54 ++++++++++++++++++++ 6 files changed, 95 insertions(+), 1 deletion(-) create mode 100644 test/cmdlineTests/dup_opt_peephole/args create mode 100644 test/cmdlineTests/dup_opt_peephole/err create mode 100644 test/cmdlineTests/dup_opt_peephole/input.sol create mode 100644 test/cmdlineTests/dup_opt_peephole/output diff --git a/Changelog.md b/Changelog.md index e4310c339..f82e99119 100644 --- a/Changelog.md +++ b/Changelog.md @@ -10,6 +10,7 @@ Bugfixes: Compiler Features: * Build System: Update internal dependency of jsoncpp to 1.9.3. * Optimizer: Add rule to remove shifts inside the byte opcode. + * Peephole Optimizer: Add rule to remove swap after dup. ### 0.6.11 (2020-07-07) diff --git a/libevmasm/PeepholeOptimiser.cpp b/libevmasm/PeepholeOptimiser.cpp index 861cae0f2..343f2f824 100644 --- a/libevmasm/PeepholeOptimiser.cpp +++ b/libevmasm/PeepholeOptimiser.cpp @@ -205,6 +205,30 @@ struct SwapComparison: SimplePeepholeOptimizerMethod } }; +/// Remove swapN after dupN +struct DupSwap: SimplePeepholeOptimizerMethod +{ + static size_t applySimple( + AssemblyItem const& _dupN, + AssemblyItem const& _swapN, + std::back_insert_iterator _out + ) + { + if ( + SemanticInformation::isDupInstruction(_dupN) && + SemanticInformation::isSwapInstruction(_swapN) && + getDupNumber(_dupN.instruction()) == getSwapNumber(_swapN.instruction()) + ) + { + *_out = _dupN; + return true; + } + else + return false; + } +}; + + struct IsZeroIsZeroJumpI: SimplePeepholeOptimizerMethod { static size_t applySimple( @@ -357,7 +381,7 @@ bool PeepholeOptimiser::optimise() applyMethods( state, PushPop(), OpPop(), DoublePush(), DoubleSwap(), CommutativeSwap(), SwapComparison(), - IsZeroIsZeroJumpI(), JumpToNext(), UnreachableCode(), + DupSwap(), IsZeroIsZeroJumpI(), JumpToNext(), UnreachableCode(), TagConjunctions(), TruthyAnd(), Identity() ); if (m_optimisedItems.size() < m_items.size() || ( diff --git a/test/cmdlineTests/dup_opt_peephole/args b/test/cmdlineTests/dup_opt_peephole/args new file mode 100644 index 000000000..d3047670e --- /dev/null +++ b/test/cmdlineTests/dup_opt_peephole/args @@ -0,0 +1 @@ +--asm \ No newline at end of file diff --git a/test/cmdlineTests/dup_opt_peephole/err b/test/cmdlineTests/dup_opt_peephole/err new file mode 100644 index 000000000..cfaa88dcf --- /dev/null +++ b/test/cmdlineTests/dup_opt_peephole/err @@ -0,0 +1,5 @@ +Warning: SPDX license identifier not provided in source file. Before publishing, consider adding a comment containing "SPDX-License-Identifier: " to each source file. Use "SPDX-License-Identifier: UNLICENSED" for non-open-source code. Please see https://spdx.org for more information. +--> dup_opt_peephole/input.sol + +Warning: Source file does not specify required compiler version! +--> dup_opt_peephole/input.sol diff --git a/test/cmdlineTests/dup_opt_peephole/input.sol b/test/cmdlineTests/dup_opt_peephole/input.sol new file mode 100644 index 000000000..1474a8f44 --- /dev/null +++ b/test/cmdlineTests/dup_opt_peephole/input.sol @@ -0,0 +1,9 @@ +contract C { + fallback() external { + assembly { + let x := calldataload(0) + x := x + sstore(0, x) + } + } +} diff --git a/test/cmdlineTests/dup_opt_peephole/output b/test/cmdlineTests/dup_opt_peephole/output new file mode 100644 index 000000000..020260c2b --- /dev/null +++ b/test/cmdlineTests/dup_opt_peephole/output @@ -0,0 +1,54 @@ + +======= dup_opt_peephole/input.sol:C ======= +EVM assembly: + /* "dup_opt_peephole/input.sol":0:111 contract C {... */ + mstore(0x40, 0x80) + callvalue + dup1 + iszero + tag_1 + jumpi + 0x00 + dup1 + revert +tag_1: + pop + dataSize(sub_0) + dup1 + dataOffset(sub_0) + 0x00 + codecopy + 0x00 + return +stop + +sub_0: assembly { + /* "dup_opt_peephole/input.sol":0:111 contract C {... */ + mstore(0x40, 0x80) + callvalue + dup1 + iszero + tag_3 + jumpi + 0x00 + dup1 + revert + tag_3: + pop + /* "dup_opt_peephole/input.sol":74:75 0 */ + 0x00 + /* "dup_opt_peephole/input.sol":61:76 calldataload(0) */ + calldataload + /* "dup_opt_peephole/input.sol":100:101 x */ + dup1 + /* "dup_opt_peephole/input.sol":97:98 0 */ + 0x00 + /* "dup_opt_peephole/input.sol":90:102 sstore(0, x) */ + sstore + /* "dup_opt_peephole/input.sol":47:106 {... */ + pop + /* "dup_opt_peephole/input.sol":0:111 contract C {... */ + stop + + auxdata: AUXDATA REMOVED +} From def0ebbb3e4a8eeb68d2e29155778eb7ede59188 Mon Sep 17 00:00:00 2001 From: Daniel Kirchner Date: Fri, 10 Jul 2020 17:39:23 +0200 Subject: [PATCH 07/22] Free variables directly after visiting RHS of Variable Declarations during EVMCodeTransform. --- Changelog.md | 1 + libyul/backends/evm/EVMCodeTransform.cpp | 14 +++--- libyul/backends/evm/EVMCodeTransform.h | 5 +- test/libyul/StackReuseCodegen.cpp | 63 ++++++++++++++++++++++++ 4 files changed, 75 insertions(+), 8 deletions(-) diff --git a/Changelog.md b/Changelog.md index e140ebea5..047b06ddf 100644 --- a/Changelog.md +++ b/Changelog.md @@ -6,6 +6,7 @@ Language Features: Compiler Features: * Code Generator: Evaluate ``keccak256`` of string literals at compile-time. * Peephole Optimizer: Remove unnecessary masking of tags. + * Yul EVM Code Transform: Free stack slots directly after visiting the right-hand-side of variable declarations instead of at the end of the statement only. Bugfixes: * Type Checker: Fix overload resolution in combination with ``{value: ...}``. diff --git a/libyul/backends/evm/EVMCodeTransform.cpp b/libyul/backends/evm/EVMCodeTransform.cpp index 6b0f06e01..041e00380 100644 --- a/libyul/backends/evm/EVMCodeTransform.cpp +++ b/libyul/backends/evm/EVMCodeTransform.cpp @@ -141,7 +141,7 @@ bool CodeTransform::unreferenced(Scope::Variable const& _var) const return !m_context->variableReferences.count(&_var) || m_context->variableReferences[&_var] == 0; } -void CodeTransform::freeUnusedVariables() +void CodeTransform::freeUnusedVariables(bool _popUnusedSlotsAtStackTop) { if (!m_allowStackOpt) return; @@ -154,11 +154,12 @@ void CodeTransform::freeUnusedVariables() deleteVariable(var); } - while (m_unusedStackSlots.count(m_assembly.stackHeight() - 1)) - { - yulAssert(m_unusedStackSlots.erase(m_assembly.stackHeight() - 1), ""); - m_assembly.appendInstruction(evmasm::Instruction::POP); - } + if (_popUnusedSlotsAtStackTop) + while (m_unusedStackSlots.count(m_assembly.stackHeight() - 1)) + { + yulAssert(m_unusedStackSlots.erase(m_assembly.stackHeight() - 1), ""); + m_assembly.appendInstruction(evmasm::Instruction::POP); + } } void CodeTransform::deleteVariable(Scope::Variable const& _var) @@ -181,6 +182,7 @@ void CodeTransform::operator()(VariableDeclaration const& _varDecl) { std::visit(*this, *_varDecl.value); expectDeposit(static_cast(numVariables), static_cast(heightAtStart)); + freeUnusedVariables(false); } else { diff --git a/libyul/backends/evm/EVMCodeTransform.h b/libyul/backends/evm/EVMCodeTransform.h index 3c9506b47..1cfa6afd7 100644 --- a/libyul/backends/evm/EVMCodeTransform.h +++ b/libyul/backends/evm/EVMCodeTransform.h @@ -161,8 +161,9 @@ protected: bool unreferenced(Scope::Variable const& _var) const; /// Marks slots of variables that are not used anymore /// and were defined in the current scope for reuse. - /// Also POPs unused topmost stack slots. - void freeUnusedVariables(); + /// Also POPs unused topmost stack slots, + /// unless @a _popUnusedSlotsAtStackTop is set to false. + void freeUnusedVariables(bool _popUnusedSlotsAtStackTop = true); /// Marks the stack slot of @a _var to be reused. void deleteVariable(Scope::Variable const& _var); diff --git a/test/libyul/StackReuseCodegen.cpp b/test/libyul/StackReuseCodegen.cpp index eec882e8c..1b6b91657 100644 --- a/test/libyul/StackReuseCodegen.cpp +++ b/test/libyul/StackReuseCodegen.cpp @@ -345,6 +345,69 @@ BOOST_AUTO_TEST_CASE(reuse_slots_function_with_gaps) ); } +BOOST_AUTO_TEST_CASE(reuse_on_decl_assign_to_last_used) +{ + string in = R"({ + let x := 5 + let y := x // y should reuse the stack slot of x + sstore(y, y) + })"; + BOOST_CHECK_EQUAL(assemble(in), + "PUSH1 0x5 " + "DUP1 SWAP1 POP " + "DUP1 DUP2 SSTORE " + "POP " + ); +} + +BOOST_AUTO_TEST_CASE(reuse_on_decl_assign_to_last_used_expr) +{ + string in = R"({ + let x := 5 + let y := add(x, 2) // y should reuse the stack slot of x + sstore(y, y) + })"; + BOOST_CHECK_EQUAL(assemble(in), + "PUSH1 0x5 " + "PUSH1 0x2 DUP2 ADD " + "SWAP1 POP " + "DUP1 DUP2 SSTORE " + "POP " + ); +} + +BOOST_AUTO_TEST_CASE(reuse_on_decl_assign_to_not_last_used) +{ + string in = R"({ + let x := 5 + let y := x // y should not reuse the stack slot of x, since x is still used below + sstore(y, x) + })"; + BOOST_CHECK_EQUAL(assemble(in), + "PUSH1 0x5 " + "DUP1 " + "DUP2 DUP2 SSTORE " + "POP POP " + ); +} + +BOOST_AUTO_TEST_CASE(reuse_on_decl_assign_not_same_scope) +{ + string in = R"({ + let x := 5 + { + let y := x // y should not reuse the stack slot of x, since x is not in the same scope + sstore(y, y) + } + })"; + BOOST_CHECK_EQUAL(assemble(in), + "PUSH1 0x5 " + "DUP1 " + "DUP1 DUP2 SSTORE " + "POP POP " + ); +} + BOOST_AUTO_TEST_SUITE_END() From 870ea98c21d4b0d6b3eea67a1ce37753610b8817 Mon Sep 17 00:00:00 2001 From: Alex Beregszaszi Date: Mon, 13 Jul 2020 15:20:29 +0100 Subject: [PATCH 08/22] Prefix EIP-615 opcodes with EIP615 in Instructions --- libevmasm/Instruction.h | 20 ++++++++--------- libyul/backends/evm/EVMAssembly.cpp | 10 ++++----- .../EVMInstructionInterpreter.cpp | 22 +++++++++---------- 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/libevmasm/Instruction.h b/libevmasm/Instruction.h index 511e3cf1b..67a77f0c9 100644 --- a/libevmasm/Instruction.h +++ b/libevmasm/Instruction.h @@ -174,16 +174,16 @@ enum class Instruction: uint8_t LOG3, ///< Makes a log entry; 3 topics. LOG4, ///< Makes a log entry; 4 topics. - JUMPTO = 0xb0, ///< alter the program counter to a jumpdest -- not part of Instructions.cpp - JUMPIF, ///< conditionally alter the program counter -- not part of Instructions.cpp - JUMPV, ///< alter the program counter to a jumpdest -- not part of Instructions.cpp - JUMPSUB, ///< alter the program counter to a beginsub -- not part of Instructions.cpp - JUMPSUBV, ///< alter the program counter to a beginsub -- not part of Instructions.cpp - BEGINSUB, ///< set a potential jumpsub destination -- not part of Instructions.cpp - BEGINDATA, ///< begin the data section -- not part of Instructions.cpp - RETURNSUB, ///< return to subroutine jumped from -- not part of Instructions.cpp - PUTLOCAL, ///< pop top of stack to local variable -- not part of Instructions.cpp - GETLOCAL, ///< push local variable to top of stack -- not part of Instructions.cpp + EIP615_JUMPTO = 0xb0, ///< alter the program counter to a jumpdest -- not part of Instructions.cpp + EIP615_JUMPIF, ///< conditionally alter the program counter -- not part of Instructions.cpp + EIP615_JUMPV, ///< alter the program counter to a jumpdest -- not part of Instructions.cpp + EIP615_JUMPSUB, ///< alter the program counter to a beginsub -- not part of Instructions.cpp + EIP615_JUMPSUBV, ///< alter the program counter to a beginsub -- not part of Instructions.cpp + EIP615_BEGINSUB, ///< set a potential jumpsub destination -- not part of Instructions.cpp + EIP615_BEGINDATA, ///< begin the data section -- not part of Instructions.cpp + EIP615_RETURNSUB, ///< return to subroutine jumped from -- not part of Instructions.cpp + EIP615_PUTLOCAL, ///< pop top of stack to local variable -- not part of Instructions.cpp + EIP615_GETLOCAL, ///< push local variable to top of stack -- not part of Instructions.cpp CREATE = 0xf0, ///< create a new account with associated code CALL, ///< message-call into an account diff --git a/libyul/backends/evm/EVMAssembly.cpp b/libyul/backends/evm/EVMAssembly.cpp index 73194ef8a..7cf3d0662 100644 --- a/libyul/backends/evm/EVMAssembly.cpp +++ b/libyul/backends/evm/EVMAssembly.cpp @@ -102,7 +102,7 @@ void EVMAssembly::appendJumpTo(LabelID _labelId, int _stackDiffAfter, JumpType _ { if (m_evm15) { - m_bytecode.push_back(uint8_t(evmasm::Instruction::JUMPTO)); + m_bytecode.push_back(uint8_t(evmasm::Instruction::EIP615_JUMPTO)); appendLabelReferenceInternal(_labelId); m_stackHeight += _stackDiffAfter; } @@ -117,7 +117,7 @@ void EVMAssembly::appendJumpToIf(LabelID _labelId, JumpType) { if (m_evm15) { - m_bytecode.push_back(uint8_t(evmasm::Instruction::JUMPIF)); + m_bytecode.push_back(uint8_t(evmasm::Instruction::EIP615_JUMPIF)); appendLabelReferenceInternal(_labelId); m_stackHeight--; } @@ -133,7 +133,7 @@ void EVMAssembly::appendBeginsub(LabelID _labelId, int _arguments) yulAssert(m_evm15, "BEGINSUB used for EVM 1.0"); yulAssert(_arguments >= 0, ""); setLabelToCurrentPosition(_labelId); - m_bytecode.push_back(uint8_t(evmasm::Instruction::BEGINSUB)); + m_bytecode.push_back(uint8_t(evmasm::Instruction::EIP615_BEGINSUB)); m_stackHeight += _arguments; } @@ -141,7 +141,7 @@ void EVMAssembly::appendJumpsub(LabelID _labelId, int _arguments, int _returns) { yulAssert(m_evm15, "JUMPSUB used for EVM 1.0"); yulAssert(_arguments >= 0 && _returns >= 0, ""); - m_bytecode.push_back(uint8_t(evmasm::Instruction::JUMPSUB)); + m_bytecode.push_back(uint8_t(evmasm::Instruction::EIP615_JUMPSUB)); appendLabelReferenceInternal(_labelId); m_stackHeight += _returns - _arguments; } @@ -150,7 +150,7 @@ void EVMAssembly::appendReturnsub(int _returns, int _stackDiffAfter) { yulAssert(m_evm15, "RETURNSUB used for EVM 1.0"); yulAssert(_returns >= 0, ""); - m_bytecode.push_back(uint8_t(evmasm::Instruction::RETURNSUB)); + m_bytecode.push_back(uint8_t(evmasm::Instruction::EIP615_RETURNSUB)); m_stackHeight += _stackDiffAfter - _returns; } diff --git a/test/tools/yulInterpreter/EVMInstructionInterpreter.cpp b/test/tools/yulInterpreter/EVMInstructionInterpreter.cpp index c17fa33e8..956db124f 100644 --- a/test/tools/yulInterpreter/EVMInstructionInterpreter.cpp +++ b/test/tools/yulInterpreter/EVMInstructionInterpreter.cpp @@ -409,17 +409,17 @@ u256 EVMInstructionInterpreter::eval( case Instruction::SWAP14: case Instruction::SWAP15: case Instruction::SWAP16: - // --------------- EVM 2.0 --------------- - case Instruction::JUMPTO: - case Instruction::JUMPIF: - case Instruction::JUMPV: - case Instruction::JUMPSUB: - case Instruction::JUMPSUBV: - case Instruction::BEGINSUB: - case Instruction::BEGINDATA: - case Instruction::RETURNSUB: - case Instruction::PUTLOCAL: - case Instruction::GETLOCAL: + // --------------- EIP-615 --------------- + case Instruction::EIP615_JUMPTO: + case Instruction::EIP615_JUMPIF: + case Instruction::EIP615_JUMPV: + case Instruction::EIP615_JUMPSUB: + case Instruction::EIP615_JUMPSUBV: + case Instruction::EIP615_BEGINSUB: + case Instruction::EIP615_BEGINDATA: + case Instruction::EIP615_RETURNSUB: + case Instruction::EIP615_PUTLOCAL: + case Instruction::EIP615_GETLOCAL: { yulAssert(false, ""); return 0; From 6a1ad7e59ceccd8a56a17d2af3e1025a321eb079 Mon Sep 17 00:00:00 2001 From: Alex Beregszaszi Date: Mon, 13 Jul 2020 19:05:34 +0100 Subject: [PATCH 09/22] Fix formatting hex strings in isoltest update --- test/libsolidity/util/BytesUtils.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/libsolidity/util/BytesUtils.cpp b/test/libsolidity/util/BytesUtils.cpp index fd2cd2fcf..7416ca9a2 100644 --- a/test/libsolidity/util/BytesUtils.cpp +++ b/test/libsolidity/util/BytesUtils.cpp @@ -22,6 +22,7 @@ #include +#include #include #include @@ -198,8 +199,7 @@ string BytesUtils::formatString(bytes const& _bytes, size_t _cutOff) if (isprint(v)) os << v; else - os << "\\x" << setw(2) << setfill('0') << hex << v; - + os << "\\x" << toHex(v); } } os << "\""; From 950612cf42b3812e5ad292d9f6e3a56f96b8f0f4 Mon Sep 17 00:00:00 2001 From: Alex Beregszaszi Date: Mon, 13 Jul 2020 18:23:42 +0100 Subject: [PATCH 10/22] Add more tests for strings with unicode (escapes) --- .../semanticTests/strings/unicode_escapes.sol | 22 +++++++++++++++++++ .../semanticTests/strings/unicode_string.sol | 7 ++++++ .../invalid_utf8_explicit_string.sol} | 0 .../string/invalid_utf8_hex_string.sol | 5 +++++ .../invalid_utf8_implicit_string.sol} | 0 .../string/invalid_utf8_in_bytes.sol | 6 +++++ .../large_utf8_codepoint.sol} | 0 .../syntaxTests/string/string_unicode.sol | 6 +++++ .../{ => string}/unicode_escape_literals.sol | 12 +--------- ...code_escape_literals_invalid_codepoint.sol | 7 ++++++ ...unicode_escape_literals_invalid_format.sol | 10 +++++++++ 11 files changed, 64 insertions(+), 11 deletions(-) create mode 100644 test/libsolidity/semanticTests/strings/unicode_escapes.sol create mode 100644 test/libsolidity/semanticTests/strings/unicode_string.sol rename test/libsolidity/syntaxTests/{nameAndTypeResolution/186_invalid_utf8_explicit.sol => string/invalid_utf8_explicit_string.sol} (100%) create mode 100644 test/libsolidity/syntaxTests/string/invalid_utf8_hex_string.sol rename test/libsolidity/syntaxTests/{nameAndTypeResolution/185_invalid_utf8_implicit.sol => string/invalid_utf8_implicit_string.sol} (100%) create mode 100644 test/libsolidity/syntaxTests/string/invalid_utf8_in_bytes.sol rename test/libsolidity/syntaxTests/{nameAndTypeResolution/187_large_utf8_codepoint.sol => string/large_utf8_codepoint.sol} (100%) create mode 100644 test/libsolidity/syntaxTests/string/string_unicode.sol rename test/libsolidity/syntaxTests/{ => string}/unicode_escape_literals.sol (60%) create mode 100644 test/libsolidity/syntaxTests/string/unicode_escape_literals_invalid_codepoint.sol create mode 100644 test/libsolidity/syntaxTests/string/unicode_escape_literals_invalid_format.sol diff --git a/test/libsolidity/semanticTests/strings/unicode_escapes.sol b/test/libsolidity/semanticTests/strings/unicode_escapes.sol new file mode 100644 index 000000000..401d8b61e --- /dev/null +++ b/test/libsolidity/semanticTests/strings/unicode_escapes.sol @@ -0,0 +1,22 @@ +contract C { + function oneByteUTF8() public pure returns (string memory) { + return "aaa\u0024aaa"; // usdollar + } + + function twoBytesUTF8() public pure returns (string memory) { + return "aaa\u00A2aaa"; // cent + } + + function threeBytesUTF8() public pure returns (string memory) { + return "aaa\u20ACaaa"; // euro + } + + function combined() public pure returns (string memory) { + return "\u0024\u00A2\u20AC"; + } +} +// ---- +// oneByteUTF8() -> 0x20, 7, "aaa$aaa" +// twoBytesUTF8() -> 0x20, 8, "aaa\xc2\xa2aaa" +// threeBytesUTF8() -> 0x20, 9, "aaa\xe2\x82\xacaaa" +// combined() -> 0x20, 6, "$\xc2\xa2\xe2\x82\xac" diff --git a/test/libsolidity/semanticTests/strings/unicode_string.sol b/test/libsolidity/semanticTests/strings/unicode_string.sol new file mode 100644 index 000000000..c139d6b8b --- /dev/null +++ b/test/libsolidity/semanticTests/strings/unicode_string.sol @@ -0,0 +1,7 @@ +contract C { + function f() public pure returns (string memory) { + return "๐Ÿ˜ƒ, ๐Ÿ˜ญ, and ๐Ÿ˜ˆ"; + } +} +// ---- +// f() -> 0x20, 0x14, "\xf0\x9f\x98\x83, \xf0\x9f\x98\xad, and \xf0\x9f\x98\x88" diff --git a/test/libsolidity/syntaxTests/nameAndTypeResolution/186_invalid_utf8_explicit.sol b/test/libsolidity/syntaxTests/string/invalid_utf8_explicit_string.sol similarity index 100% rename from test/libsolidity/syntaxTests/nameAndTypeResolution/186_invalid_utf8_explicit.sol rename to test/libsolidity/syntaxTests/string/invalid_utf8_explicit_string.sol diff --git a/test/libsolidity/syntaxTests/string/invalid_utf8_hex_string.sol b/test/libsolidity/syntaxTests/string/invalid_utf8_hex_string.sol new file mode 100644 index 000000000..d72ee5af0 --- /dev/null +++ b/test/libsolidity/syntaxTests/string/invalid_utf8_hex_string.sol @@ -0,0 +1,5 @@ +contract C { + string s = hex"a000"; +} +// ---- +// TypeError 7407: (28-37): Type literal_string (contains invalid UTF-8 sequence at position 0) is not implicitly convertible to expected type string storage ref. diff --git a/test/libsolidity/syntaxTests/nameAndTypeResolution/185_invalid_utf8_implicit.sol b/test/libsolidity/syntaxTests/string/invalid_utf8_implicit_string.sol similarity index 100% rename from test/libsolidity/syntaxTests/nameAndTypeResolution/185_invalid_utf8_implicit.sol rename to test/libsolidity/syntaxTests/string/invalid_utf8_implicit_string.sol diff --git a/test/libsolidity/syntaxTests/string/invalid_utf8_in_bytes.sol b/test/libsolidity/syntaxTests/string/invalid_utf8_in_bytes.sol new file mode 100644 index 000000000..c5af0d094 --- /dev/null +++ b/test/libsolidity/syntaxTests/string/invalid_utf8_in_bytes.sol @@ -0,0 +1,6 @@ +contract C { + bytes b1 = "\xa0\x00"; + bytes32 b2 = "\xa0\x00"; + bytes b3 = hex"a000"; +} +// ---- diff --git a/test/libsolidity/syntaxTests/nameAndTypeResolution/187_large_utf8_codepoint.sol b/test/libsolidity/syntaxTests/string/large_utf8_codepoint.sol similarity index 100% rename from test/libsolidity/syntaxTests/nameAndTypeResolution/187_large_utf8_codepoint.sol rename to test/libsolidity/syntaxTests/string/large_utf8_codepoint.sol diff --git a/test/libsolidity/syntaxTests/string/string_unicode.sol b/test/libsolidity/syntaxTests/string/string_unicode.sol new file mode 100644 index 000000000..380ad9e39 --- /dev/null +++ b/test/libsolidity/syntaxTests/string/string_unicode.sol @@ -0,0 +1,6 @@ +contract test { + function f() public pure returns (string memory) { + return "๐Ÿ˜ƒ, ๐Ÿ˜ญ, and ๐Ÿ˜ˆ"; + } +} +// ---- diff --git a/test/libsolidity/syntaxTests/unicode_escape_literals.sol b/test/libsolidity/syntaxTests/string/unicode_escape_literals.sol similarity index 60% rename from test/libsolidity/syntaxTests/unicode_escape_literals.sol rename to test/libsolidity/syntaxTests/string/unicode_escape_literals.sol index d78e6aa09..011f58b40 100644 --- a/test/libsolidity/syntaxTests/unicode_escape_literals.sol +++ b/test/libsolidity/syntaxTests/string/unicode_escape_literals.sol @@ -1,5 +1,4 @@ contract test { - function oneByteUTF8() public pure returns (bytes32) { bytes32 usdollar = "aaa\u0024aaa"; return usdollar; @@ -15,17 +14,8 @@ contract test { return eur; } - function together() public pure returns (bytes32) { + function combined() public pure returns (bytes32) { bytes32 res = "\u0024\u00A2\u20AC"; return res; } - - // this function returns an invalid unicode character - function invalidLiteral() public pure returns(bytes32) { - bytes32 invalid = "\u00xx"; - return invalid; - } - } -// ---- -// ParserError 8936: (678-681): Invalid escape sequence. diff --git a/test/libsolidity/syntaxTests/string/unicode_escape_literals_invalid_codepoint.sol b/test/libsolidity/syntaxTests/string/unicode_escape_literals_invalid_codepoint.sol new file mode 100644 index 000000000..46a0de5f4 --- /dev/null +++ b/test/libsolidity/syntaxTests/string/unicode_escape_literals_invalid_codepoint.sol @@ -0,0 +1,7 @@ +contract test { + function f() public pure returns (string memory) { + return "\xc1"; + } +} +// ---- +// TypeError 6359: (86-92): Return argument type literal_string (contains invalid UTF-8 sequence at position 0) is not implicitly convertible to expected type (type of first return variable) string memory. diff --git a/test/libsolidity/syntaxTests/string/unicode_escape_literals_invalid_format.sol b/test/libsolidity/syntaxTests/string/unicode_escape_literals_invalid_format.sol new file mode 100644 index 000000000..6afcd94af --- /dev/null +++ b/test/libsolidity/syntaxTests/string/unicode_escape_literals_invalid_format.sol @@ -0,0 +1,10 @@ +contract test { + // this function returns an invalid unicode character + function invalidLiteral() public pure returns (bytes32) { + bytes32 invalid = "\u00xx"; + return invalid; + } + +} +// ---- +// ParserError 8936: (162-165): Invalid escape sequence. From f28bede34a632a0c2417be8399c3daa864092bb0 Mon Sep 17 00:00:00 2001 From: chriseth Date: Tue, 14 Jul 2020 10:32:11 +0200 Subject: [PATCH 11/22] Refactor storageSizeUpperBound. --- libsolidity/analysis/ContractLevelChecker.cpp | 18 ++++++ libsolidity/analysis/ContractLevelChecker.h | 2 + libsolidity/analysis/TypeChecker.cpp | 16 +---- libsolidity/ast/Types.cpp | 61 ++++++------------- libsolidity/ast/Types.h | 7 ++- .../largeTypes/oversized_array.sol | 2 +- .../largeTypes/oversized_contract.sol | 2 +- .../oversized_contract_inheritance.sol | 12 ++++ .../largeTypes/oversized_struct.sol | 2 +- 9 files changed, 63 insertions(+), 59 deletions(-) create mode 100644 test/libsolidity/syntaxTests/largeTypes/oversized_contract_inheritance.sol diff --git a/libsolidity/analysis/ContractLevelChecker.cpp b/libsolidity/analysis/ContractLevelChecker.cpp index c0da54564..fc391f196 100644 --- a/libsolidity/analysis/ContractLevelChecker.cpp +++ b/libsolidity/analysis/ContractLevelChecker.cpp @@ -60,6 +60,7 @@ bool ContractLevelChecker::check(ContractDefinition const& _contract) checkLibraryRequirements(_contract); checkBaseABICompatibility(_contract); checkPayableFallbackWithoutReceive(_contract); + checkStorageSize(_contract); return Error::containsOnlyWarnings(m_errorReporter.errors()); } @@ -458,3 +459,20 @@ void ContractLevelChecker::checkPayableFallbackWithoutReceive(ContractDefinition SecondarySourceLocation{}.append("The payable fallback function is defined here.", fallback->location()) ); } + +void ContractLevelChecker::checkStorageSize(ContractDefinition const& _contract) +{ + bigint size = 0; + vector variables; + for (ContractDefinition const* contract: boost::adaptors::reverse(_contract.annotation().linearizedBaseContracts)) + for (VariableDeclaration const* variable: contract->stateVariables()) + if (!(variable->isConstant() || variable->immutable())) + { + size += variable->annotation().type->storageSizeUpperBound(); + if (size >= bigint(1) << 256) + { + m_errorReporter.typeError(7676_error, _contract.location(), "Contract too large for storage."); + break; + } + } +} diff --git a/libsolidity/analysis/ContractLevelChecker.h b/libsolidity/analysis/ContractLevelChecker.h index e40cb8b0d..3a0d7d740 100644 --- a/libsolidity/analysis/ContractLevelChecker.h +++ b/libsolidity/analysis/ContractLevelChecker.h @@ -83,6 +83,8 @@ private: /// Warns if the contract has a payable fallback, but no receive ether function. void checkPayableFallbackWithoutReceive(ContractDefinition const& _contract); + /// Error if the contract requires too much storage + void checkStorageSize(ContractDefinition const& _contract); OverrideChecker m_overrideChecker; langutil::ErrorReporter& m_errorReporter; diff --git a/libsolidity/analysis/TypeChecker.cpp b/libsolidity/analysis/TypeChecker.cpp index aa316cb48..0c47f2073 100644 --- a/libsolidity/analysis/TypeChecker.cpp +++ b/libsolidity/analysis/TypeChecker.cpp @@ -92,19 +92,7 @@ bool TypeChecker::visit(ContractDefinition const& _contract) for (auto const& n: _contract.subNodes()) n->accept(*this); - bigint size = 0; - vector variables; - for (ContractDefinition const* contract: boost::adaptors::reverse(m_currentContract->annotation().linearizedBaseContracts)) - for (VariableDeclaration const* variable: contract->stateVariables()) - if (!(variable->isConstant() || variable->immutable())) - { - size += storageSizeUpperBound(*(variable->annotation().type)); - if (size >= bigint(1) << 256) - { - m_errorReporter.typeError(7676_error, m_currentContract->location(), "Contract too large for storage."); - break; - } - } + m_currentContract = nullptr; return false; } @@ -566,7 +554,7 @@ bool TypeChecker::visit(VariableDeclaration const& _variable) " in small quantities per transaction."; }; - if (storageSizeUpperBound(*varType) >= bigint(1) << 64) + if (varType->storageSizeUpperBound() >= bigint(1) << 64) { if (_variable.isStateVariable()) m_errorReporter.warning(3408_error, _variable.location(), collisionMessage(_variable.name(), true)); diff --git a/libsolidity/ast/Types.cpp b/libsolidity/ast/Types.cpp index 546e6ee63..825fc8d0c 100644 --- a/libsolidity/ast/Types.cpp +++ b/libsolidity/ast/Types.cpp @@ -63,37 +63,6 @@ struct TypeComp }; using TypeSet = std::set; -bigint storageSizeUpperBoundInner( - Type const& _type, - set& _structsSeen -) -{ - switch (_type.category()) - { - case Type::Category::Array: - { - auto const& t = dynamic_cast(_type); - if (!t.isDynamicallySized()) - return storageSizeUpperBoundInner(*t.baseType(), _structsSeen) * t.length(); - break; - } - case Type::Category::Struct: - { - auto const& t = dynamic_cast(_type); - solAssert(!_structsSeen.count(&t.structDefinition()), "Recursive struct."); - bigint size = 1; - _structsSeen.insert(&t.structDefinition()); - for (auto const& m: t.members(nullptr)) - size += storageSizeUpperBoundInner(*m.type, _structsSeen); - _structsSeen.erase(&t.structDefinition()); - return size; - } - default: - break; - } - return bigint(1); -} - void oversizedSubtypesInner( Type const& _type, bool _includeType, @@ -106,7 +75,7 @@ void oversizedSubtypesInner( case Type::Category::Array: { auto const& t = dynamic_cast(_type); - if (_includeType && storageSizeUpperBound(t) >= bigint(1) << 64) + if (_includeType && t.storageSizeUpperBound() >= bigint(1) << 64) _oversizedSubtypes.insert(&t); oversizedSubtypesInner(*t.baseType(), t.isDynamicallySized(), _structsSeen, _oversizedSubtypes); break; @@ -116,7 +85,7 @@ void oversizedSubtypesInner( auto const& t = dynamic_cast(_type); if (_structsSeen.count(&t.structDefinition())) return; - if (_includeType && storageSizeUpperBound(t) >= bigint(1) << 64) + if (_includeType && t.storageSizeUpperBound() >= bigint(1) << 64) _oversizedSubtypes.insert(&t); _structsSeen.insert(&t.structDefinition()); for (auto const& m: t.members(nullptr)) @@ -231,12 +200,6 @@ util::Result transformParametersToExternal(TypePointers const& _pa } -bigint solidity::frontend::storageSizeUpperBound(frontend::Type const& _type) -{ - set structsSeen; - return storageSizeUpperBoundInner(_type, structsSeen); -} - vector solidity::frontend::oversizedSubtypes(frontend::Type const& _type) { set structsSeen; @@ -1847,7 +1810,7 @@ BoolResult ArrayType::validForLocation(DataLocation _loc) const break; } case DataLocation::Storage: - if (storageSizeUpperBound(*this) >= bigint(1) << 256) + if (storageSizeUpperBound() >= bigint(1) << 256) return BoolResult::err("Type too large for storage."); break; } @@ -1888,6 +1851,14 @@ bool ArrayType::isDynamicallyEncoded() const return isDynamicallySized() || baseType()->isDynamicallyEncoded(); } +bigint ArrayType::storageSizeUpperBound() const +{ + if (isDynamicallySized()) + return 1; + else + return length() * baseType()->storageSizeUpperBound(); +} + u256 ArrayType::storageSize() const { if (isDynamicallySized()) @@ -2347,6 +2318,14 @@ u256 StructType::memoryDataSize() const return size; } +bigint StructType::storageSizeUpperBound() const +{ + bigint size = 1; + for (auto const& member: members(nullptr)) + size += member.type->storageSizeUpperBound(); + return size; +} + u256 StructType::storageSize() const { return max(1, members(nullptr).storageSize()); @@ -2491,7 +2470,7 @@ BoolResult StructType::validForLocation(DataLocation _loc) const if ( _loc == DataLocation::Storage && - storageSizeUpperBound(*this) >= bigint(1) << 256 + storageSizeUpperBound() >= bigint(1) << 256 ) return BoolResult::err("Type too large for storage."); diff --git a/libsolidity/ast/Types.h b/libsolidity/ast/Types.h index b05021630..155fa7f14 100644 --- a/libsolidity/ast/Types.h +++ b/libsolidity/ast/Types.h @@ -59,7 +59,6 @@ using BoolResult = util::Result; namespace solidity::frontend { -bigint storageSizeUpperBound(frontend::Type const& _type); std::vector oversizedSubtypes(frontend::Type const& _type); inline rational makeRational(bigint const& _numerator, bigint const& _denominator) @@ -251,6 +250,10 @@ public: /// @returns the number of storage slots required to hold this value in storage. /// For dynamically "allocated" types, it returns the size of the statically allocated head, virtual u256 storageSize() const { return 1; } + /// @returns an upper bound on the total storage size required by this type, descending + /// into structs and statically-sized arrays. This is mainly to ensure that the storage + /// slot allocation algorithm does not overflow, it is not a protection against collisions. + virtual bigint storageSizeUpperBound() const { return 1; } /// Multiple small types can be packed into a single storage slot. If such a packing is possible /// this function @returns the size in bytes smaller than 32. Data is moved to the next slot if /// it does not fit. @@ -788,6 +791,7 @@ public: unsigned calldataEncodedTailSize() const override; bool isDynamicallySized() const override { return m_hasDynamicLength; } bool isDynamicallyEncoded() const override; + bigint storageSizeUpperBound() const override; u256 storageSize() const override; bool canLiveOutsideStorage() const override { return m_baseType->canLiveOutsideStorage(); } bool nameable() const override { return true; } @@ -952,6 +956,7 @@ public: unsigned calldataEncodedTailSize() const override; bool isDynamicallyEncoded() const override; u256 memoryDataSize() const override; + bigint storageSizeUpperBound() const override; u256 storageSize() const override; bool canLiveOutsideStorage() const override { return true; } bool nameable() const override { return true; } diff --git a/test/libsolidity/syntaxTests/largeTypes/oversized_array.sol b/test/libsolidity/syntaxTests/largeTypes/oversized_array.sol index 951840fd4..123773403 100644 --- a/test/libsolidity/syntaxTests/largeTypes/oversized_array.sol +++ b/test/libsolidity/syntaxTests/largeTypes/oversized_array.sol @@ -4,5 +4,5 @@ contract C { uint[2**255][2] a; } // ---- -// TypeError 1534: (77-94): Type too large for storage. // TypeError 7676: (60-97): Contract too large for storage. +// TypeError 1534: (77-94): Type too large for storage. diff --git a/test/libsolidity/syntaxTests/largeTypes/oversized_contract.sol b/test/libsolidity/syntaxTests/largeTypes/oversized_contract.sol index 69f8b1024..44c7274e6 100644 --- a/test/libsolidity/syntaxTests/largeTypes/oversized_contract.sol +++ b/test/libsolidity/syntaxTests/largeTypes/oversized_contract.sol @@ -5,6 +5,6 @@ contract C { uint[2**255] b; } // ---- +// TypeError 7676: (60-114): Contract too large for storage. // Warning 3408: (77-91): Variable "a" covers a large part of storage and thus makes collisions likely. Either use mappings or dynamic arrays and allow their size to be increased only in small quantities per transaction. // Warning 3408: (97-111): Variable "b" covers a large part of storage and thus makes collisions likely. Either use mappings or dynamic arrays and allow their size to be increased only in small quantities per transaction. -// TypeError 7676: (60-114): Contract too large for storage. diff --git a/test/libsolidity/syntaxTests/largeTypes/oversized_contract_inheritance.sol b/test/libsolidity/syntaxTests/largeTypes/oversized_contract_inheritance.sol new file mode 100644 index 000000000..24073d4c0 --- /dev/null +++ b/test/libsolidity/syntaxTests/largeTypes/oversized_contract_inheritance.sol @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-3.0 +pragma solidity >= 0.0; +contract C { + uint[2**255] a; +} +contract D is C { + uint[2**255] b; +} +// ---- +// TypeError 7676: (95-134): Contract too large for storage. +// Warning 3408: (77-91): Variable "a" covers a large part of storage and thus makes collisions likely. Either use mappings or dynamic arrays and allow their size to be increased only in small quantities per transaction. +// Warning 3408: (117-131): Variable "b" covers a large part of storage and thus makes collisions likely. Either use mappings or dynamic arrays and allow their size to be increased only in small quantities per transaction. diff --git a/test/libsolidity/syntaxTests/largeTypes/oversized_struct.sol b/test/libsolidity/syntaxTests/largeTypes/oversized_struct.sol index a3f3a4160..757ec9e4a 100644 --- a/test/libsolidity/syntaxTests/largeTypes/oversized_struct.sol +++ b/test/libsolidity/syntaxTests/largeTypes/oversized_struct.sol @@ -8,5 +8,5 @@ contract C { S s; } // ---- -// TypeError 1534: (146-149): Type too large for storage. // TypeError 7676: (60-152): Contract too large for storage. +// TypeError 1534: (146-149): Type too large for storage. From 9c62ea182dad60467bbc39a6914aba7b703b9286 Mon Sep 17 00:00:00 2001 From: Daniel Kirchner Date: Tue, 14 Jul 2020 10:08:24 +0200 Subject: [PATCH 12/22] Fix segfault in oversized objects check in combination with var. --- libsolidity/analysis/TypeChecker.cpp | 6 +++++- .../iceRegressionTests/oversized_var.sol | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 test/libsolidity/syntaxTests/iceRegressionTests/oversized_var.sol diff --git a/libsolidity/analysis/TypeChecker.cpp b/libsolidity/analysis/TypeChecker.cpp index aa316cb48..02cbaaadd 100644 --- a/libsolidity/analysis/TypeChecker.cpp +++ b/libsolidity/analysis/TypeChecker.cpp @@ -571,7 +571,11 @@ bool TypeChecker::visit(VariableDeclaration const& _variable) if (_variable.isStateVariable()) m_errorReporter.warning(3408_error, _variable.location(), collisionMessage(_variable.name(), true)); else - m_errorReporter.warning(2332_error, _variable.typeName()->location(), collisionMessage(varType->canonicalName(), false)); + m_errorReporter.warning( + 2332_error, + _variable.typeName() ? _variable.typeName()->location() : _variable.location(), + collisionMessage(varType->canonicalName(), false) + ); } vector oversizedSubtypes = frontend::oversizedSubtypes(*varType); for (Type const* subtype: oversizedSubtypes) diff --git a/test/libsolidity/syntaxTests/iceRegressionTests/oversized_var.sol b/test/libsolidity/syntaxTests/iceRegressionTests/oversized_var.sol new file mode 100644 index 000000000..0e50ae138 --- /dev/null +++ b/test/libsolidity/syntaxTests/iceRegressionTests/oversized_var.sol @@ -0,0 +1,15 @@ +contract b { + struct c { + uint [2 ** 253] a; + } + + c d; + function e() public { + var d = d; + } +} +// ---- +// Warning 2519: (105-110): This declaration shadows an existing declaration. +// Warning 3408: (66-69): Variable "d" covers a large part of storage and thus makes collisions likely. Either use mappings or dynamic arrays and allow their size to be increased only in small quantities per transaction. +// Warning 2332: (105-110): Type "b.c" covers a large part of storage and thus makes collisions likely. Either use mappings or dynamic arrays and allow their size to be increased only in small quantities per transaction. +// SyntaxError 1719: (105-114): Use of the "var" keyword is disallowed. Use explicit declaration `struct b.c storage pointer d = ...ยด instead. From eab12ecf77e3d352981fa4a71297ccfc8a9f3aea Mon Sep 17 00:00:00 2001 From: Alex Beregszaszi Date: Tue, 14 Jul 2020 14:25:17 +0100 Subject: [PATCH 13/22] Nicer error message for private payable functions --- libsolidity/analysis/TypeChecker.cpp | 2 +- .../syntaxTests/nameAndTypeResolution/356_payable_internal.sol | 2 +- .../syntaxTests/nameAndTypeResolution/357_payable_private.sol | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libsolidity/analysis/TypeChecker.cpp b/libsolidity/analysis/TypeChecker.cpp index 762361080..94d9544df 100644 --- a/libsolidity/analysis/TypeChecker.cpp +++ b/libsolidity/analysis/TypeChecker.cpp @@ -337,7 +337,7 @@ bool TypeChecker::visit(FunctionDefinition const& _function) if (_function.libraryFunction()) m_errorReporter.typeError(7708_error, _function.location(), "Library functions cannot be payable."); if (_function.isOrdinary() && !_function.isPartOfExternalInterface()) - m_errorReporter.typeError(5587_error, _function.location(), "Internal functions cannot be payable."); + m_errorReporter.typeError(5587_error, _function.location(), "\"internal\" and \"private\" functions cannot be payable."); } auto checkArgumentAndReturnParameter = [&](VariableDeclaration const& var) { if (type(var)->category() == Type::Category::Mapping) diff --git a/test/libsolidity/syntaxTests/nameAndTypeResolution/356_payable_internal.sol b/test/libsolidity/syntaxTests/nameAndTypeResolution/356_payable_internal.sol index 2bf491b5d..a21a18d9c 100644 --- a/test/libsolidity/syntaxTests/nameAndTypeResolution/356_payable_internal.sol +++ b/test/libsolidity/syntaxTests/nameAndTypeResolution/356_payable_internal.sol @@ -2,4 +2,4 @@ contract test { function f() payable internal {} } // ---- -// TypeError 5587: (20-52): Internal functions cannot be payable. +// TypeError 5587: (20-52): "internal" and "private" functions cannot be payable. diff --git a/test/libsolidity/syntaxTests/nameAndTypeResolution/357_payable_private.sol b/test/libsolidity/syntaxTests/nameAndTypeResolution/357_payable_private.sol index fcc0dba35..3436ce76f 100644 --- a/test/libsolidity/syntaxTests/nameAndTypeResolution/357_payable_private.sol +++ b/test/libsolidity/syntaxTests/nameAndTypeResolution/357_payable_private.sol @@ -2,4 +2,4 @@ contract test { function f() payable private {} } // ---- -// TypeError 5587: (20-51): Internal functions cannot be payable. +// TypeError 5587: (20-51): "internal" and "private" functions cannot be payable. From aed424f23e9f671e968377381ef138eb3c34489d Mon Sep 17 00:00:00 2001 From: Alexander Arlt Date: Tue, 23 Jun 2020 12:17:15 -0500 Subject: [PATCH 14/22] Initial docker image creation workflows. --- .circleci/config.yml | 19 +----- .github/workflows/buildpack-deps.yml | 42 ++++++++++++ .gitignore | 2 +- scripts/ci/build.sh | 23 +++++++ scripts/ci/build_ossfuzz.sh | 15 ++++ scripts/ci/buildpack-deps_test_emscripten.sh | 1 + ...pack-deps_test_ubuntu1604.clang.ossfuzz.sh | 1 + scripts/ci/buildpack-deps_test_ubuntu1804.sh | 1 + .../buildpack-deps_test_ubuntu2004.clang.sh | 1 + scripts/ci/buildpack-deps_test_ubuntu2004.sh | 1 + scripts/ci/docker_upgrade.sh | 68 +++++++++++++++++++ .../buildpack-deps}/Dockerfile.emscripten | 2 + .../Dockerfile.ubuntu1604.clang.ossfuzz | 2 + .../buildpack-deps}/Dockerfile.ubuntu1804 | 2 + .../buildpack-deps}/Dockerfile.ubuntu2004 | 2 + .../Dockerfile.ubuntu2004.clang | 2 + scripts/docker/buildpack-deps/README.md | 35 ++++++++++ .../docker/buildpack-deps}/emscripten.jam | 0 18 files changed, 201 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/buildpack-deps.yml create mode 100755 scripts/ci/build.sh create mode 100755 scripts/ci/build_ossfuzz.sh create mode 120000 scripts/ci/buildpack-deps_test_emscripten.sh create mode 120000 scripts/ci/buildpack-deps_test_ubuntu1604.clang.ossfuzz.sh create mode 120000 scripts/ci/buildpack-deps_test_ubuntu1804.sh create mode 120000 scripts/ci/buildpack-deps_test_ubuntu2004.clang.sh create mode 120000 scripts/ci/buildpack-deps_test_ubuntu2004.sh create mode 100755 scripts/ci/docker_upgrade.sh rename {.circleci/docker => scripts/docker/buildpack-deps}/Dockerfile.emscripten (99%) rename {.circleci/docker => scripts/docker/buildpack-deps}/Dockerfile.ubuntu1604.clang.ossfuzz (99%) rename {.circleci/docker => scripts/docker/buildpack-deps}/Dockerfile.ubuntu1804 (99%) rename {.circleci/docker => scripts/docker/buildpack-deps}/Dockerfile.ubuntu2004 (99%) rename {.circleci/docker => scripts/docker/buildpack-deps}/Dockerfile.ubuntu2004.clang (99%) create mode 100644 scripts/docker/buildpack-deps/README.md rename {.circleci/docker => scripts/docker/buildpack-deps}/emscripten.jam (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index d17f13baa..288749974 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -33,26 +33,11 @@ defaults: - run_build: &run_build name: Build - command: | - set -ex - if [ "$CIRCLE_BRANCH" = release -o -n "$CIRCLE_TAG" -o -n "$FORCE_RELEASE" ]; then echo -n > prerelease.txt; else date -u +"nightly.%Y.%-m.%-d" > prerelease.txt; fi - echo -n "$CIRCLE_SHA1" > commit_hash.txt - mkdir -p build - cd build - [ -n "$COVERAGE" -a "$CIRCLE_BRANCH" != release -a -z "$CIRCLE_TAG" ] && CMAKE_OPTIONS="$CMAKE_OPTIONS -DCOVERAGE=ON" - cmake .. -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-Release} $CMAKE_OPTIONS -G "Unix Makefiles" - make -j4 + command: scripts/ci/build.sh - run_build_ossfuzz: &run_build_ossfuzz name: Build_ossfuzz - command: | - mkdir -p build - cd build - protoc --proto_path=../test/tools/ossfuzz yulProto.proto --cpp_out=../test/tools/ossfuzz - protoc --proto_path=../test/tools/ossfuzz abiV2Proto.proto --cpp_out=../test/tools/ossfuzz - protoc --proto_path=../test/tools/ossfuzz solProto.proto --cpp_out=../test/tools/ossfuzz - cmake .. -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-Release} $CMAKE_OPTIONS - make ossfuzz ossfuzz_proto ossfuzz_abiv2 -j4 + command: scripts/ci/build_ossfuzz.sh - run_proofs: &run_proofs name: Correctness proofs for optimization rules diff --git a/.github/workflows/buildpack-deps.yml b/.github/workflows/buildpack-deps.yml new file mode 100644 index 000000000..48d83b77c --- /dev/null +++ b/.github/workflows/buildpack-deps.yml @@ -0,0 +1,42 @@ +name: buildpack-deps + +on: + pull_request: + branches: [ develop ] + paths: + - 'scripts/docker/buildpack-deps/Dockerfile.emscripten' + - 'scripts/docker/buildpack-deps/Dockerfile.ubuntu1604.clang.ossfuzz' + - 'scripts/docker/buildpack-deps/Dockerfile.ubuntu1804' + - 'scripts/docker/buildpack-deps/Dockerfile.ubuntu2004.clang' + - 'scripts/docker/buildpack-deps/Dockerfile.ubuntu2004' + +jobs: + buildpack-deps: + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DOCKER_REPOSITORY: docker.pkg.github.com/${{ github.repository }} + IMAGE_NAME: buildpack-deps + + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + image_variant: [emscripten, ubuntu1604.clang.ossfuzz, ubuntu1804, ubuntu2004.clang, ubuntu2004] + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Upgrade ${{ env.IMAGE_NAME }}-${{ matrix.image_variant }} + run: | + echo "${GITHUB_TOKEN}" | docker login docker.pkg.github.com -u "${GITHUB_ACTOR}" --password-stdin + scripts/ci/docker_upgrade.sh ${{ env.IMAGE_NAME }} ${{ matrix.image_variant }} ${{ env.DOCKER_REPOSITORY }} + docker logout docker.pkg.github.com + + - name: comment PR + if: "env.DOCKER_IMAGE" + uses: aarlt/comment-on-pr@v1.2.0 + with: + msg: "`${{ env.DOCKER_IMAGE }} ${{ env.DOCKER_REPO_DIGEST }}`." \ No newline at end of file diff --git a/.gitignore b/.gitignore index 67e450759..749860301 100644 --- a/.gitignore +++ b/.gitignore @@ -32,7 +32,7 @@ prerelease.txt # Build directory build/ -build*/ +/build*/ emscripten_build/ docs/_build __pycache__ diff --git a/scripts/ci/build.sh b/scripts/ci/build.sh new file mode 100755 index 000000000..1c8824abf --- /dev/null +++ b/scripts/ci/build.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -ex + +ROOTDIR="$(dirname "$0")/../.." +cd "${ROOTDIR}" + +# shellcheck disable=SC2166 +if [ "$CIRCLE_BRANCH" = release -o -n "$CIRCLE_TAG" -o -n "$FORCE_RELEASE" ]; then echo -n >prerelease.txt; else date -u +"nightly.%Y.%-m.%-d" >prerelease.txt; fi +if [ -n "$CIRCLE_SHA1" ] +then + echo -n "$CIRCLE_SHA1" >commit_hash.txt +fi + +mkdir -p build +cd build + +# shellcheck disable=SC2166 +[ -n "$COVERAGE" -a "$CIRCLE_BRANCH" != release -a -z "$CIRCLE_TAG" ] && CMAKE_OPTIONS="$CMAKE_OPTIONS -DCOVERAGE=ON" + +# shellcheck disable=SC2086 +cmake .. -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE:-Release}" $CMAKE_OPTIONS -G "Unix Makefiles" + +make -j 4 diff --git a/scripts/ci/build_ossfuzz.sh b/scripts/ci/build_ossfuzz.sh new file mode 100755 index 000000000..42f7b3a39 --- /dev/null +++ b/scripts/ci/build_ossfuzz.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -e + +ROOTDIR="$(dirname "$0")/../.." +BUILDDIR="${ROOTDIR}/build" + +mkdir -p "${BUILDDIR}" +cd "${BUILDDIR}" + +protoc --proto_path=../test/tools/ossfuzz yulProto.proto --cpp_out=../test/tools/ossfuzz +protoc --proto_path=../test/tools/ossfuzz abiV2Proto.proto --cpp_out=../test/tools/ossfuzz +protoc --proto_path=../test/tools/ossfuzz solProto.proto --cpp_out=../test/tools/ossfuzz +cmake .. -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE:-Release}" -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/libfuzzer.cmake + +make ossfuzz ossfuzz_proto ossfuzz_abiv2 -j 4 diff --git a/scripts/ci/buildpack-deps_test_emscripten.sh b/scripts/ci/buildpack-deps_test_emscripten.sh new file mode 120000 index 000000000..6e838fba7 --- /dev/null +++ b/scripts/ci/buildpack-deps_test_emscripten.sh @@ -0,0 +1 @@ +../../scripts/travis-emscripten/build_emscripten.sh \ No newline at end of file diff --git a/scripts/ci/buildpack-deps_test_ubuntu1604.clang.ossfuzz.sh b/scripts/ci/buildpack-deps_test_ubuntu1604.clang.ossfuzz.sh new file mode 120000 index 000000000..86beda7dd --- /dev/null +++ b/scripts/ci/buildpack-deps_test_ubuntu1604.clang.ossfuzz.sh @@ -0,0 +1 @@ +build_ossfuzz.sh \ No newline at end of file diff --git a/scripts/ci/buildpack-deps_test_ubuntu1804.sh b/scripts/ci/buildpack-deps_test_ubuntu1804.sh new file mode 120000 index 000000000..c07a74de4 --- /dev/null +++ b/scripts/ci/buildpack-deps_test_ubuntu1804.sh @@ -0,0 +1 @@ +build.sh \ No newline at end of file diff --git a/scripts/ci/buildpack-deps_test_ubuntu2004.clang.sh b/scripts/ci/buildpack-deps_test_ubuntu2004.clang.sh new file mode 120000 index 000000000..c07a74de4 --- /dev/null +++ b/scripts/ci/buildpack-deps_test_ubuntu2004.clang.sh @@ -0,0 +1 @@ +build.sh \ No newline at end of file diff --git a/scripts/ci/buildpack-deps_test_ubuntu2004.sh b/scripts/ci/buildpack-deps_test_ubuntu2004.sh new file mode 120000 index 000000000..c07a74de4 --- /dev/null +++ b/scripts/ci/buildpack-deps_test_ubuntu2004.sh @@ -0,0 +1 @@ +build.sh \ No newline at end of file diff --git a/scripts/ci/docker_upgrade.sh b/scripts/ci/docker_upgrade.sh new file mode 100755 index 000000000..a8212c2c2 --- /dev/null +++ b/scripts/ci/docker_upgrade.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +set -e + +function error() { + echo >&2 "ERROR: ${1} Aborting." && false +} + +function warning() { + echo >&2 "WARNING: ${1}" +} + +[[ $# == 3 ]] || error "Expected exactly 3 parameters: '${0} '." + +IMAGE_NAME="${1}" +IMAGE_VARIANT="${2}" +DOCKER_REPOSITORY="${3}" +DOCKERFILE="scripts/docker/${IMAGE_NAME}/Dockerfile.${IMAGE_VARIANT}" + +echo "-- check_dockerfile_was_changed" + +# exit, if the dockerfile was not changed. +if git diff --quiet origin/develop HEAD -- "${DOCKERFILE}"; then + echo "${DOCKERFILE} was not changed. Nothing to do." + exit 0 +fi + +echo "-- check_version" + +PREV_VERSION=$(git diff origin/develop HEAD -- "${DOCKERFILE}" | grep -e '^\s*-LABEL\s\+version=".*"\s*$' | awk -F'"' '{ print $2 }') +NEXT_VERSION=$(git diff origin/develop HEAD -- "${DOCKERFILE}" | grep -e '^\s*+LABEL\s\+version=".*"\s*$' | awk -F'"' '{ print $2 }') + +[[ $NEXT_VERSION != "" ]] || error "No version label defined in Dockerfile. You may need to add 'LABEL version' in '${DOCKERFILE}'." + +[[ $PREV_VERSION != "" ]] || { + warning "no previous version found. Will set \$PREV_VERSION = 0." + PREV_VERSION=0 +} + +if [[ $((PREV_VERSION + 1)) != $((NEXT_VERSION)) ]]; then + error "Version label in Dockerfile was not incremented. You may need to change 'LABEL version' in '${DOCKERFILE}'." +fi + +echo "-- build_docker" + +# This is a workaround: we run `docker build` twice to prevent the `layer does not exist` problem. +# See https://github.com/moby/moby/issues/37965. +docker build "scripts/docker/${IMAGE_NAME}" --file "scripts/docker/${IMAGE_NAME}/Dockerfile.${IMAGE_VARIANT}" --tag "${IMAGE_NAME}" || + docker build "scripts/docker/${IMAGE_NAME}" --file "scripts/docker/${IMAGE_NAME}/Dockerfile.${IMAGE_VARIANT}" --tag "${IMAGE_NAME}" + +echo "-- test_docker @ '${PWD}'" + +docker run --rm --volume "${PWD}:/root/project" "${IMAGE_NAME}" "/root/project/scripts/ci/${IMAGE_NAME}_test_${IMAGE_VARIANT}.sh" + +echo "-- push_docker" + +VERSION=$(docker inspect --format='{{.Config.Labels.version}}' "${IMAGE_NAME}") +DOCKER_IMAGE_ID="${DOCKER_REPOSITORY}/${IMAGE_NAME}-${IMAGE_VARIANT}" + +docker tag "${IMAGE_NAME}" "${DOCKER_IMAGE_ID}:${VERSION}" +docker push "${DOCKER_IMAGE_ID}:${VERSION}" + +REPO_DIGEST=$(docker inspect --format='{{.RepoDigests}}' "${DOCKER_IMAGE_ID}:${VERSION}") + +docker tag "${IMAGE_NAME}" "${DOCKER_IMAGE_ID}:latest" +docker push "${DOCKER_IMAGE_ID}:latest" + +echo "::set-env name=DOCKER_IMAGE::${DOCKER_IMAGE_ID}:${VERSION}" +echo "::set-env name=DOCKER_REPO_DIGEST::${REPO_DIGEST}" diff --git a/.circleci/docker/Dockerfile.emscripten b/scripts/docker/buildpack-deps/Dockerfile.emscripten similarity index 99% rename from .circleci/docker/Dockerfile.emscripten rename to scripts/docker/buildpack-deps/Dockerfile.emscripten index d2456bb2a..8885e1302 100644 --- a/.circleci/docker/Dockerfile.emscripten +++ b/scripts/docker/buildpack-deps/Dockerfile.emscripten @@ -29,6 +29,7 @@ # make version=1.39.15 build # FROM emscripten/emsdk:1.39.15 AS base +LABEL version="0" ADD emscripten.jam /usr/src RUN set -ex; \ @@ -63,3 +64,4 @@ RUN set -ex; \ cxxflags="-s DISABLE_EXCEPTION_CATCHING=0 -Wno-unused-local-typedef -Wno-variadic-macros -Wno-c99-extensions -Wno-all" \ --prefix=/emsdk/emscripten/sdk/system install; \ rm -r /usr/src/boost_1_73_0 + diff --git a/.circleci/docker/Dockerfile.ubuntu1604.clang.ossfuzz b/scripts/docker/buildpack-deps/Dockerfile.ubuntu1604.clang.ossfuzz similarity index 99% rename from .circleci/docker/Dockerfile.ubuntu1604.clang.ossfuzz rename to scripts/docker/buildpack-deps/Dockerfile.ubuntu1604.clang.ossfuzz index f7dfc7001..add0bfec4 100644 --- a/.circleci/docker/Dockerfile.ubuntu1604.clang.ossfuzz +++ b/scripts/docker/buildpack-deps/Dockerfile.ubuntu1604.clang.ossfuzz @@ -22,6 +22,7 @@ # (c) 2016-2019 solidity contributors. #------------------------------------------------------------------------------ FROM gcr.io/oss-fuzz-base/base-clang as base +LABEL version="0" ARG DEBIAN_FRONTEND=noninteractive @@ -99,3 +100,4 @@ FROM base COPY --from=libraries /usr/lib /usr/lib COPY --from=libraries /usr/bin /usr/bin COPY --from=libraries /usr/include /usr/include + diff --git a/.circleci/docker/Dockerfile.ubuntu1804 b/scripts/docker/buildpack-deps/Dockerfile.ubuntu1804 similarity index 99% rename from .circleci/docker/Dockerfile.ubuntu1804 rename to scripts/docker/buildpack-deps/Dockerfile.ubuntu1804 index fa651900b..34f1e674b 100644 --- a/.circleci/docker/Dockerfile.ubuntu1804 +++ b/scripts/docker/buildpack-deps/Dockerfile.ubuntu1804 @@ -22,6 +22,7 @@ # (c) 2016-2019 solidity contributors. #------------------------------------------------------------------------------ FROM buildpack-deps:bionic AS base +LABEL version="0" ARG DEBIAN_FRONTEND=noninteractive @@ -91,3 +92,4 @@ FROM base COPY --from=libraries /usr/lib /usr/lib COPY --from=libraries /usr/bin /usr/bin COPY --from=libraries /usr/include /usr/include + diff --git a/.circleci/docker/Dockerfile.ubuntu2004 b/scripts/docker/buildpack-deps/Dockerfile.ubuntu2004 similarity index 99% rename from .circleci/docker/Dockerfile.ubuntu2004 rename to scripts/docker/buildpack-deps/Dockerfile.ubuntu2004 index a021b6533..6c5486bd2 100644 --- a/.circleci/docker/Dockerfile.ubuntu2004 +++ b/scripts/docker/buildpack-deps/Dockerfile.ubuntu2004 @@ -22,6 +22,7 @@ # (c) 2016-2019 solidity contributors. #------------------------------------------------------------------------------ FROM buildpack-deps:focal AS base +LABEL version="0" ARG DEBIAN_FRONTEND=noninteractive @@ -60,3 +61,4 @@ FROM base COPY --from=libraries /usr/lib /usr/lib COPY --from=libraries /usr/bin /usr/bin COPY --from=libraries /usr/include /usr/include + diff --git a/.circleci/docker/Dockerfile.ubuntu2004.clang b/scripts/docker/buildpack-deps/Dockerfile.ubuntu2004.clang similarity index 99% rename from .circleci/docker/Dockerfile.ubuntu2004.clang rename to scripts/docker/buildpack-deps/Dockerfile.ubuntu2004.clang index c8d057eb1..b38fe4f1a 100644 --- a/.circleci/docker/Dockerfile.ubuntu2004.clang +++ b/scripts/docker/buildpack-deps/Dockerfile.ubuntu2004.clang @@ -22,6 +22,7 @@ # (c) 2016-2019 solidity contributors. #------------------------------------------------------------------------------ FROM buildpack-deps:focal AS base +LABEL version="0" ARG DEBIAN_FRONTEND=noninteractive @@ -62,3 +63,4 @@ FROM base COPY --from=libraries /usr/lib /usr/lib COPY --from=libraries /usr/bin /usr/bin COPY --from=libraries /usr/include /usr/include + diff --git a/scripts/docker/buildpack-deps/README.md b/scripts/docker/buildpack-deps/README.md new file mode 100644 index 000000000..004a89e91 --- /dev/null +++ b/scripts/docker/buildpack-deps/README.md @@ -0,0 +1,35 @@ +# buildpack-deps docker images + +The `buildpack-deps` docker images are used to compile and test solidity within our CI. + +## GitHub Workflow + +The creation of the images are triggered by a single workflow, defined in `.github/workflows/buildpack-deps.yml`. +For each resulting `buildpack-deps` docker image a strategy is defined in the workflow file - the image variant. +The workflow gets triggered, if any Dockerfile defined in `scripts/docker/buildpack-deps/Dockerfile.*` were changed +within the PR. + +### Versioning + +The version of the docker images can be defined within the Dockerfile with `LABEL version`. A new docker image +will only be created and pushed, if the new version is incremented by `1` compared with the version of the Dockerfile +located in `develop`. + +### Build, Test & Push + +Note that the whole workflow - including all defined strategies (image variants) - will be triggered, +even if only a single Dockerfile was change. The full workflow will only gets executed, if the corresponding +Dockerfile was changed. The execution of workflows of unchanged Dockerfiles will not continue and just return success. +See `scripts/ci/docker_upgrade.sh`. + +If the version check was successful, the docker image will be built using the Dockerfile located in +`scripts/docker/buildpack-deps/Dockerfile.*`. + +The resulting docker image will be tested by executing +the corresponding `scripts/ci/buildpack-deps_test_*` scripts. These scripts are normally symlinked to `scripts/ci/build.sh`, +except for the `buildpack-deps-ubuntu1604.clang.ossfuzz` docker image, that is symlinked to `scripts/ci/build_ossfuzz.sh`. +These scripts `scripts/ci/build.sh` and `scripts/ci/build_ossfuzz.sh` are also used by CircleCI, see `.circleci/config.yml`. + +If the tests passed successfully, the docker image will get tagged by the version defined within the corresponding `Dockerfile`. +Finally, a comment will be added to the PR that contains the full repository, version and repository digest +of the freshly created docker image. \ No newline at end of file diff --git a/.circleci/docker/emscripten.jam b/scripts/docker/buildpack-deps/emscripten.jam similarity index 100% rename from .circleci/docker/emscripten.jam rename to scripts/docker/buildpack-deps/emscripten.jam From ca313e40030a8c52e8427947820f0594d94edffb Mon Sep 17 00:00:00 2001 From: chriseth Date: Wed, 15 Jul 2020 12:10:56 +0200 Subject: [PATCH 15/22] Use more processors to build base builds. --- .circleci/config.yml | 5 +++++ .travis.yml | 1 + scripts/ci/build.sh | 2 +- scripts/travis-emscripten/build_emscripten.sh | 2 +- 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 288749974..826c04785 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -407,8 +407,11 @@ jobs: - persist_to_workspace: *artifacts_executables b_ubu: &build_ubuntu2004 + resource_class: xlarge docker: - image: ethereum/solidity-buildpack-deps:ubuntu2004-<< pipeline.parameters.ubuntu-2004-docker-image-rev >> + environment: + MAKEFLAGS: -j 10 steps: - checkout - run: *run_build @@ -583,9 +586,11 @@ jobs: - store_artifacts: *artifacts_test_results b_ems: + resource_class: xlarge docker: - image: ethereum/solidity-buildpack-deps:emsdk-1.39.15-2 environment: + MAKEFLAGS: -j 10 TERM: xterm steps: - checkout diff --git a/.travis.yml b/.travis.yml index ef7832777..e20a4d202 100644 --- a/.travis.yml +++ b/.travis.yml @@ -47,6 +47,7 @@ env: - SOLC_TESTS=On - SOLC_STOREBYTECODE=Off - SOLC_DOCKER=Off + - MAKEFLAGS="-j 4" matrix: include: diff --git a/scripts/ci/build.sh b/scripts/ci/build.sh index 1c8824abf..a939412f6 100755 --- a/scripts/ci/build.sh +++ b/scripts/ci/build.sh @@ -20,4 +20,4 @@ cd build # shellcheck disable=SC2086 cmake .. -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE:-Release}" $CMAKE_OPTIONS -G "Unix Makefiles" -make -j 4 +make diff --git a/scripts/travis-emscripten/build_emscripten.sh b/scripts/travis-emscripten/build_emscripten.sh index 5c7e3f052..3162e13b6 100755 --- a/scripts/travis-emscripten/build_emscripten.sh +++ b/scripts/travis-emscripten/build_emscripten.sh @@ -53,7 +53,7 @@ cmake \ -DBoost_USE_STATIC_RUNTIME=1 \ -DTESTS=0 \ .. -make -j 4 soljson +make soljson # Patch soljson.js for backwards compatibility. # TODO: remove this with 0.7. # "viiiii" encodes the signature of the callback function. From 46653b2d434f026933aa4a8a244a1152ceb7009d Mon Sep 17 00:00:00 2001 From: Leonardo Alt Date: Wed, 15 Jul 2020 18:10:14 +0200 Subject: [PATCH 16/22] Fix ICE when bitwise operator on fixed bytes --- Changelog.md | 1 + libsmtutil/SolverInterface.h | 23 +++++++++++-------- libsolidity/formal/SMTEncoder.cpp | 6 +++-- libsolidity/formal/SymbolicTypes.cpp | 4 ++++ .../operators/bitwise_and_fixed_bytes.sol | 8 +++++++ .../operators/bitwise_xor_fixed_bytes.sol | 9 ++++++++ 6 files changed, 40 insertions(+), 11 deletions(-) create mode 100644 test/libsolidity/smtCheckerTests/operators/bitwise_and_fixed_bytes.sol create mode 100644 test/libsolidity/smtCheckerTests/operators/bitwise_xor_fixed_bytes.sol diff --git a/Changelog.md b/Changelog.md index 047b06ddf..4953a4e49 100644 --- a/Changelog.md +++ b/Changelog.md @@ -9,6 +9,7 @@ Compiler Features: * Yul EVM Code Transform: Free stack slots directly after visiting the right-hand-side of variable declarations instead of at the end of the statement only. Bugfixes: + * SMTChecker: Fix internal error when using bitwise operators on fixed bytes type. * Type Checker: Fix overload resolution in combination with ``{value: ...}``. * Type Checker: Fix internal compiler error related to oversized types. diff --git a/libsmtutil/SolverInterface.h b/libsmtutil/SolverInterface.h index be12c3a80..a62a39b37 100644 --- a/libsmtutil/SolverInterface.h +++ b/libsmtutil/SolverInterface.h @@ -59,10 +59,10 @@ class Expression public: explicit Expression(bool _v): Expression(_v ? "true" : "false", Kind::Bool) {} explicit Expression(std::shared_ptr _sort, std::string _name = ""): Expression(std::move(_name), {}, _sort) {} - Expression(size_t _number): Expression(std::to_string(_number), Kind::Int) {} - Expression(u256 const& _number): Expression(_number.str(), Kind::Int) {} - Expression(s256 const& _number): Expression(_number.str(), Kind::Int) {} - Expression(bigint const& _number): Expression(_number.str(), Kind::Int) {} + Expression(size_t _number): Expression(std::to_string(_number), {}, SortProvider::sintSort) {} + Expression(u256 const& _number): Expression(_number.str(), {}, SortProvider::sintSort) {} + Expression(s256 const& _number): Expression(_number.str(), {}, SortProvider::sintSort) {} + Expression(bigint const& _number): Expression(_number.str(), {}, SortProvider::sintSort) {} Expression(Expression const&) = default; Expression(Expression&&) = default; @@ -262,23 +262,28 @@ public: } friend Expression operator+(Expression _a, Expression _b) { - return Expression("+", std::move(_a), std::move(_b), Kind::Int); + auto intSort = _a.sort; + return Expression("+", {std::move(_a), std::move(_b)}, intSort); } friend Expression operator-(Expression _a, Expression _b) { - return Expression("-", std::move(_a), std::move(_b), Kind::Int); + auto intSort = _a.sort; + return Expression("-", {std::move(_a), std::move(_b)}, intSort); } friend Expression operator*(Expression _a, Expression _b) { - return Expression("*", std::move(_a), std::move(_b), Kind::Int); + auto intSort = _a.sort; + return Expression("*", {std::move(_a), std::move(_b)}, intSort); } friend Expression operator/(Expression _a, Expression _b) { - return Expression("/", std::move(_a), std::move(_b), Kind::Int); + auto intSort = _a.sort; + return Expression("/", {std::move(_a), std::move(_b)}, intSort); } friend Expression operator%(Expression _a, Expression _b) { - return Expression("mod", std::move(_a), std::move(_b), Kind::Int); + auto intSort = _a.sort; + return Expression("mod", {std::move(_a), std::move(_b)}, intSort); } friend Expression operator&(Expression _a, Expression _b) { diff --git a/libsolidity/formal/SMTEncoder.cpp b/libsolidity/formal/SMTEncoder.cpp index eea0b24e5..6a4030511 100644 --- a/libsolidity/formal/SMTEncoder.cpp +++ b/libsolidity/formal/SMTEncoder.cpp @@ -1376,9 +1376,11 @@ void SMTEncoder::bitwiseOperation(BinaryOperation const& _op) bvSize = fixedType->numBits(); isSigned = fixedType->isSigned(); } + else if (auto const* fixedBytesType = dynamic_cast(commonType)) + bvSize = fixedBytesType->numBytes() * 8; - auto bvLeft = smtutil::Expression::int2bv(expr(_op.leftExpression()), bvSize); - auto bvRight = smtutil::Expression::int2bv(expr(_op.rightExpression()), bvSize); + auto bvLeft = smtutil::Expression::int2bv(expr(_op.leftExpression(), commonType), bvSize); + auto bvRight = smtutil::Expression::int2bv(expr(_op.rightExpression(), commonType), bvSize); optional result; if (_op.getOperator() == Token::BitAnd) diff --git a/libsolidity/formal/SymbolicTypes.cpp b/libsolidity/formal/SymbolicTypes.cpp index a46b88e03..096f496c4 100644 --- a/libsolidity/formal/SymbolicTypes.cpp +++ b/libsolidity/formal/SymbolicTypes.cpp @@ -441,7 +441,11 @@ optional symbolicTypeConversion(TypePointer _from, TypePoin // case they'd need to be encoded as numbers. if (auto strType = dynamic_cast(_from)) if (_to->category() == frontend::Type::Category::FixedBytes) + { + if (strType->value().empty()) + return smtutil::Expression(size_t(0)); return smtutil::Expression(u256(toHex(util::asBytes(strType->value()), util::HexPrefix::Add))); + } return std::nullopt; } diff --git a/test/libsolidity/smtCheckerTests/operators/bitwise_and_fixed_bytes.sol b/test/libsolidity/smtCheckerTests/operators/bitwise_and_fixed_bytes.sol new file mode 100644 index 000000000..c3212bb23 --- /dev/null +++ b/test/libsolidity/smtCheckerTests/operators/bitwise_and_fixed_bytes.sol @@ -0,0 +1,8 @@ +pragma experimental SMTChecker; +contract C { + function f() public pure returns (byte) { + return (byte("") & ("")); + } +} +// ---- +// Warning 5084: (101-109): Type conversion is not yet fully supported and might yield false positives. diff --git a/test/libsolidity/smtCheckerTests/operators/bitwise_xor_fixed_bytes.sol b/test/libsolidity/smtCheckerTests/operators/bitwise_xor_fixed_bytes.sol new file mode 100644 index 000000000..3852563be --- /dev/null +++ b/test/libsolidity/smtCheckerTests/operators/bitwise_xor_fixed_bytes.sol @@ -0,0 +1,9 @@ +pragma experimental SMTChecker; +contract Simp { + function f3() public pure returns (byte) { + bytes memory y = "def"; + return y[0] ^ "e"; + } +} +// ---- +// Warning 1093: (142-152): Assertion checker does not yet implement this bitwise operator. From 99d831d7d150fc03941cbd0482d51dc54d228fae Mon Sep 17 00:00:00 2001 From: Daniel Kirchner Date: Wed, 15 Jul 2020 19:50:19 +0200 Subject: [PATCH 17/22] Extract recursive function detection from Semantics into the CallGraph. --- libyul/optimiser/CallGraphGenerator.cpp | 41 +++++++++++++++++++++++++ libyul/optimiser/CallGraphGenerator.h | 4 +++ libyul/optimiser/Semantics.cpp | 22 +------------ 3 files changed, 46 insertions(+), 21 deletions(-) diff --git a/libyul/optimiser/CallGraphGenerator.cpp b/libyul/optimiser/CallGraphGenerator.cpp index d3805e7f7..565518af0 100644 --- a/libyul/optimiser/CallGraphGenerator.cpp +++ b/libyul/optimiser/CallGraphGenerator.cpp @@ -30,6 +30,47 @@ using namespace solidity; using namespace solidity::yul; using namespace solidity::util; +namespace +{ +// TODO: This algorithm is non-optimal. +struct CallGraphCycleFinder +{ + CallGraph const& callGraph; + set containedInCycle{}; + set visited{}; + vector currentPath{}; + + void visit(YulString _function) + { + if (visited.count(_function)) + return; + if ( + auto it = find(currentPath.begin(), currentPath.end(), _function); + it != currentPath.end() + ) + containedInCycle.insert(it, currentPath.end()); + else + { + currentPath.emplace_back(_function); + if (callGraph.functionCalls.count(_function)) + for (auto const& child: callGraph.functionCalls.at(_function)) + visit(child); + currentPath.pop_back(); + visited.insert(_function); + } + } +}; +} + +set CallGraph::recursiveFunctions() const +{ + CallGraphCycleFinder cycleFinder{*this}; + // Visiting the root only is not enough, since there may be disconnected recursive functions. + for (auto const& call: functionCalls) + cycleFinder.visit(call.first); + return cycleFinder.containedInCycle; +} + CallGraph CallGraphGenerator::callGraph(Block const& _ast) { CallGraphGenerator gen; diff --git a/libyul/optimiser/CallGraphGenerator.h b/libyul/optimiser/CallGraphGenerator.h index 2a05c7564..2b57f8592 100644 --- a/libyul/optimiser/CallGraphGenerator.h +++ b/libyul/optimiser/CallGraphGenerator.h @@ -35,6 +35,10 @@ struct CallGraph { std::map> functionCalls; std::set functionsWithLoops; + /// @returns the set of functions contained in cycles in the call graph, i.e. + /// functions that are part of a (mutual) recursion. + /// Note that this does not include functions that merely call recursive functions. + std::set recursiveFunctions() const; }; /** diff --git a/libyul/optimiser/Semantics.cpp b/libyul/optimiser/Semantics.cpp index f9d831b7a..6100b1cdc 100644 --- a/libyul/optimiser/Semantics.cpp +++ b/libyul/optimiser/Semantics.cpp @@ -102,33 +102,13 @@ map SideEffectsPropagator::sideEffects( // is actually a bit different from "not movable". map ret; - for (auto const& function: _directCallGraph.functionsWithLoops) + for (auto const& function: _directCallGraph.functionsWithLoops + _directCallGraph.recursiveFunctions()) { ret[function].movable = false; ret[function].sideEffectFree = false; ret[function].sideEffectFreeIfNoMSize = false; } - // Detect recursive functions. - for (auto const& call: _directCallGraph.functionCalls) - { - // TODO we could shortcut the search as soon as we find a - // function that has as bad side-effects as we can - // ever achieve via recursion. - auto search = [&](YulString const& _functionName, util::CycleDetector& _cycleDetector, size_t) { - for (auto const& callee: _directCallGraph.functionCalls.at(_functionName)) - if (!_dialect.builtin(callee)) - if (_cycleDetector.run(callee)) - return; - }; - if (util::CycleDetector(search).run(call.first)) - { - ret[call.first].movable = false; - ret[call.first].sideEffectFree = false; - ret[call.first].sideEffectFreeIfNoMSize = false; - } - } - for (auto const& call: _directCallGraph.functionCalls) { YulString funName = call.first; From 0b9bb84c39a25b0361d8fe30eddf47c8c4c441d4 Mon Sep 17 00:00:00 2001 From: chriseth Date: Wed, 15 Jul 2020 13:05:08 +0200 Subject: [PATCH 18/22] Run tests in parallel. --- .circleci/config.yml | 1 + .circleci/soltest_all.sh | 47 ++++++++++++++++++++++++++++++++++++---- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 826c04785..df0eba03d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -139,6 +139,7 @@ defaults: - test_ubuntu2004: &test_ubuntu2004 docker: - image: ethereum/solidity-buildpack-deps:ubuntu2004-<< pipeline.parameters.ubuntu-2004-docker-image-rev >> + parallelism: 6 steps: - checkout - attach_workspace: diff --git a/.circleci/soltest_all.sh b/.circleci/soltest_all.sh index 86f172a04..7b1fb053b 100755 --- a/.circleci/soltest_all.sh +++ b/.circleci/soltest_all.sh @@ -28,10 +28,49 @@ set -e REPODIR="$(realpath $(dirname $0)/..)" -EVM=istanbul OPTIMIZE=1 ABI_ENCODER_V2=1 ${REPODIR}/.circleci/soltest.sh +EVM_VALUES=(homestead byzantium constantinople petersburg istanbul) +OPTIMIZE_VALUES=(0 1) +STEPS=$(( 1 + ${#EVM_VALUES[@]} * ${#OPTIMIZE_VALUES[@]} )) -for OPTIMIZE in 0 1; do - for EVM in homestead byzantium constantinople petersburg istanbul; do - EVM=$EVM OPTIMIZE=$OPTIMIZE BOOST_TEST_ARGS="-t !@nooptions" ${REPODIR}/.circleci/soltest.sh +if (( $CIRCLE_NODE_TOTAL )) && (( $CIRCLE_NODE_TOTAL > 1 )) +then + # Run step 1 as the only step on the first executor + # and evenly distribute the other steps among + # the other executors. + # The first step takes much longer than the other steps. + if (( $CIRCLE_NODE_INDEX == 0 )) + then + RUN_STEPS="1" + else + export CIRCLE_NODE_INDEX=$(($CIRCLE_NODE_INDEX - 1)) + export CIRCLE_NODE_TOTAL=$(($CIRCLE_NODE_TOTAL - 1)) + RUN_STEPS=$(seq 2 "$STEPS" | circleci tests split) + fi +else + RUN_STEPS=$(seq "$STEPS") +fi + +# turn newlines into spaces +RUN_STEPS=$(echo $RUN_STEPS) + +echo "Running steps $RUN_STEPS..." + +STEP=1 + +[[ " $RUN_STEPS " =~ " $STEP " ]] && EVM=istanbul OPTIMIZE=1 ABI_ENCODER_V2=1 "${REPODIR}/.circleci/soltest.sh" +STEP=$(($STEP + 1)) + +for OPTIMIZE in ${OPTIMIZE_VALUES[@]} +do + for EVM in ${EVM_VALUES[@]} + do + [[ " $RUN_STEPS " =~ " $STEP " ]] && EVM="$EVM" OPTIMIZE="$OPTIMIZE" BOOST_TEST_ARGS="-t !@nooptions" "${REPODIR}/.circleci/soltest.sh" + STEP=$(($STEP + 1)) done done + +if (($STEP != $STEPS + 1)) +then + echo "Step counter not properly adjusted!" >2 + exit 1 +fi From f47981d484606172af1a393ea2d107eee48e11be Mon Sep 17 00:00:00 2001 From: Mathias Baumann Date: Mon, 8 Jun 2020 17:50:55 +0200 Subject: [PATCH 19/22] Prepare testcase for avoiding double cleanup --- test/libsolidity/SolidityOptimizer.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/libsolidity/SolidityOptimizer.cpp b/test/libsolidity/SolidityOptimizer.cpp index 4062b1928..68c500e8c 100644 --- a/test/libsolidity/SolidityOptimizer.cpp +++ b/test/libsolidity/SolidityOptimizer.cpp @@ -708,6 +708,18 @@ BOOST_AUTO_TEST_CASE(shift_optimizer_bug) compareVersions("g(uint256)", u256(-1)); } +BOOST_AUTO_TEST_CASE(avoid_double_cleanup) +{ + char const* sourceCode = R"( + contract C { + receive() external payable { + abi.encodePacked(uint200(0)); + } + } + )"; + compileBothVersions(sourceCode, 0, "C", 50); + BOOST_CHECK_EQUAL(numInstructions(m_nonOptimizedBytecode, Instruction::AND), 2); +} BOOST_AUTO_TEST_SUITE_END() From d3af598c49b42f62c56f250f72546686a2bd83a6 Mon Sep 17 00:00:00 2001 From: Mathias Baumann Date: Mon, 8 Jun 2020 17:52:40 +0200 Subject: [PATCH 20/22] CodeGen: Avoid double cleanup when copying to memory --- Changelog.md | 1 + libsolidity/codegen/CompilerUtils.cpp | 19 +++++++++++++------ libsolidity/codegen/CompilerUtils.h | 6 ++++-- test/libsolidity/SolidityOptimizer.cpp | 3 ++- 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/Changelog.md b/Changelog.md index 4953a4e49..0f0e533b9 100644 --- a/Changelog.md +++ b/Changelog.md @@ -12,6 +12,7 @@ Bugfixes: * SMTChecker: Fix internal error when using bitwise operators on fixed bytes type. * Type Checker: Fix overload resolution in combination with ``{value: ...}``. * Type Checker: Fix internal compiler error related to oversized types. + * Code Generator: Avoid double cleanup when copying to memory. Compiler Features: * Build System: Update internal dependency of jsoncpp to 1.9.3. diff --git a/libsolidity/codegen/CompilerUtils.cpp b/libsolidity/codegen/CompilerUtils.cpp index b3fc9da2e..2d7b38b25 100644 --- a/libsolidity/codegen/CompilerUtils.cpp +++ b/libsolidity/codegen/CompilerUtils.cpp @@ -180,7 +180,7 @@ void CompilerUtils::storeInMemory(unsigned _offset) m_context << u256(_offset) << Instruction::MSTORE; } -void CompilerUtils::storeInMemoryDynamic(Type const& _type, bool _padToWordBoundaries) +void CompilerUtils::storeInMemoryDynamic(Type const& _type, bool _padToWordBoundaries, bool _cleanup) { // process special types (Reference, StringLiteral, Function) if (auto ref = dynamic_cast(&_type)) @@ -189,7 +189,7 @@ void CompilerUtils::storeInMemoryDynamic(Type const& _type, bool _padToWordBound ref->location() == DataLocation::Memory, "Only in-memory reference type can be stored." ); - storeInMemoryDynamic(*TypeProvider::uint256(), _padToWordBoundaries); + storeInMemoryDynamic(*TypeProvider::uint256(), _padToWordBoundaries, _cleanup); } else if (auto str = dynamic_cast(&_type)) { @@ -212,7 +212,7 @@ void CompilerUtils::storeInMemoryDynamic(Type const& _type, bool _padToWordBound } else if (_type.isValueType()) { - unsigned numBytes = prepareMemoryStore(_type, _padToWordBoundaries); + unsigned numBytes = prepareMemoryStore(_type, _padToWordBoundaries, _cleanup); m_context << Instruction::DUP2 << Instruction::MSTORE; m_context << u256(numBytes) << Instruction::ADD; } @@ -463,6 +463,7 @@ void CompilerUtils::encodeToMemory( } else { + bool needCleanup = true; copyToStackTop(argSize - stackPos + dynPointers + 2, _givenTypes[i]->sizeOnStack()); solAssert(!!targetType, "Externalable type expected."); TypePointer type = targetType; @@ -481,7 +482,11 @@ void CompilerUtils::encodeToMemory( ) type = _givenTypes[i]; // delay conversion else + { convertType(*_givenTypes[i], *targetType, true); + needCleanup = false; + } + if (auto arrayType = dynamic_cast(type)) ArrayUtils(m_context).copyArrayToMemory(*arrayType, _padToWordBoundaries); else if (auto arraySliceType = dynamic_cast(type)) @@ -495,7 +500,7 @@ void CompilerUtils::encodeToMemory( ArrayUtils(m_context).copyArrayToMemory(arraySliceType->arrayType(), _padToWordBoundaries); } else - storeInMemoryDynamic(*type, _padToWordBoundaries); + storeInMemoryDynamic(*type, _padToWordBoundaries, needCleanup); } stackPos += _givenTypes[i]->sizeOnStack(); } @@ -1499,7 +1504,7 @@ void CompilerUtils::rightShiftNumberOnStack(unsigned _bits) m_context << (u256(1) << _bits) << Instruction::SWAP1 << Instruction::DIV; } -unsigned CompilerUtils::prepareMemoryStore(Type const& _type, bool _padToWords) +unsigned CompilerUtils::prepareMemoryStore(Type const& _type, bool _padToWords, bool _cleanup) { solAssert( _type.sizeOnStack() == 1, @@ -1522,7 +1527,9 @@ unsigned CompilerUtils::prepareMemoryStore(Type const& _type, bool _padToWords) bool leftAligned = _type.category() == Type::Category::FixedBytes; - convertType(_type, _type, true); + if (_cleanup) + convertType(_type, _type, true); + if (numBytes != 32 && !leftAligned && !_padToWords) // shift the value accordingly before storing leftShiftNumberOnStack((32 - numBytes) * 8); diff --git a/libsolidity/codegen/CompilerUtils.h b/libsolidity/codegen/CompilerUtils.h index a3c266bde..a65edb8f2 100644 --- a/libsolidity/codegen/CompilerUtils.h +++ b/libsolidity/codegen/CompilerUtils.h @@ -107,10 +107,11 @@ public: /// and also updates that. For reference types, only copies the data pointer. Fails for /// non-memory-references. /// @param _padToWords if true, adds zeros to pad to multiple of 32 bytes. Array elements + /// @param _cleanup if true, adds code to cleanup the value before storing it. /// are always padded (except for byte arrays), regardless of this parameter. /// Stack pre: memory_offset value... /// Stack post: (memory_offset+length) - void storeInMemoryDynamic(Type const& _type, bool _padToWords = true); + void storeInMemoryDynamic(Type const& _type, bool _padToWords = true, bool _cleanup = true); /// Creates code that unpacks the arguments according to their types specified by a vector of TypePointers. /// From memory if @a _fromMemory is true, otherwise from call data. @@ -309,7 +310,8 @@ private: void cleanHigherOrderBits(IntegerType const& _typeOnStack); /// Prepares the given type for storing in memory by shifting it if necessary. - unsigned prepareMemoryStore(Type const& _type, bool _padToWords); + /// @param _cleanup if true, also cleanup the value when preparing to store it in memory + unsigned prepareMemoryStore(Type const& _type, bool _padToWords, bool _cleanup = true); /// Loads type from memory assuming memory offset is on stack top. unsigned loadFromMemoryHelper(Type const& _type, bool _fromCalldata, bool _padToWords); diff --git a/test/libsolidity/SolidityOptimizer.cpp b/test/libsolidity/SolidityOptimizer.cpp index 68c500e8c..c4f7a2e71 100644 --- a/test/libsolidity/SolidityOptimizer.cpp +++ b/test/libsolidity/SolidityOptimizer.cpp @@ -718,7 +718,8 @@ BOOST_AUTO_TEST_CASE(avoid_double_cleanup) } )"; compileBothVersions(sourceCode, 0, "C", 50); - BOOST_CHECK_EQUAL(numInstructions(m_nonOptimizedBytecode, Instruction::AND), 2); + // Check that there is no double AND instruction in the resulting code + BOOST_CHECK_EQUAL(numInstructions(m_nonOptimizedBytecode, Instruction::AND), 1); } BOOST_AUTO_TEST_SUITE_END() From f7e180fb8f1283a4e0bd16d9f16bafd8daaf196d Mon Sep 17 00:00:00 2001 From: Alexander Arlt Date: Wed, 15 Jul 2020 10:42:15 -0500 Subject: [PATCH 21/22] Push images to docker hub. --- .github/workflows/buildpack-deps.yml | 7 ++++--- scripts/ci/docker_upgrade.sh | 13 +++++-------- scripts/docker/buildpack-deps/Dockerfile.emscripten | 2 +- .../Dockerfile.ubuntu1604.clang.ossfuzz | 2 +- scripts/docker/buildpack-deps/Dockerfile.ubuntu1804 | 2 +- scripts/docker/buildpack-deps/Dockerfile.ubuntu2004 | 2 +- .../buildpack-deps/Dockerfile.ubuntu2004.clang | 2 +- 7 files changed, 14 insertions(+), 16 deletions(-) diff --git a/.github/workflows/buildpack-deps.yml b/.github/workflows/buildpack-deps.yml index 48d83b77c..2e2ea547f 100644 --- a/.github/workflows/buildpack-deps.yml +++ b/.github/workflows/buildpack-deps.yml @@ -14,7 +14,8 @@ jobs: buildpack-deps: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - DOCKER_REPOSITORY: docker.pkg.github.com/${{ github.repository }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + DOCKER_REPOSITORY: solbuildpackpusher/solidity-buildpack-deps IMAGE_NAME: buildpack-deps runs-on: ubuntu-latest @@ -31,9 +32,9 @@ jobs: - name: Upgrade ${{ env.IMAGE_NAME }}-${{ matrix.image_variant }} run: | - echo "${GITHUB_TOKEN}" | docker login docker.pkg.github.com -u "${GITHUB_ACTOR}" --password-stdin + echo ${DOCKERHUB_TOKEN} | docker login -u solbuildpackpusher --password-stdin scripts/ci/docker_upgrade.sh ${{ env.IMAGE_NAME }} ${{ matrix.image_variant }} ${{ env.DOCKER_REPOSITORY }} - docker logout docker.pkg.github.com + docker logout - name: comment PR if: "env.DOCKER_IMAGE" diff --git a/scripts/ci/docker_upgrade.sh b/scripts/ci/docker_upgrade.sh index a8212c2c2..16d622230 100755 --- a/scripts/ci/docker_upgrade.sh +++ b/scripts/ci/docker_upgrade.sh @@ -54,15 +54,12 @@ docker run --rm --volume "${PWD}:/root/project" "${IMAGE_NAME}" "/root/project/s echo "-- push_docker" VERSION=$(docker inspect --format='{{.Config.Labels.version}}' "${IMAGE_NAME}") -DOCKER_IMAGE_ID="${DOCKER_REPOSITORY}/${IMAGE_NAME}-${IMAGE_VARIANT}" +DOCKER_IMAGE_ID="${DOCKER_REPOSITORY}:${IMAGE_VARIANT}" -docker tag "${IMAGE_NAME}" "${DOCKER_IMAGE_ID}:${VERSION}" -docker push "${DOCKER_IMAGE_ID}:${VERSION}" +docker tag "${IMAGE_NAME}" "${DOCKER_IMAGE_ID}-${VERSION}" +docker push "${DOCKER_IMAGE_ID}-${VERSION}" -REPO_DIGEST=$(docker inspect --format='{{.RepoDigests}}' "${DOCKER_IMAGE_ID}:${VERSION}") +REPO_DIGEST=$(docker inspect --format='{{.RepoDigests}}' "${DOCKER_IMAGE_ID}-${VERSION}") -docker tag "${IMAGE_NAME}" "${DOCKER_IMAGE_ID}:latest" -docker push "${DOCKER_IMAGE_ID}:latest" - -echo "::set-env name=DOCKER_IMAGE::${DOCKER_IMAGE_ID}:${VERSION}" +echo "::set-env name=DOCKER_IMAGE::${DOCKER_IMAGE_ID}-${VERSION}" echo "::set-env name=DOCKER_REPO_DIGEST::${REPO_DIGEST}" diff --git a/scripts/docker/buildpack-deps/Dockerfile.emscripten b/scripts/docker/buildpack-deps/Dockerfile.emscripten index 8885e1302..2d9ce401d 100644 --- a/scripts/docker/buildpack-deps/Dockerfile.emscripten +++ b/scripts/docker/buildpack-deps/Dockerfile.emscripten @@ -29,7 +29,7 @@ # make version=1.39.15 build # FROM emscripten/emsdk:1.39.15 AS base -LABEL version="0" +LABEL version="1" ADD emscripten.jam /usr/src RUN set -ex; \ diff --git a/scripts/docker/buildpack-deps/Dockerfile.ubuntu1604.clang.ossfuzz b/scripts/docker/buildpack-deps/Dockerfile.ubuntu1604.clang.ossfuzz index add0bfec4..ceea17d81 100644 --- a/scripts/docker/buildpack-deps/Dockerfile.ubuntu1604.clang.ossfuzz +++ b/scripts/docker/buildpack-deps/Dockerfile.ubuntu1604.clang.ossfuzz @@ -22,7 +22,7 @@ # (c) 2016-2019 solidity contributors. #------------------------------------------------------------------------------ FROM gcr.io/oss-fuzz-base/base-clang as base -LABEL version="0" +LABEL version="1" ARG DEBIAN_FRONTEND=noninteractive diff --git a/scripts/docker/buildpack-deps/Dockerfile.ubuntu1804 b/scripts/docker/buildpack-deps/Dockerfile.ubuntu1804 index 34f1e674b..51eed0a9e 100644 --- a/scripts/docker/buildpack-deps/Dockerfile.ubuntu1804 +++ b/scripts/docker/buildpack-deps/Dockerfile.ubuntu1804 @@ -22,7 +22,7 @@ # (c) 2016-2019 solidity contributors. #------------------------------------------------------------------------------ FROM buildpack-deps:bionic AS base -LABEL version="0" +LABEL version="1" ARG DEBIAN_FRONTEND=noninteractive diff --git a/scripts/docker/buildpack-deps/Dockerfile.ubuntu2004 b/scripts/docker/buildpack-deps/Dockerfile.ubuntu2004 index 6c5486bd2..591e430af 100644 --- a/scripts/docker/buildpack-deps/Dockerfile.ubuntu2004 +++ b/scripts/docker/buildpack-deps/Dockerfile.ubuntu2004 @@ -22,7 +22,7 @@ # (c) 2016-2019 solidity contributors. #------------------------------------------------------------------------------ FROM buildpack-deps:focal AS base -LABEL version="0" +LABEL version="1" ARG DEBIAN_FRONTEND=noninteractive diff --git a/scripts/docker/buildpack-deps/Dockerfile.ubuntu2004.clang b/scripts/docker/buildpack-deps/Dockerfile.ubuntu2004.clang index b38fe4f1a..c1663fdba 100644 --- a/scripts/docker/buildpack-deps/Dockerfile.ubuntu2004.clang +++ b/scripts/docker/buildpack-deps/Dockerfile.ubuntu2004.clang @@ -22,7 +22,7 @@ # (c) 2016-2019 solidity contributors. #------------------------------------------------------------------------------ FROM buildpack-deps:focal AS base -LABEL version="0" +LABEL version="1" ARG DEBIAN_FRONTEND=noninteractive From 34c34bda264d2adee6ac54adea9bdcaf7f383662 Mon Sep 17 00:00:00 2001 From: Daniel Kirchner Date: Wed, 15 Jul 2020 12:44:35 +0200 Subject: [PATCH 22/22] Switch CI to new docker images. --- .circleci/config.yml | 45 ++++++++++++++++++++----------------- .travis.yml | 2 +- scripts/build_emscripten.sh | 3 ++- 3 files changed, 27 insertions(+), 23 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index df0eba03d..4a48caf54 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,18 +7,21 @@ # - ems: Emscripten version: 2.1 parameters: - ubuntu-1804-docker-image-rev: + ubuntu-1804-docker-image: type: string - default: "4" - ubuntu-2004-docker-image-rev: + default: "solbuildpackpusher/solidity-buildpack-deps@sha256:4484ac3da8fdc337cc77a7a7be1af71cd0f28f9c890d934f1d6ae7532beb66b1" + ubuntu-2004-docker-image: type: string - default: "2" - ubuntu-2004-clang-docker-image-rev: + default: "solbuildpackpusher/solidity-buildpack-deps@sha256:48b5bb6b91ac7dddfe9345c88876ebed126c652258800f114caed69db73b29bf" + ubuntu-2004-clang-docker-image: type: string - default: "2" - ubuntu-1604-clang-ossfuzz-docker-image-rev: + default: "solbuildpackpusher/solidity-buildpack-deps@sha256:d8775de58167db5a11690fdb6ef639317fe1e579ec5d46e9732d2d903b55d135" + ubuntu-1604-clang-ossfuzz-docker-image: type: string - default: "2" + default: "solbuildpackpusher/solidity-buildpack-deps@sha256:db52f3257396814215744a19904e42c07e040ab36b68be72a27ba71ad2f1083c" + emscripten-docker-image: + type: string + default: "solbuildpackpusher/solidity-buildpack-deps@sha256:d557d015918c3cf68b0d22839bab41013f0757b651a7fef21595f89721dbebcc" defaults: @@ -116,7 +119,7 @@ defaults: - test_ubuntu1604_clang: &test_ubuntu1604_clang docker: - - image: ethereum/solidity-buildpack-deps:ubuntu1604-clang-ossfuzz-<< pipeline.parameters.ubuntu-1604-clang-ossfuzz-docker-image-rev >> + - image: << pipeline.parameters.ubuntu-1604-clang-ossfuzz-docker-image >> steps: - checkout - attach_workspace: @@ -127,7 +130,7 @@ defaults: - test_ubuntu2004_clang: &test_ubuntu2004_clang docker: - - image: ethereum/solidity-buildpack-deps:ubuntu2004-clang-<< pipeline.parameters.ubuntu-2004-clang-docker-image-rev >> + - image: << pipeline.parameters.ubuntu-2004-clang-docker-image >> steps: - checkout - attach_workspace: @@ -138,7 +141,7 @@ defaults: - test_ubuntu2004: &test_ubuntu2004 docker: - - image: ethereum/solidity-buildpack-deps:ubuntu2004-<< pipeline.parameters.ubuntu-2004-docker-image-rev >> + - image: << pipeline.parameters.ubuntu-2004-docker-image >> parallelism: 6 steps: - checkout @@ -374,7 +377,7 @@ jobs: chk_docs_pragma_min_version: docker: - - image: ethereum/solidity-buildpack-deps:ubuntu2004-<< pipeline.parameters.ubuntu-2004-docker-image-rev >> + - image: << pipeline.parameters.ubuntu-2004-docker-image >> environment: TERM: xterm steps: @@ -383,7 +386,7 @@ jobs: b_ubu_clang: &build_ubuntu2004_clang docker: - - image: ethereum/solidity-buildpack-deps:ubuntu2004-clang-<< pipeline.parameters.ubuntu-2004-clang-docker-image-rev >> + - image: << pipeline.parameters.ubuntu-2004-clang-docker-image >> environment: CC: clang CXX: clang++ @@ -396,7 +399,7 @@ jobs: b_ubu_asan_clang: &build_ubuntu2004_clang docker: - - image: ethereum/solidity-buildpack-deps:ubuntu2004-clang-<< pipeline.parameters.ubuntu-2004-clang-docker-image-rev >> + - image: << pipeline.parameters.ubuntu-2004-clang-docker-image >> environment: CC: clang CXX: clang++ @@ -410,7 +413,7 @@ jobs: b_ubu: &build_ubuntu2004 resource_class: xlarge docker: - - image: ethereum/solidity-buildpack-deps:ubuntu2004-<< pipeline.parameters.ubuntu-2004-docker-image-rev >> + - image: << pipeline.parameters.ubuntu-2004-docker-image >> environment: MAKEFLAGS: -j 10 steps: @@ -427,7 +430,7 @@ jobs: b_ubu18: &build_ubuntu1804 docker: - - image: ethereum/solidity-buildpack-deps:ubuntu1804-<< pipeline.parameters.ubuntu-1804-docker-image-rev >> + - image: << pipeline.parameters.ubuntu-1804-docker-image >> environment: CMAKE_OPTIONS: -DCMAKE_CXX_FLAGS=-O2 CMAKE_BUILD_TYPE: RelWithDebugInfo @@ -481,7 +484,7 @@ jobs: b_ubu_ossfuzz: &build_ubuntu1604_clang docker: - - image: ethereum/solidity-buildpack-deps:ubuntu1604-clang-ossfuzz-<< pipeline.parameters.ubuntu-1604-clang-ossfuzz-docker-image-rev >> + - image: << pipeline.parameters.ubuntu-1604-clang-ossfuzz-docker-image >> environment: CC: clang CXX: clang++ @@ -589,7 +592,7 @@ jobs: b_ems: resource_class: xlarge docker: - - image: ethereum/solidity-buildpack-deps:emsdk-1.39.15-2 + - image: << pipeline.parameters.emscripten-docker-image >> environment: MAKEFLAGS: -j 10 TERM: xterm @@ -625,7 +628,7 @@ jobs: b_docs: docker: - - image: ethereum/solidity-buildpack-deps:ubuntu2004-<< pipeline.parameters.ubuntu-2004-docker-image-rev >> + - image: << pipeline.parameters.ubuntu-2004-docker-image >> steps: - checkout - run: *setup_prerelease_commit_hash @@ -641,7 +644,7 @@ jobs: t_ubu_soltest_enforce_yul: &t_ubu_soltest_enforce_yul docker: - - image: ethereum/solidity-buildpack-deps:ubuntu2004-<< pipeline.parameters.ubuntu-2004-docker-image-rev >> + - image: << pipeline.parameters.ubuntu-2004-docker-image >> environment: EVM: constantinople SOLTEST_FLAGS: --enforce-via-yul @@ -667,7 +670,7 @@ jobs: t_ubu_cli: &t_ubu_cli docker: - - image: ethereum/solidity-buildpack-deps:ubuntu2004-<< pipeline.parameters.ubuntu-2004-docker-image-rev >> + - image: << pipeline.parameters.ubuntu-2004-docker-image >> environment: TERM: xterm steps: diff --git a/.travis.yml b/.travis.yml index e20a4d202..3d00a974d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -113,7 +113,7 @@ matrix: before_install: - nvm install 10 - nvm use 10 - - docker pull ethereum/solidity-buildpack-deps:emsdk-1.39.15-2 + - docker pull solbuildpackpusher/solidity-buildpack-deps@sha256:d557d015918c3cf68b0d22839bab41013f0757b651a7fef21595f89721dbebcc env: - SOLC_EMSCRIPTEN=On - SOLC_INSTALL_DEPS_TRAVIS=Off diff --git a/scripts/build_emscripten.sh b/scripts/build_emscripten.sh index 7a052aa77..f4b0fe647 100755 --- a/scripts/build_emscripten.sh +++ b/scripts/build_emscripten.sh @@ -34,5 +34,6 @@ else BUILD_DIR="$1" fi -docker run -v $(pwd):/root/project -w /root/project ethereum/solidity-buildpack-deps:emsdk-1.39.15-2 \ +docker run -v $(pwd):/root/project -w /root/project \ + solbuildpackpusher/solidity-buildpack-deps@sha256:d557d015918c3cf68b0d22839bab41013f0757b651a7fef21595f89721dbebcc \ ./scripts/travis-emscripten/build_emscripten.sh $BUILD_DIR