From 5749a5d7c9d3d0c92a651543b97d8f447d339a39 Mon Sep 17 00:00:00 2001 From: Alex Beregszaszi Date: Mon, 26 Aug 2019 13:09:55 +0100 Subject: [PATCH 01/20] Introduce istanbul and berlin versions --- docs/using-the-compiler.rst | 4 +++- liblangutil/EVMVersion.h | 8 ++++++-- solc/CommandLineInterface.cpp | 2 +- test/EVMHost.cpp | 4 ++++ 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/docs/using-the-compiler.rst b/docs/using-the-compiler.rst index b63c36f19..5b2e9b8ca 100644 --- a/docs/using-the-compiler.rst +++ b/docs/using-the-compiler.rst @@ -122,6 +122,8 @@ at each version. Backward compatibility is not guaranteed between each version. - Shifting operators use shifting opcodes and thus need less gas. - ``petersburg`` (**default**) - The compiler behaves the same way as with constantinople. +- ``istanbul`` (**experimental**) +- ``berlin`` (**experimental**) .. _compiler-api: @@ -229,7 +231,7 @@ Input Description }, // Version of the EVM to compile for. // Affects type checking and code generation. Can be homestead, - // tangerineWhistle, spuriousDragon, byzantium, constantinople or petersburg + // tangerineWhistle, spuriousDragon, byzantium, constantinople, petersburg, istanbul or berlin "evmVersion": "byzantium", // Metadata settings (optional) "metadata": { diff --git a/liblangutil/EVMVersion.h b/liblangutil/EVMVersion.h index f80a18708..6fb6f7e3e 100644 --- a/liblangutil/EVMVersion.h +++ b/liblangutil/EVMVersion.h @@ -48,10 +48,12 @@ public: static EVMVersion byzantium() { return {Version::Byzantium}; } static EVMVersion constantinople() { return {Version::Constantinople}; } static EVMVersion petersburg() { return {Version::Petersburg}; } + static EVMVersion istanbul() { return {Version::Istanbul}; } + static EVMVersion berlin() { return {Version::Berlin}; } static boost::optional fromString(std::string const& _version) { - for (auto const& v: {homestead(), tangerineWhistle(), spuriousDragon(), byzantium(), constantinople(), petersburg()}) + for (auto const& v: {homestead(), tangerineWhistle(), spuriousDragon(), byzantium(), constantinople(), petersburg(), istanbul(), berlin()}) if (_version == v.name()) return v; return {}; @@ -70,6 +72,8 @@ public: case Version::Byzantium: return "byzantium"; case Version::Constantinople: return "constantinople"; case Version::Petersburg: return "petersburg"; + case Version::Istanbul: return "istanbul"; + case Version::Berlin: return "berlin"; } return "INVALID"; } @@ -88,7 +92,7 @@ public: bool canOverchargeGasForCall() const { return *this >= tangerineWhistle(); } private: - enum class Version { Homestead, TangerineWhistle, SpuriousDragon, Byzantium, Constantinople, Petersburg }; + enum class Version { Homestead, TangerineWhistle, SpuriousDragon, Byzantium, Constantinople, Petersburg, Istanbul, Berlin }; EVMVersion(Version _version): m_version(_version) {} diff --git a/solc/CommandLineInterface.cpp b/solc/CommandLineInterface.cpp index 34d4e8894..79d85d82f 100644 --- a/solc/CommandLineInterface.cpp +++ b/solc/CommandLineInterface.cpp @@ -641,7 +641,7 @@ Allowed options)", ( g_strEVMVersion.c_str(), po::value()->value_name("version"), - "Select desired EVM version. Either homestead, tangerineWhistle, spuriousDragon, byzantium, constantinople or petersburg (default)." + "Select desired EVM version. Either homestead, tangerineWhistle, spuriousDragon, byzantium, constantinople, petersburg (default), istanbul or berlin." ) (g_argOptimize.c_str(), "Enable bytecode optimizer.") ( diff --git a/test/EVMHost.cpp b/test/EVMHost.cpp index f18abc10d..5122f3846 100644 --- a/test/EVMHost.cpp +++ b/test/EVMHost.cpp @@ -75,6 +75,10 @@ EVMHost::EVMHost(langutil::EVMVersion _evmVersion, evmc::vm* _vm): m_evmVersion = EVMC_BYZANTIUM; else if (_evmVersion == langutil::EVMVersion::constantinople()) m_evmVersion = EVMC_CONSTANTINOPLE; + else if (_evmVersion == langutil::EVMVersion::istanbul()) + assertThrow(false, Exception, "Istanbul is not supported yet."); + else if (_evmVersion == langutil::EVMVersion::berlin()) + assertThrow(false, Exception, "Berlin is not supported yet."); else //if (_evmVersion == langutil::EVMVersion::petersburg()) m_evmVersion = EVMC_PETERSBURG; } From bcf0c1810cb050ca14739260673791e9359f31a9 Mon Sep 17 00:00:00 2001 From: Bhargava Shastry Date: Tue, 27 Aug 2019 14:57:45 +0200 Subject: [PATCH 02/20] Yul proto fuzzer: Do not reuse variable names --- test/tools/ossfuzz/protoToYul.cpp | 170 +++++++++++++++++++----------- test/tools/ossfuzz/protoToYul.h | 46 ++++++-- 2 files changed, 142 insertions(+), 74 deletions(-) diff --git a/test/tools/ossfuzz/protoToYul.cpp b/test/tools/ossfuzz/protoToYul.cpp index c2db37e46..50ef07f04 100644 --- a/test/tools/ossfuzz/protoToYul.cpp +++ b/test/tools/ossfuzz/protoToYul.cpp @@ -22,8 +22,10 @@ #include -#include #include +#include +#include +#include using namespace std; using namespace yul::test::yul_fuzzer; @@ -84,11 +86,10 @@ string ProtoConverter::visit(Literal const& _x) } } -// Reference any index in [0, m_numLiveVars-1] void ProtoConverter::visit(VarRef const& _x) { - yulAssert(m_numLiveVars > 0, "Proto fuzzer: No variables to reference."); - m_output << "x_" << (static_cast(_x.varnum()) % m_numLiveVars); + yulAssert(m_variables.size() > 0, "Proto fuzzer: No variables to reference."); + m_output << m_variables[_x.varnum() % m_variables.size()]; } void ProtoConverter::visit(Expression const& _x) @@ -202,25 +203,28 @@ void ProtoConverter::visit(BinaryOp const& _x) void ProtoConverter::visit(VarDecl const& _x) { - m_output << "let x_" << m_numLiveVars << " := "; + string varName = newVarName(); + m_output << "let " << varName << " := "; visit(_x.expr()); - m_numVarsPerScope.top()++; - m_numLiveVars++; m_output << "\n"; + m_scopes.top().insert(varName); + m_variables.push_back(varName); } void ProtoConverter::visit(EmptyVarDecl const&) { - m_output << "let x_" << m_numLiveVars++ << "\n"; - m_numVarsPerScope.top()++; + string varName = newVarName(); + m_output << "let " << varName << "\n"; + m_scopes.top().insert(varName); + m_variables.push_back(varName); } void ProtoConverter::visit(MultiVarDecl const& _x) { size_t funcId = (static_cast(_x.func_index()) % m_functionVecMultiReturnValue.size()); - int numInParams = m_functionVecMultiReturnValue.at(funcId).first; - int numOutParams = m_functionVecMultiReturnValue.at(funcId).second; + unsigned numInParams = m_functionVecMultiReturnValue.at(funcId).first; + unsigned numOutParams = m_functionVecMultiReturnValue.at(funcId).second; // Ensure that the chosen function returns at least 2 and at most 4 values yulAssert( @@ -228,28 +232,25 @@ void ProtoConverter::visit(MultiVarDecl const& _x) "Proto fuzzer: Multi variable declaration calls a function with either too few or too many output params." ); - // We must start variable numbering past the number of live variables at this point in time. - // This creates let x_p,..., x_k := - // (k-p)+1 = numOutParams - m_output << - "let " << - dev::suffixedVariableNameList("x_", m_numLiveVars, m_numLiveVars + numOutParams) << - " := "; + // Obtain variable name suffix + unsigned startIdx = counter(); + m_output << "let "; + vector varsVec = createVars(startIdx, startIdx + numOutParams); + m_output << " := "; // Create RHS of multi var decl m_output << "foo_" << functionTypeToString(NumFunctionReturns::Multiple) << "_" << funcId; m_output << "("; visitFunctionInputParams(_x, numInParams); m_output << ")\n"; - // Update live variables in scope and in total to account for the variables created by this - // multi variable declaration. - m_numVarsPerScope.top() += numOutParams; - m_numLiveVars += numOutParams; + // Add newly minted vars in the multidecl statement to current scope + addToScope(varsVec); } void ProtoConverter::visit(TypedVarDecl const& _x) { - m_output << "let x_" << m_numLiveVars; + string varName = newVarName(); + m_output << "let " << varName; switch (_x.type()) { case TypedVarDecl::BOOL: @@ -308,8 +309,8 @@ void ProtoConverter::visit(TypedVarDecl const& _x) m_output << " : u256\n"; break; } - m_numVarsPerScope.top()++; - m_numLiveVars++; + m_scopes.top().insert(varName); + m_variables.push_back(varName); } void ProtoConverter::visit(UnaryOp const& _x) @@ -693,7 +694,7 @@ void ProtoConverter::visit(CaseStmt const& _x) // a case statement containing a case literal that has already been used in a // previous case statement. If the hash (u256 value) matches a previous hash, // then we simply don't create a new case statement. - string noDoubleQuoteStr = ""; + string noDoubleQuoteStr{""}; if (literal.size() > 2) { // Ensure that all characters in the string literal except the first @@ -876,76 +877,119 @@ void ProtoConverter::visit(Statement const& _x) } } -void ProtoConverter::visit(Block const& _x) +void ProtoConverter::openScope(vector const& _funcParams) { + m_scopes.push({}); + addToScope(_funcParams); +} + +void ProtoConverter::closeScope() +{ + for (auto const& var: m_scopes.top()) + { + unsigned numErased = m_variables.size(); + m_variables.erase(remove(m_variables.begin(), m_variables.end(), var), m_variables.end()); + numErased -= m_variables.size(); + yulAssert(numErased == 1, "Proto fuzzer: More than one variable went out of scope"); + } + m_scopes.pop(); +} + +void ProtoConverter::addToScope(vector const& _vars) +{ + for (string const& i: _vars) + { + m_variables.push_back(i); + m_scopes.top().insert(i); + } +} + +void ProtoConverter::visit(Block const& _x, vector _funcParams) +{ + openScope(_funcParams); if (_x.statements_size() > 0) { - m_numVarsPerScope.push(0); m_output << "{\n"; for (auto const& st: _x.statements()) visit(st); m_output << "}\n"; - m_numLiveVars -= m_numVarsPerScope.top(); - m_numVarsPerScope.pop(); } else m_output << "{}\n"; + closeScope(); } -void ProtoConverter::visit(SpecialBlock const& _x) +void ProtoConverter::visit(SpecialBlock const& _x, vector _funcParams) { - m_numVarsPerScope.push(0); + openScope(_funcParams); m_output << "{\n"; visit(_x.var()); if (_x.statements_size() > 0) for (auto const& st: _x.statements()) visit(st); - m_numLiveVars -= m_numVarsPerScope.top(); - m_numVarsPerScope.pop(); m_output << "}\n"; + closeScope(); +} + +vector ProtoConverter::createVars(unsigned _startIdx, unsigned _endIdx) +{ + yulAssert(_endIdx > _startIdx, "Proto fuzzer: Variable indices not in range"); + string varsStr = dev::suffixedVariableNameList("x_", _startIdx, _endIdx); + m_output << varsStr; + vector varsVec; + boost::split( + varsVec, + varsStr, + boost::algorithm::is_any_of(", "), + boost::algorithm::token_compress_on + ); + + yulAssert( + varsVec.size() == (_endIdx - _startIdx), + "Proto fuzzer: Variable count mismatch during function definition" + ); + m_counter += varsVec.size(); + return varsVec; } template void ProtoConverter::createFunctionDefAndCall(T const& _x, unsigned _numInParams, unsigned _numOutParams, NumFunctionReturns _type) { yulAssert( - ((_numInParams <= modInputParams - 1) && (_numOutParams <= modOutputParams - 1)), + ((_numInParams <= s_modInputParams - 1) && (_numOutParams <= s_modOutputParams - 1)), "Proto fuzzer: Too many function I/O parameters requested." ); - // At the time of function definition creation, the number of live variables must be 0. - // This is because we always create only as many variables as we need within function scope. - yulAssert(m_numLiveVars == 0, "Proto fuzzer: Unused live variable found."); - // Signature // This creates function foo__(x_0,...,x_n) m_output << "function foo_" << functionTypeToString(_type) << "_" << m_numFunctionSets; m_output << "("; + vector varsVec = {}; if (_numInParams > 0) - m_output << dev::suffixedVariableNameList("x_", 0, _numInParams); + // Functions must use 0 as the first variable's index until function definition + // is made a statement. Once function definition as statement is implemented, + // start index becomes m_counter. + varsVec = createVars(0, _numInParams); m_output << ")"; - // Book keeping for variables in function scope and in nested scopes - m_numVarsPerScope.push(_numInParams); - m_numLiveVars += _numInParams; - + vector outVarsVec = {}; // This creates -> x_n+1,...,x_r if (_numOutParams > 0) { - m_output << " -> " << dev::suffixedVariableNameList("x_", _numInParams, _numInParams + _numOutParams); - // More bookkeeping - m_numVarsPerScope.top() += _numOutParams; - m_numLiveVars += _numOutParams; + m_output << " -> "; + if (varsVec.empty()) + varsVec = createVars(_numInParams, _numInParams + _numOutParams); + else + { + outVarsVec = createVars(_numInParams, _numInParams + _numOutParams); + varsVec.insert(varsVec.end(), outVarsVec.begin(), outVarsVec.end()); + } } + m_output << "\n"; // Body - visit(_x.statements()); - - // Ensure that variable stack is balanced - m_numLiveVars -= m_numVarsPerScope.top(); - m_numVarsPerScope.pop(); - yulAssert(m_numLiveVars == 0, "Proto fuzzer: Variable stack after function definition is unbalanced."); + visit(_x.statements(), varsVec); // Manually create a multi assignment using global variables // This prints a_0, ..., a_k-1 for this function that returns "k" values @@ -970,23 +1014,23 @@ void ProtoConverter::createFunctionDefAndCall(T const& _x, unsigned _numInParams void ProtoConverter::visit(FunctionDefinitionNoReturnVal const& _x) { - unsigned numInParams = _x.num_input_params() % modInputParams; + unsigned numInParams = _x.num_input_params() % s_modInputParams; unsigned numOutParams = 0; createFunctionDefAndCall(_x, numInParams, numOutParams, NumFunctionReturns::None); } void ProtoConverter::visit(FunctionDefinitionSingleReturnVal const& _x) { - unsigned numInParams = _x.num_input_params() % modInputParams; + unsigned numInParams = _x.num_input_params() % s_modInputParams; unsigned numOutParams = 1; createFunctionDefAndCall(_x, numInParams, numOutParams, NumFunctionReturns::Single); } void ProtoConverter::visit(FunctionDefinitionMultiReturnVal const& _x) { - unsigned numInParams = _x.num_input_params() % modInputParams; - // Synthesize at least 2 return parameters and at most (modOutputParams - 1) - unsigned numOutParams = std::max(2, _x.num_output_params() % modOutputParams); + unsigned numInParams = _x.num_input_params() % s_modInputParams; + // Synthesize at least 2 return parameters and at most (s_modOutputParams - 1) + unsigned numOutParams = std::max(2, _x.num_output_params() % s_modOutputParams); createFunctionDefAndCall(_x, numInParams, numOutParams, NumFunctionReturns::Multiple); } @@ -1014,7 +1058,7 @@ void ProtoConverter::visit(Program const& _x) m_output << "{\n"; // Create globals at the beginning // This creates let a_0, a_1, a_2, a_3 (followed by a new line) - m_output << "let " << dev::suffixedVariableNameList("a_", 0, modOutputParams - 1) << "\n"; + m_output << "let " << dev::suffixedVariableNameList("a_", 0, s_modOutputParams - 1) << "\n"; // Register function interface. Useful while visiting multi var decl/assignment statements. for (auto const& f: _x.funcs()) registerFunction(f); @@ -1037,8 +1081,8 @@ void ProtoConverter::registerFunction(FunctionDefinition const& _x) // No return and single return functions explicitly state the number of values returned registerFunction(_x.fd_zero(), NumFunctionReturns::None); registerFunction(_x.fd_one(), NumFunctionReturns::Single); - // A multi return function can have between two and (modOutputParams - 1) parameters - unsigned numOutParams = std::max(2, _x.fd_multi().num_output_params() % modOutputParams); + // A multi return function can have between two and (s_modOutputParams - 1) parameters + unsigned numOutParams = std::max(2, _x.fd_multi().num_output_params() % s_modOutputParams); registerFunction(_x.fd_multi(), NumFunctionReturns::Multiple, numOutParams); } @@ -1053,4 +1097,4 @@ std::string ProtoConverter::functionTypeToString(NumFunctionReturns _type) case NumFunctionReturns::Multiple: return "multireturn"; } -} +} \ No newline at end of file diff --git a/test/tools/ossfuzz/protoToYul.h b/test/tools/ossfuzz/protoToYul.h index c6e0ab8be..a3f617a82 100644 --- a/test/tools/ossfuzz/protoToYul.h +++ b/test/tools/ossfuzz/protoToYul.h @@ -40,8 +40,6 @@ class ProtoConverter public: ProtoConverter() { - m_numLiveVars = 0; - m_numVarsPerScope.push(m_numLiveVars); m_numFunctionSets = 0; m_inForBodyScope = false; m_inForInitScope = false; @@ -55,8 +53,18 @@ public: private: void visit(BinaryOp const&); - void visit(Block const&); - void visit(SpecialBlock const&); + /// Visits a basic block optionally adding @a _funcParams to scope. + /// @param _block Reference to a basic block of yul statements. + /// @param _funcParams List of function parameter names, defaults to + /// an empty vector. + void visit(Block const& _block, std::vector _funcParams = {}); + + /// Visits a basic block that contains a variable declaration at the + /// very beginning, optionally adding @a _funcParams to scope. + /// @param _block Reference to a basic block of yul statements. + /// @param _funcParams List of function parameter names, defaults to + /// an empty vector. + void visit(SpecialBlock const& _block, std::vector _funcParams = {}); std::string visit(Literal const&); void visit(VarRef const&); void visit(Expression const&); @@ -93,8 +101,21 @@ private: void visit(Program const&); void registerFunction(FunctionDefinition const&); + /// Creates a new scope, and optionally adds @a _funcParams to it + void openScope(std::vector const& _funcParams); + /// Closes current scope + void closeScope(); + /// Adds @a _vars to current scope + void addToScope(std::vector const& _vars); + std::string createHex(std::string const& _hexBytes); + /// Returns a new variable name. + std::string newVarName() + { + return "x_" + std::to_string(counter()); + } + /// Accepts an arbitrary string, removes all characters that are neither /// alphabets nor digits from it and returns the said string. std::string createAlphaNum(std::string const& _strBytes); @@ -112,10 +133,13 @@ private: void createFunctionDefAndCall(T const&, unsigned, unsigned, NumFunctionReturns); std::string functionTypeToString(NumFunctionReturns _type); + /// Creates variable declarations "x_<_startIdx>",...,"x_<_endIdx - 1>" + std::vector createVars(unsigned _startIdx, unsigned _endIdx); + template void registerFunction(T const& _x, NumFunctionReturns _type, unsigned _numOutputParams = 0) { - unsigned numInputParams = _x.num_input_params() % modInputParams; + unsigned numInputParams = _x.num_input_params() % s_modInputParams; switch (_type) { case NumFunctionReturns::None: @@ -145,10 +169,10 @@ private: } std::ostringstream m_output; - // Number of live variables in inner scope of a function - std::stack m_numVarsPerScope; - // Number of live variables in function scope - unsigned m_numLiveVars; + /// Scope + std::stack> m_scopes; + /// Variables + std::vector m_variables; // Set that is used for deduplicating switch case literals std::stack> m_switchLiteralSetPerScope; // Total number of function sets. A function set contains one function of each type defined by @@ -159,8 +183,8 @@ private: std::vector m_functionVecSingleReturnValue; std::vector> m_functionVecMultiReturnValue; // mod input/output parameters impose an upper bound on the number of input/output parameters a function may have. - static unsigned constexpr modInputParams = 5; - static unsigned constexpr modOutputParams = 5; + static unsigned constexpr s_modInputParams = 5; + static unsigned constexpr s_modOutputParams = 5; // predicate to keep track of for body scope bool m_inForBodyScope; // Index used for naming loop variable of bounded for loops From 1c5845e3f256ee21e9bab893a6253139257da531 Mon Sep 17 00:00:00 2001 From: chriseth Date: Tue, 13 Aug 2019 18:38:11 +0200 Subject: [PATCH 03/20] Side-effects of user-defined functions. --- .../CommonSubexpressionEliminator.cpp | 20 ++++++++++++++ .../optimiser/CommonSubexpressionEliminator.h | 10 ++++++- libyul/optimiser/DataFlowAnalyzer.cpp | 2 +- libyul/optimiser/DataFlowAnalyzer.h | 15 ++++++++++- libyul/optimiser/Semantics.cpp | 5 +++- libyul/optimiser/Semantics.h | 12 +++++++-- libyul/optimiser/Suite.cpp | 18 ++++++++----- test/libyul/YulOptimizerTest.cpp | 10 ++++--- .../movable_functions.yul | 26 +++++++++++++++++++ test/tools/yulopti.cpp | 2 +- 10 files changed, 102 insertions(+), 18 deletions(-) create mode 100644 test/libyul/yulOptimizerTests/commonSubexpressionEliminator/movable_functions.yul diff --git a/libyul/optimiser/CommonSubexpressionEliminator.cpp b/libyul/optimiser/CommonSubexpressionEliminator.cpp index 5f182ebaf..dded41ba8 100644 --- a/libyul/optimiser/CommonSubexpressionEliminator.cpp +++ b/libyul/optimiser/CommonSubexpressionEliminator.cpp @@ -23,6 +23,9 @@ #include #include +#include +#include +#include #include #include #include @@ -31,6 +34,23 @@ using namespace std; using namespace dev; using namespace yul; +void CommonSubexpressionEliminator::run(Dialect const& _dialect, Block& _ast) +{ + CommonSubexpressionEliminator cse{ + _dialect, + SideEffectsPropagator::sideEffects(_dialect, CallGraphGenerator::callGraph(_ast)) + }; + cse(_ast); +} + +CommonSubexpressionEliminator::CommonSubexpressionEliminator( + Dialect const& _dialect, + map _functionSideEffects +): + DataFlowAnalyzer(_dialect, std::move(_functionSideEffects)) +{ +} + void CommonSubexpressionEliminator::visit(Expression& _e) { bool descend = true; diff --git a/libyul/optimiser/CommonSubexpressionEliminator.h b/libyul/optimiser/CommonSubexpressionEliminator.h index 6499e6db3..07bd47fcc 100644 --- a/libyul/optimiser/CommonSubexpressionEliminator.h +++ b/libyul/optimiser/CommonSubexpressionEliminator.h @@ -27,6 +27,7 @@ namespace yul { struct Dialect; +struct SideEffects; /** * Optimisation stage that replaces expressions known to be the current value of a variable @@ -37,7 +38,14 @@ struct Dialect; class CommonSubexpressionEliminator: public DataFlowAnalyzer { public: - CommonSubexpressionEliminator(Dialect const& _dialect): DataFlowAnalyzer(_dialect) {} + /// Runs the CSE pass. @a _ast needs to be the complete AST of the program! + static void run(Dialect const& _dialect, Block& _ast); + +private: + CommonSubexpressionEliminator( + Dialect const& _dialect, + std::map _functionSideEffects + ); protected: using ASTModifier::visit; diff --git a/libyul/optimiser/DataFlowAnalyzer.cpp b/libyul/optimiser/DataFlowAnalyzer.cpp index 7c852fb47..3f3b31f2a 100644 --- a/libyul/optimiser/DataFlowAnalyzer.cpp +++ b/libyul/optimiser/DataFlowAnalyzer.cpp @@ -211,7 +211,7 @@ void DataFlowAnalyzer::handleAssignment(set const& _variables, Expres { clearValues(_variables); - MovableChecker movableChecker{m_dialect}; + MovableChecker movableChecker{m_dialect, &m_functionSideEffects}; if (_value) movableChecker.visit(*_value); else diff --git a/libyul/optimiser/DataFlowAnalyzer.h b/libyul/optimiser/DataFlowAnalyzer.h index 5df45dcb8..b8e455ea1 100644 --- a/libyul/optimiser/DataFlowAnalyzer.h +++ b/libyul/optimiser/DataFlowAnalyzer.h @@ -26,6 +26,7 @@ #include #include #include +#include // TODO avoid #include @@ -38,6 +39,7 @@ namespace yul { struct Dialect; +struct SideEffects; /** * Base class to perform data flow analysis during AST walks. @@ -67,8 +69,16 @@ struct Dialect; class DataFlowAnalyzer: public ASTModifier { public: - explicit DataFlowAnalyzer(Dialect const& _dialect): + /// @param _functionSideEffects + /// Side-effects of user-defined functions. Worst-case side-effects are assumed + /// if this is not provided or the function is not found. + /// The parameter is mostly used to determine movability of expressions. + explicit DataFlowAnalyzer( + Dialect const& _dialect, + std::map _functionSideEffects = {} + ): m_dialect(_dialect), + m_functionSideEffects(std::move(_functionSideEffects)), m_knowledgeBase(_dialect, m_value) {} @@ -124,6 +134,9 @@ protected: ) const; Dialect const& m_dialect; + /// Side-effects of user-defined functions. Worst-case side-effects are assumed + /// if this is not provided or the function is not found. + std::map m_functionSideEffects; /// Current values of variables, always movable. std::map m_value; diff --git a/libyul/optimiser/Semantics.cpp b/libyul/optimiser/Semantics.cpp index b4f3fc10a..78d1a281a 100644 --- a/libyul/optimiser/Semantics.cpp +++ b/libyul/optimiser/Semantics.cpp @@ -64,8 +64,11 @@ void SideEffectsCollector::operator()(FunctionCall const& _functionCall) { ASTWalker::operator()(_functionCall); - if (BuiltinFunction const* f = m_dialect.builtin(_functionCall.functionName.name)) + YulString functionName = _functionCall.functionName.name; + if (BuiltinFunction const* f = m_dialect.builtin(functionName)) m_sideEffects += f->sideEffects; + else if (m_functionSideEffects && m_functionSideEffects->count(functionName)) + m_sideEffects += m_functionSideEffects->at(functionName); else m_sideEffects += SideEffects::worst(); } diff --git a/libyul/optimiser/Semantics.h b/libyul/optimiser/Semantics.h index e8cfc615e..55d44a663 100644 --- a/libyul/optimiser/Semantics.h +++ b/libyul/optimiser/Semantics.h @@ -22,6 +22,7 @@ #include #include +#include #include @@ -36,7 +37,10 @@ struct Dialect; class SideEffectsCollector: public ASTWalker { public: - explicit SideEffectsCollector(Dialect const& _dialect): m_dialect(_dialect) {} + explicit SideEffectsCollector( + Dialect const& _dialect, + std::map const* _functionSideEffects = nullptr + ): m_dialect(_dialect), m_functionSideEffects(_functionSideEffects) {} SideEffectsCollector(Dialect const& _dialect, Expression const& _expression); SideEffectsCollector(Dialect const& _dialect, Statement const& _statement); SideEffectsCollector(Dialect const& _dialect, Block const& _ast); @@ -59,6 +63,7 @@ public: private: Dialect const& m_dialect; + std::map const* m_functionSideEffects = nullptr; SideEffects m_sideEffects; }; @@ -108,7 +113,10 @@ private: class MovableChecker: public SideEffectsCollector { public: - explicit MovableChecker(Dialect const& _dialect): SideEffectsCollector(_dialect) {} + explicit MovableChecker( + Dialect const& _dialect, + std::map const* _functionSideEffects = nullptr + ): SideEffectsCollector(_dialect, _functionSideEffects) {} MovableChecker(Dialect const& _dialect, Expression const& _expression); void operator()(Identifier const& _identifier) override; diff --git a/libyul/optimiser/Suite.cpp b/libyul/optimiser/Suite.cpp index 6c0fb0fbc..6c5f160db 100644 --- a/libyul/optimiser/Suite.cpp +++ b/libyul/optimiser/Suite.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -37,6 +38,7 @@ #include #include #include +#include #include #include #include @@ -114,7 +116,8 @@ void OptimiserSuite::run( RedundantAssignEliminator::run(_dialect, ast); ExpressionSimplifier::run(_dialect, ast); - CommonSubexpressionEliminator{_dialect}(ast); + + CommonSubexpressionEliminator::run(_dialect, ast); } { @@ -126,16 +129,17 @@ void OptimiserSuite::run( DeadCodeEliminator{_dialect}(ast); UnusedPruner::runUntilStabilised(_dialect, ast, reservedIdentifiers); } + { // simplify again - CommonSubexpressionEliminator{_dialect}(ast); + CommonSubexpressionEliminator::run(_dialect, ast); UnusedPruner::runUntilStabilised(_dialect, ast, reservedIdentifiers); } { // reverse SSA SSAReverser::run(ast); - CommonSubexpressionEliminator{_dialect}(ast); + CommonSubexpressionEliminator::run(_dialect, ast); UnusedPruner::runUntilStabilised(_dialect, ast, reservedIdentifiers); ExpressionJoiner::run(ast); @@ -156,7 +160,7 @@ void OptimiserSuite::run( SSATransform::run(ast, dispenser); RedundantAssignEliminator::run(_dialect, ast); RedundantAssignEliminator::run(_dialect, ast); - CommonSubexpressionEliminator{_dialect}(ast); + CommonSubexpressionEliminator::run(_dialect, ast); } { @@ -177,12 +181,12 @@ void OptimiserSuite::run( BlockFlattener{}(ast); DeadCodeEliminator{_dialect}(ast); ControlFlowSimplifier{_dialect}(ast); - CommonSubexpressionEliminator{_dialect}(ast); + CommonSubexpressionEliminator::run(_dialect, ast); SSATransform::run(ast, dispenser); RedundantAssignEliminator::run(_dialect, ast); RedundantAssignEliminator::run(_dialect, ast); UnusedPruner::runUntilStabilised(_dialect, ast, reservedIdentifiers); - CommonSubexpressionEliminator{_dialect}(ast); + CommonSubexpressionEliminator::run(_dialect, ast); } } @@ -197,7 +201,7 @@ void OptimiserSuite::run( UnusedPruner::runUntilStabilised(_dialect, ast, reservedIdentifiers); SSAReverser::run(ast); - CommonSubexpressionEliminator{_dialect}(ast); + CommonSubexpressionEliminator::run(_dialect, ast); UnusedPruner::runUntilStabilised(_dialect, ast, reservedIdentifiers); ExpressionJoiner::run(ast); diff --git a/test/libyul/YulOptimizerTest.cpp b/test/libyul/YulOptimizerTest.cpp index 019536b9d..dcdb4adc8 100644 --- a/test/libyul/YulOptimizerTest.cpp +++ b/test/libyul/YulOptimizerTest.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -45,6 +46,7 @@ #include #include #include +#include #include #include #include @@ -149,7 +151,7 @@ TestCase::TestResult YulOptimizerTest::run(ostream& _stream, string const& _line else if (m_optimizerStep == "commonSubexpressionEliminator") { disambiguate(); - (CommonSubexpressionEliminator{*m_dialect})(*m_ast); + CommonSubexpressionEliminator::run(*m_dialect, *m_ast); } else if (m_optimizerStep == "expressionSplitter") { @@ -218,7 +220,7 @@ TestCase::TestResult YulOptimizerTest::run(ostream& _stream, string const& _line NameDispenser nameDispenser{*m_dialect, *m_ast}; ExpressionSplitter{*m_dialect, nameDispenser}(*m_ast); ForLoopInitRewriter{}(*m_ast); - CommonSubexpressionEliminator{*m_dialect}(*m_ast); + CommonSubexpressionEliminator::run(*m_dialect, *m_ast); ExpressionSimplifier::run(*m_dialect, *m_ast); UnusedPruner::runUntilStabilised(*m_dialect, *m_ast); DeadCodeEliminator{*m_dialect}(*m_ast); @@ -260,7 +262,7 @@ TestCase::TestResult YulOptimizerTest::run(ostream& _stream, string const& _line ForLoopInitRewriter{}(*m_ast); NameDispenser nameDispenser{*m_dialect, *m_ast}; ExpressionSplitter{*m_dialect, nameDispenser}(*m_ast); - CommonSubexpressionEliminator{*m_dialect}(*m_ast); + CommonSubexpressionEliminator::run(*m_dialect, *m_ast); ExpressionSimplifier::run(*m_dialect, *m_ast); LoadResolver::run(*m_dialect, *m_ast); @@ -298,7 +300,7 @@ TestCase::TestResult YulOptimizerTest::run(ostream& _stream, string const& _line RedundantAssignEliminator::run(*m_dialect, *m_ast); // reverse SSA SSAReverser::run(*m_ast); - CommonSubexpressionEliminator{*m_dialect}(*m_ast); + CommonSubexpressionEliminator::run(*m_dialect, *m_ast); UnusedPruner::runUntilStabilised(*m_dialect, *m_ast); } else if (m_optimizerStep == "stackCompressor") diff --git a/test/libyul/yulOptimizerTests/commonSubexpressionEliminator/movable_functions.yul b/test/libyul/yulOptimizerTests/commonSubexpressionEliminator/movable_functions.yul new file mode 100644 index 000000000..3b3e65dbd --- /dev/null +++ b/test/libyul/yulOptimizerTests/commonSubexpressionEliminator/movable_functions.yul @@ -0,0 +1,26 @@ +{ + function double(x) -> y { y := add(x, x) } + function double_with_se(x) -> y { y := add(x, x) mstore(40, 4) } + let i := mload(3) + let a := double(i) + let b := double(i) + let c := double_with_se(i) + let d := double_with_se(i) +} +// ==== +// step: commonSubexpressionEliminator +// ---- +// { +// function double(x) -> y +// { y := add(x, x) } +// function double_with_se(x_1) -> y_2 +// { +// y_2 := add(x_1, x_1) +// mstore(40, 4) +// } +// let i := mload(3) +// let a := double(i) +// let b := a +// let c := double_with_se(i) +// let d := double_with_se(i) +// } diff --git a/test/tools/yulopti.cpp b/test/tools/yulopti.cpp index 2729c4577..e098ae4c8 100644 --- a/test/tools/yulopti.cpp +++ b/test/tools/yulopti.cpp @@ -151,7 +151,7 @@ public: ForLoopConditionIntoBody{}(*m_ast); break; case 'c': - (CommonSubexpressionEliminator{m_dialect})(*m_ast); + CommonSubexpressionEliminator::run(m_dialect, *m_ast); break; case 'd': (VarDeclInitializer{})(*m_ast); From 127bcfc69d4908fffe6c29c99da5c07526f3ec4c Mon Sep 17 00:00:00 2001 From: chriseth Date: Thu, 29 Aug 2019 15:26:36 +0200 Subject: [PATCH 04/20] Take user function side-effects into account for unused pruner. --- libyul/optimiser/Semantics.cpp | 8 ++++-- libyul/optimiser/Semantics.h | 6 ++++- libyul/optimiser/Suite.cpp | 22 +++++++-------- libyul/optimiser/UnusedPruner.cpp | 27 ++++++++++++++----- libyul/optimiser/UnusedPruner.h | 10 ++++++- test/libyul/YulOptimizerTest.cpp | 8 +++--- .../movable_user_defined_function.yul | 13 +++++++++ test/tools/yulopti.cpp | 4 ++- 8 files changed, 72 insertions(+), 26 deletions(-) create mode 100644 test/libyul/yulOptimizerTests/unusedPruner/movable_user_defined_function.yul diff --git a/libyul/optimiser/Semantics.cpp b/libyul/optimiser/Semantics.cpp index 78d1a281a..a4784af66 100644 --- a/libyul/optimiser/Semantics.cpp +++ b/libyul/optimiser/Semantics.cpp @@ -35,8 +35,12 @@ using namespace dev; using namespace yul; -SideEffectsCollector::SideEffectsCollector(Dialect const& _dialect, Expression const& _expression): - SideEffectsCollector(_dialect) +SideEffectsCollector::SideEffectsCollector( + Dialect const& _dialect, + Expression const& _expression, + map const* _functionSideEffects +): + SideEffectsCollector(_dialect, _functionSideEffects) { visit(_expression); } diff --git a/libyul/optimiser/Semantics.h b/libyul/optimiser/Semantics.h index 55d44a663..1135b7070 100644 --- a/libyul/optimiser/Semantics.h +++ b/libyul/optimiser/Semantics.h @@ -41,7 +41,11 @@ public: Dialect const& _dialect, std::map const* _functionSideEffects = nullptr ): m_dialect(_dialect), m_functionSideEffects(_functionSideEffects) {} - SideEffectsCollector(Dialect const& _dialect, Expression const& _expression); + SideEffectsCollector( + Dialect const& _dialect, + Expression const& _expression, + std::map const* _functionSideEffects = nullptr + ); SideEffectsCollector(Dialect const& _dialect, Statement const& _statement); SideEffectsCollector(Dialect const& _dialect, Block const& _ast); diff --git a/libyul/optimiser/Suite.cpp b/libyul/optimiser/Suite.cpp index 6c5f160db..d1677bf53 100644 --- a/libyul/optimiser/Suite.cpp +++ b/libyul/optimiser/Suite.cpp @@ -87,7 +87,7 @@ void OptimiserSuite::run( DeadCodeEliminator{_dialect}(ast); FunctionGrouper{}(ast); EquivalentFunctionCombiner::run(ast); - UnusedPruner::runUntilStabilised(_dialect, ast, reservedIdentifiers); + UnusedPruner::runUntilStabilisedOnFullAST(_dialect, ast, reservedIdentifiers); BlockFlattener{}(ast); ControlFlowSimplifier{_dialect}(ast); StructuralSimplifier{_dialect}(ast); @@ -127,20 +127,20 @@ void OptimiserSuite::run( ControlFlowSimplifier{_dialect}(ast); BlockFlattener{}(ast); DeadCodeEliminator{_dialect}(ast); - UnusedPruner::runUntilStabilised(_dialect, ast, reservedIdentifiers); + UnusedPruner::runUntilStabilisedOnFullAST(_dialect, ast, reservedIdentifiers); } { // simplify again CommonSubexpressionEliminator::run(_dialect, ast); - UnusedPruner::runUntilStabilised(_dialect, ast, reservedIdentifiers); + UnusedPruner::runUntilStabilisedOnFullAST(_dialect, ast, reservedIdentifiers); } { // reverse SSA SSAReverser::run(ast); CommonSubexpressionEliminator::run(_dialect, ast); - UnusedPruner::runUntilStabilised(_dialect, ast, reservedIdentifiers); + UnusedPruner::runUntilStabilisedOnFullAST(_dialect, ast, reservedIdentifiers); ExpressionJoiner::run(ast); ExpressionJoiner::run(ast); @@ -151,7 +151,7 @@ void OptimiserSuite::run( { // run functional expression inliner ExpressionInliner(_dialect, ast).run(); - UnusedPruner::runUntilStabilised(_dialect, ast, reservedIdentifiers); + UnusedPruner::runUntilStabilisedOnFullAST(_dialect, ast, reservedIdentifiers); } { @@ -185,7 +185,7 @@ void OptimiserSuite::run( SSATransform::run(ast, dispenser); RedundantAssignEliminator::run(_dialect, ast); RedundantAssignEliminator::run(_dialect, ast); - UnusedPruner::runUntilStabilised(_dialect, ast, reservedIdentifiers); + UnusedPruner::runUntilStabilisedOnFullAST(_dialect, ast, reservedIdentifiers); CommonSubexpressionEliminator::run(_dialect, ast); } } @@ -194,19 +194,19 @@ void OptimiserSuite::run( ExpressionJoiner::run(ast); Rematerialiser::run(_dialect, ast); - UnusedPruner::runUntilStabilised(_dialect, ast, reservedIdentifiers); + UnusedPruner::runUntilStabilisedOnFullAST(_dialect, ast, reservedIdentifiers); ExpressionJoiner::run(ast); - UnusedPruner::runUntilStabilised(_dialect, ast, reservedIdentifiers); + UnusedPruner::runUntilStabilisedOnFullAST(_dialect, ast, reservedIdentifiers); ExpressionJoiner::run(ast); - UnusedPruner::runUntilStabilised(_dialect, ast, reservedIdentifiers); + UnusedPruner::runUntilStabilisedOnFullAST(_dialect, ast, reservedIdentifiers); SSAReverser::run(ast); CommonSubexpressionEliminator::run(_dialect, ast); - UnusedPruner::runUntilStabilised(_dialect, ast, reservedIdentifiers); + UnusedPruner::runUntilStabilisedOnFullAST(_dialect, ast, reservedIdentifiers); ExpressionJoiner::run(ast); Rematerialiser::run(_dialect, ast); - UnusedPruner::runUntilStabilised(_dialect, ast, reservedIdentifiers); + UnusedPruner::runUntilStabilisedOnFullAST(_dialect, ast, reservedIdentifiers); // This is a tuning parameter, but actually just prevents infinite loops. size_t stackCompressorMaxIterations = 16; diff --git a/libyul/optimiser/UnusedPruner.cpp b/libyul/optimiser/UnusedPruner.cpp index 32d8a75b6..f77c1f95c 100644 --- a/libyul/optimiser/UnusedPruner.cpp +++ b/libyul/optimiser/UnusedPruner.cpp @@ -20,12 +20,14 @@ #include +#include #include #include #include #include #include #include +#include #include @@ -37,10 +39,12 @@ UnusedPruner::UnusedPruner( Dialect const& _dialect, Block& _ast, bool _allowMSizeOptimization, + map const* _functionSideEffects, set const& _externallyUsedFunctions ): m_dialect(_dialect), - m_allowMSizeOptimization(_allowMSizeOptimization) + m_allowMSizeOptimization(_allowMSizeOptimization), + m_functionSideEffects(_functionSideEffects) { m_references = ReferencesCounter::countReferences(_ast); for (auto const& f: _externallyUsedFunctions) @@ -88,7 +92,10 @@ void UnusedPruner::operator()(Block& _block) { if (!varDecl.value) statement = Block{std::move(varDecl.location), {}}; - else if (SideEffectsCollector(m_dialect, *varDecl.value).sideEffectFree(m_allowMSizeOptimization)) + else if ( + SideEffectsCollector(m_dialect, *varDecl.value, m_functionSideEffects). + sideEffectFree(m_allowMSizeOptimization) + ) { subtractReferences(ReferencesCounter::countReferences(*varDecl.value)); statement = Block{std::move(varDecl.location), {}}; @@ -104,7 +111,10 @@ void UnusedPruner::operator()(Block& _block) else if (statement.type() == typeid(ExpressionStatement)) { ExpressionStatement& exprStmt = boost::get(statement); - if (SideEffectsCollector(m_dialect, exprStmt.expression).sideEffectFree(m_allowMSizeOptimization)) + if ( + SideEffectsCollector(m_dialect, exprStmt.expression, m_functionSideEffects). + sideEffectFree(m_allowMSizeOptimization) + ) { subtractReferences(ReferencesCounter::countReferences(exprStmt.expression)); statement = Block{std::move(exprStmt.location), {}}; @@ -120,26 +130,31 @@ void UnusedPruner::runUntilStabilised( Dialect const& _dialect, Block& _ast, bool _allowMSizeOptimization, + map const* _functionSideEffects, set const& _externallyUsedFunctions ) { while (true) { - UnusedPruner pruner(_dialect, _ast, _allowMSizeOptimization, _externallyUsedFunctions); + UnusedPruner pruner( + _dialect, _ast, _allowMSizeOptimization, _functionSideEffects, + _externallyUsedFunctions); pruner(_ast); if (!pruner.shouldRunAgain()) return; } } -void UnusedPruner::runUntilStabilised( +void UnusedPruner::runUntilStabilisedOnFullAST( Dialect const& _dialect, Block& _ast, set const& _externallyUsedFunctions ) { + map functionSideEffects = + SideEffectsPropagator::sideEffects(_dialect, CallGraphGenerator::callGraph(_ast)); bool allowMSizeOptimization = !MSizeFinder::containsMSize(_dialect, _ast); - runUntilStabilised(_dialect, _ast, allowMSizeOptimization, _externallyUsedFunctions); + runUntilStabilised(_dialect, _ast, allowMSizeOptimization, &functionSideEffects, _externallyUsedFunctions); } void UnusedPruner::runUntilStabilised( diff --git a/libyul/optimiser/UnusedPruner.h b/libyul/optimiser/UnusedPruner.h index 766b01e18..626730f65 100644 --- a/libyul/optimiser/UnusedPruner.h +++ b/libyul/optimiser/UnusedPruner.h @@ -29,6 +29,7 @@ namespace yul { struct Dialect; +struct SideEffects; /** * Optimisation stage that removes unused variables and functions and also @@ -50,6 +51,7 @@ public: Dialect const& _dialect, Block& _ast, bool _allowMSizeOptimization, + std::map const* _functionSideEffects = nullptr, std::set const& _externallyUsedFunctions = {} ); UnusedPruner( @@ -70,10 +72,15 @@ public: Dialect const& _dialect, Block& _ast, bool _allowMSizeOptimization, + std::map const* _functionSideEffects = nullptr, std::set const& _externallyUsedFunctions = {} ); - static void runUntilStabilised( + /// Run the pruner until the code does not change anymore. + /// The provided block has to be a full AST. + /// The pruner itself determines if msize is used and which user-defined functions + /// are side-effect free. + static void runUntilStabilisedOnFullAST( Dialect const& _dialect, Block& _ast, std::set const& _externallyUsedFunctions = {} @@ -97,6 +104,7 @@ private: Dialect const& m_dialect; bool m_allowMSizeOptimization = false; + std::map const* m_functionSideEffects = nullptr; bool m_shouldRunAgain = false; std::map m_references; }; diff --git a/test/libyul/YulOptimizerTest.cpp b/test/libyul/YulOptimizerTest.cpp index dcdb4adc8..88e8c4e2b 100644 --- a/test/libyul/YulOptimizerTest.cpp +++ b/test/libyul/YulOptimizerTest.cpp @@ -222,7 +222,7 @@ TestCase::TestResult YulOptimizerTest::run(ostream& _stream, string const& _line ForLoopInitRewriter{}(*m_ast); CommonSubexpressionEliminator::run(*m_dialect, *m_ast); ExpressionSimplifier::run(*m_dialect, *m_ast); - UnusedPruner::runUntilStabilised(*m_dialect, *m_ast); + UnusedPruner::runUntilStabilisedOnFullAST(*m_dialect, *m_ast); DeadCodeEliminator{*m_dialect}(*m_ast); ExpressionJoiner::run(*m_ast); ExpressionJoiner::run(*m_ast); @@ -230,7 +230,7 @@ TestCase::TestResult YulOptimizerTest::run(ostream& _stream, string const& _line else if (m_optimizerStep == "unusedPruner") { disambiguate(); - UnusedPruner::runUntilStabilised(*m_dialect, *m_ast); + UnusedPruner::runUntilStabilisedOnFullAST(*m_dialect, *m_ast); } else if (m_optimizerStep == "deadCodeEliminator") { @@ -267,7 +267,7 @@ TestCase::TestResult YulOptimizerTest::run(ostream& _stream, string const& _line LoadResolver::run(*m_dialect, *m_ast); - UnusedPruner::runUntilStabilised(*m_dialect, *m_ast); + UnusedPruner::runUntilStabilisedOnFullAST(*m_dialect, *m_ast); ExpressionJoiner::run(*m_ast); ExpressionJoiner::run(*m_ast); } @@ -301,7 +301,7 @@ TestCase::TestResult YulOptimizerTest::run(ostream& _stream, string const& _line // reverse SSA SSAReverser::run(*m_ast); CommonSubexpressionEliminator::run(*m_dialect, *m_ast); - UnusedPruner::runUntilStabilised(*m_dialect, *m_ast); + UnusedPruner::runUntilStabilisedOnFullAST(*m_dialect, *m_ast); } else if (m_optimizerStep == "stackCompressor") { diff --git a/test/libyul/yulOptimizerTests/unusedPruner/movable_user_defined_function.yul b/test/libyul/yulOptimizerTests/unusedPruner/movable_user_defined_function.yul new file mode 100644 index 000000000..0d49d713b --- /dev/null +++ b/test/libyul/yulOptimizerTests/unusedPruner/movable_user_defined_function.yul @@ -0,0 +1,13 @@ +{ + function f(x) -> t { + let b := 7 + } + function g(x) -> t { + t := x + } + let x := f(g(2)) +} +// ==== +// step: unusedPruner +// ---- +// { } diff --git a/test/tools/yulopti.cpp b/test/tools/yulopti.cpp index e098ae4c8..397b4a5f2 100644 --- a/test/tools/yulopti.cpp +++ b/test/tools/yulopti.cpp @@ -32,6 +32,7 @@ #include #include +#include #include #include #include @@ -54,6 +55,7 @@ #include #include #include +#include #include #include @@ -187,7 +189,7 @@ public: (ControlFlowSimplifier{m_dialect})(*m_ast); break; case 'u': - UnusedPruner::runUntilStabilised(m_dialect, *m_ast); + UnusedPruner::runUntilStabilisedOnFullAST(m_dialect, *m_ast); break; case 'D': DeadCodeEliminator{m_dialect}(*m_ast); From 55024d40fec0619421025231cec078bb0d67be26 Mon Sep 17 00:00:00 2001 From: chriseth Date: Thu, 15 Aug 2019 14:42:56 +0200 Subject: [PATCH 05/20] Test updates. --- .../standard_eWasm_requested/output.json | 70 ++++++------------- ...ied_function_call_equality_not_movable.yul | 4 +- ...y_with_different_kinds_of_invalidation.yul | 3 - .../unusedPruner/multi_declare.yul | 6 +- 4 files changed, 26 insertions(+), 57 deletions(-) diff --git a/test/cmdlineTests/standard_eWasm_requested/output.json b/test/cmdlineTests/standard_eWasm_requested/output.json index a0956a399..e766c2156 100644 --- a/test/cmdlineTests/standard_eWasm_requested/output.json +++ b/test/cmdlineTests/standard_eWasm_requested/output.json @@ -7,24 +7,16 @@ (local $_1 i64) (local $pos i64) (local $_2 i64) - (local $_3 i64) (local $hi i64) - (local $_4 i64) + (local $y i64) (local $hi_1 i64) - (local $hi_2 i64) - (local $hi_3 i64) - (local $hi_4 i64) (set_local $_1 (i64.const 0)) (set_local $pos (call $u256_to_i32 (get_local $_1) (get_local $_1) (get_local $_1) (i64.const 64))) - (set_local $_2 (i64.const 32)) - (set_local $_3 (i64.shr_u (get_local $_1) (i64.const 16))) - (set_local $hi (i64.shl (i64.or (i64.shl (i64.or (i64.and (i64.shl (get_local $_1) (i64.const 8)) (i64.const 65280)) (i64.and (i64.shr_u (get_local $_1) (i64.const 8)) (i64.const 255))) (i64.const 16)) (call $endian_swap_16 (get_local $_3))) (get_local $_2))) - (set_local $_4 (i64.shr_u (get_local $_1) (get_local $_2))) - (i64.store (get_local $pos) (i64.or (get_local $hi) (call $endian_swap_32 (get_local $_4)))) (set_local $hi_1 (i64.shl (call $endian_swap_16 (get_local $_1)) (i64.const 16))) - (set_local $hi_2 (i64.shl (i64.or (get_local $hi_1) (call $endian_swap_16 (get_local $_3))) (get_local $_2))) - (i64.store (i64.add (get_local $pos) (i64.const 8)) (i64.or (get_local $hi_2) (call $endian_swap_32 (get_local $_4)))) (set_local $hi_3 (i64.shl (call $endian_swap_32 (get_local $_1)) (get_local $_2))) - (i64.store (i64.add (get_local $pos) (i64.const 16)) (i64.or (get_local $hi_3) (call $endian_swap_32 (get_local $_4)))) (set_local $hi_4 (i64.shl (call $endian_swap_32 (i64.const 64)) (get_local $_2))) - (i64.store (i64.add (get_local $pos) (i64.const 24)) (i64.or (get_local $hi_4) (call $endian_swap_32 (i64.shr_u (i64.const 64) (get_local $_2))))) (call $eth.revert (call $u256_to_i32 (get_local $_1) (get_local $_1) (get_local $_1) (get_local $_1)) (call $u256_to_i32 (get_local $_1) (get_local $_1) (get_local $_1) (get_local $_1))) + (set_local $_2 (i64.const 65280)) + (set_local $hi (i64.shl (i64.or (i64.shl (i64.or (i64.and (i64.shl (get_local $_1) (i64.const 8)) (get_local $_2)) (i64.and (i64.shr_u (get_local $_1) (i64.const 8)) (i64.const 255))) (i64.const 16)) (call $endian_swap_16 (i64.shr_u (get_local $_1) (i64.const 16)))) (i64.const 32))) + (set_local $y (i64.or (get_local $hi) (call $endian_swap_32 (i64.shr_u (get_local $_1) (i64.const 32))))) + (i64.store (get_local $pos) (get_local $y)) (i64.store (i64.add (get_local $pos) (i64.const 8)) (get_local $y)) (i64.store (i64.add (get_local $pos) (i64.const 16)) (get_local $y)) (set_local $hi_1 (i64.shl (i64.or (i64.shl (i64.or (i64.and (i64.shl (i64.const 64) (i64.const 8)) (get_local $_2)) (i64.and (i64.shr_u (i64.const 64) (i64.const 8)) (i64.const 255))) (i64.const 16)) (call $endian_swap_16 (i64.shr_u (i64.const 64) (i64.const 16)))) (i64.const 32))) + (i64.store (i64.add (get_local $pos) (i64.const 24)) (i64.or (get_local $hi_1) (call $endian_swap_32 (i64.shr_u (i64.const 64) (i64.const 32))))) (call $eth.revert (call $u256_to_i32 (get_local $_1) (get_local $_1) (get_local $_1) (get_local $_1)) (call $u256_to_i32 (get_local $_1) (get_local $_1) (get_local $_1) (get_local $_1))) ) (func $u256_to_i32 @@ -74,44 +66,38 @@ (local $_1 i64) (local $pos i64) (local $hi i64) - (local $_2 i64) + (local $y i64) (local $hi_1 i64) - (local $_3 i64) (local $hi_2 i64) - (local $hi_3 i64) - (local $hi_4 i64) + (local $_2 i64) + (local $_3 i64) (local $_4 i64) (local $_5 i64) (local $_6 i64) (local $_7 i64) (local $_8 i64) (local $_9 i64) - (local $_10 i64) - (local $_11 i64) (set_local $_1 (i64.const 0)) (set_local $pos (call $u256_to_i32 (get_local $_1) (get_local $_1) (get_local $_1) (i64.const 64))) - (set_local $hi (i64.shl (call $endian_swap_16 (get_local $_1)) (i64.const 16))) - (set_local $_2 (i64.shr_u (get_local $_1) (i64.const 16))) - (set_local $hi_1 (i64.shl (i64.or (get_local $hi) (call $endian_swap_16 (get_local $_2))) (i64.const 32))) - (set_local $_3 (i64.shr_u (get_local $_1) (i64.const 32))) - (i64.store (get_local $pos) (i64.or (get_local $hi_1) (call $endian_swap_32 (get_local $_3)))) (set_local $hi_2 (i64.shl (call $endian_swap_16 (get_local $_1)) (i64.const 16))) - (set_local $hi_3 (i64.shl (i64.or (get_local $hi_2) (call $endian_swap_16 (get_local $_2))) (i64.const 32))) - (i64.store (i64.add (get_local $pos) (i64.const 8)) (i64.or (get_local $hi_3) (call $endian_swap_32 (get_local $_3)))) (set_local $hi_4 (i64.shl (call $endian_swap_32 (get_local $_1)) (i64.const 32))) - (i64.store (i64.add (get_local $pos) (i64.const 16)) (i64.or (get_local $hi_4) (call $endian_swap_32 (get_local $_3)))) (i64.store (i64.add (get_local $pos) (i64.const 24)) (call $endian_swap (i64.const 64))) (block - (set_local $_4 (datasize \"C_2_deployed\")) - (set_local $_5 (get_global $global_)) - (set_local $_6 (get_global $global__1)) - (set_local $_7 (get_global $global__2)) + (set_local $hi (i64.shl (i64.or (i64.shl (i64.or (i64.and (i64.shl (get_local $_1) (i64.const 8)) (i64.const 65280)) (i64.and (i64.shr_u (get_local $_1) (i64.const 8)) (i64.const 255))) (i64.const 16)) (call $endian_swap_16 (i64.shr_u (get_local $_1) (i64.const 16)))) (i64.const 32))) + (set_local $y (i64.or (get_local $hi) (call $endian_swap_32 (i64.shr_u (get_local $_1) (i64.const 32))))) + (i64.store (get_local $pos) (get_local $y)) (i64.store (i64.add (get_local $pos) (i64.const 8)) (get_local $y)) (i64.store (i64.add (get_local $pos) (i64.const 16)) (get_local $y)) (set_local $hi_1 (i64.shl (call $endian_swap_16 (i64.const 64)) (i64.const 16))) + (set_local $hi_2 (i64.shl (i64.or (get_local $hi_1) (call $endian_swap_16 (i64.shr_u (i64.const 64) (i64.const 16)))) (i64.const 32))) + (i64.store (i64.add (get_local $pos) (i64.const 24)) (i64.or (get_local $hi_2) (call $endian_swap_32 (i64.shr_u (i64.const 64) (i64.const 32))))) (block + (set_local $_2 (datasize \"C_2_deployed\")) + (set_local $_3 (get_global $global_)) + (set_local $_4 (get_global $global__1)) + (set_local $_5 (get_global $global__2)) ) (block - (set_local $_8 (dataoffset \"C_2_deployed\")) - (set_local $_9 (get_global $global_)) - (set_local $_10 (get_global $global__1)) - (set_local $_11 (get_global $global__2)) + (set_local $_6 (dataoffset \"C_2_deployed\")) + (set_local $_7 (get_global $global_)) + (set_local $_8 (get_global $global__1)) + (set_local $_9 (get_global $global__2)) ) - (call $eth.codeCopy (call $u256_to_i32 (get_local $_1) (get_local $_1) (get_local $_1) (get_local $_1)) (call $u256_to_i32 (get_local $_8) (get_local $_9) (get_local $_10) (get_local $_11)) (call $u256_to_i32 (get_local $_4) (get_local $_5) (get_local $_6) (get_local $_7))) (call $eth.finish (call $u256_to_i32 (get_local $_1) (get_local $_1) (get_local $_1) (get_local $_1)) (call $u256_to_i32 (get_local $_4) (get_local $_5) (get_local $_6) (get_local $_7))) + (call $eth.codeCopy (call $u256_to_i32 (get_local $_1) (get_local $_1) (get_local $_1) (get_local $_1)) (call $u256_to_i32 (get_local $_6) (get_local $_7) (get_local $_8) (get_local $_9)) (call $u256_to_i32 (get_local $_2) (get_local $_3) (get_local $_4) (get_local $_5))) (call $eth.finish (call $u256_to_i32 (get_local $_1) (get_local $_1) (get_local $_1) (get_local $_1)) (call $u256_to_i32 (get_local $_2) (get_local $_3) (get_local $_4) (get_local $_5))) ) (func $u256_to_i32 @@ -147,16 +133,6 @@ (get_local $y) ) -(func $endian_swap - (param $x i64) - (result i64) - (local $y i64) - (local $hi i64) - (set_local $hi (i64.shl (call $endian_swap_32 (get_local $x)) (i64.const 32))) - (set_local $y (i64.or (get_local $hi) (call $endian_swap_32 (i64.shr_u (get_local $x) (i64.const 32))))) - (get_local $y) -) - ) "}}}},"errors":[{"component":"general","formattedMessage":"Warning: The Yul optimiser is still experimental. Do not use it in production unless correctness of generated code is verified with extensive tests. ","message":"The Yul optimiser is still experimental. Do not use it in production unless correctness of generated code is verified with extensive tests.","severity":"warning","type":"Warning"}],"sources":{"A":{"id":0}}} diff --git a/test/libyul/yulOptimizerTests/fullSimplify/not_applied_function_call_equality_not_movable.yul b/test/libyul/yulOptimizerTests/fullSimplify/not_applied_function_call_equality_not_movable.yul index bb5b90b8b..11de59aba 100644 --- a/test/libyul/yulOptimizerTests/fullSimplify/not_applied_function_call_equality_not_movable.yul +++ b/test/libyul/yulOptimizerTests/fullSimplify/not_applied_function_call_equality_not_movable.yul @@ -1,6 +1,6 @@ // Even if the functions pass the equality check, they are not movable. { - function f() -> a { } + function f() -> a { mstore(1, 2) } let b := sub(f(), f()) mstore(0, b) } @@ -9,6 +9,6 @@ // ---- // { // function f() -> a -// { } +// { mstore(1, 2) } // mstore(0, sub(f(), f())) // } diff --git a/test/libyul/yulOptimizerTests/loadResolver/memory_with_different_kinds_of_invalidation.yul b/test/libyul/yulOptimizerTests/loadResolver/memory_with_different_kinds_of_invalidation.yul index 0bd777cf8..65312c8b4 100644 --- a/test/libyul/yulOptimizerTests/loadResolver/memory_with_different_kinds_of_invalidation.yul +++ b/test/libyul/yulOptimizerTests/loadResolver/memory_with_different_kinds_of_invalidation.yul @@ -31,8 +31,5 @@ // mstore8(calldataload(_5), 4) // sstore(_5, mload(_2)) // mstore(_2, _17) -// g() // sstore(_5, mload(_2)) -// function g() -// { } // } diff --git a/test/libyul/yulOptimizerTests/unusedPruner/multi_declare.yul b/test/libyul/yulOptimizerTests/unusedPruner/multi_declare.yul index 58c969e6d..b4f6a5a7e 100644 --- a/test/libyul/yulOptimizerTests/unusedPruner/multi_declare.yul +++ b/test/libyul/yulOptimizerTests/unusedPruner/multi_declare.yul @@ -5,8 +5,4 @@ // ==== // step: unusedPruner // ---- -// { -// function f() -> x, y -// { } -// let a, b := f() -// } +// { } From 635f638fde012a1ed210dc1f2329ef4380353740 Mon Sep 17 00:00:00 2001 From: chriseth Date: Thu, 29 Aug 2019 15:36:17 +0200 Subject: [PATCH 06/20] Changelog entry. --- Changelog.md | 1 + 1 file changed, 1 insertion(+) diff --git a/Changelog.md b/Changelog.md index bc165d4f6..bc7b5b8d0 100644 --- a/Changelog.md +++ b/Changelog.md @@ -6,6 +6,7 @@ Language Features: Compiler Features: * Optimizer: Add rule that replaces the BYTE opcode by 0 if the first argument is larger than 31. + * Yul Optimizer: Take side-effect-freeness of user-defined functions into account. Bugfixes: From a774b2d905367597e09bbb640c5c66169c10887e Mon Sep 17 00:00:00 2001 From: Leonardo Alt Date: Thu, 22 Aug 2019 15:44:30 +0200 Subject: [PATCH 07/20] [SMTChecker] Zero-initialize arrays --- libsolidity/formal/CVC4Interface.cpp | 8 ++++ libsolidity/formal/SMTLib2Interface.cpp | 3 -- libsolidity/formal/SolverInterface.h | 43 ++++++++++++++++++- libsolidity/formal/SymbolicTypes.cpp | 29 +++++++++++-- libsolidity/formal/SymbolicTypes.h | 1 + libsolidity/formal/Z3CHCInterface.cpp | 13 ++++++ libsolidity/formal/Z3Interface.cpp | 14 ++++++ test/libsolidity/SMTChecker.cpp | 4 +- .../operators/compound_add_array_index.sol | 4 +- .../operators/compound_add_mapping.sol | 4 +- .../operators/compound_mul_array_index.sol | 4 +- .../operators/compound_mul_mapping.sol | 4 +- .../operators/compound_sub_array_index.sol | 4 +- .../operators/compound_sub_mapping.sol | 4 +- .../operators/delete_array.sol | 5 +-- .../operators/delete_array_2d.sol | 4 -- .../operators/delete_array_index_2d.sol | 6 +-- .../operators/delete_function.sol | 5 +-- .../types/array_dynamic_2_fail.sol | 5 ++- .../types/array_dynamic_3_fail.sol | 5 ++- .../types/array_static_2_fail.sol | 4 +- .../types/array_static_3_fail.sol | 5 ++- .../smtCheckerTests/types/mapping_4.sol | 1 - 23 files changed, 130 insertions(+), 49 deletions(-) diff --git a/libsolidity/formal/CVC4Interface.cpp b/libsolidity/formal/CVC4Interface.cpp index 909202094..dfe0cb913 100644 --- a/libsolidity/formal/CVC4Interface.cpp +++ b/libsolidity/formal/CVC4Interface.cpp @@ -137,6 +137,8 @@ CVC4::Expr CVC4Interface::toCVC4Expr(Expression const& _expr) return m_context.mkConst(true); else if (n == "false") return m_context.mkConst(false); + else if (auto sortSort = dynamic_pointer_cast(_expr.sort)) + return m_context.mkVar(n, cvc4Sort(*sortSort->inner)); else try { @@ -187,6 +189,12 @@ CVC4::Expr CVC4Interface::toCVC4Expr(Expression const& _expr) return m_context.mkExpr(CVC4::kind::SELECT, arguments[0], arguments[1]); else if (n == "store") return m_context.mkExpr(CVC4::kind::STORE, arguments[0], arguments[1], arguments[2]); + else if (n == "const_array") + { + shared_ptr sortSort = std::dynamic_pointer_cast(_expr.arguments[0].sort); + solAssert(sortSort, ""); + return m_context.mkConst(CVC4::ArrayStoreAll(cvc4Sort(*sortSort->inner), arguments[1])); + } solAssert(false, ""); } diff --git a/libsolidity/formal/SMTLib2Interface.cpp b/libsolidity/formal/SMTLib2Interface.cpp index a23dbe559..7d6e5d457 100644 --- a/libsolidity/formal/SMTLib2Interface.cpp +++ b/libsolidity/formal/SMTLib2Interface.cpp @@ -17,8 +17,6 @@ #include -#include -#include #include #include @@ -30,7 +28,6 @@ #include #include #include -#include using namespace std; using namespace dev; diff --git a/libsolidity/formal/SolverInterface.h b/libsolidity/formal/SolverInterface.h index 5610a5fd6..0004e1726 100644 --- a/libsolidity/formal/SolverInterface.h +++ b/libsolidity/formal/SolverInterface.h @@ -17,6 +17,7 @@ #pragma once +#include #include #include #include @@ -45,7 +46,8 @@ enum class Kind Int, Bool, Function, - Array + Array, + Sort }; struct Sort @@ -110,12 +112,33 @@ struct ArraySort: public Sort SortPointer range; }; +struct SortSort: public Sort +{ + SortSort(SortPointer _inner): Sort(Kind::Sort), inner(std::move(_inner)) {} + bool operator==(Sort const& _other) const override + { + if (!Sort::operator==(_other)) + return false; + auto _otherSort = dynamic_cast(&_other); + solAssert(_otherSort, ""); + solAssert(_otherSort->inner, ""); + solAssert(inner, ""); + return *inner == *_otherSort->inner; + } + + SortPointer inner; +}; + +// Forward declaration. +SortPointer smtSort(solidity::Type const& _type); + /// C++ representation of an SMTLIB2 expression. class Expression { friend class SolverInterface; public: explicit Expression(bool _v): Expression(_v ? "true" : "false", Kind::Bool) {} + explicit Expression(solidity::TypePointer _type): Expression(_type->toString(), {}, std::make_shared(smtSort(*_type))) {} Expression(size_t _number): Expression(std::to_string(_number), Kind::Int) {} Expression(u256 const& _number): Expression(_number.str(), Kind::Int) {} Expression(s256 const& _number): Expression(_number.str(), Kind::Int) {} @@ -145,7 +168,8 @@ public: {"/", 2}, {"mod", 2}, {"select", 2}, - {"store", 3} + {"store", 3}, + {"const_array", 2} }; return operatorsArity.count(name) && operatorsArity.at(name) == arguments.size(); } @@ -202,6 +226,21 @@ public: ); } + static Expression const_array(Expression _sort, Expression _value) + { + solAssert(_sort.sort->kind == Kind::Sort, ""); + auto sortSort = std::dynamic_pointer_cast(_sort.sort); + auto arraySort = std::dynamic_pointer_cast(sortSort->inner); + solAssert(sortSort && arraySort, ""); + solAssert(_value.sort, ""); + solAssert(*arraySort->range == *_value.sort, ""); + return Expression( + "const_array", + std::vector{std::move(_sort), std::move(_value)}, + arraySort + ); + } + friend Expression operator!(Expression _a) { return Expression("not", std::move(_a), Kind::Bool); diff --git a/libsolidity/formal/SymbolicTypes.cpp b/libsolidity/formal/SymbolicTypes.cpp index 6b9f26d82..c6f964b0a 100644 --- a/libsolidity/formal/SymbolicTypes.cpp +++ b/libsolidity/formal/SymbolicTypes.cpp @@ -276,10 +276,31 @@ void setSymbolicZeroValue(SymbolicVariable const& _variable, EncodingContext& _c void setSymbolicZeroValue(Expression _expr, solidity::TypePointer const& _type, EncodingContext& _context) { solAssert(_type, ""); - if (isNumber(_type->category())) - _context.addAssertion(_expr == 0); - else if (isBool(_type->category())) - _context.addAssertion(_expr == Expression(false)); + _context.addAssertion(_expr == zeroValue(_type)); +} + +Expression zeroValue(solidity::TypePointer const& _type) +{ + solAssert(_type, ""); + if (isSupportedType(_type->category())) + { + if (isNumber(_type->category())) + return 0; + if (isBool(_type->category())) + return Expression(false); + if (isArray(_type->category()) || isMapping(_type->category())) + { + if (auto arrayType = dynamic_cast(_type)) + return Expression::const_array(Expression(arrayType), zeroValue(arrayType->baseType())); + auto mappingType = dynamic_cast(_type); + solAssert(mappingType, ""); + return Expression::const_array(Expression(mappingType), zeroValue(mappingType->valueType())); + + } + solAssert(false, ""); + } + // Unsupported types are abstracted as Int. + return 0; } void setSymbolicUnknownValue(SymbolicVariable const& _variable, EncodingContext& _context) diff --git a/libsolidity/formal/SymbolicTypes.h b/libsolidity/formal/SymbolicTypes.h index 355c25328..30a341431 100644 --- a/libsolidity/formal/SymbolicTypes.h +++ b/libsolidity/formal/SymbolicTypes.h @@ -63,6 +63,7 @@ std::pair> newSymbolicVariable(solidity: Expression minValue(solidity::IntegerType const& _type); Expression maxValue(solidity::IntegerType const& _type); +Expression zeroValue(solidity::TypePointer const& _type); void setSymbolicZeroValue(SymbolicVariable const& _variable, EncodingContext& _context); void setSymbolicZeroValue(Expression _expr, solidity::TypePointer const& _type, EncodingContext& _context); diff --git a/libsolidity/formal/Z3CHCInterface.cpp b/libsolidity/formal/Z3CHCInterface.cpp index 7b8de2c91..909a34b02 100644 --- a/libsolidity/formal/Z3CHCInterface.cpp +++ b/libsolidity/formal/Z3CHCInterface.cpp @@ -33,6 +33,17 @@ Z3CHCInterface::Z3CHCInterface(): z3::set_param("rewriter.pull_cheap_ite", true); // This needs to be set in the context. m_context->set("timeout", queryTimeout); + + // Spacer options. + // These needs to be set in the solver. + // https://github.com/Z3Prover/z3/blob/master/src/muz/base/fp_params.pyg + z3::params p(*m_context); + // These are useful for solving problems with arrays and loops. + // Use quantified lemma generalizer. + p.set("fp.spacer.q3.use_qgen", true); + // Ground pobs by using values from a model. + p.set("fp.spacer.ground_pobs", false); + m_solver.set(p); } void Z3CHCInterface::declareVariable(string const& _name, Sort const& _sort) @@ -82,9 +93,11 @@ pair> Z3CHCInterface::query(Expression const& _expr) break; } case z3::check_result::unknown: + { result = CheckResult::UNKNOWN; break; } + } // TODO retrieve model / invariants } catch (z3::exception const& _e) diff --git a/libsolidity/formal/Z3Interface.cpp b/libsolidity/formal/Z3Interface.cpp index 1039dd1f8..e07bacdb7 100644 --- a/libsolidity/formal/Z3Interface.cpp +++ b/libsolidity/formal/Z3Interface.cpp @@ -132,6 +132,12 @@ z3::expr Z3Interface::toZ3Expr(Expression const& _expr) return m_context.bool_val(true); else if (n == "false") return m_context.bool_val(false); + else if (_expr.sort->kind == Kind::Sort) + { + auto sortSort = dynamic_pointer_cast(_expr.sort); + solAssert(sortSort, ""); + return m_context.constant(n.c_str(), z3Sort(*sortSort->inner)); + } else try { @@ -178,6 +184,14 @@ z3::expr Z3Interface::toZ3Expr(Expression const& _expr) return z3::select(arguments[0], arguments[1]); else if (n == "store") return z3::store(arguments[0], arguments[1], arguments[2]); + else if (n == "const_array") + { + shared_ptr sortSort = std::dynamic_pointer_cast(_expr.arguments[0].sort); + solAssert(sortSort, ""); + auto arraySort = dynamic_pointer_cast(sortSort->inner); + solAssert(arraySort && arraySort->domain, ""); + return z3::const_array(z3Sort(*arraySort->domain), arguments[1]); + } solAssert(false, ""); } diff --git a/test/libsolidity/SMTChecker.cpp b/test/libsolidity/SMTChecker.cpp index 3e5a69a3c..c7414d824 100644 --- a/test/libsolidity/SMTChecker.cpp +++ b/test/libsolidity/SMTChecker.cpp @@ -212,7 +212,7 @@ BOOST_AUTO_TEST_CASE(compound_assignment_division) uint[] array; function f(uint x, uint p) public { require(x == 2); - require(array[p] == 10); + array[p] = 10; array[p] /= array[p] / x; assert(array[p] == x); assert(array[p] == 0); @@ -225,7 +225,7 @@ BOOST_AUTO_TEST_CASE(compound_assignment_division) mapping (uint => uint) map; function f(uint x, uint p) public { require(x == 2); - require(map[p] == 10); + map[p] = 10; map[p] /= map[p] / x; assert(map[p] == x); assert(map[p] == 0); diff --git a/test/libsolidity/smtCheckerTests/operators/compound_add_array_index.sol b/test/libsolidity/smtCheckerTests/operators/compound_add_array_index.sol index 816aff23a..c4acd3842 100644 --- a/test/libsolidity/smtCheckerTests/operators/compound_add_array_index.sol +++ b/test/libsolidity/smtCheckerTests/operators/compound_add_array_index.sol @@ -5,11 +5,11 @@ contract C uint[] array; function f(uint x, uint p) public { require(x < 100); - require(array[p] == 100); + array[p] = 100; array[p] += array[p] + x; assert(array[p] < 300); assert(array[p] < 110); } } // ---- -// Warning: (202-224): Assertion violation happens here +// Warning: (192-214): Assertion violation happens here diff --git a/test/libsolidity/smtCheckerTests/operators/compound_add_mapping.sol b/test/libsolidity/smtCheckerTests/operators/compound_add_mapping.sol index f342ab97d..eab7df2f0 100644 --- a/test/libsolidity/smtCheckerTests/operators/compound_add_mapping.sol +++ b/test/libsolidity/smtCheckerTests/operators/compound_add_mapping.sol @@ -5,11 +5,11 @@ contract C mapping (uint => uint) map; function f(uint x, uint p) public { require(x < 100); - require(map[p] == 100); + map[p] = 100; map[p] += map[p] + x; assert(map[p] < 300); assert(map[p] < 110); } } // ---- -// Warning: (208-228): Assertion violation happens here +// Warning: (198-218): Assertion violation happens here diff --git a/test/libsolidity/smtCheckerTests/operators/compound_mul_array_index.sol b/test/libsolidity/smtCheckerTests/operators/compound_mul_array_index.sol index ea837801d..110eb868e 100644 --- a/test/libsolidity/smtCheckerTests/operators/compound_mul_array_index.sol +++ b/test/libsolidity/smtCheckerTests/operators/compound_mul_array_index.sol @@ -5,11 +5,11 @@ contract C uint[] array; function f(uint x, uint p) public { require(x < 10); - require(array[p] == 10); + array[p] = 10; array[p] *= array[p] + x; assert(array[p] <= 190); assert(array[p] < 50); } } // ---- -// Warning: (201-222): Assertion violation happens here +// Warning: (191-212): Assertion violation happens here diff --git a/test/libsolidity/smtCheckerTests/operators/compound_mul_mapping.sol b/test/libsolidity/smtCheckerTests/operators/compound_mul_mapping.sol index fbcb5d88c..cd73bc687 100644 --- a/test/libsolidity/smtCheckerTests/operators/compound_mul_mapping.sol +++ b/test/libsolidity/smtCheckerTests/operators/compound_mul_mapping.sol @@ -5,11 +5,11 @@ contract C mapping (uint => uint) map; function f(uint x, uint p) public { require(x < 10); - require(map[p] == 10); + map[p] = 10; map[p] *= map[p] + x; assert(map[p] <= 190); assert(map[p] < 50); } } // ---- -// Warning: (207-226): Assertion violation happens here +// Warning: (197-216): Assertion violation happens here diff --git a/test/libsolidity/smtCheckerTests/operators/compound_sub_array_index.sol b/test/libsolidity/smtCheckerTests/operators/compound_sub_array_index.sol index b966337e9..8c8f84627 100644 --- a/test/libsolidity/smtCheckerTests/operators/compound_sub_array_index.sol +++ b/test/libsolidity/smtCheckerTests/operators/compound_sub_array_index.sol @@ -5,11 +5,11 @@ contract C uint[] array; function f(uint x, uint p) public { require(x < 100); - require(array[p] == 200); + array[p] = 200; array[p] -= array[p] - x; assert(array[p] >= 0); assert(array[p] < 90); } } // ---- -// Warning: (201-222): Assertion violation happens here +// Warning: (191-212): Assertion violation happens here diff --git a/test/libsolidity/smtCheckerTests/operators/compound_sub_mapping.sol b/test/libsolidity/smtCheckerTests/operators/compound_sub_mapping.sol index df20d3712..92e315832 100644 --- a/test/libsolidity/smtCheckerTests/operators/compound_sub_mapping.sol +++ b/test/libsolidity/smtCheckerTests/operators/compound_sub_mapping.sol @@ -5,11 +5,11 @@ contract C mapping (uint => uint) map; function f(uint x, uint p) public { require(x < 100); - require(map[p] == 200); + map[p] = 200; map[p] -= map[p] - x; assert(map[p] >= 0); assert(map[p] < 90); } } // ---- -// Warning: (207-226): Assertion violation happens here +// Warning: (197-216): Assertion violation happens here diff --git a/test/libsolidity/smtCheckerTests/operators/delete_array.sol b/test/libsolidity/smtCheckerTests/operators/delete_array.sol index 0a223d2a5..d34b0862c 100644 --- a/test/libsolidity/smtCheckerTests/operators/delete_array.sol +++ b/test/libsolidity/smtCheckerTests/operators/delete_array.sol @@ -10,12 +10,9 @@ contract C delete a; else delete a[2]; - // Assertion fails as false positive because - // setZeroValue for arrays needs \forall i . a[i] = 0 - // which is still unimplemented. assert(a[2] == 0); + assert(a[1] == 0); } } // ---- // Warning: (118-119): Condition is always true. -// Warning: (297-314): Assertion violation happens here diff --git a/test/libsolidity/smtCheckerTests/operators/delete_array_2d.sol b/test/libsolidity/smtCheckerTests/operators/delete_array_2d.sol index b8ce28f58..deff5bd10 100644 --- a/test/libsolidity/smtCheckerTests/operators/delete_array_2d.sol +++ b/test/libsolidity/smtCheckerTests/operators/delete_array_2d.sol @@ -6,10 +6,6 @@ contract C function f() public { require(a[2][3] == 4); delete a; - // Fails as false positive. - // setZeroValue needs forall for arrays. assert(a[2][3] == 0); } } -// ---- -// Warning: (194-214): Assertion violation happens here diff --git a/test/libsolidity/smtCheckerTests/operators/delete_array_index_2d.sol b/test/libsolidity/smtCheckerTests/operators/delete_array_index_2d.sol index cafc09207..3e13c7f84 100644 --- a/test/libsolidity/smtCheckerTests/operators/delete_array_index_2d.sol +++ b/test/libsolidity/smtCheckerTests/operators/delete_array_index_2d.sol @@ -9,11 +9,7 @@ contract C delete a; else delete a[2]; - // Fails as false positive since - // setZeroValue for arrays needs forall - // which is unimplemented. assert(a[2][3] == 0); + assert(a[1][1] == 0); } } -// ---- -// Warning: (266-286): Assertion violation happens here diff --git a/test/libsolidity/smtCheckerTests/operators/delete_function.sol b/test/libsolidity/smtCheckerTests/operators/delete_function.sol index 5c730b3d8..62579d7da 100644 --- a/test/libsolidity/smtCheckerTests/operators/delete_function.sol +++ b/test/libsolidity/smtCheckerTests/operators/delete_function.sol @@ -16,12 +16,9 @@ contract C g(); else h(); - // Assertion fails as false positive because - // setZeroValue for arrays needs \forall i . a[i] = 0 - // which is still unimplemented. assert(a[2] == 0); + assert(a[1] == 0); } } // ---- // Warning: (201-202): Condition is always true. -// Warning: (367-384): Assertion violation happens here diff --git a/test/libsolidity/smtCheckerTests/types/array_dynamic_2_fail.sol b/test/libsolidity/smtCheckerTests/types/array_dynamic_2_fail.sol index 725e51ba2..8555b0613 100644 --- a/test/libsolidity/smtCheckerTests/types/array_dynamic_2_fail.sol +++ b/test/libsolidity/smtCheckerTests/types/array_dynamic_2_fail.sol @@ -4,10 +4,11 @@ contract C { uint[][] array; function f(uint x, uint y, uint z, uint t) public view { - require(array[x][y] == 200); + // TODO change to = 200 when 2d assignments are supported. + require(array[x][y] < 200); require(x == z && y == t); assert(array[z][t] > 300); } } // ---- -// Warning: (183-208): Assertion violation happens here +// Warning: (243-268): Assertion violation happens here diff --git a/test/libsolidity/smtCheckerTests/types/array_dynamic_3_fail.sol b/test/libsolidity/smtCheckerTests/types/array_dynamic_3_fail.sol index 3a34ce5dd..d7eb8bccb 100644 --- a/test/libsolidity/smtCheckerTests/types/array_dynamic_3_fail.sol +++ b/test/libsolidity/smtCheckerTests/types/array_dynamic_3_fail.sol @@ -4,10 +4,11 @@ contract C { uint[][][] array; function f(uint x, uint y, uint z, uint t, uint w, uint v) public view { - require(array[x][y][z] == 200); + // TODO change to = 200 when 3d assignments are supported. + require(array[x][y][z] < 200); require(x == t && y == w && z == v); assert(array[t][w][v] > 300); } } // ---- -// Warning: (214-242): Assertion violation happens here +// Warning: (274-302): Assertion violation happens here diff --git a/test/libsolidity/smtCheckerTests/types/array_static_2_fail.sol b/test/libsolidity/smtCheckerTests/types/array_static_2_fail.sol index 789308c57..b230886de 100644 --- a/test/libsolidity/smtCheckerTests/types/array_static_2_fail.sol +++ b/test/libsolidity/smtCheckerTests/types/array_static_2_fail.sol @@ -4,10 +4,10 @@ contract C { uint[10][20] array; function f(uint x, uint y, uint z, uint t) public view { - require(array[x][y] == 200); + require(array[x][y] < 200); require(x == z && y == t); assert(array[z][t] > 300); } } // ---- -// Warning: (187-212): Assertion violation happens here +// Warning: (186-211): Assertion violation happens here diff --git a/test/libsolidity/smtCheckerTests/types/array_static_3_fail.sol b/test/libsolidity/smtCheckerTests/types/array_static_3_fail.sol index 240a11067..22a2f0506 100644 --- a/test/libsolidity/smtCheckerTests/types/array_static_3_fail.sol +++ b/test/libsolidity/smtCheckerTests/types/array_static_3_fail.sol @@ -4,10 +4,11 @@ contract C { uint[10][20][30] array; function f(uint x, uint y, uint z, uint t, uint w, uint v) public view { - require(array[x][y][z] == 200); + // TODO change to = 200 when 3d assignments are supported. + require(array[x][y][z] < 200); require(x == t && y == w && z == v); assert(array[t][w][v] > 300); } } // ---- -// Warning: (220-248): Assertion violation happens here +// Warning: (280-308): Assertion violation happens here diff --git a/test/libsolidity/smtCheckerTests/types/mapping_4.sol b/test/libsolidity/smtCheckerTests/types/mapping_4.sol index d02042113..7356a3b6b 100644 --- a/test/libsolidity/smtCheckerTests/types/mapping_4.sol +++ b/test/libsolidity/smtCheckerTests/types/mapping_4.sol @@ -9,4 +9,3 @@ contract C } } // ---- -// Warning: (125-144): Assertion violation happens here From a51577facf699e9c1873dd56565c5d2422cbbb65 Mon Sep 17 00:00:00 2001 From: Leonardo Alt Date: Mon, 2 Sep 2019 14:27:35 +0200 Subject: [PATCH 08/20] Fix Windows build --- libsolidity/formal/SMTLib2Interface.cpp | 8 ++++---- libsolidity/formal/SMTLib2Interface.h | 8 ++++---- libsolidity/formal/SMTPortfolio.cpp | 4 ++-- libsolidity/formal/SMTPortfolio.h | 6 +++--- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/libsolidity/formal/SMTLib2Interface.cpp b/libsolidity/formal/SMTLib2Interface.cpp index 7d6e5d457..cd516109e 100644 --- a/libsolidity/formal/SMTLib2Interface.cpp +++ b/libsolidity/formal/SMTLib2Interface.cpp @@ -93,12 +93,12 @@ void SMTLib2Interface::declareFunction(string const& _name, Sort const& _sort) } } -void SMTLib2Interface::addAssertion(Expression const& _expr) +void SMTLib2Interface::addAssertion(smt::Expression const& _expr) { write("(assert " + toSExpr(_expr) + ")"); } -pair> SMTLib2Interface::check(vector const& _expressionsToEvaluate) +pair> SMTLib2Interface::check(vector const& _expressionsToEvaluate) { string response = querySolver( boost::algorithm::join(m_accumulatedOutput, "\n") + @@ -122,7 +122,7 @@ pair> SMTLib2Interface::check(vector con return make_pair(result, values); } -string SMTLib2Interface::toSExpr(Expression const& _expr) +string SMTLib2Interface::toSExpr(smt::Expression const& _expr) { if (_expr.arguments.empty()) return _expr.name; @@ -166,7 +166,7 @@ void SMTLib2Interface::write(string _data) m_accumulatedOutput.back() += move(_data) + "\n"; } -string SMTLib2Interface::checkSatAndGetValuesCommand(vector const& _expressionsToEvaluate) +string SMTLib2Interface::checkSatAndGetValuesCommand(vector const& _expressionsToEvaluate) { string command; if (_expressionsToEvaluate.empty()) diff --git a/libsolidity/formal/SMTLib2Interface.h b/libsolidity/formal/SMTLib2Interface.h index d0bf4702d..a39c861c6 100644 --- a/libsolidity/formal/SMTLib2Interface.h +++ b/libsolidity/formal/SMTLib2Interface.h @@ -50,21 +50,21 @@ public: void declareVariable(std::string const&, Sort const&) override; - void addAssertion(Expression const& _expr) override; - std::pair> check(std::vector const& _expressionsToEvaluate) override; + void addAssertion(smt::Expression const& _expr) override; + std::pair> check(std::vector const& _expressionsToEvaluate) override; std::vector unhandledQueries() override { return m_unhandledQueries; } private: void declareFunction(std::string const&, Sort const&); - std::string toSExpr(Expression const& _expr); + std::string toSExpr(smt::Expression const& _expr); std::string toSmtLibSort(Sort const& _sort); std::string toSmtLibSort(std::vector const& _sort); void write(std::string _data); - std::string checkSatAndGetValuesCommand(std::vector const& _expressionsToEvaluate); + std::string checkSatAndGetValuesCommand(std::vector const& _expressionsToEvaluate); std::vector parseValues(std::string::const_iterator _start, std::string::const_iterator _end); /// Communicates with the solver via the callback. Throws SMTSolverError on error. diff --git a/libsolidity/formal/SMTPortfolio.cpp b/libsolidity/formal/SMTPortfolio.cpp index 09a311f4f..73c194a01 100644 --- a/libsolidity/formal/SMTPortfolio.cpp +++ b/libsolidity/formal/SMTPortfolio.cpp @@ -65,7 +65,7 @@ void SMTPortfolio::declareVariable(string const& _name, Sort const& _sort) s->declareVariable(_name, _sort); } -void SMTPortfolio::addAssertion(Expression const& _expr) +void SMTPortfolio::addAssertion(smt::Expression const& _expr) { for (auto const& s: m_solvers) s->addAssertion(_expr); @@ -101,7 +101,7 @@ void SMTPortfolio::addAssertion(Expression const& _expr) * * If all solvers return ERROR, the result is ERROR. */ -pair> SMTPortfolio::check(vector const& _expressionsToEvaluate) +pair> SMTPortfolio::check(vector const& _expressionsToEvaluate) { CheckResult lastResult = CheckResult::ERROR; vector finalValues; diff --git a/libsolidity/formal/SMTPortfolio.h b/libsolidity/formal/SMTPortfolio.h index 3eb0a793b..ded7985f9 100644 --- a/libsolidity/formal/SMTPortfolio.h +++ b/libsolidity/formal/SMTPortfolio.h @@ -51,9 +51,9 @@ public: void declareVariable(std::string const&, Sort const&) override; - void addAssertion(Expression const& _expr) override; + void addAssertion(smt::Expression const& _expr) override; - std::pair> check(std::vector const& _expressionsToEvaluate) override; + std::pair> check(std::vector const& _expressionsToEvaluate) override; std::vector unhandledQueries() override; unsigned solvers() override { return m_solvers.size(); } @@ -62,7 +62,7 @@ private: std::vector> m_solvers; - std::vector m_assertions; + std::vector m_assertions; }; } From 6427ec661a392b60ffc0efe6fc5aa1aa9ff08cc5 Mon Sep 17 00:00:00 2001 From: Bhargava Shastry Date: Tue, 3 Sep 2019 09:50:29 +0200 Subject: [PATCH 09/20] Yul interpreter: Out of bounds mloads return zero --- test/tools/yulInterpreter/EVMInstructionInterpreter.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/tools/yulInterpreter/EVMInstructionInterpreter.cpp b/test/tools/yulInterpreter/EVMInstructionInterpreter.cpp index 0e94f63f9..70e163bf4 100644 --- a/test/tools/yulInterpreter/EVMInstructionInterpreter.cpp +++ b/test/tools/yulInterpreter/EVMInstructionInterpreter.cpp @@ -250,8 +250,7 @@ u256 EVMInstructionInterpreter::eval( case Instruction::MLOAD: if (accessMemory(arg[0], 0x20)) return readMemoryWord(arg[0]); - else - return 0x1234 + arg[0]; + return 0; case Instruction::MSTORE: if (accessMemory(arg[0], 0x20)) writeMemoryWord(arg[0], arg[1]); From f9459fcda927850347499935a6b96e6cebb351c1 Mon Sep 17 00:00:00 2001 From: Gois Date: Tue, 3 Sep 2019 09:05:25 -0300 Subject: [PATCH 10/20] Fix typo in AssemblySwitch in grammar file --- docs/grammar.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/grammar.txt b/docs/grammar.txt index 3d2c9bc37..cf0ddd104 100644 --- a/docs/grammar.txt +++ b/docs/grammar.txt @@ -171,7 +171,7 @@ AssemblyVariableDeclaration = 'let' AssemblyIdentifierList ( ':=' AssemblyExpres AssemblyAssignment = AssemblyIdentifierList ':=' AssemblyExpression AssemblyExpression = AssemblyFunctionCall | Identifier | Literal AssemblyIf = 'if' AssemblyExpression AssemblyBlock -AssemblySwitch = 'switch' AssemblyExpression ( Case+ AssemblyDefault? | AssemblyDefault ) +AssemblySwitch = 'switch' AssemblyExpression ( AssemblyCase+ AssemblyDefault? | AssemblyDefault ) AssemblyCase = 'case' Literal AssemblyBlock AssemblyDefault = 'default' AssemblyBlock AssemblyForLoop = 'for' AssemblyBlock AssemblyExpression AssemblyBlock AssemblyBlock From d1317847142f0898e5b453e7bf1b0e23843b6bb2 Mon Sep 17 00:00:00 2001 From: Leonardo Alt Date: Tue, 3 Sep 2019 15:51:33 +0200 Subject: [PATCH 11/20] Add Load Resolver to Yulopti --- test/tools/yulopti.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/tools/yulopti.cpp b/test/tools/yulopti.cpp index 397b4a5f2..eff091714 100644 --- a/test/tools/yulopti.cpp +++ b/test/tools/yulopti.cpp @@ -58,6 +58,7 @@ #include #include #include +#include #include @@ -135,7 +136,7 @@ public: cout << " (e)xpr inline/(i)nline/(s)implify/varname c(l)eaner/(u)nusedprune/ss(a) transform/" << endl; cout << " (r)edundant assign elim./re(m)aterializer/f(o)r-loop-init-rewriter/f(O)r-loop-condition-into-body/" << endl; cout << " s(t)ructural simplifier/equi(v)alent function combiner/ssa re(V)erser/? " << endl; - cout << " co(n)trol flow simplifier/stack com(p)ressor/(D)ead code eliminator/? " << endl; + cout << " co(n)trol flow simplifier/stack com(p)ressor/(D)ead code eliminator/(L)oad resolver/? " << endl; cout.flush(); int option = readStandardInputChar(); cout << ' ' << char(option) << endl; @@ -216,6 +217,9 @@ public: StackCompressor::run(m_dialect, obj, true, 16); break; } + case 'L': + LoadResolver::run(m_dialect, *m_ast); + break; default: cout << "Unknown option." << endl; } From 59df7dbc1ba43fdd9174fa4e9edf4544f8ec2419 Mon Sep 17 00:00:00 2001 From: Alex Beregszaszi Date: Tue, 29 Jan 2019 15:15:58 +0000 Subject: [PATCH 12/20] Add upper bound assert for PushSub/PushSubSize --- libevmasm/Assembly.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libevmasm/Assembly.cpp b/libevmasm/Assembly.cpp index 494b2ed93..183cba930 100644 --- a/libevmasm/Assembly.cpp +++ b/libevmasm/Assembly.cpp @@ -596,12 +596,14 @@ LinkerObject const& Assembly::assemble() const ret.bytecode.resize(ret.bytecode.size() + bytesPerDataRef); break; case PushSub: + assertThrow(i.data() <= size_t(-1), AssemblyException, ""); ret.bytecode.push_back(dataRefPush); subRef.insert(make_pair(size_t(i.data()), ret.bytecode.size())); ret.bytecode.resize(ret.bytecode.size() + bytesPerDataRef); break; case PushSubSize: { + assertThrow(i.data() <= size_t(-1), AssemblyException, ""); auto s = m_subs.at(size_t(i.data()))->assemble().bytecode.size(); i.setPushedValue(u256(s)); uint8_t b = max(1, dev::bytesRequired(s)); From 4837ef4adab1e08c91fdb2872a9ffba9afafbcf7 Mon Sep 17 00:00:00 2001 From: Bhargava Shastry Date: Tue, 27 Aug 2019 14:57:45 +0200 Subject: [PATCH 13/20] Yul proto fuzzer: Make function definition a statement --- test/tools/ossfuzz/protoToYul.cpp | 527 ++++++++++++++++++------------ test/tools/ossfuzz/protoToYul.h | 170 +++++++--- test/tools/ossfuzz/yulProto.proto | 106 ++---- 3 files changed, 455 insertions(+), 348 deletions(-) diff --git a/test/tools/ossfuzz/protoToYul.cpp b/test/tools/ossfuzz/protoToYul.cpp index 50ef07f04..0bb7559a3 100644 --- a/test/tools/ossfuzz/protoToYul.cpp +++ b/test/tools/ossfuzz/protoToYul.cpp @@ -86,10 +86,34 @@ string ProtoConverter::visit(Literal const& _x) } } +bool ProtoConverter::varDeclAvailable() +{ + if (m_inFunctionDef) + return m_scopeVars.top().size() > 0; + else + return m_variables.size() > 0; +} + +bool ProtoConverter::functionCallNotPossible(FunctionCall_Returns _type) +{ + return _type == FunctionCall::SINGLE || + (_type == FunctionCall::MULTIASSIGN && !varDeclAvailable()); +} + void ProtoConverter::visit(VarRef const& _x) { - yulAssert(m_variables.size() > 0, "Proto fuzzer: No variables to reference."); - m_output << m_variables[_x.varnum() % m_variables.size()]; + if (m_inFunctionDef) + { + // Ensure that there is at least one variable declaration to reference in function scope. + yulAssert(m_scopeVars.top().size() > 0, "Proto fuzzer: No variables to reference."); + m_output << m_scopeVars.top()[_x.varnum() % m_scopeVars.top().size()]; + } + else + { + // Ensure that there is at least one variable declaration to reference in nested scopes. + yulAssert(m_variables.size() > 0, "Proto fuzzer: No variables to reference."); + m_output << m_variables[_x.varnum() % m_variables.size()]; + } } void ProtoConverter::visit(Expression const& _x) @@ -97,7 +121,13 @@ void ProtoConverter::visit(Expression const& _x) switch (_x.expr_oneof_case()) { case Expression::kVarref: - visit(_x.varref()); + // If the expression requires a variable reference that we cannot provide + // (because there are no variables in scope), we silently output a literal + // expression from the optimizer dictionary. + if (!varDeclAvailable()) + m_output << dictionaryToken(); + else + visit(_x.varref()); break; case Expression::kCons: m_output << visit(_x.cons()); @@ -115,7 +145,12 @@ void ProtoConverter::visit(Expression const& _x) visit(_x.nop()); break; case Expression::kFuncExpr: - visit(_x.func_expr()); + // FunctionCall must return a single value, otherwise + // we output a trivial expression "1". + if (_x.func_expr().ret() == FunctionCall::SINGLE) + visit(_x.func_expr()); + else + m_output << dictionaryToken(); break; case Expression::EXPR_ONEOF_NOT_SET: m_output << dictionaryToken(); @@ -207,46 +242,10 @@ void ProtoConverter::visit(VarDecl const& _x) m_output << "let " << varName << " := "; visit(_x.expr()); m_output << "\n"; - m_scopes.top().insert(varName); + m_scopeVars.top().push_back(varName); m_variables.push_back(varName); } -void ProtoConverter::visit(EmptyVarDecl const&) -{ - string varName = newVarName(); - m_output << "let " << varName << "\n"; - m_scopes.top().insert(varName); - m_variables.push_back(varName); -} - -void ProtoConverter::visit(MultiVarDecl const& _x) -{ - size_t funcId = (static_cast(_x.func_index()) % m_functionVecMultiReturnValue.size()); - - unsigned numInParams = m_functionVecMultiReturnValue.at(funcId).first; - unsigned numOutParams = m_functionVecMultiReturnValue.at(funcId).second; - - // Ensure that the chosen function returns at least 2 and at most 4 values - yulAssert( - ((numOutParams >= 2) && (numOutParams <= 4)), - "Proto fuzzer: Multi variable declaration calls a function with either too few or too many output params." - ); - - // Obtain variable name suffix - unsigned startIdx = counter(); - m_output << "let "; - vector varsVec = createVars(startIdx, startIdx + numOutParams); - m_output << " := "; - - // Create RHS of multi var decl - m_output << "foo_" << functionTypeToString(NumFunctionReturns::Multiple) << "_" << funcId; - m_output << "("; - visitFunctionInputParams(_x, numInParams); - m_output << ")\n"; - // Add newly minted vars in the multidecl statement to current scope - addToScope(varsVec); -} - void ProtoConverter::visit(TypedVarDecl const& _x) { string varName = newVarName(); @@ -309,7 +308,7 @@ void ProtoConverter::visit(TypedVarDecl const& _x) m_output << " : u256\n"; break; } - m_scopes.top().insert(varName); + m_scopeVars.top().push_back(varName); m_variables.push_back(varName); } @@ -501,9 +500,7 @@ void ProtoConverter::visit(AssignmentStatement const& _x) m_output << "\n"; } -// Called at the time function call is being made -template -void ProtoConverter::visitFunctionInputParams(T const& _x, unsigned _numInputParams) +void ProtoConverter::visitFunctionInputParams(FunctionCall const& _x, unsigned _numInputParams) { // We reverse the order of function input visits since it helps keep this switch case concise. switch (_numInputParams) @@ -531,83 +528,151 @@ void ProtoConverter::visitFunctionInputParams(T const& _x, unsigned _numInputPar } } -void ProtoConverter::visit(MultiAssignment const& _x) +bool ProtoConverter::functionValid(FunctionCall_Returns _type, unsigned _numOutParams) { - size_t funcId = (static_cast(_x.func_index()) % m_functionVecMultiReturnValue.size()); - unsigned numInParams = m_functionVecMultiReturnValue.at(funcId).first; - unsigned numOutParams = m_functionVecMultiReturnValue.at(funcId).second; - yulAssert( - ((numOutParams >= 2) && (numOutParams <= 4)), - "Proto fuzzer: Multi assignment calls a function that has either too many or too few output parameters." - ); - - // Convert LHS of multi assignment - // We reverse the order of out param visits since the order does not matter. This helps reduce the size of this - // switch statement. - switch (numOutParams) + switch (_type) { - case 4: - visit(_x.out_param4()); - m_output << ", "; - BOOST_FALLTHROUGH; - case 3: - visit(_x.out_param3()); - m_output << ", "; - BOOST_FALLTHROUGH; - case 2: - visit(_x.out_param2()); - m_output << ", "; - visit(_x.out_param1()); - break; - default: - yulAssert(false, "Proto fuzzer: Function call with too many input parameters."); - break; + case FunctionCall::ZERO: + return _numOutParams == 0; + case FunctionCall::SINGLE: + return _numOutParams == 1; + case FunctionCall::MULTIDECL: + case FunctionCall::MULTIASSIGN: + return _numOutParams > 1; } - m_output << " := "; - - // Convert RHS of multi assignment - m_output << "foo_" << functionTypeToString(NumFunctionReturns::Multiple) << "_" << funcId; - m_output << "("; - visitFunctionInputParams(_x, numInParams); - m_output << ")\n"; } -void ProtoConverter::visit(FunctionCallNoReturnVal const& _x) +void ProtoConverter::convertFunctionCall( + FunctionCall const& _x, + std::string _name, + unsigned _numInParams, + bool _newLine +) { - size_t funcId = (static_cast(_x.func_index()) % m_functionVecNoReturnValue.size()); - unsigned numInParams = m_functionVecNoReturnValue.at(funcId); - m_output << "foo_" << functionTypeToString(NumFunctionReturns::None) << "_" << funcId; - m_output << "("; - visitFunctionInputParams(_x, numInParams); - m_output << ")\n"; -} - -void ProtoConverter::visit(FunctionCallSingleReturnVal const& _x) -{ - size_t funcId = (static_cast(_x.func_index()) % m_functionVecSingleReturnValue.size()); - unsigned numInParams = m_functionVecSingleReturnValue.at(funcId); - m_output << "foo_" << functionTypeToString(NumFunctionReturns::Single) << "_" << funcId; - m_output << "("; - visitFunctionInputParams(_x, numInParams); + m_output << _name << "("; + visitFunctionInputParams(_x, _numInParams); m_output << ")"; + if (_newLine) + m_output << "\n"; +} + +vector ProtoConverter::createVarDecls(unsigned _start, unsigned _end, bool _isAssignment) +{ + m_output << "let "; + vector varsVec = createVars(_start, _end); + if (_isAssignment) + m_output << " := "; + else + m_output << "\n"; + return varsVec; } void ProtoConverter::visit(FunctionCall const& _x) { - switch (_x.functioncall_oneof_case()) + bool functionAvailable = m_functionSigMap.size() > 0; + unsigned numInParams, numOutParams; + string funcName; + FunctionCall_Returns funcType = _x.ret(); + if (functionAvailable) { - case FunctionCall::kCallZero: - visit(_x.call_zero()); + yulAssert(m_functions.size() > 0, "Proto fuzzer: No function in scope"); + funcName = m_functions[_x.func_index() % m_functions.size()]; + auto ret = m_functionSigMap.at(funcName); + numInParams = ret.first; + numOutParams = ret.second; + } + else + { + // If there are no functions available, calls to functions that + // return a single value may be replaced by a dictionary token. + if (funcType == FunctionCall::SINGLE) + m_output << dictionaryToken(); + return; + } + + // If function selected for function call does not meet interface + // requirements (num output values) for the function type + // specified, then we return early unless it is a function call + // that returns a single value (which may be replaced by a + // dictionary token. + if (!functionValid(funcType, numOutParams)) + { + if (funcType == FunctionCall::SINGLE) + m_output << dictionaryToken(); + return; + } + + // If we are here, it means that we have at least one valid + // function for making the function call + switch (funcType) + { + case FunctionCall::ZERO: + convertFunctionCall(_x, funcName, numInParams); break; - case FunctionCall::kCallMultidecl: - // Hack: Disallow (multi) variable declarations until scope extension is implemented for "for-init" + case FunctionCall::SINGLE: + // Since functions that return a single value are used as expressions + // we do not print a newline because it is done by the expression + // visitor. + convertFunctionCall(_x, funcName, numInParams, /*newLine=*/false); + break; + case FunctionCall::MULTIDECL: + // Hack: Disallow (multi) variable declarations until scope extension + // is implemented for "for-init" if (!m_inForInitScope) - visit(_x.call_multidecl()); + { + // Ensure that the chosen function returns at most 4 values + yulAssert( + numOutParams <= 4, + "Proto fuzzer: Function call with too many output params encountered." + ); + + // Obtain variable name suffix + unsigned startIdx = counter(); + vector varsVec = createVarDecls( + startIdx, + startIdx + numOutParams, + /*isAssignment=*/true + ); + + // Create RHS of multi var decl + convertFunctionCall(_x, funcName, numInParams); + // Add newly minted vars in the multidecl statement to current scope + addVarsToScope(varsVec); + } break; - case FunctionCall::kCallMultiassign: - visit(_x.call_multiassign()); - break; - case FunctionCall::FUNCTIONCALL_ONEOF_NOT_SET: + case FunctionCall::MULTIASSIGN: + // Ensure that the chosen function returns at most 4 values + yulAssert( + numOutParams <= 4, + "Proto fuzzer: Function call with too many output params encountered." + ); + + // Convert LHS of multi assignment + // We reverse the order of out param visits since the order does not matter. + // This helps reduce the size of this switch statement. + switch (numOutParams) + { + case 4: + visit(_x.out_param4()); + m_output << ", "; + BOOST_FALLTHROUGH; + case 3: + visit(_x.out_param3()); + m_output << ", "; + BOOST_FALLTHROUGH; + case 2: + visit(_x.out_param2()); + m_output << ", "; + visit(_x.out_param1()); + break; + default: + yulAssert(false, "Proto fuzzer: Function call with too many or too few input parameters."); + break; + } + m_output << " := "; + + // Convert RHS of multi assignment + convertFunctionCall(_x, funcName, numInParams); break; } } @@ -743,7 +808,7 @@ void ProtoConverter::visit(SwitchStmt const& _x) { if (_x.case_stmt_size() > 0 || _x.has_default_block()) { - std::set s; + std::set s; m_switchLiteralSetPerScope.push(s); m_output << "switch "; visit(_x.switch_expr()); @@ -829,7 +894,10 @@ void ProtoConverter::visit(Statement const& _x) visit(_x.decl()); break; case Statement::kAssignment: - visit(_x.assignment()); + // Create an assignment statement only if there is at least one variable + // declaration that is in scope. + if (varDeclAvailable()) + visit(_x.assignment()); break; case Statement::kIfstmt: visit(_x.ifstmt()); @@ -870,8 +938,15 @@ void ProtoConverter::visit(Statement const& _x) visit(_x.terminatestmt()); break; case Statement::kFunctioncall: + // Return early if a function call cannot be created + if (functionCallNotPossible(_x.functioncall().ret())) + return; visit(_x.functioncall()); break; + case Statement::kFuncdef: + if (!m_inForInitScope) + visit(_x.funcdef()); + break; case Statement::STMT_ONEOF_NOT_SET: break; } @@ -879,34 +954,74 @@ void ProtoConverter::visit(Statement const& _x) void ProtoConverter::openScope(vector const& _funcParams) { - m_scopes.push({}); - addToScope(_funcParams); + m_scopeVars.push({}); + m_scopeFuncs.push({}); + if (!_funcParams.empty()) + addVarsToScope(_funcParams); +} + +void ProtoConverter::updateFunctionMaps(string const& _var) +{ + unsigned erased = m_functionSigMap.erase(_var); + + for (auto const& i: m_functionDefMap) + if (i.second == _var) + { + erased += m_functionDefMap.erase(i.first); + break; + } + + yulAssert(erased == 2, "Proto fuzzer: Function maps not updated"); } void ProtoConverter::closeScope() { - for (auto const& var: m_scopes.top()) + for (auto const& var: m_scopeVars.top()) { - unsigned numErased = m_variables.size(); + unsigned numVarsRemoved = m_variables.size(); m_variables.erase(remove(m_variables.begin(), m_variables.end(), var), m_variables.end()); - numErased -= m_variables.size(); - yulAssert(numErased == 1, "Proto fuzzer: More than one variable went out of scope"); + numVarsRemoved -= m_variables.size(); + yulAssert( + numVarsRemoved == 1, + "Proto fuzzer: Nothing or too much went out of scope" + ); } - m_scopes.pop(); + m_scopeVars.pop(); + + for (auto const& f: m_scopeFuncs.top()) + { + unsigned numFuncsRemoved = m_functions.size(); + m_functions.erase(remove(m_functions.begin(), m_functions.end(), f), m_functions.end()); + numFuncsRemoved -= m_functions.size(); + yulAssert( + numFuncsRemoved == 1, + "Proto fuzzer: Nothing or too much went out of scope" + ); + updateFunctionMaps(f); + } + m_scopeFuncs.pop(); } -void ProtoConverter::addToScope(vector const& _vars) +void ProtoConverter::addVarsToScope(vector const& _vars) { for (string const& i: _vars) { m_variables.push_back(i); - m_scopes.top().insert(i); + m_scopeVars.top().push_back(i); } } void ProtoConverter::visit(Block const& _x, vector _funcParams) { openScope(_funcParams); + + // Register function declarations in this scope unless this + // scope belongs to for-init (in which function declarations + // are forbidden). + for (auto const& statement: _x.statements()) + if (statement.has_funcdef() && !m_inForInitScope) + registerFunction(&statement.funcdef()); + if (_x.statements_size() > 0) { m_output << "{\n"; @@ -919,18 +1034,6 @@ void ProtoConverter::visit(Block const& _x, vector _funcParams) closeScope(); } -void ProtoConverter::visit(SpecialBlock const& _x, vector _funcParams) -{ - openScope(_funcParams); - m_output << "{\n"; - visit(_x.var()); - if (_x.statements_size() > 0) - for (auto const& st: _x.statements()) - visit(st); - m_output << "}\n"; - closeScope(); -} - vector ProtoConverter::createVars(unsigned _startIdx, unsigned _endIdx) { yulAssert(_endIdx > _startIdx, "Proto fuzzer: Variable indices not in range"); @@ -952,24 +1055,49 @@ vector ProtoConverter::createVars(unsigned _startIdx, unsigned _endIdx) return varsVec; } -template -void ProtoConverter::createFunctionDefAndCall(T const& _x, unsigned _numInParams, unsigned _numOutParams, NumFunctionReturns _type) +void ProtoConverter::registerFunction(FunctionDef const* _x) +{ + unsigned numInParams = _x->num_input_params() % s_modInputParams; + unsigned numOutParams = _x->num_output_params() % s_modOutputParams; + NumFunctionReturns numReturns; + if (numOutParams == 0) + numReturns = NumFunctionReturns::None; + else if (numOutParams == 1) + numReturns = NumFunctionReturns::Single; + else + numReturns = NumFunctionReturns::Multiple; + + // Generate function name + string funcName = functionName(numReturns); + + // Register function + auto ret = m_functionSigMap.emplace(make_pair(funcName, make_pair(numInParams, numOutParams))); + yulAssert(ret.second, "Proto fuzzer: Function already exists."); + m_functions.push_back(funcName); + m_scopeFuncs.top().push_back(funcName); + m_functionDefMap.emplace(make_pair(_x, funcName)); +} + +void ProtoConverter::createFunctionDefAndCall( + FunctionDef const& _x, + unsigned _numInParams, + unsigned _numOutParams +) { yulAssert( ((_numInParams <= s_modInputParams - 1) && (_numOutParams <= s_modOutputParams - 1)), "Proto fuzzer: Too many function I/O parameters requested." ); - // Signature - // This creates function foo__(x_0,...,x_n) - m_output << "function foo_" << functionTypeToString(_type) << "_" << m_numFunctionSets; - m_output << "("; + // Obtain function name + yulAssert(m_functionDefMap.count(&_x), "Proto fuzzer: Unregistered function"); + string funcName = m_functionDefMap.at(&_x); + vector varsVec = {}; + m_output << "function " << funcName << "("; + unsigned startIdx = counter(); if (_numInParams > 0) - // Functions must use 0 as the first variable's index until function definition - // is made a statement. Once function definition as statement is implemented, - // start index becomes m_counter. - varsVec = createVars(0, _numInParams); + varsVec = createVars(startIdx, startIdx + _numInParams); m_output << ")"; vector outVarsVec = {}; @@ -978,68 +1106,58 @@ void ProtoConverter::createFunctionDefAndCall(T const& _x, unsigned _numInParams { m_output << " -> "; if (varsVec.empty()) - varsVec = createVars(_numInParams, _numInParams + _numOutParams); + { + yulAssert(_numInParams == 0, "Proto fuzzer: Input parameters not processed correctly"); + varsVec = createVars(startIdx, startIdx + _numOutParams); + } else { - outVarsVec = createVars(_numInParams, _numInParams + _numOutParams); + outVarsVec = createVars(startIdx + _numInParams, startIdx + _numInParams + _numOutParams); varsVec.insert(varsVec.end(), outVarsVec.begin(), outVarsVec.end()); } } + yulAssert(varsVec.size() == _numInParams + _numOutParams, "Proto fuzzer: Function parameters not processed correctly"); m_output << "\n"; + // If function definition is in for-loop body, update + bool wasInForBody = m_inForBodyScope; + m_inForBodyScope = false; + + bool wasInFunctionDef = m_inFunctionDef; + m_inFunctionDef = true; + // Body - visit(_x.statements(), varsVec); + visit(_x.block(), varsVec); + + m_inForBodyScope = wasInForBody; + m_inFunctionDef = wasInFunctionDef; // Manually create a multi assignment using global variables // This prints a_0, ..., a_k-1 for this function that returns "k" values - if (_numOutParams > 0) - m_output << dev::suffixedVariableNameList("a_", 0, _numOutParams) << " := "; - - // Call the function with the correct number of input parameters via calls to calldataload with - // incremental addresses. - m_output << "foo_" << functionTypeToString(_type) << "_" << std::to_string(m_numFunctionSets); - m_output << "("; - for (unsigned i = 0; i < _numInParams; i++) - { - m_output << "calldataload(" << std::to_string(i*32) << ")"; - if (i < _numInParams - 1) - m_output << ","; - } - m_output << ")\n"; - - for (unsigned i = 0; i < _numOutParams; i++) - m_output << "sstore(" << std::to_string(i*32) << ", a_" << std::to_string(i) << ")\n"; +// if (_numOutParams > 0) +// m_output << dev::suffixedVariableNameList("a_", 0, _numOutParams) << " := "; +// +// // Call the function with the correct number of input parameters via calls to calldataload with +// // incremental addresses. +// m_output << funcName << "("; +// for (unsigned i = 0; i < _numInParams; i++) +// { +// m_output << "calldataload(" << std::to_string(i*32) << ")"; +// if (i < _numInParams - 1) +// m_output << ","; +// } +// m_output << ")\n"; +// +// for (unsigned i = 0; i < _numOutParams; i++) +// m_output << "sstore(" << std::to_string(i*32) << ", a_" << std::to_string(i) << ")\n"; } -void ProtoConverter::visit(FunctionDefinitionNoReturnVal const& _x) +void ProtoConverter::visit(FunctionDef const& _x) { unsigned numInParams = _x.num_input_params() % s_modInputParams; - unsigned numOutParams = 0; - createFunctionDefAndCall(_x, numInParams, numOutParams, NumFunctionReturns::None); -} - -void ProtoConverter::visit(FunctionDefinitionSingleReturnVal const& _x) -{ - unsigned numInParams = _x.num_input_params() % s_modInputParams; - unsigned numOutParams = 1; - createFunctionDefAndCall(_x, numInParams, numOutParams, NumFunctionReturns::Single); -} - -void ProtoConverter::visit(FunctionDefinitionMultiReturnVal const& _x) -{ - unsigned numInParams = _x.num_input_params() % s_modInputParams; - // Synthesize at least 2 return parameters and at most (s_modOutputParams - 1) - unsigned numOutParams = std::max(2, _x.num_output_params() % s_modOutputParams); - createFunctionDefAndCall(_x, numInParams, numOutParams, NumFunctionReturns::Multiple); -} - -void ProtoConverter::visit(FunctionDefinition const& _x) -{ - visit(_x.fd_zero()); - visit(_x.fd_one()); - visit(_x.fd_multi()); - m_numFunctionSets++; + unsigned numOutParams = _x.num_output_params() % s_modOutputParams; + createFunctionDefAndCall(_x, numInParams, numOutParams); } void ProtoConverter::visit(Program const& _x) @@ -1048,25 +1166,14 @@ void ProtoConverter::visit(Program const& _x) m_inputSize = _x.ByteSizeLong(); /* Program template is as follows - * Four Globals a_0, a_1, a_2, and a_3 to hold up to four function return values - * - * Repeated function definitions followed by function calls of the respective function + * Zero or more statements. If function definition is present, it is + * called post definition. * Example: function foo(x_0) -> x_1 {} - * a_0 := foo(calldataload(0)) - * sstore(0, a_0) + * x_2 := foo(calldataload(0)) + * sstore(0, x_2) */ m_output << "{\n"; - // Create globals at the beginning - // This creates let a_0, a_1, a_2, a_3 (followed by a new line) - m_output << "let " << dev::suffixedVariableNameList("a_", 0, s_modOutputParams - 1) << "\n"; - // Register function interface. Useful while visiting multi var decl/assignment statements. - for (auto const& f: _x.funcs()) - registerFunction(f); - - for (auto const& f: _x.funcs()) - visit(f); - - yulAssert((unsigned)_x.funcs_size() == m_numFunctionSets, "Proto fuzzer: Functions not correctly registered."); + visit(_x.block()); m_output << "}\n"; } @@ -1076,25 +1183,15 @@ string ProtoConverter::programToString(Program const& _input) return m_output.str(); } -void ProtoConverter::registerFunction(FunctionDefinition const& _x) -{ - // No return and single return functions explicitly state the number of values returned - registerFunction(_x.fd_zero(), NumFunctionReturns::None); - registerFunction(_x.fd_one(), NumFunctionReturns::Single); - // A multi return function can have between two and (s_modOutputParams - 1) parameters - unsigned numOutParams = std::max(2, _x.fd_multi().num_output_params() % s_modOutputParams); - registerFunction(_x.fd_multi(), NumFunctionReturns::Multiple, numOutParams); -} - std::string ProtoConverter::functionTypeToString(NumFunctionReturns _type) { switch (_type) { case NumFunctionReturns::None: - return "noreturn"; + return "n"; case NumFunctionReturns::Single: - return "singlereturn"; + return "s"; case NumFunctionReturns::Multiple: - return "multireturn"; + return "m"; } } \ No newline at end of file diff --git a/test/tools/ossfuzz/protoToYul.h b/test/tools/ossfuzz/protoToYul.h index a3f617a82..329f4663d 100644 --- a/test/tools/ossfuzz/protoToYul.h +++ b/test/tools/ossfuzz/protoToYul.h @@ -40,12 +40,12 @@ class ProtoConverter public: ProtoConverter() { - m_numFunctionSets = 0; m_inForBodyScope = false; m_inForInitScope = false; m_numNestedForLoops = 0; m_counter = 0; m_inputSize = 0; + m_inFunctionDef = false; } ProtoConverter(ProtoConverter const&) = delete; ProtoConverter(ProtoConverter&&) = delete; @@ -53,32 +53,23 @@ public: private: void visit(BinaryOp const&); + /// Visits a basic block optionally adding @a _funcParams to scope. /// @param _block Reference to a basic block of yul statements. /// @param _funcParams List of function parameter names, defaults to /// an empty vector. void visit(Block const& _block, std::vector _funcParams = {}); - /// Visits a basic block that contains a variable declaration at the - /// very beginning, optionally adding @a _funcParams to scope. - /// @param _block Reference to a basic block of yul statements. - /// @param _funcParams List of function parameter names, defaults to - /// an empty vector. - void visit(SpecialBlock const& _block, std::vector _funcParams = {}); std::string visit(Literal const&); void visit(VarRef const&); void visit(Expression const&); void visit(VarDecl const&); - void visit(EmptyVarDecl const&); - void visit(MultiVarDecl const&); void visit(TypedVarDecl const&); void visit(UnaryOp const&); void visit(AssignmentStatement const&); - void visit(MultiAssignment const&); void visit(IfStmt const&); void visit(StoreFunc const&); void visit(Statement const&); - void visit(FunctionDefinition const&); void visit(ForStmt const&); void visit(BoundedForStmt const&); void visit(CaseStmt const&); @@ -92,21 +83,17 @@ private: void visit(RetRevStmt const&); void visit(SelfDestructStmt const&); void visit(TerminatingStmt const&); - void visit(FunctionCallNoReturnVal const&); - void visit(FunctionCallSingleReturnVal const&); void visit(FunctionCall const&); - void visit(FunctionDefinitionNoReturnVal const&); - void visit(FunctionDefinitionSingleReturnVal const&); - void visit(FunctionDefinitionMultiReturnVal const&); + void visit(FunctionDef const&); void visit(Program const&); - void registerFunction(FunctionDefinition const&); - /// Creates a new scope, and optionally adds @a _funcParams to it + /// Creates a new scope, and adds @a _funcParams to it if it + /// is non-empty. void openScope(std::vector const& _funcParams); /// Closes current scope void closeScope(); /// Adds @a _vars to current scope - void addToScope(std::vector const& _vars); + void addVarsToScope(std::vector const& _vars); std::string createHex(std::string const& _hexBytes); @@ -126,33 +113,101 @@ private: Multiple }; - template - void visitFunctionInputParams(T const&, unsigned); + void visitFunctionInputParams(FunctionCall const&, unsigned); + void createFunctionDefAndCall(FunctionDef const&, unsigned, unsigned); - template - void createFunctionDefAndCall(T const&, unsigned, unsigned, NumFunctionReturns); + /// Convert function type to a string to be used while naming a + /// function that is created by a function declaration statement. + /// @param _type Type classified according to the number of + /// values returned by function. + /// @return A string as follows. If _type is + /// None -> "n" + /// Single -> "s" + /// Multiple -> "m" std::string functionTypeToString(NumFunctionReturns _type); - /// Creates variable declarations "x_<_startIdx>",...,"x_<_endIdx - 1>" + /// Return true if at least one variable declaration is in scope, + /// false otherwise. + /// @return True in the following cases: + /// - If we are inside a function that has already declared a variable + /// - If there is at least one variable declaration that is + /// in scope + bool varDeclAvailable(); + + /// Return true if a function call cannot be made, false otherwise. + /// @param _type is an enum denoting the type of function call. It + /// can be one of NONE, SINGLE, MULTIDECL, MULTIASSIGN. + /// NONE -> Function call does not return a value + /// SINGLE -> Function call returns a single value + /// MULTIDECL -> Function call returns more than one value + /// and it is used to create a multi declaration + /// statement + /// MULTIASSIGN -> Function call returns more than one value + /// and it is used to create a multi assignment + /// statement + /// @return True if the function call cannot be created for one of the + /// following reasons + // - It is a SINGLE function call (we reserve SINGLE functions for + // expressions) + // - It is a MULTIASSIGN function call and we do not have any + // variables available for assignment. + bool functionCallNotPossible(FunctionCall_Returns _type); + + /// Checks if function call of type @a _type returns the correct number + /// of values. + /// @param _type Function call type of the function being checked + /// @param _numOutParams Number of values returned by the function + /// being checked + /// @return true if the function returns the correct number of values, + /// false otherwise + bool functionValid(FunctionCall_Returns _type, unsigned _numOutParams); + + /// Converts protobuf function call to a yul function call and appends + /// it to output stream. + /// @param _x Protobuf function call + /// @param _name Function name + /// @param _numInParams Number of input arguments accepted by function + /// @param _newLine Flag that prints a new line to the output stream if + /// true. Default value for the flag is true. + void convertFunctionCall( + FunctionCall const& _x, + std::string _name, + unsigned _numInParams, + bool _newLine = true + ); + + /// Prints a yul formatted variable declaration statement to the output + /// stream. + /// Example 1: createVarDecls(0, 1, true) returns {"x_0"} and prints + /// let x_0 := + /// Example 2: createVarDecls(0, 2, false) returns {"x_0", "x_1"} and prints + /// let x_0, x_1 + /// @param _start Start index of variable (inclusive) + /// @param _end End index of variable (exclusive) + /// @param _isAssignment Flag indicating if variable declaration is also + /// an assignment. If true, the string " := " follows the variable + /// declaration. Otherwise, a new line is follows the variable + /// declaration. + /// @return A vector of strings containing the variable names used in + /// the declaration statement. + std::vector createVarDecls(unsigned _start, unsigned _end, bool _isAssignment); + + /// Prints comma separated variable names to output stream and + /// returns a vector containing the printed variable names. + /// Example: createVars(0, 2) returns {"x_0", "x_1"} and prints + /// x_0, x_1 + /// @param _startIdx Start index of variable (inclusive) + /// @param _endIdx End index of variable (exclusive) + /// @return A vector of strings containing the printed variable names. std::vector createVars(unsigned _startIdx, unsigned _endIdx); - template - void registerFunction(T const& _x, NumFunctionReturns _type, unsigned _numOutputParams = 0) - { - unsigned numInputParams = _x.num_input_params() % s_modInputParams; - switch (_type) - { - case NumFunctionReturns::None: - m_functionVecNoReturnValue.push_back(numInputParams); - break; - case NumFunctionReturns::Single: - m_functionVecSingleReturnValue.push_back(numInputParams); - break; - case NumFunctionReturns::Multiple: - m_functionVecMultiReturnValue.push_back(std::make_pair(numInputParams, _numOutputParams)); - break; - } - } + /// Register a function declaration + /// @param _f Pointer to a FunctionDef object + void registerFunction(FunctionDef const* _f); + + /// Removes entry from m_functionMap and m_functionName + void updateFunctionMaps(std::string const& _x); + /// Returns a pseudo-random dictionary token. /// @param _p Enum that decides if the returned token is hex prefixed ("0x") or not /// @return Dictionary token at the index computed using a @@ -168,33 +223,46 @@ private: return m_counter++; } + /// Generate function name of the form "foo__". + /// @param _type Type classified according to the number of + /// values returned by function. + std::string functionName(NumFunctionReturns _type) + { + return "foo_" + functionTypeToString(_type) + "_" + std::to_string(counter()); + } + std::ostringstream m_output; - /// Scope - std::stack> m_scopes; + /// Variables in current scope + std::stack> m_scopeVars; + /// Functions in current scope + std::stack> m_scopeFuncs; /// Variables std::vector m_variables; + /// Functions + std::vector m_functions; + /// Maps FunctionDef object to its name + std::map m_functionDefMap; // Set that is used for deduplicating switch case literals std::stack> m_switchLiteralSetPerScope; - // Total number of function sets. A function set contains one function of each type defined by - // NumFunctionReturns - unsigned m_numFunctionSets; // Look-up table per function type that holds the number of input (output) function parameters - std::vector m_functionVecNoReturnValue; - std::vector m_functionVecSingleReturnValue; - std::vector> m_functionVecMultiReturnValue; + std::map> m_functionSigMap; // mod input/output parameters impose an upper bound on the number of input/output parameters a function may have. static unsigned constexpr s_modInputParams = 5; static unsigned constexpr s_modOutputParams = 5; - // predicate to keep track of for body scope + /// Predicate to keep track of for body scope. If true, break/continue + /// statements can not be created. bool m_inForBodyScope; // Index used for naming loop variable of bounded for loops unsigned m_numNestedForLoops; - // predicate to keep track of for loop init scope + /// Predicate to keep track of for loop init scope. If true, variable + /// or function declarations can not be created. bool m_inForInitScope; /// Monotonically increasing counter unsigned m_counter; /// Size of protobuf input unsigned m_inputSize; + /// Predicate that is true if inside function definition, false otherwise + bool m_inFunctionDef; }; } } diff --git a/test/tools/ossfuzz/yulProto.proto b/test/tools/ossfuzz/yulProto.proto index 4a39ab258..910903afa 100644 --- a/test/tools/ossfuzz/yulProto.proto +++ b/test/tools/ossfuzz/yulProto.proto @@ -21,54 +21,24 @@ message VarDecl { required Expression expr = 1; } -message FunctionCallNoReturnVal { - // Indexes a function that does not return anything - required uint32 func_index = 1; - required Expression in_param1 = 2; - required Expression in_param2 = 3; - required Expression in_param3 = 4; - required Expression in_param4 = 5; -} - -// Used by Expression -message FunctionCallSingleReturnVal { - // Indexes a function that returns exactly one value - required uint32 func_index = 1; - required Expression in_param1 = 2; - required Expression in_param2 = 3; - required Expression in_param3 = 4; - required Expression in_param4 = 5; -} - -message MultiVarDecl { - // Indexes a function that returns more than one value - required uint32 func_index = 1; - required Expression in_param1 = 2; - required Expression in_param2 = 3; - required Expression in_param3 = 4; - required Expression in_param4 = 5; -} - -message MultiAssignment { - // Indexes a function that returns more than one value - required uint32 func_index = 1; - required Expression in_param1 = 2; - required Expression in_param2 = 3; - required Expression in_param3 = 4; - required Expression in_param4 = 5; - required VarRef out_param1 = 6; - required VarRef out_param2 = 7; - required VarRef out_param3 = 8; - required VarRef out_param4 = 9; -} - -// We exclude function calls with single return value here and use them as expressions message FunctionCall { - oneof functioncall_oneof { - FunctionCallNoReturnVal call_zero = 1; - MultiVarDecl call_multidecl = 2; - MultiAssignment call_multiassign = 3; + enum Returns { + ZERO = 1; + SINGLE = 2; + MULTIDECL = 3; + MULTIASSIGN = 4; } + required Returns ret = 1; + // Indexes an existing function + required uint32 func_index = 2; + required Expression in_param1 = 3; + required Expression in_param2 = 4; + required Expression in_param3 = 5; + required Expression in_param4 = 6; + required VarRef out_param1 = 7; + required VarRef out_param2 = 8; + required VarRef out_param3 = 9; + required VarRef out_param4 = 10; } message TypedVarDecl { @@ -242,7 +212,7 @@ message Expression { UnaryOp unop = 4; TernaryOp top = 5; NullaryOp nop = 6; - FunctionCallSingleReturnVal func_expr = 7; + FunctionCall func_expr = 7; } } @@ -311,10 +281,12 @@ message TerminatingStmt { } } -// Stub for a VarDecl without an Expression on the RHS -message EmptyVarDecl {} +message FunctionDef { + required uint32 num_input_params = 1; + required uint32 num_output_params = 2; + required Block block = 3; +} -// TODO: Make Function definition a Statement message Statement { oneof stmt_oneof { VarDecl decl = 1; @@ -332,6 +304,7 @@ message Statement { TerminatingStmt terminatestmt = 13; FunctionCall functioncall = 14; BoundedForStmt boundedforstmt = 15; + FunctionDef funcdef = 16; } } @@ -339,39 +312,8 @@ message Block { repeated Statement statements = 1; } -// Identical to Block with the addition of an empty var right at the top -// Used by FunctionDefinitionNoReturnVal only. -message SpecialBlock { - required EmptyVarDecl var = 1; - repeated Statement statements = 2; -} - -// This ensures that proto mutator generates at least one of each type if it creates at least 1 functiondef message. -message FunctionDefinition { - required FunctionDefinitionNoReturnVal fd_zero = 1; - required FunctionDefinitionSingleReturnVal fd_one = 2; - required FunctionDefinitionMultiReturnVal fd_multi = 3; -} - -// Since this function can have 0 parameters, we hoist an empty var decl at the top via SpecialBlock. -message FunctionDefinitionNoReturnVal { - required uint32 num_input_params = 1; - required SpecialBlock statements = 2; -} - -message FunctionDefinitionSingleReturnVal { - required uint32 num_input_params = 1; - required Block statements = 2; -} - -message FunctionDefinitionMultiReturnVal { - required uint32 num_input_params = 1; - required uint32 num_output_params = 2; - required Block statements = 3; -} - message Program { - repeated FunctionDefinition funcs = 1; + required Block block = 1; } package yul.test.yul_fuzzer; From d066ba71a45f145ddbcb397e97f65f3bc679e86c Mon Sep 17 00:00:00 2001 From: Bhargava Shastry Date: Tue, 3 Sep 2019 14:28:01 +0200 Subject: [PATCH 14/20] Force call of every valid func declaration --- test/tools/ossfuzz/protoToYul.cpp | 61 ++++++++++++++++++++++--------- test/tools/ossfuzz/protoToYul.h | 7 ++++ 2 files changed, 50 insertions(+), 18 deletions(-) diff --git a/test/tools/ossfuzz/protoToYul.cpp b/test/tools/ossfuzz/protoToYul.cpp index 0bb7559a3..fef7c9cb1 100644 --- a/test/tools/ossfuzz/protoToYul.cpp +++ b/test/tools/ossfuzz/protoToYul.cpp @@ -1078,6 +1078,44 @@ void ProtoConverter::registerFunction(FunctionDef const* _x) m_functionDefMap.emplace(make_pair(_x, funcName)); } +void ProtoConverter::createFunctionCall( + string _funcName, + unsigned _numInParams, + unsigned _numOutParams +) +{ + // Prints the following to output stream "let x_i,...,x_n := " + unsigned startIdx = counter(); + vector varsVec{}; + if (_numOutParams > 0) + { + varsVec = createVarDecls( + startIdx, + startIdx + _numOutParams, + /*isAssignment=*/true + ); + } + + // Call the function with the correct number of input parameters via calls to calldataload with + // incremental addresses. + m_output << _funcName << "("; + for (unsigned i = 0; i < _numInParams; i++) + { + m_output << "calldataload(" << std::to_string(i*32) << ")"; + if (i < _numInParams - 1) + m_output << ","; + } + m_output << ")\n"; + + // Save output values to storage + for (unsigned i = 0; i < varsVec.size(); i++) + m_output << "sstore(" << std::to_string(i*32) << ", " << varsVec[i] << ")\n"; + + // Add newly minted vars to current scope + if (!varsVec.empty()) + addVarsToScope(varsVec); +} + void ProtoConverter::createFunctionDefAndCall( FunctionDef const& _x, unsigned _numInParams, @@ -1133,24 +1171,11 @@ void ProtoConverter::createFunctionDefAndCall( m_inForBodyScope = wasInForBody; m_inFunctionDef = wasInFunctionDef; - // Manually create a multi assignment using global variables - // This prints a_0, ..., a_k-1 for this function that returns "k" values -// if (_numOutParams > 0) -// m_output << dev::suffixedVariableNameList("a_", 0, _numOutParams) << " := "; -// -// // Call the function with the correct number of input parameters via calls to calldataload with -// // incremental addresses. -// m_output << funcName << "("; -// for (unsigned i = 0; i < _numInParams; i++) -// { -// m_output << "calldataload(" << std::to_string(i*32) << ")"; -// if (i < _numInParams - 1) -// m_output << ","; -// } -// m_output << ")\n"; -// -// for (unsigned i = 0; i < _numOutParams; i++) -// m_output << "sstore(" << std::to_string(i*32) << ", a_" << std::to_string(i) << ")\n"; + yulAssert( + !m_inForInitScope, + "Proto fuzzer: Trying to create function call inside for-init block" + ); + createFunctionCall(funcName, _numInParams, _numOutParams); } void ProtoConverter::visit(FunctionDef const& _x) diff --git a/test/tools/ossfuzz/protoToYul.h b/test/tools/ossfuzz/protoToYul.h index 329f4663d..782e05ef9 100644 --- a/test/tools/ossfuzz/protoToYul.h +++ b/test/tools/ossfuzz/protoToYul.h @@ -201,6 +201,13 @@ private: /// @return A vector of strings containing the printed variable names. std::vector createVars(unsigned _startIdx, unsigned _endIdx); + /// Print the yul syntax to make a call to a function named @a _funcName to + /// the output stream. + /// @param _funcName Name of the function to be called + /// @param _numInParams Number of input parameters in function signature + /// @param _numOutParams Number of output parameters in function signature + void createFunctionCall(std::string _funcName, unsigned _numInParams, unsigned _numOutParams); + /// Register a function declaration /// @param _f Pointer to a FunctionDef object void registerFunction(FunctionDef const* _f); From fce65ec811aecf2c6b3d5e359a7034323538ecb3 Mon Sep 17 00:00:00 2001 From: Bhargava Shastry Date: Tue, 3 Sep 2019 15:14:17 +0200 Subject: [PATCH 15/20] Randomize calldataload and storage slots and use dictionary tokens as function argument --- test/tools/ossfuzz/protoToYul.cpp | 78 +++++++++++++++++++++++++------ test/tools/ossfuzz/protoToYul.h | 15 ++++++ 2 files changed, 78 insertions(+), 15 deletions(-) diff --git a/test/tools/ossfuzz/protoToYul.cpp b/test/tools/ossfuzz/protoToYul.cpp index fef7c9cb1..69cceda6a 100644 --- a/test/tools/ossfuzz/protoToYul.cpp +++ b/test/tools/ossfuzz/protoToYul.cpp @@ -1078,17 +1078,67 @@ void ProtoConverter::registerFunction(FunctionDef const* _x) m_functionDefMap.emplace(make_pair(_x, funcName)); } +void ProtoConverter::fillFunctionCallInput(unsigned _numInParams) +{ + for (unsigned i = 0; i < _numInParams; i++) + { + // Throw a 4-sided dice to choose whether to populate function input + // argument from a pseudo-randomly chosen slot in one of the following + // locations: calldata, memory, storage, or yul optimizer dictionary. + unsigned diceValue = counter() % 4; + // Pseudo-randomly choose one of the first ten 32-byte + // aligned slots. + string slot = to_string((counter() % 10) * 32); + switch (diceValue) + { + case 0: + m_output << "calldataload(" << slot << ")"; + break; + case 1: + m_output << "mload(" << slot << ")"; + break; + case 2: + m_output << "sload(" << slot << ")"; + break; + case 3: + // Call to dictionaryToken() automatically picks a token + // at a pseudo-random location. + m_output << dictionaryToken(); + break; + } + if (i < _numInParams - 1) + m_output << ","; + } +} + +void ProtoConverter::saveFunctionCallOutput(vector const& _varsVec) +{ + for (auto const& var: _varsVec) + { + // Flip a dice to choose whether to save output values + // in storage or memory. + bool coinFlip = counter() % 2 == 0; + // Pseudo-randomly choose one of the first ten 32-byte + // aligned slots. + string slot = to_string((counter() % 10) * 32); + if (coinFlip) + m_output << "sstore(" << slot << ", " << var << ")\n"; + else + m_output << "mstore(" << slot << ", " << var << ")\n"; + } +} + void ProtoConverter::createFunctionCall( string _funcName, unsigned _numInParams, unsigned _numOutParams ) { - // Prints the following to output stream "let x_i,...,x_n := " - unsigned startIdx = counter(); vector varsVec{}; if (_numOutParams > 0) { + unsigned startIdx = counter(); + // Prints the following to output stream "let x_i,...,x_n := " varsVec = createVarDecls( startIdx, startIdx + _numOutParams, @@ -1096,24 +1146,22 @@ void ProtoConverter::createFunctionCall( ); } - // Call the function with the correct number of input parameters via calls to calldataload with - // incremental addresses. + // Call the function with the correct number of input parameters m_output << _funcName << "("; - for (unsigned i = 0; i < _numInParams; i++) - { - m_output << "calldataload(" << std::to_string(i*32) << ")"; - if (i < _numInParams - 1) - m_output << ","; - } + if (_numInParams > 0) + fillFunctionCallInput(_numInParams); m_output << ")\n"; - // Save output values to storage - for (unsigned i = 0; i < varsVec.size(); i++) - m_output << "sstore(" << std::to_string(i*32) << ", " << varsVec[i] << ")\n"; - - // Add newly minted vars to current scope if (!varsVec.empty()) + { + // Save values returned by function so that they are reflected + // in the interpreter trace. + saveFunctionCallOutput(varsVec); + // Add newly minted vars to current scope addVarsToScope(varsVec); + } + else + yulAssert(_numOutParams == 0, "Proto fuzzer: Function return value not saved"); } void ProtoConverter::createFunctionDefAndCall( diff --git a/test/tools/ossfuzz/protoToYul.h b/test/tools/ossfuzz/protoToYul.h index 782e05ef9..4c3691a77 100644 --- a/test/tools/ossfuzz/protoToYul.h +++ b/test/tools/ossfuzz/protoToYul.h @@ -208,6 +208,21 @@ private: /// @param _numOutParams Number of output parameters in function signature void createFunctionCall(std::string _funcName, unsigned _numInParams, unsigned _numOutParams); + /// Print the yul syntax to pass input arguments to a function that has + /// @a _numInParams number of input parameters to the output stream. + /// The input arguments are pseudo-randomly chosen from calldata, memory, + /// storage, or the yul optimizer hex dictionary. + /// @param _numInParams Number of input arguments to fill + void fillFunctionCallInput(unsigned _numInParams); + + /// Print the yul syntax to save values returned by a function call + /// to the output stream. The values are either stored to memory or + /// storage based on a simulated coin flip. The saved location is + /// decided pseudo-randomly. + /// @param _varsVec A vector of strings that reference variables + /// holding the return values of a function call. + void saveFunctionCallOutput(std::vector const& _varsVec); + /// Register a function declaration /// @param _f Pointer to a FunctionDef object void registerFunction(FunctionDef const* _f); From ef407ea8967736c9d65963b02a36b01f22d3d7d3 Mon Sep 17 00:00:00 2001 From: Daniel Kirchner Date: Wed, 4 Sep 2019 10:57:01 +0200 Subject: [PATCH 16/20] Try to fix appveyor run by chaning bytecode pull output from stderr to stdout. --- scripts/bytecodecompare/storebytecode.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/bytecodecompare/storebytecode.bat b/scripts/bytecodecompare/storebytecode.bat index ef20a3207..996b0e72a 100644 --- a/scripts/bytecodecompare/storebytecode.bat +++ b/scripts/bytecodecompare/storebytecode.bat @@ -39,5 +39,5 @@ set REPORT=%DIRECTORY%/windows.txt cp ../report.txt %REPORT% git add %REPORT% git commit -a -m "Added report." -git pull --rebase +git pull --rebase 2>&1 git push origin 2>&1 From 8485a1abeccf5f4ce9311381b831e31e25a8aea5 Mon Sep 17 00:00:00 2001 From: chriseth Date: Tue, 3 Sep 2019 17:55:00 +0200 Subject: [PATCH 17/20] Change ABI sorting order. --- Changelog.md | 1 + libsolidity/interface/ABI.cpp | 18 +++-- test/libsolidity/ABIJson/events.sol | 46 ++++++------ test/libsolidity/ABIJson/inherited.sol | 70 +++++++++---------- test/libsolidity/ABIJson/pure_function.sol | 46 ++++++------ .../ABIJson/return_param_in_abi.sol | 26 +++---- test/libsolidity/ABIJson/view_function.sol | 46 ++++++------ 7 files changed, 130 insertions(+), 123 deletions(-) diff --git a/Changelog.md b/Changelog.md index bc7b5b8d0..90ebf4bad 100644 --- a/Changelog.md +++ b/Changelog.md @@ -5,6 +5,7 @@ Language Features: Compiler Features: + * ABI Output: Change sorting order of functions from selector to kind, name. * Optimizer: Add rule that replaces the BYTE opcode by 0 if the first argument is larger than 31. * Yul Optimizer: Take side-effect-freeness of user-defined functions into account. diff --git a/libsolidity/interface/ABI.cpp b/libsolidity/interface/ABI.cpp index 5001620f9..a21053d9f 100644 --- a/libsolidity/interface/ABI.cpp +++ b/libsolidity/interface/ABI.cpp @@ -40,7 +40,10 @@ bool anyDataStoredInStorage(TypePointers const& _pointers) Json::Value ABI::generate(ContractDefinition const& _contractDef) { - Json::Value abi(Json::arrayValue); + auto compare = [](Json::Value const& _a, Json::Value const& _b) -> bool { + return make_tuple(_a["type"], _a["name"]) < make_tuple(_b["type"], _b["name"]); + }; + multiset abi(compare); for (auto it: _contractDef.interfaceFunctions()) { @@ -71,7 +74,7 @@ Json::Value ABI::generate(ContractDefinition const& _contractDef) it.second->returnParameterTypes(), _contractDef.isLibrary() ); - abi.append(std::move(method)); + abi.emplace(std::move(method)); } if (_contractDef.constructor()) { @@ -88,7 +91,7 @@ Json::Value ABI::generate(ContractDefinition const& _contractDef) constrType.parameterTypes(), _contractDef.isLibrary() ); - abi.append(std::move(method)); + abi.emplace(std::move(method)); } if (_contractDef.fallbackFunction()) { @@ -98,7 +101,7 @@ Json::Value ABI::generate(ContractDefinition const& _contractDef) method["type"] = "fallback"; method["payable"] = externalFunctionType->isPayable(); method["stateMutability"] = stateMutabilityToString(externalFunctionType->stateMutability()); - abi.append(std::move(method)); + abi.emplace(std::move(method)); } for (auto const& it: _contractDef.interfaceEvents()) { @@ -117,10 +120,13 @@ Json::Value ABI::generate(ContractDefinition const& _contractDef) params.append(std::move(param)); } event["inputs"] = std::move(params); - abi.append(std::move(event)); + abi.emplace(std::move(event)); } - return abi; + Json::Value abiJson{Json::arrayValue}; + for (auto& f: abi) + abiJson.append(std::move(f)); + return abiJson; } Json::Value ABI::formatTypeList( diff --git a/test/libsolidity/ABIJson/events.sol b/test/libsolidity/ABIJson/events.sol index 287a0298f..1988e094b 100644 --- a/test/libsolidity/ABIJson/events.sol +++ b/test/libsolidity/ABIJson/events.sol @@ -9,29 +9,6 @@ contract test { // :test // [ // { -// "constant": false, -// "inputs": -// [ -// { -// "internalType": "uint256", -// "name": "a", -// "type": "uint256" -// } -// ], -// "name": "f", -// "outputs": -// [ -// { -// "internalType": "uint256", -// "name": "d", -// "type": "uint256" -// } -// ], -// "payable": false, -// "stateMutability": "nonpayable", -// "type": "function" -// }, -// { // "anonymous": false, // "inputs": // [ @@ -76,5 +53,28 @@ contract test { // "inputs": [], // "name": "e3", // "type": "event" +// }, +// { +// "constant": false, +// "inputs": +// [ +// { +// "internalType": "uint256", +// "name": "a", +// "type": "uint256" +// } +// ], +// "name": "f", +// "outputs": +// [ +// { +// "internalType": "uint256", +// "name": "d", +// "type": "uint256" +// } +// ], +// "payable": false, +// "stateMutability": "nonpayable", +// "type": "function" // } // ] diff --git a/test/libsolidity/ABIJson/inherited.sol b/test/libsolidity/ABIJson/inherited.sol index 5d61f4aef..00abd50ef 100644 --- a/test/libsolidity/ABIJson/inherited.sol +++ b/test/libsolidity/ABIJson/inherited.sol @@ -10,6 +10,20 @@ contract Derived is Base { // :Base // [ // { +// "anonymous": false, +// "inputs": +// [ +// { +// "indexed": true, +// "internalType": "bytes32", +// "name": "evtArgBase", +// "type": "bytes32" +// } +// ], +// "name": "baseEvent", +// "type": "event" +// }, +// { // "constant": false, // "inputs": // [ @@ -31,7 +45,12 @@ contract Derived is Base { // "payable": false, // "stateMutability": "nonpayable", // "type": "function" -// }, +// } +// ] +// +// +// :Derived +// [ // { // "anonymous": false, // "inputs": @@ -45,12 +64,21 @@ contract Derived is Base { // ], // "name": "baseEvent", // "type": "event" -// } -// ] -// -// -// :Derived -// [ +// }, +// { +// "anonymous": false, +// "inputs": +// [ +// { +// "indexed": true, +// "internalType": "uint256", +// "name": "evtArgDerived", +// "type": "uint256" +// } +// ], +// "name": "derivedEvent", +// "type": "event" +// }, // { // "constant": false, // "inputs": @@ -96,33 +124,5 @@ contract Derived is Base { // "payable": false, // "stateMutability": "nonpayable", // "type": "function" -// }, -// { -// "anonymous": false, -// "inputs": -// [ -// { -// "indexed": true, -// "internalType": "uint256", -// "name": "evtArgDerived", -// "type": "uint256" -// } -// ], -// "name": "derivedEvent", -// "type": "event" -// }, -// { -// "anonymous": false, -// "inputs": -// [ -// { -// "indexed": true, -// "internalType": "bytes32", -// "name": "evtArgBase", -// "type": "bytes32" -// } -// ], -// "name": "baseEvent", -// "type": "event" // } // ] diff --git a/test/libsolidity/ABIJson/pure_function.sol b/test/libsolidity/ABIJson/pure_function.sol index f1279ab3d..072b8f1f9 100644 --- a/test/libsolidity/ABIJson/pure_function.sol +++ b/test/libsolidity/ABIJson/pure_function.sol @@ -6,6 +6,29 @@ contract test { // :test // [ // { +// "constant": true, +// "inputs": +// [ +// { +// "internalType": "uint32", +// "name": "a", +// "type": "uint32" +// } +// ], +// "name": "boo", +// "outputs": +// [ +// { +// "internalType": "uint256", +// "name": "b", +// "type": "uint256" +// } +// ], +// "payable": false, +// "stateMutability": "pure", +// "type": "function" +// }, +// { // "constant": false, // "inputs": // [ @@ -32,28 +55,5 @@ contract test { // "payable": false, // "stateMutability": "nonpayable", // "type": "function" -// }, -// { -// "constant": true, -// "inputs": -// [ -// { -// "internalType": "uint32", -// "name": "a", -// "type": "uint32" -// } -// ], -// "name": "boo", -// "outputs": -// [ -// { -// "internalType": "uint256", -// "name": "b", -// "type": "uint256" -// } -// ], -// "payable": false, -// "stateMutability": "pure", -// "type": "function" // } // ] diff --git a/test/libsolidity/ABIJson/return_param_in_abi.sol b/test/libsolidity/ABIJson/return_param_in_abi.sol index 835a88a7b..2618e9ed2 100644 --- a/test/libsolidity/ABIJson/return_param_in_abi.sol +++ b/test/libsolidity/ABIJson/return_param_in_abi.sol @@ -11,6 +11,19 @@ contract test { // :test // [ // { +// "inputs": +// [ +// { +// "internalType": "enum test.ActionChoices", +// "name": "param", +// "type": "uint8" +// } +// ], +// "payable": false, +// "stateMutability": "nonpayable", +// "type": "constructor" +// }, +// { // "constant": false, // "inputs": [], // "name": "ret", @@ -25,18 +38,5 @@ contract test { // "payable": false, // "stateMutability": "nonpayable", // "type": "function" -// }, -// { -// "inputs": -// [ -// { -// "internalType": "enum test.ActionChoices", -// "name": "param", -// "type": "uint8" -// } -// ], -// "payable": false, -// "stateMutability": "nonpayable", -// "type": "constructor" // } // ] diff --git a/test/libsolidity/ABIJson/view_function.sol b/test/libsolidity/ABIJson/view_function.sol index 6112574d5..0c55b6e6e 100644 --- a/test/libsolidity/ABIJson/view_function.sol +++ b/test/libsolidity/ABIJson/view_function.sol @@ -6,6 +6,29 @@ contract test { // :test // [ // { +// "constant": true, +// "inputs": +// [ +// { +// "internalType": "uint32", +// "name": "a", +// "type": "uint32" +// } +// ], +// "name": "boo", +// "outputs": +// [ +// { +// "internalType": "uint256", +// "name": "b", +// "type": "uint256" +// } +// ], +// "payable": false, +// "stateMutability": "view", +// "type": "function" +// }, +// { // "constant": false, // "inputs": // [ @@ -32,28 +55,5 @@ contract test { // "payable": false, // "stateMutability": "nonpayable", // "type": "function" -// }, -// { -// "constant": true, -// "inputs": -// [ -// { -// "internalType": "uint32", -// "name": "a", -// "type": "uint32" -// } -// ], -// "name": "boo", -// "outputs": -// [ -// { -// "internalType": "uint256", -// "name": "b", -// "type": "uint256" -// } -// ], -// "payable": false, -// "stateMutability": "view", -// "type": "function" // } // ] From b3dd957afdfe539d725022f4afbfcfeb9fd095df Mon Sep 17 00:00:00 2001 From: Erik Kundt Date: Wed, 4 Sep 2019 12:56:29 +0200 Subject: [PATCH 18/20] Adds Gitter notifications for nightly builds and tests. --- .circleci/config.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 66b254d3b..64e3309f6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -176,6 +176,28 @@ defaults: requires: - b_ubu_ossfuzz + # -------------------------------------------------------------------------- + # Notification Templates + - gitter_notify_failure: &gitter_notify_failure + name: Gitter notify failure + command: >- + curl -X POST -i + -i -H "Content-Type: application/json" + -H "Accept: application/json" + -H "Authorization: Bearer $GITTER_API_TOKEN" "https://api.gitter.im/v1/rooms/$GITTER_NOTIFY_ROOM_ID/chatMessages" + -d '{"text":" ❌ Nightly job **'$CIRCLE_JOB'** failed. Please check '$CIRCLE_BUILD_URL' for details."}' + when: on_fail + + - gitter_notify_success: &gitter_notify_success + name: Gitter notify success + command: >- + curl -X POST -i + -i -H "Content-Type: application/json" + -H "Accept: application/json" + -H "Authorization: Bearer $GITTER_API_TOKEN" "https://api.gitter.im/v1/rooms/$GITTER_NOTIFY_ROOM_ID/chatMessages" + -d '{"text":" ✅ Nightly job **'$CIRCLE_JOB'** succeeded. Please check '$CIRCLE_BUILD_URL' for details."}' + when: on_success + # ----------------------------------------------------------------------------------------------- jobs: @@ -341,6 +363,8 @@ jobs: mkdir -p test_results export ASAN_OPTIONS="check_initialization_order=true:detect_stack_use_after_return=true:strict_init_order=true:strict_string_checks=true:detect_invalid_pointer_pairs=2" scripts/regressions.py -o test_results + - run: *gitter_notify_failure + - run: *gitter_notify_success - store_test_results: *store_test_results - store_artifacts: *artifacts_test_results @@ -526,6 +550,8 @@ jobs: name: External GnosisSafe tests command: | test/externalTests/gnosis.sh /tmp/workspace/soljson.js || test/externalTests/gnosis.sh /tmp/workspace/soljson.js + - run: *gitter_notify_failure + - run: *gitter_notify_success t_ems_external_zeppelin: docker: @@ -540,6 +566,8 @@ jobs: name: External Zeppelin tests command: | test/externalTests/zeppelin.sh /tmp/workspace/soljson.js || test/externalTests/zeppelin.sh /tmp/workspace/soljson.js + - run: *gitter_notify_failure + - run: *gitter_notify_success t_ems_external_colony: docker: @@ -558,6 +586,8 @@ jobs: name: External ColonyNetworks tests command: | test/externalTests/colony.sh /tmp/workspace/soljson.js || test/externalTests/colony.sh /tmp/workspace/soljson.js + - run: *gitter_notify_failure + - run: *gitter_notify_success workflows: version: 2 From 6b27ef8fcb0e7f08aec0411c0d661f79429fa445 Mon Sep 17 00:00:00 2001 From: Bhargava Shastry Date: Wed, 4 Sep 2019 13:24:47 +0200 Subject: [PATCH 19/20] Yul proto fuzzer: Additional blockchain opcodes --- test/tools/ossfuzz/protoToYul.cpp | 46 +++++++++++++++++++++++++++++++ test/tools/ossfuzz/protoToYul.h | 1 + test/tools/ossfuzz/yulProto.proto | 17 ++++++++++++ 3 files changed, 64 insertions(+) diff --git a/test/tools/ossfuzz/protoToYul.cpp b/test/tools/ossfuzz/protoToYul.cpp index 69cceda6a..0533cb4ed 100644 --- a/test/tools/ossfuzz/protoToYul.cpp +++ b/test/tools/ossfuzz/protoToYul.cpp @@ -337,6 +337,12 @@ void ProtoConverter::visit(UnaryOp const& _x) case UnaryOp::EXTCODEHASH: m_output << "extcodehash"; break; + case UnaryOp::BALANCE: + m_output << "balance"; + break; + case UnaryOp::BLOCKHASH: + m_output << "blockhash"; + break; } m_output << "("; visit(_x.operand()); @@ -385,6 +391,36 @@ void ProtoConverter::visit(NullaryOp const& _x) case NullaryOp::RETURNDATASIZE: m_output << "returndatasize()"; break; + case NullaryOp::ADDRESS: + m_output << "address()"; + break; + case NullaryOp::ORIGIN: + m_output << "origin()"; + break; + case NullaryOp::CALLER: + m_output << "caller()"; + break; + case NullaryOp::CALLVALUE: + m_output << "callvalue()"; + break; + case NullaryOp::GASPRICE: + m_output << "gasprice()"; + break; + case NullaryOp::COINBASE: + m_output << "coinbase()"; + break; + case NullaryOp::TIMESTAMP: + m_output << "timestamp()"; + break; + case NullaryOp::NUMBER: + m_output << "number()"; + break; + case NullaryOp::DIFFICULTY: + m_output << "difficulty()"; + break; + case NullaryOp::GASLIMIT: + m_output << "gaslimit()"; + break; } } @@ -947,6 +983,9 @@ void ProtoConverter::visit(Statement const& _x) if (!m_inForInitScope) visit(_x.funcdef()); break; + case Statement::kPop: + visit(_x.pop()); + break; case Statement::STMT_ONEOF_NOT_SET: break; } @@ -1233,6 +1272,13 @@ void ProtoConverter::visit(FunctionDef const& _x) createFunctionDefAndCall(_x, numInParams, numOutParams); } +void ProtoConverter::visit(PopStmt const& _x) +{ + m_output << "pop("; + visit(_x.expr()); + m_output << ")\n"; +} + void ProtoConverter::visit(Program const& _x) { // Initialize input size diff --git a/test/tools/ossfuzz/protoToYul.h b/test/tools/ossfuzz/protoToYul.h index 4c3691a77..b4fe052bb 100644 --- a/test/tools/ossfuzz/protoToYul.h +++ b/test/tools/ossfuzz/protoToYul.h @@ -85,6 +85,7 @@ private: void visit(TerminatingStmt const&); void visit(FunctionCall const&); void visit(FunctionDef const&); + void visit(PopStmt const&); void visit(Program const&); /// Creates a new scope, and adds @a _funcParams to it if it diff --git a/test/tools/ossfuzz/yulProto.proto b/test/tools/ossfuzz/yulProto.proto index 910903afa..10af9edc7 100644 --- a/test/tools/ossfuzz/yulProto.proto +++ b/test/tools/ossfuzz/yulProto.proto @@ -129,6 +129,8 @@ message UnaryOp { CALLDATALOAD = 4; EXTCODESIZE = 5; EXTCODEHASH = 6; + BALANCE = 7; + BLOCKHASH = 8; } required UOp op = 1; required Expression operand = 2; @@ -172,6 +174,16 @@ message NullaryOp { CALLDATASIZE = 4; CODESIZE = 5; RETURNDATASIZE = 6; + ADDRESS = 7; + ORIGIN = 8; + CALLER = 9; + CALLVALUE = 10; + GASPRICE = 11; + COINBASE = 12; + TIMESTAMP = 13; + NUMBER = 14; + DIFFICULTY = 15; + GASLIMIT = 16; } required NOp op = 1; } @@ -287,6 +299,10 @@ message FunctionDef { required Block block = 3; } +message PopStmt { + required Expression expr = 1; +} + message Statement { oneof stmt_oneof { VarDecl decl = 1; @@ -305,6 +321,7 @@ message Statement { FunctionCall functioncall = 14; BoundedForStmt boundedforstmt = 15; FunctionDef funcdef = 16; + PopStmt pop = 17; } } From c889e6e0156a63a2b36533e2990459eb3e8d6c56 Mon Sep 17 00:00:00 2001 From: Bhargava Shastry Date: Wed, 4 Sep 2019 16:05:26 +0200 Subject: [PATCH 20/20] Yul proto fuzzer: Add create and call opcodes --- test/tools/ossfuzz/protoToYul.cpp | 68 +++++++++++++++++++++++++++++++ test/tools/ossfuzz/protoToYul.h | 2 + test/tools/ossfuzz/yulProto.proto | 33 +++++++++++++++ 3 files changed, 103 insertions(+) diff --git a/test/tools/ossfuzz/protoToYul.cpp b/test/tools/ossfuzz/protoToYul.cpp index 0533cb4ed..3f0f26f4c 100644 --- a/test/tools/ossfuzz/protoToYul.cpp +++ b/test/tools/ossfuzz/protoToYul.cpp @@ -152,6 +152,12 @@ void ProtoConverter::visit(Expression const& _x) else m_output << dictionaryToken(); break; + case Expression::kLowcall: + visit(_x.lowcall()); + break; + case Expression::kCreate: + visit(_x.create()); + break; case Expression::EXPR_ONEOF_NOT_SET: m_output << dictionaryToken(); break; @@ -713,6 +719,68 @@ void ProtoConverter::visit(FunctionCall const& _x) } } +void ProtoConverter::visit(LowLevelCall const& _x) +{ + LowLevelCall_Type type = _x.callty(); + switch (type) + { + case LowLevelCall::CALL: + m_output << "call("; + break; + case LowLevelCall::CALLCODE: + m_output << "callcode("; + break; + case LowLevelCall::DELEGATECALL: + m_output << "delegatecall("; + break; + case LowLevelCall::STATICCALL: + m_output << "staticcall("; + break; + } + visit(_x.gas()); + m_output << ", "; + visit(_x.addr()); + m_output << ", "; + if (type == LowLevelCall::CALL || LowLevelCall::CALLCODE) + { + visit(_x.wei()); + m_output << ", "; + } + visit(_x.in()); + m_output << ", "; + visit(_x.insize()); + m_output << ", "; + visit(_x.out()); + m_output << ", "; + visit(_x.outsize()); + m_output << ")"; +} + +void ProtoConverter::visit(Create const& _x) +{ + Create_Type type = _x.createty(); + switch (type) + { + case Create::CREATE: + m_output << "create("; + break; + case Create::CREATE2: + m_output << "create2("; + break; + } + visit(_x.wei()); + m_output << ", "; + visit(_x.position()); + m_output << ", "; + visit(_x.size()); + if (type == Create::CREATE2) + { + m_output << ", "; + visit(_x.value()); + } + m_output << ")"; +} + void ProtoConverter::visit(IfStmt const& _x) { m_output << "if "; diff --git a/test/tools/ossfuzz/protoToYul.h b/test/tools/ossfuzz/protoToYul.h index b4fe052bb..f2530bcd5 100644 --- a/test/tools/ossfuzz/protoToYul.h +++ b/test/tools/ossfuzz/protoToYul.h @@ -86,6 +86,8 @@ private: void visit(FunctionCall const&); void visit(FunctionDef const&); void visit(PopStmt const&); + void visit(LowLevelCall const&); + void visit(Create const&); void visit(Program const&); /// Creates a new scope, and adds @a _funcParams to it if it diff --git a/test/tools/ossfuzz/yulProto.proto b/test/tools/ossfuzz/yulProto.proto index 10af9edc7..52fd632ec 100644 --- a/test/tools/ossfuzz/yulProto.proto +++ b/test/tools/ossfuzz/yulProto.proto @@ -21,6 +21,37 @@ message VarDecl { required Expression expr = 1; } +message LowLevelCall { + enum Type { + CALL = 0; + CALLCODE = 1; + DELEGATECALL = 2; + STATICCALL = 3; + } + required Type callty = 1; + required Expression gas = 2; + required Expression addr = 3; + // Valid for call and callcode only + required Expression wei = 4; + required Expression in = 5; + required Expression insize = 6; + required Expression out = 7; + required Expression outsize = 8; +} + +message Create { + enum Type { + CREATE = 0; + CREATE2 = 1; + } + required Type createty = 1; + required Expression wei = 2; + required Expression position = 3; + required Expression size = 4; + // Valid for create2 only + required Expression value = 5; +} + message FunctionCall { enum Returns { ZERO = 1; @@ -225,6 +256,8 @@ message Expression { TernaryOp top = 5; NullaryOp nop = 6; FunctionCall func_expr = 7; + LowLevelCall lowcall = 8; + Create create = 9; } }