From e7d204383de9efe5d0052715c744dd5bcb531b7d Mon Sep 17 00:00:00 2001 From: cameel Date: Fri, 17 Jan 2020 07:19:05 +0100 Subject: [PATCH 01/17] [yul-phaser] An empty command-line application for the new tool --- tools/CMakeLists.txt | 6 ++++++ tools/yulPhaser/main.cpp | 21 +++++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 tools/yulPhaser/main.cpp diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index bdcf615af..8a71a1750 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -12,3 +12,9 @@ target_link_libraries(solidity-upgrade PRIVATE solidity Boost::boost Boost::prog include(GNUInstallDirs) install(TARGETS solidity-upgrade DESTINATION "${CMAKE_INSTALL_BINDIR}") + +add_executable(yul-phaser + yulPhaser/main.cpp +) + +install(TARGETS yul-phaser DESTINATION "${CMAKE_INSTALL_BINDIR}") diff --git a/tools/yulPhaser/main.cpp b/tools/yulPhaser/main.cpp new file mode 100644 index 000000000..4c24c8240 --- /dev/null +++ b/tools/yulPhaser/main.cpp @@ -0,0 +1,21 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +int main([[maybe_unused]] int argc, [[maybe_unused]] char** argv) +{ + return 0; +} From b75370d93eceaeb862c9beb1fd2bdf2c5ae52a7e Mon Sep 17 00:00:00 2001 From: cameel Date: Fri, 17 Jan 2020 07:24:18 +0100 Subject: [PATCH 02/17] [yul-phaser] Printing help and accepting input file on the command line --- tools/CMakeLists.txt | 1 + tools/yulPhaser/main.cpp | 86 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 86 insertions(+), 1 deletion(-) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 8a71a1750..19764052a 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -16,5 +16,6 @@ install(TARGETS solidity-upgrade DESTINATION "${CMAKE_INSTALL_BINDIR}") add_executable(yul-phaser yulPhaser/main.cpp ) +target_link_libraries(yul-phaser PRIVATE Boost::program_options) install(TARGETS yul-phaser DESTINATION "${CMAKE_INSTALL_BINDIR}") diff --git a/tools/yulPhaser/main.cpp b/tools/yulPhaser/main.cpp index 4c24c8240..66943378d 100644 --- a/tools/yulPhaser/main.cpp +++ b/tools/yulPhaser/main.cpp @@ -15,7 +15,91 @@ along with solidity. If not, see . */ -int main([[maybe_unused]] int argc, [[maybe_unused]] char** argv) + +#include + +#include +#include + +using namespace std; +namespace po = boost::program_options; + +namespace { + +struct CommandLineParsingResult +{ + int exitCode; + po::variables_map arguments; +}; + +void runAlgorithm(string const& _sourcePath) +{ + cout << "Input: " << _sourcePath << endl; +} + +CommandLineParsingResult parseCommandLine(int argc, char** argv) +{ + po::options_description description( + "yul-phaser, a tool for finding the best sequence of Yul optimisation phases.\n" + "\n" + "Usage: yul-phaser [options] \n" + "Reads as Yul code and tries to find the best order in which to run optimisation" + " phases using a genetic algorithm.\n" + "Example:\n" + "yul-phaser program.yul\n" + "\n" + "Allowed options", + po::options_description::m_default_line_length, + po::options_description::m_default_line_length - 23 + ); + + description.add_options() + ("help", "Show help message and exit.") + ("input-file", po::value()->required(), "Input file") + ; + + po::positional_options_description positionalDescription; + po::variables_map arguments; + positionalDescription.add("input-file", 1); + po::notify(arguments); + + try + { + po::command_line_parser parser(argc, argv); + parser.options(description).positional(positionalDescription); + po::store(parser.run(), arguments); + } + catch (po::error const & _exception) + { + cerr << _exception.what() << endl; + return {1, move(arguments)}; + } + + if (arguments.count("help") > 0) + { + cout << description << endl; + return {2, move(arguments)}; + } + + if (arguments.count("input-file") == 0) + { + cerr << "Missing argument: input-file." << endl; + return {1, move(arguments)}; + } + + return {0, arguments}; +} + +} + +int main(int argc, char** argv) +{ + CommandLineParsingResult parsingResult = parseCommandLine(argc, argv); + if (parsingResult.exitCode != 0) + return parsingResult.exitCode; + + runAlgorithm(parsingResult.arguments["input-file"].as()); + return 0; } From 513d41c315d392471662e7aae1c39670f01ae0ac Mon Sep 17 00:00:00 2001 From: cameel Date: Fri, 17 Jan 2020 07:34:18 +0100 Subject: [PATCH 03/17] [yul-phaser] Add Program class --- tools/CMakeLists.txt | 4 +- tools/yulPhaser/Exceptions.h | 27 +++++++ tools/yulPhaser/Program.cpp | 140 +++++++++++++++++++++++++++++++++++ tools/yulPhaser/Program.h | 108 +++++++++++++++++++++++++++ tools/yulPhaser/main.cpp | 17 ++++- 5 files changed, 293 insertions(+), 3 deletions(-) create mode 100644 tools/yulPhaser/Exceptions.h create mode 100644 tools/yulPhaser/Program.cpp create mode 100644 tools/yulPhaser/Program.h diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 19764052a..acd49c59b 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -15,7 +15,9 @@ install(TARGETS solidity-upgrade DESTINATION "${CMAKE_INSTALL_BINDIR}") add_executable(yul-phaser yulPhaser/main.cpp + yulPhaser/Program.h + yulPhaser/Program.cpp ) -target_link_libraries(yul-phaser PRIVATE Boost::program_options) +target_link_libraries(yul-phaser PRIVATE solidity Boost::program_options) install(TARGETS yul-phaser DESTINATION "${CMAKE_INSTALL_BINDIR}") diff --git a/tools/yulPhaser/Exceptions.h b/tools/yulPhaser/Exceptions.h new file mode 100644 index 000000000..ae75d19ef --- /dev/null +++ b/tools/yulPhaser/Exceptions.h @@ -0,0 +1,27 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#pragma once + +#include + +namespace solidity::phaser +{ + +struct InvalidProgram: virtual util::Exception {}; + +} diff --git a/tools/yulPhaser/Program.cpp b/tools/yulPhaser/Program.cpp new file mode 100644 index 000000000..3a167755f --- /dev/null +++ b/tools/yulPhaser/Program.cpp @@ -0,0 +1,140 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#include + +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include + +#include +#include + +using namespace std; +using namespace solidity; +using namespace solidity::langutil; +using namespace solidity::yul; +using namespace solidity::util; +using namespace solidity::phaser; + +set const Program::s_externallyUsedIdentifiers = {}; + +Program Program::load(string const& _sourcePath) +{ + Dialect const& dialect = EVMDialect::strictAssemblyForEVMObjects(EVMVersion{}); + shared_ptr ast = parseSource(dialect, loadSource(_sourcePath)); + unique_ptr analysisInfo = analyzeAST(dialect, *ast); + + Program program( + dialect, + disambiguateAST(dialect, *ast, *analysisInfo) + ); + program.optimise({ + FunctionHoister::name, + FunctionGrouper::name, + ForLoopInitRewriter::name, + }); + + return program; +} + +void Program::optimise(vector const& _optimisationSteps) +{ + applyOptimisationSteps(m_optimiserStepContext, *m_ast, _optimisationSteps); +} + +CharStream Program::loadSource(string const& _sourcePath) +{ + assertThrow(boost::filesystem::exists(_sourcePath), InvalidProgram, "Source file does not exist"); + + string sourceCode = readFileAsString(_sourcePath); + return CharStream(sourceCode, _sourcePath); +} + +shared_ptr Program::parseSource(Dialect const& _dialect, CharStream _source) +{ + ErrorList errors; + ErrorReporter errorReporter(errors); + auto scanner = make_shared(move(_source)); + Parser parser(errorReporter, _dialect); + + shared_ptr ast = parser.parse(scanner, false); + assertThrow(ast != nullptr, InvalidProgram, "Error parsing source"); + assert(errorReporter.errors().empty()); + + return ast; +} + +unique_ptr Program::analyzeAST(Dialect const& _dialect, Block const& _ast) +{ + ErrorList errors; + ErrorReporter errorReporter(errors); + auto analysisInfo = make_unique(); + AsmAnalyzer analyzer(*analysisInfo, errorReporter, _dialect); + + bool analysisSuccessful = analyzer.analyze(_ast); + assertThrow(analysisSuccessful, InvalidProgram, "Error analyzing source"); + assert(errorReporter.errors().empty()); + + return analysisInfo; +} + +unique_ptr Program::disambiguateAST( + Dialect const& _dialect, + Block const& _ast, + AsmAnalysisInfo const& _analysisInfo +) +{ + Disambiguator disambiguator(_dialect, _analysisInfo, s_externallyUsedIdentifiers); + + return make_unique(get(disambiguator(_ast))); +} + +void Program::applyOptimisationSteps( + OptimiserStepContext& _context, + Block& _ast, + vector const& _optimisationSteps +) +{ + for (string const& step: _optimisationSteps) + OptimiserSuite::allSteps().at(step)->run(_context, _ast); +} + +size_t Program::computeCodeSize(Block const& _ast) +{ + return CodeSize::codeSizeIncludingFunctions(_ast); +} diff --git a/tools/yulPhaser/Program.h b/tools/yulPhaser/Program.h new file mode 100644 index 000000000..575dc29c4 --- /dev/null +++ b/tools/yulPhaser/Program.h @@ -0,0 +1,108 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#pragma once + +#include +#include + +#include +#include +#include +#include + +namespace solidity::langutil +{ + +class CharStream; + +} + +namespace solidity::yul +{ + +struct AsmAnalysisInfo; +struct Block; +struct Dialect; +class YulString; + +} + +namespace solidity::phaser +{ + +/** + * Class representing parsed and analysed Yul program that we can apply optimisations to. + * The program is already disambiguated and has several prerequisite optimiser steps applied to it + * so that the requirements of any possible step that could be later applied by the user are + * already satisfied. + * + * The class allows the user to apply extra optimisations and obtain metrics and general + * information about the resulting syntax tree. + */ +class Program +{ +public: + static Program load(std::string const& _sourcePath); + void optimise(std::vector const& _optimisationSteps); + + size_t codeSize() const { return computeCodeSize(*m_ast); } + yul::Block const& ast() const { return *m_ast; } + +private: + Program( + yul::Dialect const& _dialect, + std::shared_ptr _ast + ): + m_ast(_ast), + m_nameDispenser(_dialect, *_ast, s_externallyUsedIdentifiers), + m_optimiserStepContext{_dialect, m_nameDispenser, s_externallyUsedIdentifiers} + {} + + static langutil::CharStream loadSource(std::string const& _sourcePath); + static std::shared_ptr parseSource( + yul::Dialect const& _dialect, + langutil::CharStream _source + ); + static std::unique_ptr analyzeAST( + yul::Dialect const& _dialect, + yul::Block const& _ast + ); + static std::unique_ptr disambiguateAST( + yul::Dialect const& _dialect, + yul::Block const& _ast, + yul::AsmAnalysisInfo const& _analysisInfo + ); + static void applyOptimisationSteps( + yul::OptimiserStepContext& _context, + yul::Block& _ast, + std::vector const& _optimisationSteps + ); + static size_t computeCodeSize(yul::Block const& _ast); + + std::shared_ptr m_ast; + yul::NameDispenser m_nameDispenser; + yul::OptimiserStepContext m_optimiserStepContext; + + // Always empty set of reserved identifiers. It could be a constructor parameter but I don't + // think it would be useful in this tool. Other tools (like yulopti) have it empty too. + // We need it to live as long as m_optimiserStepContext because it stores a reference + // but since it's empty and read-only we can just have all objects share one static instance. + static std::set const s_externallyUsedIdentifiers; +}; + +} diff --git a/tools/yulPhaser/main.cpp b/tools/yulPhaser/main.cpp index 66943378d..6814acf6e 100644 --- a/tools/yulPhaser/main.cpp +++ b/tools/yulPhaser/main.cpp @@ -15,6 +15,8 @@ along with solidity. If not, see . */ +#include +#include #include @@ -22,6 +24,8 @@ #include using namespace std; +using namespace solidity::phaser; + namespace po = boost::program_options; namespace @@ -35,7 +39,8 @@ struct CommandLineParsingResult void runAlgorithm(string const& _sourcePath) { - cout << "Input: " << _sourcePath << endl; + Program::load(_sourcePath); + cout << "Program load successful." << endl; } CommandLineParsingResult parseCommandLine(int argc, char** argv) @@ -99,7 +104,15 @@ int main(int argc, char** argv) if (parsingResult.exitCode != 0) return parsingResult.exitCode; - runAlgorithm(parsingResult.arguments["input-file"].as()); + try + { + runAlgorithm(parsingResult.arguments["input-file"].as()); + } + catch (InvalidProgram const& _exception) + { + cerr << "ERROR: " << _exception.what() << endl; + return 1; + } return 0; } From 14d726ff013a7eeba6d15396b4e8a3c649d92a18 Mon Sep 17 00:00:00 2001 From: cameel Date: Wed, 22 Jan 2020 17:33:36 +0100 Subject: [PATCH 04/17] [yul-phaser] Program: Use unique_ptr rather than shared_ptr to hold a pointer to the AST root - The class never shares the instance so unique_ptr makes more sense. --- tools/yulPhaser/Program.cpp | 7 +++---- tools/yulPhaser/Program.h | 12 ++++++------ 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/tools/yulPhaser/Program.cpp b/tools/yulPhaser/Program.cpp index 3a167755f..36c7d9204 100644 --- a/tools/yulPhaser/Program.cpp +++ b/tools/yulPhaser/Program.cpp @@ -25,7 +25,6 @@ #include #include -#include #include #include #include @@ -56,7 +55,7 @@ set const Program::s_externallyUsedIdentifiers = {}; Program Program::load(string const& _sourcePath) { Dialect const& dialect = EVMDialect::strictAssemblyForEVMObjects(EVMVersion{}); - shared_ptr ast = parseSource(dialect, loadSource(_sourcePath)); + unique_ptr ast = parseSource(dialect, loadSource(_sourcePath)); unique_ptr analysisInfo = analyzeAST(dialect, *ast); Program program( @@ -85,14 +84,14 @@ CharStream Program::loadSource(string const& _sourcePath) return CharStream(sourceCode, _sourcePath); } -shared_ptr Program::parseSource(Dialect const& _dialect, CharStream _source) +unique_ptr Program::parseSource(Dialect const& _dialect, CharStream _source) { ErrorList errors; ErrorReporter errorReporter(errors); auto scanner = make_shared(move(_source)); Parser parser(errorReporter, _dialect); - shared_ptr ast = parser.parse(scanner, false); + unique_ptr ast = parser.parse(scanner, false); assertThrow(ast != nullptr, InvalidProgram, "Error parsing source"); assert(errorReporter.errors().empty()); diff --git a/tools/yulPhaser/Program.h b/tools/yulPhaser/Program.h index 575dc29c4..b4245fa9c 100644 --- a/tools/yulPhaser/Program.h +++ b/tools/yulPhaser/Program.h @@ -19,6 +19,7 @@ #include #include +#include #include #include @@ -36,7 +37,6 @@ namespace solidity::yul { struct AsmAnalysisInfo; -struct Block; struct Dialect; class YulString; @@ -66,15 +66,15 @@ public: private: Program( yul::Dialect const& _dialect, - std::shared_ptr _ast + std::unique_ptr _ast ): - m_ast(_ast), - m_nameDispenser(_dialect, *_ast, s_externallyUsedIdentifiers), + m_ast(std::move(_ast)), + m_nameDispenser(_dialect, *m_ast, s_externallyUsedIdentifiers), m_optimiserStepContext{_dialect, m_nameDispenser, s_externallyUsedIdentifiers} {} static langutil::CharStream loadSource(std::string const& _sourcePath); - static std::shared_ptr parseSource( + static std::unique_ptr parseSource( yul::Dialect const& _dialect, langutil::CharStream _source ); @@ -94,7 +94,7 @@ private: ); static size_t computeCodeSize(yul::Block const& _ast); - std::shared_ptr m_ast; + std::unique_ptr m_ast; yul::NameDispenser m_nameDispenser; yul::OptimiserStepContext m_optimiserStepContext; From 2aa42b32e5663acaa93fa50d131e0639e237f0d5 Mon Sep 17 00:00:00 2001 From: cameel Date: Wed, 22 Jan 2020 19:38:19 +0100 Subject: [PATCH 05/17] [yul-phaser] Make Program noncopyable - Copying worked but resulted in OptimiserStepContext having a reference to NameDispenser instance of the other object. --- tools/yulPhaser/Program.h | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tools/yulPhaser/Program.h b/tools/yulPhaser/Program.h index b4245fa9c..f039ddcc5 100644 --- a/tools/yulPhaser/Program.h +++ b/tools/yulPhaser/Program.h @@ -21,6 +21,8 @@ #include #include +#include + #include #include #include @@ -54,9 +56,17 @@ namespace solidity::phaser * The class allows the user to apply extra optimisations and obtain metrics and general * information about the resulting syntax tree. */ -class Program +class Program: private boost::noncopyable { public: + Program(Program&& program): + m_ast(std::move(program.m_ast)), + m_nameDispenser(std::move(program.m_nameDispenser)), + // Creating a new instance because a copied one would have a dangling reference to the old dispenser + m_optimiserStepContext{program.m_optimiserStepContext.dialect, m_nameDispenser, s_externallyUsedIdentifiers} + {} + Program operator=(Program&& program) = delete; + static Program load(std::string const& _sourcePath); void optimise(std::vector const& _optimisationSteps); From 3baa191b9463ee20d6977d1487728b2d21aee5df Mon Sep 17 00:00:00 2001 From: cameel Date: Thu, 23 Jan 2020 10:27:50 +0100 Subject: [PATCH 06/17] [yul-phaser] Printing and JSON conversion for the Program class --- tools/yulPhaser/Program.cpp | 22 +++++++++++++++++++++- tools/yulPhaser/Program.h | 4 ++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/tools/yulPhaser/Program.cpp b/tools/yulPhaser/Program.cpp index 36c7d9204..271c047aa 100644 --- a/tools/yulPhaser/Program.cpp +++ b/tools/yulPhaser/Program.cpp @@ -25,7 +25,9 @@ #include #include +#include #include +#include #include #include #include @@ -33,10 +35,10 @@ #include #include #include - #include #include +#include #include @@ -50,6 +52,13 @@ using namespace solidity::yul; using namespace solidity::util; using namespace solidity::phaser; +namespace solidity::phaser +{ + +ostream& operator<<(ostream& _stream, Program const& _program); + +} + set const Program::s_externallyUsedIdentifiers = {}; Program Program::load(string const& _sourcePath) @@ -76,6 +85,17 @@ void Program::optimise(vector const& _optimisationSteps) applyOptimisationSteps(m_optimiserStepContext, *m_ast, _optimisationSteps); } +ostream& phaser::operator<<(ostream& _stream, Program const& _program) +{ + return _stream << AsmPrinter()(*_program.m_ast); +} + +string Program::toJson() const +{ + Json::Value serializedAst = AsmJsonConverter(0)(*m_ast); + return jsonPrettyPrint(serializedAst); +} + CharStream Program::loadSource(string const& _sourcePath) { assertThrow(boost::filesystem::exists(_sourcePath), InvalidProgram, "Source file does not exist"); diff --git a/tools/yulPhaser/Program.h b/tools/yulPhaser/Program.h index f039ddcc5..3648d6e21 100644 --- a/tools/yulPhaser/Program.h +++ b/tools/yulPhaser/Program.h @@ -24,6 +24,7 @@ #include #include +#include #include #include #include @@ -73,6 +74,9 @@ public: size_t codeSize() const { return computeCodeSize(*m_ast); } yul::Block const& ast() const { return *m_ast; } + friend std::ostream& operator<<(std::ostream& _stream, Program const& _program); + std::string toJson() const; + private: Program( yul::Dialect const& _dialect, From 21a2b69f7442708a2c07253f307cd112b0a33d69 Mon Sep 17 00:00:00 2001 From: cameel Date: Fri, 24 Jan 2020 23:22:01 +0100 Subject: [PATCH 07/17] [yul-phaser] Create OptimiserStepContext on demand instead of storing it in Program class - This also lets us get rid of the static s_externallyUsedIdentifiers. --- tools/yulPhaser/Program.cpp | 18 ++++++++++++------ tools/yulPhaser/Program.h | 22 +++++++--------------- 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/tools/yulPhaser/Program.cpp b/tools/yulPhaser/Program.cpp index 271c047aa..459fca2e6 100644 --- a/tools/yulPhaser/Program.cpp +++ b/tools/yulPhaser/Program.cpp @@ -35,6 +35,7 @@ #include #include #include +#include #include #include @@ -59,8 +60,6 @@ ostream& operator<<(ostream& _stream, Program const& _program); } -set const Program::s_externallyUsedIdentifiers = {}; - Program Program::load(string const& _sourcePath) { Dialect const& dialect = EVMDialect::strictAssemblyForEVMObjects(EVMVersion{}); @@ -82,7 +81,7 @@ Program Program::load(string const& _sourcePath) void Program::optimise(vector const& _optimisationSteps) { - applyOptimisationSteps(m_optimiserStepContext, *m_ast, _optimisationSteps); + applyOptimisationSteps(m_dialect, m_nameDispenser, *m_ast, _optimisationSteps); } ostream& phaser::operator<<(ostream& _stream, Program const& _program) @@ -138,19 +137,26 @@ unique_ptr Program::disambiguateAST( AsmAnalysisInfo const& _analysisInfo ) { - Disambiguator disambiguator(_dialect, _analysisInfo, s_externallyUsedIdentifiers); + set const externallyUsedIdentifiers = {}; + Disambiguator disambiguator(_dialect, _analysisInfo, externallyUsedIdentifiers); return make_unique(get(disambiguator(_ast))); } void Program::applyOptimisationSteps( - OptimiserStepContext& _context, + Dialect const& _dialect, + NameDispenser& _nameDispenser, Block& _ast, vector const& _optimisationSteps ) { + // An empty set of reserved identifiers. It could be a constructor parameter but I don't + // think it would be useful in this tool. Other tools (like yulopti) have it empty too. + set const externallyUsedIdentifiers = {}; + OptimiserStepContext context{_dialect, _nameDispenser, externallyUsedIdentifiers}; + for (string const& step: _optimisationSteps) - OptimiserSuite::allSteps().at(step)->run(_context, _ast); + OptimiserSuite::allSteps().at(step)->run(context, _ast); } size_t Program::computeCodeSize(Block const& _ast) diff --git a/tools/yulPhaser/Program.h b/tools/yulPhaser/Program.h index 3648d6e21..bb382b374 100644 --- a/tools/yulPhaser/Program.h +++ b/tools/yulPhaser/Program.h @@ -17,7 +17,6 @@ #pragma once -#include #include #include @@ -41,7 +40,6 @@ namespace solidity::yul struct AsmAnalysisInfo; struct Dialect; -class YulString; } @@ -62,9 +60,8 @@ class Program: private boost::noncopyable public: Program(Program&& program): m_ast(std::move(program.m_ast)), - m_nameDispenser(std::move(program.m_nameDispenser)), - // Creating a new instance because a copied one would have a dangling reference to the old dispenser - m_optimiserStepContext{program.m_optimiserStepContext.dialect, m_nameDispenser, s_externallyUsedIdentifiers} + m_dialect{program.m_dialect}, + m_nameDispenser(std::move(program.m_nameDispenser)) {} Program operator=(Program&& program) = delete; @@ -83,8 +80,8 @@ private: std::unique_ptr _ast ): m_ast(std::move(_ast)), - m_nameDispenser(_dialect, *m_ast, s_externallyUsedIdentifiers), - m_optimiserStepContext{_dialect, m_nameDispenser, s_externallyUsedIdentifiers} + m_dialect{_dialect}, + m_nameDispenser(_dialect, *m_ast, {}) {} static langutil::CharStream loadSource(std::string const& _sourcePath); @@ -102,21 +99,16 @@ private: yul::AsmAnalysisInfo const& _analysisInfo ); static void applyOptimisationSteps( - yul::OptimiserStepContext& _context, + yul::Dialect const& _dialect, + yul::NameDispenser& _nameDispenser, yul::Block& _ast, std::vector const& _optimisationSteps ); static size_t computeCodeSize(yul::Block const& _ast); std::unique_ptr m_ast; + yul::Dialect const& m_dialect; yul::NameDispenser m_nameDispenser; - yul::OptimiserStepContext m_optimiserStepContext; - - // Always empty set of reserved identifiers. It could be a constructor parameter but I don't - // think it would be useful in this tool. Other tools (like yulopti) have it empty too. - // We need it to live as long as m_optimiserStepContext because it stores a reference - // but since it's empty and read-only we can just have all objects share one static instance. - static std::set const s_externallyUsedIdentifiers; }; } From 57ab8922cd387f7d86359b5d05d6f8c67fb645ff Mon Sep 17 00:00:00 2001 From: cameel Date: Fri, 17 Jan 2020 07:42:43 +0100 Subject: [PATCH 08/17] [yul-phaser] Add random number generators using Mersenne Twister from boost --- tools/CMakeLists.txt | 2 ++ tools/yulPhaser/Random.cpp | 44 ++++++++++++++++++++++++++++++++++++++ tools/yulPhaser/Random.h | 28 ++++++++++++++++++++++++ 3 files changed, 74 insertions(+) create mode 100644 tools/yulPhaser/Random.cpp create mode 100644 tools/yulPhaser/Random.h diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index acd49c59b..ecfece5d0 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -17,6 +17,8 @@ add_executable(yul-phaser yulPhaser/main.cpp yulPhaser/Program.h yulPhaser/Program.cpp + yulPhaser/Random.h + yulPhaser/Random.cpp ) target_link_libraries(yul-phaser PRIVATE solidity Boost::program_options) diff --git a/tools/yulPhaser/Random.cpp b/tools/yulPhaser/Random.cpp new file mode 100644 index 000000000..645afb395 --- /dev/null +++ b/tools/yulPhaser/Random.cpp @@ -0,0 +1,44 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#include + +#include +#include +#include + +#include + +using namespace solidity; + +uint32_t phaser::uniformRandomInt(uint32_t _min, uint32_t _max) +{ + // TODO: Seed must be configurable + static boost::random::mt19937 generator(time(0)); + boost::random::uniform_int_distribution<> distribution(_min, _max); + + return distribution(generator); +} + +uint32_t phaser::binomialRandomInt(uint32_t _numTrials, double _successProbability) +{ + // TODO: Seed must be configurable + static boost::random::mt19937 generator(time(0)); + boost::random::binomial_distribution<> distribution(_numTrials, _successProbability); + + return distribution(generator); +} diff --git a/tools/yulPhaser/Random.h b/tools/yulPhaser/Random.h new file mode 100644 index 000000000..25091ddef --- /dev/null +++ b/tools/yulPhaser/Random.h @@ -0,0 +1,28 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#pragma once + +#include + +namespace solidity::phaser +{ + +uint32_t uniformRandomInt(uint32_t _min, uint32_t _max); +uint32_t binomialRandomInt(uint32_t _numTrials, double _successProbability); + +} From f6d955db0bcf75abc6595e017229d21ea73fd640 Mon Sep 17 00:00:00 2001 From: cameel Date: Fri, 17 Jan 2020 07:41:03 +0100 Subject: [PATCH 09/17] [yul-phaser] Add Chromosome class --- tools/CMakeLists.txt | 2 + tools/yulPhaser/Chromosome.cpp | 112 +++++++++++++++++++++++++++++++++ tools/yulPhaser/Chromosome.h | 65 +++++++++++++++++++ tools/yulPhaser/main.cpp | 5 +- 4 files changed, 182 insertions(+), 2 deletions(-) create mode 100644 tools/yulPhaser/Chromosome.cpp create mode 100644 tools/yulPhaser/Chromosome.h diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index ecfece5d0..6b98f92c5 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -15,6 +15,8 @@ install(TARGETS solidity-upgrade DESTINATION "${CMAKE_INSTALL_BINDIR}") add_executable(yul-phaser yulPhaser/main.cpp + yulPhaser/Chromosome.h + yulPhaser/Chromosome.cpp yulPhaser/Program.h yulPhaser/Program.cpp yulPhaser/Random.h diff --git a/tools/yulPhaser/Chromosome.cpp b/tools/yulPhaser/Chromosome.cpp new file mode 100644 index 000000000..6e887bd11 --- /dev/null +++ b/tools/yulPhaser/Chromosome.cpp @@ -0,0 +1,112 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#include + +#include + +#include +#include + +#include + +using namespace std; +using namespace solidity; +using namespace solidity::util; +using namespace solidity::yul; +using namespace solidity::phaser; + +namespace solidity::phaser +{ + +ostream& operator<<(ostream& _stream, Chromosome const& _chromosome); + +} + +Chromosome Chromosome::makeRandom(size_t _length) +{ + vector steps; + for (size_t i = 0; i < _length; ++i) + steps.push_back(randomOptimisationStep()); + + return Chromosome(move(steps)); +} + +ostream& phaser::operator<<(ostream& _stream, Chromosome const& _chromosome) +{ + for (auto const& stepName: _chromosome.m_optimisationSteps) + _stream << OptimiserSuite::stepNameToAbbreviationMap().at(stepName); + + return _stream; +} + +vector Chromosome::allStepNames() +{ + vector stepNames; + for (auto const& step: OptimiserSuite::allSteps()) + stepNames.push_back(step.first); + + return stepNames; +} + +vector Chromosome::allStepNamesExcept(vector const& _excludedStepNames) +{ + // This is not very efficient but vectors are small and the caller will cache the results anyway. + // What matters a bit more is that using vector rather than a set gives us O(1) access to + // random elements in other functions. + return convertContainer>( + convertContainer>(allStepNames()) - + convertContainer>(_excludedStepNames) + ); +} + +string const& Chromosome::randomOptimisationStep() +{ + static vector stepNames = allStepNamesExcept({ + // All possible steps, listed and commented-out for easy tweaking. + // The uncommented ones are not used (possibly because they fail). + //{BlockFlattener::name}, + //{CommonSubexpressionEliminator::name}, + //{ConditionalSimplifier::name}, + //{ConditionalUnsimplifier::name}, + //{ControlFlowSimplifier::name}, + //{DeadCodeEliminator::name}, + //{EquivalentFunctionCombiner::name}, + //{ExpressionInliner::name}, + //{ExpressionJoiner::name}, + //{ExpressionSimplifier::name}, + //{ExpressionSplitter::name}, + //{ForLoopConditionIntoBody::name}, + //{ForLoopConditionOutOfBody::name}, + //{ForLoopInitRewriter::name}, + //{FullInliner::name}, + //{FunctionGrouper::name}, + //{FunctionHoister::name}, + //{LiteralRematerialiser::name}, + //{LoadResolver::name}, + //{LoopInvariantCodeMotion::name}, + //{RedundantAssignEliminator::name}, + //{Rematerialiser::name}, + //{SSAReverser::name}, + //{SSATransform::name}, + //{StructuralSimplifier::name}, + //{UnusedPruner::name}, + //{VarDeclInitializer::name}, + }); + + return stepNames[uniformRandomInt(0, stepNames.size() - 1)]; +} diff --git a/tools/yulPhaser/Chromosome.h b/tools/yulPhaser/Chromosome.h new file mode 100644 index 000000000..84b33d356 --- /dev/null +++ b/tools/yulPhaser/Chromosome.h @@ -0,0 +1,65 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#pragma once + +#include +#include +#include +#include + +namespace solidity::phaser +{ + +/** + * An object that represents a sequence of optimiser steps that can be applied to a program. + * Such sequences are used in our genetic algorithm to represent individual members of the + * population. + * + * To calculate the fitness of an individual one must apply its sequence to a specific program. + * This class does not provide any means to do so. It just stores information. + * + * Once created a sequence cannot be changed. The only way to mutate it is to generate a new + * chromosome based on the old one. + */ +class Chromosome +{ +public: + Chromosome() = default; + explicit Chromosome(std::vector _optimisationSteps): + m_optimisationSteps(std::move(_optimisationSteps)) {} + static Chromosome makeRandom(size_t _length); + + size_t length() const { return m_optimisationSteps.size(); } + std::vector const& optimisationSteps() const { return m_optimisationSteps; } + + friend std::ostream& operator<<(std::ostream& _stream, Chromosome const& _chromosome); + + bool operator==(Chromosome const& _other) const { return m_optimisationSteps == _other.m_optimisationSteps; } + bool operator!=(Chromosome const& _other) const { return !(*this == _other); } + +private: + static std::vector allStepNames(); + static std::vector allStepNamesExcept( + std::vector const& _excludedStepNames + ); + static std::string const& randomOptimisationStep(); + + std::vector m_optimisationSteps; +}; + +} diff --git a/tools/yulPhaser/main.cpp b/tools/yulPhaser/main.cpp index 6814acf6e..8bf32b135 100644 --- a/tools/yulPhaser/main.cpp +++ b/tools/yulPhaser/main.cpp @@ -15,6 +15,7 @@ along with solidity. If not, see . */ +#include #include #include @@ -39,8 +40,8 @@ struct CommandLineParsingResult void runAlgorithm(string const& _sourcePath) { - Program::load(_sourcePath); - cout << "Program load successful." << endl; + Program::load(_sourcePath).optimize(Chromosome::makeRandom(15).asSequence()); + cout << "Program load and optimization successful." << endl; } CommandLineParsingResult parseCommandLine(int argc, char** argv) From f0fb046038c5d4d45d544181ebf03a196424fe28 Mon Sep 17 00:00:00 2001 From: cameel Date: Wed, 29 Jan 2020 20:00:16 +0100 Subject: [PATCH 10/17] [yul-phaser] Remove Chromosome::allStepNamesExcept() --- tools/yulPhaser/Chromosome.cpp | 44 +--------------------------------- tools/yulPhaser/Chromosome.h | 3 --- 2 files changed, 1 insertion(+), 46 deletions(-) diff --git a/tools/yulPhaser/Chromosome.cpp b/tools/yulPhaser/Chromosome.cpp index 6e887bd11..b332266ba 100644 --- a/tools/yulPhaser/Chromosome.cpp +++ b/tools/yulPhaser/Chromosome.cpp @@ -26,7 +26,6 @@ using namespace std; using namespace solidity; -using namespace solidity::util; using namespace solidity::yul; using namespace solidity::phaser; @@ -63,50 +62,9 @@ vector Chromosome::allStepNames() return stepNames; } -vector Chromosome::allStepNamesExcept(vector const& _excludedStepNames) -{ - // This is not very efficient but vectors are small and the caller will cache the results anyway. - // What matters a bit more is that using vector rather than a set gives us O(1) access to - // random elements in other functions. - return convertContainer>( - convertContainer>(allStepNames()) - - convertContainer>(_excludedStepNames) - ); -} - string const& Chromosome::randomOptimisationStep() { - static vector stepNames = allStepNamesExcept({ - // All possible steps, listed and commented-out for easy tweaking. - // The uncommented ones are not used (possibly because they fail). - //{BlockFlattener::name}, - //{CommonSubexpressionEliminator::name}, - //{ConditionalSimplifier::name}, - //{ConditionalUnsimplifier::name}, - //{ControlFlowSimplifier::name}, - //{DeadCodeEliminator::name}, - //{EquivalentFunctionCombiner::name}, - //{ExpressionInliner::name}, - //{ExpressionJoiner::name}, - //{ExpressionSimplifier::name}, - //{ExpressionSplitter::name}, - //{ForLoopConditionIntoBody::name}, - //{ForLoopConditionOutOfBody::name}, - //{ForLoopInitRewriter::name}, - //{FullInliner::name}, - //{FunctionGrouper::name}, - //{FunctionHoister::name}, - //{LiteralRematerialiser::name}, - //{LoadResolver::name}, - //{LoopInvariantCodeMotion::name}, - //{RedundantAssignEliminator::name}, - //{Rematerialiser::name}, - //{SSAReverser::name}, - //{SSATransform::name}, - //{StructuralSimplifier::name}, - //{UnusedPruner::name}, - //{VarDeclInitializer::name}, - }); + static vector stepNames = allStepNames(); return stepNames[uniformRandomInt(0, stepNames.size() - 1)]; } diff --git a/tools/yulPhaser/Chromosome.h b/tools/yulPhaser/Chromosome.h index 84b33d356..6db99dc4e 100644 --- a/tools/yulPhaser/Chromosome.h +++ b/tools/yulPhaser/Chromosome.h @@ -54,9 +54,6 @@ public: private: static std::vector allStepNames(); - static std::vector allStepNamesExcept( - std::vector const& _excludedStepNames - ); static std::string const& randomOptimisationStep(); std::vector m_optimisationSteps; From 7b7c88ae95c85611ffcf5e23973d66b4bceaaf01 Mon Sep 17 00:00:00 2001 From: cameel Date: Fri, 17 Jan 2020 07:45:10 +0100 Subject: [PATCH 11/17] [yul-phaser] Add Population class --- tools/CMakeLists.txt | 2 + tools/yulPhaser/Population.cpp | 137 +++++++++++++++++++++++++++++++++ tools/yulPhaser/Population.h | 87 +++++++++++++++++++++ tools/yulPhaser/main.cpp | 7 +- 4 files changed, 229 insertions(+), 4 deletions(-) create mode 100644 tools/yulPhaser/Population.cpp create mode 100644 tools/yulPhaser/Population.h diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 6b98f92c5..a2aa807af 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -15,6 +15,8 @@ install(TARGETS solidity-upgrade DESTINATION "${CMAKE_INSTALL_BINDIR}") add_executable(yul-phaser yulPhaser/main.cpp + yulPhaser/Population.h + yulPhaser/Population.cpp yulPhaser/Chromosome.h yulPhaser/Chromosome.cpp yulPhaser/Program.h diff --git a/tools/yulPhaser/Population.cpp b/tools/yulPhaser/Population.cpp new file mode 100644 index 000000000..a5a65fa7f --- /dev/null +++ b/tools/yulPhaser/Population.cpp @@ -0,0 +1,137 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#include + +#include + +#include +#include +#include +#include + +using namespace std; +using namespace solidity; +using namespace solidity::phaser; + +namespace solidity::phaser +{ + +ostream& operator<<(ostream& _stream, Individual const& _individual); +ostream& operator<<(ostream& _stream, Population const& _population); + +} + +ostream& phaser::operator<<(ostream& _stream, Individual const& _individual) +{ + _stream << "Fitness: "; + if (_individual.fitness.has_value()) + _stream << _individual.fitness.value(); + else + _stream << ""; + _stream << ", optimisations: " << _individual.chromosome; + + return _stream; +} + +Population::Population(string const& _sourcePath, vector const& _chromosomes): + m_sourcePath{_sourcePath} +{ + for (auto const& chromosome: _chromosomes) + m_individuals.push_back({chromosome}); +} + +Population Population::makeRandom(string const& _sourcePath, size_t _size) +{ + vector individuals; + for (size_t i = 0; i < _size; ++i) + individuals.push_back({Chromosome::makeRandom(randomChromosomeLength())}); + + return Population(_sourcePath, individuals); +} + +size_t Population::measureFitness(Chromosome const& _chromosome, string const& _sourcePath) +{ + auto program = Program::load(_sourcePath); + program.optimise(_chromosome.optimisationSteps()); + return program.codeSize(); +} + +void Population::run(optional _numRounds, ostream& _outputStream) +{ + doEvaluation(); + for (size_t round = 0; !_numRounds.has_value() || round < _numRounds.value(); ++round) + { + doMutation(); + doSelection(); + doEvaluation(); + + _outputStream << "---------- ROUND " << round << " ----------" << endl; + _outputStream << *this; + } +} + +ostream& phaser::operator<<(ostream& _stream, Population const& _population) +{ + _stream << "Source: " << _population.m_sourcePath << endl; + + auto individual = _population.m_individuals.begin(); + for (; individual != _population.m_individuals.end(); ++individual) + _stream << *individual << endl; + + return _stream; +} + +void Population::doMutation() +{ + // TODO: Implement mutation and crossover +} + +void Population::doEvaluation() +{ + for (auto& individual: m_individuals) + if (!individual.fitness.has_value()) + individual.fitness = measureFitness(individual.chromosome, m_sourcePath); +} + +void Population::doSelection() +{ + assert(all_of(m_individuals.begin(), m_individuals.end(), [](auto& i){ return i.fitness.has_value(); })); + + sort( + m_individuals.begin(), + m_individuals.end(), + [](auto const& a, auto const& b){ return a.fitness.value() < b.fitness.value(); } + ); + + randomizeWorstChromosomes(m_individuals, m_individuals.size() / 2); +} + +void Population::randomizeWorstChromosomes( + vector& _individuals, + size_t _count +) +{ + assert(_individuals.size() >= _count); + // ASSUMPTION: _individuals is sorted in ascending order + + auto individual = _individuals.begin() + (_individuals.size() - _count); + for (; individual != _individuals.end(); ++individual) + { + *individual = {Chromosome::makeRandom(randomChromosomeLength())}; + } +} diff --git a/tools/yulPhaser/Population.h b/tools/yulPhaser/Population.h new file mode 100644 index 000000000..3c5596b03 --- /dev/null +++ b/tools/yulPhaser/Population.h @@ -0,0 +1,87 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#pragma once + +#include +#include + +#include +#include +#include + +namespace solidity::phaser +{ + +/** + * Information describing the state of an individual member of the population during the course + * of the genetic algorithm. + */ +struct Individual +{ + Chromosome chromosome; + std::optional fitness = std::nullopt; + + friend std::ostream& operator<<(std::ostream& _stream, Individual const& _individual); +}; + +/** + * Represents a changing set of individuals undergoing a genetic algorithm. + * Each round of the algorithm involves mutating existing individuals, evaluating their fitness + * and selecting the best ones for the next round. + * + * An individual is a sequence of optimiser steps represented by a @a Chromosome instance. The whole + * population is associated with a fixed Yul program. By loading the source code into a @a Program + * instance the class can compute fitness of the individual. + */ +class Population +{ +public: + static constexpr size_t MaxChromosomeLength = 30; + + explicit Population(std::string const& _sourcePath, std::vector const& _chromosomes = {}); + static Population makeRandom(std::string const& _sourcePath, size_t _size); + + void run(std::optional _numRounds, std::ostream& _outputStream); + + std::vector const& individuals() const { return m_individuals; } + + static size_t randomChromosomeLength() { return binomialRandomInt(MaxChromosomeLength, 0.5); } + static size_t measureFitness(Chromosome const& _chromosome, std::string const& _sourcePath); + + friend std::ostream& operator<<(std::ostream& _stream, Population const& _population); + +private: + explicit Population(std::string const& _sourcePath, std::vector _individuals = {}): + m_sourcePath{_sourcePath}, + m_individuals{std::move(_individuals)} {} + + void doMutation(); + void doEvaluation(); + void doSelection(); + + static void randomizeWorstChromosomes( + std::vector& _individuals, + size_t _count + ); + + std::string m_sourcePath; + + std::vector m_individuals; +}; + +} diff --git a/tools/yulPhaser/main.cpp b/tools/yulPhaser/main.cpp index 8bf32b135..280262d7c 100644 --- a/tools/yulPhaser/main.cpp +++ b/tools/yulPhaser/main.cpp @@ -15,9 +15,8 @@ along with solidity. If not, see . */ -#include #include -#include +#include #include @@ -40,8 +39,8 @@ struct CommandLineParsingResult void runAlgorithm(string const& _sourcePath) { - Program::load(_sourcePath).optimize(Chromosome::makeRandom(15).asSequence()); - cout << "Program load and optimization successful." << endl; + auto population = Population::makeRandom(_sourcePath, 10); + population.run(nullopt, cout); } CommandLineParsingResult parseCommandLine(int argc, char** argv) From 785f65d0f511b8a6bda589b2a77c45e6b73dc63b Mon Sep 17 00:00:00 2001 From: cameel Date: Wed, 22 Jan 2020 20:14:36 +0100 Subject: [PATCH 12/17] [yul-phaser] Make Program and Population classes accept source code rather than file path - I need some sample .yul files for testing but I see that existing tests generally have source code hard-coded in them rather than in standalone .yul files. There are lots of .yul files but they seem to be automatically processed by a special test case rather loaded ad-hoc by manually created tests. - Program and Population required a file name until now. I'm making them accept loaded source code to be able to give them data hard-coded in a test. --- tools/yulPhaser/Population.cpp | 17 +++++++++-------- tools/yulPhaser/Population.h | 14 ++++++++------ tools/yulPhaser/Program.cpp | 16 +++------------- tools/yulPhaser/Program.h | 3 +-- tools/yulPhaser/main.cpp | 17 ++++++++++++++++- 5 files changed, 37 insertions(+), 30 deletions(-) diff --git a/tools/yulPhaser/Population.cpp b/tools/yulPhaser/Population.cpp index a5a65fa7f..ca6fe7891 100644 --- a/tools/yulPhaser/Population.cpp +++ b/tools/yulPhaser/Population.cpp @@ -26,6 +26,7 @@ using namespace std; using namespace solidity; +using namespace solidity::langutil; using namespace solidity::phaser; namespace solidity::phaser @@ -48,25 +49,25 @@ ostream& phaser::operator<<(ostream& _stream, Individual const& _individual) return _stream; } -Population::Population(string const& _sourcePath, vector const& _chromosomes): - m_sourcePath{_sourcePath} +Population::Population(CharStream _sourceCode, vector const& _chromosomes): + m_sourceCode{move(_sourceCode)} { for (auto const& chromosome: _chromosomes) m_individuals.push_back({chromosome}); } -Population Population::makeRandom(string const& _sourcePath, size_t _size) +Population Population::makeRandom(CharStream _sourceCode, size_t _size) { vector individuals; for (size_t i = 0; i < _size; ++i) individuals.push_back({Chromosome::makeRandom(randomChromosomeLength())}); - return Population(_sourcePath, individuals); + return Population(move(_sourceCode), individuals); } -size_t Population::measureFitness(Chromosome const& _chromosome, string const& _sourcePath) +size_t Population::measureFitness(Chromosome const& _chromosome, CharStream& _sourceCode) { - auto program = Program::load(_sourcePath); + auto program = Program::load(_sourceCode); program.optimise(_chromosome.optimisationSteps()); return program.codeSize(); } @@ -87,7 +88,7 @@ void Population::run(optional _numRounds, ostream& _outputStream) ostream& phaser::operator<<(ostream& _stream, Population const& _population) { - _stream << "Source: " << _population.m_sourcePath << endl; + _stream << "Stream name: " << _population.m_sourceCode.name() << endl; auto individual = _population.m_individuals.begin(); for (; individual != _population.m_individuals.end(); ++individual) @@ -105,7 +106,7 @@ void Population::doEvaluation() { for (auto& individual: m_individuals) if (!individual.fitness.has_value()) - individual.fitness = measureFitness(individual.chromosome, m_sourcePath); + individual.fitness = measureFitness(individual.chromosome, m_sourceCode); } void Population::doSelection() diff --git a/tools/yulPhaser/Population.h b/tools/yulPhaser/Population.h index 3c5596b03..6e30b6193 100644 --- a/tools/yulPhaser/Population.h +++ b/tools/yulPhaser/Population.h @@ -20,6 +20,8 @@ #include #include +#include + #include #include #include @@ -53,21 +55,21 @@ class Population public: static constexpr size_t MaxChromosomeLength = 30; - explicit Population(std::string const& _sourcePath, std::vector const& _chromosomes = {}); - static Population makeRandom(std::string const& _sourcePath, size_t _size); + explicit Population(langutil::CharStream _sourceCode, std::vector const& _chromosomes = {}); + static Population makeRandom(langutil::CharStream _sourceCode, size_t _size); void run(std::optional _numRounds, std::ostream& _outputStream); std::vector const& individuals() const { return m_individuals; } static size_t randomChromosomeLength() { return binomialRandomInt(MaxChromosomeLength, 0.5); } - static size_t measureFitness(Chromosome const& _chromosome, std::string const& _sourcePath); + static size_t measureFitness(Chromosome const& _chromosome, langutil::CharStream& _sourceCode); friend std::ostream& operator<<(std::ostream& _stream, Population const& _population); private: - explicit Population(std::string const& _sourcePath, std::vector _individuals = {}): - m_sourcePath{_sourcePath}, + explicit Population(langutil::CharStream _sourceCode, std::vector _individuals = {}): + m_sourceCode{std::move(_sourceCode)}, m_individuals{std::move(_individuals)} {} void doMutation(); @@ -79,7 +81,7 @@ private: size_t _count ); - std::string m_sourcePath; + langutil::CharStream m_sourceCode; std::vector m_individuals; }; diff --git a/tools/yulPhaser/Program.cpp b/tools/yulPhaser/Program.cpp index 459fca2e6..6137ac244 100644 --- a/tools/yulPhaser/Program.cpp +++ b/tools/yulPhaser/Program.cpp @@ -38,11 +38,8 @@ #include #include -#include #include -#include - #include #include @@ -60,10 +57,11 @@ ostream& operator<<(ostream& _stream, Program const& _program); } -Program Program::load(string const& _sourcePath) +Program Program::load(CharStream& _sourceCode) { + // ASSUMPTION: parseSource() rewinds the stream on its own Dialect const& dialect = EVMDialect::strictAssemblyForEVMObjects(EVMVersion{}); - unique_ptr ast = parseSource(dialect, loadSource(_sourcePath)); + unique_ptr ast = parseSource(dialect, _sourceCode); unique_ptr analysisInfo = analyzeAST(dialect, *ast); Program program( @@ -95,14 +93,6 @@ string Program::toJson() const return jsonPrettyPrint(serializedAst); } -CharStream Program::loadSource(string const& _sourcePath) -{ - assertThrow(boost::filesystem::exists(_sourcePath), InvalidProgram, "Source file does not exist"); - - string sourceCode = readFileAsString(_sourcePath); - return CharStream(sourceCode, _sourcePath); -} - unique_ptr Program::parseSource(Dialect const& _dialect, CharStream _source) { ErrorList errors; diff --git a/tools/yulPhaser/Program.h b/tools/yulPhaser/Program.h index bb382b374..822503f26 100644 --- a/tools/yulPhaser/Program.h +++ b/tools/yulPhaser/Program.h @@ -65,7 +65,7 @@ public: {} Program operator=(Program&& program) = delete; - static Program load(std::string const& _sourcePath); + static Program load(langutil::CharStream& _sourceCode); void optimise(std::vector const& _optimisationSteps); size_t codeSize() const { return computeCodeSize(*m_ast); } @@ -84,7 +84,6 @@ private: m_nameDispenser(_dialect, *m_ast, {}) {} - static langutil::CharStream loadSource(std::string const& _sourcePath); static std::unique_ptr parseSource( yul::Dialect const& _dialect, langutil::CharStream _source diff --git a/tools/yulPhaser/main.cpp b/tools/yulPhaser/main.cpp index 280262d7c..6c72bd0f0 100644 --- a/tools/yulPhaser/main.cpp +++ b/tools/yulPhaser/main.cpp @@ -18,13 +18,20 @@ #include #include +#include +#include +#include + +#include #include #include #include using namespace std; +using namespace solidity::langutil; using namespace solidity::phaser; +using namespace solidity::util; namespace po = boost::program_options; @@ -37,9 +44,17 @@ struct CommandLineParsingResult po::variables_map arguments; }; +CharStream loadSource(string const& _sourcePath) +{ + assertThrow(boost::filesystem::exists(_sourcePath), InvalidProgram, "Source file does not exist"); + + string sourceCode = readFileAsString(_sourcePath); + return CharStream(sourceCode, _sourcePath); +} + void runAlgorithm(string const& _sourcePath) { - auto population = Population::makeRandom(_sourcePath, 10); + auto population = Population::makeRandom(loadSource(_sourcePath), 10); population.run(nullopt, cout); } From f8e397b487cd74e51f355ff6614d756f61a49c27 Mon Sep 17 00:00:00 2001 From: cameel Date: Fri, 17 Jan 2020 07:51:45 +0100 Subject: [PATCH 13/17] [yul-phaser] Create test suite for Chromosome --- test/CMakeLists.txt | 6 ++++++ test/yulPhaser/Chromosome.cpp | 26 ++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 test/yulPhaser/Chromosome.cpp diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index e7a9d903f..938d14bcb 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -138,6 +138,11 @@ set(libyul_sources ) detect_stray_source_files("${libyul_sources}" "libyul/") +set(yul_phaser_sources + yulPhaser/Chromosome.cpp +) +detect_stray_source_files("${yul_phaser_sources}" "yulPhaser/") + add_executable(soltest ${sources} ${contracts_sources} ${libsolutil_sources} @@ -146,6 +151,7 @@ add_executable(soltest ${sources} ${libyul_sources} ${libsolidity_sources} ${libsolidity_util_sources} + ${yul_phaser_sources} ) target_link_libraries(soltest PRIVATE libsolc yul solidity yulInterpreter evmasm solutil Boost::boost Boost::program_options Boost::unit_test_framework evmc) diff --git a/test/yulPhaser/Chromosome.cpp b/test/yulPhaser/Chromosome.cpp new file mode 100644 index 000000000..fef68df73 --- /dev/null +++ b/test/yulPhaser/Chromosome.cpp @@ -0,0 +1,26 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#include + +namespace solidity::phaser::test +{ + +BOOST_AUTO_TEST_SUITE(ChromosomeTest) +BOOST_AUTO_TEST_SUITE_END() + +} From bee62cdd9e2241159debd0f2a1aece7f77ab88dc Mon Sep 17 00:00:00 2001 From: cameel Date: Fri, 17 Jan 2020 07:53:21 +0100 Subject: [PATCH 14/17] [yul-phaser] Tests for Chromosome class --- test/CMakeLists.txt | 6 +++ test/yulPhaser/Chromosome.cpp | 75 +++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 938d14bcb..6bd4f3952 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -140,6 +140,12 @@ detect_stray_source_files("${libyul_sources}" "libyul/") set(yul_phaser_sources yulPhaser/Chromosome.cpp + + # FIXME: yul-phaser is not a library so I can't just add it to target_link_libraries(). + # My current workaround is just to include its source files here but this introduces + # unnecessary duplication. Create a library or find a way to reuse the list in both places. + ../tools/yulPhaser/Chromosome.cpp + ../tools/yulPhaser/Random.cpp ) detect_stray_source_files("${yul_phaser_sources}" "yulPhaser/") diff --git a/test/yulPhaser/Chromosome.cpp b/test/yulPhaser/Chromosome.cpp index fef68df73..dd250d0ec 100644 --- a/test/yulPhaser/Chromosome.cpp +++ b/test/yulPhaser/Chromosome.cpp @@ -15,12 +15,87 @@ along with solidity. If not, see . */ +#include + +#include +#include +#include +#include + +#include + #include +using namespace std; +using namespace solidity::yul; +using namespace solidity::util; + namespace solidity::phaser::test { +BOOST_AUTO_TEST_SUITE(Phaser) BOOST_AUTO_TEST_SUITE(ChromosomeTest) + +BOOST_AUTO_TEST_CASE(makeRandom_should_create_chromosome_with_random_optimisation_steps) +{ + constexpr uint32_t numSteps = 1000; + + auto chromosome1 = Chromosome::makeRandom(numSteps); + auto chromosome2 = Chromosome::makeRandom(numSteps); + BOOST_CHECK_EQUAL(chromosome1.length(), numSteps); + BOOST_CHECK_EQUAL(chromosome2.length(), numSteps); + + multiset steps1; + multiset steps2; + for (auto const& step: chromosome1.optimisationSteps()) + steps1.insert(step); + for (auto const& step: chromosome2.optimisationSteps()) + steps2.insert(step); + + // Check if steps are different and also if they're not just a permutation of the same set. + // Technically they could be the same and still random but the probability is infinitesimally low. + BOOST_TEST(steps1 != steps2); +} + +BOOST_AUTO_TEST_CASE(constructor_should_store_optimisation_steps) +{ + vector steps = { + StructuralSimplifier::name, + BlockFlattener::name, + UnusedPruner::name, + }; + Chromosome chromosome(steps); + + BOOST_TEST(steps == chromosome.optimisationSteps()); +} + +BOOST_AUTO_TEST_CASE(constructor_should_allow_duplicate_steps) +{ + vector steps = { + StructuralSimplifier::name, + StructuralSimplifier::name, + BlockFlattener::name, + UnusedPruner::name, + BlockFlattener::name, + }; + Chromosome chromosome(steps); + + BOOST_TEST(steps == chromosome.optimisationSteps()); +} + +BOOST_AUTO_TEST_CASE(output_operator_should_create_concise_and_unambiguous_string_representation) +{ + vector allSteps; + for (auto const& step: OptimiserSuite::allSteps()) + allSteps.push_back(step.first); + Chromosome chromosome(allSteps); + + BOOST_TEST(chromosome.length() == allSteps.size()); + BOOST_TEST(chromosome.optimisationSteps() == allSteps); + BOOST_TEST(toString(chromosome) == "fcCUnDvejsxIOoighTLMrmVatud"); +} + +BOOST_AUTO_TEST_SUITE_END() BOOST_AUTO_TEST_SUITE_END() } From 24d63a93cf3b6d006d78f751dbd830979f479f51 Mon Sep 17 00:00:00 2001 From: cameel Date: Thu, 23 Jan 2020 19:02:59 +0100 Subject: [PATCH 15/17] [yul-phaser] Tests for Program class --- test/CMakeLists.txt | 2 + test/yulPhaser/Program.cpp | 266 +++++++++++++++++++++++++++++++++++++ 2 files changed, 268 insertions(+) create mode 100644 test/yulPhaser/Program.cpp diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 6bd4f3952..957d3af21 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -140,11 +140,13 @@ detect_stray_source_files("${libyul_sources}" "libyul/") set(yul_phaser_sources yulPhaser/Chromosome.cpp + yulPhaser/Program.cpp # FIXME: yul-phaser is not a library so I can't just add it to target_link_libraries(). # My current workaround is just to include its source files here but this introduces # unnecessary duplication. Create a library or find a way to reuse the list in both places. ../tools/yulPhaser/Chromosome.cpp + ../tools/yulPhaser/Program.cpp ../tools/yulPhaser/Random.cpp ) detect_stray_source_files("${yul_phaser_sources}" "yulPhaser/") diff --git a/test/yulPhaser/Program.cpp b/test/yulPhaser/Program.cpp new file mode 100644 index 000000000..26f2934ea --- /dev/null +++ b/test/yulPhaser/Program.cpp @@ -0,0 +1,266 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#include +#include + +#include +#include +#include +#include + +#include +#include +#include + +#include + +#include +#include +#include + +using namespace std; +using namespace solidity::langutil; +using namespace solidity::util; +using namespace solidity::yul; +using namespace boost::unit_test::framework; + +namespace +{ + /// If the specified block is redundant (i.e. the only thing it contains is another block) + /// the function recurses into it and returns the first non-redundant one it finds. + /// If the block isn't redundant it just returns it immediately. + Block const& skipRedundantBlocks(Block const& _block) + { + if (_block.statements.size() == 1 && holds_alternative(_block.statements[0])) + return skipRedundantBlocks(get(_block.statements[0])); + else + return _block; + } + + string stripWhitespace(string const& input) + { + regex whitespaceRegex("\\s+"); + return regex_replace(input, whitespaceRegex, ""); + } +} + +namespace solidity::phaser::test +{ + +BOOST_AUTO_TEST_SUITE(Phaser) +BOOST_AUTO_TEST_SUITE(ProgramTest) + +BOOST_AUTO_TEST_CASE(load_should_rewind_the_stream) +{ + string sourceCode( + "{\n" + " let x := 1\n" + " let y := 2\n" + "}\n" + ); + CharStream sourceStream(sourceCode, current_test_case().p_name); + sourceStream.setPosition(5); + + auto program = Program::load(sourceStream); + + BOOST_TEST(CodeSize::codeSize(program.ast()) == 2); +} + +BOOST_AUTO_TEST_CASE(load_should_disambiguate) +{ + string sourceCode( + "{\n" + " {\n" + " let x := 1\n" + " }\n" + " {\n" + " let x := 2\n" + " }\n" + "}\n" + ); + CharStream sourceStream(sourceCode, current_test_case().p_name); + auto program = Program::load(sourceStream); + + // skipRedundantBlocks() makes the test independent of whether load() includes function grouping or not. + Block const& parentBlock = skipRedundantBlocks(program.ast()); + BOOST_TEST(parentBlock.statements.size() == 2); + + Block const& innerBlock1 = get(parentBlock.statements[0]); + Block const& innerBlock2 = get(parentBlock.statements[1]); + VariableDeclaration const& declaration1 = get(innerBlock1.statements[0]); + VariableDeclaration const& declaration2 = get(innerBlock2.statements[0]); + + BOOST_TEST(declaration1.variables[0].name.str() == "x"); + BOOST_TEST(declaration2.variables[0].name.str() != "x"); +} + +BOOST_AUTO_TEST_CASE(load_should_do_function_grouping_and_hoisting) +{ + string sourceCode( + "{\n" + " function foo() -> result\n" + " {\n" + " result := 1\n" + " }\n" + " let x := 1\n" + " function bar(a) -> result\n" + " {\n" + " result := 2\n" + " }\n" + " let y := 2\n" + "}\n" + ); + CharStream sourceStream(sourceCode, current_test_case().p_name); + auto program = Program::load(sourceStream); + + BOOST_TEST(program.ast().statements.size() == 3); + BOOST_TEST(holds_alternative(program.ast().statements[0])); + BOOST_TEST(holds_alternative(program.ast().statements[1])); + BOOST_TEST(holds_alternative(program.ast().statements[2])); +} + +BOOST_AUTO_TEST_CASE(load_should_do_loop_init_rewriting) +{ + string sourceCode( + "{\n" + " for { let i := 0 } true {}\n" + " {\n" + " }\n" + "}\n" + ); + CharStream sourceStream(sourceCode, current_test_case().p_name); + auto program = Program::load(sourceStream); + + // skipRedundantBlocks() makes the test independent of whether load() includes function grouping or not. + Block const& parentBlock = skipRedundantBlocks(program.ast()); + BOOST_TEST(holds_alternative(parentBlock.statements[0])); + BOOST_TEST(holds_alternative(parentBlock.statements[1])); +} + +BOOST_AUTO_TEST_CASE(load_should_throw_InvalidProgram_if_program_cant_be_parsed) +{ + string sourceCode("invalid program\n"); + CharStream sourceStream(sourceCode, current_test_case().p_name); + + BOOST_CHECK_THROW(Program::load(sourceStream), InvalidProgram); +} + +BOOST_AUTO_TEST_CASE(load_should_throw_InvalidProgram_if_program_cant_be_analyzed) +{ + // This should be parsed just fine but fail the analysis with: + // Error: Variable not found or variable not lvalue. + string sourceCode( + "{\n" + " x := 1\n" + "}\n" + ); + CharStream sourceStream(sourceCode, current_test_case().p_name); + + BOOST_CHECK_THROW(Program::load(sourceStream), InvalidProgram); +} + +BOOST_AUTO_TEST_CASE(optimise) +{ + string sourceCode( + "{\n" + " {\n" + " if 1 { let x := 1 }\n" + " if 0 { let y := 2 }\n" + " }\n" + "}\n" + ); + CharStream sourceStream(sourceCode, current_test_case().p_name); + auto program = Program::load(sourceStream); + + [[maybe_unused]] Block const& parentBlockBefore = skipRedundantBlocks(program.ast()); + assert(parentBlockBefore.statements.size() == 2); + assert(holds_alternative(parentBlockBefore.statements[0])); + assert(holds_alternative(parentBlockBefore.statements[1])); + + program.optimise({StructuralSimplifier::name, BlockFlattener::name}); + + Block const& parentBlockAfter = program.ast(); + BOOST_TEST(parentBlockAfter.statements.size() == 1); + BOOST_TEST(holds_alternative(parentBlockAfter.statements[0])); +} + +BOOST_AUTO_TEST_CASE(output_operator) +{ + string sourceCode( + "{\n" + " let factor := 13\n" + " {\n" + " if factor\n" + " {\n" + " let variable := add(1, 2)\n" + " }\n" + " let result := factor\n" + " }\n" + " let something := 6\n" + " let something_else := mul(something, factor)\n" + "}\n" + ); + CharStream sourceStream(sourceCode, current_test_case().p_name); + auto program = Program::load(sourceStream); + + // NOTE: The snippet above was chosen so that the few optimisations applied automatically by load() + // as of now do not change the code significantly. If that changes, you may have to update it. + BOOST_TEST(stripWhitespace(toString(program)) == stripWhitespace("{" + sourceCode + "}")); +} + +BOOST_AUTO_TEST_CASE(toJson) +{ + string sourceCode( + "{\n" + " let a := 3\n" + " if a\n" + " {\n" + " let abc := add(1, 2)\n" + " }\n" + "}\n" + ); + CharStream sourceStream(sourceCode, current_test_case().p_name); + auto program = Program::load(sourceStream); + + Json::Value parsingResult; + string errors; + BOOST_TEST(jsonParseStrict(program.toJson(), parsingResult, &errors)); + BOOST_TEST(errors.empty()); +} + +BOOST_AUTO_TEST_CASE(codeSize) +{ + string sourceCode( + "{\n" + " function foo() -> result\n" + " {\n" + " result := 15\n" + " }\n" + " let a := 1\n" + "}\n" + ); + CharStream sourceStream(sourceCode, current_test_case().p_name); + auto program = Program::load(sourceStream); + + BOOST_TEST(program.codeSize() == CodeSize::codeSizeIncludingFunctions(program.ast())); +} + +BOOST_AUTO_TEST_SUITE_END() +BOOST_AUTO_TEST_SUITE_END() + +} From ccaff1b08eaf58e68e1a4029bfccffa48bcd2775 Mon Sep 17 00:00:00 2001 From: cameel Date: Fri, 24 Jan 2020 03:32:56 +0100 Subject: [PATCH 16/17] [yul-phaser] Tests for random number generators --- test/CMakeLists.txt | 1 + test/yulPhaser/Random.cpp | 95 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 test/yulPhaser/Random.cpp diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 957d3af21..2ac7f1b7e 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -141,6 +141,7 @@ detect_stray_source_files("${libyul_sources}" "libyul/") set(yul_phaser_sources yulPhaser/Chromosome.cpp yulPhaser/Program.cpp + yulPhaser/Random.cpp # FIXME: yul-phaser is not a library so I can't just add it to target_link_libraries(). # My current workaround is just to include its source files here but this introduces diff --git a/test/yulPhaser/Random.cpp b/test/yulPhaser/Random.cpp new file mode 100644 index 000000000..c69b91055 --- /dev/null +++ b/test/yulPhaser/Random.cpp @@ -0,0 +1,95 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#include + +#include + +#include + +using namespace std; + +namespace solidity::phaser::test +{ + +BOOST_AUTO_TEST_SUITE(Phaser) +BOOST_AUTO_TEST_SUITE(RandomTest) + +BOOST_AUTO_TEST_CASE(uniformRandomInt_returns_different_values_when_called_multiple_times) +{ + constexpr uint32_t numSamples = 1000; + constexpr uint32_t numOutcomes = 100; + + vector samples1; + vector samples2; + for (uint32_t i = 0; i < numSamples; ++i) + { + samples1.push_back(uniformRandomInt(0, numOutcomes - 1)); + samples2.push_back(uniformRandomInt(0, numOutcomes - 1)); + } + + vector counts1(numSamples, 0); + vector counts2(numSamples, 0); + for (uint32_t i = 0; i < numSamples; ++i) + { + ++counts1[samples1[i]]; + ++counts2[samples2[i]]; + } + + // This test rules out not only the possibility that the two sequences are the same but also + // that they're just different permutations of the same values. The test is probabilistic so + // it's technically possible for it to fail even if generator is good but the probability is + // so low that it would happen on average once very 10^125 billion years if you repeated it + // every second. The chance is much lower than 1 in 1000^100 / 100!. + // + // This does not really guarantee that the generated numbers have the right distribution or + // or that they don't come in long, repeating sequences but the implementation is very simple + // (it just calls a generator from boost) so our goal here is just to make sure it's used + // properly and we're not getting something totally non-random, e.g. the same number every time. + BOOST_TEST(counts1 != counts2); +} + +BOOST_AUTO_TEST_CASE(binomialRandomInt_returns_different_values_when_called_multiple_times) +{ + constexpr uint32_t numSamples = 1000; + constexpr uint32_t numTrials = 100; + constexpr double successProbability = 0.6; + + vector samples1; + vector samples2; + for (uint32_t i = 0; i < numSamples; ++i) + { + samples1.push_back(binomialRandomInt(numTrials, successProbability)); + samples2.push_back(binomialRandomInt(numTrials, successProbability)); + } + + vector counts1(numSamples, 0); + vector counts2(numSamples, 0); + for (uint32_t i = 0; i < numSamples; ++i) + { + ++counts1[samples1[i]]; + ++counts2[samples2[i]]; + } + + // See remark for uniformRandomInt() above. Same applies here. + BOOST_TEST(counts1 != counts2); +} + +BOOST_AUTO_TEST_SUITE_END() +BOOST_AUTO_TEST_SUITE_END() + +} From 33cf4e4769409fd330174e2e8ef0e9660e6ee962 Mon Sep 17 00:00:00 2001 From: cameel Date: Fri, 24 Jan 2020 04:56:32 +0100 Subject: [PATCH 17/17] [yul-phaser] Tests for Population class --- test/CMakeLists.txt | 2 + test/yulPhaser/Population.cpp | 176 ++++++++++++++++++++++++++++++++++ 2 files changed, 178 insertions(+) create mode 100644 test/yulPhaser/Population.cpp diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 2ac7f1b7e..7cf466e0b 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -140,6 +140,7 @@ detect_stray_source_files("${libyul_sources}" "libyul/") set(yul_phaser_sources yulPhaser/Chromosome.cpp + yulPhaser/Population.cpp yulPhaser/Program.cpp yulPhaser/Random.cpp @@ -147,6 +148,7 @@ set(yul_phaser_sources # My current workaround is just to include its source files here but this introduces # unnecessary duplication. Create a library or find a way to reuse the list in both places. ../tools/yulPhaser/Chromosome.cpp + ../tools/yulPhaser/Population.cpp ../tools/yulPhaser/Program.cpp ../tools/yulPhaser/Random.cpp ) diff --git a/test/yulPhaser/Population.cpp b/test/yulPhaser/Population.cpp new file mode 100644 index 000000000..d0ee5a0c7 --- /dev/null +++ b/test/yulPhaser/Population.cpp @@ -0,0 +1,176 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#include +#include + +#include +#include +#include +#include + +#include + +#include + +#include +#include +#include + +using namespace std; +using namespace solidity::langutil; +using namespace solidity::yul; +using namespace boost::unit_test::framework; + +namespace solidity::phaser::test +{ + +namespace +{ + bool fitnessNotSet(Individual const& individual) + { + return !individual.fitness.has_value(); + } + + bool fitnessSet(Individual const& individual) + { + return individual.fitness.has_value(); + } +} + +BOOST_AUTO_TEST_SUITE(Phaser) +BOOST_AUTO_TEST_SUITE(PopulationTest) + +string const& sampleSourceCode = + "{\n" + " let factor := 13\n" + " {\n" + " if factor\n" + " {\n" + " let variable := add(1, 2)\n" + " }\n" + " let result := factor\n" + " }\n" + " let something := 6\n" + " {\n" + " {\n" + " {\n" + " let value := 15\n" + " }\n" + " }\n" + " }\n" + " let something_else := mul(mul(something, 1), add(factor, 0))\n" + " if 1 { let x := 1 }\n" + " if 0 { let y := 2 }\n" + "}\n"; + +BOOST_AUTO_TEST_CASE(constructor_should_copy_chromosomes_and_not_compute_fitness) +{ + CharStream sourceStream(sampleSourceCode, current_test_case().p_name); + vector chromosomes = { + Chromosome::makeRandom(5), + Chromosome::makeRandom(10), + }; + Population population(sourceStream, chromosomes); + + BOOST_TEST(population.individuals().size() == 2); + BOOST_TEST(population.individuals()[0].chromosome == chromosomes[0]); + BOOST_TEST(population.individuals()[1].chromosome == chromosomes[1]); + + auto fitnessNotSet = [](auto const& individual){ return !individual.fitness.has_value(); }; + BOOST_TEST(all_of(population.individuals().begin(), population.individuals().end(), fitnessNotSet)); +} + +BOOST_AUTO_TEST_CASE(makeRandom_should_return_population_with_random_chromosomes) +{ + CharStream sourceStream(sampleSourceCode, current_test_case().p_name); + auto population1 = Population::makeRandom(sourceStream, 100); + auto population2 = Population::makeRandom(sourceStream, 100); + + BOOST_TEST(population1.individuals().size() == 100); + BOOST_TEST(population2.individuals().size() == 100); + + int numMatchingPositions = 0; + for (size_t i = 0; i < 100; ++i) + if (population1.individuals()[i].chromosome == population2.individuals()[i].chromosome) + ++numMatchingPositions; + + // Assume that the results are random if there are no more than 10 identical chromosomes on the + // same positions. One duplicate is very unlikely but still possible after billions of runs + // (especially for short chromosomes). For ten the probability is so small that we can ignore it. + BOOST_TEST(numMatchingPositions < 10); +} + +BOOST_AUTO_TEST_CASE(makeRandom_should_not_compute_fitness) +{ + CharStream sourceStream(sampleSourceCode, current_test_case().p_name); + auto population = Population::makeRandom(sourceStream, 5); + + BOOST_TEST(all_of(population.individuals().begin(), population.individuals().end(), fitnessNotSet)); +} + +BOOST_AUTO_TEST_CASE(run_should_evaluate_fitness) +{ + stringstream output; + CharStream sourceStream(sampleSourceCode, current_test_case().p_name); + auto population = Population::makeRandom(sourceStream, 5); + assert(all_of(population.individuals().begin(), population.individuals().end(), fitnessNotSet)); + + population.run(1, output); + + BOOST_TEST(all_of(population.individuals().begin(), population.individuals().end(), fitnessSet)); +} + +BOOST_AUTO_TEST_CASE(run_should_not_make_fitness_of_top_chromosomes_worse) +{ + stringstream output; + CharStream sourceStream(sampleSourceCode, current_test_case().p_name); + vector chromosomes = { + Chromosome({StructuralSimplifier::name}), + Chromosome({BlockFlattener::name}), + Chromosome({SSAReverser::name}), + Chromosome({UnusedPruner::name}), + Chromosome({StructuralSimplifier::name, BlockFlattener::name}), + }; + Population population(sourceStream, chromosomes); + + size_t initialTopFitness[2] = { + Population::measureFitness(chromosomes[0], sourceStream), + Population::measureFitness(chromosomes[1], sourceStream), + }; + + for (int i = 0; i < 6; ++i) + { + population.run(1, output); + BOOST_TEST(population.individuals().size() == 5); + BOOST_TEST(fitnessSet(population.individuals()[0])); + BOOST_TEST(fitnessSet(population.individuals()[1])); + + size_t currentTopFitness[2] = { + population.individuals()[0].fitness.value(), + population.individuals()[1].fitness.value(), + }; + BOOST_TEST(currentTopFitness[0] <= initialTopFitness[0]); + BOOST_TEST(currentTopFitness[1] <= initialTopFitness[1]); + BOOST_TEST(currentTopFitness[0] <= currentTopFitness[1]); + } +} + +BOOST_AUTO_TEST_SUITE_END() +BOOST_AUTO_TEST_SUITE_END() + +}