diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index e7a9d903f..7cf466e0b 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -138,6 +138,22 @@ set(libyul_sources ) detect_stray_source_files("${libyul_sources}" "libyul/") +set(yul_phaser_sources + yulPhaser/Chromosome.cpp + yulPhaser/Population.cpp + yulPhaser/Program.cpp + yulPhaser/Random.cpp + + # FIXME: yul-phaser is not a library so I can't just add it to target_link_libraries(). + # My current workaround is just to include its source files here but this introduces + # unnecessary duplication. Create a library or find a way to reuse the list in both places. + ../tools/yulPhaser/Chromosome.cpp + ../tools/yulPhaser/Population.cpp + ../tools/yulPhaser/Program.cpp + ../tools/yulPhaser/Random.cpp +) +detect_stray_source_files("${yul_phaser_sources}" "yulPhaser/") + add_executable(soltest ${sources} ${contracts_sources} ${libsolutil_sources} @@ -146,6 +162,7 @@ add_executable(soltest ${sources} ${libyul_sources} ${libsolidity_sources} ${libsolidity_util_sources} + ${yul_phaser_sources} ) target_link_libraries(soltest PRIVATE libsolc yul solidity yulInterpreter evmasm solutil Boost::boost Boost::program_options Boost::unit_test_framework evmc) diff --git a/test/yulPhaser/Chromosome.cpp b/test/yulPhaser/Chromosome.cpp new file mode 100644 index 000000000..dd250d0ec --- /dev/null +++ b/test/yulPhaser/Chromosome.cpp @@ -0,0 +1,101 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#include + +#include +#include +#include +#include + +#include + +#include + +using namespace std; +using namespace solidity::yul; +using namespace solidity::util; + +namespace solidity::phaser::test +{ + +BOOST_AUTO_TEST_SUITE(Phaser) +BOOST_AUTO_TEST_SUITE(ChromosomeTest) + +BOOST_AUTO_TEST_CASE(makeRandom_should_create_chromosome_with_random_optimisation_steps) +{ + constexpr uint32_t numSteps = 1000; + + auto chromosome1 = Chromosome::makeRandom(numSteps); + auto chromosome2 = Chromosome::makeRandom(numSteps); + BOOST_CHECK_EQUAL(chromosome1.length(), numSteps); + BOOST_CHECK_EQUAL(chromosome2.length(), numSteps); + + multiset steps1; + multiset steps2; + for (auto const& step: chromosome1.optimisationSteps()) + steps1.insert(step); + for (auto const& step: chromosome2.optimisationSteps()) + steps2.insert(step); + + // Check if steps are different and also if they're not just a permutation of the same set. + // Technically they could be the same and still random but the probability is infinitesimally low. + BOOST_TEST(steps1 != steps2); +} + +BOOST_AUTO_TEST_CASE(constructor_should_store_optimisation_steps) +{ + vector steps = { + StructuralSimplifier::name, + BlockFlattener::name, + UnusedPruner::name, + }; + Chromosome chromosome(steps); + + BOOST_TEST(steps == chromosome.optimisationSteps()); +} + +BOOST_AUTO_TEST_CASE(constructor_should_allow_duplicate_steps) +{ + vector steps = { + StructuralSimplifier::name, + StructuralSimplifier::name, + BlockFlattener::name, + UnusedPruner::name, + BlockFlattener::name, + }; + Chromosome chromosome(steps); + + BOOST_TEST(steps == chromosome.optimisationSteps()); +} + +BOOST_AUTO_TEST_CASE(output_operator_should_create_concise_and_unambiguous_string_representation) +{ + vector allSteps; + for (auto const& step: OptimiserSuite::allSteps()) + allSteps.push_back(step.first); + Chromosome chromosome(allSteps); + + BOOST_TEST(chromosome.length() == allSteps.size()); + BOOST_TEST(chromosome.optimisationSteps() == allSteps); + BOOST_TEST(toString(chromosome) == "fcCUnDvejsxIOoighTLMrmVatud"); +} + +BOOST_AUTO_TEST_SUITE_END() +BOOST_AUTO_TEST_SUITE_END() + +} diff --git a/test/yulPhaser/Population.cpp b/test/yulPhaser/Population.cpp new file mode 100644 index 000000000..d0ee5a0c7 --- /dev/null +++ b/test/yulPhaser/Population.cpp @@ -0,0 +1,176 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#include +#include + +#include +#include +#include +#include + +#include + +#include + +#include +#include +#include + +using namespace std; +using namespace solidity::langutil; +using namespace solidity::yul; +using namespace boost::unit_test::framework; + +namespace solidity::phaser::test +{ + +namespace +{ + bool fitnessNotSet(Individual const& individual) + { + return !individual.fitness.has_value(); + } + + bool fitnessSet(Individual const& individual) + { + return individual.fitness.has_value(); + } +} + +BOOST_AUTO_TEST_SUITE(Phaser) +BOOST_AUTO_TEST_SUITE(PopulationTest) + +string const& sampleSourceCode = + "{\n" + " let factor := 13\n" + " {\n" + " if factor\n" + " {\n" + " let variable := add(1, 2)\n" + " }\n" + " let result := factor\n" + " }\n" + " let something := 6\n" + " {\n" + " {\n" + " {\n" + " let value := 15\n" + " }\n" + " }\n" + " }\n" + " let something_else := mul(mul(something, 1), add(factor, 0))\n" + " if 1 { let x := 1 }\n" + " if 0 { let y := 2 }\n" + "}\n"; + +BOOST_AUTO_TEST_CASE(constructor_should_copy_chromosomes_and_not_compute_fitness) +{ + CharStream sourceStream(sampleSourceCode, current_test_case().p_name); + vector chromosomes = { + Chromosome::makeRandom(5), + Chromosome::makeRandom(10), + }; + Population population(sourceStream, chromosomes); + + BOOST_TEST(population.individuals().size() == 2); + BOOST_TEST(population.individuals()[0].chromosome == chromosomes[0]); + BOOST_TEST(population.individuals()[1].chromosome == chromosomes[1]); + + auto fitnessNotSet = [](auto const& individual){ return !individual.fitness.has_value(); }; + BOOST_TEST(all_of(population.individuals().begin(), population.individuals().end(), fitnessNotSet)); +} + +BOOST_AUTO_TEST_CASE(makeRandom_should_return_population_with_random_chromosomes) +{ + CharStream sourceStream(sampleSourceCode, current_test_case().p_name); + auto population1 = Population::makeRandom(sourceStream, 100); + auto population2 = Population::makeRandom(sourceStream, 100); + + BOOST_TEST(population1.individuals().size() == 100); + BOOST_TEST(population2.individuals().size() == 100); + + int numMatchingPositions = 0; + for (size_t i = 0; i < 100; ++i) + if (population1.individuals()[i].chromosome == population2.individuals()[i].chromosome) + ++numMatchingPositions; + + // Assume that the results are random if there are no more than 10 identical chromosomes on the + // same positions. One duplicate is very unlikely but still possible after billions of runs + // (especially for short chromosomes). For ten the probability is so small that we can ignore it. + BOOST_TEST(numMatchingPositions < 10); +} + +BOOST_AUTO_TEST_CASE(makeRandom_should_not_compute_fitness) +{ + CharStream sourceStream(sampleSourceCode, current_test_case().p_name); + auto population = Population::makeRandom(sourceStream, 5); + + BOOST_TEST(all_of(population.individuals().begin(), population.individuals().end(), fitnessNotSet)); +} + +BOOST_AUTO_TEST_CASE(run_should_evaluate_fitness) +{ + stringstream output; + CharStream sourceStream(sampleSourceCode, current_test_case().p_name); + auto population = Population::makeRandom(sourceStream, 5); + assert(all_of(population.individuals().begin(), population.individuals().end(), fitnessNotSet)); + + population.run(1, output); + + BOOST_TEST(all_of(population.individuals().begin(), population.individuals().end(), fitnessSet)); +} + +BOOST_AUTO_TEST_CASE(run_should_not_make_fitness_of_top_chromosomes_worse) +{ + stringstream output; + CharStream sourceStream(sampleSourceCode, current_test_case().p_name); + vector chromosomes = { + Chromosome({StructuralSimplifier::name}), + Chromosome({BlockFlattener::name}), + Chromosome({SSAReverser::name}), + Chromosome({UnusedPruner::name}), + Chromosome({StructuralSimplifier::name, BlockFlattener::name}), + }; + Population population(sourceStream, chromosomes); + + size_t initialTopFitness[2] = { + Population::measureFitness(chromosomes[0], sourceStream), + Population::measureFitness(chromosomes[1], sourceStream), + }; + + for (int i = 0; i < 6; ++i) + { + population.run(1, output); + BOOST_TEST(population.individuals().size() == 5); + BOOST_TEST(fitnessSet(population.individuals()[0])); + BOOST_TEST(fitnessSet(population.individuals()[1])); + + size_t currentTopFitness[2] = { + population.individuals()[0].fitness.value(), + population.individuals()[1].fitness.value(), + }; + BOOST_TEST(currentTopFitness[0] <= initialTopFitness[0]); + BOOST_TEST(currentTopFitness[1] <= initialTopFitness[1]); + BOOST_TEST(currentTopFitness[0] <= currentTopFitness[1]); + } +} + +BOOST_AUTO_TEST_SUITE_END() +BOOST_AUTO_TEST_SUITE_END() + +} diff --git a/test/yulPhaser/Program.cpp b/test/yulPhaser/Program.cpp new file mode 100644 index 000000000..26f2934ea --- /dev/null +++ b/test/yulPhaser/Program.cpp @@ -0,0 +1,266 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#include +#include + +#include +#include +#include +#include + +#include +#include +#include + +#include + +#include +#include +#include + +using namespace std; +using namespace solidity::langutil; +using namespace solidity::util; +using namespace solidity::yul; +using namespace boost::unit_test::framework; + +namespace +{ + /// If the specified block is redundant (i.e. the only thing it contains is another block) + /// the function recurses into it and returns the first non-redundant one it finds. + /// If the block isn't redundant it just returns it immediately. + Block const& skipRedundantBlocks(Block const& _block) + { + if (_block.statements.size() == 1 && holds_alternative(_block.statements[0])) + return skipRedundantBlocks(get(_block.statements[0])); + else + return _block; + } + + string stripWhitespace(string const& input) + { + regex whitespaceRegex("\\s+"); + return regex_replace(input, whitespaceRegex, ""); + } +} + +namespace solidity::phaser::test +{ + +BOOST_AUTO_TEST_SUITE(Phaser) +BOOST_AUTO_TEST_SUITE(ProgramTest) + +BOOST_AUTO_TEST_CASE(load_should_rewind_the_stream) +{ + string sourceCode( + "{\n" + " let x := 1\n" + " let y := 2\n" + "}\n" + ); + CharStream sourceStream(sourceCode, current_test_case().p_name); + sourceStream.setPosition(5); + + auto program = Program::load(sourceStream); + + BOOST_TEST(CodeSize::codeSize(program.ast()) == 2); +} + +BOOST_AUTO_TEST_CASE(load_should_disambiguate) +{ + string sourceCode( + "{\n" + " {\n" + " let x := 1\n" + " }\n" + " {\n" + " let x := 2\n" + " }\n" + "}\n" + ); + CharStream sourceStream(sourceCode, current_test_case().p_name); + auto program = Program::load(sourceStream); + + // skipRedundantBlocks() makes the test independent of whether load() includes function grouping or not. + Block const& parentBlock = skipRedundantBlocks(program.ast()); + BOOST_TEST(parentBlock.statements.size() == 2); + + Block const& innerBlock1 = get(parentBlock.statements[0]); + Block const& innerBlock2 = get(parentBlock.statements[1]); + VariableDeclaration const& declaration1 = get(innerBlock1.statements[0]); + VariableDeclaration const& declaration2 = get(innerBlock2.statements[0]); + + BOOST_TEST(declaration1.variables[0].name.str() == "x"); + BOOST_TEST(declaration2.variables[0].name.str() != "x"); +} + +BOOST_AUTO_TEST_CASE(load_should_do_function_grouping_and_hoisting) +{ + string sourceCode( + "{\n" + " function foo() -> result\n" + " {\n" + " result := 1\n" + " }\n" + " let x := 1\n" + " function bar(a) -> result\n" + " {\n" + " result := 2\n" + " }\n" + " let y := 2\n" + "}\n" + ); + CharStream sourceStream(sourceCode, current_test_case().p_name); + auto program = Program::load(sourceStream); + + BOOST_TEST(program.ast().statements.size() == 3); + BOOST_TEST(holds_alternative(program.ast().statements[0])); + BOOST_TEST(holds_alternative(program.ast().statements[1])); + BOOST_TEST(holds_alternative(program.ast().statements[2])); +} + +BOOST_AUTO_TEST_CASE(load_should_do_loop_init_rewriting) +{ + string sourceCode( + "{\n" + " for { let i := 0 } true {}\n" + " {\n" + " }\n" + "}\n" + ); + CharStream sourceStream(sourceCode, current_test_case().p_name); + auto program = Program::load(sourceStream); + + // skipRedundantBlocks() makes the test independent of whether load() includes function grouping or not. + Block const& parentBlock = skipRedundantBlocks(program.ast()); + BOOST_TEST(holds_alternative(parentBlock.statements[0])); + BOOST_TEST(holds_alternative(parentBlock.statements[1])); +} + +BOOST_AUTO_TEST_CASE(load_should_throw_InvalidProgram_if_program_cant_be_parsed) +{ + string sourceCode("invalid program\n"); + CharStream sourceStream(sourceCode, current_test_case().p_name); + + BOOST_CHECK_THROW(Program::load(sourceStream), InvalidProgram); +} + +BOOST_AUTO_TEST_CASE(load_should_throw_InvalidProgram_if_program_cant_be_analyzed) +{ + // This should be parsed just fine but fail the analysis with: + // Error: Variable not found or variable not lvalue. + string sourceCode( + "{\n" + " x := 1\n" + "}\n" + ); + CharStream sourceStream(sourceCode, current_test_case().p_name); + + BOOST_CHECK_THROW(Program::load(sourceStream), InvalidProgram); +} + +BOOST_AUTO_TEST_CASE(optimise) +{ + string sourceCode( + "{\n" + " {\n" + " if 1 { let x := 1 }\n" + " if 0 { let y := 2 }\n" + " }\n" + "}\n" + ); + CharStream sourceStream(sourceCode, current_test_case().p_name); + auto program = Program::load(sourceStream); + + [[maybe_unused]] Block const& parentBlockBefore = skipRedundantBlocks(program.ast()); + assert(parentBlockBefore.statements.size() == 2); + assert(holds_alternative(parentBlockBefore.statements[0])); + assert(holds_alternative(parentBlockBefore.statements[1])); + + program.optimise({StructuralSimplifier::name, BlockFlattener::name}); + + Block const& parentBlockAfter = program.ast(); + BOOST_TEST(parentBlockAfter.statements.size() == 1); + BOOST_TEST(holds_alternative(parentBlockAfter.statements[0])); +} + +BOOST_AUTO_TEST_CASE(output_operator) +{ + string sourceCode( + "{\n" + " let factor := 13\n" + " {\n" + " if factor\n" + " {\n" + " let variable := add(1, 2)\n" + " }\n" + " let result := factor\n" + " }\n" + " let something := 6\n" + " let something_else := mul(something, factor)\n" + "}\n" + ); + CharStream sourceStream(sourceCode, current_test_case().p_name); + auto program = Program::load(sourceStream); + + // NOTE: The snippet above was chosen so that the few optimisations applied automatically by load() + // as of now do not change the code significantly. If that changes, you may have to update it. + BOOST_TEST(stripWhitespace(toString(program)) == stripWhitespace("{" + sourceCode + "}")); +} + +BOOST_AUTO_TEST_CASE(toJson) +{ + string sourceCode( + "{\n" + " let a := 3\n" + " if a\n" + " {\n" + " let abc := add(1, 2)\n" + " }\n" + "}\n" + ); + CharStream sourceStream(sourceCode, current_test_case().p_name); + auto program = Program::load(sourceStream); + + Json::Value parsingResult; + string errors; + BOOST_TEST(jsonParseStrict(program.toJson(), parsingResult, &errors)); + BOOST_TEST(errors.empty()); +} + +BOOST_AUTO_TEST_CASE(codeSize) +{ + string sourceCode( + "{\n" + " function foo() -> result\n" + " {\n" + " result := 15\n" + " }\n" + " let a := 1\n" + "}\n" + ); + CharStream sourceStream(sourceCode, current_test_case().p_name); + auto program = Program::load(sourceStream); + + BOOST_TEST(program.codeSize() == CodeSize::codeSizeIncludingFunctions(program.ast())); +} + +BOOST_AUTO_TEST_SUITE_END() +BOOST_AUTO_TEST_SUITE_END() + +} diff --git a/test/yulPhaser/Random.cpp b/test/yulPhaser/Random.cpp new file mode 100644 index 000000000..c69b91055 --- /dev/null +++ b/test/yulPhaser/Random.cpp @@ -0,0 +1,95 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#include + +#include + +#include + +using namespace std; + +namespace solidity::phaser::test +{ + +BOOST_AUTO_TEST_SUITE(Phaser) +BOOST_AUTO_TEST_SUITE(RandomTest) + +BOOST_AUTO_TEST_CASE(uniformRandomInt_returns_different_values_when_called_multiple_times) +{ + constexpr uint32_t numSamples = 1000; + constexpr uint32_t numOutcomes = 100; + + vector samples1; + vector samples2; + for (uint32_t i = 0; i < numSamples; ++i) + { + samples1.push_back(uniformRandomInt(0, numOutcomes - 1)); + samples2.push_back(uniformRandomInt(0, numOutcomes - 1)); + } + + vector counts1(numSamples, 0); + vector counts2(numSamples, 0); + for (uint32_t i = 0; i < numSamples; ++i) + { + ++counts1[samples1[i]]; + ++counts2[samples2[i]]; + } + + // This test rules out not only the possibility that the two sequences are the same but also + // that they're just different permutations of the same values. The test is probabilistic so + // it's technically possible for it to fail even if generator is good but the probability is + // so low that it would happen on average once very 10^125 billion years if you repeated it + // every second. The chance is much lower than 1 in 1000^100 / 100!. + // + // This does not really guarantee that the generated numbers have the right distribution or + // or that they don't come in long, repeating sequences but the implementation is very simple + // (it just calls a generator from boost) so our goal here is just to make sure it's used + // properly and we're not getting something totally non-random, e.g. the same number every time. + BOOST_TEST(counts1 != counts2); +} + +BOOST_AUTO_TEST_CASE(binomialRandomInt_returns_different_values_when_called_multiple_times) +{ + constexpr uint32_t numSamples = 1000; + constexpr uint32_t numTrials = 100; + constexpr double successProbability = 0.6; + + vector samples1; + vector samples2; + for (uint32_t i = 0; i < numSamples; ++i) + { + samples1.push_back(binomialRandomInt(numTrials, successProbability)); + samples2.push_back(binomialRandomInt(numTrials, successProbability)); + } + + vector counts1(numSamples, 0); + vector counts2(numSamples, 0); + for (uint32_t i = 0; i < numSamples; ++i) + { + ++counts1[samples1[i]]; + ++counts2[samples2[i]]; + } + + // See remark for uniformRandomInt() above. Same applies here. + BOOST_TEST(counts1 != counts2); +} + +BOOST_AUTO_TEST_SUITE_END() +BOOST_AUTO_TEST_SUITE_END() + +} diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index bdcf615af..a2aa807af 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -12,3 +12,18 @@ target_link_libraries(solidity-upgrade PRIVATE solidity Boost::boost Boost::prog include(GNUInstallDirs) install(TARGETS solidity-upgrade DESTINATION "${CMAKE_INSTALL_BINDIR}") + +add_executable(yul-phaser + yulPhaser/main.cpp + yulPhaser/Population.h + yulPhaser/Population.cpp + yulPhaser/Chromosome.h + yulPhaser/Chromosome.cpp + yulPhaser/Program.h + yulPhaser/Program.cpp + yulPhaser/Random.h + yulPhaser/Random.cpp +) +target_link_libraries(yul-phaser PRIVATE solidity Boost::program_options) + +install(TARGETS yul-phaser DESTINATION "${CMAKE_INSTALL_BINDIR}") diff --git a/tools/yulPhaser/Chromosome.cpp b/tools/yulPhaser/Chromosome.cpp new file mode 100644 index 000000000..b332266ba --- /dev/null +++ b/tools/yulPhaser/Chromosome.cpp @@ -0,0 +1,70 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#include + +#include + +#include +#include + +#include + +using namespace std; +using namespace solidity; +using namespace solidity::yul; +using namespace solidity::phaser; + +namespace solidity::phaser +{ + +ostream& operator<<(ostream& _stream, Chromosome const& _chromosome); + +} + +Chromosome Chromosome::makeRandom(size_t _length) +{ + vector steps; + for (size_t i = 0; i < _length; ++i) + steps.push_back(randomOptimisationStep()); + + return Chromosome(move(steps)); +} + +ostream& phaser::operator<<(ostream& _stream, Chromosome const& _chromosome) +{ + for (auto const& stepName: _chromosome.m_optimisationSteps) + _stream << OptimiserSuite::stepNameToAbbreviationMap().at(stepName); + + return _stream; +} + +vector Chromosome::allStepNames() +{ + vector stepNames; + for (auto const& step: OptimiserSuite::allSteps()) + stepNames.push_back(step.first); + + return stepNames; +} + +string const& Chromosome::randomOptimisationStep() +{ + static vector stepNames = allStepNames(); + + return stepNames[uniformRandomInt(0, stepNames.size() - 1)]; +} diff --git a/tools/yulPhaser/Chromosome.h b/tools/yulPhaser/Chromosome.h new file mode 100644 index 000000000..6db99dc4e --- /dev/null +++ b/tools/yulPhaser/Chromosome.h @@ -0,0 +1,62 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#pragma once + +#include +#include +#include +#include + +namespace solidity::phaser +{ + +/** + * An object that represents a sequence of optimiser steps that can be applied to a program. + * Such sequences are used in our genetic algorithm to represent individual members of the + * population. + * + * To calculate the fitness of an individual one must apply its sequence to a specific program. + * This class does not provide any means to do so. It just stores information. + * + * Once created a sequence cannot be changed. The only way to mutate it is to generate a new + * chromosome based on the old one. + */ +class Chromosome +{ +public: + Chromosome() = default; + explicit Chromosome(std::vector _optimisationSteps): + m_optimisationSteps(std::move(_optimisationSteps)) {} + static Chromosome makeRandom(size_t _length); + + size_t length() const { return m_optimisationSteps.size(); } + std::vector const& optimisationSteps() const { return m_optimisationSteps; } + + friend std::ostream& operator<<(std::ostream& _stream, Chromosome const& _chromosome); + + bool operator==(Chromosome const& _other) const { return m_optimisationSteps == _other.m_optimisationSteps; } + bool operator!=(Chromosome const& _other) const { return !(*this == _other); } + +private: + static std::vector allStepNames(); + static std::string const& randomOptimisationStep(); + + std::vector m_optimisationSteps; +}; + +} diff --git a/tools/yulPhaser/Exceptions.h b/tools/yulPhaser/Exceptions.h new file mode 100644 index 000000000..ae75d19ef --- /dev/null +++ b/tools/yulPhaser/Exceptions.h @@ -0,0 +1,27 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#pragma once + +#include + +namespace solidity::phaser +{ + +struct InvalidProgram: virtual util::Exception {}; + +} diff --git a/tools/yulPhaser/Population.cpp b/tools/yulPhaser/Population.cpp new file mode 100644 index 000000000..ca6fe7891 --- /dev/null +++ b/tools/yulPhaser/Population.cpp @@ -0,0 +1,138 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#include + +#include + +#include +#include +#include +#include + +using namespace std; +using namespace solidity; +using namespace solidity::langutil; +using namespace solidity::phaser; + +namespace solidity::phaser +{ + +ostream& operator<<(ostream& _stream, Individual const& _individual); +ostream& operator<<(ostream& _stream, Population const& _population); + +} + +ostream& phaser::operator<<(ostream& _stream, Individual const& _individual) +{ + _stream << "Fitness: "; + if (_individual.fitness.has_value()) + _stream << _individual.fitness.value(); + else + _stream << ""; + _stream << ", optimisations: " << _individual.chromosome; + + return _stream; +} + +Population::Population(CharStream _sourceCode, vector const& _chromosomes): + m_sourceCode{move(_sourceCode)} +{ + for (auto const& chromosome: _chromosomes) + m_individuals.push_back({chromosome}); +} + +Population Population::makeRandom(CharStream _sourceCode, size_t _size) +{ + vector individuals; + for (size_t i = 0; i < _size; ++i) + individuals.push_back({Chromosome::makeRandom(randomChromosomeLength())}); + + return Population(move(_sourceCode), individuals); +} + +size_t Population::measureFitness(Chromosome const& _chromosome, CharStream& _sourceCode) +{ + auto program = Program::load(_sourceCode); + program.optimise(_chromosome.optimisationSteps()); + return program.codeSize(); +} + +void Population::run(optional _numRounds, ostream& _outputStream) +{ + doEvaluation(); + for (size_t round = 0; !_numRounds.has_value() || round < _numRounds.value(); ++round) + { + doMutation(); + doSelection(); + doEvaluation(); + + _outputStream << "---------- ROUND " << round << " ----------" << endl; + _outputStream << *this; + } +} + +ostream& phaser::operator<<(ostream& _stream, Population const& _population) +{ + _stream << "Stream name: " << _population.m_sourceCode.name() << endl; + + auto individual = _population.m_individuals.begin(); + for (; individual != _population.m_individuals.end(); ++individual) + _stream << *individual << endl; + + return _stream; +} + +void Population::doMutation() +{ + // TODO: Implement mutation and crossover +} + +void Population::doEvaluation() +{ + for (auto& individual: m_individuals) + if (!individual.fitness.has_value()) + individual.fitness = measureFitness(individual.chromosome, m_sourceCode); +} + +void Population::doSelection() +{ + assert(all_of(m_individuals.begin(), m_individuals.end(), [](auto& i){ return i.fitness.has_value(); })); + + sort( + m_individuals.begin(), + m_individuals.end(), + [](auto const& a, auto const& b){ return a.fitness.value() < b.fitness.value(); } + ); + + randomizeWorstChromosomes(m_individuals, m_individuals.size() / 2); +} + +void Population::randomizeWorstChromosomes( + vector& _individuals, + size_t _count +) +{ + assert(_individuals.size() >= _count); + // ASSUMPTION: _individuals is sorted in ascending order + + auto individual = _individuals.begin() + (_individuals.size() - _count); + for (; individual != _individuals.end(); ++individual) + { + *individual = {Chromosome::makeRandom(randomChromosomeLength())}; + } +} diff --git a/tools/yulPhaser/Population.h b/tools/yulPhaser/Population.h new file mode 100644 index 000000000..6e30b6193 --- /dev/null +++ b/tools/yulPhaser/Population.h @@ -0,0 +1,89 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#pragma once + +#include +#include + +#include + +#include +#include +#include + +namespace solidity::phaser +{ + +/** + * Information describing the state of an individual member of the population during the course + * of the genetic algorithm. + */ +struct Individual +{ + Chromosome chromosome; + std::optional fitness = std::nullopt; + + friend std::ostream& operator<<(std::ostream& _stream, Individual const& _individual); +}; + +/** + * Represents a changing set of individuals undergoing a genetic algorithm. + * Each round of the algorithm involves mutating existing individuals, evaluating their fitness + * and selecting the best ones for the next round. + * + * An individual is a sequence of optimiser steps represented by a @a Chromosome instance. The whole + * population is associated with a fixed Yul program. By loading the source code into a @a Program + * instance the class can compute fitness of the individual. + */ +class Population +{ +public: + static constexpr size_t MaxChromosomeLength = 30; + + explicit Population(langutil::CharStream _sourceCode, std::vector const& _chromosomes = {}); + static Population makeRandom(langutil::CharStream _sourceCode, size_t _size); + + void run(std::optional _numRounds, std::ostream& _outputStream); + + std::vector const& individuals() const { return m_individuals; } + + static size_t randomChromosomeLength() { return binomialRandomInt(MaxChromosomeLength, 0.5); } + static size_t measureFitness(Chromosome const& _chromosome, langutil::CharStream& _sourceCode); + + friend std::ostream& operator<<(std::ostream& _stream, Population const& _population); + +private: + explicit Population(langutil::CharStream _sourceCode, std::vector _individuals = {}): + m_sourceCode{std::move(_sourceCode)}, + m_individuals{std::move(_individuals)} {} + + void doMutation(); + void doEvaluation(); + void doSelection(); + + static void randomizeWorstChromosomes( + std::vector& _individuals, + size_t _count + ); + + langutil::CharStream m_sourceCode; + + std::vector m_individuals; +}; + +} diff --git a/tools/yulPhaser/Program.cpp b/tools/yulPhaser/Program.cpp new file mode 100644 index 000000000..6137ac244 --- /dev/null +++ b/tools/yulPhaser/Program.cpp @@ -0,0 +1,155 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#include + +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +using namespace std; +using namespace solidity; +using namespace solidity::langutil; +using namespace solidity::yul; +using namespace solidity::util; +using namespace solidity::phaser; + +namespace solidity::phaser +{ + +ostream& operator<<(ostream& _stream, Program const& _program); + +} + +Program Program::load(CharStream& _sourceCode) +{ + // ASSUMPTION: parseSource() rewinds the stream on its own + Dialect const& dialect = EVMDialect::strictAssemblyForEVMObjects(EVMVersion{}); + unique_ptr ast = parseSource(dialect, _sourceCode); + unique_ptr analysisInfo = analyzeAST(dialect, *ast); + + Program program( + dialect, + disambiguateAST(dialect, *ast, *analysisInfo) + ); + program.optimise({ + FunctionHoister::name, + FunctionGrouper::name, + ForLoopInitRewriter::name, + }); + + return program; +} + +void Program::optimise(vector const& _optimisationSteps) +{ + applyOptimisationSteps(m_dialect, m_nameDispenser, *m_ast, _optimisationSteps); +} + +ostream& phaser::operator<<(ostream& _stream, Program const& _program) +{ + return _stream << AsmPrinter()(*_program.m_ast); +} + +string Program::toJson() const +{ + Json::Value serializedAst = AsmJsonConverter(0)(*m_ast); + return jsonPrettyPrint(serializedAst); +} + +unique_ptr Program::parseSource(Dialect const& _dialect, CharStream _source) +{ + ErrorList errors; + ErrorReporter errorReporter(errors); + auto scanner = make_shared(move(_source)); + Parser parser(errorReporter, _dialect); + + unique_ptr ast = parser.parse(scanner, false); + assertThrow(ast != nullptr, InvalidProgram, "Error parsing source"); + assert(errorReporter.errors().empty()); + + return ast; +} + +unique_ptr Program::analyzeAST(Dialect const& _dialect, Block const& _ast) +{ + ErrorList errors; + ErrorReporter errorReporter(errors); + auto analysisInfo = make_unique(); + AsmAnalyzer analyzer(*analysisInfo, errorReporter, _dialect); + + bool analysisSuccessful = analyzer.analyze(_ast); + assertThrow(analysisSuccessful, InvalidProgram, "Error analyzing source"); + assert(errorReporter.errors().empty()); + + return analysisInfo; +} + +unique_ptr Program::disambiguateAST( + Dialect const& _dialect, + Block const& _ast, + AsmAnalysisInfo const& _analysisInfo +) +{ + set const externallyUsedIdentifiers = {}; + Disambiguator disambiguator(_dialect, _analysisInfo, externallyUsedIdentifiers); + + return make_unique(get(disambiguator(_ast))); +} + +void Program::applyOptimisationSteps( + Dialect const& _dialect, + NameDispenser& _nameDispenser, + Block& _ast, + vector const& _optimisationSteps +) +{ + // An empty set of reserved identifiers. It could be a constructor parameter but I don't + // think it would be useful in this tool. Other tools (like yulopti) have it empty too. + set const externallyUsedIdentifiers = {}; + OptimiserStepContext context{_dialect, _nameDispenser, externallyUsedIdentifiers}; + + for (string const& step: _optimisationSteps) + OptimiserSuite::allSteps().at(step)->run(context, _ast); +} + +size_t Program::computeCodeSize(Block const& _ast) +{ + return CodeSize::codeSizeIncludingFunctions(_ast); +} diff --git a/tools/yulPhaser/Program.h b/tools/yulPhaser/Program.h new file mode 100644 index 000000000..822503f26 --- /dev/null +++ b/tools/yulPhaser/Program.h @@ -0,0 +1,113 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#pragma once + +#include +#include + +#include + +#include +#include +#include +#include +#include + +namespace solidity::langutil +{ + +class CharStream; + +} + +namespace solidity::yul +{ + +struct AsmAnalysisInfo; +struct Dialect; + +} + +namespace solidity::phaser +{ + +/** + * Class representing parsed and analysed Yul program that we can apply optimisations to. + * The program is already disambiguated and has several prerequisite optimiser steps applied to it + * so that the requirements of any possible step that could be later applied by the user are + * already satisfied. + * + * The class allows the user to apply extra optimisations and obtain metrics and general + * information about the resulting syntax tree. + */ +class Program: private boost::noncopyable +{ +public: + Program(Program&& program): + m_ast(std::move(program.m_ast)), + m_dialect{program.m_dialect}, + m_nameDispenser(std::move(program.m_nameDispenser)) + {} + Program operator=(Program&& program) = delete; + + static Program load(langutil::CharStream& _sourceCode); + void optimise(std::vector const& _optimisationSteps); + + size_t codeSize() const { return computeCodeSize(*m_ast); } + yul::Block const& ast() const { return *m_ast; } + + friend std::ostream& operator<<(std::ostream& _stream, Program const& _program); + std::string toJson() const; + +private: + Program( + yul::Dialect const& _dialect, + std::unique_ptr _ast + ): + m_ast(std::move(_ast)), + m_dialect{_dialect}, + m_nameDispenser(_dialect, *m_ast, {}) + {} + + static std::unique_ptr parseSource( + yul::Dialect const& _dialect, + langutil::CharStream _source + ); + static std::unique_ptr analyzeAST( + yul::Dialect const& _dialect, + yul::Block const& _ast + ); + static std::unique_ptr disambiguateAST( + yul::Dialect const& _dialect, + yul::Block const& _ast, + yul::AsmAnalysisInfo const& _analysisInfo + ); + static void applyOptimisationSteps( + yul::Dialect const& _dialect, + yul::NameDispenser& _nameDispenser, + yul::Block& _ast, + std::vector const& _optimisationSteps + ); + static size_t computeCodeSize(yul::Block const& _ast); + + std::unique_ptr m_ast; + yul::Dialect const& m_dialect; + yul::NameDispenser m_nameDispenser; +}; + +} diff --git a/tools/yulPhaser/Random.cpp b/tools/yulPhaser/Random.cpp new file mode 100644 index 000000000..645afb395 --- /dev/null +++ b/tools/yulPhaser/Random.cpp @@ -0,0 +1,44 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#include + +#include +#include +#include + +#include + +using namespace solidity; + +uint32_t phaser::uniformRandomInt(uint32_t _min, uint32_t _max) +{ + // TODO: Seed must be configurable + static boost::random::mt19937 generator(time(0)); + boost::random::uniform_int_distribution<> distribution(_min, _max); + + return distribution(generator); +} + +uint32_t phaser::binomialRandomInt(uint32_t _numTrials, double _successProbability) +{ + // TODO: Seed must be configurable + static boost::random::mt19937 generator(time(0)); + boost::random::binomial_distribution<> distribution(_numTrials, _successProbability); + + return distribution(generator); +} diff --git a/tools/yulPhaser/Random.h b/tools/yulPhaser/Random.h new file mode 100644 index 000000000..25091ddef --- /dev/null +++ b/tools/yulPhaser/Random.h @@ -0,0 +1,28 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#pragma once + +#include + +namespace solidity::phaser +{ + +uint32_t uniformRandomInt(uint32_t _min, uint32_t _max); +uint32_t binomialRandomInt(uint32_t _numTrials, double _successProbability); + +} diff --git a/tools/yulPhaser/main.cpp b/tools/yulPhaser/main.cpp new file mode 100644 index 000000000..6c72bd0f0 --- /dev/null +++ b/tools/yulPhaser/main.cpp @@ -0,0 +1,133 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#include +#include + +#include +#include +#include + +#include +#include + +#include +#include + +using namespace std; +using namespace solidity::langutil; +using namespace solidity::phaser; +using namespace solidity::util; + +namespace po = boost::program_options; + +namespace +{ + +struct CommandLineParsingResult +{ + int exitCode; + po::variables_map arguments; +}; + +CharStream loadSource(string const& _sourcePath) +{ + assertThrow(boost::filesystem::exists(_sourcePath), InvalidProgram, "Source file does not exist"); + + string sourceCode = readFileAsString(_sourcePath); + return CharStream(sourceCode, _sourcePath); +} + +void runAlgorithm(string const& _sourcePath) +{ + auto population = Population::makeRandom(loadSource(_sourcePath), 10); + population.run(nullopt, cout); +} + +CommandLineParsingResult parseCommandLine(int argc, char** argv) +{ + po::options_description description( + "yul-phaser, a tool for finding the best sequence of Yul optimisation phases.\n" + "\n" + "Usage: yul-phaser [options] \n" + "Reads as Yul code and tries to find the best order in which to run optimisation" + " phases using a genetic algorithm.\n" + "Example:\n" + "yul-phaser program.yul\n" + "\n" + "Allowed options", + po::options_description::m_default_line_length, + po::options_description::m_default_line_length - 23 + ); + + description.add_options() + ("help", "Show help message and exit.") + ("input-file", po::value()->required(), "Input file") + ; + + po::positional_options_description positionalDescription; + po::variables_map arguments; + positionalDescription.add("input-file", 1); + po::notify(arguments); + + try + { + po::command_line_parser parser(argc, argv); + parser.options(description).positional(positionalDescription); + po::store(parser.run(), arguments); + } + catch (po::error const & _exception) + { + cerr << _exception.what() << endl; + return {1, move(arguments)}; + } + + if (arguments.count("help") > 0) + { + cout << description << endl; + return {2, move(arguments)}; + } + + if (arguments.count("input-file") == 0) + { + cerr << "Missing argument: input-file." << endl; + return {1, move(arguments)}; + } + + return {0, arguments}; +} + +} + +int main(int argc, char** argv) +{ + CommandLineParsingResult parsingResult = parseCommandLine(argc, argv); + if (parsingResult.exitCode != 0) + return parsingResult.exitCode; + + try + { + runAlgorithm(parsingResult.arguments["input-file"].as()); + } + catch (InvalidProgram const& _exception) + { + cerr << "ERROR: " << _exception.what() << endl; + return 1; + } + + return 0; +}