Merge pull request #8164 from imapp-pl/yul-phaser

[yul-phaser] Minimal application
This commit is contained in:
chriseth 2020-02-05 18:55:55 +01:00 committed by GitHub
commit dd7a5c3386
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 1529 additions and 0 deletions

View File

@ -138,6 +138,22 @@ set(libyul_sources
)
detect_stray_source_files("${libyul_sources}" "libyul/")
set(yul_phaser_sources
yulPhaser/Chromosome.cpp
yulPhaser/Population.cpp
yulPhaser/Program.cpp
yulPhaser/Random.cpp
# FIXME: yul-phaser is not a library so I can't just add it to target_link_libraries().
# My current workaround is just to include its source files here but this introduces
# unnecessary duplication. Create a library or find a way to reuse the list in both places.
../tools/yulPhaser/Chromosome.cpp
../tools/yulPhaser/Population.cpp
../tools/yulPhaser/Program.cpp
../tools/yulPhaser/Random.cpp
)
detect_stray_source_files("${yul_phaser_sources}" "yulPhaser/")
add_executable(soltest ${sources}
${contracts_sources}
${libsolutil_sources}
@ -146,6 +162,7 @@ add_executable(soltest ${sources}
${libyul_sources}
${libsolidity_sources}
${libsolidity_util_sources}
${yul_phaser_sources}
)
target_link_libraries(soltest PRIVATE libsolc yul solidity yulInterpreter evmasm solutil Boost::boost Boost::program_options Boost::unit_test_framework evmc)

View File

@ -0,0 +1,101 @@
/*
This file is part of solidity.
solidity is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
solidity is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with solidity. If not, see <http://www.gnu.org/licenses/>.
*/
#include <tools/yulPhaser/Chromosome.h>
#include <libyul/optimiser/BlockFlattener.h>
#include <libyul/optimiser/StructuralSimplifier.h>
#include <libyul/optimiser/Suite.h>
#include <libyul/optimiser/UnusedPruner.h>
#include <libsolutil/CommonIO.h>
#include <boost/test/unit_test.hpp>
using namespace std;
using namespace solidity::yul;
using namespace solidity::util;
namespace solidity::phaser::test
{
BOOST_AUTO_TEST_SUITE(Phaser)
BOOST_AUTO_TEST_SUITE(ChromosomeTest)
BOOST_AUTO_TEST_CASE(makeRandom_should_create_chromosome_with_random_optimisation_steps)
{
constexpr uint32_t numSteps = 1000;
auto chromosome1 = Chromosome::makeRandom(numSteps);
auto chromosome2 = Chromosome::makeRandom(numSteps);
BOOST_CHECK_EQUAL(chromosome1.length(), numSteps);
BOOST_CHECK_EQUAL(chromosome2.length(), numSteps);
multiset<string> steps1;
multiset<string> steps2;
for (auto const& step: chromosome1.optimisationSteps())
steps1.insert(step);
for (auto const& step: chromosome2.optimisationSteps())
steps2.insert(step);
// Check if steps are different and also if they're not just a permutation of the same set.
// Technically they could be the same and still random but the probability is infinitesimally low.
BOOST_TEST(steps1 != steps2);
}
BOOST_AUTO_TEST_CASE(constructor_should_store_optimisation_steps)
{
vector<string> steps = {
StructuralSimplifier::name,
BlockFlattener::name,
UnusedPruner::name,
};
Chromosome chromosome(steps);
BOOST_TEST(steps == chromosome.optimisationSteps());
}
BOOST_AUTO_TEST_CASE(constructor_should_allow_duplicate_steps)
{
vector<string> steps = {
StructuralSimplifier::name,
StructuralSimplifier::name,
BlockFlattener::name,
UnusedPruner::name,
BlockFlattener::name,
};
Chromosome chromosome(steps);
BOOST_TEST(steps == chromosome.optimisationSteps());
}
BOOST_AUTO_TEST_CASE(output_operator_should_create_concise_and_unambiguous_string_representation)
{
vector<string> allSteps;
for (auto const& step: OptimiserSuite::allSteps())
allSteps.push_back(step.first);
Chromosome chromosome(allSteps);
BOOST_TEST(chromosome.length() == allSteps.size());
BOOST_TEST(chromosome.optimisationSteps() == allSteps);
BOOST_TEST(toString(chromosome) == "fcCUnDvejsxIOoighTLMrmVatud");
}
BOOST_AUTO_TEST_SUITE_END()
BOOST_AUTO_TEST_SUITE_END()
}

View File

@ -0,0 +1,176 @@
/*
This file is part of solidity.
solidity is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
solidity is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with solidity. If not, see <http://www.gnu.org/licenses/>.
*/
#include <tools/yulPhaser/Chromosome.h>
#include <tools/yulPhaser/Population.h>
#include <libyul/optimiser/BlockFlattener.h>
#include <libyul/optimiser/SSAReverser.h>
#include <libyul/optimiser/StructuralSimplifier.h>
#include <libyul/optimiser/UnusedPruner.h>
#include <liblangutil/CharStream.h>
#include <boost/test/unit_test.hpp>
#include <optional>
#include <string>
#include <sstream>
using namespace std;
using namespace solidity::langutil;
using namespace solidity::yul;
using namespace boost::unit_test::framework;
namespace solidity::phaser::test
{
namespace
{
bool fitnessNotSet(Individual const& individual)
{
return !individual.fitness.has_value();
}
bool fitnessSet(Individual const& individual)
{
return individual.fitness.has_value();
}
}
BOOST_AUTO_TEST_SUITE(Phaser)
BOOST_AUTO_TEST_SUITE(PopulationTest)
string const& sampleSourceCode =
"{\n"
" let factor := 13\n"
" {\n"
" if factor\n"
" {\n"
" let variable := add(1, 2)\n"
" }\n"
" let result := factor\n"
" }\n"
" let something := 6\n"
" {\n"
" {\n"
" {\n"
" let value := 15\n"
" }\n"
" }\n"
" }\n"
" let something_else := mul(mul(something, 1), add(factor, 0))\n"
" if 1 { let x := 1 }\n"
" if 0 { let y := 2 }\n"
"}\n";
BOOST_AUTO_TEST_CASE(constructor_should_copy_chromosomes_and_not_compute_fitness)
{
CharStream sourceStream(sampleSourceCode, current_test_case().p_name);
vector<Chromosome> chromosomes = {
Chromosome::makeRandom(5),
Chromosome::makeRandom(10),
};
Population population(sourceStream, chromosomes);
BOOST_TEST(population.individuals().size() == 2);
BOOST_TEST(population.individuals()[0].chromosome == chromosomes[0]);
BOOST_TEST(population.individuals()[1].chromosome == chromosomes[1]);
auto fitnessNotSet = [](auto const& individual){ return !individual.fitness.has_value(); };
BOOST_TEST(all_of(population.individuals().begin(), population.individuals().end(), fitnessNotSet));
}
BOOST_AUTO_TEST_CASE(makeRandom_should_return_population_with_random_chromosomes)
{
CharStream sourceStream(sampleSourceCode, current_test_case().p_name);
auto population1 = Population::makeRandom(sourceStream, 100);
auto population2 = Population::makeRandom(sourceStream, 100);
BOOST_TEST(population1.individuals().size() == 100);
BOOST_TEST(population2.individuals().size() == 100);
int numMatchingPositions = 0;
for (size_t i = 0; i < 100; ++i)
if (population1.individuals()[i].chromosome == population2.individuals()[i].chromosome)
++numMatchingPositions;
// Assume that the results are random if there are no more than 10 identical chromosomes on the
// same positions. One duplicate is very unlikely but still possible after billions of runs
// (especially for short chromosomes). For ten the probability is so small that we can ignore it.
BOOST_TEST(numMatchingPositions < 10);
}
BOOST_AUTO_TEST_CASE(makeRandom_should_not_compute_fitness)
{
CharStream sourceStream(sampleSourceCode, current_test_case().p_name);
auto population = Population::makeRandom(sourceStream, 5);
BOOST_TEST(all_of(population.individuals().begin(), population.individuals().end(), fitnessNotSet));
}
BOOST_AUTO_TEST_CASE(run_should_evaluate_fitness)
{
stringstream output;
CharStream sourceStream(sampleSourceCode, current_test_case().p_name);
auto population = Population::makeRandom(sourceStream, 5);
assert(all_of(population.individuals().begin(), population.individuals().end(), fitnessNotSet));
population.run(1, output);
BOOST_TEST(all_of(population.individuals().begin(), population.individuals().end(), fitnessSet));
}
BOOST_AUTO_TEST_CASE(run_should_not_make_fitness_of_top_chromosomes_worse)
{
stringstream output;
CharStream sourceStream(sampleSourceCode, current_test_case().p_name);
vector<Chromosome> chromosomes = {
Chromosome({StructuralSimplifier::name}),
Chromosome({BlockFlattener::name}),
Chromosome({SSAReverser::name}),
Chromosome({UnusedPruner::name}),
Chromosome({StructuralSimplifier::name, BlockFlattener::name}),
};
Population population(sourceStream, chromosomes);
size_t initialTopFitness[2] = {
Population::measureFitness(chromosomes[0], sourceStream),
Population::measureFitness(chromosomes[1], sourceStream),
};
for (int i = 0; i < 6; ++i)
{
population.run(1, output);
BOOST_TEST(population.individuals().size() == 5);
BOOST_TEST(fitnessSet(population.individuals()[0]));
BOOST_TEST(fitnessSet(population.individuals()[1]));
size_t currentTopFitness[2] = {
population.individuals()[0].fitness.value(),
population.individuals()[1].fitness.value(),
};
BOOST_TEST(currentTopFitness[0] <= initialTopFitness[0]);
BOOST_TEST(currentTopFitness[1] <= initialTopFitness[1]);
BOOST_TEST(currentTopFitness[0] <= currentTopFitness[1]);
}
}
BOOST_AUTO_TEST_SUITE_END()
BOOST_AUTO_TEST_SUITE_END()
}

266
test/yulPhaser/Program.cpp Normal file
View File

@ -0,0 +1,266 @@
/*
This file is part of solidity.
solidity is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
solidity is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with solidity. If not, see <http://www.gnu.org/licenses/>.
*/
#include <tools/yulPhaser/Exceptions.h>
#include <tools/yulPhaser/Program.h>
#include <libyul/optimiser/BlockFlattener.h>
#include <libyul/optimiser/Metrics.h>
#include <libyul/optimiser/StructuralSimplifier.h>
#include <libyul/AsmData.h>
#include <libsolutil/CommonIO.h>
#include <libsolutil/JSON.h>
#include <liblangutil/CharStream.h>
#include <boost/test/unit_test.hpp>
#include <cassert>
#include <regex>
#include <string>
using namespace std;
using namespace solidity::langutil;
using namespace solidity::util;
using namespace solidity::yul;
using namespace boost::unit_test::framework;
namespace
{
/// If the specified block is redundant (i.e. the only thing it contains is another block)
/// the function recurses into it and returns the first non-redundant one it finds.
/// If the block isn't redundant it just returns it immediately.
Block const& skipRedundantBlocks(Block const& _block)
{
if (_block.statements.size() == 1 && holds_alternative<Block>(_block.statements[0]))
return skipRedundantBlocks(get<Block>(_block.statements[0]));
else
return _block;
}
string stripWhitespace(string const& input)
{
regex whitespaceRegex("\\s+");
return regex_replace(input, whitespaceRegex, "");
}
}
namespace solidity::phaser::test
{
BOOST_AUTO_TEST_SUITE(Phaser)
BOOST_AUTO_TEST_SUITE(ProgramTest)
BOOST_AUTO_TEST_CASE(load_should_rewind_the_stream)
{
string sourceCode(
"{\n"
" let x := 1\n"
" let y := 2\n"
"}\n"
);
CharStream sourceStream(sourceCode, current_test_case().p_name);
sourceStream.setPosition(5);
auto program = Program::load(sourceStream);
BOOST_TEST(CodeSize::codeSize(program.ast()) == 2);
}
BOOST_AUTO_TEST_CASE(load_should_disambiguate)
{
string sourceCode(
"{\n"
" {\n"
" let x := 1\n"
" }\n"
" {\n"
" let x := 2\n"
" }\n"
"}\n"
);
CharStream sourceStream(sourceCode, current_test_case().p_name);
auto program = Program::load(sourceStream);
// skipRedundantBlocks() makes the test independent of whether load() includes function grouping or not.
Block const& parentBlock = skipRedundantBlocks(program.ast());
BOOST_TEST(parentBlock.statements.size() == 2);
Block const& innerBlock1 = get<Block>(parentBlock.statements[0]);
Block const& innerBlock2 = get<Block>(parentBlock.statements[1]);
VariableDeclaration const& declaration1 = get<VariableDeclaration>(innerBlock1.statements[0]);
VariableDeclaration const& declaration2 = get<VariableDeclaration>(innerBlock2.statements[0]);
BOOST_TEST(declaration1.variables[0].name.str() == "x");
BOOST_TEST(declaration2.variables[0].name.str() != "x");
}
BOOST_AUTO_TEST_CASE(load_should_do_function_grouping_and_hoisting)
{
string sourceCode(
"{\n"
" function foo() -> result\n"
" {\n"
" result := 1\n"
" }\n"
" let x := 1\n"
" function bar(a) -> result\n"
" {\n"
" result := 2\n"
" }\n"
" let y := 2\n"
"}\n"
);
CharStream sourceStream(sourceCode, current_test_case().p_name);
auto program = Program::load(sourceStream);
BOOST_TEST(program.ast().statements.size() == 3);
BOOST_TEST(holds_alternative<Block>(program.ast().statements[0]));
BOOST_TEST(holds_alternative<FunctionDefinition>(program.ast().statements[1]));
BOOST_TEST(holds_alternative<FunctionDefinition>(program.ast().statements[2]));
}
BOOST_AUTO_TEST_CASE(load_should_do_loop_init_rewriting)
{
string sourceCode(
"{\n"
" for { let i := 0 } true {}\n"
" {\n"
" }\n"
"}\n"
);
CharStream sourceStream(sourceCode, current_test_case().p_name);
auto program = Program::load(sourceStream);
// skipRedundantBlocks() makes the test independent of whether load() includes function grouping or not.
Block const& parentBlock = skipRedundantBlocks(program.ast());
BOOST_TEST(holds_alternative<VariableDeclaration>(parentBlock.statements[0]));
BOOST_TEST(holds_alternative<ForLoop>(parentBlock.statements[1]));
}
BOOST_AUTO_TEST_CASE(load_should_throw_InvalidProgram_if_program_cant_be_parsed)
{
string sourceCode("invalid program\n");
CharStream sourceStream(sourceCode, current_test_case().p_name);
BOOST_CHECK_THROW(Program::load(sourceStream), InvalidProgram);
}
BOOST_AUTO_TEST_CASE(load_should_throw_InvalidProgram_if_program_cant_be_analyzed)
{
// This should be parsed just fine but fail the analysis with:
// Error: Variable not found or variable not lvalue.
string sourceCode(
"{\n"
" x := 1\n"
"}\n"
);
CharStream sourceStream(sourceCode, current_test_case().p_name);
BOOST_CHECK_THROW(Program::load(sourceStream), InvalidProgram);
}
BOOST_AUTO_TEST_CASE(optimise)
{
string sourceCode(
"{\n"
" {\n"
" if 1 { let x := 1 }\n"
" if 0 { let y := 2 }\n"
" }\n"
"}\n"
);
CharStream sourceStream(sourceCode, current_test_case().p_name);
auto program = Program::load(sourceStream);
[[maybe_unused]] Block const& parentBlockBefore = skipRedundantBlocks(program.ast());
assert(parentBlockBefore.statements.size() == 2);
assert(holds_alternative<If>(parentBlockBefore.statements[0]));
assert(holds_alternative<If>(parentBlockBefore.statements[1]));
program.optimise({StructuralSimplifier::name, BlockFlattener::name});
Block const& parentBlockAfter = program.ast();
BOOST_TEST(parentBlockAfter.statements.size() == 1);
BOOST_TEST(holds_alternative<VariableDeclaration>(parentBlockAfter.statements[0]));
}
BOOST_AUTO_TEST_CASE(output_operator)
{
string sourceCode(
"{\n"
" let factor := 13\n"
" {\n"
" if factor\n"
" {\n"
" let variable := add(1, 2)\n"
" }\n"
" let result := factor\n"
" }\n"
" let something := 6\n"
" let something_else := mul(something, factor)\n"
"}\n"
);
CharStream sourceStream(sourceCode, current_test_case().p_name);
auto program = Program::load(sourceStream);
// NOTE: The snippet above was chosen so that the few optimisations applied automatically by load()
// as of now do not change the code significantly. If that changes, you may have to update it.
BOOST_TEST(stripWhitespace(toString(program)) == stripWhitespace("{" + sourceCode + "}"));
}
BOOST_AUTO_TEST_CASE(toJson)
{
string sourceCode(
"{\n"
" let a := 3\n"
" if a\n"
" {\n"
" let abc := add(1, 2)\n"
" }\n"
"}\n"
);
CharStream sourceStream(sourceCode, current_test_case().p_name);
auto program = Program::load(sourceStream);
Json::Value parsingResult;
string errors;
BOOST_TEST(jsonParseStrict(program.toJson(), parsingResult, &errors));
BOOST_TEST(errors.empty());
}
BOOST_AUTO_TEST_CASE(codeSize)
{
string sourceCode(
"{\n"
" function foo() -> result\n"
" {\n"
" result := 15\n"
" }\n"
" let a := 1\n"
"}\n"
);
CharStream sourceStream(sourceCode, current_test_case().p_name);
auto program = Program::load(sourceStream);
BOOST_TEST(program.codeSize() == CodeSize::codeSizeIncludingFunctions(program.ast()));
}
BOOST_AUTO_TEST_SUITE_END()
BOOST_AUTO_TEST_SUITE_END()
}

95
test/yulPhaser/Random.cpp Normal file
View File

@ -0,0 +1,95 @@
/*
This file is part of solidity.
solidity is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
solidity is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with solidity. If not, see <http://www.gnu.org/licenses/>.
*/
#include <tools/yulPhaser/Random.h>
#include <boost/test/unit_test.hpp>
#include <cassert>
using namespace std;
namespace solidity::phaser::test
{
BOOST_AUTO_TEST_SUITE(Phaser)
BOOST_AUTO_TEST_SUITE(RandomTest)
BOOST_AUTO_TEST_CASE(uniformRandomInt_returns_different_values_when_called_multiple_times)
{
constexpr uint32_t numSamples = 1000;
constexpr uint32_t numOutcomes = 100;
vector<uint32_t> samples1;
vector<uint32_t> samples2;
for (uint32_t i = 0; i < numSamples; ++i)
{
samples1.push_back(uniformRandomInt(0, numOutcomes - 1));
samples2.push_back(uniformRandomInt(0, numOutcomes - 1));
}
vector<uint32_t> counts1(numSamples, 0);
vector<uint32_t> counts2(numSamples, 0);
for (uint32_t i = 0; i < numSamples; ++i)
{
++counts1[samples1[i]];
++counts2[samples2[i]];
}
// This test rules out not only the possibility that the two sequences are the same but also
// that they're just different permutations of the same values. The test is probabilistic so
// it's technically possible for it to fail even if generator is good but the probability is
// so low that it would happen on average once very 10^125 billion years if you repeated it
// every second. The chance is much lower than 1 in 1000^100 / 100!.
//
// This does not really guarantee that the generated numbers have the right distribution or
// or that they don't come in long, repeating sequences but the implementation is very simple
// (it just calls a generator from boost) so our goal here is just to make sure it's used
// properly and we're not getting something totally non-random, e.g. the same number every time.
BOOST_TEST(counts1 != counts2);
}
BOOST_AUTO_TEST_CASE(binomialRandomInt_returns_different_values_when_called_multiple_times)
{
constexpr uint32_t numSamples = 1000;
constexpr uint32_t numTrials = 100;
constexpr double successProbability = 0.6;
vector<uint32_t> samples1;
vector<uint32_t> samples2;
for (uint32_t i = 0; i < numSamples; ++i)
{
samples1.push_back(binomialRandomInt(numTrials, successProbability));
samples2.push_back(binomialRandomInt(numTrials, successProbability));
}
vector<uint32_t> counts1(numSamples, 0);
vector<uint32_t> counts2(numSamples, 0);
for (uint32_t i = 0; i < numSamples; ++i)
{
++counts1[samples1[i]];
++counts2[samples2[i]];
}
// See remark for uniformRandomInt() above. Same applies here.
BOOST_TEST(counts1 != counts2);
}
BOOST_AUTO_TEST_SUITE_END()
BOOST_AUTO_TEST_SUITE_END()
}

View File

@ -12,3 +12,18 @@ target_link_libraries(solidity-upgrade PRIVATE solidity Boost::boost Boost::prog
include(GNUInstallDirs)
install(TARGETS solidity-upgrade DESTINATION "${CMAKE_INSTALL_BINDIR}")
add_executable(yul-phaser
yulPhaser/main.cpp
yulPhaser/Population.h
yulPhaser/Population.cpp
yulPhaser/Chromosome.h
yulPhaser/Chromosome.cpp
yulPhaser/Program.h
yulPhaser/Program.cpp
yulPhaser/Random.h
yulPhaser/Random.cpp
)
target_link_libraries(yul-phaser PRIVATE solidity Boost::program_options)
install(TARGETS yul-phaser DESTINATION "${CMAKE_INSTALL_BINDIR}")

View File

@ -0,0 +1,70 @@
/*
This file is part of solidity.
solidity is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
solidity is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with solidity. If not, see <http://www.gnu.org/licenses/>.
*/
#include <tools/yulPhaser/Chromosome.h>
#include <tools/yulPhaser/Random.h>
#include <libyul/optimiser/Suite.h>
#include <libsolutil/CommonData.h>
#include <sstream>
using namespace std;
using namespace solidity;
using namespace solidity::yul;
using namespace solidity::phaser;
namespace solidity::phaser
{
ostream& operator<<(ostream& _stream, Chromosome const& _chromosome);
}
Chromosome Chromosome::makeRandom(size_t _length)
{
vector<string> steps;
for (size_t i = 0; i < _length; ++i)
steps.push_back(randomOptimisationStep());
return Chromosome(move(steps));
}
ostream& phaser::operator<<(ostream& _stream, Chromosome const& _chromosome)
{
for (auto const& stepName: _chromosome.m_optimisationSteps)
_stream << OptimiserSuite::stepNameToAbbreviationMap().at(stepName);
return _stream;
}
vector<string> Chromosome::allStepNames()
{
vector<string> stepNames;
for (auto const& step: OptimiserSuite::allSteps())
stepNames.push_back(step.first);
return stepNames;
}
string const& Chromosome::randomOptimisationStep()
{
static vector<string> stepNames = allStepNames();
return stepNames[uniformRandomInt(0, stepNames.size() - 1)];
}

View File

@ -0,0 +1,62 @@
/*
This file is part of solidity.
solidity is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
solidity is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with solidity. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <map>
#include <ostream>
#include <string>
#include <vector>
namespace solidity::phaser
{
/**
* An object that represents a sequence of optimiser steps that can be applied to a program.
* Such sequences are used in our genetic algorithm to represent individual members of the
* population.
*
* To calculate the fitness of an individual one must apply its sequence to a specific program.
* This class does not provide any means to do so. It just stores information.
*
* Once created a sequence cannot be changed. The only way to mutate it is to generate a new
* chromosome based on the old one.
*/
class Chromosome
{
public:
Chromosome() = default;
explicit Chromosome(std::vector<std::string> _optimisationSteps):
m_optimisationSteps(std::move(_optimisationSteps)) {}
static Chromosome makeRandom(size_t _length);
size_t length() const { return m_optimisationSteps.size(); }
std::vector<std::string> const& optimisationSteps() const { return m_optimisationSteps; }
friend std::ostream& operator<<(std::ostream& _stream, Chromosome const& _chromosome);
bool operator==(Chromosome const& _other) const { return m_optimisationSteps == _other.m_optimisationSteps; }
bool operator!=(Chromosome const& _other) const { return !(*this == _other); }
private:
static std::vector<std::string> allStepNames();
static std::string const& randomOptimisationStep();
std::vector<std::string> m_optimisationSteps;
};
}

View File

@ -0,0 +1,27 @@
/*
This file is part of solidity.
solidity is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
solidity is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with solidity. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <libsolutil/Exceptions.h>
namespace solidity::phaser
{
struct InvalidProgram: virtual util::Exception {};
}

View File

@ -0,0 +1,138 @@
/*
This file is part of solidity.
solidity is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
solidity is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with solidity. If not, see <http://www.gnu.org/licenses/>.
*/
#include <tools/yulPhaser/Population.h>
#include <tools/yulPhaser/Program.h>
#include <algorithm>
#include <cassert>
#include <iostream>
#include <numeric>
using namespace std;
using namespace solidity;
using namespace solidity::langutil;
using namespace solidity::phaser;
namespace solidity::phaser
{
ostream& operator<<(ostream& _stream, Individual const& _individual);
ostream& operator<<(ostream& _stream, Population const& _population);
}
ostream& phaser::operator<<(ostream& _stream, Individual const& _individual)
{
_stream << "Fitness: ";
if (_individual.fitness.has_value())
_stream << _individual.fitness.value();
else
_stream << "<NONE>";
_stream << ", optimisations: " << _individual.chromosome;
return _stream;
}
Population::Population(CharStream _sourceCode, vector<Chromosome> const& _chromosomes):
m_sourceCode{move(_sourceCode)}
{
for (auto const& chromosome: _chromosomes)
m_individuals.push_back({chromosome});
}
Population Population::makeRandom(CharStream _sourceCode, size_t _size)
{
vector<Individual> individuals;
for (size_t i = 0; i < _size; ++i)
individuals.push_back({Chromosome::makeRandom(randomChromosomeLength())});
return Population(move(_sourceCode), individuals);
}
size_t Population::measureFitness(Chromosome const& _chromosome, CharStream& _sourceCode)
{
auto program = Program::load(_sourceCode);
program.optimise(_chromosome.optimisationSteps());
return program.codeSize();
}
void Population::run(optional<size_t> _numRounds, ostream& _outputStream)
{
doEvaluation();
for (size_t round = 0; !_numRounds.has_value() || round < _numRounds.value(); ++round)
{
doMutation();
doSelection();
doEvaluation();
_outputStream << "---------- ROUND " << round << " ----------" << endl;
_outputStream << *this;
}
}
ostream& phaser::operator<<(ostream& _stream, Population const& _population)
{
_stream << "Stream name: " << _population.m_sourceCode.name() << endl;
auto individual = _population.m_individuals.begin();
for (; individual != _population.m_individuals.end(); ++individual)
_stream << *individual << endl;
return _stream;
}
void Population::doMutation()
{
// TODO: Implement mutation and crossover
}
void Population::doEvaluation()
{
for (auto& individual: m_individuals)
if (!individual.fitness.has_value())
individual.fitness = measureFitness(individual.chromosome, m_sourceCode);
}
void Population::doSelection()
{
assert(all_of(m_individuals.begin(), m_individuals.end(), [](auto& i){ return i.fitness.has_value(); }));
sort(
m_individuals.begin(),
m_individuals.end(),
[](auto const& a, auto const& b){ return a.fitness.value() < b.fitness.value(); }
);
randomizeWorstChromosomes(m_individuals, m_individuals.size() / 2);
}
void Population::randomizeWorstChromosomes(
vector<Individual>& _individuals,
size_t _count
)
{
assert(_individuals.size() >= _count);
// ASSUMPTION: _individuals is sorted in ascending order
auto individual = _individuals.begin() + (_individuals.size() - _count);
for (; individual != _individuals.end(); ++individual)
{
*individual = {Chromosome::makeRandom(randomChromosomeLength())};
}
}

View File

@ -0,0 +1,89 @@
/*
This file is part of solidity.
solidity is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
solidity is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with solidity. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <tools/yulPhaser/Chromosome.h>
#include <tools/yulPhaser/Random.h>
#include <liblangutil/CharStream.h>
#include <optional>
#include <ostream>
#include <vector>
namespace solidity::phaser
{
/**
* Information describing the state of an individual member of the population during the course
* of the genetic algorithm.
*/
struct Individual
{
Chromosome chromosome;
std::optional<size_t> fitness = std::nullopt;
friend std::ostream& operator<<(std::ostream& _stream, Individual const& _individual);
};
/**
* Represents a changing set of individuals undergoing a genetic algorithm.
* Each round of the algorithm involves mutating existing individuals, evaluating their fitness
* and selecting the best ones for the next round.
*
* An individual is a sequence of optimiser steps represented by a @a Chromosome instance. The whole
* population is associated with a fixed Yul program. By loading the source code into a @a Program
* instance the class can compute fitness of the individual.
*/
class Population
{
public:
static constexpr size_t MaxChromosomeLength = 30;
explicit Population(langutil::CharStream _sourceCode, std::vector<Chromosome> const& _chromosomes = {});
static Population makeRandom(langutil::CharStream _sourceCode, size_t _size);
void run(std::optional<size_t> _numRounds, std::ostream& _outputStream);
std::vector<Individual> const& individuals() const { return m_individuals; }
static size_t randomChromosomeLength() { return binomialRandomInt(MaxChromosomeLength, 0.5); }
static size_t measureFitness(Chromosome const& _chromosome, langutil::CharStream& _sourceCode);
friend std::ostream& operator<<(std::ostream& _stream, Population const& _population);
private:
explicit Population(langutil::CharStream _sourceCode, std::vector<Individual> _individuals = {}):
m_sourceCode{std::move(_sourceCode)},
m_individuals{std::move(_individuals)} {}
void doMutation();
void doEvaluation();
void doSelection();
static void randomizeWorstChromosomes(
std::vector<Individual>& _individuals,
size_t _count
);
langutil::CharStream m_sourceCode;
std::vector<Individual> m_individuals;
};
}

155
tools/yulPhaser/Program.cpp Normal file
View File

@ -0,0 +1,155 @@
/*
This file is part of solidity.
solidity is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
solidity is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with solidity. If not, see <http://www.gnu.org/licenses/>.
*/
#include <tools/yulPhaser/Program.h>
#include <tools/yulPhaser/Exceptions.h>
#include <liblangutil/CharStream.h>
#include <liblangutil/ErrorReporter.h>
#include <liblangutil/Exceptions.h>
#include <libyul/AsmAnalysis.h>
#include <libyul/AsmAnalysisInfo.h>
#include <libyul/AsmJsonConverter.h>
#include <libyul/AsmParser.h>
#include <libyul/AsmPrinter.h>
#include <libyul/YulString.h>
#include <libyul/backends/evm/EVMDialect.h>
#include <libyul/optimiser/Disambiguator.h>
#include <libyul/optimiser/ForLoopInitRewriter.h>
#include <libyul/optimiser/FunctionGrouper.h>
#include <libyul/optimiser/FunctionHoister.h>
#include <libyul/optimiser/Metrics.h>
#include <libyul/optimiser/OptimiserStep.h>
#include <libyul/optimiser/Suite.h>
#include <libsolutil/JSON.h>
#include <cassert>
#include <memory>
using namespace std;
using namespace solidity;
using namespace solidity::langutil;
using namespace solidity::yul;
using namespace solidity::util;
using namespace solidity::phaser;
namespace solidity::phaser
{
ostream& operator<<(ostream& _stream, Program const& _program);
}
Program Program::load(CharStream& _sourceCode)
{
// ASSUMPTION: parseSource() rewinds the stream on its own
Dialect const& dialect = EVMDialect::strictAssemblyForEVMObjects(EVMVersion{});
unique_ptr<Block> ast = parseSource(dialect, _sourceCode);
unique_ptr<AsmAnalysisInfo> analysisInfo = analyzeAST(dialect, *ast);
Program program(
dialect,
disambiguateAST(dialect, *ast, *analysisInfo)
);
program.optimise({
FunctionHoister::name,
FunctionGrouper::name,
ForLoopInitRewriter::name,
});
return program;
}
void Program::optimise(vector<string> const& _optimisationSteps)
{
applyOptimisationSteps(m_dialect, m_nameDispenser, *m_ast, _optimisationSteps);
}
ostream& phaser::operator<<(ostream& _stream, Program const& _program)
{
return _stream << AsmPrinter()(*_program.m_ast);
}
string Program::toJson() const
{
Json::Value serializedAst = AsmJsonConverter(0)(*m_ast);
return jsonPrettyPrint(serializedAst);
}
unique_ptr<Block> Program::parseSource(Dialect const& _dialect, CharStream _source)
{
ErrorList errors;
ErrorReporter errorReporter(errors);
auto scanner = make_shared<Scanner>(move(_source));
Parser parser(errorReporter, _dialect);
unique_ptr<Block> ast = parser.parse(scanner, false);
assertThrow(ast != nullptr, InvalidProgram, "Error parsing source");
assert(errorReporter.errors().empty());
return ast;
}
unique_ptr<AsmAnalysisInfo> Program::analyzeAST(Dialect const& _dialect, Block const& _ast)
{
ErrorList errors;
ErrorReporter errorReporter(errors);
auto analysisInfo = make_unique<AsmAnalysisInfo>();
AsmAnalyzer analyzer(*analysisInfo, errorReporter, _dialect);
bool analysisSuccessful = analyzer.analyze(_ast);
assertThrow(analysisSuccessful, InvalidProgram, "Error analyzing source");
assert(errorReporter.errors().empty());
return analysisInfo;
}
unique_ptr<Block> Program::disambiguateAST(
Dialect const& _dialect,
Block const& _ast,
AsmAnalysisInfo const& _analysisInfo
)
{
set<YulString> const externallyUsedIdentifiers = {};
Disambiguator disambiguator(_dialect, _analysisInfo, externallyUsedIdentifiers);
return make_unique<Block>(get<Block>(disambiguator(_ast)));
}
void Program::applyOptimisationSteps(
Dialect const& _dialect,
NameDispenser& _nameDispenser,
Block& _ast,
vector<string> const& _optimisationSteps
)
{
// An empty set of reserved identifiers. It could be a constructor parameter but I don't
// think it would be useful in this tool. Other tools (like yulopti) have it empty too.
set<YulString> const externallyUsedIdentifiers = {};
OptimiserStepContext context{_dialect, _nameDispenser, externallyUsedIdentifiers};
for (string const& step: _optimisationSteps)
OptimiserSuite::allSteps().at(step)->run(context, _ast);
}
size_t Program::computeCodeSize(Block const& _ast)
{
return CodeSize::codeSizeIncludingFunctions(_ast);
}

113
tools/yulPhaser/Program.h Normal file
View File

@ -0,0 +1,113 @@
/*
This file is part of solidity.
solidity is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
solidity is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with solidity. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <libyul/optimiser/NameDispenser.h>
#include <libyul/AsmData.h>
#include <boost/noncopyable.hpp>
#include <optional>
#include <ostream>
#include <set>
#include <string>
#include <vector>
namespace solidity::langutil
{
class CharStream;
}
namespace solidity::yul
{
struct AsmAnalysisInfo;
struct Dialect;
}
namespace solidity::phaser
{
/**
* Class representing parsed and analysed Yul program that we can apply optimisations to.
* The program is already disambiguated and has several prerequisite optimiser steps applied to it
* so that the requirements of any possible step that could be later applied by the user are
* already satisfied.
*
* The class allows the user to apply extra optimisations and obtain metrics and general
* information about the resulting syntax tree.
*/
class Program: private boost::noncopyable
{
public:
Program(Program&& program):
m_ast(std::move(program.m_ast)),
m_dialect{program.m_dialect},
m_nameDispenser(std::move(program.m_nameDispenser))
{}
Program operator=(Program&& program) = delete;
static Program load(langutil::CharStream& _sourceCode);
void optimise(std::vector<std::string> const& _optimisationSteps);
size_t codeSize() const { return computeCodeSize(*m_ast); }
yul::Block const& ast() const { return *m_ast; }
friend std::ostream& operator<<(std::ostream& _stream, Program const& _program);
std::string toJson() const;
private:
Program(
yul::Dialect const& _dialect,
std::unique_ptr<yul::Block> _ast
):
m_ast(std::move(_ast)),
m_dialect{_dialect},
m_nameDispenser(_dialect, *m_ast, {})
{}
static std::unique_ptr<yul::Block> parseSource(
yul::Dialect const& _dialect,
langutil::CharStream _source
);
static std::unique_ptr<yul::AsmAnalysisInfo> analyzeAST(
yul::Dialect const& _dialect,
yul::Block const& _ast
);
static std::unique_ptr<yul::Block> disambiguateAST(
yul::Dialect const& _dialect,
yul::Block const& _ast,
yul::AsmAnalysisInfo const& _analysisInfo
);
static void applyOptimisationSteps(
yul::Dialect const& _dialect,
yul::NameDispenser& _nameDispenser,
yul::Block& _ast,
std::vector<std::string> const& _optimisationSteps
);
static size_t computeCodeSize(yul::Block const& _ast);
std::unique_ptr<yul::Block> m_ast;
yul::Dialect const& m_dialect;
yul::NameDispenser m_nameDispenser;
};
}

View File

@ -0,0 +1,44 @@
/*
This file is part of solidity.
solidity is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
solidity is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with solidity. If not, see <http://www.gnu.org/licenses/>.
*/
#include <tools/yulPhaser/Random.h>
#include <boost/random/mersenne_twister.hpp>
#include <boost/random/uniform_int_distribution.hpp>
#include <boost/random/binomial_distribution.hpp>
#include <ctime>
using namespace solidity;
uint32_t phaser::uniformRandomInt(uint32_t _min, uint32_t _max)
{
// TODO: Seed must be configurable
static boost::random::mt19937 generator(time(0));
boost::random::uniform_int_distribution<> distribution(_min, _max);
return distribution(generator);
}
uint32_t phaser::binomialRandomInt(uint32_t _numTrials, double _successProbability)
{
// TODO: Seed must be configurable
static boost::random::mt19937 generator(time(0));
boost::random::binomial_distribution<> distribution(_numTrials, _successProbability);
return distribution(generator);
}

28
tools/yulPhaser/Random.h Normal file
View File

@ -0,0 +1,28 @@
/*
This file is part of solidity.
solidity is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
solidity is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with solidity. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <cstdint>
namespace solidity::phaser
{
uint32_t uniformRandomInt(uint32_t _min, uint32_t _max);
uint32_t binomialRandomInt(uint32_t _numTrials, double _successProbability);
}

133
tools/yulPhaser/main.cpp Normal file
View File

@ -0,0 +1,133 @@
/*
This file is part of solidity.
solidity is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
solidity is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with solidity. If not, see <http://www.gnu.org/licenses/>.
*/
#include <tools/yulPhaser/Exceptions.h>
#include <tools/yulPhaser/Population.h>
#include <libsolutil/Assertions.h>
#include <libsolutil/CommonIO.h>
#include <liblangutil/CharStream.h>
#include <boost/filesystem.hpp>
#include <boost/program_options.hpp>
#include <iostream>
#include <string>
using namespace std;
using namespace solidity::langutil;
using namespace solidity::phaser;
using namespace solidity::util;
namespace po = boost::program_options;
namespace
{
struct CommandLineParsingResult
{
int exitCode;
po::variables_map arguments;
};
CharStream loadSource(string const& _sourcePath)
{
assertThrow(boost::filesystem::exists(_sourcePath), InvalidProgram, "Source file does not exist");
string sourceCode = readFileAsString(_sourcePath);
return CharStream(sourceCode, _sourcePath);
}
void runAlgorithm(string const& _sourcePath)
{
auto population = Population::makeRandom(loadSource(_sourcePath), 10);
population.run(nullopt, cout);
}
CommandLineParsingResult parseCommandLine(int argc, char** argv)
{
po::options_description description(
"yul-phaser, a tool for finding the best sequence of Yul optimisation phases.\n"
"\n"
"Usage: yul-phaser [options] <file>\n"
"Reads <file> as Yul code and tries to find the best order in which to run optimisation"
" phases using a genetic algorithm.\n"
"Example:\n"
"yul-phaser program.yul\n"
"\n"
"Allowed options",
po::options_description::m_default_line_length,
po::options_description::m_default_line_length - 23
);
description.add_options()
("help", "Show help message and exit.")
("input-file", po::value<string>()->required(), "Input file")
;
po::positional_options_description positionalDescription;
po::variables_map arguments;
positionalDescription.add("input-file", 1);
po::notify(arguments);
try
{
po::command_line_parser parser(argc, argv);
parser.options(description).positional(positionalDescription);
po::store(parser.run(), arguments);
}
catch (po::error const & _exception)
{
cerr << _exception.what() << endl;
return {1, move(arguments)};
}
if (arguments.count("help") > 0)
{
cout << description << endl;
return {2, move(arguments)};
}
if (arguments.count("input-file") == 0)
{
cerr << "Missing argument: input-file." << endl;
return {1, move(arguments)};
}
return {0, arguments};
}
}
int main(int argc, char** argv)
{
CommandLineParsingResult parsingResult = parseCommandLine(argc, argv);
if (parsingResult.exitCode != 0)
return parsingResult.exitCode;
try
{
runAlgorithm(parsingResult.arguments["input-file"].as<string>());
}
catch (InvalidProgram const& _exception)
{
cerr << "ERROR: " << _exception.what() << endl;
return 1;
}
return 0;
}