From e8bb212ec67fdbb87b47e0ac858bfb69a666d7aa Mon Sep 17 00:00:00 2001 From: Daniel Kirchner Date: Fri, 14 Feb 2020 21:59:06 +0100 Subject: [PATCH 01/92] Add hour and minute to bytecode repo directory names --- appveyor.yml | 2 +- scripts/bytecodecompare/storebytecode.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 4b66e027e..d7211e14b 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -64,7 +64,7 @@ build_script: - msbuild solidity.sln /p:Configuration=%CONFIGURATION% /m:%NUMBER_OF_PROCESSORS% /v:minimal - cd %APPVEYOR_BUILD_FOLDER% - scripts\release.bat %CONFIGURATION% 2017 - - ps: $bytecodedir = git show -s --format="%cd-%H" --date=short + - ps: $bytecodedir = git show -s --format="%cd-%H" --date="format:%Y-%m-%d-%H-%M" test_script: - cd %APPVEYOR_BUILD_FOLDER%\build\test\%CONFIGURATION% diff --git a/scripts/bytecodecompare/storebytecode.sh b/scripts/bytecodecompare/storebytecode.sh index 451490e0c..f2b3a3897 100755 --- a/scripts/bytecodecompare/storebytecode.sh +++ b/scripts/bytecodecompare/storebytecode.sh @@ -126,7 +126,7 @@ EOF git config user.email "chris@ethereum.org" git clean -f -d -x - DIRNAME=$(cd "$REPO_ROOT" && git show -s --format="%cd-%H" --date=short) + DIRNAME=$(cd "$REPO_ROOT" && git show -s --format="%cd-%H" --date="format:%Y-%m-%d-%H-%M") mkdir -p "$DIRNAME" REPORT="$DIRNAME/$ZIP_SUFFIX.txt" cp ../report.txt "$REPORT" From 9641d93167105d1a3bc26c51c72e49e5392e6616 Mon Sep 17 00:00:00 2001 From: Chris Chinchilla Date: Thu, 5 Dec 2019 16:58:03 +0100 Subject: [PATCH 02/92] Clarify array copying semantics --- docs/control-structures.rst | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/docs/control-structures.rst b/docs/control-structures.rst index 248641e34..407d2523c 100644 --- a/docs/control-structures.rst +++ b/docs/control-structures.rst @@ -343,18 +343,8 @@ i.e. the following is not valid: ``(x, uint y) = (1, 2);`` Complications for Arrays and Structs ------------------------------------ -The semantics of assignments are a bit more complicated for -non-value types like arrays and structs. -Assigning *to* a state variable always creates an independent -copy. On the other hand, assigning to a local variable creates -an independent copy only for elementary types, i.e. static -types that fit into 32 bytes. If structs or arrays (including -``bytes`` and ``string``) are assigned from a state variable -to a local variable, the local variable holds a reference to -the original state variable. A second assignment to the local -variable does not modify the state but only changes the -reference. Assignments to members (or elements) of the local -variable *do* change the state. +The semantics of assignments are more complicated for non-value types like arrays and structs, +including ``bytes`` and ``string``, see :ref:`Data location and assignment behaviour ` for details. In the example below the call to ``g(x)`` has no effect on ``x`` because it creates an independent copy of the storage value in memory. However, ``h(x)`` successfully modifies ``x`` From 9dd9a68c08f3d6aa04388ad797a807aaa30b0dff Mon Sep 17 00:00:00 2001 From: chriseth Date: Tue, 18 Feb 2020 16:42:02 +0100 Subject: [PATCH 03/92] Set version to 0.6.4. --- CMakeLists.txt | 2 +- Changelog.md | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1ab3e3741..4278770f1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,7 +10,7 @@ include(EthPolicy) eth_policy() # project name and version should be set after cmake_policy CMP0048 -set(PROJECT_VERSION "0.6.3") +set(PROJECT_VERSION "0.6.4") project(solidity VERSION ${PROJECT_VERSION} LANGUAGES C CXX) include(TestBigEndian) diff --git a/Changelog.md b/Changelog.md index f00d555e9..c83b26e83 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,15 @@ +### 0.6.4 (unreleased) + +Language Features: + + +Compiler Features: + + +Bugfixes: + + + ### 0.6.3 (2020-02-18) Language Features: @@ -6,7 +18,6 @@ Language Features: * Report source locations for structured documentation errors. - Compiler Features: * AST: Add a new node for doxygen-style, structured documentation that can be received by contract, function, event and modifier definitions. * Code Generator: Use ``calldatacopy`` instead of ``codecopy`` to zero out memory past input. From b3f595ce9d8b83438602b04cd30b314914a58c9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Thu, 6 Feb 2020 07:10:59 +0100 Subject: [PATCH 04/92] [yul-phaser] Population: Remove unused include for --- tools/yulPhaser/Population.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/yulPhaser/Population.cpp b/tools/yulPhaser/Population.cpp index a28db27a4..43158ac43 100644 --- a/tools/yulPhaser/Population.cpp +++ b/tools/yulPhaser/Population.cpp @@ -21,7 +21,6 @@ #include #include -#include #include using namespace std; From 71dcbf9df53c3073a1201fe5b0e8a113b59b191b Mon Sep 17 00:00:00 2001 From: cameel Date: Wed, 5 Feb 2020 14:25:27 +0100 Subject: [PATCH 05/92] [yul-phaser] Population: Remove ambiguous default from one of the constructors --- tools/yulPhaser/Population.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/yulPhaser/Population.h b/tools/yulPhaser/Population.h index 276db83ff..983e40dd8 100644 --- a/tools/yulPhaser/Population.h +++ b/tools/yulPhaser/Population.h @@ -67,7 +67,7 @@ public: friend std::ostream& operator<<(std::ostream& _stream, Population const& _population); private: - explicit Population(Program _program, std::vector _individuals = {}): + explicit Population(Program _program, std::vector _individuals): m_program{std::move(_program)}, m_individuals{std::move(_individuals)} {} From 837ea96da72ec0fd0caf7e43518269e4fab1117c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Sat, 15 Feb 2020 00:09:43 +0100 Subject: [PATCH 06/92] [yul-phaser] Move stripWhitespace() from Program tests to Common --- test/yulPhaser/Common.cpp | 8 ++++++++ test/yulPhaser/Common.h | 5 +++++ test/yulPhaser/CommonTest.cpp | 14 ++++++++++++++ test/yulPhaser/Program.cpp | 9 ++------- 4 files changed, 29 insertions(+), 7 deletions(-) diff --git a/test/yulPhaser/Common.cpp b/test/yulPhaser/Common.cpp index 3cef4ff2b..c3653eaec 100644 --- a/test/yulPhaser/Common.cpp +++ b/test/yulPhaser/Common.cpp @@ -19,6 +19,8 @@ #include +#include + using namespace std; using namespace solidity; using namespace solidity::yul; @@ -32,3 +34,9 @@ map phaser::test::enumerateOptmisationSteps() return stepIndices; } + +string phaser::test::stripWhitespace(string const& input) +{ + regex whitespaceRegex("\\s+"); + return regex_replace(input, whitespaceRegex, ""); +} diff --git a/test/yulPhaser/Common.h b/test/yulPhaser/Common.h index a3dd24bce..3dcf63288 100644 --- a/test/yulPhaser/Common.h +++ b/test/yulPhaser/Common.h @@ -42,6 +42,11 @@ namespace solidity::phaser::test /// integers. std::map enumerateOptmisationSteps(); +// STRING UTILITIES + +/// Returns the input string with all the whitespace characters (spaces, line endings, etc.) removed. +std::string stripWhitespace(std::string const& input); + // STATISTICAL UTILITIES /// Calculates the mean value of a series of samples given in a vector. diff --git a/test/yulPhaser/CommonTest.cpp b/test/yulPhaser/CommonTest.cpp index d7268c669..cdcba6f53 100644 --- a/test/yulPhaser/CommonTest.cpp +++ b/test/yulPhaser/CommonTest.cpp @@ -51,6 +51,20 @@ BOOST_AUTO_TEST_CASE(enumerateOptimisationSteps_should_assing_indices_to_all_ava } } +BOOST_AUTO_TEST_CASE(stripWhitespace_should_remove_all_whitespace_characters_from_a_string) +{ + BOOST_TEST(stripWhitespace("") == ""); + BOOST_TEST(stripWhitespace(" ") == ""); + BOOST_TEST(stripWhitespace(" \n\t\v ") == ""); + + BOOST_TEST(stripWhitespace("abc") == "abc"); + BOOST_TEST(stripWhitespace(" abc") == "abc"); + BOOST_TEST(stripWhitespace("abc ") == "abc"); + BOOST_TEST(stripWhitespace(" a b c ") == "abc"); + BOOST_TEST(stripWhitespace(" a b\tc\n") == "abc"); + BOOST_TEST(stripWhitespace(" a b \n\n c \n\t\v") == "abc"); +} + BOOST_AUTO_TEST_CASE(mean_should_calculate_statistical_mean) { BOOST_TEST(mean({0}) == 0.0); diff --git a/test/yulPhaser/Program.cpp b/test/yulPhaser/Program.cpp index 44c2bfa3c..7d9c81cd3 100644 --- a/test/yulPhaser/Program.cpp +++ b/test/yulPhaser/Program.cpp @@ -15,6 +15,8 @@ along with solidity. If not, see . */ +#include + #include #include @@ -30,7 +32,6 @@ #include #include -#include #include using namespace std; @@ -51,12 +52,6 @@ namespace else return _block; } - - string stripWhitespace(string const& input) - { - regex whitespaceRegex("\\s+"); - return regex_replace(input, whitespaceRegex, ""); - } } namespace solidity::phaser::test From 38f79a17618037af6b0248f48ec928e6e0a2b22a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Fri, 14 Feb 2020 19:41:43 +0100 Subject: [PATCH 07/92] [yul-phaser] Common: Add chromosomeLengths() --- test/yulPhaser/Common.cpp | 9 +++++++++ test/yulPhaser/Common.h | 6 ++++++ test/yulPhaser/CommonTest.cpp | 15 +++++++++++++++ 3 files changed, 30 insertions(+) diff --git a/test/yulPhaser/Common.cpp b/test/yulPhaser/Common.cpp index c3653eaec..38c6e445b 100644 --- a/test/yulPhaser/Common.cpp +++ b/test/yulPhaser/Common.cpp @@ -25,6 +25,15 @@ using namespace std; using namespace solidity; using namespace solidity::yul; +vector phaser::test::chromosomeLengths(Population const& _population) +{ + vector lengths; + for (auto const& individual: _population.individuals()) + lengths.push_back(individual.chromosome.length()); + + return lengths; +} + map phaser::test::enumerateOptmisationSteps() { map stepIndices; diff --git a/test/yulPhaser/Common.h b/test/yulPhaser/Common.h index 3dcf63288..c4da6c99c 100644 --- a/test/yulPhaser/Common.h +++ b/test/yulPhaser/Common.h @@ -28,6 +28,8 @@ #pragma once +#include + #include #include #include @@ -37,6 +39,10 @@ namespace solidity::phaser::test { // CHROMOSOME AND POPULATION HELPERS + +/// Returns a vector containing lengths of all chromosomes in the population (in the same order). +std::vector chromosomeLengths(Population const& _population); + /// Assigns indices from 0 to N to all optimisation steps available in the OptimiserSuite. /// This is a convenience helper to make it easier to test their distribution with tools made for /// integers. diff --git a/test/yulPhaser/CommonTest.cpp b/test/yulPhaser/CommonTest.cpp index cdcba6f53..be686446f 100644 --- a/test/yulPhaser/CommonTest.cpp +++ b/test/yulPhaser/CommonTest.cpp @@ -19,11 +19,14 @@ #include +#include + #include #include using namespace std; +using namespace solidity::langutil; using namespace solidity::yul; using namespace boost::test_tools; @@ -33,6 +36,18 @@ namespace solidity::phaser::test BOOST_AUTO_TEST_SUITE(Phaser) BOOST_AUTO_TEST_SUITE(CommonTest) +BOOST_AUTO_TEST_CASE(chromosomeLengths_should_return_lengths_of_all_chromosomes_in_a_population) +{ + CharStream sourceStream("{}", ""); + auto program = Program::load(sourceStream); + + Population population1(program, {Chromosome(), Chromosome("a"), Chromosome("aa"), Chromosome("aaa")}); + BOOST_TEST((chromosomeLengths(population1) == vector{0, 1, 2, 3})); + + Population population2(program); + BOOST_TEST((chromosomeLengths(population2) == vector{})); +} + BOOST_AUTO_TEST_CASE(enumerateOptimisationSteps_should_assing_indices_to_all_available_optimisation_steps) { map stepsAndAbbreviations = OptimiserSuite::stepNameToAbbreviationMap(); From d22c59aa0e6c3e8dfb46e27802111a61cac8478e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Sat, 8 Feb 2020 01:51:04 +0100 Subject: [PATCH 08/92] [yul-phaser] Chromosome: Add a constructor that reads steps from an abbreviation string --- test/yulPhaser/Chromosome.cpp | 30 +++++++++++++++++++++++++++++- test/yulPhaser/Population.cpp | 10 +++++----- tools/yulPhaser/Chromosome.cpp | 6 ++++++ tools/yulPhaser/Chromosome.h | 1 + 4 files changed, 41 insertions(+), 6 deletions(-) diff --git a/test/yulPhaser/Chromosome.cpp b/test/yulPhaser/Chromosome.cpp index 13f2f6811..5865c501d 100644 --- a/test/yulPhaser/Chromosome.cpp +++ b/test/yulPhaser/Chromosome.cpp @@ -21,8 +21,18 @@ #include #include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include +#include #include #include @@ -39,6 +49,24 @@ namespace solidity::phaser::test BOOST_AUTO_TEST_SUITE(Phaser) BOOST_AUTO_TEST_SUITE(ChromosomeTest) +BOOST_AUTO_TEST_CASE(constructor_should_convert_from_string_to_optimisation_steps) +{ + vector expectedSteps{ + ConditionalSimplifier::name, + FunctionHoister::name, + RedundantAssignEliminator::name, + ForLoopConditionOutOfBody::name, + Rematerialiser::name, + ForLoopConditionOutOfBody::name, + ExpressionSimplifier::name, + ForLoopInitRewriter::name, + LoopInvariantCodeMotion::name, + ExpressionInliner::name + }; + + BOOST_TEST(Chromosome("ChrOmOsoMe").optimisationSteps() == expectedSteps); +} + BOOST_AUTO_TEST_CASE(makeRandom_should_create_chromosome_with_random_optimisation_steps) { constexpr uint32_t numSteps = 1000; diff --git a/test/yulPhaser/Population.cpp b/test/yulPhaser/Population.cpp index d2c11a886..17907333d 100644 --- a/test/yulPhaser/Population.cpp +++ b/test/yulPhaser/Population.cpp @@ -142,11 +142,11 @@ BOOST_AUTO_TEST_CASE(run_should_not_make_fitness_of_top_chromosomes_worse) stringstream output; CharStream sourceStream(sampleSourceCode, current_test_case().p_name); vector chromosomes = { - Chromosome({StructuralSimplifier::name}), - Chromosome({BlockFlattener::name}), - Chromosome({SSAReverser::name}), - Chromosome({UnusedPruner::name}), - Chromosome({StructuralSimplifier::name, BlockFlattener::name}), + Chromosome(vector{StructuralSimplifier::name}), + Chromosome(vector{BlockFlattener::name}), + Chromosome(vector{SSAReverser::name}), + Chromosome(vector{UnusedPruner::name}), + Chromosome(vector{StructuralSimplifier::name, BlockFlattener::name}), }; auto program = Program::load(sourceStream); Population population(program, chromosomes); diff --git a/tools/yulPhaser/Chromosome.cpp b/tools/yulPhaser/Chromosome.cpp index ae30a87e9..832c2cf80 100644 --- a/tools/yulPhaser/Chromosome.cpp +++ b/tools/yulPhaser/Chromosome.cpp @@ -36,6 +36,12 @@ ostream& operator<<(ostream& _stream, Chromosome const& _chromosome); } +Chromosome::Chromosome(string const& _optimisationSteps) +{ + for (char abbreviation: _optimisationSteps) + m_optimisationSteps.push_back(OptimiserSuite::stepAbbreviationToNameMap().at(abbreviation)); +} + Chromosome Chromosome::makeRandom(size_t _length) { vector steps; diff --git a/tools/yulPhaser/Chromosome.h b/tools/yulPhaser/Chromosome.h index bf91dbd30..7d8f1b912 100644 --- a/tools/yulPhaser/Chromosome.h +++ b/tools/yulPhaser/Chromosome.h @@ -42,6 +42,7 @@ public: Chromosome() = default; explicit Chromosome(std::vector _optimisationSteps): m_optimisationSteps(std::move(_optimisationSteps)) {} + explicit Chromosome(std::string const& _optimisationSteps); static Chromosome makeRandom(size_t _length); size_t length() const { return m_optimisationSteps.size(); } From e771f00971bf61a170be677c0f46b9dc78933dc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Thu, 13 Feb 2020 22:25:25 +0100 Subject: [PATCH 09/92] [yul-phaser] Population: Extract Program construction in tests into a fixture --- test/yulPhaser/Population.cpp | 90 ++++++++++++++++++----------------- 1 file changed, 47 insertions(+), 43 deletions(-) diff --git a/test/yulPhaser/Population.cpp b/test/yulPhaser/Population.cpp index 17907333d..15297f7d7 100644 --- a/test/yulPhaser/Population.cpp +++ b/test/yulPhaser/Population.cpp @@ -53,40 +53,50 @@ namespace } } +class PopulationFixture +{ +protected: + PopulationFixture(): + m_sourceStream(SampleSourceCode, ""), + m_program(Program::load(m_sourceStream)) {} + + static constexpr char SampleSourceCode[] = + "{\n" + " let factor := 13\n" + " {\n" + " if factor\n" + " {\n" + " let variable := add(1, 2)\n" + " }\n" + " let result := factor\n" + " }\n" + " let something := 6\n" + " {\n" + " {\n" + " {\n" + " let value := 15\n" + " }\n" + " }\n" + " }\n" + " let something_else := mul(mul(something, 1), add(factor, 0))\n" + " if 1 { let x := 1 }\n" + " if 0 { let y := 2 }\n" + "}\n"; + + CharStream m_sourceStream; + Program m_program; +}; + BOOST_AUTO_TEST_SUITE(Phaser) BOOST_AUTO_TEST_SUITE(PopulationTest) -string const& sampleSourceCode = - "{\n" - " let factor := 13\n" - " {\n" - " if factor\n" - " {\n" - " let variable := add(1, 2)\n" - " }\n" - " let result := factor\n" - " }\n" - " let something := 6\n" - " {\n" - " {\n" - " {\n" - " let value := 15\n" - " }\n" - " }\n" - " }\n" - " let something_else := mul(mul(something, 1), add(factor, 0))\n" - " if 1 { let x := 1 }\n" - " if 0 { let y := 2 }\n" - "}\n"; - -BOOST_AUTO_TEST_CASE(constructor_should_copy_chromosomes_and_not_compute_fitness) +BOOST_FIXTURE_TEST_CASE(constructor_should_copy_chromosomes_and_not_compute_fitness, PopulationFixture) { - CharStream sourceStream(sampleSourceCode, current_test_case().p_name); vector chromosomes = { Chromosome::makeRandom(5), Chromosome::makeRandom(10), }; - Population population(Program::load(sourceStream), chromosomes); + Population population(m_program, chromosomes); BOOST_TEST(population.individuals().size() == 2); BOOST_TEST(population.individuals()[0].chromosome == chromosomes[0]); @@ -96,12 +106,10 @@ BOOST_AUTO_TEST_CASE(constructor_should_copy_chromosomes_and_not_compute_fitness BOOST_TEST(all_of(population.individuals().begin(), population.individuals().end(), fitnessNotSet)); } -BOOST_AUTO_TEST_CASE(makeRandom_should_return_population_with_random_chromosomes) +BOOST_FIXTURE_TEST_CASE(makeRandom_should_return_population_with_random_chromosomes, PopulationFixture) { - CharStream sourceStream(sampleSourceCode, current_test_case().p_name); - auto program = Program::load(sourceStream); - auto population1 = Population::makeRandom(program, 100); - auto population2 = Population::makeRandom(program, 100); + auto population1 = Population::makeRandom(m_program, 100); + auto population2 = Population::makeRandom(m_program, 100); BOOST_TEST(population1.individuals().size() == 100); BOOST_TEST(population2.individuals().size() == 100); @@ -117,19 +125,17 @@ BOOST_AUTO_TEST_CASE(makeRandom_should_return_population_with_random_chromosomes BOOST_TEST(numMatchingPositions < 10); } -BOOST_AUTO_TEST_CASE(makeRandom_should_not_compute_fitness) +BOOST_FIXTURE_TEST_CASE(makeRandom_should_not_compute_fitness, PopulationFixture) { - CharStream sourceStream(sampleSourceCode, current_test_case().p_name); - auto population = Population::makeRandom(Program::load(sourceStream), 5); + auto population = Population::makeRandom(m_program, 5); BOOST_TEST(all_of(population.individuals().begin(), population.individuals().end(), fitnessNotSet)); } -BOOST_AUTO_TEST_CASE(run_should_evaluate_fitness) +BOOST_FIXTURE_TEST_CASE(run_should_evaluate_fitness, PopulationFixture) { stringstream output; - CharStream sourceStream(sampleSourceCode, current_test_case().p_name); - auto population = Population::makeRandom(Program::load(sourceStream), 5); + auto population = Population::makeRandom(m_program, 5); assert(all_of(population.individuals().begin(), population.individuals().end(), fitnessNotSet)); population.run(1, output); @@ -137,10 +143,9 @@ BOOST_AUTO_TEST_CASE(run_should_evaluate_fitness) BOOST_TEST(all_of(population.individuals().begin(), population.individuals().end(), fitnessSet)); } -BOOST_AUTO_TEST_CASE(run_should_not_make_fitness_of_top_chromosomes_worse) +BOOST_FIXTURE_TEST_CASE(run_should_not_make_fitness_of_top_chromosomes_worse, PopulationFixture) { stringstream output; - CharStream sourceStream(sampleSourceCode, current_test_case().p_name); vector chromosomes = { Chromosome(vector{StructuralSimplifier::name}), Chromosome(vector{BlockFlattener::name}), @@ -148,12 +153,11 @@ BOOST_AUTO_TEST_CASE(run_should_not_make_fitness_of_top_chromosomes_worse) Chromosome(vector{UnusedPruner::name}), Chromosome(vector{StructuralSimplifier::name, BlockFlattener::name}), }; - auto program = Program::load(sourceStream); - Population population(program, chromosomes); + Population population(m_program, chromosomes); size_t initialTopFitness[2] = { - Population::measureFitness(chromosomes[0], program), - Population::measureFitness(chromosomes[1], program), + Population::measureFitness(chromosomes[0], m_program), + Population::measureFitness(chromosomes[1], m_program), }; for (int i = 0; i < 6; ++i) From 806891f494ed447d71fc781923b4c98fc6640f61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Thu, 6 Feb 2020 06:19:55 +0100 Subject: [PATCH 10/92] [yul-phaser] Population: Customizable chromosome length in makeRandom() --- test/yulPhaser/Population.cpp | 59 +++++++++++++++++++++++++++++++--- tools/yulPhaser/Population.cpp | 24 ++++++++++++-- tools/yulPhaser/Population.h | 16 +++++++-- tools/yulPhaser/main.cpp | 7 +++- 4 files changed, 96 insertions(+), 10 deletions(-) diff --git a/test/yulPhaser/Population.cpp b/test/yulPhaser/Population.cpp index 15297f7d7..15bb48d04 100644 --- a/test/yulPhaser/Population.cpp +++ b/test/yulPhaser/Population.cpp @@ -15,6 +15,8 @@ along with solidity. If not, see . */ +#include + #include #include #include @@ -106,10 +108,59 @@ BOOST_FIXTURE_TEST_CASE(constructor_should_copy_chromosomes_and_not_compute_fitn BOOST_TEST(all_of(population.individuals().begin(), population.individuals().end(), fitnessNotSet)); } +BOOST_FIXTURE_TEST_CASE(makeRandom_should_get_chromosome_lengths_from_specified_generator, PopulationFixture) +{ + size_t chromosomeCount = 30; + size_t maxLength = 5; + assert(chromosomeCount % maxLength == 0); + + auto nextLength = [counter = 0, maxLength]() mutable { return counter++ % maxLength; }; + auto population = Population::makeRandom(m_program, chromosomeCount, nextLength); + + // We can't rely on the order since the population sorts its chromosomes immediately but + // we can check the number of occurrences of each length. + for (size_t length = 0; length < maxLength; ++length) + BOOST_TEST( + count_if( + population.individuals().begin(), + population.individuals().end(), + [&length](auto const& individual) { return individual.chromosome.length() == length; } + ) == chromosomeCount / maxLength + ); +} + +BOOST_FIXTURE_TEST_CASE(makeRandom_should_get_chromosome_lengths_from_specified_range, PopulationFixture) +{ + auto population = Population::makeRandom(m_program, 100, 5, 10); + BOOST_TEST(all_of( + population.individuals().begin(), + population.individuals().end(), + [](auto const& individual){ return 5 <= individual.chromosome.length() && individual.chromosome.length() <= 10; } + )); +} + +BOOST_FIXTURE_TEST_CASE(makeRandom_should_use_random_chromosome_length, PopulationFixture) +{ + SimulationRNG::reset(1); + constexpr int populationSize = 200; + constexpr int minLength = 5; + constexpr int maxLength = 10; + constexpr double relativeTolerance = 0.05; + + auto population = Population::makeRandom(m_program, populationSize, minLength, maxLength); + vector samples = chromosomeLengths(population); + + const double expectedValue = (maxLength + minLength) / 2.0; + const double variance = ((maxLength - minLength + 1) * (maxLength - minLength + 1) - 1) / 12.0; + + BOOST_TEST(abs(mean(samples) - expectedValue) < expectedValue * relativeTolerance); + BOOST_TEST(abs(meanSquaredError(samples, expectedValue) - variance) < variance * relativeTolerance); +} + BOOST_FIXTURE_TEST_CASE(makeRandom_should_return_population_with_random_chromosomes, PopulationFixture) { - auto population1 = Population::makeRandom(m_program, 100); - auto population2 = Population::makeRandom(m_program, 100); + auto population1 = Population::makeRandom(m_program, 100, 30, 30); + auto population2 = Population::makeRandom(m_program, 100, 30, 30); BOOST_TEST(population1.individuals().size() == 100); BOOST_TEST(population2.individuals().size() == 100); @@ -127,7 +178,7 @@ BOOST_FIXTURE_TEST_CASE(makeRandom_should_return_population_with_random_chromoso BOOST_FIXTURE_TEST_CASE(makeRandom_should_not_compute_fitness, PopulationFixture) { - auto population = Population::makeRandom(m_program, 5); + auto population = Population::makeRandom(m_program, 3, 5, 10); BOOST_TEST(all_of(population.individuals().begin(), population.individuals().end(), fitnessNotSet)); } @@ -135,7 +186,7 @@ BOOST_FIXTURE_TEST_CASE(makeRandom_should_not_compute_fitness, PopulationFixture BOOST_FIXTURE_TEST_CASE(run_should_evaluate_fitness, PopulationFixture) { stringstream output; - auto population = Population::makeRandom(m_program, 5); + auto population = Population::makeRandom(m_program, 5, 5, 10); assert(all_of(population.individuals().begin(), population.individuals().end(), fitnessNotSet)); population.run(1, output); diff --git a/tools/yulPhaser/Population.cpp b/tools/yulPhaser/Population.cpp index 43158ac43..c51c3cb0c 100644 --- a/tools/yulPhaser/Population.cpp +++ b/tools/yulPhaser/Population.cpp @@ -55,15 +55,33 @@ Population::Population(Program _program, vector const& _chromosomes) m_individuals.push_back({chromosome}); } -Population Population::makeRandom(Program _program, size_t _size) +Population Population::makeRandom( + Program _program, + size_t _size, + function _chromosomeLengthGenerator +) { vector individuals; for (size_t i = 0; i < _size; ++i) - individuals.push_back({Chromosome::makeRandom(randomChromosomeLength())}); + individuals.push_back({Chromosome::makeRandom(_chromosomeLengthGenerator())}); return Population(move(_program), individuals); } +Population Population::makeRandom( + Program _program, + size_t _size, + size_t _minChromosomeLength, + size_t _maxChromosomeLength +) +{ + return makeRandom( + move(_program), + _size, + std::bind(uniformChromosomeLength, _minChromosomeLength, _maxChromosomeLength) + ); +} + size_t Population::measureFitness(Chromosome const& _chromosome, Program const& _program) { Program programCopy = _program; @@ -130,6 +148,6 @@ void Population::randomizeWorstChromosomes( auto individual = _individuals.begin() + (_individuals.size() - _count); for (; individual != _individuals.end(); ++individual) { - *individual = {Chromosome::makeRandom(randomChromosomeLength())}; + *individual = {Chromosome::makeRandom(binomialChromosomeLength(MaxChromosomeLength))}; } } diff --git a/tools/yulPhaser/Population.h b/tools/yulPhaser/Population.h index 983e40dd8..c06e0c42a 100644 --- a/tools/yulPhaser/Population.h +++ b/tools/yulPhaser/Population.h @@ -55,13 +55,25 @@ public: static constexpr size_t MaxChromosomeLength = 30; explicit Population(Program _program, std::vector const& _chromosomes = {}); - static Population makeRandom(Program _program, size_t _size); + + static Population makeRandom( + Program _program, + size_t _size, + std::function _chromosomeLengthGenerator + ); + static Population makeRandom( + Program _program, + size_t _size, + size_t _minChromosomeLength, + size_t _maxChromosomeLength + ); void run(std::optional _numRounds, std::ostream& _outputStream); std::vector const& individuals() const { return m_individuals; } - static size_t randomChromosomeLength() { return SimulationRNG::binomialInt(MaxChromosomeLength, 0.5); } + static size_t uniformChromosomeLength(size_t _min, size_t _max) { return SimulationRNG::uniformInt(_min, _max); } + static size_t binomialChromosomeLength(size_t _max) { return SimulationRNG::binomialInt(_max, 0.5); } static size_t measureFitness(Chromosome const& _chromosome, Program const& _program); friend std::ostream& operator<<(std::ostream& _stream, Population const& _population); diff --git a/tools/yulPhaser/main.cpp b/tools/yulPhaser/main.cpp index b5c522ce7..ed846dc4f 100644 --- a/tools/yulPhaser/main.cpp +++ b/tools/yulPhaser/main.cpp @@ -28,6 +28,7 @@ #include #include +#include #include using namespace std; @@ -70,7 +71,11 @@ CharStream loadSource(string const& _sourcePath) void runAlgorithm(string const& _sourcePath) { CharStream sourceCode = loadSource(_sourcePath); - auto population = Population::makeRandom(Program::load(sourceCode), 10); + auto population = Population::makeRandom( + Program::load(sourceCode), + 10, + bind(Population::binomialChromosomeLength, Population::MaxChromosomeLength) + ); population.run(nullopt, cout); } From 823e715902d6167231f30716aeaaaafa97593807 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Fri, 14 Feb 2020 06:07:28 +0100 Subject: [PATCH 11/92] [yul-phaser] Population+Chromosome: Better tests for makeRandom() --- test/yulPhaser/Chromosome.cpp | 36 ++++++++++++++++++++--------------- test/yulPhaser/Population.cpp | 27 ++++++++++++++------------ 2 files changed, 36 insertions(+), 27 deletions(-) diff --git a/test/yulPhaser/Chromosome.cpp b/test/yulPhaser/Chromosome.cpp index 5865c501d..23de34a2d 100644 --- a/test/yulPhaser/Chromosome.cpp +++ b/test/yulPhaser/Chromosome.cpp @@ -67,25 +67,31 @@ BOOST_AUTO_TEST_CASE(constructor_should_convert_from_string_to_optimisation_step BOOST_TEST(Chromosome("ChrOmOsoMe").optimisationSteps() == expectedSteps); } -BOOST_AUTO_TEST_CASE(makeRandom_should_create_chromosome_with_random_optimisation_steps) +BOOST_AUTO_TEST_CASE(makeRandom_should_return_different_chromosome_each_time) { - constexpr uint32_t numSteps = 1000; + SimulationRNG::reset(1); + for (size_t i = 0; i < 10; ++i) + BOOST_TEST(Chromosome::makeRandom(100) != Chromosome::makeRandom(100)); +} - auto chromosome1 = Chromosome::makeRandom(numSteps); - auto chromosome2 = Chromosome::makeRandom(numSteps); - BOOST_CHECK_EQUAL(chromosome1.length(), numSteps); - BOOST_CHECK_EQUAL(chromosome2.length(), numSteps); +BOOST_AUTO_TEST_CASE(makeRandom_should_use_every_possible_step_with_the_same_probability) +{ + SimulationRNG::reset(1); + constexpr int samplesPerStep = 100; + constexpr double relativeTolerance = 0.01; - multiset steps1; - multiset steps2; - for (auto const& step: chromosome1.optimisationSteps()) - steps1.insert(step); - for (auto const& step: chromosome2.optimisationSteps()) - steps2.insert(step); + map stepIndices = enumerateOptmisationSteps(); + auto chromosome = Chromosome::makeRandom(stepIndices.size() * samplesPerStep); - // Check if steps are different and also if they're not just a permutation of the same set. - // Technically they could be the same and still random but the probability is infinitesimally low. - BOOST_TEST(steps1 != steps2); + vector samples; + for (auto& step: chromosome.optimisationSteps()) + samples.push_back(stepIndices.at(step)); + + const double expectedValue = (stepIndices.size() - 1) / 2.0; + const double variance = (stepIndices.size() * stepIndices.size() - 1) / 12.0; + + BOOST_TEST(abs(mean(samples) - expectedValue) < expectedValue * relativeTolerance); + BOOST_TEST(abs(meanSquaredError(samples, expectedValue) - variance) < variance * relativeTolerance); } BOOST_AUTO_TEST_CASE(constructor_should_store_optimisation_steps) diff --git a/test/yulPhaser/Population.cpp b/test/yulPhaser/Population.cpp index 15bb48d04..d241117de 100644 --- a/test/yulPhaser/Population.cpp +++ b/test/yulPhaser/Population.cpp @@ -159,21 +159,24 @@ BOOST_FIXTURE_TEST_CASE(makeRandom_should_use_random_chromosome_length, Populati BOOST_FIXTURE_TEST_CASE(makeRandom_should_return_population_with_random_chromosomes, PopulationFixture) { - auto population1 = Population::makeRandom(m_program, 100, 30, 30); - auto population2 = Population::makeRandom(m_program, 100, 30, 30); + SimulationRNG::reset(1); + constexpr int populationSize = 100; + constexpr int chromosomeLength = 30; + constexpr double relativeTolerance = 0.01; - BOOST_TEST(population1.individuals().size() == 100); - BOOST_TEST(population2.individuals().size() == 100); + map stepIndices = enumerateOptmisationSteps(); + auto population = Population::makeRandom(m_program, populationSize, chromosomeLength, chromosomeLength); - int numMatchingPositions = 0; - for (size_t i = 0; i < 100; ++i) - if (population1.individuals()[i].chromosome == population2.individuals()[i].chromosome) - ++numMatchingPositions; + vector samples; + for (auto& individual: population.individuals()) + for (auto& step: individual.chromosome.optimisationSteps()) + samples.push_back(stepIndices.at(step)); - // Assume that the results are random if there are no more than 10 identical chromosomes on the - // same positions. One duplicate is very unlikely but still possible after billions of runs - // (especially for short chromosomes). For ten the probability is so small that we can ignore it. - BOOST_TEST(numMatchingPositions < 10); + const double expectedValue = (stepIndices.size() - 1) / 2.0; + const double variance = (stepIndices.size() * stepIndices.size() - 1) / 12.0; + + BOOST_TEST(abs(mean(samples) - expectedValue) < expectedValue * relativeTolerance); + BOOST_TEST(abs(meanSquaredError(samples, expectedValue) - variance) < variance * relativeTolerance); } BOOST_FIXTURE_TEST_CASE(makeRandom_should_not_compute_fitness, PopulationFixture) From 40a666953878957dd746ab43835bf8e246173346 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Tue, 11 Feb 2020 19:20:01 +0100 Subject: [PATCH 12/92] [yul-phaser] Population: Extract a function for comparing fitness of individuals - Mostly for readability and convenience. This significantly shortens calls to sort(). - I could define it as Individual::operator< instead but it would be inconsistent with operator== because it does not compare the chromosomes, only fitness. It could result in an unintuitive situation where (a <= b <= a) does not necessarily imply (a == b). --- test/yulPhaser/Population.cpp | 18 ++++++++++++++++++ tools/yulPhaser/Population.cpp | 14 ++++++++------ tools/yulPhaser/Population.h | 3 +++ 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/test/yulPhaser/Population.cpp b/test/yulPhaser/Population.cpp index d241117de..695d6b397 100644 --- a/test/yulPhaser/Population.cpp +++ b/test/yulPhaser/Population.cpp @@ -92,6 +92,24 @@ protected: BOOST_AUTO_TEST_SUITE(Phaser) BOOST_AUTO_TEST_SUITE(PopulationTest) +BOOST_AUTO_TEST_CASE(isFitter_should_use_fitness_as_the_main_criterion) +{ + BOOST_TEST(isFitter(Individual{Chromosome("a"), 5}, Individual{Chromosome("a"), 10})); + BOOST_TEST(!isFitter(Individual{Chromosome("a"), 10}, Individual{Chromosome("a"), 5})); + + BOOST_TEST(isFitter(Individual{Chromosome("aaa"), 5}, Individual{Chromosome("aaaaa"), 10})); + BOOST_TEST(!isFitter(Individual{Chromosome("aaaaa"), 10}, Individual{Chromosome("aaa"), 5})); + + BOOST_TEST(isFitter(Individual{Chromosome("aaaaa"), 5}, Individual{Chromosome("aaa"), 10})); + BOOST_TEST(!isFitter(Individual{Chromosome("aaa"), 10}, Individual{Chromosome("aaaaa"), 5})); +} + +BOOST_AUTO_TEST_CASE(isFitter_should_return_false_for_identical_individuals) +{ + BOOST_TEST(!isFitter(Individual{Chromosome("a"), 3}, Individual{Chromosome("a"), 3})); + BOOST_TEST(!isFitter(Individual{Chromosome("acT"), 0}, Individual{Chromosome("acT"), 0})); +} + BOOST_FIXTURE_TEST_CASE(constructor_should_copy_chromosomes_and_not_compute_fitness, PopulationFixture) { vector chromosomes = { diff --git a/tools/yulPhaser/Population.cpp b/tools/yulPhaser/Population.cpp index c51c3cb0c..c4b529a91 100644 --- a/tools/yulPhaser/Population.cpp +++ b/tools/yulPhaser/Population.cpp @@ -48,6 +48,13 @@ ostream& phaser::operator<<(ostream& _stream, Individual const& _individual) return _stream; } +bool phaser::isFitter(Individual const& a, Individual const& b) +{ + assert(a.fitness.has_value() && b.fitness.has_value()); + + return a.fitness.value() < b.fitness.value(); +} + Population::Population(Program _program, vector const& _chromosomes): m_program{move(_program)} { @@ -128,12 +135,7 @@ void Population::doSelection() { assert(all_of(m_individuals.begin(), m_individuals.end(), [](auto& i){ return i.fitness.has_value(); })); - sort( - m_individuals.begin(), - m_individuals.end(), - [](auto const& a, auto const& b){ return a.fitness.value() < b.fitness.value(); } - ); - + sort(m_individuals.begin(), m_individuals.end(), isFitter); randomizeWorstChromosomes(m_individuals, m_individuals.size() / 2); } diff --git a/tools/yulPhaser/Population.h b/tools/yulPhaser/Population.h index c06e0c42a..0c6c69be7 100644 --- a/tools/yulPhaser/Population.h +++ b/tools/yulPhaser/Population.h @@ -40,6 +40,9 @@ struct Individual friend std::ostream& operator<<(std::ostream& _stream, Individual const& _individual); }; +/// Determines which individual is better by comparing fitness values. +bool isFitter(Individual const& a, Individual const& b); + /** * Represents a changing set of individuals undergoing a genetic algorithm. * Each round of the algorithm involves mutating existing individuals, evaluating their fitness From ecb30c670fef1a6eeace327336d91d214b79ac07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Tue, 11 Feb 2020 22:25:17 +0100 Subject: [PATCH 13/92] [yul-phaser] Population: Make ordering of individuals with same fitness deterministic and prioritise shorter chromosomes - Before this change the order of chromosomes with the same fitness in a population depended on the initial order set when the population was first created. Now it only depends on the individual. - The length comparison is not strictly necessary (lexicographical order covers that) but it makes the intention clear and the comparison slightly faster when chromosomes have different lengths. --- test/yulPhaser/Population.cpp | 12 ++++++++++++ tools/yulPhaser/Population.cpp | 9 ++++++++- tools/yulPhaser/Population.h | 4 +++- 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/test/yulPhaser/Population.cpp b/test/yulPhaser/Population.cpp index 695d6b397..bb2f4cc05 100644 --- a/test/yulPhaser/Population.cpp +++ b/test/yulPhaser/Population.cpp @@ -104,6 +104,18 @@ BOOST_AUTO_TEST_CASE(isFitter_should_use_fitness_as_the_main_criterion) BOOST_TEST(!isFitter(Individual{Chromosome("aaa"), 10}, Individual{Chromosome("aaaaa"), 5})); } +BOOST_AUTO_TEST_CASE(isFitter_should_use_alphabetical_order_when_fitness_is_the_same) +{ + BOOST_TEST(isFitter(Individual{Chromosome("a"), 3}, Individual{Chromosome("c"), 3})); + BOOST_TEST(!isFitter(Individual{Chromosome("c"), 3}, Individual{Chromosome("a"), 3})); + + BOOST_TEST(isFitter(Individual{Chromosome("a"), 3}, Individual{Chromosome("aa"), 3})); + BOOST_TEST(!isFitter(Individual{Chromosome("aa"), 3}, Individual{Chromosome("a"), 3})); + + BOOST_TEST(isFitter(Individual{Chromosome("T"), 3}, Individual{Chromosome("a"), 3})); + BOOST_TEST(!isFitter(Individual{Chromosome("a"), 3}, Individual{Chromosome("T"), 3})); +} + BOOST_AUTO_TEST_CASE(isFitter_should_return_false_for_identical_individuals) { BOOST_TEST(!isFitter(Individual{Chromosome("a"), 3}, Individual{Chromosome("a"), 3})); diff --git a/tools/yulPhaser/Population.cpp b/tools/yulPhaser/Population.cpp index c4b529a91..890e13b43 100644 --- a/tools/yulPhaser/Population.cpp +++ b/tools/yulPhaser/Population.cpp @@ -19,6 +19,8 @@ #include +#include + #include #include #include @@ -26,6 +28,7 @@ using namespace std; using namespace solidity; using namespace solidity::langutil; +using namespace solidity::util; using namespace solidity::phaser; namespace solidity::phaser @@ -52,7 +55,11 @@ bool phaser::isFitter(Individual const& a, Individual const& b) { assert(a.fitness.has_value() && b.fitness.has_value()); - return a.fitness.value() < b.fitness.value(); + return ( + (a.fitness.value() < b.fitness.value()) || + (a.fitness.value() == b.fitness.value() && a.chromosome.length() < b.chromosome.length()) || + (a.fitness.value() == b.fitness.value() && a.chromosome.length() == b.chromosome.length() && toString(a.chromosome) < toString(b.chromosome)) + ); } Population::Population(Program _program, vector const& _chromosomes): diff --git a/tools/yulPhaser/Population.h b/tools/yulPhaser/Population.h index 0c6c69be7..43f8643e1 100644 --- a/tools/yulPhaser/Population.h +++ b/tools/yulPhaser/Population.h @@ -40,7 +40,9 @@ struct Individual friend std::ostream& operator<<(std::ostream& _stream, Individual const& _individual); }; -/// Determines which individual is better by comparing fitness values. +/// Determines which individual is better by comparing fitness values. If fitness is the same +/// takes into account all the other properties of the individual to make the comparison +/// deterministic as long as the individuals are not equal. bool isFitter(Individual const& a, Individual const& b); /** From d9c5e2dc9f8b967430d75c0c276ca94d4c98535d Mon Sep 17 00:00:00 2001 From: cameel Date: Wed, 5 Feb 2020 16:51:43 +0100 Subject: [PATCH 14/92] [yul-phaser] Population: Add operator+() --- test/yulPhaser/Population.cpp | 9 +++++++++ tools/yulPhaser/Population.cpp | 8 ++++++++ tools/yulPhaser/Population.h | 12 ++++++++++++ 3 files changed, 29 insertions(+) diff --git a/test/yulPhaser/Population.cpp b/test/yulPhaser/Population.cpp index bb2f4cc05..be90382c7 100644 --- a/test/yulPhaser/Population.cpp +++ b/test/yulPhaser/Population.cpp @@ -261,6 +261,15 @@ BOOST_FIXTURE_TEST_CASE(run_should_not_make_fitness_of_top_chromosomes_worse, Po } } +BOOST_FIXTURE_TEST_CASE(plus_operator_should_add_two_populations, PopulationFixture) +{ + BOOST_CHECK_EQUAL( + Population(m_program, {Chromosome("ac"), Chromosome("cx")}) + + Population(m_program, {Chromosome("g"), Chromosome("h"), Chromosome("iI")}), + Population(m_program, {Chromosome("ac"), Chromosome("cx"), Chromosome("g"), Chromosome("h"), Chromosome("iI")}) + ); +} + BOOST_AUTO_TEST_SUITE_END() BOOST_AUTO_TEST_SUITE_END() diff --git a/tools/yulPhaser/Population.cpp b/tools/yulPhaser/Population.cpp index 890e13b43..e8a302683 100644 --- a/tools/yulPhaser/Population.cpp +++ b/tools/yulPhaser/Population.cpp @@ -19,6 +19,7 @@ #include +#include #include #include @@ -117,6 +118,13 @@ void Population::run(optional _numRounds, ostream& _outputStream) } } +Population operator+(Population _a, Population _b) +{ + assert(toString(_a.m_program) == toString(_b.m_program)); + + return Population(_a.m_program, move(_a.m_individuals) + move(_b.m_individuals)); +} + ostream& phaser::operator<<(ostream& _stream, Population const& _population) { auto individual = _population.m_individuals.begin(); diff --git a/tools/yulPhaser/Population.h b/tools/yulPhaser/Population.h index 43f8643e1..7171a7045 100644 --- a/tools/yulPhaser/Population.h +++ b/tools/yulPhaser/Population.h @@ -28,6 +28,17 @@ namespace solidity::phaser { +class Population; + +} + +// This operator+ must be declared in the global namespace. Otherwise it would shadow global +// operator+ overloads from CommonData.h (e.g. the one for vector) in the namespace it was declared in. +solidity::phaser::Population operator+(solidity::phaser::Population _a, solidity::phaser::Population _b); + +namespace solidity::phaser +{ + /** * Information describing the state of an individual member of the population during the course * of the genetic algorithm. @@ -74,6 +85,7 @@ public: ); void run(std::optional _numRounds, std::ostream& _outputStream); + friend Population (::operator+)(Population _a, Population _b); std::vector const& individuals() const { return m_individuals; } From 31d8d5930af52b258d6def957484271fc22fb003 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Tue, 11 Feb 2020 11:25:00 +0100 Subject: [PATCH 15/92] [yul-phaser] Population: Equality operators for populations and individuals --- tools/yulPhaser/Population.cpp | 7 +++++++ tools/yulPhaser/Population.h | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/tools/yulPhaser/Population.cpp b/tools/yulPhaser/Population.cpp index e8a302683..a0b64999b 100644 --- a/tools/yulPhaser/Population.cpp +++ b/tools/yulPhaser/Population.cpp @@ -125,6 +125,13 @@ Population operator+(Population _a, Population _b) return Population(_a.m_program, move(_a.m_individuals) + move(_b.m_individuals)); } +bool Population::operator==(Population const& _other) const +{ + // TODO: Comparing programs is pretty heavy but it's just a stopgap. It will soon be replaced + // by a comparison of fitness metric associated with the population (once metrics are introduced). + return m_individuals == _other.m_individuals && toString(m_program) == toString(_other.m_program); +} + ostream& phaser::operator<<(ostream& _stream, Population const& _population) { auto individual = _population.m_individuals.begin(); diff --git a/tools/yulPhaser/Population.h b/tools/yulPhaser/Population.h index 7171a7045..466c8f6f7 100644 --- a/tools/yulPhaser/Population.h +++ b/tools/yulPhaser/Population.h @@ -48,6 +48,9 @@ struct Individual Chromosome chromosome; std::optional fitness = std::nullopt; + bool operator==(Individual const& _other) const { return fitness == _other.fitness && chromosome == _other.chromosome; } + bool operator!=(Individual const& _other) const { return !(*this == _other); } + friend std::ostream& operator<<(std::ostream& _stream, Individual const& _individual); }; @@ -93,6 +96,9 @@ public: static size_t binomialChromosomeLength(size_t _max) { return SimulationRNG::binomialInt(_max, 0.5); } static size_t measureFitness(Chromosome const& _chromosome, Program const& _program); + bool operator==(Population const& _other) const; + bool operator!=(Population const& _other) const { return !(*this == _other); } + friend std::ostream& operator<<(std::ostream& _stream, Population const& _population); private: From 4a29726f76ecc8375b06b64a5c53de1f4d50af26 Mon Sep 17 00:00:00 2001 From: a3d4 Date: Tue, 11 Feb 2020 01:47:57 +0100 Subject: [PATCH 16/92] Adjusted solc path and fixed remapping tests in cmdlineTests.sh under mingw64. --- test/cmdlineTests.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/test/cmdlineTests.sh b/test/cmdlineTests.sh index 754cd0bac..fd0375407 100755 --- a/test/cmdlineTests.sh +++ b/test/cmdlineTests.sh @@ -33,7 +33,19 @@ set -e REPO_ROOT=$(cd $(dirname "$0")/.. && pwd) SOLIDITY_BUILD_DIR=${SOLIDITY_BUILD_DIR:-build} source "${REPO_ROOT}/scripts/common.sh" -SOLC="$REPO_ROOT/${SOLIDITY_BUILD_DIR}/solc/solc" + +case "$OSTYPE" in + msys) + SOLC="$REPO_ROOT/${SOLIDITY_BUILD_DIR}/solc/Release/solc.exe" + + # prevents msys2 path translation for a remapping test + export MSYS2_ARG_CONV_EXCL="=" + ;; + *) + SOLC="$REPO_ROOT/${SOLIDITY_BUILD_DIR}/solc/solc" + ;; +esac + INTERACTIVE=true if ! tty -s || [ "$CI" ] then From 096129fbc47fa07c6189b415e7515e171fd6964a Mon Sep 17 00:00:00 2001 From: cameel Date: Wed, 5 Feb 2020 14:56:55 +0100 Subject: [PATCH 17/92] [yul-phaser] Base class for fitness metrics --- test/CMakeLists.txt | 1 + tools/CMakeLists.txt | 2 ++ tools/yulPhaser/FitnessMetrics.cpp | 18 +++++++++++ tools/yulPhaser/FitnessMetrics.h | 48 ++++++++++++++++++++++++++++++ 4 files changed, 69 insertions(+) create mode 100644 tools/yulPhaser/FitnessMetrics.cpp create mode 100644 tools/yulPhaser/FitnessMetrics.h diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 37a78287d..d7ac67114 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -151,6 +151,7 @@ set(yul_phaser_sources # My current workaround is just to include its source files here but this introduces # unnecessary duplication. Create a library or find a way to reuse the list in both places. ../tools/yulPhaser/Chromosome.cpp + ../tools/yulPhaser/FitnessMetrics.cpp ../tools/yulPhaser/Population.cpp ../tools/yulPhaser/Program.cpp ../tools/yulPhaser/SimulationRNG.cpp diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index cd33c4006..d3101ad7b 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -17,6 +17,8 @@ add_executable(yul-phaser yulPhaser/main.cpp yulPhaser/Population.h yulPhaser/Population.cpp + yulPhaser/FitnessMetrics.h + yulPhaser/FitnessMetrics.cpp yulPhaser/Chromosome.h yulPhaser/Chromosome.cpp yulPhaser/Program.h diff --git a/tools/yulPhaser/FitnessMetrics.cpp b/tools/yulPhaser/FitnessMetrics.cpp new file mode 100644 index 000000000..a1014476b --- /dev/null +++ b/tools/yulPhaser/FitnessMetrics.cpp @@ -0,0 +1,18 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#include diff --git a/tools/yulPhaser/FitnessMetrics.h b/tools/yulPhaser/FitnessMetrics.h new file mode 100644 index 000000000..712ae1ab5 --- /dev/null +++ b/tools/yulPhaser/FitnessMetrics.h @@ -0,0 +1,48 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ +/** + * Contains an abstract base class representing a fitness metric and its concrete implementations. + */ + +#pragma once + +#include + +#include + +namespace solidity::phaser +{ + +/** + * Abstract base class for fitness metrics. + * + * The main feature is the @a evaluate() method that can tell how good a given chromosome is. + * The lower the value, the better the fitness is. The result should be deterministic and depend + * only on the chromosome and metric's state (which is constant). + */ +class FitnessMetric +{ +public: + FitnessMetric() = default; + FitnessMetric(FitnessMetric const&) = delete; + FitnessMetric& operator=(FitnessMetric const&) = delete; + virtual ~FitnessMetric() = default; + + virtual size_t evaluate(Chromosome const& _chromosome) const = 0; +}; + +} From 2238919c76b258d53079ae9a085e632d7db87fcf Mon Sep 17 00:00:00 2001 From: cameel Date: Wed, 5 Feb 2020 14:58:06 +0100 Subject: [PATCH 18/92] [yul-phaser] Add ProgramSize metric --- test/CMakeLists.txt | 1 + test/yulPhaser/FitnessMetrics.cpp | 80 ++++++++++++++++++++++++++++++ tools/yulPhaser/FitnessMetrics.cpp | 10 ++++ tools/yulPhaser/FitnessMetrics.h | 16 ++++++ 4 files changed, 107 insertions(+) create mode 100644 test/yulPhaser/FitnessMetrics.cpp diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index d7ac67114..3b2fef6f0 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -143,6 +143,7 @@ set(yul_phaser_sources yulPhaser/Common.cpp yulPhaser/CommonTest.cpp yulPhaser/Chromosome.cpp + yulPhaser/FitnessMetrics.cpp yulPhaser/Population.cpp yulPhaser/Program.cpp yulPhaser/SimulationRNG.cpp diff --git a/test/yulPhaser/FitnessMetrics.cpp b/test/yulPhaser/FitnessMetrics.cpp new file mode 100644 index 000000000..7d3c3ab52 --- /dev/null +++ b/test/yulPhaser/FitnessMetrics.cpp @@ -0,0 +1,80 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#include + +#include +#include + +#include + +#include + +using namespace std; +using namespace solidity::langutil; +using namespace solidity::yul; + +namespace solidity::phaser::test +{ + +class FitnessMetricFixture +{ +protected: + FitnessMetricFixture(): + m_sourceStream(SampleSourceCode, ""), + m_program(Program::load(m_sourceStream)) {} + + static constexpr char SampleSourceCode[] = + "{\n" + " function foo() -> result\n" + " {\n" + " let x := 1\n" + " result := 15\n" + " }\n" + " function bar() -> result\n" + " {\n" + " result := 15\n" + " }\n" + " mstore(foo(), bar())\n" + "}\n"; + + CharStream m_sourceStream; + Program m_program; +}; + +BOOST_AUTO_TEST_SUITE(Phaser) +BOOST_AUTO_TEST_SUITE(FitnessMetricsTest) +BOOST_AUTO_TEST_SUITE(ProgramSizeTest) + +BOOST_FIXTURE_TEST_CASE(evaluate_should_compute_size_of_the_optimised_program, FitnessMetricFixture) +{ + Chromosome chromosome(vector{UnusedPruner::name, EquivalentFunctionCombiner::name}); + + Program optimisedProgram = m_program; + optimisedProgram.optimise(chromosome.optimisationSteps()); + assert(m_program.codeSize() != optimisedProgram.codeSize()); + + BOOST_TEST(ProgramSize(m_program).evaluate(chromosome) != m_program.codeSize()); + BOOST_TEST(ProgramSize(m_program).evaluate(chromosome) == optimisedProgram.codeSize()); +} + +BOOST_AUTO_TEST_SUITE_END() +BOOST_AUTO_TEST_SUITE_END() +BOOST_AUTO_TEST_SUITE_END() + +} + diff --git a/tools/yulPhaser/FitnessMetrics.cpp b/tools/yulPhaser/FitnessMetrics.cpp index a1014476b..27e2e3094 100644 --- a/tools/yulPhaser/FitnessMetrics.cpp +++ b/tools/yulPhaser/FitnessMetrics.cpp @@ -16,3 +16,13 @@ */ #include + +using namespace std; +using namespace solidity::phaser; + +size_t ProgramSize::evaluate(Chromosome const& _chromosome) const +{ + Program programCopy = m_program; + programCopy.optimise(_chromosome.optimisationSteps()); + return programCopy.codeSize(); +} diff --git a/tools/yulPhaser/FitnessMetrics.h b/tools/yulPhaser/FitnessMetrics.h index 712ae1ab5..025b0ccec 100644 --- a/tools/yulPhaser/FitnessMetrics.h +++ b/tools/yulPhaser/FitnessMetrics.h @@ -21,6 +21,7 @@ #pragma once #include +#include #include @@ -45,4 +46,19 @@ public: virtual size_t evaluate(Chromosome const& _chromosome) const = 0; }; +/** + * Fitness metric based on the size of a specific program after applying the optimisations from the + * chromosome to it. + */ +class ProgramSize: public FitnessMetric +{ +public: + ProgramSize(Program _program): m_program(std::move(_program)) {} + + size_t evaluate(Chromosome const& _chromosome) const override; + +private: + Program m_program; +}; + } From 751caf0ed3a5c7838bbaaec90426cfd85b8b80bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Sat, 15 Feb 2020 01:40:18 +0100 Subject: [PATCH 19/92] [yul-phaser] Common: Add ChromosomeLengthMetric --- test/yulPhaser/Common.h | 14 ++++++++++++++ test/yulPhaser/CommonTest.cpp | 17 ++++++++++------- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/test/yulPhaser/Common.h b/test/yulPhaser/Common.h index c4da6c99c..63ca45c50 100644 --- a/test/yulPhaser/Common.h +++ b/test/yulPhaser/Common.h @@ -28,6 +28,8 @@ #pragma once +#include +#include #include #include @@ -38,6 +40,18 @@ namespace solidity::phaser::test { +/** + * Fitness metric that only takes into account the number of optimisation steps in the chromosome. + * Recommended for use in tests because it's much faster than ProgramSize metric and it's very + * easy to guess the result at a glance. + */ +class ChromosomeLengthMetric: public FitnessMetric +{ +public: + using FitnessMetric::FitnessMetric; + size_t evaluate(Chromosome const& _chromosome) const override { return _chromosome.length(); } +}; + // CHROMOSOME AND POPULATION HELPERS /// Returns a vector containing lengths of all chromosomes in the population (in the same order). diff --git a/test/yulPhaser/CommonTest.cpp b/test/yulPhaser/CommonTest.cpp index be686446f..feded8292 100644 --- a/test/yulPhaser/CommonTest.cpp +++ b/test/yulPhaser/CommonTest.cpp @@ -19,14 +19,11 @@ #include -#include - #include #include using namespace std; -using namespace solidity::langutil; using namespace solidity::yul; using namespace boost::test_tools; @@ -36,15 +33,21 @@ namespace solidity::phaser::test BOOST_AUTO_TEST_SUITE(Phaser) BOOST_AUTO_TEST_SUITE(CommonTest) +BOOST_AUTO_TEST_CASE(ChromosomeLengthMetric_evaluate_should_return_chromosome_length) +{ + BOOST_TEST(ChromosomeLengthMetric{}.evaluate(Chromosome()) == 0); + BOOST_TEST(ChromosomeLengthMetric{}.evaluate(Chromosome("a")) == 1); + BOOST_TEST(ChromosomeLengthMetric{}.evaluate(Chromosome("aaaaa")) == 5); +} + BOOST_AUTO_TEST_CASE(chromosomeLengths_should_return_lengths_of_all_chromosomes_in_a_population) { - CharStream sourceStream("{}", ""); - auto program = Program::load(sourceStream); + shared_ptr fitnessMetric = make_shared(); - Population population1(program, {Chromosome(), Chromosome("a"), Chromosome("aa"), Chromosome("aaa")}); + Population population1(fitnessMetric, {Chromosome(), Chromosome("a"), Chromosome("aa"), Chromosome("aaa")}); BOOST_TEST((chromosomeLengths(population1) == vector{0, 1, 2, 3})); - Population population2(program); + Population population2(fitnessMetric); BOOST_TEST((chromosomeLengths(population2) == vector{})); } From 930a9918a61b0240f6183e820ce0b68f3dfc0cb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Wed, 5 Feb 2020 19:24:46 +0100 Subject: [PATCH 20/92] [yul-phaser] ProgramSize: Add an option to repeat the optimisation sequence several times --- test/yulPhaser/FitnessMetrics.cpp | 33 ++++++++++++++++++++++++++++++ tools/yulPhaser/FitnessMetrics.cpp | 4 +++- tools/yulPhaser/FitnessMetrics.h | 5 ++++- 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/test/yulPhaser/FitnessMetrics.cpp b/test/yulPhaser/FitnessMetrics.cpp index 7d3c3ab52..58561806d 100644 --- a/test/yulPhaser/FitnessMetrics.cpp +++ b/test/yulPhaser/FitnessMetrics.cpp @@ -72,6 +72,39 @@ BOOST_FIXTURE_TEST_CASE(evaluate_should_compute_size_of_the_optimised_program, F BOOST_TEST(ProgramSize(m_program).evaluate(chromosome) == optimisedProgram.codeSize()); } +BOOST_FIXTURE_TEST_CASE(evaluate_should_repeat_the_optimisation_specified_number_of_times, FitnessMetricFixture) +{ + Chromosome chromosome(vector{UnusedPruner::name, EquivalentFunctionCombiner::name}); + + Program programOptimisedOnce = m_program; + programOptimisedOnce.optimise(chromosome.optimisationSteps()); + Program programOptimisedTwice = programOptimisedOnce; + programOptimisedTwice.optimise(chromosome.optimisationSteps()); + assert(m_program.codeSize() != programOptimisedOnce.codeSize()); + assert(m_program.codeSize() != programOptimisedTwice.codeSize()); + assert(programOptimisedOnce.codeSize() != programOptimisedTwice.codeSize()); + + ProgramSize metric(m_program, 2); + + BOOST_TEST(metric.evaluate(chromosome) != m_program.codeSize()); + BOOST_TEST(metric.evaluate(chromosome) != programOptimisedOnce.codeSize()); + BOOST_TEST(metric.evaluate(chromosome) == programOptimisedTwice.codeSize()); +} + +BOOST_FIXTURE_TEST_CASE(evaluate_should_not_optimise_if_number_of_repetitions_is_zero, FitnessMetricFixture) +{ + Chromosome chromosome(vector{UnusedPruner::name, EquivalentFunctionCombiner::name}); + + Program optimisedProgram = m_program; + optimisedProgram.optimise(chromosome.optimisationSteps()); + assert(m_program.codeSize() != optimisedProgram.codeSize()); + + ProgramSize metric(m_program, 0); + + BOOST_TEST(metric.evaluate(chromosome) == m_program.codeSize()); + BOOST_TEST(metric.evaluate(chromosome) != optimisedProgram.codeSize()); +} + BOOST_AUTO_TEST_SUITE_END() BOOST_AUTO_TEST_SUITE_END() BOOST_AUTO_TEST_SUITE_END() diff --git a/tools/yulPhaser/FitnessMetrics.cpp b/tools/yulPhaser/FitnessMetrics.cpp index 27e2e3094..be7ea5497 100644 --- a/tools/yulPhaser/FitnessMetrics.cpp +++ b/tools/yulPhaser/FitnessMetrics.cpp @@ -23,6 +23,8 @@ using namespace solidity::phaser; size_t ProgramSize::evaluate(Chromosome const& _chromosome) const { Program programCopy = m_program; - programCopy.optimise(_chromosome.optimisationSteps()); + for (size_t i = 0; i < m_repetitionCount; ++i) + programCopy.optimise(_chromosome.optimisationSteps()); + return programCopy.codeSize(); } diff --git a/tools/yulPhaser/FitnessMetrics.h b/tools/yulPhaser/FitnessMetrics.h index 025b0ccec..7fac5f080 100644 --- a/tools/yulPhaser/FitnessMetrics.h +++ b/tools/yulPhaser/FitnessMetrics.h @@ -53,12 +53,15 @@ public: class ProgramSize: public FitnessMetric { public: - ProgramSize(Program _program): m_program(std::move(_program)) {} + explicit ProgramSize(Program _program, size_t _repetitionCount = 1): + m_program(std::move(_program)), + m_repetitionCount(_repetitionCount) {} size_t evaluate(Chromosome const& _chromosome) const override; private: Program m_program; + size_t m_repetitionCount; }; } From 41f36f421db46bbfde1e1a754bbb37267d768c96 Mon Sep 17 00:00:00 2001 From: cameel Date: Wed, 5 Feb 2020 15:15:17 +0100 Subject: [PATCH 21/92] [yul-phaser] Population: Extract conversion from chromosomes to individuals into a separate function --- tools/yulPhaser/Population.cpp | 18 +++++++++++------- tools/yulPhaser/Population.h | 9 ++++++++- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/tools/yulPhaser/Population.cpp b/tools/yulPhaser/Population.cpp index a0b64999b..02d356bb1 100644 --- a/tools/yulPhaser/Population.cpp +++ b/tools/yulPhaser/Population.cpp @@ -63,13 +63,6 @@ bool phaser::isFitter(Individual const& a, Individual const& b) ); } -Population::Population(Program _program, vector const& _chromosomes): - m_program{move(_program)} -{ - for (auto const& chromosome: _chromosomes) - m_individuals.push_back({chromosome}); -} - Population Population::makeRandom( Program _program, size_t _size, @@ -175,3 +168,14 @@ void Population::randomizeWorstChromosomes( *individual = {Chromosome::makeRandom(binomialChromosomeLength(MaxChromosomeLength))}; } } + +vector Population::chromosomesToIndividuals( + vector _chromosomes +) +{ + vector individuals; + for (auto& chromosome: _chromosomes) + individuals.push_back({move(chromosome)}); + + return individuals; +} diff --git a/tools/yulPhaser/Population.h b/tools/yulPhaser/Population.h index 466c8f6f7..ec384ecc8 100644 --- a/tools/yulPhaser/Population.h +++ b/tools/yulPhaser/Population.h @@ -73,7 +73,11 @@ class Population public: static constexpr size_t MaxChromosomeLength = 30; - explicit Population(Program _program, std::vector const& _chromosomes = {}); + explicit Population(Program _program, std::vector _chromosomes = {}): + Population( + std::move(_program), + chromosomesToIndividuals(std::move(_chromosomes)) + ) {} static Population makeRandom( Program _program, @@ -114,6 +118,9 @@ private: std::vector& _individuals, size_t _count ); + static std::vector chromosomesToIndividuals( + std::vector _chromosomes + ); Program m_program; std::vector m_individuals; From 85074e7a8a7956e4fc2f926f6cd4630301cd8f54 Mon Sep 17 00:00:00 2001 From: cameel Date: Wed, 5 Feb 2020 16:04:17 +0100 Subject: [PATCH 22/92] [yul-phaser] Population: Extract sorting from doSelection() into sortIndividuals() in Population class --- tools/yulPhaser/Population.cpp | 12 +++++++++--- tools/yulPhaser/Population.h | 1 + 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/tools/yulPhaser/Population.cpp b/tools/yulPhaser/Population.cpp index 02d356bb1..dbd406613 100644 --- a/tools/yulPhaser/Population.cpp +++ b/tools/yulPhaser/Population.cpp @@ -148,9 +148,7 @@ void Population::doEvaluation() void Population::doSelection() { - assert(all_of(m_individuals.begin(), m_individuals.end(), [](auto& i){ return i.fitness.has_value(); })); - - sort(m_individuals.begin(), m_individuals.end(), isFitter); + m_individuals = sortedIndividuals(move(m_individuals)); randomizeWorstChromosomes(m_individuals, m_individuals.size() / 2); } @@ -179,3 +177,11 @@ vector Population::chromosomesToIndividuals( return individuals; } + +vector Population::sortedIndividuals(vector _individuals) +{ + assert(all_of(_individuals.begin(), _individuals.end(), [](auto& i){ return i.fitness.has_value(); })); + + sort(_individuals.begin(), _individuals.end(), isFitter); + return _individuals; +} diff --git a/tools/yulPhaser/Population.h b/tools/yulPhaser/Population.h index ec384ecc8..1e8a116b8 100644 --- a/tools/yulPhaser/Population.h +++ b/tools/yulPhaser/Population.h @@ -121,6 +121,7 @@ private: static std::vector chromosomesToIndividuals( std::vector _chromosomes ); + static std::vector sortedIndividuals(std::vector _individuals); Program m_program; std::vector m_individuals; From 66fdc1c374dd26d937d0f8eff9e96a6af84324a3 Mon Sep 17 00:00:00 2001 From: cameel Date: Wed, 5 Feb 2020 15:58:35 +0100 Subject: [PATCH 23/92] [yul-phaser] Population: Store fitness metric rather than program directly - In the console app use ProgramSize metric when creating the population. --- test/yulPhaser/Population.cpp | 56 +++++++++------------------------- tools/yulPhaser/Population.cpp | 31 ++++++++----------- tools/yulPhaser/Population.h | 27 ++++++++-------- tools/yulPhaser/main.cpp | 4 ++- 4 files changed, 45 insertions(+), 73 deletions(-) diff --git a/test/yulPhaser/Population.cpp b/test/yulPhaser/Population.cpp index be90382c7..4fb796c3e 100644 --- a/test/yulPhaser/Population.cpp +++ b/test/yulPhaser/Population.cpp @@ -58,35 +58,7 @@ namespace class PopulationFixture { protected: - PopulationFixture(): - m_sourceStream(SampleSourceCode, ""), - m_program(Program::load(m_sourceStream)) {} - - static constexpr char SampleSourceCode[] = - "{\n" - " let factor := 13\n" - " {\n" - " if factor\n" - " {\n" - " let variable := add(1, 2)\n" - " }\n" - " let result := factor\n" - " }\n" - " let something := 6\n" - " {\n" - " {\n" - " {\n" - " let value := 15\n" - " }\n" - " }\n" - " }\n" - " let something_else := mul(mul(something, 1), add(factor, 0))\n" - " if 1 { let x := 1 }\n" - " if 0 { let y := 2 }\n" - "}\n"; - - CharStream m_sourceStream; - Program m_program; + shared_ptr m_fitnessMetric = make_shared(); }; BOOST_AUTO_TEST_SUITE(Phaser) @@ -128,7 +100,7 @@ BOOST_FIXTURE_TEST_CASE(constructor_should_copy_chromosomes_and_not_compute_fitn Chromosome::makeRandom(5), Chromosome::makeRandom(10), }; - Population population(m_program, chromosomes); + Population population(m_fitnessMetric, chromosomes); BOOST_TEST(population.individuals().size() == 2); BOOST_TEST(population.individuals()[0].chromosome == chromosomes[0]); @@ -145,7 +117,7 @@ BOOST_FIXTURE_TEST_CASE(makeRandom_should_get_chromosome_lengths_from_specified_ assert(chromosomeCount % maxLength == 0); auto nextLength = [counter = 0, maxLength]() mutable { return counter++ % maxLength; }; - auto population = Population::makeRandom(m_program, chromosomeCount, nextLength); + auto population = Population::makeRandom(m_fitnessMetric, chromosomeCount, nextLength); // We can't rely on the order since the population sorts its chromosomes immediately but // we can check the number of occurrences of each length. @@ -161,7 +133,7 @@ BOOST_FIXTURE_TEST_CASE(makeRandom_should_get_chromosome_lengths_from_specified_ BOOST_FIXTURE_TEST_CASE(makeRandom_should_get_chromosome_lengths_from_specified_range, PopulationFixture) { - auto population = Population::makeRandom(m_program, 100, 5, 10); + auto population = Population::makeRandom(m_fitnessMetric, 100, 5, 10); BOOST_TEST(all_of( population.individuals().begin(), population.individuals().end(), @@ -177,7 +149,7 @@ BOOST_FIXTURE_TEST_CASE(makeRandom_should_use_random_chromosome_length, Populati constexpr int maxLength = 10; constexpr double relativeTolerance = 0.05; - auto population = Population::makeRandom(m_program, populationSize, minLength, maxLength); + auto population = Population::makeRandom(m_fitnessMetric, populationSize, minLength, maxLength); vector samples = chromosomeLengths(population); const double expectedValue = (maxLength + minLength) / 2.0; @@ -195,7 +167,7 @@ BOOST_FIXTURE_TEST_CASE(makeRandom_should_return_population_with_random_chromoso constexpr double relativeTolerance = 0.01; map stepIndices = enumerateOptmisationSteps(); - auto population = Population::makeRandom(m_program, populationSize, chromosomeLength, chromosomeLength); + auto population = Population::makeRandom(m_fitnessMetric, populationSize, chromosomeLength, chromosomeLength); vector samples; for (auto& individual: population.individuals()) @@ -211,7 +183,7 @@ BOOST_FIXTURE_TEST_CASE(makeRandom_should_return_population_with_random_chromoso BOOST_FIXTURE_TEST_CASE(makeRandom_should_not_compute_fitness, PopulationFixture) { - auto population = Population::makeRandom(m_program, 3, 5, 10); + auto population = Population::makeRandom(m_fitnessMetric, 3, 5, 10); BOOST_TEST(all_of(population.individuals().begin(), population.individuals().end(), fitnessNotSet)); } @@ -219,7 +191,7 @@ BOOST_FIXTURE_TEST_CASE(makeRandom_should_not_compute_fitness, PopulationFixture BOOST_FIXTURE_TEST_CASE(run_should_evaluate_fitness, PopulationFixture) { stringstream output; - auto population = Population::makeRandom(m_program, 5, 5, 10); + auto population = Population::makeRandom(m_fitnessMetric, 5, 5, 10); assert(all_of(population.individuals().begin(), population.individuals().end(), fitnessNotSet)); population.run(1, output); @@ -237,11 +209,11 @@ BOOST_FIXTURE_TEST_CASE(run_should_not_make_fitness_of_top_chromosomes_worse, Po Chromosome(vector{UnusedPruner::name}), Chromosome(vector{StructuralSimplifier::name, BlockFlattener::name}), }; - Population population(m_program, chromosomes); + Population population(m_fitnessMetric, chromosomes); size_t initialTopFitness[2] = { - Population::measureFitness(chromosomes[0], m_program), - Population::measureFitness(chromosomes[1], m_program), + m_fitnessMetric->evaluate(chromosomes[0]), + m_fitnessMetric->evaluate(chromosomes[1]), }; for (int i = 0; i < 6; ++i) @@ -264,9 +236,9 @@ BOOST_FIXTURE_TEST_CASE(run_should_not_make_fitness_of_top_chromosomes_worse, Po BOOST_FIXTURE_TEST_CASE(plus_operator_should_add_two_populations, PopulationFixture) { BOOST_CHECK_EQUAL( - Population(m_program, {Chromosome("ac"), Chromosome("cx")}) + - Population(m_program, {Chromosome("g"), Chromosome("h"), Chromosome("iI")}), - Population(m_program, {Chromosome("ac"), Chromosome("cx"), Chromosome("g"), Chromosome("h"), Chromosome("iI")}) + Population(m_fitnessMetric, {Chromosome("ac"), Chromosome("cx")}) + + Population(m_fitnessMetric, {Chromosome("g"), Chromosome("h"), Chromosome("iI")}), + Population(m_fitnessMetric, {Chromosome("ac"), Chromosome("cx"), Chromosome("g"), Chromosome("h"), Chromosome("iI")}) ); } diff --git a/tools/yulPhaser/Population.cpp b/tools/yulPhaser/Population.cpp index dbd406613..58309299a 100644 --- a/tools/yulPhaser/Population.cpp +++ b/tools/yulPhaser/Population.cpp @@ -17,7 +17,6 @@ #include -#include #include #include @@ -64,7 +63,7 @@ bool phaser::isFitter(Individual const& a, Individual const& b) } Population Population::makeRandom( - Program _program, + shared_ptr _fitnessMetric, size_t _size, function _chromosomeLengthGenerator ) @@ -73,30 +72,23 @@ Population Population::makeRandom( for (size_t i = 0; i < _size; ++i) individuals.push_back({Chromosome::makeRandom(_chromosomeLengthGenerator())}); - return Population(move(_program), individuals); + return Population(move(_fitnessMetric), move(individuals)); } Population Population::makeRandom( - Program _program, + shared_ptr _fitnessMetric, size_t _size, size_t _minChromosomeLength, size_t _maxChromosomeLength ) { return makeRandom( - move(_program), + move(_fitnessMetric), _size, std::bind(uniformChromosomeLength, _minChromosomeLength, _maxChromosomeLength) ); } -size_t Population::measureFitness(Chromosome const& _chromosome, Program const& _program) -{ - Program programCopy = _program; - programCopy.optimise(_chromosome.optimisationSteps()); - return programCopy.codeSize(); -} - void Population::run(optional _numRounds, ostream& _outputStream) { doEvaluation(); @@ -113,16 +105,19 @@ void Population::run(optional _numRounds, ostream& _outputStream) Population operator+(Population _a, Population _b) { - assert(toString(_a.m_program) == toString(_b.m_program)); + // This operator is meant to be used only with populations sharing the same metric (and, to make + // things simple, "the same" here means the same exact object in memory). + assert(_a.m_fitnessMetric == _b.m_fitnessMetric); - return Population(_a.m_program, move(_a.m_individuals) + move(_b.m_individuals)); + return Population(_a.m_fitnessMetric, move(_a.m_individuals) + move(_b.m_individuals)); } bool Population::operator==(Population const& _other) const { - // TODO: Comparing programs is pretty heavy but it's just a stopgap. It will soon be replaced - // by a comparison of fitness metric associated with the population (once metrics are introduced). - return m_individuals == _other.m_individuals && toString(m_program) == toString(_other.m_program); + // We consider populations identical only if they share the same exact instance of the metric. + // It might be possible to define some notion of equality for metric objects but it would + // be an overkill since mixing populations using different metrics is not a common use case. + return m_individuals == _other.m_individuals && m_fitnessMetric == _other.m_fitnessMetric; } ostream& phaser::operator<<(ostream& _stream, Population const& _population) @@ -143,7 +138,7 @@ void Population::doEvaluation() { for (auto& individual: m_individuals) if (!individual.fitness.has_value()) - individual.fitness = measureFitness(individual.chromosome, m_program); + individual.fitness = m_fitnessMetric->evaluate(individual.chromosome); } void Population::doSelection() diff --git a/tools/yulPhaser/Population.h b/tools/yulPhaser/Population.h index 1e8a116b8..5387cea28 100644 --- a/tools/yulPhaser/Population.h +++ b/tools/yulPhaser/Population.h @@ -18,7 +18,7 @@ #pragma once #include -#include +#include #include #include @@ -64,28 +64,31 @@ bool isFitter(Individual const& a, Individual const& b); * Each round of the algorithm involves mutating existing individuals, evaluating their fitness * and selecting the best ones for the next round. * - * An individual is a sequence of optimiser steps represented by a @a Chromosome instance. The whole - * population is associated with a fixed Yul program. By applying the steps to the @a Program - * instance the class can compute fitness of the individual. + * An individual is a sequence of optimiser steps represented by a @a Chromosome instance. + * Individuals are stored together with a fitness value that can be computed by the fitness metric + * associated with the population. */ class Population { public: static constexpr size_t MaxChromosomeLength = 30; - explicit Population(Program _program, std::vector _chromosomes = {}): + explicit Population( + std::shared_ptr _fitnessMetric, + std::vector _chromosomes = {} + ): Population( - std::move(_program), + std::move(_fitnessMetric), chromosomesToIndividuals(std::move(_chromosomes)) ) {} static Population makeRandom( - Program _program, + std::shared_ptr _fitnessMetric, size_t _size, std::function _chromosomeLengthGenerator ); static Population makeRandom( - Program _program, + std::shared_ptr _fitnessMetric, size_t _size, size_t _minChromosomeLength, size_t _maxChromosomeLength @@ -94,11 +97,11 @@ public: void run(std::optional _numRounds, std::ostream& _outputStream); friend Population (::operator+)(Population _a, Population _b); + std::shared_ptr fitnessMetric() const { return m_fitnessMetric; } std::vector const& individuals() const { return m_individuals; } static size_t uniformChromosomeLength(size_t _min, size_t _max) { return SimulationRNG::uniformInt(_min, _max); } static size_t binomialChromosomeLength(size_t _max) { return SimulationRNG::binomialInt(_max, 0.5); } - static size_t measureFitness(Chromosome const& _chromosome, Program const& _program); bool operator==(Population const& _other) const; bool operator!=(Population const& _other) const { return !(*this == _other); } @@ -106,8 +109,8 @@ public: friend std::ostream& operator<<(std::ostream& _stream, Population const& _population); private: - explicit Population(Program _program, std::vector _individuals): - m_program{std::move(_program)}, + explicit Population(std::shared_ptr _fitnessMetric, std::vector _individuals): + m_fitnessMetric(std::move(_fitnessMetric)), m_individuals{std::move(_individuals)} {} void doMutation(); @@ -123,7 +126,7 @@ private: ); static std::vector sortedIndividuals(std::vector _individuals); - Program m_program; + std::shared_ptr m_fitnessMetric; std::vector m_individuals; }; diff --git a/tools/yulPhaser/main.cpp b/tools/yulPhaser/main.cpp index ed846dc4f..f8226e1b9 100644 --- a/tools/yulPhaser/main.cpp +++ b/tools/yulPhaser/main.cpp @@ -17,6 +17,7 @@ #include #include +#include #include #include @@ -71,8 +72,9 @@ CharStream loadSource(string const& _sourcePath) void runAlgorithm(string const& _sourcePath) { CharStream sourceCode = loadSource(_sourcePath); + shared_ptr fitnessMetric = make_shared(Program::load(sourceCode)); auto population = Population::makeRandom( - Program::load(sourceCode), + fitnessMetric, 10, bind(Population::binomialChromosomeLength, Population::MaxChromosomeLength) ); From 76842ac3fdd63fed559ccbbdf070c41dc2b0df41 Mon Sep 17 00:00:00 2001 From: cameel Date: Wed, 5 Feb 2020 16:18:53 +0100 Subject: [PATCH 24/92] [yul-phaser] Population: Evaluate fitness immediately when an individual is added or modified - This removes the explicit evaluation phase. - Fitness is no longer optional in Individual --- test/yulPhaser/Population.cpp | 42 +++++++------------------------- tools/yulPhaser/Population.cpp | 44 +++++++++++++--------------------- tools/yulPhaser/Population.h | 10 ++++---- 3 files changed, 32 insertions(+), 64 deletions(-) diff --git a/test/yulPhaser/Population.cpp b/test/yulPhaser/Population.cpp index 4fb796c3e..1b8e76da0 100644 --- a/test/yulPhaser/Population.cpp +++ b/test/yulPhaser/Population.cpp @@ -42,19 +42,6 @@ using namespace boost::unit_test::framework; namespace solidity::phaser::test { -namespace -{ - bool fitnessNotSet(Individual const& individual) - { - return !individual.fitness.has_value(); - } - - bool fitnessSet(Individual const& individual) - { - return individual.fitness.has_value(); - } -} - class PopulationFixture { protected: @@ -94,7 +81,7 @@ BOOST_AUTO_TEST_CASE(isFitter_should_return_false_for_identical_individuals) BOOST_TEST(!isFitter(Individual{Chromosome("acT"), 0}, Individual{Chromosome("acT"), 0})); } -BOOST_FIXTURE_TEST_CASE(constructor_should_copy_chromosomes_and_not_compute_fitness, PopulationFixture) +BOOST_FIXTURE_TEST_CASE(constructor_should_copy_chromosomes_and_compute_fitness, PopulationFixture) { vector chromosomes = { Chromosome::makeRandom(5), @@ -106,8 +93,8 @@ BOOST_FIXTURE_TEST_CASE(constructor_should_copy_chromosomes_and_not_compute_fitn BOOST_TEST(population.individuals()[0].chromosome == chromosomes[0]); BOOST_TEST(population.individuals()[1].chromosome == chromosomes[1]); - auto fitnessNotSet = [](auto const& individual){ return !individual.fitness.has_value(); }; - BOOST_TEST(all_of(population.individuals().begin(), population.individuals().end(), fitnessNotSet)); + BOOST_TEST(population.individuals()[0].fitness == m_fitnessMetric->evaluate(population.individuals()[0].chromosome)); + BOOST_TEST(population.individuals()[1].fitness == m_fitnessMetric->evaluate(population.individuals()[1].chromosome)); } BOOST_FIXTURE_TEST_CASE(makeRandom_should_get_chromosome_lengths_from_specified_generator, PopulationFixture) @@ -181,22 +168,13 @@ BOOST_FIXTURE_TEST_CASE(makeRandom_should_return_population_with_random_chromoso BOOST_TEST(abs(meanSquaredError(samples, expectedValue) - variance) < variance * relativeTolerance); } -BOOST_FIXTURE_TEST_CASE(makeRandom_should_not_compute_fitness, PopulationFixture) +BOOST_FIXTURE_TEST_CASE(makeRandom_should_compute_fitness, PopulationFixture) { auto population = Population::makeRandom(m_fitnessMetric, 3, 5, 10); - BOOST_TEST(all_of(population.individuals().begin(), population.individuals().end(), fitnessNotSet)); -} - -BOOST_FIXTURE_TEST_CASE(run_should_evaluate_fitness, PopulationFixture) -{ - stringstream output; - auto population = Population::makeRandom(m_fitnessMetric, 5, 5, 10); - assert(all_of(population.individuals().begin(), population.individuals().end(), fitnessNotSet)); - - population.run(1, output); - - BOOST_TEST(all_of(population.individuals().begin(), population.individuals().end(), fitnessSet)); + BOOST_TEST(population.individuals()[0].fitness == m_fitnessMetric->evaluate(population.individuals()[0].chromosome)); + BOOST_TEST(population.individuals()[1].fitness == m_fitnessMetric->evaluate(population.individuals()[1].chromosome)); + BOOST_TEST(population.individuals()[2].fitness == m_fitnessMetric->evaluate(population.individuals()[2].chromosome)); } BOOST_FIXTURE_TEST_CASE(run_should_not_make_fitness_of_top_chromosomes_worse, PopulationFixture) @@ -220,12 +198,10 @@ BOOST_FIXTURE_TEST_CASE(run_should_not_make_fitness_of_top_chromosomes_worse, Po { population.run(1, output); BOOST_TEST(population.individuals().size() == 5); - BOOST_TEST(fitnessSet(population.individuals()[0])); - BOOST_TEST(fitnessSet(population.individuals()[1])); size_t currentTopFitness[2] = { - population.individuals()[0].fitness.value(), - population.individuals()[1].fitness.value(), + population.individuals()[0].fitness, + population.individuals()[1].fitness, }; BOOST_TEST(currentTopFitness[0] <= initialTopFitness[0]); BOOST_TEST(currentTopFitness[1] <= initialTopFitness[1]); diff --git a/tools/yulPhaser/Population.cpp b/tools/yulPhaser/Population.cpp index 58309299a..d258fbf3d 100644 --- a/tools/yulPhaser/Population.cpp +++ b/tools/yulPhaser/Population.cpp @@ -41,11 +41,7 @@ ostream& operator<<(ostream& _stream, Population const& _population); ostream& phaser::operator<<(ostream& _stream, Individual const& _individual) { - _stream << "Fitness: "; - if (_individual.fitness.has_value()) - _stream << _individual.fitness.value(); - else - _stream << ""; + _stream << "Fitness: " << _individual.fitness; _stream << ", optimisations: " << _individual.chromosome; return _stream; @@ -53,12 +49,10 @@ ostream& phaser::operator<<(ostream& _stream, Individual const& _individual) bool phaser::isFitter(Individual const& a, Individual const& b) { - assert(a.fitness.has_value() && b.fitness.has_value()); - return ( - (a.fitness.value() < b.fitness.value()) || - (a.fitness.value() == b.fitness.value() && a.chromosome.length() < b.chromosome.length()) || - (a.fitness.value() == b.fitness.value() && a.chromosome.length() == b.chromosome.length() && toString(a.chromosome) < toString(b.chromosome)) + (a.fitness < b.fitness) || + (a.fitness == b.fitness && a.chromosome.length() < b.chromosome.length()) || + (a.fitness == b.fitness && a.chromosome.length() == b.chromosome.length() && toString(a.chromosome) < toString(b.chromosome)) ); } @@ -68,11 +62,11 @@ Population Population::makeRandom( function _chromosomeLengthGenerator ) { - vector individuals; + vector chromosomes; for (size_t i = 0; i < _size; ++i) - individuals.push_back({Chromosome::makeRandom(_chromosomeLengthGenerator())}); + chromosomes.push_back(Chromosome::makeRandom(_chromosomeLengthGenerator())); - return Population(move(_fitnessMetric), move(individuals)); + return Population(move(_fitnessMetric), move(chromosomes)); } Population Population::makeRandom( @@ -91,12 +85,10 @@ Population Population::makeRandom( void Population::run(optional _numRounds, ostream& _outputStream) { - doEvaluation(); for (size_t round = 0; !_numRounds.has_value() || round < _numRounds.value(); ++round) { doMutation(); doSelection(); - doEvaluation(); _outputStream << "---------- ROUND " << round << " ----------" << endl; _outputStream << *this; @@ -134,20 +126,14 @@ void Population::doMutation() // TODO: Implement mutation and crossover } -void Population::doEvaluation() -{ - for (auto& individual: m_individuals) - if (!individual.fitness.has_value()) - individual.fitness = m_fitnessMetric->evaluate(individual.chromosome); -} - void Population::doSelection() { m_individuals = sortedIndividuals(move(m_individuals)); - randomizeWorstChromosomes(m_individuals, m_individuals.size() / 2); + randomizeWorstChromosomes(*m_fitnessMetric, m_individuals, m_individuals.size() / 2); } void Population::randomizeWorstChromosomes( + FitnessMetric const& _fitnessMetric, vector& _individuals, size_t _count ) @@ -158,25 +144,29 @@ void Population::randomizeWorstChromosomes( auto individual = _individuals.begin() + (_individuals.size() - _count); for (; individual != _individuals.end(); ++individual) { - *individual = {Chromosome::makeRandom(binomialChromosomeLength(MaxChromosomeLength))}; + auto chromosome = Chromosome::makeRandom(binomialChromosomeLength(MaxChromosomeLength)); + size_t fitness = _fitnessMetric.evaluate(chromosome); + *individual = {move(chromosome), fitness}; } } vector Population::chromosomesToIndividuals( + FitnessMetric const& _fitnessMetric, vector _chromosomes ) { vector individuals; for (auto& chromosome: _chromosomes) - individuals.push_back({move(chromosome)}); + { + size_t fitness = _fitnessMetric.evaluate(chromosome); + individuals.push_back({move(chromosome), fitness}); + } return individuals; } vector Population::sortedIndividuals(vector _individuals) { - assert(all_of(_individuals.begin(), _individuals.end(), [](auto& i){ return i.fitness.has_value(); })); - sort(_individuals.begin(), _individuals.end(), isFitter); return _individuals; } diff --git a/tools/yulPhaser/Population.h b/tools/yulPhaser/Population.h index 5387cea28..0dbac3235 100644 --- a/tools/yulPhaser/Population.h +++ b/tools/yulPhaser/Population.h @@ -46,7 +46,7 @@ namespace solidity::phaser struct Individual { Chromosome chromosome; - std::optional fitness = std::nullopt; + size_t fitness; bool operator==(Individual const& _other) const { return fitness == _other.fitness && chromosome == _other.chromosome; } bool operator!=(Individual const& _other) const { return !(*this == _other); } @@ -67,6 +67,7 @@ bool isFitter(Individual const& a, Individual const& b); * An individual is a sequence of optimiser steps represented by a @a Chromosome instance. * Individuals are stored together with a fitness value that can be computed by the fitness metric * associated with the population. + * The fitness is computed using the metric as soon as an individual is inserted into the population. */ class Population { @@ -78,8 +79,8 @@ public: std::vector _chromosomes = {} ): Population( - std::move(_fitnessMetric), - chromosomesToIndividuals(std::move(_chromosomes)) + _fitnessMetric, + chromosomesToIndividuals(*_fitnessMetric, std::move(_chromosomes)) ) {} static Population makeRandom( @@ -114,14 +115,15 @@ private: m_individuals{std::move(_individuals)} {} void doMutation(); - void doEvaluation(); void doSelection(); static void randomizeWorstChromosomes( + FitnessMetric const& _fitnessMetric, std::vector& _individuals, size_t _count ); static std::vector chromosomesToIndividuals( + FitnessMetric const& _fitnessMetric, std::vector _chromosomes ); static std::vector sortedIndividuals(std::vector _individuals); From cef01c961a02e1af9efbca41eb10d76b582ef911 Mon Sep 17 00:00:00 2001 From: cameel Date: Wed, 5 Feb 2020 16:49:50 +0100 Subject: [PATCH 25/92] [yul-phaser] Population: Keep the individuals always sorted --- test/yulPhaser/Population.cpp | 16 ++++++++++------ tools/yulPhaser/Population.cpp | 2 +- tools/yulPhaser/Population.h | 5 ++--- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/test/yulPhaser/Population.cpp b/test/yulPhaser/Population.cpp index 1b8e76da0..c4b127888 100644 --- a/test/yulPhaser/Population.cpp +++ b/test/yulPhaser/Population.cpp @@ -81,20 +81,24 @@ BOOST_AUTO_TEST_CASE(isFitter_should_return_false_for_identical_individuals) BOOST_TEST(!isFitter(Individual{Chromosome("acT"), 0}, Individual{Chromosome("acT"), 0})); } -BOOST_FIXTURE_TEST_CASE(constructor_should_copy_chromosomes_and_compute_fitness, PopulationFixture) +BOOST_FIXTURE_TEST_CASE(constructor_should_copy_chromosomes_compute_fitness_and_sort_chromosomes, PopulationFixture) { vector chromosomes = { Chromosome::makeRandom(5), + Chromosome::makeRandom(15), Chromosome::makeRandom(10), }; Population population(m_fitnessMetric, chromosomes); - BOOST_TEST(population.individuals().size() == 2); - BOOST_TEST(population.individuals()[0].chromosome == chromosomes[0]); - BOOST_TEST(population.individuals()[1].chromosome == chromosomes[1]); + vector const& individuals = population.individuals(); - BOOST_TEST(population.individuals()[0].fitness == m_fitnessMetric->evaluate(population.individuals()[0].chromosome)); - BOOST_TEST(population.individuals()[1].fitness == m_fitnessMetric->evaluate(population.individuals()[1].chromosome)); + BOOST_TEST(individuals.size() == 3); + BOOST_TEST(individuals[0].fitness == 5); + BOOST_TEST(individuals[1].fitness == 10); + BOOST_TEST(individuals[2].fitness == 15); + BOOST_TEST(individuals[0].chromosome == chromosomes[0]); + BOOST_TEST(individuals[1].chromosome == chromosomes[2]); + BOOST_TEST(individuals[2].chromosome == chromosomes[1]); } BOOST_FIXTURE_TEST_CASE(makeRandom_should_get_chromosome_lengths_from_specified_generator, PopulationFixture) diff --git a/tools/yulPhaser/Population.cpp b/tools/yulPhaser/Population.cpp index d258fbf3d..923e75e6a 100644 --- a/tools/yulPhaser/Population.cpp +++ b/tools/yulPhaser/Population.cpp @@ -128,8 +128,8 @@ void Population::doMutation() void Population::doSelection() { - m_individuals = sortedIndividuals(move(m_individuals)); randomizeWorstChromosomes(*m_fitnessMetric, m_individuals, m_individuals.size() / 2); + m_individuals = sortedIndividuals(move(m_individuals)); } void Population::randomizeWorstChromosomes( diff --git a/tools/yulPhaser/Population.h b/tools/yulPhaser/Population.h index 0dbac3235..bd58c397a 100644 --- a/tools/yulPhaser/Population.h +++ b/tools/yulPhaser/Population.h @@ -65,8 +65,7 @@ bool isFitter(Individual const& a, Individual const& b); * and selecting the best ones for the next round. * * An individual is a sequence of optimiser steps represented by a @a Chromosome instance. - * Individuals are stored together with a fitness value that can be computed by the fitness metric - * associated with the population. + * Individuals are always ordered by their fitness (based on @_fitnessMetric and @a isFitter()). * The fitness is computed using the metric as soon as an individual is inserted into the population. */ class Population @@ -112,7 +111,7 @@ public: private: explicit Population(std::shared_ptr _fitnessMetric, std::vector _individuals): m_fitnessMetric(std::move(_fitnessMetric)), - m_individuals{std::move(_individuals)} {} + m_individuals{sortedIndividuals(std::move(_individuals))} {} void doMutation(); void doSelection(); From 2291cf78ac5afa42598facb7f8e1f8b2cf99558d Mon Sep 17 00:00:00 2001 From: cameel Date: Wed, 5 Feb 2020 17:02:32 +0100 Subject: [PATCH 26/92] [yul-phaser] Population: Add constructors to Individual to simplify initialization --- test/yulPhaser/Population.cpp | 28 ++++++++++++++-------------- tools/yulPhaser/Population.cpp | 5 +---- tools/yulPhaser/Population.h | 7 +++++++ 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/test/yulPhaser/Population.cpp b/test/yulPhaser/Population.cpp index c4b127888..55c346575 100644 --- a/test/yulPhaser/Population.cpp +++ b/test/yulPhaser/Population.cpp @@ -53,32 +53,32 @@ BOOST_AUTO_TEST_SUITE(PopulationTest) BOOST_AUTO_TEST_CASE(isFitter_should_use_fitness_as_the_main_criterion) { - BOOST_TEST(isFitter(Individual{Chromosome("a"), 5}, Individual{Chromosome("a"), 10})); - BOOST_TEST(!isFitter(Individual{Chromosome("a"), 10}, Individual{Chromosome("a"), 5})); + BOOST_TEST(isFitter(Individual(Chromosome("a"), 5), Individual(Chromosome("a"), 10))); + BOOST_TEST(!isFitter(Individual(Chromosome("a"), 10), Individual(Chromosome("a"), 5))); - BOOST_TEST(isFitter(Individual{Chromosome("aaa"), 5}, Individual{Chromosome("aaaaa"), 10})); - BOOST_TEST(!isFitter(Individual{Chromosome("aaaaa"), 10}, Individual{Chromosome("aaa"), 5})); + BOOST_TEST(isFitter(Individual(Chromosome("aaa"), 5), Individual(Chromosome("aaaaa"), 10))); + BOOST_TEST(!isFitter(Individual(Chromosome("aaaaa"), 10), Individual(Chromosome("aaa"), 5))); - BOOST_TEST(isFitter(Individual{Chromosome("aaaaa"), 5}, Individual{Chromosome("aaa"), 10})); - BOOST_TEST(!isFitter(Individual{Chromosome("aaa"), 10}, Individual{Chromosome("aaaaa"), 5})); + BOOST_TEST(isFitter(Individual(Chromosome("aaaaa"), 5), Individual(Chromosome("aaa"), 10))); + BOOST_TEST(!isFitter(Individual(Chromosome("aaa"), 10), Individual(Chromosome("aaaaa"), 5))); } BOOST_AUTO_TEST_CASE(isFitter_should_use_alphabetical_order_when_fitness_is_the_same) { - BOOST_TEST(isFitter(Individual{Chromosome("a"), 3}, Individual{Chromosome("c"), 3})); - BOOST_TEST(!isFitter(Individual{Chromosome("c"), 3}, Individual{Chromosome("a"), 3})); + BOOST_TEST(isFitter(Individual(Chromosome("a"), 3), Individual(Chromosome("c"), 3))); + BOOST_TEST(!isFitter(Individual(Chromosome("c"), 3), Individual(Chromosome("a"), 3))); - BOOST_TEST(isFitter(Individual{Chromosome("a"), 3}, Individual{Chromosome("aa"), 3})); - BOOST_TEST(!isFitter(Individual{Chromosome("aa"), 3}, Individual{Chromosome("a"), 3})); + BOOST_TEST(isFitter(Individual(Chromosome("a"), 3), Individual(Chromosome("aa"), 3))); + BOOST_TEST(!isFitter(Individual(Chromosome("aa"), 3), Individual(Chromosome("a"), 3))); - BOOST_TEST(isFitter(Individual{Chromosome("T"), 3}, Individual{Chromosome("a"), 3})); - BOOST_TEST(!isFitter(Individual{Chromosome("a"), 3}, Individual{Chromosome("T"), 3})); + BOOST_TEST(isFitter(Individual(Chromosome("T"), 3), Individual(Chromosome("a"), 3))); + BOOST_TEST(!isFitter(Individual(Chromosome("a"), 3), Individual(Chromosome("T"), 3))); } BOOST_AUTO_TEST_CASE(isFitter_should_return_false_for_identical_individuals) { - BOOST_TEST(!isFitter(Individual{Chromosome("a"), 3}, Individual{Chromosome("a"), 3})); - BOOST_TEST(!isFitter(Individual{Chromosome("acT"), 0}, Individual{Chromosome("acT"), 0})); + BOOST_TEST(!isFitter(Individual(Chromosome("a"), 3), Individual(Chromosome("a"), 3))); + BOOST_TEST(!isFitter(Individual(Chromosome("acT"), 0), Individual(Chromosome("acT"), 0))); } BOOST_FIXTURE_TEST_CASE(constructor_should_copy_chromosomes_compute_fitness_and_sort_chromosomes, PopulationFixture) diff --git a/tools/yulPhaser/Population.cpp b/tools/yulPhaser/Population.cpp index 923e75e6a..2fed8b90d 100644 --- a/tools/yulPhaser/Population.cpp +++ b/tools/yulPhaser/Population.cpp @@ -157,10 +157,7 @@ vector Population::chromosomesToIndividuals( { vector individuals; for (auto& chromosome: _chromosomes) - { - size_t fitness = _fitnessMetric.evaluate(chromosome); - individuals.push_back({move(chromosome), fitness}); - } + individuals.emplace_back(move(chromosome), _fitnessMetric); return individuals; } diff --git a/tools/yulPhaser/Population.h b/tools/yulPhaser/Population.h index bd58c397a..bd15674d6 100644 --- a/tools/yulPhaser/Population.h +++ b/tools/yulPhaser/Population.h @@ -48,6 +48,13 @@ struct Individual Chromosome chromosome; size_t fitness; + Individual(Chromosome _chromosome, size_t _fitness): + chromosome(std::move(_chromosome)), + fitness(_fitness) {} + Individual(Chromosome _chromosome, FitnessMetric const& _fitnessMetric): + chromosome(std::move(_chromosome)), + fitness(_fitnessMetric.evaluate(chromosome)) {} + bool operator==(Individual const& _other) const { return fitness == _other.fitness && chromosome == _other.chromosome; } bool operator!=(Individual const& _other) const { return !(*this == _other); } From e8192e9aa34315b6bf29d4ee7a4e8ed095154c9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Wed, 5 Feb 2020 19:25:15 +0100 Subject: [PATCH 27/92] [yul-phaser] main: Set the number of optimisation sequence repetitions to 5 --- tools/yulPhaser/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/yulPhaser/main.cpp b/tools/yulPhaser/main.cpp index f8226e1b9..eea43e5fb 100644 --- a/tools/yulPhaser/main.cpp +++ b/tools/yulPhaser/main.cpp @@ -72,7 +72,7 @@ CharStream loadSource(string const& _sourcePath) void runAlgorithm(string const& _sourcePath) { CharStream sourceCode = loadSource(_sourcePath); - shared_ptr fitnessMetric = make_shared(Program::load(sourceCode)); + shared_ptr fitnessMetric = make_shared(Program::load(sourceCode), 5); auto population = Population::makeRandom( fitnessMetric, 10, From 98fb71f03f9013ba9f1d32e8ae0167579c2f1962 Mon Sep 17 00:00:00 2001 From: Bhargava Shastry Date: Thu, 20 Feb 2020 11:34:49 +0530 Subject: [PATCH 28/92] circleci: Use custom pipeline parameters per docker image revision in config --- .circleci/README.md | 2 +- .circleci/config.yml | 22 ++++++++++++++-------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/.circleci/README.md b/.circleci/README.md index 466aaa76a..b48adba24 100644 --- a/.circleci/README.md +++ b/.circleci/README.md @@ -11,7 +11,7 @@ docker build -t ethereum/solidity-buildpack-deps:ubuntu1904- -f Docker docker push ethereum/solidity-buildpack-deps:ubuntu1904- ``` -The current revision is stored in a [circle ci pipeline parameter](https://github.com/CircleCI-Public/api-preview-docs/blob/master/docs/pipeline-parameters.md#pipeline-parameters) called `docker-image-rev`. Please update the value assigned to this parameter at the time of a docker image update. Please verify that the value assigned to the parameter matches the revision part of the docker image tag (`` in the docker build/push snippet shown above). Otherwise, the docker image used by circle ci and the one actually pushed to docker hub will differ. +The current revisions per docker image are stored in [circle ci pipeline parameters](https://github.com/CircleCI-Public/api-preview-docs/blob/master/docs/pipeline-parameters.md#pipeline-parameters) called `-docker-image-rev` (e.g., `ubuntu-1904-docker-image-rev`). Please update the value assigned to the parameter(s) corresponding to the docker image(s) being updated at the time of the update. Please verify that the value assigned to the parameter matches the revision part of the docker image tag (`` in the docker build/push snippet shown above). Otherwise, the docker image used by circle ci and the one actually pushed to docker hub will differ. Once the docker image has been built and pushed to Dockerhub, you can find it at: diff --git a/.circleci/config.yml b/.circleci/config.yml index 1009d8217..3d95b0a73 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,7 +7,13 @@ # - ems: Emscripten version: 2.1 parameters: - docker-image-rev: + ubuntu-1804-docker-image-rev: + type: string + default: "4" + ubuntu-1904-docker-image-rev: + type: string + default: "4" + ubuntu-1904-clang-docker-image-rev: type: string default: "4" @@ -115,7 +121,7 @@ defaults: - test_ubuntu1904_clang: &test_ubuntu1904_clang docker: - - image: ethereum/solidity-buildpack-deps:ubuntu1904-clang-<< pipeline.parameters.docker-image-rev >> + - image: ethereum/solidity-buildpack-deps:ubuntu1904-clang-<< pipeline.parameters.ubuntu-1904-clang-docker-image-rev >> steps: - checkout - attach_workspace: @@ -126,7 +132,7 @@ defaults: - test_ubuntu1904: &test_ubuntu1904 docker: - - image: ethereum/solidity-buildpack-deps:ubuntu1904-<< pipeline.parameters.docker-image-rev >> + - image: ethereum/solidity-buildpack-deps:ubuntu1904-<< pipeline.parameters.ubuntu-1904-docker-image-rev >> steps: - checkout - attach_workspace: @@ -312,7 +318,7 @@ jobs: b_ubu_clang: &build_ubuntu1904_clang docker: - - image: ethereum/solidity-buildpack-deps:ubuntu1904-clang-<< pipeline.parameters.docker-image-rev >> + - image: ethereum/solidity-buildpack-deps:ubuntu1904-clang-<< pipeline.parameters.ubuntu-1904-clang-docker-image-rev >> environment: CC: clang CXX: clang++ @@ -324,7 +330,7 @@ jobs: b_ubu: &build_ubuntu1904 docker: - - image: ethereum/solidity-buildpack-deps:ubuntu1904-<< pipeline.parameters.docker-image-rev >> + - image: ethereum/solidity-buildpack-deps:ubuntu1904-<< pipeline.parameters.ubuntu-1904-docker-image-rev >> steps: - checkout - run: *run_build @@ -339,7 +345,7 @@ jobs: b_ubu18: &build_ubuntu1804 docker: - - image: ethereum/solidity-buildpack-deps:ubuntu1804-<< pipeline.parameters.docker-image-rev >> + - image: ethereum/solidity-buildpack-deps:ubuntu1804-<< pipeline.parameters.ubuntu-1804-docker-image-rev >> environment: CMAKE_OPTIONS: -DCMAKE_CXX_FLAGS=-O2 CMAKE_BUILD_TYPE: RelWithDebugInfo @@ -546,7 +552,7 @@ jobs: b_docs: docker: - - image: ethereum/solidity-buildpack-deps:ubuntu1904-<< pipeline.parameters.docker-image-rev >> + - image: ethereum/solidity-buildpack-deps:ubuntu1904-<< pipeline.parameters.ubuntu-1904-docker-image-rev >> steps: - checkout - run: *setup_prerelease_commit_hash @@ -571,7 +577,7 @@ jobs: t_ubu_cli: &t_ubu_cli docker: - - image: ethereum/solidity-buildpack-deps:ubuntu1904-<< pipeline.parameters.docker-image-rev >> + - image: ethereum/solidity-buildpack-deps:ubuntu1904-<< pipeline.parameters.ubuntu-1904-docker-image-rev >> environment: TERM: xterm steps: From dd9009eba6cc6f901123174a5932090a611c73ba Mon Sep 17 00:00:00 2001 From: Djordje Mijovic Date: Tue, 18 Feb 2020 12:57:48 +0100 Subject: [PATCH 29/92] TestFileParser: Adding new keyword wei for expressing function value --- Changelog.md | 3 +- test/libsolidity/SemanticTest.cpp | 6 +-- .../semanticTests/functionCall/value_test.sol | 8 +++ .../inlineAssembly/selfbalance.sol | 2 +- .../receive/empty_calldata_calls_receive.sol | 3 +- .../semanticTests/revertStrings/transfer.sol | 2 +- .../libsolidity/semanticTests/smoke/basic.sol | 3 +- .../semanticTests/smoke/constructor.sol | 2 +- .../semanticTests/smoke/fallback.sol | 4 +- test/libsolidity/util/BytesUtils.cpp | 2 +- test/libsolidity/util/SoltestTypes.h | 33 +++++++++++- test/libsolidity/util/TestFileParser.cpp | 16 ++++-- test/libsolidity/util/TestFileParser.h | 3 +- test/libsolidity/util/TestFileParserTests.cpp | 50 +++++++++++++------ test/libsolidity/util/TestFunctionCall.cpp | 4 +- .../util/TestFunctionCallTests.cpp | 26 +++++----- 16 files changed, 117 insertions(+), 50 deletions(-) create mode 100644 test/libsolidity/semanticTests/functionCall/value_test.sol diff --git a/Changelog.md b/Changelog.md index c83b26e83..9981ea25f 100644 --- a/Changelog.md +++ b/Changelog.md @@ -7,7 +7,7 @@ Compiler Features: Bugfixes: - + * isoltest: Added new keyword `wei` to express function value in semantic tests ### 0.6.3 (2020-02-18) @@ -31,7 +31,6 @@ Bugfixes: * Type Checker: Make invalid calls to uncallable types fatal errors instead of regular. - ### 0.6.2 (2020-01-27) Language Features: diff --git a/test/libsolidity/SemanticTest.cpp b/test/libsolidity/SemanticTest.cpp index 35ea80bd9..b1ef9372a 100644 --- a/test/libsolidity/SemanticTest.cpp +++ b/test/libsolidity/SemanticTest.cpp @@ -133,7 +133,7 @@ TestCase::TestResult SemanticTest::run(ostream& _stream, string const& _linePref else { if (test.call().isConstructor) - deploy("", test.call().value, test.call().arguments.rawBytes(), libraries); + deploy("", test.call().value.value, test.call().arguments.rawBytes(), libraries); else soltestAssert(deploy("", 0, bytes(), libraries), "Failed to deploy contract."); constructed = true; @@ -151,7 +151,7 @@ TestCase::TestResult SemanticTest::run(ostream& _stream, string const& _linePref { bytes output; if (test.call().useCallWithoutSignature) - output = callLowLevel(test.call().arguments.rawBytes(), test.call().value); + output = callLowLevel(test.call().arguments.rawBytes(), test.call().value.value); else { soltestAssert( @@ -161,7 +161,7 @@ TestCase::TestResult SemanticTest::run(ostream& _stream, string const& _linePref output = callContractFunctionWithValueNoEncoding( test.call().signature, - test.call().value, + test.call().value.value, test.call().arguments.rawBytes() ); } diff --git a/test/libsolidity/semanticTests/functionCall/value_test.sol b/test/libsolidity/semanticTests/functionCall/value_test.sol new file mode 100644 index 000000000..582cda9f6 --- /dev/null +++ b/test/libsolidity/semanticTests/functionCall/value_test.sol @@ -0,0 +1,8 @@ +contract C { + function f() public payable returns (uint) { + return msg.value; + } +} +// ---- +// f(), 1 ether -> 1000000000000000000 +// f(), 1 wei -> 1 \ No newline at end of file diff --git a/test/libsolidity/semanticTests/inlineAssembly/selfbalance.sol b/test/libsolidity/semanticTests/inlineAssembly/selfbalance.sol index 8a0a5caae..b16678833 100644 --- a/test/libsolidity/semanticTests/inlineAssembly/selfbalance.sol +++ b/test/libsolidity/semanticTests/inlineAssembly/selfbalance.sol @@ -9,4 +9,4 @@ contract C { // EVMVersion: >=istanbul // compileViaYul: also // ---- -// f(), 254 ether -> 254 +// f(), 254 wei -> 254 diff --git a/test/libsolidity/semanticTests/receive/empty_calldata_calls_receive.sol b/test/libsolidity/semanticTests/receive/empty_calldata_calls_receive.sol index 42f564139..ce04cd737 100644 --- a/test/libsolidity/semanticTests/receive/empty_calldata_calls_receive.sol +++ b/test/libsolidity/semanticTests/receive/empty_calldata_calls_receive.sol @@ -6,7 +6,8 @@ contract A { // x() -> 0 // () // x() -> 1 -// (), 1 ether +// (), 1 wei // x() -> 2 +// x(), 1 wei -> FAILURE // (): hex"00" -> FAILURE // (), 1 ether: hex"00" -> FAILURE diff --git a/test/libsolidity/semanticTests/revertStrings/transfer.sol b/test/libsolidity/semanticTests/revertStrings/transfer.sol index fcf971a5c..0d766bd1e 100644 --- a/test/libsolidity/semanticTests/revertStrings/transfer.sol +++ b/test/libsolidity/semanticTests/revertStrings/transfer.sol @@ -21,7 +21,7 @@ contract C { // EVMVersion: >=byzantium // revertStrings: debug // ---- -// (), 10 ether -> +// (), 10 wei -> // g() -> 10 // f() -> FAILURE, hex"08c379a0", 0x20, 10, "no_receive" // h() -> FAILURE diff --git a/test/libsolidity/semanticTests/smoke/basic.sol b/test/libsolidity/semanticTests/smoke/basic.sol index 403b13c7b..892ee8702 100644 --- a/test/libsolidity/semanticTests/smoke/basic.sol +++ b/test/libsolidity/semanticTests/smoke/basic.sol @@ -30,7 +30,8 @@ contract C { } // ---- // d() -> -// e(), 1 ether -> 1 +// e(), 1 wei -> 1 +// e(), 1 ether -> 1000000000000000000 // f(uint256): 3 -> 3, 3 // g() -> 2, 3 // h(uint256,uint256): 1, -2 -> 3 diff --git a/test/libsolidity/semanticTests/smoke/constructor.sol b/test/libsolidity/semanticTests/smoke/constructor.sol index 9aac2ed7b..e3bee950f 100644 --- a/test/libsolidity/semanticTests/smoke/constructor.sol +++ b/test/libsolidity/semanticTests/smoke/constructor.sol @@ -11,7 +11,7 @@ contract C { } } // ---- -// constructor(), 2 ether: 3 -> +// constructor(), 2 wei: 3 -> // state() -> 3 // balance() -> 2 // update(uint256): 4 diff --git a/test/libsolidity/semanticTests/smoke/fallback.sol b/test/libsolidity/semanticTests/smoke/fallback.sol index 72bc27ac7..ff0e65bff 100644 --- a/test/libsolidity/semanticTests/smoke/fallback.sol +++ b/test/libsolidity/semanticTests/smoke/fallback.sol @@ -16,8 +16,8 @@ contract A { // data() -> 2 // externalData() -> 0x20, 2, left(0x42ef) // balance() -> 0 -// (), 1 ether +// (), 1 wei // balance() -> 1 -// (), 2 ether: hex"fefe" +// (), 2 wei: hex"fefe" // balance() -> 2 // externalData() -> 0x20, 2, left(0xfefe) \ No newline at end of file diff --git a/test/libsolidity/util/BytesUtils.cpp b/test/libsolidity/util/BytesUtils.cpp index 6377fbe2e..0c5518afb 100644 --- a/test/libsolidity/util/BytesUtils.cpp +++ b/test/libsolidity/util/BytesUtils.cpp @@ -225,7 +225,7 @@ string BytesUtils::formatRawBytes( { bytes byteRange{it, it + static_cast(parameter.abiType.size)}; - os << _linePrefix << byteRange; + os << _linePrefix << formatBytes(byteRange, parameter.abiType); if (¶meter != ¶meters.back()) os << endl; diff --git a/test/libsolidity/util/SoltestTypes.h b/test/libsolidity/util/SoltestTypes.h index 0fec97eb1..4c1cb01fd 100644 --- a/test/libsolidity/util/SoltestTypes.h +++ b/test/libsolidity/util/SoltestTypes.h @@ -50,6 +50,7 @@ namespace solidity::frontend::test T(Identifier, "identifier", 0) \ /* type keywords */ \ K(Ether, "ether", 0) \ + K(Wei, "wei", 0) \ K(Hex, "hex", 0) \ K(Boolean, "boolean", 0) \ /* special keywords */ \ @@ -229,6 +230,32 @@ struct FunctionCallArgs } }; +/// Units that can be used to express function value +enum class FunctionValueUnit +{ + Wei, + Ether +}; + +/// Holds value along with unit it was expressed in originally. +/// Value should be always converted to wei, no meter on which unit it was originally +struct FunctionValue +{ + u256 value; + FunctionValueUnit unit = FunctionValueUnit::Wei; +}; + +inline bool operator==(FunctionValue const& _a, FunctionValue const& _b) +{ + return _a.value == _b.value; +} + +inline std::ostream& operator<<(std::ostream& _os, FunctionValue const& _v) +{ + _os << _v.value << (_v.unit == FunctionValueUnit::Wei ? " wei" : " ether"); + return _os; +} + /** * Represents a function call read from an input stream. It contains the signature, the * arguments, an optional ether value and an expected execution result. @@ -237,8 +264,10 @@ struct FunctionCall { /// Signature of the function call, e.g. `f(uint256, uint256)`. std::string signature; - /// Optional `ether` value that can be send with the call. - u256 value; + /// Optional value that can be sent with the call. + /// Value is expressed in wei, smallest unit of ether + /// Value has a field unit which represents denomination on which value was expressed originally + FunctionValue value; /// Object that holds all function parameters in their `bytes` /// representations given by the contract ABI. FunctionCallArgs arguments; diff --git a/test/libsolidity/util/TestFileParser.cpp b/test/libsolidity/util/TestFileParser.cpp index 4be96c88d..50ca68f63 100644 --- a/test/libsolidity/util/TestFileParser.cpp +++ b/test/libsolidity/util/TestFileParser.cpp @@ -88,6 +88,7 @@ vector TestFileParser::parseFunctionCall tie(call.signature, call.useCallWithoutSignature) = parseFunctionSignature(); if (accept(Token::Comma, true)) call.value = parseFunctionCallValue(); + if (accept(Token::Colon, true)) call.arguments = parseFunctionCallArguments(); @@ -199,13 +200,19 @@ pair TestFileParser::parseFunctionSignature() return {signature, !hasName}; } -u256 TestFileParser::parseFunctionCallValue() +FunctionValue TestFileParser::parseFunctionCallValue() { try { - u256 value{parseDecimalNumber()}; - expect(Token::Ether); - return value; + u256 value{ parseDecimalNumber() }; + Token token = m_scanner.currentToken(); + if (token != Token::Ether && token != Token::Wei) + throw Error(Error::Type::ParserError, "Invalid value unit provided. Coins can be wei or ether."); + + m_scanner.scanNextToken(); + + FunctionValueUnit unit = token == Token::Wei ? FunctionValueUnit::Wei : FunctionValueUnit::Ether; + return { (unit == FunctionValueUnit::Wei ? u256(1) : exp256(u256(10), u256(18))) * value, unit }; } catch (std::exception const&) { @@ -467,6 +474,7 @@ void TestFileParser::Scanner::scanNextToken() if (_literal == "true") return TokenDesc{Token::Boolean, _literal}; if (_literal == "false") return TokenDesc{Token::Boolean, _literal}; if (_literal == "ether") return TokenDesc{Token::Ether, _literal}; + if (_literal == "wei") return TokenDesc{Token::Wei, _literal}; if (_literal == "left") return TokenDesc{Token::Left, _literal}; if (_literal == "library") return TokenDesc{Token::Library, _literal}; if (_literal == "right") return TokenDesc{Token::Right, _literal}; diff --git a/test/libsolidity/util/TestFileParser.h b/test/libsolidity/util/TestFileParser.h index a10bb1114..502279d2d 100644 --- a/test/libsolidity/util/TestFileParser.h +++ b/test/libsolidity/util/TestFileParser.h @@ -39,6 +39,7 @@ namespace solidity::frontend::test * // f(uint256, uint256): 1, 1 # Signature and comma-separated list of arguments # * // -> 1, 1 # Expected result value # * // g(), 2 ether # (Optional) Ether to be send with the call # + * // g(), 1 wei # (Optional) Wei to be sent with the call # * // -> 2, 3 * // h(uint256), 1 ether: 42 * // -> FAILURE # If REVERT or other EVM failure was detected # @@ -134,7 +135,7 @@ private: /// Parses the optional ether value that can be passed alongside the /// function call arguments. Throws an InvalidEtherValueEncoding exception /// if given value cannot be converted to `u256`. - u256 parseFunctionCallValue(); + FunctionValue parseFunctionCallValue(); /// Parses a comma-separated list of arguments passed with a function call. /// Does not check for a potential mismatch between the signature and the number diff --git a/test/libsolidity/util/TestFileParserTests.cpp b/test/libsolidity/util/TestFileParserTests.cpp index c7e4cb22a..82d885b9e 100644 --- a/test/libsolidity/util/TestFileParserTests.cpp +++ b/test/libsolidity/util/TestFileParserTests.cpp @@ -51,7 +51,7 @@ void testFunctionCall( bool _failure = true, bytes _arguments = bytes{}, bytes _expectations = bytes{}, - u256 _value = 0, + FunctionValue _value = { 0 }, string _argumentComment = "", string _expectationComment = "", vector _rawArguments = vector{}, @@ -143,7 +143,7 @@ BOOST_AUTO_TEST_CASE(call_arguments_comments_success) false, fmt::encodeArgs(1, 1), fmt::encodeArgs(), - 0, + {0}, " Comment on the parameters. ", " This call should not return a value, but still succeed. " ); @@ -154,7 +154,7 @@ BOOST_AUTO_TEST_CASE(call_arguments_comments_success) false, fmt::encodeArgs(), fmt::encodeArgs(1), - 0, + {0}, " Comment on no parameters. ", " This comment should be parsed. " ); @@ -176,7 +176,7 @@ BOOST_AUTO_TEST_CASE(simple_single_line_call_comment_success) false, fmt::encodeArgs(1), fmt::encodeArgs(), - 0, + {0}, "", " f(uint256) does not return a value. " ); @@ -275,7 +275,7 @@ BOOST_AUTO_TEST_CASE(call_comments) false, fmt::encodeArgs(), fmt::encodeArgs(1), - 0, + {0}, " Parameter comment ", " Expectation comment " ); @@ -286,7 +286,7 @@ BOOST_AUTO_TEST_CASE(call_comments) false, fmt::encodeArgs(), fmt::encodeArgs(1), - 0, + {0}, " Parameter comment ", " Expectation comment " ); @@ -295,7 +295,7 @@ BOOST_AUTO_TEST_CASE(call_comments) BOOST_AUTO_TEST_CASE(call_arguments) { char const* source = R"( - // f(uint256), 314 ether: 5 # optional ether value # + // f(uint256), 314 wei: 5 # optional wei value # // -> 4 )"; auto const calls = parse(source); @@ -307,7 +307,27 @@ BOOST_AUTO_TEST_CASE(call_arguments) false, fmt::encodeArgs(5), fmt::encodeArgs(4), - 314, + {314}, + " optional wei value " + ); +} + +BOOST_AUTO_TEST_CASE(call_arguments_ether) +{ + char const* source = R"( + // f(uint256), 1 ether: 5 # optional ether value # + // -> 4 + )"; + auto const calls = parse(source); + BOOST_REQUIRE_EQUAL(calls.size(), 1); + testFunctionCall( + calls.at(0), + Mode::MultiLine, + "f(uint256)", + false, + fmt::encodeArgs(5), + fmt::encodeArgs(4), + {exp256(u256(10), u256(18)) , FunctionValueUnit::Ether}, " optional ether value " ); } @@ -521,7 +541,7 @@ BOOST_AUTO_TEST_CASE(call_arguments_tuple_of_tuples) false, fmt::encodeArgs(), fmt::encodeArgs(), - 0, + {0}, " f(S memory s, uint256 b) " ); } @@ -565,7 +585,7 @@ BOOST_AUTO_TEST_CASE(call_arguments_mismatch) false, fmt::encodeArgs(1, 2), fmt::encodeArgs(1), - 0, + {0}, " This only throws at runtime " ); } @@ -594,7 +614,7 @@ BOOST_AUTO_TEST_CASE(call_multiple_arguments) BOOST_AUTO_TEST_CASE(call_multiple_arguments_mixed_format) { char const* source = R"( - // test(uint256, uint256), 314 ether: + // test(uint256, uint256), 314 wei: // 1, -2 // -> -1, 2 )"; @@ -607,7 +627,7 @@ BOOST_AUTO_TEST_CASE(call_multiple_arguments_mixed_format) false, fmt::encodeArgs(1, -2), fmt::encodeArgs(-1, 2), - 314 + {314} ); } @@ -668,7 +688,7 @@ BOOST_AUTO_TEST_CASE(call_raw_arguments) false, fmt::encodeArgs(1, -2, -3), fmt::encodeArgs(), - 0, + {0}, "", "", {"1", "-2", "-3"} @@ -899,7 +919,7 @@ BOOST_AUTO_TEST_CASE(constructor) false, {}, {}, - 0, + {0}, "", "", {}, @@ -921,7 +941,7 @@ BOOST_AUTO_TEST_CASE(library) false, {}, {}, - 0, + {0}, "", "", {}, diff --git a/test/libsolidity/util/TestFunctionCall.cpp b/test/libsolidity/util/TestFunctionCall.cpp index 22dc82aa8..43f190349 100644 --- a/test/libsolidity/util/TestFunctionCall.cpp +++ b/test/libsolidity/util/TestFunctionCall.cpp @@ -63,8 +63,8 @@ string TestFunctionCall::format( /// Formats the function signature. This is the same independent from the display-mode. stream << _linePrefix << newline << ws << m_call.signature; - if (m_call.value > u256(0)) - stream << comma << ws << m_call.value << ws << ether; + if (m_call.value.value > u256(0)) + stream << comma << ws << m_call.value.value << ws << ether; if (!m_call.arguments.rawBytes().empty()) { string output = formatRawParameters(m_call.arguments.parameters, _linePrefix); diff --git a/test/libsolidity/util/TestFunctionCallTests.cpp b/test/libsolidity/util/TestFunctionCallTests.cpp index f3f977bbf..7b34cbe8e 100644 --- a/test/libsolidity/util/TestFunctionCallTests.cpp +++ b/test/libsolidity/util/TestFunctionCallTests.cpp @@ -40,7 +40,7 @@ BOOST_AUTO_TEST_CASE(format_unsigned_singleline) Parameter param{expectedBytes, "1", abiType, FormatInfo{}}; FunctionCallExpectations expectations{vector{param}, false, string{}}; FunctionCallArgs arguments{vector{param}, string{}}; - FunctionCall call{"f(uint8)", 0, arguments, expectations}; + FunctionCall call{"f(uint8)", {0}, arguments, expectations}; call.omitsArrow = false; TestFunctionCall test{call}; @@ -60,7 +60,7 @@ BOOST_AUTO_TEST_CASE(format_unsigned_singleline_signed_encoding) Parameter param{expectedBytes, "1", abiType, FormatInfo{}}; FunctionCallExpectations expectations{vector{param}, false, string{}}; FunctionCallArgs arguments{vector{param}, string{}}; - FunctionCall call{"f(uint8)", 0, arguments, expectations}; + FunctionCall call{"f(uint8)", {0}, arguments, expectations}; call.omitsArrow = false; TestFunctionCall test{call}; @@ -80,7 +80,7 @@ BOOST_AUTO_TEST_CASE(format_unsigned_multiline) Parameter result{expectedBytes, "1", abiType, FormatInfo{}}; FunctionCallExpectations expectations{vector{result}, false, string{}}; FunctionCallArgs arguments{vector{}, string{}}; - FunctionCall call{"f(uint8)", 0, arguments, expectations}; + FunctionCall call{"f(uint8)", {0}, arguments, expectations}; call.omitsArrow = false; call.displayMode = FunctionCall::DisplayMode::MultiLine; TestFunctionCall test{call}; @@ -95,7 +95,7 @@ BOOST_AUTO_TEST_CASE(format_multiple_unsigned_singleline) Parameter param{expectedBytes, "1", abiType, FormatInfo{}}; FunctionCallExpectations expectations{vector{param, param}, false, string{}}; FunctionCallArgs arguments{vector{param, param}, string{}}; - FunctionCall call{"f(uint8, uint8)", 0, arguments, expectations}; + FunctionCall call{"f(uint8, uint8)", {0}, arguments, expectations}; call.omitsArrow = false; TestFunctionCall test{call}; @@ -109,7 +109,7 @@ BOOST_AUTO_TEST_CASE(format_signed_singleline) Parameter param{expectedBytes, "-1", abiType, FormatInfo{}}; FunctionCallExpectations expectations{vector{param}, false, string{}}; FunctionCallArgs arguments{vector{param}, string{}}; - FunctionCall call{"f(int8)", 0, arguments, expectations}; + FunctionCall call{"f(int8)", {0}, arguments, expectations}; call.omitsArrow = false; TestFunctionCall test{call}; @@ -129,7 +129,7 @@ BOOST_AUTO_TEST_CASE(format_hex_singleline) Parameter param{expectedBytes, "0x31", abiType, FormatInfo{}}; FunctionCallExpectations expectations{vector{param}, false, string{}}; FunctionCallArgs arguments{vector{param}, string{}}; - FunctionCall call{"f(bytes32)", 0, arguments, expectations}; + FunctionCall call{"f(bytes32)", {0}, arguments, expectations}; call.omitsArrow = false; TestFunctionCall test{call}; @@ -151,7 +151,7 @@ BOOST_AUTO_TEST_CASE(format_hex_string_singleline) Parameter param{expectedBytes, "hex\"4200ef\"", abiType, FormatInfo{}}; FunctionCallExpectations expectations{vector{param}, false, string{}}; FunctionCallArgs arguments{vector{param}, string{}}; - FunctionCall call{"f(string)", 0, arguments, expectations}; + FunctionCall call{"f(string)", {0}, arguments, expectations}; call.omitsArrow = false; TestFunctionCall test{call}; @@ -165,7 +165,7 @@ BOOST_AUTO_TEST_CASE(format_bool_true_singleline) Parameter param{expectedBytes, "true", abiType, FormatInfo{}}; FunctionCallExpectations expectations{vector{param}, false, string{}}; FunctionCallArgs arguments{vector{param}, string{}}; - FunctionCall call{"f(bool)", 0, arguments, expectations}; + FunctionCall call{"f(bool)", {0}, arguments, expectations}; call.omitsArrow = false; TestFunctionCall test{call}; @@ -186,7 +186,7 @@ BOOST_AUTO_TEST_CASE(format_bool_false_singleline) Parameter param{expectedBytes, "false", abiType, FormatInfo{}}; FunctionCallExpectations expectations{vector{param}, false, string{}}; FunctionCallArgs arguments{vector{param}, string{}}; - FunctionCall call{"f(bool)", 0, arguments, expectations}; + FunctionCall call{"f(bool)", {0}, arguments, expectations}; call.omitsArrow = false; TestFunctionCall test{call}; @@ -200,7 +200,7 @@ BOOST_AUTO_TEST_CASE(format_bool_left_singleline) Parameter param{expectedBytes, "left(false)", abiType, FormatInfo{}}; FunctionCallExpectations expectations{vector{param}, false, string{}}; FunctionCallArgs arguments{vector{param}, string{}}; - FunctionCall call{"f(bool)", 0, arguments, expectations}; + FunctionCall call{"f(bool)", {0}, arguments, expectations}; call.omitsArrow = false; TestFunctionCall test{call}; @@ -215,7 +215,7 @@ BOOST_AUTO_TEST_CASE(format_hex_number_right_singleline) Parameter param{expectedBytes, "right(0x42)", abiType, FormatInfo{}}; FunctionCallExpectations expectations{vector{param}, false, string{}}; FunctionCallArgs arguments{vector{param}, string{}}; - FunctionCall call{"f(bool)", 0, arguments, expectations}; + FunctionCall call{"f(bool)", {0}, arguments, expectations}; call.omitsArrow = false; TestFunctionCall test{call}; @@ -229,7 +229,7 @@ BOOST_AUTO_TEST_CASE(format_empty_byte_range) Parameter param{expectedBytes, "1", abiType, FormatInfo{}}; FunctionCallExpectations expectations{vector{param}, false, string{}}; FunctionCallArgs arguments{vector{}, string{}}; - FunctionCall call{"f()", 0, arguments, expectations}; + FunctionCall call{"f()", {0}, arguments, expectations}; call.omitsArrow = false; TestFunctionCall test{call}; @@ -243,7 +243,7 @@ BOOST_AUTO_TEST_CASE(format_failure_singleline) Parameter param{expectedBytes, "1", abiType, FormatInfo{}}; FunctionCallExpectations expectations{vector{}, true, string{}}; FunctionCallArgs arguments{vector{param}, string{}}; - FunctionCall call{"f(uint8)", 0, arguments, expectations}; + FunctionCall call{"f(uint8)", {0}, arguments, expectations}; call.omitsArrow = false; TestFunctionCall test{call}; From 8b6bfabfee72561cb95b0b0125389d9c412e0e85 Mon Sep 17 00:00:00 2001 From: Bhargava Shastry Date: Thu, 20 Feb 2020 15:25:20 +0530 Subject: [PATCH 30/92] Docker: Upgrade libprotobuf-mutator inside clang docker image --- .circleci/config.yml | 2 +- .circleci/docker/Dockerfile.clang.ubuntu1904 | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3d95b0a73..f8f142412 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,7 +15,7 @@ parameters: default: "4" ubuntu-1904-clang-docker-image-rev: type: string - default: "4" + default: "5" defaults: diff --git a/.circleci/docker/Dockerfile.clang.ubuntu1904 b/.circleci/docker/Dockerfile.clang.ubuntu1904 index 14c9b1339..cf790c84e 100644 --- a/.circleci/docker/Dockerfile.clang.ubuntu1904 +++ b/.circleci/docker/Dockerfile.clang.ubuntu1904 @@ -51,9 +51,10 @@ ENV CC clang ENV CXX clang++ # Boost -RUN git clone --recursive -b boost-1.69.0 https://github.com/boostorg/boost.git \ +RUN git clone -b boost-1.69.0 https://github.com/boostorg/boost.git \ /usr/src/boost; \ cd /usr/src/boost; \ + git submodule update --init --recursive; \ ./bootstrap.sh --with-toolset=clang --prefix=/usr; \ ./b2 toolset=clang headers; \ ./b2 toolset=clang variant=release \ @@ -76,7 +77,7 @@ RUN set -ex; \ git clone https://github.com/google/libprotobuf-mutator.git \ /usr/src/libprotobuf-mutator; \ cd /usr/src/libprotobuf-mutator; \ - git checkout d1fe8a7d8ae18f3d454f055eba5213c291986f21; \ + git checkout 3521f47a2828da9ace403e4ecc4aece1a84feb36; \ mkdir build; \ cd build; \ cmake .. -GNinja -DLIB_PROTO_MUTATOR_DOWNLOAD_PROTOBUF=ON \ From 1f51716227853d222a0c970c31365939f66c1e73 Mon Sep 17 00:00:00 2001 From: chriseth Date: Thu, 16 Jan 2020 18:56:05 +0100 Subject: [PATCH 31/92] Type checking for Yul. --- libyul/AsmAnalysis.cpp | 110 +++++++++++++++++++++++++++++------------ libyul/AsmAnalysis.h | 9 +++- libyul/Dialect.cpp | 9 ++++ libyul/Dialect.h | 7 ++- 4 files changed, 101 insertions(+), 34 deletions(-) diff --git a/libyul/AsmAnalysis.cpp b/libyul/AsmAnalysis.cpp index 89db52d8d..91f818100 100644 --- a/libyul/AsmAnalysis.cpp +++ b/libyul/AsmAnalysis.cpp @@ -82,7 +82,6 @@ AsmAnalysisInfo AsmAnalyzer::analyzeStrictAssertCorrect(Dialect const& _dialect, vector AsmAnalyzer::operator()(Literal const& _literal) { expectValidType(_literal.type, _literal.location); - if (_literal.kind == LiteralKind::String && _literal.value.str().size() > 32) typeError( _literal.location, @@ -93,6 +92,13 @@ vector AsmAnalyzer::operator()(Literal const& _literal) else if (_literal.kind == LiteralKind::Boolean) yulAssert(_literal.value == "true"_yulstring || _literal.value == "false"_yulstring, ""); + if (!m_dialect.validTypeForLiteral(_literal.kind, _literal.value, _literal.type)) + typeError( + _literal.location, + "Invalid type \"" + _literal.type.str() + "\" for literal \"" + _literal.value.str() + "\"." + ); + + return {_literal.type}; } @@ -179,7 +185,8 @@ void AsmAnalyzer::operator()(Assignment const& _assignment) ); for (size_t i = 0; i < numVariables; ++i) - checkAssignment(_assignment.variableNames[i]); + if (i < types.size()) + checkAssignment(_assignment.variableNames[i], types[i]); } void AsmAnalyzer::operator()(VariableDeclaration const& _varDecl) @@ -193,6 +200,9 @@ void AsmAnalyzer::operator()(VariableDeclaration const& _varDecl) yul::IdentifierContext::VariableDeclaration, m_currentScope->insideFunction() ); + for (auto const& variable: _varDecl.variables) + expectValidType(variable.type, variable.location); + if (_varDecl.value) { vector types = std::visit(*this, *_varDecl.value); @@ -204,15 +214,25 @@ void AsmAnalyzer::operator()(VariableDeclaration const& _varDecl) to_string(types.size()) + " values." ); + + for (size_t i = 0; i < _varDecl.variables.size(); ++i) + { + YulString givenType = m_dialect.defaultType; + if (i < types.size()) + givenType = types[i]; + TypedName const& variable = _varDecl.variables[i]; + if (variable.type != givenType) + typeError( + variable.location, + "Assigning value of type \"" + givenType.str() + "\" to variable of type \"" + variable.type.str() + "." + ); + } } for (TypedName const& variable: _varDecl.variables) - { - expectValidType(variable.type, variable.location); m_activeVariables.insert(&std::get( m_currentScope->identifiers.at(variable.name)) ); - } } void AsmAnalyzer::operator()(FunctionDefinition const& _funDef) @@ -291,6 +311,11 @@ vector AsmAnalyzer::operator()(FunctionCall const& _funCall) ); } } + std::reverse(argTypes.begin(), argTypes.end()); + + if (parameterTypes && parameterTypes->size() == argTypes.size()) + for (size_t i = 0; i < parameterTypes->size(); ++i) + expectType((*parameterTypes)[i], argTypes[i], locationOf(_funCall.arguments[i])); if (m_success) { @@ -306,7 +331,7 @@ vector AsmAnalyzer::operator()(FunctionCall const& _funCall) void AsmAnalyzer::operator()(If const& _if) { - expectExpression(*_if.condition); + expectBoolExpression(*_if.condition); (*this)(_if.body); } @@ -315,32 +340,15 @@ void AsmAnalyzer::operator()(Switch const& _switch) { yulAssert(_switch.expression, ""); - expectExpression(*_switch.expression); - - YulString caseType; - bool mismatchingTypes = false; - for (auto const& _case: _switch.cases) - if (_case.value) - { - if (caseType.empty()) - caseType = _case.value->type; - else if (caseType != _case.value->type) - { - mismatchingTypes = true; - break; - } - } - if (mismatchingTypes) - m_errorReporter.typeError( - _switch.location, - "Switch cases have non-matching types." - ); + YulString valueType = expectExpression(*_switch.expression); set cases; for (auto const& _case: _switch.cases) { if (_case.value) { + expectType(valueType, _case.value->type, _case.value->location); + // We cannot use "expectExpression" here because *_case.value is not an // Expression and would be converted to an Expression otherwise. (*this)(*_case.value); @@ -366,8 +374,7 @@ void AsmAnalyzer::operator()(ForLoop const& _for) // condition, the body and the post part inside. m_currentScope = &scope(&_for.pre); - expectExpression(*_for.condition); - + expectBoolExpression(*_for.condition); // backup outer for-loop & create new state auto outerForLoop = m_currentForLoop; m_currentForLoop = &_for; @@ -403,10 +410,24 @@ YulString AsmAnalyzer::expectExpression(Expression const& _expr) return types.empty() ? m_dialect.defaultType : types.front(); } -void AsmAnalyzer::checkAssignment(Identifier const& _variable) +void AsmAnalyzer::expectBoolExpression(Expression const& _expr) +{ + YulString type = expectExpression(_expr); + if (type != m_dialect.boolType) + typeError(locationOf(_expr), + "Expected a value of boolean type \"" + + m_dialect.boolType.str() + + "\" but got \"" + + type.str() + + "\"" + ); +} + +void AsmAnalyzer::checkAssignment(Identifier const& _variable, YulString _valueType) { yulAssert(!_variable.name.empty(), ""); size_t numErrorsBefore = m_errorReporter.errors().size(); + YulString const* variableType = nullptr; bool found = false; if (Scope::Identifier const* var = m_currentScope->lookup(_variable.name)) { @@ -418,6 +439,8 @@ void AsmAnalyzer::checkAssignment(Identifier const& _variable) _variable.location, "Variable " + _variable.name.str() + " used before it was declared." ); + else + variableType = &std::get(*var).type; found = true; } else if (m_resolver) @@ -427,6 +450,7 @@ void AsmAnalyzer::checkAssignment(Identifier const& _variable) if (variableSize != size_t(-1)) { found = true; + variableType = &m_dialect.defaultType; yulAssert(variableSize == 1, "Invalid stack size of external reference."); } } @@ -438,6 +462,17 @@ void AsmAnalyzer::checkAssignment(Identifier const& _variable) if (numErrorsBefore == m_errorReporter.errors().size()) declarationError(_variable.location, "Variable not found or variable not lvalue."); } + if (variableType && *variableType != _valueType) + typeError(_variable.location, + "Assigning a value of type \"" + + _valueType.str() + + "\" to a variable of type \"" + + variableType->str() + + "\"." + ); + + if (m_success) + yulAssert(variableType, ""); } Scope& AsmAnalyzer::scope(Block const* _block) @@ -447,15 +482,28 @@ Scope& AsmAnalyzer::scope(Block const* _block) yulAssert(scopePtr, "Scope requested but not present."); return *scopePtr; } + void AsmAnalyzer::expectValidType(YulString _type, SourceLocation const& _location) { - if (!_type.empty() && !m_dialect.types.count(_type)) - m_errorReporter.typeError( + if (!m_dialect.types.count(_type)) + typeError( _location, "\"" + _type.str() + "\" is not a valid type (user defined types are not yet supported)." ); } +void AsmAnalyzer::expectType(YulString _expectedType, YulString _givenType, SourceLocation const& _location) +{ + if (_expectedType != _givenType) + typeError(_location, + "Expected a value of type \"" + + _expectedType.str() + + "\" but got \"" + + _givenType.str() + + "\"" + ); +} + bool AsmAnalyzer::warnOnInstructions(std::string const& _instructionIdentifier, langutil::SourceLocation const& _location) { auto const builtin = EVMDialect::strictAssemblyForEVM(EVMVersion{}).builtin(YulString(_instructionIdentifier)); diff --git a/libyul/AsmAnalysis.h b/libyul/AsmAnalysis.h index dded6d5ab..853f9f654 100644 --- a/libyul/AsmAnalysis.h +++ b/libyul/AsmAnalysis.h @@ -96,13 +96,18 @@ private: /// Visits the expression, expects that it evaluates to exactly one value and /// returns the type. Reports errors on errors and returns the default type. YulString expectExpression(Expression const& _expr); + /// Vists the expression and expects it to return a single boolean value. + /// Reports an error otherwise. + void expectBoolExpression(Expression const& _expr); bool expectDeposit(int _deposit, int _oldHeight, langutil::SourceLocation const& _location); - /// Verifies that a variable to be assigned to exists and can be assigned to. - void checkAssignment(Identifier const& _variable); + /// Verifies that a variable to be assigned to exists, can be assigned to + /// and has the same type as the value. + void checkAssignment(Identifier const& _variable, YulString _valueType); Scope& scope(Block const* _block); void expectValidType(YulString _type, langutil::SourceLocation const& _location); + void expectType(YulString _expectedType, YulString _givenType, langutil::SourceLocation const& _location); bool warnOnInstructions(evmasm::Instruction _instr, langutil::SourceLocation const& _location); bool warnOnInstructions(std::string const& _instrIdentifier, langutil::SourceLocation const& _location); diff --git a/libyul/Dialect.cpp b/libyul/Dialect.cpp index 6bc056a75..4d5cccd41 100644 --- a/libyul/Dialect.cpp +++ b/libyul/Dialect.cpp @@ -19,10 +19,19 @@ */ #include +#include using namespace solidity::yul; using namespace std; +bool Dialect::validTypeForLiteral(LiteralKind _kind, YulString, YulString _type) const +{ + if (_kind == LiteralKind::Boolean) + return _type == boolType; + else + return true; +} + Dialect const& Dialect::yulDeprecated() { static unique_ptr dialect; diff --git a/libyul/Dialect.h b/libyul/Dialect.h index c25ba1ac3..3d261fdee 100644 --- a/libyul/Dialect.h +++ b/libyul/Dialect.h @@ -33,6 +33,7 @@ namespace solidity::yul class YulString; using Type = YulString; +enum class LiteralKind; struct BuiltinFunction { @@ -52,7 +53,7 @@ struct Dialect: boost::noncopyable YulString defaultType; /// Type used for the literals "true" and "false". YulString boolType; - std::set types; + std::set types = {{}}; /// @returns the builtin function of the given name or a nullptr if it is not a builtin function. virtual BuiltinFunction const* builtin(YulString /*_name*/) const { return nullptr; } @@ -61,6 +62,10 @@ struct Dialect: boost::noncopyable virtual BuiltinFunction const* equalityFunction() const { return nullptr; } virtual BuiltinFunction const* booleanNegationFunction() const { return nullptr; } + /// Check whether the given type is legal for the given literal value. + /// Should only be called if the type exists in the dialect at all. + virtual bool validTypeForLiteral(LiteralKind _kind, YulString _value, YulString _type) const; + virtual std::set fixedFunctionNames() const { return {}; } Dialect() = default; From 6eec968365e22c28975dce970ddd93223316f41d Mon Sep 17 00:00:00 2001 From: chriseth Date: Wed, 22 Jan 2020 20:11:33 +0100 Subject: [PATCH 32/92] Test updates. --- test/libyul/Inliner.cpp | 2 +- test/libyul/Parser.cpp | 22 ++----------------- .../disambiguator/for_statement.yul | 7 ++++-- 3 files changed, 8 insertions(+), 23 deletions(-) diff --git a/test/libyul/Inliner.cpp b/test/libyul/Inliner.cpp index e564ac2b8..de47f4885 100644 --- a/test/libyul/Inliner.cpp +++ b/test/libyul/Inliner.cpp @@ -85,7 +85,7 @@ BOOST_AUTO_TEST_CASE(simple_inside_structures) BOOST_CHECK_EQUAL(inlinableFunctions("{" "function g(a:u256) -> b:u256 { b := a }" "for {" - "} 1:u256 {" + "} true {" "function f() -> x:u256 { x := g(2:u256) }" "}" "{" diff --git a/test/libyul/Parser.cpp b/test/libyul/Parser.cpp index 8ec551258..020f3d4c4 100644 --- a/test/libyul/Parser.cpp +++ b/test/libyul/Parser.cpp @@ -245,16 +245,6 @@ BOOST_AUTO_TEST_CASE(optional_types) BOOST_CHECK(successParse("{ function f(a:u256) -> b {} }")); } -BOOST_AUTO_TEST_CASE(invalid_types) -{ - /// testing invalid literal - /// NOTE: these will need to change when types are compared - CHECK_ERROR("{ let x:bool := 1:invalid }", TypeError, "\"invalid\" is not a valid type (user defined types are not yet supported)."); - /// testing invalid variable declaration - CHECK_ERROR("{ let x:invalid := 1:bool }", TypeError, "\"invalid\" is not a valid type (user defined types are not yet supported)."); - CHECK_ERROR("{ function f(a:invalid) {} }", TypeError, "\"invalid\" is not a valid type (user defined types are not yet supported)."); -} - BOOST_AUTO_TEST_CASE(number_literals) { BOOST_CHECK(successParse("{ let x:u256 := 1:u256 }")); @@ -268,7 +258,7 @@ BOOST_AUTO_TEST_CASE(builtin_types) { BOOST_CHECK(successParse("{ let x:bool := true:bool }")); BOOST_CHECK(successParse("{ let x:u8 := 1:u8 }")); - BOOST_CHECK(successParse("{ let x:s8 := 1:u8 }")); + BOOST_CHECK(successParse("{ let x:s8 := 1:s8 }")); BOOST_CHECK(successParse("{ let x:u32 := 1:u32 }")); BOOST_CHECK(successParse("{ let x:s32 := 1:s32 }")); BOOST_CHECK(successParse("{ let x:u64 := 1:u64 }")); @@ -495,15 +485,7 @@ BOOST_AUTO_TEST_CASE(if_statement_invalid) { CHECK_ERROR("{ if let x:u256 {} }", ParserError, "Literal or identifier expected."); CHECK_ERROR("{ if true:bool let x:u256 := 3:u256 }", ParserError, "Expected '{' but got reserved keyword 'let'"); - // TODO change this to an error once we check types. - BOOST_CHECK(successParse("{ if 42:u256 { } }")); -} - -BOOST_AUTO_TEST_CASE(switch_case_types) -{ - CHECK_ERROR("{ switch 0:u256 case 0:u256 {} case 1:u32 {} }", TypeError, "Switch cases have non-matching types."); - // The following should be an error in the future, but this is not yet detected. - BOOST_CHECK(successParse("{ switch 0:u256 case 0:u32 {} case 1:u32 {} }")); + CHECK_ERROR("{ if 42:u256 { } }", TypeError, "Expected a value of boolean type"); } BOOST_AUTO_TEST_CASE(switch_duplicate_case) diff --git a/test/libyul/yulOptimizerTests/disambiguator/for_statement.yul b/test/libyul/yulOptimizerTests/disambiguator/for_statement.yul index ce79ef5e6..c6b246ad0 100644 --- a/test/libyul/yulOptimizerTests/disambiguator/for_statement.yul +++ b/test/libyul/yulOptimizerTests/disambiguator/for_statement.yul @@ -1,7 +1,8 @@ { { let a:u256, b:u256 } { - for { let a:u256 } a { a := a } { + function eq(x: u256, y: u256) -> z: bool {} + for { let a:u256 } eq(a, a) { a := a } { let b:u256 := a } } @@ -13,7 +14,9 @@ // { // { let a, b } // { -// for { let a_1 } a_1 { a_1 := a_1 } +// function eq(x, y) -> z:bool +// { } +// for { let a_1 } eq(a_1, a_1) { a_1 := a_1 } // { let b_2 := a_1 } // } // } From 9140bc52c44a32d828e51ac480159c7fd1c4a448 Mon Sep 17 00:00:00 2001 From: chriseth Date: Wed, 19 Feb 2020 16:30:35 +0100 Subject: [PATCH 33/92] Tests for invalid types. --- test/libyul/yulSyntaxTests/invalid_type.yul | 7 +++++++ test/libyul/yulSyntaxTests/invalid_type2.yul | 8 ++++++++ test/libyul/yulSyntaxTests/invalid_type3.yul | 8 ++++++++ test/libyul/yulSyntaxTests/invalid_type4.yul | 9 +++++++++ 4 files changed, 32 insertions(+) create mode 100644 test/libyul/yulSyntaxTests/invalid_type.yul create mode 100644 test/libyul/yulSyntaxTests/invalid_type2.yul create mode 100644 test/libyul/yulSyntaxTests/invalid_type3.yul create mode 100644 test/libyul/yulSyntaxTests/invalid_type4.yul diff --git a/test/libyul/yulSyntaxTests/invalid_type.yul b/test/libyul/yulSyntaxTests/invalid_type.yul new file mode 100644 index 000000000..ec16ebf01 --- /dev/null +++ b/test/libyul/yulSyntaxTests/invalid_type.yul @@ -0,0 +1,7 @@ +{ + let x: invalidType +} +// ==== +// dialect: evmTyped +// ---- +// TypeError: (10-24): "invalidType" is not a valid type (user defined types are not yet supported). diff --git a/test/libyul/yulSyntaxTests/invalid_type2.yul b/test/libyul/yulSyntaxTests/invalid_type2.yul new file mode 100644 index 000000000..04593e666 --- /dev/null +++ b/test/libyul/yulSyntaxTests/invalid_type2.yul @@ -0,0 +1,8 @@ +{ + let x := 1:invalidType +} +// ==== +// dialect: evmTyped +// ---- +// TypeError: (15-28): "invalidType" is not a valid type (user defined types are not yet supported). +// TypeError: (10-11): Assigning value of type "invalidType" to variable of type "u256. diff --git a/test/libyul/yulSyntaxTests/invalid_type3.yul b/test/libyul/yulSyntaxTests/invalid_type3.yul new file mode 100644 index 000000000..cf92ce7b8 --- /dev/null +++ b/test/libyul/yulSyntaxTests/invalid_type3.yul @@ -0,0 +1,8 @@ +{ + function f(a: invalidType) -> b: invalidType {} +} +// ==== +// dialect: evmTyped +// ---- +// TypeError: (17-31): "invalidType" is not a valid type (user defined types are not yet supported). +// TypeError: (36-50): "invalidType" is not a valid type (user defined types are not yet supported). diff --git a/test/libyul/yulSyntaxTests/invalid_type4.yul b/test/libyul/yulSyntaxTests/invalid_type4.yul new file mode 100644 index 000000000..faf217a88 --- /dev/null +++ b/test/libyul/yulSyntaxTests/invalid_type4.yul @@ -0,0 +1,9 @@ +{ + switch 1 + case 8: invalidType {} +} +// ==== +// dialect: evmTyped +// ---- +// TypeError: (24-38): Expected a value of type "u256" but got "invalidType" +// TypeError: (24-38): "invalidType" is not a valid type (user defined types are not yet supported). From 2ee748b7f57bcd5696e8b778329a987d41020135 Mon Sep 17 00:00:00 2001 From: chriseth Date: Thu, 6 Feb 2020 19:44:14 +0100 Subject: [PATCH 34/92] Add tests --- test/libyul/yulSyntaxTests/assignment_fail.yul | 12 ++++++++++++ test/libyul/yulSyntaxTests/for_loop_condition.yul | 7 +++++++ .../yulSyntaxTests/for_loop_condition_fail.yul | 8 ++++++++ .../yulSyntaxTests/for_loop_condition_fail_ewasm.yul | 8 ++++++++ test/libyul/yulSyntaxTests/type_check_cases.yul | 8 ++++++++ test/libyul/yulSyntaxTests/type_check_cases_fail.yul | 10 ++++++++++ .../type_check_cases_fail_evmtyped.yul | 10 ++++++++++ .../yulSyntaxTests/type_check_if_condition.yul | 7 +++++++ .../yulSyntaxTests/type_check_if_condition_fail.yul | 8 ++++++++ .../yulSyntaxTests/user_defined_functions_fail.yul | 12 ++++++++++++ .../yulSyntaxTests/user_defined_functions_fine.yul | 10 ++++++++++ 11 files changed, 100 insertions(+) create mode 100644 test/libyul/yulSyntaxTests/assignment_fail.yul create mode 100644 test/libyul/yulSyntaxTests/for_loop_condition.yul create mode 100644 test/libyul/yulSyntaxTests/for_loop_condition_fail.yul create mode 100644 test/libyul/yulSyntaxTests/for_loop_condition_fail_ewasm.yul create mode 100644 test/libyul/yulSyntaxTests/type_check_cases.yul create mode 100644 test/libyul/yulSyntaxTests/type_check_cases_fail.yul create mode 100644 test/libyul/yulSyntaxTests/type_check_cases_fail_evmtyped.yul create mode 100644 test/libyul/yulSyntaxTests/type_check_if_condition.yul create mode 100644 test/libyul/yulSyntaxTests/type_check_if_condition_fail.yul create mode 100644 test/libyul/yulSyntaxTests/user_defined_functions_fail.yul create mode 100644 test/libyul/yulSyntaxTests/user_defined_functions_fine.yul diff --git a/test/libyul/yulSyntaxTests/assignment_fail.yul b/test/libyul/yulSyntaxTests/assignment_fail.yul new file mode 100644 index 000000000..59c18bfd3 --- /dev/null +++ b/test/libyul/yulSyntaxTests/assignment_fail.yul @@ -0,0 +1,12 @@ +{ + let x:u256 + let y := x + let z:bool + z := y + y := z +} +// ==== +// dialect: evmTyped +// ---- +// TypeError: (51-52): Assigning a value of type "u256" to a variable of type "bool". +// TypeError: (62-63): Assigning a value of type "bool" to a variable of type "u256". diff --git a/test/libyul/yulSyntaxTests/for_loop_condition.yul b/test/libyul/yulSyntaxTests/for_loop_condition.yul new file mode 100644 index 000000000..72ed7b71f --- /dev/null +++ b/test/libyul/yulSyntaxTests/for_loop_condition.yul @@ -0,0 +1,7 @@ +{ + let x:bool + for {} x {} {} +} +// ==== +// dialect: evmTyped +// ---- diff --git a/test/libyul/yulSyntaxTests/for_loop_condition_fail.yul b/test/libyul/yulSyntaxTests/for_loop_condition_fail.yul new file mode 100644 index 000000000..903f5471a --- /dev/null +++ b/test/libyul/yulSyntaxTests/for_loop_condition_fail.yul @@ -0,0 +1,8 @@ +{ + let x + for {} x {} {} +} +// ==== +// dialect: evmTyped +// ---- +// TypeError: (23-24): Expected a value of boolean type "bool" but got "u256" diff --git a/test/libyul/yulSyntaxTests/for_loop_condition_fail_ewasm.yul b/test/libyul/yulSyntaxTests/for_loop_condition_fail_ewasm.yul new file mode 100644 index 000000000..cbe266f9f --- /dev/null +++ b/test/libyul/yulSyntaxTests/for_loop_condition_fail_ewasm.yul @@ -0,0 +1,8 @@ +{ + let x + for {} x {} {} +} +// ==== +// dialect: ewasm +// ---- +// TypeError: (23-24): Expected a value of boolean type "i32" but got "i64" diff --git a/test/libyul/yulSyntaxTests/type_check_cases.yul b/test/libyul/yulSyntaxTests/type_check_cases.yul new file mode 100644 index 000000000..4e09457c8 --- /dev/null +++ b/test/libyul/yulSyntaxTests/type_check_cases.yul @@ -0,0 +1,8 @@ +{ + switch 7:i64 + case 0:i64 {} + case 2:i64 {} +} +// ==== +// dialect: ewasm +// ---- diff --git a/test/libyul/yulSyntaxTests/type_check_cases_fail.yul b/test/libyul/yulSyntaxTests/type_check_cases_fail.yul new file mode 100644 index 000000000..c799bec4b --- /dev/null +++ b/test/libyul/yulSyntaxTests/type_check_cases_fail.yul @@ -0,0 +1,10 @@ +{ + switch 7:i32 + case 0:i64 {} + case 2:i64 {} +} +// ==== +// dialect: ewasm +// ---- +// TypeError: (28-33): Expected a value of type "i32" but got "i64" +// TypeError: (46-51): Expected a value of type "i32" but got "i64" diff --git a/test/libyul/yulSyntaxTests/type_check_cases_fail_evmtyped.yul b/test/libyul/yulSyntaxTests/type_check_cases_fail_evmtyped.yul new file mode 100644 index 000000000..c1db211f0 --- /dev/null +++ b/test/libyul/yulSyntaxTests/type_check_cases_fail_evmtyped.yul @@ -0,0 +1,10 @@ +{ + switch 7 + case true:bool {} + case true:bool {} +} +// ==== +// dialect: evmTyped +// ---- +// TypeError: (24-33): Expected a value of type "u256" but got "bool" +// TypeError: (46-55): Expected a value of type "u256" but got "bool" diff --git a/test/libyul/yulSyntaxTests/type_check_if_condition.yul b/test/libyul/yulSyntaxTests/type_check_if_condition.yul new file mode 100644 index 000000000..eaccf63f6 --- /dev/null +++ b/test/libyul/yulSyntaxTests/type_check_if_condition.yul @@ -0,0 +1,7 @@ +{ + let x:i32 + if x {} +} +// ==== +// dialect: ewasm +// ---- diff --git a/test/libyul/yulSyntaxTests/type_check_if_condition_fail.yul b/test/libyul/yulSyntaxTests/type_check_if_condition_fail.yul new file mode 100644 index 000000000..a79d50e34 --- /dev/null +++ b/test/libyul/yulSyntaxTests/type_check_if_condition_fail.yul @@ -0,0 +1,8 @@ +{ + let x:i64 + if x {} +} +// ==== +// dialect: ewasm +// ---- +// TypeError: (23-24): Expected a value of boolean type "i32" but got "i64" diff --git a/test/libyul/yulSyntaxTests/user_defined_functions_fail.yul b/test/libyul/yulSyntaxTests/user_defined_functions_fail.yul new file mode 100644 index 000000000..324f937d4 --- /dev/null +++ b/test/libyul/yulSyntaxTests/user_defined_functions_fail.yul @@ -0,0 +1,12 @@ +{ + function f(a:u256, b:u256, c:bool) -> r:bool, t { + r := lt(a, b) + t := bool_to_u256(not(c)) + } + let x, y: bool := f(1, 2: u256, true) +} +// ==== +// dialect: evmTyped +// ---- +// TypeError: (126-127): Assigning value of type "bool" to variable of type "u256. +// TypeError: (129-136): Assigning value of type "u256" to variable of type "bool. diff --git a/test/libyul/yulSyntaxTests/user_defined_functions_fine.yul b/test/libyul/yulSyntaxTests/user_defined_functions_fine.yul new file mode 100644 index 000000000..e24ea923e --- /dev/null +++ b/test/libyul/yulSyntaxTests/user_defined_functions_fine.yul @@ -0,0 +1,10 @@ +{ + function f(a:u256, b:u256, c:bool) -> r:bool, t { + r := lt(a, b) + t := bool_to_u256(not(c)) + } + let x: bool, y: u256 := f(1, 2: u256, true) +} +// ==== +// dialect: evmTyped +// ---- From 915cb65106f1f1006974df752842f011b74a3ede Mon Sep 17 00:00:00 2001 From: chriseth Date: Tue, 11 Feb 2020 21:57:52 +0100 Subject: [PATCH 35/92] Collect types together with names. --- libyul/CMakeLists.txt | 2 + libyul/optimiser/TypeInfo.cpp | 98 +++++++++++++++++++++++++++++++++++ libyul/optimiser/TypeInfo.h | 61 ++++++++++++++++++++++ 3 files changed, 161 insertions(+) create mode 100644 libyul/optimiser/TypeInfo.cpp create mode 100644 libyul/optimiser/TypeInfo.h diff --git a/libyul/CMakeLists.txt b/libyul/CMakeLists.txt index 12d2866c2..65269e9e0 100644 --- a/libyul/CMakeLists.txt +++ b/libyul/CMakeLists.txt @@ -154,6 +154,8 @@ add_library(yul optimiser/Suite.h optimiser/SyntacticalEquality.cpp optimiser/SyntacticalEquality.h + optimiser/TypeInfo.cpp + optimiser/TypeInfo.h optimiser/UnusedPruner.cpp optimiser/UnusedPruner.h optimiser/VarDeclInitializer.cpp diff --git a/libyul/optimiser/TypeInfo.cpp b/libyul/optimiser/TypeInfo.cpp new file mode 100644 index 000000000..4b5715baa --- /dev/null +++ b/libyul/optimiser/TypeInfo.cpp @@ -0,0 +1,98 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ +/** + * Helper class that keeps track of the types while performing optimizations. + */ + +#include + +#include + +#include +#include + +#include + +using namespace std; +using namespace solidity::yul; +using namespace solidity::util; + +class TypeInfo::TypeCollector: public ASTWalker +{ +public: + explicit TypeCollector(Block const& _block) + { + (*this)(_block); + } + + using ASTWalker::operator(); + void operator()(VariableDeclaration const& _varDecl) override + { + for (auto const& var: _varDecl.variables) + variableTypes[var.name] = var.type; + } + void operator()(FunctionDefinition const& _funDef) override + { + ASTWalker::operator()(_funDef); + + auto& funType = functionTypes[_funDef.name]; + for (auto const arg: _funDef.parameters) + { + funType.parameters.emplace_back(arg.type); + variableTypes[arg.name] = arg.type; + } + for (auto const ret: _funDef.returnVariables) + { + funType.returns.emplace_back(ret.type); + variableTypes[ret.name] = ret.type; + } + } + + std::map variableTypes; + std::map functionTypes; +}; + + +TypeInfo::TypeInfo(Dialect const& _dialect, Block const& _ast): + m_dialect(_dialect) +{ + TypeCollector types(_ast); + m_functionTypes = std::move(types.functionTypes); + m_variableTypes = std::move(types.variableTypes); +} + +YulString TypeInfo::typeOf(Expression const& _expression) const +{ + return std::visit(GenericVisitor{ + [&](FunctionCall const& _funCall) { + YulString name = _funCall.functionName.name; + vector const* retTypes = nullptr; + if (BuiltinFunction const* fun = m_dialect.builtin(name)) + retTypes = &fun->returns; + else + retTypes = &m_functionTypes.at(name).returns; + yulAssert(retTypes && retTypes->size() == 1, "Call to typeOf for non-single-value expression."); + return retTypes->front(); + }, + [&](Identifier const& _identifier) { + return m_variableTypes.at(_identifier.name); + }, + [&](Literal const& _literal) { + return _literal.type; + } + }, _expression); +} diff --git a/libyul/optimiser/TypeInfo.h b/libyul/optimiser/TypeInfo.h new file mode 100644 index 000000000..deb537ea3 --- /dev/null +++ b/libyul/optimiser/TypeInfo.h @@ -0,0 +1,61 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ +/** + * Helper class that keeps track of the types while performing optimizations. + */ +#pragma once + +#include +#include + +#include +#include + +namespace solidity::yul +{ +struct Dialect; + +/** + * Helper class that keeps track of the types while performing optimizations. + * + * Only works on disambiguated sources! + */ +class TypeInfo +{ +public: + TypeInfo(Dialect const& _dialect, Block const& _ast); + + void setVariableType(YulString _name, YulString _type) { m_variableTypes[_name] = _type; } + + /// @returns the type of an expression that is assumed to return exactly one value. + YulString typeOf(Expression const& _expression) const; + +private: + class TypeCollector; + + struct FunctionType + { + std::vector parameters; + std::vector returns; + }; + + Dialect const& m_dialect; + std::map m_variableTypes; + std::map m_functionTypes; +}; + +} From b9b36cd89e81265d3e243a17bb504af0b47da495 Mon Sep 17 00:00:00 2001 From: chriseth Date: Fri, 14 Feb 2020 15:33:54 +0100 Subject: [PATCH 36/92] Properly assign types in ExpressionSplitter. --- libyul/optimiser/ExpressionSplitter.cpp | 10 ++++- libyul/optimiser/ExpressionSplitter.h | 13 ++++-- .../expressionSplitter/typed.yul | 42 +++++++++++++++++++ 3 files changed, 60 insertions(+), 5 deletions(-) create mode 100644 test/libyul/yulOptimizerTests/expressionSplitter/typed.yul diff --git a/libyul/optimiser/ExpressionSplitter.cpp b/libyul/optimiser/ExpressionSplitter.cpp index d95e11a81..7b2ac14f6 100644 --- a/libyul/optimiser/ExpressionSplitter.cpp +++ b/libyul/optimiser/ExpressionSplitter.cpp @@ -23,11 +23,13 @@ #include #include +#include #include #include #include +#include #include @@ -39,7 +41,8 @@ using namespace solidity::langutil; void ExpressionSplitter::run(OptimiserStepContext& _context, Block& _ast) { - ExpressionSplitter{_context.dialect, _context.dispenser}(_ast); + TypeInfo typeInfo(_context.dialect, _ast); + ExpressionSplitter{_context.dialect, _context.dispenser, typeInfo}(_ast); } void ExpressionSplitter::operator()(FunctionCall& _funCall) @@ -103,10 +106,13 @@ void ExpressionSplitter::outlineExpression(Expression& _expr) SourceLocation location = locationOf(_expr); YulString var = m_nameDispenser.newName({}); + YulString type = m_typeInfo.typeOf(_expr); m_statementsToPrefix.emplace_back(VariableDeclaration{ location, - {{TypedName{location, var, {}}}}, + {{TypedName{location, var, type}}}, make_unique(std::move(_expr)) }); _expr = Identifier{location, var}; + m_typeInfo.setVariableType(var, type); } + diff --git a/libyul/optimiser/ExpressionSplitter.h b/libyul/optimiser/ExpressionSplitter.h index dfd640ac3..107be2590 100644 --- a/libyul/optimiser/ExpressionSplitter.h +++ b/libyul/optimiser/ExpressionSplitter.h @@ -30,9 +30,9 @@ namespace solidity::yul { -class NameCollector; struct Dialect; struct OptimiserStepContext; +class TypeInfo; /** * Optimiser component that modifies an AST in place, turning complex @@ -68,8 +68,14 @@ public: void operator()(Block& _block) override; private: - explicit ExpressionSplitter(Dialect const& _dialect, NameDispenser& _nameDispenser): - m_dialect(_dialect), m_nameDispenser(_nameDispenser) + explicit ExpressionSplitter( + Dialect const& _dialect, + NameDispenser& _nameDispenser, + TypeInfo& _typeInfo + ): + m_dialect(_dialect), + m_nameDispenser(_nameDispenser), + m_typeInfo(_typeInfo) { } /// Replaces the expression by a variable if it is a function call or functional @@ -82,6 +88,7 @@ private: std::vector m_statementsToPrefix; Dialect const& m_dialect; NameDispenser& m_nameDispenser; + TypeInfo& m_typeInfo; }; } diff --git a/test/libyul/yulOptimizerTests/expressionSplitter/typed.yul b/test/libyul/yulOptimizerTests/expressionSplitter/typed.yul new file mode 100644 index 000000000..37f40f5e6 --- /dev/null +++ b/test/libyul/yulOptimizerTests/expressionSplitter/typed.yul @@ -0,0 +1,42 @@ +{ + function fun(x: i32, y) -> t: i32, z: i32 { + z := i32.add(x, i32.add(z, z)) + + } + i64.store(i32.load(5:i32), i64.load(8:i32)) + let i := 0 + for {} i32.eqz(i32.load(9:i32)) { i := i64.add(i, 1) } { + let f: i32, g: i32 := fun(i32.load(1:i32), i64.load(i32.load(0: i32))) + } +} +// ==== +// dialect: ewasm +// step: expressionSplitter +// ---- +// { +// function fun(x:i32, y) -> t:i32, z:i32 +// { +// let _1:i32 := i32.add(z, z) +// z := i32.add(x, _1) +// } +// let _2:i32 := 8:i32 +// let _3 := i64.load(_2) +// let _4:i32 := 5:i32 +// let _5:i32 := i32.load(_4) +// i64.store(_5, _3) +// let i := 0 +// for { } +// i32.eqz(i32.load(9:i32)) +// { +// let _6 := 1 +// i := i64.add(i, _6) +// } +// { +// let _7:i32 := 0:i32 +// let _8:i32 := i32.load(_7) +// let _9 := i64.load(_8) +// let _10:i32 := 1:i32 +// let _11:i32 := i32.load(_10) +// let f:i32, g:i32 := fun(_11, _9) +// } +// } From e728cd76b6cfb5f2a19206dd24808799203c68f5 Mon Sep 17 00:00:00 2001 From: chriseth Date: Wed, 19 Feb 2020 00:18:13 +0100 Subject: [PATCH 37/92] Introduce typed named functions to dialect. --- libyul/Dialect.h | 4 ++-- libyul/backends/evm/EVMDialect.cpp | 22 ++++++++++++++++++++++ libyul/backends/evm/EVMDialect.h | 8 ++++++-- libyul/backends/wasm/WasmDialect.cpp | 19 ++++++++++++++++++- libyul/backends/wasm/WasmDialect.h | 12 ++++++------ 5 files changed, 54 insertions(+), 11 deletions(-) diff --git a/libyul/Dialect.h b/libyul/Dialect.h index 3d261fdee..4a1aff0fc 100644 --- a/libyul/Dialect.h +++ b/libyul/Dialect.h @@ -58,8 +58,8 @@ struct Dialect: boost::noncopyable /// @returns the builtin function of the given name or a nullptr if it is not a builtin function. virtual BuiltinFunction const* builtin(YulString /*_name*/) const { return nullptr; } - virtual BuiltinFunction const* discardFunction() const { return nullptr; } - virtual BuiltinFunction const* equalityFunction() const { return nullptr; } + virtual BuiltinFunction const* discardFunction(YulString /* _type */) const { return nullptr; } + virtual BuiltinFunction const* equalityFunction(YulString /* _type */) const { return nullptr; } virtual BuiltinFunction const* booleanNegationFunction() const { return nullptr; } /// Check whether the given type is legal for the given literal value. diff --git a/libyul/backends/evm/EVMDialect.cpp b/libyul/backends/evm/EVMDialect.cpp index 39fc65632..27a2b85e3 100644 --- a/libyul/backends/evm/EVMDialect.cpp +++ b/libyul/backends/evm/EVMDialect.cpp @@ -290,6 +290,28 @@ EVMDialectTyped::EVMDialectTyped(langutil::EVMVersion _evmVersion, bool _objectA m_functions["u256_to_bool"_yulstring].returns = {"bool"_yulstring}; } +BuiltinFunctionForEVM const* EVMDialectTyped::discardFunction(YulString _type) const +{ + if (_type == "bool"_yulstring) + return builtin("popbool"_yulstring); + else + { + yulAssert(_type == defaultType, ""); + return builtin("pop"_yulstring); + } +} + +BuiltinFunctionForEVM const* EVMDialectTyped::equalityFunction(YulString _type) const +{ + if (_type == "bool"_yulstring) + return nullptr; + else + { + yulAssert(_type == defaultType, ""); + return builtin("eq"_yulstring); + } +} + EVMDialectTyped const& EVMDialectTyped::instance(langutil::EVMVersion _version) { static map> dialects; diff --git a/libyul/backends/evm/EVMDialect.h b/libyul/backends/evm/EVMDialect.h index 40842fa57..2141ab98f 100644 --- a/libyul/backends/evm/EVMDialect.h +++ b/libyul/backends/evm/EVMDialect.h @@ -68,8 +68,8 @@ struct EVMDialect: public Dialect /// @returns the builtin function of the given name or a nullptr if it is not a builtin function. BuiltinFunctionForEVM const* builtin(YulString _name) const override; - BuiltinFunctionForEVM const* discardFunction() const override { return builtin("pop"_yulstring); } - BuiltinFunctionForEVM const* equalityFunction() const override { return builtin("eq"_yulstring); } + BuiltinFunctionForEVM const* discardFunction(YulString /*_type*/) const override { return builtin("pop"_yulstring); } + BuiltinFunctionForEVM const* equalityFunction(YulString /*_type*/) const override { return builtin("eq"_yulstring); } BuiltinFunctionForEVM const* booleanNegationFunction() const override { return builtin("iszero"_yulstring); } static EVMDialect const& strictAssemblyForEVM(langutil::EVMVersion _version); @@ -102,6 +102,10 @@ struct EVMDialectTyped: public EVMDialect /// Constructor, should only be used internally. Use the factory function below. EVMDialectTyped(langutil::EVMVersion _evmVersion, bool _objectAccess); + BuiltinFunctionForEVM const* discardFunction(YulString _type) const override; + BuiltinFunctionForEVM const* equalityFunction(YulString _type) const override; + BuiltinFunctionForEVM const* booleanNegationFunction() const override { return builtin("not"_yulstring); } + static EVMDialectTyped const& instance(langutil::EVMVersion _version); }; diff --git a/libyul/backends/wasm/WasmDialect.cpp b/libyul/backends/wasm/WasmDialect.cpp index 947326306..7f2149adc 100644 --- a/libyul/backends/wasm/WasmDialect.cpp +++ b/libyul/backends/wasm/WasmDialect.cpp @@ -91,8 +91,9 @@ WasmDialect::WasmDialect() m_functions["i64.load"_yulstring].sideEffects.sideEffectFreeIfNoMSize = true; // Drop is actually overloaded for all types, but Yul does not support that. - // We could introduce "i32.drop". + // Because of that, we introduce "i32.drop". addFunction("drop", {i64}, {}); + addFunction("i32.drop", {i32}, {}); addFunction("nop", {}, {}); addFunction("unreachable", {}, {}, false); @@ -114,6 +115,22 @@ BuiltinFunction const* WasmDialect::builtin(YulString _name) const return nullptr; } +BuiltinFunction const* WasmDialect::discardFunction(YulString _type) const +{ + if (_type == "i32"_yulstring) + return builtin("i32.drop"_yulstring); + yulAssert(_type == "i64"_yulstring, ""); + return builtin("drop"_yulstring); +} + +BuiltinFunction const* WasmDialect::equalityFunction(YulString _type) const +{ + if (_type == "i32"_yulstring) + return builtin("i32.eq"_yulstring); + yulAssert(_type == "i64"_yulstring, ""); + return builtin("i64.eq"_yulstring); +} + WasmDialect const& WasmDialect::instance() { static std::unique_ptr dialect; diff --git a/libyul/backends/wasm/WasmDialect.h b/libyul/backends/wasm/WasmDialect.h index cf1a76d9d..1de65dfdc 100644 --- a/libyul/backends/wasm/WasmDialect.h +++ b/libyul/backends/wasm/WasmDialect.h @@ -35,19 +35,19 @@ struct Object; /** * Yul dialect for Wasm as a backend. * - * Builtin functions are a subset of the wasm instructions, always implicitly assuming - * unsigned 64 bit types. + * Builtin functions are a subset of the wasm instructions. + * + * There is a builtin function `i32.drop` that takes an i32, while `drop` takes i64. * - * !This is subject to changes! */ struct WasmDialect: public Dialect { WasmDialect(); BuiltinFunction const* builtin(YulString _name) const override; - BuiltinFunction const* discardFunction() const override { return builtin("drop"_yulstring); } - BuiltinFunction const* equalityFunction() const override { return builtin("i64.eq"_yulstring); } - BuiltinFunction const* booleanNegationFunction() const override { return builtin("i64.eqz"_yulstring); } + BuiltinFunction const* discardFunction(YulString _type) const override; + BuiltinFunction const* equalityFunction(YulString _type) const override; + BuiltinFunction const* booleanNegationFunction() const override { return builtin("i32.eqz"_yulstring); } std::set fixedFunctionNames() const override { return {"main"_yulstring}; } From e75cace78d47b8f95a30e25b02db77c5dc9c508a Mon Sep 17 00:00:00 2001 From: chriseth Date: Wed, 19 Feb 2020 00:36:50 +0100 Subject: [PATCH 38/92] Properly assign types in control flow simplifier. --- libyul/optimiser/ControlFlowSimplifier.cpp | 123 +++++++++++---------- libyul/optimiser/ControlFlowSimplifier.h | 10 +- 2 files changed, 73 insertions(+), 60 deletions(-) diff --git a/libyul/optimiser/ControlFlowSimplifier.cpp b/libyul/optimiser/ControlFlowSimplifier.cpp index 8af8e6bb0..1640fa1b0 100644 --- a/libyul/optimiser/ControlFlowSimplifier.cpp +++ b/libyul/optimiser/ControlFlowSimplifier.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -38,14 +39,13 @@ namespace ExpressionStatement makeDiscardCall( langutil::SourceLocation const& _location, - Dialect const& _dialect, + BuiltinFunction const& _discardFunction, Expression&& _expression ) { - yulAssert(_dialect.discardFunction(), "No discard function available."); return {_location, FunctionCall{ _location, - Identifier{_location, _dialect.discardFunction()->name}, + Identifier{_location, _discardFunction.name}, {std::move(_expression)} }}; } @@ -74,62 +74,12 @@ void removeEmptyCasesFromSwitch(Switch& _switchStmt) ); } -OptionalStatements reduceNoCaseSwitch(Dialect const& _dialect, Switch& _switchStmt) -{ - yulAssert(_switchStmt.cases.empty(), "Expected no case!"); - if (!_dialect.discardFunction()) - return {}; - - auto loc = locationOf(*_switchStmt.expression); - - return make_vector(makeDiscardCall( - loc, - _dialect, - std::move(*_switchStmt.expression) - )); -} - -OptionalStatements reduceSingleCaseSwitch(Dialect const& _dialect, Switch& _switchStmt) -{ - yulAssert(_switchStmt.cases.size() == 1, "Expected only one case!"); - - auto& switchCase = _switchStmt.cases.front(); - auto loc = locationOf(*_switchStmt.expression); - if (switchCase.value) - { - if (!_dialect.equalityFunction()) - return {}; - return make_vector(If{ - std::move(_switchStmt.location), - make_unique(FunctionCall{ - loc, - Identifier{loc, _dialect.equalityFunction()->name}, - {std::move(*switchCase.value), std::move(*_switchStmt.expression)} - }), - std::move(switchCase.body) - }); - } - else - { - if (!_dialect.discardFunction()) - return {}; - - return make_vector( - makeDiscardCall( - loc, - _dialect, - std::move(*_switchStmt.expression) - ), - std::move(switchCase.body) - ); - } -} - } void ControlFlowSimplifier::run(OptimiserStepContext& _context, Block& _ast) { - ControlFlowSimplifier{_context.dialect}(_ast); + TypeInfo typeInfo(_context.dialect, _ast); + ControlFlowSimplifier{_context.dialect, typeInfo}(_ast); } void ControlFlowSimplifier::operator()(Block& _block) @@ -194,12 +144,12 @@ void ControlFlowSimplifier::simplify(std::vector& _statements) GenericVisitor visitor{ VisitorFallback{}, [&](If& _ifStmt) -> OptionalStatements { - if (_ifStmt.body.statements.empty() && m_dialect.discardFunction()) + if (_ifStmt.body.statements.empty() && m_dialect.discardFunction(m_dialect.boolType)) { OptionalStatements s = vector{}; s->emplace_back(makeDiscardCall( _ifStmt.location, - m_dialect, + *m_dialect.discardFunction(m_dialect.boolType), std::move(*_ifStmt.condition) )); return s; @@ -211,9 +161,9 @@ void ControlFlowSimplifier::simplify(std::vector& _statements) removeEmptyCasesFromSwitch(_switchStmt); if (_switchStmt.cases.empty()) - return reduceNoCaseSwitch(m_dialect, _switchStmt); + return reduceNoCaseSwitch(_switchStmt); else if (_switchStmt.cases.size() == 1) - return reduceSingleCaseSwitch(m_dialect, _switchStmt); + return reduceSingleCaseSwitch(_switchStmt); return {}; } @@ -231,3 +181,58 @@ void ControlFlowSimplifier::simplify(std::vector& _statements) } ); } + +OptionalStatements ControlFlowSimplifier::reduceNoCaseSwitch(Switch& _switchStmt) const +{ + yulAssert(_switchStmt.cases.empty(), "Expected no case!"); + BuiltinFunction const* discardFunction = + m_dialect.discardFunction(m_typeInfo.typeOf(*_switchStmt.expression)); + if (!discardFunction) + return {}; + + auto loc = locationOf(*_switchStmt.expression); + + return make_vector(makeDiscardCall( + loc, + *discardFunction, + std::move(*_switchStmt.expression) + )); +} + +OptionalStatements ControlFlowSimplifier::reduceSingleCaseSwitch(Switch& _switchStmt) const +{ + yulAssert(_switchStmt.cases.size() == 1, "Expected only one case!"); + + auto& switchCase = _switchStmt.cases.front(); + auto loc = locationOf(*_switchStmt.expression); + YulString type = m_typeInfo.typeOf(*_switchStmt.expression); + if (switchCase.value) + { + if (!m_dialect.equalityFunction(type)) + return {}; + return make_vector(If{ + std::move(_switchStmt.location), + make_unique(FunctionCall{ + loc, + Identifier{loc, m_dialect.equalityFunction(type)->name}, + {std::move(*switchCase.value), std::move(*_switchStmt.expression)} + }), + std::move(switchCase.body) + }); + } + else + { + if (!m_dialect.discardFunction(type)) + return {}; + + return make_vector( + makeDiscardCall( + loc, + *m_dialect.discardFunction(type), + std::move(*_switchStmt.expression) + ), + std::move(switchCase.body) + ); + } +} + diff --git a/libyul/optimiser/ControlFlowSimplifier.h b/libyul/optimiser/ControlFlowSimplifier.h index 5713f12a1..f8ea1af1e 100644 --- a/libyul/optimiser/ControlFlowSimplifier.h +++ b/libyul/optimiser/ControlFlowSimplifier.h @@ -23,6 +23,7 @@ namespace solidity::yul { struct Dialect; struct OptimiserStepContext; +class TypeInfo; /** * Simplifies several control-flow structures: @@ -61,11 +62,18 @@ public: void visit(Statement& _st) override; private: - ControlFlowSimplifier(Dialect const& _dialect): m_dialect(_dialect) {} + ControlFlowSimplifier(Dialect const& _dialect, TypeInfo const& _typeInfo): + m_dialect(_dialect), + m_typeInfo(_typeInfo) + {} void simplify(std::vector& _statements); + std::optional> reduceNoCaseSwitch(Switch& _switchStmt) const; + std::optional> reduceSingleCaseSwitch(Switch& _switchStmt) const; + Dialect const& m_dialect; + TypeInfo const& m_typeInfo; size_t m_numBreakStatements = 0; size_t m_numContinueStatements = 0; }; From a52c9af5b9fc26877f14b0fc451ff4b3e3cec6e4 Mon Sep 17 00:00:00 2001 From: Djordje Mijovic Date: Tue, 18 Feb 2020 16:11:21 +0100 Subject: [PATCH 39/92] Adding vardecl optimization for boolean types --- libyul/Dialect.cpp | 8 +++++++ libyul/Dialect.h | 3 +++ libyul/optimiser/UnusedPruner.cpp | 4 ++-- libyul/optimiser/VarDeclInitializer.cpp | 12 ++++++---- libyul/optimiser/VarDeclInitializer.h | 7 +++++- test/libyul/YulOptimizerTest.cpp | 2 ++ .../varDeclInitializer/typed.yul | 23 +++++++++++++++++++ 7 files changed, 52 insertions(+), 7 deletions(-) create mode 100644 test/libyul/yulOptimizerTests/varDeclInitializer/typed.yul diff --git a/libyul/Dialect.cpp b/libyul/Dialect.cpp index 4d5cccd41..407fdea2f 100644 --- a/libyul/Dialect.cpp +++ b/libyul/Dialect.cpp @@ -23,6 +23,14 @@ using namespace solidity::yul; using namespace std; +using namespace solidity::langutil; + +Literal Dialect::zeroLiteralForType(solidity::yul::YulString _type) const +{ + if (_type == boolType && _type != defaultType) + return {SourceLocation{}, LiteralKind::Boolean, "false"_yulstring, _type}; + return {SourceLocation{}, LiteralKind::Number, "0"_yulstring, _type}; +} bool Dialect::validTypeForLiteral(LiteralKind _kind, YulString, YulString _type) const { diff --git a/libyul/Dialect.h b/libyul/Dialect.h index 4a1aff0fc..c137791ee 100644 --- a/libyul/Dialect.h +++ b/libyul/Dialect.h @@ -34,6 +34,7 @@ namespace solidity::yul class YulString; using Type = YulString; enum class LiteralKind; +struct Literal; struct BuiltinFunction { @@ -66,6 +67,8 @@ struct Dialect: boost::noncopyable /// Should only be called if the type exists in the dialect at all. virtual bool validTypeForLiteral(LiteralKind _kind, YulString _value, YulString _type) const; + virtual Literal zeroLiteralForType(YulString _type) const; + virtual std::set fixedFunctionNames() const { return {}; } Dialect() = default; diff --git a/libyul/optimiser/UnusedPruner.cpp b/libyul/optimiser/UnusedPruner.cpp index b7925790a..9dc7cedc3 100644 --- a/libyul/optimiser/UnusedPruner.cpp +++ b/libyul/optimiser/UnusedPruner.cpp @@ -100,10 +100,10 @@ void UnusedPruner::operator()(Block& _block) subtractReferences(ReferencesCounter::countReferences(*varDecl.value)); statement = Block{std::move(varDecl.location), {}}; } - else if (varDecl.variables.size() == 1 && m_dialect.discardFunction()) + else if (varDecl.variables.size() == 1 && m_dialect.discardFunction(varDecl.variables.front().type)) statement = ExpressionStatement{varDecl.location, FunctionCall{ varDecl.location, - {varDecl.location, m_dialect.discardFunction()->name}, + {varDecl.location, m_dialect.discardFunction(varDecl.variables.front().type)->name}, {*std::move(varDecl.value)} }}; } diff --git a/libyul/optimiser/VarDeclInitializer.cpp b/libyul/optimiser/VarDeclInitializer.cpp index bad601f0e..66c5b4505 100644 --- a/libyul/optimiser/VarDeclInitializer.cpp +++ b/libyul/optimiser/VarDeclInitializer.cpp @@ -20,6 +20,7 @@ #include #include +#include using namespace std; using namespace solidity; @@ -32,14 +33,14 @@ void VarDeclInitializer::operator()(Block& _block) using OptionalStatements = std::optional>; util::GenericVisitor visitor{ util::VisitorFallback{}, - [](VariableDeclaration& _varDecl) -> OptionalStatements + [this](VariableDeclaration& _varDecl) -> OptionalStatements { if (_varDecl.value) return {}; - Literal zero{{}, LiteralKind::Number, YulString{"0"}, {}}; + if (_varDecl.variables.size() == 1) { - _varDecl.value = make_unique(std::move(zero)); + _varDecl.value = make_unique(m_dialect.zeroLiteralForType(_varDecl.variables.front().type)); return {}; } else @@ -47,7 +48,10 @@ void VarDeclInitializer::operator()(Block& _block) OptionalStatements ret{vector{}}; langutil::SourceLocation loc{std::move(_varDecl.location)}; for (auto& var: _varDecl.variables) - ret->emplace_back(VariableDeclaration{loc, {std::move(var)}, make_unique(zero)}); + { + unique_ptr expr = make_unique(m_dialect.zeroLiteralForType(var.type)); + ret->emplace_back(VariableDeclaration{loc, {std::move(var)}, std::move(expr)}); + } return ret; } } diff --git a/libyul/optimiser/VarDeclInitializer.h b/libyul/optimiser/VarDeclInitializer.h index 556ce081b..75bb1493c 100644 --- a/libyul/optimiser/VarDeclInitializer.h +++ b/libyul/optimiser/VarDeclInitializer.h @@ -34,9 +34,14 @@ class VarDeclInitializer: public ASTModifier { public: static constexpr char const* name{"VarDeclInitializer"}; - static void run(OptimiserStepContext&, Block& _ast) { VarDeclInitializer{}(_ast); } + static void run(OptimiserStepContext& _ctx, Block& _ast) { VarDeclInitializer{_ctx.dialect}(_ast); } void operator()(Block& _block) override; + +private: + explicit VarDeclInitializer(Dialect const& _dialect): m_dialect(_dialect) {} + + Dialect const& m_dialect; }; } diff --git a/test/libyul/YulOptimizerTest.cpp b/test/libyul/YulOptimizerTest.cpp index 3ac745d6f..699a767d0 100644 --- a/test/libyul/YulOptimizerTest.cpp +++ b/test/libyul/YulOptimizerTest.cpp @@ -111,6 +111,8 @@ YulOptimizerTest::YulOptimizerTest(string const& _filename) m_dialect = &WasmDialect::instance(); else if (dialectName == "evm") m_dialect = &EVMDialect::strictAssemblyForEVMObjects(solidity::test::CommonOptions::get().evmVersion()); + else if (dialectName == "evmTyped") + m_dialect = &EVMDialectTyped::instance(solidity::test::CommonOptions::get().evmVersion()); else BOOST_THROW_EXCEPTION(runtime_error("Invalid dialect " + dialectName)); diff --git a/test/libyul/yulOptimizerTests/varDeclInitializer/typed.yul b/test/libyul/yulOptimizerTests/varDeclInitializer/typed.yul new file mode 100644 index 000000000..912246bea --- /dev/null +++ b/test/libyul/yulOptimizerTests/varDeclInitializer/typed.yul @@ -0,0 +1,23 @@ +{ + let a1 + let a2: bool + let b1, b2: bool + function f(a:u256, b:u256, c:bool) -> r:bool, t { + let x1: bool, x2 + } +} +// ==== +// dialect: evmTyped +// step: varDeclInitializer +// ---- +// { +// let a1 := 0 +// let a2:bool := false +// let b1 := 0 +// let b2:bool := false +// function f(a, b, c:bool) -> r:bool, t +// { +// let x1:bool := false +// let x2 := 0 +// } +// } From a52305d3bd8cb1792175e8333560569cb566a3e1 Mon Sep 17 00:00:00 2001 From: chriseth Date: Wed, 19 Feb 2020 16:39:59 +0100 Subject: [PATCH 40/92] Use bool type in conditional simplifier and for loop condition into body. --- libyul/optimiser/ConditionalSimplifier.cpp | 7 +------ libyul/optimiser/ForLoopConditionIntoBody.cpp | 6 +++--- .../conditionalSimplifier/add_correct_type.yul | 18 ++++++++++++++++++ .../add_correct_type_wasm.yul | 18 ++++++++++++++++++ .../forLoopConditionIntoBody/cond_types.yul | 2 +- .../forLoopConditionIntoBody/empty_body.yul | 2 +- .../forLoopConditionIntoBody/nested.yul | 6 +++--- .../forLoopConditionIntoBody/simple.yul | 2 +- .../fullSuite/no_move_loop_orig.yul | 2 +- 9 files changed, 47 insertions(+), 16 deletions(-) create mode 100644 test/libyul/yulOptimizerTests/conditionalSimplifier/add_correct_type.yul create mode 100644 test/libyul/yulOptimizerTests/conditionalSimplifier/add_correct_type_wasm.yul diff --git a/libyul/optimiser/ConditionalSimplifier.cpp b/libyul/optimiser/ConditionalSimplifier.cpp index 858ac8999..453d2d9ce 100644 --- a/libyul/optimiser/ConditionalSimplifier.cpp +++ b/libyul/optimiser/ConditionalSimplifier.cpp @@ -77,12 +77,7 @@ void ConditionalSimplifier::operator()(Block& _block) Assignment{ location, {Identifier{location, condition}}, - make_unique(Literal{ - location, - LiteralKind::Number, - "0"_yulstring, - {} - }) + make_unique(m_dialect.zeroLiteralForType(m_dialect.boolType)) } ); } diff --git a/libyul/optimiser/ForLoopConditionIntoBody.cpp b/libyul/optimiser/ForLoopConditionIntoBody.cpp index a45b33635..22c3e76dc 100644 --- a/libyul/optimiser/ForLoopConditionIntoBody.cpp +++ b/libyul/optimiser/ForLoopConditionIntoBody.cpp @@ -56,9 +56,9 @@ void ForLoopConditionIntoBody::operator()(ForLoop& _forLoop) _forLoop.condition = make_unique( Literal { loc, - LiteralKind::Number, - "1"_yulstring, - {} + LiteralKind::Boolean, + "true"_yulstring, + m_dialect.boolType } ); } diff --git a/test/libyul/yulOptimizerTests/conditionalSimplifier/add_correct_type.yul b/test/libyul/yulOptimizerTests/conditionalSimplifier/add_correct_type.yul new file mode 100644 index 000000000..f0b46072f --- /dev/null +++ b/test/libyul/yulOptimizerTests/conditionalSimplifier/add_correct_type.yul @@ -0,0 +1,18 @@ +{ + let y:bool := false + for {} true { } { + if y { break } + } +} +// ==== +// dialect: yul +// step: conditionalSimplifier +// ---- +// { +// let y:bool := false +// for { } true { } +// { +// if y { break } +// y := false +// } +// } diff --git a/test/libyul/yulOptimizerTests/conditionalSimplifier/add_correct_type_wasm.yul b/test/libyul/yulOptimizerTests/conditionalSimplifier/add_correct_type_wasm.yul new file mode 100644 index 000000000..ab523246e --- /dev/null +++ b/test/libyul/yulOptimizerTests/conditionalSimplifier/add_correct_type_wasm.yul @@ -0,0 +1,18 @@ +{ + let y:i32 := 0:i32 + for {} true { } { + if y { break } + } +} +// ==== +// dialect: ewasm +// step: conditionalSimplifier +// ---- +// { +// let y:i32 := 0:i32 +// for { } true { } +// { +// if y { break } +// y := false +// } +// } diff --git a/test/libyul/yulOptimizerTests/forLoopConditionIntoBody/cond_types.yul b/test/libyul/yulOptimizerTests/forLoopConditionIntoBody/cond_types.yul index 9b401e638..95e171475 100644 --- a/test/libyul/yulOptimizerTests/forLoopConditionIntoBody/cond_types.yul +++ b/test/libyul/yulOptimizerTests/forLoopConditionIntoBody/cond_types.yul @@ -19,7 +19,7 @@ // { } // for { } a { } // { } -// for { } 1 { } +// for { } true { } // { // if iszero(add(a, a)) { break } // } diff --git a/test/libyul/yulOptimizerTests/forLoopConditionIntoBody/empty_body.yul b/test/libyul/yulOptimizerTests/forLoopConditionIntoBody/empty_body.yul index 6a570750f..6f1ad5458 100644 --- a/test/libyul/yulOptimizerTests/forLoopConditionIntoBody/empty_body.yul +++ b/test/libyul/yulOptimizerTests/forLoopConditionIntoBody/empty_body.yul @@ -5,7 +5,7 @@ // step: forLoopConditionIntoBody // ---- // { -// for { let a := 1 } 1 { a := add(a, 1) } +// for { let a := 1 } true { a := add(a, 1) } // { // if iszero(iszero(eq(a, 10))) { break } // } diff --git a/test/libyul/yulOptimizerTests/forLoopConditionIntoBody/nested.yul b/test/libyul/yulOptimizerTests/forLoopConditionIntoBody/nested.yul index 257467d28..292284a65 100644 --- a/test/libyul/yulOptimizerTests/forLoopConditionIntoBody/nested.yul +++ b/test/libyul/yulOptimizerTests/forLoopConditionIntoBody/nested.yul @@ -19,16 +19,16 @@ // { // let random := 42 // for { -// for { let a := 1 } 1 { } +// for { let a := 1 } true { } // { // if iszero(iszero(eq(a, 10))) { break } // a := add(a, 1) // } // let b := 1 // } -// 1 +// true // { -// for { let c := 1 } 1 { c := add(c, 1) } +// for { let c := 1 } true { c := add(c, 1) } // { // if iszero(iszero(eq(c, 2))) { break } // b := add(b, 1) diff --git a/test/libyul/yulOptimizerTests/forLoopConditionIntoBody/simple.yul b/test/libyul/yulOptimizerTests/forLoopConditionIntoBody/simple.yul index 0e3c45ffe..f47026462 100644 --- a/test/libyul/yulOptimizerTests/forLoopConditionIntoBody/simple.yul +++ b/test/libyul/yulOptimizerTests/forLoopConditionIntoBody/simple.yul @@ -9,7 +9,7 @@ // ---- // { // let random := 42 -// for { let a := 1 } 1 { a := add(a, 1) } +// for { let a := 1 } true { a := add(a, 1) } // { // if iszero(iszero(eq(a, 10))) { break } // a := add(a, 1) diff --git a/test/libyul/yulOptimizerTests/fullSuite/no_move_loop_orig.yul b/test/libyul/yulOptimizerTests/fullSuite/no_move_loop_orig.yul index 77c384021..4efe81d89 100644 --- a/test/libyul/yulOptimizerTests/fullSuite/no_move_loop_orig.yul +++ b/test/libyul/yulOptimizerTests/fullSuite/no_move_loop_orig.yul @@ -14,7 +14,7 @@ // { // let _1 := iszero(caller()) // for { } -// 1 +// true // { // for { } iszero(_1) { } // { } From 6b272faec003a5177acbf70cb0e8b4a4c3357c2f Mon Sep 17 00:00:00 2001 From: chriseth Date: Wed, 19 Feb 2020 16:56:53 +0100 Subject: [PATCH 41/92] Some wasm related type fixes. --- libyul/backends/wasm/EVMToEwasmTranslator.cpp | 8 ++++++-- libyul/backends/wasm/WordSizeTransform.h | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/libyul/backends/wasm/EVMToEwasmTranslator.cpp b/libyul/backends/wasm/EVMToEwasmTranslator.cpp index 19bd98da7..a5b88d053 100644 --- a/libyul/backends/wasm/EVMToEwasmTranslator.cpp +++ b/libyul/backends/wasm/EVMToEwasmTranslator.cpp @@ -1251,8 +1251,12 @@ Object EVMToEwasmTranslator::run(Object const& _object) AsmAnalyzer analyzer(*ret.analysisInfo, errorReporter, WasmDialect::instance(), {}, _object.dataNames()); if (!analyzer.analyze(*ret.code)) { - // TODO the errors here are "wrong" because they have invalid source references! - string message; + string message = "Invalid code generated after EVM to wasm translation.\n"; + message += "Note that the source locations in the errors below will reference the original, not the translated code.\n"; + message += "Translated code:\n"; + message += "----------------------------------\n"; + message += ret.toString(&WasmDialect::instance()); + message += "----------------------------------\n"; for (auto const& err: errors) message += langutil::SourceReferenceFormatter::formatErrorInformation(*err); yulAssert(false, message); diff --git a/libyul/backends/wasm/WordSizeTransform.h b/libyul/backends/wasm/WordSizeTransform.h index 71e526b7f..2e4db8446 100644 --- a/libyul/backends/wasm/WordSizeTransform.h +++ b/libyul/backends/wasm/WordSizeTransform.h @@ -53,7 +53,7 @@ namespace solidity::yul * take four times the parameters and each of type u64. * In addition, it uses a single other builtin function called `or_bool` that * takes four u64 parameters and is supposed to return the logical disjunction - * of them as a u64 value. If this name is already used somewhere, it is renamed. + * of them as a i32 value. If this name is already used somewhere, it is renamed. * * Prerequisite: Disambiguator, ForLoopConditionIntoBody, ExpressionSplitter */ From bddbcbe6a4a78efa3e40d86eac4657c593bb5d31 Mon Sep 17 00:00:00 2001 From: chriseth Date: Thu, 20 Feb 2020 16:17:09 +0100 Subject: [PATCH 42/92] Use bool type in word size transform. --- libyul/backends/wasm/EVMToEwasmTranslator.cpp | 2 +- libyul/backends/wasm/WordSizeTransform.cpp | 47 ++++++++++++++----- libyul/backends/wasm/WordSizeTransform.h | 14 ++---- test/libyul/YulOptimizerTest.cpp | 2 +- .../wordSizeTransform/switch_3.yul | 8 ++-- .../wordSizeTransform/switch_4.yul | 12 ++--- .../wordSizeTransform/switch_5.yul | 2 +- 7 files changed, 52 insertions(+), 35 deletions(-) diff --git a/libyul/backends/wasm/EVMToEwasmTranslator.cpp b/libyul/backends/wasm/EVMToEwasmTranslator.cpp index a5b88d053..dc69d913a 100644 --- a/libyul/backends/wasm/EVMToEwasmTranslator.cpp +++ b/libyul/backends/wasm/EVMToEwasmTranslator.cpp @@ -1235,7 +1235,7 @@ Object EVMToEwasmTranslator::run(Object const& _object) MainFunction{}(ast); ForLoopConditionIntoBody::run(context, ast); ExpressionSplitter::run(context, ast); - WordSizeTransform::run(m_dialect, WasmDialect::instance().defaultType, ast, nameDispenser); + WordSizeTransform::run(m_dialect, WasmDialect::instance(), ast, nameDispenser); NameDisplacer{nameDispenser, m_polyfillFunctions}(ast); for (auto const& st: m_polyfill->statements) diff --git a/libyul/backends/wasm/WordSizeTransform.cpp b/libyul/backends/wasm/WordSizeTransform.cpp index 9ebf31738..eec04fae6 100644 --- a/libyul/backends/wasm/WordSizeTransform.cpp +++ b/libyul/backends/wasm/WordSizeTransform.cpp @@ -45,7 +45,7 @@ void WordSizeTransform::operator()(FunctionCall& _fc) if (fun->literalArguments) { for (Expression& arg: _fc.arguments) - get(arg).type = m_defaultType; + get(arg).type = m_targetDialect.defaultType; return; } @@ -106,12 +106,17 @@ void WordSizeTransform::operator()(Block& _block) for (int i = 0; i < 3; i++) ret.push_back(VariableDeclaration{ varDecl.location, - {TypedName{varDecl.location, newLhs[i], m_defaultType}}, - make_unique(Literal{locationOf(*varDecl.value), LiteralKind::Number, "0"_yulstring, m_defaultType}) + {TypedName{varDecl.location, newLhs[i], m_targetDialect.defaultType}}, + make_unique(Literal{ + locationOf(*varDecl.value), + LiteralKind::Number, + "0"_yulstring, + m_targetDialect.defaultType + }) }); ret.push_back(VariableDeclaration{ varDecl.location, - {TypedName{varDecl.location, newLhs[3], m_defaultType}}, + {TypedName{varDecl.location, newLhs[3], m_targetDialect.defaultType}}, std::move(varDecl.value) }); return {std::move(ret)}; @@ -133,7 +138,7 @@ void WordSizeTransform::operator()(Block& _block) ret.push_back( VariableDeclaration{ varDecl.location, - {TypedName{varDecl.location, newLhs[i], m_defaultType}}, + {TypedName{varDecl.location, newLhs[i], m_targetDialect.defaultType}}, std::move(newRhs[i]) } ); @@ -163,7 +168,12 @@ void WordSizeTransform::operator()(Block& _block) ret.push_back(Assignment{ assignment.location, {Identifier{assignment.location, newLhs[i]}}, - make_unique(Literal{locationOf(*assignment.value), LiteralKind::Number, "0"_yulstring, m_defaultType}) + make_unique(Literal{ + locationOf(*assignment.value), + LiteralKind::Number, + "0"_yulstring, + m_targetDialect.defaultType + }) }); ret.push_back(Assignment{ assignment.location, @@ -209,14 +219,25 @@ void WordSizeTransform::operator()(Block& _block) void WordSizeTransform::run( Dialect const& _inputDialect, - YulString _targetDefaultType, + Dialect const& _targetDialect, Block& _ast, NameDispenser& _nameDispenser ) { // Free the name `or_bool`. NameDisplacer{_nameDispenser, {"or_bool"_yulstring}}(_ast); - WordSizeTransform{_inputDialect, _nameDispenser, _targetDefaultType}(_ast); + WordSizeTransform{_inputDialect, _targetDialect, _nameDispenser}(_ast); +} + +WordSizeTransform::WordSizeTransform( + Dialect const& _inputDialect, + Dialect const& _targetDialect, + NameDispenser& _nameDispenser +): + m_inputDialect(_inputDialect), + m_targetDialect(_targetDialect), + m_nameDispenser(_nameDispenser) +{ } void WordSizeTransform::rewriteVarDeclList(TypedNameList& _nameList) @@ -227,7 +248,7 @@ void WordSizeTransform::rewriteVarDeclList(TypedNameList& _nameList) { TypedNameList ret; for (auto newName: generateU64IdentifierNames(_n.name)) - ret.emplace_back(TypedName{_n.location, newName, m_defaultType}); + ret.emplace_back(TypedName{_n.location, newName, m_targetDialect.defaultType}); return ret; } ); @@ -291,7 +312,7 @@ vector WordSizeTransform::handleSwitchInternal( for (auto& c: cases) { - Literal label{_location, LiteralKind::Number, YulString(c.first.str()), m_defaultType}; + Literal label{_location, LiteralKind::Number, YulString(c.first.str()), m_targetDialect.defaultType}; ret.cases.emplace_back(Case{ c.second.front().location, make_unique(std::move(label)), @@ -312,7 +333,7 @@ vector WordSizeTransform::handleSwitchInternal( Assignment{ _location, {{_location, _runDefaultFlag}}, - make_unique(Literal{_location, LiteralKind::Number, "1"_yulstring, m_defaultType}) + make_unique(Literal{_location, LiteralKind::Boolean, "true"_yulstring, m_targetDialect.boolType}) } )} }); @@ -337,7 +358,7 @@ std::vector WordSizeTransform::handleSwitch(Switch& _switch) _switch.cases.pop_back(); ret.emplace_back(VariableDeclaration{ _switch.location, - {TypedName{_switch.location, runDefaultFlag, m_defaultType}}, + {TypedName{_switch.location, runDefaultFlag, m_targetDialect.boolType}}, {} }); } @@ -392,7 +413,7 @@ array, 4> WordSizeTransform::expandValue(Expression const lit.location, LiteralKind::Number, YulString(currentVal.str()), - m_defaultType + m_targetDialect.defaultType } ); } diff --git a/libyul/backends/wasm/WordSizeTransform.h b/libyul/backends/wasm/WordSizeTransform.h index 2e4db8446..67dfe8864 100644 --- a/libyul/backends/wasm/WordSizeTransform.h +++ b/libyul/backends/wasm/WordSizeTransform.h @@ -69,7 +69,7 @@ public: static void run( Dialect const& _inputDialect, - YulString _targetDefaultType, + Dialect const& _targetDialect, Block& _ast, NameDispenser& _nameDispenser ); @@ -77,13 +77,9 @@ public: private: explicit WordSizeTransform( Dialect const& _inputDialect, - NameDispenser& _nameDispenser, - YulString _defaultType - ): - m_inputDialect(_inputDialect), - m_nameDispenser(_nameDispenser), - m_defaultType(_defaultType) - { } + Dialect const& _targetDialect, + NameDispenser& _nameDispenser + ); void rewriteVarDeclList(std::vector&); void rewriteIdentifierList(std::vector&); @@ -103,8 +99,8 @@ private: std::vector expandValueToVector(Expression const& _e); Dialect const& m_inputDialect; + Dialect const& m_targetDialect; NameDispenser& m_nameDispenser; - YulString m_defaultType; /// maps original u256 variable's name to corresponding u64 variables' names std::map> m_variableMapping; }; diff --git a/test/libyul/YulOptimizerTest.cpp b/test/libyul/YulOptimizerTest.cpp index 699a767d0..e488460fc 100644 --- a/test/libyul/YulOptimizerTest.cpp +++ b/test/libyul/YulOptimizerTest.cpp @@ -359,7 +359,7 @@ TestCase::TestResult YulOptimizerTest::run(ostream& _stream, string const& _line { disambiguate(); ExpressionSplitter::run(*m_context, *m_ast); - WordSizeTransform::run(*m_dialect, ""_yulstring, *m_ast, *m_nameDispenser); + WordSizeTransform::run(*m_dialect, *m_dialect, *m_ast, *m_nameDispenser); } else if (m_optimizerStep == "fullSuite") { diff --git a/test/libyul/yulOptimizerTests/wordSizeTransform/switch_3.yul b/test/libyul/yulOptimizerTests/wordSizeTransform/switch_3.yul index 50a36f61f..617eebb64 100644 --- a/test/libyul/yulOptimizerTests/wordSizeTransform/switch_3.yul +++ b/test/libyul/yulOptimizerTests/wordSizeTransform/switch_3.yul @@ -67,13 +67,13 @@ // let _10_3 := 3 // sstore(_10_0, _10_1, _10_2, _10_3, _9_0, _9_1, _9_2, _9_3) // } -// default { run_default := 1 } +// default { run_default := true } // } -// default { run_default := 1 } +// default { run_default := true } // } -// default { run_default := 1 } +// default { run_default := true } // } -// default { run_default := 1 } +// default { run_default := true } // if run_default // { // let _11_0 := 0 diff --git a/test/libyul/yulOptimizerTests/wordSizeTransform/switch_4.yul b/test/libyul/yulOptimizerTests/wordSizeTransform/switch_4.yul index 106941302..20f81d614 100644 --- a/test/libyul/yulOptimizerTests/wordSizeTransform/switch_4.yul +++ b/test/libyul/yulOptimizerTests/wordSizeTransform/switch_4.yul @@ -45,9 +45,9 @@ // let _8_3 := 2 // sstore(_8_0, _8_1, _8_2, _8_3, _7_0, _7_1, _7_2, _7_3) // } -// default { run_default := 1 } +// default { run_default := true } // } -// default { run_default := 1 } +// default { run_default := true } // } // case 536870912 { // switch _2_2 @@ -75,13 +75,13 @@ // let _10_3 := 3 // sstore(_10_0, _10_1, _10_2, _10_3, _9_0, _9_1, _9_2, _9_3) // } -// default { run_default := 1 } +// default { run_default := true } // } -// default { run_default := 1 } +// default { run_default := true } // } -// default { run_default := 1 } +// default { run_default := true } // } -// default { run_default := 1 } +// default { run_default := true } // if run_default // { // let _11_0 := 0 diff --git a/test/libyul/yulOptimizerTests/wordSizeTransform/switch_5.yul b/test/libyul/yulOptimizerTests/wordSizeTransform/switch_5.yul index eb05e0a6c..2a42adefe 100644 --- a/test/libyul/yulOptimizerTests/wordSizeTransform/switch_5.yul +++ b/test/libyul/yulOptimizerTests/wordSizeTransform/switch_5.yul @@ -13,7 +13,7 @@ // let _2_0, _2_1, _2_2, _2_3 := calldataload(_1_0, _1_1, _1_2, _1_3) // let run_default // switch _2_0 -// default { run_default := 1 } +// default { run_default := true } // if run_default // { // let _3_0 := 0 From 2efda4129baa3b545f3417587e6cc605f145b41e Mon Sep 17 00:00:00 2001 From: Djordje Mijovic Date: Wed, 19 Feb 2020 15:36:05 +0100 Subject: [PATCH 43/92] Adding test for multi return values including bool in evmTyped dialect. Calling zeroLiteralForType from inliner --- libyul/optimiser/FullInliner.cpp | 11 +++++----- libyul/optimiser/FullInliner.h | 11 ++++++---- .../fullInliner/multi_return_typed.yul | 22 +++++++++++++++++++ 3 files changed, 35 insertions(+), 9 deletions(-) create mode 100644 test/libyul/yulOptimizerTests/fullInliner/multi_return_typed.yul diff --git a/libyul/optimiser/FullInliner.cpp b/libyul/optimiser/FullInliner.cpp index 4b72e4ad9..32036184a 100644 --- a/libyul/optimiser/FullInliner.cpp +++ b/libyul/optimiser/FullInliner.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include #include @@ -41,11 +42,11 @@ using namespace solidity::yul; void FullInliner::run(OptimiserStepContext& _context, Block& _ast) { - FullInliner{_ast, _context.dispenser}.run(); + FullInliner{_ast, _context.dispenser, _context.dialect}.run(); } -FullInliner::FullInliner(Block& _ast, NameDispenser& _dispenser): - m_ast(_ast), m_nameDispenser(_dispenser) +FullInliner::FullInliner(Block& _ast, NameDispenser& _dispenser, Dialect const& _dialect): + m_ast(_ast), m_nameDispenser(_dispenser), m_dialect(_dialect) { // Determine constants SSAValueTracker tracker; @@ -139,7 +140,7 @@ void FullInliner::updateCodeSize(FunctionDefinition const& _fun) void FullInliner::handleBlock(YulString _currentFunctionName, Block& _block) { - InlineModifier{*this, m_nameDispenser, _currentFunctionName}(_block); + InlineModifier{*this, m_nameDispenser, _currentFunctionName, m_dialect}(_block); } bool FullInliner::recursive(FunctionDefinition const& _fun) const @@ -198,7 +199,7 @@ vector InlineModifier::performInline(Statement& _statement, FunctionC if (_value) varDecl.value = make_unique(std::move(*_value)); else - varDecl.value = make_unique(Literal{{}, LiteralKind::Number, YulString{"0"}, {}}); + varDecl.value = make_unique(m_dialect.zeroLiteralForType(varDecl.variables.front().type)); newStatements.emplace_back(std::move(varDecl)); }; diff --git a/libyul/optimiser/FullInliner.h b/libyul/optimiser/FullInliner.h index 788eea99b..81c608de8 100644 --- a/libyul/optimiser/FullInliner.h +++ b/libyul/optimiser/FullInliner.h @@ -69,7 +69,7 @@ class FullInliner: public ASTModifier { public: static constexpr char const* name{"FullInliner"}; - static void run(OptimiserStepContext&, Block& _ast); + static void run(OptimiserStepContext& _context, Block& _ast); /// Inlining heuristic. /// @param _callSite the name of the function in which the function call is located. @@ -89,7 +89,7 @@ public: void tentativelyUpdateCodeSize(YulString _function, YulString _callSite); private: - FullInliner(Block& _ast, NameDispenser& _dispenser); + FullInliner(Block& _ast, NameDispenser& _dispenser, Dialect const& _dialect); void run(); void updateCodeSize(FunctionDefinition const& _fun); @@ -108,6 +108,7 @@ private: std::set m_constants; std::map m_functionSizes; NameDispenser& m_nameDispenser; + Dialect const& m_dialect; }; /** @@ -117,10 +118,11 @@ private: class InlineModifier: public ASTModifier { public: - InlineModifier(FullInliner& _driver, NameDispenser& _nameDispenser, YulString _functionName): + InlineModifier(FullInliner& _driver, NameDispenser& _nameDispenser, YulString _functionName, Dialect const& _dialect): m_currentFunction(std::move(_functionName)), m_driver(_driver), - m_nameDispenser(_nameDispenser) + m_nameDispenser(_nameDispenser), + m_dialect(_dialect) { } void operator()(Block& _block) override; @@ -132,6 +134,7 @@ private: YulString m_currentFunction; FullInliner& m_driver; NameDispenser& m_nameDispenser; + Dialect const& m_dialect; }; /** diff --git a/test/libyul/yulOptimizerTests/fullInliner/multi_return_typed.yul b/test/libyul/yulOptimizerTests/fullInliner/multi_return_typed.yul new file mode 100644 index 000000000..f9f01a193 --- /dev/null +++ b/test/libyul/yulOptimizerTests/fullInliner/multi_return_typed.yul @@ -0,0 +1,22 @@ +{ + function f(a: u256) -> x: bool, y:u256 { + y := mul(a, a) + } + let r: bool, s: u256 := f(mload(3)) +} +// ==== +// dialect: evmTyped +// step: fullInliner +// ---- +// { +// { +// let a_3 := mload(3) +// let x_4:bool := false +// let y_5 := 0 +// y_5 := mul(a_3, a_3) +// let r:bool := x_4 +// let s := y_5 +// } +// function f(a) -> x:bool, y +// { y := mul(a, a) } +// } From c8915972046cc45edc37f1803a09c807df5b2140 Mon Sep 17 00:00:00 2001 From: Djordje Mijovic Date: Wed, 19 Feb 2020 17:34:08 +0100 Subject: [PATCH 44/92] Adding ssa type check and test for that one --- libyul/optimiser/SSATransform.cpp | 38 ++++++++++++----- libyul/optimiser/TypeInfo.cpp | 5 +++ libyul/optimiser/TypeInfo.h | 3 ++ .../yulOptimizerTests/ssaTransform/typed.yul | 41 +++++++++++++++++++ .../ssaTransform/typed_for.yul | 25 +++++++++++ .../ssaTransform/typed_switch.yul | 25 +++++++++++ 6 files changed, 127 insertions(+), 10 deletions(-) create mode 100644 test/libyul/yulOptimizerTests/ssaTransform/typed.yul create mode 100644 test/libyul/yulOptimizerTests/ssaTransform/typed_for.yul create mode 100644 test/libyul/yulOptimizerTests/ssaTransform/typed_switch.yul diff --git a/libyul/optimiser/SSATransform.cpp b/libyul/optimiser/SSATransform.cpp index 91454b9ee..948a0e6a6 100644 --- a/libyul/optimiser/SSATransform.cpp +++ b/libyul/optimiser/SSATransform.cpp @@ -27,6 +27,8 @@ #include +#include + using namespace std; using namespace solidity; using namespace solidity::yul; @@ -42,8 +44,14 @@ namespace class IntroduceSSA: public ASTModifier { public: - explicit IntroduceSSA(NameDispenser& _nameDispenser, set const& _variablesToReplace): - m_nameDispenser(_nameDispenser), m_variablesToReplace(_variablesToReplace) + explicit IntroduceSSA( + NameDispenser& _nameDispenser, + set const& _variablesToReplace, + TypeInfo& _typeInfo + ): + m_nameDispenser(_nameDispenser), + m_variablesToReplace(_variablesToReplace), + m_typeInfo(_typeInfo) { } void operator()(Block& _block) override; @@ -51,6 +59,7 @@ public: private: NameDispenser& m_nameDispenser; set const& m_variablesToReplace; + TypeInfo const& m_typeInfo; }; @@ -83,10 +92,10 @@ void IntroduceSSA::operator()(Block& _block) { YulString oldName = var.name; YulString newName = m_nameDispenser.newName(oldName); - newVariables.emplace_back(TypedName{loc, newName, {}}); + newVariables.emplace_back(TypedName{loc, newName, var.type}); statements.emplace_back(VariableDeclaration{ loc, - {TypedName{loc, oldName, {}}}, + {TypedName{loc, oldName, var.type}}, make_unique(Identifier{loc, newName}) }); } @@ -110,7 +119,11 @@ void IntroduceSSA::operator()(Block& _block) { YulString oldName = var.name; YulString newName = m_nameDispenser.newName(oldName); - newVariables.emplace_back(TypedName{loc, newName, {}}); + newVariables.emplace_back(TypedName{ + loc, + newName, + m_typeInfo.typeOfVariable(oldName) + }); statements.emplace_back(Assignment{ loc, {Identifier{loc, oldName}}, @@ -136,9 +149,12 @@ class IntroduceControlFlowSSA: public ASTModifier public: explicit IntroduceControlFlowSSA( NameDispenser& _nameDispenser, - set const& _variablesToReplace + set const& _variablesToReplace, + TypeInfo const& _typeInfo ): - m_nameDispenser(_nameDispenser), m_variablesToReplace(_variablesToReplace) + m_nameDispenser(_nameDispenser), + m_variablesToReplace(_variablesToReplace), + m_typeInfo(_typeInfo) { } void operator()(FunctionDefinition& _function) override; @@ -153,6 +169,7 @@ private: set m_variablesInScope; /// Set of variables that do not have a specific value. set m_variablesToReassign; + TypeInfo const& m_typeInfo; }; void IntroduceControlFlowSSA::operator()(FunctionDefinition& _function) @@ -221,7 +238,7 @@ void IntroduceControlFlowSSA::operator()(Block& _block) YulString newName = m_nameDispenser.newName(toReassign); toPrepend.emplace_back(VariableDeclaration{ locationOf(_s), - {TypedName{locationOf(_s), newName, {}}}, + {TypedName{locationOf(_s), newName, m_typeInfo.typeOfVariable(toReassign)}}, make_unique(Identifier{locationOf(_s), toReassign}) }); assignedVariables.insert(toReassign); @@ -375,10 +392,11 @@ void PropagateValues::operator()(Block& _block) void SSATransform::run(OptimiserStepContext& _context, Block& _ast) { + TypeInfo typeInfo(_context.dialect, _ast); Assignments assignments; assignments(_ast); - IntroduceSSA{_context.dispenser, assignments.names()}(_ast); - IntroduceControlFlowSSA{_context.dispenser, assignments.names()}(_ast); + IntroduceSSA{_context.dispenser, assignments.names(), typeInfo}(_ast); + IntroduceControlFlowSSA{_context.dispenser, assignments.names(), typeInfo}(_ast); PropagateValues{assignments.names()}(_ast); } diff --git a/libyul/optimiser/TypeInfo.cpp b/libyul/optimiser/TypeInfo.cpp index 4b5715baa..ea2d81a83 100644 --- a/libyul/optimiser/TypeInfo.cpp +++ b/libyul/optimiser/TypeInfo.cpp @@ -96,3 +96,8 @@ YulString TypeInfo::typeOf(Expression const& _expression) const } }, _expression); } + +YulString TypeInfo::typeOfVariable(YulString _name) const +{ + return m_variableTypes.at(_name); +} diff --git a/libyul/optimiser/TypeInfo.h b/libyul/optimiser/TypeInfo.h index deb537ea3..48c6c1fba 100644 --- a/libyul/optimiser/TypeInfo.h +++ b/libyul/optimiser/TypeInfo.h @@ -44,6 +44,9 @@ public: /// @returns the type of an expression that is assumed to return exactly one value. YulString typeOf(Expression const& _expression) const; + /// \returns the type of variable + YulString typeOfVariable(YulString _name) const; + private: class TypeCollector; diff --git a/test/libyul/yulOptimizerTests/ssaTransform/typed.yul b/test/libyul/yulOptimizerTests/ssaTransform/typed.yul new file mode 100644 index 000000000..f9d48a227 --- /dev/null +++ b/test/libyul/yulOptimizerTests/ssaTransform/typed.yul @@ -0,0 +1,41 @@ +{ + let b:bool := true + let c:bool := false + c := b + b := false + + let a:u256 := 1 + a := add(a, 1) + if c { + a := add(a, 1) + } + a := add(a, 1) + mstore(a, 1) +} +// ==== +// dialect: evmTyped +// step: ssaTransform +// ---- +// { +// let b_1:bool := true +// let b:bool := b_1 +// let c_2:bool := false +// let c:bool := c_2 +// let c_3:bool := b_1 +// c := c_3 +// let b_4:bool := false +// b := b_4 +// let a_5 := 1 +// let a := a_5 +// let a_6 := add(a_5, 1) +// a := a_6 +// if c_3 +// { +// let a_7 := add(a_6, 1) +// a := a_7 +// } +// let a_9 := a +// let a_8 := add(a_9, 1) +// a := a_8 +// mstore(a_8, 1) +// } diff --git a/test/libyul/yulOptimizerTests/ssaTransform/typed_for.yul b/test/libyul/yulOptimizerTests/ssaTransform/typed_for.yul new file mode 100644 index 000000000..685c42a24 --- /dev/null +++ b/test/libyul/yulOptimizerTests/ssaTransform/typed_for.yul @@ -0,0 +1,25 @@ +{ + let b:bool := true + let c:bool := false + for {} b {} { + c := true + } + let d: bool := c +} +// ==== +// dialect: evmTyped +// step: ssaTransform +// ---- +// { +// let b:bool := true +// let c_1:bool := false +// let c:bool := c_1 +// for { } b { } +// { +// let c_3:bool := c +// let c_2:bool := true +// c := c_2 +// } +// let c_4:bool := c +// let d:bool := c_4 +// } diff --git a/test/libyul/yulOptimizerTests/ssaTransform/typed_switch.yul b/test/libyul/yulOptimizerTests/ssaTransform/typed_switch.yul new file mode 100644 index 000000000..7d61b1031 --- /dev/null +++ b/test/libyul/yulOptimizerTests/ssaTransform/typed_switch.yul @@ -0,0 +1,25 @@ +{ + let b:bool := true + let c:bool := false + switch b + case true { c := true} + case false { } + let d: bool := c +} +// ==== +// dialect: evmTyped +// step: ssaTransform +// ---- +// { +// let b:bool := true +// let c_1:bool := false +// let c:bool := c_1 +// switch b +// case true { +// let c_2:bool := true +// c := c_2 +// } +// case false { } +// let c_3:bool := c +// let d:bool := c_3 +// } From 8524e3f48d17f1c063c984d7341b6565c497a7d5 Mon Sep 17 00:00:00 2001 From: Djordje Mijovic Date: Mon, 24 Feb 2020 18:54:23 +0100 Subject: [PATCH 45/92] Standard-JSON-Interface: Fix a bug (#8371) related to empty filenames and imports. --- Changelog.md | 1 + libsolidity/interface/CompilerStack.cpp | 1 - test/cmdlineTests/standard_empty_file_name/exit | 1 + test/cmdlineTests/standard_empty_file_name/input.json | 10 ++++++++++ test/cmdlineTests/standard_empty_file_name/output.json | 4 ++++ 5 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 test/cmdlineTests/standard_empty_file_name/exit create mode 100644 test/cmdlineTests/standard_empty_file_name/input.json create mode 100644 test/cmdlineTests/standard_empty_file_name/output.json diff --git a/Changelog.md b/Changelog.md index 9981ea25f..10c32d884 100644 --- a/Changelog.md +++ b/Changelog.md @@ -8,6 +8,7 @@ Compiler Features: Bugfixes: * isoltest: Added new keyword `wei` to express function value in semantic tests + * Standard-JSON-Interface: Fix a bug related to empty filenames and imports. ### 0.6.3 (2020-02-18) diff --git a/libsolidity/interface/CompilerStack.cpp b/libsolidity/interface/CompilerStack.cpp index 89ab13818..6cf044c1c 100644 --- a/libsolidity/interface/CompilerStack.cpp +++ b/libsolidity/interface/CompilerStack.cpp @@ -1005,7 +1005,6 @@ void CompilerStack::resolveImports() if (ImportDirective const* import = dynamic_cast(node.get())) { string const& path = import->annotation().absolutePath; - solAssert(!path.empty(), ""); solAssert(m_sources.count(path), ""); import->annotation().sourceUnit = m_sources[path].ast.get(); toposort(&m_sources[path]); diff --git a/test/cmdlineTests/standard_empty_file_name/exit b/test/cmdlineTests/standard_empty_file_name/exit new file mode 100644 index 000000000..573541ac9 --- /dev/null +++ b/test/cmdlineTests/standard_empty_file_name/exit @@ -0,0 +1 @@ +0 diff --git a/test/cmdlineTests/standard_empty_file_name/input.json b/test/cmdlineTests/standard_empty_file_name/input.json new file mode 100644 index 000000000..95c2cdd30 --- /dev/null +++ b/test/cmdlineTests/standard_empty_file_name/input.json @@ -0,0 +1,10 @@ +{ + "language": "Solidity", + "sources": + { + "": + { + "content": "pragma solidity >=0.0; import {A} from \".\";" + } + } +} diff --git a/test/cmdlineTests/standard_empty_file_name/output.json b/test/cmdlineTests/standard_empty_file_name/output.json new file mode 100644 index 000000000..b31ceb152 --- /dev/null +++ b/test/cmdlineTests/standard_empty_file_name/output.json @@ -0,0 +1,4 @@ +{"errors":[{"component":"general","formattedMessage":":1:24: DeclarationError: Declaration \"A\" not found in \"\" (referenced as \".\"). +pragma solidity >=0.0; import {A} from \".\"; + ^------------------^ +","message":"Declaration \"A\" not found in \"\" (referenced as \".\").","severity":"error","type":"DeclarationError"}],"sources":{}} From 24eb39ca1146905efaf2f44807e35eec2035500d Mon Sep 17 00:00:00 2001 From: Bhargava Shastry Date: Thu, 20 Feb 2020 20:02:28 +0530 Subject: [PATCH 46/92] Docker: Add image identical to one used by ossfuzz builder --- .circleci/config.yml | 32 ++++-- .../Dockerfile.ubuntu1604.clang.ossfuzz | 101 ++++++++++++++++++ ...ubuntu1904 => Dockerfile.ubuntu1904.clang} | 0 cmake/toolchains/libfuzzer.cmake | 6 +- 4 files changed, 132 insertions(+), 7 deletions(-) create mode 100644 .circleci/docker/Dockerfile.ubuntu1604.clang.ossfuzz rename .circleci/docker/{Dockerfile.clang.ubuntu1904 => Dockerfile.ubuntu1904.clang} (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index f8f142412..8bee0b83b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,6 +16,9 @@ parameters: ubuntu-1904-clang-docker-image-rev: type: string default: "5" + ubuntu-1604-clang-ossfuzz-docker-image-rev: + type: string + default: "1" defaults: @@ -119,6 +122,17 @@ defaults: name: command line tests command: ./test/cmdlineTests.sh + - test_ubuntu1604_clang: &test_ubuntu1604_clang + docker: + - image: ethereum/solidity-buildpack-deps:ubuntu1604-clang-ossfuzz-<< pipeline.parameters.ubuntu-1604-clang-ossfuzz-docker-image-rev >> + steps: + - checkout + - attach_workspace: + at: build + - run: *run_soltest + - store_test_results: *store_test_results + - store_artifacts: *artifacts_test_results + - test_ubuntu1904_clang: &test_ubuntu1904_clang docker: - image: ethereum/solidity-buildpack-deps:ubuntu1904-clang-<< pipeline.parameters.ubuntu-1904-clang-docker-image-rev >> @@ -166,6 +180,11 @@ defaults: requires: - b_ubu + - workflow_ubuntu1604_clang: &workflow_ubuntu1604_clang + <<: *workflow_trigger_on_tags + requires: + - b_ubu_ossfuzz + - workflow_ubuntu1904_clang: &workflow_ubuntu1904_clang <<: *workflow_trigger_on_tags requires: @@ -196,7 +215,7 @@ defaults: requires: - b_ems - - workflow_ubuntu1904_ossfuzz: &workflow_ubuntu1904_ossfuzz + - workflow_ubuntu1604_ossfuzz: &workflow_ubuntu1604_ossfuzz <<: *workflow_trigger_on_tags requires: - b_ubu_ossfuzz @@ -397,12 +416,13 @@ jobs: - checkout - run: *run_build - b_ubu_ossfuzz: - <<: *build_ubuntu1904_clang + b_ubu_ossfuzz: &build_ubuntu1604_clang + docker: + - image: ethereum/solidity-buildpack-deps:ubuntu1604-clang-ossfuzz-<< pipeline.parameters.ubuntu-1604-clang-ossfuzz-docker-image-rev >> environment: - TERM: xterm CC: clang CXX: clang++ + TERM: xterm CMAKE_OPTIONS: -DCMAKE_TOOLCHAIN_FILE=cmake/toolchains/libfuzzer.cmake steps: - checkout @@ -411,7 +431,7 @@ jobs: - persist_to_workspace: *artifacts_executables_ossfuzz t_ubu_ossfuzz: &t_ubu_ossfuzz - <<: *test_ubuntu1904_clang + <<: *test_ubuntu1604_clang steps: - checkout - attach_workspace: @@ -799,7 +819,7 @@ workflows: jobs: # OSSFUZZ builds and (regression) tests - b_ubu_ossfuzz: *workflow_trigger_on_tags - - t_ubu_ossfuzz: *workflow_ubuntu1904_ossfuzz + - t_ubu_ossfuzz: *workflow_ubuntu1604_ossfuzz # Code Coverage enabled build and tests - b_ubu_codecov: *workflow_trigger_on_tags diff --git a/.circleci/docker/Dockerfile.ubuntu1604.clang.ossfuzz b/.circleci/docker/Dockerfile.ubuntu1604.clang.ossfuzz new file mode 100644 index 000000000..8e2ee01b0 --- /dev/null +++ b/.circleci/docker/Dockerfile.ubuntu1604.clang.ossfuzz @@ -0,0 +1,101 @@ +# vim:syntax=dockerfile +#------------------------------------------------------------------------------ +# Dockerfile for building and testing Solidity Compiler on CI +# Target: Ubuntu 16.04 (Xenial Xerus) ossfuzz Clang variant +# URL: https://hub.docker.com/r/ethereum/solidity-buildpack-deps +# +# This file is part of solidity. +# +# solidity is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# solidity is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with solidity. If not, see +# +# (c) 2016-2019 solidity contributors. +#------------------------------------------------------------------------------ +FROM gcr.io/oss-fuzz-base/base-clang as base + +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt-get update; \ + apt-get -qqy install --no-install-recommends \ + build-essential \ + software-properties-common \ + ninja-build git wget \ + libbz2-dev zlib1g-dev git; \ + apt-get install -qy python-pip python-sphinx; + +# Install cmake 3.14 (minimum requirement is cmake 3.10) +RUN wget https://github.com/Kitware/CMake/releases/download/v3.14.5/cmake-3.14.5-Linux-x86_64.sh; \ + chmod +x cmake-3.14.5-Linux-x86_64.sh; \ + ./cmake-3.14.5-Linux-x86_64.sh --skip-license --prefix="/usr" + +FROM base AS libraries + +# Boost +RUN git clone -b boost-1.69.0 https://github.com/boostorg/boost.git \ + /usr/src/boost; \ + cd /usr/src/boost; \ + git submodule update --init --recursive; \ + ./bootstrap.sh --with-toolset=clang --prefix=/usr; \ + ./b2 toolset=clang cxxflags="-stdlib=libc++" linkflags="-stdlib=libc++" headers; \ + ./b2 toolset=clang cxxflags="-stdlib=libc++" linkflags="-stdlib=libc++" \ + link=static variant=release runtime-link=static \ + system filesystem unit_test_framework program_options \ + install -j $(($(nproc)/2)); \ + rm -rf /usr/src/boost + +# Z3 +RUN git clone --depth 1 -b z3-4.8.7 https://github.com/Z3Prover/z3.git \ + /usr/src/z3; \ + cd /usr/src/z3; \ + mkdir build; \ + cd build; \ + LDFLAGS=$CXXFLAGS cmake -DZ3_BUILD_LIBZ3_SHARED=OFF -DCMAKE_INSTALL_PREFIX=/usr \ + -DCMAKE_BUILD_TYPE=Release ..; \ + make libz3 -j; \ + make install; \ + rm -rf /usr/src/z3 + +# OSSFUZZ: libprotobuf-mutator +RUN set -ex; \ + git clone https://github.com/google/libprotobuf-mutator.git \ + /usr/src/libprotobuf-mutator; \ + cd /usr/src/libprotobuf-mutator; \ + git checkout 3521f47a2828da9ace403e4ecc4aece1a84feb36; \ + mkdir build; \ + cd build; \ + cmake .. -GNinja -DLIB_PROTO_MUTATOR_DOWNLOAD_PROTOBUF=ON \ + -DLIB_PROTO_MUTATOR_TESTING=OFF -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="/usr"; \ + ninja; \ + cp -vpr external.protobuf/bin/* /usr/bin/; \ + cp -vpr external.protobuf/include/* /usr/include/; \ + cp -vpr external.protobuf/lib/* /usr/lib/; \ + ninja install/strip; \ + rm -rf /usr/src/libprotobuf-mutator + +# EVMONE +RUN set -ex; \ + cd /usr/src; \ + git clone --branch="v0.4.0" --recurse-submodules https://github.com/ethereum/evmone.git; \ + cd evmone; \ + mkdir build; \ + cd build; \ + cmake -G Ninja -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX="/usr" ..; \ + ninja; \ + ninja install/strip; \ + rm -rf /usr/src/evmone + +FROM base +COPY --from=libraries /usr/lib /usr/lib +COPY --from=libraries /usr/bin /usr/bin +COPY --from=libraries /usr/include /usr/include diff --git a/.circleci/docker/Dockerfile.clang.ubuntu1904 b/.circleci/docker/Dockerfile.ubuntu1904.clang similarity index 100% rename from .circleci/docker/Dockerfile.clang.ubuntu1904 rename to .circleci/docker/Dockerfile.ubuntu1904.clang diff --git a/cmake/toolchains/libfuzzer.cmake b/cmake/toolchains/libfuzzer.cmake index 53c36d3c9..a734df975 100644 --- a/cmake/toolchains/libfuzzer.cmake +++ b/cmake/toolchains/libfuzzer.cmake @@ -8,4 +8,8 @@ set(OSSFUZZ ON CACHE BOOL "Enable fuzzer build" FORCE) # Use libfuzzer as the fuzzing back-end set(LIB_FUZZING_ENGINE "-fsanitize=fuzzer" CACHE STRING "Use libfuzzer back-end" FORCE) # clang/libfuzzer specific flags for UBSan instrumentation -set(CMAKE_CXX_FLAGS "-O1 -gline-tables-only -fsanitize=undefined -fsanitize=fuzzer-no-link -stdlib=libstdc++" CACHE STRING "Custom compilation flags" FORCE) +set(CMAKE_CXX_FLAGS "-O1 -fno-omit-frame-pointer -gline-tables-only -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -I /usr/local/include/c++/v1 -fsanitize=undefined -fsanitize=fuzzer-no-link -stdlib=libc++" CACHE STRING "Custom compilation flags" FORCE) +# Link statically against boost libraries +set(BOOST_FOUND ON CACHE BOOL "" FORCE) +set(Boost_USE_STATIC_LIBS ON CACHE BOOL "Link against static Boost libraries" FORCE) +set(Boost_USE_STATIC_RUNTIME ON CACHE BOOL "Link against static Boost runtime library" FORCE) From 45041e5d3a155782150ca2a09debb1d4cbfd7eae Mon Sep 17 00:00:00 2001 From: chriseth Date: Mon, 24 Feb 2020 15:53:32 +0100 Subject: [PATCH 47/92] Allow access to ``_slot`` for local storage pointer variables. --- Changelog.md | 1 + docs/assembly.rst | 6 +++- libsolidity/analysis/TypeChecker.cpp | 16 +++++++-- libsolidity/codegen/ContractCompiler.cpp | 2 +- .../inlineAssembly/slot_access.sol | 33 +++++++++++++++++++ .../storage_reference_assignment.sol | 3 +- .../storage_reference_assignment_statevar.sol | 12 +++++++ .../inlineAssembly/storage_slot_assign.yul | 12 +++++++ 8 files changed, 78 insertions(+), 7 deletions(-) create mode 100644 test/libsolidity/semanticTests/inlineAssembly/slot_access.sol create mode 100644 test/libsolidity/syntaxTests/inlineAssembly/storage_reference_assignment_statevar.sol create mode 100644 test/libsolidity/syntaxTests/inlineAssembly/storage_slot_assign.yul diff --git a/Changelog.md b/Changelog.md index 9981ea25f..6a829845a 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,6 +1,7 @@ ### 0.6.4 (unreleased) Language Features: + * Inline Assembly: Allow assigning to `_slot` of local storage variable pointers. Compiler Features: diff --git a/docs/assembly.rst b/docs/assembly.rst index 67c1ab6f6..7e498cd5a 100644 --- a/docs/assembly.rst +++ b/docs/assembly.rst @@ -172,6 +172,11 @@ Assignments are possible to assembly-local variables and to function-local variables. Take care that when you assign to variables that point to memory or storage, you will only change the pointer and not the data. +You can assign to the ``_slot`` part of a local storage variable pointer. +For these (structs, arrays or mappings), the ``_offset`` part is always zero. +It is not possible to assign to the ``_slot`` or ``_offset`` part of a state variable, +though. + Things to Avoid @@ -225,4 +230,3 @@ first slot of the array and followed by the array elements. Statically-sized memory arrays do not have a length field, but it might be added later to allow better convertibility between statically- and dynamically-sized arrays, so do not rely on this. - diff --git a/libsolidity/analysis/TypeChecker.cpp b/libsolidity/analysis/TypeChecker.cpp index e41425c6e..2f9526f87 100644 --- a/libsolidity/analysis/TypeChecker.cpp +++ b/libsolidity/analysis/TypeChecker.cpp @@ -677,10 +677,20 @@ bool TypeChecker::visit(InlineAssembly const& _inlineAssembly) m_errorReporter.typeError(_identifier.location, "The suffixes _offset and _slot can only be used on storage variables."); return size_t(-1); } - else if (_context != yul::IdentifierContext::RValue) + else if (_context == yul::IdentifierContext::LValue) { - m_errorReporter.typeError(_identifier.location, "Storage variables cannot be assigned to."); - return size_t(-1); + if (var->isStateVariable()) + { + m_errorReporter.typeError(_identifier.location, "State variables cannot be assigned to - you have to use \"sstore()\"."); + return size_t(-1); + } + else if (ref->second.isOffset) + { + m_errorReporter.typeError(_identifier.location, "Only _slot can be assigned to."); + return size_t(-1); + } + else + solAssert(ref->second.isSlot, ""); } } else if (!var->isConstant() && var->isStateVariable()) diff --git a/libsolidity/codegen/ContractCompiler.cpp b/libsolidity/codegen/ContractCompiler.cpp index 298437b1d..a99aefb5f 100644 --- a/libsolidity/codegen/ContractCompiler.cpp +++ b/libsolidity/codegen/ContractCompiler.cpp @@ -781,7 +781,7 @@ bool ContractCompiler::visit(InlineAssembly const& _inlineAssembly) else { // lvalue context - solAssert(!ref->second.isOffset && !ref->second.isSlot, ""); + solAssert(!ref->second.isOffset, ""); auto variable = dynamic_cast(decl); solAssert( !!variable && m_context.isLocalVariable(variable), diff --git a/test/libsolidity/semanticTests/inlineAssembly/slot_access.sol b/test/libsolidity/semanticTests/inlineAssembly/slot_access.sol new file mode 100644 index 000000000..4f3be7489 --- /dev/null +++ b/test/libsolidity/semanticTests/inlineAssembly/slot_access.sol @@ -0,0 +1,33 @@ +contract C { + struct S { + uint a; + uint b; + } + + mapping(uint => S) public mappingAccess; + + function data() internal view returns (S storage _data) { + // We need to assign it from somewhere, otherwise we would + // get an "uninitialized access" error. + _data = mappingAccess[20]; + + bytes32 slot = keccak256(abi.encode(uint(1), uint(0))); + assembly { + _data_slot := slot + } + } + + function set(uint x) public { + data().a = x; + } + + function get() public view returns (uint) { + return data().a; + } +} +// ---- +// get() -> 0 +// mappingAccess(uint256): 1 -> 0, 0 +// set(uint256): 4 +// get() -> 4 +// mappingAccess(uint256): 1 -> 4, 0 diff --git a/test/libsolidity/syntaxTests/inlineAssembly/storage_reference_assignment.sol b/test/libsolidity/syntaxTests/inlineAssembly/storage_reference_assignment.sol index 164265781..91362d719 100644 --- a/test/libsolidity/syntaxTests/inlineAssembly/storage_reference_assignment.sol +++ b/test/libsolidity/syntaxTests/inlineAssembly/storage_reference_assignment.sol @@ -9,5 +9,4 @@ contract C { } } // ---- -// TypeError: (114-120): Storage variables cannot be assigned to. -// TypeError: (138-146): Storage variables cannot be assigned to. +// TypeError: (138-146): Only _slot can be assigned to. diff --git a/test/libsolidity/syntaxTests/inlineAssembly/storage_reference_assignment_statevar.sol b/test/libsolidity/syntaxTests/inlineAssembly/storage_reference_assignment_statevar.sol new file mode 100644 index 000000000..d37008774 --- /dev/null +++ b/test/libsolidity/syntaxTests/inlineAssembly/storage_reference_assignment_statevar.sol @@ -0,0 +1,12 @@ +contract C { + uint[] x; + fallback() external { + assembly { + x_slot := 1 + x_offset := 2 + } + } +} +// ---- +// TypeError: (84-90): State variables cannot be assigned to - you have to use "sstore()". +// TypeError: (108-116): State variables cannot be assigned to - you have to use "sstore()". diff --git a/test/libsolidity/syntaxTests/inlineAssembly/storage_slot_assign.yul b/test/libsolidity/syntaxTests/inlineAssembly/storage_slot_assign.yul new file mode 100644 index 000000000..91362d719 --- /dev/null +++ b/test/libsolidity/syntaxTests/inlineAssembly/storage_slot_assign.yul @@ -0,0 +1,12 @@ +contract C { + uint[] x; + fallback() external { + uint[] storage y = x; + assembly { + y_slot := 1 + y_offset := 2 + } + } +} +// ---- +// TypeError: (138-146): Only _slot can be assigned to. From 26bae6b4599cc00122fa0c3a38937f4f6395e454 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Fri, 14 Feb 2020 20:37:49 +0100 Subject: [PATCH 48/92] [yul-phaser] Common: Add countSubstringOccurrences() --- test/yulPhaser/Common.cpp | 15 +++++++++++++++ test/yulPhaser/Common.h | 4 ++++ test/yulPhaser/CommonTest.cpp | 19 +++++++++++++++++++ 3 files changed, 38 insertions(+) diff --git a/test/yulPhaser/Common.cpp b/test/yulPhaser/Common.cpp index 38c6e445b..93aa432fa 100644 --- a/test/yulPhaser/Common.cpp +++ b/test/yulPhaser/Common.cpp @@ -49,3 +49,18 @@ string phaser::test::stripWhitespace(string const& input) regex whitespaceRegex("\\s+"); return regex_replace(input, whitespaceRegex, ""); } + +size_t phaser::test::countSubstringOccurrences(string const& _inputString, string const& _substring) +{ + assert(_substring.size() > 0); + + size_t count = 0; + size_t lastOccurrence = 0; + while ((lastOccurrence = _inputString.find(_substring, lastOccurrence)) != string::npos) + { + ++count; + lastOccurrence += _substring.size(); + } + + return count; +} diff --git a/test/yulPhaser/Common.h b/test/yulPhaser/Common.h index 63ca45c50..e73260d7a 100644 --- a/test/yulPhaser/Common.h +++ b/test/yulPhaser/Common.h @@ -67,6 +67,10 @@ std::map enumerateOptmisationSteps(); /// Returns the input string with all the whitespace characters (spaces, line endings, etc.) removed. std::string stripWhitespace(std::string const& input); +/// Counts the number of times one strinng can be found inside another. Only non-overlapping +/// occurrences are counted. +size_t countSubstringOccurrences(std::string const& _inputString, std::string const& _substring); + // STATISTICAL UTILITIES /// Calculates the mean value of a series of samples given in a vector. diff --git a/test/yulPhaser/CommonTest.cpp b/test/yulPhaser/CommonTest.cpp index feded8292..9bcab9923 100644 --- a/test/yulPhaser/CommonTest.cpp +++ b/test/yulPhaser/CommonTest.cpp @@ -83,6 +83,25 @@ BOOST_AUTO_TEST_CASE(stripWhitespace_should_remove_all_whitespace_characters_fro BOOST_TEST(stripWhitespace(" a b \n\n c \n\t\v") == "abc"); } +BOOST_AUTO_TEST_CASE(countSubstringOccurrences_should_count_non_overlapping_substring_occurrences_in_a_string) +{ + BOOST_TEST(countSubstringOccurrences("aaabcdcbaaa", "a") == 6); + BOOST_TEST(countSubstringOccurrences("aaabcdcbaaa", "aa") == 2); + BOOST_TEST(countSubstringOccurrences("aaabcdcbaaa", "aaa") == 2); + BOOST_TEST(countSubstringOccurrences("aaabcdcbaaa", "aaab") == 1); + BOOST_TEST(countSubstringOccurrences("aaabcdcbaaa", "b") == 2); + BOOST_TEST(countSubstringOccurrences("aaabcdcbaaa", "d") == 1); + BOOST_TEST(countSubstringOccurrences("aaabcdcbaaa", "cdc") == 1); + + BOOST_TEST(countSubstringOccurrences("aaabcdcbaaa", "x") == 0); + BOOST_TEST(countSubstringOccurrences("aaabcdcbaaa", "aaaa") == 0); + BOOST_TEST(countSubstringOccurrences("aaabcdcbaaa", "dcd") == 0); + + BOOST_TEST(countSubstringOccurrences("", "a") == 0); + BOOST_TEST(countSubstringOccurrences("", "aa") == 0); + BOOST_TEST(countSubstringOccurrences("a", "aa") == 0); +} + BOOST_AUTO_TEST_CASE(mean_should_calculate_statistical_mean) { BOOST_TEST(mean({0}) == 0.0); From 11bdf358df078d9fb69136b79a788164f4636b22 Mon Sep 17 00:00:00 2001 From: cameel Date: Wed, 5 Feb 2020 14:56:55 +0100 Subject: [PATCH 49/92] [yul-phaser] Base class for genetic algorithms --- test/CMakeLists.txt | 2 + test/yulPhaser/GeneticAlgorithms.cpp | 96 +++++++++++++++++++++++++++ tools/CMakeLists.txt | 2 + tools/yulPhaser/GeneticAlgorithms.cpp | 32 +++++++++ tools/yulPhaser/GeneticAlgorithms.h | 66 ++++++++++++++++++ 5 files changed, 198 insertions(+) create mode 100644 test/yulPhaser/GeneticAlgorithms.cpp create mode 100644 tools/yulPhaser/GeneticAlgorithms.cpp create mode 100644 tools/yulPhaser/GeneticAlgorithms.h diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 3b2fef6f0..70d547a0b 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -144,6 +144,7 @@ set(yul_phaser_sources yulPhaser/CommonTest.cpp yulPhaser/Chromosome.cpp yulPhaser/FitnessMetrics.cpp + yulPhaser/GeneticAlgorithms.cpp yulPhaser/Population.cpp yulPhaser/Program.cpp yulPhaser/SimulationRNG.cpp @@ -153,6 +154,7 @@ set(yul_phaser_sources # unnecessary duplication. Create a library or find a way to reuse the list in both places. ../tools/yulPhaser/Chromosome.cpp ../tools/yulPhaser/FitnessMetrics.cpp + ../tools/yulPhaser/GeneticAlgorithms.cpp ../tools/yulPhaser/Population.cpp ../tools/yulPhaser/Program.cpp ../tools/yulPhaser/SimulationRNG.cpp diff --git a/test/yulPhaser/GeneticAlgorithms.cpp b/test/yulPhaser/GeneticAlgorithms.cpp new file mode 100644 index 000000000..3528f2be0 --- /dev/null +++ b/test/yulPhaser/GeneticAlgorithms.cpp @@ -0,0 +1,96 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#include + +#include +#include +#include +#include + +#include + +#include + +#include +#include + +#include + +using namespace std; +using namespace boost::unit_test::framework; +using namespace boost::test_tools; +using namespace solidity::langutil; +using namespace solidity::util; + +namespace solidity::phaser::test +{ + +class DummyAlgorithm: public GeneticAlgorithm +{ +public: + using GeneticAlgorithm::GeneticAlgorithm; + void runNextRound() override { ++m_currentRound; } + + size_t m_currentRound = 0; +}; + +class GeneticAlgorithmFixture +{ +protected: + shared_ptr m_fitnessMetric = make_shared(); + output_test_stream m_output; +}; + +BOOST_AUTO_TEST_SUITE(Phaser) +BOOST_AUTO_TEST_SUITE(GeneticAlgorithmsTest) +BOOST_AUTO_TEST_SUITE(GeneticAlgorithmTest) + +BOOST_FIXTURE_TEST_CASE(run_should_call_runNextRound_once_per_round, GeneticAlgorithmFixture) +{ + DummyAlgorithm algorithm(Population(m_fitnessMetric), m_output); + + BOOST_TEST(algorithm.m_currentRound == 0); + algorithm.run(10); + BOOST_TEST(algorithm.m_currentRound == 10); + algorithm.run(3); + BOOST_TEST(algorithm.m_currentRound == 13); +} + +BOOST_FIXTURE_TEST_CASE(run_should_print_the_top_chromosome, GeneticAlgorithmFixture) +{ + // run() is allowed to print more but should at least print the first one + + DummyAlgorithm algorithm( + // NOTE: Chromosomes chosen so that they're not substrings of each other and are not + // words likely to appear in the output in normal circumstances. + Population(m_fitnessMetric, {Chromosome("fcCUnDve"), Chromosome("jsxIOo"), Chromosome("ighTLM")}), + m_output + ); + + BOOST_TEST(m_output.is_empty()); + algorithm.run(1); + BOOST_TEST(countSubstringOccurrences(m_output.str(), toString(algorithm.population().individuals()[0].chromosome)) == 1); + algorithm.run(3); + BOOST_TEST(countSubstringOccurrences(m_output.str(), toString(algorithm.population().individuals()[0].chromosome)) == 4); +} + +BOOST_AUTO_TEST_SUITE_END() +BOOST_AUTO_TEST_SUITE_END() +BOOST_AUTO_TEST_SUITE_END() + +} diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index d3101ad7b..601b249e6 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -15,6 +15,8 @@ install(TARGETS solidity-upgrade DESTINATION "${CMAKE_INSTALL_BINDIR}") add_executable(yul-phaser yulPhaser/main.cpp + yulPhaser/GeneticAlgorithms.h + yulPhaser/GeneticAlgorithms.cpp yulPhaser/Population.h yulPhaser/Population.cpp yulPhaser/FitnessMetrics.h diff --git a/tools/yulPhaser/GeneticAlgorithms.cpp b/tools/yulPhaser/GeneticAlgorithms.cpp new file mode 100644 index 000000000..6e69565f4 --- /dev/null +++ b/tools/yulPhaser/GeneticAlgorithms.cpp @@ -0,0 +1,32 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#include + +using namespace std; +using namespace solidity::phaser; + +void GeneticAlgorithm::run(optional _numRounds) +{ + for (size_t round = 0; !_numRounds.has_value() || round < _numRounds.value(); ++round) + { + runNextRound(); + + m_outputStream << "---------- ROUND " << round << " ----------" << endl; + m_outputStream << m_population; + } +} diff --git a/tools/yulPhaser/GeneticAlgorithms.h b/tools/yulPhaser/GeneticAlgorithms.h new file mode 100644 index 000000000..c2700416b --- /dev/null +++ b/tools/yulPhaser/GeneticAlgorithms.h @@ -0,0 +1,66 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ +/** + * Contains an abstract base class representing a genetic algorithm and its concrete implementations. + */ + +#pragma once + +#include + +#include +#include + +namespace solidity::phaser +{ + +/** + * Abstract base class for genetic algorithms. + * + * The main feature is the @a run() method that executes the algorithm, updating the internal + * population during each round and printing the results to the stream provided to the constructor. + * + * Derived classes can provide specific methods for updating the population by implementing + * the @a runNextRound() method. + */ +class GeneticAlgorithm +{ +public: + GeneticAlgorithm(Population _initialPopulation, std::ostream& _outputStream): + m_population(std::move(_initialPopulation)), + m_outputStream(_outputStream) {} + + GeneticAlgorithm(GeneticAlgorithm const&) = delete; + GeneticAlgorithm& operator=(GeneticAlgorithm const&) = delete; + virtual ~GeneticAlgorithm() = default; + + Population const& population() const { return m_population; } + + void run(std::optional _numRounds = std::nullopt); + + /// The method that actually implements the algorithm. Should use @a m_population as input and + /// replace it with the updated state after the round. + virtual void runNextRound() = 0; + +protected: + Population m_population; + +private: + std::ostream& m_outputStream; +}; + +} From 3c41bfbc4e22b7a34dee6886803ebd60ca4cdc10 Mon Sep 17 00:00:00 2001 From: cameel Date: Wed, 5 Feb 2020 14:56:55 +0100 Subject: [PATCH 50/92] [yul-phaser] Base class for selections --- tools/CMakeLists.txt | 2 ++ tools/yulPhaser/Selections.cpp | 18 ++++++++++++ tools/yulPhaser/Selections.h | 52 ++++++++++++++++++++++++++++++++++ 3 files changed, 72 insertions(+) create mode 100644 tools/yulPhaser/Selections.cpp create mode 100644 tools/yulPhaser/Selections.h diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 601b249e6..21c0b8c91 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -23,6 +23,8 @@ add_executable(yul-phaser yulPhaser/FitnessMetrics.cpp yulPhaser/Chromosome.h yulPhaser/Chromosome.cpp + yulPhaser/Selections.h + yulPhaser/Selections.cpp yulPhaser/Program.h yulPhaser/Program.cpp yulPhaser/SimulationRNG.h diff --git a/tools/yulPhaser/Selections.cpp b/tools/yulPhaser/Selections.cpp new file mode 100644 index 000000000..19451f45c --- /dev/null +++ b/tools/yulPhaser/Selections.cpp @@ -0,0 +1,18 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#include diff --git a/tools/yulPhaser/Selections.h b/tools/yulPhaser/Selections.h new file mode 100644 index 000000000..1155c6708 --- /dev/null +++ b/tools/yulPhaser/Selections.h @@ -0,0 +1,52 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ +/** + * Contains an abstract base class representing a selection of elements from a collection + * and its concrete implementations. + */ + +#pragma once + +#include + +namespace solidity::phaser +{ + +/** + * Abstract base class for selections of elements from a collection. + * + * An instance of this class represents a specific method of selecting a set of elements from + * containers of arbitrary sizes. The set of selected elements is always a subset of the container + * but may indicate the same element more than once. The selection may or may not be fixed - it's + * up to a specific implementation whether subsequent calls for the same container produce the same + * indices or not. + * + * Derived classes are meant to override the @a materialise() method. + * This method is expected to produce indices of selected elements given the size of the collection. + */ +class Selection +{ +public: + Selection() = default; + Selection(Selection const&) = delete; + Selection& operator=(Selection const&) = delete; + virtual ~Selection() = default; + + virtual std::vector materialise(size_t _poolSize) const = 0; +}; + +} From 83b8ab8012d4a0d33e5e65812b46d63d77611a2e Mon Sep 17 00:00:00 2001 From: cameel Date: Wed, 5 Feb 2020 14:57:29 +0100 Subject: [PATCH 51/92] [yul-phaser] Add RangeSelection, MosaicSelection and RandomSelection classes --- test/CMakeLists.txt | 2 + test/yulPhaser/Selections.cpp | 206 +++++++++++++++++++++++++++++++++ tools/yulPhaser/Selections.cpp | 42 +++++++ tools/yulPhaser/Selections.h | 69 +++++++++++ 4 files changed, 319 insertions(+) create mode 100644 test/yulPhaser/Selections.cpp diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 70d547a0b..3ea080686 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -147,6 +147,7 @@ set(yul_phaser_sources yulPhaser/GeneticAlgorithms.cpp yulPhaser/Population.cpp yulPhaser/Program.cpp + yulPhaser/Selections.cpp yulPhaser/SimulationRNG.cpp # FIXME: yul-phaser is not a library so I can't just add it to target_link_libraries(). @@ -157,6 +158,7 @@ set(yul_phaser_sources ../tools/yulPhaser/GeneticAlgorithms.cpp ../tools/yulPhaser/Population.cpp ../tools/yulPhaser/Program.cpp + ../tools/yulPhaser/Selections.cpp ../tools/yulPhaser/SimulationRNG.cpp ) detect_stray_source_files("${yul_phaser_sources}" "yulPhaser/") diff --git a/test/yulPhaser/Selections.cpp b/test/yulPhaser/Selections.cpp new file mode 100644 index 000000000..ce870ce80 --- /dev/null +++ b/test/yulPhaser/Selections.cpp @@ -0,0 +1,206 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#include + +#include +#include + +#include + +#include + +#include +#include + +using namespace std; + +namespace solidity::phaser::test +{ + +BOOST_AUTO_TEST_SUITE(Phaser) +BOOST_AUTO_TEST_SUITE(SelectionsTest) +BOOST_AUTO_TEST_SUITE(RangeSelectionTest) + +BOOST_AUTO_TEST_CASE(materialise) +{ + BOOST_TEST(RangeSelection(0.0, 1.0).materialise(10) == vector({0, 1, 2, 3, 4, 5, 6, 7, 8, 9})); + BOOST_TEST(RangeSelection(0.0, 0.1).materialise(10) == vector({0})); + BOOST_TEST(RangeSelection(0.0, 0.2).materialise(10) == vector({0, 1})); + BOOST_TEST(RangeSelection(0.0, 0.7).materialise(10) == vector({0, 1, 2, 3, 4, 5, 6})); + + BOOST_TEST(RangeSelection(0.9, 1.0).materialise(10) == vector({ 9})); + BOOST_TEST(RangeSelection(0.8, 1.0).materialise(10) == vector({ 8, 9})); + BOOST_TEST(RangeSelection(0.5, 1.0).materialise(10) == vector({ 5, 6, 7, 8, 9})); + + BOOST_TEST(RangeSelection(0.3, 0.6).materialise(10) == vector({ 3, 4, 5 })); + BOOST_TEST(RangeSelection(0.2, 0.7).materialise(10) == vector({ 2, 3, 4, 5, 6 })); + BOOST_TEST(RangeSelection(0.4, 0.7).materialise(10) == vector({ 4, 5, 6 })); + + BOOST_TEST(RangeSelection(0.4, 0.7).materialise(5) == vector({2, 3})); +} + +BOOST_AUTO_TEST_CASE(materialise_should_round_indices) +{ + BOOST_TEST(RangeSelection(0.01, 0.99).materialise(10) == vector({0, 1, 2, 3, 4, 5, 6, 7, 8, 9})); + BOOST_TEST(RangeSelection(0.04, 0.96).materialise(10) == vector({0, 1, 2, 3, 4, 5, 6, 7, 8, 9})); + BOOST_TEST(RangeSelection(0.05, 0.95).materialise(10) == vector({ 1, 2, 3, 4, 5, 6, 7, 8, 9})); + BOOST_TEST(RangeSelection(0.06, 0.94).materialise(10) == vector({ 1, 2, 3, 4, 5, 6, 7, 8 })); +} + +BOOST_AUTO_TEST_CASE(materialise_should_handle_empty_collections) +{ + BOOST_TEST(RangeSelection(0.0, 0.0).materialise(0).empty()); + BOOST_TEST(RangeSelection(0.0, 1.0).materialise(0).empty()); + BOOST_TEST(RangeSelection(0.5, 1.0).materialise(0).empty()); + BOOST_TEST(RangeSelection(0.0, 0.5).materialise(0).empty()); + BOOST_TEST(RangeSelection(0.2, 0.7).materialise(0).empty()); +} + +BOOST_AUTO_TEST_CASE(materialise_should_handle_empty_selection_ranges) +{ + BOOST_TEST(RangeSelection(0.0, 0.0).materialise(1).empty()); + BOOST_TEST(RangeSelection(1.0, 1.0).materialise(1).empty()); + + BOOST_TEST(RangeSelection(0.0, 0.0).materialise(100).empty()); + BOOST_TEST(RangeSelection(1.0, 1.0).materialise(100).empty()); + BOOST_TEST(RangeSelection(0.5, 0.5).materialise(100).empty()); + + BOOST_TEST(RangeSelection(0.45, 0.54).materialise(10).empty()); + BOOST_TEST(!RangeSelection(0.45, 0.54).materialise(100).empty()); + BOOST_TEST(RangeSelection(0.045, 0.054).materialise(100).empty()); +} + +BOOST_AUTO_TEST_SUITE_END() +BOOST_AUTO_TEST_SUITE(MosaicSelectionTest) + +BOOST_AUTO_TEST_CASE(materialise) +{ + BOOST_TEST(MosaicSelection({1}, 0.5).materialise(4) == vector({1, 1})); + BOOST_TEST(MosaicSelection({1}, 1.0).materialise(4) == vector({1, 1, 1, 1})); + BOOST_TEST(MosaicSelection({1}, 2.0).materialise(4) == vector({1, 1, 1, 1, 1, 1, 1, 1})); + BOOST_TEST(MosaicSelection({1}, 1.0).materialise(2) == vector({1, 1})); + + BOOST_TEST(MosaicSelection({0, 1}, 0.5).materialise(4) == vector({0, 1})); + BOOST_TEST(MosaicSelection({0, 1}, 1.0).materialise(4) == vector({0, 1, 0, 1})); + BOOST_TEST(MosaicSelection({0, 1}, 2.0).materialise(4) == vector({0, 1, 0, 1, 0, 1, 0, 1})); + BOOST_TEST(MosaicSelection({0, 1}, 1.0).materialise(2) == vector({0, 1})); + + BOOST_TEST(MosaicSelection({3, 2, 1, 0}, 0.5).materialise(4) == vector({3, 2})); + BOOST_TEST(MosaicSelection({3, 2, 1, 0}, 1.0).materialise(4) == vector({3, 2, 1, 0})); + BOOST_TEST(MosaicSelection({3, 2, 1, 0}, 2.0).materialise(4) == vector({3, 2, 1, 0, 3, 2, 1, 0})); + BOOST_TEST(MosaicSelection({1, 0, 1, 0}, 1.0).materialise(2) == vector({1, 0})); +} + +BOOST_AUTO_TEST_CASE(materialise_should_round_indices) +{ + BOOST_TEST(MosaicSelection({4, 3, 2, 1, 0}, 0.49).materialise(5) == vector({4, 3})); + BOOST_TEST(MosaicSelection({4, 3, 2, 1, 0}, 0.50).materialise(5) == vector({4, 3, 2})); + BOOST_TEST(MosaicSelection({4, 3, 2, 1, 0}, 0.51).materialise(5) == vector({4, 3, 2})); +} + +BOOST_AUTO_TEST_CASE(materialise_should_handle_empty_collections) +{ + BOOST_TEST(MosaicSelection({1}, 1.0).materialise(0).empty()); + BOOST_TEST(MosaicSelection({1, 3}, 2.0).materialise(0).empty()); + BOOST_TEST(MosaicSelection({5, 4, 3, 2}, 0.5).materialise(0).empty()); +} + +BOOST_AUTO_TEST_CASE(materialise_should_handle_empty_selections) +{ + BOOST_TEST(MosaicSelection({1}, 0.0).materialise(8).empty()); + BOOST_TEST(MosaicSelection({1, 3}, 0.0).materialise(8).empty()); + BOOST_TEST(MosaicSelection({5, 4, 3, 2}, 0.0).materialise(8).empty()); +} + +BOOST_AUTO_TEST_CASE(materialise_should_clamp_indices_at_collection_size) +{ + BOOST_TEST(MosaicSelection({4, 3, 2, 1, 0}, 1.0).materialise(4) == vector({3, 3, 2, 1})); + BOOST_TEST(MosaicSelection({4, 3, 2, 1, 0}, 2.0).materialise(3) == vector({2, 2, 2, 1, 0, 2})); + BOOST_TEST(MosaicSelection({4, 3, 2, 1, 0}, 1.0).materialise(1) == vector({0})); + BOOST_TEST(MosaicSelection({4, 3, 2, 1, 0}, 7.0).materialise(1) == vector({0, 0, 0, 0, 0, 0, 0})); +} + +BOOST_AUTO_TEST_SUITE_END() +BOOST_AUTO_TEST_SUITE(RandomSelectionTest) + +BOOST_AUTO_TEST_CASE(materialise_should_return_random_values_with_equal_probabilities) +{ + constexpr int collectionSize = 10; + constexpr int selectionSize = 100; + constexpr double relativeTolerance = 0.1; + constexpr double expectedValue = (collectionSize - 1) / 2.0; + constexpr double variance = (collectionSize * collectionSize - 1) / 12.0; + + SimulationRNG::reset(1); + vector samples = RandomSelection(selectionSize).materialise(collectionSize); + + BOOST_TEST(abs(mean(samples) - expectedValue) < expectedValue * relativeTolerance); + BOOST_TEST(abs(meanSquaredError(samples, expectedValue) - variance) < variance * relativeTolerance); +} + +BOOST_AUTO_TEST_CASE(materialise_should_return_only_values_that_can_be_used_as_collection_indices) +{ + const size_t collectionSize = 200; + + vector indices = RandomSelection(0.5).materialise(collectionSize); + + BOOST_TEST(indices.size() == 100); + BOOST_TEST(all_of(indices.begin(), indices.end(), [&](auto const& index){ return index <= collectionSize; })); +} + +BOOST_AUTO_TEST_CASE(materialise_should_return_number_of_indices_thats_a_fraction_of_collection_size) +{ + BOOST_TEST(RandomSelection(0.0).materialise(10).size() == 0); + BOOST_TEST(RandomSelection(0.3).materialise(10).size() == 3); + BOOST_TEST(RandomSelection(0.5).materialise(10).size() == 5); + BOOST_TEST(RandomSelection(0.7).materialise(10).size() == 7); + BOOST_TEST(RandomSelection(1.0).materialise(10).size() == 10); +} + +BOOST_AUTO_TEST_CASE(materialise_should_support_number_of_indices_bigger_than_collection_size) +{ + BOOST_TEST(RandomSelection(2.0).materialise(5).size() == 10); + BOOST_TEST(RandomSelection(1.5).materialise(10).size() == 15); + BOOST_TEST(RandomSelection(10.0).materialise(10).size() == 100); +} + +BOOST_AUTO_TEST_CASE(materialise_should_round_the_number_of_indices_to_the_nearest_integer) +{ + BOOST_TEST(RandomSelection(0.49).materialise(3).size() == 1); + BOOST_TEST(RandomSelection(0.50).materialise(3).size() == 2); + BOOST_TEST(RandomSelection(0.51).materialise(3).size() == 2); + + BOOST_TEST(RandomSelection(1.51).materialise(3).size() == 5); + + BOOST_TEST(RandomSelection(0.01).materialise(2).size() == 0); + BOOST_TEST(RandomSelection(0.01).materialise(3).size() == 0); +} + +BOOST_AUTO_TEST_CASE(materialise_should_return_no_indices_if_collection_is_empty) +{ + BOOST_TEST(RandomSelection(0.0).materialise(0).empty()); + BOOST_TEST(RandomSelection(0.5).materialise(0).empty()); + BOOST_TEST(RandomSelection(1.0).materialise(0).empty()); + BOOST_TEST(RandomSelection(2.0).materialise(0).empty()); +} + +BOOST_AUTO_TEST_SUITE_END() +BOOST_AUTO_TEST_SUITE_END() +BOOST_AUTO_TEST_SUITE_END() + +} diff --git a/tools/yulPhaser/Selections.cpp b/tools/yulPhaser/Selections.cpp index 19451f45c..abc080fde 100644 --- a/tools/yulPhaser/Selections.cpp +++ b/tools/yulPhaser/Selections.cpp @@ -16,3 +16,45 @@ */ #include + +#include + +#include + +using namespace std; +using namespace solidity::phaser; + +vector RangeSelection::materialise(size_t _poolSize) const +{ + size_t beginIndex = static_cast(round(_poolSize * m_startPercent)); + size_t endIndex = static_cast(round(_poolSize * m_endPercent)); + vector selection; + + for (size_t i = beginIndex; i < endIndex; ++i) + selection.push_back(i); + + return selection; +} + +vector MosaicSelection::materialise(size_t _poolSize) const +{ + size_t count = static_cast(round(_poolSize * m_selectionSize)); + + vector selection; + for (size_t i = 0; i < count; ++i) + selection.push_back(min(m_pattern[i % m_pattern.size()], _poolSize - 1)); + + return selection; +} + +vector RandomSelection::materialise(size_t _poolSize) const +{ + size_t count = static_cast(round(_poolSize * m_selectionSize)); + + vector selection; + for (size_t i = 0; i < count; ++i) + selection.push_back(SimulationRNG::uniformInt(0, _poolSize - 1)); + + return selection; +} + diff --git a/tools/yulPhaser/Selections.h b/tools/yulPhaser/Selections.h index 1155c6708..46d975bbd 100644 --- a/tools/yulPhaser/Selections.h +++ b/tools/yulPhaser/Selections.h @@ -21,6 +21,7 @@ #pragma once +#include #include namespace solidity::phaser @@ -49,4 +50,72 @@ public: virtual std::vector materialise(size_t _poolSize) const = 0; }; +/** + * A selection that selects a contiguous slice of the container. Start and end of this part are + * specified as percentages of its size. + */ +class RangeSelection: public Selection +{ +public: + explicit RangeSelection(double _startPercent = 0.0, double _endPercent = 1.0): + m_startPercent(_startPercent), + m_endPercent(_endPercent) + { + assert(0 <= m_startPercent && m_startPercent <= m_endPercent && m_endPercent <= 1.0); + } + + std::vector materialise(size_t _poolSize) const override; + +private: + double m_startPercent; + double m_endPercent; +}; + +/** + * A selection that selects elements at specific, fixed positions indicated by a repeating "pattern". + * If the positions in the pattern exceed the size of the container, they are capped at the maximum + * available position. Always selects as many elements as the size of the container multiplied by + * @a _selectionSize (unless the container is empty). + * + * E.g. if the pattern is {0, 9} and collection size is 5, the selection will materialise into + * {0, 4, 0, 4, 0}. If the size is 3, it will be {0, 2, 0}. + */ +class MosaicSelection: public Selection +{ +public: + explicit MosaicSelection(std::vector _pattern, double _selectionSize = 1.0): + m_pattern(move(_pattern)), + m_selectionSize(_selectionSize) + { + assert(m_pattern.size() > 0 || _selectionSize == 0.0); + } + + std::vector materialise(size_t _poolSize) const override; + +private: + std::vector m_pattern; + double m_selectionSize; +}; + +/** + * A selection that randomly selects elements from a container. The resulting set of indices may + * contain duplicates and is different on each call to @a materialise(). Always selects as many + * elements as the size of the container multiplied by @a _selectionSize (unless the container is + * empty). + */ +class RandomSelection: public Selection +{ +public: + explicit RandomSelection(double _selectionSize): + m_selectionSize(_selectionSize) + { + assert(_selectionSize >= 0); + } + + std::vector materialise(size_t _poolSize) const override; + +private: + double m_selectionSize; +}; + } From 4665b7a7e4d5fe5d4135a5ea67b06a7f1c7e5814 Mon Sep 17 00:00:00 2001 From: cameel Date: Wed, 5 Feb 2020 16:55:50 +0100 Subject: [PATCH 52/92] [yul-phaser] Population: Add select() method --- test/yulPhaser/Population.cpp | 36 ++++++++++++++++++++++++++++++++++ tools/yulPhaser/Population.cpp | 10 ++++++++++ tools/yulPhaser/Population.h | 3 +++ 3 files changed, 49 insertions(+) diff --git a/test/yulPhaser/Population.cpp b/test/yulPhaser/Population.cpp index 55c346575..dfee5f74a 100644 --- a/test/yulPhaser/Population.cpp +++ b/test/yulPhaser/Population.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -222,6 +223,41 @@ BOOST_FIXTURE_TEST_CASE(plus_operator_should_add_two_populations, PopulationFixt ); } +BOOST_FIXTURE_TEST_CASE(select_should_return_population_containing_individuals_indicated_by_selection, PopulationFixture) +{ + Population population(m_fitnessMetric, {Chromosome("a"), Chromosome("c"), Chromosome("g"), Chromosome("h")}); + RangeSelection selection(0.25, 0.75); + assert(selection.materialise(population.individuals().size()) == (vector{1, 2})); + + BOOST_TEST( + population.select(selection) == + Population(m_fitnessMetric, {population.individuals()[1].chromosome, population.individuals()[2].chromosome}) + ); +} + +BOOST_FIXTURE_TEST_CASE(select_should_include_duplicates_if_selection_contains_duplicates, PopulationFixture) +{ + Population population(m_fitnessMetric, {Chromosome("a"), Chromosome("c")}); + MosaicSelection selection({0, 1}, 2.0); + assert(selection.materialise(population.individuals().size()) == (vector{0, 1, 0, 1})); + + BOOST_TEST(population.select(selection) == Population(m_fitnessMetric, { + population.individuals()[0].chromosome, + population.individuals()[1].chromosome, + population.individuals()[0].chromosome, + population.individuals()[1].chromosome, + })); +} + +BOOST_FIXTURE_TEST_CASE(select_should_return_empty_population_if_selection_is_empty, PopulationFixture) +{ + Population population(m_fitnessMetric, {Chromosome("a"), Chromosome("c")}); + RangeSelection selection(0.0, 0.0); + assert(selection.materialise(population.individuals().size()).empty()); + + BOOST_TEST(population.select(selection).individuals().empty()); +} + BOOST_AUTO_TEST_SUITE_END() BOOST_AUTO_TEST_SUITE_END() diff --git a/tools/yulPhaser/Population.cpp b/tools/yulPhaser/Population.cpp index 2fed8b90d..12acbe417 100644 --- a/tools/yulPhaser/Population.cpp +++ b/tools/yulPhaser/Population.cpp @@ -17,6 +17,7 @@ #include +#include #include #include @@ -95,6 +96,15 @@ void Population::run(optional _numRounds, ostream& _outputStream) } } +Population Population::select(Selection const& _selection) const +{ + vector selectedIndividuals; + for (size_t i: _selection.materialise(m_individuals.size())) + selectedIndividuals.emplace_back(m_individuals[i]); + + return Population(m_fitnessMetric, selectedIndividuals); +} + Population operator+(Population _a, Population _b) { // This operator is meant to be used only with populations sharing the same metric (and, to make diff --git a/tools/yulPhaser/Population.h b/tools/yulPhaser/Population.h index bd15674d6..f66efc9a0 100644 --- a/tools/yulPhaser/Population.h +++ b/tools/yulPhaser/Population.h @@ -39,6 +39,8 @@ solidity::phaser::Population operator+(solidity::phaser::Population _a, solidity namespace solidity::phaser { +class Selection; + /** * Information describing the state of an individual member of the population during the course * of the genetic algorithm. @@ -102,6 +104,7 @@ public: ); void run(std::optional _numRounds, std::ostream& _outputStream); + Population select(Selection const& _selection) const; friend Population (::operator+)(Population _a, Population _b); std::shared_ptr fitnessMetric() const { return m_fitnessMetric; } From 67fbafab8fddb42e0d84271bcac5ecce95835c9d Mon Sep 17 00:00:00 2001 From: cameel Date: Wed, 5 Feb 2020 14:58:48 +0100 Subject: [PATCH 53/92] [yul-phaser] Add RandomAlgorithm --- test/yulPhaser/GeneticAlgorithms.cpp | 43 +++++++++++++++++++++++ tools/yulPhaser/GeneticAlgorithms.cpp | 18 ++++++++++ tools/yulPhaser/GeneticAlgorithms.h | 49 +++++++++++++++++++++++++++ 3 files changed, 110 insertions(+) diff --git a/test/yulPhaser/GeneticAlgorithms.cpp b/test/yulPhaser/GeneticAlgorithms.cpp index 3528f2be0..aaa0a0b05 100644 --- a/test/yulPhaser/GeneticAlgorithms.cpp +++ b/test/yulPhaser/GeneticAlgorithms.cpp @@ -89,6 +89,49 @@ BOOST_FIXTURE_TEST_CASE(run_should_print_the_top_chromosome, GeneticAlgorithmFix BOOST_TEST(countSubstringOccurrences(m_output.str(), toString(algorithm.population().individuals()[0].chromosome)) == 4); } +BOOST_AUTO_TEST_SUITE_END() +BOOST_AUTO_TEST_SUITE(RandomAlgorithmTest) + +BOOST_FIXTURE_TEST_CASE(runNextRound_should_preserve_elite_and_randomise_rest_of_population, GeneticAlgorithmFixture) +{ + auto population = Population::makeRandom(m_fitnessMetric, 4, 3, 3) + Population::makeRandom(m_fitnessMetric, 4, 5, 5); + RandomAlgorithm algorithm(population, m_output, {0.5, 1, 1}); + assert((chromosomeLengths(algorithm.population()) == vector{3, 3, 3, 3, 5, 5, 5, 5})); + + algorithm.runNextRound(); + BOOST_TEST((chromosomeLengths(algorithm.population()) == vector{1, 1, 1, 1, 3, 3, 3, 3})); +} + +BOOST_FIXTURE_TEST_CASE(runNextRound_should_not_replace_elite_with_worse_individuals, GeneticAlgorithmFixture) +{ + auto population = Population::makeRandom(m_fitnessMetric, 4, 3, 3) + Population::makeRandom(m_fitnessMetric, 4, 5, 5); + RandomAlgorithm algorithm(population, m_output, {0.5, 7, 7}); + assert((chromosomeLengths(algorithm.population()) == vector{3, 3, 3, 3, 5, 5, 5, 5})); + + algorithm.runNextRound(); + BOOST_TEST((chromosomeLengths(algorithm.population()) == vector{3, 3, 3, 3, 7, 7, 7, 7})); +} + +BOOST_FIXTURE_TEST_CASE(runNextRound_should_replace_all_chromosomes_if_zero_size_elite, GeneticAlgorithmFixture) +{ + auto population = Population::makeRandom(m_fitnessMetric, 4, 3, 3) + Population::makeRandom(m_fitnessMetric, 4, 5, 5); + RandomAlgorithm algorithm(population, m_output, {0.0, 1, 1}); + assert((chromosomeLengths(algorithm.population()) == vector{3, 3, 3, 3, 5, 5, 5, 5})); + + algorithm.runNextRound(); + BOOST_TEST((chromosomeLengths(algorithm.population()) == vector{1, 1, 1, 1, 1, 1, 1, 1})); +} + +BOOST_FIXTURE_TEST_CASE(runNextRound_should_not_replace_any_chromosomes_if_whole_population_is_the_elite, GeneticAlgorithmFixture) +{ + auto population = Population::makeRandom(m_fitnessMetric, 4, 3, 3) + Population::makeRandom(m_fitnessMetric, 4, 5, 5); + RandomAlgorithm algorithm(population, m_output, {1.0, 1, 1}); + assert((chromosomeLengths(algorithm.population()) == vector{3, 3, 3, 3, 5, 5, 5, 5})); + + algorithm.runNextRound(); + BOOST_TEST((chromosomeLengths(algorithm.population()) == vector{3, 3, 3, 3, 5, 5, 5, 5})); +} + BOOST_AUTO_TEST_SUITE_END() BOOST_AUTO_TEST_SUITE_END() BOOST_AUTO_TEST_SUITE_END() diff --git a/tools/yulPhaser/GeneticAlgorithms.cpp b/tools/yulPhaser/GeneticAlgorithms.cpp index 6e69565f4..756a410f7 100644 --- a/tools/yulPhaser/GeneticAlgorithms.cpp +++ b/tools/yulPhaser/GeneticAlgorithms.cpp @@ -16,6 +16,7 @@ */ #include +#include using namespace std; using namespace solidity::phaser; @@ -30,3 +31,20 @@ void GeneticAlgorithm::run(optional _numRounds) m_outputStream << m_population; } } + +void RandomAlgorithm::runNextRound() +{ + RangeSelection elite(0.0, m_options.elitePoolSize); + + Population elitePopulation = m_population.select(elite); + size_t replacementCount = m_population.individuals().size() - elitePopulation.individuals().size(); + + m_population = + move(elitePopulation) + + Population::makeRandom( + m_population.fitnessMetric(), + replacementCount, + m_options.minChromosomeLength, + m_options.maxChromosomeLength + ); +} diff --git a/tools/yulPhaser/GeneticAlgorithms.h b/tools/yulPhaser/GeneticAlgorithms.h index c2700416b..a7600d894 100644 --- a/tools/yulPhaser/GeneticAlgorithms.h +++ b/tools/yulPhaser/GeneticAlgorithms.h @@ -63,4 +63,53 @@ private: std::ostream& m_outputStream; }; +/** + * Completely random genetic algorithm, + * + * The algorithm simply replaces the worst chromosomes with entirely new ones, generated + * randomly and not based on any member of the current population. Only a constant proportion of the + * chromosomes (the elite) is preserved in each round. + * + * Preserves the size of the population. You can use @a elitePoolSize to make the algorithm + * generational (replacing most members in each round) or steady state (replacing only one member). + * Both versions are equivalent in terms of the outcome but the generational one converges in a + * smaller number of rounds while the steady state one does less work per round. This may matter + * in case of metrics that take a long time to compute though in case of this particular + * algorithm the same result could also be achieved by simply making the population smaller. + */ +class RandomAlgorithm: public GeneticAlgorithm +{ +public: + struct Options + { + double elitePoolSize; ///< Percentage of the population treated as the elite + size_t minChromosomeLength; ///< Minimum length of newly generated chromosomes + size_t maxChromosomeLength; ///< Maximum length of newly generated chromosomes + + bool isValid() const + { + return ( + 0 <= elitePoolSize && elitePoolSize <= 1.0 && + minChromosomeLength <= maxChromosomeLength + ); + } + }; + + explicit RandomAlgorithm( + Population _initialPopulation, + std::ostream& _outputStream, + Options const& _options + ): + GeneticAlgorithm(_initialPopulation, _outputStream), + m_options(_options) + { + assert(_options.isValid()); + } + + void runNextRound() override; + +private: + Options m_options; +}; + } From e1b8b64f05a5b6efd860e59aabd6ac5ded6aaa9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Thu, 6 Feb 2020 07:59:08 +0100 Subject: [PATCH 54/92] [yul-phaser] Population: Remove no longer used methods for running algorithm steps - They have been superseded by objects from GeneticAlgorithms.h --- test/yulPhaser/Population.cpp | 32 -------------------------- tools/yulPhaser/Population.cpp | 41 ---------------------------------- tools/yulPhaser/Population.h | 19 +++++----------- 3 files changed, 5 insertions(+), 87 deletions(-) diff --git a/test/yulPhaser/Population.cpp b/test/yulPhaser/Population.cpp index dfee5f74a..98532c9dc 100644 --- a/test/yulPhaser/Population.cpp +++ b/test/yulPhaser/Population.cpp @@ -182,38 +182,6 @@ BOOST_FIXTURE_TEST_CASE(makeRandom_should_compute_fitness, PopulationFixture) BOOST_TEST(population.individuals()[2].fitness == m_fitnessMetric->evaluate(population.individuals()[2].chromosome)); } -BOOST_FIXTURE_TEST_CASE(run_should_not_make_fitness_of_top_chromosomes_worse, PopulationFixture) -{ - stringstream output; - vector chromosomes = { - Chromosome(vector{StructuralSimplifier::name}), - Chromosome(vector{BlockFlattener::name}), - Chromosome(vector{SSAReverser::name}), - Chromosome(vector{UnusedPruner::name}), - Chromosome(vector{StructuralSimplifier::name, BlockFlattener::name}), - }; - Population population(m_fitnessMetric, chromosomes); - - size_t initialTopFitness[2] = { - m_fitnessMetric->evaluate(chromosomes[0]), - m_fitnessMetric->evaluate(chromosomes[1]), - }; - - for (int i = 0; i < 6; ++i) - { - population.run(1, output); - BOOST_TEST(population.individuals().size() == 5); - - size_t currentTopFitness[2] = { - population.individuals()[0].fitness, - population.individuals()[1].fitness, - }; - BOOST_TEST(currentTopFitness[0] <= initialTopFitness[0]); - BOOST_TEST(currentTopFitness[1] <= initialTopFitness[1]); - BOOST_TEST(currentTopFitness[0] <= currentTopFitness[1]); - } -} - BOOST_FIXTURE_TEST_CASE(plus_operator_should_add_two_populations, PopulationFixture) { BOOST_CHECK_EQUAL( diff --git a/tools/yulPhaser/Population.cpp b/tools/yulPhaser/Population.cpp index 12acbe417..b39f5dad5 100644 --- a/tools/yulPhaser/Population.cpp +++ b/tools/yulPhaser/Population.cpp @@ -84,18 +84,6 @@ Population Population::makeRandom( ); } -void Population::run(optional _numRounds, ostream& _outputStream) -{ - for (size_t round = 0; !_numRounds.has_value() || round < _numRounds.value(); ++round) - { - doMutation(); - doSelection(); - - _outputStream << "---------- ROUND " << round << " ----------" << endl; - _outputStream << *this; - } -} - Population Population::select(Selection const& _selection) const { vector selectedIndividuals; @@ -131,35 +119,6 @@ ostream& phaser::operator<<(ostream& _stream, Population const& _population) return _stream; } -void Population::doMutation() -{ - // TODO: Implement mutation and crossover -} - -void Population::doSelection() -{ - randomizeWorstChromosomes(*m_fitnessMetric, m_individuals, m_individuals.size() / 2); - m_individuals = sortedIndividuals(move(m_individuals)); -} - -void Population::randomizeWorstChromosomes( - FitnessMetric const& _fitnessMetric, - vector& _individuals, - size_t _count -) -{ - assert(_individuals.size() >= _count); - // ASSUMPTION: _individuals is sorted in ascending order - - auto individual = _individuals.begin() + (_individuals.size() - _count); - for (; individual != _individuals.end(); ++individual) - { - auto chromosome = Chromosome::makeRandom(binomialChromosomeLength(MaxChromosomeLength)); - size_t fitness = _fitnessMetric.evaluate(chromosome); - *individual = {move(chromosome), fitness}; - } -} - vector Population::chromosomesToIndividuals( FitnessMetric const& _fitnessMetric, vector _chromosomes diff --git a/tools/yulPhaser/Population.h b/tools/yulPhaser/Population.h index f66efc9a0..f4f42346d 100644 --- a/tools/yulPhaser/Population.h +++ b/tools/yulPhaser/Population.h @@ -69,19 +69,19 @@ struct Individual bool isFitter(Individual const& a, Individual const& b); /** - * Represents a changing set of individuals undergoing a genetic algorithm. - * Each round of the algorithm involves mutating existing individuals, evaluating their fitness - * and selecting the best ones for the next round. + * Represents a snapshot of a population undergoing a genetic algorithm. Consists of a set of + * chromosomes with associated fitness values. * * An individual is a sequence of optimiser steps represented by a @a Chromosome instance. * Individuals are always ordered by their fitness (based on @_fitnessMetric and @a isFitter()). * The fitness is computed using the metric as soon as an individual is inserted into the population. + * + * The population is immutable. Selections, mutations and crossover work by producing a new + * instance and copying the individuals. */ class Population { public: - static constexpr size_t MaxChromosomeLength = 30; - explicit Population( std::shared_ptr _fitnessMetric, std::vector _chromosomes = {} @@ -103,7 +103,6 @@ public: size_t _maxChromosomeLength ); - void run(std::optional _numRounds, std::ostream& _outputStream); Population select(Selection const& _selection) const; friend Population (::operator+)(Population _a, Population _b); @@ -123,14 +122,6 @@ private: m_fitnessMetric(std::move(_fitnessMetric)), m_individuals{sortedIndividuals(std::move(_individuals))} {} - void doMutation(); - void doSelection(); - - static void randomizeWorstChromosomes( - FitnessMetric const& _fitnessMetric, - std::vector& _individuals, - size_t _count - ); static std::vector chromosomesToIndividuals( FitnessMetric const& _fitnessMetric, std::vector _chromosomes From 4aac7d167387e65ca53b1ce9bac9809b30d6eaf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Thu, 13 Feb 2020 20:35:50 +0100 Subject: [PATCH 55/92] [yul-phaser] main: Switch from using Population::run() to RandomAlgorithm --- tools/yulPhaser/main.cpp | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/tools/yulPhaser/main.cpp b/tools/yulPhaser/main.cpp index eea43e5fb..d3ea9d713 100644 --- a/tools/yulPhaser/main.cpp +++ b/tools/yulPhaser/main.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -29,7 +30,6 @@ #include #include -#include #include using namespace std; @@ -71,14 +71,26 @@ CharStream loadSource(string const& _sourcePath) void runAlgorithm(string const& _sourcePath) { + constexpr size_t minChromosomeLength = 12; + constexpr size_t maxChromosomeLength = 30; + CharStream sourceCode = loadSource(_sourcePath); shared_ptr fitnessMetric = make_shared(Program::load(sourceCode), 5); auto population = Population::makeRandom( fitnessMetric, 10, - bind(Population::binomialChromosomeLength, Population::MaxChromosomeLength) + minChromosomeLength, + maxChromosomeLength ); - population.run(nullopt, cout); + RandomAlgorithm( + population, + cout, + { + /* elitePoolSize = */ 0.5, + /* minChromosomeLength = */ minChromosomeLength, + /* maxChromosomeLength = */ maxChromosomeLength, + } + ).run(); } CommandLineParsingResult parseCommandLine(int argc, char** argv) From 3aaca31c31e80151f4b3208194a76a329f38e79b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Wed, 5 Feb 2020 19:25:15 +0100 Subject: [PATCH 56/92] [yul-phaser] main: Change the number of chromosomes preserved in every round of the algorithm to 1 --- tools/yulPhaser/main.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/yulPhaser/main.cpp b/tools/yulPhaser/main.cpp index d3ea9d713..26a2ea005 100644 --- a/tools/yulPhaser/main.cpp +++ b/tools/yulPhaser/main.cpp @@ -71,6 +71,7 @@ CharStream loadSource(string const& _sourcePath) void runAlgorithm(string const& _sourcePath) { + constexpr size_t populationSize = 10; constexpr size_t minChromosomeLength = 12; constexpr size_t maxChromosomeLength = 30; @@ -78,7 +79,7 @@ void runAlgorithm(string const& _sourcePath) shared_ptr fitnessMetric = make_shared(Program::load(sourceCode), 5); auto population = Population::makeRandom( fitnessMetric, - 10, + populationSize, minChromosomeLength, maxChromosomeLength ); @@ -86,7 +87,7 @@ void runAlgorithm(string const& _sourcePath) population, cout, { - /* elitePoolSize = */ 0.5, + /* elitePoolSize = */ 1.0 / populationSize, /* minChromosomeLength = */ minChromosomeLength, /* maxChromosomeLength = */ maxChromosomeLength, } From 66d733fbacd66ce91c448873d6fe0ab831fb5f0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Wed, 5 Feb 2020 19:25:15 +0100 Subject: [PATCH 57/92] [yul-phaser] main: Increase the number of chromosomes in the initial population to 20 --- tools/yulPhaser/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/yulPhaser/main.cpp b/tools/yulPhaser/main.cpp index 26a2ea005..321ebe088 100644 --- a/tools/yulPhaser/main.cpp +++ b/tools/yulPhaser/main.cpp @@ -71,7 +71,7 @@ CharStream loadSource(string const& _sourcePath) void runAlgorithm(string const& _sourcePath) { - constexpr size_t populationSize = 10; + constexpr size_t populationSize = 20; constexpr size_t minChromosomeLength = 12; constexpr size_t maxChromosomeLength = 30; From 40cae442d5ca32e6803e101839448b85b65d2d46 Mon Sep 17 00:00:00 2001 From: chriseth Date: Tue, 25 Feb 2020 17:55:14 +0100 Subject: [PATCH 58/92] Fix re-printing value expectations. --- test/libsolidity/util/SoltestTypes.h | 13 +------------ test/libsolidity/util/TestFileParserTests.cpp | 3 ++- test/libsolidity/util/TestFunctionCall.cpp | 10 +++++++++- 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/test/libsolidity/util/SoltestTypes.h b/test/libsolidity/util/SoltestTypes.h index 4c1cb01fd..26b92d535 100644 --- a/test/libsolidity/util/SoltestTypes.h +++ b/test/libsolidity/util/SoltestTypes.h @@ -238,24 +238,13 @@ enum class FunctionValueUnit }; /// Holds value along with unit it was expressed in originally. -/// Value should be always converted to wei, no meter on which unit it was originally +/// @a value is always in wei - it is converted back when stringifying again. struct FunctionValue { u256 value; FunctionValueUnit unit = FunctionValueUnit::Wei; }; -inline bool operator==(FunctionValue const& _a, FunctionValue const& _b) -{ - return _a.value == _b.value; -} - -inline std::ostream& operator<<(std::ostream& _os, FunctionValue const& _v) -{ - _os << _v.value << (_v.unit == FunctionValueUnit::Wei ? " wei" : " ether"); - return _os; -} - /** * Represents a function call read from an input stream. It contains the signature, the * arguments, an optional ether value and an expected execution result. diff --git a/test/libsolidity/util/TestFileParserTests.cpp b/test/libsolidity/util/TestFileParserTests.cpp index 82d885b9e..f074fc1bb 100644 --- a/test/libsolidity/util/TestFileParserTests.cpp +++ b/test/libsolidity/util/TestFileParserTests.cpp @@ -64,7 +64,8 @@ void testFunctionCall( ABI_CHECK(_call.arguments.rawBytes(), _arguments); ABI_CHECK(_call.expectations.rawBytes(), _expectations); BOOST_REQUIRE_EQUAL(_call.displayMode, _mode); - BOOST_REQUIRE_EQUAL(_call.value, _value); + BOOST_REQUIRE_EQUAL(_call.value.value, _value.value); + BOOST_REQUIRE_EQUAL(size_t(_call.value.unit), size_t(_value.unit)); BOOST_REQUIRE_EQUAL(_call.arguments.comment, _argumentComment); BOOST_REQUIRE_EQUAL(_call.expectations.comment, _expectationComment); diff --git a/test/libsolidity/util/TestFunctionCall.cpp b/test/libsolidity/util/TestFunctionCall.cpp index 43f190349..f961c615c 100644 --- a/test/libsolidity/util/TestFunctionCall.cpp +++ b/test/libsolidity/util/TestFunctionCall.cpp @@ -52,6 +52,7 @@ string TestFunctionCall::format( string comma = formatToken(Token::Comma); string comment = formatToken(Token::Comment); string ether = formatToken(Token::Ether); + string wei = formatToken(Token::Wei); string newline = formatToken(Token::Newline); string failure = formatToken(Token::Failure); @@ -64,7 +65,14 @@ string TestFunctionCall::format( /// Formats the function signature. This is the same independent from the display-mode. stream << _linePrefix << newline << ws << m_call.signature; if (m_call.value.value > u256(0)) - stream << comma << ws << m_call.value.value << ws << ether; + { + if (m_call.value.unit == FunctionValueUnit::Ether) + stream << comma << ws << (m_call.value.value / exp256(10, 18)) << ws << ether; + else if (m_call.value.unit == FunctionValueUnit::Wei) + stream << comma << ws << m_call.value.value << ws << wei; + else + soltestAssert(false, ""); + } if (!m_call.arguments.rawBytes().empty()) { string output = formatRawParameters(m_call.arguments.parameters, _linePrefix); From ec083c487837ebc409cda9a7531b843fe8a437fa Mon Sep 17 00:00:00 2001 From: Djordje Mijovic Date: Mon, 24 Feb 2020 18:11:20 +0100 Subject: [PATCH 59/92] Adding source location support to AssemblyStack and thus debugging Yul sources --- Changelog.md | 1 + libevmasm/AssemblyItem.cpp | 91 ++++++++++++++++++ libevmasm/AssemblyItem.h | 9 +- liblangutil/Scanner.h | 1 + libsolidity/interface/CompilerStack.cpp | 95 +------------------ libsolidity/interface/CompilerStack.h | 3 - libsolidity/interface/StandardCompiler.cpp | 2 +- libyul/AssemblyStack.cpp | 6 ++ libyul/AssemblyStack.h | 3 + test/cmdlineTests/standard_yul/output.json | 2 +- .../standard_yul_object/output.json | 2 +- .../standard_yul_object_name/output.json | 2 +- .../standard_yul_optimized/output.json | 2 +- test/libyul/ObjectCompilerTest.cpp | 3 + test/libyul/objectCompiler/data.yul | 1 + test/libyul/objectCompiler/datacopy.yul | 1 + .../libyul/objectCompiler/dataoffset_code.yul | 1 + .../libyul/objectCompiler/dataoffset_data.yul | 1 + .../libyul/objectCompiler/dataoffset_self.yul | 1 + test/libyul/objectCompiler/datasize_code.yul | 1 + test/libyul/objectCompiler/datasize_data.yul | 1 + test/libyul/objectCompiler/datasize_self.yul | 1 + .../libyul/objectCompiler/function_series.yul | 1 + .../libyul/objectCompiler/namedObjectCode.yul | 1 + .../objectCompiler/nested_optimizer.yul | 1 + test/libyul/objectCompiler/simple.yul | 1 + .../objectCompiler/simple_optimizer.yul | 1 + test/libyul/objectCompiler/subObject.yul | 1 + test/libyul/objectCompiler/subSubObject.yul | 1 + 29 files changed, 136 insertions(+), 101 deletions(-) diff --git a/Changelog.md b/Changelog.md index 9981ea25f..d1f0b76fa 100644 --- a/Changelog.md +++ b/Changelog.md @@ -4,6 +4,7 @@ Language Features: Compiler Features: + * AssemblyStack: Support for source locations (source mappings) and thus debugging Yul sources. Bugfixes: diff --git a/libevmasm/AssemblyItem.cpp b/libevmasm/AssemblyItem.cpp index b9b6c4acd..c238a3a3c 100644 --- a/libevmasm/AssemblyItem.cpp +++ b/libevmasm/AssemblyItem.cpp @@ -19,12 +19,14 @@ #include #include +#include #include using namespace std; using namespace solidity; using namespace solidity::evmasm; +using namespace solidity::langutil; static_assert(sizeof(size_t) <= 8, "size_t must be at most 64-bits wide"); @@ -281,3 +283,92 @@ ostream& solidity::evmasm::operator<<(ostream& _out, AssemblyItem const& _item) } return _out; } + +std::string AssemblyItem::computeSourceMapping( + AssemblyItems const& _items, + map const& _sourceIndicesMap +) +{ + string ret; + + int prevStart = -1; + int prevLength = -1; + int prevSourceIndex = -1; + size_t prevModifierDepth = -1; + char prevJump = 0; + for (auto const& item: _items) + { + if (!ret.empty()) + ret += ";"; + + SourceLocation const& location = item.location(); + int length = location.start != -1 && location.end != -1 ? location.end - location.start : -1; + int sourceIndex = + location.source && _sourceIndicesMap.count(location.source->name()) ? + _sourceIndicesMap.at(location.source->name()) : + -1; + char jump = '-'; + if (item.getJumpType() == evmasm::AssemblyItem::JumpType::IntoFunction) + jump = 'i'; + else if (item.getJumpType() == evmasm::AssemblyItem::JumpType::OutOfFunction) + jump = 'o'; + size_t modifierDepth = item.m_modifierDepth; + + unsigned components = 5; + if (modifierDepth == prevModifierDepth) + { + components--; + if (jump == prevJump) + { + components--; + if (sourceIndex == prevSourceIndex) + { + components--; + if (length == prevLength) + { + components--; + if (location.start == prevStart) + components--; + } + } + } + } + + if (components-- > 0) + { + if (location.start != prevStart) + ret += to_string(location.start); + if (components-- > 0) + { + ret += ':'; + if (length != prevLength) + ret += to_string(length); + if (components-- > 0) + { + ret += ':'; + if (sourceIndex != prevSourceIndex) + ret += to_string(sourceIndex); + if (components-- > 0) + { + ret += ':'; + if (jump != prevJump) + ret += jump; + if (components-- > 0) + { + ret += ':'; + if (modifierDepth != prevModifierDepth) + ret += to_string(modifierDepth); + } + } + } + } + } + + prevStart = location.start; + prevLength = length; + prevSourceIndex = sourceIndex; + prevJump = jump; + prevModifierDepth = modifierDepth; + } + return ret; +} diff --git a/libevmasm/AssemblyItem.h b/libevmasm/AssemblyItem.h index 31f175103..e506a9fdb 100644 --- a/libevmasm/AssemblyItem.h +++ b/libevmasm/AssemblyItem.h @@ -48,6 +48,8 @@ enum AssemblyItemType { }; class Assembly; +class AssemblyItem; +using AssemblyItems = std::vector; class AssemblyItem { @@ -122,6 +124,11 @@ public: } bool operator!=(Instruction _instr) const { return !operator==(_instr); } + static std::string computeSourceMapping( + AssemblyItems const& _items, + std::map const& _sourceIndicesMap + ); + /// @returns an upper bound for the number of bytes required by this item, assuming that /// the value of a jump tag takes @a _addressLength bytes. unsigned bytesRequired(unsigned _addressLength) const; @@ -157,8 +164,6 @@ private: mutable std::shared_ptr m_pushedValue; }; -using AssemblyItems = std::vector; - inline size_t bytesRequired(AssemblyItems const& _items, size_t _addressLength) { size_t size = 0; diff --git a/liblangutil/Scanner.h b/liblangutil/Scanner.h index 8f6cdab87..5b365f134 100644 --- a/liblangutil/Scanner.h +++ b/liblangutil/Scanner.h @@ -98,6 +98,7 @@ public: std::string const& source() const noexcept { return m_source->source(); } std::shared_ptr charStream() noexcept { return m_source; } + std::shared_ptr charStream() const noexcept { return m_source; } /// Resets the scanner as if newly constructed with _source as input. void reset(CharStream _source); diff --git a/libsolidity/interface/CompilerStack.cpp b/libsolidity/interface/CompilerStack.cpp index 89ab13818..8509d512f 100644 --- a/libsolidity/interface/CompilerStack.cpp +++ b/libsolidity/interface/CompilerStack.cpp @@ -565,7 +565,7 @@ string const* CompilerStack::sourceMapping(string const& _contractName) const if (!c.sourceMapping) { if (auto items = assemblyItems(_contractName)) - c.sourceMapping = make_unique(computeSourceMapping(*items)); + c.sourceMapping = make_unique(evmasm::AssemblyItem::computeSourceMapping(*items, sourceIndices())); } return c.sourceMapping.get(); } @@ -579,7 +579,9 @@ string const* CompilerStack::runtimeSourceMapping(string const& _contractName) c if (!c.runtimeSourceMapping) { if (auto items = runtimeAssemblyItems(_contractName)) - c.runtimeSourceMapping = make_unique(computeSourceMapping(*items)); + c.runtimeSourceMapping = make_unique( + evmasm::AssemblyItem::computeSourceMapping(*items, sourceIndices()) + ); } return c.runtimeSourceMapping.get(); } @@ -1390,95 +1392,6 @@ bytes CompilerStack::createCBORMetadata(string const& _metadata, bool _experimen return encoder.serialise(); } -string CompilerStack::computeSourceMapping(evmasm::AssemblyItems const& _items) const -{ - if (m_stackState != CompilationSuccessful) - BOOST_THROW_EXCEPTION(CompilerError() << errinfo_comment("Compilation was not successful.")); - - string ret; - map sourceIndicesMap = sourceIndices(); - int prevStart = -1; - int prevLength = -1; - int prevSourceIndex = -1; - size_t prevModifierDepth = -1; - char prevJump = 0; - for (auto const& item: _items) - { - if (!ret.empty()) - ret += ";"; - - SourceLocation const& location = item.location(); - int length = location.start != -1 && location.end != -1 ? location.end - location.start : -1; - int sourceIndex = - location.source && sourceIndicesMap.count(location.source->name()) ? - sourceIndicesMap.at(location.source->name()) : - -1; - char jump = '-'; - if (item.getJumpType() == evmasm::AssemblyItem::JumpType::IntoFunction) - jump = 'i'; - else if (item.getJumpType() == evmasm::AssemblyItem::JumpType::OutOfFunction) - jump = 'o'; - size_t modifierDepth = item.m_modifierDepth; - - unsigned components = 5; - if (modifierDepth == prevModifierDepth) - { - components--; - if (jump == prevJump) - { - components--; - if (sourceIndex == prevSourceIndex) - { - components--; - if (length == prevLength) - { - components--; - if (location.start == prevStart) - components--; - } - } - } - } - - if (components-- > 0) - { - if (location.start != prevStart) - ret += to_string(location.start); - if (components-- > 0) - { - ret += ':'; - if (length != prevLength) - ret += to_string(length); - if (components-- > 0) - { - ret += ':'; - if (sourceIndex != prevSourceIndex) - ret += to_string(sourceIndex); - if (components-- > 0) - { - ret += ':'; - if (jump != prevJump) - ret += jump; - if (components-- > 0) - { - ret += ':'; - if (modifierDepth != prevModifierDepth) - ret += to_string(modifierDepth); - } - } - } - } - } - - prevStart = location.start; - prevLength = length; - prevSourceIndex = sourceIndex; - prevJump = jump; - prevModifierDepth = modifierDepth; - } - return ret; -} - namespace { diff --git a/libsolidity/interface/CompilerStack.h b/libsolidity/interface/CompilerStack.h index 099375879..b0ea0603f 100644 --- a/libsolidity/interface/CompilerStack.h +++ b/libsolidity/interface/CompilerStack.h @@ -401,9 +401,6 @@ private: /// @returns the metadata CBOR for the given serialised metadata JSON. bytes createCBORMetadata(std::string const& _metadata, bool _experimentalMode); - /// @returns the computer source mapping string. - std::string computeSourceMapping(evmasm::AssemblyItems const& _items) const; - /// @returns the contract ABI as a JSON object. /// This will generate the JSON object and store it in the Contract object if it is not present yet. Json::Value const& contractABI(Contract const&) const; diff --git a/libsolidity/interface/StandardCompiler.cpp b/libsolidity/interface/StandardCompiler.cpp index 9f837a996..77e00df05 100644 --- a/libsolidity/interface/StandardCompiler.cpp +++ b/libsolidity/interface/StandardCompiler.cpp @@ -1081,7 +1081,7 @@ Json::Value StandardCompiler::compileYul(InputsAndSettings _inputsAndSettings) { "evm.bytecode", "evm.bytecode.object", "evm.bytecode.opcodes", "evm.bytecode.sourceMap", "evm.bytecode.linkReferences" }, wildcardMatchesExperimental )) - output["contracts"][sourceName][contractName]["evm"]["bytecode"] = collectEVMObject(*object.bytecode, nullptr); + output["contracts"][sourceName][contractName]["evm"]["bytecode"] = collectEVMObject(*object.bytecode, object.sourceMappings.get()); if (isArtifactRequested(_inputsAndSettings.outputSelection, sourceName, contractName, "irOptimized", wildcardMatchesExperimental)) output["contracts"][sourceName][contractName]["irOptimized"] = stack.print(); diff --git a/libyul/AssemblyStack.cpp b/libyul/AssemblyStack.cpp index 9f7cc57a5..7c0eca276 100644 --- a/libyul/AssemblyStack.cpp +++ b/libyul/AssemblyStack.cpp @@ -204,6 +204,12 @@ MachineAssemblyObject AssemblyStack::assemble(Machine _machine) const compileEVM(adapter, false, m_optimiserSettings.optimizeStackAllocation); object.bytecode = make_shared(assembly.assemble()); object.assembly = assembly.assemblyString(); + object.sourceMappings = make_unique( + evmasm::AssemblyItem::computeSourceMapping( + assembly.items(), + {{scanner().charStream() ? scanner().charStream()->name() : "", 0}} + ) + ); return object; } case Machine::EVM15: diff --git a/libyul/AssemblyStack.h b/libyul/AssemblyStack.h index 1104d6b05..60efd9674 100644 --- a/libyul/AssemblyStack.h +++ b/libyul/AssemblyStack.h @@ -48,6 +48,7 @@ struct MachineAssemblyObject { std::shared_ptr bytecode; std::string assembly; + std::unique_ptr sourceMappings; }; /* @@ -114,6 +115,8 @@ private: std::shared_ptr m_parserResult; langutil::ErrorList m_errors; langutil::ErrorReporter m_errorReporter; + + std::unique_ptr m_sourceMappings; }; } diff --git a/test/cmdlineTests/standard_yul/output.json b/test/cmdlineTests/standard_yul/output.json index b2c711e7b..ac6c8f7cb 100644 --- a/test/cmdlineTests/standard_yul/output.json +++ b/test/cmdlineTests/standard_yul/output.json @@ -14,7 +14,7 @@ sstore /* \"A\":0:42 */ pop -","bytecode":{"linkReferences":{},"object":"bytecode removed","opcodes":"opcodes removed","sourceMap":""}},"ir":"object \"object\" { +","bytecode":{"linkReferences":{},"object":"bytecode removed","opcodes":"opcodes removed","sourceMap":"sourceMap removed"}},"ir":"object \"object\" { code { let x := mload(0) sstore(add(x, 0), 0) diff --git a/test/cmdlineTests/standard_yul_object/output.json b/test/cmdlineTests/standard_yul_object/output.json index f7bf6c1b8..946e5773b 100644 --- a/test/cmdlineTests/standard_yul_object/output.json +++ b/test/cmdlineTests/standard_yul_object/output.json @@ -13,7 +13,7 @@ pop stop data_4e03657aea45a94fc7d47ba826c8d667c0d1e6e33a64a036ec44f58fa12d6c45 616263 -","bytecode":{"linkReferences":{},"object":"bytecode removed","opcodes":"opcodes removed","sourceMap":""}},"ir":"object \"NamedObject\" { +","bytecode":{"linkReferences":{},"object":"bytecode removed","opcodes":"opcodes removed","sourceMap":"sourceMap removed"}},"ir":"object \"NamedObject\" { code { let x := dataoffset(\"DataName\") sstore(add(x, 0), 0) diff --git a/test/cmdlineTests/standard_yul_object_name/output.json b/test/cmdlineTests/standard_yul_object_name/output.json index 92b4fab5b..2fdd9bc11 100644 --- a/test/cmdlineTests/standard_yul_object_name/output.json +++ b/test/cmdlineTests/standard_yul_object_name/output.json @@ -22,7 +22,7 @@ sub_0: assembly { /* \"A\":137:149 */ revert } -","bytecode":{"linkReferences":{},"object":"bytecode removed","opcodes":"opcodes removed","sourceMap":""}},"ir":"object \"NamedObject\" { +","bytecode":{"linkReferences":{},"object":"bytecode removed","opcodes":"opcodes removed","sourceMap":"sourceMap removed"}},"ir":"object \"NamedObject\" { code { let x := dataoffset(\"DataName\") sstore(add(x, 0), 0) diff --git a/test/cmdlineTests/standard_yul_optimized/output.json b/test/cmdlineTests/standard_yul_optimized/output.json index c5e1067f9..532ff00d9 100644 --- a/test/cmdlineTests/standard_yul_optimized/output.json +++ b/test/cmdlineTests/standard_yul_optimized/output.json @@ -5,7 +5,7 @@ mload /* \"A\":20:40 */ sstore -","bytecode":{"linkReferences":{},"object":"bytecode removed","opcodes":"opcodes removed","sourceMap":""}},"ir":"object \"object\" { +","bytecode":{"linkReferences":{},"object":"bytecode removed","opcodes":"opcodes removed","sourceMap":"sourceMap removed"}},"ir":"object \"object\" { code { let x := mload(0) sstore(add(x, 0), 0) diff --git a/test/libyul/ObjectCompilerTest.cpp b/test/libyul/ObjectCompilerTest.cpp index 7472b124f..f1b284042 100644 --- a/test/libyul/ObjectCompilerTest.cpp +++ b/test/libyul/ObjectCompilerTest.cpp @@ -74,6 +74,7 @@ TestCase::TestResult ObjectCompilerTest::run(ostream& _stream, string const& _li MachineAssemblyObject obj = stack.assemble(AssemblyStack::Machine::EVM); solAssert(obj.bytecode, ""); + solAssert(obj.sourceMappings, ""); m_obtainedResult = "Assembly:\n" + obj.assembly; if (obj.bytecode->bytecode.empty()) @@ -84,6 +85,8 @@ TestCase::TestResult ObjectCompilerTest::run(ostream& _stream, string const& _li toHex(obj.bytecode->bytecode) + "\nOpcodes: " + boost::trim_copy(evmasm::disassemble(obj.bytecode->bytecode)) + + "\nSourceMappings:" + + (obj.sourceMappings->empty() ? "" : " " + *obj.sourceMappings) + "\n"; if (m_expectation != m_obtainedResult) diff --git a/test/libyul/objectCompiler/data.yul b/test/libyul/objectCompiler/data.yul index daa22d21c..611715b65 100644 --- a/test/libyul/objectCompiler/data.yul +++ b/test/libyul/objectCompiler/data.yul @@ -9,3 +9,4 @@ object "a" { // data_acaf3289d7b601cbd114fb36c4d29c85bbfd5e133f14cb355c3fd8d99367964f 48656c6c6f2c20576f726c6421 // Bytecode: fe // Opcodes: INVALID +// SourceMappings: diff --git a/test/libyul/objectCompiler/datacopy.yul b/test/libyul/objectCompiler/datacopy.yul index 2259e5dc9..1a725bab0 100644 --- a/test/libyul/objectCompiler/datacopy.yul +++ b/test/libyul/objectCompiler/datacopy.yul @@ -46,3 +46,4 @@ object "a" { // } // Bytecode: 600b600d600039600b6000f3fe6000600055600d600052fe // Opcodes: PUSH1 0xB PUSH1 0xD PUSH1 0x0 CODECOPY PUSH1 0xB PUSH1 0x0 RETURN INVALID PUSH1 0x0 PUSH1 0x0 SSTORE PUSH1 0xD PUSH1 0x0 MSTORE INVALID +// SourceMappings: 26:47:0:-:0;;35:1;26:47;78:26;85:1;78:26 diff --git a/test/libyul/objectCompiler/dataoffset_code.yul b/test/libyul/objectCompiler/dataoffset_code.yul index 725267f27..ed6242090 100644 --- a/test/libyul/objectCompiler/dataoffset_code.yul +++ b/test/libyul/objectCompiler/dataoffset_code.yul @@ -27,3 +27,4 @@ object "a" { // } // Bytecode: 6006600055fe6008600055fe // Opcodes: PUSH1 0x6 PUSH1 0x0 SSTORE INVALID PUSH1 0x8 PUSH1 0x0 SSTORE INVALID +// SourceMappings: 22:28:0:-:0;29:1;22:28 diff --git a/test/libyul/objectCompiler/dataoffset_data.yul b/test/libyul/objectCompiler/dataoffset_data.yul index 9a0a461dc..6848ce9b9 100644 --- a/test/libyul/objectCompiler/dataoffset_data.yul +++ b/test/libyul/objectCompiler/dataoffset_data.yul @@ -14,3 +14,4 @@ object "a" { // data_acaf3289d7b601cbd114fb36c4d29c85bbfd5e133f14cb355c3fd8d99367964f 48656c6c6f2c20576f726c6421 // Bytecode: 6006600055fe48656c6c6f2c20576f726c6421 // Opcodes: PUSH1 0x6 PUSH1 0x0 SSTORE INVALID 0x48 PUSH6 0x6C6C6F2C2057 PUSH16 0x726C6421000000000000000000000000 +// SourceMappings: 22:30:0:-:0;29:1;22:30 diff --git a/test/libyul/objectCompiler/dataoffset_self.yul b/test/libyul/objectCompiler/dataoffset_self.yul index b77407353..6709bc98d 100644 --- a/test/libyul/objectCompiler/dataoffset_self.yul +++ b/test/libyul/objectCompiler/dataoffset_self.yul @@ -14,3 +14,4 @@ object "a" { // data_acaf3289d7b601cbd114fb36c4d29c85bbfd5e133f14cb355c3fd8d99367964f 48656c6c6f2c20576f726c6421 // Bytecode: 6000600055fe // Opcodes: PUSH1 0x0 PUSH1 0x0 SSTORE INVALID +// SourceMappings: 22:26:0:-:0;29:1;22:26 diff --git a/test/libyul/objectCompiler/datasize_code.yul b/test/libyul/objectCompiler/datasize_code.yul index cff685150..c48b1eba9 100644 --- a/test/libyul/objectCompiler/datasize_code.yul +++ b/test/libyul/objectCompiler/datasize_code.yul @@ -27,3 +27,4 @@ object "a" { // } // Bytecode: 6006600055fe // Opcodes: PUSH1 0x6 PUSH1 0x0 SSTORE INVALID +// SourceMappings: 22:26:0:-:0;29:1;22:26 diff --git a/test/libyul/objectCompiler/datasize_data.yul b/test/libyul/objectCompiler/datasize_data.yul index f83414697..191d162df 100644 --- a/test/libyul/objectCompiler/datasize_data.yul +++ b/test/libyul/objectCompiler/datasize_data.yul @@ -14,3 +14,4 @@ object "a" { // data_acaf3289d7b601cbd114fb36c4d29c85bbfd5e133f14cb355c3fd8d99367964f 48656c6c6f2c20576f726c6421 // Bytecode: 600d600055fe // Opcodes: PUSH1 0xD PUSH1 0x0 SSTORE INVALID +// SourceMappings: 22:28:0:-:0;29:1;22:28 diff --git a/test/libyul/objectCompiler/datasize_self.yul b/test/libyul/objectCompiler/datasize_self.yul index 4579fe67c..cfd96555a 100644 --- a/test/libyul/objectCompiler/datasize_self.yul +++ b/test/libyul/objectCompiler/datasize_self.yul @@ -14,3 +14,4 @@ object "a" { // data_acaf3289d7b601cbd114fb36c4d29c85bbfd5e133f14cb355c3fd8d99367964f 48656c6c6f2c20576f726c6421 // Bytecode: 6006600055fe // Opcodes: PUSH1 0x6 PUSH1 0x0 SSTORE INVALID +// SourceMappings: 22:24:0:-:0;29:1;22:24 diff --git a/test/libyul/objectCompiler/function_series.yul b/test/libyul/objectCompiler/function_series.yul index 88d2e171f..90dc88753 100644 --- a/test/libyul/objectCompiler/function_series.yul +++ b/test/libyul/objectCompiler/function_series.yul @@ -28,3 +28,4 @@ object "Contract" { // sstore // Bytecode: 6009565b5b565b5b565b6001600055 // Opcodes: PUSH1 0x9 JUMP JUMPDEST JUMPDEST JUMP JUMPDEST JUMPDEST JUMP JUMPDEST PUSH1 0x1 PUSH1 0x0 SSTORE +// SourceMappings: 33:15:0:-:0;;;46:2;;53:15;66:2;;;83:1;80;73:12 diff --git a/test/libyul/objectCompiler/namedObjectCode.yul b/test/libyul/objectCompiler/namedObjectCode.yul index 4fc6891c3..b13bdae5e 100644 --- a/test/libyul/objectCompiler/namedObjectCode.yul +++ b/test/libyul/objectCompiler/namedObjectCode.yul @@ -11,3 +11,4 @@ object "a" { // sstore // Bytecode: 6001600055 // Opcodes: PUSH1 0x1 PUSH1 0x0 SSTORE +// SourceMappings: 32:1:0:-:0;29;22:12 diff --git a/test/libyul/objectCompiler/nested_optimizer.yul b/test/libyul/objectCompiler/nested_optimizer.yul index 1d5e568f1..e4512f03d 100644 --- a/test/libyul/objectCompiler/nested_optimizer.yul +++ b/test/libyul/objectCompiler/nested_optimizer.yul @@ -38,3 +38,4 @@ object "a" { // } // Bytecode: 600060003555fe // Opcodes: PUSH1 0x0 PUSH1 0x0 CALLDATALOAD SSTORE INVALID +// SourceMappings: 48:1:0:-:0;;35:15;107:20 diff --git a/test/libyul/objectCompiler/simple.yul b/test/libyul/objectCompiler/simple.yul index d41b527cb..3e7d1859f 100644 --- a/test/libyul/objectCompiler/simple.yul +++ b/test/libyul/objectCompiler/simple.yul @@ -11,3 +11,4 @@ // sstore // Bytecode: 6001600055 // Opcodes: PUSH1 0x1 PUSH1 0x0 SSTORE +// SourceMappings: 14:1:0:-:0;11;4:12 diff --git a/test/libyul/objectCompiler/simple_optimizer.yul b/test/libyul/objectCompiler/simple_optimizer.yul index 3d00e45d3..40234a1db 100644 --- a/test/libyul/objectCompiler/simple_optimizer.yul +++ b/test/libyul/objectCompiler/simple_optimizer.yul @@ -17,3 +17,4 @@ // sstore // Bytecode: 600060003555 // Opcodes: PUSH1 0x0 PUSH1 0x0 CALLDATALOAD SSTORE +// SourceMappings: 26:1:0:-:0;;13:15;79:20 diff --git a/test/libyul/objectCompiler/subObject.yul b/test/libyul/objectCompiler/subObject.yul index 98ea4d079..f99b071af 100644 --- a/test/libyul/objectCompiler/subObject.yul +++ b/test/libyul/objectCompiler/subObject.yul @@ -19,3 +19,4 @@ object "a" { // } // Bytecode: fe // Opcodes: INVALID +// SourceMappings: diff --git a/test/libyul/objectCompiler/subSubObject.yul b/test/libyul/objectCompiler/subSubObject.yul index 5e01f6ddd..a36f97619 100644 --- a/test/libyul/objectCompiler/subSubObject.yul +++ b/test/libyul/objectCompiler/subSubObject.yul @@ -37,3 +37,4 @@ object "a" { // } // Bytecode: fe // Opcodes: INVALID +// SourceMappings: From 9815a618b007fc95a9b8ff7f2df733ee47793fad Mon Sep 17 00:00:00 2001 From: Bhargava Shastry Date: Wed, 26 Feb 2020 11:52:40 +0100 Subject: [PATCH 60/92] Docker: install curl in ubuntu16.04 ossfuzz base image --- .circleci/config.yml | 2 +- .circleci/docker/Dockerfile.ubuntu1604.clang.ossfuzz | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8bee0b83b..0319ca152 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -18,7 +18,7 @@ parameters: default: "5" ubuntu-1604-clang-ossfuzz-docker-image-rev: type: string - default: "1" + default: "2" defaults: diff --git a/.circleci/docker/Dockerfile.ubuntu1604.clang.ossfuzz b/.circleci/docker/Dockerfile.ubuntu1604.clang.ossfuzz index 8e2ee01b0..f7dfc7001 100644 --- a/.circleci/docker/Dockerfile.ubuntu1604.clang.ossfuzz +++ b/.circleci/docker/Dockerfile.ubuntu1604.clang.ossfuzz @@ -30,7 +30,7 @@ RUN apt-get update; \ build-essential \ software-properties-common \ ninja-build git wget \ - libbz2-dev zlib1g-dev git; \ + libbz2-dev zlib1g-dev git curl; \ apt-get install -qy python-pip python-sphinx; # Install cmake 3.14 (minimum requirement is cmake 3.10) From d1b6a4a649557b0740c860d2b0e4106bc4a8e8c7 Mon Sep 17 00:00:00 2001 From: Erik Kundt Date: Wed, 26 Feb 2020 21:37:52 +0100 Subject: [PATCH 61/92] Fixes raw bytes warning in semantic test framework. --- test/libsolidity/util/BytesUtils.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/libsolidity/util/BytesUtils.cpp b/test/libsolidity/util/BytesUtils.cpp index 0c5518afb..6377fbe2e 100644 --- a/test/libsolidity/util/BytesUtils.cpp +++ b/test/libsolidity/util/BytesUtils.cpp @@ -225,7 +225,7 @@ string BytesUtils::formatRawBytes( { bytes byteRange{it, it + static_cast(parameter.abiType.size)}; - os << _linePrefix << formatBytes(byteRange, parameter.abiType); + os << _linePrefix << byteRange; if (¶meter != ¶meters.back()) os << endl; From bc3261936471b0cc4f7b2c9ae1be3179d52ca71d Mon Sep 17 00:00:00 2001 From: chriseth Date: Wed, 26 Feb 2020 15:50:34 +0100 Subject: [PATCH 62/92] Enable optimized IR output via the commandline. --- Changelog.md | 1 + solc/CommandLineInterface.cpp | 66 ++++++++++++++++++++++------------- solc/CommandLineInterface.h | 1 + 3 files changed, 44 insertions(+), 24 deletions(-) diff --git a/Changelog.md b/Changelog.md index f151365ae..9f99cea33 100644 --- a/Changelog.md +++ b/Changelog.md @@ -6,6 +6,7 @@ Language Features: Compiler Features: * AssemblyStack: Support for source locations (source mappings) and thus debugging Yul sources. + * Commandline Interface: Enable output of experimental optimized IR via ``--ir-optimized``. Bugfixes: diff --git a/solc/CommandLineInterface.cpp b/solc/CommandLineInterface.cpp index ccb192ba1..e91f6446c 100644 --- a/solc/CommandLineInterface.cpp +++ b/solc/CommandLineInterface.cpp @@ -127,6 +127,7 @@ static string const g_strInterface = "interface"; static string const g_strYul = "yul"; static string const g_strYulDialect = "yul-dialect"; static string const g_strIR = "ir"; +static string const g_strIROptimized = "ir-optimized"; static string const g_strIPFS = "ipfs"; static string const g_strLicense = "license"; static string const g_strLibraries = "libraries"; @@ -190,6 +191,7 @@ static string const g_argImportAst = g_strImportAst; static string const g_argInputFile = g_strInputFile; static string const g_argYul = g_strYul; static string const g_argIR = g_strIR; +static string const g_argIROptimized = g_strIROptimized; static string const g_argEwasm = g_strEwasm; static string const g_argLibraries = g_strLibraries; static string const g_argLink = g_strLink; @@ -336,36 +338,50 @@ void CommandLineInterface::handleOpcode(string const& _contract) void CommandLineInterface::handleIR(string const& _contractName) { - if (m_args.count(g_argIR)) + if (!m_args.count(g_argIR)) + return; + + if (m_args.count(g_argOutputDir)) + createFile(m_compiler->filesystemFriendlyName(_contractName) + ".yul", m_compiler->yulIR(_contractName)); + else { - if (m_args.count(g_argOutputDir)) - createFile(m_compiler->filesystemFriendlyName(_contractName) + ".yul", m_compiler->yulIR(_contractName)); - else - { - sout() << "IR:" << endl; - sout() << m_compiler->yulIR(_contractName) << endl; - } + sout() << "IR:" << endl; + sout() << m_compiler->yulIR(_contractName) << endl; + } +} + +void CommandLineInterface::handleIROptimized(string const& _contractName) +{ + if (!m_args.count(g_argIROptimized)) + return; + + if (m_args.count(g_argOutputDir)) + createFile(m_compiler->filesystemFriendlyName(_contractName) + "_opt.yul", m_compiler->yulIROptimized(_contractName)); + else + { + sout() << "Optimized IR:" << endl; + sout() << m_compiler->yulIROptimized(_contractName) << endl; } } void CommandLineInterface::handleEwasm(string const& _contractName) { - if (m_args.count(g_argEwasm)) + if (!m_args.count(g_argEwasm)) + return; + + if (m_args.count(g_argOutputDir)) { - if (m_args.count(g_argOutputDir)) - { - createFile(m_compiler->filesystemFriendlyName(_contractName) + ".wast", m_compiler->ewasm(_contractName)); - createFile( - m_compiler->filesystemFriendlyName(_contractName) + ".wasm", - asString(m_compiler->ewasmObject(_contractName).bytecode) - ); - } - else - { - sout() << "Ewasm text:" << endl; - sout() << m_compiler->ewasm(_contractName) << endl; - sout() << "Ewasm binary (hex): " << m_compiler->ewasmObject(_contractName).toHex() << endl; - } + createFile(m_compiler->filesystemFriendlyName(_contractName) + ".wast", m_compiler->ewasm(_contractName)); + createFile( + m_compiler->filesystemFriendlyName(_contractName) + ".wasm", + asString(m_compiler->ewasmObject(_contractName).bytecode) + ); + } + else + { + sout() << "Ewasm text:" << endl; + sout() << m_compiler->ewasm(_contractName) << endl; + sout() << "Ewasm binary (hex): " << m_compiler->ewasmObject(_contractName).toHex() << endl; } } @@ -812,6 +828,7 @@ Allowed options)", (g_argBinaryRuntime.c_str(), "Binary of the runtime part of the contracts in hex.") (g_argAbi.c_str(), "ABI specification of the contracts.") (g_argIR.c_str(), "Intermediate Representation (IR) of all contracts (EXPERIMENTAL).") + (g_argIROptimized.c_str(), "Optimized intermediate Representation (IR) of all contracts (EXPERIMENTAL).") (g_argEwasm.c_str(), "Ewasm text representation of all contracts (EXPERIMENTAL).") (g_argSignatureHashes.c_str(), "Function signature hashes of the contracts.") (g_argNatspecUser.c_str(), "Natspec user documentation of all contracts.") @@ -1123,7 +1140,7 @@ bool CommandLineInterface::processInput() m_compiler->setRevertStringBehaviour(m_revertStrings); // TODO: Perhaps we should not compile unless requested - m_compiler->enableIRGeneration(m_args.count(g_argIR)); + m_compiler->enableIRGeneration(m_args.count(g_argIR) || m_args.count(g_argIROptimized)); m_compiler->enableEwasmGeneration(m_args.count(g_argEwasm)); OptimiserSettings settings = m_args.count(g_argOptimize) ? OptimiserSettings::standard() : OptimiserSettings::minimal(); @@ -1631,6 +1648,7 @@ void CommandLineInterface::outputCompilationResults() handleBytecode(contract); handleIR(contract); + handleIROptimized(contract); handleEwasm(contract); handleSignatureHashes(contract); handleMetadata(contract); diff --git a/solc/CommandLineInterface.h b/solc/CommandLineInterface.h index 553f9e69b..417501907 100644 --- a/solc/CommandLineInterface.h +++ b/solc/CommandLineInterface.h @@ -65,6 +65,7 @@ private: void handleBinary(std::string const& _contract); void handleOpcode(std::string const& _contract); void handleIR(std::string const& _contract); + void handleIROptimized(std::string const& _contract); void handleEwasm(std::string const& _contract); void handleBytecode(std::string const& _contract); void handleSignatureHashes(std::string const& _contract); From ed02aae1d9cad7514df6131ed326dc33f87923f2 Mon Sep 17 00:00:00 2001 From: Bhargava Shastry Date: Fri, 28 Feb 2020 15:41:22 +0100 Subject: [PATCH 63/92] Update solidity fuzzing dictionary with >0.6.0 keywords --- test/tools/ossfuzz/config/solidity.dict | 13 ++++++++++++- test/tools/ossfuzz/config/strict_assembly.dict | 1 + 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/test/tools/ossfuzz/config/solidity.dict b/test/tools/ossfuzz/config/solidity.dict index 9fe773870..8190e7c55 100644 --- a/test/tools/ossfuzz/config/solidity.dict +++ b/test/tools/ossfuzz/config/solidity.dict @@ -119,7 +119,7 @@ "bytes8 " "bytes9 " "constant " -"constructor " +"constructor() " "continue;" "contract " "delete " @@ -212,3 +212,14 @@ "|" "}" "~" +"override" +"virtual" +" is " +"receive() " +"fallback() " +"catch Error() {}" +"catch (bytes memory ) {}" +"{value: 1, gas: 2}" +"{salt: "salt", value: 10}" +"leave" +"a[1:2]" diff --git a/test/tools/ossfuzz/config/strict_assembly.dict b/test/tools/ossfuzz/config/strict_assembly.dict index 4415c87f4..7f4fe4b5b 100644 --- a/test/tools/ossfuzz/config/strict_assembly.dict +++ b/test/tools/ossfuzz/config/strict_assembly.dict @@ -89,3 +89,4 @@ "xor(" "{" "}" +"leave" From f10c6500b2378e652c5e7e91f255501d9acec4ba Mon Sep 17 00:00:00 2001 From: chriseth Date: Tue, 25 Feb 2020 18:25:19 +0100 Subject: [PATCH 64/92] Immutable is not reserved anymore. --- liblangutil/Token.h | 2 +- test/libsolidity/SolidityParser.cpp | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/liblangutil/Token.h b/liblangutil/Token.h index ed6cff3e1..d9d4471cb 100644 --- a/liblangutil/Token.h +++ b/liblangutil/Token.h @@ -166,6 +166,7 @@ namespace solidity::langutil K(Indexed, "indexed", 0) \ K(Interface, "interface", 0) \ K(Internal, "internal", 0) \ + K(Immutable, "immutable", 0) \ K(Import, "import", 0) \ K(Is, "is", 0) \ K(Library, "library", 0) \ @@ -243,7 +244,6 @@ namespace solidity::langutil K(Default, "default", 0) \ K(Define, "define", 0) \ K(Final, "final", 0) \ - K(Immutable, "immutable", 0) \ K(Implements, "implements", 0) \ K(In, "in", 0) \ K(Inline, "inline", 0) \ diff --git a/test/libsolidity/SolidityParser.cpp b/test/libsolidity/SolidityParser.cpp index 3f70f9160..192a26898 100644 --- a/test/libsolidity/SolidityParser.cpp +++ b/test/libsolidity/SolidityParser.cpp @@ -540,7 +540,6 @@ BOOST_AUTO_TEST_CASE(keyword_is_reserved) "default", "define", "final", - "immutable", "implements", "in", "inline", From 1488a1ceb807633beef3349ad785ee98be45742d Mon Sep 17 00:00:00 2001 From: chriseth Date: Tue, 25 Feb 2020 18:42:16 +0100 Subject: [PATCH 65/92] Refactor isConstant to add "immutable". --- libsolidity/ast/AST.cpp | 2 +- libsolidity/ast/AST.h | 10 ++++++---- libsolidity/ast/ASTJsonImporter.cpp | 8 +++++++- libsolidity/parsing/Parser.cpp | 6 +++--- 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/libsolidity/ast/AST.cpp b/libsolidity/ast/AST.cpp index 90084f9c4..6b03c7b58 100644 --- a/libsolidity/ast/AST.cpp +++ b/libsolidity/ast/AST.cpp @@ -390,7 +390,7 @@ DeclarationAnnotation& Declaration::annotation() const bool VariableDeclaration::isLValue() const { // Constant declared variables are Read-Only - if (m_isConstant) + if (isConstant()) return false; // External function arguments of reference type are Read-Only if (isExternalCallableParameter() && dynamic_cast(type())) diff --git a/libsolidity/ast/AST.h b/libsolidity/ast/AST.h index 336c221c6..adb957cfe 100644 --- a/libsolidity/ast/AST.h +++ b/libsolidity/ast/AST.h @@ -814,6 +814,7 @@ class VariableDeclaration: public Declaration { public: enum Location { Unspecified, Storage, Memory, CallData }; + enum class Constantness { Mutable, Immutable, Constant }; VariableDeclaration( int64_t _id, @@ -824,7 +825,7 @@ public: Visibility _visibility, bool _isStateVar = false, bool _isIndexed = false, - bool _isConstant = false, + Constantness _constantness = Constantness::Mutable, ASTPointer const& _overrides = nullptr, Location _referenceLocation = Location::Unspecified ): @@ -833,7 +834,7 @@ public: m_value(_value), m_isStateVariable(_isStateVar), m_isIndexed(_isIndexed), - m_isConstant(_isConstant), + m_constantness(_constantness), m_overrides(_overrides), m_location(_referenceLocation) {} @@ -877,7 +878,7 @@ public: bool hasReferenceOrMappingType() const; bool isStateVariable() const { return m_isStateVariable; } bool isIndexed() const { return m_isIndexed; } - bool isConstant() const { return m_isConstant; } + bool isConstant() const { return m_constantness == Constantness::Constant; } ASTPointer const& overrides() const { return m_overrides; } Location referenceLocation() const { return m_location; } /// @returns a set of allowed storage locations for the variable. @@ -904,7 +905,8 @@ private: ASTPointer m_value; bool m_isStateVariable = false; ///< Whether or not this is a contract state variable bool m_isIndexed = false; ///< Whether this is an indexed variable (used by events). - bool m_isConstant = false; ///< Whether the variable is a compile-time constant. + /// Whether the variable is "constant", "immutable" or non-marked (mutable). + Constantness m_constantness = Constantness::Mutable; ASTPointer m_overrides; ///< Contains the override specifier node Location m_location = Location::Unspecified; ///< Location of the variable if it is of reference type. }; diff --git a/libsolidity/ast/ASTJsonImporter.cpp b/libsolidity/ast/ASTJsonImporter.cpp index aa28e3f17..26a7c7b7e 100644 --- a/libsolidity/ast/ASTJsonImporter.cpp +++ b/libsolidity/ast/ASTJsonImporter.cpp @@ -411,6 +411,12 @@ ASTPointer ASTJsonImporter::createVariableDeclaration(Json: { astAssert(_node["name"].isString(), "Expected 'name' to be a string!"); + VariableDeclaration::Constantness constantness{}; + if (memberAsBool(_node, "constant")) + constantness = VariableDeclaration::Constantness::Constant; + else + constantness = VariableDeclaration::Constantness::Mutable; + return createASTNode( _node, nullOrCast(member(_node, "typeName")), @@ -419,7 +425,7 @@ ASTPointer ASTJsonImporter::createVariableDeclaration(Json: visibility(_node), memberAsBool(_node, "stateVariable"), _node.isMember("indexed") ? memberAsBool(_node, "indexed") : false, - memberAsBool(_node, "constant"), + constantness, _node["overrides"].isNull() ? nullptr : createOverrideSpecifier(member(_node, "overrides")), location(_node) ); diff --git a/libsolidity/parsing/Parser.cpp b/libsolidity/parsing/Parser.cpp index 99aa6e2a1..edf43377f 100644 --- a/libsolidity/parsing/Parser.cpp +++ b/libsolidity/parsing/Parser.cpp @@ -695,7 +695,7 @@ ASTPointer Parser::parseVariableDeclaration( ); bool isIndexed = false; - bool isDeclaredConst = false; + VariableDeclaration::Constantness constantness = VariableDeclaration::Constantness::Mutable; ASTPointer overrides = nullptr; Visibility visibility(Visibility::Default); VariableDeclaration::Location location = VariableDeclaration::Location::Unspecified; @@ -731,7 +731,7 @@ ASTPointer Parser::parseVariableDeclaration( if (_options.allowIndexed && token == Token::Indexed) isIndexed = true; else if (token == Token::Constant) - isDeclaredConst = true; + constantness = VariableDeclaration::Constantness::Constant; else if (_options.allowLocationSpecifier && TokenTraits::isLocationSpecifier(token)) { if (location != VariableDeclaration::Location::Unspecified) @@ -790,7 +790,7 @@ ASTPointer Parser::parseVariableDeclaration( visibility, _options.isStateVariable, isIndexed, - isDeclaredConst, + constantness, overrides, location ); From 90fa56c7191d4353d8cdbc778c0f71a4459f4a56 Mon Sep 17 00:00:00 2001 From: Daniel Kirchner Date: Mon, 2 Mar 2020 16:32:30 +0100 Subject: [PATCH 66/92] Allow use of yul util functions in legacy code generation. --- libsolidity/codegen/ABIFunctions.cpp | 11 +--------- libsolidity/codegen/ABIFunctions.h | 7 ------- libsolidity/codegen/CompilerContext.cpp | 14 +++++++++++++ libsolidity/codegen/CompilerContext.h | 20 ++++++++++++++++++- libsolidity/codegen/ContractCompiler.cpp | 8 ++++---- .../codegen/MultiUseYulFunctionCollector.cpp | 12 +++++++++-- .../codegen/MultiUseYulFunctionCollector.h | 19 ++++++++++++++++-- libsolidity/codegen/ir/IRGenerator.cpp | 6 +++--- 8 files changed, 68 insertions(+), 29 deletions(-) diff --git a/libsolidity/codegen/ABIFunctions.cpp b/libsolidity/codegen/ABIFunctions.cpp index 0ad044a54..b69f47346 100644 --- a/libsolidity/codegen/ABIFunctions.cpp +++ b/libsolidity/codegen/ABIFunctions.cpp @@ -240,13 +240,6 @@ string ABIFunctions::tupleDecoder(TypePointers const& _types, bool _fromMemory) }); } -pair> ABIFunctions::requestedFunctions() -{ - std::set empty; - swap(empty, m_externallyUsedFunctions); - return make_pair(m_functionCollector->requestedFunctions(), std::move(empty)); -} - string ABIFunctions::EncodingOptions::toFunctionNameSuffix() const { string suffix; @@ -1504,9 +1497,7 @@ string ABIFunctions::createFunction(string const& _name, function con string ABIFunctions::createExternallyUsedFunction(string const& _name, function const& _creator) { - string name = createFunction(_name, _creator); - m_externallyUsedFunctions.insert(name); - return name; + return m_functionCollector->createExternallyUsedFunction(_name, _creator); } size_t ABIFunctions::headSize(TypePointers const& _targetTypes) diff --git a/libsolidity/codegen/ABIFunctions.h b/libsolidity/codegen/ABIFunctions.h index ce3efe6ea..74bfdf7fa 100644 --- a/libsolidity/codegen/ABIFunctions.h +++ b/libsolidity/codegen/ABIFunctions.h @@ -104,12 +104,6 @@ public: /// stack slot, it takes exactly that number of values. std::string tupleDecoder(TypePointers const& _types, bool _fromMemory = false); - /// @returns concatenation of all generated functions and a set of the - /// externally used functions. - /// Clears the internal list, i.e. calling it again will result in an - /// empty return value. - std::pair> requestedFunctions(); - private: struct EncodingOptions { @@ -260,7 +254,6 @@ private: langutil::EVMVersion m_evmVersion; RevertStrings const m_revertStrings; std::shared_ptr m_functionCollector; - std::set m_externallyUsedFunctions; YulUtilFunctions m_utils; }; diff --git a/libsolidity/codegen/CompilerContext.cpp b/libsolidity/codegen/CompilerContext.cpp index 8c1099855..f65ccae52 100644 --- a/libsolidity/codegen/CompilerContext.cpp +++ b/libsolidity/codegen/CompilerContext.cpp @@ -94,6 +94,20 @@ void CompilerContext::callLowLevelFunction( *this << retTag.tag(); } +void CompilerContext::callYulUtilFunction( + string const& _name, + unsigned _inArgs, + unsigned _outArgs +) +{ + m_functionCollector->markAsExternallyUsed(_name); + auto retTag = pushNewTag(); + CompilerUtils(*this).moveIntoStack(_inArgs); + appendJumpTo(namedTag(_name)); + adjustStackOffset(int(_outArgs) - 1 - _inArgs); + *this << retTag.tag(); +} + evmasm::AssemblyItem CompilerContext::lowLevelFunctionTag( string const& _name, unsigned _inArgs, diff --git a/libsolidity/codegen/CompilerContext.h b/libsolidity/codegen/CompilerContext.h index c37afbfa5..d9393e23a 100644 --- a/libsolidity/codegen/CompilerContext.h +++ b/libsolidity/codegen/CompilerContext.h @@ -65,7 +65,8 @@ public: m_evmVersion(_evmVersion), m_revertStrings(_revertStrings), m_runtimeContext(_runtimeContext), - m_abiFunctions(m_evmVersion, m_revertStrings) + m_abiFunctions(m_evmVersion, m_revertStrings, m_functionCollector), + m_yulUtilFunctions(m_evmVersion, m_revertStrings, m_functionCollector) { if (m_runtimeContext) m_runtimeSub = size_t(m_asm->newSub(m_runtimeContext->m_asm).data()); @@ -131,6 +132,14 @@ public: unsigned _outArgs, std::function const& _generator ); + + /// Appends a call to a yul util function and registers the function as externally used. + void callYulUtilFunction( + std::string const& _name, + unsigned _inArgs, + unsigned _outArgs + ); + /// Returns the tag of the named low-level function and inserts the generator into the /// list of low-level-functions to be generated, unless it already exists. /// Note that the generator should not assume that objects are still alive when it is called, @@ -144,6 +153,11 @@ public: /// Generates the code for missing low-level functions, i.e. calls the generators passed above. void appendMissingLowLevelFunctions(); ABIFunctions& abiFunctions() { return m_abiFunctions; } + YulUtilFunctions& utilFunctions() { return m_yulUtilFunctions; } + std::pair> requestedYulFunctions() + { + return m_functionCollector->requestedFunctions(); + } ModifierDefinition const& resolveVirtualFunctionModifier(ModifierDefinition const& _modifier) const; /// Returns the distance of the given local variable from the bottom of the stack (of the current function). @@ -355,8 +369,12 @@ private: size_t m_runtimeSub = -1; /// An index of low-level function labels by name. std::map m_lowLevelFunctions; + // Collector for yul functions. + std::shared_ptr m_functionCollector = std::make_shared(); /// Container for ABI functions to be generated. ABIFunctions m_abiFunctions; + /// Container for Yul Util functions to be generated. + YulUtilFunctions m_yulUtilFunctions; /// The queue of low-level functions to generate. std::queue>> m_lowLevelFunctionGenerationQueue; }; diff --git a/libsolidity/codegen/ContractCompiler.cpp b/libsolidity/codegen/ContractCompiler.cpp index a99aefb5f..47187b1c8 100644 --- a/libsolidity/codegen/ContractCompiler.cpp +++ b/libsolidity/codegen/ContractCompiler.cpp @@ -1267,12 +1267,12 @@ void ContractCompiler::appendMissingFunctions() solAssert(m_context.nextFunctionToCompile() != function, "Compiled the wrong function?"); } m_context.appendMissingLowLevelFunctions(); - auto abiFunctions = m_context.abiFunctions().requestedFunctions(); - if (!abiFunctions.first.empty()) + auto yulFunctions = m_context.requestedYulFunctions(); + if (!yulFunctions.first.empty()) m_context.appendInlineAssembly( - "{" + move(abiFunctions.first) + "}", + "{" + move(yulFunctions.first) + "}", {}, - abiFunctions.second, + yulFunctions.second, true, m_optimiserSettings ); diff --git a/libsolidity/codegen/MultiUseYulFunctionCollector.cpp b/libsolidity/codegen/MultiUseYulFunctionCollector.cpp index 2be3d29ae..8af292eda 100644 --- a/libsolidity/codegen/MultiUseYulFunctionCollector.cpp +++ b/libsolidity/codegen/MultiUseYulFunctionCollector.cpp @@ -30,13 +30,15 @@ using namespace std; using namespace solidity; using namespace solidity::frontend; -string MultiUseYulFunctionCollector::requestedFunctions() +pair> MultiUseYulFunctionCollector::requestedFunctions() { string result; for (auto const& f: m_requestedFunctions) result += f.second; m_requestedFunctions.clear(); - return result; + std::set empty; + swap(empty, m_externallyUsedFunctions); + return make_pair(result, std::move(empty)); } string MultiUseYulFunctionCollector::createFunction(string const& _name, function const& _creator) @@ -50,3 +52,9 @@ string MultiUseYulFunctionCollector::createFunction(string const& _name, functio } return _name; } + +string MultiUseYulFunctionCollector::createExternallyUsedFunction(string const& _name, function const& _creator) +{ + m_externallyUsedFunctions.insert(_name); + return createFunction(_name, _creator); +} diff --git a/libsolidity/codegen/MultiUseYulFunctionCollector.h b/libsolidity/codegen/MultiUseYulFunctionCollector.h index d839a31be..a1c733274 100644 --- a/libsolidity/codegen/MultiUseYulFunctionCollector.h +++ b/libsolidity/codegen/MultiUseYulFunctionCollector.h @@ -23,6 +23,7 @@ #include #include +#include #include namespace solidity::frontend @@ -40,14 +41,28 @@ public: /// cases. std::string createFunction(std::string const& _name, std::function const& _creator); - /// @returns concatenation of all generated functions. + /// Helper function that uses @a _creator to create a function and add it to + /// @a m_requestedFunctions if it has not been created yet and returns @a _name in both + /// cases. + std::string createExternallyUsedFunction(std::string const& _name, std::function const& _creator); + + /// Manually mark a function as externally used. + void markAsExternallyUsed(std::string const& _name) + { + m_externallyUsedFunctions.insert(_name); + } + + /// @returns concatenation of all generated functions and a set of the + /// externally used functions. /// Clears the internal list, i.e. calling it again will result in an /// empty return value. - std::string requestedFunctions(); + std::pair> requestedFunctions(); private: /// Map from function name to code for a multi-use function. std::map m_requestedFunctions; + // Set of externally used functions. + std::set m_externallyUsedFunctions; }; } diff --git a/libsolidity/codegen/ir/IRGenerator.cpp b/libsolidity/codegen/ir/IRGenerator.cpp index 9c2d8807c..0f2ffaf35 100644 --- a/libsolidity/codegen/ir/IRGenerator.cpp +++ b/libsolidity/codegen/ir/IRGenerator.cpp @@ -107,7 +107,7 @@ string IRGenerator::generate(ContractDefinition const& _contract) for (auto const* contract: _contract.annotation().linearizedBaseContracts) for (auto const* fun: contract->definedFunctions()) generateFunction(*fun); - t("functions", m_context.functionCollector()->requestedFunctions()); + t("functions", m_context.functionCollector()->requestedFunctions().first); resetContext(_contract); m_context.setInheritanceHierarchy(_contract.annotation().linearizedBaseContracts); @@ -116,7 +116,7 @@ string IRGenerator::generate(ContractDefinition const& _contract) for (auto const* contract: _contract.annotation().linearizedBaseContracts) for (auto const* fun: contract->definedFunctions()) generateFunction(*fun); - t("runtimeFunctions", m_context.functionCollector()->requestedFunctions()); + t("runtimeFunctions", m_context.functionCollector()->requestedFunctions().first); return t.render(); } @@ -383,7 +383,7 @@ string IRGenerator::memoryInit() void IRGenerator::resetContext(ContractDefinition const& _contract) { solAssert( - m_context.functionCollector()->requestedFunctions().empty(), + m_context.functionCollector()->requestedFunctions().first.empty(), "Reset context while it still had functions." ); m_context = IRGenerationContext(m_evmVersion, m_context.revertStrings(), m_optimiserSettings); From 96a230af507187aa42d849a95f45cb551a61cd80 Mon Sep 17 00:00:00 2001 From: Leonardo Alt Date: Mon, 2 Mar 2020 22:10:15 +0100 Subject: [PATCH 67/92] [SMTChecker] Fix ICEs with tuples --- Changelog.md | 1 + libsolidity/formal/SMTEncoder.cpp | 15 +++++++++++---- .../smtCheckerTests/types/function_in_tuple_1.sol | 10 ++++++++++ .../smtCheckerTests/types/function_in_tuple_2.sol | 10 ++++++++++ .../types/tuple_single_element_1.sol | 9 +++++++++ .../types/tuple_single_element_2.sol | 9 +++++++++ .../types/tuple_single_non_tuple_element.sol | 9 +++++++++ 7 files changed, 59 insertions(+), 4 deletions(-) create mode 100644 test/libsolidity/smtCheckerTests/types/function_in_tuple_1.sol create mode 100644 test/libsolidity/smtCheckerTests/types/function_in_tuple_2.sol create mode 100644 test/libsolidity/smtCheckerTests/types/tuple_single_element_1.sol create mode 100644 test/libsolidity/smtCheckerTests/types/tuple_single_element_2.sol create mode 100644 test/libsolidity/smtCheckerTests/types/tuple_single_non_tuple_element.sol diff --git a/Changelog.md b/Changelog.md index f151365ae..fc5d3d0d3 100644 --- a/Changelog.md +++ b/Changelog.md @@ -11,6 +11,7 @@ Compiler Features: Bugfixes: * isoltest: Added new keyword `wei` to express function value in semantic tests * Standard-JSON-Interface: Fix a bug related to empty filenames and imports. + * SMTChecker: Fix internal errors when analysing tuples. ### 0.6.3 (2020-02-18) diff --git a/libsolidity/formal/SMTEncoder.cpp b/libsolidity/formal/SMTEncoder.cpp index ac81a5d87..3b2f48816 100644 --- a/libsolidity/formal/SMTEncoder.cpp +++ b/libsolidity/formal/SMTEncoder.cpp @@ -407,19 +407,26 @@ void SMTEncoder::endVisit(TupleExpression const& _tuple) auto const& symbTuple = dynamic_pointer_cast(m_context.expression(_tuple)); solAssert(symbTuple, ""); auto const& symbComponents = symbTuple->components(); - auto const& tupleComponents = _tuple.components(); - solAssert(symbComponents.size() == _tuple.components().size(), ""); + auto const* tupleComponents = &_tuple.components(); + while (tupleComponents->size() == 1) + { + auto innerTuple = dynamic_pointer_cast(tupleComponents->front()); + solAssert(innerTuple, ""); + tupleComponents = &innerTuple->components(); + } + solAssert(symbComponents.size() == tupleComponents->size(), ""); for (unsigned i = 0; i < symbComponents.size(); ++i) { auto sComponent = symbComponents.at(i); - auto tComponent = tupleComponents.at(i); + auto tComponent = tupleComponents->at(i); if (sComponent && tComponent) { if (auto varDecl = identifierToVariable(*tComponent)) m_context.addAssertion(sComponent->currentValue() == currentValue(*varDecl)); else { - solAssert(m_context.knownExpression(*tComponent), ""); + if (!m_context.knownExpression(*tComponent)) + createExpr(*tComponent); m_context.addAssertion(sComponent->currentValue() == expr(*tComponent)); } } diff --git a/test/libsolidity/smtCheckerTests/types/function_in_tuple_1.sol b/test/libsolidity/smtCheckerTests/types/function_in_tuple_1.sol new file mode 100644 index 000000000..4d1d954f7 --- /dev/null +++ b/test/libsolidity/smtCheckerTests/types/function_in_tuple_1.sol @@ -0,0 +1,10 @@ +pragma experimental SMTChecker; + +contract K { + function f() public pure { + (abi.encode, 2); + } +} +// ---- +// Warning: (76-91): Statement has no effect. +// Warning: (77-80): Assertion checker does not yet implement type abi diff --git a/test/libsolidity/smtCheckerTests/types/function_in_tuple_2.sol b/test/libsolidity/smtCheckerTests/types/function_in_tuple_2.sol new file mode 100644 index 000000000..5b9c2f41c --- /dev/null +++ b/test/libsolidity/smtCheckerTests/types/function_in_tuple_2.sol @@ -0,0 +1,10 @@ +pragma experimental SMTChecker; + +contract K { + function f() public pure { + (abi.encode, ""); + } +} +// ---- +// Warning: (76-92): Statement has no effect. +// Warning: (77-80): Assertion checker does not yet implement type abi diff --git a/test/libsolidity/smtCheckerTests/types/tuple_single_element_1.sol b/test/libsolidity/smtCheckerTests/types/tuple_single_element_1.sol new file mode 100644 index 000000000..f14ca9bc1 --- /dev/null +++ b/test/libsolidity/smtCheckerTests/types/tuple_single_element_1.sol @@ -0,0 +1,9 @@ +pragma experimental SMTChecker; + +contract C { + function f() public pure { + (("", 2)); + } +} +// ---- +// Warning: (76-85): Statement has no effect. diff --git a/test/libsolidity/smtCheckerTests/types/tuple_single_element_2.sol b/test/libsolidity/smtCheckerTests/types/tuple_single_element_2.sol new file mode 100644 index 000000000..29b9c9747 --- /dev/null +++ b/test/libsolidity/smtCheckerTests/types/tuple_single_element_2.sol @@ -0,0 +1,9 @@ +pragma experimental SMTChecker; + +contract C { + function f() public pure { + (("", "")); + } +} +// ---- +// Warning: (76-86): Statement has no effect. diff --git a/test/libsolidity/smtCheckerTests/types/tuple_single_non_tuple_element.sol b/test/libsolidity/smtCheckerTests/types/tuple_single_non_tuple_element.sol new file mode 100644 index 000000000..2a99a552e --- /dev/null +++ b/test/libsolidity/smtCheckerTests/types/tuple_single_non_tuple_element.sol @@ -0,0 +1,9 @@ +pragma experimental SMTChecker; + +contract C { + function f() public pure { + (2); + } +} +// ---- +// Warning: (76-79): Statement has no effect. From 3bee348525de032cdc510dc62b7366c90cf13d89 Mon Sep 17 00:00:00 2001 From: Leonardo Alt Date: Tue, 11 Feb 2020 22:52:35 -0300 Subject: [PATCH 68/92] Change CHC encoding to functions forest instead of explicit CFG --- libsolidity/formal/CHC.cpp | 385 +++++++++++++----- libsolidity/formal/CHC.h | 77 +++- libsolidity/formal/EncodingContext.h | 1 + libsolidity/formal/ModelChecker.cpp | 4 +- libsolidity/formal/ModelChecker.h | 6 +- libsolidity/formal/Z3Interface.h | 6 +- .../functions/library_constant.sol | 26 ++ .../functions/library_constant_2.sol | 9 + .../invariants/loop_nested.sol | 5 + .../invariants/loop_nested_for.sol | 5 + ...r_loop_array_assignment_storage_memory.sol | 6 +- ..._loop_array_assignment_storage_storage.sol | 8 +- .../operators/delete_array_index_2d.sol | 4 +- 13 files changed, 426 insertions(+), 116 deletions(-) create mode 100644 test/libsolidity/smtCheckerTests/functions/library_constant.sol create mode 100644 test/libsolidity/smtCheckerTests/functions/library_constant_2.sol diff --git a/libsolidity/formal/CHC.cpp b/libsolidity/formal/CHC.cpp index 25201474b..af50b2722 100644 --- a/libsolidity/formal/CHC.cpp +++ b/libsolidity/formal/CHC.cpp @@ -27,6 +27,8 @@ #include +#include + using namespace std; using namespace solidity; using namespace solidity::langutil; @@ -75,16 +77,39 @@ void CHC::analyze(SourceUnit const& _source) m_context.setAssertionAccumulation(false); m_variableUsage.setFunctionInlining(false); + resetSourceAnalysis(); + auto boolSort = make_shared(smt::Kind::Bool); auto genesisSort = make_shared( vector(), boolSort ); m_genesisPredicate = createSymbolicBlock(genesisSort, "genesis"); - auto genesis = (*m_genesisPredicate)({}); - addRule(genesis, genesis.name); + addRule(genesis(), "genesis"); - _source.accept(*this); + set sources; + sources.insert(&_source); + for (auto const& source: _source.referencedSourceUnits(true)) + sources.insert(source); + for (auto const* source: sources) + defineInterfacesAndSummaries(*source); + for (auto const* source: sources) + source->accept(*this); + + for (auto const& [scope, target]: m_verificationTargets) + { + auto assertions = transactionAssertions(scope); + for (auto const* assertion: assertions) + { + createErrorBlock(); + connectBlocks(target.value, error(), target.constraints && (target.errorId == assertion->id())); + auto [result, model] = query(error(), assertion->location()); + // This should be fine but it's a bug in the old compiler + (void)model; + if (result == smt::CheckResult::UNSATISFIABLE) + m_safeAssertions.insert(assertion); + } + } } vector CHC::unhandledQueries() const @@ -97,26 +122,15 @@ vector CHC::unhandledQueries() const bool CHC::visit(ContractDefinition const& _contract) { - if (!shouldVisit(_contract)) - return false; - - reset(); + resetContractAnalysis(); initContract(_contract); - m_stateVariables = _contract.stateVariablesIncludingInherited(); - - for (auto const& var: m_stateVariables) - // SMT solvers do not support function types as arguments. - if (var->type()->category() == Type::Category::Function) - m_stateSorts.push_back(make_shared(smt::Kind::Int)); - else - m_stateSorts.push_back(smt::smtSort(*var->type())); + m_stateVariables = stateVariablesIncludingInheritedAndPrivate(_contract); + m_stateSorts = stateSorts(_contract); clearIndices(&_contract); - string suffix = _contract.name() + "_" + to_string(_contract.id()); - m_interfacePredicate = createSymbolicBlock(interfaceSort(), "interface_" + suffix); // TODO create static instances for Bool/Int sorts in SolverInterface. auto boolSort = make_shared(smt::Kind::Bool); @@ -125,10 +139,12 @@ bool CHC::visit(ContractDefinition const& _contract) boolSort ); + string suffix = _contract.name() + "_" + to_string(_contract.id()); m_errorPredicate = createSymbolicBlock(errorFunctionSort, "error_" + suffix); - m_constructorPredicate = createSymbolicBlock(constructorSort(), "implicit_constructor_" + to_string(_contract.id())); + m_constructorSummaryPredicate = createSymbolicBlock(constructorSort(), "summary_constructor_" + suffix); + m_implicitConstructorPredicate = createSymbolicBlock(interfaceSort(), "implicit_constructor_" + suffix); auto stateExprs = currentStateVariables(); - setCurrentBlock(*m_interfacePredicate, &stateExprs); + setCurrentBlock(*m_interfaces.at(m_currentContract), &stateExprs); SMTEncoder::visit(_contract); return false; @@ -136,33 +152,33 @@ bool CHC::visit(ContractDefinition const& _contract) void CHC::endVisit(ContractDefinition const& _contract) { - if (!shouldVisit(_contract)) - return; - for (auto const& var: m_stateVariables) { solAssert(m_context.knownVariable(*var), ""); + auto const& symbVar = m_context.variable(*var); + symbVar->resetIndex(); m_context.setZeroValue(*var); + symbVar->increaseIndex(); } - auto genesisPred = (*m_genesisPredicate)({}); - auto implicitConstructor = (*m_constructorPredicate)(currentStateVariables()); - connectBlocks(genesisPred, implicitConstructor); + auto implicitConstructor = (*m_implicitConstructorPredicate)(initialStateVariables()); + connectBlocks(genesis(), implicitConstructor); m_currentBlock = implicitConstructor; + m_context.addAssertion(m_error.currentValue() == 0); if (auto constructor = _contract.constructor()) constructor->accept(*this); else inlineConstructorHierarchy(_contract); - connectBlocks(m_currentBlock, interface()); + auto summary = predicate(*m_constructorSummaryPredicate, vector{m_error.currentValue()} + currentStateVariables()); + connectBlocks(m_currentBlock, summary); - for (unsigned i = 0; i < m_verificationTargets.size(); ++i) - { - auto const& target = m_verificationTargets.at(i); - auto errorAppl = error(i + 1); - if (query(errorAppl, target->location())) - m_safeAssertions.insert(target); - } + clearIndices(m_currentContract, nullptr); + auto stateExprs = vector{m_error.currentValue()} + currentStateVariables(); + setCurrentBlock(*m_constructorSummaryPredicate, &stateExprs); + + addVerificationTarget(m_currentContract, m_currentBlock, smt::Expression(true), m_error.currentValue()); + connectBlocks(m_currentBlock, interface(), m_error.currentValue() == 0); SMTEncoder::endVisit(_contract); } @@ -182,7 +198,7 @@ bool CHC::visit(FunctionDefinition const& _function) return false; } - solAssert(!m_currentFunction, "Inlining internal function calls not yet implemented"); + solAssert(!m_currentFunction, "Function inlining should not happen in CHC."); m_currentFunction = &_function; initFunction(_function); @@ -193,7 +209,17 @@ bool CHC::visit(FunctionDefinition const& _function) auto functionPred = predicate(*functionEntryBlock, currentFunctionVariables()); auto bodyPred = predicate(*bodyBlock); - connectBlocks(m_currentBlock, functionPred); + if (_function.isConstructor()) + connectBlocks(m_currentBlock, functionPred); + else + connectBlocks(genesis(), functionPred); + + m_context.addAssertion(m_error.currentValue() == 0); + for (auto const* var: m_stateVariables) + m_context.addAssertion(m_context.variable(*var)->valueAtIndex(0) == currentValue(*var)); + for (auto const& var: _function.parameters()) + m_context.addAssertion(m_context.variable(*var)->valueAtIndex(0) == currentValue(*var)); + connectBlocks(functionPred, bodyPred); setCurrentBlock(*bodyBlock); @@ -225,18 +251,30 @@ void CHC::endVisit(FunctionDefinition const& _function) // This is done in endVisit(ContractDefinition). if (_function.isConstructor()) { - auto constructorExit = createSymbolicBlock(interfaceSort(), "constructor_exit_" + to_string(_function.id())); - connectBlocks(m_currentBlock, predicate(*constructorExit, currentStateVariables())); + string suffix = m_currentContract->name() + "_" + to_string(m_currentContract->id()); + auto constructorExit = createSymbolicBlock(constructorSort(), "constructor_exit_" + suffix); + connectBlocks(m_currentBlock, predicate(*constructorExit, vector{m_error.currentValue()} + currentStateVariables())); + clearIndices(m_currentContract, m_currentFunction); - auto stateExprs = currentStateVariables(); + auto stateExprs = vector{m_error.currentValue()} + currentStateVariables(); setCurrentBlock(*constructorExit, &stateExprs); } else { - connectBlocks(m_currentBlock, interface()); - clearIndices(m_currentContract, m_currentFunction); - auto stateExprs = currentStateVariables(); - setCurrentBlock(*m_interfacePredicate, &stateExprs); + auto assertionError = m_error.currentValue(); + auto sum = summary(_function); + connectBlocks(m_currentBlock, sum); + + auto iface = interface(); + + auto stateExprs = initialStateVariables(); + setCurrentBlock(*m_interfaces.at(m_currentContract), &stateExprs); + + if (_function.isPublic()) + { + addVerificationTarget(&_function, m_currentBlock, sum, assertionError); + connectBlocks(m_currentBlock, iface, sum && (assertionError == 0)); + } } m_currentFunction = nullptr; } @@ -468,12 +506,23 @@ void CHC::visitAssert(FunctionCall const& _funCall) solAssert(args.size() == 1, ""); solAssert(args.front()->annotation().type->category() == Type::Category::Bool, ""); - createErrorBlock(); + solAssert(m_currentContract, ""); + solAssert(m_currentFunction, ""); + if (m_currentFunction->isConstructor()) + m_functionAssertions[m_currentContract].insert(&_funCall); + else + m_functionAssertions[m_currentFunction].insert(&_funCall); - smt::Expression assertNeg = !(m_context.expression(*args.front())->currentValue()); - connectBlocks(m_currentBlock, error(), currentPathConditions() && assertNeg); + auto previousError = m_error.currentValue(); + m_error.increaseIndex(); - m_verificationTargets.push_back(&_funCall); + connectBlocks( + m_currentBlock, + m_currentFunction->isConstructor() ? summary(*m_currentContract) : summary(*m_currentFunction), + currentPathConditions() && !m_context.expression(*args.front())->currentValue() && (m_error.currentValue() == _funCall.id()) + ); + + m_context.addAssertion(m_error.currentValue() == previousError); } void CHC::unknownFunctionCall(FunctionCall const&) @@ -488,15 +537,23 @@ void CHC::unknownFunctionCall(FunctionCall const&) m_unknownFunctionCallSeen = true; } -void CHC::reset() +void CHC::resetSourceAnalysis() +{ + m_verificationTargets.clear(); + m_safeAssertions.clear(); + m_functionAssertions.clear(); + m_callGraph.clear(); + m_summaries.clear(); +} + +void CHC::resetContractAnalysis() { m_stateSorts.clear(); m_stateVariables.clear(); - m_verificationTargets.clear(); - m_safeAssertions.clear(); m_unknownFunctionCallSeen = false; m_breakDest = nullptr; m_continueDest = nullptr; + m_error.resetIndex(); } void CHC::eraseKnowledge() @@ -521,17 +578,6 @@ void CHC::clearIndices(ContractDefinition const* _contract, FunctionDefinition c } } - -bool CHC::shouldVisit(ContractDefinition const& _contract) const -{ - if ( - _contract.isLibrary() || - _contract.isInterface() - ) - return false; - return true; -} - bool CHC::shouldVisit(FunctionDefinition const& _function) const { if ( @@ -547,7 +593,8 @@ void CHC::setCurrentBlock( vector const* _arguments ) { - m_context.popSolver(); + if (m_context.solverStackHeigh() > 0) + m_context.popSolver(); solAssert(m_currentContract, ""); clearIndices(m_currentContract, m_currentFunction); m_context.pushSolver(); @@ -557,10 +604,42 @@ void CHC::setCurrentBlock( m_currentBlock = predicate(_block); } +set CHC::transactionAssertions(ASTNode const* _txRoot) +{ + set assertions; + solidity::util::BreadthFirstSearch{{_txRoot}}.run([&](auto const* function, auto&& _addChild) { + assertions.insert(m_functionAssertions[function].begin(), m_functionAssertions[function].end()); + for (auto const* called: m_callGraph[function]) + _addChild(called); + }); + return assertions; +} + +vector CHC::stateVariablesIncludingInheritedAndPrivate(ContractDefinition const& _contract) +{ + vector stateVars; + for (auto const& contract: _contract.annotation().linearizedBaseContracts) + for (auto var: contract->stateVariables()) + stateVars.push_back(var); + return stateVars; +} + +vector CHC::stateSorts(ContractDefinition const& _contract) +{ + vector stateSorts; + for (auto const& var: stateVariablesIncludingInheritedAndPrivate(_contract)) + stateSorts.push_back(smt::smtSortAbstractFunction(*var->type())); + return stateSorts; +} + smt::SortPointer CHC::constructorSort() { - // TODO this will change once we support function calls. - return interfaceSort(); + auto boolSort = make_shared(smt::Kind::Bool); + auto intSort = make_shared(smt::Kind::Int); + return make_shared( + vector{intSort} + m_stateSorts, + boolSort + ); } smt::SortPointer CHC::interfaceSort() @@ -572,20 +651,38 @@ smt::SortPointer CHC::interfaceSort() ); } +smt::SortPointer CHC::interfaceSort(ContractDefinition const& _contract) +{ + auto boolSort = make_shared(smt::Kind::Bool); + return make_shared( + stateSorts(_contract), + boolSort + ); +} + +/// A function in the symbolic CFG requires: +/// - Index of failed assertion. 0 means no assertion failed. +/// - 2 sets of state variables: +/// - State variables at the beginning of the current function, immutable +/// - Current state variables +/// At the beginning of the function these must equal set 1 +/// - 2 sets of input variables: +/// - Input variables at the beginning of the current function, immutable +/// - Current input variables +/// At the beginning of the function these must equal set 1 +/// - 1 set of output variables smt::SortPointer CHC::sort(FunctionDefinition const& _function) { auto boolSort = make_shared(smt::Kind::Bool); - vector varSorts; - for (auto const& var: _function.parameters() + _function.returnParameters()) - { - // SMT solvers do not support function types as arguments. - if (var->type()->category() == Type::Category::Function) - varSorts.push_back(make_shared(smt::Kind::Int)); - else - varSorts.push_back(smt::smtSort(*var->type())); - } + auto intSort = make_shared(smt::Kind::Int); + vector inputSorts; + for (auto const& var: _function.parameters()) + inputSorts.push_back(smt::smtSortAbstractFunction(*var->type())); + vector outputSorts; + for (auto const& var: _function.returnParameters()) + outputSorts.push_back(smt::smtSortAbstractFunction(*var->type())); return make_shared( - m_stateSorts + varSorts, + vector{intSort} + m_stateSorts + inputSorts + m_stateSorts + inputSorts + outputSorts, boolSort ); } @@ -601,19 +698,31 @@ smt::SortPointer CHC::sort(ASTNode const* _node) auto boolSort = make_shared(smt::Kind::Bool); vector varSorts; for (auto const& var: m_currentFunction->localVariables()) - { - // SMT solvers do not support function types as arguments. - if (var->type()->category() == Type::Category::Function) - varSorts.push_back(make_shared(smt::Kind::Int)); - else - varSorts.push_back(smt::smtSort(*var->type())); - } + varSorts.push_back(smt::smtSortAbstractFunction(*var->type())); return make_shared( fSort->domain + varSorts, boolSort ); } +smt::SortPointer CHC::summarySort(FunctionDefinition const& _function, ContractDefinition const& _contract) +{ + auto stateVariables = stateVariablesIncludingInheritedAndPrivate(_contract); + auto sorts = stateSorts(_contract); + + auto boolSort = make_shared(smt::Kind::Bool); + auto intSort = make_shared(smt::Kind::Int); + vector inputSorts, outputSorts; + for (auto const& var: _function.parameters()) + inputSorts.push_back(smt::smtSortAbstractFunction(*var->type())); + for (auto const& var: _function.returnParameters()) + outputSorts.push_back(smt::smtSortAbstractFunction(*var->type())); + return make_shared( + vector{intSort} + sorts + inputSorts + sorts + outputSorts, + boolSort + ); +} + unique_ptr CHC::createSymbolicBlock(smt::SortPointer _sort, string const& _name) { auto block = make_unique( @@ -625,12 +734,33 @@ unique_ptr CHC::createSymbolicBlock(smt::SortPoin return block; } +void CHC::defineInterfacesAndSummaries(SourceUnit const& _source) +{ + for (auto const& node: _source.nodes()) + if (auto const* contract = dynamic_cast(node.get())) + for (auto const* base: contract->annotation().linearizedBaseContracts) + { + string suffix = base->name() + "_" + to_string(base->id()); + m_interfaces[base] = createSymbolicBlock(interfaceSort(*base), "interface_" + suffix); + for (auto const* var: stateVariablesIncludingInheritedAndPrivate(*base)) + if (!m_context.knownVariable(*var)) + createVariable(*var); + for (auto const* function: base->definedFunctions()) + m_summaries[contract].emplace(function, createSummaryBlock(*function, *contract)); + } +} + smt::Expression CHC::interface() { vector paramExprs; for (auto const& var: m_stateVariables) paramExprs.push_back(m_context.variable(*var)->currentValue()); - return (*m_interfacePredicate)(paramExprs); + return (*m_interfaces.at(m_currentContract))(paramExprs); +} + +smt::Expression CHC::interface(ContractDefinition const& _contract) +{ + return (*m_interfaces.at(&_contract))(stateVariablesAtIndex(0, _contract)); } smt::Expression CHC::error() @@ -643,6 +773,27 @@ smt::Expression CHC::error(unsigned _idx) return m_errorPredicate->functionValueAtIndex(_idx)({}); } +smt::Expression CHC::summary(ContractDefinition const&) +{ + return (*m_constructorSummaryPredicate)( + vector{m_error.currentValue()} + + currentStateVariables() + ); +} + +smt::Expression CHC::summary(FunctionDefinition const& _function) +{ + vector args{m_error.currentValue()}; + auto contract = _function.annotation().contract; + args += contract->isLibrary() ? stateVariablesAtIndex(0, *contract) : initialStateVariables(); + for (auto const& var: _function.parameters()) + args.push_back(m_context.variable(*var)->valueAtIndex(0)); + args += contract->isLibrary() ? stateVariablesAtIndex(1, *contract) : currentStateVariables(); + for (auto const& var: _function.returnParameters()) + args.push_back(m_context.variable(*var)->currentValue()); + return (*m_summaries.at(m_currentContract).at(&_function))(args); +} + unique_ptr CHC::createBlock(ASTNode const* _node, string const& _prefix) { return createSymbolicBlock(sort(_node), @@ -653,6 +804,15 @@ unique_ptr CHC::createBlock(ASTNode const* _node, predicateName(_node)); } +unique_ptr CHC::createSummaryBlock(FunctionDefinition const& _function, ContractDefinition const& _contract) +{ + return createSymbolicBlock(summarySort(_function, _contract), + "summary_" + + uniquePrefix() + + "_" + + predicateName(&_function, &_contract)); +} + void CHC::createErrorBlock() { solAssert(m_errorPredicate, ""); @@ -669,6 +829,28 @@ void CHC::connectBlocks(smt::Expression const& _from, smt::Expression const& _to addRule(edge, _from.name + "_to_" + _to.name); } +vector CHC::initialStateVariables() +{ + return stateVariablesAtIndex(0); +} + +vector CHC::stateVariablesAtIndex(int _index) +{ + solAssert(m_currentContract, ""); + vector exprs; + for (auto const& var: m_stateVariables) + exprs.push_back(m_context.variable(*var)->valueAtIndex(_index)); + return exprs; +} + +vector CHC::stateVariablesAtIndex(int _index, ContractDefinition const& _contract) +{ + vector exprs; + for (auto const& var: stateVariablesIncludingInheritedAndPrivate(_contract)) + exprs.push_back(m_context.variable(*var)->valueAtIndex(_index)); + return exprs; +} + vector CHC::currentStateVariables() { solAssert(m_currentContract, ""); @@ -680,11 +862,22 @@ vector CHC::currentStateVariables() vector CHC::currentFunctionVariables() { - vector paramExprs; - if (m_currentFunction) - for (auto const& var: m_currentFunction->parameters() + m_currentFunction->returnParameters()) - paramExprs.push_back(m_context.variable(*var)->currentValue()); - return currentStateVariables() + paramExprs; + vector initInputExprs; + vector mutableInputExprs; + for (auto const& var: m_currentFunction->parameters()) + { + initInputExprs.push_back(m_context.variable(*var)->valueAtIndex(0)); + mutableInputExprs.push_back(m_context.variable(*var)->currentValue()); + } + vector returnExprs; + for (auto const& var: m_currentFunction->returnParameters()) + returnExprs.push_back(m_context.variable(*var)->currentValue()); + return vector{m_error.currentValue()} + + initialStateVariables() + + initInputExprs + + currentStateVariables() + + mutableInputExprs + + returnExprs; } vector CHC::currentBlockVariables() @@ -696,7 +889,7 @@ vector CHC::currentBlockVariables() return currentFunctionVariables() + paramExprs; } -string CHC::predicateName(ASTNode const* _node) +string CHC::predicateName(ASTNode const* _node, ContractDefinition const* _contract) { string prefix; if (auto funDef = dynamic_cast(_node)) @@ -705,7 +898,12 @@ string CHC::predicateName(ASTNode const* _node) if (!funDef->name().empty()) prefix += "_" + funDef->name() + "_"; } - return prefix + to_string(_node->id()); + else if (m_currentFunction && !m_currentFunction->name().empty()) + prefix += m_currentFunction->name(); + + auto contract = _contract ? _contract : m_currentContract; + solAssert(contract, ""); + return prefix + "_" + to_string(_node->id()) + "_" + to_string(contract->id()); } smt::Expression CHC::predicate(smt::SymbolicFunctionVariable const& _block) @@ -726,7 +924,7 @@ void CHC::addRule(smt::Expression const& _rule, string const& _ruleName) m_interface->addRule(_rule, _ruleName); } -bool CHC::query(smt::Expression const& _query, langutil::SourceLocation const& _location) +pair> CHC::query(smt::Expression const& _query, langutil::SourceLocation const& _location) { smt::CheckResult result; vector values; @@ -736,7 +934,7 @@ bool CHC::query(smt::Expression const& _query, langutil::SourceLocation const& _ case smt::CheckResult::SATISFIABLE: break; case smt::CheckResult::UNSATISFIABLE: - return true; + break; case smt::CheckResult::UNKNOWN: break; case smt::CheckResult::CONFLICTING: @@ -746,7 +944,12 @@ bool CHC::query(smt::Expression const& _query, langutil::SourceLocation const& _ m_outerErrorReporter.warning(_location, "Error trying to invoke SMT solver."); break; } - return false; + return {result, values}; +} + +void CHC::addVerificationTarget(ASTNode const* _scope, smt::Expression _from, smt::Expression _constraints, smt::Expression _errorId) +{ + m_verificationTargets.emplace(_scope, CHCVerificationTarget{{VerificationTarget::Type::Assert, _from, _constraints}, _errorId}); } string CHC::uniquePrefix() diff --git a/libsolidity/formal/CHC.h b/libsolidity/formal/CHC.h index 6f912fbd8..889c21b6a 100644 --- a/libsolidity/formal/CHC.h +++ b/libsolidity/formal/CHC.h @@ -79,22 +79,38 @@ private: void unknownFunctionCall(FunctionCall const& _funCall); //@} + struct IdCompare + { + bool operator()(ASTNode const* lhs, ASTNode const* rhs) const + { + return lhs->id() < rhs->id(); + } + }; + /// Helpers. //@{ - void reset(); + void resetSourceAnalysis(); + void resetContractAnalysis(); void eraseKnowledge(); void clearIndices(ContractDefinition const* _contract, FunctionDefinition const* _function = nullptr) override; - bool shouldVisit(ContractDefinition const& _contract) const; bool shouldVisit(FunctionDefinition const& _function) const; void setCurrentBlock(smt::SymbolicFunctionVariable const& _block, std::vector const* _arguments = nullptr); + std::set transactionAssertions(ASTNode const* _txRoot); + static std::vector stateVariablesIncludingInheritedAndPrivate(ContractDefinition const& _contract); //@} /// Sort helpers. //@{ + static std::vector stateSorts(ContractDefinition const& _contract); smt::SortPointer constructorSort(); smt::SortPointer interfaceSort(); + static smt::SortPointer interfaceSort(ContractDefinition const& _const); smt::SortPointer sort(FunctionDefinition const& _function); smt::SortPointer sort(ASTNode const* _block); + /// @returns the sort of a predicate that represents the summary of _function in the scope of _contract. + /// The _contract is also needed because the same function might be in many contracts due to inheritance, + /// where the sort changes because the set of state variables might change. + static smt::SortPointer summarySort(FunctionDefinition const& _function, ContractDefinition const& _contract); //@} /// Predicate helpers. @@ -102,14 +118,24 @@ private: /// @returns a new block of given _sort and _name. std::unique_ptr createSymbolicBlock(smt::SortPointer _sort, std::string const& _name); + /// Creates summary predicates for all functions of all contracts + /// in a given _source. + void defineInterfacesAndSummaries(SourceUnit const& _source); + + /// Genesis predicate. + smt::Expression genesis() { return (*m_genesisPredicate)({}); } /// Interface predicate over current variables. smt::Expression interface(); + smt::Expression interface(ContractDefinition const& _contract); /// Error predicate over current variables. smt::Expression error(); smt::Expression error(unsigned _idx); /// Creates a block for the given _node. std::unique_ptr createBlock(ASTNode const* _node, std::string const& _prefix = ""); + /// Creates a call block for the given function _function from contract _contract. + /// The contract is needed here because of inheritance. + std::unique_ptr createSummaryBlock(FunctionDefinition const& _function, ContractDefinition const& _contract); /// Creates a new error block to be used by an assertion. /// Also registers the predicate. @@ -117,6 +143,11 @@ private: void connectBlocks(smt::Expression const& _from, smt::Expression const& _to, smt::Expression const& _constraints = smt::Expression(true)); + /// @returns the symbolic values of the state variables at the beginning + /// of the current transaction. + std::vector initialStateVariables(); + std::vector stateVariablesAtIndex(int _index); + std::vector stateVariablesAtIndex(int _index, ContractDefinition const& _contract); /// @returns the current symbolic values of the current state variables. std::vector currentStateVariables(); @@ -128,19 +159,26 @@ private: std::vector currentBlockVariables(); /// @returns the predicate name for a given node. - std::string predicateName(ASTNode const* _node); + std::string predicateName(ASTNode const* _node, ContractDefinition const* _contract = nullptr); /// @returns a predicate application over the current scoped variables. smt::Expression predicate(smt::SymbolicFunctionVariable const& _block); /// @returns a predicate application over @param _arguments. smt::Expression predicate(smt::SymbolicFunctionVariable const& _block, std::vector const& _arguments); + /// @returns a predicate that defines a constructor summary. + smt::Expression summary(ContractDefinition const& _contract); + /// @returns a predicate that defines a function summary. + smt::Expression summary(FunctionDefinition const& _function); //@} /// Solver related. //@{ /// Adds Horn rule to the solver. void addRule(smt::Expression const& _rule, std::string const& _ruleName); - /// @returns true if query is unsatisfiable (safe). - bool query(smt::Expression const& _query, langutil::SourceLocation const& _location); + /// @returns if query is unsatisfiable (safe). + /// @returns otherwise. + std::pair> query(smt::Expression const& _query, langutil::SourceLocation const& _location); + + void addVerificationTarget(ASTNode const* _scope, smt::Expression _from, smt::Expression _constraints, smt::Expression _errorId); //@} /// Misc. @@ -157,15 +195,29 @@ private: /// Implicit constructor predicate. /// Explicit constructors are handled as functions. - std::unique_ptr m_constructorPredicate; + std::unique_ptr m_implicitConstructorPredicate; + + /// Constructor summary predicate, exists after the constructor + /// (implicit or explicit) and before the interface. + std::unique_ptr m_constructorSummaryPredicate; /// Artificial Interface predicate. /// Single entry block for all functions. - std::unique_ptr m_interfacePredicate; + std::map> m_interfaces; /// Artificial Error predicate. /// Single error block for all assertions. std::unique_ptr m_errorPredicate; + + /// Function predicates. + std::map>> m_summaries; + + smt::SymbolicIntVariable m_error{ + TypeProvider::uint256(), + TypeProvider::uint256(), + "error", + m_context + }; //@} /// Variables. @@ -180,7 +232,12 @@ private: /// Verification targets. //@{ - std::vector m_verificationTargets; + struct CHCVerificationTarget: VerificationTarget + { + smt::Expression errorId; + }; + + std::map m_verificationTargets; /// Assertions proven safe. std::set m_safeAssertions; @@ -190,6 +247,10 @@ private: //@{ FunctionDefinition const* m_currentFunction = nullptr; + std::map, IdCompare> m_callGraph; + + std::map, IdCompare> m_functionAssertions; + /// The current block. smt::Expression m_currentBlock = smt::Expression(true); diff --git a/libsolidity/formal/EncodingContext.h b/libsolidity/formal/EncodingContext.h index 9b75065f6..648f7dc33 100644 --- a/libsolidity/formal/EncodingContext.h +++ b/libsolidity/formal/EncodingContext.h @@ -140,6 +140,7 @@ public: void pushSolver(); void popSolver(); void addAssertion(Expression const& _e); + unsigned solverStackHeigh() { return m_assertions.size(); } const SolverInterface* solver() { solAssert(m_solver, ""); diff --git a/libsolidity/formal/ModelChecker.cpp b/libsolidity/formal/ModelChecker.cpp index c69849a06..ebc1a8751 100644 --- a/libsolidity/formal/ModelChecker.cpp +++ b/libsolidity/formal/ModelChecker.cpp @@ -29,9 +29,9 @@ ModelChecker::ModelChecker( ReadCallback::Callback const& _smtCallback, smt::SMTSolverChoice _enabledSolvers ): + m_context(), m_bmc(m_context, _errorReporter, _smtlib2Responses, _smtCallback, _enabledSolvers), - m_chc(m_context, _errorReporter, _smtlib2Responses, _smtCallback, _enabledSolvers), - m_context() + m_chc(m_context, _errorReporter, _smtlib2Responses, _smtCallback, _enabledSolvers) { } diff --git a/libsolidity/formal/ModelChecker.h b/libsolidity/formal/ModelChecker.h index ac19ba270..546f2df86 100644 --- a/libsolidity/formal/ModelChecker.h +++ b/libsolidity/formal/ModelChecker.h @@ -62,14 +62,14 @@ public: static smt::SMTSolverChoice availableSolvers(); private: + /// Stores the context of the encoding. + smt::EncodingContext m_context; + /// Bounded Model Checker engine. BMC m_bmc; /// Constrained Horn Clauses engine. CHC m_chc; - - /// Stores the context of the encoding. - smt::EncodingContext m_context; }; } diff --git a/libsolidity/formal/Z3Interface.h b/libsolidity/formal/Z3Interface.h index a6247a541..3791af4c5 100644 --- a/libsolidity/formal/Z3Interface.h +++ b/libsolidity/formal/Z3Interface.h @@ -58,11 +58,11 @@ private: z3::sort z3Sort(smt::Sort const& _sort); z3::sort_vector z3Sort(std::vector const& _sorts); - std::map m_constants; - std::map m_functions; - z3::context m_context; z3::solver m_solver; + + std::map m_constants; + std::map m_functions; }; } diff --git a/test/libsolidity/smtCheckerTests/functions/library_constant.sol b/test/libsolidity/smtCheckerTests/functions/library_constant.sol new file mode 100644 index 000000000..e32b8565b --- /dev/null +++ b/test/libsolidity/smtCheckerTests/functions/library_constant.sol @@ -0,0 +1,26 @@ +pragma experimental SMTChecker; + +library l1 { + + uint private constant TON = 1000; + function f1() public pure { + assert(TON == 1000); + assert(TON == 2000); + } + function f2(uint x, uint y) internal pure returns (uint) { + return x + y; + } +} + +contract C { + function f(uint x) public pure { + uint z = l1.f2(x, 1); + assert(z == x + 1); + } +} +// ---- +// Warning: (136-155): Assertion violation happens here +// Warning: (229-234): Overflow (resulting value larger than 2**256 - 1) happens here +// Warning: (300-302): Assertion checker does not yet implement type type(library l1) +// Warning: (229-234): Overflow (resulting value larger than 2**256 - 1) happens here +// Warning: (327-332): Overflow (resulting value larger than 2**256 - 1) happens here diff --git a/test/libsolidity/smtCheckerTests/functions/library_constant_2.sol b/test/libsolidity/smtCheckerTests/functions/library_constant_2.sol new file mode 100644 index 000000000..9c04e6f9a --- /dev/null +++ b/test/libsolidity/smtCheckerTests/functions/library_constant_2.sol @@ -0,0 +1,9 @@ +pragma experimental SMTChecker; + +library l1 { + + uint private constant TON = 1000; + function f1() public pure { + assert(TON == 1000); + } +} diff --git a/test/libsolidity/smtCheckerTests/invariants/loop_nested.sol b/test/libsolidity/smtCheckerTests/invariants/loop_nested.sol index 867403b61..1fb380649 100644 --- a/test/libsolidity/smtCheckerTests/invariants/loop_nested.sol +++ b/test/libsolidity/smtCheckerTests/invariants/loop_nested.sol @@ -17,3 +17,8 @@ contract Simple { } // ==== // SMTSolvers: z3 +// ---- +// Warning: (172-187): Error trying to invoke SMT solver. +// Warning: (195-209): Error trying to invoke SMT solver. +// Warning: (172-187): Assertion violation happens here +// Warning: (195-209): Assertion violation happens here diff --git a/test/libsolidity/smtCheckerTests/invariants/loop_nested_for.sol b/test/libsolidity/smtCheckerTests/invariants/loop_nested_for.sol index 9cfd4569f..e7fde4d5a 100644 --- a/test/libsolidity/smtCheckerTests/invariants/loop_nested_for.sol +++ b/test/libsolidity/smtCheckerTests/invariants/loop_nested_for.sol @@ -14,3 +14,8 @@ contract Simple { } // ==== // SMTSolvers: z3 +// ---- +// Warning: (164-179): Error trying to invoke SMT solver. +// Warning: (187-201): Error trying to invoke SMT solver. +// Warning: (164-179): Assertion violation happens here +// Warning: (187-201): Assertion violation happens here diff --git a/test/libsolidity/smtCheckerTests/loops/for_loop_array_assignment_storage_memory.sol b/test/libsolidity/smtCheckerTests/loops/for_loop_array_assignment_storage_memory.sol index 9e7a83866..0b301505b 100644 --- a/test/libsolidity/smtCheckerTests/loops/for_loop_array_assignment_storage_memory.sol +++ b/test/libsolidity/smtCheckerTests/loops/for_loop_array_assignment_storage_memory.sol @@ -12,12 +12,12 @@ contract LoopFor2 { b[i] = i + 1; c[i] = b[i]; } + // This is safe but too hard to solve currently. assert(b[0] == c[0]); assert(a[0] == 900); assert(b[0] == 900); } } -// ==== -// SMTSolvers: z3 // ---- -// Warning: (312-331): Assertion violation happens here +// Warning: (316-336): Assertion violation happens here +// Warning: (363-382): Assertion violation happens here diff --git a/test/libsolidity/smtCheckerTests/loops/for_loop_array_assignment_storage_storage.sol b/test/libsolidity/smtCheckerTests/loops/for_loop_array_assignment_storage_storage.sol index 8146a7f19..333273781 100644 --- a/test/libsolidity/smtCheckerTests/loops/for_loop_array_assignment_storage_storage.sol +++ b/test/libsolidity/smtCheckerTests/loops/for_loop_array_assignment_storage_storage.sol @@ -12,13 +12,13 @@ contract LoopFor2 { b[i] = i + 1; c[i] = b[i]; } + // This is safe but too hard to prove currently. assert(b[0] == c[0]); assert(a[0] == 900); assert(b[0] == 900); } } -// ==== -// SMTSolvers: z3 // ---- -// Warning: (290-309): Assertion violation happens here -// Warning: (313-332): Assertion violation happens here +// Warning: (317-337): Assertion violation happens here +// Warning: (341-360): Assertion violation happens here +// Warning: (364-383): Assertion violation happens here diff --git a/test/libsolidity/smtCheckerTests/operators/delete_array_index_2d.sol b/test/libsolidity/smtCheckerTests/operators/delete_array_index_2d.sol index 06ef66be9..9cd7de9e6 100644 --- a/test/libsolidity/smtCheckerTests/operators/delete_array_index_2d.sol +++ b/test/libsolidity/smtCheckerTests/operators/delete_array_index_2d.sol @@ -13,5 +13,5 @@ contract C assert(a[1][1] == 0); } } -// ==== -// SMTSolvers: z3 +// ---- +// Warning: (184-204): Assertion violation happens here From 24d6e6295e75ee9da587ced7471df9e89713b21b Mon Sep 17 00:00:00 2001 From: Daniel Kirchner Date: Mon, 2 Mar 2020 17:20:49 +0100 Subject: [PATCH 69/92] Reuse the mechanism for abi functions and move tracking of used functions to CompilerContext --- libsolidity/codegen/ABIFunctions.cpp | 11 +++-------- libsolidity/codegen/ABIFunctions.h | 6 ------ libsolidity/codegen/CompilerContext.cpp | 9 ++++++++- libsolidity/codegen/CompilerContext.h | 11 +++++++---- libsolidity/codegen/CompilerUtils.cpp | 16 +++------------- .../codegen/MultiUseYulFunctionCollector.cpp | 12 ++---------- .../codegen/MultiUseYulFunctionCollector.h | 19 ++----------------- libsolidity/codegen/ir/IRGenerator.cpp | 6 +++--- 8 files changed, 28 insertions(+), 62 deletions(-) diff --git a/libsolidity/codegen/ABIFunctions.cpp b/libsolidity/codegen/ABIFunctions.cpp index b69f47346..b6d690898 100644 --- a/libsolidity/codegen/ABIFunctions.cpp +++ b/libsolidity/codegen/ABIFunctions.cpp @@ -55,7 +55,7 @@ string ABIFunctions::tupleEncoder( functionName += t->identifier() + "_"; functionName += options.toFunctionNameSuffix(); - return createExternallyUsedFunction(functionName, [&]() { + return createFunction(functionName, [&]() { // Note that the values are in reverse due to the difference in calling semantics. Whiskers templ(R"( function (headStart ) -> tail { @@ -121,7 +121,7 @@ string ABIFunctions::tupleEncoderPacked( functionName += t->identifier() + "_"; functionName += options.toFunctionNameSuffix(); - return createExternallyUsedFunction(functionName, [&]() { + return createFunction(functionName, [&]() { solAssert(!_givenTypes.empty(), ""); // Note that the values are in reverse due to the difference in calling semantics. @@ -173,7 +173,7 @@ string ABIFunctions::tupleDecoder(TypePointers const& _types, bool _fromMemory) if (_fromMemory) functionName += "_fromMemory"; - return createExternallyUsedFunction(functionName, [&]() { + return createFunction(functionName, [&]() { TypePointers decodingTypes; for (auto const& t: _types) decodingTypes.emplace_back(t->decodingType()); @@ -1495,11 +1495,6 @@ string ABIFunctions::createFunction(string const& _name, function con return m_functionCollector->createFunction(_name, _creator); } -string ABIFunctions::createExternallyUsedFunction(string const& _name, function const& _creator) -{ - return m_functionCollector->createExternallyUsedFunction(_name, _creator); -} - size_t ABIFunctions::headSize(TypePointers const& _targetTypes) { size_t headSize = 0; diff --git a/libsolidity/codegen/ABIFunctions.h b/libsolidity/codegen/ABIFunctions.h index 74bfdf7fa..1099e3593 100644 --- a/libsolidity/codegen/ABIFunctions.h +++ b/libsolidity/codegen/ABIFunctions.h @@ -31,7 +31,6 @@ #include #include -#include #include namespace solidity::frontend @@ -233,11 +232,6 @@ private: /// cases. std::string createFunction(std::string const& _name, std::function const& _creator); - /// Helper function that uses @a _creator to create a function and add it to - /// @a m_requestedFunctions if it has not been created yet and returns @a _name in both - /// cases. Also adds it to the list of externally used functions. - std::string createExternallyUsedFunction(std::string const& _name, std::function const& _creator); - /// @returns the size of the static part of the encoding of the given types. static size_t headSize(TypePointers const& _targetTypes); diff --git a/libsolidity/codegen/CompilerContext.cpp b/libsolidity/codegen/CompilerContext.cpp index f65ccae52..ee12b9b3e 100644 --- a/libsolidity/codegen/CompilerContext.cpp +++ b/libsolidity/codegen/CompilerContext.cpp @@ -100,7 +100,7 @@ void CompilerContext::callYulUtilFunction( unsigned _outArgs ) { - m_functionCollector->markAsExternallyUsed(_name); + m_externallyUsedFunctions.insert(_name); auto retTag = pushNewTag(); CompilerUtils(*this).moveIntoStack(_inArgs); appendJumpTo(namedTag(_name)); @@ -147,6 +147,13 @@ void CompilerContext::appendMissingLowLevelFunctions() } } +pair> CompilerContext::requestedYulFunctions() +{ + set empty; + swap(empty, m_externallyUsedFunctions); + return make_pair(m_functionCollector->requestedFunctions(), std::move(empty)); +} + void CompilerContext::addVariable( VariableDeclaration const& _declaration, unsigned _offsetToCurrent diff --git a/libsolidity/codegen/CompilerContext.h b/libsolidity/codegen/CompilerContext.h index d9393e23a..6d554da82 100644 --- a/libsolidity/codegen/CompilerContext.h +++ b/libsolidity/codegen/CompilerContext.h @@ -154,10 +154,11 @@ public: void appendMissingLowLevelFunctions(); ABIFunctions& abiFunctions() { return m_abiFunctions; } YulUtilFunctions& utilFunctions() { return m_yulUtilFunctions; } - std::pair> requestedYulFunctions() - { - return m_functionCollector->requestedFunctions(); - } + /// @returns concatenation of all generated functions and a set of the + /// externally used functions. + /// Clears the internal list, i.e. calling it again will result in an + /// empty return value. + std::pair> requestedYulFunctions(); ModifierDefinition const& resolveVirtualFunctionModifier(ModifierDefinition const& _modifier) const; /// Returns the distance of the given local variable from the bottom of the stack (of the current function). @@ -371,6 +372,8 @@ private: std::map m_lowLevelFunctions; // Collector for yul functions. std::shared_ptr m_functionCollector = std::make_shared(); + /// Set of externally used yul functions. + std::set m_externallyUsedFunctions; /// Container for ABI functions to be generated. ABIFunctions m_abiFunctions; /// Container for Yul Util functions to be generated. diff --git a/libsolidity/codegen/CompilerUtils.cpp b/libsolidity/codegen/CompilerUtils.cpp index 4c75081ff..a0d38eae5 100644 --- a/libsolidity/codegen/CompilerUtils.cpp +++ b/libsolidity/codegen/CompilerUtils.cpp @@ -595,31 +595,21 @@ void CompilerUtils::abiEncodeV2( // stack: <$value0> <$value1> ... <$value(n-1)> <$headStart> - auto ret = m_context.pushNewTag(); - moveIntoStack(sizeOnStack(_givenTypes) + 1); - string encoderName = _padToWordBoundaries ? m_context.abiFunctions().tupleEncoder(_givenTypes, _targetTypes, _encodeAsLibraryTypes) : m_context.abiFunctions().tupleEncoderPacked(_givenTypes, _targetTypes); - m_context.appendJumpTo(m_context.namedTag(encoderName)); - m_context.adjustStackOffset(-int(sizeOnStack(_givenTypes)) - 1); - m_context << ret.tag(); + m_context.callYulUtilFunction(encoderName, sizeOnStack(_givenTypes) + 1, 1); } void CompilerUtils::abiDecodeV2(TypePointers const& _parameterTypes, bool _fromMemory) { // stack: [stack top] - auto ret = m_context.pushNewTag(); - moveIntoStack(2); - // stack: [stack top] m_context << Instruction::DUP2 << Instruction::ADD; m_context << Instruction::SWAP1; - // stack: + // stack: string decoderName = m_context.abiFunctions().tupleDecoder(_parameterTypes, _fromMemory); - m_context.appendJumpTo(m_context.namedTag(decoderName)); - m_context.adjustStackOffset(int(sizeOnStack(_parameterTypes)) - 3); - m_context << ret.tag(); + m_context.callYulUtilFunction(decoderName, 2, sizeOnStack(_parameterTypes)); } void CompilerUtils::zeroInitialiseMemoryArray(ArrayType const& _type) diff --git a/libsolidity/codegen/MultiUseYulFunctionCollector.cpp b/libsolidity/codegen/MultiUseYulFunctionCollector.cpp index 8af292eda..2be3d29ae 100644 --- a/libsolidity/codegen/MultiUseYulFunctionCollector.cpp +++ b/libsolidity/codegen/MultiUseYulFunctionCollector.cpp @@ -30,15 +30,13 @@ using namespace std; using namespace solidity; using namespace solidity::frontend; -pair> MultiUseYulFunctionCollector::requestedFunctions() +string MultiUseYulFunctionCollector::requestedFunctions() { string result; for (auto const& f: m_requestedFunctions) result += f.second; m_requestedFunctions.clear(); - std::set empty; - swap(empty, m_externallyUsedFunctions); - return make_pair(result, std::move(empty)); + return result; } string MultiUseYulFunctionCollector::createFunction(string const& _name, function const& _creator) @@ -52,9 +50,3 @@ string MultiUseYulFunctionCollector::createFunction(string const& _name, functio } return _name; } - -string MultiUseYulFunctionCollector::createExternallyUsedFunction(string const& _name, function const& _creator) -{ - m_externallyUsedFunctions.insert(_name); - return createFunction(_name, _creator); -} diff --git a/libsolidity/codegen/MultiUseYulFunctionCollector.h b/libsolidity/codegen/MultiUseYulFunctionCollector.h index a1c733274..d839a31be 100644 --- a/libsolidity/codegen/MultiUseYulFunctionCollector.h +++ b/libsolidity/codegen/MultiUseYulFunctionCollector.h @@ -23,7 +23,6 @@ #include #include -#include #include namespace solidity::frontend @@ -41,28 +40,14 @@ public: /// cases. std::string createFunction(std::string const& _name, std::function const& _creator); - /// Helper function that uses @a _creator to create a function and add it to - /// @a m_requestedFunctions if it has not been created yet and returns @a _name in both - /// cases. - std::string createExternallyUsedFunction(std::string const& _name, std::function const& _creator); - - /// Manually mark a function as externally used. - void markAsExternallyUsed(std::string const& _name) - { - m_externallyUsedFunctions.insert(_name); - } - - /// @returns concatenation of all generated functions and a set of the - /// externally used functions. + /// @returns concatenation of all generated functions. /// Clears the internal list, i.e. calling it again will result in an /// empty return value. - std::pair> requestedFunctions(); + std::string requestedFunctions(); private: /// Map from function name to code for a multi-use function. std::map m_requestedFunctions; - // Set of externally used functions. - std::set m_externallyUsedFunctions; }; } diff --git a/libsolidity/codegen/ir/IRGenerator.cpp b/libsolidity/codegen/ir/IRGenerator.cpp index 0f2ffaf35..9c2d8807c 100644 --- a/libsolidity/codegen/ir/IRGenerator.cpp +++ b/libsolidity/codegen/ir/IRGenerator.cpp @@ -107,7 +107,7 @@ string IRGenerator::generate(ContractDefinition const& _contract) for (auto const* contract: _contract.annotation().linearizedBaseContracts) for (auto const* fun: contract->definedFunctions()) generateFunction(*fun); - t("functions", m_context.functionCollector()->requestedFunctions().first); + t("functions", m_context.functionCollector()->requestedFunctions()); resetContext(_contract); m_context.setInheritanceHierarchy(_contract.annotation().linearizedBaseContracts); @@ -116,7 +116,7 @@ string IRGenerator::generate(ContractDefinition const& _contract) for (auto const* contract: _contract.annotation().linearizedBaseContracts) for (auto const* fun: contract->definedFunctions()) generateFunction(*fun); - t("runtimeFunctions", m_context.functionCollector()->requestedFunctions().first); + t("runtimeFunctions", m_context.functionCollector()->requestedFunctions()); return t.render(); } @@ -383,7 +383,7 @@ string IRGenerator::memoryInit() void IRGenerator::resetContext(ContractDefinition const& _contract) { solAssert( - m_context.functionCollector()->requestedFunctions().first.empty(), + m_context.functionCollector()->requestedFunctions().empty(), "Reset context while it still had functions." ); m_context = IRGenerationContext(m_evmVersion, m_context.revertStrings(), m_optimiserSettings); From 561e5d9b27439e56cbd136abecfde5727594155b Mon Sep 17 00:00:00 2001 From: Daniel Kirchner Date: Mon, 2 Mar 2020 17:23:58 +0100 Subject: [PATCH 70/92] Rename variables and review suggestion. --- libsolidity/codegen/CompilerContext.cpp | 10 +++++----- libsolidity/codegen/CompilerContext.h | 14 +++++++------- libsolidity/codegen/CompilerUtils.cpp | 4 ++-- libsolidity/codegen/ContractCompiler.cpp | 8 ++++---- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/libsolidity/codegen/CompilerContext.cpp b/libsolidity/codegen/CompilerContext.cpp index ee12b9b3e..1d1eb92b9 100644 --- a/libsolidity/codegen/CompilerContext.cpp +++ b/libsolidity/codegen/CompilerContext.cpp @@ -94,14 +94,14 @@ void CompilerContext::callLowLevelFunction( *this << retTag.tag(); } -void CompilerContext::callYulUtilFunction( +void CompilerContext::callYulFunction( string const& _name, unsigned _inArgs, unsigned _outArgs ) { - m_externallyUsedFunctions.insert(_name); - auto retTag = pushNewTag(); + m_externallyUsedYulFunctions.insert(_name); + auto const retTag = pushNewTag(); CompilerUtils(*this).moveIntoStack(_inArgs); appendJumpTo(namedTag(_name)); adjustStackOffset(int(_outArgs) - 1 - _inArgs); @@ -150,8 +150,8 @@ void CompilerContext::appendMissingLowLevelFunctions() pair> CompilerContext::requestedYulFunctions() { set empty; - swap(empty, m_externallyUsedFunctions); - return make_pair(m_functionCollector->requestedFunctions(), std::move(empty)); + swap(empty, m_externallyUsedYulFunctions); + return make_pair(m_yulFunctionCollector->requestedFunctions(), std::move(empty)); } void CompilerContext::addVariable( diff --git a/libsolidity/codegen/CompilerContext.h b/libsolidity/codegen/CompilerContext.h index 6d554da82..76c72770d 100644 --- a/libsolidity/codegen/CompilerContext.h +++ b/libsolidity/codegen/CompilerContext.h @@ -65,8 +65,8 @@ public: m_evmVersion(_evmVersion), m_revertStrings(_revertStrings), m_runtimeContext(_runtimeContext), - m_abiFunctions(m_evmVersion, m_revertStrings, m_functionCollector), - m_yulUtilFunctions(m_evmVersion, m_revertStrings, m_functionCollector) + m_abiFunctions(m_evmVersion, m_revertStrings, m_yulFunctionCollector), + m_yulUtilFunctions(m_evmVersion, m_revertStrings, m_yulFunctionCollector) { if (m_runtimeContext) m_runtimeSub = size_t(m_asm->newSub(m_runtimeContext->m_asm).data()); @@ -133,8 +133,8 @@ public: std::function const& _generator ); - /// Appends a call to a yul util function and registers the function as externally used. - void callYulUtilFunction( + /// Appends a call to a yul function and registers the function as externally used. + void callYulFunction( std::string const& _name, unsigned _inArgs, unsigned _outArgs @@ -370,10 +370,10 @@ private: size_t m_runtimeSub = -1; /// An index of low-level function labels by name. std::map m_lowLevelFunctions; - // Collector for yul functions. - std::shared_ptr m_functionCollector = std::make_shared(); + /// Collector for yul functions. + std::shared_ptr m_yulFunctionCollector = std::make_shared(); /// Set of externally used yul functions. - std::set m_externallyUsedFunctions; + std::set m_externallyUsedYulFunctions; /// Container for ABI functions to be generated. ABIFunctions m_abiFunctions; /// Container for Yul Util functions to be generated. diff --git a/libsolidity/codegen/CompilerUtils.cpp b/libsolidity/codegen/CompilerUtils.cpp index a0d38eae5..b39633377 100644 --- a/libsolidity/codegen/CompilerUtils.cpp +++ b/libsolidity/codegen/CompilerUtils.cpp @@ -599,7 +599,7 @@ void CompilerUtils::abiEncodeV2( _padToWordBoundaries ? m_context.abiFunctions().tupleEncoder(_givenTypes, _targetTypes, _encodeAsLibraryTypes) : m_context.abiFunctions().tupleEncoderPacked(_givenTypes, _targetTypes); - m_context.callYulUtilFunction(encoderName, sizeOnStack(_givenTypes) + 1, 1); + m_context.callYulFunction(encoderName, sizeOnStack(_givenTypes) + 1, 1); } void CompilerUtils::abiDecodeV2(TypePointers const& _parameterTypes, bool _fromMemory) @@ -609,7 +609,7 @@ void CompilerUtils::abiDecodeV2(TypePointers const& _parameterTypes, bool _fromM m_context << Instruction::SWAP1; // stack: string decoderName = m_context.abiFunctions().tupleDecoder(_parameterTypes, _fromMemory); - m_context.callYulUtilFunction(decoderName, 2, sizeOnStack(_parameterTypes)); + m_context.callYulFunction(decoderName, 2, sizeOnStack(_parameterTypes)); } void CompilerUtils::zeroInitialiseMemoryArray(ArrayType const& _type) diff --git a/libsolidity/codegen/ContractCompiler.cpp b/libsolidity/codegen/ContractCompiler.cpp index 47187b1c8..8b9061af3 100644 --- a/libsolidity/codegen/ContractCompiler.cpp +++ b/libsolidity/codegen/ContractCompiler.cpp @@ -1267,12 +1267,12 @@ void ContractCompiler::appendMissingFunctions() solAssert(m_context.nextFunctionToCompile() != function, "Compiled the wrong function?"); } m_context.appendMissingLowLevelFunctions(); - auto yulFunctions = m_context.requestedYulFunctions(); - if (!yulFunctions.first.empty()) + auto [yulFunctions, externallyUsedYulFunctions] = m_context.requestedYulFunctions(); + if (!yulFunctions.empty()) m_context.appendInlineAssembly( - "{" + move(yulFunctions.first) + "}", + "{" + move(yulFunctions) + "}", {}, - yulFunctions.second, + externallyUsedYulFunctions, true, m_optimiserSettings ); From 495abee7693810b012032741e0723c55e8a26024 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Tue, 3 Mar 2020 12:21:16 +0100 Subject: [PATCH 71/92] [test] Fixes ExecutionFramework providing different contract addresses when running test cases for both, old and new yul codegen. --- test/ExecutionFramework.cpp | 8 ++++++-- test/ExecutionFramework.h | 2 ++ test/libsolidity/SemanticTest.cpp | 1 + 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/test/ExecutionFramework.cpp b/test/ExecutionFramework.cpp index cc54e7a7b..170f5638c 100644 --- a/test/ExecutionFramework.cpp +++ b/test/ExecutionFramework.cpp @@ -53,12 +53,16 @@ ExecutionFramework::ExecutionFramework(langutil::EVMVersion _evmVersion): m_optimiserSettings = solidity::frontend::OptimiserSettings::full(); else if (solidity::test::CommonOptions::get().optimize) m_optimiserSettings = solidity::frontend::OptimiserSettings::standard(); - m_evmHost->reset(); + reset(); +} + +void ExecutionFramework::reset() +{ + m_evmHost->reset(); for (size_t i = 0; i < 10; i++) m_evmHost->accounts[EVMHost::convertToEVMC(account(i))].balance = EVMHost::convertToEVMC(u256(1) << 100); - } std::pair ExecutionFramework::compareAndCreateMessage( diff --git a/test/ExecutionFramework.h b/test/ExecutionFramework.h index 6a3ffc0fb..e55a12aa6 100644 --- a/test/ExecutionFramework.h +++ b/test/ExecutionFramework.h @@ -251,6 +251,8 @@ private: } protected: + void reset(); + void sendMessage(bytes const& _data, bool _isCreation, u256 const& _value = 0); void sendEther(Address const& _to, u256 const& _value); size_t currentTimestamp(); diff --git a/test/libsolidity/SemanticTest.cpp b/test/libsolidity/SemanticTest.cpp index b1ef9372a..b2c4628da 100644 --- a/test/libsolidity/SemanticTest.cpp +++ b/test/libsolidity/SemanticTest.cpp @@ -101,6 +101,7 @@ TestCase::TestResult SemanticTest::run(ostream& _stream, string const& _linePref { for(bool compileViaYul: set{!m_runWithoutYul, m_runWithYul}) { + reset(); bool success = true; m_compileViaYul = compileViaYul; From 58c6b9070530d91ca1c7851a55b106dba3a8be99 Mon Sep 17 00:00:00 2001 From: Djordje Mijovic Date: Tue, 3 Mar 2020 12:15:59 +0100 Subject: [PATCH 72/92] Deprecated warning for .value() and .gas() on function and constructror calls --- Changelog.md | 1 + docs/types/value-types.rst | 2 +- libsolidity/analysis/TypeChecker.cpp | 18 +++++++++++++++++- .../semanticTests/tryCatch/assert.sol | 2 +- .../semanticTests/tryCatch/trivial.sol | 2 +- .../types/function_type_members.sol | 3 +++ .../constructor/constructor_payable.sol | 10 ++++++++++ .../functionCalls/calloptions_repeated.sol | 1 + .../new_with_invalid_calloptions.sol | 8 ++++---- .../functionTypes/call_gas_on_function.sol | 7 +++++++ .../call_value_library_function.sol | 10 ++++++++++ ...call_value_on_non_payable_function_type.sol | 2 +- .../call_value_on_payable_function_type.sol | 2 +- ...ue_options_on_non_payable_function_type.sol | 8 ++++++++ .../warn_deprecate_gas_function.sol | 8 ++++++++ .../warn_deprecate_value_constructor.sol | 11 +++++++++++ .../warn_deprecate_value_function.sol | 8 ++++++++ .../348_unused_return_value_call_value.sol | 3 +++ .../361_calling_payable.sol | 9 +++++++-- .../362_calling_nonpayable.sol | 6 ++++-- ...not_warn_msg_value_in_internal_function.sol | 1 + ...unspecified_encoding_internal_functions.sol | 1 + .../viewPureChecker/gas_value_without_call.sol | 13 +++++++++++++ .../gas_with_call_nonpayable.sol | 15 +++++++++++++-- .../viewPureChecker/staticcall_gas_view.sol | 4 ++++ .../value_with_call_nonpayable.sol | 13 +++++++++++-- 26 files changed, 150 insertions(+), 18 deletions(-) create mode 100644 test/libsolidity/syntaxTests/constructor/constructor_payable.sol create mode 100644 test/libsolidity/syntaxTests/functionTypes/call_gas_on_function.sol create mode 100644 test/libsolidity/syntaxTests/functionTypes/call_value_library_function.sol create mode 100644 test/libsolidity/syntaxTests/functionTypes/call_value_options_on_non_payable_function_type.sol create mode 100644 test/libsolidity/syntaxTests/functionTypes/warn_deprecate_gas_function.sol create mode 100644 test/libsolidity/syntaxTests/functionTypes/warn_deprecate_value_constructor.sol create mode 100644 test/libsolidity/syntaxTests/functionTypes/warn_deprecate_value_function.sol diff --git a/Changelog.md b/Changelog.md index f151365ae..9a84d3012 100644 --- a/Changelog.md +++ b/Changelog.md @@ -2,6 +2,7 @@ Language Features: * Inline Assembly: Allow assigning to `_slot` of local storage variable pointers. + * General: Deprecated `value(...)` and `gas(...)` in favor of `{value: ...}` and `{gas: ...}` Compiler Features: diff --git a/docs/types/value-types.rst b/docs/types/value-types.rst index 9dae37912..8984a5b8e 100644 --- a/docs/types/value-types.rst +++ b/docs/types/value-types.rst @@ -646,7 +646,7 @@ External (or public) functions have the following members: Example that shows how to use the members:: pragma solidity >=0.4.16 <0.7.0; - + // This will report a warning contract Example { function f() public payable returns (bytes4) { diff --git a/libsolidity/analysis/TypeChecker.cpp b/libsolidity/analysis/TypeChecker.cpp index 2f9526f87..f713ea289 100644 --- a/libsolidity/analysis/TypeChecker.cpp +++ b/libsolidity/analysis/TypeChecker.cpp @@ -2312,7 +2312,11 @@ bool TypeChecker::visit(FunctionCallOptions const& _functionCallOptions) else if (!expressionFunctionType->isPayable()) m_errorReporter.typeError( _functionCallOptions.location(), - "Cannot set option \"value\" on a non-payable function type." + kind == FunctionType::Kind::Creation ? + "Cannot set option \"value\", since the constructor of " + + expressionFunctionType->returnParameterTypes().front()->toString() + + " is not payable." : + "Cannot set option \"value\" on a non-payable function type." ); else { @@ -2522,12 +2526,24 @@ bool TypeChecker::visit(MemberAccess const& _memberAccess) annotation.type = possibleMembers.front().type; if (auto funType = dynamic_cast(annotation.type)) + { solAssert( !funType->bound() || exprType->isImplicitlyConvertibleTo(*funType->selfType()), "Function \"" + memberName + "\" cannot be called on an object of type " + exprType->toString() + " (expected " + funType->selfType()->toString() + ")." ); + if ( + dynamic_cast(exprType) && + !annotation.referencedDeclaration && + (memberName == "value" || memberName == "gas") + ) + m_errorReporter.warning( + _memberAccess.location(), + "Using \"." + memberName + "(...)\" is deprecated. Use \"{" + memberName + ": ...}\" instead." + ); + } + if (auto const* structType = dynamic_cast(exprType)) annotation.isLValue = !structType->dataStoredIn(DataLocation::CallData); else if (exprType->category() == Type::Category::Array) diff --git a/test/libsolidity/semanticTests/tryCatch/assert.sol b/test/libsolidity/semanticTests/tryCatch/assert.sol index 367ab287b..8b6a7b996 100644 --- a/test/libsolidity/semanticTests/tryCatch/assert.sol +++ b/test/libsolidity/semanticTests/tryCatch/assert.sol @@ -4,7 +4,7 @@ contract C { } function f(bool x) public returns (uint) { // Set the gas to make this work on pre-byzantium VMs - try this.g.gas(8000)(x) { + try this.g{gas: 8000}(x) { return 1; } catch { return 2; diff --git a/test/libsolidity/semanticTests/tryCatch/trivial.sol b/test/libsolidity/semanticTests/tryCatch/trivial.sol index e2f8739bd..d43477e99 100644 --- a/test/libsolidity/semanticTests/tryCatch/trivial.sol +++ b/test/libsolidity/semanticTests/tryCatch/trivial.sol @@ -4,7 +4,7 @@ contract C { } function f(bool x) public returns (uint) { // Set the gas to make this work on pre-byzantium VMs - try this.g.gas(8000)(x) { + try this.g{gas: 8000}(x) { return 1; } catch { return 2; diff --git a/test/libsolidity/smtCheckerTests/types/function_type_members.sol b/test/libsolidity/smtCheckerTests/types/function_type_members.sol index 2296fa798..923767dd7 100644 --- a/test/libsolidity/smtCheckerTests/types/function_type_members.sol +++ b/test/libsolidity/smtCheckerTests/types/function_type_members.sol @@ -3,9 +3,12 @@ contract C { function f(function(uint) external payable g) internal { g.selector; g.gas(2).value(3)(4); + g{gas: 2, value: 3}(4); } } // ---- +// Warning: (122-127): Using ".gas(...)" is deprecated. Use "{gas: ...}" instead. +// Warning: (122-136): Using ".value(...)" is deprecated. Use "{value: ...}" instead. // Warning: (108-118): Assertion checker does not yet support this expression. // Warning: (122-130): Assertion checker does not yet implement this type of function call. // Warning: (122-139): Assertion checker does not yet implement this type of function call. diff --git a/test/libsolidity/syntaxTests/constructor/constructor_payable.sol b/test/libsolidity/syntaxTests/constructor/constructor_payable.sol new file mode 100644 index 000000000..e5c6ac28d --- /dev/null +++ b/test/libsolidity/syntaxTests/constructor/constructor_payable.sol @@ -0,0 +1,10 @@ +contract C { + constructor() public payable { } +} + +contract D { + function createC() public returns (C) { + C c = (new C){value: 1}(); + return c; + } +} diff --git a/test/libsolidity/syntaxTests/functionCalls/calloptions_repeated.sol b/test/libsolidity/syntaxTests/functionCalls/calloptions_repeated.sol index 2ffe4b3ed..181b325b0 100644 --- a/test/libsolidity/syntaxTests/functionCalls/calloptions_repeated.sol +++ b/test/libsolidity/syntaxTests/functionCalls/calloptions_repeated.sol @@ -15,6 +15,7 @@ contract C { // TypeError: (78-110): Option "gas" has already been set. // TypeError: (120-154): Option "gas" has already been set. // TypeError: (164-198): Option "value" has already been set. +// Warning: (208-222): Using ".value(...)" is deprecated. Use "{value: ...}" instead. // TypeError: (208-242): Option "value" has already been set. // TypeError: (252-293): Option "value" has already been set. // TypeError: (252-293): Option "gas" has already been set. diff --git a/test/libsolidity/syntaxTests/functionCalls/new_with_invalid_calloptions.sol b/test/libsolidity/syntaxTests/functionCalls/new_with_invalid_calloptions.sol index a8cabe0de..29798dde6 100644 --- a/test/libsolidity/syntaxTests/functionCalls/new_with_invalid_calloptions.sol +++ b/test/libsolidity/syntaxTests/functionCalls/new_with_invalid_calloptions.sol @@ -14,14 +14,14 @@ contract C { // ==== // EVMVersion: >=constantinople // ---- -// TypeError: (64-98): Cannot set option "value" on a non-payable function type. +// TypeError: (64-98): Cannot set option "value", since the constructor of contract D is not payable. // TypeError: (64-98): Function call option "gas" cannot be used with "new". // TypeError: (102-123): Unknown call option "slt". Valid options are "salt", "value" and "gas". -// TypeError: (102-123): Cannot set option "value" on a non-payable function type. +// TypeError: (102-123): Cannot set option "value", since the constructor of contract D is not payable. // TypeError: (127-139): Unknown call option "val". Valid options are "salt", "value" and "gas". // TypeError: (143-172): Duplicate option "salt". -// TypeError: (176-199): Cannot set option "value" on a non-payable function type. -// TypeError: (176-199): Cannot set option "value" on a non-payable function type. +// TypeError: (176-199): Cannot set option "value", since the constructor of contract D is not payable. +// TypeError: (176-199): Cannot set option "value", since the constructor of contract D is not payable. // TypeError: (203-220): Unknown call option "random". Valid options are "salt", "value" and "gas". // TypeError: (224-242): Unknown call option "what". Valid options are "salt", "value" and "gas". // TypeError: (246-259): Function call option "gas" cannot be used with "new". diff --git a/test/libsolidity/syntaxTests/functionTypes/call_gas_on_function.sol b/test/libsolidity/syntaxTests/functionTypes/call_gas_on_function.sol new file mode 100644 index 000000000..381fb6ea0 --- /dev/null +++ b/test/libsolidity/syntaxTests/functionTypes/call_gas_on_function.sol @@ -0,0 +1,7 @@ +contract C { + function (uint) external returns (uint) x; + function f() public { + x{gas: 2}(1); + } +} + diff --git a/test/libsolidity/syntaxTests/functionTypes/call_value_library_function.sol b/test/libsolidity/syntaxTests/functionTypes/call_value_library_function.sol new file mode 100644 index 000000000..97eebfd23 --- /dev/null +++ b/test/libsolidity/syntaxTests/functionTypes/call_value_library_function.sol @@ -0,0 +1,10 @@ +library L { + function value(function()internal a, uint256 b) internal {} +} +contract C { + using L for function()internal; + function f() public { + function()internal x; + x.value(42); + } +} diff --git a/test/libsolidity/syntaxTests/functionTypes/call_value_on_non_payable_function_type.sol b/test/libsolidity/syntaxTests/functionTypes/call_value_on_non_payable_function_type.sol index 5efdf2407..822d3eb28 100644 --- a/test/libsolidity/syntaxTests/functionTypes/call_value_on_non_payable_function_type.sol +++ b/test/libsolidity/syntaxTests/functionTypes/call_value_on_non_payable_function_type.sol @@ -1,7 +1,7 @@ contract C { function (uint) external returns (uint) x; function f() public { - x.value(2)(); + x.value(2)(1); } } // ---- diff --git a/test/libsolidity/syntaxTests/functionTypes/call_value_on_payable_function_type.sol b/test/libsolidity/syntaxTests/functionTypes/call_value_on_payable_function_type.sol index ca2a01964..1fd0e7188 100644 --- a/test/libsolidity/syntaxTests/functionTypes/call_value_on_payable_function_type.sol +++ b/test/libsolidity/syntaxTests/functionTypes/call_value_on_payable_function_type.sol @@ -1,6 +1,6 @@ contract C { function (uint) external payable returns (uint) x; function f() public { - x.value(2)(1); + x{value: 2}(1); } } diff --git a/test/libsolidity/syntaxTests/functionTypes/call_value_options_on_non_payable_function_type.sol b/test/libsolidity/syntaxTests/functionTypes/call_value_options_on_non_payable_function_type.sol new file mode 100644 index 000000000..30f36f219 --- /dev/null +++ b/test/libsolidity/syntaxTests/functionTypes/call_value_options_on_non_payable_function_type.sol @@ -0,0 +1,8 @@ +contract C { + function (uint) external returns (uint) x; + function g() public { + x{value: 2}(1); + } +} +// ---- +// TypeError: (94-105): Cannot set option "value" on a non-payable function type. diff --git a/test/libsolidity/syntaxTests/functionTypes/warn_deprecate_gas_function.sol b/test/libsolidity/syntaxTests/functionTypes/warn_deprecate_gas_function.sol new file mode 100644 index 000000000..36d42e349 --- /dev/null +++ b/test/libsolidity/syntaxTests/functionTypes/warn_deprecate_gas_function.sol @@ -0,0 +1,8 @@ +contract C { + function (uint) external payable returns (uint) x; + function f() public { + x.gas(2)(1); + } +} +// ---- +// Warning: (102-107): Using ".gas(...)" is deprecated. Use "{gas: ...}" instead. diff --git a/test/libsolidity/syntaxTests/functionTypes/warn_deprecate_value_constructor.sol b/test/libsolidity/syntaxTests/functionTypes/warn_deprecate_value_constructor.sol new file mode 100644 index 000000000..28a573c93 --- /dev/null +++ b/test/libsolidity/syntaxTests/functionTypes/warn_deprecate_value_constructor.sol @@ -0,0 +1,11 @@ +contract C { + constructor() payable public {} +} +contract D { + function createC() public returns (C) { + C c = (new C).value(2)(); + return c; + } +} +// ---- +// Warning: (122-135): Using ".value(...)" is deprecated. Use "{value: ...}" instead. diff --git a/test/libsolidity/syntaxTests/functionTypes/warn_deprecate_value_function.sol b/test/libsolidity/syntaxTests/functionTypes/warn_deprecate_value_function.sol new file mode 100644 index 000000000..854ea28fa --- /dev/null +++ b/test/libsolidity/syntaxTests/functionTypes/warn_deprecate_value_function.sol @@ -0,0 +1,8 @@ +contract C { + function (uint) external payable returns (uint) x; + function f() public { + x.value(2)(1); + } +} +// ---- +// Warning: (102-109): Using ".value(...)" is deprecated. Use "{value: ...}" instead. diff --git a/test/libsolidity/syntaxTests/nameAndTypeResolution/348_unused_return_value_call_value.sol b/test/libsolidity/syntaxTests/nameAndTypeResolution/348_unused_return_value_call_value.sol index 1ac7c6f33..dffa55fdf 100644 --- a/test/libsolidity/syntaxTests/nameAndTypeResolution/348_unused_return_value_call_value.sol +++ b/test/libsolidity/syntaxTests/nameAndTypeResolution/348_unused_return_value_call_value.sol @@ -1,7 +1,10 @@ contract test { function f() public { address(0x12).call.value(2)("abc"); + address(0x12).call{value: 2}("abc"); } } // ---- +// Warning: (50-74): Using ".value(...)" is deprecated. Use "{value: ...}" instead. // Warning: (50-84): Return value of low-level calls not used. +// Warning: (94-129): Return value of low-level calls not used. diff --git a/test/libsolidity/syntaxTests/nameAndTypeResolution/361_calling_payable.sol b/test/libsolidity/syntaxTests/nameAndTypeResolution/361_calling_payable.sol index 8ef4d5799..a3c5a5f30 100644 --- a/test/libsolidity/syntaxTests/nameAndTypeResolution/361_calling_payable.sol +++ b/test/libsolidity/syntaxTests/nameAndTypeResolution/361_calling_payable.sol @@ -1,6 +1,11 @@ contract receiver { function pay() payable public {} } contract test { - function f() public { (new receiver()).pay.value(10)(); } + function f() public { (new receiver()).pay{value: 10}(); } + function g() public { (new receiver()).pay.value(10)(); } receiver r = new receiver(); - function g() public { r.pay.value(10)(); } + function h() public { r.pay{value: 10}(); } + function i() public { r.pay.value(10)(); } } +// ---- +// Warning: (160-186): Using ".value(...)" is deprecated. Use "{value: ...}" instead. +// Warning: (303-314): Using ".value(...)" is deprecated. Use "{value: ...}" instead. diff --git a/test/libsolidity/syntaxTests/nameAndTypeResolution/362_calling_nonpayable.sol b/test/libsolidity/syntaxTests/nameAndTypeResolution/362_calling_nonpayable.sol index 1c04be755..6f5706446 100644 --- a/test/libsolidity/syntaxTests/nameAndTypeResolution/362_calling_nonpayable.sol +++ b/test/libsolidity/syntaxTests/nameAndTypeResolution/362_calling_nonpayable.sol @@ -1,6 +1,8 @@ contract receiver { function nopay() public {} } contract test { - function f() public { (new receiver()).nopay.value(10)(); } + function f() public { (new receiver()).nopay{value: 10}(); } + function g() public { (new receiver()).nopay.value(10)(); } } // ---- -// TypeError: (91-119): Member "value" is only available for payable functions. +// TypeError: (91-124): Cannot set option "value" on a non-payable function type. +// TypeError: (156-184): Member "value" is only available for payable functions. diff --git a/test/libsolidity/syntaxTests/nameAndTypeResolution/399_does_not_warn_msg_value_in_internal_function.sol b/test/libsolidity/syntaxTests/nameAndTypeResolution/399_does_not_warn_msg_value_in_internal_function.sol index 8492e691e..00d8ad4c8 100644 --- a/test/libsolidity/syntaxTests/nameAndTypeResolution/399_does_not_warn_msg_value_in_internal_function.sol +++ b/test/libsolidity/syntaxTests/nameAndTypeResolution/399_does_not_warn_msg_value_in_internal_function.sol @@ -3,3 +3,4 @@ contract C { msg.value; } } + diff --git a/test/libsolidity/syntaxTests/specialFunctions/types_with_unspecified_encoding_internal_functions.sol b/test/libsolidity/syntaxTests/specialFunctions/types_with_unspecified_encoding_internal_functions.sol index a30e428a6..ea11703f9 100644 --- a/test/libsolidity/syntaxTests/specialFunctions/types_with_unspecified_encoding_internal_functions.sol +++ b/test/libsolidity/syntaxTests/specialFunctions/types_with_unspecified_encoding_internal_functions.sol @@ -5,6 +5,7 @@ contract C { } } // ---- +// Warning: (105-115): Using ".gas(...)" is deprecated. Use "{gas: ...}" instead. // TypeError: (91-100): This type cannot be encoded. // TypeError: (102-103): This type cannot be encoded. // TypeError: (105-115): This type cannot be encoded. diff --git a/test/libsolidity/syntaxTests/viewPureChecker/gas_value_without_call.sol b/test/libsolidity/syntaxTests/viewPureChecker/gas_value_without_call.sol index 2df3bbe4f..77dab9af2 100644 --- a/test/libsolidity/syntaxTests/viewPureChecker/gas_value_without_call.sol +++ b/test/libsolidity/syntaxTests/viewPureChecker/gas_value_without_call.sol @@ -2,12 +2,25 @@ contract C { function f() external payable {} function g(address a) external pure { a.call.value(42); + a.call{value: 42}; a.call.gas(42); + a.call{gas: 42}; a.staticcall.gas(42); + a.staticcall{gas: 42}; a.delegatecall.gas(42); + a.delegatecall{gas: 42}; } function h() external view { this.f.value(42); + this.f{value: 42}; this.f.gas(42); + this.f{gas: 42}; } } +// ---- +// Warning: (91-103): Using ".value(...)" is deprecated. Use "{value: ...}" instead. +// Warning: (132-142): Using ".gas(...)" is deprecated. Use "{gas: ...}" instead. +// Warning: (169-185): Using ".gas(...)" is deprecated. Use "{gas: ...}" instead. +// Warning: (218-236): Using ".gas(...)" is deprecated. Use "{gas: ...}" instead. +// Warning: (304-316): Using ".value(...)" is deprecated. Use "{value: ...}" instead. +// Warning: (345-355): Using ".gas(...)" is deprecated. Use "{gas: ...}" instead. diff --git a/test/libsolidity/syntaxTests/viewPureChecker/gas_with_call_nonpayable.sol b/test/libsolidity/syntaxTests/viewPureChecker/gas_with_call_nonpayable.sol index 0a58a516c..4a0da038e 100644 --- a/test/libsolidity/syntaxTests/viewPureChecker/gas_with_call_nonpayable.sol +++ b/test/libsolidity/syntaxTests/viewPureChecker/gas_with_call_nonpayable.sol @@ -1,16 +1,27 @@ contract C { function f(address a) external view returns (bool success) { (success,) = a.call.gas(42)(""); + (success,) = a.call{gas: 42}(""); } function g(address a) external view returns (bool success) { (success,) = a.call.gas(42)(""); + (success,) = a.call{gas: 42}(""); } function h() external payable {} function i() external view { this.h.gas(42)(); } + function j() external view { + this.h{gas: 42}(); + } } // ---- +// Warning: (90-100): Using ".gas(...)" is deprecated. Use "{gas: ...}" instead. +// Warning: (226-236): Using ".gas(...)" is deprecated. Use "{gas: ...}" instead. +// Warning: (351-361): Using ".gas(...)" is deprecated. Use "{gas: ...}" instead. // TypeError: (90-108): Function declared as view, but this expression (potentially) modifies the state and thus requires non-payable (the default) or payable. -// TypeError: (190-208): Function declared as view, but this expression (potentially) modifies the state and thus requires non-payable (the default) or payable. -// TypeError: (279-295): Function declared as view, but this expression (potentially) modifies the state and thus requires non-payable (the default) or payable. +// TypeError: (125-144): Function declared as view, but this expression (potentially) modifies the state and thus requires non-payable (the default) or payable. +// TypeError: (226-244): Function declared as view, but this expression (potentially) modifies the state and thus requires non-payable (the default) or payable. +// TypeError: (261-280): Function declared as view, but this expression (potentially) modifies the state and thus requires non-payable (the default) or payable. +// TypeError: (351-367): Function declared as view, but this expression (potentially) modifies the state and thus requires non-payable (the default) or payable. +// TypeError: (404-421): Function declared as view, but this expression (potentially) modifies the state and thus requires non-payable (the default) or payable. diff --git a/test/libsolidity/syntaxTests/viewPureChecker/staticcall_gas_view.sol b/test/libsolidity/syntaxTests/viewPureChecker/staticcall_gas_view.sol index 6f8b31bfc..2b4821c53 100644 --- a/test/libsolidity/syntaxTests/viewPureChecker/staticcall_gas_view.sol +++ b/test/libsolidity/syntaxTests/viewPureChecker/staticcall_gas_view.sol @@ -3,9 +3,13 @@ contract C { function test(address a) external view returns (bool status) { // This used to incorrectly raise an error about violating the view mutability. (status,) = a.staticcall.gas(42)(""); + (status,) = a.staticcall{gas: 42}(""); this.f.gas(42)(); + this.f{gas: 42}(); } } // ==== // EVMVersion: >=byzantium // ---- +// Warning: (207-223): Using ".gas(...)" is deprecated. Use "{gas: ...}" instead. +// Warning: (276-286): Using ".gas(...)" is deprecated. Use "{gas: ...}" instead. diff --git a/test/libsolidity/syntaxTests/viewPureChecker/value_with_call_nonpayable.sol b/test/libsolidity/syntaxTests/viewPureChecker/value_with_call_nonpayable.sol index cf5d885c9..6b82f2fab 100644 --- a/test/libsolidity/syntaxTests/viewPureChecker/value_with_call_nonpayable.sol +++ b/test/libsolidity/syntaxTests/viewPureChecker/value_with_call_nonpayable.sol @@ -1,16 +1,25 @@ contract C { function f(address a) external view returns (bool success) { (success,) = a.call.value(42)(""); + (success,) = a.call{value: 42}(""); } function g(address a) external view returns (bool success) { (success,) = a.call.value(42)(""); + (success,) = a.call{value: 42}(""); } function h() external payable {} function i() external view { this.h.value(42)(); + this.h{value: 42}(); } } // ---- +// Warning: (90-102): Using ".value(...)" is deprecated. Use "{value: ...}" instead. +// Warning: (230-242): Using ".value(...)" is deprecated. Use "{value: ...}" instead. +// Warning: (359-371): Using ".value(...)" is deprecated. Use "{value: ...}" instead. // TypeError: (90-110): Function declared as view, but this expression (potentially) modifies the state and thus requires non-payable (the default) or payable. -// TypeError: (192-212): Function declared as view, but this expression (potentially) modifies the state and thus requires non-payable (the default) or payable. -// TypeError: (283-301): Function declared as view, but this expression (potentially) modifies the state and thus requires non-payable (the default) or payable. +// TypeError: (127-148): Function declared as view, but this expression (potentially) modifies the state and thus requires non-payable (the default) or payable. +// TypeError: (230-250): Function declared as view, but this expression (potentially) modifies the state and thus requires non-payable (the default) or payable. +// TypeError: (267-288): Function declared as view, but this expression (potentially) modifies the state and thus requires non-payable (the default) or payable. +// TypeError: (359-377): Function declared as view, but this expression (potentially) modifies the state and thus requires non-payable (the default) or payable. +// TypeError: (381-400): Function declared as view, but this expression (potentially) modifies the state and thus requires non-payable (the default) or payable. From 919888ddbc470a628676586fcc1b21a7030b1042 Mon Sep 17 00:00:00 2001 From: Daniel Kirchner Date: Mon, 2 Mar 2020 16:33:12 +0100 Subject: [PATCH 73/92] Use yul function for calldata tail access, fix checks and add additional revert reason. --- libsolidity/codegen/CompilerUtils.cpp | 56 ++----------------- libsolidity/codegen/YulUtilFunctions.cpp | 9 ++- .../revertStrings/calldata_tail_short.sol | 9 +++ 3 files changed, 21 insertions(+), 53 deletions(-) create mode 100644 test/libsolidity/semanticTests/revertStrings/calldata_tail_short.sol diff --git a/libsolidity/codegen/CompilerUtils.cpp b/libsolidity/codegen/CompilerUtils.cpp index b39633377..da82c1822 100644 --- a/libsolidity/codegen/CompilerUtils.cpp +++ b/libsolidity/codegen/CompilerUtils.cpp @@ -121,56 +121,12 @@ void CompilerUtils::returnDataToArray() void CompilerUtils::accessCalldataTail(Type const& _type) { - solAssert(_type.dataStoredIn(DataLocation::CallData), ""); - solAssert(_type.isDynamicallyEncoded(), ""); - - unsigned int tailSize = _type.calldataEncodedTailSize(); - solAssert(tailSize > 1, ""); - - // returns the absolute offset of the tail in "base_ref" - m_context.appendInlineAssembly(Whiskers(R"({ - let rel_offset_of_tail := calldataload(ptr_to_tail) - if iszero(slt(rel_offset_of_tail, sub(sub(calldatasize(), base_ref), sub(, 1)))) { } - base_ref := add(base_ref, rel_offset_of_tail) - })") - ("neededLength", toCompactHexWithPrefix(tailSize)) - ("revertString", m_context.revertReasonIfDebug("Invalid calldata tail offset")) - .render(), {"base_ref", "ptr_to_tail"}); - // stack layout: - - if (!_type.isDynamicallySized()) - { - m_context << Instruction::POP; - // stack layout: - solAssert( - _type.category() == Type::Category::Struct || - _type.category() == Type::Category::Array, - "Invalid dynamically encoded base type on tail access." - ); - } - else - { - auto const* arrayType = dynamic_cast(&_type); - solAssert(!!arrayType, "Invalid dynamically sized type."); - unsigned int calldataStride = arrayType->calldataStride(); - solAssert(calldataStride > 0, ""); - - // returns the absolute offset of the tail in "base_ref" - // and the length of the tail in "length" - m_context.appendInlineAssembly( - Whiskers(R"({ - length := calldataload(base_ref) - base_ref := add(base_ref, 0x20) - if gt(length, 0xffffffffffffffff) { } - if sgt(base_ref, sub(calldatasize(), mul(length, ))) { revert(0, 0) } - })") - ("calldataStride", toCompactHexWithPrefix(calldataStride)) - ("revertString", m_context.revertReasonIfDebug("Invalid calldata tail length")) - .render(), - {"base_ref", "length"} - ); - // stack layout: - } + m_context << Instruction::SWAP1; + m_context.callYulFunction( + m_context.utilFunctions().accessCalldataTailFunction(_type), + 2, + _type.isDynamicallySized() ? 2 : 1 + ); } unsigned CompilerUtils::loadFromMemory( diff --git a/libsolidity/codegen/YulUtilFunctions.cpp b/libsolidity/codegen/YulUtilFunctions.cpp index 0472dd588..7a5036109 100644 --- a/libsolidity/codegen/YulUtilFunctions.cpp +++ b/libsolidity/codegen/YulUtilFunctions.cpp @@ -942,13 +942,13 @@ string YulUtilFunctions::accessCalldataTailFunction(Type const& _type) return Whiskers(R"( function (base_ref, ptr_to_tail) -> addr, length { let rel_offset_of_tail := calldataload(ptr_to_tail) - if iszero(slt(rel_offset_of_tail, sub(sub(calldatasize(), base_ref), sub(, 1)))) { revert(0, 0) } + if iszero(slt(rel_offset_of_tail, sub(sub(calldatasize(), base_ref), sub(, 1)))) { } addr := add(base_ref, rel_offset_of_tail) length := calldataload(addr) - if gt(length, 0xffffffffffffffff) { revert(0, 0) } - if sgt(base_ref, sub(calldatasize(), mul(length, ))) { revert(0, 0) } + if gt(length, 0xffffffffffffffff) { } addr := add(addr, 32) + if sgt(addr, sub(calldatasize(), mul(length, ))) { } } )") @@ -956,6 +956,9 @@ string YulUtilFunctions::accessCalldataTailFunction(Type const& _type) ("dynamicallySized", _type.isDynamicallySized()) ("neededLength", toCompactHexWithPrefix(_type.calldataEncodedTailSize())) ("calldataStride", toCompactHexWithPrefix(_type.isDynamicallySized() ? dynamic_cast(_type).calldataStride() : 0)) + ("invalidCalldataTailOffset", revertReasonIfDebug("Invalid calldata tail offset")) + ("invalidCalldataTailLength", revertReasonIfDebug("Invalid calldata tail length")) + ("shortCalldataTail", revertReasonIfDebug("Calldata tail too short")) .render(); }); } diff --git a/test/libsolidity/semanticTests/revertStrings/calldata_tail_short.sol b/test/libsolidity/semanticTests/revertStrings/calldata_tail_short.sol new file mode 100644 index 000000000..b6cc5800d --- /dev/null +++ b/test/libsolidity/semanticTests/revertStrings/calldata_tail_short.sol @@ -0,0 +1,9 @@ +pragma experimental ABIEncoderV2; +contract C { + function f(uint256[][] calldata x) external { x[0]; } +} +// ==== +// EVMVersion: >=byzantium +// revertStrings: debug +// ---- +// f(uint256[][]): 0x20, 1, 0x20, 2, 0x42 -> FAILURE, hex"08c379a0", 0x20, 23, "Calldata tail too short" From 857ed12b056489eb53131b196bf8ea5e4257c956 Mon Sep 17 00:00:00 2001 From: Daniel Kirchner Date: Mon, 2 Mar 2020 18:08:19 +0100 Subject: [PATCH 74/92] Use plain members and references instead of shared pointers for MultiUseYulFunctionCollector --- libsolidity/codegen/ABIFunctions.cpp | 2 +- libsolidity/codegen/ABIFunctions.h | 6 +- libsolidity/codegen/CompilerContext.cpp | 2 +- libsolidity/codegen/CompilerContext.h | 2 +- libsolidity/codegen/YulUtilFunctions.cpp | 110 +++++++++--------- libsolidity/codegen/YulUtilFunctions.h | 6 +- .../codegen/ir/IRGenerationContext.cpp | 2 +- libsolidity/codegen/ir/IRGenerationContext.h | 7 +- libsolidity/codegen/ir/IRGenerator.cpp | 13 +-- 9 files changed, 74 insertions(+), 76 deletions(-) diff --git a/libsolidity/codegen/ABIFunctions.cpp b/libsolidity/codegen/ABIFunctions.cpp index b6d690898..5cca1c1e4 100644 --- a/libsolidity/codegen/ABIFunctions.cpp +++ b/libsolidity/codegen/ABIFunctions.cpp @@ -1492,7 +1492,7 @@ string ABIFunctions::arrayStoreLengthForEncodingFunction(ArrayType const& _type, string ABIFunctions::createFunction(string const& _name, function const& _creator) { - return m_functionCollector->createFunction(_name, _creator); + return m_functionCollector.createFunction(_name, _creator); } size_t ABIFunctions::headSize(TypePointers const& _targetTypes) diff --git a/libsolidity/codegen/ABIFunctions.h b/libsolidity/codegen/ABIFunctions.h index 1099e3593..e760917ac 100644 --- a/libsolidity/codegen/ABIFunctions.h +++ b/libsolidity/codegen/ABIFunctions.h @@ -57,11 +57,11 @@ public: explicit ABIFunctions( langutil::EVMVersion _evmVersion, RevertStrings _revertStrings, - std::shared_ptr _functionCollector = std::make_shared() + MultiUseYulFunctionCollector& _functionCollector ): m_evmVersion(_evmVersion), m_revertStrings(_revertStrings), - m_functionCollector(std::move(_functionCollector)), + m_functionCollector(_functionCollector), m_utils(_evmVersion, m_revertStrings, m_functionCollector) {} @@ -247,7 +247,7 @@ private: langutil::EVMVersion m_evmVersion; RevertStrings const m_revertStrings; - std::shared_ptr m_functionCollector; + MultiUseYulFunctionCollector& m_functionCollector; YulUtilFunctions m_utils; }; diff --git a/libsolidity/codegen/CompilerContext.cpp b/libsolidity/codegen/CompilerContext.cpp index 1d1eb92b9..cb195c49f 100644 --- a/libsolidity/codegen/CompilerContext.cpp +++ b/libsolidity/codegen/CompilerContext.cpp @@ -151,7 +151,7 @@ pair> CompilerContext::requestedYulFunctions() { set empty; swap(empty, m_externallyUsedYulFunctions); - return make_pair(m_yulFunctionCollector->requestedFunctions(), std::move(empty)); + return {m_yulFunctionCollector.requestedFunctions(), std::move(empty)}; } void CompilerContext::addVariable( diff --git a/libsolidity/codegen/CompilerContext.h b/libsolidity/codegen/CompilerContext.h index 76c72770d..8c5775239 100644 --- a/libsolidity/codegen/CompilerContext.h +++ b/libsolidity/codegen/CompilerContext.h @@ -371,7 +371,7 @@ private: /// An index of low-level function labels by name. std::map m_lowLevelFunctions; /// Collector for yul functions. - std::shared_ptr m_yulFunctionCollector = std::make_shared(); + MultiUseYulFunctionCollector m_yulFunctionCollector; /// Set of externally used yul functions. std::set m_externallyUsedYulFunctions; /// Container for ABI functions to be generated. diff --git a/libsolidity/codegen/YulUtilFunctions.cpp b/libsolidity/codegen/YulUtilFunctions.cpp index 0472dd588..bcb309c99 100644 --- a/libsolidity/codegen/YulUtilFunctions.cpp +++ b/libsolidity/codegen/YulUtilFunctions.cpp @@ -39,7 +39,7 @@ using namespace solidity::frontend; string YulUtilFunctions::combineExternalFunctionIdFunction() { string functionName = "combine_external_function_id"; - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (addr, selector) -> combined { combined := (or((addr), and(selector, 0xffffffff))) @@ -55,7 +55,7 @@ string YulUtilFunctions::combineExternalFunctionIdFunction() string YulUtilFunctions::splitExternalFunctionIdFunction() { string functionName = "split_external_function_id"; - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (combined) -> addr, selector { combined := (combined) @@ -73,7 +73,7 @@ string YulUtilFunctions::splitExternalFunctionIdFunction() string YulUtilFunctions::copyToMemoryFunction(bool _fromCalldata) { string functionName = "copy_" + string(_fromCalldata ? "calldata" : "memory") + "_to_memory"; - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { if (_fromCalldata) { return Whiskers(R"( @@ -116,7 +116,7 @@ string YulUtilFunctions::requireOrAssertFunction(bool _assert, Type const* _mess solAssert(!_assert || !_messageType, "Asserts can't have messages!"); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { if (!_messageType) return Whiskers(R"( function (condition) { @@ -166,7 +166,7 @@ string YulUtilFunctions::requireOrAssertFunction(bool _assert, Type const* _mess string YulUtilFunctions::leftAlignFunction(Type const& _type) { string functionName = string("leftAlign_") + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { Whiskers templ(R"( function (value) -> aligned { @@ -228,7 +228,7 @@ string YulUtilFunctions::shiftLeftFunction(size_t _numBits) solAssert(_numBits < 256, ""); string functionName = "shift_left_" + to_string(_numBits); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (value) -> newValue { @@ -251,7 +251,7 @@ string YulUtilFunctions::shiftLeftFunction(size_t _numBits) string YulUtilFunctions::shiftLeftFunctionDynamic() { string functionName = "shift_left_dynamic"; - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (bits, value) -> newValue { @@ -277,7 +277,7 @@ string YulUtilFunctions::shiftRightFunction(size_t _numBits) // the opcodes SAR and SDIV behave differently with regards to rounding! string functionName = "shift_right_" + to_string(_numBits) + "_unsigned"; - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (value) -> newValue { @@ -303,7 +303,7 @@ string YulUtilFunctions::shiftRightFunctionDynamic() // the opcodes SAR and SDIV behave differently with regards to rounding! string const functionName = "shift_right_unsigned_dynamic"; - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (bits, value) -> newValue { @@ -328,7 +328,7 @@ string YulUtilFunctions::updateByteSliceFunction(size_t _numBytes, size_t _shift size_t numBits = _numBytes * 8; size_t shiftBits = _shiftBytes * 8; string functionName = "update_byte_slice_" + to_string(_numBytes) + "_shift_" + to_string(_shiftBytes); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (value, toInsert) -> result { @@ -350,7 +350,7 @@ string YulUtilFunctions::updateByteSliceFunctionDynamic(size_t _numBytes) solAssert(_numBytes <= 32, ""); size_t numBits = _numBytes * 8; string functionName = "update_byte_slice_dynamic" + to_string(_numBytes); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (value, shiftBytes, toInsert) -> result { @@ -371,7 +371,7 @@ string YulUtilFunctions::updateByteSliceFunctionDynamic(size_t _numBytes) string YulUtilFunctions::roundUpFunction() { string functionName = "round_up_to_mul_of_32"; - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (value) -> result { @@ -389,7 +389,7 @@ string YulUtilFunctions::overflowCheckedIntAddFunction(IntegerType const& _type) // TODO: Consider to add a special case for unsigned 256-bit integers // and use the following instead: // sum := add(x, y) if lt(sum, x) { revert(0, 0) } - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (x, y) -> sum { @@ -416,7 +416,7 @@ string YulUtilFunctions::overflowCheckedIntAddFunction(IntegerType const& _type) string YulUtilFunctions::overflowCheckedIntMulFunction(IntegerType const& _type) { string functionName = "checked_mul_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return // Multiplication by zero could be treated separately and directly return zero. Whiskers(R"( @@ -448,7 +448,7 @@ string YulUtilFunctions::overflowCheckedIntMulFunction(IntegerType const& _type) string YulUtilFunctions::overflowCheckedIntDivFunction(IntegerType const& _type) { string functionName = "checked_div_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (x, y) -> r { @@ -473,7 +473,7 @@ string YulUtilFunctions::overflowCheckedIntDivFunction(IntegerType const& _type) string YulUtilFunctions::checkedIntModFunction(IntegerType const& _type) { string functionName = "checked_mod_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (x, y) -> r { @@ -490,7 +490,7 @@ string YulUtilFunctions::checkedIntModFunction(IntegerType const& _type) string YulUtilFunctions::overflowCheckedIntSubFunction(IntegerType const& _type) { string functionName = "checked_sub_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&] { + return m_functionCollector.createFunction(functionName, [&] { return Whiskers(R"( function (x, y) -> diff { @@ -516,7 +516,7 @@ string YulUtilFunctions::overflowCheckedIntSubFunction(IntegerType const& _type) string YulUtilFunctions::arrayLengthFunction(ArrayType const& _type) { string functionName = "array_length_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { Whiskers w(R"( function (value) -> length { @@ -564,7 +564,7 @@ std::string YulUtilFunctions::resizeDynamicArrayFunction(ArrayType const& _type) solUnimplementedAssert(_type.baseType()->storageSize() == 1, ""); string functionName = "resize_array_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (array, newLen) { if gt(newLen, ) { @@ -604,7 +604,7 @@ string YulUtilFunctions::storageArrayPopFunction(ArrayType const& _type) solUnimplementedAssert(_type.baseType()->storageBytes() <= 32, "Base type is not yet implemented."); string functionName = "array_pop_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (array) { let oldLen := (array) @@ -632,7 +632,7 @@ string YulUtilFunctions::storageArrayPushFunction(ArrayType const& _type) solUnimplementedAssert(_type.baseType()->storageBytes() <= 32, "Base type is not yet implemented."); string functionName = "array_push_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (array, value) { let oldLen := (array) @@ -659,7 +659,7 @@ string YulUtilFunctions::storageArrayPushZeroFunction(ArrayType const& _type) solUnimplementedAssert(_type.baseType()->storageBytes() <= 32, "Base type is not yet implemented."); string functionName = "array_push_zero_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (array) -> slot, offset { let oldLen := (array) @@ -684,7 +684,7 @@ string YulUtilFunctions::clearStorageRangeFunction(Type const& _type) solAssert(_type.storageBytes() >= 32, "Expected smaller value for storage bytes"); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (start, end) { for {} lt(start, end) { start := add(start, ) } @@ -715,7 +715,7 @@ string YulUtilFunctions::clearStorageArrayFunction(ArrayType const& _type) string functionName = "clear_storage_array_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (slot) { @@ -745,7 +745,7 @@ string YulUtilFunctions::clearStorageArrayFunction(ArrayType const& _type) string YulUtilFunctions::arrayConvertLengthToSize(ArrayType const& _type) { string functionName = "array_convert_length_to_size_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { Type const& baseType = *_type.baseType(); switch (_type.location()) @@ -798,7 +798,7 @@ string YulUtilFunctions::arrayAllocationSizeFunction(ArrayType const& _type) { solAssert(_type.dataStoredIn(DataLocation::Memory), ""); string functionName = "array_allocation_size_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { Whiskers w(R"( function (length) -> size { // Make sure we can allocate memory without overflow @@ -825,7 +825,7 @@ string YulUtilFunctions::arrayAllocationSizeFunction(ArrayType const& _type) string YulUtilFunctions::arrayDataAreaFunction(ArrayType const& _type) { string functionName = "array_dataslot_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { // No special processing for calldata arrays, because they are stored as // offset of the data area and length on the stack, so the offset already // points to the data area. @@ -858,7 +858,7 @@ string YulUtilFunctions::storageArrayIndexAccessFunction(ArrayType const& _type) solUnimplementedAssert(_type.baseType()->storageBytes() > 16, ""); string functionName = "storage_array_index_access_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (array, index) -> slot, offset { if iszero(lt(index, (array))) { @@ -886,7 +886,7 @@ string YulUtilFunctions::storageArrayIndexAccessFunction(ArrayType const& _type) string YulUtilFunctions::memoryArrayIndexAccessFunction(ArrayType const& _type) { string functionName = "memory_array_index_access_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (baseRef, index) -> addr { if iszero(lt(index, (baseRef))) { @@ -912,7 +912,7 @@ string YulUtilFunctions::calldataArrayIndexAccessFunction(ArrayType const& _type { solAssert(_type.dataStoredIn(DataLocation::CallData), ""); string functionName = "calldata_array_index_access_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (base_ref, length, index) -> addr, len { if iszero(lt(index, length)) { invalid() } @@ -938,7 +938,7 @@ string YulUtilFunctions::accessCalldataTailFunction(Type const& _type) solAssert(_type.isDynamicallyEncoded(), ""); solAssert(_type.dataStoredIn(DataLocation::CallData), ""); string functionName = "access_calldata_tail_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (base_ref, ptr_to_tail) -> addr, length { let rel_offset_of_tail := calldataload(ptr_to_tail) @@ -966,7 +966,7 @@ string YulUtilFunctions::nextArrayElementFunction(ArrayType const& _type) if (_type.dataStoredIn(DataLocation::Storage)) solAssert(_type.baseType()->storageBytes() > 16, ""); string functionName = "array_nextElement_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { Whiskers templ(R"( function (ptr) -> next { next := add(ptr, ) @@ -1002,7 +1002,7 @@ string YulUtilFunctions::mappingIndexAccessFunction(MappingType const& _mappingT solAssert(_keyType.sizeOnStack() <= 1, ""); string functionName = "mapping_index_access_" + _mappingType.identifier() + "_of_" + _keyType.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { if (_mappingType.keyType()->isDynamicallySized()) return Whiskers(R"( function (slot ) -> dataSlot { @@ -1050,7 +1050,7 @@ string YulUtilFunctions::readFromStorage(Type const& _type, size_t _offset, bool to_string(_offset) + "_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&] { + return m_functionCollector.createFunction(functionName, [&] { solAssert(_type.sizeOnStack() == 1, ""); return Whiskers(R"( function (slot) -> value { @@ -1071,7 +1071,7 @@ string YulUtilFunctions::readFromStorageDynamic(Type const& _type, bool _splitFu string(_splitFunctionTypes ? "split_" : "") + "_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&] { + return m_functionCollector.createFunction(functionName, [&] { solAssert(_type.sizeOnStack() == 1, ""); return Whiskers(R"( function (slot, offset) -> value { @@ -1101,7 +1101,7 @@ string YulUtilFunctions::updateStorageValueFunction(Type const& _type, std::opti (_offset.has_value() ? ("offset_" + to_string(*_offset)) : "") + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&] { + return m_functionCollector.createFunction(functionName, [&] { if (_type.isValueType()) { solAssert(_type.storageBytes() <= 32, "Invalid storage bytes size."); @@ -1141,7 +1141,7 @@ string YulUtilFunctions::writeToMemoryFunction(Type const& _type) string("write_to_memory_") + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&] { + return m_functionCollector.createFunction(functionName, [&] { solAssert(!dynamic_cast(&_type), ""); if (auto ref = dynamic_cast(&_type)) { @@ -1201,7 +1201,7 @@ string YulUtilFunctions::extractFromStorageValueDynamic(Type const& _type, bool "extract_from_storage_value_dynamic" + string(_splitFunctionTypes ? "split_" : "") + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&] { + return m_functionCollector.createFunction(functionName, [&] { return Whiskers(R"( function (slot_value, offset) -> value { value := ((mul(offset, 8), slot_value)) @@ -1224,7 +1224,7 @@ string YulUtilFunctions::extractFromStorageValue(Type const& _type, size_t _offs "offset_" + to_string(_offset) + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&] { + return m_functionCollector.createFunction(functionName, [&] { return Whiskers(R"( function (slot_value) -> value { value := ((slot_value)) @@ -1243,7 +1243,7 @@ string YulUtilFunctions::cleanupFromStorageFunction(Type const& _type, bool _spl solUnimplementedAssert(!_splitFunctionTypes, ""); string functionName = string("cleanup_from_storage_") + (_splitFunctionTypes ? "split_" : "") + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&] { + return m_functionCollector.createFunction(functionName, [&] { Whiskers templ(R"( function (value) -> cleaned { cleaned := @@ -1275,7 +1275,7 @@ string YulUtilFunctions::prepareStoreFunction(Type const& _type) solUnimplementedAssert(_type.category() != Type::Category::Function, ""); string functionName = "prepare_store_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { Whiskers templ(R"( function (value) -> ret { ret := @@ -1293,7 +1293,7 @@ string YulUtilFunctions::prepareStoreFunction(Type const& _type) string YulUtilFunctions::allocationFunction() { string functionName = "allocateMemory"; - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (size) -> memPtr { memPtr := mload() @@ -1314,7 +1314,7 @@ string YulUtilFunctions::allocateMemoryArrayFunction(ArrayType const& _type) solUnimplementedAssert(!_type.isByteArray(), ""); string functionName = "allocate_memory_array_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (length) -> memPtr { memPtr := ((length)) @@ -1341,7 +1341,7 @@ string YulUtilFunctions::conversionFunction(Type const& _from, Type const& _to) _from.identifier() + "_to_" + _to.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { Whiskers templ(R"( function (value) -> converted { @@ -1519,7 +1519,7 @@ string YulUtilFunctions::conversionFunction(Type const& _from, Type const& _to) string YulUtilFunctions::cleanupFunction(Type const& _type) { string functionName = string("cleanup_") + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { Whiskers templ(R"( function (value) -> cleaned { @@ -1606,7 +1606,7 @@ string YulUtilFunctions::cleanupFunction(Type const& _type) string YulUtilFunctions::validatorFunction(Type const& _type, bool _revertOnFailure) { string functionName = string("validator_") + (_revertOnFailure ? "revert_" : "assert_") + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { Whiskers templ(R"( function (value) { if iszero() { } @@ -1667,7 +1667,7 @@ string YulUtilFunctions::packedHashFunction( size_t sizeOnStack = 0; for (Type const* t: _givenTypes) sizeOnStack += t->sizeOnStack(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { Whiskers templ(R"( function () -> hash { let pos := mload() @@ -1688,7 +1688,7 @@ string YulUtilFunctions::forwardingRevertFunction() { bool forward = m_evmVersion.supportsReturndata(); string functionName = "revert_forward_" + to_string(forward); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { if (forward) return Whiskers(R"( function () { @@ -1715,7 +1715,7 @@ std::string YulUtilFunctions::decrementCheckedFunction(Type const& _type) string const functionName = "decrement_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { u256 minintval; // Smallest admissible value to decrement @@ -1743,7 +1743,7 @@ std::string YulUtilFunctions::incrementCheckedFunction(Type const& _type) string const functionName = "increment_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { u256 maxintval; // Biggest admissible value to increment @@ -1774,7 +1774,7 @@ string YulUtilFunctions::negateNumberCheckedFunction(Type const& _type) u256 const minintval = 0 - (u256(1) << (type.numBits() - 1)) + 1; - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (_value) -> ret { if slt(_value, ) { revert(0,0) } @@ -1794,7 +1794,7 @@ string YulUtilFunctions::zeroValueFunction(Type const& _type) string const functionName = "zero_value_for_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function () -> ret { @@ -1810,7 +1810,7 @@ string YulUtilFunctions::storageSetToZeroFunction(Type const& _type) { string const functionName = "storage_set_to_zero_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { if (_type.isValueType()) return Whiskers(R"( function (slot, offset) { @@ -1842,7 +1842,7 @@ string YulUtilFunctions::conversionFunctionSpecial(Type const& _from, Type const _from.identifier() + "_to_" + _to.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { if ( auto fromTuple = dynamic_cast(&_from), toTuple = dynamic_cast(&_to); fromTuple && toTuple && fromTuple->components().size() == toTuple->components().size() @@ -1950,7 +1950,7 @@ string YulUtilFunctions::readFromMemoryOrCalldata(Type const& _type, bool _fromC if (_fromCalldata) solAssert(!_type.isDynamicallyEncoded(), ""); - return m_functionCollector->createFunction(functionName, [&] { + return m_functionCollector.createFunction(functionName, [&] { if (auto refType = dynamic_cast(&_type)) { solAssert(refType->sizeOnStack() == 1, ""); diff --git a/libsolidity/codegen/YulUtilFunctions.h b/libsolidity/codegen/YulUtilFunctions.h index 8c92d7b4f..cf50c785f 100644 --- a/libsolidity/codegen/YulUtilFunctions.h +++ b/libsolidity/codegen/YulUtilFunctions.h @@ -47,11 +47,11 @@ public: explicit YulUtilFunctions( langutil::EVMVersion _evmVersion, RevertStrings _revertStrings, - std::shared_ptr _functionCollector + MultiUseYulFunctionCollector& _functionCollector ): m_evmVersion(_evmVersion), m_revertStrings(_revertStrings), - m_functionCollector(std::move(_functionCollector)) + m_functionCollector(_functionCollector) {} /// @returns a function that combines the address and selector to a single value @@ -306,7 +306,7 @@ private: langutil::EVMVersion m_evmVersion; RevertStrings m_revertStrings; - std::shared_ptr m_functionCollector; + MultiUseYulFunctionCollector& m_functionCollector; }; } diff --git a/libsolidity/codegen/ir/IRGenerationContext.cpp b/libsolidity/codegen/ir/IRGenerationContext.cpp index 43cb6a040..7184247f4 100644 --- a/libsolidity/codegen/ir/IRGenerationContext.cpp +++ b/libsolidity/codegen/ir/IRGenerationContext.cpp @@ -100,7 +100,7 @@ string IRGenerationContext::newYulVariable() string IRGenerationContext::internalDispatch(size_t _in, size_t _out) { string funName = "dispatch_internal_in_" + to_string(_in) + "_out_" + to_string(_out); - return m_functions->createFunction(funName, [&]() { + return m_functions.createFunction(funName, [&]() { Whiskers templ(R"( function (fun ) { switch fun diff --git a/libsolidity/codegen/ir/IRGenerationContext.h b/libsolidity/codegen/ir/IRGenerationContext.h index d4c29ab10..473b62482 100644 --- a/libsolidity/codegen/ir/IRGenerationContext.h +++ b/libsolidity/codegen/ir/IRGenerationContext.h @@ -56,11 +56,10 @@ public: ): m_evmVersion(_evmVersion), m_revertStrings(_revertStrings), - m_optimiserSettings(std::move(_optimiserSettings)), - m_functions(std::make_shared()) + m_optimiserSettings(std::move(_optimiserSettings)) {} - std::shared_ptr functionCollector() const { return m_functions; } + MultiUseYulFunctionCollector& functionCollector() { return m_functions; } /// Sets the current inheritance hierarchy from derived to base. void setInheritanceHierarchy(std::vector _hierarchy) @@ -108,7 +107,7 @@ private: std::map m_localVariables; /// Storage offsets of state variables std::map> m_stateVariables; - std::shared_ptr m_functions; + MultiUseYulFunctionCollector m_functions; size_t m_varCounter = 0; }; diff --git a/libsolidity/codegen/ir/IRGenerator.cpp b/libsolidity/codegen/ir/IRGenerator.cpp index 9c2d8807c..8da576055 100644 --- a/libsolidity/codegen/ir/IRGenerator.cpp +++ b/libsolidity/codegen/ir/IRGenerator.cpp @@ -107,7 +107,7 @@ string IRGenerator::generate(ContractDefinition const& _contract) for (auto const* contract: _contract.annotation().linearizedBaseContracts) for (auto const* fun: contract->definedFunctions()) generateFunction(*fun); - t("functions", m_context.functionCollector()->requestedFunctions()); + t("functions", m_context.functionCollector().requestedFunctions()); resetContext(_contract); m_context.setInheritanceHierarchy(_contract.annotation().linearizedBaseContracts); @@ -116,7 +116,7 @@ string IRGenerator::generate(ContractDefinition const& _contract) for (auto const* contract: _contract.annotation().linearizedBaseContracts) for (auto const* fun: contract->definedFunctions()) generateFunction(*fun); - t("runtimeFunctions", m_context.functionCollector()->requestedFunctions()); + t("runtimeFunctions", m_context.functionCollector().requestedFunctions()); return t.render(); } @@ -130,7 +130,7 @@ string IRGenerator::generate(Block const& _block) string IRGenerator::generateFunction(FunctionDefinition const& _function) { string functionName = m_context.functionName(_function); - return m_context.functionCollector()->createFunction(functionName, [&]() { + return m_context.functionCollector().createFunction(functionName, [&]() { Whiskers t(R"( function () { @@ -160,7 +160,7 @@ string IRGenerator::generateGetter(VariableDeclaration const& _varDecl) solAssert(_varDecl.isStateVariable(), ""); if (auto const* mappingType = dynamic_cast(type)) - return m_context.functionCollector()->createFunction(functionName, [&]() { + return m_context.functionCollector().createFunction(functionName, [&]() { pair slot_offset = m_context.storageLocationOfVariable(_varDecl); solAssert(slot_offset.second == 0, ""); FunctionType funType(_varDecl); @@ -209,7 +209,7 @@ string IRGenerator::generateGetter(VariableDeclaration const& _varDecl) { solUnimplementedAssert(type->isValueType(), ""); - return m_context.functionCollector()->createFunction(functionName, [&]() { + return m_context.functionCollector().createFunction(functionName, [&]() { pair slot_offset = m_context.storageLocationOfVariable(_varDecl); return Whiskers(R"( @@ -383,11 +383,10 @@ string IRGenerator::memoryInit() void IRGenerator::resetContext(ContractDefinition const& _contract) { solAssert( - m_context.functionCollector()->requestedFunctions().empty(), + m_context.functionCollector().requestedFunctions().empty(), "Reset context while it still had functions." ); m_context = IRGenerationContext(m_evmVersion, m_context.revertStrings(), m_optimiserSettings); - m_utils = YulUtilFunctions(m_evmVersion, m_context.revertStrings(), m_context.functionCollector()); m_context.setInheritanceHierarchy(_contract.annotation().linearizedBaseContracts); for (auto const& var: ContractType(_contract).stateVariables()) From 38b219d140b646705cebf2a92bb2d9b2838a9866 Mon Sep 17 00:00:00 2001 From: Djordje Mijovic Date: Wed, 4 Mar 2020 16:16:30 +0100 Subject: [PATCH 75/92] Throwing stack too deep ICE in case of calling encode with too many arguments instead of invalid opcode --- libsolidity/codegen/CompilerUtils.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libsolidity/codegen/CompilerUtils.cpp b/libsolidity/codegen/CompilerUtils.cpp index 4c75081ff..e958da05e 100644 --- a/libsolidity/codegen/CompilerUtils.cpp +++ b/libsolidity/codegen/CompilerUtils.cpp @@ -539,6 +539,10 @@ void CompilerUtils::encodeToMemory( if (targetType->isDynamicallySized() && !_copyDynamicDataInPlace) { // copy tail pointer (=mem_end - mem_start) to memory + solAssert( + (2 + dynPointers) <= 16, + "Stack too deep(" + to_string(2 + dynPointers) + "), try using fewer variables." + ); m_context << dupInstruction(2 + dynPointers) << Instruction::DUP2; m_context << Instruction::SUB; m_context << dupInstruction(2 + dynPointers - thisDynPointer); From 7483c6f13e818c8c75aa2e11014d0d7b7bd38a31 Mon Sep 17 00:00:00 2001 From: Bhargava Shastry Date: Thu, 5 Mar 2020 10:37:52 +0100 Subject: [PATCH 76/92] ossfuzz: Update README.md with steps to build fuzzers via docker --- test/tools/ossfuzz/README.md | 68 ++++++++++++++++++++++++++++++------ 1 file changed, 58 insertions(+), 10 deletions(-) diff --git a/test/tools/ossfuzz/README.md b/test/tools/ossfuzz/README.md index 70469513c..c360eb9a8 100644 --- a/test/tools/ossfuzz/README.md +++ b/test/tools/ossfuzz/README.md @@ -1,8 +1,63 @@ ## Intro -[oss-fuzz][1] is Google's fuzzing infrastructure that performs continuous fuzzing. What this means is that, each and every upstream commit is automatically fetched by the infrastructure and fuzzed. +[oss-fuzz][1] is Google's fuzzing infrastructure that performs continuous fuzzing. What this means is that, each and every upstream commit is automatically fetched by the infrastructure and fuzzed on a daily basis. -## What does this directory contain? +## How to build fuzzers? + +We have multiple fuzzers, some based on string input and others on protobuf input. To build them, please do the following: + +- Create a local docker image from `Dockerfile.ubuntu1604.clang.ossfuzz` in the `.circleci/docker` sub-directory. Please note that this step is going to take at least an hour to complete. Therefore, it is recommended to do it when you are away from computer (and the computer is plugged to power since we do not want a battery drain). + +``` +$ cd .circleci/docker +$ docker build -t solidity-ossfuzz-local -f Dockerfile.ubuntu1604.clang.ossfuzz . +``` + +- Create and login into the docker container sourced from the image built in the previous step from the solidity parent directory + +``` +## Host +$ cd solidity +$ docker run -v `pwd`:/src/solidity -ti solidity-ossfuzz-local /bin/bash +## Docker shell +$ cd /src/solidity +``` + +- Run cmake and build fuzzer harnesses + +``` +## Docker shell +$ cd /src/solidity +$ rm -rf fuzzer-build && mkdir fuzzer-build && cd fuzzer-build +## Compile protobuf C++ bindings +$ protoc --proto_path=../test/tools/ossfuzz yulProto.proto --cpp_out=../test/tools/ossfuzz +$ protoc --proto_path=../test/tools/ossfuzz abiV2Proto.proto --cpp_out=../test/tools/ossfuzz +## Run cmake +$ export CC=clang CXX=clang++ +$ cmake -DCMAKE_TOOLCHAIN_FILE=cmake/toolchains/libfuzzer.cmake -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-Release} .. +$ make ossfuzz ossfuzz_proto ossfuzz_abiv2 -j +``` + +## Why the elaborate docker image to build fuzzers? + +For the following reasons: + +- Fuzzing binaries **must** link against libc++ and not libstdc++ + - This is [because][2] (1) MemorySanitizer (which flags uses of uninitialized memory) depends on libc++; and (2) because libc++ is instrumented (to check for memory and type errors) and libstdc++ not, the former may find more bugs. + +- Linking against libc++ requires us to compile everything solidity depends on from source (and link these against libc++ as well) + +- To reproduce the compiler versions used by upstream oss-fuzz bots, we need to reuse their docker image containing the said compiler versions + +- Some fuzzers depend on libprotobuf, libprotobuf-mutator, libevmone etc. which may not be available locally; even if they were they might not be the right versions + +## What is LIB\_FUZZING\_ENGINE? + +oss-fuzz contains multiple fuzzer back-ends i.e., fuzzers. Each back-end may require different linker flags. oss-fuzz builder bot defines the correct linker flags via a bash environment variable called `LIB_FUZZING_ENGINE`. + +For the solidity ossfuzz CI build, we use the libFuzzer back-end. This back-end requires us to manually set the `LIB_FUZZING_ENGINE` to `-fsanitize=fuzzer`. + +## What does the ossfuzz directory contain? To help oss-fuzz do this, we (as project maintainers) need to provide the following: @@ -17,12 +72,5 @@ To be consistent and aid better evaluation of the utility of the fuzzing diction - Incomplete tokens including function calls such as `msg.sender.send()` are abbreviated `.send(` to provide some leeway to the fuzzer to sythesize variants such as `address(this).send()` - Language keywords are suffixed by a whitespace with the exception of those that end a line of code such as `break;` and `continue;` -## What is libFuzzingEngine.a? - -`libFuzzingEngine.a` is an oss-fuzz-related dependency. It is present in the Dockerized environment in which Solidity's oss-fuzz code will be built. - -## Is this directory relevant for routine Solidity CI builds? - -No. This is the reason why the `add_subdirectory(ossfuzz)` cmake directive is nested under the `if (OSSFUZZ)` predicate. `OSSFUZZ` is a solidity-wide cmake option that is invoked by the ossfuzz solidity-builder-bot in order to compile solidity fuzzer binaries. - [1]: https://github.com/google/oss-fuzz +[2]: https://github.com/google/oss-fuzz/issues/1114#issuecomment-360660201 From 420f57aec301e7629721d8220699c7d6239e7333 Mon Sep 17 00:00:00 2001 From: Daniel Kirchner Date: Thu, 5 Mar 2020 12:56:14 +0100 Subject: [PATCH 77/92] Fix yul ast import for blocks, switches and string literals. --- Changelog.md | 1 + libsolidity/ast/AsmJsonImporter.cpp | 19 ++- scripts/ASTImportTest.sh | 3 +- .../ASTJSON/assembly/empty_block.json | 95 ++++++++++++++ .../ASTJSON/assembly/empty_block.sol | 7 + .../ASTJSON/assembly/empty_block_legacy.json | 123 ++++++++++++++++++ .../ASTJSON/assembly/switch_default.json | 116 +++++++++++++++++ .../ASTJSON/assembly/switch_default.sol | 7 + .../assembly/switch_default_legacy.json | 123 ++++++++++++++++++ 9 files changed, 488 insertions(+), 6 deletions(-) create mode 100644 test/libsolidity/ASTJSON/assembly/empty_block.json create mode 100644 test/libsolidity/ASTJSON/assembly/empty_block.sol create mode 100644 test/libsolidity/ASTJSON/assembly/empty_block_legacy.json create mode 100644 test/libsolidity/ASTJSON/assembly/switch_default.json create mode 100644 test/libsolidity/ASTJSON/assembly/switch_default.sol create mode 100644 test/libsolidity/ASTJSON/assembly/switch_default_legacy.json diff --git a/Changelog.md b/Changelog.md index d187d2527..81522f511 100644 --- a/Changelog.md +++ b/Changelog.md @@ -13,6 +13,7 @@ Bugfixes: * isoltest: Added new keyword `wei` to express function value in semantic tests * Standard-JSON-Interface: Fix a bug related to empty filenames and imports. * SMTChecker: Fix internal errors when analysing tuples. + * Yul AST Import: correctly import blocks as statements, switch statements and string literals. ### 0.6.3 (2020-02-18) diff --git a/libsolidity/ast/AsmJsonImporter.cpp b/libsolidity/ast/AsmJsonImporter.cpp index a47ea27f7..302388841 100644 --- a/libsolidity/ast/AsmJsonImporter.cpp +++ b/libsolidity/ast/AsmJsonImporter.cpp @@ -104,6 +104,8 @@ yul::Statement AsmJsonImporter::createStatement(Json::Value const& _node) return createContinue(_node); else if (nodeType == "Leave") return createLeave(_node); + else if (nodeType == "Block") + return createBlock(_node); else astAssert(false, "Invalid nodeType as statement"); } @@ -158,10 +160,9 @@ yul::Literal AsmJsonImporter::createLiteral(Json::Value const& _node) lit.value = YulString{member(_node, "value").asString()}; lit.type= YulString{member(_node, "type").asString()}; - langutil::Scanner scanner{langutil::CharStream(lit.value.str(), "")}; - if (kind == "number") { + langutil::Scanner scanner{langutil::CharStream(lit.value.str(), "")}; lit.kind = yul::LiteralKind::Number; astAssert( scanner.currentToken() == Token::Number, @@ -170,6 +171,7 @@ yul::Literal AsmJsonImporter::createLiteral(Json::Value const& _node) } else if (kind == "bool") { + langutil::Scanner scanner{langutil::CharStream(lit.value.str(), "")}; lit.kind = yul::LiteralKind::Boolean; astAssert( scanner.currentToken() == Token::TrueLiteral || @@ -180,7 +182,10 @@ yul::Literal AsmJsonImporter::createLiteral(Json::Value const& _node) else if (kind == "string") { lit.kind = yul::LiteralKind::String; - astAssert(scanner.currentToken() == Token::StringLiteral, "Expected string literal!"); + astAssert( + lit.value.str().size() <= 32, + "String literal too long (" + to_string(lit.value.str().size()) + " > 32)" + ); } else solAssert(false, "unknown type of literal"); @@ -268,7 +273,11 @@ yul::If AsmJsonImporter::createIf(Json::Value const& _node) yul::Case AsmJsonImporter::createCase(Json::Value const& _node) { auto caseStatement = createAsmNode(_node); - caseStatement.value = member(_node, "value").asString() == "default" ? nullptr : make_unique(createLiteral(member(_node, "value"))); + auto const& value = member(_node, "value"); + if (value.isString()) + astAssert(value.asString() == "default", "Expected default case"); + else + caseStatement.value = make_unique(createLiteral(value)); caseStatement.body = createBlock(member(_node, "body")); return caseStatement; } @@ -276,7 +285,7 @@ yul::Case AsmJsonImporter::createCase(Json::Value const& _node) yul::Switch AsmJsonImporter::createSwitch(Json::Value const& _node) { auto switchStatement = createAsmNode(_node); - switchStatement.expression = make_unique(createExpression(member(_node, "value"))); + switchStatement.expression = make_unique(createExpression(member(_node, "expression"))); for (auto const& var: member(_node, "cases")) switchStatement.cases.emplace_back(createCase(var)); return switchStatement; diff --git a/scripts/ASTImportTest.sh b/scripts/ASTImportTest.sh index 800c5a514..f9172df0c 100755 --- a/scripts/ASTImportTest.sh +++ b/scripts/ASTImportTest.sh @@ -10,6 +10,7 @@ SOLC=${REPO_ROOT}/${SOLIDITY_BUILD_DIR}/solc/solc SPLITSOURCES=${REPO_ROOT}/scripts/splitSources.py SYNTAXTESTS_DIR="${REPO_ROOT}/test/libsolidity/syntaxTests" +ASTJSONTESTS_DIR="${REPO_ROOT}/test/libsolidity/ASTJSON" NSOURCES="$(find $SYNTAXTESTS_DIR -type f | wc -l)" # DEV_DIR="${REPO_ROOT}/../tmp/contracts/" @@ -75,7 +76,7 @@ echo "Looking at $NSOURCES .sol files..." WORKINGDIR=$PWD # for solfile in $(find $DEV_DIR -name *.sol) -for solfile in $(find $SYNTAXTESTS_DIR -name *.sol) +for solfile in $(find $SYNTAXTESTS_DIR $ASTJSONTESTS_DIR -name *.sol) do echo -n "." # create a temporary sub-directory diff --git a/test/libsolidity/ASTJSON/assembly/empty_block.json b/test/libsolidity/ASTJSON/assembly/empty_block.json new file mode 100644 index 000000000..575b1f7f4 --- /dev/null +++ b/test/libsolidity/ASTJSON/assembly/empty_block.json @@ -0,0 +1,95 @@ +{ + "absolutePath": "a", + "exportedSymbols": + { + "C": + [ + 6 + ] + }, + "id": 7, + "nodeType": "SourceUnit", + "nodes": + [ + { + "abstract": false, + "baseContracts": [], + "contractDependencies": [], + "contractKind": "contract", + "documentation": null, + "fullyImplemented": true, + "id": 6, + "linearizedBaseContracts": + [ + 6 + ], + "name": "C", + "nodeType": "ContractDefinition", + "nodes": + [ + { + "body": + { + "id": 4, + "nodeType": "Block", + "src": "42:31:1", + "statements": + [ + { + "AST": + { + "nodeType": "YulBlock", + "src": "61:6:1", + "statements": + [ + { + "nodeType": "YulBlock", + "src": "63:2:1", + "statements": [] + } + ] + }, + "evmVersion": %EVMVERSION%, + "externalReferences": [], + "id": 3, + "nodeType": "InlineAssembly", + "src": "52:15:1" + } + ] + }, + "documentation": null, + "functionSelector": "e2179b8e", + "id": 5, + "implemented": true, + "kind": "function", + "modifiers": [], + "name": "g", + "nodeType": "FunctionDefinition", + "overrides": null, + "parameters": + { + "id": 1, + "nodeType": "ParameterList", + "parameters": [], + "src": "27:2:1" + }, + "returnParameters": + { + "id": 2, + "nodeType": "ParameterList", + "parameters": [], + "src": "42:0:1" + }, + "scope": 6, + "src": "17:56:1", + "stateMutability": "view", + "virtual": false, + "visibility": "public" + } + ], + "scope": 7, + "src": "0:75:1" + } + ], + "src": "0:76:1" +} diff --git a/test/libsolidity/ASTJSON/assembly/empty_block.sol b/test/libsolidity/ASTJSON/assembly/empty_block.sol new file mode 100644 index 000000000..23c2babbe --- /dev/null +++ b/test/libsolidity/ASTJSON/assembly/empty_block.sol @@ -0,0 +1,7 @@ +contract C { + function g() view public { + assembly { {} } + } +} + +// ---- diff --git a/test/libsolidity/ASTJSON/assembly/empty_block_legacy.json b/test/libsolidity/ASTJSON/assembly/empty_block_legacy.json new file mode 100644 index 000000000..108a76a00 --- /dev/null +++ b/test/libsolidity/ASTJSON/assembly/empty_block_legacy.json @@ -0,0 +1,123 @@ +{ + "attributes": + { + "absolutePath": "a", + "exportedSymbols": + { + "C": + [ + 6 + ] + } + }, + "children": + [ + { + "attributes": + { + "abstract": false, + "baseContracts": + [ + null + ], + "contractDependencies": + [ + null + ], + "contractKind": "contract", + "documentation": null, + "fullyImplemented": true, + "linearizedBaseContracts": + [ + 6 + ], + "name": "C", + "scope": 7 + }, + "children": + [ + { + "attributes": + { + "documentation": null, + "functionSelector": "e2179b8e", + "implemented": true, + "isConstructor": false, + "kind": "function", + "modifiers": + [ + null + ], + "name": "g", + "overrides": null, + "scope": 6, + "stateMutability": "view", + "virtual": false, + "visibility": "public" + }, + "children": + [ + { + "attributes": + { + "parameters": + [ + null + ] + }, + "children": [], + "id": 1, + "name": "ParameterList", + "src": "27:2:1" + }, + { + "attributes": + { + "parameters": + [ + null + ] + }, + "children": [], + "id": 2, + "name": "ParameterList", + "src": "42:0:1" + }, + { + "children": + [ + { + "attributes": + { + "evmVersion": %EVMVERSION%, + "externalReferences": + [ + null + ], + "operations": "{ { } }" + }, + "children": [], + "id": 3, + "name": "InlineAssembly", + "src": "52:15:1" + } + ], + "id": 4, + "name": "Block", + "src": "42:31:1" + } + ], + "id": 5, + "name": "FunctionDefinition", + "src": "17:56:1" + } + ], + "id": 6, + "name": "ContractDefinition", + "src": "0:75:1" + } + ], + "id": 7, + "name": "SourceUnit", + "src": "0:76:1" +} diff --git a/test/libsolidity/ASTJSON/assembly/switch_default.json b/test/libsolidity/ASTJSON/assembly/switch_default.json new file mode 100644 index 000000000..c0bf26438 --- /dev/null +++ b/test/libsolidity/ASTJSON/assembly/switch_default.json @@ -0,0 +1,116 @@ +{ + "absolutePath": "a", + "exportedSymbols": + { + "C": + [ + 6 + ] + }, + "id": 7, + "nodeType": "SourceUnit", + "nodes": + [ + { + "abstract": false, + "baseContracts": [], + "contractDependencies": [], + "contractKind": "contract", + "documentation": null, + "fullyImplemented": true, + "id": 6, + "linearizedBaseContracts": + [ + 6 + ], + "name": "C", + "nodeType": "ContractDefinition", + "nodes": + [ + { + "body": + { + "id": 4, + "nodeType": "Block", + "src": "42:48:1", + "statements": + [ + { + "AST": + { + "nodeType": "YulBlock", + "src": "61:23:1", + "statements": + [ + { + "cases": + [ + { + "body": + { + "nodeType": "YulBlock", + "src": "80:2:1", + "statements": [] + }, + "nodeType": "YulCase", + "src": "72:10:1", + "value": "default" + } + ], + "expression": + { + "kind": "number", + "nodeType": "YulLiteral", + "src": "70:1:1", + "type": "", + "value": "0" + }, + "nodeType": "YulSwitch", + "src": "63:19:1" + } + ] + }, + "evmVersion": %EVMVERSION%, + "externalReferences": [], + "id": 3, + "nodeType": "InlineAssembly", + "src": "52:32:1" + } + ] + }, + "documentation": null, + "functionSelector": "e2179b8e", + "id": 5, + "implemented": true, + "kind": "function", + "modifiers": [], + "name": "g", + "nodeType": "FunctionDefinition", + "overrides": null, + "parameters": + { + "id": 1, + "nodeType": "ParameterList", + "parameters": [], + "src": "27:2:1" + }, + "returnParameters": + { + "id": 2, + "nodeType": "ParameterList", + "parameters": [], + "src": "42:0:1" + }, + "scope": 6, + "src": "17:73:1", + "stateMutability": "view", + "virtual": false, + "visibility": "public" + } + ], + "scope": 7, + "src": "0:92:1" + } + ], + "src": "0:93:1" +} diff --git a/test/libsolidity/ASTJSON/assembly/switch_default.sol b/test/libsolidity/ASTJSON/assembly/switch_default.sol new file mode 100644 index 000000000..8bfba2c11 --- /dev/null +++ b/test/libsolidity/ASTJSON/assembly/switch_default.sol @@ -0,0 +1,7 @@ +contract C { + function g() view public { + assembly { switch 0 default {} } + } +} + +// ---- diff --git a/test/libsolidity/ASTJSON/assembly/switch_default_legacy.json b/test/libsolidity/ASTJSON/assembly/switch_default_legacy.json new file mode 100644 index 000000000..2d5caa0fa --- /dev/null +++ b/test/libsolidity/ASTJSON/assembly/switch_default_legacy.json @@ -0,0 +1,123 @@ +{ + "attributes": + { + "absolutePath": "a", + "exportedSymbols": + { + "C": + [ + 6 + ] + } + }, + "children": + [ + { + "attributes": + { + "abstract": false, + "baseContracts": + [ + null + ], + "contractDependencies": + [ + null + ], + "contractKind": "contract", + "documentation": null, + "fullyImplemented": true, + "linearizedBaseContracts": + [ + 6 + ], + "name": "C", + "scope": 7 + }, + "children": + [ + { + "attributes": + { + "documentation": null, + "functionSelector": "e2179b8e", + "implemented": true, + "isConstructor": false, + "kind": "function", + "modifiers": + [ + null + ], + "name": "g", + "overrides": null, + "scope": 6, + "stateMutability": "view", + "virtual": false, + "visibility": "public" + }, + "children": + [ + { + "attributes": + { + "parameters": + [ + null + ] + }, + "children": [], + "id": 1, + "name": "ParameterList", + "src": "27:2:1" + }, + { + "attributes": + { + "parameters": + [ + null + ] + }, + "children": [], + "id": 2, + "name": "ParameterList", + "src": "42:0:1" + }, + { + "children": + [ + { + "attributes": + { + "evmVersion": %EVMVERSION%, + "externalReferences": + [ + null + ], + "operations": "{\n switch 0\n default { }\n}" + }, + "children": [], + "id": 3, + "name": "InlineAssembly", + "src": "52:32:1" + } + ], + "id": 4, + "name": "Block", + "src": "42:48:1" + } + ], + "id": 5, + "name": "FunctionDefinition", + "src": "17:73:1" + } + ], + "id": 6, + "name": "ContractDefinition", + "src": "0:92:1" + } + ], + "id": 7, + "name": "SourceUnit", + "src": "0:93:1" +} From 7f38cbb91d320c56118fc79dec4d4a711c53d2e6 Mon Sep 17 00:00:00 2001 From: Daniel Kirchner Date: Fri, 6 Mar 2020 10:44:51 +0100 Subject: [PATCH 78/92] Fix calling unimplemented base function. --- Changelog.md | 1 + libsolidity/analysis/TypeChecker.cpp | 27 +++++++++---------- libsolidity/ast/Types.cpp | 10 ++++++- .../functionCall/call_unimplemented_base.sol | 14 ++++++++++ .../functionCalls/call_unimplemented_base.sol | 8 ++++++ 5 files changed, 45 insertions(+), 15 deletions(-) create mode 100644 test/libsolidity/semanticTests/functionCall/call_unimplemented_base.sol create mode 100644 test/libsolidity/syntaxTests/functionCalls/call_unimplemented_base.sol diff --git a/Changelog.md b/Changelog.md index d187d2527..693874608 100644 --- a/Changelog.md +++ b/Changelog.md @@ -10,6 +10,7 @@ Compiler Features: Bugfixes: + * Inheritance: Fix incorrect error on calling unimplemented base functions. * isoltest: Added new keyword `wei` to express function value in semantic tests * Standard-JSON-Interface: Fix a bug related to empty filenames and imports. * SMTChecker: Fix internal errors when analysing tuples. diff --git a/libsolidity/analysis/TypeChecker.cpp b/libsolidity/analysis/TypeChecker.cpp index f713ea289..5cf003bed 100644 --- a/libsolidity/analysis/TypeChecker.cpp +++ b/libsolidity/analysis/TypeChecker.cpp @@ -1703,22 +1703,21 @@ void TypeChecker::typeCheckFunctionCall( if (_functionType->kind() == FunctionType::Kind::Declaration) { - m_errorReporter.typeError( - _functionCall.location(), - "Cannot call function via contract type name." - ); + if ( + m_scope->derivesFrom(*_functionType->declaration().annotation().contract) && + !dynamic_cast(_functionType->declaration()).isImplemented() + ) + m_errorReporter.typeError( + _functionCall.location(), + "Cannot call unimplemented base function." + ); + else + m_errorReporter.typeError( + _functionCall.location(), + "Cannot call function via contract type name." + ); return; } - if (_functionType->kind() == FunctionType::Kind::Internal && _functionType->hasDeclaration()) - if (auto const* functionDefinition = dynamic_cast(&_functionType->declaration())) - // functionDefinition->annotation().contract != m_scope ensures that this is a qualified access, - // e.g. ``A.f();`` instead of a simple function call like ``f();`` (the latter is valid for unimplemented - // functions). - if (functionDefinition->annotation().contract != m_scope && !functionDefinition->isImplemented()) - m_errorReporter.typeError( - _functionCall.location(), - "Cannot call unimplemented base function." - ); // Check for unsupported use of bare static call if ( diff --git a/libsolidity/ast/Types.cpp b/libsolidity/ast/Types.cpp index c4d4c9fd4..258088adf 100644 --- a/libsolidity/ast/Types.cpp +++ b/libsolidity/ast/Types.cpp @@ -3471,7 +3471,15 @@ MemberList::MemberMap TypeType::nativeMembers(ContractDefinition const* _current continue; if (!contract.isLibrary() && inDerivingScope && declaration->isVisibleInDerivedContracts()) - members.emplace_back(declaration->name(), declaration->type(), declaration); + { + if ( + auto const* functionDefinition = dynamic_cast(declaration); + functionDefinition && !functionDefinition->isImplemented() + ) + members.emplace_back(declaration->name(), declaration->typeViaContractName(), declaration); + else + members.emplace_back(declaration->name(), declaration->type(), declaration); + } else if ( (contract.isLibrary() && declaration->isVisibleAsLibraryMember()) || declaration->isVisibleViaContractTypeAccess() diff --git a/test/libsolidity/semanticTests/functionCall/call_unimplemented_base.sol b/test/libsolidity/semanticTests/functionCall/call_unimplemented_base.sol new file mode 100644 index 000000000..3041f0e4b --- /dev/null +++ b/test/libsolidity/semanticTests/functionCall/call_unimplemented_base.sol @@ -0,0 +1,14 @@ +abstract contract I +{ + function a() internal view virtual returns(uint256); +} +abstract contract V is I +{ + function b() public view returns(uint256) { return a(); } +} +contract C is V +{ + function a() internal view override returns (uint256) { return 42;} +} +// ---- +// b() -> 42 diff --git a/test/libsolidity/syntaxTests/functionCalls/call_unimplemented_base.sol b/test/libsolidity/syntaxTests/functionCalls/call_unimplemented_base.sol new file mode 100644 index 000000000..82c6eecb9 --- /dev/null +++ b/test/libsolidity/syntaxTests/functionCalls/call_unimplemented_base.sol @@ -0,0 +1,8 @@ +abstract contract I +{ + function a() internal view virtual returns(uint256); +} +abstract contract V is I +{ + function b() public view returns(uint256) { return a(); } +} From 092827b7ad062fd59ec294a326c6b306c4ce62e3 Mon Sep 17 00:00:00 2001 From: Djordje Mijovic Date: Thu, 5 Mar 2020 19:26:30 +0100 Subject: [PATCH 79/92] Adding sol->yul for f.selector and f.address --- libsolidity/ast/Types.cpp | 5 ++++- .../codegen/ir/IRGeneratorForStatements.cpp | 12 ++++++++++-- .../semanticTests/viaYul/function_address.sol | 17 +++++++++++++++++ .../semanticTests/viaYul/function_selector.sol | 13 +++++++++++++ 4 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 test/libsolidity/semanticTests/viaYul/function_address.sol create mode 100644 test/libsolidity/semanticTests/viaYul/function_selector.sol diff --git a/libsolidity/ast/Types.cpp b/libsolidity/ast/Types.cpp index c4d4c9fd4..78a504ddc 100644 --- a/libsolidity/ast/Types.cpp +++ b/libsolidity/ast/Types.cpp @@ -2924,7 +2924,10 @@ vector> FunctionType::makeStackItems() const { case Kind::External: case Kind::DelegateCall: - slots = {make_tuple("address", TypeProvider::address()), make_tuple("functionIdentifier", TypeProvider::fixedBytes(4))}; + slots = { + make_tuple("address", TypeProvider::address()), + make_tuple("functionIdentifier", TypeProvider::uint(32)) + }; break; case Kind::BareCall: case Kind::BareCallCode: diff --git a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp index 38d16eb69..b40768be2 100644 --- a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp +++ b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp @@ -800,11 +800,19 @@ void IRGeneratorForStatements::endVisit(MemberAccess const& _memberAccess) case Type::Category::Function: if (member == "selector") { - solUnimplementedAssert(false, ""); + solUnimplementedAssert( + dynamic_cast(*_memberAccess.expression().annotation().type).kind() == + FunctionType::Kind::External, "" + ); + define(IRVariable{_memberAccess}, IRVariable(_memberAccess.expression()).part("functionIdentifier")); } else if (member == "address") { - solUnimplementedAssert(false, ""); + solUnimplementedAssert( + dynamic_cast(*_memberAccess.expression().annotation().type).kind() == + FunctionType::Kind::External, "" + ); + define(IRVariable{_memberAccess}, IRVariable(_memberAccess.expression()).part("address")); } else solAssert( diff --git a/test/libsolidity/semanticTests/viaYul/function_address.sol b/test/libsolidity/semanticTests/viaYul/function_address.sol new file mode 100644 index 000000000..402ad1481 --- /dev/null +++ b/test/libsolidity/semanticTests/viaYul/function_address.sol @@ -0,0 +1,17 @@ +contract C { + function f() external returns (address) { + return this.f.address; + } + function g() external returns (bool) { + return this.f.address == address(this); + } + function h(function() external a) public returns (address) { + return a.address; + } +} +// ==== +// compileViaYul: also +// ---- +// f() -> 0x0fdd67305928fcac8d213d1e47bfa6165cd0b87b +// g() -> true +// h(function): left(0x1122334400112233445566778899AABBCCDDEEFF42424242) -> 0x1122334400112233445566778899AABBCCDDEEFF diff --git a/test/libsolidity/semanticTests/viaYul/function_selector.sol b/test/libsolidity/semanticTests/viaYul/function_selector.sol new file mode 100644 index 000000000..40faef1dd --- /dev/null +++ b/test/libsolidity/semanticTests/viaYul/function_selector.sol @@ -0,0 +1,13 @@ +contract C { + function f() external returns (bytes4) { + return this.f.selector; + } + function h(function() external a) public returns (bytes4) { + return a.selector; + } +} +// ==== +// compileViaYul: also +// ---- +// f() -> left(0x26121ff0) +// h(function): left(0x1122334400112233445566778899AABBCCDDEEFF42424242) -> left(0x42424242) From cdfb8723892d0b32bfa98de5a558f3f244d8fc9f Mon Sep 17 00:00:00 2001 From: chriseth Date: Sun, 8 Mar 2020 19:15:21 +0100 Subject: [PATCH 80/92] [DOCS] Fix pre-computation of salted address. --- docs/control-structures.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/control-structures.rst b/docs/control-structures.rst index 407d2523c..d5448d0b3 100644 --- a/docs/control-structures.rst +++ b/docs/control-structures.rst @@ -252,7 +252,7 @@ which only need to be created if there is a dispute. /// This complicated expression just tells you how the address /// can be pre-computed. It is just there for illustration. /// You actually only need ``new D{salt: salt}(arg)``. - address predictedAddress = address(bytes20(keccak256(abi.encodePacked( + address predictedAddress = address(uint(keccak256(abi.encodePacked( byte(0xff), address(this), salt, From 2b804017fe25b230e28c8c854a50259e1cb64f8c Mon Sep 17 00:00:00 2001 From: chriseth Date: Mon, 9 Mar 2020 09:42:03 +0100 Subject: [PATCH 81/92] Fix yul links. --- docs/assembly.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/assembly.rst b/docs/assembly.rst index 7e498cd5a..f908c0f57 100644 --- a/docs/assembly.rst +++ b/docs/assembly.rst @@ -11,7 +11,7 @@ You can interleave Solidity statements with inline assembly in a language close to the one of the Ethereum virtual machine. This gives you more fine-grained control, which is especially useful when you are enhancing the language by writing libraries. -The language used for inline assembly in Solidity is called `Yul `_ +The language used for inline assembly in Solidity is called :ref:`Yul ` and it is documented in its own section. This section will only cover how the inline assembly code can interface with the surrounding Solidity code. @@ -24,7 +24,7 @@ how the inline assembly code can interface with the surrounding Solidity code. An inline assembly block is marked by ``assembly { ... }``, where the code inside -the curly braces is code in the `Yul `_ language. +the curly braces is code in the :ref:`Yul ` language. The inline assembly code can access local Solidity variables as explained below. From c8cbb980026413fb88bedc9cbdafd3cd979da215 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Tue, 3 Mar 2020 12:18:26 +0100 Subject: [PATCH 82/92] [Sol2Yul] Fixes appendExternalFunctionCall for argumentStrings.size() == 0. --- libsolidity/codegen/ir/IRGeneratorForStatements.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp index 38d16eb69..f35b81f4e 100644 --- a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp +++ b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp @@ -47,6 +47,7 @@ using namespace std; using namespace solidity; using namespace solidity::util; using namespace solidity::frontend; +using namespace std::string_literals; namespace { @@ -1204,7 +1205,7 @@ void IRGeneratorForStatements::appendExternalFunctionCall( argumentTypes.emplace_back(&type(*arg)); argumentStrings.emplace_back(IRVariable(*arg).commaSeparatedList()); } - string argumentString = ", " + joinHumanReadable(argumentStrings); + string argumentString = argumentStrings.empty() ? ""s : (", " + joinHumanReadable(argumentStrings)); solUnimplementedAssert(funKind != FunctionType::Kind::ECRecover, ""); From 2153a1ef1df8a004bc3fed4c0afca25b19a2ad81 Mon Sep 17 00:00:00 2001 From: Bhargava Shastry Date: Mon, 9 Mar 2020 10:27:38 +0100 Subject: [PATCH 83/92] Update test/tools/ossfuzz/README.md Address review comments --- test/tools/ossfuzz/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/tools/ossfuzz/README.md b/test/tools/ossfuzz/README.md index c360eb9a8..944595a3f 100644 --- a/test/tools/ossfuzz/README.md +++ b/test/tools/ossfuzz/README.md @@ -6,14 +6,14 @@ We have multiple fuzzers, some based on string input and others on protobuf input. To build them, please do the following: -- Create a local docker image from `Dockerfile.ubuntu1604.clang.ossfuzz` in the `.circleci/docker` sub-directory. Please note that this step is going to take at least an hour to complete. Therefore, it is recommended to do it when you are away from computer (and the computer is plugged to power since we do not want a battery drain). +- Create a local docker image from `Dockerfile.ubuntu1604.clang.ossfuzz` in the `.circleci/docker` sub-directory. Please note that this step is likely to take at least an hour to complete. Therefore, it is recommended to do it when you are away from the computer (and the computer is plugged to power since we do not want a battery drain). ``` $ cd .circleci/docker $ docker build -t solidity-ossfuzz-local -f Dockerfile.ubuntu1604.clang.ossfuzz . ``` -- Create and login into the docker container sourced from the image built in the previous step from the solidity parent directory +- Login to the docker container sourced from the image built in the previous step from the solidity parent directory ``` ## Host From e210026e7455f61a93a2b89bdc50f4d2f5c2097a Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Tue, 3 Mar 2020 12:20:04 +0100 Subject: [PATCH 84/92] [Sol2Yul] Implements function-to-function cast. --- libsolidity/codegen/YulUtilFunctions.cpp | 29 +++++++++++++++++++ .../viaYul/conversion/function_cast.sol | 23 +++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 test/libsolidity/semanticTests/viaYul/conversion/function_cast.sol diff --git a/libsolidity/codegen/YulUtilFunctions.cpp b/libsolidity/codegen/YulUtilFunctions.cpp index 0472dd588..a2af5f777 100644 --- a/libsolidity/codegen/YulUtilFunctions.cpp +++ b/libsolidity/codegen/YulUtilFunctions.cpp @@ -1333,6 +1333,35 @@ string YulUtilFunctions::allocateMemoryArrayFunction(ArrayType const& _type) string YulUtilFunctions::conversionFunction(Type const& _from, Type const& _to) { + if (_from.category() == Type::Category::Function) + { + solAssert(_to.category() == Type::Category::Function, ""); + FunctionType const& fromType = dynamic_cast(_from); + FunctionType const& targetType = dynamic_cast(_to); + solAssert( + fromType.isImplicitlyConvertibleTo(targetType) && + fromType.sizeOnStack() == targetType.sizeOnStack() && + (fromType.kind() == FunctionType::Kind::Internal || fromType.kind() == FunctionType::Kind::External) && + fromType.kind() == targetType.kind(), + "Invalid function type conversion requested." + ); + string const functionName = + "convert_" + + _from.identifier() + + "_to_" + + _to.identifier(); + return m_functionCollector->createFunction(functionName, [&]() { + return Whiskers(R"( + function (addr, functionId) -> outAddr, outFunctionId { + outAddr := addr + outFunctionId := functionId + } + )") + ("functionName", functionName) + .render(); + }); + } + if (_from.sizeOnStack() != 1 || _to.sizeOnStack() != 1) return conversionFunctionSpecial(_from, _to); diff --git a/test/libsolidity/semanticTests/viaYul/conversion/function_cast.sol b/test/libsolidity/semanticTests/viaYul/conversion/function_cast.sol new file mode 100644 index 000000000..08462c5f8 --- /dev/null +++ b/test/libsolidity/semanticTests/viaYul/conversion/function_cast.sol @@ -0,0 +1,23 @@ +contract C { + function f(uint x) public pure returns (uint) { + return 2 * x; + } + function g() public view returns (function (uint) external returns (uint)) { + return this.f; + } + function h(uint x) public returns (uint) { + return this.g()(x) + 1; + } + function t() external view returns ( + function(uint) external returns (uint) a, + function(uint) external view returns (uint) b) { + a = this.f; + b = this.f; + } +} +// ==== +// compileViaYul: also +// ---- +// f(uint256): 2 -> 4 +// h(uint256): 2 -> 5 +// t() -> 0xFDD67305928FCAC8D213D1E47BFA6165CD0B87BB3DE648B0000000000000000, 0xFDD67305928FCAC8D213D1E47BFA6165CD0B87BB3DE648B0000000000000000 From 105d89bea8119ab0dc9780b3fbf0d02a6d0ca737 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Mon, 9 Mar 2020 11:42:34 +0100 Subject: [PATCH 85/92] Compilation fix. --- libsolidity/codegen/YulUtilFunctions.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libsolidity/codegen/YulUtilFunctions.cpp b/libsolidity/codegen/YulUtilFunctions.cpp index 38ed88f4c..83e8272fe 100644 --- a/libsolidity/codegen/YulUtilFunctions.cpp +++ b/libsolidity/codegen/YulUtilFunctions.cpp @@ -1350,7 +1350,7 @@ string YulUtilFunctions::conversionFunction(Type const& _from, Type const& _to) _from.identifier() + "_to_" + _to.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (addr, functionId) -> outAddr, outFunctionId { outAddr := addr From 37e01a19c0c24f32ecdb978aa7c17db71fd5c769 Mon Sep 17 00:00:00 2001 From: chriseth Date: Sun, 8 Mar 2020 17:27:43 +0100 Subject: [PATCH 86/92] Fix scoping following try/catch. --- Changelog.md | 1 + libsolidity/analysis/ReferencesResolver.cpp | 16 ++++++++++++++++ libsolidity/analysis/ReferencesResolver.h | 2 ++ .../libsolidity/syntaxTests/tryCatch/scoping.sol | 10 ++++++++++ 4 files changed, 29 insertions(+) create mode 100644 test/libsolidity/syntaxTests/tryCatch/scoping.sol diff --git a/Changelog.md b/Changelog.md index a86b3ac59..92e31b9dc 100644 --- a/Changelog.md +++ b/Changelog.md @@ -12,6 +12,7 @@ Compiler Features: Bugfixes: * Inheritance: Fix incorrect error on calling unimplemented base functions. * isoltest: Added new keyword `wei` to express function value in semantic tests + * Reference Resolver: Fix scoping issue following try/catch statements. * Standard-JSON-Interface: Fix a bug related to empty filenames and imports. * SMTChecker: Fix internal errors when analysing tuples. * Yul AST Import: correctly import blocks as statements, switch statements and string literals. diff --git a/libsolidity/analysis/ReferencesResolver.cpp b/libsolidity/analysis/ReferencesResolver.cpp index 4fe2d01c9..6867f049c 100644 --- a/libsolidity/analysis/ReferencesResolver.cpp +++ b/libsolidity/analysis/ReferencesResolver.cpp @@ -67,6 +67,22 @@ void ReferencesResolver::endVisit(Block const& _block) m_resolver.setScope(_block.scope()); } +bool ReferencesResolver::visit(TryCatchClause const& _tryCatchClause) +{ + if (!m_resolveInsideCode) + return false; + m_resolver.setScope(&_tryCatchClause); + return true; +} + +void ReferencesResolver::endVisit(TryCatchClause const& _tryCatchClause) +{ + if (!m_resolveInsideCode) + return; + + m_resolver.setScope(_tryCatchClause.scope()); +} + bool ReferencesResolver::visit(ForStatement const& _for) { if (!m_resolveInsideCode) diff --git a/libsolidity/analysis/ReferencesResolver.h b/libsolidity/analysis/ReferencesResolver.h index c560be31e..488562f21 100644 --- a/libsolidity/analysis/ReferencesResolver.h +++ b/libsolidity/analysis/ReferencesResolver.h @@ -70,6 +70,8 @@ private: bool visit(Block const& _block) override; void endVisit(Block const& _block) override; + bool visit(TryCatchClause const& _tryCatchClause) override; + void endVisit(TryCatchClause const& _tryCatchClause) override; bool visit(ForStatement const& _for) override; void endVisit(ForStatement const& _for) override; void endVisit(VariableDeclarationStatement const& _varDeclStatement) override; diff --git a/test/libsolidity/syntaxTests/tryCatch/scoping.sol b/test/libsolidity/syntaxTests/tryCatch/scoping.sol new file mode 100644 index 000000000..3c902c336 --- /dev/null +++ b/test/libsolidity/syntaxTests/tryCatch/scoping.sol @@ -0,0 +1,10 @@ +contract Test { + // This checks a scoping error, + // the variable "a" was not visible + // at the assignment. + function test(address _ext) external { + try Test(_ext).test(_ext) {} catch {} + uint a = 1; + a = 3; + } +} From 29b770c43439addcb29fd79ddbf6bc37dfed0f5e Mon Sep 17 00:00:00 2001 From: a3d4 Date: Tue, 18 Feb 2020 17:13:13 +0100 Subject: [PATCH 87/92] Introduced TestCase::shouldRun(). --- test/TestCase.cpp | 35 ++++++++++++++++++----------- test/TestCase.h | 9 +++++--- test/boostTest.cpp | 3 ++- test/libsolidity/SMTCheckerTest.cpp | 23 ++++++++----------- test/libsolidity/SMTCheckerTest.h | 2 -- test/libsolidity/SemanticTest.cpp | 10 +++------ test/libsolidity/SemanticTest.h | 2 -- test/libyul/SyntaxTest.cpp | 9 +++----- test/libyul/SyntaxTest.h | 5 +---- test/tools/isoltest.cpp | 3 ++- 10 files changed, 48 insertions(+), 53 deletions(-) diff --git a/test/TestCase.cpp b/test/TestCase.cpp index 01dc51e7c..6b0a16e27 100644 --- a/test/TestCase.cpp +++ b/test/TestCase.cpp @@ -15,6 +15,7 @@ along with solidity. If not, see . */ +#include #include #include @@ -52,14 +53,18 @@ bool TestCase::isTestFilename(boost::filesystem::path const& _filename) !boost::starts_with(_filename.string(), "."); } -bool TestCase::validateSettings(langutil::EVMVersion) +void TestCase::validateSettings() { if (!m_settings.empty()) throw runtime_error( "Unknown setting(s): " + util::joinHumanReadable(m_settings | boost::adaptors::map_keys) ); - return true; +} + +bool TestCase::shouldRun() +{ + return m_shouldRun; } pair, size_t> TestCase::parseSourcesAndSettingsWithLineNumbers(istream& _stream) @@ -157,20 +162,19 @@ void TestCase::expect(string::iterator& _it, string::iterator _end, string::valu ++_it; } -bool EVMVersionRestrictedTestCase::validateSettings(langutil::EVMVersion _evmVersion) +void EVMVersionRestrictedTestCase::validateSettings() { if (!m_settings.count("EVMVersion")) - return true; + return; string versionString = m_settings["EVMVersion"]; m_validatedSettings["EVMVersion"] = versionString; m_settings.erase("EVMVersion"); - if (!TestCase::validateSettings(_evmVersion)) - return false; + TestCase::validateSettings(); if (versionString.empty()) - return true; + return; string comparator; size_t versionBegin = 0; @@ -188,18 +192,23 @@ bool EVMVersionRestrictedTestCase::validateSettings(langutil::EVMVersion _evmVer if (!version) BOOST_THROW_EXCEPTION(runtime_error{"Invalid EVM version: \"" + versionString + "\""}); + langutil::EVMVersion evmVersion = solidity::test::CommonOptions::get().evmVersion(); + bool comparisonResult; if (comparator == ">") - return _evmVersion > version; + comparisonResult = evmVersion > version; else if (comparator == ">=") - return _evmVersion >= version; + comparisonResult = evmVersion >= version; else if (comparator == "<") - return _evmVersion < version; + comparisonResult = evmVersion < version; else if (comparator == "<=") - return _evmVersion <= version; + comparisonResult = evmVersion <= version; else if (comparator == "=") - return _evmVersion == version; + comparisonResult = evmVersion == version; else if (comparator == "!") - return !(_evmVersion == version); + comparisonResult = !(evmVersion == version); else BOOST_THROW_EXCEPTION(runtime_error{"Invalid EVM comparator: \"" + comparator + "\""}); + + if (!comparisonResult) + m_shouldRun = false; } diff --git a/test/TestCase.h b/test/TestCase.h index 927490537..d6afff8b8 100644 --- a/test/TestCase.h +++ b/test/TestCase.h @@ -70,10 +70,12 @@ public: /// Validates the settings, i.e. moves them from m_settings to m_validatedSettings. /// Throws a runtime exception if any setting is left at this class (i.e. unknown setting). + virtual void validateSettings(); + /// Returns true, if the test case is supported in the current environment and false /// otherwise which causes this test to be skipped. /// This might check e.g. for restrictions on the EVM version. - virtual bool validateSettings(langutil::EVMVersion /*_evmVersion*/); + bool shouldRun(); protected: std::pair, std::size_t> parseSourcesAndSettingsWithLineNumbers(std::istream& _file); @@ -102,13 +104,14 @@ protected: std::map m_settings; /// Updated settings after validation. std::map m_validatedSettings; + + bool m_shouldRun = true; }; class EVMVersionRestrictedTestCase: public TestCase { public: - /// Returns true, if the test case is supported for EVM version @arg _evmVersion, false otherwise. - bool validateSettings(langutil::EVMVersion _evmVersion) override; + void validateSettings() override; }; } diff --git a/test/boostTest.cpp b/test/boostTest.cpp index 2b5606546..3137a5085 100644 --- a/test/boostTest.cpp +++ b/test/boostTest.cpp @@ -94,7 +94,8 @@ int registerTests( { stringstream errorStream; auto testCase = _testCaseCreator(config); - if (testCase->validateSettings(solidity::test::CommonOptions::get().evmVersion())) + testCase->validateSettings(); + if (testCase->shouldRun()) switch (testCase->run(errorStream)) { case TestCase::TestResult::Success: diff --git a/test/libsolidity/SMTCheckerTest.cpp b/test/libsolidity/SMTCheckerTest.cpp index 7db668a2d..912349182 100644 --- a/test/libsolidity/SMTCheckerTest.cpp +++ b/test/libsolidity/SMTCheckerTest.cpp @@ -44,6 +44,15 @@ SMTCheckerTest::SMTCheckerTest(string const& _filename, langutil::EVMVersion _ev } else m_enabledSolvers = smt::SMTSolverChoice::All(); + + auto available = ModelChecker::availableSolvers(); + if (!available.z3) + m_enabledSolvers.z3 = false; + if (!available.cvc4) + m_enabledSolvers.cvc4 = false; + + if (m_enabledSolvers.none()) + m_shouldRun = false; } TestCase::TestResult SMTCheckerTest::run(ostream& _stream, string const& _linePrefix, bool _formatted) @@ -55,17 +64,3 @@ TestCase::TestResult SMTCheckerTest::run(ostream& _stream, string const& _linePr return printExpectationAndError(_stream, _linePrefix, _formatted) ? TestResult::Success : TestResult::Failure; } - -bool SMTCheckerTest::validateSettings(langutil::EVMVersion _evmVersion) -{ - auto available = ModelChecker::availableSolvers(); - if (!available.z3) - m_enabledSolvers.z3 = false; - if (!available.cvc4) - m_enabledSolvers.cvc4 = false; - - if (m_enabledSolvers.none()) - return false; - - return SyntaxTest::validateSettings(_evmVersion); -} diff --git a/test/libsolidity/SMTCheckerTest.h b/test/libsolidity/SMTCheckerTest.h index ce28409e6..ad38af25c 100644 --- a/test/libsolidity/SMTCheckerTest.h +++ b/test/libsolidity/SMTCheckerTest.h @@ -37,8 +37,6 @@ public: TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override; - bool validateSettings(langutil::EVMVersion _evmVersion) override; - protected: /// This is set via option SMTSolvers in the test. /// The possible options are `all`, `z3`, `cvc4`, `none`, diff --git a/test/libsolidity/SemanticTest.cpp b/test/libsolidity/SemanticTest.cpp index b2c4628da..97c83a3e5 100644 --- a/test/libsolidity/SemanticTest.cpp +++ b/test/libsolidity/SemanticTest.cpp @@ -71,6 +71,9 @@ SemanticTest::SemanticTest(string const& _filename, langutil::EVMVersion _evmVer m_settings.erase("ABIEncoderV1Only"); } + if (m_runWithABIEncoderV1Only && solidity::test::CommonOptions::get().useABIEncoderV2) + m_shouldRun = false; + if (m_settings.count("revertStrings")) { auto revertStrings = revertStringsFromString(m_settings["revertStrings"]); @@ -90,13 +93,6 @@ SemanticTest::SemanticTest(string const& _filename, langutil::EVMVersion _evmVer soltestAssert(!m_tests.empty(), "No tests specified in " + _filename); } -bool SemanticTest::validateSettings(langutil::EVMVersion _evmVersion) -{ - if (m_runWithABIEncoderV1Only && solidity::test::CommonOptions::get().useABIEncoderV2) - return false; - return EVMVersionRestrictedTestCase::validateSettings(_evmVersion); -} - TestCase::TestResult SemanticTest::run(ostream& _stream, string const& _linePrefix, bool _formatted) { for(bool compileViaYul: set{!m_runWithoutYul, m_runWithYul}) diff --git a/test/libsolidity/SemanticTest.h b/test/libsolidity/SemanticTest.h index 5cd8e5cc7..0ea486cad 100644 --- a/test/libsolidity/SemanticTest.h +++ b/test/libsolidity/SemanticTest.h @@ -44,8 +44,6 @@ public: explicit SemanticTest(std::string const& _filename, langutil::EVMVersion _evmVersion); - bool validateSettings(langutil::EVMVersion _evmVersion) override; - TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override; void printSource(std::ostream &_stream, std::string const& _linePrefix = "", bool _formatted = false) const override; void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix = "") const override; diff --git a/test/libyul/SyntaxTest.cpp b/test/libyul/SyntaxTest.cpp index be1b9990b..75e38c844 100644 --- a/test/libyul/SyntaxTest.cpp +++ b/test/libyul/SyntaxTest.cpp @@ -114,13 +114,12 @@ void SyntaxTest::parseAndAnalyze() } -bool SyntaxTest::validateSettings(langutil::EVMVersion _evmVersion) +void SyntaxTest::validateSettings() { - if (!CommonSyntaxTest::validateSettings(_evmVersion)) - return false; + CommonSyntaxTest::validateSettings(); if (!m_settings.count("dialect")) - return true; + return; string const dialect = m_settings["dialect"]; m_validatedSettings["dialect"] = dialect; @@ -134,6 +133,4 @@ bool SyntaxTest::validateSettings(langutil::EVMVersion _evmVersion) joinHumanReadable(validDialectNames(), ", ", " and ") + "." }); - - return true; } diff --git a/test/libyul/SyntaxTest.h b/test/libyul/SyntaxTest.h index e355a5932..087a6326c 100644 --- a/test/libyul/SyntaxTest.h +++ b/test/libyul/SyntaxTest.h @@ -42,10 +42,7 @@ public: /// Validates the settings, i.e. moves them from m_settings to m_validatedSettings. /// Throws a runtime exception if any setting is left at this class (i.e. unknown setting). - /// Returns true, if the test case is supported in the current environment and false - /// otherwise which causes this test to be skipped. - /// This might check e.g. for restrictions on the EVM version. - bool validateSettings(langutil::EVMVersion _evmVersion) override; + void validateSettings() override; protected: void parseAndAnalyze() override; }; diff --git a/test/tools/isoltest.cpp b/test/tools/isoltest.cpp index a8c8748fb..e439aba64 100644 --- a/test/tools/isoltest.cpp +++ b/test/tools/isoltest.cpp @@ -161,7 +161,8 @@ TestTool::Result TestTool::process() (AnsiColorized(cout, formatted, {BOLD}) << m_name << ": ").flush(); m_test = m_testCaseCreator(TestCase::Config{m_path.string(), m_options.evmVersion()}); - if (m_test->validateSettings(m_options.evmVersion())) + m_test->validateSettings(); + if (m_test->shouldRun()) switch (TestCase::TestResult result = m_test->run(outputMessages, " ", formatted)) { case TestCase::TestResult::Success: From 809e3503baa8b1aa4d2bdcfb81d0ae8213fb1878 Mon Sep 17 00:00:00 2001 From: Daniel Kirchner Date: Thu, 5 Mar 2020 10:47:01 +0100 Subject: [PATCH 88/92] Control flow analysis for inline assembly. --- Changelog.md | 1 + libevmasm/SemanticInformation.cpp | 20 ++ libevmasm/SemanticInformation.h | 2 + libsolidity/analysis/ControlFlowAnalyzer.cpp | 10 +- libsolidity/analysis/ControlFlowAnalyzer.h | 4 +- libsolidity/analysis/ControlFlowBuilder.cpp | 186 +++++++++++++++--- libsolidity/analysis/ControlFlowBuilder.h | 25 ++- libsolidity/analysis/ControlFlowGraph.h | 22 ++- libyul/ControlFlowSideEffects.h | 38 ++++ libyul/Dialect.h | 2 + libyul/backends/evm/EVMDialect.cpp | 2 + libyul/backends/wasm/WasmDialect.cpp | 16 +- .../controlFlow/leave_inside_function.sol | 11 ++ .../controlFlow/leave_outside_function.sol | 10 + .../storageReturn/assembly/for_err.sol | 29 +++ .../storageReturn/assembly/for_fine.sol | 15 ++ .../storageReturn/assembly/if_err.sol | 11 ++ .../assembly/returning_function.sol | 13 ++ .../assembly/reverting_function.sol | 13 ++ .../storageReturn/assembly/stub.sol | 10 + .../storageReturn/assembly/switch_err.sol | 27 +++ .../storageReturn/assembly/switch_fine.sol | 25 +++ .../uninitializedAccess/assembly.sol | 2 +- .../assembly/double_revert.sol | 17 ++ .../unreachableCode/assembly/for_break.sol | 13 ++ .../unreachableCode/assembly/for_continue.sol | 12 ++ .../unreachableCode/assembly/return.sol | 17 ++ .../unreachableCode/assembly/revert.sol | 17 ++ test/libyul/Parser.cpp | 2 +- 29 files changed, 526 insertions(+), 46 deletions(-) create mode 100644 libyul/ControlFlowSideEffects.h create mode 100644 test/libsolidity/syntaxTests/controlFlow/leave_inside_function.sol create mode 100644 test/libsolidity/syntaxTests/controlFlow/leave_outside_function.sol create mode 100644 test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/for_err.sol create mode 100644 test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/for_fine.sol create mode 100644 test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/if_err.sol create mode 100644 test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/returning_function.sol create mode 100644 test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/reverting_function.sol create mode 100644 test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/stub.sol create mode 100644 test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/switch_err.sol create mode 100644 test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/switch_fine.sol create mode 100644 test/libsolidity/syntaxTests/controlFlow/unreachableCode/assembly/double_revert.sol create mode 100644 test/libsolidity/syntaxTests/controlFlow/unreachableCode/assembly/for_break.sol create mode 100644 test/libsolidity/syntaxTests/controlFlow/unreachableCode/assembly/for_continue.sol create mode 100644 test/libsolidity/syntaxTests/controlFlow/unreachableCode/assembly/return.sol create mode 100644 test/libsolidity/syntaxTests/controlFlow/unreachableCode/assembly/revert.sol diff --git a/Changelog.md b/Changelog.md index 192272595..0b955c0cb 100644 --- a/Changelog.md +++ b/Changelog.md @@ -3,6 +3,7 @@ Language Features: * Inline Assembly: Allow assigning to `_slot` of local storage variable pointers. * General: Deprecated `value(...)` and `gas(...)` in favor of `{value: ...}` and `{gas: ...}` + * Inline Assembly: Perform control flow analysis on inline assembly. Allows storage returns to be set in assembly only. Compiler Features: diff --git a/libevmasm/SemanticInformation.cpp b/libevmasm/SemanticInformation.cpp index b7fa069d2..4ea9a01ae 100644 --- a/libevmasm/SemanticInformation.cpp +++ b/libevmasm/SemanticInformation.cpp @@ -155,6 +155,26 @@ bool SemanticInformation::terminatesControlFlow(Instruction _instruction) } } +bool SemanticInformation::reverts(AssemblyItem const& _item) +{ + if (_item.type() != Operation) + return false; + else + return reverts(_item.instruction()); +} + +bool SemanticInformation::reverts(Instruction _instruction) +{ + switch (_instruction) + { + case Instruction::INVALID: + case Instruction::REVERT: + return true; + default: + return false; + } +} + bool SemanticInformation::isDeterministic(AssemblyItem const& _item) { if (_item.type() != Operation) diff --git a/libevmasm/SemanticInformation.h b/libevmasm/SemanticInformation.h index f08c3d73d..39a1b2439 100644 --- a/libevmasm/SemanticInformation.h +++ b/libevmasm/SemanticInformation.h @@ -47,6 +47,8 @@ struct SemanticInformation static bool altersControlFlow(AssemblyItem const& _item); static bool terminatesControlFlow(AssemblyItem const& _item); static bool terminatesControlFlow(Instruction _instruction); + static bool reverts(AssemblyItem const& _item); + static bool reverts(Instruction _instruction); /// @returns false if the value put on the stack by _item depends on anything else than /// the information in the current block header, memory, storage or stack. static bool isDeterministic(AssemblyItem const& _item); diff --git a/libsolidity/analysis/ControlFlowAnalyzer.cpp b/libsolidity/analysis/ControlFlowAnalyzer.cpp index c8a824c54..1a001eb1e 100644 --- a/libsolidity/analysis/ControlFlowAnalyzer.cpp +++ b/libsolidity/analysis/ControlFlowAnalyzer.cpp @@ -37,7 +37,7 @@ bool ControlFlowAnalyzer::visit(FunctionDefinition const& _function) { auto const& functionFlow = m_cfg.functionFlow(_function); checkUninitializedAccess(functionFlow.entry, functionFlow.exit); - checkUnreachable(functionFlow.entry, functionFlow.exit, functionFlow.revert); + checkUnreachable(functionFlow.entry, functionFlow.exit, functionFlow.revert, functionFlow.transactionReturn); } return false; } @@ -137,7 +137,7 @@ void ControlFlowAnalyzer::checkUninitializedAccess(CFGNode const* _entry, CFGNod m_errorReporter.typeError( variableOccurrence->occurrence() ? - variableOccurrence->occurrence()->location() : + *variableOccurrence->occurrence() : variableOccurrence->declaration().location(), ssl, string("This variable is of storage pointer type and can be ") + @@ -148,7 +148,7 @@ void ControlFlowAnalyzer::checkUninitializedAccess(CFGNode const* _entry, CFGNod } } -void ControlFlowAnalyzer::checkUnreachable(CFGNode const* _entry, CFGNode const* _exit, CFGNode const* _revert) const +void ControlFlowAnalyzer::checkUnreachable(CFGNode const* _entry, CFGNode const* _exit, CFGNode const* _revert, CFGNode const* _transactionReturn) const { // collect all nodes reachable from the entry point std::set reachable = util::BreadthFirstSearch{{_entry}}.run( @@ -158,10 +158,10 @@ void ControlFlowAnalyzer::checkUnreachable(CFGNode const* _entry, CFGNode const* } ).visited; - // traverse all paths backwards from exit and revert + // traverse all paths backwards from exit, revert and transaction return // and extract (valid) source locations of unreachable nodes into sorted set std::set unreachable; - util::BreadthFirstSearch{{_exit, _revert}}.run( + util::BreadthFirstSearch{{_exit, _revert, _transactionReturn}}.run( [&](CFGNode const* _node, auto&& _addChild) { if (!reachable.count(_node) && _node->location.isValid()) unreachable.insert(_node->location); diff --git a/libsolidity/analysis/ControlFlowAnalyzer.h b/libsolidity/analysis/ControlFlowAnalyzer.h index 24af990ab..a839c5679 100644 --- a/libsolidity/analysis/ControlFlowAnalyzer.h +++ b/libsolidity/analysis/ControlFlowAnalyzer.h @@ -36,9 +36,9 @@ public: private: /// Checks for uninitialized variable accesses in the control flow between @param _entry and @param _exit. void checkUninitializedAccess(CFGNode const* _entry, CFGNode const* _exit) const; - /// Checks for unreachable code, i.e. code ending in @param _exit or @param _revert + /// Checks for unreachable code, i.e. code ending in @param _exit, @param _revert or @param _transactionReturn /// that can not be reached from @param _entry. - void checkUnreachable(CFGNode const* _entry, CFGNode const* _exit, CFGNode const* _revert) const; + void checkUnreachable(CFGNode const* _entry, CFGNode const* _exit, CFGNode const* _revert, CFGNode const* _transactionReturn) const; CFG const& m_cfg; langutil::ErrorReporter& m_errorReporter; diff --git a/libsolidity/analysis/ControlFlowBuilder.cpp b/libsolidity/analysis/ControlFlowBuilder.cpp index c7504897b..f4b6b5d5f 100644 --- a/libsolidity/analysis/ControlFlowBuilder.cpp +++ b/libsolidity/analysis/ControlFlowBuilder.cpp @@ -16,6 +16,8 @@ */ #include +#include +#include using namespace solidity; using namespace solidity::langutil; @@ -26,10 +28,12 @@ ControlFlowBuilder::ControlFlowBuilder(CFG::NodeContainer& _nodeContainer, Funct m_nodeContainer(_nodeContainer), m_currentNode(_functionFlow.entry), m_returnNode(_functionFlow.exit), - m_revertNode(_functionFlow.revert) + m_revertNode(_functionFlow.revert), + m_transactionReturnNode(_functionFlow.transactionReturn) { } + unique_ptr ControlFlowBuilder::createFunctionFlow( CFG::NodeContainer& _nodeContainer, FunctionDefinition const& _function @@ -39,6 +43,7 @@ unique_ptr ControlFlowBuilder::createFunctionFlow( functionFlow->entry = _nodeContainer.newNode(); functionFlow->exit = _nodeContainer.newNode(); functionFlow->revert = _nodeContainer.newNode(); + functionFlow->transactionReturn = _nodeContainer.newNode(); ControlFlowBuilder builder(_nodeContainer, *functionFlow); builder.appendControlFlow(_function); @@ -131,17 +136,17 @@ bool ControlFlowBuilder::visit(ForStatement const& _forStatement) if (_forStatement.condition()) appendControlFlow(*_forStatement.condition()); - auto loopExpression = newLabel(); + auto postPart = newLabel(); auto nodes = splitFlow<2>(); auto afterFor = nodes[1]; m_currentNode = nodes[0]; { - BreakContinueScope scope(*this, afterFor, loopExpression); + BreakContinueScope scope(*this, afterFor, postPart); appendControlFlow(_forStatement.body()); } - placeAndConnectLabel(loopExpression); + placeAndConnectLabel(postPart); if (auto expression = _forStatement.loopExpression()) appendControlFlow(*expression); @@ -315,8 +320,7 @@ bool ControlFlowBuilder::visit(FunctionDefinition const& _functionDefinition) appendControlFlow(*returnParameter); m_returnNode->variableOccurrences.emplace_back( *returnParameter, - VariableOccurrence::Kind::Return, - nullptr + VariableOccurrence::Kind::Return ); } @@ -345,7 +349,7 @@ bool ControlFlowBuilder::visit(Return const& _return) m_currentNode->variableOccurrences.emplace_back( *returnParameter, VariableOccurrence::Kind::Assignment, - &_return + _return.location() ); } connect(m_currentNode, m_returnNode); @@ -363,18 +367,158 @@ bool ControlFlowBuilder::visit(FunctionTypeName const& _functionTypeName) bool ControlFlowBuilder::visit(InlineAssembly const& _inlineAssembly) { - solAssert(!!m_currentNode, ""); - visitNode(_inlineAssembly); - for (auto const& ref: _inlineAssembly.annotation().externalReferences) + solAssert(!!m_currentNode && !m_inlineAssembly, ""); + + m_inlineAssembly = &_inlineAssembly; + (*this)(_inlineAssembly.operations()); + m_inlineAssembly = nullptr; + + return false; +} + +void ControlFlowBuilder::visit(yul::Statement const& _statement) +{ + solAssert(m_currentNode && m_inlineAssembly, ""); + m_currentNode->location = langutil::SourceLocation::smallestCovering(m_currentNode->location, locationOf(_statement)); + ASTWalker::visit(_statement); +} + +void ControlFlowBuilder::operator()(yul::If const& _if) +{ + solAssert(m_currentNode && m_inlineAssembly, ""); + visit(*_if.condition); + + auto nodes = splitFlow<2>(); + m_currentNode = nodes[0]; + (*this)(_if.body); + nodes[0] = m_currentNode; + mergeFlow(nodes, nodes[1]); +} + +void ControlFlowBuilder::operator()(yul::Switch const& _switch) +{ + solAssert(m_currentNode && m_inlineAssembly, ""); + visit(*_switch.expression); + + auto beforeSwitch = m_currentNode; + + auto nodes = splitFlow(_switch.cases.size()); + for (size_t i = 0u; i < _switch.cases.size(); ++i) { - if (auto variableDeclaration = dynamic_cast(ref.second.declaration)) + m_currentNode = nodes[i]; + (*this)(_switch.cases[i].body); + nodes[i] = m_currentNode; + } + mergeFlow(nodes); + + bool hasDefault = util::contains_if(_switch.cases, [](yul::Case const& _case) { return !_case.value; }); + if (!hasDefault) + connect(beforeSwitch, m_currentNode); +} + +void ControlFlowBuilder::operator()(yul::ForLoop const& _forLoop) +{ + solAssert(m_currentNode && m_inlineAssembly, ""); + + (*this)(_forLoop.pre); + + auto condition = createLabelHere(); + + if (_forLoop.condition) + visit(*_forLoop.condition); + + auto loopExpression = newLabel(); + auto nodes = splitFlow<2>(); + auto afterFor = nodes[1]; + m_currentNode = nodes[0]; + + { + BreakContinueScope scope(*this, afterFor, loopExpression); + (*this)(_forLoop.body); + } + + placeAndConnectLabel(loopExpression); + + (*this)(_forLoop.post); + + connect(m_currentNode, condition); + m_currentNode = afterFor; +} + +void ControlFlowBuilder::operator()(yul::Break const&) +{ + solAssert(m_currentNode && m_inlineAssembly, ""); + solAssert(m_breakJump, ""); + connect(m_currentNode, m_breakJump); + m_currentNode = newLabel(); +} + +void ControlFlowBuilder::operator()(yul::Continue const&) +{ + solAssert(m_currentNode && m_inlineAssembly, ""); + solAssert(m_continueJump, ""); + connect(m_currentNode, m_continueJump); + m_currentNode = newLabel(); +} + +void ControlFlowBuilder::operator()(yul::Identifier const& _identifier) +{ + solAssert(m_currentNode && m_inlineAssembly, ""); + auto const& externalReferences = m_inlineAssembly->annotation().externalReferences; + if (externalReferences.count(&_identifier)) + { + if (auto const* declaration = dynamic_cast(externalReferences.at(&_identifier).declaration)) m_currentNode->variableOccurrences.emplace_back( - *variableDeclaration, - VariableOccurrence::Kind::InlineAssembly, - &_inlineAssembly + *declaration, + VariableOccurrence::Kind::Access, + _identifier.location ); } - return true; +} + +void ControlFlowBuilder::operator()(yul::Assignment const& _assignment) +{ + solAssert(m_currentNode && m_inlineAssembly, ""); + visit(*_assignment.value); + auto const& externalReferences = m_inlineAssembly->annotation().externalReferences; + for (auto const& variable: _assignment.variableNames) + if (externalReferences.count(&variable)) + if (auto const* declaration = dynamic_cast(externalReferences.at(&variable).declaration)) + m_currentNode->variableOccurrences.emplace_back( + *declaration, + VariableOccurrence::Kind::Assignment, + variable.location + ); +} + +void ControlFlowBuilder::operator()(yul::FunctionCall const& _functionCall) +{ + using namespace yul; + solAssert(m_currentNode && m_inlineAssembly, ""); + yul::ASTWalker::operator()(_functionCall); + + if (auto const *builtinFunction = m_inlineAssembly->dialect().builtin(_functionCall.functionName.name)) + if (builtinFunction->controlFlowSideEffects.terminates) + { + if (builtinFunction->controlFlowSideEffects.reverts) + connect(m_currentNode, m_revertNode); + else + connect(m_currentNode, m_transactionReturnNode); + m_currentNode = newLabel(); + } +} + +void ControlFlowBuilder::operator()(yul::FunctionDefinition const&) +{ + solAssert(m_currentNode && m_inlineAssembly, ""); + // External references cannot be accessed from within functions, so we can ignore their control flow. + // TODO: we might still want to track if they always revert or return, though. +} + +void ControlFlowBuilder::operator()(yul::Leave const&) +{ + // This has to be implemented, if we ever decide to visit functions. + solUnimplementedAssert(false, ""); } bool ControlFlowBuilder::visit(VariableDeclaration const& _variableDeclaration) @@ -384,8 +528,7 @@ bool ControlFlowBuilder::visit(VariableDeclaration const& _variableDeclaration) m_currentNode->variableOccurrences.emplace_back( _variableDeclaration, - VariableOccurrence::Kind::Declaration, - nullptr + VariableOccurrence::Kind::Declaration ); // Handle declaration with immediate assignment. @@ -393,14 +536,13 @@ bool ControlFlowBuilder::visit(VariableDeclaration const& _variableDeclaration) m_currentNode->variableOccurrences.emplace_back( _variableDeclaration, VariableOccurrence::Kind::Assignment, - _variableDeclaration.value().get() + _variableDeclaration.value()->location() ); // Function arguments are considered to be immediately assigned as well (they are "externally assigned"). else if (_variableDeclaration.isCallableOrCatchParameter() && !_variableDeclaration.isReturnParameter()) m_currentNode->variableOccurrences.emplace_back( _variableDeclaration, - VariableOccurrence::Kind::Assignment, - nullptr + VariableOccurrence::Kind::Assignment ); return true; } @@ -434,7 +576,7 @@ bool ControlFlowBuilder::visit(VariableDeclarationStatement const& _variableDecl m_currentNode->variableOccurrences.emplace_back( *var, VariableOccurrence::Kind::Assignment, - expression + expression ? std::make_optional(expression->location()) : std::optional{} ); } } @@ -452,7 +594,7 @@ bool ControlFlowBuilder::visit(Identifier const& _identifier) static_cast(_identifier).annotation().lValueRequested ? VariableOccurrence::Kind::Assignment : VariableOccurrence::Kind::Access, - &_identifier + _identifier.location() ); return true; diff --git a/libsolidity/analysis/ControlFlowBuilder.h b/libsolidity/analysis/ControlFlowBuilder.h index ecdedfbfe..ee1c3ab79 100644 --- a/libsolidity/analysis/ControlFlowBuilder.h +++ b/libsolidity/analysis/ControlFlowBuilder.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -30,7 +31,7 @@ namespace solidity::frontend { * Modifiers are not yet applied to the functions. This is done in a second * step in the CFG class. */ -class ControlFlowBuilder: private ASTConstVisitor +class ControlFlowBuilder: private ASTConstVisitor, private yul::ASTWalker { public: static std::unique_ptr createFunctionFlow( @@ -39,7 +40,10 @@ public: ); private: - explicit ControlFlowBuilder(CFG::NodeContainer& _nodeContainer, FunctionFlow const& _functionFlow); + explicit ControlFlowBuilder( + CFG::NodeContainer& _nodeContainer, + FunctionFlow const& _functionFlow + ); // Visits for constructing the control flow. bool visit(BinaryOperation const& _operation) override; @@ -62,6 +66,17 @@ private: // Visits for filling variable occurrences. bool visit(FunctionTypeName const& _functionTypeName) override; bool visit(InlineAssembly const& _inlineAssembly) override; + void visit(yul::Statement const& _statement) override; + void operator()(yul::If const& _if) override; + void operator()(yul::Switch const& _switch) override; + void operator()(yul::ForLoop const& _for) override; + void operator()(yul::Break const&) override; + void operator()(yul::Continue const&) override; + void operator()(yul::Identifier const& _identifier) override; + void operator()(yul::Assignment const& _assignment) override; + void operator()(yul::FunctionCall const& _functionCall) override; + void operator()(yul::FunctionDefinition const& _functionDefinition) override; + void operator()(yul::Leave const& _leave) override; bool visit(VariableDeclaration const& _variableDeclaration) override; bool visit(VariableDeclarationStatement const& _variableDeclarationStatement) override; bool visit(Identifier const& _identifier) override; @@ -70,6 +85,9 @@ protected: bool visitNode(ASTNode const&) override; private: + using ASTConstVisitor::visit; + using yul::ASTWalker::visit; + using yul::ASTWalker::operator(); /// Appends the control flow of @a _node to the current control flow. void appendControlFlow(ASTNode const& _node); @@ -136,6 +154,7 @@ private: CFGNode* m_currentNode = nullptr; CFGNode* m_returnNode = nullptr; CFGNode* m_revertNode = nullptr; + CFGNode* m_transactionReturnNode = nullptr; /// The current jump destination of break Statements. CFGNode* m_breakJump = nullptr; @@ -145,6 +164,8 @@ private: CFGNode* m_placeholderEntry = nullptr; CFGNode* m_placeholderExit = nullptr; + InlineAssembly const* m_inlineAssembly = nullptr; + /// Helper class that replaces the break and continue jump destinations for the /// current scope and restores the originals at the end of the scope. class BreakContinueScope diff --git a/libsolidity/analysis/ControlFlowGraph.h b/libsolidity/analysis/ControlFlowGraph.h index e40bb7623..93e2e3b89 100644 --- a/libsolidity/analysis/ControlFlowGraph.h +++ b/libsolidity/analysis/ControlFlowGraph.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -33,8 +34,8 @@ namespace solidity::frontend /** * Occurrence of a variable in a block of control flow. * Stores the declaration of the referenced variable, the - * kind of the occurrence and possibly the node at which - * it occurred. + * kind of the occurrence and possibly the source location + * at which it occurred. */ class VariableOccurrence { @@ -47,7 +48,7 @@ public: Assignment, InlineAssembly }; - VariableOccurrence(VariableDeclaration const& _declaration, Kind _kind, ASTNode const* _occurrence): + VariableOccurrence(VariableDeclaration const& _declaration, Kind _kind, std::optional const& _occurrence = {}): m_declaration(_declaration), m_occurrenceKind(_kind), m_occurrence(_occurrence) { } @@ -57,8 +58,8 @@ public: { if (m_occurrence && _rhs.m_occurrence) { - if (m_occurrence->id() < _rhs.m_occurrence->id()) return true; - if (_rhs.m_occurrence->id() < m_occurrence->id()) return false; + if (*m_occurrence < *_rhs.m_occurrence) return true; + if (*_rhs.m_occurrence < *m_occurrence) return false; } else if (_rhs.m_occurrence) return true; @@ -74,14 +75,14 @@ public: VariableDeclaration const& declaration() const { return m_declaration; } Kind kind() const { return m_occurrenceKind; }; - ASTNode const* occurrence() const { return m_occurrence; } + std::optional const& occurrence() const { return m_occurrence; } private: /// Declaration of the occurring variable. VariableDeclaration const& m_declaration; /// Kind of occurrence. Kind m_occurrenceKind = Kind::Access; - /// AST node at which the variable occurred, if available (may be nullptr). - ASTNode const* m_occurrence = nullptr; + /// Source location at which the variable occurred, if available (may be nullptr). + std::optional m_occurrence; }; /** @@ -119,6 +120,10 @@ struct FunctionFlow /// This node is empty does not have any exits, but may have multiple entries /// (e.g. all assert, require, revert and throw statements). CFGNode* revert = nullptr; + /// Transaction return node. Destination node for inline assembly "return" calls. + /// This node is empty and does not have any exits, but may have multiple entries + /// (e.g. all inline assembly return calls). + CFGNode* transactionReturn = nullptr; }; class CFG: private ASTConstVisitor @@ -140,7 +145,6 @@ public: std::vector> m_nodes; }; private: - langutil::ErrorReporter& m_errorReporter; /// Node container. diff --git a/libyul/ControlFlowSideEffects.h b/libyul/ControlFlowSideEffects.h new file mode 100644 index 000000000..5621a122b --- /dev/null +++ b/libyul/ControlFlowSideEffects.h @@ -0,0 +1,38 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#pragma once + +#include + +namespace solidity::yul +{ + +/** + * Side effects of code related to control flow. + */ +struct ControlFlowSideEffects +{ + /// If true, this code terminates the control flow. + /// State may or may not be reverted as indicated by the ``reverts`` flag. + bool terminates = false; + /// If true, this code reverts all state changes in the transaction. + /// Whenever this is true, ``terminates`` has to be true as well. + bool reverts = false; +}; + +} diff --git a/libyul/Dialect.h b/libyul/Dialect.h index c137791ee..a2a596177 100644 --- a/libyul/Dialect.h +++ b/libyul/Dialect.h @@ -22,6 +22,7 @@ #include #include +#include #include @@ -42,6 +43,7 @@ struct BuiltinFunction std::vector parameters; std::vector returns; SideEffects sideEffects; + ControlFlowSideEffects controlFlowSideEffects; /// If true, this is the msize instruction. bool isMSize = false; /// If true, can only accept literals as arguments and they cannot be moved to variables. diff --git a/libyul/backends/evm/EVMDialect.cpp b/libyul/backends/evm/EVMDialect.cpp index 27a2b85e3..ea6586323 100644 --- a/libyul/backends/evm/EVMDialect.cpp +++ b/libyul/backends/evm/EVMDialect.cpp @@ -52,6 +52,8 @@ pair createEVMFunction( f.parameters.resize(info.args); f.returns.resize(info.ret); f.sideEffects = EVMDialect::sideEffectsOfInstruction(_instruction); + f.controlFlowSideEffects.terminates = evmasm::SemanticInformation::terminatesControlFlow(_instruction); + f.controlFlowSideEffects.reverts = evmasm::SemanticInformation::reverts(_instruction); f.isMSize = _instruction == evmasm::Instruction::MSIZE; f.literalArguments = false; f.instruction = _instruction; diff --git a/libyul/backends/wasm/WasmDialect.cpp b/libyul/backends/wasm/WasmDialect.cpp index 7f2149adc..b8dc9f32f 100644 --- a/libyul/backends/wasm/WasmDialect.cpp +++ b/libyul/backends/wasm/WasmDialect.cpp @@ -99,6 +99,8 @@ WasmDialect::WasmDialect() addFunction("unreachable", {}, {}, false); m_functions["unreachable"_yulstring].sideEffects.invalidatesStorage = false; m_functions["unreachable"_yulstring].sideEffects.invalidatesMemory = false; + m_functions["unreachable"_yulstring].controlFlowSideEffects.terminates = true; + m_functions["unreachable"_yulstring].controlFlowSideEffects.reverts = true; addFunction("datasize", {i64}, {i64}, true, true); addFunction("dataoffset", {i64}, {i64}, true, true); @@ -147,7 +149,12 @@ void WasmDialect::addEthereumExternals() static string const i64{"i64"}; static string const i32{"i32"}; static string const i32ptr{"i32"}; // Uses "i32" on purpose. - struct External { string name; vector parameters; vector returns; }; + struct External { + string name; + vector parameters; + vector returns; + ControlFlowSideEffects controlFlowSideEffects = {}; + }; static vector externals{ {"getAddress", {i32ptr}, {}}, {"getExternalBalance", {i32ptr, i32ptr}, {}}, @@ -175,11 +182,11 @@ void WasmDialect::addEthereumExternals() {"log", {i32ptr, i32, i32, i32ptr, i32ptr, i32ptr, i32ptr}, {}}, {"getBlockNumber", {}, {i64}}, {"getTxOrigin", {i32ptr}, {}}, - {"finish", {i32ptr, i32}, {}}, - {"revert", {i32ptr, i32}, {}}, + {"finish", {i32ptr, i32}, {}, {true, false}}, + {"revert", {i32ptr, i32}, {}, {true, true}}, {"getReturnDataSize", {}, {i32}}, {"returnDataCopy", {i32ptr, i32, i32}, {}}, - {"selfDestruct", {i32ptr}, {}}, + {"selfDestruct", {i32ptr}, {}, {true, false}}, {"getBlockTimestamp", {}, {i64}} }; for (External const& ext: externals) @@ -193,6 +200,7 @@ void WasmDialect::addEthereumExternals() f.returns.emplace_back(YulString(p)); // TODO some of them are side effect free. f.sideEffects = SideEffects::worst(); + f.controlFlowSideEffects = ext.controlFlowSideEffects; f.isMSize = false; f.sideEffects.invalidatesStorage = (ext.name == "storageStore"); f.literalArguments = false; diff --git a/test/libsolidity/syntaxTests/controlFlow/leave_inside_function.sol b/test/libsolidity/syntaxTests/controlFlow/leave_inside_function.sol new file mode 100644 index 000000000..245106e13 --- /dev/null +++ b/test/libsolidity/syntaxTests/controlFlow/leave_inside_function.sol @@ -0,0 +1,11 @@ +contract C { + function f() public pure { + assembly { + function f() { + // Make sure this doesn't trigger the unimplemented assertion in the control flow builder. + leave + } + } + } +} +// ---- diff --git a/test/libsolidity/syntaxTests/controlFlow/leave_outside_function.sol b/test/libsolidity/syntaxTests/controlFlow/leave_outside_function.sol new file mode 100644 index 000000000..772a108e2 --- /dev/null +++ b/test/libsolidity/syntaxTests/controlFlow/leave_outside_function.sol @@ -0,0 +1,10 @@ +contract C { + function f() public pure { + assembly { + // Make sure this doesn't trigger the unimplemented assertion in the control flow builder. + leave + } + } +} +// ---- +// SyntaxError: (178-183): Keyword "leave" can only be used inside a function. diff --git a/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/for_err.sol b/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/for_err.sol new file mode 100644 index 000000000..909d4f333 --- /dev/null +++ b/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/for_err.sol @@ -0,0 +1,29 @@ +contract C { + struct S { bool f; } + S s; + function f() internal pure returns (S storage c) { + assembly { + for {} eq(0,0) { c_slot := s_slot } {} + } + } + function g() internal pure returns (S storage c) { + assembly { + for {} eq(0,1) { c_slot := s_slot } {} + } + } + function h() internal pure returns (S storage c) { + assembly { + for {} eq(0,0) {} { c_slot := s_slot } + } + } + function i() internal pure returns (S storage c) { + assembly { + for {} eq(0,1) {} { c_slot := s_slot } + } + } +} +// ---- +// TypeError: (87-98): This variable is of storage pointer type and can be returned without prior assignment, which would lead to undefined behaviour. +// TypeError: (228-239): This variable is of storage pointer type and can be returned without prior assignment, which would lead to undefined behaviour. +// TypeError: (369-380): This variable is of storage pointer type and can be returned without prior assignment, which would lead to undefined behaviour. +// TypeError: (510-521): This variable is of storage pointer type and can be returned without prior assignment, which would lead to undefined behaviour. diff --git a/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/for_fine.sol b/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/for_fine.sol new file mode 100644 index 000000000..ac3532aaf --- /dev/null +++ b/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/for_fine.sol @@ -0,0 +1,15 @@ +contract C { + struct S { bool f; } + S s; + function f() internal pure returns (S storage c) { + assembly { + for { c_slot := s_slot } iszero(0) {} {} + } + } + function g() internal pure returns (S storage c) { + assembly { + for { c_slot := s_slot } iszero(1) {} {} + } + } +} +// ---- diff --git a/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/if_err.sol b/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/if_err.sol new file mode 100644 index 000000000..2f79ee10f --- /dev/null +++ b/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/if_err.sol @@ -0,0 +1,11 @@ +contract C { + struct S { bool f; } + S s; + function f(bool flag) internal pure returns (S storage c) { + assembly { + if flag { c_slot := s_slot } + } + } +} +// ---- +// TypeError: (96-107): This variable is of storage pointer type and can be returned without prior assignment, which would lead to undefined behaviour. diff --git a/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/returning_function.sol b/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/returning_function.sol new file mode 100644 index 000000000..81b6bcfe7 --- /dev/null +++ b/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/returning_function.sol @@ -0,0 +1,13 @@ +contract C { + struct S { bool f; } + S s; + function f() internal pure returns (S storage c) { + // this should warn about unreachable code, but currently function flow is ignored + assembly { + function f() { return(0, 0) } + f() + c_slot := s_slot + } + } +} +// ---- diff --git a/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/reverting_function.sol b/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/reverting_function.sol new file mode 100644 index 000000000..4619584ef --- /dev/null +++ b/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/reverting_function.sol @@ -0,0 +1,13 @@ +contract C { + struct S { bool f; } + S s; + function f() internal pure returns (S storage c) { + // this could be allowed, but currently control flow for functions is not analysed + assembly { + function f() { revert(0, 0) } + f() + } + } +} +// ---- +// TypeError: (87-98): This variable is of storage pointer type and can be returned without prior assignment, which would lead to undefined behaviour. diff --git a/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/stub.sol b/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/stub.sol new file mode 100644 index 000000000..e5873f3d1 --- /dev/null +++ b/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/stub.sol @@ -0,0 +1,10 @@ +contract C { + struct S { bool f; } + S s; + function f() internal pure returns (S storage c) { + assembly { + c_slot := s_slot + } + } +} +// ---- diff --git a/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/switch_err.sol b/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/switch_err.sol new file mode 100644 index 000000000..0644d4b8f --- /dev/null +++ b/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/switch_err.sol @@ -0,0 +1,27 @@ +contract C { + struct S { bool f; } + S s; + function f(uint256 a) internal pure returns (S storage c) { + assembly { + switch a + case 0 { c_slot := s_slot } + } + } + function g(bool flag) internal pure returns (S storage c) { + assembly { + switch flag + case 0 { c_slot := s_slot } + case 1 { c_slot := s_slot } + } + } + function h(uint256 a) internal pure returns (S storage c) { + assembly { + switch a + case 0 { c_slot := s_slot } + default { return(0,0) } + } + } +} +// ---- +// TypeError: (96-107): This variable is of storage pointer type and can be returned without prior assignment, which would lead to undefined behaviour. +// TypeError: (256-267): This variable is of storage pointer type and can be returned without prior assignment, which would lead to undefined behaviour. diff --git a/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/switch_fine.sol b/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/switch_fine.sol new file mode 100644 index 000000000..9e73ecff1 --- /dev/null +++ b/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/switch_fine.sol @@ -0,0 +1,25 @@ +contract C { + struct S { bool f; } + S s; + function f(uint256 a) internal pure returns (S storage c) { + assembly { + switch a + default { c_slot := s_slot } + } + } + function g(bool flag) internal pure returns (S storage c) { + assembly { + switch flag + case 0 { c_slot := s_slot } + default { c_slot := s_slot } + } + } + function h(uint256 a) internal pure returns (S storage c) { + assembly { + switch a + case 0 { revert(0, 0) } + default { c_slot := s_slot } + } + } +} +// ---- diff --git a/test/libsolidity/syntaxTests/controlFlow/uninitializedAccess/assembly.sol b/test/libsolidity/syntaxTests/controlFlow/uninitializedAccess/assembly.sol index e73d9cde6..89a13717e 100644 --- a/test/libsolidity/syntaxTests/controlFlow/uninitializedAccess/assembly.sol +++ b/test/libsolidity/syntaxTests/controlFlow/uninitializedAccess/assembly.sol @@ -6,4 +6,4 @@ contract C { } } // ---- -// TypeError: (92-116): This variable is of storage pointer type and can be accessed without prior assignment, which would lead to undefined behaviour. +// TypeError: (107-113): This variable is of storage pointer type and can be accessed without prior assignment, which would lead to undefined behaviour. diff --git a/test/libsolidity/syntaxTests/controlFlow/unreachableCode/assembly/double_revert.sol b/test/libsolidity/syntaxTests/controlFlow/unreachableCode/assembly/double_revert.sol new file mode 100644 index 000000000..8a441a5b2 --- /dev/null +++ b/test/libsolidity/syntaxTests/controlFlow/unreachableCode/assembly/double_revert.sol @@ -0,0 +1,17 @@ +contract C { + function f() public pure { + assembly { + revert(0, 0) + revert(0, 0) + } + } + function g() public pure { + assembly { + revert(0, 0) + } + revert(); + } +} +// ---- +// Warning: (100-112): Unreachable code. +// Warning: (222-230): Unreachable code. diff --git a/test/libsolidity/syntaxTests/controlFlow/unreachableCode/assembly/for_break.sol b/test/libsolidity/syntaxTests/controlFlow/unreachableCode/assembly/for_break.sol new file mode 100644 index 000000000..466bf4fae --- /dev/null +++ b/test/libsolidity/syntaxTests/controlFlow/unreachableCode/assembly/for_break.sol @@ -0,0 +1,13 @@ +contract C { + function f() public pure { + assembly { + for { let a := 0} lt(a,1) { a := add(a, 1) } { + break + let b := 42 + } + } + } +} +// ---- +// Warning: (103-117): Unreachable code. +// Warning: (160-171): Unreachable code. diff --git a/test/libsolidity/syntaxTests/controlFlow/unreachableCode/assembly/for_continue.sol b/test/libsolidity/syntaxTests/controlFlow/unreachableCode/assembly/for_continue.sol new file mode 100644 index 000000000..be09967f1 --- /dev/null +++ b/test/libsolidity/syntaxTests/controlFlow/unreachableCode/assembly/for_continue.sol @@ -0,0 +1,12 @@ +contract C { + function f() public pure { + assembly { + for { let a := 0} lt(a,1) { a := add(a, 1) } { + continue + let b := 42 + } + } + } +} +// ---- +// Warning: (163-174): Unreachable code. diff --git a/test/libsolidity/syntaxTests/controlFlow/unreachableCode/assembly/return.sol b/test/libsolidity/syntaxTests/controlFlow/unreachableCode/assembly/return.sol new file mode 100644 index 000000000..9b13441b4 --- /dev/null +++ b/test/libsolidity/syntaxTests/controlFlow/unreachableCode/assembly/return.sol @@ -0,0 +1,17 @@ +contract C { + function f(uint256 y) public pure returns (uint256 x) { + assembly { + return(0, 0) + x := y + } + } + function g(uint256 y) public pure returns (uint256 x) { + assembly { + return(0, 0) + } + x = y; + } +} +// ---- +// Warning: (129-135): Unreachable code. +// Warning: (274-279): Unreachable code. diff --git a/test/libsolidity/syntaxTests/controlFlow/unreachableCode/assembly/revert.sol b/test/libsolidity/syntaxTests/controlFlow/unreachableCode/assembly/revert.sol new file mode 100644 index 000000000..66b6ae382 --- /dev/null +++ b/test/libsolidity/syntaxTests/controlFlow/unreachableCode/assembly/revert.sol @@ -0,0 +1,17 @@ +contract C { + function f(uint256 y) public pure returns (uint256 x) { + assembly { + revert(0, 0) + x := y + } + } + function g(uint256 y) public pure returns (uint256 x) { + assembly { + revert(0, 0) + } + x = y; + } +} +// ---- +// Warning: (129-135): Unreachable code. +// Warning: (274-279): Unreachable code. diff --git a/test/libyul/Parser.cpp b/test/libyul/Parser.cpp index 020f3d4c4..93b528721 100644 --- a/test/libyul/Parser.cpp +++ b/test/libyul/Parser.cpp @@ -539,7 +539,7 @@ BOOST_AUTO_TEST_CASE(builtins_analysis) { return _name == "builtin"_yulstring ? &f : nullptr; } - BuiltinFunction f{"builtin"_yulstring, vector(2), vector(3), {}}; + BuiltinFunction f{"builtin"_yulstring, vector(2), vector(3), {}, {}}; }; SimpleDialect dialect; From d541e222a2ee7194080cbe33edb842664fda6589 Mon Sep 17 00:00:00 2001 From: chriseth Date: Tue, 10 Mar 2020 09:45:13 +0100 Subject: [PATCH 89/92] Prepare changelog for 0.6.4. --- Changelog.md | 5 ++--- docs/bugs_by_version.json | 4 ++++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Changelog.md b/Changelog.md index 0b955c0cb..19e89adf5 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,8 +1,8 @@ -### 0.6.4 (unreleased) +### 0.6.4 (2020-03-10) Language Features: - * Inline Assembly: Allow assigning to `_slot` of local storage variable pointers. * General: Deprecated `value(...)` and `gas(...)` in favor of `{value: ...}` and `{gas: ...}` + * Inline Assembly: Allow assigning to `_slot` of local storage variable pointers. * Inline Assembly: Perform control flow analysis on inline assembly. Allows storage returns to be set in assembly only. @@ -13,7 +13,6 @@ Compiler Features: Bugfixes: * Inheritance: Fix incorrect error on calling unimplemented base functions. - * isoltest: Added new keyword `wei` to express function value in semantic tests * Reference Resolver: Fix scoping issue following try/catch statements. * Standard-JSON-Interface: Fix a bug related to empty filenames and imports. * SMTChecker: Fix internal errors when analysing tuples. diff --git a/docs/bugs_by_version.json b/docs/bugs_by_version.json index 3bdc7d9e5..dc32e2687 100644 --- a/docs/bugs_by_version.json +++ b/docs/bugs_by_version.json @@ -888,5 +888,9 @@ "0.6.3": { "bugs": [], "released": "2020-02-18" + }, + "0.6.4": { + "bugs": [], + "released": "2020-03-10" } } \ No newline at end of file From 5d7a37024854990f175c4e93a881b259b6be18b1 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Mon, 24 Feb 2020 18:06:58 +0100 Subject: [PATCH 90/92] YulUtilFunctions: convertionFunction() to also handle array string/memory casts. --- libsolidity/codegen/YulUtilFunctions.cpp | 34 +++++++++++++------ .../conversions/string_to_bytes.sol | 9 +++++ 2 files changed, 32 insertions(+), 11 deletions(-) create mode 100644 test/libsolidity/semanticTests/conversions/string_to_bytes.sol diff --git a/libsolidity/codegen/YulUtilFunctions.cpp b/libsolidity/codegen/YulUtilFunctions.cpp index 83e8272fe..9abe860b3 100644 --- a/libsolidity/codegen/YulUtilFunctions.cpp +++ b/libsolidity/codegen/YulUtilFunctions.cpp @@ -1469,22 +1469,34 @@ string YulUtilFunctions::conversionFunction(Type const& _from, Type const& _to) break; case Type::Category::Array: { - bool equal = _from == _to; - - if (!equal) + if (_from == _to) + body = "converted := value"; + else { ArrayType const& from = dynamic_cast(_from); ArrayType const& to = dynamic_cast(_to); - if (*from.mobileType() == *to.mobileType()) - equal = true; + switch (to.location()) + { + case DataLocation::Storage: + // Other cases are done explicitly in LValue::storeValue, and only possible by assignment. + solAssert( + (to.isPointer() || (from.isByteArray() && to.isByteArray())) && + from.location() == DataLocation::Storage, + "Invalid conversion to storage type." + ); + body = "converted := value"; + break; + case DataLocation::Memory: + // Copy the array to a free position in memory, unless it is already in memory. + solUnimplementedAssert(from.location() == DataLocation::Memory, "Not implemented yet."); + body = "converted := value"; + break; + case DataLocation::CallData: + solUnimplemented("Conversion of calldata types not yet implemented."); + break; + } } - - if (equal) - body = "converted := value"; - else - solUnimplementedAssert(false, "Array conversion not implemented."); - break; } case Type::Category::Struct: diff --git a/test/libsolidity/semanticTests/conversions/string_to_bytes.sol b/test/libsolidity/semanticTests/conversions/string_to_bytes.sol new file mode 100644 index 000000000..ca258ca26 --- /dev/null +++ b/test/libsolidity/semanticTests/conversions/string_to_bytes.sol @@ -0,0 +1,9 @@ +contract C { + function f(string memory s) public pure returns (bytes memory t) { + t = bytes(s); + } +} +// ==== +// compileViaYul: also +// ---- +// f(string): 32, 5, "Hello" -> 32, 5, "Hello" From a3d5af30c6455d12e5f4849f5cfdd18fceb90bf6 Mon Sep 17 00:00:00 2001 From: chriseth Date: Tue, 10 Mar 2020 12:55:23 +0100 Subject: [PATCH 91/92] Mention ControlFlowSideEffects explicitly to ease the burden on MSVC. --- libyul/backends/wasm/WasmDialect.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/libyul/backends/wasm/WasmDialect.cpp b/libyul/backends/wasm/WasmDialect.cpp index b8dc9f32f..c9817af76 100644 --- a/libyul/backends/wasm/WasmDialect.cpp +++ b/libyul/backends/wasm/WasmDialect.cpp @@ -149,11 +149,12 @@ void WasmDialect::addEthereumExternals() static string const i64{"i64"}; static string const i32{"i32"}; static string const i32ptr{"i32"}; // Uses "i32" on purpose. - struct External { + struct External + { string name; vector parameters; vector returns; - ControlFlowSideEffects controlFlowSideEffects = {}; + ControlFlowSideEffects controlFlowSideEffects = ControlFlowSideEffects{}; }; static vector externals{ {"getAddress", {i32ptr}, {}}, @@ -182,11 +183,11 @@ void WasmDialect::addEthereumExternals() {"log", {i32ptr, i32, i32, i32ptr, i32ptr, i32ptr, i32ptr}, {}}, {"getBlockNumber", {}, {i64}}, {"getTxOrigin", {i32ptr}, {}}, - {"finish", {i32ptr, i32}, {}, {true, false}}, - {"revert", {i32ptr, i32}, {}, {true, true}}, + {"finish", {i32ptr, i32}, {}, ControlFlowSideEffects{true, false}}, + {"revert", {i32ptr, i32}, {}, ControlFlowSideEffects{true, true}}, {"getReturnDataSize", {}, {i32}}, {"returnDataCopy", {i32ptr, i32, i32}, {}}, - {"selfDestruct", {i32ptr}, {}, {true, false}}, + {"selfDestruct", {i32ptr}, {}, ControlFlowSideEffects{true, false}}, {"getBlockTimestamp", {}, {i64}} }; for (External const& ext: externals) From 437ab3d24ce33e74eb4be2228d92fc4ddec15554 Mon Sep 17 00:00:00 2001 From: a3d4 Date: Tue, 10 Mar 2020 14:02:16 +0100 Subject: [PATCH 92/92] Fixed ControlFlowBuilder compilation error. --- libsolidity/analysis/ControlFlowBuilder.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libsolidity/analysis/ControlFlowBuilder.h b/libsolidity/analysis/ControlFlowBuilder.h index ee1c3ab79..2d0910d6a 100644 --- a/libsolidity/analysis/ControlFlowBuilder.h +++ b/libsolidity/analysis/ControlFlowBuilder.h @@ -76,7 +76,7 @@ private: void operator()(yul::Assignment const& _assignment) override; void operator()(yul::FunctionCall const& _functionCall) override; void operator()(yul::FunctionDefinition const& _functionDefinition) override; - void operator()(yul::Leave const& _leave) override; + void operator()(yul::Leave const& _leaveStatement) override; bool visit(VariableDeclaration const& _variableDeclaration) override; bool visit(VariableDeclarationStatement const& _variableDeclarationStatement) override; bool visit(Identifier const& _identifier) override;