From d7e3d3f75f8496b6a69fc02a54a9f976a0376516 Mon Sep 17 00:00:00 2001 From: Bhargava Shastry Date: Thu, 21 Jan 2021 17:53:22 +0100 Subject: [PATCH 1/3] Add visitors for automatic test case generation. --- test/tools/ossfuzz/Generators.h | 32 ++++ .../SolidityCustomMutatorInterface.cpp | 5 +- test/tools/ossfuzz/SolidityGenerator.cpp | 119 +++++++++++- test/tools/ossfuzz/SolidityGenerator.h | 175 +++++++++++++++++- 4 files changed, 320 insertions(+), 11 deletions(-) create mode 100644 test/tools/ossfuzz/Generators.h diff --git a/test/tools/ossfuzz/Generators.h b/test/tools/ossfuzz/Generators.h new file mode 100644 index 000000000..fb0a5e604 --- /dev/null +++ b/test/tools/ossfuzz/Generators.h @@ -0,0 +1,32 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ +// SPDX-License-Identifier: GPL-3.0 +/** + * Convenience macros for Solidity generator type declarations. + */ + +#pragma once + +/* + * Alphabetically sorted Generator types. + * SEP must appear between two elements and ENDSEP must + * appear after the last element. + */ +#define GENERATORLIST(MACRO, SEP, ENDSEP) \ + MACRO(PragmaGenerator) SEP \ + MACRO(SourceUnitGenerator) SEP \ + MACRO(TestCaseGenerator) ENDSEP diff --git a/test/tools/ossfuzz/SolidityCustomMutatorInterface.cpp b/test/tools/ossfuzz/SolidityCustomMutatorInterface.cpp index 16c71f970..fd4670e9b 100644 --- a/test/tools/ossfuzz/SolidityCustomMutatorInterface.cpp +++ b/test/tools/ossfuzz/SolidityCustomMutatorInterface.cpp @@ -27,7 +27,8 @@ using namespace solidity::test::fuzzer; // Prototype as we can't use the FuzzerInterface.h header. extern "C" size_t LLVMFuzzerMutate(uint8_t* _data, size_t _size, size_t _maxSize); extern "C" size_t LLVMFuzzerCustomMutator(uint8_t* _data, size_t size, size_t _maxSize, unsigned int seed); - +namespace +{ /// Define Solidity's custom mutator by implementing libFuzzer's /// custom mutator external interface. extern "C" size_t LLVMFuzzerCustomMutator( @@ -37,10 +38,12 @@ extern "C" size_t LLVMFuzzerCustomMutator( unsigned int _seed ) { + solAssert(_data, "libFuzzerInterface: libFuzzer supplied bad buffer"); if (_maxSize <= _size || _size == 0) return LLVMFuzzerMutate(_data, _size, _maxSize); return SolidityCustomMutatorInterface{_data, _size, _maxSize, _seed}.generate(); } +} SolidityCustomMutatorInterface::SolidityCustomMutatorInterface( uint8_t* _data, diff --git a/test/tools/ossfuzz/SolidityGenerator.cpp b/test/tools/ossfuzz/SolidityGenerator.cpp index 6e9198b87..bd00e4ce6 100644 --- a/test/tools/ossfuzz/SolidityGenerator.cpp +++ b/test/tools/ossfuzz/SolidityGenerator.cpp @@ -23,13 +23,120 @@ using namespace solidity::test::fuzzer; using namespace solidity::util; using namespace std; +using PrngUtil = solidity::test::fuzzer::GenerationProbability; + +GeneratorBase::GeneratorBase(std::shared_ptr _mutator) +{ + mutator = std::move(_mutator); + rand = mutator->randomEngine(); +} + +string GeneratorBase::visitChildren() +{ + ostringstream os; + // Randomise visit order + vector randomisedChildren; + for (auto child: generators) + randomisedChildren.push_back(child); + shuffle(randomisedChildren.begin(), randomisedChildren.end(), *rand); + for (auto child: randomisedChildren) + os << std::visit(GeneratorVisitor{}, child); + return os.str(); +} + +void TestCaseGenerator::setup() +{ + addGenerators( + { + mutator->generator() + } + ); +} + +string TestCaseGenerator::visit() +{ + ostringstream os; + for (unsigned i = 0; i < PrngUtil{}.distributionOneToN(s_maxSourceUnits, rand); i++) + { + string sourcePath = path(); + os << "\n" + << "==== Source: " + << sourcePath + << " ====" + << "\n"; + m_numSourceUnits++; + os << visitChildren(); + generator()->reset(); + } + return os.str(); +} + +void SourceUnitGenerator::setup() +{ + addGenerators( + { + mutator->generator(), + } + ); +} + +string SourceUnitGenerator::visit() +{ + string sourceUnit = visitChildren(); + reset(); + return sourceUnit; +} + +void SourceUnitGenerator::reset() +{ + for (auto& g: generators) + std::visit(ResetVisitor{}, g); +} + +string PragmaGenerator::visit() +{ + static constexpr const char* preamble = R"( + pragma solidity >= 0.0.0; + pragma experimental SMTChecker; + )"; + // Choose equally at random from coder v1 and v2 + string abiPragma = "pragma abicoder v" + + to_string(PrngUtil{}.distributionOneToN(2, rand)) + + ";\n"; + return preamble + abiPragma; +} + +template +shared_ptr SolidityGenerator::generator() +{ + for (auto& g: m_generators) + if (holds_alternative>(g)) + return get>(g); + solAssert(false, ""); +} + +SolidityGenerator::SolidityGenerator(unsigned _seed) +{ + m_rand = make_shared(_seed); + m_generators = {}; +} + +template +void SolidityGenerator::createGenerators() +{ + if constexpr (I < std::variant_size_v) + { + createGenerator>(); + createGenerators(); + } +} string SolidityGenerator::generateTestProgram() { - // TODO: Add generators for grammar elements of - // Solidity antlr4 grammar. Currently, the generated - // test program consists of a version pragma only. - return Whiskers(R"(pragma ;)") - ("directive", "solidity >= 0.0.0") - .render(); + createGenerators(); + for (auto& g: m_generators) + std::visit(AddDependenciesVisitor{}, g); + string program = generator()->visit(); + destroyGenerators(); + return program; } diff --git a/test/tools/ossfuzz/SolidityGenerator.h b/test/tools/ossfuzz/SolidityGenerator.h index 55c3c1f3a..d57d11e05 100644 --- a/test/tools/ossfuzz/SolidityGenerator.h +++ b/test/tools/ossfuzz/SolidityGenerator.h @@ -22,21 +22,188 @@ #pragma once +#include + +#include + +#include #include +#include +#include namespace solidity::test::fuzzer { -using RandomEngine = std::mt19937_64; +/// Forward declarations +class SolidityGenerator; -class SolidityGenerator +/// Type declarations +#define SEMICOLON() ; +#define FORWARDDECLAREGENERATORS(G) class G +GENERATORLIST(FORWARDDECLAREGENERATORS, SEMICOLON(), SEMICOLON()) +#undef FORWARDDECLAREGENERATORS +#undef SEMICOLON + +#define COMMA() , +using GeneratorPtr = std::variant< +#define VARIANTOFSHARED(G) std::shared_ptr +GENERATORLIST(VARIANTOFSHARED, COMMA(), ) +>; +#undef VARIANTOFSHARED +using Generator = std::variant< +#define VARIANTOFGENERATOR(G) G +GENERATORLIST(VARIANTOFGENERATOR, COMMA(), ) +>; +#undef VARIANTOFGENERATOR +#undef COMMA +using RandomEngine = std::mt19937_64; +using Distribution = std::uniform_int_distribution; + +struct GenerationProbability +{ + static size_t distributionOneToN(size_t _n, std::shared_ptr _rand) + { + return Distribution(1, _n)(*_rand); + } +}; + +struct AddDependenciesVisitor +{ + template + void operator()(T const& _t) + { + _t->setup(); + } +}; + +struct GeneratorVisitor +{ + template + std::string operator()(T const& _t) + { + return _t->visit(); + } +}; + +struct ResetVisitor +{ + template + void operator()(T const& _t) + { + _t->reset(); + } +}; + +struct GeneratorBase +{ + GeneratorBase(std::shared_ptr _mutator); + template + std::shared_ptr generator() + { + for (auto& g: generators) + if (std::holds_alternative>(g)) + return std::get>(g); + solAssert(false, ""); + } + /// Generator + virtual std::string visit() = 0; + std::string visitChildren(); + void addGenerators(std::set _generators) + { + for (auto& g: _generators) + generators.insert(g); + } + virtual void reset() = 0; + virtual std::string name() = 0; + virtual void setup() = 0; + virtual ~GeneratorBase() + { + generators.clear(); + } + std::shared_ptr mutator; + /// Random engine shared by Solidity mutators + std::shared_ptr rand; + std::set generators; +}; + +class TestCaseGenerator: public GeneratorBase { public: - SolidityGenerator(uint64_t _seed): m_rand(_seed) + TestCaseGenerator(std::shared_ptr _mutator): + GeneratorBase(std::move(_mutator)), + m_numSourceUnits(0) {} + void setup() override; + void reset() override {} + std::string visit() override; + std::string name() override + { + return "Test case generator"; + } +private: + std::string path() const + { + return m_sourceUnitNamePrefix + std::to_string(m_numSourceUnits) + ".sol"; + } + size_t m_numSourceUnits; + std::string const m_sourceUnitNamePrefix = "su"; + static constexpr unsigned s_maxSourceUnits = 3; +}; + +class SourceUnitGenerator: public GeneratorBase +{ +public: + SourceUnitGenerator(std::shared_ptr _mutator): + GeneratorBase(std::move(_mutator)) + {} + void setup() override; + std::string visit() override; + void reset() override; + std::string name() override { return "Source unit generator"; } +}; + +class PragmaGenerator: public GeneratorBase +{ +public: + PragmaGenerator(std::shared_ptr _mutator): + GeneratorBase(std::move(_mutator)) + {} + void setup() override {} + void reset() override {} + std::string visit() override; + std::string name() override { return "Pragma generator"; } +}; + +class SolidityGenerator: public std::enable_shared_from_this +{ +public: + explicit SolidityGenerator(unsigned _seed); + + std::string visit(); + template + std::shared_ptr generator(); + std::shared_ptr randomEngine() + { + return m_rand; + } /// @returns a pseudo randomly generated test program std::string generateTestProgram(); private: + template + void createGenerator() + { + m_generators.insert( + std::make_shared(shared_from_this()) + ); + } + template + void createGenerators(); + void destroyGenerators() + { + m_generators.clear(); + } /// Random number generator - RandomEngine const m_rand; + std::shared_ptr m_rand; + /// Sub generators + std::set m_generators; }; } From f5e0faaa37998bd828d3b2357bbb888ff0fabb07 Mon Sep 17 00:00:00 2001 From: Bhargava Shastry Date: Mon, 25 Jan 2021 14:01:43 +0100 Subject: [PATCH 2/3] Add documentation and/or comments. Co-authored-by: Leonardo --- test/tools/ossfuzz/Generators.h | 14 +++++++++++ test/tools/ossfuzz/SolidityGenerator.cpp | 16 +++++-------- test/tools/ossfuzz/SolidityGenerator.h | 30 ++++++++++++++++++++---- 3 files changed, 46 insertions(+), 14 deletions(-) diff --git a/test/tools/ossfuzz/Generators.h b/test/tools/ossfuzz/Generators.h index fb0a5e604..675984857 100644 --- a/test/tools/ossfuzz/Generators.h +++ b/test/tools/ossfuzz/Generators.h @@ -25,6 +25,20 @@ * Alphabetically sorted Generator types. * SEP must appear between two elements and ENDSEP must * appear after the last element. + * + * This macro applies another macro (MACRO) that is + * passed as input to this macro on the list of Generator + * types. Example that uses forward declaration: + * + * #define MACRO(G) class G; + * #define SEMICOLON() ; + * + * GENERATORLIST(MACRO, SEMICOLON(), SEMICOLON()) + * + * produces + * + * class PragmaGenerator;class SourceUnitGenerator;class TestCaseGenerator; + * */ #define GENERATORLIST(MACRO, SEP, ENDSEP) \ MACRO(PragmaGenerator) SEP \ diff --git a/test/tools/ossfuzz/SolidityGenerator.cpp b/test/tools/ossfuzz/SolidityGenerator.cpp index bd00e4ce6..28a69d1b1 100644 --- a/test/tools/ossfuzz/SolidityGenerator.cpp +++ b/test/tools/ossfuzz/SolidityGenerator.cpp @@ -46,11 +46,9 @@ string GeneratorBase::visitChildren() void TestCaseGenerator::setup() { - addGenerators( - { - mutator->generator() - } - ); + addGenerators({ + mutator->generator() + }); } string TestCaseGenerator::visit() @@ -73,11 +71,9 @@ string TestCaseGenerator::visit() void SourceUnitGenerator::setup() { - addGenerators( - { - mutator->generator(), - } - ); + addGenerators({ + mutator->generator(), + }); } string SourceUnitGenerator::visit() diff --git a/test/tools/ossfuzz/SolidityGenerator.h b/test/tools/ossfuzz/SolidityGenerator.h index d57d11e05..3d435d887 100644 --- a/test/tools/ossfuzz/SolidityGenerator.h +++ b/test/tools/ossfuzz/SolidityGenerator.h @@ -60,6 +60,8 @@ using Distribution = std::uniform_int_distribution; struct GenerationProbability { + /// @returns an unsigned integer in the range [1, @param _n] chosen + /// uniformly at random. static size_t distributionOneToN(size_t _n, std::shared_ptr _rand) { return Distribution(1, _n)(*_rand); @@ -104,24 +106,37 @@ struct GeneratorBase return std::get>(g); solAssert(false, ""); } - /// Generator + /// Virtual visitor that returns a string representing + /// the generation of the Solidity grammar element. virtual std::string visit() = 0; + /// Visitor that invokes child grammar elements of + /// this grammar element returning their string + /// representations. std::string visitChildren(); + /// Adds generators for child grammar elements of + /// this grammar element. void addGenerators(std::set _generators) { - for (auto& g: _generators) - generators.insert(g); + generators += _generators; } + /// Virtual reset method used to reset test state or + /// a portion of it if necessary e.g., remove scoped + /// variables. virtual void reset() = 0; + /// Virtual method to obtain string name of generator. virtual std::string name() = 0; + /// Virtual method to add generators that this grammar + /// element depends on. virtual void setup() = 0; virtual ~GeneratorBase() { generators.clear(); } + /// Shared pointer to the mutator instance std::shared_ptr mutator; /// Random engine shared by Solidity mutators std::shared_ptr rand; + /// Set of generators used by this generator. std::set generators; }; @@ -144,8 +159,11 @@ private: { return m_sourceUnitNamePrefix + std::to_string(m_numSourceUnits) + ".sol"; } + /// Number of source units in test input size_t m_numSourceUnits; + /// String prefix of source unit names std::string const m_sourceUnitNamePrefix = "su"; + /// Maximum number of source units per test input static constexpr unsigned s_maxSourceUnits = 3; }; @@ -178,14 +196,18 @@ class SolidityGenerator: public std::enable_shared_from_this public: explicit SolidityGenerator(unsigned _seed); + /// Returns a multi-source test case. std::string visit(); + /// Returns the generator of type @param T. template std::shared_ptr generator(); + /// Returns a shared ptr to underlying random + /// number generator. std::shared_ptr randomEngine() { return m_rand; } - /// @returns a pseudo randomly generated test program + /// Returns a pseudo randomly generated test case. std::string generateTestProgram(); private: template From be733d5f61d5a661fc5e1fec22bb1190c2d26ef7 Mon Sep 17 00:00:00 2001 From: Bhargava Shastry Date: Tue, 26 Jan 2021 16:11:18 +0100 Subject: [PATCH 3/3] Reorganized code for better readability. --- test/tools/ossfuzz/SolidityGenerator.cpp | 13 ++------- test/tools/ossfuzz/SolidityGenerator.h | 34 ++++++++++-------------- 2 files changed, 16 insertions(+), 31 deletions(-) diff --git a/test/tools/ossfuzz/SolidityGenerator.cpp b/test/tools/ossfuzz/SolidityGenerator.cpp index 28a69d1b1..ac903226e 100644 --- a/test/tools/ossfuzz/SolidityGenerator.cpp +++ b/test/tools/ossfuzz/SolidityGenerator.cpp @@ -64,7 +64,6 @@ string TestCaseGenerator::visit() << "\n"; m_numSourceUnits++; os << visitChildren(); - generator()->reset(); } return os.str(); } @@ -78,15 +77,7 @@ void SourceUnitGenerator::setup() string SourceUnitGenerator::visit() { - string sourceUnit = visitChildren(); - reset(); - return sourceUnit; -} - -void SourceUnitGenerator::reset() -{ - for (auto& g: generators) - std::visit(ResetVisitor{}, g); + return visitChildren(); } string PragmaGenerator::visit() @@ -132,7 +123,7 @@ string SolidityGenerator::generateTestProgram() createGenerators(); for (auto& g: m_generators) std::visit(AddDependenciesVisitor{}, g); - string program = generator()->visit(); + string program = generator()->generate(); destroyGenerators(); return program; } diff --git a/test/tools/ossfuzz/SolidityGenerator.h b/test/tools/ossfuzz/SolidityGenerator.h index 3d435d887..54ffe5462 100644 --- a/test/tools/ossfuzz/SolidityGenerator.h +++ b/test/tools/ossfuzz/SolidityGenerator.h @@ -82,16 +82,7 @@ struct GeneratorVisitor template std::string operator()(T const& _t) { - return _t->visit(); - } -}; - -struct ResetVisitor -{ - template - void operator()(T const& _t) - { - _t->reset(); + return _t->generate(); } }; @@ -106,9 +97,19 @@ struct GeneratorBase return std::get>(g); solAssert(false, ""); } + /// Returns test fragment created by this generator. + std::string generate() + { + std::string generatedCode = visit(); + endVisit(); + return generatedCode; + } /// Virtual visitor that returns a string representing /// the generation of the Solidity grammar element. virtual std::string visit() = 0; + /// Method called after visiting this generator. Used + /// for clearing state if necessary. + virtual void endVisit() {} /// Visitor that invokes child grammar elements of /// this grammar element returning their string /// representations. @@ -119,15 +120,12 @@ struct GeneratorBase { generators += _generators; } - /// Virtual reset method used to reset test state or - /// a portion of it if necessary e.g., remove scoped - /// variables. - virtual void reset() = 0; /// Virtual method to obtain string name of generator. virtual std::string name() = 0; /// Virtual method to add generators that this grammar - /// element depends on. - virtual void setup() = 0; + /// element depends on. If not overridden, there are + /// no dependencies. + virtual void setup() {} virtual ~GeneratorBase() { generators.clear(); @@ -148,7 +146,6 @@ public: m_numSourceUnits(0) {} void setup() override; - void reset() override {} std::string visit() override; std::string name() override { @@ -175,7 +172,6 @@ public: {} void setup() override; std::string visit() override; - void reset() override; std::string name() override { return "Source unit generator"; } }; @@ -185,8 +181,6 @@ public: PragmaGenerator(std::shared_ptr _mutator): GeneratorBase(std::move(_mutator)) {} - void setup() override {} - void reset() override {} std::string visit() override; std::string name() override { return "Pragma generator"; } };