Merge remote-tracking branch 'origin/develop' into breaking

This commit is contained in:
chriseth
2020-12-14 11:33:40 +01:00
112 changed files with 950 additions and 226 deletions
+5
View File
@@ -35,6 +35,9 @@ namespace po = boost::program_options;
namespace solidity::test
{
namespace
{
auto const description = R"(isoltest, tool for interactively managing test contracts.
Usage: isoltest [Options]
Interactively validates test contracts.
@@ -51,6 +54,8 @@ std::string editorPath()
return std::string{};
}
}
IsolTestOptions::IsolTestOptions(std::string* _editor):
CommonOptions(description)
{
+22 -4
View File
@@ -28,10 +28,28 @@ using namespace std;
static constexpr size_t abiCoderHeapSize = 1024 * 512;
DEFINE_PROTO_FUZZER(Contract const&)
DEFINE_PROTO_FUZZER(Contract const& _contract)
{
ProtoConverter converter;
string contractSource = converter.contractToString(_contract);
if (const char* dump_path = getenv("PROTO_FUZZER_DUMP_PATH"))
{
// With libFuzzer binary run this to generate the solidity source file x.sol from a proto input:
// PROTO_FUZZER_DUMP_PATH=x.sol ./a.out proto-input
ofstream of(dump_path);
of << contractSource;
}
string typeString = converter.isabelleTypeString();
string valueString = converter.isabelleValueString();
std::cout << typeString << std::endl;
std::cout << valueString << std::endl;
abicoder::ABICoder coder(abiCoderHeapSize);
auto [encodeStatus, encodedData] = coder.encode("bool", "true");
solAssert(encodeStatus, "Isabelle abicoder fuzzer: Encoding failed");
if (!typeString.empty())
{
auto [encodeStatus, encodedData] = coder.encode(typeString, valueString);
solAssert(encodeStatus, "Isabelle abicoder fuzzer: Encoding failed");
}
return;
}
}
+10 -11
View File
@@ -10,14 +10,13 @@ add_dependencies(ossfuzz
strictasm_assembly_ossfuzz
)
if (OSSFUZZ)
add_custom_target(ossfuzz_proto)
add_dependencies(ossfuzz_proto
sol_proto_ossfuzz
yul_proto_ossfuzz
yul_proto_diff_ossfuzz
yul_proto_diff_custom_mutate_ossfuzz
sol_proto_ossfuzz
yul_proto_ossfuzz
yul_proto_diff_ossfuzz
yul_proto_diff_custom_mutate_ossfuzz
)
add_custom_target(ossfuzz_abiv2)
@@ -85,7 +84,7 @@ if (OSSFUZZ)
protobuf.a
)
set_target_properties(yul_proto_ossfuzz PROPERTIES LINK_FLAGS ${LIB_FUZZING_ENGINE})
target_compile_options(yul_proto_ossfuzz PUBLIC ${COMPILE_OPTIONS} -Wno-sign-conversion)
target_compile_options(yul_proto_ossfuzz PUBLIC ${COMPILE_OPTIONS} -Wno-sign-conversion -Wno-suggest-destructor-override -Wno-inconsistent-missing-destructor-override)
add_executable(yul_proto_diff_ossfuzz yulProto_diff_ossfuzz.cpp yulFuzzerCommon.cpp protoToYul.cpp yulProto.pb.cc)
target_include_directories(yul_proto_diff_ossfuzz PRIVATE /usr/include/libprotobuf-mutator)
@@ -96,7 +95,7 @@ if (OSSFUZZ)
protobuf.a
)
set_target_properties(yul_proto_diff_ossfuzz PROPERTIES LINK_FLAGS ${LIB_FUZZING_ENGINE})
target_compile_options(yul_proto_diff_ossfuzz PUBLIC ${COMPILE_OPTIONS} -Wno-sign-conversion)
target_compile_options(yul_proto_diff_ossfuzz PUBLIC ${COMPILE_OPTIONS} -Wno-sign-conversion -Wno-suggest-destructor-override -Wno-inconsistent-missing-destructor-override)
add_executable(yul_proto_diff_custom_mutate_ossfuzz
yulProto_diff_ossfuzz.cpp
@@ -113,7 +112,7 @@ if (OSSFUZZ)
protobuf.a
)
set_target_properties(yul_proto_diff_custom_mutate_ossfuzz PROPERTIES LINK_FLAGS ${LIB_FUZZING_ENGINE})
target_compile_options(yul_proto_diff_custom_mutate_ossfuzz PUBLIC ${COMPILE_OPTIONS} -Wno-sign-conversion)
target_compile_options(yul_proto_diff_custom_mutate_ossfuzz PUBLIC ${COMPILE_OPTIONS} -Wno-sign-conversion -Wno-suggest-destructor-override -Wno-inconsistent-missing-destructor-override)
add_executable(abiv2_proto_ossfuzz
../../EVMHost.cpp
@@ -133,7 +132,7 @@ if (OSSFUZZ)
protobuf.a
)
set_target_properties(abiv2_proto_ossfuzz PROPERTIES LINK_FLAGS ${LIB_FUZZING_ENGINE})
target_compile_options(abiv2_proto_ossfuzz PUBLIC ${COMPILE_OPTIONS} -Wno-sign-conversion)
target_compile_options(abiv2_proto_ossfuzz PUBLIC ${COMPILE_OPTIONS} -Wno-sign-conversion -Wno-suggest-destructor-override -Wno-inconsistent-missing-destructor-override)
add_executable(abiv2_isabelle_ossfuzz
../../EVMHost.cpp
@@ -155,7 +154,7 @@ if (OSSFUZZ)
gmp
)
set_target_properties(abiv2_isabelle_ossfuzz PROPERTIES LINK_FLAGS ${LIB_FUZZING_ENGINE})
target_compile_options(abiv2_isabelle_ossfuzz PUBLIC ${COMPILE_OPTIONS} -Wno-sign-conversion)
target_compile_options(abiv2_isabelle_ossfuzz PUBLIC ${COMPILE_OPTIONS} -Wno-sign-conversion -Wno-suggest-destructor-override -Wno-inconsistent-missing-destructor-override)
add_executable(sol_proto_ossfuzz
solProtoFuzzer.cpp
@@ -175,7 +174,7 @@ if (OSSFUZZ)
protobuf.a
)
set_target_properties(sol_proto_ossfuzz PROPERTIES LINK_FLAGS ${LIB_FUZZING_ENGINE})
target_compile_options(sol_proto_ossfuzz PUBLIC ${COMPILE_OPTIONS} -Wno-sign-conversion)
target_compile_options(sol_proto_ossfuzz PUBLIC ${COMPILE_OPTIONS} -Wno-sign-conversion -Wno-suggest-destructor-override -Wno-inconsistent-missing-destructor-override)
else()
add_library(solc_opt_ossfuzz
solc_opt_ossfuzz.cpp
@@ -24,10 +24,9 @@
using namespace std;
using namespace solidity::test::fuzzer;
namespace
{
/// Forward declare libFuzzer's default mutator definition
// Prototype as we can't use the FuzzerInterface.h header.
extern "C" size_t LLVMFuzzerMutate(uint8_t* _data, size_t _size, size_t _maxSize);
extern "C" size_t LLVMFuzzerCustomMutator(uint8_t* _data, size_t size, size_t _maxSize, unsigned int seed);
/// Define Solidity's custom mutator by implementing libFuzzer's
/// custom mutator external interface.
@@ -42,7 +41,6 @@ extern "C" size_t LLVMFuzzerCustomMutator(
return LLVMFuzzerMutate(_data, _size, _maxSize);
return SolidityCustomMutatorInterface{_data, _size, _maxSize, _seed}.generate();
}
}
SolidityCustomMutatorInterface::SolidityCustomMutatorInterface(
uint8_t* _data,
+1 -1
View File
@@ -32,4 +32,4 @@ string SolidityGenerator::generateTestProgram()
return Whiskers(R"(pragma <directive>;)")
("directive", "solidity >= 0.0.0")
.render();
}
}
+1 -1
View File
@@ -39,4 +39,4 @@ private:
/// Random number generator
RandomEngine const m_rand;
};
}
}
+1 -1
View File
@@ -82,4 +82,4 @@ evmc::result AbiV2Utility::deployContract(EVMHost& _hostContext, bytes const& _c
evmc_message message = initializeMessage(_code);
message.kind = EVMC_CREATE;
return _hostContext.call(message);
}
}
+2 -4
View File
@@ -32,10 +32,8 @@ message FixedByteType {
required uint32 width = 1;
}
// address, address payable
message AddressType {
required bool payable = 1;
}
// address
message AddressType {}
message ValueType {
oneof value_type_oneof {
+4 -1
View File
@@ -20,6 +20,9 @@
using namespace std;
// Prototype as we can't use the FuzzerInterface.h header.
extern "C" int LLVMFuzzerTestOneInput(uint8_t const* _data, size_t _size);
extern "C" int LLVMFuzzerTestOneInput(uint8_t const* _data, size_t _size)
{
if (_size <= 250)
@@ -28,4 +31,4 @@ extern "C" int LLVMFuzzerTestOneInput(uint8_t const* _data, size_t _size)
FuzzerUtil::testConstantOptimizer(input, /*quiet=*/true);
}
return 0;
}
}
+91 -10
View File
@@ -1,6 +1,7 @@
#include <test/tools/ossfuzz/protoToAbiV2.h>
#include <boost/preprocessor.hpp>
#include <regex>
/// Convenience macros
/// Returns a valid Solidity integer width w such that 8 <= w <= 256.
@@ -315,6 +316,10 @@ pair<string, string> ProtoConverter::assignChecker(
m_counter += acVisitor.counted();
m_checks << assignCheckStrPair.second;
appendToIsabelleValueString(
acVisitor.isabelleValueString(),
((m_varCounter == 1) ? Delimiter::SKIP : Delimiter::ADD)
);
// State variables cannot be assigned in contract-scope
// Therefore, we buffer their assignments and
@@ -333,12 +338,12 @@ std::string ProtoConverter::equalityChecksAsString()
return m_checks.str();
}
std::string ProtoConverter::delimiterToString(Delimiter _delimiter)
std::string ProtoConverter::delimiterToString(Delimiter _delimiter, bool _space)
{
switch (_delimiter)
{
case Delimiter::ADD:
return ", ";
return _space ? ", " : ",";
case Delimiter::SKIP:
return "";
}
@@ -448,7 +453,15 @@ void ProtoConverter::appendToIsabelleTypeString(
Delimiter _delimiter
)
{
m_isabelleTypeString << delimiterToString(_delimiter) << _typeString;
m_isabelleTypeString << delimiterToString(_delimiter, false) << _typeString;
}
void ProtoConverter::appendToIsabelleValueString(
std::string const& _valueString,
Delimiter _delimiter
)
{
m_isabelleValueString << delimiterToString(_delimiter, false) << _valueString;
}
std::string ProtoConverter::typedParametersAsString(CalleeType _calleeType)
@@ -736,6 +749,15 @@ string ProtoConverter::isabelleTypeString() const
return typeString;
}
string ProtoConverter::isabelleValueString() const
{
string valueString = m_isabelleValueString.str();
if (!valueString.empty())
return "(" + valueString + ")";
else
return valueString;
}
string ProtoConverter::contractToString(Contract const& _input)
{
visit(_input);
@@ -777,9 +799,9 @@ string TypeVisitor::visit(FixedByteType const& _type)
return m_baseType;
}
string TypeVisitor::visit(AddressType const& _type)
string TypeVisitor::visit(AddressType const&)
{
m_baseType = getAddressTypeAsString(_type);
m_baseType = "address";
m_structTupleString.addTypeStringToTuple(m_baseType);
return m_baseType;
}
@@ -890,33 +912,61 @@ string TypeVisitor::visit(StructType const& _type)
}
/// AssignCheckVisitor implementation
void AssignCheckVisitor::ValueStream::appendValue(string& _value)
{
solAssert(!_value.empty(), "Abiv2 fuzzer: Empty value");
index++;
if (index > 1)
stream << ",";
stream << _value;
}
pair<string, string> AssignCheckVisitor::visit(BoolType const& _type)
{
string value = ValueGetterVisitor(counter()).visit(_type);
if (!m_forcedVisit)
m_valueStream.appendValue(value);
return assignAndCheckStringPair(m_varName, m_paramName, value, value, DataType::VALUE);
}
pair<string, string> AssignCheckVisitor::visit(IntegerType const& _type)
{
string value = ValueGetterVisitor(counter()).visit(_type);
if (!m_forcedVisit)
m_valueStream.appendValue(value);
return assignAndCheckStringPair(m_varName, m_paramName, value, value, DataType::VALUE);
}
pair<string, string> AssignCheckVisitor::visit(FixedByteType const& _type)
{
string value = ValueGetterVisitor(counter()).visit(_type);
if (!m_forcedVisit)
{
string isabelleValue = ValueGetterVisitor{}.isabelleBytesValueAsString(value);
m_valueStream.appendValue(isabelleValue);
}
return assignAndCheckStringPair(m_varName, m_paramName, value, value, DataType::VALUE);
}
pair<string, string> AssignCheckVisitor::visit(AddressType const& _type)
{
string value = ValueGetterVisitor(counter()).visit(_type);
if (!m_forcedVisit)
{
string isabelleValue = ValueGetterVisitor{}.isabelleAddressValueAsString(value);
m_valueStream.appendValue(isabelleValue);
}
return assignAndCheckStringPair(m_varName, m_paramName, value, value, DataType::VALUE);
}
pair<string, string> AssignCheckVisitor::visit(DynamicByteArrayType const& _type)
{
string value = ValueGetterVisitor(counter()).visit(_type);
if (!m_forcedVisit)
{
string isabelleValue = ValueGetterVisitor{}.isabelleBytesValueAsString(value);
m_valueStream.appendValue(isabelleValue);
}
DataType dataType = _type.type() == DynamicByteArrayType::BYTES ? DataType::BYTES : DataType::STRING;
return assignAndCheckStringPair(m_varName, m_paramName, value, value, dataType);
}
@@ -981,6 +1031,8 @@ pair<string, string> AssignCheckVisitor::visit(ArrayType const& _type)
pair<string, string> assignCheckBuffer;
string wasVarName = m_varName;
string wasParamName = m_paramName;
if (!m_forcedVisit)
m_valueStream.startArray();
for (unsigned i = 0; i < length; i++)
{
m_varName = wasVarName + "[" + to_string(i) + "]";
@@ -991,12 +1043,18 @@ pair<string, string> AssignCheckVisitor::visit(ArrayType const& _type)
if (i < length - 1)
m_structCounter = wasStructCounter;
}
// Since struct visitor won't be called for zero-length
// arrays, struct counter will not get incremented. Therefore,
// we need to manually force a recursive struct visit.
if (length == 0 && TypeVisitor().arrayOfStruct(_type))
{
bool previousState = m_forcedVisit;
m_forcedVisit = true;
visit(_type.t());
m_forcedVisit = previousState;
}
if (!m_forcedVisit)
m_valueStream.endArray();
m_varName = wasVarName;
m_paramName = wasParamName;
@@ -1022,6 +1080,8 @@ pair<string, string> AssignCheckVisitor::visit(StructType const& _type)
string wasVarName = m_varName;
string wasParamName = m_paramName;
if (!m_forcedVisit)
m_valueStream.startStruct();
for (auto const& t: _type.t())
{
m_varName = wasVarName + ".m" + to_string(i);
@@ -1035,6 +1095,8 @@ pair<string, string> AssignCheckVisitor::visit(StructType const& _type)
assignCheckBuffer.second += assign.second;
i++;
}
if (!m_forcedVisit)
m_valueStream.endStruct();
m_varName = wasVarName;
m_paramName = wasParamName;
return assignCheckBuffer;
@@ -1117,11 +1179,11 @@ string ValueGetterVisitor::visit(AddressType const&)
return addressValueAsString(counter());
}
string ValueGetterVisitor::visit(DynamicByteArrayType const& _type)
string ValueGetterVisitor::visit(DynamicByteArrayType const&)
{
return bytesArrayValueAsString(
counter(),
getDataTypeOfDynBytesType(_type) == DataType::BYTES
true
);
}
@@ -1195,10 +1257,29 @@ std::string ValueGetterVisitor::fixedByteValueAsString(unsigned _width, unsigned
std::string ValueGetterVisitor::addressValueAsString(unsigned _counter)
{
// TODO: Isabelle encoder expects address literal to be exactly
return "address(" + maskUnsignedIntToHex(_counter, 40) + ")";
}
std::string ValueGetterVisitor::isabelleAddressValueAsString(std::string& _solAddressString)
{
// Isabelle encoder expects address literal to be exactly
// 20 bytes and a hex string.
// Example: 0x0102030405060708090a0102030405060708090a
return "address(" + maskUnsignedIntToHex(_counter, 40) + ")";
std::regex const addressPattern("address\\((.*)\\)");
std::smatch match;
solAssert(std::regex_match(_solAddressString, match, addressPattern), "Abiv2 fuzzer: Invalid address string");
std::string addressHex = match[1].str();
addressHex.erase(2, 24);
return addressHex;
}
std::string ValueGetterVisitor::isabelleBytesValueAsString(std::string& _solBytesString)
{
std::regex const bytesPattern("hex\"(.*)\"");
std::smatch match;
solAssert(std::regex_match(_solBytesString, match, bytesPattern), "Abiv2 fuzzer: Invalid bytes string");
std::string bytesHex = match[1].str();
return "0x" + bytesHex;
}
std::string ValueGetterVisitor::variableLengthValueAsString(
+58 -19
View File
@@ -153,6 +153,7 @@ public:
ProtoConverter(ProtoConverter&&) = delete;
std::string contractToString(Contract const& _input);
std::string isabelleTypeString() const;
std::string isabelleValueString() const;
private:
enum class Delimiter
{
@@ -295,6 +296,13 @@ private:
Delimiter _delimiter
);
/// Append @a _valueString to value string meant to be
/// passed to Isabelle coder API.
void appendToIsabelleValueString(
std::string const& _valueString,
Delimiter _delimiter
);
/// Returns a Solidity variable declaration statement
/// @param _type: string containing Solidity type of the
/// variable to be declared.
@@ -367,7 +375,7 @@ private:
}
/// Convert delimter to a comma or null string.
static std::string delimiterToString(Delimiter _delimiter);
static std::string delimiterToString(Delimiter _delimiter, bool _space = true);
/// Contains the test program
std::ostringstream m_output;
@@ -379,6 +387,9 @@ private:
std::ostringstream m_typedParamsPublic;
/// Contains type string to be passed to Isabelle API
std::ostringstream m_isabelleTypeString;
/// Contains values to be encoded in the format accepted
/// by the Isabelle API.
std::ostringstream m_isabelleValueString;
/// Contains type stream to be used in returndata coder function
/// signature
std::ostringstream m_types;
@@ -476,19 +487,6 @@ public:
return "bytes" + std::to_string(getFixedByteWidth(_x));
}
static std::string getAddressTypeAsString(AddressType const& _x)
{
return (_x.payable() ? "address payable" : "address");
}
static DataType getDataTypeOfDynBytesType(DynamicByteArrayType const& _x)
{
if (_x.type() == DynamicByteArrayType::STRING)
return DataType::STRING;
else
return DataType::BYTES;
}
// Convert _counter to string and return its keccak256 hash
static u256 hashUnsignedInt(unsigned _counter)
{
@@ -531,7 +529,13 @@ public:
// this linear equation to make the number derived from
// _counter approach a uniform distribution over
// [0, s_maxDynArrayLength]
return (_counter + 879) * 32 % (s_maxDynArrayLength + 1);
auto v = (_counter + 879) * 32 % (s_maxDynArrayLength + 1);
/// Always return an even number because Isabelle string
/// values are formatted as hex literals
if (v % 2 == 1)
return v + 1;
else
return v;
}
static std::string bytesArrayTypeAsString(DynamicByteArrayType const& _x)
@@ -658,10 +662,6 @@ private:
{
stream << ")";
}
std::string operator()()
{
return stream.str();
}
void addTypeStringToTuple(std::string& _typeString);
void addArrayBracketToType(std::string& _arrayBracket);
};
@@ -735,7 +735,42 @@ public:
{
return m_structCounter - m_structStart;
}
std::string isabelleValueString()
{
return m_valueStream.stream.str();
}
private:
struct ValueStream
{
ValueStream() = default;
unsigned index = 0;
std::ostringstream stream;
void startStruct()
{
if (index >= 1)
stream << ",";
index = 0;
stream << "(";
}
void endStruct()
{
stream << ")";
}
void startArray()
{
if (index >= 1)
stream << ",";
index = 0;
stream << "[";
}
void endArray()
{
stream << "]";
index++;
}
void appendValue(std::string& _value);
};
std::string indentation()
{
return std::string(m_indentation * 1, '\t');
@@ -764,6 +799,8 @@ private:
bool m_stateVar;
unsigned m_structCounter;
unsigned m_structStart;
ValueStream m_valueStream;
bool m_forcedVisit = false;
};
/// Returns a valid value (as a string) for a given type.
@@ -786,6 +823,8 @@ public:
solAssert(false, "ABIv2 proto fuzzer: Cannot call valuegettervisitor on complex type");
}
using AbiV2ProtoVisitor<std::string>::visit;
static std::string isabelleAddressValueAsString(std::string& _solAddressString);
static std::string isabelleBytesValueAsString(std::string& _solFixedBytesString);
private:
unsigned counter()
{
@@ -25,6 +25,9 @@
using namespace solidity::frontend::test;
using namespace std;
// Prototype as we can't use the FuzzerInterface.h header.
extern "C" int LLVMFuzzerTestOneInput(uint8_t const* _data, size_t _size);
extern "C" int LLVMFuzzerTestOneInput(uint8_t const* _data, size_t _size)
{
if (_size <= 600)
+3
View File
@@ -25,6 +25,9 @@
using namespace solidity::frontend::test;
using namespace std;
// Prototype as we can't use the FuzzerInterface.h header.
extern "C" int LLVMFuzzerTestOneInput(uint8_t const* _data, size_t _size);
extern "C" int LLVMFuzzerTestOneInput(uint8_t const* _data, size_t _size)
{
if (_size <= 600)
@@ -24,6 +24,9 @@ using namespace solidity;
using namespace solidity::yul;
using namespace std;
// Prototype as we can't use the FuzzerInterface.h header.
extern "C" int LLVMFuzzerTestOneInput(uint8_t const* _data, size_t _size);
extern "C" int LLVMFuzzerTestOneInput(uint8_t const* _data, size_t _size)
{
if (_size > 600)
@@ -43,6 +43,9 @@ using namespace solidity::util;
using namespace solidity::langutil;
using namespace solidity::yul::test::yul_fuzzer;
// Prototype as we can't use the FuzzerInterface.h header.
extern "C" int LLVMFuzzerTestOneInput(uint8_t const* _data, size_t _size);
extern "C" int LLVMFuzzerTestOneInput(uint8_t const* _data, size_t _size)
{
if (_size > 600)
@@ -24,6 +24,9 @@ using namespace solidity::util;
using namespace solidity::yul;
using namespace std;
// Prototype as we can't use the FuzzerInterface.h header.
extern "C" int LLVMFuzzerTestOneInput(uint8_t const* _data, size_t _size);
extern "C" int LLVMFuzzerTestOneInput(uint8_t const* _data, size_t _size)
{
if (_size > 600)