ABI decoder.

This commit is contained in:
chriseth 2017-09-01 13:59:21 +02:00
parent 7c69d88f93
commit bdc1ff8ec7
7 changed files with 505 additions and 4 deletions

View File

@ -2,6 +2,8 @@
Features:
* Allow constant variables to be used as array length
* Code Generator: New ABI decoder which supports structs and arbitrarily nested
arrays and checks input size (activate using ``pragma experimental ABIEncoderV2;``.
* Syntax Checker: Turn the usage of ``callcode`` into an error as experimental 0.5.0 feature.
* Type Checker: Improve address checksum warning.
* Type Checker: More detailed errors for invalid array lengths (such as division by zero).

View File

@ -22,9 +22,13 @@
#include <libsolidity/codegen/ABIFunctions.h>
#include <libsolidity/ast/AST.h>
#include <libsolidity/codegen/CompilerUtils.h>
#include <libdevcore/Whiskers.h>
#include <libsolidity/ast/AST.h>
#include <boost/algorithm/string/join.hpp>
#include <boost/range/adaptor/reversed.hpp>
using namespace std;
using namespace dev;
@ -99,6 +103,73 @@ string ABIFunctions::tupleEncoder(
});
}
string ABIFunctions::tupleDecoder(TypePointers const& _types, bool _fromMemory)
{
string functionName = string("abi_decode_tuple_");
for (auto const& t: _types)
functionName += t->identifier();
if (_fromMemory)
functionName += "_fromMemory";
return createFunction(functionName, [&]() {
solAssert(!_types.empty(), "");
TypePointers decodingTypes;
for (auto const& t: _types)
decodingTypes.emplace_back(t->decodingType());
Whiskers templ(R"(
function <functionName>(headStart, dataEnd) -> <valueReturnParams> {
switch slt(sub(dataEnd, headStart), <minimumSize>) case 1 { revert(0, 0) }
<decodeElements>
}
)");
templ("functionName", functionName);
templ("minimumSize", to_string(headSize(decodingTypes)));
string decodeElements;
vector<string> valueReturnParams;
size_t headPos = 0;
size_t stackPos = 0;
for (size_t i = 0; i < _types.size(); ++i)
{
solAssert(_types[i], "");
solAssert(decodingTypes[i], "");
size_t sizeOnStack = _types[i]->sizeOnStack();
solAssert(sizeOnStack == decodingTypes[i]->sizeOnStack(), "");
solAssert(sizeOnStack > 0, "");
vector<string> valueNamesLocal;
for (size_t j = 0; j < sizeOnStack; j++)
{
valueNamesLocal.push_back("value" + to_string(stackPos));
valueReturnParams.push_back("value" + to_string(stackPos));
stackPos++;
}
bool dynamic = decodingTypes[i]->isDynamicallyEncoded();
Whiskers elementTempl(R"(
{
let offset := )" + string(
dynamic ?
"<load>(add(headStart, <pos>))" :
"<pos>"
) + R"(
<values> := <abiDecode>(add(headStart, offset), dataEnd)
}
)");
elementTempl("load", _fromMemory ? "mload" : "calldataload");
elementTempl("values", boost::algorithm::join(valueNamesLocal, ", "));
elementTempl("pos", to_string(headPos));
elementTempl("abiDecode", abiDecodingFunction(*_types[i], _fromMemory, true));
decodeElements += elementTempl.render();
headPos += dynamic ? 0x20 : decodingTypes[i]->calldataEncodedSize();
}
templ("valueReturnParams", boost::algorithm::join(valueReturnParams, ", "));
templ("decodeElements", decodeElements);
return templ.render();
});
}
string ABIFunctions::requestedFunctions()
{
string result;
@ -141,10 +212,9 @@ string ABIFunctions::cleanupFunction(Type const& _type, bool _revertOnFailure)
solUnimplemented("Fixed point types not implemented.");
break;
case Type::Category::Array:
solAssert(false, "Array cleanup requested.");
break;
case Type::Category::Struct:
solAssert(false, "Struct cleanup requested.");
solAssert(_type.dataStoredIn(DataLocation::Storage), "Cleanup requested for non-storage reference type.");
templ("body", "cleaned := value");
break;
case Type::Category::FixedBytes:
{
@ -367,6 +437,24 @@ string ABIFunctions::combineExternalFunctionIdFunction()
});
}
string ABIFunctions::splitExternalFunctionIdFunction()
{
string functionName = "split_external_function_id";
return createFunction(functionName, [&]() {
return Whiskers(R"(
function <functionName>(combined) -> addr, selector {
combined := <shr64>(combined)
selector := and(combined, 0xffffffff)
addr := <shr32>(combined)
}
)")
("functionName", functionName)
("shr32", shiftRightFunction(32, false))
("shr64", shiftRightFunction(64, false))
.render();
});
}
string ABIFunctions::abiEncodingFunction(
Type const& _from,
Type const& _to,
@ -963,6 +1051,290 @@ string ABIFunctions::abiEncodingFunctionFunctionType(
});
}
string ABIFunctions::abiDecodingFunction(Type const& _type, bool _fromMemory, bool _forUseOnStack)
{
TypePointer decodingType = _type.decodingType();
solAssert(decodingType, "");
if (auto arrayType = dynamic_cast<ArrayType const*>(decodingType.get()))
{
if (arrayType->dataStoredIn(DataLocation::CallData))
{
solAssert(!_fromMemory, "");
return abiDecodingFunctionCalldataArray(*arrayType);
}
else if (arrayType->isByteArray())
return abiDecodingFunctionByteArray(*arrayType, _fromMemory);
else
return abiDecodingFunctionArray(*arrayType, _fromMemory);
}
else if (auto const* structType = dynamic_cast<StructType const*>(decodingType.get()))
return abiDecodingFunctionStruct(*structType, _fromMemory);
else if (auto const* functionType = dynamic_cast<FunctionType const*>(decodingType.get()))
return abiDecodingFunctionFunctionType(*functionType, _fromMemory, _forUseOnStack);
solAssert(decodingType->sizeOnStack() == 1, "");
solAssert(decodingType->isValueType(), "");
solAssert(decodingType->calldataEncodedSize() == 32, "");
solAssert(!decodingType->isDynamicallyEncoded(), "");
string functionName =
"abi_decode_" +
_type.identifier() +
(_fromMemory ? "_fromMemory" : "");
return createFunction(functionName, [&]() {
Whiskers templ(R"(
function <functionName>(offset, end) -> value {
value := <cleanup>(<load>(offset))
}
)");
templ("functionName", functionName);
templ("load", _fromMemory ? "mload" : "calldataload");
// Cleanup itself should use the type and not decodingType, because e.g.
// the decoding type of an enum is a plain int.
templ("cleanup", cleanupFunction(_type, true));
return templ.render();
});
}
string ABIFunctions::abiDecodingFunctionArray(ArrayType const& _type, bool _fromMemory)
{
solAssert(_type.dataStoredIn(DataLocation::Memory), "");
solAssert(!_type.isByteArray(), "");
string functionName =
"abi_decode_" +
_type.identifier() +
(_fromMemory ? "_fromMemory" : "");
solAssert(!_type.dataStoredIn(DataLocation::Storage), "");
return createFunction(functionName, [&]() {
string load = _fromMemory ? "mload" : "calldataload";
bool dynamicBase = _type.baseType()->isDynamicallyEncoded();
Whiskers templ(
R"(
// <readableTypeName>
function <functionName>(offset, end) -> array {
let length := <retrieveLength>
array := <allocate>(<allocationSize>(length))
let dst := array
<storeLength> // might update offset and dst
let src := offset
<staticBoundsCheck>
for { let i := 0 } lt(i, length) { i := add(i, 1) }
{
let elementPos := <retrieveElementPos>
<dynamicBoundsCheck>
mstore(dst, <decodingFun>(elementPos, end))
dst := add(dst, 0x20)
src := add(src, <baseEncodedSize>)
}
}
)"
);
templ("functionName", functionName);
templ("readableTypeName", _type.toString(true));
templ("retrieveLength", !_type.isDynamicallySized() ? toCompactHexWithPrefix(_type.length()) : load + "(offset)");
templ("allocate", allocationFunction());
templ("allocationSize", arrayAllocationSizeFunction(_type));
if (_type.isDynamicallySized())
templ("storeLength", "mstore(array, length) offset := add(offset, 0x20) dst := add(dst, 0x20)");
else
templ("storeLength", "");
if (dynamicBase)
{
templ("staticBoundsCheck", "");
// The dynamic bounds check might not be needed (because we have an additional check
// one level deeper), but we keep it in just in case. This at least prevents
// the part one level deeper from reading the length from an out of bounds position.
templ("dynamicBoundsCheck", "switch gt(elementPos, end) case 1 { revert(0, 0) }");
templ("retrieveElementPos", "add(offset, " + load + "(src))");
templ("baseEncodedSize", "0x20");
}
else
{
string baseEncodedSize = toCompactHexWithPrefix(_type.baseType()->calldataEncodedSize());
templ("staticBoundsCheck", "switch gt(add(src, mul(length, " + baseEncodedSize + ")), end) case 1 { revert(0, 0) }");
templ("dynamicBoundsCheck", "");
templ("retrieveElementPos", "src");
templ("baseEncodedSize", baseEncodedSize);
}
templ("decodingFun", abiDecodingFunction(*_type.baseType(), _fromMemory, false));
return templ.render();
});
}
string ABIFunctions::abiDecodingFunctionCalldataArray(ArrayType const& _type)
{
// This does not work with arrays of complex types - the array access
// is not yet implemented in Solidity.
solAssert(_type.dataStoredIn(DataLocation::CallData), "");
if (!_type.isDynamicallySized())
solAssert(_type.length() < u256("0xffffffffffffffff"), "");
solAssert(!_type.baseType()->isDynamicallyEncoded(), "");
solAssert(_type.baseType()->calldataEncodedSize() < u256("0xffffffffffffffff"), "");
string functionName =
"abi_decode_" +
_type.identifier();
return createFunction(functionName, [&]() {
string templ;
if (_type.isDynamicallySized())
templ = R"(
// <readableTypeName>
function <functionName>(offset, end) -> arrayPos, length {
length := calldataload(offset)
switch gt(length, 0xffffffffffffffff) case 1 { revert(0, 0) }
arrayPos := add(offset, 0x20)
switch gt(add(arrayPos, mul(<length>, <baseEncodedSize>)), end) case 1 { revert(0, 0) }
}
)";
else
templ = R"(
// <readableTypeName>
function <functionName>(offset, end) -> arrayPos {
arrayPos := offset
switch gt(add(arrayPos, mul(<length>, <baseEncodedSize>)), end) case 1 { revert(0, 0) }
}
)";
Whiskers w{templ};
w("functionName", functionName);
w("readableTypeName", _type.toString(true));
w("baseEncodedSize", toCompactHexWithPrefix(_type.isByteArray() ? 1 : _type.baseType()->calldataEncodedSize()));
w("length", _type.isDynamicallyEncoded() ? "length" : toCompactHexWithPrefix(_type.length()));
return w.render();
});
}
string ABIFunctions::abiDecodingFunctionByteArray(ArrayType const& _type, bool _fromMemory)
{
solAssert(_type.dataStoredIn(DataLocation::Memory), "");
solAssert(_type.isByteArray(), "");
string functionName =
"abi_decode_" +
_type.identifier() +
(_fromMemory ? "_fromMemory" : "");
return createFunction(functionName, [&]() {
Whiskers templ(
R"(
function <functionName>(offset, end) -> array {
let length := <load>(offset)
array := <allocate>(<allocationSize>(length))
mstore(array, length)
let src := add(offset, 0x20)
let dst := add(array, 0x20)
switch gt(add(src, length), end) case 1 { revert(0, 0) }
<copyToMemFun>(src, dst, length)
}
)"
);
templ("functionName", functionName);
templ("load", _fromMemory ? "mload" : "calldataload");
templ("allocate", allocationFunction());
templ("allocationSize", arrayAllocationSizeFunction(_type));
templ("copyToMemFun", copyToMemoryFunction(!_fromMemory));
return templ.render();
});
}
string ABIFunctions::abiDecodingFunctionStruct(StructType const& _type, bool _fromMemory)
{
string functionName =
"abi_decode_" +
_type.identifier() +
(_fromMemory ? "_fromMemory" : "");
solUnimplementedAssert(!_type.dataStoredIn(DataLocation::CallData), "");
return createFunction(functionName, [&]() {
Whiskers templ(R"(
// <readableTypeName>
function <functionName>(headStart, end) -> value {
switch slt(sub(end, headStart), <minimumSize>) case 1 { revert(0, 0) }
value := <allocate>(<memorySize>)
<#members>
{
// <memberName>
<decode>
}
</members>
}
)");
templ("functionName", functionName);
templ("readableTypeName", _type.toString(true));
templ("allocate", allocationFunction());
solAssert(_type.memorySize() < u256("0xffffffffffffffff"), "");
templ("memorySize", toCompactHexWithPrefix(_type.memorySize()));
size_t headPos = 0;
vector<map<string, string>> members;
for (auto const& member: _type.members(nullptr))
{
solAssert(member.type, "");
solAssert(member.type->canLiveOutsideStorage(), "");
auto decodingType = member.type->decodingType();
solAssert(decodingType, "");
bool dynamic = decodingType->isDynamicallyEncoded();
Whiskers memberTempl(R"(
let offset := )" + string(dynamic ? "<load>(add(headStart, <pos>))" : "<pos>" ) + R"(
mstore(add(value, <memoryOffset>), <abiDecode>(add(headStart, offset), end))
)");
memberTempl("load", _fromMemory ? "mload" : "calldataload");
memberTempl("pos", to_string(headPos));
memberTempl("memoryOffset", toCompactHexWithPrefix(_type.memoryOffsetOfMember(member.name)));
memberTempl("abiDecode", abiDecodingFunction(*member.type, _fromMemory, false));
members.push_back({});
members.back()["decode"] = memberTempl.render();
members.back()["memberName"] = member.name;
headPos += dynamic ? 0x20 : decodingType->calldataEncodedSize();
}
templ("members", members);
templ("minimumSize", toCompactHexWithPrefix(headPos));
return templ.render();
});
}
string ABIFunctions::abiDecodingFunctionFunctionType(FunctionType const& _type, bool _fromMemory, bool _forUseOnStack)
{
solAssert(_type.kind() == FunctionType::Kind::External, "");
string functionName =
"abi_decode_" +
_type.identifier() +
(_fromMemory ? "_fromMemory" : "") +
(_forUseOnStack ? "_onStack" : "");
return createFunction(functionName, [&]() {
if (_forUseOnStack)
{
return Whiskers(R"(
function <functionName>(offset, end) -> addr, function_selector {
addr, function_selector := <splitExtFun>(<load>(offset))
}
)")
("functionName", functionName)
("load", _fromMemory ? "mload" : "calldataload")
("splitExtFun", splitExternalFunctionIdFunction())
.render();
}
else
{
return Whiskers(R"(
function <functionName>(offset, end) -> fun {
fun := <cleanExtFun>(<load>(offset))
}
)")
("functionName", functionName)
("load", _fromMemory ? "mload" : "calldataload")
("cleanExtFun", cleanupCombinedExternalFunctionIdFunction())
.render();
}
});
}
string ABIFunctions::copyToMemoryFunction(bool _fromCalldata)
{
string functionName = "copy_" + string(_fromCalldata ? "calldata" : "memory") + "_to_memory";
@ -1098,6 +1470,33 @@ string ABIFunctions::arrayLengthFunction(ArrayType const& _type)
});
}
string ABIFunctions::arrayAllocationSizeFunction(ArrayType const& _type)
{
solAssert(_type.dataStoredIn(DataLocation::Memory), "");
string functionName = "array_allocation_size_" + _type.identifier();
return createFunction(functionName, [&]() {
Whiskers w(R"(
function <functionName>(length) -> size {
// Make sure we can allocate memory without overflow
switch gt(length, 0xffffffffffffffff) case 1 { revert(0, 0) }
size := <allocationSize>
<addLengthSlot>
}
)");
w("functionName", functionName);
if (_type.isByteArray())
// Round up
w("allocationSize", "and(add(length, 0x1f), not(0x1f))");
else
w("allocationSize", "mul(length, 0x20)");
if (_type.isDynamicallySized())
w("addLengthSlot", "size := add(size, 0x20)");
else
w("addLengthSlot", "");
return w.render();
});
}
string ABIFunctions::arrayDataAreaFunction(ArrayType const& _type)
{
string functionName = "array_dataslot_" + _type.identifier();
@ -1189,6 +1588,24 @@ string ABIFunctions::nextArrayElementFunction(ArrayType const& _type)
});
}
string ABIFunctions::allocationFunction()
{
string functionName = "allocateMemory";
return createFunction(functionName, [&]() {
return Whiskers(R"(
function <functionName>(size) -> memPtr {
memPtr := mload(<freeMemoryPointer>)
let newFreePtr := add(memPtr, size)
switch lt(newFreePtr, memPtr) case 1 { revert(0, 0) }
mstore(<freeMemoryPointer>, newFreePtr)
}
)")
("freeMemoryPointer", to_string(CompilerUtils::freeMemoryPointer))
("functionName", functionName)
.render();
});
}
string ABIFunctions::createFunction(string const& _name, function<string ()> const& _creator)
{
if (!m_requestedFunctions.count(_name))

View File

@ -66,6 +66,16 @@ public:
bool _encodeAsLibraryTypes = false
);
/// @returns name of an assembly function to ABI-decode values of @a _types
/// into memory. If @a _fromMemory is true, decodes from memory instead of
/// from calldata.
/// Can allocate memory.
/// Inputs: <source_offset> <source_end> (layout reversed on stack)
/// Outputs: <value0> <value1> ... <valuen>
/// The values represent stack slots. If a type occupies more or less than one
/// stack slot, it takes exactly that number of values.
std::string tupleDecoder(TypePointers const& _types, bool _fromMemory = false);
/// @returns concatenation of all generated functions.
std::string requestedFunctions();
@ -87,6 +97,10 @@ private:
/// for use in the ABI.
std::string combineExternalFunctionIdFunction();
/// @returns a function that splits the address and selector from a single value
/// for use in the ABI.
std::string splitExternalFunctionIdFunction();
/// @returns the name of the ABI encoding function with the given type
/// and queues the generation of the function to the requested functions.
/// @param _fromStack if false, the input value was just loaded from storage
@ -146,6 +160,29 @@ private:
bool _fromStack
);
/// @returns the name of the ABI decodinf function for the given type
/// and queues the generation of the function to the requested functions.
/// The caller has to ensure that no out of bounds access (at least to the static
/// part) can happen inside this function.
/// @param _fromMemory if decoding from memory instead of from calldata
/// @param _forUseOnStack if the decoded value is stored on stack or in memory.
std::string abiDecodingFunction(
Type const& _Type,
bool _fromMemory,
bool _forUseOnStack
);
/// Part of @a abiDecodingFunction for "regular" array types.
std::string abiDecodingFunctionArray(ArrayType const& _type, bool _fromMemory);
/// Part of @a abiDecodingFunction for calldata array types.
std::string abiDecodingFunctionCalldataArray(ArrayType const& _type);
/// Part of @a abiDecodingFunction for byte array types.
std::string abiDecodingFunctionByteArray(ArrayType const& _type, bool _fromMemory);
/// Part of @a abiDecodingFunction for struct types.
std::string abiDecodingFunctionStruct(StructType const& _type, bool _fromMemory);
/// Part of @a abiDecodingFunction for array types.
std::string abiDecodingFunctionFunctionType(FunctionType const& _type, bool _fromMemory, bool _forUseOnStack);
/// @returns a function that copies raw bytes of dynamic length from calldata
/// or memory to memory.
/// Pads with zeros and might write more than exactly length.
@ -158,6 +195,10 @@ private:
std::string roundUpFunction();
std::string arrayLengthFunction(ArrayType const& _type);
/// @returns the name of a function that computes the number of bytes required
/// to store an array in memory given its length (internally encoded, not ABI encoded).
/// The function reverts for too large lengthes.
std::string arrayAllocationSizeFunction(ArrayType const& _type);
/// @returns the name of a function that converts a storage slot number
/// or a memory pointer to the slot number / memory pointer for the data position of an array
/// which is stored in that slot / memory area.
@ -166,6 +207,12 @@ private:
/// Only works for memory arrays and storage arrays that store one item per slot.
std::string nextArrayElementFunction(ArrayType const& _type);
/// @returns the name of a function that allocates memory.
/// Modifies the "free memory pointer"
/// Arguments: size
/// Return value: pointer
std::string allocationFunction();
/// Helper function that uses @a _creator to create a function and add it to
/// @a m_requestedFunctions if it has not been created yet and returns @a _name in both
/// cases.

View File

@ -318,6 +318,7 @@ void CompilerContext::appendInlineAssembly(
ErrorList errors;
ErrorReporter errorReporter(errors);
// cout << _assembly << endl;
auto scanner = make_shared<Scanner>(CharStream(_assembly), "--CODEGEN--");
auto parserResult = assembly::Parser(errorReporter).parse(scanner);
#ifdef SOL_OUTPUT_ASM

View File

@ -319,6 +319,23 @@ void CompilerUtils::abiEncodeV2(
m_context << ret.tag();
}
void CompilerUtils::abiDecodeV2(TypePointers const& _parameterTypes, bool _fromMemory)
{
// stack: <source_offset>
auto ret = m_context.pushNewTag();
m_context << Instruction::SWAP1;
if (_fromMemory)
// TODO pass correct size for the memory case
m_context << (u256(1) << 63);
else
m_context << Instruction::CALLDATASIZE;
m_context << Instruction::SWAP1;
string decoderName = m_context.abiFunctions().tupleDecoder(_parameterTypes, _fromMemory);
m_context.appendJumpTo(m_context.namedTag(decoderName));
m_context.adjustStackOffset(int(sizeOnStack(_parameterTypes)) - 3);
m_context << ret.tag();
}
void CompilerUtils::zeroInitialiseMemoryArray(ArrayType const& _type)
{
auto repeat = m_context.newTag();

View File

@ -146,6 +146,13 @@ public:
bool _encodeAsLibraryTypes = false
);
/// Decodes data from ABI encoding into internal encoding. If @a _fromMemory is set to true,
/// the data is taken from memory instead of from calldata.
/// Can allocate memory.
/// Stack pre: <source_offset>
/// Stack post: <value0> <value1> ... <valuen>
void abiDecodeV2(TypePointers const& _parameterTypes, bool _fromMemory = false);
/// Zero-initialises (the data part of) an already allocated memory array.
/// Length has to be nonzero!
/// Stack pre: <length> <memptr>

View File

@ -322,6 +322,15 @@ void ContractCompiler::appendCalldataUnpacker(TypePointers const& _typeParameter
{
// We do not check the calldata size, everything is zero-padded
if (m_context.experimentalFeatureActive(ExperimentalFeature::ABIEncoderV2))
{
// Use the new JULIA-based decoding function
auto stackHeightBefore = m_context.stackHeight();
CompilerUtils(m_context).abiDecodeV2(_typeParameters, _fromMemory);
solAssert(m_context.stackHeight() - stackHeightBefore == CompilerUtils(m_context).sizeOnStack(_typeParameters) - 1, "");
return;
}
//@todo this does not yet support nested dynamic arrays
// Retain the offset pointer as base_offset, the point from which the data offsets are computed.
@ -892,6 +901,7 @@ void ContractCompiler::appendMissingFunctions()
}
m_context.appendMissingLowLevelFunctions();
string abiFunctions = m_context.abiFunctions().requestedFunctions();
// cout << abiFunctions << endl;
if (!abiFunctions.empty())
m_context.appendInlineAssembly("{" + move(abiFunctions) + "}", {}, true);
}