Tools to compute UnixFS IPFS hash.

libdevcore/CMakeLists.txt

@@ -12,10 +12,13 @@ set(sources
 	FixedHash.h
 	IndentedWriter.cpp
 	IndentedWriter.h
+	IpfsHash.cpp
+	IpfsHash.h
 	JSON.cpp
 	JSON.h
 	Keccak256.cpp
 	Keccak256.h
+	picosha2.h
 	Result.h
 	StringUtils.cpp
 	StringUtils.h

libdevcore/Exceptions.h

@@ -47,6 +47,7 @@ private:
 DEV_SIMPLE_EXCEPTION(InvalidAddress);
 DEV_SIMPLE_EXCEPTION(BadHexCharacter);
 DEV_SIMPLE_EXCEPTION(FileError);
+DEV_SIMPLE_EXCEPTION(DataTooLong);
 
 // error information to be added to exceptions
 using errinfo_invalidSymbol = boost::error_info<struct tag_invalidSymbol, char>;

libdevcore/IpfsHash.cpp

@@ -0,0 +1,93 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <libdevcore/IpfsHash.h>
+
+#include <libdevcore/Exceptions.h>
+#include <libdevcore/picosha2.h>
+#include <libdevcore/CommonData.h>
+
+using namespace std;
+using namespace dev;
+
+namespace
+{
+bytes varintEncoding(size_t _n)
+{
+	bytes encoded;
+	while (_n > 0x7f)
+	{
+		encoded.emplace_back(uint8_t(0x80 | (_n & 0x7f)));
+		_n >>= 7;
+	}
+	encoded.emplace_back(_n);
+	return encoded;
+}
+
+string base58Encode(bytes const& _data)
+{
+	static string const alphabet{"123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"};
+	bigint data(toHex(_data, HexPrefix::Add));
+	string output;
+	while (data)
+	{
+		output += alphabet[size_t(data % alphabet.size())];
+		data /= alphabet.size();
+	}
+	reverse(output.begin(), output.end());
+	return output;
+}
+}
+
+bytes dev::ipfsHash(string _data)
+{
+	if (_data.length() >= 1024 * 256)
+		BOOST_THROW_EXCEPTION(
+			DataTooLong() <<
+			errinfo_comment("Ipfs hash for large (chunked) files not yet implemented.")
+		);
+
+	bytes lengthAsVarint = varintEncoding(_data.size());
+
+	bytes protobufEncodedData;
+	// Type: File
+	protobufEncodedData += bytes{0x08, 0x02};
+	if (!_data.empty())
+	{
+		// Data (length delimited bytes)
+		protobufEncodedData += bytes{0x12};
+		protobufEncodedData += lengthAsVarint;
+		protobufEncodedData += asBytes(std::move(_data));
+	}
+	// filesize: length as varint
+	protobufEncodedData += bytes{0x18} + lengthAsVarint;
+
+	// PBDag:
+	// Data: (length delimited bytes)
+	size_t protobufLength = protobufEncodedData.size();
+	bytes blockData = bytes{0x0a} + varintEncoding(protobufLength) + std::move(protobufEncodedData);
+	// TODO Handle "large" files with multiple blocks
+
+	// Multihash: sha2-256, 256 bits
+	bytes hash = bytes{0x12, 0x20} + picosha2::hash256(std::move(blockData));
+	return hash;
+}
+
+string dev::ipfsHashBase58(string _data)
+{
+	return base58Encode(ipfsHash(std::move(_data)));
+}

libdevcore/IpfsHash.h

@@ -0,0 +1,37 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+
+#include <libdevcore/Common.h>
+
+#include <string>
+
+namespace dev
+{
+
+/// Compute the "ipfs hash" of a file with the content @a _data.
+/// The output will be the multihash of the UnixFS protobuf encoded data.
+/// As hash function it will use sha2-256.
+/// The effect is that the hash should be identical to the one produced by
+/// the command `ipfs add <filename>`.
+bytes ipfsHash(std::string _data);
+
+/// Compute the "ipfs hash" as above, but encoded in base58 as used by ipfs / bitcoin.
+std::string ipfsHashBase58(std::string _data);
+
+}

libdevcore/picosha2.h

@@ -0,0 +1,288 @@
+/*
+The MIT License (MIT)
+
+Copyright (C) 2014 okdshin
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+#pragma once
+
+//picosha2:20140213
+#include <cstdint>
+#include <iostream>
+#include <vector>
+#include <iterator>
+#include <cassert>
+#include <sstream>
+#include <algorithm>
+
+namespace picosha2
+{
+
+namespace detail
+{
+
+inline uint8_t mask_8bit(uint8_t x)
+{
+	return x & 0xff;
+}
+
+inline uint32_t mask_32bit(uint32_t x)
+{
+	return x & 0xffffffff;
+}
+
+static uint32_t const add_constant[64] = {
+	0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5,
+	0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
+	0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3,
+	0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
+	0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc,
+	0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
+	0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7,
+	0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
+	0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13,
+	0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
+	0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3,
+	0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
+	0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5,
+	0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
+	0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208,
+	0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2
+};
+
+static uint32_t const initial_message_digest[8] = {
+	0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a,
+	0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19
+};
+
+inline uint32_t ch(uint32_t x, uint32_t y, uint32_t z)
+{
+	return (x & y) ^ ((~x) & z);
+}
+
+inline uint32_t maj(uint32_t x, uint32_t y, uint32_t z)
+{
+	return (x & y) ^ (x & z) ^ (y & z);
+}
+
+inline uint32_t rotr(uint32_t x, std::size_t n)
+{
+	assert(n < 32);
+	return mask_32bit((x >> n) | (x << (32 - n)));
+}
+
+inline uint32_t bsig0(uint32_t x)
+{
+	return rotr(x, 2) ^ rotr(x, 13) ^ rotr(x, 22);
+}
+
+inline uint32_t bsig1(uint32_t x)
+{
+	return rotr(x, 6) ^ rotr(x, 11) ^ rotr(x, 25);
+}
+
+inline uint32_t shr(uint32_t x, std::size_t n)
+{
+	assert(n < 32);
+	return x >> n;
+}
+
+inline uint32_t ssig0(uint32_t x)
+{
+	return rotr(x, 7) ^ rotr(x, 18) ^ shr(x, 3);
+}
+
+inline uint32_t ssig1(uint32_t x)
+{
+	return rotr(x, 17) ^ rotr(x, 19) ^ shr(x, 10);
+}
+
+template<typename RaIter1, typename RaIter2>
+void hash256_block(RaIter1 message_digest, RaIter2 first, RaIter2 last)
+{
+	(void)last; // FIXME: check this is valid
+	uint32_t w[64];
+	std::fill(w, w+64, 0);
+	for (std::size_t i = 0; i < 16; ++i)
+		w[i] = (static_cast<uint32_t>(mask_8bit(*(first + i * 4))) << 24)
+			| (static_cast<uint32_t>(mask_8bit(*(first + i * 4 + 1))) << 16)
+			| (static_cast<uint32_t>(mask_8bit(*(first + i * 4 + 2))) << 8)
+			| (static_cast<uint32_t>(mask_8bit(*(first + i * 4 + 3))));
+	for (std::size_t i = 16; i < 64; ++i)
+		w[i] = mask_32bit(ssig1(w[i-2])+w[i-7]+ssig0(w[i-15])+w[i-16]);
+
+	uint32_t a = *message_digest;
+	uint32_t b = *(message_digest + 1);
+	uint32_t c = *(message_digest + 2);
+	uint32_t d = *(message_digest + 3);
+	uint32_t e = *(message_digest + 4);
+	uint32_t f = *(message_digest + 5);
+	uint32_t g = *(message_digest + 6);
+	uint32_t h = *(message_digest + 7);
+
+	for (std::size_t i = 0; i < 64; ++i)
+	{
+		uint32_t temp1 = h+bsig1(e)+ch(e,f,g)+add_constant[i]+w[i];
+		uint32_t temp2 = bsig0(a)+maj(a,b,c);
+		h = g;
+		g = f;
+		f = e;
+		e = mask_32bit(d+temp1);
+		d = c;
+		c = b;
+		b = a;
+		a = mask_32bit(temp1+temp2);
+	}
+	*message_digest += a;
+	*(message_digest+1) += b;
+	*(message_digest+2) += c;
+	*(message_digest+3) += d;
+	*(message_digest+4) += e;
+	*(message_digest+5) += f;
+	*(message_digest+6) += g;
+	*(message_digest+7) += h;
+	for (std::size_t i = 0; i < 8; ++i)
+		*(message_digest+i) = mask_32bit(*(message_digest+i));
+}
+
+}//namespace detail
+
+class hash256_one_by_one
+{
+public:
+	hash256_one_by_one()
+	{
+		init();
+	}
+
+	void init()
+	{
+		buffer_.clear();
+		std::fill(data_length_digits_, data_length_digits_ + 4, 0);
+		std::copy(detail::initial_message_digest, detail::initial_message_digest+8, h_);
+	}
+
+	template<typename RaIter>
+	void process(RaIter first, RaIter last)
+	{
+		add_to_data_length(std::distance(first, last));
+		std::copy(first, last, std::back_inserter(buffer_));
+		std::size_t i = 0;
+		for (;i + 64 <= buffer_.size(); i+=64)
+			detail::hash256_block(h_, buffer_.begin()+i, buffer_.begin()+i+64);
+		buffer_.erase(buffer_.begin(), buffer_.begin()+i);
+	}
+
+	void finish()
+	{
+		uint8_t temp[64];
+		std::fill(temp, temp+64, 0);
+		std::size_t remains = buffer_.size();
+		std::copy(buffer_.begin(), buffer_.end(), temp);
+		temp[remains] = 0x80;
+
+		if (remains > 55)
+		{
+			std::fill(temp+remains+1, temp+64, 0);
+			detail::hash256_block(h_, temp, temp+64);
+			std::fill(temp, temp+64-4, 0);
+		}
+		else
+			std::fill(temp+remains+1, temp+64-4, 0);
+
+		write_data_bit_length(&(temp[56]));
+		detail::hash256_block(h_, temp, temp+64);
+	}
+
+	template<typename OutIter>
+	void get_hash_bytes(OutIter first, OutIter last) const
+	{
+		for (uint32_t const* iter = h_; iter != h_ + 8; ++iter)
+			for (std::size_t i = 0; i < 4 && first != last; ++i)
+				*(first++) = detail::mask_8bit(static_cast<uint8_t>(*iter >> (24 - 8 * i)));
+	}
+
+private:
+	void add_to_data_length(uint32_t n)
+	{
+		uint32_t carry = 0;
+		data_length_digits_[0] += n;
+		for (std::size_t i = 0; i < 4; ++i)
+		{
+			data_length_digits_[i] += carry;
+			if (data_length_digits_[i] >= 65536u)
+			{
+				carry = data_length_digits_[i] >> 16;
+				data_length_digits_[i] &= 65535u;
+			}
+			else
+				break;
+		}
+	}
+	void write_data_bit_length(uint8_t* begin)
+	{
+		uint32_t data_bit_length_digits[4];
+		std::copy(
+			data_length_digits_, data_length_digits_ + 4,
+			data_bit_length_digits
+		);
+
+		// convert byte length to bit length (multiply 8 or shift 3 times left)
+		uint32_t carry = 0;
+		for (std::size_t i = 0; i < 4; ++i)
+		{
+			uint32_t before_val = data_bit_length_digits[i];
+			data_bit_length_digits[i] <<= 3;
+			data_bit_length_digits[i] |= carry;
+			data_bit_length_digits[i] &= 65535u;
+			carry = (before_val >> (16-3)) & 65535u;
+		}
+
+		// write data_bit_length
+		for (int i = 3; i >= 0; --i)
+		{
+			(*begin++) = static_cast<uint8_t>(data_bit_length_digits[i] >> 8);
+			(*begin++) = static_cast<uint8_t>(data_bit_length_digits[i]);
+		}
+	}
+	std::vector<uint8_t> buffer_;
+	uint32_t data_length_digits_[4]; //as 64bit integer (16bit x 4 integer)
+	uint32_t h_[8];
+};
+
+template<typename RaIter, typename OutIter>
+void hash256(RaIter first, RaIter last, OutIter first2, OutIter last2)
+{
+	hash256_one_by_one hasher;
+	//hasher.init();
+	hasher.process(first, last);
+	hasher.finish();
+	hasher.get_hash_bytes(first2, last2);
+}
+
+template <typename RaContainer>
+std::vector<uint8_t> hash256(RaContainer const& _src)
+{
+	std::vector<uint8_t> ret(32);
+	hash256(_src.begin(), _src.end(), ret.begin(), ret.end());
+	return ret;
+}
+
+}//namespace picosha2

libsolidity/interface/CompilerStack.cpp

@@ -769,7 +769,6 @@ h256 const& CompilerStack::Source::swarmHash() const
 	return swarmHashCached;
 }
 
-
 StringMap CompilerStack::loadMissingSources(SourceUnit const& _ast, std::string const& _sourcePath)
 {
 	solAssert(m_stackState < ParsingSuccessful, "");

test/libdevcore/IpfsHash.cpp

@@ -0,0 +1,76 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/**
+ * Unit tests for the ipfs hash computation routine.
+ */
+
+#include <libdevcore/IpfsHash.h>
+
+#include <test/Options.h>
+
+using namespace std;
+
+namespace dev
+{
+namespace test
+{
+
+BOOST_AUTO_TEST_SUITE(IpfsHash)
+
+BOOST_AUTO_TEST_CASE(test_small)
+{
+	BOOST_CHECK_EQUAL(ipfsHashBase58({}), "QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH");
+	BOOST_CHECK_EQUAL(ipfsHashBase58("x"), "QmULKig5Fxrs2sC4qt9nNduucXfb92AFYQ6Hi3YRqDmrYC");
+	BOOST_CHECK_EQUAL(ipfsHashBase58("Solidity\n"), "QmSsm9M7PQRBnyiz1smizk8hZw3URfk8fSeHzeTo3oZidS");
+	BOOST_CHECK_EQUAL(ipfsHashBase58(string(size_t(200), char(0))), "QmSXR1N23uWzsANi8wpxMPw5dmmhqBVUAb4hUrHVLpNaMr");
+	BOOST_CHECK_EQUAL(ipfsHashBase58(string(size_t(10250), char(0))), "QmVJJBB3gKKBWYC9QTywpH8ZL1bDeTDJ17B63Af5kino9i");
+	BOOST_CHECK_EQUAL(ipfsHashBase58(string(size_t(100000), char(0))), "QmYgKa25YqEGpQmmZtPPFMNK3kpqqneHk6nMSEUYryEX1C");
+	BOOST_CHECK_EQUAL(ipfsHashBase58(string(size_t(121071), char(0))), "QmdMdRshQmqvyc92N82r7AKYdUF5FRh4DJo6GtrmEk3wgj");
+}
+
+BOOST_AUTO_TEST_CASE(test_medium)
+{
+	size_t length = 131071;
+	string data;
+	data.resize(length, 0);
+	BOOST_REQUIRE_EQUAL(data.size(), length);
+	BOOST_CHECK_EQUAL(ipfsHashBase58(data), "QmSxYSToKHsPqqRdRnsM9gmr3EYS6dakhVaHgbFdgYQWi6");
+}
+
+BOOST_AUTO_TEST_CASE(test_largest_unchunked)
+{
+	size_t length = 1024 * 256 - 1;
+	string data;
+	data.resize(length, 0);
+	BOOST_REQUIRE_EQUAL(data.size(), length);
+	BOOST_CHECK_EQUAL(ipfsHashBase58(data), "QmbNDspMkzkMFKyS3eCJGedG7GWRQHSCzJCZLjxP7wyVAx");
+}
+
+// TODO This needs chunking implemented
+//BOOST_AUTO_TEST_CASE(test_large)
+//{
+//	size_t length = 1310710;
+//	string data;
+//	data.resize(length, 0);
+//	BOOST_REQUIRE_EQUAL(data.size(), length);
+//	BOOST_CHECK_EQUAL(ipfsHashBase58(data), "QmNg7BJo8gEMDK8yGQbHEwPtycesnE6FUULX5iVd5TAL9f");
+//}
+
+BOOST_AUTO_TEST_SUITE_END()
+
+}
+}