internalized ipld types and dag_putters; adjust converters/publishers to work with them

pkg/ipfs/ipld/btc_header.go

@@ -0,0 +1,202 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package ipld
+
+import (
+	"bytes"
+	"encoding/hex"
+	"fmt"
+
+	"github.com/btcsuite/btcd/wire"
+	"github.com/ipfs/go-cid"
+	node "github.com/ipfs/go-ipld-format"
+	mh "github.com/multiformats/go-multihash"
+)
+
+type BtcHeader struct {
+	*wire.BlockHeader
+
+	rawdata []byte
+	cid     cid.Cid
+}
+
+// Static (compile time) check that BtcBtcHeader satisfies the node.Node interface.
+var _ node.Node = (*BtcHeader)(nil)
+
+/*
+  INPUT
+*/
+
+// NewBtcHeader converts a *wire.Header into an BtcHeader IPLD node
+func NewBtcHeader(header *wire.BlockHeader) (*BtcHeader, error) {
+	w := bytes.NewBuffer(make([]byte, 0, 80))
+	if err := header.Serialize(w); err != nil {
+		return nil, err
+	}
+	rawdata := w.Bytes()
+	c, err := rawdataToCid(MBitcoinHeader, rawdata, mh.DBL_SHA2_256)
+	if err != nil {
+		return nil, err
+	}
+	return &BtcHeader{
+		BlockHeader: header,
+		cid:         c,
+		rawdata:     rawdata,
+	}, nil
+}
+
+/*
+  OUTPUT
+*/
+
+// DecodeBtcHeader takes a cid and its raw binary data
+// from IPFS and returns an BtcHeader object for further processing.
+func DecodeBtcHeader(c cid.Cid, b []byte) (*BtcHeader, error) {
+	var h *wire.BlockHeader
+	w := bytes.NewBuffer(b)
+	if err := h.Deserialize(w); err != nil {
+		return nil, err
+	}
+	return &BtcHeader{
+		BlockHeader: h,
+		cid:         c,
+		rawdata:     b,
+	}, nil
+}
+
+/*
+   Block INTERFACE
+*/
+
+func (b *BtcHeader) Cid() cid.Cid {
+	return b.cid
+}
+
+func (b *BtcHeader) RawData() []byte {
+	return b.rawdata
+}
+
+func (b *BtcHeader) String() string {
+	return fmt.Sprintf("<BtcHeader %s>", b.cid)
+}
+
+func (b *BtcHeader) Loggable() map[string]interface{} {
+	// TODO: more helpful info here
+	return map[string]interface{}{
+		"type": "bitcoin_block",
+	}
+}
+
+/*
+   Node INTERFACE
+*/
+
+func (b *BtcHeader) Links() []*node.Link {
+	return []*node.Link{
+		{
+			Name: "tx",
+			Cid:  sha256ToCid(MBitcoinTx, b.MerkleRoot.CloneBytes()),
+		},
+		{
+			Name: "parent",
+			Cid:  sha256ToCid(MBitcoinHeader, b.PrevBlock.CloneBytes()),
+		},
+	}
+}
+
+// Resolve attempts to traverse a path through this block.
+func (b *BtcHeader) Resolve(path []string) (interface{}, []string, error) {
+	if len(path) == 0 {
+		return nil, nil, fmt.Errorf("zero length path")
+	}
+	switch path[0] {
+	case "version":
+		return b.Version, path[1:], nil
+	case "timestamp":
+		return b.Timestamp, path[1:], nil
+	case "bits":
+		return b.Bits, path[1:], nil
+	case "nonce":
+		return b.Nonce, path[1:], nil
+	case "parent":
+		return &node.Link{Cid: sha256ToCid(MBitcoinHeader, b.PrevBlock.CloneBytes())}, path[1:], nil
+	case "tx":
+		return &node.Link{Cid: sha256ToCid(MBitcoinTx, b.MerkleRoot.CloneBytes())}, path[1:], nil
+	default:
+		return nil, nil, fmt.Errorf("no such link")
+	}
+}
+
+// ResolveLink is a helper function that allows easier traversal of links through blocks
+func (b *BtcHeader) ResolveLink(path []string) (*node.Link, []string, error) {
+	out, rest, err := b.Resolve(path)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	lnk, ok := out.(*node.Link)
+	if !ok {
+		return nil, nil, fmt.Errorf("object at path was not a link")
+	}
+
+	return lnk, rest, nil
+}
+
+func cidToHash(c cid.Cid) []byte {
+	h := []byte(c.Hash())
+	return h[len(h)-32:]
+}
+
+func hashToCid(hv []byte, t uint64) cid.Cid {
+	h, _ := mh.Encode(hv, mh.DBL_SHA2_256)
+	return cid.NewCidV1(t, h)
+}
+
+func (b *BtcHeader) Size() (uint64, error) {
+	return uint64(len(b.rawdata)), nil
+}
+
+func (b *BtcHeader) Stat() (*node.NodeStat, error) {
+	return &node.NodeStat{}, nil
+}
+
+func (b *BtcHeader) Tree(p string, depth int) []string {
+	// TODO: this isnt a correct implementation yet
+	return []string{"difficulty", "nonce", "version", "timestamp", "tx", "parent"}
+}
+
+func (b *BtcHeader) BTCSha() []byte {
+	blkmh, _ := mh.Sum(b.rawdata, mh.DBL_SHA2_256, -1)
+	return blkmh[2:]
+}
+
+func (b *BtcHeader) HexHash() string {
+	return hex.EncodeToString(revString(b.BTCSha()))
+}
+
+func (b *BtcHeader) Copy() node.Node {
+	nb := *b // cheating shallow copy
+	return &nb
+}
+
+func revString(s []byte) []byte {
+	b := make([]byte, len(s))
+	for i, v := range []byte(s) {
+		b[len(b)-(i+1)] = v
+	}
+	return b
+}

pkg/ipfs/ipld/btc_tx.go

@@ -0,0 +1,261 @@
+package ipld
+
+import (
+	"bytes"
+	"encoding/hex"
+	"fmt"
+	"strconv"
+
+	"github.com/btcsuite/btcd/wire"
+	cid "github.com/ipfs/go-cid"
+	node "github.com/ipfs/go-ipld-format"
+	mh "github.com/multiformats/go-multihash"
+)
+
+type BtcTx struct {
+	*wire.MsgTx
+
+	rawdata []byte
+	cid     cid.Cid
+}
+
+// Static (compile time) check that BtcBtcHeader satisfies the node.Node interface.
+var _ node.Node = (*BtcTx)(nil)
+
+/*
+  INPUT
+*/
+
+// NewBtcTx converts a *wire.MsgTx into an BtcTx IPLD node
+func NewBtcTx(tx *wire.MsgTx) (*BtcTx, error) {
+	w := bytes.NewBuffer(make([]byte, 0, tx.SerializeSize()))
+	if err := tx.Serialize(w); err != nil {
+		return nil, err
+	}
+	rawdata := w.Bytes()
+	c, err := rawdataToCid(MBitcoinTx, rawdata, mh.DBL_SHA2_256)
+	if err != nil {
+		return nil, err
+	}
+	return &BtcTx{
+		MsgTx:   tx,
+		cid:     c,
+		rawdata: rawdata,
+	}, nil
+}
+
+/*
+  OUTPUT
+*/
+
+// DecodeBtcTx takes a cid and its raw binary data
+// from IPFS and returns an BtcTx object for further processing.
+func DecodeBtcTx(c cid.Cid, b []byte) (*BtcTx, error) {
+	var tx *wire.MsgTx
+	w := bytes.NewBuffer(b)
+	if err := tx.Deserialize(w); err != nil {
+		return nil, err
+	}
+	return &BtcTx{
+		MsgTx:   tx,
+		cid:     c,
+		rawdata: b,
+	}, nil
+}
+
+/*
+   Block INTERFACE
+*/
+
+func (t *BtcTx) Cid() cid.Cid {
+	return t.cid
+}
+
+func (t *BtcTx) RawData() []byte {
+	return t.rawdata
+}
+
+func (t *BtcTx) String() string {
+	return fmt.Sprintf("<BtcTx %s>", t.cid)
+}
+
+func (t *BtcTx) Loggable() map[string]interface{} {
+	return map[string]interface{}{
+		"type": "bitcoinTx",
+	}
+}
+
+/*
+   Node INTERFACE
+*/
+
+func (t *BtcTx) Links() []*node.Link {
+	var out []*node.Link
+	for i, in := range t.MsgTx.TxIn {
+		lnk := &node.Link{Cid: sha256ToCid(MBitcoinTx, in.PreviousOutPoint.Hash.CloneBytes())}
+		lnk.Name = fmt.Sprintf("inputs/%d/prevTx", i)
+		out = append(out, lnk)
+	}
+	return out
+}
+
+func (t *BtcTx) Resolve(path []string) (interface{}, []string, error) {
+	switch path[0] {
+	case "version":
+		return t.Version, path[1:], nil
+	case "lockTime":
+		return t.LockTime, path[1:], nil
+	case "inputs":
+		if len(path) == 1 {
+			return t.MsgTx.TxIn, nil, nil
+		}
+
+		index, err := strconv.Atoi(path[1])
+		if err != nil {
+			return nil, nil, err
+		}
+
+		if index >= len(t.MsgTx.TxIn) || index < 0 {
+			return nil, nil, fmt.Errorf("index out of range")
+		}
+
+		inp := t.MsgTx.TxIn[index]
+		if len(path) == 2 {
+			return inp, nil, nil
+		}
+
+		switch path[2] {
+		case "prevTx":
+			return &node.Link{Cid: sha256ToCid(MBitcoinTx, inp.PreviousOutPoint.Hash.CloneBytes())}, path[3:], nil
+		case "seqNo":
+			return inp.Sequence, path[3:], nil
+		case "script":
+			return inp.SignatureScript, path[3:], nil
+		default:
+			return nil, nil, fmt.Errorf("no such link")
+		}
+	case "outputs":
+		if len(path) == 1 {
+			return t.TxOut, nil, nil
+		}
+
+		index, err := strconv.Atoi(path[1])
+		if err != nil {
+			return nil, nil, err
+		}
+
+		if index >= len(t.TxOut) || index < 0 {
+			return nil, nil, fmt.Errorf("index out of range")
+		}
+
+		outp := t.TxOut[index]
+		if len(path) == 2 {
+			return outp, path[2:], nil
+		}
+
+		switch path[2] {
+		case "value":
+			return outp.Value, path[3:], nil
+		case "script":
+			/*
+				if outp.Script[0] == 0x6a { // OP_RETURN
+					c, err := cid.Decode(string(outp.Script[1:]))
+					if err == nil {
+						return &node.Link{Cid: c}, path[3:], nil
+					}
+				}
+			*/
+			return outp.PkScript, path[3:], nil
+		default:
+			return nil, nil, fmt.Errorf("no such link")
+		}
+	default:
+		return nil, nil, fmt.Errorf("no such link")
+	}
+}
+
+func (t *BtcTx) ResolveLink(path []string) (*node.Link, []string, error) {
+	i, rest, err := t.Resolve(path)
+	if err != nil {
+		return nil, rest, err
+	}
+
+	lnk, ok := i.(*node.Link)
+	if !ok {
+		return nil, nil, fmt.Errorf("value was not a link")
+	}
+
+	return lnk, rest, nil
+}
+
+func (t *BtcTx) Size() (uint64, error) {
+	return uint64(len(t.RawData())), nil
+}
+
+func (t *BtcTx) Stat() (*node.NodeStat, error) {
+	return &node.NodeStat{}, nil
+}
+
+func (t *BtcTx) Copy() node.Node {
+	nt := *t // cheating shallow copy
+	return &nt
+}
+
+func (t *BtcTx) Tree(p string, depth int) []string {
+	if depth == 0 {
+		return nil
+	}
+
+	switch p {
+	case "inputs":
+		return t.treeInputs(nil, depth+1)
+	case "outputs":
+		return t.treeOutputs(nil, depth+1)
+	case "":
+		out := []string{"version", "timeLock", "inputs", "outputs"}
+		out = t.treeInputs(out, depth)
+		out = t.treeOutputs(out, depth)
+		return out
+	default:
+		return nil
+	}
+}
+
+func (t *BtcTx) treeInputs(out []string, depth int) []string {
+	if depth < 2 {
+		return out
+	}
+
+	for i := range t.TxIn {
+		inp := "inputs/" + fmt.Sprint(i)
+		out = append(out, inp)
+		if depth > 2 {
+			out = append(out, inp+"/prevTx", inp+"/seqNo", inp+"/script")
+		}
+	}
+	return out
+}
+
+func (t *BtcTx) treeOutputs(out []string, depth int) []string {
+	if depth < 2 {
+		return out
+	}
+
+	for i := range t.TxOut {
+		o := "outputs/" + fmt.Sprint(i)
+		out = append(out, o)
+		if depth > 2 {
+			out = append(out, o+"/script", o+"/value")
+		}
+	}
+	return out
+}
+
+func (t *BtcTx) BTCSha() []byte {
+	mh, _ := mh.Sum(t.RawData(), mh.DBL_SHA2_256, -1)
+	return []byte(mh[2:])
+}
+
+func (t *BtcTx) HexHash() string {
+	return hex.EncodeToString(revString(t.BTCSha()))
+}

pkg/ipfs/ipld/eth_account.go

@@ -0,0 +1,175 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package ipld
+
+import (
+	"encoding/json"
+	"fmt"
+	"math/big"
+
+	"github.com/ipfs/go-cid"
+	node "github.com/ipfs/go-ipld-format"
+)
+
+// EthAccountSnapshot (eth-account-snapshot codec 0x97)
+// represents an ethereum account, i.e. a wallet address or
+// a smart contract
+type EthAccountSnapshot struct {
+	*EthAccount
+
+	cid     cid.Cid
+	rawdata []byte
+}
+
+// EthAccount is the building block of EthAccountSnapshot.
+// Or, is the former stripped of its cid and rawdata components.
+type EthAccount struct {
+	Nonce    uint64
+	Balance  *big.Int
+	Root     []byte // This is the storage root trie
+	CodeHash []byte // This is the hash of the EVM code
+}
+
+// Static (compile time) check that EthAccountSnapshot satisfies the
+// node.Node interface.
+var _ node.Node = (*EthAccountSnapshot)(nil)
+
+/*
+  INPUT
+*/
+
+// Input should be managed by EthStateTrie
+
+/*
+   OUTPUT
+*/
+
+// Output should be managed by EthStateTrie
+
+/*
+   Block INTERFACE
+*/
+
+// RawData returns the binary of the RLP encode of the account snapshot.
+func (as *EthAccountSnapshot) RawData() []byte {
+	return as.rawdata
+}
+
+// Cid returns the cid of the transaction.
+func (as *EthAccountSnapshot) Cid() cid.Cid {
+	return as.cid
+}
+
+// String is a helper for output
+func (as *EthAccountSnapshot) String() string {
+	return fmt.Sprintf("<EthereumAccountSnapshot %s>", as.cid)
+}
+
+// Loggable returns in a map the type of IPLD Link.
+func (as *EthAccountSnapshot) Loggable() map[string]interface{} {
+	return map[string]interface{}{
+		"type": "eth-account-snapshot",
+	}
+}
+
+/*
+   Node INTERFACE
+*/
+
+// Resolve resolves a path through this node, stopping at any link boundary
+// and returning the object found as well as the remaining path to traverse
+func (as *EthAccountSnapshot) Resolve(p []string) (interface{}, []string, error) {
+	if len(p) == 0 {
+		return as, nil, nil
+	}
+
+	if len(p) > 1 {
+		return nil, nil, fmt.Errorf("unexpected path elements past %s", p[0])
+	}
+
+	switch p[0] {
+	case "balance":
+		return as.Balance, nil, nil
+	case "codeHash":
+		return &node.Link{Cid: keccak256ToCid(RawBinary, as.CodeHash)}, nil, nil
+	case "nonce":
+		return as.Nonce, nil, nil
+	case "root":
+		return &node.Link{Cid: keccak256ToCid(MEthStorageTrie, as.Root)}, nil, nil
+	default:
+		return nil, nil, fmt.Errorf("no such link")
+	}
+}
+
+// Tree lists all paths within the object under 'path', and up to the given depth.
+// To list the entire object (similar to `find .`) pass "" and -1
+func (as *EthAccountSnapshot) Tree(p string, depth int) []string {
+	if p != "" || depth == 0 {
+		return nil
+	}
+	return []string{"balance", "codeHash", "nonce", "root"}
+}
+
+// ResolveLink is a helper function that calls resolve and asserts the
+// output is a link
+func (as *EthAccountSnapshot) ResolveLink(p []string) (*node.Link, []string, error) {
+	obj, rest, err := as.Resolve(p)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if lnk, ok := obj.(*node.Link); ok {
+		return lnk, rest, nil
+	}
+
+	return nil, nil, fmt.Errorf("resolved item was not a link")
+}
+
+// Copy will go away. It is here to comply with the interface.
+func (as *EthAccountSnapshot) Copy() node.Node {
+	panic("dont use this yet")
+}
+
+// Links is a helper function that returns all links within this object
+func (as *EthAccountSnapshot) Links() []*node.Link {
+	return nil
+}
+
+// Stat will go away. It is here to comply with the interface.
+func (as *EthAccountSnapshot) Stat() (*node.NodeStat, error) {
+	return &node.NodeStat{}, nil
+}
+
+// Size will go away. It is here to comply with the interface.
+func (as *EthAccountSnapshot) Size() (uint64, error) {
+	return 0, nil
+}
+
+/*
+  EthAccountSnapshot functions
+*/
+
+// MarshalJSON processes the transaction into readable JSON format.
+func (as *EthAccountSnapshot) MarshalJSON() ([]byte, error) {
+	out := map[string]interface{}{
+		"balance":  as.Balance,
+		"codeHash": keccak256ToCid(RawBinary, as.CodeHash),
+		"nonce":    as.Nonce,
+		"root":     keccak256ToCid(MEthStorageTrie, as.Root),
+	}
+	return json.Marshal(out)
+}

pkg/ipfs/ipld/eth_header.go

@@ -0,0 +1,292 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package ipld
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ipfs/go-cid"
+	node "github.com/ipfs/go-ipld-format"
+	mh "github.com/multiformats/go-multihash"
+)
+
+// EthHeader (eth-block, codec 0x90), represents an ethereum block header
+type EthHeader struct {
+	*types.Header
+
+	cid     cid.Cid
+	rawdata []byte
+}
+
+// Static (compile time) check that EthHeader satisfies the node.Node interface.
+var _ node.Node = (*EthHeader)(nil)
+
+/*
+  INPUT
+*/
+
+// NewEthHeader converts a *types.Header into an EthHeader IPLD node
+func NewEthHeader(header *types.Header) (*EthHeader, error) {
+	headerRLP, err := rlp.EncodeToBytes(header)
+	if err != nil {
+		return nil, err
+	}
+	c, err := rawdataToCid(MEthHeader, headerRLP, mh.KECCAK_256)
+	if err != nil {
+		return nil, err
+	}
+	return &EthHeader{
+		Header:  header,
+		cid:     c,
+		rawdata: headerRLP,
+	}, nil
+}
+
+/*
+  OUTPUT
+*/
+
+// DecodeEthHeader takes a cid and its raw binary data
+// from IPFS and returns an EthHeader object for further processing.
+func DecodeEthHeader(c cid.Cid, b []byte) (*EthHeader, error) {
+	var h *types.Header
+	if err := rlp.DecodeBytes(b, h); err != nil {
+		return nil, err
+	}
+	return &EthHeader{
+		Header:  h,
+		cid:     c,
+		rawdata: b,
+	}, nil
+}
+
+/*
+  Block INTERFACE
+*/
+
+// RawData returns the binary of the RLP encode of the block header.
+func (b *EthHeader) RawData() []byte {
+	return b.rawdata
+}
+
+// Cid returns the cid of the block header.
+func (b *EthHeader) Cid() cid.Cid {
+	return b.cid
+}
+
+// String is a helper for output
+func (b *EthHeader) String() string {
+	return fmt.Sprintf("<EthHeader %s>", b.cid)
+}
+
+// Loggable returns a map the type of IPLD Link.
+func (b *EthHeader) Loggable() map[string]interface{} {
+	return map[string]interface{}{
+		"type": "eth-block",
+	}
+}
+
+/*
+  Node INTERFACE
+*/
+
+// Resolve resolves a path through this node, stopping at any link boundary
+// and returning the object found as well as the remaining path to traverse
+func (b *EthHeader) Resolve(p []string) (interface{}, []string, error) {
+	if len(p) == 0 {
+		return b, nil, nil
+	}
+
+	first, rest := p[0], p[1:]
+
+	switch first {
+	case "parent":
+		return &node.Link{Cid: commonHashToCid(MEthHeader, b.ParentHash)}, rest, nil
+	case "receipts":
+		return &node.Link{Cid: commonHashToCid(MEthTxReceiptTrie, b.ReceiptHash)}, rest, nil
+	case "root":
+		return &node.Link{Cid: commonHashToCid(MEthStateTrie, b.Root)}, rest, nil
+	case "tx":
+		return &node.Link{Cid: commonHashToCid(MEthTxTrie, b.TxHash)}, rest, nil
+	case "uncles":
+		return &node.Link{Cid: commonHashToCid(MEthHeaderList, b.UncleHash)}, rest, nil
+	}
+
+	if len(p) != 1 {
+		return nil, nil, fmt.Errorf("unexpected path elements past %s", first)
+	}
+
+	switch first {
+	case "bloom":
+		return b.Bloom, nil, nil
+	case "coinbase":
+		return b.Coinbase, nil, nil
+	case "difficulty":
+		return b.Difficulty, nil, nil
+	case "extra":
+		// This is a []byte. By default they are marshalled into Base64.
+		return fmt.Sprintf("0x%x", b.Extra), nil, nil
+	case "gaslimit":
+		return b.GasLimit, nil, nil
+	case "gasused":
+		return b.GasUsed, nil, nil
+	case "mixdigest":
+		return b.MixDigest, nil, nil
+	case "nonce":
+		return b.Nonce, nil, nil
+	case "number":
+		return b.Number, nil, nil
+	case "time":
+		return b.Time, nil, nil
+	default:
+		return nil, nil, fmt.Errorf("no such link")
+	}
+}
+
+// Tree lists all paths within the object under 'path', and up to the given depth.
+// To list the entire object (similar to `find .`) pass "" and -1
+func (b *EthHeader) Tree(p string, depth int) []string {
+	if p != "" || depth == 0 {
+		return nil
+	}
+
+	return []string{
+		"time",
+		"bloom",
+		"coinbase",
+		"difficulty",
+		"extra",
+		"gaslimit",
+		"gasused",
+		"mixdigest",
+		"nonce",
+		"number",
+		"parent",
+		"receipts",
+		"root",
+		"tx",
+		"uncles",
+	}
+}
+
+// ResolveLink is a helper function that allows easier traversal of links through blocks
+func (b *EthHeader) ResolveLink(p []string) (*node.Link, []string, error) {
+	obj, rest, err := b.Resolve(p)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if lnk, ok := obj.(*node.Link); ok {
+		return lnk, rest, nil
+	}
+
+	return nil, nil, fmt.Errorf("resolved item was not a link")
+}
+
+// Copy will go away. It is here to comply with the Node interface.
+func (b *EthHeader) Copy() node.Node {
+	panic("implement me")
+}
+
+// Links is a helper function that returns all links within this object
+// HINT: Use `ipfs refs <cid>`
+func (b *EthHeader) Links() []*node.Link {
+	return []*node.Link{
+		{Cid: commonHashToCid(MEthHeader, b.ParentHash)},
+		{Cid: commonHashToCid(MEthTxReceiptTrie, b.ReceiptHash)},
+		{Cid: commonHashToCid(MEthStateTrie, b.Root)},
+		{Cid: commonHashToCid(MEthTxTrie, b.TxHash)},
+		{Cid: commonHashToCid(MEthHeaderList, b.UncleHash)},
+	}
+}
+
+// Stat will go away. It is here to comply with the Node interface.
+func (b *EthHeader) Stat() (*node.NodeStat, error) {
+	return &node.NodeStat{}, nil
+}
+
+// Size will go away. It is here to comply with the Node interface.
+func (b *EthHeader) Size() (uint64, error) {
+	return 0, nil
+}
+
+/*
+  EthHeader functions
+*/
+
+// MarshalJSON processes the block header into readable JSON format,
+// converting the right links into their cids, and keeping the original
+// hex hash, allowing the user to simplify external queries.
+func (b *EthHeader) MarshalJSON() ([]byte, error) {
+	out := map[string]interface{}{
+		"time":       b.Time,
+		"bloom":      b.Bloom,
+		"coinbase":   b.Coinbase,
+		"difficulty": b.Difficulty,
+		"extra":      fmt.Sprintf("0x%x", b.Extra),
+		"gaslimit":   b.GasLimit,
+		"gasused":    b.GasUsed,
+		"mixdigest":  b.MixDigest,
+		"nonce":      b.Nonce,
+		"number":     b.Number,
+		"parent":     commonHashToCid(MEthHeader, b.ParentHash),
+		"receipts":   commonHashToCid(MEthTxReceiptTrie, b.ReceiptHash),
+		"root":       commonHashToCid(MEthStateTrie, b.Root),
+		"tx":         commonHashToCid(MEthTxTrie, b.TxHash),
+		"uncles":     commonHashToCid(MEthHeaderList, b.UncleHash),
+	}
+	return json.Marshal(out)
+}
+
+// objJSONBlock defines the output of the JSON RPC API for either
+// "eth_BlockByHash" or "eth_BlockByHeader".
+type objJSONBlock struct {
+	Result objJSONBlockResult `json:"result"`
+}
+
+// objJSONBLockResult is the  nested struct that takes
+// the contents of the JSON field "result".
+type objJSONBlockResult struct {
+	types.Header           // Use its fields and unmarshaler
+	*objJSONBlockResultExt // Add these fields to the parsing
+}
+
+// objJSONBLockResultExt facilitates the composition
+// of the field "result", adding to the
+// `types.Header` fields, both ommers (their hashes) and transactions.
+type objJSONBlockResultExt struct {
+	OmmerHashes  []common.Hash        `json:"uncles"`
+	Transactions []*types.Transaction `json:"transactions"`
+}
+
+// UnmarshalJSON overrides the function types.Header.UnmarshalJSON, allowing us
+// to parse the fields of Header, plus ommer hashes and transactions.
+// (yes, ommer hashes. You will need to "eth_getUncleCountByBlockHash" per each ommer)
+func (o *objJSONBlockResult) UnmarshalJSON(input []byte) error {
+	err := o.Header.UnmarshalJSON(input)
+	if err != nil {
+		return err
+	}
+
+	o.objJSONBlockResultExt = &objJSONBlockResultExt{}
+	err = json.Unmarshal(input, o.objJSONBlockResultExt)
+	return err
+}

pkg/ipfs/ipld/eth_receipt.go

@@ -0,0 +1,199 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package ipld
+
+import (
+	"encoding/json"
+	"fmt"
+	"strconv"
+
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ipfs/go-cid"
+	node "github.com/ipfs/go-ipld-format"
+	mh "github.com/multiformats/go-multihash"
+)
+
+type EthReceipt struct {
+	*types.ReceiptForStorage
+
+	rawdata []byte
+	cid     cid.Cid
+}
+
+// Static (compile time) check that EthReceipt satisfies the node.Node interface.
+var _ node.Node = (*EthReceipt)(nil)
+
+/*
+  INPUT
+*/
+
+// NewReceipt converts a types.ReceiptForStorage to an EthReceipt IPLD node
+func NewReceipt(receipt *types.ReceiptForStorage) (*EthReceipt, error) {
+	receiptRLP, err := rlp.EncodeToBytes(receipt)
+	if err != nil {
+		return nil, err
+	}
+	c, err := rawdataToCid(MEthTxReceipt, receiptRLP, mh.KECCAK_256)
+	if err != nil {
+		return nil, err
+	}
+	return &EthReceipt{
+		ReceiptForStorage: receipt,
+		cid:               c,
+		rawdata:           receiptRLP,
+	}, nil
+}
+
+/*
+ OUTPUT
+*/
+
+// DecodeEthReceipt takes a cid and its raw binary data
+// from IPFS and returns an EthReceipt object for further processing.
+func DecodeEthReceipt(c cid.Cid, b []byte) (*EthReceipt, error) {
+	var r *types.ReceiptForStorage
+	if err := rlp.DecodeBytes(b, r); err != nil {
+		return nil, err
+	}
+	return &EthReceipt{
+		ReceiptForStorage: r,
+		cid:               c,
+		rawdata:           b,
+	}, nil
+}
+
+/*
+  Block INTERFACE
+*/
+
+func (node *EthReceipt) RawData() []byte {
+	return node.rawdata
+}
+
+func (node *EthReceipt) Cid() cid.Cid {
+	return node.cid
+}
+
+// String is a helper for output
+func (r *EthReceipt) String() string {
+	return fmt.Sprintf("<EthereumReceipt %s>", r.cid)
+}
+
+// Loggable returns in a map the type of IPLD Link.
+func (r *EthReceipt) Loggable() map[string]interface{} {
+	return map[string]interface{}{
+		"type": "eth-receipt",
+	}
+}
+
+// Resolve resolves a path through this node, stopping at any link boundary
+// and returning the object found as well as the remaining path to traverse
+func (r *EthReceipt) Resolve(p []string) (interface{}, []string, error) {
+	if len(p) == 0 {
+		return r, nil, nil
+	}
+
+	if len(p) > 1 {
+		return nil, nil, fmt.Errorf("unexpected path elements past %s", p[0])
+	}
+
+	switch p[0] {
+
+	case "root":
+		return r.PostState, nil, nil
+	case "status":
+		return r.Status, nil, nil
+	case "cumulativeGasUsed":
+		return r.CumulativeGasUsed, nil, nil
+	case "logsBloom":
+		return r.Bloom, nil, nil
+	case "logs":
+		return r.Logs, nil, nil
+	case "transactionHash":
+		return r.TxHash, nil, nil
+	case "contractAddress":
+		return r.ContractAddress, nil, nil
+	case "gasUsed":
+		return r.GasUsed, nil, nil
+	default:
+		return nil, nil, fmt.Errorf("no such link")
+	}
+}
+
+// Tree lists all paths within the object under 'path', and up to the given depth.
+// To list the entire object (similar to `find .`) pass "" and -1
+func (r *EthReceipt) Tree(p string, depth int) []string {
+	if p != "" || depth == 0 {
+		return nil
+	}
+	return []string{"root", "status", "cumulativeGasUsed", "logsBloom", "logs", "transactionHash", "contractAddress", "gasUsed"}
+}
+
+// ResolveLink is a helper function that calls resolve and asserts the
+// output is a link
+func (r *EthReceipt) ResolveLink(p []string) (*node.Link, []string, error) {
+	obj, rest, err := r.Resolve(p)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if lnk, ok := obj.(*node.Link); ok {
+		return lnk, rest, nil
+	}
+
+	return nil, nil, fmt.Errorf("resolved item was not a link")
+}
+
+// Copy will go away. It is here to comply with the Node interface.
+func (*EthReceipt) Copy() node.Node {
+	panic("implement me")
+}
+
+// Links is a helper function that returns all links within this object
+func (*EthReceipt) Links() []*node.Link {
+	return nil
+}
+
+// Stat will go away. It is here to comply with the interface.
+func (r *EthReceipt) Stat() (*node.NodeStat, error) {
+	return &node.NodeStat{}, nil
+}
+
+// Size will go away. It is here to comply with the interface.
+func (r *EthReceipt) Size() (uint64, error) {
+	return strconv.ParseUint((*types.Receipt)(r.ReceiptForStorage).Size().String(), 10, 64)
+}
+
+/*
+  EthReceipt functions
+*/
+
+// MarshalJSON processes the receipt into readable JSON format.
+func (r *EthReceipt) MarshalJSON() ([]byte, error) {
+	out := map[string]interface{}{
+		"root":              r.PostState,
+		"status":            r.Status,
+		"cumulativeGasUsed": r.CumulativeGasUsed,
+		"logsBloom":         r.Bloom,
+		"logs":              r.Logs,
+		"transactionHash":   r.TxHash,
+		"contractAddress":   r.ContractAddress,
+		"gasUsed":           r.GasUsed,
+	}
+	return json.Marshal(out)
+}

pkg/ipfs/ipld/eth_state.go

@@ -0,0 +1,109 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package ipld
+
+import (
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ipfs/go-cid"
+	node "github.com/ipfs/go-ipld-format"
+	mh "github.com/multiformats/go-multihash"
+)
+
+// EthStateTrie (eth-state-trie, codec 0x96), represents
+// a node from the state trie in ethereum.
+type EthStateTrie struct {
+	*TrieNode
+}
+
+// Static (compile time) check that EthStateTrie satisfies the node.Node interface.
+var _ node.Node = (*EthStateTrie)(nil)
+
+/*
+  INPUT
+*/
+
+// FromStateTrieRLP takes the RLP bytes of an ethereum
+// state trie node to return it as an IPLD node for further processing.
+func FromStateTrieRLP(stateNodeRLP []byte) (*EthStateTrie, error) {
+	c, err := rawdataToCid(MEthStateTrie, stateNodeRLP, mh.KECCAK_256)
+	if err != nil {
+		return nil, err
+	}
+	return DecodeEthStateTrie(c, stateNodeRLP)
+}
+
+/*
+  OUTPUT
+*/
+
+// DecodeEthStateTrie returns an EthStateTrie object from its cid and rawdata.
+func DecodeEthStateTrie(c cid.Cid, b []byte) (*EthStateTrie, error) {
+	tn, err := decodeTrieNode(c, b, decodeEthStateTrieLeaf)
+	if err != nil {
+		return nil, err
+	}
+	return &EthStateTrie{TrieNode: tn}, nil
+}
+
+// decodeEthStateTrieLeaf parses a eth-tx-trie leaf
+// from decoded RLP elements
+func decodeEthStateTrieLeaf(i []interface{}) ([]interface{}, error) {
+	var account EthAccount
+	if err := rlp.DecodeBytes(i[1].([]byte), &account); err != nil {
+		return nil, err
+	}
+	c, err := rawdataToCid(MEthAccountSnapshot, i[1].([]byte), mh.KECCAK_256)
+	if err != nil {
+		return nil, err
+	}
+	return []interface{}{
+		i[0].([]byte),
+		&EthAccountSnapshot{
+			EthAccount: &account,
+			cid:        c,
+			rawdata:    i[1].([]byte),
+		},
+	}, nil
+}
+
+/*
+  Block INTERFACE
+*/
+
+// RawData returns the binary of the RLP encode of the state trie node.
+func (st *EthStateTrie) RawData() []byte {
+	return st.rawdata
+}
+
+// Cid returns the cid of the state trie node.
+func (st *EthStateTrie) Cid() cid.Cid {
+	return st.cid
+}
+
+// String is a helper for output
+func (st *EthStateTrie) String() string {
+	return fmt.Sprintf("<EthereumStateTrie %s>", st.cid)
+}
+
+// Loggable returns in a map the type of IPLD Link.
+func (st *EthStateTrie) Loggable() map[string]interface{} {
+	return map[string]interface{}{
+		"type": "eth-state-trie",
+	}
+}

pkg/ipfs/ipld/eth_storage.go

@@ -0,0 +1,96 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package ipld
+
+import (
+	"fmt"
+
+	"github.com/ipfs/go-cid"
+	node "github.com/ipfs/go-ipld-format"
+	mh "github.com/multiformats/go-multihash"
+)
+
+// EthStorageTrie (eth-storage-trie, codec 0x98), represents
+// a node from the storage trie in ethereum.
+type EthStorageTrie struct {
+	*TrieNode
+}
+
+// Static (compile time) check that EthStorageTrie satisfies the node.Node interface.
+var _ node.Node = (*EthStorageTrie)(nil)
+
+/*
+  INPUT
+*/
+
+// FromStorageTrieRLP takes the RLP bytes of an ethereum
+// storage trie node to return it as an IPLD node for further processing.
+func FromStorageTrieRLP(storageNodeRLP []byte) (*EthStorageTrie, error) {
+	c, err := rawdataToCid(MEthStorageTrie, storageNodeRLP, mh.KECCAK_256)
+	if err != nil {
+		return nil, err
+	}
+	return DecodeEthStorageTrie(c, storageNodeRLP)
+}
+
+/*
+  OUTPUT
+*/
+
+// DecodeEthStorageTrie returns an EthStorageTrie object from its cid and rawdata.
+func DecodeEthStorageTrie(c cid.Cid, b []byte) (*EthStorageTrie, error) {
+	tn, err := decodeTrieNode(c, b, decodeEthStorageTrieLeaf)
+	if err != nil {
+		return nil, err
+	}
+	return &EthStorageTrie{TrieNode: tn}, nil
+}
+
+// decodeEthStorageTrieLeaf parses a eth-tx-trie leaf
+// from decoded RLP elements
+func decodeEthStorageTrieLeaf(i []interface{}) ([]interface{}, error) {
+	return []interface{}{
+		i[0].([]byte),
+		i[1].([]byte),
+	}, nil
+}
+
+/*
+  Block INTERFACE
+*/
+
+// RawData returns the binary of the RLP encode of the storage trie node.
+func (st *EthStorageTrie) RawData() []byte {
+	return st.rawdata
+}
+
+// Cid returns the cid of the storage trie node.
+func (st *EthStorageTrie) Cid() cid.Cid {
+	return st.cid
+}
+
+// String is a helper for output
+func (st *EthStorageTrie) String() string {
+	return fmt.Sprintf("<EthereumStorageTrie %s>", st.cid)
+}
+
+// Loggable returns in a map the type of IPLD Link.
+func (st *EthStorageTrie) Loggable() map[string]interface{} {
+	return map[string]interface{}{
+		"type": "eth-storage-trie",
+	}
+}

pkg/ipfs/ipld/eth_tx.go

@@ -0,0 +1,215 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package ipld
+
+import (
+	"encoding/json"
+	"fmt"
+	"strconv"
+
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ipfs/go-cid"
+	node "github.com/ipfs/go-ipld-format"
+	mh "github.com/multiformats/go-multihash"
+)
+
+// EthTx (eth-tx codec 0x93) represents an ethereum transaction
+type EthTx struct {
+	*types.Transaction
+
+	cid     cid.Cid
+	rawdata []byte
+}
+
+// Static (compile time) check that EthTx satisfies the node.Node interface.
+var _ node.Node = (*EthTx)(nil)
+
+/*
+  INPUT
+*/
+
+// NewEthTx converts a *types.Transaction to an EthTx IPLD node
+func NewEthTx(tx *types.Transaction) (*EthTx, error) {
+	txRLP, err := rlp.EncodeToBytes(tx)
+	if err != nil {
+		return nil, err
+	}
+	c, err := rawdataToCid(MEthTx, txRLP, mh.KECCAK_256)
+	if err != nil {
+		return nil, err
+	}
+	return &EthTx{
+		Transaction: tx,
+		cid:         c,
+		rawdata:     txRLP,
+	}, nil
+}
+
+/*
+ OUTPUT
+*/
+
+// DecodeEthTx takes a cid and its raw binary data
+// from IPFS and returns an EthTx object for further processing.
+func DecodeEthTx(c cid.Cid, b []byte) (*EthTx, error) {
+	var t *types.Transaction
+	if err := rlp.DecodeBytes(b, t); err != nil {
+		return nil, err
+	}
+	return &EthTx{
+		Transaction: t,
+		cid:         c,
+		rawdata:     b,
+	}, nil
+}
+
+/*
+  Block INTERFACE
+*/
+
+// RawData returns the binary of the RLP encode of the transaction.
+func (t *EthTx) RawData() []byte {
+	return t.rawdata
+}
+
+// Cid returns the cid of the transaction.
+func (t *EthTx) Cid() cid.Cid {
+	return t.cid
+}
+
+// String is a helper for output
+func (t *EthTx) String() string {
+	return fmt.Sprintf("<EthereumTx %s>", t.cid)
+}
+
+// Loggable returns in a map the type of IPLD Link.
+func (t *EthTx) Loggable() map[string]interface{} {
+	return map[string]interface{}{
+		"type": "eth-tx",
+	}
+}
+
+/*
+  Node INTERFACE
+*/
+
+// Resolve resolves a path through this node, stopping at any link boundary
+// and returning the object found as well as the remaining path to traverse
+func (t *EthTx) Resolve(p []string) (interface{}, []string, error) {
+	if len(p) == 0 {
+		return t, nil, nil
+	}
+
+	if len(p) > 1 {
+		return nil, nil, fmt.Errorf("unexpected path elements past %s", p[0])
+	}
+
+	switch p[0] {
+
+	case "gas":
+		return t.Gas(), nil, nil
+	case "gasPrice":
+		return t.GasPrice(), nil, nil
+	case "input":
+		return fmt.Sprintf("%x", t.Data()), nil, nil
+	case "nonce":
+		return t.Nonce(), nil, nil
+	case "r":
+		_, r, _ := t.RawSignatureValues()
+		return hexutil.EncodeBig(r), nil, nil
+	case "s":
+		_, _, s := t.RawSignatureValues()
+		return hexutil.EncodeBig(s), nil, nil
+	case "toAddress":
+		return t.To(), nil, nil
+	case "v":
+		v, _, _ := t.RawSignatureValues()
+		return hexutil.EncodeBig(v), nil, nil
+	case "value":
+		return hexutil.EncodeBig(t.Value()), nil, nil
+	default:
+		return nil, nil, fmt.Errorf("no such link")
+	}
+}
+
+// Tree lists all paths within the object under 'path', and up to the given depth.
+// To list the entire object (similar to `find .`) pass "" and -1
+func (t *EthTx) Tree(p string, depth int) []string {
+	if p != "" || depth == 0 {
+		return nil
+	}
+	return []string{"gas", "gasPrice", "input", "nonce", "r", "s", "toAddress", "v", "value"}
+}
+
+// ResolveLink is a helper function that calls resolve and asserts the
+// output is a link
+func (t *EthTx) ResolveLink(p []string) (*node.Link, []string, error) {
+	obj, rest, err := t.Resolve(p)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if lnk, ok := obj.(*node.Link); ok {
+		return lnk, rest, nil
+	}
+
+	return nil, nil, fmt.Errorf("resolved item was not a link")
+}
+
+// Copy will go away. It is here to comply with the interface.
+func (t *EthTx) Copy() node.Node {
+	panic("implement me")
+}
+
+// Links is a helper function that returns all links within this object
+func (t *EthTx) Links() []*node.Link {
+	return nil
+}
+
+// Stat will go away. It is here to comply with the interface.
+func (t *EthTx) Stat() (*node.NodeStat, error) {
+	return &node.NodeStat{}, nil
+}
+
+// Size will go away. It is here to comply with the interface.
+func (t *EthTx) Size() (uint64, error) {
+	return strconv.ParseUint(t.Transaction.Size().String(), 10, 64)
+}
+
+/*
+  EthTx functions
+*/
+
+// MarshalJSON processes the transaction into readable JSON format.
+func (t *EthTx) MarshalJSON() ([]byte, error) {
+	v, r, s := t.RawSignatureValues()
+
+	out := map[string]interface{}{
+		"gas":       t.Gas(),
+		"gasPrice":  hexutil.EncodeBig(t.GasPrice()),
+		"input":     fmt.Sprintf("%x", t.Data()),
+		"nonce":     t.Nonce(),
+		"r":         hexutil.EncodeBig(r),
+		"s":         hexutil.EncodeBig(s),
+		"toAddress": t.To(),
+		"v":         hexutil.EncodeBig(v),
+		"value":     hexutil.EncodeBig(t.Value()),
+	}
+	return json.Marshal(out)
+}

pkg/ipfs/ipld/shared.go

@@ -0,0 +1,102 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package ipld
+
+import (
+	"bytes"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ipfs/go-cid"
+	mh "github.com/multiformats/go-multihash"
+)
+
+// IPLD Codecs for Ethereum
+// See the authoritative document:
+// https://github.com/multiformats/multicodec/blob/master/table.csv
+const (
+	RawBinary           = 0x55
+	MEthHeader          = 0x90
+	MEthHeaderList      = 0x91
+	MEthTxTrie          = 0x92
+	MEthTx              = 0x93
+	MEthTxReceiptTrie   = 0x94
+	MEthTxReceipt       = 0x95
+	MEthStateTrie       = 0x96
+	MEthAccountSnapshot = 0x97
+	MEthStorageTrie     = 0x98
+	MBitcoinHeader      = 0xb0
+	MBitcoinTx          = 0xb1
+)
+
+// rawdataToCid takes the desired codec and a slice of bytes
+// and returns the proper cid of the object.
+func rawdataToCid(codec uint64, rawdata []byte, multiHash uint64) (cid.Cid, error) {
+	c, err := cid.Prefix{
+		Codec:    codec,
+		Version:  1,
+		MhType:   multiHash,
+		MhLength: -1,
+	}.Sum(rawdata)
+	if err != nil {
+		return cid.Cid{}, err
+	}
+	return c, nil
+}
+
+// keccak256ToCid takes a keccak256 hash and returns its cid based on
+// the codec given.
+func keccak256ToCid(codec uint64, h []byte) cid.Cid {
+	buf, err := mh.Encode(h, mh.KECCAK_256)
+	if err != nil {
+		panic(err)
+	}
+
+	return cid.NewCidV1(codec, mh.Multihash(buf))
+}
+
+// commonHashToCid takes a go-ethereum common.Hash and returns its
+// cid based on the codec given,
+func commonHashToCid(codec uint64, h common.Hash) cid.Cid {
+	mhash, err := mh.Encode(h[:], mh.KECCAK_256)
+	if err != nil {
+		panic(err)
+	}
+
+	return cid.NewCidV1(codec, mhash)
+}
+
+// sha256ToCid takes a sha246 hash and returns its cid based on the
+// codec given
+func sha256ToCid(codec uint64, h []byte) cid.Cid {
+	hash, err := mh.Encode(h, mh.DBL_SHA2_256)
+	if err != nil {
+		panic(err)
+	}
+
+	return cid.NewCidV1(codec, hash)
+}
+
+// getRLP encodes the given object to RLP returning its bytes.
+func getRLP(object interface{}) []byte {
+	buf := new(bytes.Buffer)
+	if err := rlp.Encode(buf, object); err != nil {
+		panic(err)
+	}
+
+	return buf.Bytes()
+}

pkg/ipfs/ipld/trie_node.go

@@ -0,0 +1,440 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package ipld
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ipfs/go-cid"
+	node "github.com/ipfs/go-ipld-format"
+)
+
+// TrieNode is the general abstraction for
+//ethereum IPLD trie nodes.
+type TrieNode struct {
+	// leaf, extension or branch
+	nodeKind string
+
+	// If leaf or extension: [0] is key, [1] is val.
+	// If branch: [0] - [16] are children.
+	elements []interface{}
+
+	// IPLD block information
+	cid     cid.Cid
+	rawdata []byte
+}
+
+/*
+  OUTPUT
+*/
+
+type trieNodeLeafDecoder func([]interface{}) ([]interface{}, error)
+
+// decodeTrieNode returns a TrieNode object from an IPLD block's
+// cid and rawdata.
+func decodeTrieNode(c cid.Cid, b []byte,
+	leafDecoder trieNodeLeafDecoder) (*TrieNode, error) {
+	var (
+		i, decoded, elements []interface{}
+		nodeKind             string
+		err                  error
+	)
+
+	err = rlp.DecodeBytes(b, &i)
+	if err != nil {
+		return nil, err
+	}
+
+	codec := c.Type()
+	switch len(i) {
+	case 2:
+		nodeKind, decoded, err = decodeCompactKey(i)
+		if err != nil {
+			return nil, err
+		}
+
+		if nodeKind == "extension" {
+			elements, err = parseTrieNodeExtension(decoded, codec)
+		}
+		if nodeKind == "leaf" {
+			elements, err = leafDecoder(decoded)
+		}
+		if nodeKind != "extension" && nodeKind != "leaf" {
+			return nil, fmt.Errorf("unexpected nodeKind returned from decoder")
+		}
+	case 17:
+		nodeKind = "branch"
+		elements, err = parseTrieNodeBranch(i, codec)
+		if err != nil {
+			return nil, err
+		}
+	default:
+		return nil, fmt.Errorf("unknown trie node type")
+	}
+
+	return &TrieNode{
+		nodeKind: nodeKind,
+		elements: elements,
+		rawdata:  b,
+		cid:      c,
+	}, nil
+}
+
+// decodeCompactKey takes a compact key, and returns its nodeKind and value.
+func decodeCompactKey(i []interface{}) (string, []interface{}, error) {
+	first := i[0].([]byte)
+	last := i[1].([]byte)
+
+	switch first[0] / 16 {
+	case '\x00':
+		return "extension", []interface{}{
+			nibbleToByte(first)[2:],
+			last,
+		}, nil
+	case '\x01':
+		return "extension", []interface{}{
+			nibbleToByte(first)[1:],
+			last,
+		}, nil
+	case '\x02':
+		return "leaf", []interface{}{
+			nibbleToByte(first)[2:],
+			last,
+		}, nil
+	case '\x03':
+		return "leaf", []interface{}{
+			nibbleToByte(first)[1:],
+			last,
+		}, nil
+	default:
+		return "", nil, fmt.Errorf("unknown hex prefix")
+	}
+}
+
+// parseTrieNodeExtension helper improves readability
+func parseTrieNodeExtension(i []interface{}, codec uint64) ([]interface{}, error) {
+	return []interface{}{
+		i[0].([]byte),
+		keccak256ToCid(codec, i[1].([]byte)),
+	}, nil
+}
+
+// parseTrieNodeBranch helper improves readability
+func parseTrieNodeBranch(i []interface{}, codec uint64) ([]interface{}, error) {
+	var out []interface{}
+
+	for _, vi := range i {
+		v := vi.([]byte)
+
+		switch len(v) {
+		case 0:
+			out = append(out, nil)
+		case 32:
+			out = append(out, keccak256ToCid(codec, v))
+		default:
+			return nil, fmt.Errorf("unrecognized object: %v", v)
+		}
+	}
+
+	return out, nil
+}
+
+/*
+  Node INTERFACE
+*/
+
+// Resolve resolves a path through this node, stopping at any link boundary
+// and returning the object found as well as the remaining path to traverse
+func (t *TrieNode) Resolve(p []string) (interface{}, []string, error) {
+	switch t.nodeKind {
+	case "extension":
+		return t.resolveTrieNodeExtension(p)
+	case "leaf":
+		return t.resolveTrieNodeLeaf(p)
+	case "branch":
+		return t.resolveTrieNodeBranch(p)
+	default:
+		return nil, nil, fmt.Errorf("nodeKind case not implemented")
+	}
+}
+
+// Tree lists all paths within the object under 'path', and up to the given depth.
+// To list the entire object (similar to `find .`) pass "" and -1
+func (t *TrieNode) Tree(p string, depth int) []string {
+	if p != "" || depth == 0 {
+		return nil
+	}
+
+	var out []string
+
+	switch t.nodeKind {
+	case "extension":
+		var val string
+		for _, e := range t.elements[0].([]byte) {
+			val += fmt.Sprintf("%x", e)
+		}
+		return []string{val}
+	case "branch":
+		for i, elem := range t.elements {
+			if _, ok := elem.(*cid.Cid); ok {
+				out = append(out, fmt.Sprintf("%x", i))
+			}
+		}
+		return out
+
+	default:
+		return nil
+	}
+}
+
+// ResolveLink is a helper function that calls resolve and asserts the
+// output is a link
+func (t *TrieNode) ResolveLink(p []string) (*node.Link, []string, error) {
+	obj, rest, err := t.Resolve(p)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	lnk, ok := obj.(*node.Link)
+	if !ok {
+		return nil, nil, fmt.Errorf("was not a link")
+	}
+
+	return lnk, rest, nil
+}
+
+// Copy will go away. It is here to comply with the interface.
+func (t *TrieNode) Copy() node.Node {
+	panic("dont use this yet")
+}
+
+// Links is a helper function that returns all links within this object
+func (t *TrieNode) Links() []*node.Link {
+	var out []*node.Link
+
+	for _, i := range t.elements {
+		c, ok := i.(cid.Cid)
+		if ok {
+			out = append(out, &node.Link{Cid: c})
+		}
+	}
+
+	return out
+}
+
+// Stat will go away. It is here to comply with the interface.
+func (t *TrieNode) Stat() (*node.NodeStat, error) {
+	return &node.NodeStat{}, nil
+}
+
+// Size will go away. It is here to comply with the interface.
+func (t *TrieNode) Size() (uint64, error) {
+	return 0, nil
+}
+
+/*
+  TrieNode functions
+*/
+
+// MarshalJSON processes the transaction trie into readable JSON format.
+func (t *TrieNode) MarshalJSON() ([]byte, error) {
+	var out map[string]interface{}
+
+	switch t.nodeKind {
+	case "extension":
+		fallthrough
+	case "leaf":
+		var hexPrefix string
+		for _, e := range t.elements[0].([]byte) {
+			hexPrefix += fmt.Sprintf("%x", e)
+		}
+
+		// if we got a byte we need to do this casting otherwise
+		// it will be marshaled to a base64 encoded value
+		if _, ok := t.elements[1].([]byte); ok {
+			var hexVal string
+			for _, e := range t.elements[1].([]byte) {
+				hexVal += fmt.Sprintf("%x", e)
+			}
+
+			t.elements[1] = hexVal
+		}
+
+		out = map[string]interface{}{
+			"type":    t.nodeKind,
+			hexPrefix: t.elements[1],
+		}
+
+	case "branch":
+		out = map[string]interface{}{
+			"type": "branch",
+			"0":    t.elements[0],
+			"1":    t.elements[1],
+			"2":    t.elements[2],
+			"3":    t.elements[3],
+			"4":    t.elements[4],
+			"5":    t.elements[5],
+			"6":    t.elements[6],
+			"7":    t.elements[7],
+			"8":    t.elements[8],
+			"9":    t.elements[9],
+			"a":    t.elements[10],
+			"b":    t.elements[11],
+			"c":    t.elements[12],
+			"d":    t.elements[13],
+			"e":    t.elements[14],
+			"f":    t.elements[15],
+		}
+	default:
+		return nil, fmt.Errorf("nodeKind %s not supported", t.nodeKind)
+	}
+
+	return json.Marshal(out)
+}
+
+// nibbleToByte expands the nibbles of a byte slice into their own bytes.
+func nibbleToByte(k []byte) []byte {
+	var out []byte
+
+	for _, b := range k {
+		out = append(out, b/16)
+		out = append(out, b%16)
+	}
+
+	return out
+}
+
+// Resolve reading conveniences
+func (t *TrieNode) resolveTrieNodeExtension(p []string) (interface{}, []string, error) {
+	nibbles := t.elements[0].([]byte)
+	idx, rest := shiftFromPath(p, len(nibbles))
+	if len(idx) < len(nibbles) {
+		return nil, nil, fmt.Errorf("not enough nibbles to traverse this extension")
+	}
+
+	for _, i := range idx {
+		if getHexIndex(string(i)) == -1 {
+			return nil, nil, fmt.Errorf("invalid path element")
+		}
+	}
+
+	for i, n := range nibbles {
+		if string(idx[i]) != fmt.Sprintf("%x", n) {
+			return nil, nil, fmt.Errorf("no such link in this extension")
+		}
+	}
+
+	return &node.Link{Cid: t.elements[1].(cid.Cid)}, rest, nil
+}
+
+func (t *TrieNode) resolveTrieNodeLeaf(p []string) (interface{}, []string, error) {
+	nibbles := t.elements[0].([]byte)
+
+	if len(nibbles) != 0 {
+		idx, rest := shiftFromPath(p, len(nibbles))
+		if len(idx) < len(nibbles) {
+			return nil, nil, fmt.Errorf("not enough nibbles to traverse this leaf")
+		}
+
+		for _, i := range idx {
+			if getHexIndex(string(i)) == -1 {
+				return nil, nil, fmt.Errorf("invalid path element")
+			}
+		}
+
+		for i, n := range nibbles {
+			if string(idx[i]) != fmt.Sprintf("%x", n) {
+				return nil, nil, fmt.Errorf("no such link in this extension")
+			}
+		}
+
+		p = rest
+	}
+
+	link, ok := t.elements[1].(node.Node)
+	if !ok {
+		return nil, nil, fmt.Errorf("leaf children is not an IPLD node")
+	}
+
+	return link.Resolve(p)
+}
+
+func (t *TrieNode) resolveTrieNodeBranch(p []string) (interface{}, []string, error) {
+	idx, rest := shiftFromPath(p, 1)
+	hidx := getHexIndex(idx)
+	if hidx == -1 {
+		return nil, nil, fmt.Errorf("incorrect path")
+	}
+
+	child := t.elements[hidx]
+	if child != nil {
+		return &node.Link{Cid: child.(cid.Cid)}, rest, nil
+	}
+	return nil, nil, fmt.Errorf("no such link in this branch")
+}
+
+// shiftFromPath extracts from a given path (as a slice of strings)
+// the given number of elements as a single string, returning whatever
+// it has not taken.
+//
+// Examples:
+// ["0", "a", "something"] and 1 -> "0" and ["a", "something"]
+// ["ab", "c", "d", "1"] and 2 -> "ab" and ["c", "d", "1"]
+// ["abc", "d", "1"] and 2 -> "ab" and ["c", "d", "1"]
+func shiftFromPath(p []string, i int) (string, []string) {
+	var (
+		out  string
+		rest []string
+	)
+
+	for _, pe := range p {
+		re := ""
+		for _, c := range pe {
+			if len(out) < i {
+				out += string(c)
+			} else {
+				re += string(c)
+			}
+		}
+
+		if len(out) == i && re != "" {
+			rest = append(rest, re)
+		}
+	}
+
+	return out, rest
+}
+
+// getHexIndex returns to you the integer 0 - 15 equivalent to your
+// string character if applicable, or -1 otherwise.
+func getHexIndex(s string) int {
+	if len(s) != 1 {
+		return -1
+	}
+
+	c := byte(s[0])
+	switch {
+	case '0' <= c && c <= '9':
+		return int(c - '0')
+	case 'a' <= c && c <= 'f':
+		return int(c - 'a' + 10)
+	}
+
+	return -1
+}