fix state and storage iplds/dag_putters; add support for eth rct trie and eth and btc tx tries

pkg/ipfs/dag_putters/btc_header.go

@@ -19,8 +19,6 @@ package dag_putters
 import (
 	"fmt"
 
-	"github.com/btcsuite/btcd/wire"
-
 	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
 	"github.com/vulcanize/vulcanizedb/pkg/ipfs/ipld"
 )
@@ -34,16 +32,12 @@ func NewBtcHeaderDagPutter(adder *ipfs.IPFS) *BtcHeaderDagPutter {
 }
 
 func (bhdp *BtcHeaderDagPutter) DagPut(raw interface{}) ([]string, error) {
-	header, ok := raw.(*wire.BlockHeader)
+	header, ok := raw.(*ipld.BtcHeader)
 	if !ok {
-		return nil, fmt.Errorf("BtcHeaderDagPutter expected input type %T got %T", &wire.BlockHeader{}, raw)
+		return nil, fmt.Errorf("BtcHeaderDagPutter expected input type %T got %T", &ipld.BtcHeader{}, raw)
 	}
-	node, err := ipld.NewBtcHeader(header)
+	if err := bhdp.adder.Add(header); err != nil {
-	if err != nil {
 		return nil, err
 	}
-	if err := bhdp.adder.Add(node); err != nil {
+	return []string{header.Cid().String()}, nil
-		return nil, err
-	}
-	return []string{node.Cid().String()}, nil
 }

pkg/ipfs/dag_putters/btc_tx.go

@@ -19,8 +19,6 @@ package dag_putters
 import (
 	"fmt"
 
-	"github.com/btcsuite/btcutil"
-
 	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
 	"github.com/vulcanize/vulcanizedb/pkg/ipfs/ipld"
 )
@@ -34,20 +32,16 @@ func NewBtcTxDagPutter(adder *ipfs.IPFS) *BtcTxDagPutter {
 }
 
 func (etdp *BtcTxDagPutter) DagPut(raw interface{}) ([]string, error) {
-	transactions, ok := raw.([]*btcutil.Tx)
+	transactions, ok := raw.([]*ipld.BtcTx)
 	if !ok {
-		return nil, fmt.Errorf("BtcTxDagPutter expected input type %T got %T", []*btcutil.Tx{}, raw)
+		return nil, fmt.Errorf("BtcTxDagPutter expected input type %T got %T", []*ipld.BtcTx{}, raw)
 	}
 	cids := make([]string, len(transactions))
 	for i, transaction := range transactions {
-		node, err := ipld.NewBtcTx(transaction.MsgTx())
+		if err := etdp.adder.Add(transaction); err != nil {
-		if err != nil {
 			return nil, err
 		}
-		if err := etdp.adder.Add(node); err != nil {
+		cids[i] = transaction.Cid().String()
-			return nil, err
-		}
-		cids[i] = node.Cid().String()
 	}
 	return cids, nil
 }

pkg/ipfs/dag_putters/eth_header.go

@@ -19,8 +19,6 @@ package dag_putters
 import (
 	"fmt"
 
-	"github.com/ethereum/go-ethereum/core/types"
-
 	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
 	"github.com/vulcanize/vulcanizedb/pkg/ipfs/ipld"
 )
@@ -34,16 +32,12 @@ func NewEthBlockHeaderDagPutter(adder *ipfs.IPFS) *EthHeaderDagPutter {
 }
 
 func (bhdp *EthHeaderDagPutter) DagPut(raw interface{}) ([]string, error) {
-	header, ok := raw.(*types.Header)
+	header, ok := raw.(*ipld.EthHeader)
 	if !ok {
-		return nil, fmt.Errorf("EthHeaderDagPutter expected input type %T got %T", &types.Header{}, raw)
+		return nil, fmt.Errorf("EthHeaderDagPutter expected input type %T got %T", &ipld.EthHeader{}, raw)
 	}
-	node, err := ipld.NewEthHeader(header)
+	if err := bhdp.adder.Add(header); err != nil {
-	if err != nil {
 		return nil, err
 	}
-	if err := bhdp.adder.Add(node); err != nil {
+	return []string{header.Cid().String()}, nil
-		return nil, err
-	}
-	return []string{node.Cid().String()}, nil
 }

pkg/ipfs/dag_putters/eth_receipt.go

@@ -19,8 +19,6 @@ package dag_putters
 import (
 	"fmt"
 
-	"github.com/ethereum/go-ethereum/core/types"
-
 	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
 	"github.com/vulcanize/vulcanizedb/pkg/ipfs/ipld"
 )
@@ -34,20 +32,16 @@ func NewEthReceiptDagPutter(adder *ipfs.IPFS) *EthReceiptDagPutter {
 }
 
 func (erdp *EthReceiptDagPutter) DagPut(raw interface{}) ([]string, error) {
-	receipts, ok := raw.(types.Receipts)
+	receipts, ok := raw.([]*ipld.EthReceipt)
 	if !ok {
-		return nil, fmt.Errorf("EthReceiptDagPutter expected input type %T got type %T", types.Receipts{}, raw)
+		return nil, fmt.Errorf("EthReceiptDagPutter expected input type %T got type %T", []*ipld.EthReceipt{}, raw)
 	}
 	cids := make([]string, len(receipts))
 	for i, receipt := range receipts {
-		node, err := ipld.NewReceipt(receipt)
+		if err := erdp.adder.Add(receipt); err != nil {
-		if err != nil {
 			return nil, err
 		}
-		if err := erdp.adder.Add(node); err != nil {
+		cids[i] = receipt.Cid().String()
-			return nil, err
-		}
-		cids[i] = node.Cid().String()
 	}
 	return cids, nil
 }

pkg/ipfs/dag_putters/eth_receipt_trie.go

@@ -0,0 +1,47 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package dag_putters
+
+import (
+	"fmt"
+
+	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
+	"github.com/vulcanize/vulcanizedb/pkg/ipfs/ipld"
+)
+
+type EthRctTrieDagPutter struct {
+	adder *ipfs.IPFS
+}
+
+func NewEthRctTrieDagPutter(adder *ipfs.IPFS) *EthRctTrieDagPutter {
+	return &EthRctTrieDagPutter{adder: adder}
+}
+
+func (etdp *EthRctTrieDagPutter) DagPut(raw interface{}) ([]string, error) {
+	rctTrieNodes, ok := raw.([]*ipld.EthRctTrie)
+	if !ok {
+		return nil, fmt.Errorf("EthRctTrieDagPutter expected input type %T got %T", []*ipld.EthRctTrie{}, raw)
+	}
+	cids := make([]string, len(rctTrieNodes))
+	for i, rctNode := range rctTrieNodes {
+		if err := etdp.adder.Add(rctNode); err != nil {
+			return nil, err
+		}
+		cids[i] = rctNode.Cid().String()
+	}
+	return cids, nil
+}

pkg/ipfs/dag_putters/eth_state.go

@@ -32,16 +32,12 @@ func NewEthStateDagPutter(adder *ipfs.IPFS) *EthStateDagPutter {
 }
 
 func (erdp *EthStateDagPutter) DagPut(raw interface{}) ([]string, error) {
-	stateNodeRLP, ok := raw.([]byte)
+	stateNode, ok := raw.(*ipld.EthStateTrie)
 	if !ok {
-		return nil, fmt.Errorf("EthStateDagPutter expected input type %T got %T", []byte{}, raw)
+		return nil, fmt.Errorf("EthStateDagPutter expected input type %T got %T", &ipld.EthStateTrie{}, raw)
 	}
-	node, err := ipld.FromStateTrieRLP(stateNodeRLP)
+	if err := erdp.adder.Add(stateNode); err != nil {
-	if err != nil {
 		return nil, err
 	}
-	if err := erdp.adder.Add(node); err != nil {
+	return []string{stateNode.Cid().String()}, nil
-		return nil, err
-	}
-	return []string{node.Cid().String()}, nil
 }

pkg/ipfs/dag_putters/eth_storage.go

@@ -32,16 +32,12 @@ func NewEthStorageDagPutter(adder *ipfs.IPFS) *EthStorageDagPutter {
 }
 
 func (erdp *EthStorageDagPutter) DagPut(raw interface{}) ([]string, error) {
-	storageNodeRLP, ok := raw.([]byte)
+	storageNode, ok := raw.(*ipld.EthStorageTrie)
 	if !ok {
-		return nil, fmt.Errorf("EthStorageDagPutter expected input type %T got %T", []byte{}, raw)
+		return nil, fmt.Errorf("EthStorageDagPutter expected input type %T got %T", &ipld.EthStorageTrie{}, raw)
 	}
-	node, err := ipld.FromStorageTrieRLP(storageNodeRLP)
+	if err := erdp.adder.Add(storageNode); err != nil {
-	if err != nil {
 		return nil, err
 	}
-	if err := erdp.adder.Add(node); err != nil {
+	return []string{storageNode.Cid().String()}, nil
-		return nil, err
-	}
-	return []string{node.Cid().String()}, nil
 }

pkg/ipfs/dag_putters/eth_tx.go

@@ -19,8 +19,6 @@ package dag_putters
 import (
 	"fmt"
 
-	"github.com/ethereum/go-ethereum/core/types"
-
 	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
 	"github.com/vulcanize/vulcanizedb/pkg/ipfs/ipld"
 )
@@ -34,20 +32,16 @@ func NewEthTxsDagPutter(adder *ipfs.IPFS) *EthTxsDagPutter {
 }
 
 func (etdp *EthTxsDagPutter) DagPut(raw interface{}) ([]string, error) {
-	transactions, ok := raw.(types.Transactions)
+	transactions, ok := raw.([]*ipld.EthTx)
 	if !ok {
-		return nil, fmt.Errorf("EthTxsDagPutter expected input type %T got %T", types.Transactions{}, raw)
+		return nil, fmt.Errorf("EthTxsDagPutter expected input type %T got %T", []*ipld.EthTx{}, raw)
 	}
 	cids := make([]string, len(transactions))
 	for i, transaction := range transactions {
-		node, err := ipld.NewEthTx(transaction)
+		if err := etdp.adder.Add(transaction); err != nil {
-		if err != nil {
 			return nil, err
 		}
-		if err := etdp.adder.Add(node); err != nil {
+		cids[i] = transaction.Cid().String()
-			return nil, err
-		}
-		cids[i] = node.Cid().String()
 	}
 	return cids, nil
 }

pkg/ipfs/dag_putters/eth_tx_trie.go

@@ -0,0 +1,47 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package dag_putters
+
+import (
+	"fmt"
+
+	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
+	"github.com/vulcanize/vulcanizedb/pkg/ipfs/ipld"
+)
+
+type EthTxTrieDagPutter struct {
+	adder *ipfs.IPFS
+}
+
+func NewEthTxTrieDagPutter(adder *ipfs.IPFS) *EthTxTrieDagPutter {
+	return &EthTxTrieDagPutter{adder: adder}
+}
+
+func (etdp *EthTxTrieDagPutter) DagPut(raw interface{}) ([]string, error) {
+	txTrieNodes, ok := raw.([]*ipld.EthTxTrie)
+	if !ok {
+		return nil, fmt.Errorf("EthTxTrieDagPutter expected input type %T got %T", []*ipld.EthTxTrie{}, raw)
+	}
+	cids := make([]string, len(txTrieNodes))
+	for i, txNode := range txTrieNodes {
+		if err := etdp.adder.Add(txNode); err != nil {
+			return nil, err
+		}
+		cids[i] = txNode.Cid().String()
+	}
+	return cids, nil
+}

pkg/ipfs/ipld/btc_block.go

@@ -0,0 +1,74 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package ipld
+
+import (
+	"github.com/btcsuite/btcd/wire"
+	"github.com/btcsuite/btcutil"
+	node "github.com/ipfs/go-ipld-format"
+)
+
+// FromHeaderAndTxs takes a block header and txs and processes it
+// to return it a set of IPLD nodes for further processing.
+func FromHeaderAndTxs(header *wire.BlockHeader, txs []*btcutil.Tx) (*BtcHeader, []*BtcTx, []*BtcTxTrie, error) {
+	var txNodes []*BtcTx
+	for _, tx := range txs {
+		txNode, err := NewBtcTx(tx.MsgTx())
+		if err != nil {
+			return nil, nil, nil, err
+		}
+		txNodes = append(txNodes, txNode)
+	}
+	txTrie, err := mkMerkleTree(txNodes)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+	headerNode, err := NewBtcHeader(header)
+	return headerNode, txNodes, txTrie, err
+}
+
+func mkMerkleTree(txs []*BtcTx) ([]*BtcTxTrie, error) {
+	layer := make([]node.Node, len(txs))
+	for i, tx := range txs {
+		layer[i] = tx
+	}
+	var out []*BtcTxTrie
+	var next []node.Node
+	for len(layer) > 1 {
+		if len(layer)%2 != 0 {
+			layer = append(layer, layer[len(layer)-1])
+		}
+		for i := 0; i < len(layer)/2; i++ {
+			var left, right node.Node
+			left = layer[i*2]
+			right = layer[(i*2)+1]
+
+			t := &BtcTxTrie{
+				Left:  &node.Link{Cid: left.Cid()},
+				Right: &node.Link{Cid: right.Cid()},
+			}
+
+			out = append(out, t)
+			next = append(next, t)
+		}
+
+		layer = next
+		next = nil
+	}
+
+	return out, nil
+}

pkg/ipfs/ipld/btc_tx.go

@@ -1,3 +1,19 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 package ipld
 
 import (
@@ -7,7 +23,7 @@ import (
 	"strconv"
 
 	"github.com/btcsuite/btcd/wire"
-	cid "github.com/ipfs/go-cid"
+	"github.com/ipfs/go-cid"
 	node "github.com/ipfs/go-ipld-format"
 	mh "github.com/multiformats/go-multihash"
 )

pkg/ipfs/ipld/btc_tx_trie.go

@@ -0,0 +1,110 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package ipld
+
+import (
+	"fmt"
+
+	"github.com/ipfs/go-cid"
+	node "github.com/ipfs/go-ipld-format"
+	mh "github.com/multiformats/go-multihash"
+)
+
+type BtcTxTrie struct {
+	Left  *node.Link
+	Right *node.Link
+}
+
+func (t *BtcTxTrie) BTCSha() []byte {
+	return cidToHash(t.Cid())
+}
+
+func (t *BtcTxTrie) Cid() cid.Cid {
+	h, _ := mh.Sum(t.RawData(), mh.DBL_SHA2_256, -1)
+	return cid.NewCidV1(cid.BitcoinTx, h)
+}
+
+func (t *BtcTxTrie) Links() []*node.Link {
+	return []*node.Link{t.Left, t.Right}
+}
+
+func (t *BtcTxTrie) RawData() []byte {
+	out := make([]byte, 64)
+	lbytes := cidToHash(t.Left.Cid)
+	copy(out[:32], lbytes)
+
+	rbytes := cidToHash(t.Right.Cid)
+	copy(out[32:], rbytes)
+
+	return out
+}
+
+func (t *BtcTxTrie) Loggable() map[string]interface{} {
+	return map[string]interface{}{
+		"type": "bitcoin_tx_tree",
+	}
+}
+
+func (t *BtcTxTrie) Resolve(path []string) (interface{}, []string, error) {
+	if len(path) == 0 {
+		return nil, nil, fmt.Errorf("zero length path")
+	}
+
+	switch path[0] {
+	case "0":
+		return t.Left, path[1:], nil
+	case "1":
+		return t.Right, path[1:], nil
+	default:
+		return nil, nil, fmt.Errorf("no such link")
+	}
+}
+
+func (t *BtcTxTrie) Copy() node.Node {
+	nt := *t
+	return &nt
+}
+
+func (t *BtcTxTrie) ResolveLink(path []string) (*node.Link, []string, error) {
+	out, rest, err := t.Resolve(path)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	lnk, ok := out.(*node.Link)
+	if ok {
+		return lnk, rest, nil
+	}
+
+	return nil, nil, fmt.Errorf("path did not lead to link")
+}
+
+func (t *BtcTxTrie) Size() (uint64, error) {
+	return uint64(len(t.RawData())), nil
+}
+
+func (t *BtcTxTrie) Stat() (*node.NodeStat, error) {
+	return &node.NodeStat{}, nil
+}
+
+func (t *BtcTxTrie) String() string {
+	return fmt.Sprintf("[bitcoin transaction tree]")
+}
+
+func (t *BtcTxTrie) Tree(p string, depth int) []string {
+	return []string{"0", "1"}
+}

pkg/ipfs/ipld/eth_block.go

@@ -0,0 +1,97 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package ipld
+
+import (
+	"bytes"
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/core/types"
+	mh "github.com/multiformats/go-multihash"
+)
+
+// FromBlock takes a block and processes it
+// to return it a set of IPLD nodes for further processing.
+func FromBlock(block *types.Block, receipts []*types.Receipt) (*EthHeader, []*EthTx, []*EthTxTrie, []*EthReceipt, []*EthRctTrie, error) {
+	// process the eth-header object
+	headerRawData := getRLP(block.Header())
+	cid, err := RawdataToCid(MEthHeader, headerRawData, mh.KECCAK_256)
+	if err != nil {
+		return nil, nil, nil, nil, nil, err
+	}
+	ethHeader := &EthHeader{
+		Header:  block.Header(),
+		cid:     cid,
+		rawdata: headerRawData,
+	}
+
+	// Process the found eth-tx objects
+	ethTxNodes, ethTxTrieNodes, err := processTransactions(block.Transactions(),
+		block.Header().TxHash[:])
+	if err != nil {
+		return nil, nil, nil, nil, nil, err
+	}
+	// process the eth-rct objects
+	ethRctNodes, ethRctTrieNodes, err := processReceipts(receipts,
+		block.Header().ReceiptHash[:])
+
+	return ethHeader, ethTxNodes, ethTxTrieNodes, ethRctNodes, ethRctTrieNodes, nil
+}
+
+// processTransactions will take the found transactions in a parsed block body
+// to return IPLD node slices for eth-tx and eth-tx-trie
+func processTransactions(txs []*types.Transaction, expectedTxRoot []byte) ([]*EthTx, []*EthTxTrie, error) {
+	var ethTxNodes []*EthTx
+	transactionTrie := newTxTrie()
+
+	for idx, tx := range txs {
+		ethTx, err := NewEthTx(tx)
+		if err != nil {
+			return nil, nil, err
+		}
+		ethTxNodes = append(ethTxNodes, ethTx)
+		transactionTrie.add(idx, ethTx.RawData())
+	}
+
+	if !bytes.Equal(transactionTrie.rootHash(), expectedTxRoot) {
+		return nil, nil, fmt.Errorf("wrong transaction hash computed")
+	}
+
+	return ethTxNodes, transactionTrie.getNodes(), nil
+}
+
+// processReceipts will take in receipts
+// to return IPLD node slices for eth-rct and eth-rct-trie
+func processReceipts(rcts []*types.Receipt, expectedRctRoot []byte) ([]*EthReceipt, []*EthRctTrie, error) {
+	var ethRctNodes []*EthReceipt
+	receiptTrie := newRctTrie()
+
+	for idx, rct := range rcts {
+		ethRct, err := NewReceipt(rct)
+		if err != nil {
+			return nil, nil, err
+		}
+		ethRctNodes = append(ethRctNodes, ethRct)
+		receiptTrie.add(idx, ethRct.RawData())
+	}
+
+	if !bytes.Equal(receiptTrie.rootHash(), expectedRctRoot) {
+		return nil, nil, fmt.Errorf("wrong receipt hash computed")
+	}
+
+	return ethRctNodes, receiptTrie.getNodes(), nil
+}

pkg/ipfs/ipld/eth_header.go

@@ -20,7 +20,6 @@ import (
 	"encoding/json"
 	"fmt"
 
-	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ipfs/go-cid"
@@ -60,6 +59,24 @@ func NewEthHeader(header *types.Header) (*EthHeader, error) {
 	}, nil
 }
 
+/*
+ OUTPUT
+*/
+
+// DecodeEthHeader takes a cid and its raw binary data
+// from IPFS and returns an EthTx object for further processing.
+func DecodeEthHeader(c cid.Cid, b []byte) (*EthHeader, error) {
+	var h *types.Header
+	if err := rlp.DecodeBytes(b, h); err != nil {
+		return nil, err
+	}
+	return &EthHeader{
+		Header:  h,
+		cid:     c,
+		rawdata: b,
+	}, nil
+}
+
 /*
   Block INTERFACE
 */
@@ -237,38 +254,3 @@ func (b *EthHeader) MarshalJSON() ([]byte, error) {
 	}
 	return json.Marshal(out)
 }
-
-// objJSONBlock defines the output of the JSON RPC API for either
-// "eth_BlockByHash" or "eth_BlockByHeader".
-type objJSONBlock struct {
-	Result objJSONBlockResult `json:"result"`
-}
-
-// objJSONBLockResult is the  nested struct that takes
-// the contents of the JSON field "result".
-type objJSONBlockResult struct {
-	types.Header           // Use its fields and unmarshaler
-	*objJSONBlockResultExt // Add these fields to the parsing
-}
-
-// objJSONBLockResultExt facilitates the composition
-// of the field "result", adding to the
-// `types.Header` fields, both ommers (their hashes) and transactions.
-type objJSONBlockResultExt struct {
-	OmmerHashes  []common.Hash        `json:"uncles"`
-	Transactions []*types.Transaction `json:"transactions"`
-}
-
-// UnmarshalJSON overrides the function types.Header.UnmarshalJSON, allowing us
-// to parse the fields of Header, plus ommer hashes and transactions.
-// (yes, ommer hashes. You will need to "eth_getUncleCountByBlockHash" per each ommer)
-func (o *objJSONBlockResult) UnmarshalJSON(input []byte) error {
-	err := o.Header.UnmarshalJSON(input)
-	if err != nil {
-		return err
-	}
-
-	o.objJSONBlockResultExt = &objJSONBlockResultExt{}
-	err = json.Unmarshal(input, o.objJSONBlockResultExt)
-	return err
-}

pkg/ipfs/ipld/eth_receipt.go

@@ -59,6 +59,24 @@ func NewReceipt(receipt *types.Receipt) (*EthReceipt, error) {
 	}, nil
 }
 
+/*
+ OUTPUT
+*/
+
+// DecodeEthReceipt takes a cid and its raw binary data
+// from IPFS and returns an EthTx object for further processing.
+func DecodeEthReceipt(c cid.Cid, b []byte) (*EthReceipt, error) {
+	var r *types.Receipt
+	if err := rlp.DecodeBytes(b, r); err != nil {
+		return nil, err
+	}
+	return &EthReceipt{
+		Receipt: r,
+		cid:     c,
+		rawdata: b,
+	}, nil
+}
+
 /*
   Block INTERFACE
 */

pkg/ipfs/ipld/eth_receipt_trie.go

@@ -0,0 +1,152 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package ipld
+
+import (
+	"fmt"
+
+	"github.com/ipfs/go-cid"
+	node "github.com/ipfs/go-ipld-format"
+	"github.com/multiformats/go-multihash"
+
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/rlp"
+)
+
+// EthRctTrie (eth-tx-trie codec 0x92) represents
+// a node from the transaction trie in ethereum.
+type EthRctTrie struct {
+	*TrieNode
+}
+
+// Static (compile time) check that EthRctTrie satisfies the node.Node interface.
+var _ node.Node = (*EthRctTrie)(nil)
+
+/*
+ INPUT
+*/
+
+// To create a proper trie of the eth-tx-trie objects, it is required
+// to input all transactions belonging to a forest in a single step.
+// We are adding the transactions, and creating its trie on
+// block body parsing time.
+
+/*
+  OUTPUT
+*/
+
+// DecodeEthRctTrie returns an EthRctTrie object from its cid and rawdata.
+func DecodeEthRctTrie(c cid.Cid, b []byte) (*EthRctTrie, error) {
+	tn, err := decodeTrieNode(c, b, decodeEthRctTrieLeaf)
+	if err != nil {
+		return nil, err
+	}
+	return &EthRctTrie{TrieNode: tn}, nil
+}
+
+// decodeEthRctTrieLeaf parses a eth-rct-trie leaf
+//from decoded RLP elements
+func decodeEthRctTrieLeaf(i []interface{}) ([]interface{}, error) {
+	var r types.Receipt
+	err := rlp.DecodeBytes(i[1].([]byte), &r)
+	if err != nil {
+		return nil, err
+	}
+	c, err := RawdataToCid(MEthTxReceipt, i[1].([]byte), multihash.KECCAK_256)
+	if err != nil {
+		return nil, err
+	}
+	return []interface{}{
+		i[0].([]byte),
+		&EthReceipt{
+			Receipt: &r,
+			cid:     c,
+			rawdata: i[1].([]byte),
+		},
+	}, nil
+}
+
+/*
+  Block INTERFACE
+*/
+
+// RawData returns the binary of the RLP encode of the transaction.
+func (t *EthRctTrie) RawData() []byte {
+	return t.rawdata
+}
+
+// Cid returns the cid of the transaction.
+func (t *EthRctTrie) Cid() cid.Cid {
+	return t.cid
+}
+
+// String is a helper for output
+func (t *EthRctTrie) String() string {
+	return fmt.Sprintf("<EthereumRctTrie %s>", t.cid)
+}
+
+// Loggable returns in a map the type of IPLD Link.
+func (t *EthRctTrie) Loggable() map[string]interface{} {
+	return map[string]interface{}{
+		"type": "eth-rct-trie",
+	}
+}
+
+/*
+  EthRctTrie functions
+*/
+
+// rctTrie wraps a localTrie for use on the receipt trie.
+type rctTrie struct {
+	*localTrie
+}
+
+// newRctTrie initializes and returns a rctTrie.
+func newRctTrie() *rctTrie {
+	return &rctTrie{
+		localTrie: newLocalTrie(),
+	}
+}
+
+// getNodes invokes the localTrie, which computes the root hash of the
+// transaction trie and returns its database keys, to return a slice
+// of EthRctTrie nodes.
+func (rt *rctTrie) getNodes() []*EthRctTrie {
+	keys := rt.getKeys()
+	var out []*EthRctTrie
+	it := rt.trie.NodeIterator([]byte{})
+	for it.Next(true) {
+
+	}
+	for _, k := range keys {
+		rawdata, err := rt.db.Get(k)
+		if err != nil {
+			panic(err)
+		}
+		c, err := RawdataToCid(MEthTxReceiptTrie, rawdata, multihash.KECCAK_256)
+		if err != nil {
+			return nil
+		}
+		tn := &TrieNode{
+			cid:     c,
+			rawdata: rawdata,
+		}
+		out = append(out, &EthRctTrie{TrieNode: tn})
+	}
+
+	return out
+}

pkg/ipfs/ipld/eth_state.go

@@ -21,14 +21,15 @@ import (
 
 	"github.com/ipfs/go-cid"
 	node "github.com/ipfs/go-ipld-format"
-	mh "github.com/multiformats/go-multihash"
+	"github.com/multiformats/go-multihash"
+
+	"github.com/ethereum/go-ethereum/rlp"
 )
 
 // EthStateTrie (eth-state-trie, codec 0x96), represents
-// a node from the state trie in ethereum.
+// a node from the satte trie in ethereum.
 type EthStateTrie struct {
-	cid     cid.Cid
+	*TrieNode
-	rawdata []byte
 }
 
 // Static (compile time) check that EthStateTrie satisfies the node.Node interface.
@@ -38,16 +39,51 @@ var _ node.Node = (*EthStateTrie)(nil)
   INPUT
 */
 
-// FromStateTrieRLP takes the RLP bytes of an ethereum
+// FromStateTrieRLP takes the RLP representation of an ethereum
 // state trie node to return it as an IPLD node for further processing.
-func FromStateTrieRLP(stateNodeRLP []byte) (*EthStateTrie, error) {
+func FromStateTrieRLP(raw []byte) (*EthStateTrie, error) {
-	c, err := RawdataToCid(MEthStateTrie, stateNodeRLP, mh.KECCAK_256)
+	c, err := RawdataToCid(MEthStateTrie, raw, multihash.KECCAK_256)
 	if err != nil {
 		return nil, err
 	}
-	return &EthStateTrie{
+	// Let's run the whole mile and process the nodeKind and
+	// its elements, in case somebody would need this function
+	// to parse an RLP element from the filesystem
+	return DecodeEthStateTrie(c, raw)
+}
+
+/*
+  OUTPUT
+*/
+
+// DecodeEthStateTrie returns an EthStateTrie object from its cid and rawdata.
+func DecodeEthStateTrie(c cid.Cid, b []byte) (*EthStateTrie, error) {
+	tn, err := decodeTrieNode(c, b, decodeEthStateTrieLeaf)
+	if err != nil {
+		return nil, err
+	}
+	return &EthStateTrie{TrieNode: tn}, nil
+}
+
+// decodeEthStateTrieLeaf parses a eth-tx-trie leaf
+// from decoded RLP elements
+func decodeEthStateTrieLeaf(i []interface{}) ([]interface{}, error) {
+	var account EthAccount
+	err := rlp.DecodeBytes(i[1].([]byte), &account)
+	if err != nil {
+		return nil, err
+	}
+	c, err := RawdataToCid(MEthAccountSnapshot, i[1].([]byte), multihash.KECCAK_256)
+	if err != nil {
+		return nil, err
+	}
+	return []interface{}{
+		i[0].([]byte),
+		&EthAccountSnapshot{
+			EthAccount: &account,
 			cid:        c,
-		rawdata: stateNodeRLP,
+			rawdata:    i[1].([]byte),
+		},
 	}, nil
 }
 
@@ -70,35 +106,6 @@ func (st *EthStateTrie) String() string {
 	return fmt.Sprintf("<EthereumStateTrie %s>", st.cid)
 }
 
-// Copy will go away. It is here to comply with the Node interface.
-func (*EthStateTrie) Copy() node.Node {
-	panic("implement me")
-}
-
-func (*EthStateTrie) Links() []*node.Link {
-	panic("implement me")
-}
-
-func (*EthStateTrie) Resolve(path []string) (interface{}, []string, error) {
-	panic("implement me")
-}
-
-func (*EthStateTrie) ResolveLink(path []string) (*node.Link, []string, error) {
-	panic("implement me")
-}
-
-func (*EthStateTrie) Size() (uint64, error) {
-	panic("implement me")
-}
-
-func (*EthStateTrie) Stat() (*node.NodeStat, error) {
-	panic("implement me")
-}
-
-func (*EthStateTrie) Tree(path string, depth int) []string {
-	panic("implement me")
-}
-
 // Loggable returns in a map the type of IPLD Link.
 func (st *EthStateTrie) Loggable() map[string]interface{} {
 	return map[string]interface{}{

pkg/ipfs/ipld/eth_storage.go

@@ -21,14 +21,13 @@ import (
 
 	"github.com/ipfs/go-cid"
 	node "github.com/ipfs/go-ipld-format"
-	mh "github.com/multiformats/go-multihash"
+	"github.com/multiformats/go-multihash"
 )
 
 // EthStorageTrie (eth-storage-trie, codec 0x98), represents
 // a node from the storage trie in ethereum.
 type EthStorageTrie struct {
-	cid     cid.Cid
+	*TrieNode
-	rawdata []byte
 }
 
 // Static (compile time) check that EthStorageTrie satisfies the node.Node interface.
@@ -38,16 +37,39 @@ var _ node.Node = (*EthStorageTrie)(nil)
   INPUT
 */
 
-// FromStorageTrieRLP takes the RLP bytes of an ethereum
+// FromStorageTrieRLP takes the RLP representation of an ethereum
 // storage trie node to return it as an IPLD node for further processing.
-func FromStorageTrieRLP(storageNodeRLP []byte) (*EthStorageTrie, error) {
+func FromStorageTrieRLP(raw []byte) (*EthStorageTrie, error) {
-	c, err := RawdataToCid(MEthStorageTrie, storageNodeRLP, mh.KECCAK_256)
+	c, err := RawdataToCid(MEthStorageTrie, raw, multihash.KECCAK_256)
 	if err != nil {
 		return nil, err
 	}
-	return &EthStorageTrie{
+
-		cid:     c,
+	// Let's run the whole mile and process the nodeKind and
-		rawdata: storageNodeRLP,
+	// its elements, in case somebody would need this function
+	// to parse an RLP element from the filesystem
+	return DecodeEthStorageTrie(c, raw)
+}
+
+/*
+  OUTPUT
+*/
+
+// DecodeEthStorageTrie returns an EthStorageTrie object from its cid and rawdata.
+func DecodeEthStorageTrie(c cid.Cid, b []byte) (*EthStorageTrie, error) {
+	tn, err := decodeTrieNode(c, b, decodeEthStorageTrieLeaf)
+	if err != nil {
+		return nil, err
+	}
+	return &EthStorageTrie{TrieNode: tn}, nil
+}
+
+// decodeEthStorageTrieLeaf parses a eth-tx-trie leaf
+// from decoded RLP elements
+func decodeEthStorageTrieLeaf(i []interface{}) ([]interface{}, error) {
+	return []interface{}{
+		i[0].([]byte),
+		i[1].([]byte),
 	}, nil
 }
 
@@ -70,35 +92,6 @@ func (st *EthStorageTrie) String() string {
 	return fmt.Sprintf("<EthereumStorageTrie %s>", st.cid)
 }
 
-// Copy will go away. It is here to comply with the Node interface.
-func (*EthStorageTrie) Copy() node.Node {
-	panic("implement me")
-}
-
-func (*EthStorageTrie) Links() []*node.Link {
-	panic("implement me")
-}
-
-func (*EthStorageTrie) Resolve(path []string) (interface{}, []string, error) {
-	panic("implement me")
-}
-
-func (*EthStorageTrie) ResolveLink(path []string) (*node.Link, []string, error) {
-	panic("implement me")
-}
-
-func (*EthStorageTrie) Size() (uint64, error) {
-	panic("implement me")
-}
-
-func (*EthStorageTrie) Stat() (*node.NodeStat, error) {
-	panic("implement me")
-}
-
-func (*EthStorageTrie) Tree(path string, depth int) []string {
-	panic("implement me")
-}
-
 // Loggable returns in a map the type of IPLD Link.
 func (st *EthStorageTrie) Loggable() map[string]interface{} {
 	return map[string]interface{}{

pkg/ipfs/ipld/eth_tx_trie.go

@@ -0,0 +1,152 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package ipld
+
+import (
+	"fmt"
+
+	"github.com/ipfs/go-cid"
+	node "github.com/ipfs/go-ipld-format"
+	"github.com/multiformats/go-multihash"
+
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/rlp"
+)
+
+// EthTxTrie (eth-tx-trie codec 0x92) represents
+// a node from the transaction trie in ethereum.
+type EthTxTrie struct {
+	*TrieNode
+}
+
+// Static (compile time) check that EthTxTrie satisfies the node.Node interface.
+var _ node.Node = (*EthTxTrie)(nil)
+
+/*
+ INPUT
+*/
+
+// To create a proper trie of the eth-tx-trie objects, it is required
+// to input all transactions belonging to a forest in a single step.
+// We are adding the transactions, and creating its trie on
+// block body parsing time.
+
+/*
+  OUTPUT
+*/
+
+// DecodeEthTxTrie returns an EthTxTrie object from its cid and rawdata.
+func DecodeEthTxTrie(c cid.Cid, b []byte) (*EthTxTrie, error) {
+	tn, err := decodeTrieNode(c, b, decodeEthTxTrieLeaf)
+	if err != nil {
+		return nil, err
+	}
+	return &EthTxTrie{TrieNode: tn}, nil
+}
+
+// decodeEthTxTrieLeaf parses a eth-tx-trie leaf
+//from decoded RLP elements
+func decodeEthTxTrieLeaf(i []interface{}) ([]interface{}, error) {
+	var t types.Transaction
+	err := rlp.DecodeBytes(i[1].([]byte), &t)
+	if err != nil {
+		return nil, err
+	}
+	c, err := RawdataToCid(MEthTx, i[1].([]byte), multihash.KECCAK_256)
+	if err != nil {
+		return nil, err
+	}
+	return []interface{}{
+		i[0].([]byte),
+		&EthTx{
+			Transaction: &t,
+			cid:         c,
+			rawdata:     i[1].([]byte),
+		},
+	}, nil
+}
+
+/*
+  Block INTERFACE
+*/
+
+// RawData returns the binary of the RLP encode of the transaction.
+func (t *EthTxTrie) RawData() []byte {
+	return t.rawdata
+}
+
+// Cid returns the cid of the transaction.
+func (t *EthTxTrie) Cid() cid.Cid {
+	return t.cid
+}
+
+// String is a helper for output
+func (t *EthTxTrie) String() string {
+	return fmt.Sprintf("<EthereumTxTrie %s>", t.cid)
+}
+
+// Loggable returns in a map the type of IPLD Link.
+func (t *EthTxTrie) Loggable() map[string]interface{} {
+	return map[string]interface{}{
+		"type": "eth-tx-trie",
+	}
+}
+
+/*
+  EthTxTrie functions
+*/
+
+// txTrie wraps a localTrie for use on the transaction trie.
+type txTrie struct {
+	*localTrie
+}
+
+// newTxTrie initializes and returns a txTrie.
+func newTxTrie() *txTrie {
+	return &txTrie{
+		localTrie: newLocalTrie(),
+	}
+}
+
+// getNodes invokes the localTrie, which computes the root hash of the
+// transaction trie and returns its database keys, to return a slice
+// of EthTxTrie nodes.
+func (tt *txTrie) getNodes() []*EthTxTrie {
+	keys := tt.getKeys()
+	var out []*EthTxTrie
+	it := tt.trie.NodeIterator([]byte{})
+	for it.Next(true) {
+
+	}
+	for _, k := range keys {
+		rawdata, err := tt.db.Get(k)
+		if err != nil {
+			panic(err)
+		}
+		c, err := RawdataToCid(MEthTxTrie, rawdata, multihash.KECCAK_256)
+		if err != nil {
+			return nil
+		}
+		tn := &TrieNode{
+			cid:     c,
+			rawdata: rawdata,
+		}
+		out = append(out, &EthTxTrie{TrieNode: tn})
+	}
+
+	return out
+}

pkg/ipfs/ipld/shared.go

@@ -17,7 +17,13 @@
 package ipld
 
 import (
+	"bytes"
+
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/rawdb"
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/trie"
 	"github.com/ipfs/go-cid"
 	mh "github.com/multiformats/go-multihash"
 )
@@ -87,3 +93,56 @@ func sha256ToCid(codec uint64, h []byte) cid.Cid {
 
 	return cid.NewCidV1(codec, hash)
 }
+
+// getRLP encodes the given object to RLP returning its bytes.
+func getRLP(object interface{}) []byte {
+	buf := new(bytes.Buffer)
+	if err := rlp.Encode(buf, object); err != nil {
+		panic(err)
+	}
+
+	return buf.Bytes()
+}
+
+// localTrie wraps a go-ethereum trie and its underlying memory db.
+// It contributes to the creation of the trie node objects.
+type localTrie struct {
+	keys [][]byte
+	db   ethdb.Database
+	trie *trie.Trie
+}
+
+// newLocalTrie initializes and returns a localTrie object
+func newLocalTrie() *localTrie {
+	var err error
+	lt := &localTrie{}
+	lt.db = rawdb.NewMemoryDatabase()
+	lt.trie, err = trie.New(common.Hash{}, trie.NewDatabase(lt.db))
+	if err != nil {
+		panic(err)
+	}
+	return lt
+}
+
+// add receives the index of an object and its rawdata value
+// and includes it into the localTrie
+func (lt *localTrie) add(idx int, rawdata []byte) {
+	key, err := rlp.EncodeToBytes(uint(idx))
+	if err != nil {
+		panic(err)
+	}
+	lt.keys = append(lt.keys, key)
+	lt.trie.Update(key, rawdata)
+}
+
+// rootHash returns the computed trie root.
+// Useful for sanity checks on parsed data.
+func (lt *localTrie) rootHash() []byte {
+	return lt.trie.Hash().Bytes()
+}
+
+// getKeys returns the stored keys of the memory database
+// of the localTrie for further processing.
+func (lt *localTrie) getKeys() [][]byte {
+	return lt.keys
+}

pkg/ipfs/ipld/trie_node.go

@@ -0,0 +1,440 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package ipld
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ipfs/go-cid"
+	node "github.com/ipfs/go-ipld-format"
+)
+
+// TrieNode is the general abstraction for
+//ethereum IPLD trie nodes.
+type TrieNode struct {
+	// leaf, extension or branch
+	nodeKind string
+
+	// If leaf or extension: [0] is key, [1] is val.
+	// If branch: [0] - [16] are children.
+	elements []interface{}
+
+	// IPLD block information
+	cid     cid.Cid
+	rawdata []byte
+}
+
+/*
+  OUTPUT
+*/
+
+type trieNodeLeafDecoder func([]interface{}) ([]interface{}, error)
+
+// decodeTrieNode returns a TrieNode object from an IPLD block's
+// cid and rawdata.
+func decodeTrieNode(c cid.Cid, b []byte,
+	leafDecoder trieNodeLeafDecoder) (*TrieNode, error) {
+	var (
+		i, decoded, elements []interface{}
+		nodeKind             string
+		err                  error
+	)
+
+	err = rlp.DecodeBytes(b, &i)
+	if err != nil {
+		return nil, err
+	}
+
+	codec := c.Type()
+	switch len(i) {
+	case 2:
+		nodeKind, decoded, err = decodeCompactKey(i)
+		if err != nil {
+			return nil, err
+		}
+
+		if nodeKind == "extension" {
+			elements, err = parseTrieNodeExtension(decoded, codec)
+		}
+		if nodeKind == "leaf" {
+			elements, err = leafDecoder(decoded)
+		}
+		if nodeKind != "extension" && nodeKind != "leaf" {
+			return nil, fmt.Errorf("unexpected nodeKind returned from decoder")
+		}
+	case 17:
+		nodeKind = "branch"
+		elements, err = parseTrieNodeBranch(i, codec)
+		if err != nil {
+			return nil, err
+		}
+	default:
+		return nil, fmt.Errorf("unknown trie node type")
+	}
+
+	return &TrieNode{
+		nodeKind: nodeKind,
+		elements: elements,
+		rawdata:  b,
+		cid:      c,
+	}, nil
+}
+
+// decodeCompactKey takes a compact key, and returns its nodeKind and value.
+func decodeCompactKey(i []interface{}) (string, []interface{}, error) {
+	first := i[0].([]byte)
+	last := i[1].([]byte)
+
+	switch first[0] / 16 {
+	case '\x00':
+		return "extension", []interface{}{
+			nibbleToByte(first)[2:],
+			last,
+		}, nil
+	case '\x01':
+		return "extension", []interface{}{
+			nibbleToByte(first)[1:],
+			last,
+		}, nil
+	case '\x02':
+		return "leaf", []interface{}{
+			nibbleToByte(first)[2:],
+			last,
+		}, nil
+	case '\x03':
+		return "leaf", []interface{}{
+			nibbleToByte(first)[1:],
+			last,
+		}, nil
+	default:
+		return "", nil, fmt.Errorf("unknown hex prefix")
+	}
+}
+
+// parseTrieNodeExtension helper improves readability
+func parseTrieNodeExtension(i []interface{}, codec uint64) ([]interface{}, error) {
+	return []interface{}{
+		i[0].([]byte),
+		keccak256ToCid(codec, i[1].([]byte)),
+	}, nil
+}
+
+// parseTrieNodeBranch helper improves readability
+func parseTrieNodeBranch(i []interface{}, codec uint64) ([]interface{}, error) {
+	var out []interface{}
+
+	for _, vi := range i {
+		v := vi.([]byte)
+
+		switch len(v) {
+		case 0:
+			out = append(out, nil)
+		case 32:
+			out = append(out, keccak256ToCid(codec, v))
+		default:
+			return nil, fmt.Errorf("unrecognized object: %v", v)
+		}
+	}
+
+	return out, nil
+}
+
+/*
+  Node INTERFACE
+*/
+
+// Resolve resolves a path through this node, stopping at any link boundary
+// and returning the object found as well as the remaining path to traverse
+func (t *TrieNode) Resolve(p []string) (interface{}, []string, error) {
+	switch t.nodeKind {
+	case "extension":
+		return t.resolveTrieNodeExtension(p)
+	case "leaf":
+		return t.resolveTrieNodeLeaf(p)
+	case "branch":
+		return t.resolveTrieNodeBranch(p)
+	default:
+		return nil, nil, fmt.Errorf("nodeKind case not implemented")
+	}
+}
+
+// Tree lists all paths within the object under 'path', and up to the given depth.
+// To list the entire object (similar to `find .`) pass "" and -1
+func (t *TrieNode) Tree(p string, depth int) []string {
+	if p != "" || depth == 0 {
+		return nil
+	}
+
+	var out []string
+
+	switch t.nodeKind {
+	case "extension":
+		var val string
+		for _, e := range t.elements[0].([]byte) {
+			val += fmt.Sprintf("%x", e)
+		}
+		return []string{val}
+	case "branch":
+		for i, elem := range t.elements {
+			if _, ok := elem.(*cid.Cid); ok {
+				out = append(out, fmt.Sprintf("%x", i))
+			}
+		}
+		return out
+
+	default:
+		return nil
+	}
+}
+
+// ResolveLink is a helper function that calls resolve and asserts the
+// output is a link
+func (t *TrieNode) ResolveLink(p []string) (*node.Link, []string, error) {
+	obj, rest, err := t.Resolve(p)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	lnk, ok := obj.(*node.Link)
+	if !ok {
+		return nil, nil, fmt.Errorf("was not a link")
+	}
+
+	return lnk, rest, nil
+}
+
+// Copy will go away. It is here to comply with the interface.
+func (t *TrieNode) Copy() node.Node {
+	panic("dont use this yet")
+}
+
+// Links is a helper function that returns all links within this object
+func (t *TrieNode) Links() []*node.Link {
+	var out []*node.Link
+
+	for _, i := range t.elements {
+		c, ok := i.(cid.Cid)
+		if ok {
+			out = append(out, &node.Link{Cid: c})
+		}
+	}
+
+	return out
+}
+
+// Stat will go away. It is here to comply with the interface.
+func (t *TrieNode) Stat() (*node.NodeStat, error) {
+	return &node.NodeStat{}, nil
+}
+
+// Size will go away. It is here to comply with the interface.
+func (t *TrieNode) Size() (uint64, error) {
+	return 0, nil
+}
+
+/*
+  TrieNode functions
+*/
+
+// MarshalJSON processes the transaction trie into readable JSON format.
+func (t *TrieNode) MarshalJSON() ([]byte, error) {
+	var out map[string]interface{}
+
+	switch t.nodeKind {
+	case "extension":
+		fallthrough
+	case "leaf":
+		var hexPrefix string
+		for _, e := range t.elements[0].([]byte) {
+			hexPrefix += fmt.Sprintf("%x", e)
+		}
+
+		// if we got a byte we need to do this casting otherwise
+		// it will be marshaled to a base64 encoded value
+		if _, ok := t.elements[1].([]byte); ok {
+			var hexVal string
+			for _, e := range t.elements[1].([]byte) {
+				hexVal += fmt.Sprintf("%x", e)
+			}
+
+			t.elements[1] = hexVal
+		}
+
+		out = map[string]interface{}{
+			"type":    t.nodeKind,
+			hexPrefix: t.elements[1],
+		}
+
+	case "branch":
+		out = map[string]interface{}{
+			"type": "branch",
+			"0":    t.elements[0],
+			"1":    t.elements[1],
+			"2":    t.elements[2],
+			"3":    t.elements[3],
+			"4":    t.elements[4],
+			"5":    t.elements[5],
+			"6":    t.elements[6],
+			"7":    t.elements[7],
+			"8":    t.elements[8],
+			"9":    t.elements[9],
+			"a":    t.elements[10],
+			"b":    t.elements[11],
+			"c":    t.elements[12],
+			"d":    t.elements[13],
+			"e":    t.elements[14],
+			"f":    t.elements[15],
+		}
+	default:
+		return nil, fmt.Errorf("nodeKind %s not supported", t.nodeKind)
+	}
+
+	return json.Marshal(out)
+}
+
+// nibbleToByte expands the nibbles of a byte slice into their own bytes.
+func nibbleToByte(k []byte) []byte {
+	var out []byte
+
+	for _, b := range k {
+		out = append(out, b/16)
+		out = append(out, b%16)
+	}
+
+	return out
+}
+
+// Resolve reading conveniences
+func (t *TrieNode) resolveTrieNodeExtension(p []string) (interface{}, []string, error) {
+	nibbles := t.elements[0].([]byte)
+	idx, rest := shiftFromPath(p, len(nibbles))
+	if len(idx) < len(nibbles) {
+		return nil, nil, fmt.Errorf("not enough nibbles to traverse this extension")
+	}
+
+	for _, i := range idx {
+		if getHexIndex(string(i)) == -1 {
+			return nil, nil, fmt.Errorf("invalid path element")
+		}
+	}
+
+	for i, n := range nibbles {
+		if string(idx[i]) != fmt.Sprintf("%x", n) {
+			return nil, nil, fmt.Errorf("no such link in this extension")
+		}
+	}
+
+	return &node.Link{Cid: t.elements[1].(cid.Cid)}, rest, nil
+}
+
+func (t *TrieNode) resolveTrieNodeLeaf(p []string) (interface{}, []string, error) {
+	nibbles := t.elements[0].([]byte)
+
+	if len(nibbles) != 0 {
+		idx, rest := shiftFromPath(p, len(nibbles))
+		if len(idx) < len(nibbles) {
+			return nil, nil, fmt.Errorf("not enough nibbles to traverse this leaf")
+		}
+
+		for _, i := range idx {
+			if getHexIndex(string(i)) == -1 {
+				return nil, nil, fmt.Errorf("invalid path element")
+			}
+		}
+
+		for i, n := range nibbles {
+			if string(idx[i]) != fmt.Sprintf("%x", n) {
+				return nil, nil, fmt.Errorf("no such link in this extension")
+			}
+		}
+
+		p = rest
+	}
+
+	link, ok := t.elements[1].(node.Node)
+	if !ok {
+		return nil, nil, fmt.Errorf("leaf children is not an IPLD node")
+	}
+
+	return link.Resolve(p)
+}
+
+func (t *TrieNode) resolveTrieNodeBranch(p []string) (interface{}, []string, error) {
+	idx, rest := shiftFromPath(p, 1)
+	hidx := getHexIndex(idx)
+	if hidx == -1 {
+		return nil, nil, fmt.Errorf("incorrect path")
+	}
+
+	child := t.elements[hidx]
+	if child != nil {
+		return &node.Link{Cid: child.(cid.Cid)}, rest, nil
+	}
+	return nil, nil, fmt.Errorf("no such link in this branch")
+}
+
+// shiftFromPath extracts from a given path (as a slice of strings)
+// the given number of elements as a single string, returning whatever
+// it has not taken.
+//
+// Examples:
+// ["0", "a", "something"] and 1 -> "0" and ["a", "something"]
+// ["ab", "c", "d", "1"] and 2 -> "ab" and ["c", "d", "1"]
+// ["abc", "d", "1"] and 2 -> "ab" and ["c", "d", "1"]
+func shiftFromPath(p []string, i int) (string, []string) {
+	var (
+		out  string
+		rest []string
+	)
+
+	for _, pe := range p {
+		re := ""
+		for _, c := range pe {
+			if len(out) < i {
+				out += string(c)
+			} else {
+				re += string(c)
+			}
+		}
+
+		if len(out) == i && re != "" {
+			rest = append(rest, re)
+		}
+	}
+
+	return out, rest
+}
+
+// getHexIndex returns to you the integer 0 - 15 equivalent to your
+// string character if applicable, or -1 otherwise.
+func getHexIndex(s string) int {
+	if len(s) != 1 {
+		return -1
+	}
+
+	c := byte(s[0])
+	switch {
+	case '0' <= c && c <= '9':
+		return int(c - '0')
+	case 'a' <= c && c <= 'f':
+		return int(c - 'a' + 10)
+	}
+
+	return -1
+}