internalized ipld types and dag_putters; adjust converters/publishers to work with them

cmd/coldImport.go

@@ -20,12 +20,12 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
-	"github.com/vulcanize/vulcanizedb/pkg/eth/crypto"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/ethereum"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres/repositories"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/cold_import"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/converters/cold_db"
 	vulcCommon "github.com/vulcanize/vulcanizedb/pkg/eth/converters/common"
+	"github.com/vulcanize/vulcanizedb/pkg/eth/crypto"
+	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/ethereum"
+	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres/repositories"
 	"github.com/vulcanize/vulcanizedb/pkg/fs"
 	"github.com/vulcanize/vulcanizedb/utils"
 )

cmd/headerSync.go

@@ -22,10 +22,10 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
+	"github.com/vulcanize/vulcanizedb/pkg/eth"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres/repositories"
-	"github.com/vulcanize/vulcanizedb/pkg/eth"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/history"
 	"github.com/vulcanize/vulcanizedb/utils"
 )

cmd/root.go

@@ -29,10 +29,10 @@ import (
 	"github.com/spf13/viper"
 
 	"github.com/vulcanize/vulcanizedb/pkg/config"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
 	"github.com/vulcanize/vulcanizedb/pkg/eth"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/client"
 	vRpc "github.com/vulcanize/vulcanizedb/pkg/eth/converters/rpc"
+	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/node"
 )
 

cmd/streamEthSubscribe.go

@@ -29,8 +29,8 @@ import (
 	"github.com/spf13/viper"
 
 	"github.com/vulcanize/vulcanizedb/libraries/shared/streamer"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/client"
+	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
 	"github.com/vulcanize/vulcanizedb/pkg/super_node"
 	"github.com/vulcanize/vulcanizedb/pkg/super_node/config"
 	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"

integration_test/contract_test.go

@@ -25,10 +25,10 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
 	"github.com/vulcanize/vulcanizedb/pkg/eth"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/client"
 	rpc2 "github.com/vulcanize/vulcanizedb/pkg/eth/converters/rpc"
+	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/node"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/testing"
 	"github.com/vulcanize/vulcanizedb/test_config"

integration_test/geth_blockchain_test.go

@@ -23,13 +23,13 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
 	"github.com/vulcanize/vulcanizedb/pkg/eth"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/client"
 	rpc2 "github.com/vulcanize/vulcanizedb/pkg/eth/converters/rpc"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/node"
+	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/fakes"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/history"
+	"github.com/vulcanize/vulcanizedb/pkg/eth/node"
 	"github.com/vulcanize/vulcanizedb/test_config"
 )
 

integration_test/integration_test_suite_test.go

@@ -17,10 +17,11 @@
 package integration_test
 
 import (
-	"github.com/sirupsen/logrus"
 	"io/ioutil"
 	"testing"
 
+	"github.com/sirupsen/logrus"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )

pkg/eth/history/block_validator_test.go

@@ -22,8 +22,8 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"github.com/vulcanize/vulcanizedb/pkg/eth/history"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/fakes"
+	"github.com/vulcanize/vulcanizedb/pkg/eth/history"
 )
 
 var _ = Describe("Blocks validator", func() {

pkg/eth/history/header_validator_test.go

@@ -23,8 +23,8 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"github.com/vulcanize/vulcanizedb/pkg/eth/history"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/fakes"
+	"github.com/vulcanize/vulcanizedb/pkg/eth/history"
 )
 
 var _ = Describe("Header validator", func() {

pkg/eth/history/populate_blocks_test.go

@@ -21,8 +21,8 @@ import (
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/history"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/fakes"
+	"github.com/vulcanize/vulcanizedb/pkg/eth/history"
 )
 
 var _ = Describe("Populating blocks", func() {

pkg/eth/history/populate_headers_test.go

@@ -22,8 +22,8 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"github.com/vulcanize/vulcanizedb/pkg/eth/history"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/fakes"
+	"github.com/vulcanize/vulcanizedb/pkg/eth/history"
 )
 
 var _ = Describe("Populating headers", func() {

pkg/eth/history/validation_window_test.go

@@ -22,8 +22,8 @@ import (
 
 	"math/big"
 
-	"github.com/vulcanize/vulcanizedb/pkg/eth/history"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/fakes"
+	"github.com/vulcanize/vulcanizedb/pkg/eth/history"
 )
 
 var _ = Describe("Validation window", func() {

pkg/eth/node/node_test.go

@@ -23,8 +23,8 @@ import (
 	. "github.com/onsi/gomega"
 
 	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/node"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/fakes"
+	"github.com/vulcanize/vulcanizedb/pkg/eth/node"
 )
 
 var EmpytHeaderHash = "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347"

pkg/ipfs/builders.go

@@ -23,6 +23,7 @@ import (
 	"github.com/ipfs/go-ipfs/core"
 	"github.com/ipfs/go-ipfs/plugin/loader"
 	"github.com/ipfs/go-ipfs/repo/fsrepo"
+	ipld "github.com/ipfs/go-ipld-format"
 )
 
 // InitIPFSPlugins is used to initialized IPFS plugins before creating a new IPFS node
@@ -56,3 +57,29 @@ func InitIPFSBlockService(ipfsPath string) (blockservice.BlockService, error) {
 	}
 	return ipfsNode.Blocks, nil
 }
+
+type IPFS struct {
+	n   *core.IpfsNode
+	ctx context.Context
+}
+
+func (ipfs IPFS) Add(node ipld.Node) error {
+	return ipfs.n.DAG.Add(ipfs.n.Context(), node)
+}
+
+func InitIPFSNode(repoPath string) (*IPFS, error) {
+	r, err := fsrepo.Open(repoPath)
+	if err != nil {
+		return nil, err
+	}
+	ctx := context.Background()
+	cfg := &core.BuildCfg{
+		Online: false,
+		Repo:   r,
+	}
+	ipfsNode, err := core.NewNode(ctx, cfg)
+	if err != nil {
+		return nil, err
+	}
+	return &IPFS{n: ipfsNode, ctx: ctx}, nil
+}

pkg/ipfs/dag_putters/btc_header.go

@@ -0,0 +1,49 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package dag_putters
+
+import (
+	"fmt"
+
+	"github.com/btcsuite/btcd/wire"
+
+	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
+	"github.com/vulcanize/vulcanizedb/pkg/ipfs/ipld"
+)
+
+type BtcHeaderDagPutter struct {
+	adder *ipfs.IPFS
+}
+
+func NewBtcHeaderDagPutter(adder *ipfs.IPFS) *BtcHeaderDagPutter {
+	return &BtcHeaderDagPutter{adder: adder}
+}
+
+func (bhdp *BtcHeaderDagPutter) DagPut(raw interface{}) ([]string, error) {
+	header, ok := raw.(*wire.BlockHeader)
+	if !ok {
+		return nil, fmt.Errorf("BtcHeaderDagPutter expected input type %T got %T", &wire.BlockHeader{}, raw)
+	}
+	node, err := ipld.NewBtcHeader(header)
+	if err != nil {
+		return nil, err
+	}
+	if err := bhdp.adder.Add(node); err != nil {
+		return nil, err
+	}
+	return []string{node.Cid().String()}, nil
+}

pkg/ipfs/dag_putters/btc_tx.go

@@ -0,0 +1,53 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package dag_putters
+
+import (
+	"fmt"
+
+	"github.com/btcsuite/btcd/wire"
+
+	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
+	"github.com/vulcanize/vulcanizedb/pkg/ipfs/ipld"
+)
+
+type BtcTxDagPutter struct {
+	adder *ipfs.IPFS
+}
+
+func NewBtcTxDagPutter(adder *ipfs.IPFS) *BtcTxDagPutter {
+	return &BtcTxDagPutter{adder: adder}
+}
+
+func (etdp *BtcTxDagPutter) DagPut(raw interface{}) ([]string, error) {
+	transactions, ok := raw.([]*wire.MsgTx)
+	if !ok {
+		return nil, fmt.Errorf("BtcTxDagPutter expected input type %T got %T", []*wire.MsgTx{}, raw)
+	}
+	cids := make([]string, len(transactions))
+	for i, transaction := range transactions {
+		node, err := ipld.NewBtcTx(transaction)
+		if err != nil {
+			return nil, err
+		}
+		if err := etdp.adder.Add(node); err != nil {
+			return nil, err
+		}
+		cids[i] = node.Cid().String()
+	}
+	return cids, nil
+}

pkg/ipfs/dag_putters/eth_header.go

@@ -0,0 +1,49 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package dag_putters
+
+import (
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/core/types"
+
+	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
+	"github.com/vulcanize/vulcanizedb/pkg/ipfs/ipld"
+)
+
+type EthHeaderDagPutter struct {
+	adder *ipfs.IPFS
+}
+
+func NewEthBlockHeaderDagPutter(adder *ipfs.IPFS) *EthHeaderDagPutter {
+	return &EthHeaderDagPutter{adder: adder}
+}
+
+func (bhdp *EthHeaderDagPutter) DagPut(raw interface{}) ([]string, error) {
+	header, ok := raw.(*types.Header)
+	if !ok {
+		return nil, fmt.Errorf("EthHeaderDagPutter expected input type %T got %T", &types.Header{}, raw)
+	}
+	node, err := ipld.NewEthHeader(header)
+	if err != nil {
+		return nil, err
+	}
+	if err := bhdp.adder.Add(node); err != nil {
+		return nil, err
+	}
+	return []string{node.Cid().String()}, nil
+}

pkg/ipfs/dag_putters/eth_receipt.go

@@ -0,0 +1,53 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package dag_putters
+
+import (
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/core/types"
+
+	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
+	"github.com/vulcanize/vulcanizedb/pkg/ipfs/ipld"
+)
+
+type EthReceiptDagPutter struct {
+	adder *ipfs.IPFS
+}
+
+func NewEthReceiptDagPutter(adder *ipfs.IPFS) *EthReceiptDagPutter {
+	return &EthReceiptDagPutter{adder: adder}
+}
+
+func (erdp *EthReceiptDagPutter) DagPut(raw interface{}) ([]string, error) {
+	receipts, ok := raw.(types.Receipts)
+	if !ok {
+		return nil, fmt.Errorf("EthReceiptDagPutter expected input type %T got type %T", types.Receipts{}, raw)
+	}
+	cids := make([]string, len(receipts))
+	for i, receipt := range receipts {
+		node, err := ipld.NewReceipt((*types.ReceiptForStorage)(receipt))
+		if err != nil {
+			return nil, err
+		}
+		if err := erdp.adder.Add(node); err != nil {
+			return nil, err
+		}
+		cids[i] = node.Cid().String()
+	}
+	return cids, nil
+}

pkg/ipfs/dag_putters/eth_state.go

@@ -0,0 +1,47 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package dag_putters
+
+import (
+	"fmt"
+
+	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
+	"github.com/vulcanize/vulcanizedb/pkg/ipfs/ipld"
+)
+
+type EthStateDagPutter struct {
+	adder *ipfs.IPFS
+}
+
+func NewEthStateDagPutter(adder *ipfs.IPFS) *EthStateDagPutter {
+	return &EthStateDagPutter{adder: adder}
+}
+
+func (erdp *EthStateDagPutter) DagPut(raw interface{}) ([]string, error) {
+	stateNodeRLP, ok := raw.([]byte)
+	if !ok {
+		return nil, fmt.Errorf("EthStateDagPutter expected input type %T got %T", []byte{}, raw)
+	}
+	node, err := ipld.FromStateTrieRLP(stateNodeRLP)
+	if err != nil {
+		return nil, err
+	}
+	if err := erdp.adder.Add(node); err != nil {
+		return nil, err
+	}
+	return []string{node.Cid().String()}, nil
+}

pkg/ipfs/dag_putters/eth_storage.go

@@ -0,0 +1,47 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package dag_putters
+
+import (
+	"fmt"
+
+	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
+	"github.com/vulcanize/vulcanizedb/pkg/ipfs/ipld"
+)
+
+type EthStorageDagPutter struct {
+	adder *ipfs.IPFS
+}
+
+func NewEthStorageDagPutter(adder *ipfs.IPFS) *EthStorageDagPutter {
+	return &EthStorageDagPutter{adder: adder}
+}
+
+func (erdp *EthStorageDagPutter) DagPut(raw interface{}) ([]string, error) {
+	storageNodeRLP, ok := raw.([]byte)
+	if !ok {
+		return nil, fmt.Errorf("EthStorageDagPutter expected input type %T got %T", []byte{}, raw)
+	}
+	node, err := ipld.FromStorageTrieRLP(storageNodeRLP)
+	if err != nil {
+		return nil, err
+	}
+	if err := erdp.adder.Add(node); err != nil {
+		return nil, err
+	}
+	return []string{node.Cid().String()}, nil
+}

pkg/ipfs/dag_putters/eth_tx.go

@@ -0,0 +1,53 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package dag_putters
+
+import (
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/core/types"
+
+	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
+	"github.com/vulcanize/vulcanizedb/pkg/ipfs/ipld"
+)
+
+type EthTxsDagPutter struct {
+	adder *ipfs.IPFS
+}
+
+func NewEthTxsDagPutter(adder *ipfs.IPFS) *EthTxsDagPutter {
+	return &EthTxsDagPutter{adder: adder}
+}
+
+func (etdp *EthTxsDagPutter) DagPut(raw interface{}) ([]string, error) {
+	transactions, ok := raw.(types.Transactions)
+	if !ok {
+		return nil, fmt.Errorf("EthTxsDagPutter expected input type %T got %T", types.Transactions{}, raw)
+	}
+	cids := make([]string, len(transactions))
+	for i, transaction := range transactions {
+		node, err := ipld.NewEthTx(transaction)
+		if err != nil {
+			return nil, err
+		}
+		if err := etdp.adder.Add(node); err != nil {
+			return nil, err
+		}
+		cids[i] = node.Cid().String()
+	}
+	return cids, nil
+}

pkg/ipfs/ipld/btc_header.go

@@ -0,0 +1,202 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package ipld
+
+import (
+	"bytes"
+	"encoding/hex"
+	"fmt"
+
+	"github.com/btcsuite/btcd/wire"
+	"github.com/ipfs/go-cid"
+	node "github.com/ipfs/go-ipld-format"
+	mh "github.com/multiformats/go-multihash"
+)
+
+type BtcHeader struct {
+	*wire.BlockHeader
+
+	rawdata []byte
+	cid     cid.Cid
+}
+
+// Static (compile time) check that BtcBtcHeader satisfies the node.Node interface.
+var _ node.Node = (*BtcHeader)(nil)
+
+/*
+  INPUT
+*/
+
+// NewBtcHeader converts a *wire.Header into an BtcHeader IPLD node
+func NewBtcHeader(header *wire.BlockHeader) (*BtcHeader, error) {
+	w := bytes.NewBuffer(make([]byte, 0, 80))
+	if err := header.Serialize(w); err != nil {
+		return nil, err
+	}
+	rawdata := w.Bytes()
+	c, err := rawdataToCid(MBitcoinHeader, rawdata, mh.DBL_SHA2_256)
+	if err != nil {
+		return nil, err
+	}
+	return &BtcHeader{
+		BlockHeader: header,
+		cid:         c,
+		rawdata:     rawdata,
+	}, nil
+}
+
+/*
+  OUTPUT
+*/
+
+// DecodeBtcHeader takes a cid and its raw binary data
+// from IPFS and returns an BtcHeader object for further processing.
+func DecodeBtcHeader(c cid.Cid, b []byte) (*BtcHeader, error) {
+	var h *wire.BlockHeader
+	w := bytes.NewBuffer(b)
+	if err := h.Deserialize(w); err != nil {
+		return nil, err
+	}
+	return &BtcHeader{
+		BlockHeader: h,
+		cid:         c,
+		rawdata:     b,
+	}, nil
+}
+
+/*
+   Block INTERFACE
+*/
+
+func (b *BtcHeader) Cid() cid.Cid {
+	return b.cid
+}
+
+func (b *BtcHeader) RawData() []byte {
+	return b.rawdata
+}
+
+func (b *BtcHeader) String() string {
+	return fmt.Sprintf("<BtcHeader %s>", b.cid)
+}
+
+func (b *BtcHeader) Loggable() map[string]interface{} {
+	// TODO: more helpful info here
+	return map[string]interface{}{
+		"type": "bitcoin_block",
+	}
+}
+
+/*
+   Node INTERFACE
+*/
+
+func (b *BtcHeader) Links() []*node.Link {
+	return []*node.Link{
+		{
+			Name: "tx",
+			Cid:  sha256ToCid(MBitcoinTx, b.MerkleRoot.CloneBytes()),
+		},
+		{
+			Name: "parent",
+			Cid:  sha256ToCid(MBitcoinHeader, b.PrevBlock.CloneBytes()),
+		},
+	}
+}
+
+// Resolve attempts to traverse a path through this block.
+func (b *BtcHeader) Resolve(path []string) (interface{}, []string, error) {
+	if len(path) == 0 {
+		return nil, nil, fmt.Errorf("zero length path")
+	}
+	switch path[0] {
+	case "version":
+		return b.Version, path[1:], nil
+	case "timestamp":
+		return b.Timestamp, path[1:], nil
+	case "bits":
+		return b.Bits, path[1:], nil
+	case "nonce":
+		return b.Nonce, path[1:], nil
+	case "parent":
+		return &node.Link{Cid: sha256ToCid(MBitcoinHeader, b.PrevBlock.CloneBytes())}, path[1:], nil
+	case "tx":
+		return &node.Link{Cid: sha256ToCid(MBitcoinTx, b.MerkleRoot.CloneBytes())}, path[1:], nil
+	default:
+		return nil, nil, fmt.Errorf("no such link")
+	}
+}
+
+// ResolveLink is a helper function that allows easier traversal of links through blocks
+func (b *BtcHeader) ResolveLink(path []string) (*node.Link, []string, error) {
+	out, rest, err := b.Resolve(path)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	lnk, ok := out.(*node.Link)
+	if !ok {
+		return nil, nil, fmt.Errorf("object at path was not a link")
+	}
+
+	return lnk, rest, nil
+}
+
+func cidToHash(c cid.Cid) []byte {
+	h := []byte(c.Hash())
+	return h[len(h)-32:]
+}
+
+func hashToCid(hv []byte, t uint64) cid.Cid {
+	h, _ := mh.Encode(hv, mh.DBL_SHA2_256)
+	return cid.NewCidV1(t, h)
+}
+
+func (b *BtcHeader) Size() (uint64, error) {
+	return uint64(len(b.rawdata)), nil
+}
+
+func (b *BtcHeader) Stat() (*node.NodeStat, error) {
+	return &node.NodeStat{}, nil
+}
+
+func (b *BtcHeader) Tree(p string, depth int) []string {
+	// TODO: this isnt a correct implementation yet
+	return []string{"difficulty", "nonce", "version", "timestamp", "tx", "parent"}
+}
+
+func (b *BtcHeader) BTCSha() []byte {
+	blkmh, _ := mh.Sum(b.rawdata, mh.DBL_SHA2_256, -1)
+	return blkmh[2:]
+}
+
+func (b *BtcHeader) HexHash() string {
+	return hex.EncodeToString(revString(b.BTCSha()))
+}
+
+func (b *BtcHeader) Copy() node.Node {
+	nb := *b // cheating shallow copy
+	return &nb
+}
+
+func revString(s []byte) []byte {
+	b := make([]byte, len(s))
+	for i, v := range []byte(s) {
+		b[len(b)-(i+1)] = v
+	}
+	return b
+}

pkg/ipfs/ipld/btc_tx.go

@@ -0,0 +1,261 @@
+package ipld
+
+import (
+	"bytes"
+	"encoding/hex"
+	"fmt"
+	"strconv"
+
+	"github.com/btcsuite/btcd/wire"
+	cid "github.com/ipfs/go-cid"
+	node "github.com/ipfs/go-ipld-format"
+	mh "github.com/multiformats/go-multihash"
+)
+
+type BtcTx struct {
+	*wire.MsgTx
+
+	rawdata []byte
+	cid     cid.Cid
+}
+
+// Static (compile time) check that BtcBtcHeader satisfies the node.Node interface.
+var _ node.Node = (*BtcTx)(nil)
+
+/*
+  INPUT
+*/
+
+// NewBtcTx converts a *wire.MsgTx into an BtcTx IPLD node
+func NewBtcTx(tx *wire.MsgTx) (*BtcTx, error) {
+	w := bytes.NewBuffer(make([]byte, 0, tx.SerializeSize()))
+	if err := tx.Serialize(w); err != nil {
+		return nil, err
+	}
+	rawdata := w.Bytes()
+	c, err := rawdataToCid(MBitcoinTx, rawdata, mh.DBL_SHA2_256)
+	if err != nil {
+		return nil, err
+	}
+	return &BtcTx{
+		MsgTx:   tx,
+		cid:     c,
+		rawdata: rawdata,
+	}, nil
+}
+
+/*
+  OUTPUT
+*/
+
+// DecodeBtcTx takes a cid and its raw binary data
+// from IPFS and returns an BtcTx object for further processing.
+func DecodeBtcTx(c cid.Cid, b []byte) (*BtcTx, error) {
+	var tx *wire.MsgTx
+	w := bytes.NewBuffer(b)
+	if err := tx.Deserialize(w); err != nil {
+		return nil, err
+	}
+	return &BtcTx{
+		MsgTx:   tx,
+		cid:     c,
+		rawdata: b,
+	}, nil
+}
+
+/*
+   Block INTERFACE
+*/
+
+func (t *BtcTx) Cid() cid.Cid {
+	return t.cid
+}
+
+func (t *BtcTx) RawData() []byte {
+	return t.rawdata
+}
+
+func (t *BtcTx) String() string {
+	return fmt.Sprintf("<BtcTx %s>", t.cid)
+}
+
+func (t *BtcTx) Loggable() map[string]interface{} {
+	return map[string]interface{}{
+		"type": "bitcoinTx",
+	}
+}
+
+/*
+   Node INTERFACE
+*/
+
+func (t *BtcTx) Links() []*node.Link {
+	var out []*node.Link
+	for i, in := range t.MsgTx.TxIn {
+		lnk := &node.Link{Cid: sha256ToCid(MBitcoinTx, in.PreviousOutPoint.Hash.CloneBytes())}
+		lnk.Name = fmt.Sprintf("inputs/%d/prevTx", i)
+		out = append(out, lnk)
+	}
+	return out
+}
+
+func (t *BtcTx) Resolve(path []string) (interface{}, []string, error) {
+	switch path[0] {
+	case "version":
+		return t.Version, path[1:], nil
+	case "lockTime":
+		return t.LockTime, path[1:], nil
+	case "inputs":
+		if len(path) == 1 {
+			return t.MsgTx.TxIn, nil, nil
+		}
+
+		index, err := strconv.Atoi(path[1])
+		if err != nil {
+			return nil, nil, err
+		}
+
+		if index >= len(t.MsgTx.TxIn) || index < 0 {
+			return nil, nil, fmt.Errorf("index out of range")
+		}
+
+		inp := t.MsgTx.TxIn[index]
+		if len(path) == 2 {
+			return inp, nil, nil
+		}
+
+		switch path[2] {
+		case "prevTx":
+			return &node.Link{Cid: sha256ToCid(MBitcoinTx, inp.PreviousOutPoint.Hash.CloneBytes())}, path[3:], nil
+		case "seqNo":
+			return inp.Sequence, path[3:], nil
+		case "script":
+			return inp.SignatureScript, path[3:], nil
+		default:
+			return nil, nil, fmt.Errorf("no such link")
+		}
+	case "outputs":
+		if len(path) == 1 {
+			return t.TxOut, nil, nil
+		}
+
+		index, err := strconv.Atoi(path[1])
+		if err != nil {
+			return nil, nil, err
+		}
+
+		if index >= len(t.TxOut) || index < 0 {
+			return nil, nil, fmt.Errorf("index out of range")
+		}
+
+		outp := t.TxOut[index]
+		if len(path) == 2 {
+			return outp, path[2:], nil
+		}
+
+		switch path[2] {
+		case "value":
+			return outp.Value, path[3:], nil
+		case "script":
+			/*
+				if outp.Script[0] == 0x6a { // OP_RETURN
+					c, err := cid.Decode(string(outp.Script[1:]))
+					if err == nil {
+						return &node.Link{Cid: c}, path[3:], nil
+					}
+				}
+			*/
+			return outp.PkScript, path[3:], nil
+		default:
+			return nil, nil, fmt.Errorf("no such link")
+		}
+	default:
+		return nil, nil, fmt.Errorf("no such link")
+	}
+}
+
+func (t *BtcTx) ResolveLink(path []string) (*node.Link, []string, error) {
+	i, rest, err := t.Resolve(path)
+	if err != nil {
+		return nil, rest, err
+	}
+
+	lnk, ok := i.(*node.Link)
+	if !ok {
+		return nil, nil, fmt.Errorf("value was not a link")
+	}
+
+	return lnk, rest, nil
+}
+
+func (t *BtcTx) Size() (uint64, error) {
+	return uint64(len(t.RawData())), nil
+}
+
+func (t *BtcTx) Stat() (*node.NodeStat, error) {
+	return &node.NodeStat{}, nil
+}
+
+func (t *BtcTx) Copy() node.Node {
+	nt := *t // cheating shallow copy
+	return &nt
+}
+
+func (t *BtcTx) Tree(p string, depth int) []string {
+	if depth == 0 {
+		return nil
+	}
+
+	switch p {
+	case "inputs":
+		return t.treeInputs(nil, depth+1)
+	case "outputs":
+		return t.treeOutputs(nil, depth+1)
+	case "":
+		out := []string{"version", "timeLock", "inputs", "outputs"}
+		out = t.treeInputs(out, depth)
+		out = t.treeOutputs(out, depth)
+		return out
+	default:
+		return nil
+	}
+}
+
+func (t *BtcTx) treeInputs(out []string, depth int) []string {
+	if depth < 2 {
+		return out
+	}
+
+	for i := range t.TxIn {
+		inp := "inputs/" + fmt.Sprint(i)
+		out = append(out, inp)
+		if depth > 2 {
+			out = append(out, inp+"/prevTx", inp+"/seqNo", inp+"/script")
+		}
+	}
+	return out
+}
+
+func (t *BtcTx) treeOutputs(out []string, depth int) []string {
+	if depth < 2 {
+		return out
+	}
+
+	for i := range t.TxOut {
+		o := "outputs/" + fmt.Sprint(i)
+		out = append(out, o)
+		if depth > 2 {
+			out = append(out, o+"/script", o+"/value")
+		}
+	}
+	return out
+}
+
+func (t *BtcTx) BTCSha() []byte {
+	mh, _ := mh.Sum(t.RawData(), mh.DBL_SHA2_256, -1)
+	return []byte(mh[2:])
+}
+
+func (t *BtcTx) HexHash() string {
+	return hex.EncodeToString(revString(t.BTCSha()))
+}

pkg/ipfs/ipld/eth_account.go

@@ -0,0 +1,175 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package ipld
+
+import (
+	"encoding/json"
+	"fmt"
+	"math/big"
+
+	"github.com/ipfs/go-cid"
+	node "github.com/ipfs/go-ipld-format"
+)
+
+// EthAccountSnapshot (eth-account-snapshot codec 0x97)
+// represents an ethereum account, i.e. a wallet address or
+// a smart contract
+type EthAccountSnapshot struct {
+	*EthAccount
+
+	cid     cid.Cid
+	rawdata []byte
+}
+
+// EthAccount is the building block of EthAccountSnapshot.
+// Or, is the former stripped of its cid and rawdata components.
+type EthAccount struct {
+	Nonce    uint64
+	Balance  *big.Int
+	Root     []byte // This is the storage root trie
+	CodeHash []byte // This is the hash of the EVM code
+}
+
+// Static (compile time) check that EthAccountSnapshot satisfies the
+// node.Node interface.
+var _ node.Node = (*EthAccountSnapshot)(nil)
+
+/*
+  INPUT
+*/
+
+// Input should be managed by EthStateTrie
+
+/*
+   OUTPUT
+*/
+
+// Output should be managed by EthStateTrie
+
+/*
+   Block INTERFACE
+*/
+
+// RawData returns the binary of the RLP encode of the account snapshot.
+func (as *EthAccountSnapshot) RawData() []byte {
+	return as.rawdata
+}
+
+// Cid returns the cid of the transaction.
+func (as *EthAccountSnapshot) Cid() cid.Cid {
+	return as.cid
+}
+
+// String is a helper for output
+func (as *EthAccountSnapshot) String() string {
+	return fmt.Sprintf("<EthereumAccountSnapshot %s>", as.cid)
+}
+
+// Loggable returns in a map the type of IPLD Link.
+func (as *EthAccountSnapshot) Loggable() map[string]interface{} {
+	return map[string]interface{}{
+		"type": "eth-account-snapshot",
+	}
+}
+
+/*
+   Node INTERFACE
+*/
+
+// Resolve resolves a path through this node, stopping at any link boundary
+// and returning the object found as well as the remaining path to traverse
+func (as *EthAccountSnapshot) Resolve(p []string) (interface{}, []string, error) {
+	if len(p) == 0 {
+		return as, nil, nil
+	}
+
+	if len(p) > 1 {
+		return nil, nil, fmt.Errorf("unexpected path elements past %s", p[0])
+	}
+
+	switch p[0] {
+	case "balance":
+		return as.Balance, nil, nil
+	case "codeHash":
+		return &node.Link{Cid: keccak256ToCid(RawBinary, as.CodeHash)}, nil, nil
+	case "nonce":
+		return as.Nonce, nil, nil
+	case "root":
+		return &node.Link{Cid: keccak256ToCid(MEthStorageTrie, as.Root)}, nil, nil
+	default:
+		return nil, nil, fmt.Errorf("no such link")
+	}
+}
+
+// Tree lists all paths within the object under 'path', and up to the given depth.
+// To list the entire object (similar to `find .`) pass "" and -1
+func (as *EthAccountSnapshot) Tree(p string, depth int) []string {
+	if p != "" || depth == 0 {
+		return nil
+	}
+	return []string{"balance", "codeHash", "nonce", "root"}
+}
+
+// ResolveLink is a helper function that calls resolve and asserts the
+// output is a link
+func (as *EthAccountSnapshot) ResolveLink(p []string) (*node.Link, []string, error) {
+	obj, rest, err := as.Resolve(p)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if lnk, ok := obj.(*node.Link); ok {
+		return lnk, rest, nil
+	}
+
+	return nil, nil, fmt.Errorf("resolved item was not a link")
+}
+
+// Copy will go away. It is here to comply with the interface.
+func (as *EthAccountSnapshot) Copy() node.Node {
+	panic("dont use this yet")
+}
+
+// Links is a helper function that returns all links within this object
+func (as *EthAccountSnapshot) Links() []*node.Link {
+	return nil
+}
+
+// Stat will go away. It is here to comply with the interface.
+func (as *EthAccountSnapshot) Stat() (*node.NodeStat, error) {
+	return &node.NodeStat{}, nil
+}
+
+// Size will go away. It is here to comply with the interface.
+func (as *EthAccountSnapshot) Size() (uint64, error) {
+	return 0, nil
+}
+
+/*
+  EthAccountSnapshot functions
+*/
+
+// MarshalJSON processes the transaction into readable JSON format.
+func (as *EthAccountSnapshot) MarshalJSON() ([]byte, error) {
+	out := map[string]interface{}{
+		"balance":  as.Balance,
+		"codeHash": keccak256ToCid(RawBinary, as.CodeHash),
+		"nonce":    as.Nonce,
+		"root":     keccak256ToCid(MEthStorageTrie, as.Root),
+	}
+	return json.Marshal(out)
+}

pkg/ipfs/ipld/eth_header.go

@@ -0,0 +1,292 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package ipld
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ipfs/go-cid"
+	node "github.com/ipfs/go-ipld-format"
+	mh "github.com/multiformats/go-multihash"
+)
+
+// EthHeader (eth-block, codec 0x90), represents an ethereum block header
+type EthHeader struct {
+	*types.Header
+
+	cid     cid.Cid
+	rawdata []byte
+}
+
+// Static (compile time) check that EthHeader satisfies the node.Node interface.
+var _ node.Node = (*EthHeader)(nil)
+
+/*
+  INPUT
+*/
+
+// NewEthHeader converts a *types.Header into an EthHeader IPLD node
+func NewEthHeader(header *types.Header) (*EthHeader, error) {
+	headerRLP, err := rlp.EncodeToBytes(header)
+	if err != nil {
+		return nil, err
+	}
+	c, err := rawdataToCid(MEthHeader, headerRLP, mh.KECCAK_256)
+	if err != nil {
+		return nil, err
+	}
+	return &EthHeader{
+		Header:  header,
+		cid:     c,
+		rawdata: headerRLP,
+	}, nil
+}
+
+/*
+  OUTPUT
+*/
+
+// DecodeEthHeader takes a cid and its raw binary data
+// from IPFS and returns an EthHeader object for further processing.
+func DecodeEthHeader(c cid.Cid, b []byte) (*EthHeader, error) {
+	var h *types.Header
+	if err := rlp.DecodeBytes(b, h); err != nil {
+		return nil, err
+	}
+	return &EthHeader{
+		Header:  h,
+		cid:     c,
+		rawdata: b,
+	}, nil
+}
+
+/*
+  Block INTERFACE
+*/
+
+// RawData returns the binary of the RLP encode of the block header.
+func (b *EthHeader) RawData() []byte {
+	return b.rawdata
+}
+
+// Cid returns the cid of the block header.
+func (b *EthHeader) Cid() cid.Cid {
+	return b.cid
+}
+
+// String is a helper for output
+func (b *EthHeader) String() string {
+	return fmt.Sprintf("<EthHeader %s>", b.cid)
+}
+
+// Loggable returns a map the type of IPLD Link.
+func (b *EthHeader) Loggable() map[string]interface{} {
+	return map[string]interface{}{
+		"type": "eth-block",
+	}
+}
+
+/*
+  Node INTERFACE
+*/
+
+// Resolve resolves a path through this node, stopping at any link boundary
+// and returning the object found as well as the remaining path to traverse
+func (b *EthHeader) Resolve(p []string) (interface{}, []string, error) {
+	if len(p) == 0 {
+		return b, nil, nil
+	}
+
+	first, rest := p[0], p[1:]
+
+	switch first {
+	case "parent":
+		return &node.Link{Cid: commonHashToCid(MEthHeader, b.ParentHash)}, rest, nil
+	case "receipts":
+		return &node.Link{Cid: commonHashToCid(MEthTxReceiptTrie, b.ReceiptHash)}, rest, nil
+	case "root":
+		return &node.Link{Cid: commonHashToCid(MEthStateTrie, b.Root)}, rest, nil
+	case "tx":
+		return &node.Link{Cid: commonHashToCid(MEthTxTrie, b.TxHash)}, rest, nil
+	case "uncles":
+		return &node.Link{Cid: commonHashToCid(MEthHeaderList, b.UncleHash)}, rest, nil
+	}
+
+	if len(p) != 1 {
+		return nil, nil, fmt.Errorf("unexpected path elements past %s", first)
+	}
+
+	switch first {
+	case "bloom":
+		return b.Bloom, nil, nil
+	case "coinbase":
+		return b.Coinbase, nil, nil
+	case "difficulty":
+		return b.Difficulty, nil, nil
+	case "extra":
+		// This is a []byte. By default they are marshalled into Base64.
+		return fmt.Sprintf("0x%x", b.Extra), nil, nil
+	case "gaslimit":
+		return b.GasLimit, nil, nil
+	case "gasused":
+		return b.GasUsed, nil, nil
+	case "mixdigest":
+		return b.MixDigest, nil, nil
+	case "nonce":
+		return b.Nonce, nil, nil
+	case "number":
+		return b.Number, nil, nil
+	case "time":
+		return b.Time, nil, nil
+	default:
+		return nil, nil, fmt.Errorf("no such link")
+	}
+}
+
+// Tree lists all paths within the object under 'path', and up to the given depth.
+// To list the entire object (similar to `find .`) pass "" and -1
+func (b *EthHeader) Tree(p string, depth int) []string {
+	if p != "" || depth == 0 {
+		return nil
+	}
+
+	return []string{
+		"time",
+		"bloom",
+		"coinbase",
+		"difficulty",
+		"extra",
+		"gaslimit",
+		"gasused",
+		"mixdigest",
+		"nonce",
+		"number",
+		"parent",
+		"receipts",
+		"root",
+		"tx",
+		"uncles",
+	}
+}
+
+// ResolveLink is a helper function that allows easier traversal of links through blocks
+func (b *EthHeader) ResolveLink(p []string) (*node.Link, []string, error) {
+	obj, rest, err := b.Resolve(p)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if lnk, ok := obj.(*node.Link); ok {
+		return lnk, rest, nil
+	}
+
+	return nil, nil, fmt.Errorf("resolved item was not a link")
+}
+
+// Copy will go away. It is here to comply with the Node interface.
+func (b *EthHeader) Copy() node.Node {
+	panic("implement me")
+}
+
+// Links is a helper function that returns all links within this object
+// HINT: Use `ipfs refs <cid>`
+func (b *EthHeader) Links() []*node.Link {
+	return []*node.Link{
+		{Cid: commonHashToCid(MEthHeader, b.ParentHash)},
+		{Cid: commonHashToCid(MEthTxReceiptTrie, b.ReceiptHash)},
+		{Cid: commonHashToCid(MEthStateTrie, b.Root)},
+		{Cid: commonHashToCid(MEthTxTrie, b.TxHash)},
+		{Cid: commonHashToCid(MEthHeaderList, b.UncleHash)},
+	}
+}
+
+// Stat will go away. It is here to comply with the Node interface.
+func (b *EthHeader) Stat() (*node.NodeStat, error) {
+	return &node.NodeStat{}, nil
+}
+
+// Size will go away. It is here to comply with the Node interface.
+func (b *EthHeader) Size() (uint64, error) {
+	return 0, nil
+}
+
+/*
+  EthHeader functions
+*/
+
+// MarshalJSON processes the block header into readable JSON format,
+// converting the right links into their cids, and keeping the original
+// hex hash, allowing the user to simplify external queries.
+func (b *EthHeader) MarshalJSON() ([]byte, error) {
+	out := map[string]interface{}{
+		"time":       b.Time,
+		"bloom":      b.Bloom,
+		"coinbase":   b.Coinbase,
+		"difficulty": b.Difficulty,
+		"extra":      fmt.Sprintf("0x%x", b.Extra),
+		"gaslimit":   b.GasLimit,
+		"gasused":    b.GasUsed,
+		"mixdigest":  b.MixDigest,
+		"nonce":      b.Nonce,
+		"number":     b.Number,
+		"parent":     commonHashToCid(MEthHeader, b.ParentHash),
+		"receipts":   commonHashToCid(MEthTxReceiptTrie, b.ReceiptHash),
+		"root":       commonHashToCid(MEthStateTrie, b.Root),
+		"tx":         commonHashToCid(MEthTxTrie, b.TxHash),
+		"uncles":     commonHashToCid(MEthHeaderList, b.UncleHash),
+	}
+	return json.Marshal(out)
+}
+
+// objJSONBlock defines the output of the JSON RPC API for either
+// "eth_BlockByHash" or "eth_BlockByHeader".
+type objJSONBlock struct {
+	Result objJSONBlockResult `json:"result"`
+}
+
+// objJSONBLockResult is the  nested struct that takes
+// the contents of the JSON field "result".
+type objJSONBlockResult struct {
+	types.Header           // Use its fields and unmarshaler
+	*objJSONBlockResultExt // Add these fields to the parsing
+}
+
+// objJSONBLockResultExt facilitates the composition
+// of the field "result", adding to the
+// `types.Header` fields, both ommers (their hashes) and transactions.
+type objJSONBlockResultExt struct {
+	OmmerHashes  []common.Hash        `json:"uncles"`
+	Transactions []*types.Transaction `json:"transactions"`
+}
+
+// UnmarshalJSON overrides the function types.Header.UnmarshalJSON, allowing us
+// to parse the fields of Header, plus ommer hashes and transactions.
+// (yes, ommer hashes. You will need to "eth_getUncleCountByBlockHash" per each ommer)
+func (o *objJSONBlockResult) UnmarshalJSON(input []byte) error {
+	err := o.Header.UnmarshalJSON(input)
+	if err != nil {
+		return err
+	}
+
+	o.objJSONBlockResultExt = &objJSONBlockResultExt{}
+	err = json.Unmarshal(input, o.objJSONBlockResultExt)
+	return err
+}

pkg/ipfs/ipld/eth_receipt.go

@@ -0,0 +1,199 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package ipld
+
+import (
+	"encoding/json"
+	"fmt"
+	"strconv"
+
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ipfs/go-cid"
+	node "github.com/ipfs/go-ipld-format"
+	mh "github.com/multiformats/go-multihash"
+)
+
+type EthReceipt struct {
+	*types.ReceiptForStorage
+
+	rawdata []byte
+	cid     cid.Cid
+}
+
+// Static (compile time) check that EthReceipt satisfies the node.Node interface.
+var _ node.Node = (*EthReceipt)(nil)
+
+/*
+  INPUT
+*/
+
+// NewReceipt converts a types.ReceiptForStorage to an EthReceipt IPLD node
+func NewReceipt(receipt *types.ReceiptForStorage) (*EthReceipt, error) {
+	receiptRLP, err := rlp.EncodeToBytes(receipt)
+	if err != nil {
+		return nil, err
+	}
+	c, err := rawdataToCid(MEthTxReceipt, receiptRLP, mh.KECCAK_256)
+	if err != nil {
+		return nil, err
+	}
+	return &EthReceipt{
+		ReceiptForStorage: receipt,
+		cid:               c,
+		rawdata:           receiptRLP,
+	}, nil
+}
+
+/*
+ OUTPUT
+*/
+
+// DecodeEthReceipt takes a cid and its raw binary data
+// from IPFS and returns an EthReceipt object for further processing.
+func DecodeEthReceipt(c cid.Cid, b []byte) (*EthReceipt, error) {
+	var r *types.ReceiptForStorage
+	if err := rlp.DecodeBytes(b, r); err != nil {
+		return nil, err
+	}
+	return &EthReceipt{
+		ReceiptForStorage: r,
+		cid:               c,
+		rawdata:           b,
+	}, nil
+}
+
+/*
+  Block INTERFACE
+*/
+
+func (node *EthReceipt) RawData() []byte {
+	return node.rawdata
+}
+
+func (node *EthReceipt) Cid() cid.Cid {
+	return node.cid
+}
+
+// String is a helper for output
+func (r *EthReceipt) String() string {
+	return fmt.Sprintf("<EthereumReceipt %s>", r.cid)
+}
+
+// Loggable returns in a map the type of IPLD Link.
+func (r *EthReceipt) Loggable() map[string]interface{} {
+	return map[string]interface{}{
+		"type": "eth-receipt",
+	}
+}
+
+// Resolve resolves a path through this node, stopping at any link boundary
+// and returning the object found as well as the remaining path to traverse
+func (r *EthReceipt) Resolve(p []string) (interface{}, []string, error) {
+	if len(p) == 0 {
+		return r, nil, nil
+	}
+
+	if len(p) > 1 {
+		return nil, nil, fmt.Errorf("unexpected path elements past %s", p[0])
+	}
+
+	switch p[0] {
+
+	case "root":
+		return r.PostState, nil, nil
+	case "status":
+		return r.Status, nil, nil
+	case "cumulativeGasUsed":
+		return r.CumulativeGasUsed, nil, nil
+	case "logsBloom":
+		return r.Bloom, nil, nil
+	case "logs":
+		return r.Logs, nil, nil
+	case "transactionHash":
+		return r.TxHash, nil, nil
+	case "contractAddress":
+		return r.ContractAddress, nil, nil
+	case "gasUsed":
+		return r.GasUsed, nil, nil
+	default:
+		return nil, nil, fmt.Errorf("no such link")
+	}
+}
+
+// Tree lists all paths within the object under 'path', and up to the given depth.
+// To list the entire object (similar to `find .`) pass "" and -1
+func (r *EthReceipt) Tree(p string, depth int) []string {
+	if p != "" || depth == 0 {
+		return nil
+	}
+	return []string{"root", "status", "cumulativeGasUsed", "logsBloom", "logs", "transactionHash", "contractAddress", "gasUsed"}
+}
+
+// ResolveLink is a helper function that calls resolve and asserts the
+// output is a link
+func (r *EthReceipt) ResolveLink(p []string) (*node.Link, []string, error) {
+	obj, rest, err := r.Resolve(p)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if lnk, ok := obj.(*node.Link); ok {
+		return lnk, rest, nil
+	}
+
+	return nil, nil, fmt.Errorf("resolved item was not a link")
+}
+
+// Copy will go away. It is here to comply with the Node interface.
+func (*EthReceipt) Copy() node.Node {
+	panic("implement me")
+}
+
+// Links is a helper function that returns all links within this object
+func (*EthReceipt) Links() []*node.Link {
+	return nil
+}
+
+// Stat will go away. It is here to comply with the interface.
+func (r *EthReceipt) Stat() (*node.NodeStat, error) {
+	return &node.NodeStat{}, nil
+}
+
+// Size will go away. It is here to comply with the interface.
+func (r *EthReceipt) Size() (uint64, error) {
+	return strconv.ParseUint((*types.Receipt)(r.ReceiptForStorage).Size().String(), 10, 64)
+}
+
+/*
+  EthReceipt functions
+*/
+
+// MarshalJSON processes the receipt into readable JSON format.
+func (r *EthReceipt) MarshalJSON() ([]byte, error) {
+	out := map[string]interface{}{
+		"root":              r.PostState,
+		"status":            r.Status,
+		"cumulativeGasUsed": r.CumulativeGasUsed,
+		"logsBloom":         r.Bloom,
+		"logs":              r.Logs,
+		"transactionHash":   r.TxHash,
+		"contractAddress":   r.ContractAddress,
+		"gasUsed":           r.GasUsed,
+	}
+	return json.Marshal(out)
+}

pkg/ipfs/ipld/eth_state.go

@@ -0,0 +1,109 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package ipld
+
+import (
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ipfs/go-cid"
+	node "github.com/ipfs/go-ipld-format"
+	mh "github.com/multiformats/go-multihash"
+)
+
+// EthStateTrie (eth-state-trie, codec 0x96), represents
+// a node from the state trie in ethereum.
+type EthStateTrie struct {
+	*TrieNode
+}
+
+// Static (compile time) check that EthStateTrie satisfies the node.Node interface.
+var _ node.Node = (*EthStateTrie)(nil)
+
+/*
+  INPUT
+*/
+
+// FromStateTrieRLP takes the RLP bytes of an ethereum
+// state trie node to return it as an IPLD node for further processing.
+func FromStateTrieRLP(stateNodeRLP []byte) (*EthStateTrie, error) {
+	c, err := rawdataToCid(MEthStateTrie, stateNodeRLP, mh.KECCAK_256)
+	if err != nil {
+		return nil, err
+	}
+	return DecodeEthStateTrie(c, stateNodeRLP)
+}
+
+/*
+  OUTPUT
+*/
+
+// DecodeEthStateTrie returns an EthStateTrie object from its cid and rawdata.
+func DecodeEthStateTrie(c cid.Cid, b []byte) (*EthStateTrie, error) {
+	tn, err := decodeTrieNode(c, b, decodeEthStateTrieLeaf)
+	if err != nil {
+		return nil, err
+	}
+	return &EthStateTrie{TrieNode: tn}, nil
+}
+
+// decodeEthStateTrieLeaf parses a eth-tx-trie leaf
+// from decoded RLP elements
+func decodeEthStateTrieLeaf(i []interface{}) ([]interface{}, error) {
+	var account EthAccount
+	if err := rlp.DecodeBytes(i[1].([]byte), &account); err != nil {
+		return nil, err
+	}
+	c, err := rawdataToCid(MEthAccountSnapshot, i[1].([]byte), mh.KECCAK_256)
+	if err != nil {
+		return nil, err
+	}
+	return []interface{}{
+		i[0].([]byte),
+		&EthAccountSnapshot{
+			EthAccount: &account,
+			cid:        c,
+			rawdata:    i[1].([]byte),
+		},
+	}, nil
+}
+
+/*
+  Block INTERFACE
+*/
+
+// RawData returns the binary of the RLP encode of the state trie node.
+func (st *EthStateTrie) RawData() []byte {
+	return st.rawdata
+}
+
+// Cid returns the cid of the state trie node.
+func (st *EthStateTrie) Cid() cid.Cid {
+	return st.cid
+}
+
+// String is a helper for output
+func (st *EthStateTrie) String() string {
+	return fmt.Sprintf("<EthereumStateTrie %s>", st.cid)
+}
+
+// Loggable returns in a map the type of IPLD Link.
+func (st *EthStateTrie) Loggable() map[string]interface{} {
+	return map[string]interface{}{
+		"type": "eth-state-trie",
+	}
+}

pkg/ipfs/ipld/eth_storage.go

@@ -0,0 +1,96 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package ipld
+
+import (
+	"fmt"
+
+	"github.com/ipfs/go-cid"
+	node "github.com/ipfs/go-ipld-format"
+	mh "github.com/multiformats/go-multihash"
+)
+
+// EthStorageTrie (eth-storage-trie, codec 0x98), represents
+// a node from the storage trie in ethereum.
+type EthStorageTrie struct {
+	*TrieNode
+}
+
+// Static (compile time) check that EthStorageTrie satisfies the node.Node interface.
+var _ node.Node = (*EthStorageTrie)(nil)
+
+/*
+  INPUT
+*/
+
+// FromStorageTrieRLP takes the RLP bytes of an ethereum
+// storage trie node to return it as an IPLD node for further processing.
+func FromStorageTrieRLP(storageNodeRLP []byte) (*EthStorageTrie, error) {
+	c, err := rawdataToCid(MEthStorageTrie, storageNodeRLP, mh.KECCAK_256)
+	if err != nil {
+		return nil, err
+	}
+	return DecodeEthStorageTrie(c, storageNodeRLP)
+}
+
+/*
+  OUTPUT
+*/
+
+// DecodeEthStorageTrie returns an EthStorageTrie object from its cid and rawdata.
+func DecodeEthStorageTrie(c cid.Cid, b []byte) (*EthStorageTrie, error) {
+	tn, err := decodeTrieNode(c, b, decodeEthStorageTrieLeaf)
+	if err != nil {
+		return nil, err
+	}
+	return &EthStorageTrie{TrieNode: tn}, nil
+}
+
+// decodeEthStorageTrieLeaf parses a eth-tx-trie leaf
+// from decoded RLP elements
+func decodeEthStorageTrieLeaf(i []interface{}) ([]interface{}, error) {
+	return []interface{}{
+		i[0].([]byte),
+		i[1].([]byte),
+	}, nil
+}
+
+/*
+  Block INTERFACE
+*/
+
+// RawData returns the binary of the RLP encode of the storage trie node.
+func (st *EthStorageTrie) RawData() []byte {
+	return st.rawdata
+}
+
+// Cid returns the cid of the storage trie node.
+func (st *EthStorageTrie) Cid() cid.Cid {
+	return st.cid
+}
+
+// String is a helper for output
+func (st *EthStorageTrie) String() string {
+	return fmt.Sprintf("<EthereumStorageTrie %s>", st.cid)
+}
+
+// Loggable returns in a map the type of IPLD Link.
+func (st *EthStorageTrie) Loggable() map[string]interface{} {
+	return map[string]interface{}{
+		"type": "eth-storage-trie",
+	}
+}

pkg/ipfs/ipld/eth_tx.go

@@ -0,0 +1,215 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package ipld
+
+import (
+	"encoding/json"
+	"fmt"
+	"strconv"
+
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ipfs/go-cid"
+	node "github.com/ipfs/go-ipld-format"
+	mh "github.com/multiformats/go-multihash"
+)
+
+// EthTx (eth-tx codec 0x93) represents an ethereum transaction
+type EthTx struct {
+	*types.Transaction
+
+	cid     cid.Cid
+	rawdata []byte
+}
+
+// Static (compile time) check that EthTx satisfies the node.Node interface.
+var _ node.Node = (*EthTx)(nil)
+
+/*
+  INPUT
+*/
+
+// NewEthTx converts a *types.Transaction to an EthTx IPLD node
+func NewEthTx(tx *types.Transaction) (*EthTx, error) {
+	txRLP, err := rlp.EncodeToBytes(tx)
+	if err != nil {
+		return nil, err
+	}
+	c, err := rawdataToCid(MEthTx, txRLP, mh.KECCAK_256)
+	if err != nil {
+		return nil, err
+	}
+	return &EthTx{
+		Transaction: tx,
+		cid:         c,
+		rawdata:     txRLP,
+	}, nil
+}
+
+/*
+ OUTPUT
+*/
+
+// DecodeEthTx takes a cid and its raw binary data
+// from IPFS and returns an EthTx object for further processing.
+func DecodeEthTx(c cid.Cid, b []byte) (*EthTx, error) {
+	var t *types.Transaction
+	if err := rlp.DecodeBytes(b, t); err != nil {
+		return nil, err
+	}
+	return &EthTx{
+		Transaction: t,
+		cid:         c,
+		rawdata:     b,
+	}, nil
+}
+
+/*
+  Block INTERFACE
+*/
+
+// RawData returns the binary of the RLP encode of the transaction.
+func (t *EthTx) RawData() []byte {
+	return t.rawdata
+}
+
+// Cid returns the cid of the transaction.
+func (t *EthTx) Cid() cid.Cid {
+	return t.cid
+}
+
+// String is a helper for output
+func (t *EthTx) String() string {
+	return fmt.Sprintf("<EthereumTx %s>", t.cid)
+}
+
+// Loggable returns in a map the type of IPLD Link.
+func (t *EthTx) Loggable() map[string]interface{} {
+	return map[string]interface{}{
+		"type": "eth-tx",
+	}
+}
+
+/*
+  Node INTERFACE
+*/
+
+// Resolve resolves a path through this node, stopping at any link boundary
+// and returning the object found as well as the remaining path to traverse
+func (t *EthTx) Resolve(p []string) (interface{}, []string, error) {
+	if len(p) == 0 {
+		return t, nil, nil
+	}
+
+	if len(p) > 1 {
+		return nil, nil, fmt.Errorf("unexpected path elements past %s", p[0])
+	}
+
+	switch p[0] {
+
+	case "gas":
+		return t.Gas(), nil, nil
+	case "gasPrice":
+		return t.GasPrice(), nil, nil
+	case "input":
+		return fmt.Sprintf("%x", t.Data()), nil, nil
+	case "nonce":
+		return t.Nonce(), nil, nil
+	case "r":
+		_, r, _ := t.RawSignatureValues()
+		return hexutil.EncodeBig(r), nil, nil
+	case "s":
+		_, _, s := t.RawSignatureValues()
+		return hexutil.EncodeBig(s), nil, nil
+	case "toAddress":
+		return t.To(), nil, nil
+	case "v":
+		v, _, _ := t.RawSignatureValues()
+		return hexutil.EncodeBig(v), nil, nil
+	case "value":
+		return hexutil.EncodeBig(t.Value()), nil, nil
+	default:
+		return nil, nil, fmt.Errorf("no such link")
+	}
+}
+
+// Tree lists all paths within the object under 'path', and up to the given depth.
+// To list the entire object (similar to `find .`) pass "" and -1
+func (t *EthTx) Tree(p string, depth int) []string {
+	if p != "" || depth == 0 {
+		return nil
+	}
+	return []string{"gas", "gasPrice", "input", "nonce", "r", "s", "toAddress", "v", "value"}
+}
+
+// ResolveLink is a helper function that calls resolve and asserts the
+// output is a link
+func (t *EthTx) ResolveLink(p []string) (*node.Link, []string, error) {
+	obj, rest, err := t.Resolve(p)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if lnk, ok := obj.(*node.Link); ok {
+		return lnk, rest, nil
+	}
+
+	return nil, nil, fmt.Errorf("resolved item was not a link")
+}
+
+// Copy will go away. It is here to comply with the interface.
+func (t *EthTx) Copy() node.Node {
+	panic("implement me")
+}
+
+// Links is a helper function that returns all links within this object
+func (t *EthTx) Links() []*node.Link {
+	return nil
+}
+
+// Stat will go away. It is here to comply with the interface.
+func (t *EthTx) Stat() (*node.NodeStat, error) {
+	return &node.NodeStat{}, nil
+}
+
+// Size will go away. It is here to comply with the interface.
+func (t *EthTx) Size() (uint64, error) {
+	return strconv.ParseUint(t.Transaction.Size().String(), 10, 64)
+}
+
+/*
+  EthTx functions
+*/
+
+// MarshalJSON processes the transaction into readable JSON format.
+func (t *EthTx) MarshalJSON() ([]byte, error) {
+	v, r, s := t.RawSignatureValues()
+
+	out := map[string]interface{}{
+		"gas":       t.Gas(),
+		"gasPrice":  hexutil.EncodeBig(t.GasPrice()),
+		"input":     fmt.Sprintf("%x", t.Data()),
+		"nonce":     t.Nonce(),
+		"r":         hexutil.EncodeBig(r),
+		"s":         hexutil.EncodeBig(s),
+		"toAddress": t.To(),
+		"v":         hexutil.EncodeBig(v),
+		"value":     hexutil.EncodeBig(t.Value()),
+	}
+	return json.Marshal(out)
+}

pkg/ipfs/ipld/shared.go

@@ -0,0 +1,102 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package ipld
+
+import (
+	"bytes"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ipfs/go-cid"
+	mh "github.com/multiformats/go-multihash"
+)
+
+// IPLD Codecs for Ethereum
+// See the authoritative document:
+// https://github.com/multiformats/multicodec/blob/master/table.csv
+const (
+	RawBinary           = 0x55
+	MEthHeader          = 0x90
+	MEthHeaderList      = 0x91
+	MEthTxTrie          = 0x92
+	MEthTx              = 0x93
+	MEthTxReceiptTrie   = 0x94
+	MEthTxReceipt       = 0x95
+	MEthStateTrie       = 0x96
+	MEthAccountSnapshot = 0x97
+	MEthStorageTrie     = 0x98
+	MBitcoinHeader      = 0xb0
+	MBitcoinTx          = 0xb1
+)
+
+// rawdataToCid takes the desired codec and a slice of bytes
+// and returns the proper cid of the object.
+func rawdataToCid(codec uint64, rawdata []byte, multiHash uint64) (cid.Cid, error) {
+	c, err := cid.Prefix{
+		Codec:    codec,
+		Version:  1,
+		MhType:   multiHash,
+		MhLength: -1,
+	}.Sum(rawdata)
+	if err != nil {
+		return cid.Cid{}, err
+	}
+	return c, nil
+}
+
+// keccak256ToCid takes a keccak256 hash and returns its cid based on
+// the codec given.
+func keccak256ToCid(codec uint64, h []byte) cid.Cid {
+	buf, err := mh.Encode(h, mh.KECCAK_256)
+	if err != nil {
+		panic(err)
+	}
+
+	return cid.NewCidV1(codec, mh.Multihash(buf))
+}
+
+// commonHashToCid takes a go-ethereum common.Hash and returns its
+// cid based on the codec given,
+func commonHashToCid(codec uint64, h common.Hash) cid.Cid {
+	mhash, err := mh.Encode(h[:], mh.KECCAK_256)
+	if err != nil {
+		panic(err)
+	}
+
+	return cid.NewCidV1(codec, mhash)
+}
+
+// sha256ToCid takes a sha246 hash and returns its cid based on the
+// codec given
+func sha256ToCid(codec uint64, h []byte) cid.Cid {
+	hash, err := mh.Encode(h, mh.DBL_SHA2_256)
+	if err != nil {
+		panic(err)
+	}
+
+	return cid.NewCidV1(codec, hash)
+}
+
+// getRLP encodes the given object to RLP returning its bytes.
+func getRLP(object interface{}) []byte {
+	buf := new(bytes.Buffer)
+	if err := rlp.Encode(buf, object); err != nil {
+		panic(err)
+	}
+
+	return buf.Bytes()
+}

pkg/ipfs/ipld/trie_node.go

@@ -0,0 +1,440 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package ipld
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ipfs/go-cid"
+	node "github.com/ipfs/go-ipld-format"
+)
+
+// TrieNode is the general abstraction for
+//ethereum IPLD trie nodes.
+type TrieNode struct {
+	// leaf, extension or branch
+	nodeKind string
+
+	// If leaf or extension: [0] is key, [1] is val.
+	// If branch: [0] - [16] are children.
+	elements []interface{}
+
+	// IPLD block information
+	cid     cid.Cid
+	rawdata []byte
+}
+
+/*
+  OUTPUT
+*/
+
+type trieNodeLeafDecoder func([]interface{}) ([]interface{}, error)
+
+// decodeTrieNode returns a TrieNode object from an IPLD block's
+// cid and rawdata.
+func decodeTrieNode(c cid.Cid, b []byte,
+	leafDecoder trieNodeLeafDecoder) (*TrieNode, error) {
+	var (
+		i, decoded, elements []interface{}
+		nodeKind             string
+		err                  error
+	)
+
+	err = rlp.DecodeBytes(b, &i)
+	if err != nil {
+		return nil, err
+	}
+
+	codec := c.Type()
+	switch len(i) {
+	case 2:
+		nodeKind, decoded, err = decodeCompactKey(i)
+		if err != nil {
+			return nil, err
+		}
+
+		if nodeKind == "extension" {
+			elements, err = parseTrieNodeExtension(decoded, codec)
+		}
+		if nodeKind == "leaf" {
+			elements, err = leafDecoder(decoded)
+		}
+		if nodeKind != "extension" && nodeKind != "leaf" {
+			return nil, fmt.Errorf("unexpected nodeKind returned from decoder")
+		}
+	case 17:
+		nodeKind = "branch"
+		elements, err = parseTrieNodeBranch(i, codec)
+		if err != nil {
+			return nil, err
+		}
+	default:
+		return nil, fmt.Errorf("unknown trie node type")
+	}
+
+	return &TrieNode{
+		nodeKind: nodeKind,
+		elements: elements,
+		rawdata:  b,
+		cid:      c,
+	}, nil
+}
+
+// decodeCompactKey takes a compact key, and returns its nodeKind and value.
+func decodeCompactKey(i []interface{}) (string, []interface{}, error) {
+	first := i[0].([]byte)
+	last := i[1].([]byte)
+
+	switch first[0] / 16 {
+	case '\x00':
+		return "extension", []interface{}{
+			nibbleToByte(first)[2:],
+			last,
+		}, nil
+	case '\x01':
+		return "extension", []interface{}{
+			nibbleToByte(first)[1:],
+			last,
+		}, nil
+	case '\x02':
+		return "leaf", []interface{}{
+			nibbleToByte(first)[2:],
+			last,
+		}, nil
+	case '\x03':
+		return "leaf", []interface{}{
+			nibbleToByte(first)[1:],
+			last,
+		}, nil
+	default:
+		return "", nil, fmt.Errorf("unknown hex prefix")
+	}
+}
+
+// parseTrieNodeExtension helper improves readability
+func parseTrieNodeExtension(i []interface{}, codec uint64) ([]interface{}, error) {
+	return []interface{}{
+		i[0].([]byte),
+		keccak256ToCid(codec, i[1].([]byte)),
+	}, nil
+}
+
+// parseTrieNodeBranch helper improves readability
+func parseTrieNodeBranch(i []interface{}, codec uint64) ([]interface{}, error) {
+	var out []interface{}
+
+	for _, vi := range i {
+		v := vi.([]byte)
+
+		switch len(v) {
+		case 0:
+			out = append(out, nil)
+		case 32:
+			out = append(out, keccak256ToCid(codec, v))
+		default:
+			return nil, fmt.Errorf("unrecognized object: %v", v)
+		}
+	}
+
+	return out, nil
+}
+
+/*
+  Node INTERFACE
+*/
+
+// Resolve resolves a path through this node, stopping at any link boundary
+// and returning the object found as well as the remaining path to traverse
+func (t *TrieNode) Resolve(p []string) (interface{}, []string, error) {
+	switch t.nodeKind {
+	case "extension":
+		return t.resolveTrieNodeExtension(p)
+	case "leaf":
+		return t.resolveTrieNodeLeaf(p)
+	case "branch":
+		return t.resolveTrieNodeBranch(p)
+	default:
+		return nil, nil, fmt.Errorf("nodeKind case not implemented")
+	}
+}
+
+// Tree lists all paths within the object under 'path', and up to the given depth.
+// To list the entire object (similar to `find .`) pass "" and -1
+func (t *TrieNode) Tree(p string, depth int) []string {
+	if p != "" || depth == 0 {
+		return nil
+	}
+
+	var out []string
+
+	switch t.nodeKind {
+	case "extension":
+		var val string
+		for _, e := range t.elements[0].([]byte) {
+			val += fmt.Sprintf("%x", e)
+		}
+		return []string{val}
+	case "branch":
+		for i, elem := range t.elements {
+			if _, ok := elem.(*cid.Cid); ok {
+				out = append(out, fmt.Sprintf("%x", i))
+			}
+		}
+		return out
+
+	default:
+		return nil
+	}
+}
+
+// ResolveLink is a helper function that calls resolve and asserts the
+// output is a link
+func (t *TrieNode) ResolveLink(p []string) (*node.Link, []string, error) {
+	obj, rest, err := t.Resolve(p)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	lnk, ok := obj.(*node.Link)
+	if !ok {
+		return nil, nil, fmt.Errorf("was not a link")
+	}
+
+	return lnk, rest, nil
+}
+
+// Copy will go away. It is here to comply with the interface.
+func (t *TrieNode) Copy() node.Node {
+	panic("dont use this yet")
+}
+
+// Links is a helper function that returns all links within this object
+func (t *TrieNode) Links() []*node.Link {
+	var out []*node.Link
+
+	for _, i := range t.elements {
+		c, ok := i.(cid.Cid)
+		if ok {
+			out = append(out, &node.Link{Cid: c})
+		}
+	}
+
+	return out
+}
+
+// Stat will go away. It is here to comply with the interface.
+func (t *TrieNode) Stat() (*node.NodeStat, error) {
+	return &node.NodeStat{}, nil
+}
+
+// Size will go away. It is here to comply with the interface.
+func (t *TrieNode) Size() (uint64, error) {
+	return 0, nil
+}
+
+/*
+  TrieNode functions
+*/
+
+// MarshalJSON processes the transaction trie into readable JSON format.
+func (t *TrieNode) MarshalJSON() ([]byte, error) {
+	var out map[string]interface{}
+
+	switch t.nodeKind {
+	case "extension":
+		fallthrough
+	case "leaf":
+		var hexPrefix string
+		for _, e := range t.elements[0].([]byte) {
+			hexPrefix += fmt.Sprintf("%x", e)
+		}
+
+		// if we got a byte we need to do this casting otherwise
+		// it will be marshaled to a base64 encoded value
+		if _, ok := t.elements[1].([]byte); ok {
+			var hexVal string
+			for _, e := range t.elements[1].([]byte) {
+				hexVal += fmt.Sprintf("%x", e)
+			}
+
+			t.elements[1] = hexVal
+		}
+
+		out = map[string]interface{}{
+			"type":    t.nodeKind,
+			hexPrefix: t.elements[1],
+		}
+
+	case "branch":
+		out = map[string]interface{}{
+			"type": "branch",
+			"0":    t.elements[0],
+			"1":    t.elements[1],
+			"2":    t.elements[2],
+			"3":    t.elements[3],
+			"4":    t.elements[4],
+			"5":    t.elements[5],
+			"6":    t.elements[6],
+			"7":    t.elements[7],
+			"8":    t.elements[8],
+			"9":    t.elements[9],
+			"a":    t.elements[10],
+			"b":    t.elements[11],
+			"c":    t.elements[12],
+			"d":    t.elements[13],
+			"e":    t.elements[14],
+			"f":    t.elements[15],
+		}
+	default:
+		return nil, fmt.Errorf("nodeKind %s not supported", t.nodeKind)
+	}
+
+	return json.Marshal(out)
+}
+
+// nibbleToByte expands the nibbles of a byte slice into their own bytes.
+func nibbleToByte(k []byte) []byte {
+	var out []byte
+
+	for _, b := range k {
+		out = append(out, b/16)
+		out = append(out, b%16)
+	}
+
+	return out
+}
+
+// Resolve reading conveniences
+func (t *TrieNode) resolveTrieNodeExtension(p []string) (interface{}, []string, error) {
+	nibbles := t.elements[0].([]byte)
+	idx, rest := shiftFromPath(p, len(nibbles))
+	if len(idx) < len(nibbles) {
+		return nil, nil, fmt.Errorf("not enough nibbles to traverse this extension")
+	}
+
+	for _, i := range idx {
+		if getHexIndex(string(i)) == -1 {
+			return nil, nil, fmt.Errorf("invalid path element")
+		}
+	}
+
+	for i, n := range nibbles {
+		if string(idx[i]) != fmt.Sprintf("%x", n) {
+			return nil, nil, fmt.Errorf("no such link in this extension")
+		}
+	}
+
+	return &node.Link{Cid: t.elements[1].(cid.Cid)}, rest, nil
+}
+
+func (t *TrieNode) resolveTrieNodeLeaf(p []string) (interface{}, []string, error) {
+	nibbles := t.elements[0].([]byte)
+
+	if len(nibbles) != 0 {
+		idx, rest := shiftFromPath(p, len(nibbles))
+		if len(idx) < len(nibbles) {
+			return nil, nil, fmt.Errorf("not enough nibbles to traverse this leaf")
+		}
+
+		for _, i := range idx {
+			if getHexIndex(string(i)) == -1 {
+				return nil, nil, fmt.Errorf("invalid path element")
+			}
+		}
+
+		for i, n := range nibbles {
+			if string(idx[i]) != fmt.Sprintf("%x", n) {
+				return nil, nil, fmt.Errorf("no such link in this extension")
+			}
+		}
+
+		p = rest
+	}
+
+	link, ok := t.elements[1].(node.Node)
+	if !ok {
+		return nil, nil, fmt.Errorf("leaf children is not an IPLD node")
+	}
+
+	return link.Resolve(p)
+}
+
+func (t *TrieNode) resolveTrieNodeBranch(p []string) (interface{}, []string, error) {
+	idx, rest := shiftFromPath(p, 1)
+	hidx := getHexIndex(idx)
+	if hidx == -1 {
+		return nil, nil, fmt.Errorf("incorrect path")
+	}
+
+	child := t.elements[hidx]
+	if child != nil {
+		return &node.Link{Cid: child.(cid.Cid)}, rest, nil
+	}
+	return nil, nil, fmt.Errorf("no such link in this branch")
+}
+
+// shiftFromPath extracts from a given path (as a slice of strings)
+// the given number of elements as a single string, returning whatever
+// it has not taken.
+//
+// Examples:
+// ["0", "a", "something"] and 1 -> "0" and ["a", "something"]
+// ["ab", "c", "d", "1"] and 2 -> "ab" and ["c", "d", "1"]
+// ["abc", "d", "1"] and 2 -> "ab" and ["c", "d", "1"]
+func shiftFromPath(p []string, i int) (string, []string) {
+	var (
+		out  string
+		rest []string
+	)
+
+	for _, pe := range p {
+		re := ""
+		for _, c := range pe {
+			if len(out) < i {
+				out += string(c)
+			} else {
+				re += string(c)
+			}
+		}
+
+		if len(out) == i && re != "" {
+			rest = append(rest, re)
+		}
+	}
+
+	return out, rest
+}
+
+// getHexIndex returns to you the integer 0 - 15 equivalent to your
+// string character if applicable, or -1 otherwise.
+func getHexIndex(s string) int {
+	if len(s) != 1 {
+		return -1
+	}
+
+	c := byte(s[0])
+	switch {
+	case '0' <= c && c <= '9':
+		return int(c - '0')
+	case 'a' <= c && c <= 'f':
+		return int(c - 'a' + 10)
+	}
+
+	return -1
+}

pkg/super_node/btc/converter.go

@@ -17,11 +17,7 @@
 package btc
 
 import (
-	"bytes"
 	"fmt"
-
-	"github.com/btcsuite/btcd/wire"
-	"github.com/btcsuite/btcutil"
 )
 
 // PayloadConverter satisfies the PayloadConverter interface for bitcoin
@@ -39,18 +35,8 @@ func (pc *PayloadConverter) Convert(payload interface{}) (interface{}, error) {
 	if !ok {
 		return nil, fmt.Errorf("btc converter: expected payload type %T got %T", BlockPayload{}, payload)
 	}
-	msgBlock := wire.NewMsgBlock(btcBlockPayload.Header)
-	for _, tx := range btcBlockPayload.Txs {
-		msgBlock.AddTransaction(tx.MsgTx())
-	}
-	w := bytes.NewBuffer(make([]byte, 0, msgBlock.SerializeSize()))
-	if err := msgBlock.Serialize(w); err != nil {
-		return nil, err
-	}
-	utilBlock := btcutil.NewBlockFromBlockAndBytes(msgBlock, w.Bytes())
-	utilBlock.SetHeight(btcBlockPayload.Height)
 	txMeta := make([]TxModel, len(btcBlockPayload.Txs))
-	for _, tx := range utilBlock.Transactions() {
+	for _, tx := range btcBlockPayload.Txs {
 		index := tx.Index()
 		txModel := TxModel{
 			TxHash:     tx.Hash().String(),
@@ -63,7 +49,7 @@ func (pc *PayloadConverter) Convert(payload interface{}) (interface{}, error) {
 		txMeta[index] = txModel
 	}
 	return IPLDPayload{
-		Block:      utilBlock,
+		BlockPayload: btcBlockPayload,
 		TxMetaData:   txMeta,
 	}, nil
 }

pkg/super_node/btc/types.go

@@ -37,7 +37,7 @@ type BlockPayload struct {
 // Returned by PayloadConverter
 // Passed to IPLDPublisher and ResponseFilterer
 type IPLDPayload struct {
-	Block      *btcutil.Block
+	BlockPayload
 	TxMetaData []TxModel
 }
 

pkg/super_node/config/btc_subscription.go

@@ -0,0 +1,17 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package config

pkg/super_node/config/config.go

@@ -52,11 +52,11 @@ type SuperNode struct {
 	// Sync params
 	Sync     bool
 	Workers  int
-	WSClient core.RPCClient
+	WSClient interface{}
 	NodeInfo core.Node
 	// Backfiller params
 	BackFill   bool
-	HTTPClient core.RPCClient
+	HTTPClient interface{}
 	Frequency  time.Duration
 	BatchSize  uint64
 }
@@ -147,7 +147,7 @@ func (sn *SuperNode) BackFillFields() error {
 	return nil
 }
 
-func getNodeAndClient(chain ChainType, path string) (core.Node, core.RPCClient, error) {
+func getNodeAndClient(chain ChainType, path string) (core.Node, interface{}, error) {
 	switch chain {
 	case Ethereum:
 		rawRPCClient, err := rpc.Dial(path)

pkg/super_node/eth/converter.go

@@ -50,17 +50,10 @@ func (pc *PayloadConverter) Convert(payload interface{}) (interface{}, error) {
 	if err := rlp.DecodeBytes(stateDiffPayload.BlockRlp, block); err != nil {
 		return nil, err
 	}
-	// Process and publish headers
-	header := block.Header()
-	headerRlp, err := rlp.EncodeToBytes(header)
-	if err != nil {
-		return nil, err
-	}
 	trxLen := len(block.Transactions())
 	convertedPayload := &IPLDPayload{
 		TotalDifficulty: stateDiffPayload.TotalDifficulty,
 		Block:           block,
-		HeaderRLP:       headerRlp,
 		TxMetaData:      make([]TxModel, 0, trxLen),
 		Receipts:        make(types.Receipts, 0, trxLen),
 		ReceiptMetaData: make([]ReceiptModel, 0, trxLen),

pkg/super_node/eth/converter_test.go

@@ -44,7 +44,9 @@ var _ = Describe("Converter", func() {
 			expectedBody, err := rlp.EncodeToBytes(mocks.MockBlock.Body())
 			Expect(err).ToNot(HaveOccurred())
 			Expect(gotBody).To(Equal(expectedBody))
-			Expect(convertedPayload.HeaderRLP).To(Equal(mocks.MockHeaderRlp))
+			gotHeader, err := rlp.EncodeToBytes(convertedPayload.Block.Header())
+			Expect(err).ToNot(HaveOccurred())
+			Expect(gotHeader).To(Equal(mocks.MockHeaderRlp))
 			Expect(convertedPayload.TxMetaData).To(Equal(mocks.MockTrxMeta))
 			Expect(convertedPayload.ReceiptMetaData).To(Equal(mocks.MockRctMeta))
 		})

pkg/super_node/eth/filterer.go

@@ -76,7 +76,11 @@ func (s *ResponseFilterer) Filter(filter, payload interface{}) (interface{}, err
 
 func (s *ResponseFilterer) filterHeaders(headerFilter config.HeaderFilter, response *StreamPayload, payload *IPLDPayload) error {
 	if !headerFilter.Off {
-		response.HeadersRlp = append(response.HeadersRlp, payload.HeaderRLP)
+		headerRLP, err := rlp.EncodeToBytes(payload.Block.Header())
+		if err != nil {
+			return err
+		}
+		response.HeadersRlp = append(response.HeadersRlp, headerRLP)
 		if headerFilter.Uncles {
 			response.UnclesRlp = make([][]byte, 0, len(payload.Block.Body().Uncles))
 			for _, uncle := range payload.Block.Body().Uncles {

pkg/super_node/eth/mocks/dag_putters.go

@@ -25,22 +25,26 @@ import (
 // DagPutter is a mock for testing the ipfs publisher
 type DagPutter struct {
 	CIDsToReturn []string
+	PassedRaw    interface{}
 	ErrToReturn  error
 }
 
 // DagPut returns the pre-loaded CIDs or error
 func (dp *DagPutter) DagPut(raw interface{}) ([]string, error) {
+	dp.PassedRaw = raw
 	return dp.CIDsToReturn, dp.ErrToReturn
 }
 
 // MappedDagPutter is a mock for testing the ipfs publisher
 type MappedDagPutter struct {
 	CIDsToReturn map[common.Hash][]string
+	PassedRaw    interface{}
 	ErrToReturn  error
 }
 
 // DagPut returns the pre-loaded CIDs or error
 func (mdp *MappedDagPutter) DagPut(raw interface{}) ([]string, error) {
+	mdp.PassedRaw = raw
 	if mdp.CIDsToReturn == nil {
 		return nil, errors.New("mapped dag putter needs to be initialized with a map of cids to return")
 	}

pkg/super_node/eth/mocks/streamer.go

@@ -19,6 +19,7 @@ package mocks
 import (
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/ethereum/go-ethereum/statediff"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
 )
 
 // StateDiffStreamer is the underlying struct for the Streamer interface
@@ -30,7 +31,7 @@ type StateDiffStreamer struct {
 }
 
 // Stream is the main loop for subscribing to data from the Geth state diff process
-func (sds *StateDiffStreamer) Stream(payloadChan chan interface{}) (*rpc.ClientSubscription, error) {
+func (sds *StateDiffStreamer) Stream(payloadChan chan interface{}) (shared.ClientSubscription, error) {
 	sds.PassedPayloadChan = payloadChan
 
 	go func() {

pkg/super_node/eth/mocks/test_data.go

@@ -237,7 +237,6 @@ var (
 		TotalDifficulty: big.NewInt(1337),
 		Block:           MockBlock,
 		Receipts:        MockReceipts,
-		HeaderRLP:       MockHeaderRlp,
 		TxMetaData:      MockTrxMeta,
 		ReceiptMetaData: MockRctMeta,
 		StorageNodes:    MockStorageNodes,

pkg/super_node/eth/publisher.go

@@ -20,26 +20,21 @@ import (
 	"errors"
 	"fmt"
 
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/rlp"
-
-	"github.com/vulcanize/eth-block-extractor/pkg/ipfs"
-	"github.com/vulcanize/eth-block-extractor/pkg/ipfs/eth_block_header"
-	"github.com/vulcanize/eth-block-extractor/pkg/ipfs/eth_block_receipts"
-	"github.com/vulcanize/eth-block-extractor/pkg/ipfs/eth_block_transactions"
-	"github.com/vulcanize/eth-block-extractor/pkg/ipfs/eth_state_trie"
-	"github.com/vulcanize/eth-block-extractor/pkg/ipfs/eth_storage_trie"
-	rlp2 "github.com/vulcanize/eth-block-extractor/pkg/wrappers/rlp"
+	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
+	"github.com/vulcanize/vulcanizedb/pkg/ipfs/dag_putters"
 )
 
 // IPLDPublisher satisfies the IPLDPublisher for ethereum
 type IPLDPublisher struct {
-	HeaderPutter      ipfs.DagPutter
-	TransactionPutter ipfs.DagPutter
-	ReceiptPutter     ipfs.DagPutter
-	StatePutter       ipfs.DagPutter
-	StoragePutter     ipfs.DagPutter
+	HeaderPutter      shared.DagPutter
+	TransactionPutter shared.DagPutter
+	ReceiptPutter     shared.DagPutter
+	StatePutter       shared.DagPutter
+	StoragePutter     shared.DagPutter
 }
 
 // NewIPLDPublisher creates a pointer to a new Publisher which satisfies the IPLDPublisher interface
@@ -49,11 +44,11 @@ func NewIPLDPublisher(ipfsPath string) (*IPLDPublisher, error) {
 		return nil, err
 	}
 	return &IPLDPublisher{
-		HeaderPutter:      eth_block_header.NewBlockHeaderDagPutter(node, rlp2.RlpDecoder{}),
-		TransactionPutter: eth_block_transactions.NewBlockTransactionsDagPutter(node),
-		ReceiptPutter:     eth_block_receipts.NewEthBlockReceiptDagPutter(node),
-		StatePutter:       eth_state_trie.NewStateTrieDagPutter(node),
-		StoragePutter:     eth_storage_trie.NewStorageTrieDagPutter(node),
+		HeaderPutter:      dag_putters.NewEthBlockHeaderDagPutter(node),
+		TransactionPutter: dag_putters.NewEthTxsDagPutter(node),
+		ReceiptPutter:     dag_putters.NewEthReceiptDagPutter(node),
+		StatePutter:       dag_putters.NewEthStateDagPutter(node),
+		StoragePutter:     dag_putters.NewEthStorageDagPutter(node),
 	}, nil
 }
 
@@ -64,7 +59,7 @@ func (pub *IPLDPublisher) Publish(payload interface{}) (interface{}, error) {
 		return nil, fmt.Errorf("eth publisher expected payload type %T got %T", &IPLDPayload{}, payload)
 	}
 	// Process and publish headers
-	headerCid, err := pub.publishHeader(ipldPayload.HeaderRLP)
+	headerCid, err := pub.publishHeader(ipldPayload.Block.Header())
 	if err != nil {
 		return nil, err
 	}
@@ -79,11 +74,7 @@ func (pub *IPLDPublisher) Publish(payload interface{}) (interface{}, error) {
 	// Process and publish uncles
 	uncleCids := make([]UncleModel, 0, len(ipldPayload.Block.Uncles()))
 	for _, uncle := range ipldPayload.Block.Uncles() {
-		uncleRlp, err := rlp.EncodeToBytes(uncle)
-		if err != nil {
-			return nil, err
-		}
-		uncleCid, err := pub.publishHeader(uncleRlp)
+		uncleCid, err := pub.publishHeader(uncle)
 		if err != nil {
 			return nil, err
 		}
@@ -95,7 +86,7 @@ func (pub *IPLDPublisher) Publish(payload interface{}) (interface{}, error) {
 	}
 
 	// Process and publish transactions
-	transactionCids, err := pub.publishTransactions(ipldPayload.Block.Body(), ipldPayload.TxMetaData)
+	transactionCids, err := pub.publishTransactions(ipldPayload.Block.Body().Transactions, ipldPayload.TxMetaData)
 	if err != nil {
 		return nil, err
 	}
@@ -129,19 +120,16 @@ func (pub *IPLDPublisher) Publish(payload interface{}) (interface{}, error) {
 	}, nil
 }
 
-func (pub *IPLDPublisher) publishHeader(headerRLP []byte) (string, error) {
-	headerCids, err := pub.HeaderPutter.DagPut(headerRLP)
+func (pub *IPLDPublisher) publishHeader(header *types.Header) (string, error) {
+	cids, err := pub.HeaderPutter.DagPut(header)
 	if err != nil {
 		return "", err
 	}
-	if len(headerCids) != 1 {
-		return "", errors.New("single CID expected to be returned for header")
-	}
-	return headerCids[0], nil
+	return cids[0], nil
 }
 
-func (pub *IPLDPublisher) publishTransactions(blockBody *types.Body, trxMeta []TxModel) ([]TxModel, error) {
-	transactionCids, err := pub.TransactionPutter.DagPut(blockBody)
+func (pub *IPLDPublisher) publishTransactions(transactions types.Transactions, trxMeta []TxModel) ([]TxModel, error) {
+	transactionCids, err := pub.TransactionPutter.DagPut(transactions)
 	if err != nil {
 		return nil, err
 	}
@@ -187,16 +175,13 @@ func (pub *IPLDPublisher) publishReceipts(receipts types.Receipts, receiptMeta [
 func (pub *IPLDPublisher) publishStateNodes(stateNodes []TrieNode) ([]StateNodeModel, error) {
 	stateNodeCids := make([]StateNodeModel, 0, len(stateNodes))
 	for _, node := range stateNodes {
-		stateNodeCid, err := pub.StatePutter.DagPut(node.Value)
+		cids, err := pub.StatePutter.DagPut(node.Value)
 		if err != nil {
 			return nil, err
 		}
-		if len(stateNodeCid) != 1 {
-			return nil, errors.New("single CID expected to be returned for state leaf")
-		}
 		stateNodeCids = append(stateNodeCids, StateNodeModel{
 			StateKey: node.Key.String(),
-			CID:      stateNodeCid[0],
+			CID:      cids[0],
 			Leaf:     node.Leaf,
 		})
 	}
@@ -208,17 +193,14 @@ func (pub *IPLDPublisher) publishStorageNodes(storageNodes map[common.Hash][]Tri
 	for addrKey, storageTrie := range storageNodes {
 		storageLeafCids[addrKey] = make([]StorageNodeModel, 0, len(storageTrie))
 		for _, node := range storageTrie {
-			storageNodeCid, err := pub.StoragePutter.DagPut(node.Value)
+			cids, err := pub.StoragePutter.DagPut(node.Value)
 			if err != nil {
 				return nil, err
 			}
-			if len(storageNodeCid) != 1 {
-				return nil, errors.New("single CID expected to be returned for storage leaf")
-			}
 			// Map storage node cids to their state key hashes
 			storageLeafCids[addrKey] = append(storageLeafCids[addrKey], StorageNodeModel{
 				StorageKey: node.Key.Hex(),
-				CID:        storageNodeCid[0],
+				CID:        cids[0],
 				Leaf:       node.Leaf,
 			})
 		}

pkg/super_node/eth/types.go

@@ -31,7 +31,6 @@ import (
 type IPLDPayload struct {
 	TotalDifficulty *big.Int
 	Block           *types.Block
-	HeaderRLP       []byte
 	TxMetaData      []TxModel
 	Receipts        types.Receipts
 	ReceiptMetaData []ReceiptModel

pkg/super_node/shared/intefaces.go

@@ -69,3 +69,8 @@ type ClientSubscription interface {
 	Err() <-chan error
 	Unsubscribe()
 }
+
+// DagPutter is a general interface for a dag putter
+type DagPutter interface {
+	DagPut(raw interface{}) ([]string, error)
+}

utils/utils.go

@@ -25,9 +25,9 @@ import (
 	"github.com/sirupsen/logrus"
 
 	"github.com/vulcanize/vulcanizedb/pkg/config"
+	"github.com/vulcanize/vulcanizedb/pkg/eth"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/eth"
 )
 
 func LoadPostgres(database config.Database, node core.Node) postgres.DB {