diff --git a/cmd/coldImport.go b/cmd/coldImport.go index a98ac0b4..4e8a8a44 100644 --- a/cmd/coldImport.go +++ b/cmd/coldImport.go @@ -20,12 +20,12 @@ import ( log "github.com/sirupsen/logrus" "github.com/spf13/cobra" - "github.com/vulcanize/vulcanizedb/pkg/eth/crypto" - "github.com/vulcanize/vulcanizedb/pkg/eth/datastore/ethereum" - "github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/eth/cold_import" "github.com/vulcanize/vulcanizedb/pkg/eth/converters/cold_db" vulcCommon "github.com/vulcanize/vulcanizedb/pkg/eth/converters/common" + "github.com/vulcanize/vulcanizedb/pkg/eth/crypto" + "github.com/vulcanize/vulcanizedb/pkg/eth/datastore/ethereum" + "github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/fs" "github.com/vulcanize/vulcanizedb/utils" ) diff --git a/cmd/headerSync.go b/cmd/headerSync.go index 22b384cc..e08e68d0 100644 --- a/cmd/headerSync.go +++ b/cmd/headerSync.go @@ -22,10 +22,10 @@ import ( log "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "github.com/vulcanize/vulcanizedb/pkg/eth" "github.com/vulcanize/vulcanizedb/pkg/eth/core" "github.com/vulcanize/vulcanizedb/pkg/eth/datastore" "github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres/repositories" - "github.com/vulcanize/vulcanizedb/pkg/eth" "github.com/vulcanize/vulcanizedb/pkg/eth/history" "github.com/vulcanize/vulcanizedb/utils" ) diff --git a/cmd/root.go b/cmd/root.go index 3e9dc2cc..1b02abd1 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -29,10 +29,10 @@ import ( "github.com/spf13/viper" "github.com/vulcanize/vulcanizedb/pkg/config" - "github.com/vulcanize/vulcanizedb/pkg/eth/core" "github.com/vulcanize/vulcanizedb/pkg/eth" "github.com/vulcanize/vulcanizedb/pkg/eth/client" vRpc "github.com/vulcanize/vulcanizedb/pkg/eth/converters/rpc" + "github.com/vulcanize/vulcanizedb/pkg/eth/core" "github.com/vulcanize/vulcanizedb/pkg/eth/node" ) diff --git a/cmd/streamEthSubscribe.go b/cmd/streamEthSubscribe.go index cbc8021a..f27b5b88 100644 --- a/cmd/streamEthSubscribe.go +++ b/cmd/streamEthSubscribe.go @@ -29,8 +29,8 @@ import ( "github.com/spf13/viper" "github.com/vulcanize/vulcanizedb/libraries/shared/streamer" - "github.com/vulcanize/vulcanizedb/pkg/eth/core" "github.com/vulcanize/vulcanizedb/pkg/eth/client" + "github.com/vulcanize/vulcanizedb/pkg/eth/core" "github.com/vulcanize/vulcanizedb/pkg/super_node" "github.com/vulcanize/vulcanizedb/pkg/super_node/config" "github.com/vulcanize/vulcanizedb/pkg/super_node/eth" diff --git a/integration_test/contract_test.go b/integration_test/contract_test.go index 5e423ada..d04c4662 100644 --- a/integration_test/contract_test.go +++ b/integration_test/contract_test.go @@ -25,10 +25,10 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/vulcanize/vulcanizedb/pkg/eth/core" "github.com/vulcanize/vulcanizedb/pkg/eth" "github.com/vulcanize/vulcanizedb/pkg/eth/client" rpc2 "github.com/vulcanize/vulcanizedb/pkg/eth/converters/rpc" + "github.com/vulcanize/vulcanizedb/pkg/eth/core" "github.com/vulcanize/vulcanizedb/pkg/eth/node" "github.com/vulcanize/vulcanizedb/pkg/eth/testing" "github.com/vulcanize/vulcanizedb/test_config" diff --git a/integration_test/geth_blockchain_test.go b/integration_test/geth_blockchain_test.go index a2bb6b6c..fb8521b9 100644 --- a/integration_test/geth_blockchain_test.go +++ b/integration_test/geth_blockchain_test.go @@ -23,13 +23,13 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/vulcanize/vulcanizedb/pkg/eth/core" "github.com/vulcanize/vulcanizedb/pkg/eth" "github.com/vulcanize/vulcanizedb/pkg/eth/client" rpc2 "github.com/vulcanize/vulcanizedb/pkg/eth/converters/rpc" - "github.com/vulcanize/vulcanizedb/pkg/eth/node" + "github.com/vulcanize/vulcanizedb/pkg/eth/core" "github.com/vulcanize/vulcanizedb/pkg/eth/fakes" "github.com/vulcanize/vulcanizedb/pkg/eth/history" + "github.com/vulcanize/vulcanizedb/pkg/eth/node" "github.com/vulcanize/vulcanizedb/test_config" ) diff --git a/integration_test/integration_test_suite_test.go b/integration_test/integration_test_suite_test.go index d0db85e5..7ea0a67f 100644 --- a/integration_test/integration_test_suite_test.go +++ b/integration_test/integration_test_suite_test.go @@ -17,10 +17,11 @@ package integration_test import ( - "github.com/sirupsen/logrus" "io/ioutil" "testing" + "github.com/sirupsen/logrus" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) diff --git a/pkg/eth/history/block_validator_test.go b/pkg/eth/history/block_validator_test.go index b82a58d1..8771455c 100644 --- a/pkg/eth/history/block_validator_test.go +++ b/pkg/eth/history/block_validator_test.go @@ -22,8 +22,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/vulcanize/vulcanizedb/pkg/eth/history" "github.com/vulcanize/vulcanizedb/pkg/eth/fakes" + "github.com/vulcanize/vulcanizedb/pkg/eth/history" ) var _ = Describe("Blocks validator", func() { diff --git a/pkg/eth/history/header_validator_test.go b/pkg/eth/history/header_validator_test.go index 2578c29b..32035b06 100644 --- a/pkg/eth/history/header_validator_test.go +++ b/pkg/eth/history/header_validator_test.go @@ -23,8 +23,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/vulcanize/vulcanizedb/pkg/eth/history" "github.com/vulcanize/vulcanizedb/pkg/eth/fakes" + "github.com/vulcanize/vulcanizedb/pkg/eth/history" ) var _ = Describe("Header validator", func() { diff --git a/pkg/eth/history/populate_blocks_test.go b/pkg/eth/history/populate_blocks_test.go index 5e00c3d7..aa0fe7de 100644 --- a/pkg/eth/history/populate_blocks_test.go +++ b/pkg/eth/history/populate_blocks_test.go @@ -21,8 +21,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/vulcanize/vulcanizedb/pkg/eth/history" "github.com/vulcanize/vulcanizedb/pkg/eth/fakes" + "github.com/vulcanize/vulcanizedb/pkg/eth/history" ) var _ = Describe("Populating blocks", func() { diff --git a/pkg/eth/history/populate_headers_test.go b/pkg/eth/history/populate_headers_test.go index c85fd2a7..018a45df 100644 --- a/pkg/eth/history/populate_headers_test.go +++ b/pkg/eth/history/populate_headers_test.go @@ -22,8 +22,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/vulcanize/vulcanizedb/pkg/eth/history" "github.com/vulcanize/vulcanizedb/pkg/eth/fakes" + "github.com/vulcanize/vulcanizedb/pkg/eth/history" ) var _ = Describe("Populating headers", func() { diff --git a/pkg/eth/history/validation_window_test.go b/pkg/eth/history/validation_window_test.go index 1a832d7e..baeaa491 100644 --- a/pkg/eth/history/validation_window_test.go +++ b/pkg/eth/history/validation_window_test.go @@ -22,8 +22,8 @@ import ( "math/big" - "github.com/vulcanize/vulcanizedb/pkg/eth/history" "github.com/vulcanize/vulcanizedb/pkg/eth/fakes" + "github.com/vulcanize/vulcanizedb/pkg/eth/history" ) var _ = Describe("Validation window", func() { diff --git a/pkg/eth/node/node_test.go b/pkg/eth/node/node_test.go index ac90d2a9..5afda719 100644 --- a/pkg/eth/node/node_test.go +++ b/pkg/eth/node/node_test.go @@ -23,8 +23,8 @@ import ( . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/eth/core" - "github.com/vulcanize/vulcanizedb/pkg/eth/node" "github.com/vulcanize/vulcanizedb/pkg/eth/fakes" + "github.com/vulcanize/vulcanizedb/pkg/eth/node" ) var EmpytHeaderHash = "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347" diff --git a/pkg/ipfs/builders.go b/pkg/ipfs/builders.go index 8baf22b3..d76bb16b 100644 --- a/pkg/ipfs/builders.go +++ b/pkg/ipfs/builders.go @@ -23,6 +23,7 @@ import ( "github.com/ipfs/go-ipfs/core" "github.com/ipfs/go-ipfs/plugin/loader" "github.com/ipfs/go-ipfs/repo/fsrepo" + ipld "github.com/ipfs/go-ipld-format" ) // InitIPFSPlugins is used to initialized IPFS plugins before creating a new IPFS node @@ -56,3 +57,29 @@ func InitIPFSBlockService(ipfsPath string) (blockservice.BlockService, error) { } return ipfsNode.Blocks, nil } + +type IPFS struct { + n *core.IpfsNode + ctx context.Context +} + +func (ipfs IPFS) Add(node ipld.Node) error { + return ipfs.n.DAG.Add(ipfs.n.Context(), node) +} + +func InitIPFSNode(repoPath string) (*IPFS, error) { + r, err := fsrepo.Open(repoPath) + if err != nil { + return nil, err + } + ctx := context.Background() + cfg := &core.BuildCfg{ + Online: false, + Repo: r, + } + ipfsNode, err := core.NewNode(ctx, cfg) + if err != nil { + return nil, err + } + return &IPFS{n: ipfsNode, ctx: ctx}, nil +} diff --git a/pkg/ipfs/dag_putters/btc_header.go b/pkg/ipfs/dag_putters/btc_header.go new file mode 100644 index 00000000..4687becb --- /dev/null +++ b/pkg/ipfs/dag_putters/btc_header.go @@ -0,0 +1,49 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package dag_putters + +import ( + "fmt" + + "github.com/btcsuite/btcd/wire" + + "github.com/vulcanize/vulcanizedb/pkg/ipfs" + "github.com/vulcanize/vulcanizedb/pkg/ipfs/ipld" +) + +type BtcHeaderDagPutter struct { + adder *ipfs.IPFS +} + +func NewBtcHeaderDagPutter(adder *ipfs.IPFS) *BtcHeaderDagPutter { + return &BtcHeaderDagPutter{adder: adder} +} + +func (bhdp *BtcHeaderDagPutter) DagPut(raw interface{}) ([]string, error) { + header, ok := raw.(*wire.BlockHeader) + if !ok { + return nil, fmt.Errorf("BtcHeaderDagPutter expected input type %T got %T", &wire.BlockHeader{}, raw) + } + node, err := ipld.NewBtcHeader(header) + if err != nil { + return nil, err + } + if err := bhdp.adder.Add(node); err != nil { + return nil, err + } + return []string{node.Cid().String()}, nil +} diff --git a/pkg/ipfs/dag_putters/btc_tx.go b/pkg/ipfs/dag_putters/btc_tx.go new file mode 100644 index 00000000..cab4088d --- /dev/null +++ b/pkg/ipfs/dag_putters/btc_tx.go @@ -0,0 +1,53 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package dag_putters + +import ( + "fmt" + + "github.com/btcsuite/btcd/wire" + + "github.com/vulcanize/vulcanizedb/pkg/ipfs" + "github.com/vulcanize/vulcanizedb/pkg/ipfs/ipld" +) + +type BtcTxDagPutter struct { + adder *ipfs.IPFS +} + +func NewBtcTxDagPutter(adder *ipfs.IPFS) *BtcTxDagPutter { + return &BtcTxDagPutter{adder: adder} +} + +func (etdp *BtcTxDagPutter) DagPut(raw interface{}) ([]string, error) { + transactions, ok := raw.([]*wire.MsgTx) + if !ok { + return nil, fmt.Errorf("BtcTxDagPutter expected input type %T got %T", []*wire.MsgTx{}, raw) + } + cids := make([]string, len(transactions)) + for i, transaction := range transactions { + node, err := ipld.NewBtcTx(transaction) + if err != nil { + return nil, err + } + if err := etdp.adder.Add(node); err != nil { + return nil, err + } + cids[i] = node.Cid().String() + } + return cids, nil +} diff --git a/pkg/ipfs/dag_putters/eth_header.go b/pkg/ipfs/dag_putters/eth_header.go new file mode 100644 index 00000000..b06fd086 --- /dev/null +++ b/pkg/ipfs/dag_putters/eth_header.go @@ -0,0 +1,49 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package dag_putters + +import ( + "fmt" + + "github.com/ethereum/go-ethereum/core/types" + + "github.com/vulcanize/vulcanizedb/pkg/ipfs" + "github.com/vulcanize/vulcanizedb/pkg/ipfs/ipld" +) + +type EthHeaderDagPutter struct { + adder *ipfs.IPFS +} + +func NewEthBlockHeaderDagPutter(adder *ipfs.IPFS) *EthHeaderDagPutter { + return &EthHeaderDagPutter{adder: adder} +} + +func (bhdp *EthHeaderDagPutter) DagPut(raw interface{}) ([]string, error) { + header, ok := raw.(*types.Header) + if !ok { + return nil, fmt.Errorf("EthHeaderDagPutter expected input type %T got %T", &types.Header{}, raw) + } + node, err := ipld.NewEthHeader(header) + if err != nil { + return nil, err + } + if err := bhdp.adder.Add(node); err != nil { + return nil, err + } + return []string{node.Cid().String()}, nil +} diff --git a/pkg/ipfs/dag_putters/eth_receipt.go b/pkg/ipfs/dag_putters/eth_receipt.go new file mode 100644 index 00000000..a153e36d --- /dev/null +++ b/pkg/ipfs/dag_putters/eth_receipt.go @@ -0,0 +1,53 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package dag_putters + +import ( + "fmt" + + "github.com/ethereum/go-ethereum/core/types" + + "github.com/vulcanize/vulcanizedb/pkg/ipfs" + "github.com/vulcanize/vulcanizedb/pkg/ipfs/ipld" +) + +type EthReceiptDagPutter struct { + adder *ipfs.IPFS +} + +func NewEthReceiptDagPutter(adder *ipfs.IPFS) *EthReceiptDagPutter { + return &EthReceiptDagPutter{adder: adder} +} + +func (erdp *EthReceiptDagPutter) DagPut(raw interface{}) ([]string, error) { + receipts, ok := raw.(types.Receipts) + if !ok { + return nil, fmt.Errorf("EthReceiptDagPutter expected input type %T got type %T", types.Receipts{}, raw) + } + cids := make([]string, len(receipts)) + for i, receipt := range receipts { + node, err := ipld.NewReceipt((*types.ReceiptForStorage)(receipt)) + if err != nil { + return nil, err + } + if err := erdp.adder.Add(node); err != nil { + return nil, err + } + cids[i] = node.Cid().String() + } + return cids, nil +} diff --git a/pkg/ipfs/dag_putters/eth_state.go b/pkg/ipfs/dag_putters/eth_state.go new file mode 100644 index 00000000..b031f66c --- /dev/null +++ b/pkg/ipfs/dag_putters/eth_state.go @@ -0,0 +1,47 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package dag_putters + +import ( + "fmt" + + "github.com/vulcanize/vulcanizedb/pkg/ipfs" + "github.com/vulcanize/vulcanizedb/pkg/ipfs/ipld" +) + +type EthStateDagPutter struct { + adder *ipfs.IPFS +} + +func NewEthStateDagPutter(adder *ipfs.IPFS) *EthStateDagPutter { + return &EthStateDagPutter{adder: adder} +} + +func (erdp *EthStateDagPutter) DagPut(raw interface{}) ([]string, error) { + stateNodeRLP, ok := raw.([]byte) + if !ok { + return nil, fmt.Errorf("EthStateDagPutter expected input type %T got %T", []byte{}, raw) + } + node, err := ipld.FromStateTrieRLP(stateNodeRLP) + if err != nil { + return nil, err + } + if err := erdp.adder.Add(node); err != nil { + return nil, err + } + return []string{node.Cid().String()}, nil +} diff --git a/pkg/ipfs/dag_putters/eth_storage.go b/pkg/ipfs/dag_putters/eth_storage.go new file mode 100644 index 00000000..f1c20c9b --- /dev/null +++ b/pkg/ipfs/dag_putters/eth_storage.go @@ -0,0 +1,47 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package dag_putters + +import ( + "fmt" + + "github.com/vulcanize/vulcanizedb/pkg/ipfs" + "github.com/vulcanize/vulcanizedb/pkg/ipfs/ipld" +) + +type EthStorageDagPutter struct { + adder *ipfs.IPFS +} + +func NewEthStorageDagPutter(adder *ipfs.IPFS) *EthStorageDagPutter { + return &EthStorageDagPutter{adder: adder} +} + +func (erdp *EthStorageDagPutter) DagPut(raw interface{}) ([]string, error) { + storageNodeRLP, ok := raw.([]byte) + if !ok { + return nil, fmt.Errorf("EthStorageDagPutter expected input type %T got %T", []byte{}, raw) + } + node, err := ipld.FromStorageTrieRLP(storageNodeRLP) + if err != nil { + return nil, err + } + if err := erdp.adder.Add(node); err != nil { + return nil, err + } + return []string{node.Cid().String()}, nil +} diff --git a/pkg/ipfs/dag_putters/eth_tx.go b/pkg/ipfs/dag_putters/eth_tx.go new file mode 100644 index 00000000..b4339ac0 --- /dev/null +++ b/pkg/ipfs/dag_putters/eth_tx.go @@ -0,0 +1,53 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package dag_putters + +import ( + "fmt" + + "github.com/ethereum/go-ethereum/core/types" + + "github.com/vulcanize/vulcanizedb/pkg/ipfs" + "github.com/vulcanize/vulcanizedb/pkg/ipfs/ipld" +) + +type EthTxsDagPutter struct { + adder *ipfs.IPFS +} + +func NewEthTxsDagPutter(adder *ipfs.IPFS) *EthTxsDagPutter { + return &EthTxsDagPutter{adder: adder} +} + +func (etdp *EthTxsDagPutter) DagPut(raw interface{}) ([]string, error) { + transactions, ok := raw.(types.Transactions) + if !ok { + return nil, fmt.Errorf("EthTxsDagPutter expected input type %T got %T", types.Transactions{}, raw) + } + cids := make([]string, len(transactions)) + for i, transaction := range transactions { + node, err := ipld.NewEthTx(transaction) + if err != nil { + return nil, err + } + if err := etdp.adder.Add(node); err != nil { + return nil, err + } + cids[i] = node.Cid().String() + } + return cids, nil +} diff --git a/pkg/ipfs/ipld/btc_header.go b/pkg/ipfs/ipld/btc_header.go new file mode 100644 index 00000000..4a09d4ab --- /dev/null +++ b/pkg/ipfs/ipld/btc_header.go @@ -0,0 +1,202 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ipld + +import ( + "bytes" + "encoding/hex" + "fmt" + + "github.com/btcsuite/btcd/wire" + "github.com/ipfs/go-cid" + node "github.com/ipfs/go-ipld-format" + mh "github.com/multiformats/go-multihash" +) + +type BtcHeader struct { + *wire.BlockHeader + + rawdata []byte + cid cid.Cid +} + +// Static (compile time) check that BtcBtcHeader satisfies the node.Node interface. +var _ node.Node = (*BtcHeader)(nil) + +/* + INPUT +*/ + +// NewBtcHeader converts a *wire.Header into an BtcHeader IPLD node +func NewBtcHeader(header *wire.BlockHeader) (*BtcHeader, error) { + w := bytes.NewBuffer(make([]byte, 0, 80)) + if err := header.Serialize(w); err != nil { + return nil, err + } + rawdata := w.Bytes() + c, err := rawdataToCid(MBitcoinHeader, rawdata, mh.DBL_SHA2_256) + if err != nil { + return nil, err + } + return &BtcHeader{ + BlockHeader: header, + cid: c, + rawdata: rawdata, + }, nil +} + +/* + OUTPUT +*/ + +// DecodeBtcHeader takes a cid and its raw binary data +// from IPFS and returns an BtcHeader object for further processing. +func DecodeBtcHeader(c cid.Cid, b []byte) (*BtcHeader, error) { + var h *wire.BlockHeader + w := bytes.NewBuffer(b) + if err := h.Deserialize(w); err != nil { + return nil, err + } + return &BtcHeader{ + BlockHeader: h, + cid: c, + rawdata: b, + }, nil +} + +/* + Block INTERFACE +*/ + +func (b *BtcHeader) Cid() cid.Cid { + return b.cid +} + +func (b *BtcHeader) RawData() []byte { + return b.rawdata +} + +func (b *BtcHeader) String() string { + return fmt.Sprintf("", b.cid) +} + +func (b *BtcHeader) Loggable() map[string]interface{} { + // TODO: more helpful info here + return map[string]interface{}{ + "type": "bitcoin_block", + } +} + +/* + Node INTERFACE +*/ + +func (b *BtcHeader) Links() []*node.Link { + return []*node.Link{ + { + Name: "tx", + Cid: sha256ToCid(MBitcoinTx, b.MerkleRoot.CloneBytes()), + }, + { + Name: "parent", + Cid: sha256ToCid(MBitcoinHeader, b.PrevBlock.CloneBytes()), + }, + } +} + +// Resolve attempts to traverse a path through this block. +func (b *BtcHeader) Resolve(path []string) (interface{}, []string, error) { + if len(path) == 0 { + return nil, nil, fmt.Errorf("zero length path") + } + switch path[0] { + case "version": + return b.Version, path[1:], nil + case "timestamp": + return b.Timestamp, path[1:], nil + case "bits": + return b.Bits, path[1:], nil + case "nonce": + return b.Nonce, path[1:], nil + case "parent": + return &node.Link{Cid: sha256ToCid(MBitcoinHeader, b.PrevBlock.CloneBytes())}, path[1:], nil + case "tx": + return &node.Link{Cid: sha256ToCid(MBitcoinTx, b.MerkleRoot.CloneBytes())}, path[1:], nil + default: + return nil, nil, fmt.Errorf("no such link") + } +} + +// ResolveLink is a helper function that allows easier traversal of links through blocks +func (b *BtcHeader) ResolveLink(path []string) (*node.Link, []string, error) { + out, rest, err := b.Resolve(path) + if err != nil { + return nil, nil, err + } + + lnk, ok := out.(*node.Link) + if !ok { + return nil, nil, fmt.Errorf("object at path was not a link") + } + + return lnk, rest, nil +} + +func cidToHash(c cid.Cid) []byte { + h := []byte(c.Hash()) + return h[len(h)-32:] +} + +func hashToCid(hv []byte, t uint64) cid.Cid { + h, _ := mh.Encode(hv, mh.DBL_SHA2_256) + return cid.NewCidV1(t, h) +} + +func (b *BtcHeader) Size() (uint64, error) { + return uint64(len(b.rawdata)), nil +} + +func (b *BtcHeader) Stat() (*node.NodeStat, error) { + return &node.NodeStat{}, nil +} + +func (b *BtcHeader) Tree(p string, depth int) []string { + // TODO: this isnt a correct implementation yet + return []string{"difficulty", "nonce", "version", "timestamp", "tx", "parent"} +} + +func (b *BtcHeader) BTCSha() []byte { + blkmh, _ := mh.Sum(b.rawdata, mh.DBL_SHA2_256, -1) + return blkmh[2:] +} + +func (b *BtcHeader) HexHash() string { + return hex.EncodeToString(revString(b.BTCSha())) +} + +func (b *BtcHeader) Copy() node.Node { + nb := *b // cheating shallow copy + return &nb +} + +func revString(s []byte) []byte { + b := make([]byte, len(s)) + for i, v := range []byte(s) { + b[len(b)-(i+1)] = v + } + return b +} diff --git a/pkg/ipfs/ipld/btc_tx.go b/pkg/ipfs/ipld/btc_tx.go new file mode 100644 index 00000000..e92b29c1 --- /dev/null +++ b/pkg/ipfs/ipld/btc_tx.go @@ -0,0 +1,261 @@ +package ipld + +import ( + "bytes" + "encoding/hex" + "fmt" + "strconv" + + "github.com/btcsuite/btcd/wire" + cid "github.com/ipfs/go-cid" + node "github.com/ipfs/go-ipld-format" + mh "github.com/multiformats/go-multihash" +) + +type BtcTx struct { + *wire.MsgTx + + rawdata []byte + cid cid.Cid +} + +// Static (compile time) check that BtcBtcHeader satisfies the node.Node interface. +var _ node.Node = (*BtcTx)(nil) + +/* + INPUT +*/ + +// NewBtcTx converts a *wire.MsgTx into an BtcTx IPLD node +func NewBtcTx(tx *wire.MsgTx) (*BtcTx, error) { + w := bytes.NewBuffer(make([]byte, 0, tx.SerializeSize())) + if err := tx.Serialize(w); err != nil { + return nil, err + } + rawdata := w.Bytes() + c, err := rawdataToCid(MBitcoinTx, rawdata, mh.DBL_SHA2_256) + if err != nil { + return nil, err + } + return &BtcTx{ + MsgTx: tx, + cid: c, + rawdata: rawdata, + }, nil +} + +/* + OUTPUT +*/ + +// DecodeBtcTx takes a cid and its raw binary data +// from IPFS and returns an BtcTx object for further processing. +func DecodeBtcTx(c cid.Cid, b []byte) (*BtcTx, error) { + var tx *wire.MsgTx + w := bytes.NewBuffer(b) + if err := tx.Deserialize(w); err != nil { + return nil, err + } + return &BtcTx{ + MsgTx: tx, + cid: c, + rawdata: b, + }, nil +} + +/* + Block INTERFACE +*/ + +func (t *BtcTx) Cid() cid.Cid { + return t.cid +} + +func (t *BtcTx) RawData() []byte { + return t.rawdata +} + +func (t *BtcTx) String() string { + return fmt.Sprintf("", t.cid) +} + +func (t *BtcTx) Loggable() map[string]interface{} { + return map[string]interface{}{ + "type": "bitcoinTx", + } +} + +/* + Node INTERFACE +*/ + +func (t *BtcTx) Links() []*node.Link { + var out []*node.Link + for i, in := range t.MsgTx.TxIn { + lnk := &node.Link{Cid: sha256ToCid(MBitcoinTx, in.PreviousOutPoint.Hash.CloneBytes())} + lnk.Name = fmt.Sprintf("inputs/%d/prevTx", i) + out = append(out, lnk) + } + return out +} + +func (t *BtcTx) Resolve(path []string) (interface{}, []string, error) { + switch path[0] { + case "version": + return t.Version, path[1:], nil + case "lockTime": + return t.LockTime, path[1:], nil + case "inputs": + if len(path) == 1 { + return t.MsgTx.TxIn, nil, nil + } + + index, err := strconv.Atoi(path[1]) + if err != nil { + return nil, nil, err + } + + if index >= len(t.MsgTx.TxIn) || index < 0 { + return nil, nil, fmt.Errorf("index out of range") + } + + inp := t.MsgTx.TxIn[index] + if len(path) == 2 { + return inp, nil, nil + } + + switch path[2] { + case "prevTx": + return &node.Link{Cid: sha256ToCid(MBitcoinTx, inp.PreviousOutPoint.Hash.CloneBytes())}, path[3:], nil + case "seqNo": + return inp.Sequence, path[3:], nil + case "script": + return inp.SignatureScript, path[3:], nil + default: + return nil, nil, fmt.Errorf("no such link") + } + case "outputs": + if len(path) == 1 { + return t.TxOut, nil, nil + } + + index, err := strconv.Atoi(path[1]) + if err != nil { + return nil, nil, err + } + + if index >= len(t.TxOut) || index < 0 { + return nil, nil, fmt.Errorf("index out of range") + } + + outp := t.TxOut[index] + if len(path) == 2 { + return outp, path[2:], nil + } + + switch path[2] { + case "value": + return outp.Value, path[3:], nil + case "script": + /* + if outp.Script[0] == 0x6a { // OP_RETURN + c, err := cid.Decode(string(outp.Script[1:])) + if err == nil { + return &node.Link{Cid: c}, path[3:], nil + } + } + */ + return outp.PkScript, path[3:], nil + default: + return nil, nil, fmt.Errorf("no such link") + } + default: + return nil, nil, fmt.Errorf("no such link") + } +} + +func (t *BtcTx) ResolveLink(path []string) (*node.Link, []string, error) { + i, rest, err := t.Resolve(path) + if err != nil { + return nil, rest, err + } + + lnk, ok := i.(*node.Link) + if !ok { + return nil, nil, fmt.Errorf("value was not a link") + } + + return lnk, rest, nil +} + +func (t *BtcTx) Size() (uint64, error) { + return uint64(len(t.RawData())), nil +} + +func (t *BtcTx) Stat() (*node.NodeStat, error) { + return &node.NodeStat{}, nil +} + +func (t *BtcTx) Copy() node.Node { + nt := *t // cheating shallow copy + return &nt +} + +func (t *BtcTx) Tree(p string, depth int) []string { + if depth == 0 { + return nil + } + + switch p { + case "inputs": + return t.treeInputs(nil, depth+1) + case "outputs": + return t.treeOutputs(nil, depth+1) + case "": + out := []string{"version", "timeLock", "inputs", "outputs"} + out = t.treeInputs(out, depth) + out = t.treeOutputs(out, depth) + return out + default: + return nil + } +} + +func (t *BtcTx) treeInputs(out []string, depth int) []string { + if depth < 2 { + return out + } + + for i := range t.TxIn { + inp := "inputs/" + fmt.Sprint(i) + out = append(out, inp) + if depth > 2 { + out = append(out, inp+"/prevTx", inp+"/seqNo", inp+"/script") + } + } + return out +} + +func (t *BtcTx) treeOutputs(out []string, depth int) []string { + if depth < 2 { + return out + } + + for i := range t.TxOut { + o := "outputs/" + fmt.Sprint(i) + out = append(out, o) + if depth > 2 { + out = append(out, o+"/script", o+"/value") + } + } + return out +} + +func (t *BtcTx) BTCSha() []byte { + mh, _ := mh.Sum(t.RawData(), mh.DBL_SHA2_256, -1) + return []byte(mh[2:]) +} + +func (t *BtcTx) HexHash() string { + return hex.EncodeToString(revString(t.BTCSha())) +} diff --git a/pkg/ipfs/ipld/eth_account.go b/pkg/ipfs/ipld/eth_account.go new file mode 100644 index 00000000..5d80af1d --- /dev/null +++ b/pkg/ipfs/ipld/eth_account.go @@ -0,0 +1,175 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ipld + +import ( + "encoding/json" + "fmt" + "math/big" + + "github.com/ipfs/go-cid" + node "github.com/ipfs/go-ipld-format" +) + +// EthAccountSnapshot (eth-account-snapshot codec 0x97) +// represents an ethereum account, i.e. a wallet address or +// a smart contract +type EthAccountSnapshot struct { + *EthAccount + + cid cid.Cid + rawdata []byte +} + +// EthAccount is the building block of EthAccountSnapshot. +// Or, is the former stripped of its cid and rawdata components. +type EthAccount struct { + Nonce uint64 + Balance *big.Int + Root []byte // This is the storage root trie + CodeHash []byte // This is the hash of the EVM code +} + +// Static (compile time) check that EthAccountSnapshot satisfies the +// node.Node interface. +var _ node.Node = (*EthAccountSnapshot)(nil) + +/* + INPUT +*/ + +// Input should be managed by EthStateTrie + +/* + OUTPUT +*/ + +// Output should be managed by EthStateTrie + +/* + Block INTERFACE +*/ + +// RawData returns the binary of the RLP encode of the account snapshot. +func (as *EthAccountSnapshot) RawData() []byte { + return as.rawdata +} + +// Cid returns the cid of the transaction. +func (as *EthAccountSnapshot) Cid() cid.Cid { + return as.cid +} + +// String is a helper for output +func (as *EthAccountSnapshot) String() string { + return fmt.Sprintf("", as.cid) +} + +// Loggable returns in a map the type of IPLD Link. +func (as *EthAccountSnapshot) Loggable() map[string]interface{} { + return map[string]interface{}{ + "type": "eth-account-snapshot", + } +} + +/* + Node INTERFACE +*/ + +// Resolve resolves a path through this node, stopping at any link boundary +// and returning the object found as well as the remaining path to traverse +func (as *EthAccountSnapshot) Resolve(p []string) (interface{}, []string, error) { + if len(p) == 0 { + return as, nil, nil + } + + if len(p) > 1 { + return nil, nil, fmt.Errorf("unexpected path elements past %s", p[0]) + } + + switch p[0] { + case "balance": + return as.Balance, nil, nil + case "codeHash": + return &node.Link{Cid: keccak256ToCid(RawBinary, as.CodeHash)}, nil, nil + case "nonce": + return as.Nonce, nil, nil + case "root": + return &node.Link{Cid: keccak256ToCid(MEthStorageTrie, as.Root)}, nil, nil + default: + return nil, nil, fmt.Errorf("no such link") + } +} + +// Tree lists all paths within the object under 'path', and up to the given depth. +// To list the entire object (similar to `find .`) pass "" and -1 +func (as *EthAccountSnapshot) Tree(p string, depth int) []string { + if p != "" || depth == 0 { + return nil + } + return []string{"balance", "codeHash", "nonce", "root"} +} + +// ResolveLink is a helper function that calls resolve and asserts the +// output is a link +func (as *EthAccountSnapshot) ResolveLink(p []string) (*node.Link, []string, error) { + obj, rest, err := as.Resolve(p) + if err != nil { + return nil, nil, err + } + + if lnk, ok := obj.(*node.Link); ok { + return lnk, rest, nil + } + + return nil, nil, fmt.Errorf("resolved item was not a link") +} + +// Copy will go away. It is here to comply with the interface. +func (as *EthAccountSnapshot) Copy() node.Node { + panic("dont use this yet") +} + +// Links is a helper function that returns all links within this object +func (as *EthAccountSnapshot) Links() []*node.Link { + return nil +} + +// Stat will go away. It is here to comply with the interface. +func (as *EthAccountSnapshot) Stat() (*node.NodeStat, error) { + return &node.NodeStat{}, nil +} + +// Size will go away. It is here to comply with the interface. +func (as *EthAccountSnapshot) Size() (uint64, error) { + return 0, nil +} + +/* + EthAccountSnapshot functions +*/ + +// MarshalJSON processes the transaction into readable JSON format. +func (as *EthAccountSnapshot) MarshalJSON() ([]byte, error) { + out := map[string]interface{}{ + "balance": as.Balance, + "codeHash": keccak256ToCid(RawBinary, as.CodeHash), + "nonce": as.Nonce, + "root": keccak256ToCid(MEthStorageTrie, as.Root), + } + return json.Marshal(out) +} diff --git a/pkg/ipfs/ipld/eth_header.go b/pkg/ipfs/ipld/eth_header.go new file mode 100644 index 00000000..7ae89c28 --- /dev/null +++ b/pkg/ipfs/ipld/eth_header.go @@ -0,0 +1,292 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ipld + +import ( + "encoding/json" + "fmt" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ipfs/go-cid" + node "github.com/ipfs/go-ipld-format" + mh "github.com/multiformats/go-multihash" +) + +// EthHeader (eth-block, codec 0x90), represents an ethereum block header +type EthHeader struct { + *types.Header + + cid cid.Cid + rawdata []byte +} + +// Static (compile time) check that EthHeader satisfies the node.Node interface. +var _ node.Node = (*EthHeader)(nil) + +/* + INPUT +*/ + +// NewEthHeader converts a *types.Header into an EthHeader IPLD node +func NewEthHeader(header *types.Header) (*EthHeader, error) { + headerRLP, err := rlp.EncodeToBytes(header) + if err != nil { + return nil, err + } + c, err := rawdataToCid(MEthHeader, headerRLP, mh.KECCAK_256) + if err != nil { + return nil, err + } + return &EthHeader{ + Header: header, + cid: c, + rawdata: headerRLP, + }, nil +} + +/* + OUTPUT +*/ + +// DecodeEthHeader takes a cid and its raw binary data +// from IPFS and returns an EthHeader object for further processing. +func DecodeEthHeader(c cid.Cid, b []byte) (*EthHeader, error) { + var h *types.Header + if err := rlp.DecodeBytes(b, h); err != nil { + return nil, err + } + return &EthHeader{ + Header: h, + cid: c, + rawdata: b, + }, nil +} + +/* + Block INTERFACE +*/ + +// RawData returns the binary of the RLP encode of the block header. +func (b *EthHeader) RawData() []byte { + return b.rawdata +} + +// Cid returns the cid of the block header. +func (b *EthHeader) Cid() cid.Cid { + return b.cid +} + +// String is a helper for output +func (b *EthHeader) String() string { + return fmt.Sprintf("", b.cid) +} + +// Loggable returns a map the type of IPLD Link. +func (b *EthHeader) Loggable() map[string]interface{} { + return map[string]interface{}{ + "type": "eth-block", + } +} + +/* + Node INTERFACE +*/ + +// Resolve resolves a path through this node, stopping at any link boundary +// and returning the object found as well as the remaining path to traverse +func (b *EthHeader) Resolve(p []string) (interface{}, []string, error) { + if len(p) == 0 { + return b, nil, nil + } + + first, rest := p[0], p[1:] + + switch first { + case "parent": + return &node.Link{Cid: commonHashToCid(MEthHeader, b.ParentHash)}, rest, nil + case "receipts": + return &node.Link{Cid: commonHashToCid(MEthTxReceiptTrie, b.ReceiptHash)}, rest, nil + case "root": + return &node.Link{Cid: commonHashToCid(MEthStateTrie, b.Root)}, rest, nil + case "tx": + return &node.Link{Cid: commonHashToCid(MEthTxTrie, b.TxHash)}, rest, nil + case "uncles": + return &node.Link{Cid: commonHashToCid(MEthHeaderList, b.UncleHash)}, rest, nil + } + + if len(p) != 1 { + return nil, nil, fmt.Errorf("unexpected path elements past %s", first) + } + + switch first { + case "bloom": + return b.Bloom, nil, nil + case "coinbase": + return b.Coinbase, nil, nil + case "difficulty": + return b.Difficulty, nil, nil + case "extra": + // This is a []byte. By default they are marshalled into Base64. + return fmt.Sprintf("0x%x", b.Extra), nil, nil + case "gaslimit": + return b.GasLimit, nil, nil + case "gasused": + return b.GasUsed, nil, nil + case "mixdigest": + return b.MixDigest, nil, nil + case "nonce": + return b.Nonce, nil, nil + case "number": + return b.Number, nil, nil + case "time": + return b.Time, nil, nil + default: + return nil, nil, fmt.Errorf("no such link") + } +} + +// Tree lists all paths within the object under 'path', and up to the given depth. +// To list the entire object (similar to `find .`) pass "" and -1 +func (b *EthHeader) Tree(p string, depth int) []string { + if p != "" || depth == 0 { + return nil + } + + return []string{ + "time", + "bloom", + "coinbase", + "difficulty", + "extra", + "gaslimit", + "gasused", + "mixdigest", + "nonce", + "number", + "parent", + "receipts", + "root", + "tx", + "uncles", + } +} + +// ResolveLink is a helper function that allows easier traversal of links through blocks +func (b *EthHeader) ResolveLink(p []string) (*node.Link, []string, error) { + obj, rest, err := b.Resolve(p) + if err != nil { + return nil, nil, err + } + + if lnk, ok := obj.(*node.Link); ok { + return lnk, rest, nil + } + + return nil, nil, fmt.Errorf("resolved item was not a link") +} + +// Copy will go away. It is here to comply with the Node interface. +func (b *EthHeader) Copy() node.Node { + panic("implement me") +} + +// Links is a helper function that returns all links within this object +// HINT: Use `ipfs refs ` +func (b *EthHeader) Links() []*node.Link { + return []*node.Link{ + {Cid: commonHashToCid(MEthHeader, b.ParentHash)}, + {Cid: commonHashToCid(MEthTxReceiptTrie, b.ReceiptHash)}, + {Cid: commonHashToCid(MEthStateTrie, b.Root)}, + {Cid: commonHashToCid(MEthTxTrie, b.TxHash)}, + {Cid: commonHashToCid(MEthHeaderList, b.UncleHash)}, + } +} + +// Stat will go away. It is here to comply with the Node interface. +func (b *EthHeader) Stat() (*node.NodeStat, error) { + return &node.NodeStat{}, nil +} + +// Size will go away. It is here to comply with the Node interface. +func (b *EthHeader) Size() (uint64, error) { + return 0, nil +} + +/* + EthHeader functions +*/ + +// MarshalJSON processes the block header into readable JSON format, +// converting the right links into their cids, and keeping the original +// hex hash, allowing the user to simplify external queries. +func (b *EthHeader) MarshalJSON() ([]byte, error) { + out := map[string]interface{}{ + "time": b.Time, + "bloom": b.Bloom, + "coinbase": b.Coinbase, + "difficulty": b.Difficulty, + "extra": fmt.Sprintf("0x%x", b.Extra), + "gaslimit": b.GasLimit, + "gasused": b.GasUsed, + "mixdigest": b.MixDigest, + "nonce": b.Nonce, + "number": b.Number, + "parent": commonHashToCid(MEthHeader, b.ParentHash), + "receipts": commonHashToCid(MEthTxReceiptTrie, b.ReceiptHash), + "root": commonHashToCid(MEthStateTrie, b.Root), + "tx": commonHashToCid(MEthTxTrie, b.TxHash), + "uncles": commonHashToCid(MEthHeaderList, b.UncleHash), + } + return json.Marshal(out) +} + +// objJSONBlock defines the output of the JSON RPC API for either +// "eth_BlockByHash" or "eth_BlockByHeader". +type objJSONBlock struct { + Result objJSONBlockResult `json:"result"` +} + +// objJSONBLockResult is the nested struct that takes +// the contents of the JSON field "result". +type objJSONBlockResult struct { + types.Header // Use its fields and unmarshaler + *objJSONBlockResultExt // Add these fields to the parsing +} + +// objJSONBLockResultExt facilitates the composition +// of the field "result", adding to the +// `types.Header` fields, both ommers (their hashes) and transactions. +type objJSONBlockResultExt struct { + OmmerHashes []common.Hash `json:"uncles"` + Transactions []*types.Transaction `json:"transactions"` +} + +// UnmarshalJSON overrides the function types.Header.UnmarshalJSON, allowing us +// to parse the fields of Header, plus ommer hashes and transactions. +// (yes, ommer hashes. You will need to "eth_getUncleCountByBlockHash" per each ommer) +func (o *objJSONBlockResult) UnmarshalJSON(input []byte) error { + err := o.Header.UnmarshalJSON(input) + if err != nil { + return err + } + + o.objJSONBlockResultExt = &objJSONBlockResultExt{} + err = json.Unmarshal(input, o.objJSONBlockResultExt) + return err +} diff --git a/pkg/ipfs/ipld/eth_receipt.go b/pkg/ipfs/ipld/eth_receipt.go new file mode 100644 index 00000000..bbceb09f --- /dev/null +++ b/pkg/ipfs/ipld/eth_receipt.go @@ -0,0 +1,199 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ipld + +import ( + "encoding/json" + "fmt" + "strconv" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ipfs/go-cid" + node "github.com/ipfs/go-ipld-format" + mh "github.com/multiformats/go-multihash" +) + +type EthReceipt struct { + *types.ReceiptForStorage + + rawdata []byte + cid cid.Cid +} + +// Static (compile time) check that EthReceipt satisfies the node.Node interface. +var _ node.Node = (*EthReceipt)(nil) + +/* + INPUT +*/ + +// NewReceipt converts a types.ReceiptForStorage to an EthReceipt IPLD node +func NewReceipt(receipt *types.ReceiptForStorage) (*EthReceipt, error) { + receiptRLP, err := rlp.EncodeToBytes(receipt) + if err != nil { + return nil, err + } + c, err := rawdataToCid(MEthTxReceipt, receiptRLP, mh.KECCAK_256) + if err != nil { + return nil, err + } + return &EthReceipt{ + ReceiptForStorage: receipt, + cid: c, + rawdata: receiptRLP, + }, nil +} + +/* + OUTPUT +*/ + +// DecodeEthReceipt takes a cid and its raw binary data +// from IPFS and returns an EthReceipt object for further processing. +func DecodeEthReceipt(c cid.Cid, b []byte) (*EthReceipt, error) { + var r *types.ReceiptForStorage + if err := rlp.DecodeBytes(b, r); err != nil { + return nil, err + } + return &EthReceipt{ + ReceiptForStorage: r, + cid: c, + rawdata: b, + }, nil +} + +/* + Block INTERFACE +*/ + +func (node *EthReceipt) RawData() []byte { + return node.rawdata +} + +func (node *EthReceipt) Cid() cid.Cid { + return node.cid +} + +// String is a helper for output +func (r *EthReceipt) String() string { + return fmt.Sprintf("", r.cid) +} + +// Loggable returns in a map the type of IPLD Link. +func (r *EthReceipt) Loggable() map[string]interface{} { + return map[string]interface{}{ + "type": "eth-receipt", + } +} + +// Resolve resolves a path through this node, stopping at any link boundary +// and returning the object found as well as the remaining path to traverse +func (r *EthReceipt) Resolve(p []string) (interface{}, []string, error) { + if len(p) == 0 { + return r, nil, nil + } + + if len(p) > 1 { + return nil, nil, fmt.Errorf("unexpected path elements past %s", p[0]) + } + + switch p[0] { + + case "root": + return r.PostState, nil, nil + case "status": + return r.Status, nil, nil + case "cumulativeGasUsed": + return r.CumulativeGasUsed, nil, nil + case "logsBloom": + return r.Bloom, nil, nil + case "logs": + return r.Logs, nil, nil + case "transactionHash": + return r.TxHash, nil, nil + case "contractAddress": + return r.ContractAddress, nil, nil + case "gasUsed": + return r.GasUsed, nil, nil + default: + return nil, nil, fmt.Errorf("no such link") + } +} + +// Tree lists all paths within the object under 'path', and up to the given depth. +// To list the entire object (similar to `find .`) pass "" and -1 +func (r *EthReceipt) Tree(p string, depth int) []string { + if p != "" || depth == 0 { + return nil + } + return []string{"root", "status", "cumulativeGasUsed", "logsBloom", "logs", "transactionHash", "contractAddress", "gasUsed"} +} + +// ResolveLink is a helper function that calls resolve and asserts the +// output is a link +func (r *EthReceipt) ResolveLink(p []string) (*node.Link, []string, error) { + obj, rest, err := r.Resolve(p) + if err != nil { + return nil, nil, err + } + + if lnk, ok := obj.(*node.Link); ok { + return lnk, rest, nil + } + + return nil, nil, fmt.Errorf("resolved item was not a link") +} + +// Copy will go away. It is here to comply with the Node interface. +func (*EthReceipt) Copy() node.Node { + panic("implement me") +} + +// Links is a helper function that returns all links within this object +func (*EthReceipt) Links() []*node.Link { + return nil +} + +// Stat will go away. It is here to comply with the interface. +func (r *EthReceipt) Stat() (*node.NodeStat, error) { + return &node.NodeStat{}, nil +} + +// Size will go away. It is here to comply with the interface. +func (r *EthReceipt) Size() (uint64, error) { + return strconv.ParseUint((*types.Receipt)(r.ReceiptForStorage).Size().String(), 10, 64) +} + +/* + EthReceipt functions +*/ + +// MarshalJSON processes the receipt into readable JSON format. +func (r *EthReceipt) MarshalJSON() ([]byte, error) { + out := map[string]interface{}{ + "root": r.PostState, + "status": r.Status, + "cumulativeGasUsed": r.CumulativeGasUsed, + "logsBloom": r.Bloom, + "logs": r.Logs, + "transactionHash": r.TxHash, + "contractAddress": r.ContractAddress, + "gasUsed": r.GasUsed, + } + return json.Marshal(out) +} diff --git a/pkg/ipfs/ipld/eth_state.go b/pkg/ipfs/ipld/eth_state.go new file mode 100644 index 00000000..bcc9cb34 --- /dev/null +++ b/pkg/ipfs/ipld/eth_state.go @@ -0,0 +1,109 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ipld + +import ( + "fmt" + + "github.com/ethereum/go-ethereum/rlp" + "github.com/ipfs/go-cid" + node "github.com/ipfs/go-ipld-format" + mh "github.com/multiformats/go-multihash" +) + +// EthStateTrie (eth-state-trie, codec 0x96), represents +// a node from the state trie in ethereum. +type EthStateTrie struct { + *TrieNode +} + +// Static (compile time) check that EthStateTrie satisfies the node.Node interface. +var _ node.Node = (*EthStateTrie)(nil) + +/* + INPUT +*/ + +// FromStateTrieRLP takes the RLP bytes of an ethereum +// state trie node to return it as an IPLD node for further processing. +func FromStateTrieRLP(stateNodeRLP []byte) (*EthStateTrie, error) { + c, err := rawdataToCid(MEthStateTrie, stateNodeRLP, mh.KECCAK_256) + if err != nil { + return nil, err + } + return DecodeEthStateTrie(c, stateNodeRLP) +} + +/* + OUTPUT +*/ + +// DecodeEthStateTrie returns an EthStateTrie object from its cid and rawdata. +func DecodeEthStateTrie(c cid.Cid, b []byte) (*EthStateTrie, error) { + tn, err := decodeTrieNode(c, b, decodeEthStateTrieLeaf) + if err != nil { + return nil, err + } + return &EthStateTrie{TrieNode: tn}, nil +} + +// decodeEthStateTrieLeaf parses a eth-tx-trie leaf +// from decoded RLP elements +func decodeEthStateTrieLeaf(i []interface{}) ([]interface{}, error) { + var account EthAccount + if err := rlp.DecodeBytes(i[1].([]byte), &account); err != nil { + return nil, err + } + c, err := rawdataToCid(MEthAccountSnapshot, i[1].([]byte), mh.KECCAK_256) + if err != nil { + return nil, err + } + return []interface{}{ + i[0].([]byte), + &EthAccountSnapshot{ + EthAccount: &account, + cid: c, + rawdata: i[1].([]byte), + }, + }, nil +} + +/* + Block INTERFACE +*/ + +// RawData returns the binary of the RLP encode of the state trie node. +func (st *EthStateTrie) RawData() []byte { + return st.rawdata +} + +// Cid returns the cid of the state trie node. +func (st *EthStateTrie) Cid() cid.Cid { + return st.cid +} + +// String is a helper for output +func (st *EthStateTrie) String() string { + return fmt.Sprintf("", st.cid) +} + +// Loggable returns in a map the type of IPLD Link. +func (st *EthStateTrie) Loggable() map[string]interface{} { + return map[string]interface{}{ + "type": "eth-state-trie", + } +} diff --git a/pkg/ipfs/ipld/eth_storage.go b/pkg/ipfs/ipld/eth_storage.go new file mode 100644 index 00000000..d1dc1c9f --- /dev/null +++ b/pkg/ipfs/ipld/eth_storage.go @@ -0,0 +1,96 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ipld + +import ( + "fmt" + + "github.com/ipfs/go-cid" + node "github.com/ipfs/go-ipld-format" + mh "github.com/multiformats/go-multihash" +) + +// EthStorageTrie (eth-storage-trie, codec 0x98), represents +// a node from the storage trie in ethereum. +type EthStorageTrie struct { + *TrieNode +} + +// Static (compile time) check that EthStorageTrie satisfies the node.Node interface. +var _ node.Node = (*EthStorageTrie)(nil) + +/* + INPUT +*/ + +// FromStorageTrieRLP takes the RLP bytes of an ethereum +// storage trie node to return it as an IPLD node for further processing. +func FromStorageTrieRLP(storageNodeRLP []byte) (*EthStorageTrie, error) { + c, err := rawdataToCid(MEthStorageTrie, storageNodeRLP, mh.KECCAK_256) + if err != nil { + return nil, err + } + return DecodeEthStorageTrie(c, storageNodeRLP) +} + +/* + OUTPUT +*/ + +// DecodeEthStorageTrie returns an EthStorageTrie object from its cid and rawdata. +func DecodeEthStorageTrie(c cid.Cid, b []byte) (*EthStorageTrie, error) { + tn, err := decodeTrieNode(c, b, decodeEthStorageTrieLeaf) + if err != nil { + return nil, err + } + return &EthStorageTrie{TrieNode: tn}, nil +} + +// decodeEthStorageTrieLeaf parses a eth-tx-trie leaf +// from decoded RLP elements +func decodeEthStorageTrieLeaf(i []interface{}) ([]interface{}, error) { + return []interface{}{ + i[0].([]byte), + i[1].([]byte), + }, nil +} + +/* + Block INTERFACE +*/ + +// RawData returns the binary of the RLP encode of the storage trie node. +func (st *EthStorageTrie) RawData() []byte { + return st.rawdata +} + +// Cid returns the cid of the storage trie node. +func (st *EthStorageTrie) Cid() cid.Cid { + return st.cid +} + +// String is a helper for output +func (st *EthStorageTrie) String() string { + return fmt.Sprintf("", st.cid) +} + +// Loggable returns in a map the type of IPLD Link. +func (st *EthStorageTrie) Loggable() map[string]interface{} { + return map[string]interface{}{ + "type": "eth-storage-trie", + } +} diff --git a/pkg/ipfs/ipld/eth_tx.go b/pkg/ipfs/ipld/eth_tx.go new file mode 100644 index 00000000..a18bfa39 --- /dev/null +++ b/pkg/ipfs/ipld/eth_tx.go @@ -0,0 +1,215 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ipld + +import ( + "encoding/json" + "fmt" + "strconv" + + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ipfs/go-cid" + node "github.com/ipfs/go-ipld-format" + mh "github.com/multiformats/go-multihash" +) + +// EthTx (eth-tx codec 0x93) represents an ethereum transaction +type EthTx struct { + *types.Transaction + + cid cid.Cid + rawdata []byte +} + +// Static (compile time) check that EthTx satisfies the node.Node interface. +var _ node.Node = (*EthTx)(nil) + +/* + INPUT +*/ + +// NewEthTx converts a *types.Transaction to an EthTx IPLD node +func NewEthTx(tx *types.Transaction) (*EthTx, error) { + txRLP, err := rlp.EncodeToBytes(tx) + if err != nil { + return nil, err + } + c, err := rawdataToCid(MEthTx, txRLP, mh.KECCAK_256) + if err != nil { + return nil, err + } + return &EthTx{ + Transaction: tx, + cid: c, + rawdata: txRLP, + }, nil +} + +/* + OUTPUT +*/ + +// DecodeEthTx takes a cid and its raw binary data +// from IPFS and returns an EthTx object for further processing. +func DecodeEthTx(c cid.Cid, b []byte) (*EthTx, error) { + var t *types.Transaction + if err := rlp.DecodeBytes(b, t); err != nil { + return nil, err + } + return &EthTx{ + Transaction: t, + cid: c, + rawdata: b, + }, nil +} + +/* + Block INTERFACE +*/ + +// RawData returns the binary of the RLP encode of the transaction. +func (t *EthTx) RawData() []byte { + return t.rawdata +} + +// Cid returns the cid of the transaction. +func (t *EthTx) Cid() cid.Cid { + return t.cid +} + +// String is a helper for output +func (t *EthTx) String() string { + return fmt.Sprintf("", t.cid) +} + +// Loggable returns in a map the type of IPLD Link. +func (t *EthTx) Loggable() map[string]interface{} { + return map[string]interface{}{ + "type": "eth-tx", + } +} + +/* + Node INTERFACE +*/ + +// Resolve resolves a path through this node, stopping at any link boundary +// and returning the object found as well as the remaining path to traverse +func (t *EthTx) Resolve(p []string) (interface{}, []string, error) { + if len(p) == 0 { + return t, nil, nil + } + + if len(p) > 1 { + return nil, nil, fmt.Errorf("unexpected path elements past %s", p[0]) + } + + switch p[0] { + + case "gas": + return t.Gas(), nil, nil + case "gasPrice": + return t.GasPrice(), nil, nil + case "input": + return fmt.Sprintf("%x", t.Data()), nil, nil + case "nonce": + return t.Nonce(), nil, nil + case "r": + _, r, _ := t.RawSignatureValues() + return hexutil.EncodeBig(r), nil, nil + case "s": + _, _, s := t.RawSignatureValues() + return hexutil.EncodeBig(s), nil, nil + case "toAddress": + return t.To(), nil, nil + case "v": + v, _, _ := t.RawSignatureValues() + return hexutil.EncodeBig(v), nil, nil + case "value": + return hexutil.EncodeBig(t.Value()), nil, nil + default: + return nil, nil, fmt.Errorf("no such link") + } +} + +// Tree lists all paths within the object under 'path', and up to the given depth. +// To list the entire object (similar to `find .`) pass "" and -1 +func (t *EthTx) Tree(p string, depth int) []string { + if p != "" || depth == 0 { + return nil + } + return []string{"gas", "gasPrice", "input", "nonce", "r", "s", "toAddress", "v", "value"} +} + +// ResolveLink is a helper function that calls resolve and asserts the +// output is a link +func (t *EthTx) ResolveLink(p []string) (*node.Link, []string, error) { + obj, rest, err := t.Resolve(p) + if err != nil { + return nil, nil, err + } + + if lnk, ok := obj.(*node.Link); ok { + return lnk, rest, nil + } + + return nil, nil, fmt.Errorf("resolved item was not a link") +} + +// Copy will go away. It is here to comply with the interface. +func (t *EthTx) Copy() node.Node { + panic("implement me") +} + +// Links is a helper function that returns all links within this object +func (t *EthTx) Links() []*node.Link { + return nil +} + +// Stat will go away. It is here to comply with the interface. +func (t *EthTx) Stat() (*node.NodeStat, error) { + return &node.NodeStat{}, nil +} + +// Size will go away. It is here to comply with the interface. +func (t *EthTx) Size() (uint64, error) { + return strconv.ParseUint(t.Transaction.Size().String(), 10, 64) +} + +/* + EthTx functions +*/ + +// MarshalJSON processes the transaction into readable JSON format. +func (t *EthTx) MarshalJSON() ([]byte, error) { + v, r, s := t.RawSignatureValues() + + out := map[string]interface{}{ + "gas": t.Gas(), + "gasPrice": hexutil.EncodeBig(t.GasPrice()), + "input": fmt.Sprintf("%x", t.Data()), + "nonce": t.Nonce(), + "r": hexutil.EncodeBig(r), + "s": hexutil.EncodeBig(s), + "toAddress": t.To(), + "v": hexutil.EncodeBig(v), + "value": hexutil.EncodeBig(t.Value()), + } + return json.Marshal(out) +} diff --git a/pkg/ipfs/ipld/shared.go b/pkg/ipfs/ipld/shared.go new file mode 100644 index 00000000..0d87c0ee --- /dev/null +++ b/pkg/ipfs/ipld/shared.go @@ -0,0 +1,102 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ipld + +import ( + "bytes" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ipfs/go-cid" + mh "github.com/multiformats/go-multihash" +) + +// IPLD Codecs for Ethereum +// See the authoritative document: +// https://github.com/multiformats/multicodec/blob/master/table.csv +const ( + RawBinary = 0x55 + MEthHeader = 0x90 + MEthHeaderList = 0x91 + MEthTxTrie = 0x92 + MEthTx = 0x93 + MEthTxReceiptTrie = 0x94 + MEthTxReceipt = 0x95 + MEthStateTrie = 0x96 + MEthAccountSnapshot = 0x97 + MEthStorageTrie = 0x98 + MBitcoinHeader = 0xb0 + MBitcoinTx = 0xb1 +) + +// rawdataToCid takes the desired codec and a slice of bytes +// and returns the proper cid of the object. +func rawdataToCid(codec uint64, rawdata []byte, multiHash uint64) (cid.Cid, error) { + c, err := cid.Prefix{ + Codec: codec, + Version: 1, + MhType: multiHash, + MhLength: -1, + }.Sum(rawdata) + if err != nil { + return cid.Cid{}, err + } + return c, nil +} + +// keccak256ToCid takes a keccak256 hash and returns its cid based on +// the codec given. +func keccak256ToCid(codec uint64, h []byte) cid.Cid { + buf, err := mh.Encode(h, mh.KECCAK_256) + if err != nil { + panic(err) + } + + return cid.NewCidV1(codec, mh.Multihash(buf)) +} + +// commonHashToCid takes a go-ethereum common.Hash and returns its +// cid based on the codec given, +func commonHashToCid(codec uint64, h common.Hash) cid.Cid { + mhash, err := mh.Encode(h[:], mh.KECCAK_256) + if err != nil { + panic(err) + } + + return cid.NewCidV1(codec, mhash) +} + +// sha256ToCid takes a sha246 hash and returns its cid based on the +// codec given +func sha256ToCid(codec uint64, h []byte) cid.Cid { + hash, err := mh.Encode(h, mh.DBL_SHA2_256) + if err != nil { + panic(err) + } + + return cid.NewCidV1(codec, hash) +} + +// getRLP encodes the given object to RLP returning its bytes. +func getRLP(object interface{}) []byte { + buf := new(bytes.Buffer) + if err := rlp.Encode(buf, object); err != nil { + panic(err) + } + + return buf.Bytes() +} diff --git a/pkg/ipfs/ipld/trie_node.go b/pkg/ipfs/ipld/trie_node.go new file mode 100644 index 00000000..4534138e --- /dev/null +++ b/pkg/ipfs/ipld/trie_node.go @@ -0,0 +1,440 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ipld + +import ( + "encoding/json" + "fmt" + + "github.com/ethereum/go-ethereum/rlp" + "github.com/ipfs/go-cid" + node "github.com/ipfs/go-ipld-format" +) + +// TrieNode is the general abstraction for +//ethereum IPLD trie nodes. +type TrieNode struct { + // leaf, extension or branch + nodeKind string + + // If leaf or extension: [0] is key, [1] is val. + // If branch: [0] - [16] are children. + elements []interface{} + + // IPLD block information + cid cid.Cid + rawdata []byte +} + +/* + OUTPUT +*/ + +type trieNodeLeafDecoder func([]interface{}) ([]interface{}, error) + +// decodeTrieNode returns a TrieNode object from an IPLD block's +// cid and rawdata. +func decodeTrieNode(c cid.Cid, b []byte, + leafDecoder trieNodeLeafDecoder) (*TrieNode, error) { + var ( + i, decoded, elements []interface{} + nodeKind string + err error + ) + + err = rlp.DecodeBytes(b, &i) + if err != nil { + return nil, err + } + + codec := c.Type() + switch len(i) { + case 2: + nodeKind, decoded, err = decodeCompactKey(i) + if err != nil { + return nil, err + } + + if nodeKind == "extension" { + elements, err = parseTrieNodeExtension(decoded, codec) + } + if nodeKind == "leaf" { + elements, err = leafDecoder(decoded) + } + if nodeKind != "extension" && nodeKind != "leaf" { + return nil, fmt.Errorf("unexpected nodeKind returned from decoder") + } + case 17: + nodeKind = "branch" + elements, err = parseTrieNodeBranch(i, codec) + if err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("unknown trie node type") + } + + return &TrieNode{ + nodeKind: nodeKind, + elements: elements, + rawdata: b, + cid: c, + }, nil +} + +// decodeCompactKey takes a compact key, and returns its nodeKind and value. +func decodeCompactKey(i []interface{}) (string, []interface{}, error) { + first := i[0].([]byte) + last := i[1].([]byte) + + switch first[0] / 16 { + case '\x00': + return "extension", []interface{}{ + nibbleToByte(first)[2:], + last, + }, nil + case '\x01': + return "extension", []interface{}{ + nibbleToByte(first)[1:], + last, + }, nil + case '\x02': + return "leaf", []interface{}{ + nibbleToByte(first)[2:], + last, + }, nil + case '\x03': + return "leaf", []interface{}{ + nibbleToByte(first)[1:], + last, + }, nil + default: + return "", nil, fmt.Errorf("unknown hex prefix") + } +} + +// parseTrieNodeExtension helper improves readability +func parseTrieNodeExtension(i []interface{}, codec uint64) ([]interface{}, error) { + return []interface{}{ + i[0].([]byte), + keccak256ToCid(codec, i[1].([]byte)), + }, nil +} + +// parseTrieNodeBranch helper improves readability +func parseTrieNodeBranch(i []interface{}, codec uint64) ([]interface{}, error) { + var out []interface{} + + for _, vi := range i { + v := vi.([]byte) + + switch len(v) { + case 0: + out = append(out, nil) + case 32: + out = append(out, keccak256ToCid(codec, v)) + default: + return nil, fmt.Errorf("unrecognized object: %v", v) + } + } + + return out, nil +} + +/* + Node INTERFACE +*/ + +// Resolve resolves a path through this node, stopping at any link boundary +// and returning the object found as well as the remaining path to traverse +func (t *TrieNode) Resolve(p []string) (interface{}, []string, error) { + switch t.nodeKind { + case "extension": + return t.resolveTrieNodeExtension(p) + case "leaf": + return t.resolveTrieNodeLeaf(p) + case "branch": + return t.resolveTrieNodeBranch(p) + default: + return nil, nil, fmt.Errorf("nodeKind case not implemented") + } +} + +// Tree lists all paths within the object under 'path', and up to the given depth. +// To list the entire object (similar to `find .`) pass "" and -1 +func (t *TrieNode) Tree(p string, depth int) []string { + if p != "" || depth == 0 { + return nil + } + + var out []string + + switch t.nodeKind { + case "extension": + var val string + for _, e := range t.elements[0].([]byte) { + val += fmt.Sprintf("%x", e) + } + return []string{val} + case "branch": + for i, elem := range t.elements { + if _, ok := elem.(*cid.Cid); ok { + out = append(out, fmt.Sprintf("%x", i)) + } + } + return out + + default: + return nil + } +} + +// ResolveLink is a helper function that calls resolve and asserts the +// output is a link +func (t *TrieNode) ResolveLink(p []string) (*node.Link, []string, error) { + obj, rest, err := t.Resolve(p) + if err != nil { + return nil, nil, err + } + + lnk, ok := obj.(*node.Link) + if !ok { + return nil, nil, fmt.Errorf("was not a link") + } + + return lnk, rest, nil +} + +// Copy will go away. It is here to comply with the interface. +func (t *TrieNode) Copy() node.Node { + panic("dont use this yet") +} + +// Links is a helper function that returns all links within this object +func (t *TrieNode) Links() []*node.Link { + var out []*node.Link + + for _, i := range t.elements { + c, ok := i.(cid.Cid) + if ok { + out = append(out, &node.Link{Cid: c}) + } + } + + return out +} + +// Stat will go away. It is here to comply with the interface. +func (t *TrieNode) Stat() (*node.NodeStat, error) { + return &node.NodeStat{}, nil +} + +// Size will go away. It is here to comply with the interface. +func (t *TrieNode) Size() (uint64, error) { + return 0, nil +} + +/* + TrieNode functions +*/ + +// MarshalJSON processes the transaction trie into readable JSON format. +func (t *TrieNode) MarshalJSON() ([]byte, error) { + var out map[string]interface{} + + switch t.nodeKind { + case "extension": + fallthrough + case "leaf": + var hexPrefix string + for _, e := range t.elements[0].([]byte) { + hexPrefix += fmt.Sprintf("%x", e) + } + + // if we got a byte we need to do this casting otherwise + // it will be marshaled to a base64 encoded value + if _, ok := t.elements[1].([]byte); ok { + var hexVal string + for _, e := range t.elements[1].([]byte) { + hexVal += fmt.Sprintf("%x", e) + } + + t.elements[1] = hexVal + } + + out = map[string]interface{}{ + "type": t.nodeKind, + hexPrefix: t.elements[1], + } + + case "branch": + out = map[string]interface{}{ + "type": "branch", + "0": t.elements[0], + "1": t.elements[1], + "2": t.elements[2], + "3": t.elements[3], + "4": t.elements[4], + "5": t.elements[5], + "6": t.elements[6], + "7": t.elements[7], + "8": t.elements[8], + "9": t.elements[9], + "a": t.elements[10], + "b": t.elements[11], + "c": t.elements[12], + "d": t.elements[13], + "e": t.elements[14], + "f": t.elements[15], + } + default: + return nil, fmt.Errorf("nodeKind %s not supported", t.nodeKind) + } + + return json.Marshal(out) +} + +// nibbleToByte expands the nibbles of a byte slice into their own bytes. +func nibbleToByte(k []byte) []byte { + var out []byte + + for _, b := range k { + out = append(out, b/16) + out = append(out, b%16) + } + + return out +} + +// Resolve reading conveniences +func (t *TrieNode) resolveTrieNodeExtension(p []string) (interface{}, []string, error) { + nibbles := t.elements[0].([]byte) + idx, rest := shiftFromPath(p, len(nibbles)) + if len(idx) < len(nibbles) { + return nil, nil, fmt.Errorf("not enough nibbles to traverse this extension") + } + + for _, i := range idx { + if getHexIndex(string(i)) == -1 { + return nil, nil, fmt.Errorf("invalid path element") + } + } + + for i, n := range nibbles { + if string(idx[i]) != fmt.Sprintf("%x", n) { + return nil, nil, fmt.Errorf("no such link in this extension") + } + } + + return &node.Link{Cid: t.elements[1].(cid.Cid)}, rest, nil +} + +func (t *TrieNode) resolveTrieNodeLeaf(p []string) (interface{}, []string, error) { + nibbles := t.elements[0].([]byte) + + if len(nibbles) != 0 { + idx, rest := shiftFromPath(p, len(nibbles)) + if len(idx) < len(nibbles) { + return nil, nil, fmt.Errorf("not enough nibbles to traverse this leaf") + } + + for _, i := range idx { + if getHexIndex(string(i)) == -1 { + return nil, nil, fmt.Errorf("invalid path element") + } + } + + for i, n := range nibbles { + if string(idx[i]) != fmt.Sprintf("%x", n) { + return nil, nil, fmt.Errorf("no such link in this extension") + } + } + + p = rest + } + + link, ok := t.elements[1].(node.Node) + if !ok { + return nil, nil, fmt.Errorf("leaf children is not an IPLD node") + } + + return link.Resolve(p) +} + +func (t *TrieNode) resolveTrieNodeBranch(p []string) (interface{}, []string, error) { + idx, rest := shiftFromPath(p, 1) + hidx := getHexIndex(idx) + if hidx == -1 { + return nil, nil, fmt.Errorf("incorrect path") + } + + child := t.elements[hidx] + if child != nil { + return &node.Link{Cid: child.(cid.Cid)}, rest, nil + } + return nil, nil, fmt.Errorf("no such link in this branch") +} + +// shiftFromPath extracts from a given path (as a slice of strings) +// the given number of elements as a single string, returning whatever +// it has not taken. +// +// Examples: +// ["0", "a", "something"] and 1 -> "0" and ["a", "something"] +// ["ab", "c", "d", "1"] and 2 -> "ab" and ["c", "d", "1"] +// ["abc", "d", "1"] and 2 -> "ab" and ["c", "d", "1"] +func shiftFromPath(p []string, i int) (string, []string) { + var ( + out string + rest []string + ) + + for _, pe := range p { + re := "" + for _, c := range pe { + if len(out) < i { + out += string(c) + } else { + re += string(c) + } + } + + if len(out) == i && re != "" { + rest = append(rest, re) + } + } + + return out, rest +} + +// getHexIndex returns to you the integer 0 - 15 equivalent to your +// string character if applicable, or -1 otherwise. +func getHexIndex(s string) int { + if len(s) != 1 { + return -1 + } + + c := byte(s[0]) + switch { + case '0' <= c && c <= '9': + return int(c - '0') + case 'a' <= c && c <= 'f': + return int(c - 'a' + 10) + } + + return -1 +} diff --git a/pkg/super_node/btc/converter.go b/pkg/super_node/btc/converter.go index 0a1bba07..28616de6 100644 --- a/pkg/super_node/btc/converter.go +++ b/pkg/super_node/btc/converter.go @@ -17,11 +17,7 @@ package btc import ( - "bytes" "fmt" - - "github.com/btcsuite/btcd/wire" - "github.com/btcsuite/btcutil" ) // PayloadConverter satisfies the PayloadConverter interface for bitcoin @@ -39,18 +35,8 @@ func (pc *PayloadConverter) Convert(payload interface{}) (interface{}, error) { if !ok { return nil, fmt.Errorf("btc converter: expected payload type %T got %T", BlockPayload{}, payload) } - msgBlock := wire.NewMsgBlock(btcBlockPayload.Header) - for _, tx := range btcBlockPayload.Txs { - msgBlock.AddTransaction(tx.MsgTx()) - } - w := bytes.NewBuffer(make([]byte, 0, msgBlock.SerializeSize())) - if err := msgBlock.Serialize(w); err != nil { - return nil, err - } - utilBlock := btcutil.NewBlockFromBlockAndBytes(msgBlock, w.Bytes()) - utilBlock.SetHeight(btcBlockPayload.Height) txMeta := make([]TxModel, len(btcBlockPayload.Txs)) - for _, tx := range utilBlock.Transactions() { + for _, tx := range btcBlockPayload.Txs { index := tx.Index() txModel := TxModel{ TxHash: tx.Hash().String(), @@ -63,7 +49,7 @@ func (pc *PayloadConverter) Convert(payload interface{}) (interface{}, error) { txMeta[index] = txModel } return IPLDPayload{ - Block: utilBlock, - TxMetaData: txMeta, + BlockPayload: btcBlockPayload, + TxMetaData: txMeta, }, nil } diff --git a/pkg/super_node/btc/types.go b/pkg/super_node/btc/types.go index 77cb3eae..74107958 100644 --- a/pkg/super_node/btc/types.go +++ b/pkg/super_node/btc/types.go @@ -37,7 +37,7 @@ type BlockPayload struct { // Returned by PayloadConverter // Passed to IPLDPublisher and ResponseFilterer type IPLDPayload struct { - Block *btcutil.Block + BlockPayload TxMetaData []TxModel } diff --git a/pkg/super_node/config/btc_subscription.go b/pkg/super_node/config/btc_subscription.go new file mode 100644 index 00000000..50d7e290 --- /dev/null +++ b/pkg/super_node/config/btc_subscription.go @@ -0,0 +1,17 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package config diff --git a/pkg/super_node/config/config.go b/pkg/super_node/config/config.go index ce4e096f..75d1a246 100644 --- a/pkg/super_node/config/config.go +++ b/pkg/super_node/config/config.go @@ -52,11 +52,11 @@ type SuperNode struct { // Sync params Sync bool Workers int - WSClient core.RPCClient + WSClient interface{} NodeInfo core.Node // Backfiller params BackFill bool - HTTPClient core.RPCClient + HTTPClient interface{} Frequency time.Duration BatchSize uint64 } @@ -147,7 +147,7 @@ func (sn *SuperNode) BackFillFields() error { return nil } -func getNodeAndClient(chain ChainType, path string) (core.Node, core.RPCClient, error) { +func getNodeAndClient(chain ChainType, path string) (core.Node, interface{}, error) { switch chain { case Ethereum: rawRPCClient, err := rpc.Dial(path) diff --git a/pkg/super_node/eth/converter.go b/pkg/super_node/eth/converter.go index e86b7fd1..a01dec9a 100644 --- a/pkg/super_node/eth/converter.go +++ b/pkg/super_node/eth/converter.go @@ -50,17 +50,10 @@ func (pc *PayloadConverter) Convert(payload interface{}) (interface{}, error) { if err := rlp.DecodeBytes(stateDiffPayload.BlockRlp, block); err != nil { return nil, err } - // Process and publish headers - header := block.Header() - headerRlp, err := rlp.EncodeToBytes(header) - if err != nil { - return nil, err - } trxLen := len(block.Transactions()) convertedPayload := &IPLDPayload{ TotalDifficulty: stateDiffPayload.TotalDifficulty, Block: block, - HeaderRLP: headerRlp, TxMetaData: make([]TxModel, 0, trxLen), Receipts: make(types.Receipts, 0, trxLen), ReceiptMetaData: make([]ReceiptModel, 0, trxLen), diff --git a/pkg/super_node/eth/converter_test.go b/pkg/super_node/eth/converter_test.go index cff68b15..bbff68cb 100644 --- a/pkg/super_node/eth/converter_test.go +++ b/pkg/super_node/eth/converter_test.go @@ -44,7 +44,9 @@ var _ = Describe("Converter", func() { expectedBody, err := rlp.EncodeToBytes(mocks.MockBlock.Body()) Expect(err).ToNot(HaveOccurred()) Expect(gotBody).To(Equal(expectedBody)) - Expect(convertedPayload.HeaderRLP).To(Equal(mocks.MockHeaderRlp)) + gotHeader, err := rlp.EncodeToBytes(convertedPayload.Block.Header()) + Expect(err).ToNot(HaveOccurred()) + Expect(gotHeader).To(Equal(mocks.MockHeaderRlp)) Expect(convertedPayload.TxMetaData).To(Equal(mocks.MockTrxMeta)) Expect(convertedPayload.ReceiptMetaData).To(Equal(mocks.MockRctMeta)) }) diff --git a/pkg/super_node/eth/filterer.go b/pkg/super_node/eth/filterer.go index 5048c7e7..4e660e25 100644 --- a/pkg/super_node/eth/filterer.go +++ b/pkg/super_node/eth/filterer.go @@ -76,7 +76,11 @@ func (s *ResponseFilterer) Filter(filter, payload interface{}) (interface{}, err func (s *ResponseFilterer) filterHeaders(headerFilter config.HeaderFilter, response *StreamPayload, payload *IPLDPayload) error { if !headerFilter.Off { - response.HeadersRlp = append(response.HeadersRlp, payload.HeaderRLP) + headerRLP, err := rlp.EncodeToBytes(payload.Block.Header()) + if err != nil { + return err + } + response.HeadersRlp = append(response.HeadersRlp, headerRLP) if headerFilter.Uncles { response.UnclesRlp = make([][]byte, 0, len(payload.Block.Body().Uncles)) for _, uncle := range payload.Block.Body().Uncles { diff --git a/pkg/super_node/eth/mocks/dag_putters.go b/pkg/super_node/eth/mocks/dag_putters.go index 7461344c..2c2cb8bc 100644 --- a/pkg/super_node/eth/mocks/dag_putters.go +++ b/pkg/super_node/eth/mocks/dag_putters.go @@ -25,22 +25,26 @@ import ( // DagPutter is a mock for testing the ipfs publisher type DagPutter struct { CIDsToReturn []string + PassedRaw interface{} ErrToReturn error } // DagPut returns the pre-loaded CIDs or error func (dp *DagPutter) DagPut(raw interface{}) ([]string, error) { + dp.PassedRaw = raw return dp.CIDsToReturn, dp.ErrToReturn } // MappedDagPutter is a mock for testing the ipfs publisher type MappedDagPutter struct { CIDsToReturn map[common.Hash][]string + PassedRaw interface{} ErrToReturn error } // DagPut returns the pre-loaded CIDs or error func (mdp *MappedDagPutter) DagPut(raw interface{}) ([]string, error) { + mdp.PassedRaw = raw if mdp.CIDsToReturn == nil { return nil, errors.New("mapped dag putter needs to be initialized with a map of cids to return") } diff --git a/pkg/super_node/eth/mocks/streamer.go b/pkg/super_node/eth/mocks/streamer.go index 83186b16..cc8a6b97 100644 --- a/pkg/super_node/eth/mocks/streamer.go +++ b/pkg/super_node/eth/mocks/streamer.go @@ -19,6 +19,7 @@ package mocks import ( "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/statediff" + "github.com/vulcanize/vulcanizedb/pkg/super_node/shared" ) // StateDiffStreamer is the underlying struct for the Streamer interface @@ -30,7 +31,7 @@ type StateDiffStreamer struct { } // Stream is the main loop for subscribing to data from the Geth state diff process -func (sds *StateDiffStreamer) Stream(payloadChan chan interface{}) (*rpc.ClientSubscription, error) { +func (sds *StateDiffStreamer) Stream(payloadChan chan interface{}) (shared.ClientSubscription, error) { sds.PassedPayloadChan = payloadChan go func() { diff --git a/pkg/super_node/eth/mocks/test_data.go b/pkg/super_node/eth/mocks/test_data.go index a328f25c..361121a9 100644 --- a/pkg/super_node/eth/mocks/test_data.go +++ b/pkg/super_node/eth/mocks/test_data.go @@ -237,7 +237,6 @@ var ( TotalDifficulty: big.NewInt(1337), Block: MockBlock, Receipts: MockReceipts, - HeaderRLP: MockHeaderRlp, TxMetaData: MockTrxMeta, ReceiptMetaData: MockRctMeta, StorageNodes: MockStorageNodes, diff --git a/pkg/super_node/eth/publisher.go b/pkg/super_node/eth/publisher.go index 128125bd..9720514c 100644 --- a/pkg/super_node/eth/publisher.go +++ b/pkg/super_node/eth/publisher.go @@ -20,26 +20,21 @@ import ( "errors" "fmt" + "github.com/vulcanize/vulcanizedb/pkg/super_node/shared" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/rlp" - - "github.com/vulcanize/eth-block-extractor/pkg/ipfs" - "github.com/vulcanize/eth-block-extractor/pkg/ipfs/eth_block_header" - "github.com/vulcanize/eth-block-extractor/pkg/ipfs/eth_block_receipts" - "github.com/vulcanize/eth-block-extractor/pkg/ipfs/eth_block_transactions" - "github.com/vulcanize/eth-block-extractor/pkg/ipfs/eth_state_trie" - "github.com/vulcanize/eth-block-extractor/pkg/ipfs/eth_storage_trie" - rlp2 "github.com/vulcanize/eth-block-extractor/pkg/wrappers/rlp" + "github.com/vulcanize/vulcanizedb/pkg/ipfs" + "github.com/vulcanize/vulcanizedb/pkg/ipfs/dag_putters" ) // IPLDPublisher satisfies the IPLDPublisher for ethereum type IPLDPublisher struct { - HeaderPutter ipfs.DagPutter - TransactionPutter ipfs.DagPutter - ReceiptPutter ipfs.DagPutter - StatePutter ipfs.DagPutter - StoragePutter ipfs.DagPutter + HeaderPutter shared.DagPutter + TransactionPutter shared.DagPutter + ReceiptPutter shared.DagPutter + StatePutter shared.DagPutter + StoragePutter shared.DagPutter } // NewIPLDPublisher creates a pointer to a new Publisher which satisfies the IPLDPublisher interface @@ -49,11 +44,11 @@ func NewIPLDPublisher(ipfsPath string) (*IPLDPublisher, error) { return nil, err } return &IPLDPublisher{ - HeaderPutter: eth_block_header.NewBlockHeaderDagPutter(node, rlp2.RlpDecoder{}), - TransactionPutter: eth_block_transactions.NewBlockTransactionsDagPutter(node), - ReceiptPutter: eth_block_receipts.NewEthBlockReceiptDagPutter(node), - StatePutter: eth_state_trie.NewStateTrieDagPutter(node), - StoragePutter: eth_storage_trie.NewStorageTrieDagPutter(node), + HeaderPutter: dag_putters.NewEthBlockHeaderDagPutter(node), + TransactionPutter: dag_putters.NewEthTxsDagPutter(node), + ReceiptPutter: dag_putters.NewEthReceiptDagPutter(node), + StatePutter: dag_putters.NewEthStateDagPutter(node), + StoragePutter: dag_putters.NewEthStorageDagPutter(node), }, nil } @@ -64,7 +59,7 @@ func (pub *IPLDPublisher) Publish(payload interface{}) (interface{}, error) { return nil, fmt.Errorf("eth publisher expected payload type %T got %T", &IPLDPayload{}, payload) } // Process and publish headers - headerCid, err := pub.publishHeader(ipldPayload.HeaderRLP) + headerCid, err := pub.publishHeader(ipldPayload.Block.Header()) if err != nil { return nil, err } @@ -79,11 +74,7 @@ func (pub *IPLDPublisher) Publish(payload interface{}) (interface{}, error) { // Process and publish uncles uncleCids := make([]UncleModel, 0, len(ipldPayload.Block.Uncles())) for _, uncle := range ipldPayload.Block.Uncles() { - uncleRlp, err := rlp.EncodeToBytes(uncle) - if err != nil { - return nil, err - } - uncleCid, err := pub.publishHeader(uncleRlp) + uncleCid, err := pub.publishHeader(uncle) if err != nil { return nil, err } @@ -95,7 +86,7 @@ func (pub *IPLDPublisher) Publish(payload interface{}) (interface{}, error) { } // Process and publish transactions - transactionCids, err := pub.publishTransactions(ipldPayload.Block.Body(), ipldPayload.TxMetaData) + transactionCids, err := pub.publishTransactions(ipldPayload.Block.Body().Transactions, ipldPayload.TxMetaData) if err != nil { return nil, err } @@ -129,19 +120,16 @@ func (pub *IPLDPublisher) Publish(payload interface{}) (interface{}, error) { }, nil } -func (pub *IPLDPublisher) publishHeader(headerRLP []byte) (string, error) { - headerCids, err := pub.HeaderPutter.DagPut(headerRLP) +func (pub *IPLDPublisher) publishHeader(header *types.Header) (string, error) { + cids, err := pub.HeaderPutter.DagPut(header) if err != nil { return "", err } - if len(headerCids) != 1 { - return "", errors.New("single CID expected to be returned for header") - } - return headerCids[0], nil + return cids[0], nil } -func (pub *IPLDPublisher) publishTransactions(blockBody *types.Body, trxMeta []TxModel) ([]TxModel, error) { - transactionCids, err := pub.TransactionPutter.DagPut(blockBody) +func (pub *IPLDPublisher) publishTransactions(transactions types.Transactions, trxMeta []TxModel) ([]TxModel, error) { + transactionCids, err := pub.TransactionPutter.DagPut(transactions) if err != nil { return nil, err } @@ -187,16 +175,13 @@ func (pub *IPLDPublisher) publishReceipts(receipts types.Receipts, receiptMeta [ func (pub *IPLDPublisher) publishStateNodes(stateNodes []TrieNode) ([]StateNodeModel, error) { stateNodeCids := make([]StateNodeModel, 0, len(stateNodes)) for _, node := range stateNodes { - stateNodeCid, err := pub.StatePutter.DagPut(node.Value) + cids, err := pub.StatePutter.DagPut(node.Value) if err != nil { return nil, err } - if len(stateNodeCid) != 1 { - return nil, errors.New("single CID expected to be returned for state leaf") - } stateNodeCids = append(stateNodeCids, StateNodeModel{ StateKey: node.Key.String(), - CID: stateNodeCid[0], + CID: cids[0], Leaf: node.Leaf, }) } @@ -208,17 +193,14 @@ func (pub *IPLDPublisher) publishStorageNodes(storageNodes map[common.Hash][]Tri for addrKey, storageTrie := range storageNodes { storageLeafCids[addrKey] = make([]StorageNodeModel, 0, len(storageTrie)) for _, node := range storageTrie { - storageNodeCid, err := pub.StoragePutter.DagPut(node.Value) + cids, err := pub.StoragePutter.DagPut(node.Value) if err != nil { return nil, err } - if len(storageNodeCid) != 1 { - return nil, errors.New("single CID expected to be returned for storage leaf") - } // Map storage node cids to their state key hashes storageLeafCids[addrKey] = append(storageLeafCids[addrKey], StorageNodeModel{ StorageKey: node.Key.Hex(), - CID: storageNodeCid[0], + CID: cids[0], Leaf: node.Leaf, }) } diff --git a/pkg/super_node/eth/types.go b/pkg/super_node/eth/types.go index 48d01c12..011965fb 100644 --- a/pkg/super_node/eth/types.go +++ b/pkg/super_node/eth/types.go @@ -31,7 +31,6 @@ import ( type IPLDPayload struct { TotalDifficulty *big.Int Block *types.Block - HeaderRLP []byte TxMetaData []TxModel Receipts types.Receipts ReceiptMetaData []ReceiptModel diff --git a/pkg/super_node/shared/intefaces.go b/pkg/super_node/shared/intefaces.go index 1364e0be..c619bb83 100644 --- a/pkg/super_node/shared/intefaces.go +++ b/pkg/super_node/shared/intefaces.go @@ -69,3 +69,8 @@ type ClientSubscription interface { Err() <-chan error Unsubscribe() } + +// DagPutter is a general interface for a dag putter +type DagPutter interface { + DagPut(raw interface{}) ([]string, error) +} diff --git a/utils/utils.go b/utils/utils.go index 3bfa83b8..87608f94 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -25,9 +25,9 @@ import ( "github.com/sirupsen/logrus" "github.com/vulcanize/vulcanizedb/pkg/config" + "github.com/vulcanize/vulcanizedb/pkg/eth" "github.com/vulcanize/vulcanizedb/pkg/eth/core" "github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/eth" ) func LoadPostgres(database config.Database, node core.Node) postgres.DB {