From 68d8e6814433b54b3db20d190e69cd3bd4990626 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Thu, 30 Jan 2020 16:46:08 -0600 Subject: [PATCH] btc ipld publisher --- pkg/ipfs/dag_putters/btc_tx.go | 8 +- pkg/super_node/btc/btc_suite_test.go | 35 +++++++++ pkg/super_node/btc/filterer.go | 17 +++++ pkg/super_node/btc/indexer.go | 17 +++++ pkg/super_node/btc/ipld_fetcher.go | 17 +++++ pkg/super_node/btc/models.go | 38 ++++++++++ pkg/super_node/btc/publisher.go | 106 +++++++++++++++++++++++++++ pkg/super_node/btc/resolver.go | 17 +++++ pkg/super_node/btc/retriever.go | 17 +++++ pkg/super_node/btc/streamer_test.go | 17 +++++ pkg/super_node/btc/types.go | 8 -- 11 files changed, 285 insertions(+), 12 deletions(-) create mode 100644 pkg/super_node/btc/btc_suite_test.go create mode 100644 pkg/super_node/btc/filterer.go create mode 100644 pkg/super_node/btc/indexer.go create mode 100644 pkg/super_node/btc/ipld_fetcher.go create mode 100644 pkg/super_node/btc/models.go create mode 100644 pkg/super_node/btc/publisher.go create mode 100644 pkg/super_node/btc/resolver.go create mode 100644 pkg/super_node/btc/retriever.go create mode 100644 pkg/super_node/btc/streamer_test.go diff --git a/pkg/ipfs/dag_putters/btc_tx.go b/pkg/ipfs/dag_putters/btc_tx.go index cab4088d..77f0dcb7 100644 --- a/pkg/ipfs/dag_putters/btc_tx.go +++ b/pkg/ipfs/dag_putters/btc_tx.go @@ -19,7 +19,7 @@ package dag_putters import ( "fmt" - "github.com/btcsuite/btcd/wire" + "github.com/btcsuite/btcutil" "github.com/vulcanize/vulcanizedb/pkg/ipfs" "github.com/vulcanize/vulcanizedb/pkg/ipfs/ipld" @@ -34,13 +34,13 @@ func NewBtcTxDagPutter(adder *ipfs.IPFS) *BtcTxDagPutter { } func (etdp *BtcTxDagPutter) DagPut(raw interface{}) ([]string, error) { - transactions, ok := raw.([]*wire.MsgTx) + transactions, ok := raw.([]*btcutil.Tx) if !ok { - return nil, fmt.Errorf("BtcTxDagPutter expected input type %T got %T", []*wire.MsgTx{}, raw) + return nil, fmt.Errorf("BtcTxDagPutter expected input type %T got %T", []*btcutil.Tx{}, raw) } cids := make([]string, len(transactions)) for i, transaction := range transactions { - node, err := ipld.NewBtcTx(transaction) + node, err := ipld.NewBtcTx(transaction.MsgTx()) if err != nil { return nil, err } diff --git a/pkg/super_node/btc/btc_suite_test.go b/pkg/super_node/btc/btc_suite_test.go new file mode 100644 index 00000000..ca21ebee --- /dev/null +++ b/pkg/super_node/btc/btc_suite_test.go @@ -0,0 +1,35 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package btc_test + +import ( + "io/ioutil" + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/sirupsen/logrus" +) + +func TestETHSuperNode(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Super Node BTC Suite Test") +} + +var _ = BeforeSuite(func() { + logrus.SetOutput(ioutil.Discard) +}) diff --git a/pkg/super_node/btc/filterer.go b/pkg/super_node/btc/filterer.go new file mode 100644 index 00000000..8dd3c1ae --- /dev/null +++ b/pkg/super_node/btc/filterer.go @@ -0,0 +1,17 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package btc diff --git a/pkg/super_node/btc/indexer.go b/pkg/super_node/btc/indexer.go new file mode 100644 index 00000000..8dd3c1ae --- /dev/null +++ b/pkg/super_node/btc/indexer.go @@ -0,0 +1,17 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package btc diff --git a/pkg/super_node/btc/ipld_fetcher.go b/pkg/super_node/btc/ipld_fetcher.go new file mode 100644 index 00000000..8dd3c1ae --- /dev/null +++ b/pkg/super_node/btc/ipld_fetcher.go @@ -0,0 +1,17 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package btc diff --git a/pkg/super_node/btc/models.go b/pkg/super_node/btc/models.go new file mode 100644 index 00000000..42ef9678 --- /dev/null +++ b/pkg/super_node/btc/models.go @@ -0,0 +1,38 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package btc + +type HeaderModel struct { + ID int64 `db:"id"` + BlockNumber string `db:"block_number"` + BlockHash string `db:"block_hash"` + ParentHash string `db:"parent_hash"` + CID string `db:"cid"` + // TotalDifficulty string `db:"td"` Not sure we can support this unless we modify btcd +} + +type TxModel struct { + ID int64 `db:"id"` + HeaderID int64 `db:"header_id"` + Index int64 `db:"index"` + TxHash string `db:"tx_hash"` + CID string `db:"cid"` + HasWitness bool `db:"has_witness"` + WitnessHash string `db:"witness_hash"` + //Dst string `db:"dst"` + //Src string `db:"src"` +} diff --git a/pkg/super_node/btc/publisher.go b/pkg/super_node/btc/publisher.go new file mode 100644 index 00000000..1d2a2062 --- /dev/null +++ b/pkg/super_node/btc/publisher.go @@ -0,0 +1,106 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package btc + +import ( + "errors" + "fmt" + "strconv" + + "github.com/btcsuite/btcd/wire" + "github.com/btcsuite/btcutil" + + "github.com/vulcanize/vulcanizedb/pkg/ipfs" + "github.com/vulcanize/vulcanizedb/pkg/ipfs/dag_putters" + "github.com/vulcanize/vulcanizedb/pkg/super_node/shared" +) + +// IPLDPublisher satisfies the IPLDPublisher for ethereum +type IPLDPublisher struct { + HeaderPutter shared.DagPutter + TransactionPutter shared.DagPutter +} + +// NewIPLDPublisher creates a pointer to a new Publisher which satisfies the IPLDPublisher interface +func NewIPLDPublisher(ipfsPath string) (*IPLDPublisher, error) { + node, err := ipfs.InitIPFSNode(ipfsPath) + if err != nil { + return nil, err + } + return &IPLDPublisher{ + HeaderPutter: dag_putters.NewBtcHeaderDagPutter(node), + TransactionPutter: dag_putters.NewBtcTxDagPutter(node), + }, nil +} + +// Publish publishes an IPLDPayload to IPFS and returns the corresponding CIDPayload +func (pub *IPLDPublisher) Publish(payload interface{}) (interface{}, error) { + ipldPayload, ok := payload.(*IPLDPayload) + if !ok { + return nil, fmt.Errorf("eth publisher expected payload type %T got %T", &IPLDPayload{}, payload) + } + // Process and publish headers + headerCid, err := pub.publishHeader(ipldPayload.Header) + if err != nil { + return nil, err + } + header := HeaderModel{ + CID: headerCid, + ParentHash: ipldPayload.Header.PrevBlock.String(), + BlockNumber: strconv.Itoa(int(ipldPayload.Height)), + BlockHash: ipldPayload.Header.BlockHash().String(), + } + // Process and publish transactions + transactionCids, err := pub.publishTransactions(ipldPayload.Txs, ipldPayload.TxMetaData) + if err != nil { + return nil, err + } + // Package CIDs and their metadata into a single struct + return &CIDPayload{ + HeaderCID: header, + TransactionCIDs: transactionCids, + }, nil +} + +func (pub *IPLDPublisher) publishHeader(header *wire.BlockHeader) (string, error) { + cids, err := pub.HeaderPutter.DagPut(header) + if err != nil { + return "", err + } + return cids[0], nil +} + +func (pub *IPLDPublisher) publishTransactions(transactions []*btcutil.Tx, trxMeta []TxModel) ([]TxModel, error) { + transactionCids, err := pub.TransactionPutter.DagPut(transactions) + if err != nil { + return nil, err + } + if len(transactionCids) != len(trxMeta) { + return nil, errors.New("expected one CID for each transaction") + } + mappedTrxCids := make([]TxModel, len(transactionCids)) + for i, cid := range transactionCids { + mappedTrxCids[i] = TxModel{ + CID: cid, + Index: trxMeta[i].Index, + TxHash: trxMeta[i].TxHash, + HasWitness: trxMeta[i].HasWitness, + WitnessHash: trxMeta[i].WitnessHash, + } + } + return mappedTrxCids, nil +} diff --git a/pkg/super_node/btc/resolver.go b/pkg/super_node/btc/resolver.go new file mode 100644 index 00000000..8dd3c1ae --- /dev/null +++ b/pkg/super_node/btc/resolver.go @@ -0,0 +1,17 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package btc diff --git a/pkg/super_node/btc/retriever.go b/pkg/super_node/btc/retriever.go new file mode 100644 index 00000000..8dd3c1ae --- /dev/null +++ b/pkg/super_node/btc/retriever.go @@ -0,0 +1,17 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package btc diff --git a/pkg/super_node/btc/streamer_test.go b/pkg/super_node/btc/streamer_test.go new file mode 100644 index 00000000..8dd3c1ae --- /dev/null +++ b/pkg/super_node/btc/streamer_test.go @@ -0,0 +1,17 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package btc diff --git a/pkg/super_node/btc/types.go b/pkg/super_node/btc/types.go index 74107958..41714a94 100644 --- a/pkg/super_node/btc/types.go +++ b/pkg/super_node/btc/types.go @@ -22,7 +22,6 @@ import ( "github.com/btcsuite/btcd/wire" "github.com/btcsuite/btcutil" - "github.com/ethereum/go-ethereum/common" "github.com/ipfs/go-block-format" ) @@ -41,13 +40,6 @@ type IPLDPayload struct { TxMetaData []TxModel } -// Trie struct used to flag node as leaf or not -type TrieNode struct { - Key common.Hash - Value []byte - Leaf bool -} - // CIDPayload is a struct to hold all the CIDs and their associated meta data for indexing in Postgres // Returned by IPLDPublisher // Passed to CIDIndexer