direct fetching of iplds from pg-ipfs for btc and eth

pkg/super_node/btc/ipld_fetcher.go

@@ -40,6 +40,7 @@ type IPLDFetcher struct {
 }
 
 // NewIPLDFetcher creates a pointer to a new IPLDFetcher
+// It interfaces with PG-IPFS through an internalized IPFS node interface
 func NewIPLDFetcher(ipfsPath string) (*IPLDFetcher, error) {
 	blockService, err := ipfs.InitIPFSBlockService(ipfsPath)
 	if err != nil {

pkg/super_node/btc/ipld_pg_fetcher.go

@@ -0,0 +1,98 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package btc
+
+import (
+	"fmt"
+
+	"github.com/jmoiron/sqlx"
+	log "github.com/sirupsen/logrus"
+
+	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
+	"github.com/vulcanize/vulcanizedb/pkg/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+)
+
+// IPLDPGFetcher satisfies the IPLDFetcher interface for ethereum
+// it interfaces directly with PG-IPFS instead of going through a node-interface or remote node
+type IPLDPGFetcher struct {
+	db *postgres.DB
+}
+
+// NewIPLDPGFetcher creates a pointer to a new IPLDPGFetcher
+func NewIPLDPGFetcher(db *postgres.DB) *IPLDPGFetcher {
+	return &IPLDPGFetcher{
+		db: db,
+	}
+}
+
+// Fetch is the exported method for fetching and returning all the IPLDS specified in the CIDWrapper
+func (f *IPLDPGFetcher) Fetch(cids shared.CIDsForFetching) (shared.IPLDs, error) {
+	cidWrapper, ok := cids.(*CIDWrapper)
+	if !ok {
+		return nil, fmt.Errorf("btc fetcher: expected cids type %T got %T", &CIDWrapper{}, cids)
+	}
+	log.Debug("fetching iplds")
+	iplds := IPLDs{}
+	iplds.BlockNumber = cidWrapper.BlockNumber
+	var err error
+	tx, err := f.db.Beginx()
+	if err != nil {
+		return nil, err
+	}
+	iplds.Header, err = f.FetchHeader(tx, cidWrapper.Header)
+	if err != nil {
+		shared.Rollback(tx)
+		return nil, fmt.Errorf("btc pg fetcher: header fetching error: %s", err.Error())
+	}
+	iplds.Transactions, err = f.FetchTrxs(tx, cidWrapper.Transactions)
+	if err != nil {
+		shared.Rollback(tx)
+		return nil, fmt.Errorf("btc pg fetcher: transaction fetching error: %s", err.Error())
+	}
+	return iplds, tx.Commit()
+}
+
+// FetchHeaders fetches headers
+func (f *IPLDPGFetcher) FetchHeader(tx *sqlx.Tx, c HeaderModel) (ipfs.BlockModel, error) {
+	log.Debug("fetching header ipld")
+	headerBytes, err := shared.FetchIPLD(tx, c.CID)
+	if err != nil {
+		return ipfs.BlockModel{}, err
+	}
+	return ipfs.BlockModel{
+		Data: headerBytes,
+		CID:  c.CID,
+	}, nil
+}
+
+// FetchTrxs fetches transactions
+func (f *IPLDPGFetcher) FetchTrxs(tx *sqlx.Tx, cids []TxModel) ([]ipfs.BlockModel, error) {
+	log.Debug("fetching transaction iplds")
+	trxIPLDs := make([]ipfs.BlockModel, len(cids))
+	for i, c := range cids {
+		trxBytes, err := shared.FetchIPLD(tx, c.CID)
+		if err != nil {
+			return nil, err
+		}
+		trxIPLDs[i] = ipfs.BlockModel{
+			Data: trxBytes,
+			CID:  c.CID,
+		}
+	}
+	return trxIPLDs, nil
+}

pkg/super_node/btc/publishAndIndexer_test.go

@@ -18,6 +18,7 @@ package btc_test
 
 import (
 	"bytes"
+	"fmt"
 
 	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-ipfs-blockstore"
@@ -111,6 +112,7 @@ var _ = Describe("PublishAndIndexer", func() {
 				Expect(err).ToNot(HaveOccurred())
 				mhKey := dshelp.CidToDsKey(dc)
 				prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
+				fmt.Printf("mhKey: %s\r\n", prefixedKey)
 				var data []byte
 				err = db.Get(&data, ipfsPgGet, prefixedKey)
 				Expect(err).ToNot(HaveOccurred())