api test; return full ipld models to subscribers

pkg/super_node/btc/filterer.go

@@ -21,6 +21,10 @@ import (
 	"fmt"
 	"math/big"
 
+	"github.com/multiformats/go-multihash"
+
+	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
+	"github.com/vulcanize/vulcanizedb/pkg/ipfs/ipld"
 	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
 )
 
@@ -63,7 +67,15 @@ func (s *ResponseFilterer) filterHeaders(headerFilter HeaderFilter, response *IP
 		if err := payload.Header.Serialize(headerBuffer); err != nil {
 			return err
 		}
-		response.Headers = append(response.Headers, headerBuffer.Bytes())
+		data := headerBuffer.Bytes()
+		cid, err := ipld.RawdataToCid(ipld.MBitcoinHeader, data, multihash.DBL_SHA2_256)
+		if err != nil {
+			return err
+		}
+		response.Headers = append(response.Headers, ipfs.BlockModel{
+			Data: data,
+			CID:  cid.String(),
+		})
 	}
 	return nil
 }
@@ -77,14 +89,22 @@ func checkRange(start, end, actual int64) bool {
 
 func (s *ResponseFilterer) filterTransactions(trxFilter TxFilter, response *IPLDs, payload ConvertedPayload) error {
 	if !trxFilter.Off {
-		response.Transactions = make([][]byte, 0, len(payload.TxMetaData))
+		response.Transactions = make([]ipfs.BlockModel, 0, len(payload.TxMetaData))
 		for i, txMeta := range payload.TxMetaData {
 			if checkTransaction(txMeta, trxFilter) {
 				trxBuffer := new(bytes.Buffer)
 				if err := payload.Txs[i].MsgTx().Serialize(trxBuffer); err != nil {
 					return err
 				}
-				response.Transactions = append(response.Transactions, trxBuffer.Bytes())
+				data := trxBuffer.Bytes()
+				cid, err := ipld.RawdataToCid(ipld.MBitcoinTx, data, multihash.DBL_SHA2_256)
+				if err != nil {
+					return err
+				}
+				response.Transactions = append(response.Transactions, ipfs.BlockModel{
+					Data: data,
+					CID:  cid.String(),
+				})
 			}
 		}
 	}

pkg/super_node/btc/ipld_fetcher.go

@@ -21,14 +21,13 @@ import (
 	"errors"
 	"fmt"
 
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
-
 	"github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-blockservice"
 	"github.com/ipfs/go-cid"
 	log "github.com/sirupsen/logrus"
 
 	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
 )
 
 var (
@@ -74,7 +73,7 @@ func (f *IPLDFetcher) Fetch(cids shared.CIDsForFetching) (shared.IPLDs, error) {
 
 // FetchHeaders fetches headers
 // It uses the f.fetchBatch method
-func (f *IPLDFetcher) FetchHeaders(cids []HeaderModel) ([][]byte, error) {
+func (f *IPLDFetcher) FetchHeaders(cids []HeaderModel) ([]ipfs.BlockModel, error) {
 	log.Debug("fetching header iplds")
 	headerCids := make([]cid.Cid, len(cids))
 	for i, c := range cids {
@@ -85,20 +84,23 @@ func (f *IPLDFetcher) FetchHeaders(cids []HeaderModel) ([][]byte, error) {
 		headerCids[i] = dc
 	}
 	headers := f.fetchBatch(headerCids)
-	headersBytes := make([][]byte, len(headers))
+	headerIPLDs := make([]ipfs.BlockModel, len(headers))
 	for i, header := range headers {
-		headersBytes[i] = header.RawData()
+		headerIPLDs[i] = ipfs.BlockModel{
+			Data: header.RawData(),
+			CID:  header.Cid().String(),
+		}
 	}
-	if len(headersBytes) != len(headerCids) {
+	if len(headerIPLDs) != len(headerCids) {
 		log.Errorf("ipfs fetcher: number of header blocks returned (%d) does not match number expected (%d)", len(headers), len(headerCids))
-		return headersBytes, errUnexpectedNumberOfIPLDs
+		return headerIPLDs, errUnexpectedNumberOfIPLDs
 	}
-	return headersBytes, nil
+	return headerIPLDs, nil
 }
 
 // FetchTrxs fetches transactions
 // It uses the f.fetchBatch method
-func (f *IPLDFetcher) FetchTrxs(cids []TxModel) ([][]byte, error) {
+func (f *IPLDFetcher) FetchTrxs(cids []TxModel) ([]ipfs.BlockModel, error) {
 	log.Debug("fetching transaction iplds")
 	trxCids := make([]cid.Cid, len(cids))
 	for i, c := range cids {
@@ -109,15 +111,18 @@ func (f *IPLDFetcher) FetchTrxs(cids []TxModel) ([][]byte, error) {
 		trxCids[i] = dc
 	}
 	trxs := f.fetchBatch(trxCids)
-	trxBytes := make([][]byte, len(trxs))
+	trxIPLDs := make([]ipfs.BlockModel, len(trxs))
 	for i, trx := range trxs {
-		trxBytes[i] = trx.RawData()
+		trxIPLDs[i] = ipfs.BlockModel{
+			Data: trx.RawData(),
+			CID:  trx.Cid().String(),
+		}
 	}
-	if len(trxBytes) != len(trxCids) {
+	if len(trxIPLDs) != len(trxCids) {
 		log.Errorf("ipfs fetcher: number of transaction blocks returned (%d) does not match number expected (%d)", len(trxs), len(trxCids))
-		return trxBytes, errUnexpectedNumberOfIPLDs
+		return trxIPLDs, errUnexpectedNumberOfIPLDs
 	}
-	return trxBytes, nil
+	return trxIPLDs, nil
 }
 
 // fetch is used to fetch a single cid

pkg/super_node/btc/types.go

@@ -19,6 +19,8 @@ package btc
 import (
 	"math/big"
 
+	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
+
 	"github.com/btcsuite/btcd/wire"
 	"github.com/btcsuite/btcutil"
 )
@@ -64,8 +66,8 @@ type CIDWrapper struct {
 // Returned by IPLDFetcher and ResponseFilterer
 type IPLDs struct {
 	BlockNumber  *big.Int
-	Headers      [][]byte
-	Transactions [][]byte
+	Headers      []ipfs.BlockModel
+	Transactions []ipfs.BlockModel
 }
 
 // Height satisfies the StreamedIPLDs interface