trimming down to ipfs watchers

pkg/btc/btc_suite_test.go

@@ -0,0 +1,35 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package btc_test
+
+import (
+	"io/ioutil"
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/sirupsen/logrus"
+)
+
+func TestBTCSuperNode(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Super Node BTC Suite Test")
+}
+
+var _ = BeforeSuite(func() {
+	logrus.SetOutput(ioutil.Discard)
+})

pkg/btc/cid_retriever.go

@@ -0,0 +1,302 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package btc
+
+import (
+	"database/sql"
+	"fmt"
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/jmoiron/sqlx"
+	"github.com/lib/pq"
+	log "github.com/sirupsen/logrus"
+
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/postgres"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
+	"github.com/vulcanize/ipfs-chain-watcher/utils"
+)
+
+// CIDRetriever satisfies the CIDRetriever interface for bitcoin
+type CIDRetriever struct {
+	db *postgres.DB
+}
+
+// NewCIDRetriever returns a pointer to a new CIDRetriever which supports the CIDRetriever interface
+func NewCIDRetriever(db *postgres.DB) *CIDRetriever {
+	return &CIDRetriever{
+		db: db,
+	}
+}
+
+// RetrieveFirstBlockNumber is used to retrieve the first block number in the db
+func (bcr *CIDRetriever) RetrieveFirstBlockNumber() (int64, error) {
+	var blockNumber int64
+	err := bcr.db.Get(&blockNumber, "SELECT block_number FROM btc.header_cids ORDER BY block_number ASC LIMIT 1")
+	return blockNumber, err
+}
+
+// RetrieveLastBlockNumber is used to retrieve the latest block number in the db
+func (bcr *CIDRetriever) RetrieveLastBlockNumber() (int64, error) {
+	var blockNumber int64
+	err := bcr.db.Get(&blockNumber, "SELECT block_number FROM btc.header_cids ORDER BY block_number DESC LIMIT 1 ")
+	return blockNumber, err
+}
+
+// Retrieve is used to retrieve all of the CIDs which conform to the passed StreamFilters
+func (bcr *CIDRetriever) Retrieve(filter shared.SubscriptionSettings, blockNumber int64) ([]shared.CIDsForFetching, bool, error) {
+	streamFilter, ok := filter.(*SubscriptionSettings)
+	if !ok {
+		return nil, true, fmt.Errorf("btc retriever expected filter type %T got %T", &SubscriptionSettings{}, filter)
+	}
+	log.Debug("retrieving cids")
+
+	// Begin new db tx
+	tx, err := bcr.db.Beginx()
+	if err != nil {
+		return nil, true, err
+	}
+	defer func() {
+		if p := recover(); p != nil {
+			shared.Rollback(tx)
+			panic(p)
+		} else if err != nil {
+			shared.Rollback(tx)
+		} else {
+			err = tx.Commit()
+		}
+	}()
+
+	// Retrieve cached header CIDs
+	headers, err := bcr.RetrieveHeaderCIDs(tx, blockNumber)
+	if err != nil {
+		log.Error("header cid retrieval error")
+		return nil, true, err
+	}
+	cws := make([]shared.CIDsForFetching, len(headers))
+	empty := true
+	for i, header := range headers {
+		cw := new(CIDWrapper)
+		cw.BlockNumber = big.NewInt(blockNumber)
+		if !streamFilter.HeaderFilter.Off {
+			cw.Header = header
+			empty = false
+		}
+		// Retrieve cached trx CIDs
+		if !streamFilter.TxFilter.Off {
+			cw.Transactions, err = bcr.RetrieveTxCIDs(tx, streamFilter.TxFilter, header.ID)
+			if err != nil {
+				log.Error("transaction cid retrieval error")
+				return nil, true, err
+			}
+			if len(cw.Transactions) > 0 {
+				empty = false
+			}
+		}
+		cws[i] = cw
+	}
+
+	return cws, empty, err
+}
+
+// RetrieveHeaderCIDs retrieves and returns all of the header cids at the provided blockheight
+func (bcr *CIDRetriever) RetrieveHeaderCIDs(tx *sqlx.Tx, blockNumber int64) ([]HeaderModel, error) {
+	log.Debug("retrieving header cids for block ", blockNumber)
+	headers := make([]HeaderModel, 0)
+	pgStr := `SELECT * FROM btc.header_cids
+				WHERE block_number = $1`
+	return headers, tx.Select(&headers, pgStr, blockNumber)
+}
+
+// RetrieveTxCIDs retrieves and returns all of the trx cids at the provided blockheight that conform to the provided filter parameters
+// also returns the ids for the returned transaction cids
+func (bcr *CIDRetriever) RetrieveTxCIDs(tx *sqlx.Tx, txFilter TxFilter, headerID int64) ([]TxModel, error) {
+	log.Debug("retrieving transaction cids for header id ", headerID)
+	args := make([]interface{}, 0, 3)
+	results := make([]TxModel, 0)
+	id := 1
+	pgStr := fmt.Sprintf(`SELECT transaction_cids.id, transaction_cids.header_id,
+ 			transaction_cids.tx_hash, transaction_cids.cid,
+ 			transaction_cids.segwit, transaction_cids.witness_hash, transaction_cids.index
+ 			FROM btc.transaction_cids, btc.header_cids, btc.tx_inputs, btc.tx_outputs
+			WHERE transaction_cids.header_id = header_cids.id
+			AND tx_inputs.tx_id = transaction_cids.id
+			AND tx_outputs.tx_id = transaction_cids.id
+			AND header_cids.id = $%d`, id)
+	args = append(args, headerID)
+	id++
+	if txFilter.Segwit {
+		pgStr += ` AND transaction_cids.segwit = true`
+	}
+	if txFilter.MultiSig {
+		pgStr += ` AND tx_outputs.required_sigs > 1`
+	}
+	if len(txFilter.WitnessHashes) > 0 {
+		pgStr += fmt.Sprintf(` AND transaction_cids.witness_hash = ANY($%d::VARCHAR(66)[])`, id)
+		args = append(args, pq.Array(txFilter.WitnessHashes))
+		id++
+	}
+	if len(txFilter.Addresses) > 0 {
+		pgStr += fmt.Sprintf(` AND tx_outputs.addresses && $%d::VARCHAR(66)[]`, id)
+		args = append(args, pq.Array(txFilter.Addresses))
+		id++
+	}
+	if len(txFilter.Indexes) > 0 {
+		pgStr += fmt.Sprintf(` AND transaction_cids.index = ANY($%d::INTEGER[])`, id)
+		args = append(args, pq.Array(txFilter.Indexes))
+		id++
+	}
+	if len(txFilter.PkScriptClasses) > 0 {
+		pgStr += fmt.Sprintf(` AND tx_outputs.script_class = ANY($%d::INTEGER[])`, id)
+		args = append(args, pq.Array(txFilter.PkScriptClasses))
+	}
+	return results, tx.Select(&results, pgStr, args...)
+}
+
+// RetrieveGapsInData is used to find the the block numbers at which we are missing data in the db
+func (bcr *CIDRetriever) RetrieveGapsInData(validationLevel int) ([]shared.Gap, error) {
+	log.Info("searching for gaps in the btc super node database")
+	startingBlock, err := bcr.RetrieveFirstBlockNumber()
+	if err != nil {
+		return nil, fmt.Errorf("btc CIDRetriever RetrieveFirstBlockNumber error: %v", err)
+	}
+	var initialGap []shared.Gap
+	if startingBlock != 0 {
+		stop := uint64(startingBlock - 1)
+		log.Infof("found gap at the beginning of the btc sync from 0 to %d", stop)
+		initialGap = []shared.Gap{{
+			Start: 0,
+			Stop:  stop,
+		}}
+	}
+
+	pgStr := `SELECT header_cids.block_number + 1 AS start, min(fr.block_number) - 1 AS stop FROM btc.header_cids
+				LEFT JOIN btc.header_cids r on btc.header_cids.block_number = r.block_number - 1
+				LEFT JOIN btc.header_cids fr on btc.header_cids.block_number < fr.block_number
+				WHERE r.block_number is NULL and fr.block_number IS NOT NULL
+				GROUP BY header_cids.block_number, r.block_number`
+	results := make([]struct {
+		Start uint64 `db:"start"`
+		Stop  uint64 `db:"stop"`
+	}, 0)
+	if err := bcr.db.Select(&results, pgStr); err != nil && err != sql.ErrNoRows {
+		return nil, err
+	}
+	emptyGaps := make([]shared.Gap, len(results))
+	for i, res := range results {
+		emptyGaps[i] = shared.Gap{
+			Start: res.Start,
+			Stop:  res.Stop,
+		}
+	}
+
+	// Find sections of blocks where we are below the validation level
+	// There will be no overlap between these "gaps" and the ones above
+	pgStr = `SELECT block_number FROM btc.header_cids
+			WHERE times_validated < $1
+			ORDER BY block_number`
+	var heights []uint64
+	if err := bcr.db.Select(&heights, pgStr, validationLevel); err != nil && err != sql.ErrNoRows {
+		return nil, err
+	}
+	return append(append(initialGap, emptyGaps...), utils.MissingHeightsToGaps(heights)...), nil
+}
+
+// RetrieveBlockByHash returns all of the CIDs needed to compose an entire block, for a given block hash
+func (bcr *CIDRetriever) RetrieveBlockByHash(blockHash common.Hash) (HeaderModel, []TxModel, error) {
+	log.Debug("retrieving block cids for block hash ", blockHash.String())
+
+	// Begin new db tx
+	tx, err := bcr.db.Beginx()
+	if err != nil {
+		return HeaderModel{}, nil, err
+	}
+	defer func() {
+		if p := recover(); p != nil {
+			shared.Rollback(tx)
+			panic(p)
+		} else if err != nil {
+			shared.Rollback(tx)
+		} else {
+			err = tx.Commit()
+		}
+	}()
+
+	headerCID, err := bcr.RetrieveHeaderCIDByHash(tx, blockHash)
+	if err != nil {
+		log.Error("header cid retrieval error")
+		return HeaderModel{}, nil, err
+	}
+	txCIDs, err := bcr.RetrieveTxCIDsByHeaderID(tx, headerCID.ID)
+	if err != nil {
+		log.Error("tx cid retrieval error")
+	}
+	return headerCID, txCIDs, err
+}
+
+// RetrieveBlockByNumber returns all of the CIDs needed to compose an entire block, for a given block number
+func (bcr *CIDRetriever) RetrieveBlockByNumber(blockNumber int64) (HeaderModel, []TxModel, error) {
+	log.Debug("retrieving block cids for block number ", blockNumber)
+
+	// Begin new db tx
+	tx, err := bcr.db.Beginx()
+	if err != nil {
+		return HeaderModel{}, nil, err
+	}
+	defer func() {
+		if p := recover(); p != nil {
+			shared.Rollback(tx)
+			panic(p)
+		} else if err != nil {
+			shared.Rollback(tx)
+		} else {
+			err = tx.Commit()
+		}
+	}()
+
+	headerCID, err := bcr.RetrieveHeaderCIDs(tx, blockNumber)
+	if err != nil {
+		log.Error("header cid retrieval error")
+		return HeaderModel{}, nil, err
+	}
+	if len(headerCID) < 1 {
+		return HeaderModel{}, nil, fmt.Errorf("header cid retrieval error, no header CIDs found at block %d", blockNumber)
+	}
+	txCIDs, err := bcr.RetrieveTxCIDsByHeaderID(tx, headerCID[0].ID)
+	if err != nil {
+		log.Error("tx cid retrieval error")
+	}
+	return headerCID[0], txCIDs, err
+}
+
+// RetrieveHeaderCIDByHash returns the header for the given block hash
+func (bcr *CIDRetriever) RetrieveHeaderCIDByHash(tx *sqlx.Tx, blockHash common.Hash) (HeaderModel, error) {
+	log.Debug("retrieving header cids for block hash ", blockHash.String())
+	pgStr := `SELECT * FROM btc.header_cids
+			WHERE block_hash = $1`
+	var headerCID HeaderModel
+	return headerCID, tx.Get(&headerCID, pgStr, blockHash.String())
+}
+
+// RetrieveTxCIDsByHeaderID retrieves all tx CIDs for the given header id
+func (bcr *CIDRetriever) RetrieveTxCIDsByHeaderID(tx *sqlx.Tx, headerID int64) ([]TxModel, error) {
+	log.Debug("retrieving tx cids for block id ", headerID)
+	pgStr := `SELECT * FROM btc.transaction_cids
+			WHERE header_id = $1`
+	var txCIDs []TxModel
+	return txCIDs, tx.Select(&txCIDs, pgStr, headerID)
+}

pkg/btc/cleaner.go

@@ -0,0 +1,193 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package btc
+
+import (
+	"fmt"
+
+	"github.com/jmoiron/sqlx"
+	"github.com/sirupsen/logrus"
+
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/postgres"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
+)
+
+// Cleaner satisfies the shared.Cleaner interface fo bitcoin
+type Cleaner struct {
+	db *postgres.DB
+}
+
+// NewCleaner returns a new Cleaner struct that satisfies the shared.Cleaner interface
+func NewCleaner(db *postgres.DB) *Cleaner {
+	return &Cleaner{
+		db: db,
+	}
+}
+
+// ResetValidation resets the validation level to 0 to enable revalidation
+func (c *Cleaner) ResetValidation(rngs [][2]uint64) error {
+	tx, err := c.db.Beginx()
+	if err != nil {
+		return err
+	}
+	for _, rng := range rngs {
+		logrus.Infof("btc db cleaner resetting validation level to 0 for block range %d to %d", rng[0], rng[1])
+		pgStr := `UPDATE btc.header_cids
+				SET times_validated = 0
+				WHERE block_number BETWEEN $1 AND $2`
+		if _, err := tx.Exec(pgStr, rng[0], rng[1]); err != nil {
+			shared.Rollback(tx)
+			return err
+		}
+	}
+	return tx.Commit()
+}
+
+// Clean removes the specified data from the db within the provided block range
+func (c *Cleaner) Clean(rngs [][2]uint64, t shared.DataType) error {
+	tx, err := c.db.Beginx()
+	if err != nil {
+		return err
+	}
+	for _, rng := range rngs {
+		logrus.Infof("btc db cleaner cleaning up block range %d to %d", rng[0], rng[1])
+		if err := c.clean(tx, rng, t); err != nil {
+			shared.Rollback(tx)
+			return err
+		}
+	}
+	if err := tx.Commit(); err != nil {
+		return err
+	}
+	logrus.Infof("btc db cleaner vacuum analyzing cleaned tables to free up space from deleted rows")
+	return c.vacuumAnalyze(t)
+}
+
+func (c *Cleaner) clean(tx *sqlx.Tx, rng [2]uint64, t shared.DataType) error {
+	switch t {
+	case shared.Full, shared.Headers:
+		return c.cleanFull(tx, rng)
+	case shared.Transactions:
+		if err := c.cleanTransactionIPLDs(tx, rng); err != nil {
+			return err
+		}
+		return c.cleanTransactionMetaData(tx, rng)
+	default:
+		return fmt.Errorf("btc cleaner unrecognized type: %s", t.String())
+	}
+}
+
+func (c *Cleaner) vacuumAnalyze(t shared.DataType) error {
+	switch t {
+	case shared.Full, shared.Headers:
+		if err := c.vacuumHeaders(); err != nil {
+			return err
+		}
+		if err := c.vacuumTxs(); err != nil {
+			return err
+		}
+		if err := c.vacuumTxInputs(); err != nil {
+			return err
+		}
+		if err := c.vacuumTxOutputs(); err != nil {
+			return err
+		}
+	case shared.Transactions:
+		if err := c.vacuumTxs(); err != nil {
+			return err
+		}
+		if err := c.vacuumTxInputs(); err != nil {
+			return err
+		}
+		if err := c.vacuumTxOutputs(); err != nil {
+			return err
+		}
+	default:
+		return fmt.Errorf("btc cleaner unrecognized type: %s", t.String())
+	}
+	return c.vacuumIPLDs()
+}
+
+func (c *Cleaner) vacuumHeaders() error {
+	_, err := c.db.Exec(`VACUUM ANALYZE btc.header_cids`)
+	return err
+}
+
+func (c *Cleaner) vacuumTxs() error {
+	_, err := c.db.Exec(`VACUUM ANALYZE btc.transaction_cids`)
+	return err
+}
+
+func (c *Cleaner) vacuumTxInputs() error {
+	_, err := c.db.Exec(`VACUUM ANALYZE btc.tx_inputs`)
+	return err
+}
+
+func (c *Cleaner) vacuumTxOutputs() error {
+	_, err := c.db.Exec(`VACUUM ANALYZE btc.tx_outputs`)
+	return err
+}
+
+func (c *Cleaner) vacuumIPLDs() error {
+	_, err := c.db.Exec(`VACUUM ANALYZE public.blocks`)
+	return err
+}
+
+func (c *Cleaner) cleanFull(tx *sqlx.Tx, rng [2]uint64) error {
+	if err := c.cleanTransactionIPLDs(tx, rng); err != nil {
+		return err
+	}
+	if err := c.cleanHeaderIPLDs(tx, rng); err != nil {
+		return err
+	}
+	return c.cleanHeaderMetaData(tx, rng)
+}
+
+func (c *Cleaner) cleanTransactionIPLDs(tx *sqlx.Tx, rng [2]uint64) error {
+	pgStr := `DELETE FROM public.blocks A
+			USING btc.transaction_cids B, btc.header_cids C
+			WHERE A.key = B.cid
+			AND B.header_id = C.id
+			AND C.block_number BETWEEN $1 AND $2`
+	_, err := tx.Exec(pgStr, rng[0], rng[1])
+	return err
+}
+
+func (c *Cleaner) cleanTransactionMetaData(tx *sqlx.Tx, rng [2]uint64) error {
+	pgStr := `DELETE FROM btc.transaction_cids A
+			USING btc.header_cids B
+			WHERE A.header_id = B.id
+			AND B.block_number BETWEEN $1 AND $2`
+	_, err := tx.Exec(pgStr, rng[0], rng[1])
+	return err
+}
+
+func (c *Cleaner) cleanHeaderIPLDs(tx *sqlx.Tx, rng [2]uint64) error {
+	pgStr := `DELETE FROM public.blocks A
+			USING btc.header_cids B
+			WHERE A.key = B.cid
+			AND B.block_number BETWEEN $1 AND $2`
+	_, err := tx.Exec(pgStr, rng[0], rng[1])
+	return err
+}
+
+func (c *Cleaner) cleanHeaderMetaData(tx *sqlx.Tx, rng [2]uint64) error {
+	pgStr := `DELETE FROM btc.header_cids
+			WHERE block_number BETWEEN $1 AND $2`
+	_, err := tx.Exec(pgStr, rng[0], rng[1])
+	return err
+}

pkg/btc/cleaner_test.go

@@ -0,0 +1,340 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package btc_test
+
+import (
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/crypto"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/btc"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/postgres"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
+)
+
+var (
+	// Block 0
+	// header variables
+	blockHash1   = crypto.Keccak256Hash([]byte{00, 02})
+	blocKNumber1 = big.NewInt(0)
+	headerCid1   = "mockHeader1CID"
+	parentHash   = crypto.Keccak256Hash([]byte{00, 01})
+	headerModel1 = btc.HeaderModel{
+		BlockHash:   blockHash1.String(),
+		BlockNumber: blocKNumber1.String(),
+		ParentHash:  parentHash.String(),
+		CID:         headerCid1,
+	}
+
+	// tx variables
+	tx1CID    = "mockTx1CID"
+	tx2CID    = "mockTx2CID"
+	tx1Hash   = crypto.Keccak256Hash([]byte{01, 01})
+	tx2Hash   = crypto.Keccak256Hash([]byte{01, 02})
+	opHash    = crypto.Keccak256Hash([]byte{02, 01})
+	txModels1 = []btc.TxModelWithInsAndOuts{
+		{
+			Index:  0,
+			CID:    tx1CID,
+			TxHash: tx1Hash.String(),
+			SegWit: true,
+			TxInputs: []btc.TxInput{
+				{
+					Index:                 0,
+					TxWitness:             []string{"mockWitness"},
+					SignatureScript:       []byte{01},
+					PreviousOutPointIndex: 0,
+					PreviousOutPointHash:  opHash.String(),
+				},
+			},
+			TxOutputs: []btc.TxOutput{
+				{
+					Index:        0,
+					Value:        50000000,
+					PkScript:     []byte{02},
+					ScriptClass:  0,
+					RequiredSigs: 1,
+				},
+			},
+		},
+		{
+			Index:  1,
+			CID:    tx2CID,
+			TxHash: tx2Hash.String(),
+			SegWit: true,
+		},
+	}
+	mockCIDPayload1 = &btc.CIDPayload{
+		HeaderCID:       headerModel1,
+		TransactionCIDs: txModels1,
+	}
+
+	// Block 1
+	// header variables
+	blockHash2   = crypto.Keccak256Hash([]byte{00, 03})
+	blocKNumber2 = big.NewInt(1)
+	headerCid2   = "mockHeaderCID2"
+	headerModel2 = btc.HeaderModel{
+		BlockNumber: blocKNumber2.String(),
+		BlockHash:   blockHash2.String(),
+		ParentHash:  blockHash1.String(),
+		CID:         headerCid2,
+	}
+
+	// tx variables
+	tx3CID    = "mockTx3CID"
+	tx3Hash   = crypto.Keccak256Hash([]byte{01, 03})
+	txModels2 = []btc.TxModelWithInsAndOuts{
+		{
+			Index:  0,
+			CID:    tx3CID,
+			TxHash: tx3Hash.String(),
+			SegWit: true,
+		},
+	}
+	mockCIDPayload2 = &btc.CIDPayload{
+		HeaderCID:       headerModel2,
+		TransactionCIDs: txModels2,
+	}
+	rngs = [][2]uint64{{0, 1}}
+	cids = []string{
+		headerCid1,
+		headerCid2,
+		tx1CID,
+		tx2CID,
+		tx3CID,
+	}
+	mockData = []byte{'\x01'}
+)
+
+var _ = Describe("Cleaner", func() {
+	var (
+		db      *postgres.DB
+		repo    *btc.CIDIndexer
+		cleaner *btc.Cleaner
+	)
+	BeforeEach(func() {
+		var err error
+		db, err = shared.SetupDB()
+		Expect(err).ToNot(HaveOccurred())
+		repo = btc.NewCIDIndexer(db)
+		cleaner = btc.NewCleaner(db)
+	})
+
+	Describe("Clean", func() {
+		BeforeEach(func() {
+			err := repo.Index(mockCIDPayload1)
+			Expect(err).ToNot(HaveOccurred())
+			err = repo.Index(mockCIDPayload2)
+			Expect(err).ToNot(HaveOccurred())
+
+			for _, cid := range cids {
+				_, err = db.Exec(`INSERT INTO public.blocks (key, data) VALUES ($1, $2)`, cid, mockData)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			tx, err := db.Beginx()
+			Expect(err).ToNot(HaveOccurred())
+			var startingIPFSBlocksCount int
+			pgStr := `SELECT COUNT(*) FROM public.blocks`
+			err = tx.Get(&startingIPFSBlocksCount, pgStr)
+			Expect(err).ToNot(HaveOccurred())
+			var startingTxCount int
+			pgStr = `SELECT COUNT(*) FROM btc.transaction_cids`
+			err = tx.Get(&startingTxCount, pgStr)
+			Expect(err).ToNot(HaveOccurred())
+			var startingHeaderCount int
+			pgStr = `SELECT COUNT(*) FROM btc.header_cids`
+			err = tx.Get(&startingHeaderCount, pgStr)
+			Expect(err).ToNot(HaveOccurred())
+
+			err = tx.Commit()
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(startingIPFSBlocksCount).To(Equal(5))
+			Expect(startingTxCount).To(Equal(3))
+			Expect(startingHeaderCount).To(Equal(2))
+		})
+		AfterEach(func() {
+			btc.TearDownDB(db)
+		})
+		It("Cleans everything", func() {
+			err := cleaner.Clean(rngs, shared.Full)
+			Expect(err).ToNot(HaveOccurred())
+
+			tx, err := db.Beginx()
+			Expect(err).ToNot(HaveOccurred())
+			var txCount int
+			pgStr := `SELECT COUNT(*) FROM btc.transaction_cids`
+			err = tx.Get(&txCount, pgStr)
+			Expect(err).ToNot(HaveOccurred())
+			var txInCount int
+			pgStr = `SELECT COUNT(*) FROM btc.tx_inputs`
+			err = tx.Get(&txInCount, pgStr)
+			Expect(err).ToNot(HaveOccurred())
+			var txOutCount int
+			pgStr = `SELECT COUNT(*) FROM btc.tx_outputs`
+			err = tx.Get(&txOutCount, pgStr)
+			Expect(err).ToNot(HaveOccurred())
+			var headerCount int
+			pgStr = `SELECT COUNT(*) FROM btc.header_cids`
+			err = tx.Get(&headerCount, pgStr)
+			Expect(err).ToNot(HaveOccurred())
+			var blocksCount int
+			pgStr = `SELECT COUNT(*) FROM public.blocks`
+			err = tx.Get(&blocksCount, pgStr)
+			Expect(err).ToNot(HaveOccurred())
+
+			err = tx.Commit()
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(blocksCount).To(Equal(0))
+			Expect(txCount).To(Equal(0))
+			Expect(txInCount).To(Equal(0))
+			Expect(txOutCount).To(Equal(0))
+			Expect(headerCount).To(Equal(0))
+		})
+		It("Cleans headers and all linked data", func() {
+			err := cleaner.Clean(rngs, shared.Headers)
+			Expect(err).ToNot(HaveOccurred())
+
+			tx, err := db.Beginx()
+			Expect(err).ToNot(HaveOccurred())
+			var txCount int
+			pgStr := `SELECT COUNT(*) FROM btc.transaction_cids`
+			err = tx.Get(&txCount, pgStr)
+			Expect(err).ToNot(HaveOccurred())
+			var txInCount int
+			pgStr = `SELECT COUNT(*) FROM btc.tx_inputs`
+			err = tx.Get(&txInCount, pgStr)
+			Expect(err).ToNot(HaveOccurred())
+			var txOutCount int
+			pgStr = `SELECT COUNT(*) FROM btc.tx_outputs`
+			err = tx.Get(&txOutCount, pgStr)
+			Expect(err).ToNot(HaveOccurred())
+			var headerCount int
+			pgStr = `SELECT COUNT(*) FROM btc.header_cids`
+			err = tx.Get(&headerCount, pgStr)
+			Expect(err).ToNot(HaveOccurred())
+			var blocksCount int
+			pgStr = `SELECT COUNT(*) FROM public.blocks`
+			err = tx.Get(&blocksCount, pgStr)
+			Expect(err).ToNot(HaveOccurred())
+
+			err = tx.Commit()
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(blocksCount).To(Equal(0))
+			Expect(txCount).To(Equal(0))
+			Expect(txInCount).To(Equal(0))
+			Expect(txOutCount).To(Equal(0))
+			Expect(headerCount).To(Equal(0))
+		})
+		It("Cleans transactions", func() {
+			err := cleaner.Clean(rngs, shared.Transactions)
+			Expect(err).ToNot(HaveOccurred())
+
+			tx, err := db.Beginx()
+			Expect(err).ToNot(HaveOccurred())
+			var txCount int
+			pgStr := `SELECT COUNT(*) FROM btc.transaction_cids`
+			err = tx.Get(&txCount, pgStr)
+			Expect(err).ToNot(HaveOccurred())
+			var txInCount int
+			pgStr = `SELECT COUNT(*) FROM btc.tx_inputs`
+			err = tx.Get(&txInCount, pgStr)
+			Expect(err).ToNot(HaveOccurred())
+			var txOutCount int
+			pgStr = `SELECT COUNT(*) FROM btc.tx_outputs`
+			err = tx.Get(&txOutCount, pgStr)
+			Expect(err).ToNot(HaveOccurred())
+			var headerCount int
+			pgStr = `SELECT COUNT(*) FROM btc.header_cids`
+			err = tx.Get(&headerCount, pgStr)
+			Expect(err).ToNot(HaveOccurred())
+			var blocksCount int
+			pgStr = `SELECT COUNT(*) FROM public.blocks`
+			err = tx.Get(&blocksCount, pgStr)
+			Expect(err).ToNot(HaveOccurred())
+
+			err = tx.Commit()
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(blocksCount).To(Equal(2))
+			Expect(txCount).To(Equal(0))
+			Expect(txInCount).To(Equal(0))
+			Expect(txOutCount).To(Equal(0))
+			Expect(headerCount).To(Equal(2))
+		})
+	})
+
+	Describe("ResetValidation", func() {
+		BeforeEach(func() {
+			err := repo.Index(mockCIDPayload1)
+			Expect(err).ToNot(HaveOccurred())
+			err = repo.Index(mockCIDPayload2)
+			Expect(err).ToNot(HaveOccurred())
+
+			var validationTimes []int
+			pgStr := `SELECT times_validated FROM btc.header_cids`
+			err = db.Select(&validationTimes, pgStr)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(validationTimes)).To(Equal(2))
+			Expect(validationTimes[0]).To(Equal(1))
+			Expect(validationTimes[1]).To(Equal(1))
+
+			err = repo.Index(mockCIDPayload1)
+			Expect(err).ToNot(HaveOccurred())
+
+			validationTimes = []int{}
+			pgStr = `SELECT times_validated FROM btc.header_cids ORDER BY block_number`
+			err = db.Select(&validationTimes, pgStr)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(validationTimes)).To(Equal(2))
+			Expect(validationTimes[0]).To(Equal(2))
+			Expect(validationTimes[1]).To(Equal(1))
+		})
+		AfterEach(func() {
+			btc.TearDownDB(db)
+		})
+		It("Resets the validation level", func() {
+			err := cleaner.ResetValidation(rngs)
+			Expect(err).ToNot(HaveOccurred())
+
+			var validationTimes []int
+			pgStr := `SELECT times_validated FROM btc.header_cids`
+			err = db.Select(&validationTimes, pgStr)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(validationTimes)).To(Equal(2))
+			Expect(validationTimes[0]).To(Equal(0))
+			Expect(validationTimes[1]).To(Equal(0))
+
+			err = repo.Index(mockCIDPayload2)
+			Expect(err).ToNot(HaveOccurred())
+
+			validationTimes = []int{}
+			pgStr = `SELECT times_validated FROM btc.header_cids ORDER BY block_number`
+			err = db.Select(&validationTimes, pgStr)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(validationTimes)).To(Equal(2))
+			Expect(validationTimes[0]).To(Equal(0))
+			Expect(validationTimes[1]).To(Equal(1))
+		})
+	})
+})

pkg/btc/converter.go

@@ -0,0 +1,102 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package btc
+
+import (
+	"encoding/hex"
+	"fmt"
+
+	"github.com/btcsuite/btcd/chaincfg"
+	"github.com/btcsuite/btcd/txscript"
+
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
+)
+
+// PayloadConverter satisfies the PayloadConverter interface for bitcoin
+type PayloadConverter struct {
+	chainConfig *chaincfg.Params
+}
+
+// NewPayloadConverter creates a pointer to a new PayloadConverter which satisfies the PayloadConverter interface
+func NewPayloadConverter(chainConfig *chaincfg.Params) *PayloadConverter {
+	return &PayloadConverter{
+		chainConfig: chainConfig,
+	}
+}
+
+// Convert method is used to convert a bitcoin BlockPayload to an IPLDPayload
+// Satisfies the shared.PayloadConverter interface
+func (pc *PayloadConverter) Convert(payload shared.RawChainData) (shared.ConvertedData, error) {
+	btcBlockPayload, ok := payload.(BlockPayload)
+	if !ok {
+		return nil, fmt.Errorf("btc converter: expected payload type %T got %T", BlockPayload{}, payload)
+	}
+	txMeta := make([]TxModelWithInsAndOuts, len(btcBlockPayload.Txs))
+	for i, tx := range btcBlockPayload.Txs {
+		txModel := TxModelWithInsAndOuts{
+			TxHash:    tx.Hash().String(),
+			Index:     int64(i),
+			SegWit:    tx.HasWitness(),
+			TxOutputs: make([]TxOutput, len(tx.MsgTx().TxOut)),
+			TxInputs:  make([]TxInput, len(tx.MsgTx().TxIn)),
+		}
+		if tx.HasWitness() {
+			txModel.WitnessHash = tx.WitnessHash().String()
+		}
+		for i, in := range tx.MsgTx().TxIn {
+			txModel.TxInputs[i] = TxInput{
+				Index:                 int64(i),
+				SignatureScript:       in.SignatureScript,
+				PreviousOutPointHash:  in.PreviousOutPoint.Hash.String(),
+				PreviousOutPointIndex: in.PreviousOutPoint.Index,
+				TxWitness:             convertBytesToHexArray(in.Witness),
+			}
+		}
+		for i, out := range tx.MsgTx().TxOut {
+			scriptClass, addresses, numberOfSigs, err := txscript.ExtractPkScriptAddrs(out.PkScript, pc.chainConfig)
+			// if we receive an error but the txscript type isn't NonStandardTy then something went wrong
+			if err != nil && scriptClass != txscript.NonStandardTy {
+				return nil, err
+			}
+			stringAddrs := make([]string, len(addresses))
+			for i, addr := range addresses {
+				stringAddrs[i] = addr.EncodeAddress()
+			}
+			txModel.TxOutputs[i] = TxOutput{
+				Index:        int64(i),
+				Value:        out.Value,
+				PkScript:     out.PkScript,
+				RequiredSigs: int64(numberOfSigs),
+				ScriptClass:  uint8(scriptClass),
+				Addresses:    stringAddrs,
+			}
+		}
+		txMeta[i] = txModel
+	}
+	return ConvertedPayload{
+		BlockPayload: btcBlockPayload,
+		TxMetaData:   txMeta,
+	}, nil
+}
+
+func convertBytesToHexArray(bytea [][]byte) []string {
+	var strs []string
+	for _, b := range bytea {
+		strs = append(strs, hex.EncodeToString(b))
+	}
+	return strs
+}

pkg/btc/converter_test.go

@@ -0,0 +1,43 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package btc_test
+
+import (
+	"github.com/btcsuite/btcd/chaincfg"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/btc"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/btc/mocks"
+)
+
+var _ = Describe("Converter", func() {
+	Describe("Convert", func() {
+		It("Converts mock BlockPayloads into the expected IPLDPayloads", func() {
+			converter := btc.NewPayloadConverter(&chaincfg.MainNetParams)
+			payload, err := converter.Convert(mocks.MockBlockPayload)
+			Expect(err).ToNot(HaveOccurred())
+			convertedPayload, ok := payload.(btc.ConvertedPayload)
+			Expect(ok).To(BeTrue())
+			Expect(convertedPayload).To(Equal(mocks.MockConvertedPayload))
+			Expect(convertedPayload.BlockHeight).To(Equal(mocks.MockBlockHeight))
+			Expect(convertedPayload.Header).To(Equal(&mocks.MockBlock.Header))
+			Expect(convertedPayload.Txs).To(Equal(mocks.MockTransactions))
+			Expect(convertedPayload.TxMetaData).To(Equal(mocks.MockTxsMetaData))
+		})
+	})
+})

pkg/btc/filterer.go

@@ -0,0 +1,159 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package btc
+
+import (
+	"bytes"
+	"fmt"
+	"math/big"
+
+	"github.com/multiformats/go-multihash"
+
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/ipfs"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/ipfs/ipld"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
+)
+
+// ResponseFilterer satisfies the ResponseFilterer interface for bitcoin
+type ResponseFilterer struct{}
+
+// NewResponseFilterer creates a new Filterer satisfying the ResponseFilterer interface
+func NewResponseFilterer() *ResponseFilterer {
+	return &ResponseFilterer{}
+}
+
+// Filter is used to filter through btc data to extract and package requested data into a Payload
+func (s *ResponseFilterer) Filter(filter shared.SubscriptionSettings, payload shared.ConvertedData) (shared.IPLDs, error) {
+	btcFilters, ok := filter.(*SubscriptionSettings)
+	if !ok {
+		return IPLDs{}, fmt.Errorf("btc filterer expected filter type %T got %T", &SubscriptionSettings{}, filter)
+	}
+	btcPayload, ok := payload.(ConvertedPayload)
+	if !ok {
+		return IPLDs{}, fmt.Errorf("btc filterer expected payload type %T got %T", ConvertedPayload{}, payload)
+	}
+	height := int64(btcPayload.BlockPayload.BlockHeight)
+	if checkRange(btcFilters.Start.Int64(), btcFilters.End.Int64(), height) {
+		response := new(IPLDs)
+		if err := s.filterHeaders(btcFilters.HeaderFilter, response, btcPayload); err != nil {
+			return IPLDs{}, err
+		}
+		if err := s.filterTransactions(btcFilters.TxFilter, response, btcPayload); err != nil {
+			return IPLDs{}, err
+		}
+		response.BlockNumber = big.NewInt(height)
+		return *response, nil
+	}
+	return IPLDs{}, nil
+}
+
+func (s *ResponseFilterer) filterHeaders(headerFilter HeaderFilter, response *IPLDs, payload ConvertedPayload) error {
+	if !headerFilter.Off {
+		headerBuffer := new(bytes.Buffer)
+		if err := payload.Header.Serialize(headerBuffer); err != nil {
+			return err
+		}
+		data := headerBuffer.Bytes()
+		cid, err := ipld.RawdataToCid(ipld.MBitcoinHeader, data, multihash.DBL_SHA2_256)
+		if err != nil {
+			return err
+		}
+		response.Header = ipfs.BlockModel{
+			Data: data,
+			CID:  cid.String(),
+		}
+	}
+	return nil
+}
+
+func checkRange(start, end, actual int64) bool {
+	if (end <= 0 || end >= actual) && start <= actual {
+		return true
+	}
+	return false
+}
+
+func (s *ResponseFilterer) filterTransactions(trxFilter TxFilter, response *IPLDs, payload ConvertedPayload) error {
+	if !trxFilter.Off {
+		response.Transactions = make([]ipfs.BlockModel, 0, len(payload.TxMetaData))
+		for i, txMeta := range payload.TxMetaData {
+			if checkTransaction(txMeta, trxFilter) {
+				trxBuffer := new(bytes.Buffer)
+				if err := payload.Txs[i].MsgTx().Serialize(trxBuffer); err != nil {
+					return err
+				}
+				data := trxBuffer.Bytes()
+				cid, err := ipld.RawdataToCid(ipld.MBitcoinTx, data, multihash.DBL_SHA2_256)
+				if err != nil {
+					return err
+				}
+				response.Transactions = append(response.Transactions, ipfs.BlockModel{
+					Data: data,
+					CID:  cid.String(),
+				})
+			}
+		}
+	}
+	return nil
+}
+
+// checkTransaction returns true if the provided transaction has a hit on the filter
+func checkTransaction(txMeta TxModelWithInsAndOuts, txFilter TxFilter) bool {
+	passesSegwitFilter := false
+	if !txFilter.Segwit || (txFilter.Segwit && txMeta.SegWit) {
+		passesSegwitFilter = true
+	}
+	passesMultiSigFilter := !txFilter.MultiSig
+	if txFilter.MultiSig {
+		for _, out := range txMeta.TxOutputs {
+			if out.RequiredSigs > 1 {
+				passesMultiSigFilter = true
+			}
+		}
+	}
+	passesWitnessFilter := len(txFilter.WitnessHashes) == 0
+	for _, wantedWitnessHash := range txFilter.WitnessHashes {
+		if wantedWitnessHash == txMeta.WitnessHash {
+			passesWitnessFilter = true
+		}
+	}
+	passesAddressFilter := len(txFilter.Addresses) == 0
+	for _, wantedAddress := range txFilter.Addresses {
+		for _, out := range txMeta.TxOutputs {
+			for _, actualAddress := range out.Addresses {
+				if wantedAddress == actualAddress {
+					passesAddressFilter = true
+				}
+			}
+		}
+	}
+	passesIndexFilter := len(txFilter.Indexes) == 0
+	for _, wantedIndex := range txFilter.Indexes {
+		if wantedIndex == txMeta.Index {
+			passesIndexFilter = true
+		}
+	}
+	passesPkScriptClassFilter := len(txFilter.PkScriptClasses) == 0
+	for _, wantedPkScriptClass := range txFilter.PkScriptClasses {
+		for _, out := range txMeta.TxOutputs {
+			if out.ScriptClass == wantedPkScriptClass {
+				passesPkScriptClassFilter = true
+			}
+		}
+	}
+	return passesSegwitFilter && passesMultiSigFilter && passesWitnessFilter && passesAddressFilter && passesIndexFilter && passesPkScriptClassFilter
+}

pkg/btc/http_streamer.go

@@ -0,0 +1,104 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package btc
+
+import (
+	"bytes"
+	"time"
+
+	"github.com/btcsuite/btcd/rpcclient"
+	"github.com/sirupsen/logrus"
+
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
+)
+
+// HTTPPayloadStreamer satisfies the PayloadStreamer interface for bitcoin over http endpoints (since bitcoin core doesn't support websockets)
+type HTTPPayloadStreamer struct {
+	Config   *rpcclient.ConnConfig
+	lastHash []byte
+}
+
+// NewHTTPPayloadStreamer creates a pointer to a new PayloadStreamer which satisfies the PayloadStreamer interface for bitcoin
+func NewHTTPPayloadStreamer(clientConfig *rpcclient.ConnConfig) *HTTPPayloadStreamer {
+	return &HTTPPayloadStreamer{
+		Config: clientConfig,
+	}
+}
+
+// Stream is the main loop for subscribing to data from the btc block notifications
+// Satisfies the shared.PayloadStreamer interface
+func (ps *HTTPPayloadStreamer) Stream(payloadChan chan shared.RawChainData) (shared.ClientSubscription, error) {
+	logrus.Debug("streaming block payloads from btc")
+	client, err := rpcclient.New(ps.Config, nil)
+	if err != nil {
+		return nil, err
+	}
+	ticker := time.NewTicker(time.Second * 5)
+	errChan := make(chan error)
+	go func() {
+		for {
+			// start at
+			select {
+			case <-ticker.C:
+				height, err := client.GetBlockCount()
+				if err != nil {
+					errChan <- err
+					continue
+				}
+				blockHash, err := client.GetBlockHash(height)
+				if err != nil {
+					errChan <- err
+					continue
+				}
+				blockHashBytes := blockHash.CloneBytes()
+				if bytes.Equal(blockHashBytes, ps.lastHash) {
+					continue
+				}
+				block, err := client.GetBlock(blockHash)
+				if err != nil {
+					errChan <- err
+					continue
+				}
+				ps.lastHash = blockHashBytes
+				payloadChan <- BlockPayload{
+					Header:      &block.Header,
+					BlockHeight: height,
+					Txs:         msgTxsToUtilTxs(block.Transactions),
+				}
+			default:
+			}
+		}
+	}()
+	return &HTTPClientSubscription{client: client, errChan: errChan}, nil
+}
+
+// HTTPClientSubscription is a wrapper around the underlying bitcoind rpc client
+// to fit the shared.ClientSubscription interface
+type HTTPClientSubscription struct {
+	client  *rpcclient.Client
+	errChan chan error
+}
+
+// Unsubscribe satisfies the rpc.Subscription interface
+func (bcs *HTTPClientSubscription) Unsubscribe() {
+	bcs.client.Shutdown()
+}
+
+// Err() satisfies the rpc.Subscription interface
+func (bcs *HTTPClientSubscription) Err() <-chan error {
+	return bcs.errChan
+}

pkg/btc/indexer.go

@@ -0,0 +1,132 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package btc
+
+import (
+	"fmt"
+
+	"github.com/sirupsen/logrus"
+
+	"github.com/jmoiron/sqlx"
+	"github.com/lib/pq"
+
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/postgres"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
+)
+
+type CIDIndexer struct {
+	db *postgres.DB
+}
+
+func NewCIDIndexer(db *postgres.DB) *CIDIndexer {
+	return &CIDIndexer{
+		db: db,
+	}
+}
+
+func (in *CIDIndexer) Index(cids shared.CIDsForIndexing) error {
+	cidWrapper, ok := cids.(*CIDPayload)
+	if !ok {
+		return fmt.Errorf("btc indexer expected cids type %T got %T", &CIDPayload{}, cids)
+	}
+
+	// Begin new db tx
+	tx, err := in.db.Beginx()
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if p := recover(); p != nil {
+			shared.Rollback(tx)
+			panic(p)
+		} else if err != nil {
+			shared.Rollback(tx)
+		} else {
+			err = tx.Commit()
+		}
+	}()
+
+	headerID, err := in.indexHeaderCID(tx, cidWrapper.HeaderCID)
+	if err != nil {
+		logrus.Error("btc indexer error when indexing header")
+		return err
+	}
+	err = in.indexTransactionCIDs(tx, cidWrapper.TransactionCIDs, headerID)
+	if err != nil {
+		logrus.Error("btc indexer error when indexing transactions")
+	}
+	return err
+}
+
+func (in *CIDIndexer) indexHeaderCID(tx *sqlx.Tx, header HeaderModel) (int64, error) {
+	var headerID int64
+	err := tx.QueryRowx(`INSERT INTO btc.header_cids (block_number, block_hash, parent_hash, cid, timestamp, bits, node_id, times_validated)
+							VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
+							ON CONFLICT (block_number, block_hash) DO UPDATE SET (parent_hash, cid, timestamp, bits, node_id, times_validated) = ($3, $4, $5, $6, $7, btc.header_cids.times_validated + 1)
+							RETURNING id`,
+		header.BlockNumber, header.BlockHash, header.ParentHash, header.CID, header.Timestamp, header.Bits, in.db.NodeID, 1).Scan(&headerID)
+	return headerID, err
+}
+
+func (in *CIDIndexer) indexTransactionCIDs(tx *sqlx.Tx, transactions []TxModelWithInsAndOuts, headerID int64) error {
+	for _, transaction := range transactions {
+		txID, err := in.indexTransactionCID(tx, transaction, headerID)
+		if err != nil {
+			logrus.Error("btc indexer error when indexing header")
+			return err
+		}
+		for _, input := range transaction.TxInputs {
+			if err := in.indexTxInput(tx, input, txID); err != nil {
+				logrus.Error("btc indexer error when indexing tx inputs")
+				return err
+			}
+		}
+		for _, output := range transaction.TxOutputs {
+			if err := in.indexTxOutput(tx, output, txID); err != nil {
+				logrus.Error("btc indexer error when indexing tx outputs")
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (in *CIDIndexer) indexTransactionCID(tx *sqlx.Tx, transaction TxModelWithInsAndOuts, headerID int64) (int64, error) {
+	var txID int64
+	err := tx.QueryRowx(`INSERT INTO btc.transaction_cids (header_id, tx_hash, index, cid, segwit, witness_hash)
+							VALUES ($1, $2, $3, $4, $5, $6)
+							ON CONFLICT (tx_hash) DO UPDATE SET (header_id, index, cid, segwit, witness_hash) = ($1, $3, $4, $5, $6)
+							RETURNING id`,
+		headerID, transaction.TxHash, transaction.Index, transaction.CID, transaction.SegWit, transaction.WitnessHash).Scan(&txID)
+	return txID, err
+}
+
+func (in *CIDIndexer) indexTxInput(tx *sqlx.Tx, txInput TxInput, txID int64) error {
+	_, err := tx.Exec(`INSERT INTO btc.tx_inputs (tx_id, index, witness, sig_script, outpoint_tx_hash, outpoint_index)
+						VALUES ($1, $2, $3, $4, $5, $6)
+						ON CONFLICT (tx_id, index) DO UPDATE SET (witness, sig_script, outpoint_tx_hash, outpoint_index) = ($3, $4, $5, $6)`,
+		txID, txInput.Index, pq.Array(txInput.TxWitness), txInput.SignatureScript, txInput.PreviousOutPointHash, txInput.PreviousOutPointIndex)
+	return err
+}
+
+func (in *CIDIndexer) indexTxOutput(tx *sqlx.Tx, txOuput TxOutput, txID int64) error {
+	_, err := tx.Exec(`INSERT INTO btc.tx_outputs (tx_id, index, value, pk_script, script_class, addresses, required_sigs)
+							VALUES ($1, $2, $3, $4, $5, $6, $7)
+							ON CONFLICT (tx_id, index) DO UPDATE SET (value, pk_script, script_class, addresses, required_sigs) = ($3, $4, $5, $6, $7)`,
+		txID, txOuput.Index, txOuput.Value, txOuput.PkScript, txOuput.ScriptClass, txOuput.Addresses, txOuput.RequiredSigs)
+	return err
+}

pkg/btc/indexer_test.go

@@ -0,0 +1,87 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package btc_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/btc"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/btc/mocks"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/postgres"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
+)
+
+var _ = Describe("Indexer", func() {
+	var (
+		db   *postgres.DB
+		err  error
+		repo *btc.CIDIndexer
+	)
+	BeforeEach(func() {
+		db, err = shared.SetupDB()
+		Expect(err).ToNot(HaveOccurred())
+		repo = btc.NewCIDIndexer(db)
+	})
+	AfterEach(func() {
+		btc.TearDownDB(db)
+	})
+
+	Describe("Index", func() {
+		It("Indexes CIDs and related metadata into vulcanizedb", func() {
+			err = repo.Index(&mocks.MockCIDPayload)
+			Expect(err).ToNot(HaveOccurred())
+			pgStr := `SELECT * FROM btc.header_cids
+				WHERE block_number = $1`
+			// check header was properly indexed
+			header := new(btc.HeaderModel)
+			err = db.Get(header, pgStr, mocks.MockHeaderMetaData.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(header.CID).To(Equal(mocks.MockHeaderMetaData.CID))
+			Expect(header.BlockNumber).To(Equal(mocks.MockHeaderMetaData.BlockNumber))
+			Expect(header.Bits).To(Equal(mocks.MockHeaderMetaData.Bits))
+			Expect(header.Timestamp).To(Equal(mocks.MockHeaderMetaData.Timestamp))
+			Expect(header.BlockHash).To(Equal(mocks.MockHeaderMetaData.BlockHash))
+			Expect(header.ParentHash).To(Equal(mocks.MockHeaderMetaData.ParentHash))
+			// check trxs were properly indexed
+			trxs := make([]btc.TxModel, 0)
+			pgStr = `SELECT transaction_cids.id, transaction_cids.header_id, transaction_cids.index,
+				transaction_cids.tx_hash, transaction_cids.cid, transaction_cids.segwit, transaction_cids.witness_hash
+				FROM btc.transaction_cids INNER JOIN btc.header_cids ON (transaction_cids.header_id = header_cids.id)
+				WHERE header_cids.block_number = $1`
+			err = db.Select(&trxs, pgStr, mocks.MockHeaderMetaData.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(trxs)).To(Equal(3))
+			for _, tx := range trxs {
+				Expect(tx.SegWit).To(Equal(false))
+				Expect(tx.HeaderID).To(Equal(header.ID))
+				Expect(tx.WitnessHash).To(Equal(""))
+				switch tx.Index {
+				case 0:
+					Expect(tx.CID).To(Equal("mockTrxCID1"))
+					Expect(tx.TxHash).To(Equal(mocks.MockBlock.Transactions[0].TxHash().String()))
+				case 1:
+					Expect(tx.CID).To(Equal("mockTrxCID2"))
+					Expect(tx.TxHash).To(Equal(mocks.MockBlock.Transactions[1].TxHash().String()))
+				case 2:
+					Expect(tx.CID).To(Equal("mockTrxCID3"))
+					Expect(tx.TxHash).To(Equal(mocks.MockBlock.Transactions[2].TxHash().String()))
+				}
+			}
+		})
+	})
+})

pkg/btc/ipld_fetcher.go

@@ -0,0 +1,135 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package btc
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-blockservice"
+	"github.com/ipfs/go-cid"
+	log "github.com/sirupsen/logrus"
+
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/ipfs"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
+)
+
+var (
+	errUnexpectedNumberOfIPLDs = errors.New("ipfs batch fetch returned unexpected number of IPLDs")
+)
+
+// IPLDFetcher satisfies the IPLDFetcher interface for ethereum
+type IPLDFetcher struct {
+	BlockService blockservice.BlockService
+}
+
+// NewIPLDFetcher creates a pointer to a new IPLDFetcher
+// It interfaces with PG-IPFS through an internalized IPFS node interface
+func NewIPLDFetcher(ipfsPath string) (*IPLDFetcher, error) {
+	blockService, err := ipfs.InitIPFSBlockService(ipfsPath)
+	if err != nil {
+		return nil, err
+	}
+	return &IPLDFetcher{
+		BlockService: blockService,
+	}, nil
+}
+
+// Fetch is the exported method for fetching and returning all the IPLDS specified in the CIDWrapper
+func (f *IPLDFetcher) Fetch(cids shared.CIDsForFetching) (shared.IPLDs, error) {
+	cidWrapper, ok := cids.(*CIDWrapper)
+	if !ok {
+		return nil, fmt.Errorf("btc fetcher: expected cids type %T got %T", &CIDWrapper{}, cids)
+	}
+	log.Debug("fetching iplds")
+	iplds := IPLDs{}
+	iplds.BlockNumber = cidWrapper.BlockNumber
+	var err error
+	iplds.Header, err = f.FetchHeader(cidWrapper.Header)
+	if err != nil {
+		return nil, err
+	}
+	iplds.Transactions, err = f.FetchTrxs(cidWrapper.Transactions)
+	if err != nil {
+		return nil, err
+	}
+	return iplds, nil
+}
+
+// FetchHeaders fetches headers
+// It uses the f.fetch method
+func (f *IPLDFetcher) FetchHeader(c HeaderModel) (ipfs.BlockModel, error) {
+	log.Debug("fetching header ipld")
+	dc, err := cid.Decode(c.CID)
+	if err != nil {
+		return ipfs.BlockModel{}, err
+	}
+	header, err := f.fetch(dc)
+	if err != nil {
+		return ipfs.BlockModel{}, err
+	}
+	return ipfs.BlockModel{
+		Data: header.RawData(),
+		CID:  header.Cid().String(),
+	}, nil
+}
+
+// FetchTrxs fetches transactions
+// It uses the f.fetchBatch method
+func (f *IPLDFetcher) FetchTrxs(cids []TxModel) ([]ipfs.BlockModel, error) {
+	log.Debug("fetching transaction iplds")
+	trxCids := make([]cid.Cid, len(cids))
+	for i, c := range cids {
+		dc, err := cid.Decode(c.CID)
+		if err != nil {
+			return nil, err
+		}
+		trxCids[i] = dc
+	}
+	trxs := f.fetchBatch(trxCids)
+	trxIPLDs := make([]ipfs.BlockModel, len(trxs))
+	for i, trx := range trxs {
+		trxIPLDs[i] = ipfs.BlockModel{
+			Data: trx.RawData(),
+			CID:  trx.Cid().String(),
+		}
+	}
+	if len(trxIPLDs) != len(trxCids) {
+		log.Errorf("ipfs fetcher: number of transaction blocks returned (%d) does not match number expected (%d)", len(trxs), len(trxCids))
+		return trxIPLDs, errUnexpectedNumberOfIPLDs
+	}
+	return trxIPLDs, nil
+}
+
+// fetch is used to fetch a single cid
+func (f *IPLDFetcher) fetch(cid cid.Cid) (blocks.Block, error) {
+	return f.BlockService.GetBlock(context.Background(), cid)
+}
+
+// fetchBatch is used to fetch a batch of IPFS data blocks by cid
+// There is no guarantee all are fetched, and no error in such a case, so
+// downstream we will need to confirm which CIDs were fetched in the result set
+func (f *IPLDFetcher) fetchBatch(cids []cid.Cid) []blocks.Block {
+	fetchedBlocks := make([]blocks.Block, 0, len(cids))
+	blockChan := f.BlockService.GetBlocks(context.Background(), cids)
+	for block := range blockChan {
+		fetchedBlocks = append(fetchedBlocks, block)
+	}
+	return fetchedBlocks
+}

pkg/btc/ipld_pg_fetcher.go

@@ -0,0 +1,107 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package btc
+
+import (
+	"fmt"
+
+	"github.com/jmoiron/sqlx"
+	log "github.com/sirupsen/logrus"
+
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/ipfs"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/postgres"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
+)
+
+// IPLDPGFetcher satisfies the IPLDFetcher interface for ethereum
+// it interfaces directly with PG-IPFS instead of going through a node-interface or remote node
+type IPLDPGFetcher struct {
+	db *postgres.DB
+}
+
+// NewIPLDPGFetcher creates a pointer to a new IPLDPGFetcher
+func NewIPLDPGFetcher(db *postgres.DB) *IPLDPGFetcher {
+	return &IPLDPGFetcher{
+		db: db,
+	}
+}
+
+// Fetch is the exported method for fetching and returning all the IPLDS specified in the CIDWrapper
+func (f *IPLDPGFetcher) Fetch(cids shared.CIDsForFetching) (shared.IPLDs, error) {
+	cidWrapper, ok := cids.(*CIDWrapper)
+	if !ok {
+		return nil, fmt.Errorf("btc fetcher: expected cids type %T got %T", &CIDWrapper{}, cids)
+	}
+	log.Debug("fetching iplds")
+	iplds := IPLDs{}
+	iplds.BlockNumber = cidWrapper.BlockNumber
+
+	tx, err := f.db.Beginx()
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		if p := recover(); p != nil {
+			shared.Rollback(tx)
+			panic(p)
+		} else if err != nil {
+			shared.Rollback(tx)
+		} else {
+			err = tx.Commit()
+		}
+	}()
+
+	iplds.Header, err = f.FetchHeader(tx, cidWrapper.Header)
+	if err != nil {
+		return nil, fmt.Errorf("btc pg fetcher: header fetching error: %s", err.Error())
+	}
+	iplds.Transactions, err = f.FetchTrxs(tx, cidWrapper.Transactions)
+	if err != nil {
+		return nil, fmt.Errorf("btc pg fetcher: transaction fetching error: %s", err.Error())
+	}
+	return iplds, err
+}
+
+// FetchHeaders fetches headers
+func (f *IPLDPGFetcher) FetchHeader(tx *sqlx.Tx, c HeaderModel) (ipfs.BlockModel, error) {
+	log.Debug("fetching header ipld")
+	headerBytes, err := shared.FetchIPLD(tx, c.CID)
+	if err != nil {
+		return ipfs.BlockModel{}, err
+	}
+	return ipfs.BlockModel{
+		Data: headerBytes,
+		CID:  c.CID,
+	}, nil
+}
+
+// FetchTrxs fetches transactions
+func (f *IPLDPGFetcher) FetchTrxs(tx *sqlx.Tx, cids []TxModel) ([]ipfs.BlockModel, error) {
+	log.Debug("fetching transaction iplds")
+	trxIPLDs := make([]ipfs.BlockModel, len(cids))
+	for i, c := range cids {
+		trxBytes, err := shared.FetchIPLD(tx, c.CID)
+		if err != nil {
+			return nil, err
+		}
+		trxIPLDs[i] = ipfs.BlockModel{
+			Data: trxBytes,
+			CID:  c.CID,
+		}
+	}
+	return trxIPLDs, nil
+}

pkg/btc/mocks/converter.go

@@ -0,0 +1,64 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"fmt"
+
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/btc"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
+)
+
+// PayloadConverter is the underlying struct for the Converter interface
+type PayloadConverter struct {
+	PassedStatediffPayload btc.BlockPayload
+	ReturnIPLDPayload      btc.ConvertedPayload
+	ReturnErr              error
+}
+
+// Convert method is used to convert a geth statediff.Payload to a IPLDPayload
+func (pc *PayloadConverter) Convert(payload shared.RawChainData) (shared.ConvertedData, error) {
+	stateDiffPayload, ok := payload.(btc.BlockPayload)
+	if !ok {
+		return nil, fmt.Errorf("convert expected payload type %T got %T", btc.BlockPayload{}, payload)
+	}
+	pc.PassedStatediffPayload = stateDiffPayload
+	return pc.ReturnIPLDPayload, pc.ReturnErr
+}
+
+// IterativePayloadConverter is the underlying struct for the Converter interface
+type IterativePayloadConverter struct {
+	PassedStatediffPayload []btc.BlockPayload
+	ReturnIPLDPayload      []btc.ConvertedPayload
+	ReturnErr              error
+	iteration              int
+}
+
+// Convert method is used to convert a geth statediff.Payload to a IPLDPayload
+func (pc *IterativePayloadConverter) Convert(payload shared.RawChainData) (shared.ConvertedData, error) {
+	stateDiffPayload, ok := payload.(btc.BlockPayload)
+	if !ok {
+		return nil, fmt.Errorf("convert expected payload type %T got %T", btc.BlockPayload{}, payload)
+	}
+	pc.PassedStatediffPayload = append(pc.PassedStatediffPayload, stateDiffPayload)
+	if len(pc.PassedStatediffPayload) < pc.iteration+1 {
+		return nil, fmt.Errorf("IterativePayloadConverter does not have a payload to return at iteration %d", pc.iteration)
+	}
+	returnPayload := pc.ReturnIPLDPayload[pc.iteration]
+	pc.iteration++
+	return returnPayload, pc.ReturnErr
+}

pkg/btc/mocks/indexer.go

@@ -0,0 +1,40 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"fmt"
+
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/btc"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
+)
+
+// CIDIndexer is the underlying struct for the Indexer interface
+type CIDIndexer struct {
+	PassedCIDPayload []*btc.CIDPayload
+	ReturnErr        error
+}
+
+// Index indexes a cidPayload in Postgres
+func (repo *CIDIndexer) Index(cids shared.CIDsForIndexing) error {
+	cidPayload, ok := cids.(*btc.CIDPayload)
+	if !ok {
+		return fmt.Errorf("index expected cids type %T got %T", &btc.CIDPayload{}, cids)
+	}
+	repo.PassedCIDPayload = append(repo.PassedCIDPayload, cidPayload)
+	return repo.ReturnErr
+}

pkg/btc/mocks/publisher.go

@@ -0,0 +1,65 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"fmt"
+
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/btc"
+
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
+)
+
+// IPLDPublisher is the underlying struct for the Publisher interface
+type IPLDPublisher struct {
+	PassedIPLDPayload btc.ConvertedPayload
+	ReturnCIDPayload  *btc.CIDPayload
+	ReturnErr         error
+}
+
+// Publish publishes an IPLDPayload to IPFS and returns the corresponding CIDPayload
+func (pub *IPLDPublisher) Publish(payload shared.ConvertedData) (shared.CIDsForIndexing, error) {
+	ipldPayload, ok := payload.(btc.ConvertedPayload)
+	if !ok {
+		return nil, fmt.Errorf("publish expected payload type %T got %T", &btc.ConvertedPayload{}, payload)
+	}
+	pub.PassedIPLDPayload = ipldPayload
+	return pub.ReturnCIDPayload, pub.ReturnErr
+}
+
+// IterativeIPLDPublisher is the underlying struct for the Publisher interface; used in testing
+type IterativeIPLDPublisher struct {
+	PassedIPLDPayload []btc.ConvertedPayload
+	ReturnCIDPayload  []*btc.CIDPayload
+	ReturnErr         error
+	iteration         int
+}
+
+// Publish publishes an IPLDPayload to IPFS and returns the corresponding CIDPayload
+func (pub *IterativeIPLDPublisher) Publish(payload shared.ConvertedData) (shared.CIDsForIndexing, error) {
+	ipldPayload, ok := payload.(btc.ConvertedPayload)
+	if !ok {
+		return nil, fmt.Errorf("publish expected payload type %T got %T", &btc.ConvertedPayload{}, payload)
+	}
+	pub.PassedIPLDPayload = append(pub.PassedIPLDPayload, ipldPayload)
+	if len(pub.ReturnCIDPayload) < pub.iteration+1 {
+		return nil, fmt.Errorf("IterativeIPLDPublisher does not have a payload to return at iteration %d", pub.iteration)
+	}
+	returnPayload := pub.ReturnCIDPayload[pub.iteration]
+	pub.iteration++
+	return returnPayload, pub.ReturnErr
+}

pkg/btc/mocks/test_data.go

@@ -0,0 +1,744 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"strconv"
+	"time"
+
+	"github.com/btcsuite/btcd/chaincfg"
+	"github.com/btcsuite/btcd/chaincfg/chainhash"
+	"github.com/btcsuite/btcd/txscript"
+	"github.com/btcsuite/btcd/wire"
+	"github.com/btcsuite/btcutil"
+
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/btc"
+)
+
+var (
+	MockBlockHeight int64 = 1337
+	MockBlock             = wire.MsgBlock{
+		Header: wire.BlockHeader{
+			Version: 1,
+			PrevBlock: chainhash.Hash([32]byte{ // Make go vet happy.
+				0x50, 0x12, 0x01, 0x19, 0x17, 0x2a, 0x61, 0x04,
+				0x21, 0xa6, 0xc3, 0x01, 0x1d, 0xd3, 0x30, 0xd9,
+				0xdf, 0x07, 0xb6, 0x36, 0x16, 0xc2, 0xcc, 0x1f,
+				0x1c, 0xd0, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00,
+			}), // 000000000002d01c1fccc21636b607dfd930d31d01c3a62104612a1719011250
+			MerkleRoot: chainhash.Hash([32]byte{ // Make go vet happy.
+				0x66, 0x57, 0xa9, 0x25, 0x2a, 0xac, 0xd5, 0xc0,
+				0xb2, 0x94, 0x09, 0x96, 0xec, 0xff, 0x95, 0x22,
+				0x28, 0xc3, 0x06, 0x7c, 0xc3, 0x8d, 0x48, 0x85,
+				0xef, 0xb5, 0xa4, 0xac, 0x42, 0x47, 0xe9, 0xf3,
+			}), // f3e94742aca4b5ef85488dc37c06c3282295ffec960994b2c0d5ac2a25a95766
+			Timestamp: time.Unix(1293623863, 0), // 2010-12-29 11:57:43 +0000 UTC
+			Bits:      0x1b04864c,               // 453281356
+			Nonce:     0x10572b0f,               // 274148111
+		},
+		Transactions: []*wire.MsgTx{
+			{
+				Version: 1,
+				TxIn: []*wire.TxIn{
+					{
+						PreviousOutPoint: wire.OutPoint{
+							Hash:  chainhash.Hash{},
+							Index: 0xffffffff,
+						},
+						SignatureScript: []byte{
+							0x04, 0x4c, 0x86, 0x04, 0x1b, 0x02, 0x06, 0x02,
+						},
+						Sequence: 0xffffffff,
+					},
+				},
+				TxOut: []*wire.TxOut{
+					{
+						Value: 0x12a05f200, // 5000000000
+						PkScript: []byte{
+							0x41, // OP_DATA_65
+							0x04, 0x1b, 0x0e, 0x8c, 0x25, 0x67, 0xc1, 0x25,
+							0x36, 0xaa, 0x13, 0x35, 0x7b, 0x79, 0xa0, 0x73,
+							0xdc, 0x44, 0x44, 0xac, 0xb8, 0x3c, 0x4e, 0xc7,
+							0xa0, 0xe2, 0xf9, 0x9d, 0xd7, 0x45, 0x75, 0x16,
+							0xc5, 0x81, 0x72, 0x42, 0xda, 0x79, 0x69, 0x24,
+							0xca, 0x4e, 0x99, 0x94, 0x7d, 0x08, 0x7f, 0xed,
+							0xf9, 0xce, 0x46, 0x7c, 0xb9, 0xf7, 0xc6, 0x28,
+							0x70, 0x78, 0xf8, 0x01, 0xdf, 0x27, 0x6f, 0xdf,
+							0x84, // 65-byte signature
+							0xac, // OP_CHECKSIG
+						},
+					},
+				},
+				LockTime: 0,
+			},
+			{
+				Version: 1,
+				TxIn: []*wire.TxIn{
+					{
+						PreviousOutPoint: wire.OutPoint{
+							Hash: chainhash.Hash([32]byte{ // Make go vet happy.
+								0x03, 0x2e, 0x38, 0xe9, 0xc0, 0xa8, 0x4c, 0x60,
+								0x46, 0xd6, 0x87, 0xd1, 0x05, 0x56, 0xdc, 0xac,
+								0xc4, 0x1d, 0x27, 0x5e, 0xc5, 0x5f, 0xc0, 0x07,
+								0x79, 0xac, 0x88, 0xfd, 0xf3, 0x57, 0xa1, 0x87,
+							}), // 87a157f3fd88ac7907c05fc55e271dc4acdc5605d187d646604ca8c0e9382e03
+							Index: 0,
+						},
+						SignatureScript: []byte{
+							0x49, // OP_DATA_73
+							0x30, 0x46, 0x02, 0x21, 0x00, 0xc3, 0x52, 0xd3,
+							0xdd, 0x99, 0x3a, 0x98, 0x1b, 0xeb, 0xa4, 0xa6,
+							0x3a, 0xd1, 0x5c, 0x20, 0x92, 0x75, 0xca, 0x94,
+							0x70, 0xab, 0xfc, 0xd5, 0x7d, 0xa9, 0x3b, 0x58,
+							0xe4, 0xeb, 0x5d, 0xce, 0x82, 0x02, 0x21, 0x00,
+							0x84, 0x07, 0x92, 0xbc, 0x1f, 0x45, 0x60, 0x62,
+							0x81, 0x9f, 0x15, 0xd3, 0x3e, 0xe7, 0x05, 0x5c,
+							0xf7, 0xb5, 0xee, 0x1a, 0xf1, 0xeb, 0xcc, 0x60,
+							0x28, 0xd9, 0xcd, 0xb1, 0xc3, 0xaf, 0x77, 0x48,
+							0x01, // 73-byte signature
+							0x41, // OP_DATA_65
+							0x04, 0xf4, 0x6d, 0xb5, 0xe9, 0xd6, 0x1a, 0x9d,
+							0xc2, 0x7b, 0x8d, 0x64, 0xad, 0x23, 0xe7, 0x38,
+							0x3a, 0x4e, 0x6c, 0xa1, 0x64, 0x59, 0x3c, 0x25,
+							0x27, 0xc0, 0x38, 0xc0, 0x85, 0x7e, 0xb6, 0x7e,
+							0xe8, 0xe8, 0x25, 0xdc, 0xa6, 0x50, 0x46, 0xb8,
+							0x2c, 0x93, 0x31, 0x58, 0x6c, 0x82, 0xe0, 0xfd,
+							0x1f, 0x63, 0x3f, 0x25, 0xf8, 0x7c, 0x16, 0x1b,
+							0xc6, 0xf8, 0xa6, 0x30, 0x12, 0x1d, 0xf2, 0xb3,
+							0xd3, // 65-byte pubkey
+						},
+						Sequence: 0xffffffff,
+					},
+				},
+				TxOut: []*wire.TxOut{
+					{
+						Value: 0x2123e300, // 556000000
+						PkScript: []byte{
+							0x76, // OP_DUP
+							0xa9, // OP_HASH160
+							0x14, // OP_DATA_20
+							0xc3, 0x98, 0xef, 0xa9, 0xc3, 0x92, 0xba, 0x60,
+							0x13, 0xc5, 0xe0, 0x4e, 0xe7, 0x29, 0x75, 0x5e,
+							0xf7, 0xf5, 0x8b, 0x32,
+							0x88, // OP_EQUALVERIFY
+							0xac, // OP_CHECKSIG
+						},
+					},
+					{
+						Value: 0x108e20f00, // 4444000000
+						PkScript: []byte{
+							0x76, // OP_DUP
+							0xa9, // OP_HASH160
+							0x14, // OP_DATA_20
+							0x94, 0x8c, 0x76, 0x5a, 0x69, 0x14, 0xd4, 0x3f,
+							0x2a, 0x7a, 0xc1, 0x77, 0xda, 0x2c, 0x2f, 0x6b,
+							0x52, 0xde, 0x3d, 0x7c,
+							0x88, // OP_EQUALVERIFY
+							0xac, // OP_CHECKSIG
+						},
+					},
+				},
+				LockTime: 0,
+			},
+			{
+				Version: 1,
+				TxIn: []*wire.TxIn{
+					{
+						PreviousOutPoint: wire.OutPoint{
+							Hash: chainhash.Hash([32]byte{ // Make go vet happy.
+								0xc3, 0x3e, 0xbf, 0xf2, 0xa7, 0x09, 0xf1, 0x3d,
+								0x9f, 0x9a, 0x75, 0x69, 0xab, 0x16, 0xa3, 0x27,
+								0x86, 0xaf, 0x7d, 0x7e, 0x2d, 0xe0, 0x92, 0x65,
+								0xe4, 0x1c, 0x61, 0xd0, 0x78, 0x29, 0x4e, 0xcf,
+							}), // cf4e2978d0611ce46592e02d7e7daf8627a316ab69759a9f3df109a7f2bf3ec3
+							Index: 1,
+						},
+						SignatureScript: []byte{
+							0x47, // OP_DATA_71
+							0x30, 0x44, 0x02, 0x20, 0x03, 0x2d, 0x30, 0xdf,
+							0x5e, 0xe6, 0xf5, 0x7f, 0xa4, 0x6c, 0xdd, 0xb5,
+							0xeb, 0x8d, 0x0d, 0x9f, 0xe8, 0xde, 0x6b, 0x34,
+							0x2d, 0x27, 0x94, 0x2a, 0xe9, 0x0a, 0x32, 0x31,
+							0xe0, 0xba, 0x33, 0x3e, 0x02, 0x20, 0x3d, 0xee,
+							0xe8, 0x06, 0x0f, 0xdc, 0x70, 0x23, 0x0a, 0x7f,
+							0x5b, 0x4a, 0xd7, 0xd7, 0xbc, 0x3e, 0x62, 0x8c,
+							0xbe, 0x21, 0x9a, 0x88, 0x6b, 0x84, 0x26, 0x9e,
+							0xae, 0xb8, 0x1e, 0x26, 0xb4, 0xfe, 0x01,
+							0x41, // OP_DATA_65
+							0x04, 0xae, 0x31, 0xc3, 0x1b, 0xf9, 0x12, 0x78,
+							0xd9, 0x9b, 0x83, 0x77, 0xa3, 0x5b, 0xbc, 0xe5,
+							0xb2, 0x7d, 0x9f, 0xff, 0x15, 0x45, 0x68, 0x39,
+							0xe9, 0x19, 0x45, 0x3f, 0xc7, 0xb3, 0xf7, 0x21,
+							0xf0, 0xba, 0x40, 0x3f, 0xf9, 0x6c, 0x9d, 0xee,
+							0xb6, 0x80, 0xe5, 0xfd, 0x34, 0x1c, 0x0f, 0xc3,
+							0xa7, 0xb9, 0x0d, 0xa4, 0x63, 0x1e, 0xe3, 0x95,
+							0x60, 0x63, 0x9d, 0xb4, 0x62, 0xe9, 0xcb, 0x85,
+							0x0f, // 65-byte pubkey
+						},
+						Sequence: 0xffffffff,
+					},
+				},
+				TxOut: []*wire.TxOut{
+					{
+						Value: 0xf4240, // 1000000
+						PkScript: []byte{
+							0x76, // OP_DUP
+							0xa9, // OP_HASH160
+							0x14, // OP_DATA_20
+							0xb0, 0xdc, 0xbf, 0x97, 0xea, 0xbf, 0x44, 0x04,
+							0xe3, 0x1d, 0x95, 0x24, 0x77, 0xce, 0x82, 0x2d,
+							0xad, 0xbe, 0x7e, 0x10,
+							0x88, // OP_EQUALVERIFY
+							0xac, // OP_CHECKSIG
+						},
+					},
+					{
+						Value: 0x11d260c0, // 299000000
+						PkScript: []byte{
+							0x76, // OP_DUP
+							0xa9, // OP_HASH160
+							0x14, // OP_DATA_20
+							0x6b, 0x12, 0x81, 0xee, 0xc2, 0x5a, 0xb4, 0xe1,
+							0xe0, 0x79, 0x3f, 0xf4, 0xe0, 0x8a, 0xb1, 0xab,
+							0xb3, 0x40, 0x9c, 0xd9,
+							0x88, // OP_EQUALVERIFY
+							0xac, // OP_CHECKSIG
+						},
+					},
+				},
+				LockTime: 0,
+			},
+		},
+	}
+	MockTransactions = []*btcutil.Tx{
+		btcutil.NewTx(MockBlock.Transactions[0]),
+		btcutil.NewTx(MockBlock.Transactions[1]),
+		btcutil.NewTx(MockBlock.Transactions[2]),
+	}
+	MockBlockPayload = btc.BlockPayload{
+		Header:      &MockBlock.Header,
+		Txs:         MockTransactions,
+		BlockHeight: MockBlockHeight,
+	}
+	sClass1, addresses1, numOfSigs1, _ = txscript.ExtractPkScriptAddrs([]byte{
+		0x41, // OP_DATA_65
+		0x04, 0x1b, 0x0e, 0x8c, 0x25, 0x67, 0xc1, 0x25,
+		0x36, 0xaa, 0x13, 0x35, 0x7b, 0x79, 0xa0, 0x73,
+		0xdc, 0x44, 0x44, 0xac, 0xb8, 0x3c, 0x4e, 0xc7,
+		0xa0, 0xe2, 0xf9, 0x9d, 0xd7, 0x45, 0x75, 0x16,
+		0xc5, 0x81, 0x72, 0x42, 0xda, 0x79, 0x69, 0x24,
+		0xca, 0x4e, 0x99, 0x94, 0x7d, 0x08, 0x7f, 0xed,
+		0xf9, 0xce, 0x46, 0x7c, 0xb9, 0xf7, 0xc6, 0x28,
+		0x70, 0x78, 0xf8, 0x01, 0xdf, 0x27, 0x6f, 0xdf,
+		0x84, // 65-byte signature
+		0xac, // OP_CHECKSIG
+	}, &chaincfg.MainNetParams)
+	sClass2a, addresses2a, numOfSigs2a, _ = txscript.ExtractPkScriptAddrs([]byte{
+		0x76, // OP_DUP
+		0xa9, // OP_HASH160
+		0x14, // OP_DATA_20
+		0xc3, 0x98, 0xef, 0xa9, 0xc3, 0x92, 0xba, 0x60,
+		0x13, 0xc5, 0xe0, 0x4e, 0xe7, 0x29, 0x75, 0x5e,
+		0xf7, 0xf5, 0x8b, 0x32,
+		0x88, // OP_EQUALVERIFY
+		0xac, // OP_CHECKSIG
+	}, &chaincfg.MainNetParams)
+	sClass2b, addresses2b, numOfSigs2b, _ = txscript.ExtractPkScriptAddrs([]byte{
+		0x76, // OP_DUP
+		0xa9, // OP_HASH160
+		0x14, // OP_DATA_20
+		0x94, 0x8c, 0x76, 0x5a, 0x69, 0x14, 0xd4, 0x3f,
+		0x2a, 0x7a, 0xc1, 0x77, 0xda, 0x2c, 0x2f, 0x6b,
+		0x52, 0xde, 0x3d, 0x7c,
+		0x88, // OP_EQUALVERIFY
+		0xac, // OP_CHECKSIG
+	}, &chaincfg.MainNetParams)
+	sClass3a, addresses3a, numOfSigs3a, _ = txscript.ExtractPkScriptAddrs([]byte{
+		0x76, // OP_DUP
+		0xa9, // OP_HASH160
+		0x14, // OP_DATA_20
+		0xb0, 0xdc, 0xbf, 0x97, 0xea, 0xbf, 0x44, 0x04,
+		0xe3, 0x1d, 0x95, 0x24, 0x77, 0xce, 0x82, 0x2d,
+		0xad, 0xbe, 0x7e, 0x10,
+		0x88, // OP_EQUALVERIFY
+		0xac, // OP_CHECKSIG
+	}, &chaincfg.MainNetParams)
+	sClass3b, addresses3b, numOfSigs3b, _ = txscript.ExtractPkScriptAddrs([]byte{
+		0x76, // OP_DUP
+		0xa9, // OP_HASH160
+		0x14, // OP_DATA_20
+		0x6b, 0x12, 0x81, 0xee, 0xc2, 0x5a, 0xb4, 0xe1,
+		0xe0, 0x79, 0x3f, 0xf4, 0xe0, 0x8a, 0xb1, 0xab,
+		0xb3, 0x40, 0x9c, 0xd9,
+		0x88, // OP_EQUALVERIFY
+		0xac, // OP_CHECKSIG
+	}, &chaincfg.MainNetParams)
+	MockTxsMetaData = []btc.TxModelWithInsAndOuts{
+		{
+			TxHash: MockBlock.Transactions[0].TxHash().String(),
+			Index:  0,
+			SegWit: MockBlock.Transactions[0].HasWitness(),
+			TxInputs: []btc.TxInput{
+				{
+					Index: 0,
+					SignatureScript: []byte{
+						0x04, 0x4c, 0x86, 0x04, 0x1b, 0x02, 0x06, 0x02,
+					},
+					PreviousOutPointHash:  chainhash.Hash{}.String(),
+					PreviousOutPointIndex: 0xffffffff,
+				},
+			},
+			TxOutputs: []btc.TxOutput{
+				{
+					Value: 5000000000,
+					Index: 0,
+					PkScript: []byte{
+						0x41, // OP_DATA_65
+						0x04, 0x1b, 0x0e, 0x8c, 0x25, 0x67, 0xc1, 0x25,
+						0x36, 0xaa, 0x13, 0x35, 0x7b, 0x79, 0xa0, 0x73,
+						0xdc, 0x44, 0x44, 0xac, 0xb8, 0x3c, 0x4e, 0xc7,
+						0xa0, 0xe2, 0xf9, 0x9d, 0xd7, 0x45, 0x75, 0x16,
+						0xc5, 0x81, 0x72, 0x42, 0xda, 0x79, 0x69, 0x24,
+						0xca, 0x4e, 0x99, 0x94, 0x7d, 0x08, 0x7f, 0xed,
+						0xf9, 0xce, 0x46, 0x7c, 0xb9, 0xf7, 0xc6, 0x28,
+						0x70, 0x78, 0xf8, 0x01, 0xdf, 0x27, 0x6f, 0xdf,
+						0x84, // 65-byte signature
+						0xac, // OP_CHECKSIG
+					},
+					ScriptClass:  uint8(sClass1),
+					RequiredSigs: int64(numOfSigs1),
+					Addresses:    stringSliceFromAddresses(addresses1),
+				},
+			},
+		},
+		{
+			TxHash: MockBlock.Transactions[1].TxHash().String(),
+			Index:  1,
+			SegWit: MockBlock.Transactions[1].HasWitness(),
+			TxInputs: []btc.TxInput{
+				{
+					Index: 0,
+					PreviousOutPointHash: chainhash.Hash([32]byte{ // Make go vet happy.
+						0x03, 0x2e, 0x38, 0xe9, 0xc0, 0xa8, 0x4c, 0x60,
+						0x46, 0xd6, 0x87, 0xd1, 0x05, 0x56, 0xdc, 0xac,
+						0xc4, 0x1d, 0x27, 0x5e, 0xc5, 0x5f, 0xc0, 0x07,
+						0x79, 0xac, 0x88, 0xfd, 0xf3, 0x57, 0xa1, 0x87,
+					}).String(),
+					PreviousOutPointIndex: 0,
+					SignatureScript: []byte{
+						0x49, // OP_DATA_73
+						0x30, 0x46, 0x02, 0x21, 0x00, 0xc3, 0x52, 0xd3,
+						0xdd, 0x99, 0x3a, 0x98, 0x1b, 0xeb, 0xa4, 0xa6,
+						0x3a, 0xd1, 0x5c, 0x20, 0x92, 0x75, 0xca, 0x94,
+						0x70, 0xab, 0xfc, 0xd5, 0x7d, 0xa9, 0x3b, 0x58,
+						0xe4, 0xeb, 0x5d, 0xce, 0x82, 0x02, 0x21, 0x00,
+						0x84, 0x07, 0x92, 0xbc, 0x1f, 0x45, 0x60, 0x62,
+						0x81, 0x9f, 0x15, 0xd3, 0x3e, 0xe7, 0x05, 0x5c,
+						0xf7, 0xb5, 0xee, 0x1a, 0xf1, 0xeb, 0xcc, 0x60,
+						0x28, 0xd9, 0xcd, 0xb1, 0xc3, 0xaf, 0x77, 0x48,
+						0x01, // 73-byte signature
+						0x41, // OP_DATA_65
+						0x04, 0xf4, 0x6d, 0xb5, 0xe9, 0xd6, 0x1a, 0x9d,
+						0xc2, 0x7b, 0x8d, 0x64, 0xad, 0x23, 0xe7, 0x38,
+						0x3a, 0x4e, 0x6c, 0xa1, 0x64, 0x59, 0x3c, 0x25,
+						0x27, 0xc0, 0x38, 0xc0, 0x85, 0x7e, 0xb6, 0x7e,
+						0xe8, 0xe8, 0x25, 0xdc, 0xa6, 0x50, 0x46, 0xb8,
+						0x2c, 0x93, 0x31, 0x58, 0x6c, 0x82, 0xe0, 0xfd,
+						0x1f, 0x63, 0x3f, 0x25, 0xf8, 0x7c, 0x16, 0x1b,
+						0xc6, 0xf8, 0xa6, 0x30, 0x12, 0x1d, 0xf2, 0xb3,
+						0xd3, // 65-byte pubkey
+					},
+				},
+			},
+			TxOutputs: []btc.TxOutput{
+				{
+					Index: 0,
+					Value: 556000000,
+					PkScript: []byte{
+						0x76, // OP_DUP
+						0xa9, // OP_HASH160
+						0x14, // OP_DATA_20
+						0xc3, 0x98, 0xef, 0xa9, 0xc3, 0x92, 0xba, 0x60,
+						0x13, 0xc5, 0xe0, 0x4e, 0xe7, 0x29, 0x75, 0x5e,
+						0xf7, 0xf5, 0x8b, 0x32,
+						0x88, // OP_EQUALVERIFY
+						0xac, // OP_CHECKSIG
+					},
+					ScriptClass:  uint8(sClass2a),
+					RequiredSigs: int64(numOfSigs2a),
+					Addresses:    stringSliceFromAddresses(addresses2a),
+				},
+				{
+					Index: 1,
+					Value: 4444000000,
+					PkScript: []byte{
+						0x76, // OP_DUP
+						0xa9, // OP_HASH160
+						0x14, // OP_DATA_20
+						0x94, 0x8c, 0x76, 0x5a, 0x69, 0x14, 0xd4, 0x3f,
+						0x2a, 0x7a, 0xc1, 0x77, 0xda, 0x2c, 0x2f, 0x6b,
+						0x52, 0xde, 0x3d, 0x7c,
+						0x88, // OP_EQUALVERIFY
+						0xac, // OP_CHECKSIG
+					},
+					ScriptClass:  uint8(sClass2b),
+					RequiredSigs: int64(numOfSigs2b),
+					Addresses:    stringSliceFromAddresses(addresses2b),
+				},
+			},
+		},
+		{
+			TxHash: MockBlock.Transactions[2].TxHash().String(),
+			Index:  2,
+			SegWit: MockBlock.Transactions[2].HasWitness(),
+			TxInputs: []btc.TxInput{
+				{
+					Index: 0,
+					PreviousOutPointHash: chainhash.Hash([32]byte{ // Make go vet happy.
+						0xc3, 0x3e, 0xbf, 0xf2, 0xa7, 0x09, 0xf1, 0x3d,
+						0x9f, 0x9a, 0x75, 0x69, 0xab, 0x16, 0xa3, 0x27,
+						0x86, 0xaf, 0x7d, 0x7e, 0x2d, 0xe0, 0x92, 0x65,
+						0xe4, 0x1c, 0x61, 0xd0, 0x78, 0x29, 0x4e, 0xcf,
+					}).String(),
+					PreviousOutPointIndex: 1,
+					SignatureScript: []byte{
+						0x47, // OP_DATA_71
+						0x30, 0x44, 0x02, 0x20, 0x03, 0x2d, 0x30, 0xdf,
+						0x5e, 0xe6, 0xf5, 0x7f, 0xa4, 0x6c, 0xdd, 0xb5,
+						0xeb, 0x8d, 0x0d, 0x9f, 0xe8, 0xde, 0x6b, 0x34,
+						0x2d, 0x27, 0x94, 0x2a, 0xe9, 0x0a, 0x32, 0x31,
+						0xe0, 0xba, 0x33, 0x3e, 0x02, 0x20, 0x3d, 0xee,
+						0xe8, 0x06, 0x0f, 0xdc, 0x70, 0x23, 0x0a, 0x7f,
+						0x5b, 0x4a, 0xd7, 0xd7, 0xbc, 0x3e, 0x62, 0x8c,
+						0xbe, 0x21, 0x9a, 0x88, 0x6b, 0x84, 0x26, 0x9e,
+						0xae, 0xb8, 0x1e, 0x26, 0xb4, 0xfe, 0x01,
+						0x41, // OP_DATA_65
+						0x04, 0xae, 0x31, 0xc3, 0x1b, 0xf9, 0x12, 0x78,
+						0xd9, 0x9b, 0x83, 0x77, 0xa3, 0x5b, 0xbc, 0xe5,
+						0xb2, 0x7d, 0x9f, 0xff, 0x15, 0x45, 0x68, 0x39,
+						0xe9, 0x19, 0x45, 0x3f, 0xc7, 0xb3, 0xf7, 0x21,
+						0xf0, 0xba, 0x40, 0x3f, 0xf9, 0x6c, 0x9d, 0xee,
+						0xb6, 0x80, 0xe5, 0xfd, 0x34, 0x1c, 0x0f, 0xc3,
+						0xa7, 0xb9, 0x0d, 0xa4, 0x63, 0x1e, 0xe3, 0x95,
+						0x60, 0x63, 0x9d, 0xb4, 0x62, 0xe9, 0xcb, 0x85,
+						0x0f, // 65-byte pubkey
+					},
+				},
+			},
+			TxOutputs: []btc.TxOutput{
+				{
+					Index: 0,
+					Value: 1000000,
+					PkScript: []byte{
+						0x76, // OP_DUP
+						0xa9, // OP_HASH160
+						0x14, // OP_DATA_20
+						0xb0, 0xdc, 0xbf, 0x97, 0xea, 0xbf, 0x44, 0x04,
+						0xe3, 0x1d, 0x95, 0x24, 0x77, 0xce, 0x82, 0x2d,
+						0xad, 0xbe, 0x7e, 0x10,
+						0x88, // OP_EQUALVERIFY
+						0xac, // OP_CHECKSIG
+					},
+					ScriptClass:  uint8(sClass3a),
+					RequiredSigs: int64(numOfSigs3a),
+					Addresses:    stringSliceFromAddresses(addresses3a),
+				},
+				{
+					Index: 1,
+					Value: 299000000,
+					PkScript: []byte{
+						0x76, // OP_DUP
+						0xa9, // OP_HASH160
+						0x14, // OP_DATA_20
+						0x6b, 0x12, 0x81, 0xee, 0xc2, 0x5a, 0xb4, 0xe1,
+						0xe0, 0x79, 0x3f, 0xf4, 0xe0, 0x8a, 0xb1, 0xab,
+						0xb3, 0x40, 0x9c, 0xd9,
+						0x88, // OP_EQUALVERIFY
+						0xac, // OP_CHECKSIG
+					},
+					ScriptClass:  uint8(sClass3b),
+					RequiredSigs: int64(numOfSigs3b),
+					Addresses:    stringSliceFromAddresses(addresses3b),
+				},
+			},
+		},
+	}
+	MockTxsMetaDataPostPublish = []btc.TxModelWithInsAndOuts{
+		{
+			CID:    "mockTrxCID1",
+			TxHash: MockBlock.Transactions[0].TxHash().String(),
+			Index:  0,
+			SegWit: MockBlock.Transactions[0].HasWitness(),
+			TxInputs: []btc.TxInput{
+				{
+					Index: 0,
+					SignatureScript: []byte{
+						0x04, 0x4c, 0x86, 0x04, 0x1b, 0x02, 0x06, 0x02,
+					},
+					PreviousOutPointHash:  chainhash.Hash{}.String(),
+					PreviousOutPointIndex: 0xffffffff,
+				},
+			},
+			TxOutputs: []btc.TxOutput{
+				{
+					Value: 5000000000,
+					Index: 0,
+					PkScript: []byte{
+						0x41, // OP_DATA_65
+						0x04, 0x1b, 0x0e, 0x8c, 0x25, 0x67, 0xc1, 0x25,
+						0x36, 0xaa, 0x13, 0x35, 0x7b, 0x79, 0xa0, 0x73,
+						0xdc, 0x44, 0x44, 0xac, 0xb8, 0x3c, 0x4e, 0xc7,
+						0xa0, 0xe2, 0xf9, 0x9d, 0xd7, 0x45, 0x75, 0x16,
+						0xc5, 0x81, 0x72, 0x42, 0xda, 0x79, 0x69, 0x24,
+						0xca, 0x4e, 0x99, 0x94, 0x7d, 0x08, 0x7f, 0xed,
+						0xf9, 0xce, 0x46, 0x7c, 0xb9, 0xf7, 0xc6, 0x28,
+						0x70, 0x78, 0xf8, 0x01, 0xdf, 0x27, 0x6f, 0xdf,
+						0x84, // 65-byte signature
+						0xac, // OP_CHECKSIG
+					},
+					ScriptClass:  uint8(sClass1),
+					RequiredSigs: int64(numOfSigs1),
+					Addresses:    stringSliceFromAddresses(addresses1),
+				},
+			},
+		},
+		{
+			CID:    "mockTrxCID2",
+			TxHash: MockBlock.Transactions[1].TxHash().String(),
+			Index:  1,
+			SegWit: MockBlock.Transactions[1].HasWitness(),
+			TxInputs: []btc.TxInput{
+				{
+					Index: 0,
+					PreviousOutPointHash: chainhash.Hash([32]byte{ // Make go vet happy.
+						0x03, 0x2e, 0x38, 0xe9, 0xc0, 0xa8, 0x4c, 0x60,
+						0x46, 0xd6, 0x87, 0xd1, 0x05, 0x56, 0xdc, 0xac,
+						0xc4, 0x1d, 0x27, 0x5e, 0xc5, 0x5f, 0xc0, 0x07,
+						0x79, 0xac, 0x88, 0xfd, 0xf3, 0x57, 0xa1, 0x87,
+					}).String(),
+					PreviousOutPointIndex: 0,
+					SignatureScript: []byte{
+						0x49, // OP_DATA_73
+						0x30, 0x46, 0x02, 0x21, 0x00, 0xc3, 0x52, 0xd3,
+						0xdd, 0x99, 0x3a, 0x98, 0x1b, 0xeb, 0xa4, 0xa6,
+						0x3a, 0xd1, 0x5c, 0x20, 0x92, 0x75, 0xca, 0x94,
+						0x70, 0xab, 0xfc, 0xd5, 0x7d, 0xa9, 0x3b, 0x58,
+						0xe4, 0xeb, 0x5d, 0xce, 0x82, 0x02, 0x21, 0x00,
+						0x84, 0x07, 0x92, 0xbc, 0x1f, 0x45, 0x60, 0x62,
+						0x81, 0x9f, 0x15, 0xd3, 0x3e, 0xe7, 0x05, 0x5c,
+						0xf7, 0xb5, 0xee, 0x1a, 0xf1, 0xeb, 0xcc, 0x60,
+						0x28, 0xd9, 0xcd, 0xb1, 0xc3, 0xaf, 0x77, 0x48,
+						0x01, // 73-byte signature
+						0x41, // OP_DATA_65
+						0x04, 0xf4, 0x6d, 0xb5, 0xe9, 0xd6, 0x1a, 0x9d,
+						0xc2, 0x7b, 0x8d, 0x64, 0xad, 0x23, 0xe7, 0x38,
+						0x3a, 0x4e, 0x6c, 0xa1, 0x64, 0x59, 0x3c, 0x25,
+						0x27, 0xc0, 0x38, 0xc0, 0x85, 0x7e, 0xb6, 0x7e,
+						0xe8, 0xe8, 0x25, 0xdc, 0xa6, 0x50, 0x46, 0xb8,
+						0x2c, 0x93, 0x31, 0x58, 0x6c, 0x82, 0xe0, 0xfd,
+						0x1f, 0x63, 0x3f, 0x25, 0xf8, 0x7c, 0x16, 0x1b,
+						0xc6, 0xf8, 0xa6, 0x30, 0x12, 0x1d, 0xf2, 0xb3,
+						0xd3, // 65-byte pubkey
+					},
+				},
+			},
+			TxOutputs: []btc.TxOutput{
+				{
+					Index: 0,
+					Value: 556000000,
+					PkScript: []byte{
+						0x76, // OP_DUP
+						0xa9, // OP_HASH160
+						0x14, // OP_DATA_20
+						0xc3, 0x98, 0xef, 0xa9, 0xc3, 0x92, 0xba, 0x60,
+						0x13, 0xc5, 0xe0, 0x4e, 0xe7, 0x29, 0x75, 0x5e,
+						0xf7, 0xf5, 0x8b, 0x32,
+						0x88, // OP_EQUALVERIFY
+						0xac, // OP_CHECKSIG
+					},
+					ScriptClass:  uint8(sClass2a),
+					RequiredSigs: int64(numOfSigs2a),
+					Addresses:    stringSliceFromAddresses(addresses2a),
+				},
+				{
+					Index: 1,
+					Value: 4444000000,
+					PkScript: []byte{
+						0x76, // OP_DUP
+						0xa9, // OP_HASH160
+						0x14, // OP_DATA_20
+						0x94, 0x8c, 0x76, 0x5a, 0x69, 0x14, 0xd4, 0x3f,
+						0x2a, 0x7a, 0xc1, 0x77, 0xda, 0x2c, 0x2f, 0x6b,
+						0x52, 0xde, 0x3d, 0x7c,
+						0x88, // OP_EQUALVERIFY
+						0xac, // OP_CHECKSIG
+					},
+					ScriptClass:  uint8(sClass2b),
+					RequiredSigs: int64(numOfSigs2b),
+					Addresses:    stringSliceFromAddresses(addresses2b),
+				},
+			},
+		},
+		{
+			CID:    "mockTrxCID3",
+			TxHash: MockBlock.Transactions[2].TxHash().String(),
+			Index:  2,
+			SegWit: MockBlock.Transactions[2].HasWitness(),
+			TxInputs: []btc.TxInput{
+				{
+					Index: 0,
+					PreviousOutPointHash: chainhash.Hash([32]byte{ // Make go vet happy.
+						0xc3, 0x3e, 0xbf, 0xf2, 0xa7, 0x09, 0xf1, 0x3d,
+						0x9f, 0x9a, 0x75, 0x69, 0xab, 0x16, 0xa3, 0x27,
+						0x86, 0xaf, 0x7d, 0x7e, 0x2d, 0xe0, 0x92, 0x65,
+						0xe4, 0x1c, 0x61, 0xd0, 0x78, 0x29, 0x4e, 0xcf,
+					}).String(),
+					PreviousOutPointIndex: 1,
+					SignatureScript: []byte{
+						0x47, // OP_DATA_71
+						0x30, 0x44, 0x02, 0x20, 0x03, 0x2d, 0x30, 0xdf,
+						0x5e, 0xe6, 0xf5, 0x7f, 0xa4, 0x6c, 0xdd, 0xb5,
+						0xeb, 0x8d, 0x0d, 0x9f, 0xe8, 0xde, 0x6b, 0x34,
+						0x2d, 0x27, 0x94, 0x2a, 0xe9, 0x0a, 0x32, 0x31,
+						0xe0, 0xba, 0x33, 0x3e, 0x02, 0x20, 0x3d, 0xee,
+						0xe8, 0x06, 0x0f, 0xdc, 0x70, 0x23, 0x0a, 0x7f,
+						0x5b, 0x4a, 0xd7, 0xd7, 0xbc, 0x3e, 0x62, 0x8c,
+						0xbe, 0x21, 0x9a, 0x88, 0x6b, 0x84, 0x26, 0x9e,
+						0xae, 0xb8, 0x1e, 0x26, 0xb4, 0xfe, 0x01,
+						0x41, // OP_DATA_65
+						0x04, 0xae, 0x31, 0xc3, 0x1b, 0xf9, 0x12, 0x78,
+						0xd9, 0x9b, 0x83, 0x77, 0xa3, 0x5b, 0xbc, 0xe5,
+						0xb2, 0x7d, 0x9f, 0xff, 0x15, 0x45, 0x68, 0x39,
+						0xe9, 0x19, 0x45, 0x3f, 0xc7, 0xb3, 0xf7, 0x21,
+						0xf0, 0xba, 0x40, 0x3f, 0xf9, 0x6c, 0x9d, 0xee,
+						0xb6, 0x80, 0xe5, 0xfd, 0x34, 0x1c, 0x0f, 0xc3,
+						0xa7, 0xb9, 0x0d, 0xa4, 0x63, 0x1e, 0xe3, 0x95,
+						0x60, 0x63, 0x9d, 0xb4, 0x62, 0xe9, 0xcb, 0x85,
+						0x0f, // 65-byte pubkey
+					},
+				},
+			},
+			TxOutputs: []btc.TxOutput{
+				{
+					Index: 0,
+					Value: 1000000,
+					PkScript: []byte{
+						0x76, // OP_DUP
+						0xa9, // OP_HASH160
+						0x14, // OP_DATA_20
+						0xb0, 0xdc, 0xbf, 0x97, 0xea, 0xbf, 0x44, 0x04,
+						0xe3, 0x1d, 0x95, 0x24, 0x77, 0xce, 0x82, 0x2d,
+						0xad, 0xbe, 0x7e, 0x10,
+						0x88, // OP_EQUALVERIFY
+						0xac, // OP_CHECKSIG
+					},
+					ScriptClass:  uint8(sClass3a),
+					RequiredSigs: int64(numOfSigs3a),
+					Addresses:    stringSliceFromAddresses(addresses3a),
+				},
+				{
+					Index: 1,
+					Value: 299000000,
+					PkScript: []byte{
+						0x76, // OP_DUP
+						0xa9, // OP_HASH160
+						0x14, // OP_DATA_20
+						0x6b, 0x12, 0x81, 0xee, 0xc2, 0x5a, 0xb4, 0xe1,
+						0xe0, 0x79, 0x3f, 0xf4, 0xe0, 0x8a, 0xb1, 0xab,
+						0xb3, 0x40, 0x9c, 0xd9,
+						0x88, // OP_EQUALVERIFY
+						0xac, // OP_CHECKSIG
+					},
+					ScriptClass:  uint8(sClass3b),
+					RequiredSigs: int64(numOfSigs3b),
+					Addresses:    stringSliceFromAddresses(addresses3b),
+				},
+			},
+		},
+	}
+	MockHeaderMetaData = btc.HeaderModel{
+		CID:         "mockHeaderCID",
+		ParentHash:  MockBlock.Header.PrevBlock.String(),
+		BlockNumber: strconv.Itoa(int(MockBlockHeight)),
+		BlockHash:   MockBlock.Header.BlockHash().String(),
+		Timestamp:   MockBlock.Header.Timestamp.UnixNano(),
+		Bits:        MockBlock.Header.Bits,
+	}
+	MockConvertedPayload = btc.ConvertedPayload{
+		BlockPayload: MockBlockPayload,
+		TxMetaData:   MockTxsMetaData,
+	}
+	MockCIDPayload = btc.CIDPayload{
+		HeaderCID:       MockHeaderMetaData,
+		TransactionCIDs: MockTxsMetaDataPostPublish,
+	}
+	DummyCIDPayloadForFKReference = btc.CIDPayload{
+		HeaderCID: btc.HeaderModel{
+			CID:         "dummyHeader",
+			ParentHash:  "",
+			BlockHash:   "",
+			BlockNumber: "1336",
+			Bits:        1,
+			Timestamp:   1000000000,
+		},
+		TransactionCIDs: []btc.TxModelWithInsAndOuts{
+			{
+				TxHash: "87a157f3fd88ac7907c05fc55e271dc4acdc5605d187d646604ca8c0e9382e03",
+				CID:    "dummyTx1",
+				Index:  0,
+				TxOutputs: []btc.TxOutput{
+					{
+						Index:        0,
+						RequiredSigs: 0,
+						Value:        0,
+						PkScript:     []byte{},
+						ScriptClass:  0,
+					},
+				},
+			},
+			{
+				TxHash: "cf4e2978d0611ce46592e02d7e7daf8627a316ab69759a9f3df109a7f2bf3ec3",
+				CID:    "dummyTx2",
+				Index:  1,
+				TxOutputs: []btc.TxOutput{
+					{
+						Index:        0,
+						RequiredSigs: 0,
+						Value:        0,
+						PkScript:     []byte{},
+						ScriptClass:  0,
+					},
+					{
+						Index:        1,
+						RequiredSigs: 0,
+						Value:        0,
+						PkScript:     []byte{},
+						ScriptClass:  0,
+					},
+				},
+			},
+		},
+	}
+)
+
+func stringSliceFromAddresses(addrs []btcutil.Address) []string {
+	strs := make([]string, len(addrs))
+	for i, addr := range addrs {
+		strs[i] = addr.EncodeAddress()
+	}
+	return strs
+}

pkg/btc/models.go

@@ -0,0 +1,79 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package btc
+
+import "github.com/lib/pq"
+
+// HeaderModel is the db model for btc.header_cids table
+type HeaderModel struct {
+	ID             int64  `db:"id"`
+	BlockNumber    string `db:"block_number"`
+	BlockHash      string `db:"block_hash"`
+	ParentHash     string `db:"parent_hash"`
+	CID            string `db:"cid"`
+	Timestamp      int64  `db:"timestamp"`
+	Bits           uint32 `db:"bits"`
+	NodeID         int64  `db:"node_id"`
+	TimesValidated int64  `db:"times_validated"`
+}
+
+// TxModel is the db model for btc.transaction_cids table
+type TxModel struct {
+	ID          int64  `db:"id"`
+	HeaderID    int64  `db:"header_id"`
+	Index       int64  `db:"index"`
+	TxHash      string `db:"tx_hash"`
+	CID         string `db:"cid"`
+	SegWit      bool   `db:"segwit"`
+	WitnessHash string `db:"witness_hash"`
+}
+
+// TxModelWithInsAndOuts is the db model for btc.transaction_cids table that includes the children tx_input and tx_output tables
+type TxModelWithInsAndOuts struct {
+	ID          int64  `db:"id"`
+	HeaderID    int64  `db:"header_id"`
+	Index       int64  `db:"index"`
+	TxHash      string `db:"tx_hash"`
+	CID         string `db:"cid"`
+	SegWit      bool   `db:"segwit"`
+	WitnessHash string `db:"witness_hash"`
+	TxInputs    []TxInput
+	TxOutputs   []TxOutput
+}
+
+// TxInput is the db model for btc.tx_inputs table
+type TxInput struct {
+	ID                    int64    `db:"id"`
+	TxID                  int64    `db:"tx_id"`
+	Index                 int64    `db:"index"`
+	TxWitness             []string `db:"witness"`
+	SignatureScript       []byte   `db:"sig_script"`
+	PreviousOutPointIndex uint32   `db:"outpoint_tx_hash"`
+	PreviousOutPointHash  string   `db:"outpoint_index"`
+}
+
+// TxOutput is the db model for btc.tx_outputs table
+type TxOutput struct {
+	ID           int64          `db:"id"`
+	TxID         int64          `db:"tx_id"`
+	Index        int64          `db:"index"`
+	Value        int64          `db:"value"`
+	PkScript     []byte         `db:"pk_script"`
+	ScriptClass  uint8          `db:"script_class"`
+	RequiredSigs int64          `db:"required_sigs"`
+	Addresses    pq.StringArray `db:"addresses"`
+}

pkg/btc/payload_fetcher.go

@@ -0,0 +1,76 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package btc
+
+import (
+	"fmt"
+
+	"github.com/btcsuite/btcd/rpcclient"
+	"github.com/btcsuite/btcd/wire"
+	"github.com/btcsuite/btcutil"
+
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
+)
+
+// PayloadFetcher satisfies the PayloadFetcher interface for bitcoin
+type PayloadFetcher struct {
+	// PayloadFetcher is thread-safe as long as the underlying client is thread-safe, since it has/modifies no other state
+	// http.Client is thread-safe
+	client *rpcclient.Client
+}
+
+// NewStateDiffFetcher returns a PayloadFetcher
+func NewPayloadFetcher(c *rpcclient.ConnConfig) (*PayloadFetcher, error) {
+	client, err := rpcclient.New(c, nil)
+	if err != nil {
+		return nil, err
+	}
+	return &PayloadFetcher{
+		client: client,
+	}, nil
+}
+
+// FetchAt fetches the block payloads at the given block heights
+func (fetcher *PayloadFetcher) FetchAt(blockHeights []uint64) ([]shared.RawChainData, error) {
+	blockPayloads := make([]shared.RawChainData, len(blockHeights))
+	for i, height := range blockHeights {
+		hash, err := fetcher.client.GetBlockHash(int64(height))
+		if err != nil {
+			return nil, fmt.Errorf("bitcoin PayloadFetcher GetBlockHash err at blockheight %d: %s", height, err.Error())
+		}
+		block, err := fetcher.client.GetBlock(hash)
+		if err != nil {
+			return nil, fmt.Errorf("bitcoin PayloadFetcher GetBlock err at blockheight %d: %s", height, err.Error())
+		}
+		blockPayloads[i] = BlockPayload{
+			BlockHeight: int64(height),
+			Header:      &block.Header,
+			Txs:         msgTxsToUtilTxs(block.Transactions),
+		}
+	}
+	return blockPayloads, nil
+}
+
+func msgTxsToUtilTxs(msgs []*wire.MsgTx) []*btcutil.Tx {
+	txs := make([]*btcutil.Tx, len(msgs))
+	for i, msg := range msgs {
+		tx := btcutil.NewTx(msg)
+		tx.SetIndex(i)
+		txs[i] = tx
+	}
+	return txs
+}

pkg/btc/publish_and_indexer.go

@@ -0,0 +1,124 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package btc
+
+import (
+	"fmt"
+	"strconv"
+
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/ipfs/ipld"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/postgres"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
+)
+
+// IPLDPublisherAndIndexer satisfies the IPLDPublisher interface for bitcoin
+// It interfaces directly with the public.blocks table of PG-IPFS rather than going through an ipfs intermediary
+// It publishes and indexes IPLDs together in a single sqlx.Tx
+type IPLDPublisherAndIndexer struct {
+	indexer *CIDIndexer
+}
+
+// NewIPLDPublisherAndIndexer creates a pointer to a new IPLDPublisherAndIndexer which satisfies the IPLDPublisher interface
+func NewIPLDPublisherAndIndexer(db *postgres.DB) *IPLDPublisherAndIndexer {
+	return &IPLDPublisherAndIndexer{
+		indexer: NewCIDIndexer(db),
+	}
+}
+
+// Publish publishes an IPLDPayload to IPFS and returns the corresponding CIDPayload
+func (pub *IPLDPublisherAndIndexer) Publish(payload shared.ConvertedData) (shared.CIDsForIndexing, error) {
+	ipldPayload, ok := payload.(ConvertedPayload)
+	if !ok {
+		return nil, fmt.Errorf("btc publisher expected payload type %T got %T", ConvertedPayload{}, payload)
+	}
+	// Generate the iplds
+	headerNode, txNodes, txTrieNodes, err := ipld.FromHeaderAndTxs(ipldPayload.Header, ipldPayload.Txs)
+	if err != nil {
+		return nil, err
+	}
+
+	// Begin new db tx
+	tx, err := pub.indexer.db.Beginx()
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		if p := recover(); p != nil {
+			shared.Rollback(tx)
+			panic(p)
+		} else if err != nil {
+			shared.Rollback(tx)
+		} else {
+			err = tx.Commit()
+		}
+	}()
+
+	// Publish trie nodes
+	for _, node := range txTrieNodes {
+		if err := shared.PublishIPLD(tx, node); err != nil {
+			return nil, err
+		}
+	}
+
+	// Publish and index header
+	if err := shared.PublishIPLD(tx, headerNode); err != nil {
+		return nil, err
+	}
+	header := HeaderModel{
+		CID:         headerNode.Cid().String(),
+		ParentHash:  ipldPayload.Header.PrevBlock.String(),
+		BlockNumber: strconv.Itoa(int(ipldPayload.BlockPayload.BlockHeight)),
+		BlockHash:   ipldPayload.Header.BlockHash().String(),
+		Timestamp:   ipldPayload.Header.Timestamp.UnixNano(),
+		Bits:        ipldPayload.Header.Bits,
+	}
+	headerID, err := pub.indexer.indexHeaderCID(tx, header)
+	if err != nil {
+		return nil, err
+	}
+
+	// Publish and index txs
+	for i, txNode := range txNodes {
+		if err := shared.PublishIPLD(tx, txNode); err != nil {
+			return nil, err
+		}
+		txModel := ipldPayload.TxMetaData[i]
+		txModel.CID = txNode.Cid().String()
+		txID, err := pub.indexer.indexTransactionCID(tx, txModel, headerID)
+		if err != nil {
+			return nil, err
+		}
+		for _, input := range txModel.TxInputs {
+			if err := pub.indexer.indexTxInput(tx, input, txID); err != nil {
+				return nil, err
+			}
+		}
+		for _, output := range txModel.TxOutputs {
+			if err := pub.indexer.indexTxOutput(tx, output, txID); err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	// This IPLDPublisher does both publishing and indexing, we do not need to pass anything forward to the indexer
+	return nil, err
+}
+
+// Index satisfies the shared.CIDIndexer interface
+func (pub *IPLDPublisherAndIndexer) Index(cids shared.CIDsForIndexing) error {
+	return nil
+}

pkg/btc/publish_and_indexer_test.go

@@ -0,0 +1,121 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package btc_test
+
+import (
+	"bytes"
+
+	"github.com/ipfs/go-cid"
+	"github.com/ipfs/go-ipfs-blockstore"
+	"github.com/ipfs/go-ipfs-ds-help"
+	"github.com/multiformats/go-multihash"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/ipfs/ipld"
+
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/btc"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/btc/mocks"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/postgres"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
+)
+
+var _ = Describe("PublishAndIndexer", func() {
+	var (
+		db        *postgres.DB
+		err       error
+		repo      *btc.IPLDPublisherAndIndexer
+		ipfsPgGet = `SELECT data FROM public.blocks
+					WHERE key = $1`
+	)
+	BeforeEach(func() {
+		db, err = shared.SetupDB()
+		Expect(err).ToNot(HaveOccurred())
+		repo = btc.NewIPLDPublisherAndIndexer(db)
+	})
+	AfterEach(func() {
+		btc.TearDownDB(db)
+	})
+
+	Describe("Publish", func() {
+		It("Published and indexes header and transaction IPLDs in a single tx", func() {
+			emptyReturn, err := repo.Publish(mocks.MockConvertedPayload)
+			Expect(emptyReturn).To(BeNil())
+			Expect(err).ToNot(HaveOccurred())
+			pgStr := `SELECT * FROM btc.header_cids
+				WHERE block_number = $1`
+			// check header was properly indexed
+			buf := bytes.NewBuffer(make([]byte, 0, 80))
+			err = mocks.MockBlock.Header.Serialize(buf)
+			Expect(err).ToNot(HaveOccurred())
+			headerBytes := buf.Bytes()
+			c, _ := ipld.RawdataToCid(ipld.MBitcoinHeader, headerBytes, multihash.DBL_SHA2_256)
+			header := new(btc.HeaderModel)
+			err = db.Get(header, pgStr, mocks.MockHeaderMetaData.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(header.CID).To(Equal(c.String()))
+			Expect(header.BlockNumber).To(Equal(mocks.MockHeaderMetaData.BlockNumber))
+			Expect(header.Bits).To(Equal(mocks.MockHeaderMetaData.Bits))
+			Expect(header.Timestamp).To(Equal(mocks.MockHeaderMetaData.Timestamp))
+			Expect(header.BlockHash).To(Equal(mocks.MockHeaderMetaData.BlockHash))
+			Expect(header.ParentHash).To(Equal(mocks.MockHeaderMetaData.ParentHash))
+			dc, err := cid.Decode(header.CID)
+			Expect(err).ToNot(HaveOccurred())
+			mhKey := dshelp.CidToDsKey(dc)
+			prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
+			var data []byte
+			err = db.Get(&data, ipfsPgGet, prefixedKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(data).To(Equal(headerBytes))
+
+			// check that txs were properly indexed
+			trxs := make([]btc.TxModel, 0)
+			pgStr = `SELECT transaction_cids.id, transaction_cids.header_id, transaction_cids.index,
+				transaction_cids.tx_hash, transaction_cids.cid, transaction_cids.segwit, transaction_cids.witness_hash
+				FROM btc.transaction_cids INNER JOIN btc.header_cids ON (transaction_cids.header_id = header_cids.id)
+				WHERE header_cids.block_number = $1`
+			err = db.Select(&trxs, pgStr, mocks.MockHeaderMetaData.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(trxs)).To(Equal(3))
+			txData := make([][]byte, len(mocks.MockTransactions))
+			txCIDs := make([]string, len(mocks.MockTransactions))
+			for i, m := range mocks.MockTransactions {
+				buf := bytes.NewBuffer(make([]byte, 0))
+				err = m.MsgTx().Serialize(buf)
+				Expect(err).ToNot(HaveOccurred())
+				tx := buf.Bytes()
+				txData[i] = tx
+				c, _ := ipld.RawdataToCid(ipld.MBitcoinTx, tx, multihash.DBL_SHA2_256)
+				txCIDs[i] = c.String()
+			}
+			for _, tx := range trxs {
+				Expect(tx.SegWit).To(Equal(false))
+				Expect(tx.HeaderID).To(Equal(header.ID))
+				Expect(tx.WitnessHash).To(Equal(""))
+				Expect(tx.CID).To(Equal(txCIDs[tx.Index]))
+				Expect(tx.TxHash).To(Equal(mocks.MockBlock.Transactions[tx.Index].TxHash().String()))
+				dc, err := cid.Decode(tx.CID)
+				Expect(err).ToNot(HaveOccurred())
+				mhKey := dshelp.CidToDsKey(dc)
+				prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
+				var data []byte
+				err = db.Get(&data, ipfsPgGet, prefixedKey)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(data).To(Equal(txData[tx.Index]))
+			}
+		})
+	})
+})

pkg/btc/publisher.go

@@ -0,0 +1,117 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package btc
+
+import (
+	"fmt"
+	"strconv"
+
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/ipfs"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/ipfs/dag_putters"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/ipfs/ipld"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
+)
+
+// IPLDPublisher satisfies the IPLDPublisher for ethereum
+type IPLDPublisher struct {
+	HeaderPutter          ipfs.DagPutter
+	TransactionPutter     ipfs.DagPutter
+	TransactionTriePutter ipfs.DagPutter
+}
+
+// NewIPLDPublisher creates a pointer to a new Publisher which satisfies the IPLDPublisher interface
+func NewIPLDPublisher(ipfsPath string) (*IPLDPublisher, error) {
+	node, err := ipfs.InitIPFSNode(ipfsPath)
+	if err != nil {
+		return nil, err
+	}
+	return &IPLDPublisher{
+		HeaderPutter:          dag_putters.NewBtcHeaderDagPutter(node),
+		TransactionPutter:     dag_putters.NewBtcTxDagPutter(node),
+		TransactionTriePutter: dag_putters.NewBtcTxTrieDagPutter(node),
+	}, nil
+}
+
+// Publish publishes an IPLDPayload to IPFS and returns the corresponding CIDPayload
+func (pub *IPLDPublisher) Publish(payload shared.ConvertedData) (shared.CIDsForIndexing, error) {
+	ipldPayload, ok := payload.(ConvertedPayload)
+	if !ok {
+		return nil, fmt.Errorf("eth publisher expected payload type %T got %T", &ConvertedPayload{}, payload)
+	}
+	// Generate nodes
+	headerNode, txNodes, txTrieNodes, err := ipld.FromHeaderAndTxs(ipldPayload.Header, ipldPayload.Txs)
+	if err != nil {
+		return nil, err
+	}
+	// Process and publish headers
+	headerCid, err := pub.publishHeader(headerNode)
+	if err != nil {
+		return nil, err
+	}
+	header := HeaderModel{
+		CID:         headerCid,
+		ParentHash:  ipldPayload.Header.PrevBlock.String(),
+		BlockNumber: strconv.Itoa(int(ipldPayload.BlockPayload.BlockHeight)),
+		BlockHash:   ipldPayload.Header.BlockHash().String(),
+		Timestamp:   ipldPayload.Header.Timestamp.UnixNano(),
+		Bits:        ipldPayload.Header.Bits,
+	}
+	// Process and publish transactions
+	transactionCids, err := pub.publishTransactions(txNodes, txTrieNodes, ipldPayload.TxMetaData)
+	if err != nil {
+		return nil, err
+	}
+	// Package CIDs and their metadata into a single struct
+	return &CIDPayload{
+		HeaderCID:       header,
+		TransactionCIDs: transactionCids,
+	}, nil
+}
+
+func (pub *IPLDPublisher) publishHeader(header *ipld.BtcHeader) (string, error) {
+	cid, err := pub.HeaderPutter.DagPut(header)
+	if err != nil {
+		return "", err
+	}
+	return cid, nil
+}
+
+func (pub *IPLDPublisher) publishTransactions(transactions []*ipld.BtcTx, txTrie []*ipld.BtcTxTrie, trxMeta []TxModelWithInsAndOuts) ([]TxModelWithInsAndOuts, error) {
+	txCids := make([]TxModelWithInsAndOuts, len(transactions))
+	for i, tx := range transactions {
+		cid, err := pub.TransactionPutter.DagPut(tx)
+		if err != nil {
+			return nil, err
+		}
+		txCids[i] = TxModelWithInsAndOuts{
+			CID:         cid,
+			Index:       trxMeta[i].Index,
+			TxHash:      trxMeta[i].TxHash,
+			SegWit:      trxMeta[i].SegWit,
+			WitnessHash: trxMeta[i].WitnessHash,
+			TxInputs:    trxMeta[i].TxInputs,
+			TxOutputs:   trxMeta[i].TxOutputs,
+		}
+	}
+	for _, txNode := range txTrie {
+		// We don't do anything with the tx trie cids atm
+		if _, err := pub.TransactionTriePutter.DagPut(txNode); err != nil {
+			return nil, err
+		}
+	}
+	return txCids, nil
+}

pkg/btc/publisher_test.go

@@ -0,0 +1,81 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package btc_test
+
+import (
+	"bytes"
+
+	"github.com/ethereum/go-ethereum/common"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/btc"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/btc/mocks"
+	mocks2 "github.com/vulcanize/ipfs-chain-watcher/pkg/ipfs/mocks"
+)
+
+var (
+	mockHeaderDagPutter  *mocks2.MappedDagPutter
+	mockTrxDagPutter     *mocks2.MappedDagPutter
+	mockTrxTrieDagPutter *mocks2.DagPutter
+)
+
+var _ = Describe("Publisher", func() {
+	BeforeEach(func() {
+		mockHeaderDagPutter = new(mocks2.MappedDagPutter)
+		mockTrxDagPutter = new(mocks2.MappedDagPutter)
+		mockTrxTrieDagPutter = new(mocks2.DagPutter)
+	})
+
+	Describe("Publish", func() {
+		It("Publishes the passed IPLDPayload objects to IPFS and returns a CIDPayload for indexing", func() {
+			by := new(bytes.Buffer)
+			err := mocks.MockConvertedPayload.BlockPayload.Header.Serialize(by)
+			Expect(err).ToNot(HaveOccurred())
+			headerBytes := by.Bytes()
+			err = mocks.MockTransactions[0].MsgTx().Serialize(by)
+			Expect(err).ToNot(HaveOccurred())
+			tx1Bytes := by.Bytes()
+			err = mocks.MockTransactions[1].MsgTx().Serialize(by)
+			Expect(err).ToNot(HaveOccurred())
+			tx2Bytes := by.Bytes()
+			err = mocks.MockTransactions[2].MsgTx().Serialize(by)
+			Expect(err).ToNot(HaveOccurred())
+			tx3Bytes := by.Bytes()
+			mockHeaderDagPutter.CIDsToReturn = map[common.Hash]string{
+				common.BytesToHash(headerBytes): "mockHeaderCID",
+			}
+			mockTrxDagPutter.CIDsToReturn = map[common.Hash]string{
+				common.BytesToHash(tx1Bytes): "mockTrxCID1",
+				common.BytesToHash(tx2Bytes): "mockTrxCID2",
+				common.BytesToHash(tx3Bytes): "mockTrxCID3",
+			}
+			publisher := btc.IPLDPublisher{
+				HeaderPutter:          mockHeaderDagPutter,
+				TransactionPutter:     mockTrxDagPutter,
+				TransactionTriePutter: mockTrxTrieDagPutter,
+			}
+			payload, err := publisher.Publish(mocks.MockConvertedPayload)
+			Expect(err).ToNot(HaveOccurred())
+			cidPayload, ok := payload.(*btc.CIDPayload)
+			Expect(ok).To(BeTrue())
+			Expect(cidPayload).To(Equal(&mocks.MockCIDPayload))
+			Expect(cidPayload.HeaderCID).To(Equal(mocks.MockHeaderMetaData))
+			Expect(cidPayload.TransactionCIDs).To(Equal(mocks.MockTxsMetaDataPostPublish))
+		})
+	})
+})

pkg/btc/streamer.go

@@ -0,0 +1,86 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package btc
+
+import (
+	"github.com/btcsuite/btcd/rpcclient"
+	"github.com/btcsuite/btcd/wire"
+	"github.com/btcsuite/btcutil"
+	"github.com/sirupsen/logrus"
+
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
+)
+
+const (
+	PayloadChanBufferSize = 20000 // the max eth sub buffer size
+)
+
+// PayloadStreamer satisfies the PayloadStreamer interface for bitcoin
+type PayloadStreamer struct {
+	Config *rpcclient.ConnConfig
+}
+
+// NewPayloadStreamer creates a pointer to a new PayloadStreamer which satisfies the PayloadStreamer interface for bitcoin
+func NewPayloadStreamer(clientConfig *rpcclient.ConnConfig) *PayloadStreamer {
+	return &PayloadStreamer{
+		Config: clientConfig,
+	}
+}
+
+// Stream is the main loop for subscribing to data from the btc block notifications
+// Satisfies the shared.PayloadStreamer interface
+func (ps *PayloadStreamer) Stream(payloadChan chan shared.RawChainData) (shared.ClientSubscription, error) {
+	logrus.Info("streaming block payloads from btc")
+	blockNotificationHandler := rpcclient.NotificationHandlers{
+		// Notification handler for block connections, forwards new block data to the payloadChan
+		OnFilteredBlockConnected: func(height int32, header *wire.BlockHeader, txs []*btcutil.Tx) {
+			payloadChan <- BlockPayload{
+				BlockHeight: int64(height),
+				Header:      header,
+				Txs:         txs,
+			}
+		},
+	}
+	// Create a new client, and connect to btc ws server
+	client, err := rpcclient.New(ps.Config, &blockNotificationHandler)
+	if err != nil {
+		return nil, err
+	}
+	// Register for block connect notifications.
+	if err := client.NotifyBlocks(); err != nil {
+		return nil, err
+	}
+	client.WaitForShutdown()
+	return &ClientSubscription{client: client}, nil
+}
+
+// ClientSubscription is a wrapper around the underlying btcd rpc client
+// to fit the shared.ClientSubscription interface
+type ClientSubscription struct {
+	client *rpcclient.Client
+}
+
+// Unsubscribe satisfies the rpc.Subscription interface
+func (bcs *ClientSubscription) Unsubscribe() {
+	bcs.client.Shutdown()
+}
+
+// Err() satisfies the rpc.Subscription interface with a dummy err channel
+func (bcs *ClientSubscription) Err() <-chan error {
+	errChan := make(chan error)
+	return errChan
+}

pkg/btc/subscription_config.go

@@ -0,0 +1,115 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package btc
+
+import (
+	"errors"
+	"math/big"
+
+	"github.com/spf13/viper"
+
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
+)
+
+// SubscriptionSettings config is used by a subscriber to specify what bitcoin data to stream from the super node
+type SubscriptionSettings struct {
+	BackFill     bool
+	BackFillOnly bool
+	Start        *big.Int
+	End          *big.Int // set to 0 or a negative value to have no ending block
+	HeaderFilter HeaderFilter
+	TxFilter     TxFilter
+}
+
+// HeaderFilter contains filter settings for headers
+type HeaderFilter struct {
+	Off bool
+}
+
+// TxFilter contains filter settings for txs
+type TxFilter struct {
+	Off             bool
+	Segwit          bool     // allow filtering for segwit trxs
+	WitnessHashes   []string // allow filtering for specific witness hashes
+	Indexes         []int64  // allow filtering for specific transaction indexes (e.g. 0 for coinbase transactions)
+	PkScriptClasses []uint8  // allow filtering for txs that have at least one tx output with the specified pkscript class
+	MultiSig        bool     // allow filtering for txs that have at least one tx output that requires more than one signature
+	Addresses       []string // allow filtering for txs that have at least one tx output with at least one of the provided addresses
+}
+
+// Init is used to initialize a EthSubscription struct with env variables
+func NewBtcSubscriptionConfig() (*SubscriptionSettings, error) {
+	sc := new(SubscriptionSettings)
+	// Below default to false, which means we do not backfill by default
+	sc.BackFill = viper.GetBool("superNode.btcSubscription.historicalData")
+	sc.BackFillOnly = viper.GetBool("superNode.btcSubscription.historicalDataOnly")
+	// Below default to 0
+	// 0 start means we start at the beginning and 0 end means we continue indefinitely
+	sc.Start = big.NewInt(viper.GetInt64("superNode.btcSubscription.startingBlock"))
+	sc.End = big.NewInt(viper.GetInt64("superNode.btcSubscription.endingBlock"))
+	// Below default to false, which means we get all headers by default
+	sc.HeaderFilter = HeaderFilter{
+		Off: viper.GetBool("superNode.btcSubscription.headerFilter.off"),
+	}
+	// Below defaults to false and two slices of length 0
+	// Which means we get all transactions by default
+	pksc := viper.Get("superNode.btcSubscription.txFilter.pkScriptClass")
+	pkScriptClasses, ok := pksc.([]uint8)
+	if !ok {
+		return nil, errors.New("superNode.btcSubscription.txFilter.pkScriptClass needs to be an array of uint8s")
+	}
+	is := viper.Get("superNode.btcSubscription.txFilter.indexes")
+	indexes, ok := is.([]int64)
+	if !ok {
+		return nil, errors.New("superNode.btcSubscription.txFilter.indexes needs to be an array of int64s")
+	}
+	sc.TxFilter = TxFilter{
+		Off:             viper.GetBool("superNode.btcSubscription.txFilter.off"),
+		Segwit:          viper.GetBool("superNode.btcSubscription.txFilter.segwit"),
+		WitnessHashes:   viper.GetStringSlice("superNode.btcSubscription.txFilter.witnessHashes"),
+		PkScriptClasses: pkScriptClasses,
+		Indexes:         indexes,
+		MultiSig:        viper.GetBool("superNode.btcSubscription.txFilter.multiSig"),
+		Addresses:       viper.GetStringSlice("superNode.btcSubscription.txFilter.addresses"),
+	}
+	return sc, nil
+}
+
+// StartingBlock satisfies the SubscriptionSettings() interface
+func (sc *SubscriptionSettings) StartingBlock() *big.Int {
+	return sc.Start
+}
+
+// EndingBlock satisfies the SubscriptionSettings() interface
+func (sc *SubscriptionSettings) EndingBlock() *big.Int {
+	return sc.End
+}
+
+// HistoricalData satisfies the SubscriptionSettings() interface
+func (sc *SubscriptionSettings) HistoricalData() bool {
+	return sc.BackFill
+}
+
+// HistoricalDataOnly satisfies the SubscriptionSettings() interface
+func (sc *SubscriptionSettings) HistoricalDataOnly() bool {
+	return sc.BackFillOnly
+}
+
+// ChainType satisfies the SubscriptionSettings() interface
+func (sc *SubscriptionSettings) ChainType() shared.ChainType {
+	return shared.Bitcoin
+}

pkg/btc/test_helpers.go

@@ -0,0 +1,43 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package btc
+
+import (
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/postgres"
+)
+
+// TearDownDB is used to tear down the super node dbs after tests
+func TearDownDB(db *postgres.DB) {
+	tx, err := db.Beginx()
+	Expect(err).NotTo(HaveOccurred())
+
+	_, err = tx.Exec(`DELETE FROM btc.header_cids`)
+	Expect(err).NotTo(HaveOccurred())
+	_, err = tx.Exec(`DELETE FROM btc.transaction_cids`)
+	Expect(err).NotTo(HaveOccurred())
+	_, err = tx.Exec(`DELETE FROM btc.tx_inputs`)
+	Expect(err).NotTo(HaveOccurred())
+	_, err = tx.Exec(`DELETE FROM btc.tx_outputs`)
+	Expect(err).NotTo(HaveOccurred())
+	_, err = tx.Exec(`DELETE FROM blocks`)
+	Expect(err).NotTo(HaveOccurred())
+
+	err = tx.Commit()
+	Expect(err).NotTo(HaveOccurred())
+}

pkg/btc/types.go

@@ -0,0 +1,76 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package btc
+
+import (
+	"math/big"
+
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/ipfs"
+
+	"github.com/btcsuite/btcd/wire"
+	"github.com/btcsuite/btcutil"
+)
+
+// BlockPayload packages the block and tx data received from block connection notifications
+type BlockPayload struct {
+	BlockHeight int64
+	Header      *wire.BlockHeader
+	Txs         []*btcutil.Tx
+}
+
+// ConvertedPayload is a custom type which packages raw BTC data for publishing to IPFS and filtering to subscribers
+// Returned by PayloadConverter
+// Passed to IPLDPublisher and ResponseFilterer
+type ConvertedPayload struct {
+	BlockPayload
+	TxMetaData []TxModelWithInsAndOuts
+}
+
+// Height satisfies the StreamedIPLDs interface
+func (cp ConvertedPayload) Height() int64 {
+	return cp.BlockPayload.BlockHeight
+}
+
+// CIDPayload is a struct to hold all the CIDs and their associated meta data for indexing in Postgres
+// Returned by IPLDPublisher
+// Passed to CIDIndexer
+type CIDPayload struct {
+	HeaderCID       HeaderModel
+	TransactionCIDs []TxModelWithInsAndOuts
+}
+
+// CIDWrapper is used to direct fetching of IPLDs from IPFS
+// Returned by CIDRetriever
+// Passed to IPLDFetcher
+type CIDWrapper struct {
+	BlockNumber  *big.Int
+	Header       HeaderModel
+	Transactions []TxModel
+}
+
+// IPLDs is used to package raw IPLD block data fetched from IPFS and returned by the server
+// Returned by IPLDFetcher and ResponseFilterer
+type IPLDs struct {
+	BlockNumber  *big.Int
+	Header       ipfs.BlockModel
+	Transactions []ipfs.BlockModel
+}
+
+// Height satisfies the StreamedIPLDs interface
+func (i IPLDs) Height() int64 {
+	return i.BlockNumber.Int64()
+}