btc converter, publisher, and indexer unit tests

db/migrations/00041_create_btc_header_cids_table.sql

@@ -5,9 +5,8 @@ CREATE TABLE btc.header_cids (
 	block_hash   VARCHAR(66) NOT NULL,
 	parent_hash  VARCHAR(66) NOT NULL,
 	cid          TEXT NOT NULL,
-	version      INTEGER NOT NULL,
+	timestamp    NUMERIC NOT NULL,
-	timestamp    INTEGER NOT NULL,
+	bits         BIGINT NOT NULL,
-	bits         INTEGER NOT NULL,
 	UNIQUE (block_number, block_hash)
 );
 

db/migrations/00043_create_btc_tx_inputs_table.sql

@@ -5,8 +5,8 @@ CREATE TABLE btc.tx_inputs (
 	index            INTEGER NOT NULL,
 	witness          BYTEA[],
 	sig_script       BYTEA NOT NULL,
-	outpoint_tx_id INTEGER NOT NULL REFERENCES btc.transaction_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
+	outpoint_tx_hash VARCHAR(66) REFERENCES btc.transaction_cids (tx_hash) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-	outpoint_index INTEGER NOT NULL,
+	outpoint_index   BIGINT NOT NULL,
 	UNIQUE (tx_id, index)
 );
 

db/migrations/00044_create_btc_tx_outputs_table.sql

@@ -3,7 +3,7 @@ CREATE TABLE btc.tx_outputs (
   id            SERIAL PRIMARY KEY,
 	tx_id         INTEGER NOT NULL REFERENCES btc.transaction_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
 	index         INTEGER NOT NULL,
-	value         INTEGER NOT NULL,
+	value         BIGINT NOT NULL,
 	pk_script     BYTEA NOT NULL,
 	script_class  INTEGER NOT NULL,
 	addresses     VARCHAR(66)[],

db/schema.sql

@@ -42,9 +42,8 @@ CREATE TABLE btc.header_cids (
     block_hash character varying(66) NOT NULL,
     parent_hash character varying(66) NOT NULL,
     cid text NOT NULL,
-    version integer NOT NULL,
+    "timestamp" numeric NOT NULL,
-    "timestamp" integer NOT NULL,
+    bits bigint NOT NULL
-    bits integer NOT NULL
 );
 
 
@@ -113,8 +112,8 @@ CREATE TABLE btc.tx_inputs (
     index integer NOT NULL,
     witness bytea[],
     sig_script bytea NOT NULL,
-    outpoint_tx_id integer NOT NULL,
+    outpoint_tx_hash character varying(66),
-    outpoint_index integer NOT NULL
+    outpoint_index bigint NOT NULL
 );
 
 
@@ -146,7 +145,7 @@ CREATE TABLE btc.tx_outputs (
     id integer NOT NULL,
     tx_id integer NOT NULL,
     index integer NOT NULL,
-    value integer NOT NULL,
+    value bigint NOT NULL,
     pk_script bytea NOT NULL,
     script_class integer NOT NULL,
     addresses character varying(66)[],
@@ -1747,11 +1746,11 @@ ALTER TABLE ONLY btc.transaction_cids
 
 
 --
--- Name: tx_inputs tx_inputs_outpoint_tx_id_fkey; Type: FK CONSTRAINT; Schema: btc; Owner: -
+-- Name: tx_inputs tx_inputs_outpoint_tx_hash_fkey; Type: FK CONSTRAINT; Schema: btc; Owner: -
 --
 
 ALTER TABLE ONLY btc.tx_inputs
-    ADD CONSTRAINT tx_inputs_outpoint_tx_id_fkey FOREIGN KEY (outpoint_tx_id) REFERENCES btc.transaction_cids(id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
+    ADD CONSTRAINT tx_inputs_outpoint_tx_hash_fkey FOREIGN KEY (outpoint_tx_hash) REFERENCES btc.transaction_cids(tx_hash) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
 
 
 --

pkg/super_node/btc/btc_suite_test.go

@@ -25,7 +25,7 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-func TestETHSuperNode(t *testing.T) {
+func TestBTCSuperNode(t *testing.T) {
 	RegisterFailHandler(Fail)
 	RunSpecs(t, "Super Node BTC Suite Test")
 }

pkg/super_node/btc/converter.go

@@ -32,7 +32,9 @@ type PayloadConverter struct {
 
 // NewPayloadConverter creates a pointer to a new PayloadConverter which satisfies the PayloadConverter interface
 func NewPayloadConverter(chainConfig *chaincfg.Params) *PayloadConverter {
-	return &PayloadConverter{}
+	return &PayloadConverter{
+		chainConfig: chainConfig,
+	}
 }
 
 // Convert method is used to convert a bitcoin BlockPayload to an IPLDPayload
@@ -43,11 +45,10 @@ func (pc *PayloadConverter) Convert(payload shared.RawChainData) (shared.Streame
 		return nil, fmt.Errorf("btc converter: expected payload type %T got %T", BlockPayload{}, payload)
 	}
 	txMeta := make([]TxModelWithInsAndOuts, len(btcBlockPayload.Txs))
-	for _, tx := range btcBlockPayload.Txs {
+	for i, tx := range btcBlockPayload.Txs {
-		index := tx.Index()
 		txModel := TxModelWithInsAndOuts{
 			TxHash:    tx.Hash().String(),
-			Index:     int64(index),
+			Index:     int64(i),
 			SegWit:    tx.HasWitness(),
 			TxOutputs: make([]TxOutput, len(tx.MsgTx().TxOut)),
 			TxInputs:  make([]TxInput, len(tx.MsgTx().TxIn)),
@@ -79,11 +80,11 @@ func (pc *PayloadConverter) Convert(payload shared.RawChainData) (shared.Streame
 				Value:        out.Value,
 				PkScript:     out.PkScript,
 				RequiredSigs: int64(numberOfSigs),
-				ScriptClass:  (uint8)(scriptClass),
+				ScriptClass:  uint8(scriptClass),
 				Addresses:    stringAddrs,
 			}
 		}
-		txMeta[index] = txModel
+		txMeta[i] = txModel
 	}
 	return IPLDPayload{
 		BlockPayload: btcBlockPayload,

pkg/super_node/btc/converter_test.go

@@ -0,0 +1,43 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package btc_test
+
+import (
+	"github.com/btcsuite/btcd/chaincfg"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/btc"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/btc/mocks"
+)
+
+var _ = Describe("Converter", func() {
+	Describe("Convert", func() {
+		It("Converts mock BlockPayloads into the expected IPLDPayloads", func() {
+			converter := btc.NewPayloadConverter(&chaincfg.MainNetParams)
+			payload, err := converter.Convert(mocks.MockBlockPayload)
+			Expect(err).ToNot(HaveOccurred())
+			convertedPayload, ok := payload.(btc.IPLDPayload)
+			Expect(ok).To(BeTrue())
+			Expect(convertedPayload).To(Equal(mocks.MockIPLDPayload))
+			Expect(convertedPayload.Height).To(Equal(mocks.MockBlockHeight))
+			Expect(convertedPayload.Header).To(Equal(&mocks.MockBlock.Header))
+			Expect(convertedPayload.Txs).To(Equal(mocks.MockTransactions))
+			Expect(convertedPayload.TxMetaData).To(Equal(mocks.MockTxsMetaData))
+		})
+	})
+})

pkg/super_node/btc/filterer.go

@@ -21,12 +21,10 @@ import (
 	"fmt"
 	"math/big"
 
-	"github.com/btcsuite/btcutil"
-
 	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
 )
 
-// ResponseFilterer satisfies the ResponseFilterer interface for ethereum
+// ResponseFilterer satisfies the ResponseFilterer interface for bitcoin
 type ResponseFilterer struct{}
 
 // NewResponseFilterer creates a new Filterer satisfying the ResponseFilterer interface
@@ -34,15 +32,15 @@ func NewResponseFilterer() *ResponseFilterer {
 	return &ResponseFilterer{}
 }
 
-// Filter is used to filter through eth data to extract and package requested data into a Payload
+// Filter is used to filter through btc data to extract and package requested data into a Payload
 func (s *ResponseFilterer) Filter(filter shared.SubscriptionSettings, payload shared.StreamedIPLDs) (shared.ServerResponse, error) {
 	btcFilters, ok := filter.(*SubscriptionSettings)
 	if !ok {
-		return StreamResponse{}, fmt.Errorf("eth filterer expected filter type %T got %T", &SubscriptionSettings{}, filter)
+		return StreamResponse{}, fmt.Errorf("btc filterer expected filter type %T got %T", &SubscriptionSettings{}, filter)
 	}
 	btcPayload, ok := payload.(IPLDPayload)
 	if !ok {
-		return StreamResponse{}, fmt.Errorf("eth filterer expected payload type %T got %T", IPLDPayload{}, payload)
+		return StreamResponse{}, fmt.Errorf("btc filterer expected payload type %T got %T", IPLDPayload{}, payload)
 	}
 	height := int64(btcPayload.Height)
 	if checkRange(btcFilters.Start.Int64(), btcFilters.End.Int64(), height) {
@@ -79,10 +77,10 @@ func checkRange(start, end, actual int64) bool {
 
 func (s *ResponseFilterer) filterTransactions(trxFilter TxFilter, response *StreamResponse, payload IPLDPayload) error {
 	if !trxFilter.Off {
-		for _, trx := range payload.Txs {
+		for i, txMeta := range payload.TxMetaData {
-			if checkTransaction(trx, trxFilter) {
+			if checkTransaction(txMeta, trxFilter) {
 				trxBuffer := new(bytes.Buffer)
-				if err := trx.MsgTx().Serialize(trxBuffer); err != nil {
+				if err := payload.Txs[i].MsgTx().Serialize(trxBuffer); err != nil {
 					return err
 				}
 				response.SerializedTxs = append(response.SerializedTxs, trxBuffer.Bytes())
@@ -93,6 +91,48 @@ func (s *ResponseFilterer) filterTransactions(trxFilter TxFilter, response *Stre
 }
 
 // checkTransaction returns true if the provided transaction has a hit on the filter
-func checkTransaction(trx *btcutil.Tx, txFilter TxFilter) bool {
+func checkTransaction(txMeta TxModelWithInsAndOuts, txFilter TxFilter) bool {
-	panic("implement me")
+	passesSegwitFilter := false
+	if !txFilter.Segwit || (txFilter.Segwit && txMeta.SegWit) {
+		passesSegwitFilter = true
+	}
+	passesMultiSigFilter := !txFilter.MultiSig
+	if txFilter.MultiSig {
+		for _, out := range txMeta.TxOutputs {
+			if out.RequiredSigs > 1 {
+				passesMultiSigFilter = true
+			}
+		}
+	}
+	passesWitnessFilter := len(txFilter.WitnessHashes) == 0
+	for _, wantedWitnessHash := range txFilter.WitnessHashes {
+		if wantedWitnessHash == txMeta.WitnessHash {
+			passesWitnessFilter = true
+		}
+	}
+	passesAddressFilter := len(txFilter.Addresses) == 0
+	for _, wantedAddress := range txFilter.Addresses {
+		for _, out := range txMeta.TxOutputs {
+			for _, actualAddress := range out.Addresses {
+				if wantedAddress == actualAddress {
+					passesAddressFilter = true
+				}
+			}
+		}
+	}
+	passesIndexFilter := len(txFilter.Indexes) == 0
+	for _, wantedIndex := range txFilter.Indexes {
+		if wantedIndex == txMeta.Index {
+			passesIndexFilter = true
+		}
+	}
+	passesPkScriptClassFilter := len(txFilter.PkScriptClasses) == 0
+	for _, wantedPkScriptClass := range txFilter.PkScriptClasses {
+		for _, out := range txMeta.TxOutputs {
+			if out.ScriptClass == wantedPkScriptClass {
+				passesPkScriptClassFilter = true
+			}
+		}
+	}
+	return passesSegwitFilter && passesMultiSigFilter && passesWitnessFilter && passesAddressFilter && passesIndexFilter && passesPkScriptClassFilter
 }

pkg/super_node/btc/filterer_test.go

@@ -0,0 +1,17 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package btc

pkg/super_node/btc/indexer.go

@@ -19,6 +19,8 @@ package btc
 import (
 	"fmt"
 
+	"github.com/sirupsen/logrus"
+
 	"github.com/jmoiron/sqlx"
 	"github.com/lib/pq"
 
@@ -47,9 +49,12 @@ func (in *CIDIndexer) Index(cids shared.CIDsForIndexing) error {
 	}
 	headerID, err := in.indexHeaderCID(tx, cidWrapper.HeaderCID)
 	if err != nil {
+		println("err")
+		logrus.Error("btc indexer error when indexing header")
 		return err
 	}
 	if err := in.indexTransactionCIDs(tx, cidWrapper.TransactionCIDs, headerID); err != nil {
+		logrus.Error("btc indexer error when indexing transactions")
 		return err
 	}
 	return tx.Commit()
@@ -57,11 +62,11 @@ func (in *CIDIndexer) Index(cids shared.CIDsForIndexing) error {
 
 func (in *CIDIndexer) indexHeaderCID(tx *sqlx.Tx, header HeaderModel) (int64, error) {
 	var headerID int64
-	err := tx.QueryRowx(`INSERT INTO btc.header_cids (block_number, block_hash, parent_hash, cid, version, timestamp, bits)
+	err := tx.QueryRowx(`INSERT INTO btc.header_cids (block_number, block_hash, parent_hash, cid, timestamp, bits)
-							VALUES ($1, $2, $3, $4, $5, $6, $7) 
+							VALUES ($1, $2, $3, $4, $5, $6) 
-							ON CONFLICT (block_number, block_hash) DO UPDATE SET (parent_hash, cid, version, timestamp, bits) = ($3, $4, $5, $6, $7)
+							ON CONFLICT (block_number, block_hash) DO UPDATE SET (parent_hash, cid, timestamp, bits) = ($3, $4, $5, $6)
 							RETURNING id`,
-		header.BlockNumber, header.BlockHash, header.ParentHash, header.CID, header.Version, header.Timestamp, header.Bits).Scan(&headerID)
+		header.BlockNumber, header.BlockHash, header.ParentHash, header.CID, header.Timestamp, header.Bits).Scan(&headerID)
 	return headerID, err
 }
 
@@ -69,15 +74,21 @@ func (in *CIDIndexer) indexTransactionCIDs(tx *sqlx.Tx, transactions []TxModelWi
 	for _, transaction := range transactions {
 		txID, err := in.indexTransactionCID(tx, transaction, headerID)
 		if err != nil {
+			println(0)
+			logrus.Error("btc indexer error when indexing header")
 			return err
 		}
 		for _, input := range transaction.TxInputs {
 			if err := in.indexTxInput(tx, input, txID); err != nil {
+				println(1)
+				logrus.Error("btc indexer error when indexing tx inputs")
 				return err
 			}
 		}
 		for _, output := range transaction.TxOutputs {
 			if err := in.indexTxOutput(tx, output, txID); err != nil {
+				println(2)
+				logrus.Error("btc indexer error when indexing tx outputs")
 				return err
 			}
 		}
@@ -96,25 +107,25 @@ func (in *CIDIndexer) indexTransactionCID(tx *sqlx.Tx, transaction TxModelWithIn
 }
 
 func (in *CIDIndexer) indexTxInput(tx *sqlx.Tx, txInput TxInput, txID int64) error {
-	var referencedOutPutTxID int64
+	// resolve zero-value hash to null value (coinbase tx input with no referenced outputs)
-	if err := tx.Get(&referencedOutPutTxID, `SELECT id FROM btc.transaction_cids
+	if txInput.PreviousOutPointHash == "0000000000000000000000000000000000000000000000000000000000000000" {
-						WHERE tx_hash = $1`, txInput.PreviousOutPointHash); err != nil {
+		_, err := tx.Exec(`INSERT INTO btc.tx_inputs (tx_id, index, witness, sig_script, outpoint_tx_hash, outpoint_index)
+						VALUES ($1, $2, $3, $4, $5, $6)
+						ON CONFLICT (tx_id, index) DO UPDATE SET (witness, sig_script, outpoint_tx_hash, outpoint_index) = ($3, $4, $5, $6)`,
+			txID, txInput.Index, pq.Array(txInput.TxWitness), txInput.SignatureScript, nil, txInput.PreviousOutPointIndex)
 		return err
 	}
-	if referencedOutPutTxID == 0 {
+	_, err := tx.Exec(`INSERT INTO btc.tx_inputs (tx_id, index, witness, sig_script, outpoint_tx_hash, outpoint_index)
-		return fmt.Errorf("btc indexer could not find the tx hash %s referenced in tx input of tx id %d", txInput.PreviousOutPointHash, txID)
-	}
-	_, err := tx.Exec(`INSERT INTO btc.tx_inputs (tx_id, index, witness, sig_script, outpoint_tx_id, outpoint_index)
 						VALUES ($1, $2, $3, $4, $5, $6)
-						ON CONFLICT (tx_id, index) DO UPDATE SET (witness, sig_script, outpoint_tx_id, outpoint_index) = ($3, $4, $5, $6)`,
+						ON CONFLICT (tx_id, index) DO UPDATE SET (witness, sig_script, outpoint_tx_hash, outpoint_index) = ($3, $4, $5, $6)`,
-		txID, txInput.Index, pq.Array(txInput.TxWitness), txInput.SignatureScript, referencedOutPutTxID, txInput.PreviousOutPointIndex)
+		txID, txInput.Index, pq.Array(txInput.TxWitness), txInput.SignatureScript, txInput.PreviousOutPointHash, txInput.PreviousOutPointIndex)
 	return err
 }
 
 func (in *CIDIndexer) indexTxOutput(tx *sqlx.Tx, txOuput TxOutput, txID int64) error {
 	_, err := tx.Exec(`INSERT INTO btc.tx_outputs (tx_id, index, value, pk_script, script_class, addresses, required_sigs)
 							VALUES ($1, $2, $3, $4, $5, $6, $7)
-							ON CONFLICT (ix_id, index) DO UPDATE SET (value, pk_script, script_class, addresses, required_sigs) = ($3, $4, $5, $6, $7)`,
+							ON CONFLICT (tx_id, index) DO UPDATE SET (value, pk_script, script_class, addresses, required_sigs) = ($3, $4, $5, $6, $7)`,
 		txID, txOuput.Index, txOuput.Value, txOuput.PkScript, txOuput.ScriptClass, txOuput.Addresses, txOuput.RequiredSigs)
 	return err
 }

pkg/super_node/btc/indexer_test.go

@@ -0,0 +1,89 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package btc_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/btc"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/btc/mocks"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+)
+
+var _ = Describe("Indexer", func() {
+	var (
+		db   *postgres.DB
+		err  error
+		repo *btc.CIDIndexer
+	)
+	BeforeEach(func() {
+		db, err = shared.SetupDB()
+		Expect(err).ToNot(HaveOccurred())
+		repo = btc.NewCIDIndexer(db)
+	})
+	AfterEach(func() {
+		btc.TearDownDB(db)
+	})
+
+	Describe("Index", func() {
+		It("Indexes CIDs and related metadata into vulcanizedb", func() {
+			err = repo.Index(&mocks.DummyCIDPayloadForFKReference)
+			Expect(err).ToNot(HaveOccurred())
+			err = repo.Index(&mocks.MockCIDPayload)
+			Expect(err).ToNot(HaveOccurred())
+			pgStr := `SELECT * FROM btc.header_cids
+				WHERE block_number = $1`
+			// check header was properly indexed
+			header := new(btc.HeaderModel)
+			err = db.Get(header, pgStr, mocks.MockHeaderMetaData.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(header.CID).To(Equal(mocks.MockHeaderMetaData.CID))
+			Expect(header.BlockNumber).To(Equal(mocks.MockHeaderMetaData.BlockNumber))
+			Expect(header.Bits).To(Equal(mocks.MockHeaderMetaData.Bits))
+			Expect(header.Timestamp).To(Equal(mocks.MockHeaderMetaData.Timestamp))
+			Expect(header.BlockHash).To(Equal(mocks.MockHeaderMetaData.BlockHash))
+			Expect(header.ParentHash).To(Equal(mocks.MockHeaderMetaData.ParentHash))
+			// check trxs were properly indexed
+			trxs := make([]btc.TxModel, 0)
+			pgStr = `SELECT transaction_cids.id, transaction_cids.header_id, transaction_cids.index,
+				transaction_cids.tx_hash, transaction_cids.cid, transaction_cids.segwit, transaction_cids.witness_hash
+				FROM btc.transaction_cids INNER JOIN btc.header_cids ON (transaction_cids.header_id = header_cids.id)
+				WHERE header_cids.block_number = $1`
+			err = db.Select(&trxs, pgStr, mocks.MockHeaderMetaData.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(trxs)).To(Equal(3))
+			for _, tx := range trxs {
+				Expect(tx.SegWit).To(Equal(false))
+				Expect(tx.HeaderID).To(Equal(header.ID))
+				Expect(tx.WitnessHash).To(Equal(""))
+				switch tx.Index {
+				case 0:
+					Expect(tx.CID).To(Equal("mockTrxCID1"))
+					Expect(tx.TxHash).To(Equal(mocks.MockBlock.Transactions[0].TxHash().String()))
+				case 1:
+					Expect(tx.CID).To(Equal("mockTrxCID2"))
+					Expect(tx.TxHash).To(Equal(mocks.MockBlock.Transactions[1].TxHash().String()))
+				case 2:
+					Expect(tx.CID).To(Equal("mockTrxCID3"))
+					Expect(tx.TxHash).To(Equal(mocks.MockBlock.Transactions[2].TxHash().String()))
+				}
+			}
+		})
+	})
+})

pkg/super_node/btc/ipld_fetcher_test.go

@@ -0,0 +1,17 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package btc

pkg/super_node/btc/mocks/test_data.go

@@ -0,0 +1,718 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"strconv"
+	"time"
+
+	"github.com/btcsuite/btcd/chaincfg"
+	"github.com/btcsuite/btcd/chaincfg/chainhash"
+	"github.com/btcsuite/btcd/txscript"
+	"github.com/btcsuite/btcd/wire"
+	"github.com/btcsuite/btcutil"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/btc"
+)
+
+var (
+	MockBlockHeight int32 = 1337
+	MockBlock             = wire.MsgBlock{
+		Header: wire.BlockHeader{
+			Version: 1,
+			PrevBlock: chainhash.Hash([32]byte{ // Make go vet happy.
+				0x50, 0x12, 0x01, 0x19, 0x17, 0x2a, 0x61, 0x04,
+				0x21, 0xa6, 0xc3, 0x01, 0x1d, 0xd3, 0x30, 0xd9,
+				0xdf, 0x07, 0xb6, 0x36, 0x16, 0xc2, 0xcc, 0x1f,
+				0x1c, 0xd0, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00,
+			}), // 000000000002d01c1fccc21636b607dfd930d31d01c3a62104612a1719011250
+			MerkleRoot: chainhash.Hash([32]byte{ // Make go vet happy.
+				0x66, 0x57, 0xa9, 0x25, 0x2a, 0xac, 0xd5, 0xc0,
+				0xb2, 0x94, 0x09, 0x96, 0xec, 0xff, 0x95, 0x22,
+				0x28, 0xc3, 0x06, 0x7c, 0xc3, 0x8d, 0x48, 0x85,
+				0xef, 0xb5, 0xa4, 0xac, 0x42, 0x47, 0xe9, 0xf3,
+			}), // f3e94742aca4b5ef85488dc37c06c3282295ffec960994b2c0d5ac2a25a95766
+			Timestamp: time.Unix(1293623863, 0), // 2010-12-29 11:57:43 +0000 UTC
+			Bits:      0x1b04864c,               // 453281356
+			Nonce:     0x10572b0f,               // 274148111
+		},
+		Transactions: []*wire.MsgTx{
+			{
+				Version: 1,
+				TxIn: []*wire.TxIn{
+					{
+						PreviousOutPoint: wire.OutPoint{
+							Hash:  chainhash.Hash{},
+							Index: 0xffffffff,
+						},
+						SignatureScript: []byte{
+							0x04, 0x4c, 0x86, 0x04, 0x1b, 0x02, 0x06, 0x02,
+						},
+						Sequence: 0xffffffff,
+					},
+				},
+				TxOut: []*wire.TxOut{
+					{
+						Value: 0x12a05f200, // 5000000000
+						PkScript: []byte{
+							0x41, // OP_DATA_65
+							0x04, 0x1b, 0x0e, 0x8c, 0x25, 0x67, 0xc1, 0x25,
+							0x36, 0xaa, 0x13, 0x35, 0x7b, 0x79, 0xa0, 0x73,
+							0xdc, 0x44, 0x44, 0xac, 0xb8, 0x3c, 0x4e, 0xc7,
+							0xa0, 0xe2, 0xf9, 0x9d, 0xd7, 0x45, 0x75, 0x16,
+							0xc5, 0x81, 0x72, 0x42, 0xda, 0x79, 0x69, 0x24,
+							0xca, 0x4e, 0x99, 0x94, 0x7d, 0x08, 0x7f, 0xed,
+							0xf9, 0xce, 0x46, 0x7c, 0xb9, 0xf7, 0xc6, 0x28,
+							0x70, 0x78, 0xf8, 0x01, 0xdf, 0x27, 0x6f, 0xdf,
+							0x84, // 65-byte signature
+							0xac, // OP_CHECKSIG
+						},
+					},
+				},
+				LockTime: 0,
+			},
+			{
+				Version: 1,
+				TxIn: []*wire.TxIn{
+					{
+						PreviousOutPoint: wire.OutPoint{
+							Hash: chainhash.Hash([32]byte{ // Make go vet happy.
+								0x03, 0x2e, 0x38, 0xe9, 0xc0, 0xa8, 0x4c, 0x60,
+								0x46, 0xd6, 0x87, 0xd1, 0x05, 0x56, 0xdc, 0xac,
+								0xc4, 0x1d, 0x27, 0x5e, 0xc5, 0x5f, 0xc0, 0x07,
+								0x79, 0xac, 0x88, 0xfd, 0xf3, 0x57, 0xa1, 0x87,
+							}), // 87a157f3fd88ac7907c05fc55e271dc4acdc5605d187d646604ca8c0e9382e03
+							Index: 0,
+						},
+						SignatureScript: []byte{
+							0x49, // OP_DATA_73
+							0x30, 0x46, 0x02, 0x21, 0x00, 0xc3, 0x52, 0xd3,
+							0xdd, 0x99, 0x3a, 0x98, 0x1b, 0xeb, 0xa4, 0xa6,
+							0x3a, 0xd1, 0x5c, 0x20, 0x92, 0x75, 0xca, 0x94,
+							0x70, 0xab, 0xfc, 0xd5, 0x7d, 0xa9, 0x3b, 0x58,
+							0xe4, 0xeb, 0x5d, 0xce, 0x82, 0x02, 0x21, 0x00,
+							0x84, 0x07, 0x92, 0xbc, 0x1f, 0x45, 0x60, 0x62,
+							0x81, 0x9f, 0x15, 0xd3, 0x3e, 0xe7, 0x05, 0x5c,
+							0xf7, 0xb5, 0xee, 0x1a, 0xf1, 0xeb, 0xcc, 0x60,
+							0x28, 0xd9, 0xcd, 0xb1, 0xc3, 0xaf, 0x77, 0x48,
+							0x01, // 73-byte signature
+							0x41, // OP_DATA_65
+							0x04, 0xf4, 0x6d, 0xb5, 0xe9, 0xd6, 0x1a, 0x9d,
+							0xc2, 0x7b, 0x8d, 0x64, 0xad, 0x23, 0xe7, 0x38,
+							0x3a, 0x4e, 0x6c, 0xa1, 0x64, 0x59, 0x3c, 0x25,
+							0x27, 0xc0, 0x38, 0xc0, 0x85, 0x7e, 0xb6, 0x7e,
+							0xe8, 0xe8, 0x25, 0xdc, 0xa6, 0x50, 0x46, 0xb8,
+							0x2c, 0x93, 0x31, 0x58, 0x6c, 0x82, 0xe0, 0xfd,
+							0x1f, 0x63, 0x3f, 0x25, 0xf8, 0x7c, 0x16, 0x1b,
+							0xc6, 0xf8, 0xa6, 0x30, 0x12, 0x1d, 0xf2, 0xb3,
+							0xd3, // 65-byte pubkey
+						},
+						Sequence: 0xffffffff,
+					},
+				},
+				TxOut: []*wire.TxOut{
+					{
+						Value: 0x2123e300, // 556000000
+						PkScript: []byte{
+							0x76, // OP_DUP
+							0xa9, // OP_HASH160
+							0x14, // OP_DATA_20
+							0xc3, 0x98, 0xef, 0xa9, 0xc3, 0x92, 0xba, 0x60,
+							0x13, 0xc5, 0xe0, 0x4e, 0xe7, 0x29, 0x75, 0x5e,
+							0xf7, 0xf5, 0x8b, 0x32,
+							0x88, // OP_EQUALVERIFY
+							0xac, // OP_CHECKSIG
+						},
+					},
+					{
+						Value: 0x108e20f00, // 4444000000
+						PkScript: []byte{
+							0x76, // OP_DUP
+							0xa9, // OP_HASH160
+							0x14, // OP_DATA_20
+							0x94, 0x8c, 0x76, 0x5a, 0x69, 0x14, 0xd4, 0x3f,
+							0x2a, 0x7a, 0xc1, 0x77, 0xda, 0x2c, 0x2f, 0x6b,
+							0x52, 0xde, 0x3d, 0x7c,
+							0x88, // OP_EQUALVERIFY
+							0xac, // OP_CHECKSIG
+						},
+					},
+				},
+				LockTime: 0,
+			},
+			{
+				Version: 1,
+				TxIn: []*wire.TxIn{
+					{
+						PreviousOutPoint: wire.OutPoint{
+							Hash: chainhash.Hash([32]byte{ // Make go vet happy.
+								0xc3, 0x3e, 0xbf, 0xf2, 0xa7, 0x09, 0xf1, 0x3d,
+								0x9f, 0x9a, 0x75, 0x69, 0xab, 0x16, 0xa3, 0x27,
+								0x86, 0xaf, 0x7d, 0x7e, 0x2d, 0xe0, 0x92, 0x65,
+								0xe4, 0x1c, 0x61, 0xd0, 0x78, 0x29, 0x4e, 0xcf,
+							}), // cf4e2978d0611ce46592e02d7e7daf8627a316ab69759a9f3df109a7f2bf3ec3
+							Index: 1,
+						},
+						SignatureScript: []byte{
+							0x47, // OP_DATA_71
+							0x30, 0x44, 0x02, 0x20, 0x03, 0x2d, 0x30, 0xdf,
+							0x5e, 0xe6, 0xf5, 0x7f, 0xa4, 0x6c, 0xdd, 0xb5,
+							0xeb, 0x8d, 0x0d, 0x9f, 0xe8, 0xde, 0x6b, 0x34,
+							0x2d, 0x27, 0x94, 0x2a, 0xe9, 0x0a, 0x32, 0x31,
+							0xe0, 0xba, 0x33, 0x3e, 0x02, 0x20, 0x3d, 0xee,
+							0xe8, 0x06, 0x0f, 0xdc, 0x70, 0x23, 0x0a, 0x7f,
+							0x5b, 0x4a, 0xd7, 0xd7, 0xbc, 0x3e, 0x62, 0x8c,
+							0xbe, 0x21, 0x9a, 0x88, 0x6b, 0x84, 0x26, 0x9e,
+							0xae, 0xb8, 0x1e, 0x26, 0xb4, 0xfe, 0x01,
+							0x41, // OP_DATA_65
+							0x04, 0xae, 0x31, 0xc3, 0x1b, 0xf9, 0x12, 0x78,
+							0xd9, 0x9b, 0x83, 0x77, 0xa3, 0x5b, 0xbc, 0xe5,
+							0xb2, 0x7d, 0x9f, 0xff, 0x15, 0x45, 0x68, 0x39,
+							0xe9, 0x19, 0x45, 0x3f, 0xc7, 0xb3, 0xf7, 0x21,
+							0xf0, 0xba, 0x40, 0x3f, 0xf9, 0x6c, 0x9d, 0xee,
+							0xb6, 0x80, 0xe5, 0xfd, 0x34, 0x1c, 0x0f, 0xc3,
+							0xa7, 0xb9, 0x0d, 0xa4, 0x63, 0x1e, 0xe3, 0x95,
+							0x60, 0x63, 0x9d, 0xb4, 0x62, 0xe9, 0xcb, 0x85,
+							0x0f, // 65-byte pubkey
+						},
+						Sequence: 0xffffffff,
+					},
+				},
+				TxOut: []*wire.TxOut{
+					{
+						Value: 0xf4240, // 1000000
+						PkScript: []byte{
+							0x76, // OP_DUP
+							0xa9, // OP_HASH160
+							0x14, // OP_DATA_20
+							0xb0, 0xdc, 0xbf, 0x97, 0xea, 0xbf, 0x44, 0x04,
+							0xe3, 0x1d, 0x95, 0x24, 0x77, 0xce, 0x82, 0x2d,
+							0xad, 0xbe, 0x7e, 0x10,
+							0x88, // OP_EQUALVERIFY
+							0xac, // OP_CHECKSIG
+						},
+					},
+					{
+						Value: 0x11d260c0, // 299000000
+						PkScript: []byte{
+							0x76, // OP_DUP
+							0xa9, // OP_HASH160
+							0x14, // OP_DATA_20
+							0x6b, 0x12, 0x81, 0xee, 0xc2, 0x5a, 0xb4, 0xe1,
+							0xe0, 0x79, 0x3f, 0xf4, 0xe0, 0x8a, 0xb1, 0xab,
+							0xb3, 0x40, 0x9c, 0xd9,
+							0x88, // OP_EQUALVERIFY
+							0xac, // OP_CHECKSIG
+						},
+					},
+				},
+				LockTime: 0,
+			},
+		},
+	}
+	MockTransactions = []*btcutil.Tx{
+		btcutil.NewTx(MockBlock.Transactions[0]),
+		btcutil.NewTx(MockBlock.Transactions[1]),
+		btcutil.NewTx(MockBlock.Transactions[2]),
+	}
+	MockBlockPayload = btc.BlockPayload{
+		Header: &MockBlock.Header,
+		Txs:    MockTransactions,
+		Height: MockBlockHeight,
+	}
+	sClass1, addresses1, numOfSigs1, _ = txscript.ExtractPkScriptAddrs([]byte{
+		0x41, // OP_DATA_65
+		0x04, 0x1b, 0x0e, 0x8c, 0x25, 0x67, 0xc1, 0x25,
+		0x36, 0xaa, 0x13, 0x35, 0x7b, 0x79, 0xa0, 0x73,
+		0xdc, 0x44, 0x44, 0xac, 0xb8, 0x3c, 0x4e, 0xc7,
+		0xa0, 0xe2, 0xf9, 0x9d, 0xd7, 0x45, 0x75, 0x16,
+		0xc5, 0x81, 0x72, 0x42, 0xda, 0x79, 0x69, 0x24,
+		0xca, 0x4e, 0x99, 0x94, 0x7d, 0x08, 0x7f, 0xed,
+		0xf9, 0xce, 0x46, 0x7c, 0xb9, 0xf7, 0xc6, 0x28,
+		0x70, 0x78, 0xf8, 0x01, 0xdf, 0x27, 0x6f, 0xdf,
+		0x84, // 65-byte signature
+		0xac, // OP_CHECKSIG
+	}, &chaincfg.MainNetParams)
+	sClass2a, addresses2a, numOfSigs2a, _ = txscript.ExtractPkScriptAddrs([]byte{
+		0x76, // OP_DUP
+		0xa9, // OP_HASH160
+		0x14, // OP_DATA_20
+		0xc3, 0x98, 0xef, 0xa9, 0xc3, 0x92, 0xba, 0x60,
+		0x13, 0xc5, 0xe0, 0x4e, 0xe7, 0x29, 0x75, 0x5e,
+		0xf7, 0xf5, 0x8b, 0x32,
+		0x88, // OP_EQUALVERIFY
+		0xac, // OP_CHECKSIG
+	}, &chaincfg.MainNetParams)
+	sClass2b, addresses2b, numOfSigs2b, _ = txscript.ExtractPkScriptAddrs([]byte{
+		0x76, // OP_DUP
+		0xa9, // OP_HASH160
+		0x14, // OP_DATA_20
+		0x94, 0x8c, 0x76, 0x5a, 0x69, 0x14, 0xd4, 0x3f,
+		0x2a, 0x7a, 0xc1, 0x77, 0xda, 0x2c, 0x2f, 0x6b,
+		0x52, 0xde, 0x3d, 0x7c,
+		0x88, // OP_EQUALVERIFY
+		0xac, // OP_CHECKSIG
+	}, &chaincfg.MainNetParams)
+	sClass3a, addresses3a, numOfSigs3a, _ = txscript.ExtractPkScriptAddrs([]byte{
+		0x76, // OP_DUP
+		0xa9, // OP_HASH160
+		0x14, // OP_DATA_20
+		0xb0, 0xdc, 0xbf, 0x97, 0xea, 0xbf, 0x44, 0x04,
+		0xe3, 0x1d, 0x95, 0x24, 0x77, 0xce, 0x82, 0x2d,
+		0xad, 0xbe, 0x7e, 0x10,
+		0x88, // OP_EQUALVERIFY
+		0xac, // OP_CHECKSIG
+	}, &chaincfg.MainNetParams)
+	sClass3b, addresses3b, numOfSigs3b, _ = txscript.ExtractPkScriptAddrs([]byte{
+		0x76, // OP_DUP
+		0xa9, // OP_HASH160
+		0x14, // OP_DATA_20
+		0x6b, 0x12, 0x81, 0xee, 0xc2, 0x5a, 0xb4, 0xe1,
+		0xe0, 0x79, 0x3f, 0xf4, 0xe0, 0x8a, 0xb1, 0xab,
+		0xb3, 0x40, 0x9c, 0xd9,
+		0x88, // OP_EQUALVERIFY
+		0xac, // OP_CHECKSIG
+	}, &chaincfg.MainNetParams)
+	MockTxsMetaData = []btc.TxModelWithInsAndOuts{
+		{
+			TxHash: MockBlock.Transactions[0].TxHash().String(),
+			Index:  0,
+			SegWit: MockBlock.Transactions[0].HasWitness(),
+			TxInputs: []btc.TxInput{
+				{
+					Index: 0,
+					SignatureScript: []byte{
+						0x04, 0x4c, 0x86, 0x04, 0x1b, 0x02, 0x06, 0x02,
+					},
+					PreviousOutPointHash:  chainhash.Hash{}.String(),
+					PreviousOutPointIndex: 0xffffffff,
+				},
+			},
+			TxOutputs: []btc.TxOutput{
+				{
+					Value: 5000000000,
+					Index: 0,
+					PkScript: []byte{
+						0x41, // OP_DATA_65
+						0x04, 0x1b, 0x0e, 0x8c, 0x25, 0x67, 0xc1, 0x25,
+						0x36, 0xaa, 0x13, 0x35, 0x7b, 0x79, 0xa0, 0x73,
+						0xdc, 0x44, 0x44, 0xac, 0xb8, 0x3c, 0x4e, 0xc7,
+						0xa0, 0xe2, 0xf9, 0x9d, 0xd7, 0x45, 0x75, 0x16,
+						0xc5, 0x81, 0x72, 0x42, 0xda, 0x79, 0x69, 0x24,
+						0xca, 0x4e, 0x99, 0x94, 0x7d, 0x08, 0x7f, 0xed,
+						0xf9, 0xce, 0x46, 0x7c, 0xb9, 0xf7, 0xc6, 0x28,
+						0x70, 0x78, 0xf8, 0x01, 0xdf, 0x27, 0x6f, 0xdf,
+						0x84, // 65-byte signature
+						0xac, // OP_CHECKSIG
+					},
+					ScriptClass:  uint8(sClass1),
+					RequiredSigs: int64(numOfSigs1),
+					Addresses:    stringSliceFromAddresses(addresses1),
+				},
+			},
+		},
+		{
+			TxHash: MockBlock.Transactions[1].TxHash().String(),
+			Index:  1,
+			SegWit: MockBlock.Transactions[1].HasWitness(),
+			TxInputs: []btc.TxInput{
+				{
+					Index: 0,
+					PreviousOutPointHash: chainhash.Hash([32]byte{ // Make go vet happy.
+						0x03, 0x2e, 0x38, 0xe9, 0xc0, 0xa8, 0x4c, 0x60,
+						0x46, 0xd6, 0x87, 0xd1, 0x05, 0x56, 0xdc, 0xac,
+						0xc4, 0x1d, 0x27, 0x5e, 0xc5, 0x5f, 0xc0, 0x07,
+						0x79, 0xac, 0x88, 0xfd, 0xf3, 0x57, 0xa1, 0x87,
+					}).String(),
+					PreviousOutPointIndex: 0,
+					SignatureScript: []byte{
+						0x49, // OP_DATA_73
+						0x30, 0x46, 0x02, 0x21, 0x00, 0xc3, 0x52, 0xd3,
+						0xdd, 0x99, 0x3a, 0x98, 0x1b, 0xeb, 0xa4, 0xa6,
+						0x3a, 0xd1, 0x5c, 0x20, 0x92, 0x75, 0xca, 0x94,
+						0x70, 0xab, 0xfc, 0xd5, 0x7d, 0xa9, 0x3b, 0x58,
+						0xe4, 0xeb, 0x5d, 0xce, 0x82, 0x02, 0x21, 0x00,
+						0x84, 0x07, 0x92, 0xbc, 0x1f, 0x45, 0x60, 0x62,
+						0x81, 0x9f, 0x15, 0xd3, 0x3e, 0xe7, 0x05, 0x5c,
+						0xf7, 0xb5, 0xee, 0x1a, 0xf1, 0xeb, 0xcc, 0x60,
+						0x28, 0xd9, 0xcd, 0xb1, 0xc3, 0xaf, 0x77, 0x48,
+						0x01, // 73-byte signature
+						0x41, // OP_DATA_65
+						0x04, 0xf4, 0x6d, 0xb5, 0xe9, 0xd6, 0x1a, 0x9d,
+						0xc2, 0x7b, 0x8d, 0x64, 0xad, 0x23, 0xe7, 0x38,
+						0x3a, 0x4e, 0x6c, 0xa1, 0x64, 0x59, 0x3c, 0x25,
+						0x27, 0xc0, 0x38, 0xc0, 0x85, 0x7e, 0xb6, 0x7e,
+						0xe8, 0xe8, 0x25, 0xdc, 0xa6, 0x50, 0x46, 0xb8,
+						0x2c, 0x93, 0x31, 0x58, 0x6c, 0x82, 0xe0, 0xfd,
+						0x1f, 0x63, 0x3f, 0x25, 0xf8, 0x7c, 0x16, 0x1b,
+						0xc6, 0xf8, 0xa6, 0x30, 0x12, 0x1d, 0xf2, 0xb3,
+						0xd3, // 65-byte pubkey
+					},
+				},
+			},
+			TxOutputs: []btc.TxOutput{
+				{
+					Index: 0,
+					Value: 556000000,
+					PkScript: []byte{
+						0x76, // OP_DUP
+						0xa9, // OP_HASH160
+						0x14, // OP_DATA_20
+						0xc3, 0x98, 0xef, 0xa9, 0xc3, 0x92, 0xba, 0x60,
+						0x13, 0xc5, 0xe0, 0x4e, 0xe7, 0x29, 0x75, 0x5e,
+						0xf7, 0xf5, 0x8b, 0x32,
+						0x88, // OP_EQUALVERIFY
+						0xac, // OP_CHECKSIG
+					},
+					ScriptClass:  uint8(sClass2a),
+					RequiredSigs: int64(numOfSigs2a),
+					Addresses:    stringSliceFromAddresses(addresses2a),
+				},
+				{
+					Index: 1,
+					Value: 4444000000,
+					PkScript: []byte{
+						0x76, // OP_DUP
+						0xa9, // OP_HASH160
+						0x14, // OP_DATA_20
+						0x94, 0x8c, 0x76, 0x5a, 0x69, 0x14, 0xd4, 0x3f,
+						0x2a, 0x7a, 0xc1, 0x77, 0xda, 0x2c, 0x2f, 0x6b,
+						0x52, 0xde, 0x3d, 0x7c,
+						0x88, // OP_EQUALVERIFY
+						0xac, // OP_CHECKSIG
+					},
+					ScriptClass:  uint8(sClass2b),
+					RequiredSigs: int64(numOfSigs2b),
+					Addresses:    stringSliceFromAddresses(addresses2b),
+				},
+			},
+		},
+		{
+			TxHash: MockBlock.Transactions[2].TxHash().String(),
+			Index:  2,
+			SegWit: MockBlock.Transactions[2].HasWitness(),
+			TxInputs: []btc.TxInput{
+				{
+					Index: 0,
+					PreviousOutPointHash: chainhash.Hash([32]byte{ // Make go vet happy.
+						0xc3, 0x3e, 0xbf, 0xf2, 0xa7, 0x09, 0xf1, 0x3d,
+						0x9f, 0x9a, 0x75, 0x69, 0xab, 0x16, 0xa3, 0x27,
+						0x86, 0xaf, 0x7d, 0x7e, 0x2d, 0xe0, 0x92, 0x65,
+						0xe4, 0x1c, 0x61, 0xd0, 0x78, 0x29, 0x4e, 0xcf,
+					}).String(),
+					PreviousOutPointIndex: 1,
+					SignatureScript: []byte{
+						0x47, // OP_DATA_71
+						0x30, 0x44, 0x02, 0x20, 0x03, 0x2d, 0x30, 0xdf,
+						0x5e, 0xe6, 0xf5, 0x7f, 0xa4, 0x6c, 0xdd, 0xb5,
+						0xeb, 0x8d, 0x0d, 0x9f, 0xe8, 0xde, 0x6b, 0x34,
+						0x2d, 0x27, 0x94, 0x2a, 0xe9, 0x0a, 0x32, 0x31,
+						0xe0, 0xba, 0x33, 0x3e, 0x02, 0x20, 0x3d, 0xee,
+						0xe8, 0x06, 0x0f, 0xdc, 0x70, 0x23, 0x0a, 0x7f,
+						0x5b, 0x4a, 0xd7, 0xd7, 0xbc, 0x3e, 0x62, 0x8c,
+						0xbe, 0x21, 0x9a, 0x88, 0x6b, 0x84, 0x26, 0x9e,
+						0xae, 0xb8, 0x1e, 0x26, 0xb4, 0xfe, 0x01,
+						0x41, // OP_DATA_65
+						0x04, 0xae, 0x31, 0xc3, 0x1b, 0xf9, 0x12, 0x78,
+						0xd9, 0x9b, 0x83, 0x77, 0xa3, 0x5b, 0xbc, 0xe5,
+						0xb2, 0x7d, 0x9f, 0xff, 0x15, 0x45, 0x68, 0x39,
+						0xe9, 0x19, 0x45, 0x3f, 0xc7, 0xb3, 0xf7, 0x21,
+						0xf0, 0xba, 0x40, 0x3f, 0xf9, 0x6c, 0x9d, 0xee,
+						0xb6, 0x80, 0xe5, 0xfd, 0x34, 0x1c, 0x0f, 0xc3,
+						0xa7, 0xb9, 0x0d, 0xa4, 0x63, 0x1e, 0xe3, 0x95,
+						0x60, 0x63, 0x9d, 0xb4, 0x62, 0xe9, 0xcb, 0x85,
+						0x0f, // 65-byte pubkey
+					},
+				},
+			},
+			TxOutputs: []btc.TxOutput{
+				{
+					Index: 0,
+					Value: 1000000,
+					PkScript: []byte{
+						0x76, // OP_DUP
+						0xa9, // OP_HASH160
+						0x14, // OP_DATA_20
+						0xb0, 0xdc, 0xbf, 0x97, 0xea, 0xbf, 0x44, 0x04,
+						0xe3, 0x1d, 0x95, 0x24, 0x77, 0xce, 0x82, 0x2d,
+						0xad, 0xbe, 0x7e, 0x10,
+						0x88, // OP_EQUALVERIFY
+						0xac, // OP_CHECKSIG
+					},
+					ScriptClass:  uint8(sClass3a),
+					RequiredSigs: int64(numOfSigs3a),
+					Addresses:    stringSliceFromAddresses(addresses3a),
+				},
+				{
+					Index: 1,
+					Value: 299000000,
+					PkScript: []byte{
+						0x76, // OP_DUP
+						0xa9, // OP_HASH160
+						0x14, // OP_DATA_20
+						0x6b, 0x12, 0x81, 0xee, 0xc2, 0x5a, 0xb4, 0xe1,
+						0xe0, 0x79, 0x3f, 0xf4, 0xe0, 0x8a, 0xb1, 0xab,
+						0xb3, 0x40, 0x9c, 0xd9,
+						0x88, // OP_EQUALVERIFY
+						0xac, // OP_CHECKSIG
+					},
+					ScriptClass:  uint8(sClass3b),
+					RequiredSigs: int64(numOfSigs3b),
+					Addresses:    stringSliceFromAddresses(addresses3b),
+				},
+			},
+		},
+	}
+	MockTxsMetaDataPostPublish = []btc.TxModelWithInsAndOuts{
+		{
+			CID:    "mockTrxCID1",
+			TxHash: MockBlock.Transactions[0].TxHash().String(),
+			Index:  0,
+			SegWit: MockBlock.Transactions[0].HasWitness(),
+			TxInputs: []btc.TxInput{
+				{
+					Index: 0,
+					SignatureScript: []byte{
+						0x04, 0x4c, 0x86, 0x04, 0x1b, 0x02, 0x06, 0x02,
+					},
+					PreviousOutPointHash:  chainhash.Hash{}.String(),
+					PreviousOutPointIndex: 0xffffffff,
+				},
+			},
+			TxOutputs: []btc.TxOutput{
+				{
+					Value: 5000000000,
+					Index: 0,
+					PkScript: []byte{
+						0x41, // OP_DATA_65
+						0x04, 0x1b, 0x0e, 0x8c, 0x25, 0x67, 0xc1, 0x25,
+						0x36, 0xaa, 0x13, 0x35, 0x7b, 0x79, 0xa0, 0x73,
+						0xdc, 0x44, 0x44, 0xac, 0xb8, 0x3c, 0x4e, 0xc7,
+						0xa0, 0xe2, 0xf9, 0x9d, 0xd7, 0x45, 0x75, 0x16,
+						0xc5, 0x81, 0x72, 0x42, 0xda, 0x79, 0x69, 0x24,
+						0xca, 0x4e, 0x99, 0x94, 0x7d, 0x08, 0x7f, 0xed,
+						0xf9, 0xce, 0x46, 0x7c, 0xb9, 0xf7, 0xc6, 0x28,
+						0x70, 0x78, 0xf8, 0x01, 0xdf, 0x27, 0x6f, 0xdf,
+						0x84, // 65-byte signature
+						0xac, // OP_CHECKSIG
+					},
+					ScriptClass:  uint8(sClass1),
+					RequiredSigs: int64(numOfSigs1),
+					Addresses:    stringSliceFromAddresses(addresses1),
+				},
+			},
+		},
+		{
+			CID:    "mockTrxCID2",
+			TxHash: MockBlock.Transactions[1].TxHash().String(),
+			Index:  1,
+			SegWit: MockBlock.Transactions[1].HasWitness(),
+			TxInputs: []btc.TxInput{
+				{
+					Index: 0,
+					PreviousOutPointHash: chainhash.Hash([32]byte{ // Make go vet happy.
+						0x03, 0x2e, 0x38, 0xe9, 0xc0, 0xa8, 0x4c, 0x60,
+						0x46, 0xd6, 0x87, 0xd1, 0x05, 0x56, 0xdc, 0xac,
+						0xc4, 0x1d, 0x27, 0x5e, 0xc5, 0x5f, 0xc0, 0x07,
+						0x79, 0xac, 0x88, 0xfd, 0xf3, 0x57, 0xa1, 0x87,
+					}).String(),
+					PreviousOutPointIndex: 0,
+					SignatureScript: []byte{
+						0x49, // OP_DATA_73
+						0x30, 0x46, 0x02, 0x21, 0x00, 0xc3, 0x52, 0xd3,
+						0xdd, 0x99, 0x3a, 0x98, 0x1b, 0xeb, 0xa4, 0xa6,
+						0x3a, 0xd1, 0x5c, 0x20, 0x92, 0x75, 0xca, 0x94,
+						0x70, 0xab, 0xfc, 0xd5, 0x7d, 0xa9, 0x3b, 0x58,
+						0xe4, 0xeb, 0x5d, 0xce, 0x82, 0x02, 0x21, 0x00,
+						0x84, 0x07, 0x92, 0xbc, 0x1f, 0x45, 0x60, 0x62,
+						0x81, 0x9f, 0x15, 0xd3, 0x3e, 0xe7, 0x05, 0x5c,
+						0xf7, 0xb5, 0xee, 0x1a, 0xf1, 0xeb, 0xcc, 0x60,
+						0x28, 0xd9, 0xcd, 0xb1, 0xc3, 0xaf, 0x77, 0x48,
+						0x01, // 73-byte signature
+						0x41, // OP_DATA_65
+						0x04, 0xf4, 0x6d, 0xb5, 0xe9, 0xd6, 0x1a, 0x9d,
+						0xc2, 0x7b, 0x8d, 0x64, 0xad, 0x23, 0xe7, 0x38,
+						0x3a, 0x4e, 0x6c, 0xa1, 0x64, 0x59, 0x3c, 0x25,
+						0x27, 0xc0, 0x38, 0xc0, 0x85, 0x7e, 0xb6, 0x7e,
+						0xe8, 0xe8, 0x25, 0xdc, 0xa6, 0x50, 0x46, 0xb8,
+						0x2c, 0x93, 0x31, 0x58, 0x6c, 0x82, 0xe0, 0xfd,
+						0x1f, 0x63, 0x3f, 0x25, 0xf8, 0x7c, 0x16, 0x1b,
+						0xc6, 0xf8, 0xa6, 0x30, 0x12, 0x1d, 0xf2, 0xb3,
+						0xd3, // 65-byte pubkey
+					},
+				},
+			},
+			TxOutputs: []btc.TxOutput{
+				{
+					Index: 0,
+					Value: 556000000,
+					PkScript: []byte{
+						0x76, // OP_DUP
+						0xa9, // OP_HASH160
+						0x14, // OP_DATA_20
+						0xc3, 0x98, 0xef, 0xa9, 0xc3, 0x92, 0xba, 0x60,
+						0x13, 0xc5, 0xe0, 0x4e, 0xe7, 0x29, 0x75, 0x5e,
+						0xf7, 0xf5, 0x8b, 0x32,
+						0x88, // OP_EQUALVERIFY
+						0xac, // OP_CHECKSIG
+					},
+					ScriptClass:  uint8(sClass2a),
+					RequiredSigs: int64(numOfSigs2a),
+					Addresses:    stringSliceFromAddresses(addresses2a),
+				},
+				{
+					Index: 1,
+					Value: 4444000000,
+					PkScript: []byte{
+						0x76, // OP_DUP
+						0xa9, // OP_HASH160
+						0x14, // OP_DATA_20
+						0x94, 0x8c, 0x76, 0x5a, 0x69, 0x14, 0xd4, 0x3f,
+						0x2a, 0x7a, 0xc1, 0x77, 0xda, 0x2c, 0x2f, 0x6b,
+						0x52, 0xde, 0x3d, 0x7c,
+						0x88, // OP_EQUALVERIFY
+						0xac, // OP_CHECKSIG
+					},
+					ScriptClass:  uint8(sClass2b),
+					RequiredSigs: int64(numOfSigs2b),
+					Addresses:    stringSliceFromAddresses(addresses2b),
+				},
+			},
+		},
+		{
+			CID:    "mockTrxCID3",
+			TxHash: MockBlock.Transactions[2].TxHash().String(),
+			Index:  2,
+			SegWit: MockBlock.Transactions[2].HasWitness(),
+			TxInputs: []btc.TxInput{
+				{
+					Index: 0,
+					PreviousOutPointHash: chainhash.Hash([32]byte{ // Make go vet happy.
+						0xc3, 0x3e, 0xbf, 0xf2, 0xa7, 0x09, 0xf1, 0x3d,
+						0x9f, 0x9a, 0x75, 0x69, 0xab, 0x16, 0xa3, 0x27,
+						0x86, 0xaf, 0x7d, 0x7e, 0x2d, 0xe0, 0x92, 0x65,
+						0xe4, 0x1c, 0x61, 0xd0, 0x78, 0x29, 0x4e, 0xcf,
+					}).String(),
+					PreviousOutPointIndex: 1,
+					SignatureScript: []byte{
+						0x47, // OP_DATA_71
+						0x30, 0x44, 0x02, 0x20, 0x03, 0x2d, 0x30, 0xdf,
+						0x5e, 0xe6, 0xf5, 0x7f, 0xa4, 0x6c, 0xdd, 0xb5,
+						0xeb, 0x8d, 0x0d, 0x9f, 0xe8, 0xde, 0x6b, 0x34,
+						0x2d, 0x27, 0x94, 0x2a, 0xe9, 0x0a, 0x32, 0x31,
+						0xe0, 0xba, 0x33, 0x3e, 0x02, 0x20, 0x3d, 0xee,
+						0xe8, 0x06, 0x0f, 0xdc, 0x70, 0x23, 0x0a, 0x7f,
+						0x5b, 0x4a, 0xd7, 0xd7, 0xbc, 0x3e, 0x62, 0x8c,
+						0xbe, 0x21, 0x9a, 0x88, 0x6b, 0x84, 0x26, 0x9e,
+						0xae, 0xb8, 0x1e, 0x26, 0xb4, 0xfe, 0x01,
+						0x41, // OP_DATA_65
+						0x04, 0xae, 0x31, 0xc3, 0x1b, 0xf9, 0x12, 0x78,
+						0xd9, 0x9b, 0x83, 0x77, 0xa3, 0x5b, 0xbc, 0xe5,
+						0xb2, 0x7d, 0x9f, 0xff, 0x15, 0x45, 0x68, 0x39,
+						0xe9, 0x19, 0x45, 0x3f, 0xc7, 0xb3, 0xf7, 0x21,
+						0xf0, 0xba, 0x40, 0x3f, 0xf9, 0x6c, 0x9d, 0xee,
+						0xb6, 0x80, 0xe5, 0xfd, 0x34, 0x1c, 0x0f, 0xc3,
+						0xa7, 0xb9, 0x0d, 0xa4, 0x63, 0x1e, 0xe3, 0x95,
+						0x60, 0x63, 0x9d, 0xb4, 0x62, 0xe9, 0xcb, 0x85,
+						0x0f, // 65-byte pubkey
+					},
+				},
+			},
+			TxOutputs: []btc.TxOutput{
+				{
+					Index: 0,
+					Value: 1000000,
+					PkScript: []byte{
+						0x76, // OP_DUP
+						0xa9, // OP_HASH160
+						0x14, // OP_DATA_20
+						0xb0, 0xdc, 0xbf, 0x97, 0xea, 0xbf, 0x44, 0x04,
+						0xe3, 0x1d, 0x95, 0x24, 0x77, 0xce, 0x82, 0x2d,
+						0xad, 0xbe, 0x7e, 0x10,
+						0x88, // OP_EQUALVERIFY
+						0xac, // OP_CHECKSIG
+					},
+					ScriptClass:  uint8(sClass3a),
+					RequiredSigs: int64(numOfSigs3a),
+					Addresses:    stringSliceFromAddresses(addresses3a),
+				},
+				{
+					Index: 1,
+					Value: 299000000,
+					PkScript: []byte{
+						0x76, // OP_DUP
+						0xa9, // OP_HASH160
+						0x14, // OP_DATA_20
+						0x6b, 0x12, 0x81, 0xee, 0xc2, 0x5a, 0xb4, 0xe1,
+						0xe0, 0x79, 0x3f, 0xf4, 0xe0, 0x8a, 0xb1, 0xab,
+						0xb3, 0x40, 0x9c, 0xd9,
+						0x88, // OP_EQUALVERIFY
+						0xac, // OP_CHECKSIG
+					},
+					ScriptClass:  uint8(sClass3b),
+					RequiredSigs: int64(numOfSigs3b),
+					Addresses:    stringSliceFromAddresses(addresses3b),
+				},
+			},
+		},
+	}
+	MockHeaderMetaData = btc.HeaderModel{
+		CID:         "mockHeaderCID",
+		ParentHash:  MockBlock.Header.PrevBlock.String(),
+		BlockNumber: strconv.Itoa(int(MockBlockHeight)),
+		BlockHash:   MockBlock.Header.BlockHash().String(),
+		Timestamp:   MockBlock.Header.Timestamp.UnixNano(),
+		Bits:        MockBlock.Header.Bits,
+	}
+	MockIPLDPayload = btc.IPLDPayload{
+		BlockPayload: MockBlockPayload,
+		TxMetaData:   MockTxsMetaData,
+	}
+	MockCIDPayload = btc.CIDPayload{
+		HeaderCID:       MockHeaderMetaData,
+		TransactionCIDs: MockTxsMetaDataPostPublish,
+	}
+	DummyCIDPayloadForFKReference = btc.CIDPayload{
+		HeaderCID: btc.HeaderModel{
+			CID:         "dummyHeader",
+			ParentHash:  "",
+			BlockHash:   "",
+			BlockNumber: "1336",
+			Bits:        1,
+			Timestamp:   1000000000,
+		},
+		TransactionCIDs: []btc.TxModelWithInsAndOuts{
+			{
+				TxHash: "87a157f3fd88ac7907c05fc55e271dc4acdc5605d187d646604ca8c0e9382e03",
+				CID:    "dummyTx1",
+				Index:  0,
+			},
+			{
+				TxHash: "cf4e2978d0611ce46592e02d7e7daf8627a316ab69759a9f3df109a7f2bf3ec3",
+				CID:    "dummyTx2",
+				Index:  1,
+			},
+		},
+	}
+)
+
+func stringSliceFromAddresses(addrs []btcutil.Address) []string {
+	strs := make([]string, len(addrs))
+	for i, addr := range addrs {
+		strs[i] = addr.EncodeAddress()
+	}
+	return strs
+}

pkg/super_node/btc/models.go

@@ -25,7 +25,6 @@ type HeaderModel struct {
 	BlockHash   string `db:"block_hash"`
 	ParentHash  string `db:"parent_hash"`
 	CID         string `db:"cid"`
-	Version     int32  `db:"version"`
 	Timestamp   int64  `db:"timestamp"`
 	Bits        uint32 `db:"bits"`
 }
@@ -61,9 +60,8 @@ type TxInput struct {
 	Index                 int64    `db:"index"`
 	TxWitness             [][]byte `db:"witness"`
 	SignatureScript       []byte   `db:"sig_script"`
-	PreviousOutPointTxID  int64    `db:"outpoint_tx_id"`
 	PreviousOutPointIndex uint32   `db:"outpoint_index"`
-	PreviousOutPointHash  string
+	PreviousOutPointHash  string   `db:"outpoint_tx_hash"`
 }
 
 // TxOutput is the db model for btc.tx_outputs table

pkg/super_node/btc/payload_fetcher_test.go

@@ -0,0 +1,17 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package btc

pkg/super_node/btc/publisher.go

@@ -63,7 +63,6 @@ func (pub *IPLDPublisher) Publish(payload shared.StreamedIPLDs) (shared.CIDsForI
 		ParentHash:  ipldPayload.Header.PrevBlock.String(),
 		BlockNumber: strconv.Itoa(int(ipldPayload.Height)),
 		BlockHash:   ipldPayload.Header.BlockHash().String(),
-		Version:     ipldPayload.Header.Version,
 		Timestamp:   ipldPayload.Header.Timestamp.UnixNano(),
 		Bits:        ipldPayload.Header.Bits,
 	}

pkg/super_node/btc/publisher_test.go

@@ -0,0 +1,56 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package btc_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	mocks2 "github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/btc"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/btc/mocks"
+)
+
+var (
+	mockHeaderDagPutter *mocks2.DagPutter
+	mockTrxDagPutter    *mocks2.DagPutter
+)
+
+var _ = Describe("Publisher", func() {
+	BeforeEach(func() {
+		mockHeaderDagPutter = new(mocks2.DagPutter)
+		mockTrxDagPutter = new(mocks2.DagPutter)
+	})
+
+	Describe("Publish", func() {
+		It("Publishes the passed IPLDPayload objects to IPFS and returns a CIDPayload for indexing", func() {
+			mockHeaderDagPutter.CIDsToReturn = []string{"mockHeaderCID"}
+			mockTrxDagPutter.CIDsToReturn = []string{"mockTrxCID1", "mockTrxCID2", "mockTrxCID3"}
+			publisher := btc.IPLDPublisher{
+				HeaderPutter:      mockHeaderDagPutter,
+				TransactionPutter: mockTrxDagPutter,
+			}
+			payload, err := publisher.Publish(mocks.MockIPLDPayload)
+			Expect(err).ToNot(HaveOccurred())
+			cidPayload, ok := payload.(*btc.CIDPayload)
+			Expect(ok).To(BeTrue())
+			Expect(cidPayload).To(Equal(&mocks.MockCIDPayload))
+			Expect(cidPayload.HeaderCID).To(Equal(mocks.MockHeaderMetaData))
+			Expect(cidPayload.TransactionCIDs).To(Equal(mocks.MockTxsMetaDataPostPublish))
+		})
+	})
+})

pkg/super_node/btc/resolver_test.go

@@ -0,0 +1,17 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package btc

pkg/super_node/btc/retriever.go

@@ -20,6 +20,8 @@ import (
 	"fmt"
 	"math/big"
 
+	"github.com/lib/pq"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/jmoiron/sqlx"
 	log "github.com/sirupsen/logrus"
@@ -58,7 +60,7 @@ func (ecr *CIDRetriever) RetrieveLastBlockNumber() (int64, error) {
 func (ecr *CIDRetriever) Retrieve(filter shared.SubscriptionSettings, blockNumber int64) (shared.CIDsForFetching, bool, error) {
 	streamFilter, ok := filter.(*SubscriptionSettings)
 	if !ok {
-		return nil, true, fmt.Errorf("eth retriever expected filter type %T got %T", &SubscriptionSettings{}, filter)
+		return nil, true, fmt.Errorf("btc retriever expected filter type %T got %T", &SubscriptionSettings{}, filter)
 	}
 	log.Debug("retrieving cids")
 	tx, err := ecr.db.Beginx()
@@ -90,10 +92,6 @@ func (ecr *CIDRetriever) Retrieve(filter shared.SubscriptionSettings, blockNumbe
 			return nil, true, err
 		}
 	}
-	trxIds := make([]int64, 0, len(cw.Transactions))
-	for _, tx := range cw.Transactions {
-		trxIds = append(trxIds, tx.ID)
-	}
 	return cw, empty(cw), tx.Commit()
 }
 
@@ -113,18 +111,70 @@ func (ecr *CIDRetriever) RetrieveHeaderCIDs(tx *sqlx.Tx, blockNumber int64) ([]H
 	return headers, tx.Select(&headers, pgStr, blockNumber)
 }
 
+/*
+type TxModel struct {
+	ID          int64  `db:"id"`
+	HeaderID    int64  `db:"header_id"`
+	Index       int64  `db:"index"`
+	TxHash      string `db:"tx_hash"`
+	CID         string `db:"cid"`
+	SegWit      bool   `db:"segwit"`
+	WitnessHash string `db:"witness_hash"`
+}
+// TxFilter contains filter settings for txs
+type TxFilter struct {
+	Off bool
+	Index         int64    // allow filtering by index so that we can filter for only coinbase transactions (index 0) if we want to
+	Segwit        bool     // allow filtering for segwit trxs
+	WitnessHashes []string // allow filtering for specific witness hashes
+	PkScriptClass uint8 // allow filtering for txs that have at least one tx output with the specified pkscript class
+	MultiSig bool  // allow filtering for txs that have at least one tx output that requires more than one signature
+	Addresses []string // allow filtering for txs that have at least one tx output with at least one of the provided addresses
+}
+*/
+
 // RetrieveTxCIDs retrieves and returns all of the trx cids at the provided blockheight that conform to the provided filter parameters
 // also returns the ids for the returned transaction cids
 func (ecr *CIDRetriever) RetrieveTxCIDs(tx *sqlx.Tx, txFilter TxFilter, blockNumber int64) ([]TxModel, error) {
 	log.Debug("retrieving transaction cids for block ", blockNumber)
 	args := make([]interface{}, 0, 3)
 	results := make([]TxModel, 0)
-	pgStr := `SELECT transaction_cids.id, transaction_cids.header_id,
+	id := 1
+	pgStr := fmt.Sprintf(`SELECT transaction_cids.id, transaction_cids.header_id,
  			transaction_cids.tx_hash, transaction_cids.cid,
- 			transaction_cids.dst, transaction_cids.src, transaction_cids.index
+ 			transaction_cids.segwit, transaction_cids.witness_hash, transaction_cids.index
- 			FROM eth.transaction_cids INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id)
+ 			FROM btc.transaction_cids, btc.header_cids, btc.tx_inputs, btc.tx_outputs
-			WHERE header_cids.block_number = $1`
+			WHERE transaction_cids.header_id = header_cids.id
+			AND tx_inputs.tx_id = transaction_cids.id
+			AND tx_outputs.tx_id = transaction_cids.id
+			AND header_cids.block_number = $%d`, id)
 	args = append(args, blockNumber)
+	id++
+	if txFilter.Segwit {
+		pgStr += ` AND transaction_cids.segwit = true`
+	}
+	if txFilter.MultiSig {
+		pgStr += ` AND tx_outputs.required_sigs > 1`
+	}
+	if len(txFilter.WitnessHashes) > 0 {
+		pgStr += fmt.Sprintf(` AND transaction_cids.witness_hash = ANY($%d::VARCHAR(66)[])`, id)
+		args = append(args, pq.Array(txFilter.WitnessHashes))
+		id++
+	}
+	if len(txFilter.Addresses) > 0 {
+		pgStr += fmt.Sprintf(` AND tx_outputs.addresses && $%d::VARCHAR(66)[]`, id)
+		args = append(args, pq.Array(txFilter.Addresses))
+		id++
+	}
+	if len(txFilter.Indexes) > 0 {
+		pgStr += fmt.Sprintf(` AND transaction_cids.index = ANY($%d::INTEGER[])`, id)
+		args = append(args, pq.Array(txFilter.Indexes))
+		id++
+	}
+	if len(txFilter.PkScriptClasses) > 0 {
+		pgStr += fmt.Sprintf(` AND tx_outputs.script_class = ANY($%d::INTEGER[])`, id)
+		args = append(args, pq.Array(txFilter.PkScriptClasses))
+	}
 	return results, tx.Select(&results, pgStr, args...)
 }
 

pkg/super_node/btc/retriever_test.go

@@ -0,0 +1,17 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package btc

pkg/super_node/btc/subscription_config.go

@@ -17,6 +17,7 @@
 package btc
 
 import (
+	"errors"
 	"math/big"
 
 	"github.com/spf13/viper"
@@ -24,29 +25,6 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
 )
 
-/*
-// HeaderModel is the db model for btc.header_cids table
-// TxInput is the db model for btc.tx_inputs table
-type TxInput struct {
-	ID                    int64    `db:"id"`
-	TxID                  int64    `db:"tx_id"`
-	Index                 int64    `db:"index"`
-	TxWitness             [][]byte `db:"tx_witness"`
-	SignatureScript       []byte   `db:"sig_script"`
-	PreviousOutPointHash  string   `db:"outpoint_hash"`
-	PreviousOutPointIndex uint32   `db:"outpoint_index"`
-}
-
-// TxOutput is the db model for btc.tx_outputs table
-type TxOutput struct {
-	ID       int64  `db:"id"`
-	TxID     int64  `db:"tx_id"`
-	Index    int64  `db:"index"`
-	Value    int64  `db:"value"`
-	PkScript []byte `db:"pk_script"`
-}
-
-*/
 // SubscriptionSettings config is used by a subscriber to specify what bitcoin data to stream from the super node
 type SubscriptionSettings struct {
 	BackFill     bool
@@ -65,12 +43,12 @@ type HeaderFilter struct {
 // TxFilter contains filter settings for txs
 type TxFilter struct {
 	Off             bool
-	// Top level trx filters
-	Index         int64    // allow filtering by index so that we can filter for only coinbase transactions (index 0) if we want to
 	Segwit          bool     // allow filtering for segwit trxs
 	WitnessHashes   []string // allow filtering for specific witness hashes
-	// TODO: trx input filters
+	Indexes         []int64  // allow filtering for specific transaction indexes (e.g. 0 for coinbase transactions)
-	// TODO: trx output filters
+	PkScriptClasses []uint8  // allow filtering for txs that have at least one tx output with the specified pkscript class
+	MultiSig        bool     // allow filtering for txs that have at least one tx output that requires more than one signature
+	Addresses       []string // allow filtering for txs that have at least one tx output with at least one of the provided addresses
 }
 
 // Init is used to initialize a EthSubscription struct with env variables
@@ -83,17 +61,30 @@ func NewEthSubscriptionConfig() (*SubscriptionSettings, error) {
 	// 0 start means we start at the beginning and 0 end means we continue indefinitely
 	sc.Start = big.NewInt(viper.GetInt64("superNode.btcSubscription.startingBlock"))
 	sc.End = big.NewInt(viper.GetInt64("superNode.btcSubscription.endingBlock"))
-	// Below default to false, which means we get all headers and no uncles by default
+	// Below default to false, which means we get all headers by default
 	sc.HeaderFilter = HeaderFilter{
 		Off: viper.GetBool("superNode.btcSubscription.headerFilter.off"),
 	}
 	// Below defaults to false and two slices of length 0
 	// Which means we get all transactions by default
+	pksc := viper.Get("superNode.btcSubscription.txFilter.pkScriptClass")
+	pkScriptClasses, ok := pksc.([]uint8)
+	if !ok {
+		return nil, errors.New("superNode.btcSubscription.txFilter.pkScriptClass needs to be an array of uint8s")
+	}
+	is := viper.Get("superNode.btcSubscription.txFilter.indexes")
+	indexes, ok := is.([]int64)
+	if !ok {
+		return nil, errors.New("superNode.btcSubscription.txFilter.indexes needs to be an array of int64s")
+	}
 	sc.TxFilter = TxFilter{
 		Off:             viper.GetBool("superNode.btcSubscription.txFilter.off"),
-		Index:         viper.GetInt64("superNode.btcSubscription.txFilter.index"),
 		Segwit:          viper.GetBool("superNode.btcSubscription.txFilter.segwit"),
 		WitnessHashes:   viper.GetStringSlice("superNode.btcSubscription.txFilter.witnessHashes"),
+		PkScriptClasses: pkScriptClasses,
+		Indexes:         indexes,
+		MultiSig:        viper.GetBool("superNode.btcSubscription.txFilter.multiSig"),
+		Addresses:       viper.GetStringSlice("superNode.btcSubscription.txFilter.addresses"),
 	}
 	return sc, nil
 }

pkg/super_node/btc/test_helpers.go

@@ -0,0 +1,43 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package btc
+
+import (
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres"
+)
+
+// TearDownDB is used to tear down the super node dbs after tests
+func TearDownDB(db *postgres.DB) {
+	tx, err := db.Beginx()
+	Expect(err).NotTo(HaveOccurred())
+
+	_, err = tx.Exec(`DELETE FROM btc.header_cids`)
+	Expect(err).NotTo(HaveOccurred())
+	_, err = tx.Exec(`DELETE FROM btc.transaction_cids`)
+	Expect(err).NotTo(HaveOccurred())
+	_, err = tx.Exec(`DELETE FROM btc.tx_inputs`)
+	Expect(err).NotTo(HaveOccurred())
+	_, err = tx.Exec(`DELETE FROM btc.tx_outputs`)
+	Expect(err).NotTo(HaveOccurred())
+	_, err = tx.Exec(`DELETE FROM blocks`)
+	Expect(err).NotTo(HaveOccurred())
+
+	err = tx.Commit()
+	Expect(err).NotTo(HaveOccurred())
+}

pkg/super_node/eth/indexer_test.go

@@ -19,24 +19,23 @@ package eth_test
 import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
 
 	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
-	eth2 "github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
 	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth/mocks"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
 )
 
 var _ = Describe("Indexer", func() {
 	var (
 		db   *postgres.DB
 		err  error
-		repo *eth2.CIDIndexer
+		repo *eth.CIDIndexer
 	)
 	BeforeEach(func() {
-		db, err = eth.SetupDB()
+		db, err = shared.SetupDB()
 		Expect(err).ToNot(HaveOccurred())
-		repo = eth2.NewCIDIndexer(db)
+		repo = eth.NewCIDIndexer(db)
 	})
 	AfterEach(func() {
 		eth.TearDownDB(db)

pkg/super_node/eth/resolver.go

@@ -41,48 +41,24 @@ func (eir *IPLDResolver) Resolve(iplds shared.FetchedIPLDs) (shared.ServerRespon
 	}
 	return StreamResponse{
 		BlockNumber:     ipfsBlocks.BlockNumber,
-		HeadersRlp:      eir.ResolveHeaders(ipfsBlocks.Headers),
+		HeadersRlp:      eir.resolve(ipfsBlocks.Headers),
-		UnclesRlp:       eir.ResolveUncles(ipfsBlocks.Uncles),
+		UnclesRlp:       eir.resolve(ipfsBlocks.Uncles),
-		TransactionsRlp: eir.ResolveTransactions(ipfsBlocks.Transactions),
+		TransactionsRlp: eir.resolve(ipfsBlocks.Transactions),
-		ReceiptsRlp:     eir.ResolveReceipts(ipfsBlocks.Receipts),
+		ReceiptsRlp:     eir.resolve(ipfsBlocks.Receipts),
-		StateNodesRlp:   eir.ResolveState(ipfsBlocks.StateNodes),
+		StateNodesRlp:   eir.resolveState(ipfsBlocks.StateNodes),
-		StorageNodesRlp: eir.ResolveStorage(ipfsBlocks.StorageNodes),
+		StorageNodesRlp: eir.resolveStorage(ipfsBlocks.StorageNodes),
 	}, nil
 }
 
-func (eir *IPLDResolver) ResolveHeaders(iplds []blocks.Block) [][]byte {
+func (eir *IPLDResolver) resolve(iplds []blocks.Block) [][]byte {
-	headerRlps := make([][]byte, 0, len(iplds))
+	rlps := make([][]byte, 0, len(iplds))
 	for _, ipld := range iplds {
-		headerRlps = append(headerRlps, ipld.RawData())
+		rlps = append(rlps, ipld.RawData())
 	}
-	return headerRlps
+	return rlps
 }
 
-func (eir *IPLDResolver) ResolveUncles(iplds []blocks.Block) [][]byte {
+func (eir *IPLDResolver) resolveState(iplds map[common.Hash]blocks.Block) map[common.Hash][]byte {
-	uncleRlps := make([][]byte, 0, len(iplds))
-	for _, ipld := range iplds {
-		uncleRlps = append(uncleRlps, ipld.RawData())
-	}
-	return uncleRlps
-}
-
-func (eir *IPLDResolver) ResolveTransactions(iplds []blocks.Block) [][]byte {
-	trxs := make([][]byte, 0, len(iplds))
-	for _, ipld := range iplds {
-		trxs = append(trxs, ipld.RawData())
-	}
-	return trxs
-}
-
-func (eir *IPLDResolver) ResolveReceipts(iplds []blocks.Block) [][]byte {
-	rcts := make([][]byte, 0, len(iplds))
-	for _, ipld := range iplds {
-		rcts = append(rcts, ipld.RawData())
-	}
-	return rcts
-}
-
-func (eir *IPLDResolver) ResolveState(iplds map[common.Hash]blocks.Block) map[common.Hash][]byte {
 	stateNodes := make(map[common.Hash][]byte, len(iplds))
 	for key, ipld := range iplds {
 		stateNodes[key] = ipld.RawData()
@@ -90,7 +66,7 @@ func (eir *IPLDResolver) ResolveState(iplds map[common.Hash]blocks.Block) map[co
 	return stateNodes
 }
 
-func (eir *IPLDResolver) ResolveStorage(iplds map[common.Hash]map[common.Hash]blocks.Block) map[common.Hash]map[common.Hash][]byte {
+func (eir *IPLDResolver) resolveStorage(iplds map[common.Hash]map[common.Hash]blocks.Block) map[common.Hash]map[common.Hash][]byte {
 	storageNodes := make(map[common.Hash]map[common.Hash][]byte)
 	for stateKey, storageIPLDs := range iplds {
 		storageNodes[stateKey] = make(map[common.Hash][]byte)

pkg/super_node/eth/retriever.go

@@ -176,18 +176,21 @@ func (ecr *CIDRetriever) RetrieveTxCIDs(tx *sqlx.Tx, txFilter TxFilter, blockNum
 	log.Debug("retrieving transaction cids for block ", blockNumber)
 	args := make([]interface{}, 0, 3)
 	results := make([]TxModel, 0)
-	pgStr := `SELECT transaction_cids.id, transaction_cids.header_id,
+	id := 1
+	pgStr := fmt.Sprintf(`SELECT transaction_cids.id, transaction_cids.header_id,
  			transaction_cids.tx_hash, transaction_cids.cid,
  			transaction_cids.dst, transaction_cids.src, transaction_cids.index
  			FROM eth.transaction_cids INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id)
-			WHERE header_cids.block_number = $1`
+			WHERE header_cids.block_number = $%d`, id)
 	args = append(args, blockNumber)
+	id++
 	if len(txFilter.Dst) > 0 {
-		pgStr += ` AND transaction_cids.dst = ANY($2::VARCHAR(66)[])`
+		pgStr += fmt.Sprintf(` AND transaction_cids.dst = ANY($%d::VARCHAR(66)[])`, id)
 		args = append(args, pq.Array(txFilter.Dst))
+		id++
 	}
 	if len(txFilter.Src) > 0 {
-		pgStr += ` AND transaction_cids.src = ANY($3::VARCHAR(66)[])`
+		pgStr += fmt.Sprintf(` AND transaction_cids.src = ANY($%d::VARCHAR(66)[])`, id)
 		args = append(args, pq.Array(txFilter.Src))
 	}
 	return results, tx.Select(&results, pgStr, args...)
@@ -309,26 +312,26 @@ func (ecr *CIDRetriever) RetrieveStateCIDs(tx *sqlx.Tx, stateFilter StateFilter,
 func (ecr *CIDRetriever) RetrieveStorageCIDs(tx *sqlx.Tx, storageFilter StorageFilter, blockNumber int64) ([]StorageNodeWithStateKeyModel, error) {
 	log.Debug("retrieving storage cids for block ", blockNumber)
 	args := make([]interface{}, 0, 3)
-	pgStr := `SELECT storage_cids.id, storage_cids.state_id, storage_cids.storage_key,
+	id := 1
+	pgStr := fmt.Sprintf(`SELECT storage_cids.id, storage_cids.state_id, storage_cids.storage_key,
  			storage_cids.leaf, storage_cids.cid, state_cids.state_key FROM eth.storage_cids, eth.state_cids, eth.header_cids
 			WHERE storage_cids.state_id = state_cids.id 
 			AND state_cids.header_id = header_cids.id
-			AND header_cids.block_number = $1`
+			AND header_cids.block_number = $%d`, id)
 	args = append(args, blockNumber)
+	id++
 	addrLen := len(storageFilter.Addresses)
 	if addrLen > 0 {
 		keys := make([]string, addrLen)
 		for i, addr := range storageFilter.Addresses {
 			keys[i] = crypto.Keccak256Hash(common.HexToAddress(addr).Bytes()).String()
 		}
-		pgStr += ` AND state_cids.state_key = ANY($2::VARCHAR(66)[])`
+		pgStr += fmt.Sprintf(` AND state_cids.state_key = ANY($%d::VARCHAR(66)[])`, id)
 		args = append(args, pq.Array(keys))
-		if len(storageFilter.StorageKeys) > 0 {
+		id++
-			pgStr += ` AND storage_cids.storage_key = ANY($3::VARCHAR(66)[])`
-			args = append(args, pq.Array(storageFilter.StorageKeys))
 	}
-	} else if len(storageFilter.StorageKeys) > 0 {
+	if len(storageFilter.StorageKeys) > 0 {
-		pgStr += ` AND storage_cids.storage_key = ANY($2::VARCHAR(66)[])`
+		pgStr += fmt.Sprintf(` AND storage_cids.storage_key = ANY($%d::VARCHAR(66)[])`, id)
 		args = append(args, pq.Array(storageFilter.StorageKeys))
 	}
 	if !storageFilter.IntermediateNodes {

pkg/super_node/eth/retriever_test.go

@@ -212,7 +212,7 @@ var _ = Describe("Retriever", func() {
 	)
 	BeforeEach(func() {
 		var err error
-		db, err = eth.SetupDB()
+		db, err = shared.SetupDB()
 		Expect(err).ToNot(HaveOccurred())
 		repo = eth2.NewCIDIndexer(db)
 		retriever = eth2.NewCIDRetriever(db)

pkg/super_node/eth/test_helpers.go

@@ -19,20 +19,9 @@ package eth
 import (
 	. "github.com/onsi/gomega"
 
-	"github.com/vulcanize/vulcanizedb/pkg/config"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres"
 )
 
-// SetupDB is use to setup a db for super node tests
-func SetupDB() (*postgres.DB, error) {
-	return postgres.NewDB(config.Database{
-		Hostname: "localhost",
-		Name:     "vulcanize_testing",
-		Port:     5432,
-	}, core.Node{})
-}
-
 // TearDownDB is used to tear down the super node dbs after tests
 func TearDownDB(db *postgres.DB) {
 	tx, err := db.Beginx()

pkg/super_node/shared/functions.go

@@ -18,7 +18,6 @@ package shared
 
 import (
 	"bytes"
-	"reflect"
 )
 
 // ListContainsString used to check if a list of strings contains a particular string
@@ -50,8 +49,3 @@ func ListContainsGap(gapList []Gap, gap Gap) bool {
 	}
 	return false
 }
-
-// IsPointer returns true if the concrete type underneath the provided interface is a pointer
-func IsPointer(i interface{}) bool {
-	return reflect.ValueOf(i).Type().Kind() == reflect.Ptr
-}

pkg/super_node/shared/test_helpers.go

@@ -0,0 +1,32 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package shared
+
+import (
+	"github.com/vulcanize/vulcanizedb/pkg/config"
+	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
+	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres"
+)
+
+// SetupDB is use to setup a db for super node tests
+func SetupDB() (*postgres.DB, error) {
+	return postgres.NewDB(config.Database{
+		Hostname: "localhost",
+		Name:     "vulcanize_testing",
+		Port:     5432,
+	}, core.Node{})
+}