TxOutputs: extract and index pkscript metadata (script type, addresses, #required sigs); TxInputs: outpoint_hash => outpoint_tx_id that references transaction_cids.id

db/migrations/00042_create_btc_transaction_cids_table.sql

@@ -3,11 +3,10 @@ CREATE TABLE btc.transaction_cids (
   id           SERIAL PRIMARY KEY,
 	header_id    INTEGER NOT NULL REFERENCES btc.header_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
 	index        INTEGER NOT NULL,
-	tx_hash      VARCHAR(66) NOT NULL,
+	tx_hash      VARCHAR(66) NOT NULL UNIQUE,
 	cid          TEXT NOT NULL,
-	has_witness  BOOL NOT NULL,
+	segwit       BOOL NOT NULL,
-	witness_hash VARCHAR(66),
+	witness_hash VARCHAR(66)
-	UNIQUE (header_id, tx_hash)
 );
 
 -- +goose Down

db/migrations/00043_create_btc_tx_inputs_table.sql

@@ -3,9 +3,9 @@ CREATE TABLE btc.tx_inputs (
   id             SERIAL PRIMARY KEY,
 	tx_id          INTEGER NOT NULL REFERENCES btc.transaction_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
 	index          INTEGER NOT NULL,
-	tx_witness     BYTEA[],
+	witness        BYTEA[],
 	sig_script     BYTEA NOT NULL,
-	outpoint_hash  VARCHAR(66) NOT NULL,
+	outpoint_tx_id INTEGER NOT NULL REFERENCES btc.transaction_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
 	outpoint_index INTEGER NOT NULL,
 	UNIQUE (tx_id, index)
 );

db/migrations/00044_create_btc_tx_outputs_table.sql

@@ -1,10 +1,13 @@
 -- +goose Up
 CREATE TABLE btc.tx_outputs (
-  id           SERIAL PRIMARY KEY,
+  id            SERIAL PRIMARY KEY,
-	tx_id        INTEGER NOT NULL REFERENCES btc.transaction_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
+	tx_id         INTEGER NOT NULL REFERENCES btc.transaction_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-	index        INTEGER NOT NULL,
+	index         INTEGER NOT NULL,
-	value        INTEGER NOT NULL,
+	value         INTEGER NOT NULL,
-	pk_script    BYTEA NOT NULL,
+	pk_script     BYTEA NOT NULL,
+	script_class  INTEGER NOT NULL,
+	addresses     VARCHAR(66)[],
+	required_sigs INTEGER NOT NULL,
 	UNIQUE (tx_id, index)
 );
 

db/schema.sql

@@ -78,7 +78,7 @@ CREATE TABLE btc.transaction_cids (
     index integer NOT NULL,
     tx_hash character varying(66) NOT NULL,
     cid text NOT NULL,
-    has_witness boolean NOT NULL,
+    segwit boolean NOT NULL,
     witness_hash character varying(66)
 );
 
@@ -111,10 +111,9 @@ CREATE TABLE btc.tx_inputs (
     id integer NOT NULL,
     tx_id integer NOT NULL,
     index integer NOT NULL,
-    tx_witness bytea[],
+    witness bytea[],
-    sequence integer NOT NULL,
+    sig_script bytea NOT NULL,
-    script bytea NOT NULL,
+    outpoint_tx_id integer NOT NULL,
-    outpoint_hash character varying(66) NOT NULL,
     outpoint_index integer NOT NULL
 );
 
@@ -148,7 +147,10 @@ CREATE TABLE btc.tx_outputs (
     tx_id integer NOT NULL,
     index integer NOT NULL,
     value integer NOT NULL,
-    script bytea NOT NULL
+    pk_script bytea NOT NULL,
+    script_class integer NOT NULL,
+    addresses character varying(66)[],
+    required_sigs integer NOT NULL
 );
 
 
@@ -1275,14 +1277,6 @@ ALTER TABLE ONLY btc.header_cids
     ADD CONSTRAINT header_cids_pkey PRIMARY KEY (id);
 
 
---
--- Name: transaction_cids transaction_cids_header_id_tx_hash_key; Type: CONSTRAINT; Schema: btc; Owner: -
---
-
-ALTER TABLE ONLY btc.transaction_cids
-    ADD CONSTRAINT transaction_cids_header_id_tx_hash_key UNIQUE (header_id, tx_hash);
-
-
 --
 -- Name: transaction_cids transaction_cids_pkey; Type: CONSTRAINT; Schema: btc; Owner: -
 --
@@ -1291,6 +1285,14 @@ ALTER TABLE ONLY btc.transaction_cids
     ADD CONSTRAINT transaction_cids_pkey PRIMARY KEY (id);
 
 
+--
+-- Name: transaction_cids transaction_cids_tx_hash_key; Type: CONSTRAINT; Schema: btc; Owner: -
+--
+
+ALTER TABLE ONLY btc.transaction_cids
+    ADD CONSTRAINT transaction_cids_tx_hash_key UNIQUE (tx_hash);
+
+
 --
 -- Name: tx_inputs tx_inputs_pkey; Type: CONSTRAINT; Schema: btc; Owner: -
 --
@@ -1744,6 +1746,14 @@ ALTER TABLE ONLY btc.transaction_cids
     ADD CONSTRAINT transaction_cids_header_id_fkey FOREIGN KEY (header_id) REFERENCES btc.header_cids(id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
 
 
+--
+-- Name: tx_inputs tx_inputs_outpoint_tx_id_fkey; Type: FK CONSTRAINT; Schema: btc; Owner: -
+--
+
+ALTER TABLE ONLY btc.tx_inputs
+    ADD CONSTRAINT tx_inputs_outpoint_tx_id_fkey FOREIGN KEY (outpoint_tx_id) REFERENCES btc.transaction_cids(id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
+
+
 --
 -- Name: tx_inputs tx_inputs_tx_id_fkey; Type: FK CONSTRAINT; Schema: btc; Owner: -
 --

pkg/super_node/btc/converter.go

@@ -19,14 +19,19 @@ package btc
 import (
 	"fmt"
 
+	"github.com/btcsuite/btcd/chaincfg"
+	"github.com/btcsuite/btcd/txscript"
+
 	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
 )
 
 // PayloadConverter satisfies the PayloadConverter interface for bitcoin
-type PayloadConverter struct{}
+type PayloadConverter struct{
+	chainConfig *chaincfg.Params
+}
 
 // NewPayloadConverter creates a pointer to a new PayloadConverter which satisfies the PayloadConverter interface
-func NewPayloadConverter() *PayloadConverter {
+func NewPayloadConverter(chainConfig *chaincfg.Params) *PayloadConverter {
 	return &PayloadConverter{}
 }
 
@@ -41,11 +46,11 @@ func (pc *PayloadConverter) Convert(payload shared.RawChainData) (shared.Streame
 	for _, tx := range btcBlockPayload.Txs {
 		index := tx.Index()
 		txModel := TxModelWithInsAndOuts{
-			TxHash:     tx.Hash().String(),
+			TxHash:    tx.Hash().String(),
-			Index:      int64(tx.Index()),
+			Index:     int64(index),
-			HasWitness: tx.HasWitness(),
+			SegWit:    tx.HasWitness(),
-			TxOutputs:  make([]TxOutput, len(tx.MsgTx().TxOut)),
+			TxOutputs: make([]TxOutput, len(tx.MsgTx().TxOut)),
-			TxInputs:   make([]TxInput, len(tx.MsgTx().TxIn)),
+			TxInputs:  make([]TxInput, len(tx.MsgTx().TxIn)),
 		}
 		if tx.HasWitness() {
 			txModel.WitnessHash = tx.WitnessHash().String()
@@ -60,10 +65,22 @@ func (pc *PayloadConverter) Convert(payload shared.RawChainData) (shared.Streame
 			}
 		}
 		for i, out := range tx.MsgTx().TxOut {
+			scriptClass, addresses, numberOfSigs, err := txscript.ExtractPkScriptAddrs(out.PkScript, pc.chainConfig)
+			// if we receive an error but the txscript type isn't NonStandardTy then something went wrong
+			if err != nil && scriptClass != txscript.NonStandardTy {
+				return nil, err
+			}
+			stringAddrs := make([]string, len(addresses))
+			for i, addr := range addresses {
+				stringAddrs[i] = addr.EncodeAddress()
+			}
 			txModel.TxOutputs[i] = TxOutput{
-				Index:    int64(i),
+				Index:        int64(i),
-				Value:    out.Value,
+				Value:        out.Value,
-				PkScript: out.PkScript,
+				PkScript:     out.PkScript,
+				RequiredSigs: int64(numberOfSigs),
+				ScriptClass:  (uint8)(scriptClass),
+				Addresses:    stringAddrs,
 			}
 		}
 		txMeta[index] = txModel

pkg/super_node/btc/indexer.go

@@ -87,26 +87,34 @@ func (in *CIDIndexer) indexTransactionCIDs(tx *sqlx.Tx, transactions []TxModelWi
 
 func (in *CIDIndexer) indexTransactionCID(tx *sqlx.Tx, transaction TxModelWithInsAndOuts, headerID int64) (int64, error) {
 	var txID int64
-	err := tx.QueryRowx(`INSERT INTO btc.transaction_cids (header_id, tx_hash, index, cid, has_witness, witness_hash)
+	err := tx.QueryRowx(`INSERT INTO btc.transaction_cids (header_id, tx_hash, index, cid, segwit, witness_hash)
 							VALUES ($1, $2, $3, $4, $5, $6)
-							ON CONFLICT (header_id, tx_hash) DO UPDATE SET (index, cid, has_witness, witness_hash) = ($3, $4, $5, $6)
+							ON CONFLICT (tx_hash) DO UPDATE SET (header_id, index, cid, segwit, witness_hash) = ($1, $3, $4, $5, $6)
 							RETURNING id`,
-		headerID, transaction.TxHash, transaction.Index, transaction.CID, transaction.HasWitness, transaction.WitnessHash).Scan(&txID)
+		headerID, transaction.TxHash, transaction.Index, transaction.CID, transaction.SegWit, transaction.WitnessHash).Scan(&txID)
 	return txID, err
 }
 
 func (in *CIDIndexer) indexTxInput(tx *sqlx.Tx, txInput TxInput, txID int64) error {
-	_, err := tx.Exec(`INSERT INTO btc.tx_inputs (tx_id, index, tx_witness, sig_script, outpoint_hash, outpoint_index)
+	var referencedOutPutTxID int64
+	if err := tx.Get(&referencedOutPutTxID, `SELECT id FROM btc.transaction_cids
+						WHERE tx_hash = $1`, txInput.PreviousOutPointHash); err != nil {
+		return err
+	}
+	if referencedOutPutTxID == 0 {
+		return fmt.Errorf("btc indexer could not find the tx hash %s referenced in tx input of tx id %d", txInput.PreviousOutPointHash, txID)
+	}
+	_, err := tx.Exec(`INSERT INTO btc.tx_inputs (tx_id, index, witness, sig_script, outpoint_tx_id, outpoint_index)
 						VALUES ($1, $2, $3, $4, $5, $6)
-						ON CONFLICT (tx_id, index) DO UPDATE SET (tx_witness, sig_script, outpoint_hash, outpoint_index) = ($3, $4, $5, $6)`,
+						ON CONFLICT (tx_id, index) DO UPDATE SET (witness, sig_script, outpoint_tx_id, outpoint_index) = ($3, $4, $5, $6)`,
-		txID, txInput.Index, pq.Array(txInput.TxWitness), txInput.SignatureScript, txInput.PreviousOutPointHash, txInput.PreviousOutPointIndex)
+		txID, txInput.Index, pq.Array(txInput.TxWitness), txInput.SignatureScript, referencedOutPutTxID, txInput.PreviousOutPointIndex)
 	return err
 }
 
 func (in *CIDIndexer) indexTxOutput(tx *sqlx.Tx, txOuput TxOutput, txID int64) error {
-	_, err := tx.Exec(`INSERT INTO btc.tx_outputs (tx_id, index, value, pk_script)
+	_, err := tx.Exec(`INSERT INTO btc.tx_outputs (tx_id, index, value, pk_script, script_class, addresses, required_sigs)
-							VALUES ($1, $2, $3, $4)
+							VALUES ($1, $2, $3, $4, $5, $6, $7)
-							ON CONFLICT (ix_id, index) DO UPDATE SET (value, pk_script) = ($3, $4)`,
+							ON CONFLICT (ix_id, index) DO UPDATE SET (value, pk_script, script_class, addresses, required_sigs) = ($3, $4, $5, $6, $7)`,
-		txID, txOuput.Index, txOuput.Value, txOuput.PkScript)
+		txID, txOuput.Index, txOuput.Value, txOuput.PkScript, txOuput.ScriptClass, txOuput.Addresses, txOuput.RequiredSigs)
 	return err
 }

pkg/super_node/btc/models.go

@@ -16,6 +16,8 @@
 
 package btc
 
+import "github.com/lib/pq"
+
 // HeaderModel is the db model for btc.header_cids table
 type HeaderModel struct {
 	ID          int64  `db:"id"`
@@ -35,7 +37,7 @@ type TxModel struct {
 	Index       int64  `db:"index"`
 	TxHash      string `db:"tx_hash"`
 	CID         string `db:"cid"`
-	HasWitness  bool   `db:"has_witness"`
+	SegWit      bool   `db:"segwit"`
 	WitnessHash string `db:"witness_hash"`
 }
 
@@ -46,7 +48,7 @@ type TxModelWithInsAndOuts struct {
 	Index       int64  `db:"index"`
 	TxHash      string `db:"tx_hash"`
 	CID         string `db:"cid"`
-	HasWitness  bool   `db:"has_witness"`
+	SegWit      bool   `db:"segwit"`
 	WitnessHash string `db:"witness_hash"`
 	TxInputs    []TxInput
 	TxOutputs   []TxOutput
@@ -57,17 +59,21 @@ type TxInput struct {
 	ID                    int64    `db:"id"`
 	TxID                  int64    `db:"tx_id"`
 	Index                 int64    `db:"index"`
-	TxWitness             [][]byte `db:"tx_witness"`
+	TxWitness             [][]byte `db:"witness"`
 	SignatureScript       []byte   `db:"sig_script"`
-	PreviousOutPointHash  string   `db:"outpoint_hash"`
+	PreviousOutPointTxID  int64    `db:"outpoint_tx_id"`
 	PreviousOutPointIndex uint32   `db:"outpoint_index"`
+	PreviousOutPointHash  string
 }
 
 // TxOutput is the db model for btc.tx_outputs table
 type TxOutput struct {
-	ID       int64  `db:"id"`
+	ID           int64          `db:"id"`
-	TxID     int64  `db:"tx_id"`
+	TxID         int64          `db:"tx_id"`
-	Index    int64  `db:"index"`
+	Index        int64          `db:"index"`
-	Value    int64  `db:"value"`
+	Value        int64          `db:"value"`
-	PkScript []byte `db:"pk_script"`
+	PkScript     []byte         `db:"pk_script"`
+	ScriptClass  uint8          `db:"script_class"`
+	RequiredSigs int64          `db:"required_sigs"`
+	Addresses    pq.StringArray `db:"addresses"`
 }

pkg/super_node/btc/publisher.go

@@ -101,7 +101,7 @@ func (pub *IPLDPublisher) publishTransactions(transactions []*btcutil.Tx, trxMet
 			CID:         cid,
 			Index:       trxMeta[i].Index,
 			TxHash:      trxMeta[i].TxHash,
-			HasWitness:  trxMeta[i].HasWitness,
+			SegWit:      trxMeta[i].SegWit,
 			WitnessHash: trxMeta[i].WitnessHash,
 			TxInputs:    trxMeta[i].TxInputs,
 			TxOutputs:   trxMeta[i].TxOutputs,

pkg/super_node/btc/types.go

@@ -77,8 +77,8 @@ type IPLDWrapper struct {
 // Passed to client subscriptions
 type StreamPayload struct {
 	BlockNumber       *big.Int `json:"blockNumber"`
-	HeadersBytes      [][]byte `json:"headerBytes"`
+	SerializedHeaders [][]byte `json:"headerBytes"`
-	TransactionsBytes [][]byte `json:"transactionBytes"`
+	SerializedTxs     [][]byte `json:"transactionBytes"`
 
 	encoded []byte
 	err     error

pkg/super_node/constructors.go

@@ -18,6 +18,7 @@ package super_node
 
 import (
 	"fmt"
+	"github.com/btcsuite/btcd/chaincfg"
 
 	"github.com/btcsuite/btcd/rpcclient"
 	"github.com/ethereum/go-ethereum/params"
@@ -106,7 +107,7 @@ func NewPayloadConverter(chain shared.ChainType) (shared.PayloadConverter, error
 	case shared.Ethereum:
 		return eth.NewPayloadConverter(params.MainnetChainConfig), nil
 	case shared.Bitcoin:
-		return btc.NewPayloadConverter(), nil
+		return btc.NewPayloadConverter(&chaincfg.MainNetParams), nil
 	default:
 		return nil, fmt.Errorf("invalid chain %T for converter constructor", chain)
 	}