From f33cc3f34b2e646dc557d2d3d4349fc65df38c17 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Tue, 4 Feb 2020 15:20:49 -0600 Subject: [PATCH] TxOutputs: extract and index pkscript metadata (script type, addresses, #required sigs); TxInputs: outpoint_hash => outpoint_tx_id that references transaction_cids.id --- ...0042_create_btc_transaction_cids_table.sql | 7 ++-- .../00043_create_btc_tx_inputs_table.sql | 4 +- .../00044_create_btc_tx_outputs_table.sql | 13 ++++--- db/schema.sql | 38 ++++++++++++------- pkg/super_node/btc/converter.go | 37 +++++++++++++----- pkg/super_node/btc/indexer.go | 28 +++++++++----- pkg/super_node/btc/models.go | 24 +++++++----- pkg/super_node/btc/publisher.go | 2 +- pkg/super_node/btc/types.go | 4 +- pkg/super_node/constructors.go | 3 +- 10 files changed, 102 insertions(+), 58 deletions(-) diff --git a/db/migrations/00042_create_btc_transaction_cids_table.sql b/db/migrations/00042_create_btc_transaction_cids_table.sql index 95411a2b..2648c6ef 100644 --- a/db/migrations/00042_create_btc_transaction_cids_table.sql +++ b/db/migrations/00042_create_btc_transaction_cids_table.sql @@ -3,11 +3,10 @@ CREATE TABLE btc.transaction_cids ( id SERIAL PRIMARY KEY, header_id INTEGER NOT NULL REFERENCES btc.header_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, index INTEGER NOT NULL, - tx_hash VARCHAR(66) NOT NULL, + tx_hash VARCHAR(66) NOT NULL UNIQUE, cid TEXT NOT NULL, - has_witness BOOL NOT NULL, - witness_hash VARCHAR(66), - UNIQUE (header_id, tx_hash) + segwit BOOL NOT NULL, + witness_hash VARCHAR(66) ); -- +goose Down diff --git a/db/migrations/00043_create_btc_tx_inputs_table.sql b/db/migrations/00043_create_btc_tx_inputs_table.sql index 5a70f4c3..61619af8 100644 --- a/db/migrations/00043_create_btc_tx_inputs_table.sql +++ b/db/migrations/00043_create_btc_tx_inputs_table.sql @@ -3,9 +3,9 @@ CREATE TABLE btc.tx_inputs ( id SERIAL PRIMARY KEY, tx_id INTEGER NOT NULL REFERENCES btc.transaction_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, index INTEGER NOT NULL, - tx_witness BYTEA[], + witness BYTEA[], sig_script BYTEA NOT NULL, - outpoint_hash VARCHAR(66) NOT NULL, + outpoint_tx_id INTEGER NOT NULL REFERENCES btc.transaction_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, outpoint_index INTEGER NOT NULL, UNIQUE (tx_id, index) ); diff --git a/db/migrations/00044_create_btc_tx_outputs_table.sql b/db/migrations/00044_create_btc_tx_outputs_table.sql index 836455a3..ae95b72d 100644 --- a/db/migrations/00044_create_btc_tx_outputs_table.sql +++ b/db/migrations/00044_create_btc_tx_outputs_table.sql @@ -1,10 +1,13 @@ -- +goose Up CREATE TABLE btc.tx_outputs ( - id SERIAL PRIMARY KEY, - tx_id INTEGER NOT NULL REFERENCES btc.transaction_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, - index INTEGER NOT NULL, - value INTEGER NOT NULL, - pk_script BYTEA NOT NULL, + id SERIAL PRIMARY KEY, + tx_id INTEGER NOT NULL REFERENCES btc.transaction_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, + index INTEGER NOT NULL, + value INTEGER NOT NULL, + pk_script BYTEA NOT NULL, + script_class INTEGER NOT NULL, + addresses VARCHAR(66)[], + required_sigs INTEGER NOT NULL, UNIQUE (tx_id, index) ); diff --git a/db/schema.sql b/db/schema.sql index bd167767..932d51a5 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -78,7 +78,7 @@ CREATE TABLE btc.transaction_cids ( index integer NOT NULL, tx_hash character varying(66) NOT NULL, cid text NOT NULL, - has_witness boolean NOT NULL, + segwit boolean NOT NULL, witness_hash character varying(66) ); @@ -111,10 +111,9 @@ CREATE TABLE btc.tx_inputs ( id integer NOT NULL, tx_id integer NOT NULL, index integer NOT NULL, - tx_witness bytea[], - sequence integer NOT NULL, - script bytea NOT NULL, - outpoint_hash character varying(66) NOT NULL, + witness bytea[], + sig_script bytea NOT NULL, + outpoint_tx_id integer NOT NULL, outpoint_index integer NOT NULL ); @@ -148,7 +147,10 @@ CREATE TABLE btc.tx_outputs ( tx_id integer NOT NULL, index integer NOT NULL, value integer NOT NULL, - script bytea NOT NULL + pk_script bytea NOT NULL, + script_class integer NOT NULL, + addresses character varying(66)[], + required_sigs integer NOT NULL ); @@ -1275,14 +1277,6 @@ ALTER TABLE ONLY btc.header_cids ADD CONSTRAINT header_cids_pkey PRIMARY KEY (id); --- --- Name: transaction_cids transaction_cids_header_id_tx_hash_key; Type: CONSTRAINT; Schema: btc; Owner: - --- - -ALTER TABLE ONLY btc.transaction_cids - ADD CONSTRAINT transaction_cids_header_id_tx_hash_key UNIQUE (header_id, tx_hash); - - -- -- Name: transaction_cids transaction_cids_pkey; Type: CONSTRAINT; Schema: btc; Owner: - -- @@ -1291,6 +1285,14 @@ ALTER TABLE ONLY btc.transaction_cids ADD CONSTRAINT transaction_cids_pkey PRIMARY KEY (id); +-- +-- Name: transaction_cids transaction_cids_tx_hash_key; Type: CONSTRAINT; Schema: btc; Owner: - +-- + +ALTER TABLE ONLY btc.transaction_cids + ADD CONSTRAINT transaction_cids_tx_hash_key UNIQUE (tx_hash); + + -- -- Name: tx_inputs tx_inputs_pkey; Type: CONSTRAINT; Schema: btc; Owner: - -- @@ -1744,6 +1746,14 @@ ALTER TABLE ONLY btc.transaction_cids ADD CONSTRAINT transaction_cids_header_id_fkey FOREIGN KEY (header_id) REFERENCES btc.header_cids(id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; +-- +-- Name: tx_inputs tx_inputs_outpoint_tx_id_fkey; Type: FK CONSTRAINT; Schema: btc; Owner: - +-- + +ALTER TABLE ONLY btc.tx_inputs + ADD CONSTRAINT tx_inputs_outpoint_tx_id_fkey FOREIGN KEY (outpoint_tx_id) REFERENCES btc.transaction_cids(id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; + + -- -- Name: tx_inputs tx_inputs_tx_id_fkey; Type: FK CONSTRAINT; Schema: btc; Owner: - -- diff --git a/pkg/super_node/btc/converter.go b/pkg/super_node/btc/converter.go index 600d9bb0..61fded86 100644 --- a/pkg/super_node/btc/converter.go +++ b/pkg/super_node/btc/converter.go @@ -19,14 +19,19 @@ package btc import ( "fmt" + "github.com/btcsuite/btcd/chaincfg" + "github.com/btcsuite/btcd/txscript" + "github.com/vulcanize/vulcanizedb/pkg/super_node/shared" ) // PayloadConverter satisfies the PayloadConverter interface for bitcoin -type PayloadConverter struct{} +type PayloadConverter struct{ + chainConfig *chaincfg.Params +} // NewPayloadConverter creates a pointer to a new PayloadConverter which satisfies the PayloadConverter interface -func NewPayloadConverter() *PayloadConverter { +func NewPayloadConverter(chainConfig *chaincfg.Params) *PayloadConverter { return &PayloadConverter{} } @@ -41,11 +46,11 @@ func (pc *PayloadConverter) Convert(payload shared.RawChainData) (shared.Streame for _, tx := range btcBlockPayload.Txs { index := tx.Index() txModel := TxModelWithInsAndOuts{ - TxHash: tx.Hash().String(), - Index: int64(tx.Index()), - HasWitness: tx.HasWitness(), - TxOutputs: make([]TxOutput, len(tx.MsgTx().TxOut)), - TxInputs: make([]TxInput, len(tx.MsgTx().TxIn)), + TxHash: tx.Hash().String(), + Index: int64(index), + SegWit: tx.HasWitness(), + TxOutputs: make([]TxOutput, len(tx.MsgTx().TxOut)), + TxInputs: make([]TxInput, len(tx.MsgTx().TxIn)), } if tx.HasWitness() { txModel.WitnessHash = tx.WitnessHash().String() @@ -60,10 +65,22 @@ func (pc *PayloadConverter) Convert(payload shared.RawChainData) (shared.Streame } } for i, out := range tx.MsgTx().TxOut { + scriptClass, addresses, numberOfSigs, err := txscript.ExtractPkScriptAddrs(out.PkScript, pc.chainConfig) + // if we receive an error but the txscript type isn't NonStandardTy then something went wrong + if err != nil && scriptClass != txscript.NonStandardTy { + return nil, err + } + stringAddrs := make([]string, len(addresses)) + for i, addr := range addresses { + stringAddrs[i] = addr.EncodeAddress() + } txModel.TxOutputs[i] = TxOutput{ - Index: int64(i), - Value: out.Value, - PkScript: out.PkScript, + Index: int64(i), + Value: out.Value, + PkScript: out.PkScript, + RequiredSigs: int64(numberOfSigs), + ScriptClass: (uint8)(scriptClass), + Addresses: stringAddrs, } } txMeta[index] = txModel diff --git a/pkg/super_node/btc/indexer.go b/pkg/super_node/btc/indexer.go index 5e705a8b..1f7554f9 100644 --- a/pkg/super_node/btc/indexer.go +++ b/pkg/super_node/btc/indexer.go @@ -87,26 +87,34 @@ func (in *CIDIndexer) indexTransactionCIDs(tx *sqlx.Tx, transactions []TxModelWi func (in *CIDIndexer) indexTransactionCID(tx *sqlx.Tx, transaction TxModelWithInsAndOuts, headerID int64) (int64, error) { var txID int64 - err := tx.QueryRowx(`INSERT INTO btc.transaction_cids (header_id, tx_hash, index, cid, has_witness, witness_hash) + err := tx.QueryRowx(`INSERT INTO btc.transaction_cids (header_id, tx_hash, index, cid, segwit, witness_hash) VALUES ($1, $2, $3, $4, $5, $6) - ON CONFLICT (header_id, tx_hash) DO UPDATE SET (index, cid, has_witness, witness_hash) = ($3, $4, $5, $6) + ON CONFLICT (tx_hash) DO UPDATE SET (header_id, index, cid, segwit, witness_hash) = ($1, $3, $4, $5, $6) RETURNING id`, - headerID, transaction.TxHash, transaction.Index, transaction.CID, transaction.HasWitness, transaction.WitnessHash).Scan(&txID) + headerID, transaction.TxHash, transaction.Index, transaction.CID, transaction.SegWit, transaction.WitnessHash).Scan(&txID) return txID, err } func (in *CIDIndexer) indexTxInput(tx *sqlx.Tx, txInput TxInput, txID int64) error { - _, err := tx.Exec(`INSERT INTO btc.tx_inputs (tx_id, index, tx_witness, sig_script, outpoint_hash, outpoint_index) + var referencedOutPutTxID int64 + if err := tx.Get(&referencedOutPutTxID, `SELECT id FROM btc.transaction_cids + WHERE tx_hash = $1`, txInput.PreviousOutPointHash); err != nil { + return err + } + if referencedOutPutTxID == 0 { + return fmt.Errorf("btc indexer could not find the tx hash %s referenced in tx input of tx id %d", txInput.PreviousOutPointHash, txID) + } + _, err := tx.Exec(`INSERT INTO btc.tx_inputs (tx_id, index, witness, sig_script, outpoint_tx_id, outpoint_index) VALUES ($1, $2, $3, $4, $5, $6) - ON CONFLICT (tx_id, index) DO UPDATE SET (tx_witness, sig_script, outpoint_hash, outpoint_index) = ($3, $4, $5, $6)`, - txID, txInput.Index, pq.Array(txInput.TxWitness), txInput.SignatureScript, txInput.PreviousOutPointHash, txInput.PreviousOutPointIndex) + ON CONFLICT (tx_id, index) DO UPDATE SET (witness, sig_script, outpoint_tx_id, outpoint_index) = ($3, $4, $5, $6)`, + txID, txInput.Index, pq.Array(txInput.TxWitness), txInput.SignatureScript, referencedOutPutTxID, txInput.PreviousOutPointIndex) return err } func (in *CIDIndexer) indexTxOutput(tx *sqlx.Tx, txOuput TxOutput, txID int64) error { - _, err := tx.Exec(`INSERT INTO btc.tx_outputs (tx_id, index, value, pk_script) - VALUES ($1, $2, $3, $4) - ON CONFLICT (ix_id, index) DO UPDATE SET (value, pk_script) = ($3, $4)`, - txID, txOuput.Index, txOuput.Value, txOuput.PkScript) + _, err := tx.Exec(`INSERT INTO btc.tx_outputs (tx_id, index, value, pk_script, script_class, addresses, required_sigs) + VALUES ($1, $2, $3, $4, $5, $6, $7) + ON CONFLICT (ix_id, index) DO UPDATE SET (value, pk_script, script_class, addresses, required_sigs) = ($3, $4, $5, $6, $7)`, + txID, txOuput.Index, txOuput.Value, txOuput.PkScript, txOuput.ScriptClass, txOuput.Addresses, txOuput.RequiredSigs) return err } diff --git a/pkg/super_node/btc/models.go b/pkg/super_node/btc/models.go index 13cbad1f..580dfc9e 100644 --- a/pkg/super_node/btc/models.go +++ b/pkg/super_node/btc/models.go @@ -16,6 +16,8 @@ package btc +import "github.com/lib/pq" + // HeaderModel is the db model for btc.header_cids table type HeaderModel struct { ID int64 `db:"id"` @@ -35,7 +37,7 @@ type TxModel struct { Index int64 `db:"index"` TxHash string `db:"tx_hash"` CID string `db:"cid"` - HasWitness bool `db:"has_witness"` + SegWit bool `db:"segwit"` WitnessHash string `db:"witness_hash"` } @@ -46,7 +48,7 @@ type TxModelWithInsAndOuts struct { Index int64 `db:"index"` TxHash string `db:"tx_hash"` CID string `db:"cid"` - HasWitness bool `db:"has_witness"` + SegWit bool `db:"segwit"` WitnessHash string `db:"witness_hash"` TxInputs []TxInput TxOutputs []TxOutput @@ -57,17 +59,21 @@ type TxInput struct { ID int64 `db:"id"` TxID int64 `db:"tx_id"` Index int64 `db:"index"` - TxWitness [][]byte `db:"tx_witness"` + TxWitness [][]byte `db:"witness"` SignatureScript []byte `db:"sig_script"` - PreviousOutPointHash string `db:"outpoint_hash"` + PreviousOutPointTxID int64 `db:"outpoint_tx_id"` PreviousOutPointIndex uint32 `db:"outpoint_index"` + PreviousOutPointHash string } // TxOutput is the db model for btc.tx_outputs table type TxOutput struct { - ID int64 `db:"id"` - TxID int64 `db:"tx_id"` - Index int64 `db:"index"` - Value int64 `db:"value"` - PkScript []byte `db:"pk_script"` + ID int64 `db:"id"` + TxID int64 `db:"tx_id"` + Index int64 `db:"index"` + Value int64 `db:"value"` + PkScript []byte `db:"pk_script"` + ScriptClass uint8 `db:"script_class"` + RequiredSigs int64 `db:"required_sigs"` + Addresses pq.StringArray `db:"addresses"` } diff --git a/pkg/super_node/btc/publisher.go b/pkg/super_node/btc/publisher.go index fa5b6185..3b391bee 100644 --- a/pkg/super_node/btc/publisher.go +++ b/pkg/super_node/btc/publisher.go @@ -101,7 +101,7 @@ func (pub *IPLDPublisher) publishTransactions(transactions []*btcutil.Tx, trxMet CID: cid, Index: trxMeta[i].Index, TxHash: trxMeta[i].TxHash, - HasWitness: trxMeta[i].HasWitness, + SegWit: trxMeta[i].SegWit, WitnessHash: trxMeta[i].WitnessHash, TxInputs: trxMeta[i].TxInputs, TxOutputs: trxMeta[i].TxOutputs, diff --git a/pkg/super_node/btc/types.go b/pkg/super_node/btc/types.go index bcd96059..e1d6c629 100644 --- a/pkg/super_node/btc/types.go +++ b/pkg/super_node/btc/types.go @@ -77,8 +77,8 @@ type IPLDWrapper struct { // Passed to client subscriptions type StreamPayload struct { BlockNumber *big.Int `json:"blockNumber"` - HeadersBytes [][]byte `json:"headerBytes"` - TransactionsBytes [][]byte `json:"transactionBytes"` + SerializedHeaders [][]byte `json:"headerBytes"` + SerializedTxs [][]byte `json:"transactionBytes"` encoded []byte err error diff --git a/pkg/super_node/constructors.go b/pkg/super_node/constructors.go index 210966b1..b73cb34b 100644 --- a/pkg/super_node/constructors.go +++ b/pkg/super_node/constructors.go @@ -18,6 +18,7 @@ package super_node import ( "fmt" + "github.com/btcsuite/btcd/chaincfg" "github.com/btcsuite/btcd/rpcclient" "github.com/ethereum/go-ethereum/params" @@ -106,7 +107,7 @@ func NewPayloadConverter(chain shared.ChainType) (shared.PayloadConverter, error case shared.Ethereum: return eth.NewPayloadConverter(params.MainnetChainConfig), nil case shared.Bitcoin: - return btc.NewPayloadConverter(), nil + return btc.NewPayloadConverter(&chaincfg.MainNetParams), nil default: return nil, fmt.Errorf("invalid chain %T for converter constructor", chain) }