From 2c50615b81bea01d276abc04070efad2af10b55f Mon Sep 17 00:00:00 2001 From: i-norden Date: Sat, 18 Mar 2023 13:27:10 -0500 Subject: [PATCH] insert node_ids pq.StringArray --- statediff/indexer/database/file/csv_writer.go | 2 +- statediff/indexer/database/file/indexer.go | 4 +- statediff/indexer/database/file/interfaces.go | 1 - statediff/indexer/database/file/sql_writer.go | 10 +---- statediff/indexer/database/sql/indexer.go | 1 - statediff/indexer/database/sql/interfaces.go | 1 - .../indexer/database/sql/postgres/database.go | 6 +-- statediff/indexer/database/sql/writer.go | 6 ++- statediff/indexer/models/models.go | 37 +++++++------------ 9 files changed, 26 insertions(+), 42 deletions(-) diff --git a/statediff/indexer/database/file/csv_writer.go b/statediff/indexer/database/file/csv_writer.go index 55f3584d2..be6f43304 100644 --- a/statediff/indexer/database/file/csv_writer.go +++ b/statediff/indexer/database/file/csv_writer.go @@ -250,7 +250,7 @@ func (csw *CSVWriter) upsertIPLDRaw(blockNumber string, codec, mh uint64, raw [] func (csw *CSVWriter) upsertHeaderCID(header models.HeaderModel) { var values []interface{} values = append(values, header.BlockNumber, header.BlockHash, header.ParentHash, header.CID, - header.TotalDifficulty, header.NodeIDs[0], header.Reward, header.StateRoot, header.TxRoot, + header.TotalDifficulty, header.NodeIDs, header.Reward, header.StateRoot, header.TxRoot, header.RctRoot, header.UnclesHash, header.Bloom, strconv.FormatUint(header.Timestamp, 10), header.Coinbase) csw.rows <- tableRow{types.TableHeader, values} indexerMetrics.blocks.Inc(1) diff --git a/statediff/indexer/database/file/indexer.go b/statediff/indexer/database/file/indexer.go index 42ead7cd9..a1ed0edc7 100644 --- a/statediff/indexer/database/file/indexer.go +++ b/statediff/indexer/database/file/indexer.go @@ -29,7 +29,7 @@ import ( blockstore "github.com/ipfs/go-ipfs-blockstore" dshelp "github.com/ipfs/go-ipfs-ds-help" - + "github.com/lib/pq" "github.com/multiformats/go-multihash" "github.com/ethereum/go-ethereum/common" @@ -238,7 +238,7 @@ func (sdi *StateDiffIndexer) processHeader(header *types.Header, headerNode ipld } headerID := header.Hash().String() sdi.fileWriter.upsertHeaderCID(models.HeaderModel{ - NodeIDs: []string{sdi.nodeID}, + NodeIDs: pq.StringArray([]string{sdi.nodeID}), CID: headerNode.Cid().String(), ParentHash: header.ParentHash.String(), BlockNumber: header.Number.String(), diff --git a/statediff/indexer/database/file/interfaces.go b/statediff/indexer/database/file/interfaces.go index 802d6f782..e322c35b9 100644 --- a/statediff/indexer/database/file/interfaces.go +++ b/statediff/indexer/database/file/interfaces.go @@ -39,7 +39,6 @@ type FileWriter interface { upsertHeaderCID(header models.HeaderModel) upsertUncleCID(uncle models.UncleModel) upsertTransactionCID(transaction models.TxModel) - upsertAccessListElement(accessListElement models.AccessListElementModel) upsertReceiptCID(rct *models.ReceiptModel) upsertLogCID(logs []*models.LogsModel) upsertStateCID(stateNode models.StateNodeModel) diff --git a/statediff/indexer/database/file/sql_writer.go b/statediff/indexer/database/file/sql_writer.go index 73b20e1bb..91f250074 100644 --- a/statediff/indexer/database/file/sql_writer.go +++ b/statediff/indexer/database/file/sql_writer.go @@ -136,7 +136,7 @@ func (sqw *SQLWriter) flush() error { } const ( - nodeInsert = "INSERT INTO nodes (genesis_block, network_id, node_ids, client_name, chain_id) VALUES " + + nodeInsert = "INSERT INTO nodes (genesis_block, network_id, node_id, client_name, chain_id) VALUES " + "('%s', '%s', '%s', '%s', %d);\n" ipldInsert = "INSERT INTO public.blocks (block_number, key, data) VALUES ('%s', '%s', '\\x%x');\n" @@ -204,7 +204,7 @@ func (sqw *SQLWriter) upsertIPLDRaw(blockNumber string, codec, mh uint64, raw [] func (sqw *SQLWriter) upsertHeaderCID(header models.HeaderModel) { stmt := fmt.Sprintf(headerInsert, header.BlockNumber, header.BlockHash, header.ParentHash, header.CID, - header.TotalDifficulty, header.NodeIDs[0], header.Reward, header.StateRoot, header.TxRoot, + header.TotalDifficulty, header.NodeIDs, header.Reward, header.StateRoot, header.TxRoot, header.RctRoot, header.UnclesHash, header.Bloom, header.Timestamp, header.Coinbase) sqw.stmts <- []byte(stmt) indexerMetrics.blocks.Inc(1) @@ -221,12 +221,6 @@ func (sqw *SQLWriter) upsertTransactionCID(transaction models.TxModel) { indexerMetrics.transactions.Inc(1) } -func (sqw *SQLWriter) upsertAccessListElement(accessListElement models.AccessListElementModel) { - sqw.stmts <- []byte(fmt.Sprintf(alInsert, accessListElement.BlockNumber, accessListElement.TxID, accessListElement.Index, accessListElement.Address, - formatPostgresStringArray(accessListElement.StorageKeys))) - indexerMetrics.accessListEntries.Inc(1) -} - func (sqw *SQLWriter) upsertReceiptCID(rct *models.ReceiptModel) { sqw.stmts <- []byte(fmt.Sprintf(rctInsert, rct.BlockNumber, rct.HeaderID, rct.TxID, rct.CID, rct.Contract, rct.ContractHash, rct.PostState, rct.PostStatus)) diff --git a/statediff/indexer/database/sql/indexer.go b/statediff/indexer/database/sql/indexer.go index 0e695edd5..018d037c5 100644 --- a/statediff/indexer/database/sql/indexer.go +++ b/statediff/indexer/database/sql/indexer.go @@ -28,7 +28,6 @@ import ( blockstore "github.com/ipfs/go-ipfs-blockstore" dshelp "github.com/ipfs/go-ipfs-ds-help" - "github.com/multiformats/go-multihash" "github.com/ethereum/go-ethereum/common" diff --git a/statediff/indexer/database/sql/interfaces.go b/statediff/indexer/database/sql/interfaces.go index 613a09251..77e95f1b2 100644 --- a/statediff/indexer/database/sql/interfaces.go +++ b/statediff/indexer/database/sql/interfaces.go @@ -46,7 +46,6 @@ type Statements interface { InsertHeaderStm() string InsertUncleStm() string InsertTxStm() string - InsertAccessListElementStm() string InsertRctStm() string InsertLogStm() string InsertStateStm() string diff --git a/statediff/indexer/database/sql/postgres/database.go b/statediff/indexer/database/sql/postgres/database.go index 5bd90dfcb..ab6bbb253 100644 --- a/statediff/indexer/database/sql/postgres/database.go +++ b/statediff/indexer/database/sql/postgres/database.go @@ -40,11 +40,11 @@ type DB struct { // Stm == Statement func (db *DB) InsertHeaderStm() string { if db.upsert { - return `INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncles_hash, bloom, timestamp, mh_key, times_validated, coinbase) + return `INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_ids, reward, state_root, tx_root, receipt_root, uncles_hash, bloom, timestamp, mh_key, times_validated, coinbase) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16) - ON CONFLICT (block_hash, block_number) DO UPDATE SET (parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncles_hash, bloom, timestamp, mh_key, times_validated, coinbase) = ($3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, eth.header_cids.times_validated + 1, $16)` + ON CONFLICT (block_hash, block_number) DO UPDATE SET (parent_hash, cid, td, node_ids, reward, state_root, tx_root, receipt_root, uncles_hash, bloom, timestamp, mh_key, times_validated, coinbase) = ($3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, eth.header_cids.times_validated + 1, $16)` } - return `INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncles_hash, bloom, timestamp, mh_key, times_validated, coinbase) + return `INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_ids, reward, state_root, tx_root, receipt_root, uncles_hash, bloom, timestamp, mh_key, times_validated, coinbase) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16) ON CONFLICT (block_hash, block_number) DO NOTHING` } diff --git a/statediff/indexer/database/sql/writer.go b/statediff/indexer/database/sql/writer.go index fb46cd1cf..43006a8a1 100644 --- a/statediff/indexer/database/sql/writer.go +++ b/statediff/indexer/database/sql/writer.go @@ -19,6 +19,8 @@ package sql import ( "fmt" + "github.com/lib/pq" + "github.com/ethereum/go-ethereum/statediff/indexer/models" ) @@ -40,13 +42,13 @@ func (w *Writer) Close() error { } /* -INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncles_hash, bloom, timestamp, mh_key, times_validated, coinbase) +INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_ids, reward, state_root, tx_root, receipt_root, uncles_hash, bloom, timestamp, mh_key, times_validated, coinbase) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) ON CONFLICT (block_hash, block_number) DO NOTHING */ func (w *Writer) upsertHeaderCID(tx Tx, header models.HeaderModel) error { _, err := tx.Exec(w.db.Context(), w.db.InsertHeaderStm(), - header.BlockNumber, header.BlockHash, header.ParentHash, header.CID, header.TotalDifficulty, w.db.NodeID(), + header.BlockNumber, header.BlockHash, header.ParentHash, header.CID, header.TotalDifficulty, pq.StringArray([]string{w.db.NodeID()}), header.Reward, header.StateRoot, header.TxRoot, header.RctRoot, header.UnclesHash, header.Bloom, header.Timestamp, header.Coinbase) if err != nil { diff --git a/statediff/indexer/models/models.go b/statediff/indexer/models/models.go index 2b9a48f13..b2a7286fd 100644 --- a/statediff/indexer/models/models.go +++ b/statediff/indexer/models/models.go @@ -27,20 +27,20 @@ type IPLDModel struct { // HeaderModel is the db model for eth.header_cids type HeaderModel struct { - BlockNumber string `db:"block_number"` - BlockHash string `db:"block_hash"` - ParentHash string `db:"parent_hash"` - CID string `db:"cid"` - TotalDifficulty string `db:"td"` - NodeIDs []string `db:"node_ids"` - Reward string `db:"reward"` - StateRoot string `db:"state_root"` - UnclesHash string `db:"uncles_hash"` - TxRoot string `db:"tx_root"` - RctRoot string `db:"receipt_root"` - Bloom []byte `db:"bloom"` - Timestamp uint64 `db:"timestamp"` - Coinbase string `db:"coinbase"` + BlockNumber string `db:"block_number"` + BlockHash string `db:"block_hash"` + ParentHash string `db:"parent_hash"` + CID string `db:"cid"` + TotalDifficulty string `db:"td"` + NodeIDs pq.StringArray `db:"node_ids"` + Reward string `db:"reward"` + StateRoot string `db:"state_root"` + UnclesHash string `db:"uncles_hash"` + TxRoot string `db:"tx_root"` + RctRoot string `db:"receipt_root"` + Bloom []byte `db:"bloom"` + Timestamp uint64 `db:"timestamp"` + Coinbase string `db:"coinbase"` } // UncleModel is the db model for eth.uncle_cids @@ -67,15 +67,6 @@ type TxModel struct { Value string `db:"value"` } -// AccessListElementModel is the db model for eth.access_list_entry -type AccessListElementModel struct { - BlockNumber string `db:"block_number"` - Index int64 `db:"index"` - TxID string `db:"tx_id"` - Address string `db:"address"` - StorageKeys pq.StringArray `db:"storage_keys"` -} - // ReceiptModel is the db model for eth.receipt_cids type ReceiptModel struct { BlockNumber string `db:"block_number"`