cerc-io/go-ethereum
14
0
Fork 0
Code Issues 9 Pull Requests Packages Projects Releases 66 Wiki Activity

update types, models, and indexers and writers to match removal of partial_path columns from state_cids and storage_cids

Browse Source
This commit is contained in:
i-norden 2023-02-15 15:20:50 -06:00
parent 583de67b10
commit 1e574d4df8
7 changed files with 24 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
statediff/indexer/database/file/csv_writer.go
View File

@ -251,8 +251,8 @@ func (csw *CSVWriter) upsertIPLDRaw(blockNumber string, codec, mh uint64, raw []
func (csw *CSVWriter) upsertHeaderCID(header models.HeaderModel) { func (csw *CSVWriter) upsertHeaderCID(header models.HeaderModel) {
var values []interface{} var values []interface{}
values = append(values, header.BlockNumber, header.BlockHash, header.ParentHash, header.CID, values = append(values, header.BlockNumber, header.BlockHash, header.ParentHash, header.CID,
header.TotalDifficulty, header.NodeID, header.Reward, header.StateRoot, header.TxRoot, header.TotalDifficulty, header.NodeIDs[0], header.Reward, header.StateRoot, header.TxRoot,
header.RctRoot, header.UnclesHash, header.Bloom, strconv.FormatUint(header.Timestamp, 10), header.MhKey, 1, header.Coinbase) header.RctRoot, header.UnclesHash, header.Bloom, strconv.FormatUint(header.Timestamp, 10), header.Coinbase)
csw.rows <- tableRow{types.TableHeader, values} csw.rows <- tableRow{types.TableHeader, values}
indexerMetrics.blocks.Inc(1) indexerMetrics.blocks.Inc(1)
} }
@ -260,14 +260,14 @@ func (csw *CSVWriter) upsertHeaderCID(header models.HeaderModel) {
func (csw *CSVWriter) upsertUncleCID(uncle models.UncleModel) { func (csw *CSVWriter) upsertUncleCID(uncle models.UncleModel) {
var values []interface{} var values []interface{}
values = append(values, uncle.BlockNumber, uncle.BlockHash, uncle.HeaderID, uncle.ParentHash, uncle.CID, values = append(values, uncle.BlockNumber, uncle.BlockHash, uncle.HeaderID, uncle.ParentHash, uncle.CID,
uncle.Reward, uncle.MhKey, uncle.Index) uncle.Reward, uncle.Index)
csw.rows <- tableRow{types.TableUncle, values} csw.rows <- tableRow{types.TableUncle, values}
} }
func (csw *CSVWriter) upsertTransactionCID(transaction models.TxModel) { func (csw *CSVWriter) upsertTransactionCID(transaction models.TxModel) {
var values []interface{} var values []interface{}
values = append(values, transaction.BlockNumber, transaction.HeaderID, transaction.TxHash, transaction.CID, transaction.Dst, values = append(values, transaction.BlockNumber, transaction.HeaderID, transaction.TxHash, transaction.CID, transaction.Dst,
transaction.Src, transaction.Index, transaction.MhKey, transaction.Data, transaction.Type, transaction.Value) transaction.Src, transaction.Index, transaction.Type, transaction.Value)
csw.rows <- tableRow{types.TableTransaction, values} csw.rows <- tableRow{types.TableTransaction, values}
indexerMetrics.transactions.Inc(1) indexerMetrics.transactions.Inc(1)
} }
@ -281,8 +281,8 @@ func (csw *CSVWriter) upsertAccessListElement(accessListElement models.AccessLis
func (csw *CSVWriter) upsertReceiptCID(rct *models.ReceiptModel) { func (csw *CSVWriter) upsertReceiptCID(rct *models.ReceiptModel) {
var values []interface{} var values []interface{}
values = append(values, rct.BlockNumber, rct.HeaderID, rct.TxID, rct.LeafCID, rct.Contract, rct.ContractHash, rct.LeafMhKey, values = append(values, rct.BlockNumber, rct.HeaderID, rct.TxID, rct.CID, rct.Contract, rct.ContractHash,
rct.PostState, rct.PostStatus, rct.LogRoot) rct.PostState, rct.PostStatus)
csw.rows <- tableRow{types.TableReceipt, values} csw.rows <- tableRow{types.TableReceipt, values}
indexerMetrics.receipts.Inc(1) indexerMetrics.receipts.Inc(1)
} }
@ -290,8 +290,8 @@ func (csw *CSVWriter) upsertReceiptCID(rct *models.ReceiptModel) {
func (csw *CSVWriter) upsertLogCID(logs []*models.LogsModel) { func (csw *CSVWriter) upsertLogCID(logs []*models.LogsModel) {
for _, l := range logs { for _, l := range logs {
var values []interface{} var values []interface{}
values = append(values, l.BlockNumber, l.HeaderID, l.LeafCID, l.LeafMhKey, l.ReceiptID, l.Address, l.Index, l.Topic0, values = append(values, l.BlockNumber, l.HeaderID, l.CID, l.ReceiptID, l.Address, l.Index, l.Topic0,
l.Topic1, l.Topic2, l.Topic3, l.Data) l.Topic1, l.Topic2, l.Topic3)
csw.rows <- tableRow{types.TableLog, values} csw.rows <- tableRow{types.TableLog, values}
indexerMetrics.logs.Inc(1) indexerMetrics.logs.Inc(1)
} }
@ -304,18 +304,11 @@ func (csw *CSVWriter) upsertStateCID(stateNode models.StateNodeModel) {
} }
var values []interface{} var values []interface{}
values = append(values, stateNode.BlockNumber, stateNode.HeaderID, stateKey, stateNode.CID, stateNode.Path, values = append(values, stateNode.BlockNumber, stateNode.HeaderID, stateKey, stateNode.CID,
stateNode.NodeType, true, stateNode.MhKey) true, stateNode.Balance, strconv.FormatUint(stateNode.Nonce, 10), stateNode.CodeHash, stateNode.StorageRoot, stateNode.Removed)
csw.rows <- tableRow{types.TableStateNode, values} csw.rows <- tableRow{types.TableStateNode, values}
} }
func (csw *CSVWriter) upsertStateAccount(stateAccount models.StateAccountModel) {
var values []interface{}
values = append(values, stateAccount.BlockNumber, stateAccount.HeaderID, stateAccount.StatePath, stateAccount.Balance,
strconv.FormatUint(stateAccount.Nonce, 10), stateAccount.CodeHash, stateAccount.StorageRoot)
csw.rows <- tableRow{types.TableStateAccount, values}
}
func (csw *CSVWriter) upsertStorageCID(storageCID models.StorageNodeModel) { func (csw *CSVWriter) upsertStorageCID(storageCID models.StorageNodeModel) {
var storageKey string var storageKey string
if storageCID.StorageKey != nullHash.String() { if storageCID.StorageKey != nullHash.String() {
@ -323,8 +316,8 @@ func (csw *CSVWriter) upsertStorageCID(storageCID models.StorageNodeModel) {
} }
var values []interface{} var values []interface{}
values = append(values, storageCID.BlockNumber, storageCID.HeaderID, storageCID.StatePath, storageKey, storageCID.CID, values = append(values, storageCID.BlockNumber, storageCID.HeaderID, storageCID.StateKey, storageKey, storageCID.CID,
storageCID.Path, storageCID.NodeType, true, storageCID.MhKey) true, storageCID.Value, storageCID.Removed)
csw.rows <- tableRow{types.TableStorageNode, values} csw.rows <- tableRow{types.TableStorageNode, values}
} }

4
statediff/indexer/database/file/indexer.go
View File

@ -415,7 +415,7 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt
} }
// publish the state node // publish the state node
var stateModel models.StateNodeModel var stateModel models.StateNodeModel
if stateNode.NodeType == sdtypes.Removed { if stateNode.Removed {
if atomic.LoadUint32(sdi.removedCacheFlag) == 0 { if atomic.LoadUint32(sdi.removedCacheFlag) == 0 {
atomic.StoreUint32(sdi.removedCacheFlag, 1) atomic.StoreUint32(sdi.removedCacheFlag, 1)
sdi.fileWriter.upsertIPLDDirect(tx.BlockNumber, shared.RemovedNodeMhKey, []byte{}) sdi.fileWriter.upsertIPLDDirect(tx.BlockNumber, shared.RemovedNodeMhKey, []byte{})
@ -423,7 +423,6 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt
stateModel = models.StateNodeModel{ stateModel = models.StateNodeModel{
BlockNumber: tx.BlockNumber, BlockNumber: tx.BlockNumber,
HeaderID: headerID, HeaderID: headerID,
Path: stateNode.Path,
StateKey: common.BytesToHash(stateNode.LeafKey).String(), StateKey: common.BytesToHash(stateNode.LeafKey).String(),
CID: shared.RemovedNodeStateCID, CID: shared.RemovedNodeStateCID,
Removed: true, Removed: true,
@ -436,7 +435,6 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt
stateModel = models.StateNodeModel{ stateModel = models.StateNodeModel{
BlockNumber: tx.BlockNumber, BlockNumber: tx.BlockNumber,
HeaderID: headerID, HeaderID: headerID,
Path: stateNode.Path,
StateKey: common.BytesToHash(stateNode.LeafKey).String(), StateKey: common.BytesToHash(stateNode.LeafKey).String(),
CID: stateCIDStr, CID: stateCIDStr,
Removed: false, Removed: false,

12
statediff/indexer/database/file/sql_writer.go
View File

@ -161,11 +161,11 @@ const (
logInsert = "INSERT INTO eth.log_cids (block_number, header_id, cid, rct_id, address, index, topic0, topic1, topic2, " + logInsert = "INSERT INTO eth.log_cids (block_number, header_id, cid, rct_id, address, index, topic0, topic1, topic2, " +
"topic3) VALUES ('%s', '%s', '%s', '%s', '%s', %d, '%s', '%s', '%s', '%s');\n" "topic3) VALUES ('%s', '%s', '%s', '%s', '%s', %d, '%s', '%s', '%s', '%s');\n"
stateInsert = "INSERT INTO eth.state_cids (block_number, header_id, state_leaf_key, cid, partial_path, removed, diff, " + stateInsert = "INSERT INTO eth.state_cids (block_number, header_id, state_leaf_key, cid, removed, diff, " +
"balance, nonce, code_hash, storage_root) VALUES ('%s', '%s', '%s', '%s', '\\x%x', %t, %t, '%s', %d, '\\x%x', '%s');\n" "balance, nonce, code_hash, storage_root) VALUES ('%s', '%s', '%s', '%s', %t, %t, '%s', %d, '\\x%x', '%s');\n"
storageInsert = "INSERT INTO eth.storage_cids (block_number, header_id, state_leaf_key, storage_leaf_key, cid, partial_path, " + storageInsert = "INSERT INTO eth.storage_cids (block_number, header_id, state_leaf_key, storage_leaf_key, cid, " +
"removed, diff, val) VALUES ('%s', '%s', '%s', '%s', '%s', '\\x%x', %t, %t, '\\x%x');\n" "removed, diff, val) VALUES ('%s', '%s', '%s', '%s', '%s', %t, %t, '\\x%x');\n"
) )
func (sqw *SQLWriter) upsertNode(node nodeinfo.Info) { func (sqw *SQLWriter) upsertNode(node nodeinfo.Info) {
@ -246,13 +246,13 @@ func (sqw *SQLWriter) upsertLogCID(logs []*models.LogsModel) {
} }
func (sqw *SQLWriter) upsertStateCID(stateNode models.StateNodeModel) { func (sqw *SQLWriter) upsertStateCID(stateNode models.StateNodeModel) {
sqw.stmts <- []byte(fmt.Sprintf(stateInsert, stateNode.BlockNumber, stateNode.HeaderID, stateNode.StateKey, stateNode.CID, stateNode.Path, sqw.stmts <- []byte(fmt.Sprintf(stateInsert, stateNode.BlockNumber, stateNode.HeaderID, stateNode.StateKey, stateNode.CID,
stateNode.Removed, true, stateNode.Balance, stateNode.Nonce, stateNode.CodeHash, stateNode.StorageRoot)) stateNode.Removed, true, stateNode.Balance, stateNode.Nonce, stateNode.CodeHash, stateNode.StorageRoot))
} }
func (sqw *SQLWriter) upsertStorageCID(storageCID models.StorageNodeModel) { func (sqw *SQLWriter) upsertStorageCID(storageCID models.StorageNodeModel) {
sqw.stmts <- []byte(fmt.Sprintf(storageInsert, storageCID.BlockNumber, storageCID.HeaderID, storageCID.StateKey, storageCID.StorageKey, storageCID.CID, sqw.stmts <- []byte(fmt.Sprintf(storageInsert, storageCID.BlockNumber, storageCID.HeaderID, storageCID.StateKey, storageCID.StorageKey, storageCID.CID,
storageCID.Path, storageCID.Removed, true, storageCID.Value)) storageCID.Removed, true, storageCID.Value))
} }
// LoadWatchedAddresses loads watched addresses from a file // LoadWatchedAddresses loads watched addresses from a file

4
statediff/indexer/database/sql/indexer.go
View File

@ -429,12 +429,11 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt
} }
// publish the state node // publish the state node
var stateModel models.StateNodeModel var stateModel models.StateNodeModel
if stateNode.NodeType == sdtypes.Removed { if stateNode.Removed {
tx.cacheRemoved(shared.RemovedNodeMhKey, []byte{}) tx.cacheRemoved(shared.RemovedNodeMhKey, []byte{})
stateModel = models.StateNodeModel{ stateModel = models.StateNodeModel{
BlockNumber: tx.BlockNumber, BlockNumber: tx.BlockNumber,
HeaderID: headerID, HeaderID: headerID,
Path: stateNode.Path,
StateKey: common.BytesToHash(stateNode.LeafKey).String(), StateKey: common.BytesToHash(stateNode.LeafKey).String(),
CID: shared.RemovedNodeStateCID, CID: shared.RemovedNodeStateCID,
Removed: true, Removed: true,
@ -447,7 +446,6 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt
stateModel = models.StateNodeModel{ stateModel = models.StateNodeModel{
BlockNumber: tx.BlockNumber, BlockNumber: tx.BlockNumber,
HeaderID: headerID, HeaderID: headerID,
Path: stateNode.Path,
StateKey: common.BytesToHash(stateNode.LeafKey).String(), StateKey: common.BytesToHash(stateNode.LeafKey).String(),
CID: stateCIDStr, CID: stateCIDStr,
Removed: false, Removed: false,

8
statediff/indexer/database/sql/writer.go
View File

@ -132,12 +132,12 @@ func (w *Writer) upsertLogCID(tx Tx, logs []*models.LogsModel) error {
} }
/* /*
INSERT INTO eth.state_cids (block_number, header_id, state_leaf_key, cid, partial_path, removed, diff, balance, nonce, code_hash, storage_root) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) INSERT INTO eth.state_cids (block_number, header_id, state_leaf_key, cid, removed, diff, balance, nonce, code_hash, storage_root) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
ON CONFLICT (header_id, state_leaf_key, block_number) DO NOTHING ON CONFLICT (header_id, state_leaf_key, block_number) DO NOTHING
*/ */
func (w *Writer) upsertStateCID(tx Tx, stateNode models.StateNodeModel) error { func (w *Writer) upsertStateCID(tx Tx, stateNode models.StateNodeModel) error {
_, err := tx.Exec(w.db.Context(), w.db.InsertStateStm(), _, err := tx.Exec(w.db.Context(), w.db.InsertStateStm(),
stateNode.BlockNumber, stateNode.HeaderID, stateNode.StateKey, stateNode.CID, stateNode.Path, stateNode.Removed, true, stateNode.BlockNumber, stateNode.HeaderID, stateNode.StateKey, stateNode.CID, stateNode.Removed, true,
stateNode.Balance, stateNode.Nonce, stateNode.CodeHash, stateNode.StorageRoot) stateNode.Balance, stateNode.Nonce, stateNode.CodeHash, stateNode.StorageRoot)
if err != nil { if err != nil {
return insertError{"eth.state_cids", err, w.db.InsertStateStm(), stateNode} return insertError{"eth.state_cids", err, w.db.InsertStateStm(), stateNode}
@ -146,12 +146,12 @@ func (w *Writer) upsertStateCID(tx Tx, stateNode models.StateNodeModel) error {
} }
/* /*
INSERT INTO eth.storage_cids (block_number, header_id, state_leaf_key, storage_leaf_key, cid, storage_path, removed, diff, val) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) INSERT INTO eth.storage_cids (block_number, header_id, state_leaf_key, storage_leaf_key, cid, removed, diff, val) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
ON CONFLICT (header_id, state_leaf_key, storage_leaf_key, block_number) DO NOTHING ON CONFLICT (header_id, state_leaf_key, storage_leaf_key, block_number) DO NOTHING
*/ */
func (w *Writer) upsertStorageCID(tx Tx, storageCID models.StorageNodeModel) error { func (w *Writer) upsertStorageCID(tx Tx, storageCID models.StorageNodeModel) error {
_, err := tx.Exec(w.db.Context(), w.db.InsertStorageStm(), _, err := tx.Exec(w.db.Context(), w.db.InsertStorageStm(),
storageCID.BlockNumber, storageCID.HeaderID, storageCID.StateKey, storageCID.StorageKey, storageCID.CID, storageCID.Path, storageCID.BlockNumber, storageCID.HeaderID, storageCID.StateKey, storageCID.StorageKey, storageCID.CID,
storageCID.Removed, true, storageCID.Value) storageCID.Removed, true, storageCID.Value)
if err != nil { if err != nil {
return insertError{"eth.storage_cids", err, w.db.InsertStorageStm(), storageCID} return insertError{"eth.storage_cids", err, w.db.InsertStorageStm(), storageCID}

2
statediff/indexer/models/models.go
View File

@ -92,7 +92,6 @@ type ReceiptModel struct {
type StateNodeModel struct { type StateNodeModel struct {
BlockNumber string `db:"block_number"` BlockNumber string `db:"block_number"`
HeaderID string `db:"header_id"` HeaderID string `db:"header_id"`
Path []byte `db:"partial_path"`
StateKey string `db:"state_leaf_key"` StateKey string `db:"state_leaf_key"`
Removed bool `db:"removed"` Removed bool `db:"removed"`
CID string `db:"cid"` CID string `db:"cid"`
@ -108,7 +107,6 @@ type StorageNodeModel struct {
BlockNumber string `db:"block_number"` BlockNumber string `db:"block_number"`
HeaderID string `db:"header_id"` HeaderID string `db:"header_id"`
StateKey []byte `db:"state_leaf_key"` StateKey []byte `db:"state_leaf_key"`
Path []byte `db:"partial_path"`
StorageKey string `db:"storage_leaf_key"` StorageKey string `db:"storage_leaf_key"`
Removed bool `db:"removed"` Removed bool `db:"removed"`
CID string `db:"cid"` CID string `db:"cid"`

4
statediff/types/types.go
View File

@ -43,8 +43,6 @@ type AccountMap map[string]AccountWrapper
type AccountWrapper struct { type AccountWrapper struct {
Account *types.StateAccount Account *types.StateAccount
Removed bool Removed bool
Path []byte
NodeValue []byte
LeafKey []byte LeafKey []byte
LeafNodeHash []byte LeafNodeHash []byte
} }
@ -52,7 +50,6 @@ type AccountWrapper struct {
// StateNode holds the data for a single state diff node // StateNode holds the data for a single state diff node
type StateNode struct { type StateNode struct {
Removed bool `json:"removed" gencodec:"required"` Removed bool `json:"removed" gencodec:"required"`
Path []byte `json:"path" gencodec:"required"`
NodeValue []byte `json:"value" gencodec:"required"` NodeValue []byte `json:"value" gencodec:"required"`
StorageNodes []StorageNode `json:"storage"` StorageNodes []StorageNode `json:"storage"`
LeafKey []byte `json:"leafKey"` LeafKey []byte `json:"leafKey"`
@ -62,7 +59,6 @@ type StateNode struct {
// StorageNode holds the data for a single storage diff node // StorageNode holds the data for a single storage diff node
type StorageNode struct { type StorageNode struct {
Removed bool `json:"removed" gencodec:"required"` Removed bool `json:"removed" gencodec:"required"`
Path []byte `json:"path" gencodec:"required"`
NodeValue []byte `json:"value" gencodec:"required"` NodeValue []byte `json:"value" gencodec:"required"`
LeafKey []byte `json:"leafKey"` LeafKey []byte `json:"leafKey"`
NodeHash []byte `json:"hash"` NodeHash []byte `json:"hash"`