Fix linting errors.

This commit is contained in:
Arijit Das 2021-12-14 12:20:19 +05:30
parent 8b4ebfa355
commit e4de336833
6 changed files with 90 additions and 75 deletions

View File

@ -37,6 +37,7 @@ var rootCmd = &cobra.Command{
PersistentPreRun: initFuncs, PersistentPreRun: initFuncs,
} }
// Execute executes root Command.
func Execute() { func Execute() {
log.Info("----- Starting vDB -----") log.Info("----- Starting vDB -----")
if err := rootCmd.Execute(); err != nil { if err := rootCmd.Execute(); err != nil {

View File

@ -19,6 +19,7 @@ import (
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/spf13/viper" "github.com/spf13/viper"
"github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/snapshot" "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/snapshot"
) )
@ -37,7 +38,7 @@ var stateSnapshotCmd = &cobra.Command{
} }
func stateSnapshot() { func stateSnapshot() {
snapConfig := snapshot.Config{} snapConfig := &snapshot.Config{}
snapConfig.Init() snapConfig.Init()
snapshotService, err := snapshot.NewSnapshotService(snapConfig) snapshotService, err := snapshot.NewSnapshotService(snapConfig)
if err != nil { if err != nil {

View File

@ -16,38 +16,40 @@
package snapshot package snapshot
import ( import (
"github.com/ethereum/go-ethereum/statediff/indexer/node" ethNode "github.com/ethereum/go-ethereum/statediff/indexer/node"
"github.com/ethereum/go-ethereum/statediff/indexer/postgres" "github.com/ethereum/go-ethereum/statediff/indexer/postgres"
"github.com/spf13/viper" "github.com/spf13/viper"
) )
const ( const (
ancientDbPath = "ANCIENT_DB_PATH" ancientDBPath = "ANCIENT_DB_PATH"
ethClientName = "ETH_CLIENT_NAME" ethClientName = "ETH_CLIENT_NAME"
ethGenesisBlock = "ETH_GENESIS_BLOCK" ethGenesisBlock = "ETH_GENESIS_BLOCK"
ethNetworkId = "ETH_NETWORK_ID" ethNetworkID = "ETH_NETWORK_ID"
ethNodeId = "ETH_NODE_ID" ethNodeID = "ETH_NODE_ID"
lvlDbPath = "LVL_DB_PATH" lvlDBPath = "LVL_DB_PATH"
) )
// Config is config parameters for DB.
type Config struct { type Config struct {
LevelDBPath string LevelDBPath string
AncientDBPath string AncientDBPath string
Node node.Info Node ethNode.Info
connectionURI string connectionURI string
DBConfig postgres.ConnectionConfig DBConfig postgres.ConnectionConfig
} }
// Init Initialises config
func (c *Config) Init() { func (c *Config) Init() {
c.dbInit() c.dbInit()
viper.BindEnv("leveldb.path", lvlDbPath) viper.BindEnv("leveldb.path", lvlDBPath)
viper.BindEnv("ethereum.nodeID", ethNodeId) viper.BindEnv("ethereum.nodeID", ethNodeID)
viper.BindEnv("ethereum.clientName", ethClientName) viper.BindEnv("ethereum.clientName", ethClientName)
viper.BindEnv("ethereum.genesisBlock", ethGenesisBlock) viper.BindEnv("ethereum.genesisBlock", ethGenesisBlock)
viper.BindEnv("ethereum.networkID", ethNetworkId) viper.BindEnv("ethereum.networkID", ethNetworkID)
viper.BindEnv("leveldb.ancient", ancientDbPath) viper.BindEnv("leveldb.ancient", ancientDBPath)
c.Node = node.Info{ c.Node = ethNode.Info{
ID: viper.GetString("ethereum.nodeID"), ID: viper.GetString("ethereum.nodeID"),
ClientName: viper.GetString("ethereum.clientName"), ClientName: viper.GetString("ethereum.clientName"),
GenesisBlock: viper.GetString("ethereum.genesisBlock"), GenesisBlock: viper.GetString("ethereum.genesisBlock"),

View File

@ -21,43 +21,43 @@ import (
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
) )
// Node for holding trie node information // node for holding trie node information
type Node struct { type node struct {
NodeType NodeType nodeType nodeType
Path []byte path []byte
Key common.Hash key common.Hash
Value []byte value []byte
} }
// NodeType for explicitly setting type of node // nodeType for explicitly setting type of node
type NodeType int type nodeType int
const ( const (
Branch NodeType = iota branch nodeType = iota
Extension extension
Leaf leaf
Removed removed
Unknown unknown
) )
// CheckKeyType checks what type of key we have // CheckKeyType checks what type of key we have
func CheckKeyType(elements []interface{}) (NodeType, error) { func CheckKeyType(elements []interface{}) (nodeType, error) {
if len(elements) > 2 { if len(elements) > 2 {
return Branch, nil return branch, nil
} }
if len(elements) < 2 { if len(elements) < 2 {
return Unknown, fmt.Errorf("node cannot be less than two elements in length") return unknown, fmt.Errorf("node cannot be less than two elements in length")
} }
switch elements[0].([]byte)[0] / 16 { switch elements[0].([]byte)[0] / 16 {
case '\x00': case '\x00':
return Extension, nil return extension, nil
case '\x01': case '\x01':
return Extension, nil return extension, nil
case '\x02': case '\x02':
return Leaf, nil return leaf, nil
case '\x03': case '\x03':
return Leaf, nil return leaf, nil
default: default:
return Unknown, fmt.Errorf("unknown hex prefix") return unknown, fmt.Errorf("unknown hex prefix")
} }
} }

View File

@ -29,11 +29,13 @@ import (
"github.com/ethereum/go-ethereum/statediff/indexer/shared" "github.com/ethereum/go-ethereum/statediff/indexer/shared"
) )
// Publisher is wrapper around DB.
type Publisher struct { type Publisher struct {
db *postgres.DB db *postgres.DB
currBatchSize uint currBatchSize uint
} }
// NewPublisher creates Publisher
func NewPublisher(db *postgres.DB) *Publisher { func NewPublisher(db *postgres.DB) *Publisher {
return &Publisher{ return &Publisher{
db: db, db: db,
@ -81,11 +83,11 @@ func (p *Publisher) PublishHeader(header *types.Header) (int64, error) {
} }
// PublishStateNode writes the state node to the ipfs backing datastore and adds secondary indexes in the state_cids table // PublishStateNode writes the state node to the ipfs backing datastore and adds secondary indexes in the state_cids table
func (p *Publisher) PublishStateNode(node Node, headerID int64, tx *sqlx.Tx) (int64, error) { func (p *Publisher) PublishStateNode(node *node, headerID int64, tx *sqlx.Tx) (int64, error) {
var stateID int64 var stateID int64
var stateKey string var stateKey string
if !bytes.Equal(node.Key.Bytes(), nullHash.Bytes()) { if !bytes.Equal(node.key.Bytes(), nullHash.Bytes()) {
stateKey = node.Key.Hex() stateKey = node.key.Hex()
} }
defer func() { defer func() {
@ -95,7 +97,7 @@ func (p *Publisher) PublishStateNode(node Node, headerID int64, tx *sqlx.Tx) (in
} }
}() }()
stateCIDStr, mhKey, err := shared.PublishRaw(tx, ipld.MEthStateTrie, multihash.KECCAK_256, node.Value) stateCIDStr, mhKey, err := shared.PublishRaw(tx, ipld.MEthStateTrie, multihash.KECCAK_256, node.value)
if err != nil { if err != nil {
return 0, err return 0, err
} }
@ -103,17 +105,17 @@ func (p *Publisher) PublishStateNode(node Node, headerID int64, tx *sqlx.Tx) (in
err = tx.QueryRowx(`INSERT INTO eth.state_cids (header_id, state_leaf_key, cid, state_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7) err = tx.QueryRowx(`INSERT INTO eth.state_cids (header_id, state_leaf_key, cid, state_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7)
ON CONFLICT (header_id, state_path) DO UPDATE SET (state_leaf_key, cid, node_type, diff, mh_key) = ($2, $3, $5, $6, $7) ON CONFLICT (header_id, state_path) DO UPDATE SET (state_leaf_key, cid, node_type, diff, mh_key) = ($2, $3, $5, $6, $7)
RETURNING id`, RETURNING id`,
headerID, stateKey, stateCIDStr, node.Path, node.NodeType, false, mhKey).Scan(&stateID) headerID, stateKey, stateCIDStr, node.path, node.nodeType, false, mhKey).Scan(&stateID)
p.currBatchSize += 2 p.currBatchSize += 2
return stateID, err return stateID, err
} }
// PublishStorageNode writes the storage node to the ipfs backing pg datastore and adds secondary indexes in the storage_cids table // PublishStorageNode writes the storage node to the ipfs backing pg datastore and adds secondary indexes in the storage_cids table
func (p *Publisher) PublishStorageNode(node Node, stateID int64, tx *sqlx.Tx) error { func (p *Publisher) PublishStorageNode(node *node, stateID int64, tx *sqlx.Tx) error {
var storageKey string var storageKey string
if !bytes.Equal(node.Key.Bytes(), nullHash.Bytes()) { if !bytes.Equal(node.key.Bytes(), nullHash.Bytes()) {
storageKey = node.Key.Hex() storageKey = node.key.Hex()
} }
defer func() { defer func() {
@ -123,14 +125,14 @@ func (p *Publisher) PublishStorageNode(node Node, stateID int64, tx *sqlx.Tx) er
} }
}() }()
storageCIDStr, mhKey, err := shared.PublishRaw(tx, ipld.MEthStorageTrie, multihash.KECCAK_256, node.Value) storageCIDStr, mhKey, err := shared.PublishRaw(tx, ipld.MEthStorageTrie, multihash.KECCAK_256, node.value)
if err != nil { if err != nil {
return err return err
} }
_, err = tx.Exec(`INSERT INTO eth.storage_cids (state_id, storage_leaf_key, cid, storage_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7) _, err = tx.Exec(`INSERT INTO eth.storage_cids (state_id, storage_leaf_key, cid, storage_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7)
ON CONFLICT (state_id, storage_path) DO UPDATE SET (storage_leaf_key, cid, node_type, diff, mh_key) = ($2, $3, $5, $6, $7)`, ON CONFLICT (state_id, storage_path) DO UPDATE SET (storage_leaf_key, cid, node_type, diff, mh_key) = ($2, $3, $5, $6, $7)`,
stateID, storageKey, storageCIDStr, node.Path, node.NodeType, false, mhKey) stateID, storageKey, storageCIDStr, node.path, node.nodeType, false, mhKey)
if err != nil { if err != nil {
return err return err
} }

View File

@ -27,11 +27,10 @@ import (
"github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
"github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie"
"github.com/jmoiron/sqlx" "github.com/jmoiron/sqlx"
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
) )
var ( var (
@ -43,6 +42,8 @@ var (
defaultBatchSize = uint(100) defaultBatchSize = uint(100)
) )
// Service holds ethDB and stateDB to read data from lvldb and Publisher
// to publish trie in postgres DB.
type Service struct { type Service struct {
ethDB ethdb.Database ethDB ethdb.Database
stateDB state.Database stateDB state.Database
@ -50,7 +51,8 @@ type Service struct {
maxBatchSize uint maxBatchSize uint
} }
func NewSnapshotService(con Config) (*Service, error) { // NewSnapshotService creates Service.
func NewSnapshotService(con *Config) (*Service, error) {
pgDB, err := postgres.NewDB(con.connectionURI, con.DBConfig, con.Node) pgDB, err := postgres.NewDB(con.connectionURI, con.DBConfig, con.Node)
if err != nil { if err != nil {
return nil, err return nil, err
@ -69,6 +71,7 @@ func NewSnapshotService(con Config) (*Service, error) {
}, nil }, nil
} }
// CreateLatestSnapshot creates snapshot for the latest block.
func (s *Service) CreateLatestSnapshot() error { func (s *Service) CreateLatestSnapshot() error {
// extract header from lvldb and publish to PG-IPFS // extract header from lvldb and publish to PG-IPFS
// hold onto the headerID so that we can link the state nodes to this header // hold onto the headerID so that we can link the state nodes to this header
@ -101,6 +104,7 @@ func (s *Service) CreateLatestSnapshot() error {
return s.createSnapshot(t.NodeIterator([]byte{}), trieDB, headerID) return s.createSnapshot(t.NodeIterator([]byte{}), trieDB, headerID)
} }
// CreateSnapshot creates snapshot for given block height.
func (s *Service) CreateSnapshot(height uint64) error { func (s *Service) CreateSnapshot(height uint64) error {
// extract header from lvldb and publish to PG-IPFS // extract header from lvldb and publish to PG-IPFS
// hold onto the headerID so that we can link the state nodes to this header // hold onto the headerID so that we can link the state nodes to this header
@ -153,17 +157,17 @@ func (s *Service) createSnapshot(it trie.NodeIterator, trieDB *trie.Database, he
copy(nodePath, it.Path()) copy(nodePath, it.Path())
var ( var (
node []byte nodeData []byte
ty NodeType ty nodeType
) )
node, err = trieDB.Node(it.Hash()) nodeData, err = trieDB.Node(it.Hash())
if err != nil { if err != nil {
return err return err
} }
var nodeElements []interface{} var nodeElements []interface{}
if err = rlp.DecodeBytes(node, &nodeElements); err != nil { if err = rlp.DecodeBytes(nodeData, &nodeElements); err != nil {
return err return err
} }
@ -172,25 +176,25 @@ func (s *Service) createSnapshot(it trie.NodeIterator, trieDB *trie.Database, he
return err return err
} }
stateNode := Node{ stateNode := &node{
NodeType: ty, nodeType: ty,
Path: nodePath, path: nodePath,
Value: node, value: nodeData,
} }
switch ty { switch ty {
case Leaf: case leaf:
// if the node is a leaf, decode the account and publish the associated storage trie nodes if there are any // if the node is a leaf, decode the account and publish the associated storage trie nodes if there are any
var account types.StateAccount var account types.StateAccount
if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil { if err = rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil {
return fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", nodePath, err) return fmt.Errorf("error decoding account for leaf node at path %x nerror: %w", nodePath, err)
} }
partialPath := trie.CompactToHex(nodeElements[0].([]byte)) partialPath := trie.CompactToHex(nodeElements[0].([]byte))
valueNodePath := append(nodePath, partialPath...) valueNodePath := append(nodePath, partialPath...)
encodedPath := trie.HexToCompact(valueNodePath) encodedPath := trie.HexToCompact(valueNodePath)
leafKey := encodedPath[1:] leafKey := encodedPath[1:]
stateNode.Key = common.BytesToHash(leafKey) stateNode.key = common.BytesToHash(leafKey)
stateID, err := s.ipfsPublisher.PublishStateNode(stateNode, headerID, tx) stateID, err := s.ipfsPublisher.PublishStateNode(stateNode, headerID, tx)
if err != nil { if err != nil {
@ -199,23 +203,22 @@ func (s *Service) createSnapshot(it trie.NodeIterator, trieDB *trie.Database, he
// publish any non-nil code referenced by codehash // publish any non-nil code referenced by codehash
if !bytes.Equal(account.CodeHash, emptyCodeHash) { if !bytes.Equal(account.CodeHash, emptyCodeHash) {
key := common.BytesToHash(account.CodeHash) codeBytes := rawdb.ReadCode(s.ethDB, common.BytesToHash(account.CodeHash))
codeBytes := rawdb.ReadCode(s.ethDB, key)
if len(codeBytes) == 0 { if len(codeBytes) == 0 {
logrus.Error("Code is missing", "account", common.BytesToHash(it.LeafKey())) logrus.Error("Code is missing", "account", common.BytesToHash(it.LeafKey()))
return errors.New("missing code") return errors.New("missing code")
} }
if err := s.ipfsPublisher.PublishCode(codeBytes, tx); err != nil { if err = s.ipfsPublisher.PublishCode(codeBytes, tx); err != nil {
return err return err
} }
} }
if tx, err = s.storageSnapshot(account.Root, stateID, tx); err != nil { if tx, err = s.storageSnapshot(account.Root, stateID, tx); err != nil {
return fmt.Errorf("failed building storage snapshot for account %+v\r\nerror: %v", account, err) return fmt.Errorf("failed building storage snapshot for account %+v\r\nerror: %w", account, err)
} }
case Extension, Branch: case extension, branch:
stateNode.Key = common.BytesToHash([]byte{}) stateNode.key = common.BytesToHash([]byte{})
if _, err := s.ipfsPublisher.PublishStateNode(stateNode, headerID, tx); err != nil { if _, err := s.ipfsPublisher.PublishStateNode(stateNode, headerID, tx); err != nil {
return err return err
} }
@ -254,36 +257,42 @@ func (s *Service) storageSnapshot(sr common.Hash, stateID int64, tx *sqlx.Tx) (*
nodePath := make([]byte, len(it.Path())) nodePath := make([]byte, len(it.Path()))
copy(nodePath, it.Path()) copy(nodePath, it.Path())
node, err := s.stateDB.TrieDB().Node(it.Hash())
var (
nodeData []byte
ty nodeType
)
nodeData, err = s.stateDB.TrieDB().Node(it.Hash())
if err != nil { if err != nil {
return nil, err return nil, err
} }
var nodeElements []interface{} var nodeElements []interface{}
if err := rlp.DecodeBytes(node, &nodeElements); err != nil { if err = rlp.DecodeBytes(nodeData, &nodeElements); err != nil {
return nil, err return nil, err
} }
ty, err := CheckKeyType(nodeElements) ty, err = CheckKeyType(nodeElements)
if err != nil { if err != nil {
return nil, err return nil, err
} }
storageNode := Node{ storageNode := &node{
NodeType: ty, nodeType: ty,
Path: nodePath, path: nodePath,
Value: node, value: nodeData,
} }
switch ty { switch ty {
case Leaf: case leaf:
partialPath := trie.CompactToHex(nodeElements[0].([]byte)) partialPath := trie.CompactToHex(nodeElements[0].([]byte))
valueNodePath := append(nodePath, partialPath...) valueNodePath := append(nodePath, partialPath...)
encodedPath := trie.HexToCompact(valueNodePath) encodedPath := trie.HexToCompact(valueNodePath)
leafKey := encodedPath[1:] leafKey := encodedPath[1:]
storageNode.Key = common.BytesToHash(leafKey) storageNode.key = common.BytesToHash(leafKey)
case Extension, Branch: case extension, branch:
storageNode.Key = common.BytesToHash([]byte{}) storageNode.key = common.BytesToHash([]byte{})
default: default:
return nil, errors.New("unexpected node type") return nil, errors.New("unexpected node type")
} }