diff --git a/cmd/root.go b/cmd/root.go index 1cae6ed..3e664d3 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -37,6 +37,7 @@ var rootCmd = &cobra.Command{ PersistentPreRun: initFuncs, } +// Execute executes root Command. func Execute() { log.Info("----- Starting vDB -----") if err := rootCmd.Execute(); err != nil { diff --git a/cmd/stateSnapshot.go b/cmd/stateSnapshot.go index d16df12..5eef0a9 100644 --- a/cmd/stateSnapshot.go +++ b/cmd/stateSnapshot.go @@ -19,6 +19,7 @@ import ( "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/viper" + "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/snapshot" ) @@ -37,7 +38,7 @@ var stateSnapshotCmd = &cobra.Command{ } func stateSnapshot() { - snapConfig := snapshot.Config{} + snapConfig := &snapshot.Config{} snapConfig.Init() snapshotService, err := snapshot.NewSnapshotService(snapConfig) if err != nil { diff --git a/pkg/snapshot/config.go b/pkg/snapshot/config.go index 1406f3e..20471b5 100644 --- a/pkg/snapshot/config.go +++ b/pkg/snapshot/config.go @@ -16,38 +16,40 @@ package snapshot import ( - "github.com/ethereum/go-ethereum/statediff/indexer/node" + ethNode "github.com/ethereum/go-ethereum/statediff/indexer/node" "github.com/ethereum/go-ethereum/statediff/indexer/postgres" "github.com/spf13/viper" ) const ( - ancientDbPath = "ANCIENT_DB_PATH" + ancientDBPath = "ANCIENT_DB_PATH" ethClientName = "ETH_CLIENT_NAME" ethGenesisBlock = "ETH_GENESIS_BLOCK" - ethNetworkId = "ETH_NETWORK_ID" - ethNodeId = "ETH_NODE_ID" - lvlDbPath = "LVL_DB_PATH" + ethNetworkID = "ETH_NETWORK_ID" + ethNodeID = "ETH_NODE_ID" + lvlDBPath = "LVL_DB_PATH" ) +// Config is config parameters for DB. type Config struct { LevelDBPath string AncientDBPath string - Node node.Info + Node ethNode.Info connectionURI string DBConfig postgres.ConnectionConfig } +// Init Initialises config func (c *Config) Init() { c.dbInit() - viper.BindEnv("leveldb.path", lvlDbPath) - viper.BindEnv("ethereum.nodeID", ethNodeId) + viper.BindEnv("leveldb.path", lvlDBPath) + viper.BindEnv("ethereum.nodeID", ethNodeID) viper.BindEnv("ethereum.clientName", ethClientName) viper.BindEnv("ethereum.genesisBlock", ethGenesisBlock) - viper.BindEnv("ethereum.networkID", ethNetworkId) - viper.BindEnv("leveldb.ancient", ancientDbPath) + viper.BindEnv("ethereum.networkID", ethNetworkID) + viper.BindEnv("leveldb.ancient", ancientDBPath) - c.Node = node.Info{ + c.Node = ethNode.Info{ ID: viper.GetString("ethereum.nodeID"), ClientName: viper.GetString("ethereum.clientName"), GenesisBlock: viper.GetString("ethereum.genesisBlock"), diff --git a/pkg/snapshot/node_type.go b/pkg/snapshot/node_type.go index 6777b69..2ff9374 100644 --- a/pkg/snapshot/node_type.go +++ b/pkg/snapshot/node_type.go @@ -21,43 +21,43 @@ import ( "github.com/ethereum/go-ethereum/common" ) -// Node for holding trie node information -type Node struct { - NodeType NodeType - Path []byte - Key common.Hash - Value []byte +// node for holding trie node information +type node struct { + nodeType nodeType + path []byte + key common.Hash + value []byte } -// NodeType for explicitly setting type of node -type NodeType int +// nodeType for explicitly setting type of node +type nodeType int const ( - Branch NodeType = iota - Extension - Leaf - Removed - Unknown + branch nodeType = iota + extension + leaf + removed + unknown ) // CheckKeyType checks what type of key we have -func CheckKeyType(elements []interface{}) (NodeType, error) { +func CheckKeyType(elements []interface{}) (nodeType, error) { if len(elements) > 2 { - return Branch, nil + return branch, nil } if len(elements) < 2 { - return Unknown, fmt.Errorf("node cannot be less than two elements in length") + return unknown, fmt.Errorf("node cannot be less than two elements in length") } switch elements[0].([]byte)[0] / 16 { case '\x00': - return Extension, nil + return extension, nil case '\x01': - return Extension, nil + return extension, nil case '\x02': - return Leaf, nil + return leaf, nil case '\x03': - return Leaf, nil + return leaf, nil default: - return Unknown, fmt.Errorf("unknown hex prefix") + return unknown, fmt.Errorf("unknown hex prefix") } } diff --git a/pkg/snapshot/publisher.go b/pkg/snapshot/publisher.go index cc81eb1..d0ec420 100644 --- a/pkg/snapshot/publisher.go +++ b/pkg/snapshot/publisher.go @@ -29,11 +29,13 @@ import ( "github.com/ethereum/go-ethereum/statediff/indexer/shared" ) +// Publisher is wrapper around DB. type Publisher struct { db *postgres.DB currBatchSize uint } +// NewPublisher creates Publisher func NewPublisher(db *postgres.DB) *Publisher { return &Publisher{ db: db, @@ -81,11 +83,11 @@ func (p *Publisher) PublishHeader(header *types.Header) (int64, error) { } // PublishStateNode writes the state node to the ipfs backing datastore and adds secondary indexes in the state_cids table -func (p *Publisher) PublishStateNode(node Node, headerID int64, tx *sqlx.Tx) (int64, error) { +func (p *Publisher) PublishStateNode(node *node, headerID int64, tx *sqlx.Tx) (int64, error) { var stateID int64 var stateKey string - if !bytes.Equal(node.Key.Bytes(), nullHash.Bytes()) { - stateKey = node.Key.Hex() + if !bytes.Equal(node.key.Bytes(), nullHash.Bytes()) { + stateKey = node.key.Hex() } defer func() { @@ -95,7 +97,7 @@ func (p *Publisher) PublishStateNode(node Node, headerID int64, tx *sqlx.Tx) (in } }() - stateCIDStr, mhKey, err := shared.PublishRaw(tx, ipld.MEthStateTrie, multihash.KECCAK_256, node.Value) + stateCIDStr, mhKey, err := shared.PublishRaw(tx, ipld.MEthStateTrie, multihash.KECCAK_256, node.value) if err != nil { return 0, err } @@ -103,17 +105,17 @@ func (p *Publisher) PublishStateNode(node Node, headerID int64, tx *sqlx.Tx) (in err = tx.QueryRowx(`INSERT INTO eth.state_cids (header_id, state_leaf_key, cid, state_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (header_id, state_path) DO UPDATE SET (state_leaf_key, cid, node_type, diff, mh_key) = ($2, $3, $5, $6, $7) RETURNING id`, - headerID, stateKey, stateCIDStr, node.Path, node.NodeType, false, mhKey).Scan(&stateID) + headerID, stateKey, stateCIDStr, node.path, node.nodeType, false, mhKey).Scan(&stateID) p.currBatchSize += 2 return stateID, err } // PublishStorageNode writes the storage node to the ipfs backing pg datastore and adds secondary indexes in the storage_cids table -func (p *Publisher) PublishStorageNode(node Node, stateID int64, tx *sqlx.Tx) error { +func (p *Publisher) PublishStorageNode(node *node, stateID int64, tx *sqlx.Tx) error { var storageKey string - if !bytes.Equal(node.Key.Bytes(), nullHash.Bytes()) { - storageKey = node.Key.Hex() + if !bytes.Equal(node.key.Bytes(), nullHash.Bytes()) { + storageKey = node.key.Hex() } defer func() { @@ -123,14 +125,14 @@ func (p *Publisher) PublishStorageNode(node Node, stateID int64, tx *sqlx.Tx) er } }() - storageCIDStr, mhKey, err := shared.PublishRaw(tx, ipld.MEthStorageTrie, multihash.KECCAK_256, node.Value) + storageCIDStr, mhKey, err := shared.PublishRaw(tx, ipld.MEthStorageTrie, multihash.KECCAK_256, node.value) if err != nil { return err } _, err = tx.Exec(`INSERT INTO eth.storage_cids (state_id, storage_leaf_key, cid, storage_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (state_id, storage_path) DO UPDATE SET (storage_leaf_key, cid, node_type, diff, mh_key) = ($2, $3, $5, $6, $7)`, - stateID, storageKey, storageCIDStr, node.Path, node.NodeType, false, mhKey) + stateID, storageKey, storageCIDStr, node.path, node.nodeType, false, mhKey) if err != nil { return err } diff --git a/pkg/snapshot/service.go b/pkg/snapshot/service.go index 6497032..a1dc3e5 100644 --- a/pkg/snapshot/service.go +++ b/pkg/snapshot/service.go @@ -27,11 +27,10 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/statediff/indexer/postgres" "github.com/ethereum/go-ethereum/trie" "github.com/jmoiron/sqlx" "github.com/sirupsen/logrus" - - "github.com/ethereum/go-ethereum/statediff/indexer/postgres" ) var ( @@ -43,6 +42,8 @@ var ( defaultBatchSize = uint(100) ) +// Service holds ethDB and stateDB to read data from lvldb and Publisher +// to publish trie in postgres DB. type Service struct { ethDB ethdb.Database stateDB state.Database @@ -50,7 +51,8 @@ type Service struct { maxBatchSize uint } -func NewSnapshotService(con Config) (*Service, error) { +// NewSnapshotService creates Service. +func NewSnapshotService(con *Config) (*Service, error) { pgDB, err := postgres.NewDB(con.connectionURI, con.DBConfig, con.Node) if err != nil { return nil, err @@ -69,6 +71,7 @@ func NewSnapshotService(con Config) (*Service, error) { }, nil } +// CreateLatestSnapshot creates snapshot for the latest block. func (s *Service) CreateLatestSnapshot() error { // extract header from lvldb and publish to PG-IPFS // hold onto the headerID so that we can link the state nodes to this header @@ -101,6 +104,7 @@ func (s *Service) CreateLatestSnapshot() error { return s.createSnapshot(t.NodeIterator([]byte{}), trieDB, headerID) } +// CreateSnapshot creates snapshot for given block height. func (s *Service) CreateSnapshot(height uint64) error { // extract header from lvldb and publish to PG-IPFS // hold onto the headerID so that we can link the state nodes to this header @@ -153,17 +157,17 @@ func (s *Service) createSnapshot(it trie.NodeIterator, trieDB *trie.Database, he copy(nodePath, it.Path()) var ( - node []byte - ty NodeType + nodeData []byte + ty nodeType ) - node, err = trieDB.Node(it.Hash()) + nodeData, err = trieDB.Node(it.Hash()) if err != nil { return err } var nodeElements []interface{} - if err = rlp.DecodeBytes(node, &nodeElements); err != nil { + if err = rlp.DecodeBytes(nodeData, &nodeElements); err != nil { return err } @@ -172,25 +176,25 @@ func (s *Service) createSnapshot(it trie.NodeIterator, trieDB *trie.Database, he return err } - stateNode := Node{ - NodeType: ty, - Path: nodePath, - Value: node, + stateNode := &node{ + nodeType: ty, + path: nodePath, + value: nodeData, } switch ty { - case Leaf: + case leaf: // if the node is a leaf, decode the account and publish the associated storage trie nodes if there are any var account types.StateAccount - if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil { - return fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", nodePath, err) + if err = rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil { + return fmt.Errorf("error decoding account for leaf node at path %x nerror: %w", nodePath, err) } partialPath := trie.CompactToHex(nodeElements[0].([]byte)) valueNodePath := append(nodePath, partialPath...) encodedPath := trie.HexToCompact(valueNodePath) leafKey := encodedPath[1:] - stateNode.Key = common.BytesToHash(leafKey) + stateNode.key = common.BytesToHash(leafKey) stateID, err := s.ipfsPublisher.PublishStateNode(stateNode, headerID, tx) if err != nil { @@ -199,23 +203,22 @@ func (s *Service) createSnapshot(it trie.NodeIterator, trieDB *trie.Database, he // publish any non-nil code referenced by codehash if !bytes.Equal(account.CodeHash, emptyCodeHash) { - key := common.BytesToHash(account.CodeHash) - codeBytes := rawdb.ReadCode(s.ethDB, key) + codeBytes := rawdb.ReadCode(s.ethDB, common.BytesToHash(account.CodeHash)) if len(codeBytes) == 0 { logrus.Error("Code is missing", "account", common.BytesToHash(it.LeafKey())) return errors.New("missing code") } - if err := s.ipfsPublisher.PublishCode(codeBytes, tx); err != nil { + if err = s.ipfsPublisher.PublishCode(codeBytes, tx); err != nil { return err } } if tx, err = s.storageSnapshot(account.Root, stateID, tx); err != nil { - return fmt.Errorf("failed building storage snapshot for account %+v\r\nerror: %v", account, err) + return fmt.Errorf("failed building storage snapshot for account %+v\r\nerror: %w", account, err) } - case Extension, Branch: - stateNode.Key = common.BytesToHash([]byte{}) + case extension, branch: + stateNode.key = common.BytesToHash([]byte{}) if _, err := s.ipfsPublisher.PublishStateNode(stateNode, headerID, tx); err != nil { return err } @@ -254,36 +257,42 @@ func (s *Service) storageSnapshot(sr common.Hash, stateID int64, tx *sqlx.Tx) (* nodePath := make([]byte, len(it.Path())) copy(nodePath, it.Path()) - node, err := s.stateDB.TrieDB().Node(it.Hash()) + + var ( + nodeData []byte + ty nodeType + ) + + nodeData, err = s.stateDB.TrieDB().Node(it.Hash()) if err != nil { return nil, err } var nodeElements []interface{} - if err := rlp.DecodeBytes(node, &nodeElements); err != nil { + if err = rlp.DecodeBytes(nodeData, &nodeElements); err != nil { return nil, err } - ty, err := CheckKeyType(nodeElements) + ty, err = CheckKeyType(nodeElements) if err != nil { return nil, err } - storageNode := Node{ - NodeType: ty, - Path: nodePath, - Value: node, + storageNode := &node{ + nodeType: ty, + path: nodePath, + value: nodeData, } switch ty { - case Leaf: + case leaf: partialPath := trie.CompactToHex(nodeElements[0].([]byte)) valueNodePath := append(nodePath, partialPath...) encodedPath := trie.HexToCompact(valueNodePath) leafKey := encodedPath[1:] - storageNode.Key = common.BytesToHash(leafKey) - case Extension, Branch: - storageNode.Key = common.BytesToHash([]byte{}) + storageNode.key = common.BytesToHash(leafKey) + case extension, branch: + storageNode.key = common.BytesToHash([]byte{}) default: return nil, errors.New("unexpected node type") }