cerc-io/ipld-eth-state-snapshot
16
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

Fix linting errors.

Browse Source
This commit is contained in:
Arijit Das 2021-12-14 12:20:19 +05:30
parent 8b4ebfa355
commit e4de336833
6 changed files with 90 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cmd/root.go
View File

@ -37,6 +37,7 @@ var rootCmd = &cobra.Command{
PersistentPreRun: initFuncs,
}
// Execute executes root Command.
func Execute() {
log.Info("----- Starting vDB -----")
if err := rootCmd.Execute(); err != nil {

3
cmd/stateSnapshot.go
View File

@ -19,6 +19,7 @@ import (
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/snapshot"
)
@ -37,7 +38,7 @@ var stateSnapshotCmd = &cobra.Command{
}
func stateSnapshot() {
snapConfig := snapshot.Config{}
snapConfig := &snapshot.Config{}
snapConfig.Init()
snapshotService, err := snapshot.NewSnapshotService(snapConfig)
if err != nil {

24
pkg/snapshot/config.go
View File

@ -16,38 +16,40 @@
package snapshot
import (
"github.com/ethereum/go-ethereum/statediff/indexer/node"
ethNode "github.com/ethereum/go-ethereum/statediff/indexer/node"
"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
"github.com/spf13/viper"
)
const (
ancientDbPath = "ANCIENT_DB_PATH"
ancientDBPath = "ANCIENT_DB_PATH"
ethClientName = "ETH_CLIENT_NAME"
ethGenesisBlock = "ETH_GENESIS_BLOCK"
ethNetworkId = "ETH_NETWORK_ID"
ethNodeId = "ETH_NODE_ID"
lvlDbPath = "LVL_DB_PATH"
ethNetworkID = "ETH_NETWORK_ID"
ethNodeID = "ETH_NODE_ID"
lvlDBPath = "LVL_DB_PATH"
)
// Config is config parameters for DB.
type Config struct {
LevelDBPath string
AncientDBPath string
Node node.Info
Node ethNode.Info
connectionURI string
DBConfig postgres.ConnectionConfig
}
// Init Initialises config
func (c *Config) Init() {
c.dbInit()
viper.BindEnv("leveldb.path", lvlDbPath)
viper.BindEnv("ethereum.nodeID", ethNodeId)
viper.BindEnv("leveldb.path", lvlDBPath)
viper.BindEnv("ethereum.nodeID", ethNodeID)
viper.BindEnv("ethereum.clientName", ethClientName)
viper.BindEnv("ethereum.genesisBlock", ethGenesisBlock)
viper.BindEnv("ethereum.networkID", ethNetworkId)
viper.BindEnv("leveldb.ancient", ancientDbPath)
viper.BindEnv("ethereum.networkID", ethNetworkID)
viper.BindEnv("leveldb.ancient", ancientDBPath)
c.Node = node.Info{
c.Node = ethNode.Info{
ID: viper.GetString("ethereum.nodeID"),
ClientName: viper.GetString("ethereum.clientName"),
GenesisBlock: viper.GetString("ethereum.genesisBlock"),

42
pkg/snapshot/node_type.go
View File

@ -21,43 +21,43 @@ import (
"github.com/ethereum/go-ethereum/common"
)
// Node for holding trie node information
type Node struct {
NodeType NodeType
Path []byte
Key common.Hash
Value []byte
// node for holding trie node information
type node struct {
nodeType nodeType
path []byte
key common.Hash
value []byte
}
// NodeType for explicitly setting type of node
type NodeType int
// nodeType for explicitly setting type of node
type nodeType int
const (
Branch NodeType = iota
Extension
Leaf
Removed
Unknown
branch nodeType = iota
extension
leaf
removed
unknown
)
// CheckKeyType checks what type of key we have
func CheckKeyType(elements []interface{}) (NodeType, error) {
func CheckKeyType(elements []interface{}) (nodeType, error) {
if len(elements) > 2 {
return Branch, nil
return branch, nil
}
if len(elements) < 2 {
return Unknown, fmt.Errorf("node cannot be less than two elements in length")
return unknown, fmt.Errorf("node cannot be less than two elements in length")
}
switch elements[0].([]byte)[0] / 16 {
case '\x00':
return Extension, nil
return extension, nil
case '\x01':
return Extension, nil
return extension, nil
case '\x02':
return Leaf, nil
return leaf, nil
case '\x03':
return Leaf, nil
return leaf, nil
default:
return Unknown, fmt.Errorf("unknown hex prefix")
return unknown, fmt.Errorf("unknown hex prefix")
}
}

22
pkg/snapshot/publisher.go
View File

@ -29,11 +29,13 @@ import (
"github.com/ethereum/go-ethereum/statediff/indexer/shared"
)
// Publisher is wrapper around DB.
type Publisher struct {
db *postgres.DB
currBatchSize uint
}
// NewPublisher creates Publisher
func NewPublisher(db *postgres.DB) *Publisher {
return &Publisher{
db: db,
@ -81,11 +83,11 @@ func (p *Publisher) PublishHeader(header *types.Header) (int64, error) {
}
// PublishStateNode writes the state node to the ipfs backing datastore and adds secondary indexes in the state_cids table
func (p *Publisher) PublishStateNode(node Node, headerID int64, tx *sqlx.Tx) (int64, error) {
func (p *Publisher) PublishStateNode(node *node, headerID int64, tx *sqlx.Tx) (int64, error) {
var stateID int64
var stateKey string
if !bytes.Equal(node.Key.Bytes(), nullHash.Bytes()) {
stateKey = node.Key.Hex()
if !bytes.Equal(node.key.Bytes(), nullHash.Bytes()) {
stateKey = node.key.Hex()
}
defer func() {
@ -95,7 +97,7 @@ func (p *Publisher) PublishStateNode(node Node, headerID int64, tx *sqlx.Tx) (in
}
}()
stateCIDStr, mhKey, err := shared.PublishRaw(tx, ipld.MEthStateTrie, multihash.KECCAK_256, node.Value)
stateCIDStr, mhKey, err := shared.PublishRaw(tx, ipld.MEthStateTrie, multihash.KECCAK_256, node.value)
if err != nil {
return 0, err
}
@ -103,17 +105,17 @@ func (p *Publisher) PublishStateNode(node Node, headerID int64, tx *sqlx.Tx) (in
err = tx.QueryRowx(`INSERT INTO eth.state_cids (header_id, state_leaf_key, cid, state_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7)
ON CONFLICT (header_id, state_path) DO UPDATE SET (state_leaf_key, cid, node_type, diff, mh_key) = ($2, $3, $5, $6, $7)
RETURNING id`,
headerID, stateKey, stateCIDStr, node.Path, node.NodeType, false, mhKey).Scan(&stateID)
headerID, stateKey, stateCIDStr, node.path, node.nodeType, false, mhKey).Scan(&stateID)
p.currBatchSize += 2
return stateID, err
}
// PublishStorageNode writes the storage node to the ipfs backing pg datastore and adds secondary indexes in the storage_cids table
func (p *Publisher) PublishStorageNode(node Node, stateID int64, tx *sqlx.Tx) error {
func (p *Publisher) PublishStorageNode(node *node, stateID int64, tx *sqlx.Tx) error {
var storageKey string
if !bytes.Equal(node.Key.Bytes(), nullHash.Bytes()) {
storageKey = node.Key.Hex()
if !bytes.Equal(node.key.Bytes(), nullHash.Bytes()) {
storageKey = node.key.Hex()
}
defer func() {
@ -123,14 +125,14 @@ func (p *Publisher) PublishStorageNode(node Node, stateID int64, tx *sqlx.Tx) er
}
}()
storageCIDStr, mhKey, err := shared.PublishRaw(tx, ipld.MEthStorageTrie, multihash.KECCAK_256, node.Value)
storageCIDStr, mhKey, err := shared.PublishRaw(tx, ipld.MEthStorageTrie, multihash.KECCAK_256, node.value)
if err != nil {
return err
}
_, err = tx.Exec(`INSERT INTO eth.storage_cids (state_id, storage_leaf_key, cid, storage_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7)
ON CONFLICT (state_id, storage_path) DO UPDATE SET (storage_leaf_key, cid, node_type, diff, mh_key) = ($2, $3, $5, $6, $7)`,
stateID, storageKey, storageCIDStr, node.Path, node.NodeType, false, mhKey)
stateID, storageKey, storageCIDStr, node.path, node.nodeType, false, mhKey)
if err != nil {
return err
}

73
pkg/snapshot/service.go
View File

@ -27,11 +27,10 @@ import (
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
"github.com/ethereum/go-ethereum/trie"
"github.com/jmoiron/sqlx"
"github.com/sirupsen/logrus"
"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
)
var (
@ -43,6 +42,8 @@ var (
defaultBatchSize = uint(100)
)
// Service holds ethDB and stateDB to read data from lvldb and Publisher
// to publish trie in postgres DB.
type Service struct {
ethDB ethdb.Database
stateDB state.Database
@ -50,7 +51,8 @@ type Service struct {
maxBatchSize uint
}
func NewSnapshotService(con Config) (*Service, error) {
// NewSnapshotService creates Service.
func NewSnapshotService(con *Config) (*Service, error) {
pgDB, err := postgres.NewDB(con.connectionURI, con.DBConfig, con.Node)
if err != nil {
return nil, err
@ -69,6 +71,7 @@ func NewSnapshotService(con Config) (*Service, error) {
}, nil
}
// CreateLatestSnapshot creates snapshot for the latest block.
func (s *Service) CreateLatestSnapshot() error {
// extract header from lvldb and publish to PG-IPFS
// hold onto the headerID so that we can link the state nodes to this header
@ -101,6 +104,7 @@ func (s *Service) CreateLatestSnapshot() error {
return s.createSnapshot(t.NodeIterator([]byte{}), trieDB, headerID)
}
// CreateSnapshot creates snapshot for given block height.
func (s *Service) CreateSnapshot(height uint64) error {
// extract header from lvldb and publish to PG-IPFS
// hold onto the headerID so that we can link the state nodes to this header
@ -153,17 +157,17 @@ func (s *Service) createSnapshot(it trie.NodeIterator, trieDB *trie.Database, he
copy(nodePath, it.Path())
var (
node []byte
ty NodeType
nodeData []byte
ty nodeType
)
node, err = trieDB.Node(it.Hash())
nodeData, err = trieDB.Node(it.Hash())
if err != nil {
return err
}
var nodeElements []interface{}
if err = rlp.DecodeBytes(node, &nodeElements); err != nil {
if err = rlp.DecodeBytes(nodeData, &nodeElements); err != nil {
return err
}
@ -172,25 +176,25 @@ func (s *Service) createSnapshot(it trie.NodeIterator, trieDB *trie.Database, he
return err
}
stateNode := Node{
NodeType: ty,
Path: nodePath,
Value: node,
stateNode := &node{
nodeType: ty,
path: nodePath,
value: nodeData,
}
switch ty {
case Leaf:
case leaf:
// if the node is a leaf, decode the account and publish the associated storage trie nodes if there are any
var account types.StateAccount
if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil {
return fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", nodePath, err)
if err = rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil {
return fmt.Errorf("error decoding account for leaf node at path %x nerror: %w", nodePath, err)
}
partialPath := trie.CompactToHex(nodeElements[0].([]byte))
valueNodePath := append(nodePath, partialPath...)
encodedPath := trie.HexToCompact(valueNodePath)
leafKey := encodedPath[1:]
stateNode.Key = common.BytesToHash(leafKey)
stateNode.key = common.BytesToHash(leafKey)
stateID, err := s.ipfsPublisher.PublishStateNode(stateNode, headerID, tx)
if err != nil {
@ -199,23 +203,22 @@ func (s *Service) createSnapshot(it trie.NodeIterator, trieDB *trie.Database, he
// publish any non-nil code referenced by codehash
if !bytes.Equal(account.CodeHash, emptyCodeHash) {
key := common.BytesToHash(account.CodeHash)
codeBytes := rawdb.ReadCode(s.ethDB, key)
codeBytes := rawdb.ReadCode(s.ethDB, common.BytesToHash(account.CodeHash))
if len(codeBytes) == 0 {
logrus.Error("Code is missing", "account", common.BytesToHash(it.LeafKey()))
return errors.New("missing code")
}
if err := s.ipfsPublisher.PublishCode(codeBytes, tx); err != nil {
if err = s.ipfsPublisher.PublishCode(codeBytes, tx); err != nil {
return err
}
}
if tx, err = s.storageSnapshot(account.Root, stateID, tx); err != nil {
return fmt.Errorf("failed building storage snapshot for account %+v\r\nerror: %v", account, err)
return fmt.Errorf("failed building storage snapshot for account %+v\r\nerror: %w", account, err)
}
case Extension, Branch:
stateNode.Key = common.BytesToHash([]byte{})
case extension, branch:
stateNode.key = common.BytesToHash([]byte{})
if _, err := s.ipfsPublisher.PublishStateNode(stateNode, headerID, tx); err != nil {
return err
}
@ -254,36 +257,42 @@ func (s *Service) storageSnapshot(sr common.Hash, stateID int64, tx *sqlx.Tx) (*
nodePath := make([]byte, len(it.Path()))
copy(nodePath, it.Path())
node, err := s.stateDB.TrieDB().Node(it.Hash())
var (
nodeData []byte
ty nodeType
)
nodeData, err = s.stateDB.TrieDB().Node(it.Hash())
if err != nil {
return nil, err
}
var nodeElements []interface{}
if err := rlp.DecodeBytes(node, &nodeElements); err != nil {
if err = rlp.DecodeBytes(nodeData, &nodeElements); err != nil {
return nil, err
}
ty, err := CheckKeyType(nodeElements)
ty, err = CheckKeyType(nodeElements)
if err != nil {
return nil, err
}
storageNode := Node{
NodeType: ty,
Path: nodePath,
Value: node,
storageNode := &node{
nodeType: ty,
path: nodePath,
value: nodeData,
}
switch ty {
case Leaf:
case leaf:
partialPath := trie.CompactToHex(nodeElements[0].([]byte))
valueNodePath := append(nodePath, partialPath...)
encodedPath := trie.HexToCompact(valueNodePath)
leafKey := encodedPath[1:]
storageNode.Key = common.BytesToHash(leafKey)
case Extension, Branch:
storageNode.Key = common.BytesToHash([]byte{})
storageNode.key = common.BytesToHash(leafKey)
case extension, branch:
storageNode.key = common.BytesToHash([]byte{})
default:
return nil, errors.New("unexpected node type")
}