cerc-io/ipld-eth-state-snapshot
16
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

number of workers instead of divide depth

Browse Source 
update w/ iterator change
This commit is contained in:
Roy Crihfield 2020-08-31 11:03:37 -05:00
parent 96ad583b06
commit 169887f39f
2 changed files with 9 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
cmd/stateSnapshot.go
View File

@ -44,8 +44,8 @@ func stateSnapshot() {
logWithCommand.Fatal(err) logWithCommand.Fatal(err)
} }
height := viper.GetInt64("snapshot.blockHeight") height := viper.GetInt64("snapshot.blockHeight")
divideDepth := viper.GetInt("snapshot.divideDepth") workers := viper.GetUint("snapshot.workers")
params := snapshot.SnapshotParams{DivideDepth: divideDepth} params := snapshot.SnapshotParams{Workers: workers}
if height < 0 { if height < 0 {
if err := snapshotService.CreateLatestSnapshot(params); err != nil { if err := snapshotService.CreateLatestSnapshot(params); err != nil {
logWithCommand.Fatal(err) logWithCommand.Fatal(err)
@ -65,10 +65,10 @@ func init() {
stateSnapshotCmd.PersistentFlags().String("leveldb-path", "", "path to primary datastore") stateSnapshotCmd.PersistentFlags().String("leveldb-path", "", "path to primary datastore")
stateSnapshotCmd.PersistentFlags().String("ancient-path", "", "path to ancient datastore") stateSnapshotCmd.PersistentFlags().String("ancient-path", "", "path to ancient datastore")
stateSnapshotCmd.PersistentFlags().String("block-height", "", "blockheight to extract state at") stateSnapshotCmd.PersistentFlags().String("block-height", "", "blockheight to extract state at")
stateSnapshotCmd.PersistentFlags().Int("divide-depth", 0, "trie depth to divide concurrent work at") stateSnapshotCmd.PersistentFlags().Int("workers", 0, "number of concurrent workers to use")
viper.BindPFlag("leveldb.path", stateSnapshotCmd.PersistentFlags().Lookup("leveldb-path")) viper.BindPFlag("leveldb.path", stateSnapshotCmd.PersistentFlags().Lookup("leveldb-path"))
viper.BindPFlag("leveldb.ancient", stateSnapshotCmd.PersistentFlags().Lookup("ancient-path")) viper.BindPFlag("leveldb.ancient", stateSnapshotCmd.PersistentFlags().Lookup("ancient-path"))
viper.BindPFlag("snapshot.blockHeight", stateSnapshotCmd.PersistentFlags().Lookup("block-height")) viper.BindPFlag("snapshot.blockHeight", stateSnapshotCmd.PersistentFlags().Lookup("block-height"))
viper.BindPFlag("snapshot.divideDepth", stateSnapshotCmd.PersistentFlags().Lookup("divide-depth")) viper.BindPFlag("snapshot.workers", stateSnapshotCmd.PersistentFlags().Lookup("workers"))
} }

11
pkg/snapshot/service.go
View File

@ -39,7 +39,6 @@ var (
emptyNode, _ = rlp.EncodeToBytes([]byte{}) emptyNode, _ = rlp.EncodeToBytes([]byte{})
emptyCodeHash = crypto.Keccak256([]byte{}) emptyCodeHash = crypto.Keccak256([]byte{})
emptyContractRoot = crypto.Keccak256Hash(emptyNode) emptyContractRoot = crypto.Keccak256Hash(emptyNode)
spawnDepth = 2
) )
type Service struct { type Service struct {
@ -67,7 +66,7 @@ func NewSnapshotService(con ServiceConfig) (*Service, error) {
type SnapshotParams struct { type SnapshotParams struct {
Height uint64 Height uint64
DivideDepth int Workers uint
} }
func (s *Service) CreateSnapshot(params SnapshotParams) error { func (s *Service) CreateSnapshot(params SnapshotParams) error {
@ -88,8 +87,8 @@ func (s *Service) CreateSnapshot(params SnapshotParams) error {
if err != nil { if err != nil {
return err return err
} }
if params.DivideDepth > 0 { if params.Workers > 0 {
return s.createSnapshotAsync(t, headerID, params.DivideDepth) return s.createSnapshotAsync(t, headerID, params.Workers)
} else { } else {
return s.createSnapshot(t.NodeIterator(nil), headerID) return s.createSnapshot(t.NodeIterator(nil), headerID)
} }
@ -200,12 +199,12 @@ func (s *Service) createSnapshot(it iter.NodeIterator, headerID int64) error {
} }
// Full-trie snapshot using goroutines // Full-trie snapshot using goroutines
func (s *Service) createSnapshotAsync(tree state.Trie, headerID int64, depth int) error { func (s *Service) createSnapshotAsync(tree state.Trie, headerID int64, workers uint) error {
errors := make(chan error) errors := make(chan error)
finished := make(chan bool) finished := make(chan bool)
var wg sync.WaitGroup var wg sync.WaitGroup
iter.VisitSubtries(tree, depth, func (it iter.NodeIterator) { iter.VisitSubtries(tree, workers, func (it iter.NodeIterator) {
wg.Add(1) wg.Add(1)
go func() { go func() {
defer wg.Done() defer wg.Done()