From 7957c5a92b19dde4ad440cc0720c60373fb3b46c Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Wed, 28 Nov 2018 16:15:19 -0600 Subject: [PATCH 1/6] Add a state diff service --- cmd/geth/usage.go | 6 +++++ cmd/utils/flags.go | 13 +++++++++++ statediff/service.go | 54 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 73 insertions(+) create mode 100644 statediff/service.go diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 82f17e0ee..152ac059b 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -245,6 +245,12 @@ var AppHelpFlagGroups = []flagGroup{ utils.MinerLegacyExtraDataFlag, }, }, + { + Name: "STATE DIFF", + Flags: []cli.Flag{ + utils.StateDiffFlag, + }, + }, { Name: "MISC", }, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index c18d98383..b16f9cdca 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -58,6 +58,7 @@ import ( "github.com/ethereum/go-ethereum/params" whisper "github.com/ethereum/go-ethereum/whisper/whisperv6" "gopkg.in/urfave/cli.v1" + "github.com/ethereum/go-ethereum/statediff" ) var ( @@ -1338,6 +1339,18 @@ func RegisterEthStatsService(stack *node.Node, url string) { } } +func RegisterStateDiffService(stack *node.Node) { + if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) { + var ethServ *eth.Ethereum + ctx.Service(ðServ) + chainDb := ethServ.ChainDb() + blockChain := ethServ.BlockChain() + return statediff.NewStateDiffService(chainDb, blockChain) + }); err != nil { + Fatalf("Failed to register State Diff Service", err) + } +} + func SetupMetrics(ctx *cli.Context) { if metrics.Enabled { log.Info("Enabling metrics collection") diff --git a/statediff/service.go b/statediff/service.go new file mode 100644 index 000000000..695e3d638 --- /dev/null +++ b/statediff/service.go @@ -0,0 +1,54 @@ +package statediff + +import ( + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/ethdb" + "fmt" +) + +type StateDiffService struct { + builder *builder + extractor *extractor + blockchain *core.BlockChain +} + +func NewStateDiffService(db ethdb.Database, blockChain *core.BlockChain) (*StateDiffService, error) { + config := Config{} + extractor, _ := NewExtractor(db, config) + return &StateDiffService{ + blockchain: blockChain, + extractor: extractor, + }, nil +} + +func (StateDiffService) Protocols() []p2p.Protocol { + return []p2p.Protocol{} + +} + +func (StateDiffService) APIs() []rpc.API { + return []rpc.API{} +} + +func (sds *StateDiffService) Start(server *p2p.Server) error { + fmt.Println("starting the state diff service") + blockChannel := make(chan core.ChainHeadEvent) + sds.blockchain.SubscribeChainHeadEvent(blockChannel) + for { + select { + case <-blockChannel: + headOfChainEvent := <-blockChannel + previousBlock := headOfChainEvent.Block + //TODO: figure out the best way to get the previous block + currentBlock := headOfChainEvent.Block + sds.extractor.ExtractStateDiff(*previousBlock, *currentBlock) + } + } + return nil +} +func (StateDiffService) Stop() error { + fmt.Println("stopping the state diff service") + return nil +} \ No newline at end of file From 6fd20597a77fa0a1a5d0fd83d9c6bb82542589bc Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Wed, 28 Nov 2018 16:20:25 -0600 Subject: [PATCH 2/6] Remove diff extractor from blockchain --- core/blockchain.go | 19 ------------------- eth/backend.go | 1 - 2 files changed, 20 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 77295b888..1b48020a6 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -44,7 +44,6 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" "github.com/hashicorp/golang-lru" - "github.com/ethereum/go-ethereum/statediff" ) var ( @@ -73,7 +72,6 @@ type CacheConfig struct { TrieCleanLimit int // Memory allowance (MB) to use for caching trie nodes in memory TrieDirtyLimit int // Memory limit (MB) at which to start flushing dirty trie nodes to disk TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk - StateDiff statediff.Config // Settings for state diff extraction } // BlockChain represents the canonical chain given a database with a genesis @@ -135,8 +133,6 @@ type BlockChain struct { badBlocks *lru.Cache // Bad block cache shouldPreserve func(*types.Block) bool // Function used to determine whether should preserve the given block. - - diffExtractor statediff.Extractor // State diff processing interface } // NewBlockChain returns a fully initialised block chain using information @@ -178,12 +174,6 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par bc.SetProcessor(NewStateProcessor(chainConfig, bc, engine)) var err error - if cacheConfig.StateDiff.On { - bc.diffExtractor, err = statediff.NewExtractor(db, cacheConfig.StateDiff) - if err != nil { - return nil, err - } - } bc.hc, err = NewHeaderChain(db, chainConfig, engine, bc.getProcInterrupt) if err != nil { @@ -1216,15 +1206,6 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty } proctime := time.Since(bstart) - // If extracting statediffs, do so now - if bc.cacheConfig.StateDiff.On { - // Currently not doing anything with returned cid... - _, err = bc.diffExtractor.ExtractStateDiff(*parent, *block) - if err != nil { - return i, events, coalescedLogs, err - } - } - // Write the block to the chain and get the status. status, err := bc.WriteBlockWithState(block, receipts, state) if err != nil { diff --git a/eth/backend.go b/eth/backend.go index d96175643..0de0a1980 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -159,7 +159,6 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { TrieCleanLimit: config.TrieCleanCache, TrieDirtyLimit: config.TrieDirtyCache, TrieTimeLimit: config.TrieTimeout, - StateDiff: config.StateDiff.On, } ) eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, eth.chainConfig, eth.engine, vmConfig, eth.shouldPreserve) From 32a0a238216db3fd988bf42f818cefb9d6fe5c9c Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Wed, 28 Nov 2018 16:51:16 -0600 Subject: [PATCH 3/6] Update imports --- statediff/ipfs/dag_putter.go | 3 ++- statediff/publisher.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/statediff/ipfs/dag_putter.go b/statediff/ipfs/dag_putter.go index 92ffa1e8a..4f99bdae3 100644 --- a/statediff/ipfs/dag_putter.go +++ b/statediff/ipfs/dag_putter.go @@ -22,9 +22,10 @@ package ipfs import ( "bytes" "encoding/gob" - "github.com/i-norden/go-ethereum/statediff" ipld "gx/ipfs/QmWi2BYBL5gJ3CiAiQchg6rn1A8iBsrWy51EYxvHVjFvLb/go-ipld-format" + "github.com/ethereum/go-ethereum/statediff" + "github.com/ethereum/go-ethereum/common" ) const ( diff --git a/statediff/publisher.go b/statediff/publisher.go index 163f8d3b0..48ad7cf64 100644 --- a/statediff/publisher.go +++ b/statediff/publisher.go @@ -21,7 +21,7 @@ package statediff import ( "errors" - "github.com/i-norden/go-ethereum/statediff/ipfs" + "github.com/ethereum/go-ethereum/statediff/ipfs" ) type Publisher interface { From 70d531dc1ef8418d1f03341891fbd06d20a7a93a Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Thu, 29 Nov 2018 11:05:00 -0600 Subject: [PATCH 4/6] Move statediff on/off check to geth cmd config --- cmd/geth/config.go | 4 ++++ cmd/utils/flags.go | 11 ----------- eth/config.go | 4 ---- 3 files changed, 4 insertions(+), 15 deletions(-) diff --git a/cmd/geth/config.go b/cmd/geth/config.go index b0749d232..a05e78b11 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -178,6 +178,10 @@ func makeFullNode(ctx *cli.Context) *node.Node { if cfg.Ethstats.URL != "" { utils.RegisterEthStatsService(stack, cfg.Ethstats.URL) } + + if ctx.GlobalBool(utils.StateDiffFlag.Name) { + utils.RegisterStateDiffService(stack) + } return stack } diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index b16f9cdca..54e3dd83d 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1134,13 +1134,6 @@ func SetShhConfig(ctx *cli.Context, stack *node.Node, cfg *whisper.Config) { } } -// Check if state diff flags are on and applies them to eth context -func setStateDiff(ctx *cli.Context, cfg *eth.Config) { - if ctx.GlobalBool(StateDiffFlag.Name) && cfg.NoPruning && cfg.SyncMode == downloader.FullSync { - cfg.StateDiff.On = true - } -} - // SetEthConfig applies eth-related command line flags to the config. func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { // Avoid conflicting network flags @@ -1176,10 +1169,6 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { } cfg.NoPruning = ctx.GlobalString(GCModeFlag.Name) == "archive" - if ctx.GlobalIsSet(StateDiffFlag.Name) { - setStateDiff(ctx, cfg) - } - if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheTrieFlag.Name) { cfg.TrieCleanCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheTrieFlag.Name) / 100 } diff --git a/eth/config.go b/eth/config.go index c675b6d08..601f4735e 100644 --- a/eth/config.go +++ b/eth/config.go @@ -17,7 +17,6 @@ package eth import ( - "github.com/ethereum/go-ethereum/statediff" "math/big" "os" "os/user" @@ -129,9 +128,6 @@ type Config struct { EWASMInterpreter string // Type of the EVM interpreter ("" for default) EVMInterpreter string - - // Config for state diff building - StateDiff statediff.Config } type configMarshaling struct { From 71147861f70b808669f453550ee5bdf44db0610d Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Thu, 29 Nov 2018 13:52:02 -0600 Subject: [PATCH 5/6] Update starting state diff service --- statediff/service.go | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/statediff/service.go b/statediff/service.go index 695e3d638..6870ce047 100644 --- a/statediff/service.go +++ b/statediff/service.go @@ -1,11 +1,12 @@ package statediff import ( - "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/ethdb" - "fmt" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/event" + "log" ) type StateDiffService struct { @@ -19,7 +20,7 @@ func NewStateDiffService(db ethdb.Database, blockChain *core.BlockChain) (*State extractor, _ := NewExtractor(db, config) return &StateDiffService{ blockchain: blockChain, - extractor: extractor, + extractor: extractor, }, nil } @@ -32,23 +33,32 @@ func (StateDiffService) APIs() []rpc.API { return []rpc.API{} } -func (sds *StateDiffService) Start(server *p2p.Server) error { - fmt.Println("starting the state diff service") - blockChannel := make(chan core.ChainHeadEvent) - sds.blockchain.SubscribeChainHeadEvent(blockChannel) +func (sds *StateDiffService) loop (sub event.Subscription, events chan core.ChainHeadEvent) { + defer sub.Unsubscribe() + for { select { - case <-blockChannel: - headOfChainEvent := <-blockChannel - previousBlock := headOfChainEvent.Block + case ev, ok := <-events: + if !ok { + log.Fatalf("Error getting chain head event from subscription.") + } + log.Println("doing something with an event", ev) + previousBlock := ev.Block //TODO: figure out the best way to get the previous block - currentBlock := headOfChainEvent.Block + currentBlock := ev.Block sds.extractor.ExtractStateDiff(*previousBlock, *currentBlock) } } + +} +func (sds *StateDiffService) Start(server *p2p.Server) error { + events := make(chan core.ChainHeadEvent, 10) + sub := sds.blockchain.SubscribeChainHeadEvent(events) + + go sds.loop(sub, events) + return nil } func (StateDiffService) Stop() error { - fmt.Println("stopping the state diff service") return nil -} \ No newline at end of file +} From cf4ed9c2b378904a24cb42fe90d6af4e99a0aa66 Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Thu, 29 Nov 2018 14:29:24 -0600 Subject: [PATCH 6/6] Add debugging logs for creating diff --- statediff/builder.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/statediff/builder.go b/statediff/builder.go index 433353591..edfb50ca8 100644 --- a/statediff/builder.go +++ b/statediff/builder.go @@ -49,10 +49,13 @@ func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, block // Generate tries for old and new states oldTrie, err := trie.New(oldStateRoot, sdb.trieDB) if err != nil { + log.Debug("error creating oldTrie", err) + //getting this error: error creating oldTrie missing trie node ddfbb83966d870891aa47147269447a83564d1defaefad5f9844a3a3a2a08433 (path ) return nil, err } newTrie, err := trie.New(newStateRoot, sdb.trieDB) if err != nil { + log.Debug("error creating newTrie", err) return nil, err } @@ -61,6 +64,7 @@ func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, block newIt := newTrie.NodeIterator([]byte{}) creations, err := sdb.collectDiffNodes(oldIt, newIt) if err != nil { + log.Debug("error collecting creation diff nodes", err) return nil, err } @@ -69,6 +73,7 @@ func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, block newIt = newTrie.NodeIterator(make([]byte, 0)) deletions, err := sdb.collectDiffNodes(newIt, oldIt) if err != nil { + log.Debug("error collecting deletion diff nodes", err) return nil, err } @@ -80,14 +85,17 @@ func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, block // Build and return the statediff updatedAccounts, err := sdb.buildDiffIncremental(creations, deletions, updatedKeys) if err != nil { + log.Debug("error building diff incremental for updated", err) return nil, err } createdAccounts, err := sdb.buildDiffEventual(creations, true) if err != nil { + log.Debug("error building diff incremental for created", err) return nil, err } deletedAccounts, err := sdb.buildDiffEventual(deletions, false) if err != nil { + log.Debug("error building diff incremental for deleted", err) return nil, err }