make proofs and paths optional + compress service loop into single for loop (may be missing something here)

This commit is contained in:
Ian Norden 2019-05-13 10:31:42 -05:00
parent 71b41b5c77
commit 5c62d0bcb1
10 changed files with 235 additions and 145 deletions

View File

@ -147,6 +147,8 @@ var (
utils.EWASMInterpreterFlag,
utils.EVMInterpreterFlag,
utils.StateDiffFlag,
utils.StateDiffPathsAndProofs,
utils.StateDiffLeafNodesOnly,
configFileFlag,
}

View File

@ -255,6 +255,8 @@ var AppHelpFlagGroups = []flagGroup{
Name: "STATE DIFF",
Flags: []cli.Flag{
utils.StateDiffFlag,
utils.StateDiffPathsAndProofs,
utils.StateDiffLeafNodesOnly,
},
},
{

View File

@ -761,6 +761,14 @@ var (
Name: "statediff",
Usage: "Enables the calculation of state diffs between each block, persists these state diffs the configured persistence mode.",
}
StateDiffPathsAndProofs = cli.BoolFlag{
Name: "statediff.pathsandproofs",
Usage: "Path and proof sets for the state and storage nodes are generated",
}
StateDiffLeafNodesOnly = cli.BoolFlag{
Name: "statediff.leafs",
Usage: "Consider only leaf nodes of the storage and state tries",
}
)
// MakeDataDir retrieves the currently requested data directory, terminating
@ -1634,12 +1642,16 @@ func RegisterGraphQLService(stack *node.Node, endpoint string, cors, vhosts []st
// RegisterStateDiffService configures and registers a service to stream state diff data over RPC
func RegisterStateDiffService(stack *node.Node, ctx *cli.Context) {
config := statediff.Config{
PathsAndProofs: ctx.GlobalBool(StateDiffPathsAndProofs.Name),
LeafsOnly: ctx.GlobalBool(StateDiffLeafNodesOnly.Name),
}
if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) {
var ethServ *eth.Ethereum
ctx.Service(&ethServ)
chainDb := ethServ.ChainDb()
blockChain := ethServ.BlockChain()
return statediff.NewStateDiffService(chainDb, blockChain)
return statediff.NewStateDiffService(chainDb, blockChain, config)
}); err != nil {
Fatalf("Failed to register State Diff Service", err)
}

View File

@ -20,6 +20,7 @@
package statediff
import (
"fmt"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/core"
@ -37,13 +38,15 @@ type Builder interface {
type builder struct {
chainDB ethdb.Database
config Config
blockChain *core.BlockChain
}
// NewBuilder is used to create a builder
func NewBuilder(db ethdb.Database, blockChain *core.BlockChain) Builder {
// NewBuilder is used to create a state diff builder
func NewBuilder(db ethdb.Database, blockChain *core.BlockChain, config Config) Builder {
return &builder{
chainDB: db,
config: config,
blockChain: blockChain,
}
}
@ -54,13 +57,11 @@ func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, block
stateCache := sdb.blockChain.StateCache()
oldTrie, err := stateCache.OpenTrie(oldStateRoot)
if err != nil {
log.Error("Error creating trie for oldStateRoot", "error", err)
return StateDiff{}, err
return StateDiff{}, fmt.Errorf("error creating trie for oldStateRoot: %v", err)
}
newTrie, err := stateCache.OpenTrie(newStateRoot)
if err != nil {
log.Error("Error creating trie for newStateRoot", "error", err)
return StateDiff{}, err
return StateDiff{}, fmt.Errorf("error creating trie for newStateRoot: %v", err)
}
// Find created accounts
@ -68,8 +69,7 @@ func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, block
newIt := newTrie.NodeIterator([]byte{})
creations, err := sdb.collectDiffNodes(oldIt, newIt)
if err != nil {
log.Error("Error collecting creation diff nodes", "error", err)
return StateDiff{}, err
return StateDiff{}, fmt.Errorf("error collecting creation diff nodes: %v", err)
}
// Find deleted accounts
@ -77,8 +77,7 @@ func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, block
newIt = newTrie.NodeIterator([]byte{})
deletions, err := sdb.collectDiffNodes(newIt, oldIt)
if err != nil {
log.Error("Error collecting deletion diff nodes", "error", err)
return StateDiff{}, err
return StateDiff{}, fmt.Errorf("error collecting deletion diff nodes: %v", err)
}
// Find all the diffed keys
@ -89,18 +88,15 @@ func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, block
// Build and return the statediff
updatedAccounts, err := sdb.buildDiffIncremental(creations, deletions, updatedKeys)
if err != nil {
log.Error("Error building diff for updated accounts", "error", err)
return StateDiff{}, err
return StateDiff{}, fmt.Errorf("error building diff for updated accounts: %v", err)
}
createdAccounts, err := sdb.buildDiffEventual(creations)
if err != nil {
log.Error("Error building diff for created accounts", "error", err)
return StateDiff{}, err
return StateDiff{}, fmt.Errorf("error building diff for created accounts: %v", err)
}
deletedAccounts, err := sdb.buildDiffEventual(deletions)
if err != nil {
log.Error("Error building diff for deleted accounts", "error", err)
return StateDiff{}, err
return StateDiff{}, fmt.Errorf("error building diff for deleted accounts: %v", err)
}
return StateDiff{
@ -116,38 +112,69 @@ func (sdb *builder) collectDiffNodes(a, b trie.NodeIterator) (AccountsMap, error
var diffAccounts = make(AccountsMap)
it, _ := trie.NewDifferenceIterator(a, b)
for {
log.Debug("Current Path and Hash", "path", pathToStr(it), "hashold", it.Hash())
if it.Leaf() {
leafProof := make([][]byte, len(it.LeafProof()))
copy(leafProof, it.LeafProof())
leafPath := make([]byte, len(it.Path()))
copy(leafPath, it.Path())
leafKey := make([]byte, len(it.LeafKey()))
copy(leafKey, it.LeafKey())
leafKeyHash := common.BytesToHash(leafKey)
leafValue := make([]byte, len(it.LeafBlob()))
copy(leafValue, it.LeafBlob())
// lookup account state
var account state.Account
if err := rlp.DecodeBytes(leafValue, &account); err != nil {
log.Error("Error looking up account via address", "address", leafKeyHash, "error", err)
return nil, err
if sdb.config.PathsAndProofs {
for {
log.Debug("Current Path and Hash", "path", pathToStr(it), "hashold", it.Hash())
if it.Leaf() {
leafProof := make([][]byte, len(it.LeafProof()))
copy(leafProof, it.LeafProof())
leafPath := make([]byte, len(it.Path()))
copy(leafPath, it.Path())
leafKey := make([]byte, len(it.LeafKey()))
copy(leafKey, it.LeafKey())
leafKeyHash := common.BytesToHash(leafKey)
leafValue := make([]byte, len(it.LeafBlob()))
copy(leafValue, it.LeafBlob())
// lookup account state
var account state.Account
if err := rlp.DecodeBytes(leafValue, &account); err != nil {
return nil, fmt.Errorf("error looking up account via address: %s, error: %v", leafKeyHash.Hex(), err)
}
aw := accountWrapper{
Account: account,
RawKey: leafKey,
RawValue: leafValue,
Proof: leafProof,
Path: leafPath,
}
// record account to diffs (creation if we are looking at new - old; deletion if old - new)
log.Debug("Account lookup successful", "address", leafKeyHash, "account", account)
diffAccounts[leafKeyHash] = aw
}
aw := accountWrapper{
Account: account,
RawKey: leafKey,
RawValue: leafValue,
Proof: leafProof,
Path: leafPath,
cont := it.Next(true)
if !cont {
break
}
// record account to diffs (creation if we are looking at new - old; deletion if old - new)
log.Debug("Account lookup successful", "address", leafKeyHash, "account", account)
diffAccounts[leafKeyHash] = aw
}
cont := it.Next(true)
if !cont {
break
} else {
for {
log.Debug("Current Path and Hash", "path", pathToStr(it), "old hash", it.Hash())
if it.Leaf() {
leafKey := make([]byte, len(it.LeafKey()))
copy(leafKey, it.LeafKey())
leafKeyHash := common.BytesToHash(leafKey)
leafValue := make([]byte, len(it.LeafBlob()))
copy(leafValue, it.LeafBlob())
// lookup account state
var account state.Account
if err := rlp.DecodeBytes(leafValue, &account); err != nil {
return nil, fmt.Errorf("error looking up account via address: %s, error: %v", leafKeyHash.Hex(), err)
}
aw := accountWrapper{
Account: account,
RawKey: leafKey,
RawValue: leafValue,
Proof: nil,
Path: nil,
}
// record account to diffs (creation if we are looking at new - old; deletion if old - new)
log.Debug("Account lookup successful", "address", leafKeyHash, "account", account)
diffAccounts[leafKeyHash] = aw
}
cont := it.Next(true)
if !cont {
break
}
}
}
@ -159,8 +186,7 @@ func (sdb *builder) buildDiffEventual(accounts AccountsMap) (AccountDiffsMap, er
for _, val := range accounts {
storageDiffs, err := sdb.buildStorageDiffsEventual(val.Account.Root)
if err != nil {
log.Error("Failed building eventual storage diffs", "Address", common.BytesToHash(val.RawKey), "error", err)
return nil, err
return nil, fmt.Errorf("failed building eventual storage diffs for address: %s, error: %v", common.BytesToHash(val.RawKey), err)
}
accountDiffs[common.BytesToHash(val.RawKey)] = AccountDiff{
Key: val.RawKey,
@ -209,7 +235,7 @@ func (sdb *builder) buildStorageDiffsEventual(sr common.Hash) ([]StorageDiff, er
return nil, err
}
it := sTrie.NodeIterator(make([]byte, 0))
storageDiffs := buildStorageDiffsFromTrie(it)
storageDiffs := sdb.buildStorageDiffsFromTrie(it)
return storageDiffs, nil
}
@ -229,35 +255,58 @@ func (sdb *builder) buildStorageDiffsIncremental(oldSR common.Hash, newSR common
oldIt := oldTrie.NodeIterator(make([]byte, 0))
newIt := newTrie.NodeIterator(make([]byte, 0))
it, _ := trie.NewDifferenceIterator(oldIt, newIt)
storageDiffs := buildStorageDiffsFromTrie(it)
storageDiffs := sdb.buildStorageDiffsFromTrie(it)
return storageDiffs, nil
}
func buildStorageDiffsFromTrie(it trie.NodeIterator) []StorageDiff {
func (sdb *builder) buildStorageDiffsFromTrie(it trie.NodeIterator) []StorageDiff {
storageDiffs := make([]StorageDiff, 0)
for {
log.Debug("Iterating over state at path ", "path", pathToStr(it))
if it.Leaf() {
log.Debug("Found leaf in storage", "path", pathToStr(it))
leafProof := make([][]byte, len(it.LeafProof()))
copy(leafProof, it.LeafProof())
leafPath := make([]byte, len(it.Path()))
copy(leafPath, it.Path())
leafKey := make([]byte, len(it.LeafKey()))
copy(leafKey, it.LeafKey())
leafValue := make([]byte, len(it.LeafBlob()))
copy(leafValue, it.LeafBlob())
storageDiffs = append(storageDiffs, StorageDiff{
Key: leafKey,
Value: leafValue,
Path: leafPath,
Proof: leafProof,
})
if sdb.config.PathsAndProofs {
for {
log.Debug("Iterating over state at path ", "path", pathToStr(it))
if it.Leaf() {
log.Debug("Found leaf in storage", "path", pathToStr(it))
leafProof := make([][]byte, len(it.LeafProof()))
copy(leafProof, it.LeafProof())
leafPath := make([]byte, len(it.Path()))
copy(leafPath, it.Path())
leafKey := make([]byte, len(it.LeafKey()))
copy(leafKey, it.LeafKey())
leafValue := make([]byte, len(it.LeafBlob()))
copy(leafValue, it.LeafBlob())
storageDiffs = append(storageDiffs, StorageDiff{
Key: leafKey,
Value: leafValue,
Path: leafPath,
Proof: leafProof,
})
}
cont := it.Next(true)
if !cont {
break
}
}
cont := it.Next(true)
if !cont {
break
} else {
for {
log.Debug("Iterating over state at path ", "path", pathToStr(it))
if it.Leaf() {
log.Debug("Found leaf in storage", "path", pathToStr(it))
leafKey := make([]byte, len(it.LeafKey()))
copy(leafKey, it.LeafKey())
leafValue := make([]byte, len(it.LeafBlob()))
copy(leafValue, it.LeafBlob())
storageDiffs = append(storageDiffs, StorageDiff{
Key: leafKey,
Value: leafValue,
Path: nil,
Proof: nil,
})
}
cont := it.Next(true)
if !cont {
break
}
}
}

View File

@ -56,7 +56,10 @@ func TestBuilder(t *testing.T) {
block1 = blockMap[block1Hash]
block2 = blockMap[block2Hash]
block3 = blockMap[block3Hash]
builder = statediff.NewBuilder(testhelpers.Testdb, chain)
config := statediff.Config{
PathsAndProofs: true,
}
builder = statediff.NewBuilder(testhelpers.Testdb, chain, config)
type arguments struct {
oldStateRoot common.Hash

23
statediff/config.go Normal file
View File

@ -0,0 +1,23 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package statediff
// Config is used to carry in parameters from CLI configuration
type Config struct {
PathsAndProofs bool
LeafsOnly bool
}

View File

@ -18,18 +18,17 @@ package statediff
import (
"bytes"
"encoding/json"
"fmt"
"sync"
"github.com/ethereum/go-ethereum/node"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/event"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/node"
"github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/rpc"
)
@ -66,25 +65,12 @@ type Service struct {
Subscriptions map[rpc.ID]Subscription
}
// Subscription struct holds our subscription channels
type Subscription struct {
PayloadChan chan<- Payload
QuitChan chan<- bool
}
// Payload packages the data to send to StateDiffingService subscriptions
type Payload struct {
BlockRlp []byte `json:"blockRlp" gencodec:"required"`
StateDiffRlp []byte `json:"stateDiffRlp" gencodec:"required"`
Err error `json:"error"`
}
// NewStateDiffService creates a new StateDiffingService
func NewStateDiffService(db ethdb.Database, blockChain *core.BlockChain) (*Service, error) {
func NewStateDiffService(db ethdb.Database, blockChain *core.BlockChain, config Config) (*Service, error) {
return &Service{
Mutex: sync.Mutex{},
BlockChain: blockChain,
Builder: NewBuilder(db, blockChain),
Builder: NewBuilder(db, blockChain, config),
QuitChan: make(chan bool),
Subscriptions: make(map[rpc.ID]Subscription),
}, nil
@ -109,70 +95,61 @@ func (sds *Service) APIs() []rpc.API {
// Loop is the main processing method
func (sds *Service) Loop(chainEventCh chan core.ChainEvent) {
chainEventSub := sds.BlockChain.SubscribeChainEvent(chainEventCh)
defer chainEventSub.Unsubscribe()
blocksCh := make(chan *types.Block, 10)
errCh := chainEventSub.Err()
go func() {
HandleChainEventChLoop:
for {
select {
//Notify chain event channel of events
case chainEvent := <-chainEventCh:
log.Debug("Event received from chainEventCh", "event", chainEvent)
blocksCh <- chainEvent.Block
//if node stopped
case err := <-errCh:
log.Warn("Error from chain event subscription, breaking loop.", "error", err)
close(sds.QuitChan)
break HandleChainEventChLoop
case <-sds.QuitChan:
break HandleChainEventChLoop
}
}
}()
//loop through chain events until no more
HandleBlockChLoop:
for {
select {
case block := <-blocksCh:
currentBlock := block
//Notify chain event channel of events
case chainEvent := <-chainEventCh:
log.Debug("Event received from chainEventCh", "event", chainEvent)
currentBlock := chainEvent.Block
parentHash := currentBlock.ParentHash()
parentBlock := sds.BlockChain.GetBlockByHash(parentHash)
if parentBlock == nil {
log.Error("Parent block is nil, skipping this block",
"parent block hash", parentHash.String(),
"current block number", currentBlock.Number())
break HandleBlockChLoop
continue
}
stateDiff, err := sds.Builder.BuildStateDiff(parentBlock.Root(), currentBlock.Root(), currentBlock.Number().Int64(), currentBlock.Hash())
if err != nil {
if err := sds.process(currentBlock, parentBlock); err != nil {
log.Error("Error building statediff", "block number", currentBlock.Number(), "error", err)
}
rlpBuff := new(bytes.Buffer)
currentBlock.EncodeRLP(rlpBuff)
blockRlp := rlpBuff.Bytes()
stateDiffRlp, _ := rlp.EncodeToBytes(stateDiff)
payload := Payload{
BlockRlp: blockRlp,
StateDiffRlp: stateDiffRlp,
Err: err,
}
// If we have any websocket subscription listening in, send the data to them
sds.send(payload)
case err := <-errCh:
log.Warn("Error from chain event subscription, breaking loop.", "error", err)
sds.close()
return
case <-sds.QuitChan:
log.Debug("Quitting the statediff block channel")
log.Info("Quitting the statediff block channel")
sds.close()
return
}
}
}
// process method builds the state diff payload from the current and parent block and streams it to listening subscriptions
func (sds *Service) process(currentBlock, parentBlock *types.Block) error {
stateDiff, err := sds.Builder.BuildStateDiff(parentBlock.Root(), currentBlock.Root(), currentBlock.Number().Int64(), currentBlock.Hash())
if err != nil {
return err
}
rlpBuff := new(bytes.Buffer)
currentBlock.EncodeRLP(rlpBuff)
blockRlp := rlpBuff.Bytes()
stateDiffBytes, _ := json.Marshal(stateDiff)
payload := Payload{
BlockRlp: blockRlp,
StateDiff: stateDiffBytes,
Err: err,
}
// If we have any websocket subscription listening in, send the data to them
sds.send(payload)
return nil
}
// Subscribe is used by the API to subscribe to the StateDiffingService loop
func (sds *Service) Subscribe(id rpc.ID, sub chan<- Payload, quitChan chan<- bool) {
log.Info("Subscribing to the statediff service")

View File

@ -18,6 +18,7 @@ package mocks
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"sync"
@ -26,7 +27,6 @@ import (
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/statediff"
)
@ -63,7 +63,6 @@ func (sds *MockStateDiffService) APIs() []rpc.API {
// Loop mock method
func (sds *MockStateDiffService) Loop(chan core.ChainEvent) {
//loop through chain events until no more
HandleBlockChLoop:
for {
select {
case block := <-sds.BlockChan:
@ -74,7 +73,7 @@ HandleBlockChLoop:
log.Error("Parent block is nil, skipping this block",
"parent block hash", parentHash.String(),
"current block number", currentBlock.Number())
break HandleBlockChLoop
continue
}
stateDiff, err := sds.Builder.BuildStateDiff(parentBlock.Root(), currentBlock.Root(), currentBlock.Number().Int64(), currentBlock.Hash())
@ -84,11 +83,11 @@ HandleBlockChLoop:
rlpBuff := new(bytes.Buffer)
currentBlock.EncodeRLP(rlpBuff)
blockRlp := rlpBuff.Bytes()
stateDiffRlp, _ := rlp.EncodeToBytes(stateDiff)
stateDiffBytes, _ := json.Marshal(stateDiff)
payload := statediff.Payload{
BlockRlp: blockRlp,
StateDiffRlp: stateDiffRlp,
Err: err,
BlockRlp: blockRlp,
StateDiff: stateDiffBytes,
Err: err,
}
// If we have any websocket subscription listening in, send the data to them
sds.send(payload)

View File

@ -18,6 +18,7 @@ package mocks
import (
"bytes"
"encoding/json"
"math/big"
"sync"
"testing"
@ -63,9 +64,12 @@ func TestAPI(t *testing.T) {
blockChan := make(chan *types.Block)
parentBlockChain := make(chan *types.Block)
serviceQuitChan := make(chan bool)
config := statediff.Config{
PathsAndProofs: true,
}
mockService := MockStateDiffService{
Mutex: sync.Mutex{},
Builder: statediff.NewBuilder(testhelpers.Testdb, chain),
Builder: statediff.NewBuilder(testhelpers.Testdb, chain, config),
BlockChan: blockChan,
ParentBlockChan: parentBlockChain,
QuitChan: serviceQuitChan,
@ -79,7 +83,7 @@ func TestAPI(t *testing.T) {
blockChan <- block1
parentBlockChain <- block0
expectedBlockRlp, _ := rlp.EncodeToBytes(block1)
expectedStateDiff := &statediff.StateDiff{
expectedStateDiff := statediff.StateDiff{
BlockNumber: block1.Number().Int64(),
BlockHash: block1.Hash(),
CreatedAccounts: statediff.AccountDiffsMap{
@ -112,14 +116,20 @@ func TestAPI(t *testing.T) {
},
},
}
expectedStateDiffRlp, _ := rlp.EncodeToBytes(expectedStateDiff)
expectedStateDiffBytes, err := json.Marshal(expectedStateDiff)
if err != nil {
t.Error(err)
}
select {
case payload := <-payloadChan:
if !bytes.Equal(payload.BlockRlp, expectedBlockRlp) {
t.Errorf("payload does not have expected block\r\actual: %v\r\nexpected: %v", payload.BlockRlp, expectedBlockRlp)
}
if !bytes.Equal(payload.StateDiffRlp, expectedStateDiffRlp) {
t.Errorf("payload does not have expected state diff\r\actual: %v\r\nexpected: %v", payload.StateDiffRlp, expectedStateDiffRlp)
if !bytes.Equal(payload.StateDiff, expectedStateDiffBytes) {
t.Errorf("payload does not have expected state diff\r\actual: %v\r\nexpected: %v", payload.StateDiff, expectedStateDiffBytes)
}
if payload.Err != nil {
t.Errorf("payload should not contain an error, but does: %v", payload.Err)
}
case <-quitChan:
t.Errorf("channel quit before delivering payload")

View File

@ -26,6 +26,19 @@ import (
"github.com/ethereum/go-ethereum/core/state"
)
// Subscription struct holds our subscription channels
type Subscription struct {
PayloadChan chan<- Payload
QuitChan chan<- bool
}
// Payload packages the data to send to StateDiffingService subscriptions
type Payload struct {
BlockRlp []byte `json:"blockRlp" gencodec:"required"`
StateDiff []byte `json:"stateDiff" gencodec:"required"`
Err error `json:"error"`
}
// AccountsMap is a mapping of keccak256(address) => accountWrapper
type AccountsMap map[common.Hash]accountWrapper