cerc-io/go-ethereum
18
0
Fork 0
Code Issues 9 Pull Requests Packages Projects Releases 66 Wiki Activity

integrating state diff extracting, building, and persisting into geth processes

Browse Source
This commit is contained in:
Ian Norden 2018-11-25 23:52:53 -06:00
parent 53a6b1c689
commit 26e8088cc5
14 changed files with 343 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cmd/geth/main.go
View File

@ -132,6 +132,7 @@ var (
utils.GpoPercentileFlag,
utils.EWASMInterpreterFlag,
utils.EVMInterpreterFlag,
utils.StateDiffFlag,
configFileFlag,
}

17
cmd/utils/flags.go
View File

@ -626,6 +626,12 @@ var (
Usage: "External EVM configuration (default = built-in interpreter)",
Value: "",
}
// Statediff flags
StateDiffFlag = cli.BoolFlag{
Name: "statediff",
Usage: "Enables the calculation of state diffs between each block, persists these state diffs in ipfs",
}
)
// MakeDataDir retrieves the currently requested data directory, terminating
@ -1127,6 +1133,13 @@ func SetShhConfig(ctx *cli.Context, stack *node.Node, cfg *whisper.Config) {
}
}
// Check if state diff flags are on and applies them to eth context
func setStateDiff(ctx *cli.Context, cfg *eth.Config) {
if ctx.GlobalBool(StateDiffFlag.Name) && cfg.NoPruning && cfg.SyncMode == downloader.FullSync {
cfg.StateDiff.On = true
}
}
// SetEthConfig applies eth-related command line flags to the config.
func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
// Avoid conflicting network flags
@ -1162,6 +1175,10 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
}
cfg.NoPruning = ctx.GlobalString(GCModeFlag.Name) == "archive"
if ctx.GlobalIsSet(StateDiffFlag.Name) {
setStateDiff(ctx, cfg)
}
if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheTrieFlag.Name) {
cfg.TrieCleanCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheTrieFlag.Name) / 100
}

14
core/blockchain.go
View File

@ -20,6 +20,7 @@ package core
import (
"errors"
"fmt"
"github.com/ethereum/go-ethereum/statediff"
"io"
"math/big"
mrand "math/rand"
@ -72,6 +73,7 @@ type CacheConfig struct {
TrieCleanLimit int // Memory allowance (MB) to use for caching trie nodes in memory
TrieDirtyLimit int // Memory limit (MB) at which to start flushing dirty trie nodes to disk
TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk
StateDiff bool // Whether or not to calculate and persist state diffs
}
// BlockChain represents the canonical chain given a database with a genesis
@ -133,6 +135,8 @@ type BlockChain struct {
badBlocks *lru.Cache // Bad block cache
shouldPreserve func(*types.Block) bool // Function used to determine whether should preserve the given block.
diffExtractor statediff.Extractor // State diff processing interface
}
// NewBlockChain returns a fully initialised block chain using information
@ -173,6 +177,10 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par
bc.SetValidator(NewBlockValidator(chainConfig, bc, engine))
bc.SetProcessor(NewStateProcessor(chainConfig, bc, engine))
if cacheConfig.StateDiff {
bc.diffExtractor = statediff.NewExtractor(db)
}
var err error
bc.hc, err = NewHeaderChain(db, chainConfig, engine, bc.getProcInterrupt)
if err != nil {
@ -1187,6 +1195,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty
parent = chain[i-1]
}
state, err := state.New(parent.Root(), bc.stateCache)
if err != nil {
return i, events, coalescedLogs, err
}
@ -1204,6 +1213,11 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty
}
proctime := time.Since(bstart)
// If extracting statediffs, do so now
if bc.cacheConfig.StateDiff {
bc.diffExtractor.Extract(*parent, *block)
}
// Write the block to the chain and get the status.
status, err := bc.WriteBlockWithState(block, receipts, state)
if err != nil {

8
eth/backend.go
View File

@ -154,7 +154,13 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) {
EWASMInterpreter: config.EWASMInterpreter,
EVMInterpreter: config.EVMInterpreter,
}
cacheConfig = &core.CacheConfig{Disabled: config.NoPruning, TrieCleanLimit: config.TrieCleanCache, TrieDirtyLimit: config.TrieDirtyCache, TrieTimeLimit: config.TrieTimeout}
cacheConfig = &core.CacheConfig{
Disabled: config.NoPruning,
TrieCleanLimit: config.TrieCleanCache,
TrieDirtyLimit: config.TrieDirtyCache,
TrieTimeLimit: config.TrieTimeout,
StateDiff: config.StateDiff.On,
}
)
eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, eth.chainConfig, eth.engine, vmConfig, eth.shouldPreserve)
if err != nil {

4
eth/config.go
View File

@ -17,6 +17,7 @@
package eth
import (
"github.com/ethereum/go-ethereum/statediff"
"math/big"
"os"
"os/user"
@ -128,6 +129,9 @@ type Config struct {
EWASMInterpreter string
// Type of the EVM interpreter ("" for default)
EVMInterpreter string
// Config for state diff building
StateDiff statediff.Config
}
type configMarshaling struct {

43
statediff/statediff_builder.go → statediff/builder.go
View File

@ -1,3 +1,22 @@
// Copyright 2015 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// Contains a batch of utility type declarations used by the tests. As the node
// operates on unique types, a lot of them are needed to check various features.
package statediff
import (
@ -9,24 +28,24 @@ import (
"github.com/ethereum/go-ethereum/trie"
)
type StateDiffBuilder interface {
CreateStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (*StateDiff, error)
type Builder interface {
BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (*StateDiff, error)
}
type stateDiffBuilder struct {
type builder struct {
chainDB ethdb.Database
trieDB *trie.Database
cachedTrie *trie.Trie
}
func NewStateDiffBuilder(db ethdb.Database) *stateDiffBuilder {
return &stateDiffBuilder{
func NewBuilder(db ethdb.Database) *builder {
return &builder{
chainDB: db,
trieDB: trie.NewDatabase(db),
}
}
func (sdb *stateDiffBuilder) CreateStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (*StateDiff, error) {
func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (*StateDiff, error) {
// Generate tries for old and new states
oldTrie, err := trie.New(oldStateRoot, sdb.trieDB)
if err != nil {
@ -81,7 +100,7 @@ func (sdb *stateDiffBuilder) CreateStateDiff(oldStateRoot, newStateRoot common.H
}, nil
}
func (sdb *stateDiffBuilder) collectDiffNodes(a, b trie.NodeIterator) (map[common.Address]*state.Account, error) {
func (sdb *builder) collectDiffNodes(a, b trie.NodeIterator) (map[common.Address]*state.Account, error) {
var diffAccounts map[common.Address]*state.Account
it, _ := trie.NewDifferenceIterator(a, b)
@ -117,7 +136,7 @@ func (sdb *stateDiffBuilder) collectDiffNodes(a, b trie.NodeIterator) (map[commo
return diffAccounts, nil
}
func (sdb *stateDiffBuilder) buildDiffEventual(accounts map[common.Address]*state.Account, created bool) (map[common.Address]AccountDiffEventual, error) {
func (sdb *builder) buildDiffEventual(accounts map[common.Address]*state.Account, created bool) (map[common.Address]AccountDiffEventual, error) {
accountDiffs := make(map[common.Address]AccountDiffEventual)
for addr, val := range accounts {
sr := val.Root
@ -178,7 +197,7 @@ func (sdb *stateDiffBuilder) buildDiffEventual(accounts map[common.Address]*stat
return accountDiffs, nil
}
func (sdb *stateDiffBuilder) buildDiffIncremental(creations map[common.Address]*state.Account, deletions map[common.Address]*state.Account, updatedKeys *[]string) (map[common.Address]AccountDiffIncremental, error) {
func (sdb *builder) buildDiffIncremental(creations map[common.Address]*state.Account, deletions map[common.Address]*state.Account, updatedKeys *[]string) (map[common.Address]AccountDiffIncremental, error) {
updatedAccounts := make(map[common.Address]AccountDiffIncremental)
for _, val := range *updatedKeys {
createdAcc := creations[common.HexToAddress(val)]
@ -221,7 +240,7 @@ func (sdb *stateDiffBuilder) buildDiffIncremental(creations map[common.Address]*
return updatedAccounts, nil
}
func (sdb *stateDiffBuilder) buildStorageDiffsEventual(sr common.Hash, creation bool) (map[string]diffString, error) {
func (sdb *builder) buildStorageDiffsEventual(sr common.Hash, creation bool) (map[string]diffString, error) {
log.Debug("Storage Root For Eventual Diff", "root", sr.Hex())
sTrie, err := trie.New(sr, sdb.trieDB)
if err != nil {
@ -249,7 +268,7 @@ func (sdb *stateDiffBuilder) buildStorageDiffsEventual(sr common.Hash, creation
return storageDiffs, nil
}
func (sdb *stateDiffBuilder) buildStorageDiffsIncremental(oldSR common.Hash, newSR common.Hash) (map[string]diffString, error) {
func (sdb *builder) buildStorageDiffsIncremental(oldSR common.Hash, newSR common.Hash) (map[string]diffString, error) {
log.Debug("Storage Roots for Incremental Diff", "old", oldSR.Hex(), "new", newSR.Hex())
oldTrie, err := trie.New(oldSR, sdb.trieDB)
if err != nil {
@ -285,7 +304,7 @@ func (sdb *stateDiffBuilder) buildStorageDiffsIncremental(oldSR common.Hash, new
return storageDiffs, nil
}
func (sdb *stateDiffBuilder) addressByPath(path []byte) (*common.Address, error) {
func (sdb *builder) addressByPath(path []byte) (*common.Address, error) {
// db := core.PreimageTable(sdb.chainDb)
log.Debug("Looking up address from path", "path", common.ToHex(append([]byte("secure-key-"), path...)))
// if addrBytes,err := db.Get(path); err != nil {

21
statediff/builder_test.go Normal file
View File

@ -0,0 +1,21 @@
// Copyright 2015 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// Contains a batch of utility type declarations used by the tests. As the node
// operates on unique types, a lot of them are needed to check various features.
package statediff_test

80
statediff/config.go Normal file
View File

@ -0,0 +1,80 @@
// Copyright 2015 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// Contains a batch of utility type declarations used by the tests. As the node
// operates on unique types, a lot of them are needed to check various features.
package statediff
import "fmt"
type Config struct {
On bool
Mode StateDiffMode
}
type StateDiffMode int
const (
IPFS StateDiffMode = iota
LDB
SQL
)
func (mode StateDiffMode) IsValid() bool {
return mode >= IPFS && mode <= SQL
}
// String implements the stringer interface.
func (mode StateDiffMode) String() string {
switch mode {
case IPFS:
return "ipfs"
case LDB:
return "ldb"
case SQL:
return "sql"
default:
return "unknown"
}
}
func (mode StateDiffMode) MarshalText() ([]byte, error) {
switch mode {
case IPFS:
return []byte("ipfs"), nil
case LDB:
return []byte("ldb"), nil
case SQL:
return []byte("sql"), nil
default:
return nil, fmt.Errorf("unknown state diff storage mode %d", mode)
}
}
func (mode *StateDiffMode) UnmarshalText(text []byte) error {
switch string(text) {
case "ipfs":
*mode = IPFS
case "ldb":
*mode = LDB
case "sql":
*mode = SQL
default:
return fmt.Errorf(`unknown state diff storage mode %q, want "ipfs", "ldb" or "sql"`, text)
}
return nil
}

50
statediff/extractor.go Normal file
View File

@ -0,0 +1,50 @@
// Copyright 2015 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// Contains a batch of utility type declarations used by the tests. As the node
// operates on unique types, a lot of them are needed to check various features.
package statediff
import (
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/ethdb"
)
type Extractor interface {
ExtractStateDiff(parent, current types.Block) error
}
type extractor struct {
b *builder
p *persister
}
func NewExtractor(db ethdb.Database) *extractor {
return &extractor{
b: NewBuilder(db),
p: NewPersister(),
}
}
func (e *extractor) Extract(parent, current types.Block) error {
stateDiff, err := e.b.BuildStateDiff(parent.Root(), current.Root(), current.Number().Int64(), current.Hash())
if err != nil {
return err
}
return e.p.PersistStateDiff(stateDiff)
}

20
statediff/extractor_test.go Normal file
View File

@ -0,0 +1,20 @@
// Copyright 2015 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// Contains a batch of utility type declarations used by the tests. As the node
// operates on unique types, a lot of them are needed to check various features.
package statediff_test

19
statediff/helpers.go
View File

@ -1,3 +1,22 @@
// Copyright 2015 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// Contains a batch of utility type declarations used by the tests. As the node
// operates on unique types, a lot of them are needed to check various features.
package statediff
import (

40
statediff/persister.go Normal file
View File

@ -0,0 +1,40 @@
// Copyright 2015 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// Contains a batch of utility type declarations used by the tests. As the node
// operates on unique types, a lot of them are needed to check various features.
package statediff
type Persister interface {
PersistStateDiff(sd *StateDiff) error
}
type persister struct {
}
func NewPersister() *persister {
return &persister{
}
}
func (p *persister) PersistStateDiff(sd *StateDiff) error {
//TODO: Persist state diff in IPFS
return nil
}

20
statediff/persister_test.go Normal file
View File

@ -0,0 +1,20 @@
// Copyright 2015 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// Contains a batch of utility type declarations used by the tests. As the node
// operates on unique types, a lot of them are needed to check various features.
package statediff_test

19
statediff/statediff.go
View File

@ -1,3 +1,22 @@
// Copyright 2015 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// Contains a batch of utility type declarations used by the tests. As the node
// operates on unique types, a lot of them are needed to check various features.
package statediff
import (