From 2caeba151db2a7b4ab66ce8f966b50c5028eefbc Mon Sep 17 00:00:00 2001 From: Aleksandr Bezobchuk Date: Wed, 18 Jul 2018 14:14:14 -0400 Subject: [PATCH] Merge pull request #446: Restructure Importing Test --- README.md | 4 +- copied.go | 96 ----------------------- core/ethdb.go | 1 + state/database.go | 18 +++-- state/trie.go | 1 + test/importer/doc.go | 8 ++ main.go => test/importer/importer.go | 113 +++++++++------------------ test/importer/log.go | 66 ++++++++++++++++ test/importer/utils.go | 43 ++++++++++ test/run.go | 67 ++++++++++++++++ 10 files changed, 239 insertions(+), 178 deletions(-) delete mode 100644 copied.go create mode 100644 test/importer/doc.go rename main.go => test/importer/importer.go (68%) create mode 100644 test/importer/log.go create mode 100644 test/importer/utils.go create mode 100644 test/run.go diff --git a/README.md b/README.md index 731db7a1..2596e491 100644 --- a/README.md +++ b/README.md @@ -36,9 +36,11 @@ $ make tools deps install There is an included Ethereum Mainnet blockchain file in `data/blockchain` that provides an easy way to run the demo of parsing Mainnet Ethereum blocks. The dump in `data/` only includes up to block `97638`. To run this, type the following command: ```bash -$ go run main.go copied.go +$ go run test/run.go ``` +By default, state will be dumped into `$HOME/.ethermint`. See `--help` for further usage. + ### Community The following chat channels and forums are a great spot to ask questions about Ethermint: diff --git a/copied.go b/copied.go deleted file mode 100644 index 4d23bae3..00000000 --- a/copied.go +++ /dev/null @@ -1,96 +0,0 @@ -// Code copied from go-ethereum -package main - -import ( - "fmt" - "math/big" - - "github.com/ethereum/go-ethereum/common/math" - "github.com/ethereum/go-ethereum/consensus/ethash" - "github.com/ethereum/go-ethereum/core/state" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/params" -) - -// Some weird constants to avoid constant memory allocs for them. -var ( - big8 = big.NewInt(8) - big32 = big.NewInt(32) -) - -// AccumulateRewards credits the coinbase of the given block with the mining -// reward. The total reward consists of the static block reward and rewards for -// included uncles. The coinbase of each uncle block is also rewarded. -func accumulateRewards(config *params.ChainConfig, state *state.StateDB, header *types.Header, uncles []*types.Header) { - // Select the correct block reward based on chain progression - blockReward := ethash.FrontierBlockReward - if config.IsByzantium(header.Number) { - blockReward = ethash.ByzantiumBlockReward - } - // Accumulate the rewards for the miner and any included uncles - reward := new(big.Int).Set(blockReward) - r := new(big.Int) - for _, uncle := range uncles { - r.Add(uncle.Number, big8) - r.Sub(r, header.Number) - r.Mul(r, blockReward) - r.Div(r, big8) - state.AddBalance(uncle.Coinbase, r) - - r.Div(blockReward, big32) - reward.Add(reward, r) - } - state.AddBalance(header.Coinbase, reward) -} - -// StructLogRes stores a structured log emitted by the EVM while replaying a -// transaction in debug mode -type StructLogRes struct { - Pc uint64 `json:"pc"` - Op string `json:"op"` - Gas uint64 `json:"gas"` - GasCost uint64 `json:"gasCost"` - Depth int `json:"depth"` - Error error `json:"error,omitempty"` - Stack *[]string `json:"stack,omitempty"` - Memory *[]string `json:"memory,omitempty"` - Storage *map[string]string `json:"storage,omitempty"` -} - -// formatLogs formats EVM returned structured logs for json output -func FormatLogs(logs []vm.StructLog) []StructLogRes { - formatted := make([]StructLogRes, len(logs)) - for index, trace := range logs { - formatted[index] = StructLogRes{ - Pc: trace.Pc, - Op: trace.Op.String(), - Gas: trace.Gas, - GasCost: trace.GasCost, - Depth: trace.Depth, - Error: trace.Err, - } - if trace.Stack != nil { - stack := make([]string, len(trace.Stack)) - for i, stackValue := range trace.Stack { - stack[i] = fmt.Sprintf("%x", math.PaddedBigBytes(stackValue, 32)) - } - formatted[index].Stack = &stack - } - if trace.Memory != nil { - memory := make([]string, 0, (len(trace.Memory)+31)/32) - for i := 0; i+32 <= len(trace.Memory); i += 32 { - memory = append(memory, fmt.Sprintf("%x", trace.Memory[i:i+32])) - } - formatted[index].Memory = &memory - } - if trace.Storage != nil { - storage := make(map[string]string) - for i, storageValue := range trace.Storage { - storage[fmt.Sprintf("%x", i)] = fmt.Sprintf("%x", storageValue) - } - formatted[index].Storage = &storage - } - } - return formatted -} diff --git a/core/ethdb.go b/core/ethdb.go index b2f0a07a..89211509 100644 --- a/core/ethdb.go +++ b/core/ethdb.go @@ -2,6 +2,7 @@ package core import ( ethdb "github.com/ethereum/go-ethereum/ethdb" + dbm "github.com/tendermint/tendermint/libs/db" ) diff --git a/state/database.go b/state/database.go index e5dcbb24..637b3519 100644 --- a/state/database.go +++ b/state/database.go @@ -2,28 +2,31 @@ package state import ( "github.com/cosmos/cosmos-sdk/store" - "github.com/cosmos/cosmos-sdk/types" sdk "github.com/cosmos/cosmos-sdk/types" + "github.com/cosmos/ethermint/core" + ethcommon "github.com/ethereum/go-ethereum/common" ethstate "github.com/ethereum/go-ethereum/core/state" ethtrie "github.com/ethereum/go-ethereum/trie" + lru "github.com/hashicorp/golang-lru" + dbm "github.com/tendermint/tendermint/libs/db" ) var ( // AccountsKey is the key used for storing Ethereum accounts in the Cosmos // SDK multi-store. - AccountsKey = types.NewKVStoreKey("account") + AccountsKey = sdk.NewKVStoreKey("account") // StorageKey is the key used for storing Ethereum contract storage in the // Cosmos SDK multi-store. - StorageKey = types.NewKVStoreKey("storage") + StorageKey = sdk.NewKVStoreKey("storage") // CodeKey is the key used for storing Ethereum contract code in the Cosmos // SDK multi-store. - CodeKey = types.NewKVStoreKey("code") + CodeKey = sdk.NewKVStoreKey("code") ) const ( @@ -75,8 +78,8 @@ func NewDatabase(stateDB, codeDB dbm.DB) (*Database, error) { // Create the underlying multi-store stores that will persist account and // account storage data. - db.stateStore.MountStoreWithDB(AccountsKey, types.StoreTypeIAVL, nil) - db.stateStore.MountStoreWithDB(StorageKey, types.StoreTypeIAVL, nil) + db.stateStore.MountStoreWithDB(AccountsKey, sdk.StoreTypeIAVL, nil) + db.stateStore.MountStoreWithDB(StorageKey, sdk.StoreTypeIAVL, nil) // Load the latest account state from the Cosmos SDK multi-store. if err := db.stateStore.LoadLatestVersion(); err != nil { @@ -95,6 +98,7 @@ func NewDatabase(stateDB, codeDB dbm.DB) (*Database, error) { return db, nil } +// LatestVersion returns the latest version of the underlying mult-store. func (db *Database) LatestVersion() int64 { return db.stateStore.LastCommitID().Version } @@ -192,7 +196,7 @@ func (db *Database) ContractCodeSize(addrHash, codeHash ethcommon.Hash) (int, er // Commit commits the underlying Cosmos SDK multi-store returning the commit // ID. -func (db *Database) Commit() types.CommitID { +func (db *Database) Commit() sdk.CommitID { return db.stateStore.Commit() } diff --git a/state/trie.go b/state/trie.go index 86f10c1a..0574d11c 100644 --- a/state/trie.go +++ b/state/trie.go @@ -4,6 +4,7 @@ import ( "encoding/binary" "github.com/cosmos/cosmos-sdk/store" + ethcommon "github.com/ethereum/go-ethereum/common" ethdb "github.com/ethereum/go-ethereum/ethdb" ethtrie "github.com/ethereum/go-ethereum/trie" diff --git a/test/importer/doc.go b/test/importer/doc.go new file mode 100644 index 00000000..0993adc4 --- /dev/null +++ b/test/importer/doc.go @@ -0,0 +1,8 @@ +// The implementation below is to be considered highly unstable and a continual +// WIP. It is a means to replicate and test replaying Ethereum transactions +// using the Cosmos SDK and the EVM. The ultimate result will be what is known +// as Ethermint. +// +// Note: Some code is directly copied from go-ethereum. + +package importer diff --git a/main.go b/test/importer/importer.go similarity index 68% rename from main.go rename to test/importer/importer.go index 3ed6f1f4..db08339b 100644 --- a/main.go +++ b/test/importer/importer.go @@ -1,24 +1,18 @@ -// The implementation below is to be considered highly unstable and a continual -// WIP. It is a means to replicate and test replaying Ethereum transactions -// using the Cosmos SDK and the EVM. The ultimate result will be what is known -// as Ethermint. -package main +package importer import ( "bytes" "encoding/binary" "encoding/json" - "flag" "fmt" "io" "os" - "os/signal" - "runtime/pprof" - "syscall" + "path" "time" "github.com/cosmos/ethermint/core" "github.com/cosmos/ethermint/state" + ethcommon "github.com/ethereum/go-ethereum/common" ethmisc "github.com/ethereum/go-ethereum/consensus/misc" ethcore "github.com/ethereum/go-ethereum/core" @@ -27,60 +21,32 @@ import ( ethvm "github.com/ethereum/go-ethereum/core/vm" ethparams "github.com/ethereum/go-ethereum/params" ethrlp "github.com/ethereum/go-ethereum/rlp" - dbm "github.com/tendermint/tendermint/libs/db" ) -var cpuprofile = flag.String("cpu-profile", "", "write cpu profile `file`") -var blockchain = flag.String("blockchain", "data/blockchain", "file containing blocks to load") -var datadir = flag.String("datadir", "", "directory for ethermint data") - var ( - // TODO: Document... miner501 = ethcommon.HexToAddress("0x35e8e5dC5FBd97c5b421A80B596C030a2Be2A04D") genInvestor = ethcommon.HexToAddress("0x756F45E3FA69347A9A973A725E3C98bC4db0b5a0") ) -// TODO: Document... -func main() { - flag.Parse() +// Importer implements a structure to facilitate testing of importing mainnet +// Ethereum blocks and transactions using the Cosmos SDK components and the +// EVM. +type Importer struct { + EthermintDB *state.Database + BlockchainFile string + Datadir string + InterruptCh <-chan bool +} - if *cpuprofile != "" { - f, err := os.Create(*cpuprofile) - if err != nil { - fmt.Printf("could not create CPU profile: %v\n", err) - return - } - - if err := pprof.StartCPUProfile(f); err != nil { - fmt.Printf("could not start CPU profile: %v\n", err) - return - } - - defer pprof.StopCPUProfile() - } - - sigs := make(chan os.Signal, 1) - interruptCh := make(chan bool, 1) - signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) - go func() { - <-sigs - interruptCh <- true - }() - - stateDB := dbm.NewDB("state", dbm.LevelDBBackend, *datadir) - codeDB := dbm.NewDB("code", dbm.LevelDBBackend, *datadir) - - ethermintDB, err := state.NewDatabase(stateDB, codeDB) - if err != nil { - panic(err) - } - - // Only create genesis if it is a brand new database - if ethermintDB.LatestVersion() == 0 { +// Import performs an import given an Importer that has a Geth stateDB +// implementation and a blockchain exported file. +func (imp *Importer) Import() { + // only create genesis if it is a brand new database + if imp.EthermintDB.LatestVersion() == 0 { // start with empty root hash (i.e. empty state) - gethStateDB, err := ethstate.New(ethcommon.Hash{}, ethermintDB) + gethStateDB, err := ethstate.New(ethcommon.Hash{}, imp.EthermintDB) if err != nil { - panic(err) + panic(fmt.Sprintf("failed to instantiate geth state.StateDB: %v", err)) } genBlock := ethcore.DefaultGenesisBlock() @@ -104,7 +70,7 @@ func main() { panic(err) } - commitID := ethermintDB.Commit() + commitID := imp.EthermintDB.Commit() fmt.Printf("commitID after genesis: %v\n", commitID) fmt.Printf("genesis state root hash: %x\n", genRoot[:]) @@ -112,9 +78,7 @@ func main() { // file with blockchain data exported from geth by using "geth exportdb" // command. - // - // TODO: Allow this to be configurable - input, err := os.Open(*blockchain) + input, err := os.Open(imp.BlockchainFile) if err != nil { panic(err) } @@ -133,16 +97,17 @@ func main() { root501 ethcommon.Hash // root hash after block 501 ) - var prevRoot ethcommon.Hash - binary.BigEndian.PutUint64(prevRoot[:8], uint64(ethermintDB.LatestVersion())) + var prevRoot ethcommon.Hash + binary.BigEndian.PutUint64(prevRoot[:8], uint64(imp.EthermintDB.LatestVersion())) - ethermintDB.Tracing = true + imp.EthermintDB.Tracing = true chainContext := core.NewChainContext() vmConfig := ethvm.Config{} n := 0 startTime := time.Now() interrupt := false + var lastSkipped uint64 for !interrupt { if err = stream.Decode(&block); err == io.EOF { @@ -153,12 +118,13 @@ func main() { } // don't import blocks already imported - if block.NumberU64() < uint64(ethermintDB.LatestVersion()) { + if block.NumberU64() < uint64(imp.EthermintDB.LatestVersion()) { lastSkipped = block.NumberU64() continue } + if lastSkipped > 0 { - fmt.Printf("Skipped blocks up to %d\n", lastSkipped) + fmt.Printf("skipped blocks up to %d\n", lastSkipped) lastSkipped = 0 } @@ -166,7 +132,7 @@ func main() { chainContext.Coinbase = header.Coinbase chainContext.SetHeader(block.NumberU64(), header) - gethStateDB, err := ethstate.New(prevRoot, ethermintDB) + gethStateDB, err := ethstate.New(prevRoot, imp.EthermintDB) if err != nil { panic(fmt.Errorf("failed to instantiate geth state.StateDB at block %d: %v", block.NumberU64(), err)) } @@ -186,7 +152,6 @@ func main() { gethStateDB.Prepare(tx.Hash(), block.Hash(), i) txHash := tx.Hash() - // TODO: Why this address? if bytes.Equal(txHash[:], ethcommon.FromHex("0xc438cfcc3b74a28741bda361032f1c6362c34aa0e1cedff693f31ec7d6a12717")) { vmConfig.Tracer = ethvm.NewStructLogger(ðvm.LogConfig{}) vmConfig.Debug = true @@ -194,9 +159,9 @@ func main() { receipt, _, err := ethcore.ApplyTransaction(chainConfig, chainContext, nil, gp, gethStateDB, header, tx, usedGas, vmConfig) if vmConfig.Tracer != nil { - w, err := os.Create("structlogs.txt") + w, err := os.Create(path.Join(imp.Datadir, "structlogs.txt")) if err != nil { - panic(err) + panic(fmt.Sprintf("failed to create file for VM tracing: %v", err)) } encoder := json.NewEncoder(w) @@ -232,8 +197,7 @@ func main() { } // commit block in Ethermint - ethermintDB.Commit() - //fmt.Printf("commitID after block %d: %v\n", block.NumberU64(), commitID) + imp.EthermintDB.Commit() switch block.NumberU64() { case 500: @@ -249,26 +213,27 @@ func main() { // Check for interrupts select { - case interrupt = <-interruptCh: - fmt.Printf("Interrupted, please wait for cleanup...\n") + case interrupt = <-imp.InterruptCh: + fmt.Println("interrupted, please wait for cleanup...") default: } } fmt.Printf("processed %d blocks\n", n) - ethermintDB.Tracing = true + imp.EthermintDB.Tracing = true // try to create a new geth stateDB from root of the block 500 - fmt.Printf("root500: %x\n", root500[:]) + fmt.Printf("root at block 500: %x\n", root500[:]) - state500, err := ethstate.New(root500, ethermintDB) + state500, err := ethstate.New(root500, imp.EthermintDB) if err != nil { panic(err) } + miner501BalanceAt500 := state500.GetBalance(miner501) - state501, err := ethstate.New(root501, ethermintDB) + state501, err := ethstate.New(root501, imp.EthermintDB) if err != nil { panic(err) } diff --git a/test/importer/log.go b/test/importer/log.go new file mode 100644 index 00000000..d8f6c137 --- /dev/null +++ b/test/importer/log.go @@ -0,0 +1,66 @@ +package importer + +import ( + "fmt" + + ethmath "github.com/ethereum/go-ethereum/common/math" + ethvm "github.com/ethereum/go-ethereum/core/vm" +) + +// StructLogRes stores a structured log emitted by the EVM while replaying a +// transaction in debug mode. +type StructLogRes struct { + Pc uint64 `json:"pc"` + Op string `json:"op"` + Gas uint64 `json:"gas"` + GasCost uint64 `json:"gasCost"` + Depth int `json:"depth"` + Error error `json:"error,omitempty"` + Stack *[]string `json:"stack,omitempty"` + Memory *[]string `json:"memory,omitempty"` + Storage *map[string]string `json:"storage,omitempty"` +} + +// FormatLogs formats EVM returned structured logs for json output. +func FormatLogs(logs []ethvm.StructLog) []StructLogRes { + formatted := make([]StructLogRes, len(logs)) + for index, trace := range logs { + formatted[index] = StructLogRes{ + Pc: trace.Pc, + Op: trace.Op.String(), + Gas: trace.Gas, + GasCost: trace.GasCost, + Depth: trace.Depth, + Error: trace.Err, + } + + if trace.Stack != nil { + stack := make([]string, len(trace.Stack)) + for i, stackValue := range trace.Stack { + stack[i] = fmt.Sprintf("%x", ethmath.PaddedBigBytes(stackValue, 32)) + } + + formatted[index].Stack = &stack + } + + if trace.Memory != nil { + memory := make([]string, 0, (len(trace.Memory)+31)/32) + for i := 0; i+32 <= len(trace.Memory); i += 32 { + memory = append(memory, fmt.Sprintf("%x", trace.Memory[i:i+32])) + } + + formatted[index].Memory = &memory + } + + if trace.Storage != nil { + storage := make(map[string]string) + for i, storageValue := range trace.Storage { + storage[fmt.Sprintf("%x", i)] = fmt.Sprintf("%x", storageValue) + } + + formatted[index].Storage = &storage + } + } + + return formatted +} diff --git a/test/importer/utils.go b/test/importer/utils.go new file mode 100644 index 00000000..4e29b2a8 --- /dev/null +++ b/test/importer/utils.go @@ -0,0 +1,43 @@ +package importer + +import ( + "math/big" + + "github.com/ethereum/go-ethereum/consensus/ethash" + ethstate "github.com/ethereum/go-ethereum/core/state" + ethtypes "github.com/ethereum/go-ethereum/core/types" + ethparams "github.com/ethereum/go-ethereum/params" +) + +// Some weird constants to avoid constant memory allocs for them. +var ( + big8 = big.NewInt(8) + big32 = big.NewInt(32) +) + +// accumulateRewards credits the coinbase of the given block with the mining +// reward. The total reward consists of the static block reward and rewards for +// included uncles. The coinbase of each uncle block is also rewarded. +func accumulateRewards(config *ethparams.ChainConfig, state *ethstate.StateDB, header *ethtypes.Header, uncles []*ethtypes.Header) { + // select the correct block reward based on chain progression + blockReward := ethash.FrontierBlockReward + if config.IsByzantium(header.Number) { + blockReward = ethash.ByzantiumBlockReward + } + + // accumulate the rewards for the miner and any included uncles + reward := new(big.Int).Set(blockReward) + r := new(big.Int) + + for _, uncle := range uncles { + r.Add(uncle.Number, big8) + r.Sub(r, header.Number) + r.Mul(r, blockReward) + r.Div(r, big8) + state.AddBalance(uncle.Coinbase, r) + r.Div(blockReward, big32) + reward.Add(reward, r) + } + + state.AddBalance(header.Coinbase, reward) +} diff --git a/test/run.go b/test/run.go new file mode 100644 index 00000000..20a86fb3 --- /dev/null +++ b/test/run.go @@ -0,0 +1,67 @@ +package main + +import ( + "flag" + "fmt" + "os" + "os/signal" + "path" + "runtime/pprof" + "syscall" + + "github.com/cosmos/ethermint/state" + "github.com/cosmos/ethermint/test/importer" + + dbm "github.com/tendermint/tendermint/libs/db" +) + +var ( + cpuprofile = flag.String("cpu-profile", "", "write cpu profile `file`") + blockchain = flag.String("blockchain", "data/blockchain", "file containing blocks to load") + datadir = flag.String("datadir", path.Join(os.Getenv("HOME"), ".ethermint"), "directory for ethermint data") +) + +func main() { + flag.Parse() + + if *cpuprofile != "" { + f, err := os.Create(*cpuprofile) + if err != nil { + fmt.Printf("could not create CPU profile: %v\n", err) + return + } + + if err := pprof.StartCPUProfile(f); err != nil { + fmt.Printf("could not start CPU profile: %v\n", err) + return + } + + defer pprof.StopCPUProfile() + } + + sigs := make(chan os.Signal, 1) + interruptCh := make(chan bool, 1) + signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) + + go func() { + <-sigs + interruptCh <- true + }() + + stateDB := dbm.NewDB("state", dbm.LevelDBBackend, *datadir) + codeDB := dbm.NewDB("code", dbm.LevelDBBackend, *datadir) + + ethermintDB, err := state.NewDatabase(stateDB, codeDB) + if err != nil { + panic(fmt.Sprintf("failed to initialize geth Database: %v", err)) + } + + importer := importer.Importer{ + EthermintDB: ethermintDB, + BlockchainFile: *blockchain, + Datadir: *datadir, + InterruptCh: interruptCh, + } + + importer.Import() +}