Merge pull request #446: Restructure Importing Test

README.md

@@ -36,9 +36,11 @@ $ make tools deps install
 There is an included Ethereum Mainnet blockchain file in `data/blockchain` that provides an easy way to run the demo of parsing Mainnet Ethereum blocks. The dump in `data/` only includes up to block `97638`. To run this, type the following command:
 
 ```bash
-$ go run main.go copied.go
+$ go run test/run.go
 ```
 
+By default, state will be dumped into `$HOME/.ethermint`. See `--help` for further usage.
+
 ### Community
 
 The following chat channels and forums are a great spot to ask questions about Ethermint:

copied.go

@@ -1,96 +0,0 @@
-// Code copied from go-ethereum
-package main
-
-import (
-	"fmt"
-	"math/big"
-
-	"github.com/ethereum/go-ethereum/common/math"
-	"github.com/ethereum/go-ethereum/consensus/ethash"
-	"github.com/ethereum/go-ethereum/core/state"
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/core/vm"
-	"github.com/ethereum/go-ethereum/params"
-)
-
-// Some weird constants to avoid constant memory allocs for them.
-var (
-	big8  = big.NewInt(8)
-	big32 = big.NewInt(32)
-)
-
-// AccumulateRewards credits the coinbase of the given block with the mining
-// reward. The total reward consists of the static block reward and rewards for
-// included uncles. The coinbase of each uncle block is also rewarded.
-func accumulateRewards(config *params.ChainConfig, state *state.StateDB, header *types.Header, uncles []*types.Header) {
-	// Select the correct block reward based on chain progression
-	blockReward := ethash.FrontierBlockReward
-	if config.IsByzantium(header.Number) {
-		blockReward = ethash.ByzantiumBlockReward
-	}
-	// Accumulate the rewards for the miner and any included uncles
-	reward := new(big.Int).Set(blockReward)
-	r := new(big.Int)
-	for _, uncle := range uncles {
-		r.Add(uncle.Number, big8)
-		r.Sub(r, header.Number)
-		r.Mul(r, blockReward)
-		r.Div(r, big8)
-		state.AddBalance(uncle.Coinbase, r)
-
-		r.Div(blockReward, big32)
-		reward.Add(reward, r)
-	}
-	state.AddBalance(header.Coinbase, reward)
-}
-
-// StructLogRes stores a structured log emitted by the EVM while replaying a
-// transaction in debug mode
-type StructLogRes struct {
-	Pc      uint64             `json:"pc"`
-	Op      string             `json:"op"`
-	Gas     uint64             `json:"gas"`
-	GasCost uint64             `json:"gasCost"`
-	Depth   int                `json:"depth"`
-	Error   error              `json:"error,omitempty"`
-	Stack   *[]string          `json:"stack,omitempty"`
-	Memory  *[]string          `json:"memory,omitempty"`
-	Storage *map[string]string `json:"storage,omitempty"`
-}
-
-// formatLogs formats EVM returned structured logs for json output
-func FormatLogs(logs []vm.StructLog) []StructLogRes {
-	formatted := make([]StructLogRes, len(logs))
-	for index, trace := range logs {
-		formatted[index] = StructLogRes{
-			Pc:      trace.Pc,
-			Op:      trace.Op.String(),
-			Gas:     trace.Gas,
-			GasCost: trace.GasCost,
-			Depth:   trace.Depth,
-			Error:   trace.Err,
-		}
-		if trace.Stack != nil {
-			stack := make([]string, len(trace.Stack))
-			for i, stackValue := range trace.Stack {
-				stack[i] = fmt.Sprintf("%x", math.PaddedBigBytes(stackValue, 32))
-			}
-			formatted[index].Stack = &stack
-		}
-		if trace.Memory != nil {
-			memory := make([]string, 0, (len(trace.Memory)+31)/32)
-			for i := 0; i+32 <= len(trace.Memory); i += 32 {
-				memory = append(memory, fmt.Sprintf("%x", trace.Memory[i:i+32]))
-			}
-			formatted[index].Memory = &memory
-		}
-		if trace.Storage != nil {
-			storage := make(map[string]string)
-			for i, storageValue := range trace.Storage {
-				storage[fmt.Sprintf("%x", i)] = fmt.Sprintf("%x", storageValue)
-			}
-			formatted[index].Storage = &storage
-		}
-	}
-	return formatted
-}

core/ethdb.go

@@ -2,6 +2,7 @@ package core
 
 import (
 	ethdb "github.com/ethereum/go-ethereum/ethdb"
+
 	dbm "github.com/tendermint/tendermint/libs/db"
 )
 

state/database.go

@@ -2,28 +2,31 @@ package state
 
 import (
 	"github.com/cosmos/cosmos-sdk/store"
-	"github.com/cosmos/cosmos-sdk/types"
 	sdk "github.com/cosmos/cosmos-sdk/types"
+
 	"github.com/cosmos/ethermint/core"
+
 	ethcommon "github.com/ethereum/go-ethereum/common"
 	ethstate "github.com/ethereum/go-ethereum/core/state"
 	ethtrie "github.com/ethereum/go-ethereum/trie"
+
 	lru "github.com/hashicorp/golang-lru"
+
 	dbm "github.com/tendermint/tendermint/libs/db"
 )
 
 var (
 	// AccountsKey is the key used for storing Ethereum accounts in the Cosmos
 	// SDK multi-store.
-	AccountsKey = types.NewKVStoreKey("account")
+	AccountsKey = sdk.NewKVStoreKey("account")
 
 	// StorageKey is the key used for storing Ethereum contract storage in the
 	// Cosmos SDK multi-store.
-	StorageKey = types.NewKVStoreKey("storage")
+	StorageKey = sdk.NewKVStoreKey("storage")
 
 	// CodeKey is the key used for storing Ethereum contract code in the Cosmos
 	// SDK multi-store.
-	CodeKey = types.NewKVStoreKey("code")
+	CodeKey = sdk.NewKVStoreKey("code")
 )
 
 const (
@@ -75,8 +78,8 @@ func NewDatabase(stateDB, codeDB dbm.DB) (*Database, error) {
 
 	// Create the underlying multi-store stores that will persist account and
 	// account storage data.
-	db.stateStore.MountStoreWithDB(AccountsKey, types.StoreTypeIAVL, nil)
+	db.stateStore.MountStoreWithDB(AccountsKey, sdk.StoreTypeIAVL, nil)
-	db.stateStore.MountStoreWithDB(StorageKey, types.StoreTypeIAVL, nil)
+	db.stateStore.MountStoreWithDB(StorageKey, sdk.StoreTypeIAVL, nil)
 
 	// Load the latest account state from the Cosmos SDK multi-store.
 	if err := db.stateStore.LoadLatestVersion(); err != nil {
@@ -95,6 +98,7 @@ func NewDatabase(stateDB, codeDB dbm.DB) (*Database, error) {
 	return db, nil
 }
 
+// LatestVersion returns the latest version of the underlying mult-store.
 func (db *Database) LatestVersion() int64 {
 	return db.stateStore.LastCommitID().Version
 }
@@ -192,7 +196,7 @@ func (db *Database) ContractCodeSize(addrHash, codeHash ethcommon.Hash) (int, er
 
 // Commit commits the underlying Cosmos SDK multi-store returning the commit
 // ID.
-func (db *Database) Commit() types.CommitID {
+func (db *Database) Commit() sdk.CommitID {
 	return db.stateStore.Commit()
 }
 

state/trie.go

@@ -4,6 +4,7 @@ import (
 	"encoding/binary"
 
 	"github.com/cosmos/cosmos-sdk/store"
+
 	ethcommon "github.com/ethereum/go-ethereum/common"
 	ethdb "github.com/ethereum/go-ethereum/ethdb"
 	ethtrie "github.com/ethereum/go-ethereum/trie"

test/importer/doc.go

@@ -0,0 +1,8 @@
+// The implementation below is to be considered highly unstable and a continual
+// WIP. It is a means to replicate and test replaying Ethereum transactions
+// using the Cosmos SDK and the EVM. The ultimate result will be what is known
+// as Ethermint.
+//
+// Note: Some code is directly  copied from go-ethereum.
+
+package importer

test/importer/importer.go

@@ -1,24 +1,18 @@
-// The implementation below is to be considered highly unstable and a continual
+package importer
-// WIP. It is a means to replicate and test replaying Ethereum transactions
-// using the Cosmos SDK and the EVM. The ultimate result will be what is known
-// as Ethermint.
-package main
 
 import (
 	"bytes"
 	"encoding/binary"
 	"encoding/json"
-	"flag"
 	"fmt"
 	"io"
 	"os"
-	"os/signal"
+	"path"
-	"runtime/pprof"
-	"syscall"
 	"time"
 
 	"github.com/cosmos/ethermint/core"
 	"github.com/cosmos/ethermint/state"
+
 	ethcommon "github.com/ethereum/go-ethereum/common"
 	ethmisc "github.com/ethereum/go-ethereum/consensus/misc"
 	ethcore "github.com/ethereum/go-ethereum/core"
@@ -27,60 +21,32 @@ import (
 	ethvm "github.com/ethereum/go-ethereum/core/vm"
 	ethparams "github.com/ethereum/go-ethereum/params"
 	ethrlp "github.com/ethereum/go-ethereum/rlp"
-	dbm "github.com/tendermint/tendermint/libs/db"
 )
 
-var cpuprofile = flag.String("cpu-profile", "", "write cpu profile `file`")
-var blockchain = flag.String("blockchain", "data/blockchain", "file containing blocks to load")
-var datadir = flag.String("datadir", "", "directory for ethermint data")
-
 var (
-	// TODO: Document...
 	miner501    = ethcommon.HexToAddress("0x35e8e5dC5FBd97c5b421A80B596C030a2Be2A04D")
 	genInvestor = ethcommon.HexToAddress("0x756F45E3FA69347A9A973A725E3C98bC4db0b5a0")
 )
 
-// TODO: Document...
+// Importer implements a structure to facilitate testing of importing mainnet
-func main() {
+// Ethereum blocks and transactions using the Cosmos SDK components and the
-	flag.Parse()
+// EVM.
+type Importer struct {
+	EthermintDB    *state.Database
+	BlockchainFile string
+	Datadir        string
+	InterruptCh    <-chan bool
+}
 
-	if *cpuprofile != "" {
+// Import performs an import given an Importer that has a Geth stateDB
-		f, err := os.Create(*cpuprofile)
+// implementation and a blockchain exported file.
-		if err != nil {
+func (imp *Importer) Import() {
-			fmt.Printf("could not create CPU profile: %v\n", err)
+	// only create genesis if it is a brand new database
-			return
+	if imp.EthermintDB.LatestVersion() == 0 {
-		}
-
-		if err := pprof.StartCPUProfile(f); err != nil {
-			fmt.Printf("could not start CPU profile: %v\n", err)
-			return
-		}
-
-		defer pprof.StopCPUProfile()
-	}
-
-	sigs := make(chan os.Signal, 1)
-	interruptCh := make(chan bool, 1)
-	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
-	go func() {
-        <-sigs
-        interruptCh <- true
-    }()
-
-	stateDB := dbm.NewDB("state", dbm.LevelDBBackend, *datadir)
-	codeDB := dbm.NewDB("code", dbm.LevelDBBackend, *datadir)
-
-	ethermintDB, err := state.NewDatabase(stateDB, codeDB)
-	if err != nil {
-		panic(err)
-	}
-
-	// Only create genesis if it is a brand new database
-	if ethermintDB.LatestVersion() == 0 {
 		// start with empty root hash (i.e. empty state)
-		gethStateDB, err := ethstate.New(ethcommon.Hash{}, ethermintDB)
+		gethStateDB, err := ethstate.New(ethcommon.Hash{}, imp.EthermintDB)
 		if err != nil {
-			panic(err)
+			panic(fmt.Sprintf("failed to instantiate geth state.StateDB: %v", err))
 		}
 
 		genBlock := ethcore.DefaultGenesisBlock()
@@ -104,7 +70,7 @@ func main() {
 			panic(err)
 		}
 
-		commitID := ethermintDB.Commit()
+		commitID := imp.EthermintDB.Commit()
 
 		fmt.Printf("commitID after genesis: %v\n", commitID)
 		fmt.Printf("genesis state root hash: %x\n", genRoot[:])
@@ -112,9 +78,7 @@ func main() {
 
 	// file with blockchain data exported from geth by using "geth exportdb"
 	// command.
-	//
+	input, err := os.Open(imp.BlockchainFile)
-	// TODO: Allow this to be configurable
-	input, err := os.Open(*blockchain)
 	if err != nil {
 		panic(err)
 	}
@@ -133,16 +97,17 @@ func main() {
 		root501 ethcommon.Hash // root hash after block 501
 	)
 
-	var prevRoot ethcommon.Hash 
+	var prevRoot ethcommon.Hash
-	binary.BigEndian.PutUint64(prevRoot[:8], uint64(ethermintDB.LatestVersion()))
+	binary.BigEndian.PutUint64(prevRoot[:8], uint64(imp.EthermintDB.LatestVersion()))
 
-	ethermintDB.Tracing = true
+	imp.EthermintDB.Tracing = true
 	chainContext := core.NewChainContext()
 	vmConfig := ethvm.Config{}
 
 	n := 0
 	startTime := time.Now()
 	interrupt := false
+
 	var lastSkipped uint64
 	for !interrupt {
 		if err = stream.Decode(&block); err == io.EOF {
@@ -153,12 +118,13 @@ func main() {
 		}
 
 		// don't import blocks already imported
-		if block.NumberU64() < uint64(ethermintDB.LatestVersion()) {
+		if block.NumberU64() < uint64(imp.EthermintDB.LatestVersion()) {
 			lastSkipped = block.NumberU64()
 			continue
 		}
+
 		if lastSkipped > 0 {
-			fmt.Printf("Skipped blocks up to %d\n", lastSkipped)
+			fmt.Printf("skipped blocks up to %d\n", lastSkipped)
 			lastSkipped = 0
 		}
 
@@ -166,7 +132,7 @@ func main() {
 		chainContext.Coinbase = header.Coinbase
 		chainContext.SetHeader(block.NumberU64(), header)
 
-		gethStateDB, err := ethstate.New(prevRoot, ethermintDB)
+		gethStateDB, err := ethstate.New(prevRoot, imp.EthermintDB)
 		if err != nil {
 			panic(fmt.Errorf("failed to instantiate geth state.StateDB at block %d: %v", block.NumberU64(), err))
 		}
@@ -186,7 +152,6 @@ func main() {
 			gethStateDB.Prepare(tx.Hash(), block.Hash(), i)
 
 			txHash := tx.Hash()
-			// TODO: Why this address?
 			if bytes.Equal(txHash[:], ethcommon.FromHex("0xc438cfcc3b74a28741bda361032f1c6362c34aa0e1cedff693f31ec7d6a12717")) {
 				vmConfig.Tracer = ethvm.NewStructLogger(&ethvm.LogConfig{})
 				vmConfig.Debug = true
@@ -194,9 +159,9 @@ func main() {
 
 			receipt, _, err := ethcore.ApplyTransaction(chainConfig, chainContext, nil, gp, gethStateDB, header, tx, usedGas, vmConfig)
 			if vmConfig.Tracer != nil {
-				w, err := os.Create("structlogs.txt")
+				w, err := os.Create(path.Join(imp.Datadir, "structlogs.txt"))
 				if err != nil {
-					panic(err)
+					panic(fmt.Sprintf("failed to create file for VM tracing: %v", err))
 				}
 
 				encoder := json.NewEncoder(w)
@@ -232,8 +197,7 @@ func main() {
 		}
 
 		// commit block in Ethermint
-		ethermintDB.Commit()
+		imp.EthermintDB.Commit()
-		//fmt.Printf("commitID after block %d: %v\n", block.NumberU64(), commitID)
 
 		switch block.NumberU64() {
 		case 500:
@@ -249,26 +213,27 @@ func main() {
 
 		// Check for interrupts
 		select {
-		case interrupt = <-interruptCh:
+		case interrupt = <-imp.InterruptCh:
-			fmt.Printf("Interrupted, please wait for cleanup...\n")
+			fmt.Println("interrupted, please wait for cleanup...")
 		default:
 		}
 	}
 
 	fmt.Printf("processed %d blocks\n", n)
 
-	ethermintDB.Tracing = true
+	imp.EthermintDB.Tracing = true
 
 	// try to create a new geth stateDB from root of the block 500
-	fmt.Printf("root500: %x\n", root500[:])
+	fmt.Printf("root at block 500: %x\n", root500[:])
 
-	state500, err := ethstate.New(root500, ethermintDB)
+	state500, err := ethstate.New(root500, imp.EthermintDB)
 	if err != nil {
 		panic(err)
 	}
+
 	miner501BalanceAt500 := state500.GetBalance(miner501)
 
-	state501, err := ethstate.New(root501, ethermintDB)
+	state501, err := ethstate.New(root501, imp.EthermintDB)
 	if err != nil {
 		panic(err)
 	}

test/importer/log.go

@@ -0,0 +1,66 @@
+package importer
+
+import (
+	"fmt"
+
+	ethmath "github.com/ethereum/go-ethereum/common/math"
+	ethvm "github.com/ethereum/go-ethereum/core/vm"
+)
+
+// StructLogRes stores a structured log emitted by the EVM while replaying a
+// transaction in debug mode.
+type StructLogRes struct {
+	Pc      uint64             `json:"pc"`
+	Op      string             `json:"op"`
+	Gas     uint64             `json:"gas"`
+	GasCost uint64             `json:"gasCost"`
+	Depth   int                `json:"depth"`
+	Error   error              `json:"error,omitempty"`
+	Stack   *[]string          `json:"stack,omitempty"`
+	Memory  *[]string          `json:"memory,omitempty"`
+	Storage *map[string]string `json:"storage,omitempty"`
+}
+
+// FormatLogs formats EVM returned structured logs for json output.
+func FormatLogs(logs []ethvm.StructLog) []StructLogRes {
+	formatted := make([]StructLogRes, len(logs))
+	for index, trace := range logs {
+		formatted[index] = StructLogRes{
+			Pc:      trace.Pc,
+			Op:      trace.Op.String(),
+			Gas:     trace.Gas,
+			GasCost: trace.GasCost,
+			Depth:   trace.Depth,
+			Error:   trace.Err,
+		}
+
+		if trace.Stack != nil {
+			stack := make([]string, len(trace.Stack))
+			for i, stackValue := range trace.Stack {
+				stack[i] = fmt.Sprintf("%x", ethmath.PaddedBigBytes(stackValue, 32))
+			}
+
+			formatted[index].Stack = &stack
+		}
+
+		if trace.Memory != nil {
+			memory := make([]string, 0, (len(trace.Memory)+31)/32)
+			for i := 0; i+32 <= len(trace.Memory); i += 32 {
+				memory = append(memory, fmt.Sprintf("%x", trace.Memory[i:i+32]))
+			}
+
+			formatted[index].Memory = &memory
+		}
+
+		if trace.Storage != nil {
+			storage := make(map[string]string)
+			for i, storageValue := range trace.Storage {
+				storage[fmt.Sprintf("%x", i)] = fmt.Sprintf("%x", storageValue)
+			}
+
+			formatted[index].Storage = &storage
+		}
+	}
+
+	return formatted
+}

test/importer/utils.go

@@ -0,0 +1,43 @@
+package importer
+
+import (
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/consensus/ethash"
+	ethstate "github.com/ethereum/go-ethereum/core/state"
+	ethtypes "github.com/ethereum/go-ethereum/core/types"
+	ethparams "github.com/ethereum/go-ethereum/params"
+)
+
+// Some weird constants to avoid constant memory allocs for them.
+var (
+	big8  = big.NewInt(8)
+	big32 = big.NewInt(32)
+)
+
+// accumulateRewards credits the coinbase of the given block with the mining
+// reward. The total reward consists of the static block reward and rewards for
+// included uncles. The coinbase of each uncle block is also rewarded.
+func accumulateRewards(config *ethparams.ChainConfig, state *ethstate.StateDB, header *ethtypes.Header, uncles []*ethtypes.Header) {
+	// select the correct block reward based on chain progression
+	blockReward := ethash.FrontierBlockReward
+	if config.IsByzantium(header.Number) {
+		blockReward = ethash.ByzantiumBlockReward
+	}
+
+	// accumulate the rewards for the miner and any included uncles
+	reward := new(big.Int).Set(blockReward)
+	r := new(big.Int)
+
+	for _, uncle := range uncles {
+		r.Add(uncle.Number, big8)
+		r.Sub(r, header.Number)
+		r.Mul(r, blockReward)
+		r.Div(r, big8)
+		state.AddBalance(uncle.Coinbase, r)
+		r.Div(blockReward, big32)
+		reward.Add(reward, r)
+	}
+
+	state.AddBalance(header.Coinbase, reward)
+}

test/run.go

@@ -0,0 +1,67 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"os/signal"
+	"path"
+	"runtime/pprof"
+	"syscall"
+
+	"github.com/cosmos/ethermint/state"
+	"github.com/cosmos/ethermint/test/importer"
+
+	dbm "github.com/tendermint/tendermint/libs/db"
+)
+
+var (
+	cpuprofile = flag.String("cpu-profile", "", "write cpu profile `file`")
+	blockchain = flag.String("blockchain", "data/blockchain", "file containing blocks to load")
+	datadir    = flag.String("datadir", path.Join(os.Getenv("HOME"), ".ethermint"), "directory for ethermint data")
+)
+
+func main() {
+	flag.Parse()
+
+	if *cpuprofile != "" {
+		f, err := os.Create(*cpuprofile)
+		if err != nil {
+			fmt.Printf("could not create CPU profile: %v\n", err)
+			return
+		}
+
+		if err := pprof.StartCPUProfile(f); err != nil {
+			fmt.Printf("could not start CPU profile: %v\n", err)
+			return
+		}
+
+		defer pprof.StopCPUProfile()
+	}
+
+	sigs := make(chan os.Signal, 1)
+	interruptCh := make(chan bool, 1)
+	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
+
+	go func() {
+		<-sigs
+		interruptCh <- true
+	}()
+
+	stateDB := dbm.NewDB("state", dbm.LevelDBBackend, *datadir)
+	codeDB := dbm.NewDB("code", dbm.LevelDBBackend, *datadir)
+
+	ethermintDB, err := state.NewDatabase(stateDB, codeDB)
+	if err != nil {
+		panic(fmt.Sprintf("failed to initialize geth Database: %v", err))
+	}
+
+	importer := importer.Importer{
+		EthermintDB:    ethermintDB,
+		BlockchainFile: *blockchain,
+		Datadir:        *datadir,
+		InterruptCh:    interruptCh,
+	}
+
+	importer.Import()
+}