cmd/evm: add --prestate, --sender, --json flags for fuzzing (#14476)

cmd/evm/json_logger.go

@@ -0,0 +1,60 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"encoding/json"
+	"io"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/math"
+	"github.com/ethereum/go-ethereum/core/vm"
+)
+
+type JSONLogger struct {
+	encoder *json.Encoder
+}
+
+func NewJSONLogger(writer io.Writer) *JSONLogger {
+	return &JSONLogger{json.NewEncoder(writer)}
+}
+
+// CaptureState outputs state information on the logger.
+func (l *JSONLogger) CaptureState(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost uint64, memory *vm.Memory, stack *vm.Stack, contract *vm.Contract, depth int, err error) error {
+	return l.encoder.Encode(vm.StructLog{
+		Pc:      pc,
+		Op:      op,
+		Gas:     gas + cost,
+		GasCost: cost,
+		Memory:  memory.Data(),
+		Stack:   stack.Data(),
+		Storage: nil,
+		Depth:   depth,
+		Err:     err,
+	})
+}
+
+// CaptureEnd is triggered at end of execution.
+func (l *JSONLogger) CaptureEnd(output []byte, gasUsed uint64, t time.Duration) error {
+	type endLog struct {
+		Output  string              `json:"output"`
+		GasUsed math.HexOrDecimal64 `json:"gasUsed"`
+		Time    time.Duration       `json:"time"`
+	}
+	return l.encoder.Encode(endLog{common.Bytes2Hex(output), math.HexOrDecimal64(gasUsed), t})
+}

cmd/evm/main.go

@@ -90,6 +90,18 @@ var (
 		Name:  "nogasmetering",
 		Usage: "disable gas metering",
 	}
+	GenesisFlag = cli.StringFlag{
+		Name:  "prestate",
+		Usage: "JSON file with prestate (genesis) config",
+	}
+	MachineFlag = cli.BoolFlag{
+		Name:  "json",
+		Usage: "output trace logs in machine readable format (json)",
+	}
+	SenderFlag = cli.StringFlag{
+		Name:  "sender",
+		Usage: "The transaction origin",
+	}
 )
 
 func init() {
@@ -108,6 +120,9 @@ func init() {
 		MemProfileFlag,
 		CPUProfileFlag,
 		StatDumpFlag,
+		GenesisFlag,
+		MachineFlag,
+		SenderFlag,
 	}
 	app.Commands = []cli.Command{
 		compileCommand,

cmd/evm/runner.go

@@ -18,6 +18,7 @@ package main
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -29,11 +30,13 @@ import (
 	"github.com/ethereum/go-ethereum/cmd/evm/internal/compiler"
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/core/vm/runtime"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/params"
 	cli "gopkg.in/urfave/cli.v1"
 )
 
@@ -45,17 +48,59 @@ var runCommand = cli.Command{
 	Description: `The run command runs arbitrary EVM code.`,
 }
 
+// readGenesis will read the given JSON format genesis file and return
+// the initialized Genesis structure
+func readGenesis(genesisPath string) *core.Genesis {
+	// Make sure we have a valid genesis JSON
+	//genesisPath := ctx.Args().First()
+	if len(genesisPath) == 0 {
+		utils.Fatalf("Must supply path to genesis JSON file")
+	}
+	file, err := os.Open(genesisPath)
+	if err != nil {
+		utils.Fatalf("Failed to read genesis file: %v", err)
+	}
+	defer file.Close()
+
+	genesis := new(core.Genesis)
+	if err := json.NewDecoder(file).Decode(genesis); err != nil {
+		utils.Fatalf("invalid genesis file: %v", err)
+	}
+	return genesis
+}
+
 func runCmd(ctx *cli.Context) error {
 	glogger := log.NewGlogHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(false)))
 	glogger.Verbosity(log.Lvl(ctx.GlobalInt(VerbosityFlag.Name)))
 	log.Root().SetHandler(glogger)
 
 	var (
-		db, _      = ethdb.NewMemDatabase()
+		tracer      vm.Tracer
-		statedb, _ = state.New(common.Hash{}, db)
+		debugLogger *vm.StructLogger
-		sender     = common.StringToAddress("sender")
+		statedb     *state.StateDB
-		logger     = vm.NewStructLogger(nil)
+		chainConfig *params.ChainConfig
+		sender      = common.StringToAddress("sender")
 	)
+	if ctx.GlobalBool(MachineFlag.Name) {
+		tracer = NewJSONLogger(os.Stdout)
+	} else if ctx.GlobalBool(DebugFlag.Name) {
+		debugLogger = vm.NewStructLogger(nil)
+		tracer = debugLogger
+	} else {
+		debugLogger = vm.NewStructLogger(nil)
+	}
+	if ctx.GlobalString(GenesisFlag.Name) != "" {
+		gen := readGenesis(ctx.GlobalString(GenesisFlag.Name))
+		_, statedb = gen.ToBlock()
+		chainConfig = gen.Config
+	} else {
+		var db, _ = ethdb.NewMemDatabase()
+		statedb, _ = state.New(common.Hash{}, db)
+	}
+	if ctx.GlobalString(SenderFlag.Name) != "" {
+		sender = common.HexToAddress(ctx.GlobalString(SenderFlag.Name))
+	}
+
 	statedb.CreateAccount(sender)
 
 	var (
@@ -95,16 +140,16 @@ func runCmd(ctx *cli.Context) error {
 		}
 		code = common.Hex2Bytes(string(bytes.TrimRight(hexcode, "\n")))
 	}
-
+	initialGas := ctx.GlobalUint64(GasFlag.Name)
 	runtimeConfig := runtime.Config{
 		Origin:   sender,
 		State:    statedb,
-		GasLimit: ctx.GlobalUint64(GasFlag.Name),
+		GasLimit: initialGas,
 		GasPrice: utils.GlobalBig(ctx, PriceFlag.Name),
 		Value:    utils.GlobalBig(ctx, ValueFlag.Name),
 		EVMConfig: vm.Config{
-			Tracer:             logger,
+			Tracer:             tracer,
-			Debug:              ctx.GlobalBool(DebugFlag.Name),
+			Debug:              ctx.GlobalBool(DebugFlag.Name) || ctx.GlobalBool(MachineFlag.Name),
 			DisableGasMetering: ctx.GlobalBool(DisableGasMeteringFlag.Name),
 		},
 	}
@@ -122,15 +167,19 @@ func runCmd(ctx *cli.Context) error {
 		defer pprof.StopCPUProfile()
 	}
 
+	if chainConfig != nil {
+		runtimeConfig.ChainConfig = chainConfig
+	}
 	tstart := time.Now()
+	var leftOverGas uint64
 	if ctx.GlobalBool(CreateFlag.Name) {
 		input := append(code, common.Hex2Bytes(ctx.GlobalString(InputFlag.Name))...)
-		ret, _, err = runtime.Create(input, &runtimeConfig)
+		ret, _, leftOverGas, err = runtime.Create(input, &runtimeConfig)
 	} else {
 		receiver := common.StringToAddress("receiver")
 		statedb.SetCode(receiver, code)
 
-		ret, err = runtime.Call(receiver, common.Hex2Bytes(ctx.GlobalString(InputFlag.Name)), &runtimeConfig)
+		ret, leftOverGas, err = runtime.Call(receiver, common.Hex2Bytes(ctx.GlobalString(InputFlag.Name)), &runtimeConfig)
 	}
 	execTime := time.Since(tstart)
 
@@ -153,8 +202,10 @@ func runCmd(ctx *cli.Context) error {
 	}
 
 	if ctx.GlobalBool(DebugFlag.Name) {
-		fmt.Fprintln(os.Stderr, "#### TRACE ####")
+		if debugLogger != nil {
-		vm.WriteTrace(os.Stderr, logger.StructLogs())
+			fmt.Fprintln(os.Stderr, "#### TRACE ####")
+			vm.WriteTrace(os.Stderr, debugLogger.StructLogs())
+		}
 		fmt.Fprintln(os.Stderr, "#### LOGS ####")
 		vm.WriteLogs(os.Stderr, statedb.Logs())
 	}
@@ -167,14 +218,18 @@ heap objects:       %d
 allocations:        %d
 total allocations:  %d
 GC calls:           %d
+Gas used:           %d
 
-`, execTime, mem.HeapObjects, mem.Alloc, mem.TotalAlloc, mem.NumGC)
+`, execTime, mem.HeapObjects, mem.Alloc, mem.TotalAlloc, mem.NumGC, initialGas-leftOverGas)
+	}
+	if tracer != nil {
+		tracer.CaptureEnd(ret, initialGas-leftOverGas, execTime)
+	} else {
+		fmt.Printf("0x%x\n", ret)
 	}
 
-	fmt.Printf("0x%x", ret)
 	if err != nil {
-		fmt.Printf(" error: %v", err)
+		fmt.Printf(" error: %v\n", err)
 	}
-	fmt.Println()
 	return nil
 }

core/vm/gen_structlog.go

@@ -0,0 +1,95 @@
+// Code generated by github.com/fjl/gencodec. DO NOT EDIT.
+
+package vm
+
+import (
+	"encoding/json"
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/common/math"
+)
+
+func (s StructLog) MarshalJSON() ([]byte, error) {
+	type StructLog struct {
+		Pc         uint64                      `json:"pc"`
+		Op         OpCode                      `json:"op"`
+		Gas        math.HexOrDecimal64         `json:"gas"`
+		GasCost    math.HexOrDecimal64         `json:"gasCost"`
+		Memory     hexutil.Bytes               `json:"memory"`
+		Stack      []*math.HexOrDecimal256     `json:"stack"`
+		Storage    map[common.Hash]common.Hash `json:"-"`
+		Depth      int                         `json:"depth"`
+		Err        error                       `json:"error"`
+		OpName     string                      `json:"opName"`
+		MemorySize int                         `json:"memSize"`
+	}
+	var enc StructLog
+	enc.Pc = s.Pc
+	enc.Op = s.Op
+	enc.Gas = math.HexOrDecimal64(s.Gas)
+	enc.GasCost = math.HexOrDecimal64(s.GasCost)
+	enc.Memory = s.Memory
+	if s.Stack != nil {
+		enc.Stack = make([]*math.HexOrDecimal256, len(s.Stack))
+		for k, v := range s.Stack {
+			enc.Stack[k] = (*math.HexOrDecimal256)(v)
+		}
+	}
+	enc.Storage = s.Storage
+	enc.Depth = s.Depth
+	enc.Err = s.Err
+	enc.OpName = s.OpName()
+	enc.MemorySize = s.MemorySize()
+	return json.Marshal(&enc)
+}
+
+func (s *StructLog) UnmarshalJSON(input []byte) error {
+	type StructLog struct {
+		Pc      *uint64                     `json:"pc"`
+		Op      *OpCode                     `json:"op"`
+		Gas     *math.HexOrDecimal64        `json:"gas"`
+		GasCost *math.HexOrDecimal64        `json:"gasCost"`
+		Memory  hexutil.Bytes               `json:"memory"`
+		Stack   []*math.HexOrDecimal256     `json:"stack"`
+		Storage map[common.Hash]common.Hash `json:"-"`
+		Depth   *int                        `json:"depth"`
+		Err     *error                      `json:"error"`
+	}
+	var dec StructLog
+	if err := json.Unmarshal(input, &dec); err != nil {
+		return err
+	}
+	if dec.Pc != nil {
+		s.Pc = *dec.Pc
+	}
+	if dec.Op != nil {
+		s.Op = *dec.Op
+	}
+	if dec.Gas != nil {
+		s.Gas = uint64(*dec.Gas)
+	}
+	if dec.GasCost != nil {
+		s.GasCost = uint64(*dec.GasCost)
+	}
+	if dec.Memory != nil {
+		s.Memory = dec.Memory
+	}
+	if dec.Stack != nil {
+		s.Stack = make([]*big.Int, len(dec.Stack))
+		for k, v := range dec.Stack {
+			s.Stack[k] = (*big.Int)(v)
+		}
+	}
+	if dec.Storage != nil {
+		s.Storage = dec.Storage
+	}
+	if dec.Depth != nil {
+		s.Depth = *dec.Depth
+	}
+	if dec.Err != nil {
+		s.Err = *dec.Err
+	}
+	return nil
+}

core/vm/logger.go

@@ -21,8 +21,10 @@ import (
 	"fmt"
 	"io"
 	"math/big"
+	"time"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/common/math"
 	"github.com/ethereum/go-ethereum/core/types"
 )
@@ -47,18 +49,38 @@ type LogConfig struct {
 	Limit          int  // maximum length of output, but zero means unlimited
 }
 
+//go:generate gencodec -type StructLog -field-override structLogMarshaling -out gen_structlog.go
+
 // StructLog is emitted to the EVM each cycle and lists information about the current internal state
 // prior to the execution of the statement.
 type StructLog struct {
-	Pc      uint64
+	Pc      uint64                      `json:"pc"`
-	Op      OpCode
+	Op      OpCode                      `json:"op"`
-	Gas     uint64
+	Gas     uint64                      `json:"gas"`
-	GasCost uint64
+	GasCost uint64                      `json:"gasCost"`
-	Memory  []byte
+	Memory  []byte                      `json:"memory"`
-	Stack   []*big.Int
+	Stack   []*big.Int                  `json:"stack"`
-	Storage map[common.Hash]common.Hash
+	Storage map[common.Hash]common.Hash `json:"-"`
-	Depth   int
+	Depth   int                         `json:"depth"`
-	Err     error
+	Err     error                       `json:"error"`
+}
+
+// overrides for gencodec
+type structLogMarshaling struct {
+	Stack      []*math.HexOrDecimal256
+	Gas        math.HexOrDecimal64
+	GasCost    math.HexOrDecimal64
+	Memory     hexutil.Bytes
+	OpName     string `json:"opName"`
+	MemorySize int    `json:"memSize"`
+}
+
+func (s *StructLog) OpName() string {
+	return s.Op.String()
+}
+
+func (s *StructLog) MemorySize() int {
+	return len(s.Memory)
 }
 
 // Tracer is used to collect execution traces from an EVM transaction
@@ -68,6 +90,7 @@ type StructLog struct {
 // if you need to retain them beyond the current call.
 type Tracer interface {
 	CaptureState(env *EVM, pc uint64, op OpCode, gas, cost uint64, memory *Memory, stack *Stack, contract *Contract, depth int, err error) error
+	CaptureEnd(output []byte, gasUsed uint64, t time.Duration) error
 }
 
 // StructLogger is an EVM state logger and implements Tracer.
@@ -82,7 +105,7 @@ type StructLogger struct {
 	changedValues map[common.Address]Storage
 }
 
-// NewLogger returns a new logger
+// NewStructLogger returns a new logger
 func NewStructLogger(cfg *LogConfig) *StructLogger {
 	logger := &StructLogger{
 		changedValues: make(map[common.Address]Storage),
@@ -93,9 +116,9 @@ func NewStructLogger(cfg *LogConfig) *StructLogger {
 	return logger
 }
 
-// captureState logs a new structured log message and pushes it out to the environment
+// CaptureState logs a new structured log message and pushes it out to the environment
 //
-// captureState also tracks SSTORE ops to track dirty values.
+// CaptureState also tracks SSTORE ops to track dirty values.
 func (l *StructLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost uint64, memory *Memory, stack *Stack, contract *Contract, depth int, err error) error {
 	// check if already accumulated the specified number of logs
 	if l.cfg.Limit != 0 && l.cfg.Limit <= len(l.logs) {
@@ -164,6 +187,11 @@ func (l *StructLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost ui
 	return nil
 }
 
+func (l *StructLogger) CaptureEnd(output []byte, gasUsed uint64, t time.Duration) error {
+	fmt.Printf("0x%x", output)
+	return nil
+}
+
 // StructLogs returns a list of captured log entries
 func (l *StructLogger) StructLogs() []StructLog {
 	return l.logs

core/vm/runtime/runtime.go

@@ -125,7 +125,7 @@ func Execute(code, input []byte, cfg *Config) ([]byte, *state.StateDB, error) {
 }
 
 // Create executes the code using the EVM create method
-func Create(input []byte, cfg *Config) ([]byte, common.Address, error) {
+func Create(input []byte, cfg *Config) ([]byte, common.Address, uint64, error) {
 	if cfg == nil {
 		cfg = new(Config)
 	}
@@ -141,13 +141,13 @@ func Create(input []byte, cfg *Config) ([]byte, common.Address, error) {
 	)
 
 	// Call the code with the given configuration.
-	code, address, _, err := vmenv.Create(
+	code, address, leftOverGas, err := vmenv.Create(
 		sender,
 		input,
 		cfg.GasLimit,
 		cfg.Value,
 	)
-	return code, address, err
+	return code, address, leftOverGas, err
 }
 
 // Call executes the code given by the contract's address. It will return the
@@ -155,14 +155,14 @@ func Create(input []byte, cfg *Config) ([]byte, common.Address, error) {
 //
 // Call, unlike Execute, requires a config and also requires the State field to
 // be set.
-func Call(address common.Address, input []byte, cfg *Config) ([]byte, error) {
+func Call(address common.Address, input []byte, cfg *Config) ([]byte, uint64, error) {
 	setDefaults(cfg)
 
 	vmenv := NewEnv(cfg, cfg.State)
 
 	sender := cfg.State.GetOrNewStateObject(cfg.Origin)
 	// Call the code with the given configuration.
-	ret, _, err := vmenv.Call(
+	ret, leftOverGas, err := vmenv.Call(
 		sender,
 		address,
 		input,
@@ -170,5 +170,5 @@ func Call(address common.Address, input []byte, cfg *Config) ([]byte, error) {
 		cfg.Value,
 	)
 
-	return ret, err
+	return ret, leftOverGas, err
 }

core/vm/runtime/runtime_test.go

@@ -106,7 +106,7 @@ func TestCall(t *testing.T) {
 		byte(vm.RETURN),
 	})
 
-	ret, err := Call(address, nil, &Config{State: state})
+	ret, _, err := Call(address, nil, &Config{State: state})
 	if err != nil {
 		t.Fatal("didn't expect error", err)
 	}

internal/ethapi/tracer.go

@@ -21,6 +21,7 @@ import (
 	"errors"
 	"fmt"
 	"math/big"
+	"time"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
@@ -344,6 +345,12 @@ func (jst *JavascriptTracer) CaptureState(env *vm.EVM, pc uint64, op vm.OpCode,
 	return nil
 }
 
+// CaptureEnd is called after the call finishes
+func (jst *JavascriptTracer) CaptureEnd(output []byte, gasUsed uint64, t time.Duration) error {
+	//TODO! @Arachnid please figure out of there's anything we can use this method for
+	return nil
+}
+
 // GetResult calls the Javascript 'result' function and returns its value, or any accumulated error
 func (jst *JavascriptTracer) GetResult() (result interface{}, err error) {
 	if jst.err != nil {