diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
new file mode 100644
index 000000000..8eb113c34
--- /dev/null
+++ b/.github/workflows/build.yml
@@ -0,0 +1,12 @@
+name: Docker Build
+
+on: [pull_request]
+
+jobs:
+ build:
+ name: Run docker build
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2
+ - name: Run docker build
+ run: docker build -t vulcanize/go-ethereum -f Dockerfile.amd64 .
diff --git a/.github/workflows/on-master.yaml b/.github/workflows/on-master.yaml
new file mode 100644
index 000000000..e53dc88b5
--- /dev/null
+++ b/.github/workflows/on-master.yaml
@@ -0,0 +1,27 @@
+name: Docker Build and publish to Github
+
+on:
+ push:
+ branches:
+ - v1.9.24-statediff
+ - v1.9.23-statediff
+ - v1.9.11-statediff
+
+jobs:
+ build:
+ name: Run docker build and publish
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2
+ - name: Run docker build
+ run: docker build -t vulcanize/go-ethereum -f Dockerfile.amd64 .
+ - name: Get the version
+ id: vars
+ run: echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
+ - name: Tag docker image
+ run: docker tag vulcanize/go-ethereum docker.pkg.github.com/vulcanize/go-ethereum/go-ethereum:${{steps.vars.outputs.sha}}
+ - name: Docker Login
+ run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
+ - name: Docker Push
+ run: docker push docker.pkg.github.com/vulcanize/go-ethereum/go-ethereum:${{steps.vars.outputs.sha}}
+
diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml
new file mode 100644
index 000000000..3fadf7926
--- /dev/null
+++ b/.github/workflows/publish.yaml
@@ -0,0 +1,34 @@
+name: Publish geth to release
+on:
+ release:
+ types: [published]
+jobs:
+ push_to_registries:
+ name: Publish assets to Release
+ runs-on: ubuntu-latest
+ steps:
+ - name: Get the version
+ id: vars
+ run: |
+ echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
+ - name: Docker Login to Github Registry
+ run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
+ - name: Docker Pull
+ run: docker pull docker.pkg.github.com/vulcanize/go-ethereum/go-ethereum:${{steps.vars.outputs.sha}}
+ - name: Copy ethereum binary file
+ run: docker run --rm --entrypoint cat docker.pkg.github.com/vulcanize/go-ethereum/go-ethereum:${{steps.vars.outputs.sha}} /go-ethereum/build/bin/geth > geth-linux-amd64
+ - name: Get release
+ id: get_release
+ uses: bruceadams/get-release@v1.2.0
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ - name: Upload Release Asset
+ id: upload-release-asset
+ uses: actions/upload-release-asset@v1
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ with:
+ upload_url: ${{ steps.get_release.outputs.upload_url }}
+ asset_path: geth-linux-amd64
+ asset_name: geth-linux-amd64
+ asset_content_type: application/octet-stream
diff --git a/Dockerfile.amd64 b/Dockerfile.amd64
new file mode 100644
index 000000000..7a35376c9
--- /dev/null
+++ b/Dockerfile.amd64
@@ -0,0 +1,7 @@
+# Build Geth in a stock Go builder container
+FROM golang:1.15.5 as builder
+
+#RUN apk add --no-cache make gcc musl-dev linux-headers git
+
+ADD . /go-ethereum
+RUN cd /go-ethereum && make geth
diff --git a/cmd/geth/config.go b/cmd/geth/config.go
index bf1fc55b1..dfb0d4aa7 100644
--- a/cmd/geth/config.go
+++ b/cmd/geth/config.go
@@ -25,6 +25,8 @@ import (
"unicode"
"gopkg.in/urfave/cli.v1"
+ "github.com/ethereum/go-ethereum/eth/downloader"
+ "github.com/ethereum/go-ethereum/statediff"
"github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/eth"
@@ -145,6 +147,9 @@ func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) {
cfg.Ethstats.URL = ctx.GlobalString(utils.EthStatsURLFlag.Name)
}
utils.SetShhConfig(ctx, stack)
+ if ctx.GlobalBool(utils.StateDiffFlag.Name) {
+ cfg.Eth.Diffing = true
+ }
return stack, cfg
}
@@ -162,18 +167,67 @@ func checkWhisper(ctx *cli.Context) {
func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) {
stack, cfg := makeConfigNode(ctx)
+ if cfg.Eth.SyncMode == downloader.LightSync {
+ return makeLightNode(ctx, stack, cfg)
+ }
+
backend := utils.RegisterEthService(stack, &cfg.Eth)
checkWhisper(ctx)
+
+ if ctx.GlobalBool(utils.StateDiffFlag.Name) {
+ var dbParams *statediff.DBParams
+ if ctx.GlobalIsSet(utils.StateDiffDBFlag.Name) {
+ dbParams = new(statediff.DBParams)
+ dbParams.ConnectionURL = ctx.GlobalString(utils.StateDiffDBFlag.Name)
+ if ctx.GlobalIsSet(utils.StateDiffDBNodeIDFlag.Name) {
+ dbParams.ID = ctx.GlobalString(utils.StateDiffDBNodeIDFlag.Name)
+ } else {
+ utils.Fatalf("Must specify node ID for statediff DB output")
+ }
+ if ctx.GlobalIsSet(utils.StateDiffDBClientNameFlag.Name) {
+ dbParams.ClientName = ctx.GlobalString(utils.StateDiffDBClientNameFlag.Name)
+ } else {
+ utils.Fatalf("Must specify client name for statediff DB output")
+ }
+ } else {
+ if ctx.GlobalBool(utils.StateDiffWritingFlag.Name) {
+ utils.Fatalf("Must pass DB parameters if enabling statediff write loop")
+ }
+ }
+ params := statediff.ServiceParams{
+ DBParams: dbParams,
+ EnableWriteLoop: ctx.GlobalBool(utils.StateDiffWritingFlag.Name),
+ NumWorkers: ctx.GlobalUint(utils.StateDiffWorkersFlag.Name),
+ }
+ utils.RegisterStateDiffService(stack, backend, params)
+ }
+
// Configure GraphQL if requested
if ctx.GlobalIsSet(utils.GraphQLEnabledFlag.Name) {
- utils.RegisterGraphQLService(stack, backend, cfg.Node)
+ utils.RegisterGraphQLService(stack, backend.APIBackend, cfg.Node)
}
// Add the Ethereum Stats daemon if requested.
if cfg.Ethstats.URL != "" {
- utils.RegisterEthStatsService(stack, backend, cfg.Ethstats.URL)
+ utils.RegisterEthStatsService(stack, backend.APIBackend, cfg.Ethstats.URL)
}
- return stack, backend
+ return stack, backend.APIBackend
+}
+
+func makeLightNode(ctx *cli.Context, stack *node.Node, cfg gethConfig) (*node.Node, ethapi.Backend) {
+ backend := utils.RegisterLesEthService(stack, &cfg.Eth)
+
+ checkWhisper(ctx)
+
+ // Configure GraphQL if requested
+ if ctx.GlobalIsSet(utils.GraphQLEnabledFlag.Name) {
+ utils.RegisterGraphQLService(stack, backend.ApiBackend, cfg.Node)
+ }
+ // Add the Ethereum Stats daemon if requested.
+ if cfg.Ethstats.URL != "" {
+ utils.RegisterEthStatsService(stack, backend.ApiBackend, cfg.Ethstats.URL)
+ }
+ return stack, backend.ApiBackend
}
// dumpConfig is the dumpconfig command.
diff --git a/cmd/geth/main.go b/cmd/geth/main.go
index e587a5f86..01fedd489 100644
--- a/cmd/geth/main.go
+++ b/cmd/geth/main.go
@@ -158,6 +158,12 @@ var (
utils.GpoMaxGasPriceFlag,
utils.EWASMInterpreterFlag,
utils.EVMInterpreterFlag,
+ utils.StateDiffFlag,
+ utils.StateDiffDBFlag,
+ utils.StateDiffDBNodeIDFlag,
+ utils.StateDiffDBClientNameFlag,
+ utils.StateDiffWritingFlag,
+ utils.StateDiffWorkersFlag,
configFileFlag,
}
diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go
index 0e70451ed..d059912ec 100644
--- a/cmd/geth/usage.go
+++ b/cmd/geth/usage.go
@@ -235,6 +235,17 @@ var AppHelpFlagGroups = []flags.FlagGroup{
utils.LegacyGraphQLPortFlag,
}, debug.DeprecatedFlags...),
},
+ {
+ Name: "STATE DIFF",
+ Flags: []cli.Flag{
+ utils.StateDiffFlag,
+ utils.StateDiffDBFlag,
+ utils.StateDiffDBNodeIDFlag,
+ utils.StateDiffDBClientNameFlag,
+ utils.StateDiffWritingFlag,
+ utils.StateDiffWorkersFlag,
+ },
+ },
{
Name: "MISC",
Flags: []cli.Flag{
diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go
index 051bdd630..0ea7d203a 100644
--- a/cmd/utils/flags.go
+++ b/cmd/utils/flags.go
@@ -63,6 +63,8 @@ import (
"github.com/ethereum/go-ethereum/p2p/nat"
"github.com/ethereum/go-ethereum/p2p/netutil"
"github.com/ethereum/go-ethereum/params"
+ "github.com/ethereum/go-ethereum/statediff"
+
pcsclite "github.com/gballet/go-libpcsclite"
"gopkg.in/urfave/cli.v1"
)
@@ -726,6 +728,31 @@ var (
Usage: "External EVM configuration (default = built-in interpreter)",
Value: "",
}
+
+ StateDiffFlag = cli.BoolFlag{
+ Name: "statediff",
+ Usage: "Enables the processing of state diffs between each block",
+ }
+ StateDiffDBFlag = cli.StringFlag{
+ Name: "statediff.db",
+ Usage: "PostgreSQL database connection string for writing state diffs",
+ }
+ StateDiffDBNodeIDFlag = cli.StringFlag{
+ Name: "statediff.dbnodeid",
+ Usage: "Node ID to use when writing state diffs to database",
+ }
+ StateDiffDBClientNameFlag = cli.StringFlag{
+ Name: "statediff.dbclientname",
+ Usage: "Client name to use when writing state diffs to database",
+ }
+ StateDiffWritingFlag = cli.BoolFlag{
+ Name: "statediff.writing",
+ Usage: "Activates progressive writing of state diffs to database as new block are synced",
+ }
+ StateDiffWorkersFlag = cli.UintFlag{
+ Name: "statediff.workers",
+ Usage: "Number of concurrent workers to use during statediff processing (0 = 1)",
+ }
)
// MakeDataDir retrieves the currently requested data directory, terminating
@@ -991,6 +1018,9 @@ func setWS(ctx *cli.Context, cfg *node.Config) {
if ctx.GlobalIsSet(WSApiFlag.Name) {
cfg.WSModules = SplitAndTrim(ctx.GlobalString(WSApiFlag.Name))
}
+ if ctx.GlobalBool(StateDiffFlag.Name) {
+ cfg.WSModules = append(cfg.WSModules, "statediff")
+ }
}
// setIPC creates an IPC path configuration from the set command line flags,
@@ -1690,14 +1720,8 @@ func SetDNSDiscoveryDefaults(cfg *eth.Config, genesis common.Hash) {
}
// RegisterEthService adds an Ethereum client to the stack.
-func RegisterEthService(stack *node.Node, cfg *eth.Config) ethapi.Backend {
- if cfg.SyncMode == downloader.LightSync {
- backend, err := les.New(stack, cfg)
- if err != nil {
- Fatalf("Failed to register the Ethereum service: %v", err)
- }
- return backend.ApiBackend
- }
+// RegisterEthService adds an Ethereum client to the stack.
+func RegisterEthService(stack *node.Node, cfg *eth.Config) *eth.Ethereum {
backend, err := eth.New(stack, cfg)
if err != nil {
Fatalf("Failed to register the Ethereum service: %v", err)
@@ -1708,7 +1732,16 @@ func RegisterEthService(stack *node.Node, cfg *eth.Config) ethapi.Backend {
Fatalf("Failed to create the LES server: %v", err)
}
}
- return backend.APIBackend
+ return backend
+}
+
+// RegisterLesEthService adds an Ethereum les client to the stack.
+func RegisterLesEthService(stack *node.Node, cfg *eth.Config) *les.LightEthereum {
+ backend, err := les.New(stack, cfg)
+ if err != nil {
+ Fatalf("Failed to register the Ethereum service: %v", err)
+ }
+ return backend
}
// RegisterEthStatsService configures the Ethereum Stats daemon and adds it to
@@ -1726,6 +1759,13 @@ func RegisterGraphQLService(stack *node.Node, backend ethapi.Backend, cfg node.C
}
}
+// RegisterStateDiffService configures and registers a service to stream state diff data over RPC
+func RegisterStateDiffService(stack *node.Node, ethServ *eth.Ethereum, params statediff.ServiceParams) {
+ if err := statediff.New(stack, ethServ, params); err != nil {
+ Fatalf("Failed to register the Statediff service: %v", err)
+ }
+}
+
func SetupMetrics(ctx *cli.Context) {
if metrics.Enabled {
log.Info("Enabling metrics collection")
diff --git a/core/blockchain.go b/core/blockchain.go
index bc1db49f3..08f8927d8 100644
--- a/core/blockchain.go
+++ b/core/blockchain.go
@@ -132,6 +132,7 @@ type CacheConfig struct {
Preimages bool // Whether to store preimage of trie key to the disk
SnapshotWait bool // Wait for snapshot construction on startup. TODO(karalabe): This is a dirty hack for testing, nuke it
+ StateDiffing bool // Whether or not the statediffing service is running
}
// defaultCacheConfig are the default caching values if none are specified by the
@@ -212,6 +213,10 @@ type BlockChain struct {
shouldPreserve func(*types.Block) bool // Function used to determine whether should preserve the given block.
terminateInsert func(common.Hash, uint64) bool // Testing hook used to terminate ancient receipt chain insertion.
writeLegacyJournal bool // Testing flag used to flush the snapshot journal in legacy format.
+
+ // Locked roots and their mutex
+ trieLock sync.Mutex
+ lockedRoots map[common.Hash]bool
}
// NewBlockChain returns a fully initialised block chain using information
@@ -228,7 +233,6 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par
txLookupCache, _ := lru.New(txLookupCacheLimit)
futureBlocks, _ := lru.New(maxFutureBlocks)
badBlocks, _ := lru.New(badBlockLimit)
-
bc := &BlockChain{
chainConfig: chainConfig,
cacheConfig: cacheConfig,
@@ -250,6 +254,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par
engine: engine,
vmConfig: vmConfig,
badBlocks: badBlocks,
+ lockedRoots: make(map[common.Hash]bool),
}
bc.validator = NewBlockValidator(chainConfig, bc, engine)
bc.prefetcher = newStatePrefetcher(chainConfig, bc, engine)
@@ -1040,7 +1045,10 @@ func (bc *BlockChain) Stop() {
}
}
for !bc.triegc.Empty() {
- triedb.Dereference(bc.triegc.PopItem().(common.Hash))
+ pruneRoot := bc.triegc.PopItem().(common.Hash)
+ if !bc.TrieLocked(pruneRoot) {
+ triedb.Dereference(pruneRoot)
+ }
}
if size, _ := triedb.Size(); size != 0 {
log.Error("Dangling trie nodes after full cleanup")
@@ -1546,6 +1554,11 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.
triedb.Reference(root, common.Hash{}) // metadata reference to keep trie alive
bc.triegc.Push(root, -int64(block.NumberU64()))
+ // If we are statediffing, lock the trie until the statediffing service is done using it
+ if bc.cacheConfig.StateDiffing {
+ bc.LockTrie(root)
+ }
+
if current := block.NumberU64(); current > TriesInMemory {
// If we exceeded our memory allowance, flush matured singleton nodes to disk
var (
@@ -1584,7 +1597,11 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.
bc.triegc.Push(root, number)
break
}
- triedb.Dereference(root.(common.Hash))
+ pruneRoot := root.(common.Hash)
+ if !bc.TrieLocked(pruneRoot) {
+ log.Debug("Dereferencing", "root", root.(common.Hash).Hex())
+ triedb.Dereference(pruneRoot)
+ }
}
}
}
@@ -2556,3 +2573,28 @@ func (bc *BlockChain) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscript
func (bc *BlockChain) SubscribeBlockProcessingEvent(ch chan<- bool) event.Subscription {
return bc.scope.Track(bc.blockProcFeed.Subscribe(ch))
}
+
+// TrieLocked returns whether the trie associated with the provided root is locked for use
+func (bc *BlockChain) TrieLocked(root common.Hash) bool {
+ bc.trieLock.Lock()
+ locked, ok := bc.lockedRoots[root]
+ bc.trieLock.Unlock()
+ if !ok {
+ return false
+ }
+ return locked
+}
+
+// LockTrie prevents dereferencing of the provided root
+func (bc *BlockChain) LockTrie(root common.Hash) {
+ bc.trieLock.Lock()
+ bc.lockedRoots[root] = true
+ bc.trieLock.Unlock()
+}
+
+// UnlockTrie allows dereferencing of the provided root- provided it was previously locked
+func (bc *BlockChain) UnlockTrie(root common.Hash) {
+ bc.trieLock.Lock()
+ bc.lockedRoots[root] = false
+ bc.trieLock.Unlock()
+}
diff --git a/eth/backend.go b/eth/backend.go
index 03b0b319b..5fcf522c2 100644
--- a/eth/backend.go
+++ b/eth/backend.go
@@ -170,6 +170,7 @@ func New(stack *node.Node, config *Config) (*Ethereum, error) {
TrieTimeLimit: config.TrieTimeout,
SnapshotLimit: config.SnapshotCache,
Preimages: config.Preimages,
+ StateDiffing: config.Diffing,
}
)
eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, chainConfig, eth.engine, vmConfig, eth.shouldPreserve, &config.TxLookupLimit)
diff --git a/eth/config.go b/eth/config.go
index 0d90376d9..b9e6e4bcd 100644
--- a/eth/config.go
+++ b/eth/config.go
@@ -187,4 +187,8 @@ type Config struct {
// CheckpointOracle is the configuration for checkpoint oracle.
CheckpointOracle *params.CheckpointOracleConfig `toml:",omitempty"`
+
+ // Signify whether or not we are producing statediffs
+ // If we are, do not dereference state roots until the statediffing service is done with them
+ Diffing bool
}
diff --git a/go.mod b/go.mod
index 0f47809ef..6d8dfc901 100644
--- a/go.mod
+++ b/go.mod
@@ -35,20 +35,30 @@ require (
github.com/holiman/uint256 v1.1.1
github.com/huin/goupnp v1.0.0
github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883
+ github.com/ipfs/go-block-format v0.0.2
+ github.com/ipfs/go-cid v0.0.7
+ github.com/ipfs/go-ipfs-blockstore v1.0.1
+ github.com/ipfs/go-ipfs-ds-help v1.0.0
+ github.com/ipfs/go-ipld-format v0.2.0
github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458
github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e
- github.com/julienschmidt/httprouter v1.1.1-0.20170430222011-975b5c4c7c21
+ github.com/jmoiron/sqlx v1.2.0
+ github.com/julienschmidt/httprouter v1.2.0
github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356
- github.com/kr/pretty v0.1.0 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
- github.com/mattn/go-colorable v0.1.0
- github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035
+ github.com/lib/pq v1.8.0
+ github.com/mattn/go-colorable v0.1.1
+ github.com/mattn/go-isatty v0.0.5
+ github.com/multiformats/go-multihash v0.0.14
github.com/naoina/go-stringutil v0.1.0 // indirect
github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416
github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c
+ github.com/onsi/ginkgo v1.14.0
+ github.com/onsi/gomega v1.10.1
github.com/pborman/uuid v0.0.0-20170112150404-1b00554d8222
github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7
- github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150
+ github.com/prometheus/client_golang v0.9.3
+ github.com/prometheus/tsdb v0.7.1
github.com/rjeczalik/notify v0.9.1
github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00
github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521 // indirect
diff --git a/go.sum b/go.sum
index d1bd60858..8a06d1129 100644
--- a/go.sum
+++ b/go.sum
@@ -1,3 +1,16 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
+cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
+cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
+cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
+dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4=
github.com/Azure/azure-pipeline-go v0.2.2 h1:6oiIS9yaG6XCCzhgAgKFfIWyo4LLCiDhZot6ltoThhY=
github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc=
@@ -32,9 +45,16 @@ github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah
github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847 h1:rtI0fD4oG/8eVokGVPYJEW1F88p1ZNgXiEIs9thEE4A=
github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ=
+github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
+github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
+github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/aws/aws-sdk-go v1.25.48 h1:J82DYDGZHOKHdhx6hD24Tm30c2C3GchYGfN0mf9iKUk=
github.com/aws/aws-sdk-go v1.25.48/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6 h1:Eey/GGQ/E5Xp1P2Lyx1qj007hLZfbi0+CoVeJruGCtI=
github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ=
github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk=
@@ -43,8 +63,14 @@ github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9 h1:J82+/8rub3qSy0HxEnoYD8cs+HDlHWYrqYXe2Vqxluk=
github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U=
+github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
+github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -66,6 +92,8 @@ github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c h1:JHHhtb9XWJrGNMcr
github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
github.com/fatih/color v1.3.0 h1:YehCCcyeQ6Km0D6+IapqPinWBK6y+0eB5umvZXK9WPs=
github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
+github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc h1:jtW8jbpkO4YirRSyepBOH8E+2HEw6/hKkBvFPwhUN8c=
github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
@@ -74,18 +102,30 @@ github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWo
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI=
github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/go-kit/kit v0.8.0 h1:Wz+5lgoB0kkuqLEc6NVmwRknTKP6dTGbSqvhZtBI/j0=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-logfmt/logfmt v0.3.0 h1:8HUsc87TaSWLKwrnumgC8/YconD2fJQsRJAsWaPg2ic=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E=
github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
github.com/go-sourcemap/sourcemap v2.1.2+incompatible h1:0b/xya7BKGhXuqFESKM4oIiRo9WOt2ebz7KxfreD6ug=
github.com/go-sourcemap/sourcemap v2.1.2+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
+github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
+github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
@@ -108,10 +148,16 @@ github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa h1:Q75Upo5UN4JbPFURXZ8nLKYUvF85dyFRop/vQ0Rv+64=
github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989 h1:giknQ4mEuDFmmHSrGcbargOuLHQGtywqo4mheITex54=
github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
+github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277 h1:E0whKxgp2ojts0FDgUA8dl62bmH0LxKanMoBr6MDTDM=
github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc=
+github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU=
+github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48=
github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
github.com/holiman/uint256 v1.1.1 h1:4JywC80b+/hSfljFlEBLHrrh+CIONLDz9NuFl0af4Mw=
@@ -123,16 +169,54 @@ github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7
github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o=
github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883 h1:FSeK4fZCo8u40n2JMnyAsd6x7+SbvoOMHvQOU/n10P4=
github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY=
+github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
+github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
+github.com/ipfs/go-block-format v0.0.2 h1:qPDvcP19izTjU8rgo6p7gTXZlkMkF5bz5G3fqIsSCPE=
+github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=
+github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
+github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
+github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog=
+github.com/ipfs/go-cid v0.0.7 h1:ysQJVJA3fNDF1qigJbsSQOdjhVLsOEoPdh0+R97k3jY=
+github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I=
+github.com/ipfs/go-datastore v0.4.1/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA=
+github.com/ipfs/go-datastore v0.4.2 h1:h8/n7WPzhp239kkLws+epN3Ic7YtcBPgcaXfEfdVDWM=
+github.com/ipfs/go-datastore v0.4.2/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA=
+github.com/ipfs/go-ipfs-blockstore v1.0.1 h1:fnuVj4XdZp4yExhd0CnUwAiMNJHiPnfInhiuwz4lW1w=
+github.com/ipfs/go-ipfs-blockstore v1.0.1/go.mod h1:MGNZlHNEnR4KGgPHM3/k8lBySIOK2Ve+0KjZubKlaOE=
+github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw=
+github.com/ipfs/go-ipfs-ds-help v1.0.0 h1:bEQ8hMGs80h0sR8O4tfDgV6B01aaF9qeTrujrTLYV3g=
+github.com/ipfs/go-ipfs-ds-help v1.0.0/go.mod h1:ujAbkeIgkKAWtxxNkoZHWLCyk5JpPoKnGyCcsoF6ueE=
+github.com/ipfs/go-ipfs-util v0.0.1 h1:Wz9bL2wB2YBJqggkA4dD7oSmqB4cAnpNbGrlHJulv50=
+github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc=
+github.com/ipfs/go-ipld-format v0.2.0 h1:xGlJKkArkmBvowr+GMCX0FEZtkro71K1AwiKnL37mwA=
+github.com/ipfs/go-ipld-format v0.2.0/go.mod h1:3l3C1uKoadTPbeNfrDi+xMInYKlx2Cvg1BuydPSdzQs=
+github.com/ipfs/go-log v0.0.1 h1:9XTUN/rW64BCG1YhPK9Hoy3q8nr4gOmHHBpgFdfw6Lc=
+github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM=
+github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg=
+github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY=
github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 h1:6OvNmYgJyexcZ3pYbTI9jWx5tHo1Dee/tWbLMfPe2TA=
github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
+github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8 h1:bspPhN+oKYFk5fcGNuQzp6IGzYQSenLEgH3s6jkXrWw=
+github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY=
github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e h1:UvSe12bq+Uj2hWd8aOlwPmoZ+CITRFrdit+sDGfAg8U=
github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA=
+github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
+github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/julienschmidt/httprouter v1.1.1-0.20170430222011-975b5c4c7c21 h1:F/iKcka0K2LgnKy/fgSBf235AETtm1n1TvBzqu40LE0=
github.com/julienschmidt/httprouter v1.1.1-0.20170430222011-975b5c4c7c21/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356 h1:I/yrLt2WilKxlQKCM52clh5rGzTKpVctGT1lH4Dc8Jw=
github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU=
+github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
@@ -142,17 +226,64 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg=
+github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
+github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM=
+github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
+github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-colorable v0.1.0 h1:v2XXALHHh6zHfYTJ+cSkwtyffnaOyR1MXaA91mTrb8o=
github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg=
+github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc=
github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d h1:oNAwILwmgWKFpuU+dXvI6dl9jG2mAWAZLX3r9s0PPiw=
github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc=
+github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035 h1:USWjF42jDCSEeikX/G1g40ZWnsPXN5WkZ4jMHZWyBK4=
github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-isatty v0.0.5 h1:tHXDdz1cpzGaovsTB+TVB8q90WEokoVmfMqoVcrLUgw=
+github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y=
github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
+github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
+github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g=
+github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ=
+github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=
+github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771 h1:MHkK1uRtFbVqvAgvWxafZe54+5uBxLluGylDiKgdhwo=
+github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
+github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
+github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
+github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
+github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
+github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
+github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8=
+github.com/mr-tron/base58 v1.1.3 h1:v+sk57XuaCKGXpWtVBX8YJzO7hMGx4Aajh4TQbdEFdc=
+github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
+github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp8Nq/kkI=
+github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA=
+github.com/multiformats/go-base36 v0.1.0 h1:JR6TyF7JjGd3m6FbLU2cOxhC0Li8z8dLNGQ89tUg4F4=
+github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM=
+github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs=
+github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk=
+github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc=
+github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U=
+github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
+github.com/multiformats/go-multihash v0.0.14 h1:QoBceQYQQtNUuf6s7wHxnE2c8bhbMqhfGzNI032se/I=
+github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
+github.com/multiformats/go-varint v0.0.5 h1:XVZwSo04Cs3j/jS0uAEPpT3JY6DzMcVLLoWOSnCxOjg=
+github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hzifhks=
github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0=
github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416 h1:shk/vn9oCoOTmwcouEdwIeOtOGA/ELRUw/GwvxwfT+0=
@@ -164,16 +295,22 @@ github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXW
github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c h1:1RHs3tNxjXGHeul8z2t6H2N2TlAqpKe5yryJztRx4Jk=
github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA=
github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
+github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE=
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
+github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU=
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
+github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/pborman/uuid v0.0.0-20170112150404-1b00554d8222 h1:goeTyGkArOZIVOMA0dQbyuPWGNQJZGPwPu/QS9GlpnA=
github.com/pborman/uuid v0.0.0-20170112150404-1b00554d8222/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34=
+github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
+github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 h1:oYW+YCJ1pachXTQmzR3rNLYGGz4g/UgFcjb28p/viDM=
github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -181,14 +318,27 @@ github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v0.9.3 h1:9iH4JKXLzFbOAdtqv/a+j8aewx2Y8lAjAydhbaScPF8=
+github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/common v0.4.0 h1:7etb9YClo3a6HjLzfl6rIQaU+FDfi0VSX39io3aQ+DM=
+github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 h1:sofwID9zm4tzrgykg80hfFph1mryUeLRsUfoocVVmRY=
+github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150 h1:ZeU+auZj1iNzN8iVhff6M38Mfu73FQiJve/GEXYJBjE=
github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA=
+github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
github.com/rjeczalik/notify v0.9.1 h1:CLCKso/QK1snAlnhNR/CNvNiFU2saUtjV0bx3EwNeCE=
github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho=
+github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00 h1:8DPul/X0IT/1TNMIxoKLwdemEOBBHDC/K4EB16Cw5WE=
github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521 h1:3hxavr+IHMsQBrYUPQM5v0CgENFktkkbg1sfpgM3h20=
@@ -197,7 +347,15 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD
github.com/shirou/gopsutil v2.20.5+incompatible h1:tYH07UPoQt0OCQdgWWMgYHy3/a9bcxNpBIysykNIP7I=
github.com/shirou/gopsutil v2.20.5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM=
+github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
+github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 h1:Gb2Tyox57NRNuZ2d3rmvB3pcmbu7O1RS3m8WRx7ilrg=
github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q=
github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570 h1:gIlAHnH1vJb5vwEjIp5kBj/eu99p/bl0Ay2goiPe5xE=
@@ -205,6 +363,7 @@ github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570/go.mod h1:8
github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3 h1:njlZPzLwU639dk2kqnCPPv+wNjq7Xb6EfUxe/oX0/NM=
github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3/go.mod h1:hpGUWaI9xL8pRQCTXQgocU38Qw1g0Us7n5PxxTwTCYU=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
@@ -214,10 +373,17 @@ github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca/go.mod h1:u2MKk
github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef h1:wHSqTBrZW24CsNJDfeh9Ex6Pm0Rcpc7qrgKBiL44vF4=
github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs=
github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
+github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc h1:9lDbC6Rz4bwmou+oE6Dt4Cb2BGMur5eR/GYptkKUVHo=
+github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM=
github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208 h1:1cngl9mPEoITZG8s8cVcUy5CeIBYhEESkOB7m6Gmkrk=
github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208/go.mod h1:IotVbo4F+mw0EzQ08zFqg7pK3FebNXpaMsRy2RT+Ees=
+go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk=
+go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
@@ -225,6 +391,7 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
golang.org/x/mobile v0.0.0-20200801112145-973feb4309de h1:OVJ6QQUBAesB8CZijKDSsXX7xYVtUhrkY0gwMfbi4p4=
golang.org/x/mobile v0.0.0-20200801112145-973feb4309de/go.mod h1:skQtrUTUwhdJvXM/2KKJzY8pDgNr9I/FOMqDVRPBUS4=
@@ -234,6 +401,8 @@ golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd h1:ePuNC7PZ6O5BzgPn9bZayER
golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -244,10 +413,16 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -259,14 +434,18 @@ golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8 h1:AvbQYmiaaaza3cW3QXRyPo5kYgpFIzOAfeAAN7m3qQ4=
golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69 h1:yBHHx+XZqXJBm6Exke3N7V9gnlsyXxoCPEb1yVenjfk=
golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
@@ -287,8 +466,12 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno=
+gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU=
gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c=
gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6 h1:a6cXbcDDUkSBlpnkWV1bJ+vv3mOgQEltEJ2rPxroVu0=
@@ -297,6 +480,7 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkep
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/urfave/cli.v1 v1.20.0 h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0=
gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
@@ -304,3 +488,8 @@ gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go
index 030bdb37a..4aa91bef2 100644
--- a/internal/ethapi/api.go
+++ b/internal/ethapi/api.go
@@ -932,7 +932,12 @@ func (s *PublicBlockChainAPI) Call(ctx context.Context, args CallArgs, blockNrOr
if overrides != nil {
accounts = *overrides
}
- result, err := DoCall(ctx, s.b, args, blockNrOrHash, accounts, vm.Config{}, 5*time.Second, s.b.RPCGasCap())
+ timeout := 5 * time.Second
+ d, ok := ctx.Deadline()
+ if ok {
+ timeout = d.Sub(time.Now())
+ }
+ result, err := DoCall(ctx, s.b, args, blockNrOrHash, accounts, vm.Config{}, timeout, s.b.RPCGasCap())
if err != nil {
return nil, err
}
diff --git a/params/version.go b/params/version.go
index 3237b51f8..3c7f6e7cb 100644
--- a/params/version.go
+++ b/params/version.go
@@ -24,7 +24,7 @@ const (
VersionMajor = 1 // Major version component of the current release
VersionMinor = 9 // Minor version component of the current release
VersionPatch = 25 // Patch version component of the current release
- VersionMeta = "stable" // Version metadata to append to the version string
+ VersionMeta = "statediff-0.0.13" // Version metadata to append to the version string
)
// Version holds the textual version string.
diff --git a/rpc/http.go b/rpc/http.go
index 87a96e49e..ddbbf6fe4 100644
--- a/rpc/http.go
+++ b/rpc/http.go
@@ -32,7 +32,7 @@ import (
)
const (
- maxRequestContentLength = 1024 * 1024 * 5
+ maxRequestContentLength = 1024 * 1024 * 12
contentType = "application/json"
)
diff --git a/statediff/api.go b/statediff/api.go
new file mode 100644
index 000000000..33ecedd67
--- /dev/null
+++ b/statediff/api.go
@@ -0,0 +1,139 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package statediff
+
+import (
+ "context"
+
+ "github.com/ethereum/go-ethereum/log"
+ "github.com/ethereum/go-ethereum/rpc"
+ . "github.com/ethereum/go-ethereum/statediff/types"
+)
+
+// APIName is the namespace used for the state diffing service API
+const APIName = "statediff"
+
+// APIVersion is the version of the state diffing service API
+const APIVersion = "0.0.1"
+
+// PublicStateDiffAPI provides an RPC subscription interface
+// that can be used to stream out state diffs as they
+// are produced by a full node
+type PublicStateDiffAPI struct {
+ sds IService
+}
+
+// NewPublicStateDiffAPI creates an rpc subscription interface for the underlying statediff service
+func NewPublicStateDiffAPI(sds IService) *PublicStateDiffAPI {
+ return &PublicStateDiffAPI{
+ sds: sds,
+ }
+}
+
+// Stream is the public method to setup a subscription that fires off statediff service payloads as they are created
+func (api *PublicStateDiffAPI) Stream(ctx context.Context, params Params) (*rpc.Subscription, error) {
+ // ensure that the RPC connection supports subscriptions
+ notifier, supported := rpc.NotifierFromContext(ctx)
+ if !supported {
+ return nil, rpc.ErrNotificationsUnsupported
+ }
+
+ // create subscription and start waiting for events
+ rpcSub := notifier.CreateSubscription()
+
+ go func() {
+ // subscribe to events from the statediff service
+ payloadChannel := make(chan Payload, chainEventChanSize)
+ quitChan := make(chan bool, 1)
+ api.sds.Subscribe(rpcSub.ID, payloadChannel, quitChan, params)
+ // loop and await payloads and relay them to the subscriber with the notifier
+ for {
+ select {
+ case payload := <-payloadChannel:
+ if err := notifier.Notify(rpcSub.ID, payload); err != nil {
+ log.Error("Failed to send state diff packet; error: " + err.Error())
+ if err := api.sds.Unsubscribe(rpcSub.ID); err != nil {
+ log.Error("Failed to unsubscribe from the state diff service; error: " + err.Error())
+ }
+ return
+ }
+ case err := <-rpcSub.Err():
+ if err != nil {
+ log.Error("State diff service rpcSub error: " + err.Error())
+ err = api.sds.Unsubscribe(rpcSub.ID)
+ if err != nil {
+ log.Error("Failed to unsubscribe from the state diff service; error: " + err.Error())
+ }
+ return
+ }
+ case <-quitChan:
+ // don't need to unsubscribe, service does so before sending the quit signal
+ return
+ }
+ }
+ }()
+
+ return rpcSub, nil
+}
+
+// StateDiffAt returns a state diff payload at the specific blockheight
+func (api *PublicStateDiffAPI) StateDiffAt(ctx context.Context, blockNumber uint64, params Params) (*Payload, error) {
+ return api.sds.StateDiffAt(blockNumber, params)
+}
+
+// StateTrieAt returns a state trie payload at the specific blockheight
+func (api *PublicStateDiffAPI) StateTrieAt(ctx context.Context, blockNumber uint64, params Params) (*Payload, error) {
+ return api.sds.StateTrieAt(blockNumber, params)
+}
+
+// StreamCodeAndCodeHash writes all of the codehash=>code pairs out to a websocket channel
+func (api *PublicStateDiffAPI) StreamCodeAndCodeHash(ctx context.Context, blockNumber uint64) (*rpc.Subscription, error) {
+ // ensure that the RPC connection supports subscriptions
+ notifier, supported := rpc.NotifierFromContext(ctx)
+ if !supported {
+ return nil, rpc.ErrNotificationsUnsupported
+ }
+
+ // create subscription and start waiting for events
+ rpcSub := notifier.CreateSubscription()
+ payloadChan := make(chan CodeAndCodeHash, chainEventChanSize)
+ quitChan := make(chan bool)
+ api.sds.StreamCodeAndCodeHash(blockNumber, payloadChan, quitChan)
+ go func() {
+ for {
+ select {
+ case payload := <-payloadChan:
+ if err := notifier.Notify(rpcSub.ID, payload); err != nil {
+ log.Error("Failed to send code and codehash packet", "err", err)
+ return
+ }
+ case err := <-rpcSub.Err():
+ log.Error("State diff service rpcSub error", "err", err)
+ return
+ case <-quitChan:
+ return
+ }
+ }
+ }()
+
+ return rpcSub, nil
+}
+
+// WriteStateDiffAt writes a state diff object directly to DB at the specific blockheight
+func (api *PublicStateDiffAPI) WriteStateDiffAt(ctx context.Context, blockNumber uint64, params Params) error {
+ return api.sds.WriteStateDiffAt(blockNumber, params)
+}
diff --git a/statediff/builder.go b/statediff/builder.go
new file mode 100644
index 000000000..c9757b794
--- /dev/null
+++ b/statediff/builder.go
@@ -0,0 +1,754 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+// Contains a batch of utility type declarations used by the tests. As the node
+// operates on unique types, a lot of them are needed to check various features.
+
+package statediff
+
+import (
+ "bytes"
+ "fmt"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/core/state"
+ "github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/crypto"
+ "github.com/ethereum/go-ethereum/log"
+ "github.com/ethereum/go-ethereum/rlp"
+ . "github.com/ethereum/go-ethereum/statediff/types"
+ "github.com/ethereum/go-ethereum/trie"
+)
+
+var (
+ nullHashBytes = common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000000")
+ emptyNode, _ = rlp.EncodeToBytes([]byte{})
+ emptyContractRoot = crypto.Keccak256Hash(emptyNode)
+ nullCodeHash = crypto.Keccak256Hash([]byte{}).Bytes()
+)
+
+// Builder interface exposes the method for building a state diff between two blocks
+type Builder interface {
+ BuildStateDiffObject(args Args, params Params) (StateObject, error)
+ BuildStateTrieObject(current *types.Block) (StateObject, error)
+ WriteStateDiffObject(args StateRoots, params Params, output StateNodeSink, codeOutput CodeSink) error
+}
+
+type builder struct {
+ stateCache state.Database
+}
+
+func resolveNode(it trie.NodeIterator, trieDB *trie.Database) (StateNode, []interface{}, error) {
+ nodePath := make([]byte, len(it.Path()))
+ copy(nodePath, it.Path())
+ node, err := trieDB.Node(it.Hash())
+ if err != nil {
+ return StateNode{}, nil, err
+ }
+ var nodeElements []interface{}
+ if err := rlp.DecodeBytes(node, &nodeElements); err != nil {
+ return StateNode{}, nil, err
+ }
+ ty, err := CheckKeyType(nodeElements)
+ if err != nil {
+ return StateNode{}, nil, err
+ }
+ return StateNode{
+ NodeType: ty,
+ Path: nodePath,
+ NodeValue: node,
+ }, nodeElements, nil
+}
+
+// convenience
+func stateNodeAppender(nodes *[]StateNode) StateNodeSink {
+ return func(node StateNode) error {
+ *nodes = append(*nodes, node)
+ return nil
+ }
+}
+func storageNodeAppender(nodes *[]StorageNode) StorageNodeSink {
+ return func(node StorageNode) error {
+ *nodes = append(*nodes, node)
+ return nil
+ }
+}
+func codeMappingAppender(codeAndCodeHashes *[]CodeAndCodeHash) CodeSink {
+ return func(c CodeAndCodeHash) error {
+ *codeAndCodeHashes = append(*codeAndCodeHashes, c)
+ return nil
+ }
+}
+
+// NewBuilder is used to create a statediff builder
+func NewBuilder(stateCache state.Database) Builder {
+ return &builder{
+ stateCache: stateCache, // state cache is safe for concurrent reads
+ }
+}
+
+// BuildStateTrieObject builds a state trie object from the provided block
+func (sdb *builder) BuildStateTrieObject(current *types.Block) (StateObject, error) {
+ currentTrie, err := sdb.stateCache.OpenTrie(current.Root())
+ if err != nil {
+ return StateObject{}, fmt.Errorf("error creating trie for block %d: %v", current.Number(), err)
+ }
+ it := currentTrie.NodeIterator([]byte{})
+ stateNodes, codeAndCodeHashes, err := sdb.buildStateTrie(it)
+ if err != nil {
+ return StateObject{}, fmt.Errorf("error collecting state nodes for block %d: %v", current.Number(), err)
+ }
+ return StateObject{
+ BlockNumber: current.Number(),
+ BlockHash: current.Hash(),
+ Nodes: stateNodes,
+ CodeAndCodeHashes: codeAndCodeHashes,
+ }, nil
+}
+
+func (sdb *builder) buildStateTrie(it trie.NodeIterator) ([]StateNode, []CodeAndCodeHash, error) {
+ stateNodes := make([]StateNode, 0)
+ codeAndCodeHashes := make([]CodeAndCodeHash, 0)
+ for it.Next(true) {
+ // skip value nodes
+ if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
+ continue
+ }
+ node, nodeElements, err := resolveNode(it, sdb.stateCache.TrieDB())
+ if err != nil {
+ return nil, nil, err
+ }
+ switch node.NodeType {
+ case Leaf:
+ var account state.Account
+ if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil {
+ return nil, nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", node.Path, err)
+ }
+ partialPath := trie.CompactToHex(nodeElements[0].([]byte))
+ valueNodePath := append(node.Path, partialPath...)
+ encodedPath := trie.HexToCompact(valueNodePath)
+ leafKey := encodedPath[1:]
+ node.LeafKey = leafKey
+ if !bytes.Equal(account.CodeHash, nullCodeHash) {
+ var storageNodes []StorageNode
+ err := sdb.buildStorageNodesEventual(account.Root, nil, true, storageNodeAppender(&storageNodes))
+ if err != nil {
+ return nil, nil, fmt.Errorf("failed building eventual storage diffs for account %+v\r\nerror: %v", account, err)
+ }
+ node.StorageNodes = storageNodes
+ // emit codehash => code mappings for cod
+ codeHash := common.BytesToHash(account.CodeHash)
+ code, err := sdb.stateCache.ContractCode(common.Hash{}, codeHash)
+ if err != nil {
+ return nil, nil, fmt.Errorf("failed to retrieve code for codehash %s\r\n error: %v", codeHash.String(), err)
+ }
+ codeAndCodeHashes = append(codeAndCodeHashes, CodeAndCodeHash{
+ Hash: codeHash,
+ Code: code,
+ })
+ }
+ stateNodes = append(stateNodes, node)
+ case Extension, Branch:
+ stateNodes = append(stateNodes, node)
+ default:
+ return nil, nil, fmt.Errorf("unexpected node type %s", node.NodeType)
+ }
+ }
+ return stateNodes, codeAndCodeHashes, it.Error()
+}
+
+// BuildStateDiffObject builds a statediff object from two blocks and the provided parameters
+func (sdb *builder) BuildStateDiffObject(args Args, params Params) (StateObject, error) {
+ var stateNodes []StateNode
+ var codeAndCodeHashes []CodeAndCodeHash
+ err := sdb.WriteStateDiffObject(
+ StateRoots{OldStateRoot: args.OldStateRoot, NewStateRoot: args.NewStateRoot},
+ params, stateNodeAppender(&stateNodes), codeMappingAppender(&codeAndCodeHashes))
+ if err != nil {
+ return StateObject{}, err
+ }
+ return StateObject{
+ BlockHash: args.BlockHash,
+ BlockNumber: args.BlockNumber,
+ Nodes: stateNodes,
+ CodeAndCodeHashes: codeAndCodeHashes,
+ }, nil
+}
+
+// Writes a statediff object to output callback
+func (sdb *builder) WriteStateDiffObject(args StateRoots, params Params, output StateNodeSink, codeOutput CodeSink) error {
+ if !params.IntermediateStateNodes || len(params.WatchedAddresses) > 0 {
+ // if we are watching only specific accounts then we are only diffing leaf nodes
+ return sdb.buildStateDiffWithoutIntermediateStateNodes(args, params, output, codeOutput)
+ } else {
+ return sdb.buildStateDiffWithIntermediateStateNodes(args, params, output, codeOutput)
+ }
+}
+
+func (sdb *builder) buildStateDiffWithIntermediateStateNodes(args StateRoots, params Params, output StateNodeSink, codeOutput CodeSink) error {
+ // Load tries for old and new states
+ oldTrie, err := sdb.stateCache.OpenTrie(args.OldStateRoot)
+ if err != nil {
+ return fmt.Errorf("error creating trie for oldStateRoot: %v", err)
+ }
+ newTrie, err := sdb.stateCache.OpenTrie(args.NewStateRoot)
+ if err != nil {
+ return fmt.Errorf("error creating trie for newStateRoot: %v", err)
+ }
+
+ // collect a slice of all the intermediate nodes that were touched and exist at B
+ // a map of their leafkey to all the accounts that were touched and exist at B
+ // and a slice of all the paths for the nodes in both of the above sets
+ diffAccountsAtB, diffPathsAtB, err := sdb.createdAndUpdatedStateWithIntermediateNodes(
+ oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}),
+ output)
+ if err != nil {
+ return fmt.Errorf("error collecting createdAndUpdatedNodes: %v", err)
+ }
+
+ // collect a slice of all the nodes that existed at a path in A that doesn't exist in B
+ // a map of their leafkey to all the accounts that were touched and exist at A
+ diffAccountsAtA, err := sdb.deletedOrUpdatedState(
+ oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}),
+ diffPathsAtB, output)
+ if err != nil {
+ return fmt.Errorf("error collecting deletedOrUpdatedNodes: %v", err)
+ }
+
+ // collect and sort the leafkey keys for both account mappings into a slice
+ createKeys := sortKeys(diffAccountsAtB)
+ deleteKeys := sortKeys(diffAccountsAtA)
+
+ // and then find the intersection of these keys
+ // these are the leafkeys for the accounts which exist at both A and B but are different
+ // this also mutates the passed in createKeys and deleteKeys, removing the intersection keys
+ // and leaving the truly created or deleted keys in place
+ updatedKeys := findIntersection(createKeys, deleteKeys)
+
+ // build the diff nodes for the updated accounts using the mappings at both A and B as directed by the keys found as the intersection of the two
+ err = sdb.buildAccountUpdates(
+ diffAccountsAtB, diffAccountsAtA, updatedKeys,
+ params.WatchedStorageSlots, params.IntermediateStorageNodes, output)
+ if err != nil {
+ return fmt.Errorf("error building diff for updated accounts: %v", err)
+ }
+ // build the diff nodes for created accounts
+ err = sdb.buildAccountCreations(diffAccountsAtB, params.WatchedStorageSlots, params.IntermediateStorageNodes, output, codeOutput)
+ if err != nil {
+ return fmt.Errorf("error building diff for created accounts: %v", err)
+ }
+ return nil
+}
+
+func (sdb *builder) buildStateDiffWithoutIntermediateStateNodes(args StateRoots, params Params, output StateNodeSink, codeOutput CodeSink) error {
+ // Load tries for old (A) and new (B) states
+ oldTrie, err := sdb.stateCache.OpenTrie(args.OldStateRoot)
+ if err != nil {
+ return fmt.Errorf("error creating trie for oldStateRoot: %v", err)
+ }
+ newTrie, err := sdb.stateCache.OpenTrie(args.NewStateRoot)
+ if err != nil {
+ return fmt.Errorf("error creating trie for newStateRoot: %v", err)
+ }
+
+ // collect a map of their leafkey to all the accounts that were touched and exist at B
+ // and a slice of all the paths for the nodes in both of the above sets
+ diffAccountsAtB, diffPathsAtB, err := sdb.createdAndUpdatedState(
+ oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}),
+ params.WatchedAddresses)
+ if err != nil {
+ return fmt.Errorf("error collecting createdAndUpdatedNodes: %v", err)
+ }
+
+ // collect a slice of all the nodes that existed at a path in A that doesn't exist in B
+ // a map of their leafkey to all the accounts that were touched and exist at A
+ diffAccountsAtA, err := sdb.deletedOrUpdatedState(
+ oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}),
+ diffPathsAtB, output)
+ if err != nil {
+ return fmt.Errorf("error collecting deletedOrUpdatedNodes: %v", err)
+ }
+
+ // collect and sort the leafkeys for both account mappings into a slice
+ createKeys := sortKeys(diffAccountsAtB)
+ deleteKeys := sortKeys(diffAccountsAtA)
+
+ // and then find the intersection of these keys
+ // these are the leafkeys for the accounts which exist at both A and B but are different
+ // this also mutates the passed in createKeys and deleteKeys, removing in intersection keys
+ // and leaving the truly created or deleted keys in place
+ updatedKeys := findIntersection(createKeys, deleteKeys)
+
+ // build the diff nodes for the updated accounts using the mappings at both A and B as directed by the keys found as the intersection of the two
+ err = sdb.buildAccountUpdates(
+ diffAccountsAtB, diffAccountsAtA, updatedKeys,
+ params.WatchedStorageSlots, params.IntermediateStorageNodes, output)
+ if err != nil {
+ return fmt.Errorf("error building diff for updated accounts: %v", err)
+ }
+ // build the diff nodes for created accounts
+ err = sdb.buildAccountCreations(diffAccountsAtB, params.WatchedStorageSlots, params.IntermediateStorageNodes, output, codeOutput)
+ if err != nil {
+ return fmt.Errorf("error building diff for created accounts: %v", err)
+ }
+ return nil
+}
+
+// createdAndUpdatedState returns
+// a mapping of their leafkeys to all the accounts that exist in a different state at B than A
+// and a slice of the paths for all of the nodes included in both
+func (sdb *builder) createdAndUpdatedState(a, b trie.NodeIterator, watchedAddresses []common.Address) (AccountMap, map[string]bool, error) {
+ diffPathsAtB := make(map[string]bool)
+ diffAcountsAtB := make(AccountMap)
+ it, _ := trie.NewDifferenceIterator(a, b)
+ for it.Next(true) {
+ // skip value nodes
+ if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
+ continue
+ }
+ node, nodeElements, err := resolveNode(it, sdb.stateCache.TrieDB())
+ if err != nil {
+ return nil, nil, err
+ }
+ if node.NodeType == Leaf {
+ // created vs updated is important for leaf nodes since we need to diff their storage
+ // so we need to map all changed accounts at B to their leafkey, since account can change pathes but not leafkey
+ var account state.Account
+ if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil {
+ return nil, nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", node.Path, err)
+ }
+ partialPath := trie.CompactToHex(nodeElements[0].([]byte))
+ valueNodePath := append(node.Path, partialPath...)
+ encodedPath := trie.HexToCompact(valueNodePath)
+ leafKey := encodedPath[1:]
+ if isWatchedAddress(watchedAddresses, leafKey) {
+ diffAcountsAtB[common.Bytes2Hex(leafKey)] = accountWrapper{
+ NodeType: node.NodeType,
+ Path: node.Path,
+ NodeValue: node.NodeValue,
+ LeafKey: leafKey,
+ Account: &account,
+ }
+ }
+ }
+ // add both intermediate and leaf node paths to the list of diffPathsAtB
+ diffPathsAtB[common.Bytes2Hex(node.Path)] = true
+ }
+ return diffAcountsAtB, diffPathsAtB, it.Error()
+}
+
+// createdAndUpdatedStateWithIntermediateNodes returns
+// a slice of all the intermediate nodes that exist in a different state at B than A
+// a mapping of their leafkeys to all the accounts that exist in a different state at B than A
+// and a slice of the paths for all of the nodes included in both
+func (sdb *builder) createdAndUpdatedStateWithIntermediateNodes(a, b trie.NodeIterator, output StateNodeSink) (AccountMap, map[string]bool, error) {
+ diffPathsAtB := make(map[string]bool)
+ diffAcountsAtB := make(AccountMap)
+ it, _ := trie.NewDifferenceIterator(a, b)
+ for it.Next(true) {
+ // skip value nodes
+ if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
+ continue
+ }
+ node, nodeElements, err := resolveNode(it, sdb.stateCache.TrieDB())
+ if err != nil {
+ return nil, nil, err
+ }
+ switch node.NodeType {
+ case Leaf:
+ // created vs updated is important for leaf nodes since we need to diff their storage
+ // so we need to map all changed accounts at B to their leafkey, since account can change paths but not leafkey
+ var account state.Account
+ if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil {
+ return nil, nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", node.Path, err)
+ }
+ partialPath := trie.CompactToHex(nodeElements[0].([]byte))
+ valueNodePath := append(node.Path, partialPath...)
+ encodedPath := trie.HexToCompact(valueNodePath)
+ leafKey := encodedPath[1:]
+ diffAcountsAtB[common.Bytes2Hex(leafKey)] = accountWrapper{
+ NodeType: node.NodeType,
+ Path: node.Path,
+ NodeValue: node.NodeValue,
+ LeafKey: leafKey,
+ Account: &account,
+ }
+ case Extension, Branch:
+ // create a diff for any intermediate node that has changed at b
+ // created vs updated makes no difference for intermediate nodes since we do not need to diff storage
+ if err := output(StateNode{
+ NodeType: node.NodeType,
+ Path: node.Path,
+ NodeValue: node.NodeValue,
+ }); err != nil {
+ return nil, nil, err
+ }
+ default:
+ return nil, nil, fmt.Errorf("unexpected node type %s", node.NodeType)
+ }
+ // add both intermediate and leaf node paths to the list of diffPathsAtB
+ diffPathsAtB[common.Bytes2Hex(node.Path)] = true
+ }
+ return diffAcountsAtB, diffPathsAtB, it.Error()
+}
+
+// deletedOrUpdatedState returns a slice of all the pathes that are emptied at B
+// and a mapping of their leafkeys to all the accounts that exist in a different state at A than B
+func (sdb *builder) deletedOrUpdatedState(a, b trie.NodeIterator, diffPathsAtB map[string]bool, output StateNodeSink) (AccountMap, error) {
+ diffAccountAtA := make(AccountMap)
+ it, _ := trie.NewDifferenceIterator(b, a)
+ for it.Next(true) {
+ // skip value nodes
+ if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
+ continue
+ }
+ node, nodeElements, err := resolveNode(it, sdb.stateCache.TrieDB())
+ if err != nil {
+ return nil, err
+ }
+ // if this node's path did not show up in diffPathsAtB
+ // that means the node at this path was deleted (or moved) in B
+ // emit an empty "removed" diff to signify as such
+ if _, ok := diffPathsAtB[common.Bytes2Hex(node.Path)]; !ok {
+ if err := output(StateNode{
+ Path: node.Path,
+ NodeValue: []byte{},
+ NodeType: Removed,
+ }); err != nil {
+ return nil, err
+ }
+ }
+ switch node.NodeType {
+ case Leaf:
+ // map all different accounts at A to their leafkey
+ var account state.Account
+ if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil {
+ return nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", node.Path, err)
+ }
+ partialPath := trie.CompactToHex(nodeElements[0].([]byte))
+ valueNodePath := append(node.Path, partialPath...)
+ encodedPath := trie.HexToCompact(valueNodePath)
+ leafKey := encodedPath[1:]
+ diffAccountAtA[common.Bytes2Hex(leafKey)] = accountWrapper{
+ NodeType: node.NodeType,
+ Path: node.Path,
+ NodeValue: node.NodeValue,
+ LeafKey: leafKey,
+ Account: &account,
+ }
+ case Extension, Branch:
+ // fall through, we did everything we need to do with these node types
+ default:
+ return nil, fmt.Errorf("unexpected node type %s", node.NodeType)
+ }
+ }
+ return diffAccountAtA, it.Error()
+}
+
+// buildAccountUpdates uses the account diffs maps for A => B and B => A and the known intersection of their leafkeys
+// to generate the statediff node objects for all of the accounts that existed at both A and B but in different states
+// needs to be called before building account creations and deletions as this mutates
+// those account maps to remove the accounts which were updated
+func (sdb *builder) buildAccountUpdates(creations, deletions AccountMap, updatedKeys []string,
+ watchedStorageKeys []common.Hash, intermediateStorageNodes bool, output StateNodeSink) error {
+ var err error
+ for _, key := range updatedKeys {
+ createdAcc := creations[key]
+ deletedAcc := deletions[key]
+ var storageDiffs []StorageNode
+ if deletedAcc.Account != nil && createdAcc.Account != nil {
+ oldSR := deletedAcc.Account.Root
+ newSR := createdAcc.Account.Root
+ err = sdb.buildStorageNodesIncremental(
+ oldSR, newSR, watchedStorageKeys, intermediateStorageNodes,
+ storageNodeAppender(&storageDiffs))
+ if err != nil {
+ return fmt.Errorf("failed building incremental storage diffs for account with leafkey %s\r\nerror: %v", key, err)
+ }
+ }
+ if err = output(StateNode{
+ NodeType: createdAcc.NodeType,
+ Path: createdAcc.Path,
+ NodeValue: createdAcc.NodeValue,
+ LeafKey: createdAcc.LeafKey,
+ StorageNodes: storageDiffs,
+ }); err != nil {
+ return err
+ }
+ delete(creations, key)
+ delete(deletions, key)
+ }
+
+ return nil
+}
+
+// buildAccountCreations returns the statediff node objects for all the accounts that exist at B but not at A
+// it also returns the code and codehash for created contract accounts
+func (sdb *builder) buildAccountCreations(accounts AccountMap, watchedStorageKeys []common.Hash, intermediateStorageNodes bool, output StateNodeSink, codeOutput CodeSink) error {
+ for _, val := range accounts {
+ diff := StateNode{
+ NodeType: val.NodeType,
+ Path: val.Path,
+ LeafKey: val.LeafKey,
+ NodeValue: val.NodeValue,
+ }
+ if !bytes.Equal(val.Account.CodeHash, nullCodeHash) {
+ // For contract creations, any storage node contained is a diff
+ var storageDiffs []StorageNode
+ err := sdb.buildStorageNodesEventual(val.Account.Root, watchedStorageKeys, intermediateStorageNodes, storageNodeAppender(&storageDiffs))
+ if err != nil {
+ return fmt.Errorf("failed building eventual storage diffs for node %x\r\nerror: %v", val.Path, err)
+ }
+ diff.StorageNodes = storageDiffs
+ // emit codehash => code mappings for cod
+ codeHash := common.BytesToHash(val.Account.CodeHash)
+ code, err := sdb.stateCache.ContractCode(common.Hash{}, codeHash)
+ if err != nil {
+ return fmt.Errorf("failed to retrieve code for codehash %s\r\n error: %v", codeHash.String(), err)
+ }
+ if err := codeOutput(CodeAndCodeHash{
+ Hash: codeHash,
+ Code: code,
+ }); err != nil {
+ return err
+ }
+ }
+ if err := output(diff); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+// buildStorageNodesEventual builds the storage diff node objects for a created account
+// i.e. it returns all the storage nodes at this state, since there is no previous state
+func (sdb *builder) buildStorageNodesEventual(sr common.Hash, watchedStorageKeys []common.Hash, intermediateNodes bool, output StorageNodeSink) error {
+ if bytes.Equal(sr.Bytes(), emptyContractRoot.Bytes()) {
+ return nil
+ }
+ log.Debug("Storage Root For Eventual Diff", "root", sr.Hex())
+ sTrie, err := sdb.stateCache.OpenTrie(sr)
+ if err != nil {
+ log.Info("error in build storage diff eventual", "error", err)
+ return err
+ }
+ it := sTrie.NodeIterator(make([]byte, 0))
+ err = sdb.buildStorageNodesFromTrie(it, watchedStorageKeys, intermediateNodes, output)
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+// buildStorageNodesFromTrie returns all the storage diff node objects in the provided node interator
+// if any storage keys are provided it will only return those leaf nodes
+// including intermediate nodes can be turned on or off
+func (sdb *builder) buildStorageNodesFromTrie(it trie.NodeIterator, watchedStorageKeys []common.Hash, intermediateNodes bool, output StorageNodeSink) error {
+ for it.Next(true) {
+ // skip value nodes
+ if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
+ continue
+ }
+ node, nodeElements, err := resolveNode(it, sdb.stateCache.TrieDB())
+ if err != nil {
+ return err
+ }
+ switch node.NodeType {
+ case Leaf:
+ partialPath := trie.CompactToHex(nodeElements[0].([]byte))
+ valueNodePath := append(node.Path, partialPath...)
+ encodedPath := trie.HexToCompact(valueNodePath)
+ leafKey := encodedPath[1:]
+ if isWatchedStorageKey(watchedStorageKeys, leafKey) {
+ if err := output(StorageNode{
+ NodeType: node.NodeType,
+ Path: node.Path,
+ NodeValue: node.NodeValue,
+ LeafKey: leafKey,
+ }); err != nil {
+ return err
+ }
+ }
+ case Extension, Branch:
+ if intermediateNodes {
+ if err := output(StorageNode{
+ NodeType: node.NodeType,
+ Path: node.Path,
+ NodeValue: node.NodeValue,
+ }); err != nil {
+ return err
+ }
+ }
+ default:
+ return fmt.Errorf("unexpected node type %s", node.NodeType)
+ }
+ }
+ return it.Error()
+}
+
+// buildStorageNodesIncremental builds the storage diff node objects for all nodes that exist in a different state at B than A
+func (sdb *builder) buildStorageNodesIncremental(oldSR common.Hash, newSR common.Hash, watchedStorageKeys []common.Hash, intermediateNodes bool, output StorageNodeSink) error {
+ if bytes.Equal(newSR.Bytes(), oldSR.Bytes()) {
+ return nil
+ }
+ log.Debug("Storage Roots for Incremental Diff", "old", oldSR.Hex(), "new", newSR.Hex())
+ oldTrie, err := sdb.stateCache.OpenTrie(oldSR)
+ if err != nil {
+ return err
+ }
+ newTrie, err := sdb.stateCache.OpenTrie(newSR)
+ if err != nil {
+ return err
+ }
+
+ diffPathsAtB, err := sdb.createdAndUpdatedStorage(
+ oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}),
+ watchedStorageKeys, intermediateNodes, output)
+ if err != nil {
+ return err
+ }
+ err = sdb.deletedOrUpdatedStorage(oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}),
+ diffPathsAtB, watchedStorageKeys, intermediateNodes, output)
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+func (sdb *builder) createdAndUpdatedStorage(a, b trie.NodeIterator, watchedKeys []common.Hash, intermediateNodes bool, output StorageNodeSink) (map[string]bool, error) {
+ diffPathsAtB := make(map[string]bool)
+ it, _ := trie.NewDifferenceIterator(a, b)
+ for it.Next(true) {
+ // skip value nodes
+ if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
+ continue
+ }
+ node, nodeElements, err := resolveNode(it, sdb.stateCache.TrieDB())
+ if err != nil {
+ return nil, err
+ }
+ switch node.NodeType {
+ case Leaf:
+ partialPath := trie.CompactToHex(nodeElements[0].([]byte))
+ valueNodePath := append(node.Path, partialPath...)
+ encodedPath := trie.HexToCompact(valueNodePath)
+ leafKey := encodedPath[1:]
+ if isWatchedStorageKey(watchedKeys, leafKey) {
+ if err := output(StorageNode{
+ NodeType: node.NodeType,
+ Path: node.Path,
+ NodeValue: node.NodeValue,
+ LeafKey: leafKey,
+ }); err != nil {
+ return nil, err
+ }
+ }
+ case Extension, Branch:
+ if intermediateNodes {
+ if err := output(StorageNode{
+ NodeType: node.NodeType,
+ Path: node.Path,
+ NodeValue: node.NodeValue,
+ }); err != nil {
+ return nil, err
+ }
+ }
+ default:
+ return nil, fmt.Errorf("unexpected node type %s", node.NodeType)
+ }
+ diffPathsAtB[common.Bytes2Hex(node.Path)] = true
+ }
+ return diffPathsAtB, it.Error()
+}
+
+func (sdb *builder) deletedOrUpdatedStorage(a, b trie.NodeIterator, diffPathsAtB map[string]bool, watchedKeys []common.Hash, intermediateNodes bool, output StorageNodeSink) error {
+ it, _ := trie.NewDifferenceIterator(b, a)
+ for it.Next(true) {
+ // skip value nodes
+ if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
+ continue
+ }
+ node, nodeElements, err := resolveNode(it, sdb.stateCache.TrieDB())
+ if err != nil {
+ return err
+ }
+ // if this node path showed up in diffPathsAtB
+ // that means this node was updated at B and we already have the updated diff for it
+ // otherwise that means this node was deleted in B and we need to add a "removed" diff to represent that event
+ if _, ok := diffPathsAtB[common.Bytes2Hex(node.Path)]; ok {
+ continue
+ }
+ switch node.NodeType {
+ case Leaf:
+ partialPath := trie.CompactToHex(nodeElements[0].([]byte))
+ valueNodePath := append(node.Path, partialPath...)
+ encodedPath := trie.HexToCompact(valueNodePath)
+ leafKey := encodedPath[1:]
+ if isWatchedStorageKey(watchedKeys, leafKey) {
+ if err := output(StorageNode{
+ NodeType: Removed,
+ Path: node.Path,
+ NodeValue: []byte{},
+ }); err != nil {
+ return err
+ }
+ }
+ case Extension, Branch:
+ if intermediateNodes {
+ if err := output(StorageNode{
+ NodeType: Removed,
+ Path: node.Path,
+ NodeValue: []byte{},
+ }); err != nil {
+ return err
+ }
+ }
+ default:
+ return fmt.Errorf("unexpected node type %s", node.NodeType)
+ }
+ }
+ return it.Error()
+}
+
+// isWatchedAddress is used to check if a state account corresponds to one of the addresses the builder is configured to watch
+func isWatchedAddress(watchedAddresses []common.Address, stateLeafKey []byte) bool {
+ // If we aren't watching any specific addresses, we are watching everything
+ if len(watchedAddresses) == 0 {
+ return true
+ }
+ for _, addr := range watchedAddresses {
+ addrHashKey := crypto.Keccak256(addr.Bytes())
+ if bytes.Equal(addrHashKey, stateLeafKey) {
+ return true
+ }
+ }
+ return false
+}
+
+// isWatchedStorageKey is used to check if a storage leaf corresponds to one of the storage slots the builder is configured to watch
+func isWatchedStorageKey(watchedKeys []common.Hash, storageLeafKey []byte) bool {
+ // If we aren't watching any specific addresses, we are watching everything
+ if len(watchedKeys) == 0 {
+ return true
+ }
+ for _, hashKey := range watchedKeys {
+ if bytes.Equal(hashKey.Bytes(), storageLeafKey) {
+ return true
+ }
+ }
+ return false
+}
diff --git a/statediff/builder_test.go b/statediff/builder_test.go
new file mode 100644
index 000000000..8ad0d9822
--- /dev/null
+++ b/statediff/builder_test.go
@@ -0,0 +1,2300 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package statediff_test
+
+import (
+ "bytes"
+ "math/big"
+ "sort"
+ "testing"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/core/state"
+ "github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/crypto"
+ "github.com/ethereum/go-ethereum/rlp"
+ "github.com/ethereum/go-ethereum/statediff"
+ "github.com/ethereum/go-ethereum/statediff/testhelpers"
+ sdtypes "github.com/ethereum/go-ethereum/statediff/types"
+)
+
+// TODO: add test that filters on address
+var (
+ contractLeafKey []byte
+ emptyDiffs = make([]sdtypes.StateNode, 0)
+ emptyStorage = make([]sdtypes.StorageNode, 0)
+ block0, block1, block2, block3, block4, block5, block6 *types.Block
+ builder statediff.Builder
+ miningReward = int64(2000000000000000000)
+ minerAddress = common.HexToAddress("0x0")
+ minerLeafKey = testhelpers.AddressToLeafKey(minerAddress)
+
+ balanceChange10000 = int64(10000)
+ balanceChange1000 = int64(1000)
+ block1BankBalance = int64(99990000)
+ block1Account1Balance = int64(10000)
+ block2Account2Balance = int64(1000)
+
+ slot0 = common.HexToHash("0")
+ slot1 = common.HexToHash("1")
+ slot2 = common.HexToHash("2")
+ slot3 = common.HexToHash("3")
+
+ slot0StorageKey = crypto.Keccak256Hash(slot0[:])
+ slot1StorageKey = crypto.Keccak256Hash(slot1[:])
+ slot2StorageKey = crypto.Keccak256Hash(slot2[:])
+ slot3StorageKey = crypto.Keccak256Hash(slot3[:])
+
+ slot0StorageValue = common.Hex2Bytes("94703c4b2bd70c169f5717101caee543299fc946c7") // prefixed AccountAddr1
+ slot1StorageValue = common.Hex2Bytes("01")
+ slot2StorageValue = common.Hex2Bytes("09")
+ slot3StorageValue = common.Hex2Bytes("03")
+
+ slot0StorageLeafNode, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("390decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563"),
+ slot0StorageValue,
+ })
+ slot1StorageLeafNode, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("310e2d527612073b26eecdfd717e6a320cf44b4afac2b0732d9fcbe2b7fa0cf6"),
+ slot1StorageValue,
+ })
+ slot2StorageLeafNode, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("305787fa12a823e0f2b7631cc41b3ba8828b3321ca811111fa75cd3aa3bb5ace"),
+ slot2StorageValue,
+ })
+ slot3StorageLeafNode, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("32575a0e9e593c00f959f8c92f12db2869c3395a3b0502d05e2516446f71f85b"),
+ slot3StorageValue,
+ })
+ slot0StorageLeafRootNode, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("20290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563"),
+ slot0StorageValue,
+ })
+
+ contractAccountAtBlock2, _ = rlp.EncodeToBytes(state.Account{
+ Nonce: 1,
+ Balance: big.NewInt(0),
+ CodeHash: common.HexToHash("0xaaea5efba4fd7b45d7ec03918ac5d8b31aa93b48986af0e6b591f0f087c80127").Bytes(),
+ Root: crypto.Keccak256Hash(block2StorageBranchRootNode),
+ })
+ contractAccountAtBlock2LeafNode, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("3114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45"),
+ contractAccountAtBlock2,
+ })
+ contractAccountAtBlock3, _ = rlp.EncodeToBytes(state.Account{
+ Nonce: 1,
+ Balance: big.NewInt(0),
+ CodeHash: common.HexToHash("0xaaea5efba4fd7b45d7ec03918ac5d8b31aa93b48986af0e6b591f0f087c80127").Bytes(),
+ Root: crypto.Keccak256Hash(block3StorageBranchRootNode),
+ })
+ contractAccountAtBlock3LeafNode, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("3114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45"),
+ contractAccountAtBlock3,
+ })
+ contractAccountAtBlock4, _ = rlp.EncodeToBytes(state.Account{
+ Nonce: 1,
+ Balance: big.NewInt(0),
+ CodeHash: common.HexToHash("0xaaea5efba4fd7b45d7ec03918ac5d8b31aa93b48986af0e6b591f0f087c80127").Bytes(),
+ Root: crypto.Keccak256Hash(block4StorageBranchRootNode),
+ })
+ contractAccountAtBlock4LeafNode, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("3114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45"),
+ contractAccountAtBlock4,
+ })
+ contractAccountAtBlock5, _ = rlp.EncodeToBytes(state.Account{
+ Nonce: 1,
+ Balance: big.NewInt(0),
+ CodeHash: common.HexToHash("0xaaea5efba4fd7b45d7ec03918ac5d8b31aa93b48986af0e6b591f0f087c80127").Bytes(),
+ Root: crypto.Keccak256Hash(slot0StorageLeafRootNode),
+ })
+ contractAccountAtBlock5LeafNode, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("3114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45"),
+ contractAccountAtBlock5,
+ })
+
+ minerAccountAtBlock1, _ = rlp.EncodeToBytes(state.Account{
+ Nonce: 0,
+ Balance: big.NewInt(miningReward),
+ CodeHash: testhelpers.NullCodeHash.Bytes(),
+ Root: testhelpers.EmptyContractRoot,
+ })
+ minerAccountAtBlock1LeafNode, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("3380c7b7ae81a58eb98d9c78de4a1fd7fd9535fc953ed2be602daaa41767312a"),
+ minerAccountAtBlock1,
+ })
+ minerAccountAtBlock2, _ = rlp.EncodeToBytes(state.Account{
+ Nonce: 0,
+ Balance: big.NewInt(miningReward + miningReward),
+ CodeHash: testhelpers.NullCodeHash.Bytes(),
+ Root: testhelpers.EmptyContractRoot,
+ })
+ minerAccountAtBlock2LeafNode, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("3380c7b7ae81a58eb98d9c78de4a1fd7fd9535fc953ed2be602daaa41767312a"),
+ minerAccountAtBlock2,
+ })
+
+ account1AtBlock1, _ = rlp.EncodeToBytes(state.Account{
+ Nonce: 0,
+ Balance: big.NewInt(balanceChange10000),
+ CodeHash: testhelpers.NullCodeHash.Bytes(),
+ Root: testhelpers.EmptyContractRoot,
+ })
+ account1AtBlock1LeafNode, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("3926db69aaced518e9b9f0f434a473e7174109c943548bb8f23be41ca76d9ad2"),
+ account1AtBlock1,
+ })
+ account1AtBlock2, _ = rlp.EncodeToBytes(state.Account{
+ Nonce: 2,
+ Balance: big.NewInt(block1Account1Balance - balanceChange1000 + balanceChange1000),
+ CodeHash: testhelpers.NullCodeHash.Bytes(),
+ Root: testhelpers.EmptyContractRoot,
+ })
+ account1AtBlock2LeafNode, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("3926db69aaced518e9b9f0f434a473e7174109c943548bb8f23be41ca76d9ad2"),
+ account1AtBlock2,
+ })
+ account1AtBlock5, _ = rlp.EncodeToBytes(state.Account{
+ Nonce: 2,
+ Balance: big.NewInt(block1Account1Balance - balanceChange1000 + balanceChange1000 + miningReward),
+ CodeHash: testhelpers.NullCodeHash.Bytes(),
+ Root: testhelpers.EmptyContractRoot,
+ })
+ account1AtBlock5LeafNode, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("3926db69aaced518e9b9f0f434a473e7174109c943548bb8f23be41ca76d9ad2"),
+ account1AtBlock5,
+ })
+ account1AtBlock6, _ = rlp.EncodeToBytes(state.Account{
+ Nonce: 3,
+ Balance: big.NewInt(block1Account1Balance - balanceChange1000 + balanceChange1000 + miningReward),
+ CodeHash: testhelpers.NullCodeHash.Bytes(),
+ Root: testhelpers.EmptyContractRoot,
+ })
+ account1AtBlock6LeafNode, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("3926db69aaced518e9b9f0f434a473e7174109c943548bb8f23be41ca76d9ad2"),
+ account1AtBlock6,
+ })
+
+ account2AtBlock2, _ = rlp.EncodeToBytes(state.Account{
+ Nonce: 0,
+ Balance: big.NewInt(balanceChange1000),
+ CodeHash: testhelpers.NullCodeHash.Bytes(),
+ Root: testhelpers.EmptyContractRoot,
+ })
+ account2AtBlock2LeafNode, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("3957f3e2f04a0764c3a0491b175f69926da61efbcc8f61fa1455fd2d2b4cdd45"),
+ account2AtBlock2,
+ })
+ account2AtBlock3, _ = rlp.EncodeToBytes(state.Account{
+ Nonce: 0,
+ Balance: big.NewInt(block2Account2Balance + miningReward),
+ CodeHash: testhelpers.NullCodeHash.Bytes(),
+ Root: testhelpers.EmptyContractRoot,
+ })
+ account2AtBlock3LeafNode, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("3957f3e2f04a0764c3a0491b175f69926da61efbcc8f61fa1455fd2d2b4cdd45"),
+ account2AtBlock3,
+ })
+ account2AtBlock4, _ = rlp.EncodeToBytes(state.Account{
+ Nonce: 0,
+ Balance: big.NewInt(block2Account2Balance + miningReward*2),
+ CodeHash: testhelpers.NullCodeHash.Bytes(),
+ Root: testhelpers.EmptyContractRoot,
+ })
+ account2AtBlock4LeafNode, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("3957f3e2f04a0764c3a0491b175f69926da61efbcc8f61fa1455fd2d2b4cdd45"),
+ account2AtBlock4,
+ })
+ account2AtBlock6, _ = rlp.EncodeToBytes(state.Account{
+ Nonce: 0,
+ Balance: big.NewInt(block2Account2Balance + miningReward*3),
+ CodeHash: testhelpers.NullCodeHash.Bytes(),
+ Root: testhelpers.EmptyContractRoot,
+ })
+ account2AtBlock6LeafNode, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("3957f3e2f04a0764c3a0491b175f69926da61efbcc8f61fa1455fd2d2b4cdd45"),
+ account2AtBlock6,
+ })
+
+ bankAccountAtBlock0, _ = rlp.EncodeToBytes(state.Account{
+ Nonce: 0,
+ Balance: big.NewInt(testhelpers.TestBankFunds.Int64()),
+ CodeHash: testhelpers.NullCodeHash.Bytes(),
+ Root: testhelpers.EmptyContractRoot,
+ })
+ bankAccountAtBlock0LeafNode, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("2000bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a"),
+ bankAccountAtBlock0,
+ })
+ bankAccountAtBlock1, _ = rlp.EncodeToBytes(state.Account{
+ Nonce: 1,
+ Balance: big.NewInt(testhelpers.TestBankFunds.Int64() - balanceChange10000),
+ CodeHash: testhelpers.NullCodeHash.Bytes(),
+ Root: testhelpers.EmptyContractRoot,
+ })
+ bankAccountAtBlock1LeafNode, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("30bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a"),
+ bankAccountAtBlock1,
+ })
+ bankAccountAtBlock2, _ = rlp.EncodeToBytes(state.Account{
+ Nonce: 2,
+ Balance: big.NewInt(block1BankBalance - balanceChange1000),
+ CodeHash: testhelpers.NullCodeHash.Bytes(),
+ Root: testhelpers.EmptyContractRoot,
+ })
+ bankAccountAtBlock2LeafNode, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("30bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a"),
+ bankAccountAtBlock2,
+ })
+ bankAccountAtBlock3, _ = rlp.EncodeToBytes(state.Account{
+ Nonce: 3,
+ Balance: big.NewInt(99989000),
+ CodeHash: testhelpers.NullCodeHash.Bytes(),
+ Root: testhelpers.EmptyContractRoot,
+ })
+ bankAccountAtBlock3LeafNode, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("30bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a"),
+ bankAccountAtBlock3,
+ })
+ bankAccountAtBlock4, _ = rlp.EncodeToBytes(state.Account{
+ Nonce: 6,
+ Balance: big.NewInt(99989000),
+ CodeHash: testhelpers.NullCodeHash.Bytes(),
+ Root: testhelpers.EmptyContractRoot,
+ })
+ bankAccountAtBlock4LeafNode, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("30bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a"),
+ bankAccountAtBlock4,
+ })
+ bankAccountAtBlock5, _ = rlp.EncodeToBytes(state.Account{
+ Nonce: 7,
+ Balance: big.NewInt(99989000),
+ CodeHash: testhelpers.NullCodeHash.Bytes(),
+ Root: testhelpers.EmptyContractRoot,
+ })
+ bankAccountAtBlock5LeafNode, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("30bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a"),
+ bankAccountAtBlock5,
+ })
+
+ block1BranchRootNode, _ = rlp.EncodeToBytes([]interface{}{
+ crypto.Keccak256(bankAccountAtBlock1LeafNode),
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ crypto.Keccak256(minerAccountAtBlock1LeafNode),
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ crypto.Keccak256(account1AtBlock1LeafNode),
+ []byte{},
+ []byte{},
+ })
+ block2BranchRootNode, _ = rlp.EncodeToBytes([]interface{}{
+ crypto.Keccak256(bankAccountAtBlock2LeafNode),
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ crypto.Keccak256(minerAccountAtBlock2LeafNode),
+ crypto.Keccak256(contractAccountAtBlock2LeafNode),
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ crypto.Keccak256(account2AtBlock2LeafNode),
+ []byte{},
+ crypto.Keccak256(account1AtBlock2LeafNode),
+ []byte{},
+ []byte{},
+ })
+ block3BranchRootNode, _ = rlp.EncodeToBytes([]interface{}{
+ crypto.Keccak256(bankAccountAtBlock3LeafNode),
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ crypto.Keccak256(minerAccountAtBlock2LeafNode),
+ crypto.Keccak256(contractAccountAtBlock3LeafNode),
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ crypto.Keccak256(account2AtBlock3LeafNode),
+ []byte{},
+ crypto.Keccak256(account1AtBlock2LeafNode),
+ []byte{},
+ []byte{},
+ })
+ block4BranchRootNode, _ = rlp.EncodeToBytes([]interface{}{
+ crypto.Keccak256(bankAccountAtBlock4LeafNode),
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ crypto.Keccak256(minerAccountAtBlock2LeafNode),
+ crypto.Keccak256(contractAccountAtBlock4LeafNode),
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ crypto.Keccak256(account2AtBlock4LeafNode),
+ []byte{},
+ crypto.Keccak256(account1AtBlock2LeafNode),
+ []byte{},
+ []byte{},
+ })
+ block5BranchRootNode, _ = rlp.EncodeToBytes([]interface{}{
+ crypto.Keccak256(bankAccountAtBlock5LeafNode),
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ crypto.Keccak256(minerAccountAtBlock2LeafNode),
+ crypto.Keccak256(contractAccountAtBlock5LeafNode),
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ crypto.Keccak256(account2AtBlock4LeafNode),
+ []byte{},
+ crypto.Keccak256(account1AtBlock5LeafNode),
+ []byte{},
+ []byte{},
+ })
+ block6BranchRootNode, _ = rlp.EncodeToBytes([]interface{}{
+ crypto.Keccak256(bankAccountAtBlock5LeafNode),
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ crypto.Keccak256(minerAccountAtBlock2LeafNode),
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ crypto.Keccak256(account2AtBlock6LeafNode),
+ []byte{},
+ crypto.Keccak256(account1AtBlock6LeafNode),
+ []byte{},
+ []byte{},
+ })
+
+ block2StorageBranchRootNode, _ = rlp.EncodeToBytes([]interface{}{
+ []byte{},
+ []byte{},
+ crypto.Keccak256(slot0StorageLeafNode),
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ crypto.Keccak256(slot1StorageLeafNode),
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ })
+ block3StorageBranchRootNode, _ = rlp.EncodeToBytes([]interface{}{
+ []byte{},
+ []byte{},
+ crypto.Keccak256(slot0StorageLeafNode),
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ crypto.Keccak256(slot1StorageLeafNode),
+ crypto.Keccak256(slot3StorageLeafNode),
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ })
+ block4StorageBranchRootNode, _ = rlp.EncodeToBytes([]interface{}{
+ []byte{},
+ []byte{},
+ crypto.Keccak256(slot0StorageLeafNode),
+ []byte{},
+ crypto.Keccak256(slot2StorageLeafNode),
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ })
+)
+
+func TestBuilder(t *testing.T) {
+ blocks, chain := testhelpers.MakeChain(3, testhelpers.Genesis, testhelpers.TestChainGen)
+ contractLeafKey = testhelpers.AddressToLeafKey(testhelpers.ContractAddr)
+ defer chain.Stop()
+ block0 = testhelpers.Genesis
+ block1 = blocks[0]
+ block2 = blocks[1]
+ block3 = blocks[2]
+ params := statediff.Params{}
+ builder = statediff.NewBuilder(chain.StateCache())
+
+ var tests = []struct {
+ name string
+ startingArguments statediff.Args
+ expected *statediff.StateObject
+ }{
+ {
+ "testEmptyDiff",
+ statediff.Args{
+ OldStateRoot: block0.Root(),
+ NewStateRoot: block0.Root(),
+ BlockNumber: block0.Number(),
+ BlockHash: block0.Hash(),
+ },
+ &statediff.StateObject{
+ BlockNumber: block0.Number(),
+ BlockHash: block0.Hash(),
+ Nodes: emptyDiffs,
+ },
+ },
+ {
+ "testBlock0",
+ //10000 transferred from testBankAddress to account1Addr
+ statediff.Args{
+ OldStateRoot: testhelpers.NullHash,
+ NewStateRoot: block0.Root(),
+ BlockNumber: block0.Number(),
+ BlockHash: block0.Hash(),
+ },
+ &statediff.StateObject{
+ BlockNumber: block0.Number(),
+ BlockHash: block0.Hash(),
+ Nodes: []sdtypes.StateNode{
+ {
+ Path: []byte{},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.BankLeafKey,
+ NodeValue: bankAccountAtBlock0LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ },
+ },
+ },
+ {
+ "testBlock1",
+ //10000 transferred from testBankAddress to account1Addr
+ statediff.Args{
+ OldStateRoot: block0.Root(),
+ NewStateRoot: block1.Root(),
+ BlockNumber: block1.Number(),
+ BlockHash: block1.Hash(),
+ },
+ &statediff.StateObject{
+ BlockNumber: block1.Number(),
+ BlockHash: block1.Hash(),
+ Nodes: []sdtypes.StateNode{
+ {
+ Path: []byte{'\x00'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.BankLeafKey,
+ NodeValue: bankAccountAtBlock1LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ {
+ Path: []byte{'\x05'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: minerLeafKey,
+ NodeValue: minerAccountAtBlock1LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ {
+ Path: []byte{'\x0e'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.Account1LeafKey,
+ NodeValue: account1AtBlock1LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ },
+ },
+ },
+ {
+ "testBlock2",
+ // 1000 transferred from testBankAddress to account1Addr
+ // 1000 transferred from account1Addr to account2Addr
+ // account1addr creates a new contract
+ statediff.Args{
+ OldStateRoot: block1.Root(),
+ NewStateRoot: block2.Root(),
+ BlockNumber: block2.Number(),
+ BlockHash: block2.Hash(),
+ },
+ &statediff.StateObject{
+ BlockNumber: block2.Number(),
+ BlockHash: block2.Hash(),
+ Nodes: []sdtypes.StateNode{
+ {
+ Path: []byte{'\x00'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.BankLeafKey,
+ NodeValue: bankAccountAtBlock2LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ {
+ Path: []byte{'\x05'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: minerLeafKey,
+ NodeValue: minerAccountAtBlock2LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ {
+ Path: []byte{'\x0e'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.Account1LeafKey,
+ NodeValue: account1AtBlock2LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ {
+ Path: []byte{'\x06'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: contractLeafKey,
+ NodeValue: contractAccountAtBlock2LeafNode,
+ StorageNodes: []sdtypes.StorageNode{
+ {
+ Path: []byte{'\x02'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: slot0StorageKey.Bytes(),
+ NodeValue: slot0StorageLeafNode,
+ },
+ {
+ Path: []byte{'\x0b'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: slot1StorageKey.Bytes(),
+ NodeValue: slot1StorageLeafNode,
+ },
+ },
+ },
+ {
+ Path: []byte{'\x0c'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.Account2LeafKey,
+ NodeValue: account2AtBlock2LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ },
+ CodeAndCodeHashes: []sdtypes.CodeAndCodeHash{
+ {
+ Hash: testhelpers.CodeHash,
+ Code: testhelpers.ByteCodeAfterDeployment,
+ },
+ },
+ },
+ },
+ {
+ "testBlock3",
+ //the contract's storage is changed
+ //and the block is mined by account 2
+ statediff.Args{
+ OldStateRoot: block2.Root(),
+ NewStateRoot: block3.Root(),
+ BlockNumber: block3.Number(),
+ BlockHash: block3.Hash(),
+ },
+ &statediff.StateObject{
+ BlockNumber: block3.Number(),
+ BlockHash: block3.Hash(),
+ Nodes: []sdtypes.StateNode{
+ {
+ Path: []byte{'\x00'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.BankLeafKey,
+ NodeValue: bankAccountAtBlock3LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ {
+ Path: []byte{'\x06'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: contractLeafKey,
+ NodeValue: contractAccountAtBlock3LeafNode,
+ StorageNodes: []sdtypes.StorageNode{
+ {
+ Path: []byte{'\x0c'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: slot3StorageKey.Bytes(),
+ NodeValue: slot3StorageLeafNode,
+ },
+ },
+ },
+ {
+ Path: []byte{'\x0c'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.Account2LeafKey,
+ NodeValue: account2AtBlock3LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ },
+ },
+ },
+ }
+
+ for _, test := range tests {
+ diff, err := builder.BuildStateDiffObject(test.startingArguments, params)
+ if err != nil {
+ t.Error(err)
+ }
+ receivedStateDiffRlp, err := rlp.EncodeToBytes(diff)
+ if err != nil {
+ t.Error(err)
+ }
+ expectedStateDiffRlp, err := rlp.EncodeToBytes(test.expected)
+ if err != nil {
+ t.Error(err)
+ }
+ sort.Slice(receivedStateDiffRlp, func(i, j int) bool { return receivedStateDiffRlp[i] < receivedStateDiffRlp[j] })
+ sort.Slice(expectedStateDiffRlp, func(i, j int) bool { return expectedStateDiffRlp[i] < expectedStateDiffRlp[j] })
+ if !bytes.Equal(receivedStateDiffRlp, expectedStateDiffRlp) {
+ t.Logf("Test failed: %s", test.name)
+ t.Errorf("actual state diff: %+v\nexpected state diff: %+v", diff, test.expected)
+ }
+ }
+}
+
+func TestBuilderWithIntermediateNodes(t *testing.T) {
+ blocks, chain := testhelpers.MakeChain(3, testhelpers.Genesis, testhelpers.TestChainGen)
+ contractLeafKey = testhelpers.AddressToLeafKey(testhelpers.ContractAddr)
+ defer chain.Stop()
+ block0 = testhelpers.Genesis
+ block1 = blocks[0]
+ block2 = blocks[1]
+ block3 = blocks[2]
+ blocks = append([]*types.Block{block0}, blocks...)
+ params := statediff.Params{
+ IntermediateStateNodes: true,
+ IntermediateStorageNodes: true,
+ }
+ builder = statediff.NewBuilder(chain.StateCache())
+
+ var tests = []struct {
+ name string
+ startingArguments statediff.Args
+ expected *statediff.StateObject
+ }{
+ {
+ "testEmptyDiff",
+ statediff.Args{
+ OldStateRoot: block0.Root(),
+ NewStateRoot: block0.Root(),
+ BlockNumber: block0.Number(),
+ BlockHash: block0.Hash(),
+ },
+ &statediff.StateObject{
+ BlockNumber: block0.Number(),
+ BlockHash: block0.Hash(),
+ Nodes: emptyDiffs,
+ },
+ },
+ {
+ "testBlock0",
+ //10000 transferred from testBankAddress to account1Addr
+ statediff.Args{
+ OldStateRoot: testhelpers.NullHash,
+ NewStateRoot: block0.Root(),
+ BlockNumber: block0.Number(),
+ BlockHash: block0.Hash(),
+ },
+ &statediff.StateObject{
+ BlockNumber: block0.Number(),
+ BlockHash: block0.Hash(),
+ Nodes: []sdtypes.StateNode{
+ {
+ Path: []byte{},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.BankLeafKey,
+ NodeValue: bankAccountAtBlock0LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ },
+ },
+ },
+ {
+ "testBlock1",
+ //10000 transferred from testBankAddress to account1Addr
+ statediff.Args{
+ OldStateRoot: block0.Root(),
+ NewStateRoot: block1.Root(),
+ BlockNumber: block1.Number(),
+ BlockHash: block1.Hash(),
+ },
+ &statediff.StateObject{
+ BlockNumber: block1.Number(),
+ BlockHash: block1.Hash(),
+ Nodes: []sdtypes.StateNode{
+ {
+ Path: []byte{},
+ NodeType: sdtypes.Branch,
+ NodeValue: block1BranchRootNode,
+ StorageNodes: emptyStorage,
+ },
+ {
+ Path: []byte{'\x00'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.BankLeafKey,
+ NodeValue: bankAccountAtBlock1LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ {
+ Path: []byte{'\x05'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: minerLeafKey,
+ NodeValue: minerAccountAtBlock1LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ {
+ Path: []byte{'\x0e'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.Account1LeafKey,
+ NodeValue: account1AtBlock1LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ },
+ },
+ },
+ {
+ "testBlock2",
+ // 1000 transferred from testBankAddress to account1Addr
+ // 1000 transferred from account1Addr to account2Addr
+ // account1addr creates a new contract
+ statediff.Args{
+ OldStateRoot: block1.Root(),
+ NewStateRoot: block2.Root(),
+ BlockNumber: block2.Number(),
+ BlockHash: block2.Hash(),
+ },
+ &statediff.StateObject{
+ BlockNumber: block2.Number(),
+ BlockHash: block2.Hash(),
+ Nodes: []sdtypes.StateNode{
+ {
+ Path: []byte{},
+ NodeType: sdtypes.Branch,
+ NodeValue: block2BranchRootNode,
+ StorageNodes: emptyStorage,
+ },
+ {
+ Path: []byte{'\x00'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.BankLeafKey,
+ NodeValue: bankAccountAtBlock2LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ {
+ Path: []byte{'\x05'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: minerLeafKey,
+ NodeValue: minerAccountAtBlock2LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ {
+ Path: []byte{'\x0e'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.Account1LeafKey,
+ NodeValue: account1AtBlock2LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ {
+ Path: []byte{'\x06'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: contractLeafKey,
+ NodeValue: contractAccountAtBlock2LeafNode,
+ StorageNodes: []sdtypes.StorageNode{
+ {
+ Path: []byte{},
+ NodeType: sdtypes.Branch,
+ NodeValue: block2StorageBranchRootNode,
+ },
+ {
+ Path: []byte{'\x02'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: slot0StorageKey.Bytes(),
+ NodeValue: slot0StorageLeafNode,
+ },
+ {
+ Path: []byte{'\x0b'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: slot1StorageKey.Bytes(),
+ NodeValue: slot1StorageLeafNode,
+ },
+ },
+ },
+ {
+ Path: []byte{'\x0c'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.Account2LeafKey,
+ NodeValue: account2AtBlock2LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ },
+ CodeAndCodeHashes: []sdtypes.CodeAndCodeHash{
+ {
+ Hash: testhelpers.CodeHash,
+ Code: testhelpers.ByteCodeAfterDeployment,
+ },
+ },
+ },
+ },
+ {
+ "testBlock3",
+ //the contract's storage is changed
+ //and the block is mined by account 2
+ statediff.Args{
+ OldStateRoot: block2.Root(),
+ NewStateRoot: block3.Root(),
+ BlockNumber: block3.Number(),
+ BlockHash: block3.Hash(),
+ },
+ &statediff.StateObject{
+ BlockNumber: block3.Number(),
+ BlockHash: block3.Hash(),
+ Nodes: []sdtypes.StateNode{
+ {
+ Path: []byte{},
+ NodeType: sdtypes.Branch,
+ NodeValue: block3BranchRootNode,
+ StorageNodes: emptyStorage,
+ },
+ {
+ Path: []byte{'\x00'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.BankLeafKey,
+ NodeValue: bankAccountAtBlock3LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ {
+ Path: []byte{'\x06'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: contractLeafKey,
+ NodeValue: contractAccountAtBlock3LeafNode,
+ StorageNodes: []sdtypes.StorageNode{
+ {
+ Path: []byte{},
+ NodeType: sdtypes.Branch,
+ NodeValue: block3StorageBranchRootNode,
+ },
+ {
+ Path: []byte{'\x0c'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: slot3StorageKey.Bytes(),
+ NodeValue: slot3StorageLeafNode,
+ },
+ },
+ },
+ {
+ Path: []byte{'\x0c'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.Account2LeafKey,
+ NodeValue: account2AtBlock3LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ },
+ },
+ },
+ }
+
+ for i, test := range tests {
+ diff, err := builder.BuildStateDiffObject(test.startingArguments, params)
+ if err != nil {
+ t.Error(err)
+ }
+ receivedStateDiffRlp, err := rlp.EncodeToBytes(diff)
+ if err != nil {
+ t.Error(err)
+ }
+ expectedStateDiffRlp, err := rlp.EncodeToBytes(test.expected)
+ if err != nil {
+ t.Error(err)
+ }
+ sort.Slice(receivedStateDiffRlp, func(i, j int) bool { return receivedStateDiffRlp[i] < receivedStateDiffRlp[j] })
+ sort.Slice(expectedStateDiffRlp, func(i, j int) bool { return expectedStateDiffRlp[i] < expectedStateDiffRlp[j] })
+ if !bytes.Equal(receivedStateDiffRlp, expectedStateDiffRlp) {
+ t.Logf("Test failed: %s", test.name)
+ t.Errorf("actual state diff: %+v\r\n\r\n\r\nexpected state diff: %+v", diff, test.expected)
+ }
+ // Let's also confirm that our root state nodes form the state root hash in the headers
+ if i > 0 {
+ block := blocks[i-1]
+ expectedStateRoot := block.Root()
+ for _, node := range test.expected.Nodes {
+ if bytes.Equal(node.Path, []byte{}) {
+ stateRoot := crypto.Keccak256Hash(node.NodeValue)
+ if !bytes.Equal(expectedStateRoot.Bytes(), stateRoot.Bytes()) {
+ t.Logf("Test failed: %s", test.name)
+ t.Errorf("actual stateroot: %x\r\nexpected stateroot: %x", stateRoot.Bytes(), expectedStateRoot.Bytes())
+ }
+ }
+ }
+ }
+ }
+}
+
+func TestBuilderWithWatchedAddressList(t *testing.T) {
+ blocks, chain := testhelpers.MakeChain(3, testhelpers.Genesis, testhelpers.TestChainGen)
+ contractLeafKey = testhelpers.AddressToLeafKey(testhelpers.ContractAddr)
+ defer chain.Stop()
+ block0 = testhelpers.Genesis
+ block1 = blocks[0]
+ block2 = blocks[1]
+ block3 = blocks[2]
+ params := statediff.Params{
+ WatchedAddresses: []common.Address{testhelpers.Account1Addr, testhelpers.ContractAddr},
+ }
+ builder = statediff.NewBuilder(chain.StateCache())
+
+ var tests = []struct {
+ name string
+ startingArguments statediff.Args
+ expected *statediff.StateObject
+ }{
+ {
+ "testEmptyDiff",
+ statediff.Args{
+ OldStateRoot: block0.Root(),
+ NewStateRoot: block0.Root(),
+ BlockNumber: block0.Number(),
+ BlockHash: block0.Hash(),
+ },
+ &statediff.StateObject{
+ BlockNumber: block0.Number(),
+ BlockHash: block0.Hash(),
+ Nodes: emptyDiffs,
+ },
+ },
+ {
+ "testBlock0",
+ //10000 transferred from testBankAddress to account1Addr
+ statediff.Args{
+ OldStateRoot: testhelpers.NullHash,
+ NewStateRoot: block0.Root(),
+ BlockNumber: block0.Number(),
+ BlockHash: block0.Hash(),
+ },
+ &statediff.StateObject{
+ BlockNumber: block0.Number(),
+ BlockHash: block0.Hash(),
+ Nodes: emptyDiffs,
+ },
+ },
+ {
+ "testBlock1",
+ //10000 transferred from testBankAddress to account1Addr
+ statediff.Args{
+ OldStateRoot: block0.Root(),
+ NewStateRoot: block1.Root(),
+ BlockNumber: block1.Number(),
+ BlockHash: block1.Hash(),
+ },
+ &statediff.StateObject{
+ BlockNumber: block1.Number(),
+ BlockHash: block1.Hash(),
+ Nodes: []sdtypes.StateNode{
+ {
+ Path: []byte{'\x0e'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.Account1LeafKey,
+ NodeValue: account1AtBlock1LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ },
+ },
+ },
+ {
+ "testBlock2",
+ //1000 transferred from testBankAddress to account1Addr
+ //1000 transferred from account1Addr to account2Addr
+ statediff.Args{
+ OldStateRoot: block1.Root(),
+ NewStateRoot: block2.Root(),
+ BlockNumber: block2.Number(),
+ BlockHash: block2.Hash(),
+ },
+ &statediff.StateObject{
+ BlockNumber: block2.Number(),
+ BlockHash: block2.Hash(),
+ Nodes: []sdtypes.StateNode{
+ {
+ Path: []byte{'\x06'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: contractLeafKey,
+ NodeValue: contractAccountAtBlock2LeafNode,
+ StorageNodes: []sdtypes.StorageNode{
+ {
+ Path: []byte{'\x02'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: slot0StorageKey.Bytes(),
+ NodeValue: slot0StorageLeafNode,
+ },
+ {
+ Path: []byte{'\x0b'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: slot1StorageKey.Bytes(),
+ NodeValue: slot1StorageLeafNode,
+ },
+ },
+ },
+ {
+ Path: []byte{'\x0e'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.Account1LeafKey,
+ NodeValue: account1AtBlock2LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ },
+ CodeAndCodeHashes: []sdtypes.CodeAndCodeHash{
+ {
+ Hash: testhelpers.CodeHash,
+ Code: testhelpers.ByteCodeAfterDeployment,
+ },
+ },
+ },
+ },
+ {
+ "testBlock3",
+ //the contract's storage is changed
+ //and the block is mined by account 2
+ statediff.Args{
+ OldStateRoot: block2.Root(),
+ NewStateRoot: block3.Root(),
+ BlockNumber: block3.Number(),
+ BlockHash: block3.Hash(),
+ },
+ &statediff.StateObject{
+ BlockNumber: block3.Number(),
+ BlockHash: block3.Hash(),
+ Nodes: []sdtypes.StateNode{
+ {
+ Path: []byte{'\x06'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: contractLeafKey,
+ NodeValue: contractAccountAtBlock3LeafNode,
+ StorageNodes: []sdtypes.StorageNode{
+ {
+ Path: []byte{'\x0c'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: slot3StorageKey.Bytes(),
+ NodeValue: slot3StorageLeafNode,
+ },
+ },
+ },
+ },
+ },
+ },
+ }
+
+ for _, test := range tests {
+ diff, err := builder.BuildStateDiffObject(test.startingArguments, params)
+ if err != nil {
+ t.Error(err)
+ }
+ receivedStateDiffRlp, err := rlp.EncodeToBytes(diff)
+ if err != nil {
+ t.Error(err)
+ }
+ expectedStateDiffRlp, err := rlp.EncodeToBytes(test.expected)
+ if err != nil {
+ t.Error(err)
+ }
+ sort.Slice(receivedStateDiffRlp, func(i, j int) bool { return receivedStateDiffRlp[i] < receivedStateDiffRlp[j] })
+ sort.Slice(expectedStateDiffRlp, func(i, j int) bool { return expectedStateDiffRlp[i] < expectedStateDiffRlp[j] })
+ if !bytes.Equal(receivedStateDiffRlp, expectedStateDiffRlp) {
+ t.Logf("Test failed: %s", test.name)
+ t.Errorf("actual state diff: %+v\nexpected state diff: %+v", diff, test.expected)
+ }
+ }
+}
+
+func TestBuilderWithWatchedAddressAndStorageKeyList(t *testing.T) {
+ blocks, chain := testhelpers.MakeChain(3, testhelpers.Genesis, testhelpers.TestChainGen)
+ contractLeafKey = testhelpers.AddressToLeafKey(testhelpers.ContractAddr)
+ defer chain.Stop()
+ block0 = testhelpers.Genesis
+ block1 = blocks[0]
+ block2 = blocks[1]
+ block3 = blocks[2]
+ params := statediff.Params{
+ WatchedAddresses: []common.Address{testhelpers.Account1Addr, testhelpers.ContractAddr},
+ WatchedStorageSlots: []common.Hash{slot1StorageKey},
+ }
+ builder = statediff.NewBuilder(chain.StateCache())
+
+ var tests = []struct {
+ name string
+ startingArguments statediff.Args
+ expected *statediff.StateObject
+ }{
+ {
+ "testEmptyDiff",
+ statediff.Args{
+ OldStateRoot: block0.Root(),
+ NewStateRoot: block0.Root(),
+ BlockNumber: block0.Number(),
+ BlockHash: block0.Hash(),
+ },
+ &statediff.StateObject{
+ BlockNumber: block0.Number(),
+ BlockHash: block0.Hash(),
+ Nodes: emptyDiffs,
+ },
+ },
+ {
+ "testBlock0",
+ //10000 transferred from testBankAddress to account1Addr
+ statediff.Args{
+ OldStateRoot: testhelpers.NullHash,
+ NewStateRoot: block0.Root(),
+ BlockNumber: block0.Number(),
+ BlockHash: block0.Hash(),
+ },
+ &statediff.StateObject{
+ BlockNumber: block0.Number(),
+ BlockHash: block0.Hash(),
+ Nodes: emptyDiffs,
+ },
+ },
+ {
+ "testBlock1",
+ //10000 transferred from testBankAddress to account1Addr
+ statediff.Args{
+ OldStateRoot: block0.Root(),
+ NewStateRoot: block1.Root(),
+ BlockNumber: block1.Number(),
+ BlockHash: block1.Hash(),
+ },
+ &statediff.StateObject{
+ BlockNumber: block1.Number(),
+ BlockHash: block1.Hash(),
+ Nodes: []sdtypes.StateNode{
+ {
+ Path: []byte{'\x0e'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.Account1LeafKey,
+ NodeValue: account1AtBlock1LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ },
+ },
+ },
+ {
+ "testBlock2",
+ //1000 transferred from testBankAddress to account1Addr
+ //1000 transferred from account1Addr to account2Addr
+ statediff.Args{
+ OldStateRoot: block1.Root(),
+ NewStateRoot: block2.Root(),
+ BlockNumber: block2.Number(),
+ BlockHash: block2.Hash(),
+ },
+ &statediff.StateObject{
+ BlockNumber: block2.Number(),
+ BlockHash: block2.Hash(),
+ Nodes: []sdtypes.StateNode{
+ {
+ Path: []byte{'\x06'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: contractLeafKey,
+ NodeValue: contractAccountAtBlock2LeafNode,
+ StorageNodes: []sdtypes.StorageNode{
+ {
+ Path: []byte{'\x0b'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: slot1StorageKey.Bytes(),
+ NodeValue: slot1StorageLeafNode,
+ },
+ },
+ },
+ {
+ Path: []byte{'\x0e'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.Account1LeafKey,
+ NodeValue: account1AtBlock2LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ },
+ CodeAndCodeHashes: []sdtypes.CodeAndCodeHash{
+ {
+ Hash: testhelpers.CodeHash,
+ Code: testhelpers.ByteCodeAfterDeployment,
+ },
+ },
+ },
+ },
+ {
+ "testBlock3",
+ //the contract's storage is changed
+ //and the block is mined by account 2
+ statediff.Args{
+ OldStateRoot: block2.Root(),
+ NewStateRoot: block3.Root(),
+ BlockNumber: block3.Number(),
+ BlockHash: block3.Hash(),
+ },
+ &statediff.StateObject{
+ BlockNumber: block3.Number(),
+ BlockHash: block3.Hash(),
+ Nodes: []sdtypes.StateNode{
+ {
+ Path: []byte{'\x06'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: contractLeafKey,
+ NodeValue: contractAccountAtBlock3LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ },
+ },
+ },
+ }
+
+ for _, test := range tests {
+ diff, err := builder.BuildStateDiffObject(test.startingArguments, params)
+ if err != nil {
+ t.Error(err)
+ }
+ receivedStateDiffRlp, err := rlp.EncodeToBytes(diff)
+ if err != nil {
+ t.Error(err)
+ }
+ expectedStateDiffRlp, err := rlp.EncodeToBytes(test.expected)
+ if err != nil {
+ t.Error(err)
+ }
+ sort.Slice(receivedStateDiffRlp, func(i, j int) bool { return receivedStateDiffRlp[i] < receivedStateDiffRlp[j] })
+ sort.Slice(expectedStateDiffRlp, func(i, j int) bool { return expectedStateDiffRlp[i] < expectedStateDiffRlp[j] })
+ if !bytes.Equal(receivedStateDiffRlp, expectedStateDiffRlp) {
+ t.Logf("Test failed: %s", test.name)
+ t.Errorf("actual state diff: %+v\nexpected state diff: %+v", diff, test.expected)
+ }
+ }
+}
+
+func TestBuilderWithRemovedAccountAndStorage(t *testing.T) {
+ blocks, chain := testhelpers.MakeChain(6, testhelpers.Genesis, testhelpers.TestChainGen)
+ contractLeafKey = testhelpers.AddressToLeafKey(testhelpers.ContractAddr)
+ defer chain.Stop()
+ block3 = blocks[2]
+ block4 = blocks[3]
+ block5 = blocks[4]
+ block6 = blocks[5]
+ params := statediff.Params{
+ IntermediateStateNodes: true,
+ IntermediateStorageNodes: true,
+ }
+ builder = statediff.NewBuilder(chain.StateCache())
+
+ var tests = []struct {
+ name string
+ startingArguments statediff.Args
+ expected *statediff.StateObject
+ }{
+ // blocks 0-3 are the same as in TestBuilderWithIntermediateNodes
+ {
+ "testBlock4",
+ statediff.Args{
+ OldStateRoot: block3.Root(),
+ NewStateRoot: block4.Root(),
+ BlockNumber: block4.Number(),
+ BlockHash: block4.Hash(),
+ },
+ &statediff.StateObject{
+ BlockNumber: block4.Number(),
+ BlockHash: block4.Hash(),
+ Nodes: []sdtypes.StateNode{
+ {
+ Path: []byte{},
+ NodeType: sdtypes.Branch,
+ NodeValue: block4BranchRootNode,
+ StorageNodes: emptyStorage,
+ },
+ {
+ Path: []byte{'\x00'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.BankLeafKey,
+ NodeValue: bankAccountAtBlock4LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ {
+ Path: []byte{'\x06'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: contractLeafKey,
+ NodeValue: contractAccountAtBlock4LeafNode,
+ StorageNodes: []sdtypes.StorageNode{
+ {
+ Path: []byte{},
+ NodeType: sdtypes.Branch,
+ NodeValue: block4StorageBranchRootNode,
+ },
+ {
+ Path: []byte{'\x04'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: slot2StorageKey.Bytes(),
+ NodeValue: slot2StorageLeafNode,
+ },
+ {
+ Path: []byte{'\x0b'},
+ NodeType: sdtypes.Removed,
+ NodeValue: []byte{},
+ },
+ {
+ Path: []byte{'\x0c'},
+ NodeType: sdtypes.Removed,
+ NodeValue: []byte{},
+ },
+ },
+ },
+ {
+ Path: []byte{'\x0c'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.Account2LeafKey,
+ NodeValue: account2AtBlock4LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ },
+ },
+ },
+ {
+ "testBlock5",
+ statediff.Args{
+ OldStateRoot: block4.Root(),
+ NewStateRoot: block5.Root(),
+ BlockNumber: block5.Number(),
+ BlockHash: block5.Hash(),
+ },
+ &statediff.StateObject{
+ BlockNumber: block5.Number(),
+ BlockHash: block5.Hash(),
+ Nodes: []sdtypes.StateNode{
+ {
+ Path: []byte{},
+ NodeType: sdtypes.Branch,
+ NodeValue: block5BranchRootNode,
+ StorageNodes: emptyStorage,
+ },
+ {
+ Path: []byte{'\x00'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.BankLeafKey,
+ NodeValue: bankAccountAtBlock5LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ {
+ Path: []byte{'\x06'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: contractLeafKey,
+ NodeValue: contractAccountAtBlock5LeafNode,
+ StorageNodes: []sdtypes.StorageNode{
+ {
+ Path: []byte{},
+ NodeType: sdtypes.Leaf,
+ NodeValue: slot0StorageLeafRootNode,
+ LeafKey: slot0StorageKey.Bytes(),
+ },
+ {
+ Path: []byte{'\x02'},
+ NodeType: sdtypes.Removed,
+ NodeValue: []byte{},
+ },
+ {
+ Path: []byte{'\x04'},
+ NodeType: sdtypes.Removed,
+ NodeValue: []byte{},
+ },
+ },
+ },
+ {
+ Path: []byte{'\x0e'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.Account1LeafKey,
+ NodeValue: account1AtBlock5LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ },
+ },
+ },
+ {
+ "testBlock6",
+ statediff.Args{
+ OldStateRoot: block5.Root(),
+ NewStateRoot: block6.Root(),
+ BlockNumber: block6.Number(),
+ BlockHash: block6.Hash(),
+ },
+ &statediff.StateObject{
+ BlockNumber: block6.Number(),
+ BlockHash: block6.Hash(),
+ Nodes: []sdtypes.StateNode{
+ {
+ Path: []byte{},
+ NodeType: sdtypes.Branch,
+ NodeValue: block6BranchRootNode,
+ StorageNodes: emptyStorage,
+ },
+ {
+ Path: []byte{'\x06'},
+ NodeType: sdtypes.Removed,
+ NodeValue: []byte{},
+ },
+ {
+ Path: []byte{'\x0c'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.Account2LeafKey,
+ NodeValue: account2AtBlock6LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ {
+ Path: []byte{'\x0e'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.Account1LeafKey,
+ NodeValue: account1AtBlock6LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ },
+ },
+ },
+ }
+
+ for _, test := range tests {
+ diff, err := builder.BuildStateDiffObject(test.startingArguments, params)
+ if err != nil {
+ t.Error(err)
+ }
+ receivedStateDiffRlp, err := rlp.EncodeToBytes(diff)
+ if err != nil {
+ t.Error(err)
+ }
+ expectedStateDiffRlp, err := rlp.EncodeToBytes(test.expected)
+ if err != nil {
+ t.Error(err)
+ }
+ sort.Slice(receivedStateDiffRlp, func(i, j int) bool { return receivedStateDiffRlp[i] < receivedStateDiffRlp[j] })
+ sort.Slice(expectedStateDiffRlp, func(i, j int) bool { return expectedStateDiffRlp[i] < expectedStateDiffRlp[j] })
+ if !bytes.Equal(receivedStateDiffRlp, expectedStateDiffRlp) {
+ t.Logf("Test failed: %s", test.name)
+ t.Errorf("actual state diff: %+v\r\n\r\n\r\nexpected state diff: %+v", diff, test.expected)
+ }
+ }
+}
+
+func TestBuilderWithRemovedAccountAndStorageWithoutIntermediateNodes(t *testing.T) {
+ blocks, chain := testhelpers.MakeChain(6, testhelpers.Genesis, testhelpers.TestChainGen)
+ contractLeafKey = testhelpers.AddressToLeafKey(testhelpers.ContractAddr)
+ defer chain.Stop()
+ block3 = blocks[2]
+ block4 = blocks[3]
+ block5 = blocks[4]
+ block6 = blocks[5]
+ params := statediff.Params{
+ IntermediateStateNodes: false,
+ IntermediateStorageNodes: false,
+ }
+ builder = statediff.NewBuilder(chain.StateCache())
+
+ var tests = []struct {
+ name string
+ startingArguments statediff.Args
+ expected *statediff.StateObject
+ }{
+ // blocks 0-3 are the same as in TestBuilderWithIntermediateNodes
+ {
+ "testBlock4",
+ statediff.Args{
+ OldStateRoot: block3.Root(),
+ NewStateRoot: block4.Root(),
+ BlockNumber: block4.Number(),
+ BlockHash: block4.Hash(),
+ },
+ &statediff.StateObject{
+ BlockNumber: block4.Number(),
+ BlockHash: block4.Hash(),
+ Nodes: []sdtypes.StateNode{
+ {
+ Path: []byte{'\x00'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.BankLeafKey,
+ NodeValue: bankAccountAtBlock4LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ {
+ Path: []byte{'\x06'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: contractLeafKey,
+ NodeValue: contractAccountAtBlock4LeafNode,
+ StorageNodes: []sdtypes.StorageNode{
+ {
+ Path: []byte{'\x04'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: slot2StorageKey.Bytes(),
+ NodeValue: slot2StorageLeafNode,
+ },
+ {
+ Path: []byte{'\x0b'},
+ NodeType: sdtypes.Removed,
+ NodeValue: []byte{},
+ },
+ {
+ Path: []byte{'\x0c'},
+ NodeType: sdtypes.Removed,
+ NodeValue: []byte{},
+ },
+ },
+ },
+ {
+ Path: []byte{'\x0c'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.Account2LeafKey,
+ NodeValue: account2AtBlock4LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ },
+ },
+ },
+ {
+ "testBlock5",
+ statediff.Args{
+ OldStateRoot: block4.Root(),
+ NewStateRoot: block5.Root(),
+ BlockNumber: block5.Number(),
+ BlockHash: block5.Hash(),
+ },
+ &statediff.StateObject{
+ BlockNumber: block5.Number(),
+ BlockHash: block5.Hash(),
+ Nodes: []sdtypes.StateNode{
+ {
+ Path: []byte{'\x00'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.BankLeafKey,
+ NodeValue: bankAccountAtBlock5LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ {
+ Path: []byte{'\x06'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: contractLeafKey,
+ NodeValue: contractAccountAtBlock5LeafNode,
+ StorageNodes: []sdtypes.StorageNode{
+ {
+ Path: []byte{},
+ NodeType: sdtypes.Leaf,
+ NodeValue: slot0StorageLeafRootNode,
+ LeafKey: slot0StorageKey.Bytes(),
+ },
+ {
+ Path: []byte{'\x02'},
+ NodeType: sdtypes.Removed,
+ NodeValue: []byte{},
+ },
+ {
+ Path: []byte{'\x04'},
+ NodeType: sdtypes.Removed,
+ NodeValue: []byte{},
+ },
+ },
+ },
+ {
+ Path: []byte{'\x0e'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.Account1LeafKey,
+ NodeValue: account1AtBlock5LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ },
+ },
+ },
+ {
+ "testBlock6",
+ statediff.Args{
+ OldStateRoot: block5.Root(),
+ NewStateRoot: block6.Root(),
+ BlockNumber: block6.Number(),
+ BlockHash: block6.Hash(),
+ },
+ &statediff.StateObject{
+ BlockNumber: block6.Number(),
+ BlockHash: block6.Hash(),
+ Nodes: []sdtypes.StateNode{
+ {
+ Path: []byte{'\x06'},
+ NodeType: sdtypes.Removed,
+ NodeValue: []byte{},
+ },
+ {
+ Path: []byte{'\x0c'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.Account2LeafKey,
+ NodeValue: account2AtBlock6LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ {
+ Path: []byte{'\x0e'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.Account1LeafKey,
+ NodeValue: account1AtBlock6LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ },
+ },
+ },
+ }
+
+ for _, test := range tests {
+ diff, err := builder.BuildStateDiffObject(test.startingArguments, params)
+ if err != nil {
+ t.Error(err)
+ }
+ receivedStateDiffRlp, err := rlp.EncodeToBytes(diff)
+ if err != nil {
+ t.Error(err)
+ }
+ expectedStateDiffRlp, err := rlp.EncodeToBytes(test.expected)
+ if err != nil {
+ t.Error(err)
+ }
+ sort.Slice(receivedStateDiffRlp, func(i, j int) bool { return receivedStateDiffRlp[i] < receivedStateDiffRlp[j] })
+ sort.Slice(expectedStateDiffRlp, func(i, j int) bool { return expectedStateDiffRlp[i] < expectedStateDiffRlp[j] })
+ if !bytes.Equal(receivedStateDiffRlp, expectedStateDiffRlp) {
+ t.Logf("Test failed: %s", test.name)
+ t.Errorf("actual state diff: %+v\r\n\r\n\r\nexpected state diff: %+v", diff, test.expected)
+ }
+ }
+}
+
+var (
+ slot00StorageValue = common.Hex2Bytes("9471562b71999873db5b286df957af199ec94617f7") // prefixed TestBankAddress
+
+ slot00StorageLeafNode, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("390decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563"),
+ slot00StorageValue,
+ })
+
+ contractAccountAtBlock01, _ = rlp.EncodeToBytes(state.Account{
+ Nonce: 1,
+ Balance: big.NewInt(0),
+ CodeHash: common.HexToHash("0xaaea5efba4fd7b45d7ec03918ac5d8b31aa93b48986af0e6b591f0f087c80127").Bytes(),
+ Root: crypto.Keccak256Hash(block01StorageBranchRootNode),
+ })
+ contractAccountAtBlock01LeafNode, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("3cb2583748c26e89ef19c2a8529b05a270f735553b4d44b6f2a1894987a71c8b"),
+ contractAccountAtBlock01,
+ })
+
+ bankAccountAtBlock01, _ = rlp.EncodeToBytes(state.Account{
+ Nonce: 1,
+ Balance: big.NewInt(testhelpers.TestBankFunds.Int64() + miningReward),
+ CodeHash: testhelpers.NullCodeHash.Bytes(),
+ Root: testhelpers.EmptyContractRoot,
+ })
+ bankAccountAtBlock01LeafNode, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("30bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a"),
+ bankAccountAtBlock01,
+ })
+ bankAccountAtBlock02, _ = rlp.EncodeToBytes(state.Account{
+ Nonce: 2,
+ Balance: big.NewInt(testhelpers.TestBankFunds.Int64() + miningReward*2),
+ CodeHash: testhelpers.NullCodeHash.Bytes(),
+ Root: testhelpers.EmptyContractRoot,
+ })
+ bankAccountAtBlock02LeafNode, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("2000bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a"),
+ bankAccountAtBlock02,
+ })
+
+ block01BranchRootNode, _ = rlp.EncodeToBytes([]interface{}{
+ crypto.Keccak256Hash(bankAccountAtBlock01LeafNode),
+ crypto.Keccak256Hash(contractAccountAtBlock01LeafNode),
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ })
+
+ block01StorageBranchRootNode, _ = rlp.EncodeToBytes([]interface{}{
+ []byte{},
+ []byte{},
+ crypto.Keccak256(slot00StorageLeafNode),
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ crypto.Keccak256(slot1StorageLeafNode),
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ })
+)
+
+func TestBuilderWithMovedAccount(t *testing.T) {
+ blocks, chain := testhelpers.MakeChain(2, testhelpers.Genesis, testhelpers.TestSelfDestructChainGen)
+ contractLeafKey = testhelpers.AddressToLeafKey(testhelpers.ContractAddr)
+ defer chain.Stop()
+ block0 = testhelpers.Genesis
+ block1 = blocks[0]
+ block2 = blocks[1]
+ params := statediff.Params{
+ IntermediateStateNodes: true,
+ IntermediateStorageNodes: true,
+ }
+ builder = statediff.NewBuilder(chain.StateCache())
+
+ var tests = []struct {
+ name string
+ startingArguments statediff.Args
+ expected *statediff.StateObject
+ }{
+ {
+ "testBlock1",
+ statediff.Args{
+ OldStateRoot: block0.Root(),
+ NewStateRoot: block1.Root(),
+ BlockNumber: block1.Number(),
+ BlockHash: block1.Hash(),
+ },
+ &statediff.StateObject{
+ BlockNumber: block1.Number(),
+ BlockHash: block1.Hash(),
+ Nodes: []sdtypes.StateNode{
+ {
+ Path: []byte{},
+ NodeType: sdtypes.Branch,
+ NodeValue: block01BranchRootNode,
+ StorageNodes: emptyStorage,
+ },
+ {
+ Path: []byte{'\x00'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.BankLeafKey,
+ NodeValue: bankAccountAtBlock01LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ {
+ Path: []byte{'\x01'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: contractLeafKey,
+ NodeValue: contractAccountAtBlock01LeafNode,
+ StorageNodes: []sdtypes.StorageNode{
+ {
+ Path: []byte{},
+ NodeType: sdtypes.Branch,
+ NodeValue: block01StorageBranchRootNode,
+ },
+ {
+ Path: []byte{'\x02'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: slot0StorageKey.Bytes(),
+ NodeValue: slot00StorageLeafNode,
+ },
+ {
+ Path: []byte{'\x0b'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: slot1StorageKey.Bytes(),
+ NodeValue: slot1StorageLeafNode,
+ },
+ },
+ },
+ },
+ CodeAndCodeHashes: []sdtypes.CodeAndCodeHash{
+ {
+ Hash: testhelpers.CodeHash,
+ Code: testhelpers.ByteCodeAfterDeployment,
+ },
+ },
+ },
+ },
+ {
+ "testBlock2",
+ statediff.Args{
+ OldStateRoot: block1.Root(),
+ NewStateRoot: block2.Root(),
+ BlockNumber: block2.Number(),
+ BlockHash: block2.Hash(),
+ },
+ &statediff.StateObject{
+ BlockNumber: block2.Number(),
+ BlockHash: block2.Hash(),
+ Nodes: []sdtypes.StateNode{
+ {
+ Path: []byte{},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.BankLeafKey,
+ NodeValue: bankAccountAtBlock02LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ {
+ Path: []byte{'\x01'},
+ NodeType: sdtypes.Removed,
+ NodeValue: []byte{},
+ },
+ {
+ Path: []byte{'\x00'},
+ NodeType: sdtypes.Removed,
+ NodeValue: []byte{},
+ },
+ },
+ },
+ },
+ }
+
+ for _, test := range tests {
+ diff, err := builder.BuildStateDiffObject(test.startingArguments, params)
+ if err != nil {
+ t.Error(err)
+ }
+ receivedStateDiffRlp, err := rlp.EncodeToBytes(diff)
+ if err != nil {
+ t.Error(err)
+ }
+ expectedStateDiffRlp, err := rlp.EncodeToBytes(test.expected)
+ if err != nil {
+ t.Error(err)
+ }
+ sort.Slice(receivedStateDiffRlp, func(i, j int) bool { return receivedStateDiffRlp[i] < receivedStateDiffRlp[j] })
+ sort.Slice(expectedStateDiffRlp, func(i, j int) bool { return expectedStateDiffRlp[i] < expectedStateDiffRlp[j] })
+ if !bytes.Equal(receivedStateDiffRlp, expectedStateDiffRlp) {
+ t.Logf("Test failed: %s", test.name)
+ t.Errorf("actual state diff: %+v\r\n\r\n\r\nexpected state diff: %+v", diff, test.expected)
+ }
+ }
+}
+
+func TestBuilderWithMovedAccountOnlyLeafs(t *testing.T) {
+ blocks, chain := testhelpers.MakeChain(2, testhelpers.Genesis, testhelpers.TestSelfDestructChainGen)
+ contractLeafKey = testhelpers.AddressToLeafKey(testhelpers.ContractAddr)
+ defer chain.Stop()
+ block0 = testhelpers.Genesis
+ block1 = blocks[0]
+ block2 = blocks[1]
+ params := statediff.Params{
+ IntermediateStateNodes: false,
+ IntermediateStorageNodes: false,
+ }
+ builder = statediff.NewBuilder(chain.StateCache())
+
+ var tests = []struct {
+ name string
+ startingArguments statediff.Args
+ expected *statediff.StateObject
+ }{
+ {
+ "testBlock1",
+ statediff.Args{
+ OldStateRoot: block0.Root(),
+ NewStateRoot: block1.Root(),
+ BlockNumber: block1.Number(),
+ BlockHash: block1.Hash(),
+ },
+ &statediff.StateObject{
+ BlockNumber: block1.Number(),
+ BlockHash: block1.Hash(),
+ Nodes: []sdtypes.StateNode{
+ {
+ Path: []byte{'\x00'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.BankLeafKey,
+ NodeValue: bankAccountAtBlock01LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ {
+ Path: []byte{'\x01'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: contractLeafKey,
+ NodeValue: contractAccountAtBlock01LeafNode,
+ StorageNodes: []sdtypes.StorageNode{
+ {
+ Path: []byte{'\x02'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: slot0StorageKey.Bytes(),
+ NodeValue: slot00StorageLeafNode,
+ },
+ {
+ Path: []byte{'\x0b'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: slot1StorageKey.Bytes(),
+ NodeValue: slot1StorageLeafNode,
+ },
+ },
+ },
+ },
+ CodeAndCodeHashes: []sdtypes.CodeAndCodeHash{
+ {
+ Hash: testhelpers.CodeHash,
+ Code: testhelpers.ByteCodeAfterDeployment,
+ },
+ },
+ },
+ },
+ {
+ "testBlock2",
+ statediff.Args{
+ OldStateRoot: block1.Root(),
+ NewStateRoot: block2.Root(),
+ BlockNumber: block2.Number(),
+ BlockHash: block2.Hash(),
+ },
+ &statediff.StateObject{
+ BlockNumber: block2.Number(),
+ BlockHash: block2.Hash(),
+ Nodes: []sdtypes.StateNode{
+ {
+ Path: []byte{},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.BankLeafKey,
+ NodeValue: bankAccountAtBlock02LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ {
+ Path: []byte{'\x01'},
+ NodeType: sdtypes.Removed,
+ NodeValue: []byte{},
+ },
+ {
+ Path: []byte{'\x00'},
+ NodeType: sdtypes.Removed,
+ NodeValue: []byte{},
+ },
+ },
+ },
+ },
+ }
+
+ for _, test := range tests {
+ diff, err := builder.BuildStateDiffObject(test.startingArguments, params)
+ if err != nil {
+ t.Error(err)
+ }
+ receivedStateDiffRlp, err := rlp.EncodeToBytes(diff)
+ if err != nil {
+ t.Error(err)
+ }
+ expectedStateDiffRlp, err := rlp.EncodeToBytes(test.expected)
+ if err != nil {
+ t.Error(err)
+ }
+ sort.Slice(receivedStateDiffRlp, func(i, j int) bool { return receivedStateDiffRlp[i] < receivedStateDiffRlp[j] })
+ sort.Slice(expectedStateDiffRlp, func(i, j int) bool { return expectedStateDiffRlp[i] < expectedStateDiffRlp[j] })
+ if !bytes.Equal(receivedStateDiffRlp, expectedStateDiffRlp) {
+ t.Logf("Test failed: %s", test.name)
+ t.Errorf("actual state diff: %+v\r\n\r\n\r\nexpected state diff: %+v", diff, test.expected)
+ }
+ }
+}
+
+func TestBuildStateTrie(t *testing.T) {
+ blocks, chain := testhelpers.MakeChain(3, testhelpers.Genesis, testhelpers.TestChainGen)
+ contractLeafKey = testhelpers.AddressToLeafKey(testhelpers.ContractAddr)
+ defer chain.Stop()
+ block1 = blocks[0]
+ block2 = blocks[1]
+ block3 = blocks[2]
+ builder = statediff.NewBuilder(chain.StateCache())
+
+ var tests = []struct {
+ name string
+ block *types.Block
+ expected *statediff.StateObject
+ }{
+ {
+ "testBlock1",
+ block1,
+ &statediff.StateObject{
+ BlockNumber: block1.Number(),
+ BlockHash: block1.Hash(),
+ Nodes: []sdtypes.StateNode{
+ {
+ Path: []byte{},
+ NodeType: sdtypes.Branch,
+ NodeValue: block1BranchRootNode,
+ StorageNodes: emptyStorage,
+ },
+ {
+ Path: []byte{'\x00'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.BankLeafKey,
+ NodeValue: bankAccountAtBlock1LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ {
+ Path: []byte{'\x05'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: minerLeafKey,
+ NodeValue: minerAccountAtBlock1LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ {
+ Path: []byte{'\x0e'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.Account1LeafKey,
+ NodeValue: account1AtBlock1LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ },
+ },
+ },
+ {
+ "testBlock2",
+ block2,
+ &statediff.StateObject{
+ BlockNumber: block2.Number(),
+ BlockHash: block2.Hash(),
+ Nodes: []sdtypes.StateNode{
+ {
+ Path: []byte{},
+ NodeType: sdtypes.Branch,
+ NodeValue: block2BranchRootNode,
+ StorageNodes: emptyStorage,
+ },
+ {
+ Path: []byte{'\x00'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.BankLeafKey,
+ NodeValue: bankAccountAtBlock2LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ {
+ Path: []byte{'\x05'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: minerLeafKey,
+ NodeValue: minerAccountAtBlock2LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ {
+ Path: []byte{'\x0e'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.Account1LeafKey,
+ NodeValue: account1AtBlock2LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ {
+ Path: []byte{'\x06'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: contractLeafKey,
+ NodeValue: contractAccountAtBlock2LeafNode,
+ StorageNodes: []sdtypes.StorageNode{
+ {
+ Path: []byte{},
+ NodeType: sdtypes.Branch,
+ NodeValue: block2StorageBranchRootNode,
+ },
+ {
+ Path: []byte{'\x02'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: slot0StorageKey.Bytes(),
+ NodeValue: slot0StorageLeafNode,
+ },
+ {
+ Path: []byte{'\x0b'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: slot1StorageKey.Bytes(),
+ NodeValue: slot1StorageLeafNode,
+ },
+ },
+ },
+ {
+ Path: []byte{'\x0c'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.Account2LeafKey,
+ NodeValue: account2AtBlock2LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ },
+ CodeAndCodeHashes: []sdtypes.CodeAndCodeHash{
+ {
+ Hash: testhelpers.CodeHash,
+ Code: testhelpers.ByteCodeAfterDeployment,
+ },
+ },
+ },
+ },
+ {
+ "testBlock3",
+ block3,
+ &statediff.StateObject{
+ BlockNumber: block3.Number(),
+ BlockHash: block3.Hash(),
+ Nodes: []sdtypes.StateNode{
+ {
+ Path: []byte{},
+ NodeType: sdtypes.Branch,
+ NodeValue: block3BranchRootNode,
+ StorageNodes: emptyStorage,
+ },
+ {
+ Path: []byte{'\x00'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.BankLeafKey,
+ NodeValue: bankAccountAtBlock3LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ {
+ Path: []byte{'\x05'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: minerLeafKey,
+ NodeValue: minerAccountAtBlock2LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ {
+ Path: []byte{'\x0e'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.Account1LeafKey,
+ NodeValue: account1AtBlock2LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ {
+ Path: []byte{'\x06'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: contractLeafKey,
+ NodeValue: contractAccountAtBlock3LeafNode,
+ StorageNodes: []sdtypes.StorageNode{
+ {
+ Path: []byte{},
+ NodeType: sdtypes.Branch,
+ NodeValue: block3StorageBranchRootNode,
+ },
+ {
+ Path: []byte{'\x02'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: slot0StorageKey.Bytes(),
+ NodeValue: slot0StorageLeafNode,
+ },
+ {
+ Path: []byte{'\x0b'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: slot1StorageKey.Bytes(),
+ NodeValue: slot1StorageLeafNode,
+ },
+ {
+ Path: []byte{'\x0c'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: slot3StorageKey.Bytes(),
+ NodeValue: slot3StorageLeafNode,
+ },
+ },
+ },
+ {
+ Path: []byte{'\x0c'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.Account2LeafKey,
+ NodeValue: account2AtBlock3LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ },
+ CodeAndCodeHashes: []sdtypes.CodeAndCodeHash{
+ {
+ Hash: testhelpers.CodeHash,
+ Code: testhelpers.ByteCodeAfterDeployment,
+ },
+ },
+ },
+ },
+ }
+
+ for _, test := range tests {
+ diff, err := builder.BuildStateTrieObject(test.block)
+ if err != nil {
+ t.Error(err)
+ }
+ receivedStateTrieRlp, err := rlp.EncodeToBytes(diff)
+ if err != nil {
+ t.Error(err)
+ }
+ expectedStateTrieRlp, err := rlp.EncodeToBytes(test.expected)
+ if err != nil {
+ t.Error(err)
+ }
+ sort.Slice(receivedStateTrieRlp, func(i, j int) bool { return receivedStateTrieRlp[i] < receivedStateTrieRlp[j] })
+ sort.Slice(expectedStateTrieRlp, func(i, j int) bool { return expectedStateTrieRlp[i] < expectedStateTrieRlp[j] })
+ if !bytes.Equal(receivedStateTrieRlp, expectedStateTrieRlp) {
+ t.Logf("Test failed: %s", test.name)
+ t.Errorf("actual state trie: %+v\r\n\r\n\r\nexpected state trie: %+v", diff, test.expected)
+ }
+ }
+}
+
+/*
+pragma solidity ^0.5.10;
+
+contract test {
+ address payable owner;
+
+ modifier onlyOwner {
+ require(
+ msg.sender == owner,
+ "Only owner can call this function."
+ );
+ _;
+ }
+
+ uint256[100] data;
+
+ constructor() public {
+ owner = msg.sender;
+ data = [1];
+ }
+
+ function Put(uint256 addr, uint256 value) public {
+ data[addr] = value;
+ }
+
+ function close() public onlyOwner { //onlyOwner is custom modifier
+ selfdestruct(owner); // `owner` is the owners address
+ }
+}
+*/
diff --git a/statediff/doc.go b/statediff/doc.go
new file mode 100644
index 000000000..184bc242c
--- /dev/null
+++ b/statediff/doc.go
@@ -0,0 +1,57 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+/*
+Package statediff provides an auxiliary service that processes state diff objects from incoming chain events,
+relaying the objects to any rpc subscriptions.
+
+This work is adapted from work by Charles Crain at https://github.com/jpmorganchase/quorum/blob/9b7fd9af8082795eeeb6863d9746f12b82dd5078/statediff/statediff.go
+
+The service is spun up using the below CLI flags
+--statediff: boolean flag, turns on the service
+--statediff.streamblock: boolean flag, configures the service to associate and stream out the rest of the block data with the state diffs.
+--statediff.intermediatenodes: boolean flag, tells service to include intermediate (branch and extension) nodes; default (false) processes leaf nodes only.
+--statediff.watchedaddresses: string slice flag, used to limit the state diffing process to the given addresses. Usage: --statediff.watchedaddresses=addr1 --statediff.watchedaddresses=addr2 --statediff.watchedaddresses=addr3
+
+If you wish to use the websocket endpoint to subscribe to the statediff service, be sure to open up the Websocket RPC server with the `--ws` flag. The IPC-RPC server is turned on by default.
+
+The statediffing services works only with `--syncmode="full", but -importantly- does not require garbage collection to be turned off (does not require an archival node).
+
+e.g.
+
+$ ./geth --statediff --statediff.streamblock --ws --syncmode "full"
+
+This starts up the geth node in full sync mode, starts up the statediffing service, and opens up the websocket endpoint to subscribe to the service.
+Because the "streamblock" flag has been turned on, the service will strean out block data (headers, transactions, and receipts) along with the diffed state and storage leafs.
+
+Rpc subscriptions to the service can be created using the rpc.Client.Subscribe() method,
+with the "statediff" namespace, a statediff.Payload channel, and the name of the statediff api's rpc method- "stream".
+
+e.g.
+
+cli, _ := rpc.Dial("ipcPathOrWsURL")
+stateDiffPayloadChan := make(chan statediff.Payload, 20000)
+rpcSub, err := cli.Subscribe(context.Background(), "statediff", stateDiffPayloadChan, "stream"})
+for {
+ select {
+ case stateDiffPayload := <- stateDiffPayloadChan:
+ processPayload(stateDiffPayload)
+ case err := <- rpcSub.Err():
+ log.Error(err)
+ }
+}
+*/
+package statediff
diff --git a/statediff/helpers.go b/statediff/helpers.go
new file mode 100644
index 000000000..51ac5c1be
--- /dev/null
+++ b/statediff/helpers.go
@@ -0,0 +1,98 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+// Contains a batch of utility type declarations used by the tests. As the node
+// operates on unique types, a lot of them are needed to check various features.
+
+package statediff
+
+import (
+ "fmt"
+ "sort"
+ "strings"
+
+ sdtypes "github.com/ethereum/go-ethereum/statediff/types"
+)
+
+func sortKeys(data AccountMap) []string {
+ keys := make([]string, 0, len(data))
+ for key := range data {
+ keys = append(keys, key)
+ }
+ sort.Strings(keys)
+
+ return keys
+}
+
+// findIntersection finds the set of strings from both arrays that are equivalent
+// a and b must first be sorted
+// this is used to find which keys have been both "deleted" and "created" i.e. they were updated
+func findIntersection(a, b []string) []string {
+ lenA := len(a)
+ lenB := len(b)
+ iOfA, iOfB := 0, 0
+ updates := make([]string, 0)
+ if iOfA >= lenA || iOfB >= lenB {
+ return updates
+ }
+ for {
+ switch strings.Compare(a[iOfA], b[iOfB]) {
+ // -1 when a[iOfA] < b[iOfB]
+ case -1:
+ iOfA++
+ if iOfA >= lenA {
+ return updates
+ }
+ // 0 when a[iOfA] == b[iOfB]
+ case 0:
+ updates = append(updates, a[iOfA])
+ iOfA++
+ iOfB++
+ if iOfA >= lenA || iOfB >= lenB {
+ return updates
+ }
+ // 1 when a[iOfA] > b[iOfB]
+ case 1:
+ iOfB++
+ if iOfB >= lenB {
+ return updates
+ }
+ }
+ }
+
+}
+
+// CheckKeyType checks what type of key we have
+func CheckKeyType(elements []interface{}) (sdtypes.NodeType, error) {
+ if len(elements) > 2 {
+ return sdtypes.Branch, nil
+ }
+ if len(elements) < 2 {
+ return sdtypes.Unknown, fmt.Errorf("node cannot be less than two elements in length")
+ }
+ switch elements[0].([]byte)[0] / 16 {
+ case '\x00':
+ return sdtypes.Extension, nil
+ case '\x01':
+ return sdtypes.Extension, nil
+ case '\x02':
+ return sdtypes.Leaf, nil
+ case '\x03':
+ return sdtypes.Leaf, nil
+ default:
+ return sdtypes.Unknown, fmt.Errorf("unknown hex prefix")
+ }
+}
diff --git a/statediff/indexer/helpers.go b/statediff/indexer/helpers.go
new file mode 100644
index 000000000..bb62fd079
--- /dev/null
+++ b/statediff/indexer/helpers.go
@@ -0,0 +1,55 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package indexer
+
+import (
+ "fmt"
+
+ "github.com/ethereum/go-ethereum/params"
+ "github.com/ethereum/go-ethereum/statediff/types"
+)
+
+func ResolveFromNodeType(nodeType types.NodeType) int {
+ switch nodeType {
+ case types.Branch:
+ return 0
+ case types.Extension:
+ return 1
+ case types.Leaf:
+ return 2
+ case types.Removed:
+ return 3
+ default:
+ return -1
+ }
+}
+
+// ChainConfig returns the appropriate ethereum chain config for the provided chain id
+func ChainConfig(chainID uint64) (*params.ChainConfig, error) {
+ switch chainID {
+ case 1:
+ return params.MainnetChainConfig, nil
+ case 3:
+ return params.RopstenChainConfig, nil
+ case 4:
+ return params.RinkebyChainConfig, nil
+ case 5:
+ return params.GoerliChainConfig, nil
+ default:
+ return nil, fmt.Errorf("chain config for chainid %d not available", chainID)
+ }
+}
diff --git a/statediff/indexer/indexer.go b/statediff/indexer/indexer.go
new file mode 100644
index 000000000..2a6bac3f2
--- /dev/null
+++ b/statediff/indexer/indexer.go
@@ -0,0 +1,423 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+// This package provides an interface for pushing and indexing IPLD objects into a Postgres database
+// Metrics for reporting processing and connection stats are defined in ./metrics.go
+package indexer
+
+import (
+ "fmt"
+ "math/big"
+ "time"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/core/state"
+ "github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/crypto"
+ "github.com/ethereum/go-ethereum/log"
+ "github.com/ethereum/go-ethereum/metrics"
+ "github.com/ethereum/go-ethereum/params"
+ "github.com/ethereum/go-ethereum/rlp"
+
+ node "github.com/ipfs/go-ipld-format"
+ "github.com/jmoiron/sqlx"
+ "github.com/multiformats/go-multihash"
+
+ "github.com/ethereum/go-ethereum/statediff/indexer/ipfs/ipld"
+ "github.com/ethereum/go-ethereum/statediff/indexer/models"
+ "github.com/ethereum/go-ethereum/statediff/indexer/postgres"
+ "github.com/ethereum/go-ethereum/statediff/indexer/shared"
+ sdtypes "github.com/ethereum/go-ethereum/statediff/types"
+)
+
+var (
+ indexerMetrics = RegisterIndexerMetrics(metrics.DefaultRegistry)
+ dbMetrics = RegisterDBMetrics(metrics.DefaultRegistry)
+)
+
+// Indexer interface to allow substitution of mocks for testing
+type Indexer interface {
+ PushBlock(block *types.Block, receipts types.Receipts, totalDifficulty *big.Int) (*BlockTx, error)
+ PushStateNode(tx *BlockTx, stateNode sdtypes.StateNode) error
+ PushCodeAndCodeHash(tx *BlockTx, codeAndCodeHash sdtypes.CodeAndCodeHash) error
+ ReportDBMetrics(delay time.Duration, quit <-chan bool)
+}
+
+// StateDiffIndexer satisfies the Indexer interface for ethereum statediff objects
+type StateDiffIndexer struct {
+ chainConfig *params.ChainConfig
+ dbWriter *PostgresCIDWriter
+}
+
+// NewStateDiffIndexer creates a pointer to a new PayloadConverter which satisfies the PayloadConverter interface
+func NewStateDiffIndexer(chainConfig *params.ChainConfig, db *postgres.DB) *StateDiffIndexer {
+ return &StateDiffIndexer{
+ chainConfig: chainConfig,
+ dbWriter: NewPostgresCIDWriter(db),
+ }
+}
+
+type BlockTx struct {
+ dbtx *sqlx.Tx
+ BlockNumber uint64
+ headerID int64
+ err error
+ Close func() error
+}
+
+// Reporting function to run as goroutine
+func (sdi *StateDiffIndexer) ReportDBMetrics(delay time.Duration, quit <-chan bool) {
+ if !metrics.Enabled {
+ return
+ }
+ ticker := time.NewTicker(delay)
+ go func() {
+ for {
+ select {
+ case <-ticker.C:
+ dbMetrics.Update(sdi.dbWriter.db.Stats())
+ case <-quit:
+ ticker.Stop()
+ return
+ }
+ }
+ }()
+}
+
+// Pushes and indexes block data in database, except state & storage nodes (includes header, uncles, transactions & receipts)
+// Returns an initiated DB transaction which must be Closed via defer to commit or rollback
+func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receipts, totalDifficulty *big.Int) (*BlockTx, error) {
+ start, t := time.Now(), time.Now()
+ blockHash := block.Hash()
+ blockHashStr := blockHash.String()
+ height := block.NumberU64()
+ traceMsg := fmt.Sprintf("indexer stats for statediff at %d with hash %s:\r\n", height, blockHashStr)
+ transactions := block.Transactions()
+ // Derive any missing fields
+ if err := receipts.DeriveFields(sdi.chainConfig, blockHash, height, transactions); err != nil {
+ return nil, err
+ }
+ // Generate the block iplds
+ headerNode, uncleNodes, txNodes, txTrieNodes, rctNodes, rctTrieNodes, err := ipld.FromBlockAndReceipts(block, receipts)
+ if err != nil {
+ return nil, err
+ }
+ if len(txNodes) != len(txTrieNodes) && len(rctNodes) != len(rctTrieNodes) && len(txNodes) != len(rctNodes) {
+ return nil, fmt.Errorf("expected number of transactions (%d), transaction trie nodes (%d), receipts (%d), and receipt trie nodes (%d)to be equal", len(txNodes), len(txTrieNodes), len(rctNodes), len(rctTrieNodes))
+ }
+ // Calculate reward
+ reward := CalcEthBlockReward(block.Header(), block.Uncles(), block.Transactions(), receipts)
+ t = time.Now()
+ // Begin new db tx for everything
+ tx, err := sdi.dbWriter.db.Beginx()
+ if err != nil {
+ return nil, err
+ }
+ blocktx := BlockTx{
+ dbtx: tx,
+ // handle transaction commit or rollback for any return case
+ Close: func() error {
+ var err error
+ if p := recover(); p != nil {
+ shared.Rollback(tx)
+ panic(p)
+ } else {
+ tDiff := time.Now().Sub(t)
+ indexerMetrics.tStateStoreCodeProcessing.Update(tDiff)
+ traceMsg += fmt.Sprintf("state, storage, and code storage processing time: %s\r\n", tDiff.String())
+ t = time.Now()
+ err = tx.Commit()
+ tDiff = time.Now().Sub(t)
+ indexerMetrics.tPostgresCommit.Update(tDiff)
+ traceMsg += fmt.Sprintf("postgres transaction commit duration: %s\r\n", tDiff.String())
+ }
+ traceMsg += fmt.Sprintf(" TOTAL PROCESSING DURATION: %s\r\n", time.Now().Sub(start).String())
+ log.Debug(traceMsg)
+ return err
+ },
+ }
+ tDiff := time.Now().Sub(t)
+ indexerMetrics.tFreePostgres.Update(tDiff)
+
+ traceMsg += fmt.Sprintf("time spent waiting for free postgres tx: %s:\r\n", tDiff.String())
+ t = time.Now()
+
+ // Publish and index header, collect headerID
+ headerID, err := sdi.processHeader(tx, block.Header(), headerNode, reward, totalDifficulty)
+ if err != nil {
+ return nil, err
+ }
+ tDiff = time.Now().Sub(t)
+ indexerMetrics.tHeaderProcessing.Update(tDiff)
+ traceMsg += fmt.Sprintf("header processing time: %s\r\n", tDiff.String())
+ t = time.Now()
+ // Publish and index uncles
+ if err := sdi.processUncles(tx, headerID, height, uncleNodes); err != nil {
+ return nil, err
+ }
+ tDiff = time.Now().Sub(t)
+ indexerMetrics.tUncleProcessing.Update(tDiff)
+ traceMsg += fmt.Sprintf("uncle processing time: %s\r\n", tDiff.String())
+ t = time.Now()
+ // Publish and index receipts and txs
+ if err := sdi.processReceiptsAndTxs(tx, processArgs{
+ headerID: headerID,
+ blockNumber: block.Number(),
+ receipts: receipts,
+ txs: transactions,
+ rctNodes: rctNodes,
+ rctTrieNodes: rctTrieNodes,
+ txNodes: txNodes,
+ txTrieNodes: txTrieNodes,
+ }); err != nil {
+ return nil, err
+ }
+ tDiff = time.Now().Sub(t)
+ indexerMetrics.tTxAndRecProcessing.Update(tDiff)
+ traceMsg += fmt.Sprintf("tx and receipt processing time: %s\r\n", tDiff.String())
+ t = time.Now()
+
+ blocktx.BlockNumber = height
+ blocktx.headerID = headerID
+ return &blocktx, err
+}
+
+// processHeader publishes and indexes a header IPLD in Postgres
+// it returns the headerID
+func (sdi *StateDiffIndexer) processHeader(tx *sqlx.Tx, header *types.Header, headerNode node.Node, reward, td *big.Int) (int64, error) {
+ // publish header
+ if err := shared.PublishIPLD(tx, headerNode); err != nil {
+ return 0, err
+ }
+ // index header
+ return sdi.dbWriter.upsertHeaderCID(tx, models.HeaderModel{
+ CID: headerNode.Cid().String(),
+ MhKey: shared.MultihashKeyFromCID(headerNode.Cid()),
+ ParentHash: header.ParentHash.String(),
+ BlockNumber: header.Number.String(),
+ BlockHash: header.Hash().String(),
+ TotalDifficulty: td.String(),
+ Reward: reward.String(),
+ Bloom: header.Bloom.Bytes(),
+ StateRoot: header.Root.String(),
+ RctRoot: header.ReceiptHash.String(),
+ TxRoot: header.TxHash.String(),
+ UncleRoot: header.UncleHash.String(),
+ Timestamp: header.Time,
+ })
+}
+
+func (sdi *StateDiffIndexer) processUncles(tx *sqlx.Tx, headerID int64, blockNumber uint64, uncleNodes []*ipld.EthHeader) error {
+ // publish and index uncles
+ for _, uncleNode := range uncleNodes {
+ if err := shared.PublishIPLD(tx, uncleNode); err != nil {
+ return err
+ }
+ uncleReward := CalcUncleMinerReward(blockNumber, uncleNode.Number.Uint64())
+ uncle := models.UncleModel{
+ CID: uncleNode.Cid().String(),
+ MhKey: shared.MultihashKeyFromCID(uncleNode.Cid()),
+ ParentHash: uncleNode.ParentHash.String(),
+ BlockHash: uncleNode.Hash().String(),
+ Reward: uncleReward.String(),
+ }
+ if err := sdi.dbWriter.upsertUncleCID(tx, uncle, headerID); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// processArgs bundles arugments to processReceiptsAndTxs
+type processArgs struct {
+ headerID int64
+ blockNumber *big.Int
+ receipts types.Receipts
+ txs types.Transactions
+ rctNodes []*ipld.EthReceipt
+ rctTrieNodes []*ipld.EthRctTrie
+ txNodes []*ipld.EthTx
+ txTrieNodes []*ipld.EthTxTrie
+}
+
+// processReceiptsAndTxs publishes and indexes receipt and transaction IPLDs in Postgres
+func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *sqlx.Tx, args processArgs) error {
+ // Process receipts and txs
+ signer := types.MakeSigner(sdi.chainConfig, args.blockNumber)
+ for i, receipt := range args.receipts {
+ // tx that corresponds with this receipt
+ trx := args.txs[i]
+ from, err := types.Sender(signer, trx)
+ if err != nil {
+ return err
+ }
+
+ // Publishing
+ // publish trie nodes, these aren't indexed directly
+ if err := shared.PublishIPLD(tx, args.txTrieNodes[i]); err != nil {
+ return err
+ }
+ if err := shared.PublishIPLD(tx, args.rctTrieNodes[i]); err != nil {
+ return err
+ }
+ // publish the txs and receipts
+ txNode, rctNode := args.txNodes[i], args.rctNodes[i]
+ if err := shared.PublishIPLD(tx, txNode); err != nil {
+ return err
+ }
+ if err := shared.PublishIPLD(tx, rctNode); err != nil {
+ return err
+ }
+
+ // Indexing
+ // extract topic and contract data from the receipt for indexing
+ topicSets := make([][]string, 4)
+ mappedContracts := make(map[string]bool) // use map to avoid duplicate addresses
+ for _, log := range receipt.Logs {
+ for i, topic := range log.Topics {
+ topicSets[i] = append(topicSets[i], topic.Hex())
+ }
+ mappedContracts[log.Address.String()] = true
+ }
+ // these are the contracts seen in the logs
+ logContracts := make([]string, 0, len(mappedContracts))
+ for addr := range mappedContracts {
+ logContracts = append(logContracts, addr)
+ }
+ // this is the contract address if this receipt is for a contract creation tx
+ contract := shared.HandleZeroAddr(receipt.ContractAddress)
+ var contractHash string
+ isDeployment := contract != ""
+ if isDeployment {
+ contractHash = crypto.Keccak256Hash(common.HexToAddress(contract).Bytes()).String()
+ // if tx is a contract deployment, publish the data (code)
+ // codec doesn't matter in this case sine we are not interested in the cid and the db key is multihash-derived
+ // TODO: THE DATA IS NOT DIRECTLY THE CONTRACT CODE; THERE IS A MISSING PROCESSING STEP HERE
+ // the contractHash => contract code is not currently correct
+ if _, err := shared.PublishRaw(tx, ipld.MEthStorageTrie, multihash.KECCAK_256, trx.Data()); err != nil {
+ return err
+ }
+ }
+ // index tx first so that the receipt can reference it by FK
+ txModel := models.TxModel{
+ Dst: shared.HandleZeroAddrPointer(trx.To()),
+ Src: shared.HandleZeroAddr(from),
+ TxHash: trx.Hash().String(),
+ Index: int64(i),
+ Data: trx.Data(),
+ Deployment: isDeployment,
+ CID: txNode.Cid().String(),
+ MhKey: shared.MultihashKeyFromCID(txNode.Cid()),
+ }
+ txID, err := sdi.dbWriter.upsertTransactionCID(tx, txModel, args.headerID)
+ if err != nil {
+ return err
+ }
+ // index the receipt
+ rctModel := models.ReceiptModel{
+ Topic0s: topicSets[0],
+ Topic1s: topicSets[1],
+ Topic2s: topicSets[2],
+ Topic3s: topicSets[3],
+ Contract: contract,
+ ContractHash: contractHash,
+ LogContracts: logContracts,
+ CID: rctNode.Cid().String(),
+ MhKey: shared.MultihashKeyFromCID(rctNode.Cid()),
+ }
+ if err := sdi.dbWriter.upsertReceiptCID(tx, rctModel, txID); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (sdi *StateDiffIndexer) PushStateNode(tx *BlockTx, stateNode sdtypes.StateNode) error {
+ // publish the state node
+ stateCIDStr, err := shared.PublishRaw(tx.dbtx, ipld.MEthStateTrie, multihash.KECCAK_256, stateNode.NodeValue)
+ if err != nil {
+ return err
+ }
+ mhKey, _ := shared.MultihashKeyFromCIDString(stateCIDStr)
+ stateModel := models.StateNodeModel{
+ Path: stateNode.Path,
+ StateKey: common.BytesToHash(stateNode.LeafKey).String(),
+ CID: stateCIDStr,
+ MhKey: mhKey,
+ NodeType: ResolveFromNodeType(stateNode.NodeType),
+ }
+ // index the state node, collect the stateID to reference by FK
+ stateID, err := sdi.dbWriter.upsertStateCID(tx.dbtx, stateModel, tx.headerID)
+ if err != nil {
+ return err
+ }
+ // if we have a leaf, decode and index the account data
+ if stateNode.NodeType == sdtypes.Leaf {
+ var i []interface{}
+ if err := rlp.DecodeBytes(stateNode.NodeValue, &i); err != nil {
+ return fmt.Errorf("error decoding state leaf node rlp: %s", err.Error())
+ }
+ if len(i) != 2 {
+ return fmt.Errorf("eth IPLDPublisher expected state leaf node rlp to decode into two elements")
+ }
+ var account state.Account
+ if err := rlp.DecodeBytes(i[1].([]byte), &account); err != nil {
+ return fmt.Errorf("error decoding state account rlp: %s", err.Error())
+ }
+ accountModel := models.StateAccountModel{
+ Balance: account.Balance.String(),
+ Nonce: account.Nonce,
+ CodeHash: account.CodeHash,
+ StorageRoot: account.Root.String(),
+ }
+ if err := sdi.dbWriter.upsertStateAccount(tx.dbtx, accountModel, stateID); err != nil {
+ return err
+ }
+ }
+ // if there are any storage nodes associated with this node, publish and index them
+ for _, storageNode := range stateNode.StorageNodes {
+ storageCIDStr, err := shared.PublishRaw(tx.dbtx, ipld.MEthStorageTrie, multihash.KECCAK_256, storageNode.NodeValue)
+ if err != nil {
+ return err
+ }
+ mhKey, _ := shared.MultihashKeyFromCIDString(storageCIDStr)
+ storageModel := models.StorageNodeModel{
+ Path: storageNode.Path,
+ StorageKey: common.BytesToHash(storageNode.LeafKey).String(),
+ CID: storageCIDStr,
+ MhKey: mhKey,
+ NodeType: ResolveFromNodeType(storageNode.NodeType),
+ }
+ if err := sdi.dbWriter.upsertStorageCID(tx.dbtx, storageModel, stateID); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+// Publishes code and codehash pairs to the ipld database
+func (sdi *StateDiffIndexer) PushCodeAndCodeHash(tx *BlockTx, codeAndCodeHash sdtypes.CodeAndCodeHash) error {
+ // codec doesn't matter since db key is multihash-based
+ mhKey, err := shared.MultihashKeyFromKeccak256(codeAndCodeHash.Hash)
+ if err != nil {
+ return err
+ }
+ if err := shared.PublishDirect(tx.dbtx, mhKey, codeAndCodeHash.Code); err != nil {
+ return err
+ }
+ return nil
+}
diff --git a/statediff/indexer/indexer_test.go b/statediff/indexer/indexer_test.go
new file mode 100644
index 000000000..1a7c76763
--- /dev/null
+++ b/statediff/indexer/indexer_test.go
@@ -0,0 +1,292 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package indexer_test
+
+import (
+ "testing"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/params"
+ "github.com/ipfs/go-cid"
+ "github.com/ipfs/go-ipfs-blockstore"
+ "github.com/ipfs/go-ipfs-ds-help"
+
+ ind "github.com/ethereum/go-ethereum/statediff/indexer"
+ "github.com/ethereum/go-ethereum/statediff/indexer/mocks"
+ eth "github.com/ethereum/go-ethereum/statediff/indexer/models"
+ "github.com/ethereum/go-ethereum/statediff/indexer/postgres"
+ "github.com/ethereum/go-ethereum/statediff/indexer/shared"
+)
+
+var (
+ db *postgres.DB
+ err error
+ indexer *ind.StateDiffIndexer
+ ipfsPgGet = `SELECT data FROM public.blocks
+ WHERE key = $1`
+)
+
+func expectTrue(t *testing.T, value bool) {
+ if !value {
+ t.Fatalf("Assertion failed")
+ }
+}
+
+func setup(t *testing.T) {
+ db, err = shared.SetupDB()
+ if err != nil {
+ t.Fatal(err)
+ }
+ indexer = ind.NewStateDiffIndexer(params.MainnetChainConfig, db)
+ var tx *ind.BlockTx
+ tx, err = indexer.PushBlock(
+ mocks.MockBlock,
+ mocks.MockReceipts,
+ mocks.MockBlock.Difficulty())
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer tx.Close()
+ for _, node := range mocks.StateDiffs {
+ err = indexer.PushStateNode(tx, node)
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ shared.ExpectEqual(t, tx.BlockNumber, mocks.BlockNumber.Uint64())
+}
+
+func tearDown(t *testing.T) {
+ ind.TearDownDB(t, db)
+}
+
+func TestPublishAndIndexer(t *testing.T) {
+ t.Run("Publish and index header IPLDs in a single tx", func(t *testing.T) {
+ setup(t)
+ defer tearDown(t)
+ pgStr := `SELECT cid, td, reward, id
+ FROM eth.header_cids
+ WHERE block_number = $1`
+ // check header was properly indexed
+ type res struct {
+ CID string
+ TD string
+ Reward string
+ ID int
+ }
+ header := new(res)
+ err = db.QueryRowx(pgStr, 1).StructScan(header)
+ if err != nil {
+ t.Fatal(err)
+ }
+ shared.ExpectEqual(t, header.CID, mocks.HeaderCID.String())
+ shared.ExpectEqual(t, header.TD, mocks.MockBlock.Difficulty().String())
+ shared.ExpectEqual(t, header.Reward, "5000000000000011250")
+ dc, err := cid.Decode(header.CID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ mhKey := dshelp.MultihashToDsKey(dc.Hash())
+ prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
+ var data []byte
+ err = db.Get(&data, ipfsPgGet, prefixedKey)
+ if err != nil {
+ t.Fatal(err)
+ }
+ shared.ExpectEqual(t, data, mocks.MockHeaderRlp)
+ })
+
+ t.Run("Publish and index transaction IPLDs in a single tx", func(t *testing.T) {
+ setup(t)
+ defer tearDown(t)
+ // check that txs were properly indexed
+ trxs := make([]string, 0)
+ pgStr := `SELECT transaction_cids.cid FROM eth.transaction_cids INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id)
+ WHERE header_cids.block_number = $1`
+ err = db.Select(&trxs, pgStr, 1)
+ if err != nil {
+ t.Fatal(err)
+ }
+ shared.ExpectEqual(t, len(trxs), 3)
+ expectTrue(t, shared.ListContainsString(trxs, mocks.Trx1CID.String()))
+ expectTrue(t, shared.ListContainsString(trxs, mocks.Trx2CID.String()))
+ expectTrue(t, shared.ListContainsString(trxs, mocks.Trx3CID.String()))
+ // and published
+ for _, c := range trxs {
+ dc, err := cid.Decode(c)
+ if err != nil {
+ t.Fatal(err)
+ }
+ mhKey := dshelp.MultihashToDsKey(dc.Hash())
+ prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
+ var data []byte
+ err = db.Get(&data, ipfsPgGet, prefixedKey)
+ if err != nil {
+ t.Fatal(err)
+ }
+ switch c {
+ case mocks.Trx1CID.String():
+ shared.ExpectEqual(t, data, mocks.MockTransactions.GetRlp(0))
+ case mocks.Trx2CID.String():
+ shared.ExpectEqual(t, data, mocks.MockTransactions.GetRlp(1))
+ case mocks.Trx3CID.String():
+ shared.ExpectEqual(t, data, mocks.MockTransactions.GetRlp(2))
+ }
+ }
+ })
+
+ t.Run("Publish and index receipt IPLDs in a single tx", func(t *testing.T) {
+ setup(t)
+ defer tearDown(t)
+ // check receipts were properly indexed
+ rcts := make([]string, 0)
+ pgStr := `SELECT receipt_cids.cid FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids
+ WHERE receipt_cids.tx_id = transaction_cids.id
+ AND transaction_cids.header_id = header_cids.id
+ AND header_cids.block_number = $1`
+ err = db.Select(&rcts, pgStr, 1)
+ if err != nil {
+ t.Fatal(err)
+ }
+ shared.ExpectEqual(t, len(rcts), 3)
+ expectTrue(t, shared.ListContainsString(rcts, mocks.Rct1CID.String()))
+ expectTrue(t, shared.ListContainsString(rcts, mocks.Rct2CID.String()))
+ expectTrue(t, shared.ListContainsString(rcts, mocks.Rct3CID.String()))
+ // and published
+ for _, c := range rcts {
+ dc, err := cid.Decode(c)
+ if err != nil {
+ t.Fatal(err)
+ }
+ mhKey := dshelp.MultihashToDsKey(dc.Hash())
+ prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
+ var data []byte
+ err = db.Get(&data, ipfsPgGet, prefixedKey)
+ if err != nil {
+ t.Fatal(err)
+ }
+ switch c {
+ case mocks.Rct1CID.String():
+ shared.ExpectEqual(t, data, mocks.MockReceipts.GetRlp(0))
+ case mocks.Rct2CID.String():
+ shared.ExpectEqual(t, data, mocks.MockReceipts.GetRlp(1))
+ case mocks.Rct3CID.String():
+ shared.ExpectEqual(t, data, mocks.MockReceipts.GetRlp(2))
+ }
+ }
+ })
+
+ t.Run("Publish and index state IPLDs in a single tx", func(t *testing.T) {
+ setup(t)
+ defer tearDown(t)
+ // check that state nodes were properly indexed and published
+ stateNodes := make([]eth.StateNodeModel, 0)
+ pgStr := `SELECT state_cids.id, state_cids.cid, state_cids.state_leaf_key, state_cids.node_type, state_cids.state_path, state_cids.header_id
+ FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
+ WHERE header_cids.block_number = $1`
+ err = db.Select(&stateNodes, pgStr, 1)
+ if err != nil {
+ t.Fatal(err)
+ }
+ shared.ExpectEqual(t, len(stateNodes), 2)
+ for _, stateNode := range stateNodes {
+ var data []byte
+ dc, err := cid.Decode(stateNode.CID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ mhKey := dshelp.MultihashToDsKey(dc.Hash())
+ prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
+ err = db.Get(&data, ipfsPgGet, prefixedKey)
+ if err != nil {
+ t.Fatal(err)
+ }
+ pgStr = `SELECT * from eth.state_accounts WHERE state_id = $1`
+ var account eth.StateAccountModel
+ err = db.Get(&account, pgStr, stateNode.ID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if stateNode.CID == mocks.State1CID.String() {
+ shared.ExpectEqual(t, stateNode.NodeType, 2)
+ shared.ExpectEqual(t, stateNode.StateKey, common.BytesToHash(mocks.ContractLeafKey).Hex())
+ shared.ExpectEqual(t, stateNode.Path, []byte{'\x06'})
+ shared.ExpectEqual(t, data, mocks.ContractLeafNode)
+ shared.ExpectEqual(t, account, eth.StateAccountModel{
+ ID: account.ID,
+ StateID: stateNode.ID,
+ Balance: "0",
+ CodeHash: mocks.ContractCodeHash.Bytes(),
+ StorageRoot: mocks.ContractRoot,
+ Nonce: 1,
+ })
+ }
+ if stateNode.CID == mocks.State2CID.String() {
+ shared.ExpectEqual(t, stateNode.NodeType, 2)
+ shared.ExpectEqual(t, stateNode.StateKey, common.BytesToHash(mocks.AccountLeafKey).Hex())
+ shared.ExpectEqual(t, stateNode.Path, []byte{'\x0c'})
+ shared.ExpectEqual(t, data, mocks.AccountLeafNode)
+ shared.ExpectEqual(t, account, eth.StateAccountModel{
+ ID: account.ID,
+ StateID: stateNode.ID,
+ Balance: "1000",
+ CodeHash: mocks.AccountCodeHash.Bytes(),
+ StorageRoot: mocks.AccountRoot,
+ Nonce: 0,
+ })
+ }
+ }
+ pgStr = `SELECT * from eth.state_accounts WHERE state_id = $1`
+ })
+
+ t.Run("Publish and index storage IPLDs in a single tx", func(t *testing.T) {
+ setup(t)
+ defer tearDown(t)
+ // check that storage nodes were properly indexed
+ storageNodes := make([]eth.StorageNodeWithStateKeyModel, 0)
+ pgStr := `SELECT storage_cids.cid, state_cids.state_leaf_key, storage_cids.storage_leaf_key, storage_cids.node_type, storage_cids.storage_path
+ FROM eth.storage_cids, eth.state_cids, eth.header_cids
+ WHERE storage_cids.state_id = state_cids.id
+ AND state_cids.header_id = header_cids.id
+ AND header_cids.block_number = $1`
+ err = db.Select(&storageNodes, pgStr, 1)
+ if err != nil {
+ t.Fatal(err)
+ }
+ shared.ExpectEqual(t, len(storageNodes), 1)
+ shared.ExpectEqual(t, storageNodes[0], eth.StorageNodeWithStateKeyModel{
+ CID: mocks.StorageCID.String(),
+ NodeType: 2,
+ StorageKey: common.BytesToHash(mocks.StorageLeafKey).Hex(),
+ StateKey: common.BytesToHash(mocks.ContractLeafKey).Hex(),
+ Path: []byte{},
+ })
+ var data []byte
+ dc, err := cid.Decode(storageNodes[0].CID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ mhKey := dshelp.MultihashToDsKey(dc.Hash())
+ prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
+ err = db.Get(&data, ipfsPgGet, prefixedKey)
+ if err != nil {
+ t.Fatal(err)
+ }
+ shared.ExpectEqual(t, data, mocks.StorageLeafNode)
+ })
+}
diff --git a/statediff/indexer/ipfs/ipld/eth_account.go b/statediff/indexer/ipfs/ipld/eth_account.go
new file mode 100644
index 000000000..5d80af1d9
--- /dev/null
+++ b/statediff/indexer/ipfs/ipld/eth_account.go
@@ -0,0 +1,175 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package ipld
+
+import (
+ "encoding/json"
+ "fmt"
+ "math/big"
+
+ "github.com/ipfs/go-cid"
+ node "github.com/ipfs/go-ipld-format"
+)
+
+// EthAccountSnapshot (eth-account-snapshot codec 0x97)
+// represents an ethereum account, i.e. a wallet address or
+// a smart contract
+type EthAccountSnapshot struct {
+ *EthAccount
+
+ cid cid.Cid
+ rawdata []byte
+}
+
+// EthAccount is the building block of EthAccountSnapshot.
+// Or, is the former stripped of its cid and rawdata components.
+type EthAccount struct {
+ Nonce uint64
+ Balance *big.Int
+ Root []byte // This is the storage root trie
+ CodeHash []byte // This is the hash of the EVM code
+}
+
+// Static (compile time) check that EthAccountSnapshot satisfies the
+// node.Node interface.
+var _ node.Node = (*EthAccountSnapshot)(nil)
+
+/*
+ INPUT
+*/
+
+// Input should be managed by EthStateTrie
+
+/*
+ OUTPUT
+*/
+
+// Output should be managed by EthStateTrie
+
+/*
+ Block INTERFACE
+*/
+
+// RawData returns the binary of the RLP encode of the account snapshot.
+func (as *EthAccountSnapshot) RawData() []byte {
+ return as.rawdata
+}
+
+// Cid returns the cid of the transaction.
+func (as *EthAccountSnapshot) Cid() cid.Cid {
+ return as.cid
+}
+
+// String is a helper for output
+func (as *EthAccountSnapshot) String() string {
+ return fmt.Sprintf("", as.cid)
+}
+
+// Loggable returns in a map the type of IPLD Link.
+func (as *EthAccountSnapshot) Loggable() map[string]interface{} {
+ return map[string]interface{}{
+ "type": "eth-account-snapshot",
+ }
+}
+
+/*
+ Node INTERFACE
+*/
+
+// Resolve resolves a path through this node, stopping at any link boundary
+// and returning the object found as well as the remaining path to traverse
+func (as *EthAccountSnapshot) Resolve(p []string) (interface{}, []string, error) {
+ if len(p) == 0 {
+ return as, nil, nil
+ }
+
+ if len(p) > 1 {
+ return nil, nil, fmt.Errorf("unexpected path elements past %s", p[0])
+ }
+
+ switch p[0] {
+ case "balance":
+ return as.Balance, nil, nil
+ case "codeHash":
+ return &node.Link{Cid: keccak256ToCid(RawBinary, as.CodeHash)}, nil, nil
+ case "nonce":
+ return as.Nonce, nil, nil
+ case "root":
+ return &node.Link{Cid: keccak256ToCid(MEthStorageTrie, as.Root)}, nil, nil
+ default:
+ return nil, nil, fmt.Errorf("no such link")
+ }
+}
+
+// Tree lists all paths within the object under 'path', and up to the given depth.
+// To list the entire object (similar to `find .`) pass "" and -1
+func (as *EthAccountSnapshot) Tree(p string, depth int) []string {
+ if p != "" || depth == 0 {
+ return nil
+ }
+ return []string{"balance", "codeHash", "nonce", "root"}
+}
+
+// ResolveLink is a helper function that calls resolve and asserts the
+// output is a link
+func (as *EthAccountSnapshot) ResolveLink(p []string) (*node.Link, []string, error) {
+ obj, rest, err := as.Resolve(p)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ if lnk, ok := obj.(*node.Link); ok {
+ return lnk, rest, nil
+ }
+
+ return nil, nil, fmt.Errorf("resolved item was not a link")
+}
+
+// Copy will go away. It is here to comply with the interface.
+func (as *EthAccountSnapshot) Copy() node.Node {
+ panic("dont use this yet")
+}
+
+// Links is a helper function that returns all links within this object
+func (as *EthAccountSnapshot) Links() []*node.Link {
+ return nil
+}
+
+// Stat will go away. It is here to comply with the interface.
+func (as *EthAccountSnapshot) Stat() (*node.NodeStat, error) {
+ return &node.NodeStat{}, nil
+}
+
+// Size will go away. It is here to comply with the interface.
+func (as *EthAccountSnapshot) Size() (uint64, error) {
+ return 0, nil
+}
+
+/*
+ EthAccountSnapshot functions
+*/
+
+// MarshalJSON processes the transaction into readable JSON format.
+func (as *EthAccountSnapshot) MarshalJSON() ([]byte, error) {
+ out := map[string]interface{}{
+ "balance": as.Balance,
+ "codeHash": keccak256ToCid(RawBinary, as.CodeHash),
+ "nonce": as.Nonce,
+ "root": keccak256ToCid(MEthStorageTrie, as.Root),
+ }
+ return json.Marshal(out)
+}
diff --git a/statediff/indexer/ipfs/ipld/eth_header.go b/statediff/indexer/ipfs/ipld/eth_header.go
new file mode 100644
index 000000000..c33931d4f
--- /dev/null
+++ b/statediff/indexer/ipfs/ipld/eth_header.go
@@ -0,0 +1,256 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package ipld
+
+import (
+ "encoding/json"
+ "fmt"
+
+ "github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/rlp"
+ "github.com/ipfs/go-cid"
+ node "github.com/ipfs/go-ipld-format"
+ mh "github.com/multiformats/go-multihash"
+)
+
+// EthHeader (eth-block, codec 0x90), represents an ethereum block header
+type EthHeader struct {
+ *types.Header
+
+ cid cid.Cid
+ rawdata []byte
+}
+
+// Static (compile time) check that EthHeader satisfies the node.Node interface.
+var _ node.Node = (*EthHeader)(nil)
+
+/*
+ INPUT
+*/
+
+// NewEthHeader converts a *types.Header into an EthHeader IPLD node
+func NewEthHeader(header *types.Header) (*EthHeader, error) {
+ headerRLP, err := rlp.EncodeToBytes(header)
+ if err != nil {
+ return nil, err
+ }
+ c, err := RawdataToCid(MEthHeader, headerRLP, mh.KECCAK_256)
+ if err != nil {
+ return nil, err
+ }
+ return &EthHeader{
+ Header: header,
+ cid: c,
+ rawdata: headerRLP,
+ }, nil
+}
+
+/*
+ OUTPUT
+*/
+
+// DecodeEthHeader takes a cid and its raw binary data
+// from IPFS and returns an EthTx object for further processing.
+func DecodeEthHeader(c cid.Cid, b []byte) (*EthHeader, error) {
+ var h *types.Header
+ if err := rlp.DecodeBytes(b, h); err != nil {
+ return nil, err
+ }
+ return &EthHeader{
+ Header: h,
+ cid: c,
+ rawdata: b,
+ }, nil
+}
+
+/*
+ Block INTERFACE
+*/
+
+// RawData returns the binary of the RLP encode of the block header.
+func (b *EthHeader) RawData() []byte {
+ return b.rawdata
+}
+
+// Cid returns the cid of the block header.
+func (b *EthHeader) Cid() cid.Cid {
+ return b.cid
+}
+
+// String is a helper for output
+func (b *EthHeader) String() string {
+ return fmt.Sprintf("", b.cid)
+}
+
+// Loggable returns a map the type of IPLD Link.
+func (b *EthHeader) Loggable() map[string]interface{} {
+ return map[string]interface{}{
+ "type": "eth-block",
+ }
+}
+
+/*
+ Node INTERFACE
+*/
+
+// Resolve resolves a path through this node, stopping at any link boundary
+// and returning the object found as well as the remaining path to traverse
+func (b *EthHeader) Resolve(p []string) (interface{}, []string, error) {
+ if len(p) == 0 {
+ return b, nil, nil
+ }
+
+ first, rest := p[0], p[1:]
+
+ switch first {
+ case "parent":
+ return &node.Link{Cid: commonHashToCid(MEthHeader, b.ParentHash)}, rest, nil
+ case "receipts":
+ return &node.Link{Cid: commonHashToCid(MEthTxReceiptTrie, b.ReceiptHash)}, rest, nil
+ case "root":
+ return &node.Link{Cid: commonHashToCid(MEthStateTrie, b.Root)}, rest, nil
+ case "tx":
+ return &node.Link{Cid: commonHashToCid(MEthTxTrie, b.TxHash)}, rest, nil
+ case "uncles":
+ return &node.Link{Cid: commonHashToCid(MEthHeaderList, b.UncleHash)}, rest, nil
+ }
+
+ if len(p) != 1 {
+ return nil, nil, fmt.Errorf("unexpected path elements past %s", first)
+ }
+
+ switch first {
+ case "bloom":
+ return b.Bloom, nil, nil
+ case "coinbase":
+ return b.Coinbase, nil, nil
+ case "difficulty":
+ return b.Difficulty, nil, nil
+ case "extra":
+ // This is a []byte. By default they are marshalled into Base64.
+ return fmt.Sprintf("0x%x", b.Extra), nil, nil
+ case "gaslimit":
+ return b.GasLimit, nil, nil
+ case "gasused":
+ return b.GasUsed, nil, nil
+ case "mixdigest":
+ return b.MixDigest, nil, nil
+ case "nonce":
+ return b.Nonce, nil, nil
+ case "number":
+ return b.Number, nil, nil
+ case "time":
+ return b.Time, nil, nil
+ default:
+ return nil, nil, fmt.Errorf("no such link")
+ }
+}
+
+// Tree lists all paths within the object under 'path', and up to the given depth.
+// To list the entire object (similar to `find .`) pass "" and -1
+func (b *EthHeader) Tree(p string, depth int) []string {
+ if p != "" || depth == 0 {
+ return nil
+ }
+
+ return []string{
+ "time",
+ "bloom",
+ "coinbase",
+ "difficulty",
+ "extra",
+ "gaslimit",
+ "gasused",
+ "mixdigest",
+ "nonce",
+ "number",
+ "parent",
+ "receipts",
+ "root",
+ "tx",
+ "uncles",
+ }
+}
+
+// ResolveLink is a helper function that allows easier traversal of links through blocks
+func (b *EthHeader) ResolveLink(p []string) (*node.Link, []string, error) {
+ obj, rest, err := b.Resolve(p)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ if lnk, ok := obj.(*node.Link); ok {
+ return lnk, rest, nil
+ }
+
+ return nil, nil, fmt.Errorf("resolved item was not a link")
+}
+
+// Copy will go away. It is here to comply with the Node interface.
+func (b *EthHeader) Copy() node.Node {
+ panic("implement me")
+}
+
+// Links is a helper function that returns all links within this object
+// HINT: Use `ipfs refs `
+func (b *EthHeader) Links() []*node.Link {
+ return []*node.Link{
+ {Cid: commonHashToCid(MEthHeader, b.ParentHash)},
+ {Cid: commonHashToCid(MEthTxReceiptTrie, b.ReceiptHash)},
+ {Cid: commonHashToCid(MEthStateTrie, b.Root)},
+ {Cid: commonHashToCid(MEthTxTrie, b.TxHash)},
+ {Cid: commonHashToCid(MEthHeaderList, b.UncleHash)},
+ }
+}
+
+// Stat will go away. It is here to comply with the Node interface.
+func (b *EthHeader) Stat() (*node.NodeStat, error) {
+ return &node.NodeStat{}, nil
+}
+
+// Size will go away. It is here to comply with the Node interface.
+func (b *EthHeader) Size() (uint64, error) {
+ return 0, nil
+}
+
+/*
+ EthHeader functions
+*/
+
+// MarshalJSON processes the block header into readable JSON format,
+// converting the right links into their cids, and keeping the original
+// hex hash, allowing the user to simplify external queries.
+func (b *EthHeader) MarshalJSON() ([]byte, error) {
+ out := map[string]interface{}{
+ "time": b.Time,
+ "bloom": b.Bloom,
+ "coinbase": b.Coinbase,
+ "difficulty": b.Difficulty,
+ "extra": fmt.Sprintf("0x%x", b.Extra),
+ "gaslimit": b.GasLimit,
+ "gasused": b.GasUsed,
+ "mixdigest": b.MixDigest,
+ "nonce": b.Nonce,
+ "number": b.Number,
+ "parent": commonHashToCid(MEthHeader, b.ParentHash),
+ "receipts": commonHashToCid(MEthTxReceiptTrie, b.ReceiptHash),
+ "root": commonHashToCid(MEthStateTrie, b.Root),
+ "tx": commonHashToCid(MEthTxTrie, b.TxHash),
+ "uncles": commonHashToCid(MEthHeaderList, b.UncleHash),
+ }
+ return json.Marshal(out)
+}
diff --git a/statediff/indexer/ipfs/ipld/eth_parser.go b/statediff/indexer/ipfs/ipld/eth_parser.go
new file mode 100644
index 000000000..f02d7d401
--- /dev/null
+++ b/statediff/indexer/ipfs/ipld/eth_parser.go
@@ -0,0 +1,97 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package ipld
+
+import (
+ "bytes"
+ "fmt"
+
+ "github.com/ethereum/go-ethereum/core/types"
+)
+
+// FromBlockAndReceipts takes a block and processes it
+// to return it a set of IPLD nodes for further processing.
+func FromBlockAndReceipts(block *types.Block, receipts []*types.Receipt) (*EthHeader, []*EthHeader, []*EthTx, []*EthTxTrie, []*EthReceipt, []*EthRctTrie, error) {
+ // Process the header
+ headerNode, err := NewEthHeader(block.Header())
+ if err != nil {
+ return nil, nil, nil, nil, nil, nil, err
+ }
+ // Process the uncles
+ uncleNodes := make([]*EthHeader, len(block.Uncles()))
+ for i, uncle := range block.Uncles() {
+ uncleNode, err := NewEthHeader(uncle)
+ if err != nil {
+ return nil, nil, nil, nil, nil, nil, err
+ }
+ uncleNodes[i] = uncleNode
+ }
+ // Process the txs
+ ethTxNodes, ethTxTrieNodes, err := processTransactions(block.Transactions(),
+ block.Header().TxHash[:])
+ if err != nil {
+ return nil, nil, nil, nil, nil, nil, err
+ }
+ // Process the receipts
+ ethRctNodes, ethRctTrieNodes, err := processReceipts(receipts,
+ block.Header().ReceiptHash[:])
+ return headerNode, uncleNodes, ethTxNodes, ethTxTrieNodes, ethRctNodes, ethRctTrieNodes, err
+}
+
+// processTransactions will take the found transactions in a parsed block body
+// to return IPLD node slices for eth-tx and eth-tx-trie
+func processTransactions(txs []*types.Transaction, expectedTxRoot []byte) ([]*EthTx, []*EthTxTrie, error) {
+ var ethTxNodes []*EthTx
+ transactionTrie := newTxTrie()
+
+ for idx, tx := range txs {
+ ethTx, err := NewEthTx(tx)
+ if err != nil {
+ return nil, nil, err
+ }
+ ethTxNodes = append(ethTxNodes, ethTx)
+ transactionTrie.add(idx, ethTx.RawData())
+ }
+
+ if !bytes.Equal(transactionTrie.rootHash(), expectedTxRoot) {
+ return nil, nil, fmt.Errorf("wrong transaction hash computed")
+ }
+
+ return ethTxNodes, transactionTrie.getNodes(), nil
+}
+
+// processReceipts will take in receipts
+// to return IPLD node slices for eth-rct and eth-rct-trie
+func processReceipts(rcts []*types.Receipt, expectedRctRoot []byte) ([]*EthReceipt, []*EthRctTrie, error) {
+ var ethRctNodes []*EthReceipt
+ receiptTrie := newRctTrie()
+
+ for idx, rct := range rcts {
+ ethRct, err := NewReceipt(rct)
+ if err != nil {
+ return nil, nil, err
+ }
+ ethRctNodes = append(ethRctNodes, ethRct)
+ receiptTrie.add(idx, ethRct.RawData())
+ }
+
+ if !bytes.Equal(receiptTrie.rootHash(), expectedRctRoot) {
+ return nil, nil, fmt.Errorf("wrong receipt hash computed")
+ }
+
+ return ethRctNodes, receiptTrie.getNodes(), nil
+}
diff --git a/statediff/indexer/ipfs/ipld/eth_receipt.go b/statediff/indexer/ipfs/ipld/eth_receipt.go
new file mode 100644
index 000000000..cfa46b36e
--- /dev/null
+++ b/statediff/indexer/ipfs/ipld/eth_receipt.go
@@ -0,0 +1,199 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package ipld
+
+import (
+ "encoding/json"
+ "fmt"
+ "strconv"
+
+ "github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/rlp"
+ "github.com/ipfs/go-cid"
+ node "github.com/ipfs/go-ipld-format"
+ mh "github.com/multiformats/go-multihash"
+)
+
+type EthReceipt struct {
+ *types.Receipt
+
+ rawdata []byte
+ cid cid.Cid
+}
+
+// Static (compile time) check that EthReceipt satisfies the node.Node interface.
+var _ node.Node = (*EthReceipt)(nil)
+
+/*
+ INPUT
+*/
+
+// NewReceipt converts a types.ReceiptForStorage to an EthReceipt IPLD node
+func NewReceipt(receipt *types.Receipt) (*EthReceipt, error) {
+ receiptRLP, err := rlp.EncodeToBytes(receipt)
+ if err != nil {
+ return nil, err
+ }
+ c, err := RawdataToCid(MEthTxReceipt, receiptRLP, mh.KECCAK_256)
+ if err != nil {
+ return nil, err
+ }
+ return &EthReceipt{
+ Receipt: receipt,
+ cid: c,
+ rawdata: receiptRLP,
+ }, nil
+}
+
+/*
+ OUTPUT
+*/
+
+// DecodeEthReceipt takes a cid and its raw binary data
+// from IPFS and returns an EthTx object for further processing.
+func DecodeEthReceipt(c cid.Cid, b []byte) (*EthReceipt, error) {
+ var r *types.Receipt
+ if err := rlp.DecodeBytes(b, r); err != nil {
+ return nil, err
+ }
+ return &EthReceipt{
+ Receipt: r,
+ cid: c,
+ rawdata: b,
+ }, nil
+}
+
+/*
+ Block INTERFACE
+*/
+
+func (node *EthReceipt) RawData() []byte {
+ return node.rawdata
+}
+
+func (node *EthReceipt) Cid() cid.Cid {
+ return node.cid
+}
+
+// String is a helper for output
+func (r *EthReceipt) String() string {
+ return fmt.Sprintf("", r.cid)
+}
+
+// Loggable returns in a map the type of IPLD Link.
+func (r *EthReceipt) Loggable() map[string]interface{} {
+ return map[string]interface{}{
+ "type": "eth-receipt",
+ }
+}
+
+// Resolve resolves a path through this node, stopping at any link boundary
+// and returning the object found as well as the remaining path to traverse
+func (r *EthReceipt) Resolve(p []string) (interface{}, []string, error) {
+ if len(p) == 0 {
+ return r, nil, nil
+ }
+
+ if len(p) > 1 {
+ return nil, nil, fmt.Errorf("unexpected path elements past %s", p[0])
+ }
+
+ switch p[0] {
+
+ case "root":
+ return r.PostState, nil, nil
+ case "status":
+ return r.Status, nil, nil
+ case "cumulativeGasUsed":
+ return r.CumulativeGasUsed, nil, nil
+ case "logsBloom":
+ return r.Bloom, nil, nil
+ case "logs":
+ return r.Logs, nil, nil
+ case "transactionHash":
+ return r.TxHash, nil, nil
+ case "contractAddress":
+ return r.ContractAddress, nil, nil
+ case "gasUsed":
+ return r.GasUsed, nil, nil
+ default:
+ return nil, nil, fmt.Errorf("no such link")
+ }
+}
+
+// Tree lists all paths within the object under 'path', and up to the given depth.
+// To list the entire object (similar to `find .`) pass "" and -1
+func (r *EthReceipt) Tree(p string, depth int) []string {
+ if p != "" || depth == 0 {
+ return nil
+ }
+ return []string{"root", "status", "cumulativeGasUsed", "logsBloom", "logs", "transactionHash", "contractAddress", "gasUsed"}
+}
+
+// ResolveLink is a helper function that calls resolve and asserts the
+// output is a link
+func (r *EthReceipt) ResolveLink(p []string) (*node.Link, []string, error) {
+ obj, rest, err := r.Resolve(p)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ if lnk, ok := obj.(*node.Link); ok {
+ return lnk, rest, nil
+ }
+
+ return nil, nil, fmt.Errorf("resolved item was not a link")
+}
+
+// Copy will go away. It is here to comply with the Node interface.
+func (*EthReceipt) Copy() node.Node {
+ panic("implement me")
+}
+
+// Links is a helper function that returns all links within this object
+func (*EthReceipt) Links() []*node.Link {
+ return nil
+}
+
+// Stat will go away. It is here to comply with the interface.
+func (r *EthReceipt) Stat() (*node.NodeStat, error) {
+ return &node.NodeStat{}, nil
+}
+
+// Size will go away. It is here to comply with the interface.
+func (r *EthReceipt) Size() (uint64, error) {
+ return strconv.ParseUint(r.Receipt.Size().String(), 10, 64)
+}
+
+/*
+ EthReceipt functions
+*/
+
+// MarshalJSON processes the receipt into readable JSON format.
+func (r *EthReceipt) MarshalJSON() ([]byte, error) {
+ out := map[string]interface{}{
+ "root": r.PostState,
+ "status": r.Status,
+ "cumulativeGasUsed": r.CumulativeGasUsed,
+ "logsBloom": r.Bloom,
+ "logs": r.Logs,
+ "transactionHash": r.TxHash,
+ "contractAddress": r.ContractAddress,
+ "gasUsed": r.GasUsed,
+ }
+ return json.Marshal(out)
+}
diff --git a/statediff/indexer/ipfs/ipld/eth_receipt_trie.go b/statediff/indexer/ipfs/ipld/eth_receipt_trie.go
new file mode 100644
index 000000000..6a1b7e406
--- /dev/null
+++ b/statediff/indexer/ipfs/ipld/eth_receipt_trie.go
@@ -0,0 +1,152 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package ipld
+
+import (
+ "fmt"
+
+ "github.com/ipfs/go-cid"
+ node "github.com/ipfs/go-ipld-format"
+ "github.com/multiformats/go-multihash"
+
+ "github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/rlp"
+)
+
+// EthRctTrie (eth-tx-trie codec 0x92) represents
+// a node from the transaction trie in ethereum.
+type EthRctTrie struct {
+ *TrieNode
+}
+
+// Static (compile time) check that EthRctTrie satisfies the node.Node interface.
+var _ node.Node = (*EthRctTrie)(nil)
+
+/*
+ INPUT
+*/
+
+// To create a proper trie of the eth-tx-trie objects, it is required
+// to input all transactions belonging to a forest in a single step.
+// We are adding the transactions, and creating its trie on
+// block body parsing time.
+
+/*
+ OUTPUT
+*/
+
+// DecodeEthRctTrie returns an EthRctTrie object from its cid and rawdata.
+func DecodeEthRctTrie(c cid.Cid, b []byte) (*EthRctTrie, error) {
+ tn, err := decodeTrieNode(c, b, decodeEthRctTrieLeaf)
+ if err != nil {
+ return nil, err
+ }
+ return &EthRctTrie{TrieNode: tn}, nil
+}
+
+// decodeEthRctTrieLeaf parses a eth-rct-trie leaf
+//from decoded RLP elements
+func decodeEthRctTrieLeaf(i []interface{}) ([]interface{}, error) {
+ var r types.Receipt
+ err := rlp.DecodeBytes(i[1].([]byte), &r)
+ if err != nil {
+ return nil, err
+ }
+ c, err := RawdataToCid(MEthTxReceipt, i[1].([]byte), multihash.KECCAK_256)
+ if err != nil {
+ return nil, err
+ }
+ return []interface{}{
+ i[0].([]byte),
+ &EthReceipt{
+ Receipt: &r,
+ cid: c,
+ rawdata: i[1].([]byte),
+ },
+ }, nil
+}
+
+/*
+ Block INTERFACE
+*/
+
+// RawData returns the binary of the RLP encode of the transaction.
+func (t *EthRctTrie) RawData() []byte {
+ return t.rawdata
+}
+
+// Cid returns the cid of the transaction.
+func (t *EthRctTrie) Cid() cid.Cid {
+ return t.cid
+}
+
+// String is a helper for output
+func (t *EthRctTrie) String() string {
+ return fmt.Sprintf("", t.cid)
+}
+
+// Loggable returns in a map the type of IPLD Link.
+func (t *EthRctTrie) Loggable() map[string]interface{} {
+ return map[string]interface{}{
+ "type": "eth-rct-trie",
+ }
+}
+
+/*
+ EthRctTrie functions
+*/
+
+// rctTrie wraps a localTrie for use on the receipt trie.
+type rctTrie struct {
+ *localTrie
+}
+
+// newRctTrie initializes and returns a rctTrie.
+func newRctTrie() *rctTrie {
+ return &rctTrie{
+ localTrie: newLocalTrie(),
+ }
+}
+
+// getNodes invokes the localTrie, which computes the root hash of the
+// transaction trie and returns its database keys, to return a slice
+// of EthRctTrie nodes.
+func (rt *rctTrie) getNodes() []*EthRctTrie {
+ keys := rt.getKeys()
+ var out []*EthRctTrie
+ it := rt.trie.NodeIterator([]byte{})
+ for it.Next(true) {
+
+ }
+ for _, k := range keys {
+ rawdata, err := rt.db.Get(k)
+ if err != nil {
+ panic(err)
+ }
+ c, err := RawdataToCid(MEthTxReceiptTrie, rawdata, multihash.KECCAK_256)
+ if err != nil {
+ return nil
+ }
+ tn := &TrieNode{
+ cid: c,
+ rawdata: rawdata,
+ }
+ out = append(out, &EthRctTrie{TrieNode: tn})
+ }
+
+ return out
+}
diff --git a/statediff/indexer/ipfs/ipld/eth_state.go b/statediff/indexer/ipfs/ipld/eth_state.go
new file mode 100644
index 000000000..a127f956e
--- /dev/null
+++ b/statediff/indexer/ipfs/ipld/eth_state.go
@@ -0,0 +1,114 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package ipld
+
+import (
+ "fmt"
+
+ "github.com/ipfs/go-cid"
+ node "github.com/ipfs/go-ipld-format"
+ "github.com/multiformats/go-multihash"
+
+ "github.com/ethereum/go-ethereum/rlp"
+)
+
+// EthStateTrie (eth-state-trie, codec 0x96), represents
+// a node from the satte trie in ethereum.
+type EthStateTrie struct {
+ *TrieNode
+}
+
+// Static (compile time) check that EthStateTrie satisfies the node.Node interface.
+var _ node.Node = (*EthStateTrie)(nil)
+
+/*
+ INPUT
+*/
+
+// FromStateTrieRLP takes the RLP representation of an ethereum
+// state trie node to return it as an IPLD node for further processing.
+func FromStateTrieRLP(raw []byte) (*EthStateTrie, error) {
+ c, err := RawdataToCid(MEthStateTrie, raw, multihash.KECCAK_256)
+ if err != nil {
+ return nil, err
+ }
+ // Let's run the whole mile and process the nodeKind and
+ // its elements, in case somebody would need this function
+ // to parse an RLP element from the filesystem
+ return DecodeEthStateTrie(c, raw)
+}
+
+/*
+ OUTPUT
+*/
+
+// DecodeEthStateTrie returns an EthStateTrie object from its cid and rawdata.
+func DecodeEthStateTrie(c cid.Cid, b []byte) (*EthStateTrie, error) {
+ tn, err := decodeTrieNode(c, b, decodeEthStateTrieLeaf)
+ if err != nil {
+ return nil, err
+ }
+ return &EthStateTrie{TrieNode: tn}, nil
+}
+
+// decodeEthStateTrieLeaf parses a eth-tx-trie leaf
+// from decoded RLP elements
+func decodeEthStateTrieLeaf(i []interface{}) ([]interface{}, error) {
+ var account EthAccount
+ err := rlp.DecodeBytes(i[1].([]byte), &account)
+ if err != nil {
+ return nil, err
+ }
+ c, err := RawdataToCid(MEthAccountSnapshot, i[1].([]byte), multihash.KECCAK_256)
+ if err != nil {
+ return nil, err
+ }
+ return []interface{}{
+ i[0].([]byte),
+ &EthAccountSnapshot{
+ EthAccount: &account,
+ cid: c,
+ rawdata: i[1].([]byte),
+ },
+ }, nil
+}
+
+/*
+ Block INTERFACE
+*/
+
+// RawData returns the binary of the RLP encode of the state trie node.
+func (st *EthStateTrie) RawData() []byte {
+ return st.rawdata
+}
+
+// Cid returns the cid of the state trie node.
+func (st *EthStateTrie) Cid() cid.Cid {
+ return st.cid
+}
+
+// String is a helper for output
+func (st *EthStateTrie) String() string {
+ return fmt.Sprintf("", st.cid)
+}
+
+// Loggable returns in a map the type of IPLD Link.
+func (st *EthStateTrie) Loggable() map[string]interface{} {
+ return map[string]interface{}{
+ "type": "eth-state-trie",
+ }
+}
diff --git a/statediff/indexer/ipfs/ipld/eth_storage.go b/statediff/indexer/ipfs/ipld/eth_storage.go
new file mode 100644
index 000000000..779cad4d9
--- /dev/null
+++ b/statediff/indexer/ipfs/ipld/eth_storage.go
@@ -0,0 +1,100 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package ipld
+
+import (
+ "fmt"
+
+ "github.com/ipfs/go-cid"
+ node "github.com/ipfs/go-ipld-format"
+ "github.com/multiformats/go-multihash"
+)
+
+// EthStorageTrie (eth-storage-trie, codec 0x98), represents
+// a node from the storage trie in ethereum.
+type EthStorageTrie struct {
+ *TrieNode
+}
+
+// Static (compile time) check that EthStorageTrie satisfies the node.Node interface.
+var _ node.Node = (*EthStorageTrie)(nil)
+
+/*
+ INPUT
+*/
+
+// FromStorageTrieRLP takes the RLP representation of an ethereum
+// storage trie node to return it as an IPLD node for further processing.
+func FromStorageTrieRLP(raw []byte) (*EthStorageTrie, error) {
+ c, err := RawdataToCid(MEthStorageTrie, raw, multihash.KECCAK_256)
+ if err != nil {
+ return nil, err
+ }
+
+ // Let's run the whole mile and process the nodeKind and
+ // its elements, in case somebody would need this function
+ // to parse an RLP element from the filesystem
+ return DecodeEthStorageTrie(c, raw)
+}
+
+/*
+ OUTPUT
+*/
+
+// DecodeEthStorageTrie returns an EthStorageTrie object from its cid and rawdata.
+func DecodeEthStorageTrie(c cid.Cid, b []byte) (*EthStorageTrie, error) {
+ tn, err := decodeTrieNode(c, b, decodeEthStorageTrieLeaf)
+ if err != nil {
+ return nil, err
+ }
+ return &EthStorageTrie{TrieNode: tn}, nil
+}
+
+// decodeEthStorageTrieLeaf parses a eth-tx-trie leaf
+// from decoded RLP elements
+func decodeEthStorageTrieLeaf(i []interface{}) ([]interface{}, error) {
+ return []interface{}{
+ i[0].([]byte),
+ i[1].([]byte),
+ }, nil
+}
+
+/*
+ Block INTERFACE
+*/
+
+// RawData returns the binary of the RLP encode of the storage trie node.
+func (st *EthStorageTrie) RawData() []byte {
+ return st.rawdata
+}
+
+// Cid returns the cid of the storage trie node.
+func (st *EthStorageTrie) Cid() cid.Cid {
+ return st.cid
+}
+
+// String is a helper for output
+func (st *EthStorageTrie) String() string {
+ return fmt.Sprintf("", st.cid)
+}
+
+// Loggable returns in a map the type of IPLD Link.
+func (st *EthStorageTrie) Loggable() map[string]interface{} {
+ return map[string]interface{}{
+ "type": "eth-storage-trie",
+ }
+}
diff --git a/statediff/indexer/ipfs/ipld/eth_tx.go b/statediff/indexer/ipfs/ipld/eth_tx.go
new file mode 100644
index 000000000..4fc4d20a6
--- /dev/null
+++ b/statediff/indexer/ipfs/ipld/eth_tx.go
@@ -0,0 +1,215 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package ipld
+
+import (
+ "encoding/json"
+ "fmt"
+ "strconv"
+
+ "github.com/ethereum/go-ethereum/common/hexutil"
+ "github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/rlp"
+ "github.com/ipfs/go-cid"
+ node "github.com/ipfs/go-ipld-format"
+ mh "github.com/multiformats/go-multihash"
+)
+
+// EthTx (eth-tx codec 0x93) represents an ethereum transaction
+type EthTx struct {
+ *types.Transaction
+
+ cid cid.Cid
+ rawdata []byte
+}
+
+// Static (compile time) check that EthTx satisfies the node.Node interface.
+var _ node.Node = (*EthTx)(nil)
+
+/*
+ INPUT
+*/
+
+// NewEthTx converts a *types.Transaction to an EthTx IPLD node
+func NewEthTx(tx *types.Transaction) (*EthTx, error) {
+ txRLP, err := rlp.EncodeToBytes(tx)
+ if err != nil {
+ return nil, err
+ }
+ c, err := RawdataToCid(MEthTx, txRLP, mh.KECCAK_256)
+ if err != nil {
+ return nil, err
+ }
+ return &EthTx{
+ Transaction: tx,
+ cid: c,
+ rawdata: txRLP,
+ }, nil
+}
+
+/*
+ OUTPUT
+*/
+
+// DecodeEthTx takes a cid and its raw binary data
+// from IPFS and returns an EthTx object for further processing.
+func DecodeEthTx(c cid.Cid, b []byte) (*EthTx, error) {
+ var t *types.Transaction
+ if err := rlp.DecodeBytes(b, t); err != nil {
+ return nil, err
+ }
+ return &EthTx{
+ Transaction: t,
+ cid: c,
+ rawdata: b,
+ }, nil
+}
+
+/*
+ Block INTERFACE
+*/
+
+// RawData returns the binary of the RLP encode of the transaction.
+func (t *EthTx) RawData() []byte {
+ return t.rawdata
+}
+
+// Cid returns the cid of the transaction.
+func (t *EthTx) Cid() cid.Cid {
+ return t.cid
+}
+
+// String is a helper for output
+func (t *EthTx) String() string {
+ return fmt.Sprintf("", t.cid)
+}
+
+// Loggable returns in a map the type of IPLD Link.
+func (t *EthTx) Loggable() map[string]interface{} {
+ return map[string]interface{}{
+ "type": "eth-tx",
+ }
+}
+
+/*
+ Node INTERFACE
+*/
+
+// Resolve resolves a path through this node, stopping at any link boundary
+// and returning the object found as well as the remaining path to traverse
+func (t *EthTx) Resolve(p []string) (interface{}, []string, error) {
+ if len(p) == 0 {
+ return t, nil, nil
+ }
+
+ if len(p) > 1 {
+ return nil, nil, fmt.Errorf("unexpected path elements past %s", p[0])
+ }
+
+ switch p[0] {
+
+ case "gas":
+ return t.Gas(), nil, nil
+ case "gasPrice":
+ return t.GasPrice(), nil, nil
+ case "input":
+ return fmt.Sprintf("%x", t.Data()), nil, nil
+ case "nonce":
+ return t.Nonce(), nil, nil
+ case "r":
+ _, r, _ := t.RawSignatureValues()
+ return hexutil.EncodeBig(r), nil, nil
+ case "s":
+ _, _, s := t.RawSignatureValues()
+ return hexutil.EncodeBig(s), nil, nil
+ case "toAddress":
+ return t.To(), nil, nil
+ case "v":
+ v, _, _ := t.RawSignatureValues()
+ return hexutil.EncodeBig(v), nil, nil
+ case "value":
+ return hexutil.EncodeBig(t.Value()), nil, nil
+ default:
+ return nil, nil, fmt.Errorf("no such link")
+ }
+}
+
+// Tree lists all paths within the object under 'path', and up to the given depth.
+// To list the entire object (similar to `find .`) pass "" and -1
+func (t *EthTx) Tree(p string, depth int) []string {
+ if p != "" || depth == 0 {
+ return nil
+ }
+ return []string{"gas", "gasPrice", "input", "nonce", "r", "s", "toAddress", "v", "value"}
+}
+
+// ResolveLink is a helper function that calls resolve and asserts the
+// output is a link
+func (t *EthTx) ResolveLink(p []string) (*node.Link, []string, error) {
+ obj, rest, err := t.Resolve(p)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ if lnk, ok := obj.(*node.Link); ok {
+ return lnk, rest, nil
+ }
+
+ return nil, nil, fmt.Errorf("resolved item was not a link")
+}
+
+// Copy will go away. It is here to comply with the interface.
+func (t *EthTx) Copy() node.Node {
+ panic("implement me")
+}
+
+// Links is a helper function that returns all links within this object
+func (t *EthTx) Links() []*node.Link {
+ return nil
+}
+
+// Stat will go away. It is here to comply with the interface.
+func (t *EthTx) Stat() (*node.NodeStat, error) {
+ return &node.NodeStat{}, nil
+}
+
+// Size will go away. It is here to comply with the interface.
+func (t *EthTx) Size() (uint64, error) {
+ return strconv.ParseUint(t.Transaction.Size().String(), 10, 64)
+}
+
+/*
+ EthTx functions
+*/
+
+// MarshalJSON processes the transaction into readable JSON format.
+func (t *EthTx) MarshalJSON() ([]byte, error) {
+ v, r, s := t.RawSignatureValues()
+
+ out := map[string]interface{}{
+ "gas": t.Gas(),
+ "gasPrice": hexutil.EncodeBig(t.GasPrice()),
+ "input": fmt.Sprintf("%x", t.Data()),
+ "nonce": t.Nonce(),
+ "r": hexutil.EncodeBig(r),
+ "s": hexutil.EncodeBig(s),
+ "toAddress": t.To(),
+ "v": hexutil.EncodeBig(v),
+ "value": hexutil.EncodeBig(t.Value()),
+ }
+ return json.Marshal(out)
+}
diff --git a/statediff/indexer/ipfs/ipld/eth_tx_trie.go b/statediff/indexer/ipfs/ipld/eth_tx_trie.go
new file mode 100644
index 000000000..6f106f6d8
--- /dev/null
+++ b/statediff/indexer/ipfs/ipld/eth_tx_trie.go
@@ -0,0 +1,152 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package ipld
+
+import (
+ "fmt"
+
+ "github.com/ipfs/go-cid"
+ node "github.com/ipfs/go-ipld-format"
+ "github.com/multiformats/go-multihash"
+
+ "github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/rlp"
+)
+
+// EthTxTrie (eth-tx-trie codec 0x92) represents
+// a node from the transaction trie in ethereum.
+type EthTxTrie struct {
+ *TrieNode
+}
+
+// Static (compile time) check that EthTxTrie satisfies the node.Node interface.
+var _ node.Node = (*EthTxTrie)(nil)
+
+/*
+ INPUT
+*/
+
+// To create a proper trie of the eth-tx-trie objects, it is required
+// to input all transactions belonging to a forest in a single step.
+// We are adding the transactions, and creating its trie on
+// block body parsing time.
+
+/*
+ OUTPUT
+*/
+
+// DecodeEthTxTrie returns an EthTxTrie object from its cid and rawdata.
+func DecodeEthTxTrie(c cid.Cid, b []byte) (*EthTxTrie, error) {
+ tn, err := decodeTrieNode(c, b, decodeEthTxTrieLeaf)
+ if err != nil {
+ return nil, err
+ }
+ return &EthTxTrie{TrieNode: tn}, nil
+}
+
+// decodeEthTxTrieLeaf parses a eth-tx-trie leaf
+//from decoded RLP elements
+func decodeEthTxTrieLeaf(i []interface{}) ([]interface{}, error) {
+ var t types.Transaction
+ err := rlp.DecodeBytes(i[1].([]byte), &t)
+ if err != nil {
+ return nil, err
+ }
+ c, err := RawdataToCid(MEthTx, i[1].([]byte), multihash.KECCAK_256)
+ if err != nil {
+ return nil, err
+ }
+ return []interface{}{
+ i[0].([]byte),
+ &EthTx{
+ Transaction: &t,
+ cid: c,
+ rawdata: i[1].([]byte),
+ },
+ }, nil
+}
+
+/*
+ Block INTERFACE
+*/
+
+// RawData returns the binary of the RLP encode of the transaction.
+func (t *EthTxTrie) RawData() []byte {
+ return t.rawdata
+}
+
+// Cid returns the cid of the transaction.
+func (t *EthTxTrie) Cid() cid.Cid {
+ return t.cid
+}
+
+// String is a helper for output
+func (t *EthTxTrie) String() string {
+ return fmt.Sprintf("", t.cid)
+}
+
+// Loggable returns in a map the type of IPLD Link.
+func (t *EthTxTrie) Loggable() map[string]interface{} {
+ return map[string]interface{}{
+ "type": "eth-tx-trie",
+ }
+}
+
+/*
+ EthTxTrie functions
+*/
+
+// txTrie wraps a localTrie for use on the transaction trie.
+type txTrie struct {
+ *localTrie
+}
+
+// newTxTrie initializes and returns a txTrie.
+func newTxTrie() *txTrie {
+ return &txTrie{
+ localTrie: newLocalTrie(),
+ }
+}
+
+// getNodes invokes the localTrie, which computes the root hash of the
+// transaction trie and returns its database keys, to return a slice
+// of EthTxTrie nodes.
+func (tt *txTrie) getNodes() []*EthTxTrie {
+ keys := tt.getKeys()
+ var out []*EthTxTrie
+ it := tt.trie.NodeIterator([]byte{})
+ for it.Next(true) {
+
+ }
+ for _, k := range keys {
+ rawdata, err := tt.db.Get(k)
+ if err != nil {
+ panic(err)
+ }
+ c, err := RawdataToCid(MEthTxTrie, rawdata, multihash.KECCAK_256)
+ if err != nil {
+ return nil
+ }
+ tn := &TrieNode{
+ cid: c,
+ rawdata: rawdata,
+ }
+ out = append(out, &EthTxTrie{TrieNode: tn})
+ }
+
+ return out
+}
diff --git a/statediff/indexer/ipfs/ipld/shared.go b/statediff/indexer/ipfs/ipld/shared.go
new file mode 100644
index 000000000..cfdd7446a
--- /dev/null
+++ b/statediff/indexer/ipfs/ipld/shared.go
@@ -0,0 +1,126 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package ipld
+
+import (
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/core/rawdb"
+ "github.com/ethereum/go-ethereum/ethdb"
+ "github.com/ethereum/go-ethereum/rlp"
+ "github.com/ethereum/go-ethereum/trie"
+ "github.com/ipfs/go-cid"
+ mh "github.com/multiformats/go-multihash"
+)
+
+// IPLD Codecs for Ethereum
+// See the authoritative document:
+// https://github.com/multiformats/multicodec/blob/master/table.csv
+const (
+ RawBinary = 0x55
+ MEthHeader = 0x90
+ MEthHeaderList = 0x91
+ MEthTxTrie = 0x92
+ MEthTx = 0x93
+ MEthTxReceiptTrie = 0x94
+ MEthTxReceipt = 0x95
+ MEthStateTrie = 0x96
+ MEthAccountSnapshot = 0x97
+ MEthStorageTrie = 0x98
+)
+
+// RawdataToCid takes the desired codec and a slice of bytes
+// and returns the proper cid of the object.
+func RawdataToCid(codec uint64, rawdata []byte, multiHash uint64) (cid.Cid, error) {
+ c, err := cid.Prefix{
+ Codec: codec,
+ Version: 1,
+ MhType: multiHash,
+ MhLength: -1,
+ }.Sum(rawdata)
+ if err != nil {
+ return cid.Cid{}, err
+ }
+ return c, nil
+}
+
+// keccak256ToCid takes a keccak256 hash and returns its cid based on
+// the codec given.
+func keccak256ToCid(codec uint64, h []byte) cid.Cid {
+ buf, err := mh.Encode(h, mh.KECCAK_256)
+ if err != nil {
+ panic(err)
+ }
+
+ return cid.NewCidV1(codec, mh.Multihash(buf))
+}
+
+// commonHashToCid takes a go-ethereum common.Hash and returns its
+// cid based on the codec given,
+func commonHashToCid(codec uint64, h common.Hash) cid.Cid {
+ mhash, err := mh.Encode(h[:], mh.KECCAK_256)
+ if err != nil {
+ panic(err)
+ }
+
+ return cid.NewCidV1(codec, mhash)
+}
+
+// localTrie wraps a go-ethereum trie and its underlying memory db.
+// It contributes to the creation of the trie node objects.
+type localTrie struct {
+ keys [][]byte
+ db ethdb.Database
+ trie *trie.Trie
+}
+
+// newLocalTrie initializes and returns a localTrie object
+func newLocalTrie() *localTrie {
+ var err error
+ lt := &localTrie{}
+ lt.db = rawdb.NewMemoryDatabase()
+ lt.trie, err = trie.New(common.Hash{}, trie.NewDatabase(lt.db))
+ if err != nil {
+ panic(err)
+ }
+ return lt
+}
+
+// add receives the index of an object and its rawdata value
+// and includes it into the localTrie
+func (lt *localTrie) add(idx int, rawdata []byte) {
+ key, err := rlp.EncodeToBytes(uint(idx))
+ if err != nil {
+ panic(err)
+ }
+ lt.keys = append(lt.keys, key)
+ if err := lt.db.Put(key, rawdata); err != nil {
+ panic(err)
+ }
+ lt.trie.Update(key, rawdata)
+}
+
+// rootHash returns the computed trie root.
+// Useful for sanity checks on parsed data.
+func (lt *localTrie) rootHash() []byte {
+ return lt.trie.Hash().Bytes()
+}
+
+// getKeys returns the stored keys of the memory database
+// of the localTrie for further processing.
+func (lt *localTrie) getKeys() [][]byte {
+ return lt.keys
+}
diff --git a/statediff/indexer/ipfs/ipld/trie_node.go b/statediff/indexer/ipfs/ipld/trie_node.go
new file mode 100644
index 000000000..788f76db8
--- /dev/null
+++ b/statediff/indexer/ipfs/ipld/trie_node.go
@@ -0,0 +1,444 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package ipld
+
+import (
+ "encoding/json"
+ "fmt"
+
+ "github.com/ethereum/go-ethereum/rlp"
+ "github.com/ipfs/go-cid"
+ node "github.com/ipfs/go-ipld-format"
+)
+
+// TrieNode is the general abstraction for
+//ethereum IPLD trie nodes.
+type TrieNode struct {
+ // leaf, extension or branch
+ nodeKind string
+
+ // If leaf or extension: [0] is key, [1] is val.
+ // If branch: [0] - [16] are children.
+ elements []interface{}
+
+ // IPLD block information
+ cid cid.Cid
+ rawdata []byte
+}
+
+/*
+ OUTPUT
+*/
+
+type trieNodeLeafDecoder func([]interface{}) ([]interface{}, error)
+
+// decodeTrieNode returns a TrieNode object from an IPLD block's
+// cid and rawdata.
+func decodeTrieNode(c cid.Cid, b []byte,
+ leafDecoder trieNodeLeafDecoder) (*TrieNode, error) {
+ var (
+ i, decoded, elements []interface{}
+ nodeKind string
+ err error
+ )
+
+ if err = rlp.DecodeBytes(b, &i); err != nil {
+ return nil, err
+ }
+
+ codec := c.Type()
+ switch len(i) {
+ case 2:
+ nodeKind, decoded, err = decodeCompactKey(i)
+ if err != nil {
+ return nil, err
+ }
+
+ if nodeKind == "extension" {
+ elements, err = parseTrieNodeExtension(decoded, codec)
+ }
+ if nodeKind == "leaf" {
+ elements, err = leafDecoder(decoded)
+ }
+ if nodeKind != "extension" && nodeKind != "leaf" {
+ return nil, fmt.Errorf("unexpected nodeKind returned from decoder")
+ }
+ case 17:
+ nodeKind = "branch"
+ elements, err = parseTrieNodeBranch(i, codec)
+ if err != nil {
+ return nil, err
+ }
+ default:
+ return nil, fmt.Errorf("unknown trie node type")
+ }
+
+ return &TrieNode{
+ nodeKind: nodeKind,
+ elements: elements,
+ rawdata: b,
+ cid: c,
+ }, nil
+}
+
+// decodeCompactKey takes a compact key, and returns its nodeKind and value.
+func decodeCompactKey(i []interface{}) (string, []interface{}, error) {
+ first := i[0].([]byte)
+ last := i[1].([]byte)
+
+ switch first[0] / 16 {
+ case '\x00':
+ return "extension", []interface{}{
+ nibbleToByte(first)[2:],
+ last,
+ }, nil
+ case '\x01':
+ return "extension", []interface{}{
+ nibbleToByte(first)[1:],
+ last,
+ }, nil
+ case '\x02':
+ return "leaf", []interface{}{
+ nibbleToByte(first)[2:],
+ last,
+ }, nil
+ case '\x03':
+ return "leaf", []interface{}{
+ nibbleToByte(first)[1:],
+ last,
+ }, nil
+ default:
+ return "", nil, fmt.Errorf("unknown hex prefix")
+ }
+}
+
+// parseTrieNodeExtension helper improves readability
+func parseTrieNodeExtension(i []interface{}, codec uint64) ([]interface{}, error) {
+ return []interface{}{
+ i[0].([]byte),
+ keccak256ToCid(codec, i[1].([]byte)),
+ }, nil
+}
+
+// parseTrieNodeBranch helper improves readability
+func parseTrieNodeBranch(i []interface{}, codec uint64) ([]interface{}, error) {
+ var out []interface{}
+
+ for i, vi := range i {
+ v, ok := vi.([]byte)
+ // Sometimes this throws "panic: interface conversion: interface {} is []interface {}, not []uint8"
+ // Figure out why, and if it is okay to continue
+ if !ok {
+ return nil, fmt.Errorf("unable to decode branch node entry into []byte at position: %d value: %+v", i, vi)
+ }
+
+ switch len(v) {
+ case 0:
+ out = append(out, nil)
+ case 32:
+ out = append(out, keccak256ToCid(codec, v))
+ default:
+ return nil, fmt.Errorf("unrecognized object: %v", v)
+ }
+ }
+
+ return out, nil
+}
+
+/*
+ Node INTERFACE
+*/
+
+// Resolve resolves a path through this node, stopping at any link boundary
+// and returning the object found as well as the remaining path to traverse
+func (t *TrieNode) Resolve(p []string) (interface{}, []string, error) {
+ switch t.nodeKind {
+ case "extension":
+ return t.resolveTrieNodeExtension(p)
+ case "leaf":
+ return t.resolveTrieNodeLeaf(p)
+ case "branch":
+ return t.resolveTrieNodeBranch(p)
+ default:
+ return nil, nil, fmt.Errorf("nodeKind case not implemented")
+ }
+}
+
+// Tree lists all paths within the object under 'path', and up to the given depth.
+// To list the entire object (similar to `find .`) pass "" and -1
+func (t *TrieNode) Tree(p string, depth int) []string {
+ if p != "" || depth == 0 {
+ return nil
+ }
+
+ var out []string
+
+ switch t.nodeKind {
+ case "extension":
+ var val string
+ for _, e := range t.elements[0].([]byte) {
+ val += fmt.Sprintf("%x", e)
+ }
+ return []string{val}
+ case "branch":
+ for i, elem := range t.elements {
+ if _, ok := elem.(*cid.Cid); ok {
+ out = append(out, fmt.Sprintf("%x", i))
+ }
+ }
+ return out
+
+ default:
+ return nil
+ }
+}
+
+// ResolveLink is a helper function that calls resolve and asserts the
+// output is a link
+func (t *TrieNode) ResolveLink(p []string) (*node.Link, []string, error) {
+ obj, rest, err := t.Resolve(p)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ lnk, ok := obj.(*node.Link)
+ if !ok {
+ return nil, nil, fmt.Errorf("was not a link")
+ }
+
+ return lnk, rest, nil
+}
+
+// Copy will go away. It is here to comply with the interface.
+func (t *TrieNode) Copy() node.Node {
+ panic("dont use this yet")
+}
+
+// Links is a helper function that returns all links within this object
+func (t *TrieNode) Links() []*node.Link {
+ var out []*node.Link
+
+ for _, i := range t.elements {
+ c, ok := i.(cid.Cid)
+ if ok {
+ out = append(out, &node.Link{Cid: c})
+ }
+ }
+
+ return out
+}
+
+// Stat will go away. It is here to comply with the interface.
+func (t *TrieNode) Stat() (*node.NodeStat, error) {
+ return &node.NodeStat{}, nil
+}
+
+// Size will go away. It is here to comply with the interface.
+func (t *TrieNode) Size() (uint64, error) {
+ return 0, nil
+}
+
+/*
+ TrieNode functions
+*/
+
+// MarshalJSON processes the transaction trie into readable JSON format.
+func (t *TrieNode) MarshalJSON() ([]byte, error) {
+ var out map[string]interface{}
+
+ switch t.nodeKind {
+ case "extension":
+ fallthrough
+ case "leaf":
+ var hexPrefix string
+ for _, e := range t.elements[0].([]byte) {
+ hexPrefix += fmt.Sprintf("%x", e)
+ }
+
+ // if we got a byte we need to do this casting otherwise
+ // it will be marshaled to a base64 encoded value
+ if _, ok := t.elements[1].([]byte); ok {
+ var hexVal string
+ for _, e := range t.elements[1].([]byte) {
+ hexVal += fmt.Sprintf("%x", e)
+ }
+
+ t.elements[1] = hexVal
+ }
+
+ out = map[string]interface{}{
+ "type": t.nodeKind,
+ hexPrefix: t.elements[1],
+ }
+
+ case "branch":
+ out = map[string]interface{}{
+ "type": "branch",
+ "0": t.elements[0],
+ "1": t.elements[1],
+ "2": t.elements[2],
+ "3": t.elements[3],
+ "4": t.elements[4],
+ "5": t.elements[5],
+ "6": t.elements[6],
+ "7": t.elements[7],
+ "8": t.elements[8],
+ "9": t.elements[9],
+ "a": t.elements[10],
+ "b": t.elements[11],
+ "c": t.elements[12],
+ "d": t.elements[13],
+ "e": t.elements[14],
+ "f": t.elements[15],
+ }
+ default:
+ return nil, fmt.Errorf("nodeKind %s not supported", t.nodeKind)
+ }
+
+ return json.Marshal(out)
+}
+
+// nibbleToByte expands the nibbles of a byte slice into their own bytes.
+func nibbleToByte(k []byte) []byte {
+ var out []byte
+
+ for _, b := range k {
+ out = append(out, b/16)
+ out = append(out, b%16)
+ }
+
+ return out
+}
+
+// Resolve reading conveniences
+func (t *TrieNode) resolveTrieNodeExtension(p []string) (interface{}, []string, error) {
+ nibbles := t.elements[0].([]byte)
+ idx, rest := shiftFromPath(p, len(nibbles))
+ if len(idx) < len(nibbles) {
+ return nil, nil, fmt.Errorf("not enough nibbles to traverse this extension")
+ }
+
+ for _, i := range idx {
+ if getHexIndex(string(i)) == -1 {
+ return nil, nil, fmt.Errorf("invalid path element")
+ }
+ }
+
+ for i, n := range nibbles {
+ if string(idx[i]) != fmt.Sprintf("%x", n) {
+ return nil, nil, fmt.Errorf("no such link in this extension")
+ }
+ }
+
+ return &node.Link{Cid: t.elements[1].(cid.Cid)}, rest, nil
+}
+
+func (t *TrieNode) resolveTrieNodeLeaf(p []string) (interface{}, []string, error) {
+ nibbles := t.elements[0].([]byte)
+
+ if len(nibbles) != 0 {
+ idx, rest := shiftFromPath(p, len(nibbles))
+ if len(idx) < len(nibbles) {
+ return nil, nil, fmt.Errorf("not enough nibbles to traverse this leaf")
+ }
+
+ for _, i := range idx {
+ if getHexIndex(string(i)) == -1 {
+ return nil, nil, fmt.Errorf("invalid path element")
+ }
+ }
+
+ for i, n := range nibbles {
+ if string(idx[i]) != fmt.Sprintf("%x", n) {
+ return nil, nil, fmt.Errorf("no such link in this extension")
+ }
+ }
+
+ p = rest
+ }
+
+ link, ok := t.elements[1].(node.Node)
+ if !ok {
+ return nil, nil, fmt.Errorf("leaf children is not an IPLD node")
+ }
+
+ return link.Resolve(p)
+}
+
+func (t *TrieNode) resolveTrieNodeBranch(p []string) (interface{}, []string, error) {
+ idx, rest := shiftFromPath(p, 1)
+ hidx := getHexIndex(idx)
+ if hidx == -1 {
+ return nil, nil, fmt.Errorf("incorrect path")
+ }
+
+ child := t.elements[hidx]
+ if child != nil {
+ return &node.Link{Cid: child.(cid.Cid)}, rest, nil
+ }
+ return nil, nil, fmt.Errorf("no such link in this branch")
+}
+
+// shiftFromPath extracts from a given path (as a slice of strings)
+// the given number of elements as a single string, returning whatever
+// it has not taken.
+//
+// Examples:
+// ["0", "a", "something"] and 1 -> "0" and ["a", "something"]
+// ["ab", "c", "d", "1"] and 2 -> "ab" and ["c", "d", "1"]
+// ["abc", "d", "1"] and 2 -> "ab" and ["c", "d", "1"]
+func shiftFromPath(p []string, i int) (string, []string) {
+ var (
+ out string
+ rest []string
+ )
+
+ for _, pe := range p {
+ re := ""
+ for _, c := range pe {
+ if len(out) < i {
+ out += string(c)
+ } else {
+ re += string(c)
+ }
+ }
+
+ if len(out) == i && re != "" {
+ rest = append(rest, re)
+ }
+ }
+
+ return out, rest
+}
+
+// getHexIndex returns to you the integer 0 - 15 equivalent to your
+// string character if applicable, or -1 otherwise.
+func getHexIndex(s string) int {
+ if len(s) != 1 {
+ return -1
+ }
+
+ c := byte(s[0])
+ switch {
+ case '0' <= c && c <= '9':
+ return int(c - '0')
+ case 'a' <= c && c <= 'f':
+ return int(c - 'a' + 10)
+ }
+
+ return -1
+}
diff --git a/statediff/indexer/ipfs/models.go b/statediff/indexer/ipfs/models.go
new file mode 100644
index 000000000..eb0312beb
--- /dev/null
+++ b/statediff/indexer/ipfs/models.go
@@ -0,0 +1,22 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package ipfs
+
+type BlockModel struct {
+ CID string `db:"key"`
+ Data []byte `db:"data"`
+}
diff --git a/statediff/indexer/metrics.go b/statediff/indexer/metrics.go
new file mode 100644
index 000000000..fc0727eda
--- /dev/null
+++ b/statediff/indexer/metrics.go
@@ -0,0 +1,124 @@
+package indexer
+
+import (
+ "database/sql"
+ "strings"
+
+ "github.com/ethereum/go-ethereum/metrics"
+)
+
+const (
+ namespace = "statediff"
+)
+
+// Build a fully qualified metric name
+func metricName(subsystem, name string) string {
+ if name == "" {
+ return ""
+ }
+ parts := []string{namespace, name}
+ if subsystem != "" {
+ parts = []string{namespace, subsystem, name}
+ }
+ // Prometheus uses _ but geth metrics uses / and replaces
+ return strings.Join(parts, "/")
+}
+
+type indexerMetricsHandles struct {
+ // The total number of processed blocks
+ blocks metrics.Counter
+ // The total number of processed transactions
+ transactions metrics.Counter
+ // The total number of processed receipts
+ receipts metrics.Counter
+ // Time spent waiting for free postgres tx
+ tFreePostgres metrics.Timer
+ // Postgres transaction commit duration
+ tPostgresCommit metrics.Timer
+ // Header processing time
+ tHeaderProcessing metrics.Timer
+ // Uncle processing time
+ tUncleProcessing metrics.Timer
+ // Tx and receipt processing time
+ tTxAndRecProcessing metrics.Timer
+ // State, storage, and code combined processing time
+ tStateStoreCodeProcessing metrics.Timer
+}
+
+func RegisterIndexerMetrics(reg metrics.Registry) indexerMetricsHandles {
+ ctx := indexerMetricsHandles{
+ blocks: metrics.NewCounter(),
+ transactions: metrics.NewCounter(),
+ receipts: metrics.NewCounter(),
+ tFreePostgres: metrics.NewTimer(),
+ tPostgresCommit: metrics.NewTimer(),
+ tHeaderProcessing: metrics.NewTimer(),
+ tUncleProcessing: metrics.NewTimer(),
+ tTxAndRecProcessing: metrics.NewTimer(),
+ tStateStoreCodeProcessing: metrics.NewTimer(),
+ }
+ subsys := "indexer"
+ reg.Register(metricName(subsys, "blocks"), ctx.blocks)
+ reg.Register(metricName(subsys, "transactions"), ctx.transactions)
+ reg.Register(metricName(subsys, "receipts"), ctx.receipts)
+ reg.Register(metricName(subsys, "t_free_postgres"), ctx.tFreePostgres)
+ reg.Register(metricName(subsys, "t_postgres_commit"), ctx.tPostgresCommit)
+ reg.Register(metricName(subsys, "t_header_processing"), ctx.tHeaderProcessing)
+ reg.Register(metricName(subsys, "t_uncle_processing"), ctx.tUncleProcessing)
+ reg.Register(metricName(subsys, "t_tx_receipt_processing"), ctx.tTxAndRecProcessing)
+ reg.Register(metricName(subsys, "t_state_store_code_processing"), ctx.tStateStoreCodeProcessing)
+ return ctx
+}
+
+type dbMetricsHandles struct {
+ // Maximum number of open connections to the database
+ maxOpen metrics.Gauge
+ // The number of established connections both in use and idle
+ open metrics.Gauge
+ // The number of connections currently in use
+ inUse metrics.Gauge
+ // The number of idle connections
+ idle metrics.Gauge
+ // The total number of connections waited for
+ waitedFor metrics.Counter
+ // The total time blocked waiting for a new connection
+ blockedMilliseconds metrics.Counter
+ // The total number of connections closed due to SetMaxIdleConns
+ closedMaxIdle metrics.Counter
+ // The total number of connections closed due to SetConnMaxLifetime
+ closedMaxLifetime metrics.Counter
+}
+
+func RegisterDBMetrics(reg metrics.Registry) dbMetricsHandles {
+ ctx := dbMetricsHandles{
+ maxOpen: metrics.NewGauge(),
+ open: metrics.NewGauge(),
+ inUse: metrics.NewGauge(),
+ idle: metrics.NewGauge(),
+ waitedFor: metrics.NewCounter(),
+ blockedMilliseconds: metrics.NewCounter(),
+ closedMaxIdle: metrics.NewCounter(),
+ closedMaxLifetime: metrics.NewCounter(),
+ }
+ subsys := "connections"
+ reg.Register(metricName(subsys, "max_open"), ctx.maxOpen)
+ reg.Register(metricName(subsys, "open"), ctx.open)
+ reg.Register(metricName(subsys, "in_use"), ctx.inUse)
+ reg.Register(metricName(subsys, "idle"), ctx.idle)
+ reg.Register(metricName(subsys, "waited_for"), ctx.waitedFor)
+ reg.Register(metricName(subsys, "blocked_milliseconds"), ctx.blockedMilliseconds)
+ reg.Register(metricName(subsys, "closed_max_idle"), ctx.closedMaxIdle)
+ reg.Register(metricName(subsys, "closed_max_lifetime"), ctx.closedMaxLifetime)
+ return ctx
+}
+
+func (met *dbMetricsHandles) Update(stats sql.DBStats) {
+ met.maxOpen.Update(int64(stats.MaxOpenConnections))
+ met.open.Update(int64(stats.OpenConnections))
+ met.inUse.Update(int64(stats.InUse))
+ met.idle.Update(int64(stats.Idle))
+ met.waitedFor.Inc(int64(stats.WaitCount))
+ met.blockedMilliseconds.Inc(int64(stats.WaitDuration.Milliseconds()))
+ met.closedMaxIdle.Inc(int64(stats.MaxIdleClosed))
+ met.closedMaxLifetime.Inc(int64(stats.MaxLifetimeClosed))
+}
diff --git a/statediff/indexer/mocks/test_data.go b/statediff/indexer/mocks/test_data.go
new file mode 100644
index 000000000..e2a2af54b
--- /dev/null
+++ b/statediff/indexer/mocks/test_data.go
@@ -0,0 +1,194 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package mocks
+
+import (
+ "crypto/ecdsa"
+ "crypto/elliptic"
+ "crypto/rand"
+ "math/big"
+
+ "github.com/ethereum/go-ethereum/trie"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/core/state"
+ "github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/crypto"
+ "github.com/ethereum/go-ethereum/log"
+ "github.com/ethereum/go-ethereum/params"
+ "github.com/ethereum/go-ethereum/rlp"
+ "github.com/multiformats/go-multihash"
+
+ "github.com/ethereum/go-ethereum/statediff/indexer/ipfs/ipld"
+ "github.com/ethereum/go-ethereum/statediff/testhelpers"
+ sdtypes "github.com/ethereum/go-ethereum/statediff/types"
+)
+
+// Test variables
+var (
+ // block data
+ BlockNumber = big.NewInt(1)
+ MockHeader = types.Header{
+ Time: 0,
+ Number: new(big.Int).Set(BlockNumber),
+ Root: common.HexToHash("0x0"),
+ TxHash: common.HexToHash("0x0"),
+ ReceiptHash: common.HexToHash("0x0"),
+ Difficulty: big.NewInt(5000000),
+ Extra: []byte{},
+ }
+ MockTransactions, MockReceipts, SenderAddr = createTransactionsAndReceipts()
+ ReceiptsRlp, _ = rlp.EncodeToBytes(MockReceipts)
+ MockBlock = types.NewBlock(&MockHeader, MockTransactions, nil, MockReceipts, new(trie.Trie))
+ MockHeaderRlp, _ = rlp.EncodeToBytes(MockBlock.Header())
+ Address = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592")
+ AnotherAddress = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476593")
+ ContractAddress = crypto.CreateAddress(SenderAddr, MockTransactions[2].Nonce())
+ ContractHash = crypto.Keccak256Hash(ContractAddress.Bytes()).String()
+ MockContractByteCode = []byte{0, 1, 2, 3, 4, 5}
+ mockTopic11 = common.HexToHash("0x04")
+ mockTopic12 = common.HexToHash("0x06")
+ mockTopic21 = common.HexToHash("0x05")
+ mockTopic22 = common.HexToHash("0x07")
+ MockLog1 = &types.Log{
+ Address: Address,
+ Topics: []common.Hash{mockTopic11, mockTopic12},
+ Data: []byte{},
+ }
+ MockLog2 = &types.Log{
+ Address: AnotherAddress,
+ Topics: []common.Hash{mockTopic21, mockTopic22},
+ Data: []byte{},
+ }
+ HeaderCID, _ = ipld.RawdataToCid(ipld.MEthHeader, MockHeaderRlp, multihash.KECCAK_256)
+ Trx1CID, _ = ipld.RawdataToCid(ipld.MEthTx, MockTransactions.GetRlp(0), multihash.KECCAK_256)
+ Trx2CID, _ = ipld.RawdataToCid(ipld.MEthTx, MockTransactions.GetRlp(1), multihash.KECCAK_256)
+ Trx3CID, _ = ipld.RawdataToCid(ipld.MEthTx, MockTransactions.GetRlp(2), multihash.KECCAK_256)
+ Rct1CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, MockReceipts.GetRlp(0), multihash.KECCAK_256)
+ Rct2CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, MockReceipts.GetRlp(1), multihash.KECCAK_256)
+ Rct3CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, MockReceipts.GetRlp(2), multihash.KECCAK_256)
+ State1CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, ContractLeafNode, multihash.KECCAK_256)
+ State2CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, AccountLeafNode, multihash.KECCAK_256)
+ StorageCID, _ = ipld.RawdataToCid(ipld.MEthStorageTrie, StorageLeafNode, multihash.KECCAK_256)
+
+ // statediff data
+ storageLocation = common.HexToHash("0")
+ StorageLeafKey = crypto.Keccak256Hash(storageLocation[:]).Bytes()
+ StorageValue = common.Hex2Bytes("01")
+ StoragePartialPath = common.Hex2Bytes("20290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563")
+ StorageLeafNode, _ = rlp.EncodeToBytes([]interface{}{
+ StoragePartialPath,
+ StorageValue,
+ })
+
+ nonce1 = uint64(1)
+ ContractRoot = "0x821e2556a290c86405f8160a2d662042a431ba456b9db265c79bb837c04be5f0"
+ ContractCodeHash = common.HexToHash("0x753f98a8d4328b15636e46f66f2cb4bc860100aa17967cc145fcd17d1d4710ea")
+ ContractLeafKey = testhelpers.AddressToLeafKey(ContractAddress)
+ ContractAccount, _ = rlp.EncodeToBytes(state.Account{
+ Nonce: nonce1,
+ Balance: big.NewInt(0),
+ CodeHash: ContractCodeHash.Bytes(),
+ Root: common.HexToHash(ContractRoot),
+ })
+ ContractPartialPath = common.Hex2Bytes("3114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45")
+ ContractLeafNode, _ = rlp.EncodeToBytes([]interface{}{
+ ContractPartialPath,
+ ContractAccount,
+ })
+
+ nonce0 = uint64(0)
+ AccountRoot = "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"
+ AccountCodeHash = common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")
+ AccountLeafKey = testhelpers.Account2LeafKey
+ Account, _ = rlp.EncodeToBytes(state.Account{
+ Nonce: nonce0,
+ Balance: big.NewInt(1000),
+ CodeHash: AccountCodeHash.Bytes(),
+ Root: common.HexToHash(AccountRoot),
+ })
+ AccountPartialPath = common.Hex2Bytes("3957f3e2f04a0764c3a0491b175f69926da61efbcc8f61fa1455fd2d2b4cdd45")
+ AccountLeafNode, _ = rlp.EncodeToBytes([]interface{}{
+ AccountPartialPath,
+ Account,
+ })
+
+ StateDiffs = []sdtypes.StateNode{
+ {
+ Path: []byte{'\x06'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: ContractLeafKey,
+ NodeValue: ContractLeafNode,
+ StorageNodes: []sdtypes.StorageNode{
+ {
+ Path: []byte{},
+ NodeType: sdtypes.Leaf,
+ LeafKey: StorageLeafKey,
+ NodeValue: StorageLeafNode,
+ },
+ },
+ },
+ {
+ Path: []byte{'\x0c'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: AccountLeafKey,
+ NodeValue: AccountLeafNode,
+ StorageNodes: []sdtypes.StorageNode{},
+ },
+ }
+)
+
+// createTransactionsAndReceipts is a helper function to generate signed mock transactions and mock receipts with mock logs
+func createTransactionsAndReceipts() (types.Transactions, types.Receipts, common.Address) {
+ // make transactions
+ trx1 := types.NewTransaction(0, Address, big.NewInt(1000), 50, big.NewInt(100), []byte{})
+ trx2 := types.NewTransaction(1, AnotherAddress, big.NewInt(2000), 100, big.NewInt(200), []byte{})
+ trx3 := types.NewContractCreation(2, big.NewInt(1500), 75, big.NewInt(150), MockContractByteCode)
+ transactionSigner := types.MakeSigner(params.MainnetChainConfig, new(big.Int).Set(BlockNumber))
+ mockCurve := elliptic.P256()
+ mockPrvKey, err := ecdsa.GenerateKey(mockCurve, rand.Reader)
+ if err != nil {
+ log.Crit(err.Error())
+ }
+ signedTrx1, err := types.SignTx(trx1, transactionSigner, mockPrvKey)
+ if err != nil {
+ log.Crit(err.Error())
+ }
+ signedTrx2, err := types.SignTx(trx2, transactionSigner, mockPrvKey)
+ if err != nil {
+ log.Crit(err.Error())
+ }
+ signedTrx3, err := types.SignTx(trx3, transactionSigner, mockPrvKey)
+ if err != nil {
+ log.Crit(err.Error())
+ }
+ SenderAddr, err := types.Sender(transactionSigner, signedTrx1) // same for both trx
+ if err != nil {
+ log.Crit(err.Error())
+ }
+ // make receipts
+ mockReceipt1 := types.NewReceipt(common.HexToHash("0x0").Bytes(), false, 50)
+ mockReceipt1.Logs = []*types.Log{MockLog1}
+ mockReceipt1.TxHash = signedTrx1.Hash()
+ mockReceipt2 := types.NewReceipt(common.HexToHash("0x1").Bytes(), false, 100)
+ mockReceipt2.Logs = []*types.Log{MockLog2}
+ mockReceipt2.TxHash = signedTrx2.Hash()
+ mockReceipt3 := types.NewReceipt(common.HexToHash("0x2").Bytes(), false, 75)
+ mockReceipt3.Logs = []*types.Log{}
+ mockReceipt3.TxHash = signedTrx3.Hash()
+ return types.Transactions{signedTrx1, signedTrx2, signedTrx3}, types.Receipts{mockReceipt1, mockReceipt2, mockReceipt3}, SenderAddr
+}
diff --git a/statediff/indexer/models/models.go b/statediff/indexer/models/models.go
new file mode 100644
index 000000000..8e8ebf624
--- /dev/null
+++ b/statediff/indexer/models/models.go
@@ -0,0 +1,126 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package models
+
+import "github.com/lib/pq"
+
+// HeaderModel is the db model for eth.header_cids
+type HeaderModel struct {
+ ID int64 `db:"id"`
+ BlockNumber string `db:"block_number"`
+ BlockHash string `db:"block_hash"`
+ ParentHash string `db:"parent_hash"`
+ CID string `db:"cid"`
+ MhKey string `db:"mh_key"`
+ TotalDifficulty string `db:"td"`
+ NodeID int64 `db:"node_id"`
+ Reward string `db:"reward"`
+ StateRoot string `db:"state_root"`
+ UncleRoot string `db:"uncle_root"`
+ TxRoot string `db:"tx_root"`
+ RctRoot string `db:"receipt_root"`
+ Bloom []byte `db:"bloom"`
+ Timestamp uint64 `db:"timestamp"`
+ TimesValidated int64 `db:"times_validated"`
+}
+
+// UncleModel is the db model for eth.uncle_cids
+type UncleModel struct {
+ ID int64 `db:"id"`
+ HeaderID int64 `db:"header_id"`
+ BlockHash string `db:"block_hash"`
+ ParentHash string `db:"parent_hash"`
+ CID string `db:"cid"`
+ MhKey string `db:"mh_key"`
+ Reward string `db:"reward"`
+}
+
+// TxModel is the db model for eth.transaction_cids
+type TxModel struct {
+ ID int64 `db:"id"`
+ HeaderID int64 `db:"header_id"`
+ Index int64 `db:"index"`
+ TxHash string `db:"tx_hash"`
+ CID string `db:"cid"`
+ MhKey string `db:"mh_key"`
+ Dst string `db:"dst"`
+ Src string `db:"src"`
+ Data []byte `db:"tx_data"`
+ Deployment bool `db:"deployment"`
+}
+
+// ReceiptModel is the db model for eth.receipt_cids
+type ReceiptModel struct {
+ ID int64 `db:"id"`
+ TxID int64 `db:"tx_id"`
+ CID string `db:"cid"`
+ MhKey string `db:"mh_key"`
+ Contract string `db:"contract"`
+ ContractHash string `db:"contract_hash"`
+ LogContracts pq.StringArray `db:"log_contracts"`
+ Topic0s pq.StringArray `db:"topic0s"`
+ Topic1s pq.StringArray `db:"topic1s"`
+ Topic2s pq.StringArray `db:"topic2s"`
+ Topic3s pq.StringArray `db:"topic3s"`
+}
+
+// StateNodeModel is the db model for eth.state_cids
+type StateNodeModel struct {
+ ID int64 `db:"id"`
+ HeaderID int64 `db:"header_id"`
+ Path []byte `db:"state_path"`
+ StateKey string `db:"state_leaf_key"`
+ NodeType int `db:"node_type"`
+ CID string `db:"cid"`
+ MhKey string `db:"mh_key"`
+ Diff bool `db:"diff"`
+}
+
+// StorageNodeModel is the db model for eth.storage_cids
+type StorageNodeModel struct {
+ ID int64 `db:"id"`
+ StateID int64 `db:"state_id"`
+ Path []byte `db:"storage_path"`
+ StorageKey string `db:"storage_leaf_key"`
+ NodeType int `db:"node_type"`
+ CID string `db:"cid"`
+ MhKey string `db:"mh_key"`
+ Diff bool `db:"diff"`
+}
+
+// StorageNodeWithStateKeyModel is a db model for eth.storage_cids + eth.state_cids.state_key
+type StorageNodeWithStateKeyModel struct {
+ ID int64 `db:"id"`
+ StateID int64 `db:"state_id"`
+ Path []byte `db:"storage_path"`
+ StateKey string `db:"state_leaf_key"`
+ StorageKey string `db:"storage_leaf_key"`
+ NodeType int `db:"node_type"`
+ CID string `db:"cid"`
+ MhKey string `db:"mh_key"`
+ Diff bool `db:"diff"`
+}
+
+// StateAccountModel is a db model for an eth state account (decoded value of state leaf node)
+type StateAccountModel struct {
+ ID int64 `db:"id"`
+ StateID int64 `db:"state_id"`
+ Balance string `db:"balance"`
+ Nonce uint64 `db:"nonce"`
+ CodeHash []byte `db:"code_hash"`
+ StorageRoot string `db:"storage_root"`
+}
diff --git a/statediff/indexer/node/node.go b/statediff/indexer/node/node.go
new file mode 100644
index 000000000..527546efa
--- /dev/null
+++ b/statediff/indexer/node/node.go
@@ -0,0 +1,25 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package node
+
+type Info struct {
+ GenesisBlock string
+ NetworkID string
+ ChainID uint64
+ ID string
+ ClientName string
+}
diff --git a/statediff/indexer/postgres/config.go b/statediff/indexer/postgres/config.go
new file mode 100644
index 000000000..c2de0a6bf
--- /dev/null
+++ b/statediff/indexer/postgres/config.go
@@ -0,0 +1,59 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package postgres
+
+import (
+ "fmt"
+)
+
+// Env variables
+const (
+ DATABASE_NAME = "DATABASE_NAME"
+ DATABASE_HOSTNAME = "DATABASE_HOSTNAME"
+ DATABASE_PORT = "DATABASE_PORT"
+ DATABASE_USER = "DATABASE_USER"
+ DATABASE_PASSWORD = "DATABASE_PASSWORD"
+ DATABASE_MAX_IDLE_CONNECTIONS = "DATABASE_MAX_IDLE_CONNECTIONS"
+ DATABASE_MAX_OPEN_CONNECTIONS = "DATABASE_MAX_OPEN_CONNECTIONS"
+ DATABASE_MAX_CONN_LIFETIME = "DATABASE_MAX_CONN_LIFETIME"
+)
+
+type ConnectionParams struct {
+ Hostname string
+ Name string
+ User string
+ Password string
+ Port int
+}
+
+type ConnectionConfig struct {
+ MaxIdle int
+ MaxOpen int
+ MaxLifetime int
+}
+
+func DbConnectionString(params ConnectionParams) string {
+ if len(params.User) > 0 && len(params.Password) > 0 {
+ return fmt.Sprintf("postgresql://%s:%s@%s:%d/%s?sslmode=disable",
+ params.User, params.Password, params.Hostname, params.Port, params.Name)
+ }
+ if len(params.User) > 0 && len(params.Password) == 0 {
+ return fmt.Sprintf("postgresql://%s@%s:%d/%s?sslmode=disable",
+ params.User, params.Hostname, params.Port, params.Name)
+ }
+ return fmt.Sprintf("postgresql://%s:%d/%s?sslmode=disable", params.Hostname, params.Port, params.Name)
+}
diff --git a/statediff/indexer/postgres/errors.go b/statediff/indexer/postgres/errors.go
new file mode 100644
index 000000000..f368f900b
--- /dev/null
+++ b/statediff/indexer/postgres/errors.go
@@ -0,0 +1,53 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package postgres
+
+import (
+ "fmt"
+)
+
+const (
+ BeginTransactionFailedMsg = "failed to begin transaction"
+ DbConnectionFailedMsg = "db connection failed"
+ DeleteQueryFailedMsg = "delete query failed"
+ InsertQueryFailedMsg = "insert query failed"
+ SettingNodeFailedMsg = "unable to set db node"
+)
+
+func ErrBeginTransactionFailed(beginErr error) error {
+ return formatError(BeginTransactionFailedMsg, beginErr.Error())
+}
+
+func ErrDBConnectionFailed(connectErr error) error {
+ return formatError(DbConnectionFailedMsg, connectErr.Error())
+}
+
+func ErrDBDeleteFailed(deleteErr error) error {
+ return formatError(DeleteQueryFailedMsg, deleteErr.Error())
+}
+
+func ErrDBInsertFailed(insertErr error) error {
+ return formatError(InsertQueryFailedMsg, insertErr.Error())
+}
+
+func ErrUnableToSetNode(setErr error) error {
+ return formatError(SettingNodeFailedMsg, setErr.Error())
+}
+
+func formatError(msg, err string) error {
+ return fmt.Errorf("%s: %s", msg, err)
+}
diff --git a/statediff/indexer/postgres/postgres.go b/statediff/indexer/postgres/postgres.go
new file mode 100644
index 000000000..455dac306
--- /dev/null
+++ b/statediff/indexer/postgres/postgres.go
@@ -0,0 +1,76 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package postgres
+
+import (
+ "time"
+
+ "github.com/jmoiron/sqlx"
+ _ "github.com/lib/pq" //postgres driver
+
+ "github.com/ethereum/go-ethereum/statediff/indexer/node"
+)
+
+type DB struct {
+ *sqlx.DB
+ Node node.Info
+ NodeID int64
+}
+
+func NewDB(connectString string, config ConnectionConfig, node node.Info) (*DB, error) {
+ db, connectErr := sqlx.Connect("postgres", connectString)
+ if connectErr != nil {
+ return &DB{}, ErrDBConnectionFailed(connectErr)
+ }
+ if config.MaxOpen > 0 {
+ db.SetMaxOpenConns(config.MaxOpen)
+ }
+ if config.MaxIdle > 0 {
+ db.SetMaxIdleConns(config.MaxIdle)
+ }
+ if config.MaxLifetime > 0 {
+ lifetime := time.Duration(config.MaxLifetime) * time.Second
+ db.SetConnMaxLifetime(lifetime)
+ }
+ pg := DB{DB: db, Node: node}
+ nodeErr := pg.CreateNode(&node)
+ if nodeErr != nil {
+ return &DB{}, ErrUnableToSetNode(nodeErr)
+ }
+ return &pg, nil
+}
+
+func (db *DB) CreateNode(node *node.Info) error {
+ var nodeID int64
+ err := db.QueryRow(
+ `INSERT INTO nodes (genesis_block, network_id, node_id, client_name, chain_id)
+ VALUES ($1, $2, $3, $4, $5)
+ ON CONFLICT (genesis_block, network_id, node_id, chain_id)
+ DO UPDATE
+ SET genesis_block = $1,
+ network_id = $2,
+ node_id = $3,
+ client_name = $4,
+ chain_id = $5
+ RETURNING id`,
+ node.GenesisBlock, node.NetworkID, node.ID, node.ClientName, node.ChainID).Scan(&nodeID)
+ if err != nil {
+ return ErrUnableToSetNode(err)
+ }
+ db.NodeID = nodeID
+ return nil
+}
diff --git a/statediff/indexer/postgres/postgres_suite_test.go b/statediff/indexer/postgres/postgres_suite_test.go
new file mode 100644
index 000000000..e11e9ea77
--- /dev/null
+++ b/statediff/indexer/postgres/postgres_suite_test.go
@@ -0,0 +1,25 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package postgres_test
+
+import (
+ "github.com/ethereum/go-ethereum/log"
+)
+
+func init() {
+ log.Root().SetHandler(log.DiscardHandler())
+}
diff --git a/statediff/indexer/postgres/postgres_test.go b/statediff/indexer/postgres/postgres_test.go
new file mode 100644
index 000000000..f86055352
--- /dev/null
+++ b/statediff/indexer/postgres/postgres_test.go
@@ -0,0 +1,131 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package postgres_test
+
+import (
+ "fmt"
+ "strings"
+ "testing"
+
+ "math/big"
+
+ "github.com/jmoiron/sqlx"
+ _ "github.com/lib/pq"
+
+ "github.com/ethereum/go-ethereum/statediff/indexer/node"
+ "github.com/ethereum/go-ethereum/statediff/indexer/postgres"
+ "github.com/ethereum/go-ethereum/statediff/indexer/shared"
+)
+
+var DBParams postgres.ConnectionParams
+
+func expectContainsSubstring(t *testing.T, full string, sub string) {
+ if !strings.Contains(full, sub) {
+ t.Fatalf("Expected \"%v\" to contain substring \"%v\"\n", full, sub)
+ }
+}
+
+func TestPostgresDB(t *testing.T) {
+ var sqlxdb *sqlx.DB
+
+ t.Run("connects to the database", func(t *testing.T) {
+ var err error
+ pgConfig := postgres.DbConnectionString(DBParams)
+
+ sqlxdb, err = sqlx.Connect("postgres", pgConfig)
+
+ if err != nil {
+ t.Fatal(err)
+ }
+ if sqlxdb == nil {
+ t.Fatal("DB is nil")
+ }
+ })
+
+ t.Run("serializes big.Int to db", func(t *testing.T) {
+ // postgres driver doesn't support go big.Int type
+ // various casts in golang uint64, int64, overflow for
+ // transaction value (in wei) even though
+ // postgres numeric can handle an arbitrary
+ // sized int, so use string representation of big.Int
+ // and cast on insert
+
+ pgConnectString := postgres.DbConnectionString(DBParams)
+ db, err := sqlx.Connect("postgres", pgConnectString)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ bi := new(big.Int)
+ bi.SetString("34940183920000000000", 10)
+ shared.ExpectEqual(t, bi.String(), "34940183920000000000")
+
+ defer db.Exec(`DROP TABLE IF EXISTS example`)
+ _, err = db.Exec("CREATE TABLE example ( id INTEGER, data NUMERIC )")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ sqlStatement := `
+ INSERT INTO example (id, data)
+ VALUES (1, cast($1 AS NUMERIC))`
+ _, err = db.Exec(sqlStatement, bi.String())
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ var data string
+ err = db.QueryRow(`SELECT data FROM example WHERE id = 1`).Scan(&data)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ shared.ExpectEqual(t, bi.String(), data)
+ actual := new(big.Int)
+ actual.SetString(data, 10)
+ shared.ExpectEqual(t, actual, bi)
+ })
+
+ t.Run("throws error when can't connect to the database", func(t *testing.T) {
+ invalidDatabase := postgres.ConnectionParams{}
+ node := node.Info{GenesisBlock: "GENESIS", NetworkID: "1", ID: "x123", ClientName: "geth"}
+
+ _, err := postgres.NewDB(postgres.DbConnectionString(invalidDatabase),
+ postgres.ConnectionConfig{}, node)
+
+ if err == nil {
+ t.Fatal("Expected an error")
+ }
+
+ expectContainsSubstring(t, err.Error(), postgres.DbConnectionFailedMsg)
+ })
+
+ t.Run("throws error when can't create node", func(t *testing.T) {
+ badHash := fmt.Sprintf("x %s", strings.Repeat("1", 100))
+ node := node.Info{GenesisBlock: badHash, NetworkID: "1", ID: "x123", ClientName: "geth"}
+
+ _, err := postgres.NewDB(postgres.DbConnectionString(DBParams), postgres.ConnectionConfig{}, node)
+
+ if err == nil {
+ t.Fatal("Expected an error")
+ }
+ expectContainsSubstring(t, err.Error(), postgres.SettingNodeFailedMsg)
+ })
+}
diff --git a/statediff/indexer/reward.go b/statediff/indexer/reward.go
new file mode 100644
index 000000000..47e3f17b9
--- /dev/null
+++ b/statediff/indexer/reward.go
@@ -0,0 +1,76 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package indexer
+
+import (
+ "math/big"
+
+ "github.com/ethereum/go-ethereum/core/types"
+)
+
+func CalcEthBlockReward(header *types.Header, uncles []*types.Header, txs types.Transactions, receipts types.Receipts) *big.Int {
+ staticBlockReward := staticRewardByBlockNumber(header.Number.Uint64())
+ transactionFees := calcEthTransactionFees(txs, receipts)
+ uncleInclusionRewards := calcEthUncleInclusionRewards(header, uncles)
+ tmp := transactionFees.Add(transactionFees, uncleInclusionRewards)
+ return tmp.Add(tmp, staticBlockReward)
+}
+
+func CalcUncleMinerReward(blockNumber, uncleBlockNumber uint64) *big.Int {
+ staticBlockReward := staticRewardByBlockNumber(blockNumber)
+ rewardDiv8 := staticBlockReward.Div(staticBlockReward, big.NewInt(8))
+ mainBlock := new(big.Int).SetUint64(blockNumber)
+ uncleBlock := new(big.Int).SetUint64(uncleBlockNumber)
+ uncleBlockPlus8 := uncleBlock.Add(uncleBlock, big.NewInt(8))
+ uncleBlockPlus8MinusMainBlock := uncleBlockPlus8.Sub(uncleBlockPlus8, mainBlock)
+ return rewardDiv8.Mul(rewardDiv8, uncleBlockPlus8MinusMainBlock)
+}
+
+func staticRewardByBlockNumber(blockNumber uint64) *big.Int {
+ staticBlockReward := new(big.Int)
+ //https://blog.ethereum.org/2017/10/12/byzantium-hf-announcement/
+ if blockNumber >= 7280000 {
+ staticBlockReward.SetString("2000000000000000000", 10)
+ } else if blockNumber >= 4370000 {
+ staticBlockReward.SetString("3000000000000000000", 10)
+ } else {
+ staticBlockReward.SetString("5000000000000000000", 10)
+ }
+ return staticBlockReward
+}
+
+func calcEthTransactionFees(txs types.Transactions, receipts types.Receipts) *big.Int {
+ transactionFees := new(big.Int)
+ for i, transaction := range txs {
+ receipt := receipts[i]
+ gasPrice := big.NewInt(transaction.GasPrice().Int64())
+ gasUsed := big.NewInt(int64(receipt.GasUsed))
+ transactionFee := gasPrice.Mul(gasPrice, gasUsed)
+ transactionFees = transactionFees.Add(transactionFees, transactionFee)
+ }
+ return transactionFees
+}
+
+func calcEthUncleInclusionRewards(header *types.Header, uncles []*types.Header) *big.Int {
+ uncleInclusionRewards := new(big.Int)
+ for range uncles {
+ staticBlockReward := staticRewardByBlockNumber(header.Number.Uint64())
+ staticBlockReward.Div(staticBlockReward, big.NewInt(32))
+ uncleInclusionRewards.Add(uncleInclusionRewards, staticBlockReward)
+ }
+ return uncleInclusionRewards
+}
diff --git a/statediff/indexer/shared/chain_type.go b/statediff/indexer/shared/chain_type.go
new file mode 100644
index 000000000..c3dedfe38
--- /dev/null
+++ b/statediff/indexer/shared/chain_type.go
@@ -0,0 +1,78 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package shared
+
+import (
+ "errors"
+ "strings"
+)
+
+// ChainType enum for specifying blockchain
+type ChainType int
+
+const (
+ UnknownChain ChainType = iota
+ Ethereum
+ Bitcoin
+ Omni
+ EthereumClassic
+)
+
+func (c ChainType) String() string {
+ switch c {
+ case Ethereum:
+ return "Ethereum"
+ case Bitcoin:
+ return "Bitcoin"
+ case Omni:
+ return "Omni"
+ case EthereumClassic:
+ return "EthereumClassic"
+ default:
+ return ""
+ }
+}
+
+func (c ChainType) API() string {
+ switch c {
+ case Ethereum:
+ return "eth"
+ case Bitcoin:
+ return "btc"
+ case Omni:
+ return "omni"
+ case EthereumClassic:
+ return "etc"
+ default:
+ return ""
+ }
+}
+
+func NewChainType(name string) (ChainType, error) {
+ switch strings.ToLower(name) {
+ case "ethereum", "eth":
+ return Ethereum, nil
+ case "bitcoin", "btc", "xbt":
+ return Bitcoin, nil
+ case "omni":
+ return Omni, nil
+ case "classic", "etc":
+ return EthereumClassic, nil
+ default:
+ return UnknownChain, errors.New("invalid name for chain")
+ }
+}
diff --git a/statediff/indexer/shared/constants.go b/statediff/indexer/shared/constants.go
new file mode 100644
index 000000000..3dc2994c4
--- /dev/null
+++ b/statediff/indexer/shared/constants.go
@@ -0,0 +1,22 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package shared
+
+const (
+ DefaultMaxBatchSize uint64 = 100
+ DefaultMaxBatchNumber int64 = 50
+)
diff --git a/statediff/indexer/shared/data_type.go b/statediff/indexer/shared/data_type.go
new file mode 100644
index 000000000..01fed57f7
--- /dev/null
+++ b/statediff/indexer/shared/data_type.go
@@ -0,0 +1,101 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package shared
+
+import (
+ "fmt"
+ "strings"
+)
+
+// DataType is an enum to loosely represent type of chain data
+type DataType int
+
+const (
+ UnknownDataType DataType = iota - 1
+ Full
+ Headers
+ Uncles
+ Transactions
+ Receipts
+ State
+ Storage
+)
+
+// String() method to resolve ReSyncType enum
+func (r DataType) String() string {
+ switch r {
+ case Full:
+ return "full"
+ case Headers:
+ return "headers"
+ case Uncles:
+ return "uncles"
+ case Transactions:
+ return "transactions"
+ case Receipts:
+ return "receipts"
+ case State:
+ return "state"
+ case Storage:
+ return "storage"
+ default:
+ return "unknown"
+ }
+}
+
+// GenerateDataTypeFromString
+func GenerateDataTypeFromString(str string) (DataType, error) {
+ switch strings.ToLower(str) {
+ case "full", "f":
+ return Full, nil
+ case "headers", "header", "h":
+ return Headers, nil
+ case "uncles", "u":
+ return Uncles, nil
+ case "transactions", "transaction", "trxs", "txs", "trx", "tx", "t":
+ return Transactions, nil
+ case "receipts", "receipt", "rcts", "rct", "r":
+ return Receipts, nil
+ case "state":
+ return State, nil
+ case "storage":
+ return Storage, nil
+ default:
+ return UnknownDataType, fmt.Errorf("unrecognized resync type: %s", str)
+ }
+}
+
+func SupportedDataType(d DataType) (bool, error) {
+ switch d {
+ case Full:
+ return true, nil
+ case Headers:
+ return true, nil
+ case Uncles:
+ return true, nil
+ case Transactions:
+ return true, nil
+ case Receipts:
+ return true, nil
+ case State:
+ return true, nil
+ case Storage:
+ return true, nil
+ default:
+ return true, nil
+ }
+}
diff --git a/statediff/indexer/shared/functions.go b/statediff/indexer/shared/functions.go
new file mode 100644
index 000000000..257100289
--- /dev/null
+++ b/statediff/indexer/shared/functions.go
@@ -0,0 +1,124 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package shared
+
+import (
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/log"
+ "github.com/ethereum/go-ethereum/statediff/indexer/ipfs/ipld"
+
+ "github.com/ipfs/go-cid"
+ "github.com/ipfs/go-ipfs-blockstore"
+ "github.com/ipfs/go-ipfs-ds-help"
+ node "github.com/ipfs/go-ipld-format"
+ "github.com/jmoiron/sqlx"
+ "github.com/multiformats/go-multihash"
+)
+
+// HandleZeroAddrPointer will return an emtpy string for a nil address pointer
+func HandleZeroAddrPointer(to *common.Address) string {
+ if to == nil {
+ return ""
+ }
+ return to.Hex()
+}
+
+// HandleZeroAddr will return an empty string for a 0 value address
+func HandleZeroAddr(to common.Address) string {
+ if to.Hex() == "0x0000000000000000000000000000000000000000" {
+ return ""
+ }
+ return to.Hex()
+}
+
+// Rollback sql transaction and log any error
+func Rollback(tx *sqlx.Tx) {
+ if err := tx.Rollback(); err != nil {
+ log.Error(err.Error())
+ }
+}
+
+// PublishIPLD is used to insert an IPLD into Postgres blockstore with the provided tx
+func PublishIPLD(tx *sqlx.Tx, i node.Node) error {
+ dbKey := dshelp.MultihashToDsKey(i.Cid().Hash())
+ prefixedKey := blockstore.BlockPrefix.String() + dbKey.String()
+ raw := i.RawData()
+ _, err := tx.Exec(`INSERT INTO public.blocks (key, data) VALUES ($1, $2) ON CONFLICT (key) DO NOTHING`, prefixedKey, raw)
+ return err
+}
+
+// FetchIPLD is used to retrieve an ipld from Postgres blockstore with the provided tx and cid string
+func FetchIPLD(tx *sqlx.Tx, cid string) ([]byte, error) {
+ mhKey, err := MultihashKeyFromCIDString(cid)
+ if err != nil {
+ return nil, err
+ }
+ pgStr := `SELECT data FROM public.blocks WHERE key = $1`
+ var block []byte
+ return block, tx.Get(&block, pgStr, mhKey)
+}
+
+// FetchIPLDByMhKey is used to retrieve an ipld from Postgres blockstore with the provided tx and mhkey string
+func FetchIPLDByMhKey(tx *sqlx.Tx, mhKey string) ([]byte, error) {
+ pgStr := `SELECT data FROM public.blocks WHERE key = $1`
+ var block []byte
+ return block, tx.Get(&block, pgStr, mhKey)
+}
+
+// MultihashKeyFromCID converts a cid into a blockstore-prefixed multihash db key string
+func MultihashKeyFromCID(c cid.Cid) string {
+ dbKey := dshelp.MultihashToDsKey(c.Hash())
+ return blockstore.BlockPrefix.String() + dbKey.String()
+}
+
+// MultihashKeyFromCIDString converts a cid string into a blockstore-prefixed multihash db key string
+func MultihashKeyFromCIDString(c string) (string, error) {
+ dc, err := cid.Decode(c)
+ if err != nil {
+ return "", err
+ }
+ dbKey := dshelp.MultihashToDsKey(dc.Hash())
+ return blockstore.BlockPrefix.String() + dbKey.String(), nil
+}
+
+// PublishRaw derives a cid from raw bytes and provided codec and multihash type, and writes it to the db tx
+func PublishRaw(tx *sqlx.Tx, codec, mh uint64, raw []byte) (string, error) {
+ c, err := ipld.RawdataToCid(codec, raw, mh)
+ if err != nil {
+ return "", err
+ }
+ dbKey := dshelp.MultihashToDsKey(c.Hash())
+ prefixedKey := blockstore.BlockPrefix.String() + dbKey.String()
+ _, err = tx.Exec(`INSERT INTO public.blocks (key, data) VALUES ($1, $2) ON CONFLICT (key) DO NOTHING`, prefixedKey, raw)
+ return c.String(), err
+}
+
+// MultihashKeyFromKeccak256 converts keccak256 hash bytes into a blockstore-prefixed multihash db key string
+func MultihashKeyFromKeccak256(hash common.Hash) (string, error) {
+ mh, err := multihash.Encode(hash.Bytes(), multihash.KECCAK_256)
+ if err != nil {
+ return "", err
+ }
+ dbKey := dshelp.MultihashToDsKey(mh)
+ return blockstore.BlockPrefix.String() + dbKey.String(), nil
+}
+
+// PublishDirect diretly writes a previously derived mhkey => value pair to the ipld database
+func PublishDirect(tx *sqlx.Tx, key string, value []byte) error {
+ _, err := tx.Exec(`INSERT INTO public.blocks (key, data) VALUES ($1, $2) ON CONFLICT (key) DO NOTHING`, key, value)
+ return err
+}
diff --git a/statediff/indexer/shared/test_helpers.go b/statediff/indexer/shared/test_helpers.go
new file mode 100644
index 000000000..b2e13f832
--- /dev/null
+++ b/statediff/indexer/shared/test_helpers.go
@@ -0,0 +1,68 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package shared
+
+import (
+ "reflect"
+ "testing"
+
+ "github.com/ipfs/go-cid"
+ "github.com/multiformats/go-multihash"
+
+ "github.com/ethereum/go-ethereum/statediff/indexer/node"
+ "github.com/ethereum/go-ethereum/statediff/indexer/postgres"
+)
+
+func ExpectEqual(t *testing.T, got interface{}, want interface{}) {
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("Expected: %v\nActual: %v", want, got)
+ }
+}
+
+// SetupDB is use to setup a db for watcher tests
+func SetupDB() (*postgres.DB, error) {
+ uri := postgres.DbConnectionString(postgres.ConnectionParams{
+ User: "postgres",
+ Password: "",
+ Hostname: "localhost",
+ Name: "vulcanize_testing",
+ Port: 5432,
+ })
+ return postgres.NewDB(uri, postgres.ConnectionConfig{}, node.Info{})
+}
+
+// ListContainsString used to check if a list of strings contains a particular string
+func ListContainsString(sss []string, s string) bool {
+ for _, str := range sss {
+ if s == str {
+ return true
+ }
+ }
+ return false
+}
+
+// TestCID creates a basic CID for testing purposes
+func TestCID(b []byte) cid.Cid {
+ pref := cid.Prefix{
+ Version: 1,
+ Codec: cid.Raw,
+ MhType: multihash.KECCAK_256,
+ MhLength: -1,
+ }
+ c, _ := pref.Sum(b)
+ return c
+}
diff --git a/statediff/indexer/shared/types.go b/statediff/indexer/shared/types.go
new file mode 100644
index 000000000..a9792522b
--- /dev/null
+++ b/statediff/indexer/shared/types.go
@@ -0,0 +1,42 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package shared
+
+import "github.com/ethereum/go-ethereum/common"
+import "github.com/ethereum/go-ethereum/statediff/types"
+import "github.com/ethereum/go-ethereum/statediff/indexer/models"
+
+// Trie struct used to flag node as leaf or not
+type TrieNode struct {
+ Path []byte
+ LeafKey common.Hash
+ Value []byte
+ Type types.NodeType
+}
+
+// CIDPayload is a struct to hold all the CIDs and their associated meta data for indexing in Postgres
+// Returned by IPLDPublisher
+// Passed to CIDIndexer
+type CIDPayload struct {
+ HeaderCID models.HeaderModel
+ UncleCIDs []models.UncleModel
+ TransactionCIDs []models.TxModel
+ ReceiptCIDs map[common.Hash]models.ReceiptModel
+ StateNodeCIDs []models.StateNodeModel
+ StateAccounts map[string]models.StateAccountModel
+ StorageNodeCIDs map[string][]models.StorageNodeModel
+}
diff --git a/statediff/indexer/test_helpers.go b/statediff/indexer/test_helpers.go
new file mode 100644
index 000000000..024bb58f0
--- /dev/null
+++ b/statediff/indexer/test_helpers.go
@@ -0,0 +1,60 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package indexer
+
+import (
+ "testing"
+
+ "github.com/ethereum/go-ethereum/statediff/indexer/postgres"
+)
+
+// TearDownDB is used to tear down the watcher dbs after tests
+func TearDownDB(t *testing.T, db *postgres.DB) {
+ tx, err := db.Beginx()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ _, err = tx.Exec(`DELETE FROM eth.header_cids`)
+ if err != nil {
+ t.Fatal(err)
+ }
+ _, err = tx.Exec(`DELETE FROM eth.transaction_cids`)
+ if err != nil {
+ t.Fatal(err)
+ }
+ _, err = tx.Exec(`DELETE FROM eth.receipt_cids`)
+ if err != nil {
+ t.Fatal(err)
+ }
+ _, err = tx.Exec(`DELETE FROM eth.state_cids`)
+ if err != nil {
+ t.Fatal(err)
+ }
+ _, err = tx.Exec(`DELETE FROM eth.storage_cids`)
+ if err != nil {
+ t.Fatal(err)
+ }
+ _, err = tx.Exec(`DELETE FROM blocks`)
+ if err != nil {
+ t.Fatal(err)
+ }
+ err = tx.Commit()
+ if err != nil {
+ t.Fatal(err)
+ }
+}
diff --git a/statediff/indexer/writer.go b/statediff/indexer/writer.go
new file mode 100644
index 000000000..24bef7ed3
--- /dev/null
+++ b/statediff/indexer/writer.go
@@ -0,0 +1,137 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package indexer
+
+import (
+ "github.com/jmoiron/sqlx"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/statediff/indexer/models"
+ "github.com/ethereum/go-ethereum/statediff/indexer/postgres"
+ "github.com/ethereum/go-ethereum/statediff/indexer/shared"
+)
+
+var (
+ nullHash = common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000")
+)
+
+// Handles processing and writing of indexed IPLD objects to Postgres
+type PostgresCIDWriter struct {
+ db *postgres.DB
+}
+
+// NewPostgresCIDWriter creates a new pointer to a Indexer which satisfies the PostgresCIDWriter interface
+func NewPostgresCIDWriter(db *postgres.DB) *PostgresCIDWriter {
+ return &PostgresCIDWriter{
+ db: db,
+ }
+}
+
+func (in *PostgresCIDWriter) upsertHeaderCID(tx *sqlx.Tx, header models.HeaderModel) (int64, error) {
+ var headerID int64
+ err := tx.QueryRowx(`INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated)
+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)
+ ON CONFLICT (block_number, block_hash) DO UPDATE SET (parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated) = ($3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, eth.header_cids.times_validated + 1)
+ RETURNING id`,
+ header.BlockNumber, header.BlockHash, header.ParentHash, header.CID, header.TotalDifficulty, in.db.NodeID, header.Reward, header.StateRoot, header.TxRoot,
+ header.RctRoot, header.UncleRoot, header.Bloom, header.Timestamp, header.MhKey, 1).Scan(&headerID)
+ if err == nil {
+ indexerMetrics.blocks.Inc(1)
+ }
+ return headerID, err
+}
+
+func (in *PostgresCIDWriter) upsertUncleCID(tx *sqlx.Tx, uncle models.UncleModel, headerID int64) error {
+ _, err := tx.Exec(`INSERT INTO eth.uncle_cids (block_hash, header_id, parent_hash, cid, reward, mh_key) VALUES ($1, $2, $3, $4, $5, $6)
+ ON CONFLICT (header_id, block_hash) DO UPDATE SET (parent_hash, cid, reward, mh_key) = ($3, $4, $5, $6)`,
+ uncle.BlockHash, headerID, uncle.ParentHash, uncle.CID, uncle.Reward, uncle.MhKey)
+ return err
+}
+
+func (in *PostgresCIDWriter) upsertTransactionAndReceiptCIDs(tx *sqlx.Tx, payload shared.CIDPayload, headerID int64) error {
+ for _, trxCidMeta := range payload.TransactionCIDs {
+ var txID int64
+ err := tx.QueryRowx(`INSERT INTO eth.transaction_cids (header_id, tx_hash, cid, dst, src, index, mh_key, tx_data) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
+ ON CONFLICT (header_id, tx_hash) DO UPDATE SET (cid, dst, src, index, mh_key, tx_data) = ($3, $4, $5, $6, $7, $8)
+ RETURNING id`,
+ headerID, trxCidMeta.TxHash, trxCidMeta.CID, trxCidMeta.Dst, trxCidMeta.Src, trxCidMeta.Index, trxCidMeta.MhKey, trxCidMeta.Data).Scan(&txID)
+ if err != nil {
+ return err
+ }
+ indexerMetrics.transactions.Inc(1)
+ receiptCidMeta, ok := payload.ReceiptCIDs[common.HexToHash(trxCidMeta.TxHash)]
+ if ok {
+ if err := in.upsertReceiptCID(tx, receiptCidMeta, txID); err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+}
+
+func (in *PostgresCIDWriter) upsertTransactionCID(tx *sqlx.Tx, transaction models.TxModel, headerID int64) (int64, error) {
+ var txID int64
+ err := tx.QueryRowx(`INSERT INTO eth.transaction_cids (header_id, tx_hash, cid, dst, src, index, mh_key, tx_data) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
+ ON CONFLICT (header_id, tx_hash) DO UPDATE SET (cid, dst, src, index, mh_key, tx_data) = ($3, $4, $5, $6, $7, $8)
+ RETURNING id`,
+ headerID, transaction.TxHash, transaction.CID, transaction.Dst, transaction.Src, transaction.Index, transaction.MhKey, transaction.Data).Scan(&txID)
+ if err == nil {
+ indexerMetrics.transactions.Inc(1)
+ }
+ return txID, err
+}
+
+func (in *PostgresCIDWriter) upsertReceiptCID(tx *sqlx.Tx, rct models.ReceiptModel, txID int64) error {
+ _, err := tx.Exec(`INSERT INTO eth.receipt_cids (tx_id, cid, contract, contract_hash, topic0s, topic1s, topic2s, topic3s, log_contracts, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
+ ON CONFLICT (tx_id) DO UPDATE SET (cid, contract, contract_hash, topic0s, topic1s, topic2s, topic3s, log_contracts, mh_key) = ($2, $3, $4, $5, $6, $7, $8, $9, $10)`,
+ txID, rct.CID, rct.Contract, rct.ContractHash, rct.Topic0s, rct.Topic1s, rct.Topic2s, rct.Topic3s, rct.LogContracts, rct.MhKey)
+ if err == nil {
+ indexerMetrics.receipts.Inc(1)
+ }
+ return err
+}
+
+func (in *PostgresCIDWriter) upsertStateCID(tx *sqlx.Tx, stateNode models.StateNodeModel, headerID int64) (int64, error) {
+ var stateID int64
+ var stateKey string
+ if stateNode.StateKey != nullHash.String() {
+ stateKey = stateNode.StateKey
+ }
+ err := tx.QueryRowx(`INSERT INTO eth.state_cids (header_id, state_leaf_key, cid, state_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7)
+ ON CONFLICT (header_id, state_path) DO UPDATE SET (state_leaf_key, cid, node_type, diff, mh_key) = ($2, $3, $5, $6, $7)
+ RETURNING id`,
+ headerID, stateKey, stateNode.CID, stateNode.Path, stateNode.NodeType, true, stateNode.MhKey).Scan(&stateID)
+ return stateID, err
+}
+
+func (in *PostgresCIDWriter) upsertStateAccount(tx *sqlx.Tx, stateAccount models.StateAccountModel, stateID int64) error {
+ _, err := tx.Exec(`INSERT INTO eth.state_accounts (state_id, balance, nonce, code_hash, storage_root) VALUES ($1, $2, $3, $4, $5)
+ ON CONFLICT (state_id) DO UPDATE SET (balance, nonce, code_hash, storage_root) = ($2, $3, $4, $5)`,
+ stateID, stateAccount.Balance, stateAccount.Nonce, stateAccount.CodeHash, stateAccount.StorageRoot)
+ return err
+}
+
+func (in *PostgresCIDWriter) upsertStorageCID(tx *sqlx.Tx, storageCID models.StorageNodeModel, stateID int64) error {
+ var storageKey string
+ if storageCID.StorageKey != nullHash.String() {
+ storageKey = storageCID.StorageKey
+ }
+ _, err := tx.Exec(`INSERT INTO eth.storage_cids (state_id, storage_leaf_key, cid, storage_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7)
+ ON CONFLICT (state_id, storage_path) DO UPDATE SET (storage_leaf_key, cid, node_type, diff, mh_key) = ($2, $3, $5, $6, $7)`,
+ stateID, storageKey, storageCID.CID, storageCID.Path, storageCID.NodeType, true, storageCID.MhKey)
+ return err
+}
diff --git a/statediff/mainnet_tests/block0_rlp b/statediff/mainnet_tests/block0_rlp
new file mode 100644
index 000000000..eb912911d
Binary files /dev/null and b/statediff/mainnet_tests/block0_rlp differ
diff --git a/statediff/mainnet_tests/block1_rlp b/statediff/mainnet_tests/block1_rlp
new file mode 100644
index 000000000..d286fafde
Binary files /dev/null and b/statediff/mainnet_tests/block1_rlp differ
diff --git a/statediff/mainnet_tests/block2_rlp b/statediff/mainnet_tests/block2_rlp
new file mode 100644
index 000000000..6a8b81600
Binary files /dev/null and b/statediff/mainnet_tests/block2_rlp differ
diff --git a/statediff/mainnet_tests/block3_rlp b/statediff/mainnet_tests/block3_rlp
new file mode 100644
index 000000000..86f90a83a
Binary files /dev/null and b/statediff/mainnet_tests/block3_rlp differ
diff --git a/statediff/mainnet_tests/builder_test.go b/statediff/mainnet_tests/builder_test.go
new file mode 100644
index 000000000..6412331ff
--- /dev/null
+++ b/statediff/mainnet_tests/builder_test.go
@@ -0,0 +1,685 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package statediff_test
+
+import (
+ "bytes"
+ "io/ioutil"
+ "log"
+ "math/big"
+ "os"
+ "sort"
+ "testing"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/consensus/ethash"
+ "github.com/ethereum/go-ethereum/core"
+ "github.com/ethereum/go-ethereum/core/rawdb"
+ "github.com/ethereum/go-ethereum/core/state"
+ "github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/core/vm"
+ "github.com/ethereum/go-ethereum/crypto"
+ "github.com/ethereum/go-ethereum/ethdb"
+ "github.com/ethereum/go-ethereum/params"
+ "github.com/ethereum/go-ethereum/rlp"
+ "github.com/ethereum/go-ethereum/statediff"
+ "github.com/ethereum/go-ethereum/statediff/testhelpers"
+ sdtypes "github.com/ethereum/go-ethereum/statediff/types"
+)
+
+var (
+ db ethdb.Database
+ genesisBlock, block0, block1, block2, block3 *types.Block
+ block1CoinbaseAddr, block2CoinbaseAddr, block3CoinbaseAddr common.Address
+ block1CoinbaseHash, block2CoinbaseHash, block3CoinbaseHash common.Hash
+ builder statediff.Builder
+ emptyStorage = make([]sdtypes.StorageNode, 0)
+
+ // block 1 data
+ block1CoinbaseAccount, _ = rlp.EncodeToBytes(state.Account{
+ Nonce: 0,
+ Balance: big.NewInt(5000000000000000000),
+ CodeHash: testhelpers.NullCodeHash.Bytes(),
+ Root: testhelpers.EmptyContractRoot,
+ })
+ block1CoinbaseLeafNode, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("38251692195afc818c92b485fcb8a4691af89cbe5a2ab557b83a4261be2a9a"),
+ block1CoinbaseAccount,
+ })
+ block1CoinbaseLeafNodeHash = crypto.Keccak256(block1CoinbaseLeafNode)
+ block1x040bBranchNode, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("cc947d5ebb80600bad471f12c6ad5e4981e3525ecf8a2d982cc032536ae8b66d"),
+ common.Hex2Bytes("e80e52462e635a834e90e86ccf7673a6430384aac17004d626f4db831f0624bc"),
+ common.Hex2Bytes("59a8f11f60cb0a8488831f242da02944a26fd269d0608a44b8b873ded9e59e1b"),
+ common.Hex2Bytes("1ffb51e987e3cbd2e1dc1a64508d2e2b265477e21698b0d10fdf137f35027f40"),
+ []byte{},
+ common.Hex2Bytes("ce5077f49a13ff8199d0e77715fdd7bfd6364774effcd5499bd93cba54b3c644"),
+ common.Hex2Bytes("f5146783c048e66ce1a776ae990b4255e5fba458ece77fcb83ff6e91d6637a88"),
+ common.Hex2Bytes("6a0558b6c38852e985cf01c2156517c1c6a1e64c787a953c347825f050b236c6"),
+ common.Hex2Bytes("56b6e93958b99aaae158cc2329e71a1865ba6f39c67b096922c5cf3ed86b0ae5"),
+ []byte{},
+ common.Hex2Bytes("50d317a89a3405367d66668902f2c9f273a8d0d7d5d790dc516bca142f4a84af"),
+ common.Hex2Bytes("c72ca72750fdc1af3e6da5c7c5d82c54e4582f15b488a8aa1674058a99825dae"),
+ common.Hex2Bytes("e1a489df7b18cde818da6d38e235b026c2e61bcd3d34880b3ed0d67e0e4f0159"),
+ common.Hex2Bytes("b58d5062f2609fd2d68f00d14ab33fef2b373853877cf40bf64729e85b8fdc54"),
+ block1CoinbaseLeafNodeHash,
+ []byte{},
+ []byte{},
+ })
+ block1x040bBranchNodeHash = crypto.Keccak256(block1x040bBranchNode)
+ block1x04BranchNode, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("a9317a59365ca09cefcd384018696590afffc432e35a97e8f85aa48907bf3247"),
+ common.Hex2Bytes("e0bc229254ce7a6a736c3953e570ab18b4a7f5f2a9aa3c3057b5f17d250a1cad"),
+ common.Hex2Bytes("a2484ec8884dbe0cf24ece99d67df0d1fe78992d67cc777636a817cb2ef205aa"),
+ common.Hex2Bytes("12b78d4078c607747f06bb88bd08f839eaae0e3ac6854e5f65867d4f78abb84e"),
+ common.Hex2Bytes("359a51862df5462e4cd302f69cb338512f21eb37ce0791b9a562e72ec48b7dbf"),
+ common.Hex2Bytes("13f8d617b6a734da9235b6ac80bdd7aeaff6120c39aa223638d88f22d4ba4007"),
+ common.Hex2Bytes("02055c6400e0ec3440a8bb8fdfd7d6b6c57b7bf83e37d7e4e983d416fdd8314e"),
+ common.Hex2Bytes("4b1cca9eb3e47e805e7f4c80671a9fcd589fd6ddbe1790c3f3e177e8ede01b9e"),
+ common.Hex2Bytes("70c3815efb23b986018089e009a38e6238b8850b3efd33831913ca6fa9240249"),
+ common.Hex2Bytes("7084699d2e72a193fd75bb6108ae797b4661696eba2d631d521fc94acc7b3247"),
+ common.Hex2Bytes("b2b3cd9f1e46eb583a6185d9a96b4e80125e3d75e6191fdcf684892ef52935cb"),
+ block1x040bBranchNodeHash,
+ common.Hex2Bytes("34d9ff0fee6c929424e52268dedbc596d10786e909c5a68d6466c2aba17387ce"),
+ common.Hex2Bytes("7484d5e44b6ee6b10000708c37e035b42b818475620f9316beffc46531d1eebf"),
+ common.Hex2Bytes("30c8a283adccf2742272563cd3d6710c89ba21eac0118bf5310cfb231bcca77f"),
+ common.Hex2Bytes("4bae8558d2385b8d3bc6e6ede20bdbc5dbb0b5384c316ba8985682f88d2e506d"),
+ []byte{},
+ })
+ block1x04BranchNodeHash = crypto.Keccak256(block1x04BranchNode)
+ block1RootBranchNode, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("90dcaf88c40c7bbc95a912cbdde67c175767b31173df9ee4b0d733bfdd511c43"),
+ common.Hex2Bytes("babe369f6b12092f49181ae04ca173fb68d1a5456f18d20fa32cba73954052bd"),
+ common.Hex2Bytes("473ecf8a7e36a829e75039a3b055e51b8332cbf03324ab4af2066bbd6fbf0021"),
+ common.Hex2Bytes("bbda34753d7aa6c38e603f360244e8f59611921d9e1f128372fec0d586d4f9e0"),
+ block1x04BranchNodeHash,
+ common.Hex2Bytes("a5f3f2f7542148c973977c8a1e154c4300fec92f755f7846f1b734d3ab1d90e7"),
+ common.Hex2Bytes("e823850f50bf72baae9d1733a36a444ab65d0a6faaba404f0583ce0ca4dad92d"),
+ common.Hex2Bytes("f7a00cbe7d4b30b11faea3ae61b7f1f2b315b61d9f6bd68bfe587ad0eeceb721"),
+ common.Hex2Bytes("7117ef9fc932f1a88e908eaead8565c19b5645dc9e5b1b6e841c5edbdfd71681"),
+ common.Hex2Bytes("69eb2de283f32c11f859d7bcf93da23990d3e662935ed4d6b39ce3673ec84472"),
+ common.Hex2Bytes("203d26456312bbc4da5cd293b75b840fc5045e493d6f904d180823ec22bfed8e"),
+ common.Hex2Bytes("9287b5c21f2254af4e64fca76acc5cd87399c7f1ede818db4326c98ce2dc2208"),
+ common.Hex2Bytes("6fc2d754e304c48ce6a517753c62b1a9c1d5925b89707486d7fc08919e0a94ec"),
+ common.Hex2Bytes("7b1c54f15e299bd58bdfef9741538c7828b5d7d11a489f9c20d052b3471df475"),
+ common.Hex2Bytes("51f9dd3739a927c89e357580a4c97b40234aa01ed3d5e0390dc982a7975880a0"),
+ common.Hex2Bytes("89d613f26159af43616fd9455bb461f4869bfede26f2130835ed067a8b967bfb"),
+ []byte{},
+ })
+
+ // block 2 data
+ block2CoinbaseAccount, _ = rlp.EncodeToBytes(state.Account{
+ Nonce: 0,
+ Balance: big.NewInt(5000000000000000000),
+ CodeHash: testhelpers.NullCodeHash.Bytes(),
+ Root: testhelpers.EmptyContractRoot,
+ })
+ block2CoinbaseLeafNode, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("20679cbcf198c1741a6f4e4473845659a30caa8b26f8d37a0be2e2bc0d8892"),
+ block2CoinbaseAccount,
+ })
+ block2CoinbaseLeafNodeHash = crypto.Keccak256(block2CoinbaseLeafNode)
+ block2MovedPremineBalance, _ = new(big.Int).SetString("4000000000000000000000", 10)
+ block2MovedPremineAccount, _ = rlp.EncodeToBytes(state.Account{
+ Nonce: 0,
+ Balance: block2MovedPremineBalance,
+ CodeHash: testhelpers.NullCodeHash.Bytes(),
+ Root: testhelpers.EmptyContractRoot,
+ })
+ block2MovedPremineLeafNode, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("20f2e24db7943eab4415f99e109698863b0fecca1cf9ffc500f38cefbbe29e"),
+ block2MovedPremineAccount,
+ })
+ block2MovedPremineLeafNodeHash = crypto.Keccak256(block2MovedPremineLeafNode)
+ block2x00080dBranchNode, _ = rlp.EncodeToBytes([]interface{}{
+ block2MovedPremineLeafNodeHash,
+ []byte{},
+ []byte{},
+ []byte{},
+ block2CoinbaseLeafNodeHash,
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ })
+ block2x00080dBranchNodeHash = crypto.Keccak256(block2x00080dBranchNode)
+ block2x0008BranchNode, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("def97a26f824fc3911cf7f8c41dfc9bc93cc36ae2248de22ecae01d6950b2dc9"),
+ common.Hex2Bytes("234a575e2c5badab8de0f6515b6723195323a0562fbe1316255888637043f1c1"),
+ common.Hex2Bytes("29659740af1c23306ee8f8294c71a5632ace8c80b1eb61cfdf7022f47ff52305"),
+ common.Hex2Bytes("cf2681d23bb666d89dec8123bce9e626240a7e2ce7a1e8316b1ee88181c9471c"),
+ common.Hex2Bytes("18d8de6967fe34b9fd411c74fecc45f8a737961791e70d8ece967bb07cf4d4dc"),
+ common.Hex2Bytes("7cad60c7cbca8c79c2db5a8fc1baa9381484d43d6c37dfb97718c3a109d47dfc"),
+ common.Hex2Bytes("2138f5a9062b750b6320e5fac5b134da90a9edbda06ef3e1ae64fb1366ca998c"),
+ common.Hex2Bytes("532826502a9661fcae7c0f5d2a4c8cb287dfc521e828349543c5a461a9d591ed"),
+ common.Hex2Bytes("30543537413dd086d4b1560f46b90e8da0f43de5584a138ab036d74e84657523"),
+ common.Hex2Bytes("c98042928af640bfa1142aca895cd76e146332dce94ddad3426e74ed519ca1e0"),
+ common.Hex2Bytes("43de3e62cc3148193899d018dff813c04c5b636ce95bd7e828416204292d9ff9"),
+ []byte{},
+ common.Hex2Bytes("78d533b9182bb42f6c16e9ebd5734f0d280179ba1c9b6316c2c1df73f7dd8a54"),
+ block2x00080dBranchNodeHash,
+ common.Hex2Bytes("934b736b57a892aaa15a03c7e37746bb096313727135f9841cb64c263785cf81"),
+ common.Hex2Bytes("38ce97150e90dfd7258901a0ddee72d8e30760a3d0419dbb80135c66588739a2"),
+ []byte{},
+ })
+ block2x0008BranchNodeHash = crypto.Keccak256(block2x0008BranchNode)
+ block2x00BranchNode, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("e45a9e85cab1b6eb18b30df2c6acc448bbac6a30d81646823b31223e16e5063e"),
+ common.Hex2Bytes("33bd7171d556b981f6849064eb09412b24fedc0812127db936067043f53db1b9"),
+ common.Hex2Bytes("ca56945f074da4f15587404593faf3a50d17ea0e21a418ad6ec99bdf4bf3f914"),
+ common.Hex2Bytes("da23e9004f782df128eea1adff77952dc85f91b7f7ca4893aac5f21d24c3a1c9"),
+ common.Hex2Bytes("ba5ec61fa780ee02af19db99677c37560fc4f0df5c278d9dfa2837f30f72bc6b"),
+ common.Hex2Bytes("8310ad91625c2e3429a74066b7e2e0c958325e4e7fa3ec486b73b7c8300cfef7"),
+ common.Hex2Bytes("732e5c103bf4d5adfef83773026809d9405539b67e93293a02342e83ad2fb766"),
+ common.Hex2Bytes("30d14ff0c2aab57d1fbaf498ab14519b4e9d94f149a3dc15f0eec5adf8df25e1"),
+ block2x0008BranchNodeHash,
+ common.Hex2Bytes("5a43bd92e55aa78df60e70b6b53b6366c4080fd6a5bdd7b533b46aff4a75f6f2"),
+ common.Hex2Bytes("a0c410aa59efe416b1213166fab680ce330bd46c3ebf877ff14609ee6a383600"),
+ common.Hex2Bytes("2f41e918786e557293068b1eda9b3f9f86ed4e65a6a5363ee3262109f6e08b17"),
+ common.Hex2Bytes("01f42a40f02f6f24bb97b09c4d3934e8b03be7cfbb902acc1c8fd67a7a5abace"),
+ common.Hex2Bytes("0acbdce2787a6ea177209bd13bfc9d0779d7e2b5249e0211a2974164e14312f5"),
+ common.Hex2Bytes("dadbe113e4132e0c0c3cd4867e0a2044d0e5a3d44b350677ed42fc9244d004d4"),
+ common.Hex2Bytes("aa7441fefc17d76aedfcaf692fe71014b94c1547b6d129562b34fc5995ca0d1a"),
+ []byte{},
+ })
+ block2x00BranchNodeHash = crypto.Keccak256(block2x00BranchNode)
+ block2RootBranchNode, _ = rlp.EncodeToBytes([]interface{}{
+ block2x00BranchNodeHash,
+ common.Hex2Bytes("babe369f6b12092f49181ae04ca173fb68d1a5456f18d20fa32cba73954052bd"),
+ common.Hex2Bytes("473ecf8a7e36a829e75039a3b055e51b8332cbf03324ab4af2066bbd6fbf0021"),
+ common.Hex2Bytes("bbda34753d7aa6c38e603f360244e8f59611921d9e1f128372fec0d586d4f9e0"),
+ block1x04BranchNodeHash,
+ common.Hex2Bytes("a5f3f2f7542148c973977c8a1e154c4300fec92f755f7846f1b734d3ab1d90e7"),
+ common.Hex2Bytes("e823850f50bf72baae9d1733a36a444ab65d0a6faaba404f0583ce0ca4dad92d"),
+ common.Hex2Bytes("f7a00cbe7d4b30b11faea3ae61b7f1f2b315b61d9f6bd68bfe587ad0eeceb721"),
+ common.Hex2Bytes("7117ef9fc932f1a88e908eaead8565c19b5645dc9e5b1b6e841c5edbdfd71681"),
+ common.Hex2Bytes("69eb2de283f32c11f859d7bcf93da23990d3e662935ed4d6b39ce3673ec84472"),
+ common.Hex2Bytes("203d26456312bbc4da5cd293b75b840fc5045e493d6f904d180823ec22bfed8e"),
+ common.Hex2Bytes("9287b5c21f2254af4e64fca76acc5cd87399c7f1ede818db4326c98ce2dc2208"),
+ common.Hex2Bytes("6fc2d754e304c48ce6a517753c62b1a9c1d5925b89707486d7fc08919e0a94ec"),
+ common.Hex2Bytes("7b1c54f15e299bd58bdfef9741538c7828b5d7d11a489f9c20d052b3471df475"),
+ common.Hex2Bytes("51f9dd3739a927c89e357580a4c97b40234aa01ed3d5e0390dc982a7975880a0"),
+ common.Hex2Bytes("89d613f26159af43616fd9455bb461f4869bfede26f2130835ed067a8b967bfb"),
+ []byte{},
+ })
+
+ // block3 data
+ // path 060e0f
+ blcok3CoinbaseBalance, _ = new(big.Int).SetString("5156250000000000000", 10)
+ block3CoinbaseAccount, _ = rlp.EncodeToBytes(state.Account{
+ Nonce: 0,
+ Balance: blcok3CoinbaseBalance,
+ CodeHash: testhelpers.NullCodeHash.Bytes(),
+ Root: testhelpers.EmptyContractRoot,
+ })
+ block3CoinbaseLeafNode, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("3a174f00e64521a535f35e67c1aa241951c791639b2f3d060f49c5d9fa8b9e"),
+ block3CoinbaseAccount,
+ })
+ block3CoinbaseLeafNodeHash = crypto.Keccak256(block3CoinbaseLeafNode)
+ // path 0c0e050703
+ block3MovedPremineBalance1, _ = new(big.Int).SetString("3750000000000000000", 10)
+ block3MovedPremineAccount1, _ = rlp.EncodeToBytes(state.Account{
+ Nonce: 0,
+ Balance: block3MovedPremineBalance1,
+ CodeHash: testhelpers.NullCodeHash.Bytes(),
+ Root: testhelpers.EmptyContractRoot,
+ })
+ block3MovedPremineLeafNode1, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("3ced93917e658d10e2d9009470dad72b63c898d173721194a12f2ae5e190"), // ce573ced93917e658d10e2d9009470dad72b63c898d173721194a12f2ae5e190
+ block3MovedPremineAccount1,
+ })
+ block3MovedPremineLeafNodeHash1 = crypto.Keccak256(block3MovedPremineLeafNode1)
+ // path 0c0e050708
+ block3MovedPremineBalance2, _ = new(big.Int).SetString("1999944000000000000000", 10)
+ block3MovedPremineAccount2, _ = rlp.EncodeToBytes(state.Account{
+ Nonce: 0,
+ Balance: block3MovedPremineBalance2,
+ CodeHash: testhelpers.NullCodeHash.Bytes(),
+ Root: testhelpers.EmptyContractRoot,
+ })
+ block3MovedPremineLeafNode2, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("33bc1e69eedf90f402e11f6862da14ed8e50156635a04d6393bbae154012"), // ce5783bc1e69eedf90f402e11f6862da14ed8e50156635a04d6393bbae154012
+ block3MovedPremineAccount2,
+ })
+ block3MovedPremineLeafNodeHash2 = crypto.Keccak256(block3MovedPremineLeafNode2)
+
+ block3x0c0e0507BranchNode, _ = rlp.EncodeToBytes([]interface{}{
+ []byte{},
+ []byte{},
+ []byte{},
+ block3MovedPremineLeafNodeHash1,
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ block3MovedPremineLeafNodeHash2,
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ })
+ block3x0c0e0507BranchNodeHash = crypto.Keccak256(block3x0c0e0507BranchNode)
+
+ block3x0c0e05BranchNode, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("452e3beb503b1d87ae7c672b98a8e3fd043a671405502562ae1043dc97151a50"),
+ []byte{},
+ common.Hex2Bytes("2f5bb16f77086f67ce8c4258cb9061cb299e597b2ad4ad6d7ccc474d6d88e85e"),
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ block3x0c0e0507BranchNodeHash,
+ []byte{},
+ common.Hex2Bytes("44623e5a9319f83870db0ea4611a25fca1e1da3eeea2be4a091dfc15ab45689e"),
+ common.Hex2Bytes("b41e047a97f44fa4cb8146467b88c8f4705811029d9e170abb0aba7d0af9f0da"),
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ []byte{},
+ })
+ block3x0c0e05BranchNodeHash = crypto.Keccak256(block3x0c0e05BranchNode)
+
+ block3x060eBranchNode, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("94d77c7c30b88829c9989948b206cda5e532b38b49534261c517aebf4a3e6fdb"),
+ common.Hex2Bytes("a5cf57a50da8204964e834a12a53f9bed7afc9b700a4a81b440122d60c7603a7"),
+ []byte{},
+ common.Hex2Bytes("3730ec0571f34b6c3b178dc26ccb31a3f50c29da9b1921e41b9477ddab41b0fe"),
+ []byte{},
+ common.Hex2Bytes("543952bb9566c2018cf8d7b90d6a7903cdfff3d79ac36189be5322de42fc3fc0"),
+ []byte{},
+ common.Hex2Bytes("c4a49b66f0bcc08531e50cdea5577a281d111fa542eaefd9a9aead8febb0735e"),
+ common.Hex2Bytes("362ad58916c71463b98c079649fc486c5f082c4f548bd4ab501515f0c5641cb4"),
+ common.Hex2Bytes("36aae109f6f55f0bd05eb05bb365af2332dfe5f06d3d17903e88534c319eb709"),
+ common.Hex2Bytes("430dcfc5cc49a6b490dd54138920e8f94e427239c2bccc14705cfd4ff6cc4383"),
+ common.Hex2Bytes("73ed77563dfed2fdb38900b474db88b2270f449167e0d877fda9e2229f119fe8"),
+ common.Hex2Bytes("5dfe06013f2a41f1779194ceb07769d019f518b2a694a82fa1661e60fd973eaa"),
+ common.Hex2Bytes("80bdfd85fbb6b45850bad6e34136aaa1b04711e47469fa2f0d19eca52089efb5"),
+ []byte{},
+ block3CoinbaseLeafNodeHash,
+ []byte{},
+ })
+ block3x060eBranchNodeHash = crypto.Keccak256(block3x060eBranchNode)
+
+ block3x0c0eBranchNode, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("70647f11b2b995d718f9e8aceb44c8839e0055641930d216fa6090280a9d63d5"),
+ common.Hex2Bytes("fdfb17cd2fba2a14219981cb7886a1977cd85dbef5c767c562f4a5f547febff0"),
+ common.Hex2Bytes("ff87313253ec6f860142b7bf62efb4cb07ea668c57aa90cbe9ef22b72fee15c7"),
+ common.Hex2Bytes("3a77b3c26a54ad37bdf4e19c1bce93493ec0f79d9ad90190b70bc840b54918e1"),
+ common.Hex2Bytes("af1b3b14324561b68f2e24dbcc28673ab35ce3fd0230fe2bc86b3d1931745195"),
+ block3x0c0e05BranchNodeHash,
+ common.Hex2Bytes("647dcbfe6aabcd9d219ff40422af4326bfc1ec66703195a78eb48618ddef248d"),
+ common.Hex2Bytes("2d2bf06159cc8928283c3419a03f08ea34c493a9d002a0ec76d5c429508ccaf4"),
+ common.Hex2Bytes("d7147251b3f48f25e1e4c6d8f83a00b1eca66e99a4ea0d238942ce72d0ba6414"),
+ common.Hex2Bytes("cb859370869967594fb29f4e2904413310146733d7fcbd11407f3e47626e0e34"),
+ common.Hex2Bytes("b93ab9b0bd83963860fbe0b7d543879cfde756ea1618d2a40d85483058cc5a26"),
+ common.Hex2Bytes("45aee096499d209931457ce251c5c7e5543f22524f67785ff8f0f3f02588b0ed"),
+ []byte{},
+ common.Hex2Bytes("aa2ae9379797c5066bba646108074ae8677e82c923d584b6d1c1268ca3708c5c"),
+ common.Hex2Bytes("e6eb055f0d8e194c083471479a3de87fa0f90c0f4aaa518416ec1e469ec32e3a"),
+ common.Hex2Bytes("0cc9c50fc7eba162fb17f2e04e3599c13abbf210d9781864d0edec401ecaebba"),
+ []byte{},
+ })
+ block3x0c0eBranchNodeHash = crypto.Keccak256(block3x0c0eBranchNode)
+
+ block3x06BranchNode, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("68f7ff8c074d6e4cccd55b5b1c2116a6dd7047d4332090e6db8839362991b0ae"),
+ common.Hex2Bytes("c446eb4377c750701374c56e50759e6ba68b7adf4d543e718c8b28a99ae3b6ad"),
+ common.Hex2Bytes("ef2c49ec64cb65eae0d99684e74c8af2bd0206c9a0214d9d3eddf0881dd8412a"),
+ common.Hex2Bytes("7096c4cc7e8125f0b142d8644ad681f8a8142e210c806f33f3f7004f0e9d6002"),
+ common.Hex2Bytes("bc9a8ae647b234cd6607b6b0245e3b3d5ec4f7ea006e7eda1f92d02f0ea91116"),
+ common.Hex2Bytes("a87720deb92ff2f899e809befab9970a61c86148c4fa09d04b77505ee4a5bda5"),
+ common.Hex2Bytes("2460e5b6ded7c0001de29c15db124614432fef6486370cc9970f63b0d95fd5e2"),
+ common.Hex2Bytes("ed1c447d4a32bc31e9e32259dc63da10df91231e786332e3df122b301b1f8fc3"),
+ common.Hex2Bytes("0d27dfc201d995c2323b792860dbca087da7cc56d1698c39b7c4b9277729c5ca"),
+ common.Hex2Bytes("f6d2be168d9c17643c9ea80c29322b364604cdfd36eef40123d83fad364e43fa"),
+ common.Hex2Bytes("004bf1c30a5730f464de1a0ba4ac5b5618df66d6106073d08742166e33a7eeb5"),
+ common.Hex2Bytes("7298d019a57a1b04ac31ed874d654ba0d3c249704c5d9efa1d08959fc89e0779"),
+ common.Hex2Bytes("fb3d50b7af6f839e371ff8ebd0322e94e6b6fb7888416737f88cf55bcf5859ec"),
+ common.Hex2Bytes("4e7a2618fa1fc560a73c24839657adf7e48d600ecfb12333678115936597a913"),
+ block3x060eBranchNodeHash,
+ common.Hex2Bytes("1909706c5db040f54c19f4050659ad484982145b02474653917de379f15ebb36"),
+ []byte{},
+ })
+ block3x06BranchNodeHash = crypto.Keccak256(block3x06BranchNode)
+
+ block3x0cBranchNode, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("dae48f5b47930c28bb116fbd55e52cd47242c71bf55373b55eb2805ee2e4a929"),
+ common.Hex2Bytes("0f1f37f337ec800e2e5974e2e7355f10f1a4832b39b846d916c3597a460e0676"),
+ common.Hex2Bytes("da8f627bb8fbeead17b318e0a8e4f528db310f591bb6ab2deda4a9f7ca902ab5"),
+ common.Hex2Bytes("971c662648d58295d0d0aa4b8055588da0037619951217c22052802549d94a2f"),
+ common.Hex2Bytes("ccc701efe4b3413fd6a61a6c9f40e955af774649a8d9fd212d046a5a39ddbb67"),
+ common.Hex2Bytes("d607cdb32e2bd635ee7f2f9e07bc94ddbd09b10ec0901b66628e15667aec570b"),
+ common.Hex2Bytes("5b89203dc940e6fa70ec19ad4e01d01849d3a5baa0a8f9c0525256ed490b159f"),
+ common.Hex2Bytes("b84227d48df68aecc772939a59afa9e1a4ab578f7b698bdb1289e29b6044668e"),
+ common.Hex2Bytes("fd1c992070b94ace57e48cbf6511a16aa770c645f9f5efba87bbe59d0a042913"),
+ common.Hex2Bytes("e16a7ccea6748ae90de92f8aef3b3dc248a557b9ac4e296934313f24f7fced5f"),
+ common.Hex2Bytes("42373cf4a00630d94de90d0a23b8f38ced6b0f7cb818b8925fee8f0c2a28a25a"),
+ common.Hex2Bytes("5f89d2161c1741ff428864f7889866484cef622de5023a46e795dfdec336319f"),
+ common.Hex2Bytes("7597a017664526c8c795ce1da27b8b72455c49657113e0455552dbc068c5ba31"),
+ common.Hex2Bytes("d5be9089012fda2c585a1b961e988ea5efcd3a06988e150a8682091f694b37c5"),
+ block3x0c0eBranchNodeHash,
+ common.Hex2Bytes("49bf6e8df0acafd0eff86defeeb305568e44d52d2235cf340ae15c6034e2b241"),
+ []byte{},
+ })
+ block3x0cBranchNodeHash = crypto.Keccak256(block3x0cBranchNode)
+
+ block3RootBranchNode, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("f646da473c426e79f1c796b00d4873f47de1dbe1c9d19d63993a05eeb8b4041d"),
+ common.Hex2Bytes("babe369f6b12092f49181ae04ca173fb68d1a5456f18d20fa32cba73954052bd"),
+ common.Hex2Bytes("473ecf8a7e36a829e75039a3b055e51b8332cbf03324ab4af2066bbd6fbf0021"),
+ common.Hex2Bytes("bbda34753d7aa6c38e603f360244e8f59611921d9e1f128372fec0d586d4f9e0"),
+ common.Hex2Bytes("d9cff5d5f2418afd16a4da5c221fdc8bd47520c5927922f69a68177b64da6ac0"),
+ common.Hex2Bytes("a5f3f2f7542148c973977c8a1e154c4300fec92f755f7846f1b734d3ab1d90e7"),
+ block3x06BranchNodeHash,
+ common.Hex2Bytes("f7a00cbe7d4b30b11faea3ae61b7f1f2b315b61d9f6bd68bfe587ad0eeceb721"),
+ common.Hex2Bytes("7117ef9fc932f1a88e908eaead8565c19b5645dc9e5b1b6e841c5edbdfd71681"),
+ common.Hex2Bytes("69eb2de283f32c11f859d7bcf93da23990d3e662935ed4d6b39ce3673ec84472"),
+ common.Hex2Bytes("203d26456312bbc4da5cd293b75b840fc5045e493d6f904d180823ec22bfed8e"),
+ common.Hex2Bytes("9287b5c21f2254af4e64fca76acc5cd87399c7f1ede818db4326c98ce2dc2208"),
+ block3x0cBranchNodeHash,
+ common.Hex2Bytes("7b1c54f15e299bd58bdfef9741538c7828b5d7d11a489f9c20d052b3471df475"),
+ common.Hex2Bytes("51f9dd3739a927c89e357580a4c97b40234aa01ed3d5e0390dc982a7975880a0"),
+ common.Hex2Bytes("89d613f26159af43616fd9455bb461f4869bfede26f2130835ed067a8b967bfb"),
+ []byte{},
+ })
+)
+
+func init() {
+ db = rawdb.NewMemoryDatabase()
+ genesisBlock = core.DefaultGenesisBlock().MustCommit(db)
+ genBy, err := rlp.EncodeToBytes(genesisBlock)
+ if err != nil {
+ log.Fatal(err)
+ }
+ var block0RLP []byte
+ block0, block0RLP, err = loadBlockFromRLPFile("./block0_rlp")
+ if err != nil {
+ log.Fatal(err)
+ }
+ if !bytes.Equal(genBy, block0RLP) {
+ log.Fatal("mainnet genesis blocks do not match")
+ }
+ block1, _, err = loadBlockFromRLPFile("./block1_rlp")
+ if err != nil {
+ log.Fatal(err)
+ }
+ block1CoinbaseAddr = block1.Coinbase()
+ block1CoinbaseHash = crypto.Keccak256Hash(block1CoinbaseAddr.Bytes())
+ block2, _, err = loadBlockFromRLPFile("./block2_rlp")
+ if err != nil {
+ log.Fatal(err)
+ }
+ block2CoinbaseAddr = block2.Coinbase()
+ block2CoinbaseHash = crypto.Keccak256Hash(block2CoinbaseAddr.Bytes())
+ block3, _, err = loadBlockFromRLPFile("./block3_rlp")
+ if err != nil {
+ log.Fatal(err)
+ }
+ block3CoinbaseAddr = block3.Coinbase()
+ block3CoinbaseHash = crypto.Keccak256Hash(block3CoinbaseAddr.Bytes())
+}
+
+func loadBlockFromRLPFile(filename string) (*types.Block, []byte, error) {
+ f, err := os.Open(filename)
+ if err != nil {
+ return nil, nil, err
+ }
+ defer f.Close()
+ blockRLP, err := ioutil.ReadAll(f)
+ if err != nil {
+ return nil, nil, err
+ }
+ block := new(types.Block)
+ return block, blockRLP, rlp.DecodeBytes(blockRLP, block)
+}
+
+func TestBuilderOnMainnetBlocks(t *testing.T) {
+ chain, _ := core.NewBlockChain(db, nil, params.MainnetChainConfig, ethash.NewFaker(), vm.Config{}, nil)
+ _, err := chain.InsertChain([]*types.Block{block1, block2, block3})
+ if err != nil {
+ t.Error(err)
+ }
+ params := statediff.Params{
+ IntermediateStateNodes: true,
+ }
+ builder = statediff.NewBuilder(chain.StateCache())
+
+ var tests = []struct {
+ name string
+ startingArguments statediff.Args
+ expected *statediff.StateObject
+ }{
+ // note that block0 (genesis) has over 1000 nodes due to the pre-allocation for the crowd-sale
+ // it is not feasible to write a unit test of that size at this time
+ {
+ "testBlock1",
+ //10000 transferred from testBankAddress to account1Addr
+ statediff.Args{
+ OldStateRoot: block0.Root(),
+ NewStateRoot: block1.Root(),
+ BlockNumber: block1.Number(),
+ BlockHash: block1.Hash(),
+ },
+ &statediff.StateObject{
+ BlockNumber: block1.Number(),
+ BlockHash: block1.Hash(),
+ Nodes: []sdtypes.StateNode{
+ {
+ Path: []byte{},
+ NodeType: sdtypes.Branch,
+ StorageNodes: emptyStorage,
+ NodeValue: block1RootBranchNode,
+ },
+ {
+ Path: []byte{'\x04'},
+ NodeType: sdtypes.Branch,
+ StorageNodes: emptyStorage,
+ NodeValue: block1x04BranchNode,
+ },
+ {
+ Path: []byte{'\x04', '\x0b'},
+ NodeType: sdtypes.Branch,
+ StorageNodes: emptyStorage,
+ NodeValue: block1x040bBranchNode,
+ },
+ {
+ Path: []byte{'\x04', '\x0b', '\x0e'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: block1CoinbaseHash.Bytes(),
+ NodeValue: block1CoinbaseLeafNode,
+ StorageNodes: emptyStorage,
+ },
+ },
+ },
+ },
+ {
+ "testBlock2",
+ // 1000 transferred from testBankAddress to account1Addr
+ // 1000 transferred from account1Addr to account2Addr
+ // account1addr creates a new contract
+ statediff.Args{
+ OldStateRoot: block1.Root(),
+ NewStateRoot: block2.Root(),
+ BlockNumber: block2.Number(),
+ BlockHash: block2.Hash(),
+ },
+ &statediff.StateObject{
+ BlockNumber: block2.Number(),
+ BlockHash: block2.Hash(),
+ Nodes: []sdtypes.StateNode{
+ {
+ Path: []byte{},
+ NodeType: sdtypes.Branch,
+ StorageNodes: emptyStorage,
+ NodeValue: block2RootBranchNode,
+ },
+ {
+ Path: []byte{'\x00'},
+ NodeType: sdtypes.Branch,
+ StorageNodes: emptyStorage,
+ NodeValue: block2x00BranchNode,
+ },
+ {
+ Path: []byte{'\x00', '\x08'},
+ NodeType: sdtypes.Branch,
+ StorageNodes: emptyStorage,
+ NodeValue: block2x0008BranchNode,
+ },
+ {
+ Path: []byte{'\x00', '\x08', '\x0d'},
+ NodeType: sdtypes.Branch,
+ StorageNodes: emptyStorage,
+ NodeValue: block2x00080dBranchNode,
+ },
+ // this new leaf at x00 x08 x0d x00 was "created" when a premine account (leaf) was moved from path x00 x08 x0d
+ // this occurred because of the creation of the new coinbase receiving account (leaf) at x00 x08 x0d x04
+ // which necessitates we create a branch at x00 x08 x0d (as shown in the below UpdateAccounts)
+ {
+ Path: []byte{'\x00', '\x08', '\x0d', '\x00'},
+ NodeType: sdtypes.Leaf,
+ StorageNodes: emptyStorage,
+ LeafKey: common.HexToHash("08d0f2e24db7943eab4415f99e109698863b0fecca1cf9ffc500f38cefbbe29e").Bytes(),
+ NodeValue: block2MovedPremineLeafNode,
+ },
+ {
+ Path: []byte{'\x00', '\x08', '\x0d', '\x04'},
+ NodeType: sdtypes.Leaf,
+ StorageNodes: emptyStorage,
+ LeafKey: block2CoinbaseHash.Bytes(),
+ NodeValue: block2CoinbaseLeafNode,
+ },
+ },
+ },
+ },
+ {
+ "testBlock3",
+ //the contract's storage is changed
+ //and the block is mined by account 2
+ statediff.Args{
+ OldStateRoot: block2.Root(),
+ NewStateRoot: block3.Root(),
+ BlockNumber: block3.Number(),
+ BlockHash: block3.Hash(),
+ },
+ &statediff.StateObject{
+ BlockNumber: block3.Number(),
+ BlockHash: block3.Hash(),
+ Nodes: []sdtypes.StateNode{
+ {
+ Path: []byte{},
+ NodeType: sdtypes.Branch,
+ StorageNodes: emptyStorage,
+ NodeValue: block3RootBranchNode,
+ },
+ {
+ Path: []byte{'\x06'},
+ NodeType: sdtypes.Branch,
+ StorageNodes: emptyStorage,
+ NodeValue: block3x06BranchNode,
+ },
+ {
+ Path: []byte{'\x06', '\x0e'},
+ NodeType: sdtypes.Branch,
+ StorageNodes: emptyStorage,
+ NodeValue: block3x060eBranchNode,
+ },
+ {
+ Path: []byte{'\x0c'},
+ NodeType: sdtypes.Branch,
+ StorageNodes: emptyStorage,
+ NodeValue: block3x0cBranchNode,
+ },
+ {
+ Path: []byte{'\x0c', '\x0e'},
+ NodeType: sdtypes.Branch,
+ StorageNodes: emptyStorage,
+ NodeValue: block3x0c0eBranchNode,
+ },
+ {
+ Path: []byte{'\x0c', '\x0e', '\x05'},
+ NodeType: sdtypes.Branch,
+ StorageNodes: emptyStorage,
+ NodeValue: block3x0c0e05BranchNode,
+ },
+ {
+ Path: []byte{'\x0c', '\x0e', '\x05', '\x07'},
+ NodeType: sdtypes.Branch,
+ StorageNodes: emptyStorage,
+ NodeValue: block3x0c0e0507BranchNode,
+ },
+ { // How was this account created???
+ Path: []byte{'\x0c', '\x0e', '\x05', '\x07', '\x03'},
+ NodeType: sdtypes.Leaf,
+ StorageNodes: emptyStorage,
+ LeafKey: common.HexToHash("ce573ced93917e658d10e2d9009470dad72b63c898d173721194a12f2ae5e190").Bytes(),
+ NodeValue: block3MovedPremineLeafNode1,
+ },
+ { // This account (leaf) used to be at 0c 0e 05 07, likely moves because of the new account above
+ Path: []byte{'\x0c', '\x0e', '\x05', '\x07', '\x08'},
+ NodeType: sdtypes.Leaf,
+ StorageNodes: emptyStorage,
+ LeafKey: common.HexToHash("ce5783bc1e69eedf90f402e11f6862da14ed8e50156635a04d6393bbae154012").Bytes(),
+ NodeValue: block3MovedPremineLeafNode2,
+ },
+ { // this is the new account created due to the coinbase mining a block, it's creation shouldn't affect 0x 0e 05 07
+ Path: []byte{'\x06', '\x0e', '\x0f'},
+ NodeType: sdtypes.Leaf,
+ StorageNodes: emptyStorage,
+ LeafKey: block3CoinbaseHash.Bytes(),
+ NodeValue: block3CoinbaseLeafNode,
+ },
+ },
+ },
+ },
+ }
+
+ for _, test := range tests {
+ diff, err := builder.BuildStateDiffObject(test.startingArguments, params)
+ if err != nil {
+ t.Error(err)
+ }
+ receivedStateDiffRlp, err := rlp.EncodeToBytes(diff)
+ if err != nil {
+ t.Error(err)
+ }
+ expectedStateDiffRlp, err := rlp.EncodeToBytes(test.expected)
+ if err != nil {
+ t.Error(err)
+ }
+ sort.Slice(receivedStateDiffRlp, func(i, j int) bool { return receivedStateDiffRlp[i] < receivedStateDiffRlp[j] })
+ sort.Slice(expectedStateDiffRlp, func(i, j int) bool { return expectedStateDiffRlp[i] < expectedStateDiffRlp[j] })
+ if !bytes.Equal(receivedStateDiffRlp, expectedStateDiffRlp) {
+ t.Logf("Test failed: %s", test.name)
+ t.Errorf("actual state diff: %+v\nexpected state diff: %+v", diff, test.expected)
+ }
+ }
+}
diff --git a/statediff/metrics.go b/statediff/metrics.go
new file mode 100644
index 000000000..7e7d6e328
--- /dev/null
+++ b/statediff/metrics.go
@@ -0,0 +1,54 @@
+package statediff
+
+import (
+ "strings"
+
+ "github.com/ethereum/go-ethereum/metrics"
+)
+
+const (
+ namespace = "statediff"
+)
+
+// Build a fully qualified metric name
+func metricName(subsystem, name string) string {
+ if name == "" {
+ return ""
+ }
+ parts := []string{namespace, name}
+ if subsystem != "" {
+ parts = []string{namespace, subsystem, name}
+ }
+ // Prometheus uses _ but geth metrics uses / and replaces
+ return strings.Join(parts, "/")
+}
+
+type statediffMetricsHandles struct {
+ // Height of latest synced by core.BlockChain
+ // FIXME
+ lastSyncHeight metrics.Gauge
+ // Height of the latest block received from chainEvent channel
+ lastEventHeight metrics.Gauge
+ // Height of latest state diff
+ lastStatediffHeight metrics.Gauge
+ // Current length of chainEvent channels
+ serviceLoopChannelLen metrics.Gauge
+ writeLoopChannelLen metrics.Gauge
+}
+
+func RegisterStatediffMetrics(reg metrics.Registry) statediffMetricsHandles {
+ ctx := statediffMetricsHandles{
+ lastSyncHeight: metrics.NewGauge(),
+ lastEventHeight: metrics.NewGauge(),
+ lastStatediffHeight: metrics.NewGauge(),
+ serviceLoopChannelLen: metrics.NewGauge(),
+ writeLoopChannelLen: metrics.NewGauge(),
+ }
+ subsys := "service"
+ reg.Register(metricName(subsys, "last_sync_height"), ctx.lastSyncHeight)
+ reg.Register(metricName(subsys, "last_event_height"), ctx.lastEventHeight)
+ reg.Register(metricName(subsys, "last_statediff_height"), ctx.lastStatediffHeight)
+ reg.Register(metricName(subsys, "service_loop_channel_len"), ctx.serviceLoopChannelLen)
+ reg.Register(metricName(subsys, "write_loop_channel_len"), ctx.writeLoopChannelLen)
+ return ctx
+}
diff --git a/statediff/service.go b/statediff/service.go
new file mode 100644
index 000000000..2f16fc4ab
--- /dev/null
+++ b/statediff/service.go
@@ -0,0 +1,627 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package statediff
+
+import (
+ "bytes"
+ "math/big"
+ "strconv"
+ "sync"
+ "sync/atomic"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/core"
+ "github.com/ethereum/go-ethereum/core/state"
+ "github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/crypto"
+ "github.com/ethereum/go-ethereum/eth"
+ "github.com/ethereum/go-ethereum/event"
+ "github.com/ethereum/go-ethereum/log"
+ "github.com/ethereum/go-ethereum/metrics"
+ "github.com/ethereum/go-ethereum/node"
+ "github.com/ethereum/go-ethereum/p2p"
+ "github.com/ethereum/go-ethereum/rlp"
+ "github.com/ethereum/go-ethereum/rpc"
+ "github.com/ethereum/go-ethereum/trie"
+
+ ind "github.com/ethereum/go-ethereum/statediff/indexer"
+ nodeinfo "github.com/ethereum/go-ethereum/statediff/indexer/node"
+ "github.com/ethereum/go-ethereum/statediff/indexer/postgres"
+ . "github.com/ethereum/go-ethereum/statediff/types"
+)
+
+const chainEventChanSize = 20000
+
+var writeLoopParams = Params{
+ IntermediateStateNodes: true,
+ IntermediateStorageNodes: true,
+ IncludeBlock: true,
+ IncludeReceipts: true,
+ IncludeTD: true,
+ IncludeCode: true,
+}
+
+var statediffMetrics = RegisterStatediffMetrics(metrics.DefaultRegistry)
+
+type blockChain interface {
+ SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription
+ GetBlockByHash(hash common.Hash) *types.Block
+ GetBlockByNumber(number uint64) *types.Block
+ GetReceiptsByHash(hash common.Hash) types.Receipts
+ GetTdByHash(hash common.Hash) *big.Int
+ UnlockTrie(root common.Hash)
+ StateCache() state.Database
+}
+
+// IService is the state-diffing service interface
+type IService interface {
+ // Start() and Stop()
+ node.Lifecycle
+ // Method to getting API(s) for this service
+ APIs() []rpc.API
+ // Main event loop for processing state diffs
+ Loop(chainEventCh chan core.ChainEvent)
+ // Method to subscribe to receive state diff processing output
+ Subscribe(id rpc.ID, sub chan<- Payload, quitChan chan<- bool, params Params)
+ // Method to unsubscribe from state diff processing
+ Unsubscribe(id rpc.ID) error
+ // Method to get state diff object at specific block
+ StateDiffAt(blockNumber uint64, params Params) (*Payload, error)
+ // Method to get state trie object at specific block
+ StateTrieAt(blockNumber uint64, params Params) (*Payload, error)
+ // Method to stream out all code and codehash pairs
+ StreamCodeAndCodeHash(blockNumber uint64, outChan chan<- CodeAndCodeHash, quitChan chan<- bool)
+ // Method to write state diff object directly to DB
+ WriteStateDiffAt(blockNumber uint64, params Params) error
+ // Event loop for progressively processing and writing diffs directly to DB
+ WriteLoop(chainEventCh chan core.ChainEvent)
+}
+
+// Wraps consructor parameters
+type ServiceParams struct {
+ DBParams *DBParams
+ // Whether to enable writing state diffs directly to track blochain head
+ EnableWriteLoop bool
+ // Size of the worker pool
+ NumWorkers uint
+}
+
+// Service is the underlying struct for the state diffing service
+type Service struct {
+ // Used to sync access to the Subscriptions
+ sync.Mutex
+ // Used to build the state diff objects
+ Builder Builder
+ // Used to subscribe to chain events (blocks)
+ BlockChain blockChain
+ // Used to signal shutdown of the service
+ QuitChan chan bool
+ // A mapping of rpc.IDs to their subscription channels, mapped to their subscription type (hash of the Params rlp)
+ Subscriptions map[common.Hash]map[rpc.ID]Subscription
+ // A mapping of subscription params rlp hash to the corresponding subscription params
+ SubscriptionTypes map[common.Hash]Params
+ // Cache the last block so that we can avoid having to lookup the next block's parent
+ BlockCache blockCache
+ // Whether or not we have any subscribers; only if we do, do we processes state diffs
+ subscribers int32
+ // Interface for publishing statediffs as PG-IPLD objects
+ indexer ind.Indexer
+ // Whether to enable writing state diffs directly to track blochain head
+ enableWriteLoop bool
+ // Size of the worker pool
+ numWorkers uint
+}
+
+// Wrap the cached last block for safe access from different service loops
+type blockCache struct {
+ sync.Mutex
+ blocks map[common.Hash]*types.Block
+ maxSize uint
+}
+
+func NewBlockCache(max uint) blockCache {
+ return blockCache{
+ blocks: make(map[common.Hash]*types.Block),
+ maxSize: max,
+ }
+}
+
+// New creates a new statediff.Service
+// func New(stack *node.Node, ethServ *eth.Ethereum, dbParams *DBParams, enableWriteLoop bool) error {
+func New(stack *node.Node, ethServ *eth.Ethereum, params ServiceParams) error {
+ blockChain := ethServ.BlockChain()
+ var indexer ind.Indexer
+ if params.DBParams != nil {
+ info := nodeinfo.Info{
+ GenesisBlock: blockChain.Genesis().Hash().Hex(),
+ NetworkID: strconv.FormatUint(ethServ.NetVersion(), 10),
+ ChainID: blockChain.Config().ChainID.Uint64(),
+ ID: params.DBParams.ID,
+ ClientName: params.DBParams.ClientName,
+ }
+
+ // TODO: pass max idle, open, lifetime?
+ db, err := postgres.NewDB(params.DBParams.ConnectionURL, postgres.ConnectionConfig{}, info)
+ if err != nil {
+ return err
+ }
+ indexer = ind.NewStateDiffIndexer(blockChain.Config(), db)
+ }
+ workers := params.NumWorkers
+ if workers == 0 {
+ workers = 1
+ }
+ sds := &Service{
+ Mutex: sync.Mutex{},
+ BlockChain: blockChain,
+ Builder: NewBuilder(blockChain.StateCache()),
+ QuitChan: make(chan bool),
+ Subscriptions: make(map[common.Hash]map[rpc.ID]Subscription),
+ SubscriptionTypes: make(map[common.Hash]Params),
+ BlockCache: NewBlockCache(workers),
+ indexer: indexer,
+ enableWriteLoop: params.EnableWriteLoop,
+ numWorkers: workers,
+ }
+ stack.RegisterLifecycle(sds)
+ stack.RegisterAPIs(sds.APIs())
+ return nil
+}
+
+// Protocols exports the services p2p protocols, this service has none
+func (sds *Service) Protocols() []p2p.Protocol {
+ return []p2p.Protocol{}
+}
+
+// APIs returns the RPC descriptors the statediff.Service offers
+func (sds *Service) APIs() []rpc.API {
+ return []rpc.API{
+ {
+ Namespace: APIName,
+ Version: APIVersion,
+ Service: NewPublicStateDiffAPI(sds),
+ Public: true,
+ },
+ }
+}
+
+// Return the parent block of currentBlock, using the cached block if available;
+// and cache the passed block
+func (lbc *blockCache) getParentBlock(currentBlock *types.Block, bc blockChain) *types.Block {
+ lbc.Lock()
+ parentHash := currentBlock.ParentHash()
+ var parentBlock *types.Block
+ if block, ok := lbc.blocks[parentHash]; ok {
+ parentBlock = block
+ if len(lbc.blocks) > int(lbc.maxSize) {
+ delete(lbc.blocks, parentHash)
+ }
+ } else {
+ parentBlock = bc.GetBlockByHash(parentHash)
+ }
+ lbc.blocks[currentBlock.Hash()] = currentBlock
+ lbc.Unlock()
+ return parentBlock
+}
+
+type workerParams struct {
+ chainEventCh <-chan core.ChainEvent
+ errCh <-chan error
+ wg *sync.WaitGroup
+ id uint
+}
+
+func (sds *Service) WriteLoop(chainEventCh chan core.ChainEvent) {
+ chainEventSub := sds.BlockChain.SubscribeChainEvent(chainEventCh)
+ defer chainEventSub.Unsubscribe()
+ errCh := chainEventSub.Err()
+ var wg sync.WaitGroup
+ // Process metrics for chain events, then forward to workers
+ chainEventFwd := make(chan core.ChainEvent, chainEventChanSize)
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ for {
+ select {
+ case chainEvent := <-chainEventCh:
+ statediffMetrics.lastEventHeight.Update(int64(chainEvent.Block.Number().Uint64()))
+ statediffMetrics.writeLoopChannelLen.Update(int64(len(chainEventCh)))
+ chainEventFwd <- chainEvent
+ case <-sds.QuitChan:
+ return
+ }
+ }
+ }()
+ wg.Add(int(sds.numWorkers))
+ for worker := uint(0); worker < sds.numWorkers; worker++ {
+ params := workerParams{chainEventCh: chainEventFwd, errCh: errCh, wg: &wg, id: worker}
+ go sds.writeLoopWorker(params)
+ }
+ wg.Wait()
+}
+
+func (sds *Service) writeLoopWorker(params workerParams) {
+ defer params.wg.Done()
+ for {
+ select {
+ //Notify chain event channel of events
+ case chainEvent := <-params.chainEventCh:
+ log.Debug("WriteLoop(): chain event received", "event", chainEvent)
+ currentBlock := chainEvent.Block
+ parentBlock := sds.BlockCache.getParentBlock(currentBlock, sds.BlockChain)
+ if parentBlock == nil {
+ log.Error("Parent block is nil, skipping this block", "block height", currentBlock.Number())
+ continue
+ }
+ log.Info("Writing state diff", "block height", currentBlock.Number().Uint64(), "worker", params.id)
+ err := sds.writeStateDiff(currentBlock, parentBlock.Root(), writeLoopParams)
+ if err != nil {
+ log.Error("statediff.Service.WriteLoop: processing error", "block height", currentBlock.Number().Uint64(), "error", err.Error(), "worker", params.id)
+ continue
+ }
+ // TODO: how to handle with concurrent workers
+ statediffMetrics.lastStatediffHeight.Update(int64(currentBlock.Number().Uint64()))
+ case err := <-params.errCh:
+ log.Warn("Error from chain event subscription", "error", err, "worker", params.id)
+ sds.close()
+ return
+ case <-sds.QuitChan:
+ log.Info("Quitting the statediff writing process", "worker", params.id)
+ sds.close()
+ return
+ }
+ }
+}
+
+// Loop is the main processing method
+func (sds *Service) Loop(chainEventCh chan core.ChainEvent) {
+ chainEventSub := sds.BlockChain.SubscribeChainEvent(chainEventCh)
+ defer chainEventSub.Unsubscribe()
+ errCh := chainEventSub.Err()
+ for {
+ select {
+ //Notify chain event channel of events
+ case chainEvent := <-chainEventCh:
+ statediffMetrics.serviceLoopChannelLen.Update(int64(len(chainEventCh)))
+ log.Debug("Loop(): chain event received", "event", chainEvent)
+ // if we don't have any subscribers, do not process a statediff
+ if atomic.LoadInt32(&sds.subscribers) == 0 {
+ log.Debug("Currently no subscribers to the statediffing service; processing is halted")
+ continue
+ }
+ currentBlock := chainEvent.Block
+ parentBlock := sds.BlockCache.getParentBlock(currentBlock, sds.BlockChain)
+ if parentBlock == nil {
+ log.Error("Parent block is nil, skipping this block", "block height", currentBlock.Number())
+ continue
+ }
+ sds.streamStateDiff(currentBlock, parentBlock.Root())
+ case err := <-errCh:
+ log.Warn("Error from chain event subscription", "error", err)
+ sds.close()
+ return
+ case <-sds.QuitChan:
+ log.Info("Quitting the statediffing process")
+ sds.close()
+ return
+ }
+ }
+}
+
+// streamStateDiff method builds the state diff payload for each subscription according to their subscription type and sends them the result
+func (sds *Service) streamStateDiff(currentBlock *types.Block, parentRoot common.Hash) {
+ sds.Lock()
+ for ty, subs := range sds.Subscriptions {
+ params, ok := sds.SubscriptionTypes[ty]
+ if !ok {
+ log.Error("no parameter set associated with this subscription", "subscription type", ty.Hex())
+ sds.closeType(ty)
+ continue
+ }
+ // create payload for this subscription type
+ payload, err := sds.processStateDiff(currentBlock, parentRoot, params)
+ if err != nil {
+ log.Error("statediff processing error", "block height", currentBlock.Number().Uint64(), "parameters", params, "error", err.Error())
+ continue
+ }
+ for id, sub := range subs {
+ select {
+ case sub.PayloadChan <- *payload:
+ log.Debug("sending statediff payload at head", "height", currentBlock.Number(), "subscription id", id)
+ default:
+ log.Info("unable to send statediff payload; channel has no receiver", "subscription id", id)
+ }
+ }
+ }
+ sds.Unlock()
+}
+
+// StateDiffAt returns a state diff object payload at the specific blockheight
+// This operation cannot be performed back past the point of db pruning; it requires an archival node for historical data
+func (sds *Service) StateDiffAt(blockNumber uint64, params Params) (*Payload, error) {
+ currentBlock := sds.BlockChain.GetBlockByNumber(blockNumber)
+ log.Info("sending state diff", "block height", blockNumber)
+ if blockNumber == 0 {
+ return sds.processStateDiff(currentBlock, common.Hash{}, params)
+ }
+ parentBlock := sds.BlockChain.GetBlockByHash(currentBlock.ParentHash())
+ return sds.processStateDiff(currentBlock, parentBlock.Root(), params)
+}
+
+// processStateDiff method builds the state diff payload from the current block, parent state root, and provided params
+func (sds *Service) processStateDiff(currentBlock *types.Block, parentRoot common.Hash, params Params) (*Payload, error) {
+ stateDiff, err := sds.Builder.BuildStateDiffObject(Args{
+ NewStateRoot: currentBlock.Root(),
+ OldStateRoot: parentRoot,
+ BlockHash: currentBlock.Hash(),
+ BlockNumber: currentBlock.Number(),
+ }, params)
+ // allow dereferencing of parent, keep current locked as it should be the next parent
+ sds.BlockChain.UnlockTrie(parentRoot)
+ if err != nil {
+ return nil, err
+ }
+ stateDiffRlp, err := rlp.EncodeToBytes(stateDiff)
+ if err != nil {
+ return nil, err
+ }
+ log.Info("state diff size", "at block height", currentBlock.Number().Uint64(), "rlp byte size", len(stateDiffRlp))
+ return sds.newPayload(stateDiffRlp, currentBlock, params)
+}
+
+func (sds *Service) newPayload(stateObject []byte, block *types.Block, params Params) (*Payload, error) {
+ payload := &Payload{
+ StateObjectRlp: stateObject,
+ }
+ if params.IncludeBlock {
+ blockBuff := new(bytes.Buffer)
+ if err := block.EncodeRLP(blockBuff); err != nil {
+ return nil, err
+ }
+ payload.BlockRlp = blockBuff.Bytes()
+ }
+ if params.IncludeTD {
+ payload.TotalDifficulty = sds.BlockChain.GetTdByHash(block.Hash())
+ }
+ if params.IncludeReceipts {
+ receiptBuff := new(bytes.Buffer)
+ receipts := sds.BlockChain.GetReceiptsByHash(block.Hash())
+ if err := rlp.Encode(receiptBuff, receipts); err != nil {
+ return nil, err
+ }
+ payload.ReceiptsRlp = receiptBuff.Bytes()
+ }
+ return payload, nil
+}
+
+// StateTrieAt returns a state trie object payload at the specified blockheight
+// This operation cannot be performed back past the point of db pruning; it requires an archival node for historical data
+func (sds *Service) StateTrieAt(blockNumber uint64, params Params) (*Payload, error) {
+ currentBlock := sds.BlockChain.GetBlockByNumber(blockNumber)
+ log.Info("sending state trie", "block height", blockNumber)
+ return sds.processStateTrie(currentBlock, params)
+}
+
+func (sds *Service) processStateTrie(block *types.Block, params Params) (*Payload, error) {
+ stateNodes, err := sds.Builder.BuildStateTrieObject(block)
+ if err != nil {
+ return nil, err
+ }
+ stateTrieRlp, err := rlp.EncodeToBytes(stateNodes)
+ if err != nil {
+ return nil, err
+ }
+ log.Info("state trie size", "at block height", block.Number().Uint64(), "rlp byte size", len(stateTrieRlp))
+ return sds.newPayload(stateTrieRlp, block, params)
+}
+
+// Subscribe is used by the API to subscribe to the service loop
+func (sds *Service) Subscribe(id rpc.ID, sub chan<- Payload, quitChan chan<- bool, params Params) {
+ log.Info("Subscribing to the statediff service")
+ if atomic.CompareAndSwapInt32(&sds.subscribers, 0, 1) {
+ log.Info("State diffing subscription received; beginning statediff processing")
+ }
+ // Subscription type is defined as the hash of the rlp-serialized subscription params
+ by, err := rlp.EncodeToBytes(params)
+ if err != nil {
+ log.Error("State diffing params need to be rlp-serializable")
+ return
+ }
+ subscriptionType := crypto.Keccak256Hash(by)
+ // Add subscriber
+ sds.Lock()
+ if sds.Subscriptions[subscriptionType] == nil {
+ sds.Subscriptions[subscriptionType] = make(map[rpc.ID]Subscription)
+ }
+ sds.Subscriptions[subscriptionType][id] = Subscription{
+ PayloadChan: sub,
+ QuitChan: quitChan,
+ }
+ sds.SubscriptionTypes[subscriptionType] = params
+ sds.Unlock()
+}
+
+// Unsubscribe is used to unsubscribe from the service loop
+func (sds *Service) Unsubscribe(id rpc.ID) error {
+ log.Info("Unsubscribing from the statediff service", "subscription id", id)
+ sds.Lock()
+ for ty := range sds.Subscriptions {
+ delete(sds.Subscriptions[ty], id)
+ if len(sds.Subscriptions[ty]) == 0 {
+ // If we removed the last subscription of this type, remove the subscription type outright
+ delete(sds.Subscriptions, ty)
+ delete(sds.SubscriptionTypes, ty)
+ }
+ }
+ if len(sds.Subscriptions) == 0 {
+ if atomic.CompareAndSwapInt32(&sds.subscribers, 1, 0) {
+ log.Info("No more subscriptions; halting statediff processing")
+ }
+ }
+ sds.Unlock()
+ return nil
+}
+
+// Start is used to begin the service
+func (sds *Service) Start() error {
+ log.Info("Starting statediff service")
+
+ chainEventCh := make(chan core.ChainEvent, chainEventChanSize)
+ go sds.Loop(chainEventCh)
+
+ if sds.enableWriteLoop {
+ log.Info("Starting statediff DB write loop", "params", writeLoopParams)
+ chainEventCh := make(chan core.ChainEvent, chainEventChanSize)
+ go sds.WriteLoop(chainEventCh)
+ }
+
+ return nil
+}
+
+// Stop is used to close down the service
+func (sds *Service) Stop() error {
+ log.Info("Stopping statediff service")
+ close(sds.QuitChan)
+ return nil
+}
+
+// close is used to close all listening subscriptions
+func (sds *Service) close() {
+ sds.Lock()
+ for ty, subs := range sds.Subscriptions {
+ for id, sub := range subs {
+ select {
+ case sub.QuitChan <- true:
+ log.Info("closing subscription", "id", id)
+ default:
+ log.Info("unable to close subscription; channel has no receiver", "subscription id", id)
+ }
+ delete(sds.Subscriptions[ty], id)
+ }
+ delete(sds.Subscriptions, ty)
+ delete(sds.SubscriptionTypes, ty)
+ }
+ sds.Unlock()
+}
+
+// closeType is used to close all subscriptions of given type
+// closeType needs to be called with subscription access locked
+func (sds *Service) closeType(subType common.Hash) {
+ subs := sds.Subscriptions[subType]
+ for id, sub := range subs {
+ sendNonBlockingQuit(id, sub)
+ }
+ delete(sds.Subscriptions, subType)
+ delete(sds.SubscriptionTypes, subType)
+}
+
+func sendNonBlockingQuit(id rpc.ID, sub Subscription) {
+ select {
+ case sub.QuitChan <- true:
+ log.Info("closing subscription", "id", id)
+ default:
+ log.Info("unable to close subscription; channel has no receiver", "subscription id", id)
+ }
+}
+
+// StreamCodeAndCodeHash subscription method for extracting all the codehash=>code mappings that exist in the trie at the provided height
+func (sds *Service) StreamCodeAndCodeHash(blockNumber uint64, outChan chan<- CodeAndCodeHash, quitChan chan<- bool) {
+ current := sds.BlockChain.GetBlockByNumber(blockNumber)
+ log.Info("sending code and codehash", "block height", blockNumber)
+ currentTrie, err := sds.BlockChain.StateCache().OpenTrie(current.Root())
+ if err != nil {
+ log.Error("error creating trie for block", "block height", current.Number(), "err", err)
+ close(quitChan)
+ return
+ }
+ it := currentTrie.NodeIterator([]byte{})
+ leafIt := trie.NewIterator(it)
+ go func() {
+ defer close(quitChan)
+ for leafIt.Next() {
+ select {
+ case <-sds.QuitChan:
+ return
+ default:
+ }
+ account := new(state.Account)
+ if err := rlp.DecodeBytes(leafIt.Value, account); err != nil {
+ log.Error("error decoding state account", "err", err)
+ return
+ }
+ codeHash := common.BytesToHash(account.CodeHash)
+ code, err := sds.BlockChain.StateCache().ContractCode(common.Hash{}, codeHash)
+ if err != nil {
+ log.Error("error collecting contract code", "err", err)
+ return
+ }
+ outChan <- CodeAndCodeHash{
+ Hash: codeHash,
+ Code: code,
+ }
+ }
+ }()
+}
+
+// WriteStateDiffAt writes a state diff at the specific blockheight directly to the database
+// This operation cannot be performed back past the point of db pruning; it requires an archival node
+// for historical data
+func (sds *Service) WriteStateDiffAt(blockNumber uint64, params Params) error {
+ currentBlock := sds.BlockChain.GetBlockByNumber(blockNumber)
+ parentRoot := common.Hash{}
+ if blockNumber != 0 {
+ parentBlock := sds.BlockChain.GetBlockByHash(currentBlock.ParentHash())
+ parentRoot = parentBlock.Root()
+ }
+ return sds.writeStateDiff(currentBlock, parentRoot, params)
+}
+
+// Writes a state diff from the current block, parent state root, and provided params
+func (sds *Service) writeStateDiff(block *types.Block, parentRoot common.Hash, params Params) error {
+ // log.Info("Writing state diff", "block height", block.Number().Uint64())
+ var totalDifficulty *big.Int
+ var receipts types.Receipts
+ if params.IncludeTD {
+ totalDifficulty = sds.BlockChain.GetTdByHash(block.Hash())
+ }
+ if params.IncludeReceipts {
+ receipts = sds.BlockChain.GetReceiptsByHash(block.Hash())
+ }
+ tx, err := sds.indexer.PushBlock(block, receipts, totalDifficulty)
+ if err != nil {
+ return err
+ }
+ // defer handling of commit/rollback for any return case
+ defer tx.Close()
+ output := func(node StateNode) error {
+ return sds.indexer.PushStateNode(tx, node)
+ }
+ codeOutput := func(c CodeAndCodeHash) error {
+ return sds.indexer.PushCodeAndCodeHash(tx, c)
+ }
+ err = sds.Builder.WriteStateDiffObject(StateRoots{
+ NewStateRoot: block.Root(),
+ OldStateRoot: parentRoot,
+ }, params, output, codeOutput)
+
+ // allow dereferencing of parent, keep current locked as it should be the next parent
+ sds.BlockChain.UnlockTrie(parentRoot)
+ if err != nil {
+ return err
+ }
+ return nil
+}
diff --git a/statediff/service_test.go b/statediff/service_test.go
new file mode 100644
index 000000000..ca9a483a5
--- /dev/null
+++ b/statediff/service_test.go
@@ -0,0 +1,291 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package statediff_test
+
+import (
+ "bytes"
+ "math/big"
+ "math/rand"
+ "reflect"
+ "sync"
+ "testing"
+
+ "github.com/ethereum/go-ethereum/trie"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/core"
+ "github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/rlp"
+ "github.com/ethereum/go-ethereum/rpc"
+ statediff "github.com/ethereum/go-ethereum/statediff"
+ "github.com/ethereum/go-ethereum/statediff/testhelpers/mocks"
+)
+
+func TestServiceLoop(t *testing.T) {
+ testErrorInChainEventLoop(t)
+ testErrorInBlockLoop(t)
+}
+
+var (
+ eventsChannel = make(chan core.ChainEvent, 1)
+
+ parentRoot1 = common.HexToHash("0x01")
+ parentRoot2 = common.HexToHash("0x02")
+ parentHeader1 = types.Header{Number: big.NewInt(rand.Int63()), Root: parentRoot1}
+ parentHeader2 = types.Header{Number: big.NewInt(rand.Int63()), Root: parentRoot2}
+
+ parentBlock1 = types.NewBlock(&parentHeader1, nil, nil, nil, new(trie.Trie))
+ parentBlock2 = types.NewBlock(&parentHeader2, nil, nil, nil, new(trie.Trie))
+
+ parentHash1 = parentBlock1.Hash()
+ parentHash2 = parentBlock2.Hash()
+
+ testRoot1 = common.HexToHash("0x03")
+ testRoot2 = common.HexToHash("0x04")
+ testRoot3 = common.HexToHash("0x04")
+ header1 = types.Header{ParentHash: parentHash1, Root: testRoot1, Number: big.NewInt(1)}
+ header2 = types.Header{ParentHash: parentHash2, Root: testRoot2, Number: big.NewInt(2)}
+ header3 = types.Header{ParentHash: common.HexToHash("parent hash"), Root: testRoot3, Number: big.NewInt(3)}
+
+ testBlock1 = types.NewBlock(&header1, nil, nil, nil, new(trie.Trie))
+ testBlock2 = types.NewBlock(&header2, nil, nil, nil, new(trie.Trie))
+ testBlock3 = types.NewBlock(&header3, nil, nil, nil, new(trie.Trie))
+
+ receiptRoot1 = common.HexToHash("0x05")
+ receiptRoot2 = common.HexToHash("0x06")
+ receiptRoot3 = common.HexToHash("0x07")
+ testReceipts1 = []*types.Receipt{types.NewReceipt(receiptRoot1.Bytes(), false, 1000), types.NewReceipt(receiptRoot2.Bytes(), false, 2000)}
+ testReceipts2 = []*types.Receipt{types.NewReceipt(receiptRoot3.Bytes(), false, 3000)}
+
+ event1 = core.ChainEvent{Block: testBlock1}
+ event2 = core.ChainEvent{Block: testBlock2}
+ event3 = core.ChainEvent{Block: testBlock3}
+
+ defaultParams = statediff.Params{
+ IncludeBlock: true,
+ IncludeReceipts: true,
+ IncludeTD: true,
+ }
+)
+
+func testErrorInChainEventLoop(t *testing.T) {
+ //the first chain event causes and error (in blockchain mock)
+ builder := mocks.Builder{}
+ blockChain := mocks.BlockChain{}
+ serviceQuit := make(chan bool)
+ service := statediff.Service{
+ Mutex: sync.Mutex{},
+ Builder: &builder,
+ BlockChain: &blockChain,
+ QuitChan: serviceQuit,
+ Subscriptions: make(map[common.Hash]map[rpc.ID]statediff.Subscription),
+ SubscriptionTypes: make(map[common.Hash]statediff.Params),
+ BlockCache: statediff.NewBlockCache(1),
+ }
+ payloadChan := make(chan statediff.Payload, 2)
+ quitChan := make(chan bool)
+ service.Subscribe(rpc.NewID(), payloadChan, quitChan, defaultParams)
+ testRoot2 = common.HexToHash("0xTestRoot2")
+ blockMapping := make(map[common.Hash]*types.Block)
+ blockMapping[parentBlock1.Hash()] = parentBlock1
+ blockMapping[parentBlock2.Hash()] = parentBlock2
+ blockChain.SetBlocksForHashes(blockMapping)
+ blockChain.SetChainEvents([]core.ChainEvent{event1, event2, event3})
+ blockChain.SetReceiptsForHash(testBlock1.Hash(), testReceipts1)
+ blockChain.SetReceiptsForHash(testBlock2.Hash(), testReceipts2)
+
+ payloads := make([]statediff.Payload, 0, 2)
+ wg := new(sync.WaitGroup)
+ go func() {
+ wg.Add(1)
+ for i := 0; i < 2; i++ {
+ select {
+ case payload := <-payloadChan:
+ payloads = append(payloads, payload)
+ case <-quitChan:
+ }
+ }
+ wg.Done()
+ }()
+ service.Loop(eventsChannel)
+ wg.Wait()
+ if len(payloads) != 2 {
+ t.Error("Test failure:", t.Name())
+ t.Logf("Actual number of payloads does not equal expected.\nactual: %+v\nexpected: 3", len(payloads))
+ }
+
+ testReceipts1Rlp, err := rlp.EncodeToBytes(testReceipts1)
+ if err != nil {
+ t.Error(err)
+ }
+ testReceipts2Rlp, err := rlp.EncodeToBytes(testReceipts2)
+ if err != nil {
+ t.Error(err)
+ }
+ expectedReceiptsRlp := [][]byte{testReceipts1Rlp, testReceipts2Rlp, nil}
+ for i, payload := range payloads {
+ if !bytes.Equal(payload.ReceiptsRlp, expectedReceiptsRlp[i]) {
+ t.Error("Test failure:", t.Name())
+ t.Logf("Actual receipt rlp for payload %d does not equal expected.\nactual: %+v\nexpected: %+v", i, payload.ReceiptsRlp, expectedReceiptsRlp[i])
+ }
+ }
+
+ if !reflect.DeepEqual(builder.Params, defaultParams) {
+ t.Error("Test failure:", t.Name())
+ t.Logf("Actual params does not equal expected.\nactual:%+v\nexpected: %+v", builder.Params, defaultParams)
+ }
+ if !bytes.Equal(builder.Args.BlockHash.Bytes(), testBlock2.Hash().Bytes()) {
+ t.Error("Test failure:", t.Name())
+ t.Logf("Actual blockhash does not equal expected.\nactual:%x\nexpected: %x", builder.Args.BlockHash.Bytes(), testBlock2.Hash().Bytes())
+ }
+ if !bytes.Equal(builder.Args.OldStateRoot.Bytes(), parentBlock2.Root().Bytes()) {
+ t.Error("Test failure:", t.Name())
+ t.Logf("Actual root does not equal expected.\nactual:%x\nexpected: %x", builder.Args.OldStateRoot.Bytes(), parentBlock2.Root().Bytes())
+ }
+ if !bytes.Equal(builder.Args.NewStateRoot.Bytes(), testBlock2.Root().Bytes()) {
+ t.Error("Test failure:", t.Name())
+ t.Logf("Actual root does not equal expected.\nactual:%x\nexpected: %x", builder.Args.NewStateRoot.Bytes(), testBlock2.Root().Bytes())
+ }
+ //look up the parent block from its hash
+ expectedHashes := []common.Hash{testBlock1.ParentHash(), testBlock2.ParentHash()}
+ if !reflect.DeepEqual(blockChain.HashesLookedUp, expectedHashes) {
+ t.Error("Test failure:", t.Name())
+ t.Logf("Actual looked up parent hashes does not equal expected.\nactual:%+v\nexpected: %+v", blockChain.HashesLookedUp, expectedHashes)
+ }
+}
+
+func testErrorInBlockLoop(t *testing.T) {
+ //second block's parent block can't be found
+ builder := mocks.Builder{}
+ blockChain := mocks.BlockChain{}
+ service := statediff.Service{
+ Builder: &builder,
+ BlockChain: &blockChain,
+ QuitChan: make(chan bool),
+ Subscriptions: make(map[common.Hash]map[rpc.ID]statediff.Subscription),
+ SubscriptionTypes: make(map[common.Hash]statediff.Params),
+ BlockCache: statediff.NewBlockCache(1),
+ }
+ payloadChan := make(chan statediff.Payload)
+ quitChan := make(chan bool)
+ service.Subscribe(rpc.NewID(), payloadChan, quitChan, defaultParams)
+ blockMapping := make(map[common.Hash]*types.Block)
+ blockMapping[parentBlock1.Hash()] = parentBlock1
+ blockChain.SetBlocksForHashes(blockMapping)
+ blockChain.SetChainEvents([]core.ChainEvent{event1, event2})
+ // Need to have listeners on the channels or the subscription will be closed and the processing halted
+ go func() {
+ select {
+ case <-payloadChan:
+ case <-quitChan:
+ }
+ }()
+ service.Loop(eventsChannel)
+ if !reflect.DeepEqual(builder.Params, defaultParams) {
+ t.Error("Test failure:", t.Name())
+ t.Logf("Actual params does not equal expected.\nactual:%+v\nexpected: %+v", builder.Params, defaultParams)
+ }
+ if !bytes.Equal(builder.Args.BlockHash.Bytes(), testBlock1.Hash().Bytes()) {
+ t.Error("Test failure:", t.Name())
+ t.Logf("Actual blockhash does not equal expected.\nactual:%+v\nexpected: %x", builder.Args.BlockHash.Bytes(), testBlock1.Hash().Bytes())
+ }
+ if !bytes.Equal(builder.Args.OldStateRoot.Bytes(), parentBlock1.Root().Bytes()) {
+ t.Error("Test failure:", t.Name())
+ t.Logf("Actual old state root does not equal expected.\nactual:%+v\nexpected: %x", builder.Args.OldStateRoot.Bytes(), parentBlock1.Root().Bytes())
+ }
+ if !bytes.Equal(builder.Args.NewStateRoot.Bytes(), testBlock1.Root().Bytes()) {
+ t.Error("Test failure:", t.Name())
+ t.Logf("Actual new state root does not equal expected.\nactual:%+v\nexpected: %x", builder.Args.NewStateRoot.Bytes(), testBlock1.Root().Bytes())
+ }
+}
+
+func TestGetStateDiffAt(t *testing.T) {
+ testErrorInStateDiffAt(t)
+}
+
+func testErrorInStateDiffAt(t *testing.T) {
+ mockStateDiff := statediff.StateObject{
+ BlockNumber: testBlock1.Number(),
+ BlockHash: testBlock1.Hash(),
+ }
+ expectedStateDiffRlp, err := rlp.EncodeToBytes(mockStateDiff)
+ if err != nil {
+ t.Error(err)
+ }
+ expectedReceiptsRlp, err := rlp.EncodeToBytes(testReceipts1)
+ if err != nil {
+ t.Error(err)
+ }
+ expectedBlockRlp, err := rlp.EncodeToBytes(testBlock1)
+ if err != nil {
+ t.Error(err)
+ }
+ expectedStateDiffPayload := statediff.Payload{
+ StateObjectRlp: expectedStateDiffRlp,
+ ReceiptsRlp: expectedReceiptsRlp,
+ BlockRlp: expectedBlockRlp,
+ }
+ expectedStateDiffPayloadRlp, err := rlp.EncodeToBytes(expectedStateDiffPayload)
+ if err != nil {
+ t.Error(err)
+ }
+ builder := mocks.Builder{}
+ builder.SetStateDiffToBuild(mockStateDiff)
+ blockChain := mocks.BlockChain{}
+ blockMapping := make(map[common.Hash]*types.Block)
+ blockMapping[parentBlock1.Hash()] = parentBlock1
+ blockChain.SetBlocksForHashes(blockMapping)
+ blockChain.SetBlockForNumber(testBlock1, testBlock1.NumberU64())
+ blockChain.SetReceiptsForHash(testBlock1.Hash(), testReceipts1)
+ service := statediff.Service{
+ Mutex: sync.Mutex{},
+ Builder: &builder,
+ BlockChain: &blockChain,
+ QuitChan: make(chan bool),
+ Subscriptions: make(map[common.Hash]map[rpc.ID]statediff.Subscription),
+ SubscriptionTypes: make(map[common.Hash]statediff.Params),
+ BlockCache: statediff.NewBlockCache(1),
+ }
+ stateDiffPayload, err := service.StateDiffAt(testBlock1.NumberU64(), defaultParams)
+ if err != nil {
+ t.Error(err)
+ }
+ stateDiffPayloadRlp, err := rlp.EncodeToBytes(stateDiffPayload)
+ if err != nil {
+ t.Error(err)
+ }
+ if !reflect.DeepEqual(builder.Params, defaultParams) {
+ t.Error("Test failure:", t.Name())
+ t.Logf("Actual params does not equal expected.\nactual:%+v\nexpected: %+v", builder.Params, defaultParams)
+ }
+ if !bytes.Equal(builder.Args.BlockHash.Bytes(), testBlock1.Hash().Bytes()) {
+ t.Error("Test failure:", t.Name())
+ t.Logf("Actual blockhash does not equal expected.\nactual:%+v\nexpected: %x", builder.Args.BlockHash.Bytes(), testBlock1.Hash().Bytes())
+ }
+ if !bytes.Equal(builder.Args.OldStateRoot.Bytes(), parentBlock1.Root().Bytes()) {
+ t.Error("Test failure:", t.Name())
+ t.Logf("Actual old state root does not equal expected.\nactual:%+v\nexpected: %x", builder.Args.OldStateRoot.Bytes(), parentBlock1.Root().Bytes())
+ }
+ if !bytes.Equal(builder.Args.NewStateRoot.Bytes(), testBlock1.Root().Bytes()) {
+ t.Error("Test failure:", t.Name())
+ t.Logf("Actual new state root does not equal expected.\nactual:%+v\nexpected: %x", builder.Args.NewStateRoot.Bytes(), testBlock1.Root().Bytes())
+ }
+ if !bytes.Equal(expectedStateDiffPayloadRlp, stateDiffPayloadRlp) {
+ t.Error("Test failure:", t.Name())
+ t.Logf("Actual state diff payload does not equal expected.\nactual:%+v\nexpected: %+v", expectedStateDiffPayload, stateDiffPayload)
+ }
+}
diff --git a/statediff/testhelpers/helpers.go b/statediff/testhelpers/helpers.go
new file mode 100644
index 000000000..7fd320b25
--- /dev/null
+++ b/statediff/testhelpers/helpers.go
@@ -0,0 +1,124 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package testhelpers
+
+import (
+ "math/big"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/consensus/ethash"
+ "github.com/ethereum/go-ethereum/core"
+ "github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/core/vm"
+ "github.com/ethereum/go-ethereum/crypto"
+ "github.com/ethereum/go-ethereum/params"
+)
+
+// MakeChain creates a chain of n blocks starting at and including parent.
+// the returned hash chain is ordered head->parent.
+func MakeChain(n int, parent *types.Block, chainGen func(int, *core.BlockGen)) ([]*types.Block, *core.BlockChain) {
+ config := params.TestChainConfig
+ blocks, _ := core.GenerateChain(config, parent, ethash.NewFaker(), Testdb, n, chainGen)
+ chain, _ := core.NewBlockChain(Testdb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil)
+ return blocks, chain
+}
+
+func TestSelfDestructChainGen(i int, block *core.BlockGen) {
+ signer := types.HomesteadSigner{}
+ switch i {
+ case 0:
+ // Block 1 is mined by Account1Addr
+ // Account1Addr creates a new contract
+ block.SetCoinbase(TestBankAddress)
+ tx, _ := types.SignTx(types.NewContractCreation(0, big.NewInt(0), 1000000, big.NewInt(0), ContractCode), signer, TestBankKey)
+ ContractAddr = crypto.CreateAddress(TestBankAddress, 0)
+ block.AddTx(tx)
+ case 1:
+ // Block 2 is mined by Account1Addr
+ // Account1Addr self-destructs the contract
+ block.SetCoinbase(TestBankAddress)
+ data := common.Hex2Bytes("43D726D6")
+ tx, _ := types.SignTx(types.NewTransaction(1, ContractAddr, big.NewInt(0), 100000, nil, data), signer, TestBankKey)
+ block.AddTx(tx)
+ }
+}
+
+func TestChainGen(i int, block *core.BlockGen) {
+ signer := types.HomesteadSigner{}
+ switch i {
+ case 0:
+ // In block 1, the test bank sends account #1 some ether.
+ tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), Account1Addr, big.NewInt(10000), params.TxGas, nil, nil), signer, TestBankKey)
+ block.AddTx(tx)
+ case 1:
+ // In block 2, the test bank sends some more ether to account #1.
+ // Account1Addr passes it on to account #2.
+ // Account1Addr creates a test contract.
+ tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), Account1Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, TestBankKey)
+ nonce := block.TxNonce(Account1Addr)
+ tx2, _ := types.SignTx(types.NewTransaction(nonce, Account2Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, Account1Key)
+ nonce++
+ tx3, _ := types.SignTx(types.NewContractCreation(nonce, big.NewInt(0), 1000000, big.NewInt(0), ContractCode), signer, Account1Key)
+ ContractAddr = crypto.CreateAddress(Account1Addr, nonce)
+ block.AddTx(tx1)
+ block.AddTx(tx2)
+ block.AddTx(tx3)
+ case 2:
+ // Block 3 has a single tx from the bankAccount to the contract, that transfers no value
+ // Block 3 is mined by Account2Addr
+ block.SetCoinbase(Account2Addr)
+ //put function: c16431b9
+ //close function: 43d726d6
+ data := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003")
+ tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), ContractAddr, big.NewInt(0), 100000, nil, data), signer, TestBankKey)
+ block.AddTx(tx)
+ case 3:
+ // Block 4 has three txs from bankAccount to the contract, that transfer no value
+ // Two set the two original slot positions to 0 and one sets another position to a new value
+ // Block 4 is mined by Account2Addr
+ block.SetCoinbase(Account2Addr)
+ data1 := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")
+ data2 := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000")
+ data3 := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000009")
+
+ nonce := block.TxNonce(TestBankAddress)
+ tx1, _ := types.SignTx(types.NewTransaction(nonce, ContractAddr, big.NewInt(0), 100000, nil, data1), signer, TestBankKey)
+ nonce++
+ tx2, _ := types.SignTx(types.NewTransaction(nonce, ContractAddr, big.NewInt(0), 100000, nil, data2), signer, TestBankKey)
+ nonce++
+ tx3, _ := types.SignTx(types.NewTransaction(nonce, ContractAddr, big.NewInt(0), 100000, nil, data3), signer, TestBankKey)
+ block.AddTx(tx1)
+ block.AddTx(tx2)
+ block.AddTx(tx3)
+ case 4:
+ // Block 5 has one tx from bankAccount to the contract, that transfers no value
+ // It sets the remaining storage value to zero
+ // Block 5 is mined by Account1Addr
+ block.SetCoinbase(Account1Addr)
+ data := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000")
+ nonce := block.TxNonce(TestBankAddress)
+ tx, _ := types.SignTx(types.NewTransaction(nonce, ContractAddr, big.NewInt(0), 100000, nil, data), signer, TestBankKey)
+ block.AddTx(tx)
+ case 5:
+ // Block 6 has a tx from Account1Key which self-destructs the contract, it transfers no value
+ // Block 6 is mined by Account2Addr
+ block.SetCoinbase(Account2Addr)
+ data := common.Hex2Bytes("43D726D6")
+ tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(Account1Addr), ContractAddr, big.NewInt(0), 100000, nil, data), signer, Account1Key)
+ block.AddTx(tx)
+ }
+}
diff --git a/statediff/testhelpers/mocks/blockchain.go b/statediff/testhelpers/mocks/blockchain.go
new file mode 100644
index 000000000..b0111e64c
--- /dev/null
+++ b/statediff/testhelpers/mocks/blockchain.go
@@ -0,0 +1,134 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package mocks
+
+import (
+ "errors"
+ "math/big"
+ "time"
+
+ "github.com/ethereum/go-ethereum/core/state"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/core"
+ "github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/event"
+)
+
+// BlockChain is a mock blockchain for testing
+type BlockChain struct {
+ HashesLookedUp []common.Hash
+ blocksToReturnByHash map[common.Hash]*types.Block
+ blocksToReturnByNumber map[uint64]*types.Block
+ callCount int
+ ChainEvents []core.ChainEvent
+ Receipts map[common.Hash]types.Receipts
+ TDByHash map[common.Hash]*big.Int
+}
+
+// SetBlocksForHashes mock method
+func (blockChain *BlockChain) SetBlocksForHashes(blocks map[common.Hash]*types.Block) {
+ if blockChain.blocksToReturnByHash == nil {
+ blockChain.blocksToReturnByHash = make(map[common.Hash]*types.Block)
+ }
+ blockChain.blocksToReturnByHash = blocks
+}
+
+// GetBlockByHash mock method
+func (blockChain *BlockChain) GetBlockByHash(hash common.Hash) *types.Block {
+ blockChain.HashesLookedUp = append(blockChain.HashesLookedUp, hash)
+
+ var block *types.Block
+ if len(blockChain.blocksToReturnByHash) > 0 {
+ block = blockChain.blocksToReturnByHash[hash]
+ }
+
+ return block
+}
+
+// SetChainEvents mock method
+func (blockChain *BlockChain) SetChainEvents(chainEvents []core.ChainEvent) {
+ blockChain.ChainEvents = chainEvents
+}
+
+// SubscribeChainEvent mock method
+func (blockChain *BlockChain) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription {
+ subErr := errors.New("subscription error")
+
+ var eventCounter int
+ subscription := event.NewSubscription(func(quit <-chan struct{}) error {
+ for _, chainEvent := range blockChain.ChainEvents {
+ if eventCounter > 1 {
+ time.Sleep(250 * time.Millisecond)
+ return subErr
+ }
+ select {
+ case ch <- chainEvent:
+ case <-quit:
+ return nil
+ }
+ eventCounter++
+ }
+ return nil
+ })
+
+ return subscription
+}
+
+// SetReceiptsForHash test method
+func (blockChain *BlockChain) SetReceiptsForHash(hash common.Hash, receipts types.Receipts) {
+ if blockChain.Receipts == nil {
+ blockChain.Receipts = make(map[common.Hash]types.Receipts)
+ }
+ blockChain.Receipts[hash] = receipts
+}
+
+// GetReceiptsByHash mock method
+func (blockChain *BlockChain) GetReceiptsByHash(hash common.Hash) types.Receipts {
+ return blockChain.Receipts[hash]
+}
+
+// SetBlockForNumber test method
+func (blockChain *BlockChain) SetBlockForNumber(block *types.Block, number uint64) {
+ if blockChain.blocksToReturnByNumber == nil {
+ blockChain.blocksToReturnByNumber = make(map[uint64]*types.Block)
+ }
+ blockChain.blocksToReturnByNumber[number] = block
+}
+
+// GetBlockByNumber mock method
+func (blockChain *BlockChain) GetBlockByNumber(number uint64) *types.Block {
+ return blockChain.blocksToReturnByNumber[number]
+}
+
+// GetTdByHash mock method
+func (blockChain *BlockChain) GetTdByHash(hash common.Hash) *big.Int {
+ return blockChain.TDByHash[hash]
+}
+
+func (blockChain *BlockChain) SetTdByHash(hash common.Hash, td *big.Int) {
+ if blockChain.TDByHash == nil {
+ blockChain.TDByHash = make(map[common.Hash]*big.Int)
+ }
+ blockChain.TDByHash[hash] = td
+}
+
+func (blockChain *BlockChain) UnlockTrie(root common.Hash) {}
+
+func (BlockChain *BlockChain) StateCache() state.Database {
+ return nil
+}
diff --git a/statediff/testhelpers/mocks/builder.go b/statediff/testhelpers/mocks/builder.go
new file mode 100644
index 000000000..ff9faf3ec
--- /dev/null
+++ b/statediff/testhelpers/mocks/builder.go
@@ -0,0 +1,67 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package mocks
+
+import (
+ "github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/statediff"
+ sdtypes "github.com/ethereum/go-ethereum/statediff/types"
+)
+
+// Builder is a mock state diff builder
+type Builder struct {
+ Args statediff.Args
+ Params statediff.Params
+ StateRoots statediff.StateRoots
+ stateDiff statediff.StateObject
+ block *types.Block
+ stateTrie statediff.StateObject
+ builderError error
+}
+
+// BuildStateDiffObject mock method
+func (builder *Builder) BuildStateDiffObject(args statediff.Args, params statediff.Params) (statediff.StateObject, error) {
+ builder.Args = args
+ builder.Params = params
+
+ return builder.stateDiff, builder.builderError
+}
+
+// BuildStateDiffObject mock method
+func (builder *Builder) WriteStateDiffObject(args statediff.StateRoots, params statediff.Params, output sdtypes.StateNodeSink, codeOutput sdtypes.CodeSink) error {
+ builder.StateRoots = args
+ builder.Params = params
+
+ return builder.builderError
+}
+
+// BuildStateTrieObject mock method
+func (builder *Builder) BuildStateTrieObject(block *types.Block) (statediff.StateObject, error) {
+ builder.block = block
+
+ return builder.stateTrie, builder.builderError
+}
+
+// SetStateDiffToBuild mock method
+func (builder *Builder) SetStateDiffToBuild(stateDiff statediff.StateObject) {
+ builder.stateDiff = stateDiff
+}
+
+// SetBuilderError mock method
+func (builder *Builder) SetBuilderError(err error) {
+ builder.builderError = err
+}
diff --git a/statediff/testhelpers/mocks/service.go b/statediff/testhelpers/mocks/service.go
new file mode 100644
index 000000000..64e48a325
--- /dev/null
+++ b/statediff/testhelpers/mocks/service.go
@@ -0,0 +1,322 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package mocks
+
+import (
+ "bytes"
+ "errors"
+ "fmt"
+ "sync"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/crypto"
+ "github.com/ethereum/go-ethereum/rlp"
+
+ "github.com/ethereum/go-ethereum/core"
+ "github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/log"
+ "github.com/ethereum/go-ethereum/p2p"
+ "github.com/ethereum/go-ethereum/rpc"
+ "github.com/ethereum/go-ethereum/statediff"
+ sdtypes "github.com/ethereum/go-ethereum/statediff/types"
+)
+
+// MockStateDiffService is a mock state diff service
+type MockStateDiffService struct {
+ sync.Mutex
+ Builder statediff.Builder
+ BlockChain *BlockChain
+ ReturnProtocol []p2p.Protocol
+ ReturnAPIs []rpc.API
+ BlockChan chan *types.Block
+ ParentBlockChan chan *types.Block
+ QuitChan chan bool
+ Subscriptions map[common.Hash]map[rpc.ID]statediff.Subscription
+ SubscriptionTypes map[common.Hash]statediff.Params
+}
+
+// Protocols mock method
+func (sds *MockStateDiffService) Protocols() []p2p.Protocol {
+ return []p2p.Protocol{}
+}
+
+// APIs mock method
+func (sds *MockStateDiffService) APIs() []rpc.API {
+ return []rpc.API{
+ {
+ Namespace: statediff.APIName,
+ Version: statediff.APIVersion,
+ Service: statediff.NewPublicStateDiffAPI(sds),
+ Public: true,
+ },
+ }
+}
+
+// Loop mock method
+func (sds *MockStateDiffService) Loop(chan core.ChainEvent) {
+ //loop through chain events until no more
+ for {
+ select {
+ case block := <-sds.BlockChan:
+ currentBlock := block
+ parentBlock := <-sds.ParentBlockChan
+ parentHash := parentBlock.Hash()
+ if parentBlock == nil {
+ log.Error("Parent block is nil, skipping this block",
+ "parent block hash", parentHash.String(),
+ "current block number", currentBlock.Number())
+ continue
+ }
+ sds.streamStateDiff(currentBlock, parentBlock.Root())
+ case <-sds.QuitChan:
+ log.Debug("Quitting the statediff block channel")
+ sds.close()
+ return
+ }
+ }
+}
+
+// streamStateDiff method builds the state diff payload for each subscription according to their subscription type and sends them the result
+func (sds *MockStateDiffService) streamStateDiff(currentBlock *types.Block, parentRoot common.Hash) {
+ sds.Lock()
+ for ty, subs := range sds.Subscriptions {
+ params, ok := sds.SubscriptionTypes[ty]
+ if !ok {
+ log.Error(fmt.Sprintf("subscriptions type %s do not have a parameter set associated with them", ty.Hex()))
+ sds.closeType(ty)
+ continue
+ }
+ // create payload for this subscription type
+ payload, err := sds.processStateDiff(currentBlock, parentRoot, params)
+ if err != nil {
+ log.Error(fmt.Sprintf("statediff processing error for subscriptions with parameters: %+v", params))
+ sds.closeType(ty)
+ continue
+ }
+ for id, sub := range subs {
+ select {
+ case sub.PayloadChan <- *payload:
+ log.Debug(fmt.Sprintf("sending statediff payload to subscription %s", id))
+ default:
+ log.Info(fmt.Sprintf("unable to send statediff payload to subscription %s; channel has no receiver", id))
+ }
+ }
+ }
+ sds.Unlock()
+}
+
+// StateDiffAt mock method
+func (sds *MockStateDiffService) StateDiffAt(blockNumber uint64, params statediff.Params) (*statediff.Payload, error) {
+ currentBlock := sds.BlockChain.GetBlockByNumber(blockNumber)
+ log.Info(fmt.Sprintf("sending state diff at %d", blockNumber))
+ if blockNumber == 0 {
+ return sds.processStateDiff(currentBlock, common.Hash{}, params)
+ }
+ parentBlock := sds.BlockChain.GetBlockByHash(currentBlock.ParentHash())
+ return sds.processStateDiff(currentBlock, parentBlock.Root(), params)
+}
+
+// processStateDiff method builds the state diff payload from the current block, parent state root, and provided params
+func (sds *MockStateDiffService) processStateDiff(currentBlock *types.Block, parentRoot common.Hash, params statediff.Params) (*statediff.Payload, error) {
+ stateDiff, err := sds.Builder.BuildStateDiffObject(statediff.Args{
+ NewStateRoot: currentBlock.Root(),
+ OldStateRoot: parentRoot,
+ BlockHash: currentBlock.Hash(),
+ BlockNumber: currentBlock.Number(),
+ }, params)
+ if err != nil {
+ return nil, err
+ }
+ stateDiffRlp, err := rlp.EncodeToBytes(stateDiff)
+ if err != nil {
+ return nil, err
+ }
+ return sds.newPayload(stateDiffRlp, currentBlock, params)
+}
+
+func (sds *MockStateDiffService) newPayload(stateObject []byte, block *types.Block, params statediff.Params) (*statediff.Payload, error) {
+ payload := &statediff.Payload{
+ StateObjectRlp: stateObject,
+ }
+ if params.IncludeBlock {
+ blockBuff := new(bytes.Buffer)
+ if err := block.EncodeRLP(blockBuff); err != nil {
+ return nil, err
+ }
+ payload.BlockRlp = blockBuff.Bytes()
+ }
+ if params.IncludeTD {
+ payload.TotalDifficulty = sds.BlockChain.GetTdByHash(block.Hash())
+ }
+ if params.IncludeReceipts {
+ receiptBuff := new(bytes.Buffer)
+ receipts := sds.BlockChain.GetReceiptsByHash(block.Hash())
+ if err := rlp.Encode(receiptBuff, receipts); err != nil {
+ return nil, err
+ }
+ payload.ReceiptsRlp = receiptBuff.Bytes()
+ }
+ return payload, nil
+}
+
+// WriteStateDiffAt mock method
+func (sds *MockStateDiffService) WriteStateDiffAt(blockNumber uint64, params statediff.Params) error {
+ // TODO: something useful here
+ return nil
+}
+
+// Loop mock method
+func (sds *MockStateDiffService) WriteLoop(chan core.ChainEvent) {
+ //loop through chain events until no more
+ for {
+ select {
+ case block := <-sds.BlockChan:
+ currentBlock := block
+ parentBlock := <-sds.ParentBlockChan
+ parentHash := parentBlock.Hash()
+ if parentBlock == nil {
+ log.Error("Parent block is nil, skipping this block",
+ "parent block hash", parentHash.String(),
+ "current block number", currentBlock.Number())
+ continue
+ }
+ // TODO:
+ // sds.writeStateDiff(currentBlock, parentBlock.Root(), statediff.Params{})
+ case <-sds.QuitChan:
+ log.Debug("Quitting the statediff block channel")
+ sds.close()
+ return
+ }
+ }
+}
+
+// StateTrieAt mock method
+func (sds *MockStateDiffService) StateTrieAt(blockNumber uint64, params statediff.Params) (*statediff.Payload, error) {
+ currentBlock := sds.BlockChain.GetBlockByNumber(blockNumber)
+ log.Info(fmt.Sprintf("sending state trie at %d", blockNumber))
+ return sds.stateTrieAt(currentBlock, params)
+}
+
+func (sds *MockStateDiffService) stateTrieAt(block *types.Block, params statediff.Params) (*statediff.Payload, error) {
+ stateNodes, err := sds.Builder.BuildStateTrieObject(block)
+ if err != nil {
+ return nil, err
+ }
+ stateTrieRlp, err := rlp.EncodeToBytes(stateNodes)
+ if err != nil {
+ return nil, err
+ }
+ return sds.newPayload(stateTrieRlp, block, params)
+}
+
+// Subscribe is used by the API to subscribe to the service loop
+func (sds *MockStateDiffService) Subscribe(id rpc.ID, sub chan<- statediff.Payload, quitChan chan<- bool, params statediff.Params) {
+ // Subscription type is defined as the hash of the rlp-serialized subscription params
+ by, err := rlp.EncodeToBytes(params)
+ if err != nil {
+ return
+ }
+ subscriptionType := crypto.Keccak256Hash(by)
+ // Add subscriber
+ sds.Lock()
+ if sds.Subscriptions[subscriptionType] == nil {
+ sds.Subscriptions[subscriptionType] = make(map[rpc.ID]statediff.Subscription)
+ }
+ sds.Subscriptions[subscriptionType][id] = statediff.Subscription{
+ PayloadChan: sub,
+ QuitChan: quitChan,
+ }
+ sds.SubscriptionTypes[subscriptionType] = params
+ sds.Unlock()
+}
+
+// Unsubscribe is used to unsubscribe from the service loop
+func (sds *MockStateDiffService) Unsubscribe(id rpc.ID) error {
+ sds.Lock()
+ for ty := range sds.Subscriptions {
+ delete(sds.Subscriptions[ty], id)
+ if len(sds.Subscriptions[ty]) == 0 {
+ // If we removed the last subscription of this type, remove the subscription type outright
+ delete(sds.Subscriptions, ty)
+ delete(sds.SubscriptionTypes, ty)
+ }
+ }
+ sds.Unlock()
+ return nil
+}
+
+// close is used to close all listening subscriptions
+func (sds *MockStateDiffService) close() {
+ sds.Lock()
+ for ty, subs := range sds.Subscriptions {
+ for id, sub := range subs {
+ select {
+ case sub.QuitChan <- true:
+ log.Info(fmt.Sprintf("closing subscription %s", id))
+ default:
+ log.Info(fmt.Sprintf("unable to close subscription %s; channel has no receiver", id))
+ }
+ delete(sds.Subscriptions[ty], id)
+ }
+ delete(sds.Subscriptions, ty)
+ delete(sds.SubscriptionTypes, ty)
+ }
+ sds.Unlock()
+}
+
+// Start mock method
+func (sds *MockStateDiffService) Start() error {
+ log.Info("Starting mock statediff service")
+ if sds.ParentBlockChan == nil || sds.BlockChan == nil {
+ return errors.New("MockStateDiffingService needs to be configured with a MockParentBlockChan and MockBlockChan")
+ }
+ chainEventCh := make(chan core.ChainEvent, 10)
+ go sds.Loop(chainEventCh)
+
+ return nil
+}
+
+// Stop mock method
+func (sds *MockStateDiffService) Stop() error {
+ log.Info("Stopping mock statediff service")
+ close(sds.QuitChan)
+ return nil
+}
+
+// closeType is used to close all subscriptions of given type
+// closeType needs to be called with subscription access locked
+func (sds *MockStateDiffService) closeType(subType common.Hash) {
+ subs := sds.Subscriptions[subType]
+ for id, sub := range subs {
+ sendNonBlockingQuit(id, sub)
+ }
+ delete(sds.Subscriptions, subType)
+ delete(sds.SubscriptionTypes, subType)
+}
+
+func (sds *MockStateDiffService) StreamCodeAndCodeHash(blockNumber uint64, outChan chan<- sdtypes.CodeAndCodeHash, quitChan chan<- bool) {
+ panic("implement me")
+}
+
+func sendNonBlockingQuit(id rpc.ID, sub statediff.Subscription) {
+ select {
+ case sub.QuitChan <- true:
+ log.Info(fmt.Sprintf("closing subscription %s", id))
+ default:
+ log.Info("unable to close subscription %s; channel has no receiver", id)
+ }
+}
diff --git a/statediff/testhelpers/mocks/service_test.go b/statediff/testhelpers/mocks/service_test.go
new file mode 100644
index 000000000..4b8d89e41
--- /dev/null
+++ b/statediff/testhelpers/mocks/service_test.go
@@ -0,0 +1,238 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package mocks
+
+import (
+ "bytes"
+ "math/big"
+ "sort"
+ "sync"
+ "testing"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/core/state"
+ "github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/rlp"
+ "github.com/ethereum/go-ethereum/rpc"
+ "github.com/ethereum/go-ethereum/statediff"
+ "github.com/ethereum/go-ethereum/statediff/testhelpers"
+ sdtypes "github.com/ethereum/go-ethereum/statediff/types"
+)
+
+var (
+ emptyStorage = make([]sdtypes.StorageNode, 0)
+ block0, block1 *types.Block
+ minerLeafKey = testhelpers.AddressToLeafKey(common.HexToAddress("0x0"))
+ account1, _ = rlp.EncodeToBytes(state.Account{
+ Nonce: uint64(0),
+ Balance: big.NewInt(10000),
+ CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(),
+ Root: common.HexToHash("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"),
+ })
+ account1LeafNode, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("3926db69aaced518e9b9f0f434a473e7174109c943548bb8f23be41ca76d9ad2"),
+ account1,
+ })
+ minerAccount, _ = rlp.EncodeToBytes(state.Account{
+ Nonce: uint64(0),
+ Balance: big.NewInt(2000000000000000000),
+ CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(),
+ Root: common.HexToHash("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"),
+ })
+ minerAccountLeafNode, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("3380c7b7ae81a58eb98d9c78de4a1fd7fd9535fc953ed2be602daaa41767312a"),
+ minerAccount,
+ })
+ bankAccount, _ = rlp.EncodeToBytes(state.Account{
+ Nonce: uint64(1),
+ Balance: big.NewInt(testhelpers.TestBankFunds.Int64() - 10000),
+ CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(),
+ Root: common.HexToHash("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"),
+ })
+ bankAccountLeafNode, _ = rlp.EncodeToBytes([]interface{}{
+ common.Hex2Bytes("30bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a"),
+ bankAccount,
+ })
+ mockTotalDifficulty = big.NewInt(1337)
+ params = statediff.Params{
+ IntermediateStateNodes: false,
+ IncludeTD: true,
+ IncludeBlock: true,
+ IncludeReceipts: true,
+ }
+)
+
+func TestAPI(t *testing.T) {
+ testSubscriptionAPI(t)
+ testHTTPAPI(t)
+}
+
+func testSubscriptionAPI(t *testing.T) {
+ blocks, chain := testhelpers.MakeChain(1, testhelpers.Genesis, testhelpers.TestChainGen)
+ defer chain.Stop()
+ block0 = testhelpers.Genesis
+ block1 = blocks[0]
+ expectedBlockRlp, _ := rlp.EncodeToBytes(block1)
+ mockReceipt := &types.Receipt{
+ BlockNumber: block1.Number(),
+ BlockHash: block1.Hash(),
+ }
+ expectedReceiptBytes, _ := rlp.EncodeToBytes(types.Receipts{mockReceipt})
+ expectedStateDiff := statediff.StateObject{
+ BlockNumber: block1.Number(),
+ BlockHash: block1.Hash(),
+ Nodes: []sdtypes.StateNode{
+ {
+ Path: []byte{'\x05'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: minerLeafKey,
+ NodeValue: minerAccountLeafNode,
+ StorageNodes: emptyStorage,
+ },
+ {
+ Path: []byte{'\x0e'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.Account1LeafKey,
+ NodeValue: account1LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ {
+ Path: []byte{'\x00'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.BankLeafKey,
+ NodeValue: bankAccountLeafNode,
+ StorageNodes: emptyStorage,
+ },
+ },
+ }
+ expectedStateDiffBytes, _ := rlp.EncodeToBytes(expectedStateDiff)
+ blockChan := make(chan *types.Block)
+ parentBlockChain := make(chan *types.Block)
+ serviceQuitChan := make(chan bool)
+ mockBlockChain := &BlockChain{}
+ mockBlockChain.SetReceiptsForHash(block1.Hash(), types.Receipts{mockReceipt})
+ mockBlockChain.SetTdByHash(block1.Hash(), mockTotalDifficulty)
+ mockService := MockStateDiffService{
+ Mutex: sync.Mutex{},
+ Builder: statediff.NewBuilder(chain.StateCache()),
+ BlockChan: blockChan,
+ BlockChain: mockBlockChain,
+ ParentBlockChan: parentBlockChain,
+ QuitChan: serviceQuitChan,
+ Subscriptions: make(map[common.Hash]map[rpc.ID]statediff.Subscription),
+ SubscriptionTypes: make(map[common.Hash]statediff.Params),
+ }
+ mockService.Start()
+ id := rpc.NewID()
+ payloadChan := make(chan statediff.Payload)
+ quitChan := make(chan bool)
+ mockService.Subscribe(id, payloadChan, quitChan, params)
+ blockChan <- block1
+ parentBlockChain <- block0
+
+ sort.Slice(expectedStateDiffBytes, func(i, j int) bool { return expectedStateDiffBytes[i] < expectedStateDiffBytes[j] })
+ select {
+ case payload := <-payloadChan:
+ if !bytes.Equal(payload.BlockRlp, expectedBlockRlp) {
+ t.Errorf("payload does not have expected block\r\nactual block rlp: %v\r\nexpected block rlp: %v", payload.BlockRlp, expectedBlockRlp)
+ }
+ sort.Slice(payload.StateObjectRlp, func(i, j int) bool { return payload.StateObjectRlp[i] < payload.StateObjectRlp[j] })
+ if !bytes.Equal(payload.StateObjectRlp, expectedStateDiffBytes) {
+ t.Errorf("payload does not have expected state diff\r\nactual state diff rlp: %v\r\nexpected state diff rlp: %v", payload.StateObjectRlp, expectedStateDiffBytes)
+ }
+ if !bytes.Equal(expectedReceiptBytes, payload.ReceiptsRlp) {
+ t.Errorf("payload does not have expected receipts\r\nactual receipt rlp: %v\r\nexpected receipt rlp: %v", payload.ReceiptsRlp, expectedReceiptBytes)
+ }
+ if !bytes.Equal(payload.TotalDifficulty.Bytes(), mockTotalDifficulty.Bytes()) {
+ t.Errorf("payload does not have expected total difficulty\r\nactual td: %d\r\nexpected td: %d", payload.TotalDifficulty.Int64(), mockTotalDifficulty.Int64())
+ }
+ case <-quitChan:
+ t.Errorf("channel quit before delivering payload")
+ }
+}
+
+func testHTTPAPI(t *testing.T) {
+ blocks, chain := testhelpers.MakeChain(1, testhelpers.Genesis, testhelpers.TestChainGen)
+ defer chain.Stop()
+ block0 = testhelpers.Genesis
+ block1 = blocks[0]
+ expectedBlockRlp, _ := rlp.EncodeToBytes(block1)
+ mockReceipt := &types.Receipt{
+ BlockNumber: block1.Number(),
+ BlockHash: block1.Hash(),
+ }
+ expectedReceiptBytes, _ := rlp.EncodeToBytes(types.Receipts{mockReceipt})
+ expectedStateDiff := statediff.StateObject{
+ BlockNumber: block1.Number(),
+ BlockHash: block1.Hash(),
+ Nodes: []sdtypes.StateNode{
+ {
+ Path: []byte{'\x05'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: minerLeafKey,
+ NodeValue: minerAccountLeafNode,
+ StorageNodes: emptyStorage,
+ },
+ {
+ Path: []byte{'\x0e'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.Account1LeafKey,
+ NodeValue: account1LeafNode,
+ StorageNodes: emptyStorage,
+ },
+ {
+ Path: []byte{'\x00'},
+ NodeType: sdtypes.Leaf,
+ LeafKey: testhelpers.BankLeafKey,
+ NodeValue: bankAccountLeafNode,
+ StorageNodes: emptyStorage,
+ },
+ },
+ }
+ expectedStateDiffBytes, _ := rlp.EncodeToBytes(expectedStateDiff)
+ mockBlockChain := &BlockChain{}
+ mockBlockChain.SetBlocksForHashes(map[common.Hash]*types.Block{
+ block0.Hash(): block0,
+ block1.Hash(): block1,
+ })
+ mockBlockChain.SetBlockForNumber(block1, block1.Number().Uint64())
+ mockBlockChain.SetReceiptsForHash(block1.Hash(), types.Receipts{mockReceipt})
+ mockBlockChain.SetTdByHash(block1.Hash(), big.NewInt(1337))
+ mockService := MockStateDiffService{
+ Mutex: sync.Mutex{},
+ Builder: statediff.NewBuilder(chain.StateCache()),
+ BlockChain: mockBlockChain,
+ }
+ payload, err := mockService.StateDiffAt(block1.Number().Uint64(), params)
+ if err != nil {
+ t.Error(err)
+ }
+ sort.Slice(payload.StateObjectRlp, func(i, j int) bool { return payload.StateObjectRlp[i] < payload.StateObjectRlp[j] })
+ sort.Slice(expectedStateDiffBytes, func(i, j int) bool { return expectedStateDiffBytes[i] < expectedStateDiffBytes[j] })
+ if !bytes.Equal(payload.BlockRlp, expectedBlockRlp) {
+ t.Errorf("payload does not have expected block\r\nactual block rlp: %v\r\nexpected block rlp: %v", payload.BlockRlp, expectedBlockRlp)
+ }
+ if !bytes.Equal(payload.StateObjectRlp, expectedStateDiffBytes) {
+ t.Errorf("payload does not have expected state diff\r\nactual state diff rlp: %v\r\nexpected state diff rlp: %v", payload.StateObjectRlp, expectedStateDiffBytes)
+ }
+ if !bytes.Equal(expectedReceiptBytes, payload.ReceiptsRlp) {
+ t.Errorf("payload does not have expected receipts\r\nactual receipt rlp: %v\r\nexpected receipt rlp: %v", payload.ReceiptsRlp, expectedReceiptBytes)
+ }
+ if !bytes.Equal(payload.TotalDifficulty.Bytes(), mockTotalDifficulty.Bytes()) {
+ t.Errorf("paylaod does not have the expected total difficulty\r\nactual td: %d\r\nexpected td: %d", payload.TotalDifficulty.Int64(), mockTotalDifficulty.Int64())
+ }
+}
diff --git a/statediff/testhelpers/test_data.go b/statediff/testhelpers/test_data.go
new file mode 100644
index 000000000..cc5374da9
--- /dev/null
+++ b/statediff/testhelpers/test_data.go
@@ -0,0 +1,73 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package testhelpers
+
+import (
+ "math/big"
+ "math/rand"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/core"
+ "github.com/ethereum/go-ethereum/core/rawdb"
+ "github.com/ethereum/go-ethereum/crypto"
+ "github.com/ethereum/go-ethereum/rlp"
+)
+
+// AddressToLeafKey hashes an returns an address
+func AddressToLeafKey(address common.Address) []byte {
+ return crypto.Keccak256(address[:])
+}
+
+// AddressToEncodedPath hashes an address and appends the even-number leaf flag to it
+func AddressToEncodedPath(address common.Address) []byte {
+ addrHash := crypto.Keccak256(address[:])
+ decodedPath := append(EvenLeafFlag, addrHash...)
+ return decodedPath
+}
+
+// Test variables
+var (
+ EvenLeafFlag = []byte{byte(2) << 4}
+ BlockNumber = big.NewInt(rand.Int63())
+ BlockHash = "0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"
+ NullCodeHash = crypto.Keccak256Hash([]byte{})
+ StoragePath = common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes()
+ StorageKey = common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001").Bytes()
+ StorageValue = common.Hex2Bytes("0x03")
+ NullHash = common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000")
+
+ Testdb = rawdb.NewMemoryDatabase()
+ TestBankKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
+ TestBankAddress = crypto.PubkeyToAddress(TestBankKey.PublicKey) //0x71562b71999873DB5b286dF957af199Ec94617F7
+ BankLeafKey = AddressToLeafKey(TestBankAddress)
+ TestBankFunds = big.NewInt(100000000)
+ Genesis = core.GenesisBlockForTesting(Testdb, TestBankAddress, TestBankFunds)
+
+ Account1Key, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a")
+ Account2Key, _ = crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee")
+ Account1Addr = crypto.PubkeyToAddress(Account1Key.PublicKey) //0x703c4b2bD70c169f5717101CaeE543299Fc946C7
+ Account2Addr = crypto.PubkeyToAddress(Account2Key.PublicKey) //0x0D3ab14BBaD3D99F4203bd7a11aCB94882050E7e
+ Account1LeafKey = AddressToLeafKey(Account1Addr)
+ Account2LeafKey = AddressToLeafKey(Account2Addr)
+ ContractCode = common.Hex2Bytes("608060405234801561001057600080fd5b50336000806101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff1602179055506040518060200160405280600160ff16815250600190600161007492919061007a565b506100e4565b82606481019282156100ae579160200282015b828111156100ad578251829060ff1690559160200191906001019061008d565b5b5090506100bb91906100bf565b5090565b6100e191905b808211156100dd5760008160009055506001016100c5565b5090565b90565b6101ca806100f36000396000f3fe608060405234801561001057600080fd5b50600436106100365760003560e01c806343d726d61461003b578063c16431b914610045575b600080fd5b61004361007d565b005b61007b6004803603604081101561005b57600080fd5b81019080803590602001909291908035906020019092919050505061015c565b005b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff1614610122576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260228152602001806101746022913960400191505060405180910390fd5b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16ff5b806001836064811061016a57fe5b0181905550505056fe4f6e6c79206f776e65722063616e2063616c6c20746869732066756e6374696f6e2ea265627a7a72305820e3747183708fb6bff3f6f7a80fb57dcc1c19f83f9cb25457a3ed5c0424bde66864736f6c634300050a0032")
+ ByteCodeAfterDeployment = common.Hex2Bytes("608060405234801561001057600080fd5b50600436106100365760003560e01c806343d726d61461003b578063c16431b914610045575b600080fd5b61004361007d565b005b61007b6004803603604081101561005b57600080fd5b81019080803590602001909291908035906020019092919050505061015c565b005b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff1614610122576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260228152602001806101746022913960400191505060405180910390fd5b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16ff5b806001836064811061016a57fe5b0181905550505056fe4f6e6c79206f776e65722063616e2063616c6c20746869732066756e6374696f6e2ea265627a7a72305820e3747183708fb6bff3f6f7a80fb57dcc1c19f83f9cb25457a3ed5c0424bde66864736f6c634300050a0032")
+ CodeHash = common.HexToHash("0xaaea5efba4fd7b45d7ec03918ac5d8b31aa93b48986af0e6b591f0f087c80127")
+ ContractAddr common.Address
+
+ EmptyRootNode, _ = rlp.EncodeToBytes([]byte{})
+ EmptyContractRoot = crypto.Keccak256Hash(EmptyRootNode)
+)
diff --git a/statediff/types.go b/statediff/types.go
new file mode 100644
index 000000000..148567dd7
--- /dev/null
+++ b/statediff/types.go
@@ -0,0 +1,113 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+// Contains a batch of utility type declarations used by the tests. As the node
+// operates on unique types, a lot of them are needed to check various features.
+
+package statediff
+
+import (
+ "encoding/json"
+ "math/big"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/core/state"
+ "github.com/ethereum/go-ethereum/statediff/types"
+)
+
+// Subscription struct holds our subscription channels
+type Subscription struct {
+ PayloadChan chan<- Payload
+ QuitChan chan<- bool
+}
+
+// DBParams holds params for Postgres db connection
+type DBParams struct {
+ ConnectionURL string
+ ID string
+ ClientName string
+}
+
+// Params is used to carry in parameters from subscribing/requesting clients configuration
+type Params struct {
+ IntermediateStateNodes bool
+ IntermediateStorageNodes bool
+ IncludeBlock bool
+ IncludeReceipts bool
+ IncludeTD bool
+ IncludeCode bool
+ WatchedAddresses []common.Address
+ WatchedStorageSlots []common.Hash
+}
+
+// Args bundles the arguments for the state diff builder
+type Args struct {
+ OldStateRoot, NewStateRoot, BlockHash common.Hash
+ BlockNumber *big.Int
+}
+
+type StateRoots struct {
+ OldStateRoot, NewStateRoot common.Hash
+}
+
+// Payload packages the data to send to statediff subscriptions
+type Payload struct {
+ BlockRlp []byte `json:"blockRlp"`
+ TotalDifficulty *big.Int `json:"totalDifficulty"`
+ ReceiptsRlp []byte `json:"receiptsRlp"`
+ StateObjectRlp []byte `json:"stateObjectRlp" gencodec:"required"`
+
+ encoded []byte
+ err error
+}
+
+func (sd *Payload) ensureEncoded() {
+ if sd.encoded == nil && sd.err == nil {
+ sd.encoded, sd.err = json.Marshal(sd)
+ }
+}
+
+// Length to implement Encoder interface for Payload
+func (sd *Payload) Length() int {
+ sd.ensureEncoded()
+ return len(sd.encoded)
+}
+
+// Encode to implement Encoder interface for Payload
+func (sd *Payload) Encode() ([]byte, error) {
+ sd.ensureEncoded()
+ return sd.encoded, sd.err
+}
+
+// StateObject is the final output structure from the builder
+type StateObject struct {
+ BlockNumber *big.Int `json:"blockNumber" gencodec:"required"`
+ BlockHash common.Hash `json:"blockHash" gencodec:"required"`
+ Nodes []types.StateNode `json:"nodes" gencodec:"required"`
+ CodeAndCodeHashes []types.CodeAndCodeHash `json:"codeMapping"`
+}
+
+// AccountMap is a mapping of hex encoded path => account wrapper
+type AccountMap map[string]accountWrapper
+
+// accountWrapper is used to temporary associate the unpacked node with its raw values
+type accountWrapper struct {
+ Account *state.Account
+ NodeType types.NodeType
+ Path []byte
+ NodeValue []byte
+ LeafKey []byte
+}
diff --git a/statediff/types/types.go b/statediff/types/types.go
new file mode 100644
index 000000000..08e2124fa
--- /dev/null
+++ b/statediff/types/types.go
@@ -0,0 +1,61 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+// Contains a batch of utility type declarations used by the tests. As the node
+// operates on unique types, a lot of them are needed to check various features.
+
+package types
+
+import "github.com/ethereum/go-ethereum/common"
+
+// NodeType for explicitly setting type of node
+type NodeType string
+
+const (
+ Unknown NodeType = "Unknown"
+ Leaf NodeType = "Leaf"
+ Extension NodeType = "Extension"
+ Branch NodeType = "Branch"
+ Removed NodeType = "Removed" // used to represent pathes which have been emptied
+)
+
+// StateNode holds the data for a single state diff node
+type StateNode struct {
+ NodeType NodeType `json:"nodeType" gencodec:"required"`
+ Path []byte `json:"path" gencodec:"required"`
+ NodeValue []byte `json:"value" gencodec:"required"`
+ StorageNodes []StorageNode `json:"storage"`
+ LeafKey []byte `json:"leafKey"`
+}
+
+// StorageNode holds the data for a single storage diff node
+type StorageNode struct {
+ NodeType NodeType `json:"nodeType" gencodec:"required"`
+ Path []byte `json:"path" gencodec:"required"`
+ NodeValue []byte `json:"value" gencodec:"required"`
+ LeafKey []byte `json:"leafKey"`
+}
+
+// CodeAndCodeHash struct for holding codehash => code mappings
+// we can't use an actual map because they are not rlp serializable
+type CodeAndCodeHash struct {
+ Hash common.Hash `json:"codeHash"`
+ Code []byte `json:"code"`
+}
+
+type StateNodeSink func(StateNode) error
+type StorageNodeSink func(StorageNode) error
+type CodeSink func(CodeAndCodeHash) error
diff --git a/trie/encoding.go b/trie/encoding.go
index 8ee0022ef..3a5f632e2 100644
--- a/trie/encoding.go
+++ b/trie/encoding.go
@@ -34,6 +34,11 @@ package trie
// in the case of an odd number. All remaining nibbles (now an even number) fit properly
// into the remaining bytes. Compact encoding is used for nodes stored on disk.
+// HexToCompact converts a hex path to the compact encoded format
+func HexToCompact(hex []byte) []byte {
+ return hexToCompact(hex)
+}
+
func hexToCompact(hex []byte) []byte {
terminator := byte(0)
if hasTerm(hex) {
@@ -55,8 +60,8 @@ func hexToCompact(hex []byte) []byte {
// needed for the representation
func hexToCompactInPlace(hex []byte) int {
var (
- hexLen = len(hex) // length of the hex input
- firstByte = byte(0)
+ hexLen= len(hex) // length of the hex input
+ firstByte= byte(0)
)
// Check if we have a terminator there
if hexLen > 0 && hex[hexLen-1] == 16 {
@@ -64,9 +69,9 @@ func hexToCompactInPlace(hex []byte) int {
hexLen-- // last part was the terminator, ignore that
}
var (
- binLen = hexLen/2 + 1
- ni = 0 // index in hex
- bi = 1 // index in bin (compact)
+ binLen= hexLen/2 + 1
+ ni= 0 // index in hex
+ bi= 1 // index in bin (compact)
)
if hexLen&1 == 1 {
firstByte |= 1 << 4 // odd flag
@@ -80,6 +85,11 @@ func hexToCompactInPlace(hex []byte) int {
return binLen
}
+// CompactToHex converts a compact encoded path to hex format
+func CompactToHex(compact []byte) []byte {
+ return compactToHex(compact)
+}
+
func compactToHex(compact []byte) []byte {
if len(compact) == 0 {
return compact
@@ -105,9 +115,9 @@ func keybytesToHex(str []byte) []byte {
return nibbles
}
-// hexToKeybytes turns hex nibbles into key bytes.
+// hexToKeyBytes turns hex nibbles into key bytes.
// This can only be used for keys of even length.
-func hexToKeybytes(hex []byte) []byte {
+func hexToKeyBytes(hex []byte) []byte {
if hasTerm(hex) {
hex = hex[:len(hex)-1]
}
diff --git a/trie/encoding_test.go b/trie/encoding_test.go
index 16393313f..7ade0a095 100644
--- a/trie/encoding_test.go
+++ b/trie/encoding_test.go
@@ -71,8 +71,8 @@ func TestHexKeybytes(t *testing.T) {
if h := keybytesToHex(test.key); !bytes.Equal(h, test.hexOut) {
t.Errorf("keybytesToHex(%x) -> %x, want %x", test.key, h, test.hexOut)
}
- if k := hexToKeybytes(test.hexIn); !bytes.Equal(k, test.key) {
- t.Errorf("hexToKeybytes(%x) -> %x, want %x", test.hexIn, k, test.key)
+ if k := hexToKeyBytes(test.hexIn); !bytes.Equal(k, test.key) {
+ t.Errorf("hexToKeyBytes(%x) -> %x, want %x", test.hexIn, k, test.key)
}
}
}
@@ -135,6 +135,6 @@ func BenchmarkKeybytesToHex(b *testing.B) {
func BenchmarkHexToKeybytes(b *testing.B) {
testBytes := []byte{7, 6, 6, 5, 7, 2, 6, 2, 16}
for i := 0; i < b.N; i++ {
- hexToKeybytes(testBytes)
+ hexToKeyBytes(testBytes)
}
}
diff --git a/trie/iterator.go b/trie/iterator.go
index 76d437c40..c4fce17df 100644
--- a/trie/iterator.go
+++ b/trie/iterator.go
@@ -164,7 +164,7 @@ func (it *nodeIterator) Leaf() bool {
func (it *nodeIterator) LeafKey() []byte {
if len(it.stack) > 0 {
if _, ok := it.stack[len(it.stack)-1].node.(valueNode); ok {
- return hexToKeybytes(it.path)
+ return hexToKeyBytes(it.path)
}
}
panic("not at leaf")
diff --git a/trie/sync.go b/trie/sync.go
index bc93ddd3f..a39351b06 100644
--- a/trie/sync.go
+++ b/trie/sync.go
@@ -82,7 +82,7 @@ func newSyncPath(path []byte) SyncPath {
if len(path) < 64 {
return SyncPath{hexToCompact(path)}
}
- return SyncPath{hexToKeybytes(path[:64]), hexToCompact(path[64:])}
+ return SyncPath{hexToKeyBytes(path[:64]), hexToCompact(path[64:])}
}
// SyncResult is a response with requested data along with it's hash.