Compare commits
5 Commits
7b47871269
...
be7b6f5a33
Author | SHA1 | Date | |
---|---|---|---|
be7b6f5a33 | |||
07614a8719 | |||
90f64c2a94 | |||
73292c4046 | |||
472b2a7f61 |
@ -9,13 +9,13 @@ ADD . /go/src/github.com/vulcanize/eth-statediff-service
|
||||
#RUN git clone https://github.com/vulcanize/eth-statediff-service.git /go/src/github.com/vulcanize/eth-statediff-service
|
||||
|
||||
WORKDIR /go/src/github.com/vulcanize/eth-statediff-service
|
||||
RUN GO111MODULE=on GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o eth-statediff-service .
|
||||
RUN GO111MODULE=on CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o eth-statediff-service .
|
||||
|
||||
# app container
|
||||
FROM alpine
|
||||
|
||||
ARG USER="vdm"
|
||||
ARG CONFIG_FILE="./environments/config.toml"
|
||||
ARG USER="vdbm"
|
||||
ARG CONFIG_FILE="./environments/docker.toml"
|
||||
ARG EXPOSE_PORT=8545
|
||||
|
||||
RUN adduser -Du 5000 $USER adm
|
||||
|
12
Makefile
12
Makefile
@ -1,12 +0,0 @@
|
||||
## Build docker image
|
||||
.PHONY: docker-build
|
||||
docker-build:
|
||||
docker build -t vulcanize/eth-statediff-service .
|
||||
|
||||
.PHONY: test
|
||||
test: | $(GOOSE)
|
||||
go test -p 1 ./pkg/... -v
|
||||
|
||||
build:
|
||||
go fmt ./...
|
||||
go build
|
130
README.md
130
README.md
@ -10,15 +10,7 @@ Stand up a statediffing service directly on top of a go-ethereum LevelDB instanc
|
||||
This service can serve historical state data over the same rpc interface as
|
||||
[statediffing geth](https://github.com/vulcanize/go-ethereum/releases/tag/v1.9.11-statediff-0.0.5) without needing to run a full node
|
||||
|
||||
## Setup
|
||||
|
||||
Build the binary:
|
||||
|
||||
```bash
|
||||
make build
|
||||
```
|
||||
|
||||
### Local Setup
|
||||
## Configuration
|
||||
|
||||
* Create a chain config file `chain.json` according to chain config in genesis json file used by local geth.
|
||||
|
||||
@ -42,7 +34,7 @@ make build
|
||||
}
|
||||
```
|
||||
|
||||
* Change the following in [config file](./environments/config.toml)
|
||||
* Sample database and chain configuration (from [environments/config.toml](./environments/config.toml)):
|
||||
|
||||
```toml
|
||||
[leveldb]
|
||||
@ -53,7 +45,7 @@ make build
|
||||
|
||||
[ethereum]
|
||||
chainConfig = "./chain.json" # Path to custom chain config file
|
||||
chainID = 41337 # Same chain ID as in chain.json
|
||||
chainID = 41337 # Same chain ID as in chain.json
|
||||
|
||||
[database]
|
||||
# Update database config
|
||||
@ -65,22 +57,42 @@ make build
|
||||
type = "postgres"
|
||||
```
|
||||
|
||||
* To write statediff for a range of block make changes in [config file](./environments/config.toml)
|
||||
* Service and metrics configuration:
|
||||
|
||||
```toml
|
||||
[prerun]
|
||||
only = false
|
||||
ranges = [
|
||||
[8, 15] # Block number range for which to write statediff.
|
||||
]
|
||||
[statediff]
|
||||
serviceWorkers = 1 # Number of diffs to process concurrently
|
||||
workerQueueSize = 1024 # Size of buffer for block range requests
|
||||
trieWorkers = 4 # Number of state subtries to process concurrently
|
||||
|
||||
[cache]
|
||||
database = 1024 # Trie node cache size in MB
|
||||
trie = 1024 # LevelDB cache size in MiB
|
||||
|
||||
[prom]
|
||||
dbStats = false
|
||||
metrics = true
|
||||
http = true
|
||||
httpAddr = "localhost"
|
||||
httpPort = "8889"
|
||||
```
|
||||
|
||||
* To use remote LevelDB RPC endpoint change the following in [config file](./environments/config.toml)
|
||||
* To use a remote LevelDB RPC endpoint change the following in [config file](./environments/config.toml)
|
||||
```toml
|
||||
[leveldb]
|
||||
mode = "remote"
|
||||
url = "http://127.0.0.1:8082/" # Remote LevelDB RPC url
|
||||
```
|
||||
|
||||
* When using the `run` command to write diffs for specific block ranges, add this:
|
||||
```toml
|
||||
[run]
|
||||
only = false
|
||||
ranges = [
|
||||
[8, 15] # Block number range for which to write statediff.
|
||||
]
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
### `serve`
|
||||
@ -96,88 +108,18 @@ Example:
|
||||
```
|
||||
|
||||
Available RPC methods are:
|
||||
* `statediff_stateTrieAt()`
|
||||
* `statediff_streamCodeAndCodeHash()`
|
||||
* `statediff_stateDiffAt()`
|
||||
* `statediff_writeStateDiffAt()`
|
||||
* `statediff_writeStateDiffsInRange()`
|
||||
|
||||
e.g. `curl -X POST -H 'Content-Type: application/json' --data '{"jsonrpc":"2.0","method":"statediff_writeStateDiffsInRange","params":['"$BEGIN"', '"$END"', {"intermediateStateNodes":true,"intermediateStorageNodes":true,"includeBlock":true,"includeReceipts":true,"includeTD":true,"includeCode":true}],"id":1}' "$HOST":"$PORT"`
|
||||
e.g. `curl -X POST -H 'Content-Type: application/json' --data '{"jsonrpc":"2.0","method":"statediff_writeStateDiffsInRange","params":['"$BEGIN"', '"$END"', {"includeBlock":true,"includeReceipts":true,"includeTD":true,"includeCode":true}],"id":1}' "$HOST":"$PORT"`
|
||||
|
||||
The process can be configured locally with sets of ranges to process as a "prerun" to processing directed by the server endpoints.
|
||||
This is done by turning "prerun" on in the config (`statediff.prerun = true`) and defining ranged and params in the
|
||||
`prerun` section of the config as shown below.
|
||||
### `run`
|
||||
|
||||
## Configuration
|
||||
Produces diffs for specific block ranges.
|
||||
|
||||
An example config file:
|
||||
Example:
|
||||
|
||||
```toml
|
||||
[leveldb]
|
||||
mode = "local"
|
||||
# path and ancient LevelDB paths required in local mode
|
||||
path = "/Users/user/Library/Ethereum/geth/chaindata"
|
||||
ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient"
|
||||
# url for leveldb-ethdb-rpc endpoint required in remote mode
|
||||
url = "http://127.0.0.1:8082/"
|
||||
|
||||
[server]
|
||||
ipcPath = ".ipc"
|
||||
httpPath = "127.0.0.1:8545"
|
||||
|
||||
[statediff]
|
||||
prerun = true
|
||||
serviceWorkers = 1
|
||||
workerQueueSize = 1024
|
||||
trieWorkers = 4
|
||||
|
||||
[prerun]
|
||||
only = false
|
||||
ranges = [
|
||||
[0, 1000]
|
||||
]
|
||||
[prerun.params]
|
||||
intermediateStateNodes = true
|
||||
intermediateStorageNodes = true
|
||||
includeBlock = true
|
||||
includeReceipts = true
|
||||
includeTD = true
|
||||
includeCode = true
|
||||
watchedAddresses = []
|
||||
|
||||
[log]
|
||||
file = ""
|
||||
level = "info"
|
||||
|
||||
[eth]
|
||||
chainID = 1
|
||||
|
||||
[database]
|
||||
name = "vulcanize_test"
|
||||
hostname = "localhost"
|
||||
port = 5432
|
||||
user = "vulcanize"
|
||||
password = "..."
|
||||
type = "postgres"
|
||||
driver = "sqlx"
|
||||
dumpDestination = ""
|
||||
filePath = ""
|
||||
|
||||
[cache]
|
||||
database = 1024
|
||||
trie = 1024
|
||||
|
||||
[prom]
|
||||
dbStats = false
|
||||
metrics = true
|
||||
http = true
|
||||
httpAddr = "localhost"
|
||||
httpPort = "8889"
|
||||
|
||||
[ethereum]
|
||||
nodeID = ""
|
||||
clientName = "eth-statediff-service"
|
||||
genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3"
|
||||
networkID = 1
|
||||
chainID = 1
|
||||
```bash
|
||||
./eth-statediff-service run --config environments/config.toml --run.ranges '[8,15]'
|
||||
```
|
||||
|
28
cmd/env.go
28
cmd/env.go
@ -35,7 +35,6 @@ const (
|
||||
LVLDB_ANCIENT = "LVLDB_ANCIENT"
|
||||
LVLDB_URL = "LVLDB_URL"
|
||||
|
||||
STATEDIFF_PRERUN = "STATEDIFF_PRERUN"
|
||||
STATEDIFF_TRIE_WORKERS = "STATEDIFF_TRIE_WORKERS"
|
||||
STATEDIFF_SERVICE_WORKERS = "STATEDIFF_SERVICE_WORKERS"
|
||||
STATEDIFF_WORKER_QUEUE_SIZE = "STATEDIFF_WORKER_QUEUE_SIZE"
|
||||
@ -49,15 +48,10 @@ const (
|
||||
PROM_HTTP_PORT = "PROM_HTTP_PORT"
|
||||
PROM_DB_STATS = "PROM_DB_STATS"
|
||||
|
||||
PRERUN_ONLY = "PRERUN_ONLY"
|
||||
PRERUN_RANGE_START = "PRERUN_RANGE_START"
|
||||
PRERUN_RANGE_STOP = "PRERUN_RANGE_STOP"
|
||||
PRERUN_INTERMEDIATE_STATE_NODES = "PRERUN_INTERMEDIATE_STATE_NODES"
|
||||
PRERUN_INTERMEDIATE_STORAGE_NODES = "PRERUN_INTERMEDIATE_STORAGE_NODES"
|
||||
PRERUN_INCLUDE_BLOCK = "PRERUN_INCLUDE_BLOCK"
|
||||
PRERUN_INCLUDE_RECEIPTS = "PRERUN_INCLUDE_RECEIPTS"
|
||||
PRERUN_INCLUDE_TD = "PRERUN_INCLUDE_TD"
|
||||
PRERUN_INCLUDE_CODE = "PRERUN_INCLUDE_CODE"
|
||||
RUN_INCLUDE_BLOCK = "RUN_INCLUDE_BLOCK"
|
||||
RUN_INCLUDE_RECEIPTS = "RUN_INCLUDE_RECEIPTS"
|
||||
RUN_INCLUDE_TD = "RUN_INCLUDE_TD"
|
||||
RUN_INCLUDE_CODE = "RUN_INCLUDE_CODE"
|
||||
|
||||
LOG_LEVEL = "LOG_LEVEL"
|
||||
LOG_FILE_PATH = "LOG_FILE_PATH"
|
||||
@ -129,16 +123,10 @@ func init() {
|
||||
viper.BindEnv("statediff.trieWorkers", STATEDIFF_TRIE_WORKERS)
|
||||
viper.BindEnv("statediff.workerQueueSize", STATEDIFF_WORKER_QUEUE_SIZE)
|
||||
|
||||
viper.BindEnv("statediff.prerun", STATEDIFF_PRERUN)
|
||||
viper.BindEnv("prerun.only", PRERUN_ONLY)
|
||||
viper.BindEnv("prerun.start", PRERUN_RANGE_START)
|
||||
viper.BindEnv("prerun.stop", PRERUN_RANGE_STOP)
|
||||
viper.BindEnv("prerun.params.intermediateStateNodes", PRERUN_INTERMEDIATE_STATE_NODES)
|
||||
viper.BindEnv("prerun.params.intermediateStorageNodes", PRERUN_INTERMEDIATE_STORAGE_NODES)
|
||||
viper.BindEnv("prerun.params.includeBlock", PRERUN_INCLUDE_BLOCK)
|
||||
viper.BindEnv("prerun.params.includeReceipts", PRERUN_INCLUDE_RECEIPTS)
|
||||
viper.BindEnv("prerun.params.includeTD", PRERUN_INCLUDE_TD)
|
||||
viper.BindEnv("prerun.params.includeCode", PRERUN_INCLUDE_CODE)
|
||||
viper.BindEnv("run.params.includeBlock", RUN_INCLUDE_BLOCK)
|
||||
viper.BindEnv("run.params.includeReceipts", RUN_INCLUDE_RECEIPTS)
|
||||
viper.BindEnv("run.params.includeTD", RUN_INCLUDE_TD)
|
||||
viper.BindEnv("run.params.includeCode", RUN_INCLUDE_CODE)
|
||||
|
||||
viper.BindEnv("log.level", LOG_LEVEL)
|
||||
viper.BindEnv("log.file", LOG_FILE_PATH)
|
||||
|
19
cmd/root.go
19
cmd/root.go
@ -45,17 +45,16 @@ var (
|
||||
|
||||
var rootCmd = &cobra.Command{
|
||||
Use: "eth-statediff-service",
|
||||
PersistentPreRun: initFuncs,
|
||||
PersistentPreRun: setupLoggingAndMetrics,
|
||||
}
|
||||
|
||||
func Execute() {
|
||||
log.Info("----- Starting vDB -----")
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func initFuncs(cmd *cobra.Command, args []string) {
|
||||
func setupLoggingAndMetrics(cmd *cobra.Command, args []string) {
|
||||
logfile := viper.GetString("log.file")
|
||||
if logfile != "" {
|
||||
file, err := os.OpenFile(logfile,
|
||||
@ -121,7 +120,6 @@ func init() {
|
||||
rootCmd.PersistentFlags().String("ancient-path", "", "path to ancient datastore")
|
||||
rootCmd.PersistentFlags().String("leveldb-url", "", "url to primary leveldb-ethdb-rpc server")
|
||||
|
||||
rootCmd.PersistentFlags().Bool("prerun", false, "turn on prerun of toml configured ranges")
|
||||
rootCmd.PersistentFlags().Int("service-workers", 0, "number of range requests to process concurrently")
|
||||
rootCmd.PersistentFlags().Int("trie-workers", 0, "number of workers to use for trie traversal and processing")
|
||||
rootCmd.PersistentFlags().Int("worker-queue-size", 0, "size of the range request queue for service workers")
|
||||
@ -158,11 +156,6 @@ func init() {
|
||||
rootCmd.PersistentFlags().Bool("prom-db-stats", false, "enables prometheus db stats")
|
||||
rootCmd.PersistentFlags().Bool("prom-metrics", false, "enable prometheus metrics")
|
||||
|
||||
rootCmd.PersistentFlags().Bool("prerun-only", false, "only process pre-configured ranges; exit afterwards")
|
||||
rootCmd.PersistentFlags().Int("prerun-start", 0, "start height for a prerun range")
|
||||
rootCmd.PersistentFlags().Int("prerun-stop", 0, "stop height for a prerun range")
|
||||
rootCmd.PersistentFlags().Bool("prerun-intermediate-state-nodes", true, "include intermediate state nodes in state diff")
|
||||
rootCmd.PersistentFlags().Bool("prerun-intermediate-storage-nodes", true, "include intermediate storage nodes in state diff")
|
||||
rootCmd.PersistentFlags().Bool("prerun-include-block", true, "include block data in the statediff payload")
|
||||
rootCmd.PersistentFlags().Bool("prerun-include-receipts", true, "include receipts in the statediff payload")
|
||||
rootCmd.PersistentFlags().Bool("prerun-include-td", true, "include td in the statediff payload")
|
||||
@ -174,7 +167,6 @@ func init() {
|
||||
viper.BindPFlag("log.file", rootCmd.PersistentFlags().Lookup("log-file"))
|
||||
viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level"))
|
||||
|
||||
viper.BindPFlag("statediff.prerun", rootCmd.PersistentFlags().Lookup("prerun"))
|
||||
viper.BindPFlag("statediff.serviceWorkers", rootCmd.PersistentFlags().Lookup("service-workers"))
|
||||
viper.BindPFlag("statediff.trieWorkers", rootCmd.PersistentFlags().Lookup("trie-workers"))
|
||||
viper.BindPFlag("statediff.workerQueueSize", rootCmd.PersistentFlags().Lookup("worker-queue-size"))
|
||||
@ -216,11 +208,6 @@ func init() {
|
||||
viper.BindPFlag("prom.dbStats", rootCmd.PersistentFlags().Lookup("prom-db-stats"))
|
||||
viper.BindPFlag("prom.metrics", rootCmd.PersistentFlags().Lookup("prom-metrics"))
|
||||
|
||||
viper.BindPFlag("prerun.only", rootCmd.PersistentFlags().Lookup("prerun-only"))
|
||||
viper.BindPFlag("prerun.start", rootCmd.PersistentFlags().Lookup("prerun-start"))
|
||||
viper.BindPFlag("prerun.stop", rootCmd.PersistentFlags().Lookup("prerun-stop"))
|
||||
viper.BindPFlag("prerun.params.intermediateStateNodes", rootCmd.PersistentFlags().Lookup("prerun-intermediate-state-nodes"))
|
||||
viper.BindPFlag("prerun.params.intermediateStorageNodes", rootCmd.PersistentFlags().Lookup("prerun-intermediate-storage-nodes"))
|
||||
viper.BindPFlag("prerun.params.includeBlock", rootCmd.PersistentFlags().Lookup("prerun-include-block"))
|
||||
viper.BindPFlag("prerun.params.includeReceipts", rootCmd.PersistentFlags().Lookup("prerun-include-receipts"))
|
||||
viper.BindPFlag("prerun.params.includeTD", rootCmd.PersistentFlags().Lookup("prerun-include-td"))
|
||||
@ -320,7 +307,7 @@ func getConfig(nodeInfo node.Info) (interfaces.Config, error) {
|
||||
case dump.STDOUT:
|
||||
indexerConfig = dump.Config{Dump: os.Stderr}
|
||||
case dump.DISCARD:
|
||||
indexerConfig = dump.Config{Dump: dump.NewDiscardWriterCloser()}
|
||||
indexerConfig = dump.Config{Dump: dump.Discard}
|
||||
default:
|
||||
return nil, fmt.Errorf("unrecognized dump destination: %s", dumpDst)
|
||||
}
|
||||
|
91
cmd/run.go
Normal file
91
cmd/run.go
Normal file
@ -0,0 +1,91 @@
|
||||
// Copyright © 2023 Vulcanize, Inc
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/signal"
|
||||
"sync"
|
||||
|
||||
statediff "github.com/cerc-io/plugeth-statediff"
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/viper"
|
||||
|
||||
pkg "github.com/vulcanize/eth-statediff-service/pkg"
|
||||
)
|
||||
|
||||
// serveCmd represents the serve command
|
||||
var runCmd = &cobra.Command{
|
||||
Use: "run",
|
||||
Short: "Produce diffs for a specific block range",
|
||||
Long: `Usage
|
||||
|
||||
./eth-statediff-service run --config={path to toml config file}`,
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
subCommand = cmd.CalledAs()
|
||||
logWithCommand = *logrus.WithField("SubCommand", subCommand)
|
||||
runRanges()
|
||||
},
|
||||
}
|
||||
|
||||
func runRanges() {
|
||||
service := createStateDiffService()
|
||||
|
||||
// start service and servers
|
||||
var wg sync.WaitGroup
|
||||
if err := service.Loop(&wg); err != nil {
|
||||
logWithCommand.Fatalf("unable to start statediff service: %v", err)
|
||||
}
|
||||
ranges := getConfiguredRanges()
|
||||
service.Run(ranges)
|
||||
|
||||
// clean shutdown
|
||||
shutdown := make(chan os.Signal)
|
||||
signal.Notify(shutdown, os.Interrupt)
|
||||
<-shutdown
|
||||
logWithCommand.Info("Received interrupt signal, shutting down")
|
||||
service.Stop()
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func getConfiguredRanges() []pkg.RangeRequest {
|
||||
params := statediff.Params{
|
||||
IncludeBlock: viper.GetBool("run.params.includeBlock"),
|
||||
IncludeReceipts: viper.GetBool("run.params.includeReceipts"),
|
||||
IncludeTD: viper.GetBool("run.params.includeTD"),
|
||||
IncludeCode: viper.GetBool("run.params.includeCode"),
|
||||
}
|
||||
var addrStrs []string
|
||||
viper.UnmarshalKey("run.params.watchedAddresses", &addrStrs)
|
||||
addrs := make([]common.Address, len(addrStrs))
|
||||
for i, addrStr := range addrStrs {
|
||||
addrs[i] = common.HexToAddress(addrStr)
|
||||
}
|
||||
params.WatchedAddresses = addrs
|
||||
var rawRanges []blockRange
|
||||
viper.UnmarshalKey("run.ranges", &rawRanges)
|
||||
blockRanges := make([]pkg.RangeRequest, len(rawRanges))
|
||||
for i, rawRange := range rawRanges {
|
||||
blockRanges[i] = pkg.RangeRequest{
|
||||
Start: rawRange[0],
|
||||
Stop: rawRange[1],
|
||||
Params: params,
|
||||
}
|
||||
}
|
||||
return blockRanges
|
||||
}
|
23
cmd/serve.go
23
cmd/serve.go
@ -50,27 +50,16 @@ func init() {
|
||||
func serve() {
|
||||
logWithCommand.Info("Running eth-statediff-service serve command")
|
||||
|
||||
statediffService, err := createStateDiffService()
|
||||
if err != nil {
|
||||
logWithCommand.Fatal(err)
|
||||
}
|
||||
|
||||
// short circuit if we only want to perform prerun
|
||||
if viper.GetBool("prerun.only") {
|
||||
if err := statediffService.Run(nil); err != nil {
|
||||
logWithCommand.Fatal("unable to perform prerun: %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
service := createStateDiffService()
|
||||
|
||||
// start service and servers
|
||||
logWithCommand.Info("Starting statediff service")
|
||||
wg := new(sync.WaitGroup)
|
||||
if err := statediffService.Loop(wg); err != nil {
|
||||
var wg sync.WaitGroup
|
||||
if err := service.Loop(&wg); err != nil {
|
||||
logWithCommand.Fatalf("unable to start statediff service: %v", err)
|
||||
}
|
||||
logWithCommand.Info("Starting RPC servers")
|
||||
if err := startServers(statediffService); err != nil {
|
||||
if err := startServers(service); err != nil {
|
||||
logWithCommand.Fatal(err)
|
||||
}
|
||||
logWithCommand.Info("RPC servers successfully spun up; awaiting requests")
|
||||
@ -80,11 +69,11 @@ func serve() {
|
||||
signal.Notify(shutdown, os.Interrupt)
|
||||
<-shutdown
|
||||
logWithCommand.Info("Received interrupt signal, shutting down")
|
||||
statediffService.Stop()
|
||||
service.Stop()
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func startServers(serv sd.StateDiffService) error {
|
||||
func startServers(serv *sd.Service) error {
|
||||
ipcPath := viper.GetString("server.ipcPath")
|
||||
httpPath := viper.GetString("server.httpPath")
|
||||
if ipcPath == "" && httpPath == "" {
|
||||
|
60
cmd/util.go
60
cmd/util.go
@ -2,10 +2,13 @@ package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/cerc-io/plugeth-statediff"
|
||||
gethsd "github.com/cerc-io/plugeth-statediff"
|
||||
ind "github.com/cerc-io/plugeth-statediff/indexer"
|
||||
"github.com/cerc-io/plugeth-statediff/utils/log"
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/params"
|
||||
"github.com/ethereum/go-ethereum/trie"
|
||||
@ -16,7 +19,7 @@ import (
|
||||
|
||||
type blockRange [2]uint64
|
||||
|
||||
func createStateDiffService() (sd.StateDiffService, error) {
|
||||
func createStateDiffService() *sd.Service {
|
||||
// load some necessary params
|
||||
logWithCommand.Info("Loading statediff service parameters")
|
||||
mode := viper.GetString("leveldb.mode")
|
||||
@ -41,13 +44,7 @@ func createStateDiffService() (sd.StateDiffService, error) {
|
||||
var chainConf *params.ChainConfig
|
||||
var err error
|
||||
chainConfigPath := viper.GetString("ethereum.chainConfig")
|
||||
|
||||
if chainConfigPath != "" {
|
||||
chainConf, err = statediff.LoadConfig(chainConfigPath)
|
||||
} else {
|
||||
chainConf, err = statediff.ChainConfig(nodeInfo.ChainID)
|
||||
}
|
||||
|
||||
chainConf, err = LoadConfig(chainConfigPath)
|
||||
if err != nil {
|
||||
logWithCommand.Fatal(err)
|
||||
}
|
||||
@ -84,53 +81,10 @@ func createStateDiffService() (sd.StateDiffService, error) {
|
||||
logWithCommand.Fatal(err)
|
||||
}
|
||||
logWithCommand.Info("Creating statediff service")
|
||||
sdConf := sd.Config{
|
||||
sdConf := sd.ServiceConfig{
|
||||
ServiceWorkers: viper.GetUint("statediff.serviceWorkers"),
|
||||
TrieWorkers: viper.GetUint("statediff.trieWorkers"),
|
||||
WorkerQueueSize: viper.GetUint("statediff.workerQueueSize"),
|
||||
PreRuns: setupPreRunRanges(),
|
||||
}
|
||||
return sd.NewStateDiffService(lvlDBReader, indexer, sdConf)
|
||||
}
|
||||
|
||||
func setupPreRunRanges() []sd.RangeRequest {
|
||||
if !viper.GetBool("statediff.prerun") {
|
||||
return nil
|
||||
}
|
||||
preRunParams := gethsd.Params{
|
||||
IntermediateStateNodes: viper.GetBool("prerun.params.intermediateStateNodes"),
|
||||
IntermediateStorageNodes: viper.GetBool("prerun.params.intermediateStorageNodes"),
|
||||
IncludeBlock: viper.GetBool("prerun.params.includeBlock"),
|
||||
IncludeReceipts: viper.GetBool("prerun.params.includeReceipts"),
|
||||
IncludeTD: viper.GetBool("prerun.params.includeTD"),
|
||||
IncludeCode: viper.GetBool("prerun.params.includeCode"),
|
||||
}
|
||||
var addrStrs []string
|
||||
viper.UnmarshalKey("prerun.params.watchedAddresses", &addrStrs)
|
||||
addrs := make([]common.Address, len(addrStrs))
|
||||
for i, addrStr := range addrStrs {
|
||||
addrs[i] = common.HexToAddress(addrStr)
|
||||
}
|
||||
preRunParams.WatchedAddresses = addrs
|
||||
var rawRanges []blockRange
|
||||
viper.UnmarshalKey("prerun.ranges", &rawRanges)
|
||||
blockRanges := make([]sd.RangeRequest, len(rawRanges))
|
||||
for i, rawRange := range rawRanges {
|
||||
blockRanges[i] = sd.RangeRequest{
|
||||
Start: rawRange[0],
|
||||
Stop: rawRange[1],
|
||||
Params: preRunParams,
|
||||
}
|
||||
}
|
||||
if viper.IsSet("prerun.start") && viper.IsSet("prerun.stop") {
|
||||
hardStart := viper.GetInt("prerun.start")
|
||||
hardStop := viper.GetInt("prerun.stop")
|
||||
blockRanges = append(blockRanges, sd.RangeRequest{
|
||||
Start: uint64(hardStart),
|
||||
Stop: uint64(hardStop),
|
||||
Params: preRunParams,
|
||||
})
|
||||
}
|
||||
|
||||
return blockRanges
|
||||
}
|
||||
|
@ -1,21 +1,19 @@
|
||||
version: '3.2'
|
||||
|
||||
services:
|
||||
eth-statediff-service:
|
||||
build:
|
||||
context: ./
|
||||
cache_from:
|
||||
- alpine:latest
|
||||
- golang:1.16
|
||||
- golang:1.18
|
||||
dockerfile: ./Dockerfile
|
||||
args:
|
||||
USER: "vdbm"
|
||||
CONFIG_FILE: ./environments/example.toml
|
||||
CONFIG_FILE: ./environments/docker.toml
|
||||
EXPOSE_PORT: 8545
|
||||
environment:
|
||||
- VDB_COMMAND=serve
|
||||
volumes:
|
||||
- eth-statediff-service-data:/root/.ethereum/
|
||||
- eth-statediff-service-data:/vdbm/.ethereum/
|
||||
ports:
|
||||
- "127.0.0.1:8545:8545"
|
||||
|
||||
|
@ -1,61 +0,0 @@
|
||||
[leveldb]
|
||||
mode = "local"
|
||||
path = "/app/geth-rw/chaindata"
|
||||
ancient = "/app/geth-rw/chaindata/ancient"
|
||||
url = "http://127.0.0.1:8082/"
|
||||
|
||||
[server]
|
||||
ipcPath = ""
|
||||
httpPath = "0.0.0.0:8545"
|
||||
|
||||
[statediff]
|
||||
prerun = true
|
||||
serviceWorkers = 1
|
||||
workerQueueSize = 1024
|
||||
trieWorkers = 16
|
||||
|
||||
[prerun]
|
||||
only = true
|
||||
ranges = []
|
||||
[prerun.params]
|
||||
intermediateStateNodes = true
|
||||
intermediateStorageNodes = true
|
||||
includeBlock = true
|
||||
includeReceipts = true
|
||||
includeTD = true
|
||||
includeCode = true
|
||||
watchedAddresses = []
|
||||
|
||||
[log]
|
||||
file = ""
|
||||
level = "info"
|
||||
|
||||
[database]
|
||||
name = ""
|
||||
hostname = ""
|
||||
port = 5432
|
||||
user = ""
|
||||
password = ""
|
||||
type = "postgres"
|
||||
driver = "sqlx"
|
||||
dumpDestination = ""
|
||||
filePath = ""
|
||||
|
||||
[cache]
|
||||
database = 1024
|
||||
trie = 4096
|
||||
|
||||
[prom]
|
||||
dbStats = false
|
||||
metrics = true
|
||||
http = true
|
||||
httpAddr = "0.0.0.0"
|
||||
httpPort = 9100
|
||||
|
||||
[ethereum]
|
||||
chainConfig = ""
|
||||
nodeID = ""
|
||||
clientName = "eth-statediff-service"
|
||||
genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3"
|
||||
networkID = 1
|
||||
chainID = 1
|
@ -1,7 +1,7 @@
|
||||
[leveldb]
|
||||
mode = "local"
|
||||
path = "/Users/user/Library/Ethereum/geth/chaindata"
|
||||
ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient"
|
||||
path = "/home/user/.ethereum/geth/chaindata"
|
||||
ancient = "/home/user/.ethereum/geth/chaindata/ancient"
|
||||
url = "http://127.0.0.1:8082/"
|
||||
|
||||
[server]
|
||||
@ -9,19 +9,15 @@
|
||||
httpPath = "127.0.0.1:8545"
|
||||
|
||||
[statediff]
|
||||
prerun = true
|
||||
serviceWorkers = 1
|
||||
workerQueueSize = 1024
|
||||
trieWorkers = 4
|
||||
|
||||
[prerun]
|
||||
only = false
|
||||
[run]
|
||||
ranges = [
|
||||
[0, 1000]
|
||||
]
|
||||
[prerun.params]
|
||||
intermediateStateNodes = true
|
||||
intermediateStorageNodes = true
|
||||
[run.params]
|
||||
includeBlock = true
|
||||
includeReceipts = true
|
||||
includeTD = true
|
||||
@ -33,15 +29,15 @@
|
||||
level = "info"
|
||||
|
||||
[database]
|
||||
name = "vulcanize_test"
|
||||
hostname = "localhost"
|
||||
port = 5432
|
||||
user = "vulcanize"
|
||||
password = "..."
|
||||
type = "postgres"
|
||||
driver = "sqlx"
|
||||
name = "cerc_testing"
|
||||
hostname = "localhost"
|
||||
port = 5432
|
||||
user = "vdbm"
|
||||
password = "password"
|
||||
type = "postgres"
|
||||
driver = "sqlx"
|
||||
dumpDestination = ""
|
||||
filePath = ""
|
||||
filePath = ""
|
||||
|
||||
[cache]
|
||||
database = 1024
|
||||
|
142
go.mod
142
go.mod
@ -3,17 +3,145 @@ module github.com/vulcanize/eth-statediff-service
|
||||
go 1.19
|
||||
|
||||
require (
|
||||
github.com/cerc-io/plugeth-statediff v0.0.0-00010101000000-000000000000
|
||||
github.com/ethereum/go-ethereum v1.12.0
|
||||
github.com/jmoiron/sqlx v1.2.0
|
||||
github.com/prometheus/client_golang v1.4.0
|
||||
github.com/sirupsen/logrus v1.7.0
|
||||
github.com/prometheus/client_golang v1.16.0
|
||||
github.com/sirupsen/logrus v1.9.0
|
||||
github.com/spf13/cobra v1.3.0
|
||||
github.com/spf13/viper v1.10.1
|
||||
github.com/vulcanize/go-eth-state-node-iterator v1.0.3
|
||||
github.com/vulcanize/leveldb-ethdb-rpc v0.1.2
|
||||
)
|
||||
|
||||
replace (
|
||||
github.com/cerc-io/plugeth-statediff => git.vdb.to/cerc-io/plugeth-statediff 840d4a4e5d42110e694941afa734dac8fea126fe
|
||||
github.com/ethereum/go-ethereum => git.vdb.to/cerc-io/plugeth v0.0.0-20230808125822-691dc334fab1
|
||||
// github.com/openrelayxyz/plugeth-utils => git.vdb.to/cerc-io/plugeth-utils v0.0.0-20230706160122-cd41de354c46
|
||||
require (
|
||||
github.com/DataDog/zstd v1.5.5 // indirect
|
||||
github.com/VictoriaMetrics/fastcache v1.12.1 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect
|
||||
github.com/cerc-io/eth-iterator-utils v1.2.0 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||
github.com/cockroachdb/errors v1.10.0 // indirect
|
||||
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
|
||||
github.com/cockroachdb/pebble v0.0.0-20230720154706-692f3b61a3c4 // indirect
|
||||
github.com/cockroachdb/redact v1.1.5 // indirect
|
||||
github.com/cockroachdb/tokenbucket v0.0.0-20230613231145-182959a1fad6 // indirect
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/deckarep/golang-set/v2 v2.3.0 // indirect
|
||||
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
|
||||
github.com/deepmap/oapi-codegen v1.8.2 // indirect
|
||||
github.com/edsrzf/mmap-go v1.0.0 // indirect
|
||||
github.com/fjl/memsize v0.0.1 // indirect
|
||||
github.com/fsnotify/fsnotify v1.6.0 // indirect
|
||||
github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff // indirect
|
||||
github.com/georgysavva/scany v0.2.9 // indirect
|
||||
github.com/getsentry/sentry-go v0.22.0 // indirect
|
||||
github.com/go-ole/go-ole v1.2.6 // indirect
|
||||
github.com/go-stack/stack v1.8.1 // indirect
|
||||
github.com/gofrs/flock v0.8.1 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
|
||||
github.com/golang/protobuf v1.5.3 // indirect
|
||||
github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect
|
||||
github.com/google/uuid v1.3.0 // indirect
|
||||
github.com/gorilla/websocket v1.5.0 // indirect
|
||||
github.com/graph-gophers/graphql-go v1.3.0 // indirect
|
||||
github.com/hashicorp/go-bexpr v0.1.12 // indirect
|
||||
github.com/hashicorp/hcl v1.0.0 // indirect
|
||||
github.com/holiman/bloomfilter/v2 v2.0.3 // indirect
|
||||
github.com/holiman/uint256 v1.2.3 // indirect
|
||||
github.com/huin/goupnp v1.2.0 // indirect
|
||||
github.com/inconshreveable/log15 v2.16.0+incompatible // indirect
|
||||
github.com/inconshreveable/mousetrap v1.0.0 // indirect
|
||||
github.com/influxdata/influxdb-client-go/v2 v2.4.0 // indirect
|
||||
github.com/influxdata/influxdb1-client v0.0.0-20220302092344-a9ab5670611c // indirect
|
||||
github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 // indirect
|
||||
github.com/ipfs/go-cid v0.4.1 // indirect
|
||||
github.com/jackc/chunkreader/v2 v2.0.1 // indirect
|
||||
github.com/jackc/pgconn v1.10.0 // indirect
|
||||
github.com/jackc/pgio v1.0.0 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgproto3/v2 v2.1.1 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
|
||||
github.com/jackc/pgtype v1.8.1 // indirect
|
||||
github.com/jackc/pgx/v4 v4.13.0 // indirect
|
||||
github.com/jackc/puddle v1.1.3 // indirect
|
||||
github.com/jackpal/go-nat-pmp v1.0.2 // indirect
|
||||
github.com/klauspost/compress v1.16.7 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
|
||||
github.com/kr/pretty v0.3.1 // indirect
|
||||
github.com/kr/text v0.2.0 // indirect
|
||||
github.com/lib/pq v1.10.9 // indirect
|
||||
github.com/magiconair/properties v1.8.5 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-isatty v0.0.19 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.14 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
|
||||
github.com/minio/sha256-simd v1.0.1 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/mitchellh/pointerstructure v1.2.1 // indirect
|
||||
github.com/mr-tron/base58 v1.2.0 // indirect
|
||||
github.com/multiformats/go-base32 v0.1.0 // indirect
|
||||
github.com/multiformats/go-base36 v0.2.0 // indirect
|
||||
github.com/multiformats/go-multibase v0.2.0 // indirect
|
||||
github.com/multiformats/go-multihash v0.2.3 // indirect
|
||||
github.com/multiformats/go-varint v0.0.7 // indirect
|
||||
github.com/olekukonko/tablewriter v0.0.5 // indirect
|
||||
github.com/openrelayxyz/plugeth-utils v1.2.0 // indirect
|
||||
github.com/opentracing/opentracing-go v1.2.0 // indirect
|
||||
github.com/pelletier/go-toml v1.9.4 // indirect
|
||||
github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 // indirect
|
||||
github.com/pganalyze/pg_query_go/v4 v4.2.1 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/prometheus/client_model v0.4.0 // indirect
|
||||
github.com/prometheus/common v0.44.0 // indirect
|
||||
github.com/prometheus/procfs v0.11.0 // indirect
|
||||
github.com/rivo/uniseg v0.4.4 // indirect
|
||||
github.com/rogpeppe/go-internal v1.11.0 // indirect
|
||||
github.com/rs/cors v1.9.0 // indirect
|
||||
github.com/russross/blackfriday/v2 v2.1.0 // indirect
|
||||
github.com/shirou/gopsutil v3.21.11+incompatible // indirect
|
||||
github.com/shopspring/decimal v1.2.0 // indirect
|
||||
github.com/spaolacci/murmur3 v1.1.0 // indirect
|
||||
github.com/spf13/afero v1.6.0 // indirect
|
||||
github.com/spf13/cast v1.4.1 // indirect
|
||||
github.com/spf13/jwalterweatherman v1.1.0 // indirect
|
||||
github.com/spf13/pflag v1.0.5 // indirect
|
||||
github.com/status-im/keycard-go v0.2.0 // indirect
|
||||
github.com/stretchr/objx v0.5.0 // indirect
|
||||
github.com/stretchr/testify v1.8.2 // indirect
|
||||
github.com/subosito/gotenv v1.2.0 // indirect
|
||||
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect
|
||||
github.com/thoas/go-funk v0.9.3 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.11 // indirect
|
||||
github.com/tklauser/numcpus v0.6.1 // indirect
|
||||
github.com/tyler-smith/go-bip39 v1.1.0 // indirect
|
||||
github.com/urfave/cli/v2 v2.25.7 // indirect
|
||||
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.3 // indirect
|
||||
golang.org/x/crypto v0.11.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 // indirect
|
||||
golang.org/x/net v0.10.0 // indirect
|
||||
golang.org/x/sync v0.3.0 // indirect
|
||||
golang.org/x/sys v0.10.0 // indirect
|
||||
golang.org/x/term v0.10.0 // indirect
|
||||
golang.org/x/text v0.11.0 // indirect
|
||||
golang.org/x/time v0.3.0 // indirect
|
||||
google.golang.org/protobuf v1.31.0 // indirect
|
||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
|
||||
gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
lukechampine.com/blake3 v1.2.1 // indirect
|
||||
)
|
||||
|
||||
replace (
|
||||
github.com/cerc-io/eth-iterator-utils => git.vdb.to/cerc-io/eth-iterator-utils v0.0.0-20230803115933-6bb6d4e27fd2
|
||||
// github.com/cerc-io/plugeth-statediff => git.vdb.to/cerc-io/plugeth-statediff v0.1.0
|
||||
github.com/cerc-io/plugeth-statediff => ../plugeth-statediff
|
||||
// github.com/ethereum/go-ethereum => git.vdb.to/cerc-io/plugeth v0.0.0-20230808125822-691dc334fab1
|
||||
github.com/ethereum/go-ethereum => ../plugeth
|
||||
github.com/openrelayxyz/plugeth-utils => git.vdb.to/cerc-io/plugeth-utils v0.0.0-20230706160122-cd41de354c46
|
||||
)
|
||||
|
@ -30,11 +30,11 @@ const APIVersion = "0.0.1"
|
||||
// PublicStateDiffAPI provides an RPC interface
|
||||
// that can be used to fetch historical diffs from LevelDB directly
|
||||
type PublicStateDiffAPI struct {
|
||||
sds StateDiffService
|
||||
sds *Service
|
||||
}
|
||||
|
||||
// NewPublicStateDiffAPI creates an rpc interface for the underlying statediff service
|
||||
func NewPublicStateDiffAPI(sds StateDiffService) *PublicStateDiffAPI {
|
||||
func NewPublicStateDiffAPI(sds *Service) *PublicStateDiffAPI {
|
||||
return &PublicStateDiffAPI{
|
||||
sds: sds,
|
||||
}
|
||||
@ -45,11 +45,6 @@ func (api *PublicStateDiffAPI) StateDiffAt(ctx context.Context, blockNumber uint
|
||||
return api.sds.StateDiffAt(blockNumber, params)
|
||||
}
|
||||
|
||||
// StateTrieAt returns a state trie payload at the specific blockheight
|
||||
func (api *PublicStateDiffAPI) StateTrieAt(ctx context.Context, blockNumber uint64, params sd.Params) (*sd.Payload, error) {
|
||||
return api.sds.StateTrieAt(blockNumber, params)
|
||||
}
|
||||
|
||||
// WriteStateDiffAt writes a state diff object directly to DB at the specific blockheight
|
||||
func (api *PublicStateDiffAPI) WriteStateDiffAt(ctx context.Context, blockNumber uint64, params sd.Params) error {
|
||||
return api.sds.WriteStateDiffAt(blockNumber, params)
|
||||
|
886
pkg/builder.go
886
pkg/builder.go
@ -1,886 +0,0 @@
|
||||
// Copyright 2019 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
// Contains a batch of utility type declarations used by the tests. As the node
|
||||
// operates on unique types, a lot of them are needed to check various features.
|
||||
|
||||
package statediff
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"math/bits"
|
||||
"sync"
|
||||
|
||||
sd "github.com/cerc-io/plugeth-statediff"
|
||||
sdtrie "github.com/cerc-io/plugeth-statediff/trie_helpers"
|
||||
sdtypes "github.com/cerc-io/plugeth-statediff/types"
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/state"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/crypto"
|
||||
"github.com/ethereum/go-ethereum/log"
|
||||
"github.com/ethereum/go-ethereum/rlp"
|
||||
"github.com/ethereum/go-ethereum/trie"
|
||||
"github.com/sirupsen/logrus"
|
||||
|
||||
iter "github.com/vulcanize/go-eth-state-node-iterator"
|
||||
)
|
||||
|
||||
var (
|
||||
nullHashBytes = common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000000")
|
||||
emptyNode, _ = rlp.EncodeToBytes(&[]byte{})
|
||||
emptyContractRoot = crypto.Keccak256Hash(emptyNode)
|
||||
nullCodeHash = crypto.Keccak256Hash([]byte{}).Bytes()
|
||||
)
|
||||
|
||||
// Builder interface exposes the method for building a state diff between two blocks
|
||||
type Builder interface {
|
||||
BuildStateDiffObject(args sd.Args, params sd.Params) (sdtypes.StateObject, error)
|
||||
BuildStateTrieObject(current *types.Block) (sdtypes.StateObject, error)
|
||||
WriteStateDiffObject(args sdtypes.StateRoots, params sd.Params, output sdtypes.StateNodeSink, codeOutput sdtypes.CodeSink) error
|
||||
}
|
||||
|
||||
type builder struct {
|
||||
stateCache state.Database
|
||||
numWorkers uint
|
||||
}
|
||||
|
||||
type iterPair struct {
|
||||
older, newer trie.NodeIterator
|
||||
}
|
||||
|
||||
func resolveNode(it trie.NodeIterator, trieDB *trie.Database) (sdtypes.StateNode, []interface{}, error) {
|
||||
nodePath := make([]byte, len(it.Path()))
|
||||
copy(nodePath, it.Path())
|
||||
node, err := trieDB.Node(it.Hash())
|
||||
if err != nil {
|
||||
return sdtypes.StateNode{}, nil, err
|
||||
}
|
||||
var nodeElements []interface{}
|
||||
if err := rlp.DecodeBytes(node, &nodeElements); err != nil {
|
||||
return sdtypes.StateNode{}, nil, err
|
||||
}
|
||||
ty, err := sdtrie.CheckKeyType(nodeElements)
|
||||
if err != nil {
|
||||
return sdtypes.StateNode{}, nil, err
|
||||
}
|
||||
return sdtypes.StateNode{
|
||||
NodeType: ty,
|
||||
Path: nodePath,
|
||||
NodeValue: node,
|
||||
}, nodeElements, nil
|
||||
}
|
||||
|
||||
// convenience
|
||||
func stateNodeAppender(nodes *[]sdtypes.StateNode) sdtypes.StateNodeSink {
|
||||
return func(node sdtypes.StateNode) error {
|
||||
*nodes = append(*nodes, node)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
func storageNodeAppender(nodes *[]sdtypes.StorageNode) sdtypes.StorageNodeSink {
|
||||
return func(node sdtypes.StorageNode) error {
|
||||
*nodes = append(*nodes, node)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
func codeMappingAppender(data *[]sdtypes.CodeAndCodeHash) sdtypes.CodeSink {
|
||||
return func(c sdtypes.CodeAndCodeHash) error {
|
||||
*data = append(*data, c)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// NewBuilder is used to create a statediff builder
|
||||
func NewBuilder(stateCache state.Database, workers uint) (Builder, error) {
|
||||
if workers == 0 {
|
||||
workers = 1
|
||||
}
|
||||
if bits.OnesCount(workers) != 1 {
|
||||
return nil, fmt.Errorf("workers must be a power of 2")
|
||||
}
|
||||
return &builder{
|
||||
stateCache: stateCache, // state cache is safe for concurrent reads
|
||||
numWorkers: workers,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// BuildStateTrieObject builds a state trie object from the provided block
|
||||
func (sdb *builder) BuildStateTrieObject(current *types.Block) (sdtypes.StateObject, error) {
|
||||
currentTrie, err := sdb.stateCache.OpenTrie(current.Root())
|
||||
if err != nil {
|
||||
return sdtypes.StateObject{}, fmt.Errorf("error creating trie for block %d: %v", current.Number(), err)
|
||||
}
|
||||
it := currentTrie.NodeIterator([]byte{})
|
||||
stateNodes, codeAndCodeHashes, err := sdb.buildStateTrie(it)
|
||||
if err != nil {
|
||||
return sdtypes.StateObject{}, fmt.Errorf("error collecting state nodes for block %d: %v", current.Number(), err)
|
||||
}
|
||||
return sdtypes.StateObject{
|
||||
BlockNumber: current.Number(),
|
||||
BlockHash: current.Hash(),
|
||||
Nodes: stateNodes,
|
||||
CodeAndCodeHashes: codeAndCodeHashes,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (sdb *builder) buildStateTrie(it trie.NodeIterator) ([]sdtypes.StateNode, []sdtypes.CodeAndCodeHash, error) {
|
||||
stateNodes := make([]sdtypes.StateNode, 0)
|
||||
codeAndCodeHashes := make([]sdtypes.CodeAndCodeHash, 0)
|
||||
for it.Next(true) {
|
||||
// skip value nodes
|
||||
if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
|
||||
continue
|
||||
}
|
||||
node, nodeElements, err := resolveNode(it, sdb.stateCache.TrieDB())
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
switch node.NodeType {
|
||||
case sdtypes.Leaf:
|
||||
var account types.StateAccount
|
||||
if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil {
|
||||
return nil, nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", node.Path, err)
|
||||
}
|
||||
partialPath := trie.CompactToHex(nodeElements[0].([]byte))
|
||||
valueNodePath := append(node.Path, partialPath...)
|
||||
encodedPath := trie.HexToCompact(valueNodePath)
|
||||
leafKey := encodedPath[1:]
|
||||
node.LeafKey = leafKey
|
||||
if !bytes.Equal(account.CodeHash, nullCodeHash) {
|
||||
var storageNodes []sdtypes.StorageNode
|
||||
err := sdb.buildStorageNodesEventual(account.Root, true, storageNodeAppender(&storageNodes))
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed building eventual storage diffs for account %+v\r\nerror: %v", account, err)
|
||||
}
|
||||
node.StorageNodes = storageNodes
|
||||
// emit codehash => code mappings for code
|
||||
codeHash := common.BytesToHash(account.CodeHash)
|
||||
code, err := sdb.stateCache.ContractCode(common.Hash{}, codeHash)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to retrieve code for codehash %s\r\n error: %v", codeHash.String(), err)
|
||||
}
|
||||
codeAndCodeHashes = append(codeAndCodeHashes, sdtypes.CodeAndCodeHash{
|
||||
Hash: codeHash,
|
||||
Code: code,
|
||||
})
|
||||
}
|
||||
stateNodes = append(stateNodes, node)
|
||||
case sdtypes.Extension, sdtypes.Branch:
|
||||
stateNodes = append(stateNodes, node)
|
||||
default:
|
||||
return nil, nil, fmt.Errorf("unexpected node type %s", node.NodeType)
|
||||
}
|
||||
}
|
||||
return stateNodes, codeAndCodeHashes, it.Error()
|
||||
}
|
||||
|
||||
// BuildStateDiffObject builds a statediff object from two blocks and the provided parameters
|
||||
func (sdb *builder) BuildStateDiffObject(args sd.Args, params sd.Params) (sdtypes.StateObject, error) {
|
||||
var stateNodes []sdtypes.StateNode
|
||||
var codeAndCodeHashes []sdtypes.CodeAndCodeHash
|
||||
err := sdb.WriteStateDiffObject(
|
||||
sdtypes.StateRoots{OldStateRoot: args.OldStateRoot, NewStateRoot: args.NewStateRoot},
|
||||
params, stateNodeAppender(&stateNodes), codeMappingAppender(&codeAndCodeHashes))
|
||||
if err != nil {
|
||||
return sdtypes.StateObject{}, err
|
||||
}
|
||||
return sdtypes.StateObject{
|
||||
BlockHash: args.BlockHash,
|
||||
BlockNumber: args.BlockNumber,
|
||||
Nodes: stateNodes,
|
||||
CodeAndCodeHashes: codeAndCodeHashes,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// WriteStateDiffObject writes a statediff object to output callback
|
||||
func (sdb *builder) WriteStateDiffObject(args sdtypes.StateRoots, params sd.Params, output sdtypes.StateNodeSink, codeOutput sdtypes.CodeSink) error {
|
||||
if len(params.WatchedAddresses) > 0 {
|
||||
// if we are watching only specific accounts then we are only diffing leaf nodes
|
||||
log.Info("Ignoring intermediate state nodes because WatchedAddresses was passed")
|
||||
params.IntermediateStateNodes = false
|
||||
}
|
||||
|
||||
// Load tries for old and new states
|
||||
oldTrie, err := sdb.stateCache.OpenTrie(args.OldStateRoot)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error creating trie for oldStateRoot: %v", err)
|
||||
}
|
||||
newTrie, err := sdb.stateCache.OpenTrie(args.NewStateRoot)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error creating trie for newStateRoot: %v", err)
|
||||
}
|
||||
|
||||
// Split old and new tries into corresponding subtrie iterators
|
||||
oldIters1 := iter.SubtrieIterators(oldTrie, sdb.numWorkers)
|
||||
oldIters2 := iter.SubtrieIterators(oldTrie, sdb.numWorkers)
|
||||
newIters1 := iter.SubtrieIterators(newTrie, sdb.numWorkers)
|
||||
newIters2 := iter.SubtrieIterators(newTrie, sdb.numWorkers)
|
||||
|
||||
// Create iterators ahead of time to avoid race condition in state.Trie access
|
||||
// We do two state iterations per subtrie: one for new/updated nodes,
|
||||
// one for deleted/updated nodes; prepare 2 iterator instances for each task
|
||||
var iterPairs [][]iterPair
|
||||
for i := uint(0); i < sdb.numWorkers; i++ {
|
||||
iterPairs = append(iterPairs, []iterPair{
|
||||
{older: oldIters1[i], newer: newIters1[i]},
|
||||
{older: oldIters2[i], newer: newIters2[i]},
|
||||
})
|
||||
}
|
||||
|
||||
// Dispatch workers to process trie data; sync and collect results here via channels
|
||||
nodeChan := make(chan sdtypes.StateNode)
|
||||
codeChan := make(chan sdtypes.CodeAndCodeHash)
|
||||
|
||||
go func() {
|
||||
nodeSender := func(node sdtypes.StateNode) error { nodeChan <- node; return nil }
|
||||
codeSender := func(code sdtypes.CodeAndCodeHash) error { codeChan <- code; return nil }
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for w := uint(0); w < sdb.numWorkers; w++ {
|
||||
wg.Add(1)
|
||||
go func(worker uint) {
|
||||
defer wg.Done()
|
||||
if err := sdb.buildStateDiff(iterPairs[worker], params, nodeSender, codeSender); err != nil {
|
||||
logrus.Errorf("buildStateDiff error for worker %d, pparams %+v", worker, params)
|
||||
}
|
||||
}(w)
|
||||
}
|
||||
wg.Wait()
|
||||
close(nodeChan)
|
||||
close(codeChan)
|
||||
}()
|
||||
|
||||
for nodeChan != nil || codeChan != nil {
|
||||
select {
|
||||
case node, more := <-nodeChan:
|
||||
if more {
|
||||
if err := output(node); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
nodeChan = nil
|
||||
}
|
||||
case codeAndCodeHash, more := <-codeChan:
|
||||
if more {
|
||||
if err := codeOutput(codeAndCodeHash); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
codeChan = nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sdb *builder) buildStateDiff(args []iterPair, params sd.Params, output sdtypes.StateNodeSink, codeOutput sdtypes.CodeSink) error {
|
||||
// collect a slice of all the intermediate nodes that were touched and exist at B
|
||||
// a map of their leafkey to all the accounts that were touched and exist at B
|
||||
// and a slice of all the paths for the nodes in both of the above sets
|
||||
var diffAccountsAtB AccountMap
|
||||
var diffPathsAtB map[string]bool
|
||||
var err error
|
||||
if params.IntermediateStateNodes {
|
||||
diffAccountsAtB, diffPathsAtB, err = sdb.createdAndUpdatedStateWithIntermediateNodes(args[0], output)
|
||||
} else {
|
||||
diffAccountsAtB, diffPathsAtB, err = sdb.createdAndUpdatedState(args[0], params.WatchedAddressesLeafKeys())
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("error collecting createdAndUpdatedNodes: %v", err)
|
||||
}
|
||||
|
||||
// collect a slice of all the nodes that existed at a path in A that doesn't exist in B
|
||||
// a map of their leafkey to all the accounts that were touched and exist at A
|
||||
diffAccountsAtA, err := sdb.deletedOrUpdatedState(args[1], diffAccountsAtB, diffPathsAtB, params.WatchedAddressesLeafKeys(), params.IntermediateStorageNodes, output)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error collecting deletedOrUpdatedNodes: %v", err)
|
||||
}
|
||||
|
||||
// collect and sort the leafkeys for both account mappings into a slice
|
||||
createKeys := sortKeys(diffAccountsAtB)
|
||||
deleteKeys := sortKeys(diffAccountsAtA)
|
||||
|
||||
// and then find the intersection of these keys
|
||||
// these are the leafkeys for the accounts which exist at both A and B but are different
|
||||
// this also mutates the passed in createKeys and deleteKeys, removing the intersection keys
|
||||
// and leaving the truly created or deleted keys in place
|
||||
updatedKeys := findIntersection(createKeys, deleteKeys)
|
||||
|
||||
// build the diff nodes for the updated accounts using the mappings at both A and B as directed by the keys found as the intersection of the two
|
||||
err = sdb.buildAccountUpdates(
|
||||
diffAccountsAtB, diffAccountsAtA, updatedKeys, params.IntermediateStorageNodes, output)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error building diff for updated accounts: %v", err)
|
||||
}
|
||||
// build the diff nodes for created accounts
|
||||
err = sdb.buildAccountCreations(diffAccountsAtB, params.IntermediateStorageNodes, output, codeOutput)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error building diff for created accounts: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// createdAndUpdatedState returns
|
||||
// a mapping of their leafkeys to all the accounts that exist in a different state at B than A
|
||||
// and a slice of the paths for all of the nodes included in both
|
||||
func (sdb *builder) createdAndUpdatedState(iters iterPair, watchedAddressesLeafKeys map[common.Hash]struct{}) (AccountMap, map[string]bool, error) {
|
||||
diffPathsAtB := make(map[string]bool)
|
||||
diffAcountsAtB := make(AccountMap)
|
||||
it, _ := trie.NewDifferenceIterator(iters.older, iters.newer)
|
||||
for it.Next(true) {
|
||||
if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
|
||||
continue
|
||||
}
|
||||
node, nodeElements, err := sdtrie.ResolveNode(it, sdb.stateCache.TrieDB())
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
switch node.NodeType {
|
||||
case sdtypes.Leaf:
|
||||
// created vs updated is important for leaf nodes since we need to diff their storage
|
||||
// so we need to map all changed accounts at B to their leafkey, since account can change pathes but not leafkey
|
||||
var account types.StateAccount
|
||||
if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil {
|
||||
return nil, nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", node.Path, err)
|
||||
}
|
||||
partialPath := trie.CompactToHex(nodeElements[0].([]byte))
|
||||
valueNodePath := append(node.Path, partialPath...)
|
||||
encodedPath := trie.HexToCompact(valueNodePath)
|
||||
leafKey := encodedPath[1:]
|
||||
if isWatchedAddress(watchedAddressesLeafKeys, leafKey) {
|
||||
diffAcountsAtB[common.Bytes2Hex(leafKey)] = accountWrapper{
|
||||
NodeType: node.NodeType,
|
||||
Path: node.Path,
|
||||
NodeValue: node.NodeValue,
|
||||
LeafKey: leafKey,
|
||||
Account: &account,
|
||||
}
|
||||
}
|
||||
}
|
||||
// add both intermediate and leaf node paths to the list of diffPathsAtB
|
||||
diffPathsAtB[common.Bytes2Hex(node.Path)] = true
|
||||
}
|
||||
return diffAcountsAtB, diffPathsAtB, it.Error()
|
||||
}
|
||||
|
||||
// createdAndUpdatedStateWithIntermediateNodes returns
|
||||
// a slice of all the intermediate nodes that exist in a different state at B than A
|
||||
// a mapping of their leafkeys to all the accounts that exist in a different state at B than A
|
||||
// and a slice of the paths for all of the nodes included in both
|
||||
func (sdb *builder) createdAndUpdatedStateWithIntermediateNodes(iters iterPair, output sdtypes.StateNodeSink) (AccountMap, map[string]bool, error) {
|
||||
diffPathsAtB := make(map[string]bool)
|
||||
diffAcountsAtB := make(AccountMap)
|
||||
it, _ := trie.NewDifferenceIterator(iters.older, iters.newer)
|
||||
for it.Next(true) {
|
||||
if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
|
||||
continue
|
||||
}
|
||||
node, nodeElements, err := resolveNode(it, sdb.stateCache.TrieDB())
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
switch node.NodeType {
|
||||
case sdtypes.Leaf:
|
||||
// created vs updated is important for leaf nodes since we need to diff their storage
|
||||
// so we need to map all changed accounts at B to their leafkey, since account can change paths but not leafkey
|
||||
var account types.StateAccount
|
||||
if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil {
|
||||
return nil, nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", node.Path, err)
|
||||
}
|
||||
partialPath := trie.CompactToHex(nodeElements[0].([]byte))
|
||||
valueNodePath := append(node.Path, partialPath...)
|
||||
encodedPath := trie.HexToCompact(valueNodePath)
|
||||
leafKey := encodedPath[1:]
|
||||
diffAcountsAtB[common.Bytes2Hex(leafKey)] = accountWrapper{
|
||||
NodeType: node.NodeType,
|
||||
Path: node.Path,
|
||||
NodeValue: node.NodeValue,
|
||||
LeafKey: leafKey,
|
||||
Account: &account,
|
||||
}
|
||||
case sdtypes.Extension, sdtypes.Branch:
|
||||
// create a diff for any intermediate node that has changed at b
|
||||
// created vs updated makes no difference for intermediate nodes since we do not need to diff storage
|
||||
if err := output(sdtypes.StateNode{
|
||||
NodeType: node.NodeType,
|
||||
Path: node.Path,
|
||||
NodeValue: node.NodeValue,
|
||||
}); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
default:
|
||||
return nil, nil, fmt.Errorf("unexpected node type %s", node.NodeType)
|
||||
}
|
||||
// add both intermediate and leaf node paths to the list of diffPathsAtB
|
||||
diffPathsAtB[common.Bytes2Hex(node.Path)] = true
|
||||
}
|
||||
return diffAcountsAtB, diffPathsAtB, it.Error()
|
||||
}
|
||||
|
||||
// deletedOrUpdatedState returns a slice of all the paths that are emptied at B
|
||||
// and a mapping of their leafkeys to all the accounts that exist in a different state at A than B
|
||||
func (sdb *builder) deletedOrUpdatedState(iters iterPair, diffAccountsAtB AccountMap, diffPathsAtB map[string]bool, watchedAddressesLeafKeys map[common.Hash]struct{}, intermediateStorageNodes bool, output sdtypes.StateNodeSink) (AccountMap, error) {
|
||||
diffAccountAtA := make(AccountMap)
|
||||
it, _ := trie.NewDifferenceIterator(iters.newer, iters.older)
|
||||
for it.Next(true) {
|
||||
// skip value nodes
|
||||
if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
|
||||
continue
|
||||
}
|
||||
node, nodeElements, err := sdtrie.ResolveNode(it, sdb.stateCache.TrieDB())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch node.NodeType {
|
||||
case sdtypes.Leaf:
|
||||
// map all different accounts at A to their leafkey
|
||||
var account types.StateAccount
|
||||
if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil {
|
||||
return nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", node.Path, err)
|
||||
}
|
||||
partialPath := trie.CompactToHex(nodeElements[0].([]byte))
|
||||
valueNodePath := append(node.Path, partialPath...)
|
||||
encodedPath := trie.HexToCompact(valueNodePath)
|
||||
leafKey := encodedPath[1:]
|
||||
if isWatchedAddress(watchedAddressesLeafKeys, leafKey) {
|
||||
diffAccountAtA[common.Bytes2Hex(leafKey)] = accountWrapper{
|
||||
NodeType: node.NodeType,
|
||||
Path: node.Path,
|
||||
NodeValue: node.NodeValue,
|
||||
LeafKey: leafKey,
|
||||
Account: &account,
|
||||
}
|
||||
// if this node's path did not show up in diffPathsAtB
|
||||
// that means the node at this path was deleted (or moved) in B
|
||||
if _, ok := diffPathsAtB[common.Bytes2Hex(node.Path)]; !ok {
|
||||
var diff sdtypes.StateNode
|
||||
// if this node's leaf key also did not show up in diffAccountsAtB
|
||||
// that means the node was deleted
|
||||
// in that case, emit an empty "removed" diff state node
|
||||
// include empty "removed" diff storage nodes for all the storage slots
|
||||
if _, ok := diffAccountsAtB[common.Bytes2Hex(leafKey)]; !ok {
|
||||
diff = sdtypes.StateNode{
|
||||
NodeType: sdtypes.Removed,
|
||||
Path: node.Path,
|
||||
LeafKey: leafKey,
|
||||
NodeValue: []byte{},
|
||||
}
|
||||
|
||||
var storageDiffs []sdtypes.StorageNode
|
||||
err := sdb.buildRemovedAccountStorageNodes(account.Root, intermediateStorageNodes, storageNodeAppender(&storageDiffs))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed building storage diffs for removed node %x\r\nerror: %v", node.Path, err)
|
||||
}
|
||||
diff.StorageNodes = storageDiffs
|
||||
} else {
|
||||
// emit an empty "removed" diff with empty leaf key if the account was moved
|
||||
diff = sdtypes.StateNode{
|
||||
NodeType: sdtypes.Removed,
|
||||
Path: node.Path,
|
||||
NodeValue: []byte{},
|
||||
}
|
||||
}
|
||||
|
||||
if err := output(diff); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
case sdtypes.Extension, sdtypes.Branch:
|
||||
// if this node's path did not show up in diffPathsAtB
|
||||
// that means the node at this path was deleted (or moved) in B
|
||||
// emit an empty "removed" diff to signify as such
|
||||
if _, ok := diffPathsAtB[common.Bytes2Hex(node.Path)]; !ok {
|
||||
if err := output(sdtypes.StateNode{
|
||||
Path: node.Path,
|
||||
NodeValue: []byte{},
|
||||
NodeType: sdtypes.Removed,
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
// fall through, we did everything we need to do with these node types
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected node type %s", node.NodeType)
|
||||
}
|
||||
}
|
||||
return diffAccountAtA, it.Error()
|
||||
}
|
||||
|
||||
// buildAccountUpdates uses the account diffs maps for A => B and B => A and the known intersection of their leafkeys
|
||||
// to generate the statediff node objects for all of the accounts that existed at both A and B but in different states
|
||||
// needs to be called before building account creations and deletions as this mutates
|
||||
// those account maps to remove the accounts which were updated
|
||||
func (sdb *builder) buildAccountUpdates(creations, deletions AccountMap, updatedKeys []string, intermediateStorageNodes bool, output sdtypes.StateNodeSink) error {
|
||||
var err error
|
||||
for _, key := range updatedKeys {
|
||||
createdAcc := creations[key]
|
||||
deletedAcc := deletions[key]
|
||||
var storageDiffs []sdtypes.StorageNode
|
||||
if deletedAcc.Account != nil && createdAcc.Account != nil {
|
||||
oldSR := deletedAcc.Account.Root
|
||||
newSR := createdAcc.Account.Root
|
||||
err = sdb.buildStorageNodesIncremental(oldSR, newSR, intermediateStorageNodes, storageNodeAppender(&storageDiffs))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed building incremental storage diffs for account with leafkey %s\r\nerror: %v", key, err)
|
||||
}
|
||||
}
|
||||
if err = output(sdtypes.StateNode{
|
||||
NodeType: createdAcc.NodeType,
|
||||
Path: createdAcc.Path,
|
||||
NodeValue: createdAcc.NodeValue,
|
||||
LeafKey: createdAcc.LeafKey,
|
||||
StorageNodes: storageDiffs,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
delete(creations, key)
|
||||
delete(deletions, key)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// buildAccountCreations returns the statediff node objects for all the accounts that exist at B but not at A
|
||||
// it also returns the code and codehash for created contract accounts
|
||||
func (sdb *builder) buildAccountCreations(accounts AccountMap, intermediateStorageNodes bool, output sdtypes.StateNodeSink, codeOutput sdtypes.CodeSink) error {
|
||||
for _, val := range accounts {
|
||||
diff := sdtypes.StateNode{
|
||||
NodeType: val.NodeType,
|
||||
Path: val.Path,
|
||||
LeafKey: val.LeafKey,
|
||||
NodeValue: val.NodeValue,
|
||||
}
|
||||
if !bytes.Equal(val.Account.CodeHash, nullCodeHash) {
|
||||
// For contract creations, any storage node contained is a diff
|
||||
var storageDiffs []sdtypes.StorageNode
|
||||
err := sdb.buildStorageNodesEventual(val.Account.Root, intermediateStorageNodes, storageNodeAppender(&storageDiffs))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed building eventual storage diffs for node %x\r\nerror: %v", val.Path, err)
|
||||
}
|
||||
diff.StorageNodes = storageDiffs
|
||||
// emit codehash => code mappings for code
|
||||
codeHash := common.BytesToHash(val.Account.CodeHash)
|
||||
code, err := sdb.stateCache.ContractCode(common.Hash{}, codeHash)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to retrieve code for codehash %s\r\n error: %v", codeHash.String(), err)
|
||||
}
|
||||
if err := codeOutput(sdtypes.CodeAndCodeHash{
|
||||
Hash: codeHash,
|
||||
Code: code,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := output(diff); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// buildStorageNodesEventual builds the storage diff node objects for a created account
|
||||
// i.e. it returns all the storage nodes at this state, since there is no previous state
|
||||
func (sdb *builder) buildStorageNodesEventual(sr common.Hash, intermediateNodes bool, output sdtypes.StorageNodeSink) error {
|
||||
if bytes.Equal(sr.Bytes(), emptyContractRoot.Bytes()) {
|
||||
return nil
|
||||
}
|
||||
log.Debug("Storage Root For Eventual Diff", "root", sr.Hex())
|
||||
sTrie, err := sdb.stateCache.OpenTrie(sr)
|
||||
if err != nil {
|
||||
log.Info("error in build storage diff eventual", "error", err)
|
||||
return err
|
||||
}
|
||||
it := sTrie.NodeIterator(make([]byte, 0))
|
||||
err = sdb.buildStorageNodesFromTrie(it, intermediateNodes, output)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// buildStorageNodesFromTrie returns all the storage diff node objects in the provided node iterator
|
||||
// if any storage keys are provided it will only return those leaf nodes
|
||||
// including intermediate nodes can be turned on or off
|
||||
func (sdb *builder) buildStorageNodesFromTrie(it trie.NodeIterator, intermediateNodes bool, output sdtypes.StorageNodeSink) error {
|
||||
for it.Next(true) {
|
||||
if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
|
||||
continue
|
||||
}
|
||||
node, nodeElements, err := resolveNode(it, sdb.stateCache.TrieDB())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch node.NodeType {
|
||||
case sdtypes.Leaf:
|
||||
partialPath := trie.CompactToHex(nodeElements[0].([]byte))
|
||||
valueNodePath := append(node.Path, partialPath...)
|
||||
encodedPath := trie.HexToCompact(valueNodePath)
|
||||
leafKey := encodedPath[1:]
|
||||
if err := output(sdtypes.StorageNode{
|
||||
NodeType: node.NodeType,
|
||||
Path: node.Path,
|
||||
NodeValue: node.NodeValue,
|
||||
LeafKey: leafKey,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
case sdtypes.Extension, sdtypes.Branch:
|
||||
if intermediateNodes {
|
||||
if err := output(sdtypes.StorageNode{
|
||||
NodeType: node.NodeType,
|
||||
Path: node.Path,
|
||||
NodeValue: node.NodeValue,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("unexpected node type %s", node.NodeType)
|
||||
}
|
||||
}
|
||||
return it.Error()
|
||||
}
|
||||
|
||||
// buildRemovedAccountStorageNodes builds the "removed" diffs for all the storage nodes for a destroyed account
|
||||
func (sdb *builder) buildRemovedAccountStorageNodes(sr common.Hash, intermediateNodes bool, output sdtypes.StorageNodeSink) error {
|
||||
if bytes.Equal(sr.Bytes(), emptyContractRoot.Bytes()) {
|
||||
return nil
|
||||
}
|
||||
log.Debug("Storage Root For Removed Diffs", "root", sr.Hex())
|
||||
sTrie, err := sdb.stateCache.OpenTrie(sr)
|
||||
if err != nil {
|
||||
log.Info("error in build removed account storage diffs", "error", err)
|
||||
return err
|
||||
}
|
||||
it := sTrie.NodeIterator(make([]byte, 0))
|
||||
err = sdb.buildRemovedStorageNodesFromTrie(it, intermediateNodes, output)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// buildRemovedStorageNodesFromTrie returns diffs for all the storage nodes in the provided node interator
|
||||
// including intermediate nodes can be turned on or off
|
||||
func (sdb *builder) buildRemovedStorageNodesFromTrie(it trie.NodeIterator, intermediateNodes bool, output sdtypes.StorageNodeSink) error {
|
||||
for it.Next(true) {
|
||||
// skip value nodes
|
||||
if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
|
||||
continue
|
||||
}
|
||||
node, nodeElements, err := resolveNode(it, sdb.stateCache.TrieDB())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch node.NodeType {
|
||||
case sdtypes.Leaf:
|
||||
partialPath := trie.CompactToHex(nodeElements[0].([]byte))
|
||||
valueNodePath := append(node.Path, partialPath...)
|
||||
encodedPath := trie.HexToCompact(valueNodePath)
|
||||
leafKey := encodedPath[1:]
|
||||
if err := output(sdtypes.StorageNode{
|
||||
NodeType: sdtypes.Removed,
|
||||
Path: node.Path,
|
||||
NodeValue: []byte{},
|
||||
LeafKey: leafKey,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
case sdtypes.Extension, sdtypes.Branch:
|
||||
if intermediateNodes {
|
||||
if err := output(sdtypes.StorageNode{
|
||||
NodeType: sdtypes.Removed,
|
||||
Path: node.Path,
|
||||
NodeValue: []byte{},
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("unexpected node type %s", node.NodeType)
|
||||
}
|
||||
}
|
||||
return it.Error()
|
||||
}
|
||||
|
||||
// buildStorageNodesIncremental builds the storage diff node objects for all nodes that exist in a different state at B than A
|
||||
func (sdb *builder) buildStorageNodesIncremental(oldSR common.Hash, newSR common.Hash, intermediateNodes bool, output sdtypes.StorageNodeSink) error {
|
||||
if bytes.Equal(newSR.Bytes(), oldSR.Bytes()) {
|
||||
return nil
|
||||
}
|
||||
log.Debug("Storage Roots for Incremental Diff", "old", oldSR.Hex(), "new", newSR.Hex())
|
||||
oldTrie, err := sdb.stateCache.OpenTrie(oldSR)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
newTrie, err := sdb.stateCache.OpenTrie(newSR)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
diffSlotsAtB, diffPathsAtB, err := sdb.createdAndUpdatedStorage(oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), intermediateNodes, output)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = sdb.deletedOrUpdatedStorage(oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), diffSlotsAtB, diffPathsAtB, intermediateNodes, output)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sdb *builder) createdAndUpdatedStorage(a, b trie.NodeIterator, intermediateNodes bool, output sdtypes.StorageNodeSink) (map[string]bool, map[string]bool, error) {
|
||||
diffPathsAtB := make(map[string]bool)
|
||||
diffSlotsAtB := make(map[string]bool)
|
||||
it, _ := trie.NewDifferenceIterator(a, b)
|
||||
for it.Next(true) {
|
||||
if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
|
||||
continue
|
||||
}
|
||||
node, nodeElements, err := resolveNode(it, sdb.stateCache.TrieDB())
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
switch node.NodeType {
|
||||
case sdtypes.Leaf:
|
||||
partialPath := trie.CompactToHex(nodeElements[0].([]byte))
|
||||
valueNodePath := append(node.Path, partialPath...)
|
||||
encodedPath := trie.HexToCompact(valueNodePath)
|
||||
leafKey := encodedPath[1:]
|
||||
diffSlotsAtB[common.Bytes2Hex(leafKey)] = true
|
||||
if err := output(sdtypes.StorageNode{
|
||||
NodeType: node.NodeType,
|
||||
Path: node.Path,
|
||||
NodeValue: node.NodeValue,
|
||||
LeafKey: leafKey,
|
||||
}); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
case sdtypes.Extension, sdtypes.Branch:
|
||||
if intermediateNodes {
|
||||
if err := output(sdtypes.StorageNode{
|
||||
NodeType: node.NodeType,
|
||||
Path: node.Path,
|
||||
NodeValue: node.NodeValue,
|
||||
}); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
}
|
||||
default:
|
||||
return nil, nil, fmt.Errorf("unexpected node type %s", node.NodeType)
|
||||
}
|
||||
diffPathsAtB[common.Bytes2Hex(node.Path)] = true
|
||||
}
|
||||
return diffSlotsAtB, diffPathsAtB, it.Error()
|
||||
}
|
||||
|
||||
func (sdb *builder) deletedOrUpdatedStorage(a, b trie.NodeIterator, diffSlotsAtB, diffPathsAtB map[string]bool, intermediateNodes bool, output sdtypes.StorageNodeSink) error {
|
||||
it, _ := trie.NewDifferenceIterator(b, a)
|
||||
for it.Next(true) {
|
||||
// skip value nodes
|
||||
if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
|
||||
continue
|
||||
}
|
||||
node, nodeElements, err := sdtrie.ResolveNode(it, sdb.stateCache.TrieDB())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
switch node.NodeType {
|
||||
case sdtypes.Leaf:
|
||||
partialPath := trie.CompactToHex(nodeElements[0].([]byte))
|
||||
valueNodePath := append(node.Path, partialPath...)
|
||||
encodedPath := trie.HexToCompact(valueNodePath)
|
||||
leafKey := encodedPath[1:]
|
||||
|
||||
// if this node's path did not show up in diffPathsAtB
|
||||
// that means the node at this path was deleted (or moved) in B
|
||||
if _, ok := diffPathsAtB[common.Bytes2Hex(node.Path)]; !ok {
|
||||
// if this node's leaf key also did not show up in diffSlotsAtB
|
||||
// that means the node was deleted
|
||||
// in that case, emit an empty "removed" diff storage node
|
||||
if _, ok := diffSlotsAtB[common.Bytes2Hex(leafKey)]; !ok {
|
||||
if err := output(sdtypes.StorageNode{
|
||||
NodeType: sdtypes.Removed,
|
||||
Path: node.Path,
|
||||
NodeValue: []byte{},
|
||||
LeafKey: leafKey,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
// emit an empty "removed" diff with empty leaf key if the account was moved
|
||||
if err := output(sdtypes.StorageNode{
|
||||
NodeType: sdtypes.Removed,
|
||||
Path: node.Path,
|
||||
NodeValue: []byte{},
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
case sdtypes.Extension, sdtypes.Branch:
|
||||
// if this node's path did not show up in diffPathsAtB
|
||||
// that means the node at this path was deleted in B
|
||||
// in that case, emit an empty "removed" diff storage node
|
||||
if _, ok := diffPathsAtB[common.Bytes2Hex(node.Path)]; !ok {
|
||||
if intermediateNodes {
|
||||
if err := output(sdtypes.StorageNode{
|
||||
NodeType: sdtypes.Removed,
|
||||
Path: node.Path,
|
||||
NodeValue: []byte{},
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("unexpected node type %s", node.NodeType)
|
||||
}
|
||||
}
|
||||
return it.Error()
|
||||
}
|
||||
|
||||
// isWatchedAddress is used to check if a state account corresponds to one of the addresses the builder is configured to watch
|
||||
func isWatchedAddress(watchedAddressesLeafKeys map[common.Hash]struct{}, stateLeafKey []byte) bool {
|
||||
// If we aren't watching any specific addresses, we are watching everything
|
||||
if len(watchedAddressesLeafKeys) == 0 {
|
||||
return true
|
||||
}
|
||||
|
||||
_, ok := watchedAddressesLeafKeys[common.BytesToHash(stateLeafKey)]
|
||||
return ok
|
||||
}
|
||||
|
||||
// isWatchedStorageKey is used to check if a storage leaf corresponds to one of the storage slots the builder is configured to watch
|
||||
func isWatchedStorageKey(watchedKeys []common.Hash, storageLeafKey []byte) bool {
|
||||
// If we aren't watching any specific addresses, we are watching everything
|
||||
if len(watchedKeys) == 0 {
|
||||
return true
|
||||
}
|
||||
for _, hashKey := range watchedKeys {
|
||||
if bytes.Equal(hashKey.Bytes(), storageLeafKey) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
2523
pkg/builder_test.go
2523
pkg/builder_test.go
File diff suppressed because it is too large
Load Diff
@ -1,9 +1,8 @@
|
||||
package statediff
|
||||
|
||||
// Config holds config params for the statediffing service
|
||||
type Config struct {
|
||||
// ServiceConfig holds config params for the statediffing service
|
||||
type ServiceConfig struct {
|
||||
ServiceWorkers uint
|
||||
TrieWorkers uint
|
||||
WorkerQueueSize uint
|
||||
PreRuns []RangeRequest
|
||||
}
|
||||
|
@ -22,9 +22,11 @@ package statediff
|
||||
import (
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
sdtypes "github.com/cerc-io/plugeth-statediff/types"
|
||||
)
|
||||
|
||||
func sortKeys(data AccountMap) []string {
|
||||
func sortKeys(data sdtypes.AccountMap) []string {
|
||||
keys := make([]string, 0, len(data))
|
||||
for key := range data {
|
||||
keys = append(keys, key)
|
||||
|
@ -54,22 +54,29 @@ type LvLDBReaderConfig struct {
|
||||
DBCacheSize int
|
||||
}
|
||||
|
||||
// NewLvlDBReader creates a new Read using LevelDB
|
||||
// NewLvlDBReader creates a new Reader using LevelDB
|
||||
func NewLvlDBReader(conf LvLDBReaderConfig) (*LvlDBReader, error) {
|
||||
var edb ethdb.Database
|
||||
var err error
|
||||
|
||||
if conf.Mode == "local" {
|
||||
edb, err = rawdb.NewLevelDBDatabaseWithFreezer(conf.Path, conf.DBCacheSize, 256, conf.AncientPath, "eth-statediff-service", true)
|
||||
}
|
||||
switch conf.Mode {
|
||||
case "local":
|
||||
edb, err = rawdb.NewLevelDBDatabase(conf.Path, conf.DBCacheSize, 256, "eth-statediff-service", true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if conf.Mode == "remote" {
|
||||
edb, err = rawdb.NewDatabaseWithFreezer(edb, conf.AncientPath, "eth-statediff-service", true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
case "remote":
|
||||
edb, err = client.NewDatabaseClient(conf.Url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &LvlDBReader{
|
||||
ethDB: edb,
|
||||
stateDB: state.NewDatabaseWithConfig(edb, conf.TrieConfig),
|
||||
|
@ -31,7 +31,7 @@ import (
|
||||
func StartHTTPEndpoint(endpoint string, apis []rpc.API, modules []string, cors []string, vhosts []string, timeouts rpc.HTTPTimeouts) (*rpc.Server, error) {
|
||||
|
||||
srv := rpc.NewServer()
|
||||
err := node.RegisterApis(apis, modules, srv, false)
|
||||
err := node.RegisterApis(apis, modules, srv)
|
||||
if err != nil {
|
||||
utils.Fatalf("Could not register HTTP API: %w", err)
|
||||
}
|
||||
|
132
pkg/service.go
132
pkg/service.go
@ -22,12 +22,12 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
sd "github.com/cerc-io/plugeth-statediff"
|
||||
"github.com/cerc-io/plugeth-statediff"
|
||||
"github.com/cerc-io/plugeth-statediff/adapt"
|
||||
"github.com/cerc-io/plugeth-statediff/indexer/interfaces"
|
||||
sdtypes "github.com/cerc-io/plugeth-statediff/types"
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/node"
|
||||
"github.com/ethereum/go-ethereum/p2p"
|
||||
"github.com/ethereum/go-ethereum/rlp"
|
||||
"github.com/ethereum/go-ethereum/rpc"
|
||||
@ -38,35 +38,10 @@ import (
|
||||
|
||||
const defaultQueueSize = 1024
|
||||
|
||||
// StateDiffService is the state-diffing service interface
|
||||
type StateDiffService interface {
|
||||
// Lifecycle Start() and Stop()
|
||||
node.Lifecycle
|
||||
// APIs and Protocols() interface for node service registration
|
||||
APIs() []rpc.API
|
||||
Protocols() []p2p.Protocol
|
||||
// Loop is the main event loop for processing state diffs
|
||||
Loop(wg *sync.WaitGroup) error
|
||||
// Run is a one-off command to run on a predefined set of ranges
|
||||
Run(ranges []RangeRequest) error
|
||||
// StateDiffAt method to get state diff object at specific block
|
||||
StateDiffAt(blockNumber uint64, params sd.Params) (*sd.Payload, error)
|
||||
// StateDiffFor method to get state diff object at specific block
|
||||
StateDiffFor(blockHash common.Hash, params sd.Params) (*sd.Payload, error)
|
||||
// StateTrieAt method to get state trie object at specific block
|
||||
StateTrieAt(blockNumber uint64, params sd.Params) (*sd.Payload, error)
|
||||
// WriteStateDiffAt method to write state diff object directly to DB
|
||||
WriteStateDiffAt(blockNumber uint64, params sd.Params) error
|
||||
// WriteStateDiffFor method to get state trie object at specific block
|
||||
WriteStateDiffFor(blockHash common.Hash, params sd.Params) error
|
||||
// WriteStateDiffsInRange method to wrtie state diff objects within the range directly to the DB
|
||||
WriteStateDiffsInRange(start, stop uint64, params sd.Params) error
|
||||
}
|
||||
|
||||
// Service is the underlying struct for the state diffing service
|
||||
type Service struct {
|
||||
// Used to build the state diff objects
|
||||
Builder Builder
|
||||
Builder statediff.Builder
|
||||
// Used to read data from LevelDB
|
||||
lvlDBReader Reader
|
||||
// Used to signal shutdown of the service
|
||||
@ -77,27 +52,24 @@ type Service struct {
|
||||
queue chan RangeRequest
|
||||
// number of ranges we can work over concurrently
|
||||
workers uint
|
||||
// ranges configured locally
|
||||
preruns []RangeRequest
|
||||
}
|
||||
|
||||
// NewStateDiffService creates a new Service
|
||||
func NewStateDiffService(lvlDBReader Reader, indexer interfaces.StateDiffIndexer, conf Config) (*Service, error) {
|
||||
b, err := NewBuilder(lvlDBReader.StateDB(), conf.TrieWorkers)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
func NewStateDiffService(lvlDBReader Reader, indexer interfaces.StateDiffIndexer, conf ServiceConfig) *Service {
|
||||
builder := statediff.NewBuilderWithOptions(
|
||||
adapt.GethStateView(lvlDBReader.StateDB()),
|
||||
statediff.BuilderOptions{SubtrieWorkers: conf.TrieWorkers},
|
||||
)
|
||||
if conf.WorkerQueueSize == 0 {
|
||||
conf.WorkerQueueSize = defaultQueueSize
|
||||
}
|
||||
return &Service{
|
||||
lvlDBReader: lvlDBReader,
|
||||
Builder: b,
|
||||
Builder: builder,
|
||||
indexer: indexer,
|
||||
workers: conf.ServiceWorkers,
|
||||
queue: make(chan RangeRequest, conf.WorkerQueueSize),
|
||||
preruns: conf.PreRuns,
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Protocols exports the services p2p protocols, this service has none
|
||||
@ -119,17 +91,7 @@ func (sds *Service) APIs() []rpc.API {
|
||||
|
||||
// Run does a one-off processing run on the provided RangeRequests + any pre-runs, exiting afterwards
|
||||
func (sds *Service) Run(rngs []RangeRequest) error {
|
||||
for _, preRun := range sds.preruns {
|
||||
logrus.Infof("processing prerun range (%d, %d)", preRun.Start, preRun.Stop)
|
||||
for i := preRun.Start; i <= preRun.Stop; i++ {
|
||||
if err := sds.WriteStateDiffAt(i, preRun.Params); err != nil {
|
||||
return fmt.Errorf("error writing statediff at height %d in range (%d, %d) : %v", i, preRun.Start, preRun.Stop, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
sds.preruns = nil
|
||||
for _, rng := range rngs {
|
||||
logrus.Infof("processing prerun range (%d, %d)", rng.Start, rng.Stop)
|
||||
for i := rng.Start; i <= rng.Stop; i++ {
|
||||
if err := sds.WriteStateDiffAt(i, rng.Params); err != nil {
|
||||
return fmt.Errorf("error writing statediff at height %d in range (%d, %d) : %v", i, rng.Start, rng.Stop, err)
|
||||
@ -177,18 +139,12 @@ func (sds *Service) Loop(wg *sync.WaitGroup) error {
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
for _, preRun := range sds.preruns {
|
||||
if err := sds.WriteStateDiffsInRange(preRun.Start, preRun.Stop, preRun.Params); err != nil {
|
||||
close(sds.quitChan)
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// StateDiffAt returns a state diff object payload at the specific blockheight
|
||||
// This operation cannot be performed back past the point of db pruning; it requires an archival node for historical data
|
||||
func (sds *Service) StateDiffAt(blockNumber uint64, params sd.Params) (*sd.Payload, error) {
|
||||
func (sds *Service) StateDiffAt(blockNumber uint64, params statediff.Params) (*statediff.Payload, error) {
|
||||
currentBlock, err := sds.lvlDBReader.GetBlockByNumber(blockNumber)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@ -196,7 +152,7 @@ func (sds *Service) StateDiffAt(blockNumber uint64, params sd.Params) (*sd.Paylo
|
||||
logrus.Infof("sending state diff at block %d", blockNumber)
|
||||
|
||||
// compute leaf keys of watched addresses in the params
|
||||
params.ComputeWatchedAddressesLeafKeys()
|
||||
params.ComputeWatchedAddressesLeafPaths()
|
||||
|
||||
if blockNumber == 0 {
|
||||
return sds.processStateDiff(currentBlock, common.Hash{}, params)
|
||||
@ -210,7 +166,7 @@ func (sds *Service) StateDiffAt(blockNumber uint64, params sd.Params) (*sd.Paylo
|
||||
|
||||
// StateDiffFor returns a state diff object payload for the specific blockhash
|
||||
// This operation cannot be performed back past the point of db pruning; it requires an archival node for historical data
|
||||
func (sds *Service) StateDiffFor(blockHash common.Hash, params sd.Params) (*sd.Payload, error) {
|
||||
func (sds *Service) StateDiffFor(blockHash common.Hash, params statediff.Params) (*statediff.Payload, error) {
|
||||
currentBlock, err := sds.lvlDBReader.GetBlockByHash(blockHash)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@ -218,7 +174,7 @@ func (sds *Service) StateDiffFor(blockHash common.Hash, params sd.Params) (*sd.P
|
||||
logrus.Infof("sending state diff at block %s", blockHash.Hex())
|
||||
|
||||
// compute leaf keys of watched addresses in the params
|
||||
params.ComputeWatchedAddressesLeafKeys()
|
||||
params.ComputeWatchedAddressesLeafPaths()
|
||||
|
||||
if currentBlock.NumberU64() == 0 {
|
||||
return sds.processStateDiff(currentBlock, common.Hash{}, params)
|
||||
@ -231,8 +187,8 @@ func (sds *Service) StateDiffFor(blockHash common.Hash, params sd.Params) (*sd.P
|
||||
}
|
||||
|
||||
// processStateDiff method builds the state diff payload from the current block, parent state root, and provided params
|
||||
func (sds *Service) processStateDiff(currentBlock *types.Block, parentRoot common.Hash, params sd.Params) (*sd.Payload, error) {
|
||||
stateDiff, err := sds.Builder.BuildStateDiffObject(sd.Args{
|
||||
func (sds *Service) processStateDiff(currentBlock *types.Block, parentRoot common.Hash, params statediff.Params) (*statediff.Payload, error) {
|
||||
stateDiff, err := sds.Builder.BuildStateDiffObject(statediff.Args{
|
||||
BlockHash: currentBlock.Hash(),
|
||||
BlockNumber: currentBlock.Number(),
|
||||
OldStateRoot: parentRoot,
|
||||
@ -249,8 +205,8 @@ func (sds *Service) processStateDiff(currentBlock *types.Block, parentRoot commo
|
||||
return sds.newPayload(stateDiffRlp, currentBlock, params)
|
||||
}
|
||||
|
||||
func (sds *Service) newPayload(stateObject []byte, block *types.Block, params sd.Params) (*sd.Payload, error) {
|
||||
payload := &sd.Payload{
|
||||
func (sds *Service) newPayload(stateObject []byte, block *types.Block, params statediff.Params) (*statediff.Payload, error) {
|
||||
payload := &statediff.Payload{
|
||||
StateObjectRlp: stateObject,
|
||||
}
|
||||
if params.IncludeBlock {
|
||||
@ -281,34 +237,6 @@ func (sds *Service) newPayload(stateObject []byte, block *types.Block, params sd
|
||||
return payload, nil
|
||||
}
|
||||
|
||||
// StateTrieAt returns a state trie object payload at the specified blockheight
|
||||
// This operation cannot be performed back past the point of db pruning; it requires an archival node for historical data
|
||||
func (sds *Service) StateTrieAt(blockNumber uint64, params sd.Params) (*sd.Payload, error) {
|
||||
currentBlock, err := sds.lvlDBReader.GetBlockByNumber(blockNumber)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
logrus.Infof("sending state trie at block %d", blockNumber)
|
||||
|
||||
// compute leaf keys of watched addresses in the params
|
||||
params.ComputeWatchedAddressesLeafKeys()
|
||||
|
||||
return sds.processStateTrie(currentBlock, params)
|
||||
}
|
||||
|
||||
func (sds *Service) processStateTrie(block *types.Block, params sd.Params) (*sd.Payload, error) {
|
||||
stateNodes, err := sds.Builder.BuildStateTrieObject(block)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stateTrieRlp, err := rlp.EncodeToBytes(&stateNodes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
logrus.Infof("state trie object at block %d is %d bytes in length", block.Number().Uint64(), len(stateTrieRlp))
|
||||
return sds.newPayload(stateTrieRlp, block, params)
|
||||
}
|
||||
|
||||
// Start is used to begin the service
|
||||
func (sds *Service) Start() error {
|
||||
logrus.Info("starting statediff service")
|
||||
@ -325,7 +253,7 @@ func (sds *Service) Stop() error {
|
||||
// WriteStateDiffAt writes a state diff at the specific blockheight directly to the database
|
||||
// This operation cannot be performed back past the point of db pruning; it requires an archival node
|
||||
// for historical data
|
||||
func (sds *Service) WriteStateDiffAt(blockNumber uint64, params sd.Params) error {
|
||||
func (sds *Service) WriteStateDiffAt(blockNumber uint64, params statediff.Params) error {
|
||||
logrus.Infof("Writing state diff at block %d", blockNumber)
|
||||
t := time.Now()
|
||||
currentBlock, err := sds.lvlDBReader.GetBlockByNumber(blockNumber)
|
||||
@ -334,7 +262,7 @@ func (sds *Service) WriteStateDiffAt(blockNumber uint64, params sd.Params) error
|
||||
}
|
||||
|
||||
// compute leaf keys of watched addresses in the params
|
||||
params.ComputeWatchedAddressesLeafKeys()
|
||||
params.ComputeWatchedAddressesLeafPaths()
|
||||
|
||||
parentRoot := common.Hash{}
|
||||
if blockNumber != 0 {
|
||||
@ -350,7 +278,7 @@ func (sds *Service) WriteStateDiffAt(blockNumber uint64, params sd.Params) error
|
||||
// WriteStateDiffFor writes a state diff for the specific blockHash directly to the database
|
||||
// This operation cannot be performed back past the point of db pruning; it requires an archival node
|
||||
// for historical data
|
||||
func (sds *Service) WriteStateDiffFor(blockHash common.Hash, params sd.Params) error {
|
||||
func (sds *Service) WriteStateDiffFor(blockHash common.Hash, params statediff.Params) error {
|
||||
logrus.Infof("Writing state diff for block %s", blockHash.Hex())
|
||||
t := time.Now()
|
||||
currentBlock, err := sds.lvlDBReader.GetBlockByHash(blockHash)
|
||||
@ -359,7 +287,7 @@ func (sds *Service) WriteStateDiffFor(blockHash common.Hash, params sd.Params) e
|
||||
}
|
||||
|
||||
// compute leaf keys of watched addresses in the params
|
||||
params.ComputeWatchedAddressesLeafKeys()
|
||||
params.ComputeWatchedAddressesLeafPaths()
|
||||
|
||||
parentRoot := common.Hash{}
|
||||
if currentBlock.NumberU64() != 0 {
|
||||
@ -373,7 +301,7 @@ func (sds *Service) WriteStateDiffFor(blockHash common.Hash, params sd.Params) e
|
||||
}
|
||||
|
||||
// Writes a state diff from the current block, parent state root, and provided params
|
||||
func (sds *Service) writeStateDiff(block *types.Block, parentRoot common.Hash, params sd.Params, t time.Time) error {
|
||||
func (sds *Service) writeStateDiff(block *types.Block, parentRoot common.Hash, params statediff.Params, t time.Time) error {
|
||||
var totalDifficulty *big.Int
|
||||
var receipts types.Receipts
|
||||
var err error
|
||||
@ -398,28 +326,30 @@ func (sds *Service) writeStateDiff(block *types.Block, parentRoot common.Hash, p
|
||||
return err
|
||||
}
|
||||
// defer handling of commit/rollback for any return case
|
||||
output := func(node sdtypes.StateNode) error {
|
||||
output := func(node sdtypes.StateLeafNode) error {
|
||||
return sds.indexer.PushStateNode(tx, node, block.Hash().String())
|
||||
}
|
||||
codeOutput := func(c sdtypes.CodeAndCodeHash) error {
|
||||
return sds.indexer.PushCodeAndCodeHash(tx, c)
|
||||
codeOutput := func(c sdtypes.IPLD) error {
|
||||
return sds.indexer.PushIPLD(tx, c)
|
||||
}
|
||||
prom.SetTimeMetric(prom.T_BLOCK_PROCESSING, time.Now().Sub(t))
|
||||
t = time.Now()
|
||||
err = sds.Builder.WriteStateDiffObject(sdtypes.StateRoots{
|
||||
err = sds.Builder.WriteStateDiff(statediff.Args{
|
||||
NewStateRoot: block.Root(),
|
||||
OldStateRoot: parentRoot,
|
||||
BlockNumber: block.Number(),
|
||||
BlockHash: block.Hash(),
|
||||
}, params, output, codeOutput)
|
||||
prom.SetTimeMetric(prom.T_STATE_PROCESSING, time.Now().Sub(t))
|
||||
t = time.Now()
|
||||
err = tx.Submit(err)
|
||||
err = tx.Submit()
|
||||
prom.SetLastProcessedHeight(height)
|
||||
prom.SetTimeMetric(prom.T_POSTGRES_TX_COMMIT, time.Now().Sub(t))
|
||||
return err
|
||||
}
|
||||
|
||||
// WriteStateDiffsInRange adds a RangeRequest to the work queue
|
||||
func (sds *Service) WriteStateDiffsInRange(start, stop uint64, params sd.Params) error {
|
||||
func (sds *Service) WriteStateDiffsInRange(start, stop uint64, params statediff.Params) error {
|
||||
if stop < start {
|
||||
return fmt.Errorf("invalid block range (%d, %d): stop height must be greater or equal to start height", start, stop)
|
||||
}
|
||||
|
14
pkg/types.go
14
pkg/types.go
@ -21,22 +21,8 @@ package statediff
|
||||
|
||||
import (
|
||||
sd "github.com/cerc-io/plugeth-statediff"
|
||||
sdTypes "github.com/cerc-io/plugeth-statediff/types"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
)
|
||||
|
||||
// AccountMap is a mapping of hex encoded path => account wrapper
|
||||
type AccountMap map[string]accountWrapper
|
||||
|
||||
// accountWrapper is used to temporary associate the unpacked node with its raw values
|
||||
type accountWrapper struct {
|
||||
Account *types.StateAccount
|
||||
NodeType sdTypes.NodeType
|
||||
Path []byte
|
||||
NodeValue []byte
|
||||
LeafKey []byte
|
||||
}
|
||||
|
||||
// RangeRequest holds range quest work params
|
||||
type RangeRequest struct {
|
||||
Start, Stop uint64
|
||||
|
@ -17,4 +17,4 @@ mkdir -p /app/geth-rw && \
|
||||
sudo mount -t overlay overlay -o lowerdir=/app/geth-ro,upperdir=/tmp/overlay/upper,workdir=/tmp/overlay/work /app/geth-rw && \
|
||||
|
||||
echo "Running the statediff service" && \
|
||||
sudo ./eth-statediff-service "$VDB_COMMAND" --config=config.toml
|
||||
exec sudo ./eth-statediff-service "$VDB_COMMAND" --config=config.toml
|
||||
|
Loading…
Reference in New Issue
Block a user