cerc-io/ipld-eth-state-snapshot
16
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #25 from vulcanize/release-v3.0.1

Browse Source 
fix minor bug
This commit is contained in:
Ian Norden 2022-03-31 10:34:36 -05:00 committed by GitHub
commit 560199f930
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 121 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -1,2 +1,2 @@
.idea/ .idea/
eth-pg-ipfs-state-snapshot ipld-eth-state-snapshot

33
README.md
View File

@ -1,26 +1,35 @@
# eth-pg-ipfs-state-snapshot # ipld-eth-state-snapshot
> Tool for extracting the entire Ethereum state at a particular block height from leveldb into Postgres-backed IPFS > Tool for extracting the entire Ethereum state at a particular block height from leveldb into Postgres-backed IPFS
[![Go Report Card](https://goreportcard.com/badge/github.com/vulcanize/eth-pg-ipfs-state-snapshot)](https://goreportcard.com/report/github.com/vulcanize/eth-pg-ipfs-state-snapshot) [![Go Report Card](https://goreportcard.com/badge/github.com/vulcanize/ipld-eth-state-snapshot)](https://goreportcard.com/report/github.com/vulcanize/ipld-eth-state-snapshot)
## Usage ## Usage
./eth-pg-ipfs-state-snapshot stateSnapshot --config={path to toml config file} ./ipld-eth-state-snapshot stateSnapshot --config={path to toml config file}
### Config
Config format: Config format:
```toml ```toml
[database] [snapshot]
name = "vulcanize_public" mode = "file" # indicates output mode ("postgres" or "file")
hostname = "localhost" workers = 4 # degree of concurrency, the state trie is subdivided into sectiosn that are traversed and processed concurrently
port = 5432 blockHeight = -1 # blockheight to perform the snapshot at (-1 indicates to use the latest blockheight found in leveldb)
user = "postgres" recoveryFile = "recovery_file" # specifies a file to output recovery information on error or premature closure
[leveldb] [leveldb]
path = "/Users/user/Library/Ethereum/geth/chaindata" path = "/Users/user/Library/Ethereum/geth/chaindata" # path to geth leveldb
ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient" ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient" # path to geth ancient database
[snapshot] [database]
blockHeight = 0 name = "vulcanize_public" # postgres database name
hostname = "localhost" # postgres host
port = 5432 # postgres port
user = "postgres" # postgres user
password = "" # postgres password
[file]
outputDir = "output_dir/" # when operating in 'file' output mode, this is the directory the files are written to
``` ```

2
cmd/root.go
View File

@ -33,7 +33,7 @@ var (
) )
var rootCmd = &cobra.Command{ var rootCmd = &cobra.Command{
Use: "eth-pg-ipfs-state-snapshot", Use: "ipld-eth-state-snapshot",
PersistentPreRun: initFuncs, PersistentPreRun: initFuncs,
} }

32
cmd/stateSnapshot.go
View File

@ -16,11 +16,13 @@
package cmd package cmd
import ( import (
"fmt"
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/spf13/viper" "github.com/spf13/viper"
"github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/snapshot" "github.com/vulcanize/ipld-eth-state-snapshot/pkg/snapshot"
) )
// stateSnapshotCmd represents the stateSnapshot command // stateSnapshotCmd represents the stateSnapshot command
@ -29,7 +31,7 @@ var stateSnapshotCmd = &cobra.Command{
Short: "Extract the entire Ethereum state from leveldb and publish into PG-IPFS", Short: "Extract the entire Ethereum state from leveldb and publish into PG-IPFS",
Long: `Usage Long: `Usage
./eth-pg-ipfs-state-snapshot stateSnapshot --config={path to toml config file}`, ./ipld-eth-state-snapshot stateSnapshot --config={path to toml config file}`,
Run: func(cmd *cobra.Command, args []string) { Run: func(cmd *cobra.Command, args []string) {
subCommand = cmd.CalledAs() subCommand = cmd.CalledAs()
logWithCommand = *logrus.WithField("SubCommand", subCommand) logWithCommand = *logrus.WithField("SubCommand", subCommand)
@ -38,19 +40,26 @@ var stateSnapshotCmd = &cobra.Command{
} }
func stateSnapshot() { func stateSnapshot() {
config := snapshot.NewConfig() modeStr := viper.GetString("snapshot.mode")
mode := snapshot.SnapshotMode(modeStr)
config, err := snapshot.NewConfig(mode)
if err != nil {
logWithCommand.Fatal("unable to initialize config: %v", err)
}
logWithCommand.Infof("opening levelDB and ancient data at %s and %s",
config.Eth.LevelDBPath, config.Eth.AncientDBPath)
edb, err := snapshot.NewLevelDB(config.Eth) edb, err := snapshot.NewLevelDB(config.Eth)
if err != nil { if err != nil {
logWithCommand.Fatal(err) logWithCommand.Fatal(err)
} }
height := viper.GetInt64("snapshot.blockHeight")
recoveryFile := viper.GetString("snapshot.recoveryFile") recoveryFile := viper.GetString("snapshot.recoveryFile")
if recoveryFile == "" { if recoveryFile == "" {
recoveryFile = "./snapshot_recovery" recoveryFile = fmt.Sprintf("./%d_snapshot_recovery", height)
logWithCommand.Infof("no recovery file set, creating default: %s", recoveryFile)
} }
mode := viper.GetString("snapshot.mode") pub, err := snapshot.NewPublisher(mode, config)
pub, err := snapshot.NewPublisher(snapshot.SnapshotMode(mode), config)
if err != nil { if err != nil {
logWithCommand.Fatal(err) logWithCommand.Fatal(err)
} }
@ -59,7 +68,6 @@ func stateSnapshot() {
if err != nil { if err != nil {
logWithCommand.Fatal(err) logWithCommand.Fatal(err)
} }
height := viper.GetInt64("snapshot.blockHeight")
workers := viper.GetUint("snapshot.workers") workers := viper.GetUint("snapshot.workers")
if height < 0 { if height < 0 {
@ -81,10 +89,16 @@ func init() {
stateSnapshotCmd.PersistentFlags().String("leveldb-path", "", "path to primary datastore") stateSnapshotCmd.PersistentFlags().String("leveldb-path", "", "path to primary datastore")
stateSnapshotCmd.PersistentFlags().String("ancient-path", "", "path to ancient datastore") stateSnapshotCmd.PersistentFlags().String("ancient-path", "", "path to ancient datastore")
stateSnapshotCmd.PersistentFlags().String("block-height", "", "blockheight to extract state at") stateSnapshotCmd.PersistentFlags().String("block-height", "", "blockheight to extract state at")
stateSnapshotCmd.PersistentFlags().Int("workers", 0, "number of concurrent workers to use") stateSnapshotCmd.PersistentFlags().Int("workers", 1, "number of concurrent workers to use")
stateSnapshotCmd.PersistentFlags().String("recovery-file", "", "file to recover from a previous iteration")
stateSnapshotCmd.PersistentFlags().String("snapshot-mode", "postgres", "output mode for snapshot ('file' or 'postgres')")
stateSnapshotCmd.PersistentFlags().String("output-dir", "", "directory for writing ouput to while operating in 'file' mode")
viper.BindPFlag("leveldb.path", stateSnapshotCmd.PersistentFlags().Lookup("leveldb-path")) viper.BindPFlag("leveldb.path", stateSnapshotCmd.PersistentFlags().Lookup("leveldb-path"))
viper.BindPFlag("leveldb.ancient", stateSnapshotCmd.PersistentFlags().Lookup("ancient-path")) viper.BindPFlag("leveldb.ancient", stateSnapshotCmd.PersistentFlags().Lookup("ancient-path"))
viper.BindPFlag("snapshot.blockHeight", stateSnapshotCmd.PersistentFlags().Lookup("block-height")) viper.BindPFlag("snapshot.blockHeight", stateSnapshotCmd.PersistentFlags().Lookup("block-height"))
viper.BindPFlag("snapshot.workers", stateSnapshotCmd.PersistentFlags().Lookup("workers")) viper.BindPFlag("snapshot.workers", stateSnapshotCmd.PersistentFlags().Lookup("workers"))
viper.BindPFlag("snapshot.recoveryFile", stateSnapshotCmd.PersistentFlags().Lookup("recovery-file"))
viper.BindPFlag("snapshot.mode", stateSnapshotCmd.PersistentFlags().Lookup("snapshot-mode"))
viper.BindPFlag("file.outputDir", stateSnapshotCmd.PersistentFlags().Lookup("output-dir"))
} }

8
environments/example.toml
View File

@ -9,4 +9,10 @@
ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient" ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient"
[snapshot] [snapshot]
blockHeight = -1 mode = "file"
workers = 4
blockHeight = -1
recoveryFile = "recovery_file"
[file]
outputDir = "output_dir/"

3
fixture/service.go
View File

@ -5,9 +5,8 @@ import (
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/types"
// "github.com/jmoiron/sqlx"
snapt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/types" snapt "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types"
) )
var Block1_Header = types.Header{ var Block1_Header = types.Header{

6
go.mod
View File

@ -1,11 +1,11 @@
module github.com/vulcanize/eth-pg-ipfs-state-snapshot module github.com/vulcanize/ipld-eth-state-snapshot
go 1.15 go 1.15
require ( require (
github.com/btcsuite/btcd v0.22.0-beta // indirect github.com/btcsuite/btcd v0.22.0-beta // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/ethereum/go-ethereum v1.10.15 github.com/ethereum/go-ethereum v1.10.16
github.com/fsnotify/fsnotify v1.5.1 // indirect github.com/fsnotify/fsnotify v1.5.1 // indirect
github.com/go-kit/kit v0.10.0 // indirect github.com/go-kit/kit v0.10.0 // indirect
github.com/golang/mock v1.6.0 github.com/golang/mock v1.6.0
@ -43,4 +43,4 @@ require (
lukechampine.com/blake3 v1.1.7 // indirect lukechampine.com/blake3 v1.1.7 // indirect
) )
replace github.com/ethereum/go-ethereum v1.10.15 => github.com/vulcanize/go-ethereum v1.10.15-statediff-3.0.1 replace github.com/ethereum/go-ethereum v1.10.16 => github.com/vulcanize/go-ethereum v1.10.16-statediff-3.0.2

8
go.sum
View File

@ -146,6 +146,7 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea h1:j4317fAZh7X6GqbFowYdYdI0L9bwxL07jyPZIdepyZ0= github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea h1:j4317fAZh7X6GqbFowYdYdI0L9bwxL07jyPZIdepyZ0=
github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ=
github.com/deckarep/golang-set v1.8.0/go.mod h1:5nI87KwE7wgsBU1F4GKAw2Qod7p5kyS383rP6+o6qqo=
github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218=
github.com/deepmap/oapi-codegen v1.6.0/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE9iAUlWl9Al3M= github.com/deepmap/oapi-codegen v1.6.0/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE9iAUlWl9Al3M=
github.com/deepmap/oapi-codegen v1.8.2/go.mod h1:YLgSKSDv/bZQB7N4ws6luhozi3cEdRktEqrX88CvjIw= github.com/deepmap/oapi-codegen v1.8.2/go.mod h1:YLgSKSDv/bZQB7N4ws6luhozi3cEdRktEqrX88CvjIw=
@ -168,6 +169,7 @@ github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4s
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0=
github.com/ethereum/go-ethereum v1.10.15/go.mod h1:W3yfrFyL9C1pHcwY5hmRHVDaorTiQxhYBkKyu5mEDHw=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c= github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c=
github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0=
@ -290,6 +292,7 @@ github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoA
github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc=
github.com/graph-gophers/graphql-go v1.3.0/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc=
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
@ -468,6 +471,7 @@ github.com/jackc/puddle v1.2.1 h1:gI8os0wpRXFd4FiAY2dWiqRK037tjj3t7rKFeO4X5iw=
github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 h1:6OvNmYgJyexcZ3pYbTI9jWx5tHo1Dee/tWbLMfPe2TA= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 h1:6OvNmYgJyexcZ3pYbTI9jWx5tHo1Dee/tWbLMfPe2TA=
github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA=
github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY= github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY=
github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o= github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o=
@ -497,6 +501,7 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V
github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0= github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0=
github.com/karalabe/usb v0.0.0-20211005121534-4c5740d64559/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= github.com/karalabe/usb v0.0.0-20211005121534-4c5740d64559/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU=
github.com/karalabe/usb v0.0.2/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU=
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
@ -809,6 +814,8 @@ github.com/vulcanize/go-eth-state-node-iterator v1.0.1 h1:lI8+moQ0Nv62NwmiCxgvWk
github.com/vulcanize/go-eth-state-node-iterator v1.0.1/go.mod h1:uWhleTvUEZ+cEkNRIAmBpZ14KilTP71OxY5NZDrpNlo= github.com/vulcanize/go-eth-state-node-iterator v1.0.1/go.mod h1:uWhleTvUEZ+cEkNRIAmBpZ14KilTP71OxY5NZDrpNlo=
github.com/vulcanize/go-ethereum v1.10.15-statediff-3.0.1 h1:MX7WcTwxpxOoYVMifrs9vsHDmM6gKOFF+2KnP19LddI= github.com/vulcanize/go-ethereum v1.10.15-statediff-3.0.1 h1:MX7WcTwxpxOoYVMifrs9vsHDmM6gKOFF+2KnP19LddI=
github.com/vulcanize/go-ethereum v1.10.15-statediff-3.0.1/go.mod h1:XO9WLkNXfwoJN05BZj0//xgOWHJyUrUPdnudbQfKlUo= github.com/vulcanize/go-ethereum v1.10.15-statediff-3.0.1/go.mod h1:XO9WLkNXfwoJN05BZj0//xgOWHJyUrUPdnudbQfKlUo=
github.com/vulcanize/go-ethereum v1.10.16-statediff-3.0.2 h1:H3SLHZdvTyKYbFc1CO2b8A9XF3BcakcXtvThKPbgT8k=
github.com/vulcanize/go-ethereum v1.10.16-statediff-3.0.2/go.mod h1:NI+tCVeIQBPrMfJUZvTLjhCieb7CZcmNPbJVlXbncxU=
github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM=
github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
@ -946,6 +953,7 @@ golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d/go.mod h1:m0MpNAwzfU5UDzcl9v
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211209124913-491a49abca63 h1:iocB37TsdFuN6IBRZ+ry36wrkoV51/tl5vOWqkcPGvY= golang.org/x/net v0.0.0-20211209124913-491a49abca63 h1:iocB37TsdFuN6IBRZ+ry36wrkoV51/tl5vOWqkcPGvY=

2
main.go
View File

@ -18,7 +18,7 @@ package main
import ( import (
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
"github.com/vulcanize/eth-pg-ipfs-state-snapshot/cmd" "github.com/vulcanize/ipld-eth-state-snapshot/cmd"
) )
func main() { func main() {

30
pkg/snapshot/config.go
View File

@ -16,8 +16,11 @@
package snapshot package snapshot
import ( import (
"fmt"
"time" "time"
"github.com/sirupsen/logrus"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
ethNode "github.com/ethereum/go-ethereum/statediff/indexer/node" ethNode "github.com/ethereum/go-ethereum/statediff/indexer/node"
"github.com/spf13/viper" "github.com/spf13/viper"
@ -38,6 +41,8 @@ type SnapshotMode string
const ( const (
PgSnapshot SnapshotMode = "postgres" PgSnapshot SnapshotMode = "postgres"
FileSnapshot SnapshotMode = "file" FileSnapshot SnapshotMode = "file"
defaultOutputDir = "./snapshot_output"
) )
// Config contains params for both databases the service uses // Config contains params for both databases the service uses
@ -64,18 +69,17 @@ type FileConfig struct {
OutputDir string OutputDir string
} }
func NewConfig() *Config { func NewConfig(mode SnapshotMode) (*Config, error) {
ret := &Config{ ret := &Config{
&EthConfig{}, &EthConfig{},
&DBConfig{}, &DBConfig{},
&FileConfig{}, &FileConfig{},
} }
ret.Init() return ret, ret.Init(mode)
return ret
} }
// Init Initialises config // Init Initialises config
func (c *Config) Init() { func (c *Config) Init(mode SnapshotMode) error {
viper.BindEnv("ethereum.nodeID", ETH_NODE_ID) viper.BindEnv("ethereum.nodeID", ETH_NODE_ID)
viper.BindEnv("ethereum.clientName", ETH_CLIENT_NAME) viper.BindEnv("ethereum.clientName", ETH_CLIENT_NAME)
viper.BindEnv("ethereum.genesisBlock", ETH_GENESIS_BLOCK) viper.BindEnv("ethereum.genesisBlock", ETH_GENESIS_BLOCK)
@ -95,8 +99,15 @@ func (c *Config) Init() {
c.Eth.AncientDBPath = viper.GetString("leveldb.ancient") c.Eth.AncientDBPath = viper.GetString("leveldb.ancient")
c.Eth.LevelDBPath = viper.GetString("leveldb.path") c.Eth.LevelDBPath = viper.GetString("leveldb.path")
c.DB.Init() switch mode {
c.File.Init() case FileSnapshot:
c.File.Init()
case PgSnapshot:
c.DB.Init()
default:
return fmt.Errorf("no output mode specified")
}
return nil
} }
func (c *DBConfig) Init() { func (c *DBConfig) Init() {
@ -125,7 +136,12 @@ func (c *DBConfig) Init() {
c.URI = dbParams.DbConnectionString() c.URI = dbParams.DbConnectionString()
} }
func (c *FileConfig) Init() { func (c *FileConfig) Init() error {
viper.BindEnv("file.outputDir", "FILE_OUTPUT_DIR") viper.BindEnv("file.outputDir", "FILE_OUTPUT_DIR")
c.OutputDir = viper.GetString("file.outputDir") c.OutputDir = viper.GetString("file.outputDir")
if c.OutputDir == "" {
logrus.Infof("no output directory set, using default: %s", defaultOutputDir)
c.OutputDir = defaultOutputDir
}
return nil
} }

8
pkg/snapshot/file/publisher.go
View File

@ -34,7 +34,7 @@ import (
"github.com/ethereum/go-ethereum/statediff/indexer/ipld" "github.com/ethereum/go-ethereum/statediff/indexer/ipld"
nodeinfo "github.com/ethereum/go-ethereum/statediff/indexer/node" nodeinfo "github.com/ethereum/go-ethereum/statediff/indexer/node"
"github.com/ethereum/go-ethereum/statediff/indexer/shared" "github.com/ethereum/go-ethereum/statediff/indexer/shared"
snapt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/types" snapt "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types"
) )
var _ snapt.Publisher = (*publisher)(nil) var _ snapt.Publisher = (*publisher)(nil)
@ -123,8 +123,8 @@ func makeFileWriters(dir string, tables []*snapt.Table) (fileWriters, error) {
// with the Postgres COPY command. // with the Postgres COPY command.
// The output directory will be created if it does not exist. // The output directory will be created if it does not exist.
func NewPublisher(path string, node nodeinfo.Info) (*publisher, error) { func NewPublisher(path string, node nodeinfo.Info) (*publisher, error) {
if err := os.MkdirAll(path, 0755); err != nil { if err := os.MkdirAll(path, 0777); err != nil {
return nil, err return nil, fmt.Errorf("unable to make MkdirAll for path: %s err: %s", path, err)
} }
writers, err := makeFileWriters(path, perBlockTables) writers, err := makeFileWriters(path, perBlockTables)
if err != nil { if err != nil {
@ -285,7 +285,7 @@ func (p *publisher) logNodeCounters() {
} }
func (p *publisher) printNodeCounters(msg string) { func (p *publisher) printNodeCounters(msg string) {
log.WithFields(log.Fields{ logrus.WithFields(logrus.Fields{
"runtime": time.Now().Sub(p.startTime).String(), "runtime": time.Now().Sub(p.startTime).String(),
"state nodes": atomic.LoadUint64(&p.stateNodeCounter), "state nodes": atomic.LoadUint64(&p.stateNodeCounter),
"storage nodes": atomic.LoadUint64(&p.storageNodeCounter), "storage nodes": atomic.LoadUint64(&p.storageNodeCounter),

6
pkg/snapshot/file/publisher_test.go
View File

@ -12,9 +12,9 @@ import (
"github.com/ethereum/go-ethereum/statediff/indexer/ipld" "github.com/ethereum/go-ethereum/statediff/indexer/ipld"
"github.com/jackc/pgx/v4" "github.com/jackc/pgx/v4"
fixt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/fixture" fixt "github.com/vulcanize/ipld-eth-state-snapshot/fixture"
snapt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/types" snapt "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types"
"github.com/vulcanize/eth-pg-ipfs-state-snapshot/test" "github.com/vulcanize/ipld-eth-state-snapshot/test"
) )
var ( var (

7
pkg/snapshot/mock/util.go
View File

@ -1,7 +1,10 @@
package mock package mock
import "fmt" import (
import "github.com/golang/mock/gomock" "fmt"
"github.com/golang/mock/gomock"
)
type anyOfMatcher struct { type anyOfMatcher struct {
values []interface{} values []interface{}

2
pkg/snapshot/pg/publisher.go
View File

@ -33,7 +33,7 @@ import (
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
"github.com/ethereum/go-ethereum/statediff/indexer/ipld" "github.com/ethereum/go-ethereum/statediff/indexer/ipld"
"github.com/ethereum/go-ethereum/statediff/indexer/shared" "github.com/ethereum/go-ethereum/statediff/indexer/shared"
snapt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/types" snapt "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types"
) )
var _ snapt.Publisher = (*publisher)(nil) var _ snapt.Publisher = (*publisher)(nil)

6
pkg/snapshot/pg/publisher_test.go
View File

@ -9,9 +9,9 @@ import (
"github.com/ethereum/go-ethereum/statediff/indexer/ipld" "github.com/ethereum/go-ethereum/statediff/indexer/ipld"
"github.com/jackc/pgx/v4" "github.com/jackc/pgx/v4"
fixt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/fixture" fixt "github.com/vulcanize/ipld-eth-state-snapshot/fixture"
snapt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/types" snapt "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types"
"github.com/vulcanize/eth-pg-ipfs-state-snapshot/test" "github.com/vulcanize/ipld-eth-state-snapshot/test"
) )
var ( var (

12
pkg/snapshot/service.go
View File

@ -31,8 +31,8 @@ import (
"github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
. "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/types"
iter "github.com/vulcanize/go-eth-state-node-iterator" iter "github.com/vulcanize/go-eth-state-node-iterator"
. "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types"
) )
var ( var (
@ -55,9 +55,15 @@ type Service struct {
} }
func NewLevelDB(con *EthConfig) (ethdb.Database, error) { func NewLevelDB(con *EthConfig) (ethdb.Database, error) {
return rawdb.NewLevelDBDatabaseWithFreezer( println(con.LevelDBPath)
con.LevelDBPath, 1024, 256, con.AncientDBPath, "eth-pg-ipfs-state-snapshot", false, println(con.AncientDBPath)
edb, err := rawdb.NewLevelDBDatabaseWithFreezer(
con.LevelDBPath, 1024, 256, con.AncientDBPath, "ipld-eth-state-snapshot", true,
) )
if err != nil {
return nil, fmt.Errorf("unable to create NewLevelDBDatabaseWithFreezer: %s", err)
}
return edb, nil
} }
// NewSnapshotService creates Service. // NewSnapshotService creates Service.

8
pkg/snapshot/service_test.go
View File

@ -8,10 +8,10 @@ import (
"github.com/golang/mock/gomock" "github.com/golang/mock/gomock"
fixt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/fixture" fixt "github.com/vulcanize/ipld-eth-state-snapshot/fixture"
mock "github.com/vulcanize/eth-pg-ipfs-state-snapshot/mocks/snapshot" mock "github.com/vulcanize/ipld-eth-state-snapshot/mocks/snapshot"
snapt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/types" snapt "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types"
"github.com/vulcanize/eth-pg-ipfs-state-snapshot/test" "github.com/vulcanize/ipld-eth-state-snapshot/test"
) )
func testConfig(leveldbpath, ancientdbpath string) *Config { func testConfig(leveldbpath, ancientdbpath string) *Config {

6
pkg/snapshot/util.go
View File

@ -6,9 +6,9 @@ import (
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
file "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/snapshot/file" file "github.com/vulcanize/ipld-eth-state-snapshot/pkg/snapshot/file"
pg "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/snapshot/pg" pg "github.com/vulcanize/ipld-eth-state-snapshot/pkg/snapshot/pg"
snapt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/types" snapt "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types"
) )
func NewPublisher(mode SnapshotMode, config *Config) (snapt.Publisher, error) { func NewPublisher(mode SnapshotMode, config *Config) (snapt.Publisher, error) {