Ian Norden
GitHub
commit
560199f930
2
.gitignore
vendored
2
.gitignore
vendored
@ -1,2 +1,2 @@
|
||||
.idea/
|
||||
eth-pg-ipfs-state-snapshot
|
||||
ipld-eth-state-snapshot
|
||||
|
||||
33
README.md
33
README.md
@ -1,26 +1,35 @@
|
||||
# eth-pg-ipfs-state-snapshot
|
||||
# ipld-eth-state-snapshot
|
||||
|
||||
> Tool for extracting the entire Ethereum state at a particular block height from leveldb into Postgres-backed IPFS
|
||||
|
||||
[](https://goreportcard.com/report/github.com/vulcanize/eth-pg-ipfs-state-snapshot)
|
||||
[](https://goreportcard.com/report/github.com/vulcanize/ipld-eth-state-snapshot)
|
||||
|
||||
## Usage
|
||||
|
||||
./eth-pg-ipfs-state-snapshot stateSnapshot --config={path to toml config file}
|
||||
./ipld-eth-state-snapshot stateSnapshot --config={path to toml config file}
|
||||
|
||||
### Config
|
||||
|
||||
Config format:
|
||||
|
||||
```toml
|
||||
[database]
|
||||
name = "vulcanize_public"
|
||||
hostname = "localhost"
|
||||
port = 5432
|
||||
user = "postgres"
|
||||
[snapshot]
|
||||
mode = "file" # indicates output mode ("postgres" or "file")
|
||||
workers = 4 # degree of concurrency, the state trie is subdivided into sectiosn that are traversed and processed concurrently
|
||||
blockHeight = -1 # blockheight to perform the snapshot at (-1 indicates to use the latest blockheight found in leveldb)
|
||||
recoveryFile = "recovery_file" # specifies a file to output recovery information on error or premature closure
|
||||
|
||||
[leveldb]
|
||||
path = "/Users/user/Library/Ethereum/geth/chaindata"
|
||||
ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient"
|
||||
path = "/Users/user/Library/Ethereum/geth/chaindata" # path to geth leveldb
|
||||
ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient" # path to geth ancient database
|
||||
|
||||
[snapshot]
|
||||
blockHeight = 0
|
||||
[database]
|
||||
name = "vulcanize_public" # postgres database name
|
||||
hostname = "localhost" # postgres host
|
||||
port = 5432 # postgres port
|
||||
user = "postgres" # postgres user
|
||||
password = "" # postgres password
|
||||
|
||||
[file]
|
||||
outputDir = "output_dir/" # when operating in 'file' output mode, this is the directory the files are written to
|
||||
```
|
||||
|
||||
@ -33,7 +33,7 @@ var (
|
||||
)
|
||||
|
||||
var rootCmd = &cobra.Command{
|
||||
Use: "eth-pg-ipfs-state-snapshot",
|
||||
Use: "ipld-eth-state-snapshot",
|
||||
PersistentPreRun: initFuncs,
|
||||
}
|
||||
|
||||
|
||||
@ -16,11 +16,13 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/viper"
|
||||
|
||||
"github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/snapshot"
|
||||
"github.com/vulcanize/ipld-eth-state-snapshot/pkg/snapshot"
|
||||
)
|
||||
|
||||
// stateSnapshotCmd represents the stateSnapshot command
|
||||
@ -29,7 +31,7 @@ var stateSnapshotCmd = &cobra.Command{
|
||||
Short: "Extract the entire Ethereum state from leveldb and publish into PG-IPFS",
|
||||
Long: `Usage
|
||||
|
||||
./eth-pg-ipfs-state-snapshot stateSnapshot --config={path to toml config file}`,
|
||||
./ipld-eth-state-snapshot stateSnapshot --config={path to toml config file}`,
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
subCommand = cmd.CalledAs()
|
||||
logWithCommand = *logrus.WithField("SubCommand", subCommand)
|
||||
@ -38,19 +40,26 @@ var stateSnapshotCmd = &cobra.Command{
|
||||
}
|
||||
|
||||
func stateSnapshot() {
|
||||
config := snapshot.NewConfig()
|
||||
modeStr := viper.GetString("snapshot.mode")
|
||||
mode := snapshot.SnapshotMode(modeStr)
|
||||
config, err := snapshot.NewConfig(mode)
|
||||
if err != nil {
|
||||
logWithCommand.Fatal("unable to initialize config: %v", err)
|
||||
}
|
||||
logWithCommand.Infof("opening levelDB and ancient data at %s and %s",
|
||||
config.Eth.LevelDBPath, config.Eth.AncientDBPath)
|
||||
edb, err := snapshot.NewLevelDB(config.Eth)
|
||||
if err != nil {
|
||||
logWithCommand.Fatal(err)
|
||||
}
|
||||
|
||||
height := viper.GetInt64("snapshot.blockHeight")
|
||||
recoveryFile := viper.GetString("snapshot.recoveryFile")
|
||||
if recoveryFile == "" {
|
||||
recoveryFile = "./snapshot_recovery"
|
||||
recoveryFile = fmt.Sprintf("./%d_snapshot_recovery", height)
|
||||
logWithCommand.Infof("no recovery file set, creating default: %s", recoveryFile)
|
||||
}
|
||||
|
||||
mode := viper.GetString("snapshot.mode")
|
||||
pub, err := snapshot.NewPublisher(snapshot.SnapshotMode(mode), config)
|
||||
pub, err := snapshot.NewPublisher(mode, config)
|
||||
if err != nil {
|
||||
logWithCommand.Fatal(err)
|
||||
}
|
||||
@ -59,7 +68,6 @@ func stateSnapshot() {
|
||||
if err != nil {
|
||||
logWithCommand.Fatal(err)
|
||||
}
|
||||
height := viper.GetInt64("snapshot.blockHeight")
|
||||
workers := viper.GetUint("snapshot.workers")
|
||||
|
||||
if height < 0 {
|
||||
@ -81,10 +89,16 @@ func init() {
|
||||
stateSnapshotCmd.PersistentFlags().String("leveldb-path", "", "path to primary datastore")
|
||||
stateSnapshotCmd.PersistentFlags().String("ancient-path", "", "path to ancient datastore")
|
||||
stateSnapshotCmd.PersistentFlags().String("block-height", "", "blockheight to extract state at")
|
||||
stateSnapshotCmd.PersistentFlags().Int("workers", 0, "number of concurrent workers to use")
|
||||
stateSnapshotCmd.PersistentFlags().Int("workers", 1, "number of concurrent workers to use")
|
||||
stateSnapshotCmd.PersistentFlags().String("recovery-file", "", "file to recover from a previous iteration")
|
||||
stateSnapshotCmd.PersistentFlags().String("snapshot-mode", "postgres", "output mode for snapshot ('file' or 'postgres')")
|
||||
stateSnapshotCmd.PersistentFlags().String("output-dir", "", "directory for writing ouput to while operating in 'file' mode")
|
||||
|
||||
viper.BindPFlag("leveldb.path", stateSnapshotCmd.PersistentFlags().Lookup("leveldb-path"))
|
||||
viper.BindPFlag("leveldb.ancient", stateSnapshotCmd.PersistentFlags().Lookup("ancient-path"))
|
||||
viper.BindPFlag("snapshot.blockHeight", stateSnapshotCmd.PersistentFlags().Lookup("block-height"))
|
||||
viper.BindPFlag("snapshot.workers", stateSnapshotCmd.PersistentFlags().Lookup("workers"))
|
||||
viper.BindPFlag("snapshot.recoveryFile", stateSnapshotCmd.PersistentFlags().Lookup("recovery-file"))
|
||||
viper.BindPFlag("snapshot.mode", stateSnapshotCmd.PersistentFlags().Lookup("snapshot-mode"))
|
||||
viper.BindPFlag("file.outputDir", stateSnapshotCmd.PersistentFlags().Lookup("output-dir"))
|
||||
}
|
||||
|
||||
@ -9,4 +9,10 @@
|
||||
ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient"
|
||||
|
||||
[snapshot]
|
||||
mode = "file"
|
||||
workers = 4
|
||||
blockHeight = -1
|
||||
recoveryFile = "recovery_file"
|
||||
|
||||
[file]
|
||||
outputDir = "output_dir/"
|
||||
|
||||
@ -5,9 +5,8 @@ import (
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
// "github.com/jmoiron/sqlx"
|
||||
|
||||
snapt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/types"
|
||||
snapt "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types"
|
||||
)
|
||||
|
||||
var Block1_Header = types.Header{
|
||||
|
||||
6
go.mod
6
go.mod
@ -1,11 +1,11 @@
|
||||
module github.com/vulcanize/eth-pg-ipfs-state-snapshot
|
||||
module github.com/vulcanize/ipld-eth-state-snapshot
|
||||
|
||||
go 1.15
|
||||
|
||||
require (
|
||||
github.com/btcsuite/btcd v0.22.0-beta // indirect
|
||||
github.com/cespare/xxhash/v2 v2.1.2 // indirect
|
||||
github.com/ethereum/go-ethereum v1.10.15
|
||||
github.com/ethereum/go-ethereum v1.10.16
|
||||
github.com/fsnotify/fsnotify v1.5.1 // indirect
|
||||
github.com/go-kit/kit v0.10.0 // indirect
|
||||
github.com/golang/mock v1.6.0
|
||||
@ -43,4 +43,4 @@ require (
|
||||
lukechampine.com/blake3 v1.1.7 // indirect
|
||||
)
|
||||
|
||||
replace github.com/ethereum/go-ethereum v1.10.15 => github.com/vulcanize/go-ethereum v1.10.15-statediff-3.0.1
|
||||
replace github.com/ethereum/go-ethereum v1.10.16 => github.com/vulcanize/go-ethereum v1.10.16-statediff-3.0.2
|
||||
|
||||
8
go.sum
8
go.sum
@ -146,6 +146,7 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea h1:j4317fAZh7X6GqbFowYdYdI0L9bwxL07jyPZIdepyZ0=
|
||||
github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ=
|
||||
github.com/deckarep/golang-set v1.8.0/go.mod h1:5nI87KwE7wgsBU1F4GKAw2Qod7p5kyS383rP6+o6qqo=
|
||||
github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218=
|
||||
github.com/deepmap/oapi-codegen v1.6.0/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE9iAUlWl9Al3M=
|
||||
github.com/deepmap/oapi-codegen v1.8.2/go.mod h1:YLgSKSDv/bZQB7N4ws6luhozi3cEdRktEqrX88CvjIw=
|
||||
@ -168,6 +169,7 @@ github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4s
|
||||
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||
github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0=
|
||||
github.com/ethereum/go-ethereum v1.10.15/go.mod h1:W3yfrFyL9C1pHcwY5hmRHVDaorTiQxhYBkKyu5mEDHw=
|
||||
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
|
||||
github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c=
|
||||
github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0=
|
||||
@ -290,6 +292,7 @@ github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoA
|
||||
github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
|
||||
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc=
|
||||
github.com/graph-gophers/graphql-go v1.3.0/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc=
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
|
||||
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
|
||||
@ -468,6 +471,7 @@ github.com/jackc/puddle v1.2.1 h1:gI8os0wpRXFd4FiAY2dWiqRK037tjj3t7rKFeO4X5iw=
|
||||
github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
|
||||
github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 h1:6OvNmYgJyexcZ3pYbTI9jWx5tHo1Dee/tWbLMfPe2TA=
|
||||
github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
|
||||
github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
|
||||
github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA=
|
||||
github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY=
|
||||
github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o=
|
||||
@ -497,6 +501,7 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V
|
||||
github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
|
||||
github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0=
|
||||
github.com/karalabe/usb v0.0.0-20211005121534-4c5740d64559/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU=
|
||||
github.com/karalabe/usb v0.0.2/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU=
|
||||
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
|
||||
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
|
||||
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
|
||||
@ -809,6 +814,8 @@ github.com/vulcanize/go-eth-state-node-iterator v1.0.1 h1:lI8+moQ0Nv62NwmiCxgvWk
|
||||
github.com/vulcanize/go-eth-state-node-iterator v1.0.1/go.mod h1:uWhleTvUEZ+cEkNRIAmBpZ14KilTP71OxY5NZDrpNlo=
|
||||
github.com/vulcanize/go-ethereum v1.10.15-statediff-3.0.1 h1:MX7WcTwxpxOoYVMifrs9vsHDmM6gKOFF+2KnP19LddI=
|
||||
github.com/vulcanize/go-ethereum v1.10.15-statediff-3.0.1/go.mod h1:XO9WLkNXfwoJN05BZj0//xgOWHJyUrUPdnudbQfKlUo=
|
||||
github.com/vulcanize/go-ethereum v1.10.16-statediff-3.0.2 h1:H3SLHZdvTyKYbFc1CO2b8A9XF3BcakcXtvThKPbgT8k=
|
||||
github.com/vulcanize/go-ethereum v1.10.16-statediff-3.0.2/go.mod h1:NI+tCVeIQBPrMfJUZvTLjhCieb7CZcmNPbJVlXbncxU=
|
||||
github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM=
|
||||
github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
|
||||
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
|
||||
@ -946,6 +953,7 @@ golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d/go.mod h1:m0MpNAwzfU5UDzcl9v
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
|
||||
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
|
||||
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20211209124913-491a49abca63 h1:iocB37TsdFuN6IBRZ+ry36wrkoV51/tl5vOWqkcPGvY=
|
||||
|
||||
2
main.go
2
main.go
@ -18,7 +18,7 @@ package main
|
||||
import (
|
||||
"github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/vulcanize/eth-pg-ipfs-state-snapshot/cmd"
|
||||
"github.com/vulcanize/ipld-eth-state-snapshot/cmd"
|
||||
)
|
||||
|
||||
func main() {
|
||||
|
||||
@ -16,8 +16,11 @@
|
||||
package snapshot
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
|
||||
ethNode "github.com/ethereum/go-ethereum/statediff/indexer/node"
|
||||
"github.com/spf13/viper"
|
||||
@ -38,6 +41,8 @@ type SnapshotMode string
|
||||
const (
|
||||
PgSnapshot SnapshotMode = "postgres"
|
||||
FileSnapshot SnapshotMode = "file"
|
||||
|
||||
defaultOutputDir = "./snapshot_output"
|
||||
)
|
||||
|
||||
// Config contains params for both databases the service uses
|
||||
@ -64,18 +69,17 @@ type FileConfig struct {
|
||||
OutputDir string
|
||||
}
|
||||
|
||||
func NewConfig() *Config {
|
||||
func NewConfig(mode SnapshotMode) (*Config, error) {
|
||||
ret := &Config{
|
||||
&EthConfig{},
|
||||
&DBConfig{},
|
||||
&FileConfig{},
|
||||
}
|
||||
ret.Init()
|
||||
return ret
|
||||
return ret, ret.Init(mode)
|
||||
}
|
||||
|
||||
// Init Initialises config
|
||||
func (c *Config) Init() {
|
||||
func (c *Config) Init(mode SnapshotMode) error {
|
||||
viper.BindEnv("ethereum.nodeID", ETH_NODE_ID)
|
||||
viper.BindEnv("ethereum.clientName", ETH_CLIENT_NAME)
|
||||
viper.BindEnv("ethereum.genesisBlock", ETH_GENESIS_BLOCK)
|
||||
@ -95,8 +99,15 @@ func (c *Config) Init() {
|
||||
c.Eth.AncientDBPath = viper.GetString("leveldb.ancient")
|
||||
c.Eth.LevelDBPath = viper.GetString("leveldb.path")
|
||||
|
||||
c.DB.Init()
|
||||
c.File.Init()
|
||||
switch mode {
|
||||
case FileSnapshot:
|
||||
c.File.Init()
|
||||
case PgSnapshot:
|
||||
c.DB.Init()
|
||||
default:
|
||||
return fmt.Errorf("no output mode specified")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *DBConfig) Init() {
|
||||
@ -125,7 +136,12 @@ func (c *DBConfig) Init() {
|
||||
c.URI = dbParams.DbConnectionString()
|
||||
}
|
||||
|
||||
func (c *FileConfig) Init() {
|
||||
func (c *FileConfig) Init() error {
|
||||
viper.BindEnv("file.outputDir", "FILE_OUTPUT_DIR")
|
||||
c.OutputDir = viper.GetString("file.outputDir")
|
||||
if c.OutputDir == "" {
|
||||
logrus.Infof("no output directory set, using default: %s", defaultOutputDir)
|
||||
c.OutputDir = defaultOutputDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -34,7 +34,7 @@ import (
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
|
||||
nodeinfo "github.com/ethereum/go-ethereum/statediff/indexer/node"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/shared"
|
||||
snapt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/types"
|
||||
snapt "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types"
|
||||
)
|
||||
|
||||
var _ snapt.Publisher = (*publisher)(nil)
|
||||
@ -123,8 +123,8 @@ func makeFileWriters(dir string, tables []*snapt.Table) (fileWriters, error) {
|
||||
// with the Postgres COPY command.
|
||||
// The output directory will be created if it does not exist.
|
||||
func NewPublisher(path string, node nodeinfo.Info) (*publisher, error) {
|
||||
if err := os.MkdirAll(path, 0755); err != nil {
|
||||
return nil, err
|
||||
if err := os.MkdirAll(path, 0777); err != nil {
|
||||
return nil, fmt.Errorf("unable to make MkdirAll for path: %s err: %s", path, err)
|
||||
}
|
||||
writers, err := makeFileWriters(path, perBlockTables)
|
||||
if err != nil {
|
||||
@ -285,7 +285,7 @@ func (p *publisher) logNodeCounters() {
|
||||
}
|
||||
|
||||
func (p *publisher) printNodeCounters(msg string) {
|
||||
log.WithFields(log.Fields{
|
||||
logrus.WithFields(logrus.Fields{
|
||||
"runtime": time.Now().Sub(p.startTime).String(),
|
||||
"state nodes": atomic.LoadUint64(&p.stateNodeCounter),
|
||||
"storage nodes": atomic.LoadUint64(&p.storageNodeCounter),
|
||||
|
||||
@ -12,9 +12,9 @@ import (
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
|
||||
"github.com/jackc/pgx/v4"
|
||||
|
||||
fixt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/fixture"
|
||||
snapt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/types"
|
||||
"github.com/vulcanize/eth-pg-ipfs-state-snapshot/test"
|
||||
fixt "github.com/vulcanize/ipld-eth-state-snapshot/fixture"
|
||||
snapt "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types"
|
||||
"github.com/vulcanize/ipld-eth-state-snapshot/test"
|
||||
)
|
||||
|
||||
var (
|
||||
|
||||
@ -1,7 +1,10 @@
|
||||
package mock
|
||||
|
||||
import "fmt"
|
||||
import "github.com/golang/mock/gomock"
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/golang/mock/gomock"
|
||||
)
|
||||
|
||||
type anyOfMatcher struct {
|
||||
values []interface{}
|
||||
|
||||
@ -33,7 +33,7 @@ import (
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/shared"
|
||||
snapt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/types"
|
||||
snapt "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types"
|
||||
)
|
||||
|
||||
var _ snapt.Publisher = (*publisher)(nil)
|
||||
|
||||
@ -9,9 +9,9 @@ import (
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
|
||||
"github.com/jackc/pgx/v4"
|
||||
|
||||
fixt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/fixture"
|
||||
snapt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/types"
|
||||
"github.com/vulcanize/eth-pg-ipfs-state-snapshot/test"
|
||||
fixt "github.com/vulcanize/ipld-eth-state-snapshot/fixture"
|
||||
snapt "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types"
|
||||
"github.com/vulcanize/ipld-eth-state-snapshot/test"
|
||||
)
|
||||
|
||||
var (
|
||||
|
||||
@ -31,8 +31,8 @@ import (
|
||||
"github.com/ethereum/go-ethereum/trie"
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
. "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/types"
|
||||
iter "github.com/vulcanize/go-eth-state-node-iterator"
|
||||
. "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types"
|
||||
)
|
||||
|
||||
var (
|
||||
@ -55,9 +55,15 @@ type Service struct {
|
||||
}
|
||||
|
||||
func NewLevelDB(con *EthConfig) (ethdb.Database, error) {
|
||||
return rawdb.NewLevelDBDatabaseWithFreezer(
|
||||
con.LevelDBPath, 1024, 256, con.AncientDBPath, "eth-pg-ipfs-state-snapshot", false,
|
||||
println(con.LevelDBPath)
|
||||
println(con.AncientDBPath)
|
||||
edb, err := rawdb.NewLevelDBDatabaseWithFreezer(
|
||||
con.LevelDBPath, 1024, 256, con.AncientDBPath, "ipld-eth-state-snapshot", true,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to create NewLevelDBDatabaseWithFreezer: %s", err)
|
||||
}
|
||||
return edb, nil
|
||||
}
|
||||
|
||||
// NewSnapshotService creates Service.
|
||||
|
||||
@ -8,10 +8,10 @@ import (
|
||||
|
||||
"github.com/golang/mock/gomock"
|
||||
|
||||
fixt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/fixture"
|
||||
mock "github.com/vulcanize/eth-pg-ipfs-state-snapshot/mocks/snapshot"
|
||||
snapt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/types"
|
||||
"github.com/vulcanize/eth-pg-ipfs-state-snapshot/test"
|
||||
fixt "github.com/vulcanize/ipld-eth-state-snapshot/fixture"
|
||||
mock "github.com/vulcanize/ipld-eth-state-snapshot/mocks/snapshot"
|
||||
snapt "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types"
|
||||
"github.com/vulcanize/ipld-eth-state-snapshot/test"
|
||||
)
|
||||
|
||||
func testConfig(leveldbpath, ancientdbpath string) *Config {
|
||||
|
||||
@ -6,9 +6,9 @@ import (
|
||||
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
|
||||
|
||||
file "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/snapshot/file"
|
||||
pg "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/snapshot/pg"
|
||||
snapt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/types"
|
||||
file "github.com/vulcanize/ipld-eth-state-snapshot/pkg/snapshot/file"
|
||||
pg "github.com/vulcanize/ipld-eth-state-snapshot/pkg/snapshot/pg"
|
||||
snapt "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types"
|
||||
)
|
||||
|
||||
func NewPublisher(mode SnapshotMode, config *Config) (snapt.Publisher, error) {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user