From 060eb5cf67761e26aab359143d551fc3db79b180 Mon Sep 17 00:00:00 2001 From: i-norden Date: Wed, 30 Mar 2022 18:57:30 -0500 Subject: [PATCH] add some logs and guards, update module name, update readme --- .gitignore | 2 +- README.md | 33 ++++++++++++++++++----------- cmd/root.go | 2 +- cmd/stateSnapshot.go | 32 ++++++++++++++++++++-------- environments/example.toml | 8 ++++++- fixture/service.go | 2 +- go.mod | 6 +++--- go.sum | 8 +++++++ main.go | 2 +- pkg/snapshot/config.go | 30 ++++++++++++++++++++------ pkg/snapshot/file/publisher.go | 6 +++--- pkg/snapshot/file/publisher_test.go | 6 +++--- pkg/snapshot/pg/publisher.go | 2 +- pkg/snapshot/pg/publisher_test.go | 6 +++--- pkg/snapshot/service.go | 12 ++++++++--- pkg/snapshot/service_test.go | 8 +++---- pkg/snapshot/util.go | 6 +++--- 17 files changed, 115 insertions(+), 56 deletions(-) diff --git a/.gitignore b/.gitignore index 9d3f7cf..5d7a86c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ .idea/ -eth-pg-ipfs-state-snapshot +ipld-eth-state-snapshot diff --git a/README.md b/README.md index bc087a6..01fda86 100644 --- a/README.md +++ b/README.md @@ -1,26 +1,35 @@ -# eth-pg-ipfs-state-snapshot +# ipld-eth-state-snapshot > Tool for extracting the entire Ethereum state at a particular block height from leveldb into Postgres-backed IPFS -[![Go Report Card](https://goreportcard.com/badge/github.com/vulcanize/eth-pg-ipfs-state-snapshot)](https://goreportcard.com/report/github.com/vulcanize/eth-pg-ipfs-state-snapshot) +[![Go Report Card](https://goreportcard.com/badge/github.com/vulcanize/ipld-eth-state-snapshot)](https://goreportcard.com/report/github.com/vulcanize/ipld-eth-state-snapshot) ## Usage -./eth-pg-ipfs-state-snapshot stateSnapshot --config={path to toml config file} +./ipld-eth-state-snapshot stateSnapshot --config={path to toml config file} + +### Config Config format: ```toml -[database] - name = "vulcanize_public" - hostname = "localhost" - port = 5432 - user = "postgres" +[snapshot] + mode = "file" # indicates output mode ("postgres" or "file") + workers = 4 # degree of concurrency, the state trie is subdivided into sectiosn that are traversed and processed concurrently + blockHeight = -1 # blockheight to perform the snapshot at (-1 indicates to use the latest blockheight found in leveldb) + recoveryFile = "recovery_file" # specifies a file to output recovery information on error or premature closure [leveldb] - path = "/Users/user/Library/Ethereum/geth/chaindata" - ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient" + path = "/Users/user/Library/Ethereum/geth/chaindata" # path to geth leveldb + ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient" # path to geth ancient database -[snapshot] - blockHeight = 0 +[database] + name = "vulcanize_public" # postgres database name + hostname = "localhost" # postgres host + port = 5432 # postgres port + user = "postgres" # postgres user + password = "" # postgres password + +[file] + outputDir = "output_dir/" # when operating in 'file' output mode, this is the directory the files are written to ``` diff --git a/cmd/root.go b/cmd/root.go index 3e664d3..259006e 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -33,7 +33,7 @@ var ( ) var rootCmd = &cobra.Command{ - Use: "eth-pg-ipfs-state-snapshot", + Use: "ipld-eth-state-snapshot", PersistentPreRun: initFuncs, } diff --git a/cmd/stateSnapshot.go b/cmd/stateSnapshot.go index bacfe9f..4c79fb2 100644 --- a/cmd/stateSnapshot.go +++ b/cmd/stateSnapshot.go @@ -16,11 +16,13 @@ package cmd import ( + "fmt" + "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/viper" - "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/snapshot" + "github.com/vulcanize/ipld-eth-state-snapshot/pkg/snapshot" ) // stateSnapshotCmd represents the stateSnapshot command @@ -29,7 +31,7 @@ var stateSnapshotCmd = &cobra.Command{ Short: "Extract the entire Ethereum state from leveldb and publish into PG-IPFS", Long: `Usage -./eth-pg-ipfs-state-snapshot stateSnapshot --config={path to toml config file}`, +./ipld-eth-state-snapshot stateSnapshot --config={path to toml config file}`, Run: func(cmd *cobra.Command, args []string) { subCommand = cmd.CalledAs() logWithCommand = *logrus.WithField("SubCommand", subCommand) @@ -38,19 +40,26 @@ var stateSnapshotCmd = &cobra.Command{ } func stateSnapshot() { - config := snapshot.NewConfig() + modeStr := viper.GetString("snapshot.mode") + mode := snapshot.SnapshotMode(modeStr) + config, err := snapshot.NewConfig(mode) + if err != nil { + logWithCommand.Fatal("unable to initialize config: %v", err) + } + logWithCommand.Infof("opening levelDB and ancient data at %s and %s", + config.Eth.LevelDBPath, config.Eth.AncientDBPath) edb, err := snapshot.NewLevelDB(config.Eth) if err != nil { logWithCommand.Fatal(err) } - + height := viper.GetInt64("snapshot.blockHeight") recoveryFile := viper.GetString("snapshot.recoveryFile") if recoveryFile == "" { - recoveryFile = "./snapshot_recovery" + recoveryFile = fmt.Sprintf("./%d_snapshot_recovery", height) + logWithCommand.Infof("no recovery file set, creating default: %s", recoveryFile) } - mode := viper.GetString("snapshot.mode") - pub, err := snapshot.NewPublisher(snapshot.SnapshotMode(mode), config) + pub, err := snapshot.NewPublisher(mode, config) if err != nil { logWithCommand.Fatal(err) } @@ -59,7 +68,6 @@ func stateSnapshot() { if err != nil { logWithCommand.Fatal(err) } - height := viper.GetInt64("snapshot.blockHeight") workers := viper.GetUint("snapshot.workers") if height < 0 { @@ -81,10 +89,16 @@ func init() { stateSnapshotCmd.PersistentFlags().String("leveldb-path", "", "path to primary datastore") stateSnapshotCmd.PersistentFlags().String("ancient-path", "", "path to ancient datastore") stateSnapshotCmd.PersistentFlags().String("block-height", "", "blockheight to extract state at") - stateSnapshotCmd.PersistentFlags().Int("workers", 0, "number of concurrent workers to use") + stateSnapshotCmd.PersistentFlags().Int("workers", 1, "number of concurrent workers to use") + stateSnapshotCmd.PersistentFlags().String("recovery-file", "", "file to recover from a previous iteration") + stateSnapshotCmd.PersistentFlags().String("snapshot-mode", "postgres", "output mode for snapshot ('file' or 'postgres')") + stateSnapshotCmd.PersistentFlags().String("output-dir", "", "directory for writing ouput to while operating in 'file' mode") viper.BindPFlag("leveldb.path", stateSnapshotCmd.PersistentFlags().Lookup("leveldb-path")) viper.BindPFlag("leveldb.ancient", stateSnapshotCmd.PersistentFlags().Lookup("ancient-path")) viper.BindPFlag("snapshot.blockHeight", stateSnapshotCmd.PersistentFlags().Lookup("block-height")) viper.BindPFlag("snapshot.workers", stateSnapshotCmd.PersistentFlags().Lookup("workers")) + viper.BindPFlag("snapshot.recoveryFile", stateSnapshotCmd.PersistentFlags().Lookup("recovery-file")) + viper.BindPFlag("snapshot.mode", stateSnapshotCmd.PersistentFlags().Lookup("snapshot-mode")) + viper.BindPFlag("file.outputDir", stateSnapshotCmd.PersistentFlags().Lookup("output-dir")) } diff --git a/environments/example.toml b/environments/example.toml index dde7825..6741edf 100644 --- a/environments/example.toml +++ b/environments/example.toml @@ -9,4 +9,10 @@ ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient" [snapshot] - blockHeight = -1 \ No newline at end of file + mode = "file" + workers = 4 + blockHeight = -1 + recoveryFile = "recovery_file" + +[file] + outputDir = "output_dir/" diff --git a/fixture/service.go b/fixture/service.go index 2b7b05e..3665ceb 100644 --- a/fixture/service.go +++ b/fixture/service.go @@ -6,7 +6,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" - snapt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/types" + snapt "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types" ) var Block1_Header = types.Header{ diff --git a/go.mod b/go.mod index db93f35..0a16f70 100644 --- a/go.mod +++ b/go.mod @@ -1,11 +1,11 @@ -module github.com/vulcanize/eth-pg-ipfs-state-snapshot +module github.com/vulcanize/ipld-eth-state-snapshot go 1.15 require ( github.com/btcsuite/btcd v0.22.0-beta // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect - github.com/ethereum/go-ethereum v1.10.15 + github.com/ethereum/go-ethereum v1.10.16 github.com/fsnotify/fsnotify v1.5.1 // indirect github.com/go-kit/kit v0.10.0 // indirect github.com/golang/mock v1.6.0 @@ -43,4 +43,4 @@ require ( lukechampine.com/blake3 v1.1.7 // indirect ) -replace github.com/ethereum/go-ethereum v1.10.15 => github.com/vulcanize/go-ethereum v1.10.15-statediff-3.0.1 +replace github.com/ethereum/go-ethereum v1.10.16 => github.com/vulcanize/go-ethereum v1.10.16-statediff-3.0.2 diff --git a/go.sum b/go.sum index 7f9bae0..5530067 100644 --- a/go.sum +++ b/go.sum @@ -146,6 +146,7 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea h1:j4317fAZh7X6GqbFowYdYdI0L9bwxL07jyPZIdepyZ0= github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= +github.com/deckarep/golang-set v1.8.0/go.mod h1:5nI87KwE7wgsBU1F4GKAw2Qod7p5kyS383rP6+o6qqo= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= github.com/deepmap/oapi-codegen v1.6.0/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE9iAUlWl9Al3M= github.com/deepmap/oapi-codegen v1.8.2/go.mod h1:YLgSKSDv/bZQB7N4ws6luhozi3cEdRktEqrX88CvjIw= @@ -168,6 +169,7 @@ github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4s github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= +github.com/ethereum/go-ethereum v1.10.15/go.mod h1:W3yfrFyL9C1pHcwY5hmRHVDaorTiQxhYBkKyu5mEDHw= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c= github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= @@ -290,6 +292,7 @@ github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoA github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= +github.com/graph-gophers/graphql-go v1.3.0/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= @@ -468,6 +471,7 @@ github.com/jackc/puddle v1.2.1 h1:gI8os0wpRXFd4FiAY2dWiqRK037tjj3t7rKFeO4X5iw= github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 h1:6OvNmYgJyexcZ3pYbTI9jWx5tHo1Dee/tWbLMfPe2TA= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= +github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY= github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o= @@ -497,6 +501,7 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0= github.com/karalabe/usb v0.0.0-20211005121534-4c5740d64559/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= +github.com/karalabe/usb v0.0.2/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -809,6 +814,8 @@ github.com/vulcanize/go-eth-state-node-iterator v1.0.1 h1:lI8+moQ0Nv62NwmiCxgvWk github.com/vulcanize/go-eth-state-node-iterator v1.0.1/go.mod h1:uWhleTvUEZ+cEkNRIAmBpZ14KilTP71OxY5NZDrpNlo= github.com/vulcanize/go-ethereum v1.10.15-statediff-3.0.1 h1:MX7WcTwxpxOoYVMifrs9vsHDmM6gKOFF+2KnP19LddI= github.com/vulcanize/go-ethereum v1.10.15-statediff-3.0.1/go.mod h1:XO9WLkNXfwoJN05BZj0//xgOWHJyUrUPdnudbQfKlUo= +github.com/vulcanize/go-ethereum v1.10.16-statediff-3.0.2 h1:H3SLHZdvTyKYbFc1CO2b8A9XF3BcakcXtvThKPbgT8k= +github.com/vulcanize/go-ethereum v1.10.16-statediff-3.0.2/go.mod h1:NI+tCVeIQBPrMfJUZvTLjhCieb7CZcmNPbJVlXbncxU= github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= @@ -946,6 +953,7 @@ golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211209124913-491a49abca63 h1:iocB37TsdFuN6IBRZ+ry36wrkoV51/tl5vOWqkcPGvY= diff --git a/main.go b/main.go index e866716..27277df 100644 --- a/main.go +++ b/main.go @@ -18,7 +18,7 @@ package main import ( "github.com/sirupsen/logrus" - "github.com/vulcanize/eth-pg-ipfs-state-snapshot/cmd" + "github.com/vulcanize/ipld-eth-state-snapshot/cmd" ) func main() { diff --git a/pkg/snapshot/config.go b/pkg/snapshot/config.go index e097c94..ab48889 100644 --- a/pkg/snapshot/config.go +++ b/pkg/snapshot/config.go @@ -16,8 +16,11 @@ package snapshot import ( + "fmt" "time" + "github.com/sirupsen/logrus" + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" ethNode "github.com/ethereum/go-ethereum/statediff/indexer/node" "github.com/spf13/viper" @@ -38,6 +41,8 @@ type SnapshotMode string const ( PgSnapshot SnapshotMode = "postgres" FileSnapshot SnapshotMode = "file" + + defaultOutputDir = "./snapshot_output" ) // Config contains params for both databases the service uses @@ -64,18 +69,17 @@ type FileConfig struct { OutputDir string } -func NewConfig() *Config { +func NewConfig(mode SnapshotMode) (*Config, error) { ret := &Config{ &EthConfig{}, &DBConfig{}, &FileConfig{}, } - ret.Init() - return ret + return ret, ret.Init(mode) } // Init Initialises config -func (c *Config) Init() { +func (c *Config) Init(mode SnapshotMode) error { viper.BindEnv("ethereum.nodeID", ETH_NODE_ID) viper.BindEnv("ethereum.clientName", ETH_CLIENT_NAME) viper.BindEnv("ethereum.genesisBlock", ETH_GENESIS_BLOCK) @@ -95,8 +99,15 @@ func (c *Config) Init() { c.Eth.AncientDBPath = viper.GetString("leveldb.ancient") c.Eth.LevelDBPath = viper.GetString("leveldb.path") - c.DB.Init() - c.File.Init() + switch mode { + case FileSnapshot: + c.File.Init() + case PgSnapshot: + c.DB.Init() + default: + return fmt.Errorf("no output mode specified") + } + return nil } func (c *DBConfig) Init() { @@ -125,7 +136,12 @@ func (c *DBConfig) Init() { c.URI = dbParams.DbConnectionString() } -func (c *FileConfig) Init() { +func (c *FileConfig) Init() error { viper.BindEnv("file.outputDir", "FILE_OUTPUT_DIR") c.OutputDir = viper.GetString("file.outputDir") + if c.OutputDir == "" { + logrus.Infof("no output directory set, using default: %s", defaultOutputDir) + c.OutputDir = defaultOutputDir + } + return nil } diff --git a/pkg/snapshot/file/publisher.go b/pkg/snapshot/file/publisher.go index 8abc8be..f20dfb8 100644 --- a/pkg/snapshot/file/publisher.go +++ b/pkg/snapshot/file/publisher.go @@ -34,7 +34,7 @@ import ( "github.com/ethereum/go-ethereum/statediff/indexer/ipld" nodeinfo "github.com/ethereum/go-ethereum/statediff/indexer/node" "github.com/ethereum/go-ethereum/statediff/indexer/shared" - snapt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/types" + snapt "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types" ) var _ snapt.Publisher = (*publisher)(nil) @@ -123,8 +123,8 @@ func makeFileWriters(dir string, tables []*snapt.Table) (fileWriters, error) { // with the Postgres COPY command. // The output directory will be created if it does not exist. func NewPublisher(path string, node nodeinfo.Info) (*publisher, error) { - if err := os.MkdirAll(path, 0755); err != nil { - return nil, err + if err := os.MkdirAll(path, 0777); err != nil { + return nil, fmt.Errorf("unable to make MkdirAll for path: %s err: %s", path, err) } writers, err := makeFileWriters(path, perBlockTables) if err != nil { diff --git a/pkg/snapshot/file/publisher_test.go b/pkg/snapshot/file/publisher_test.go index 382ec0c..f2a591c 100644 --- a/pkg/snapshot/file/publisher_test.go +++ b/pkg/snapshot/file/publisher_test.go @@ -12,9 +12,9 @@ import ( "github.com/ethereum/go-ethereum/statediff/indexer/ipld" "github.com/jackc/pgx/v4" - fixt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/fixture" - snapt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/types" - "github.com/vulcanize/eth-pg-ipfs-state-snapshot/test" + fixt "github.com/vulcanize/ipld-eth-state-snapshot/fixture" + snapt "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types" + "github.com/vulcanize/ipld-eth-state-snapshot/test" ) var ( diff --git a/pkg/snapshot/pg/publisher.go b/pkg/snapshot/pg/publisher.go index 4f014a3..3b7f4a2 100644 --- a/pkg/snapshot/pg/publisher.go +++ b/pkg/snapshot/pg/publisher.go @@ -33,7 +33,7 @@ import ( "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" "github.com/ethereum/go-ethereum/statediff/indexer/ipld" "github.com/ethereum/go-ethereum/statediff/indexer/shared" - snapt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/types" + snapt "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types" ) var _ snapt.Publisher = (*publisher)(nil) diff --git a/pkg/snapshot/pg/publisher_test.go b/pkg/snapshot/pg/publisher_test.go index a6e9842..e4ba321 100644 --- a/pkg/snapshot/pg/publisher_test.go +++ b/pkg/snapshot/pg/publisher_test.go @@ -9,9 +9,9 @@ import ( "github.com/ethereum/go-ethereum/statediff/indexer/ipld" "github.com/jackc/pgx/v4" - fixt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/fixture" - snapt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/types" - "github.com/vulcanize/eth-pg-ipfs-state-snapshot/test" + fixt "github.com/vulcanize/ipld-eth-state-snapshot/fixture" + snapt "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types" + "github.com/vulcanize/ipld-eth-state-snapshot/test" ) var ( diff --git a/pkg/snapshot/service.go b/pkg/snapshot/service.go index 8978dd0..bdd7bdc 100644 --- a/pkg/snapshot/service.go +++ b/pkg/snapshot/service.go @@ -31,8 +31,8 @@ import ( "github.com/ethereum/go-ethereum/trie" log "github.com/sirupsen/logrus" - . "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/types" iter "github.com/vulcanize/go-eth-state-node-iterator" + . "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types" ) var ( @@ -55,9 +55,15 @@ type Service struct { } func NewLevelDB(con *EthConfig) (ethdb.Database, error) { - return rawdb.NewLevelDBDatabaseWithFreezer( - con.LevelDBPath, 1024, 256, con.AncientDBPath, "eth-pg-ipfs-state-snapshot", false, + println(con.LevelDBPath) + println(con.AncientDBPath) + edb, err := rawdb.NewLevelDBDatabaseWithFreezer( + con.LevelDBPath, 1024, 256, con.AncientDBPath, "ipld-eth-state-snapshot", true, ) + if err != nil { + return nil, fmt.Errorf("unable to create NewLevelDBDatabaseWithFreezer: %s", err) + } + return edb, nil } // NewSnapshotService creates Service. diff --git a/pkg/snapshot/service_test.go b/pkg/snapshot/service_test.go index 3dacc5f..f1263fe 100644 --- a/pkg/snapshot/service_test.go +++ b/pkg/snapshot/service_test.go @@ -8,10 +8,10 @@ import ( "github.com/golang/mock/gomock" - fixt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/fixture" - mock "github.com/vulcanize/eth-pg-ipfs-state-snapshot/mocks/snapshot" - snapt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/types" - "github.com/vulcanize/eth-pg-ipfs-state-snapshot/test" + fixt "github.com/vulcanize/ipld-eth-state-snapshot/fixture" + mock "github.com/vulcanize/ipld-eth-state-snapshot/mocks/snapshot" + snapt "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types" + "github.com/vulcanize/ipld-eth-state-snapshot/test" ) func testConfig(leveldbpath, ancientdbpath string) *Config { diff --git a/pkg/snapshot/util.go b/pkg/snapshot/util.go index a05d0d9..c5673d0 100644 --- a/pkg/snapshot/util.go +++ b/pkg/snapshot/util.go @@ -6,9 +6,9 @@ import ( "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" - file "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/snapshot/file" - pg "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/snapshot/pg" - snapt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/types" + file "github.com/vulcanize/ipld-eth-state-snapshot/pkg/snapshot/file" + pg "github.com/vulcanize/ipld-eth-state-snapshot/pkg/snapshot/pg" + snapt "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types" ) func NewPublisher(mode SnapshotMode, config *Config) (snapt.Publisher, error) {