Merge pull request #69 from cerc-io/ian/v4_dev

Update to use v4 vdb geth v1.11.5

.github/workflows/on-pr.yaml

@@ -15,7 +15,7 @@ jobs:
 
       - uses: actions/setup-go@v3
         with:
-          go-version: ">=1.18.0"
+          go-version: ">=1.19.0"
           check-latest: true
 
       - name: Checkout code

fixture/chain2data/CURRENT

@@ -0,0 +1 @@
+MANIFEST-000005

fixture/chain2data/LOG

@@ -0,0 +1,15 @@
+=============== Mar 31, 2023 (CDT) ===============
+12:22:01.281499 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed
+12:22:01.281586 table@recovery F·2
+12:22:01.281592 table@recovery recovering @2
+12:22:01.281901 table@recovery recovered @2 Gk·13 Ck·0 Cb·0 S·918 Q·13
+12:22:01.281909 table@recovery recovering @4
+12:22:01.282725 table@recovery recovered @4 Gk·416 Ck·0 Cb·0 S·35872 Q·430
+12:22:01.282737 table@recovery recovered F·2 N·429 Gk·429 Ck·0 Q·430
+12:22:01.298688 version@stat F·[2] S·35KiB[35KiB] Sc·[0.50]
+12:22:01.308338 db@open opening
+12:22:01.308738 version@stat F·[2] S·35KiB[35KiB] Sc·[0.50]
+12:22:01.314746 db@janitor F·4 G·0
+12:22:01.314778 db@open done T·6.427167ms
+12:22:01.317510 db@close closing
+12:22:01.317545 db@close done T·34.833µs

fixture/chaindata/CURRENT

@@ -0,0 +1 @@
+MANIFEST-000003

fixture/chaindata/LOG

@@ -0,0 +1,13 @@
+=============== Mar 31, 2023 (CDT) ===============
+12:22:01.090151 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed
+12:22:01.090565 table@recovery F·1
+12:22:01.090576 table@recovery recovering @2
+12:22:01.106640 table@recovery recovered @2 Gk·146827 Ck·0 Cb·0 S·5765300 Q·146827
+12:22:01.106661 table@recovery recovered F·1 N·146827 Gk·146827 Ck·0 Q·146827
+12:22:01.133566 version@stat F·[1] S·5MiB[5MiB] Sc·[0.25]
+12:22:01.145740 db@open opening
+12:22:01.146636 version@stat F·[1] S·5MiB[5MiB] Sc·[0.25]
+12:22:01.151049 db@janitor F·3 G·0
+12:22:01.151090 db@open done T·5.315792ms
+12:22:01.161329 db@close closing
+12:22:01.161389 db@close done T·58.667µs

fixture/chaindata/ancient/bodies.meta

@@ -0,0 +1 @@
+<EFBFBD>
<01>

fixture/chaindata/ancient/diffs.meta

@@ -0,0 +1 @@
+<EFBFBD>
<01>

fixture/chaindata/ancient/hashes.meta

@@ -0,0 +1 @@
+<EFBFBD>
<01>

fixture/chaindata/ancient/headers.meta

@@ -0,0 +1 @@
+<EFBFBD>
<01>

fixture/chaindata/ancient/receipts.meta

@@ -0,0 +1 @@
+<EFBFBD>
<01>

go.mod

@@ -1,36 +1,40 @@
 module github.com/cerc-io/ipld-eth-state-snapshot
 
-go 1.18
+go 1.19
 
 require (
-	github.com/cerc-io/go-eth-state-node-iterator v1.1.9
-	github.com/ethereum/go-ethereum v1.10.26
+	github.com/ethereum/go-ethereum v1.11.5
 	github.com/golang/mock v1.6.0
 	github.com/ipfs/go-cid v0.2.0
 	github.com/ipfs/go-ipfs-blockstore v1.2.0
 	github.com/ipfs/go-ipfs-ds-help v1.1.0
 	github.com/jmoiron/sqlx v1.3.5
 	github.com/multiformats/go-multihash v0.1.0
-	github.com/prometheus/client_golang v1.3.0
+	github.com/prometheus/client_golang v1.14.0
 	github.com/sirupsen/logrus v1.9.0
 	github.com/spf13/cobra v1.5.0
 	github.com/spf13/viper v1.12.0
-	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
+	golang.org/x/sync v0.1.0
 )
 
 require (
+	github.com/DataDog/zstd v1.5.2 // indirect
 	github.com/VictoriaMetrics/fastcache v1.6.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
-	github.com/cespare/xxhash/v2 v2.1.2 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/deckarep/golang-set v1.8.0 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cockroachdb/errors v1.9.1 // indirect
+	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
+	github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811 // indirect
+	github.com/cockroachdb/redact v1.1.3 // indirect
+	github.com/deckarep/golang-set/v2 v2.1.0 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
-	github.com/fsnotify/fsnotify v1.5.4 // indirect
-	github.com/georgysavva/scany v0.2.9 // indirect
-	github.com/go-kit/kit v0.10.0 // indirect
+	github.com/fsnotify/fsnotify v1.6.0 // indirect
+	github.com/georgysavva/scany v1.2.1 // indirect
+	github.com/getsentry/sentry-go v0.17.0 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
-	github.com/go-stack/stack v1.8.0 // indirect
+	github.com/go-stack/stack v1.8.1 // indirect
+	github.com/gofrs/flock v0.8.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
@@ -39,6 +43,7 @@ require (
 	github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/holiman/bloomfilter/v2 v2.0.3 // indirect
+	github.com/holiman/uint256 v1.2.0 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/ipfs/bbloom v0.0.4 // indirect
 	github.com/ipfs/go-block-format v0.0.3 // indirect
@@ -58,12 +63,15 @@ require (
 	github.com/jackc/pgx/v4 v4.15.0 // indirect
 	github.com/jackc/puddle v1.2.1 // indirect
 	github.com/jbenet/goprocess v0.1.4 // indirect
+	github.com/klauspost/compress v1.15.15 // indirect
 	github.com/klauspost/cpuid/v2 v2.0.9 // indirect
-	github.com/lib/pq v1.10.6 // indirect
+	github.com/kr/pretty v0.3.1 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/lib/pq v1.10.7 // indirect
 	github.com/magiconair/properties v1.8.6 // indirect
-	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/mattn/go-isatty v0.0.16 // indirect
 	github.com/mattn/go-runewidth v0.0.9 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect
 	github.com/minio/sha256-simd v1.0.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
@@ -75,39 +83,37 @@ require (
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
 	github.com/pelletier/go-toml v1.9.5 // indirect
-	github.com/pelletier/go-toml/v2 v2.0.1 // indirect
+	github.com/pelletier/go-toml/v2 v2.0.5 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_model v0.1.0 // indirect
-	github.com/prometheus/common v0.7.0 // indirect
-	github.com/prometheus/procfs v0.0.8 // indirect
-	github.com/prometheus/tsdb v0.10.0 // indirect
+	github.com/prometheus/client_model v0.3.0 // indirect
+	github.com/prometheus/common v0.37.1 // indirect
+	github.com/prometheus/procfs v0.9.0 // indirect
+	github.com/rogpeppe/go-internal v1.9.0 // indirect
 	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/spf13/afero v1.8.2 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/stretchr/objx v0.2.0 // indirect
-	github.com/stretchr/testify v1.7.2 // indirect
 	github.com/subosito/gotenv v1.3.0 // indirect
 	github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a // indirect
-	github.com/tklauser/go-sysconf v0.3.5 // indirect
-	github.com/tklauser/numcpus v0.2.2 // indirect
+	github.com/tklauser/go-sysconf v0.3.11 // indirect
+	github.com/tklauser/numcpus v0.6.0 // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/goleak v1.1.11 // indirect
 	go.uber.org/multierr v1.7.0 // indirect
 	go.uber.org/zap v1.19.1 // indirect
-	golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 // indirect
-	golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect
-	golang.org/x/text v0.3.7 // indirect
-	google.golang.org/protobuf v1.28.0 // indirect
-	gopkg.in/ini.v1 v1.66.4 // indirect
+	golang.org/x/crypto v0.6.0 // indirect
+	golang.org/x/exp v0.0.0-20230206171751-46f607a40771 // indirect
+	golang.org/x/sys v0.5.0 // indirect
+	golang.org/x/text v0.7.0 // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	lukechampine.com/blake3 v1.1.7 // indirect
 )
 
-replace github.com/ethereum/go-ethereum v1.10.26 => github.com/cerc-io/go-ethereum v1.10.26-statediff-4.2.2-alpha
+replace github.com/ethereum/go-ethereum v1.11.5 => github.com/cerc-io/go-ethereum v1.11.5-statediff-4.3.9-alpha

pkg/prom/db_stats_collector.go

@@ -19,12 +19,12 @@ package prom
 import (
 	"github.com/prometheus/client_golang/prometheus"
 
-	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
+	mets "github.com/ethereum/go-ethereum/statediff/indexer/database/metrics"
 )
 
 // DBStatsGetter is an interface that gets sql.DBStats.
 type DBStatsGetter interface {
-	Stats() sql.Stats
+	Stats() mets.DbStats
 }
 
 // DBStatsCollector implements the prometheus.Collector interface.

pkg/snapshot/file/publisher_test.go

@@ -9,9 +9,9 @@ import (
 	"path/filepath"
 	"testing"
 
-	"github.com/ethereum/go-ethereum/statediff/indexer/test_helpers"
 	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
 	"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
+	"github.com/ethereum/go-ethereum/statediff/indexer/test_helpers"
 
 	fixt "github.com/cerc-io/ipld-eth-state-snapshot/fixture"
 	snapt "github.com/cerc-io/ipld-eth-state-snapshot/pkg/types"
@@ -91,7 +91,7 @@ func TestPgCopy(t *testing.T) {
 	ctx := context.Background()
 	driver, err := postgres.NewSQLXDriver(ctx, pgConfig, nodeInfo)
 	test.NoError(t, err)
-	db := postgres.NewPostgresDB(driver)
+	db := postgres.NewPostgresDB(driver, false)
 
 	test_helpers.TearDownDB(t, db)
 

pkg/snapshot/in_place_snapshot_test.go

@@ -80,7 +80,7 @@ func TestCreateInPlaceSnapshot(t *testing.T) {
 	ctx := context.Background()
 	driver, err := postgres.NewSQLXDriver(ctx, pgConfig, nodeInfo)
 	test.NoError(t, err)
-	db := postgres.NewPostgresDB(driver)
+	db := postgres.NewPostgresDB(driver, false)
 
 	test_helpers.TearDownDB(t, db)
 

pkg/snapshot/pg/publisher_test.go

@@ -4,9 +4,9 @@ import (
 	"context"
 	"testing"
 
-	"github.com/ethereum/go-ethereum/statediff/indexer/test_helpers"
 	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
 	"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
+	"github.com/ethereum/go-ethereum/statediff/indexer/test_helpers"
 
 	fixt "github.com/cerc-io/ipld-eth-state-snapshot/fixture"
 	snapt "github.com/cerc-io/ipld-eth-state-snapshot/pkg/types"
@@ -46,7 +46,7 @@ func TestBasic(t *testing.T) {
 	ctx := context.Background()
 	driver, err := postgres.NewSQLXDriver(ctx, pgConfig, nodeInfo)
 	test.NoError(t, err)
-	db := postgres.NewPostgresDB(driver)
+	db := postgres.NewPostgresDB(driver, false)
 
 	test_helpers.TearDownDB(t, db)
 

pkg/snapshot/service.go

@@ -30,10 +30,10 @@ import (
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/trie"
+	iter "github.com/ethereum/go-ethereum/trie/concurrent_iterator"
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/sync/errgroup"
 
-	iter "github.com/cerc-io/go-eth-state-node-iterator"
 	"github.com/cerc-io/ipld-eth-state-snapshot/pkg/prom"
 	. "github.com/cerc-io/ipld-eth-state-snapshot/pkg/types"
 )
@@ -59,9 +59,8 @@ type Service struct {
 }
 
 func NewLevelDB(con *EthConfig) (ethdb.Database, error) {
-	edb, err := rawdb.NewLevelDBDatabaseWithFreezer(
-		con.LevelDBPath, 1024, 256, con.AncientDBPath, "ipld-eth-state-snapshot", true,
-	)
+	kvdb, _ := rawdb.NewLevelDBDatabase(con.LevelDBPath, 1024, 256, "ipld-eth-state-snapshot", true)
+	edb, err := rawdb.NewDatabaseWithFreezer(kvdb, con.AncientDBPath, "ipld-eth-state-snapshot", true)
 	if err != nil {
 		return nil, fmt.Errorf("unable to create NewLevelDBDatabaseWithFreezer: %s", err)
 	}

pkg/snapshot/tracker.go

@@ -11,9 +11,8 @@ import (
 
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/trie"
+	iter "github.com/ethereum/go-ethereum/trie/concurrent_iterator"
 	log "github.com/sirupsen/logrus"
-
-	iter "github.com/cerc-io/go-eth-state-node-iterator"
 )
 
 type trackedIter struct {

pkg/snapshot/util.go

@@ -23,7 +23,7 @@ func NewPublisher(mode SnapshotMode, config *Config) (snapt.Publisher, error) {
 
 		prom.RegisterDBCollector(config.DB.ConnConfig.DatabaseName, driver)
 
-		return pg.NewPublisher(postgres.NewPostgresDB(driver)), nil
+		return pg.NewPublisher(postgres.NewPostgresDB(driver, false)), nil
 	case FileSnapshot:
 		return file.NewPublisher(config.File.OutputDir, config.Eth.NodeInfo)
 	}