cerc-io/ipld-eth-state-snapshot
16
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #72 from cerc-io/ian/v5_rebase

Browse Source 
Rebase
This commit is contained in:
Ian Norden 2023-04-12 13:12:35 -05:00 committed by GitHub
commit fb5a95d874
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
42 changed files with 577 additions and 325 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/on-pr.yaml vendored
View File

@ -15,7 +15,7 @@ jobs:
- uses: actions/setup-go@v3 - uses: actions/setup-go@v3
with: with:
go-version: ">=1.18.0" go-version: ">=1.19.0"
check-latest: true check-latest: true
- name: Checkout code - name: Checkout code

16
README.md
View File

@ -191,3 +191,19 @@ Config format:
``` ```
* NOTE: `COPY` command on CSVs inserts empty strings as `NULL` in the DB. Passing `FORCE_NOT_NULL <COLUMN_NAME>` forces it to insert empty strings instead. This is required to maintain compatibility of the imported snapshot data with the data generated by statediffing. Reference: https://www.postgresql.org/docs/14/sql-copy.html * NOTE: `COPY` command on CSVs inserts empty strings as `NULL` in the DB. Passing `FORCE_NOT_NULL <COLUMN_NAME>` forces it to insert empty strings instead. This is required to maintain compatibility of the imported snapshot data with the data generated by statediffing. Reference: https://www.postgresql.org/docs/14/sql-copy.html
### Troubleshooting
* Run the following command to find any rows (in data dumps in `file` mode) having unexpected number of columns:
```bash
./scripts/find-bad-rows.sh -i <input-file> -c <expected-columns> -o [output-file] -d true
```
* Run the following command to select rows (from data dumps in `file` mode) other than the ones having unexpected number of columns:
```bash
./scripts/filter-bad-rows.sh -i <input-file> -c <expected-columns> -o <output-file>
```
* See [scripts](./scripts) for more details.

2
cmd/inplaceStateSnapshot.go
View File

@ -21,7 +21,7 @@ import (
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/spf13/viper" "github.com/spf13/viper"
"github.com/vulcanize/ipld-eth-state-snapshot/pkg/snapshot" "github.com/cerc-io/ipld-eth-state-snapshot/pkg/snapshot"
) )
// inPlaceStateSnapshotCmd represents the inPlaceStateSnapshot command // inPlaceStateSnapshotCmd represents the inPlaceStateSnapshot command

4
cmd/root.go
View File

@ -25,8 +25,8 @@ import (
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/spf13/viper" "github.com/spf13/viper"
"github.com/vulcanize/ipld-eth-state-snapshot/pkg/prom" "github.com/cerc-io/ipld-eth-state-snapshot/pkg/prom"
"github.com/vulcanize/ipld-eth-state-snapshot/pkg/snapshot" "github.com/cerc-io/ipld-eth-state-snapshot/pkg/snapshot"
) )
var ( var (

2
cmd/stateSnapshot.go
View File

@ -22,7 +22,7 @@ import (
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/spf13/viper" "github.com/spf13/viper"
"github.com/vulcanize/ipld-eth-state-snapshot/pkg/snapshot" "github.com/cerc-io/ipld-eth-state-snapshot/pkg/snapshot"
) )
// stateSnapshotCmd represents the stateSnapshot command // stateSnapshotCmd represents the stateSnapshot command

2
docker-compose.yml
View File

@ -5,7 +5,7 @@ services:
restart: on-failure restart: on-failure
depends_on: depends_on:
- ipld-eth-db - ipld-eth-db
image: vulcanize/ipld-eth-db:v4.2.1-alpha image: git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:v4.2.3-alpha
environment: environment:
DATABASE_USER: "vdbm" DATABASE_USER: "vdbm"
DATABASE_NAME: "vulcanize_testing" DATABASE_NAME: "vulcanize_testing"

0
fixture/chain2data/000006.log Normal file
View File

1
fixture/chain2data/CURRENT Normal file
View File

@ -0,0 +1 @@
MANIFEST-000005

0
fixture/chain2data/LOCK Normal file
View File

15
fixture/chain2data/LOG Normal file
View File

@ -0,0 +1,15 @@
=============== Mar 31, 2023 (CDT) ===============
12:22:01.281499 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed
12:22:01.281586 table@recovery F·2
12:22:01.281592 table@recovery recovering @2
12:22:01.281901 table@recovery recovered @2 Gk·13 Ck·0 Cb·0 S·918 Q·13
12:22:01.281909 table@recovery recovering @4
12:22:01.282725 table@recovery recovered @4 Gk·416 Ck·0 Cb·0 S·35872 Q·430
12:22:01.282737 table@recovery recovered F·2 N·429 Gk·429 Ck·0 Q·430
12:22:01.298688 version@stat F·[2] S·35KiB[35KiB] Sc·[0.50]
12:22:01.308338 db@open opening
12:22:01.308738 version@stat F·[2] S·35KiB[35KiB] Sc·[0.50]
12:22:01.314746 db@janitor F·4 G·0
12:22:01.314778 db@open done T·6.427167ms
12:22:01.317510 db@close closing
12:22:01.317545 db@close done T·34.833µs

BIN
fixture/chain2data/MANIFEST-000005 Normal file
View File

Binary file not shown.

0
fixture/chain2data/ancient/FLOCK Normal file
View File

0
fixture/chaindata/000004.log Normal file
View File

1
fixture/chaindata/CURRENT Normal file
View File

@ -0,0 +1 @@
MANIFEST-000003

0
fixture/chaindata/LOCK Normal file
View File

13
fixture/chaindata/LOG Normal file
View File

@ -0,0 +1,13 @@
=============== Mar 31, 2023 (CDT) ===============
12:22:01.090151 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed
12:22:01.090565 table@recovery F·1
12:22:01.090576 table@recovery recovering @2
12:22:01.106640 table@recovery recovered @2 Gk·146827 Ck·0 Cb·0 S·5765300 Q·146827
12:22:01.106661 table@recovery recovered F·1 N·146827 Gk·146827 Ck·0 Q·146827
12:22:01.133566 version@stat F·[1] S·5MiB[5MiB] Sc·[0.25]
12:22:01.145740 db@open opening
12:22:01.146636 version@stat F·[1] S·5MiB[5MiB] Sc·[0.25]
12:22:01.151049 db@janitor F·3 G·0
12:22:01.151090 db@open done T·5.315792ms
12:22:01.161329 db@close closing
12:22:01.161389 db@close done T·58.667µs

BIN
fixture/chaindata/MANIFEST-000003 Normal file
View File

Binary file not shown.

0
fixture/chaindata/ancient/FLOCK Normal file
View File

1
fixture/chaindata/ancient/bodies.meta Normal file
View File

@ -0,0 +1 @@
<EFBFBD><01>

1
fixture/chaindata/ancient/diffs.meta Normal file
View File

@ -0,0 +1 @@
<EFBFBD><01>

1
fixture/chaindata/ancient/hashes.meta Normal file
View File

@ -0,0 +1 @@
<EFBFBD><01>

1
fixture/chaindata/ancient/headers.meta Normal file
View File

@ -0,0 +1 @@
<EFBFBD><01>

1
fixture/chaindata/ancient/receipts.meta Normal file
View File

@ -0,0 +1 @@
<EFBFBD><01>

2
fixture/in_place_snapshot.go
View File

@ -18,9 +18,9 @@ package fixture
import ( import (
"math/big" "math/big"
snapt "github.com/cerc-io/ipld-eth-state-snapshot/pkg/types"
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/types"
snapt "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types"
) )
type Block struct { type Block struct {

2
fixture/service.go
View File

@ -6,7 +6,7 @@ import (
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/types"
snapt "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types" snapt "github.com/cerc-io/ipld-eth-state-snapshot/pkg/types"
) )
var Block1_Header = types.Header{ var Block1_Header = types.Header{

71
go.mod
View File

@ -1,42 +1,49 @@
module github.com/vulcanize/ipld-eth-state-snapshot module github.com/cerc-io/ipld-eth-state-snapshot
go 1.18 go 1.19
require ( require (
github.com/ethereum/go-ethereum v1.10.21 github.com/ethereum/go-ethereum v1.11.5
github.com/golang/mock v1.6.0 github.com/golang/mock v1.6.0
github.com/ipfs/go-cid v0.2.0 github.com/ipfs/go-cid v0.2.0
github.com/ipfs/go-ipfs-blockstore v1.2.0 github.com/ipfs/go-ipfs-blockstore v1.2.0
github.com/ipfs/go-ipfs-ds-help v1.1.0 github.com/ipfs/go-ipfs-ds-help v1.1.0
github.com/jmoiron/sqlx v1.3.5 github.com/jmoiron/sqlx v1.3.5
github.com/multiformats/go-multihash v0.1.0 github.com/multiformats/go-multihash v0.1.0
github.com/prometheus/client_golang v1.3.0 github.com/prometheus/client_golang v1.14.0
github.com/sirupsen/logrus v1.9.0 github.com/sirupsen/logrus v1.9.0
github.com/spf13/cobra v1.5.0 github.com/spf13/cobra v1.5.0
github.com/spf13/viper v1.12.0 github.com/spf13/viper v1.12.0
github.com/vulcanize/go-eth-state-node-iterator v1.1.4 golang.org/x/sync v0.1.0
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
) )
require ( require (
github.com/DataDog/zstd v1.5.2 // indirect
github.com/VictoriaMetrics/fastcache v1.6.0 // indirect github.com/VictoriaMetrics/fastcache v1.6.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect github.com/beorn7/perks v1.0.1 // indirect
github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect github.com/cockroachdb/errors v1.9.1 // indirect
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811 // indirect
github.com/cockroachdb/redact v1.1.3 // indirect
github.com/deckarep/golang-set/v2 v2.1.0 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
github.com/fsnotify/fsnotify v1.5.4 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/georgysavva/scany v0.2.9 // indirect github.com/georgysavva/scany v1.2.1 // indirect
github.com/go-kit/kit v0.10.0 // indirect github.com/getsentry/sentry-go v0.17.0 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-ole/go-ole v1.2.6 // indirect
github.com/go-stack/stack v1.8.0 // indirect github.com/go-stack/stack v1.8.1 // indirect
github.com/gofrs/flock v0.8.1 // indirect
github.com/gogo/protobuf v1.3.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/protobuf v1.5.2 // indirect github.com/golang/protobuf v1.5.2 // indirect
github.com/golang/snappy v0.0.4 // indirect github.com/golang/snappy v0.0.4 // indirect
github.com/google/uuid v1.3.0 // indirect github.com/google/uuid v1.3.0 // indirect
github.com/gorilla/websocket v1.4.2 // indirect
github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect
github.com/hashicorp/hcl v1.0.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect
github.com/holiman/bloomfilter/v2 v2.0.3 // indirect github.com/holiman/bloomfilter/v2 v2.0.3 // indirect
github.com/holiman/uint256 v1.2.0 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/ipfs/bbloom v0.0.4 // indirect github.com/ipfs/bbloom v0.0.4 // indirect
github.com/ipfs/go-block-format v0.0.3 // indirect github.com/ipfs/go-block-format v0.0.3 // indirect
@ -56,12 +63,15 @@ require (
github.com/jackc/pgx/v4 v4.15.0 // indirect github.com/jackc/pgx/v4 v4.15.0 // indirect
github.com/jackc/puddle v1.2.1 // indirect github.com/jackc/puddle v1.2.1 // indirect
github.com/jbenet/goprocess v0.1.4 // indirect github.com/jbenet/goprocess v0.1.4 // indirect
github.com/klauspost/compress v1.15.15 // indirect
github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/klauspost/cpuid/v2 v2.0.9 // indirect
github.com/lib/pq v1.10.2 // indirect github.com/kr/pretty v0.3.1 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/lib/pq v1.10.7 // indirect
github.com/magiconair/properties v1.8.6 // indirect github.com/magiconair/properties v1.8.6 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect github.com/mattn/go-isatty v0.0.16 // indirect
github.com/mattn/go-runewidth v0.0.9 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect
github.com/minio/sha256-simd v1.0.0 // indirect github.com/minio/sha256-simd v1.0.0 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect
@ -73,38 +83,37 @@ require (
github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml v1.9.5 // indirect
github.com/pelletier/go-toml/v2 v2.0.1 // indirect github.com/pelletier/go-toml/v2 v2.0.5 // indirect
github.com/pkg/errors v0.9.1 // indirect github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect
github.com/prometheus/client_model v0.1.0 // indirect github.com/prometheus/common v0.37.1 // indirect
github.com/prometheus/common v0.7.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect
github.com/prometheus/procfs v0.0.8 // indirect github.com/rogpeppe/go-internal v1.9.0 // indirect
github.com/prometheus/tsdb v0.10.0 // indirect
github.com/shirou/gopsutil v3.21.11+incompatible // indirect github.com/shirou/gopsutil v3.21.11+incompatible // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/spf13/afero v1.8.2 // indirect github.com/spf13/afero v1.8.2 // indirect
github.com/spf13/cast v1.5.0 // indirect github.com/spf13/cast v1.5.0 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect github.com/spf13/pflag v1.0.5 // indirect
github.com/stretchr/objx v0.2.0 // indirect
github.com/stretchr/testify v1.7.2 // indirect
github.com/subosito/gotenv v1.3.0 // indirect github.com/subosito/gotenv v1.3.0 // indirect
github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a // indirect github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a // indirect
github.com/tklauser/go-sysconf v0.3.5 // indirect github.com/tklauser/go-sysconf v0.3.11 // indirect
github.com/tklauser/numcpus v0.2.2 // indirect github.com/tklauser/numcpus v0.6.0 // indirect
github.com/yusufpapurcu/wmi v1.2.2 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect
go.uber.org/atomic v1.9.0 // indirect go.uber.org/atomic v1.9.0 // indirect
go.uber.org/goleak v1.1.11 // indirect go.uber.org/goleak v1.1.11 // indirect
go.uber.org/multierr v1.7.0 // indirect go.uber.org/multierr v1.7.0 // indirect
go.uber.org/zap v1.19.1 // indirect go.uber.org/zap v1.19.1 // indirect
golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect golang.org/x/crypto v0.6.0 // indirect
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect golang.org/x/exp v0.0.0-20230206171751-46f607a40771 // indirect
golang.org/x/text v0.3.7 // indirect golang.org/x/sys v0.5.0 // indirect
google.golang.org/protobuf v1.28.0 // indirect golang.org/x/text v0.7.0 // indirect
gopkg.in/ini.v1 v1.66.4 // indirect google.golang.org/protobuf v1.28.1 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect
lukechampine.com/blake3 v1.1.7 // indirect lukechampine.com/blake3 v1.1.7 // indirect
) )
replace github.com/ethereum/go-ethereum v1.10.21 => github.com/vulcanize/go-ethereum v1.10.21-statediff-4.1.2-alpha replace github.com/ethereum/go-ethereum v1.11.5 => github.com/cerc-io/go-ethereum v1.11.5-statediff-4.3.9-alpha

533
go.sum
View File

File diff suppressed because it is too large Load Diff

2
main.go
View File

@ -18,7 +18,7 @@ package main
import ( import (
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
"github.com/vulcanize/ipld-eth-state-snapshot/cmd" "github.com/cerc-io/ipld-eth-state-snapshot/cmd"
) )
func main() { func main() {

4
pkg/prom/db_stats_collector.go
View File

@ -19,12 +19,12 @@ package prom
import ( import (
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql" mets "github.com/ethereum/go-ethereum/statediff/indexer/database/metrics"
) )
// DBStatsGetter is an interface that gets sql.DBStats. // DBStatsGetter is an interface that gets sql.DBStats.
type DBStatsGetter interface { type DBStatsGetter interface {
Stats() sql.Stats Stats() mets.DbStats
} }
// DBStatsCollector implements the prometheus.Collector interface. // DBStatsCollector implements the prometheus.Collector interface.

4
pkg/snapshot/file/publisher.go
View File

@ -32,11 +32,11 @@ import (
"github.com/multiformats/go-multihash" "github.com/multiformats/go-multihash"
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
"github.com/cerc-io/ipld-eth-state-snapshot/pkg/prom"
snapt "github.com/cerc-io/ipld-eth-state-snapshot/pkg/types"
"github.com/ethereum/go-ethereum/statediff/indexer/ipld" "github.com/ethereum/go-ethereum/statediff/indexer/ipld"
nodeinfo "github.com/ethereum/go-ethereum/statediff/indexer/node" nodeinfo "github.com/ethereum/go-ethereum/statediff/indexer/node"
"github.com/ethereum/go-ethereum/statediff/indexer/shared" "github.com/ethereum/go-ethereum/statediff/indexer/shared"
"github.com/vulcanize/ipld-eth-state-snapshot/pkg/prom"
snapt "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types"
) )
var _ snapt.Publisher = (*publisher)(nil) var _ snapt.Publisher = (*publisher)(nil)

12
pkg/snapshot/file/publisher_test.go
View File

@ -9,13 +9,13 @@ import (
"path/filepath" "path/filepath"
"testing" "testing"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
"github.com/ethereum/go-ethereum/statediff/indexer/ipld" "github.com/ethereum/go-ethereum/statediff/indexer/ipld"
"github.com/ethereum/go-ethereum/statediff/indexer/test_helpers"
fixt "github.com/vulcanize/ipld-eth-state-snapshot/fixture" fixt "github.com/cerc-io/ipld-eth-state-snapshot/fixture"
snapt "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types" snapt "github.com/cerc-io/ipld-eth-state-snapshot/pkg/types"
"github.com/vulcanize/ipld-eth-state-snapshot/test" "github.com/cerc-io/ipld-eth-state-snapshot/test"
) )
var ( var (
@ -91,9 +91,9 @@ func TestPgCopy(t *testing.T) {
ctx := context.Background() ctx := context.Background()
driver, err := postgres.NewSQLXDriver(ctx, pgConfig, nodeInfo) driver, err := postgres.NewSQLXDriver(ctx, pgConfig, nodeInfo)
test.NoError(t, err) test.NoError(t, err)
db := postgres.NewPostgresDB(driver) db := postgres.NewPostgresDB(driver, false)
sql.TearDownDB(t, db) test_helpers.TearDownDB(t, db)
// copy from files // copy from files
pgCopyStatement := `COPY %s FROM '%s' CSV` pgCopyStatement := `COPY %s FROM '%s' CSV`

2
pkg/snapshot/in_place_snapshot.go
View File

@ -16,9 +16,9 @@
package snapshot package snapshot
import ( import (
. "github.com/cerc-io/ipld-eth-state-snapshot/pkg/types"
"github.com/jmoiron/sqlx" "github.com/jmoiron/sqlx"
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
. "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types"
) )
const ( const (

14
pkg/snapshot/in_place_snapshot_test.go
View File

@ -21,16 +21,16 @@ import (
"testing" "testing"
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
"github.com/ethereum/go-ethereum/statediff/indexer/ipld" "github.com/ethereum/go-ethereum/statediff/indexer/ipld"
"github.com/ethereum/go-ethereum/statediff/indexer/models" "github.com/ethereum/go-ethereum/statediff/indexer/models"
"github.com/ethereum/go-ethereum/statediff/indexer/test_helpers"
"github.com/multiformats/go-multihash" "github.com/multiformats/go-multihash"
fixt "github.com/vulcanize/ipld-eth-state-snapshot/fixture" fixt "github.com/cerc-io/ipld-eth-state-snapshot/fixture"
"github.com/vulcanize/ipld-eth-state-snapshot/pkg/snapshot/pg" "github.com/cerc-io/ipld-eth-state-snapshot/pkg/snapshot/pg"
snapt "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types" snapt "github.com/cerc-io/ipld-eth-state-snapshot/pkg/types"
"github.com/vulcanize/ipld-eth-state-snapshot/test" "github.com/cerc-io/ipld-eth-state-snapshot/test"
) )
var ( var (
@ -80,9 +80,9 @@ func TestCreateInPlaceSnapshot(t *testing.T) {
ctx := context.Background() ctx := context.Background()
driver, err := postgres.NewSQLXDriver(ctx, pgConfig, nodeInfo) driver, err := postgres.NewSQLXDriver(ctx, pgConfig, nodeInfo)
test.NoError(t, err) test.NoError(t, err)
db := postgres.NewPostgresDB(driver) db := postgres.NewPostgresDB(driver, false)
sql.TearDownDB(t, db) test_helpers.TearDownDB(t, db)
_ = writeData(t, db) _ = writeData(t, db)

4
pkg/snapshot/pg/publisher.go
View File

@ -31,12 +31,12 @@ import (
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/cerc-io/ipld-eth-state-snapshot/pkg/prom"
snapt "github.com/cerc-io/ipld-eth-state-snapshot/pkg/types"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql" "github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
"github.com/ethereum/go-ethereum/statediff/indexer/ipld" "github.com/ethereum/go-ethereum/statediff/indexer/ipld"
"github.com/ethereum/go-ethereum/statediff/indexer/shared" "github.com/ethereum/go-ethereum/statediff/indexer/shared"
"github.com/vulcanize/ipld-eth-state-snapshot/pkg/prom"
snapt "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types"
) )
var _ snapt.Publisher = (*publisher)(nil) var _ snapt.Publisher = (*publisher)(nil)

12
pkg/snapshot/pg/publisher_test.go
View File

@ -4,13 +4,13 @@ import (
"context" "context"
"testing" "testing"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
"github.com/ethereum/go-ethereum/statediff/indexer/ipld" "github.com/ethereum/go-ethereum/statediff/indexer/ipld"
"github.com/ethereum/go-ethereum/statediff/indexer/test_helpers"
fixt "github.com/vulcanize/ipld-eth-state-snapshot/fixture" fixt "github.com/cerc-io/ipld-eth-state-snapshot/fixture"
snapt "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types" snapt "github.com/cerc-io/ipld-eth-state-snapshot/pkg/types"
"github.com/vulcanize/ipld-eth-state-snapshot/test" "github.com/cerc-io/ipld-eth-state-snapshot/test"
) )
var ( var (
@ -46,9 +46,9 @@ func TestBasic(t *testing.T) {
ctx := context.Background() ctx := context.Background()
driver, err := postgres.NewSQLXDriver(ctx, pgConfig, nodeInfo) driver, err := postgres.NewSQLXDriver(ctx, pgConfig, nodeInfo)
test.NoError(t, err) test.NoError(t, err)
db := postgres.NewPostgresDB(driver) db := postgres.NewPostgresDB(driver, false)
sql.TearDownDB(t, db) test_helpers.TearDownDB(t, db)
_ = writeData(t, db) _ = writeData(t, db)

11
pkg/snapshot/service.go
View File

@ -30,12 +30,12 @@ import (
"github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie"
iter "github.com/ethereum/go-ethereum/trie/concurrent_iterator"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"golang.org/x/sync/errgroup" "golang.org/x/sync/errgroup"
iter "github.com/vulcanize/go-eth-state-node-iterator" "github.com/cerc-io/ipld-eth-state-snapshot/pkg/prom"
"github.com/vulcanize/ipld-eth-state-snapshot/pkg/prom" . "github.com/cerc-io/ipld-eth-state-snapshot/pkg/types"
. "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types"
) )
var ( var (
@ -59,9 +59,8 @@ type Service struct {
} }
func NewLevelDB(con *EthConfig) (ethdb.Database, error) { func NewLevelDB(con *EthConfig) (ethdb.Database, error) {
edb, err := rawdb.NewLevelDBDatabaseWithFreezer( kvdb, _ := rawdb.NewLevelDBDatabase(con.LevelDBPath, 1024, 256, "ipld-eth-state-snapshot", true)
con.LevelDBPath, 1024, 256, con.AncientDBPath, "ipld-eth-state-snapshot", true, edb, err := rawdb.NewDatabaseWithFreezer(kvdb, con.AncientDBPath, "ipld-eth-state-snapshot", true)
)
if err != nil { if err != nil {
return nil, fmt.Errorf("unable to create NewLevelDBDatabaseWithFreezer: %s", err) return nil, fmt.Errorf("unable to create NewLevelDBDatabaseWithFreezer: %s", err)
} }

8
pkg/snapshot/service_test.go
View File

@ -15,10 +15,10 @@ import (
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/golang/mock/gomock" "github.com/golang/mock/gomock"
fixt "github.com/vulcanize/ipld-eth-state-snapshot/fixture" fixt "github.com/cerc-io/ipld-eth-state-snapshot/fixture"
mock "github.com/vulcanize/ipld-eth-state-snapshot/mocks/snapshot" mock "github.com/cerc-io/ipld-eth-state-snapshot/mocks/snapshot"
snapt "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types" snapt "github.com/cerc-io/ipld-eth-state-snapshot/pkg/types"
"github.com/vulcanize/ipld-eth-state-snapshot/test" "github.com/cerc-io/ipld-eth-state-snapshot/test"
) )
var ( var (

3
pkg/snapshot/tracker.go
View File

@ -11,9 +11,8 @@ import (
"github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie"
iter "github.com/ethereum/go-ethereum/trie/concurrent_iterator"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
iter "github.com/vulcanize/go-eth-state-node-iterator"
) )
type trackedIter struct { type trackedIter struct {

10
pkg/snapshot/util.go
View File

@ -7,10 +7,10 @@ import (
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
"github.com/vulcanize/ipld-eth-state-snapshot/pkg/prom" "github.com/cerc-io/ipld-eth-state-snapshot/pkg/prom"
file "github.com/vulcanize/ipld-eth-state-snapshot/pkg/snapshot/file" file "github.com/cerc-io/ipld-eth-state-snapshot/pkg/snapshot/file"
pg "github.com/vulcanize/ipld-eth-state-snapshot/pkg/snapshot/pg" pg "github.com/cerc-io/ipld-eth-state-snapshot/pkg/snapshot/pg"
snapt "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types" snapt "github.com/cerc-io/ipld-eth-state-snapshot/pkg/types"
) )
func NewPublisher(mode SnapshotMode, config *Config) (snapt.Publisher, error) { func NewPublisher(mode SnapshotMode, config *Config) (snapt.Publisher, error) {
@ -23,7 +23,7 @@ func NewPublisher(mode SnapshotMode, config *Config) (snapt.Publisher, error) {
prom.RegisterDBCollector(config.DB.ConnConfig.DatabaseName, driver) prom.RegisterDBCollector(config.DB.ConnConfig.DatabaseName, driver)
return pg.NewPublisher(postgres.NewPostgresDB(driver)), nil return pg.NewPublisher(postgres.NewPostgresDB(driver, false)), nil
case FileSnapshot: case FileSnapshot:
return file.NewPublisher(config.File.OutputDir, config.Eth.NodeInfo) return file.NewPublisher(config.File.OutputDir, config.Eth.NodeInfo)
} }

73
scripts/README.md Normal file
View File

@ -0,0 +1,73 @@
## Data Validation
* For a given table in the `ipld-eth-db` schema, we know the number of columns to be expected in each row in the data dump:
| Table | Expected columns |
| ----------------- |:----------------:|
| public.nodes | 5 |
| public.blocks | 3 |
| eth.header_cids | 16 |
| eth.state_cids | 8 |
| eth.storage_cids | 9 |
### Find Bad Data
* Run the following command to find any rows having unexpected number of columns:
```bash
./scripts/find-bad-rows.sh -i <input-file> -c <expected-columns> -o [output-file] -d [include-data]
```
* `input-file` `-i`: Input data file path
* `expected-columns` `-c`: Expected number of columns in each row of the input file
* `output-file` `-o`: Output destination file path (default: `STDOUT`)
* `include-data` `-d`: Whether to include the data row in the output (`true | false`) (default: `false`)
* The output is of format: row number, number of columns, the data row
Eg:
```bash
./scripts/find-bad-rows.sh -i eth.state_cids.csv -c 8 -o res.txt -d true
```
Output:
```
1 9 1500000,xxxxxxxx,0x83952d392f9b0059eea94b10d1a095eefb1943ea91595a16c6698757127d4e1c,,baglacgzasvqcntdahkxhufdnkm7a22s2eetj6mx6nzkarwxtkvy4x3bubdgq,\x0f,0,f,/blocks/,DMQJKYBGZRQDVLT2CRWVGPQNNJNCCJU7GL7G4VAI3LZVK4OL5Q2ARTI
```
Eg:
```bash
./scripts/find-bad-rows.sh -i public.nodes.csv -c 5 -o res.txt -d true
./scripts/find-bad-rows.sh -i public.blocks.csv -c 3 -o res.txt -d true
./scripts/find-bad-rows.sh -i eth.header_cids.csv -c 16 -o res.txt -d true
./scripts/find-bad-rows.sh -i eth.state_cids.csv -c 8 -o res.txt -d true
./scripts/find-bad-rows.sh -i eth.storage_cids.csv -c 9 -o res.txt -d true
```
## Data Cleanup
* In case of column count mismatch, data from `file` mode dumps can't be imported readily into `ipld-eth-db`.
### Filter Bad Data
* Run the following command to filter out rows having unexpected number of columns:
```bash
./scripts/filter-bad-rows.sh -i <input-file> -c <expected-columns> -o <output-file>
```
* `input-file` `-i`: Input data file path
* `expected-columns` `-c`: Expected number of columns in each row of the input file
* `output-file` `-o`: Output destination file path
Eg:
```bash
./scripts/filter-bad-rows.sh -i public.nodes.csv -c 5 -o cleaned-public.nodes.csv
./scripts/filter-bad-rows.sh -i public.blocks.csv -c 3 -o cleaned-public.blocks.csv
./scripts/filter-bad-rows.sh -i eth.header_cids.csv -c 16 -o cleaned-eth.header_cids.csv
./scripts/filter-bad-rows.sh -i eth.state_cids.csv -c 8 -o cleaned-eth.state_cids.csv
./scripts/filter-bad-rows.sh -i eth.storage_cids.csv -c 9 -o cleaned-eth.storage_cids.csv
```

29
scripts/filter-bad-rows.sh Executable file
View File

@ -0,0 +1,29 @@
#!/bin/bash
# flags
# -i <input-file>: Input data file path
# -c <expected-columns>: Expected number of columns in each row of the input file
# -o [output-file]: Output destination file path
# eg: ./scripts/filter-bad-rows.sh -i eth.state_cids.csv -c 8 -o cleaned-eth.state_cids.csv
while getopts i:c:o: OPTION
do
case "${OPTION}" in
i) inputFile=${OPTARG};;
c) expectedColumns=${OPTARG};;
o) outputFile=${OPTARG};;
esac
done
timestamp=$(date +%s)
# select only rows having expected number of columns
if [ -z "${outputFile}" ]; then
echo "Invalid destination file arg (-o) ${outputFile}"
else
awk -F"," "NF==${expectedColumns}" ${inputFile} > ${outputFile}
fi
difference=$(($(date +%s)-timestamp))
echo Time taken: $(date -d@${difference} -u +%H:%M:%S)

43
scripts/find-bad-rows.sh Executable file
View File

@ -0,0 +1,43 @@
#!/bin/bash
# flags
# -i <input-file>: Input data file path
# -c <expected-columns>: Expected number of columns in each row of the input file
# -o [output-file]: Output destination file path (default: STDOUT)
# -d [include-data]: Whether to include the data row in output (true | false) (default: false)
# eg: ./scripts/find-bad-rows.sh -i eth.state_cids.csv -c 8 -o res.txt -d true
# output: 1 9 1500000,xxxxxxxx,0x83952d392f9b0059eea94b10d1a095eefb1943ea91595a16c6698757127d4e1c,,
# baglacgzasvqcntdahkxhufdnkm7a22s2eetj6mx6nzkarwxtkvy4x3bubdgq,\x0f,0,f,/blocks/,
# DMQJKYBGZRQDVLT2CRWVGPQNNJNCCJU7GL7G4VAI3LZVK4OL5Q2ARTI
while getopts i:c:o:d: OPTION
do
case "${OPTION}" in
i) inputFile=${OPTARG};;
c) expectedColumns=${OPTARG};;
o) outputFile=${OPTARG};;
d) data=${OPTARG};;
esac
done
timestamp=$(date +%s)
# if data requested, dump row number, number of columns and the row
if [ "${data}" = true ] ; then
if [ -z "${outputFile}" ]; then
awk -F"," "NF!=${expectedColumns} {print NR, NF, \$0}" < ${inputFile}
else
awk -F"," "NF!=${expectedColumns} {print NR, NF, \$0}" < ${inputFile} > ${outputFile}
fi
# else, dump only row number, number of columns
else
if [ -z "${outputFile}" ]; then
awk -F"," "NF!=${expectedColumns} {print NR, NF}" < ${inputFile}
else
awk -F"," "NF!=${expectedColumns} {print NR, NF}" < ${inputFile} > ${outputFile}
fi
fi
difference=$(($(date +%s)-timestamp))
echo Time taken: $(date -d@${difference} -u +%H:%M:%S)