From c4b4f232e9bec62abedb08326d55ae8c1c130cd0 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Wed, 9 Feb 2022 23:19:10 +0800 Subject: [PATCH 1/6] create csv file publisher; update geth and schema --- Makefile | 2 +- cmd/stateSnapshot.go | 17 +- fixture/chaindata.go | 27 +++ fixture/service.go | 4 +- go.mod | 9 +- go.sum | 158 ++++++++++++++- pkg/snapshot/config.go | 87 +++++--- pkg/snapshot/file/publisher.go | 298 ++++++++++++++++++++++++++++ pkg/snapshot/file/publisher_test.go | 136 +++++++++++++ pkg/snapshot/mock/publisher.go | 64 ------ pkg/snapshot/{ => pg}/publisher.go | 158 ++++++++------- pkg/snapshot/pg/publisher_test.go | 78 ++++++++ pkg/snapshot/service.go | 44 ++-- pkg/snapshot/service_test.go | 71 ++----- pkg/snapshot/util.go | 26 +++ pkg/types/node_type.go | 3 +- pkg/types/publisher.go | 19 +- pkg/types/schema.go | 73 +++++++ pkg/types/table.go | 79 ++++++++ pkg/types/util.go | 33 +++ test/helper.go | 52 +++++ 21 files changed, 1167 insertions(+), 271 deletions(-) create mode 100644 fixture/chaindata.go create mode 100644 pkg/snapshot/file/publisher.go create mode 100644 pkg/snapshot/file/publisher_test.go delete mode 100644 pkg/snapshot/mock/publisher.go rename pkg/snapshot/{ => pg}/publisher.go (53%) create mode 100644 pkg/snapshot/pg/publisher_test.go create mode 100644 pkg/snapshot/util.go create mode 100644 pkg/types/schema.go create mode 100644 pkg/types/table.go create mode 100644 pkg/types/util.go create mode 100644 test/helper.go diff --git a/Makefile b/Makefile index f5afb82..76d7f0c 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ mockgen_cmd=mockgen mocks: mocks/snapshot/publisher.go mocks/snapshot/publisher.go: pkg/types/publisher.go - $(mockgen_cmd) -package snapshot_mock -destination $@ -source $< Publisher + $(mockgen_cmd) -package snapshot_mock -destination $@ -source $< Publisher Tx clean: rm -f mocks/snapshot/publisher.go diff --git a/cmd/stateSnapshot.go b/cmd/stateSnapshot.go index 119be17..eefc609 100644 --- a/cmd/stateSnapshot.go +++ b/cmd/stateSnapshot.go @@ -38,22 +38,25 @@ var stateSnapshotCmd = &cobra.Command{ } func stateSnapshot() { - snapConfig := snapshot.NewConfig() - pgDB, err := snapshot.NewPostgresDB(snapConfig.DB) - if err != nil { - logWithCommand.Fatal(err) - } - edb, err := snapshot.NewLevelDB(snapConfig.Eth) + config := snapshot.NewConfig() + edb, err := snapshot.NewLevelDB(config.Eth) if err != nil { logWithCommand.Fatal(err) } - snapshotService, err := snapshot.NewSnapshotService(edb, snapshot.NewPublisher(pgDB)) + mode := viper.GetString("snapshot.mode") + pub, err := snapshot.NewPublisher(snapshot.SnapshotMode(mode), config) + if err != nil { + logWithCommand.Fatal(err) + } + + snapshotService, err := snapshot.NewSnapshotService(edb, pub) if err != nil { logWithCommand.Fatal(err) } height := viper.GetInt64("snapshot.blockHeight") workers := viper.GetUint("snapshot.workers") + if height < 0 { if err := snapshotService.CreateLatestSnapshot(workers); err != nil { logWithCommand.Fatal(err) diff --git a/fixture/chaindata.go b/fixture/chaindata.go new file mode 100644 index 0000000..6cc4c29 --- /dev/null +++ b/fixture/chaindata.go @@ -0,0 +1,27 @@ +package fixture + +import ( + "os" + "path/filepath" + "runtime" +) + +// TODO: embed some mainnet data +// import "embed" +//_go:embed mainnet_data.tar.gz + +var ( + ChaindataPath, AncientdataPath string +) + +func init() { + _, path, _, _ := runtime.Caller(0) + wd := filepath.Dir(path) + + ChaindataPath = filepath.Join(wd, "..", "fixture", "chaindata") + AncientdataPath = filepath.Join(ChaindataPath, "ancient") + + if _, err := os.Stat(ChaindataPath); err != nil { + panic("must populate chaindata at " + ChaindataPath) + } +} diff --git a/fixture/service.go b/fixture/service.go index 812e2ff..25a48a2 100644 --- a/fixture/service.go +++ b/fixture/service.go @@ -10,7 +10,7 @@ import ( snapt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/types" ) -var PublishHeader = &types.Header{ +var Header1 = types.Header{ ParentHash: common.HexToHash("0x6341fd3daf94b748c72ced5a5b26028f2474f5f00d824504e4fa37a75767e177"), UncleHash: common.HexToHash("0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347"), Coinbase: common.HexToAddress("0x0000000000000000000000000000000000000000"), @@ -29,7 +29,7 @@ var PublishHeader = &types.Header{ BaseFee: nil, } -var StateNode = snapt.Node{ +var StateNode1 = snapt.Node{ NodeType: 0, Path: []byte{12, 0}, Key: common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000"), diff --git a/go.mod b/go.mod index 83ee9da..5dd22c3 100644 --- a/go.mod +++ b/go.mod @@ -12,12 +12,13 @@ require ( github.com/google/go-cmp v0.5.6 // indirect github.com/google/uuid v1.3.0 // indirect github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c // indirect - github.com/ipfs/go-cid v0.1.0 // indirect + github.com/ipfs/go-cid v0.1.0 github.com/ipfs/go-datastore v0.5.1 // indirect - github.com/ipfs/go-ipfs-blockstore v1.1.2 // indirect + github.com/ipfs/go-ipfs-blockstore v1.1.2 + github.com/ipfs/go-ipfs-ds-help v1.1.0 github.com/ipfs/go-log v1.0.5 // indirect github.com/ipfs/go-log/v2 v2.4.0 // indirect - github.com/jmoiron/sqlx v1.2.0 + github.com/jackc/pgx/v4 v4.15.0 github.com/kr/pretty v0.3.0 // indirect github.com/multiformats/go-base32 v0.0.4 // indirect github.com/multiformats/go-multihash v0.1.0 @@ -42,4 +43,4 @@ require ( lukechampine.com/blake3 v1.1.7 // indirect ) -replace github.com/ethereum/go-ethereum v1.10.15 => github.com/vulcanize/go-ethereum v1.10.15-statediff-2.0.0 +replace github.com/ethereum/go-ethereum v1.10.15 => github.com/vulcanize/go-ethereum v1.10.15-statediff-3.0.1 diff --git a/go.sum b/go.sum index 4ff9acd..84a034b 100644 --- a/go.sum +++ b/go.sum @@ -37,6 +37,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= +github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= +github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= @@ -101,6 +103,7 @@ github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOC github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk= github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -113,6 +116,10 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.14.0/go.mod h1:EnwdgGMaFOruiPZRFSgn+TsQ3hQ7C/YWzIGLeu5c304= +github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/cockroachdb/cockroach-go/v2 v2.0.3 h1:ZA346ACHIZctef6trOTwBAEvPVm1k0uLm/bb2Atc+S8= +github.com/cockroachdb/cockroach-go/v2 v2.0.3/go.mod h1:hAuDgiVgDVkfirP9JnhXEfcXEPRKBpYdGz+l7mvYSzw= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ= @@ -124,6 +131,7 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -141,6 +149,7 @@ github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vs github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= github.com/deepmap/oapi-codegen v1.6.0/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE9iAUlWl9Al3M= github.com/deepmap/oapi-codegen v1.8.2/go.mod h1:YLgSKSDv/bZQB7N4ws6luhozi3cEdRktEqrX88CvjIw= +github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8/go.mod h1:VMaSuZ+SZcx/wljOQKvp5srsbCiKDEb6K2wC4+PiBmQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= @@ -158,7 +167,9 @@ github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaB github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c= github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= @@ -167,7 +178,10 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= +github.com/georgysavva/scany v0.2.9 h1:Xt6rjYpHnMClTm/g+oZTnoSxUwiln5GqMNU+QeLNHQU= +github.com/georgysavva/scany v0.2.9/go.mod h1:yeOeC1BdIdl6hOwy8uefL2WNSlseFzbhlG/frrh65SA= github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -180,6 +194,7 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.10.0 h1:dXFJfIHVvUcpSgDOV+Ne6t7jXri8Tfv2uOLHUZ2XNuo= github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4= @@ -195,7 +210,10 @@ github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= +github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -203,6 +221,7 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -283,6 +302,7 @@ github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoP github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpxn4uE= github.com/hashicorp/go-bexpr v0.1.10/go.mod h1:oxlubA2vC/gFVfX1A6JGp7ls7uCDlfJn732ehYYg+g0= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= @@ -312,6 +332,7 @@ github.com/holiman/uint256 v1.2.0 h1:gpSYcPLWGv4sG43I2mVLiDZCNDh/EpGjSk8tmtxitHM github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= +github.com/huin/goupnp v1.0.2 h1:RfGLP+h3mvisuWEyybxNq5Eft3NWhHLPeUN72kpKZoI= github.com/huin/goupnp v1.0.2/go.mod h1:0dxJBVBHqTMjIUMkESDTNgOOx/Mw5wYIfyFmdzSamkM= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -367,6 +388,85 @@ github.com/ipfs/go-log/v2 v2.4.0 h1:iR/2o9PGWanVJrBgIH5Ff8mPGOwpqLaPIAFqSnsdlzk= github.com/ipfs/go-log/v2 v2.4.0/go.mod h1:nPZnh7Cj7lwS3LpRU5Mwr2ol1c2gXIEXuF6aywqrtmo= github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= +github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/fake v0.0.0-20150926172116-812a484cc733/go.mod h1:WrMFNQdiFJ80sQsxDoMokWK1W5TQtxBFNpzWTD84ibQ= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= +github.com/jackc/pgconn v1.4.0/go.mod h1:Y2O3ZDF0q4mMacyWV3AstPJpeHXWGEetiFttmq5lahk= +github.com/jackc/pgconn v1.5.0/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI= +github.com/jackc/pgconn v1.5.1-0.20200601181101-fa742c524853/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI= +github.com/jackc/pgconn v1.6.4/go.mod h1:w2pne1C2tZgP+TvjqLpOigGzNqjBgQW9dUw/4Chex78= +github.com/jackc/pgconn v1.7.0/go.mod h1:sF/lPpNEMEOp+IYhyQGdAvrG20gWf6A1tKlr0v7JMeA= +github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= +github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= +github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgconn v1.10.0/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgconn v1.11.0 h1:HiHArx4yFbwl91X3qqIHtUFoiIfLNJXCQRsnzkiwwaQ= +github.com/jackc/pgconn v1.11.0/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65 h1:DadwsjnMwFjfWc9y5Wi/+Zz7xoE5ALHsRQlOctkOiHc= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.0.2/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.0.5/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.2.0 h1:r7JypeP2D3onoQTCxWdTpCtJ4D+qpKr0TxvoyMhZ5ns= +github.com/jackc/pgproto3/v2 v2.2.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= +github.com/jackc/pgtype v1.2.0/go.mod h1:5m2OfMh1wTK7x+Fk952IDmI4nw3nPrvtQdM0ZT4WpC0= +github.com/jackc/pgtype v1.3.0/go.mod h1:b0JqxHvPmljG+HQ5IsvQ0yqeSi4nGcDTVjFoiLDb0Ik= +github.com/jackc/pgtype v1.3.1-0.20200510190516-8cd94a14c75a/go.mod h1:vaogEUkALtxZMCH411K+tKzNpwzCKU+AnPzBKZ+I+Po= +github.com/jackc/pgtype v1.3.1-0.20200606141011-f6355165a91c/go.mod h1:cvk9Bgu/VzJ9/lxTO5R5sf80p0DiucVtN7ZxvaC4GmQ= +github.com/jackc/pgtype v1.4.2/go.mod h1:JCULISAZBFGrHaOXIIFiyfzW5VY0GRitRr8NeJsrdig= +github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= +github.com/jackc/pgtype v1.8.1/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgtype v1.10.0 h1:ILnBWrRMSXGczYvmkYD6PsYyVFUNLTnIUJHHDLmqk38= +github.com/jackc/pgtype v1.10.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgx v3.6.2+incompatible h1:2zP5OD7kiyR3xzRYMhOcXVvkDZsImVXfj+yIyTQf3/o= +github.com/jackc/pgx v3.6.2+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/pgx/v4 v4.5.0/go.mod h1:EpAKPLdnTorwmPUUsqrPxy5fphV18j9q3wrfRXgo+kA= +github.com/jackc/pgx/v4 v4.6.0/go.mod h1:vPh43ZzxijXUVJ+t/EmXBtFmbFVO72cuneCT9oAlxAg= +github.com/jackc/pgx/v4 v4.6.1-0.20200510190926-94ba730bb1e9/go.mod h1:t3/cdRQl6fOLDxqtlyhe9UWgfIi9R8+8v8GKV5TRA/o= +github.com/jackc/pgx/v4 v4.6.1-0.20200606145419-4e5062306904/go.mod h1:ZDaNWkt9sW1JMiNn0kdYBaLelIhw7Pg4qd+Vk6tw7Hg= +github.com/jackc/pgx/v4 v4.8.1/go.mod h1:4HOLxrl8wToZJReD04/yB20GDwf4KBYETvlHciCnwW0= +github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= +github.com/jackc/pgx/v4 v4.13.0/go.mod h1:9P4X524sErlaxj0XSGZk7s+LD0eOyu1ZDUrrpznYDF0= +github.com/jackc/pgx/v4 v4.15.0 h1:B7dTkXsdILD3MF987WGGCcg+tvLW6bZJdEcqVFeU//w= +github.com/jackc/pgx/v4 v4.15.0/go.mod h1:D/zyOyXiaM1TmVWnOM18p0xdDtdakRBa0RsVGI3U3bw= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.2/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.2.1 h1:gI8os0wpRXFd4FiAY2dWiqRK037tjj3t7rKFeO4X5iw= +github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 h1:6OvNmYgJyexcZ3pYbTI9jWx5tHo1Dee/tWbLMfPe2TA= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY= @@ -375,6 +475,9 @@ github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZl github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jinzhu/gorm v1.9.12/go.mod h1:vhTjlKSJUTWNtcbQtrMBFCxy7eXTzeCAzfL5fBZT/Qs= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= @@ -408,6 +511,7 @@ github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02 github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -417,6 +521,7 @@ github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= @@ -425,6 +530,11 @@ github.com/labstack/echo/v4 v4.2.1/go.mod h1:AA49e0DZ8kk5jTOOCKNuPR6oTnBS0dYiM4F github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.4.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM= @@ -440,13 +550,16 @@ github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIG github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= @@ -458,8 +571,8 @@ github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/Qd github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= -github.com/mattn/go-sqlite3 v1.14.7 h1:fxWBnXkxfM6sRiuH3bqJ4CfzZojMOLVc0UTsTglEghA= -github.com/mattn/go-sqlite3 v1.14.7/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/mattn/go-sqlite3 v2.0.1+incompatible h1:xQ15muvnzGBHpIpdrNi1DA5x0+TcBZzsIDwmw9uTHzw= +github.com/mattn/go-sqlite3 v2.0.1+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= @@ -479,6 +592,7 @@ github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:F github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/pointerstructure v1.2.0 h1:O+i9nHnXS3l/9Wu7r4NrEdwA2VFTicjUEN1uBnDo34A= github.com/mitchellh/pointerstructure v1.2.0/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -596,23 +710,35 @@ github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52/go.mod h1:RDpi1RftBQPUCDRw6SmxeaREsAaRKnOclghuzp/WRzc= +github.com/rjeczalik/notify v0.9.1 h1:CLCKso/QK1snAlnhNR/CNvNiFU2saUtjV0bx3EwNeCE= github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU= github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/shopspring/decimal v0.0.0-20200419222939-1884f454f8ea/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= @@ -641,12 +767,15 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= +github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 h1:Gb2Tyox57NRNuZ2d3rmvB3pcmbu7O1RS3m8WRx7ilrg= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -665,6 +794,7 @@ github.com/tklauser/numcpus v0.2.2 h1:oyhllyrScuYI6g+h/zUvNXNp1wy7x8qQy3t/piefld github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef h1:wHSqTBrZW24CsNJDfeh9Ex6Pm0Rcpc7qrgKBiL44vF4= github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= @@ -675,8 +805,8 @@ github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPU github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/vulcanize/go-eth-state-node-iterator v1.0.0 h1:FQ4s0K5TnRD44p3vO4uQtjfE8C1Gr2EDMT2vVGS3Lu8= github.com/vulcanize/go-eth-state-node-iterator v1.0.0/go.mod h1:uWhleTvUEZ+cEkNRIAmBpZ14KilTP71OxY5NZDrpNlo= -github.com/vulcanize/go-ethereum v1.10.15-statediff-2.0.0 h1:/BiYPUHnubh46YVtASGt4MPlFR96Rc+iJuTyOI8KZa4= -github.com/vulcanize/go-ethereum v1.10.15-statediff-2.0.0/go.mod h1:9L+QY31AnWnX2/2HDOySCjQoYUdWNGBRMezFJVfH73E= +github.com/vulcanize/go-ethereum v1.10.15-statediff-3.0.1 h1:MX7WcTwxpxOoYVMifrs9vsHDmM6gKOFF+2KnP19LddI= +github.com/vulcanize/go-ethereum v1.10.15-statediff-3.0.1/go.mod h1:XO9WLkNXfwoJN05BZj0//xgOWHJyUrUPdnudbQfKlUo= github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= @@ -686,6 +816,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= @@ -722,21 +853,29 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= -golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= +golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b h1:QAqMVf3pSa6eeTsuklijukjXBlj7Es2QQplab+/RbQ4= golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -822,6 +961,7 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -835,6 +975,7 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -892,6 +1033,7 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -906,12 +1048,14 @@ golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -935,6 +1079,8 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.8 h1:P1HhGGuLW4aAclzjtmJdf0mJOjVUZUzOTqkAkWL+l6w= golang.org/x/tools v0.1.8/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1007,6 +1153,7 @@ gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qS gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU= @@ -1015,6 +1162,7 @@ gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6/go.mod h1:uAJ gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/urfave/cli.v1 v1.20.0 h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0= gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= diff --git a/pkg/snapshot/config.go b/pkg/snapshot/config.go index 4c20199..e097c94 100644 --- a/pkg/snapshot/config.go +++ b/pkg/snapshot/config.go @@ -16,8 +16,10 @@ package snapshot import ( + "time" + + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" ethNode "github.com/ethereum/go-ethereum/statediff/indexer/node" - "github.com/ethereum/go-ethereum/statediff/indexer/postgres" "github.com/spf13/viper" ) @@ -30,40 +32,56 @@ const ( LVL_DB_PATH = "LVL_DB_PATH" ) +// SnapshotMode specifies the snapshot data output method +type SnapshotMode string + +const ( + PgSnapshot SnapshotMode = "postgres" + FileSnapshot SnapshotMode = "file" +) + // Config contains params for both databases the service uses type Config struct { - DB *DBConfig - Eth *EthConfig -} - -// DBConfig is config parameters for DB. -type DBConfig struct { - Node ethNode.Info - URI string - ConnConfig postgres.ConnectionConfig + Eth *EthConfig + DB *DBConfig + File *FileConfig } // EthConfig is config parameters for the chain. type EthConfig struct { LevelDBPath string AncientDBPath string + NodeInfo ethNode.Info +} + +// DBConfig is config parameters for DB. +type DBConfig struct { + URI string + ConnConfig postgres.Config +} + +type FileConfig struct { + OutputDir string } func NewConfig() *Config { - ret := &Config{&DBConfig{}, &EthConfig{}} + ret := &Config{ + &EthConfig{}, + &DBConfig{}, + &FileConfig{}, + } ret.Init() return ret } // Init Initialises config func (c *Config) Init() { - c.DB.dbInit() viper.BindEnv("ethereum.nodeID", ETH_NODE_ID) viper.BindEnv("ethereum.clientName", ETH_CLIENT_NAME) viper.BindEnv("ethereum.genesisBlock", ETH_GENESIS_BLOCK) viper.BindEnv("ethereum.networkID", ETH_NETWORK_ID) - c.DB.Node = ethNode.Info{ + c.Eth.NodeInfo = ethNode.Info{ ID: viper.GetString("ethereum.nodeID"), ClientName: viper.GetString("ethereum.clientName"), GenesisBlock: viper.GetString("ethereum.genesisBlock"), @@ -76,29 +94,38 @@ func (c *Config) Init() { c.Eth.AncientDBPath = viper.GetString("leveldb.ancient") c.Eth.LevelDBPath = viper.GetString("leveldb.path") + + c.DB.Init() + c.File.Init() } -func (c *DBConfig) dbInit() { - viper.BindEnv("database.name", postgres.DATABASE_NAME) - viper.BindEnv("database.hostname", postgres.DATABASE_HOSTNAME) - viper.BindEnv("database.port", postgres.DATABASE_PORT) - viper.BindEnv("database.user", postgres.DATABASE_USER) - viper.BindEnv("database.password", postgres.DATABASE_PASSWORD) - viper.BindEnv("database.maxIdle", postgres.DATABASE_MAX_IDLE_CONNECTIONS) - viper.BindEnv("database.maxOpen", postgres.DATABASE_MAX_OPEN_CONNECTIONS) - viper.BindEnv("database.maxLifetime", postgres.DATABASE_MAX_CONN_LIFETIME) +func (c *DBConfig) Init() { + viper.BindEnv("database.name", "DATABASE_NAME") + viper.BindEnv("database.hostname", "DATABASE_HOSTNAME") + viper.BindEnv("database.port", "DATABASE_PORT") + viper.BindEnv("database.user", "DATABASE_USER") + viper.BindEnv("database.password", "DATABASE_PASSWORD") + viper.BindEnv("database.maxIdle", "DATABASE_MAX_IDLE_CONNECTIONS") + viper.BindEnv("database.maxOpen", "DATABASE_MAX_OPEN_CONNECTIONS") + viper.BindEnv("database.maxLifetime", "DATABASE_MAX_CONN_LIFETIME") - dbParams := postgres.ConnectionParams{} + dbParams := postgres.Config{} // DB params - dbParams.Name = viper.GetString("database.name") + dbParams.DatabaseName = viper.GetString("database.name") dbParams.Hostname = viper.GetString("database.hostname") dbParams.Port = viper.GetInt("database.port") - dbParams.User = viper.GetString("database.user") + dbParams.Username = viper.GetString("database.user") dbParams.Password = viper.GetString("database.password") - - c.URI = postgres.DbConnectionString(dbParams) // Connection config - c.ConnConfig.MaxIdle = viper.GetInt("database.maxIdle") - c.ConnConfig.MaxOpen = viper.GetInt("database.maxOpen") - c.ConnConfig.MaxLifetime = viper.GetInt("database.maxLifetime") + dbParams.MaxIdle = viper.GetInt("database.maxIdle") + dbParams.MaxConns = viper.GetInt("database.maxOpen") + dbParams.MaxConnLifetime = time.Duration(viper.GetInt("database.maxLifetime")) * time.Second + + c.ConnConfig = dbParams + c.URI = dbParams.DbConnectionString() +} + +func (c *FileConfig) Init() { + viper.BindEnv("file.outputDir", "FILE_OUTPUT_DIR") + c.OutputDir = viper.GetString("file.outputDir") } diff --git a/pkg/snapshot/file/publisher.go b/pkg/snapshot/file/publisher.go new file mode 100644 index 0000000..7b3d5c7 --- /dev/null +++ b/pkg/snapshot/file/publisher.go @@ -0,0 +1,298 @@ +// Copyright © 2020 Vulcanize, Inc +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package publisher + +import ( + // "bufio" + "encoding/csv" + "fmt" + "os" + "path/filepath" + "sync/atomic" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ipfs/go-cid" + blockstore "github.com/ipfs/go-ipfs-blockstore" + dshelp "github.com/ipfs/go-ipfs-ds-help" + "github.com/multiformats/go-multihash" + "github.com/sirupsen/logrus" + + "github.com/ethereum/go-ethereum/statediff/indexer/ipld" + nodeinfo "github.com/ethereum/go-ethereum/statediff/indexer/node" + "github.com/ethereum/go-ethereum/statediff/indexer/shared" + snapt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/types" +) + +var _ snapt.Publisher = (*publisher)(nil) + +var ( + // tables written once per block + perBlockTables = []*snapt.Table{ + &snapt.TableIPLDBlock, + &snapt.TableNodeInfo, + &snapt.TableHeader, + } + // tables written during state iteration + perNodeTables = []*snapt.Table{ + &snapt.TableIPLDBlock, + &snapt.TableStateNode, + &snapt.TableStorageNode, + } +) + +const logInterval = 1 * time.Minute + +type publisher struct { + dir string // dir containing output files + writers fileWriters + + nodeInfo nodeinfo.Info + + startTime time.Time + currBatchSize uint + stateNodeCounter uint64 + storageNodeCounter uint64 + codeNodeCounter uint64 + txCounter uint32 +} + +type fileWriter struct { + *csv.Writer +} + +// fileWriters wraps the file writers for each output table +type fileWriters map[*snapt.Table]fileWriter + +type fileTx struct { + fileWriters + // index uint +} + +func (tx fileWriters) Commit() error { + for _, w := range tx { + w.Flush() + if err := w.Error(); err != nil { + return err + } + } + return nil +} +func (fileWriters) Rollback() error { return nil } // TODO: delete the file? + +func newFileWriter(path string) (ret fileWriter, err error) { + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) + if err != nil { + return + } + buf := file //bufio.NewWriter(file) + ret = fileWriter{csv.NewWriter(buf)} + return +} + +func (tx fileWriters) write(tbl *snapt.Table, args ...interface{}) error { + row := tbl.ToCsvRow(args...) + return tx[tbl].Write(row) +} + +func makeFileWriters(dir string, tables []*snapt.Table) (fileWriters, error) { + if err := os.MkdirAll(dir, 0755); err != nil { + return nil, err + } + writers := map[*snapt.Table]fileWriter{} + for _, tbl := range tables { + w, err := newFileWriter(TableFile(dir, tbl.Name)) + if err != nil { + return nil, err + } + writers[tbl] = w + } + return writers, nil +} + +// NewPublisher creates a publisher which writes to per-table CSV files which can be imported +// with the Postgres COPY command. +// The output directory will be created if it does not exist. +func NewPublisher(path string, node nodeinfo.Info) (*publisher, error) { + if err := os.MkdirAll(path, 0755); err != nil { + return nil, err + } + writers, err := makeFileWriters(path, perBlockTables) + if err != nil { + return nil, err + } + pub := &publisher{ + writers: writers, + dir: path, + nodeInfo: node, + startTime: time.Now(), + } + go pub.logNodeCounters() + return pub, nil +} + +func TableFile(dir, name string) string { return filepath.Join(dir, name+".csv") } + +func (p *publisher) txDir(index uint32) string { + return filepath.Join(p.dir, fmt.Sprintf("%010d", index)) +} + +func (p *publisher) BeginTx() (snapt.Tx, error) { + index := atomic.LoadUint32(&p.txCounter) + dir := p.txDir(index) + writers, err := makeFileWriters(dir, perNodeTables) + if err != nil { + return nil, err + } + + atomic.AddUint32(&p.txCounter, 1) + return fileTx{writers}, nil +} + +// PublishRaw derives a cid from raw bytes and provided codec and multihash type, and writes it to the db tx +// returns the CID and blockstore prefixed multihash key +func (tx fileWriters) publishRaw(codec uint64, raw []byte) (cid, prefixedKey string, err error) { + c, err := ipld.RawdataToCid(codec, raw, multihash.KECCAK_256) + if err != nil { + return + } + cid = c.String() + prefixedKey, err = tx.publishIPLD(c, raw) + return +} + +func (tx fileWriters) publishIPLD(c cid.Cid, raw []byte) (string, error) { + dbKey := dshelp.MultihashToDsKey(c.Hash()) + prefixedKey := blockstore.BlockPrefix.String() + dbKey.String() + return prefixedKey, tx.write(&snapt.TableIPLDBlock, prefixedKey, raw) +} + +// PublishHeader writes the header to the ipfs backing pg datastore and adds secondary +// indexes in the header_cids table +func (p *publisher) PublishHeader(header *types.Header) error { + headerNode, err := ipld.NewEthHeader(header) + if err != nil { + return err + } + if _, err = p.writers.publishIPLD(headerNode.Cid(), headerNode.RawData()); err != nil { + return err + } + + mhKey := shared.MultihashKeyFromCID(headerNode.Cid()) + err = p.writers.write(&snapt.TableNodeInfo, p.nodeInfo.GenesisBlock, p.nodeInfo.NetworkID, p.nodeInfo.ID, + p.nodeInfo.ClientName, p.nodeInfo.ChainID) + if err != nil { + return err + } + err = p.writers.write(&snapt.TableHeader, header.Number.String(), header.Hash().Hex(), header.ParentHash.Hex(), + headerNode.Cid().String(), 0, p.nodeInfo.ID, 0, header.Root.Hex(), header.TxHash.Hex(), + header.ReceiptHash.Hex(), header.UncleHash.Hex(), header.Bloom.Bytes(), header.Time, mhKey, + 0, header.Coinbase.String()) + if err != nil { + return err + } + return p.writers.Commit() +} + +// PublishStateNode writes the state node to the ipfs backing datastore and adds secondary indexes +// in the state_cids table +func (p *publisher) PublishStateNode(node *snapt.Node, headerID string, tx_ snapt.Tx) error { + var stateKey string + if !snapt.IsNullHash(node.Key) { + stateKey = node.Key.Hex() + } + + tx := tx_.(fileTx) + stateCIDStr, mhKey, err := tx.publishRaw(ipld.MEthStateTrie, node.Value) + if err != nil { + return err + } + + err = tx.write(&snapt.TableStateNode, headerID, stateKey, stateCIDStr, node.Path, + node.NodeType, false, mhKey) + if err != nil { + return err + } + // increment state node counter. + atomic.AddUint64(&p.stateNodeCounter, 1) + // increment current batch size counter + p.currBatchSize += 2 + return err +} + +// PublishStorageNode writes the storage node to the ipfs backing pg datastore and adds secondary +// indexes in the storage_cids table +func (p *publisher) PublishStorageNode(node *snapt.Node, headerID string, statePath []byte, tx_ snapt.Tx) error { + var storageKey string + if !snapt.IsNullHash(node.Key) { + storageKey = node.Key.Hex() + } + + tx := tx_.(fileTx) + storageCIDStr, mhKey, err := tx.publishRaw(ipld.MEthStorageTrie, node.Value) + if err != nil { + return err + } + + err = tx.write(&snapt.TableStorageNode, headerID, statePath, storageKey, storageCIDStr, node.Path, + node.NodeType, false, mhKey) + if err != nil { + return err + } + // increment storage node counter. + atomic.AddUint64(&p.storageNodeCounter, 1) + // increment current batch size counter + p.currBatchSize += 2 + return nil +} + +// PublishCode writes code to the ipfs backing pg datastore +func (p *publisher) PublishCode(codeHash common.Hash, codeBytes []byte, tx_ snapt.Tx) error { + // no codec for code, doesn't matter though since blockstore key is multihash-derived + mhKey, err := shared.MultihashKeyFromKeccak256(codeHash) + if err != nil { + return fmt.Errorf("error deriving multihash key from codehash: %v", err) + } + + tx := tx_.(fileTx) + if err = tx.write(&snapt.TableIPLDBlock, mhKey, codeBytes); err != nil { + return fmt.Errorf("error publishing code IPLD: %v", err) + } + // increment code node counter. + atomic.AddUint64(&p.codeNodeCounter, 1) + p.currBatchSize++ + return nil +} + +func (p *publisher) PrepareTxForBatch(tx snapt.Tx, maxBatchSize uint) (snapt.Tx, error) { + return tx, nil +} + +// logNodeCounters periodically logs the number of node processed. +func (p *publisher) logNodeCounters() { + t := time.NewTicker(logInterval) + for range t.C { + p.printNodeCounters() + } +} + +func (p *publisher) printNodeCounters() { + logrus.Infof("runtime: %s", time.Now().Sub(p.startTime).String()) + logrus.Infof("processed state nodes: %d", atomic.LoadUint64(&p.stateNodeCounter)) + logrus.Infof("processed storage nodes: %d", atomic.LoadUint64(&p.storageNodeCounter)) + logrus.Infof("processed code nodes: %d", atomic.LoadUint64(&p.codeNodeCounter)) +} diff --git a/pkg/snapshot/file/publisher_test.go b/pkg/snapshot/file/publisher_test.go new file mode 100644 index 0000000..9505373 --- /dev/null +++ b/pkg/snapshot/file/publisher_test.go @@ -0,0 +1,136 @@ +package publisher + +import ( + "context" + "encoding/csv" + "fmt" + "io" + "os" + "path/filepath" + "testing" + + "github.com/ethereum/go-ethereum/statediff/indexer/ipld" + "github.com/jackc/pgx/v4" + + fixt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/fixture" + snapt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/types" + "github.com/vulcanize/eth-pg-ipfs-state-snapshot/test" +) + +var ( + pgConfig = test.DefaultPgConfig + nodeInfo = test.DefaultNodeInfo + // tables ordered according to fkey depedencies + allTables = []*snapt.Table{ + &snapt.TableIPLDBlock, + &snapt.TableNodeInfo, + &snapt.TableHeader, + &snapt.TableStateNode, + &snapt.TableStorageNode, + } +) + +func writeFiles(t *testing.T, dir string) *publisher { + pub, err := NewPublisher(dir, nodeInfo) + test.NoError(t, err) + test.NoError(t, pub.PublishHeader(&fixt.Header1)) + tx, err := pub.BeginTx() + test.NoError(t, err) + + headerID := fixt.Header1.Hash().String() + test.NoError(t, pub.PublishStateNode(&fixt.StateNode1, headerID, tx)) + + test.NoError(t, tx.Commit()) + return pub +} + +// verify that we can parse the csvs +// TODO check actual data +func verifyFileData(t *testing.T, path string, tbl *snapt.Table) { + file, err := os.Open(path) + test.NoError(t, err) + r := csv.NewReader(file) + test.NoError(t, err) + r.FieldsPerRecord = len(tbl.Columns) + + for { + _, err := r.Read() + if err == io.EOF { + break + } + test.NoError(t, err) + // _, err = tbl.fromCsvRow(row) + // test.NoError(t, err) + } +} + +func TestWriting(t *testing.T) { + dir := t.TempDir() + // tempdir like /tmp/TempFoo/001/, TempFoo defaults to 0700 + test.NoError(t, os.Chmod(filepath.Dir(dir), 0755)) + // dir, _ := os.Getwd() + // dir += "/dump" + + pub := writeFiles(t, dir) + + for _, tbl := range perBlockTables { + verifyFileData(t, TableFile(pub.dir, tbl.Name), tbl) + } + for i := uint32(0); i < pub.txCounter; i++ { + for _, tbl := range perNodeTables { + verifyFileData(t, TableFile(pub.txDir(i), tbl.Name), tbl) + } + } +} + +// Note: DB user requires role membership "pg_read_server_files" +func TestPgCopy(t *testing.T) { + test.NeedsDB(t) + + dir := t.TempDir() + test.NoError(t, os.Chmod(filepath.Dir(dir), 0755)) + pub := writeFiles(t, dir) + + ctx := context.Background() + conn, err := pgx.Connect(ctx, pgConfig.DbConnectionString()) + test.NoError(t, err) + + // clear existing test data + pgDeleteTable := `DELETE FROM %s` + for _, tbl := range allTables { + _, err = conn.Exec(ctx, fmt.Sprintf(pgDeleteTable, tbl.Name)) + test.NoError(t, err) + } + + // copy from files + pgCopyStatement := `COPY %s FROM '%s' CSV` + for _, tbl := range perBlockTables { + stm := fmt.Sprintf(pgCopyStatement, tbl.Name, TableFile(pub.dir, tbl.Name)) + _, err = conn.Exec(ctx, stm) + test.NoError(t, err) + } + for i := uint32(0); i < pub.txCounter; i++ { + for _, tbl := range perNodeTables { + stm := fmt.Sprintf(pgCopyStatement, tbl.Name, TableFile(pub.txDir(i), tbl.Name)) + _, err = conn.Exec(ctx, stm) + test.NoError(t, err) + } + } + + // check header was successfully committed + pgQueryHeader := `SELECT cid, block_hash + FROM eth.header_cids + WHERE block_number = $1` + type res struct { + CID string + BlockHash string + } + var header res + err = conn.QueryRow(ctx, pgQueryHeader, fixt.Header1.Number.Uint64()).Scan( + &header.CID, &header.BlockHash) + test.NoError(t, err) + + headerNode, err := ipld.NewEthHeader(&fixt.Header1) + test.ExpectEqual(t, headerNode.Cid().String(), header.CID) + test.ExpectEqual(t, fixt.Header1.Hash().String(), header.BlockHash) +} diff --git a/pkg/snapshot/mock/publisher.go b/pkg/snapshot/mock/publisher.go deleted file mode 100644 index bd2bd56..0000000 --- a/pkg/snapshot/mock/publisher.go +++ /dev/null @@ -1,64 +0,0 @@ -package mock - -import ( - "fmt" - "strings" - "testing" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - "github.com/jmoiron/sqlx" - - "github.com/golang/mock/gomock" - - mocks "github.com/vulcanize/eth-pg-ipfs-state-snapshot/mocks/snapshot" - snapt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/types" -) - -type MockPublisher struct { - *mocks.MockPublisher -} - -func NewMockPublisher(t *testing.T) *MockPublisher { - ctl := gomock.NewController(t) - return &MockPublisher{mocks.NewMockPublisher(ctl)} -} - -func dump(funcname string, xs ...interface{}) { - if true { - return - } - fmt.Printf(">> %s", funcname) - fmt.Printf(strings.Repeat(" %+v", len(xs))+"\n", xs...) -} - -func (p *MockPublisher) PublishHeader(header *types.Header) (int64, error) { - // fmt.Printf("PublishHeader %+v\n", header) - dump("PublishHeader", header) - return p.MockPublisher.PublishHeader(header) -} -func (p *MockPublisher) PublishStateNode(node *snapt.Node, headerID int64, tx *sqlx.Tx) (int64, error) { - dump("PublishStateNode", node, headerID) - return p.MockPublisher.PublishStateNode(node, headerID, tx) - -} -func (p *MockPublisher) PublishStorageNode(node *snapt.Node, stateID int64, tx *sqlx.Tx) error { - dump("PublishStorageNode", node, stateID) - return p.MockPublisher.PublishStorageNode(node, stateID, tx) -} -func (p *MockPublisher) PublishCode(codeHash common.Hash, codeBytes []byte, tx *sqlx.Tx) error { - dump("PublishCode", codeHash, codeBytes) - return p.MockPublisher.PublishCode(codeHash, codeBytes, tx) -} -func (p *MockPublisher) BeginTx() (*sqlx.Tx, error) { - dump("BeginTx") - return p.MockPublisher.BeginTx() -} -func (p *MockPublisher) CommitTx(tx *sqlx.Tx) error { - dump("CommitTx", tx) - return p.MockPublisher.CommitTx(tx) -} -func (p *MockPublisher) PrepareTxForBatch(tx *sqlx.Tx, batchSize uint) (*sqlx.Tx, error) { - dump("PrepareTxForBatch", tx, batchSize) - return p.MockPublisher.PrepareTxForBatch(tx, batchSize) -} diff --git a/pkg/snapshot/publisher.go b/pkg/snapshot/pg/publisher.go similarity index 53% rename from pkg/snapshot/publisher.go rename to pkg/snapshot/pg/publisher.go index 5cd2e42..54958e7 100644 --- a/pkg/snapshot/publisher.go +++ b/pkg/snapshot/pg/publisher.go @@ -13,26 +13,31 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -package snapshot +package pg import ( - "bytes" + "context" "fmt" "sync/atomic" "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" - "github.com/jmoiron/sqlx" + "github.com/ipfs/go-cid" + blockstore "github.com/ipfs/go-ipfs-blockstore" + dshelp "github.com/ipfs/go-ipfs-ds-help" "github.com/multiformats/go-multihash" "github.com/sirupsen/logrus" - "github.com/ethereum/go-ethereum/statediff/indexer/ipfs/ipld" - "github.com/ethereum/go-ethereum/statediff/indexer/postgres" + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql" + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" + "github.com/ethereum/go-ethereum/statediff/indexer/ipld" "github.com/ethereum/go-ethereum/statediff/indexer/shared" snapt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/types" ) +var _ snapt.Publisher = (*publisher)(nil) + const logInterval = 1 * time.Minute // Publisher is wrapper around DB. @@ -48,128 +53,146 @@ type publisher struct { // NewPublisher creates Publisher func NewPublisher(db *postgres.DB) *publisher { return &publisher{ - db: db, - currBatchSize: 0, - startTime: time.Now(), + db: db, + startTime: time.Now(), } } -func (p *publisher) BeginTx() (*sqlx.Tx, error) { - tx, err := p.db.Beginx() +type pubTx struct { + sql.Tx + callback func() +} + +func (tx pubTx) Rollback() error { return tx.Tx.Rollback(context.Background()) } +func (tx pubTx) Commit() error { + if tx.callback != nil { + defer tx.callback() + } + return tx.Tx.Commit(context.Background()) +} +func (tx pubTx) Exec(sql string, args ...interface{}) (sql.Result, error) { + return tx.Tx.Exec(context.Background(), sql, args...) +} + +func (p *publisher) BeginTx() (snapt.Tx, error) { + tx, err := p.db.Begin(context.Background()) if err != nil { return nil, err } go p.logNodeCounters() - return tx, nil + return pubTx{tx, func() { + logrus.Info("----- final counts -----") + p.printNodeCounters() + }}, nil } -func (p *publisher) CommitTx(tx *sqlx.Tx) error { - logrus.Info("----- final counts -----") - p.printNodeCounters() - return tx.Commit() +// PublishRaw derives a cid from raw bytes and provided codec and multihash type, and writes it to the db tx +// returns the CID and blockstore prefixed multihash key +func (tx pubTx) publishRaw(codec uint64, raw []byte) (cid, prefixedKey string, err error) { + c, err := ipld.RawdataToCid(codec, raw, multihash.KECCAK_256) + if err != nil { + return + } + cid = c.String() + prefixedKey, err = tx.publishIPLD(c, raw) + return +} + +func (tx pubTx) publishIPLD(c cid.Cid, raw []byte) (string, error) { + dbKey := dshelp.MultihashToDsKey(c.Hash()) + prefixedKey := blockstore.BlockPrefix.String() + dbKey.String() + _, err := tx.Exec(snapt.TableIPLDBlock.ToInsertStatement(), prefixedKey, raw) + return prefixedKey, err } // PublishHeader writes the header to the ipfs backing pg datastore and adds secondary indexes in the header_cids table -func (p *publisher) PublishHeader(header *types.Header) (int64, error) { +func (p *publisher) PublishHeader(header *types.Header) (err error) { headerNode, err := ipld.NewEthHeader(header) if err != nil { - return 0, err + return err } - tx, err := p.db.Beginx() + tx_, err := p.db.Begin(context.Background()) if err != nil { - return 0, err + return err + } + tx := pubTx{tx_, nil} + defer func() { err = snapt.CommitOrRollback(tx, err) }() + + if _, err = tx.publishIPLD(headerNode.Cid(), headerNode.RawData()); err != nil { + return err } - defer func() { - if p := recover(); p != nil { - shared.Rollback(tx) - panic(p) - } else if err != nil { - shared.Rollback(tx) - } else { - err = tx.Commit() - } - }() - - if err = shared.PublishIPLD(tx, headerNode); err != nil { - return 0, err - } - - mhKey, _ := shared.MultihashKeyFromCIDString(headerNode.Cid().String()) - var headerID int64 - err = tx.QueryRowx(`INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) - ON CONFLICT (block_number, block_hash) DO UPDATE SET block_number = header_cids.block_number - RETURNING id`, - header.Number.Uint64(), header.Hash().Hex(), header.ParentHash.Hex(), headerNode.Cid().String(), "0", p.db.NodeID, "0", header.Root.Hex(), header.TxHash.Hex(), - header.ReceiptHash.Hex(), header.UncleHash.Hex(), header.Bloom.Bytes(), header.Time, mhKey, 0).Scan(&headerID) - - return headerID, err + mhKey := shared.MultihashKeyFromCID(headerNode.Cid()) + _, err = tx.Exec(snapt.TableHeader.ToInsertStatement(), header.Number.Uint64(), header.Hash().Hex(), + header.ParentHash.Hex(), headerNode.Cid().String(), "0", p.db.NodeID(), "0", + header.Root.Hex(), header.TxHash.Hex(), header.ReceiptHash.Hex(), header.UncleHash.Hex(), + header.Bloom.Bytes(), header.Time, mhKey, 0, header.Coinbase.String()) + return err } // PublishStateNode writes the state node to the ipfs backing datastore and adds secondary indexes in the state_cids table -func (p *publisher) PublishStateNode(node *snapt.Node, headerID int64, tx *sqlx.Tx) (int64, error) { - var stateID int64 +func (p *publisher) PublishStateNode(node *snapt.Node, headerID string, tx_ snapt.Tx) error { var stateKey string - if !bytes.Equal(node.Key.Bytes(), nullHash.Bytes()) { + if !snapt.IsNullHash(node.Key) { stateKey = node.Key.Hex() } - stateCIDStr, mhKey, err := shared.PublishRaw(tx, ipld.MEthStateTrie, multihash.KECCAK_256, node.Value) + tx := tx_.(pubTx) + stateCIDStr, mhKey, err := tx.publishRaw(ipld.MEthStateTrie, node.Value) if err != nil { - return 0, err + return err } - err = tx.QueryRowx(`INSERT INTO eth.state_cids (header_id, state_leaf_key, cid, state_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7) - ON CONFLICT (header_id, state_path) DO UPDATE SET (state_leaf_key, cid, node_type, diff, mh_key) = ($2, $3, $5, $6, $7) - RETURNING id`, - headerID, stateKey, stateCIDStr, node.Path, node.NodeType, false, mhKey).Scan(&stateID) - + _, err = tx.Exec(snapt.TableStateNode.ToInsertStatement(), + headerID, stateKey, stateCIDStr, node.Path, node.NodeType, false, mhKey) + if err != nil { + return err + } // increment state node counter. atomic.AddUint64(&p.stateNodeCounter, 1) // increment current batch size counter p.currBatchSize += 2 - return stateID, err + return err } // PublishStorageNode writes the storage node to the ipfs backing pg datastore and adds secondary indexes in the storage_cids table -func (p *publisher) PublishStorageNode(node *snapt.Node, stateID int64, tx *sqlx.Tx) error { +func (p *publisher) PublishStorageNode(node *snapt.Node, headerID string, statePath []byte, tx_ snapt.Tx) error { var storageKey string - if !bytes.Equal(node.Key.Bytes(), nullHash.Bytes()) { + if !snapt.IsNullHash(node.Key) { storageKey = node.Key.Hex() } - storageCIDStr, mhKey, err := shared.PublishRaw(tx, ipld.MEthStorageTrie, multihash.KECCAK_256, node.Value) + tx := tx_.(pubTx) + storageCIDStr, mhKey, err := tx.publishRaw(ipld.MEthStorageTrie, node.Value) if err != nil { return err } - _, err = tx.Exec(`INSERT INTO eth.storage_cids (state_id, storage_leaf_key, cid, storage_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7) - ON CONFLICT (state_id, storage_path) DO UPDATE SET (storage_leaf_key, cid, node_type, diff, mh_key) = ($2, $3, $5, $6, $7)`, - stateID, storageKey, storageCIDStr, node.Path, node.NodeType, false, mhKey) + _, err = tx.Exec(snapt.TableStorageNode.ToInsertStatement(), + headerID, statePath, storageKey, storageCIDStr, node.Path, node.NodeType, false, mhKey) if err != nil { return err } - // increment storage node counter. atomic.AddUint64(&p.storageNodeCounter, 1) // increment current batch size counter p.currBatchSize += 2 - return nil + return err } // PublishCode writes code to the ipfs backing pg datastore -func (p *publisher) PublishCode(codeHash common.Hash, codeBytes []byte, tx *sqlx.Tx) error { +func (p *publisher) PublishCode(codeHash common.Hash, codeBytes []byte, tx_ snapt.Tx) error { // no codec for code, doesn't matter though since blockstore key is multihash-derived mhKey, err := shared.MultihashKeyFromKeccak256(codeHash) if err != nil { return fmt.Errorf("error deriving multihash key from codehash: %v", err) } - if err = shared.PublishDirect(tx, mhKey, codeBytes); err != nil { + tx := tx_.(pubTx) + if _, err = tx.Exec(snapt.TableIPLDBlock.ToInsertStatement(), mhKey, codeBytes); err != nil { return fmt.Errorf("error publishing code IPLD: %v", err) } @@ -180,7 +203,7 @@ func (p *publisher) PublishCode(codeHash common.Hash, codeBytes []byte, tx *sqlx return nil } -func (p *publisher) PrepareTxForBatch(tx *sqlx.Tx, maxBatchSize uint) (*sqlx.Tx, error) { +func (p *publisher) PrepareTxForBatch(tx snapt.Tx, maxBatchSize uint) (snapt.Tx, error) { var err error // maximum batch size reached, commit the current transaction and begin a new transaction. if maxBatchSize <= p.currBatchSize { @@ -188,7 +211,8 @@ func (p *publisher) PrepareTxForBatch(tx *sqlx.Tx, maxBatchSize uint) (*sqlx.Tx, return nil, err } - tx, err = p.db.Beginx() + tx_, err := p.db.Begin(context.Background()) + tx = pubTx{Tx: tx_} if err != nil { return nil, err } diff --git a/pkg/snapshot/pg/publisher_test.go b/pkg/snapshot/pg/publisher_test.go new file mode 100644 index 0000000..c4860aa --- /dev/null +++ b/pkg/snapshot/pg/publisher_test.go @@ -0,0 +1,78 @@ +package pg + +import ( + "context" + "fmt" + "testing" + + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" + "github.com/ethereum/go-ethereum/statediff/indexer/ipld" + "github.com/jackc/pgx/v4" + + fixt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/fixture" + snapt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/types" + "github.com/vulcanize/eth-pg-ipfs-state-snapshot/test" +) + +var ( + pgConfig = test.DefaultPgConfig + nodeInfo = test.DefaultNodeInfo + // tables ordered according to fkey depedencies + allTables = []*snapt.Table{ + &snapt.TableIPLDBlock, + &snapt.TableNodeInfo, + &snapt.TableHeader, + &snapt.TableStateNode, + &snapt.TableStorageNode, + } +) + +func writeData(t *testing.T) *publisher { + driver, err := postgres.NewPGXDriver(context.Background(), pgConfig, nodeInfo) + test.NoError(t, err) + pub := NewPublisher(postgres.NewPostgresDB(driver)) + test.NoError(t, pub.PublishHeader(&fixt.Header1)) + tx, err := pub.BeginTx() + test.NoError(t, err) + + headerID := fixt.Header1.Hash().String() + test.NoError(t, pub.PublishStateNode(&fixt.StateNode1, headerID, tx)) + + test.NoError(t, tx.Commit()) + return pub +} + +// Note: DB user requires role membership "pg_read_server_files" +func TestBasic(t *testing.T) { + test.NeedsDB(t) + + ctx := context.Background() + conn, err := pgx.Connect(ctx, pgConfig.DbConnectionString()) + test.NoError(t, err) + + // clear existing test data + pgDeleteTable := `DELETE FROM %s` + for _, tbl := range allTables { + _, err = conn.Exec(ctx, fmt.Sprintf(pgDeleteTable, tbl.Name)) + test.NoError(t, err) + } + + _ = writeData(t) + + // check header was successfully committed + pgQueryHeader := `SELECT cid, block_hash + FROM eth.header_cids + WHERE block_number = $1` + type res struct { + CID string + BlockHash string + } + var header res + err = conn.QueryRow(ctx, pgQueryHeader, fixt.Header1.Number.Uint64()).Scan( + &header.CID, &header.BlockHash) + test.NoError(t, err) + + headerNode, err := ipld.NewEthHeader(&fixt.Header1) + test.ExpectEqual(t, headerNode.Cid().String(), header.CID) + test.ExpectEqual(t, fixt.Header1.Hash().String(), header.BlockHash) +} diff --git a/pkg/snapshot/service.go b/pkg/snapshot/service.go index 3838438..4146032 100644 --- a/pkg/snapshot/service.go +++ b/pkg/snapshot/service.go @@ -28,10 +28,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/statediff/indexer/postgres" - "github.com/ethereum/go-ethereum/statediff/indexer/shared" "github.com/ethereum/go-ethereum/trie" - "github.com/jmoiron/sqlx" "github.com/sirupsen/logrus" . "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/types" @@ -39,7 +36,6 @@ import ( ) var ( - nullHash = common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000") emptyNode, _ = rlp.EncodeToBytes([]byte{}) emptyCodeHash = crypto.Keccak256([]byte{}) emptyContractRoot = crypto.Keccak256Hash(emptyNode) @@ -56,14 +52,6 @@ type Service struct { maxBatchSize uint } -func NewPostgresDB(con *DBConfig) (*postgres.DB, error) { - pgDB, err := postgres.NewDB(con.URI, con.ConnConfig, con.Node) - if err != nil { - return nil, err - } - return pgDB, nil -} - func NewLevelDB(con *EthConfig) (ethdb.Database, error) { return rawdb.NewLevelDBDatabaseWithFreezer( con.LevelDBPath, 1024, 256, con.AncientDBPath, "eth-pg-ipfs-state-snapshot", false, @@ -97,7 +85,7 @@ func (s *Service) CreateSnapshot(params SnapshotParams) error { logrus.Infof("head hash: %s head height: %d", hash.Hex(), params.Height) - headerID, err := s.ipfsPublisher.PublishHeader(header) + err := s.ipfsPublisher.PublishHeader(header) if err != nil { return err } @@ -106,6 +94,8 @@ func (s *Service) CreateSnapshot(params SnapshotParams) error { if err != nil { return err } + headerID := header.Hash().String() + if params.Workers > 0 { return s.createSnapshotAsync(t, headerID, params.Workers) } else { @@ -135,7 +125,7 @@ func resolveNode(it trie.NodeIterator, trieDB *trie.Database) (*nodeResult, erro if it.Leaf() { return nil, nil } - if bytes.Equal(nullHash.Bytes(), it.Hash().Bytes()) { + if IsNullHash(it.Hash()) { return nil, nil } @@ -163,22 +153,12 @@ func resolveNode(it trie.NodeIterator, trieDB *trie.Database) (*nodeResult, erro }, nil } -func (s *Service) createSnapshot(it trie.NodeIterator, headerID int64) error { +func (s *Service) createSnapshot(it trie.NodeIterator, headerID string) error { tx, err := s.ipfsPublisher.BeginTx() if err != nil { return err } - - defer func() { - if rec := recover(); rec != nil { - shared.Rollback(tx) - panic(rec) - } else if err != nil { - shared.Rollback(tx) - } else { - err = s.ipfsPublisher.CommitTx(tx) - } - }() + defer func() { err = CommitOrRollback(tx, err) }() for it.Next(true) { res, err := resolveNode(it, s.stateDB.TrieDB()) @@ -207,7 +187,7 @@ func (s *Service) createSnapshot(it trie.NodeIterator, headerID int64) error { encodedPath := trie.HexToCompact(valueNodePath) leafKey := encodedPath[1:] res.node.Key = common.BytesToHash(leafKey) - stateID, err := s.ipfsPublisher.PublishStateNode(&res.node, headerID, tx) + err := s.ipfsPublisher.PublishStateNode(&res.node, headerID, tx) if err != nil { return err } @@ -226,12 +206,12 @@ func (s *Service) createSnapshot(it trie.NodeIterator, headerID int64) error { } } - if tx, err = s.storageSnapshot(account.Root, stateID, tx); err != nil { + if tx, err = s.storageSnapshot(account.Root, headerID, res.node.Path, tx); err != nil { return fmt.Errorf("failed building storage snapshot for account %+v\r\nerror: %w", account, err) } case Extension, Branch: res.node.Key = common.BytesToHash([]byte{}) - if _, err := s.ipfsPublisher.PublishStateNode(&res.node, headerID, tx); err != nil { + if err := s.ipfsPublisher.PublishStateNode(&res.node, headerID, tx); err != nil { return err } default: @@ -243,7 +223,7 @@ func (s *Service) createSnapshot(it trie.NodeIterator, headerID int64) error { } // Full-trie concurrent snapshot -func (s *Service) createSnapshotAsync(tree state.Trie, headerID int64, workers uint) error { +func (s *Service) createSnapshotAsync(tree state.Trie, headerID string, workers uint) error { errors := make(chan error) var wg sync.WaitGroup for _, it := range iter.SubtrieIterators(tree, workers) { @@ -267,7 +247,7 @@ func (s *Service) createSnapshotAsync(tree state.Trie, headerID int64, workers u return nil } -func (s *Service) storageSnapshot(sr common.Hash, stateID int64, tx *sqlx.Tx) (*sqlx.Tx, error) { +func (s *Service) storageSnapshot(sr common.Hash, headerID string, statePath []byte, tx Tx) (Tx, error) { if bytes.Equal(sr.Bytes(), emptyContractRoot.Bytes()) { return tx, nil } @@ -311,7 +291,7 @@ func (s *Service) storageSnapshot(sr common.Hash, stateID int64, tx *sqlx.Tx) (* default: return nil, errors.New("unexpected node type") } - if err = s.ipfsPublisher.PublishStorageNode(&res.node, stateID, tx); err != nil { + if err = s.ipfsPublisher.PublishStorageNode(&res.node, headerID, statePath, tx); err != nil { return nil, err } } diff --git a/pkg/snapshot/service_test.go b/pkg/snapshot/service_test.go index 8fbac83..bc5b81b 100644 --- a/pkg/snapshot/service_test.go +++ b/pkg/snapshot/service_test.go @@ -1,84 +1,57 @@ package snapshot import ( - "fmt" - "os" - "path/filepath" "testing" "github.com/golang/mock/gomock" - ethNode "github.com/ethereum/go-ethereum/statediff/indexer/node" - "github.com/ethereum/go-ethereum/statediff/indexer/postgres" - fixt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/fixture" - "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/snapshot/mock" + mock "github.com/vulcanize/eth-pg-ipfs-state-snapshot/mocks/snapshot" + "github.com/vulcanize/eth-pg-ipfs-state-snapshot/test" ) func testConfig(leveldbpath, ancientdbpath string) *Config { - dbParams := postgres.ConnectionParams{ - Name: "snapshot_test", - Hostname: "localhost", - Port: 5432, - User: "tester", - Password: "test_pw", - } - connconfig := postgres.ConnectionConfig{ - MaxIdle: 0, - MaxLifetime: 0, - MaxOpen: 4, - } - nodeinfo := ethNode.Info{ - ID: "eth_node_id", - ClientName: "eth_client", - GenesisBlock: "X", - NetworkID: "eth_network", - ChainID: 0, - } - return &Config{ - DB: &DBConfig{ - Node: nodeinfo, - URI: postgres.DbConnectionString(dbParams), - ConnConfig: connconfig, - }, Eth: &EthConfig{ LevelDBPath: leveldbpath, AncientDBPath: ancientdbpath, + NodeInfo: test.DefaultNodeInfo, + }, + DB: &DBConfig{ + URI: test.DefaultPgConfig.DbConnectionString(), + ConnConfig: test.DefaultPgConfig, }, } } func TestCreateSnapshot(t *testing.T) { - wd, err := os.Getwd() - if err != nil { - panic(err) - } - fixture_path := filepath.Join(wd, "..", "..", "fixture") - datadir := filepath.Join(fixture_path, "chaindata") - if _, err := os.Stat(datadir); err != nil { - t.Fatal("no chaindata:", err) - } - config := testConfig(datadir, filepath.Join(datadir, "ancient")) - fmt.Printf("config: %+v %+v\n", config.DB, config.Eth) + config := testConfig(fixt.ChaindataPath, fixt.AncientdataPath) edb, err := NewLevelDB(config.Eth) if err != nil { t.Fatal(err) } - workers := 8 + workers := 4 - pub := mock.NewMockPublisher(t) - pub.EXPECT().PublishHeader(gomock.Eq(fixt.PublishHeader)) + ctl := gomock.NewController(t) + tx := mock.NewMockTx(ctl) + pub := mock.NewMockPublisher(ctl) + + pub.EXPECT().PublishHeader(gomock.Eq(&fixt.Header1)) pub.EXPECT().BeginTx(). + Return(tx, nil). Times(workers) pub.EXPECT().PrepareTxForBatch(gomock.Any(), gomock.Any()). + Return(tx, nil). Times(workers) - pub.EXPECT().PublishStateNode(gomock.Any(), mock.AnyOf(int64(0), int64(1)), gomock.Any()). + pub.EXPECT().PublishStateNode(gomock.Any(), gomock.Any(), gomock.Any()). Times(workers) // TODO: fixtures for storage node // pub.EXPECT().PublishStorageNode(gomock.Eq(fixt.StorageNode), gomock.Eq(int64(0)), gomock.Any()) - pub.EXPECT().CommitTx(gomock.Any()). + // pub.EXPECT().CommitTx(gomock.Any()). + // Times(workers) + + tx.EXPECT().Commit(). Times(workers) service, err := NewSnapshotService(edb, pub) @@ -94,6 +67,6 @@ func TestCreateSnapshot(t *testing.T) { // err = service.CreateLatestSnapshot(0) // if err != nil { - // t.Fatal(err) + // t.Fatal(err) // } } diff --git a/pkg/snapshot/util.go b/pkg/snapshot/util.go new file mode 100644 index 0000000..a05d0d9 --- /dev/null +++ b/pkg/snapshot/util.go @@ -0,0 +1,26 @@ +package snapshot + +import ( + "context" + "fmt" + + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" + + file "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/snapshot/file" + pg "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/snapshot/pg" + snapt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/types" +) + +func NewPublisher(mode SnapshotMode, config *Config) (snapt.Publisher, error) { + switch mode { + case PgSnapshot: + driver, err := postgres.NewPGXDriver(context.Background(), config.DB.ConnConfig, config.Eth.NodeInfo) + if err != nil { + return nil, err + } + return pg.NewPublisher(postgres.NewPostgresDB(driver)), nil + case FileSnapshot: + return file.NewPublisher(config.File.OutputDir, config.Eth.NodeInfo) + } + return nil, fmt.Errorf("invalid snapshot mode: %s", mode) +} diff --git a/pkg/types/node_type.go b/pkg/types/node_type.go index f33dcd0..ce63835 100644 --- a/pkg/types/node_type.go +++ b/pkg/types/node_type.go @@ -22,13 +22,12 @@ import ( ) // node for holding trie node information -type node struct { +type Node struct { NodeType nodeType Path []byte Key common.Hash Value []byte } -type Node = node // nodeType for explicitly setting type of node type nodeType int diff --git a/pkg/types/publisher.go b/pkg/types/publisher.go index 8e6ece8..ccb9276 100644 --- a/pkg/types/publisher.go +++ b/pkg/types/publisher.go @@ -3,15 +3,18 @@ package types import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" - "github.com/jmoiron/sqlx" ) type Publisher interface { - PublishHeader(header *types.Header) (int64, error) - PublishStateNode(node *Node, headerID int64, tx *sqlx.Tx) (int64, error) - PublishStorageNode(node *Node, stateID int64, tx *sqlx.Tx) error - PublishCode(codeHash common.Hash, codeBytes []byte, tx *sqlx.Tx) error - BeginTx() (*sqlx.Tx, error) - CommitTx(*sqlx.Tx) error - PrepareTxForBatch(tx *sqlx.Tx, batchSize uint) (*sqlx.Tx, error) + PublishHeader(header *types.Header) error + PublishStateNode(node *Node, headerID string, tx Tx) error + PublishStorageNode(node *Node, headerID string, statePath []byte, tx Tx) error + PublishCode(codeHash common.Hash, codeBytes []byte, tx Tx) error + BeginTx() (Tx, error) + PrepareTxForBatch(tx Tx, batchSize uint) (Tx, error) +} + +type Tx interface { + Rollback() error + Commit() error } diff --git a/pkg/types/schema.go b/pkg/types/schema.go new file mode 100644 index 0000000..1430f23 --- /dev/null +++ b/pkg/types/schema.go @@ -0,0 +1,73 @@ +package types + +var TableIPLDBlock = Table{ + `public.blocks`, + []column{ + column{"key", text}, + column{"data", bytea}, + }, + `ON CONFLICT (key) DO NOTHING`, +} + +var TableNodeInfo = Table{ + Name: `public.nodes`, + Columns: []column{ + column{"genesis_block", varchar}, + column{"network_id", varchar}, + column{"node_id", varchar}, + column{"client_name", varchar}, + column{"chain_id", integer}, + }, +} + +var TableHeader = Table{ + "eth.header_cids", + []column{ + column{"block_number", bigint}, + column{"block_hash", varchar}, + column{"parent_hash", varchar}, + column{"cid", text}, + column{"td", numeric}, + column{"node_id", varchar}, + column{"reward", numeric}, + column{"state_root", varchar}, + column{"tx_root", varchar}, + column{"receipt_root", varchar}, + column{"uncle_root", varchar}, + column{"bloom", bytea}, + column{"timestamp", numeric}, + column{"mh_key", text}, + column{"times_validated", integer}, + column{"coinbase", varchar}, + }, + "ON CONFLICT (block_hash) DO UPDATE SET (parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated, coinbase) = (EXCLUDED.parent_hash, EXCLUDED.cid, EXCLUDED.td, EXCLUDED.node_id, EXCLUDED.reward, EXCLUDED.state_root, EXCLUDED.tx_root, EXCLUDED.receipt_root, EXCLUDED.uncle_root, EXCLUDED.bloom, EXCLUDED.timestamp, EXCLUDED.mh_key, eth.header_cids.times_validated + 1, EXCLUDED.coinbase)", +} + +var TableStateNode = Table{ + "eth.state_cids", + []column{ + column{"header_id", varchar}, + column{"state_leaf_key", varchar}, + column{"cid", text}, + column{"state_path", bytea}, + column{"node_type", integer}, + column{"diff", boolean}, + column{"mh_key", text}, + }, + `ON CONFLICT (header_id, state_path) DO UPDATE SET (state_leaf_key, cid, node_type, diff, mh_key) = (EXCLUDED.state_leaf_key, EXCLUDED.cid, EXCLUDED.node_type, EXCLUDED.diff, EXCLUDED.mh_key)`, +} + +var TableStorageNode = Table{ + "eth.storage_cids", + []column{ + column{"header_id", varchar}, + column{"state_path", bytea}, + column{"storage_leaf_key", varchar}, + column{"cid", text}, + column{"storage_path", bytea}, + column{"node_type", integer}, + column{"diff", boolean}, + column{"mh_key", text}, + }, + "ON CONFLICT (header_id, state_path, storage_path) DO UPDATE SET (storage_leaf_key, cid, node_type, diff, mh_key) = (EXCLUDED.storage_leaf_key, EXCLUDED.cid, EXCLUDED.node_type, EXCLUDED.diff, EXCLUDED.mh_key)", +} diff --git a/pkg/types/table.go b/pkg/types/table.go new file mode 100644 index 0000000..36299b4 --- /dev/null +++ b/pkg/types/table.go @@ -0,0 +1,79 @@ +package types + +import ( + "fmt" + "strings" +) + +type colType int + +const ( + integer colType = iota + boolean + bigint + numeric + bytea + varchar + text +) + +type column struct { + name string + typ colType +} +type Table struct { + Name string + Columns []column + conflictClause string +} + +func (tbl *Table) ToCsvRow(args ...interface{}) []string { + var row []string + for i, col := range tbl.Columns { + row = append(row, col.typ.formatter()(args[i])) + } + return row +} + +func (tbl *Table) ToInsertStatement() string { + var colnames, placeholders []string + for i, col := range tbl.Columns { + colnames = append(colnames, col.name) + placeholders = append(placeholders, fmt.Sprintf("$%d", i+1)) + } + return fmt.Sprintf( + "INSERT INTO %s (%s) VALUES (%s) %s", + tbl.Name, strings.Join(colnames, ", "), strings.Join(placeholders, ", "), tbl.conflictClause, + ) +} + +type colfmt = func(interface{}) string + +func sprintf(f string) colfmt { + return func(x interface{}) string { return fmt.Sprintf(f, x) } +} + +func (typ colType) formatter() colfmt { + switch typ { + case integer: + return sprintf("%d") + case boolean: + return func(x interface{}) string { + if x.(bool) { + return "t" + } + return "f" + } + case bigint: + return sprintf("%s") + case numeric: + return sprintf("%d") + case bytea: + return sprintf(`\x%x`) + case varchar: + return sprintf("%s") + case text: + return sprintf("%s") + } + panic("unreachable") +} diff --git a/pkg/types/util.go b/pkg/types/util.go new file mode 100644 index 0000000..32fe245 --- /dev/null +++ b/pkg/types/util.go @@ -0,0 +1,33 @@ +package types + +import ( + "bytes" + + "github.com/sirupsen/logrus" + + "github.com/ethereum/go-ethereum/common" +) + +var nullHash = common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000") + +func IsNullHash(hash common.Hash) bool { + return bytes.Equal(hash.Bytes(), nullHash.Bytes()) +} + +func CommitOrRollback(tx Tx, err error) error { + var rberr error + defer func() { + if rberr != nil { + logrus.Errorf("rollback failed: %s", rberr) + } + }() + if rec := recover(); rec != nil { + rberr = tx.Rollback() + panic(rec) + } else if err != nil { + rberr = tx.Rollback() + } else { + err = tx.Commit() + } + return err +} diff --git a/test/helper.go b/test/helper.go new file mode 100644 index 0000000..c5955e4 --- /dev/null +++ b/test/helper.go @@ -0,0 +1,52 @@ +package test + +import ( + "os" + "reflect" + "testing" + + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" + ethnode "github.com/ethereum/go-ethereum/statediff/indexer/node" +) + +var ( + DefaultNodeInfo = ethnode.Info{ + ID: "test_nodeid", + ClientName: "test_client", + GenesisBlock: "TEST_GENESIS", + NetworkID: "test_network", + ChainID: 0, + } + DefaultPgConfig = postgres.Config{ + Hostname: "localhost", + Port: 5432, + DatabaseName: "vulcanize_test", + Username: "vulcanize", + Password: "vulcanize_password", + + MaxIdle: 0, + MaxConnLifetime: 0, + MaxConns: 4, + } +) + +func NeedsDB(t *testing.T) { + t.Helper() + if os.Getenv("TEST_WITH_DB") == "" { + t.Skip("set TEST_WITH_DB to enable test") + } +} + +func NoError(t *testing.T, err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } +} + +// ExpectEqual asserts the provided interfaces are deep equal +func ExpectEqual(t *testing.T, want interface{}, got interface{}) { + if !reflect.DeepEqual(want, got) { + t.Fatalf("Expected: %v\nActual: %v", want, got) + } +} From 73cc0f191b2f6859664de3001df1d8159abd2d2d Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Tue, 15 Feb 2022 18:41:31 +0800 Subject: [PATCH 2/6] clean up - can rm type here --- pkg/types/schema.go | 76 ++++++++++++++++++++++----------------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/pkg/types/schema.go b/pkg/types/schema.go index 1430f23..a21dee8 100644 --- a/pkg/types/schema.go +++ b/pkg/types/schema.go @@ -3,8 +3,8 @@ package types var TableIPLDBlock = Table{ `public.blocks`, []column{ - column{"key", text}, - column{"data", bytea}, + {"key", text}, + {"data", bytea}, }, `ON CONFLICT (key) DO NOTHING`, } @@ -12,33 +12,33 @@ var TableIPLDBlock = Table{ var TableNodeInfo = Table{ Name: `public.nodes`, Columns: []column{ - column{"genesis_block", varchar}, - column{"network_id", varchar}, - column{"node_id", varchar}, - column{"client_name", varchar}, - column{"chain_id", integer}, + {"genesis_block", varchar}, + {"network_id", varchar}, + {"node_id", varchar}, + {"client_name", varchar}, + {"chain_id", integer}, }, } var TableHeader = Table{ "eth.header_cids", []column{ - column{"block_number", bigint}, - column{"block_hash", varchar}, - column{"parent_hash", varchar}, - column{"cid", text}, - column{"td", numeric}, - column{"node_id", varchar}, - column{"reward", numeric}, - column{"state_root", varchar}, - column{"tx_root", varchar}, - column{"receipt_root", varchar}, - column{"uncle_root", varchar}, - column{"bloom", bytea}, - column{"timestamp", numeric}, - column{"mh_key", text}, - column{"times_validated", integer}, - column{"coinbase", varchar}, + {"block_number", bigint}, + {"block_hash", varchar}, + {"parent_hash", varchar}, + {"cid", text}, + {"td", numeric}, + {"node_id", varchar}, + {"reward", numeric}, + {"state_root", varchar}, + {"tx_root", varchar}, + {"receipt_root", varchar}, + {"uncle_root", varchar}, + {"bloom", bytea}, + {"timestamp", numeric}, + {"mh_key", text}, + {"times_validated", integer}, + {"coinbase", varchar}, }, "ON CONFLICT (block_hash) DO UPDATE SET (parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated, coinbase) = (EXCLUDED.parent_hash, EXCLUDED.cid, EXCLUDED.td, EXCLUDED.node_id, EXCLUDED.reward, EXCLUDED.state_root, EXCLUDED.tx_root, EXCLUDED.receipt_root, EXCLUDED.uncle_root, EXCLUDED.bloom, EXCLUDED.timestamp, EXCLUDED.mh_key, eth.header_cids.times_validated + 1, EXCLUDED.coinbase)", } @@ -46,13 +46,13 @@ var TableHeader = Table{ var TableStateNode = Table{ "eth.state_cids", []column{ - column{"header_id", varchar}, - column{"state_leaf_key", varchar}, - column{"cid", text}, - column{"state_path", bytea}, - column{"node_type", integer}, - column{"diff", boolean}, - column{"mh_key", text}, + {"header_id", varchar}, + {"state_leaf_key", varchar}, + {"cid", text}, + {"state_path", bytea}, + {"node_type", integer}, + {"diff", boolean}, + {"mh_key", text}, }, `ON CONFLICT (header_id, state_path) DO UPDATE SET (state_leaf_key, cid, node_type, diff, mh_key) = (EXCLUDED.state_leaf_key, EXCLUDED.cid, EXCLUDED.node_type, EXCLUDED.diff, EXCLUDED.mh_key)`, } @@ -60,14 +60,14 @@ var TableStateNode = Table{ var TableStorageNode = Table{ "eth.storage_cids", []column{ - column{"header_id", varchar}, - column{"state_path", bytea}, - column{"storage_leaf_key", varchar}, - column{"cid", text}, - column{"storage_path", bytea}, - column{"node_type", integer}, - column{"diff", boolean}, - column{"mh_key", text}, + {"header_id", varchar}, + {"state_path", bytea}, + {"storage_leaf_key", varchar}, + {"cid", text}, + {"storage_path", bytea}, + {"node_type", integer}, + {"diff", boolean}, + {"mh_key", text}, }, "ON CONFLICT (header_id, state_path, storage_path) DO UPDATE SET (storage_leaf_key, cid, node_type, diff, mh_key) = (EXCLUDED.storage_leaf_key, EXCLUDED.cid, EXCLUDED.node_type, EXCLUDED.diff, EXCLUDED.mh_key)", } From 3fda647ee910862051c1f3aa6c2c078afd074f43 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Wed, 16 Feb 2022 11:13:51 +0800 Subject: [PATCH 3/6] clean up --- pkg/snapshot/file/publisher.go | 1 - pkg/snapshot/file/publisher_test.go | 4 ---- 2 files changed, 5 deletions(-) diff --git a/pkg/snapshot/file/publisher.go b/pkg/snapshot/file/publisher.go index 7b3d5c7..d7d46ac 100644 --- a/pkg/snapshot/file/publisher.go +++ b/pkg/snapshot/file/publisher.go @@ -16,7 +16,6 @@ package publisher import ( - // "bufio" "encoding/csv" "fmt" "os" diff --git a/pkg/snapshot/file/publisher_test.go b/pkg/snapshot/file/publisher_test.go index 9505373..ac8c9fe 100644 --- a/pkg/snapshot/file/publisher_test.go +++ b/pkg/snapshot/file/publisher_test.go @@ -59,8 +59,6 @@ func verifyFileData(t *testing.T, path string, tbl *snapt.Table) { break } test.NoError(t, err) - // _, err = tbl.fromCsvRow(row) - // test.NoError(t, err) } } @@ -68,8 +66,6 @@ func TestWriting(t *testing.T) { dir := t.TempDir() // tempdir like /tmp/TempFoo/001/, TempFoo defaults to 0700 test.NoError(t, os.Chmod(filepath.Dir(dir), 0755)) - // dir, _ := os.Getwd() - // dir += "/dump" pub := writeFiles(t, dir) From ef4ae3a0d4367c56c3dd5165609a78d8a498fd8b Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Wed, 16 Feb 2022 19:27:02 +0800 Subject: [PATCH 4/6] revisions --- pkg/snapshot/file/publisher.go | 17 +++++++---------- pkg/snapshot/pg/publisher.go | 20 ++++++++++---------- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/pkg/snapshot/file/publisher.go b/pkg/snapshot/file/publisher.go index d7d46ac..f151ac2 100644 --- a/pkg/snapshot/file/publisher.go +++ b/pkg/snapshot/file/publisher.go @@ -77,10 +77,7 @@ type fileWriter struct { // fileWriters wraps the file writers for each output table type fileWriters map[*snapt.Table]fileWriter -type fileTx struct { - fileWriters - // index uint -} +type fileTx struct{ fileWriters } func (tx fileWriters) Commit() error { for _, w := range tx { @@ -209,13 +206,13 @@ func (p *publisher) PublishHeader(header *types.Header) error { // PublishStateNode writes the state node to the ipfs backing datastore and adds secondary indexes // in the state_cids table -func (p *publisher) PublishStateNode(node *snapt.Node, headerID string, tx_ snapt.Tx) error { +func (p *publisher) PublishStateNode(node *snapt.Node, headerID string, snapTx snapt.Tx) error { var stateKey string if !snapt.IsNullHash(node.Key) { stateKey = node.Key.Hex() } - tx := tx_.(fileTx) + tx := snapTx.(fileTx) stateCIDStr, mhKey, err := tx.publishRaw(ipld.MEthStateTrie, node.Value) if err != nil { return err @@ -235,13 +232,13 @@ func (p *publisher) PublishStateNode(node *snapt.Node, headerID string, tx_ snap // PublishStorageNode writes the storage node to the ipfs backing pg datastore and adds secondary // indexes in the storage_cids table -func (p *publisher) PublishStorageNode(node *snapt.Node, headerID string, statePath []byte, tx_ snapt.Tx) error { +func (p *publisher) PublishStorageNode(node *snapt.Node, headerID string, statePath []byte, snapTx snapt.Tx) error { var storageKey string if !snapt.IsNullHash(node.Key) { storageKey = node.Key.Hex() } - tx := tx_.(fileTx) + tx := snapTx.(fileTx) storageCIDStr, mhKey, err := tx.publishRaw(ipld.MEthStorageTrie, node.Value) if err != nil { return err @@ -260,14 +257,14 @@ func (p *publisher) PublishStorageNode(node *snapt.Node, headerID string, stateP } // PublishCode writes code to the ipfs backing pg datastore -func (p *publisher) PublishCode(codeHash common.Hash, codeBytes []byte, tx_ snapt.Tx) error { +func (p *publisher) PublishCode(codeHash common.Hash, codeBytes []byte, snapTx snapt.Tx) error { // no codec for code, doesn't matter though since blockstore key is multihash-derived mhKey, err := shared.MultihashKeyFromKeccak256(codeHash) if err != nil { return fmt.Errorf("error deriving multihash key from codehash: %v", err) } - tx := tx_.(fileTx) + tx := snapTx.(fileTx) if err = tx.write(&snapt.TableIPLDBlock, mhKey, codeBytes); err != nil { return fmt.Errorf("error publishing code IPLD: %v", err) } diff --git a/pkg/snapshot/pg/publisher.go b/pkg/snapshot/pg/publisher.go index 54958e7..a32d626 100644 --- a/pkg/snapshot/pg/publisher.go +++ b/pkg/snapshot/pg/publisher.go @@ -112,11 +112,11 @@ func (p *publisher) PublishHeader(header *types.Header) (err error) { return err } - tx_, err := p.db.Begin(context.Background()) + snapTx, err := p.db.Begin(context.Background()) if err != nil { return err } - tx := pubTx{tx_, nil} + tx := pubTx{snapTx, nil} defer func() { err = snapt.CommitOrRollback(tx, err) }() if _, err = tx.publishIPLD(headerNode.Cid(), headerNode.RawData()); err != nil { @@ -132,13 +132,13 @@ func (p *publisher) PublishHeader(header *types.Header) (err error) { } // PublishStateNode writes the state node to the ipfs backing datastore and adds secondary indexes in the state_cids table -func (p *publisher) PublishStateNode(node *snapt.Node, headerID string, tx_ snapt.Tx) error { +func (p *publisher) PublishStateNode(node *snapt.Node, headerID string, snapTx snapt.Tx) error { var stateKey string if !snapt.IsNullHash(node.Key) { stateKey = node.Key.Hex() } - tx := tx_.(pubTx) + tx := snapTx.(pubTx) stateCIDStr, mhKey, err := tx.publishRaw(ipld.MEthStateTrie, node.Value) if err != nil { return err @@ -158,13 +158,13 @@ func (p *publisher) PublishStateNode(node *snapt.Node, headerID string, tx_ snap } // PublishStorageNode writes the storage node to the ipfs backing pg datastore and adds secondary indexes in the storage_cids table -func (p *publisher) PublishStorageNode(node *snapt.Node, headerID string, statePath []byte, tx_ snapt.Tx) error { +func (p *publisher) PublishStorageNode(node *snapt.Node, headerID string, statePath []byte, snapTx snapt.Tx) error { var storageKey string if !snapt.IsNullHash(node.Key) { storageKey = node.Key.Hex() } - tx := tx_.(pubTx) + tx := snapTx.(pubTx) storageCIDStr, mhKey, err := tx.publishRaw(ipld.MEthStorageTrie, node.Value) if err != nil { return err @@ -184,14 +184,14 @@ func (p *publisher) PublishStorageNode(node *snapt.Node, headerID string, stateP } // PublishCode writes code to the ipfs backing pg datastore -func (p *publisher) PublishCode(codeHash common.Hash, codeBytes []byte, tx_ snapt.Tx) error { +func (p *publisher) PublishCode(codeHash common.Hash, codeBytes []byte, snapTx snapt.Tx) error { // no codec for code, doesn't matter though since blockstore key is multihash-derived mhKey, err := shared.MultihashKeyFromKeccak256(codeHash) if err != nil { return fmt.Errorf("error deriving multihash key from codehash: %v", err) } - tx := tx_.(pubTx) + tx := snapTx.(pubTx) if _, err = tx.Exec(snapt.TableIPLDBlock.ToInsertStatement(), mhKey, codeBytes); err != nil { return fmt.Errorf("error publishing code IPLD: %v", err) } @@ -211,8 +211,8 @@ func (p *publisher) PrepareTxForBatch(tx snapt.Tx, maxBatchSize uint) (snapt.Tx, return nil, err } - tx_, err := p.db.Begin(context.Background()) - tx = pubTx{Tx: tx_} + snapTx, err := p.db.Begin(context.Background()) + tx = pubTx{Tx: snapTx} if err != nil { return nil, err } From 1a65d9e022d70716fc89de5163ab8c74086ba330 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Thu, 17 Feb 2022 14:34:54 +0800 Subject: [PATCH 5/6] string, not ptr, as map index --- pkg/snapshot/file/publisher.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/snapshot/file/publisher.go b/pkg/snapshot/file/publisher.go index f151ac2..c2a2d05 100644 --- a/pkg/snapshot/file/publisher.go +++ b/pkg/snapshot/file/publisher.go @@ -75,7 +75,7 @@ type fileWriter struct { } // fileWriters wraps the file writers for each output table -type fileWriters map[*snapt.Table]fileWriter +type fileWriters map[string]fileWriter type fileTx struct{ fileWriters } @@ -102,20 +102,20 @@ func newFileWriter(path string) (ret fileWriter, err error) { func (tx fileWriters) write(tbl *snapt.Table, args ...interface{}) error { row := tbl.ToCsvRow(args...) - return tx[tbl].Write(row) + return tx[tbl.Name].Write(row) } func makeFileWriters(dir string, tables []*snapt.Table) (fileWriters, error) { if err := os.MkdirAll(dir, 0755); err != nil { return nil, err } - writers := map[*snapt.Table]fileWriter{} + writers := fileWriters{} for _, tbl := range tables { w, err := newFileWriter(TableFile(dir, tbl.Name)) if err != nil { return nil, err } - writers[tbl] = w + writers[tbl.Name] = w } return writers, nil } From e0df3399e1ade6fc51a65a730384e11ecfe96110 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Thu, 17 Feb 2022 14:35:20 +0800 Subject: [PATCH 6/6] clanup --- pkg/snapshot/file/publisher.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/snapshot/file/publisher.go b/pkg/snapshot/file/publisher.go index c2a2d05..1d70a81 100644 --- a/pkg/snapshot/file/publisher.go +++ b/pkg/snapshot/file/publisher.go @@ -95,8 +95,7 @@ func newFileWriter(path string) (ret fileWriter, err error) { if err != nil { return } - buf := file //bufio.NewWriter(file) - ret = fileWriter{csv.NewWriter(buf)} + ret = fileWriter{csv.NewWriter(file)} return }