create csv file publisher; update geth and schema
This commit is contained in:
parent
d109173e5d
commit
c4b4f232e9
2
Makefile
2
Makefile
@ -6,7 +6,7 @@ mockgen_cmd=mockgen
|
||||
mocks: mocks/snapshot/publisher.go
|
||||
|
||||
mocks/snapshot/publisher.go: pkg/types/publisher.go
|
||||
$(mockgen_cmd) -package snapshot_mock -destination $@ -source $< Publisher
|
||||
$(mockgen_cmd) -package snapshot_mock -destination $@ -source $< Publisher Tx
|
||||
|
||||
clean:
|
||||
rm -f mocks/snapshot/publisher.go
|
||||
|
||||
@ -38,22 +38,25 @@ var stateSnapshotCmd = &cobra.Command{
|
||||
}
|
||||
|
||||
func stateSnapshot() {
|
||||
snapConfig := snapshot.NewConfig()
|
||||
pgDB, err := snapshot.NewPostgresDB(snapConfig.DB)
|
||||
if err != nil {
|
||||
logWithCommand.Fatal(err)
|
||||
}
|
||||
edb, err := snapshot.NewLevelDB(snapConfig.Eth)
|
||||
config := snapshot.NewConfig()
|
||||
edb, err := snapshot.NewLevelDB(config.Eth)
|
||||
if err != nil {
|
||||
logWithCommand.Fatal(err)
|
||||
}
|
||||
|
||||
snapshotService, err := snapshot.NewSnapshotService(edb, snapshot.NewPublisher(pgDB))
|
||||
mode := viper.GetString("snapshot.mode")
|
||||
pub, err := snapshot.NewPublisher(snapshot.SnapshotMode(mode), config)
|
||||
if err != nil {
|
||||
logWithCommand.Fatal(err)
|
||||
}
|
||||
|
||||
snapshotService, err := snapshot.NewSnapshotService(edb, pub)
|
||||
if err != nil {
|
||||
logWithCommand.Fatal(err)
|
||||
}
|
||||
height := viper.GetInt64("snapshot.blockHeight")
|
||||
workers := viper.GetUint("snapshot.workers")
|
||||
|
||||
if height < 0 {
|
||||
if err := snapshotService.CreateLatestSnapshot(workers); err != nil {
|
||||
logWithCommand.Fatal(err)
|
||||
|
||||
27
fixture/chaindata.go
Normal file
27
fixture/chaindata.go
Normal file
@ -0,0 +1,27 @@
|
||||
package fixture
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
)
|
||||
|
||||
// TODO: embed some mainnet data
|
||||
// import "embed"
|
||||
//_go:embed mainnet_data.tar.gz
|
||||
|
||||
var (
|
||||
ChaindataPath, AncientdataPath string
|
||||
)
|
||||
|
||||
func init() {
|
||||
_, path, _, _ := runtime.Caller(0)
|
||||
wd := filepath.Dir(path)
|
||||
|
||||
ChaindataPath = filepath.Join(wd, "..", "fixture", "chaindata")
|
||||
AncientdataPath = filepath.Join(ChaindataPath, "ancient")
|
||||
|
||||
if _, err := os.Stat(ChaindataPath); err != nil {
|
||||
panic("must populate chaindata at " + ChaindataPath)
|
||||
}
|
||||
}
|
||||
@ -10,7 +10,7 @@ import (
|
||||
snapt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/types"
|
||||
)
|
||||
|
||||
var PublishHeader = &types.Header{
|
||||
var Header1 = types.Header{
|
||||
ParentHash: common.HexToHash("0x6341fd3daf94b748c72ced5a5b26028f2474f5f00d824504e4fa37a75767e177"),
|
||||
UncleHash: common.HexToHash("0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347"),
|
||||
Coinbase: common.HexToAddress("0x0000000000000000000000000000000000000000"),
|
||||
@ -29,7 +29,7 @@ var PublishHeader = &types.Header{
|
||||
BaseFee: nil,
|
||||
}
|
||||
|
||||
var StateNode = snapt.Node{
|
||||
var StateNode1 = snapt.Node{
|
||||
NodeType: 0,
|
||||
Path: []byte{12, 0},
|
||||
Key: common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000"),
|
||||
|
||||
9
go.mod
9
go.mod
@ -12,12 +12,13 @@ require (
|
||||
github.com/google/go-cmp v0.5.6 // indirect
|
||||
github.com/google/uuid v1.3.0 // indirect
|
||||
github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c // indirect
|
||||
github.com/ipfs/go-cid v0.1.0 // indirect
|
||||
github.com/ipfs/go-cid v0.1.0
|
||||
github.com/ipfs/go-datastore v0.5.1 // indirect
|
||||
github.com/ipfs/go-ipfs-blockstore v1.1.2 // indirect
|
||||
github.com/ipfs/go-ipfs-blockstore v1.1.2
|
||||
github.com/ipfs/go-ipfs-ds-help v1.1.0
|
||||
github.com/ipfs/go-log v1.0.5 // indirect
|
||||
github.com/ipfs/go-log/v2 v2.4.0 // indirect
|
||||
github.com/jmoiron/sqlx v1.2.0
|
||||
github.com/jackc/pgx/v4 v4.15.0
|
||||
github.com/kr/pretty v0.3.0 // indirect
|
||||
github.com/multiformats/go-base32 v0.0.4 // indirect
|
||||
github.com/multiformats/go-multihash v0.1.0
|
||||
@ -42,4 +43,4 @@ require (
|
||||
lukechampine.com/blake3 v1.1.7 // indirect
|
||||
)
|
||||
|
||||
replace github.com/ethereum/go-ethereum v1.10.15 => github.com/vulcanize/go-ethereum v1.10.15-statediff-2.0.0
|
||||
replace github.com/ethereum/go-ethereum v1.10.15 => github.com/vulcanize/go-ethereum v1.10.15-statediff-3.0.1
|
||||
|
||||
158
go.sum
158
go.sum
@ -37,6 +37,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
|
||||
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
|
||||
github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
|
||||
github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
|
||||
github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc=
|
||||
github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
|
||||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
||||
github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
|
||||
github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
|
||||
@ -101,6 +103,7 @@ github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOC
|
||||
github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
|
||||
github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk=
|
||||
github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s=
|
||||
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
|
||||
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
|
||||
@ -113,6 +116,10 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn
|
||||
github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/cloudflare/cloudflare-go v0.14.0/go.mod h1:EnwdgGMaFOruiPZRFSgn+TsQ3hQ7C/YWzIGLeu5c304=
|
||||
github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I=
|
||||
github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
|
||||
github.com/cockroachdb/cockroach-go/v2 v2.0.3 h1:ZA346ACHIZctef6trOTwBAEvPVm1k0uLm/bb2Atc+S8=
|
||||
github.com/cockroachdb/cockroach-go/v2 v2.0.3/go.mod h1:hAuDgiVgDVkfirP9JnhXEfcXEPRKBpYdGz+l7mvYSzw=
|
||||
github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
|
||||
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
|
||||
github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ=
|
||||
@ -124,6 +131,7 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee
|
||||
github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
|
||||
github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
|
||||
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
|
||||
github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
|
||||
github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
|
||||
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
|
||||
@ -141,6 +149,7 @@ github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vs
|
||||
github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218=
|
||||
github.com/deepmap/oapi-codegen v1.6.0/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE9iAUlWl9Al3M=
|
||||
github.com/deepmap/oapi-codegen v1.8.2/go.mod h1:YLgSKSDv/bZQB7N4ws6luhozi3cEdRktEqrX88CvjIw=
|
||||
github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
|
||||
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
|
||||
github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8/go.mod h1:VMaSuZ+SZcx/wljOQKvp5srsbCiKDEb6K2wC4+PiBmQ=
|
||||
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
|
||||
@ -158,7 +167,9 @@ github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaB
|
||||
github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g=
|
||||
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||
github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0=
|
||||
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
|
||||
github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c=
|
||||
github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0=
|
||||
github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
|
||||
github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4=
|
||||
@ -167,7 +178,10 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo
|
||||
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
|
||||
github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI=
|
||||
github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU=
|
||||
github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI=
|
||||
github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww=
|
||||
github.com/georgysavva/scany v0.2.9 h1:Xt6rjYpHnMClTm/g+oZTnoSxUwiln5GqMNU+QeLNHQU=
|
||||
github.com/georgysavva/scany v0.2.9/go.mod h1:yeOeC1BdIdl6hOwy8uefL2WNSlseFzbhlG/frrh65SA=
|
||||
github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4=
|
||||
github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
@ -180,6 +194,7 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2
|
||||
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-kit/kit v0.10.0 h1:dXFJfIHVvUcpSgDOV+Ne6t7jXri8Tfv2uOLHUZ2XNuo=
|
||||
github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o=
|
||||
github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
|
||||
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
|
||||
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
|
||||
github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4=
|
||||
@ -195,7 +210,10 @@ github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG
|
||||
github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
|
||||
github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
|
||||
github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
|
||||
github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw=
|
||||
github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
|
||||
github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
|
||||
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
|
||||
github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
|
||||
@ -203,6 +221,7 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV
|
||||
github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
|
||||
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
|
||||
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
|
||||
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
|
||||
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
|
||||
github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
@ -283,6 +302,7 @@ github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoP
|
||||
github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
|
||||
github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
|
||||
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpxn4uE=
|
||||
github.com/hashicorp/go-bexpr v0.1.10/go.mod h1:oxlubA2vC/gFVfX1A6JGp7ls7uCDlfJn732ehYYg+g0=
|
||||
github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
|
||||
github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
|
||||
@ -312,6 +332,7 @@ github.com/holiman/uint256 v1.2.0 h1:gpSYcPLWGv4sG43I2mVLiDZCNDh/EpGjSk8tmtxitHM
|
||||
github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg=
|
||||
github.com/huin/goupnp v1.0.2 h1:RfGLP+h3mvisuWEyybxNq5Eft3NWhHLPeUN72kpKZoI=
|
||||
github.com/huin/goupnp v1.0.2/go.mod h1:0dxJBVBHqTMjIUMkESDTNgOOx/Mw5wYIfyFmdzSamkM=
|
||||
github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o=
|
||||
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
|
||||
@ -367,6 +388,85 @@ github.com/ipfs/go-log/v2 v2.4.0 h1:iR/2o9PGWanVJrBgIH5Ff8mPGOwpqLaPIAFqSnsdlzk=
|
||||
github.com/ipfs/go-log/v2 v2.4.0/go.mod h1:nPZnh7Cj7lwS3LpRU5Mwr2ol1c2gXIEXuF6aywqrtmo=
|
||||
github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg=
|
||||
github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY=
|
||||
github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0=
|
||||
github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo=
|
||||
github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
|
||||
github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8=
|
||||
github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
|
||||
github.com/jackc/fake v0.0.0-20150926172116-812a484cc733/go.mod h1:WrMFNQdiFJ80sQsxDoMokWK1W5TQtxBFNpzWTD84ibQ=
|
||||
github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA=
|
||||
github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE=
|
||||
github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s=
|
||||
github.com/jackc/pgconn v1.4.0/go.mod h1:Y2O3ZDF0q4mMacyWV3AstPJpeHXWGEetiFttmq5lahk=
|
||||
github.com/jackc/pgconn v1.5.0/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI=
|
||||
github.com/jackc/pgconn v1.5.1-0.20200601181101-fa742c524853/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI=
|
||||
github.com/jackc/pgconn v1.6.4/go.mod h1:w2pne1C2tZgP+TvjqLpOigGzNqjBgQW9dUw/4Chex78=
|
||||
github.com/jackc/pgconn v1.7.0/go.mod h1:sF/lPpNEMEOp+IYhyQGdAvrG20gWf6A1tKlr0v7JMeA=
|
||||
github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o=
|
||||
github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY=
|
||||
github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI=
|
||||
github.com/jackc/pgconn v1.10.0/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI=
|
||||
github.com/jackc/pgconn v1.11.0 h1:HiHArx4yFbwl91X3qqIHtUFoiIfLNJXCQRsnzkiwwaQ=
|
||||
github.com/jackc/pgconn v1.11.0/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI=
|
||||
github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE=
|
||||
github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8=
|
||||
github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE=
|
||||
github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c=
|
||||
github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65 h1:DadwsjnMwFjfWc9y5Wi/+Zz7xoE5ALHsRQlOctkOiHc=
|
||||
github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak=
|
||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||
github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A=
|
||||
github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78=
|
||||
github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA=
|
||||
github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg=
|
||||
github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM=
|
||||
github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM=
|
||||
github.com/jackc/pgproto3/v2 v2.0.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
|
||||
github.com/jackc/pgproto3/v2 v2.0.2/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
|
||||
github.com/jackc/pgproto3/v2 v2.0.5/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
|
||||
github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
|
||||
github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
|
||||
github.com/jackc/pgproto3/v2 v2.2.0 h1:r7JypeP2D3onoQTCxWdTpCtJ4D+qpKr0TxvoyMhZ5ns=
|
||||
github.com/jackc/pgproto3/v2 v2.2.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
|
||||
github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E=
|
||||
github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg=
|
||||
github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E=
|
||||
github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg=
|
||||
github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc=
|
||||
github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw=
|
||||
github.com/jackc/pgtype v1.2.0/go.mod h1:5m2OfMh1wTK7x+Fk952IDmI4nw3nPrvtQdM0ZT4WpC0=
|
||||
github.com/jackc/pgtype v1.3.0/go.mod h1:b0JqxHvPmljG+HQ5IsvQ0yqeSi4nGcDTVjFoiLDb0Ik=
|
||||
github.com/jackc/pgtype v1.3.1-0.20200510190516-8cd94a14c75a/go.mod h1:vaogEUkALtxZMCH411K+tKzNpwzCKU+AnPzBKZ+I+Po=
|
||||
github.com/jackc/pgtype v1.3.1-0.20200606141011-f6355165a91c/go.mod h1:cvk9Bgu/VzJ9/lxTO5R5sf80p0DiucVtN7ZxvaC4GmQ=
|
||||
github.com/jackc/pgtype v1.4.2/go.mod h1:JCULISAZBFGrHaOXIIFiyfzW5VY0GRitRr8NeJsrdig=
|
||||
github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM=
|
||||
github.com/jackc/pgtype v1.8.1/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4=
|
||||
github.com/jackc/pgtype v1.10.0 h1:ILnBWrRMSXGczYvmkYD6PsYyVFUNLTnIUJHHDLmqk38=
|
||||
github.com/jackc/pgtype v1.10.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4=
|
||||
github.com/jackc/pgx v3.6.2+incompatible h1:2zP5OD7kiyR3xzRYMhOcXVvkDZsImVXfj+yIyTQf3/o=
|
||||
github.com/jackc/pgx v3.6.2+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I=
|
||||
github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y=
|
||||
github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM=
|
||||
github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc=
|
||||
github.com/jackc/pgx/v4 v4.5.0/go.mod h1:EpAKPLdnTorwmPUUsqrPxy5fphV18j9q3wrfRXgo+kA=
|
||||
github.com/jackc/pgx/v4 v4.6.0/go.mod h1:vPh43ZzxijXUVJ+t/EmXBtFmbFVO72cuneCT9oAlxAg=
|
||||
github.com/jackc/pgx/v4 v4.6.1-0.20200510190926-94ba730bb1e9/go.mod h1:t3/cdRQl6fOLDxqtlyhe9UWgfIi9R8+8v8GKV5TRA/o=
|
||||
github.com/jackc/pgx/v4 v4.6.1-0.20200606145419-4e5062306904/go.mod h1:ZDaNWkt9sW1JMiNn0kdYBaLelIhw7Pg4qd+Vk6tw7Hg=
|
||||
github.com/jackc/pgx/v4 v4.8.1/go.mod h1:4HOLxrl8wToZJReD04/yB20GDwf4KBYETvlHciCnwW0=
|
||||
github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs=
|
||||
github.com/jackc/pgx/v4 v4.13.0/go.mod h1:9P4X524sErlaxj0XSGZk7s+LD0eOyu1ZDUrrpznYDF0=
|
||||
github.com/jackc/pgx/v4 v4.15.0 h1:B7dTkXsdILD3MF987WGGCcg+tvLW6bZJdEcqVFeU//w=
|
||||
github.com/jackc/pgx/v4 v4.15.0/go.mod h1:D/zyOyXiaM1TmVWnOM18p0xdDtdakRBa0RsVGI3U3bw=
|
||||
github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
|
||||
github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
|
||||
github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
|
||||
github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
|
||||
github.com/jackc/puddle v1.1.2/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
|
||||
github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
|
||||
github.com/jackc/puddle v1.2.1 h1:gI8os0wpRXFd4FiAY2dWiqRK037tjj3t7rKFeO4X5iw=
|
||||
github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
|
||||
github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 h1:6OvNmYgJyexcZ3pYbTI9jWx5tHo1Dee/tWbLMfPe2TA=
|
||||
github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
|
||||
github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA=
|
||||
github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY=
|
||||
@ -375,6 +475,9 @@ github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZl
|
||||
github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU=
|
||||
github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
|
||||
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
|
||||
github.com/jinzhu/gorm v1.9.12/go.mod h1:vhTjlKSJUTWNtcbQtrMBFCxy7eXTzeCAzfL5fBZT/Qs=
|
||||
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
|
||||
github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
|
||||
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
|
||||
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
|
||||
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
|
||||
@ -408,6 +511,7 @@ github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02
|
||||
github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg=
|
||||
github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
|
||||
@ -417,6 +521,7 @@ github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn
|
||||
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
|
||||
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
@ -425,6 +530,11 @@ github.com/labstack/echo/v4 v4.2.1/go.mod h1:AA49e0DZ8kk5jTOOCKNuPR6oTnBS0dYiM4F
|
||||
github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
|
||||
github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
|
||||
github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
|
||||
github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
|
||||
github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
|
||||
github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
|
||||
github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
|
||||
github.com/lib/pq v1.4.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
|
||||
github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8=
|
||||
github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
||||
github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM=
|
||||
@ -440,13 +550,16 @@ github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIG
|
||||
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
|
||||
github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
|
||||
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
|
||||
github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
|
||||
github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
|
||||
github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
|
||||
github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
|
||||
github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc=
|
||||
github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc=
|
||||
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
||||
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
||||
github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
|
||||
github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
|
||||
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
|
||||
github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
|
||||
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
|
||||
@ -458,8 +571,8 @@ github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/Qd
|
||||
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
|
||||
github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
|
||||
github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
|
||||
github.com/mattn/go-sqlite3 v1.14.7 h1:fxWBnXkxfM6sRiuH3bqJ4CfzZojMOLVc0UTsTglEghA=
|
||||
github.com/mattn/go-sqlite3 v1.14.7/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
|
||||
github.com/mattn/go-sqlite3 v2.0.1+incompatible h1:xQ15muvnzGBHpIpdrNi1DA5x0+TcBZzsIDwmw9uTHzw=
|
||||
github.com/mattn/go-sqlite3 v2.0.1+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
|
||||
github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
||||
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
|
||||
@ -479,6 +592,7 @@ github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:F
|
||||
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag=
|
||||
github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
|
||||
github.com/mitchellh/pointerstructure v1.2.0 h1:O+i9nHnXS3l/9Wu7r4NrEdwA2VFTicjUEN1uBnDo34A=
|
||||
github.com/mitchellh/pointerstructure v1.2.0/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
@ -596,23 +710,35 @@ github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA
|
||||
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
|
||||
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
|
||||
github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52/go.mod h1:RDpi1RftBQPUCDRw6SmxeaREsAaRKnOclghuzp/WRzc=
|
||||
github.com/rjeczalik/notify v0.9.1 h1:CLCKso/QK1snAlnhNR/CNvNiFU2saUtjV0bx3EwNeCE=
|
||||
github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho=
|
||||
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k=
|
||||
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
|
||||
github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
|
||||
github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
|
||||
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
|
||||
github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
|
||||
github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc=
|
||||
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
|
||||
github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
|
||||
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
|
||||
github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo=
|
||||
github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo=
|
||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||
github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU=
|
||||
github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
|
||||
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
|
||||
github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
|
||||
github.com/shopspring/decimal v0.0.0-20200419222939-1884f454f8ea/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
|
||||
github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ=
|
||||
github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
|
||||
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
|
||||
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
|
||||
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
|
||||
github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I=
|
||||
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
|
||||
@ -641,12 +767,15 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn
|
||||
github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
|
||||
github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM=
|
||||
github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
|
||||
github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 h1:Gb2Tyox57NRNuZ2d3rmvB3pcmbu7O1RS3m8WRx7ilrg=
|
||||
github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q=
|
||||
github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
|
||||
github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
|
||||
github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
|
||||
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
|
||||
github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
@ -665,6 +794,7 @@ github.com/tklauser/numcpus v0.2.2 h1:oyhllyrScuYI6g+h/zUvNXNp1wy7x8qQy3t/piefld
|
||||
github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM=
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
|
||||
github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef h1:wHSqTBrZW24CsNJDfeh9Ex6Pm0Rcpc7qrgKBiL44vF4=
|
||||
github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs=
|
||||
github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
|
||||
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
|
||||
@ -675,8 +805,8 @@ github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPU
|
||||
github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
|
||||
github.com/vulcanize/go-eth-state-node-iterator v1.0.0 h1:FQ4s0K5TnRD44p3vO4uQtjfE8C1Gr2EDMT2vVGS3Lu8=
|
||||
github.com/vulcanize/go-eth-state-node-iterator v1.0.0/go.mod h1:uWhleTvUEZ+cEkNRIAmBpZ14KilTP71OxY5NZDrpNlo=
|
||||
github.com/vulcanize/go-ethereum v1.10.15-statediff-2.0.0 h1:/BiYPUHnubh46YVtASGt4MPlFR96Rc+iJuTyOI8KZa4=
|
||||
github.com/vulcanize/go-ethereum v1.10.15-statediff-2.0.0/go.mod h1:9L+QY31AnWnX2/2HDOySCjQoYUdWNGBRMezFJVfH73E=
|
||||
github.com/vulcanize/go-ethereum v1.10.15-statediff-3.0.1 h1:MX7WcTwxpxOoYVMifrs9vsHDmM6gKOFF+2KnP19LddI=
|
||||
github.com/vulcanize/go-ethereum v1.10.15-statediff-3.0.1/go.mod h1:XO9WLkNXfwoJN05BZj0//xgOWHJyUrUPdnudbQfKlUo=
|
||||
github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM=
|
||||
github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
|
||||
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
|
||||
@ -686,6 +816,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
|
||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
|
||||
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
|
||||
go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
|
||||
go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
|
||||
@ -722,21 +853,29 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf
|
||||
golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
|
||||
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
|
||||
golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
|
||||
golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
|
||||
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
|
||||
golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
|
||||
golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
|
||||
golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b h1:QAqMVf3pSa6eeTsuklijukjXBlj7Es2QQplab+/RbQ4=
|
||||
golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
@ -822,6 +961,7 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
@ -835,6 +975,7 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
|
||||
golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
@ -892,6 +1033,7 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxb
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE=
|
||||
golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
@ -906,12 +1048,14 @@ golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3
|
||||
golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
@ -935,6 +1079,8 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
|
||||
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
|
||||
golang.org/x/tools v0.1.8 h1:P1HhGGuLW4aAclzjtmJdf0mJOjVUZUzOTqkAkWL+l6w=
|
||||
golang.org/x/tools v0.1.8/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
|
||||
golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
@ -1007,6 +1153,7 @@ gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qS
|
||||
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o=
|
||||
gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s=
|
||||
gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno=
|
||||
gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU=
|
||||
@ -1015,6 +1162,7 @@ gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6/go.mod h1:uAJ
|
||||
gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||
gopkg.in/urfave/cli.v1 v1.20.0 h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0=
|
||||
gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0=
|
||||
gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
|
||||
gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
|
||||
|
||||
@ -16,8 +16,10 @@
|
||||
package snapshot
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
|
||||
ethNode "github.com/ethereum/go-ethereum/statediff/indexer/node"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
@ -30,40 +32,56 @@ const (
|
||||
LVL_DB_PATH = "LVL_DB_PATH"
|
||||
)
|
||||
|
||||
// SnapshotMode specifies the snapshot data output method
|
||||
type SnapshotMode string
|
||||
|
||||
const (
|
||||
PgSnapshot SnapshotMode = "postgres"
|
||||
FileSnapshot SnapshotMode = "file"
|
||||
)
|
||||
|
||||
// Config contains params for both databases the service uses
|
||||
type Config struct {
|
||||
DB *DBConfig
|
||||
Eth *EthConfig
|
||||
}
|
||||
|
||||
// DBConfig is config parameters for DB.
|
||||
type DBConfig struct {
|
||||
Node ethNode.Info
|
||||
URI string
|
||||
ConnConfig postgres.ConnectionConfig
|
||||
DB *DBConfig
|
||||
File *FileConfig
|
||||
}
|
||||
|
||||
// EthConfig is config parameters for the chain.
|
||||
type EthConfig struct {
|
||||
LevelDBPath string
|
||||
AncientDBPath string
|
||||
NodeInfo ethNode.Info
|
||||
}
|
||||
|
||||
// DBConfig is config parameters for DB.
|
||||
type DBConfig struct {
|
||||
URI string
|
||||
ConnConfig postgres.Config
|
||||
}
|
||||
|
||||
type FileConfig struct {
|
||||
OutputDir string
|
||||
}
|
||||
|
||||
func NewConfig() *Config {
|
||||
ret := &Config{&DBConfig{}, &EthConfig{}}
|
||||
ret := &Config{
|
||||
&EthConfig{},
|
||||
&DBConfig{},
|
||||
&FileConfig{},
|
||||
}
|
||||
ret.Init()
|
||||
return ret
|
||||
}
|
||||
|
||||
// Init Initialises config
|
||||
func (c *Config) Init() {
|
||||
c.DB.dbInit()
|
||||
viper.BindEnv("ethereum.nodeID", ETH_NODE_ID)
|
||||
viper.BindEnv("ethereum.clientName", ETH_CLIENT_NAME)
|
||||
viper.BindEnv("ethereum.genesisBlock", ETH_GENESIS_BLOCK)
|
||||
viper.BindEnv("ethereum.networkID", ETH_NETWORK_ID)
|
||||
|
||||
c.DB.Node = ethNode.Info{
|
||||
c.Eth.NodeInfo = ethNode.Info{
|
||||
ID: viper.GetString("ethereum.nodeID"),
|
||||
ClientName: viper.GetString("ethereum.clientName"),
|
||||
GenesisBlock: viper.GetString("ethereum.genesisBlock"),
|
||||
@ -76,29 +94,38 @@ func (c *Config) Init() {
|
||||
|
||||
c.Eth.AncientDBPath = viper.GetString("leveldb.ancient")
|
||||
c.Eth.LevelDBPath = viper.GetString("leveldb.path")
|
||||
|
||||
c.DB.Init()
|
||||
c.File.Init()
|
||||
}
|
||||
|
||||
func (c *DBConfig) dbInit() {
|
||||
viper.BindEnv("database.name", postgres.DATABASE_NAME)
|
||||
viper.BindEnv("database.hostname", postgres.DATABASE_HOSTNAME)
|
||||
viper.BindEnv("database.port", postgres.DATABASE_PORT)
|
||||
viper.BindEnv("database.user", postgres.DATABASE_USER)
|
||||
viper.BindEnv("database.password", postgres.DATABASE_PASSWORD)
|
||||
viper.BindEnv("database.maxIdle", postgres.DATABASE_MAX_IDLE_CONNECTIONS)
|
||||
viper.BindEnv("database.maxOpen", postgres.DATABASE_MAX_OPEN_CONNECTIONS)
|
||||
viper.BindEnv("database.maxLifetime", postgres.DATABASE_MAX_CONN_LIFETIME)
|
||||
func (c *DBConfig) Init() {
|
||||
viper.BindEnv("database.name", "DATABASE_NAME")
|
||||
viper.BindEnv("database.hostname", "DATABASE_HOSTNAME")
|
||||
viper.BindEnv("database.port", "DATABASE_PORT")
|
||||
viper.BindEnv("database.user", "DATABASE_USER")
|
||||
viper.BindEnv("database.password", "DATABASE_PASSWORD")
|
||||
viper.BindEnv("database.maxIdle", "DATABASE_MAX_IDLE_CONNECTIONS")
|
||||
viper.BindEnv("database.maxOpen", "DATABASE_MAX_OPEN_CONNECTIONS")
|
||||
viper.BindEnv("database.maxLifetime", "DATABASE_MAX_CONN_LIFETIME")
|
||||
|
||||
dbParams := postgres.ConnectionParams{}
|
||||
dbParams := postgres.Config{}
|
||||
// DB params
|
||||
dbParams.Name = viper.GetString("database.name")
|
||||
dbParams.DatabaseName = viper.GetString("database.name")
|
||||
dbParams.Hostname = viper.GetString("database.hostname")
|
||||
dbParams.Port = viper.GetInt("database.port")
|
||||
dbParams.User = viper.GetString("database.user")
|
||||
dbParams.Username = viper.GetString("database.user")
|
||||
dbParams.Password = viper.GetString("database.password")
|
||||
|
||||
c.URI = postgres.DbConnectionString(dbParams)
|
||||
// Connection config
|
||||
c.ConnConfig.MaxIdle = viper.GetInt("database.maxIdle")
|
||||
c.ConnConfig.MaxOpen = viper.GetInt("database.maxOpen")
|
||||
c.ConnConfig.MaxLifetime = viper.GetInt("database.maxLifetime")
|
||||
dbParams.MaxIdle = viper.GetInt("database.maxIdle")
|
||||
dbParams.MaxConns = viper.GetInt("database.maxOpen")
|
||||
dbParams.MaxConnLifetime = time.Duration(viper.GetInt("database.maxLifetime")) * time.Second
|
||||
|
||||
c.ConnConfig = dbParams
|
||||
c.URI = dbParams.DbConnectionString()
|
||||
}
|
||||
|
||||
func (c *FileConfig) Init() {
|
||||
viper.BindEnv("file.outputDir", "FILE_OUTPUT_DIR")
|
||||
c.OutputDir = viper.GetString("file.outputDir")
|
||||
}
|
||||
|
||||
298
pkg/snapshot/file/publisher.go
Normal file
298
pkg/snapshot/file/publisher.go
Normal file
@ -0,0 +1,298 @@
|
||||
// Copyright © 2020 Vulcanize, Inc
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package publisher
|
||||
|
||||
import (
|
||||
// "bufio"
|
||||
"encoding/csv"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ipfs/go-cid"
|
||||
blockstore "github.com/ipfs/go-ipfs-blockstore"
|
||||
dshelp "github.com/ipfs/go-ipfs-ds-help"
|
||||
"github.com/multiformats/go-multihash"
|
||||
"github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
|
||||
nodeinfo "github.com/ethereum/go-ethereum/statediff/indexer/node"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/shared"
|
||||
snapt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/types"
|
||||
)
|
||||
|
||||
var _ snapt.Publisher = (*publisher)(nil)
|
||||
|
||||
var (
|
||||
// tables written once per block
|
||||
perBlockTables = []*snapt.Table{
|
||||
&snapt.TableIPLDBlock,
|
||||
&snapt.TableNodeInfo,
|
||||
&snapt.TableHeader,
|
||||
}
|
||||
// tables written during state iteration
|
||||
perNodeTables = []*snapt.Table{
|
||||
&snapt.TableIPLDBlock,
|
||||
&snapt.TableStateNode,
|
||||
&snapt.TableStorageNode,
|
||||
}
|
||||
)
|
||||
|
||||
const logInterval = 1 * time.Minute
|
||||
|
||||
type publisher struct {
|
||||
dir string // dir containing output files
|
||||
writers fileWriters
|
||||
|
||||
nodeInfo nodeinfo.Info
|
||||
|
||||
startTime time.Time
|
||||
currBatchSize uint
|
||||
stateNodeCounter uint64
|
||||
storageNodeCounter uint64
|
||||
codeNodeCounter uint64
|
||||
txCounter uint32
|
||||
}
|
||||
|
||||
type fileWriter struct {
|
||||
*csv.Writer
|
||||
}
|
||||
|
||||
// fileWriters wraps the file writers for each output table
|
||||
type fileWriters map[*snapt.Table]fileWriter
|
||||
|
||||
type fileTx struct {
|
||||
fileWriters
|
||||
// index uint
|
||||
}
|
||||
|
||||
func (tx fileWriters) Commit() error {
|
||||
for _, w := range tx {
|
||||
w.Flush()
|
||||
if err := w.Error(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (fileWriters) Rollback() error { return nil } // TODO: delete the file?
|
||||
|
||||
func newFileWriter(path string) (ret fileWriter, err error) {
|
||||
file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
buf := file //bufio.NewWriter(file)
|
||||
ret = fileWriter{csv.NewWriter(buf)}
|
||||
return
|
||||
}
|
||||
|
||||
func (tx fileWriters) write(tbl *snapt.Table, args ...interface{}) error {
|
||||
row := tbl.ToCsvRow(args...)
|
||||
return tx[tbl].Write(row)
|
||||
}
|
||||
|
||||
func makeFileWriters(dir string, tables []*snapt.Table) (fileWriters, error) {
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
writers := map[*snapt.Table]fileWriter{}
|
||||
for _, tbl := range tables {
|
||||
w, err := newFileWriter(TableFile(dir, tbl.Name))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
writers[tbl] = w
|
||||
}
|
||||
return writers, nil
|
||||
}
|
||||
|
||||
// NewPublisher creates a publisher which writes to per-table CSV files which can be imported
|
||||
// with the Postgres COPY command.
|
||||
// The output directory will be created if it does not exist.
|
||||
func NewPublisher(path string, node nodeinfo.Info) (*publisher, error) {
|
||||
if err := os.MkdirAll(path, 0755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
writers, err := makeFileWriters(path, perBlockTables)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pub := &publisher{
|
||||
writers: writers,
|
||||
dir: path,
|
||||
nodeInfo: node,
|
||||
startTime: time.Now(),
|
||||
}
|
||||
go pub.logNodeCounters()
|
||||
return pub, nil
|
||||
}
|
||||
|
||||
func TableFile(dir, name string) string { return filepath.Join(dir, name+".csv") }
|
||||
|
||||
func (p *publisher) txDir(index uint32) string {
|
||||
return filepath.Join(p.dir, fmt.Sprintf("%010d", index))
|
||||
}
|
||||
|
||||
func (p *publisher) BeginTx() (snapt.Tx, error) {
|
||||
index := atomic.LoadUint32(&p.txCounter)
|
||||
dir := p.txDir(index)
|
||||
writers, err := makeFileWriters(dir, perNodeTables)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
atomic.AddUint32(&p.txCounter, 1)
|
||||
return fileTx{writers}, nil
|
||||
}
|
||||
|
||||
// PublishRaw derives a cid from raw bytes and provided codec and multihash type, and writes it to the db tx
|
||||
// returns the CID and blockstore prefixed multihash key
|
||||
func (tx fileWriters) publishRaw(codec uint64, raw []byte) (cid, prefixedKey string, err error) {
|
||||
c, err := ipld.RawdataToCid(codec, raw, multihash.KECCAK_256)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
cid = c.String()
|
||||
prefixedKey, err = tx.publishIPLD(c, raw)
|
||||
return
|
||||
}
|
||||
|
||||
func (tx fileWriters) publishIPLD(c cid.Cid, raw []byte) (string, error) {
|
||||
dbKey := dshelp.MultihashToDsKey(c.Hash())
|
||||
prefixedKey := blockstore.BlockPrefix.String() + dbKey.String()
|
||||
return prefixedKey, tx.write(&snapt.TableIPLDBlock, prefixedKey, raw)
|
||||
}
|
||||
|
||||
// PublishHeader writes the header to the ipfs backing pg datastore and adds secondary
|
||||
// indexes in the header_cids table
|
||||
func (p *publisher) PublishHeader(header *types.Header) error {
|
||||
headerNode, err := ipld.NewEthHeader(header)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err = p.writers.publishIPLD(headerNode.Cid(), headerNode.RawData()); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
mhKey := shared.MultihashKeyFromCID(headerNode.Cid())
|
||||
err = p.writers.write(&snapt.TableNodeInfo, p.nodeInfo.GenesisBlock, p.nodeInfo.NetworkID, p.nodeInfo.ID,
|
||||
p.nodeInfo.ClientName, p.nodeInfo.ChainID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = p.writers.write(&snapt.TableHeader, header.Number.String(), header.Hash().Hex(), header.ParentHash.Hex(),
|
||||
headerNode.Cid().String(), 0, p.nodeInfo.ID, 0, header.Root.Hex(), header.TxHash.Hex(),
|
||||
header.ReceiptHash.Hex(), header.UncleHash.Hex(), header.Bloom.Bytes(), header.Time, mhKey,
|
||||
0, header.Coinbase.String())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return p.writers.Commit()
|
||||
}
|
||||
|
||||
// PublishStateNode writes the state node to the ipfs backing datastore and adds secondary indexes
|
||||
// in the state_cids table
|
||||
func (p *publisher) PublishStateNode(node *snapt.Node, headerID string, tx_ snapt.Tx) error {
|
||||
var stateKey string
|
||||
if !snapt.IsNullHash(node.Key) {
|
||||
stateKey = node.Key.Hex()
|
||||
}
|
||||
|
||||
tx := tx_.(fileTx)
|
||||
stateCIDStr, mhKey, err := tx.publishRaw(ipld.MEthStateTrie, node.Value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = tx.write(&snapt.TableStateNode, headerID, stateKey, stateCIDStr, node.Path,
|
||||
node.NodeType, false, mhKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// increment state node counter.
|
||||
atomic.AddUint64(&p.stateNodeCounter, 1)
|
||||
// increment current batch size counter
|
||||
p.currBatchSize += 2
|
||||
return err
|
||||
}
|
||||
|
||||
// PublishStorageNode writes the storage node to the ipfs backing pg datastore and adds secondary
|
||||
// indexes in the storage_cids table
|
||||
func (p *publisher) PublishStorageNode(node *snapt.Node, headerID string, statePath []byte, tx_ snapt.Tx) error {
|
||||
var storageKey string
|
||||
if !snapt.IsNullHash(node.Key) {
|
||||
storageKey = node.Key.Hex()
|
||||
}
|
||||
|
||||
tx := tx_.(fileTx)
|
||||
storageCIDStr, mhKey, err := tx.publishRaw(ipld.MEthStorageTrie, node.Value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = tx.write(&snapt.TableStorageNode, headerID, statePath, storageKey, storageCIDStr, node.Path,
|
||||
node.NodeType, false, mhKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// increment storage node counter.
|
||||
atomic.AddUint64(&p.storageNodeCounter, 1)
|
||||
// increment current batch size counter
|
||||
p.currBatchSize += 2
|
||||
return nil
|
||||
}
|
||||
|
||||
// PublishCode writes code to the ipfs backing pg datastore
|
||||
func (p *publisher) PublishCode(codeHash common.Hash, codeBytes []byte, tx_ snapt.Tx) error {
|
||||
// no codec for code, doesn't matter though since blockstore key is multihash-derived
|
||||
mhKey, err := shared.MultihashKeyFromKeccak256(codeHash)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error deriving multihash key from codehash: %v", err)
|
||||
}
|
||||
|
||||
tx := tx_.(fileTx)
|
||||
if err = tx.write(&snapt.TableIPLDBlock, mhKey, codeBytes); err != nil {
|
||||
return fmt.Errorf("error publishing code IPLD: %v", err)
|
||||
}
|
||||
// increment code node counter.
|
||||
atomic.AddUint64(&p.codeNodeCounter, 1)
|
||||
p.currBatchSize++
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *publisher) PrepareTxForBatch(tx snapt.Tx, maxBatchSize uint) (snapt.Tx, error) {
|
||||
return tx, nil
|
||||
}
|
||||
|
||||
// logNodeCounters periodically logs the number of node processed.
|
||||
func (p *publisher) logNodeCounters() {
|
||||
t := time.NewTicker(logInterval)
|
||||
for range t.C {
|
||||
p.printNodeCounters()
|
||||
}
|
||||
}
|
||||
|
||||
func (p *publisher) printNodeCounters() {
|
||||
logrus.Infof("runtime: %s", time.Now().Sub(p.startTime).String())
|
||||
logrus.Infof("processed state nodes: %d", atomic.LoadUint64(&p.stateNodeCounter))
|
||||
logrus.Infof("processed storage nodes: %d", atomic.LoadUint64(&p.storageNodeCounter))
|
||||
logrus.Infof("processed code nodes: %d", atomic.LoadUint64(&p.codeNodeCounter))
|
||||
}
|
||||
136
pkg/snapshot/file/publisher_test.go
Normal file
136
pkg/snapshot/file/publisher_test.go
Normal file
@ -0,0 +1,136 @@
|
||||
package publisher
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/csv"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
|
||||
"github.com/jackc/pgx/v4"
|
||||
|
||||
fixt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/fixture"
|
||||
snapt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/types"
|
||||
"github.com/vulcanize/eth-pg-ipfs-state-snapshot/test"
|
||||
)
|
||||
|
||||
var (
|
||||
pgConfig = test.DefaultPgConfig
|
||||
nodeInfo = test.DefaultNodeInfo
|
||||
// tables ordered according to fkey depedencies
|
||||
allTables = []*snapt.Table{
|
||||
&snapt.TableIPLDBlock,
|
||||
&snapt.TableNodeInfo,
|
||||
&snapt.TableHeader,
|
||||
&snapt.TableStateNode,
|
||||
&snapt.TableStorageNode,
|
||||
}
|
||||
)
|
||||
|
||||
func writeFiles(t *testing.T, dir string) *publisher {
|
||||
pub, err := NewPublisher(dir, nodeInfo)
|
||||
test.NoError(t, err)
|
||||
test.NoError(t, pub.PublishHeader(&fixt.Header1))
|
||||
tx, err := pub.BeginTx()
|
||||
test.NoError(t, err)
|
||||
|
||||
headerID := fixt.Header1.Hash().String()
|
||||
test.NoError(t, pub.PublishStateNode(&fixt.StateNode1, headerID, tx))
|
||||
|
||||
test.NoError(t, tx.Commit())
|
||||
return pub
|
||||
}
|
||||
|
||||
// verify that we can parse the csvs
|
||||
// TODO check actual data
|
||||
func verifyFileData(t *testing.T, path string, tbl *snapt.Table) {
|
||||
file, err := os.Open(path)
|
||||
test.NoError(t, err)
|
||||
r := csv.NewReader(file)
|
||||
test.NoError(t, err)
|
||||
r.FieldsPerRecord = len(tbl.Columns)
|
||||
|
||||
for {
|
||||
_, err := r.Read()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
test.NoError(t, err)
|
||||
// _, err = tbl.fromCsvRow(row)
|
||||
// test.NoError(t, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriting(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
// tempdir like /tmp/TempFoo/001/, TempFoo defaults to 0700
|
||||
test.NoError(t, os.Chmod(filepath.Dir(dir), 0755))
|
||||
// dir, _ := os.Getwd()
|
||||
// dir += "/dump"
|
||||
|
||||
pub := writeFiles(t, dir)
|
||||
|
||||
for _, tbl := range perBlockTables {
|
||||
verifyFileData(t, TableFile(pub.dir, tbl.Name), tbl)
|
||||
}
|
||||
for i := uint32(0); i < pub.txCounter; i++ {
|
||||
for _, tbl := range perNodeTables {
|
||||
verifyFileData(t, TableFile(pub.txDir(i), tbl.Name), tbl)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Note: DB user requires role membership "pg_read_server_files"
|
||||
func TestPgCopy(t *testing.T) {
|
||||
test.NeedsDB(t)
|
||||
|
||||
dir := t.TempDir()
|
||||
test.NoError(t, os.Chmod(filepath.Dir(dir), 0755))
|
||||
pub := writeFiles(t, dir)
|
||||
|
||||
ctx := context.Background()
|
||||
conn, err := pgx.Connect(ctx, pgConfig.DbConnectionString())
|
||||
test.NoError(t, err)
|
||||
|
||||
// clear existing test data
|
||||
pgDeleteTable := `DELETE FROM %s`
|
||||
for _, tbl := range allTables {
|
||||
_, err = conn.Exec(ctx, fmt.Sprintf(pgDeleteTable, tbl.Name))
|
||||
test.NoError(t, err)
|
||||
}
|
||||
|
||||
// copy from files
|
||||
pgCopyStatement := `COPY %s FROM '%s' CSV`
|
||||
for _, tbl := range perBlockTables {
|
||||
stm := fmt.Sprintf(pgCopyStatement, tbl.Name, TableFile(pub.dir, tbl.Name))
|
||||
_, err = conn.Exec(ctx, stm)
|
||||
test.NoError(t, err)
|
||||
}
|
||||
for i := uint32(0); i < pub.txCounter; i++ {
|
||||
for _, tbl := range perNodeTables {
|
||||
stm := fmt.Sprintf(pgCopyStatement, tbl.Name, TableFile(pub.txDir(i), tbl.Name))
|
||||
_, err = conn.Exec(ctx, stm)
|
||||
test.NoError(t, err)
|
||||
}
|
||||
}
|
||||
|
||||
// check header was successfully committed
|
||||
pgQueryHeader := `SELECT cid, block_hash
|
||||
FROM eth.header_cids
|
||||
WHERE block_number = $1`
|
||||
type res struct {
|
||||
CID string
|
||||
BlockHash string
|
||||
}
|
||||
var header res
|
||||
err = conn.QueryRow(ctx, pgQueryHeader, fixt.Header1.Number.Uint64()).Scan(
|
||||
&header.CID, &header.BlockHash)
|
||||
test.NoError(t, err)
|
||||
|
||||
headerNode, err := ipld.NewEthHeader(&fixt.Header1)
|
||||
test.ExpectEqual(t, headerNode.Cid().String(), header.CID)
|
||||
test.ExpectEqual(t, fixt.Header1.Hash().String(), header.BlockHash)
|
||||
}
|
||||
@ -1,64 +0,0 @@
|
||||
package mock
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/jmoiron/sqlx"
|
||||
|
||||
"github.com/golang/mock/gomock"
|
||||
|
||||
mocks "github.com/vulcanize/eth-pg-ipfs-state-snapshot/mocks/snapshot"
|
||||
snapt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/types"
|
||||
)
|
||||
|
||||
type MockPublisher struct {
|
||||
*mocks.MockPublisher
|
||||
}
|
||||
|
||||
func NewMockPublisher(t *testing.T) *MockPublisher {
|
||||
ctl := gomock.NewController(t)
|
||||
return &MockPublisher{mocks.NewMockPublisher(ctl)}
|
||||
}
|
||||
|
||||
func dump(funcname string, xs ...interface{}) {
|
||||
if true {
|
||||
return
|
||||
}
|
||||
fmt.Printf(">> %s", funcname)
|
||||
fmt.Printf(strings.Repeat(" %+v", len(xs))+"\n", xs...)
|
||||
}
|
||||
|
||||
func (p *MockPublisher) PublishHeader(header *types.Header) (int64, error) {
|
||||
// fmt.Printf("PublishHeader %+v\n", header)
|
||||
dump("PublishHeader", header)
|
||||
return p.MockPublisher.PublishHeader(header)
|
||||
}
|
||||
func (p *MockPublisher) PublishStateNode(node *snapt.Node, headerID int64, tx *sqlx.Tx) (int64, error) {
|
||||
dump("PublishStateNode", node, headerID)
|
||||
return p.MockPublisher.PublishStateNode(node, headerID, tx)
|
||||
|
||||
}
|
||||
func (p *MockPublisher) PublishStorageNode(node *snapt.Node, stateID int64, tx *sqlx.Tx) error {
|
||||
dump("PublishStorageNode", node, stateID)
|
||||
return p.MockPublisher.PublishStorageNode(node, stateID, tx)
|
||||
}
|
||||
func (p *MockPublisher) PublishCode(codeHash common.Hash, codeBytes []byte, tx *sqlx.Tx) error {
|
||||
dump("PublishCode", codeHash, codeBytes)
|
||||
return p.MockPublisher.PublishCode(codeHash, codeBytes, tx)
|
||||
}
|
||||
func (p *MockPublisher) BeginTx() (*sqlx.Tx, error) {
|
||||
dump("BeginTx")
|
||||
return p.MockPublisher.BeginTx()
|
||||
}
|
||||
func (p *MockPublisher) CommitTx(tx *sqlx.Tx) error {
|
||||
dump("CommitTx", tx)
|
||||
return p.MockPublisher.CommitTx(tx)
|
||||
}
|
||||
func (p *MockPublisher) PrepareTxForBatch(tx *sqlx.Tx, batchSize uint) (*sqlx.Tx, error) {
|
||||
dump("PrepareTxForBatch", tx, batchSize)
|
||||
return p.MockPublisher.PrepareTxForBatch(tx, batchSize)
|
||||
}
|
||||
@ -13,26 +13,31 @@
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package snapshot
|
||||
package pg
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/jmoiron/sqlx"
|
||||
"github.com/ipfs/go-cid"
|
||||
blockstore "github.com/ipfs/go-ipfs-blockstore"
|
||||
dshelp "github.com/ipfs/go-ipfs-ds-help"
|
||||
"github.com/multiformats/go-multihash"
|
||||
"github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/ipfs/ipld"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/shared"
|
||||
snapt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/types"
|
||||
)
|
||||
|
||||
var _ snapt.Publisher = (*publisher)(nil)
|
||||
|
||||
const logInterval = 1 * time.Minute
|
||||
|
||||
// Publisher is wrapper around DB.
|
||||
@ -49,127 +54,145 @@ type publisher struct {
|
||||
func NewPublisher(db *postgres.DB) *publisher {
|
||||
return &publisher{
|
||||
db: db,
|
||||
currBatchSize: 0,
|
||||
startTime: time.Now(),
|
||||
}
|
||||
}
|
||||
|
||||
func (p *publisher) BeginTx() (*sqlx.Tx, error) {
|
||||
tx, err := p.db.Beginx()
|
||||
type pubTx struct {
|
||||
sql.Tx
|
||||
callback func()
|
||||
}
|
||||
|
||||
func (tx pubTx) Rollback() error { return tx.Tx.Rollback(context.Background()) }
|
||||
func (tx pubTx) Commit() error {
|
||||
if tx.callback != nil {
|
||||
defer tx.callback()
|
||||
}
|
||||
return tx.Tx.Commit(context.Background())
|
||||
}
|
||||
func (tx pubTx) Exec(sql string, args ...interface{}) (sql.Result, error) {
|
||||
return tx.Tx.Exec(context.Background(), sql, args...)
|
||||
}
|
||||
|
||||
func (p *publisher) BeginTx() (snapt.Tx, error) {
|
||||
tx, err := p.db.Begin(context.Background())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
go p.logNodeCounters()
|
||||
return tx, nil
|
||||
}
|
||||
|
||||
func (p *publisher) CommitTx(tx *sqlx.Tx) error {
|
||||
return pubTx{tx, func() {
|
||||
logrus.Info("----- final counts -----")
|
||||
p.printNodeCounters()
|
||||
return tx.Commit()
|
||||
}}, nil
|
||||
}
|
||||
|
||||
// PublishRaw derives a cid from raw bytes and provided codec and multihash type, and writes it to the db tx
|
||||
// returns the CID and blockstore prefixed multihash key
|
||||
func (tx pubTx) publishRaw(codec uint64, raw []byte) (cid, prefixedKey string, err error) {
|
||||
c, err := ipld.RawdataToCid(codec, raw, multihash.KECCAK_256)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
cid = c.String()
|
||||
prefixedKey, err = tx.publishIPLD(c, raw)
|
||||
return
|
||||
}
|
||||
|
||||
func (tx pubTx) publishIPLD(c cid.Cid, raw []byte) (string, error) {
|
||||
dbKey := dshelp.MultihashToDsKey(c.Hash())
|
||||
prefixedKey := blockstore.BlockPrefix.String() + dbKey.String()
|
||||
_, err := tx.Exec(snapt.TableIPLDBlock.ToInsertStatement(), prefixedKey, raw)
|
||||
return prefixedKey, err
|
||||
}
|
||||
|
||||
// PublishHeader writes the header to the ipfs backing pg datastore and adds secondary indexes in the header_cids table
|
||||
func (p *publisher) PublishHeader(header *types.Header) (int64, error) {
|
||||
func (p *publisher) PublishHeader(header *types.Header) (err error) {
|
||||
headerNode, err := ipld.NewEthHeader(header)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
return err
|
||||
}
|
||||
|
||||
tx, err := p.db.Beginx()
|
||||
tx_, err := p.db.Begin(context.Background())
|
||||
if err != nil {
|
||||
return 0, err
|
||||
return err
|
||||
}
|
||||
tx := pubTx{tx_, nil}
|
||||
defer func() { err = snapt.CommitOrRollback(tx, err) }()
|
||||
|
||||
if _, err = tx.publishIPLD(headerNode.Cid(), headerNode.RawData()); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if p := recover(); p != nil {
|
||||
shared.Rollback(tx)
|
||||
panic(p)
|
||||
} else if err != nil {
|
||||
shared.Rollback(tx)
|
||||
} else {
|
||||
err = tx.Commit()
|
||||
}
|
||||
}()
|
||||
|
||||
if err = shared.PublishIPLD(tx, headerNode); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
mhKey, _ := shared.MultihashKeyFromCIDString(headerNode.Cid().String())
|
||||
var headerID int64
|
||||
err = tx.QueryRowx(`INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)
|
||||
ON CONFLICT (block_number, block_hash) DO UPDATE SET block_number = header_cids.block_number
|
||||
RETURNING id`,
|
||||
header.Number.Uint64(), header.Hash().Hex(), header.ParentHash.Hex(), headerNode.Cid().String(), "0", p.db.NodeID, "0", header.Root.Hex(), header.TxHash.Hex(),
|
||||
header.ReceiptHash.Hex(), header.UncleHash.Hex(), header.Bloom.Bytes(), header.Time, mhKey, 0).Scan(&headerID)
|
||||
|
||||
return headerID, err
|
||||
mhKey := shared.MultihashKeyFromCID(headerNode.Cid())
|
||||
_, err = tx.Exec(snapt.TableHeader.ToInsertStatement(), header.Number.Uint64(), header.Hash().Hex(),
|
||||
header.ParentHash.Hex(), headerNode.Cid().String(), "0", p.db.NodeID(), "0",
|
||||
header.Root.Hex(), header.TxHash.Hex(), header.ReceiptHash.Hex(), header.UncleHash.Hex(),
|
||||
header.Bloom.Bytes(), header.Time, mhKey, 0, header.Coinbase.String())
|
||||
return err
|
||||
}
|
||||
|
||||
// PublishStateNode writes the state node to the ipfs backing datastore and adds secondary indexes in the state_cids table
|
||||
func (p *publisher) PublishStateNode(node *snapt.Node, headerID int64, tx *sqlx.Tx) (int64, error) {
|
||||
var stateID int64
|
||||
func (p *publisher) PublishStateNode(node *snapt.Node, headerID string, tx_ snapt.Tx) error {
|
||||
var stateKey string
|
||||
if !bytes.Equal(node.Key.Bytes(), nullHash.Bytes()) {
|
||||
if !snapt.IsNullHash(node.Key) {
|
||||
stateKey = node.Key.Hex()
|
||||
}
|
||||
|
||||
stateCIDStr, mhKey, err := shared.PublishRaw(tx, ipld.MEthStateTrie, multihash.KECCAK_256, node.Value)
|
||||
tx := tx_.(pubTx)
|
||||
stateCIDStr, mhKey, err := tx.publishRaw(ipld.MEthStateTrie, node.Value)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
return err
|
||||
}
|
||||
|
||||
err = tx.QueryRowx(`INSERT INTO eth.state_cids (header_id, state_leaf_key, cid, state_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7)
|
||||
ON CONFLICT (header_id, state_path) DO UPDATE SET (state_leaf_key, cid, node_type, diff, mh_key) = ($2, $3, $5, $6, $7)
|
||||
RETURNING id`,
|
||||
headerID, stateKey, stateCIDStr, node.Path, node.NodeType, false, mhKey).Scan(&stateID)
|
||||
|
||||
_, err = tx.Exec(snapt.TableStateNode.ToInsertStatement(),
|
||||
headerID, stateKey, stateCIDStr, node.Path, node.NodeType, false, mhKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// increment state node counter.
|
||||
atomic.AddUint64(&p.stateNodeCounter, 1)
|
||||
|
||||
// increment current batch size counter
|
||||
p.currBatchSize += 2
|
||||
return stateID, err
|
||||
return err
|
||||
}
|
||||
|
||||
// PublishStorageNode writes the storage node to the ipfs backing pg datastore and adds secondary indexes in the storage_cids table
|
||||
func (p *publisher) PublishStorageNode(node *snapt.Node, stateID int64, tx *sqlx.Tx) error {
|
||||
func (p *publisher) PublishStorageNode(node *snapt.Node, headerID string, statePath []byte, tx_ snapt.Tx) error {
|
||||
var storageKey string
|
||||
if !bytes.Equal(node.Key.Bytes(), nullHash.Bytes()) {
|
||||
if !snapt.IsNullHash(node.Key) {
|
||||
storageKey = node.Key.Hex()
|
||||
}
|
||||
|
||||
storageCIDStr, mhKey, err := shared.PublishRaw(tx, ipld.MEthStorageTrie, multihash.KECCAK_256, node.Value)
|
||||
tx := tx_.(pubTx)
|
||||
storageCIDStr, mhKey, err := tx.publishRaw(ipld.MEthStorageTrie, node.Value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = tx.Exec(`INSERT INTO eth.storage_cids (state_id, storage_leaf_key, cid, storage_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7)
|
||||
ON CONFLICT (state_id, storage_path) DO UPDATE SET (storage_leaf_key, cid, node_type, diff, mh_key) = ($2, $3, $5, $6, $7)`,
|
||||
stateID, storageKey, storageCIDStr, node.Path, node.NodeType, false, mhKey)
|
||||
_, err = tx.Exec(snapt.TableStorageNode.ToInsertStatement(),
|
||||
headerID, statePath, storageKey, storageCIDStr, node.Path, node.NodeType, false, mhKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// increment storage node counter.
|
||||
atomic.AddUint64(&p.storageNodeCounter, 1)
|
||||
|
||||
// increment current batch size counter
|
||||
p.currBatchSize += 2
|
||||
return nil
|
||||
return err
|
||||
}
|
||||
|
||||
// PublishCode writes code to the ipfs backing pg datastore
|
||||
func (p *publisher) PublishCode(codeHash common.Hash, codeBytes []byte, tx *sqlx.Tx) error {
|
||||
func (p *publisher) PublishCode(codeHash common.Hash, codeBytes []byte, tx_ snapt.Tx) error {
|
||||
// no codec for code, doesn't matter though since blockstore key is multihash-derived
|
||||
mhKey, err := shared.MultihashKeyFromKeccak256(codeHash)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error deriving multihash key from codehash: %v", err)
|
||||
}
|
||||
|
||||
if err = shared.PublishDirect(tx, mhKey, codeBytes); err != nil {
|
||||
tx := tx_.(pubTx)
|
||||
if _, err = tx.Exec(snapt.TableIPLDBlock.ToInsertStatement(), mhKey, codeBytes); err != nil {
|
||||
return fmt.Errorf("error publishing code IPLD: %v", err)
|
||||
}
|
||||
|
||||
@ -180,7 +203,7 @@ func (p *publisher) PublishCode(codeHash common.Hash, codeBytes []byte, tx *sqlx
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *publisher) PrepareTxForBatch(tx *sqlx.Tx, maxBatchSize uint) (*sqlx.Tx, error) {
|
||||
func (p *publisher) PrepareTxForBatch(tx snapt.Tx, maxBatchSize uint) (snapt.Tx, error) {
|
||||
var err error
|
||||
// maximum batch size reached, commit the current transaction and begin a new transaction.
|
||||
if maxBatchSize <= p.currBatchSize {
|
||||
@ -188,7 +211,8 @@ func (p *publisher) PrepareTxForBatch(tx *sqlx.Tx, maxBatchSize uint) (*sqlx.Tx,
|
||||
return nil, err
|
||||
}
|
||||
|
||||
tx, err = p.db.Beginx()
|
||||
tx_, err := p.db.Begin(context.Background())
|
||||
tx = pubTx{Tx: tx_}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
78
pkg/snapshot/pg/publisher_test.go
Normal file
78
pkg/snapshot/pg/publisher_test.go
Normal file
@ -0,0 +1,78 @@
|
||||
package pg
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
|
||||
"github.com/jackc/pgx/v4"
|
||||
|
||||
fixt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/fixture"
|
||||
snapt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/types"
|
||||
"github.com/vulcanize/eth-pg-ipfs-state-snapshot/test"
|
||||
)
|
||||
|
||||
var (
|
||||
pgConfig = test.DefaultPgConfig
|
||||
nodeInfo = test.DefaultNodeInfo
|
||||
// tables ordered according to fkey depedencies
|
||||
allTables = []*snapt.Table{
|
||||
&snapt.TableIPLDBlock,
|
||||
&snapt.TableNodeInfo,
|
||||
&snapt.TableHeader,
|
||||
&snapt.TableStateNode,
|
||||
&snapt.TableStorageNode,
|
||||
}
|
||||
)
|
||||
|
||||
func writeData(t *testing.T) *publisher {
|
||||
driver, err := postgres.NewPGXDriver(context.Background(), pgConfig, nodeInfo)
|
||||
test.NoError(t, err)
|
||||
pub := NewPublisher(postgres.NewPostgresDB(driver))
|
||||
test.NoError(t, pub.PublishHeader(&fixt.Header1))
|
||||
tx, err := pub.BeginTx()
|
||||
test.NoError(t, err)
|
||||
|
||||
headerID := fixt.Header1.Hash().String()
|
||||
test.NoError(t, pub.PublishStateNode(&fixt.StateNode1, headerID, tx))
|
||||
|
||||
test.NoError(t, tx.Commit())
|
||||
return pub
|
||||
}
|
||||
|
||||
// Note: DB user requires role membership "pg_read_server_files"
|
||||
func TestBasic(t *testing.T) {
|
||||
test.NeedsDB(t)
|
||||
|
||||
ctx := context.Background()
|
||||
conn, err := pgx.Connect(ctx, pgConfig.DbConnectionString())
|
||||
test.NoError(t, err)
|
||||
|
||||
// clear existing test data
|
||||
pgDeleteTable := `DELETE FROM %s`
|
||||
for _, tbl := range allTables {
|
||||
_, err = conn.Exec(ctx, fmt.Sprintf(pgDeleteTable, tbl.Name))
|
||||
test.NoError(t, err)
|
||||
}
|
||||
|
||||
_ = writeData(t)
|
||||
|
||||
// check header was successfully committed
|
||||
pgQueryHeader := `SELECT cid, block_hash
|
||||
FROM eth.header_cids
|
||||
WHERE block_number = $1`
|
||||
type res struct {
|
||||
CID string
|
||||
BlockHash string
|
||||
}
|
||||
var header res
|
||||
err = conn.QueryRow(ctx, pgQueryHeader, fixt.Header1.Number.Uint64()).Scan(
|
||||
&header.CID, &header.BlockHash)
|
||||
test.NoError(t, err)
|
||||
|
||||
headerNode, err := ipld.NewEthHeader(&fixt.Header1)
|
||||
test.ExpectEqual(t, headerNode.Cid().String(), header.CID)
|
||||
test.ExpectEqual(t, fixt.Header1.Hash().String(), header.BlockHash)
|
||||
}
|
||||
@ -28,10 +28,7 @@ import (
|
||||
"github.com/ethereum/go-ethereum/crypto"
|
||||
"github.com/ethereum/go-ethereum/ethdb"
|
||||
"github.com/ethereum/go-ethereum/rlp"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/shared"
|
||||
"github.com/ethereum/go-ethereum/trie"
|
||||
"github.com/jmoiron/sqlx"
|
||||
"github.com/sirupsen/logrus"
|
||||
|
||||
. "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/types"
|
||||
@ -39,7 +36,6 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
nullHash = common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000")
|
||||
emptyNode, _ = rlp.EncodeToBytes([]byte{})
|
||||
emptyCodeHash = crypto.Keccak256([]byte{})
|
||||
emptyContractRoot = crypto.Keccak256Hash(emptyNode)
|
||||
@ -56,14 +52,6 @@ type Service struct {
|
||||
maxBatchSize uint
|
||||
}
|
||||
|
||||
func NewPostgresDB(con *DBConfig) (*postgres.DB, error) {
|
||||
pgDB, err := postgres.NewDB(con.URI, con.ConnConfig, con.Node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return pgDB, nil
|
||||
}
|
||||
|
||||
func NewLevelDB(con *EthConfig) (ethdb.Database, error) {
|
||||
return rawdb.NewLevelDBDatabaseWithFreezer(
|
||||
con.LevelDBPath, 1024, 256, con.AncientDBPath, "eth-pg-ipfs-state-snapshot", false,
|
||||
@ -97,7 +85,7 @@ func (s *Service) CreateSnapshot(params SnapshotParams) error {
|
||||
|
||||
logrus.Infof("head hash: %s head height: %d", hash.Hex(), params.Height)
|
||||
|
||||
headerID, err := s.ipfsPublisher.PublishHeader(header)
|
||||
err := s.ipfsPublisher.PublishHeader(header)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -106,6 +94,8 @@ func (s *Service) CreateSnapshot(params SnapshotParams) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
headerID := header.Hash().String()
|
||||
|
||||
if params.Workers > 0 {
|
||||
return s.createSnapshotAsync(t, headerID, params.Workers)
|
||||
} else {
|
||||
@ -135,7 +125,7 @@ func resolveNode(it trie.NodeIterator, trieDB *trie.Database) (*nodeResult, erro
|
||||
if it.Leaf() {
|
||||
return nil, nil
|
||||
}
|
||||
if bytes.Equal(nullHash.Bytes(), it.Hash().Bytes()) {
|
||||
if IsNullHash(it.Hash()) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
@ -163,22 +153,12 @@ func resolveNode(it trie.NodeIterator, trieDB *trie.Database) (*nodeResult, erro
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *Service) createSnapshot(it trie.NodeIterator, headerID int64) error {
|
||||
func (s *Service) createSnapshot(it trie.NodeIterator, headerID string) error {
|
||||
tx, err := s.ipfsPublisher.BeginTx()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if rec := recover(); rec != nil {
|
||||
shared.Rollback(tx)
|
||||
panic(rec)
|
||||
} else if err != nil {
|
||||
shared.Rollback(tx)
|
||||
} else {
|
||||
err = s.ipfsPublisher.CommitTx(tx)
|
||||
}
|
||||
}()
|
||||
defer func() { err = CommitOrRollback(tx, err) }()
|
||||
|
||||
for it.Next(true) {
|
||||
res, err := resolveNode(it, s.stateDB.TrieDB())
|
||||
@ -207,7 +187,7 @@ func (s *Service) createSnapshot(it trie.NodeIterator, headerID int64) error {
|
||||
encodedPath := trie.HexToCompact(valueNodePath)
|
||||
leafKey := encodedPath[1:]
|
||||
res.node.Key = common.BytesToHash(leafKey)
|
||||
stateID, err := s.ipfsPublisher.PublishStateNode(&res.node, headerID, tx)
|
||||
err := s.ipfsPublisher.PublishStateNode(&res.node, headerID, tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -226,12 +206,12 @@ func (s *Service) createSnapshot(it trie.NodeIterator, headerID int64) error {
|
||||
}
|
||||
}
|
||||
|
||||
if tx, err = s.storageSnapshot(account.Root, stateID, tx); err != nil {
|
||||
if tx, err = s.storageSnapshot(account.Root, headerID, res.node.Path, tx); err != nil {
|
||||
return fmt.Errorf("failed building storage snapshot for account %+v\r\nerror: %w", account, err)
|
||||
}
|
||||
case Extension, Branch:
|
||||
res.node.Key = common.BytesToHash([]byte{})
|
||||
if _, err := s.ipfsPublisher.PublishStateNode(&res.node, headerID, tx); err != nil {
|
||||
if err := s.ipfsPublisher.PublishStateNode(&res.node, headerID, tx); err != nil {
|
||||
return err
|
||||
}
|
||||
default:
|
||||
@ -243,7 +223,7 @@ func (s *Service) createSnapshot(it trie.NodeIterator, headerID int64) error {
|
||||
}
|
||||
|
||||
// Full-trie concurrent snapshot
|
||||
func (s *Service) createSnapshotAsync(tree state.Trie, headerID int64, workers uint) error {
|
||||
func (s *Service) createSnapshotAsync(tree state.Trie, headerID string, workers uint) error {
|
||||
errors := make(chan error)
|
||||
var wg sync.WaitGroup
|
||||
for _, it := range iter.SubtrieIterators(tree, workers) {
|
||||
@ -267,7 +247,7 @@ func (s *Service) createSnapshotAsync(tree state.Trie, headerID int64, workers u
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Service) storageSnapshot(sr common.Hash, stateID int64, tx *sqlx.Tx) (*sqlx.Tx, error) {
|
||||
func (s *Service) storageSnapshot(sr common.Hash, headerID string, statePath []byte, tx Tx) (Tx, error) {
|
||||
if bytes.Equal(sr.Bytes(), emptyContractRoot.Bytes()) {
|
||||
return tx, nil
|
||||
}
|
||||
@ -311,7 +291,7 @@ func (s *Service) storageSnapshot(sr common.Hash, stateID int64, tx *sqlx.Tx) (*
|
||||
default:
|
||||
return nil, errors.New("unexpected node type")
|
||||
}
|
||||
if err = s.ipfsPublisher.PublishStorageNode(&res.node, stateID, tx); err != nil {
|
||||
if err = s.ipfsPublisher.PublishStorageNode(&res.node, headerID, statePath, tx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,84 +1,57 @@
|
||||
package snapshot
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/golang/mock/gomock"
|
||||
|
||||
ethNode "github.com/ethereum/go-ethereum/statediff/indexer/node"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
|
||||
|
||||
fixt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/fixture"
|
||||
"github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/snapshot/mock"
|
||||
mock "github.com/vulcanize/eth-pg-ipfs-state-snapshot/mocks/snapshot"
|
||||
"github.com/vulcanize/eth-pg-ipfs-state-snapshot/test"
|
||||
)
|
||||
|
||||
func testConfig(leveldbpath, ancientdbpath string) *Config {
|
||||
dbParams := postgres.ConnectionParams{
|
||||
Name: "snapshot_test",
|
||||
Hostname: "localhost",
|
||||
Port: 5432,
|
||||
User: "tester",
|
||||
Password: "test_pw",
|
||||
}
|
||||
connconfig := postgres.ConnectionConfig{
|
||||
MaxIdle: 0,
|
||||
MaxLifetime: 0,
|
||||
MaxOpen: 4,
|
||||
}
|
||||
nodeinfo := ethNode.Info{
|
||||
ID: "eth_node_id",
|
||||
ClientName: "eth_client",
|
||||
GenesisBlock: "X",
|
||||
NetworkID: "eth_network",
|
||||
ChainID: 0,
|
||||
}
|
||||
|
||||
return &Config{
|
||||
DB: &DBConfig{
|
||||
Node: nodeinfo,
|
||||
URI: postgres.DbConnectionString(dbParams),
|
||||
ConnConfig: connconfig,
|
||||
},
|
||||
Eth: &EthConfig{
|
||||
LevelDBPath: leveldbpath,
|
||||
AncientDBPath: ancientdbpath,
|
||||
NodeInfo: test.DefaultNodeInfo,
|
||||
},
|
||||
DB: &DBConfig{
|
||||
URI: test.DefaultPgConfig.DbConnectionString(),
|
||||
ConnConfig: test.DefaultPgConfig,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateSnapshot(t *testing.T) {
|
||||
wd, err := os.Getwd()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
fixture_path := filepath.Join(wd, "..", "..", "fixture")
|
||||
datadir := filepath.Join(fixture_path, "chaindata")
|
||||
if _, err := os.Stat(datadir); err != nil {
|
||||
t.Fatal("no chaindata:", err)
|
||||
}
|
||||
config := testConfig(datadir, filepath.Join(datadir, "ancient"))
|
||||
fmt.Printf("config: %+v %+v\n", config.DB, config.Eth)
|
||||
config := testConfig(fixt.ChaindataPath, fixt.AncientdataPath)
|
||||
|
||||
edb, err := NewLevelDB(config.Eth)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
workers := 8
|
||||
workers := 4
|
||||
|
||||
pub := mock.NewMockPublisher(t)
|
||||
pub.EXPECT().PublishHeader(gomock.Eq(fixt.PublishHeader))
|
||||
ctl := gomock.NewController(t)
|
||||
tx := mock.NewMockTx(ctl)
|
||||
pub := mock.NewMockPublisher(ctl)
|
||||
|
||||
pub.EXPECT().PublishHeader(gomock.Eq(&fixt.Header1))
|
||||
pub.EXPECT().BeginTx().
|
||||
Return(tx, nil).
|
||||
Times(workers)
|
||||
pub.EXPECT().PrepareTxForBatch(gomock.Any(), gomock.Any()).
|
||||
Return(tx, nil).
|
||||
Times(workers)
|
||||
pub.EXPECT().PublishStateNode(gomock.Any(), mock.AnyOf(int64(0), int64(1)), gomock.Any()).
|
||||
pub.EXPECT().PublishStateNode(gomock.Any(), gomock.Any(), gomock.Any()).
|
||||
Times(workers)
|
||||
// TODO: fixtures for storage node
|
||||
// pub.EXPECT().PublishStorageNode(gomock.Eq(fixt.StorageNode), gomock.Eq(int64(0)), gomock.Any())
|
||||
pub.EXPECT().CommitTx(gomock.Any()).
|
||||
// pub.EXPECT().CommitTx(gomock.Any()).
|
||||
// Times(workers)
|
||||
|
||||
tx.EXPECT().Commit().
|
||||
Times(workers)
|
||||
|
||||
service, err := NewSnapshotService(edb, pub)
|
||||
|
||||
26
pkg/snapshot/util.go
Normal file
26
pkg/snapshot/util.go
Normal file
@ -0,0 +1,26 @@
|
||||
package snapshot
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
|
||||
|
||||
file "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/snapshot/file"
|
||||
pg "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/snapshot/pg"
|
||||
snapt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/types"
|
||||
)
|
||||
|
||||
func NewPublisher(mode SnapshotMode, config *Config) (snapt.Publisher, error) {
|
||||
switch mode {
|
||||
case PgSnapshot:
|
||||
driver, err := postgres.NewPGXDriver(context.Background(), config.DB.ConnConfig, config.Eth.NodeInfo)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return pg.NewPublisher(postgres.NewPostgresDB(driver)), nil
|
||||
case FileSnapshot:
|
||||
return file.NewPublisher(config.File.OutputDir, config.Eth.NodeInfo)
|
||||
}
|
||||
return nil, fmt.Errorf("invalid snapshot mode: %s", mode)
|
||||
}
|
||||
@ -22,13 +22,12 @@ import (
|
||||
)
|
||||
|
||||
// node for holding trie node information
|
||||
type node struct {
|
||||
type Node struct {
|
||||
NodeType nodeType
|
||||
Path []byte
|
||||
Key common.Hash
|
||||
Value []byte
|
||||
}
|
||||
type Node = node
|
||||
|
||||
// nodeType for explicitly setting type of node
|
||||
type nodeType int
|
||||
|
||||
@ -3,15 +3,18 @@ package types
|
||||
import (
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/jmoiron/sqlx"
|
||||
)
|
||||
|
||||
type Publisher interface {
|
||||
PublishHeader(header *types.Header) (int64, error)
|
||||
PublishStateNode(node *Node, headerID int64, tx *sqlx.Tx) (int64, error)
|
||||
PublishStorageNode(node *Node, stateID int64, tx *sqlx.Tx) error
|
||||
PublishCode(codeHash common.Hash, codeBytes []byte, tx *sqlx.Tx) error
|
||||
BeginTx() (*sqlx.Tx, error)
|
||||
CommitTx(*sqlx.Tx) error
|
||||
PrepareTxForBatch(tx *sqlx.Tx, batchSize uint) (*sqlx.Tx, error)
|
||||
PublishHeader(header *types.Header) error
|
||||
PublishStateNode(node *Node, headerID string, tx Tx) error
|
||||
PublishStorageNode(node *Node, headerID string, statePath []byte, tx Tx) error
|
||||
PublishCode(codeHash common.Hash, codeBytes []byte, tx Tx) error
|
||||
BeginTx() (Tx, error)
|
||||
PrepareTxForBatch(tx Tx, batchSize uint) (Tx, error)
|
||||
}
|
||||
|
||||
type Tx interface {
|
||||
Rollback() error
|
||||
Commit() error
|
||||
}
|
||||
|
||||
73
pkg/types/schema.go
Normal file
73
pkg/types/schema.go
Normal file
@ -0,0 +1,73 @@
|
||||
package types
|
||||
|
||||
var TableIPLDBlock = Table{
|
||||
`public.blocks`,
|
||||
[]column{
|
||||
column{"key", text},
|
||||
column{"data", bytea},
|
||||
},
|
||||
`ON CONFLICT (key) DO NOTHING`,
|
||||
}
|
||||
|
||||
var TableNodeInfo = Table{
|
||||
Name: `public.nodes`,
|
||||
Columns: []column{
|
||||
column{"genesis_block", varchar},
|
||||
column{"network_id", varchar},
|
||||
column{"node_id", varchar},
|
||||
column{"client_name", varchar},
|
||||
column{"chain_id", integer},
|
||||
},
|
||||
}
|
||||
|
||||
var TableHeader = Table{
|
||||
"eth.header_cids",
|
||||
[]column{
|
||||
column{"block_number", bigint},
|
||||
column{"block_hash", varchar},
|
||||
column{"parent_hash", varchar},
|
||||
column{"cid", text},
|
||||
column{"td", numeric},
|
||||
column{"node_id", varchar},
|
||||
column{"reward", numeric},
|
||||
column{"state_root", varchar},
|
||||
column{"tx_root", varchar},
|
||||
column{"receipt_root", varchar},
|
||||
column{"uncle_root", varchar},
|
||||
column{"bloom", bytea},
|
||||
column{"timestamp", numeric},
|
||||
column{"mh_key", text},
|
||||
column{"times_validated", integer},
|
||||
column{"coinbase", varchar},
|
||||
},
|
||||
"ON CONFLICT (block_hash) DO UPDATE SET (parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated, coinbase) = (EXCLUDED.parent_hash, EXCLUDED.cid, EXCLUDED.td, EXCLUDED.node_id, EXCLUDED.reward, EXCLUDED.state_root, EXCLUDED.tx_root, EXCLUDED.receipt_root, EXCLUDED.uncle_root, EXCLUDED.bloom, EXCLUDED.timestamp, EXCLUDED.mh_key, eth.header_cids.times_validated + 1, EXCLUDED.coinbase)",
|
||||
}
|
||||
|
||||
var TableStateNode = Table{
|
||||
"eth.state_cids",
|
||||
[]column{
|
||||
column{"header_id", varchar},
|
||||
column{"state_leaf_key", varchar},
|
||||
column{"cid", text},
|
||||
column{"state_path", bytea},
|
||||
column{"node_type", integer},
|
||||
column{"diff", boolean},
|
||||
column{"mh_key", text},
|
||||
},
|
||||
`ON CONFLICT (header_id, state_path) DO UPDATE SET (state_leaf_key, cid, node_type, diff, mh_key) = (EXCLUDED.state_leaf_key, EXCLUDED.cid, EXCLUDED.node_type, EXCLUDED.diff, EXCLUDED.mh_key)`,
|
||||
}
|
||||
|
||||
var TableStorageNode = Table{
|
||||
"eth.storage_cids",
|
||||
[]column{
|
||||
column{"header_id", varchar},
|
||||
column{"state_path", bytea},
|
||||
column{"storage_leaf_key", varchar},
|
||||
column{"cid", text},
|
||||
column{"storage_path", bytea},
|
||||
column{"node_type", integer},
|
||||
column{"diff", boolean},
|
||||
column{"mh_key", text},
|
||||
},
|
||||
"ON CONFLICT (header_id, state_path, storage_path) DO UPDATE SET (storage_leaf_key, cid, node_type, diff, mh_key) = (EXCLUDED.storage_leaf_key, EXCLUDED.cid, EXCLUDED.node_type, EXCLUDED.diff, EXCLUDED.mh_key)",
|
||||
}
|
||||
79
pkg/types/table.go
Normal file
79
pkg/types/table.go
Normal file
@ -0,0 +1,79 @@
|
||||
package types
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type colType int
|
||||
|
||||
const (
|
||||
integer colType = iota
|
||||
boolean
|
||||
bigint
|
||||
numeric
|
||||
bytea
|
||||
varchar
|
||||
text
|
||||
)
|
||||
|
||||
type column struct {
|
||||
name string
|
||||
typ colType
|
||||
}
|
||||
type Table struct {
|
||||
Name string
|
||||
Columns []column
|
||||
conflictClause string
|
||||
}
|
||||
|
||||
func (tbl *Table) ToCsvRow(args ...interface{}) []string {
|
||||
var row []string
|
||||
for i, col := range tbl.Columns {
|
||||
row = append(row, col.typ.formatter()(args[i]))
|
||||
}
|
||||
return row
|
||||
}
|
||||
|
||||
func (tbl *Table) ToInsertStatement() string {
|
||||
var colnames, placeholders []string
|
||||
for i, col := range tbl.Columns {
|
||||
colnames = append(colnames, col.name)
|
||||
placeholders = append(placeholders, fmt.Sprintf("$%d", i+1))
|
||||
}
|
||||
return fmt.Sprintf(
|
||||
"INSERT INTO %s (%s) VALUES (%s) %s",
|
||||
tbl.Name, strings.Join(colnames, ", "), strings.Join(placeholders, ", "), tbl.conflictClause,
|
||||
)
|
||||
}
|
||||
|
||||
type colfmt = func(interface{}) string
|
||||
|
||||
func sprintf(f string) colfmt {
|
||||
return func(x interface{}) string { return fmt.Sprintf(f, x) }
|
||||
}
|
||||
|
||||
func (typ colType) formatter() colfmt {
|
||||
switch typ {
|
||||
case integer:
|
||||
return sprintf("%d")
|
||||
case boolean:
|
||||
return func(x interface{}) string {
|
||||
if x.(bool) {
|
||||
return "t"
|
||||
}
|
||||
return "f"
|
||||
}
|
||||
case bigint:
|
||||
return sprintf("%s")
|
||||
case numeric:
|
||||
return sprintf("%d")
|
||||
case bytea:
|
||||
return sprintf(`\x%x`)
|
||||
case varchar:
|
||||
return sprintf("%s")
|
||||
case text:
|
||||
return sprintf("%s")
|
||||
}
|
||||
panic("unreachable")
|
||||
}
|
||||
33
pkg/types/util.go
Normal file
33
pkg/types/util.go
Normal file
@ -0,0 +1,33 @@
|
||||
package types
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
)
|
||||
|
||||
var nullHash = common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000")
|
||||
|
||||
func IsNullHash(hash common.Hash) bool {
|
||||
return bytes.Equal(hash.Bytes(), nullHash.Bytes())
|
||||
}
|
||||
|
||||
func CommitOrRollback(tx Tx, err error) error {
|
||||
var rberr error
|
||||
defer func() {
|
||||
if rberr != nil {
|
||||
logrus.Errorf("rollback failed: %s", rberr)
|
||||
}
|
||||
}()
|
||||
if rec := recover(); rec != nil {
|
||||
rberr = tx.Rollback()
|
||||
panic(rec)
|
||||
} else if err != nil {
|
||||
rberr = tx.Rollback()
|
||||
} else {
|
||||
err = tx.Commit()
|
||||
}
|
||||
return err
|
||||
}
|
||||
52
test/helper.go
Normal file
52
test/helper.go
Normal file
@ -0,0 +1,52 @@
|
||||
package test
|
||||
|
||||
import (
|
||||
"os"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
|
||||
ethnode "github.com/ethereum/go-ethereum/statediff/indexer/node"
|
||||
)
|
||||
|
||||
var (
|
||||
DefaultNodeInfo = ethnode.Info{
|
||||
ID: "test_nodeid",
|
||||
ClientName: "test_client",
|
||||
GenesisBlock: "TEST_GENESIS",
|
||||
NetworkID: "test_network",
|
||||
ChainID: 0,
|
||||
}
|
||||
DefaultPgConfig = postgres.Config{
|
||||
Hostname: "localhost",
|
||||
Port: 5432,
|
||||
DatabaseName: "vulcanize_test",
|
||||
Username: "vulcanize",
|
||||
Password: "vulcanize_password",
|
||||
|
||||
MaxIdle: 0,
|
||||
MaxConnLifetime: 0,
|
||||
MaxConns: 4,
|
||||
}
|
||||
)
|
||||
|
||||
func NeedsDB(t *testing.T) {
|
||||
t.Helper()
|
||||
if os.Getenv("TEST_WITH_DB") == "" {
|
||||
t.Skip("set TEST_WITH_DB to enable test")
|
||||
}
|
||||
}
|
||||
|
||||
func NoError(t *testing.T, err error) {
|
||||
t.Helper()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// ExpectEqual asserts the provided interfaces are deep equal
|
||||
func ExpectEqual(t *testing.T, want interface{}, got interface{}) {
|
||||
if !reflect.DeepEqual(want, got) {
|
||||
t.Fatalf("Expected: %v\nActual: %v", want, got)
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user