cerc-io/eth-ipfs-state-validator
16
0
Fork 1
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

Use groupcache based ethdb/validator. (#12)

Browse Source 
* Use groupcache based ethdb/validator.

* Use latest ipfs-ethdb version
This commit is contained in:
Ashwin Phatak 2021-08-25 13:50:31 +05:30 committed by GitHub
parent 12ee1f4947
commit 91cd784baa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 38 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
cmd/validateTrie.go
View File

@ -24,7 +24,7 @@ import (
"github.com/spf13/cobra"
"github.com/spf13/viper"
"github.com/vulcanize/eth-ipfs-state-validator/pkg"
validator "github.com/vulcanize/eth-ipfs-state-validator/pkg"
)
// validateTrieCmd represents the validateTrie command
@ -35,7 +35,7 @@ var validateTrieCmd = &cobra.Command{
If an ipfs-path is provided it will use a blockservice, otherwise it expects Postgres db configuration in a linked config file.
It can operate at three levels:
It can operate at three levels:
"full" validates completeness of the entire state corresponding to a provided state root, including both state and storage tries
@ -99,6 +99,9 @@ func validateTrie() {
}
logWithCommand.Infof("Storage trie for contract %s and root %s is complete", addr.String(), storageRoot.String())
}
stats := v.GetCacheStats()
logWithCommand.Debugf("groupcache stats %+v", stats)
}
func newValidator() (*validator.Validator, error) {

3
go.mod
View File

@ -12,11 +12,12 @@ require (
github.com/ipfs/go-ipfs-ds-help v1.0.0
github.com/jmoiron/sqlx v1.2.0
github.com/lib/pq v1.5.2
github.com/mailgun/groupcache/v2 v2.2.1
github.com/multiformats/go-multihash v0.0.14
github.com/onsi/ginkgo v1.14.0
github.com/onsi/gomega v1.10.1
github.com/sirupsen/logrus v1.6.0
github.com/spf13/cobra v1.0.0
github.com/spf13/viper v1.7.0
github.com/vulcanize/ipfs-ethdb v0.0.2
github.com/vulcanize/ipfs-ethdb v0.0.3
)

11
go.sum
View File

@ -837,6 +837,8 @@ github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/mailgun/groupcache/v2 v2.2.1 h1:OalhvLqdhiHd76CU8gn+w6UqeEa1m60ChrVwwKkmqh4=
github.com/mailgun/groupcache/v2 v2.2.1/go.mod h1:fgFJNRQar4yVloM0SzqWhOuTF83HCO5DDXVnZQVVJ58=
github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/marten-seemann/qpack v0.2.0/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc=
@ -1037,6 +1039,8 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/segmentio/fasthash v1.0.3 h1:EI9+KE1EwvMLBWwjpRDc+fEM+prwxDYbslddQGtrmhM=
github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/shirou/gopsutil v2.20.5-0.20200531151128-663af789c085+incompatible h1:+gAR1bMhuoQnZMTWFIvp7ukynULPsteLzG+siZKLtD8=
github.com/shirou/gopsutil v2.20.5-0.20200531151128-663af789c085+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
@ -1131,8 +1135,8 @@ github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljT
github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
github.com/vulcanize/ipfs-ethdb v0.0.2 h1:Ogp9k+OMKIn2xo9Y9VoNTjd5v1kXcjL+k3cDQwqpUPU=
github.com/vulcanize/ipfs-ethdb v0.0.2/go.mod h1:9yDJ8PCVKK4auwM8KgVLVkGM05Atp2OITJvt6cSvsHs=
github.com/vulcanize/ipfs-ethdb v0.0.3 h1:Rflz7/PbZUao7Oyv9iy4VMhynDuxZ8Yy0SypovRqO8c=
github.com/vulcanize/ipfs-ethdb v0.0.3/go.mod h1:VHyO1HhUYRtxsQ0EpF06VGGP8w4zGjvd+hXL6GK35xw=
github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a h1:G++j5e0OC488te356JvdhaM8YS6nMsjLAYF7JxCv07w=
@ -1423,8 +1427,9 @@ google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM=
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=

27
pkg/validator.go
View File

@ -17,6 +17,8 @@
package validator
import (
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/crypto"
@ -24,9 +26,10 @@ import (
"github.com/ethereum/go-ethereum/trie"
"github.com/ipfs/go-blockservice"
"github.com/jmoiron/sqlx"
"github.com/mailgun/groupcache/v2"
"github.com/vulcanize/ipfs-ethdb"
"github.com/vulcanize/ipfs-ethdb/postgres"
ipfsethdb "github.com/vulcanize/ipfs-ethdb"
pgipfsethdb "github.com/vulcanize/ipfs-ethdb/postgres"
)
// Validator is used for validating Ethereum state and storage tries on PG-IPFS
@ -34,19 +37,35 @@ type Validator struct {
kvs ethdb.KeyValueStore
trieDB *trie.Database
stateDatabase state.Database
db *pgipfsethdb.Database
}
// NewPGIPFSValidator returns a new trie validator ontop of a connection pool for an IPFS backing Postgres database
func NewPGIPFSValidator(db *sqlx.DB) *Validator {
kvs := pgipfsethdb.NewKeyValueStore(db)
database := pgipfsethdb.NewDatabase(db)
kvs := pgipfsethdb.NewKeyValueStore(db, pgipfsethdb.CacheConfig{
Name: "kv",
Size: 16 * 1000 * 1000, // 16MB
ExpiryDuration: time.Hour * 8, // 8 hours
})
database := pgipfsethdb.NewDatabase(db, pgipfsethdb.CacheConfig{
Name: "db",
Size: 16 * 1000 * 1000, // 16MB
ExpiryDuration: time.Hour * 8, // 8 hours
})
return &Validator{
kvs: kvs,
trieDB: trie.NewDatabase(kvs),
stateDatabase: state.NewDatabase(database),
db: database,
}
}
func (v *Validator) GetCacheStats() groupcache.Stats {
return v.db.GetCacheStats()
}
// NewIPFSValidator returns a new trie validator ontop of an IPFS blockservice
func NewIPFSValidator(bs blockservice.BlockService) *Validator {
kvs := ipfsethdb.NewKeyValueStore(bs)