From 91cd784baa514b7a70e00c4193ba7da33c68d7ae Mon Sep 17 00:00:00 2001 From: Ashwin Phatak Date: Wed, 25 Aug 2021 13:50:31 +0530 Subject: [PATCH] Use groupcache based ethdb/validator. (#12) * Use groupcache based ethdb/validator. * Use latest ipfs-ethdb version --- cmd/validateTrie.go | 7 +++++-- go.mod | 3 ++- go.sum | 11 ++++++++--- pkg/validator.go | 27 +++++++++++++++++++++++---- 4 files changed, 38 insertions(+), 10 deletions(-) diff --git a/cmd/validateTrie.go b/cmd/validateTrie.go index 279e948..8f2fe37 100644 --- a/cmd/validateTrie.go +++ b/cmd/validateTrie.go @@ -24,7 +24,7 @@ import ( "github.com/spf13/cobra" "github.com/spf13/viper" - "github.com/vulcanize/eth-ipfs-state-validator/pkg" + validator "github.com/vulcanize/eth-ipfs-state-validator/pkg" ) // validateTrieCmd represents the validateTrie command @@ -35,7 +35,7 @@ var validateTrieCmd = &cobra.Command{ If an ipfs-path is provided it will use a blockservice, otherwise it expects Postgres db configuration in a linked config file. -It can operate at three levels: +It can operate at three levels: "full" validates completeness of the entire state corresponding to a provided state root, including both state and storage tries @@ -99,6 +99,9 @@ func validateTrie() { } logWithCommand.Infof("Storage trie for contract %s and root %s is complete", addr.String(), storageRoot.String()) } + + stats := v.GetCacheStats() + logWithCommand.Debugf("groupcache stats %+v", stats) } func newValidator() (*validator.Validator, error) { diff --git a/go.mod b/go.mod index 87b12e0..bb4834e 100644 --- a/go.mod +++ b/go.mod @@ -12,11 +12,12 @@ require ( github.com/ipfs/go-ipfs-ds-help v1.0.0 github.com/jmoiron/sqlx v1.2.0 github.com/lib/pq v1.5.2 + github.com/mailgun/groupcache/v2 v2.2.1 github.com/multiformats/go-multihash v0.0.14 github.com/onsi/ginkgo v1.14.0 github.com/onsi/gomega v1.10.1 github.com/sirupsen/logrus v1.6.0 github.com/spf13/cobra v1.0.0 github.com/spf13/viper v1.7.0 - github.com/vulcanize/ipfs-ethdb v0.0.2 + github.com/vulcanize/ipfs-ethdb v0.0.3 ) diff --git a/go.sum b/go.sum index e43972d..3c0ad27 100644 --- a/go.sum +++ b/go.sum @@ -837,6 +837,8 @@ github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mailgun/groupcache/v2 v2.2.1 h1:OalhvLqdhiHd76CU8gn+w6UqeEa1m60ChrVwwKkmqh4= +github.com/mailgun/groupcache/v2 v2.2.1/go.mod h1:fgFJNRQar4yVloM0SzqWhOuTF83HCO5DDXVnZQVVJ58= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/marten-seemann/qpack v0.2.0/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc= @@ -1037,6 +1039,8 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/segmentio/fasthash v1.0.3 h1:EI9+KE1EwvMLBWwjpRDc+fEM+prwxDYbslddQGtrmhM= +github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shirou/gopsutil v2.20.5-0.20200531151128-663af789c085+incompatible h1:+gAR1bMhuoQnZMTWFIvp7ukynULPsteLzG+siZKLtD8= github.com/shirou/gopsutil v2.20.5-0.20200531151128-663af789c085+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= @@ -1131,8 +1135,8 @@ github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljT github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= -github.com/vulcanize/ipfs-ethdb v0.0.2 h1:Ogp9k+OMKIn2xo9Y9VoNTjd5v1kXcjL+k3cDQwqpUPU= -github.com/vulcanize/ipfs-ethdb v0.0.2/go.mod h1:9yDJ8PCVKK4auwM8KgVLVkGM05Atp2OITJvt6cSvsHs= +github.com/vulcanize/ipfs-ethdb v0.0.3 h1:Rflz7/PbZUao7Oyv9iy4VMhynDuxZ8Yy0SypovRqO8c= +github.com/vulcanize/ipfs-ethdb v0.0.3/go.mod h1:VHyO1HhUYRtxsQ0EpF06VGGP8w4zGjvd+hXL6GK35xw= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a h1:G++j5e0OC488te356JvdhaM8YS6nMsjLAYF7JxCv07w= @@ -1423,8 +1427,9 @@ google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= diff --git a/pkg/validator.go b/pkg/validator.go index 144b263..d80aab4 100644 --- a/pkg/validator.go +++ b/pkg/validator.go @@ -17,6 +17,8 @@ package validator import ( + "time" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/crypto" @@ -24,9 +26,10 @@ import ( "github.com/ethereum/go-ethereum/trie" "github.com/ipfs/go-blockservice" "github.com/jmoiron/sqlx" + "github.com/mailgun/groupcache/v2" - "github.com/vulcanize/ipfs-ethdb" - "github.com/vulcanize/ipfs-ethdb/postgres" + ipfsethdb "github.com/vulcanize/ipfs-ethdb" + pgipfsethdb "github.com/vulcanize/ipfs-ethdb/postgres" ) // Validator is used for validating Ethereum state and storage tries on PG-IPFS @@ -34,19 +37,35 @@ type Validator struct { kvs ethdb.KeyValueStore trieDB *trie.Database stateDatabase state.Database + db *pgipfsethdb.Database } // NewPGIPFSValidator returns a new trie validator ontop of a connection pool for an IPFS backing Postgres database func NewPGIPFSValidator(db *sqlx.DB) *Validator { - kvs := pgipfsethdb.NewKeyValueStore(db) - database := pgipfsethdb.NewDatabase(db) + kvs := pgipfsethdb.NewKeyValueStore(db, pgipfsethdb.CacheConfig{ + Name: "kv", + Size: 16 * 1000 * 1000, // 16MB + ExpiryDuration: time.Hour * 8, // 8 hours + }) + + database := pgipfsethdb.NewDatabase(db, pgipfsethdb.CacheConfig{ + Name: "db", + Size: 16 * 1000 * 1000, // 16MB + ExpiryDuration: time.Hour * 8, // 8 hours + }) + return &Validator{ kvs: kvs, trieDB: trie.NewDatabase(kvs), stateDatabase: state.NewDatabase(database), + db: database, } } +func (v *Validator) GetCacheStats() groupcache.Stats { + return v.db.GetCacheStats() +} + // NewIPFSValidator returns a new trie validator ontop of an IPFS blockservice func NewIPFSValidator(bs blockservice.BlockService) *Validator { kvs := ipfsethdb.NewKeyValueStore(bs)