diff --git a/cmd/serve.go b/cmd/serve.go index a5b9bdb3..4662d538 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -209,25 +209,29 @@ func startIpldGraphQL(settings *s.Config) error { func startGroupCacheService(settings *s.Config) error { gcc := settings.GroupCache - pool := groupcache.NewHTTPPoolOpts(gcc.Pool.HttpEndpoint, &groupcache.HTTPPoolOptions{}) - pool.Set(gcc.Pool.PeerHttpEndpoints...) + if gcc.Pool.Enabled { + logWithCommand.Info("starting up groupcache pool HTTTP server") - httpURL, err := url.Parse(gcc.Pool.HttpEndpoint) - if err != nil { - return err - } + pool := groupcache.NewHTTPPoolOpts(gcc.Pool.HttpEndpoint, &groupcache.HTTPPoolOptions{}) + pool.Set(gcc.Pool.PeerHttpEndpoints...) - server := http.Server{ - Addr: httpURL.Host, - Handler: pool, - } - - // Start a HTTP server to listen for peer requests from the groupcache - go func() { - if err := server.ListenAndServe(); err != nil { - logWithCommand.Fatal(err) + httpURL, err := url.Parse(gcc.Pool.HttpEndpoint) + if err != nil { + return err } - }() + + server := http.Server{ + Addr: httpURL.Host, + Handler: pool, + } + + // Start a HTTP server to listen for peer requests from the groupcache + go server.ListenAndServe() + + logWithCommand.Infof("groupcache pool endpoint opened for url %s", httpURL) + } else { + logWithCommand.Info("Groupcache pool is disabled") + } return nil } @@ -292,6 +296,14 @@ func init() { serveCmd.PersistentFlags().String("eth-chain-config", "", "json chain config file location") serveCmd.PersistentFlags().Bool("eth-supports-state-diff", false, "whether or not the proxy ethereum client supports statediffing endpoints") + // groupcache flags + serveCmd.PersistentFlags().Bool("gcache-pool-enabled", false, "turn on the groupcache pool") + serveCmd.PersistentFlags().String("gcache-pool-http-path", "", "http url for groupcache node") + serveCmd.PersistentFlags().StringArray("gcache-pool-http-peers", []string{}, "http urls for groupcache peers") + serveCmd.PersistentFlags().Int("gcache-statedb-cache-size", 16, "state DB cache size in MB") + serveCmd.PersistentFlags().Int("gcache-statedb-cache-expiry", 60, "state DB cache expiry time in mins") + serveCmd.PersistentFlags().Int("gcache-statedb-log-stats-interval", 60, "state DB cache stats log interval in secs") + // and their bindings // database viper.BindPFlag("database.name", serveCmd.PersistentFlags().Lookup("database-name")) @@ -333,4 +345,12 @@ func init() { viper.BindPFlag("ethereum.rpcGasCap", serveCmd.PersistentFlags().Lookup("eth-rpc-gas-cap")) viper.BindPFlag("ethereum.chainConfig", serveCmd.PersistentFlags().Lookup("eth-chain-config")) viper.BindPFlag("ethereum.supportsStateDiff", serveCmd.PersistentFlags().Lookup("eth-supports-state-diff")) + + // groupcache flags + viper.BindPFlag("groupcache.pool.enabled", serveCmd.PersistentFlags().Lookup("gcache-pool-enabled")) + viper.BindPFlag("groupcache.pool.httpEndpoint", serveCmd.PersistentFlags().Lookup("gcache-pool-http-path")) + viper.BindPFlag("groupcache.pool.peerHttpEndpoints", serveCmd.PersistentFlags().Lookup("gcache-pool-http-peers")) + viper.BindPFlag("groupcache.statedb.cacheSizeInMB", serveCmd.PersistentFlags().Lookup("gcache-statedb-cache-size")) + viper.BindPFlag("groupcache.statedb.cacheExpiryInMins", serveCmd.PersistentFlags().Lookup("gcache-statedb-cache-expiry")) + viper.BindPFlag("groupcache.statedb.logStatsIntervalInSecs", serveCmd.PersistentFlags().Lookup("gcache-statedb-log-stats-interval")) } diff --git a/pkg/eth/backend.go b/pkg/eth/backend.go index 1af668ed..07a06601 100644 --- a/pkg/eth/backend.go +++ b/pkg/eth/backend.go @@ -44,6 +44,7 @@ import ( "github.com/ethereum/go-ethereum/statediff/indexer/postgres" ethServerShared "github.com/ethereum/go-ethereum/statediff/indexer/shared" "github.com/ethereum/go-ethereum/trie" + log "github.com/sirupsen/logrus" ipfsethdb "github.com/vulcanize/ipfs-ethdb/postgres" "github.com/vulcanize/ipld-eth-server/pkg/shared" ) @@ -85,6 +86,10 @@ const ( RetrieveCodeByMhKey = `SELECT data FROM public.blocks WHERE key = $1` ) +const ( + StateDBGroupCacheName = "statedb" +) + type Backend struct { // underlying postgres db DB *postgres.DB @@ -114,11 +119,13 @@ func NewEthBackend(db *postgres.DB, c *Config) (*Backend, error) { r := NewCIDRetriever(db) ethDB := ipfsethdb.NewDatabase(db.DB, ipfsethdb.CacheConfig{ - Name: "statedb", + Name: StateDBGroupCacheName, Size: gcc.StateDB.CacheSizeInMB, ExpiryDuration: time.Minute * time.Duration(gcc.StateDB.CacheExpiryInMins), }) + logStateDBStatsOnTimer(ethDB, gcc) + return &Backend{ DB: db, Retriever: r, @@ -834,3 +841,21 @@ func (b *Backend) BloomStatus() (uint64, uint64) { func (b *Backend) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) { panic("implement me") } + +func logStateDBStatsOnTimer(ethDB *ipfsethdb.Database, gcc *shared.GroupCacheConfig) { + // No stats logging if interval isn't a positive integer. + if gcc.StateDB.LogStatsIntervalInSecs <= 0 { + return + } + + ticker := time.NewTicker(time.Duration(gcc.StateDB.LogStatsIntervalInSecs) * time.Second) + + go func() { + for { + select { + case <-ticker.C: + log.Infof("%s groupcache stats: %+v", StateDBGroupCacheName, ethDB.GetCacheStats()) + } + } + }() +} diff --git a/pkg/serve/config.go b/pkg/serve/config.go index 5f8502c9..00967a6d 100644 --- a/pkg/serve/config.go +++ b/pkg/serve/config.go @@ -245,13 +245,24 @@ func (d *Config) dbInit() { } func (c *Config) loadGroupCacheConfig() { + viper.BindEnv("groupcache.pool.enabled", ethServerShared.GCACHE_POOL_ENABLED) + viper.BindEnv("groupcache.pool.httpEndpoint", ethServerShared.GCACHE_POOL_HTTP_PATH) + viper.BindEnv("groupcache.pool.peerHttpEndpoints", ethServerShared.GCACHE_POOL_HTTP_PEERS) + viper.BindEnv("groupcache.statedb.cacheSizeInMB", ethServerShared.GCACHE_STATEDB_CACHE_SIZE) + viper.BindEnv("groupcache.statedb.cacheExpiryInMins", ethServerShared.GCACHE_STATEDB_CACHE_EXPIRY) + viper.BindEnv("groupcache.statedb.logStatsIntervalInSecs", ethServerShared.GCACHE_STATEDB_LOG_STATS_INTERVAL) + gcc := ethServerShared.GroupCacheConfig{} + gcc.Pool.Enabled = viper.GetBool("groupcache.pool.enabled") + if gcc.Pool.Enabled { + gcc.Pool.HttpEndpoint = viper.GetString("groupcache.pool.httpEndpoint") + gcc.Pool.PeerHttpEndpoints = viper.GetStringSlice("groupcache.pool.peerHttpEndpoints") + } - gcc.Pool.HttpEndpoint = viper.GetString("groupcache.pool.httpEndpoint") - gcc.Pool.PeerHttpEndpoints = viper.GetStringSlice("groupcache.pool.peerHttpEndpoints") - + // Irrespective of whether the pool is enabled, we always use the hot/local cache. gcc.StateDB.CacheSizeInMB = viper.GetInt("groupcache.statedb.cacheSizeInMB") gcc.StateDB.CacheExpiryInMins = viper.GetInt("groupcache.statedb.cacheExpiryInMins") + gcc.StateDB.LogStatsIntervalInSecs = viper.GetInt("groupcache.statedb.logStatsIntervalInSecs") c.GroupCache = &gcc } diff --git a/pkg/shared/constants.go b/pkg/shared/constants.go index 3dc2994c..4d253d51 100644 --- a/pkg/shared/constants.go +++ b/pkg/shared/constants.go @@ -19,4 +19,11 @@ package shared const ( DefaultMaxBatchSize uint64 = 100 DefaultMaxBatchNumber int64 = 50 + + GCACHE_POOL_ENABLED = "GCACHE_POOL_ENABLED" + GCACHE_POOL_HTTP_PATH = "GCACHE_POOL_HTTP_PATH" + GCACHE_POOL_HTTP_PEERS = "GCACHE_POOL_HTTP_PEERS" + GCACHE_STATEDB_CACHE_SIZE = "GCACHE_STATEDB_CACHE_SIZE" + GCACHE_STATEDB_CACHE_EXPIRY = "GCACHE_STATEDB_CACHE_EXPIRY" + GCACHE_STATEDB_LOG_STATS_INTERVAL = "GCACHE_STATEDB_LOG_STATS_INTERVAL" ) diff --git a/pkg/shared/types.go b/pkg/shared/types.go index dc1cc29b..d4435921 100644 --- a/pkg/shared/types.go +++ b/pkg/shared/types.go @@ -17,13 +17,15 @@ package shared type PoolConfig struct { + Enabled bool HttpEndpoint string PeerHttpEndpoints []string } type GroupConfig struct { - CacheSizeInMB int - CacheExpiryInMins int + CacheSizeInMB int + CacheExpiryInMins int + LogStatsIntervalInSecs int } type GroupCacheConfig struct {