cerc-io/ipld-eth-server
12
0
Fork 1
Code Issues 10 Pull Requests 2 Actions Packages Projects Releases 37 Wiki Activity

Use groupcache pool for state db access (#91)

Browse Source 
* Use groupcache pool for state db access

* Group cache config and logging stats on timer

* Integrate state validator into server

* Use tagged ipfs-ethdb

* groupcache config for tests

* Work around duplicate registration of groupcache error in tests

* Use tagged version of eth-ipfs-state-validator

* State validation command.

* Validator for static replicas to keep cache warm

* Update docker go-version and go.mod.

* Address comments and self review.

* Fix ipfs-ethdb version.

Co-authored-by: Arijit Das <arijitad.in@gmail.com>
This commit was merged in pull request #91.
This commit is contained in:
Ashwin Phatak
2021-09-21 17:40:55 +05:30
committed by GitHub
co-authored by Arijit Das
parent 838ed033f8
commit 2de9c5bd48
14 changed files with 1208 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/eth/api_test.go
+10 -7
View File
@@ -20,7 +20,6 @@ import (
"context"
"math/big"
"strconv"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
@@ -37,9 +36,10 @@ import (
sdtypes "github.com/ethereum/go-ethereum/statediff/types"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
pgipfsethdb "github.com/vulcanize/ipfs-ethdb/postgres"
"github.com/vulcanize/ipld-eth-server/pkg/eth"
"github.com/vulcanize/ipld-eth-server/pkg/eth/test_helpers"
ethServerShared "github.com/vulcanize/ipld-eth-server/pkg/shared"
)
var (
@@ -213,12 +213,15 @@ var _ = Describe("API", func() {
indexAndPublisher := indexer.NewStateDiffIndexer(chainConfig, db)
backend, err := eth.NewEthBackend(db, &eth.Config{
ChainConfig: chainConfig,
VmConfig: vm.Config{},
VMConfig: vm.Config{},
RPCGasCap: big.NewInt(10000000000), // Max gas capacity for a rpc call.
CacheConfig: pgipfsethdb.CacheConfig{
Name: "api_test",
Size: 3000000, // 3MB
ExpiryDuration: time.Hour,
GroupCacheConfig: &ethServerShared.GroupCacheConfig{
StateDB: ethServerShared.GroupConfig{
Name: "api_test",
CacheSizeInMB: 8,
CacheExpiryInMins: 60,
LogStatsIntervalInSecs: 0,
},
},
})
Expect(err).ToNot(HaveOccurred())
pkg/eth/backend.go
+57 -16
View File
@@ -22,6 +22,7 @@ import (
"errors"
"fmt"
"math/big"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
@@ -41,9 +42,13 @@ import (
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/statediff/indexer/ipfs"
"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
"github.com/ethereum/go-ethereum/statediff/indexer/shared"
ethServerShared "github.com/ethereum/go-ethereum/statediff/indexer/shared"
"github.com/ethereum/go-ethereum/trie"
pgipfsethdb "github.com/vulcanize/ipfs-ethdb/postgres"
log "github.com/sirupsen/logrus"
validator "github.com/vulcanize/eth-ipfs-state-validator/pkg"
ipfsethdb "github.com/vulcanize/ipfs-ethdb/postgres"
"github.com/vulcanize/ipld-eth-server/pkg/shared"
)
var (
@@ -83,6 +88,10 @@ const (
RetrieveCodeByMhKey = `SELECT data FROM public.blocks WHERE key = $1`
)
const (
StateDBGroupCacheName = "statedb"
)
type Backend struct {
// underlying postgres db
DB *postgres.DB
@@ -100,17 +109,30 @@ type Backend struct {
}
type Config struct {
ChainConfig *params.ChainConfig
VmConfig vm.Config
DefaultSender *common.Address
RPCGasCap *big.Int
CacheConfig pgipfsethdb.CacheConfig
ChainConfig *params.ChainConfig
VMConfig vm.Config
DefaultSender *common.Address
RPCGasCap *big.Int
GroupCacheConfig *shared.GroupCacheConfig
}
func NewEthBackend(db *postgres.DB, c *Config) (*Backend, error) {
gcc := c.GroupCacheConfig
groupName := gcc.StateDB.Name
if groupName == "" {
groupName = StateDBGroupCacheName
}
r := NewCIDRetriever(db)
ethDB := ipfsethdb.NewDatabase(db.DB, ipfsethdb.CacheConfig{
Name: groupName,
Size: gcc.StateDB.CacheSizeInMB * 1024 * 1024,
ExpiryDuration: time.Minute * time.Duration(gcc.StateDB.CacheExpiryInMins),
})
logStateDBStatsOnTimer(ethDB, gcc)
ethDB := pgipfsethdb.NewDatabase(db.DB, c.CacheConfig)
return &Backend{
DB: db,
Retriever: r,
@@ -292,10 +314,10 @@ func (b *Backend) BlockByNumber(ctx context.Context, blockNumber rpc.BlockNumber
}
defer func() {
if p := recover(); p != nil {
shared.Rollback(tx)
ethServerShared.Rollback(tx)
panic(p)
} else if err != nil {
shared.Rollback(tx)
ethServerShared.Rollback(tx)
} else {
err = tx.Commit()
}
@@ -383,10 +405,10 @@ func (b *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Blo
}
defer func() {
if p := recover(); p != nil {
shared.Rollback(tx)
ethServerShared.Rollback(tx)
panic(p)
} else if err != nil {
shared.Rollback(tx)
ethServerShared.Rollback(tx)
} else {
err = tx.Commit()
}
@@ -592,7 +614,7 @@ func (b *Backend) GetEVM(ctx context.Context, msg core.Message, state *state.Sta
state.SetBalance(msg.From(), math.MaxBig256)
vmctx := core.NewEVMBlockContext(header, b, nil)
txContext := core.NewEVMTxContext(msg)
return vm.NewEVM(vmctx, txContext, state, b.Config.ChainConfig, b.Config.VmConfig), nil
return vm.NewEVM(vmctx, txContext, state, b.Config.ChainConfig, b.Config.VMConfig), nil
}
// GetAccountByNumberOrHash returns the account object for the provided address at the block corresponding to the provided number or hash
@@ -704,10 +726,10 @@ func (b *Backend) GetCodeByHash(ctx context.Context, address common.Address, has
}
defer func() {
if p := recover(); p != nil {
shared.Rollback(tx)
ethServerShared.Rollback(tx)
panic(p)
} else if err != nil {
shared.Rollback(tx)
ethServerShared.Rollback(tx)
} else {
err = tx.Commit()
}
@@ -717,7 +739,7 @@ func (b *Backend) GetCodeByHash(ctx context.Context, address common.Address, has
return nil, err
}
var mhKey string
mhKey, err = shared.MultihashKeyFromKeccak256(common.BytesToHash(codeHash))
mhKey, err = ethServerShared.MultihashKeyFromKeccak256(common.BytesToHash(codeHash))
if err != nil {
return nil, err
}
@@ -794,6 +816,10 @@ func (b *Backend) GetHeader(hash common.Hash, height uint64) *types.Header {
return header
}
func (b *Backend) ValidateTrie(stateRoot common.Hash) error {
return validator.NewValidator(nil, b.EthDB).ValidateTrie(stateRoot)
}
// RPCGasCap returns the configured gas cap for the rpc server
func (b *Backend) RPCGasCap() *big.Int {
return b.Config.RPCGasCap
@@ -826,3 +852,18 @@ func (b *Backend) BloomStatus() (uint64, uint64) {
func (b *Backend) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) {
panic("implement me")
}
func logStateDBStatsOnTimer(ethDB *ipfsethdb.Database, gcc *shared.GroupCacheConfig) {
// No stats logging if interval isn't a positive integer.
if gcc.StateDB.LogStatsIntervalInSecs <= 0 {
return
}
ticker := time.NewTicker(time.Duration(gcc.StateDB.LogStatsIntervalInSecs) * time.Second)
go func() {
for range ticker.C {
log.Infof("%s groupcache stats: %+v", StateDBGroupCacheName, ethDB.GetCacheStats())
}
}()
}
pkg/eth/eth_state_test.go
+9 -7
View File
@@ -21,7 +21,6 @@ import (
"context"
"io/ioutil"
"math/big"
"time"
"github.com/ethereum/go-ethereum/accounts/abi"
"github.com/ethereum/go-ethereum/common"
@@ -37,10 +36,10 @@ import (
sdtypes "github.com/ethereum/go-ethereum/statediff/types"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
pgipfsethdb "github.com/vulcanize/ipfs-ethdb/postgres"
"github.com/vulcanize/ipld-eth-server/pkg/eth"
"github.com/vulcanize/ipld-eth-server/pkg/eth/test_helpers"
ethServerShared "github.com/vulcanize/ipld-eth-server/pkg/shared"
)
var (
@@ -80,12 +79,15 @@ var _ = Describe("eth state reading tests", func() {
transformer := indexer.NewStateDiffIndexer(chainConfig, db)
backend, err = eth.NewEthBackend(db, &eth.Config{
ChainConfig: chainConfig,
VmConfig: vm.Config{},
VMConfig: vm.Config{},
RPCGasCap: big.NewInt(10000000000), // Max gas capacity for a rpc call.
CacheConfig: pgipfsethdb.CacheConfig{
Name: "eth_state",
Size: 3000000, // 3MB
ExpiryDuration: time.Hour,
GroupCacheConfig: &ethServerShared.GroupCacheConfig{
StateDB: ethServerShared.GroupConfig{
Name: "eth_state_test",
CacheSizeInMB: 8,
CacheExpiryInMins: 60,
LogStatsIntervalInSecs: 0,
},
},
})
Expect(err).ToNot(HaveOccurred())