Upgrade to v5 schema

Now uses:
* ipld direct_by_leaf StateDB for basic queries
* trie_by_cid StateDB for trie slice and proof queries

Also:
* vulcanize => cerc refactor
* Backend method to close dbs
* state tests are in multiple packages, to allow separate ginkgo suites
* removes gap-filler module
* integration tests and github workflows
* run stack-orchestrator for testnet
* fix various issues with tests, hardhat server, dockerfile
* fix cmd flags / env vars
* fix flaky tests and clean up code
* remove unused code, scripts
* remove outdated docs
* update version
This commit is contained in:
2023-05-25 21:39:48 +08:00
parent ec6707c6ab
commit 6d7487152c
84 changed files with 5118 additions and 17644 deletions
+9 -9
View File
@@ -43,12 +43,12 @@ type IPFSBlockResponse struct {
}
type EthTransactionCIDResponse struct {
CID string `json:"cid"`
TxHash string `json:"txHash"`
Index int32 `json:"index"`
Src string `json:"src"`
Dst string `json:"dst"`
BlockByMhKey IPFSBlockResponse `json:"blockByMhKey"`
CID string `json:"cid"`
TxHash string `json:"txHash"`
Index int32 `json:"index"`
Src string `json:"src"`
Dst string `json:"dst"`
BlockByCid IPFSBlockResponse `json:"blockByCid"`
}
type EthTransactionCIDByTxHash struct {
@@ -72,7 +72,7 @@ type EthHeaderCIDResponse struct {
UncleRoot string `json:"uncleRoot"`
Bloom string `json:"bloom"`
EthTransactionCIDsByHeaderId EthTransactionCIDsByHeaderIdResponse `json:"ethTransactionCidsByHeaderId"`
BlockByMhKey IPFSBlockResponse `json:"blockByMhKey"`
BlockByCid IPFSBlockResponse `json:"blockByCid"`
}
type AllEthHeaderCIDsResponse struct {
@@ -195,7 +195,7 @@ func (c *Client) AllEthHeaderCIDs(ctx context.Context, condition EthHeaderCIDCon
receiptRoot
uncleRoot
bloom
blockByMhKey {
blockByCid {
key
data
}
@@ -244,7 +244,7 @@ func (c *Client) EthTransactionCIDByTxHash(ctx context.Context, txHash string) (
index
src
dst
blockByMhKey {
blockByCid {
data
}
}
+18 -21
View File
@@ -34,9 +34,9 @@ import (
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/rpc"
"github.com/cerc-io/ipld-eth-server/v4/pkg/eth"
"github.com/cerc-io/ipld-eth-server/v4/pkg/shared"
ipld_eth_statedb "github.com/cerc-io/ipld-eth-statedb"
"github.com/cerc-io/ipld-eth-server/v5/pkg/eth"
"github.com/cerc-io/ipld-eth-server/v5/pkg/shared"
"github.com/cerc-io/ipld-eth-statedb/direct_by_leaf"
)
var (
@@ -51,8 +51,8 @@ type Account struct {
}
// getState fetches the StateDB object for an account.
func (a *Account) getState(ctx context.Context) (*ipld_eth_statedb.StateDB, error) {
state, _, err := a.backend.IPLDStateDBAndHeaderByNumberOrHash(ctx, a.blockNrOrHash)
func (a *Account) getState(ctx context.Context) (*state.StateDB, error) {
state, _, err := a.backend.IPLDDirectStateDBAndHeaderByNumberOrHash(ctx, a.blockNrOrHash)
return state, err
}
@@ -1009,30 +1009,27 @@ func (r *Resolver) GetStorageAt(ctx context.Context, args struct {
Contract common.Address
Slot common.Hash
}) (*StorageResult, error) {
cid, ipldBlock, rlpValue, err := r.backend.Retriever.RetrieveStorageAtByAddressAndStorageSlotAndBlockHash(args.Contract, args.Slot, args.BlockHash)
cid, nodeRLP, err := r.backend.Retriever.RetrieveStorageAndRLP(args.Contract, args.Slot, args.BlockHash)
if err != nil {
if err == sql.ErrNoRows {
ret := StorageResult{value: []byte{}, cid: "", ipldBlock: []byte{}}
return &ret, nil
return &StorageResult{value: []byte{}, cid: "", ipldBlock: []byte{}}, nil
}
return nil, err
}
if bytes.Equal(rlpValue, eth.EmptyNodeValue) {
return &StorageResult{value: eth.EmptyNodeValue, cid: cid, ipldBlock: ipldBlock}, nil
valueRLP, err := eth.DecodeLeafNode(nodeRLP)
if err != nil {
return nil, err
}
if bytes.Equal(valueRLP, eth.EmptyNodeValue) {
return &StorageResult{value: eth.EmptyNodeValue, cid: cid, ipldBlock: nodeRLP}, nil
}
var value interface{}
err = rlp.DecodeBytes(rlpValue, &value)
err = rlp.DecodeBytes(valueRLP, &value)
if err != nil {
return nil, err
}
ret := StorageResult{value: value.([]byte), cid: cid, ipldBlock: ipldBlock}
return &ret, nil
return &StorageResult{value: value.([]byte), cid: cid, ipldBlock: nodeRLP}, nil
}
func (r *Resolver) GetLogs(ctx context.Context, args struct {
@@ -1160,7 +1157,7 @@ func (t EthTransactionCID) Dst(ctx context.Context) string {
return t.dst
}
func (t EthTransactionCID) BlockByMhKey(ctx context.Context) IPFSBlock {
func (t EthTransactionCID) BlockByCid(ctx context.Context) IPFSBlock {
return t.ipfsBlock
}
@@ -1249,7 +1246,7 @@ func (h EthHeaderCID) EthTransactionCidsByHeaderId(ctx context.Context) EthTrans
return EthTransactionCIDsConnection{nodes: h.transactions}
}
func (h EthHeaderCID) BlockByMhKey(ctx context.Context) IPFSBlock {
func (h EthHeaderCID) BlockByCid(ctx context.Context) IPFSBlock {
return h.ipfsBlock
}
@@ -1326,7 +1323,7 @@ func (r *Resolver) AllEthHeaderCids(ctx context.Context, args struct {
td: td,
txRoot: headerCID.TxRoot,
receiptRoot: headerCID.RctRoot,
uncleRoot: headerCID.UncleRoot,
uncleRoot: headerCID.UnclesHash,
bloom: Bytes(headerCID.Bloom).String(),
ipfsBlock: IPFSBlock{
key: headerCID.IPLD.Key,
+1 -7
View File
@@ -17,11 +17,9 @@
package graphql_test
import (
"io/ioutil"
"testing"
"github.com/cerc-io/ipld-eth-server/v4/pkg/log"
. "github.com/onsi/ginkgo"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
@@ -29,7 +27,3 @@ func TestGraphQL(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "graphql test suite")
}
var _ = BeforeSuite(func() {
log.SetOutput(ioutil.Discard)
})
+106 -144
View File
@@ -30,151 +30,113 @@ import (
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/statediff"
"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
sdtypes "github.com/ethereum/go-ethereum/statediff/types"
"github.com/jmoiron/sqlx"
. "github.com/onsi/ginkgo"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/cerc-io/ipld-eth-server/v4/pkg/eth"
"github.com/cerc-io/ipld-eth-server/v4/pkg/eth/test_helpers"
"github.com/cerc-io/ipld-eth-server/v4/pkg/graphql"
"github.com/cerc-io/ipld-eth-server/v4/pkg/shared"
ethServerShared "github.com/cerc-io/ipld-eth-server/v4/pkg/shared"
"github.com/cerc-io/ipld-eth-server/v5/pkg/eth"
"github.com/cerc-io/ipld-eth-server/v5/pkg/eth/test_helpers"
"github.com/cerc-io/ipld-eth-server/v5/pkg/graphql"
"github.com/cerc-io/ipld-eth-server/v5/pkg/shared"
)
var _ = Describe("GraphQL", func() {
const (
gqlEndPoint = "127.0.0.1:8083"
)
var (
randomAddr = common.HexToAddress("0x1C3ab14BBaD3D99F4203bd7a11aCB94882050E6f")
randomHash = crypto.Keccak256Hash(randomAddr.Bytes())
blocks []*types.Block
receipts []types.Receipts
chain *core.BlockChain
db *sqlx.DB
blockHashes []common.Hash
backend *eth.Backend
graphQLServer *graphql.Service
chainConfig = params.TestChainConfig
mockTD = big.NewInt(1337)
client = graphql.NewClient(fmt.Sprintf("http://%s/graphql", gqlEndPoint))
ctx = context.Background()
blockHash common.Hash
contractAddress common.Address
)
const (
gqlEndPoint = "127.0.0.1:8083"
)
It("test init", func() {
var err error
db = shared.SetupDB()
transformer := shared.SetupTestStateDiffIndexer(ctx, chainConfig, test_helpers.Genesis.Hash())
var (
randomAddr = common.HexToAddress("0x1C3ab14BBaD3D99F4203bd7a11aCB94882050E6f")
randomHash = crypto.Keccak256Hash(randomAddr.Bytes())
blocks []*types.Block
receipts []types.Receipts
chain *core.BlockChain
db *sqlx.DB
backend *eth.Backend
graphQLServer *graphql.Service
chainConfig = &*params.TestChainConfig
client = graphql.NewClient(fmt.Sprintf("http://%s/graphql", gqlEndPoint))
mockTD = big.NewInt(1337)
ctx = context.Background()
nonCanonBlockHash common.Hash
nonCanonContractAddress common.Address
)
backend, err = eth.NewEthBackend(db, &eth.Config{
ChainConfig: chainConfig,
VMConfig: vm.Config{},
RPCGasCap: big.NewInt(10000000000),
GroupCacheConfig: &ethServerShared.GroupCacheConfig{
StateDB: ethServerShared.GroupConfig{
Name: "graphql_test",
CacheSizeInMB: 8,
CacheExpiryInMins: 60,
LogStatsIntervalInSecs: 0,
},
var _ = BeforeSuite(func() {
var err error
db = shared.SetupDB()
backend, err = eth.NewEthBackend(db, &eth.Config{
ChainConfig: chainConfig,
VMConfig: vm.Config{},
RPCGasCap: big.NewInt(10000000000),
GroupCacheConfig: &shared.GroupCacheConfig{
StateDB: shared.GroupConfig{
Name: "graphql_test",
CacheSizeInMB: 8,
CacheExpiryInMins: 60,
LogStatsIntervalInSecs: 0,
},
})
Expect(err).ToNot(HaveOccurred())
},
})
Expect(err).ToNot(HaveOccurred())
// make the test blockchain (and state)
blocks, receipts, chain = test_helpers.MakeChain(5, test_helpers.Genesis, test_helpers.TestChainGen)
params := statediff.Params{
IntermediateStateNodes: true,
IntermediateStorageNodes: true,
}
// iterate over the blocks, generating statediff payloads, and transforming the data into Postgres
builder := statediff.NewBuilder(chain.StateCache())
for i, block := range blocks {
blockHashes = append(blockHashes, block.Hash())
var args statediff.Args
var rcts types.Receipts
if i == 0 {
args = statediff.Args{
OldStateRoot: common.Hash{},
NewStateRoot: block.Root(),
BlockNumber: block.Number(),
BlockHash: block.Hash(),
}
} else {
args = statediff.Args{
OldStateRoot: blocks[i-1].Root(),
NewStateRoot: block.Root(),
BlockNumber: block.Number(),
BlockHash: block.Hash(),
}
rcts = receipts[i-1]
}
var diff sdtypes.StateObject
diff, err = builder.BuildStateDiffObject(args, params)
Expect(err).ToNot(HaveOccurred())
tx, err := transformer.PushBlock(block, rcts, mockTD)
Expect(err).ToNot(HaveOccurred())
for _, node := range diff.Nodes {
err = transformer.PushStateNode(tx, node, block.Hash().String())
Expect(err).ToNot(HaveOccurred())
}
err = tx.Submit(err)
Expect(err).ToNot(HaveOccurred())
}
// Insert some non-canonical data into the database so that we test our ability to discern canonicity
indexAndPublisher := shared.SetupTestStateDiffIndexer(ctx, chainConfig, test_helpers.Genesis.Hash())
blockHash = test_helpers.MockBlock.Hash()
contractAddress = test_helpers.ContractAddr
tx, err := indexAndPublisher.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
Expect(err).ToNot(HaveOccurred())
err = tx.Submit(err)
Expect(err).ToNot(HaveOccurred())
// The non-canonical header has a child
tx, err = indexAndPublisher.PushBlock(test_helpers.MockChild, test_helpers.MockReceipts, test_helpers.MockChild.Difficulty())
Expect(err).ToNot(HaveOccurred())
ccHash := sdtypes.CodeAndCodeHash{
Hash: test_helpers.CodeHash,
Code: test_helpers.ContractCode,
}
err = indexAndPublisher.PushCodeAndCodeHash(tx, ccHash)
Expect(err).ToNot(HaveOccurred())
err = tx.Submit(err)
Expect(err).ToNot(HaveOccurred())
graphQLServer, err = graphql.New(backend, gqlEndPoint, nil, []string{"*"}, rpc.HTTPTimeouts{})
Expect(err).ToNot(HaveOccurred())
err = graphQLServer.Start(nil)
Expect(err).ToNot(HaveOccurred())
// make the test blockchain (and state)
chainConfig.LondonBlock = big.NewInt(100)
blocks, receipts, chain = test_helpers.MakeChain(5, test_helpers.Genesis, test_helpers.TestChainGen, chainConfig)
test_helpers.IndexChain(test_helpers.IndexChainParams{
StateCache: chain.StateCache(),
ChainConfig: chainConfig,
Blocks: blocks,
Receipts: receipts,
TotalDifficulty: mockTD,
})
defer It("test teardown", func() {
err := graphQLServer.Stop()
Expect(err).ToNot(HaveOccurred())
shared.TearDownDB(db)
chain.Stop()
})
// Insert some non-canonical data into the database so that we test our ability to discern canonicity
indexAndPublisher := shared.SetupTestStateDiffIndexer(ctx, chainConfig, test_helpers.Genesis.Hash())
nonCanonBlockHash = test_helpers.MockBlock.Hash()
nonCanonContractAddress = test_helpers.ContractAddr
tx, err := indexAndPublisher.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
Expect(err).ToNot(HaveOccurred())
err = tx.Submit(err)
Expect(err).ToNot(HaveOccurred())
// The non-canonical header has a child
tx, err = indexAndPublisher.PushBlock(test_helpers.MockChild, test_helpers.MockReceipts, test_helpers.MockChild.Difficulty())
Expect(err).ToNot(HaveOccurred())
ipld := sdtypes.IPLD{
CID: ipld.Keccak256ToCid(ipld.RawBinary, test_helpers.CodeHash.Bytes()).String(),
Content: test_helpers.ContractCode,
}
err = indexAndPublisher.PushIPLD(tx, ipld)
Expect(err).ToNot(HaveOccurred())
err = tx.Submit(err)
Expect(err).ToNot(HaveOccurred())
graphQLServer, err = graphql.New(backend, gqlEndPoint, nil, []string{"*"}, rpc.HTTPTimeouts{})
Expect(err).ToNot(HaveOccurred())
err = graphQLServer.Start(nil)
Expect(err).ToNot(HaveOccurred())
})
var _ = AfterSuite(func() {
err := graphQLServer.Stop()
Expect(err).ToNot(HaveOccurred())
shared.TearDownDB(db)
chain.Stop()
})
var _ = Describe("GraphQL", func() {
Describe("eth_getLogs", func() {
It("Retrieves logs that matches the provided blockHash and contract address", func() {
logs, err := client.GetLogs(ctx, blockHash, []common.Address{contractAddress})
logs, err := client.GetLogs(ctx, nonCanonBlockHash, []common.Address{nonCanonContractAddress})
Expect(err).ToNot(HaveOccurred())
expectedLogs := []graphql.LogResponse{
@@ -191,7 +153,7 @@ var _ = Describe("GraphQL", func() {
})
It("Retrieves logs for the failed receipt status that matches the provided blockHash and another contract address", func() {
logs, err := client.GetLogs(ctx, blockHash, []common.Address{test_helpers.AnotherAddress2})
logs, err := client.GetLogs(ctx, nonCanonBlockHash, []common.Address{test_helpers.AnotherAddress2})
Expect(err).ToNot(HaveOccurred())
expectedLogs := []graphql.LogResponse{
@@ -208,7 +170,7 @@ var _ = Describe("GraphQL", func() {
})
It("Retrieves logs that matches the provided blockHash and multiple contract addresses", func() {
logs, err := client.GetLogs(ctx, blockHash, []common.Address{contractAddress, test_helpers.AnotherAddress2})
logs, err := client.GetLogs(ctx, nonCanonBlockHash, []common.Address{nonCanonContractAddress, test_helpers.AnotherAddress2})
Expect(err).ToNot(HaveOccurred())
expectedLogs := []graphql.LogResponse{
@@ -232,13 +194,13 @@ var _ = Describe("GraphQL", func() {
})
It("Retrieves all the logs for the receipt that matches the provided blockHash and nil contract address", func() {
logs, err := client.GetLogs(ctx, blockHash, nil)
logs, err := client.GetLogs(ctx, nonCanonBlockHash, nil)
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(6))
})
It("Retrieves logs with random hash", func() {
logs, err := client.GetLogs(ctx, randomHash, []common.Address{contractAddress})
logs, err := client.GetLogs(ctx, randomHash, []common.Address{nonCanonContractAddress})
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(0))
})
@@ -246,31 +208,31 @@ var _ = Describe("GraphQL", func() {
Describe("eth_getStorageAt", func() {
It("Retrieves the storage value at the provided contract address and storage leaf key at the block with the provided hash", func() {
storageRes, err := client.GetStorageAt(ctx, blockHashes[2], contractAddress, test_helpers.IndexOne)
storageRes, err := client.GetStorageAt(ctx, blocks[2].Hash(), nonCanonContractAddress, test_helpers.IndexOne)
Expect(err).ToNot(HaveOccurred())
Expect(storageRes.Value).To(Equal(common.HexToHash("01")))
storageRes, err = client.GetStorageAt(ctx, blockHashes[3], contractAddress, test_helpers.IndexOne)
storageRes, err = client.GetStorageAt(ctx, blocks[3].Hash(), nonCanonContractAddress, test_helpers.IndexOne)
Expect(err).ToNot(HaveOccurred())
Expect(storageRes.Value).To(Equal(common.HexToHash("03")))
storageRes, err = client.GetStorageAt(ctx, blockHashes[4], contractAddress, test_helpers.IndexOne)
storageRes, err = client.GetStorageAt(ctx, blocks[4].Hash(), nonCanonContractAddress, test_helpers.IndexOne)
Expect(err).ToNot(HaveOccurred())
Expect(storageRes.Value).To(Equal(common.HexToHash("09")))
})
It("Retrieves empty data if it tries to access a contract at a blockHash which does not exist", func() {
storageRes, err := client.GetStorageAt(ctx, blockHashes[0], contractAddress, test_helpers.IndexOne)
storageRes, err := client.GetStorageAt(ctx, blocks[0].Hash(), nonCanonContractAddress, test_helpers.IndexOne)
Expect(err).ToNot(HaveOccurred())
Expect(storageRes.Value).To(Equal(common.Hash{}))
storageRes, err = client.GetStorageAt(ctx, blockHashes[1], contractAddress, test_helpers.IndexOne)
storageRes, err = client.GetStorageAt(ctx, blocks[1].Hash(), nonCanonContractAddress, test_helpers.IndexOne)
Expect(err).ToNot(HaveOccurred())
Expect(storageRes.Value).To(Equal(common.Hash{}))
})
It("Retrieves empty data if it tries to access a contract slot which does not exist", func() {
storageRes, err := client.GetStorageAt(ctx, blockHashes[3], contractAddress, randomHash.Hex())
storageRes, err := client.GetStorageAt(ctx, blocks[3].Hash(), nonCanonContractAddress, randomHash.Hex())
Expect(err).ToNot(HaveOccurred())
Expect(storageRes.Value).To(Equal(common.Hash{}))
})
@@ -316,7 +278,7 @@ var _ = Describe("GraphQL", func() {
compareEthTxCID(*ethTransactionCIDResp, txCID)
Expect(ethTransactionCIDResp.BlockByMhKey.Data).To(Equal(graphql.Bytes(txCID.IPLD.Data).String()))
Expect(ethTransactionCIDResp.BlockByCid.Data).To(Equal(graphql.Bytes(txCID.IPLD.Data).String()))
})
})
})
@@ -337,7 +299,7 @@ func compareEthHeaderCID(ethHeaderCID graphql.EthHeaderCIDResponse, headerCID et
Expect(ethHeaderCID.Td).To(Equal(*new(graphql.BigInt).SetUint64(uint64(td))))
Expect(ethHeaderCID.TxRoot).To(Equal(headerCID.TxRoot))
Expect(ethHeaderCID.ReceiptRoot).To(Equal(headerCID.RctRoot))
Expect(ethHeaderCID.UncleRoot).To(Equal(headerCID.UncleRoot))
Expect(ethHeaderCID.UncleRoot).To(Equal(headerCID.UnclesHash))
Expect(ethHeaderCID.Bloom).To(Equal(graphql.Bytes(headerCID.Bloom).String()))
for tIdx, txCID := range headerCID.TransactionCIDs {
@@ -345,8 +307,8 @@ func compareEthHeaderCID(ethHeaderCID graphql.EthHeaderCIDResponse, headerCID et
compareEthTxCID(ethTxCID, txCID)
}
Expect(ethHeaderCID.BlockByMhKey.Data).To(Equal(graphql.Bytes(headerCID.IPLD.Data).String()))
Expect(ethHeaderCID.BlockByMhKey.Key).To(Equal(headerCID.IPLD.Key))
Expect(ethHeaderCID.BlockByCid.Data).To(Equal(graphql.Bytes(headerCID.IPLD.Data).String()))
Expect(ethHeaderCID.BlockByCid.Key).To(Equal(headerCID.IPLD.Key))
}
func compareEthTxCID(ethTxCID graphql.EthTransactionCIDResponse, txCID eth.TransactionCIDRecord) {
+2 -2
View File
@@ -292,7 +292,7 @@ const schema string = `
index: Int!
src: String!
dst: String!
blockByMhKey: IPFSBlock!
blockByCid: IPFSBlock!
}
type EthTransactionCidsConnection {
@@ -317,7 +317,7 @@ const schema string = `
uncleRoot: String!
bloom: String!
ethTransactionCidsByHeaderId: EthTransactionCidsConnection!
blockByMhKey: IPFSBlock!
blockByCid: IPFSBlock!
}
type EthHeaderCidsConnection {
+4 -4
View File
@@ -28,8 +28,8 @@ import (
"github.com/graph-gophers/graphql-go"
"github.com/graph-gophers/graphql-go/relay"
"github.com/cerc-io/ipld-eth-server/v4/pkg/eth"
"github.com/cerc-io/ipld-eth-server/v4/pkg/log"
"github.com/cerc-io/ipld-eth-server/v5/pkg/eth"
"github.com/cerc-io/ipld-eth-server/v5/pkg/log"
)
// Service encapsulates a GraphQL service.
@@ -76,8 +76,8 @@ func (s *Service) Start(server *p2p.Server) error {
if err != nil {
utils.Fatalf("Could not start RPC api: %v", err)
}
extapiURL := fmt.Sprintf("http://%v/", addr)
log.Infof("graphQL endpoint opened for url %s", extapiURL)
extapiURL := fmt.Sprintf("http://%v", addr)
log.Infof("GraphQL endpoint opened at %s", extapiURL)
return nil
}