deep-stack/ipld-eth-server
4
0
Fork 0
forked from cerc-io/ipld-eth-server
Code Pull Requests Activity

Upgrade to v5 schema

Browse Source 
Now uses:
* ipld direct_by_leaf StateDB for basic queries
* trie_by_cid StateDB for trie slice and proof queries

Also:
* vulcanize => cerc refactor
* Backend method to close dbs
* state tests are in multiple packages, to allow separate ginkgo suites
* removes gap-filler module
* integration tests and github workflows
* run stack-orchestrator for testnet
* fix various issues with tests, hardhat server, dockerfile
* fix cmd flags / env vars
* fix flaky tests and clean up code
* remove unused code, scripts
* remove outdated docs
* update version
This commit is contained in:
roysc
2023-05-25 21:39:48 +08:00
parent ec6707c6ab
commit 6d7487152c
84 changed files with 5118 additions and 17644 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/eth/test_helpers/chain_indexer.go
+82
View File
@@ -0,0 +1,82 @@
package test_helpers
import (
"context"
"math/big"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/statediff"
"github.com/cerc-io/ipld-eth-server/v5/pkg/shared"
)
type IndexChainParams struct {
Blocks []*types.Block
Receipts []types.Receipts
StateCache state.Database
ChainConfig *params.ChainConfig
StateDiffParams statediff.Params
TotalDifficulty *big.Int
// Whether to skip indexing state nodes (state_cids, storage_cids)
SkipStateNodes bool
// Whether to skip indexing IPLD blocks
SkipIPLDs bool
}
func IndexChain(params IndexChainParams) error {
indexer := shared.SetupTestStateDiffIndexer(context.Background(), params.ChainConfig, Genesis.Hash())
builder := statediff.NewBuilder(params.StateCache)
// iterate over the blocks, generating statediff payloads, and transforming the data into Postgres
for i, block := range params.Blocks {
var args statediff.Args
var rcts types.Receipts
if i == 0 {
args = statediff.Args{
OldStateRoot: common.Hash{},
NewStateRoot: block.Root(),
BlockNumber: block.Number(),
BlockHash: block.Hash(),
}
} else {
args = statediff.Args{
OldStateRoot: params.Blocks[i-1].Root(),
NewStateRoot: block.Root(),
BlockNumber: block.Number(),
BlockHash: block.Hash(),
}
rcts = params.Receipts[i-1]
}
diff, err := builder.BuildStateDiffObject(args, params.StateDiffParams)
if err != nil {
return err
}
tx, err := indexer.PushBlock(block, rcts, params.TotalDifficulty)
if err != nil {
return err
}
if !params.SkipStateNodes {
for _, node := range diff.Nodes {
if err = indexer.PushStateNode(tx, node, block.Hash().String()); err != nil {
return err
}
}
}
if !params.SkipIPLDs {
for _, ipld := range diff.IPLDs {
if err := indexer.PushIPLD(tx, ipld); err != nil {
return err
}
}
}
if err = tx.Submit(err); err != nil {
return err
}
}
return nil
}
pkg/eth/test_helpers/chain_maker.go
+7 -46
View File
@@ -17,7 +17,6 @@
package test_helpers
import (
"bytes"
"math/big"
"github.com/ethereum/go-ethereum/common"
@@ -28,10 +27,7 @@ import (
"github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
"github.com/ethereum/go-ethereum/statediff/test_helpers"
"github.com/ipfs/go-cid"
)
// Test variables
@@ -40,7 +36,8 @@ var (
TestBankKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
TestBankAddress = crypto.PubkeyToAddress(TestBankKey.PublicKey) //0x71562b71999873DB5b286dF957af199Ec94617F7
TestBankFunds = big.NewInt(100000000)
Genesis = test_helpers.GenesisBlockForTesting(Testdb, TestBankAddress, TestBankFunds)
Genesis = test_helpers.GenesisBlockForTesting(Testdb, TestBankAddress, TestBankFunds, big.NewInt(params.InitialBaseFee), params.MaxGasLimit)
Account1Key, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a")
Account2Key, _ = crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee")
@@ -65,11 +62,12 @@ data function sig: 73d4a13a
// MakeChain creates a chain of n blocks starting at and including parent.
// the returned hash chain is ordered head->parent.
func MakeChain(n int, parent *types.Block, chainGen func(int, *core.BlockGen)) ([]*types.Block, []types.Receipts, *core.BlockChain) {
config := params.TestChainConfig
config.LondonBlock = big.NewInt(100)
func MakeChain(n int, parent *types.Block, chainGen func(int, *core.BlockGen), config *params.ChainConfig) ([]*types.Block, []types.Receipts, *core.BlockChain) {
blocks, receipts := core.GenerateChain(config, parent, ethash.NewFaker(), Testdb, n, chainGen)
chain, _ := core.NewBlockChain(Testdb, nil, nil, nil, ethash.NewFaker(), vm.Config{}, nil, nil)
chain, err := core.NewBlockChain(Testdb, nil, nil, nil, ethash.NewFaker(), vm.Config{}, nil, nil)
if err != nil {
panic(err)
}
return append([]*types.Block{parent}, blocks...), receipts, chain
}
@@ -110,40 +108,3 @@ func TestChainGen(i int, block *core.BlockGen) {
block.AddTx(tx)
}
}
// GetRctLeafNodeData converts the receipts to receipt trie and returns the receipt leaf node IPLD data and
// corresponding CIDs
func GetRctLeafNodeData(rcts types.Receipts) ([]cid.Cid, [][]byte, error) {
receiptTrie := ipld.NewRctTrie()
for idx, rct := range rcts {
ethRct, err := ipld.NewReceipt(rct)
if err != nil {
return nil, nil, err
}
if err = receiptTrie.Add(idx, ethRct.RawData()); err != nil {
return nil, nil, err
}
}
rctLeafNodes, keys, err := receiptTrie.GetLeafNodes()
if err != nil {
return nil, nil, err
}
ethRctleafNodeCids := make([]cid.Cid, len(rctLeafNodes))
ethRctleafNodeData := make([][]byte, len(rctLeafNodes))
for i, rln := range rctLeafNodes {
var idx uint
r := bytes.NewReader(keys[i].TrieKey)
err = rlp.Decode(r, &idx)
if err != nil {
return nil, nil, err
}
ethRctleafNodeCids[idx] = rln.Cid()
ethRctleafNodeData[idx] = rln.RawData()
}
return ethRctleafNodeCids, ethRctleafNodeData, nil
}
pkg/eth/test_helpers/test_data.go
+88 -91
View File
@@ -22,17 +22,19 @@ import (
"crypto/rand"
"math/big"
"github.com/cerc-io/ipld-eth-server/v4/pkg/eth"
"github.com/cerc-io/ipld-eth-server/v4/pkg/log"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
"github.com/ethereum/go-ethereum/statediff/indexer/models"
testhelpers "github.com/ethereum/go-ethereum/statediff/test_helpers"
sdtypes "github.com/ethereum/go-ethereum/statediff/types"
"github.com/ethereum/go-ethereum/trie"
"github.com/ipfs/go-cid"
"github.com/cerc-io/ipld-eth-server/v5/pkg/eth"
"github.com/cerc-io/ipld-eth-server/v5/pkg/log"
)
// Test variables
@@ -58,6 +60,7 @@ var (
ReceiptHash: common.HexToHash("0x1"),
Difficulty: big.NewInt(500001),
Extra: []byte{},
ParentHash: Genesis.Hash(),
},
{
Time: 2,
@@ -67,9 +70,10 @@ var (
ReceiptHash: common.HexToHash("0x2"),
Difficulty: big.NewInt(500002),
Extra: []byte{},
ParentHash: Genesis.Hash(),
},
}
MockBlock = createNewBlock(&MockHeader, MockTransactions, MockUncles, MockReceipts, new(trie.Trie))
MockBlock = createNewBlock(&MockHeader, MockTransactions, MockUncles, MockReceipts, trie.NewEmpty(nil))
MockChildHeader = types.Header{
Time: 0,
Number: new(big.Int).Add(BlockNumber, common.Big1),
@@ -80,25 +84,23 @@ var (
Extra: []byte{},
ParentHash: MockBlock.Header().Hash(),
}
MockChild = types.NewBlock(&MockChildHeader, MockTransactions, MockUncles, MockReceipts, new(trie.Trie))
Address = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592")
AnotherAddress = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476593")
AnotherAddress1 = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476594")
AnotherAddress2 = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476596")
ContractAddress = crypto.CreateAddress(SenderAddr, MockTransactions[2].Nonce())
ContractHash = crypto.Keccak256Hash(ContractAddress.Bytes()).String()
MockContractByteCode = []byte{0, 1, 2, 3, 4, 5}
mockTopic11 = common.HexToHash("0x04")
mockTopic12 = common.HexToHash("0x06")
mockTopic21 = common.HexToHash("0x05")
mockTopic22 = common.HexToHash("0x07")
mockTopic31 = common.HexToHash("0x08")
mockTopic41 = common.HexToHash("0x09")
mockTopic42 = common.HexToHash("0x0a")
mockTopic43 = common.HexToHash("0x0b")
mockTopic51 = common.HexToHash("0x0c")
mockTopic61 = common.HexToHash("0x0d")
MockLog1 = &types.Log{
MockChild = types.NewBlock(&MockChildHeader, MockTransactions, MockUncles, MockReceipts, trie.NewEmpty(nil))
Address = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592")
AnotherAddress = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476593")
AnotherAddress1 = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476594")
AnotherAddress2 = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476596")
ContractAddress = crypto.CreateAddress(SenderAddr, MockTransactions[2].Nonce())
mockTopic11 = common.HexToHash("0x04")
mockTopic12 = common.HexToHash("0x06")
mockTopic21 = common.HexToHash("0x05")
mockTopic22 = common.HexToHash("0x07")
mockTopic31 = common.HexToHash("0x08")
mockTopic41 = common.HexToHash("0x09")
mockTopic42 = common.HexToHash("0x0a")
mockTopic43 = common.HexToHash("0x0b")
mockTopic51 = common.HexToHash("0x0c")
mockTopic61 = common.HexToHash("0x0d")
MockLog1 = &types.Log{
Address: Address,
Topics: []common.Hash{mockTopic11, mockTopic12},
Data: []byte{},
@@ -148,71 +150,55 @@ var (
Index: 5,
}
rctCIDs, _, _ = GetRctLeafNodeData(MockReceipts)
Rct1CID = rctCIDs[0]
Rct4CID = rctCIDs[3]
MockTrxMeta = []models.TxModel{
rctCIDs, _ = getReceiptCIDs(MockReceipts)
Rct1CID = rctCIDs[0]
Rct4CID = rctCIDs[3]
MockTrxMeta = []models.TxModel{
{
CID: "", // This is empty until we go to publish to ipfs
MhKey: "",
Src: SenderAddr.Hex(),
Dst: Address.String(),
Index: 0,
TxHash: MockTransactions[0].Hash().String(),
Data: []byte{},
},
{
CID: "",
MhKey: "",
Src: SenderAddr.Hex(),
Dst: AnotherAddress.String(),
Index: 1,
TxHash: MockTransactions[1].Hash().String(),
Data: []byte{},
},
{
CID: "",
MhKey: "",
Src: SenderAddr.Hex(),
Dst: "",
Index: 2,
TxHash: MockTransactions[2].Hash().String(),
Data: MockContractByteCode,
},
{
CID: "",
MhKey: "",
Src: SenderAddr.Hex(),
Dst: "",
Index: 3,
TxHash: MockTransactions[3].Hash().String(),
Data: []byte{},
},
}
MockRctMeta = []models.ReceiptModel{
{
LeafCID: "",
LeafMhKey: "",
Contract: "",
ContractHash: "",
CID: "",
Contract: "",
},
{
LeafCID: "",
LeafMhKey: "",
Contract: "",
ContractHash: "",
CID: "",
Contract: "",
},
{
LeafCID: "",
LeafMhKey: "",
Contract: ContractAddress.String(),
ContractHash: ContractHash,
CID: "",
Contract: ContractAddress.String(),
},
{
LeafCID: "",
LeafMhKey: "",
Contract: "",
ContractHash: "",
CID: "",
Contract: "",
},
}
@@ -226,21 +212,20 @@ var (
StorageValue,
})
nonce1 = uint64(1)
ContractRoot = "0x821e2556a290c86405f8160a2d662042a431ba456b9db265c79bb837c04be5f0"
ContractCodeHash = crypto.Keccak256Hash(MockContractByteCode)
contractPath = common.Bytes2Hex([]byte{'\x06'})
ContractLeafKey = testhelpers.AddressToLeafKey(ContractAddress)
ContractAccount, _ = rlp.EncodeToBytes(&types.StateAccount{
Nonce: nonce1,
ContractRoot = "0x821e2556a290c86405f8160a2d662042a431ba456b9db265c79bb837c04be5f0"
contractPath = common.Bytes2Hex([]byte{'\x06'})
ContractLeafKey = crypto.Keccak256(ContractAddress[:])
ContractAccount = types.StateAccount{
Nonce: uint64(1),
Balance: big.NewInt(0),
CodeHash: ContractCodeHash.Bytes(),
CodeHash: CodeHash.Bytes(),
Root: common.HexToHash(ContractRoot),
})
ContractPartialPath = common.Hex2Bytes("3114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45")
ContractLeafNode, _ = rlp.EncodeToBytes(&[]interface{}{
}
ContractAccountRLP, _ = rlp.EncodeToBytes(&ContractAccount)
ContractPartialPath = common.Hex2Bytes("3114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45")
ContractLeafNode, _ = rlp.EncodeToBytes(&[]interface{}{
ContractPartialPath,
ContractAccount,
ContractAccountRLP,
})
nonce0 = uint64(0)
@@ -248,49 +233,49 @@ var (
AccountRoot = "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"
AccountCodeHash = common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")
AccountAddresss = common.HexToAddress("0x0D3ab14BBaD3D99F4203bd7a11aCB94882050E7e")
AccountLeafKey = testhelpers.Account2LeafKey
Account, _ = rlp.EncodeToBytes(&types.StateAccount{
AccountLeafKey = crypto.Keccak256(AccountAddresss[:])
Account = types.StateAccount{
Nonce: nonce0,
Balance: AccountBalance,
CodeHash: AccountCodeHash.Bytes(),
Root: common.HexToHash(AccountRoot),
})
}
AccountRLP, _ = rlp.EncodeToBytes(&Account)
AccountPartialPath = common.Hex2Bytes("3957f3e2f04a0764c3a0491b175f69926da61efbcc8f61fa1455fd2d2b4cdd45")
AccountLeafNode, _ = rlp.EncodeToBytes(&[]interface{}{
AccountPartialPath,
Account,
AccountRLP,
})
MockStateNodes = []sdtypes.StateNode{
MockStateNodes = []sdtypes.StateLeafNode{
{
LeafKey: ContractLeafKey,
Path: []byte{'\x06'},
NodeValue: ContractLeafNode,
NodeType: sdtypes.Leaf,
StorageNodes: []sdtypes.StorageNode{
AccountWrapper: sdtypes.AccountWrapper{
Account: &ContractAccount,
LeafKey: ContractLeafKey,
CID: ipld.Keccak256ToCid(ipld.MEthStateTrie, crypto.Keccak256(ContractLeafNode)).String(),
},
StorageDiff: []sdtypes.StorageLeafNode{
{
Path: []byte{},
NodeType: sdtypes.Leaf,
LeafKey: StorageLeafKey,
NodeValue: StorageLeafNode,
LeafKey: StorageLeafKey,
Value: StorageValue,
CID: ipld.Keccak256ToCid(ipld.MEthStorageTrie, crypto.Keccak256(StorageLeafNode)).String(),
},
},
},
{
LeafKey: AccountLeafKey,
Path: []byte{'\x0c'},
NodeValue: AccountLeafNode,
NodeType: sdtypes.Leaf,
StorageNodes: []sdtypes.StorageNode{},
AccountWrapper: sdtypes.AccountWrapper{
Account: &Account,
LeafKey: AccountLeafKey,
CID: ipld.Keccak256ToCid(ipld.MEthStateTrie, crypto.Keccak256(AccountLeafNode)).String(),
},
},
}
MockStorageNodes = map[string][]sdtypes.StorageNode{
MockStorageNodes = map[string][]sdtypes.StorageLeafNode{
contractPath: {
{
LeafKey: StorageLeafKey,
NodeValue: StorageLeafNode,
NodeType: sdtypes.Leaf,
Path: []byte{},
LeafKey: StorageLeafKey,
Value: StorageValue,
CID: ipld.Keccak256ToCid(ipld.MEthStorageTrie, crypto.Keccak256(StorageLeafNode)).String(),
},
},
}
@@ -338,7 +323,7 @@ var (
Extra: []byte{},
},
}
MockLondonBlock = createNewBlock(&MockLondonHeader, MockLondonTransactions, MockLondonUncles, MockLondonReceipts, new(trie.Trie))
MockLondonBlock = createNewBlock(&MockLondonHeader, MockLondonTransactions, MockLondonUncles, MockLondonReceipts, trie.NewEmpty(nil))
)
func createNewBlock(header *types.Header, txs []*types.Transaction, uncles []*types.Header, receipts []*types.Receipt, hasher types.TrieHasher) *types.Block {
@@ -355,7 +340,7 @@ func createNewBlock(header *types.Header, txs []*types.Transaction, uncles []*ty
// createDynamicTransactionsAndReceipts is a helper function to generate signed mock transactions and mock receipts with mock logs
func createDynamicTransactionsAndReceipts(blockNumber *big.Int) (types.Transactions, types.Receipts, common.Address) {
// make transactions
config := params.TestChainConfig
config := *params.TestChainConfig
config.LondonBlock = blockNumber
trx1 := types.NewTx(&types.DynamicFeeTx{
ChainID: config.ChainID,
@@ -368,7 +353,7 @@ func createDynamicTransactionsAndReceipts(blockNumber *big.Int) (types.Transacti
Data: []byte{},
})
transactionSigner := types.MakeSigner(config, blockNumber)
transactionSigner := types.MakeSigner(&config, blockNumber)
mockCurve := elliptic.P256()
mockPrvKey, err := ecdsa.GenerateKey(mockCurve, rand.Reader)
if err != nil {
@@ -404,7 +389,7 @@ func createLegacyTransactionsAndReceipts() (types.Transactions, types.Receipts,
// make transactions
trx1 := types.NewTransaction(0, Address, big.NewInt(1000), 50, big.NewInt(100), []byte{})
trx2 := types.NewTransaction(1, AnotherAddress, big.NewInt(2000), 100, big.NewInt(200), []byte{})
trx3 := types.NewContractCreation(2, big.NewInt(1500), 75, big.NewInt(150), MockContractByteCode)
trx3 := types.NewContractCreation(2, big.NewInt(1500), 75, big.NewInt(150), ContractCode)
trx4 := types.NewTransaction(3, AnotherAddress1, big.NewInt(2000), 100, big.NewInt(200), []byte{})
transactionSigner := types.MakeSigner(params.MainnetChainConfig, new(big.Int).Set(BlockNumber))
mockCurve := elliptic.P256()
@@ -463,3 +448,15 @@ func createLegacyTransactionsAndReceipts() (types.Transactions, types.Receipts,
return types.Transactions{signedTrx1, signedTrx2, signedTrx3, signedTrx4}, types.Receipts{mockReceipt1, mockReceipt2, mockReceipt3, mockReceipt4}, SenderAddr
}
func getReceiptCIDs(rcts []*types.Receipt) ([]cid.Cid, error) {
cids := make([]cid.Cid, len(rcts))
for i, rct := range rcts {
ethRct, err := ipld.NewReceipt(rct)
if err != nil {
return nil, err
}
cids[i] = ethRct.Cid()
}
return cids, nil
}