From 13d1e032f1e1fde7c81de04e2d752497b51194b9 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Fri, 10 May 2024 17:24:34 +0800 Subject: [PATCH 01/13] handle withdrawals --- cmd/root.go | 2 +- cmd/serve.go | 1 + pkg/eth/api.go | 3 ++- pkg/eth/api_test/api_test.go | 4 ++-- pkg/eth/backend.go | 30 ++++++++++++++++++++++++++---- pkg/eth/retriever.go | 18 ++++++++++++++++++ pkg/eth/sql.go | 15 +++++++++++++++ pkg/eth/test_helpers/test_data.go | 14 +++++++++----- test/stack.yml | 4 ++-- 9 files changed, 76 insertions(+), 15 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 7c403f78..067a57ed 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -95,7 +95,7 @@ func init() { func initConfig() { if cfgFile == "" && envFile == "" { - log.Fatal("No configuration file specified, use --config , --env flag to provide configuration") + log.Warn("No configuration file specified, use --config , --env flag to provide configuration") } if cfgFile != "" { diff --git a/cmd/serve.go b/cmd/serve.go index a1fb8621..7d94cbf3 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -60,6 +60,7 @@ func serve() { logWithCommand.Fatal(err) } logWithCommand.Debugf("server config: %+v", serverConfig) + logWithCommand.Debugf("chain config: %+v", serverConfig.ChainConfig) server, err := s.NewServer(serverConfig) if err != nil { logWithCommand.Fatal(err) diff --git a/pkg/eth/api.go b/pkg/eth/api.go index d21e364a..6adcc901 100644 --- a/pkg/eth/api.go +++ b/pkg/eth/api.go @@ -1245,7 +1245,8 @@ func (pea *PublicEthAPI) rpcMarshalBlock(b *types.Block, inclTx bool, fullTx boo if inclTx { td, err := pea.B.GetTd(b.Hash()) if err != nil { - log.Errorf("error getting td for block with hash and number %s, %s: %s", b.Hash().String(), b.Number().String(), err) + err = fmt.Errorf("error getting TD for block at (%s, %s): %s", b.Number(), b.Hash(), err) + log.Error(err) return nil, err } fields["totalDifficulty"] = (*hexutil.Big)(td) diff --git a/pkg/eth/api_test/api_test.go b/pkg/eth/api_test/api_test.go index dfda4fdd..a44673e8 100644 --- a/pkg/eth/api_test/api_test.go +++ b/pkg/eth/api_test/api_test.go @@ -50,7 +50,7 @@ var ( blockHash = test_helpers.MockBlock.Header().Hash() baseFee = test_helpers.MockLondonBlock.BaseFee() ctx = context.Background() - chainConfig = &*params.TestChainConfig + chainConfig = &*params.MergedTestChainConfig expectedBlock = map[string]interface{}{ "number": (*hexutil.Big)(test_helpers.MockBlock.Number()), @@ -390,7 +390,7 @@ var _ = Describe("API", func() { Expect(block).To(BeZero()) }) It("Fetch BaseFee from london block by block hash, returns `nil` for legacy block", func() { - block, err := api.GetBlockByHash(ctx, test_helpers.MockBlock.Hash(), true) + block, err := api.GetBlockByHash(ctx, test_helpers.MockBlock.Hash(), false) Expect(err).ToNot(HaveOccurred()) _, ok := block["baseFeePerGas"] Expect(ok).To(Equal(false)) diff --git a/pkg/eth/backend.go b/pkg/eth/backend.go index 331ad0c7..34d2448e 100644 --- a/pkg/eth/backend.go +++ b/pkg/eth/backend.go @@ -271,7 +271,7 @@ func (b *Backend) BlockByNumberOrHash(ctx context.Context, blockNrOrHash rpc.Blo func (b *Backend) BlockByNumber(ctx context.Context, blockNumber rpc.BlockNumber) (*types.Block, error) { number, err := b.NormalizeBlockNumber(blockNumber) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to normalize block number: %w", err) } canonicalHash, err := b.GetCanonicalHash(uint64(number)) if err != nil { @@ -349,11 +349,16 @@ func (b *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Blo return nil, err } - // Placeholder for withdrawal processing (TODO: https://git.vdb.to/cerc-io/ipld-eth-server/pulls/265) + // Fetch withdrawals var withdrawals types.Withdrawals if b.Config.ChainConfig.IsShanghai(header.Number, header.Time) { - // All blocks after Shanghai must include a withdrawals root. - withdrawals = make(types.Withdrawals, 0) + withdrawals, err = b.GetWithdrawals(tx, hash, blockNumber) + if err != nil && err != sql.ErrNoRows { + log.Error("error fetching withdrawals: ", err) + return nil, err + } + } else if len(withdrawals) > 0 { + return nil, errors.New("withdrawals set before Shanghai activation") } // Compose everything together into a complete block @@ -501,6 +506,23 @@ func (b *Backend) GetReceiptsByBlockHashAndNumber(tx *sqlx.Tx, hash common.Hash, return rcts, nil } +// GetWithdrawals retrieves transactions for a provided block hash and number +func (b *Backend) GetWithdrawals(tx *sqlx.Tx, hash common.Hash, number uint64) (types.Withdrawals, error) { + _, rlpBytes, err := b.Retriever.RetrieveWithdrawals(tx, hash, number) + if err != nil { + return nil, err + } + + withdrawals := make(types.Withdrawals, len(rlpBytes)) + for i, bytes := range rlpBytes { + withdrawals[i] = new(types.Withdrawal) + if err := rlp.DecodeBytes(bytes, withdrawals[i]); err != nil { + return nil, err + } + } + return withdrawals, nil +} + // GetTransaction retrieves a tx by hash // It also returns the blockhash, blocknumber, and tx index associated with the transaction func (b *Backend) GetTransaction(ctx context.Context, txHash common.Hash) (bool, *types.Transaction, common.Hash, uint64, uint64, error) { diff --git a/pkg/eth/retriever.go b/pkg/eth/retriever.go index 328e75fc..e2cf4c26 100644 --- a/pkg/eth/retriever.go +++ b/pkg/eth/retriever.go @@ -506,6 +506,24 @@ func (r *Retriever) RetrieveReceiptsByBlockHash(tx *sqlx.Tx, hash common.Hash) ( return cids, rcts, txs, nil } +// RetrieveWithdrawals returns the CIDs and RLP bytes for the withdrawals corresponding to the +// provided block hash, number. Returned CIDs correspond to the leaf node data which contains the +// withdrawal object. +func (r *Retriever) RetrieveWithdrawals(tx *sqlx.Tx, hash common.Hash, number uint64) ([]string, [][]byte, error) { + results := make([]ipldResult, 0) + if err := tx.Select(&results, RetrieveWithdrawalsPgStr, hash.Hex(), number); err != nil { + return nil, nil, err + } + cids := make([]string, len(results)) + withdrawals := make([][]byte, len(results)) + + for i, res := range results { + cids[i] = res.CID + withdrawals[i] = res.Data + } + return cids, withdrawals, nil +} + // RetrieveAccountByAddressAndBlockHash returns the cid and rlp bytes for the account corresponding to the provided address and block hash // TODO: ensure this handles deleted accounts appropriately func (r *Retriever) RetrieveAccountByAddressAndBlockHash(address common.Address, hash common.Hash) (StateAccountRecord, error) { diff --git a/pkg/eth/sql.go b/pkg/eth/sql.go index 95fef81c..392d6d0d 100644 --- a/pkg/eth/sql.go +++ b/pkg/eth/sql.go @@ -107,6 +107,21 @@ WHERE header_cids.block_hash = $1 AND blocks.key = receipt_cids.cid ORDER BY eth.transaction_cids.index ASC ` + RetrieveWithdrawalsPgStr = ` +SELECT withdrawal_cids.cid, + blocks.data + FROM eth.withdrawal_cids + JOIN eth.header_cids + ON header_cids.block_hash = $1 + AND header_cids.block_number = $2 + AND header_cids.canonical + AND withdrawal_cids.block_number = header_cids.block_number + AND withdrawal_cids.header_id = header_cids.block_hash + JOIN ipld.blocks + ON blocks.block_number = header_cids.block_number + AND blocks.key = withdrawal_cids.cid + ORDER BY eth.withdrawal_cids.index ASC` + RetrieveAccountByLeafKeyAndBlockHashPgStr = ` SELECT state_cids.nonce, state_cids.balance, diff --git a/pkg/eth/test_helpers/test_data.go b/pkg/eth/test_helpers/test_data.go index afe7b289..c66c033e 100644 --- a/pkg/eth/test_helpers/test_data.go +++ b/pkg/eth/test_helpers/test_data.go @@ -53,7 +53,11 @@ var ( Extra: []byte{}, } MockTransactions, MockReceipts, SenderAddr = createLegacyTransactionsAndReceipts() - MockUncles = []*types.Header{ + MockWithdrawals = types.Withdrawals{ + {Index: 0, Validator: 1, Address: Address, Amount: 1000000000}, + {Index: 1, Validator: 5, Address: AnotherAddress, Amount: 2000000000}, + } + MockUncles = []*types.Header{ { Time: 1, Number: big.NewInt(BlockNumber1 + 1), @@ -75,7 +79,7 @@ var ( ParentHash: Genesis.Hash(), }, } - MockBlock = createNewBlock(&MockHeader, MockTransactions, MockUncles, MockReceipts, trie.NewEmpty(nil)) + MockBlock = createNewBlock(&MockHeader, MockTransactions, MockUncles, MockReceipts, nil, trie.NewEmpty(nil)) MockChildHeader = types.Header{ Time: 0, Number: big.NewInt(BlockNumber1 + 1), @@ -326,11 +330,11 @@ var ( Extra: []byte{}, }, } - MockLondonBlock = createNewBlock(&MockLondonHeader, MockLondonTransactions, MockLondonUncles, MockLondonReceipts, trie.NewEmpty(nil)) + MockLondonBlock = createNewBlock(&MockLondonHeader, MockLondonTransactions, MockLondonUncles, MockLondonReceipts, MockWithdrawals, trie.NewEmpty(nil)) ) -func createNewBlock(header *types.Header, txs []*types.Transaction, uncles []*types.Header, receipts []*types.Receipt, hasher types.TrieHasher) *types.Block { - block := types.NewBlock(header, txs, uncles, receipts, hasher) +func createNewBlock(header *types.Header, txs []*types.Transaction, uncles []*types.Header, receipts []*types.Receipt, withdrawals types.Withdrawals, hasher types.TrieHasher) *types.Block { + block := types.NewBlockWithWithdrawals(header, txs, uncles, receipts, withdrawals, hasher) bHash := block.Hash() for _, r := range receipts { for _, l := range r.Logs { diff --git a/test/stack.yml b/test/stack.yml index 8829a4d8..750d13b1 100644 --- a/test/stack.yml +++ b/test/stack.yml @@ -3,9 +3,9 @@ name: fixturenet-plugeth-tx description: "Plugeth Ethereum Fixturenet for testing ipld-eth-server" repos: - git.vdb.to/cerc-io/plugeth@v1.13.14-cerc-2 - - git.vdb.to/cerc-io/plugeth-statediff + - git.vdb.to/cerc-io/plugeth-statediff@index-withdrawals # todo: dev - git.vdb.to/cerc-io/lighthouse - - git.vdb.to/cerc-io/ipld-eth-db@v5.2.1-alpha + - git.vdb.to/cerc-io/ipld-eth-db@add-withdrawals # todo: dev - git.vdb.to/cerc-io/ipld-eth-server containers: - cerc/plugeth-statediff -- 2.45.2 From 856126b5e57222f61d89c48fb6c30585ea8556c2 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Thu, 11 Jul 2024 12:08:50 +0800 Subject: [PATCH 02/13] docker warning --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index f1aa8605..7ce3ca05 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,9 @@ -FROM golang:1.21-alpine as debugger +FROM golang:1.21-alpine AS debugger # Include dlv RUN go install github.com/go-delve/delve/cmd/dlv@latest -FROM golang:1.21-alpine as builder +FROM golang:1.21-alpine AS builder RUN apk --update --no-cache add gcc musl-dev binutils-gold git -- 2.45.2 From 7071144b0b75fac1c2464c780a37172b17163494 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Mon, 13 May 2024 12:35:49 +0800 Subject: [PATCH 03/13] update ipld-eth-db --- scripts/integration-setup.sh | 6 +++--- test/compose-db.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/integration-setup.sh b/scripts/integration-setup.sh index 55bda42b..7c7c7a0f 100755 --- a/scripts/integration-setup.sh +++ b/scripts/integration-setup.sh @@ -13,11 +13,11 @@ export BUILDKIT_PROGRESS=plain # By default assume we are running in the project root export CERC_REPO_BASE_DIR="${CERC_REPO_BASE_DIR:-..}" -# v5 migrations only go up to version 18 -echo CERC_STATEDIFF_DB_GOOSE_MIN_VER=18 >> $CONFIG_DIR/stack.env + +echo CERC_STATEDIFF_DB_GOOSE_MIN_VER=21 >> $CONFIG_DIR/stack.env # Pass this in so we can run eth_call forwarding tests, which expect no IPLD DB echo CERC_RUN_STATEDIFF=${CERC_RUN_STATEDIFF:-true} >> $CONFIG_DIR/stack.env -# don't run plugeth in the debugger +# Don't run the debugger echo CERC_REMOTE_DEBUG=false >> $CONFIG_DIR/stack.env set -x diff --git a/test/compose-db.yml b/test/compose-db.yml index 48fbea34..ab9d7666 100644 --- a/test/compose-db.yml +++ b/test/compose-db.yml @@ -5,7 +5,7 @@ services: restart: on-failure depends_on: - ipld-eth-db - image: git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:v5.2.1-alpha + image: git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:v5.3.0-alpha environment: DATABASE_USER: "vdbm" DATABASE_NAME: "cerc_testing" -- 2.45.2 From 236340ba8bb7025875bd494c9722f0ed92242e63 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Wed, 10 Jul 2024 17:40:28 +0800 Subject: [PATCH 04/13] update deps use exported ipld encoder use plugeth-statediff cleardb --- go.mod | 11 +++++++++-- go.sum | 16 ++++++++-------- pkg/eth/test_helpers/test_data.go | 2 +- pkg/shared/test_helpers.go | 26 ++------------------------ 4 files changed, 20 insertions(+), 35 deletions(-) diff --git a/go.mod b/go.mod index dbdca838..65e86173 100644 --- a/go.mod +++ b/go.mod @@ -4,10 +4,10 @@ go 1.21 require ( github.com/cerc-io/eth-ipfs-state-validator/v5 v5.2.0-alpha - github.com/cerc-io/eth-iterator-utils v0.2.0 + github.com/cerc-io/eth-iterator-utils v0.3.1 github.com/cerc-io/ipfs-ethdb/v5 v5.1.0-alpha github.com/cerc-io/ipld-eth-statedb v0.1.0 - github.com/cerc-io/plugeth-statediff v0.2.1 + github.com/cerc-io/plugeth-statediff v0.3.1 github.com/ethereum/go-ethereum v1.13.14 github.com/google/uuid v1.6.0 github.com/graph-gophers/graphql-go v1.3.0 @@ -294,3 +294,10 @@ require ( lukechampine.com/blake3 v1.2.2 // indirect rsc.io/tmplfunc v0.0.3 // indirect ) + +replace ( + github.com/cerc-io/ipld-eth-statedb => github.com/cerc-io/ipld-eth-statedb v0.1.1-0.20240711085948-7bb5bbc6aa38 + github.com/cerc-io/plugeth-statediff => github.com/cerc-io/plugeth-statediff v0.3.2-0.20240711085659-6fe622e4bf33 +) + +// replace github.com/cerc-io/plugeth-statediff => git.vdb.to/cerc-io/plugeth-statediff v0.3.2-0.20240710110326-c8ce26de6de5 diff --git a/go.sum b/go.sum index 3920e897..c7e7cbc8 100644 --- a/go.sum +++ b/go.sum @@ -81,16 +81,16 @@ github.com/ceramicnetwork/go-dag-jose v0.1.0 h1:yJ/HVlfKpnD3LdYP03AHyTvbm3BpPiz2 github.com/ceramicnetwork/go-dag-jose v0.1.0/go.mod h1:qYA1nYt0X8u4XoMAVoOV3upUVKtrxy/I670Dg5F0wjI= github.com/cerc-io/eth-ipfs-state-validator/v5 v5.2.0-alpha h1:gpEk3BQQhnkilXxP12q77LSiIAvXfTdQ+4xxIv/JWiI= github.com/cerc-io/eth-ipfs-state-validator/v5 v5.2.0-alpha/go.mod h1:xmazdaZ/CUaXt52Mvd0KU7dcbJM3glvJzA2mEpVMgbM= -github.com/cerc-io/eth-iterator-utils v0.2.0 h1:wikAfWZ0fAqLqUy/Ud/a1n9p/arVeKG8P8tRjZE7oBg= -github.com/cerc-io/eth-iterator-utils v0.2.0/go.mod h1:wDUJvwKDSOdqTIyeG+yXJ2ckzc9f2Fem614fV61DBcg= -github.com/cerc-io/eth-testing v0.4.0 h1:ivGbXnEqlXMt/3m3jbsPJaVT7ZDTenFQWCryt1Rd/Jk= -github.com/cerc-io/eth-testing v0.4.0/go.mod h1:CVsmHjFldX9gwaQSQwGmKbmh0g6Dq+bsqB2CxBf9zbk= +github.com/cerc-io/eth-iterator-utils v0.3.1 h1:h4Bp0+fUiwkyug1uCEO2LZr2qxoW1yKszV2EO/2CDB0= +github.com/cerc-io/eth-iterator-utils v0.3.1/go.mod h1:UNrjsP5bApZkqqqfU7nmnPN/dIIo9GOUUD79tmoX/s4= +github.com/cerc-io/eth-testing v0.5.1 h1:xxcQf9ymJS0911yWIrUiGvCvqfvEjYmHvhBJkCD/whs= +github.com/cerc-io/eth-testing v0.5.1/go.mod h1:p86je2PjSM7u8Qd7rMIG/Zw+tQlBoS5Emkh1ECnC5t0= github.com/cerc-io/ipfs-ethdb/v5 v5.1.0-alpha h1:+XhYHvzC3zFIvcWEb466SNDfeLrvcW3xe/d0cbVVVRA= github.com/cerc-io/ipfs-ethdb/v5 v5.1.0-alpha/go.mod h1:w5g07b6Uz+c6r/ySml58TEOJdUYAibYYF05H5ULVv2I= -github.com/cerc-io/ipld-eth-statedb v0.1.0 h1:K2Xy8hgmkSRmHaqmw0CZf9hWX0r8/r0563O7osmyVBE= -github.com/cerc-io/ipld-eth-statedb v0.1.0/go.mod h1:Lo4TKVs7bTGr2b7L9ccmo/EMV96/nGE7T/GYliuOHlM= -github.com/cerc-io/plugeth-statediff v0.2.1 h1:zUbkazJW0omFGg7z/9MJnttox4VRurLW1pFytDlRQjM= -github.com/cerc-io/plugeth-statediff v0.2.1/go.mod h1:n85L2n8Q3bQVlAVFnyk2soCLJW+YFKGe3DVRauEaS2s= +github.com/cerc-io/ipld-eth-statedb v0.1.1-0.20240711085948-7bb5bbc6aa38 h1:8KzqJObmtnwGE8uXWfoUPG3HT86RTUCIXsipviEwPfA= +github.com/cerc-io/ipld-eth-statedb v0.1.1-0.20240711085948-7bb5bbc6aa38/go.mod h1:Vb8Ko2Nq0dUnJPZplPQp7kIClMBU0zHdH/p8n6rWUqA= +github.com/cerc-io/plugeth-statediff v0.3.2-0.20240711085659-6fe622e4bf33 h1:t0hSbThg5uX4r34eIFI0Pj2WbTIquDc+lKChLqieaMc= +github.com/cerc-io/plugeth-statediff v0.3.2-0.20240711085659-6fe622e4bf33/go.mod h1:r6Mzc6k4V9KD+iN9AXa/LmmRISDsQnnIwKzZMFjJ+eE= github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk= github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= diff --git a/pkg/eth/test_helpers/test_data.go b/pkg/eth/test_helpers/test_data.go index c66c033e..52d9c49d 100644 --- a/pkg/eth/test_helpers/test_data.go +++ b/pkg/eth/test_helpers/test_data.go @@ -459,7 +459,7 @@ func createLegacyTransactionsAndReceipts() (types.Transactions, types.Receipts, func getReceiptCIDs(rcts []*types.Receipt) ([]cid.Cid, error) { cids := make([]cid.Cid, len(rcts)) for i, rct := range rcts { - ethRct, err := ipld.NewReceipt(rct) + ethRct, err := ipld.EncodeReceipt(rct) if err != nil { return nil, err } diff --git a/pkg/shared/test_helpers.go b/pkg/shared/test_helpers.go index 31b11707..94c3fcd4 100644 --- a/pkg/shared/test_helpers.go +++ b/pkg/shared/test_helpers.go @@ -27,6 +27,7 @@ import ( "github.com/cerc-io/plugeth-statediff/indexer/interfaces" "github.com/cerc-io/plugeth-statediff/indexer/models" "github.com/cerc-io/plugeth-statediff/indexer/node" + "github.com/cerc-io/plugeth-statediff/indexer/test_helpers" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/params" "github.com/jmoiron/sqlx" @@ -55,30 +56,7 @@ func SetupDB() *sqlx.DB { // TearDownDB is used to tear down the watcher dbs after tests func TearDownDB(db *sqlx.DB) { - tx, err := db.Beginx() - Expect(err).NotTo(HaveOccurred()) - _, err = tx.Exec(`DELETE FROM nodes`) - Expect(err).NotTo(HaveOccurred()) - _, err = tx.Exec(`DELETE FROM ipld.blocks`) - Expect(err).NotTo(HaveOccurred()) - _, err = tx.Exec(`DELETE FROM eth.header_cids`) - Expect(err).NotTo(HaveOccurred()) - _, err = tx.Exec(`DELETE FROM eth.uncle_cids`) - Expect(err).NotTo(HaveOccurred()) - _, err = tx.Exec(`DELETE FROM eth.transaction_cids`) - Expect(err).NotTo(HaveOccurred()) - _, err = tx.Exec(`DELETE FROM eth.receipt_cids`) - Expect(err).NotTo(HaveOccurred()) - _, err = tx.Exec(`DELETE FROM eth.state_cids`) - Expect(err).NotTo(HaveOccurred()) - _, err = tx.Exec(`DELETE FROM eth.storage_cids`) - Expect(err).NotTo(HaveOccurred()) - _, err = tx.Exec(`DELETE FROM eth.log_cids`) - Expect(err).NotTo(HaveOccurred()) - _, err = tx.Exec(`DELETE FROM eth_meta.watched_addresses`) - Expect(err).NotTo(HaveOccurred()) - - err = tx.Commit() + err := test_helpers.ClearSqlxDB(db) Expect(err).NotTo(HaveOccurred()) } -- 2.45.2 From 81c9e05b4443d4760a5194166f6ec6f3079ddabc Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Wed, 10 Jul 2024 18:13:20 +0800 Subject: [PATCH 05/13] CT: run external stack --- .github/workflows/tests.yaml | 10 ++++- ...integration-setup.sh => run-test-stack.sh} | 44 +++++++++---------- test/stack.yml | 23 ---------- 3 files changed, 30 insertions(+), 47 deletions(-) rename scripts/{integration-setup.sh => run-test-stack.sh} (53%) delete mode 100644 test/stack.yml diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 85ca4033..3370eef8 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -13,6 +13,7 @@ on: env: SO_VERSION: v1.1.0-87fffca-202404110321 + FIXTURENET_ETH_STACKS_REF: main jobs: test: @@ -60,11 +61,18 @@ jobs: ref: ${{ env.SO_VERSION }} path: ./stack-orchestrator - run: pip install ./stack-orchestrator + - name: Clone fixturenet stack repo + uses: actions/checkout@v4 + with: + repository: cerc-io/fixturenet-eth-stacks + ref: ${{ env.FIXTURENET_ETH_STACKS_REF }} + path: ./fixturenet-eth-stacks + progress: false - name: Run testnet stack env: CERC_GO_AUTH_TOKEN: ${{ secrets.CICD_REPO_TOKEN }} - run: ./scripts/integration-setup.sh + run: ./scripts/run-test-stack.sh ./fixturenet-eth-stacks/stack-orchestrator/stacks/fixturenet-plugeth - name: Run server env: ETH_FORWARD_ETH_CALLS: false diff --git a/scripts/integration-setup.sh b/scripts/run-test-stack.sh similarity index 53% rename from scripts/integration-setup.sh rename to scripts/run-test-stack.sh index 7c7c7a0f..6be1896e 100755 --- a/scripts/integration-setup.sh +++ b/scripts/run-test-stack.sh @@ -1,39 +1,37 @@ #!/bin/bash -# Builds and deploys a stack with only what we need. -# This script assumes we are running in the project root. -set -e +set -ex -laconic_so="${LACONIC_SO:-laconic-so} --stack $(readlink -f test) --verbose" +stack_dir=$(readlink -f "$1") +[[ -d "$stack_dir" ]] CONFIG_DIR=$(readlink -f "${CONFIG_DIR:-$(mktemp -d)}") +# By default assume we are running in the project root. +export CERC_REPO_BASE_DIR="${CERC_REPO_BASE_DIR:-$(git rev-parse --show-toplevel)/..}" -# Prevent conflicting tty output -export BUILDKIT_PROGRESS=plain +laconic_so="laconic-so --verbose --stack $stack_dir" -# By default assume we are running in the project root -export CERC_REPO_BASE_DIR="${CERC_REPO_BASE_DIR:-..}" - -echo CERC_STATEDIFF_DB_GOOSE_MIN_VER=21 >> $CONFIG_DIR/stack.env -# Pass this in so we can run eth_call forwarding tests, which expect no IPLD DB -echo CERC_RUN_STATEDIFF=${CERC_RUN_STATEDIFF:-true} >> $CONFIG_DIR/stack.env -# Don't run the debugger +# Don't run geth/plugeth in the debugger, it will swallow error backtraces echo CERC_REMOTE_DEBUG=false >> $CONFIG_DIR/stack.env +# Passing this lets us run eth_call forwarding tests without running ipld-eth-db +echo CERC_RUN_STATEDIFF=${CERC_RUN_STATEDIFF:-true} >> $CONFIG_DIR/stack.env -set -x if [[ -z $SKIP_BUILD ]]; then - $laconic_so setup-repositories \ - --exclude git.vdb.to/cerc-io/ipld-eth-server - # Assume the tested image has been built separately - $laconic_so build-containers \ - --exclude cerc/ipld-eth-server + # Prevent conflicting tty output + export BUILDKIT_PROGRESS=plain + + $laconic_so setup-repositories + $laconic_so build-containers fi -$laconic_so deploy \ - --exclude ipld-eth-server \ - --env-file $CONFIG_DIR/stack.env \ - --cluster test up +if ! $laconic_so deploy \ + --env-file $CONFIG_DIR/stack.env \ + --cluster test up +then + $laconic_so deploy --cluster test logs + exit 1 +fi set +x diff --git a/test/stack.yml b/test/stack.yml deleted file mode 100644 index 750d13b1..00000000 --- a/test/stack.yml +++ /dev/null @@ -1,23 +0,0 @@ -version: "1.2" -name: fixturenet-plugeth-tx -description: "Plugeth Ethereum Fixturenet for testing ipld-eth-server" -repos: - - git.vdb.to/cerc-io/plugeth@v1.13.14-cerc-2 - - git.vdb.to/cerc-io/plugeth-statediff@index-withdrawals # todo: dev - - git.vdb.to/cerc-io/lighthouse - - git.vdb.to/cerc-io/ipld-eth-db@add-withdrawals # todo: dev - - git.vdb.to/cerc-io/ipld-eth-server -containers: - - cerc/plugeth-statediff - - cerc/plugeth - - cerc/fixturenet-eth-genesis - - cerc/fixturenet-plugeth-plugeth - - cerc/lighthouse - - cerc/lighthouse-cli - - cerc/fixturenet-eth-lighthouse - - cerc/ipld-eth-db - - cerc/ipld-eth-server -pods: - - fixturenet-plugeth - - ipld-eth-db - - ipld-eth-server -- 2.45.2 From edf612de89d5bca4213c3afa1e9a03149d84db9f Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Thu, 11 Jul 2024 18:49:30 +0800 Subject: [PATCH 06/13] [dev] fnet stack ref --- .github/workflows/tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 3370eef8..e737ec01 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -13,7 +13,7 @@ on: env: SO_VERSION: v1.1.0-87fffca-202404110321 - FIXTURENET_ETH_STACKS_REF: main + FIXTURENET_ETH_STACKS_REF: fix-plugeth-flags jobs: test: -- 2.45.2 From 7378d741f19b3c714cf0ef68cfb7fee97b220552 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Fri, 12 Jul 2024 11:17:08 +0800 Subject: [PATCH 07/13] fix SO version --- .github/workflows/tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index e737ec01..e4d63f5f 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -12,7 +12,7 @@ on: - ci-test env: - SO_VERSION: v1.1.0-87fffca-202404110321 + SO_VERSION: v1.1.0-36d4969-202407091537 FIXTURENET_ETH_STACKS_REF: fix-plugeth-flags jobs: -- 2.45.2 From 4b64f4a3d50bcd5cab54461b5dcd9218b572d222 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Fri, 12 Jul 2024 15:54:37 +0800 Subject: [PATCH 08/13] fix script name --- .github/workflows/tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index e4d63f5f..97074f8c 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -87,7 +87,7 @@ jobs: env: CERC_RUN_STATEDIFF: false SKIP_BUILD: 1 - run: ./scripts/integration-setup.sh + run: ./scripts/run-test-stack.sh ./fixturenet-eth-stacks/stack-orchestrator/stacks/fixturenet-plugeth - name: Run server with call forwarding env: ETH_FORWARD_ETH_CALLS: true -- 2.45.2 From 661638523eae5e348f00456fb598237489f26aa3 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Sat, 13 Jul 2024 21:11:59 +0800 Subject: [PATCH 09/13] update deps --- go.mod | 11 ++--------- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index 65e86173..8e416145 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ require ( github.com/cerc-io/eth-ipfs-state-validator/v5 v5.2.0-alpha github.com/cerc-io/eth-iterator-utils v0.3.1 github.com/cerc-io/ipfs-ethdb/v5 v5.1.0-alpha - github.com/cerc-io/ipld-eth-statedb v0.1.0 - github.com/cerc-io/plugeth-statediff v0.3.1 + github.com/cerc-io/ipld-eth-statedb v0.1.1 + github.com/cerc-io/plugeth-statediff v0.3.2 github.com/ethereum/go-ethereum v1.13.14 github.com/google/uuid v1.6.0 github.com/graph-gophers/graphql-go v1.3.0 @@ -294,10 +294,3 @@ require ( lukechampine.com/blake3 v1.2.2 // indirect rsc.io/tmplfunc v0.0.3 // indirect ) - -replace ( - github.com/cerc-io/ipld-eth-statedb => github.com/cerc-io/ipld-eth-statedb v0.1.1-0.20240711085948-7bb5bbc6aa38 - github.com/cerc-io/plugeth-statediff => github.com/cerc-io/plugeth-statediff v0.3.2-0.20240711085659-6fe622e4bf33 -) - -// replace github.com/cerc-io/plugeth-statediff => git.vdb.to/cerc-io/plugeth-statediff v0.3.2-0.20240710110326-c8ce26de6de5 diff --git a/go.sum b/go.sum index c7e7cbc8..87119b8b 100644 --- a/go.sum +++ b/go.sum @@ -87,10 +87,10 @@ github.com/cerc-io/eth-testing v0.5.1 h1:xxcQf9ymJS0911yWIrUiGvCvqfvEjYmHvhBJkCD github.com/cerc-io/eth-testing v0.5.1/go.mod h1:p86je2PjSM7u8Qd7rMIG/Zw+tQlBoS5Emkh1ECnC5t0= github.com/cerc-io/ipfs-ethdb/v5 v5.1.0-alpha h1:+XhYHvzC3zFIvcWEb466SNDfeLrvcW3xe/d0cbVVVRA= github.com/cerc-io/ipfs-ethdb/v5 v5.1.0-alpha/go.mod h1:w5g07b6Uz+c6r/ySml58TEOJdUYAibYYF05H5ULVv2I= -github.com/cerc-io/ipld-eth-statedb v0.1.1-0.20240711085948-7bb5bbc6aa38 h1:8KzqJObmtnwGE8uXWfoUPG3HT86RTUCIXsipviEwPfA= -github.com/cerc-io/ipld-eth-statedb v0.1.1-0.20240711085948-7bb5bbc6aa38/go.mod h1:Vb8Ko2Nq0dUnJPZplPQp7kIClMBU0zHdH/p8n6rWUqA= -github.com/cerc-io/plugeth-statediff v0.3.2-0.20240711085659-6fe622e4bf33 h1:t0hSbThg5uX4r34eIFI0Pj2WbTIquDc+lKChLqieaMc= -github.com/cerc-io/plugeth-statediff v0.3.2-0.20240711085659-6fe622e4bf33/go.mod h1:r6Mzc6k4V9KD+iN9AXa/LmmRISDsQnnIwKzZMFjJ+eE= +github.com/cerc-io/ipld-eth-statedb v0.1.1 h1:QvhdO9jZqQu+NmiilW4Ef4N3qbdN4+FRHwVVbHg7Q/w= +github.com/cerc-io/ipld-eth-statedb v0.1.1/go.mod h1:+BF5xGXodtUFlNZ9W3JYsUHRmPyso+td8IgB+x/bYjM= +github.com/cerc-io/plugeth-statediff v0.3.2 h1:GuOUqDT6nJRCikyaNxDI2pA7TRnOTWOcyJGlJtwSzHY= +github.com/cerc-io/plugeth-statediff v0.3.2/go.mod h1:r6Mzc6k4V9KD+iN9AXa/LmmRISDsQnnIwKzZMFjJ+eE= github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk= github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= -- 2.45.2 From 91cc0aef2a58aa3c1f67bba1d3ebd6cdf0825dcd Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Sun, 14 Jul 2024 01:40:43 +0800 Subject: [PATCH 10/13] Revert "[dev] fnet stack ref" This reverts commit edf612de89d5bca4213c3afa1e9a03149d84db9f. --- .github/workflows/tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 97074f8c..ea8a3282 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -13,7 +13,7 @@ on: env: SO_VERSION: v1.1.0-36d4969-202407091537 - FIXTURENET_ETH_STACKS_REF: fix-plugeth-flags + FIXTURENET_ETH_STACKS_REF: main jobs: test: -- 2.45.2 From 685f1aa4dabc69921b028ce0433838a11bfeade1 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Thu, 18 Jul 2024 18:25:59 +0800 Subject: [PATCH 11/13] [wip] Run system-tests in CI --- .github/workflows/tests.yaml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index ea8a3282..e73721ce 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -13,7 +13,8 @@ on: env: SO_VERSION: v1.1.0-36d4969-202407091537 - FIXTURENET_ETH_STACKS_REF: main + FIXTURENET_ETH_STACKS_REF: roysc/set-validator-creds + SYSTEM_TESTS_REF: roysc/test-withdrawals jobs: test: @@ -83,6 +84,21 @@ jobs: go install github.com/onsi/ginkgo/v2/ginkgo ginkgo -v --label-filter '!proxy' -r ./integration + - name: Clone system-tests + uses: actions/checkout@v4 + with: + repository: cerc-io/system-tests + ref: ${{ env.SYSTEM_TESTS_REF }} + path: ./system-tests + token: ${{ secrets.CICD_REPO_TOKEN }} + progress: false + - name: Run system tests + working-directory: ./system-tests + run: | + pip install pytest + pip install -r requirements.txt + pytest -vv + - name: Run testnet stack without statediff env: CERC_RUN_STATEDIFF: false -- 2.45.2 From cecb7bf16c0e599864187e1bb63cf899a9a44897 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Thu, 25 Jul 2024 20:29:52 +0800 Subject: [PATCH 12/13] CI python linux arm install --- .github/workflows/tests.yaml | 10 +++++++++- scripts/run-test-stack.sh | 9 +++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index e73721ce..c9ba22ea 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -51,7 +51,15 @@ jobs: env: DEBIAN_FRONTEND: noninteractive run: apt-get update && apt-get install -y jq - - name: Install Python + # At present the stock setup-python action fails on Linux/aarch64 + # Conditional steps below workaroud this by using deadsnakes for that case only + - name: "Install Python for ARM on Linux" + if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }} + uses: deadsnakes/action@v3.0.1 + with: + python-version: 3.11 + - name: "Install Python cases other than ARM on Linux" + if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }} uses: actions/setup-python@v4 with: python-version: 3.11 diff --git a/scripts/run-test-stack.sh b/scripts/run-test-stack.sh index 6be1896e..3328b09d 100755 --- a/scripts/run-test-stack.sh +++ b/scripts/run-test-stack.sh @@ -1,21 +1,22 @@ #!/bin/bash -set -ex +set -e stack_dir=$(readlink -f "$1") [[ -d "$stack_dir" ]] +laconic_so="laconic-so --verbose --stack $stack_dir" + CONFIG_DIR=$(readlink -f "${CONFIG_DIR:-$(mktemp -d)}") # By default assume we are running in the project root. export CERC_REPO_BASE_DIR="${CERC_REPO_BASE_DIR:-$(git rev-parse --show-toplevel)/..}" -laconic_so="laconic-so --verbose --stack $stack_dir" - # Don't run geth/plugeth in the debugger, it will swallow error backtraces echo CERC_REMOTE_DEBUG=false >> $CONFIG_DIR/stack.env # Passing this lets us run eth_call forwarding tests without running ipld-eth-db echo CERC_RUN_STATEDIFF=${CERC_RUN_STATEDIFF:-true} >> $CONFIG_DIR/stack.env +set -x if [[ -z $SKIP_BUILD ]]; then # Prevent conflicting tty output @@ -35,7 +36,7 @@ fi set +x -# Get IPv4 endpoint of geth file server +# Get IPv4 endpoint of geth bootnode file server bootnode_endpoint=$(docker port test-fixturenet-eth-bootnode-geth-1 9898 | head -1) # Extract the chain config and ID from genesis file -- 2.45.2 From eef1912c40a75aa73c6e7ebd89d38b4acaffa88b Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Mon, 5 Aug 2024 09:10:35 -0500 Subject: [PATCH 13/13] update refs for CI --- .github/workflows/tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index c9ba22ea..87c59c0b 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -13,8 +13,8 @@ on: env: SO_VERSION: v1.1.0-36d4969-202407091537 - FIXTURENET_ETH_STACKS_REF: roysc/set-validator-creds - SYSTEM_TESTS_REF: roysc/test-withdrawals + FIXTURENET_ETH_STACKS_REF: main + SYSTEM_TESTS_REF: main jobs: test: -- 2.45.2