From b4c9f1e8642d81d58ef7deb9f3b31230b9542524 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Thu, 10 Aug 2023 17:02:50 +0800 Subject: [PATCH 1/4] Update go-ethereum and ipld-eth-db schema - index non-canonical data first in tests --- go.mod | 5 ++- go.sum | 7 ++- internal/helpers/indexing.go | 15 ++++--- pkg/validator/validator.go | 4 +- pkg/validator/validator_test.go | 80 ++++++++++++++++----------------- test/compose-db.yml | 2 +- test/stack-refs.txt | 4 +- 7 files changed, 62 insertions(+), 55 deletions(-) diff --git a/go.mod b/go.mod index 228a9ff..86b7237 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/cerc-io/ipfs-ethdb/v5 v5.0.1-alpha github.com/cerc-io/ipld-eth-server/v5 v5.0.0-alpha github.com/cerc-io/ipld-eth-statedb v0.0.5-alpha - github.com/ethereum/go-ethereum v1.11.5 + github.com/ethereum/go-ethereum v1.11.6 github.com/jmoiron/sqlx v1.3.5 github.com/onsi/ginkgo/v2 v2.9.2 github.com/onsi/gomega v1.27.4 @@ -266,6 +266,7 @@ require ( golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect @@ -274,4 +275,4 @@ require ( lukechampine.com/blake3 v1.1.7 // indirect ) -replace github.com/ethereum/go-ethereum => github.com/cerc-io/go-ethereum v1.11.5-statediff-5.0.5-alpha +replace github.com/ethereum/go-ethereum => github.com/cerc-io/go-ethereum v1.11.6-statediff-5.0.8-alpha diff --git a/go.sum b/go.sum index 81c66b0..cf2d483 100644 --- a/go.sum +++ b/go.sum @@ -50,6 +50,7 @@ github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOv github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIoKjsnZuH8vjyaysT/ses3EvZeaV/1UkF2M= github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno= github.com/CloudyKit/jet/v3 v3.0.0/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMdUywE7VMo= @@ -135,8 +136,8 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/ceramicnetwork/go-dag-jose v0.1.0 h1:yJ/HVlfKpnD3LdYP03AHyTvbm3BpPiz2oZiOeReJRdU= github.com/cerc-io/eth-ipfs-state-validator/v5 v5.0.0-alpha h1:TGCr/v0CrDsz1Mjr4000omuEGw7RNdn+OYqIivlF3+Q= github.com/cerc-io/eth-ipfs-state-validator/v5 v5.0.0-alpha/go.mod h1:t+1UYws60dkLRecMN2NXl4LlKxQBLjhDh34swi6Jcvc= -github.com/cerc-io/go-ethereum v1.11.5-statediff-5.0.5-alpha h1:Rj/5+dDbYWa5k58g7h1jNytbWa0NY8IEKExFWaI8bcA= -github.com/cerc-io/go-ethereum v1.11.5-statediff-5.0.5-alpha/go.mod h1:DIk2wFexjyzvyjuzSOtBEIAPRNZTnLXNbIHEyq1Igek= +github.com/cerc-io/go-ethereum v1.11.6-statediff-5.0.8-alpha h1:fgCXE7wM7Sy3Ioe6TDMU4uGIfB9GRLumNd0bMh45D/k= +github.com/cerc-io/go-ethereum v1.11.6-statediff-5.0.8-alpha/go.mod h1:Jc6zFIJJFvhRMDxbj7kfUFlg6NbSQiS4RyEMDavuF1w= github.com/cerc-io/ipfs-ethdb/v5 v5.0.1-alpha h1:Q+GgPclJeuArafBSJofHmkVOVKkkyxu5qu5gORYAv1E= github.com/cerc-io/ipfs-ethdb/v5 v5.0.1-alpha/go.mod h1:EGAdV/YewEADFDDVF1k9GNwy8vNWR29Xb87sRHgMIng= github.com/cerc-io/ipld-eth-server/v5 v5.0.0-alpha h1:4BcZZd9NsXNQyffD53RhNkbeC2OqXwMFGpFhtNqMCWU= @@ -2162,6 +2163,8 @@ gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= +gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= +gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= diff --git a/internal/helpers/indexing.go b/internal/helpers/indexing.go index 7d2401f..08d0f68 100644 --- a/internal/helpers/indexing.go +++ b/internal/helpers/indexing.go @@ -2,6 +2,7 @@ package helpers import ( "context" + "fmt" "math/big" "github.com/ethereum/go-ethereum/common" @@ -64,29 +65,33 @@ func IndexChain(indexer interfaces.StateDiffIndexer, params IndexChainParams) er diff, err := builder.BuildStateDiffObject(args, params.StateDiffParams) if err != nil { - return err + return fmt.Errorf("failed to build diff (block %d): %w", block.Number(), err) } tx, err := indexer.PushBlock(block, rcts, params.TotalDifficulty) if err != nil { - return err + return fmt.Errorf("failed to index block (block %d): %w", block.Number(), err) } if !params.SkipStateNodes { for _, node := range diff.Nodes { if err = indexer.PushStateNode(tx, node, block.Hash().String()); err != nil { - return err + if err != nil { + return fmt.Errorf("failed to index state node: %w", err) + } } } } if !params.SkipIPLDs { for _, ipld := range diff.IPLDs { if err := indexer.PushIPLD(tx, ipld); err != nil { - return err + if err != nil { + return fmt.Errorf("failed to index IPLD: %w", err) + } } } } if err = tx.Submit(err); err != nil { - return err + return fmt.Errorf("failed to commit diff: %w", err) } } return nil diff --git a/pkg/validator/validator.go b/pkg/validator/validator.go index 505b78c..4c61e9c 100644 --- a/pkg/validator/validator.go +++ b/pkg/validator/validator.go @@ -304,7 +304,7 @@ func applyTransactions(block *types.Block, backend *ipldeth.Backend) (*ipldstate nrOrHash := rpc.BlockNumberOrHash{BlockHash: &parentHash} statedb, _, err := backend.IPLDTrieStateDBAndHeaderByNumberOrHash(context.Background(), nrOrHash) if err != nil { - return nil, err + return nil, fmt.Errorf("error accessing state DB: %w", err) } var gp core.GasPool @@ -319,7 +319,7 @@ func applyTransactions(block *types.Block, backend *ipldeth.Backend) (*ipldstate for i, tx := range block.Transactions() { msg, err := core.TransactionToMessage(tx, signer, block.BaseFee()) if err != nil { - return nil, err + return nil, fmt.Errorf("error converting transaction to message: %w", err) } statedb.SetTxContext(tx.Hash(), i) statedb.Prepare(rules, msg.From, block.Coinbase(), msg.To, nil, nil) diff --git a/pkg/validator/validator_test.go b/pkg/validator/validator_test.go index 8ecf3a4..f425100 100644 --- a/pkg/validator/validator_test.go +++ b/pkg/validator/validator_test.go @@ -37,7 +37,7 @@ func init() { log.Root().SetHandler(log.DiscardHandler()) } -func setup(t *testing.T) *sqlx.DB { +func setupStateValidator(t *testing.T) *sqlx.DB { // Make the test blockchain and state gen := chaingen.DefaultGenContext(chainConfig, testDB) blocks, receipts, chain := gen.MakeChain(chainLength) @@ -50,42 +50,47 @@ func setup(t *testing.T) *sqlx.DB { if err != nil { t.Fatal(err) } - helpers.IndexChain(indexer, helpers.IndexChainParams{ + + { // Insert some non-canonical data into the database so that we test our ability to discern canonicity + tx, err := indexer.PushBlock(server_mocks.MockBlock, server_mocks.MockReceipts, + server_mocks.MockBlock.Difficulty()) + if err != nil { + t.Fatal(err) + } + + err = tx.Submit(err) + if err != nil { + t.Fatal(err) + } + + // The non-canonical header has a child + tx, err = indexer.PushBlock(server_mocks.MockChild, server_mocks.MockReceipts, + server_mocks.MockChild.Difficulty()) + if err != nil { + t.Fatal(err) + } + + ipld := sdtypes.IPLD{ + CID: ipld.Keccak256ToCid(ipld.RawBinary, server_mocks.CodeHash.Bytes()).String(), + Content: server_mocks.ContractCode, + } + err = indexer.PushIPLD(tx, ipld) + if err != nil { + t.Fatal(err) + } + + err = tx.Submit(err) + if err != nil { + t.Fatal(err) + } + } + + if err := helpers.IndexChain(indexer, helpers.IndexChainParams{ StateCache: chain.StateCache(), Blocks: blocks, Receipts: receipts, TotalDifficulty: mockTD, - }) - - // Insert some non-canonical data into the database so that we test our ability to discern canonicity - tx, err := indexer.PushBlock(server_mocks.MockBlock, server_mocks.MockReceipts, - server_mocks.MockBlock.Difficulty()) - if err != nil { - t.Fatal(err) - } - - err = tx.Submit(err) - if err != nil { - t.Fatal(err) - } - - // The non-canonical header has a child - tx, err = indexer.PushBlock(server_mocks.MockChild, server_mocks.MockReceipts, server_mocks.MockChild.Difficulty()) - if err != nil { - t.Fatal(err) - } - - ipld := sdtypes.IPLD{ - CID: ipld.Keccak256ToCid(ipld.RawBinary, server_mocks.CodeHash.Bytes()).String(), - Content: server_mocks.ContractCode, - } - err = indexer.PushIPLD(tx, ipld) - if err != nil { - t.Fatal(err) - } - - err = tx.Submit(err) - if err != nil { + }); err != nil { t.Fatal(err) } @@ -98,7 +103,7 @@ func setup(t *testing.T) *sqlx.DB { } func TestStateValidation(t *testing.T) { - db := setup(t) + db := setupStateValidator(t) t.Run("Validator", func(t *testing.T) { api, err := validator.EthAPI(context.Background(), db, chainConfig) @@ -119,13 +124,6 @@ func TestStateValidation(t *testing.T) { if err != nil { t.Fatal(err) } - - tx := db.MustBegin() - defer tx.Rollback() - err = validator.ValidateReferentialIntegrity(tx, i) - if err != nil { - t.Fatal(err) - } } }) } diff --git a/test/compose-db.yml b/test/compose-db.yml index d0ba3a2..15c7422 100644 --- a/test/compose-db.yml +++ b/test/compose-db.yml @@ -5,7 +5,7 @@ services: restart: on-failure depends_on: - ipld-eth-db - image: git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:v5.0.2-alpha + image: git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:v5.0.5-alpha environment: DATABASE_USER: "vdbm" DATABASE_NAME: "cerc_testing" diff --git a/test/stack-refs.txt b/test/stack-refs.txt index 0320d8a..27950ca 100644 --- a/test/stack-refs.txt +++ b/test/stack-refs.txt @@ -1,2 +1,2 @@ -github.com/cerc-io/go-ethereum v1.11.6-statediff-5.0.5-alpha -github.com/cerc-io/ipld-eth-db v5.0.2-alpha +github.com/cerc-io/go-ethereum v1.11.6-statediff-5.0.8-alpha +github.com/cerc-io/ipld-eth-db v5.0.5-alpha From a512d3f67bfc0bd3229e814cf6e83a47ad590aaf Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Thu, 10 Aug 2023 22:34:37 +0800 Subject: [PATCH 2/4] add full chain ref integ. test --- pkg/validator/ref_integrity_test.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/validator/ref_integrity_test.go b/pkg/validator/ref_integrity_test.go index 19f5cd8..a3775aa 100644 --- a/pkg/validator/ref_integrity_test.go +++ b/pkg/validator/ref_integrity_test.go @@ -240,6 +240,14 @@ var _ = Describe("referential integrity", func() { }) }) + Describe("ValidateReferentialIntegrity", func() { + It("Validates referential integrity of full chain", func() { + for i := uint64(startBlock); i <= chainLength; i++ { + err := validator.ValidateReferentialIntegrity(tx, i) + Expect(err).ToNot(HaveOccurred()) + } + }) + }) }) func deleteEntriesFrom(tx *sqlx.Tx, tableName string) error { From ed8c39dd4180c1dfe9fb8c86f8bd22f717b17713 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Fri, 11 Aug 2023 14:31:49 +0800 Subject: [PATCH 3/4] update ipld-eth-server --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 86b7237..60c111a 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.19 require ( github.com/cerc-io/ipfs-ethdb/v5 v5.0.1-alpha - github.com/cerc-io/ipld-eth-server/v5 v5.0.0-alpha + github.com/cerc-io/ipld-eth-server/v5 v5.0.1-alpha github.com/cerc-io/ipld-eth-statedb v0.0.5-alpha github.com/ethereum/go-ethereum v1.11.6 github.com/jmoiron/sqlx v1.3.5 diff --git a/go.sum b/go.sum index cf2d483..5c80d77 100644 --- a/go.sum +++ b/go.sum @@ -140,8 +140,8 @@ github.com/cerc-io/go-ethereum v1.11.6-statediff-5.0.8-alpha h1:fgCXE7wM7Sy3Ioe6 github.com/cerc-io/go-ethereum v1.11.6-statediff-5.0.8-alpha/go.mod h1:Jc6zFIJJFvhRMDxbj7kfUFlg6NbSQiS4RyEMDavuF1w= github.com/cerc-io/ipfs-ethdb/v5 v5.0.1-alpha h1:Q+GgPclJeuArafBSJofHmkVOVKkkyxu5qu5gORYAv1E= github.com/cerc-io/ipfs-ethdb/v5 v5.0.1-alpha/go.mod h1:EGAdV/YewEADFDDVF1k9GNwy8vNWR29Xb87sRHgMIng= -github.com/cerc-io/ipld-eth-server/v5 v5.0.0-alpha h1:4BcZZd9NsXNQyffD53RhNkbeC2OqXwMFGpFhtNqMCWU= -github.com/cerc-io/ipld-eth-server/v5 v5.0.0-alpha/go.mod h1:bNZ/mgRWVHc3zP2zRIDJqqH7D6HRan7GZXhzZgT99j8= +github.com/cerc-io/ipld-eth-server/v5 v5.0.1-alpha h1:P01+2SOxMLTwP/mjF46ohOAMtygmnXTcFA9LNtPVBDg= +github.com/cerc-io/ipld-eth-server/v5 v5.0.1-alpha/go.mod h1:IKMTYwo0pj4bs/7NY8BGb8cmmjhqMdkp0IIN4R1CzcU= github.com/cerc-io/ipld-eth-statedb v0.0.5-alpha h1:olsE5OCvfTrQGLE5T2Z4VYOfx8n5ONgciKdEW5I0x3I= github.com/cerc-io/ipld-eth-statedb v0.0.5-alpha/go.mod h1:6XprlGrhm65KkHesgcLwgIHWD0TvbkQ3T9ZsfBXKvsk= github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk= From 9ab1589d2d40fdedfaa5f9bbb6589e4acba551dd Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Fri, 11 Aug 2023 15:57:58 +0800 Subject: [PATCH 4/4] improve test util --- integration/util_test.go | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/integration/util_test.go b/integration/util_test.go index 3f20d19..45ba823 100644 --- a/integration/util_test.go +++ b/integration/util_test.go @@ -6,7 +6,7 @@ import ( type atomicBlockSet struct { blocks map[uint64]struct{} - sync.Mutex + sync.RWMutex } func newBlockSet() *atomicBlockSet { @@ -14,14 +14,10 @@ func newBlockSet() *atomicBlockSet { } func (set *atomicBlockSet) contains(block uint64) bool { - set.Lock() - defer set.Unlock() - for done := range set.blocks { - if done == block { - return true - } - } - return false + set.RLock() + defer set.RUnlock() + _, has := set.blocks[block] + return has } func (set *atomicBlockSet) containsAll(blocks []uint64) bool {