Merge pull request #35 from cerc-io/update-schema

Update go-ethereum and ipld-eth-db

go.mod

@@ -4,9 +4,9 @@ go 1.19
 
 require (
 	github.com/cerc-io/ipfs-ethdb/v5 v5.0.1-alpha
-	github.com/cerc-io/ipld-eth-server/v5 v5.0.0-alpha
+	github.com/cerc-io/ipld-eth-server/v5 v5.0.1-alpha
 	github.com/cerc-io/ipld-eth-statedb v0.0.5-alpha
-	github.com/ethereum/go-ethereum v1.11.5
+	github.com/ethereum/go-ethereum v1.11.6
 	github.com/jmoiron/sqlx v1.3.5
 	github.com/onsi/ginkgo/v2 v2.9.2
 	github.com/onsi/gomega v1.27.4
@@ -266,6 +266,7 @@ require (
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
+	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
 	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
@@ -274,4 +275,4 @@ require (
 	lukechampine.com/blake3 v1.1.7 // indirect
 )
 
-replace github.com/ethereum/go-ethereum => github.com/cerc-io/go-ethereum v1.11.5-statediff-5.0.5-alpha
+replace github.com/ethereum/go-ethereum => github.com/cerc-io/go-ethereum v1.11.6-statediff-5.0.8-alpha

go.sum

@@ -50,6 +50,7 @@ github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOv
 github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIoKjsnZuH8vjyaysT/ses3EvZeaV/1UkF2M=
 github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno=
 github.com/CloudyKit/jet/v3 v3.0.0/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMdUywE7VMo=
@@ -135,12 +136,12 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA
 github.com/ceramicnetwork/go-dag-jose v0.1.0 h1:yJ/HVlfKpnD3LdYP03AHyTvbm3BpPiz2oZiOeReJRdU=
 github.com/cerc-io/eth-ipfs-state-validator/v5 v5.0.0-alpha h1:TGCr/v0CrDsz1Mjr4000omuEGw7RNdn+OYqIivlF3+Q=
 github.com/cerc-io/eth-ipfs-state-validator/v5 v5.0.0-alpha/go.mod h1:t+1UYws60dkLRecMN2NXl4LlKxQBLjhDh34swi6Jcvc=
-github.com/cerc-io/go-ethereum v1.11.5-statediff-5.0.5-alpha h1:Rj/5+dDbYWa5k58g7h1jNytbWa0NY8IEKExFWaI8bcA=
+github.com/cerc-io/go-ethereum v1.11.6-statediff-5.0.8-alpha h1:fgCXE7wM7Sy3Ioe6TDMU4uGIfB9GRLumNd0bMh45D/k=
-github.com/cerc-io/go-ethereum v1.11.5-statediff-5.0.5-alpha/go.mod h1:DIk2wFexjyzvyjuzSOtBEIAPRNZTnLXNbIHEyq1Igek=
+github.com/cerc-io/go-ethereum v1.11.6-statediff-5.0.8-alpha/go.mod h1:Jc6zFIJJFvhRMDxbj7kfUFlg6NbSQiS4RyEMDavuF1w=
 github.com/cerc-io/ipfs-ethdb/v5 v5.0.1-alpha h1:Q+GgPclJeuArafBSJofHmkVOVKkkyxu5qu5gORYAv1E=
 github.com/cerc-io/ipfs-ethdb/v5 v5.0.1-alpha/go.mod h1:EGAdV/YewEADFDDVF1k9GNwy8vNWR29Xb87sRHgMIng=
-github.com/cerc-io/ipld-eth-server/v5 v5.0.0-alpha h1:4BcZZd9NsXNQyffD53RhNkbeC2OqXwMFGpFhtNqMCWU=
+github.com/cerc-io/ipld-eth-server/v5 v5.0.1-alpha h1:P01+2SOxMLTwP/mjF46ohOAMtygmnXTcFA9LNtPVBDg=
-github.com/cerc-io/ipld-eth-server/v5 v5.0.0-alpha/go.mod h1:bNZ/mgRWVHc3zP2zRIDJqqH7D6HRan7GZXhzZgT99j8=
+github.com/cerc-io/ipld-eth-server/v5 v5.0.1-alpha/go.mod h1:IKMTYwo0pj4bs/7NY8BGb8cmmjhqMdkp0IIN4R1CzcU=
 github.com/cerc-io/ipld-eth-statedb v0.0.5-alpha h1:olsE5OCvfTrQGLE5T2Z4VYOfx8n5ONgciKdEW5I0x3I=
 github.com/cerc-io/ipld-eth-statedb v0.0.5-alpha/go.mod h1:6XprlGrhm65KkHesgcLwgIHWD0TvbkQ3T9ZsfBXKvsk=
 github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk=
@@ -2162,6 +2163,8 @@ gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
 gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
 gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU=
 gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=

integration/util_test.go

@@ -6,7 +6,7 @@ import (
 
 type atomicBlockSet struct {
 	blocks map[uint64]struct{}
-	sync.Mutex
+	sync.RWMutex
 }
 
 func newBlockSet() *atomicBlockSet {
@@ -14,14 +14,10 @@ func newBlockSet() *atomicBlockSet {
 }
 
 func (set *atomicBlockSet) contains(block uint64) bool {
-	set.Lock()
+	set.RLock()
-	defer set.Unlock()
+	defer set.RUnlock()
-	for done := range set.blocks {
+	_, has := set.blocks[block]
-		if done == block {
+	return has
-			return true
-		}
-	}
-	return false
 }
 
 func (set *atomicBlockSet) containsAll(blocks []uint64) bool {

internal/helpers/indexing.go

@@ -2,6 +2,7 @@ package helpers
 
 import (
 	"context"
+	"fmt"
 	"math/big"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -64,29 +65,33 @@ func IndexChain(indexer interfaces.StateDiffIndexer, params IndexChainParams) er
 
 		diff, err := builder.BuildStateDiffObject(args, params.StateDiffParams)
 		if err != nil {
-			return err
+			return fmt.Errorf("failed to build diff (block %d): %w", block.Number(), err)
 		}
 		tx, err := indexer.PushBlock(block, rcts, params.TotalDifficulty)
 		if err != nil {
-			return err
+			return fmt.Errorf("failed to index block (block %d): %w", block.Number(), err)
 		}
 
 		if !params.SkipStateNodes {
 			for _, node := range diff.Nodes {
 				if err = indexer.PushStateNode(tx, node, block.Hash().String()); err != nil {
-					return err
+					if err != nil {
+						return fmt.Errorf("failed to index state node: %w", err)
+					}
 				}
 			}
 		}
 		if !params.SkipIPLDs {
 			for _, ipld := range diff.IPLDs {
 				if err := indexer.PushIPLD(tx, ipld); err != nil {
-					return err
+					if err != nil {
+						return fmt.Errorf("failed to index IPLD: %w", err)
+					}
 				}
 			}
 		}
 		if err = tx.Submit(err); err != nil {
-			return err
+			return fmt.Errorf("failed to commit diff: %w", err)
 		}
 	}
 	return nil

pkg/validator/ref_integrity_test.go

@@ -240,6 +240,14 @@ var _ = Describe("referential integrity", func() {
 		})
 	})
 
+	Describe("ValidateReferentialIntegrity", func() {
+		It("Validates referential integrity of full chain", func() {
+			for i := uint64(startBlock); i <= chainLength; i++ {
+				err := validator.ValidateReferentialIntegrity(tx, i)
+				Expect(err).ToNot(HaveOccurred())
+			}
+		})
+	})
 })
 
 func deleteEntriesFrom(tx *sqlx.Tx, tableName string) error {

pkg/validator/validator.go

@@ -304,7 +304,7 @@ func applyTransactions(block *types.Block, backend *ipldeth.Backend) (*ipldstate
 	nrOrHash := rpc.BlockNumberOrHash{BlockHash: &parentHash}
 	statedb, _, err := backend.IPLDTrieStateDBAndHeaderByNumberOrHash(context.Background(), nrOrHash)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("error accessing state DB: %w", err)
 	}
 
 	var gp core.GasPool
@@ -319,7 +319,7 @@ func applyTransactions(block *types.Block, backend *ipldeth.Backend) (*ipldstate
 	for i, tx := range block.Transactions() {
 		msg, err := core.TransactionToMessage(tx, signer, block.BaseFee())
 		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("error converting transaction to message: %w", err)
 		}
 		statedb.SetTxContext(tx.Hash(), i)
 		statedb.Prepare(rules, msg.From, block.Coinbase(), msg.To, nil, nil)

pkg/validator/validator_test.go

@@ -37,7 +37,7 @@ func init() {
 	log.Root().SetHandler(log.DiscardHandler())
 }
 
-func setup(t *testing.T) *sqlx.DB {
+func setupStateValidator(t *testing.T) *sqlx.DB {
 	// Make the test blockchain and state
 	gen := chaingen.DefaultGenContext(chainConfig, testDB)
 	blocks, receipts, chain := gen.MakeChain(chainLength)
@@ -50,14 +50,8 @@ func setup(t *testing.T) *sqlx.DB {
 	if err != nil {
 		t.Fatal(err)
 	}
-	helpers.IndexChain(indexer, helpers.IndexChainParams{
-		StateCache:      chain.StateCache(),
-		Blocks:          blocks,
-		Receipts:        receipts,
-		TotalDifficulty: mockTD,
-	})
 
-	// Insert some non-canonical data into the database so that we test our ability to discern canonicity
+	{ // Insert some non-canonical data into the database so that we test our ability to discern canonicity
 		tx, err := indexer.PushBlock(server_mocks.MockBlock, server_mocks.MockReceipts,
 			server_mocks.MockBlock.Difficulty())
 		if err != nil {
@@ -70,7 +64,8 @@ func setup(t *testing.T) *sqlx.DB {
 		}
 
 		// The non-canonical header has a child
-	tx, err = indexer.PushBlock(server_mocks.MockChild, server_mocks.MockReceipts, server_mocks.MockChild.Difficulty())
+		tx, err = indexer.PushBlock(server_mocks.MockChild, server_mocks.MockReceipts,
+			server_mocks.MockChild.Difficulty())
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -88,6 +83,16 @@ func setup(t *testing.T) *sqlx.DB {
 		if err != nil {
 			t.Fatal(err)
 		}
+	}
+
+	if err := helpers.IndexChain(indexer, helpers.IndexChainParams{
+		StateCache:      chain.StateCache(),
+		Blocks:          blocks,
+		Receipts:        receipts,
+		TotalDifficulty: mockTD,
+	}); err != nil {
+		t.Fatal(err)
+	}
 
 	db := helpers.SetupDB()
 
@@ -98,7 +103,7 @@ func setup(t *testing.T) *sqlx.DB {
 }
 
 func TestStateValidation(t *testing.T) {
-	db := setup(t)
+	db := setupStateValidator(t)
 
 	t.Run("Validator", func(t *testing.T) {
 		api, err := validator.EthAPI(context.Background(), db, chainConfig)
@@ -119,13 +124,6 @@ func TestStateValidation(t *testing.T) {
 			if err != nil {
 				t.Fatal(err)
 			}
-
-			tx := db.MustBegin()
-			defer tx.Rollback()
-			err = validator.ValidateReferentialIntegrity(tx, i)
-			if err != nil {
-				t.Fatal(err)
-			}
 		}
 	})
 }

test/compose-db.yml

@@ -5,7 +5,7 @@ services:
     restart: on-failure
     depends_on:
       - ipld-eth-db
-    image: git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:v5.0.2-alpha
+    image: git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:v5.0.5-alpha
     environment:
       DATABASE_USER: "vdbm"
       DATABASE_NAME: "cerc_testing"

test/stack-refs.txt

@@ -1,2 +1,2 @@
-github.com/cerc-io/go-ethereum v1.11.6-statediff-5.0.5-alpha
+github.com/cerc-io/go-ethereum v1.11.6-statediff-5.0.8-alpha
-github.com/cerc-io/ipld-eth-db v5.0.2-alpha
+github.com/cerc-io/ipld-eth-db v5.0.5-alpha