.gitea/workflows/test.yml

@@ -34,15 +34,9 @@ jobs:
       - uses: actions/checkout@v3
         with:
           path: ./plugeth-statediff
-      - uses: actions/checkout@v3
-        with:
-          repository: cerc-io/plugeth
-          ref: v1.13.14-cerc-0
-          path: ./plugeth
       - name: Build docker image
         run: |
           docker build ./plugeth-statediff -t cerc/plugeth-statediff:local
-          docker build ./plugeth -t cerc/plugeth:local
 
       - name: Install stack-orchestrator
         run: |

builder.go

@@ -168,6 +168,7 @@ func (sdb *builder) WriteStateDiff(
 
 // WriteStateDiff writes a statediff object to output sinks
 func (sdb *builder) WriteStateSnapshot(
+	ctx context.Context,
 	stateRoot common.Hash, params Params,
 	nodeSink sdtypes.StateNodeSink,
 	ipldSink sdtypes.IPLDSink,
@@ -200,7 +201,7 @@ func (sdb *builder) WriteStateSnapshot(
 		}
 	}
 	// errgroup will cancel if any group fails
-	g, ctx := errgroup.WithContext(context.Background())
+	g, ctx := errgroup.WithContext(ctx)
 	for i := range subiters {
 		func(subdiv uint) {
 			g.Go(func() error {

go.mod

@@ -48,7 +48,7 @@ require (
 	github.com/deckarep/golang-set/v2 v2.1.0 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
 	github.com/deepmap/oapi-codegen v1.8.2 // indirect
-	github.com/ethereum/c-kzg-4844/bindings/go v0.0.0-20230126171313-363c7d7593b4 // indirect
+	github.com/ethereum/c-kzg-4844 v0.4.0 // indirect
 	github.com/ferranbt/fastssz v0.1.2 // indirect
 	github.com/fjl/memsize v0.0.2 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
@@ -140,6 +140,6 @@ require (
 )
 
 replace (
-	github.com/ethereum/go-ethereum => git.vdb.to/cerc-io/plugeth v1.13.14-cerc-0
+	github.com/ethereum/go-ethereum => git.vdb.to/cerc-io/plugeth v1.13.14-cerc-2
-	github.com/openrelayxyz/plugeth-utils => git.vdb.to/cerc-io/plugeth-utils v1.5.0-cerc-0
+	github.com/openrelayxyz/plugeth-utils => git.vdb.to/cerc-io/plugeth-utils v1.5.0-cerc-1
 )

go.sum

@@ -1,8 +1,8 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-git.vdb.to/cerc-io/plugeth v1.13.14-cerc-0 h1:R57mkeo902vrDPzDZsXsjWxXSgtg/XjwNF/UdvvB7Vs=
+git.vdb.to/cerc-io/plugeth v1.13.14-cerc-2 h1:wUnIMCUP+e/F6f/JA1Ui51AagmYkxctEcyg66QJJj0o=
-git.vdb.to/cerc-io/plugeth v1.13.14-cerc-0/go.mod h1:1AMBRYYMPWAWPCK3ui469ymYlEsciWBrRJWjPX5nxy8=
+git.vdb.to/cerc-io/plugeth v1.13.14-cerc-2/go.mod h1:sUMNKCsvK1Afdogl+n8QTm9hmCX4fa0X3SqE+xru89k=
-git.vdb.to/cerc-io/plugeth-utils v1.5.0-cerc-0 h1:4GwCBbdLB8mCZINDzoUqpPq7aP4Ha5PPYCyG2h6ee6s=
+git.vdb.to/cerc-io/plugeth-utils v1.5.0-cerc-1 h1:WMdo9Pb5lAn0e2WC1CcD6/mRTWwU0r2KjFoEh0mh2rs=
-git.vdb.to/cerc-io/plugeth-utils v1.5.0-cerc-0/go.mod h1:COwKAuTZIsCouCOrIDBhvHZqpbOO1Ojgdy5KTvL8mJg=
+git.vdb.to/cerc-io/plugeth-utils v1.5.0-cerc-1/go.mod h1:Wf47tlE95PHZto1PMFRlmQAf98MBoNSRbwnQxeq0+Z0=
 github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8=
@@ -107,8 +107,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.m
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0=
 github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw=
-github.com/ethereum/c-kzg-4844/bindings/go v0.0.0-20230126171313-363c7d7593b4 h1:B2mpK+MNqgPqk2/KNi1LbqwtZDy5F7iy0mynQiBr8VA=
+github.com/ethereum/c-kzg-4844 v0.4.0 h1:3MS1s4JtA868KpJxroZoepdV0ZKBp3u/O5HcZ7R3nlY=
-github.com/ethereum/c-kzg-4844/bindings/go v0.0.0-20230126171313-363c7d7593b4/go.mod h1:y4GA2JbAUama1S4QwYjC2hefgGLU8Ul0GMtL/ADMF1c=
+github.com/ethereum/c-kzg-4844 v0.4.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0=
 github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
 github.com/ferranbt/fastssz v0.1.2 h1:Dky6dXlngF6Qjc+EfDipAkE83N5I5DE68bY6O0VLNPk=

indexer/mocks/test_data.go

@@ -42,7 +42,7 @@ var (
 	// block data
 	TestChainConfig = params.MainnetChainConfig
 	BlockNumber     = TestChainConfig.LondonBlock
-	BlockTime       = *TestChainConfig.CancunTime // TODO: verify this
+	BlockTime       = *TestChainConfig.CancunTime
 
 	// canonical block at London height
 	// includes 5 transactions: 3 Legacy + 1 EIP-2930 + 1 EIP-1559

scripts/integration-setup.sh

@@ -12,8 +12,8 @@ CONFIG_DIR=$(readlink -f "${CONFIG_DIR:-$(mktemp -d)}")
 # Point stack-orchestrator to the multi-project root
 export CERC_REPO_BASE_DIR="${CERC_REPO_BASE_DIR:-$(git rev-parse --show-toplevel)/..}"
 
-# v5 migrations only go up to version 18
+# v5 migrations only go up to version 20
-echo CERC_STATEDIFF_DB_GOOSE_MIN_VER=18 >> $CONFIG_DIR/stack.env
+echo CERC_STATEDIFF_DB_GOOSE_MIN_VER=20 >> $CONFIG_DIR/stack.env
 # don't run plugeth in the debugger
 echo CERC_REMOTE_DEBUG=false >> $CONFIG_DIR/stack.env
 
@@ -21,10 +21,10 @@ set -x
 
 if [[ -z $SKIP_BUILD ]]; then
   $laconic_so setup-repositories \
-    --exclude git.vdb.to/cerc-io/plugeth,git.vdb.to/cerc-io/plugeth-statediff
+    --exclude git.vdb.to/cerc-io/plugeth-statediff
-
+  # Assume the tested image has been built separately
   $laconic_so build-containers \
-    --exclude cerc/plugeth,cerc/plugeth-statediff
+    --exclude cerc/plugeth-statediff
 fi
 
 $laconic_so deploy \

service.go

@@ -825,7 +825,7 @@ func (sds *Service) writeStateDiff(block *types.Block, parentRoot common.Hash, p
 	}
 	defer tx.RollbackOnFailure(err)
 
-	// TODO: review/remove the need to sync here
+	// TODO: review necessity of locking here
 	var nodeMtx, ipldMtx sync.Mutex
 	nodeSink := func(node types2.StateLeafNode) error {
 		defer metrics.UpdateDuration(time.Now(), metrics.IndexerMetrics.OutputTimer)

test/compose.yml

@@ -3,7 +3,7 @@ services:
     restart: on-failure
     depends_on:
       - ipld-eth-db
-    image: git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:v5.0.5-alpha
+    image: git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:v5.2.1-alpha
     environment:
       DATABASE_USER: "vdbm"
       DATABASE_NAME: "cerc_testing"

test/stack.yml

@@ -1,8 +1,8 @@
 version: "1.2"
 name: fixturenet-plugeth-tx
-description: "plugeth Ethereum Fixturenet"
+description: "Plugeth Ethereum Fixturenet for testing plugeth-statediff"
 repos:
-  - git.vdb.to/cerc-io/plugeth@statediff
+  - git.vdb.to/cerc-io/plugeth@v1.5.0-cerc-2
   - git.vdb.to/cerc-io/plugeth-statediff
   - git.vdb.to/cerc-io/lighthouse
   - git.vdb.to/cerc-io/ipld-eth-db@v5.2.1-alpha

test_helpers/builder.go

@@ -2,6 +2,7 @@ package test_helpers
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"math/big"
 	"math/rand"
@@ -93,6 +94,7 @@ func RunStateSnapshot(
 				tr := tracker.New(recoveryFile, subtries)
 				defer tr.CloseAndSave()
 				return builder.WriteStateSnapshot(
+					context.Background(),
 					test.StateRoot, params, stateAppender, ipldAppender, tr,
 				)
 			}

test_helpers/chaingen/contract.go

@@ -0,0 +1,30 @@
+package chaingen
+
+import (
+	"strings"
+
+	"github.com/ethereum/go-ethereum/accounts/abi"
+	"github.com/ethereum/go-ethereum/common"
+)
+
+type ContractSpec struct {
+	DeploymentCode []byte
+	ABI            abi.ABI
+}
+
+func ParseContract(abiStr, binStr string) (*ContractSpec, error) {
+	parsedABI, err := abi.JSON(strings.NewReader(abiStr))
+	if err != nil {
+		return nil, err
+	}
+	data := common.Hex2Bytes(binStr)
+	return &ContractSpec{data, parsedABI}, nil
+}
+
+func MustParseContract(abiStr, binStr string) *ContractSpec {
+	spec, err := ParseContract(abiStr, binStr)
+	if err != nil {
+		panic(err)
+	}
+	return spec
+}

test_helpers/chaingen/gen.go

@@ -0,0 +1,150 @@
+package chaingen
+
+import (
+	"crypto/ecdsa"
+	"errors"
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/consensus/ethash"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/core/vm"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/params"
+)
+
+const secondsPerBlock = 12
+
+type GenContext struct {
+	ChainConfig *params.ChainConfig
+	GenFuncs    []func(int, *core.BlockGen)
+	DB          ethdb.Database
+
+	Keys      map[common.Address]*ecdsa.PrivateKey
+	Contracts map[string]*ContractSpec
+	Genesis   *types.Block
+
+	block    *core.BlockGen            // cache the current block for my methods' use
+	deployed map[common.Address]string // names of deployed contracts keyed by deployer
+	time     uint64                    // time at current block, in seconds
+}
+
+func NewGenContext(chainConfig *params.ChainConfig, db ethdb.Database) *GenContext {
+	return &GenContext{
+		ChainConfig: chainConfig,
+		DB:          db,
+		Keys:        make(map[common.Address]*ecdsa.PrivateKey),
+		Contracts:   make(map[string]*ContractSpec),
+
+		deployed: make(map[common.Address]string),
+	}
+}
+
+func (gen *GenContext) AddFunction(fn func(int, *core.BlockGen)) {
+	gen.GenFuncs = append(gen.GenFuncs, fn)
+}
+
+func (gen *GenContext) AddOwnedAccount(key *ecdsa.PrivateKey) common.Address {
+	addr := crypto.PubkeyToAddress(key.PublicKey)
+	gen.Keys[addr] = key
+	return addr
+}
+
+func (gen *GenContext) AddContract(name string, spec *ContractSpec) {
+	gen.Contracts[name] = spec
+}
+
+func (gen *GenContext) generate(i int, block *core.BlockGen) {
+	gen.block = block
+	for _, fn := range gen.GenFuncs {
+		fn(i, block)
+	}
+	gen.time += secondsPerBlock
+}
+
+// MakeChain creates a chain of n blocks starting at and including the genesis block.
+// the returned hash chain is ordered head->parent.
+func (gen *GenContext) MakeChain(n int) ([]*types.Block, []types.Receipts, *core.BlockChain) {
+	blocks, receipts := core.GenerateChain(
+		gen.ChainConfig, gen.Genesis, ethash.NewFaker(), gen.DB, n, gen.generate,
+	)
+	chain, err := core.NewBlockChain(gen.DB, nil, nil, nil, ethash.NewFaker(), vm.Config{}, nil, nil)
+	if err != nil {
+		panic(err)
+	}
+	return append([]*types.Block{gen.Genesis}, blocks...), receipts, chain
+}
+
+func (gen *GenContext) CreateSendTx(from common.Address, to common.Address, amount *big.Int) (*types.Transaction, error) {
+	return gen.createTx(from, &to, amount, params.TxGas, nil)
+}
+
+func (gen *GenContext) CreateContractTx(from common.Address, contractName string) (*types.Transaction, error) {
+	contract := gen.Contracts[contractName]
+	if contract == nil {
+		return nil, errors.New("No contract with name " + contractName)
+	}
+	return gen.createTx(from, nil, big.NewInt(0), 1000000, contract.DeploymentCode)
+}
+
+func (gen *GenContext) CreateCallTx(from common.Address, to common.Address, methodName string, args ...interface{}) (*types.Transaction, error) {
+	contractName, ok := gen.deployed[to]
+	if !ok {
+		return nil, errors.New("No contract deployed at address " + to.String())
+	}
+	contract := gen.Contracts[contractName]
+	if contract == nil {
+		return nil, errors.New("No contract with name " + contractName)
+	}
+
+	packed, err := contract.ABI.Pack(methodName, args...)
+	if err != nil {
+		panic(err)
+	}
+	return gen.createTx(from, &to, big.NewInt(0), 100000, packed)
+}
+
+func (gen *GenContext) DeployContract(from common.Address, contractName string) (common.Address, error) {
+	tx, err := gen.CreateContractTx(from, contractName)
+	if err != nil {
+		return common.Address{}, err
+	}
+	addr := crypto.CreateAddress(from, gen.block.TxNonce(from))
+	gen.deployed[addr] = contractName
+	gen.block.AddTx(tx)
+	return addr, nil
+}
+
+func (gen *GenContext) createTx(from common.Address, to *common.Address, amount *big.Int, gasLimit uint64, data []byte) (*types.Transaction, error) {
+	signer := types.MakeSigner(gen.ChainConfig, gen.block.Number(), gen.time)
+	nonce := gen.block.TxNonce(from)
+	priv, ok := gen.Keys[from]
+	if !ok {
+		return nil, errors.New("No private key for sender address" + from.String())
+	}
+
+	var tx *types.Transaction
+	if gen.ChainConfig.IsLondon(gen.block.Number()) {
+		tx = types.NewTx(&types.DynamicFeeTx{
+			ChainID:   gen.ChainConfig.ChainID,
+			Nonce:     nonce,
+			To:        to,
+			Gas:       gasLimit,
+			GasTipCap: big.NewInt(50),
+			GasFeeCap: big.NewInt(1000000000),
+			Value:     amount,
+			Data:      data,
+		})
+	} else {
+		tx = types.NewTx(&types.LegacyTx{
+			Nonce: nonce,
+			To:    to,
+			Value: amount,
+			Gas:   gasLimit,
+			Data:  data,
+		})
+	}
+	return types.SignTx(tx, signer, priv)
+}

test_helpers/db.go

@@ -0,0 +1,33 @@
+package test_helpers
+
+import (
+	"fmt"
+
+	"github.com/jmoiron/sqlx"
+)
+
+// ClearDB is used to empty the IPLD-ETH tables after tests
+func ClearDB(db *sqlx.DB) error {
+	tx, err := db.Beginx()
+	if err != nil {
+		return err
+	}
+	statements := []string{
+		`TRUNCATE nodes`,
+		`TRUNCATE ipld.blocks`,
+		`TRUNCATE eth.header_cids`,
+		`TRUNCATE eth.uncle_cids`,
+		`TRUNCATE eth.transaction_cids`,
+		`TRUNCATE eth.receipt_cids`,
+		`TRUNCATE eth.state_cids`,
+		`TRUNCATE eth.storage_cids`,
+		`TRUNCATE eth.log_cids`,
+		`TRUNCATE eth_meta.watched_addresses`,
+	}
+	for _, stm := range statements {
+		if _, err = tx.Exec(stm); err != nil {
+			return fmt.Errorf("error executing `%s`: %w", stm, err)
+		}
+	}
+	return tx.Commit()
+}

test_helpers/indexing.go

@@ -0,0 +1,100 @@
+package test_helpers
+
+import (
+	"context"
+	"fmt"
+	"math/big"
+
+	"github.com/cerc-io/plugeth-statediff"
+	"github.com/cerc-io/plugeth-statediff/adapt"
+	"github.com/cerc-io/plugeth-statediff/indexer"
+	"github.com/cerc-io/plugeth-statediff/indexer/interfaces"
+	"github.com/cerc-io/plugeth-statediff/indexer/node"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/state"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/params"
+)
+
+type IndexChainParams struct {
+	Blocks     []*types.Block
+	Receipts   []types.Receipts
+	StateCache state.Database
+
+	StateDiffParams statediff.Params
+	TotalDifficulty *big.Int
+	// Whether to skip indexing state nodes (state_cids, storage_cids)
+	SkipStateNodes bool
+	// Whether to skip indexing IPLD blocks
+	SkipIPLDs bool
+}
+
+func NewIndexer(ctx context.Context, chainConfig *params.ChainConfig, genHash common.Hash, dbconfig interfaces.Config) (interfaces.StateDiffIndexer, error) {
+	testInfo := node.Info{
+		GenesisBlock: genHash.String(),
+		NetworkID:    "1",
+		ID:           "1",
+		ClientName:   "geth",
+		ChainID:      chainConfig.ChainID.Uint64(),
+	}
+	_, indexer, err := indexer.NewStateDiffIndexer(ctx, chainConfig, testInfo, dbconfig, true)
+	return indexer, err
+}
+
+func IndexChain(indexer interfaces.StateDiffIndexer, params IndexChainParams) error {
+	builder := statediff.NewBuilder(adapt.GethStateView(params.StateCache))
+	// iterate over the blocks, generating statediff payloads, and transforming the data into Postgres
+	for i, block := range params.Blocks {
+		var args statediff.Args
+		var rcts types.Receipts
+		if i == 0 {
+			args = statediff.Args{
+				OldStateRoot: common.Hash{},
+				NewStateRoot: block.Root(),
+				BlockNumber:  block.Number(),
+				BlockHash:    block.Hash(),
+			}
+		} else {
+			args = statediff.Args{
+				OldStateRoot: params.Blocks[i-1].Root(),
+				NewStateRoot: block.Root(),
+				BlockNumber:  block.Number(),
+				BlockHash:    block.Hash(),
+			}
+			rcts = params.Receipts[i-1]
+		}
+
+		diff, err := builder.BuildStateDiffObject(args, params.StateDiffParams)
+		if err != nil {
+			return fmt.Errorf("failed to build diff (block %d): %w", block.Number(), err)
+		}
+		tx, err := indexer.PushBlock(block, rcts, params.TotalDifficulty)
+		if err != nil {
+			return fmt.Errorf("failed to index block (block %d): %w", block.Number(), err)
+		}
+		defer tx.RollbackOnFailure(err)
+
+		if !params.SkipStateNodes {
+			for _, node := range diff.Nodes {
+				if err = indexer.PushStateNode(tx, node, block.Hash().String()); err != nil {
+					if err != nil {
+						return fmt.Errorf("failed to index state node: %w", err)
+					}
+				}
+			}
+		}
+		if !params.SkipIPLDs {
+			for _, ipld := range diff.IPLDs {
+				if err := indexer.PushIPLD(tx, ipld); err != nil {
+					if err != nil {
+						return fmt.Errorf("failed to index IPLD: %w", err)
+					}
+				}
+			}
+		}
+		if err = tx.Submit(); err != nil {
+			return fmt.Errorf("failed to commit diff: %w", err)
+		}
+	}
+	return nil
+}