diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml index 41483cf..d302e68 100644 --- a/.gitea/workflows/test.yml +++ b/.gitea/workflows/test.yml @@ -34,15 +34,9 @@ jobs: - uses: actions/checkout@v3 with: path: ./plugeth-statediff - - uses: actions/checkout@v3 - with: - repository: cerc-io/plugeth - ref: v1.13.14-cerc-0 - path: ./plugeth - name: Build docker image run: | docker build ./plugeth-statediff -t cerc/plugeth-statediff:local - docker build ./plugeth -t cerc/plugeth:local - name: Install stack-orchestrator run: | diff --git a/builder.go b/builder.go index 16a02ac..c66164c 100644 --- a/builder.go +++ b/builder.go @@ -168,6 +168,7 @@ func (sdb *builder) WriteStateDiff( // WriteStateDiff writes a statediff object to output sinks func (sdb *builder) WriteStateSnapshot( + ctx context.Context, stateRoot common.Hash, params Params, nodeSink sdtypes.StateNodeSink, ipldSink sdtypes.IPLDSink, @@ -200,7 +201,7 @@ func (sdb *builder) WriteStateSnapshot( } } // errgroup will cancel if any group fails - g, ctx := errgroup.WithContext(context.Background()) + g, ctx := errgroup.WithContext(ctx) for i := range subiters { func(subdiv uint) { g.Go(func() error { diff --git a/go.mod b/go.mod index 343ea03..6810234 100644 --- a/go.mod +++ b/go.mod @@ -48,7 +48,7 @@ require ( github.com/deckarep/golang-set/v2 v2.1.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect github.com/deepmap/oapi-codegen v1.8.2 // indirect - github.com/ethereum/c-kzg-4844/bindings/go v0.0.0-20230126171313-363c7d7593b4 // indirect + github.com/ethereum/c-kzg-4844 v0.4.0 // indirect github.com/ferranbt/fastssz v0.1.2 // indirect github.com/fjl/memsize v0.0.2 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect @@ -140,6 +140,6 @@ require ( ) replace ( - github.com/ethereum/go-ethereum => git.vdb.to/cerc-io/plugeth v1.13.14-cerc-0 - github.com/openrelayxyz/plugeth-utils => git.vdb.to/cerc-io/plugeth-utils v1.5.0-cerc-0 + github.com/ethereum/go-ethereum => git.vdb.to/cerc-io/plugeth v1.13.14-cerc-2 + github.com/openrelayxyz/plugeth-utils => git.vdb.to/cerc-io/plugeth-utils v1.5.0-cerc-1 ) diff --git a/go.sum b/go.sum index 20d4151..314bbfc 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -git.vdb.to/cerc-io/plugeth v1.13.14-cerc-0 h1:R57mkeo902vrDPzDZsXsjWxXSgtg/XjwNF/UdvvB7Vs= -git.vdb.to/cerc-io/plugeth v1.13.14-cerc-0/go.mod h1:1AMBRYYMPWAWPCK3ui469ymYlEsciWBrRJWjPX5nxy8= -git.vdb.to/cerc-io/plugeth-utils v1.5.0-cerc-0 h1:4GwCBbdLB8mCZINDzoUqpPq7aP4Ha5PPYCyG2h6ee6s= -git.vdb.to/cerc-io/plugeth-utils v1.5.0-cerc-0/go.mod h1:COwKAuTZIsCouCOrIDBhvHZqpbOO1Ojgdy5KTvL8mJg= +git.vdb.to/cerc-io/plugeth v1.13.14-cerc-2 h1:wUnIMCUP+e/F6f/JA1Ui51AagmYkxctEcyg66QJJj0o= +git.vdb.to/cerc-io/plugeth v1.13.14-cerc-2/go.mod h1:sUMNKCsvK1Afdogl+n8QTm9hmCX4fa0X3SqE+xru89k= +git.vdb.to/cerc-io/plugeth-utils v1.5.0-cerc-1 h1:WMdo9Pb5lAn0e2WC1CcD6/mRTWwU0r2KjFoEh0mh2rs= +git.vdb.to/cerc-io/plugeth-utils v1.5.0-cerc-1/go.mod h1:Wf47tlE95PHZto1PMFRlmQAf98MBoNSRbwnQxeq0+Z0= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= @@ -107,8 +107,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= -github.com/ethereum/c-kzg-4844/bindings/go v0.0.0-20230126171313-363c7d7593b4 h1:B2mpK+MNqgPqk2/KNi1LbqwtZDy5F7iy0mynQiBr8VA= -github.com/ethereum/c-kzg-4844/bindings/go v0.0.0-20230126171313-363c7d7593b4/go.mod h1:y4GA2JbAUama1S4QwYjC2hefgGLU8Ul0GMtL/ADMF1c= +github.com/ethereum/c-kzg-4844 v0.4.0 h1:3MS1s4JtA868KpJxroZoepdV0ZKBp3u/O5HcZ7R3nlY= +github.com/ethereum/c-kzg-4844 v0.4.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/ferranbt/fastssz v0.1.2 h1:Dky6dXlngF6Qjc+EfDipAkE83N5I5DE68bY6O0VLNPk= diff --git a/indexer/mocks/test_data.go b/indexer/mocks/test_data.go index d1a684b..b940207 100644 --- a/indexer/mocks/test_data.go +++ b/indexer/mocks/test_data.go @@ -42,7 +42,7 @@ var ( // block data TestChainConfig = params.MainnetChainConfig BlockNumber = TestChainConfig.LondonBlock - BlockTime = *TestChainConfig.CancunTime // TODO: verify this + BlockTime = *TestChainConfig.CancunTime // canonical block at London height // includes 5 transactions: 3 Legacy + 1 EIP-2930 + 1 EIP-1559 diff --git a/scripts/integration-setup.sh b/scripts/integration-setup.sh index 0e7a8a6..97b602d 100755 --- a/scripts/integration-setup.sh +++ b/scripts/integration-setup.sh @@ -12,8 +12,8 @@ CONFIG_DIR=$(readlink -f "${CONFIG_DIR:-$(mktemp -d)}") # Point stack-orchestrator to the multi-project root export CERC_REPO_BASE_DIR="${CERC_REPO_BASE_DIR:-$(git rev-parse --show-toplevel)/..}" -# v5 migrations only go up to version 18 -echo CERC_STATEDIFF_DB_GOOSE_MIN_VER=18 >> $CONFIG_DIR/stack.env +# v5 migrations only go up to version 20 +echo CERC_STATEDIFF_DB_GOOSE_MIN_VER=20 >> $CONFIG_DIR/stack.env # don't run plugeth in the debugger echo CERC_REMOTE_DEBUG=false >> $CONFIG_DIR/stack.env @@ -21,10 +21,10 @@ set -x if [[ -z $SKIP_BUILD ]]; then $laconic_so setup-repositories \ - --exclude git.vdb.to/cerc-io/plugeth,git.vdb.to/cerc-io/plugeth-statediff - + --exclude git.vdb.to/cerc-io/plugeth-statediff + # Assume the tested image has been built separately $laconic_so build-containers \ - --exclude cerc/plugeth,cerc/plugeth-statediff + --exclude cerc/plugeth-statediff fi $laconic_so deploy \ diff --git a/service.go b/service.go index 5c729f2..7d97073 100644 --- a/service.go +++ b/service.go @@ -825,7 +825,7 @@ func (sds *Service) writeStateDiff(block *types.Block, parentRoot common.Hash, p } defer tx.RollbackOnFailure(err) - // TODO: review/remove the need to sync here + // TODO: review necessity of locking here var nodeMtx, ipldMtx sync.Mutex nodeSink := func(node types2.StateLeafNode) error { defer metrics.UpdateDuration(time.Now(), metrics.IndexerMetrics.OutputTimer) diff --git a/test/compose.yml b/test/compose.yml index 4e56536..4fcac0c 100644 --- a/test/compose.yml +++ b/test/compose.yml @@ -3,7 +3,7 @@ services: restart: on-failure depends_on: - ipld-eth-db - image: git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:v5.0.5-alpha + image: git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:v5.2.1-alpha environment: DATABASE_USER: "vdbm" DATABASE_NAME: "cerc_testing" diff --git a/test/stack.yml b/test/stack.yml index 847e84e..3c16a0b 100644 --- a/test/stack.yml +++ b/test/stack.yml @@ -1,8 +1,8 @@ version: "1.2" name: fixturenet-plugeth-tx -description: "plugeth Ethereum Fixturenet" +description: "Plugeth Ethereum Fixturenet for testing plugeth-statediff" repos: - - git.vdb.to/cerc-io/plugeth@statediff + - git.vdb.to/cerc-io/plugeth@v1.5.0-cerc-2 - git.vdb.to/cerc-io/plugeth-statediff - git.vdb.to/cerc-io/lighthouse - git.vdb.to/cerc-io/ipld-eth-db@v5.2.1-alpha diff --git a/test_helpers/builder.go b/test_helpers/builder.go index 44f46bf..18a9363 100644 --- a/test_helpers/builder.go +++ b/test_helpers/builder.go @@ -2,6 +2,7 @@ package test_helpers import ( "bytes" + "context" "fmt" "math/big" "math/rand" @@ -93,6 +94,7 @@ func RunStateSnapshot( tr := tracker.New(recoveryFile, subtries) defer tr.CloseAndSave() return builder.WriteStateSnapshot( + context.Background(), test.StateRoot, params, stateAppender, ipldAppender, tr, ) } diff --git a/test_helpers/chaingen/contract.go b/test_helpers/chaingen/contract.go new file mode 100644 index 0000000..f4759e5 --- /dev/null +++ b/test_helpers/chaingen/contract.go @@ -0,0 +1,30 @@ +package chaingen + +import ( + "strings" + + "github.com/ethereum/go-ethereum/accounts/abi" + "github.com/ethereum/go-ethereum/common" +) + +type ContractSpec struct { + DeploymentCode []byte + ABI abi.ABI +} + +func ParseContract(abiStr, binStr string) (*ContractSpec, error) { + parsedABI, err := abi.JSON(strings.NewReader(abiStr)) + if err != nil { + return nil, err + } + data := common.Hex2Bytes(binStr) + return &ContractSpec{data, parsedABI}, nil +} + +func MustParseContract(abiStr, binStr string) *ContractSpec { + spec, err := ParseContract(abiStr, binStr) + if err != nil { + panic(err) + } + return spec +} diff --git a/test_helpers/chaingen/gen.go b/test_helpers/chaingen/gen.go new file mode 100644 index 0000000..01bc90f --- /dev/null +++ b/test_helpers/chaingen/gen.go @@ -0,0 +1,150 @@ +package chaingen + +import ( + "crypto/ecdsa" + "errors" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/params" +) + +const secondsPerBlock = 12 + +type GenContext struct { + ChainConfig *params.ChainConfig + GenFuncs []func(int, *core.BlockGen) + DB ethdb.Database + + Keys map[common.Address]*ecdsa.PrivateKey + Contracts map[string]*ContractSpec + Genesis *types.Block + + block *core.BlockGen // cache the current block for my methods' use + deployed map[common.Address]string // names of deployed contracts keyed by deployer + time uint64 // time at current block, in seconds +} + +func NewGenContext(chainConfig *params.ChainConfig, db ethdb.Database) *GenContext { + return &GenContext{ + ChainConfig: chainConfig, + DB: db, + Keys: make(map[common.Address]*ecdsa.PrivateKey), + Contracts: make(map[string]*ContractSpec), + + deployed: make(map[common.Address]string), + } +} + +func (gen *GenContext) AddFunction(fn func(int, *core.BlockGen)) { + gen.GenFuncs = append(gen.GenFuncs, fn) +} + +func (gen *GenContext) AddOwnedAccount(key *ecdsa.PrivateKey) common.Address { + addr := crypto.PubkeyToAddress(key.PublicKey) + gen.Keys[addr] = key + return addr +} + +func (gen *GenContext) AddContract(name string, spec *ContractSpec) { + gen.Contracts[name] = spec +} + +func (gen *GenContext) generate(i int, block *core.BlockGen) { + gen.block = block + for _, fn := range gen.GenFuncs { + fn(i, block) + } + gen.time += secondsPerBlock +} + +// MakeChain creates a chain of n blocks starting at and including the genesis block. +// the returned hash chain is ordered head->parent. +func (gen *GenContext) MakeChain(n int) ([]*types.Block, []types.Receipts, *core.BlockChain) { + blocks, receipts := core.GenerateChain( + gen.ChainConfig, gen.Genesis, ethash.NewFaker(), gen.DB, n, gen.generate, + ) + chain, err := core.NewBlockChain(gen.DB, nil, nil, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + if err != nil { + panic(err) + } + return append([]*types.Block{gen.Genesis}, blocks...), receipts, chain +} + +func (gen *GenContext) CreateSendTx(from common.Address, to common.Address, amount *big.Int) (*types.Transaction, error) { + return gen.createTx(from, &to, amount, params.TxGas, nil) +} + +func (gen *GenContext) CreateContractTx(from common.Address, contractName string) (*types.Transaction, error) { + contract := gen.Contracts[contractName] + if contract == nil { + return nil, errors.New("No contract with name " + contractName) + } + return gen.createTx(from, nil, big.NewInt(0), 1000000, contract.DeploymentCode) +} + +func (gen *GenContext) CreateCallTx(from common.Address, to common.Address, methodName string, args ...interface{}) (*types.Transaction, error) { + contractName, ok := gen.deployed[to] + if !ok { + return nil, errors.New("No contract deployed at address " + to.String()) + } + contract := gen.Contracts[contractName] + if contract == nil { + return nil, errors.New("No contract with name " + contractName) + } + + packed, err := contract.ABI.Pack(methodName, args...) + if err != nil { + panic(err) + } + return gen.createTx(from, &to, big.NewInt(0), 100000, packed) +} + +func (gen *GenContext) DeployContract(from common.Address, contractName string) (common.Address, error) { + tx, err := gen.CreateContractTx(from, contractName) + if err != nil { + return common.Address{}, err + } + addr := crypto.CreateAddress(from, gen.block.TxNonce(from)) + gen.deployed[addr] = contractName + gen.block.AddTx(tx) + return addr, nil +} + +func (gen *GenContext) createTx(from common.Address, to *common.Address, amount *big.Int, gasLimit uint64, data []byte) (*types.Transaction, error) { + signer := types.MakeSigner(gen.ChainConfig, gen.block.Number(), gen.time) + nonce := gen.block.TxNonce(from) + priv, ok := gen.Keys[from] + if !ok { + return nil, errors.New("No private key for sender address" + from.String()) + } + + var tx *types.Transaction + if gen.ChainConfig.IsLondon(gen.block.Number()) { + tx = types.NewTx(&types.DynamicFeeTx{ + ChainID: gen.ChainConfig.ChainID, + Nonce: nonce, + To: to, + Gas: gasLimit, + GasTipCap: big.NewInt(50), + GasFeeCap: big.NewInt(1000000000), + Value: amount, + Data: data, + }) + } else { + tx = types.NewTx(&types.LegacyTx{ + Nonce: nonce, + To: to, + Value: amount, + Gas: gasLimit, + Data: data, + }) + } + return types.SignTx(tx, signer, priv) +} diff --git a/test_helpers/db.go b/test_helpers/db.go new file mode 100644 index 0000000..964b9ce --- /dev/null +++ b/test_helpers/db.go @@ -0,0 +1,33 @@ +package test_helpers + +import ( + "fmt" + + "github.com/jmoiron/sqlx" +) + +// ClearDB is used to empty the IPLD-ETH tables after tests +func ClearDB(db *sqlx.DB) error { + tx, err := db.Beginx() + if err != nil { + return err + } + statements := []string{ + `TRUNCATE nodes`, + `TRUNCATE ipld.blocks`, + `TRUNCATE eth.header_cids`, + `TRUNCATE eth.uncle_cids`, + `TRUNCATE eth.transaction_cids`, + `TRUNCATE eth.receipt_cids`, + `TRUNCATE eth.state_cids`, + `TRUNCATE eth.storage_cids`, + `TRUNCATE eth.log_cids`, + `TRUNCATE eth_meta.watched_addresses`, + } + for _, stm := range statements { + if _, err = tx.Exec(stm); err != nil { + return fmt.Errorf("error executing `%s`: %w", stm, err) + } + } + return tx.Commit() +} diff --git a/test_helpers/indexing.go b/test_helpers/indexing.go new file mode 100644 index 0000000..5c34b58 --- /dev/null +++ b/test_helpers/indexing.go @@ -0,0 +1,100 @@ +package test_helpers + +import ( + "context" + "fmt" + "math/big" + + "github.com/cerc-io/plugeth-statediff" + "github.com/cerc-io/plugeth-statediff/adapt" + "github.com/cerc-io/plugeth-statediff/indexer" + "github.com/cerc-io/plugeth-statediff/indexer/interfaces" + "github.com/cerc-io/plugeth-statediff/indexer/node" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/params" +) + +type IndexChainParams struct { + Blocks []*types.Block + Receipts []types.Receipts + StateCache state.Database + + StateDiffParams statediff.Params + TotalDifficulty *big.Int + // Whether to skip indexing state nodes (state_cids, storage_cids) + SkipStateNodes bool + // Whether to skip indexing IPLD blocks + SkipIPLDs bool +} + +func NewIndexer(ctx context.Context, chainConfig *params.ChainConfig, genHash common.Hash, dbconfig interfaces.Config) (interfaces.StateDiffIndexer, error) { + testInfo := node.Info{ + GenesisBlock: genHash.String(), + NetworkID: "1", + ID: "1", + ClientName: "geth", + ChainID: chainConfig.ChainID.Uint64(), + } + _, indexer, err := indexer.NewStateDiffIndexer(ctx, chainConfig, testInfo, dbconfig, true) + return indexer, err +} + +func IndexChain(indexer interfaces.StateDiffIndexer, params IndexChainParams) error { + builder := statediff.NewBuilder(adapt.GethStateView(params.StateCache)) + // iterate over the blocks, generating statediff payloads, and transforming the data into Postgres + for i, block := range params.Blocks { + var args statediff.Args + var rcts types.Receipts + if i == 0 { + args = statediff.Args{ + OldStateRoot: common.Hash{}, + NewStateRoot: block.Root(), + BlockNumber: block.Number(), + BlockHash: block.Hash(), + } + } else { + args = statediff.Args{ + OldStateRoot: params.Blocks[i-1].Root(), + NewStateRoot: block.Root(), + BlockNumber: block.Number(), + BlockHash: block.Hash(), + } + rcts = params.Receipts[i-1] + } + + diff, err := builder.BuildStateDiffObject(args, params.StateDiffParams) + if err != nil { + return fmt.Errorf("failed to build diff (block %d): %w", block.Number(), err) + } + tx, err := indexer.PushBlock(block, rcts, params.TotalDifficulty) + if err != nil { + return fmt.Errorf("failed to index block (block %d): %w", block.Number(), err) + } + defer tx.RollbackOnFailure(err) + + if !params.SkipStateNodes { + for _, node := range diff.Nodes { + if err = indexer.PushStateNode(tx, node, block.Hash().String()); err != nil { + if err != nil { + return fmt.Errorf("failed to index state node: %w", err) + } + } + } + } + if !params.SkipIPLDs { + for _, ipld := range diff.IPLDs { + if err := indexer.PushIPLD(tx, ipld); err != nil { + if err != nil { + return fmt.Errorf("failed to index IPLD: %w", err) + } + } + } + } + if err = tx.Submit(); err != nil { + return fmt.Errorf("failed to commit diff: %w", err) + } + } + return nil +}