cerc-io/plugeth-statediff
16
0
Fork 0
Code Issues 3 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Add test utils and module fixes for dependents #24

Merged
roysc merged 12 commits from update-czkg4844 into main 2024-05-29 09:47:10 +00:00
Conversation 1 Commits 12 Files Changed 14 +336 -26
14 changed files with 336 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.gitea/workflows/test.yml
View File

@ -34,15 +34,9 @@ jobs:
- uses: actions/checkout@v3
with:
path: ./plugeth-statediff
- uses: actions/checkout@v3
with:
repository: cerc-io/plugeth
ref: v1.13.14-cerc-0
path: ./plugeth
- name: Build docker image
run: |
docker build ./plugeth-statediff -t cerc/plugeth-statediff:local
docker build ./plugeth -t cerc/plugeth:local
- name: Install stack-orchestrator
run: |

3
builder.go
View File

@ -168,6 +168,7 @@ func (sdb *builder) WriteStateDiff(
// WriteStateDiff writes a statediff object to output sinks
func (sdb *builder) WriteStateSnapshot(
ctx context.Context,
stateRoot common.Hash, params Params,
nodeSink sdtypes.StateNodeSink,
ipldSink sdtypes.IPLDSink,
@ -200,7 +201,7 @@ func (sdb *builder) WriteStateSnapshot(
}
}
// errgroup will cancel if any group fails
g, ctx := errgroup.WithContext(context.Background())
g, ctx := errgroup.WithContext(ctx)
for i := range subiters {
func(subdiv uint) {
g.Go(func() error {

6
go.mod
View File

@ -48,7 +48,7 @@ require (
github.com/deckarep/golang-set/v2 v2.1.0 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
github.com/deepmap/oapi-codegen v1.8.2 // indirect
github.com/ethereum/c-kzg-4844/bindings/go v0.0.0-20230126171313-363c7d7593b4 // indirect
github.com/ethereum/c-kzg-4844 v0.4.0 // indirect
github.com/ferranbt/fastssz v0.1.2 // indirect
github.com/fjl/memsize v0.0.2 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
@ -140,6 +140,6 @@ require (
)
replace (
github.com/ethereum/go-ethereum => git.vdb.to/cerc-io/plugeth v1.13.14-cerc-0
github.com/openrelayxyz/plugeth-utils => git.vdb.to/cerc-io/plugeth-utils v1.5.0-cerc-0
github.com/ethereum/go-ethereum => git.vdb.to/cerc-io/plugeth v1.13.14-cerc-2
github.com/openrelayxyz/plugeth-utils => git.vdb.to/cerc-io/plugeth-utils v1.5.0-cerc-1
)

12
go.sum
View File

@ -1,8 +1,8 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
git.vdb.to/cerc-io/plugeth v1.13.14-cerc-0 h1:R57mkeo902vrDPzDZsXsjWxXSgtg/XjwNF/UdvvB7Vs=
git.vdb.to/cerc-io/plugeth v1.13.14-cerc-0/go.mod h1:1AMBRYYMPWAWPCK3ui469ymYlEsciWBrRJWjPX5nxy8=
git.vdb.to/cerc-io/plugeth-utils v1.5.0-cerc-0 h1:4GwCBbdLB8mCZINDzoUqpPq7aP4Ha5PPYCyG2h6ee6s=
git.vdb.to/cerc-io/plugeth-utils v1.5.0-cerc-0/go.mod h1:COwKAuTZIsCouCOrIDBhvHZqpbOO1Ojgdy5KTvL8mJg=
git.vdb.to/cerc-io/plugeth v1.13.14-cerc-2 h1:wUnIMCUP+e/F6f/JA1Ui51AagmYkxctEcyg66QJJj0o=
git.vdb.to/cerc-io/plugeth v1.13.14-cerc-2/go.mod h1:sUMNKCsvK1Afdogl+n8QTm9hmCX4fa0X3SqE+xru89k=
git.vdb.to/cerc-io/plugeth-utils v1.5.0-cerc-1 h1:WMdo9Pb5lAn0e2WC1CcD6/mRTWwU0r2KjFoEh0mh2rs=
git.vdb.to/cerc-io/plugeth-utils v1.5.0-cerc-1/go.mod h1:Wf47tlE95PHZto1PMFRlmQAf98MBoNSRbwnQxeq0+Z0=
github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8=
@ -107,8 +107,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.m
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0=
github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw=
github.com/ethereum/c-kzg-4844/bindings/go v0.0.0-20230126171313-363c7d7593b4 h1:B2mpK+MNqgPqk2/KNi1LbqwtZDy5F7iy0mynQiBr8VA=
github.com/ethereum/c-kzg-4844/bindings/go v0.0.0-20230126171313-363c7d7593b4/go.mod h1:y4GA2JbAUama1S4QwYjC2hefgGLU8Ul0GMtL/ADMF1c=
github.com/ethereum/c-kzg-4844 v0.4.0 h1:3MS1s4JtA868KpJxroZoepdV0ZKBp3u/O5HcZ7R3nlY=
github.com/ethereum/c-kzg-4844 v0.4.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0=
github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8=
github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
github.com/ferranbt/fastssz v0.1.2 h1:Dky6dXlngF6Qjc+EfDipAkE83N5I5DE68bY6O0VLNPk=

2
indexer/mocks/test_data.go
View File

@ -42,7 +42,7 @@ var (
// block data
TestChainConfig = params.MainnetChainConfig
BlockNumber = TestChainConfig.LondonBlock
BlockTime = *TestChainConfig.CancunTime // TODO: verify this
BlockTime = *TestChainConfig.CancunTime
// canonical block at London height
// includes 5 transactions: 3 Legacy + 1 EIP-2930 + 1 EIP-1559

10
scripts/integration-setup.sh
View File

@ -12,8 +12,8 @@ CONFIG_DIR=$(readlink -f "${CONFIG_DIR:-$(mktemp -d)}")
# Point stack-orchestrator to the multi-project root
export CERC_REPO_BASE_DIR="${CERC_REPO_BASE_DIR:-$(git rev-parse --show-toplevel)/..}"
# v5 migrations only go up to version 18
echo CERC_STATEDIFF_DB_GOOSE_MIN_VER=18 >> $CONFIG_DIR/stack.env
# v5 migrations only go up to version 20
echo CERC_STATEDIFF_DB_GOOSE_MIN_VER=20 >> $CONFIG_DIR/stack.env
# don't run plugeth in the debugger
echo CERC_REMOTE_DEBUG=false >> $CONFIG_DIR/stack.env
@ -21,10 +21,10 @@ set -x
if [[ -z $SKIP_BUILD ]]; then
$laconic_so setup-repositories \
--exclude git.vdb.to/cerc-io/plugeth,git.vdb.to/cerc-io/plugeth-statediff
--exclude git.vdb.to/cerc-io/plugeth-statediff
# Assume the tested image has been built separately
$laconic_so build-containers \
--exclude cerc/plugeth,cerc/plugeth-statediff
--exclude cerc/plugeth-statediff
fi
$laconic_so deploy \

2
service.go
View File

@ -825,7 +825,7 @@ func (sds *Service) writeStateDiff(block *types.Block, parentRoot common.Hash, p
}
defer tx.RollbackOnFailure(err)
// TODO: review/remove the need to sync here
// TODO: review necessity of locking here
var nodeMtx, ipldMtx sync.Mutex
nodeSink := func(node types2.StateLeafNode) error {
defer metrics.UpdateDuration(time.Now(), metrics.IndexerMetrics.OutputTimer)

2
test/compose.yml
View File

@ -3,7 +3,7 @@ services:
restart: on-failure
depends_on:
- ipld-eth-db
image: git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:v5.0.5-alpha
image: git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:v5.2.1-alpha
environment:
DATABASE_USER: "vdbm"
DATABASE_NAME: "cerc_testing"

4
test/stack.yml
View File

@ -1,8 +1,8 @@
version: "1.2"
name: fixturenet-plugeth-tx
description: "plugeth Ethereum Fixturenet"
description: "Plugeth Ethereum Fixturenet for testing plugeth-statediff"
repos:
- git.vdb.to/cerc-io/plugeth@statediff
- git.vdb.to/cerc-io/plugeth@v1.5.0-cerc-2
- git.vdb.to/cerc-io/plugeth-statediff
- git.vdb.to/cerc-io/lighthouse
- git.vdb.to/cerc-io/ipld-eth-db@v5.2.1-alpha

2
test_helpers/builder.go
View File

@ -2,6 +2,7 @@ package test_helpers
import (
"bytes"
"context"
"fmt"
"math/big"
"math/rand"
@ -93,6 +94,7 @@ func RunStateSnapshot(
tr := tracker.New(recoveryFile, subtries)
defer tr.CloseAndSave()
return builder.WriteStateSnapshot(
context.Background(),
test.StateRoot, params, stateAppender, ipldAppender, tr,
)
}

30
test_helpers/chaingen/contract.go Normal file
View File

@ -0,0 +1,30 @@
package chaingen
import (
"strings"
"github.com/ethereum/go-ethereum/accounts/abi"
"github.com/ethereum/go-ethereum/common"
)
type ContractSpec struct {
DeploymentCode []byte
ABI abi.ABI
}
func ParseContract(abiStr, binStr string) (*ContractSpec, error) {
parsedABI, err := abi.JSON(strings.NewReader(abiStr))
if err != nil {
return nil, err
}
data := common.Hex2Bytes(binStr)
return &ContractSpec{data, parsedABI}, nil
}
func MustParseContract(abiStr, binStr string) *ContractSpec {
spec, err := ParseContract(abiStr, binStr)
if err != nil {
panic(err)
}
return spec
}

150
test_helpers/chaingen/gen.go Normal file
View File

@ -0,0 +1,150 @@
package chaingen
import (
"crypto/ecdsa"
"errors"
"math/big"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/consensus/ethash"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/params"
)
const secondsPerBlock = 12
type GenContext struct {
ChainConfig *params.ChainConfig
GenFuncs []func(int, *core.BlockGen)
DB ethdb.Database
Keys map[common.Address]*ecdsa.PrivateKey
Contracts map[string]*ContractSpec
Genesis *types.Block
block *core.BlockGen // cache the current block for my methods' use
deployed map[common.Address]string // names of deployed contracts keyed by deployer
time uint64 // time at current block, in seconds
}
func NewGenContext(chainConfig *params.ChainConfig, db ethdb.Database) *GenContext {
return &GenContext{
ChainConfig: chainConfig,
DB: db,
Keys: make(map[common.Address]*ecdsa.PrivateKey),
Contracts: make(map[string]*ContractSpec),
deployed: make(map[common.Address]string),
}
}
func (gen *GenContext) AddFunction(fn func(int, *core.BlockGen)) {
gen.GenFuncs = append(gen.GenFuncs, fn)
}
func (gen *GenContext) AddOwnedAccount(key *ecdsa.PrivateKey) common.Address {
addr := crypto.PubkeyToAddress(key.PublicKey)
gen.Keys[addr] = key
return addr
}
func (gen *GenContext) AddContract(name string, spec *ContractSpec) {
gen.Contracts[name] = spec
}
func (gen *GenContext) generate(i int, block *core.BlockGen) {
gen.block = block
for _, fn := range gen.GenFuncs {
fn(i, block)
}
gen.time += secondsPerBlock
}
// MakeChain creates a chain of n blocks starting at and including the genesis block.
// the returned hash chain is ordered head->parent.
func (gen *GenContext) MakeChain(n int) ([]*types.Block, []types.Receipts, *core.BlockChain) {
blocks, receipts := core.GenerateChain(
gen.ChainConfig, gen.Genesis, ethash.NewFaker(), gen.DB, n, gen.generate,
)
chain, err := core.NewBlockChain(gen.DB, nil, nil, nil, ethash.NewFaker(), vm.Config{}, nil, nil)
if err != nil {
panic(err)
}
return append([]*types.Block{gen.Genesis}, blocks...), receipts, chain
}
func (gen *GenContext) CreateSendTx(from common.Address, to common.Address, amount *big.Int) (*types.Transaction, error) {
return gen.createTx(from, &to, amount, params.TxGas, nil)
}
func (gen *GenContext) CreateContractTx(from common.Address, contractName string) (*types.Transaction, error) {
contract := gen.Contracts[contractName]
if contract == nil {
return nil, errors.New("No contract with name " + contractName)
}
return gen.createTx(from, nil, big.NewInt(0), 1000000, contract.DeploymentCode)
}
func (gen *GenContext) CreateCallTx(from common.Address, to common.Address, methodName string, args ...interface{}) (*types.Transaction, error) {
contractName, ok := gen.deployed[to]
if !ok {
return nil, errors.New("No contract deployed at address " + to.String())
}
contract := gen.Contracts[contractName]
if contract == nil {
return nil, errors.New("No contract with name " + contractName)
}
packed, err := contract.ABI.Pack(methodName, args...)
if err != nil {
panic(err)
}
return gen.createTx(from, &to, big.NewInt(0), 100000, packed)
}
func (gen *GenContext) DeployContract(from common.Address, contractName string) (common.Address, error) {
tx, err := gen.CreateContractTx(from, contractName)
if err != nil {
return common.Address{}, err
}
addr := crypto.CreateAddress(from, gen.block.TxNonce(from))
gen.deployed[addr] = contractName
gen.block.AddTx(tx)
return addr, nil
}
func (gen *GenContext) createTx(from common.Address, to *common.Address, amount *big.Int, gasLimit uint64, data []byte) (*types.Transaction, error) {
signer := types.MakeSigner(gen.ChainConfig, gen.block.Number(), gen.time)
nonce := gen.block.TxNonce(from)
priv, ok := gen.Keys[from]
if !ok {
return nil, errors.New("No private key for sender address" + from.String())
}
var tx *types.Transaction
if gen.ChainConfig.IsLondon(gen.block.Number()) {
tx = types.NewTx(&types.DynamicFeeTx{
ChainID: gen.ChainConfig.ChainID,
Nonce: nonce,
To: to,
Gas: gasLimit,
GasTipCap: big.NewInt(50),
GasFeeCap: big.NewInt(1000000000),
Value: amount,
Data: data,
})
} else {
tx = types.NewTx(&types.LegacyTx{
Nonce: nonce,
To: to,
Value: amount,
Gas: gasLimit,
Data: data,
})
}
return types.SignTx(tx, signer, priv)
}

33
test_helpers/db.go Normal file
View File

@ -0,0 +1,33 @@
package test_helpers
import (
"fmt"
"github.com/jmoiron/sqlx"
)
// ClearDB is used to empty the IPLD-ETH tables after tests
func ClearDB(db *sqlx.DB) error {
tx, err := db.Beginx()
if err != nil {
return err
}
statements := []string{
`TRUNCATE nodes`,
`TRUNCATE ipld.blocks`,
`TRUNCATE eth.header_cids`,
`TRUNCATE eth.uncle_cids`,
`TRUNCATE eth.transaction_cids`,
`TRUNCATE eth.receipt_cids`,
`TRUNCATE eth.state_cids`,
`TRUNCATE eth.storage_cids`,
`TRUNCATE eth.log_cids`,
`TRUNCATE eth_meta.watched_addresses`,
}
for _, stm := range statements {
if _, err = tx.Exec(stm); err != nil {
return fmt.Errorf("error executing `%s`: %w", stm, err)
}
}
return tx.Commit()
}

100
test_helpers/indexing.go Normal file
View File

@ -0,0 +1,100 @@
package test_helpers
import (
"context"
"fmt"
"math/big"
"github.com/cerc-io/plugeth-statediff"
"github.com/cerc-io/plugeth-statediff/adapt"
"github.com/cerc-io/plugeth-statediff/indexer"
"github.com/cerc-io/plugeth-statediff/indexer/interfaces"
"github.com/cerc-io/plugeth-statediff/indexer/node"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/params"
)
type IndexChainParams struct {
Blocks []*types.Block
Receipts []types.Receipts
StateCache state.Database
StateDiffParams statediff.Params
TotalDifficulty *big.Int
// Whether to skip indexing state nodes (state_cids, storage_cids)
SkipStateNodes bool
// Whether to skip indexing IPLD blocks
SkipIPLDs bool
}
func NewIndexer(ctx context.Context, chainConfig *params.ChainConfig, genHash common.Hash, dbconfig interfaces.Config) (interfaces.StateDiffIndexer, error) {
testInfo := node.Info{
GenesisBlock: genHash.String(),
NetworkID: "1",
ID: "1",
ClientName: "geth",
ChainID: chainConfig.ChainID.Uint64(),
}
_, indexer, err := indexer.NewStateDiffIndexer(ctx, chainConfig, testInfo, dbconfig, true)
return indexer, err
}
func IndexChain(indexer interfaces.StateDiffIndexer, params IndexChainParams) error {
builder := statediff.NewBuilder(adapt.GethStateView(params.StateCache))
// iterate over the blocks, generating statediff payloads, and transforming the data into Postgres
for i, block := range params.Blocks {
var args statediff.Args
var rcts types.Receipts
if i == 0 {
args = statediff.Args{
OldStateRoot: common.Hash{},
NewStateRoot: block.Root(),
BlockNumber: block.Number(),
BlockHash: block.Hash(),
}
} else {
args = statediff.Args{
OldStateRoot: params.Blocks[i-1].Root(),
NewStateRoot: block.Root(),
BlockNumber: block.Number(),
BlockHash: block.Hash(),
}
rcts = params.Receipts[i-1]
}
diff, err := builder.BuildStateDiffObject(args, params.StateDiffParams)
if err != nil {
return fmt.Errorf("failed to build diff (block %d): %w", block.Number(), err)
}
tx, err := indexer.PushBlock(block, rcts, params.TotalDifficulty)
if err != nil {
return fmt.Errorf("failed to index block (block %d): %w", block.Number(), err)
}
defer tx.RollbackOnFailure(err)
if !params.SkipStateNodes {
for _, node := range diff.Nodes {
if err = indexer.PushStateNode(tx, node, block.Hash().String()); err != nil {
if err != nil {
return fmt.Errorf("failed to index state node: %w", err)
}
}
}
}
if !params.SkipIPLDs {
for _, ipld := range diff.IPLDs {
if err := indexer.PushIPLD(tx, ipld); err != nil {
if err != nil {
return fmt.Errorf("failed to index IPLD: %w", err)
}
}
}
}
if err = tx.Submit(); err != nil {
return fmt.Errorf("failed to commit diff: %w", err)
}
}
return nil
}