Compare commits

..

3 Commits

Author SHA1 Message Date
801979ea2d Don't run ipld-eth-server in stack
All checks were successful
Test / Run compliance tests (pull_request) Successful in 4m31s
Test / Run unit tests (pull_request) Successful in 12m54s
Test / Run integration tests (pull_request) Successful in 27m48s
and remove unused stack.yml
2024-08-06 19:13:27 -05:00
09b066b1a3 Add godoc for Indexer 2024-08-06 19:12:54 -05:00
0ee2149885 Synchronize cache in DelayedTx
Fixes a potential but unseen race condition.
2024-08-06 19:12:44 -05:00
7 changed files with 53 additions and 39 deletions

View File

@ -3,6 +3,7 @@ package sql
import (
"context"
"reflect"
"sync"
"time"
"github.com/cerc-io/plugeth-statediff/indexer/database/metrics"
@ -15,6 +16,7 @@ const copyFromCheckLimit = 100
type DelayedTx struct {
cache []interface{}
db Database
sync.RWMutex
}
type cachedStmt struct {
sql string
@ -27,6 +29,8 @@ type copyFrom struct {
rows [][]interface{}
}
type result int64
func (cf *copyFrom) appendRows(rows [][]interface{}) {
cf.rows = append(cf.rows, rows...)
}
@ -44,6 +48,8 @@ func (tx *DelayedTx) QueryRow(ctx context.Context, sql string, args ...interface
}
func (tx *DelayedTx) findPrevCopyFrom(tableName []string, columnNames []string, limit int) (*copyFrom, int) {
tx.RLock()
defer tx.RUnlock()
for pos, count := len(tx.cache)-1, 0; pos >= 0 && count < limit; pos, count = pos-1, count+1 {
prevCopy, ok := tx.cache[pos].(*copyFrom)
if ok && prevCopy.matches(tableName, columnNames) {
@ -59,6 +65,8 @@ func (tx *DelayedTx) CopyFrom(ctx context.Context, tableName []string, columnNam
"current", len(prevCopy.rows), "new", len(rows), "distance", distance)
prevCopy.appendRows(rows)
} else {
tx.Lock()
defer tx.Unlock()
tx.cache = append(tx.cache, &copyFrom{tableName, columnNames, rows})
}
@ -66,8 +74,10 @@ func (tx *DelayedTx) CopyFrom(ctx context.Context, tableName []string, columnNam
}
func (tx *DelayedTx) Exec(ctx context.Context, sql string, args ...interface{}) (Result, error) {
tx.Lock()
defer tx.Unlock()
tx.cache = append(tx.cache, cachedStmt{sql, args})
return nil, nil
return result(0), nil
}
func (tx *DelayedTx) Commit(ctx context.Context) error {
@ -85,6 +95,8 @@ func (tx *DelayedTx) Commit(ctx context.Context) error {
rollback(ctx, base)
}
}()
tx.Lock()
defer tx.Unlock()
for _, item := range tx.cache {
switch item := item.(type) {
case *copyFrom:
@ -105,6 +117,13 @@ func (tx *DelayedTx) Commit(ctx context.Context) error {
}
func (tx *DelayedTx) Rollback(ctx context.Context) error {
tx.Lock()
defer tx.Unlock()
tx.cache = nil
return nil
}
// RowsAffected satisfies sql.Result
func (r result) RowsAffected() (int64, error) {
return int64(r), nil
}

View File

@ -29,27 +29,39 @@ import (
"github.com/ethereum/go-ethereum/core/types"
)
// StateDiffIndexer interface required to index statediff data
// StateDiffIndexer describes the interface for indexing state data.
type StateDiffIndexer interface {
DetectGaps(beginBlock uint64, endBlock uint64) ([]*BlockGap, error)
CurrentBlock() (*models.HeaderModel, error)
HasBlock(hash common.Hash, number uint64) (bool, error)
// PushBlock indexes block data except for state & storage nodes: header, uncles, transactions &
// receipts. Returns an initiated DB transaction which must be committed or rolled back.
PushBlock(block *types.Block, receipts types.Receipts, totalDifficulty *big.Int) (Batch, error)
// PushHeader indexes a block header.
PushHeader(batch Batch, header *types.Header, reward, td *big.Int) (string, error)
// PushStateNode indexes a state node and its storage trie.
PushStateNode(tx Batch, stateNode sdtypes.StateLeafNode, headerID string) error
// PushIPLD indexes an IPLD node.
PushIPLD(tx Batch, ipld sdtypes.IPLD) error
ReportDBMetrics(delay time.Duration, quit <-chan bool)
// BeginTx starts a new DB transaction.
BeginTx(number *big.Int, ctx context.Context) Batch
// DetectGaps returns a list of gaps in the block range, if any.
DetectGaps(beginBlock uint64, endBlock uint64) ([]*BlockGap, error)
// CurrentBlock returns the latest indexed block.
CurrentBlock() (*models.HeaderModel, error)
// HasBlock returns true if the block is indexed.
HasBlock(hash common.Hash, number uint64) (bool, error)
// Close closes the associated output DB connection or files.
Close() error
// Methods used by WatchAddress API/functionality
LoadWatchedAddresses() ([]common.Address, error)
InsertWatchedAddresses(addresses []sdtypes.WatchAddressArg, currentBlock *big.Int) error
RemoveWatchedAddresses(addresses []sdtypes.WatchAddressArg) error
SetWatchedAddresses(args []sdtypes.WatchAddressArg, currentBlockNumber *big.Int) error
ClearWatchedAddresses() error
Close() error
ReportDBMetrics(delay time.Duration, quit <-chan bool)
}
// Batch required for indexing data atomically

View File

@ -26,7 +26,7 @@ func Initialize(ctx core.Context, pl core.PluginLoader, logger core.Logger) {
gethContext = ctx
}
func InitializeNode(stack core.Node, b core.Backend) {
func InitializeNode(_ core.Node, b core.Backend) {
backend := b.(restricted.Backend)
networkid, err := strconv.ParseUint(gethContext.String(geth_flags.NetworkIdFlag.Name), 10, 64)

View File

@ -18,14 +18,15 @@ echo CERC_REMOTE_DEBUG=false >> $CONFIG_DIR/stack.env
if [[ -z $SKIP_BUILD ]]; then
# Assume the tested image has been built separately.
$laconic_so setup-repositories \
--exclude git.vdb.to/cerc-io/plugeth-statediff
# Assume the tested image has been built separately
--exclude git.vdb.to/cerc-io/plugeth-statediff,git.vdb.to/cerc-io/ipld-eth-server
$laconic_so build-containers \
--exclude cerc/plugeth-statediff
--exclude cerc/plugeth-statediff,cerc/ipld-eth-server
fi
if ! $laconic_so deploy \
--exclude ipld-eth-server \
--env-file $CONFIG_DIR/stack.env \
--cluster test up
then

View File

@ -1,20 +0,0 @@
version: "1.2"
name: fixturenet-plugeth-tx
description: "Plugeth Ethereum Fixturenet for testing plugeth-statediff"
repos:
- git.vdb.to/cerc-io/plugeth@v1.13.14-cerc-2
- git.vdb.to/cerc-io/plugeth-statediff
- git.vdb.to/cerc-io/lighthouse
- git.vdb.to/cerc-io/ipld-eth-db@v5.4.0-alpha
containers:
- cerc/plugeth-statediff
- cerc/plugeth
- cerc/fixturenet-eth-genesis
- cerc/fixturenet-plugeth-plugeth
- cerc/lighthouse
- cerc/lighthouse-cli
- cerc/fixturenet-eth-lighthouse
- cerc/ipld-eth-db
pods:
- fixturenet-plugeth
- ipld-eth-db

View File

@ -12,14 +12,15 @@ import (
"testing"
"github.com/cerc-io/eth-iterator-utils/tracker"
statediff "github.com/cerc-io/plugeth-statediff"
"github.com/cerc-io/plugeth-statediff/adapt"
sdtypes "github.com/cerc-io/plugeth-statediff/types"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/stretchr/testify/require"
"github.com/cerc-io/plugeth-statediff"
"github.com/cerc-io/plugeth-statediff/adapt"
sdtypes "github.com/cerc-io/plugeth-statediff/types"
)
var subtrieCounts = []uint{1, 8, 32}

View File

@ -5,15 +5,16 @@ import (
"fmt"
"math/big"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/params"
"github.com/cerc-io/plugeth-statediff"
"github.com/cerc-io/plugeth-statediff/adapt"
"github.com/cerc-io/plugeth-statediff/indexer"
"github.com/cerc-io/plugeth-statediff/indexer/interfaces"
"github.com/cerc-io/plugeth-statediff/indexer/node"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/params"
)
type IndexChainParams struct {