cerc-io/plugeth-statediff
11
0
Fork 0
Code Issues 3 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Initial plugin implementation

Browse Source 
* refactor packages, flags, subscriptions
* DRY refactor builder tests
* use mockgen to generate mocks
* update README
* MODE=statediff no longer needed for unit tests
* simplify func names, clean up metrics
* move write params to service field
* sql indexer: confirm quit after ipld cache reset
  prevents negative waitgroup panic
* don't let TotalDifficulty become nil
* use forked plugeth, plugeth-utils for now
This commit is contained in:
roysc
2023-07-14 12:56:36 +08:00
parent bb00c46735
commit ebc2eb37e7
89 changed files with 3742 additions and 3160 deletions
Download Patch File Download Diff File Expand all files Collapse all files
indexer/database/dump/batch_tx.go
+2 -2
View File
@@ -20,9 +20,9 @@ import (
"fmt"
"io"
"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
"github.com/cerc-io/plugeth-statediff/indexer/ipld"
"github.com/ethereum/go-ethereum/statediff/indexer/models"
"github.com/cerc-io/plugeth-statediff/indexer/models"
)
// BatchTx wraps a void with the state necessary for building the tx concurrently during trie difference iteration
indexer/database/dump/config.go
+23 -22
View File
@@ -21,9 +21,14 @@ import (
"io"
"strings"
"github.com/ethereum/go-ethereum/statediff/indexer/shared"
"github.com/cerc-io/plugeth-statediff/indexer/shared"
)
// Config for data dump
type Config struct {
Dump io.WriteCloser
}
// DumpType to explicitly type the dump destination
type DumpType string
@@ -31,9 +36,14 @@ const (
STDOUT = "Stdout"
STDERR = "Stderr"
DISCARD = "Discard"
UNKNOWN = "Unknown"
INVALID = "Invalid"
)
// Type satisfies interfaces.Config
func (c Config) Type() shared.DBType {
return shared.DUMP
}
// ResolveDumpType resolves the dump type for the provided string
func ResolveDumpType(str string) (DumpType, error) {
switch strings.ToLower(str) {
@@ -44,36 +54,27 @@ func ResolveDumpType(str string) (DumpType, error) {
case "discard", "void", "devnull", "dev null":
return DISCARD, nil
default:
return UNKNOWN, fmt.Errorf("unrecognized dump type: %s", str)
return INVALID, fmt.Errorf("unrecognized dump type: %s", str)
}
}
// Config for data dump
type Config struct {
Dump io.WriteCloser
// Set satisfies flag.Value
func (d *DumpType) Set(v string) (err error) {
*d, err = ResolveDumpType(v)
return
}
// Type satisfies interfaces.Config
func (c Config) Type() shared.DBType {
return shared.DUMP
}
// NewDiscardWriterCloser returns a discardWrapper wrapping io.Discard
func NewDiscardWriterCloser() io.WriteCloser {
return discardWrapper{blackhole: io.Discard}
// String satisfies flag.Value
func (d *DumpType) String() string {
return strings.ToLower(string(*d))
}
// discardWrapper wraps io.Discard with io.Closer
type discardWrapper struct {
blackhole io.Writer
}
type discardWrapper struct{ io.Writer }
// Write satisfies io.Writer
func (dw discardWrapper) Write(b []byte) (int, error) {
return dw.blackhole.Write(b)
}
var Discard = discardWrapper{io.Discard}
// Close satisfies io.Closer
func (dw discardWrapper) Close() error {
func (discardWrapper) Close() error {
return nil
}
indexer/database/dump/indexer.go
+13 -12
View File
@@ -17,7 +17,7 @@
package dump
import (
"bytes"
"encoding/hex"
"fmt"
"io"
"math/big"
@@ -28,15 +28,16 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/statediff/indexer/database/metrics"
"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
"github.com/ethereum/go-ethereum/statediff/indexer/models"
"github.com/ethereum/go-ethereum/statediff/indexer/shared"
sdtypes "github.com/ethereum/go-ethereum/statediff/types"
"github.com/cerc-io/plugeth-statediff/indexer/database/metrics"
"github.com/cerc-io/plugeth-statediff/indexer/interfaces"
"github.com/cerc-io/plugeth-statediff/indexer/ipld"
"github.com/cerc-io/plugeth-statediff/indexer/models"
"github.com/cerc-io/plugeth-statediff/indexer/shared"
sdtypes "github.com/cerc-io/plugeth-statediff/types"
"github.com/cerc-io/plugeth-statediff/utils/log"
)
var _ interfaces.StateDiffIndexer = &StateDiffIndexer{}
@@ -199,8 +200,8 @@ func (sdi *StateDiffIndexer) processUncles(tx *BatchTx, headerID string, blockNu
return err
}
preparedHash := crypto.Keccak256Hash(uncleEncoding)
if !bytes.Equal(preparedHash.Bytes(), unclesHash.Bytes()) {
return fmt.Errorf("derived uncles hash (%s) does not match the hash in the header (%s)", preparedHash.Hex(), unclesHash.Hex())
if preparedHash != unclesHash {
return fmt.Errorf("derived uncles hash (%s) does not match the hash in the header (%s)", preparedHash.String(), unclesHash.String())
}
unclesCID, err := ipld.RawdataToCid(ipld.MEthHeaderList, uncleEncoding, multihash.KECCAK_256)
if err != nil {
@@ -294,7 +295,7 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *BatchTx, args processArgs
if len(receipt.PostState) == 0 {
rctModel.PostStatus = receipt.Status
} else {
rctModel.PostState = common.Bytes2Hex(receipt.PostState)
rctModel.PostState = hex.EncodeToString(receipt.PostState)
}
if _, err := fmt.Fprintf(sdi.dump, "%+v\r\n", rctModel); err != nil {
@@ -305,7 +306,7 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *BatchTx, args processArgs
for idx, l := range receipt.Logs {
topicSet := make([]string, 4)
for ti, topic := range l.Topics {
topicSet[ti] = topic.Hex()
topicSet[ti] = topic.String()
}
logDataSet[idx] = &models.LogsModel{
indexer/database/file/config.go
+22 -11
View File
@@ -20,17 +20,26 @@ import (
"fmt"
"strings"
"github.com/ethereum/go-ethereum/statediff/indexer/node"
"github.com/ethereum/go-ethereum/statediff/indexer/shared"
"github.com/cerc-io/plugeth-statediff/indexer/node"
"github.com/cerc-io/plugeth-statediff/indexer/shared"
)
// Config holds params for writing out CSV or SQL files
type Config struct {
Mode FileMode
OutputDir string
FilePath string
WatchedAddressesFilePath string
NodeInfo node.Info
}
// FileMode to explicitly type the mode of file writer we are using
type FileMode string
const (
CSV FileMode = "CSV"
SQL FileMode = "SQL"
Unknown FileMode = "Unknown"
Invalid FileMode = "Invalid"
)
// ResolveFileMode resolves a FileMode from a provided string
@@ -41,17 +50,19 @@ func ResolveFileMode(str string) (FileMode, error) {
case "sql":
return SQL, nil
default:
return Unknown, fmt.Errorf("unrecognized file type string: %s", str)
return Invalid, fmt.Errorf("unrecognized file type string: %s", str)
}
}
// Config holds params for writing out CSV or SQL files
type Config struct {
Mode FileMode
OutputDir string
FilePath string
WatchedAddressesFilePath string
NodeInfo node.Info
// Set satisfies flag.Value
func (f *FileMode) Set(v string) (err error) {
*f, err = ResolveFileMode(v)
return
}
// Set satisfies flag.Value
func (f *FileMode) String() string {
return strings.ToLower(string(*f))
}
// Type satisfies interfaces.Config
indexer/database/file/csv_indexer_legacy_test.go
+6 -6
View File
@@ -27,11 +27,11 @@ import (
"github.com/stretchr/testify/require"
"github.com/ethereum/go-ethereum/statediff/indexer/database/file"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
"github.com/ethereum/go-ethereum/statediff/indexer/shared/schema"
"github.com/ethereum/go-ethereum/statediff/indexer/test"
"github.com/ethereum/go-ethereum/statediff/indexer/test_helpers"
"github.com/cerc-io/plugeth-statediff/indexer/database/file"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql/postgres"
"github.com/cerc-io/plugeth-statediff/indexer/shared/schema"
"github.com/cerc-io/plugeth-statediff/indexer/test"
"github.com/cerc-io/plugeth-statediff/indexer/test_helpers"
)
const dbDirectory = "/file_indexer"
@@ -43,7 +43,7 @@ func setupLegacyCSVIndexer(t *testing.T) {
require.NoError(t, err)
}
ind, err = file.NewStateDiffIndexer(context.Background(), test.LegacyConfig, file.CSVTestConfig)
ind, err = file.NewStateDiffIndexer(test.LegacyConfig, file.CSVTestConfig)
require.NoError(t, err)
db, err = postgres.SetupSQLXDB()
indexer/database/file/csv_indexer_test.go
+16 -19
View File
@@ -17,7 +17,6 @@
package file_test
import (
"context"
"errors"
"math/big"
"os"
@@ -25,15 +24,13 @@ import (
"github.com/stretchr/testify/require"
"github.com/ethereum/go-ethereum/statediff/indexer/database/file"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
"github.com/ethereum/go-ethereum/statediff/indexer/mocks"
"github.com/ethereum/go-ethereum/statediff/indexer/test"
"github.com/cerc-io/plugeth-statediff/indexer/database/file"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql/postgres"
"github.com/cerc-io/plugeth-statediff/indexer/mocks"
"github.com/cerc-io/plugeth-statediff/indexer/test"
)
func setupCSVIndexer(t *testing.T) {
file.CSVTestConfig.OutputDir = "./statediffing_test"
if _, err := os.Stat(file.CSVTestConfig.OutputDir); !errors.Is(err, os.ErrNotExist) {
err := os.RemoveAll(file.CSVTestConfig.OutputDir)
require.NoError(t, err)
@@ -44,7 +41,7 @@ func setupCSVIndexer(t *testing.T) {
require.NoError(t, err)
}
ind, err = file.NewStateDiffIndexer(context.Background(), mocks.TestConfig, file.CSVTestConfig)
ind, err = file.NewStateDiffIndexer(mocks.TestConfig, file.CSVTestConfig)
require.NoError(t, err)
db, err = postgres.SetupSQLXDB()
@@ -69,7 +66,7 @@ func TestCSVFileIndexer(t *testing.T) {
dumpCSVFileData(t)
defer tearDownCSV(t)
test.TestPublishAndIndexHeaderIPLDs(t, db)
test.DoTestPublishAndIndexHeaderIPLDs(t, db)
})
t.Run("Publish and index transaction IPLDs in a single tx", func(t *testing.T) {
@@ -77,7 +74,7 @@ func TestCSVFileIndexer(t *testing.T) {
dumpCSVFileData(t)
defer tearDownCSV(t)
test.TestPublishAndIndexTransactionIPLDs(t, db)
test.DoTestPublishAndIndexTransactionIPLDs(t, db)
})
t.Run("Publish and index log IPLDs for multiple receipt of a specific block", func(t *testing.T) {
@@ -85,7 +82,7 @@ func TestCSVFileIndexer(t *testing.T) {
dumpCSVFileData(t)
defer tearDownCSV(t)
test.TestPublishAndIndexLogIPLDs(t, db)
test.DoTestPublishAndIndexLogIPLDs(t, db)
})
t.Run("Publish and index receipt IPLDs in a single tx", func(t *testing.T) {
@@ -93,7 +90,7 @@ func TestCSVFileIndexer(t *testing.T) {
dumpCSVFileData(t)
defer tearDownCSV(t)
test.TestPublishAndIndexReceiptIPLDs(t, db)
test.DoTestPublishAndIndexReceiptIPLDs(t, db)
})
t.Run("Publish and index state IPLDs in a single tx", func(t *testing.T) {
@@ -101,7 +98,7 @@ func TestCSVFileIndexer(t *testing.T) {
dumpCSVFileData(t)
defer tearDownCSV(t)
test.TestPublishAndIndexStateIPLDs(t, db)
test.DoTestPublishAndIndexStateIPLDs(t, db)
})
t.Run("Publish and index storage IPLDs in a single tx", func(t *testing.T) {
@@ -109,7 +106,7 @@ func TestCSVFileIndexer(t *testing.T) {
dumpCSVFileData(t)
defer tearDownCSV(t)
test.TestPublishAndIndexStorageIPLDs(t, db)
test.DoTestPublishAndIndexStorageIPLDs(t, db)
})
}
@@ -127,7 +124,7 @@ func TestCSVFileIndexerNonCanonical(t *testing.T) {
dumpCSVFileData(t)
defer tearDownCSV(t)
test.TestPublishAndIndexTransactionsNonCanonical(t, db)
test.DoTestPublishAndIndexTransactionsNonCanonical(t, db)
})
t.Run("Publish and index receipts", func(t *testing.T) {
@@ -135,7 +132,7 @@ func TestCSVFileIndexerNonCanonical(t *testing.T) {
dumpCSVFileData(t)
defer tearDownCSV(t)
test.TestPublishAndIndexReceiptsNonCanonical(t, db)
test.DoTestPublishAndIndexReceiptsNonCanonical(t, db)
})
t.Run("Publish and index logs", func(t *testing.T) {
@@ -143,7 +140,7 @@ func TestCSVFileIndexerNonCanonical(t *testing.T) {
dumpCSVFileData(t)
defer tearDownCSV(t)
test.TestPublishAndIndexLogsNonCanonical(t, db)
test.DoTestPublishAndIndexLogsNonCanonical(t, db)
})
t.Run("Publish and index state nodes", func(t *testing.T) {
@@ -151,7 +148,7 @@ func TestCSVFileIndexerNonCanonical(t *testing.T) {
dumpCSVFileData(t)
defer tearDownCSV(t)
test.TestPublishAndIndexStateNonCanonical(t, db)
test.DoTestPublishAndIndexStateNonCanonical(t, db)
})
t.Run("Publish and index storage nodes", func(t *testing.T) {
@@ -159,7 +156,7 @@ func TestCSVFileIndexerNonCanonical(t *testing.T) {
dumpCSVFileData(t)
defer tearDownCSV(t)
test.TestPublishAndIndexStorageNonCanonical(t, db)
test.DoTestPublishAndIndexStorageNonCanonical(t, db)
})
}
indexer/database/file/csv_writer.go
+7 -10
View File
@@ -25,15 +25,15 @@ import (
"path/filepath"
"strconv"
"github.com/ethereum/go-ethereum/common"
"github.com/thoas/go-funk"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/statediff/indexer/database/metrics"
"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
"github.com/ethereum/go-ethereum/statediff/indexer/models"
nodeinfo "github.com/ethereum/go-ethereum/statediff/indexer/node"
"github.com/ethereum/go-ethereum/statediff/indexer/shared/schema"
sdtypes "github.com/ethereum/go-ethereum/statediff/types"
"github.com/cerc-io/plugeth-statediff/indexer/database/metrics"
"github.com/cerc-io/plugeth-statediff/indexer/ipld"
"github.com/cerc-io/plugeth-statediff/indexer/models"
nodeinfo "github.com/cerc-io/plugeth-statediff/indexer/node"
"github.com/cerc-io/plugeth-statediff/indexer/shared/schema"
sdtypes "github.com/cerc-io/plugeth-statediff/types"
)
var (
@@ -92,9 +92,6 @@ func newFileWriter(path string) (ret fileWriter, err error) {
}
func makeFileWriters(dir string, tables []*schema.Table) (fileWriters, error) {
if err := os.MkdirAll(dir, 0755); err != nil {
return nil, err
}
writers := fileWriters{}
for _, tbl := range tables {
w, err := newFileWriter(TableFilePath(dir, tbl.Name))
indexer/database/file/indexer.go
+15 -17
View File
@@ -17,8 +17,6 @@
package file
import (
"bytes"
"context"
"errors"
"fmt"
"math/big"
@@ -27,21 +25,21 @@ import (
"sync/atomic"
"time"
"github.com/lib/pq"
"github.com/multiformats/go-multihash"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/statediff/indexer/database/metrics"
"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
"github.com/ethereum/go-ethereum/statediff/indexer/models"
"github.com/ethereum/go-ethereum/statediff/indexer/shared"
sdtypes "github.com/ethereum/go-ethereum/statediff/types"
"github.com/lib/pq"
"github.com/multiformats/go-multihash"
"github.com/cerc-io/plugeth-statediff/indexer/database/metrics"
"github.com/cerc-io/plugeth-statediff/indexer/interfaces"
"github.com/cerc-io/plugeth-statediff/indexer/ipld"
"github.com/cerc-io/plugeth-statediff/indexer/models"
"github.com/cerc-io/plugeth-statediff/indexer/shared"
sdtypes "github.com/cerc-io/plugeth-statediff/types"
"github.com/cerc-io/plugeth-statediff/utils/log"
)
const defaultCSVOutputDir = "./statediff_output"
@@ -63,7 +61,7 @@ type StateDiffIndexer struct {
}
// NewStateDiffIndexer creates a void implementation of interfaces.StateDiffIndexer
func NewStateDiffIndexer(ctx context.Context, chainConfig *params.ChainConfig, config Config) (*StateDiffIndexer, error) {
func NewStateDiffIndexer(chainConfig *params.ChainConfig, config Config) (*StateDiffIndexer, error) {
var err error
var writer FileWriter
@@ -176,7 +174,7 @@ func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receip
metrics.IndexerMetrics.PostgresCommitTimer.Update(tDiff)
traceMsg += fmt.Sprintf("postgres transaction commit duration: %s\r\n", tDiff.String())
traceMsg += fmt.Sprintf(" TOTAL PROCESSING DURATION: %s\r\n", time.Since(start).String())
log.Debug(traceMsg)
log.Trace(traceMsg)
return err
},
}
@@ -258,8 +256,8 @@ func (sdi *StateDiffIndexer) processUncles(headerID string, blockNumber *big.Int
return err
}
preparedHash := crypto.Keccak256Hash(uncleEncoding)
if !bytes.Equal(preparedHash.Bytes(), unclesHash.Bytes()) {
return fmt.Errorf("derived uncles hash (%s) does not match the hash in the header (%s)", preparedHash.Hex(), unclesHash.Hex())
if preparedHash != unclesHash {
return fmt.Errorf("derived uncles hash (%s) does not match the hash in the header (%s)", preparedHash.String(), unclesHash.String())
}
unclesCID, err := ipld.RawdataToCid(ipld.MEthHeaderList, uncleEncoding, multihash.KECCAK_256)
if err != nil {
@@ -358,7 +356,7 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(args processArgs) error {
sdi.fileWriter.upsertIPLDNode(args.blockNumber.String(), args.logNodes[i][idx])
topicSet := make([]string, 4)
for ti, topic := range l.Topics {
topicSet[ti] = topic.Hex()
topicSet[ti] = topic.String()
}
logDataSet[idx] = &models.LogsModel{
indexer/database/file/interfaces.go
+5 -5
View File
@@ -19,12 +19,12 @@ package file
import (
"math/big"
"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/statediff/indexer/models"
nodeinfo "github.com/ethereum/go-ethereum/statediff/indexer/node"
"github.com/ethereum/go-ethereum/statediff/types"
"github.com/cerc-io/plugeth-statediff/indexer/ipld"
"github.com/cerc-io/plugeth-statediff/indexer/models"
nodeinfo "github.com/cerc-io/plugeth-statediff/indexer/node"
"github.com/cerc-io/plugeth-statediff/types"
)
// Writer interface required by the file indexer
indexer/database/file/mainnet_tests/indexer_test.go
+9 -13
View File
@@ -24,16 +24,16 @@ import (
"os"
"testing"
"github.com/stretchr/testify/require"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/statediff/indexer/database/file"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
"github.com/ethereum/go-ethereum/statediff/indexer/test"
"github.com/ethereum/go-ethereum/statediff/indexer/test_helpers"
"github.com/stretchr/testify/require"
"github.com/cerc-io/plugeth-statediff/indexer/database/file"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql/postgres"
"github.com/cerc-io/plugeth-statediff/indexer/interfaces"
"github.com/cerc-io/plugeth-statediff/indexer/test"
"github.com/cerc-io/plugeth-statediff/indexer/test_helpers"
)
var (
@@ -44,10 +44,6 @@ var (
)
func init() {
if os.Getenv("MODE") != "statediff" {
fmt.Println("Skipping statediff test")
os.Exit(0)
}
if os.Getenv("STATEDIFF_DB") != "file" {
fmt.Println("Skipping statediff .sql file writing mode test")
os.Exit(0)
@@ -87,7 +83,7 @@ func setupMainnetIndexer(t *testing.T) {
require.NoError(t, err)
}
ind, err = file.NewStateDiffIndexer(context.Background(), chainConf, file.CSVTestConfig)
ind, err = file.NewStateDiffIndexer(chainConf, file.CSVTestConfig)
require.NoError(t, err)
db, err = postgres.SetupSQLXDB()
indexer/database/file/sql_indexer_legacy_test.go
+7 -7
View File
@@ -24,12 +24,12 @@ import (
"github.com/stretchr/testify/require"
"github.com/ethereum/go-ethereum/statediff/indexer/database/file"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
"github.com/ethereum/go-ethereum/statediff/indexer/test"
"github.com/ethereum/go-ethereum/statediff/indexer/test_helpers"
"github.com/cerc-io/plugeth-statediff/indexer/database/file"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql/postgres"
"github.com/cerc-io/plugeth-statediff/indexer/interfaces"
"github.com/cerc-io/plugeth-statediff/indexer/test"
"github.com/cerc-io/plugeth-statediff/indexer/test_helpers"
)
var (
@@ -44,7 +44,7 @@ func setupLegacySQLIndexer(t *testing.T) {
require.NoError(t, err)
}
ind, err = file.NewStateDiffIndexer(context.Background(), test.LegacyConfig, file.SQLTestConfig)
ind, err = file.NewStateDiffIndexer(test.LegacyConfig, file.SQLTestConfig)
require.NoError(t, err)
db, err = postgres.SetupSQLXDB()
indexer/database/file/sql_indexer_test.go
+16 -17
View File
@@ -17,7 +17,6 @@
package file_test
import (
"context"
"errors"
"math/big"
"os"
@@ -25,10 +24,10 @@ import (
"github.com/stretchr/testify/require"
"github.com/ethereum/go-ethereum/statediff/indexer/database/file"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
"github.com/ethereum/go-ethereum/statediff/indexer/mocks"
"github.com/ethereum/go-ethereum/statediff/indexer/test"
"github.com/cerc-io/plugeth-statediff/indexer/database/file"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql/postgres"
"github.com/cerc-io/plugeth-statediff/indexer/mocks"
"github.com/cerc-io/plugeth-statediff/indexer/test"
)
func setupIndexer(t *testing.T) {
@@ -42,7 +41,7 @@ func setupIndexer(t *testing.T) {
require.NoError(t, err)
}
ind, err = file.NewStateDiffIndexer(context.Background(), mocks.TestConfig, file.SQLTestConfig)
ind, err = file.NewStateDiffIndexer(mocks.TestConfig, file.SQLTestConfig)
require.NoError(t, err)
db, err = postgres.SetupSQLXDB()
@@ -67,7 +66,7 @@ func TestSQLFileIndexer(t *testing.T) {
dumpFileData(t)
defer tearDown(t)
test.TestPublishAndIndexHeaderIPLDs(t, db)
test.DoTestPublishAndIndexHeaderIPLDs(t, db)
})
t.Run("Publish and index transaction IPLDs in a single tx", func(t *testing.T) {
@@ -75,7 +74,7 @@ func TestSQLFileIndexer(t *testing.T) {
dumpFileData(t)
defer tearDown(t)
test.TestPublishAndIndexTransactionIPLDs(t, db)
test.DoTestPublishAndIndexTransactionIPLDs(t, db)
})
t.Run("Publish and index log IPLDs for multiple receipt of a specific block", func(t *testing.T) {
@@ -83,7 +82,7 @@ func TestSQLFileIndexer(t *testing.T) {
dumpFileData(t)
defer tearDown(t)
test.TestPublishAndIndexLogIPLDs(t, db)
test.DoTestPublishAndIndexLogIPLDs(t, db)
})
t.Run("Publish and index receipt IPLDs in a single tx", func(t *testing.T) {
@@ -91,7 +90,7 @@ func TestSQLFileIndexer(t *testing.T) {
dumpFileData(t)
defer tearDown(t)
test.TestPublishAndIndexReceiptIPLDs(t, db)
test.DoTestPublishAndIndexReceiptIPLDs(t, db)
})
t.Run("Publish and index state IPLDs in a single tx", func(t *testing.T) {
@@ -99,7 +98,7 @@ func TestSQLFileIndexer(t *testing.T) {
dumpFileData(t)
defer tearDown(t)
test.TestPublishAndIndexStateIPLDs(t, db)
test.DoTestPublishAndIndexStateIPLDs(t, db)
})
t.Run("Publish and index storage IPLDs in a single tx", func(t *testing.T) {
@@ -107,7 +106,7 @@ func TestSQLFileIndexer(t *testing.T) {
dumpFileData(t)
defer tearDown(t)
test.TestPublishAndIndexStorageIPLDs(t, db)
test.DoTestPublishAndIndexStorageIPLDs(t, db)
})
}
@@ -125,7 +124,7 @@ func TestSQLFileIndexerNonCanonical(t *testing.T) {
dumpFileData(t)
defer tearDown(t)
test.TestPublishAndIndexTransactionsNonCanonical(t, db)
test.DoTestPublishAndIndexTransactionsNonCanonical(t, db)
})
t.Run("Publish and index receipts", func(t *testing.T) {
@@ -133,7 +132,7 @@ func TestSQLFileIndexerNonCanonical(t *testing.T) {
dumpFileData(t)
defer tearDown(t)
test.TestPublishAndIndexReceiptsNonCanonical(t, db)
test.DoTestPublishAndIndexReceiptsNonCanonical(t, db)
})
t.Run("Publish and index logs", func(t *testing.T) {
@@ -141,7 +140,7 @@ func TestSQLFileIndexerNonCanonical(t *testing.T) {
dumpFileData(t)
defer tearDown(t)
test.TestPublishAndIndexLogsNonCanonical(t, db)
test.DoTestPublishAndIndexLogsNonCanonical(t, db)
})
t.Run("Publish and index state nodes", func(t *testing.T) {
@@ -149,7 +148,7 @@ func TestSQLFileIndexerNonCanonical(t *testing.T) {
dumpFileData(t)
defer tearDown(t)
test.TestPublishAndIndexStateNonCanonical(t, db)
test.DoTestPublishAndIndexStateNonCanonical(t, db)
})
t.Run("Publish and index storage nodes", func(t *testing.T) {
@@ -157,7 +156,7 @@ func TestSQLFileIndexerNonCanonical(t *testing.T) {
dumpFileData(t)
defer tearDown(t)
test.TestPublishAndIndexStorageNonCanonical(t, db)
test.DoTestPublishAndIndexStorageNonCanonical(t, db)
})
}
indexer/database/file/sql_writer.go
+8 -11
View File
@@ -24,15 +24,15 @@ import (
"math/big"
"os"
pg_query "github.com/pganalyze/pg_query_go/v2"
"github.com/ethereum/go-ethereum/common"
pg_query "github.com/pganalyze/pg_query_go/v4"
"github.com/thoas/go-funk"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/statediff/indexer/database/metrics"
"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
"github.com/ethereum/go-ethereum/statediff/indexer/models"
nodeinfo "github.com/ethereum/go-ethereum/statediff/indexer/node"
"github.com/ethereum/go-ethereum/statediff/types"
"github.com/cerc-io/plugeth-statediff/indexer/database/metrics"
"github.com/cerc-io/plugeth-statediff/indexer/ipld"
"github.com/cerc-io/plugeth-statediff/indexer/models"
nodeinfo "github.com/cerc-io/plugeth-statediff/indexer/node"
"github.com/cerc-io/plugeth-statediff/types"
)
var (
@@ -382,10 +382,7 @@ func parseWatchedAddressStatement(stmt string) (string, error) {
addressString := parseResult.Stmts[0].Stmt.GetInsertStmt().
SelectStmt.GetSelectStmt().
ValuesLists[0].GetList().
Items[0].GetAConst().
GetVal().
GetString_().
Str
Items[0].GetAConst().GetSval().Sval
return addressString, nil
}
indexer/database/metrics/metrics.go
+61 -72
View File
@@ -17,13 +17,12 @@
package metrics
import (
"fmt"
"strings"
"time"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
"github.com/cerc-io/plugeth-statediff/utils/log"
)
const (
@@ -73,73 +72,65 @@ type IndexerMetricsHandles struct {
StateStoreCodeProcessingTimer metrics.Timer
// Fine-grained code timers
BuildStateDiffWithIntermediateStateNodesTimer metrics.Timer
BuildStateDiffWithoutIntermediateStateNodesTimer metrics.Timer
CreatedAndUpdatedStateWithIntermediateNodesTimer metrics.Timer
DeletedOrUpdatedStateTimer metrics.Timer
BuildAccountUpdatesTimer metrics.Timer
BuildAccountCreationsTimer metrics.Timer
ResolveNodeTimer metrics.Timer
SortKeysTimer metrics.Timer
FindIntersectionTimer metrics.Timer
OutputTimer metrics.Timer
IPLDOutputTimer metrics.Timer
DifferenceIteratorNextTimer metrics.Timer
DifferenceIteratorCounter metrics.Counter
DeletedOrUpdatedStorageTimer metrics.Timer
CreatedAndUpdatedStorageTimer metrics.Timer
BuildStorageNodesIncrementalTimer metrics.Timer
BuildStateTrieObjectTimer metrics.Timer
BuildStateTrieTimer metrics.Timer
BuildStateDiffObjectTimer metrics.Timer
WriteStateDiffObjectTimer metrics.Timer
CreatedAndUpdatedStateTimer metrics.Timer
BuildStorageNodesEventualTimer metrics.Timer
BuildStorageNodesFromTrieTimer metrics.Timer
BuildRemovedAccountStorageNodesTimer metrics.Timer
BuildRemovedStorageNodesFromTrieTimer metrics.Timer
IsWatchedAddressTimer metrics.Timer
BuildStateDiffTimer metrics.Timer
CreatedAndUpdatedStateTimer metrics.Timer
DeletedOrUpdatedStateTimer metrics.Timer
BuildAccountUpdatesTimer metrics.Timer
BuildAccountCreationsTimer metrics.Timer
ResolveNodeTimer metrics.Timer
SortKeysTimer metrics.Timer
FindIntersectionTimer metrics.Timer
OutputTimer metrics.Timer
IPLDOutputTimer metrics.Timer
DifferenceIteratorNextTimer metrics.Timer
DifferenceIteratorCounter metrics.Counter
DeletedOrUpdatedStorageTimer metrics.Timer
CreatedAndUpdatedStorageTimer metrics.Timer
BuildStorageNodesIncrementalTimer metrics.Timer
BuildStateDiffObjectTimer metrics.Timer
WriteStateDiffObjectTimer metrics.Timer
BuildStorageNodesEventualTimer metrics.Timer
BuildStorageNodesFromTrieTimer metrics.Timer
BuildRemovedAccountStorageNodesTimer metrics.Timer
BuildRemovedStorageNodesFromTrieTimer metrics.Timer
IsWatchedAddressTimer metrics.Timer
}
func RegisterIndexerMetrics(reg metrics.Registry) IndexerMetricsHandles {
ctx := IndexerMetricsHandles{
BlocksCounter: metrics.NewCounter(),
TransactionsCounter: metrics.NewCounter(),
ReceiptsCounter: metrics.NewCounter(),
LogsCounter: metrics.NewCounter(),
AccessListEntriesCounter: metrics.NewCounter(),
FreePostgresTimer: metrics.NewTimer(),
PostgresCommitTimer: metrics.NewTimer(),
HeaderProcessingTimer: metrics.NewTimer(),
UncleProcessingTimer: metrics.NewTimer(),
TxAndRecProcessingTimer: metrics.NewTimer(),
StateStoreCodeProcessingTimer: metrics.NewTimer(),
BuildStateDiffWithIntermediateStateNodesTimer: metrics.NewTimer(),
BuildStateDiffWithoutIntermediateStateNodesTimer: metrics.NewTimer(),
CreatedAndUpdatedStateWithIntermediateNodesTimer: metrics.NewTimer(),
DeletedOrUpdatedStateTimer: metrics.NewTimer(),
BuildAccountUpdatesTimer: metrics.NewTimer(),
BuildAccountCreationsTimer: metrics.NewTimer(),
ResolveNodeTimer: metrics.NewTimer(),
SortKeysTimer: metrics.NewTimer(),
FindIntersectionTimer: metrics.NewTimer(),
OutputTimer: metrics.NewTimer(),
IPLDOutputTimer: metrics.NewTimer(),
DifferenceIteratorNextTimer: metrics.NewTimer(),
DifferenceIteratorCounter: metrics.NewCounter(),
DeletedOrUpdatedStorageTimer: metrics.NewTimer(),
CreatedAndUpdatedStorageTimer: metrics.NewTimer(),
BuildStorageNodesIncrementalTimer: metrics.NewTimer(),
BuildStateTrieObjectTimer: metrics.NewTimer(),
BuildStateTrieTimer: metrics.NewTimer(),
BuildStateDiffObjectTimer: metrics.NewTimer(),
WriteStateDiffObjectTimer: metrics.NewTimer(),
CreatedAndUpdatedStateTimer: metrics.NewTimer(),
BuildStorageNodesEventualTimer: metrics.NewTimer(),
BuildStorageNodesFromTrieTimer: metrics.NewTimer(),
BuildRemovedAccountStorageNodesTimer: metrics.NewTimer(),
BuildRemovedStorageNodesFromTrieTimer: metrics.NewTimer(),
IsWatchedAddressTimer: metrics.NewTimer(),
BlocksCounter: metrics.NewCounter(),
TransactionsCounter: metrics.NewCounter(),
ReceiptsCounter: metrics.NewCounter(),
LogsCounter: metrics.NewCounter(),
AccessListEntriesCounter: metrics.NewCounter(),
FreePostgresTimer: metrics.NewTimer(),
PostgresCommitTimer: metrics.NewTimer(),
HeaderProcessingTimer: metrics.NewTimer(),
UncleProcessingTimer: metrics.NewTimer(),
TxAndRecProcessingTimer: metrics.NewTimer(),
StateStoreCodeProcessingTimer: metrics.NewTimer(),
BuildStateDiffTimer: metrics.NewTimer(),
CreatedAndUpdatedStateTimer: metrics.NewTimer(),
DeletedOrUpdatedStateTimer: metrics.NewTimer(),
BuildAccountUpdatesTimer: metrics.NewTimer(),
BuildAccountCreationsTimer: metrics.NewTimer(),
ResolveNodeTimer: metrics.NewTimer(),
SortKeysTimer: metrics.NewTimer(),
FindIntersectionTimer: metrics.NewTimer(),
OutputTimer: metrics.NewTimer(),
IPLDOutputTimer: metrics.NewTimer(),
DifferenceIteratorNextTimer: metrics.NewTimer(),
DifferenceIteratorCounter: metrics.NewCounter(),
DeletedOrUpdatedStorageTimer: metrics.NewTimer(),
CreatedAndUpdatedStorageTimer: metrics.NewTimer(),
BuildStorageNodesIncrementalTimer: metrics.NewTimer(),
BuildStateDiffObjectTimer: metrics.NewTimer(),
WriteStateDiffObjectTimer: metrics.NewTimer(),
BuildStorageNodesEventualTimer: metrics.NewTimer(),
BuildStorageNodesFromTrieTimer: metrics.NewTimer(),
BuildRemovedAccountStorageNodesTimer: metrics.NewTimer(),
BuildRemovedStorageNodesFromTrieTimer: metrics.NewTimer(),
IsWatchedAddressTimer: metrics.NewTimer(),
}
subsys := "indexer"
reg.Register(metricName(subsys, "blocks"), ctx.BlocksCounter)
@@ -153,9 +144,8 @@ func RegisterIndexerMetrics(reg metrics.Registry) IndexerMetricsHandles {
reg.Register(metricName(subsys, "t_uncle_processing"), ctx.UncleProcessingTimer)
reg.Register(metricName(subsys, "t_tx_receipt_processing"), ctx.TxAndRecProcessingTimer)
reg.Register(metricName(subsys, "t_state_store_code_processing"), ctx.StateStoreCodeProcessingTimer)
reg.Register(metricName(subsys, "t_build_statediff_with_intermediate_state_nodes"), ctx.BuildStateDiffWithIntermediateStateNodesTimer)
reg.Register(metricName(subsys, "t_build_statediff_without_intermediate_state_nodes"), ctx.BuildStateDiffWithoutIntermediateStateNodesTimer)
reg.Register(metricName(subsys, "t_created_and_update_state_with_intermediate_nodes"), ctx.CreatedAndUpdatedStateWithIntermediateNodesTimer)
reg.Register(metricName(subsys, "t_build_statediff"), ctx.BuildStateDiffTimer)
reg.Register(metricName(subsys, "t_created_and_update_state"), ctx.CreatedAndUpdatedStateTimer)
reg.Register(metricName(subsys, "t_deleted_or_updated_state"), ctx.DeletedOrUpdatedStateTimer)
reg.Register(metricName(subsys, "t_build_account_updates"), ctx.BuildAccountUpdatesTimer)
reg.Register(metricName(subsys, "t_build_account_creations"), ctx.BuildAccountCreationsTimer)
@@ -169,8 +159,6 @@ func RegisterIndexerMetrics(reg metrics.Registry) IndexerMetricsHandles {
reg.Register(metricName(subsys, "t_created_and_updated_storage"), ctx.CreatedAndUpdatedStorageTimer)
reg.Register(metricName(subsys, "t_deleted_or_updated_storage"), ctx.DeletedOrUpdatedStorageTimer)
reg.Register(metricName(subsys, "t_build_storage_nodes_incremental"), ctx.BuildStorageNodesIncrementalTimer)
reg.Register(metricName(subsys, "t_build_state_trie_object"), ctx.BuildStateTrieObjectTimer)
reg.Register(metricName(subsys, "t_build_state_trie"), ctx.BuildStateTrieTimer)
reg.Register(metricName(subsys, "t_build_statediff_object"), ctx.BuildStateDiffObjectTimer)
reg.Register(metricName(subsys, "t_write_statediff_object"), ctx.WriteStateDiffObjectTimer)
reg.Register(metricName(subsys, "t_created_and_updated_state"), ctx.CreatedAndUpdatedStateTimer)
@@ -253,7 +241,8 @@ func (met *dbMetricsHandles) Update(stats DbStats) {
func ReportAndUpdateDuration(msg string, start time.Time, logger log.Logger, timer metrics.Timer) {
since := UpdateDuration(start, timer)
logger.Trace(fmt.Sprintf("%s duration=%dms", msg, since.Milliseconds()))
// This is very noisy so we log at Trace.
logger.Trace(msg, "duration", since)
}
func UpdateDuration(start time.Time, timer metrics.Timer) time.Duration {
indexer/database/sql/batch_tx.go
+7 -6
View File
@@ -23,9 +23,9 @@ import (
"github.com/lib/pq"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
"github.com/ethereum/go-ethereum/statediff/indexer/models"
"github.com/cerc-io/plugeth-statediff/indexer/ipld"
"github.com/cerc-io/plugeth-statediff/indexer/models"
"github.com/cerc-io/plugeth-statediff/utils/log"
)
const startingCacheCapacity = 1024 * 24
@@ -36,7 +36,7 @@ type BatchTx struct {
ctx context.Context
dbtx Tx
stm string
quit chan struct{}
quit chan (chan<- struct{})
iplds chan models.IPLDModel
ipldCache models.IPLDBatch
removedCacheFlag *uint32
@@ -81,8 +81,9 @@ func (tx *BatchTx) cache() {
tx.ipldCache.Keys = append(tx.ipldCache.Keys, i.Key)
tx.ipldCache.Values = append(tx.ipldCache.Values, i.Data)
tx.cacheWg.Done()
case <-tx.quit:
case confirm := <-tx.quit:
tx.ipldCache = models.IPLDBatch{}
confirm <- struct{}{}
return
}
}
@@ -121,6 +122,6 @@ func (tx *BatchTx) cacheRemoved(key string, value []byte) {
// rollback sql transaction and log any error
func rollback(ctx context.Context, tx Tx) {
if err := tx.Rollback(ctx); err != nil {
log.Error(err.Error())
log.Error("error during rollback", "error", err)
}
}
indexer/database/sql/indexer.go
+29 -27
View File
@@ -20,27 +20,26 @@
package sql
import (
"bytes"
"context"
"fmt"
"math/big"
"time"
"github.com/multiformats/go-multihash"
"github.com/ethereum/go-ethereum/common"
core "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp"
metrics2 "github.com/ethereum/go-ethereum/statediff/indexer/database/metrics"
"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
"github.com/ethereum/go-ethereum/statediff/indexer/models"
"github.com/ethereum/go-ethereum/statediff/indexer/shared"
sdtypes "github.com/ethereum/go-ethereum/statediff/types"
"github.com/multiformats/go-multihash"
metrics2 "github.com/cerc-io/plugeth-statediff/indexer/database/metrics"
"github.com/cerc-io/plugeth-statediff/indexer/interfaces"
"github.com/cerc-io/plugeth-statediff/indexer/ipld"
"github.com/cerc-io/plugeth-statediff/indexer/models"
"github.com/cerc-io/plugeth-statediff/indexer/shared"
sdtypes "github.com/cerc-io/plugeth-statediff/types"
"github.com/cerc-io/plugeth-statediff/utils/log"
)
var _ interfaces.StateDiffIndexer = &StateDiffIndexer{}
@@ -130,7 +129,7 @@ func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receip
BlockNumber: block.Number().String(),
stm: sdi.dbWriter.db.InsertIPLDsStm(),
iplds: make(chan models.IPLDModel),
quit: make(chan struct{}),
quit: make(chan (chan<- struct{})),
ipldCache: models.IPLDBatch{
BlockNumbers: make([]string, 0, startingCacheCapacity),
Keys: make([]string, 0, startingCacheCapacity),
@@ -140,7 +139,10 @@ func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receip
// handle transaction commit or rollback for any return case
submit: func(self *BatchTx, err error) error {
defer func() {
confirm := make(chan struct{})
self.quit <- confirm
close(self.quit)
<-confirm
close(self.iplds)
}()
if p := recover(); p != nil {
@@ -249,15 +251,15 @@ func (sdi *StateDiffIndexer) processHeader(tx *BatchTx, header *types.Header, he
}
// processUncles publishes and indexes uncle IPLDs in Postgres
func (sdi *StateDiffIndexer) processUncles(tx *BatchTx, headerID string, blockNumber *big.Int, unclesHash common.Hash, uncles []*types.Header) error {
func (sdi *StateDiffIndexer) processUncles(tx *BatchTx, headerID string, blockNumber *big.Int, unclesHash core.Hash, uncles []*types.Header) error {
// publish and index uncles
uncleEncoding, err := rlp.EncodeToBytes(uncles)
if err != nil {
return err
}
preparedHash := crypto.Keccak256Hash(uncleEncoding)
if !bytes.Equal(preparedHash.Bytes(), unclesHash.Bytes()) {
return fmt.Errorf("derived uncles hash (%s) does not match the hash in the header (%s)", preparedHash.Hex(), unclesHash.Hex())
if preparedHash != unclesHash {
return fmt.Errorf("derived uncles hash (%s) does not match the hash in the header (%s)", preparedHash.String(), unclesHash.String())
}
unclesCID, err := ipld.RawdataToCid(ipld.MEthHeaderList, uncleEncoding, multihash.KECCAK_256)
if err != nil {
@@ -350,7 +352,7 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *BatchTx, args processArgs
if len(receipt.PostState) == 0 {
rctModel.PostStatus = receipt.Status
} else {
rctModel.PostState = common.BytesToHash(receipt.PostState).String()
rctModel.PostState = core.BytesToHash(receipt.PostState).String()
}
if err := sdi.dbWriter.upsertReceiptCID(tx.dbtx, rctModel); err != nil {
@@ -363,7 +365,7 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *BatchTx, args processArgs
tx.cacheIPLD(args.logNodes[i][idx])
topicSet := make([]string, 4)
for ti, topic := range l.Topics {
topicSet[ti] = topic.Hex()
topicSet[ti] = topic.String()
}
logDataSet[idx] = &models.LogsModel{
@@ -401,7 +403,7 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt
stateModel = models.StateNodeModel{
BlockNumber: tx.BlockNumber,
HeaderID: headerID,
StateKey: common.BytesToHash(stateNode.AccountWrapper.LeafKey).String(),
StateKey: core.BytesToHash(stateNode.AccountWrapper.LeafKey).String(),
CID: shared.RemovedNodeStateCID,
Removed: true,
}
@@ -409,12 +411,12 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt
stateModel = models.StateNodeModel{
BlockNumber: tx.BlockNumber,
HeaderID: headerID,
StateKey: common.BytesToHash(stateNode.AccountWrapper.LeafKey).String(),
StateKey: core.BytesToHash(stateNode.AccountWrapper.LeafKey).String(),
CID: stateNode.AccountWrapper.CID,
Removed: false,
Balance: stateNode.AccountWrapper.Account.Balance.String(),
Nonce: stateNode.AccountWrapper.Account.Nonce,
CodeHash: common.BytesToHash(stateNode.AccountWrapper.Account.CodeHash).String(),
CodeHash: core.BytesToHash(stateNode.AccountWrapper.Account.CodeHash).String(),
StorageRoot: stateNode.AccountWrapper.Account.Root.String(),
}
}
@@ -431,8 +433,8 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt
storageModel := models.StorageNodeModel{
BlockNumber: tx.BlockNumber,
HeaderID: headerID,
StateKey: common.BytesToHash(stateNode.AccountWrapper.LeafKey).String(),
StorageKey: common.BytesToHash(storageNode.LeafKey).String(),
StateKey: core.BytesToHash(stateNode.AccountWrapper.LeafKey).String(),
StorageKey: core.BytesToHash(storageNode.LeafKey).String(),
CID: shared.RemovedNodeStorageCID,
Removed: true,
Value: []byte{},
@@ -445,8 +447,8 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt
storageModel := models.StorageNodeModel{
BlockNumber: tx.BlockNumber,
HeaderID: headerID,
StateKey: common.BytesToHash(stateNode.AccountWrapper.LeafKey).String(),
StorageKey: common.BytesToHash(storageNode.LeafKey).String(),
StateKey: core.BytesToHash(stateNode.AccountWrapper.LeafKey).String(),
StorageKey: core.BytesToHash(storageNode.LeafKey).String(),
CID: storageNode.CID,
Removed: false,
Value: storageNode.Value,
@@ -477,7 +479,7 @@ func (sdi *StateDiffIndexer) Close() error {
// Update the known gaps table with the gap information.
// LoadWatchedAddresses reads watched addresses from the database
func (sdi *StateDiffIndexer) LoadWatchedAddresses() ([]common.Address, error) {
func (sdi *StateDiffIndexer) LoadWatchedAddresses() ([]core.Address, error) {
addressStrings := make([]string, 0)
pgStr := "SELECT address FROM eth_meta.watched_addresses"
err := sdi.dbWriter.db.Select(sdi.ctx, &addressStrings, pgStr)
@@ -485,9 +487,9 @@ func (sdi *StateDiffIndexer) LoadWatchedAddresses() ([]common.Address, error) {
return nil, fmt.Errorf("error loading watched addresses: %v", err)
}
watchedAddresses := []common.Address{}
watchedAddresses := []core.Address{}
for _, addressString := range addressStrings {
watchedAddresses = append(watchedAddresses, common.HexToAddress(addressString))
watchedAddresses = append(watchedAddresses, core.HexToAddress(addressString))
}
return watchedAddresses, nil
indexer/database/sql/indexer_shared_test.go
+3 -3
View File
@@ -5,9 +5,9 @@ import (
"github.com/stretchr/testify/require"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
"github.com/ethereum/go-ethereum/statediff/indexer/test_helpers"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql"
"github.com/cerc-io/plugeth-statediff/indexer/interfaces"
"github.com/cerc-io/plugeth-statediff/indexer/test_helpers"
)
var (
indexer/database/sql/interfaces.go
+1 -1
View File
@@ -20,7 +20,7 @@ import (
"context"
"io"
"github.com/ethereum/go-ethereum/statediff/indexer/database/metrics"
"github.com/cerc-io/plugeth-statediff/indexer/database/metrics"
)
// Database interfaces required by the sql indexer
indexer/database/sql/lazy_tx.go
+2 -2
View File
@@ -4,7 +4,7 @@ import (
"context"
"reflect"
"github.com/ethereum/go-ethereum/log"
"github.com/cerc-io/plugeth-statediff/utils/log"
)
// Changing this to 1 would make sure only sequential COPYs were combined.
@@ -86,7 +86,7 @@ func (tx *DelayedTx) Commit(ctx context.Context) error {
case *copyFrom:
_, err := base.CopyFrom(ctx, item.tableName, item.columnNames, item.rows)
if err != nil {
log.Error("COPY error", "table", item.tableName, "err", err)
log.Error("COPY error", "table", item.tableName, "error", err)
return err
}
case cachedStmt:
indexer/database/sql/mainnet_tests/indexer_test.go
+5 -14
View File
@@ -18,20 +18,18 @@ package mainnet_tests
import (
"context"
"fmt"
"math/big"
"os"
"testing"
"github.com/stretchr/testify/require"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql/postgres"
"github.com/cerc-io/plugeth-statediff/indexer/interfaces"
"github.com/cerc-io/plugeth-statediff/indexer/test"
"github.com/cerc-io/plugeth-statediff/indexer/test_helpers"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
"github.com/ethereum/go-ethereum/statediff/indexer/test"
"github.com/ethereum/go-ethereum/statediff/indexer/test_helpers"
)
var (
@@ -41,13 +39,6 @@ var (
chainConf = params.MainnetChainConfig
)
func init() {
if os.Getenv("MODE") != "statediff" {
fmt.Println("Skipping statediff test")
os.Exit(0)
}
}
func TestMainnetIndexer(t *testing.T) {
conf := test_helpers.GetTestConfig()
indexer/database/sql/pgx_indexer_legacy_test.go
+3 -3
View File
@@ -22,9 +22,9 @@ import (
"github.com/stretchr/testify/require"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
"github.com/ethereum/go-ethereum/statediff/indexer/test"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql/postgres"
"github.com/cerc-io/plugeth-statediff/indexer/test"
)
func setupLegacyPGXIndexer(t *testing.T) {
indexer/database/sql/pgx_indexer_test.go
+19 -19
View File
@@ -23,10 +23,10 @@ import (
"github.com/stretchr/testify/require"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
"github.com/ethereum/go-ethereum/statediff/indexer/mocks"
"github.com/ethereum/go-ethereum/statediff/indexer/test"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql/postgres"
"github.com/cerc-io/plugeth-statediff/indexer/mocks"
"github.com/cerc-io/plugeth-statediff/indexer/test"
)
func setupPGXIndexer(t *testing.T, config postgres.Config) {
@@ -59,7 +59,7 @@ func TestPGXIndexer(t *testing.T) {
defer tearDown(t)
defer checkTxClosure(t, 1, 0, 1)
test.TestPublishAndIndexHeaderIPLDs(t, db)
test.DoTestPublishAndIndexHeaderIPLDs(t, db)
})
t.Run("Publish and index transaction IPLDs in a single tx", func(t *testing.T) {
@@ -67,7 +67,7 @@ func TestPGXIndexer(t *testing.T) {
defer tearDown(t)
defer checkTxClosure(t, 1, 0, 1)
test.TestPublishAndIndexTransactionIPLDs(t, db)
test.DoTestPublishAndIndexTransactionIPLDs(t, db)
})
t.Run("Publish and index log IPLDs for multiple receipt of a specific block", func(t *testing.T) {
@@ -75,7 +75,7 @@ func TestPGXIndexer(t *testing.T) {
defer tearDown(t)
defer checkTxClosure(t, 1, 0, 1)
test.TestPublishAndIndexLogIPLDs(t, db)
test.DoTestPublishAndIndexLogIPLDs(t, db)
})
t.Run("Publish and index receipt IPLDs in a single tx", func(t *testing.T) {
@@ -83,7 +83,7 @@ func TestPGXIndexer(t *testing.T) {
defer tearDown(t)
defer checkTxClosure(t, 1, 0, 1)
test.TestPublishAndIndexReceiptIPLDs(t, db)
test.DoTestPublishAndIndexReceiptIPLDs(t, db)
})
t.Run("Publish and index state IPLDs in a single tx", func(t *testing.T) {
@@ -91,7 +91,7 @@ func TestPGXIndexer(t *testing.T) {
defer tearDown(t)
defer checkTxClosure(t, 1, 0, 1)
test.TestPublishAndIndexStateIPLDs(t, db)
test.DoTestPublishAndIndexStateIPLDs(t, db)
})
t.Run("Publish and index storage IPLDs in a single tx", func(t *testing.T) {
@@ -99,7 +99,7 @@ func TestPGXIndexer(t *testing.T) {
defer tearDown(t)
defer checkTxClosure(t, 1, 0, 1)
test.TestPublishAndIndexStorageIPLDs(t, db)
test.DoTestPublishAndIndexStorageIPLDs(t, db)
})
t.Run("Publish and index with CopyFrom enabled.", func(t *testing.T) {
@@ -110,10 +110,10 @@ func TestPGXIndexer(t *testing.T) {
defer tearDown(t)
defer checkTxClosure(t, 1, 0, 1)
test.TestPublishAndIndexStateIPLDs(t, db)
test.TestPublishAndIndexStorageIPLDs(t, db)
test.TestPublishAndIndexReceiptIPLDs(t, db)
test.TestPublishAndIndexLogIPLDs(t, db)
test.DoTestPublishAndIndexStateIPLDs(t, db)
test.DoTestPublishAndIndexStorageIPLDs(t, db)
test.DoTestPublishAndIndexReceiptIPLDs(t, db)
test.DoTestPublishAndIndexLogIPLDs(t, db)
})
}
@@ -132,7 +132,7 @@ func TestPGXIndexerNonCanonical(t *testing.T) {
defer tearDown(t)
defer checkTxClosure(t, 1, 0, 1)
test.TestPublishAndIndexTransactionsNonCanonical(t, db)
test.DoTestPublishAndIndexTransactionsNonCanonical(t, db)
})
t.Run("Publish and index receipts", func(t *testing.T) {
@@ -140,7 +140,7 @@ func TestPGXIndexerNonCanonical(t *testing.T) {
defer tearDown(t)
defer checkTxClosure(t, 1, 0, 1)
test.TestPublishAndIndexReceiptsNonCanonical(t, db)
test.DoTestPublishAndIndexReceiptsNonCanonical(t, db)
})
t.Run("Publish and index logs", func(t *testing.T) {
@@ -148,7 +148,7 @@ func TestPGXIndexerNonCanonical(t *testing.T) {
defer tearDown(t)
defer checkTxClosure(t, 1, 0, 1)
test.TestPublishAndIndexLogsNonCanonical(t, db)
test.DoTestPublishAndIndexLogsNonCanonical(t, db)
})
t.Run("Publish and index state nodes", func(t *testing.T) {
@@ -156,7 +156,7 @@ func TestPGXIndexerNonCanonical(t *testing.T) {
defer tearDown(t)
defer checkTxClosure(t, 1, 0, 1)
test.TestPublishAndIndexStateNonCanonical(t, db)
test.DoTestPublishAndIndexStateNonCanonical(t, db)
})
t.Run("Publish and index storage nodes", func(t *testing.T) {
@@ -164,7 +164,7 @@ func TestPGXIndexerNonCanonical(t *testing.T) {
defer tearDown(t)
defer checkTxClosure(t, 1, 0, 1)
test.TestPublishAndIndexStorageNonCanonical(t, db)
test.DoTestPublishAndIndexStorageNonCanonical(t, db)
})
}
indexer/database/sql/postgres/config.go
+53 -41
View File
@@ -23,49 +23,9 @@ import (
"strings"
"time"
"github.com/ethereum/go-ethereum/statediff/indexer/shared"
"github.com/cerc-io/plugeth-statediff/indexer/shared"
)
// DriverType to explicitly type the kind of sql driver we are using
type DriverType string
const (
PGX DriverType = "PGX"
SQLX DriverType = "SQLX"
Unknown DriverType = "Unknown"
)
// Env variables
const (
DATABASE_NAME = "DATABASE_NAME"
DATABASE_HOSTNAME = "DATABASE_HOSTNAME"
DATABASE_PORT = "DATABASE_PORT"
DATABASE_USER = "DATABASE_USER"
DATABASE_PASSWORD = "DATABASE_PASSWORD"
)
// ResolveDriverType resolves a DriverType from a provided string
func ResolveDriverType(str string) (DriverType, error) {
switch strings.ToLower(str) {
case "pgx", "pgxpool":
return PGX, nil
case "sqlx":
return SQLX, nil
default:
return Unknown, fmt.Errorf("unrecognized driver type string: %s", str)
}
}
// TestConfig specifies default parameters for connecting to a testing DB
var TestConfig = Config{
Hostname: "localhost",
Port: 8077,
DatabaseName: "cerc_testing",
Username: "vdbm",
Password: "password",
Driver: SQLX,
}
// Config holds params for a Postgres db
type Config struct {
// conn string params
@@ -98,6 +58,34 @@ type Config struct {
CopyFrom bool
}
// DriverType to explicitly type the kind of sql driver we are using
type DriverType string
const (
PGX DriverType = "PGX"
SQLX DriverType = "SQLX"
Invalid DriverType = "Invalid"
)
// Env variables
const (
DATABASE_NAME = "DATABASE_NAME"
DATABASE_HOSTNAME = "DATABASE_HOSTNAME"
DATABASE_PORT = "DATABASE_PORT"
DATABASE_USER = "DATABASE_USER"
DATABASE_PASSWORD = "DATABASE_PASSWORD"
)
// TestConfig specifies default parameters for connecting to a testing DB
var TestConfig = Config{
Hostname: "localhost",
Port: 8077,
DatabaseName: "cerc_testing",
Username: "vdbm",
Password: "password",
Driver: SQLX,
}
// Type satisfies interfaces.Config
func (c Config) Type() shared.DBType {
return shared.POSTGRES
@@ -116,6 +104,7 @@ func (c Config) DbConnectionString() string {
return fmt.Sprintf("postgresql://%s:%d/%s?sslmode=disable", c.Hostname, c.Port, c.DatabaseName)
}
// WithEnv overrides the config with env variables, returning a new instance
func (c Config) WithEnv() (Config, error) {
if val := os.Getenv(DATABASE_NAME); val != "" {
c.DatabaseName = val
@@ -138,3 +127,26 @@ func (c Config) WithEnv() (Config, error) {
}
return c, nil
}
// ResolveDriverType resolves a DriverType from a provided string
func ResolveDriverType(str string) (DriverType, error) {
switch strings.ToLower(str) {
case "pgx", "pgxpool":
return PGX, nil
case "sqlx":
return SQLX, nil
default:
return Invalid, fmt.Errorf("unrecognized driver type string: %s", str)
}
}
// Set satisfies flag.Value
func (d *DriverType) Set(v string) (err error) {
*d, err = ResolveDriverType(v)
return
}
// String satisfies flag.Value
func (d *DriverType) String() string {
return strings.ToLower(string(*d))
}
indexer/database/sql/postgres/database.go
+2 -2
View File
@@ -17,8 +17,8 @@
package postgres
import (
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
"github.com/ethereum/go-ethereum/statediff/indexer/shared/schema"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql"
"github.com/cerc-io/plugeth-statediff/indexer/shared/schema"
)
var _ sql.Database = &DB{}
indexer/database/sql/postgres/log_adapter.go
+13 -17
View File
@@ -18,8 +18,9 @@ package postgres
import (
"context"
"github.com/ethereum/go-ethereum/log"
"github.com/jackc/pgx/v4"
"github.com/cerc-io/plugeth-statediff/utils/log"
)
type LogAdapter struct {
@@ -31,31 +32,26 @@ func NewLogAdapter(l log.Logger) *LogAdapter {
}
func (l *LogAdapter) Log(ctx context.Context, level pgx.LogLevel, msg string, data map[string]interface{}) {
var logger log.Logger
if data != nil {
var args = make([]interface{}, 0)
for key, value := range data {
if value != nil {
args = append(args, key, value)
}
args := make([]interface{}, 0)
for key, value := range data {
if value != nil {
args = append(args, key, value)
}
logger = l.l.New(args...)
} else {
logger = l.l
}
logger := l.l
switch level {
case pgx.LogLevelTrace:
logger.Trace(msg)
logger.Trace(msg, args...)
case pgx.LogLevelDebug:
logger.Debug(msg)
logger.Debug(msg, args...)
case pgx.LogLevelInfo:
logger.Info(msg)
logger.Info(msg, args...)
case pgx.LogLevelWarn:
logger.Warn(msg)
logger.Warn(msg, args...)
case pgx.LogLevelError:
logger.Error(msg)
logger.Error(msg, args...)
default:
logger.New("INVALID_PGX_LOG_LEVEL", level).Error(msg)
logger.Error(msg, "INVALID_PGX_LOG_LEVEL", level)
}
}
indexer/database/sql/postgres/pgx.go
+9 -7
View File
@@ -20,16 +20,16 @@ import (
"context"
"time"
"github.com/ethereum/go-ethereum/log"
"github.com/cerc-io/plugeth-statediff/utils/log"
"github.com/georgysavva/scany/pgxscan"
"github.com/jackc/pgconn"
"github.com/jackc/pgx/v4"
"github.com/jackc/pgx/v4/pgxpool"
"github.com/ethereum/go-ethereum/statediff/indexer/database/metrics"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
"github.com/ethereum/go-ethereum/statediff/indexer/node"
"github.com/cerc-io/plugeth-statediff/indexer/database/metrics"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql"
"github.com/cerc-io/plugeth-statediff/indexer/node"
)
// PGXDriver driver, implements sql.Driver
@@ -96,7 +96,7 @@ func MakeConfig(config Config) (*pgxpool.Config, error) {
}
if config.LogStatements {
conf.ConnConfig.Logger = NewLogAdapter(log.New())
conf.ConnConfig.Logger = NewLogAdapter(log.DefaultLogger)
}
return conf, nil
@@ -106,8 +106,10 @@ func (pgx *PGXDriver) createNode() error {
_, err := pgx.pool.Exec(
pgx.ctx,
createNodeStm,
pgx.nodeInfo.GenesisBlock, pgx.nodeInfo.NetworkID,
pgx.nodeInfo.ID, pgx.nodeInfo.ClientName,
pgx.nodeInfo.GenesisBlock,
pgx.nodeInfo.NetworkID,
pgx.nodeInfo.ID,
pgx.nodeInfo.ClientName,
pgx.nodeInfo.ChainID)
if err != nil {
return ErrUnableToSetNode(err)
indexer/database/sql/postgres/pgx_test.go
+2 -2
View File
@@ -26,8 +26,8 @@ import (
"github.com/jackc/pgx/v4/pgxpool"
"github.com/stretchr/testify/require"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
"github.com/ethereum/go-ethereum/statediff/indexer/node"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql/postgres"
"github.com/cerc-io/plugeth-statediff/indexer/node"
)
var (
indexer/database/sql/postgres/postgres_suite_test.go
-33
View File
@@ -1,33 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package postgres_test
import (
"fmt"
"os"
"github.com/ethereum/go-ethereum/log"
)
func init() {
if os.Getenv("MODE") != "statediff" {
fmt.Println("Skipping statediff test")
os.Exit(0)
}
log.Root().SetHandler(log.DiscardHandler())
}
indexer/database/sql/postgres/sqlx.go
+3 -3
View File
@@ -24,9 +24,9 @@ import (
"github.com/jmoiron/sqlx"
"github.com/ethereum/go-ethereum/statediff/indexer/database/metrics"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
"github.com/ethereum/go-ethereum/statediff/indexer/node"
"github.com/cerc-io/plugeth-statediff/indexer/database/metrics"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql"
"github.com/cerc-io/plugeth-statediff/indexer/node"
)
// SQLXDriver driver, implements sql.Driver
indexer/database/sql/postgres/sqlx_test.go
+2 -2
View File
@@ -26,8 +26,8 @@ import (
_ "github.com/lib/pq"
"github.com/stretchr/testify/require"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
"github.com/ethereum/go-ethereum/statediff/indexer/node"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql/postgres"
"github.com/cerc-io/plugeth-statediff/indexer/node"
)
func TestPostgresSQLX(t *testing.T) {
indexer/database/sql/postgres/test_helpers.go
+2 -2
View File
@@ -19,8 +19,8 @@ package postgres
import (
"context"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
"github.com/ethereum/go-ethereum/statediff/indexer/node"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql"
"github.com/cerc-io/plugeth-statediff/indexer/node"
)
// SetupSQLXDB is used to setup a sqlx db for tests
indexer/database/sql/sqlx_indexer_legacy_test.go
+3 -3
View File
@@ -22,9 +22,9 @@ import (
"github.com/stretchr/testify/require"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
"github.com/ethereum/go-ethereum/statediff/indexer/test"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql/postgres"
"github.com/cerc-io/plugeth-statediff/indexer/test"
)
func setupLegacySQLXIndexer(t *testing.T) {
indexer/database/sql/sqlx_indexer_test.go
+15 -15
View File
@@ -23,10 +23,10 @@ import (
"github.com/stretchr/testify/require"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
"github.com/ethereum/go-ethereum/statediff/indexer/mocks"
"github.com/ethereum/go-ethereum/statediff/indexer/test"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql/postgres"
"github.com/cerc-io/plugeth-statediff/indexer/mocks"
"github.com/cerc-io/plugeth-statediff/indexer/test"
)
func setupSQLXIndexer(t *testing.T) {
@@ -55,7 +55,7 @@ func TestSQLXIndexer(t *testing.T) {
defer tearDown(t)
defer checkTxClosure(t, 0, 0, 0)
test.TestPublishAndIndexHeaderIPLDs(t, db)
test.DoTestPublishAndIndexHeaderIPLDs(t, db)
})
t.Run("Publish and index transaction IPLDs in a single tx", func(t *testing.T) {
@@ -63,7 +63,7 @@ func TestSQLXIndexer(t *testing.T) {
defer tearDown(t)
defer checkTxClosure(t, 0, 0, 0)
test.TestPublishAndIndexTransactionIPLDs(t, db)
test.DoTestPublishAndIndexTransactionIPLDs(t, db)
})
t.Run("Publish and index log IPLDs for multiple receipt of a specific block", func(t *testing.T) {
@@ -71,7 +71,7 @@ func TestSQLXIndexer(t *testing.T) {
defer tearDown(t)
defer checkTxClosure(t, 0, 0, 0)
test.TestPublishAndIndexLogIPLDs(t, db)
test.DoTestPublishAndIndexLogIPLDs(t, db)
})
t.Run("Publish and index receipt IPLDs in a single tx", func(t *testing.T) {
@@ -79,7 +79,7 @@ func TestSQLXIndexer(t *testing.T) {
defer tearDown(t)
defer checkTxClosure(t, 0, 0, 0)
test.TestPublishAndIndexReceiptIPLDs(t, db)
test.DoTestPublishAndIndexReceiptIPLDs(t, db)
})
t.Run("Publish and index state IPLDs in a single tx", func(t *testing.T) {
@@ -87,7 +87,7 @@ func TestSQLXIndexer(t *testing.T) {
defer tearDown(t)
defer checkTxClosure(t, 0, 0, 0)
test.TestPublishAndIndexStateIPLDs(t, db)
test.DoTestPublishAndIndexStateIPLDs(t, db)
})
t.Run("Publish and index storage IPLDs in a single tx", func(t *testing.T) {
@@ -95,7 +95,7 @@ func TestSQLXIndexer(t *testing.T) {
defer tearDown(t)
defer checkTxClosure(t, 0, 0, 0)
test.TestPublishAndIndexStorageIPLDs(t, db)
test.DoTestPublishAndIndexStorageIPLDs(t, db)
})
}
@@ -114,7 +114,7 @@ func TestSQLXIndexerNonCanonical(t *testing.T) {
defer tearDown(t)
defer checkTxClosure(t, 0, 0, 0)
test.TestPublishAndIndexTransactionsNonCanonical(t, db)
test.DoTestPublishAndIndexTransactionsNonCanonical(t, db)
})
t.Run("Publish and index receipts", func(t *testing.T) {
@@ -122,7 +122,7 @@ func TestSQLXIndexerNonCanonical(t *testing.T) {
defer tearDown(t)
defer checkTxClosure(t, 0, 0, 0)
test.TestPublishAndIndexReceiptsNonCanonical(t, db)
test.DoTestPublishAndIndexReceiptsNonCanonical(t, db)
})
t.Run("Publish and index logs", func(t *testing.T) {
@@ -130,7 +130,7 @@ func TestSQLXIndexerNonCanonical(t *testing.T) {
defer tearDown(t)
defer checkTxClosure(t, 0, 0, 0)
test.TestPublishAndIndexLogsNonCanonical(t, db)
test.DoTestPublishAndIndexLogsNonCanonical(t, db)
})
t.Run("Publish and index state nodes", func(t *testing.T) {
@@ -138,7 +138,7 @@ func TestSQLXIndexerNonCanonical(t *testing.T) {
defer tearDown(t)
defer checkTxClosure(t, 0, 0, 0)
test.TestPublishAndIndexStateNonCanonical(t, db)
test.DoTestPublishAndIndexStateNonCanonical(t, db)
})
t.Run("Publish and index storage nodes", func(t *testing.T) {
@@ -146,7 +146,7 @@ func TestSQLXIndexerNonCanonical(t *testing.T) {
defer tearDown(t)
defer checkTxClosure(t, 0, 0, 0)
test.TestPublishAndIndexStorageNonCanonical(t, db)
test.DoTestPublishAndIndexStorageNonCanonical(t, db)
})
}
indexer/database/sql/writer.go
+2 -2
View File
@@ -25,8 +25,8 @@ import (
"github.com/lib/pq"
"github.com/shopspring/decimal"
"github.com/ethereum/go-ethereum/statediff/indexer/database/metrics"
"github.com/ethereum/go-ethereum/statediff/indexer/models"
"github.com/cerc-io/plugeth-statediff/indexer/database/metrics"
"github.com/cerc-io/plugeth-statediff/indexer/models"
)
// Writer handles processing and writing of indexed IPLD objects to Postgres