cerc-io/go-ethereum
13
0
Fork 0
Code Issues 9 Pull Requests Packages Projects Releases 66 Wiki Activity

V1.10.15 statediff 3.1.0 #197

Merged
telackey merged 7 commits from v1.10.15-statediff-3.1.0 into v1.10.15-statediff-v3 2022-01-31 21:40:25 +00:00
Conversation 0 Commits 7 Files Changed 49 +1859 -855
49 changed files with 1859 additions and 855 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
.github/workflows/on-pr.yml vendored
View File

@ -58,9 +58,12 @@ jobs:
- name: Checkout code
uses: actions/checkout@v2
- name: Start database
run: docker-compose -f docker-compose.yml up -d ipld-eth-db
- name: Start database v2
run: docker-compose -f docker-compose.yml up -d ipld-eth-db-v2
- name: Start database v3
run: docker-compose -f docker-compose.yml up -d ipld-eth-db-v3
- name: Run unit tests
run:
make statedifftest
make statedifftest

84
cmd/geth/config.go
View File

@ -201,40 +201,80 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) {
FilePath: ctx.GlobalString(utils.StateDiffFilePath.Name),
}
case shared.POSTGRES:
driverTypeStr := ctx.GlobalString(utils.StateDiffDBDriverTypeFlag.Name)
driverType, err := postgres.ResolveDriverType(driverTypeStr)
v2DriverTypeStr := ctx.GlobalString(utils.StateDiffV2DBDriverTypeFlag.Name)
v2DriverType, err := postgres.ResolveDriverType(v2DriverTypeStr)
if err != nil {
utils.Fatalf("%v", err)
}
pgConfig := postgres.Config{
Hostname: ctx.GlobalString(utils.StateDiffDBHostFlag.Name),
Port: ctx.GlobalInt(utils.StateDiffDBPortFlag.Name),
DatabaseName: ctx.GlobalString(utils.StateDiffDBNameFlag.Name),
Username: ctx.GlobalString(utils.StateDiffDBUserFlag.Name),
Password: ctx.GlobalString(utils.StateDiffDBPasswordFlag.Name),
v2PgConfig := postgres.Config{
Hostname: ctx.GlobalString(utils.StateDiffV2DBHostFlag.Name),
Port: ctx.GlobalInt(utils.StateDiffV2DBPortFlag.Name),
DatabaseName: ctx.GlobalString(utils.StateDiffV2DBNameFlag.Name),
Username: ctx.GlobalString(utils.StateDiffV2DBUserFlag.Name),
Password: ctx.GlobalString(utils.StateDiffV2DBPasswordFlag.Name),
ID: nodeID,
ClientName: clientName,
Driver: driverType,
Driver: v2DriverType,
}
if ctx.GlobalIsSet(utils.StateDiffDBMinConns.Name) {
pgConfig.MinConns = ctx.GlobalInt(utils.StateDiffDBMinConns.Name)
fmt.Printf("v2 config: %+v\r\n", v2PgConfig)
if ctx.GlobalIsSet(utils.StateDiffV2DBMinConns.Name) {
v2PgConfig.MinConns = ctx.GlobalInt(utils.StateDiffV2DBMinConns.Name)
}
if ctx.GlobalIsSet(utils.StateDiffDBMaxConns.Name) {
pgConfig.MaxConns = ctx.GlobalInt(utils.StateDiffDBMaxConns.Name)
if ctx.GlobalIsSet(utils.StateDiffV2DBMaxConns.Name) {
v2PgConfig.MaxConns = ctx.GlobalInt(utils.StateDiffV2DBMaxConns.Name)
}
if ctx.GlobalIsSet(utils.StateDiffDBMaxIdleConns.Name) {
pgConfig.MaxIdle = ctx.GlobalInt(utils.StateDiffDBMaxIdleConns.Name)
if ctx.GlobalIsSet(utils.StateDiffV2DBMaxIdleConns.Name) {
v2PgConfig.MaxIdle = ctx.GlobalInt(utils.StateDiffV2DBMaxIdleConns.Name)
}
if ctx.GlobalIsSet(utils.StateDiffDBMaxConnLifetime.Name) {
pgConfig.MaxConnLifetime = ctx.GlobalDuration(utils.StateDiffDBMaxConnLifetime.Name) * time.Second
if ctx.GlobalIsSet(utils.StateDiffV2DBMaxConnLifetime.Name) {
v2PgConfig.MaxConnLifetime = ctx.GlobalDuration(utils.StateDiffV2DBMaxConnLifetime.Name) * time.Second
}
if ctx.GlobalIsSet(utils.StateDiffDBMaxConnIdleTime.Name) {
pgConfig.MaxConnIdleTime = ctx.GlobalDuration(utils.StateDiffDBMaxConnIdleTime.Name) * time.Second
if ctx.GlobalIsSet(utils.StateDiffV2DBMaxConnIdleTime.Name) {
v2PgConfig.MaxConnIdleTime = ctx.GlobalDuration(utils.StateDiffV2DBMaxConnIdleTime.Name) * time.Second
}
if ctx.GlobalIsSet(utils.StateDiffDBConnTimeout.Name) {
pgConfig.ConnTimeout = ctx.GlobalDuration(utils.StateDiffDBConnTimeout.Name) * time.Second
if ctx.GlobalIsSet(utils.StateDiffV2DBConnTimeout.Name) {
v2PgConfig.ConnTimeout = ctx.GlobalDuration(utils.StateDiffV2DBConnTimeout.Name) * time.Second
}
v3DriverTypeStr := ctx.GlobalString(utils.StateDiffV3DBDriverTypeFlag.Name)
v3DriverType, err := postgres.ResolveDriverType(v3DriverTypeStr)
if err != nil {
utils.Fatalf("%v", err)
}
v3PgConfig := postgres.Config{
Hostname: ctx.GlobalString(utils.StateDiffV3DBHostFlag.Name),
Port: ctx.GlobalInt(utils.StateDiffV3DBPortFlag.Name),
DatabaseName: ctx.GlobalString(utils.StateDiffV3DBNameFlag.Name),
Username: ctx.GlobalString(utils.StateDiffV3DBUserFlag.Name),
Password: ctx.GlobalString(utils.StateDiffV3DBPasswordFlag.Name),
ID: nodeID,
ClientName: clientName,
Driver: v3DriverType,
}
fmt.Printf("v3 config: %+v\r\n", v3PgConfig)
if ctx.GlobalIsSet(utils.StateDiffV3DBMinConns.Name) {
v3PgConfig.MinConns = ctx.GlobalInt(utils.StateDiffV3DBMinConns.Name)
}
if ctx.GlobalIsSet(utils.StateDiffV3DBMaxConns.Name) {
v3PgConfig.MaxConns = ctx.GlobalInt(utils.StateDiffV3DBMaxConns.Name)
}
if ctx.GlobalIsSet(utils.StateDiffV3DBMaxIdleConns.Name) {
v3PgConfig.MaxIdle = ctx.GlobalInt(utils.StateDiffV3DBMaxIdleConns.Name)
}
if ctx.GlobalIsSet(utils.StateDiffV3DBMaxConnLifetime.Name) {
v3PgConfig.MaxConnLifetime = ctx.GlobalDuration(utils.StateDiffV3DBMaxConnLifetime.Name) * time.Second
}
if ctx.GlobalIsSet(utils.StateDiffV3DBMaxConnIdleTime.Name) {
v3PgConfig.MaxConnIdleTime = ctx.GlobalDuration(utils.StateDiffV3DBMaxConnIdleTime.Name) * time.Second
}
if ctx.GlobalIsSet(utils.StateDiffV3DBConnTimeout.Name) {
v3PgConfig.ConnTimeout = ctx.GlobalDuration(utils.StateDiffV3DBConnTimeout.Name) * time.Second
}
indexerConfig = postgres.MultiConfig{
V2: v2PgConfig,
V3: v3PgConfig,
}
indexerConfig = pgConfig
case shared.DUMP:
dumpTypeStr := ctx.GlobalString(utils.StateDiffDBDumpDst.Name)
dumpType, err := dumpdb.ResolveDumpType(dumpTypeStr)

36
cmd/geth/main.go
View File

@ -158,19 +158,31 @@ var (
utils.MinerNotifyFullFlag,
utils.StateDiffFlag,
utils.StateDiffDBTypeFlag,
utils.StateDiffDBDriverTypeFlag,
utils.StateDiffDBDumpDst,
utils.StateDiffDBNameFlag,
utils.StateDiffDBPasswordFlag,
utils.StateDiffDBUserFlag,
utils.StateDiffDBHostFlag,
utils.StateDiffDBPortFlag,
utils.StateDiffDBMaxConnLifetime,
utils.StateDiffDBMaxConnIdleTime,
utils.StateDiffDBMaxConns,
utils.StateDiffDBMinConns,
utils.StateDiffDBMaxIdleConns,
utils.StateDiffDBConnTimeout,
utils.StateDiffV2DBDriverTypeFlag,
utils.StateDiffV2DBNameFlag,
utils.StateDiffV2DBPasswordFlag,
utils.StateDiffV2DBUserFlag,
utils.StateDiffV2DBHostFlag,
utils.StateDiffV2DBPortFlag,
utils.StateDiffV2DBMaxConnLifetime,
utils.StateDiffV2DBMaxConnIdleTime,
utils.StateDiffV2DBMaxConns,
utils.StateDiffV2DBMinConns,
utils.StateDiffV2DBMaxIdleConns,
utils.StateDiffV2DBConnTimeout,
utils.StateDiffV3DBDriverTypeFlag,
utils.StateDiffV3DBNameFlag,
utils.StateDiffV3DBPasswordFlag,
utils.StateDiffV3DBUserFlag,
utils.StateDiffV3DBHostFlag,
utils.StateDiffV3DBPortFlag,
utils.StateDiffV3DBMaxConnLifetime,
utils.StateDiffV3DBMaxConnIdleTime,
utils.StateDiffV3DBMaxConns,
utils.StateDiffV3DBMinConns,
utils.StateDiffV3DBMaxIdleConns,
utils.StateDiffV3DBConnTimeout,
utils.StateDiffDBNodeIDFlag,
utils.StateDiffDBClientNameFlag,
utils.StateDiffWritingFlag,

36
cmd/geth/usage.go
View File

@ -228,19 +228,31 @@ var AppHelpFlagGroups = []flags.FlagGroup{
Flags: []cli.Flag{
utils.StateDiffFlag,
utils.StateDiffDBTypeFlag,
utils.StateDiffDBDriverTypeFlag,
utils.StateDiffDBDumpDst,
utils.StateDiffDBNameFlag,
utils.StateDiffDBPasswordFlag,
utils.StateDiffDBUserFlag,
utils.StateDiffDBHostFlag,
utils.StateDiffDBPortFlag,
utils.StateDiffDBMaxConnLifetime,
utils.StateDiffDBMaxConnIdleTime,
utils.StateDiffDBMaxConns,
utils.StateDiffDBMinConns,
utils.StateDiffDBMaxIdleConns,
utils.StateDiffDBConnTimeout,
utils.StateDiffV2DBDriverTypeFlag,
utils.StateDiffV2DBNameFlag,
utils.StateDiffV2DBPasswordFlag,
utils.StateDiffV2DBUserFlag,
utils.StateDiffV2DBHostFlag,
utils.StateDiffV2DBPortFlag,
utils.StateDiffV2DBMaxConnLifetime,
utils.StateDiffV2DBMaxConnIdleTime,
utils.StateDiffV2DBMaxConns,
utils.StateDiffV2DBMinConns,
utils.StateDiffV2DBMaxIdleConns,
utils.StateDiffV2DBConnTimeout,
utils.StateDiffV3DBDriverTypeFlag,
utils.StateDiffV3DBNameFlag,
utils.StateDiffV3DBPasswordFlag,
utils.StateDiffV3DBUserFlag,
utils.StateDiffV3DBHostFlag,
utils.StateDiffV3DBPortFlag,
utils.StateDiffV3DBMaxConnLifetime,
utils.StateDiffV3DBMaxConnIdleTime,
utils.StateDiffV3DBMaxConns,
utils.StateDiffV3DBMinConns,
utils.StateDiffV3DBMaxIdleConns,
utils.StateDiffV3DBConnTimeout,
utils.StateDiffDBNodeIDFlag,
utils.StateDiffDBClientNameFlag,
utils.StateDiffWritingFlag,

132
cmd/utils/flags.go
View File

@ -796,6 +796,7 @@ var (
Name: "catalyst",
Usage: "Catalyst mode (eth2 integration testing)",
}
StateDiffFlag = cli.BoolFlag{
Name: "statediff",
Usage: "Enables the processing of state diffs between each block",
@ -805,63 +806,118 @@ var (
Usage: "Statediff database type (current options: postgres, file, dump)",
Value: "postgres",
}
StateDiffDBDriverTypeFlag = cli.StringFlag{
Name: "statediff.db.driver",
Usage: "Statediff database driver type",
Value: "pgx",
}
StateDiffDBDumpDst = cli.StringFlag{
Name: "statediff.dump.dst",
Usage: "Statediff database dump destination (default is stdout)",
Value: "stdout",
}
StateDiffDBHostFlag = cli.StringFlag{
Name: "statediff.db.host",
Usage: "Statediff database hostname/ip",
StateDiffV2DBDriverTypeFlag = cli.StringFlag{
Name: "statediff.db.v2.driver",
Usage: "Statediff v2 database driver type",
Value: "pgx",
}
StateDiffV2DBHostFlag = cli.StringFlag{
Name: "statediff.db.v2.host",
Usage: "Statediff v2 database hostname/ip",
Value: "localhost",
}
StateDiffDBPortFlag = cli.IntFlag{
Name: "statediff.db.port",
Usage: "Statediff database port",
StateDiffV2DBPortFlag = cli.IntFlag{
Name: "statediff.db.v2.port",
Usage: "Statediff v2 database port",
Value: 5432,
}
StateDiffDBNameFlag = cli.StringFlag{
Name: "statediff.db.name",
Usage: "Statediff database name",
StateDiffV2DBNameFlag = cli.StringFlag{
Name: "statediff.db.v2.name",
Usage: "Statediff v2 database name",
}
StateDiffDBPasswordFlag = cli.StringFlag{
Name: "statediff.db.password",
Usage: "Statediff database password",
StateDiffV2DBPasswordFlag = cli.StringFlag{
Name: "statediff.db.v2.password",
Usage: "Statediff v2 database password",
}
StateDiffDBUserFlag = cli.StringFlag{
Name: "statediff.db.user",
Usage: "Statediff database username",
StateDiffV2DBUserFlag = cli.StringFlag{
Name: "statediff.db.v2.user",
Usage: "Statediff v2 database username",
Value: "postgres",
}
StateDiffDBMaxConnLifetime = cli.DurationFlag{
Name: "statediff.db.maxconnlifetime",
Usage: "Statediff database maximum connection lifetime (in seconds)",
StateDiffV2DBMaxConnLifetime = cli.DurationFlag{
Name: "statediff.db.v2.maxconnlifetime",
Usage: "Statediff v2 database maximum connection lifetime (in seconds)",
}
StateDiffDBMaxConnIdleTime = cli.DurationFlag{
Name: "statediff.db.maxconnidletime",
Usage: "Statediff database maximum connection idle time (in seconds)",
StateDiffV2DBMaxConnIdleTime = cli.DurationFlag{
Name: "statediff.db.v2.maxconnidletime",
Usage: "Statediff v2 database maximum connection idle time (in seconds)",
}
StateDiffDBMaxConns = cli.IntFlag{
Name: "statediff.db.maxconns",
Usage: "Statediff database maximum connections",
StateDiffV2DBMaxConns = cli.IntFlag{
Name: "statediff.db.v2.maxconns",
Usage: "Statediff v2 database maximum connections",
}
StateDiffDBMinConns = cli.IntFlag{
Name: "statediff.db.minconns",
Usage: "Statediff database minimum connections",
StateDiffV2DBMinConns = cli.IntFlag{
Name: "statediff.db.v2.minconns",
Usage: "Statediff v2 database minimum connections",
}
StateDiffDBMaxIdleConns = cli.IntFlag{
Name: "statediff.db.maxidleconns",
Usage: "Statediff database maximum idle connections",
StateDiffV2DBMaxIdleConns = cli.IntFlag{
Name: "statediff.db.v2.maxidleconns",
Usage: "Statediff v2 database maximum idle connections",
}
StateDiffDBConnTimeout = cli.DurationFlag{
Name: "statediff.db.conntimeout",
Usage: "Statediff database connection timeout (in seconds)",
StateDiffV2DBConnTimeout = cli.DurationFlag{
Name: "statediff.db.v2.conntimeout",
Usage: "Statediff v2 database connection timeout (in seconds)",
}
StateDiffV3DBDriverTypeFlag = cli.StringFlag{
Name: "statediff.db.v3.driver",
Usage: "Statediff v3 database driver type",
Value: "pgx",
}
StateDiffV3DBHostFlag = cli.StringFlag{
Name: "statediff.db.v3.host",
Usage: "Statediff v3 database hostname/ip",
Value: "localhost",
}
StateDiffV3DBPortFlag = cli.IntFlag{
Name: "statediff.db.v3.port",
Usage: "Statediff v3 database port",
Value: 5432,
}
StateDiffV3DBNameFlag = cli.StringFlag{
Name: "statediff.db.v3.name",
Usage: "Statediff v3 database name",
}
StateDiffV3DBPasswordFlag = cli.StringFlag{
Name: "statediff.db.v3.password",
Usage: "Statediff v3 database password",
}
StateDiffV3DBUserFlag = cli.StringFlag{
Name: "statediff.db.v3.user",
Usage: "Statediff v3 database username",
Value: "postgres",
}
StateDiffV3DBMaxConnLifetime = cli.DurationFlag{
Name: "statediff.db.v3.maxconnlifetime",
Usage: "Statediff v3 database maximum connection lifetime (in seconds)",
}
StateDiffV3DBMaxConnIdleTime = cli.DurationFlag{
Name: "statediff.db.v3.maxconnidletime",
Usage: "Statediff v3 database maximum connection idle time (in seconds)",
}
StateDiffV3DBMaxConns = cli.IntFlag{
Name: "statediff.db.v3.maxconns",
Usage: "Statediff v3 database maximum connections",
}
StateDiffV3DBMinConns = cli.IntFlag{
Name: "statediff.db.v3.minconns",
Usage: "Statediff v3 database minimum connections",
}
StateDiffV3DBMaxIdleConns = cli.IntFlag{
Name: "statediff.db.v3.maxidleconns",
Usage: "Statediff v3 database maximum idle connections",
}
StateDiffV3DBConnTimeout = cli.DurationFlag{
Name: "statediff.db.v3.conntimeout",
Usage: "Statediff v3 database connection timeout (in seconds)",
}
StateDiffDBNodeIDFlag = cli.StringFlag{
Name: "statediff.db.nodeid",
Usage: "Node ID to use when writing state diffs to database",

21
docker-compose.yml
View File

@ -1,17 +1,30 @@
version: '3.2'
services:
ipld-eth-db:
ipld-eth-db-v2:
restart: always
image: vulcanize/ipld-eth-db:v0.3.1
image: vulcanize/ipld-eth-db:v2.0.0
environment:
POSTGRES_USER: "vdbm"
POSTGRES_DB: "vulcanize_testing_v2"
POSTGRES_PASSWORD: "password"
volumes:
- v2_data:/var/lib/postgresql/data
ports:
- "127.0.0.1:5433:5432"
ipld-eth-db-v3:
restart: always
image: vulcanize/ipld-eth-db:v3.0.6
environment:
POSTGRES_USER: "vdbm"
POSTGRES_DB: "vulcanize_testing_v3"
POSTGRES_PASSWORD: "password"
volumes:
- geth_node:/var/lib/postgresql/data
- v3_data:/var/lib/postgresql/data
ports:
- "127.0.0.1:5432:5432"
volumes:
geth_node:
v2_data:
v3_data:

2
go.mod
View File

@ -47,9 +47,7 @@ require (
github.com/ipfs/go-ipfs-blockstore v1.0.1
github.com/ipfs/go-ipfs-ds-help v1.0.0
github.com/ipfs/go-ipld-format v0.2.0
github.com/jackc/fake v0.0.0-20150926172116-812a484cc733 // indirect
github.com/jackc/pgconn v1.10.0
github.com/jackc/pgx v3.6.2+incompatible
github.com/jackc/pgx/v4 v4.13.0
github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458
github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e

2
go.sum
View File

@ -305,8 +305,6 @@ github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9
github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8=
github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
github.com/jackc/fake v0.0.0-20150926172116-812a484cc733 h1:vr3AYkKovP8uR8AvSGGUK1IDqRa5lAAvEkZG1LKaCRc=
github.com/jackc/fake v0.0.0-20150926172116-812a484cc733/go.mod h1:WrMFNQdiFJ80sQsxDoMokWK1W5TQtxBFNpzWTD84ibQ=
github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA=
github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE=
github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s=

2
params/version.go
View File

@ -24,7 +24,7 @@ const (
VersionMajor = 1 // Major version component of the current release
VersionMinor = 10 // Minor version component of the current release
VersionPatch = 15 // Patch version component of the current release
VersionMeta = "statediff-3.0.1" // Version metadata to append to the version string
VersionMeta = "statediff-3.1.0" // Version metadata to append to the version string
)
// Version holds the textual version string.

47
statediff/README.md
View File

@ -33,7 +33,7 @@ type StateNode struct {
NodeType NodeType `json:"nodeType" gencodec:"required"`
Path []byte `json:"path" gencodec:"required"`
NodeValue []byte `json:"value" gencodec:"required"`
StorageNodes []StorageNode `json:"storage"`
StorageNodes []StorageNode `json:"storage"`
LeafKey []byte `json:"leafKey"`
}
@ -80,29 +80,42 @@ This service introduces a CLI flag namespace `statediff`
`--statediff.writing` is used to tell the service to write state diff objects it produces from synced ChainEvents directly to a configured Postgres database
`--statediff.workers` is used to set the number of concurrent workers to process state diff objects and write them into the database
`--statediff.db.type` is the type of database we write out to (current options: postgres, dump, file)
`--statediff.file.path` full path (including filename) to write statediff data out to when operating in file mode
`--statediff.dump.dst` is the destination to write to when operating in database dump mode (stdout, stderr, discard)
`--statediff.db.driver` is the specific driver to use for the database (current options for postgres: pgx and sqlx)
`--statediff.db.host` is the hostname/ip to dial to connect to the database
`--statediff.db.port` is the port to dial to connect to the database
`--statediff.db.name` is the name of the database to connect to
`--statediff.db.user` is the user to connect to the database as
`--statediff.db.password` is the password to use to connect to the database
`--statediff.db.conntimeout` is the connection timeout (in seconds)
`--statediff.db.maxconns` is the maximum number of database connections
`--statediff.db.minconns` is the minimum number of database connections
`--statediff.db.maxidleconns` is the maximum number of idle connections
`--statediff.db.maxconnidletime` is the maximum lifetime for an idle connection (in seconds)
`--statediff.db.maxconnlifetime` is the maximum lifetime for a connection (in seconds)
`--statediff.db.nodeid` is the node id to use in the Postgres database
`--statediff.db.clientname` is the client name to use in the Postgres database
`--statediff.file.path` full path (including filename) to write statediff data out to when operating in file mode
`--statediff.db.v2.driver` is the specific driver to use for the v2 database (current options for postgres: pgx and sqlx)
`--statediff.db.v2.host` is the hostname/ip to dial to connect to the v2 database
`--statediff.db.v2.port` is the port to dial to connect to the v2 database
`--statediff.db.v2.name` is the name of the v2 database to connect to
`--statediff.db.v2.user` is the user to connect to the v2 database as
`--statediff.db.v2.password` is the password to use to connect to the v2 database
`--statediff.db.v2.conntimeout` is the connection timeout (in seconds) for v2 database
`--statediff.db.v2.maxconns` is the maximum number of database connections for v2 database
`--statediff.db.v2.minconns` is the minimum number of database connections for v2 database
`--statediff.db.v2.maxidleconns` is the maximum number of idle connections for v2 database
`--statediff.db.v2.maxconnidletime` is the maximum lifetime for an idle connection (in seconds) for v2 database
`--statediff.db.v2.maxconnlifetime` is the maximum lifetime for a connection (in seconds) for v2 database
`--statediff.db.v3.driver` is the specific driver to use for the v3 database (current options for postgres: pgx and sqlx)
`--statediff.db.v3.host` is the hostname/ip to dial to connect to the v3 database
`--statediff.db.v3.port` is the port to dial to connect to the v3 database
`--statediff.db.v3.name` is the name of the v3 database to connect to
`--statediff.db.v3.user` is the user to connect to the v3 database as
`--statediff.db.v3.password` is the password to use to connect to the v3 database
`--statediff.db.v3.conntimeout` is the connection timeout (in seconds) for v3 database
`--statediff.db.v3.maxconns` is the maximum number of database connections for v3 database
`--statediff.db.v3.minconns` is the minimum number of database connections for v3 database
`--statediff.db.v3.maxidleconns` is the maximum number of idle connections for v3 database
`--statediff.db.v3.maxconnidletime` is the maximum lifetime for an idle connection (in seconds) for v3 database
`--statediff.db.v3.maxconnlifetime` is the maximum lifetime for a connection (in seconds) for v3 database
The service can only operate in full sync mode (`--syncmode=full`), but only the historical RPC endpoints require an archive node (`--gcmode=archive`)
e.g.
`
./build/bin/geth --syncmode=full --gcmode=archive --statediff --statediff.writing --statediff.db.type=postgres --statediff.db.driver=sqlx --statediff.db.host=localhost --statediff.db.port=5432 --statediff.db.name=vulcanize_test --statediff.db.user=postgres --statediff.db.nodeid=nodeid --statediff.db.clientname=clientname
`
`./build/bin/geth --syncmode=full --gcmode=archive --statediff --statediff.writing --statediff.db.type=postgres --statediff.db.nodeid=nodeid --statediff.db.v2.driver=sqlx
--statediff.db.v3.driver=sqlx --statediff.db.v2.host=localhost --statediff.db.v3.host=localhost --statediff.db.v2.port=5432 --statediff.db.v3.port=5432
--statediff.db.v2.name=vulcanize_dual_v2 --statediff.db.v3.name=vulcanize_dual_v3 --statediff.db.v2.user=postgres --statediff.db.v3.user=postgres
--statediff.db.clientname=clientname --statediff.workers=20`
When operating in `--statediff.db.type=file` mode, the service will write SQL statements out to the file designated by
`--statediff.file.path`. Please note that it writes out SQL statements with all `ON CONFLICT` constraint checks dropped.

30
statediff/indexer/constructor.go
View File

@ -26,6 +26,8 @@ import (
"github.com/ethereum/go-ethereum/statediff/indexer/database/file"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
v2 "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres/v2"
v3 "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres/v3"
"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
"github.com/ethereum/go-ethereum/statediff/indexer/node"
"github.com/ethereum/go-ethereum/statediff/indexer/shared"
@ -44,27 +46,41 @@ func NewStateDiffIndexer(ctx context.Context, chainConfig *params.ChainConfig, n
return file.NewStateDiffIndexer(ctx, chainConfig, fc)
case shared.POSTGRES:
log.Info("Starting statediff service in Postgres writing mode")
pgc, ok := config.(postgres.Config)
pgc, ok := config.(postgres.MultiConfig)
if !ok {
return nil, fmt.Errorf("postgres config is not the correct type: got %T, expected %T", config, postgres.Config{})
}
var err error
var driver sql.Driver
switch pgc.Driver {
var oldDriver, newDriver interfaces.Driver
switch pgc.V2.Driver {
case postgres.PGX:
driver, err = postgres.NewPGXDriver(ctx, pgc, nodeInfo)
oldDriver, err = postgres.NewPGXDriver(ctx, pgc.V2)
if err != nil {
return nil, err
}
case postgres.SQLX:
driver, err = postgres.NewSQLXDriver(ctx, pgc, nodeInfo)
oldDriver, err = postgres.NewSQLXDriver(ctx, pgc.V2)
if err != nil {
return nil, err
}
default:
return nil, fmt.Errorf("unrecongized Postgres driver type: %s", pgc.Driver)
return nil, fmt.Errorf("unrecongized Postgres driver type: %s", pgc.V2.Driver)
}
return sql.NewStateDiffIndexer(ctx, chainConfig, postgres.NewPostgresDB(driver))
switch pgc.V3.Driver {
case postgres.PGX:
newDriver, err = postgres.NewPGXDriver(ctx, pgc.V3)
if err != nil {
return nil, err
}
case postgres.SQLX:
newDriver, err = postgres.NewSQLXDriver(ctx, pgc.V3)
if err != nil {
return nil, err
}
default:
return nil, fmt.Errorf("unrecongized Postgres driver type: %s", pgc.V3.Driver)
}
return sql.NewStateDiffIndexer(ctx, chainConfig, nodeInfo, v2.NewPostgresDB(oldDriver), v3.NewPostgresDB(newDriver))
case shared.DUMP:
log.Info("Starting statediff service in data dump mode")
dumpc, ok := config.(dump.Config)

17
statediff/indexer/database/dump/batch_tx.go
View File

@ -20,9 +20,10 @@ import (
"fmt"
"io"
sharedModels "github.com/ethereum/go-ethereum/statediff/indexer/models/shared"
"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
"github.com/ethereum/go-ethereum/statediff/indexer/models"
blockstore "github.com/ipfs/go-ipfs-blockstore"
dshelp "github.com/ipfs/go-ipfs-ds-help"
node "github.com/ipfs/go-ipld-format"
@ -33,8 +34,8 @@ type BatchTx struct {
BlockNumber uint64
dump io.Writer
quit chan struct{}
iplds chan models.IPLDModel
ipldCache models.IPLDBatch
iplds chan sharedModels.IPLDModel
ipldCache sharedModels.IPLDBatch
submit func(blockTx *BatchTx, err error) error
}
@ -48,7 +49,7 @@ func (tx *BatchTx) flush() error {
if _, err := fmt.Fprintf(tx.dump, "%+v\r\n", tx.ipldCache); err != nil {
return err
}
tx.ipldCache = models.IPLDBatch{}
tx.ipldCache = sharedModels.IPLDBatch{}
return nil
}
@ -60,21 +61,21 @@ func (tx *BatchTx) cache() {
tx.ipldCache.Keys = append(tx.ipldCache.Keys, i.Key)
tx.ipldCache.Values = append(tx.ipldCache.Values, i.Data)
case <-tx.quit:
tx.ipldCache = models.IPLDBatch{}
tx.ipldCache = sharedModels.IPLDBatch{}
return
}
}
}
func (tx *BatchTx) cacheDirect(key string, value []byte) {
tx.iplds <- models.IPLDModel{
tx.iplds <- sharedModels.IPLDModel{
Key: key,
Data: value,
}
}
func (tx *BatchTx) cacheIPLD(i node.Node) {
tx.iplds <- models.IPLDModel{
tx.iplds <- sharedModels.IPLDModel{
Key: blockstore.BlockPrefix.String() + dshelp.MultihashToDsKey(i.Cid().Hash()).String(),
Data: i.RawData(),
}
@ -86,7 +87,7 @@ func (tx *BatchTx) cacheRaw(codec, mh uint64, raw []byte) (string, string, error
return "", "", err
}
prefixedKey := blockstore.BlockPrefix.String() + dshelp.MultihashToDsKey(c.Hash()).String()
tx.iplds <- models.IPLDModel{
tx.iplds <- sharedModels.IPLDModel{
Key: prefixedKey,
Data: raw,
}

71
statediff/indexer/database/dump/indexer.go
View File

@ -22,8 +22,6 @@ import (
"math/big"
"time"
ipld2 "github.com/ethereum/go-ethereum/statediff/indexer/ipld"
"github.com/ipfs/go-cid"
node "github.com/ipfs/go-ipld-format"
"github.com/multiformats/go-multihash"
@ -36,7 +34,9 @@ import (
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
"github.com/ethereum/go-ethereum/statediff/indexer/models"
ipld2 "github.com/ethereum/go-ethereum/statediff/indexer/ipld"
sharedModels "github.com/ethereum/go-ethereum/statediff/indexer/models/shared"
v3Models "github.com/ethereum/go-ethereum/statediff/indexer/models/v3"
"github.com/ethereum/go-ethereum/statediff/indexer/shared"
sdtypes "github.com/ethereum/go-ethereum/statediff/types"
)
@ -61,12 +61,15 @@ func NewStateDiffIndexer(chainConfig *params.ChainConfig, config Config) *StateD
}
}
// ReportDBMetrics has nothing to report for dump
func (sdi *StateDiffIndexer) ReportDBMetrics(time.Duration, <-chan bool) {}
// ReportOldDBMetrics has nothing to report for dump
func (sdi *StateDiffIndexer) ReportOldDBMetrics(time.Duration, <-chan bool) {}
// ReportNewDBMetrics has nothing to report for dump
func (sdi *StateDiffIndexer) ReportNewDBMetrics(time.Duration, <-chan bool) {}
// PushBlock pushes and indexes block data in sql, except state & storage nodes (includes header, uncles, transactions & receipts)
// Returns an initiated DB transaction which must be Closed via defer to commit or rollback
func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receipts, totalDifficulty *big.Int) (interfaces.Batch, error) {
func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receipts, totalDifficulty *big.Int) (interfaces.Batch, int64, error) {
start, t := time.Now(), time.Now()
blockHash := block.Hash()
blockHashStr := blockHash.String()
@ -75,20 +78,20 @@ func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receip
transactions := block.Transactions()
// Derive any missing fields
if err := receipts.DeriveFields(sdi.chainConfig, blockHash, height, transactions); err != nil {
return nil, err
return nil, 0, err
}
// Generate the block iplds
headerNode, uncleNodes, txNodes, txTrieNodes, rctNodes, rctTrieNodes, logTrieNodes, logLeafNodeCIDs, rctLeafNodeCIDs, err := ipld2.FromBlockAndReceipts(block, receipts)
if err != nil {
return nil, fmt.Errorf("error creating IPLD nodes from block and receipts: %v", err)
return nil, 0, fmt.Errorf("error creating IPLD nodes from block and receipts: %v", err)
}
if len(txNodes) != len(rctNodes) || len(rctNodes) != len(rctLeafNodeCIDs) {
return nil, fmt.Errorf("expected number of transactions (%d), receipts (%d), and receipt trie leaf nodes (%d) to be equal", len(txNodes), len(rctNodes), len(rctLeafNodeCIDs))
return nil, 0, fmt.Errorf("expected number of transactions (%d), receipts (%d), and receipt trie leaf nodes (%d) to be equal", len(txNodes), len(rctNodes), len(rctLeafNodeCIDs))
}
if len(txTrieNodes) != len(rctTrieNodes) {
return nil, fmt.Errorf("expected number of tx trie (%d) and rct trie (%d) nodes to be equal", len(txTrieNodes), len(rctTrieNodes))
return nil, 0, fmt.Errorf("expected number of tx trie (%d) and rct trie (%d) nodes to be equal", len(txTrieNodes), len(rctTrieNodes))
}
// Calculate reward
@ -104,9 +107,9 @@ func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receip
blockTx := &BatchTx{
BlockNumber: height,
dump: sdi.dump,
iplds: make(chan models.IPLDModel),
iplds: make(chan sharedModels.IPLDModel),
quit: make(chan struct{}),
ipldCache: models.IPLDBatch{},
ipldCache: sharedModels.IPLDBatch{},
submit: func(self *BatchTx, err error) error {
close(self.quit)
close(self.iplds)
@ -139,7 +142,7 @@ func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receip
var headerID string
headerID, err = sdi.processHeader(blockTx, block.Header(), headerNode, reward, totalDifficulty)
if err != nil {
return nil, err
return nil, 0, err
}
tDiff = time.Since(t)
indexerMetrics.tHeaderProcessing.Update(tDiff)
@ -148,7 +151,7 @@ func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receip
// Publish and index uncles
err = sdi.processUncles(blockTx, headerID, height, uncleNodes)
if err != nil {
return nil, err
return nil, 0, err
}
tDiff = time.Since(t)
indexerMetrics.tUncleProcessing.Update(tDiff)
@ -169,14 +172,14 @@ func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receip
rctLeafNodeCIDs: rctLeafNodeCIDs,
})
if err != nil {
return nil, err
return nil, 0, err
}
tDiff = time.Since(t)
indexerMetrics.tTxAndRecProcessing.Update(tDiff)
traceMsg += fmt.Sprintf("tx and receipt processing time: %s\r\n", tDiff.String())
t = time.Now()
return blockTx, err
return blockTx, 0, err
}
// processHeader publishes and indexes a header IPLD in Postgres
@ -185,7 +188,7 @@ func (sdi *StateDiffIndexer) processHeader(tx *BatchTx, header *types.Header, he
tx.cacheIPLD(headerNode)
headerID := header.Hash().String()
mod := models.HeaderModel{
mod := v3Models.HeaderModel{
CID: headerNode.Cid().String(),
MhKey: shared.MultihashKeyFromCID(headerNode.Cid()),
ParentHash: header.ParentHash.String(),
@ -217,7 +220,7 @@ func (sdi *StateDiffIndexer) processUncles(tx *BatchTx, headerID string, blockNu
} else {
uncleReward = shared.CalcUncleMinerReward(blockNumber, uncleNode.Number.Uint64())
}
uncle := models.UncleModel{
uncle := v3Models.UncleModel{
HeaderID: headerID,
CID: uncleNode.Cid().String(),
MhKey: shared.MultihashKeyFromCID(uncleNode.Cid()),
@ -273,7 +276,7 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *BatchTx, args processArgs
if err != nil {
return fmt.Errorf("error deriving tx sender: %v", err)
}
txModel := models.TxModel{
txModel := v3Models.TxModel{
HeaderID: args.headerID,
Dst: shared.HandleZeroAddrPointer(trx.To()),
Src: shared.HandleZeroAddr(from),
@ -295,7 +298,7 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *BatchTx, args processArgs
for k, storageKey := range accessListElement.StorageKeys {
storageKeys[k] = storageKey.Hex()
}
accessListElementModel := models.AccessListElementModel{
accessListElementModel := v3Models.AccessListElementModel{
TxID: trxID,
Index: int64(j),
Address: accessListElement.Address.Hex(),
@ -318,7 +321,7 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *BatchTx, args processArgs
return fmt.Errorf("invalid receipt leaf node cid")
}
rctModel := &models.ReceiptModel{
rctModel := &v3Models.ReceiptModel{
TxID: trxID,
Contract: contract,
ContractHash: contractHash,
@ -336,7 +339,7 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *BatchTx, args processArgs
return err
}
logDataSet := make([]*models.LogsModel, len(receipt.Logs))
logDataSet := make([]*v3Models.LogsModel, len(receipt.Logs))
for idx, l := range receipt.Logs {
topicSet := make([]string, 4)
for ti, topic := range l.Topics {
@ -347,7 +350,7 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *BatchTx, args processArgs
return fmt.Errorf("invalid log cid")
}
logDataSet[idx] = &models.LogsModel{
logDataSet[idx] = &v3Models.LogsModel{
ReceiptID: trxID,
Address: l.Address.String(),
Index: int64(l.Index),
@ -376,7 +379,7 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *BatchTx, args processArgs
}
// PushStateNode publishes and indexes a state diff node object (including any child storage nodes) in the IPLD sql
func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdtypes.StateNode, headerID string) error {
func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdtypes.StateNode, headerHash string, headerID int64) error {
tx, ok := batch.(*BatchTx)
if !ok {
return fmt.Errorf("sql batch is expected to be of type %T, got %T", &BatchTx{}, batch)
@ -385,8 +388,8 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt
if stateNode.NodeType == sdtypes.Removed {
// short circuit if it is a Removed node
// this assumes the db has been initialized and a public.blocks entry for the Removed node is present
stateModel := models.StateNodeModel{
HeaderID: headerID,
stateModel := v3Models.StateNodeModel{
HeaderID: headerHash,
Path: stateNode.Path,
StateKey: common.BytesToHash(stateNode.LeafKey).String(),
CID: shared.RemovedNodeStateCID,
@ -400,8 +403,8 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt
if err != nil {
return fmt.Errorf("error generating and cacheing state node IPLD: %v", err)
}
stateModel := models.StateNodeModel{
HeaderID: headerID,
stateModel := v3Models.StateNodeModel{
HeaderID: headerHash,
Path: stateNode.Path,
StateKey: common.BytesToHash(stateNode.LeafKey).String(),
CID: stateCIDStr,
@ -425,8 +428,8 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt
if err := rlp.DecodeBytes(i[1].([]byte), &account); err != nil {
return fmt.Errorf("error decoding state account rlp: %s", err.Error())
}
accountModel := models.StateAccountModel{
HeaderID: headerID,
accountModel := v3Models.StateAccountModel{
HeaderID: headerHash,
StatePath: stateNode.Path,
Balance: account.Balance.String(),
Nonce: account.Nonce,
@ -442,8 +445,8 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt
if storageNode.NodeType == sdtypes.Removed {
// short circuit if it is a Removed node
// this assumes the db has been initialized and a public.blocks entry for the Removed node is present
storageModel := models.StorageNodeModel{
HeaderID: headerID,
storageModel := v3Models.StorageNodeModel{
HeaderID: headerHash,
StatePath: stateNode.Path,
Path: storageNode.Path,
StorageKey: common.BytesToHash(storageNode.LeafKey).String(),
@ -460,8 +463,8 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt
if err != nil {
return fmt.Errorf("error generating and cacheing storage node IPLD: %v", err)
}
storageModel := models.StorageNodeModel{
HeaderID: headerID,
storageModel := v3Models.StorageNodeModel{
HeaderID: headerHash,
StatePath: stateNode.Path,
Path: storageNode.Path,
StorageKey: common.BytesToHash(storageNode.LeafKey).String(),

59
statediff/indexer/database/file/indexer.go
View File

@ -38,7 +38,7 @@ import (
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
ipld2 "github.com/ethereum/go-ethereum/statediff/indexer/ipld"
"github.com/ethereum/go-ethereum/statediff/indexer/models"
v3Models "github.com/ethereum/go-ethereum/statediff/indexer/models/v3"
"github.com/ethereum/go-ethereum/statediff/indexer/shared"
sdtypes "github.com/ethereum/go-ethereum/statediff/types"
)
@ -86,12 +86,15 @@ func NewStateDiffIndexer(ctx context.Context, chainConfig *params.ChainConfig, c
}, nil
}
// ReportDBMetrics has nothing to report for dump
func (sdi *StateDiffIndexer) ReportDBMetrics(time.Duration, <-chan bool) {}
// ReportOldDBMetrics has nothing to report for dump
func (sdi *StateDiffIndexer) ReportOldDBMetrics(time.Duration, <-chan bool) {}
// ReportNewDBMetrics has nothing to report for dump
func (sdi *StateDiffIndexer) ReportNewDBMetrics(time.Duration, <-chan bool) {}
// PushBlock pushes and indexes block data in sql, except state & storage nodes (includes header, uncles, transactions & receipts)
// Returns an initiated DB transaction which must be Closed via defer to commit or rollback
func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receipts, totalDifficulty *big.Int) (interfaces.Batch, error) {
func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receipts, totalDifficulty *big.Int) (interfaces.Batch, int64, error) {
start, t := time.Now(), time.Now()
blockHash := block.Hash()
blockHashStr := blockHash.String()
@ -100,20 +103,20 @@ func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receip
transactions := block.Transactions()
// Derive any missing fields
if err := receipts.DeriveFields(sdi.chainConfig, blockHash, height, transactions); err != nil {
return nil, err
return nil, 0, err
}
// Generate the block iplds
headerNode, uncleNodes, txNodes, txTrieNodes, rctNodes, rctTrieNodes, logTrieNodes, logLeafNodeCIDs, rctLeafNodeCIDs, err := ipld2.FromBlockAndReceipts(block, receipts)
if err != nil {
return nil, fmt.Errorf("error creating IPLD nodes from block and receipts: %v", err)
return nil, 0, fmt.Errorf("error creating IPLD nodes from block and receipts: %v", err)
}
if len(txNodes) != len(rctNodes) || len(rctNodes) != len(rctLeafNodeCIDs) {
return nil, fmt.Errorf("expected number of transactions (%d), receipts (%d), and receipt trie leaf nodes (%d) to be equal", len(txNodes), len(rctNodes), len(rctLeafNodeCIDs))
return nil, 0, fmt.Errorf("expected number of transactions (%d), receipts (%d), and receipt trie leaf nodes (%d) to be equal", len(txNodes), len(rctNodes), len(rctLeafNodeCIDs))
}
if len(txTrieNodes) != len(rctTrieNodes) {
return nil, fmt.Errorf("expected number of tx trie (%d) and rct trie (%d) nodes to be equal", len(txTrieNodes), len(rctTrieNodes))
return nil, 0, fmt.Errorf("expected number of tx trie (%d) and rct trie (%d) nodes to be equal", len(txTrieNodes), len(rctTrieNodes))
}
// Calculate reward
@ -176,14 +179,14 @@ func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receip
rctLeafNodeCIDs: rctLeafNodeCIDs,
})
if err != nil {
return nil, err
return nil, 0, err
}
tDiff = time.Since(t)
indexerMetrics.tTxAndRecProcessing.Update(tDiff)
traceMsg += fmt.Sprintf("tx and receipt processing time: %s\r\n", tDiff.String())
t = time.Now()
return blockTx, err
return blockTx, 0, err
}
// processHeader write a header IPLD insert SQL stmt to a file
@ -197,7 +200,7 @@ func (sdi *StateDiffIndexer) processHeader(header *types.Header, headerNode node
*baseFee = header.BaseFee.String()
}
headerID := header.Hash().String()
sdi.fileWriter.upsertHeaderCID(models.HeaderModel{
sdi.fileWriter.upsertHeaderCID(v3Models.HeaderModel{
NodeID: sdi.nodeID,
CID: headerNode.Cid().String(),
MhKey: shared.MultihashKeyFromCID(headerNode.Cid()),
@ -229,7 +232,7 @@ func (sdi *StateDiffIndexer) processUncles(headerID string, blockNumber uint64,
} else {
uncleReward = shared.CalcUncleMinerReward(blockNumber, uncleNode.Number.Uint64())
}
sdi.fileWriter.upsertUncleCID(models.UncleModel{
sdi.fileWriter.upsertUncleCID(v3Models.UncleModel{
HeaderID: headerID,
CID: uncleNode.Cid().String(),
MhKey: shared.MultihashKeyFromCID(uncleNode.Cid()),
@ -280,7 +283,7 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(args processArgs) error {
if err != nil {
return fmt.Errorf("error deriving tx sender: %v", err)
}
txModel := models.TxModel{
txModel := v3Models.TxModel{
HeaderID: args.headerID,
Dst: shared.HandleZeroAddrPointer(trx.To()),
Src: shared.HandleZeroAddr(from),
@ -300,7 +303,7 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(args processArgs) error {
for k, storageKey := range accessListElement.StorageKeys {
storageKeys[k] = storageKey.Hex()
}
accessListElementModel := models.AccessListElementModel{
accessListElementModel := v3Models.AccessListElementModel{
TxID: txID,
Index: int64(j),
Address: accessListElement.Address.Hex(),
@ -321,7 +324,7 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(args processArgs) error {
return fmt.Errorf("invalid receipt leaf node cid")
}
rctModel := &models.ReceiptModel{
rctModel := &v3Models.ReceiptModel{
TxID: txID,
Contract: contract,
ContractHash: contractHash,
@ -337,7 +340,7 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(args processArgs) error {
sdi.fileWriter.upsertReceiptCID(rctModel)
// index logs
logDataSet := make([]*models.LogsModel, len(receipt.Logs))
logDataSet := make([]*v3Models.LogsModel, len(receipt.Logs))
for idx, l := range receipt.Logs {
topicSet := make([]string, 4)
for ti, topic := range l.Topics {
@ -348,7 +351,7 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(args processArgs) error {
return fmt.Errorf("invalid log cid")
}
logDataSet[idx] = &models.LogsModel{
logDataSet[idx] = &v3Models.LogsModel{
ReceiptID: txID,
Address: l.Address.String(),
Index: int64(l.Index),
@ -374,13 +377,13 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(args processArgs) error {
}
// PushStateNode writes a state diff node object (including any child storage nodes) IPLD insert SQL stmt to a file
func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdtypes.StateNode, headerID string) error {
func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdtypes.StateNode, headerHash string, headerID int64) error {
// publish the state node
if stateNode.NodeType == sdtypes.Removed {
// short circuit if it is a Removed node
// this assumes the db has been initialized and a public.blocks entry for the Removed node is present
stateModel := models.StateNodeModel{
HeaderID: headerID,
stateModel := v3Models.StateNodeModel{
HeaderID: headerHash,
Path: stateNode.Path,
StateKey: common.BytesToHash(stateNode.LeafKey).String(),
CID: shared.RemovedNodeStateCID,
@ -394,8 +397,8 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt
if err != nil {
return fmt.Errorf("error generating and cacheing state node IPLD: %v", err)
}
stateModel := models.StateNodeModel{
HeaderID: headerID,
stateModel := v3Models.StateNodeModel{
HeaderID: headerHash,
Path: stateNode.Path,
StateKey: common.BytesToHash(stateNode.LeafKey).String(),
CID: stateCIDStr,
@ -417,8 +420,8 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt
if err := rlp.DecodeBytes(i[1].([]byte), &account); err != nil {
return fmt.Errorf("error decoding state account rlp: %s", err.Error())
}
accountModel := models.StateAccountModel{
HeaderID: headerID,
accountModel := v3Models.StateAccountModel{
HeaderID: headerHash,
StatePath: stateNode.Path,
Balance: account.Balance.String(),
Nonce: account.Nonce,
@ -432,8 +435,8 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt
if storageNode.NodeType == sdtypes.Removed {
// short circuit if it is a Removed node
// this assumes the db has been initialized and a public.blocks entry for the Removed node is present
storageModel := models.StorageNodeModel{
HeaderID: headerID,
storageModel := v3Models.StorageNodeModel{
HeaderID: headerHash,
StatePath: stateNode.Path,
Path: storageNode.Path,
StorageKey: common.BytesToHash(storageNode.LeafKey).String(),
@ -448,8 +451,8 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt
if err != nil {
return fmt.Errorf("error generating and cacheing storage node IPLD: %v", err)
}
storageModel := models.StorageNodeModel{
HeaderID: headerID,
storageModel := v3Models.StorageNodeModel{
HeaderID: headerHash,
StatePath: stateNode.Path,
Path: storageNode.Path,
StorageKey: common.BytesToHash(storageNode.LeafKey).String(),

5
statediff/indexer/database/file/indexer_legacy_test.go
View File

@ -52,7 +52,8 @@ func setupLegacy(t *testing.T) {
ind, err := file.NewStateDiffIndexer(context.Background(), legacyData.Config, file.TestConfig)
require.NoError(t, err)
var tx interfaces.Batch
tx, err = ind.PushBlock(
var headerID int64
tx, headerID, err = ind.PushBlock(
mockLegacyBlock,
legacyData.MockReceipts,
legacyData.MockBlock.Difficulty())
@ -67,7 +68,7 @@ func setupLegacy(t *testing.T) {
}
}()
for _, node := range legacyData.StateDiffs {
err = ind.PushStateNode(tx, node, legacyData.MockBlock.Hash().String())
err = ind.PushStateNode(tx, node, legacyData.MockBlock.Hash().String(), headerID)
require.NoError(t, err)
}

35
statediff/indexer/database/file/indexer_test.go
View File

@ -24,9 +24,11 @@ import (
"os"
"testing"
sharedModels "github.com/ethereum/go-ethereum/statediff/indexer/models/shared"
v3Models "github.com/ethereum/go-ethereum/statediff/indexer/models/v3"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/statediff/indexer/models"
"github.com/ethereum/go-ethereum/statediff/indexer/shared"
"github.com/ipfs/go-cid"
@ -171,7 +173,8 @@ func setup(t *testing.T) {
ind, err = file.NewStateDiffIndexer(context.Background(), mocks.TestConfig, file.TestConfig)
require.NoError(t, err)
var tx interfaces.Batch
tx, err = ind.PushBlock(
var headerID int64
tx, headerID, err = ind.PushBlock(
mockBlock,
mocks.MockReceipts,
mocks.MockBlock.Difficulty())
@ -187,7 +190,7 @@ func setup(t *testing.T) {
}
}()
for _, node := range mocks.StateDiffs {
err = ind.PushStateNode(tx, node, mockBlock.Hash().String())
err = ind.PushStateNode(tx, node, mockBlock.Hash().String(), headerID)
require.NoError(t, err)
}
@ -331,7 +334,7 @@ func TestFileIndexer(t *testing.T) {
if txRes.Value != transactions[3].Value().String() {
t.Fatalf("expected tx value %s got %s", transactions[3].Value().String(), txRes.Value)
}
accessListElementModels := make([]models.AccessListElementModel, 0)
accessListElementModels := make([]v3Models.AccessListElementModel, 0)
pgStr = `SELECT access_list_elements.* FROM eth.access_list_elements INNER JOIN eth.transaction_cids ON (tx_id = transaction_cids.tx_hash) WHERE cid = $1 ORDER BY access_list_elements.index ASC`
err = sqlxdb.Select(&accessListElementModels, pgStr, c)
if err != nil {
@ -340,11 +343,11 @@ func TestFileIndexer(t *testing.T) {
if len(accessListElementModels) != 2 {
t.Fatalf("expected two access list entries, got %d", len(accessListElementModels))
}
model1 := models.AccessListElementModel{
model1 := v3Models.AccessListElementModel{
Index: accessListElementModels[0].Index,
Address: accessListElementModels[0].Address,
}
model2 := models.AccessListElementModel{
model2 := v3Models.AccessListElementModel{
Index: accessListElementModels[1].Index,
Address: accessListElementModels[1].Address,
StorageKeys: accessListElementModels[1].StorageKeys,
@ -447,7 +450,7 @@ func TestFileIndexer(t *testing.T) {
expectTrue(t, test_helpers.ListContainsString(rcts, rct5CID.String()))
for idx, c := range rcts {
result := make([]models.IPLDModel, 0)
result := make([]sharedModels.IPLDModel, 0)
pgStr = `SELECT data
FROM eth.receipt_cids
INNER JOIN public.blocks ON (receipt_cids.leaf_mh_key = public.blocks.key)
@ -531,7 +534,7 @@ func TestFileIndexer(t *testing.T) {
defer tearDown(t)
// check that state nodes were properly indexed and published
stateNodes := make([]models.StateNodeModel, 0)
stateNodes := make([]v3Models.StateNodeModel, 0)
pgStr := `SELECT state_cids.cid, state_cids.state_leaf_key, state_cids.node_type, state_cids.state_path, state_cids.header_id
FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.block_hash)
WHERE header_cids.block_number = $1 AND node_type != 3`
@ -553,7 +556,7 @@ func TestFileIndexer(t *testing.T) {
t.Fatal(err)
}
pgStr = `SELECT * from eth.state_accounts WHERE header_id = $1 AND state_path = $2`
var account models.StateAccountModel
var account v3Models.StateAccountModel
err = sqlxdb.Get(&account, pgStr, stateNode.HeaderID, stateNode.Path)
if err != nil {
t.Fatal(err)
@ -563,7 +566,7 @@ func TestFileIndexer(t *testing.T) {
test_helpers.ExpectEqual(t, stateNode.StateKey, common.BytesToHash(mocks.ContractLeafKey).Hex())
test_helpers.ExpectEqual(t, stateNode.Path, []byte{'\x06'})
test_helpers.ExpectEqual(t, data, mocks.ContractLeafNode)
test_helpers.ExpectEqual(t, account, models.StateAccountModel{
test_helpers.ExpectEqual(t, account, v3Models.StateAccountModel{
HeaderID: account.HeaderID,
StatePath: stateNode.Path,
Balance: "0",
@ -577,7 +580,7 @@ func TestFileIndexer(t *testing.T) {
test_helpers.ExpectEqual(t, stateNode.StateKey, common.BytesToHash(mocks.AccountLeafKey).Hex())
test_helpers.ExpectEqual(t, stateNode.Path, []byte{'\x0c'})
test_helpers.ExpectEqual(t, data, mocks.AccountLeafNode)
test_helpers.ExpectEqual(t, account, models.StateAccountModel{
test_helpers.ExpectEqual(t, account, v3Models.StateAccountModel{
HeaderID: account.HeaderID,
StatePath: stateNode.Path,
Balance: "1000",
@ -589,7 +592,7 @@ func TestFileIndexer(t *testing.T) {
}
// check that Removed state nodes were properly indexed and published
stateNodes = make([]models.StateNodeModel, 0)
stateNodes = make([]v3Models.StateNodeModel, 0)
pgStr = `SELECT state_cids.cid, state_cids.state_leaf_key, state_cids.node_type, state_cids.state_path, state_cids.header_id
FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.block_hash)
WHERE header_cids.block_number = $1 AND node_type = 3`
@ -622,7 +625,7 @@ func TestFileIndexer(t *testing.T) {
defer tearDown(t)
// check that storage nodes were properly indexed
storageNodes := make([]models.StorageNodeWithStateKeyModel, 0)
storageNodes := make([]v3Models.StorageNodeWithStateKeyModel, 0)
pgStr := `SELECT storage_cids.cid, state_cids.state_leaf_key, storage_cids.storage_leaf_key, storage_cids.node_type, storage_cids.storage_path
FROM eth.storage_cids, eth.state_cids, eth.header_cids
WHERE (storage_cids.state_path, storage_cids.header_id) = (state_cids.state_path, state_cids.header_id)
@ -634,7 +637,7 @@ func TestFileIndexer(t *testing.T) {
t.Fatal(err)
}
test_helpers.ExpectEqual(t, len(storageNodes), 1)
test_helpers.ExpectEqual(t, storageNodes[0], models.StorageNodeWithStateKeyModel{
test_helpers.ExpectEqual(t, storageNodes[0], v3Models.StorageNodeWithStateKeyModel{
CID: storageCID.String(),
NodeType: 2,
StorageKey: common.BytesToHash(mocks.StorageLeafKey).Hex(),
@ -655,7 +658,7 @@ func TestFileIndexer(t *testing.T) {
test_helpers.ExpectEqual(t, data, mocks.StorageLeafNode)
// check that Removed storage nodes were properly indexed
storageNodes = make([]models.StorageNodeWithStateKeyModel, 0)
storageNodes = make([]v3Models.StorageNodeWithStateKeyModel, 0)
pgStr = `SELECT storage_cids.cid, state_cids.state_leaf_key, storage_cids.storage_leaf_key, storage_cids.node_type, storage_cids.storage_path
FROM eth.storage_cids, eth.state_cids, eth.header_cids
WHERE (storage_cids.state_path, storage_cids.header_id) = (state_cids.state_path, state_cids.header_id)
@ -667,7 +670,7 @@ func TestFileIndexer(t *testing.T) {
t.Fatal(err)
}
test_helpers.ExpectEqual(t, len(storageNodes), 1)
test_helpers.ExpectEqual(t, storageNodes[0], models.StorageNodeWithStateKeyModel{
test_helpers.ExpectEqual(t, storageNodes[0], v3Models.StorageNodeWithStateKeyModel{
CID: shared.RemovedNodeStorageCID,
NodeType: 3,
StorageKey: common.BytesToHash(mocks.RemovedLeafKey).Hex(),

5
statediff/indexer/database/file/mainnet_tests/indexer_test.go
View File

@ -88,7 +88,8 @@ func setup(t *testing.T, testBlock *types.Block, testReceipts types.Receipts) {
ind, err := file.NewStateDiffIndexer(context.Background(), chainConf, file.TestConfig)
require.NoError(t, err)
var tx interfaces.Batch
tx, err = ind.PushBlock(
var headerID int64
tx, headerID, err = ind.PushBlock(
testBlock,
testReceipts,
testBlock.Difficulty())
@ -103,7 +104,7 @@ func setup(t *testing.T, testBlock *types.Block, testReceipts types.Receipts) {
}
}()
for _, node := range mocks.StateDiffs {
err = ind.PushStateNode(tx, node, testBlock.Hash().String())
err = ind.PushStateNode(tx, node, testBlock.Hash().String(), headerID)
require.NoError(t, err)
}

30
statediff/indexer/database/file/writer.go
View File

@ -20,13 +20,15 @@ import (
"fmt"
"io"
sharedModels "github.com/ethereum/go-ethereum/statediff/indexer/models/shared"
blockstore "github.com/ipfs/go-ipfs-blockstore"
dshelp "github.com/ipfs/go-ipfs-ds-help"
node "github.com/ipfs/go-ipld-format"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
"github.com/ethereum/go-ethereum/statediff/indexer/models"
v3Models "github.com/ethereum/go-ethereum/statediff/indexer/models/v3"
nodeinfo "github.com/ethereum/go-ethereum/statediff/indexer/node"
)
@ -161,19 +163,19 @@ func (sqw *SQLWriter) upsertNode(node nodeinfo.Info) {
sqw.stmts <- []byte(fmt.Sprintf(nodeInsert, node.GenesisBlock, node.NetworkID, node.ID, node.ClientName, node.ChainID))
}
func (sqw *SQLWriter) upsertIPLD(ipld models.IPLDModel) {
func (sqw *SQLWriter) upsertIPLD(ipld sharedModels.IPLDModel) {
sqw.stmts <- []byte(fmt.Sprintf(ipldInsert, ipld.Key, ipld.Data))
}
func (sqw *SQLWriter) upsertIPLDDirect(key string, value []byte) {
sqw.upsertIPLD(models.IPLDModel{
sqw.upsertIPLD(sharedModels.IPLDModel{
Key: key,
Data: value,
})
}
func (sqw *SQLWriter) upsertIPLDNode(i node.Node) {
sqw.upsertIPLD(models.IPLDModel{
sqw.upsertIPLD(sharedModels.IPLDModel{
Key: blockstore.BlockPrefix.String() + dshelp.MultihashToDsKey(i.Cid().Hash()).String(),
Data: i.RawData(),
})
@ -185,14 +187,14 @@ func (sqw *SQLWriter) upsertIPLDRaw(codec, mh uint64, raw []byte) (string, strin
return "", "", err
}
prefixedKey := blockstore.BlockPrefix.String() + dshelp.MultihashToDsKey(c.Hash()).String()
sqw.upsertIPLD(models.IPLDModel{
sqw.upsertIPLD(sharedModels.IPLDModel{
Key: prefixedKey,
Data: raw,
})
return c.String(), prefixedKey, err
}
func (sqw *SQLWriter) upsertHeaderCID(header models.HeaderModel) {
func (sqw *SQLWriter) upsertHeaderCID(header v3Models.HeaderModel) {
stmt := fmt.Sprintf(headerInsert, header.BlockNumber, header.BlockHash, header.ParentHash, header.CID,
header.TotalDifficulty, header.NodeID, header.Reward, header.StateRoot, header.TxRoot,
header.RctRoot, header.UncleRoot, header.Bloom, header.Timestamp, header.MhKey, 1, header.Coinbase)
@ -200,30 +202,30 @@ func (sqw *SQLWriter) upsertHeaderCID(header models.HeaderModel) {
indexerMetrics.blocks.Inc(1)
}
func (sqw *SQLWriter) upsertUncleCID(uncle models.UncleModel) {
func (sqw *SQLWriter) upsertUncleCID(uncle v3Models.UncleModel) {
sqw.stmts <- []byte(fmt.Sprintf(uncleInsert, uncle.BlockHash, uncle.HeaderID, uncle.ParentHash, uncle.CID,
uncle.Reward, uncle.MhKey))
}
func (sqw *SQLWriter) upsertTransactionCID(transaction models.TxModel) {
func (sqw *SQLWriter) upsertTransactionCID(transaction v3Models.TxModel) {
sqw.stmts <- []byte(fmt.Sprintf(txInsert, transaction.HeaderID, transaction.TxHash, transaction.CID, transaction.Dst,
transaction.Src, transaction.Index, transaction.MhKey, transaction.Data, transaction.Type, transaction.Value))
indexerMetrics.transactions.Inc(1)
}
func (sqw *SQLWriter) upsertAccessListElement(accessListElement models.AccessListElementModel) {
func (sqw *SQLWriter) upsertAccessListElement(accessListElement v3Models.AccessListElementModel) {
sqw.stmts <- []byte(fmt.Sprintf(alInsert, accessListElement.TxID, accessListElement.Index, accessListElement.Address,
formatPostgresStringArray(accessListElement.StorageKeys)))
indexerMetrics.accessListEntries.Inc(1)
}
func (sqw *SQLWriter) upsertReceiptCID(rct *models.ReceiptModel) {
func (sqw *SQLWriter) upsertReceiptCID(rct *v3Models.ReceiptModel) {
sqw.stmts <- []byte(fmt.Sprintf(rctInsert, rct.TxID, rct.LeafCID, rct.Contract, rct.ContractHash, rct.LeafMhKey,
rct.PostState, rct.PostStatus, rct.LogRoot))
indexerMetrics.receipts.Inc(1)
}
func (sqw *SQLWriter) upsertLogCID(logs []*models.LogsModel) {
func (sqw *SQLWriter) upsertLogCID(logs []*v3Models.LogsModel) {
for _, l := range logs {
sqw.stmts <- []byte(fmt.Sprintf(logInsert, l.LeafCID, l.LeafMhKey, l.ReceiptID, l.Address, l.Index, l.Topic0,
l.Topic1, l.Topic2, l.Topic3, l.Data))
@ -231,7 +233,7 @@ func (sqw *SQLWriter) upsertLogCID(logs []*models.LogsModel) {
}
}
func (sqw *SQLWriter) upsertStateCID(stateNode models.StateNodeModel) {
func (sqw *SQLWriter) upsertStateCID(stateNode v3Models.StateNodeModel) {
var stateKey string
if stateNode.StateKey != nullHash.String() {
stateKey = stateNode.StateKey
@ -240,12 +242,12 @@ func (sqw *SQLWriter) upsertStateCID(stateNode models.StateNodeModel) {
stateNode.NodeType, true, stateNode.MhKey))
}
func (sqw *SQLWriter) upsertStateAccount(stateAccount models.StateAccountModel) {
func (sqw *SQLWriter) upsertStateAccount(stateAccount v3Models.StateAccountModel) {
sqw.stmts <- []byte(fmt.Sprintf(accountInsert, stateAccount.HeaderID, stateAccount.StatePath, stateAccount.Balance,
stateAccount.Nonce, stateAccount.CodeHash, stateAccount.StorageRoot))
}
func (sqw *SQLWriter) upsertStorageCID(storageCID models.StorageNodeModel) {
func (sqw *SQLWriter) upsertStorageCID(storageCID v3Models.StorageNodeModel) {
var storageKey string
if storageCID.StorageKey != nullHash.String() {
storageKey = storageCID.StorageKey

40
statediff/indexer/database/sql/batch_tx.go
View File

@ -18,6 +18,9 @@ package sql
import (
"context"
"fmt"
"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
blockstore "github.com/ipfs/go-ipfs-blockstore"
dshelp "github.com/ipfs/go-ipfs-ds-help"
@ -26,18 +29,19 @@ import (
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
"github.com/ethereum/go-ethereum/statediff/indexer/models"
modelsShared "github.com/ethereum/go-ethereum/statediff/indexer/models/shared"
)
// BatchTx wraps a sql tx with the state necessary for building the tx concurrently during trie difference iteration
type BatchTx struct {
BlockNumber uint64
ctx context.Context
dbtx Tx
stm string
quit chan struct{}
iplds chan models.IPLDModel
ipldCache models.IPLDBatch
BlockNumber uint64
ctx context.Context
oldDBTx interfaces.Tx
newDBTx interfaces.Tx
oldStmt, newStmt string
quit chan struct{}
iplds chan modelsShared.IPLDModel
ipldCache modelsShared.IPLDBatch
submit func(blockTx *BatchTx, err error) error
}
@ -48,11 +52,15 @@ func (tx *BatchTx) Submit(err error) error {
}
func (tx *BatchTx) flush() error {
_, err := tx.dbtx.Exec(tx.ctx, tx.stm, pq.Array(tx.ipldCache.Keys), pq.Array(tx.ipldCache.Values))
_, err := tx.oldDBTx.Exec(tx.ctx, tx.oldStmt, pq.Array(tx.ipldCache.Keys), pq.Array(tx.ipldCache.Values))
if err != nil {
return err
return fmt.Errorf("error flushing IPLD cache to old DB: %v", err)
}
tx.ipldCache = models.IPLDBatch{}
_, err = tx.newDBTx.Exec(tx.ctx, tx.newStmt, pq.Array(tx.ipldCache.Keys), pq.Array(tx.ipldCache.Values))
if err != nil {
return fmt.Errorf("error flushing IPLD cache to new DB: %v", err)
}
tx.ipldCache = modelsShared.IPLDBatch{}
return nil
}
@ -64,21 +72,21 @@ func (tx *BatchTx) cache() {
tx.ipldCache.Keys = append(tx.ipldCache.Keys, i.Key)
tx.ipldCache.Values = append(tx.ipldCache.Values, i.Data)
case <-tx.quit:
tx.ipldCache = models.IPLDBatch{}
tx.ipldCache = modelsShared.IPLDBatch{}
return
}
}
}
func (tx *BatchTx) cacheDirect(key string, value []byte) {
tx.iplds <- models.IPLDModel{
tx.iplds <- modelsShared.IPLDModel{
Key: key,
Data: value,
}
}
func (tx *BatchTx) cacheIPLD(i node.Node) {
tx.iplds <- models.IPLDModel{
tx.iplds <- modelsShared.IPLDModel{
Key: blockstore.BlockPrefix.String() + dshelp.MultihashToDsKey(i.Cid().Hash()).String(),
Data: i.RawData(),
}
@ -90,7 +98,7 @@ func (tx *BatchTx) cacheRaw(codec, mh uint64, raw []byte) (string, string, error
return "", "", err
}
prefixedKey := blockstore.BlockPrefix.String() + dshelp.MultihashToDsKey(c.Hash()).String()
tx.iplds <- models.IPLDModel{
tx.iplds <- modelsShared.IPLDModel{
Key: prefixedKey,
Data: raw,
}
@ -98,7 +106,7 @@ func (tx *BatchTx) cacheRaw(codec, mh uint64, raw []byte) (string, string, error
}
// rollback sql transaction and log any error
func rollback(ctx context.Context, tx Tx) {
func rollback(ctx context.Context, tx interfaces.Tx) {
if err := tx.Rollback(ctx); err != nil {
log.Error(err.Error())
}

413
statediff/indexer/database/sql/indexer.go
View File

@ -21,11 +21,13 @@ package sql
import (
"context"
"errors"
"fmt"
"math/big"
"strings"
"time"
ipld2 "github.com/ethereum/go-ethereum/statediff/indexer/ipld"
metrics2 "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/metrics"
"github.com/ipfs/go-cid"
node "github.com/ipfs/go-ipld-format"
@ -38,8 +40,14 @@ import (
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp"
v2Writer "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/v2"
v3Writer "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/v3"
"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
"github.com/ethereum/go-ethereum/statediff/indexer/models"
"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
sharedModels "github.com/ethereum/go-ethereum/statediff/indexer/models/shared"
v2Models "github.com/ethereum/go-ethereum/statediff/indexer/models/v2"
v3Models "github.com/ethereum/go-ethereum/statediff/indexer/models/v3"
nodeInfo "github.com/ethereum/go-ethereum/statediff/indexer/node"
"github.com/ethereum/go-ethereum/statediff/indexer/shared"
sdtypes "github.com/ethereum/go-ethereum/statediff/types"
)
@ -47,32 +55,46 @@ import (
var _ interfaces.StateDiffIndexer = &StateDiffIndexer{}
var (
indexerMetrics = RegisterIndexerMetrics(metrics.DefaultRegistry)
dbMetrics = RegisterDBMetrics(metrics.DefaultRegistry)
indexerMetrics = metrics2.RegisterIndexerMetrics(metrics.DefaultRegistry)
dbMetrics = metrics2.RegisterDBMetrics(metrics.DefaultRegistry)
)
// StateDiffIndexer satisfies the indexer.StateDiffIndexer interface for ethereum statediff objects on top of an SQL sql
type StateDiffIndexer struct {
ctx context.Context
chainConfig *params.ChainConfig
dbWriter *Writer
oldDBWriter *v2Writer.Writer
newDBWriter *v3Writer.Writer
}
// NewStateDiffIndexer creates a sql implementation of interfaces.StateDiffIndexer
func NewStateDiffIndexer(ctx context.Context, chainConfig *params.ChainConfig, db Database) (*StateDiffIndexer, error) {
func NewStateDiffIndexer(ctx context.Context, chainConfig *params.ChainConfig, info nodeInfo.Info, old, new interfaces.Database) (*StateDiffIndexer, error) {
// Write the removed node to the db on init
if _, err := db.Exec(ctx, db.InsertIPLDStm(), shared.RemovedNodeMhKey, []byte{}); err != nil {
return nil, err
if _, err := old.Exec(ctx, old.InsertIPLDStm(), shared.RemovedNodeMhKey, []byte{}); err != nil {
return nil, fmt.Errorf("unable to write removed node IPLD to old DB: %v", err)
}
if _, err := new.Exec(ctx, new.InsertIPLDStm(), shared.RemovedNodeMhKey, []byte{}); err != nil {
return nil, fmt.Errorf("unable to write removed node IPLD to new DB: %v", err)
}
// Write node info to the db on init
oldWriter := v2Writer.NewWriter(old)
newWriter := v3Writer.NewWriter(new)
if err := oldWriter.InsertNodeInfo(info); err != nil {
return nil, fmt.Errorf("unable to write node info to old DB: %v", err)
}
if err := newWriter.InsertNodeInfo(info); err != nil {
return nil, fmt.Errorf("unable to write node info to new DB: %v", err)
}
return &StateDiffIndexer{
ctx: ctx,
chainConfig: chainConfig,
dbWriter: NewWriter(db),
oldDBWriter: oldWriter,
newDBWriter: newWriter,
}, nil
}
// ReportDBMetrics is a reporting function to run as goroutine
func (sdi *StateDiffIndexer) ReportDBMetrics(delay time.Duration, quit <-chan bool) {
// ReportOldDBMetrics is a reporting function to run as goroutine
func (sdi *StateDiffIndexer) ReportOldDBMetrics(delay time.Duration, quit <-chan bool) {
if !metrics.Enabled {
return
}
@ -81,7 +103,26 @@ func (sdi *StateDiffIndexer) ReportDBMetrics(delay time.Duration, quit <-chan bo
for {
select {
case <-ticker.C:
dbMetrics.Update(sdi.dbWriter.db.Stats())
dbMetrics.Update(sdi.oldDBWriter.Stats())
case <-quit:
ticker.Stop()
return
}
}
}()
}
// ReportNewDBMetrics is a reporting function to run as goroutine
func (sdi *StateDiffIndexer) ReportNewDBMetrics(delay time.Duration, quit <-chan bool) {
if !metrics.Enabled {
return
}
ticker := time.NewTicker(delay)
go func() {
for {
select {
case <-ticker.C:
dbMetrics.Update(sdi.newDBWriter.DB.Stats())
case <-quit:
ticker.Stop()
return
@ -92,7 +133,7 @@ func (sdi *StateDiffIndexer) ReportDBMetrics(delay time.Duration, quit <-chan bo
// PushBlock pushes and indexes block data in sql, except state & storage nodes (includes header, uncles, transactions & receipts)
// Returns an initiated DB transaction which must be Closed via defer to commit or rollback
func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receipts, totalDifficulty *big.Int) (interfaces.Batch, error) {
func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receipts, totalDifficulty *big.Int) (interfaces.Batch, int64, error) {
start, t := time.Now(), time.Now()
blockHash := block.Hash()
blockHashStr := blockHash.String()
@ -101,20 +142,20 @@ func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receip
transactions := block.Transactions()
// Derive any missing fields
if err := receipts.DeriveFields(sdi.chainConfig, blockHash, height, transactions); err != nil {
return nil, err
return nil, 0, err
}
// Generate the block iplds
headerNode, uncleNodes, txNodes, txTrieNodes, rctNodes, rctTrieNodes, logTrieNodes, logLeafNodeCIDs, rctLeafNodeCIDs, err := ipld2.FromBlockAndReceipts(block, receipts)
headerNode, uncleNodes, txNodes, txTrieNodes, rctNodes, rctTrieNodes, logTrieNodes, logLeafNodeCIDs, rctLeafNodeCIDs, err := ipld.FromBlockAndReceipts(block, receipts)
if err != nil {
return nil, fmt.Errorf("error creating IPLD nodes from block and receipts: %v", err)
return nil, 0, fmt.Errorf("error creating IPLD nodes from block and receipts: %v", err)
}
if len(txNodes) != len(rctNodes) || len(rctNodes) != len(rctLeafNodeCIDs) {
return nil, fmt.Errorf("expected number of transactions (%d), receipts (%d), and receipt trie leaf nodes (%d) to be equal", len(txNodes), len(rctNodes), len(rctLeafNodeCIDs))
return nil, 0, fmt.Errorf("expected number of transactions (%d), receipts (%d), and receipt trie leaf nodes (%d) to be equal", len(txNodes), len(rctNodes), len(rctLeafNodeCIDs))
}
if len(txTrieNodes) != len(rctTrieNodes) {
return nil, fmt.Errorf("expected number of tx trie (%d) and rct trie (%d) nodes to be equal", len(txTrieNodes), len(rctTrieNodes))
return nil, 0, fmt.Errorf("expected number of tx trie (%d) and rct trie (%d) nodes to be equal", len(txTrieNodes), len(rctTrieNodes))
}
// Calculate reward
@ -128,26 +169,35 @@ func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receip
t = time.Now()
// Begin new db tx for everything
tx, err := sdi.dbWriter.db.Begin(sdi.ctx)
oldTx, err := sdi.oldDBWriter.DB.Begin(sdi.ctx)
if err != nil {
return nil, err
return nil, 0, err
}
newTx, err := sdi.newDBWriter.DB.Begin(sdi.ctx)
if err != nil {
rollback(sdi.ctx, oldTx)
return nil, 0, err
}
defer func() {
if p := recover(); p != nil {
rollback(sdi.ctx, tx)
rollback(sdi.ctx, newTx)
rollback(sdi.ctx, oldTx)
panic(p)
} else if err != nil {
rollback(sdi.ctx, tx)
rollback(sdi.ctx, newTx)
rollback(sdi.ctx, oldTx)
}
}()
blockTx := &BatchTx{
ctx: sdi.ctx,
BlockNumber: height,
stm: sdi.dbWriter.db.InsertIPLDsStm(),
iplds: make(chan models.IPLDModel),
oldStmt: sdi.oldDBWriter.DB.InsertIPLDsStm(),
newStmt: sdi.newDBWriter.DB.InsertIPLDsStm(),
iplds: make(chan sharedModels.IPLDModel),
quit: make(chan struct{}),
ipldCache: models.IPLDBatch{},
dbtx: tx,
ipldCache: sharedModels.IPLDBatch{},
oldDBTx: oldTx,
newDBTx: newTx,
// handle transaction commit or rollback for any return case
submit: func(self *BatchTx, err error) error {
defer func() {
@ -155,24 +205,38 @@ func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receip
close(self.iplds)
}()
if p := recover(); p != nil {
rollback(sdi.ctx, tx)
rollback(sdi.ctx, oldTx)
rollback(sdi.ctx, newTx)
panic(p)
} else if err != nil {
rollback(sdi.ctx, tx)
rollback(sdi.ctx, oldTx)
rollback(sdi.ctx, newTx)
} else {
tDiff := time.Since(t)
indexerMetrics.tStateStoreCodeProcessing.Update(tDiff)
indexerMetrics.TimeStateStoreCodeProcessing.Update(tDiff)
traceMsg += fmt.Sprintf("state, storage, and code storage processing time: %s\r\n", tDiff.String())
t = time.Now()
if err := self.flush(); err != nil {
rollback(sdi.ctx, tx)
rollback(sdi.ctx, oldTx)
rollback(sdi.ctx, newTx)
traceMsg += fmt.Sprintf(" TOTAL PROCESSING DURATION: %s\r\n", time.Since(start).String())
log.Debug(traceMsg)
return err
}
err = tx.Commit(sdi.ctx)
errs := make([]string, 0, 2)
err = oldTx.Commit(sdi.ctx)
if err != nil {
errs = append(errs, fmt.Sprintf("old DB tx commit error: %s", err.Error()))
}
err = newTx.Commit(sdi.ctx)
if err != nil {
errs = append(errs, fmt.Sprintf("new DB tx commit error: %s", err.Error()))
}
if len(errs) > 0 {
err = errors.New(strings.Join(errs, " && "))
}
tDiff = time.Since(t)
indexerMetrics.tPostgresCommit.Update(tDiff)
indexerMetrics.TimePostgresCommit.Update(tDiff)
traceMsg += fmt.Sprintf("postgres transaction commit duration: %s\r\n", tDiff.String())
}
traceMsg += fmt.Sprintf(" TOTAL PROCESSING DURATION: %s\r\n", time.Since(start).String())
@ -183,32 +247,33 @@ func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receip
go blockTx.cache()
tDiff := time.Since(t)
indexerMetrics.tFreePostgres.Update(tDiff)
indexerMetrics.TimeFreePostgres.Update(tDiff)
traceMsg += fmt.Sprintf("time spent waiting for free postgres tx: %s:\r\n", tDiff.String())
t = time.Now()
// Publish and index header, collect headerID
var headerID string
var headerID int64
headerID, err = sdi.processHeader(blockTx, block.Header(), headerNode, reward, totalDifficulty)
if err != nil {
return nil, err
return nil, 0, err
}
tDiff = time.Since(t)
indexerMetrics.tHeaderProcessing.Update(tDiff)
indexerMetrics.TimeHeaderProcessing.Update(tDiff)
traceMsg += fmt.Sprintf("header processing time: %s\r\n", tDiff.String())
t = time.Now()
// Publish and index uncles
err = sdi.processUncles(blockTx, headerID, height, uncleNodes)
err = sdi.processUncles(blockTx, blockHashStr, headerID, height, uncleNodes)
if err != nil {
return nil, err
return nil, 0, err
}
tDiff = time.Since(t)
indexerMetrics.tUncleProcessing.Update(tDiff)
indexerMetrics.TimeUncleProcessing.Update(tDiff)
traceMsg += fmt.Sprintf("uncle processing time: %s\r\n", tDiff.String())
t = time.Now()
// Publish and index receipts and txs
err = sdi.processReceiptsAndTxs(blockTx, processArgs{
headerHash: blockHashStr,
headerID: headerID,
blockNumber: block.Number(),
receipts: receipts,
@ -222,19 +287,19 @@ func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receip
rctLeafNodeCIDs: rctLeafNodeCIDs,
})
if err != nil {
return nil, err
return nil, 0, err
}
tDiff = time.Since(t)
indexerMetrics.tTxAndRecProcessing.Update(tDiff)
indexerMetrics.TimeTxAndRecProcessing.Update(tDiff)
traceMsg += fmt.Sprintf("tx and receipt processing time: %s\r\n", tDiff.String())
t = time.Now()
return blockTx, err
return blockTx, headerID, err
}
// processHeader publishes and indexes a header IPLD in Postgres
// it returns the headerID
func (sdi *StateDiffIndexer) processHeader(tx *BatchTx, header *types.Header, headerNode node.Node, reward, td *big.Int) (string, error) {
func (sdi *StateDiffIndexer) processHeader(tx *BatchTx, header *types.Header, headerNode node.Node, reward, td *big.Int) (int64, error) {
tx.cacheIPLD(headerNode)
var baseFee *string
@ -242,14 +307,33 @@ func (sdi *StateDiffIndexer) processHeader(tx *BatchTx, header *types.Header, he
baseFee = new(string)
*baseFee = header.BaseFee.String()
}
headerID := header.Hash().String()
mhKey := shared.MultihashKeyFromCID(headerNode.Cid())
// index header
return headerID, sdi.dbWriter.upsertHeaderCID(tx.dbtx, models.HeaderModel{
headerID, err := sdi.oldDBWriter.InsertHeaderCID(tx.oldDBTx, &v2Models.HeaderModel{
CID: headerNode.Cid().String(),
MhKey: shared.MultihashKeyFromCID(headerNode.Cid()),
MhKey: mhKey,
ParentHash: header.ParentHash.String(),
BlockNumber: header.Number.String(),
BlockHash: headerID,
BlockHash: header.Hash().String(),
TotalDifficulty: td.String(),
Reward: reward.String(),
Bloom: header.Bloom.Bytes(),
StateRoot: header.Root.String(),
RctRoot: header.ReceiptHash.String(),
TxRoot: header.TxHash.String(),
UncleRoot: header.UncleHash.String(),
Timestamp: header.Time,
BaseFee: baseFee,
})
if err != nil {
return 0, err
}
if err := sdi.newDBWriter.InsertHeaderCID(tx.newDBTx, &v3Models.HeaderModel{
CID: headerNode.Cid().String(),
MhKey: mhKey,
ParentHash: header.ParentHash.String(),
BlockNumber: header.Number.String(),
BlockHash: header.Hash().String(),
TotalDifficulty: td.String(),
Reward: reward.String(),
Bloom: header.Bloom.Bytes(),
@ -259,11 +343,14 @@ func (sdi *StateDiffIndexer) processHeader(tx *BatchTx, header *types.Header, he
UncleRoot: header.UncleHash.String(),
Timestamp: header.Time,
Coinbase: header.Coinbase.String(),
})
}); err != nil {
return 0, err
}
return headerID, nil
}
// processUncles publishes and indexes uncle IPLDs in Postgres
func (sdi *StateDiffIndexer) processUncles(tx *BatchTx, headerID string, blockNumber uint64, uncleNodes []*ipld2.EthHeader) error {
func (sdi *StateDiffIndexer) processUncles(tx *BatchTx, headerHash string, headerID int64, blockNumber uint64, uncleNodes []*ipld.EthHeader) error {
// publish and index uncles
for _, uncleNode := range uncleNodes {
tx.cacheIPLD(uncleNode)
@ -274,15 +361,25 @@ func (sdi *StateDiffIndexer) processUncles(tx *BatchTx, headerID string, blockNu
} else {
uncleReward = shared.CalcUncleMinerReward(blockNumber, uncleNode.Number.Uint64())
}
uncle := models.UncleModel{
mhKey := shared.MultihashKeyFromCID(uncleNode.Cid())
if err := sdi.oldDBWriter.InsertUncleCID(tx.oldDBTx, &v2Models.UncleModel{
HeaderID: headerID,
CID: uncleNode.Cid().String(),
MhKey: shared.MultihashKeyFromCID(uncleNode.Cid()),
MhKey: mhKey,
ParentHash: uncleNode.ParentHash.String(),
BlockHash: uncleNode.Hash().String(),
Reward: uncleReward.String(),
}); err != nil {
return err
}
if err := sdi.dbWriter.upsertUncleCID(tx.dbtx, uncle); err != nil {
if err := sdi.newDBWriter.InsertUncleCID(tx.newDBTx, &v3Models.UncleModel{
HeaderID: headerHash,
CID: uncleNode.Cid().String(),
MhKey: mhKey,
ParentHash: uncleNode.ParentHash.String(),
BlockHash: uncleNode.Hash().String(),
Reward: uncleReward.String(),
}); err != nil {
return err
}
}
@ -291,14 +388,15 @@ func (sdi *StateDiffIndexer) processUncles(tx *BatchTx, headerID string, blockNu
// processArgs bundles arguments to processReceiptsAndTxs
type processArgs struct {
headerID string
headerID int64
headerHash string
blockNumber *big.Int
receipts types.Receipts
txs types.Transactions
rctNodes []*ipld2.EthReceipt
rctTrieNodes []*ipld2.EthRctTrie
txNodes []*ipld2.EthTx
txTrieNodes []*ipld2.EthTxTrie
rctNodes []*ipld.EthReceipt
rctTrieNodes []*ipld.EthRctTrie
txNodes []*ipld.EthTx
txTrieNodes []*ipld.EthTxTrie
logTrieNodes [][]node.Node
logLeafNodeCIDs [][]cid.Cid
rctLeafNodeCIDs []cid.Cid
@ -317,7 +415,7 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *BatchTx, args processArgs
// index tx
trx := args.txs[i]
txID := trx.Hash().String()
txHash := trx.Hash().String()
var val string
if trx.Value() != nil {
@ -329,35 +427,57 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *BatchTx, args processArgs
if err != nil {
return fmt.Errorf("error deriving tx sender: %v", err)
}
txModel := models.TxModel{
mhKey := shared.MultihashKeyFromCID(txNode.Cid())
dst := shared.HandleZeroAddrPointer(trx.To())
src := shared.HandleZeroAddr(from)
txID, err := sdi.oldDBWriter.InsertTransactionCID(tx.oldDBTx, &v2Models.TxModel{
HeaderID: args.headerID,
Dst: shared.HandleZeroAddrPointer(trx.To()),
Src: shared.HandleZeroAddr(from),
TxHash: txID,
Dst: dst,
Src: src,
TxHash: txHash,
Index: int64(i),
Data: trx.Data(),
CID: txNode.Cid().String(),
MhKey: shared.MultihashKeyFromCID(txNode.Cid()),
MhKey: mhKey,
Type: trx.Type(),
Value: val,
}
if err := sdi.dbWriter.upsertTransactionCID(tx.dbtx, txModel); err != nil {
})
if err != nil {
return err
}
if err := sdi.newDBWriter.InsertTransactionCID(tx.newDBTx, &v3Models.TxModel{
HeaderID: args.headerHash,
Dst: dst,
Src: src,
TxHash: txHash,
Index: int64(i),
Data: trx.Data(),
CID: txNode.Cid().String(),
MhKey: mhKey,
Type: trx.Type(),
Value: val,
}); err != nil {
return err
}
// index access list if this is one
for j, accessListElement := range trx.AccessList() {
storageKeys := make([]string, len(accessListElement.StorageKeys))
for k, storageKey := range accessListElement.StorageKeys {
storageKeys[k] = storageKey.Hex()
}
accessListElementModel := models.AccessListElementModel{
if err := sdi.oldDBWriter.InsertAccessListElement(tx.oldDBTx, &v2Models.AccessListElementModel{
TxID: txID,
Index: int64(j),
Address: accessListElement.Address.Hex(),
StorageKeys: storageKeys,
}); err != nil {
return err
}
if err := sdi.dbWriter.upsertAccessListElement(tx.dbtx, accessListElementModel); err != nil {
if err := sdi.newDBWriter.InsertAccessListElement(tx.newDBTx, &v3Models.AccessListElementModel{
TxID: txHash,
Index: int64(j),
Address: accessListElement.Address.Hex(),
StorageKeys: storageKeys,
}); err != nil {
return err
}
}
@ -374,26 +494,45 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *BatchTx, args processArgs
return fmt.Errorf("invalid receipt leaf node cid")
}
rctModel := &models.ReceiptModel{
var postState string
var postStatus uint64
if len(receipt.PostState) == 0 {
postStatus = receipt.Status
} else {
postState = common.Bytes2Hex(receipt.PostState)
}
rctMhKey := shared.MultihashKeyFromCID(args.rctLeafNodeCIDs[i])
rctID, err := sdi.oldDBWriter.InsertReceiptCID(tx.oldDBTx, &v2Models.ReceiptModel{
TxID: txID,
Contract: contract,
ContractHash: contractHash,
LeafCID: args.rctLeafNodeCIDs[i].String(),
LeafMhKey: shared.MultihashKeyFromCID(args.rctLeafNodeCIDs[i]),
LeafMhKey: rctMhKey,
LogRoot: args.rctNodes[i].LogRoot.String(),
PostState: postState,
PostStatus: postStatus,
})
if err != nil {
return err
}
if len(receipt.PostState) == 0 {
rctModel.PostStatus = receipt.Status
} else {
rctModel.PostState = common.Bytes2Hex(receipt.PostState)
}
if err := sdi.dbWriter.upsertReceiptCID(tx.dbtx, rctModel); err != nil {
if err := sdi.newDBWriter.InsertReceiptCID(tx.newDBTx, &v3Models.ReceiptModel{
TxID: txHash,
Contract: contract,
ContractHash: contractHash,
LeafCID: args.rctLeafNodeCIDs[i].String(),
LeafMhKey: rctMhKey,
LogRoot: args.rctNodes[i].LogRoot.String(),
PostState: postState,
PostStatus: postStatus,
}); err != nil {
return err
}
// index logs
logDataSet := make([]*models.LogsModel, len(receipt.Logs))
rctLen := len(receipt.Logs)
oldLogDataSet := make([]*v2Models.LogsModel, rctLen)
newLogDataSet := make([]*v3Models.LogsModel, rctLen)
for idx, l := range receipt.Logs {
topicSet := make([]string, 4)
for ti, topic := range l.Topics {
@ -404,21 +543,36 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *BatchTx, args processArgs
return fmt.Errorf("invalid log cid")
}
logDataSet[idx] = &models.LogsModel{
ReceiptID: txID,
logMhKey := shared.MultihashKeyFromCID(args.logLeafNodeCIDs[i][idx])
oldLogDataSet[idx] = &v2Models.LogsModel{
ReceiptID: rctID,
Address: l.Address.String(),
Index: int64(l.Index),
Data: l.Data,
LeafCID: args.logLeafNodeCIDs[i][idx].String(),
LeafMhKey: shared.MultihashKeyFromCID(args.logLeafNodeCIDs[i][idx]),
LeafMhKey: logMhKey,
Topic0: topicSet[0],
Topic1: topicSet[1],
Topic2: topicSet[2],
Topic3: topicSet[3],
}
newLogDataSet[idx] = &v3Models.LogsModel{
ReceiptID: txHash,
Address: l.Address.String(),
Index: int64(l.Index),
Data: l.Data,
LeafCID: args.logLeafNodeCIDs[i][idx].String(),
LeafMhKey: logMhKey,
Topic0: topicSet[0],
Topic1: topicSet[1],
Topic2: topicSet[2],
Topic3: topicSet[3],
}
}
if err := sdi.dbWriter.upsertLogCID(tx.dbtx, logDataSet); err != nil {
if err := sdi.oldDBWriter.InsertLogCID(tx.oldDBTx, oldLogDataSet); err != nil {
return err
}
if err := sdi.newDBWriter.InsertLogCID(tx.newDBTx, newLogDataSet); err != nil {
return err
}
}
@ -433,7 +587,7 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *BatchTx, args processArgs
}
// PushStateNode publishes and indexes a state diff node object (including any child storage nodes) in the IPLD sql
func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdtypes.StateNode, headerID string) error {
func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdtypes.StateNode, headerHash string, headerID int64) error {
tx, ok := batch.(*BatchTx)
if !ok {
return fmt.Errorf("sql batch is expected to be of type %T, got %T", &BatchTx{}, batch)
@ -442,30 +596,50 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt
if stateNode.NodeType == sdtypes.Removed {
// short circuit if it is a Removed node
// this assumes the db has been initialized and a public.blocks entry for the Removed node is present
stateModel := models.StateNodeModel{
_, err := sdi.oldDBWriter.InsertStateCID(tx.oldDBTx, &v2Models.StateNodeModel{
HeaderID: headerID,
Path: stateNode.Path,
StateKey: common.BytesToHash(stateNode.LeafKey).String(),
CID: shared.RemovedNodeStateCID,
MhKey: shared.RemovedNodeMhKey,
NodeType: stateNode.NodeType.Int(),
})
if err != nil {
return err
}
return sdi.dbWriter.upsertStateCID(tx.dbtx, stateModel)
return sdi.newDBWriter.InsertStateCID(tx.newDBTx, &v3Models.StateNodeModel{
HeaderID: headerHash,
Path: stateNode.Path,
StateKey: common.BytesToHash(stateNode.LeafKey).String(),
CID: shared.RemovedNodeStateCID,
MhKey: shared.RemovedNodeMhKey,
NodeType: stateNode.NodeType.Int(),
})
}
stateCIDStr, stateMhKey, err := tx.cacheRaw(ipld2.MEthStateTrie, multihash.KECCAK_256, stateNode.NodeValue)
stateCIDStr, stateMhKey, err := tx.cacheRaw(ipld.MEthStateTrie, multihash.KECCAK_256, stateNode.NodeValue)
if err != nil {
return fmt.Errorf("error generating and cacheing state node IPLD: %v", err)
}
stateModel := models.StateNodeModel{
// index the state node
stateID, err := sdi.oldDBWriter.InsertStateCID(tx.oldDBTx, &v2Models.StateNodeModel{
HeaderID: headerID,
Path: stateNode.Path,
StateKey: common.BytesToHash(stateNode.LeafKey).String(),
CID: stateCIDStr,
MhKey: stateMhKey,
NodeType: stateNode.NodeType.Int(),
})
if err != nil {
return err
}
// index the state node
if err := sdi.dbWriter.upsertStateCID(tx.dbtx, stateModel); err != nil {
if err := sdi.newDBWriter.InsertStateCID(tx.newDBTx, &v3Models.StateNodeModel{
HeaderID: headerHash,
Path: stateNode.Path,
StateKey: common.BytesToHash(stateNode.LeafKey).String(),
CID: stateCIDStr,
MhKey: stateMhKey,
NodeType: stateNode.NodeType.Int(),
}); err != nil {
return err
}
// if we have a leaf, decode and index the account data
@ -481,15 +655,23 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt
if err := rlp.DecodeBytes(i[1].([]byte), &account); err != nil {
return fmt.Errorf("error decoding state account rlp: %s", err.Error())
}
accountModel := models.StateAccountModel{
HeaderID: headerID,
if err := sdi.oldDBWriter.InsertStateAccount(tx.oldDBTx, &v2Models.StateAccountModel{
StateID: stateID,
Balance: account.Balance.String(),
Nonce: account.Nonce,
CodeHash: account.CodeHash,
StorageRoot: account.Root.String(),
}); err != nil {
return err
}
if err := sdi.newDBWriter.InsertStateAccount(tx.newDBTx, &v3Models.StateAccountModel{
HeaderID: headerHash,
StatePath: stateNode.Path,
Balance: account.Balance.String(),
Nonce: account.Nonce,
CodeHash: account.CodeHash,
StorageRoot: account.Root.String(),
}
if err := sdi.dbWriter.upsertStateAccount(tx.dbtx, accountModel); err != nil {
}); err != nil {
return err
}
}
@ -498,34 +680,52 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt
if storageNode.NodeType == sdtypes.Removed {
// short circuit if it is a Removed node
// this assumes the db has been initialized and a public.blocks entry for the Removed node is present
storageModel := models.StorageNodeModel{
HeaderID: headerID,
if err := sdi.oldDBWriter.InsertStorageCID(tx.oldDBTx, &v2Models.StorageNodeModel{
StateID: stateID,
Path: storageNode.Path,
StorageKey: common.BytesToHash(storageNode.LeafKey).String(),
CID: shared.RemovedNodeStorageCID,
MhKey: shared.RemovedNodeMhKey,
NodeType: storageNode.NodeType.Int(),
}); err != nil {
return err
}
if err := sdi.newDBWriter.InsertStorageCID(tx.newDBTx, &v3Models.StorageNodeModel{
HeaderID: headerHash,
StatePath: stateNode.Path,
Path: storageNode.Path,
StorageKey: common.BytesToHash(storageNode.LeafKey).String(),
CID: shared.RemovedNodeStorageCID,
MhKey: shared.RemovedNodeMhKey,
NodeType: storageNode.NodeType.Int(),
}
if err := sdi.dbWriter.upsertStorageCID(tx.dbtx, storageModel); err != nil {
}); err != nil {
return err
}
continue
}
storageCIDStr, storageMhKey, err := tx.cacheRaw(ipld2.MEthStorageTrie, multihash.KECCAK_256, storageNode.NodeValue)
storageCIDStr, storageMhKey, err := tx.cacheRaw(ipld.MEthStorageTrie, multihash.KECCAK_256, storageNode.NodeValue)
if err != nil {
return fmt.Errorf("error generating and cacheing storage node IPLD: %v", err)
}
storageModel := models.StorageNodeModel{
HeaderID: headerID,
if err := sdi.oldDBWriter.InsertStorageCID(tx.oldDBTx, &v2Models.StorageNodeModel{
StateID: stateID,
Path: storageNode.Path,
StorageKey: common.BytesToHash(storageNode.LeafKey).String(),
CID: storageCIDStr,
MhKey: storageMhKey,
NodeType: storageNode.NodeType.Int(),
}); err != nil {
return err
}
if err := sdi.newDBWriter.InsertStorageCID(tx.newDBTx, &v3Models.StorageNodeModel{
HeaderID: headerHash,
StatePath: stateNode.Path,
Path: storageNode.Path,
StorageKey: common.BytesToHash(storageNode.LeafKey).String(),
CID: storageCIDStr,
MhKey: storageMhKey,
NodeType: storageNode.NodeType.Int(),
}
if err := sdi.dbWriter.upsertStorageCID(tx.dbtx, storageModel); err != nil {
}); err != nil {
return err
}
}
@ -550,5 +750,8 @@ func (sdi *StateDiffIndexer) PushCodeAndCodeHash(batch interfaces.Batch, codeAnd
// Close satisfies io.Closer
func (sdi *StateDiffIndexer) Close() error {
return sdi.dbWriter.Close()
if err := sdi.oldDBWriter.Close(); err != nil {
return err
}
return sdi.newDBWriter.Close()
}

3
statediff/indexer/database/sql/indexer_shared_test.go
View File

@ -12,14 +12,13 @@ import (
"github.com/multiformats/go-multihash"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
"github.com/ethereum/go-ethereum/statediff/indexer/mocks"
)
var (
db sql.Database
db interfaces.Database
err error
ind interfaces.StateDiffIndexer
ipfsPgGet = `SELECT data FROM public.blocks

87
statediff/indexer/database/sql/interfaces.go
View File

@ -1,87 +0,0 @@
// VulcanizeDB
// Copyright © 2021 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package sql
import (
"context"
"io"
"time"
)
// Database interfaces required by the sql indexer
type Database interface {
Driver
Statements
}
// Driver interface has all the methods required by a driver implementation to support the sql indexer
type Driver interface {
QueryRow(ctx context.Context, sql string, args ...interface{}) ScannableRow
Exec(ctx context.Context, sql string, args ...interface{}) (Result, error)
Select(ctx context.Context, dest interface{}, query string, args ...interface{}) error
Get(ctx context.Context, dest interface{}, query string, args ...interface{}) error
Begin(ctx context.Context) (Tx, error)
Stats() Stats
NodeID() string
Context() context.Context
io.Closer
}
// Statements interface to accommodate different SQL query syntax
type Statements interface {
InsertHeaderStm() string
InsertUncleStm() string
InsertTxStm() string
InsertAccessListElementStm() string
InsertRctStm() string
InsertLogStm() string
InsertStateStm() string
InsertAccountStm() string
InsertStorageStm() string
InsertIPLDStm() string
InsertIPLDsStm() string
}
// Tx interface to accommodate different concrete SQL transaction types
type Tx interface {
QueryRow(ctx context.Context, sql string, args ...interface{}) ScannableRow
Exec(ctx context.Context, sql string, args ...interface{}) (Result, error)
Commit(ctx context.Context) error
Rollback(ctx context.Context) error
}
// ScannableRow interface to accommodate different concrete row types
type ScannableRow interface {
Scan(dest ...interface{}) error
}
// Result interface to accommodate different concrete result types
type Result interface {
RowsAffected() (int64, error)
}
// Stats interface to accommodate different concrete sql stats types
type Stats interface {
MaxOpen() int64
Open() int64
InUse() int64
Idle() int64
WaitCount() int64
WaitDuration() time.Duration
MaxIdleClosed() int64
MaxLifetimeClosed() int64
}

17
statediff/indexer/database/sql/mainnet_tests/indexer_test.go
View File

@ -23,6 +23,8 @@ import (
"os"
"testing"
nodeinfo "github.com/ethereum/go-ethereum/statediff/indexer/node"
"github.com/stretchr/testify/require"
"github.com/ethereum/go-ethereum/core/types"
@ -36,7 +38,7 @@ import (
var (
err error
db sql.Database
db interfaces.Database
ind interfaces.StateDiffIndexer
chainConf = params.MainnetChainConfig
)
@ -76,14 +78,19 @@ func testPushBlockAndState(t *testing.T, block *types.Block, receipts types.Rece
}
func setup(t *testing.T, testBlock *types.Block, testReceipts types.Receipts) {
db, err = postgres.SetupSQLXDB()
db, err = postgres.SetupV3SQLXDB()
if err != nil {
t.Fatal(err)
}
ind, err = sql.NewStateDiffIndexer(context.Background(), chainConf, db)
dbV2, err := postgres.SetupV2SQLXDB()
if err != nil {
t.Fatal(err)
}
ind, err = sql.NewStateDiffIndexer(context.Background(), chainConf, nodeinfo.Info{}, dbV2, db)
require.NoError(t, err)
var tx interfaces.Batch
tx, err = ind.PushBlock(
var headerID int64
tx, headerID, err = ind.PushBlock(
testBlock,
testReceipts,
testBlock.Difficulty())
@ -95,7 +102,7 @@ func setup(t *testing.T, testBlock *types.Block, testReceipts types.Receipts) {
}
}()
for _, node := range mocks.StateDiffs {
err = ind.PushStateNode(tx, node, testBlock.Hash().String())
err = ind.PushStateNode(tx, node, testBlock.Hash().String(), headerID)
require.NoError(t, err)
}

100
statediff/indexer/database/sql/metrics.go → statediff/indexer/database/sql/metrics/metrics.go
View File

@ -14,11 +14,13 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package sql
package metrics
import (
"strings"
"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
"github.com/ethereum/go-ethereum/metrics"
)
@ -39,57 +41,67 @@ func metricName(subsystem, name string) string {
return strings.Join(parts, "/")
}
type indexerMetricsHandles struct {
type WriterMetricsHandles struct {
// The total number of processed blocks
blocks metrics.Counter
Blocks metrics.Counter
// The total number of processed transactions
transactions metrics.Counter
Transactions metrics.Counter
// The total number of processed receipts
receipts metrics.Counter
Receipts metrics.Counter
// The total number of processed logs
logs metrics.Counter
Logs metrics.Counter
// The total number of access list entries processed
accessListEntries metrics.Counter
// Time spent waiting for free postgres tx
tFreePostgres metrics.Timer
// Postgres transaction commit duration
tPostgresCommit metrics.Timer
// Header processing time
tHeaderProcessing metrics.Timer
// Uncle processing time
tUncleProcessing metrics.Timer
// Tx and receipt processing time
tTxAndRecProcessing metrics.Timer
// State, storage, and code combined processing time
tStateStoreCodeProcessing metrics.Timer
AccessListEntries metrics.Counter
}
func RegisterIndexerMetrics(reg metrics.Registry) indexerMetricsHandles {
ctx := indexerMetricsHandles{
blocks: metrics.NewCounter(),
transactions: metrics.NewCounter(),
receipts: metrics.NewCounter(),
logs: metrics.NewCounter(),
accessListEntries: metrics.NewCounter(),
tFreePostgres: metrics.NewTimer(),
tPostgresCommit: metrics.NewTimer(),
tHeaderProcessing: metrics.NewTimer(),
tUncleProcessing: metrics.NewTimer(),
tTxAndRecProcessing: metrics.NewTimer(),
tStateStoreCodeProcessing: metrics.NewTimer(),
type IndexerMetricsHandles struct {
// Time spent waiting for free postgres tx
TimeFreePostgres metrics.Timer
// Postgres transaction commit duration
TimePostgresCommit metrics.Timer
// Header processing time
TimeHeaderProcessing metrics.Timer
// Uncle processing time
TimeUncleProcessing metrics.Timer
// Tx and receipt processing time
TimeTxAndRecProcessing metrics.Timer
// State, storage, and code combined processing time
TimeStateStoreCodeProcessing metrics.Timer
}
func RegisterWriterMetrics(reg metrics.Registry, version string) WriterMetricsHandles {
ctx := WriterMetricsHandles{
Blocks: metrics.NewCounter(),
Transactions: metrics.NewCounter(),
Receipts: metrics.NewCounter(),
Logs: metrics.NewCounter(),
AccessListEntries: metrics.NewCounter(),
}
subsys := "writer"
reg.Register(metricName(subsys, version+"/"+"blocks"), ctx.Blocks)
reg.Register(metricName(subsys, version+"/"+"transactions"), ctx.Transactions)
reg.Register(metricName(subsys, version+"/"+"receipts"), ctx.Receipts)
reg.Register(metricName(subsys, version+"/"+"logs"), ctx.Logs)
reg.Register(metricName(subsys, version+"/"+"access_list_entries"), ctx.AccessListEntries)
return ctx
}
func RegisterIndexerMetrics(reg metrics.Registry) IndexerMetricsHandles {
ctx := IndexerMetricsHandles{
TimeFreePostgres: metrics.NewTimer(),
TimePostgresCommit: metrics.NewTimer(),
TimeHeaderProcessing: metrics.NewTimer(),
TimeUncleProcessing: metrics.NewTimer(),
TimeTxAndRecProcessing: metrics.NewTimer(),
TimeStateStoreCodeProcessing: metrics.NewTimer(),
}
subsys := "indexer"
reg.Register(metricName(subsys, "blocks"), ctx.blocks)
reg.Register(metricName(subsys, "transactions"), ctx.transactions)
reg.Register(metricName(subsys, "receipts"), ctx.receipts)
reg.Register(metricName(subsys, "logs"), ctx.logs)
reg.Register(metricName(subsys, "access_list_entries"), ctx.accessListEntries)
reg.Register(metricName(subsys, "t_free_postgres"), ctx.tFreePostgres)
reg.Register(metricName(subsys, "t_postgres_commit"), ctx.tPostgresCommit)
reg.Register(metricName(subsys, "t_header_processing"), ctx.tHeaderProcessing)
reg.Register(metricName(subsys, "t_uncle_processing"), ctx.tUncleProcessing)
reg.Register(metricName(subsys, "t_tx_receipt_processing"), ctx.tTxAndRecProcessing)
reg.Register(metricName(subsys, "t_state_store_code_processing"), ctx.tStateStoreCodeProcessing)
reg.Register(metricName(subsys, "t_free_postgres"), ctx.TimeFreePostgres)
reg.Register(metricName(subsys, "t_postgres_commit"), ctx.TimePostgresCommit)
reg.Register(metricName(subsys, "t_header_processing"), ctx.TimeHeaderProcessing)
reg.Register(metricName(subsys, "t_uncle_processing"), ctx.TimeUncleProcessing)
reg.Register(metricName(subsys, "t_tx_receipt_processing"), ctx.TimeTxAndRecProcessing)
reg.Register(metricName(subsys, "t_state_store_code_processing"), ctx.TimeStateStoreCodeProcessing)
return ctx
}
@ -135,7 +147,7 @@ func RegisterDBMetrics(reg metrics.Registry) dbMetricsHandles {
return ctx
}
func (met *dbMetricsHandles) Update(stats Stats) {
func (met *dbMetricsHandles) Update(stats interfaces.Stats) {
met.maxOpen.Update(stats.MaxOpen())
met.open.Update(stats.Open())
met.inUse.Update(stats.InUse())

13
statediff/indexer/database/sql/pgx_indexer_legacy_test.go
View File

@ -20,6 +20,8 @@ import (
"context"
"testing"
nodeinfo "github.com/ethereum/go-ethereum/statediff/indexer/node"
"github.com/multiformats/go-multihash"
"github.com/stretchr/testify/require"
@ -34,13 +36,16 @@ func setupLegacyPGX(t *testing.T) {
mockLegacyBlock = legacyData.MockBlock
legacyHeaderCID, _ = ipld.RawdataToCid(ipld.MEthHeader, legacyData.MockHeaderRlp, multihash.KECCAK_256)
db, err = postgres.SetupPGXDB()
db, err = postgres.SetupV3PGXDB()
require.NoError(t, err)
ind, err = sql.NewStateDiffIndexer(context.Background(), legacyData.Config, db)
v2DB, err := postgres.SetupV2PGXDB()
require.NoError(t, err)
ind, err = sql.NewStateDiffIndexer(context.Background(), legacyData.Config, nodeinfo.Info{}, v2DB, db)
require.NoError(t, err)
var tx interfaces.Batch
tx, err = ind.PushBlock(
tx, headerID, err = ind.PushBlock(
mockLegacyBlock,
legacyData.MockReceipts,
legacyData.MockBlock.Difficulty())
@ -52,7 +57,7 @@ func setupLegacyPGX(t *testing.T) {
}
}()
for _, node := range legacyData.StateDiffs {
err = ind.PushStateNode(tx, node, legacyData.MockBlock.Hash().String())
err = ind.PushStateNode(tx, node, legacyData.MockBlock.Hash().String(), headerID)
require.NoError(t, err)
}

114
statediff/indexer/database/sql/pgx_indexer_test.go
View File

@ -32,33 +32,35 @@ import (
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
"github.com/ethereum/go-ethereum/statediff/indexer/mocks"
"github.com/ethereum/go-ethereum/statediff/indexer/models"
sharedModels "github.com/ethereum/go-ethereum/statediff/indexer/models/shared"
v3Models "github.com/ethereum/go-ethereum/statediff/indexer/models/v3"
nodeInfo "github.com/ethereum/go-ethereum/statediff/indexer/node"
"github.com/ethereum/go-ethereum/statediff/indexer/shared"
"github.com/ethereum/go-ethereum/statediff/indexer/test_helpers"
)
func setupPGX(t *testing.T) {
db, err = postgres.SetupPGXDB()
if err != nil {
t.Fatal(err)
}
ind, err = sql.NewStateDiffIndexer(context.Background(), mocks.TestConfig, db)
db, err = postgres.SetupV3PGXDB()
require.NoError(t, err)
v2DB, err := postgres.SetupV2PGXDB()
require.NoError(t, err)
ind, err = sql.NewStateDiffIndexer(context.Background(), mocks.TestConfig, nodeInfo.Info{}, v2DB, db)
require.NoError(t, err)
var tx interfaces.Batch
tx, err = ind.PushBlock(
tx, headerID, err = ind.PushBlock(
mockBlock,
mocks.MockReceipts,
mocks.MockBlock.Difficulty())
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
defer func() {
if err := tx.Submit(err); err != nil {
t.Fatal(err)
}
}()
for _, node := range mocks.StateDiffs {
err = ind.PushStateNode(tx, node, mockBlock.Hash().String())
err = ind.PushStateNode(tx, node, mockBlock.Hash().String(), headerID)
require.NoError(t, err)
}
@ -87,24 +89,18 @@ func TestPGXIndexer(t *testing.T) {
&header.Reward,
&header.BlockHash,
&header.Coinbase)
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
test_helpers.ExpectEqual(t, header.CID, headerCID.String())
test_helpers.ExpectEqual(t, header.TD, mocks.MockBlock.Difficulty().String())
test_helpers.ExpectEqual(t, header.Reward, "2000000000000021250")
test_helpers.ExpectEqual(t, header.Coinbase, mocks.MockHeader.Coinbase.String())
dc, err := cid.Decode(header.CID)
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
mhKey := dshelp.MultihashToDsKey(dc.Hash())
prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
var data []byte
err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey)
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
test_helpers.ExpectEqual(t, data, mocks.MockHeaderRlp)
})
@ -116,9 +112,7 @@ func TestPGXIndexer(t *testing.T) {
pgStr := `SELECT transaction_cids.cid FROM eth.transaction_cids INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.block_hash)
WHERE header_cids.block_number = $1`
err = db.Select(context.Background(), &trxs, pgStr, mocks.BlockNumber.Uint64())
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
test_helpers.ExpectEqual(t, len(trxs), 5)
expectTrue(t, test_helpers.ListContainsString(trxs, trx1CID.String()))
expectTrue(t, test_helpers.ListContainsString(trxs, trx2CID.String()))
@ -197,7 +191,7 @@ func TestPGXIndexer(t *testing.T) {
if txRes.Value != transactions[3].Value().String() {
t.Fatalf("expected tx value %s got %s", transactions[3].Value().String(), txRes.Value)
}
accessListElementModels := make([]models.AccessListElementModel, 0)
accessListElementModels := make([]v3Models.AccessListElementModel, 0)
pgStr = `SELECT access_list_elements.* FROM eth.access_list_elements INNER JOIN eth.transaction_cids ON (tx_id = transaction_cids.tx_hash) WHERE cid = $1 ORDER BY access_list_elements.index ASC`
err = db.Select(context.Background(), &accessListElementModels, pgStr, c)
if err != nil {
@ -206,11 +200,11 @@ func TestPGXIndexer(t *testing.T) {
if len(accessListElementModels) != 2 {
t.Fatalf("expected two access list entries, got %d", len(accessListElementModels))
}
model1 := models.AccessListElementModel{
model1 := v3Models.AccessListElementModel{
Index: accessListElementModels[0].Index,
Address: accessListElementModels[0].Address,
}
model2 := models.AccessListElementModel{
model2 := v3Models.AccessListElementModel{
Index: accessListElementModels[1].Index,
Address: accessListElementModels[1].Address,
StorageKeys: accessListElementModels[1].StorageKeys,
@ -249,9 +243,7 @@ func TestPGXIndexer(t *testing.T) {
INNER JOIN public.blocks ON (log_cids.leaf_mh_key = blocks.key)
WHERE receipt_cids.leaf_cid = $1 ORDER BY eth.log_cids.index ASC`
err = db.Select(context.Background(), &rcts, rctsPgStr, mocks.BlockNumber.Uint64())
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
if len(rcts) != len(mocks.MockReceipts) {
t.Fatalf("expected %d receipts, got %d", len(mocks.MockReceipts), len(rcts))
}
@ -302,9 +294,7 @@ func TestPGXIndexer(t *testing.T) {
AND transaction_cids.header_id = header_cids.block_hash
AND header_cids.block_number = $1 order by transaction_cids.index`
err = db.Select(context.Background(), &rcts, pgStr, mocks.BlockNumber.Uint64())
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
test_helpers.ExpectEqual(t, len(rcts), 5)
expectTrue(t, test_helpers.ListContainsString(rcts, rct1CID.String()))
expectTrue(t, test_helpers.ListContainsString(rcts, rct2CID.String()))
@ -313,7 +303,7 @@ func TestPGXIndexer(t *testing.T) {
expectTrue(t, test_helpers.ListContainsString(rcts, rct5CID.String()))
for idx, c := range rcts {
result := make([]models.IPLDModel, 0)
result := make([]sharedModels.IPLDModel, 0)
pgStr = `SELECT data
FROM eth.receipt_cids
INNER JOIN public.blocks ON (receipt_cids.leaf_mh_key = public.blocks.key)
@ -396,14 +386,12 @@ func TestPGXIndexer(t *testing.T) {
setupPGX(t)
defer tearDown(t)
// check that state nodes were properly indexed and published
stateNodes := make([]models.StateNodeModel, 0)
stateNodes := make([]v3Models.StateNodeModel, 0)
pgStr := `SELECT state_cids.cid, state_cids.state_leaf_key, state_cids.node_type, state_cids.state_path, state_cids.header_id
FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.block_hash)
WHERE header_cids.block_number = $1 AND node_type != 3`
err = db.Select(context.Background(), &stateNodes, pgStr, mocks.BlockNumber.Uint64())
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
test_helpers.ExpectEqual(t, len(stateNodes), 2)
for _, stateNode := range stateNodes {
var data []byte
@ -418,7 +406,7 @@ func TestPGXIndexer(t *testing.T) {
t.Fatal(err)
}
pgStr = `SELECT header_id, state_path, cast(balance AS TEXT), nonce, code_hash, storage_root from eth.state_accounts WHERE header_id = $1 AND state_path = $2`
var account models.StateAccountModel
var account v3Models.StateAccountModel
err = db.Get(context.Background(), &account, pgStr, stateNode.HeaderID, stateNode.Path)
if err != nil {
t.Fatal(err)
@ -428,7 +416,7 @@ func TestPGXIndexer(t *testing.T) {
test_helpers.ExpectEqual(t, stateNode.StateKey, common.BytesToHash(mocks.ContractLeafKey).Hex())
test_helpers.ExpectEqual(t, stateNode.Path, []byte{'\x06'})
test_helpers.ExpectEqual(t, data, mocks.ContractLeafNode)
test_helpers.ExpectEqual(t, account, models.StateAccountModel{
test_helpers.ExpectEqual(t, account, v3Models.StateAccountModel{
HeaderID: account.HeaderID,
StatePath: stateNode.Path,
Balance: "0",
@ -442,7 +430,7 @@ func TestPGXIndexer(t *testing.T) {
test_helpers.ExpectEqual(t, stateNode.StateKey, common.BytesToHash(mocks.AccountLeafKey).Hex())
test_helpers.ExpectEqual(t, stateNode.Path, []byte{'\x0c'})
test_helpers.ExpectEqual(t, data, mocks.AccountLeafNode)
test_helpers.ExpectEqual(t, account, models.StateAccountModel{
test_helpers.ExpectEqual(t, account, v3Models.StateAccountModel{
HeaderID: account.HeaderID,
StatePath: stateNode.Path,
Balance: "1000",
@ -454,28 +442,22 @@ func TestPGXIndexer(t *testing.T) {
}
// check that Removed state nodes were properly indexed and published
stateNodes = make([]models.StateNodeModel, 0)
stateNodes = make([]v3Models.StateNodeModel, 0)
pgStr = `SELECT state_cids.cid, state_cids.state_leaf_key, state_cids.node_type, state_cids.state_path, state_cids.header_id
FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.block_hash)
WHERE header_cids.block_number = $1 AND node_type = 3`
err = db.Select(context.Background(), &stateNodes, pgStr, mocks.BlockNumber.Uint64())
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
test_helpers.ExpectEqual(t, len(stateNodes), 1)
stateNode := stateNodes[0]
var data []byte
dc, err := cid.Decode(stateNode.CID)
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
mhKey := dshelp.MultihashToDsKey(dc.Hash())
prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
test_helpers.ExpectEqual(t, prefixedKey, shared.RemovedNodeMhKey)
err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey)
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
test_helpers.ExpectEqual(t, stateNode.CID, shared.RemovedNodeStateCID)
test_helpers.ExpectEqual(t, stateNode.Path, []byte{'\x02'})
test_helpers.ExpectEqual(t, data, []byte{})
@ -485,7 +467,7 @@ func TestPGXIndexer(t *testing.T) {
setupPGX(t)
defer tearDown(t)
// check that storage nodes were properly indexed
storageNodes := make([]models.StorageNodeWithStateKeyModel, 0)
storageNodes := make([]v3Models.StorageNodeWithStateKeyModel, 0)
pgStr := `SELECT storage_cids.cid, state_cids.state_leaf_key, storage_cids.storage_leaf_key, storage_cids.node_type, storage_cids.storage_path
FROM eth.storage_cids, eth.state_cids, eth.header_cids
WHERE (storage_cids.state_path, storage_cids.header_id) = (state_cids.state_path, state_cids.header_id)
@ -493,11 +475,9 @@ func TestPGXIndexer(t *testing.T) {
AND header_cids.block_number = $1
AND storage_cids.node_type != 3`
err = db.Select(context.Background(), &storageNodes, pgStr, mocks.BlockNumber.Uint64())
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
test_helpers.ExpectEqual(t, len(storageNodes), 1)
test_helpers.ExpectEqual(t, storageNodes[0], models.StorageNodeWithStateKeyModel{
test_helpers.ExpectEqual(t, storageNodes[0], v3Models.StorageNodeWithStateKeyModel{
CID: storageCID.String(),
NodeType: 2,
StorageKey: common.BytesToHash(mocks.StorageLeafKey).Hex(),
@ -506,19 +486,15 @@ func TestPGXIndexer(t *testing.T) {
})
var data []byte
dc, err := cid.Decode(storageNodes[0].CID)
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
mhKey := dshelp.MultihashToDsKey(dc.Hash())
prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey)
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
test_helpers.ExpectEqual(t, data, mocks.StorageLeafNode)
// check that Removed storage nodes were properly indexed
storageNodes = make([]models.StorageNodeWithStateKeyModel, 0)
storageNodes = make([]v3Models.StorageNodeWithStateKeyModel, 0)
pgStr = `SELECT storage_cids.cid, state_cids.state_leaf_key, storage_cids.storage_leaf_key, storage_cids.node_type, storage_cids.storage_path
FROM eth.storage_cids, eth.state_cids, eth.header_cids
WHERE (storage_cids.state_path, storage_cids.header_id) = (state_cids.state_path, state_cids.header_id)
@ -526,11 +502,9 @@ func TestPGXIndexer(t *testing.T) {
AND header_cids.block_number = $1
AND storage_cids.node_type = 3`
err = db.Select(context.Background(), &storageNodes, pgStr, mocks.BlockNumber.Uint64())
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
test_helpers.ExpectEqual(t, len(storageNodes), 1)
test_helpers.ExpectEqual(t, storageNodes[0], models.StorageNodeWithStateKeyModel{
test_helpers.ExpectEqual(t, storageNodes[0], v3Models.StorageNodeWithStateKeyModel{
CID: shared.RemovedNodeStorageCID,
NodeType: 3,
StorageKey: common.BytesToHash(mocks.RemovedLeafKey).Hex(),
@ -538,16 +512,12 @@ func TestPGXIndexer(t *testing.T) {
Path: []byte{'\x03'},
})
dc, err = cid.Decode(storageNodes[0].CID)
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
mhKey = dshelp.MultihashToDsKey(dc.Hash())
prefixedKey = blockstore.BlockPrefix.String() + mhKey.String()
test_helpers.ExpectEqual(t, prefixedKey, shared.RemovedNodeMhKey)
err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey)
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
test_helpers.ExpectEqual(t, data, []byte{})
})
}

31
statediff/indexer/database/sql/postgres/config.go
View File

@ -45,7 +45,7 @@ func ResolveDriverType(str string) (DriverType, error) {
}
}
// DefaultConfig are default parameters for connecting to a Postgres sql
// DefaultConfig are default parameters for connecting to a Postgres DB
var DefaultConfig = Config{
Hostname: "localhost",
Port: 5432,
@ -54,6 +54,35 @@ var DefaultConfig = Config{
Password: "password",
}
// DefaultV2Config are default parameters for connecting to a v3 Postgres DB
var DefaultV2Config = Config{
Hostname: "localhost",
Port: 5433,
DatabaseName: "vulcanize_testing_v2",
Username: "vdbm",
Password: "password",
}
// DefaultV3Config are default parameters for connecting to a v3 Postgres DB
var DefaultV3Config = Config{
Hostname: "localhost",
Port: 5432,
DatabaseName: "vulcanize_testing_v3",
Username: "vdbm",
Password: "password",
}
// MultiConfig holds multiple configs
type MultiConfig struct {
V2 Config
V3 Config
}
// Type satisfies interfaces.Config
func (mc MultiConfig) Type() shared.DBType {
return shared.POSTGRES
}
// Config holds params for a Postgres db
type Config struct {
// conn string params

5
statediff/indexer/database/sql/postgres/errors.go
View File

@ -22,17 +22,12 @@ import (
const (
DbConnectionFailedMsg = "db connection failed"
SettingNodeFailedMsg = "unable to set db node"
)
func ErrDBConnectionFailed(connectErr error) error {
return formatError(DbConnectionFailedMsg, connectErr.Error())
}
func ErrUnableToSetNode(setErr error) error {
return formatError(SettingNodeFailedMsg, setErr.Error())
}
func formatError(msg, err string) error {
return fmt.Errorf("%s: %s", msg, err)
}

50
statediff/indexer/database/sql/postgres/pgx.go
View File

@ -25,21 +25,20 @@ import (
"github.com/jackc/pgx/v4"
"github.com/jackc/pgx/v4/pgxpool"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
"github.com/ethereum/go-ethereum/statediff/indexer/node"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
)
// PGXDriver driver, implements sql.Driver
type PGXDriver struct {
ctx context.Context
pool *pgxpool.Pool
nodeInfo node.Info
nodeID string
ctx context.Context
pool *pgxpool.Pool
}
// NewPGXDriver returns a new pgx driver
// it initializes the connection pool and creates the node info table
func NewPGXDriver(ctx context.Context, config Config, node node.Info) (*PGXDriver, error) {
func NewPGXDriver(ctx context.Context, config Config) (*PGXDriver, error) {
log.Info("connecting to database", "connection string", config.DbConnectionString())
pgConf, err := MakeConfig(config)
if err != nil {
return nil, err
@ -48,11 +47,7 @@ func NewPGXDriver(ctx context.Context, config Config, node node.Info) (*PGXDrive
if err != nil {
return nil, ErrDBConnectionFailed(err)
}
pg := &PGXDriver{ctx: ctx, pool: dbPool, nodeInfo: node}
nodeErr := pg.createNode()
if nodeErr != nil {
return &PGXDriver{}, ErrUnableToSetNode(nodeErr)
}
pg := &PGXDriver{ctx: ctx, pool: dbPool}
return pg, nil
}
@ -88,27 +83,13 @@ func MakeConfig(config Config) (*pgxpool.Config, error) {
return conf, nil
}
func (pgx *PGXDriver) createNode() error {
_, err := pgx.pool.Exec(
pgx.ctx,
createNodeStm,
pgx.nodeInfo.GenesisBlock, pgx.nodeInfo.NetworkID,
pgx.nodeInfo.ID, pgx.nodeInfo.ClientName,
pgx.nodeInfo.ChainID)
if err != nil {
return ErrUnableToSetNode(err)
}
pgx.nodeID = pgx.nodeInfo.ID
return nil
}
// QueryRow satisfies sql.Database
func (pgx *PGXDriver) QueryRow(ctx context.Context, sql string, args ...interface{}) sql.ScannableRow {
func (pgx *PGXDriver) QueryRow(ctx context.Context, sql string, args ...interface{}) interfaces.ScannableRow {
return pgx.pool.QueryRow(ctx, sql, args...)
}
// Exec satisfies sql.Database
func (pgx *PGXDriver) Exec(ctx context.Context, sql string, args ...interface{}) (sql.Result, error) {
func (pgx *PGXDriver) Exec(ctx context.Context, sql string, args ...interface{}) (interfaces.Result, error) {
res, err := pgx.pool.Exec(ctx, sql, args...)
return resultWrapper{ct: res}, err
}
@ -124,7 +105,7 @@ func (pgx *PGXDriver) Get(ctx context.Context, dest interface{}, query string, a
}
// Begin satisfies sql.Database
func (pgx *PGXDriver) Begin(ctx context.Context) (sql.Tx, error) {
func (pgx *PGXDriver) Begin(ctx context.Context) (interfaces.Tx, error) {
tx, err := pgx.pool.Begin(ctx)
if err != nil {
return nil, err
@ -132,16 +113,11 @@ func (pgx *PGXDriver) Begin(ctx context.Context) (sql.Tx, error) {
return pgxTxWrapper{tx: tx}, nil
}
func (pgx *PGXDriver) Stats() sql.Stats {
func (pgx *PGXDriver) Stats() interfaces.Stats {
stats := pgx.pool.Stat()
return pgxStatsWrapper{stats: stats}
}
// NodeID satisfies sql.Database
func (pgx *PGXDriver) NodeID() string {
return pgx.nodeID
}
// Close satisfies sql.Database/io.Closer
func (pgx *PGXDriver) Close() error {
pgx.pool.Close()
@ -212,12 +188,12 @@ type pgxTxWrapper struct {
}
// QueryRow satisfies sql.Tx
func (t pgxTxWrapper) QueryRow(ctx context.Context, sql string, args ...interface{}) sql.ScannableRow {
func (t pgxTxWrapper) QueryRow(ctx context.Context, sql string, args ...interface{}) interfaces.ScannableRow {
return t.tx.QueryRow(ctx, sql, args...)
}
// Exec satisfies sql.Tx
func (t pgxTxWrapper) Exec(ctx context.Context, sql string, args ...interface{}) (sql.Result, error) {
func (t pgxTxWrapper) Exec(ctx context.Context, sql string, args ...interface{}) (interfaces.Result, error) {
res, err := t.tx.Exec(ctx, sql, args...)
return resultWrapper{ct: res}, err
}

17
statediff/indexer/database/sql/postgres/pgx_test.go
View File

@ -18,7 +18,6 @@ package postgres_test
import (
"context"
"fmt"
"math/big"
"strings"
"testing"
@ -26,7 +25,6 @@ import (
"github.com/jackc/pgx/v4/pgxpool"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
"github.com/ethereum/go-ethereum/statediff/indexer/node"
"github.com/ethereum/go-ethereum/statediff/indexer/test_helpers"
)
@ -98,24 +96,11 @@ func TestPostgresPGX(t *testing.T) {
})
t.Run("throws error when can't connect to the database", func(t *testing.T) {
goodInfo := node.Info{GenesisBlock: "GENESIS", NetworkID: "1", ID: "x123", ClientName: "geth"}
_, err := postgres.NewPGXDriver(ctx, postgres.Config{}, goodInfo)
_, err := postgres.NewPGXDriver(ctx, postgres.Config{})
if err == nil {
t.Fatal("Expected an error")
}
expectContainsSubstring(t, err.Error(), postgres.DbConnectionFailedMsg)
})
t.Run("throws error when can't create node", func(t *testing.T) {
badHash := fmt.Sprintf("x %s", strings.Repeat("1", 100))
badInfo := node.Info{GenesisBlock: badHash, NetworkID: "1", ID: "x123", ClientName: "geth"}
_, err := postgres.NewPGXDriver(ctx, postgres.DefaultConfig, badInfo)
if err == nil {
t.Fatal("Expected an error")
}
expectContainsSubstring(t, err.Error(), postgres.SettingNodeFailedMsg)
})
}

54
statediff/indexer/database/sql/postgres/sqlx.go
View File

@ -18,27 +18,28 @@ package postgres
import (
"context"
coresql "database/sql"
"time"
coresql "database/sql"
"github.com/jmoiron/sqlx"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
"github.com/ethereum/go-ethereum/statediff/indexer/node"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
)
// SQLXDriver driver, implements sql.Driver
type SQLXDriver struct {
ctx context.Context
db *sqlx.DB
nodeInfo node.Info
nodeID string
ctx context.Context
db *sqlx.DB
}
// NewSQLXDriver returns a new sqlx driver for Postgres
// it initializes the connection pool and creates the node info table
func NewSQLXDriver(ctx context.Context, config Config, node node.Info) (*SQLXDriver, error) {
db, err := sqlx.ConnectContext(ctx, "postgres", config.DbConnectionString())
func NewSQLXDriver(ctx context.Context, config Config) (*SQLXDriver, error) {
connStr := config.DbConnectionString()
log.Info("connecting to database", "connection string", connStr)
db, err := sqlx.ConnectContext(ctx, "postgres", connStr)
if err != nil {
return &SQLXDriver{}, ErrDBConnectionFailed(err)
}
@ -52,33 +53,17 @@ func NewSQLXDriver(ctx context.Context, config Config, node node.Info) (*SQLXDri
lifetime := config.MaxConnLifetime
db.SetConnMaxLifetime(lifetime)
}
driver := &SQLXDriver{ctx: ctx, db: db, nodeInfo: node}
if err := driver.createNode(); err != nil {
return &SQLXDriver{}, ErrUnableToSetNode(err)
}
driver := &SQLXDriver{ctx: ctx, db: db}
return driver, nil
}
func (driver *SQLXDriver) createNode() error {
_, err := driver.db.Exec(
createNodeStm,
driver.nodeInfo.GenesisBlock, driver.nodeInfo.NetworkID,
driver.nodeInfo.ID, driver.nodeInfo.ClientName,
driver.nodeInfo.ChainID)
if err != nil {
return ErrUnableToSetNode(err)
}
driver.nodeID = driver.nodeInfo.ID
return nil
}
// QueryRow satisfies sql.Database
func (driver *SQLXDriver) QueryRow(_ context.Context, sql string, args ...interface{}) sql.ScannableRow {
func (driver *SQLXDriver) QueryRow(_ context.Context, sql string, args ...interface{}) interfaces.ScannableRow {
return driver.db.QueryRowx(sql, args...)
}
// Exec satisfies sql.Database
func (driver *SQLXDriver) Exec(_ context.Context, sql string, args ...interface{}) (sql.Result, error) {
func (driver *SQLXDriver) Exec(_ context.Context, sql string, args ...interface{}) (interfaces.Result, error) {
return driver.db.Exec(sql, args...)
}
@ -93,7 +78,7 @@ func (driver *SQLXDriver) Get(_ context.Context, dest interface{}, query string,
}
// Begin satisfies sql.Database
func (driver *SQLXDriver) Begin(_ context.Context) (sql.Tx, error) {
func (driver *SQLXDriver) Begin(_ context.Context) (interfaces.Tx, error) {
tx, err := driver.db.Beginx()
if err != nil {
return nil, err
@ -101,16 +86,11 @@ func (driver *SQLXDriver) Begin(_ context.Context) (sql.Tx, error) {
return sqlxTxWrapper{tx: tx}, nil
}
func (driver *SQLXDriver) Stats() sql.Stats {
func (driver *SQLXDriver) Stats() interfaces.Stats {
stats := driver.db.Stats()
return sqlxStatsWrapper{stats: stats}
}
// NodeID satisfies sql.Database
func (driver *SQLXDriver) NodeID() string {
return driver.nodeID
}
// Close satisfies sql.Database/io.Closer
func (driver *SQLXDriver) Close() error {
return driver.db.Close()
@ -170,12 +150,12 @@ type sqlxTxWrapper struct {
}
// QueryRow satisfies sql.Tx
func (t sqlxTxWrapper) QueryRow(ctx context.Context, sql string, args ...interface{}) sql.ScannableRow {
func (t sqlxTxWrapper) QueryRow(ctx context.Context, sql string, args ...interface{}) interfaces.ScannableRow {
return t.tx.QueryRowx(sql, args...)
}
// Exec satisfies sql.Tx
func (t sqlxTxWrapper) Exec(ctx context.Context, sql string, args ...interface{}) (sql.Result, error) {
func (t sqlxTxWrapper) Exec(ctx context.Context, sql string, args ...interface{}) (interfaces.Result, error) {
return t.tx.Exec(sql, args...)
}

18
statediff/indexer/database/sql/postgres/sqlx_test.go
View File

@ -17,16 +17,13 @@
package postgres_test
import (
"fmt"
"math/big"
"strings"
"testing"
"github.com/jmoiron/sqlx"
_ "github.com/lib/pq"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
"github.com/ethereum/go-ethereum/statediff/indexer/node"
"github.com/ethereum/go-ethereum/statediff/indexer/test_helpers"
)
@ -96,24 +93,11 @@ func TestPostgresSQLX(t *testing.T) {
})
t.Run("throws error when can't connect to the database", func(t *testing.T) {
goodInfo := node.Info{GenesisBlock: "GENESIS", NetworkID: "1", ID: "x123", ClientName: "geth"}
_, err := postgres.NewSQLXDriver(ctx, postgres.Config{}, goodInfo)
_, err := postgres.NewSQLXDriver(ctx, postgres.Config{})
if err == nil {
t.Fatal("Expected an error")
}
expectContainsSubstring(t, err.Error(), postgres.DbConnectionFailedMsg)
})
t.Run("throws error when can't create node", func(t *testing.T) {
badHash := fmt.Sprintf("x %s", strings.Repeat("1", 100))
badInfo := node.Info{GenesisBlock: badHash, NetworkID: "1", ID: "x123", ClientName: "geth"}
_, err := postgres.NewSQLXDriver(ctx, postgres.DefaultConfig, badInfo)
if err == nil {
t.Fatal("Expected an error")
}
expectContainsSubstring(t, err.Error(), postgres.SettingNodeFailedMsg)
})
}

39
statediff/indexer/database/sql/postgres/test_helpers.go
View File

@ -19,24 +19,43 @@ package postgres
import (
"context"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
"github.com/ethereum/go-ethereum/statediff/indexer/node"
v2 "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres/v2"
v3 "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres/v3"
"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
)
// SetupSQLXDB is used to setup a sqlx db for tests
func SetupSQLXDB() (sql.Database, error) {
driver, err := NewSQLXDriver(context.Background(), DefaultConfig, node.Info{})
// SetupV3SQLXDB is used to setup a sqlx db for tests
func SetupV3SQLXDB() (interfaces.Database, error) {
driver, err := NewSQLXDriver(context.Background(), DefaultV3Config)
if err != nil {
return nil, err
}
return NewPostgresDB(driver), nil
return v3.NewPostgresDB(driver), nil
}
// SetupPGXDB is used to setup a pgx db for tests
func SetupPGXDB() (sql.Database, error) {
driver, err := NewPGXDriver(context.Background(), DefaultConfig, node.Info{})
// SetupV3PGXDB is used to setup a pgx db for tests
func SetupV3PGXDB() (interfaces.Database, error) {
driver, err := NewPGXDriver(context.Background(), DefaultV3Config)
if err != nil {
return nil, err
}
return NewPostgresDB(driver), nil
return v3.NewPostgresDB(driver), nil
}
// SetupV2SQLXDB is used to setup a sqlx db for tests
func SetupV2SQLXDB() (interfaces.Database, error) {
driver, err := NewSQLXDriver(context.Background(), DefaultV2Config)
if err != nil {
return nil, err
}
return v2.NewPostgresDB(driver), nil
}
// SetupV2PGXDB is used to setup a pgx db for tests
func SetupV2PGXDB() (interfaces.Database, error) {
driver, err := NewPGXDriver(context.Background(), DefaultV2Config)
if err != nil {
return nil, err
}
return v2.NewPostgresDB(driver), nil
}

117
statediff/indexer/database/sql/postgres/v2/database.go Normal file
View File

@ -0,0 +1,117 @@
// VulcanizeDB
// Copyright © 2021 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package postgres
import (
"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
)
var _ interfaces.Database = &DB{}
const version = 2
// NewPostgresDB returns a postgres.DB using the provided driver
func NewPostgresDB(driver interfaces.Driver) *DB {
return &DB{driver}
}
// DB implements sql.Database using a configured driver and Postgres statement syntax
type DB struct {
interfaces.Driver
}
// InsertNodeInfoStm satisfies interfaces.Statements
func (db *DB) InsertNodeInfoStm() string {
return `INSERT INTO nodes (genesis_block, network_id, node_id, client_name, chain_id) VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (genesis_block, network_id, node_id, chain_id)
DO UPDATE SET client_name = $4
RETURNING ID`
}
// InsertHeaderStm satisfies the interfaces.Statements
func (db *DB) InsertHeaderStm() string {
return `INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated, base_fee)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16)
ON CONFLICT (block_number, block_hash) DO UPDATE SET (parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated, base_fee) = ($3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, eth.header_cids.times_validated + 1, $16)
RETURNING id`
}
// InsertUncleStm satisfies the interfaces.Statements
func (db *DB) InsertUncleStm() string {
return `INSERT INTO eth.uncle_cids (block_hash, header_id, parent_hash, cid, reward, mh_key) VALUES ($1, $2, $3, $4, $5, $6)
ON CONFLICT (header_id, block_hash) DO UPDATE SET (parent_hash, cid, reward, mh_key) = ($3, $4, $5, $6)`
}
// InsertTxStm satisfies the interfaces.Statements
func (db *DB) InsertTxStm() string {
return `INSERT INTO eth.transaction_cids (header_id, tx_hash, cid, dst, src, index, mh_key, tx_data, tx_type) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
ON CONFLICT (header_id, tx_hash) DO UPDATE SET (cid, dst, src, index, mh_key, tx_data, tx_type) = ($3, $4, $5, $6, $7, $8, $9)
RETURNING id`
}
// InsertAccessListElementStm satisfies the interfaces.Statements
func (db *DB) InsertAccessListElementStm() string {
return `INSERT INTO eth.access_list_element (tx_id, index, address, storage_keys) VALUES ($1, $2, $3, $4)
ON CONFLICT (tx_id, index) DO UPDATE SET (address, storage_keys) = ($3, $4)`
}
// InsertRctStm satisfies the interfaces.Statements
func (db *DB) InsertRctStm() string {
return `INSERT INTO eth.receipt_cids (tx_id, leaf_cid, contract, contract_hash, leaf_mh_key, post_state, post_status, log_root) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
ON CONFLICT (tx_id) DO UPDATE SET (leaf_cid, contract, contract_hash, leaf_mh_key, post_state, post_status, log_root) = ($2, $3, $4, $5, $6, $7, $8)
RETURNING id`
}
// InsertLogStm satisfies the interfaces.Statements
func (db *DB) InsertLogStm() string {
return `INSERT INTO eth.log_cids (leaf_cid, leaf_mh_key, receipt_id, address, index, topic0, topic1, topic2, topic3, log_data) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
ON CONFLICT (receipt_id, index) DO UPDATE SET (leaf_cid, leaf_mh_key ,address, topic0, topic1, topic2, topic3,log_data ) = ($1, $2, $4, $6, $7, $8, $9, $10)`
}
// InsertStateStm satisfies the interfaces.Statements
func (db *DB) InsertStateStm() string {
return `INSERT INTO eth.state_cids (header_id, state_leaf_key, cid, state_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7)
ON CONFLICT (header_id, state_path) DO UPDATE SET (state_leaf_key, cid, node_type, diff, mh_key) = ($2, $3, $5, $6, $7)
RETURNING id`
}
// InsertAccountStm satisfies the interfaces.Statements
func (db *DB) InsertAccountStm() string {
return `INSERT INTO eth.state_accounts (state_id, balance, nonce, code_hash, storage_root) VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (state_id) DO UPDATE SET (balance, nonce, code_hash, storage_root) = ($2, $3, $4, $5)`
}
// InsertStorageStm satisfies the interfaces.Statements
func (db *DB) InsertStorageStm() string {
return `INSERT INTO eth.storage_cids (state_id, storage_leaf_key, cid, storage_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7)
ON CONFLICT (state_id, storage_path) DO UPDATE SET (storage_leaf_key, cid, node_type, diff, mh_key) = ($2, $3, $5, $6, $7)`
}
// InsertIPLDStm satisfies the interfaces.Statements
func (db *DB) InsertIPLDStm() string {
return `INSERT INTO public.blocks (key, data) VALUES ($1, $2) ON CONFLICT (key) DO NOTHING`
}
// InsertIPLDsStm satisfies the interfaces.Statements
func (db *DB) InsertIPLDsStm() string {
return `INSERT INTO public.blocks (key, data) VALUES (unnest($1::TEXT[]), unnest($2::BYTEA[])) ON CONFLICT (key) DO NOTHING`
}
// Version satisfies the interfaces.Version
func (db *DB) Version() uint {
return version
}

70
statediff/indexer/database/sql/postgres/database.go → statediff/indexer/database/sql/postgres/v3/database.go
View File

@ -16,86 +16,96 @@
package postgres
import "github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
var _ sql.Database = &DB{}
const (
createNodeStm = `INSERT INTO nodes (genesis_block, network_id, node_id, client_name, chain_id) VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (node_id) DO NOTHING`
import (
"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
)
var _ interfaces.Database = &DB{}
const version = 3
// NewPostgresDB returns a postgres.DB using the provided driver
func NewPostgresDB(driver sql.Driver) *DB {
func NewPostgresDB(driver interfaces.Driver) *DB {
return &DB{driver}
}
// DB implements sql.Database using a configured driver and Postgres statement syntax
type DB struct {
sql.Driver
interfaces.Driver
}
// InsertHeaderStm satisfies the sql.Statements interface
// InsertNodeInfoStm satisfies interfaces.Statements
func (db *DB) InsertNodeInfoStm() string {
return `INSERT INTO nodes (genesis_block, network_id, node_id, client_name, chain_id) VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (node_id) DO NOTHING`
}
// InsertHeaderStm satisfies the interfaces.Statements
func (db *DB) InsertHeaderStm() string {
return `INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated, coinbase)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16)
ON CONFLICT (block_hash) DO UPDATE SET (parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated, coinbase) = ($3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, eth.header_cids.times_validated + 1, $16)`
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16)
ON CONFLICT (block_hash) DO UPDATE SET (parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated, coinbase) = ($3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, eth.header_cids.times_validated + 1, $16)`
}
// InsertUncleStm satisfies the sql.Statements interface
// InsertUncleStm satisfies the interfaces.Statements
func (db *DB) InsertUncleStm() string {
return `INSERT INTO eth.uncle_cids (block_hash, header_id, parent_hash, cid, reward, mh_key) VALUES ($1, $2, $3, $4, $5, $6)
ON CONFLICT (block_hash) DO NOTHING`
ON CONFLICT (block_hash) DO NOTHING`
}
// InsertTxStm satisfies the sql.Statements interface
// InsertTxStm satisfies the interfaces.Statements
func (db *DB) InsertTxStm() string {
return `INSERT INTO eth.transaction_cids (header_id, tx_hash, cid, dst, src, index, mh_key, tx_data, tx_type, value) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
ON CONFLICT (tx_hash) DO NOTHING`
ON CONFLICT (tx_hash) DO NOTHING`
}
// InsertAccessListElementStm satisfies the sql.Statements interface
// InsertAccessListElementStm satisfies the interfaces.Statements
func (db *DB) InsertAccessListElementStm() string {
return `INSERT INTO eth.access_list_elements (tx_id, index, address, storage_keys) VALUES ($1, $2, $3, $4)
ON CONFLICT (tx_id, index) DO NOTHING`
ON CONFLICT (tx_id, index) DO NOTHING`
}
// InsertRctStm satisfies the sql.Statements interface
// InsertRctStm satisfies the interfaces.Statements
func (db *DB) InsertRctStm() string {
return `INSERT INTO eth.receipt_cids (tx_id, leaf_cid, contract, contract_hash, leaf_mh_key, post_state, post_status, log_root) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
ON CONFLICT (tx_id) DO NOTHING`
ON CONFLICT (tx_id) DO NOTHING`
}
// InsertLogStm satisfies the sql.Statements interface
// InsertLogStm satisfies the interfaces.Statements
func (db *DB) InsertLogStm() string {
return `INSERT INTO eth.log_cids (leaf_cid, leaf_mh_key, rct_id, address, index, topic0, topic1, topic2, topic3, log_data) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
ON CONFLICT (rct_id, index) DO NOTHING`
ON CONFLICT (rct_id, index) DO NOTHING`
}
// InsertStateStm satisfies the sql.Statements interface
// InsertStateStm satisfies the interfaces.Statements
func (db *DB) InsertStateStm() string {
return `INSERT INTO eth.state_cids (header_id, state_leaf_key, cid, state_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7)
ON CONFLICT (header_id, state_path) DO UPDATE SET (state_leaf_key, cid, node_type, diff, mh_key) = ($2, $3, $5, $6, $7)`
ON CONFLICT (header_id, state_path) DO UPDATE SET (state_leaf_key, cid, node_type, diff, mh_key) = ($2, $3, $5, $6, $7)`
}
// InsertAccountStm satisfies the sql.Statements interface
// InsertAccountStm satisfies the interfaces.Statements
func (db *DB) InsertAccountStm() string {
return `INSERT INTO eth.state_accounts (header_id, state_path, balance, nonce, code_hash, storage_root) VALUES ($1, $2, $3, $4, $5, $6)
ON CONFLICT (header_id, state_path) DO NOTHING`
ON CONFLICT (header_id, state_path) DO NOTHING`
}
// InsertStorageStm satisfies the sql.Statements interface
// InsertStorageStm satisfies the interfaces.Statements
func (db *DB) InsertStorageStm() string {
return `INSERT INTO eth.storage_cids (header_id, state_path, storage_leaf_key, cid, storage_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
ON CONFLICT (header_id, state_path, storage_path) DO UPDATE SET (storage_leaf_key, cid, node_type, diff, mh_key) = ($3, $4, $6, $7, $8)`
ON CONFLICT (header_id, state_path, storage_path) DO UPDATE SET (storage_leaf_key, cid, node_type, diff, mh_key) = ($3, $4, $6, $7, $8)`
}
// InsertIPLDStm satisfies the sql.Statements interface
// InsertIPLDStm satisfies the interfaces.Statements
func (db *DB) InsertIPLDStm() string {
return `INSERT INTO public.blocks (key, data) VALUES ($1, $2) ON CONFLICT (key) DO NOTHING`
}
// InsertIPLDsStm satisfies the sql.Statements interface
// InsertIPLDsStm satisfies the interfaces.Statements
func (db *DB) InsertIPLDsStm() string {
return `INSERT INTO public.blocks (key, data) VALUES (unnest($1::TEXT[]), unnest($2::BYTEA[])) ON CONFLICT (key) DO NOTHING`
}
// Version satisfies the interfaces.Version
func (db *DB) Version() uint {
return version
}

14
statediff/indexer/database/sql/sqlx_indexer_legacy_test.go
View File

@ -20,6 +20,8 @@ import (
"context"
"testing"
nodeinfo "github.com/ethereum/go-ethereum/statediff/indexer/node"
"github.com/ipfs/go-cid"
"github.com/jmoiron/sqlx"
"github.com/multiformats/go-multihash"
@ -38,19 +40,23 @@ var (
legacyData = mocks.NewLegacyData()
mockLegacyBlock *types.Block
legacyHeaderCID cid.Cid
headerID int64
)
func setupLegacySQLX(t *testing.T) {
mockLegacyBlock = legacyData.MockBlock
legacyHeaderCID, _ = ipld.RawdataToCid(ipld.MEthHeader, legacyData.MockHeaderRlp, multihash.KECCAK_256)
db, err = postgres.SetupSQLXDB()
db, err = postgres.SetupV3SQLXDB()
require.NoError(t, err)
ind, err = sql.NewStateDiffIndexer(context.Background(), legacyData.Config, db)
v2DB, err := postgres.SetupV2PGXDB()
require.NoError(t, err)
ind, err = sql.NewStateDiffIndexer(context.Background(), legacyData.Config, nodeinfo.Info{}, v2DB, db)
require.NoError(t, err)
var tx interfaces.Batch
tx, err = ind.PushBlock(
tx, headerID, err = ind.PushBlock(
mockLegacyBlock,
legacyData.MockReceipts,
legacyData.MockBlock.Difficulty())
@ -62,7 +68,7 @@ func setupLegacySQLX(t *testing.T) {
}
}()
for _, node := range legacyData.StateDiffs {
err = ind.PushStateNode(tx, node, mockLegacyBlock.Hash().String())
err = ind.PushStateNode(tx, node, mockLegacyBlock.Hash().String(), headerID)
require.NoError(t, err)
}

125
statediff/indexer/database/sql/sqlx_indexer_test.go
View File

@ -33,33 +33,35 @@ import (
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
"github.com/ethereum/go-ethereum/statediff/indexer/mocks"
"github.com/ethereum/go-ethereum/statediff/indexer/models"
sharedModels "github.com/ethereum/go-ethereum/statediff/indexer/models/shared"
v3Models "github.com/ethereum/go-ethereum/statediff/indexer/models/v3"
nodeinfo "github.com/ethereum/go-ethereum/statediff/indexer/node"
"github.com/ethereum/go-ethereum/statediff/indexer/shared"
"github.com/ethereum/go-ethereum/statediff/indexer/test_helpers"
)
func setupSQLX(t *testing.T) {
db, err = postgres.SetupSQLXDB()
if err != nil {
t.Fatal(err)
}
ind, err = sql.NewStateDiffIndexer(context.Background(), mocks.TestConfig, db)
db, err = postgres.SetupV3SQLXDB()
require.NoError(t, err)
v2DB, err := postgres.SetupV2PGXDB()
require.NoError(t, err)
ind, err = sql.NewStateDiffIndexer(context.Background(), mocks.TestConfig, nodeinfo.Info{}, v2DB, db)
require.NoError(t, err)
var tx interfaces.Batch
tx, err = ind.PushBlock(
tx, headerID, err = ind.PushBlock(
mockBlock,
mocks.MockReceipts,
mocks.MockBlock.Difficulty())
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
defer func() {
if err := tx.Submit(err); err != nil {
t.Fatal(err)
}
}()
for _, node := range mocks.StateDiffs {
err = ind.PushStateNode(tx, node, mockBlock.Hash().String())
err = ind.PushStateNode(tx, node, mockBlock.Hash().String(), headerID)
require.NoError(t, err)
}
@ -90,24 +92,21 @@ func TestSQLXIndexer(t *testing.T) {
}
header := new(res)
err = db.QueryRow(context.Background(), pgStr, mocks.BlockNumber.Uint64()).(*sqlx.Row).StructScan(header)
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
test_helpers.ExpectEqual(t, header.CID, headerCID.String())
test_helpers.ExpectEqual(t, header.TD, mocks.MockBlock.Difficulty().String())
test_helpers.ExpectEqual(t, header.Reward, "2000000000000021250")
test_helpers.ExpectEqual(t, header.Coinbase, mocks.MockHeader.Coinbase.String())
dc, err := cid.Decode(header.CID)
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
mhKey := dshelp.MultihashToDsKey(dc.Hash())
prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
var data []byte
err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey)
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
test_helpers.ExpectEqual(t, data, mocks.MockHeaderRlp)
})
@ -119,9 +118,8 @@ func TestSQLXIndexer(t *testing.T) {
pgStr := `SELECT transaction_cids.cid FROM eth.transaction_cids INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.block_hash)
WHERE header_cids.block_number = $1`
err = db.Select(context.Background(), &trxs, pgStr, mocks.BlockNumber.Uint64())
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
test_helpers.ExpectEqual(t, len(trxs), 5)
expectTrue(t, test_helpers.ListContainsString(trxs, trx1CID.String()))
expectTrue(t, test_helpers.ListContainsString(trxs, trx2CID.String()))
@ -136,25 +134,22 @@ func TestSQLXIndexer(t *testing.T) {
}
for _, c := range trxs {
dc, err := cid.Decode(c)
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
mhKey := dshelp.MultihashToDsKey(dc.Hash())
prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
var data []byte
err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey)
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
txTypeAndValueStr := `SELECT tx_type, value FROM eth.transaction_cids WHERE cid = $1`
switch c {
case trx1CID.String():
test_helpers.ExpectEqual(t, data, tx1)
txRes := new(txResult)
err = db.QueryRow(context.Background(), txTypeAndValueStr, c).(*sqlx.Row).StructScan(txRes)
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
if txRes.TxType != 0 {
t.Fatalf("expected LegacyTxType (0), got %d", txRes.TxType)
}
@ -165,9 +160,7 @@ func TestSQLXIndexer(t *testing.T) {
test_helpers.ExpectEqual(t, data, tx2)
txRes := new(txResult)
err = db.QueryRow(context.Background(), txTypeAndValueStr, c).(*sqlx.Row).StructScan(txRes)
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
if txRes.TxType != 0 {
t.Fatalf("expected LegacyTxType (0), got %d", txRes.TxType)
}
@ -178,9 +171,7 @@ func TestSQLXIndexer(t *testing.T) {
test_helpers.ExpectEqual(t, data, tx3)
txRes := new(txResult)
err = db.QueryRow(context.Background(), txTypeAndValueStr, c).(*sqlx.Row).StructScan(txRes)
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
if txRes.TxType != 0 {
t.Fatalf("expected LegacyTxType (0), got %d", txRes.TxType)
}
@ -191,29 +182,25 @@ func TestSQLXIndexer(t *testing.T) {
test_helpers.ExpectEqual(t, data, tx4)
txRes := new(txResult)
err = db.QueryRow(context.Background(), txTypeAndValueStr, c).(*sqlx.Row).StructScan(txRes)
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
if txRes.TxType != types.AccessListTxType {
t.Fatalf("expected AccessListTxType (1), got %d", txRes.TxType)
}
if txRes.Value != transactions[3].Value().String() {
t.Fatalf("expected tx value %s got %s", transactions[3].Value().String(), txRes.Value)
}
accessListElementModels := make([]models.AccessListElementModel, 0)
accessListElementModels := make([]v3Models.AccessListElementModel, 0)
pgStr = `SELECT access_list_elements.* FROM eth.access_list_elements INNER JOIN eth.transaction_cids ON (tx_id = transaction_cids.tx_hash) WHERE cid = $1 ORDER BY access_list_elements.index ASC`
err = db.Select(context.Background(), &accessListElementModels, pgStr, c)
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
if len(accessListElementModels) != 2 {
t.Fatalf("expected two access list entries, got %d", len(accessListElementModels))
}
model1 := models.AccessListElementModel{
model1 := v3Models.AccessListElementModel{
Index: accessListElementModels[0].Index,
Address: accessListElementModels[0].Address,
}
model2 := models.AccessListElementModel{
model2 := v3Models.AccessListElementModel{
Index: accessListElementModels[1].Index,
Address: accessListElementModels[1].Address,
StorageKeys: accessListElementModels[1].StorageKeys,
@ -224,9 +211,7 @@ func TestSQLXIndexer(t *testing.T) {
test_helpers.ExpectEqual(t, data, tx5)
txRes := new(txResult)
err = db.QueryRow(context.Background(), txTypeAndValueStr, c).(*sqlx.Row).StructScan(txRes)
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
if txRes.TxType != types.DynamicFeeTxType {
t.Fatalf("expected DynamicFeeTxType (2), got %d", txRes.TxType)
}
@ -314,7 +299,7 @@ func TestSQLXIndexer(t *testing.T) {
expectTrue(t, test_helpers.ListContainsString(rcts, rct5CID.String()))
for idx, c := range rcts {
result := make([]models.IPLDModel, 0)
result := make([]sharedModels.IPLDModel, 0)
pgStr = `SELECT data
FROM eth.receipt_cids
INNER JOIN public.blocks ON (receipt_cids.leaf_mh_key = public.blocks.key)
@ -352,41 +337,31 @@ func TestSQLXIndexer(t *testing.T) {
var postStatus uint64
pgStr = `SELECT post_status FROM eth.receipt_cids WHERE leaf_cid = $1`
err = db.Get(context.Background(), &postStatus, pgStr, c)
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
test_helpers.ExpectEqual(t, postStatus, mocks.ExpectedPostStatus)
case rct2CID.String():
test_helpers.ExpectEqual(t, data, rctLeaf2)
var postState string
err = db.Get(context.Background(), &postState, postStatePgStr, c)
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
test_helpers.ExpectEqual(t, postState, mocks.ExpectedPostState1)
case rct3CID.String():
test_helpers.ExpectEqual(t, data, rctLeaf3)
var postState string
err = db.Get(context.Background(), &postState, postStatePgStr, c)
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
test_helpers.ExpectEqual(t, postState, mocks.ExpectedPostState2)
case rct4CID.String():
test_helpers.ExpectEqual(t, data, rctLeaf4)
var postState string
err = db.Get(context.Background(), &postState, postStatePgStr, c)
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
test_helpers.ExpectEqual(t, postState, mocks.ExpectedPostState3)
case rct5CID.String():
test_helpers.ExpectEqual(t, data, rctLeaf5)
var postState string
err = db.Get(context.Background(), &postState, postStatePgStr, c)
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
test_helpers.ExpectEqual(t, postState, mocks.ExpectedPostState3)
}
}
@ -396,7 +371,7 @@ func TestSQLXIndexer(t *testing.T) {
setupSQLX(t)
defer tearDown(t)
// check that state nodes were properly indexed and published
stateNodes := make([]models.StateNodeModel, 0)
stateNodes := make([]v3Models.StateNodeModel, 0)
pgStr := `SELECT state_cids.cid, state_cids.state_leaf_key, state_cids.node_type, state_cids.state_path, state_cids.header_id
FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.block_hash)
WHERE header_cids.block_number = $1 AND node_type != 3`
@ -418,7 +393,7 @@ func TestSQLXIndexer(t *testing.T) {
t.Fatal(err)
}
pgStr = `SELECT * from eth.state_accounts WHERE header_id = $1 AND state_path = $2`
var account models.StateAccountModel
var account v3Models.StateAccountModel
err = db.Get(context.Background(), &account, pgStr, stateNode.HeaderID, stateNode.Path)
if err != nil {
t.Fatal(err)
@ -428,7 +403,7 @@ func TestSQLXIndexer(t *testing.T) {
test_helpers.ExpectEqual(t, stateNode.StateKey, common.BytesToHash(mocks.ContractLeafKey).Hex())
test_helpers.ExpectEqual(t, stateNode.Path, []byte{'\x06'})
test_helpers.ExpectEqual(t, data, mocks.ContractLeafNode)
test_helpers.ExpectEqual(t, account, models.StateAccountModel{
test_helpers.ExpectEqual(t, account, v3Models.StateAccountModel{
HeaderID: account.HeaderID,
StatePath: stateNode.Path,
Balance: "0",
@ -442,7 +417,7 @@ func TestSQLXIndexer(t *testing.T) {
test_helpers.ExpectEqual(t, stateNode.StateKey, common.BytesToHash(mocks.AccountLeafKey).Hex())
test_helpers.ExpectEqual(t, stateNode.Path, []byte{'\x0c'})
test_helpers.ExpectEqual(t, data, mocks.AccountLeafNode)
test_helpers.ExpectEqual(t, account, models.StateAccountModel{
test_helpers.ExpectEqual(t, account, v3Models.StateAccountModel{
HeaderID: account.HeaderID,
StatePath: stateNode.Path,
Balance: "1000",
@ -454,7 +429,7 @@ func TestSQLXIndexer(t *testing.T) {
}
// check that Removed state nodes were properly indexed and published
stateNodes = make([]models.StateNodeModel, 0)
stateNodes = make([]v3Models.StateNodeModel, 0)
pgStr = `SELECT state_cids.cid, state_cids.state_leaf_key, state_cids.node_type, state_cids.state_path, state_cids.header_id
FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.block_hash)
WHERE header_cids.block_number = $1 AND node_type = 3`
@ -485,7 +460,7 @@ func TestSQLXIndexer(t *testing.T) {
setupSQLX(t)
defer tearDown(t)
// check that storage nodes were properly indexed
storageNodes := make([]models.StorageNodeWithStateKeyModel, 0)
storageNodes := make([]v3Models.StorageNodeWithStateKeyModel, 0)
pgStr := `SELECT storage_cids.cid, state_cids.state_leaf_key, storage_cids.storage_leaf_key, storage_cids.node_type, storage_cids.storage_path
FROM eth.storage_cids, eth.state_cids, eth.header_cids
WHERE (storage_cids.state_path, storage_cids.header_id) = (state_cids.state_path, state_cids.header_id)
@ -497,7 +472,7 @@ func TestSQLXIndexer(t *testing.T) {
t.Fatal(err)
}
test_helpers.ExpectEqual(t, len(storageNodes), 1)
test_helpers.ExpectEqual(t, storageNodes[0], models.StorageNodeWithStateKeyModel{
test_helpers.ExpectEqual(t, storageNodes[0], v3Models.StorageNodeWithStateKeyModel{
CID: storageCID.String(),
NodeType: 2,
StorageKey: common.BytesToHash(mocks.StorageLeafKey).Hex(),
@ -518,7 +493,7 @@ func TestSQLXIndexer(t *testing.T) {
test_helpers.ExpectEqual(t, data, mocks.StorageLeafNode)
// check that Removed storage nodes were properly indexed
storageNodes = make([]models.StorageNodeWithStateKeyModel, 0)
storageNodes = make([]v3Models.StorageNodeWithStateKeyModel, 0)
pgStr = `SELECT storage_cids.cid, state_cids.state_leaf_key, storage_cids.storage_leaf_key, storage_cids.node_type, storage_cids.storage_path
FROM eth.storage_cids, eth.state_cids, eth.header_cids
WHERE (storage_cids.state_path, storage_cids.header_id) = (state_cids.state_path, state_cids.header_id)
@ -530,7 +505,7 @@ func TestSQLXIndexer(t *testing.T) {
t.Fatal(err)
}
test_helpers.ExpectEqual(t, len(storageNodes), 1)
test_helpers.ExpectEqual(t, storageNodes[0], models.StorageNodeWithStateKeyModel{
test_helpers.ExpectEqual(t, storageNodes[0], v3Models.StorageNodeWithStateKeyModel{
CID: shared.RemovedNodeStorageCID,
NodeType: 3,
StorageKey: common.BytesToHash(mocks.RemovedLeafKey).Hex(),

4
statediff/indexer/database/sql/test_helpers.go
View File

@ -19,10 +19,12 @@ package sql
import (
"context"
"testing"
"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
)
// TearDownDB is used to tear down the watcher dbs after tests
func TearDownDB(t *testing.T, db Database) {
func TearDownDB(t *testing.T, db interfaces.Database) {
ctx := context.Background()
tx, err := db.Begin(ctx)
if err != nil {

223
statediff/indexer/database/sql/v2/writer.go Normal file
View File

@ -0,0 +1,223 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package sql
import (
"fmt"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/metrics"
metrics2 "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/metrics"
"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
"github.com/ethereum/go-ethereum/statediff/indexer/models/v2"
"github.com/ethereum/go-ethereum/statediff/indexer/node"
)
var (
nullHash = common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000")
writerV2Metrics = metrics2.RegisterWriterMetrics(metrics.DefaultRegistry, "v3")
)
// Writer handles processing and writing of indexed IPLD objects to Postgres
type Writer struct {
DB interfaces.Database
nodeID int64
}
// NewWriter creates a new pointer to a Writer
func NewWriter(db interfaces.Database) *Writer {
return &Writer{
DB: db,
}
}
// Close satisfies io.Closer
func (w *Writer) Close() error {
return w.DB.Close()
}
/*
InsertNodeInfo inserts a node info model
INSERT INTO nodes (genesis_block, network_id, node_id, client_name, chain_id) VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (genesis_block, network_id, node_id, chain_id) DO NOTHING RETURNING ID
*/
func (w *Writer) InsertNodeInfo(info node.Info) error {
var nodeID int64
if err := w.DB.QueryRow(w.DB.Context(), w.DB.InsertNodeInfoStm(), info.GenesisBlock, info.NetworkID, info.ID,
info.ClientName, info.ChainID).Scan(&nodeID); err != nil {
return err
}
w.nodeID = nodeID
return nil
}
/*
InsertHeaderCID inserts a header model
INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated, base_fee)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16)
ON CONFLICT (block_number, block_hash) DO UPDATE SET (parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated, base_fee) = ($3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, eth.header_cids.times_validated + 1, $16)
*/
func (w *Writer) InsertHeaderCID(tx interfaces.Tx, header *models.HeaderModel) (int64, error) {
var headerID int64
err := tx.QueryRow(w.DB.Context(), w.DB.InsertHeaderStm(),
header.BlockNumber, header.BlockHash, header.ParentHash, header.CID, header.TotalDifficulty, w.nodeID,
header.Reward, header.StateRoot, header.TxRoot, header.RctRoot, header.UncleRoot, header.Bloom,
header.Timestamp, header.MhKey, 1, header.BaseFee).Scan(&headerID)
if err != nil {
return 0, fmt.Errorf("v2 error inserting header_cids entry: %v", err)
}
writerV2Metrics.Blocks.Inc(1)
return headerID, nil
}
/*
InsertUncleCID inserts an uncle model
INSERT INTO eth.uncle_cids (block_hash, header_id, parent_hash, cid, reward, mh_key) VALUES ($1, $2, $3, $4, $5, $6)
ON CONFLICT (header_id, block_hash) DO NOTHING
*/
func (w *Writer) InsertUncleCID(tx interfaces.Tx, uncle *models.UncleModel) error {
_, err := tx.Exec(w.DB.Context(), w.DB.InsertUncleStm(),
uncle.BlockHash, uncle.HeaderID, uncle.ParentHash, uncle.CID, uncle.Reward, uncle.MhKey)
if err != nil {
return fmt.Errorf("v2 error inserting uncle_cids entry: %v", err)
}
return nil
}
/*
InsertTransactionCID inserts a tx model
INSERT INTO eth.transaction_cids (header_id, tx_hash, cid, dst, src, index, mh_key, tx_data, tx_type) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
ON CONFLICT (header_id, tx_hash) DO NOTHING
*/
func (w *Writer) InsertTransactionCID(tx interfaces.Tx, transaction *models.TxModel) (int64, error) {
var txID int64
err := tx.QueryRow(w.DB.Context(), w.DB.InsertTxStm(),
transaction.HeaderID, transaction.TxHash, transaction.CID, transaction.Dst, transaction.Src, transaction.Index,
transaction.MhKey, transaction.Data, []byte{transaction.Type}).Scan(&txID)
if err != nil {
return 0, fmt.Errorf("v2 error inserting transaction_cids entry: %v", err)
}
writerV2Metrics.Transactions.Inc(1)
return txID, nil
}
/*
InsertAccessListElement inserts an access list element model
INSERT INTO eth.access_list_elements (tx_id, index, address, storage_keys) VALUES ($1, $2, $3, $4)
ON CONFLICT (tx_id, index) DO NOTHING
*/
func (w *Writer) InsertAccessListElement(tx interfaces.Tx, accessListElement *models.AccessListElementModel) error {
_, err := tx.Exec(w.DB.Context(), w.DB.InsertAccessListElementStm(),
accessListElement.TxID, accessListElement.Index, accessListElement.Address, accessListElement.StorageKeys)
if err != nil {
return fmt.Errorf("v2 error inserting access_list_element entry: %v", err)
}
writerV2Metrics.AccessListEntries.Inc(1)
return nil
}
/*
InsertReceiptCID inserts a receipt model
INSERT INTO eth.receipt_cids (tx_id, leaf_cid, contract, contract_hash, leaf_mh_key, post_state, post_status, log_root) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
ON CONFLICT (tx_id) DO NOTHING
*/
func (w *Writer) InsertReceiptCID(tx interfaces.Tx, rct *models.ReceiptModel) (int64, error) {
var receiptID int64
err := tx.QueryRow(w.DB.Context(), w.DB.InsertRctStm(),
rct.TxID, rct.LeafCID, rct.Contract, rct.ContractHash, rct.LeafMhKey, rct.PostState, rct.PostStatus, rct.LogRoot).Scan(&receiptID)
if err != nil {
return 0, fmt.Errorf("v2 error inserting receipt_cids entry: %w", err)
}
writerV2Metrics.Receipts.Inc(1)
return receiptID, nil
}
/*
InsertLogCID inserts a log model
INSERT INTO eth.log_cids (leaf_cid, leaf_mh_key, rct_id, address, index, topic0, topic1, topic2, topic3, log_data) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
ON CONFLICT (receipt_id, index) DO NOTHING
*/
func (w *Writer) InsertLogCID(tx interfaces.Tx, logs []*models.LogsModel) error {
for _, log := range logs {
_, err := tx.Exec(w.DB.Context(), w.DB.InsertLogStm(),
log.LeafCID, log.LeafMhKey, log.ReceiptID, log.Address, log.Index, log.Topic0, log.Topic1, log.Topic2,
log.Topic3, log.Data)
if err != nil {
return fmt.Errorf("v2 error inserting logs entry: %w", err)
}
writerV2Metrics.Logs.Inc(1)
}
return nil
}
/*
InsertStateCID inserts a state model
INSERT INTO eth.state_cids (header_id, state_leaf_key, cid, state_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7)
ON CONFLICT (header_id, state_path) DO UPDATE SET (state_leaf_key, cid, node_type, diff, mh_key) = ($2, $3, $5, $6, $7)
*/
func (w *Writer) InsertStateCID(tx interfaces.Tx, stateNode *models.StateNodeModel) (int64, error) {
var stateID int64
var stateKey string
if stateNode.StateKey != nullHash.String() {
stateKey = stateNode.StateKey
}
err := tx.QueryRow(w.DB.Context(), w.DB.InsertStateStm(),
stateNode.HeaderID, stateKey, stateNode.CID, stateNode.Path, stateNode.NodeType, true, stateNode.MhKey).Scan(&stateID)
if err != nil {
return 0, fmt.Errorf("v2 error inserting state_cids entry: %v", err)
}
return stateID, nil
}
/*
InsertStateAccount inserts a state account model
INSERT INTO eth.state_accounts (state_id, balance, nonce, code_hash, storage_root) VALUES ($1, $2, $3, $4, $5, $6)
ON CONFLICT (state_id) DO NOTHING
*/
func (w *Writer) InsertStateAccount(tx interfaces.Tx, stateAccount *models.StateAccountModel) error {
_, err := tx.Exec(w.DB.Context(), w.DB.InsertAccountStm(),
stateAccount.StateID, stateAccount.Balance, stateAccount.Nonce, stateAccount.CodeHash,
stateAccount.StorageRoot)
if err != nil {
return fmt.Errorf("v2 error inserting state_accounts entry: %v", err)
}
return nil
}
/*
InsertStorageCID inserts a storage model
INSERT INTO eth.storage_cids (state_id, storage_leaf_key, cid, storage_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7)
ON CONFLICT (state_id, storage_path) DO UPDATE SET (storage_leaf_key, cid, node_type, diff, mh_key) = ($2, $3, $5, $6, $7)
*/
func (w *Writer) InsertStorageCID(tx interfaces.Tx, storageCID *models.StorageNodeModel) error {
var storageKey string
if storageCID.StorageKey != nullHash.String() {
storageKey = storageCID.StorageKey
}
_, err := tx.Exec(w.DB.Context(), w.DB.InsertStorageStm(),
storageCID.StateID, storageKey, storageCID.CID, storageCID.Path, storageCID.NodeType,
true, storageCID.MhKey)
if err != nil {
return fmt.Errorf("v2 error inserting storage_cids entry: %v", err)
}
return nil
}
// Stats returns the stats for the underlying DB
func (w *Writer) Stats() interfaces.Stats {
return w.DB.Stats()
}

107
statediff/indexer/database/sql/writer.go → statediff/indexer/database/sql/v3/writer.go
View File

@ -20,165 +20,194 @@ import (
"fmt"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/statediff/indexer/models"
"github.com/ethereum/go-ethereum/metrics"
metrics2 "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/metrics"
"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
"github.com/ethereum/go-ethereum/statediff/indexer/models/v3"
"github.com/ethereum/go-ethereum/statediff/indexer/node"
)
var (
nullHash = common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000")
nullHash = common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000")
writerV3Metrics = metrics2.RegisterWriterMetrics(metrics.DefaultRegistry, "v3")
)
// Writer handles processing and writing of indexed IPLD objects to Postgres
type Writer struct {
db Database
DB interfaces.Database
nodeID string
}
// NewWriter creates a new pointer to a Writer
func NewWriter(db Database) *Writer {
func NewWriter(db interfaces.Database) *Writer {
return &Writer{
db: db,
DB: db,
}
}
// Close satisfies io.Closer
func (w *Writer) Close() error {
return w.db.Close()
return w.DB.Close()
}
/*
InsertNodeInfo inserts a node info model
INSERT INTO nodes (genesis_block, network_id, node_id, client_name, chain_id) VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (node_id) DO NOTHING
*/
func (w *Writer) InsertNodeInfo(info node.Info) error {
if _, err := w.DB.Exec(w.DB.Context(), w.DB.InsertNodeInfoStm(), info.GenesisBlock, info.NetworkID, info.ID,
info.ClientName, info.ChainID); err != nil {
return err
}
w.nodeID = info.ID
return nil
}
/*
InsertHeaderCID inserts a header model
INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated, coinbase)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16)
ON CONFLICT (block_hash) DO UPDATE SET (block_number, parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated, coinbase) = ($1, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, eth.header_cids.times_validated + 1, $16)
*/
func (w *Writer) upsertHeaderCID(tx Tx, header models.HeaderModel) error {
_, err := tx.Exec(w.db.Context(), w.db.InsertHeaderStm(),
header.BlockNumber, header.BlockHash, header.ParentHash, header.CID, header.TotalDifficulty, w.db.NodeID(),
func (w *Writer) InsertHeaderCID(tx interfaces.Tx, header *models.HeaderModel) error {
_, err := tx.Exec(w.DB.Context(), w.DB.InsertHeaderStm(),
header.BlockNumber, header.BlockHash, header.ParentHash, header.CID, header.TotalDifficulty, w.nodeID,
header.Reward, header.StateRoot, header.TxRoot, header.RctRoot, header.UncleRoot, header.Bloom,
header.Timestamp, header.MhKey, 1, header.Coinbase)
if err != nil {
return fmt.Errorf("error upserting header_cids entry: %v", err)
return fmt.Errorf("error inserting header_cids entry: %v", err)
}
indexerMetrics.blocks.Inc(1)
writerV3Metrics.Blocks.Inc(1)
return nil
}
/*
InsertUncleCID inserts an uncle model
INSERT INTO eth.uncle_cids (block_hash, header_id, parent_hash, cid, reward, mh_key) VALUES ($1, $2, $3, $4, $5, $6)
ON CONFLICT (block_hash) DO NOTHING
*/
func (w *Writer) upsertUncleCID(tx Tx, uncle models.UncleModel) error {
_, err := tx.Exec(w.db.Context(), w.db.InsertUncleStm(),
func (w *Writer) InsertUncleCID(tx interfaces.Tx, uncle *models.UncleModel) error {
_, err := tx.Exec(w.DB.Context(), w.DB.InsertUncleStm(),
uncle.BlockHash, uncle.HeaderID, uncle.ParentHash, uncle.CID, uncle.Reward, uncle.MhKey)
if err != nil {
return fmt.Errorf("error upserting uncle_cids entry: %v", err)
return fmt.Errorf("error inserting uncle_cids entry: %v", err)
}
return nil
}
/*
InsertTransactionCID inserts a tx model
INSERT INTO eth.transaction_cids (header_id, tx_hash, cid, dst, src, index, mh_key, tx_data, tx_type, value) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
ON CONFLICT (tx_hash) DO NOTHING
*/
func (w *Writer) upsertTransactionCID(tx Tx, transaction models.TxModel) error {
_, err := tx.Exec(w.db.Context(), w.db.InsertTxStm(),
func (w *Writer) InsertTransactionCID(tx interfaces.Tx, transaction *models.TxModel) error {
_, err := tx.Exec(w.DB.Context(), w.DB.InsertTxStm(),
transaction.HeaderID, transaction.TxHash, transaction.CID, transaction.Dst, transaction.Src, transaction.Index,
transaction.MhKey, transaction.Data, transaction.Type, transaction.Value)
if err != nil {
return fmt.Errorf("error upserting transaction_cids entry: %v", err)
return fmt.Errorf("v3 error inserting transaction_cids entry: %v", err)
}
indexerMetrics.transactions.Inc(1)
writerV3Metrics.Transactions.Inc(1)
return nil
}
/*
InsertAccessListElement inserts an access list element model
INSERT INTO eth.access_list_elements (tx_id, index, address, storage_keys) VALUES ($1, $2, $3, $4)
ON CONFLICT (tx_id, index) DO NOTHING
*/
func (w *Writer) upsertAccessListElement(tx Tx, accessListElement models.AccessListElementModel) error {
_, err := tx.Exec(w.db.Context(), w.db.InsertAccessListElementStm(),
func (w *Writer) InsertAccessListElement(tx interfaces.Tx, accessListElement *models.AccessListElementModel) error {
_, err := tx.Exec(w.DB.Context(), w.DB.InsertAccessListElementStm(),
accessListElement.TxID, accessListElement.Index, accessListElement.Address, accessListElement.StorageKeys)
if err != nil {
return fmt.Errorf("error upserting access_list_element entry: %v", err)
return fmt.Errorf("v3 error inserting access_list_element entry: %v", err)
}
indexerMetrics.accessListEntries.Inc(1)
writerV3Metrics.AccessListEntries.Inc(1)
return nil
}
/*
InsertReceiptCID inserts a receipt model
INSERT INTO eth.receipt_cids (tx_id, leaf_cid, contract, contract_hash, leaf_mh_key, post_state, post_status, log_root) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
ON CONFLICT (tx_id) DO NOTHING
*/
func (w *Writer) upsertReceiptCID(tx Tx, rct *models.ReceiptModel) error {
_, err := tx.Exec(w.db.Context(), w.db.InsertRctStm(),
func (w *Writer) InsertReceiptCID(tx interfaces.Tx, rct *models.ReceiptModel) error {
_, err := tx.Exec(w.DB.Context(), w.DB.InsertRctStm(),
rct.TxID, rct.LeafCID, rct.Contract, rct.ContractHash, rct.LeafMhKey, rct.PostState, rct.PostStatus, rct.LogRoot)
if err != nil {
return fmt.Errorf("error upserting receipt_cids entry: %w", err)
return fmt.Errorf("v3 error inserting receipt_cids entry: %w", err)
}
indexerMetrics.receipts.Inc(1)
writerV3Metrics.Receipts.Inc(1)
return nil
}
/*
InsertLogCID inserts a log model
INSERT INTO eth.log_cids (leaf_cid, leaf_mh_key, rct_id, address, index, topic0, topic1, topic2, topic3, log_data) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
ON CONFLICT (rct_id, index) DO NOTHING
*/
func (w *Writer) upsertLogCID(tx Tx, logs []*models.LogsModel) error {
func (w *Writer) InsertLogCID(tx interfaces.Tx, logs []*models.LogsModel) error {
for _, log := range logs {
_, err := tx.Exec(w.db.Context(), w.db.InsertLogStm(),
_, err := tx.Exec(w.DB.Context(), w.DB.InsertLogStm(),
log.LeafCID, log.LeafMhKey, log.ReceiptID, log.Address, log.Index, log.Topic0, log.Topic1, log.Topic2,
log.Topic3, log.Data)
if err != nil {
return fmt.Errorf("error upserting logs entry: %w", err)
return fmt.Errorf("v3 error inserting logs entry: %w", err)
}
indexerMetrics.logs.Inc(1)
writerV3Metrics.Logs.Inc(1)
}
return nil
}
/*
InsertStateCID inserts a state model
INSERT INTO eth.state_cids (header_id, state_leaf_key, cid, state_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7)
ON CONFLICT (header_id, state_path) DO UPDATE SET (state_leaf_key, cid, node_type, diff, mh_key) = ($2, $3, $5, $6, $7)
*/
func (w *Writer) upsertStateCID(tx Tx, stateNode models.StateNodeModel) error {
func (w *Writer) InsertStateCID(tx interfaces.Tx, stateNode *models.StateNodeModel) error {
var stateKey string
if stateNode.StateKey != nullHash.String() {
stateKey = stateNode.StateKey
}
_, err := tx.Exec(w.db.Context(), w.db.InsertStateStm(),
_, err := tx.Exec(w.DB.Context(), w.DB.InsertStateStm(),
stateNode.HeaderID, stateKey, stateNode.CID, stateNode.Path, stateNode.NodeType, true, stateNode.MhKey)
if err != nil {
return fmt.Errorf("error upserting state_cids entry: %v", err)
return fmt.Errorf("v3 error inserting state_cids entry: %v", err)
}
return nil
}
/*
InsertStateAccount inserts a state account model
INSERT INTO eth.state_accounts (header_id, state_path, balance, nonce, code_hash, storage_root) VALUES ($1, $2, $3, $4, $5, $6)
ON CONFLICT (header_id, state_path) DO NOTHING
*/
func (w *Writer) upsertStateAccount(tx Tx, stateAccount models.StateAccountModel) error {
_, err := tx.Exec(w.db.Context(), w.db.InsertAccountStm(),
func (w *Writer) InsertStateAccount(tx interfaces.Tx, stateAccount *models.StateAccountModel) error {
_, err := tx.Exec(w.DB.Context(), w.DB.InsertAccountStm(),
stateAccount.HeaderID, stateAccount.StatePath, stateAccount.Balance, stateAccount.Nonce, stateAccount.CodeHash,
stateAccount.StorageRoot)
if err != nil {
return fmt.Errorf("error upserting state_accounts entry: %v", err)
return fmt.Errorf("v3 error inserting state_accounts entry: %v", err)
}
return nil
}
/*
InsertStorageCID inserts a storage model
INSERT INTO eth.storage_cids (header_id, state_path, storage_leaf_key, cid, storage_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
ON CONFLICT (header_id, state_path, storage_path) DO UPDATE SET (storage_leaf_key, cid, node_type, diff, mh_key) = ($3, $4, $6, $7, $8)
*/
func (w *Writer) upsertStorageCID(tx Tx, storageCID models.StorageNodeModel) error {
func (w *Writer) InsertStorageCID(tx interfaces.Tx, storageCID *models.StorageNodeModel) error {
var storageKey string
if storageCID.StorageKey != nullHash.String() {
storageKey = storageCID.StorageKey
}
_, err := tx.Exec(w.db.Context(), w.db.InsertStorageStm(),
_, err := tx.Exec(w.DB.Context(), w.DB.InsertStorageStm(),
storageCID.HeaderID, storageCID.StatePath, storageKey, storageCID.CID, storageCID.Path, storageCID.NodeType,
true, storageCID.MhKey)
if err != nil {
return fmt.Errorf("error upserting storage_cids entry: %v", err)
return fmt.Errorf("v3 error inserting storage_cids entry: %v", err)
}
return nil
}

73
statediff/indexer/interfaces/interfaces.go
View File

@ -17,6 +17,7 @@
package interfaces
import (
"context"
"io"
"math/big"
"time"
@ -28,10 +29,11 @@ import (
// StateDiffIndexer interface required to index statediff data
type StateDiffIndexer interface {
PushBlock(block *types.Block, receipts types.Receipts, totalDifficulty *big.Int) (Batch, error)
PushStateNode(tx Batch, stateNode sdtypes.StateNode, headerID string) error
PushBlock(block *types.Block, receipts types.Receipts, totalDifficulty *big.Int) (Batch, int64, error)
PushStateNode(tx Batch, stateNode sdtypes.StateNode, headerHash string, headerID int64) error
PushCodeAndCodeHash(tx Batch, codeAndCodeHash sdtypes.CodeAndCodeHash) error
ReportDBMetrics(delay time.Duration, quit <-chan bool)
ReportOldDBMetrics(delay time.Duration, quit <-chan bool)
ReportNewDBMetrics(delay time.Duration, quit <-chan bool)
io.Closer
}
@ -44,3 +46,68 @@ type Batch interface {
type Config interface {
Type() shared.DBType
}
// Database interfaces required by the sql indexer
type Database interface {
Driver
Statements
Version() uint
}
// Driver interface has all the methods required by a driver implementation to support the sql indexer
type Driver interface {
QueryRow(ctx context.Context, sql string, args ...interface{}) ScannableRow
Exec(ctx context.Context, sql string, args ...interface{}) (Result, error)
Select(ctx context.Context, dest interface{}, query string, args ...interface{}) error
Get(ctx context.Context, dest interface{}, query string, args ...interface{}) error
Begin(ctx context.Context) (Tx, error)
Stats() Stats
Context() context.Context
io.Closer
}
// Statements interface to accommodate different SQL query syntax
type Statements interface {
InsertNodeInfoStm() string
InsertHeaderStm() string
InsertUncleStm() string
InsertTxStm() string
InsertAccessListElementStm() string
InsertRctStm() string
InsertLogStm() string
InsertStateStm() string
InsertAccountStm() string
InsertStorageStm() string
InsertIPLDStm() string
InsertIPLDsStm() string
}
// Tx interface to accommodate different concrete SQL transaction types
type Tx interface {
QueryRow(ctx context.Context, sql string, args ...interface{}) ScannableRow
Exec(ctx context.Context, sql string, args ...interface{}) (Result, error)
Commit(ctx context.Context) error
Rollback(ctx context.Context) error
}
// ScannableRow interface to accommodate different concrete row types
type ScannableRow interface {
Scan(dest ...interface{}) error
}
// Result interface to accommodate different concrete result types
type Result interface {
RowsAffected() (int64, error)
}
// Stats interface to accommodate different concrete sql stats types
type Stats interface {
MaxOpen() int64
Open() int64
InUse() int64
Idle() int64
WaitCount() int64
WaitDuration() time.Duration
MaxIdleClosed() int64
MaxLifetimeClosed() int64
}

7
statediff/indexer/mocks/test_data.go
View File

@ -22,13 +22,14 @@ import (
"crypto/rand"
"math/big"
v3Models "github.com/ethereum/go-ethereum/statediff/indexer/models/v3"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/statediff/indexer/models"
"github.com/ethereum/go-ethereum/statediff/test_helpers"
sdtypes "github.com/ethereum/go-ethereum/statediff/types"
"github.com/ethereum/go-ethereum/trie"
@ -105,11 +106,11 @@ var (
Address: AnotherAddress,
StorageKeys: []common.Hash{common.BytesToHash(StorageLeafKey), common.BytesToHash(MockStorageLeafKey)},
}
AccessListEntry1Model = models.AccessListElementModel{
AccessListEntry1Model = v3Models.AccessListElementModel{
Index: 0,
Address: Address.Hex(),
}
AccessListEntry2Model = models.AccessListElementModel{
AccessListEntry2Model = v3Models.AccessListElementModel{
Index: 1,
Address: AnotherAddress.Hex(),
StorageKeys: []string{common.BytesToHash(StorageLeafKey).Hex(), common.BytesToHash(MockStorageLeafKey).Hex()},

23
statediff/indexer/models/shared/batch.go Normal file
View File

@ -0,0 +1,23 @@
// VulcanizeDB
// Copyright © 2021 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package models
// IPLDBatch holds the arguments for a batch insert of IPLD data
type IPLDBatch struct {
Keys []string
Values [][]byte
}

23
statediff/indexer/models/shared/models.go Normal file
View File

@ -0,0 +1,23 @@
// VulcanizeDB
// Copyright © 2021 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package models
// IPLDModel is the db model for public.blocks
type IPLDModel struct {
Key string `db:"key"`
Data []byte `db:"data"`
}

128
statediff/indexer/models/v2/batch.go Normal file
View File

@ -0,0 +1,128 @@
// VulcanizeDB
// Copyright © 2021 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package models
import "github.com/lib/pq"
// UncleBatch is the db model for eth.uncle_cids
type UncleBatch struct {
IDs []int64
HeaderIDs []int64
BlockHashes []string
ParentHashes []string
CIDs []string
MhKeys []string
Rewards []string
}
// TxBatch is the db model for eth.transaction_cids
type TxBatch struct {
IDs []int64
HeaderIDs []int64
Indexes []int64
TxHashes []string
CIDs []string
MhKeys []string
Dsts []string
Srcs []string
Data [][]byte
Types []uint8
}
// AccessListElementBatch is the db model for eth.access_list_entry
type AccessListElementBatch struct {
IDs []int64
Indexes []int64
TxIDs []int64
Addresses []string
StorageKeys []pq.StringArray
}
// ReceiptBatch is the db model for eth.receipt_cids
type ReceiptBatch struct {
IDs []int64
TxIDs []int64
LeafCIDs []string
LeafMhKeys []string
PostStatuses []int64
PostStates []string
Contracts []string
ContractHashes []string
LogRoots []string
}
// StateNodeBatch is the db model for eth.state_cids
type StateNodeBatch struct {
IDs []int64
HeaderIDs []int64
Paths [][]byte
StateKeys []string
NodeTypes []int
CIDs []string
MhKeys []string
Diffs []bool
}
// StorageNodeBatch is the db model for eth.storage_cids
type StorageNodeBatch struct {
IDs []int64
StateIDs []int64
Paths [][]byte
StorageKeys []string
NodeTypes []int
CIDs []string
MhKeys []string
Diffs []bool
}
// StorageNodeWithStateKeyBatch is a db model for eth.storage_cids + eth.state_cids.state_key
type StorageNodeWithStateKeyBatch struct {
IDs []int64
StateIDs []int64
Paths [][]byte
StateKeys []string
StorageKeys []string
NodeTypes []int
CIDs []string
MhKeys []string
Diffs []bool
}
// StateAccountBatch is a db model for an eth state account (decoded value of state leaf node)
type StateAccountBatch struct {
IDs []int64
StateIDs []int64
Balances []string
Nonces []int64
CodeHashes [][]byte
StorageRoots []string
}
// LogsBatch is the db model for eth.logs
type LogsBatch struct {
IDs []int64
LeafCIDs []string
LeafMhKeys []string
ReceiptIDs []int64
Addresses []string
Indexes []int64
Data [][]byte
Topic0s []string
Topic1s []string
Topic2s []string
Topic3s []string
}

149
statediff/indexer/models/v2/models.go Normal file
View File

@ -0,0 +1,149 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package models
import "github.com/lib/pq"
// HeaderModel is the db model for eth.header_cids
type HeaderModel struct {
ID int64 `db:"id"`
BlockNumber string `db:"block_number"`
BlockHash string `db:"block_hash"`
ParentHash string `db:"parent_hash"`
CID string `db:"cid"`
MhKey string `db:"mh_key"`
TotalDifficulty string `db:"td"`
NodeID int64 `db:"node_id"`
Reward string `db:"reward"`
StateRoot string `db:"state_root"`
UncleRoot string `db:"uncle_root"`
TxRoot string `db:"tx_root"`
RctRoot string `db:"receipt_root"`
Bloom []byte `db:"bloom"`
Timestamp uint64 `db:"timestamp"`
TimesValidated int64 `db:"times_validated"`
BaseFee *string `db:"base_fee"`
}
// UncleModel is the db model for eth.uncle_cids
type UncleModel struct {
ID int64 `db:"id"`
HeaderID int64 `db:"header_id"`
BlockHash string `db:"block_hash"`
ParentHash string `db:"parent_hash"`
CID string `db:"cid"`
MhKey string `db:"mh_key"`
Reward string `db:"reward"`
}
// TxModel is the db model for eth.transaction_cids
type TxModel struct {
ID int64 `db:"id"`
HeaderID int64 `db:"header_id"`
Index int64 `db:"index"`
TxHash string `db:"tx_hash"`
CID string `db:"cid"`
MhKey string `db:"mh_key"`
Dst string `db:"dst"`
Src string `db:"src"`
Data []byte `db:"tx_data"`
Type uint8 `db:"tx_type"`
}
// AccessListElementModel is the db model for eth.access_list_entry
type AccessListElementModel struct {
ID int64 `db:"id"`
Index int64 `db:"index"`
TxID int64 `db:"tx_id"`
Address string `db:"address"`
StorageKeys pq.StringArray `db:"storage_keys"`
}
// ReceiptModel is the db model for eth.receipt_cids
type ReceiptModel struct {
ID int64 `db:"id"`
TxID int64 `db:"tx_id"`
LeafCID string `db:"leaf_cid"`
LeafMhKey string `db:"leaf_mh_key"`
PostStatus uint64 `db:"post_status"`
PostState string `db:"post_state"`
Contract string `db:"contract"`
ContractHash string `db:"contract_hash"`
LogRoot string `db:"log_root"`
}
// StateNodeModel is the db model for eth.state_cids
type StateNodeModel struct {
ID int64 `db:"id"`
HeaderID int64 `db:"header_id"`
Path []byte `db:"state_path"`
StateKey string `db:"state_leaf_key"`
NodeType int `db:"node_type"`
CID string `db:"cid"`
MhKey string `db:"mh_key"`
Diff bool `db:"diff"`
}
// StorageNodeModel is the db model for eth.storage_cids
type StorageNodeModel struct {
ID int64 `db:"id"`
StateID int64 `db:"state_id"`
Path []byte `db:"storage_path"`
StorageKey string `db:"storage_leaf_key"`
NodeType int `db:"node_type"`
CID string `db:"cid"`
MhKey string `db:"mh_key"`
Diff bool `db:"diff"`
}
// StorageNodeWithStateKeyModel is a db model for eth.storage_cids + eth.state_cids.state_key
type StorageNodeWithStateKeyModel struct {
ID int64 `db:"id"`
StateID int64 `db:"state_id"`
Path []byte `db:"storage_path"`
StateKey string `db:"state_leaf_key"`
StorageKey string `db:"storage_leaf_key"`
NodeType int `db:"node_type"`
CID string `db:"cid"`
MhKey string `db:"mh_key"`
Diff bool `db:"diff"`
}
// StateAccountModel is a db model for an eth state account (decoded value of state leaf node)
type StateAccountModel struct {
ID int64 `db:"id"`
StateID int64 `db:"state_id"`
Balance string `db:"balance"`
Nonce uint64 `db:"nonce"`
CodeHash []byte `db:"code_hash"`
StorageRoot string `db:"storage_root"`
}
// LogsModel is the db model for eth.logs
type LogsModel struct {
ID int64 `db:"id"`
LeafCID string `db:"leaf_cid"`
LeafMhKey string `db:"leaf_mh_key"`
ReceiptID int64 `db:"receipt_id"`
Address string `db:"address"`
Index int64 `db:"index"`
Data []byte `db:"log_data"`
Topic0 string `db:"topic0"`
Topic1 string `db:"topic1"`
Topic2 string `db:"topic2"`
Topic3 string `db:"topic3"`
}

6
statediff/indexer/models/batch.go → statediff/indexer/models/v3/batch.go
View File

@ -18,12 +18,6 @@ package models
import "github.com/lib/pq"
// IPLDBatch holds the arguments for a batch insert of IPLD data
type IPLDBatch struct {
Keys []string
Values [][]byte
}
// UncleBatch holds the arguments for a batch insert of uncle data
type UncleBatch struct {
HeaderID []string

6
statediff/indexer/models/models.go → statediff/indexer/models/v3/models.go
View File

@ -18,12 +18,6 @@ package models
import "github.com/lib/pq"
// IPLDModel is the db model for public.blocks
type IPLDModel struct {
Key string `db:"key"`
Data []byte `db:"data"`
}
// HeaderModel is the db model for eth.header_cids
type HeaderModel struct {
BlockNumber string `db:"block_number"`

15
statediff/service.go
View File

@ -18,6 +18,7 @@ package statediff
import (
"bytes"
"fmt"
"math/big"
"strconv"
"strings"
@ -162,9 +163,10 @@ func New(stack *node.Node, ethServ *eth.Ethereum, cfg *ethconfig.Config, params
var err error
indexer, err = ind.NewStateDiffIndexer(params.Context, blockChain.Config(), info, params.IndexerConfig)
if err != nil {
return err
return fmt.Errorf("unable to initialize a new statediff indexer: %v", err)
}
indexer.ReportDBMetrics(10*time.Second, quitCh)
indexer.ReportOldDBMetrics(10*time.Second, quitCh)
indexer.ReportNewDBMetrics(10*time.Second, quitCh)
}
workers := params.NumWorkers
if workers == 0 {
@ -234,7 +236,7 @@ func (sds *Service) WriteLoop(chainEventCh chan core.ChainEvent) {
chainEventSub := sds.BlockChain.SubscribeChainEvent(chainEventCh)
defer chainEventSub.Unsubscribe()
errCh := chainEventSub.Err()
var wg sync.WaitGroup
wg := new(sync.WaitGroup)
// Process metrics for chain events, then forward to workers
chainEventFwd := make(chan core.ChainEvent, chainEventChanSize)
wg.Add(1)
@ -265,7 +267,7 @@ func (sds *Service) WriteLoop(chainEventCh chan core.ChainEvent) {
}()
wg.Add(int(sds.numWorkers))
for worker := uint(0); worker < sds.numWorkers; worker++ {
params := workerParams{chainEventCh: chainEventFwd, wg: &wg, id: worker}
params := workerParams{chainEventCh: chainEventFwd, wg: wg, id: worker}
go sds.writeLoopWorker(params)
}
wg.Wait()
@ -661,6 +663,7 @@ func (sds *Service) writeStateDiff(block *types.Block, parentRoot common.Hash, p
// log.Info("Writing state diff", "block height", block.Number().Uint64())
var totalDifficulty *big.Int
var receipts types.Receipts
var headerID int64
var err error
var tx interfaces.Batch
if params.IncludeTD {
@ -669,7 +672,7 @@ func (sds *Service) writeStateDiff(block *types.Block, parentRoot common.Hash, p
if params.IncludeReceipts {
receipts = sds.BlockChain.GetReceiptsByHash(block.Hash())
}
tx, err = sds.indexer.PushBlock(block, receipts, totalDifficulty)
tx, headerID, err = sds.indexer.PushBlock(block, receipts, totalDifficulty)
if err != nil {
return err
}
@ -680,7 +683,7 @@ func (sds *Service) writeStateDiff(block *types.Block, parentRoot common.Hash, p
}
}()
output := func(node types2.StateNode) error {
return sds.indexer.PushStateNode(tx, node, block.Hash().String())
return sds.indexer.PushStateNode(tx, node, block.Hash().String(), headerID)
}
codeOutput := func(c types2.CodeAndCodeHash) error {
return sds.indexer.PushCodeAndCodeHash(tx, c)