Initial plugin implementation

* refactor packages, flags, subscriptions
* DRY refactor builder tests
* use mockgen to generate mocks
* update README
* MODE=statediff no longer needed for unit tests
* simplify func names, clean up metrics
* move write params to service field
* sql indexer: confirm quit after ipld cache reset
  prevents negative waitgroup panic
* don't let TotalDifficulty become nil
* use forked plugeth, plugeth-utils for now

indexer/database/sql/postgres/config.go

@@ -23,49 +23,9 @@ import (
 	"strings"
 	"time"
 
-	"github.com/ethereum/go-ethereum/statediff/indexer/shared"
+	"github.com/cerc-io/plugeth-statediff/indexer/shared"
 )
 
-// DriverType to explicitly type the kind of sql driver we are using
-type DriverType string
-
-const (
-	PGX     DriverType = "PGX"
-	SQLX    DriverType = "SQLX"
-	Unknown DriverType = "Unknown"
-)
-
-// Env variables
-const (
-	DATABASE_NAME     = "DATABASE_NAME"
-	DATABASE_HOSTNAME = "DATABASE_HOSTNAME"
-	DATABASE_PORT     = "DATABASE_PORT"
-	DATABASE_USER     = "DATABASE_USER"
-	DATABASE_PASSWORD = "DATABASE_PASSWORD"
-)
-
-// ResolveDriverType resolves a DriverType from a provided string
-func ResolveDriverType(str string) (DriverType, error) {
-	switch strings.ToLower(str) {
-	case "pgx", "pgxpool":
-		return PGX, nil
-	case "sqlx":
-		return SQLX, nil
-	default:
-		return Unknown, fmt.Errorf("unrecognized driver type string: %s", str)
-	}
-}
-
-// TestConfig specifies default parameters for connecting to a testing DB
-var TestConfig = Config{
-	Hostname:     "localhost",
-	Port:         8077,
-	DatabaseName: "cerc_testing",
-	Username:     "vdbm",
-	Password:     "password",
-	Driver:       SQLX,
-}
-
 // Config holds params for a Postgres db
 type Config struct {
 	// conn string params
@@ -98,6 +58,34 @@ type Config struct {
 	CopyFrom bool
 }
 
+// DriverType to explicitly type the kind of sql driver we are using
+type DriverType string
+
+const (
+	PGX     DriverType = "PGX"
+	SQLX    DriverType = "SQLX"
+	Invalid DriverType = "Invalid"
+)
+
+// Env variables
+const (
+	DATABASE_NAME     = "DATABASE_NAME"
+	DATABASE_HOSTNAME = "DATABASE_HOSTNAME"
+	DATABASE_PORT     = "DATABASE_PORT"
+	DATABASE_USER     = "DATABASE_USER"
+	DATABASE_PASSWORD = "DATABASE_PASSWORD"
+)
+
+// TestConfig specifies default parameters for connecting to a testing DB
+var TestConfig = Config{
+	Hostname:     "localhost",
+	Port:         8077,
+	DatabaseName: "cerc_testing",
+	Username:     "vdbm",
+	Password:     "password",
+	Driver:       SQLX,
+}
+
 // Type satisfies interfaces.Config
 func (c Config) Type() shared.DBType {
 	return shared.POSTGRES
@@ -116,6 +104,7 @@ func (c Config) DbConnectionString() string {
 	return fmt.Sprintf("postgresql://%s:%d/%s?sslmode=disable", c.Hostname, c.Port, c.DatabaseName)
 }
 
+// WithEnv overrides the config with env variables, returning a new instance
 func (c Config) WithEnv() (Config, error) {
 	if val := os.Getenv(DATABASE_NAME); val != "" {
 		c.DatabaseName = val
@@ -138,3 +127,26 @@ func (c Config) WithEnv() (Config, error) {
 	}
 	return c, nil
 }
+
+// ResolveDriverType resolves a DriverType from a provided string
+func ResolveDriverType(str string) (DriverType, error) {
+	switch strings.ToLower(str) {
+	case "pgx", "pgxpool":
+		return PGX, nil
+	case "sqlx":
+		return SQLX, nil
+	default:
+		return Invalid, fmt.Errorf("unrecognized driver type string: %s", str)
+	}
+}
+
+// Set satisfies flag.Value
+func (d *DriverType) Set(v string) (err error) {
+	*d, err = ResolveDriverType(v)
+	return
+}
+
+// String satisfies flag.Value
+func (d *DriverType) String() string {
+	return strings.ToLower(string(*d))
+}

indexer/database/sql/postgres/database.go

@@ -17,8 +17,8 @@
 package postgres
 
 import (
-	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
-	"github.com/ethereum/go-ethereum/statediff/indexer/shared/schema"
+	"github.com/cerc-io/plugeth-statediff/indexer/database/sql"
+	"github.com/cerc-io/plugeth-statediff/indexer/shared/schema"
 )
 
 var _ sql.Database = &DB{}

indexer/database/sql/postgres/log_adapter.go

@@ -18,8 +18,9 @@ package postgres
 import (
 	"context"
 
-	"github.com/ethereum/go-ethereum/log"
 	"github.com/jackc/pgx/v4"
+
+	"github.com/cerc-io/plugeth-statediff/utils/log"
 )
 
 type LogAdapter struct {
@@ -31,31 +32,26 @@ func NewLogAdapter(l log.Logger) *LogAdapter {
 }
 
 func (l *LogAdapter) Log(ctx context.Context, level pgx.LogLevel, msg string, data map[string]interface{}) {
-	var logger log.Logger
-	if data != nil {
-		var args = make([]interface{}, 0)
-		for key, value := range data {
-			if value != nil {
-				args = append(args, key, value)
-			}
+	args := make([]interface{}, 0)
+	for key, value := range data {
+		if value != nil {
+			args = append(args, key, value)
 		}
-		logger = l.l.New(args...)
-	} else {
-		logger = l.l
 	}
 
+	logger := l.l
 	switch level {
 	case pgx.LogLevelTrace:
-		logger.Trace(msg)
+		logger.Trace(msg, args...)
 	case pgx.LogLevelDebug:
-		logger.Debug(msg)
+		logger.Debug(msg, args...)
 	case pgx.LogLevelInfo:
-		logger.Info(msg)
+		logger.Info(msg, args...)
 	case pgx.LogLevelWarn:
-		logger.Warn(msg)
+		logger.Warn(msg, args...)
 	case pgx.LogLevelError:
-		logger.Error(msg)
+		logger.Error(msg, args...)
 	default:
-		logger.New("INVALID_PGX_LOG_LEVEL", level).Error(msg)
+		logger.Error(msg, "INVALID_PGX_LOG_LEVEL", level)
 	}
 }

indexer/database/sql/postgres/pgx.go

@@ -20,16 +20,16 @@ import (
 	"context"
 	"time"
 
-	"github.com/ethereum/go-ethereum/log"
+	"github.com/cerc-io/plugeth-statediff/utils/log"
 
 	"github.com/georgysavva/scany/pgxscan"
 	"github.com/jackc/pgconn"
 	"github.com/jackc/pgx/v4"
 	"github.com/jackc/pgx/v4/pgxpool"
 
-	"github.com/ethereum/go-ethereum/statediff/indexer/database/metrics"
-	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
-	"github.com/ethereum/go-ethereum/statediff/indexer/node"
+	"github.com/cerc-io/plugeth-statediff/indexer/database/metrics"
+	"github.com/cerc-io/plugeth-statediff/indexer/database/sql"
+	"github.com/cerc-io/plugeth-statediff/indexer/node"
 )
 
 // PGXDriver driver, implements sql.Driver
@@ -96,7 +96,7 @@ func MakeConfig(config Config) (*pgxpool.Config, error) {
 	}
 
 	if config.LogStatements {
-		conf.ConnConfig.Logger = NewLogAdapter(log.New())
+		conf.ConnConfig.Logger = NewLogAdapter(log.DefaultLogger)
 	}
 
 	return conf, nil
@@ -106,8 +106,10 @@ func (pgx *PGXDriver) createNode() error {
 	_, err := pgx.pool.Exec(
 		pgx.ctx,
 		createNodeStm,
-		pgx.nodeInfo.GenesisBlock, pgx.nodeInfo.NetworkID,
-		pgx.nodeInfo.ID, pgx.nodeInfo.ClientName,
+		pgx.nodeInfo.GenesisBlock,
+		pgx.nodeInfo.NetworkID,
+		pgx.nodeInfo.ID,
+		pgx.nodeInfo.ClientName,
 		pgx.nodeInfo.ChainID)
 	if err != nil {
 		return ErrUnableToSetNode(err)

indexer/database/sql/postgres/pgx_test.go

@@ -26,8 +26,8 @@ import (
 	"github.com/jackc/pgx/v4/pgxpool"
 	"github.com/stretchr/testify/require"
 
-	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
-	"github.com/ethereum/go-ethereum/statediff/indexer/node"
+	"github.com/cerc-io/plugeth-statediff/indexer/database/sql/postgres"
+	"github.com/cerc-io/plugeth-statediff/indexer/node"
 )
 
 var (

indexer/database/sql/postgres/postgres_suite_test.go

@@ -1,33 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package postgres_test
-
-import (
-	"fmt"
-	"os"
-
-	"github.com/ethereum/go-ethereum/log"
-)
-
-func init() {
-	if os.Getenv("MODE") != "statediff" {
-		fmt.Println("Skipping statediff test")
-		os.Exit(0)
-	}
-
-	log.Root().SetHandler(log.DiscardHandler())
-}

indexer/database/sql/postgres/sqlx.go

@@ -24,9 +24,9 @@ import (
 
 	"github.com/jmoiron/sqlx"
 
-	"github.com/ethereum/go-ethereum/statediff/indexer/database/metrics"
-	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
-	"github.com/ethereum/go-ethereum/statediff/indexer/node"
+	"github.com/cerc-io/plugeth-statediff/indexer/database/metrics"
+	"github.com/cerc-io/plugeth-statediff/indexer/database/sql"
+	"github.com/cerc-io/plugeth-statediff/indexer/node"
 )
 
 // SQLXDriver driver, implements sql.Driver

indexer/database/sql/postgres/sqlx_test.go

@@ -26,8 +26,8 @@ import (
 	_ "github.com/lib/pq"
 	"github.com/stretchr/testify/require"
 
-	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
-	"github.com/ethereum/go-ethereum/statediff/indexer/node"
+	"github.com/cerc-io/plugeth-statediff/indexer/database/sql/postgres"
+	"github.com/cerc-io/plugeth-statediff/indexer/node"
 )
 
 func TestPostgresSQLX(t *testing.T) {

indexer/database/sql/postgres/test_helpers.go

@@ -19,8 +19,8 @@ package postgres
 import (
 	"context"
 
-	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
-	"github.com/ethereum/go-ethereum/statediff/indexer/node"
+	"github.com/cerc-io/plugeth-statediff/indexer/database/sql"
+	"github.com/cerc-io/plugeth-statediff/indexer/node"
 )
 
 // SetupSQLXDB is used to setup a sqlx db for tests