update to use new version of statediffing geth pkgs with public.blocks insert batching

bump Go version to 1.16
Merge pull request #52 from vulcanize/add-grafana-dashboard
2021-11-08 19:08:36 -06:00 · 2021-11-08 19:05:35 -06:00 · 2021-11-04 10:09:21 -05:00 · 2021-11-01 19:39:00 +05:30 · 2021-10-28 10:52:35 -05:00 · 2021-10-28 10:21:10 -05:00
13 changed files with 649 additions and 32 deletions
@@ -1,4 +1,4 @@
-FROM golang:1.13-alpine as builder
+FROM golang:1.16-alpine as builder

 RUN apk --update --no-cache add make git g++ linux-headers
 # DEBUG
@@ -15,7 +15,7 @@ RUN GO111MODULE=on GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldfl
 FROM alpine

 ARG USER="vdm"
-ARG CONFIG_FILE="./environments/example.toml"
+ARG CONFIG_FILE="./environments/config.toml"
 ARG EXPOSE_PORT=8545

 RUN adduser -Du 5000 $USER adm
@@ -29,10 +29,12 @@ const (
 	ETH_NETWORK_ID    = "ETH_NETWORK_ID"
 	ETH_CHAIN_ID      = "ETH_CHAIN_ID"

-	DB_CACHE_SIZE_MB            = "DB_CACHE_SIZE_MB"
-	TRIE_CACHE_SIZE_MB          = "TRIE_CACHE_SIZE_MB"
-	LVLDB_PATH                  = "LVLDB_PATH"
-	LVLDB_ANCIENT               = "LVLDB_ANCIENT"
+	DB_CACHE_SIZE_MB   = "DB_CACHE_SIZE_MB"
+	TRIE_CACHE_SIZE_MB = "TRIE_CACHE_SIZE_MB"
+	LVLDB_PATH         = "LVLDB_PATH"
+	LVLDB_ANCIENT      = "LVLDB_ANCIENT"
+
+	STATEDIFF_PRERUN            = "STATEDIFF_PRERUN"
 	STATEDIFF_TRIE_WORKERS      = "STATEDIFF_TRIE_WORKERS"
 	STATEDIFF_SERVICE_WORKERS   = "STATEDIFF_SERVICE_WORKERS"
 	STATEDIFF_WORKER_QUEUE_SIZE = "STATEDIFF_WORKER_QUEUE_SIZE"
@@ -44,6 +46,14 @@ const (
 	PROM_HTTP      = "PROM_HTTP"
 	PROM_HTTP_ADDR = "PROM_HTTP_ADDR"
 	PROM_HTTP_PORT = "PROM_HTTP_PORT"
+	PROM_DB_STATS  = "PROM_DB_STATS"
+
+	PRERUN_ONLY        = "PRERUN_ONLY"
+	PRERUN_RANGE_START = "PRERUN_RANGE_START"
+	PRERUN_RANGE_STOP  = "PRERUN_RANGE_STOP"
+
+	LOG_LEVEL     = "LOG_LEVEL"
+	LOG_FILE_PATH = "LOG_FILE_PATH"
 )

 // Bind env vars for eth node and DB configuration
@@ -76,8 +86,17 @@ func init() {
 	viper.BindEnv("prom.http", PROM_HTTP)
 	viper.BindEnv("prom.httpAddr", PROM_HTTP_ADDR)
 	viper.BindEnv("prom.httpPort", PROM_HTTP_PORT)
+	viper.BindEnv("prom.dbStats", PROM_DB_STATS)

 	viper.BindEnv("statediff.serviceWorkers", STATEDIFF_SERVICE_WORKERS)
 	viper.BindEnv("statediff.trieWorkers", STATEDIFF_TRIE_WORKERS)
 	viper.BindEnv("statediff.workerQueueSize", STATEDIFF_WORKER_QUEUE_SIZE)
+
+	viper.BindEnv("statediff.prerun", STATEDIFF_PRERUN)
+	viper.BindEnv("prerun.only", PRERUN_ONLY)
+	viper.BindEnv("prerun.start", PRERUN_RANGE_START)
+	viper.BindEnv("prerun.stop", PRERUN_RANGE_STOP)
+
+	viper.BindEnv("log.level", LOG_LEVEL)
+	viper.BindEnv("log.file", LOG_FILE_PATH)
 }
@@ -18,8 +18,10 @@ package cmd

 import (
 	"fmt"
+	"math/rand"
 	"os"
 	"strings"
+	"time"

 	"github.com/ethereum/go-ethereum/statediff/indexer/node"
 	"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
@@ -84,7 +86,6 @@ func initFuncs(cmd *cobra.Command, args []string) {
 }

 func logLevel() error {
-	viper.BindEnv("log.level", "LOGRUS_LEVEL")
 	lvl, err := log.ParseLevel(viper.GetString("log.level"))
 	if err != nil {
 		return err
@@ -136,8 +137,12 @@ func init() {
 	rootCmd.PersistentFlags().Bool("prom-http", false, "enable prometheus http service")
 	rootCmd.PersistentFlags().String("prom-http-addr", "127.0.0.1", "prometheus http host")
 	rootCmd.PersistentFlags().String("prom-http-port", "8080", "prometheus http port")
+	rootCmd.PersistentFlags().Bool("prom-db-stats", false, "enables prometheus db stats")
+	rootCmd.PersistentFlags().Bool("prom-metrics", false, "enable prometheus metrics")

-	rootCmd.PersistentFlags().Bool("metrics", false, "enable metrics")
+	rootCmd.PersistentFlags().Bool("prerun-only", false, "only process pre-configured ranges; exit afterwards")
+	rootCmd.PersistentFlags().Int("prerun-start", 0, "start height for a prerun range")
+	rootCmd.PersistentFlags().Int("prerun-stop", 0, "stop height for a prerun range")

 	viper.BindPFlag("server.httpPath", rootCmd.PersistentFlags().Lookup("http-path"))
 	viper.BindPFlag("server.ipcPath", rootCmd.PersistentFlags().Lookup("ipc-path"))
@@ -164,7 +169,13 @@ func init() {
 	viper.BindPFlag("prom.http", rootCmd.PersistentFlags().Lookup("prom-http"))
 	viper.BindPFlag("prom.httpAddr", rootCmd.PersistentFlags().Lookup("prom-http-addr"))
 	viper.BindPFlag("prom.httpPort", rootCmd.PersistentFlags().Lookup("prom-http-port"))
-	viper.BindPFlag("prom.metrics", rootCmd.PersistentFlags().Lookup("metrics"))
+	viper.BindPFlag("prom.dbStats", rootCmd.PersistentFlags().Lookup("prom-db-stats"))
+	viper.BindPFlag("prom.metrics", rootCmd.PersistentFlags().Lookup("prom-metrics"))
+	viper.BindPFlag("prerun.only", rootCmd.PersistentFlags().Lookup("prerun-only"))
+	viper.BindPFlag("prerun.start", rootCmd.PersistentFlags().Lookup("prerun-start"))
+	viper.BindPFlag("prerun.stop", rootCmd.PersistentFlags().Lookup("prerun-stop"))
+
+	rand.Seed(time.Now().UnixNano())
 }

 func initConfig() {
@@ -181,13 +192,50 @@ func initConfig() {
 }

 func GetEthNodeInfo() node.Info {
-	return node.Info{
-		ID:           viper.GetString("ethereum.nodeID"),
-		ClientName:   viper.GetString("ethereum.clientName"),
-		GenesisBlock: viper.GetString("ethereum.genesisBlock"),
-		NetworkID:    viper.GetString("ethereum.networkID"),
-		ChainID:      viper.GetUint64("ethereum.chainID"),
+	var nodeID, genesisBlock, networkID, clientName string
+	var chainID uint64
+	if !viper.IsSet("ethereum.nodeID") {
+		nodeID = randSeq(12)
+	} else {
+		nodeID = viper.GetString("ethereum.nodeID")
 	}
+	if !viper.IsSet("ethereum.genesisBlock") {
+		genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3"
+	} else {
+		genesisBlock = viper.GetString("ethereum.genesisBlock")
+	}
+	if !viper.IsSet("ethereum.chainID") {
+		chainID = 1
+	} else {
+		chainID = viper.GetUint64("ethereum.chainID")
+	}
+	if !viper.IsSet("ethereum.networkID") {
+		networkID = "1"
+	} else {
+		networkID = viper.GetString("ethereum.networkID")
+	}
+	if !viper.IsSet("ethereum.clientName") {
+		clientName = "eth-statediff-service"
+	} else {
+		clientName = viper.GetString("ethereum.clientName")
+	}
+	return node.Info{
+		ID:           nodeID,
+		ClientName:   clientName,
+		GenesisBlock: genesisBlock,
+		NetworkID:    networkID,
+		ChainID:      chainID,
+	}
+}
+
+var characters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")
+
+func randSeq(n int) string {
+	b := make([]rune, n)
+	for i := range b {
+		b[i] = characters[rand.Intn(len(characters))]
+	}
+	return string(b)
 }

 func GetDBParams() postgres.ConnectionParams {
@@ -55,6 +55,14 @@ func serve() {
 		logWithCommand.Fatal(err)
 	}

+	// short circuit if we only want to perform prerun
+	if viper.GetBool("prerun.only") {
+		if err := statediffService.Run(nil); err != nil {
+			logWithCommand.Fatal("unable to perform prerun: %v", err)
+		}
+		return
+	}
+
 	// start service and servers
 	logWithCommand.Info("Starting statediff service")
 	wg := new(sync.WaitGroup)
@@ -52,15 +52,17 @@ func createStateDiffService() (sd.StateDiffService, error) {
 	}

 	// create statediff service
-	logWithCommand.Info("Creating statediff service")
+	logWithCommand.Info("Setting up Postgres DB")
 	db, err := setupPostgres(nodeInfo)
 	if err != nil {
 		logWithCommand.Fatal(err)
 	}
+	logWithCommand.Info("Creating statediff indexer")
 	indexer, err := ind.NewStateDiffIndexer(chainConf, db)
 	if err != nil {
 		logWithCommand.Fatal(err)
 	}
+	logWithCommand.Info("Creating statediff service")
 	sdConf := sd.Config{
 		ServiceWorkers:  viper.GetUint("statediff.serviceWorkers"),
 		TrieWorkers:     viper.GetUint("statediff.trieWorkers"),
@@ -71,12 +73,16 @@ func createStateDiffService() (sd.StateDiffService, error) {
 }

 func setupPostgres(nodeInfo node.Info) (*postgres.DB, error) {
-	params := GetDBParams()
-	db, err := postgres.NewDB(postgres.DbConnectionString(params), GetDBConfig(), nodeInfo)
+	p := GetDBParams()
+	logWithCommand.Info("initializing DB connection pool")
+	db, err := postgres.NewDB(postgres.DbConnectionString(p), GetDBConfig(), nodeInfo)
 	if err != nil {
 		return nil, err
 	}
-	prom.RegisterDBCollector(params.Name, db.DB)
+	if viper.GetBool("prom.dbStats") {
+		logWithCommand.Info("registering DB collector")
+		prom.RegisterDBCollector(p.Name, db.DB)
+	}
 	return db, nil
 }

@@ -116,6 +122,16 @@ func setupPreRunRanges() []sd.RangeRequest {
 			Params: preRunParams,
 		}
 	}
+	if viper.IsSet("prerun.start") && viper.IsSet("prerun.stop") {
+		hardStart := viper.GetInt("prerun.start")
+		hardStop := viper.GetInt("prerun.stop")
+		blockRanges = append(blockRanges, sd.RangeRequest{
+			Start:  uint64(hardStart),
+			Stop:   uint64(hardStop),
+			Params: preRunParams,
+		})
+	}
+
 	return blockRanges
 }

@@ -6,7 +6,7 @@ services:
      context: ./
      cache_from:
      - alpine:latest
-      - golang:1.13
+      - golang:1.16
      dockerfile: ./Dockerfile
      args:
        USER: "vdbm"
@@ -0,0 +1,51 @@
+[leveldb]
+    path = "/app/geth-rw/chaindata"
+    ancient = "/app/geth-rw/chaindata/ancient"
+
+[server]
+    ipcPath = ""
+    httpPath = "0.0.0.0:8545"
+
+[statediff]
+    prerun = true
+    serviceWorkers = 1
+    workerQueueSize = 1024
+    trieWorkers = 16
+
+[prerun]
+    only = true
+    ranges = []
+    [prerun.params]
+        intermediateStateNodes = true
+        intermediateStorageNodes = true
+        includeBlock = true
+        includeReceipts = true
+        includeTD = true
+        includeCode = true
+        watchedAddresses = []
+        watchedStorageKeys = []
+
+[log]
+    file = ""
+    level = "info"
+
+[eth]
+    chainID = 1
+
+[database]
+    name     = ""
+    hostname = ""
+    port     = 5432
+    user     = ""
+    password = ""
+
+[cache]
+    database = 1024
+    trie = 4096
+
+[prom]
+    dbStats = false
+    metrics = true
+    http = true
+    httpAddr = "0.0.0.0"
+    httpPort = 9100
@@ -13,6 +13,7 @@
    trieWorkers = 4

 [prerun]
+    only = false
    ranges = [
        [0, 1000]
    ]
@@ -45,6 +46,7 @@
    trie = 1024

 [prom]
+    dbStats = false
    metrics = true
    http = true
    httpAddr = "localhost"
@@ -1,6 +1,6 @@
 module github.com/vulcanize/eth-statediff-service

-go 1.13
+go 1.16

 require (
 	github.com/ethereum/go-ethereum v1.10.9
@@ -12,4 +12,4 @@ require (
 	github.com/vulcanize/go-eth-state-node-iterator v0.0.1-alpha.0.20211014064906-d23d01ed8191
 )

-replace github.com/ethereum/go-ethereum v1.10.9 => github.com/vulcanize/go-ethereum v1.10.11-statediff-0.0.27
+replace github.com/ethereum/go-ethereum v1.10.9 => github.com/vulcanize/go-ethereum v1.10.11-statediff-0.0.27a
@@ -539,8 +539,8 @@ github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPU
 github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
 github.com/vulcanize/go-eth-state-node-iterator v0.0.1-alpha.0.20211014064906-d23d01ed8191 h1:+xSwb0xOLCAhejoai2BQZPTJW5Z+tU9xkTaOBwGnPoU=
 github.com/vulcanize/go-eth-state-node-iterator v0.0.1-alpha.0.20211014064906-d23d01ed8191/go.mod h1:FBMQp69PTnRg3kLx9qeWGfKYfVq4fpfbr69X8lEvvfs=
-github.com/vulcanize/go-ethereum v1.10.11-statediff-0.0.27 h1:ldAVLlKll2WHHKLNu8oKbkYRUVRs9T45z12KiSrRW24=
-github.com/vulcanize/go-ethereum v1.10.11-statediff-0.0.27/go.mod h1:9L+QY31AnWnX2/2HDOySCjQoYUdWNGBRMezFJVfH73E=
+github.com/vulcanize/go-ethereum v1.10.11-statediff-0.0.27a h1:syuVN9x53LTCf9N/6/xsIlUUjp2qU0AP8JzwNMtWxis=
+github.com/vulcanize/go-ethereum v1.10.11-statediff-0.0.27a/go.mod h1:9L+QY31AnWnX2/2HDOySCjQoYUdWNGBRMezFJVfH73E=
 github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc h1:9lDbC6Rz4bwmou+oE6Dt4Cb2BGMur5eR/GYptkKUVHo=
 github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM=
 github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
@@ -603,7 +603,6 @@ golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKG
 golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -817,7 +816,6 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
@@ -0,0 +1,449 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "gnetId": null,
+  "graphTooltip": 0,
+  "id": 4,
+  "iteration": 1635774999288,
+  "links": [
+    {
+      "icon": "info",
+      "tags": [],
+      "targetBlank": true,
+      "title": "Grafana Docs",
+      "tooltip": "",
+      "type": "link",
+      "url": "https://grafana.com/docs/grafana/latest/"
+    },
+    {
+      "icon": "info",
+      "tags": [],
+      "targetBlank": true,
+      "title": "Prometheus Docs",
+      "type": "link",
+      "url": "http://prometheus.io/docs/introduction/overview/"
+    }
+  ],
+  "panels": [
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "eth-statediff-service",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "links": [],
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 6,
+        "w": 18,
+        "x": 0,
+        "y": 0
+      },
+      "hiddenSeries": false,
+      "id": 3,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "percentage": false,
+      "pluginVersion": "7.1.5",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "rate(eth_statediff_service_processed_height{instance=\"$instance\"}[1m])",
+          "interval": "",
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Instance block processed (rate-1m)",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "content": "<span style=\"font-family: 'Open Sans', 'Helvetica Neue', Helvetica; font-size: 25px;vertical-align: text-top;color: #bbbfc2;margin-left: 10px;\">Prometheus</span>\n\n<p style=\"margin-top: 10px;\">You're using Prometheus, an open-source systems monitoring and alerting toolkit originally built at SoundCloud. For more information, check out the <a href=\"https://grafana.com/\">Grafana</a> and <a href=\"http://prometheus.io/\">Prometheus</a> projects.</p>",
+      "datasource": null,
+      "editable": true,
+      "error": false,
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 5,
+        "w": 6,
+        "x": 18,
+        "y": 0
+      },
+      "id": 9,
+      "links": [],
+      "mode": "html",
+      "options": {
+        "content": "<span style=\"font-family: 'Open Sans', 'Helvetica Neue', Helvetica; font-size: 25px;vertical-align: text-top;color: #bbbfc2;margin-left: 10px;\">Prometheus</span>\n\n<p style=\"margin-top: 10px;\">You're using Prometheus, an open-source systems monitoring and alerting toolkit originally built at SoundCloud. For more information, check out the <a href=\"https://grafana.com/\">Grafana</a> and <a href=\"http://prometheus.io/\">Prometheus</a> projects.</p>",
+        "mode": "html"
+      },
+      "pluginVersion": "7.1.0",
+      "style": {},
+      "title": "",
+      "transparent": true,
+      "type": "text"
+    },
+    {
+      "content": "#### Samples Ingested\nThis graph displays the count of samples ingested by the Prometheus server, as measured over the last 5 minutes, per time series in the range vector. When troubleshooting an issue on IRC or Github, this is often the first stat requested by the Prometheus team. ",
+      "datasource": null,
+      "editable": true,
+      "error": false,
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 4,
+        "x": 18,
+        "y": 5
+      },
+      "id": 8,
+      "links": [],
+      "mode": "markdown",
+      "options": {
+        "content": "#### Samples Ingested\nThis graph displays the count of samples ingested by the Prometheus server, as measured over the last 5 minutes, per time series in the range vector. When troubleshooting an issue on IRC or Github, this is often the first stat requested by the Prometheus team. ",
+        "mode": "markdown"
+      },
+      "pluginVersion": "7.1.0",
+      "style": {},
+      "title": "",
+      "transparent": true,
+      "type": "text"
+    },
+    {
+      "aliasColors": {
+        "Value": "red",
+        "aggr": "red"
+      },
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "eth-statediff-service",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "links": [],
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 6,
+        "w": 18,
+        "x": 0,
+        "y": 6
+      },
+      "hiddenSeries": false,
+      "id": 17,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "percentage": false,
+      "pluginVersion": "7.1.5",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(increase(eth_statediff_service_processed_height{instance=~\".+\"}[1m]))",
+          "interval": "",
+          "legendFormat": "aggr",
+          "refId": "C"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Aggregate block processed (rate-1m)",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "content": "#### Scrapes\nPrometheus scrapes metrics from instrumented jobs, either directly or via an intermediary push gateway for short-lived jobs. Target scrapes will show how frequently targets are scraped, as measured over the last 5 minutes, per time series in the range vector. Scrape Duration will show how long the scrapes are taking, with percentiles available as series. ",
+      "datasource": null,
+      "editable": true,
+      "error": false,
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 6,
+        "x": 18,
+        "y": 11
+      },
+      "id": 11,
+      "links": [],
+      "mode": "markdown",
+      "options": {
+        "content": "#### Scrapes\nPrometheus scrapes metrics from instrumented jobs, either directly or via an intermediary push gateway for short-lived jobs. Target scrapes will show how frequently targets are scraped, as measured over the last 5 minutes, per time series in the range vector. Scrape Duration will show how long the scrapes are taking, with percentiles available as series. ",
+        "mode": "markdown"
+      },
+      "pluginVersion": "7.1.0",
+      "style": {},
+      "title": "",
+      "transparent": true,
+      "type": "text"
+    },
+    {
+      "content": "#### Rule Evaluation Duration\nThis graph panel plots the duration for all evaluations to execute. The 50th percentile, 90th percentile and 99th percentile are shown as three separate series to help identify outliers that may be skewing the data.",
+      "datasource": null,
+      "editable": true,
+      "error": false,
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 6,
+        "x": 18,
+        "y": 18
+      },
+      "id": 15,
+      "links": [],
+      "mode": "markdown",
+      "options": {
+        "content": "#### Rule Evaluation Duration\nThis graph panel plots the duration for all evaluations to execute. The 50th percentile, 90th percentile and 99th percentile are shown as three separate series to help identify outliers that may be skewing the data.",
+        "mode": "markdown"
+      },
+      "pluginVersion": "7.1.0",
+      "style": {},
+      "title": "",
+      "transparent": true,
+      "type": "text"
+    }
+  ],
+  "refresh": false,
+  "revision": "1.0",
+  "schemaVersion": 26,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "allValue": null,
+        "current": {
+          "selected": true,
+          "text": "localhost:8888",
+          "value": "localhost:8888"
+        },
+        "datasource": "eth-statediff-service",
+        "definition": "label_values(instance)",
+        "hide": 0,
+        "includeAll": false,
+        "label": "SelectInstance",
+        "multi": false,
+        "name": "instance",
+        "options": [],
+        "query": "label_values(instance)",
+        "refresh": 2,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      }
+    ]
+  },
+  "time": {
+    "from": "now-30m",
+    "to": "now"
+  },
+  "timepicker": {
+    "now": true,
+    "refresh_intervals": [
+      "5s",
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ],
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "timezone": "browser",
+  "title": "Prometheus Block Process Stats",
+  "uid": "q1FUJHKnz",
+  "version": 14
+}
@@ -47,6 +47,8 @@ type StateDiffService interface {
 	Protocols() []p2p.Protocol
 	// Loop is the main event loop for processing state diffs
 	Loop(wg *sync.WaitGroup) error
+	// Run is a one-off command to run on a predefined set of ranges
+	Run(ranges []RangeRequest) error
 	// StateDiffAt method to get state diff object at specific block
 	StateDiffAt(blockNumber uint64, params sd.Params) (*sd.Payload, error)
 	// StateDiffFor method to get state diff object at specific block
@@ -115,11 +117,34 @@ func (sds *Service) APIs() []rpc.API {
 	}
 }

+// Run does a one-off processing run on the provided RangeRequests + any pre-runs, exiting afterwards
+func (sds *Service) Run(rngs []RangeRequest) error {
+	for _, preRun := range sds.preruns {
+		logrus.Infof("processing prerun range (%d, %d)", preRun.Start, preRun.Stop)
+		for i := preRun.Start; i <= preRun.Stop; i++ {
+			if err := sds.WriteStateDiffAt(i, preRun.Params); err != nil {
+				return fmt.Errorf("error writing statediff at height %d in range (%d, %d) : %v", i, preRun.Start, preRun.Stop, err)
+			}
+		}
+	}
+	sds.preruns = nil
+	for _, rng := range rngs {
+		logrus.Infof("processing prerun range (%d, %d)", rng.Start, rng.Stop)
+		for i := rng.Start; i <= rng.Stop; i++ {
+			if err := sds.WriteStateDiffAt(i, rng.Params); err != nil {
+				return fmt.Errorf("error writing statediff at height %d in range (%d, %d) : %v", i, rng.Start, rng.Stop, err)
+			}
+		}
+	}
+	return nil
+}
+
 // Loop is an empty service loop for awaiting rpc requests
 func (sds *Service) Loop(wg *sync.WaitGroup) error {
 	if sds.quitChan != nil {
 		return fmt.Errorf("service loop is already running")
 	}
+
 	sds.quitChan = make(chan struct{})
 	for i := 0; i < int(sds.workers); i++ {
 		wg.Add(1)
@@ -128,6 +153,7 @@ func (sds *Service) Loop(wg *sync.WaitGroup) error {
 			for {
 				select {
 				case blockRange := <-sds.queue:
+					logrus.Infof("service worker %d received range (%d, %d) off of work queue, beginning processing", id, blockRange.Start, blockRange.Stop)
 					prom.DecQueuedRanges()
 					for j := blockRange.Start; j <= blockRange.Stop; j++ {
 						if err := sds.WriteStateDiffAt(j, blockRange.Params); err != nil {
@@ -167,7 +193,7 @@ func (sds *Service) StateDiffAt(blockNumber uint64, params sd.Params) (*sd.Paylo
 	if err != nil {
 		return nil, err
 	}
-	logrus.Info(fmt.Sprintf("sending state diff at block %d", blockNumber))
+	logrus.Infof("sending state diff at block %d", blockNumber)
 	if blockNumber == 0 {
 		return sds.processStateDiff(currentBlock, common.Hash{}, params)
 	}
@@ -185,7 +211,7 @@ func (sds *Service) StateDiffFor(blockHash common.Hash, params sd.Params) (*sd.P
 	if err != nil {
 		return nil, err
 	}
-	logrus.Info(fmt.Sprintf("sending state diff at block %s", blockHash.Hex()))
+	logrus.Infof("sending state diff at block %s", blockHash.Hex())
 	if currentBlock.NumberU64() == 0 {
 		return sds.processStateDiff(currentBlock, common.Hash{}, params)
 	}
@@ -254,7 +280,7 @@ func (sds *Service) StateTrieAt(blockNumber uint64, params sd.Params) (*sd.Paylo
 	if err != nil {
 		return nil, err
 	}
-	logrus.Info(fmt.Sprintf("sending state trie at block %d", blockNumber))
+	logrus.Infof("sending state trie at block %d", blockNumber)
 	return sds.processStateTrie(currentBlock, params)
 }

@@ -288,7 +314,7 @@ func (sds *Service) Stop() error {
 // This operation cannot be performed back past the point of db pruning; it requires an archival node
 // for historical data
 func (sds *Service) WriteStateDiffAt(blockNumber uint64, params sd.Params) error {
-	logrus.Info(fmt.Sprintf("Writing state diff at block %d", blockNumber))
+	logrus.Infof("Writing state diff at block %d", blockNumber)
 	t := time.Now()
 	currentBlock, err := sds.lvlDBReader.GetBlockByNumber(blockNumber)
 	if err != nil {
@@ -309,7 +335,7 @@ func (sds *Service) WriteStateDiffAt(blockNumber uint64, params sd.Params) error
 // This operation cannot be performed back past the point of db pruning; it requires an archival node
 // for historical data
 func (sds *Service) WriteStateDiffFor(blockHash common.Hash, params sd.Params) error {
-	logrus.Info(fmt.Sprintf("Writing state diff for block %s", blockHash.Hex()))
+	logrus.Infof("Writing state diff for block %s", blockHash.Hex())
 	t := time.Now()
 	currentBlock, err := sds.lvlDBReader.GetBlockByHash(blockHash)
 	if err != nil {
@@ -366,7 +392,7 @@ func (sds *Service) writeStateDiff(block *types.Block, parentRoot common.Hash, p
 	}, params, output, codeOutput)
 	prom.SetTimeMetric(prom.T_STATE_PROCESSING, time.Now().Sub(t))
 	t = time.Now()
-	err = tx.Close(err)
+	err = tx.Close(tx, err)
 	prom.SetLastProcessedHeight(height)
 	prom.SetTimeMetric(prom.T_POSTGRES_TX_COMMIT, time.Now().Sub(t))
 	return err
Author	SHA1	Message	Date
i-norden	c5c230a2ea	update to use new version of statediffing geth pkgs with public.blocks insert batching	2021-11-08 19:08:36 -06:00
i-norden	dcf8afb96c	bump Go version to 1.16	2021-11-08 19:05:35 -06:00
Ian Norden	212bf4c4b2	Merge pull request #52 from vulcanize/add-grafana-dashboard Add grafana dashboard.	2021-11-04 10:09:21 -05:00
Arijit Das	a4a3ec1d7a	Add grafana dashboard.	2021-11-01 19:39:00 +05:30
Ian Norden	efb334aef3	Merge pull request #51 from vulcanize/prerun generate rand node id if none is provided	2021-10-28 10:52:35 -05:00
i-norden	c83e8835ab	generate rand node id if none is provided	2021-10-28 10:21:10 -05:00
S. R. Wadleigh	728642afec	Merge pull request #48 from vulcanize/make_startup_script_executable make startup_script.sh executable	2021-10-28 03:22:51 +00:00
Ian Norden	5875d1ec82	Update Dockerfile	2021-10-27 19:57:19 -05:00
erikdies	8d6919ab2b	make startup_script.sh executable	2021-10-27 20:46:25 -04:00
Ian Norden	abf72725ee	Merge pull request #47 from vulcanize/prerun cleanup	2021-10-27 17:46:13 -05:00
i-norden	8a55469403	cleanup	2021-10-27 17:45:47 -05:00
Ian Norden	a3857bffe4	Merge pull request #46 from vulcanize/prerun enable configuration of a prerun range by env variable; prerun only mode	2021-10-27 14:54:40 -05:00
i-norden	3956081874	sanitized prod config	2021-10-27 14:44:37 -05:00
i-norden	e1abb399cd	env vars for log stuff	2021-10-27 14:34:33 -05:00
i-norden	3d00e3ed05	enable configuration of a prerun range by env variable; prerun only mode	2021-10-27 13:36:19 -05:00
Ian Norden	84fff0aa03	Merge pull request #44 from vulcanize/stats make DB stats collector registration optional	2021-10-25 18:57:40 -05:00
i-norden	79cb508d83	make DB stats collector registration optional	2021-10-25 18:54:54 -05:00