wire up; customize metrics

go.mod

@@ -4,6 +4,8 @@ go 1.13
 
 require (
 	github.com/ethereum/go-ethereum v1.10.9
+	github.com/jmoiron/sqlx v1.2.0
+	github.com/prometheus/client_golang v1.0.0
 	github.com/sirupsen/logrus v1.7.0
 	github.com/spf13/cobra v1.1.1
 	github.com/spf13/viper v1.7.1
@@ -12,4 +14,3 @@ require (
 )
 
 replace github.com/ethereum/go-ethereum v1.10.9 => github.com/vulcanize/go-ethereum v1.10.11-statediff-0.0.27
-

go.sum

@@ -62,6 +62,7 @@ github.com/aws/aws-sdk-go-v2/service/sso v1.1.1/go.mod h1:SuZJxklHxLAXgLTc1iFXbE
 github.com/aws/aws-sdk-go-v2/service/sts v1.1.1/go.mod h1:Wi0EBZwiz/K44YliU0EKxqTCJGUfYTWXrrBwkq736bM=
 github.com/aws/smithy-go v1.1.0/go.mod h1:EzMw8dbp/YJL4A5/sbhGddag+NPT7q084agLbB9LgIw=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
@@ -364,6 +365,7 @@ github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsO
 github.com/mattn/go-sqlite3 v1.14.7 h1:fxWBnXkxfM6sRiuH3bqJ4CfzZojMOLVc0UTsTglEghA=
 github.com/mattn/go-sqlite3 v1.14.7/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
 github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE=
+github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g=
@@ -441,16 +443,20 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
+github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.6.0 h1:kRhiuYSXR3+uv2IbVbZhUxK5zVD/2pp3Gd2PpvPkpEo=
 github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=

pkg/prom/prom.go

@@ -29,107 +29,53 @@ const statsSubsystem = "stats"
 var (
 	metrics bool
 
-	receipts     prometheus.Counter
+	lastLoadedHeight    prometheus.Gauge
-	transactions prometheus.Counter
+	lastProcessedHeight prometheus.Gauge
-	blocks       prometheus.Counter
-	logs		prometheus.Counter
-	accessListEntries prometheus.Counter
 
-	lenPayloadChan prometheus.Gauge
+	tBlockLoad       prometheus.Histogram
-
+	tBlockProcessing prometheus.Histogram
-	tPayloadDecode             prometheus.Histogram
+	tStateProcessing prometheus.Histogram
-	tFreePostgres              prometheus.Histogram
+	tTxCommit        prometheus.Histogram
-	tPostgresCommit            prometheus.Histogram
-	tHeaderProcessing          prometheus.Histogram
-	tUncleProcessing           prometheus.Histogram
-	tTxAndRecProcessing        prometheus.Histogram
-	tStateAndStoreProcessing   prometheus.Histogram
-	tCodeAndCodeHashProcessing prometheus.Histogram
 )
 
 // Init module initialization
 func Init() {
 	metrics = true
 
-	blocks = promauto.NewCounter(prometheus.CounterOpts{
+	lastLoadedHeight = promauto.NewGauge(prometheus.GaugeOpts{
 		Namespace: namespace,
-		Name:      "blocks",
+		Name:      "loaded_height",
-		Help:      "The total number of processed blocks",
+		Help:      "The last block that was loaded for processing",
 	})
-	transactions = promauto.NewCounter(prometheus.CounterOpts{
+	lastProcessedHeight = promauto.NewGauge(prometheus.GaugeOpts{
 		Namespace: namespace,
-		Name:      "transactions",
+		Name:      "processed_height",
-		Help:      "The total number of processed transactions",
+		Help:      "The last block that was processed",
-	})
-	receipts = promauto.NewCounter(prometheus.CounterOpts{
-		Namespace: namespace,
-		Name:      "receipts",
-		Help:      "The total number of processed receipts",
-	})
-	logs = promauto.NewCounter(prometheus.CounterOpts{
-		Namespace: namespace,
-		Name:      "logs",
-		Help:      "The total number of processed logs",
-	})
-	accessListEntries = promauto.NewCounter(prometheus.CounterOpts{
-		Namespace: namespace,
-		Name:      "access_list_entries",
-		Help:      "The total number of processed access list entries",
 	})
 
-	lenPayloadChan = promauto.NewGauge(prometheus.GaugeOpts{
+	tBlockLoad = promauto.NewHistogram(prometheus.HistogramOpts{
-		Namespace: namespace,
-		Name:      "len_payload_chan",
-		Help:      "Current length of publishPayload",
-	})
-
-	tPayloadDecode = promauto.NewHistogram(prometheus.HistogramOpts{
 		Namespace: namespace,
 		Subsystem: statsSubsystem,
-		Name:      "t_payload_decode",
+		Name:      "t_block_load",
-		Help:      "Payload decoding time",
+		Help:      "Block loading time",
 	})
-	tFreePostgres = promauto.NewHistogram(prometheus.HistogramOpts{
+	tBlockProcessing = promauto.NewHistogram(prometheus.HistogramOpts{
 		Namespace: namespace,
 		Subsystem: statsSubsystem,
-		Name:      "t_free_postgres",
+		Name:      "t_block_processing",
-		Help:      "Time spent waiting for free postgres tx",
+		Help:      "Block (header, uncles, txs, rcts, tx trie, rct trie) processing time",
 	})
-	tPostgresCommit = promauto.NewHistogram(prometheus.HistogramOpts{
+	tStateProcessing = promauto.NewHistogram(prometheus.HistogramOpts{
 		Namespace: namespace,
 		Subsystem: statsSubsystem,
-		Name:      "t_postgres_commit",
+		Name:      "t_state_processing",
-		Help:      "Postgres transaction commit duration",
+		Help:      "State (state trie, storage tries, and code) processing time",
 	})
-	tHeaderProcessing = promauto.NewHistogram(prometheus.HistogramOpts{
+	tTxCommit = promauto.NewHistogram(prometheus.HistogramOpts{
 		Namespace: namespace,
 		Subsystem: statsSubsystem,
-		Name:      "t_header_processing",
+		Name:      "t_postgres_tx_commit",
-		Help:      "Header processing time",
+		Help:      "Postgres tx commit time",
-	})
-	tUncleProcessing = promauto.NewHistogram(prometheus.HistogramOpts{
-		Namespace: namespace,
-		Subsystem: statsSubsystem,
-		Name:      "t_uncle_processing",
-		Help:      "Uncle processing time",
-	})
-	tTxAndRecProcessing = promauto.NewHistogram(prometheus.HistogramOpts{
-		Namespace: namespace,
-		Subsystem: statsSubsystem,
-		Name:      "t_tx_receipt_processing",
-		Help:      "Tx and receipt processing time",
-	})
-	tStateAndStoreProcessing = promauto.NewHistogram(prometheus.HistogramOpts{
-		Namespace: namespace,
-		Subsystem: statsSubsystem,
-		Name:      "t_state_store_processing",
-		Help:      "State and storage processing time",
-	})
-	tCodeAndCodeHashProcessing = promauto.NewHistogram(prometheus.HistogramOpts{
-		Namespace: namespace,
-		Subsystem: statsSubsystem,
-		Name:      "t_code_codehash_processing",
-		Help:      "Code and codehash processing time",
 	})
 }
 
@@ -140,45 +86,17 @@ func RegisterDBCollector(name string, db *sqlx.DB) {
 	}
 }
 
-// BlockInc block counter increment
+// SetLastLoadedHeight sets last loaded height
-func BlockInc() {
+func SetLastLoadedHeight(height int64) {
 	if metrics {
-		blocks.Inc()
+		lastLoadedHeight.Set(float64(height))
 	}
 }
 
-// TransactionInc transaction counter increment
+// SetLastProcessedHeight sets last processed height
-func TransactionInc() {
+func SetLastProcessedHeight(height int64) {
 	if metrics {
-		transactions.Inc()
+		lastProcessedHeight.Set(float64(height))
-	}
-}
-
-// ReceiptInc receipt counter increment
-func ReceiptInc() {
-	if metrics {
-		receipts.Inc()
-	}
-}
-
-// LogInc log counter increment
-func LogInc() {
-	if metrics {
-		logs.Inc()
-	}
-}
-
-// AccessListElementInc access list element counter increment
-func AccessListElementInc() {
-	if metrics {
-		accessListEntries.Inc()
-	}
-}
-
-// SetLenPayloadChan set chan length
-func SetLenPayloadChan(ln int) {
-	if metrics {
-		lenPayloadChan.Set(float64(ln))
 	}
 }
 
@@ -189,21 +107,13 @@ func SetTimeMetric(name string, t time.Duration) {
 	}
 	tAsF64 := t.Seconds()
 	switch name {
-	case "t_payload_decode":
+	case "t_block_load":
-		tPayloadDecode.Observe(tAsF64)
+		tBlockLoad.Observe(tAsF64)
-	case "t_free_postgres":
+	case "t_block_processing":
-		tFreePostgres.Observe(tAsF64)
+		tBlockProcessing.Observe(tAsF64)
-	case "t_postgres_commit":
+	case "t_state_processing":
-		tPostgresCommit.Observe(tAsF64)
+		tStateProcessing.Observe(tAsF64)
-	case "t_header_processing":
+	case "t_postgres_tx_commit":
-		tHeaderProcessing.Observe(tAsF64)
+		tTxCommit.Observe(tAsF64)
-	case "t_uncle_processing":
-		tUncleProcessing.Observe(tAsF64)
-	case "t_tx_receipt_processing":
-		tTxAndRecProcessing.Observe(tAsF64)
-	case "t_state_store_processing":
-		tStateAndStoreProcessing.Observe(tAsF64)
-	case "t_code_codehash_processing":
-		tCodeAndCodeHashProcessing.Observe(tAsF64)
 	}
 }

pkg/service.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"math/big"
 	"sync"
+	"time"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/state"
@@ -31,6 +32,7 @@ import (
 	sd "github.com/ethereum/go-ethereum/statediff"
 	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
 	"github.com/sirupsen/logrus"
+	"github.com/vulcanize/eth-statediff-service/pkg/prom"
 
 	ind "github.com/ethereum/go-ethereum/statediff/indexer"
 )
@@ -251,6 +253,7 @@ func (sds *Service) Stop() error {
 // for historical data
 func (sds *Service) WriteStateDiffAt(blockNumber uint64, params sd.Params) error {
 	logrus.Info(fmt.Sprintf("Writing state diff at block %d", blockNumber))
+	t := time.Now()
 	currentBlock, err := sds.lvlDBReader.GetBlockByNumber(blockNumber)
 	if err != nil {
 		return err
@@ -263,7 +266,7 @@ func (sds *Service) WriteStateDiffAt(blockNumber uint64, params sd.Params) error
 		}
 		parentRoot = parentBlock.Root()
 	}
-	return sds.writeStateDiff(currentBlock, parentRoot, params)
+	return sds.writeStateDiff(currentBlock, parentRoot, params, t)
 }
 
 // WriteStateDiffFor writes a state diff for the specific blockHash directly to the database
@@ -271,6 +274,7 @@ func (sds *Service) WriteStateDiffAt(blockNumber uint64, params sd.Params) error
 // for historical data
 func (sds *Service) WriteStateDiffFor(blockHash common.Hash, params sd.Params) error {
 	logrus.Info(fmt.Sprintf("Writing state diff for block %s", blockHash.Hex()))
+	t := time.Now()
 	currentBlock, err := sds.lvlDBReader.GetBlockByHash(blockHash)
 	if err != nil {
 		return err
@@ -283,11 +287,11 @@ func (sds *Service) WriteStateDiffFor(blockHash common.Hash, params sd.Params) e
 		}
 		parentRoot = parentBlock.Root()
 	}
-	return sds.writeStateDiff(currentBlock, parentRoot, params)
+	return sds.writeStateDiff(currentBlock, parentRoot, params, t)
 }
 
 // Writes a state diff from the current block, parent state root, and provided params
-func (sds *Service) writeStateDiff(block *types.Block, parentRoot common.Hash, params sd.Params) error {
+func (sds *Service) writeStateDiff(block *types.Block, parentRoot common.Hash, params sd.Params, t time.Time) error {
 	var totalDifficulty *big.Int
 	var receipts types.Receipts
 	var err error
@@ -303,7 +307,10 @@ func (sds *Service) writeStateDiff(block *types.Block, parentRoot common.Hash, p
 	if err != nil {
 		return err
 	}
-
+	height := block.Number().Int64()
+	prom.SetLastLoadedHeight(height)
+	prom.SetTimeMetric("t_block_load", time.Now().Sub(t))
+	t = time.Now()
 	tx, err := sds.indexer.PushBlock(block, receipts, totalDifficulty)
 	if err != nil {
 		return err
@@ -315,9 +322,16 @@ func (sds *Service) writeStateDiff(block *types.Block, parentRoot common.Hash, p
 	codeOutput := func(c sdtypes.CodeAndCodeHash) error {
 		return sds.indexer.PushCodeAndCodeHash(tx, c)
 	}
+	prom.SetTimeMetric("t_block_processing", time.Now().Sub(t))
+	t = time.Now()
 	err = sds.Builder.WriteStateDiffObject(sd.StateRoots{
 		NewStateRoot: block.Root(),
 		OldStateRoot: parentRoot,
 	}, params, output, codeOutput)
-	return tx.Close(err)
+	prom.SetTimeMetric("t_state_processing", time.Now().Sub(t))
+	t = time.Now()
+	err = tx.Close(err)
+	prom.SetLastProcessedHeight(height)
+	prom.SetTimeMetric("t_postgres_tx_commit", time.Now().Sub(t))
+	return err
 }