Compare commits

...
9 Commits
Author SHA1 Message Date
i-norden c5c230a2ea update to use new version of statediffing geth pkgs with public.blocks insert batching 2021-11-08 19:08:36 -06:00
i-norden dcf8afb96c bump Go version to 1.16 2021-11-08 19:05:35 -06:00
Ian Norden 212bf4c4b2 Merge pull request #52 from vulcanize/add-grafana-dashboard
Add grafana dashboard.
2021-11-04 10:09:21 -05:00
Arijit Das a4a3ec1d7a Add grafana dashboard. 2021-11-01 19:39:00 +05:30
Ian Norden efb334aef3 Merge pull request #51 from vulcanize/prerun
generate rand node id if none is provided
2021-10-28 10:52:35 -05:00
i-norden c83e8835ab generate rand node id if none is provided 2021-10-28 10:21:10 -05:00
S. R. Wadleigh 728642afec Merge pull request #48 from vulcanize/make_startup_script_executable
make startup_script.sh executable
2021-10-28 03:22:51 +00:00
Ian Norden 5875d1ec82 Update Dockerfile 2021-10-27 19:57:19 -05:00
erikdies 8d6919ab2b make startup_script.sh executable 2021-10-27 20:46:25 -04:00
9 changed files with 514 additions and 26 deletions
+2 -2
View File
@@ -1,4 +1,4 @@
FROM golang:1.13-alpine as builder
FROM golang:1.16-alpine as builder
RUN apk --update --no-cache add make git g++ linux-headers
# DEBUG
@@ -15,7 +15,7 @@ RUN GO111MODULE=on GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldfl
FROM alpine
ARG USER="vdm"
ARG CONFIG_FILE="./environments/example.toml"
ARG CONFIG_FILE="./environments/config.toml"
ARG EXPOSE_PORT=8545
RUN adduser -Du 5000 $USER adm
+5 -5
View File
@@ -29,12 +29,12 @@ const (
ETH_NETWORK_ID = "ETH_NETWORK_ID"
ETH_CHAIN_ID = "ETH_CHAIN_ID"
DB_CACHE_SIZE_MB = "DB_CACHE_SIZE_MB"
TRIE_CACHE_SIZE_MB = "TRIE_CACHE_SIZE_MB"
LVLDB_PATH = "LVLDB_PATH"
LVLDB_ANCIENT = "LVLDB_ANCIENT"
DB_CACHE_SIZE_MB = "DB_CACHE_SIZE_MB"
TRIE_CACHE_SIZE_MB = "TRIE_CACHE_SIZE_MB"
LVLDB_PATH = "LVLDB_PATH"
LVLDB_ANCIENT = "LVLDB_ANCIENT"
STATEDIFF_PRERUN = "STATEDIFF_PRERUN"
STATEDIFF_PRERUN = "STATEDIFF_PRERUN"
STATEDIFF_TRIE_WORKERS = "STATEDIFF_TRIE_WORKERS"
STATEDIFF_SERVICE_WORKERS = "STATEDIFF_SERVICE_WORKERS"
STATEDIFF_WORKER_QUEUE_SIZE = "STATEDIFF_WORKER_QUEUE_SIZE"
+47 -6
View File
@@ -18,8 +18,10 @@ package cmd
import (
"fmt"
"math/rand"
"os"
"strings"
"time"
"github.com/ethereum/go-ethereum/statediff/indexer/node"
"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
@@ -172,6 +174,8 @@ func init() {
viper.BindPFlag("prerun.only", rootCmd.PersistentFlags().Lookup("prerun-only"))
viper.BindPFlag("prerun.start", rootCmd.PersistentFlags().Lookup("prerun-start"))
viper.BindPFlag("prerun.stop", rootCmd.PersistentFlags().Lookup("prerun-stop"))
rand.Seed(time.Now().UnixNano())
}
func initConfig() {
@@ -188,13 +192,50 @@ func initConfig() {
}
func GetEthNodeInfo() node.Info {
return node.Info{
ID: viper.GetString("ethereum.nodeID"),
ClientName: viper.GetString("ethereum.clientName"),
GenesisBlock: viper.GetString("ethereum.genesisBlock"),
NetworkID: viper.GetString("ethereum.networkID"),
ChainID: viper.GetUint64("ethereum.chainID"),
var nodeID, genesisBlock, networkID, clientName string
var chainID uint64
if !viper.IsSet("ethereum.nodeID") {
nodeID = randSeq(12)
} else {
nodeID = viper.GetString("ethereum.nodeID")
}
if !viper.IsSet("ethereum.genesisBlock") {
genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3"
} else {
genesisBlock = viper.GetString("ethereum.genesisBlock")
}
if !viper.IsSet("ethereum.chainID") {
chainID = 1
} else {
chainID = viper.GetUint64("ethereum.chainID")
}
if !viper.IsSet("ethereum.networkID") {
networkID = "1"
} else {
networkID = viper.GetString("ethereum.networkID")
}
if !viper.IsSet("ethereum.clientName") {
clientName = "eth-statediff-service"
} else {
clientName = viper.GetString("ethereum.clientName")
}
return node.Info{
ID: nodeID,
ClientName: clientName,
GenesisBlock: genesisBlock,
NetworkID: networkID,
ChainID: chainID,
}
}
var characters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")
func randSeq(n int) string {
b := make([]rune, n)
for i := range b {
b[i] = characters[rand.Intn(len(characters))]
}
return string(b)
}
func GetDBParams() postgres.ConnectionParams {
+1 -1
View File
@@ -6,7 +6,7 @@ services:
context: ./
cache_from:
- alpine:latest
- golang:1.13
- golang:1.16
dockerfile: ./Dockerfile
args:
USER: "vdbm"
+2 -2
View File
@@ -1,6 +1,6 @@
module github.com/vulcanize/eth-statediff-service
go 1.13
go 1.16
require (
github.com/ethereum/go-ethereum v1.10.9
@@ -12,4 +12,4 @@ require (
github.com/vulcanize/go-eth-state-node-iterator v0.0.1-alpha.0.20211014064906-d23d01ed8191
)
replace github.com/ethereum/go-ethereum v1.10.9 => github.com/vulcanize/go-ethereum v1.10.11-statediff-0.0.27
replace github.com/ethereum/go-ethereum v1.10.9 => github.com/vulcanize/go-ethereum v1.10.11-statediff-0.0.27a
+2 -4
View File
@@ -539,8 +539,8 @@ github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPU
github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
github.com/vulcanize/go-eth-state-node-iterator v0.0.1-alpha.0.20211014064906-d23d01ed8191 h1:+xSwb0xOLCAhejoai2BQZPTJW5Z+tU9xkTaOBwGnPoU=
github.com/vulcanize/go-eth-state-node-iterator v0.0.1-alpha.0.20211014064906-d23d01ed8191/go.mod h1:FBMQp69PTnRg3kLx9qeWGfKYfVq4fpfbr69X8lEvvfs=
github.com/vulcanize/go-ethereum v1.10.11-statediff-0.0.27 h1:ldAVLlKll2WHHKLNu8oKbkYRUVRs9T45z12KiSrRW24=
github.com/vulcanize/go-ethereum v1.10.11-statediff-0.0.27/go.mod h1:9L+QY31AnWnX2/2HDOySCjQoYUdWNGBRMezFJVfH73E=
github.com/vulcanize/go-ethereum v1.10.11-statediff-0.0.27a h1:syuVN9x53LTCf9N/6/xsIlUUjp2qU0AP8JzwNMtWxis=
github.com/vulcanize/go-ethereum v1.10.11-statediff-0.0.27a/go.mod h1:9L+QY31AnWnX2/2HDOySCjQoYUdWNGBRMezFJVfH73E=
github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc h1:9lDbC6Rz4bwmou+oE6Dt4Cb2BGMur5eR/GYptkKUVHo=
github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM=
github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
@@ -603,7 +603,6 @@ golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKG
golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -817,7 +816,6 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+449
View File
@@ -0,0 +1,449 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"gnetId": null,
"graphTooltip": 0,
"id": 4,
"iteration": 1635774999288,
"links": [
{
"icon": "info",
"tags": [],
"targetBlank": true,
"title": "Grafana Docs",
"tooltip": "",
"type": "link",
"url": "https://grafana.com/docs/grafana/latest/"
},
{
"icon": "info",
"tags": [],
"targetBlank": true,
"title": "Prometheus Docs",
"type": "link",
"url": "http://prometheus.io/docs/introduction/overview/"
}
],
"panels": [
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "eth-statediff-service",
"fieldConfig": {
"defaults": {
"custom": {},
"links": [],
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 6,
"w": 18,
"x": 0,
"y": 0
},
"hiddenSeries": false,
"id": 3,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"percentage": false,
"pluginVersion": "7.1.5",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "rate(eth_statediff_service_processed_height{instance=\"$instance\"}[1m])",
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Instance block processed (rate-1m)",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"content": "<span style=\"font-family: 'Open Sans', 'Helvetica Neue', Helvetica; font-size: 25px;vertical-align: text-top;color: #bbbfc2;margin-left: 10px;\">Prometheus</span>\n\n<p style=\"margin-top: 10px;\">You're using Prometheus, an open-source systems monitoring and alerting toolkit originally built at SoundCloud. For more information, check out the <a href=\"https://grafana.com/\">Grafana</a> and <a href=\"http://prometheus.io/\">Prometheus</a> projects.</p>",
"datasource": null,
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 6,
"x": 18,
"y": 0
},
"id": 9,
"links": [],
"mode": "html",
"options": {
"content": "<span style=\"font-family: 'Open Sans', 'Helvetica Neue', Helvetica; font-size: 25px;vertical-align: text-top;color: #bbbfc2;margin-left: 10px;\">Prometheus</span>\n\n<p style=\"margin-top: 10px;\">You're using Prometheus, an open-source systems monitoring and alerting toolkit originally built at SoundCloud. For more information, check out the <a href=\"https://grafana.com/\">Grafana</a> and <a href=\"http://prometheus.io/\">Prometheus</a> projects.</p>",
"mode": "html"
},
"pluginVersion": "7.1.0",
"style": {},
"title": "",
"transparent": true,
"type": "text"
},
{
"content": "#### Samples Ingested\nThis graph displays the count of samples ingested by the Prometheus server, as measured over the last 5 minutes, per time series in the range vector. When troubleshooting an issue on IRC or Github, this is often the first stat requested by the Prometheus team. ",
"datasource": null,
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 4,
"x": 18,
"y": 5
},
"id": 8,
"links": [],
"mode": "markdown",
"options": {
"content": "#### Samples Ingested\nThis graph displays the count of samples ingested by the Prometheus server, as measured over the last 5 minutes, per time series in the range vector. When troubleshooting an issue on IRC or Github, this is often the first stat requested by the Prometheus team. ",
"mode": "markdown"
},
"pluginVersion": "7.1.0",
"style": {},
"title": "",
"transparent": true,
"type": "text"
},
{
"aliasColors": {
"Value": "red",
"aggr": "red"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "eth-statediff-service",
"fieldConfig": {
"defaults": {
"custom": {},
"links": [],
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 6,
"w": 18,
"x": 0,
"y": 6
},
"hiddenSeries": false,
"id": 17,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"percentage": false,
"pluginVersion": "7.1.5",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "sum(increase(eth_statediff_service_processed_height{instance=~\".+\"}[1m]))",
"interval": "",
"legendFormat": "aggr",
"refId": "C"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Aggregate block processed (rate-1m)",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"content": "#### Scrapes\nPrometheus scrapes metrics from instrumented jobs, either directly or via an intermediary push gateway for short-lived jobs. Target scrapes will show how frequently targets are scraped, as measured over the last 5 minutes, per time series in the range vector. Scrape Duration will show how long the scrapes are taking, with percentiles available as series. ",
"datasource": null,
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 6,
"x": 18,
"y": 11
},
"id": 11,
"links": [],
"mode": "markdown",
"options": {
"content": "#### Scrapes\nPrometheus scrapes metrics from instrumented jobs, either directly or via an intermediary push gateway for short-lived jobs. Target scrapes will show how frequently targets are scraped, as measured over the last 5 minutes, per time series in the range vector. Scrape Duration will show how long the scrapes are taking, with percentiles available as series. ",
"mode": "markdown"
},
"pluginVersion": "7.1.0",
"style": {},
"title": "",
"transparent": true,
"type": "text"
},
{
"content": "#### Rule Evaluation Duration\nThis graph panel plots the duration for all evaluations to execute. The 50th percentile, 90th percentile and 99th percentile are shown as three separate series to help identify outliers that may be skewing the data.",
"datasource": null,
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 6,
"x": 18,
"y": 18
},
"id": 15,
"links": [],
"mode": "markdown",
"options": {
"content": "#### Rule Evaluation Duration\nThis graph panel plots the duration for all evaluations to execute. The 50th percentile, 90th percentile and 99th percentile are shown as three separate series to help identify outliers that may be skewing the data.",
"mode": "markdown"
},
"pluginVersion": "7.1.0",
"style": {},
"title": "",
"transparent": true,
"type": "text"
}
],
"refresh": false,
"revision": "1.0",
"schemaVersion": 26,
"style": "dark",
"tags": [],
"templating": {
"list": [
{
"allValue": null,
"current": {
"selected": true,
"text": "localhost:8888",
"value": "localhost:8888"
},
"datasource": "eth-statediff-service",
"definition": "label_values(instance)",
"hide": 0,
"includeAll": false,
"label": "SelectInstance",
"multi": false,
"name": "instance",
"options": [],
"query": "label_values(instance)",
"refresh": 2,
"regex": "",
"skipUrlSync": false,
"sort": 1,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
}
]
},
"time": {
"from": "now-30m",
"to": "now"
},
"timepicker": {
"now": true,
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
],
"time_options": [
"5m",
"15m",
"1h",
"6h",
"12h",
"24h",
"2d",
"7d",
"30d"
]
},
"timezone": "browser",
"title": "Prometheus Block Process Stats",
"uid": "q1FUJHKnz",
"version": 14
}
+6 -6
View File
@@ -193,7 +193,7 @@ func (sds *Service) StateDiffAt(blockNumber uint64, params sd.Params) (*sd.Paylo
if err != nil {
return nil, err
}
logrus.Info(fmt.Sprintf("sending state diff at block %d", blockNumber))
logrus.Infof("sending state diff at block %d", blockNumber)
if blockNumber == 0 {
return sds.processStateDiff(currentBlock, common.Hash{}, params)
}
@@ -211,7 +211,7 @@ func (sds *Service) StateDiffFor(blockHash common.Hash, params sd.Params) (*sd.P
if err != nil {
return nil, err
}
logrus.Info(fmt.Sprintf("sending state diff at block %s", blockHash.Hex()))
logrus.Infof("sending state diff at block %s", blockHash.Hex())
if currentBlock.NumberU64() == 0 {
return sds.processStateDiff(currentBlock, common.Hash{}, params)
}
@@ -280,7 +280,7 @@ func (sds *Service) StateTrieAt(blockNumber uint64, params sd.Params) (*sd.Paylo
if err != nil {
return nil, err
}
logrus.Info(fmt.Sprintf("sending state trie at block %d", blockNumber))
logrus.Infof("sending state trie at block %d", blockNumber)
return sds.processStateTrie(currentBlock, params)
}
@@ -314,7 +314,7 @@ func (sds *Service) Stop() error {
// This operation cannot be performed back past the point of db pruning; it requires an archival node
// for historical data
func (sds *Service) WriteStateDiffAt(blockNumber uint64, params sd.Params) error {
logrus.Info(fmt.Sprintf("Writing state diff at block %d", blockNumber))
logrus.Infof("Writing state diff at block %d", blockNumber)
t := time.Now()
currentBlock, err := sds.lvlDBReader.GetBlockByNumber(blockNumber)
if err != nil {
@@ -335,7 +335,7 @@ func (sds *Service) WriteStateDiffAt(blockNumber uint64, params sd.Params) error
// This operation cannot be performed back past the point of db pruning; it requires an archival node
// for historical data
func (sds *Service) WriteStateDiffFor(blockHash common.Hash, params sd.Params) error {
logrus.Info(fmt.Sprintf("Writing state diff for block %s", blockHash.Hex()))
logrus.Infof("Writing state diff for block %s", blockHash.Hex())
t := time.Now()
currentBlock, err := sds.lvlDBReader.GetBlockByHash(blockHash)
if err != nil {
@@ -392,7 +392,7 @@ func (sds *Service) writeStateDiff(block *types.Block, parentRoot common.Hash, p
}, params, output, codeOutput)
prom.SetTimeMetric(prom.T_STATE_PROCESSING, time.Now().Sub(t))
t = time.Now()
err = tx.Close(err)
err = tx.Close(tx, err)
prom.SetLastProcessedHeight(height)
prom.SetTimeMetric(prom.T_POSTGRES_TX_COMMIT, time.Now().Sub(t))
return err
Regular → Executable
View File