cmd/puppeth: integrate blockscout (#18261)

* cmd/puppeth: integrate blockscout

* cmd/puppeth: expose debug namespace for blockscout

* cmd/puppeth: fix dbdir

* cmd/puppeth: run explorer in archive mode

* cmd/puppeth: ensure node is synced

* cmd/puppeth: fix explorer docker alignment + drop unneeded exec

* cmd/puppeth: polish up config saving and reloading

* cmd/puppeth: check both web and p2p port for explorer service

cmd/puppeth/module_explorer.go

@@ -30,108 +30,86 @@ import (
 
 // explorerDockerfile is the Dockerfile required to run a block explorer.
 var explorerDockerfile = `
-FROM puppeth/explorer:latest
-
-ADD ethstats.json /ethstats.json
-ADD chain.json /chain.json
+FROM puppeth/blockscout:latest
 
+ADD genesis.json /genesis.json
 RUN \
-  echo '(cd ../eth-net-intelligence-api && pm2 start /ethstats.json)' >  explorer.sh && \
-	echo '(cd ../etherchain-light && npm start &)'                      >> explorer.sh && \
-	echo 'exec /parity/parity --chain=/chain.json --port={{.NodePort}} --tracing=on --fat-db=on --pruning=archive' >> explorer.sh
+  echo 'geth --cache 512 init /genesis.json' > explorer.sh && \
+  echo $'geth --networkid {{.NetworkID}} --syncmode "full" --gcmode "archive" --port {{.EthPort}} --bootnodes {{.Bootnodes}} --ethstats \'{{.Ethstats}}\' --cache=512 --rpc --rpcapi "net,web3,eth,shh,debug" --rpccorsdomain "*" --rpcvhosts "*" --ws --wsorigins "*" --exitwhensynced' >> explorer.sh && \
+  echo $'exec geth --networkid {{.NetworkID}} --syncmode "full" --gcmode "archive" --port {{.EthPort}} --bootnodes {{.Bootnodes}} --ethstats \'{{.Ethstats}}\' --cache=512 --rpc --rpcapi "net,web3,eth,shh,debug" --rpccorsdomain "*" --rpcvhosts "*" --ws --wsorigins "*" &' >> explorer.sh && \
+  echo '/usr/local/bin/docker-entrypoint.sh postgres &' >> explorer.sh && \
+  echo 'sleep 5' >> explorer.sh && \
+  echo 'mix do ecto.drop --force, ecto.create, ecto.migrate' >> explorer.sh && \
+  echo 'mix phx.server' >> explorer.sh
 
 ENTRYPOINT ["/bin/sh", "explorer.sh"]
 `
 
-// explorerEthstats is the configuration file for the ethstats javascript client.
-var explorerEthstats = `[
-  {
-    "name"              : "node-app",
-    "script"            : "app.js",
-    "log_date_format"   : "YYYY-MM-DD HH:mm Z",
-    "merge_logs"        : false,
-    "watch"             : false,
-    "max_restarts"      : 10,
-    "exec_interpreter"  : "node",
-    "exec_mode"         : "fork_mode",
-    "env":
-    {
-      "NODE_ENV"        : "production",
-      "RPC_HOST"        : "localhost",
-      "RPC_PORT"        : "8545",
-      "LISTENING_PORT"  : "{{.Port}}",
-      "INSTANCE_NAME"   : "{{.Name}}",
-      "CONTACT_DETAILS" : "",
-      "WS_SERVER"       : "{{.Host}}",
-      "WS_SECRET"       : "{{.Secret}}",
-      "VERBOSITY"       : 2
-    }
-  }
-]`
-
 // explorerComposefile is the docker-compose.yml file required to deploy and
 // maintain a block explorer.
 var explorerComposefile = `
 version: '2'
 services:
-  explorer:
-    build: .
-    image: {{.Network}}/explorer
-    container_name: {{.Network}}_explorer_1
-    ports:
-      - "{{.NodePort}}:{{.NodePort}}"
-      - "{{.NodePort}}:{{.NodePort}}/udp"{{if not .VHost}}
-      - "{{.WebPort}}:3000"{{end}}
-    volumes:
-      - {{.Datadir}}:/root/.local/share/io.parity.ethereum
-    environment:
-      - NODE_PORT={{.NodePort}}/tcp
-      - STATS={{.Ethstats}}{{if .VHost}}
-      - VIRTUAL_HOST={{.VHost}}
-      - VIRTUAL_PORT=3000{{end}}
-    logging:
-      driver: "json-file"
-      options:
-        max-size: "1m"
-        max-file: "10"
-    restart: always
+    explorer:
+        build: .
+        image: {{.Network}}/explorer
+        container_name: {{.Network}}_explorer_1
+        ports:
+            - "{{.EthPort}}:{{.EthPort}}"
+            - "{{.EthPort}}:{{.EthPort}}/udp"{{if not .VHost}}
+            - "{{.WebPort}}:4000"{{end}}
+        environment:
+            - ETH_PORT={{.EthPort}}
+            - ETH_NAME={{.EthName}}
+            - BLOCK_TRANSFORMER={{.Transformer}}{{if .VHost}}
+            - VIRTUAL_HOST={{.VHost}}
+            - VIRTUAL_PORT=4000{{end}}
+        volumes:
+            - {{.Datadir}}:/opt/app/.ethereum
+            - {{.DBDir}}:/var/lib/postgresql/data
+        logging:
+          driver: "json-file"
+          options:
+            max-size: "1m"
+            max-file: "10"
+        restart: always
 `
 
 // deployExplorer deploys a new block explorer container to a remote machine via
 // SSH, docker and docker-compose. If an instance with the specified network name
 // already exists there, it will be overwritten!
-func deployExplorer(client *sshClient, network string, chainspec []byte, config *explorerInfos, nocache bool) ([]byte, error) {
+func deployExplorer(client *sshClient, network string, bootnodes []string, config *explorerInfos, nocache bool, isClique bool) ([]byte, error) {
 	// Generate the content to upload to the server
 	workdir := fmt.Sprintf("%d", rand.Int63())
 	files := make(map[string][]byte)
 
 	dockerfile := new(bytes.Buffer)
 	template.Must(template.New("").Parse(explorerDockerfile)).Execute(dockerfile, map[string]interface{}{
-		"NodePort": config.nodePort,
+		"NetworkID": config.node.network,
+		"Bootnodes": strings.Join(bootnodes, ","),
+		"Ethstats":  config.node.ethstats,
+		"EthPort":   config.node.port,
 	})
 	files[filepath.Join(workdir, "Dockerfile")] = dockerfile.Bytes()
 
-	ethstats := new(bytes.Buffer)
-	template.Must(template.New("").Parse(explorerEthstats)).Execute(ethstats, map[string]interface{}{
-		"Port":   config.nodePort,
-		"Name":   config.ethstats[:strings.Index(config.ethstats, ":")],
-		"Secret": config.ethstats[strings.Index(config.ethstats, ":")+1 : strings.Index(config.ethstats, "@")],
-		"Host":   config.ethstats[strings.Index(config.ethstats, "@")+1:],
-	})
-	files[filepath.Join(workdir, "ethstats.json")] = ethstats.Bytes()
-
+	transformer := "base"
+	if isClique {
+		transformer = "clique"
+	}
 	composefile := new(bytes.Buffer)
 	template.Must(template.New("").Parse(explorerComposefile)).Execute(composefile, map[string]interface{}{
-		"Datadir":  config.datadir,
-		"Network":  network,
-		"NodePort": config.nodePort,
-		"VHost":    config.webHost,
-		"WebPort":  config.webPort,
-		"Ethstats": config.ethstats[:strings.Index(config.ethstats, ":")],
+		"Network":     network,
+		"VHost":       config.host,
+		"Ethstats":    config.node.ethstats,
+		"Datadir":     config.node.datadir,
+		"DBDir":       config.dbdir,
+		"EthPort":     config.node.port,
+		"EthName":     config.node.ethstats[:strings.Index(config.node.ethstats, ":")],
+		"WebPort":     config.port,
+		"Transformer": transformer,
 	})
 	files[filepath.Join(workdir, "docker-compose.yaml")] = composefile.Bytes()
-
-	files[filepath.Join(workdir, "chain.json")] = chainspec
+	files[filepath.Join(workdir, "genesis.json")] = config.node.genesis
 
 	// Upload the deployment files to the remote server (and clean up afterwards)
 	if out, err := client.Upload(files); err != nil {
@@ -149,22 +127,20 @@ func deployExplorer(client *sshClient, network string, chainspec []byte, config
 // explorerInfos is returned from a block explorer status check to allow reporting
 // various configuration parameters.
 type explorerInfos struct {
-	datadir  string
-	ethstats string
-	nodePort int
-	webHost  string
-	webPort  int
+	node  *nodeInfos
+	dbdir string
+	host  string
+	port  int
 }
 
 // Report converts the typed struct into a plain string->string map, containing
 // most - but not all - fields for reporting to the user.
 func (info *explorerInfos) Report() map[string]string {
 	report := map[string]string{
-		"Data directory":         info.datadir,
-		"Node listener port ":    strconv.Itoa(info.nodePort),
-		"Ethstats username":      info.ethstats,
-		"Website address ":       info.webHost,
-		"Website listener port ": strconv.Itoa(info.webPort),
+		"Website address ":        info.host,
+		"Website listener port ":  strconv.Itoa(info.port),
+		"Ethereum listener port ": strconv.Itoa(info.node.port),
+		"Ethstats username":       info.node.ethstats,
 	}
 	return report
 }
@@ -172,7 +148,7 @@ func (info *explorerInfos) Report() map[string]string {
 // checkExplorer does a health-check against a block explorer server to verify
 // whether it's running, and if yes, whether it's responsive.
 func checkExplorer(client *sshClient, network string) (*explorerInfos, error) {
-	// Inspect a possible block explorer container on the host
+	// Inspect a possible explorer container on the host
 	infos, err := inspectContainer(client, fmt.Sprintf("%s_explorer_1", network))
 	if err != nil {
 		return nil, err
@@ -181,13 +157,13 @@ func checkExplorer(client *sshClient, network string) (*explorerInfos, error) {
 		return nil, ErrServiceOffline
 	}
 	// Resolve the port from the host, or the reverse proxy
-	webPort := infos.portmap["3000/tcp"]
-	if webPort == 0 {
+	port := infos.portmap["4000/tcp"]
+	if port == 0 {
 		if proxy, _ := checkNginx(client, network); proxy != nil {
-			webPort = proxy.port
+			port = proxy.port
 		}
 	}
-	if webPort == 0 {
+	if port == 0 {
 		return nil, ErrNotExposed
 	}
 	// Resolve the host from the reverse-proxy and the config values
@@ -196,17 +172,23 @@ func checkExplorer(client *sshClient, network string) (*explorerInfos, error) {
 		host = client.server
 	}
 	// Run a sanity check to see if the devp2p is reachable
-	nodePort := infos.portmap[infos.envvars["NODE_PORT"]]
-	if err = checkPort(client.server, nodePort); err != nil {
-		log.Warn(fmt.Sprintf("Explorer devp2p port seems unreachable"), "server", client.server, "port", nodePort, "err", err)
+	p2pPort := infos.portmap[infos.envvars["ETH_PORT"]+"/tcp"]
+	if err = checkPort(host, p2pPort); err != nil {
+		log.Warn("Explorer node seems unreachable", "server", host, "port", p2pPort, "err", err)
+	}
+	if err = checkPort(host, port); err != nil {
+		log.Warn("Explorer service seems unreachable", "server", host, "port", port, "err", err)
 	}
 	// Assemble and return the useful infos
 	stats := &explorerInfos{
-		datadir:  infos.volumes["/root/.local/share/io.parity.ethereum"],
-		nodePort: nodePort,
-		webHost:  host,
-		webPort:  webPort,
-		ethstats: infos.envvars["STATS"],
+		node: &nodeInfos{
+			datadir:  infos.volumes["/opt/app/.ethereum"],
+			port:     infos.portmap[infos.envvars["ETH_PORT"]+"/tcp"],
+			ethstats: infos.envvars["ETH_NAME"],
+		},
+		dbdir: infos.volumes["/var/lib/postgresql/data"],
+		host:  host,
+		port:  port,
 	}
 	return stats, nil
 }

cmd/puppeth/wizard_dashboard.go

@@ -77,7 +77,7 @@ func (w *wizard) deployDashboard() {
 				}
 			case "explorer":
 				if infos, err := checkExplorer(client, w.network); err == nil {
-					port = infos.webPort
+					port = infos.port
 				}
 			case "wallet":
 				if infos, err := checkWallet(client, w.network); err == nil {

cmd/puppeth/wizard_explorer.go

@@ -35,10 +35,6 @@ func (w *wizard) deployExplorer() {
 		log.Error("No ethstats server configured")
 		return
 	}
-	if w.conf.Genesis.Config.Ethash == nil {
-		log.Error("Only ethash network supported")
-		return
-	}
 	// Select the server to interact with
 	server := w.selectServer()
 	if server == "" {
@@ -50,50 +46,57 @@ func (w *wizard) deployExplorer() {
 	infos, err := checkExplorer(client, w.network)
 	if err != nil {
 		infos = &explorerInfos{
-			nodePort: 30303, webPort: 80, webHost: client.server,
+			node: &nodeInfos{port: 30303},
+			port: 80,
+			host: client.server,
 		}
 	}
 	existed := err == nil
 
-	chainspec, err := newParityChainSpec(w.network, w.conf.Genesis, w.conf.bootnodes)
-	if err != nil {
-		log.Error("Failed to create chain spec for explorer", "err", err)
-		return
-	}
-	chain, _ := json.MarshalIndent(chainspec, "", "  ")
+	infos.node.genesis, _ = json.MarshalIndent(w.conf.Genesis, "", "  ")
+	infos.node.network = w.conf.Genesis.Config.ChainID.Int64()
 
 	// Figure out which port to listen on
 	fmt.Println()
-	fmt.Printf("Which port should the explorer listen on? (default = %d)\n", infos.webPort)
-	infos.webPort = w.readDefaultInt(infos.webPort)
+	fmt.Printf("Which port should the explorer listen on? (default = %d)\n", infos.port)
+	infos.port = w.readDefaultInt(infos.port)
 
 	// Figure which virtual-host to deploy ethstats on
-	if infos.webHost, err = w.ensureVirtualHost(client, infos.webPort, infos.webHost); err != nil {
+	if infos.host, err = w.ensureVirtualHost(client, infos.port, infos.host); err != nil {
 		log.Error("Failed to decide on explorer host", "err", err)
 		return
 	}
 	// Figure out where the user wants to store the persistent data
 	fmt.Println()
-	if infos.datadir == "" {
-		fmt.Printf("Where should data be stored on the remote machine?\n")
-		infos.datadir = w.readString()
+	if infos.node.datadir == "" {
+		fmt.Printf("Where should node data be stored on the remote machine?\n")
+		infos.node.datadir = w.readString()
 	} else {
-		fmt.Printf("Where should data be stored on the remote machine? (default = %s)\n", infos.datadir)
-		infos.datadir = w.readDefaultString(infos.datadir)
+		fmt.Printf("Where should node data be stored on the remote machine? (default = %s)\n", infos.node.datadir)
+		infos.node.datadir = w.readDefaultString(infos.node.datadir)
+	}
+	// Figure out where the user wants to store the persistent data for backend database
+	fmt.Println()
+	if infos.dbdir == "" {
+		fmt.Printf("Where should postgres data be stored on the remote machine?\n")
+		infos.dbdir = w.readString()
+	} else {
+		fmt.Printf("Where should postgres data be stored on the remote machine? (default = %s)\n", infos.dbdir)
+		infos.dbdir = w.readDefaultString(infos.dbdir)
 	}
 	// Figure out which port to listen on
 	fmt.Println()
-	fmt.Printf("Which TCP/UDP port should the archive node listen on? (default = %d)\n", infos.nodePort)
-	infos.nodePort = w.readDefaultInt(infos.nodePort)
+	fmt.Printf("Which TCP/UDP port should the archive node listen on? (default = %d)\n", infos.node.port)
+	infos.node.port = w.readDefaultInt(infos.node.port)
 
 	// Set a proper name to report on the stats page
 	fmt.Println()
-	if infos.ethstats == "" {
+	if infos.node.ethstats == "" {
 		fmt.Printf("What should the explorer be called on the stats page?\n")
-		infos.ethstats = w.readString() + ":" + w.conf.ethstats
+		infos.node.ethstats = w.readString() + ":" + w.conf.ethstats
 	} else {
-		fmt.Printf("What should the explorer be called on the stats page? (default = %s)\n", infos.ethstats)
-		infos.ethstats = w.readDefaultString(infos.ethstats) + ":" + w.conf.ethstats
+		fmt.Printf("What should the explorer be called on the stats page? (default = %s)\n", infos.node.ethstats)
+		infos.node.ethstats = w.readDefaultString(infos.node.ethstats) + ":" + w.conf.ethstats
 	}
 	// Try to deploy the explorer on the host
 	nocache := false
@@ -102,7 +105,7 @@ func (w *wizard) deployExplorer() {
 		fmt.Printf("Should the explorer be built from scratch (y/n)? (default = no)\n")
 		nocache = w.readDefaultYesNo(false)
 	}
-	if out, err := deployExplorer(client, w.network, chain, infos, nocache); err != nil {
+	if out, err := deployExplorer(client, w.network, w.conf.bootnodes, infos, nocache, w.conf.Genesis.Config.Clique != nil); err != nil {
 		log.Error("Failed to deploy explorer container", "err", err)
 		if len(out) > 0 {
 			fmt.Printf("%s\n", out)

cmd/puppeth/wizard_network.go

@@ -174,7 +174,7 @@ func (w *wizard) deployComponent() {
 	fmt.Println(" 1. Ethstats  - Network monitoring tool")
 	fmt.Println(" 2. Bootnode  - Entry point of the network")
 	fmt.Println(" 3. Sealer    - Full node minting new blocks")
-	fmt.Println(" 4. Explorer  - Chain analysis webservice (ethash only)")
+	fmt.Println(" 4. Explorer  - Chain analysis webservice")
 	fmt.Println(" 5. Wallet    - Browser wallet for quick sends")
 	fmt.Println(" 6. Faucet    - Crypto faucet to give away funds")
 	fmt.Println(" 7. Dashboard - Website listing above web-services")