From 16e313699ffd4feecfe275f7be85072562218bdd Mon Sep 17 00:00:00 2001 From: gary rong Date: Tue, 9 Jul 2019 01:49:11 +0800 Subject: [PATCH] cmd/puppeth: integrate blockscout (#18261) * cmd/puppeth: integrate blockscout * cmd/puppeth: expose debug namespace for blockscout * cmd/puppeth: fix dbdir * cmd/puppeth: run explorer in archive mode * cmd/puppeth: ensure node is synced * cmd/puppeth: fix explorer docker alignment + drop unneeded exec * cmd/puppeth: polish up config saving and reloading * cmd/puppeth: check both web and p2p port for explorer service --- cmd/puppeth/module_explorer.go | 174 ++++++++++++++------------------ cmd/puppeth/wizard_dashboard.go | 2 +- cmd/puppeth/wizard_explorer.go | 55 +++++----- cmd/puppeth/wizard_network.go | 2 +- 4 files changed, 109 insertions(+), 124 deletions(-) diff --git a/cmd/puppeth/module_explorer.go b/cmd/puppeth/module_explorer.go index e465fa04a..8fffe1a1c 100644 --- a/cmd/puppeth/module_explorer.go +++ b/cmd/puppeth/module_explorer.go @@ -30,108 +30,86 @@ import ( // explorerDockerfile is the Dockerfile required to run a block explorer. var explorerDockerfile = ` -FROM puppeth/explorer:latest - -ADD ethstats.json /ethstats.json -ADD chain.json /chain.json +FROM puppeth/blockscout:latest +ADD genesis.json /genesis.json RUN \ - echo '(cd ../eth-net-intelligence-api && pm2 start /ethstats.json)' > explorer.sh && \ - echo '(cd ../etherchain-light && npm start &)' >> explorer.sh && \ - echo 'exec /parity/parity --chain=/chain.json --port={{.NodePort}} --tracing=on --fat-db=on --pruning=archive' >> explorer.sh + echo 'geth --cache 512 init /genesis.json' > explorer.sh && \ + echo $'geth --networkid {{.NetworkID}} --syncmode "full" --gcmode "archive" --port {{.EthPort}} --bootnodes {{.Bootnodes}} --ethstats \'{{.Ethstats}}\' --cache=512 --rpc --rpcapi "net,web3,eth,shh,debug" --rpccorsdomain "*" --rpcvhosts "*" --ws --wsorigins "*" --exitwhensynced' >> explorer.sh && \ + echo $'exec geth --networkid {{.NetworkID}} --syncmode "full" --gcmode "archive" --port {{.EthPort}} --bootnodes {{.Bootnodes}} --ethstats \'{{.Ethstats}}\' --cache=512 --rpc --rpcapi "net,web3,eth,shh,debug" --rpccorsdomain "*" --rpcvhosts "*" --ws --wsorigins "*" &' >> explorer.sh && \ + echo '/usr/local/bin/docker-entrypoint.sh postgres &' >> explorer.sh && \ + echo 'sleep 5' >> explorer.sh && \ + echo 'mix do ecto.drop --force, ecto.create, ecto.migrate' >> explorer.sh && \ + echo 'mix phx.server' >> explorer.sh ENTRYPOINT ["/bin/sh", "explorer.sh"] ` -// explorerEthstats is the configuration file for the ethstats javascript client. -var explorerEthstats = `[ - { - "name" : "node-app", - "script" : "app.js", - "log_date_format" : "YYYY-MM-DD HH:mm Z", - "merge_logs" : false, - "watch" : false, - "max_restarts" : 10, - "exec_interpreter" : "node", - "exec_mode" : "fork_mode", - "env": - { - "NODE_ENV" : "production", - "RPC_HOST" : "localhost", - "RPC_PORT" : "8545", - "LISTENING_PORT" : "{{.Port}}", - "INSTANCE_NAME" : "{{.Name}}", - "CONTACT_DETAILS" : "", - "WS_SERVER" : "{{.Host}}", - "WS_SECRET" : "{{.Secret}}", - "VERBOSITY" : 2 - } - } -]` - // explorerComposefile is the docker-compose.yml file required to deploy and // maintain a block explorer. var explorerComposefile = ` version: '2' services: - explorer: - build: . - image: {{.Network}}/explorer - container_name: {{.Network}}_explorer_1 - ports: - - "{{.NodePort}}:{{.NodePort}}" - - "{{.NodePort}}:{{.NodePort}}/udp"{{if not .VHost}} - - "{{.WebPort}}:3000"{{end}} - volumes: - - {{.Datadir}}:/root/.local/share/io.parity.ethereum - environment: - - NODE_PORT={{.NodePort}}/tcp - - STATS={{.Ethstats}}{{if .VHost}} - - VIRTUAL_HOST={{.VHost}} - - VIRTUAL_PORT=3000{{end}} - logging: - driver: "json-file" - options: - max-size: "1m" - max-file: "10" - restart: always + explorer: + build: . + image: {{.Network}}/explorer + container_name: {{.Network}}_explorer_1 + ports: + - "{{.EthPort}}:{{.EthPort}}" + - "{{.EthPort}}:{{.EthPort}}/udp"{{if not .VHost}} + - "{{.WebPort}}:4000"{{end}} + environment: + - ETH_PORT={{.EthPort}} + - ETH_NAME={{.EthName}} + - BLOCK_TRANSFORMER={{.Transformer}}{{if .VHost}} + - VIRTUAL_HOST={{.VHost}} + - VIRTUAL_PORT=4000{{end}} + volumes: + - {{.Datadir}}:/opt/app/.ethereum + - {{.DBDir}}:/var/lib/postgresql/data + logging: + driver: "json-file" + options: + max-size: "1m" + max-file: "10" + restart: always ` // deployExplorer deploys a new block explorer container to a remote machine via // SSH, docker and docker-compose. If an instance with the specified network name // already exists there, it will be overwritten! -func deployExplorer(client *sshClient, network string, chainspec []byte, config *explorerInfos, nocache bool) ([]byte, error) { +func deployExplorer(client *sshClient, network string, bootnodes []string, config *explorerInfos, nocache bool, isClique bool) ([]byte, error) { // Generate the content to upload to the server workdir := fmt.Sprintf("%d", rand.Int63()) files := make(map[string][]byte) dockerfile := new(bytes.Buffer) template.Must(template.New("").Parse(explorerDockerfile)).Execute(dockerfile, map[string]interface{}{ - "NodePort": config.nodePort, + "NetworkID": config.node.network, + "Bootnodes": strings.Join(bootnodes, ","), + "Ethstats": config.node.ethstats, + "EthPort": config.node.port, }) files[filepath.Join(workdir, "Dockerfile")] = dockerfile.Bytes() - ethstats := new(bytes.Buffer) - template.Must(template.New("").Parse(explorerEthstats)).Execute(ethstats, map[string]interface{}{ - "Port": config.nodePort, - "Name": config.ethstats[:strings.Index(config.ethstats, ":")], - "Secret": config.ethstats[strings.Index(config.ethstats, ":")+1 : strings.Index(config.ethstats, "@")], - "Host": config.ethstats[strings.Index(config.ethstats, "@")+1:], - }) - files[filepath.Join(workdir, "ethstats.json")] = ethstats.Bytes() - + transformer := "base" + if isClique { + transformer = "clique" + } composefile := new(bytes.Buffer) template.Must(template.New("").Parse(explorerComposefile)).Execute(composefile, map[string]interface{}{ - "Datadir": config.datadir, - "Network": network, - "NodePort": config.nodePort, - "VHost": config.webHost, - "WebPort": config.webPort, - "Ethstats": config.ethstats[:strings.Index(config.ethstats, ":")], + "Network": network, + "VHost": config.host, + "Ethstats": config.node.ethstats, + "Datadir": config.node.datadir, + "DBDir": config.dbdir, + "EthPort": config.node.port, + "EthName": config.node.ethstats[:strings.Index(config.node.ethstats, ":")], + "WebPort": config.port, + "Transformer": transformer, }) files[filepath.Join(workdir, "docker-compose.yaml")] = composefile.Bytes() - - files[filepath.Join(workdir, "chain.json")] = chainspec + files[filepath.Join(workdir, "genesis.json")] = config.node.genesis // Upload the deployment files to the remote server (and clean up afterwards) if out, err := client.Upload(files); err != nil { @@ -149,22 +127,20 @@ func deployExplorer(client *sshClient, network string, chainspec []byte, config // explorerInfos is returned from a block explorer status check to allow reporting // various configuration parameters. type explorerInfos struct { - datadir string - ethstats string - nodePort int - webHost string - webPort int + node *nodeInfos + dbdir string + host string + port int } // Report converts the typed struct into a plain string->string map, containing // most - but not all - fields for reporting to the user. func (info *explorerInfos) Report() map[string]string { report := map[string]string{ - "Data directory": info.datadir, - "Node listener port ": strconv.Itoa(info.nodePort), - "Ethstats username": info.ethstats, - "Website address ": info.webHost, - "Website listener port ": strconv.Itoa(info.webPort), + "Website address ": info.host, + "Website listener port ": strconv.Itoa(info.port), + "Ethereum listener port ": strconv.Itoa(info.node.port), + "Ethstats username": info.node.ethstats, } return report } @@ -172,7 +148,7 @@ func (info *explorerInfos) Report() map[string]string { // checkExplorer does a health-check against a block explorer server to verify // whether it's running, and if yes, whether it's responsive. func checkExplorer(client *sshClient, network string) (*explorerInfos, error) { - // Inspect a possible block explorer container on the host + // Inspect a possible explorer container on the host infos, err := inspectContainer(client, fmt.Sprintf("%s_explorer_1", network)) if err != nil { return nil, err @@ -181,13 +157,13 @@ func checkExplorer(client *sshClient, network string) (*explorerInfos, error) { return nil, ErrServiceOffline } // Resolve the port from the host, or the reverse proxy - webPort := infos.portmap["3000/tcp"] - if webPort == 0 { + port := infos.portmap["4000/tcp"] + if port == 0 { if proxy, _ := checkNginx(client, network); proxy != nil { - webPort = proxy.port + port = proxy.port } } - if webPort == 0 { + if port == 0 { return nil, ErrNotExposed } // Resolve the host from the reverse-proxy and the config values @@ -196,17 +172,23 @@ func checkExplorer(client *sshClient, network string) (*explorerInfos, error) { host = client.server } // Run a sanity check to see if the devp2p is reachable - nodePort := infos.portmap[infos.envvars["NODE_PORT"]] - if err = checkPort(client.server, nodePort); err != nil { - log.Warn(fmt.Sprintf("Explorer devp2p port seems unreachable"), "server", client.server, "port", nodePort, "err", err) + p2pPort := infos.portmap[infos.envvars["ETH_PORT"]+"/tcp"] + if err = checkPort(host, p2pPort); err != nil { + log.Warn("Explorer node seems unreachable", "server", host, "port", p2pPort, "err", err) + } + if err = checkPort(host, port); err != nil { + log.Warn("Explorer service seems unreachable", "server", host, "port", port, "err", err) } // Assemble and return the useful infos stats := &explorerInfos{ - datadir: infos.volumes["/root/.local/share/io.parity.ethereum"], - nodePort: nodePort, - webHost: host, - webPort: webPort, - ethstats: infos.envvars["STATS"], + node: &nodeInfos{ + datadir: infos.volumes["/opt/app/.ethereum"], + port: infos.portmap[infos.envvars["ETH_PORT"]+"/tcp"], + ethstats: infos.envvars["ETH_NAME"], + }, + dbdir: infos.volumes["/var/lib/postgresql/data"], + host: host, + port: port, } return stats, nil } diff --git a/cmd/puppeth/wizard_dashboard.go b/cmd/puppeth/wizard_dashboard.go index 8a8370845..b699d7617 100644 --- a/cmd/puppeth/wizard_dashboard.go +++ b/cmd/puppeth/wizard_dashboard.go @@ -77,7 +77,7 @@ func (w *wizard) deployDashboard() { } case "explorer": if infos, err := checkExplorer(client, w.network); err == nil { - port = infos.webPort + port = infos.port } case "wallet": if infos, err := checkWallet(client, w.network); err == nil { diff --git a/cmd/puppeth/wizard_explorer.go b/cmd/puppeth/wizard_explorer.go index a128fb9fb..1df9cbc0f 100644 --- a/cmd/puppeth/wizard_explorer.go +++ b/cmd/puppeth/wizard_explorer.go @@ -35,10 +35,6 @@ func (w *wizard) deployExplorer() { log.Error("No ethstats server configured") return } - if w.conf.Genesis.Config.Ethash == nil { - log.Error("Only ethash network supported") - return - } // Select the server to interact with server := w.selectServer() if server == "" { @@ -50,50 +46,57 @@ func (w *wizard) deployExplorer() { infos, err := checkExplorer(client, w.network) if err != nil { infos = &explorerInfos{ - nodePort: 30303, webPort: 80, webHost: client.server, + node: &nodeInfos{port: 30303}, + port: 80, + host: client.server, } } existed := err == nil - chainspec, err := newParityChainSpec(w.network, w.conf.Genesis, w.conf.bootnodes) - if err != nil { - log.Error("Failed to create chain spec for explorer", "err", err) - return - } - chain, _ := json.MarshalIndent(chainspec, "", " ") + infos.node.genesis, _ = json.MarshalIndent(w.conf.Genesis, "", " ") + infos.node.network = w.conf.Genesis.Config.ChainID.Int64() // Figure out which port to listen on fmt.Println() - fmt.Printf("Which port should the explorer listen on? (default = %d)\n", infos.webPort) - infos.webPort = w.readDefaultInt(infos.webPort) + fmt.Printf("Which port should the explorer listen on? (default = %d)\n", infos.port) + infos.port = w.readDefaultInt(infos.port) // Figure which virtual-host to deploy ethstats on - if infos.webHost, err = w.ensureVirtualHost(client, infos.webPort, infos.webHost); err != nil { + if infos.host, err = w.ensureVirtualHost(client, infos.port, infos.host); err != nil { log.Error("Failed to decide on explorer host", "err", err) return } // Figure out where the user wants to store the persistent data fmt.Println() - if infos.datadir == "" { - fmt.Printf("Where should data be stored on the remote machine?\n") - infos.datadir = w.readString() + if infos.node.datadir == "" { + fmt.Printf("Where should node data be stored on the remote machine?\n") + infos.node.datadir = w.readString() } else { - fmt.Printf("Where should data be stored on the remote machine? (default = %s)\n", infos.datadir) - infos.datadir = w.readDefaultString(infos.datadir) + fmt.Printf("Where should node data be stored on the remote machine? (default = %s)\n", infos.node.datadir) + infos.node.datadir = w.readDefaultString(infos.node.datadir) + } + // Figure out where the user wants to store the persistent data for backend database + fmt.Println() + if infos.dbdir == "" { + fmt.Printf("Where should postgres data be stored on the remote machine?\n") + infos.dbdir = w.readString() + } else { + fmt.Printf("Where should postgres data be stored on the remote machine? (default = %s)\n", infos.dbdir) + infos.dbdir = w.readDefaultString(infos.dbdir) } // Figure out which port to listen on fmt.Println() - fmt.Printf("Which TCP/UDP port should the archive node listen on? (default = %d)\n", infos.nodePort) - infos.nodePort = w.readDefaultInt(infos.nodePort) + fmt.Printf("Which TCP/UDP port should the archive node listen on? (default = %d)\n", infos.node.port) + infos.node.port = w.readDefaultInt(infos.node.port) // Set a proper name to report on the stats page fmt.Println() - if infos.ethstats == "" { + if infos.node.ethstats == "" { fmt.Printf("What should the explorer be called on the stats page?\n") - infos.ethstats = w.readString() + ":" + w.conf.ethstats + infos.node.ethstats = w.readString() + ":" + w.conf.ethstats } else { - fmt.Printf("What should the explorer be called on the stats page? (default = %s)\n", infos.ethstats) - infos.ethstats = w.readDefaultString(infos.ethstats) + ":" + w.conf.ethstats + fmt.Printf("What should the explorer be called on the stats page? (default = %s)\n", infos.node.ethstats) + infos.node.ethstats = w.readDefaultString(infos.node.ethstats) + ":" + w.conf.ethstats } // Try to deploy the explorer on the host nocache := false @@ -102,7 +105,7 @@ func (w *wizard) deployExplorer() { fmt.Printf("Should the explorer be built from scratch (y/n)? (default = no)\n") nocache = w.readDefaultYesNo(false) } - if out, err := deployExplorer(client, w.network, chain, infos, nocache); err != nil { + if out, err := deployExplorer(client, w.network, w.conf.bootnodes, infos, nocache, w.conf.Genesis.Config.Clique != nil); err != nil { log.Error("Failed to deploy explorer container", "err", err) if len(out) > 0 { fmt.Printf("%s\n", out) diff --git a/cmd/puppeth/wizard_network.go b/cmd/puppeth/wizard_network.go index 83b10cf37..97302c0df 100644 --- a/cmd/puppeth/wizard_network.go +++ b/cmd/puppeth/wizard_network.go @@ -174,7 +174,7 @@ func (w *wizard) deployComponent() { fmt.Println(" 1. Ethstats - Network monitoring tool") fmt.Println(" 2. Bootnode - Entry point of the network") fmt.Println(" 3. Sealer - Full node minting new blocks") - fmt.Println(" 4. Explorer - Chain analysis webservice (ethash only)") + fmt.Println(" 4. Explorer - Chain analysis webservice") fmt.Println(" 5. Wallet - Browser wallet for quick sends") fmt.Println(" 6. Faucet - Crypto faucet to give away funds") fmt.Println(" 7. Dashboard - Website listing above web-services")