Update monitoring stack with additional dashboards and watcher metrics (#693)

* Include retry jobs and update default refresh intervals

* Add prometheus blackbox exporter and it's dashboard

* Add NodeJS application dashboard

* Allow UI updates

* Update watcher dashboards for upstream and external chain heads

* Update watcher dashboards with watcher config metrics

* Upgrade sushiswap and azimuth watchers

* Removed fixed title size values

* Update instructions

* Update instructions for env config

* Update instructions with setup

stack_orchestrator/data/compose/docker-compose-prom-server.yml

@@ -18,5 +18,30 @@ services:
     extra_hosts:
       - "host.docker.internal:host-gateway"
 
+  blackbox:
+    image: prom/blackbox-exporter:latest
+    restart: always
+    volumes:
+      - ../config/monitoring/blackbox.yml:/etc/blackbox_exporter/config.yml
+    ports:
+      - '9115'
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  chain-head-exporter:
+    image: cerc/watcher-ts:local
+    restart: always
+    working_dir: /app/packages/cli
+    environment:
+      ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      FIL_RPC_ENDPOINT: ${CERC_FIL_RPC_ENDPOINT}
+      ETH_RPC_API_KEY: ${CERC_INFURA_KEY}
+      PORT: ${CERC_METRICS_PORT}
+    command: ["sh", "-c", "yarn export-metrics:chain-heads"]
+    ports:
+      - '5000'
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
 volumes:
   prometheus_data: