tightening up

Reviewed-on: cerc-io/stack-orchestrator#815
Co-authored-by: David Boreham <david@bozemanpass.com>
Co-committed-by: David Boreham <david@bozemanpass.com>

.gitea/workflows/fixturenet-eth-plugeth-arm-test.yml

@@ -9,6 +9,10 @@ on:
   schedule: # Note: coordinate with other tests to not overload runners at the same time of day
     - cron: '2 14 * * *'
 
+# Needed until we can incorporate docker startup into the executor container
+env:
+  DOCKER_HOST: unix:///var/run/dind.sock
+
 
 jobs:
   test:

.gitea/workflows/fixturenet-eth-plugeth-test.yml

@@ -9,6 +9,10 @@ on:
   schedule: # Note: coordinate with other tests to not overload runners at the same time of day
     - cron: '2 14 * * *'
 
+# Needed until we can incorporate docker startup into the executor container
+env:
+  DOCKER_HOST: unix:///var/run/dind.sock
+
 
 jobs:
   test:
@@ -37,6 +41,10 @@ jobs:
         run: ./scripts/create_build_tag_file.sh
       - name: "Build local shiv package"
         run: ./scripts/build_shiv_package.sh
+      - name: Start dockerd # Also needed until we can incorporate into the executor
+        run: |
+          dockerd -H $DOCKER_HOST --userland-proxy=false &
+          sleep 5
       - name: "Run fixturenet-eth tests"
         run: ./tests/fixturenet-eth-plugeth/run-test.sh
       - name: Notify Vulcanize Slack on CI failure

.gitea/workflows/fixturenet-eth-test.yml

@@ -7,6 +7,10 @@ on:
       - '!**'
       - '.gitea/workflows/triggers/fixturenet-eth-test'
 
+# Needed until we can incorporate docker startup into the executor container
+env:
+  DOCKER_HOST: unix:///var/run/dind.sock
+
 
 jobs:
   test:
@@ -35,6 +39,10 @@ jobs:
         run: ./scripts/create_build_tag_file.sh
       - name: "Build local shiv package"
         run: ./scripts/build_shiv_package.sh
+      - name: Start dockerd # Also needed until we can incorporate into the executor
+        run: |
+          dockerd -H $DOCKER_HOST --userland-proxy=false &
+          sleep 5
       - name: "Run fixturenet-eth tests"
         run: ./tests/fixturenet-eth/run-test.sh
       - name: Notify Vulcanize Slack on CI failure

.gitea/workflows/test-deploy.yml

@@ -10,6 +10,9 @@ on:
     paths-ignore:
       - '.gitea/workflows/triggers/*'
 
+# Needed until we can incorporate docker startup into the executor container
+env:
+  DOCKER_HOST: unix:///var/run/dind.sock
 
 jobs:
   test:
@@ -38,6 +41,10 @@ jobs:
         run: ./scripts/create_build_tag_file.sh
       - name: "Build local shiv package"
         run: ./scripts/build_shiv_package.sh
+      - name: Start dockerd # Also needed until we can incorporate into the executor
+        run: |
+          dockerd -H $DOCKER_HOST --userland-proxy=false &
+          sleep 5
       - name: "Run deploy tests"
         run: ./tests/deploy/run-deploy-test.sh
       - name: Notify Vulcanize Slack on CI failure

.gitea/workflows/test-webapp.yml

@@ -10,6 +10,10 @@ on:
     paths-ignore:
       - '.gitea/workflows/triggers/*'
 
+# Needed until we can incorporate docker startup into the executor container
+env:
+  DOCKER_HOST: unix:///var/run/dind.sock
+
 jobs:
   test:
     name: "Run webapp test suite"
@@ -39,6 +43,10 @@ jobs:
         run: ./scripts/build_shiv_package.sh
       - name: "Install wget" # 20240109 - Only needed until the executors are updated.
         run: apt update && apt install -y wget
+      - name: Start dockerd # Also needed until we can incorporate into the executor
+        run: |
+          dockerd -H $DOCKER_HOST --userland-proxy=false &
+          sleep 5
       - name: "Run webapp tests"
         run: ./tests/webapp-test/run-webapp-test.sh
       - name: Notify Vulcanize Slack on CI failure

.gitea/workflows/test.yml

@@ -10,6 +10,9 @@ on:
     paths-ignore:
       - '.gitea/workflows/triggers/*'
 
+# Needed until we can incorporate docker startup into the executor container
+env:
+  DOCKER_HOST: unix:///var/run/dind.sock
 
 jobs:
   test:
@@ -38,6 +41,10 @@ jobs:
         run: ./scripts/create_build_tag_file.sh
       - name: "Build local shiv package"
         run: ./scripts/build_shiv_package.sh
+      - name: Start dockerd # Also needed until we can incorporate into the executor
+        run: |
+          dockerd -H $DOCKER_HOST --userland-proxy=false &
+          sleep 5
       - name: "Run smoke tests"
         run: ./tests/smoke-test/run-smoke-test.sh
       - name: Notify Vulcanize Slack on CI failure

stack_orchestrator/data/compose/docker-compose-fixturenet-lotus.yml

@@ -5,7 +5,7 @@ services:
     hostname: lotus-miner
     env_file:
       - ../config/fixturenet-lotus/lotus-env.env
-    image: cerc/lotus:local
+    image: cerc/lotus-fixturenet:local
     volumes:
       - ../config/fixturenet-lotus/setup-miner.sh:/docker-entrypoint-scripts.d/setup-miner.sh
       - ../config/fixturenet-lotus/fund-account.sh:/fund-account.sh
@@ -26,7 +26,7 @@ services:
     environment:
       # Use 0.0.0.0 so that calls can be made from outside the container
       - LOTUS_API_LISTENADDRESS=/ip4/0.0.0.0/tcp/1234/http
-    image: cerc/lotus:local
+    image: cerc/lotus-fixturenet:local
     volumes:
       - ../config/fixturenet-lotus/setup-node.sh:/docker-entrypoint-scripts.d/setup-node.sh
       - lotus_node_1_params:/var/tmp/filecoin-proof-parameters
@@ -42,10 +42,10 @@ services:
       - lotus-miner
     entrypoint: ["sh", "/docker-entrypoint-scripts.d/setup-node.sh"]
     ports:
-      - "1234"
-      - "2345"
-      - "3456"
-      - "1777"
+      - "1235"
+      - "2346"
+      - "3457"
+      - "1778"
 
   lotus-node-2:
     hostname: lotus-node-2
@@ -54,7 +54,7 @@ services:
     environment:
       # Use 0.0.0.0 so that calls can be made from outside the container
       - LOTUS_API_LISTENADDRESS=/ip4/0.0.0.0/tcp/1234/http
-    image: cerc/lotus:local
+    image: cerc/lotus-fixturenet:local
     volumes:
       - ../config/fixturenet-lotus/setup-node.sh:/docker-entrypoint-scripts.d/setup-node.sh
       - lotus_node_2_params:/var/tmp/filecoin-proof-parameters
@@ -70,10 +70,10 @@ services:
       - lotus-miner
     entrypoint: ["sh", "/docker-entrypoint-scripts.d/setup-node.sh"]
     ports:
-      - "1234"
-      - "2345"
-      - "3456"
-      - "1777"
+      - "1236"
+      - "2347"
+      - "3458"
+      - "1779"
 
 volumes:
   lotus_miner_params:

stack_orchestrator/data/compose/docker-compose-grafana.yml

@@ -6,16 +6,10 @@ services:
     restart: always
     environment:
       GF_SERVER_ROOT_URL: ${GF_SERVER_ROOT_URL}
-      CERC_GRAFANA_ALERTS_SUBGRAPH_IDS: ${CERC_GRAFANA_ALERTS_SUBGRAPH_IDS}
     volumes:
       - ../config/monitoring/grafana/provisioning:/etc/grafana/provisioning
       - ../config/monitoring/grafana/dashboards:/etc/grafana/dashboards
-      - ../config/monitoring/update-grafana-alerts-config.sh:/update-grafana-alerts-config.sh
       - grafana_storage:/var/lib/grafana
-    user: root
-    entrypoint: ["bash", "-c"]
-    command: |
-      "/update-grafana-alerts-config.sh && /run.sh"
     ports:
       - "3000"
     healthcheck:

stack_orchestrator/data/compose/docker-compose-mainnet-lotus.yml

@@ -0,0 +1,23 @@
+volumes:
+  parameters:
+  lotus-repo:
+  lotus_shared:
+
+services:
+  lotus:
+    environment:
+      - LOTUS_API_LISTENADDRESS=/ip4/0.0.0.0/tcp/1234/http
+      - LOTUS_LIBP2P_LISTENADDRESSES=/ip4/0.0.0.0/tcp/1235,/ip6/::/tcp/1235,/ip4/0.0.0.0/udp/1235/quic-v1,/ip6/::/udp/1235/quic-v1,/ip4/0.0.0.0/udp/1235/quic-v1/webtransport,/ip6/::/udp/1235/quic-v1/webtransport
+    image: cerc/lotus-mainnet:local
+    volumes:
+      - parameters:/var/tmp/filecoin-proof-parameters
+      - lotus-repo:/var/lib/lotus
+      - lotus_shared:/root/.lotus
+    ports:
+      - "1234"
+      - "1235"
+    deploy:
+      restart_policy:
+        condition: on-failure
+        delay: 30s
+    entrypoint: ["sh", "/docker-lotus-entrypoint.sh"]

stack_orchestrator/data/compose/docker-compose-prom-server.yml

@@ -18,14 +18,6 @@ services:
     extra_hosts:
       - "host.docker.internal:host-gateway"
 
-  alert-manager:
-    image: prom/alertmanager
-    volumes:
-      - ../config/monitoring/alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml
-      - alertmanager_data:/data
-    ports:
-      - "9093"
-
   blackbox:
     image: prom/blackbox-exporter:latest
     restart: always
@@ -63,4 +55,3 @@ services:
 
 volumes:
   prometheus_data:
-  alertmanager_data:

stack_orchestrator/data/compose/docker-compose-watcher-ajna.yml

@@ -29,7 +29,7 @@ services:
     image: cerc/watcher-ajna:local
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
     command: ["bash", "./start-job-runner.sh"]
     volumes:
       - ../config/watcher-ajna/watcher-config-template.toml:/app/environments/watcher-config-template.toml
@@ -37,7 +37,7 @@ services:
     ports:
       - "9000"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "127.0.0.1", "9000"]
+      test: ["CMD", "nc", "-v", "localhost", "9000"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -55,7 +55,7 @@ services:
     image: cerc/watcher-ajna:local
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
     command: ["bash", "./start-server.sh"]
     volumes:
       - ../config/watcher-ajna/watcher-config-template.toml:/app/environments/watcher-config-template.toml
@@ -64,7 +64,7 @@ services:
       - "3008"
       - "9001"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "127.0.0.1", "3008"]
+      test: ["CMD", "nc", "-v", "localhost", "3008"]
       interval: 20s
       timeout: 5s
       retries: 15

stack_orchestrator/data/compose/docker-compose-watcher-azimuth.yml

@@ -32,8 +32,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
+      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
       CERC_HISTORICAL_BLOCK_RANGE: 500
       CONTRACT_ADDRESS: 0x223c067F8CF28ae173EE5CafEa60cA44C335fecB
       CONTRACT_NAME: Azimuth
@@ -47,7 +47,7 @@ services:
     ports:
       - "9000"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "127.0.0.1", "9000"]
+      test: ["CMD", "nc", "-vz", "localhost", "9000"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -66,8 +66,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
+      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
     working_dir: /app/packages/azimuth-watcher
     command: "./start-server.sh"
     volumes:
@@ -77,7 +77,7 @@ services:
     ports:
       - "3001"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "127.0.0.1", "3001"]
+      test: ["CMD", "nc", "-vz", "localhost", "3001"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -94,8 +94,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
+      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
       CONTRACT_ADDRESS: 0x325f68d32BdEe6Ed86E7235ff2480e2A433D6189
       CONTRACT_NAME: Censures
       STARTING_BLOCK: 6784954
@@ -108,7 +108,7 @@ services:
     ports:
       - "9002"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "127.0.0.1", "9002"]
+      test: ["CMD", "nc", "-vz", "localhost", "9002"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -127,8 +127,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
+      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
     working_dir: /app/packages/censures-watcher
     command: "./start-server.sh"
     volumes:
@@ -138,7 +138,7 @@ services:
     ports:
       - "3002"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "127.0.0.1", "3002"]
+      test: ["CMD", "nc", "-vz", "localhost", "3002"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -155,8 +155,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
+      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
       CONTRACT_ADDRESS: 0xe7e7f69b34D7d9Bd8d61Fb22C33b22708947971A
       CONTRACT_NAME: Claims
       STARTING_BLOCK: 6784941
@@ -169,7 +169,7 @@ services:
     ports:
       - "9004"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "127.0.0.1", "9004"]
+      test: ["CMD", "nc", "-vz", "localhost", "9004"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -188,8 +188,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
+      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
     working_dir: /app/packages/claims-watcher
     command: "./start-server.sh"
     volumes:
@@ -199,7 +199,7 @@ services:
     ports:
       - "3003"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "127.0.0.1", "3003"]
+      test: ["CMD", "nc", "-vz", "localhost", "3003"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -216,8 +216,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
+      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
       CONTRACT_ADDRESS: 0x8C241098C3D3498Fe1261421633FD57986D74AeA
       CONTRACT_NAME: ConditionalStarRelease
       STARTING_BLOCK: 6828004
@@ -230,7 +230,7 @@ services:
     ports:
       - "9006"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "127.0.0.1", "9006"]
+      test: ["CMD", "nc", "-vz", "localhost", "9006"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -249,8 +249,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
+      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
     working_dir: /app/packages/conditional-star-release-watcher
     command: "./start-server.sh"
     volumes:
@@ -260,7 +260,7 @@ services:
     ports:
       - "3004"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "127.0.0.1", "3004"]
+      test: ["CMD", "nc", "-vz", "localhost", "3004"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -277,8 +277,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
+      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
       CONTRACT_ADDRESS: 0xf6b461fE1aD4bd2ce25B23Fe0aff2ac19B3dFA76
       CONTRACT_NAME: DelegatedSending
       STARTING_BLOCK: 6784956
@@ -291,7 +291,7 @@ services:
     ports:
       - "9008"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "127.0.0.1", "9008"]
+      test: ["CMD", "nc", "-vz", "localhost", "9008"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -310,8 +310,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
+      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
     working_dir: /app/packages/delegated-sending-watcher
     command: "./start-server.sh"
     volumes:
@@ -321,7 +321,7 @@ services:
     ports:
       - "3005"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "127.0.0.1", "3005"]
+      test: ["CMD", "nc", "-vz", "localhost", "3005"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -338,8 +338,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
+      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
       CONTRACT_ADDRESS: 0x33EeCbf908478C10614626A9D304bfe18B78DD73
       CONTRACT_NAME: Ecliptic
       STARTING_BLOCK: 13692129
@@ -352,7 +352,7 @@ services:
     ports:
       - "9010"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "127.0.0.1", "9010"]
+      test: ["CMD", "nc", "-vz", "localhost", "9010"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -371,8 +371,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
+      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
     working_dir: /app/packages/ecliptic-watcher
     command: "./start-server.sh"
     volumes:
@@ -382,7 +382,7 @@ services:
     ports:
       - "3006"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "127.0.0.1", "3006"]
+      test: ["CMD", "nc", "-vz", "localhost", "3006"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -399,8 +399,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
+      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
       CONTRACT_ADDRESS: 0x86cd9cd0992F04231751E3761De45cEceA5d1801
       CONTRACT_NAME: LinearStarRelease
       STARTING_BLOCK: 6784943
@@ -413,7 +413,7 @@ services:
     ports:
       - "9012"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "127.0.0.1", "9012"]
+      test: ["CMD", "nc", "-vz", "localhost", "9012"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -432,8 +432,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
+      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
     working_dir: /app/packages/linear-star-release-watcher
     command: "./start-server.sh"
     volumes:
@@ -443,7 +443,7 @@ services:
     ports:
       - "3007"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "127.0.0.1", "3007"]
+      test: ["CMD", "nc", "-vz", "localhost", "3007"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -460,8 +460,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
+      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
       CONTRACT_ADDRESS: 0x7fEcaB617c868Bb5996d99D95200D2Fa708218e4
       CONTRACT_NAME: Polls
       STARTING_BLOCK: 6784912
@@ -474,7 +474,7 @@ services:
     ports:
       - "9014"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "127.0.0.1", "9014"]
+      test: ["CMD", "nc", "-vz", "localhost", "9014"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -493,8 +493,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
+      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
     working_dir: /app/packages/polls-watcher
     command: "./start-server.sh"
     volumes:
@@ -504,7 +504,7 @@ services:
     ports:
       - "3008"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "127.0.0.1", "3008"]
+      test: ["CMD", "nc", "-vz", "localhost", "3008"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -542,7 +542,7 @@ services:
     ports:
       - "0.0.0.0:4000:4000"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "127.0.0.1", "4000"]
+      test: ["CMD", "nc", "-vz", "localhost", "4000"]
       interval: 20s
       timeout: 5s
       retries: 15

stack_orchestrator/data/compose/docker-compose-watcher-merkl-sushiswap-v3.yml

@@ -29,7 +29,7 @@ services:
     image: cerc/watcher-merkl-sushiswap-v3:local
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
     command: ["bash", "./start-job-runner.sh"]
     volumes:
       - ../config/watcher-merkl-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
@@ -37,7 +37,7 @@ services:
     ports:
       - "9002:9000"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "127.0.0.1", "9000"]
+      test: ["CMD", "nc", "-v", "localhost", "9000"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -55,7 +55,7 @@ services:
     image: cerc/watcher-merkl-sushiswap-v3:local
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
     command: ["bash", "./start-server.sh"]
     volumes:
       - ../config/watcher-merkl-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
@@ -64,7 +64,7 @@ services:
       - "127.0.0.1:3007:3008"
       - "9003:9001"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "127.0.0.1", "3008"]
+      test: ["CMD", "nc", "-v", "localhost", "3008"]
       interval: 20s
       timeout: 5s
       retries: 15

stack_orchestrator/data/compose/docker-compose-watcher-sushiswap-v3.yml

@@ -29,7 +29,7 @@ services:
     image: cerc/watcher-sushiswap-v3:local
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
     command: ["bash", "./start-job-runner.sh"]
     volumes:
       - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
@@ -37,7 +37,7 @@ services:
     ports:
       - "9000:9000"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "127.0.0.1", "9000"]
+      test: ["CMD", "nc", "-v", "localhost", "9000"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -55,7 +55,7 @@ services:
     image: cerc/watcher-sushiswap-v3:local
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
     command: ["bash", "./start-server.sh"]
     volumes:
       - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
@@ -64,7 +64,7 @@ services:
       - "127.0.0.1:3008:3008"
       - "9001:9001"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "127.0.0.1", "3008"]
+      test: ["CMD", "nc", "-v", "localhost", "3008"]
       interval: 20s
       timeout: 5s
       retries: 15

stack_orchestrator/data/config/monitoring/alertmanager/alertmanager.yml

@@ -1,10 +0,0 @@
-route:
-  receiver: 'slack'
-  # group_by: [ alertname ]
-
-receivers:
-  - name: 'slack'
-    slack_configs:
-      - send_resolved: true
-        api_url: <YOUR_SLACK_WEBHOOK_URL>
-        channel: <YOUR_SLACK_CHANNEL>

stack_orchestrator/data/config/monitoring/grafana/dashboards/watcher-dashboard.json

@@ -18,7 +18,7 @@
   "editable": true,
   "fiscalYearStartMonth": 0,
   "graphTooltip": 0,
-  "id": 6,
+  "id": 4,
   "links": [
     {
       "asDropdown": false,
@@ -55,7 +55,7 @@
         "x": 0,
         "y": 0
       },
-      "id": 29,
+      "id": 1,
       "panels": [
         {
           "datasource": {
@@ -103,7 +103,7 @@
             "textMode": "auto",
             "wideLayout": true
           },
-          "pluginVersion": "10.2.3",
+          "pluginVersion": "10.2.2",
           "targets": [
             {
               "datasource": {
@@ -173,7 +173,7 @@
             "textMode": "auto",
             "wideLayout": true
           },
-          "pluginVersion": "10.2.3",
+          "pluginVersion": "10.2.2",
           "targets": [
             {
               "datasource": {
@@ -242,7 +242,7 @@
             "textMode": "auto",
             "wideLayout": true
           },
-          "pluginVersion": "10.2.3",
+          "pluginVersion": "10.2.2",
           "targets": [
             {
               "datasource": {
@@ -438,7 +438,7 @@
             "textMode": "auto",
             "wideLayout": true
           },
-          "pluginVersion": "10.2.3",
+          "pluginVersion": "10.2.2",
           "targets": [
             {
               "datasource": {
@@ -518,7 +518,7 @@
             "textMode": "auto",
             "wideLayout": true
           },
-          "pluginVersion": "10.2.3",
+          "pluginVersion": "10.2.2",
           "targets": [
             {
               "datasource": {
@@ -629,7 +629,7 @@
             "textMode": "auto",
             "wideLayout": true
           },
-          "pluginVersion": "10.2.3",
+          "pluginVersion": "10.2.2",
           "targets": [
             {
               "datasource": {
@@ -730,7 +730,7 @@
             "textMode": "auto",
             "wideLayout": true
           },
-          "pluginVersion": "10.2.3",
+          "pluginVersion": "10.2.2",
           "targets": [
             {
               "datasource": {
@@ -854,7 +854,7 @@
             "textMode": "auto",
             "wideLayout": true
           },
-          "pluginVersion": "10.2.3",
+          "pluginVersion": "10.2.2",
           "targets": [
             {
               "datasource": {
@@ -1025,7 +1025,7 @@
             "textMode": "auto",
             "wideLayout": true
           },
-          "pluginVersion": "10.2.3",
+          "pluginVersion": "10.2.2",
           "targets": [
             {
               "datasource": {
@@ -1111,7 +1111,7 @@
             "textMode": "auto",
             "wideLayout": true
           },
-          "pluginVersion": "10.2.3",
+          "pluginVersion": "10.2.2",
           "targets": [
             {
               "datasource": {
@@ -1146,230 +1146,6 @@
           "title": "DB size (MB)",
           "type": "stat"
         },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "description": "",
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "thresholds"
-              },
-              "custom": {
-                "axisBorderShow": false,
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 0,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "insertNulls": false,
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "auto",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "dark-green",
-                    "value": null
-                  },
-                  {
-                    "color": "dark-orange",
-                    "value": 8
-                  },
-                  {
-                    "color": "dark-red",
-                    "value": 16
-                  }
-                ]
-              }
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "diff_external"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "mode": "thresholds",
-                      "seriesBy": "last"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "diff_external"
-                },
-                "properties": [
-                  {
-                    "id": "thresholds",
-                    "value": {
-                      "mode": "absolute",
-                      "steps": [
-                        {
-                          "color": "green",
-                          "value": null
-                        },
-                        {
-                          "color": "orange",
-                          "value": 8
-                        },
-                        {
-                          "color": "red",
-                          "value": 16
-                        }
-                      ]
-                    }
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 6,
-            "w": 12,
-            "x": 0,
-            "y": 12
-          },
-          "id": 33,
-          "options": {
-            "legend": {
-              "calcs": [],
-              "displayMode": "list",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "single",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "10.2.3",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "PBFA97CFB590B2093"
-              },
-              "disableTextWrap": false,
-              "editorMode": "code",
-              "expr": "latest_upstream_block_number{job=~\"$job\", instance=~\"$watcher\"}",
-              "fullMetaSearch": false,
-              "hide": false,
-              "includeNullMetadata": true,
-              "instant": false,
-              "legendFormat": "{{__name__}}",
-              "range": true,
-              "refId": "A",
-              "useBackend": false
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "PBFA97CFB590B2093"
-              },
-              "editorMode": "code",
-              "expr": "sync_status_block_number{job=~\"$job\", instance=~\"$watcher\", kind=\"latest_indexed\"}",
-              "hide": false,
-              "instant": false,
-              "legendFormat": "{{__name__}}",
-              "range": true,
-              "refId": "B"
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "PBFA97CFB590B2093"
-              },
-              "editorMode": "code",
-              "expr": "latest_block_number{job=\"chain_heads\", chain=\"$target_chain\"}",
-              "hide": false,
-              "instant": false,
-              "legendFormat": "{{__name__}}",
-              "range": true,
-              "refId": "C"
-            }
-          ],
-          "title": "diff",
-          "transformations": [
-            {
-              "id": "calculateField",
-              "options": {
-                "alias": "diff_upstream",
-                "binary": {
-                  "left": "latest_upstream_block_number",
-                  "operator": "-",
-                  "right": "sync_status_block_number"
-                },
-                "mode": "binary",
-                "reduce": {
-                  "reducer": "sum"
-                },
-                "replaceFields": false
-              }
-            },
-            {
-              "id": "calculateField",
-              "options": {
-                "alias": "diff_external",
-                "binary": {
-                  "left": "latest_block_number",
-                  "operator": "-",
-                  "right": "sync_status_block_number"
-                },
-                "mode": "binary",
-                "reduce": {
-                  "reducer": "sum"
-                },
-                "replaceFields": false
-              }
-            },
-            {
-              "id": "organize",
-              "options": {
-                "excludeByName": {
-                  "Time": false,
-                  "latest_block_number": true,
-                  "latest_upstream_block_number": true,
-                  "sync_status_block_number": true
-                },
-                "includeByName": {},
-                "indexByName": {},
-                "renameByName": {}
-              }
-            }
-          ],
-          "type": "timeseries"
-        },
         {
           "datasource": {
             "type": "prometheus",
@@ -1432,8 +1208,8 @@
           },
           "gridPos": {
             "h": 6,
-            "w": 6,
-            "x": 12,
+            "w": 12,
+            "x": 0,
             "y": 12
           },
           "id": 5,
@@ -1534,8 +1310,8 @@
           },
           "gridPos": {
             "h": 6,
-            "w": 6,
-            "x": 18,
+            "w": 12,
+            "x": 12,
             "y": 12
           },
           "id": 15,
@@ -1574,436 +1350,6 @@
           "title": "Num event-processing jobs",
           "type": "timeseries"
         },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "description": "Total number of failed ETH RPC requests by method and provider endpoint ",
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisBorderShow": false,
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 0,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "insertNulls": false,
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "auto",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green",
-                    "value": null
-                  }
-                ]
-              }
-            },
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 7,
-            "w": 12,
-            "x": 0,
-            "y": 18
-          },
-          "id": 28,
-          "options": {
-            "legend": {
-              "calcs": [],
-              "displayMode": "list",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "single",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "10.2.3",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "PBFA97CFB590B2093"
-              },
-              "disableTextWrap": false,
-              "editorMode": "code",
-              "exemplar": false,
-              "expr": "watcher_eth_rpc_errors{job=~\"$job\", instance=~\"$watcher\"}",
-              "format": "time_series",
-              "fullMetaSearch": false,
-              "includeNullMetadata": true,
-              "instant": false,
-              "legendFormat": "{{method}}, {{provider}}",
-              "range": true,
-              "refId": "A",
-              "useBackend": false
-            }
-          ],
-          "title": "ETH RPC request failures",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "description": "Time taken by last ETH RPC requests by method and provider endpoint",
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisBorderShow": false,
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 0,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "insertNulls": false,
-                "lineInterpolation": "linear",
-                "lineStyle": {
-                  "fill": "solid"
-                },
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "auto",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green",
-                    "value": null
-                  }
-                ]
-              },
-              "unit": "s"
-            },
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 7,
-            "w": 12,
-            "x": 12,
-            "y": 18
-          },
-          "id": 26,
-          "options": {
-            "legend": {
-              "calcs": [],
-              "displayMode": "list",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "single",
-              "sort": "none"
-            }
-          },
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "PBFA97CFB590B2093"
-              },
-              "editorMode": "code",
-              "expr": "watcher_eth_rpc_request_duration{job=~\"$job\", instance=~\"$watcher\"}",
-              "instant": false,
-              "legendFormat": "{{method}}, {{provider}}",
-              "range": true,
-              "refId": "A"
-            }
-          ],
-          "title": "ETH RPC request durations",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "description": "Total number of failed ETH RPC requests by method and provider endpoint  (across all watchers)",
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisBorderShow": false,
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 0,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "insertNulls": false,
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "auto",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green",
-                    "value": null
-                  }
-                ]
-              }
-            },
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 7,
-            "w": 12,
-            "x": 0,
-            "y": 25
-          },
-          "id": 30,
-          "options": {
-            "legend": {
-              "calcs": [],
-              "displayMode": "list",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "single",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "10.2.3",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "PBFA97CFB590B2093"
-              },
-              "disableTextWrap": false,
-              "editorMode": "code",
-              "exemplar": false,
-              "expr": "sum by (method, provider) (watcher_eth_rpc_errors{chain=\"$target_chain\"})",
-              "format": "time_series",
-              "fullMetaSearch": false,
-              "includeNullMetadata": true,
-              "instant": false,
-              "legendFormat": "{{method}}, {{provider}}",
-              "range": true,
-              "refId": "A",
-              "useBackend": false
-            }
-          ],
-          "title": "Total ETH RPC request failures (across all watchers)",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "description": "Configured upstream ETH RPC endpoints",
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "fixedColor": "blue",
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "align": "auto",
-                "cellOptions": {
-                  "type": "auto"
-                },
-                "filterable": false,
-                "inspect": false
-              },
-              "mappings": [
-                {
-                  "options": {
-                    "0": {
-                      "color": "red",
-                      "index": 0,
-                      "text": "inactive"
-                    },
-                    "1": {
-                      "color": "green",
-                      "index": 1,
-                      "text": "active"
-                    }
-                  },
-                  "type": "value"
-                }
-              ],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "blue",
-                    "value": null
-                  }
-                ]
-              }
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "status"
-                },
-                "properties": [
-                  {
-                    "id": "custom.cellOptions",
-                    "value": {
-                      "type": "color-text"
-                    }
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 7,
-            "w": 12,
-            "x": 12,
-            "y": 25
-          },
-          "id": 32,
-          "options": {
-            "cellHeight": "sm",
-            "footer": {
-              "countRows": false,
-              "enablePagination": false,
-              "fields": "",
-              "reducer": [
-                "sum"
-              ],
-              "show": false
-            },
-            "frameIndex": 0,
-            "showHeader": true,
-            "sortBy": [
-              {
-                "desc": true,
-                "displayName": "status"
-              }
-            ]
-          },
-          "pluginVersion": "10.2.3",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "PBFA97CFB590B2093"
-              },
-              "editorMode": "code",
-              "exemplar": false,
-              "expr": "watcher_config_upstream_endpoints{job=~\"$job\", instance=~\"$watcher\"}",
-              "format": "table",
-              "instant": true,
-              "legendFormat": "__auto",
-              "range": false,
-              "refId": "A"
-            }
-          ],
-          "title": "Configured upstream endpoints",
-          "transformations": [
-            {
-              "id": "labelsToFields",
-              "options": {
-                "mode": "columns"
-              }
-            },
-            {
-              "id": "organize",
-              "options": {
-                "excludeByName": {
-                  "Time": true,
-                  "__name__": true,
-                  "chain": true,
-                  "instance": true,
-                  "job": true
-                },
-                "includeByName": {},
-                "indexByName": {},
-                "renameByName": {
-                  "Value": "status"
-                }
-              }
-            }
-          ],
-          "type": "table"
-        },
         {
           "datasource": {
             "type": "prometheus",
@@ -2031,8 +1377,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   }
                 ]
               }
@@ -2043,7 +1388,7 @@
             "h": 5,
             "w": 8,
             "x": 0,
-            "y": 32
+            "y": 18
           },
           "id": 18,
           "options": {
@@ -2065,7 +1410,7 @@
             "textMode": "auto",
             "wideLayout": true
           },
-          "pluginVersion": "10.2.3",
+          "pluginVersion": "10.2.2",
           "targets": [
             {
               "datasource": {
@@ -2177,8 +1522,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   }
                 ]
               }
@@ -2189,7 +1533,7 @@
             "h": 5,
             "w": 8,
             "x": 8,
-            "y": 32
+            "y": 18
           },
           "id": 17,
           "options": {
@@ -2211,7 +1555,7 @@
             "textMode": "value_and_name",
             "wideLayout": true
           },
-          "pluginVersion": "10.2.3",
+          "pluginVersion": "10.2.2",
           "targets": [
             {
               "datasource": {
@@ -2325,8 +1669,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   }
                 ]
               }
@@ -2337,7 +1680,7 @@
             "h": 5,
             "w": 8,
             "x": 16,
-            "y": 32
+            "y": 18
           },
           "id": 19,
           "options": {
@@ -2359,7 +1702,7 @@
             "textMode": "auto",
             "wideLayout": true
           },
-          "pluginVersion": "10.2.3",
+          "pluginVersion": "10.2.2",
           "targets": [
             {
               "datasource": {
@@ -2449,6 +1792,7 @@
             "uid": "PBFA97CFB590B2093"
           },
           "description": "Total number of subgraph entities loaded in event processing",
+          "hide": true,
           "fieldConfig": {
             "defaults": {
               "color": {
@@ -2492,8 +1836,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   }
                 ]
               }
@@ -2504,9 +1847,8 @@
             "h": 6,
             "w": 8,
             "x": 0,
-            "y": 37
+            "y": 23
           },
-          "hide": true,
           "id": 8,
           "options": {
             "legend": {
@@ -2590,8 +1932,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   }
                 ]
               }
@@ -2602,7 +1943,7 @@
             "h": 6,
             "w": 8,
             "x": 8,
-            "y": 37
+            "y": 23
           },
           "id": 9,
           "options": {
@@ -2687,8 +2028,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   }
                 ]
               }
@@ -2699,7 +2039,7 @@
             "h": 6,
             "w": 8,
             "x": 16,
-            "y": 37
+            "y": 23
           },
           "id": 10,
           "options": {
@@ -2744,7 +2084,7 @@
     }
   ],
   "refresh": "10s",
-  "schemaVersion": 39,
+  "schemaVersion": 38,
   "tags": [
     "watcher"
   ],
@@ -2842,6 +2182,6 @@
   "timepicker": {},
   "timezone": "",
   "title": "Watchers",
-  "version": 3,
+  "version": 1,
   "weekStart": ""
 }

stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml

@@ -3,13 +3,8 @@ global:
   evaluation_interval: 15s
 
 rule_files:
-  - rules.yml
-
-alerting:
-  alertmanagers:
-   - static_configs:
-     - targets:
-        - alert-manager:9093
+  # - "first.rules"
+  # - "second.rules"
 
 scrape_configs:
   - job_name: prometheus

stack_orchestrator/data/config/monitoring/prometheus/rules.yml

@@ -1,10 +0,0 @@
-groups:
-  - name: WatcherAlerts
-    rules:
-      - alert: SushiswapWatcherHeadTracking
-        expr: latest_block_number - on(chain) group_right sync_status_block_number{job="sushi", instance="sushiswap", kind="latest_indexed"} > 16
-        for: 5m
-        labels:
-          severity: 'critical'
-        annotations:
-          summary: 'Watcher {{ $labels.instance }} of job {{ $labels.job }} is falling behind external head'

stack_orchestrator/data/config/monitoring/subgraph-alert-rules.yml

@@ -1,64 +0,0 @@
-apiVersion: 1
-groups:
-  - orgId: 1
-    name: subgraph
-    folder: SubgraphAlerts
-    interval: 30s
-    rules:
-      - uid: b2a9144b-6104-46fc-92b5-352f4e643c4c
-        title: subgraph_head_tracking
-        condition: condition
-        data:
-          - refId: diff
-            relativeTimeRange:
-              from: 600
-              to: 0
-            datasourceUid: PBFA97CFB590B2093
-            model:
-              datasource:
-                  type: prometheus
-                  uid: PBFA97CFB590B2093
-              editorMode: code
-              expr: ethereum_chain_head_number - on(network) group_right deployment_head{deployment=~"REPLACE_WITH_SUBGRAPH_IDS"}
-              instant: true
-              intervalMs: 1000
-              legendFormat: __auto
-              maxDataPoints: 43200
-              range: false
-              refId: diff
-          - refId: condition
-            relativeTimeRange:
-              from: 600
-              to: 0
-            datasourceUid: __expr__
-            model:
-              conditions:
-                  - evaluator:
-                      params:
-                          - 15
-                          - 0
-                      type: gt
-                    operator:
-                      type: and
-                    query:
-                      params: []
-                    reducer:
-                      params: []
-                      type: avg
-                    type: query
-              datasource:
-                  name: Expression
-                  type: __expr__
-                  uid: __expr__
-              expression: diff
-              intervalMs: 1000
-              maxDataPoints: 43200
-              refId: condition
-              type: threshold
-        noDataState: OK
-        execErrState: Alerting
-        for: 5m
-        annotations:
-          summary: Subgraph deployment {{ index $labels "deployment" }} is falling behind head by {{ index $values "diff" }}
-        labels: {}
-        isPaused: false

stack_orchestrator/data/config/monitoring/update-grafana-alerts-config.sh

@@ -1,7 +0,0 @@
-#!/bin/bash
-
-echo Using CERC_GRAFANA_ALERTS_SUBGRAPH_IDS ${CERC_GRAFANA_ALERTS_SUBGRAPH_IDS}
-
-# Replace subgraph ids in subgraph alerting config
-# Note: Requires the grafana container to be run with user root
-sed -i "s/REPLACE_WITH_SUBGRAPH_IDS/$CERC_GRAFANA_ALERTS_SUBGRAPH_IDS/g" /etc/grafana/provisioning/alerting/subgraph-alert-rules.yml

stack_orchestrator/data/config/watcher-ajna/start-job-runner.sh

@@ -6,16 +6,12 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u
 
-echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
+echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
 
 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
-
-# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
-RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
-
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
 
 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml

stack_orchestrator/data/config/watcher-ajna/start-server.sh

@@ -6,16 +6,12 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u
 
-echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
+echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
 
 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
-
-# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
-RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
-
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
 
 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml

stack_orchestrator/data/config/watcher-ajna/watcher-config-template.toml

@@ -58,7 +58,7 @@
 
 [upstream]
   [upstream.ethServer]
-    rpcProviderEndpoints = REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS
+    rpcProviderEndpoint = "REPLACE_WITH_CERC_ETH_RPC_ENDPOINT"
 
     # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client)
     rpcClient = true
@@ -96,6 +96,3 @@
   # Max block range of historical processing after which it waits for completion of events processing
   # If set to -1 historical processing does not wait for events processing and completes till latest canonical block
   historicalMaxFetchAhead = 10000
-
-  # Max number of retries to fetch new block after which watcher will failover to other RPC endpoints
-  maxNewBlockRetries = 3

stack_orchestrator/data/config/watcher-azimuth/start-job-runner.sh

@@ -4,19 +4,16 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
   set -x
 fi
 
-echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
-echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL_ENDPOINT}"
+echo "Using IPLD ETH RPC endpoint ${CERC_IPLD_ETH_RPC}"
+echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL}"
 echo "Using historicalLogsBlockRange ${CERC_HISTORICAL_BLOCK_RANGE:-2000}"
 
-# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
-RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
-
 # Replace env variables in template TOML file
 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}|g; \
-          s|REPLACE_WITH_CERC_IPLD_ETH_GQL_ENDPOINT|${CERC_IPLD_ETH_GQL_ENDPOINT}|g; \
+  sed -E "s|REPLACE_WITH_CERC_IPLD_ETH_RPC|${CERC_IPLD_ETH_RPC}|g; \
+          s|REPLACE_WITH_CERC_IPLD_ETH_GQL|${CERC_IPLD_ETH_GQL}|g; \
           s|REPLACE_WITH_CERC_HISTORICAL_BLOCK_RANGE|${CERC_HISTORICAL_BLOCK_RANGE:-2000}| ")
 
 # Write the modified content to a new file

stack_orchestrator/data/config/watcher-azimuth/start-server.sh

@@ -4,19 +4,16 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
   set -x
 fi
 
-echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
-echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL_ENDPOINT}"
+echo "Using IPLD ETH RPC endpoint ${CERC_IPLD_ETH_RPC}"
+echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL}"
 echo "Using historicalLogsBlockRange ${CERC_HISTORICAL_BLOCK_RANGE:-2000}"
 
-# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
-RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
-
 # Replace env variables in template TOML file
 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}|g; \
-          s|REPLACE_WITH_CERC_IPLD_ETH_GQL_ENDPOINT|${CERC_IPLD_ETH_GQL_ENDPOINT}|g; \
+  sed -E "s|REPLACE_WITH_CERC_IPLD_ETH_RPC|${CERC_IPLD_ETH_RPC}|g; \
+          s|REPLACE_WITH_CERC_IPLD_ETH_GQL|${CERC_IPLD_ETH_GQL}|g; \
           s|REPLACE_WITH_CERC_HISTORICAL_BLOCK_RANGE|${CERC_HISTORICAL_BLOCK_RANGE:-2000}| ")
 
 # Write the modified content to a new file

stack_orchestrator/data/config/watcher-azimuth/watcher-config-template.toml

@@ -13,8 +13,8 @@
 
 [upstream]
   [upstream.ethServer]
-    gqlApiEndpoint = "REPLACE_WITH_CERC_IPLD_ETH_GQL_ENDPOINT"
-    rpcProviderEndpoints = REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS
+    gqlApiEndpoint = "REPLACE_WITH_CERC_IPLD_ETH_GQL"
+    rpcProviderEndpoint = "REPLACE_WITH_CERC_IPLD_ETH_RPC"
 
 [jobQueue]
   historicalLogsBlockRange = REPLACE_WITH_CERC_HISTORICAL_BLOCK_RANGE

stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/start-job-runner.sh

@@ -6,16 +6,12 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u
 
-echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
+echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
 
 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
-
-# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
-RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
-
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
 
 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml

stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/start-server.sh

@@ -6,16 +6,12 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u
 
-echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
+echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
 
 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
-
-# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
-RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
-
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
 
 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml

stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml

@@ -58,7 +58,7 @@
 
 [upstream]
   [upstream.ethServer]
-    rpcProviderEndpoints = REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS
+    rpcProviderEndpoint = "REPLACE_WITH_CERC_ETH_RPC_ENDPOINT"
 
     # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client)
     rpcClient = true
@@ -69,7 +69,7 @@
     # Boolean flag to filter event logs by contracts
     filterLogsByAddresses = true
     # Boolean flag to filter event logs by topics
-    filterLogsByTopics = true
+    filterLogsByTopics = false
 
   [upstream.cache]
     name = "requests"
@@ -96,6 +96,3 @@
   # Max block range of historical processing after which it waits for completion of events processing
   # If set to -1 historical processing does not wait for events processing and completes till latest canonical block
   historicalMaxFetchAhead = 10000
-
-  # Max number of retries to fetch new block after which watcher will failover to other RPC endpoints
-  maxNewBlockRetries = 3

stack_orchestrator/data/config/watcher-sushiswap-v3/start-job-runner.sh

@@ -6,16 +6,12 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u
 
-echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
+echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
 
 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
-
-# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
-RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
-
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
 
 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml

stack_orchestrator/data/config/watcher-sushiswap-v3/start-server.sh

@@ -6,16 +6,12 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u
 
-echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
+echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
 
 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
-
-# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
-RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
-
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
 
 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml

stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml

@@ -58,7 +58,7 @@
 
 [upstream]
   [upstream.ethServer]
-    rpcProviderEndpoints = REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS
+    rpcProviderEndpoint = "REPLACE_WITH_CERC_ETH_RPC_ENDPOINT"
 
     # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client)
     rpcClient = true
@@ -96,6 +96,3 @@
   # Max block range of historical processing after which it waits for completion of events processing
   # If set to -1 historical processing does not wait for events processing and completes till latest canonical block
   historicalMaxFetchAhead = 10000
-
-  # Max number of retries to fetch new block after which watcher will failover to other RPC endpoints
-  maxNewBlockRetries = 3

stack_orchestrator/data/container-build/cerc-fixturenet-lotus/Dockerfile

@@ -1,5 +1,5 @@
 #####################################
-FROM golang:1.19.12-bullseye AS lotus-builder
+FROM golang:1.21.7-bullseye AS lotus-builder
 MAINTAINER Lotus Development Team
 
 RUN apt-get update && apt-get install -y ca-certificates build-essential clang ocl-icd-opencl-dev ocl-icd-libopencl1 jq libhwloc-dev
@@ -29,7 +29,6 @@ RUN set -eux; \
     rustup --version; \
     cargo --version; \
     rustc --version;
-
 COPY ./ /opt/filecoin
 WORKDIR /opt/filecoin
 
@@ -38,13 +37,13 @@ WORKDIR /opt/filecoin
 ### make configurable filecoin-ffi build
 ARG FFI_BUILD_FROM_SOURCE=0
 ENV FFI_BUILD_FROM_SOURCE=${FFI_BUILD_FROM_SOURCE}
-
+RUN ls /opt/filecoin
 RUN make clean deps
 
 ARG RUSTFLAGS=""
 ARG GOFLAGS=""
 
-#RUN make buildall
+#RUN make lotus
 RUN make 2k
 
 #####################################
@@ -75,7 +74,7 @@ COPY --from=lotus-builder /opt/filecoin/lotus-shed /usr/local/bin/
 #COPY scripts/docker-lotus-entrypoint.sh /
 #COPY myscripts/setup-node.sh /docker-entrypoint-scripts.d/setup-node.sh
 
-ARG DOCKER_LOTUS_IMPORT_SNAPSHOT https://snapshots.mainnet.filops.net/minimal/latest
+ARG DOCKER_LOTUS_IMPORT_SNAPSHOT https://forest-archive.chainsafe.dev/latest/mainnet/
 ENV DOCKER_LOTUS_IMPORT_SNAPSHOT ${DOCKER_LOTUS_IMPORT_SNAPSHOT}
 ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
 ENV LOTUS_PATH /var/lib/lotus

stack_orchestrator/data/container-build/cerc-fixturenet-lotus/build.sh

@@ -1,12 +1,12 @@
 #!/usr/bin/env bash
-# Build cerc/lotus
+# Build cerc/fixturenet-lotus
 source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
 # Use a release version tag to match the modified Dockerfile replaced in next step
-git -C ${CERC_REPO_BASE_DIR}/lotus checkout v1.23.3
+git -C ${CERC_REPO_BASE_DIR}/lotus checkout master
 
 # Replace repo's Dockerfile with modified one
 cp ${SCRIPT_DIR}/Dockerfile ${CERC_REPO_BASE_DIR}/lotus/Dockerfile
 
-docker build -t cerc/lotus:local ${build_command_args} ${CERC_REPO_BASE_DIR}/lotus
+docker build -t cerc/fixturenet-lotus:local ${build_command_args} ${CERC_REPO_BASE_DIR}/lotus

stack_orchestrator/data/container-build/cerc-mainnet-lotus/Dockerfile

@@ -0,0 +1,124 @@
+#####################################
+FROM golang:1.21.7-bullseye AS lotus-builder
+MAINTAINER Lotus Development Team
+
+RUN apt-get update && apt-get install -y ca-certificates build-essential clang ocl-icd-opencl-dev ocl-icd-libopencl1 jq libhwloc-dev
+
+ENV XDG_CACHE_HOME="/tmp"
+
+### taken from https://github.com/rust-lang/docker-rust/blob/master/1.63.0/buster/Dockerfile
+ENV RUSTUP_HOME=/usr/local/rustup \
+    CARGO_HOME=/usr/local/cargo \
+    PATH=/usr/local/cargo/bin:$PATH \
+    RUST_VERSION=1.63.0
+
+RUN set -eux; \
+    dpkgArch="$(dpkg --print-architecture)"; \
+    case "${dpkgArch##*-}" in \
+        amd64) rustArch='x86_64-unknown-linux-gnu'; rustupSha256='5cc9ffd1026e82e7fb2eec2121ad71f4b0f044e88bca39207b3f6b769aaa799c' ;; \
+        arm64) rustArch='aarch64-unknown-linux-gnu'; rustupSha256='e189948e396d47254103a49c987e7fb0e5dd8e34b200aa4481ecc4b8e41fb929' ;; \
+        *) echo >&2 "unsupported architecture: ${dpkgArch}"; exit 1 ;; \
+    esac; \
+    url="https://static.rust-lang.org/rustup/archive/1.25.1/${rustArch}/rustup-init"; \
+    wget "$url"; \
+    echo "${rustupSha256} *rustup-init" | sha256sum -c -; \
+    chmod +x rustup-init; \
+    ./rustup-init -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION --default-host ${rustArch}; \
+    rm rustup-init; \
+    chmod -R a+w $RUSTUP_HOME $CARGO_HOME; \
+    rustup --version; \
+    cargo --version; \
+    rustc --version;
+
+COPY ./ /opt/filecoin
+WORKDIR /opt/filecoin
+
+ARG FFI_BUILD_FROM_SOURCE=0
+ENV FFI_BUILD_FROM_SOURCE=${FFI_BUILD_FROM_SOURCE}
+
+#RUN make clean deps
+
+ARG RUSTFLAGS=""
+ARG GOFLAGS=""
+
+RUN make clean deps && \
+    make lotus lotus-shed lotus-stats && \
+    install -C ./lotus /usr/local/bin/lotus && \
+    install -C ./lotus-shed /usr/local/bin/lotus-shed && \
+    install -C ./lotus-stats /usr/local/bin/lotus-stats
+
+#####################################
+FROM ubuntu:20.04 AS lotus-base
+MAINTAINER Lotus Development Team
+
+# Base resources
+COPY --from=lotus-builder /etc/ssl/certs                           /etc/ssl/certs
+COPY --from=lotus-builder /lib/*/libdl.so.2         /lib/
+COPY --from=lotus-builder /lib/*/librt.so.1         /lib/
+COPY --from=lotus-builder /lib/*/libgcc_s.so.1      /lib/
+COPY --from=lotus-builder /lib/*/libutil.so.1       /lib/
+COPY --from=lotus-builder /usr/lib/*/libltdl.so.7   /lib/
+COPY --from=lotus-builder /usr/lib/*/libnuma.so.1   /lib/
+COPY --from=lotus-builder /usr/lib/*/libhwloc.so.*  /lib/
+COPY --from=lotus-builder /usr/lib/*/libOpenCL.so.1 /lib/
+
+RUN useradd -r -u 532 -U fc \
+ && mkdir -p /etc/OpenCL/vendors \
+ && echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd
+
+#####################################
+FROM lotus-base AS lotus
+MAINTAINER Lotus Development Team
+
+COPY --from=lotus-builder \
+    /usr/local/bin/lotus \
+    /usr/local/bin/lotus-shed \
+    /usr/local/bin/
+
+ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
+ENV LOTUS_PATH /var/lib/lotus
+ENV DOCKER_LOTUS_IMPORT_WALLET ""
+
+RUN mkdir /var/lib/lotus /var/tmp/filecoin-proof-parameters
+RUN chown fc: /var/lib/lotus /var/tmp/filecoin-proof-parameters
+
+VOLUME /var/lib/lotus
+VOLUME /var/tmp/filecoin-proof-parameters
+
+USER fc
+
+EXPOSE 1234
+
+
+CMD ["-help"]
+
+#####################################
+FROM lotus-base AS lotus-all-in-one
+
+# Install netcat for healthcheck
+RUN apt-get update && apt-get install -y netcat && apt-get install -y iproute2
+
+ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
+ENV LOTUS_PATH /var/lib/lotus
+
+ARG DOCKER_LOTUS_IMPORT_SNAPSHOT=https://forest-archive.chainsafe.dev/latest/mainnet/
+ENV DOCKER_LOTUS_IMPORT_SNAPSHOT=${DOCKER_LOTUS_IMPORT_SNAPSHOT}
+
+COPY --from=lotus-builder /opt/filecoin/lotus          /usr/local/bin/
+COPY --from=lotus-builder /opt/filecoin/lotus-shed     /usr/local/bin/
+COPY --from=lotus-builder /opt/filecoin/lotus-stats    /usr/local/bin/
+COPY scripts/docker-lotus-entrypoint.sh /docker-lotus-entrypoint.sh
+RUN chmod +x /docker-lotus-entrypoint.sh
+
+RUN mkdir /var/tmp/filecoin-proof-parameters
+RUN mkdir /var/lib/lotus
+RUN chown fc: /var/tmp/filecoin-proof-parameters
+RUN chown fc: /var/lib/lotus
+
+
+VOLUME /var/tmp/filecoin-proof-parameters
+VOLUME /var/lib/lotus
+
+EXPOSE 1234
+EXPOSE 1235
+

stack_orchestrator/data/container-build/cerc-mainnet-lotus/build.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+# Build cerc/mainnet-lotus
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+git stash
+
+# Use a release version tag to match the modified Dockerfile replaced in next step
+git -C ${CERC_REPO_BASE_DIR}/lotus checkout master
+
+# Replace repo's Dockerfile with modified one
+cp ${SCRIPT_DIR}/Dockerfile ${CERC_REPO_BASE_DIR}/lotus/Dockerfile
+
+docker build -t cerc/lotus-mainnet:local ${build_command_args} ${CERC_REPO_BASE_DIR}/lotus

stack_orchestrator/data/container-build/cerc-webapp-base/scripts/start-serving-app.sh

@@ -8,27 +8,21 @@ CERC_WEBAPP_FILES_DIR="${CERC_WEBAPP_FILES_DIR:-/data}"
 CERC_ENABLE_CORS="${CERC_ENABLE_CORS:-false}"
 CERC_SINGLE_PAGE_APP="${CERC_SINGLE_PAGE_APP}"
 
-if [ -z "${CERC_SINGLE_PAGE_APP}" ]; then 
-  # If there is only one HTML file, assume an SPA.
-  if [ 1 -eq $(find "${CERC_WEBAPP_FILES_DIR}" -name '*.html' | wc -l) ]; then
+if [ -z "${CERC_SINGLE_PAGE_APP}" ]; then
+  if [ 1 -eq $(find "${CERC_WEBAPP_FILES_DIR}" -name '*.html' | wc -l) ] && [ -d "${CERC_WEBAPP_FILES_DIR}/static" ]; then
     CERC_SINGLE_PAGE_APP=true
   else
     CERC_SINGLE_PAGE_APP=false
   fi
 fi
 
-# ${var,,} is a lower-case comparison
-if [ "true" == "${CERC_ENABLE_CORS,,}" ]; then
+if [ "true" == "$CERC_ENABLE_CORS" ]; then
   CERC_HTTP_EXTRA_ARGS="$CERC_HTTP_EXTRA_ARGS --cors"
 fi
 
-# ${var,,} is a lower-case comparison
-if [ "true" == "${CERC_SINGLE_PAGE_APP,,}" ]; then
-  echo "Serving content as single-page app.  If this is wrong, set 'CERC_SINGLE_PAGE_APP=false'"
+if [ "true" == "$CERC_SINGLE_PAGE_APP" ]; then
   # Create a catchall redirect back to /
   CERC_HTTP_EXTRA_ARGS="$CERC_HTTP_EXTRA_ARGS --proxy http://localhost:${CERC_LISTEN_PORT}?"
-else
-  echo "Serving content normally.  If this is a single-page app, set 'CERC_SINGLE_PAGE_APP=true'"
 fi
 
 LACONIC_HOSTED_CONFIG_FILE=${LACONIC_HOSTED_CONFIG_FILE}
@@ -45,4 +39,4 @@ if [ -f "${LACONIC_HOSTED_CONFIG_FILE}" ]; then
 fi
 
 /scripts/apply-runtime-env.sh ${CERC_WEBAPP_FILES_DIR}
-http-server $CERC_HTTP_EXTRA_ARGS -p ${CERC_LISTEN_PORT} "${CERC_WEBAPP_FILES_DIR}"
+http-server $CERC_HTTP_EXTRA_ARGS -p ${CERC_LISTEN_PORT} "${CERC_WEBAPP_FILES_DIR}"

stack_orchestrator/data/container-image-list.txt

@@ -41,7 +41,8 @@ cerc/pocket
 cerc/watcher-azimuth
 cerc/ipld-eth-state-snapshot
 cerc/watcher-gelato
-cerc/lotus
+cerc/mainnet-lotus
+cerc/fixturenet-lotus
 cerc/go-opera
 cerc/lasso
 cerc/reth

stack_orchestrator/data/pod-list.txt

@@ -27,6 +27,7 @@ fixturenet-pocket
 watcher-azimuth
 watcher-gelato
 fixturenet-lotus
+mainnet-lotus
 mainnet-go-opera
 lasso
 reth

stack_orchestrator/data/stacks/ajna/README.md

@@ -53,7 +53,7 @@ Inside deployment directory, open the `config.env` file  and set following env v
 
 ```bash
 # External Filecoin (ETH RPC) endpoint to point the watcher to
-CERC_ETH_RPC_ENDPOINTS=https://example-lotus-endpoint-1/rpc/v1,https://example-lotus-endpoint-2/rpc/v1
+CERC_ETH_RPC_ENDPOINT=https://example-lotus-endpoint/rpc/v1
 ```
 
 ### Start the deployment

stack_orchestrator/data/stacks/ajna/stack.yml

@@ -2,7 +2,7 @@ version: "1.0"
 name: ajna
 description: "Ajna watcher stack"
 repos:
-  - git.vdb.to/cerc-io/ajna-watcher-ts@v0.1.9
+  - git.vdb.to/cerc-io/ajna-watcher-ts@v0.1.1
 containers:
   - cerc/watcher-ajna
 pods:

stack_orchestrator/data/stacks/azimuth/README.md

@@ -4,7 +4,7 @@ Instructions to setup and deploy Azimuth Watcher stack
 
 ## Setup
 
-Prerequisite: External RPC endpoints
+Prerequisite: `ipld-eth-server` RPC and GQL endpoints
 
 Clone required repositories:
 
@@ -94,7 +94,7 @@ Inside the deployment directory, open the file `config.env` and add variable to
 
   ```bash
   # External RPC endpoints
-  CERC_ETH_RPC_ENDPOINTS=https://example-rpc-endpoint-1,https://example-rpc-endpoint-2
+  CERC_IPLD_ETH_RPC=
   ```
 
 * NOTE: If RPC endpoint is on the host machine, use `host.docker.internal` as the hostname to access the host port, or use the `ip a` command to find the IP address of the `docker0` interface (this will usually be something like `172.17.0.1` or `172.18.0.1`)
@@ -120,7 +120,4 @@ To stop all azimuth services and also delete data:
 
 ```bash
 laconic-so deployment --dir azimuth-deployment stop --delete-volumes
-
-# Remove deployment directory (deployment will have to be recreated for a re-run)
-rm -r azimuth-deployment
 ```

stack_orchestrator/data/stacks/azimuth/stack.yml

@@ -1,7 +1,7 @@
 version: "1.0"
 name: azimuth
 repos:
-  - github.com/cerc-io/azimuth-watcher-ts@v0.1.4
+  - github.com/cerc-io/azimuth-watcher-ts@v0.1.3
 containers:
   - cerc/watcher-azimuth
 pods:

stack_orchestrator/data/stacks/fixturenet-lotus/stack.yml

@@ -2,8 +2,8 @@ version: "1.0"
 name: fixturenet-lotus
 description: "A lotus fixturenet"
 repos:
-  - github.com/filecoin-project/lotus
+  - git.vdb.to/cerc-io/lotus
 containers:
-  - cerc/lotus
+  - cerc/fixturenet-lotus
 pods:
   - fixturenet-lotus

stack_orchestrator/data/stacks/fixturenet-sushiswap-subgraph/stack.yml

@@ -13,7 +13,7 @@ repos:
   - github.com/sushiswap/subgraphs
 containers:
   # fixturenet-lotus image
-  - cerc/lotus
+  - cerc/lotus-fixturenet
   # graph-node image
   - cerc/graph-node
   # sushiswap contract deployment images

stack_orchestrator/data/stacks/mainnet-lotus/README.md

@@ -0,0 +1,26 @@
+# Lotus stack
+
+## Clone required repositories
+```
+$ laconic-so --stack mainnet-lotus setup-repositories
+```
+## Build the stack's containers
+```
+$ laconic-so --stack mainnet-lotus build-containers
+```
+## Create a deployment of the stack
+```
+$ laconic-so --stack mainnet-lotus deploy init --map-ports-to-host any-same --output lotus-spec.yml
+```
+Update the URLs in the generated spec file to use your systems' actual IP addresses.
+```
+$ laconic-so --stack mainnet-lotus deploy create --deployment-dir lotus-deployment --spec-file lotus-spec.yml
+```
+## Start the stack
+```
+$ laconic-so deployment --dir lotus-deployment start
+```
+Check logs:
+```
+$ laconic-so deployment --dir lotus-deployment logs
+```

stack_orchestrator/data/stacks/mainnet-lotus/stack.yml

@@ -0,0 +1,9 @@
+version: "1.0"
+name: mainnet-lotus
+description: "A lotus stack"
+repos:
+  - git.vdb.to/cerc-io/lotus
+containers:
+  - cerc/mainnet-lotus
+pods:
+  - mainnet-lotus

stack_orchestrator/data/stacks/merkl-sushiswap-v3/README.md

@@ -53,7 +53,7 @@ Inside deployment directory, open the `config.env` file  and set following env v
 
 ```bash
 # External Filecoin (ETH RPC) endpoint to point the watcher to
-CERC_ETH_RPC_ENDPOINTS=https://example-lotus-endpoint-1/rpc/v1,https://example-lotus-endpoint-2/rpc/v1
+CERC_ETH_RPC_ENDPOINT=https://example-lotus-endpoint/rpc/v1
 ```
 
 ### Start the deployment

stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml

@@ -2,7 +2,7 @@ version: "1.0"
 name: merkl-sushiswap-v3
 description: "SushiSwap v3 watcher stack"
 repos:
-  - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.11
+  - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.7
 containers:
   - cerc/watcher-merkl-sushiswap-v3
 pods:

stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md

@@ -113,31 +113,21 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers
         labels:
           instance: 'ajna'
           chain: 'filecoin'
-
-  - job_name: graph-node
-    metrics_path: /metrics
-    scrape_interval: 30s
-    static_configs:
-      - targets: ['GRAPH_NODE_HOST:GRAPH_NODE_HOST_METRICS_PORT']
   ```
 
 Add scrape config as done above for any additional watcher to add it to the Watchers dashboard.
 
 ### Grafana alerts config
 
-Place the pre-configured alerts rules in Grafana provisioning directory:
+Place the pre-configured watcher alerts rules in Grafana provisioning directory:
 
   ```bash
-  # watcher alert rules
   cp monitoring-watchers-deployment/config/monitoring/watcher-alert-rules.yml monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/
-
-  # subgraph alert rules
-  cp monitoring-watchers-deployment/config/monitoring/subgraph-alert-rules.yml monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/
   ```
 
 Update the alerting contact points config (`monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/contactpoints.yml`) with desired contact points
 
-Add corresponding routes to the notification policies config (`monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/policies.yml`) with appropriate object-matchers:
+Add corresponding routes to the notification policies config (`monitoring-watchers-deployment/monitoring/grafana/provisioning/alerting/policies.yaml`) with appropriate object-matchers:
 
   ```yml
   ...
@@ -145,7 +135,7 @@ Add corresponding routes to the notification policies config (`monitoring-watche
       - receiver: SlackNotifier
       object_matchers:
         # Add matchers below
-        - ['grafana_folder', '=~', 'WatcherAlerts|SubgraphAlerts']
+        - ['grafana_folder', '=', 'WatcherAlerts']
   ```
 
 ### Env
@@ -159,9 +149,6 @@ Set the following env variables in the deployment env config file (`monitoring-w
   # Grafana server host URL to be used
   # (Optional, default: http://localhost:3000)
   GF_SERVER_ROOT_URL=
-
-  # List of subgraph ids to configure alerts for (separated by |)
-  CERC_GRAFANA_ALERTS_SUBGRAPH_IDS=
   ```
 
 ## Start the stack

stack_orchestrator/data/stacks/sushiswap-v3/README.md

@@ -53,7 +53,7 @@ Inside deployment directory, open the `config.env` file  and set following env v
 
 ```bash
 # External Filecoin (ETH RPC) endpoint to point the watcher to
-CERC_ETH_RPC_ENDPOINTS=https://example-lotus-endpoint-1/rpc/v1,https://example-lotus-endpoint-2/rpc/v1
+CERC_ETH_RPC_ENDPOINT=https://example-lotus-endpoint/rpc/v1
 ```
 
 ### Start the deployment

stack_orchestrator/data/stacks/sushiswap-v3/stack.yml

@@ -2,7 +2,7 @@ version: "1.0"
 name: sushiswap-v3
 description: "SushiSwap v3 watcher stack"
 repos:
-  - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.11
+  - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.7
 containers:
   - cerc/watcher-sushiswap-v3
 pods:

stack_orchestrator/data/stacks/sushiswap/stack.yml

@@ -11,7 +11,7 @@ repos:
   - github.com/vulcanize/uniswap-v3-info
 containers:
   ## fixturenet-lotus image
-  - cerc/lotus
+  - cerc/lotus-fixturenet
   ## sushiswap images
   - cerc/sushiswap-v3-core
   - cerc/sushiswap-v3-periphery