Upgrade ajna watcher version in stack

Upgrade sushiswap v3 watcher version
Add dashboard panel for ETH RPC count metric
2024-07-19 11:22:48 +05:30 · 2024-07-19 11:22:48 +05:30 · 2024-07-19 11:22:48 +05:30 · 2024-07-19 11:22:48 +05:30 · 2024-07-19 11:22:48 +05:30 · 2024-07-19 11:22:48 +05:30
47 changed files with 1475 additions and 378 deletions
@@ -4,3 +4,4 @@ Trigger
 Trigger
 Trigger
 Trigger
+Trigger
@@ -4,9 +4,11 @@ with open("README.md", "r", encoding="utf-8") as fh:
    long_description = fh.read()
 with open("requirements.txt", "r", encoding="utf-8") as fh:
    requirements = fh.read()
+with open("stack_orchestrator/data/version.txt", "r", encoding="utf-8") as fh:
+    version = fh.readlines()[-1].strip(" \n")
 setup(
    name='laconic-stack-orchestrator',
-    version='1.0.12',
+    version=version,
    author='Cerc',
    author_email='info@cerc.io',
    license='GNU Affero General Public License',
@@ -21,11 +21,6 @@ from stack_orchestrator.util import get_parsed_stack_config, warn_exit

 def get_containers_in_scope(stack: str):

-    # See: https://stackoverflow.com/a/20885799/1701505
-    from stack_orchestrator import data
-    with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file:
-        all_containers = container_list_file.read().splitlines()
-
    containers_in_scope = []
    if stack:
        stack_config = get_parsed_stack_config(stack)
@@ -33,11 +28,14 @@ def get_containers_in_scope(stack: str):
            warn_exit(f"stack {stack} does not define any containers")
        containers_in_scope = stack_config['containers']
    else:
-        containers_in_scope = all_containers
+        # See: https://stackoverflow.com/a/20885799/1701505
+        from stack_orchestrator import data
+        with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file:
+            containers_in_scope = container_list_file.read().splitlines()

    if opts.o.verbose:
        print(f'Containers: {containers_in_scope}')
        if stack:
            print(f"Stack: {stack}")

-    return containers_in_scope
+    return containers_in_scope
@@ -0,0 +1,10 @@
+services:
+  laconic-explorer:
+    restart: unless-stopped
+    image: cerc/ping-pub:local
+    environment:
+      - LACONIC_LACONICD_API_URL=${LACONIC_LACONICD_API_URL:-http://localhost:1317}
+      - LACONIC_LACONICD_RPC_URL=${LACONIC_LACONICD_RPC_URL:-http://localhost:26657}
+      - LACONIC_LACONICD_CHAIN_ID=${LACONIC_LACONICD_CHAIN_ID:-chain-id-not-set}
+    ports:
+      - "5173"
@@ -60,6 +60,7 @@ services:
    volumes:
      - ../config/watcher-ajna/watcher-config-template.toml:/app/environments/watcher-config-template.toml
      - ../config/watcher-ajna/start-server.sh:/app/start-server.sh
+      - ajna_watcher_gql_logs_data:/app/gql-logs
    ports:
      - "3008"
      - "9001"
@@ -74,3 +75,4 @@ services:

 volumes:
  ajna_watcher_db_data:
+  ajna_watcher_gql_logs_data:
@@ -74,6 +74,7 @@ services:
      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/azimuth-watcher/environments/watcher-config-template.toml
      - ../config/watcher-azimuth/merge-toml.js:/app/packages/azimuth-watcher/merge-toml.js
      - ../config/watcher-azimuth/start-server.sh:/app/packages/azimuth-watcher/start-server.sh
+      - azimuth_watcher_gql_logs_data:/app/packages/azimuth-watcher/gql-logs
    ports:
      - "3001"
      - "9001"
@@ -136,6 +137,7 @@ services:
      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/censures-watcher/environments/watcher-config-template.toml
      - ../config/watcher-azimuth/merge-toml.js:/app/packages/censures-watcher/merge-toml.js
      - ../config/watcher-azimuth/start-server.sh:/app/packages/censures-watcher/start-server.sh
+      - censures_watcher_gql_logs_data:/app/packages/censures-watcher/gql-logs
    ports:
      - "3002"
      - "9003"
@@ -198,6 +200,7 @@ services:
      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/claims-watcher/environments/watcher-config-template.toml
      - ../config/watcher-azimuth/merge-toml.js:/app/packages/claims-watcher/merge-toml.js
      - ../config/watcher-azimuth/start-server.sh:/app/packages/claims-watcher/start-server.sh
+      - claims_watcher_gql_logs_data:/app/packages/claims-watcher/gql-logs
    ports:
      - "3003"
      - "9005"
@@ -260,6 +263,7 @@ services:
      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/conditional-star-release-watcher/environments/watcher-config-template.toml
      - ../config/watcher-azimuth/merge-toml.js:/app/packages/conditional-star-release-watcher/merge-toml.js
      - ../config/watcher-azimuth/start-server.sh:/app/packages/conditional-star-release-watcher/start-server.sh
+      - conditional_star_release_watcher_gql_logs_data:/app/packages/conditional-star-release-watcher/gql-logs
    ports:
      - "3004"
      - "9007"
@@ -322,6 +326,7 @@ services:
      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/delegated-sending-watcher/environments/watcher-config-template.toml
      - ../config/watcher-azimuth/merge-toml.js:/app/packages/delegated-sending-watcher/merge-toml.js
      - ../config/watcher-azimuth/start-server.sh:/app/packages/delegated-sending-watcher/start-server.sh
+      - delegated_sending_watcher_gql_logs_data:/app/packages/delegated-sending-watcher/gql-logs
    ports:
      - "3005"
      - "9009"
@@ -384,6 +389,7 @@ services:
      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/ecliptic-watcher/environments/watcher-config-template.toml
      - ../config/watcher-azimuth/merge-toml.js:/app/packages/ecliptic-watcher/merge-toml.js
      - ../config/watcher-azimuth/start-server.sh:/app/packages/ecliptic-watcher/start-server.sh
+      - ecliptic_watcher_gql_logs_data:/app/packages/ecliptic-watcher/gql-logs
    ports:
      - "3006"
      - "9011"
@@ -446,6 +452,7 @@ services:
      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/linear-star-release-watcher/environments/watcher-config-template.toml
      - ../config/watcher-azimuth/merge-toml.js:/app/packages/linear-star-release-watcher/merge-toml.js
      - ../config/watcher-azimuth/start-server.sh:/app/packages/linear-star-release-watcher/start-server.sh
+      - linear_star_release_watcher_gql_logs_data:/app/packages/linear-star-release-watcher/gql-logs
    ports:
      - "3007"
      - "9013"
@@ -508,6 +515,7 @@ services:
      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/polls-watcher/environments/watcher-config-template.toml
      - ../config/watcher-azimuth/merge-toml.js:/app/packages/polls-watcher/merge-toml.js
      - ../config/watcher-azimuth/start-server.sh:/app/packages/polls-watcher/start-server.sh
+      - polls_watcher_gql_logs_data:/app/packages/polls-watcher/gql-logs
    ports:
      - "3008"
      - "9015"
@@ -560,3 +568,11 @@ services:

 volumes:
  watcher_db_data:
+  azimuth_watcher_gql_logs_data:
+  censures_watcher_gql_logs_data:
+  claims_watcher_gql_logs_data:
+  conditional_star_release_watcher_gql_logs_data:
+  delegated_sending_watcher_gql_logs_data:
+  ecliptic_watcher_gql_logs_data:
+  linear_star_release_watcher_gql_logs_data:
+  polls_watcher_gql_logs_data:
@@ -60,6 +60,7 @@ services:
    volumes:
      - ../config/watcher-merkl-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
      - ../config/watcher-merkl-sushiswap-v3/start-server.sh:/app/start-server.sh
+      - merkl_sushiswap_v3_watcher_gql_logs_data:/app/gql-logs
    ports:
      - "127.0.0.1:3007:3008"
      - "9003:9001"
@@ -74,3 +75,4 @@ services:

 volumes:
  merkl_sushiswap_v3_watcher_db_data:
+  merkl_sushiswap_v3_watcher_gql_logs_data:
@@ -60,6 +60,7 @@ services:
    volumes:
      - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
      - ../config/watcher-sushiswap-v3/start-server.sh:/app/start-server.sh
+      - sushiswap_v3_watcher_gql_logs_data:/app/gql-logs
    ports:
      - "127.0.0.1:3008:3008"
      - "9001:9001"
@@ -74,3 +75,4 @@ services:

 volumes:
  sushiswap_v3_watcher_db_data:
+  sushiswap_v3_watcher_gql_logs_data:
@@ -18,7 +18,7 @@
  "editable": true,
  "fiscalYearStartMonth": 0,
  "graphTooltip": 0,
-  "id": 6,
+  "id": 39,
  "links": [],
  "liveNow": false,
  "panels": [
@@ -1202,6 +1202,640 @@
      ],
      "title": "ETH RPC failed requests rate",
      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "gridPos": {
+        "h": 2,
+        "w": 24,
+        "x": 0,
+        "y": 22
+      },
+      "id": 32,
+      "options": {
+        "code": {
+          "language": "plaintext",
+          "showLineNumbers": false,
+          "showMiniMap": false
+        },
+        "content": "<center style=\"margin-top: 1rem\"> \n  <h1>GraphQL Server</h1> \n</center>",
+        "mode": "html"
+      },
+      "pluginVersion": "10.2.3",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "refId": "A"
+        }
+      ],
+      "transparent": true,
+      "type": "text"
+    },
+    {
+      "cards": {},
+      "color": {
+        "cardColor": "#b4ff00",
+        "colorScale": "sqrt",
+        "colorScheme": "interpolatePlasma",
+        "exponent": 0.5,
+        "mode": "spectrum"
+      },
+      "dataFormat": "tsbuckets",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "scaleDistribution": {
+              "type": "linear"
+            }
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 6,
+        "x": 0,
+        "y": 24
+      },
+      "heatmap": {},
+      "hideZeroBuckets": true,
+      "highlightCards": true,
+      "id": 29,
+      "legend": {
+        "show": false
+      },
+      "links": [],
+      "options": {
+        "calculate": false,
+        "calculation": {},
+        "cellGap": 2,
+        "cellValues": {},
+        "color": {
+          "exponent": 0.5,
+          "fill": "#b4ff00",
+          "mode": "scheme",
+          "reverse": false,
+          "scale": "exponential",
+          "scheme": "Plasma",
+          "steps": 128
+        },
+        "exemplars": {
+          "color": "rgba(255,0,255,0.7)"
+        },
+        "filterValues": {
+          "le": 1e-9
+        },
+        "legend": {
+          "show": false
+        },
+        "rowsFrame": {
+          "layout": "auto"
+        },
+        "showValue": "never",
+        "tooltip": {
+          "show": true,
+          "showColorScale": false,
+          "yHistogram": false
+        },
+        "yAxis": {
+          "axisPlacement": "left",
+          "reverse": false,
+          "unit": "short"
+        }
+      },
+      "pluginVersion": "10.2.3",
+      "repeatDirection": "v",
+      "reverseYBuckets": false,
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "sum(increase(query_execution_time_bucket{deployment=\"[[subgraph_hash]]\"}[1m])) by (le)",
+          "format": "heatmap",
+          "instant": false,
+          "intervalFactor": 1,
+          "legendFormat": "{{le}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Query Execution Time Histogram ([[subgraph_name]])",
+      "tooltip": {
+        "show": true,
+        "showHistogram": false
+      },
+      "type": "heatmap",
+      "xAxis": {
+        "show": true
+      },
+      "yAxis": {
+        "format": "short",
+        "logBase": 1,
+        "show": true
+      },
+      "yBucketBound": "auto"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {
+            "fillOpacity": 80,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineWidth": 1
+          },
+          "links": [],
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 6,
+        "x": 6,
+        "y": 24
+      },
+      "id": 26,
+      "links": [],
+      "options": {
+        "bucketOffset": 0,
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        }
+      },
+      "pluginVersion": "10.2.3",
+      "repeat": "subgraph_hash",
+      "repeatDirection": "v",
+      "targets": [
+        {
+          "datasource": {
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "sum(increase(query_execution_time_bucket{deployment=\"[[subgraph_hash]]\"}[1m])) by (le)",
+          "format": "heatmap",
+          "instant": false,
+          "intervalFactor": 1,
+          "legendFormat": "{{le}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Query Execution Time Histogram ([[subgraph_name]])",
+      "type": "histogram"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 80,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "normal"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "links": [],
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "locale"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 24
+      },
+      "id": 36,
+      "links": [],
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": false
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "pluginVersion": "10.2.3",
+      "repeatDirection": "v",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "exemplar": false,
+          "expr": "sum(increase(query_execution_time_count{deployment=\"$subgraph_hash\"}[1m]))",
+          "format": "time_series",
+          "fullMetaSearch": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "intervalFactor": 1,
+          "legendFormat": "{{subgraph_deployment}}",
+          "range": true,
+          "refId": "A",
+          "useBackend": false
+        }
+      ],
+      "title": "Queries  ([[subgraph_name]])",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "Queries per minute per minute",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 2,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "links": [],
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "locale"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 32
+      },
+      "id": 27,
+      "links": [],
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "pluginVersion": "10.2.3",
+      "repeatDirection": "v",
+      "targets": [
+        {
+          "datasource": {
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(query_execution_time_count[1m])) by (deployment)",
+          "format": "time_series",
+          "instant": false,
+          "intervalFactor": 1,
+          "legendFormat": "{{deployment}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Query Rate (All Subgraphs)",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 80,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "normal"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "links": [],
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "locale"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 32
+      },
+      "id": 67,
+      "links": [],
+      "options": {
+        "legend": {
+          "calcs": [
+            "sum"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "pluginVersion": "10.2.3",
+      "repeatDirection": "v",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "sum(increase(query_execution_time_count[1m])) by (deployment)",
+          "format": "time_series",
+          "instant": false,
+          "intervalFactor": 1,
+          "legendFormat": "{{deployment}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Queries  (All Subgraphs)",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 70,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "normal"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "links": [],
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "locale"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 40
+      },
+      "id": 41,
+      "links": [],
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "pluginVersion": "10.2.3",
+      "repeatDirection": "v",
+      "targets": [
+        {
+          "datasource": {
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "increase(query_execution_time_count{status=\"failed\"}[1m])",
+          "format": "time_series",
+          "instant": false,
+          "intervalFactor": 1,
+          "legendFormat": "{{deployment}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Failed Queries  (All Subgraphs)",
+      "type": "timeseries"
    }
  ],
  "refresh": "10s",
@@ -18,7 +18,7 @@
  "editable": true,
  "fiscalYearStartMonth": 0,
  "graphTooltip": 0,
-  "id": 4,
+  "id": 2,
  "links": [
    {
      "asDropdown": false,
@@ -282,7 +282,7 @@
          },
          "gridPos": {
            "h": 3,
-            "w": 4,
+            "w": 3,
            "x": 0,
            "y": 4
          },
@@ -303,6 +303,75 @@
            "wideLayout": true
          },
          "pluginVersion": "10.2.3",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "PBFA97CFB590B2093"
+              },
+              "disableTextWrap": false,
+              "editorMode": "code",
+              "expr": "sync_status_block_number{job=~\"$job\", instance=~\"$watcher\", kind=\"latest_processed\"}",
+              "fullMetaSearch": false,
+              "hide": false,
+              "includeNullMetadata": true,
+              "instant": false,
+              "legendFormat": "__auto",
+              "range": true,
+              "refId": "B",
+              "useBackend": false
+            }
+          ],
+          "title": "Event processed block",
+          "type": "stat"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "fixedColor": "blue",
+                "mode": "shades"
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 3,
+            "w": 3,
+            "x": 3,
+            "y": 4
+          },
+          "id": 38,
+          "options": {
+            "colorMode": "background",
+            "graphMode": "none",
+            "justifyMode": "auto",
+            "orientation": "auto",
+            "reduceOptions": {
+              "calcs": [
+                "lastNotNull"
+              ],
+              "fields": "",
+              "values": false
+            },
+            "textMode": "auto",
+            "wideLayout": true
+          },
+          "pluginVersion": "10.2.3",
          "targets": [
            {
              "datasource": {
@@ -322,7 +391,7 @@
              "useBackend": false
            }
          ],
-          "title": "Latest indexed block",
+          "title": "Inserted processed block",
          "type": "stat"
        },
        {
@@ -352,8 +421,8 @@
          },
          "gridPos": {
            "h": 3,
-            "w": 4,
-            "x": 4,
+            "w": 3,
+            "x": 6,
            "y": 4
          },
          "id": 11,
@@ -421,8 +490,8 @@
          },
          "gridPos": {
            "h": 3,
-            "w": 4,
-            "x": 8,
+            "w": 3,
+            "x": 9,
            "y": 4
          },
          "id": 12,
@@ -639,7 +708,7 @@
                "uid": "PBFA97CFB590B2093"
              },
              "editorMode": "code",
-              "expr": "sync_status_block_number{job=~\"$job\", instance=~\"$watcher\", kind=\"latest_indexed\"}",
+              "expr": "sync_status_block_number{job=~\"$job\", instance=~\"$watcher\", kind=\"latest_processed\"}",
              "hide": false,
              "instant": false,
              "legendFormat": "{{__name__}}",
@@ -750,7 +819,7 @@
                "uid": "PBFA97CFB590B2093"
              },
              "editorMode": "code",
-              "expr": "sync_status_block_number{job=~\"$job\", instance=~\"$watcher\", kind=\"latest_indexed\"}",
+              "expr": "sync_status_block_number{job=~\"$job\", instance=~\"$watcher\", kind=\"latest_processed\"}",
              "hide": false,
              "instant": false,
              "legendFormat": "{{__name__}}",
@@ -936,12 +1005,12 @@
                "uid": "PBFA97CFB590B2093"
              },
              "disableTextWrap": false,
-              "editorMode": "builder",
-              "expr": "last_processed_block_number{job=~\"$job\", instance=~\"$watcher\"}",
+              "editorMode": "code",
+              "expr": "sync_status_block_number{job=~\"$job\", instance=~\"$watcher\", kind=\"latest_processed\"}",
              "fullMetaSearch": false,
              "includeNullMetadata": true,
              "instant": false,
-              "legendFormat": "latest_processed",
+              "legendFormat": "{{kind}}",
              "range": true,
              "refId": "A",
              "useBackend": false
@@ -952,7 +1021,7 @@
                "uid": "PBFA97CFB590B2093"
              },
              "disableTextWrap": false,
-              "editorMode": "builder",
+              "editorMode": "code",
              "expr": "sync_status_block_number{job=~\"$job\", instance=~\"$watcher\", kind=\"latest_canonical\"}",
              "fullMetaSearch": false,
              "hide": false,
@@ -1395,7 +1464,7 @@
                "uid": "PBFA97CFB590B2093"
              },
              "editorMode": "code",
-              "expr": "sync_status_block_number{job=~\"$job\", instance=~\"$watcher\", kind=\"latest_indexed\"}",
+              "expr": "sync_status_block_number{job=~\"$job\", instance=~\"$watcher\", kind=\"latest_processed\"}",
              "hide": false,
              "instant": false,
              "legendFormat": "{{__name__}}",
@@ -1676,7 +1745,7 @@
            "type": "prometheus",
            "uid": "PBFA97CFB590B2093"
          },
-          "description": "Total number of failed ETH RPC requests by method and provider endpoint ",
+          "description": "Total number of ETH RPC requests by method and provider endpoint ",
          "fieldConfig": {
            "defaults": {
              "color": {
@@ -1729,7 +1798,7 @@
          },
          "gridPos": {
            "h": 6,
-            "w": 12,
+            "w": 8,
            "x": 0,
            "y": 21
          },
@@ -1756,7 +1825,7 @@
              "disableTextWrap": false,
              "editorMode": "code",
              "exemplar": false,
-              "expr": "watcher_eth_rpc_errors{job=~\"$job\", instance=~\"$watcher\"}",
+              "expr": "sum by(method) (watcher_eth_rpc_total{job=~\"$job\", instance=~\"$watcher\"})",
              "format": "time_series",
              "fullMetaSearch": false,
              "includeNullMetadata": true,
@@ -1767,7 +1836,7 @@
              "useBackend": false
            }
          ],
-          "title": "ETH RPC request failures",
+          "title": "ETH RPC Total",
          "type": "timeseries"
        },
        {
@@ -1832,8 +1901,8 @@
          },
          "gridPos": {
            "h": 6,
-            "w": 12,
-            "x": 12,
+            "w": 8,
+            "x": 8,
            "y": 21
          },
          "id": 26,
@@ -1866,105 +1935,6 @@
          "title": "ETH RPC request durations",
          "type": "timeseries"
        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "description": "Total number of failed ETH RPC requests by method and provider endpoint  (across all watchers)",
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisBorderShow": false,
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 0,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "insertNulls": false,
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "auto",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green",
-                    "value": null
-                  }
-                ]
-              }
-            },
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 6,
-            "w": 12,
-            "x": 0,
-            "y": 27
-          },
-          "id": 30,
-          "options": {
-            "legend": {
-              "calcs": [],
-              "displayMode": "list",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "single",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "10.2.3",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "PBFA97CFB590B2093"
-              },
-              "disableTextWrap": false,
-              "editorMode": "code",
-              "exemplar": false,
-              "expr": "sum by (method, provider) (watcher_eth_rpc_errors{chain=\"$target_chain\"})",
-              "format": "time_series",
-              "fullMetaSearch": false,
-              "includeNullMetadata": true,
-              "instant": false,
-              "legendFormat": "{{method}}, {{provider}}",
-              "range": true,
-              "refId": "A",
-              "useBackend": false
-            }
-          ],
-          "title": "Total ETH RPC request failures (across all watchers)",
-          "type": "timeseries"
-        },
        {
          "datasource": {
            "type": "prometheus",
@@ -2031,9 +2001,9 @@
          },
          "gridPos": {
            "h": 6,
-            "w": 12,
-            "x": 12,
-            "y": 27
+            "w": 8,
+            "x": 16,
+            "y": 21
          },
          "id": 32,
          "options": {
@@ -2101,6 +2071,204 @@
          ],
          "type": "table"
        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "description": "Total number of failed ETH RPC requests by method and provider endpoint ",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 6,
+            "w": 12,
+            "x": 0,
+            "y": 27
+          },
+          "id": 39,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "10.2.3",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "PBFA97CFB590B2093"
+              },
+              "disableTextWrap": false,
+              "editorMode": "code",
+              "exemplar": false,
+              "expr": "watcher_eth_rpc_errors{job=~\"$job\", instance=~\"$watcher\"}",
+              "format": "time_series",
+              "fullMetaSearch": false,
+              "includeNullMetadata": true,
+              "instant": false,
+              "legendFormat": "{{method}}, {{provider}}",
+              "range": true,
+              "refId": "A",
+              "useBackend": false
+            }
+          ],
+          "title": "ETH RPC request failures",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "description": "Total number of failed ETH RPC requests by method and provider endpoint  (across all watchers)",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 6,
+            "w": 12,
+            "x": 12,
+            "y": 27
+          },
+          "id": 30,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "10.2.3",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "PBFA97CFB590B2093"
+              },
+              "disableTextWrap": false,
+              "editorMode": "code",
+              "exemplar": false,
+              "expr": "sum by (method, provider) (watcher_eth_rpc_errors{chain=\"$target_chain\"})",
+              "format": "time_series",
+              "fullMetaSearch": false,
+              "includeNullMetadata": true,
+              "instant": false,
+              "legendFormat": "{{method}}, {{provider}}",
+              "range": true,
+              "refId": "A",
+              "useBackend": false
+            }
+          ],
+          "title": "Total ETH RPC request failures (across all watchers)",
+          "type": "timeseries"
+        },
        {
          "datasource": {
            "type": "prometheus",
@@ -3215,6 +3383,6 @@
  "timepicker": {},
  "timezone": "",
  "title": "Watchers",
-  "version": 4,
+  "version": 5,
  "weekStart": ""
 }
@@ -695,82 +695,6 @@ groups:
        annotations:
          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
        isPaused: false
-      - uid: merkl_sushiswap_diff_external
-        title: merkl_sushiswap_watcher_head_tracking
-        condition: condition
-        data:
-          - refId: diff
-            relativeTimeRange:
-              from: 600
-              to: 0
-            datasourceUid: PBFA97CFB590B2093
-            model:
-              datasource:
-                type: prometheus
-                uid: PBFA97CFB590B2093
-              disableTextWrap: false
-              editorMode: code
-              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="sushi", instance="merkl_sushiswap", kind="latest_indexed"}
-              fullMetaSearch: false
-              includeNullMetadata: true
-              instant: true
-              intervalMs: 1000
-              legendFormat: __auto
-              maxDataPoints: 43200
-              range: false
-              refId: diff
-              useBackend: false
-          - refId: latest_external
-            relativeTimeRange:
-              from: 600
-              to: 0
-            datasourceUid: PBFA97CFB590B2093
-            model:
-              datasource:
-                type: prometheus
-                uid: PBFA97CFB590B2093
-              editorMode: code
-              expr: latest_block_number{chain="filecoin"}
-              hide: false
-              instant: true
-              legendFormat: __auto
-              range: false
-              refId: latest_external
-          - refId: condition
-            relativeTimeRange:
-              from: 600
-              to: 0
-            datasourceUid: __expr__
-            model:
-              conditions:
-                - evaluator:
-                    params:
-                      - 0
-                      - 0
-                    type: gt
-                  operator:
-                    type: and
-                  query:
-                    params: []
-                  reducer:
-                    params: []
-                    type: avg
-                  type: query
-              datasource:
-                name: Expression
-                type: __expr__
-                uid: __expr__
-              expression: ${diff} >= 16
-              intervalMs: 1000
-              maxDataPoints: 43200
-              refId: condition
-              type: math
-        noDataState: Alerting
-        execErrState: Alerting
-        for: 15m
-        annotations:
-          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
-        isPaused: false

      # Ajna
      - uid: ajna_diff_external
@@ -849,3 +773,81 @@ groups:
        annotations:
          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
        isPaused: false
+      
+      # Secured Finance
+      - uid: secured_finance_diff_external
+        title: secured_finance_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="secured-finance", instance="secured-finance", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="filecoin"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
@@ -2,7 +2,6 @@
  host = "0.0.0.0"
  port = 3008
  kind = "active"
-  gqlPath = "/"

  # Checkpointing state.
  checkpointing = true
@@ -11,7 +10,6 @@
  checkpointInterval = 2000

  # Enable state creation
-  # CAUTION: Disable only if state creation is not desired or can be filled subsequently
  enableState = false

  subgraphPath = "./subgraph-build"
@@ -22,23 +20,30 @@
  # Interval in number of blocks at which to clear entities cache.
  clearEntitiesCacheInterval = 1000

-  # Max block range for which to return events in eventsInRange GQL query.
-  # Use -1 for skipping check on block range.
-  maxEventsBlockRange = 1000
-
  # Flag to specify whether RPC endpoint supports block hash as block tag parameter
  rpcSupportsBlockHashParam = false

-  # GQL cache settings
-  [server.gqlCache]
-    enabled = true
+  # Server GQL config
+  [server.gql]
+    path = "/"

-    # Max in-memory cache size (in bytes) (default 8 MB)
-    # maxCacheSize
+    # Max block range for which to return events in eventsInRange GQL query.
+    # Use -1 for skipping check on block range.
+    maxEventsBlockRange = 1000

-    # GQL cache-control max-age settings (in seconds)
-    maxAge = 15
-    timeTravelMaxAge = 86400 # 1 day
+    # Log directory for GQL requests
+    logDir = "./gql-logs"
+
+    # GQL cache settings
+    [server.gql.cache]
+      enabled = true
+
+      # Max in-memory cache size (in bytes) (default 8 MB)
+      # maxCacheSize
+
+      # GQL cache-control max-age settings (in seconds)
+      maxAge = 15
+      timeTravelMaxAge = 86400 # 1 day

 [metrics]
  host = "0.0.0.0"
@@ -71,6 +76,10 @@
    # Boolean flag to filter event logs by topics
    filterLogsByTopics = true

+    # Switch clients if eth_getLogs call takes more than threshold (in secs)
+    # Set to 0 for disabling switching
+    getLogsClientSwitchThresholdInSecs = 30
+
  [upstream.cache]
    name = "requests"
    enabled = false
@@ -85,6 +94,9 @@
  # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime
  blockDelayInMilliSecs = 30000

+  # Number of blocks by which block processing lags behind head
+  blockProcessingOffset = 3
+
  # Boolean to switch between modes of processing events when starting the server.
  # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
  # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head).
@@ -1,6 +1,7 @@
 [server]
  host = "0.0.0.0"
-  maxSimultaneousRequests = -1
+  [server.gql]
+    maxSimultaneousRequests = -1

 [metrics]
  host = "0.0.0.0"
@@ -2,7 +2,6 @@
  host = "0.0.0.0"
  port = 3008
  kind = "active"
-  gqlPath = '/'

  # Checkpointing state.
  checkpointing = true
@@ -22,23 +21,30 @@
  # Interval in number of blocks at which to clear entities cache.
  clearEntitiesCacheInterval = 1000

-  # Max block range for which to return events in eventsInRange GQL query.
-  # Use -1 for skipping check on block range.
-  maxEventsBlockRange = 1000
-
  # Flag to specify whether RPC endpoint supports block hash as block tag parameter
  rpcSupportsBlockHashParam = false

-  # GQL cache settings
-  [server.gqlCache]
-    enabled = true
+  # Server GQL config
+  [server.gql]
+    path = "/"

-    # Max in-memory cache size (in bytes) (default 8 MB)
-    # maxCacheSize
+    # Max block range for which to return events in eventsInRange GQL query.
+    # Use -1 for skipping check on block range.
+    maxEventsBlockRange = 1000

-    # GQL cache-control max-age settings (in seconds)
-    maxAge = 15
-    timeTravelMaxAge = 86400 # 1 day
+    # Log directory for GQL requests
+    logDir = "./gql-logs"
+
+    # GQL cache settings
+    [server.gql.cache]
+      enabled = true
+
+      # Max in-memory cache size (in bytes) (default 8 MB)
+      # maxCacheSize
+
+      # GQL cache-control max-age settings (in seconds)
+      maxAge = 15
+      timeTravelMaxAge = 86400 # 1 day

 [metrics]
  host = "0.0.0.0"
@@ -85,6 +91,9 @@
  # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime
  blockDelayInMilliSecs = 30000

+  # Number of blocks by which block processing lags behind head
+  blockProcessingOffset = 0
+
  # Boolean to switch between modes of processing events when starting the server.
  # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
  # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head).
@@ -2,7 +2,6 @@
  host = "0.0.0.0"
  port = 3008
  kind = "active"
-  gqlPath = "/"

  # Checkpointing state.
  checkpointing = true
@@ -11,7 +10,6 @@
  checkpointInterval = 2000

  # Enable state creation
-  # CAUTION: Disable only if state creation is not desired or can be filled subsequently
  enableState = false

  subgraphPath = "./subgraph-build"
@@ -22,23 +20,30 @@
  # Interval in number of blocks at which to clear entities cache.
  clearEntitiesCacheInterval = 1000

-  # Max block range for which to return events in eventsInRange GQL query.
-  # Use -1 for skipping check on block range.
-  maxEventsBlockRange = 1000
-
  # Flag to specify whether RPC endpoint supports block hash as block tag parameter
  rpcSupportsBlockHashParam = false

-  # GQL cache settings
-  [server.gqlCache]
-    enabled = true
+  # Server GQL config
+  [server.gql]
+    path = "/"

-    # Max in-memory cache size (in bytes) (default 8 MB)
-    # maxCacheSize
+    # Max block range for which to return events in eventsInRange GQL query.
+    # Use -1 for skipping check on block range.
+    maxEventsBlockRange = 1000

-    # GQL cache-control max-age settings (in seconds)
-    maxAge = 15
-    timeTravelMaxAge = 86400 # 1 day
+    # Log directory for GQL requests
+    logDir = "./gql-logs"
+
+    # GQL cache settings
+    [server.gql.cache]
+      enabled = true
+
+      # Max in-memory cache size (in bytes) (default 8 MB)
+      # maxCacheSize
+
+      # GQL cache-control max-age settings (in seconds)
+      maxAge = 15
+      timeTravelMaxAge = 86400 # 1 day

 [metrics]
  host = "0.0.0.0"
@@ -71,6 +76,10 @@
    # Boolean flag to filter event logs by topics
    filterLogsByTopics = true

+    # Switch clients if eth_getLogs call takes more than threshold (in secs)
+    # Set to 0 for disabling switching
+    getLogsClientSwitchThresholdInSecs = 30
+
  [upstream.cache]
    name = "requests"
    enabled = false
@@ -85,6 +94,9 @@
  # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime
  blockDelayInMilliSecs = 30000

+  # Number of blocks by which block processing lags behind head
+  blockProcessingOffset = 3
+
  # Boolean to switch between modes of processing events when starting the server.
  # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
  # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head).
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Build cerc/laconic-registry-cli
+# Build cerc/laconic-console-host

 source ${CERC_CONTAINER_BASE_DIR}/build-base.sh

@@ -28,6 +28,8 @@ RUN \
    && su ${USERNAME} -c "umask 0002 && npm install -g semver" \
    # Install pnpm
    && su ${USERNAME} -c "umask 0002 && npm install -g pnpm" \
+    # Install bun
+    && su ${USERNAME} -c "umask 0002 && npm install -g bun@1.1.x" \
    && npm cache clean --force > /dev/null 2>&1

 # [Optional] Uncomment this section to install additional OS packages.
@@ -14,6 +14,8 @@ if [ -z "$CERC_BUILD_TOOL" ]; then
    CERC_BUILD_TOOL=pnpm
  elif [ -f "yarn.lock" ]; then
    CERC_BUILD_TOOL=yarn
+  elif [ -f "bun.lockb" ]; then
+    CERC_BUILD_TOOL=bun
  else
    CERC_BUILD_TOOL=npm
  fi
@@ -13,6 +13,8 @@ if [ -z "$CERC_BUILD_TOOL" ]; then
    CERC_BUILD_TOOL=pnpm
  elif [ -f "yarn.lock" ]; then
    CERC_BUILD_TOOL=yarn
+  elif [ -f "bun.lockb" ]; then
+    CERC_BUILD_TOOL=bun
  else
    CERC_BUILD_TOOL=npm
  fi
@@ -5,7 +5,7 @@ fi


 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-CERC_MAX_GENERATE_TIME=${CERC_MAX_GENERATE_TIME:-60}
+CERC_MAX_GENERATE_TIME=${CERC_MAX_GENERATE_TIME:-120}
 tpid=""

 ctrl_c() {
@@ -20,6 +20,8 @@ if [ -z "$CERC_BUILD_TOOL" ]; then
    CERC_BUILD_TOOL=pnpm
  elif [ -f "yarn.lock" ]; then
    CERC_BUILD_TOOL=yarn
+  elif [ -f "bun.lockb" ]; then
+    CERC_BUILD_TOOL=bun
  else
    CERC_BUILD_TOOL=npm
  fi
@@ -0,0 +1,10 @@
+FROM cerc/ping-pub-base:local
+
+COPY ./scripts/update-explorer-config.sh /scripts
+COPY ./scripts/start-serving-explorer.sh /scripts
+COPY ./config/laconic-chaindata-template.json /config/chains/laconic-chaindata-template.json
+
+EXPOSE 5173
+
+WORKDIR /app
+CMD ["/scripts/start-serving-explorer.sh"]
@@ -0,0 +1,8 @@
+FROM cerc/webapp-base:local
+
+WORKDIR /app
+
+COPY . .
+
+RUN yarn
+
@@ -1,5 +1,8 @@
 #!/usr/bin/env bash
 # Build the ping pub image
 source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )

-docker build -t cerc/ping-pub:local ${build_command_args} -f $CERC_REPO_BASE_DIR/explorer/Dockerfile $CERC_REPO_BASE_DIR/explorer
+# Two-stage build is to allow us to pick up both the upstream repo's files, and local files here for config
+docker build -t cerc/ping-pub-base:local ${build_command_args} -f $SCRIPT_DIR/Dockerfile.base $CERC_REPO_BASE_DIR/explorer
+docker build -t cerc/ping-pub:local ${build_command_args} -f $SCRIPT_DIR/Dockerfile $SCRIPT_DIR
@@ -0,0 +1,22 @@
+{
+    "chain_name": "LACONIC_LACONICD_CHAIN_ID",
+    "registry_name": "LACONIC_LACONICD_CHAIN_ID",
+    "api": [
+        {"provider": "LX-one-tree-one-seven", "address": "LACONIC_LACONICD_API_URL"}
+    ],
+    "rpc": [
+        {"provider": "LX-tendermint-rpc", "address": "LACONIC_LACONICD_RPC_URL"}
+    ],
+    "sdk_version": "0.45.1",
+    "coin_type": "118",
+    "min_tx_fee": "800",
+    "addr_prefix": "ethm",
+    "logo": "/logos/cosmos.svg",
+    "assets": [{
+        "base": "aphoton",
+        "symbol": "LNT",
+        "exponent": "6",
+        "coingecko_id": "cosmos", 
+        "logo": "/logos/cosmos.svg"
+    }]
+}
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+${SCRIPT_DIR}/update-explorer-config.sh
+
+echo "Starting serving explorer"
+# Force cache re-build because vite is dumb and can't be restarted otherwise
+yarn serve --host --force
@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+
+# Verify that we have the config variables we need
+if [[ -z ${LACONIC_LACONICD_API_URL} ]]; then
+    echo "Error: LACONIC_LACONICD_API_URL not defined"
+    exit 1
+fi
+if [[ -z ${LACONIC_LACONICD_RPC_URL} ]]; then
+    echo "Error: LACONIC_LACONICD_RPC_URL not defined"
+    exit 1
+fi
+if [[ -z ${LACONIC_LACONICD_CHAIN_ID} ]]; then
+    echo "Error: LACONIC_LACONICD_CHAIN_ID not defined"
+    exit 1
+fi
+
+# Ping-pub explorer has endlessly confusing behavior where it
+# infers the directory from which to load chain configuration files
+# by the presence or absense of the substring "testnet" in the host name
+# (browser side -- the host name of the host in the address bar of the browser)
+# Accordingly we configure our network in both directories in order to 
+# subvert this lunacy.
+explorer_mainnet_config_dir=/app/chains/mainnet
+explorer_testnet_config_dir=/app/chains/testnet
+config_template_file=/config/chains/laconic-chaindata-template.json
+chain_config_name=laconic.json
+mainnet_config_file=${explorer_mainnet_config_dir}/${chain_config_name}
+testnet_config_file=${explorer_testnet_config_dir}/${chain_config_name}
+
+# Delete the stock config files
+rm -f ${explorer_testnet_config_dir}/*
+rm -f ${explorer_mainnet_config_dir}/*
+
+# Copy in our template file
+cp ${config_template_file} ${mainnet_config_file}
+
+# Update the file with the config variables
+sed -i "s#LACONIC_LACONICD_API_URL#${LACONIC_LACONICD_API_URL}#g" ${mainnet_config_file}
+sed -i "s#LACONIC_LACONICD_RPC_URL#${LACONIC_LACONICD_RPC_URL}#g" ${mainnet_config_file}
+sed -i "s#LACONIC_LACONICD_CHAIN_ID#${LACONIC_LACONICD_CHAIN_ID}#g" ${mainnet_config_file}
+
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  echo "Updated chaindata file:"
+  cat ${mainnet_config_file}
+fi
+
+# Copy over to the testnet directory
+cp ${mainnet_config_file} ${testnet_config_file}
@@ -28,11 +28,13 @@ RUN \
    && su ${USERNAME} -c "umask 0002 && npm install -g semver" \
    # Install pnpm
    && su ${USERNAME} -c "umask 0002 && npm install -g pnpm" \
+    # Install bun
+    && su ${USERNAME} -c "umask 0002 && npm install -g bun@1.1.x" \
    && npm cache clean --force > /dev/null 2>&1

 # [Optional] Uncomment this section to install additional OS packages.
 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
-    && apt-get -y install --no-install-recommends jq gettext-base
+    && apt-get -y install --no-install-recommends jq gettext-base git

 # [Optional] Uncomment if you want to install an additional version of node using nvm
 # ARG EXTRA_NODE_VERSION=10
@@ -4,6 +4,8 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
    set -x
 fi

+# TODO: document what this script does
+
 WORK_DIR="${1:-./}"

 cd "${WORK_DIR}" || exit 1
@@ -27,6 +27,8 @@ elif [ -f "${WORK_DIR}/package.json" ]; then
      CERC_BUILD_TOOL=pnpm
    elif [ -f "yarn.lock" ]; then
      CERC_BUILD_TOOL=yarn
+    elif [ -f "bun.lockb" ]; then
+      CERC_BUILD_TOOL=bun
    else
      CERC_BUILD_TOOL=npm
    fi
@@ -2,7 +2,7 @@ version: "1.0"
 name: ajna
 description: "Ajna watcher stack"
 repos:
-  - git.vdb.to/cerc-io/ajna-watcher-ts@v0.1.12
+  - git.vdb.to/cerc-io/ajna-watcher-ts@v0.1.16
 containers:
  - cerc/watcher-ajna
 pods:
@@ -14,10 +14,11 @@
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.

 from stack_orchestrator.util import get_yaml
-from stack_orchestrator.deploy.deploy_types import DeployCommandContext, LaconicStackSetupCommand, DeploymentContext
+from stack_orchestrator.deploy.deploy_types import DeployCommandContext, LaconicStackSetupCommand
+from stack_orchestrator.deploy.deployment_context import DeploymentContext
 from stack_orchestrator.deploy.stack_state import State
 from stack_orchestrator.deploy.deploy_util import VolumeMapping, run_container_command
-from stack_orchestrator.command_types import CommandOptions
+from stack_orchestrator.opts import opts
 from enum import Enum
 from pathlib import Path
 from shutil import copyfile, copytree
@@ -61,7 +62,7 @@ def _get_node_moniker_from_config(network_dir: Path):
    return moniker


-def _get_node_key_from_gentx(options: CommandOptions, gentx_file_name: str):
+def _get_node_key_from_gentx(gentx_file_name: str):
    gentx_file_path = Path(gentx_file_name)
    if gentx_file_path.exists():
        with open(Path(gentx_file_name), "rb") as f:
@@ -76,24 +77,24 @@ def _comma_delimited_to_list(list_str: str):
    return list_str.split(",") if list_str else []


-def _get_node_keys_from_gentx_files(options: CommandOptions, gentx_file_list: str):
+def _get_node_keys_from_gentx_files(gentx_file_list: str):
    node_keys = []
    gentx_files = _comma_delimited_to_list(gentx_file_list)
    for gentx_file in gentx_files:
-        node_key = _get_node_key_from_gentx(options, gentx_file)
+        node_key = _get_node_key_from_gentx(gentx_file)
        if node_key:
            node_keys.append(node_key)
    return node_keys


-def _copy_gentx_files(options: CommandOptions, network_dir: Path, gentx_file_list: str):
+def _copy_gentx_files(network_dir: Path, gentx_file_list: str):
    gentx_files = _comma_delimited_to_list(gentx_file_list)
    for gentx_file in gentx_files:
        gentx_file_path = Path(gentx_file)
        copyfile(gentx_file_path, os.path.join(network_dir, "config", "gentx", os.path.basename(gentx_file_path)))


-def _remove_persistent_peers(options: CommandOptions, network_dir: Path):
+def _remove_persistent_peers(network_dir: Path):
    config_file_path = _config_toml_path(network_dir)
    if not config_file_path.exists():
        print("Error: config.toml not found")
@@ -107,20 +108,45 @@ def _remove_persistent_peers(options: CommandOptions, network_dir: Path):
        output_file.write(config_file_content)


-def _insert_persistent_peers(options: CommandOptions, config_dir: Path, new_persistent_peers: str):
+def _insert_persistent_peers(config_dir: Path, new_persistent_peers: str):
    config_file_path = config_dir.joinpath("config.toml")
    if not config_file_path.exists():
        print("Error: config.toml not found")
        sys.exit(1)
    with open(config_file_path, "r") as input_file:
        config_file_content = input_file.read()
-        persistent_peers_pattern = '^persistent_peers = ""'
+        persistent_peers_pattern = r'^persistent_peers = ""'
        replace_with = f"persistent_peers = \"{new_persistent_peers}\""
        config_file_content = re.sub(persistent_peers_pattern, replace_with, config_file_content, flags=re.MULTILINE)
    with open(config_file_path, "w") as output_file:
        output_file.write(config_file_content)


+def _enable_cors(config_dir: Path):
+    config_file_path = config_dir.joinpath("config.toml")
+    if not config_file_path.exists():
+        print("Error: config.toml not found")
+        sys.exit(1)
+    with open(config_file_path, "r") as input_file:
+        config_file_content = input_file.read()
+        cors_pattern = r'^cors_allowed_origins = \[]'
+        replace_with = 'cors_allowed_origins = ["*"]'
+        config_file_content = re.sub(cors_pattern, replace_with, config_file_content, flags=re.MULTILINE)
+    with open(config_file_path, "w") as output_file:
+        output_file.write(config_file_content)
+    app_file_path = config_dir.joinpath("app.toml")
+    if not app_file_path.exists():
+        print("Error: app.toml not found")
+        sys.exit(1)
+    with open(app_file_path, "r") as input_file:
+        app_file_content = input_file.read()
+        cors_pattern = r'^enabled-unsafe-cors = false'
+        replace_with = "enabled-unsafe-cors = true"
+        app_file_content = re.sub(cors_pattern, replace_with, app_file_content, flags=re.MULTILINE)
+    with open(app_file_path, "w") as output_file:
+        output_file.write(app_file_content)
+
+
 def _phase_from_params(parameters):
    phase = SetupPhase.ILLEGAL
    if parameters.initialize_network:
@@ -150,7 +176,7 @@ def _phase_from_params(parameters):

 def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCommand, extra_args):

-    options = command_context.cluster_context.options
+    options = opts.o

    currency = "stake"  # Does this need to be a parameter?

@@ -237,7 +263,7 @@ def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCo
                print("Error: --gentx-files must be supplied")
                sys.exit(1)
            # First look in the supplied gentx files for the other nodes' keys
-            other_node_keys = _get_node_keys_from_gentx_files(options, parameters.gentx_file_list)
+            other_node_keys = _get_node_keys_from_gentx_files(parameters.gentx_file_list)
            # Add those keys to our genesis, with balances we determine here (why?)
            for other_node_key in other_node_keys:
                outputk, statusk = run_container_command(
@@ -246,7 +272,7 @@ def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCo
            if options.debug:
                print(f"Command output: {outputk}")
            # Copy the gentx json files into our network dir
-            _copy_gentx_files(options, network_dir, parameters.gentx_file_list)
+            _copy_gentx_files(network_dir, parameters.gentx_file_list)
            # Now we can run collect-gentxs
            output1, status1 = run_container_command(
                command_context, "laconicd", f"laconicd collect-gentxs --home {laconicd_home_path_in_container}", mounts)
@@ -255,7 +281,7 @@ def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCo
            print(f"Generated genesis file, please copy to other nodes as required: \
                {os.path.join(network_dir, 'config', 'genesis.json')}")
            # Last thing, collect-gentxs puts a likely bogus set of persistent_peers in config.toml so we remove that now
-            _remove_persistent_peers(options, network_dir)
+            _remove_persistent_peers(network_dir)
        # In both cases we validate the genesis file now
        output2, status1 = run_container_command(
            command_context, "laconicd", f"laconicd validate-genesis --home {laconicd_home_path_in_container}", mounts)
@@ -266,7 +292,7 @@ def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCo
        sys.exit(1)


-def create(context: DeploymentContext, extra_args):
+def create(deployment_context: DeploymentContext, extra_args):
    network_dir = extra_args[0]
    if network_dir is None:
        print("Error: --network-dir must be supplied")
@@ -285,15 +311,17 @@ def create(context: DeploymentContext, extra_args):
        sys.exit(1)
    # Copy the network directory contents into our deployment
    # TODO: change this to work with non local paths
-    deployment_config_dir = context.deployment_dir.joinpath("data", "laconicd-config")
+    deployment_config_dir = deployment_context.deployment_dir.joinpath("data", "laconicd-config")
    copytree(config_dir_path, deployment_config_dir, dirs_exist_ok=True)
    # If supplied, add the initial persistent peers to the config file
    if extra_args[1]:
        initial_persistent_peers = extra_args[1]
-        _insert_persistent_peers(context.command_context.cluster_context.options, deployment_config_dir, initial_persistent_peers)
+        _insert_persistent_peers(deployment_config_dir, initial_persistent_peers)
+    # Enable CORS headers so explorers and so on can talk to the node
+    _enable_cors(deployment_config_dir)
    # Copy the data directory contents into our deployment
    # TODO: change this to work with non local paths
-    deployment_data_dir = context.deployment_dir.joinpath("data", "laconicd-data")
+    deployment_data_dir = deployment_context.deployment_dir.joinpath("data", "laconicd-data")
    copytree(data_dir_path, deployment_data_dir, dirs_exist_ok=True)


@@ -2,14 +2,15 @@ version: "1.0"
 name: mainnet-laconic
 description: "Mainnet laconic node"
 repos:
-  - cerc-io/laconicd
-  - lirewine/debug
-  - lirewine/crypto
-  - lirewine/gem
-  - lirewine/sdk
-  - cerc-io/laconic-sdk
-  - cerc-io/laconic-registry-cli
-  - cerc-io/laconic-console
+  - git.vdb.to/cerc-io/laconicd
+  - github.com/lirewine/debug
+  - github.com/lirewine/crypto
+  - github.com/lirewine/gem
+  - github.com/lirewine/sdk
+  - git.vdb.to/cerc-io/laconic-sdk
+  - git.vdb.to/cerc-io/laconic-registry-cli
+  - git.vdb.to/cerc-io/laconic-console
+  - github.com/ping-pub/explorer
 npms:
  - laconic-sdk
  - laconic-registry-cli
@@ -23,7 +24,8 @@ containers:
  - cerc/laconic-registry-cli
  - cerc/webapp-base
  - cerc/laconic-console-host
+  - cerc/ping-pub
 pods:
  - mainnet-laconicd
  - fixturenet-laconic-console
-
+  - laconic-explorer
@@ -2,7 +2,7 @@ version: "1.0"
 name: merkl-sushiswap-v3
 description: "SushiSwap v3 watcher stack"
 repos:
-  - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.13
+  - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.14
 containers:
  - cerc/watcher-merkl-sushiswap-v3
 pods:
@@ -99,10 +99,6 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers
        labels:
          instance: 'sushiswap'
          chain: 'filecoin'
-      - targets: ['MERKLE_SUSHISWAP_WATCHER_HOST:MERKLE_SUSHISWAP_WATCHER_METRICS_PORT', 'MERKLE_SUSHISWAP_WATCHER_HOST:MERKLE_SUSHISWAP_WATCHER_GQL_METRICS_PORT']
-        labels:
-          instance: 'merkl_sushiswap'
-          chain: 'filecoin'

  - job_name: ajna
    scrape_interval: 20s
@@ -2,7 +2,7 @@ version: "1.0"
 name: sushiswap-v3
 description: "SushiSwap v3 watcher stack"
 repos:
-  - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.13
+  - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.16
 containers:
  - cerc/watcher-sushiswap-v3
 pods:
@@ -26,8 +26,15 @@ import click
 from pathlib import Path
 from stack_orchestrator import constants
 from stack_orchestrator.opts import opts
-from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config, global_options2, get_dev_root_path
-from stack_orchestrator.util import resolve_compose_file
+from stack_orchestrator.util import (
+    get_stack_path,
+    include_exclude_check,
+    get_parsed_stack_config,
+    global_options2,
+    get_dev_root_path,
+    stack_is_in_deployment,
+    resolve_compose_file,
+)
 from stack_orchestrator.deploy.deployer import Deployer, DeployerException
 from stack_orchestrator.deploy.deployer_factory import getDeployer
 from stack_orchestrator.deploy.deploy_types import ClusterContext, DeployCommandContext
@@ -60,6 +67,7 @@ def command(ctx, include, exclude, env_file, cluster, deploy_to):
    if deploy_to is None:
        deploy_to = "compose"

+    stack = get_stack_path(stack)
    ctx.obj = create_deploy_context(global_options2(ctx), None, stack, include, exclude, cluster, env_file, deploy_to)
    # Subcommand is executed now, by the magic of click

@@ -274,16 +282,12 @@ def _make_default_cluster_name(deployment, compose_dir, stack, include, exclude)

 # stack has to be either PathLike pointing to a stack yml file, or a string with the name of a known stack
 def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file):
-
    dev_root_path = get_dev_root_path(ctx)

-    # TODO: huge hack, fix this
-    # If the caller passed a path for the stack file, then we know that we can get the compose files
-    # from the same directory
-    deployment = False
-    if isinstance(stack, os.PathLike):
-        compose_dir = stack.parent.joinpath("compose")
-        deployment = True
+    # TODO: hack, this should be encapsulated by the deployment context.
+    deployment = stack_is_in_deployment(stack)
+    if deployment:
+        compose_dir = stack.joinpath("compose")
    else:
        # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
        compose_dir = Path(__file__).absolute().parent.parent.joinpath("data", "compose")
@@ -50,15 +50,15 @@ def command(ctx, dir):

 def make_deploy_context(ctx) -> DeployCommandContext:
    context: DeploymentContext = ctx.obj
-    stack_file_path = context.get_stack_file()
    env_file = context.get_env_file()
    cluster_name = context.get_cluster_id()
    if constants.deploy_to_key in context.spec.obj:
        deployment_type = context.spec.obj[constants.deploy_to_key]
    else:
        deployment_type = constants.compose_deploy_type
-    return create_deploy_context(ctx.parent.parent.obj, context, stack_file_path, None, None, cluster_name, env_file,
-                                 deployment_type)
+    stack = context.deployment_dir
+    return create_deploy_context(ctx.parent.parent.obj, context, stack, None, None,
+                                 cluster_name, env_file, deployment_type)


@command.command()
@@ -123,6 +123,7 @@ def push_images(ctx):
@click.argument('extra_args', nargs=-1)  # help: command: port <service1> <service2>
@click.pass_context
 def port(ctx, extra_args):
+    ctx.obj = make_deploy_context(ctx)
    port_operation(ctx, extra_args)


@@ -24,7 +24,7 @@ from secrets import token_hex
 import sys
 from stack_orchestrator import constants
 from stack_orchestrator.opts import opts
-from stack_orchestrator.util import (get_stack_file_path, get_parsed_deployment_spec, get_parsed_stack_config,
+from stack_orchestrator.util import (get_stack_path, get_parsed_deployment_spec, get_parsed_stack_config,
                                     global_options, get_yaml, get_pod_list, get_pod_file_path, pod_has_scripts,
                                     get_pod_script_paths, get_plugin_code_paths, error_exit, env_var_map_from_file,
                                     resolve_config_dir)
@@ -238,6 +238,11 @@ def _find_extra_config_dirs(parsed_pod_file, pod):
                        config_dir = host_path.split("/")[2]
                        if config_dir != pod:
                            config_dirs.add(config_dir)
+        for env_file in service_info.get("env_file", []):
+            if env_file.startswith("../config"):
+                config_dir = env_file.split("/")[2]
+                if config_dir != pod:
+                    config_dirs.add(config_dir)
    return config_dirs


@@ -454,7 +459,7 @@ def create_operation(deployment_command_context, spec_file, deployment_dir, netw
    _check_volume_definitions(parsed_spec)
    stack_name = parsed_spec["stack"]
    deployment_type = parsed_spec[constants.deploy_to_key]
-    stack_file = get_stack_file_path(stack_name)
+    stack_file = get_stack_path(stack_name).joinpath(constants.stack_file_name)
    parsed_stack = get_parsed_stack_config(stack_name)
    if opts.o.debug:
        print(f"parsed spec: {parsed_spec}")
@@ -467,7 +472,7 @@ def create_operation(deployment_command_context, spec_file, deployment_dir, netw
    os.mkdir(deployment_dir_path)
    # Copy spec file and the stack file into the deployment dir
    copyfile(spec_file, deployment_dir_path.joinpath(constants.spec_file_name))
-    copyfile(stack_file, deployment_dir_path.joinpath(os.path.basename(stack_file)))
+    copyfile(stack_file, deployment_dir_path.joinpath(constants.stack_file_name))
    _create_deployment_file(deployment_dir_path)
    # Copy any config varibles from the spec file into an env file suitable for compose
    _write_config_file(spec_file, deployment_dir_path.joinpath(constants.config_file_name))
@@ -29,16 +29,29 @@ def _image_needs_pushed(image: str):
    return image.endswith(":local")


+def _remote_tag_for_image(image: str, remote_repo_url: str):
+    # Turns image tags of the form: foo/bar:local into remote.repo/org/bar:deploy
+    major_parts = image.split("/", 2)
+    image_name_with_version = major_parts[1] if 2 == len(major_parts) else major_parts[0]
+    (image_name, image_version) = image_name_with_version.split(":")
+    if image_version == "local":
+        return f"{remote_repo_url}/{image_name}:deploy"
+    else:
+        return image
+
+
+# Note: do not add any calls this function
 def remote_image_exists(remote_repo_url: str, local_tag: str):
    docker = DockerClient()
    try:
-        remote_tag = remote_tag_for_image(local_tag, remote_repo_url)
+        remote_tag = _remote_tag_for_image(local_tag, remote_repo_url)
        result = docker.manifest.inspect(remote_tag)
        return True if result else False
    except Exception:  # noqa: E722
        return False


+# Note: do not add any calls this function
 def add_tags_to_image(remote_repo_url: str, local_tag: str, *additional_tags):
    if not additional_tags:
        return
@@ -47,18 +60,20 @@ def add_tags_to_image(remote_repo_url: str, local_tag: str, *additional_tags):
        raise Exception(f"{local_tag} does not exist in {remote_repo_url}")

    docker = DockerClient()
-    remote_tag = remote_tag_for_image(local_tag, remote_repo_url)
-    new_remote_tags = [remote_tag_for_image(tag, remote_repo_url) for tag in additional_tags]
+    remote_tag = _remote_tag_for_image(local_tag, remote_repo_url)
+    new_remote_tags = [_remote_tag_for_image(tag, remote_repo_url) for tag in additional_tags]
    docker.buildx.imagetools.create(sources=[remote_tag], tags=new_remote_tags)


-def remote_tag_for_image(image: str, remote_repo_url: str):
+def remote_tag_for_image_unique(image: str, remote_repo_url: str, deployment_id: str):
    # Turns image tags of the form: foo/bar:local into remote.repo/org/bar:deploy
    major_parts = image.split("/", 2)
    image_name_with_version = major_parts[1] if 2 == len(major_parts) else major_parts[0]
    (image_name, image_version) = image_name_with_version.split(":")
    if image_version == "local":
-        return f"{remote_repo_url}/{image_name}:deploy"
+        # Salt the tag with part of the deployment id to make it unique to this deployment
+        deployment_tag = deployment_id[-8:]
+        return f"{remote_repo_url}/{image_name}:deploy-{deployment_tag}"
    else:
        return image

@@ -73,14 +88,14 @@ def push_images_operation(command_context: DeployCommandContext, deployment_cont
    docker = DockerClient()
    for image in images:
        if _image_needs_pushed(image):
-            remote_tag = remote_tag_for_image(image, remote_repo_url)
+            remote_tag = remote_tag_for_image_unique(image, remote_repo_url, deployment_context.id)
            if opts.o.verbose:
                print(f"Tagging {image} to {remote_tag}")
            docker.image.tag(image, remote_tag)
    # Run docker push commands to upload
    for image in images:
        if _image_needs_pushed(image):
-            remote_tag = remote_tag_for_image(image, remote_repo_url)
+            remote_tag = remote_tag_for_image_unique(image, remote_repo_url, deployment_context.id)
            if opts.o.verbose:
                print(f"Pushing image {remote_tag}")
            docker.image.push(remote_tag)
@@ -26,7 +26,7 @@ from stack_orchestrator.deploy.k8s.helpers import envs_from_environment_variable
 from stack_orchestrator.deploy.deploy_util import parsed_pod_files_map_from_file_names, images_for_deployment
 from stack_orchestrator.deploy.deploy_types import DeployEnvVars
 from stack_orchestrator.deploy.spec import Spec, Resources, ResourceLimits
-from stack_orchestrator.deploy.images import remote_tag_for_image
+from stack_orchestrator.deploy.images import remote_tag_for_image_unique

 DEFAULT_VOLUME_RESOURCES = Resources({
    "reservations": {"storage": "2Gi"}
@@ -326,8 +326,11 @@ class ClusterInfo:
                if opts.o.debug:
                    print(f"Merged envs: {envs}")
                # Re-write the image tag for remote deployment
-                image_to_use = remote_tag_for_image(
-                    image, self.spec.get_image_registry()) if self.spec.get_image_registry() is not None else image
+                # Note self.app_name has the same value as deployment_id
+                image_to_use = remote_tag_for_image_unique(
+                    image,
+                    self.spec.get_image_registry(),
+                    self.app_name) if self.spec.get_image_registry() is not None else image
                volume_mounts = volume_mounts_for_service(self.parsed_pod_yaml_map, service_name)
                container = client.V1Container(
                    name=container_name,
@@ -24,7 +24,7 @@ import uuid

 import click

-from stack_orchestrator.deploy.images import remote_image_exists, add_tags_to_image
+from stack_orchestrator.deploy.images import remote_image_exists
 from stack_orchestrator.deploy.webapp import deploy_webapp
 from stack_orchestrator.deploy.webapp.util import (LaconicRegistryClient, TimedLogger,
                                                   build_container_image, push_container_image,
@@ -99,44 +99,62 @@ def process_app_deployment_request(

    deployment_record = laconic.get_record(app_deployment_crn)
    deployment_dir = os.path.join(deployment_parent_dir, fqdn)
+    # At present we use this to generate a unique but stable ID for the app's host container
+    # TODO: implement support to derive this transparently from the already-unique deployment id
+    unique_deployment_id = hashlib.md5(fqdn.encode()).hexdigest()[:16]
    deployment_config_file = os.path.join(deployment_dir, "config.env")
-    # TODO: Is there any reason not to simplify the hash input to the app_deployment_crn?
-    deployment_container_tag = "laconic-webapp/%s:local" % hashlib.md5(deployment_dir.encode()).hexdigest()
+    deployment_container_tag = "laconic-webapp/%s:local" % unique_deployment_id
    app_image_shared_tag = f"laconic-webapp/{app.id}:local"
    #   b. check for deployment directory (create if necessary)
    if not os.path.exists(deployment_dir):
        if deployment_record:
            raise Exception("Deployment record %s exists, but not deployment dir %s. Please remove name." %
                            (app_deployment_crn, deployment_dir))
-        print("deploy_webapp", deployment_dir)
+        logger.log(f"Creating webapp deployment in: {deployment_dir} with container id: {deployment_container_tag}")
        deploy_webapp.create_deployment(ctx, deployment_dir, deployment_container_tag,
                                        f"https://{fqdn}", kube_config, image_registry, env_filename)
    elif env_filename:
        shutil.copyfile(env_filename, deployment_config_file)

    needs_k8s_deploy = False
+    if force_rebuild:
+        logger.log("--force-rebuild is enabled so the container will always be built now, even if nothing has changed in the app")
    # 6. build container (if needed)
-    if not deployment_record or deployment_record.attributes.application != app.id:
+    # TODO: add a comment that explains what this code is doing (not clear to me)
+    if not deployment_record or deployment_record.attributes.application != app.id or force_rebuild:
        needs_k8s_deploy = True
        # check if the image already exists
        shared_tag_exists = remote_image_exists(image_registry, app_image_shared_tag)
+        # Note: in the code below, calls to add_tags_to_image() won't work at present.
+        # This is because SO deployment code in general re-names the container image
+        # to be unique to the deployment. This is done transparently
+        # and so when we call add_tags_to_image() here and try to add tags to the remote image,
+        # we get the image name wrong. Accordingly I've disabled the relevant code for now.
+        # This is safe because we are running with --force-rebuild at present
        if shared_tag_exists and not force_rebuild:
            # simply add our unique tag to the existing image and we are done
-            logger.log(f"Using existing app image {app_image_shared_tag} for {deployment_container_tag}")
-            add_tags_to_image(image_registry, app_image_shared_tag, deployment_container_tag)
+            logger.log(
+                f"(SKIPPED) Existing image found for this app: {app_image_shared_tag} "
+                "tagging it with: {deployment_container_tag} to use in this deployment"
+                )
+            # add_tags_to_image(image_registry, app_image_shared_tag, deployment_container_tag)
            logger.log("Tag complete")
        else:
            extra_build_args = []  # TODO: pull from request
-            logger.log(f"Building container image {deployment_container_tag}")
+            logger.log(f"Building container image: {deployment_container_tag}")
            build_container_image(app, deployment_container_tag, extra_build_args, logger)
            logger.log("Build complete")
-            logger.log(f"Pushing container image {deployment_container_tag}")
+            logger.log(f"Pushing container image: {deployment_container_tag}")
            push_container_image(deployment_dir, logger)
            logger.log("Push complete")
            # The build/push commands above will use the unique deployment tag, so now we need to add the shared tag.
-            logger.log(f"Updating app image tag {app_image_shared_tag} from build of {deployment_container_tag}")
-            add_tags_to_image(image_registry, deployment_container_tag, app_image_shared_tag)
+            logger.log(
+                f"(SKIPPED) Adding global app image tag: {app_image_shared_tag} to newly built image: {deployment_container_tag}"
+                )
+            # add_tags_to_image(image_registry, deployment_container_tag, app_image_shared_tag)
            logger.log("Tag complete")
+    else:
+        logger.log("Requested app is already deployed, skipping build and image push")

    # 7. update config (if needed)
    if not deployment_record or file_hash(deployment_config_file) != deployment_record.attributes.meta.config:
@@ -242,6 +242,7 @@ def determine_base_container(clone_dir, app_type="webapp"):
 def build_container_image(app_record, tag, extra_build_args=[], logger=None):
    tmpdir = tempfile.mkdtemp()

+    # TODO: determine if this code could be calling into the Python git library like setup-repositories
    try:
        record_id = app_record["id"]
        ref = app_record.attributes.repository_ref
@@ -249,6 +250,16 @@ def build_container_image(app_record, tag, extra_build_args=[], logger=None):
        clone_dir = os.path.join(tmpdir, record_id)

        logger.log(f"Cloning repository {repo} to {clone_dir} ...")
+        # Set github credentials if present running a command like:
+        # git config --global url."https://${TOKEN}:@github.com/".insteadOf "https://github.com/"
+        github_token = os.environ.get("DEPLOYER_GITHUB_TOKEN")
+        if github_token:
+            logger.log("Github token detected, setting it in the git environment")
+            git_config_args = [
+                "git", "config", "--global", f"url.https://{github_token}:@github.com/.insteadOf", "https://github.com/"
+                ]
+            result = subprocess.run(git_config_args, stdout=logger.file, stderr=logger.file)
+            result.check_returncode()
        if ref:
            # TODO: Determing branch or hash, and use depth 1 if we can.
            git_env = dict(os.environ.copy())
@@ -265,6 +276,7 @@ def build_container_image(app_record, tag, extra_build_args=[], logger=None):
                logger.log(f"git checkout failed.  Does ref {ref} exist?")
                raise e
        else:
+            # TODO: why is this code different vs the branch above (run vs check_call, and no prompt disable)?
            result = subprocess.run(["git", "clone", "--depth", "1", repo, clone_dir], stdout=logger.file, stderr=logger.file)
            result.check_returncode()

@@ -299,11 +311,12 @@ def push_container_image(deployment_dir, logger):

 def deploy_to_k8s(deploy_record, deployment_dir, logger):
    if not deploy_record:
-        command = "up"
+        command = "start"
    else:
        command = "update"

    logger.log("Deploying to k8s ...")
+    logger.log(f"Running {command} command on deployment dir: {deployment_dir}")
    result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, command],
                            stdout=logger.file, stderr=logger.file)
    result.check_returncode()
@@ -20,14 +20,12 @@ import os
 import sys
 from decouple import config
 import git
+from git.exc import GitCommandError
 from tqdm import tqdm
 import click
 import importlib.resources
-from pathlib import Path
-import yaml
-from stack_orchestrator.constants import stack_file_name
 from stack_orchestrator.opts import opts
-from stack_orchestrator.util import include_exclude_check, stack_is_external, error_exit, warn_exit
+from stack_orchestrator.util import get_parsed_stack_config, include_exclude_check, error_exit, warn_exit


 class GitProgress(git.RemoteProgress):
@@ -81,9 +79,13 @@ def _get_repo_current_branch_or_tag(full_filesystem_repo_path):
    except TypeError:
        # This means that the current ref is not a branch, so possibly a tag
        # Let's try to get the tag
-        current_repo_branch_or_tag = git.Repo(full_filesystem_repo_path).git.describe("--tags", "--exact-match")
-        # Note that git is assymetric -- the tag you told it to check out may not be the one
-        # you get back here (if there are multiple tags associated with the same commit)
+        try:
+            current_repo_branch_or_tag = git.Repo(full_filesystem_repo_path).git.describe("--tags", "--exact-match")
+            # Note that git is asymmetric -- the tag you told it to check out may not be the one
+            # you get back here (if there are multiple tags associated with the same commit)
+        except GitCommandError:
+            # If there is no matching branch or tag checked out, just use the current SHA
+            current_repo_branch_or_tag = git.Repo(full_filesystem_repo_path).commit("HEAD").hexsha
    return current_repo_branch_or_tag, is_branch


@@ -102,7 +104,7 @@ def process_repo(pull, check_only, git_ssh, dev_root_path, branches_array, fully
        full_filesystem_repo_path
        ) if is_present else (None, None)
    if not opts.o.quiet:
-        present_text = f"already exists active {'branch' if is_branch else 'tag'}: {current_repo_branch_or_tag}" if is_present \
+        present_text = f"already exists active {'branch' if is_branch else 'ref'}: {current_repo_branch_or_tag}" if is_present \
            else 'Needs to be fetched'
        print(f"Checking: {full_filesystem_repo_path}: {present_text}")
    # Quick check that it's actually a repo
@@ -120,7 +122,7 @@ def process_repo(pull, check_only, git_ssh, dev_root_path, branches_array, fully
                        origin = git_repo.remotes.origin
                        origin.pull(progress=None if opts.o.quiet else GitProgress())
                    else:
-                        print("skipping pull because this repo checked out a tag")
+                        print("skipping pull because this repo is not on a branch")
                else:
                    print("(git pull skipped)")
    if not is_present:
@@ -222,20 +224,10 @@ def command(ctx, include, exclude, git_ssh, check_only, pull, branches):

    repos_in_scope = []
    if stack:
-        if stack_is_external(stack):
-            stack_file_path = Path(stack).joinpath(stack_file_name)
-        else:
-            # In order to be compatible with Python 3.8 we need to use this hack to get the path:
-            # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
-            stack_file_path = Path(__file__).absolute().parent.parent.joinpath("data", "stacks", stack, stack_file_name)
-        if not stack_file_path.exists():
-            error_exit(f"stack {stack} does not exist")
-        with stack_file_path:
-            stack_config = yaml.safe_load(open(stack_file_path, "r"))
-            if "repos" not in stack_config or stack_config["repos"] is None:
-                warn_exit(f"stack {stack} does not define any repositories")
-            else:
-                repos_in_scope = stack_config["repos"]
+        stack_config = get_parsed_stack_config(stack)
+        if "repos" not in stack_config or stack_config["repos"] is None:
+            warn_exit(f"stack {stack} does not define any repositories")
+        repos_in_scope = stack_config["repos"]
    else:
        repos_in_scope = all_repos

@@ -20,6 +20,7 @@ import ruamel.yaml
 from pathlib import Path
 from dotenv import dotenv_values
 from typing import Mapping, Set, List
+from stack_orchestrator.constants import stack_file_name, deployment_file_name


 def include_exclude_check(s, include, exclude):
@@ -33,11 +34,14 @@ def include_exclude_check(s, include, exclude):
        return s not in exclude_list


-def get_stack_file_path(stack):
-    # In order to be compatible with Python 3.8 we need to use this hack to get the path:
-    # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
-    stack_file_path = Path(__file__).absolute().parent.joinpath("data", "stacks", stack, "stack.yml")
-    return stack_file_path
+def get_stack_path(stack):
+    if stack_is_external(stack):
+        stack_path = Path(stack)
+    else:
+        # In order to be compatible with Python 3.8 we need to use this hack to get the path:
+        # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
+        stack_path = Path(__file__).absolute().parent.joinpath("data", "stacks", stack)
+    return stack_path


 def get_dev_root_path(ctx):
@@ -52,21 +56,14 @@ def get_dev_root_path(ctx):

 # Caller can pass either the name of a stack, or a path to a stack file
 def get_parsed_stack_config(stack):
-    stack_file_path = stack if isinstance(stack, os.PathLike) else get_stack_file_path(stack)
-    try:
-        with stack_file_path:
-            stack_config = get_yaml().load(open(stack_file_path, "r"))
-            return stack_config
-    except FileNotFoundError as error:
-        # We try here to generate a useful diagnostic error
-        # First check if the stack directory is present
-        stack_directory = stack_file_path.parent
-        if os.path.exists(stack_directory):
-            print(f"Error: stack.yml file is missing from stack: {stack}")
-        else:
-            print(f"Error: stack: {stack} does not exist")
-        print(f"Exiting, error: {error}")
-        sys.exit(1)
+    stack_file_path = get_stack_path(stack).joinpath(stack_file_name)
+    if stack_file_path.exists():
+        return get_yaml().load(open(stack_file_path, "r"))
+    # We try here to generate a useful diagnostic error
+    # First check if the stack directory is present
+    if stack_file_path.parent.exists():
+        error_exit(f"stack.yml file is missing from stack: {stack}")
+    error_exit(f"stack {stack} does not exist")


 def get_pod_list(parsed_stack):
@@ -87,7 +84,7 @@ def get_plugin_code_paths(stack) -> List[Path]:
    result: Set[Path] = set()
    for pod in pods:
        if type(pod) is str:
-            result.add(get_stack_file_path(stack).parent)
+            result.add(get_stack_path(stack))
        else:
            pod_root_dir = os.path.join(get_dev_root_path(None), pod["repository"].split("/")[-1], pod["path"])
            result.add(Path(os.path.join(pod_root_dir, "stack")))
@@ -199,6 +196,10 @@ def stack_is_external(stack: str):
    return Path(stack).exists() if stack is not None else False


+def stack_is_in_deployment(stack: Path):
+    return stack.joinpath(deployment_file_name).exists()
+
+
 def get_yaml():
    # See: https://stackoverflow.com/a/45701840/1701505
    yaml = ruamel.yaml.YAML()
@@ -14,7 +14,7 @@
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.

 import click
-import importlib.resources
+from importlib import resources, metadata


@click.command()
@@ -24,8 +24,11 @@ def command(ctx):

    # See: https://stackoverflow.com/a/20885799/1701505
    from stack_orchestrator import data
-    with importlib.resources.open_text(data, "build_tag.txt") as version_file:
-        # TODO: code better version that skips comment lines
-        version_string = version_file.read().splitlines()[1]
+    if resources.is_resource(data, "build_tag.txt"):
+        with resources.open_text(data, "build_tag.txt") as version_file:
+            # TODO: code better version that skips comment lines
+            version_string = version_file.read().splitlines()[1]
+    else:
+        version_string = metadata.version("laconic-stack-orchestrator") + "-unknown"

-    print(f"Version: {version_string}")
+    print(version_string)
@@ -22,16 +22,16 @@ echo "$(date +"%Y-%m-%d %T"): Stack started"
 # Verify that the fixturenet is up and running
 $TEST_TARGET_SO --stack fixturenet-laconicd deploy --cluster laconicd ps

+# Wait for the laconid endpoint to come up
+echo "Waiting for the RPC endpoint to come up"
+docker exec laconicd-laconicd-1 sh -c "curl --retry 20 --retry-delay 3 --retry-connrefused http://127.0.0.1:9473/api"
+
 # Get the fixturenet account address
 laconicd_account_address=$(docker exec laconicd-laconicd-1 laconicd keys list | awk '/- address:/ {print $3}')

 # Copy over config
 docker exec laconicd-cli-1 cp config.yml laconic-registry-cli/

-# Wait for the laconid endpoint to come up
-echo "Waiting for the RPC endpoint to come up"
-docker exec laconicd-laconicd-1 sh -c "curl --retry 20 --retry-delay 3 --retry-connrefused http://127.0.0.1:9473/api"
-
 # Run the tests
 echo "Running the tests"
 docker exec -e TEST_ACCOUNT=$laconicd_account_address laconicd-cli-1 sh -c 'cd laconic-registry-cli && yarn && yarn test'
@@ -18,6 +18,20 @@ do
        rm -rf ${node_network_dir}
    fi
 done
+echo "Deleting any existing deployments..."
+for (( i=1 ; i<=$node_count ; i++ ));
+do
+    node_deployment_dir=${node_dir_prefix}${i}-deployment
+    node_spec_file=${node_dir_prefix}${i}-spec.yml
+    if [[ -d $node_deployment_dir ]]; then
+        echo "Deleting ${node_deployment_dir}"
+        rm -rf ${node_deployment_dir}
+    fi
+    if [[ -f $node_spec_file ]]; then
+        echo "Deleting ${node_spec_file}"
+        rm ${node_spec_file}
+    fi
+done

 echo "Initalizing ${node_count} nodes networks..."
 for (( i=1 ; i<=$node_count ; i++ )); 
@@ -56,3 +70,12 @@ do
    node_network_dir=${node_dir_prefix}${i}
    laconic-so --stack mainnet-laconic deploy setup --network-dir ${node_network_dir} --create-network --genesis-file ${genesis_file}
 done
+
+# Create deployments
+echo "Creating ${node_count} deployments..."
+for (( i=1 ; i<=$node_count ; i++ ));
+do
+    node_network_dir=${node_dir_prefix}${i}
+    laconic-so --stack mainnet-laconic deploy init --output ${node_network_dir}-spec.yml
+    laconic-so --stack mainnet-laconic deploy create --deployment-dir ${node_network_dir}-deployment --spec-file ${node_dir_prefix}${i}-spec.yml --network-dir ${node_network_dir}
+done