Add alert rule for steer protocol watcher

stack_orchestrator/data/compose/docker-compose-graph-node.yml

@@ -1,5 +1,6 @@
 services:
   graph-node:
+    restart: unless-stopped
     image: cerc/graph-node:local
     depends_on:
       db:
@@ -40,6 +41,7 @@ services:
       retries: 10
       start_period: 3s
   ipfs:
+    restart: unless-stopped
     image: ipfs/kubo:master-2023-02-20-714a968
     volumes:
       - ipfs-import:/import
@@ -49,6 +51,7 @@ services:
       - "4001"
       - "5001"
   db:
+    restart: unless-stopped
     image: postgres:14-alpine
     volumes:
       - db-data:/var/lib/postgresql/data

stack_orchestrator/data/config/monitoring/grafana/dashboards/watcher-dashboard.json

@@ -2866,6 +2866,19 @@
               "legendFormat": "{{field}}",
               "range": true,
               "refId": "E"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "PBFA97CFB590B2093"
+              },
+              "editorMode": "code",
+              "expr": "watcher_config_info{job=~\"$job\", instance=~\"$watcher\", category=\"jobqueue\", field=\"block_processing_offset\"}",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "{{field}}",
+              "range": true,
+              "refId": "F"
             }
           ],
           "title": "Watcher config (job queue)",
@@ -3284,6 +3297,6 @@
   "timepicker": {},
   "timezone": "",
   "title": "Watchers",
-  "version": 5,
+  "version": 6,
   "weekStart": ""
 }

stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml

@@ -49,6 +49,10 @@ scrape_configs:
         labels:
           instance: 'external'
           chain: 'ethereum'
+      - targets: ['sepolia-chain-head-exporter:5000']
+        labels:
+          instance: 'external'
+          chain: 'sepolia'
       - targets: ['filecoin-chain-head-exporter:5000']
         labels:
           instance: 'external'

stack_orchestrator/data/config/monitoring/subgraph-alert-rules.yml

@@ -19,7 +19,7 @@ groups:
                   type: prometheus
                   uid: PBFA97CFB590B2093
               editorMode: code
-              expr: ethereum_chain_head_number - on(network) group_right deployment_head{deployment=~"REPLACE_WITH_SUBGRAPH_IDS"}
+              expr: label_replace(latest_block_number{instance="external"}, "network", "$1", "chain", "(.*)") - on(network) group_right deployment_head{deployment=~"REPLACE_WITH_SUBGRAPH_IDS"}
               instant: true
               intervalMs: 1000
               legendFormat: __auto

stack_orchestrator/data/config/monitoring/watcher-alert-rules.yml

@@ -695,82 +695,6 @@ groups:
         annotations:
           summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
         isPaused: false
-      - uid: merkl_sushiswap_diff_external
-        title: merkl_sushiswap_watcher_head_tracking
-        condition: condition
-        data:
-          - refId: diff
-            relativeTimeRange:
-              from: 600
-              to: 0
-            datasourceUid: PBFA97CFB590B2093
-            model:
-              datasource:
-                type: prometheus
-                uid: PBFA97CFB590B2093
-              disableTextWrap: false
-              editorMode: code
-              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="sushi", instance="merkl_sushiswap", kind="latest_indexed"}
-              fullMetaSearch: false
-              includeNullMetadata: true
-              instant: true
-              intervalMs: 1000
-              legendFormat: __auto
-              maxDataPoints: 43200
-              range: false
-              refId: diff
-              useBackend: false
-          - refId: latest_external
-            relativeTimeRange:
-              from: 600
-              to: 0
-            datasourceUid: PBFA97CFB590B2093
-            model:
-              datasource:
-                type: prometheus
-                uid: PBFA97CFB590B2093
-              editorMode: code
-              expr: latest_block_number{chain="filecoin"}
-              hide: false
-              instant: true
-              legendFormat: __auto
-              range: false
-              refId: latest_external
-          - refId: condition
-            relativeTimeRange:
-              from: 600
-              to: 0
-            datasourceUid: __expr__
-            model:
-              conditions:
-                - evaluator:
-                    params:
-                      - 0
-                      - 0
-                    type: gt
-                  operator:
-                    type: and
-                  query:
-                    params: []
-                  reducer:
-                    params: []
-                    type: avg
-                  type: query
-              datasource:
-                name: Expression
-                type: __expr__
-                uid: __expr__
-              expression: ${diff} >= 16
-              intervalMs: 1000
-              maxDataPoints: 43200
-              refId: condition
-              type: math
-        noDataState: Alerting
-        execErrState: Alerting
-        for: 15m
-        annotations:
-          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
-        isPaused: false
 
       # Ajna
       - uid: ajna_diff_external
@@ -849,7 +773,7 @@ groups:
         annotations:
           summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
         isPaused: false
-      
+
       # Secured Finance
       - uid: secured_finance_diff_external
         title: secured_finance_watcher_head_tracking
@@ -927,3 +851,159 @@ groups:
         annotations:
           summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
         isPaused: false
+
+      # Borg
+      - uid: borg_diff_external
+        title: borg_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="borg", instance="borg", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="sepolia"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+
+      # Steer Protocol
+      - uid: steer_protocol_diff_external
+        title: steer_protocol_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="steer-protocol", instance="steer-protocol", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="filecoin"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false

stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py

@@ -54,7 +54,6 @@ def process_app_deployment_request(
     deployment_record_namespace,
     dns_record_namespace,
     default_dns_suffix,
-    dns_value,
     deployment_parent_dir,
     kube_config,
     image_registry,
@@ -172,7 +171,6 @@ def process_app_deployment_request(
         logger.log(
             f"Creating webapp deployment in: {deployment_dir} with container id: {deployment_container_tag}"
         )
-        # CREATES DEPLOYMENT DIR, NOT SKIPPING FOR TESTING
         deploy_webapp.create_deployment(
             ctx,
             deployment_dir,
@@ -215,24 +213,21 @@ def process_app_deployment_request(
             # add_tags_to_image(image_registry, app_image_shared_tag, deployment_container_tag)
             logger.log("Tag complete")
         else:
-            # SKIP BUILD
-            logger.log("TESTING: Skipping container build.")
-
-            # extra_build_args = []  # TODO: pull from request
-            # logger.log(f"Building container image: {deployment_container_tag}")
-            # build_container_image(
-            #     app, deployment_container_tag, extra_build_args, logger
-            # )
-            # logger.log("Build complete")
-            # logger.log(f"Pushing container image: {deployment_container_tag}")
-            # push_container_image(deployment_dir, logger)
-            # logger.log("Push complete")
-            # # The build/push commands above will use the unique deployment tag, so now we need to add the shared tag.
-            # logger.log(
-            #     f"(SKIPPED) Adding global app image tag: {app_image_shared_tag} to newly built image: {deployment_container_tag}"
-            # )
-            # # add_tags_to_image(image_registry, deployment_container_tag, app_image_shared_tag)
-            # logger.log("Tag complete")
+            extra_build_args = []  # TODO: pull from request
+            logger.log(f"Building container image: {deployment_container_tag}")
+            build_container_image(
+                app, deployment_container_tag, extra_build_args, logger
+            )
+            logger.log("Build complete")
+            logger.log(f"Pushing container image: {deployment_container_tag}")
+            push_container_image(deployment_dir, logger)
+            logger.log("Push complete")
+            # The build/push commands above will use the unique deployment tag, so now we need to add the shared tag.
+            logger.log(
+                f"(SKIPPED) Adding global app image tag: {app_image_shared_tag} to newly built image: {deployment_container_tag}"
+            )
+            # add_tags_to_image(image_registry, deployment_container_tag, app_image_shared_tag)
+            logger.log("Tag complete")
     else:
         logger.log("Requested app is already deployed, skipping build and image push")
 
@@ -245,9 +240,7 @@ def process_app_deployment_request(
 
     # 8. update k8s deployment
     if needs_k8s_deploy:
-        # SKIP DEPLOY
-        logger.log("TESTING: Skipping deployment to k8s.")
-        # deploy_to_k8s(deployment_record, deployment_dir, recreate_on_deploy, logger)
+        deploy_to_k8s(deployment_record, deployment_dir, recreate_on_deploy, logger)
 
     logger.log("Publishing deployment to registry.")
     publish_deployment(
@@ -258,7 +251,6 @@ def process_app_deployment_request(
         dns_record,
         dns_lrn,
         deployment_dir,
-        dns_value,
         app_deployment_request,
         webapp_deployer_record,
         logger,
@@ -312,7 +304,6 @@ def dump_known_requests(filename, requests, status="SEEN"):
     help="How to handle requests with an FQDN: prohibit, allow, preexisting",
     default="prohibit",
 )
-@click.option("--ip", help="IP address of the k8s deployment (to be set in DNS record)", default=None)
 @click.option("--record-namespace-dns", help="eg, lrn://laconic/dns", required=True)
 @click.option(
     "--record-namespace-deployments",
@@ -390,7 +381,6 @@ def command(  # noqa: C901
     only_update_state,
     dns_suffix,
     fqdn_policy,
-    ip,
     record_namespace_dns,
     record_namespace_deployments,
     dry_run,
@@ -439,13 +429,6 @@ def command(  # noqa: C901
         )
         sys.exit(2)
 
-    if fqdn_policy == "allow" and not ip:
-        print(
-            "--ip is required with 'allow' fqdn-policy",
-            file=sys.stderr,
-        )
-        sys.exit(2)
-
     tempdir = tempfile.mkdtemp()
     gpg = gnupg.GPG(gnupghome=tempdir)
 
@@ -682,7 +665,6 @@ def command(  # noqa: C901
                         record_namespace_deployments,
                         record_namespace_dns,
                         dns_suffix,
-                        ip,
                         os.path.abspath(deployment_parent_dir),
                         kube_config,
                         image_registry,

stack_orchestrator/deploy/webapp/undeploy_webapp_from_registry.py

@@ -72,12 +72,11 @@ def process_app_removal_request(
     # TODO(telackey): Call the function directly.  The easiest way to build the correct click context is to
     # exec the process, but it would be better to refactor so we could just call down_operation with the
     # necessary parameters
-    main_logger.log("TESTING: Skipping stopping deployment.")
-    # down_command = [sys.argv[0], "deployment", "--dir", deployment_dir, "down"]
-    # if delete_volumes:
-    #     down_command.append("--delete-volumes")
-    # result = subprocess.run(down_command)
-    # result.check_returncode()
+    down_command = [sys.argv[0], "deployment", "--dir", deployment_dir, "down"]
+    if delete_volumes:
+        down_command.append("--delete-volumes")
+    result = subprocess.run(down_command)
+    result.check_returncode()
 
     removal_record = {
         "record": {

stack_orchestrator/deploy/webapp/util.py

@@ -689,7 +689,6 @@ def publish_deployment(
     dns_record,
     dns_lrn,
     deployment_dir,
-    dns_value=None,
     app_deployment_request=None,
     webapp_deployer_record=None,
     logger=None,
@@ -722,8 +721,6 @@ def publish_deployment(
     }
     if app_deployment_request:
         new_dns_record["record"]["request"] = app_deployment_request.id
-    if dns_value:
-        new_dns_record["record"]["value"] = dns_value
 
     if logger:
         logger.log("Publishing DnsRecord.")