Add alert rule for steer protocol watcher

Part of [Service provider auctions for web deployments](https://www.notion.so/Service-provider-auctions-for-web-deployments-104a6b22d47280dbad51d28aa3a91d75)

Reviewed-on: cerc-io/stack-orchestrator#961
Reviewed-by: ashwin <ashwin@noreply.git.vdb.to>
Co-authored-by: Prathamesh Musale <prathamesh.musale0@gmail.com>
Co-committed-by: Prathamesh Musale <prathamesh.musale0@gmail.com>

stack_orchestrator/data/compose/docker-compose-graph-node.yml

@@ -1,5 +1,6 @@
 services:
   graph-node:
+    restart: unless-stopped
     image: cerc/graph-node:local
     depends_on:
       db:
@@ -40,6 +41,7 @@ services:
       retries: 10
       start_period: 3s
   ipfs:
+    restart: unless-stopped
     image: ipfs/kubo:master-2023-02-20-714a968
     volumes:
       - ipfs-import:/import
@@ -49,6 +51,7 @@ services:
       - "4001"
       - "5001"
   db:
+    restart: unless-stopped
     image: postgres:14-alpine
     volumes:
       - db-data:/var/lib/postgresql/data

stack_orchestrator/data/config/monitoring/grafana/dashboards/watcher-dashboard.json

@@ -2866,6 +2866,19 @@
               "legendFormat": "{{field}}",
               "range": true,
               "refId": "E"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "PBFA97CFB590B2093"
+              },
+              "editorMode": "code",
+              "expr": "watcher_config_info{job=~\"$job\", instance=~\"$watcher\", category=\"jobqueue\", field=\"block_processing_offset\"}",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "{{field}}",
+              "range": true,
+              "refId": "F"
             }
           ],
           "title": "Watcher config (job queue)",
@@ -3284,6 +3297,6 @@
   "timepicker": {},
   "timezone": "",
   "title": "Watchers",
-  "version": 5,
+  "version": 6,
   "weekStart": ""
 }

stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml

@@ -49,6 +49,10 @@ scrape_configs:
         labels:
           instance: 'external'
           chain: 'ethereum'
+      - targets: ['sepolia-chain-head-exporter:5000']
+        labels:
+          instance: 'external'
+          chain: 'sepolia'
       - targets: ['filecoin-chain-head-exporter:5000']
         labels:
           instance: 'external'

stack_orchestrator/data/config/monitoring/subgraph-alert-rules.yml

@@ -19,7 +19,7 @@ groups:
                   type: prometheus
                   uid: PBFA97CFB590B2093
               editorMode: code
-              expr: ethereum_chain_head_number - on(network) group_right deployment_head{deployment=~"REPLACE_WITH_SUBGRAPH_IDS"}
+              expr: label_replace(latest_block_number{instance="external"}, "network", "$1", "chain", "(.*)") - on(network) group_right deployment_head{deployment=~"REPLACE_WITH_SUBGRAPH_IDS"}
               instant: true
               intervalMs: 1000
               legendFormat: __auto

stack_orchestrator/data/config/monitoring/watcher-alert-rules.yml

@@ -695,82 +695,6 @@ groups:
         annotations:
           summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
         isPaused: false
-      - uid: merkl_sushiswap_diff_external
-        title: merkl_sushiswap_watcher_head_tracking
-        condition: condition
-        data:
-          - refId: diff
-            relativeTimeRange:
-              from: 600
-              to: 0
-            datasourceUid: PBFA97CFB590B2093
-            model:
-              datasource:
-                type: prometheus
-                uid: PBFA97CFB590B2093
-              disableTextWrap: false
-              editorMode: code
-              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="sushi", instance="merkl_sushiswap", kind="latest_indexed"}
-              fullMetaSearch: false
-              includeNullMetadata: true
-              instant: true
-              intervalMs: 1000
-              legendFormat: __auto
-              maxDataPoints: 43200
-              range: false
-              refId: diff
-              useBackend: false
-          - refId: latest_external
-            relativeTimeRange:
-              from: 600
-              to: 0
-            datasourceUid: PBFA97CFB590B2093
-            model:
-              datasource:
-                type: prometheus
-                uid: PBFA97CFB590B2093
-              editorMode: code
-              expr: latest_block_number{chain="filecoin"}
-              hide: false
-              instant: true
-              legendFormat: __auto
-              range: false
-              refId: latest_external
-          - refId: condition
-            relativeTimeRange:
-              from: 600
-              to: 0
-            datasourceUid: __expr__
-            model:
-              conditions:
-                - evaluator:
-                    params:
-                      - 0
-                      - 0
-                    type: gt
-                  operator:
-                    type: and
-                  query:
-                    params: []
-                  reducer:
-                    params: []
-                    type: avg
-                  type: query
-              datasource:
-                name: Expression
-                type: __expr__
-                uid: __expr__
-              expression: ${diff} >= 16
-              intervalMs: 1000
-              maxDataPoints: 43200
-              refId: condition
-              type: math
-        noDataState: Alerting
-        execErrState: Alerting
-        for: 15m
-        annotations:
-          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
-        isPaused: false
 
       # Ajna
       - uid: ajna_diff_external
@@ -849,7 +773,7 @@ groups:
         annotations:
           summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
         isPaused: false
-      
+
       # Secured Finance
       - uid: secured_finance_diff_external
         title: secured_finance_watcher_head_tracking
@@ -927,3 +851,159 @@ groups:
         annotations:
           summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
         isPaused: false
+
+      # Borg
+      - uid: borg_diff_external
+        title: borg_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="borg", instance="borg", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="sepolia"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+
+      # Steer Protocol
+      - uid: steer_protocol_diff_external
+        title: steer_protocol_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="steer-protocol", instance="steer-protocol", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="filecoin"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false

stack_orchestrator/deploy/webapp/registry_mutex.py

@@ -1,8 +1,59 @@
-import fcntl
 from functools import wraps
+import os
+import time
 
 # Define default file path for the lock
 DEFAULT_LOCK_FILE_PATH = "/tmp/registry_mutex_lock_file"
+LOCK_TIMEOUT = 30
+LOCK_RETRY_INTERVAL = 3
+
+
+def acquire_lock(client, lock_file_path, timeout):
+    # Lock alreay acquired by the current client
+    if client.mutex_lock_acquired:
+        return
+
+    while True:
+        try:
+            # Check if lock file exists and is potentially stale
+            if os.path.exists(lock_file_path):
+                with open(lock_file_path, 'r') as lock_file:
+                    timestamp = float(lock_file.read().strip())
+
+                # If lock is stale, remove the lock file
+                if time.time() - timestamp > timeout:
+                    print(f"Stale lock detected, removing lock file {lock_file_path}")
+                    os.remove(lock_file_path)
+                else:
+                    print(f"Lock file {lock_file_path} exists and is recent, waiting...")
+                    time.sleep(LOCK_RETRY_INTERVAL)
+                    continue
+
+            # Try to create a new lock file with the current timestamp
+            fd = os.open(lock_file_path, os.O_CREAT | os.O_EXCL | os.O_RDWR)
+            with os.fdopen(fd, 'w') as lock_file:
+                lock_file.write(str(time.time()))
+
+            client.mutex_lock_acquired = True
+            print(f"Registry lock acquired, {lock_file_path}")
+
+            # Lock successfully acquired
+            return
+
+        except FileExistsError:
+            print(f"Lock file {lock_file_path} exists, waiting...")
+            time.sleep(LOCK_RETRY_INTERVAL)
+
+
+def release_lock(client, lock_file_path):
+    try:
+        os.remove(lock_file_path)
+
+        client.mutex_lock_acquired = False
+        print(f"Registry lock released, {lock_file_path}")
+    except FileNotFoundError:
+        # Lock file already removed
+        pass
 
 
 def registry_mutex():
@@ -13,18 +64,13 @@ def registry_mutex():
             if self.mutex_lock_file:
                 lock_file_path = self.mutex_lock_file
 
-            with open(lock_file_path, 'w') as lock_file:
-                try:
-                    # Try to acquire the lock
-                    fcntl.flock(lock_file, fcntl.LOCK_EX)
-
-                    # Call the actual function
-                    result = func(self, *args, **kwargs)
-                finally:
-                    # Always release the lock
-                    fcntl.flock(lock_file, fcntl.LOCK_UN)
-
-            return result
+            # Acquire the lock before running the function
+            acquire_lock(self, lock_file_path, LOCK_TIMEOUT)
+            try:
+                return func(self, *args, **kwargs)
+            finally:
+                # Release the lock after the function completes
+                release_lock(self, lock_file_path)
 
         return wrapper
 

stack_orchestrator/deploy/webapp/util.py

@@ -117,7 +117,6 @@ class LaconicRegistryClient:
     def __init__(self, config_file, log_file=None, mutex_lock_file=None):
         self.config_file = config_file
         self.log_file = log_file
-        self.mutex_lock_file = mutex_lock_file
         self.cache = AttrDict(
             {
                 "name_or_id": {},
@@ -126,6 +125,9 @@ class LaconicRegistryClient:
             }
         )
 
+        self.mutex_lock_file = mutex_lock_file
+        self.mutex_lock_acquired = False
+
     def whoami(self, refresh=False):
         if not refresh and "whoami" in self.cache:
             return self.cache["whoami"]
@@ -844,16 +846,21 @@ def confirm_payment(laconic: LaconicRegistryClient, record, payment_address, min
         )
         return False
 
-    # Check if the payment was already used on a
+    # Check if the payment was already used on a deployment
     used = laconic.app_deployments(
-        {"deployer": payment_address, "payment": tx.hash}, all=True
+        {"deployer": record.attributes.deployer, "payment": tx.hash}, all=True
     )
     if len(used):
-        logger.log(f"{record.id}: payment {tx.hash} already used on deployment {used}")
-        return False
+        # Fetch the app name from request record
+        used_request = laconic.get_record(used[0].attributes.request, require=True)
+
+        # Check that payment was used for deployment of same application
+        if record.attributes.application != used_request.attributes.application:
+            logger.log(f"{record.id}: payment {tx.hash} already used on a different application deployment {used}")
+            return False
 
     used = laconic.app_deployment_removals(
-        {"deployer": payment_address, "payment": tx.hash}, all=True
+        {"deployer": record.attributes.deployer, "payment": tx.hash}, all=True
     )
     if len(used):
         logger.log(