Add alert rule for steer protocol watcher

stack_orchestrator/data/compose/docker-compose-graph-node.yml

@@ -1,5 +1,6 @@
 services:
   graph-node:
+    restart: unless-stopped
     image: cerc/graph-node:local
     depends_on:
       db:
@@ -40,6 +41,7 @@ services:
       retries: 10
       start_period: 3s
   ipfs:
+    restart: unless-stopped
     image: ipfs/kubo:master-2023-02-20-714a968
     volumes:
       - ipfs-import:/import
@@ -49,6 +51,7 @@ services:
       - "4001"
       - "5001"
   db:
+    restart: unless-stopped
     image: postgres:14-alpine
     volumes:
       - db-data:/var/lib/postgresql/data

stack_orchestrator/data/config/monitoring/grafana/dashboards/watcher-dashboard.json

@@ -2866,6 +2866,19 @@
               "legendFormat": "{{field}}",
               "range": true,
               "refId": "E"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "PBFA97CFB590B2093"
+              },
+              "editorMode": "code",
+              "expr": "watcher_config_info{job=~\"$job\", instance=~\"$watcher\", category=\"jobqueue\", field=\"block_processing_offset\"}",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "{{field}}",
+              "range": true,
+              "refId": "F"
             }
           ],
           "title": "Watcher config (job queue)",
@@ -3284,6 +3297,6 @@
   "timepicker": {},
   "timezone": "",
   "title": "Watchers",
-  "version": 5,
+  "version": 6,
   "weekStart": ""
 }

stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml

@@ -49,6 +49,10 @@ scrape_configs:
         labels:
           instance: 'external'
           chain: 'ethereum'
+      - targets: ['sepolia-chain-head-exporter:5000']
+        labels:
+          instance: 'external'
+          chain: 'sepolia'
       - targets: ['filecoin-chain-head-exporter:5000']
         labels:
           instance: 'external'

stack_orchestrator/data/config/monitoring/subgraph-alert-rules.yml

@@ -19,7 +19,7 @@ groups:
                   type: prometheus
                   uid: PBFA97CFB590B2093
               editorMode: code
-              expr: ethereum_chain_head_number - on(network) group_right deployment_head{deployment=~"REPLACE_WITH_SUBGRAPH_IDS"}
+              expr: label_replace(latest_block_number{instance="external"}, "network", "$1", "chain", "(.*)") - on(network) group_right deployment_head{deployment=~"REPLACE_WITH_SUBGRAPH_IDS"}
               instant: true
               intervalMs: 1000
               legendFormat: __auto

stack_orchestrator/data/config/monitoring/watcher-alert-rules.yml

@@ -695,82 +695,6 @@ groups:
         annotations:
           summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
         isPaused: false
-      - uid: merkl_sushiswap_diff_external
-        title: merkl_sushiswap_watcher_head_tracking
-        condition: condition
-        data:
-          - refId: diff
-            relativeTimeRange:
-              from: 600
-              to: 0
-            datasourceUid: PBFA97CFB590B2093
-            model:
-              datasource:
-                type: prometheus
-                uid: PBFA97CFB590B2093
-              disableTextWrap: false
-              editorMode: code
-              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="sushi", instance="merkl_sushiswap", kind="latest_indexed"}
-              fullMetaSearch: false
-              includeNullMetadata: true
-              instant: true
-              intervalMs: 1000
-              legendFormat: __auto
-              maxDataPoints: 43200
-              range: false
-              refId: diff
-              useBackend: false
-          - refId: latest_external
-            relativeTimeRange:
-              from: 600
-              to: 0
-            datasourceUid: PBFA97CFB590B2093
-            model:
-              datasource:
-                type: prometheus
-                uid: PBFA97CFB590B2093
-              editorMode: code
-              expr: latest_block_number{chain="filecoin"}
-              hide: false
-              instant: true
-              legendFormat: __auto
-              range: false
-              refId: latest_external
-          - refId: condition
-            relativeTimeRange:
-              from: 600
-              to: 0
-            datasourceUid: __expr__
-            model:
-              conditions:
-                - evaluator:
-                    params:
-                      - 0
-                      - 0
-                    type: gt
-                  operator:
-                    type: and
-                  query:
-                    params: []
-                  reducer:
-                    params: []
-                    type: avg
-                  type: query
-              datasource:
-                name: Expression
-                type: __expr__
-                uid: __expr__
-              expression: ${diff} >= 16
-              intervalMs: 1000
-              maxDataPoints: 43200
-              refId: condition
-              type: math
-        noDataState: Alerting
-        execErrState: Alerting
-        for: 15m
-        annotations:
-          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
-        isPaused: false
 
       # Ajna
       - uid: ajna_diff_external
@@ -849,7 +773,7 @@ groups:
         annotations:
           summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
         isPaused: false
-      
+
       # Secured Finance
       - uid: secured_finance_diff_external
         title: secured_finance_watcher_head_tracking
@@ -927,3 +851,159 @@ groups:
         annotations:
           summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
         isPaused: false
+
+      # Borg
+      - uid: borg_diff_external
+        title: borg_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="borg", instance="borg", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="sepolia"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+
+      # Steer Protocol
+      - uid: steer_protocol_diff_external
+        title: steer_protocol_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="steer-protocol", instance="steer-protocol", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="filecoin"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false

stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py

@@ -54,7 +54,6 @@ def process_app_deployment_request(
     deployment_record_namespace,
     dns_record_namespace,
     default_dns_suffix,
-    dns_value,
     deployment_parent_dir,
     kube_config,
     image_registry,
@@ -252,7 +251,6 @@ def process_app_deployment_request(
         dns_record,
         dns_lrn,
         deployment_dir,
-        dns_value,
         app_deployment_request,
         webapp_deployer_record,
         logger,
@@ -306,7 +304,6 @@ def dump_known_requests(filename, requests, status="SEEN"):
     help="How to handle requests with an FQDN: prohibit, allow, preexisting",
     default="prohibit",
 )
-@click.option("--ip", help="IP address of the k8s deployment (to be set in DNS record)", default=None)
 @click.option("--record-namespace-dns", help="eg, lrn://laconic/dns", required=True)
 @click.option(
     "--record-namespace-deployments",
@@ -342,17 +339,6 @@ def dump_known_requests(filename, requests, status="SEEN"):
     help="Requests must have a minimum payment to be processed (in alnt)",
     default=0,
 )
-@click.option(
-    "--atom-payment-address",
-    help="Cosmos ATOM address to receive payments",
-    default=None,
-)
-@click.option(
-    "--min-atom-payment",
-    help="Minimum required ATOM payment amount",
-    default=1,
-    type=float,
-)
 @click.option("--lrn", help="The LRN of this deployer.", required=True)
 @click.option(
     "--all-requests",
@@ -395,7 +381,6 @@ def command(  # noqa: C901
     only_update_state,
     dns_suffix,
     fqdn_policy,
-    ip,
     record_namespace_dns,
     record_namespace_deployments,
     dry_run,
@@ -405,8 +390,6 @@ def command(  # noqa: C901
     recreate_on_deploy,
     log_dir,
     min_required_payment,
-    atom_payment_address,
-    min_atom_payment,
     lrn,
     config_upload_dir,
     private_key_file,
@@ -446,13 +429,6 @@ def command(  # noqa: C901
         )
         sys.exit(2)
 
-    if fqdn_policy == "allow" and not ip:
-        print(
-            "--ip is required with 'allow' fqdn-policy",
-            file=sys.stderr,
-        )
-        sys.exit(2)
-
     tempdir = tempfile.mkdtemp()
     gpg = gnupg.GPG(gnupghome=tempdir)
 
@@ -643,8 +619,6 @@ def command(  # noqa: C901
                     payment_address,
                     min_required_payment,
                     main_logger,
-                    atom_payment_address,
-                    min_atom_payment,
                 ):
                     main_logger.log(f"{r.id}: Payment confirmed.")
                     requests_to_execute.append(r)
@@ -691,7 +665,6 @@ def command(  # noqa: C901
                         record_namespace_deployments,
                         record_namespace_dns,
                         dns_suffix,
-                        ip,
                         os.path.abspath(deployment_parent_dir),
                         kube_config,
                         image_registry,

stack_orchestrator/deploy/webapp/publish_webapp_deployer.py

@@ -46,17 +46,6 @@ from stack_orchestrator.deploy.webapp.util import LaconicRegistryClient
     help="List the minimum required payment (in alnt) to process a deployment request.",
     default=0,
 )
-@click.option(
-    "--atom-payment-address",
-    help="The Cosmos ATOM address to which payments should be made.",
-    default=None,
-)
-@click.option(
-    "--min-atom-payment",
-    help="List the minimum required payment (in ATOM) to process a deployment request.",
-    default=1,
-    type=float,
-)
 @click.option(
     "--dry-run",
     help="Don't publish anything, just report what would be done.",
@@ -71,8 +60,6 @@ def command(  # noqa: C901
     lrn,
     payment_address,
     min_required_payment,
-    atom_payment_address,
-    min_atom_payment,
     dry_run,
 ):
     laconic = LaconicRegistryClient(laconic_config)
@@ -96,10 +83,6 @@ def command(  # noqa: C901
         webapp_deployer_record["record"][
             "minimumPayment"
         ] = f"{min_required_payment}alnt"
-        
-    if atom_payment_address:
-        webapp_deployer_record["record"]["atomPaymentAddress"] = atom_payment_address
-        webapp_deployer_record["record"]["minimumAtomPayment"] = min_atom_payment
 
     if dry_run:
         yaml.dump(webapp_deployer_record, sys.stdout)

stack_orchestrator/deploy/webapp/util.py

@@ -689,7 +689,6 @@ def publish_deployment(
     dns_record,
     dns_lrn,
     deployment_dir,
-    dns_value=None,
     app_deployment_request=None,
     webapp_deployer_record=None,
     logger=None,
@@ -722,8 +721,6 @@ def publish_deployment(
     }
     if app_deployment_request:
         new_dns_record["record"]["request"] = app_deployment_request.id
-    if dns_value:
-        new_dns_record["record"]["value"] = dns_value
 
     if logger:
         logger.log("Publishing DnsRecord.")
@@ -801,7 +798,7 @@ def skip_by_tag(r, include_tags, exclude_tags):
     return False
 
 
-def confirm_payment(laconic: LaconicRegistryClient, record, payment_address, min_amount, logger, atom_payment_address=None, atom_min_amount=None):
+def confirm_payment(laconic: LaconicRegistryClient, record, payment_address, min_amount, logger):
     req_owner = laconic.get_owner(record)
     if req_owner == payment_address:
         # No need to confirm payment if the sender and recipient are the same account.
@@ -811,114 +808,67 @@ def confirm_payment(laconic: LaconicRegistryClient, record, payment_address, min
         logger.log(f"{record.id}: no payment tx info")
         return False
 
-    # Try to verify as a laconic payment first
     tx = laconic.get_tx(record.attributes.payment)
-    if tx:
-        if tx.code != 0:
-            logger.log(
-                f"{record.id}: payment tx {tx.hash} was not successful - code: {tx.code}, log: {tx.log}"
-            )
-            return False
+    if not tx:
+        logger.log(f"{record.id}: cannot locate payment tx")
+        return False
 
-        if tx.sender != req_owner:
-            logger.log(
-                f"{record.id}: payment sender {tx.sender} in tx {tx.hash} does not match deployment "
-                f"request owner {req_owner}"
-            )
-            return False
-
-        if tx.recipient != payment_address:
-            logger.log(
-                f"{record.id}: payment recipient {tx.recipient} in tx {tx.hash} does not match {payment_address}"
-            )
-            return False
-
-        pay_denom = "".join([i for i in tx.amount if not i.isdigit()])
-        if pay_denom != "alnt":
-            logger.log(
-                f"{record.id}: {pay_denom} in tx {tx.hash} is not an expected payment denomination"
-            )
-            return False
-
-        pay_amount = int("".join([i for i in tx.amount if i.isdigit()]))
-        if pay_amount < min_amount:
-            logger.log(
-                f"{record.id}: payment amount {tx.amount} is less than minimum {min_amount}"
-            )
-            return False
-
-        # Check if the payment was already used on a deployment
-        used = laconic.app_deployments(
-            {"deployer": record.attributes.deployer, "payment": tx.hash}, all=True
+    if tx.code != 0:
+        logger.log(
+            f"{record.id}: payment tx {tx.hash} was not successful - code: {tx.code}, log: {tx.log}"
         )
-        if len(used):
-            # Fetch the app name from request record
-            used_request = laconic.get_record(used[0].attributes.request, require=True)
+        return False
 
-            # Check that payment was used for deployment of same application
-            if record.attributes.application != used_request.attributes.application:
-                logger.log(f"{record.id}: payment {tx.hash} already used on a different application deployment {used}")
-                return False
-
-        used = laconic.app_deployment_removals(
-            {"deployer": record.attributes.deployer, "payment": tx.hash}, all=True
+    if tx.sender != req_owner:
+        logger.log(
+            f"{record.id}: payment sender {tx.sender} in tx {tx.hash} does not match deployment "
+            f"request owner {req_owner}"
         )
-        if len(used):
-            logger.log(
-                f"{record.id}: payment {tx.hash} already used on deployment removal {used}"
-            )
+        return False
+
+    if tx.recipient != payment_address:
+        logger.log(
+            f"{record.id}: payment recipient {tx.recipient} in tx {tx.hash} does not match {payment_address}"
+        )
+        return False
+
+    pay_denom = "".join([i for i in tx.amount if not i.isdigit()])
+    if pay_denom != "alnt":
+        logger.log(
+            f"{record.id}: {pay_denom} in tx {tx.hash} is not an expected payment denomination"
+        )
+        return False
+
+    pay_amount = int("".join([i for i in tx.amount if i.isdigit()]))
+    if pay_amount < min_amount:
+        logger.log(
+            f"{record.id}: payment amount {tx.amount} is less than minimum {min_amount}"
+        )
+        return False
+
+    # Check if the payment was already used on a deployment
+    used = laconic.app_deployments(
+        {"deployer": record.attributes.deployer, "payment": tx.hash}, all=True
+    )
+    if len(used):
+        # Fetch the app name from request record
+        used_request = laconic.get_record(used[0].attributes.request, require=True)
+
+        # Check that payment was used for deployment of same application
+        if record.attributes.application != used_request.attributes.application:
+            logger.log(f"{record.id}: payment {tx.hash} already used on a different application deployment {used}")
             return False
 
-        return True
-    
-    # If we get here, the transaction hash wasn't found in the laconic testnet
-    # Let's check if it's a valid Cosmos ATOM payment if configuration is available
-    if atom_payment_address:
-        logger.log(f"{record.id}: checking if payment is a valid Cosmos ATOM transaction")
-        
-        try:
-            import requests
-            
-            # Use the webapp-deployment-status-api to verify the ATOM payment
-            deployer_record = laconic.get_record(record.attributes.deployer)
-            if not deployer_record or not deployer_record.attributes.apiUrl:
-                logger.log(f"{record.id}: cannot find deployer API URL to verify ATOM payment")
-                return False
-                
-            api_url = deployer_record.attributes.apiUrl
-            verify_url = f"{api_url}/verify/atom-payment"
-            
-            # Make a request to the API to verify the ATOM payment
-            # Pass markAsUsed=true to prevent this transaction from being used again
-            response = requests.post(
-                verify_url,
-                json={
-                    "txHash": record.attributes.payment,
-                    "minAmount": atom_min_amount,
-                    "markAsUsed": True
-                },
-                timeout=10
-            )
-            
-            if response.status_code != 200:
-                logger.log(f"{record.id}: ATOM payment verification API request failed with status {response.status_code}")
-                return False
-                
-            result = response.json()
-            if not result.get("valid", False):
-                logger.log(f"{record.id}: ATOM payment verification failed: {result.get('reason', 'unknown reason')}")
-                return False
-                
-            # Payment is valid
-            logger.log(f"{record.id}: ATOM payment verified successfully, amount: {result.get('amount')} ATOM")
-            return True
-            
-        except Exception as e:
-            logger.log(f"{record.id}: error verifying ATOM payment: {str(e)}")
-            return False
-    
-    logger.log(f"{record.id}: payment tx {record.attributes.payment} not found in laconic testnet and ATOM payment verification not configured")
-    return False
+    used = laconic.app_deployment_removals(
+        {"deployer": record.attributes.deployer, "payment": tx.hash}, all=True
+    )
+    if len(used):
+        logger.log(
+            f"{record.id}: payment {tx.hash} already used on deployment removal {used}"
+        )
+        return False
+
+    return True
 
 
 def confirm_auction(laconic: LaconicRegistryClient, record, deployer_lrn, payment_address, logger):