Support application removal requests. (#697)

* Support application removal request.

* Git should never prompt when deploying a webapp

stack_orchestrator/data/compose/docker-compose-node-exporter.yml

@@ -0,0 +1,16 @@
+version: '3.8'
+
+services:
+  node-exporter:
+    image: prom/node-exporter:latest
+    restart: unless-stopped
+    command:
+      - '--path.rootfs=/host'
+      - '--collector.systemd'
+      - '--collector.processes'
+    network_mode: host
+    pid: host
+    ports:
+      - 9100
+    volumes:
+      - '/:/host:ro,rslave'

stack_orchestrator/data/compose/docker-compose-prom-server.yml

@@ -43,5 +43,15 @@ services:
     extra_hosts:
       - "host.docker.internal:host-gateway"
 
+  postgres-exporter:
+    image: quay.io/prometheuscommunity/postgres-exporter
+    restart: always
+    volumes:
+      - ../config/monitoring/postgres-exporter.yml:/postgres_exporter.yml
+    ports:
+      - '9187'
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
 volumes:
   prometheus_data:

stack_orchestrator/data/config/monitoring/postgres-exporter.yml

@@ -0,0 +1,8 @@
+auth_modules:
+  foo:
+    type: userpass
+    userpass:
+      username: username
+      password: password
+    options:
+      sslmode: disable

stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml

@@ -11,6 +11,13 @@ scrape_configs:
     static_configs:
       - targets: ['localhost:9090']
 
+  - job_name: node
+    static_configs:
+      # Add node-exporter targets to be monitored below
+      # - targets: ['example-host:9100']
+      #   labels:
+      #     instance: 'my-host'
+
   - job_name: 'blackbox'
     scrape_interval: 10s
     metrics_path: /probe
@@ -40,3 +47,21 @@ scrape_configs:
     static_configs:
       - targets: ['chain-head-exporter:5000']
 
+  - job_name: 'postgres'
+    scrape_interval: 30s
+    scrape_timeout: 30s
+    static_configs:
+      # Add DB targets below
+      # - targets: [example-server:5432]
+        # labels:
+          # instance: 'example-label'
+    metrics_path: /probe
+    params:
+      auth_module: [foo]
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: postgres-exporter:9187

stack_orchestrator/data/stacks/monitoring/README.md

@@ -2,9 +2,12 @@
 
 * Instructions to setup and run a Prometheus server and Grafana dashboard
 * Comes with the following built-in exporters / dashboards:
-  * [Prometheus Blackbox Exporter](https://grafana.com/grafana/dashboards/7587-prometheus-blackbox-exporter/) - for tracking HTTP endpoints
-  * [NodeJS Application Dashboard](https://grafana.com/grafana/dashboards/11159-nodejs-application-dashboard/) - for default NodeJS metrics
   * Chain Head Exporter - for tracking chain heads given external ETH RPC endpoints
+  * Watchers dashboard
+  * [Prometheus Blackbox](https://grafana.com/grafana/dashboards/7587-prometheus-blackbox-exporter/) - for tracking HTTP endpoints
+  * [NodeJS Application Dashboard](https://grafana.com/grafana/dashboards/11159-nodejs-application-dashboard/) - for default NodeJS metrics
+  * [PostgreSQL Database](https://grafana.com/grafana/dashboards/9628-postgresql-database/) - for monitoring Postgres dbs
+  * [Node Exporter Full](https://grafana.com/grafana/dashboards/1860-node-exporter-full/) - for monitoring system metrics
 * See [monitoring-watchers.md](./monitoring-watchers.md) for an example usage of the stack with pre-configured dashboards for watchers
 
 ## Setup
@@ -72,7 +75,20 @@ laconic-so --stack monitoring deploy create --spec-file monitoring-spec.yml --de
       - targets: ['<METRICS_ENDPOINT_HOST>:<METRICS_ENDPOINT_PORT>']
   ```
 
-* Also update the `blackbox` job to add any endpoints to be monitored on the Blackbox dashboard:
+* Node exporter: update the `node` job to add any node-exporter targets to be monitored:
+
+  ```yml
+  ...
+  - job_name: 'node'
+    ...
+    static_configs:
+      # Add node-exporter targets to be monitored below
+      - targets: [example-host:9100]
+        labels:
+          instance: 'my-host'
+  ```
+
+* Blackbox (in-stack exporter): update the `blackbox` job to add any endpoints to be monitored on the Blackbox dashboard:
 
   ```yml
   ...
@@ -85,6 +101,21 @@ laconic-so --stack monitoring deploy create --spec-file monitoring-spec.yml --de
         - <HTTP_ENDPOINT_2>
   ```
 
+* Postgres (in-stack exporter):
+  * Update the `postgres` job to add Postgres db targets to be monitored:
+
+    ```yml
+    ...
+    - job_name: 'postgres'
+      ...
+      static_configs:
+        # Add DB targets below
+        - targets: [example-server:5432]
+          labels:
+            instance: 'example-db'
+    ```
+  * Add database credentials to be used in `auth_modules` in the postgres-exporter config file (`monitoring-deployment/config/monitoring/postgres-exporter.yml`)
+
 Note: Use `host.docker.internal` as host to access ports on the host machine
 
 ### Grafana Config
@@ -96,6 +127,8 @@ Place the dashboard json files in grafana dashboards config directory (`monitori
 Set the following env variables in the deployment env config file (`monitoring-deployment/config.env`):
 
   ```bash
+  # For chain-head exporter
+
   # External ETH RPC endpoint (ethereum)
   # (Optional, default: https://mainnet.infura.io/v3)
   CERC_ETH_RPC_ENDPOINT=

stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md

@@ -1,6 +1,6 @@
 # Monitoring Watchers
 
-Instructions to setup and run monitoring stack with pre-configured watcher dashboards
+Instructions to setup and run monitoring stack with configured Watchers dashboard
 
 ## Create a deployment
 
@@ -35,8 +35,6 @@ laconic-so --stack monitoring deploy create --spec-file monitoring-watchers-spec
 
 ## Configure
 
-### Prometheus Config
-
 Add the following scrape configs to prometheus config file (`monitoring-watchers-deployment/config/monitoring/prometheus/prometheus.yml`) in the deployment folder:
 
   ```yml
@@ -55,27 +53,35 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers
       - targets: ['AZIMUTH_WATCHER_HOST:AZIMUTH_WATCHER_PORT']
         labels:
           instance: 'azimuth'
+          chain: 'ethereum'
       - targets: ['CENSURES_WATCHER_HOST:CENSURES_WATCHER_PORT']
         labels:
           instance: 'censures'
+          chain: 'ethereum'
       - targets: ['CLAIMS_WATCHER_HOST:CLAIMS_WATCHER_PORT']
         labels:
           instance: 'claims'
+          chain: 'ethereum'
       - targets: ['CONDITIONAL_STAR_RELEASE_WATCHER_HOST:CONDITIONAL_STAR_RELEASE_WATCHER_PORT']
         labels:
           instance: 'conditional_star_release'
+          chain: 'ethereum'
       - targets: ['DELEGATED_SENDING_WATCHER_HOST:DELEGATED_SENDING_WATCHER_PORT']
         labels:
-          instance: 'delegated_sending_watcher'
+          instance: 'delegated_sending'
+          chain: 'ethereum'
       - targets: ['ECLIPTIC_WATCHER_HOST:ECLIPTIC_WATCHER_PORT']
         labels:
           instance: 'ecliptic'
+          chain: 'ethereum'
       - targets: ['LINEAR_STAR_WATCHER_HOST:LINEAR_STAR_WATCHER_PORT']
         labels:
           instance: 'linear_star_release'
+          chain: 'ethereum'
       - targets: ['POLLS_WATCHER_HOST:POLLS_WATCHER_PORT']
         labels:
           instance: 'polls'
+          chain: 'ethereum'
 
   - job_name: sushi
     scrape_interval: 20s
@@ -85,18 +91,14 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers
       - targets: ['SUSHISWAP_WATCHER_HOST:SUSHISWAP_WATCHER_PORT']
         labels:
           instance: 'sushiswap'
+          chain: 'filecoin'
       - targets: ['MERKLE_SUSHISWAP_WATCHER_HOST:MERKLE_SUSHISWAP_WATCHER_PORT']
         labels:
           instance: 'merkl_sushiswap'
+          chain: 'filecoin'
   ```
 
-### Grafana Config
-
-In the deployment folder, copy over the pre-configured watcher dashboard JSON files to grafana dashboards config directory:
-
-```bash
-cp -r monitoring-watchers-deployment/config/monitoring/grafana/watcher-dashboards/* monitoring-watchers-deployment/config/monitoring/grafana/dashboards/
-```
+Add scrape config as done above for any additional watcher to add it to the Watchers dashboard.
 
 ### Env
 

stack_orchestrator/data/stacks/node-exporter/README.md

@@ -0,0 +1,17 @@
+# node-exporter
+
+## Start the stack
+
+```bash
+laconic-so --stack node-exporter deploy up
+```
+
+* The host node's metrics can be accessed at `http://localhost:9100/metrics`
+
+## Clean up
+
+Stop the node-exporter running in background:
+
+```bash
+laconic-so --stack node-exporter deploy down
+```

stack_orchestrator/data/stacks/node-exporter/stack.yml

@@ -0,0 +1,6 @@
+version: "0.1"
+name: node-exporter
+repos:
+containers:
+pods:
+  - node-exporter

stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py

@@ -26,7 +26,8 @@ from stack_orchestrator.deploy.webapp import deploy_webapp
 from stack_orchestrator.deploy.webapp.util import (LaconicRegistryClient,
                                                    build_container_image, push_container_image,
                                                    file_hash, deploy_to_k8s, publish_deployment,
-                                                   hostname_for_deployment_request, generate_hostname_for_app)
+                                                   hostname_for_deployment_request, generate_hostname_for_app,
+                                                   match_owner)
 
 
 def process_app_deployment_request(
@@ -57,19 +58,12 @@ def process_app_deployment_request(
     dns_crn = f"{dns_record_namespace}/{fqdn}"
     dns_record = laconic.get_record(dns_crn)
     if dns_record:
-        dns_record_owners = dns_record.owners
-        dns_request_owners = []
-        if dns_record.request:
-            prev_request = laconic.get_record(dns_record.request, require=True)
-            dns_request_owners = prev_request.owners
+        matched_owner = match_owner(app_deployment_request, dns_record)
+        if not matched_owner and dns_record.request:
+            matched_owner = match_owner(app_deployment_request, laconic.get_record(dns_record.request, require=True))
 
-        owner_match = None
-
-        for owner in app_deployment_request.owners:
-            if owner in dns_request_owners or owner in dns_record_owners:
-                owner_match = owner
-        if owner_match:
-            print("Matched DnsRecord ownership to", owner)
+        if matched_owner:
+            print("Matched DnsRecord ownership:", matched_owner)
         else:
             raise Exception("Unable to confirm ownership of DnsRecord %s for request %s" %
                             (dns_record.id, app_deployment_request.id))
@@ -153,7 +147,6 @@ def dump_known_requests(filename, requests):
 
 @click.command()
 @click.option("--kube-config", help="Provide a config file for a k8s deployment")
-@click.option("--kube-config", help="Provide a config file for a k8s deployment")
 @click.option("--laconic-config", help="Provide a config file for laconicd", required=True)
 @click.option("--image-registry", help="Provide a container image registry url for this k8s cluster")
 @click.option("--deployment-parent-dir", help="Create deployment directories beneath this directory", required=True)
@@ -237,9 +230,20 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
         if d.attributes.request:
             deployments_by_request[d.attributes.request] = d
 
+    # Find removal requests.
+    cancellation_requests = {}
+    removal_requests = laconic.app_deployment_removal_requests()
+    for r in removal_requests:
+        if r.attributes.request:
+            cancellation_requests[r.attributes.request] = r
+
     requests_to_execute = []
     for r in requests_by_name.values():
-        if r.id not in deployments_by_request:
+        if r.id in cancellation_requests and match_owner(cancellation_requests[r.id], r):
+            print(f"Found deployment cancellation request for {r.id} at {cancellation_requests[r.id].id}")
+        elif r.id in deployments_by_request:
+            print(f"Found satisfied request for {r.id} at {deployments_by_request[r.id].id}")
+        else:
             if r.id not in previous_requests:
                 print(f"Request {r.id} needs to processed.")
                 requests_to_execute.append(r)
@@ -247,8 +251,6 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
                 print(
                     f"Skipping unsatisfied request {r.id} because we have seen it before."
                 )
-        else:
-            print(f"Found satisfied request {r.id} at {deployments_by_request[r.id].names[0]}")
 
     print("Found %d unsatisfied request(s) to process." % len(requests_to_execute))
 

stack_orchestrator/deploy/webapp/undeploy_webapp_from_registry.py

@@ -0,0 +1,189 @@
+# Copyright ©2023 Vulcanize
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+import json
+import os
+import shutil
+import subprocess
+import sys
+
+import click
+
+from stack_orchestrator.deploy.webapp.util import LaconicRegistryClient, match_owner
+
+
+def process_app_removal_request(ctx,
+                                laconic: LaconicRegistryClient,
+                                app_removal_request,
+                                deployment_parent_dir,
+                                delete_volumes,
+                                delete_names):
+    deployment_record = laconic.get_record(app_removal_request.attributes.deployment, require=True)
+    dns_record = laconic.get_record(deployment_record.attributes.dns, require=True)
+    deployment_dir = os.path.join(deployment_parent_dir, dns_record.attributes.name)
+
+    if not os.path.exists(deployment_dir):
+        raise Exception("Deployment directory %s does not exist." % deployment_dir)
+
+    # Check if the removal request is from the owner of the DnsRecord or deployment record.
+    matched_owner = match_owner(app_removal_request, deployment_record, dns_record)
+
+    # Or of the original deployment request.
+    if not matched_owner and deployment_record.request:
+        matched_owner = match_owner(app_removal_request, laconic.get_record(deployment_record.request, require=True))
+
+    if matched_owner:
+        print("Matched deployment ownership:", matched_owner)
+    else:
+        raise Exception("Unable to confirm ownership of deployment %s for removal request %s" %
+                        (deployment_record.id, app_removal_request.id))
+
+    # TODO(telackey): Call the function directly.  The easiest way to build the correct click context is to
+    # exec the process, but it would be better to refactor so we could just call down_operation with the
+    # necessary parameters
+    down_command = [sys.argv[0], "deployment", "--dir", deployment_dir, "down"]
+    if delete_volumes:
+        down_command.append("--delete-volumes")
+    result = subprocess.run(down_command)
+    result.check_returncode()
+
+    removal_record = {
+        "record": {
+            "type": "ApplicationDeploymentRemovalRecord",
+            "version": "1.0.0",
+            "request": app_removal_request.id,
+            "deployment": deployment_record.id,
+        }
+    }
+    laconic.publish(removal_record)
+
+    if delete_names:
+        if deployment_record.names:
+            for name in deployment_record.names:
+                laconic.delete_name(name)
+
+        if dns_record.names:
+            for name in dns_record.names:
+                laconic.delete_name(name)
+
+    if delete_volumes:
+        shutil.rmtree(deployment_dir)
+
+
+def load_known_requests(filename):
+    if filename and os.path.exists(filename):
+        return json.load(open(filename, "r"))
+    return {}
+
+
+def dump_known_requests(filename, requests):
+    if not filename:
+        return
+    known_requests = load_known_requests(filename)
+    for r in requests:
+        known_requests[r.id] = r.createTime
+    json.dump(known_requests, open(filename, "w"))
+
+
+@click.command()
+@click.option("--laconic-config", help="Provide a config file for laconicd", required=True)
+@click.option("--deployment-parent-dir", help="Create deployment directories beneath this directory", required=True)
+@click.option("--request-id", help="The ApplicationDeploymentRemovalRequest to process")
+@click.option("--discover", help="Discover and process all pending ApplicationDeploymentRemovalRequests",
+              is_flag=True, default=False)
+@click.option("--state-file", help="File to store state about previously seen requests.")
+@click.option("--only-update-state", help="Only update the state file, don't process any requests anything.", is_flag=True)
+@click.option("--delete-names/--preserve-names", help="Delete all names associated with removed deployments.", default=True)
+@click.option("--delete-volumes/--preserve-volumes", default=True, help="delete data volumes")
+@click.option("--dry-run", help="Don't do anything, just report what would be done.", is_flag=True)
+@click.pass_context
+def command(ctx, laconic_config, deployment_parent_dir,
+            request_id, discover, state_file, only_update_state,
+            delete_names, delete_volumes, dry_run):
+    if request_id and discover:
+        print("Cannot specify both --request-id and --discover", file=sys.stderr)
+        sys.exit(2)
+
+    if not request_id and not discover:
+        print("Must specify either --request-id or --discover", file=sys.stderr)
+        sys.exit(2)
+
+    if only_update_state and not state_file:
+        print("--only-update-state requires --state-file", file=sys.stderr)
+        sys.exit(2)
+
+    laconic = LaconicRegistryClient(laconic_config)
+
+    # Find deployment removal requests.
+    # single request
+    if request_id:
+        requests = [laconic.get_record(request_id, require=True)]
+        # TODO: assert record type
+    # all requests
+    elif discover:
+        requests = laconic.app_deployment_removal_requests()
+
+    if only_update_state:
+        if not dry_run:
+            dump_known_requests(state_file, requests)
+        return
+
+    previous_requests = load_known_requests(state_file)
+    requests.sort(key=lambda r: r.createTime)
+
+    # Find deployments.
+    deployments = {}
+    for d in laconic.app_deployments(all=True):
+        deployments[d.id] = d
+
+    # Find removal requests.
+    removals_by_deployment = {}
+    removals_by_request = {}
+    for r in laconic.app_deployment_removals():
+        if r.attributes.deployment:
+            # TODO: should we handle CRNs?
+            removals_by_deployment[r.attributes.deployment] = r
+
+    requests_to_execute = []
+    for r in requests:
+        if not r.attributes.deployment:
+            print(f"Skipping removal request {r.id} since it was a cancellation.")
+        elif r.id in removals_by_request:
+            print(f"Found satisfied request for {r.id} at {removals_by_request[r.id].id}")
+        elif r.attributes.deployment in removals_by_deployment:
+            print(
+                f"Found removal record for indicated deployment {r.attributes.deployment} at "
+                f"{removals_by_deployment[r.attributes.deployment].id}")
+        else:
+            if r.id not in previous_requests:
+                print(f"Request {r.id} needs to processed.")
+                requests_to_execute.append(r)
+            else:
+                print(f"Skipping unsatisfied request {r.id} because we have seen it before.")
+
+    print("Found %d unsatisfied request(s) to process." % len(requests_to_execute))
+
+    if not dry_run:
+        for r in requests_to_execute:
+            try:
+                process_app_removal_request(
+                    ctx,
+                    laconic,
+                    r,
+                    os.path.abspath(deployment_parent_dir),
+                    delete_volumes,
+                    delete_names
+                )
+            finally:
+                dump_known_requests(state_file, [r])

stack_orchestrator/deploy/webapp/util.py

@@ -49,6 +49,14 @@ def cmd(*vargs):
         raise err
 
 
+def match_owner(recordA, *records):
+    for owner in recordA.owners:
+        for otherRecord in records:
+            if owner in otherRecord.owners:
+                return owner
+    return None
+
+
 class LaconicRegistryClient:
     def __init__(self, config_file):
         self.config_file = config_file
@@ -146,11 +154,17 @@ class LaconicRegistryClient:
             raise Exception("Cannot locate record:", name_or_id)
         return None
 
-    def app_deployment_requests(self):
-        return self.list_records({"type": "ApplicationDeploymentRequest"}, True)
+    def app_deployment_requests(self, all=True):
+        return self.list_records({"type": "ApplicationDeploymentRequest"}, all)
 
-    def app_deployments(self):
-        return self.list_records({"type": "ApplicationDeploymentRecord"})
+    def app_deployments(self, all=True):
+        return self.list_records({"type": "ApplicationDeploymentRecord"}, all)
+
+    def app_deployment_removal_requests(self, all=True):
+        return self.list_records({"type": "ApplicationDeploymentRemovalRequest"}, all)
+
+    def app_deployment_removals(self, all=True):
+        return self.list_records({"type": "ApplicationDeploymentRemovalRecord"}, all)
 
     def publish(self, record, names=[]):
         tmpdir = tempfile.mkdtemp()
@@ -165,11 +179,17 @@ class LaconicRegistryClient:
                 cmd("laconic", "-c", self.config_file, "cns", "record", "publish", "--filename", record_fname)
             )["id"]
             for name in names:
-                cmd("laconic", "-c", self.config_file, "cns", "name", "set", name, new_record_id)
+                self.set_name(name, new_record_id)
             return new_record_id
         finally:
             cmd("rm", "-rf", tmpdir)
 
+    def set_name(self, name, record_id):
+        cmd("laconic", "-c", self.config_file, "cns", "name", "set", name, record_id)
+
+    def delete_name(self, name):
+        cmd("laconic", "-c", self.config_file, "cns", "name", "delete", name)
+
 
 def file_hash(filename):
     return hashlib.sha1(open(filename).read().encode()).hexdigest()
@@ -187,9 +207,11 @@ def build_container_image(app_record, tag, extra_build_args=[]):
         print(f"Cloning repository {repo} to {clone_dir} ...")
         if ref:
             # TODO: Determing branch or hash, and use depth 1 if we can.
-            result = subprocess.run(["git", "clone", repo, clone_dir])
-            result.check_returncode()
-            subprocess.check_call(["git", "checkout", ref], cwd=clone_dir)
+            git_env = dict(os.environ.copy())
+            # Never prompt
+            git_env["GIT_TERMINAL_PROMPT"] = "0"
+            subprocess.check_call(["git", "clone", repo, clone_dir], env=git_env)
+            subprocess.check_call(["git", "checkout", ref], cwd=clone_dir, env=git_env)
         else:
             result = subprocess.run(["git", "clone", "--depth", "1", repo, clone_dir])
             result.check_returncode()

stack_orchestrator/main.py

@@ -20,7 +20,10 @@ from stack_orchestrator.repos import setup_repositories
 from stack_orchestrator.build import build_containers
 from stack_orchestrator.build import build_npms
 from stack_orchestrator.build import build_webapp
-from stack_orchestrator.deploy.webapp import run_webapp, deploy_webapp, deploy_webapp_from_registry
+from stack_orchestrator.deploy.webapp import (run_webapp,
+                                              deploy_webapp,
+                                              deploy_webapp_from_registry,
+                                              undeploy_webapp_from_registry)
 from stack_orchestrator.deploy import deploy
 from stack_orchestrator import version
 from stack_orchestrator.deploy import deployment
@@ -54,6 +57,7 @@ cli.add_command(build_webapp.command, "build-webapp")
 cli.add_command(run_webapp.command, "run-webapp")
 cli.add_command(deploy_webapp.command, "deploy-webapp")
 cli.add_command(deploy_webapp_from_registry.command, "deploy-webapp-from-registry")
+cli.add_command(undeploy_webapp_from_registry.command, "undeploy-webapp-from-registry")
 cli.add_command(deploy.command, "deploy")  # deploy is an alias for deploy-system
 cli.add_command(deploy.command, "deploy-system")
 cli.add_command(deployment.command, "deployment")