Use classic condition for checking http status code

Add http status code check to alert rule
Add a custom label to blackbox metrics and update instructions
2024-04-22 19:33:22 +05:30 · 2024-04-15 18:42:26 +05:30 · 2024-04-15 17:51:17 +05:30 · 2024-04-15 17:02:50 +05:30 · 2024-04-11 07:59:36 +00:00 · 2024-04-11 03:21:36 +00:00
34 changed files with 223 additions and 637 deletions
@@ -1,58 +0,0 @@
-name: External Stack Test
-
-on:
-  push:
-    branches: '*'
-    paths:
-      - '!**'
-      - '.gitea/workflows/triggers/test-external-stack'
-      - '.gitea/workflows/test-external-stack.yml'
-      - 'tests/external-stack/run-test.sh'
-  schedule: # Note: coordinate with other tests to not overload runners at the same time of day
-    - cron: '8 19 * * *'
-
-jobs:
-  test:
-    name: "Run external stack test suite"
-    runs-on: ubuntu-latest
-    steps:
-      - name: "Clone project repository"
-        uses: actions/checkout@v3
-      # At present the stock setup-python action fails on Linux/aarch64
-      # Conditional steps below workaroud this by using deadsnakes for that case only
-      - name: "Install Python for ARM on Linux"
-        if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
-        uses: deadsnakes/action@v3.0.1
-        with:
-          python-version: '3.8'
-      - name: "Install Python cases other than ARM on Linux"
-        if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.8'
-      - name: "Print Python version"
-        run: python3 --version
-      - name: "Install shiv"
-        run: pip install shiv
-      - name: "Generate build version file"
-        run: ./scripts/create_build_tag_file.sh
-      - name: "Build local shiv package"
-        run: ./scripts/build_shiv_package.sh
-      - name: "Run external stack tests"
-        run: ./tests/external-stack/run-test.sh
-      - name: Notify Vulcanize Slack on CI failure
-        if: ${{ always() && github.ref_name == 'main' }}
-        uses: ravsamhq/notify-slack-action@v2
-        with:
-          status: ${{ job.status }}
-          notify_when: 'failure'
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
-      - name: Notify DeepStack Slack on CI failure
-        if: ${{ always() && github.ref_name == 'main' }}
-        uses: ravsamhq/notify-slack-action@v2
-        with:
-          status: ${{ job.status }}
-          notify_when: 'failure'
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
@@ -1,2 +0,0 @@
-Change this file to trigger running the external-stack CI job
-trigger
@@ -71,7 +71,7 @@ def process_container(build_context: BuildContext) -> bool:

    # Check if this is in an external stack
    if stack_is_external(build_context.stack):
-        container_parent_dir = Path(build_context.stack).parent.parent.joinpath("container-build")
+        container_parent_dir = Path(build_context.stack).joinpath("container-build")
        temp_build_dir = container_parent_dir.joinpath(build_context.container.replace("/", "-"))
        temp_build_script_filename = temp_build_dir.joinpath("build.sh")
        # Now check if the container exists in the external stack.
@@ -5,7 +5,7 @@ services:
    hostname: lotus-miner
    env_file:
      - ../config/fixturenet-lotus/lotus-env.env
-    image: cerc/lotus-fixturenet:local
+    image: cerc/lotus:local
    volumes:
      - ../config/fixturenet-lotus/setup-miner.sh:/docker-entrypoint-scripts.d/setup-miner.sh
      - ../config/fixturenet-lotus/fund-account.sh:/fund-account.sh
@@ -26,7 +26,7 @@ services:
    environment:
      # Use 0.0.0.0 so that calls can be made from outside the container
      - LOTUS_API_LISTENADDRESS=/ip4/0.0.0.0/tcp/1234/http
-    image: cerc/lotus-fixturenet:local
+    image: cerc/lotus:local
    volumes:
      - ../config/fixturenet-lotus/setup-node.sh:/docker-entrypoint-scripts.d/setup-node.sh
      - lotus_node_1_params:/var/tmp/filecoin-proof-parameters
@@ -42,10 +42,10 @@ services:
      - lotus-miner
    entrypoint: ["sh", "/docker-entrypoint-scripts.d/setup-node.sh"]
    ports:
-      - "1235"
-      - "2346"
-      - "3457"
-      - "1778"
+      - "1234"
+      - "2345"
+      - "3456"
+      - "1777"

  lotus-node-2:
    hostname: lotus-node-2
@@ -54,7 +54,7 @@ services:
    environment:
      # Use 0.0.0.0 so that calls can be made from outside the container
      - LOTUS_API_LISTENADDRESS=/ip4/0.0.0.0/tcp/1234/http
-    image: cerc/lotus-fixturenet:local
+    image: cerc/lotus:local
    volumes:
      - ../config/fixturenet-lotus/setup-node.sh:/docker-entrypoint-scripts.d/setup-node.sh
      - lotus_node_2_params:/var/tmp/filecoin-proof-parameters
@@ -70,10 +70,10 @@ services:
      - lotus-miner
    entrypoint: ["sh", "/docker-entrypoint-scripts.d/setup-node.sh"]
    ports:
-      - "1236"
-      - "2347"
-      - "3458"
-      - "1779"
+      - "1234"
+      - "2345"
+      - "3456"
+      - "1777"

 volumes:
  lotus_miner_params:
@@ -16,12 +16,8 @@ services:
      postgres_pass: password
      postgres_db: graph-node
      ethereum: ${ETH_NETWORKS:-lotus-fixturenet:http://lotus-node-1:1234/rpc/v1}
-      # Env varaibles reference: https://git.vdb.to/cerc-io/graph-node/src/branch/master/docs/environment-variables.md
      GRAPH_LOG: debug
      ETHEREUM_REORG_THRESHOLD: 3
-      GRAPH_ETHEREUM_JSON_RPC_TIMEOUT: ${GRAPH_ETHEREUM_JSON_RPC_TIMEOUT:-180}
-      GRAPH_ETHEREUM_REQUEST_RETRIES: ${GRAPH_ETHEREUM_REQUEST_RETRIES:-10}
-      GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE: ${GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE:-2000}
    entrypoint: ["bash", "-c"]
    # Wait for ETH RPC endpoint to be up when running with fixturenet-lotus
    command: |
@@ -1,23 +0,0 @@
-volumes:
-  parameters:
-  lotus-repo:
-  lotus_shared:
-
-services:
-  lotus:
-    environment:
-      - LOTUS_API_LISTENADDRESS=/ip4/0.0.0.0/tcp/1234/http
-      - LOTUS_LIBP2P_LISTENADDRESSES=/ip4/0.0.0.0/tcp/1235,/ip6/::/tcp/1235,/ip4/0.0.0.0/udp/1235/quic-v1,/ip6/::/udp/1235/quic-v1,/ip4/0.0.0.0/udp/1235/quic-v1/webtransport,/ip6/::/udp/1235/quic-v1/webtransport
-    image: cerc/lotus-mainnet:local
-    volumes:
-      - parameters:/var/tmp/filecoin-proof-parameters
-      - lotus-repo:/var/lib/lotus
-      - lotus_shared:/root/.lotus
-    ports:
-      - "1234"
-      - "1235"
-    deploy:
-      restart_policy:
-        condition: on-failure
-        delay: 30s
-    entrypoint: ["sh", "/docker-lotus-entrypoint.sh"]
@@ -0,0 +1,115 @@
+apiVersion: 1
+groups:
+    - orgId: 1
+      name: blackbox
+      folder: BlackboxAlerts
+      interval: 30s
+      rules:
+        # Azimuth Gateway endpoint
+        - uid: azimuth_gateway
+          title: azimuth_gateway_endpoint_tracking
+          condition: condition
+          data:
+            - refId: http_status_code
+              relativeTimeRange:
+                from: 600
+                to: 0
+              datasourceUid: PBFA97CFB590B2093
+              model:
+                editorMode: code
+                expr: probe_http_status_code{destination="azimuth_gateway"}
+                instant: true
+                intervalMs: 1000
+                legendFormat: __auto
+                maxDataPoints: 43200
+                range: false
+                refId: http_status_code
+            - refId: condition
+              relativeTimeRange:
+                from: 600
+                to: 0
+              datasourceUid: __expr__
+              model:
+                conditions:
+                    - evaluator:
+                        params:
+                          - 200
+                          - 200
+                        type: outside_range
+                      operator:
+                        type: and
+                      query:
+                        params:
+                          - http_status_code
+                      reducer:
+                        params: []
+                        type: last
+                      type: query
+                datasource:
+                    name: Expression
+                    type: __expr__
+                    uid: __expr__
+                expression: ""
+                hide: false
+                refId: condition
+                type: classic_conditions
+          noDataState: Alerting
+          execErrState: Alerting
+          for: 5m
+          annotations:
+            summary: Probe failed for Azimuth gateway endpoint, http status {{ index $values "http_status_code" }}
+          isPaused: false
+        # Laconicd GQL endpoint
+        - uid: laconicd_gql
+          title: laconicd_gql_endpoint_tracking
+          condition: condition
+          data:
+            - refId: http_status_code
+              relativeTimeRange:
+                from: 600
+                to: 0
+              datasourceUid: PBFA97CFB590B2093
+              model:
+                editorMode: code
+                expr: probe_http_status_code{destination="laconicd_gql"}
+                instant: true
+                intervalMs: 1000
+                legendFormat: __auto
+                maxDataPoints: 43200
+                range: false
+                refId: http_status_code
+            - refId: condition
+              relativeTimeRange:
+                from: 600
+                to: 0
+              datasourceUid: __expr__
+              model:
+                conditions:
+                    - evaluator:
+                        params:
+                          - 200
+                          - 200
+                        type: outside_range
+                      operator:
+                        type: and
+                      query:
+                        params:
+                          - http_status_code
+                      reducer:
+                        params: []
+                        type: last
+                      type: query
+                datasource:
+                    name: Expression
+                    type: __expr__
+                    uid: __expr__
+                expression: ""
+                hide: false
+                refId: condition
+                type: classic_conditions
+          noDataState: Alerting
+          execErrState: Alerting
+          for: 5m
+          annotations:
+            summary: Probe failed for Laconicd GQL endpoint, http status {{ index $values "http_status_code" }}
+          isPaused: false
@@ -49,7 +49,7 @@
      },
      "gridPos": {
        "h": 3,
-        "w": 3,
+        "w": 4,
        "x": 0,
        "y": 0
      },
@@ -24,9 +24,10 @@ scrape_configs:
    params:
      module: [http_2xx]
    static_configs:
-      # Add URLs to be monitored below
-      - targets:
-        # - https://github.com
+      # Add URLs for targets to be monitored below
+      # - targets: [https://github.com]
+      #   labels:
+      #     destination: 'github'
    relabel_configs:
      - source_labels: [__address__]
        regex: (.*)(:80)?
@@ -1,5 +1,5 @@
 #####################################
-FROM golang:1.21.7-bullseye AS lotus-builder
+FROM golang:1.19.12-bullseye AS lotus-builder
 MAINTAINER Lotus Development Team

 RUN apt-get update && apt-get install -y ca-certificates build-essential clang ocl-icd-opencl-dev ocl-icd-libopencl1 jq libhwloc-dev
@@ -29,6 +29,7 @@ RUN set -eux; \
    rustup --version; \
    cargo --version; \
    rustc --version;
+
 COPY ./ /opt/filecoin
 WORKDIR /opt/filecoin

@@ -37,13 +38,13 @@ WORKDIR /opt/filecoin
 ### make configurable filecoin-ffi build
 ARG FFI_BUILD_FROM_SOURCE=0
 ENV FFI_BUILD_FROM_SOURCE=${FFI_BUILD_FROM_SOURCE}
-RUN ls /opt/filecoin
+
 RUN make clean deps

 ARG RUSTFLAGS=""
 ARG GOFLAGS=""

-#RUN make lotus
+#RUN make buildall
 RUN make 2k

 #####################################
@@ -74,7 +75,7 @@ COPY --from=lotus-builder /opt/filecoin/lotus-shed /usr/local/bin/
 #COPY scripts/docker-lotus-entrypoint.sh /
 #COPY myscripts/setup-node.sh /docker-entrypoint-scripts.d/setup-node.sh

-ARG DOCKER_LOTUS_IMPORT_SNAPSHOT https://forest-archive.chainsafe.dev/latest/mainnet/
+ARG DOCKER_LOTUS_IMPORT_SNAPSHOT https://snapshots.mainnet.filops.net/minimal/latest
 ENV DOCKER_LOTUS_IMPORT_SNAPSHOT ${DOCKER_LOTUS_IMPORT_SNAPSHOT}
 ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
 ENV LOTUS_PATH /var/lib/lotus
@@ -1,12 +1,12 @@
 #!/usr/bin/env bash
-# Build cerc/fixturenet-lotus
+# Build cerc/lotus
 source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )

 # Use a release version tag to match the modified Dockerfile replaced in next step
-git -C ${CERC_REPO_BASE_DIR}/lotus checkout master
+git -C ${CERC_REPO_BASE_DIR}/lotus checkout v1.23.3

 # Replace repo's Dockerfile with modified one
 cp ${SCRIPT_DIR}/Dockerfile ${CERC_REPO_BASE_DIR}/lotus/Dockerfile

-docker build -t cerc/fixturenet-lotus:local ${build_command_args} ${CERC_REPO_BASE_DIR}/lotus
+docker build -t cerc/lotus:local ${build_command_args} ${CERC_REPO_BASE_DIR}/lotus
@@ -1,124 +0,0 @@
-#####################################
-FROM golang:1.21.7-bullseye AS lotus-builder
-MAINTAINER Lotus Development Team
-
-RUN apt-get update && apt-get install -y ca-certificates build-essential clang ocl-icd-opencl-dev ocl-icd-libopencl1 jq libhwloc-dev
-
-ENV XDG_CACHE_HOME="/tmp"
-
-### taken from https://github.com/rust-lang/docker-rust/blob/master/1.63.0/buster/Dockerfile
-ENV RUSTUP_HOME=/usr/local/rustup \
-    CARGO_HOME=/usr/local/cargo \
-    PATH=/usr/local/cargo/bin:$PATH \
-    RUST_VERSION=1.63.0
-
-RUN set -eux; \
-    dpkgArch="$(dpkg --print-architecture)"; \
-    case "${dpkgArch##*-}" in \
-        amd64) rustArch='x86_64-unknown-linux-gnu'; rustupSha256='5cc9ffd1026e82e7fb2eec2121ad71f4b0f044e88bca39207b3f6b769aaa799c' ;; \
-        arm64) rustArch='aarch64-unknown-linux-gnu'; rustupSha256='e189948e396d47254103a49c987e7fb0e5dd8e34b200aa4481ecc4b8e41fb929' ;; \
-        *) echo >&2 "unsupported architecture: ${dpkgArch}"; exit 1 ;; \
-    esac; \
-    url="https://static.rust-lang.org/rustup/archive/1.25.1/${rustArch}/rustup-init"; \
-    wget "$url"; \
-    echo "${rustupSha256} *rustup-init" | sha256sum -c -; \
-    chmod +x rustup-init; \
-    ./rustup-init -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION --default-host ${rustArch}; \
-    rm rustup-init; \
-    chmod -R a+w $RUSTUP_HOME $CARGO_HOME; \
-    rustup --version; \
-    cargo --version; \
-    rustc --version;
-
-COPY ./ /opt/filecoin
-WORKDIR /opt/filecoin
-
-ARG FFI_BUILD_FROM_SOURCE=0
-ENV FFI_BUILD_FROM_SOURCE=${FFI_BUILD_FROM_SOURCE}
-
-#RUN make clean deps
-
-ARG RUSTFLAGS=""
-ARG GOFLAGS=""
-
-RUN make clean deps && \
-    make lotus lotus-shed lotus-stats && \
-    install -C ./lotus /usr/local/bin/lotus && \
-    install -C ./lotus-shed /usr/local/bin/lotus-shed && \
-    install -C ./lotus-stats /usr/local/bin/lotus-stats
-
-#####################################
-FROM ubuntu:20.04 AS lotus-base
-MAINTAINER Lotus Development Team
-
-# Base resources
-COPY --from=lotus-builder /etc/ssl/certs                           /etc/ssl/certs
-COPY --from=lotus-builder /lib/*/libdl.so.2         /lib/
-COPY --from=lotus-builder /lib/*/librt.so.1         /lib/
-COPY --from=lotus-builder /lib/*/libgcc_s.so.1      /lib/
-COPY --from=lotus-builder /lib/*/libutil.so.1       /lib/
-COPY --from=lotus-builder /usr/lib/*/libltdl.so.7   /lib/
-COPY --from=lotus-builder /usr/lib/*/libnuma.so.1   /lib/
-COPY --from=lotus-builder /usr/lib/*/libhwloc.so.*  /lib/
-COPY --from=lotus-builder /usr/lib/*/libOpenCL.so.1 /lib/
-
-RUN useradd -r -u 532 -U fc \
- && mkdir -p /etc/OpenCL/vendors \
- && echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd
-
-#####################################
-FROM lotus-base AS lotus
-MAINTAINER Lotus Development Team
-
-COPY --from=lotus-builder \
-    /usr/local/bin/lotus \
-    /usr/local/bin/lotus-shed \
-    /usr/local/bin/
-
-ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
-ENV LOTUS_PATH /var/lib/lotus
-ENV DOCKER_LOTUS_IMPORT_WALLET ""
-
-RUN mkdir /var/lib/lotus /var/tmp/filecoin-proof-parameters
-RUN chown fc: /var/lib/lotus /var/tmp/filecoin-proof-parameters
-
-VOLUME /var/lib/lotus
-VOLUME /var/tmp/filecoin-proof-parameters
-
-USER fc
-
-EXPOSE 1234
-
-
-CMD ["-help"]
-
-#####################################
-FROM lotus-base AS lotus-all-in-one
-
-# Install netcat for healthcheck
-RUN apt-get update && apt-get install -y netcat && apt-get install -y iproute2
-
-ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
-ENV LOTUS_PATH /var/lib/lotus
-
-ARG DOCKER_LOTUS_IMPORT_SNAPSHOT=https://forest-archive.chainsafe.dev/latest/mainnet/
-ENV DOCKER_LOTUS_IMPORT_SNAPSHOT=${DOCKER_LOTUS_IMPORT_SNAPSHOT}
-
-COPY --from=lotus-builder /opt/filecoin/lotus          /usr/local/bin/
-COPY --from=lotus-builder /opt/filecoin/lotus-shed     /usr/local/bin/
-COPY --from=lotus-builder /opt/filecoin/lotus-stats    /usr/local/bin/
-COPY scripts/docker-lotus-entrypoint.sh /docker-lotus-entrypoint.sh
-RUN chmod +x /docker-lotus-entrypoint.sh
-
-RUN mkdir /var/tmp/filecoin-proof-parameters
-RUN mkdir /var/lib/lotus
-RUN chown fc: /var/tmp/filecoin-proof-parameters
-RUN chown fc: /var/lib/lotus
-
-
-VOLUME /var/tmp/filecoin-proof-parameters
-VOLUME /var/lib/lotus
-
-EXPOSE 1234
-EXPOSE 1235
-
@@ -1,14 +0,0 @@
-#!/usr/bin/env bash
-# Build cerc/mainnet-lotus
-source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
-SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-
-git stash
-
-# Use a release version tag to match the modified Dockerfile replaced in next step
-git -C ${CERC_REPO_BASE_DIR}/lotus checkout master
-
-# Replace repo's Dockerfile with modified one
-cp ${SCRIPT_DIR}/Dockerfile ${CERC_REPO_BASE_DIR}/lotus/Dockerfile
-
-docker build -t cerc/lotus-mainnet:local ${build_command_args} ${CERC_REPO_BASE_DIR}/lotus
@@ -2,4 +2,4 @@
 # Build cerc/test-container
 source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-docker build -t cerc/test-container:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} $SCRIPT_DIR
+docker build -t cerc/test-container:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} $SCRIPT_DIR
@@ -41,8 +41,7 @@ cerc/pocket
 cerc/watcher-azimuth
 cerc/ipld-eth-state-snapshot
 cerc/watcher-gelato
-cerc/mainnet-lotus
-cerc/fixturenet-lotus
+cerc/lotus
 cerc/go-opera
 cerc/lasso
 cerc/reth
@@ -27,7 +27,6 @@ fixturenet-pocket
 watcher-azimuth
 watcher-gelato
 fixturenet-lotus
-mainnet-lotus
 mainnet-go-opera
 lasso
 reth
@@ -2,8 +2,8 @@ version: "1.0"
 name: fixturenet-lotus
 description: "A lotus fixturenet"
 repos:
-  - git.vdb.to/cerc-io/lotus
+  - github.com/filecoin-project/lotus
 containers:
-  - cerc/fixturenet-lotus
+  - cerc/lotus
 pods:
  - fixturenet-lotus
@@ -13,7 +13,7 @@ repos:
  - github.com/sushiswap/subgraphs
 containers:
  # fixturenet-lotus image
-  - cerc/lotus-fixturenet
+  - cerc/lotus
  # graph-node image
  - cerc/graph-node
  # sushiswap contract deployment images
@@ -76,19 +76,6 @@ export ETH_RPC_PORT=
 # The etherum network(s) graph-node will connect to
 # Set this to a space-separated list of the networks where each entry has the form NAME:URL
 export ETH_NETWORKS=
-
-# Optional:
-
-# Timeout for ETH RPC requests in seconds (default: 180s)
-export GRAPH_ETHEREUM_JSON_RPC_TIMEOUT=
-
-# Number of times to retry ETH RPC requests (default: 10)
-export GRAPH_ETHEREUM_REQUEST_RETRIES=
-
-# Maximum number of blocks to scan for triggers in each request (default: 2000)
-export GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE=
-
-# Ref: https://git.vdb.to/cerc-io/graph-node/src/branch/master/docs/environment-variables.md
 ```

 Example env file:
@@ -98,10 +85,6 @@ export ETH_RPC_HOST=filecoin.chainup.net
 export ETH_RPC_PORT=443

 export ETH_NETWORKS=filecoin:https://filecoin.chainup.net/rpc/v1
-
-export GRAPH_ETHEREUM_JSON_RPC_TIMEOUT=360
-export GRAPH_ETHEREUM_REQUEST_RETRIES=5
-export GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE=50
 ```

 Set the environment variables:
@@ -1,26 +0,0 @@
-# Lotus stack
-
-## Clone required repositories
-```
-$ laconic-so --stack mainnet-lotus setup-repositories
-```
-## Build the stack's containers
-```
-$ laconic-so --stack mainnet-lotus build-containers
-```
-## Create a deployment of the stack
-```
-$ laconic-so --stack mainnet-lotus deploy init --map-ports-to-host any-same --output lotus-spec.yml
-```
-Update the URLs in the generated spec file to use your systems' actual IP addresses.
-```
-$ laconic-so --stack mainnet-lotus deploy create --deployment-dir lotus-deployment --spec-file lotus-spec.yml
-```
-## Start the stack
-```
-$ laconic-so deployment --dir lotus-deployment start
-```
-Check logs:
-```
-$ laconic-so deployment --dir lotus-deployment logs
-```
@@ -1,9 +0,0 @@
-version: "1.0"
-name: mainnet-lotus
-description: "A lotus stack"
-repos:
-  - git.vdb.to/cerc-io/lotus
-containers:
-  - cerc/mainnet-lotus
-pods:
-  - mainnet-lotus
@@ -123,8 +123,9 @@ laconic-so --stack monitoring deploy create --spec-file monitoring-spec.yml --de
  ```yml
  ...
  - job_name: laconicd
-  static_configs:
-    - targets: ['example-host:1317']
+    ...
+    static_configs:
+      - targets: ['example-host:1317']
  ...
  ```

@@ -44,13 +44,18 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers
  - job_name: 'blackbox'
    ...
    static_configs:
-      - targets:
-        - <AZIMUTH_GATEWAY_GQL_ENDPOINT>
-        - <LACONICD_GQL_ENDPOINT>
+      - targets: [<AZIMUTH_GATEWAY_GQL_ENDPOINT>]
+        labels:
+          # Add destination label for pre-configured alerts
+          destination: 'azimuth_gateway'
+      - targets: [<LACONICD_GQL_ENDPOINT>]
+        labels:
+          destination: 'laconicd_gql'
  ...
  - job_name: laconicd
-  static_configs:
-    - targets: ['LACONICD_REST_HOST:LACONICD_REST_PORT']
+    ...
+    static_configs:
+      - targets: ['LACONICD_REST_HOST:LACONICD_REST_PORT']
  ...
  - job_name: azimuth
    scrape_interval: 10s
@@ -119,10 +124,12 @@ Add scrape config as done above for any additional watcher to add it to the Watc

 ### Grafana alerts config

-Place the pre-configured watcher alerts rules in Grafana provisioning directory:
+Place the pre-configured watcher and blackbox endpoint alerts rules in Grafana provisioning directory:

  ```bash
  cp monitoring-watchers-deployment/config/monitoring/watcher-alert-rules.yml monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/
+
+  cp monitoring-watchers-deployment/config/monitoring/blackbox-alert-rules.yml monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/
  ```

 Update the alerting contact points config (`monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/contactpoints.yml`) with desired contact points
@@ -135,7 +142,7 @@ Add corresponding routes to the notification policies config (`monitoring-watche
      - receiver: SlackNotifier
      object_matchers:
        # Add matchers below
-        - ['grafana_folder', '=', 'WatcherAlerts']
+        - ['grafana_folder', '=~', 'WatcherAlerts|BlackboxAlerts']
  ```

 ### Env
@@ -11,7 +11,7 @@ repos:
  - github.com/vulcanize/uniswap-v3-info
 containers:
  ## fixturenet-lotus image
-  - cerc/lotus-fixturenet
+  - cerc/lotus
  ## sushiswap images
  - cerc/sushiswap-v3-core
  - cerc/sushiswap-v3-periphery
@@ -2,10 +2,10 @@ version: "1.0"
 name: webapp-deployer-backend
 description: "Deployer for webapps"
 repos:
-  - git.vdb.to/cerc-io/webapp-deployment-status-api
+  - git.vdb.to/telackey/webapp-deployment-status-api
 containers:
  - cerc/webapp-deployer-backend
 pods:
  - name: webapp-deployer-backend
-    repository: git.vdb.to/cerc-io/webapp-deployment-status-api
+    repository: git.vdb.to/telackey/webapp-deployment-status-api
    path: ./
@@ -27,7 +27,6 @@ from pathlib import Path
 from stack_orchestrator import constants
 from stack_orchestrator.opts import opts
 from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config, global_options2, get_dev_root_path
-from stack_orchestrator.util import resolve_compose_file
 from stack_orchestrator.deploy.deployer import Deployer, DeployerException
 from stack_orchestrator.deploy.deployer_factory import getDeployer
 from stack_orchestrator.deploy.deploy_types import ClusterContext, DeployCommandContext
@@ -325,10 +324,7 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file):
        pod_path = pod["path"]
        if include_exclude_check(pod_name, include, exclude):
            if pod_repository is None or pod_repository == "internal":
-                if deployment:
-                    compose_file_name = os.path.join(compose_dir, f"docker-compose-{pod_path}.yml")
-                else:
-                    compose_file_name = resolve_compose_file(stack, pod_name)
+                compose_file_name = os.path.join(compose_dir, f"docker-compose-{pod_path}.yml")
            else:
                if deployment:
                    compose_file_name = os.path.join(compose_dir, f"docker-compose-{pod_name}.yml")
@@ -340,7 +336,6 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file):
                    if pod_post_start_command is not None:
                        post_start_commands.append(os.path.join(script_dir, pod_post_start_command))
                else:
-                    # TODO: fix this code for external stack with scripts
                    pod_root_dir = os.path.join(dev_root_path, pod_repository.split("/")[-1], pod["path"])
                    compose_file_name = os.path.join(pod_root_dir, f"docker-compose-{pod_name}.yml")
                    pod_pre_start_command = pod.get("pre_start_command")
@@ -16,7 +16,7 @@
 import os
 from typing import List, Any
 from stack_orchestrator.deploy.deploy_types import DeployCommandContext, VolumeMapping
-from stack_orchestrator.util import get_parsed_stack_config, get_yaml, get_pod_list, resolve_compose_file
+from stack_orchestrator.util import get_parsed_stack_config, get_yaml, get_compose_file_dir, get_pod_list
 from stack_orchestrator.opts import opts


@@ -27,7 +27,7 @@ def _container_image_from_service(stack: str, service: str):
    pods = get_pod_list(parsed_stack)
    yaml = get_yaml()
    for pod in pods:
-        pod_file_path = resolve_compose_file(stack, pod)
+        pod_file_path = os.path.join(get_compose_file_dir(), f"docker-compose-{pod}.yml")
        parsed_pod_file = yaml.load(open(pod_file_path, "r"))
        if "services" in parsed_pod_file:
            services = parsed_pod_file["services"]
@@ -26,8 +26,7 @@ from stack_orchestrator import constants
 from stack_orchestrator.opts import opts
 from stack_orchestrator.util import (get_stack_file_path, get_parsed_deployment_spec, get_parsed_stack_config,
                                     global_options, get_yaml, get_pod_list, get_pod_file_path, pod_has_scripts,
-                                     get_pod_script_paths, get_plugin_code_paths, error_exit, env_var_map_from_file,
-                                     resolve_config_dir)
+                                     get_pod_script_paths, get_plugin_code_paths, error_exit, env_var_map_from_file)
 from stack_orchestrator.deploy.spec import Spec
 from stack_orchestrator.deploy.deploy_types import LaconicStackSetupCommand
 from stack_orchestrator.deploy.deployer_factory import getDeployerConfigGenerator
@@ -44,7 +43,7 @@ def _get_ports(stack):
    pods = get_pod_list(parsed_stack)
    yaml = get_yaml()
    for pod in pods:
-        pod_file_path = get_pod_file_path(stack, parsed_stack, pod)
+        pod_file_path = get_pod_file_path(parsed_stack, pod)
        parsed_pod_file = yaml.load(open(pod_file_path, "r"))
        if "services" in parsed_pod_file:
            for svc_name, svc in parsed_pod_file["services"].items():
@@ -80,7 +79,7 @@ def _get_named_volumes(stack):
        return ret

    for pod in pods:
-        pod_file_path = get_pod_file_path(stack, parsed_stack, pod)
+        pod_file_path = get_pod_file_path(parsed_stack, pod)
        parsed_pod_file = yaml.load(open(pod_file_path, "r"))
        if "volumes" in parsed_pod_file:
            volumes = parsed_pod_file["volumes"]
@@ -481,9 +480,10 @@ def create_operation(deployment_command_context, spec_file, deployment_dir, netw
    os.mkdir(destination_compose_dir)
    destination_pods_dir = deployment_dir_path.joinpath("pods")
    os.mkdir(destination_pods_dir)
+    data_dir = Path(__file__).absolute().parent.parent.joinpath("data")
    yaml = get_yaml()
    for pod in pods:
-        pod_file_path = get_pod_file_path(stack_name, parsed_stack, pod)
+        pod_file_path = get_pod_file_path(parsed_stack, pod)
        parsed_pod_file = yaml.load(open(pod_file_path, "r"))
        extra_config_dirs = _find_extra_config_dirs(parsed_pod_file, pod)
        destination_pod_dir = destination_pods_dir.joinpath(pod)
@@ -497,7 +497,7 @@ def create_operation(deployment_command_context, spec_file, deployment_dir, netw
        config_dirs = {pod}
        config_dirs = config_dirs.union(extra_config_dirs)
        for config_dir in config_dirs:
-            source_config_dir = resolve_config_dir(stack_name, config_dir)
+            source_config_dir = data_dir.joinpath("config", config_dir)
            if os.path.exists(source_config_dir):
                destination_config_dir = deployment_dir_path.joinpath("config", config_dir)
                # If the same config dir appears in multiple pods, it may already have been copied
@@ -39,12 +39,11 @@ def process_app_deployment_request(
    app_deployment_request,
    deployment_record_namespace,
    dns_record_namespace,
-    default_dns_suffix,
+    dns_suffix,
    deployment_parent_dir,
    kube_config,
    image_registry,
    force_rebuild,
-    fqdn_policy,
    logger
 ):
    logger.log("BEGIN - process_app_deployment_request")
@@ -57,15 +56,14 @@ def process_app_deployment_request(
    requested_name = hostname_for_deployment_request(app_deployment_request, laconic)
    logger.log(f"Determined requested name: {requested_name}")

+    # HACK
    if "." in requested_name:
-        if "allow" == fqdn_policy or "preexisting" == fqdn_policy:
-            fqdn = requested_name
-        else:
-            raise Exception(f"{requested_name} is invalid: only unqualified hostnames are allowed.")
-    else:
-        fqdn = f"{requested_name}.{default_dns_suffix}"
+        raise Exception("Only unqualified hostnames allowed at this time.")
+
+    fqdn = f"{requested_name}.{dns_suffix}"

    # 3. check ownership of existing dnsrecord vs this request
+    # TODO: Support foreign DNS
    dns_crn = f"{dns_record_namespace}/{fqdn}"
    dns_record = laconic.get_record(dns_crn)
    if dns_record:
@@ -77,9 +75,7 @@ def process_app_deployment_request(
            logger.log(f"Matched DnsRecord ownership: {matched_owner}")
        else:
            raise Exception("Unable to confirm ownership of DnsRecord %s for request %s" %
-                            (dns_crn, app_deployment_request.id))
-    elif "preexisting" == fqdn_policy:
-        raise Exception(f"No pre-existing DnsRecord {dns_crn} could be found for request {app_deployment_request.id}.")
+                            (dns_record.id, app_deployment_request.id))

    # 4. get build and runtime config from request
    env_filename = None
@@ -195,7 +191,6 @@ def dump_known_requests(filename, requests, status="SEEN"):
@click.option("--state-file", help="File to store state about previously seen requests.")
@click.option("--only-update-state", help="Only update the state file, don't process any requests anything.", is_flag=True)
@click.option("--dns-suffix", help="DNS domain to use eg, laconic.servesthe.world")
-@click.option("--fqdn-policy", help="How to handle requests with an FQDN: prohibit, allow, preexisting", default="prohibit")
@click.option("--record-namespace-dns", help="eg, crn://laconic/dns")
@click.option("--record-namespace-deployments", help="eg, crn://laconic/deployments")
@click.option("--dry-run", help="Don't do anything, just report what would be done.", is_flag=True)
@@ -206,7 +201,7 @@ def dump_known_requests(filename, requests, status="SEEN"):
@click.pass_context
 def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_dir,  # noqa: C901
            request_id, discover, state_file, only_update_state,
-            dns_suffix, fqdn_policy, record_namespace_dns, record_namespace_deployments, dry_run,
+            dns_suffix, record_namespace_dns, record_namespace_deployments, dry_run,
            include_tags, exclude_tags, force_rebuild, log_dir):
    if request_id and discover:
        print("Cannot specify both --request-id and --discover", file=sys.stderr)
@@ -225,10 +220,6 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
            print("--dns-suffix, --record-namespace-dns, and --record-namespace-deployments are all required", file=sys.stderr)
            sys.exit(2)

-    if fqdn_policy not in ["prohibit", "allow", "preexisting"]:
-        print("--fqdn-policy must be one of 'prohibit', 'allow', or 'preexisting'", file=sys.stderr)
-        sys.exit(2)
-
    # Split CSV and clean up values.
    include_tags = [tag.strip() for tag in include_tags.split(",") if tag]
    exclude_tags = [tag.strip() for tag in exclude_tags.split(",") if tag]
@@ -256,10 +247,7 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
    requests_by_name = {}
    skipped_by_name = {}
    for r in requests:
-        if r.id in previous_requests and previous_requests[r.id].get("status", "") != "RETRY":
-            print(f"Skipping request {r.id}, we've already seen it.")
-            continue
-
+        # TODO: Do this _after_ filtering deployments and cancellations to minimize round trips.
        app = laconic.get_record(r.attributes.application)
        if not app:
            print("Skipping request %s, cannot locate app." % r.id)
@@ -346,7 +334,6 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
                    kube_config,
                    image_registry,
                    force_rebuild,
-                    fqdn_policy,
                    logger
                )
                status = "DEPLOYED"
@@ -17,7 +17,6 @@ import click

 from stack_orchestrator.command_types import CommandOptions
 from stack_orchestrator.repos import setup_repositories
-from stack_orchestrator.repos import fetch_stack
 from stack_orchestrator.build import build_containers, fetch_containers
 from stack_orchestrator.build import build_npms
 from stack_orchestrator.build import build_webapp
@@ -51,7 +50,6 @@ def cli(ctx, stack, quiet, verbose, dry_run, local_stack, debug, continue_on_err
    ctx.obj = command_options


-cli.add_command(fetch_stack.command, "fetch-stack")
 cli.add_command(setup_repositories.command, "setup-repositories")
 cli.add_command(build_containers.command, "build-containers")
 cli.add_command(fetch_containers.command, "fetch-containers")
@@ -1,45 +0,0 @@
-# Copyright © 2022, 2023 Vulcanize
-
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
-
-# env vars:
-# CERC_REPO_BASE_DIR defaults to ~/cerc
-
-
-import click
-import os
-
-from decouple import config
-from git import exc
-
-from stack_orchestrator.opts import opts
-from stack_orchestrator.repos.setup_repositories import process_repo
-from stack_orchestrator.util import error_exit
-
-
-@click.command()
-@click.argument('stack-locator')
-@click.option('--git-ssh', is_flag=True, default=False)
-@click.option('--check-only', is_flag=True, default=False)
-@click.option('--pull', is_flag=True, default=False)
-@click.pass_context
-def command(ctx, stack_locator, git_ssh, check_only, pull):
-    '''optionally resolve then git clone a repository containing one or more stack definitions'''
-    dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc"))
-    if not opts.o.quiet:
-        print(f"Dev Root is: {dev_root_path}")
-    try:
-        process_repo(pull, check_only, git_ssh, dev_root_path, None, stack_locator)
-    except exc.GitCommandError as error:
-        error_exit(f"\n******* git command returned error exit status:\n{error}")
@@ -26,7 +26,6 @@ import importlib.resources
 from pathlib import Path
 import yaml
 from stack_orchestrator.constants import stack_file_name
-from stack_orchestrator.opts import opts
 from stack_orchestrator.util import include_exclude_check, stack_is_external, error_exit, warn_exit


@@ -88,8 +87,8 @@ def _get_repo_current_branch_or_tag(full_filesystem_repo_path):


 # TODO: fix the messy arg list here
-def process_repo(pull, check_only, git_ssh, dev_root_path, branches_array, fully_qualified_repo):
-    if opts.o.verbose:
+def process_repo(verbose, quiet, dry_run, pull, check_only, git_ssh, dev_root_path, branches_array, fully_qualified_repo):
+    if verbose:
        print(f"Processing repo: {fully_qualified_repo}")
    repo_host, repo_path, repo_branch = host_and_path_for_repo(fully_qualified_repo)
    git_ssh_prefix = f"git@{repo_host}:"
@@ -101,7 +100,7 @@ def process_repo(pull, check_only, git_ssh, dev_root_path, branches_array, fully
    (current_repo_branch_or_tag, is_branch) = _get_repo_current_branch_or_tag(
        full_filesystem_repo_path
        ) if is_present else (None, None)
-    if not opts.o.quiet:
+    if not quiet:
        present_text = f"already exists active {'branch' if is_branch else 'tag'}: {current_repo_branch_or_tag}" if is_present \
            else 'Needs to be fetched'
        print(f"Checking: {full_filesystem_repo_path}: {present_text}")
@@ -112,25 +111,25 @@ def process_repo(pull, check_only, git_ssh, dev_root_path, branches_array, fully
            sys.exit(1)
        else:
            if pull:
-                if opts.o.verbose:
+                if verbose:
                    print(f"Running git pull for {full_filesystem_repo_path}")
                if not check_only:
                    if is_branch:
                        git_repo = git.Repo(full_filesystem_repo_path)
                        origin = git_repo.remotes.origin
-                        origin.pull(progress=None if opts.o.quiet else GitProgress())
+                        origin.pull(progress=None if quiet else GitProgress())
                    else:
                        print("skipping pull because this repo checked out a tag")
                else:
                    print("(git pull skipped)")
    if not is_present:
        # Clone
-        if opts.o.verbose:
+        if verbose:
            print(f'Running git clone for {full_github_repo_path} into {full_filesystem_repo_path}')
-        if not opts.o.dry_run:
+        if not dry_run:
            git.Repo.clone_from(full_github_repo_path,
                                full_filesystem_repo_path,
-                                progress=None if opts.o.quiet else GitProgress())
+                                progress=None if quiet else GitProgress())
        else:
            print("(git clone skipped)")
    # Checkout the requested branch, if one was specified
@@ -151,13 +150,13 @@ def process_repo(pull, check_only, git_ssh, dev_root_path, branches_array, fully
            current_repo_branch_or_tag and (
                current_repo_branch_or_tag != branch_to_checkout)
                ):
-            if not opts.o.quiet:
+            if not quiet:
                print(f"switching to branch {branch_to_checkout} in repo {repo_path}")
            git_repo = git.Repo(full_filesystem_repo_path)
            # git checkout works for both branches and tags
            git_repo.git.checkout(branch_to_checkout)
        else:
-            if opts.o.verbose:
+            if verbose:
                print(f"repo {repo_path} is already on branch/tag {branch_to_checkout}")


@@ -183,18 +182,36 @@ def parse_branches(branches_string):
@click.option('--check-only', is_flag=True, default=False)
@click.option('--pull', is_flag=True, default=False)
@click.option("--branches", help="override branches for repositories")
+@click.option('--branches-file', help="checkout branches specified in this file")
@click.pass_context
-def command(ctx, include, exclude, git_ssh, check_only, pull, branches):
+def command(ctx, include, exclude, git_ssh, check_only, pull, branches, branches_file):
    '''git clone the set of repositories required to build the complete system from source'''

-    quiet = opts.o.quiet
-    verbose = opts.o.verbose
-    stack = opts.o.stack
+    quiet = ctx.obj.quiet
+    verbose = ctx.obj.verbose
+    dry_run = ctx.obj.dry_run
+    stack = ctx.obj.stack

    branches_array = []

+    # TODO: branches file needs to be re-worked in the context of stacks
+    if branches_file:
+        if branches:
+            print("Error: can't specify both --branches and --branches-file")
+            sys.exit(1)
+        else:
+            if verbose:
+                print(f"loading branches from: {branches_file}")
+            with open(branches_file) as branches_file_open:
+                branches_array = branches_file_open.read().splitlines()
+
+    print(f"branches: {branches}")
    if branches:
-        branches_array = parse_branches(branches)
+        if branches_file:
+            print("Error: can't specify both --branches and --branches-file")
+            sys.exit(1)
+        else:
+            branches_array = parse_branches(branches)

    if branches_array and verbose:
        print(f"Branches are: {branches_array}")
@@ -254,6 +271,7 @@ def command(ctx, include, exclude, git_ssh, check_only, pull, branches):

    for repo in repos:
        try:
-            process_repo(pull, check_only, git_ssh, dev_root_path, branches_array, repo)
+            process_repo(verbose, quiet, dry_run, pull, check_only, git_ssh, dev_root_path, branches_array, repo)
        except git.exc.GitCommandError as error:
-            error_exit(f"\n******* git command returned error exit status:\n{error}")
+            print(f"\n******* git command returned error exit status:\n{error}")
+            sys.exit(1)
@@ -94,38 +94,10 @@ def get_plugin_code_paths(stack) -> List[Path]:
    return list(result)


-# # Find a config directory, looking first in any external stack
-# and if not found there, internally
-def resolve_config_dir(stack, config_dir_name: str):
-    if stack_is_external(stack):
-        # First try looking in the external stack for the compose file
-        config_base = Path(stack).parent.parent.joinpath("config")
-        proposed_dir = config_base.joinpath(config_dir_name)
-        if proposed_dir.exists():
-            return proposed_dir
-        # If we don't find it fall through to the internal case
-    config_base = get_internal_config_dir()
-    return config_base.joinpath(config_dir_name)
-
-
-# Find a compose file, looking first in any external stack
-# and if not found there, internally
-def resolve_compose_file(stack, pod_name: str):
-    if stack_is_external(stack):
-        # First try looking in the external stack for the compose file
-        compose_base = Path(stack).parent.parent.joinpath("compose")
-        proposed_file = compose_base.joinpath(f"docker-compose-{pod_name}.yml")
-        if proposed_file.exists():
-            return proposed_file
-        # If we don't find it fall through to the internal case
-    compose_base = get_internal_compose_file_dir()
-    return compose_base.joinpath(f"docker-compose-{pod_name}.yml")
-
-
-def get_pod_file_path(stack, parsed_stack, pod_name: str):
+def get_pod_file_path(parsed_stack, pod_name: str):
    pods = parsed_stack["pods"]
    if type(pods[0]) is str:
-        result = resolve_compose_file(stack, pod_name)
+        result = os.path.join(get_compose_file_dir(), f"docker-compose-{pod_name}.yml")
    else:
        for pod in pods:
            if pod["name"] == pod_name:
@@ -159,7 +131,7 @@ def pod_has_scripts(parsed_stack, pod_name: str):
    return result


-def get_internal_compose_file_dir():
+def get_compose_file_dir():
    # TODO: refactor to use common code with deploy command
    # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
    data_dir = Path(__file__).absolute().parent.joinpath("data")
@@ -167,7 +139,7 @@ def get_internal_compose_file_dir():
    return source_compose_dir


-def get_internal_config_dir():
+def get_config_file_dir():
    # TODO: refactor to use common code with deploy command
    data_dir = Path(__file__).absolute().parent.joinpath("data")
    source_config_dir = data_dir.joinpath("config")
@@ -1,185 +0,0 @@
-#!/usr/bin/env bash
-set -e
-if [ -n "$CERC_SCRIPT_DEBUG" ]; then
-  set -x
-fi
-# Dump environment variables for debugging
-echo "Environment variables:"
-env
-
-if [ "$1" == "from-path" ]; then
-    TEST_TARGET_SO="laconic-so"
-else
-    TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 )
-fi
-
-delete_cluster_exit () {
-    $TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes
-    exit 1
-}
-
-# Test basic stack-orchestrator deploy
-echo "Running stack-orchestrator external stack deploy test"
-# Set a non-default repo dir
-export CERC_REPO_BASE_DIR=~/stack-orchestrator-test/repo-base-dir
-echo "Testing this package: $TEST_TARGET_SO"
-echo "Test version command"
-reported_version_string=$( $TEST_TARGET_SO version )
-echo "Version reported is: ${reported_version_string}"
-echo "Cloning repositories into: $CERC_REPO_BASE_DIR"
-rm -rf $CERC_REPO_BASE_DIR
-mkdir -p $CERC_REPO_BASE_DIR
-# Clone the external test stack
-$TEST_TARGET_SO fetch-stack git.vdb.to/cerc-io/test-external-stack
-stack_name="$CERC_REPO_BASE_DIR/test-external-stack/stack-orchestrator/stacks/test-external-stack"
-TEST_TARGET_SO_STACK="$TEST_TARGET_SO --stack ${stack_name}"
-# Test bringing the test container up and down
-# with and without volume removal
-$TEST_TARGET_SO_STACK setup-repositories
-$TEST_TARGET_SO_STACK build-containers
-# Test deploy command execution
-$TEST_TARGET_SO_STACK deploy setup $CERC_REPO_BASE_DIR
-# Check that we now have the expected output directory
-container_output_dir=$CERC_REPO_BASE_DIR/container-output-dir
-if [ ! -d "$container_output_dir" ]; then
-    echo "deploy setup test: output directory not present"
-    echo "deploy setup test: FAILED"
-    exit 1
-fi
-if [ ! -f "$container_output_dir/output-file" ]; then
-    echo "deploy setup test: output file not present"
-    echo "deploy setup test: FAILED"
-    exit 1
-fi
-output_file_content=$(<$container_output_dir/output-file)
-if [ ! "$output_file_content" == "output-data"  ]; then
-    echo "deploy setup test: output file contents not correct"
-    echo "deploy setup test: FAILED"
-    exit 1
-fi
-# Check that we now have the expected output file
-$TEST_TARGET_SO_STACK deploy up
-# Test deploy port command
-deploy_port_output=$( $TEST_TARGET_SO_STACK deploy port test 80 )
-if [[ "$deploy_port_output" =~ ^0.0.0.0:[1-9][0-9]* ]]; then
-    echo "Deploy port test: passed"
-else
-    echo "Deploy port test: FAILED"
-    exit 1
-fi
-$TEST_TARGET_SO_STACK deploy down
-# The next time we bring the container up the volume will be old (from the previous run above)
-$TEST_TARGET_SO_STACK deploy up
-log_output_1=$( $TEST_TARGET_SO_STACK deploy logs )
-if [[ "$log_output_1" == *"filesystem is old"* ]]; then
-    echo "Retain volumes test: passed"
-else
-    echo "Retain volumes test: FAILED"
-    exit 1
-fi
-$TEST_TARGET_SO_STACK deploy down --delete-volumes
-# Now when we bring the container up the volume will be new again
-$TEST_TARGET_SO_STACK deploy up
-log_output_2=$( $TEST_TARGET_SO_STACK deploy logs )
-if [[ "$log_output_2" == *"filesystem is fresh"* ]]; then
-    echo "Delete volumes test: passed"
-else
-    echo "Delete volumes test: FAILED"
-    exit 1
-fi
-$TEST_TARGET_SO_STACK deploy down --delete-volumes
-# Basic test of creating a deployment
-test_deployment_dir=$CERC_REPO_BASE_DIR/test-deployment-dir
-test_deployment_spec=$CERC_REPO_BASE_DIR/test-deployment-spec.yml
-$TEST_TARGET_SO_STACK deploy init --output $test_deployment_spec --config CERC_TEST_PARAM_1=PASSED,CERC_TEST_PARAM_3=FAST
-# Check the file now exists
-if [ ! -f "$test_deployment_spec" ]; then
-    echo "deploy init test: spec file not present"
-    echo "deploy init test: FAILED"
-    exit 1
-fi
-echo "deploy init test: passed"
-$TEST_TARGET_SO_STACK deploy create --spec-file $test_deployment_spec --deployment-dir $test_deployment_dir
-# Check the deployment dir exists
-if [ ! -d "$test_deployment_dir" ]; then
-    echo "deploy create test: deployment directory not present"
-    echo "deploy create test: FAILED"
-    exit 1
-fi
-echo "deploy create test: passed"
-# Check the file writted by the create command in the stack now exists
-if [ ! -f "$test_deployment_dir/create-file" ]; then
-    echo "deploy create test: create output file not present"
-    echo "deploy create test: FAILED"
-    exit 1
-fi
-# And has the right content
-create_file_content=$(<$test_deployment_dir/create-file)
-if [ ! "$create_file_content" == "create-command-output-data"  ]; then
-    echo "deploy create test: create output file contents not correct"
-    echo "deploy create test: FAILED"
-    exit 1
-fi
-
-# Add a config file to be picked up by the ConfigMap before starting.
-echo "dbfc7a4d-44a7-416d-b5f3-29842cc47650" > $test_deployment_dir/data/test-config/test_config
-
-echo "deploy create output file test: passed"
-# Try to start the deployment
-$TEST_TARGET_SO deployment --dir $test_deployment_dir start
-# Check logs command works
-log_output_3=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
-if [[ "$log_output_3" == *"filesystem is fresh"* ]]; then
-    echo "deployment logs test: passed"
-else
-    echo "deployment logs test: FAILED"
-    exit 1
-fi
-# Check the config variable CERC_TEST_PARAM_1 was passed correctly
-if [[ "$log_output_3" == *"Test-param-1: PASSED"* ]]; then
-    echo "deployment config test: passed"
-else
-    echo "deployment config test: FAILED"
-    exit 1
-fi
-# Check the config variable CERC_TEST_PARAM_2 was passed correctly from the compose file
-if [[ "$log_output_3" == *"Test-param-2: CERC_TEST_PARAM_2_VALUE"* ]]; then
-    echo "deployment compose config test: passed"
-else
-    echo "deployment compose config test: FAILED"
-    exit 1
-fi
-# Check the config variable CERC_TEST_PARAM_3 was passed correctly
-if [[ "$log_output_3" == *"Test-param-3: FAST"* ]]; then
-    echo "deployment config test: passed"
-else
-    echo "deployment config test: FAILED"
-    exit 1
-fi
-
-# Check that the ConfigMap is mounted and contains the expected content.
-log_output_4=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
-if [[ "$log_output_4" == *"/config/test_config:"* ]] && [[ "$log_output_4" == *"dbfc7a4d-44a7-416d-b5f3-29842cc47650"* ]]; then
-    echo "deployment ConfigMap test: passed"
-else
-    echo "deployment ConfigMap test: FAILED"
-    delete_cluster_exit
-fi
-
-# Stop then start again and check the volume was preserved
-$TEST_TARGET_SO deployment --dir $test_deployment_dir stop
-# Sleep a bit just in case
-# sleep for longer to check if that's why the subsequent create cluster fails
-sleep 20
-$TEST_TARGET_SO deployment --dir $test_deployment_dir start
-log_output_5=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
-if [[ "$log_output_5" == *"filesystem is old"* ]]; then
-    echo "Retain volumes test: passed"
-else
-    echo "Retain volumes test: FAILED"
-    delete_cluster_exit
-fi
-
-# Stop and clean up
-$TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes
-echo "Test passed"