Skip checks on requests we've already seen (#805 )

Reviewed-on: #805 Co-authored-by: Thomas E Lackey <telackey@bozemanpass.com> Co-committed-by: Thomas E Lackey <telackey@bozemanpass.com>
Switch repo to cerc-io org. (#804 )
2024-04-15 20:27:35 +00:00 · 2024-04-15 18:59:08 +00:00 · 2024-04-15 12:20:35 +00:00 · 2024-04-11 07:59:36 +00:00 · 2024-04-11 03:21:36 +00:00 · 2024-04-08 15:51:10 +00:00
27 changed files with 75 additions and 600 deletions
@@ -1,58 +0,0 @@
-name: External Stack Test
-
-on:
-  push:
-    branches: '*'
-    paths:
-      - '!**'
-      - '.gitea/workflows/triggers/test-external-stack'
-      - '.gitea/workflows/test-external-stack.yml'
-      - 'tests/external-stack/run-test.sh'
-  schedule: # Note: coordinate with other tests to not overload runners at the same time of day
-    - cron: '8 19 * * *'
-
-jobs:
-  test:
-    name: "Run external stack test suite"
-    runs-on: ubuntu-latest
-    steps:
-      - name: "Clone project repository"
-        uses: actions/checkout@v3
-      # At present the stock setup-python action fails on Linux/aarch64
-      # Conditional steps below workaroud this by using deadsnakes for that case only
-      - name: "Install Python for ARM on Linux"
-        if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
-        uses: deadsnakes/action@v3.0.1
-        with:
-          python-version: '3.8'
-      - name: "Install Python cases other than ARM on Linux"
-        if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.8'
-      - name: "Print Python version"
-        run: python3 --version
-      - name: "Install shiv"
-        run: pip install shiv
-      - name: "Generate build version file"
-        run: ./scripts/create_build_tag_file.sh
-      - name: "Build local shiv package"
-        run: ./scripts/build_shiv_package.sh
-      - name: "Run external stack tests"
-        run: ./tests/external-stack/run-test.sh
-      - name: Notify Vulcanize Slack on CI failure
-        if: ${{ always() && github.ref_name == 'main' }}
-        uses: ravsamhq/notify-slack-action@v2
-        with:
-          status: ${{ job.status }}
-          notify_when: 'failure'
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
-      - name: Notify DeepStack Slack on CI failure
-        if: ${{ always() && github.ref_name == 'main' }}
-        uses: ravsamhq/notify-slack-action@v2
-        with:
-          status: ${{ job.status }}
-          notify_when: 'failure'
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
@@ -1,2 +0,0 @@
-Change this file to trigger running the external-stack CI job
-trigger
@@ -71,7 +71,7 @@ def process_container(build_context: BuildContext) -> bool:

    # Check if this is in an external stack
    if stack_is_external(build_context.stack):
-        container_parent_dir = Path(build_context.stack).parent.parent.joinpath("container-build")
+        container_parent_dir = Path(build_context.stack).joinpath("container-build")
        temp_build_dir = container_parent_dir.joinpath(build_context.container.replace("/", "-"))
        temp_build_script_filename = temp_build_dir.joinpath("build.sh")
        # Now check if the container exists in the external stack.
@@ -5,7 +5,7 @@ services:
    hostname: lotus-miner
    env_file:
      - ../config/fixturenet-lotus/lotus-env.env
-    image: cerc/lotus-fixturenet:local
+    image: cerc/lotus:local
    volumes:
      - ../config/fixturenet-lotus/setup-miner.sh:/docker-entrypoint-scripts.d/setup-miner.sh
      - ../config/fixturenet-lotus/fund-account.sh:/fund-account.sh
@@ -26,7 +26,7 @@ services:
    environment:
      # Use 0.0.0.0 so that calls can be made from outside the container
      - LOTUS_API_LISTENADDRESS=/ip4/0.0.0.0/tcp/1234/http
-    image: cerc/lotus-fixturenet:local
+    image: cerc/lotus:local
    volumes:
      - ../config/fixturenet-lotus/setup-node.sh:/docker-entrypoint-scripts.d/setup-node.sh
      - lotus_node_1_params:/var/tmp/filecoin-proof-parameters
@@ -42,10 +42,10 @@ services:
      - lotus-miner
    entrypoint: ["sh", "/docker-entrypoint-scripts.d/setup-node.sh"]
    ports:
-      - "1235"
-      - "2346"
-      - "3457"
-      - "1778"
+      - "1234"
+      - "2345"
+      - "3456"
+      - "1777"

  lotus-node-2:
    hostname: lotus-node-2
@@ -54,7 +54,7 @@ services:
    environment:
      # Use 0.0.0.0 so that calls can be made from outside the container
      - LOTUS_API_LISTENADDRESS=/ip4/0.0.0.0/tcp/1234/http
-    image: cerc/lotus-fixturenet:local
+    image: cerc/lotus:local
    volumes:
      - ../config/fixturenet-lotus/setup-node.sh:/docker-entrypoint-scripts.d/setup-node.sh
      - lotus_node_2_params:/var/tmp/filecoin-proof-parameters
@@ -70,10 +70,10 @@ services:
      - lotus-miner
    entrypoint: ["sh", "/docker-entrypoint-scripts.d/setup-node.sh"]
    ports:
-      - "1236"
-      - "2347"
-      - "3458"
-      - "1779"
+      - "1234"
+      - "2345"
+      - "3456"
+      - "1777"

 volumes:
  lotus_miner_params:
@@ -16,12 +16,8 @@ services:
      postgres_pass: password
      postgres_db: graph-node
      ethereum: ${ETH_NETWORKS:-lotus-fixturenet:http://lotus-node-1:1234/rpc/v1}
-      # Env varaibles reference: https://git.vdb.to/cerc-io/graph-node/src/branch/master/docs/environment-variables.md
      GRAPH_LOG: debug
      ETHEREUM_REORG_THRESHOLD: 3
-      GRAPH_ETHEREUM_JSON_RPC_TIMEOUT: ${GRAPH_ETHEREUM_JSON_RPC_TIMEOUT:-180}
-      GRAPH_ETHEREUM_REQUEST_RETRIES: ${GRAPH_ETHEREUM_REQUEST_RETRIES:-10}
-      GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE: ${GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE:-2000}
    entrypoint: ["bash", "-c"]
    # Wait for ETH RPC endpoint to be up when running with fixturenet-lotus
    command: |
@@ -1,23 +0,0 @@
-volumes:
-  parameters:
-  lotus-repo:
-  lotus_shared:
-
-services:
-  lotus:
-    environment:
-      - LOTUS_API_LISTENADDRESS=/ip4/0.0.0.0/tcp/1234/http
-      - LOTUS_LIBP2P_LISTENADDRESSES=/ip4/0.0.0.0/tcp/1235,/ip6/::/tcp/1235,/ip4/0.0.0.0/udp/1235/quic-v1,/ip6/::/udp/1235/quic-v1,/ip4/0.0.0.0/udp/1235/quic-v1/webtransport,/ip6/::/udp/1235/quic-v1/webtransport
-    image: cerc/lotus-mainnet:local
-    volumes:
-      - parameters:/var/tmp/filecoin-proof-parameters
-      - lotus-repo:/var/lib/lotus
-      - lotus_shared:/root/.lotus
-    ports:
-      - "1234"
-      - "1235"
-    deploy:
-      restart_policy:
-        condition: on-failure
-        delay: 30s
-    entrypoint: ["sh", "/docker-lotus-entrypoint.sh"]
@@ -1,5 +1,5 @@
 #####################################
-FROM golang:1.21.7-bullseye AS lotus-builder
+FROM golang:1.19.12-bullseye AS lotus-builder
 MAINTAINER Lotus Development Team

 RUN apt-get update && apt-get install -y ca-certificates build-essential clang ocl-icd-opencl-dev ocl-icd-libopencl1 jq libhwloc-dev
@@ -29,6 +29,7 @@ RUN set -eux; \
    rustup --version; \
    cargo --version; \
    rustc --version;
+
 COPY ./ /opt/filecoin
 WORKDIR /opt/filecoin

@@ -37,13 +38,13 @@ WORKDIR /opt/filecoin
 ### make configurable filecoin-ffi build
 ARG FFI_BUILD_FROM_SOURCE=0
 ENV FFI_BUILD_FROM_SOURCE=${FFI_BUILD_FROM_SOURCE}
-RUN ls /opt/filecoin
+
 RUN make clean deps

 ARG RUSTFLAGS=""
 ARG GOFLAGS=""

-#RUN make lotus
+#RUN make buildall
 RUN make 2k

 #####################################
@@ -74,7 +75,7 @@ COPY --from=lotus-builder /opt/filecoin/lotus-shed /usr/local/bin/
 #COPY scripts/docker-lotus-entrypoint.sh /
 #COPY myscripts/setup-node.sh /docker-entrypoint-scripts.d/setup-node.sh

-ARG DOCKER_LOTUS_IMPORT_SNAPSHOT https://forest-archive.chainsafe.dev/latest/mainnet/
+ARG DOCKER_LOTUS_IMPORT_SNAPSHOT https://snapshots.mainnet.filops.net/minimal/latest
 ENV DOCKER_LOTUS_IMPORT_SNAPSHOT ${DOCKER_LOTUS_IMPORT_SNAPSHOT}
 ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
 ENV LOTUS_PATH /var/lib/lotus
@@ -1,12 +1,12 @@
 #!/usr/bin/env bash
-# Build cerc/fixturenet-lotus
+# Build cerc/lotus
 source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )

 # Use a release version tag to match the modified Dockerfile replaced in next step
-git -C ${CERC_REPO_BASE_DIR}/lotus checkout master
+git -C ${CERC_REPO_BASE_DIR}/lotus checkout v1.23.3

 # Replace repo's Dockerfile with modified one
 cp ${SCRIPT_DIR}/Dockerfile ${CERC_REPO_BASE_DIR}/lotus/Dockerfile

-docker build -t cerc/fixturenet-lotus:local ${build_command_args} ${CERC_REPO_BASE_DIR}/lotus
+docker build -t cerc/lotus:local ${build_command_args} ${CERC_REPO_BASE_DIR}/lotus
@@ -1,124 +0,0 @@
-#####################################
-FROM golang:1.21.7-bullseye AS lotus-builder
-MAINTAINER Lotus Development Team
-
-RUN apt-get update && apt-get install -y ca-certificates build-essential clang ocl-icd-opencl-dev ocl-icd-libopencl1 jq libhwloc-dev
-
-ENV XDG_CACHE_HOME="/tmp"
-
-### taken from https://github.com/rust-lang/docker-rust/blob/master/1.63.0/buster/Dockerfile
-ENV RUSTUP_HOME=/usr/local/rustup \
-    CARGO_HOME=/usr/local/cargo \
-    PATH=/usr/local/cargo/bin:$PATH \
-    RUST_VERSION=1.63.0
-
-RUN set -eux; \
-    dpkgArch="$(dpkg --print-architecture)"; \
-    case "${dpkgArch##*-}" in \
-        amd64) rustArch='x86_64-unknown-linux-gnu'; rustupSha256='5cc9ffd1026e82e7fb2eec2121ad71f4b0f044e88bca39207b3f6b769aaa799c' ;; \
-        arm64) rustArch='aarch64-unknown-linux-gnu'; rustupSha256='e189948e396d47254103a49c987e7fb0e5dd8e34b200aa4481ecc4b8e41fb929' ;; \
-        *) echo >&2 "unsupported architecture: ${dpkgArch}"; exit 1 ;; \
-    esac; \
-    url="https://static.rust-lang.org/rustup/archive/1.25.1/${rustArch}/rustup-init"; \
-    wget "$url"; \
-    echo "${rustupSha256} *rustup-init" | sha256sum -c -; \
-    chmod +x rustup-init; \
-    ./rustup-init -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION --default-host ${rustArch}; \
-    rm rustup-init; \
-    chmod -R a+w $RUSTUP_HOME $CARGO_HOME; \
-    rustup --version; \
-    cargo --version; \
-    rustc --version;
-
-COPY ./ /opt/filecoin
-WORKDIR /opt/filecoin
-
-ARG FFI_BUILD_FROM_SOURCE=0
-ENV FFI_BUILD_FROM_SOURCE=${FFI_BUILD_FROM_SOURCE}
-
-#RUN make clean deps
-
-ARG RUSTFLAGS=""
-ARG GOFLAGS=""
-
-RUN make clean deps && \
-    make lotus lotus-shed lotus-stats && \
-    install -C ./lotus /usr/local/bin/lotus && \
-    install -C ./lotus-shed /usr/local/bin/lotus-shed && \
-    install -C ./lotus-stats /usr/local/bin/lotus-stats
-
-#####################################
-FROM ubuntu:20.04 AS lotus-base
-MAINTAINER Lotus Development Team
-
-# Base resources
-COPY --from=lotus-builder /etc/ssl/certs                           /etc/ssl/certs
-COPY --from=lotus-builder /lib/*/libdl.so.2         /lib/
-COPY --from=lotus-builder /lib/*/librt.so.1         /lib/
-COPY --from=lotus-builder /lib/*/libgcc_s.so.1      /lib/
-COPY --from=lotus-builder /lib/*/libutil.so.1       /lib/
-COPY --from=lotus-builder /usr/lib/*/libltdl.so.7   /lib/
-COPY --from=lotus-builder /usr/lib/*/libnuma.so.1   /lib/
-COPY --from=lotus-builder /usr/lib/*/libhwloc.so.*  /lib/
-COPY --from=lotus-builder /usr/lib/*/libOpenCL.so.1 /lib/
-
-RUN useradd -r -u 532 -U fc \
- && mkdir -p /etc/OpenCL/vendors \
- && echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd
-
-#####################################
-FROM lotus-base AS lotus
-MAINTAINER Lotus Development Team
-
-COPY --from=lotus-builder \
-    /usr/local/bin/lotus \
-    /usr/local/bin/lotus-shed \
-    /usr/local/bin/
-
-ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
-ENV LOTUS_PATH /var/lib/lotus
-ENV DOCKER_LOTUS_IMPORT_WALLET ""
-
-RUN mkdir /var/lib/lotus /var/tmp/filecoin-proof-parameters
-RUN chown fc: /var/lib/lotus /var/tmp/filecoin-proof-parameters
-
-VOLUME /var/lib/lotus
-VOLUME /var/tmp/filecoin-proof-parameters
-
-USER fc
-
-EXPOSE 1234
-
-
-CMD ["-help"]
-
-#####################################
-FROM lotus-base AS lotus-all-in-one
-
-# Install netcat for healthcheck
-RUN apt-get update && apt-get install -y netcat && apt-get install -y iproute2
-
-ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
-ENV LOTUS_PATH /var/lib/lotus
-
-ARG DOCKER_LOTUS_IMPORT_SNAPSHOT=https://forest-archive.chainsafe.dev/latest/mainnet/
-ENV DOCKER_LOTUS_IMPORT_SNAPSHOT=${DOCKER_LOTUS_IMPORT_SNAPSHOT}
-
-COPY --from=lotus-builder /opt/filecoin/lotus          /usr/local/bin/
-COPY --from=lotus-builder /opt/filecoin/lotus-shed     /usr/local/bin/
-COPY --from=lotus-builder /opt/filecoin/lotus-stats    /usr/local/bin/
-COPY scripts/docker-lotus-entrypoint.sh /docker-lotus-entrypoint.sh
-RUN chmod +x /docker-lotus-entrypoint.sh
-
-RUN mkdir /var/tmp/filecoin-proof-parameters
-RUN mkdir /var/lib/lotus
-RUN chown fc: /var/tmp/filecoin-proof-parameters
-RUN chown fc: /var/lib/lotus
-
-
-VOLUME /var/tmp/filecoin-proof-parameters
-VOLUME /var/lib/lotus
-
-EXPOSE 1234
-EXPOSE 1235
-
@@ -1,14 +0,0 @@
-#!/usr/bin/env bash
-# Build cerc/mainnet-lotus
-source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
-SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-
-git stash
-
-# Use a release version tag to match the modified Dockerfile replaced in next step
-git -C ${CERC_REPO_BASE_DIR}/lotus checkout master
-
-# Replace repo's Dockerfile with modified one
-cp ${SCRIPT_DIR}/Dockerfile ${CERC_REPO_BASE_DIR}/lotus/Dockerfile
-
-docker build -t cerc/lotus-mainnet:local ${build_command_args} ${CERC_REPO_BASE_DIR}/lotus
@@ -2,4 +2,4 @@
 # Build cerc/test-container
 source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-docker build -t cerc/test-container:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} $SCRIPT_DIR
+docker build -t cerc/test-container:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} $SCRIPT_DIR
@@ -41,8 +41,7 @@ cerc/pocket
 cerc/watcher-azimuth
 cerc/ipld-eth-state-snapshot
 cerc/watcher-gelato
-cerc/mainnet-lotus
-cerc/fixturenet-lotus
+cerc/lotus
 cerc/go-opera
 cerc/lasso
 cerc/reth
@@ -27,7 +27,6 @@ fixturenet-pocket
 watcher-azimuth
 watcher-gelato
 fixturenet-lotus
-mainnet-lotus
 mainnet-go-opera
 lasso
 reth
@@ -2,8 +2,8 @@ version: "1.0"
 name: fixturenet-lotus
 description: "A lotus fixturenet"
 repos:
-  - git.vdb.to/cerc-io/lotus
+  - github.com/filecoin-project/lotus
 containers:
-  - cerc/fixturenet-lotus
+  - cerc/lotus
 pods:
  - fixturenet-lotus
@@ -13,7 +13,7 @@ repos:
  - github.com/sushiswap/subgraphs
 containers:
  # fixturenet-lotus image
-  - cerc/lotus-fixturenet
+  - cerc/lotus
  # graph-node image
  - cerc/graph-node
  # sushiswap contract deployment images
@@ -76,19 +76,6 @@ export ETH_RPC_PORT=
 # The etherum network(s) graph-node will connect to
 # Set this to a space-separated list of the networks where each entry has the form NAME:URL
 export ETH_NETWORKS=
-
-# Optional:
-
-# Timeout for ETH RPC requests in seconds (default: 180s)
-export GRAPH_ETHEREUM_JSON_RPC_TIMEOUT=
-
-# Number of times to retry ETH RPC requests (default: 10)
-export GRAPH_ETHEREUM_REQUEST_RETRIES=
-
-# Maximum number of blocks to scan for triggers in each request (default: 2000)
-export GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE=
-
-# Ref: https://git.vdb.to/cerc-io/graph-node/src/branch/master/docs/environment-variables.md
 ```

 Example env file:
@@ -98,10 +85,6 @@ export ETH_RPC_HOST=filecoin.chainup.net
 export ETH_RPC_PORT=443

 export ETH_NETWORKS=filecoin:https://filecoin.chainup.net/rpc/v1
-
-export GRAPH_ETHEREUM_JSON_RPC_TIMEOUT=360
-export GRAPH_ETHEREUM_REQUEST_RETRIES=5
-export GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE=50
 ```

 Set the environment variables:
@@ -1,26 +0,0 @@
-# Lotus stack
-
-## Clone required repositories
-```
-$ laconic-so --stack mainnet-lotus setup-repositories
-```
-## Build the stack's containers
-```
-$ laconic-so --stack mainnet-lotus build-containers
-```
-## Create a deployment of the stack
-```
-$ laconic-so --stack mainnet-lotus deploy init --map-ports-to-host any-same --output lotus-spec.yml
-```
-Update the URLs in the generated spec file to use your systems' actual IP addresses.
-```
-$ laconic-so --stack mainnet-lotus deploy create --deployment-dir lotus-deployment --spec-file lotus-spec.yml
-```
-## Start the stack
-```
-$ laconic-so deployment --dir lotus-deployment start
-```
-Check logs:
-```
-$ laconic-so deployment --dir lotus-deployment logs
-```
@@ -1,9 +0,0 @@
-version: "1.0"
-name: mainnet-lotus
-description: "A lotus stack"
-repos:
-  - git.vdb.to/cerc-io/lotus
-containers:
-  - cerc/mainnet-lotus
-pods:
-  - mainnet-lotus
@@ -11,7 +11,7 @@ repos:
  - github.com/vulcanize/uniswap-v3-info
 containers:
  ## fixturenet-lotus image
-  - cerc/lotus-fixturenet
+  - cerc/lotus
  ## sushiswap images
  - cerc/sushiswap-v3-core
  - cerc/sushiswap-v3-periphery
@@ -27,7 +27,6 @@ from pathlib import Path
 from stack_orchestrator import constants
 from stack_orchestrator.opts import opts
 from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config, global_options2, get_dev_root_path
-from stack_orchestrator.util import resolve_compose_file
 from stack_orchestrator.deploy.deployer import Deployer, DeployerException
 from stack_orchestrator.deploy.deployer_factory import getDeployer
 from stack_orchestrator.deploy.deploy_types import ClusterContext, DeployCommandContext
@@ -325,10 +324,7 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file):
        pod_path = pod["path"]
        if include_exclude_check(pod_name, include, exclude):
            if pod_repository is None or pod_repository == "internal":
-                if deployment:
-                    compose_file_name = os.path.join(compose_dir, f"docker-compose-{pod_path}.yml")
-                else:
-                    compose_file_name = resolve_compose_file(stack, pod_name)
+                compose_file_name = os.path.join(compose_dir, f"docker-compose-{pod_path}.yml")
            else:
                if deployment:
                    compose_file_name = os.path.join(compose_dir, f"docker-compose-{pod_name}.yml")
@@ -340,7 +336,6 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file):
                    if pod_post_start_command is not None:
                        post_start_commands.append(os.path.join(script_dir, pod_post_start_command))
                else:
-                    # TODO: fix this code for external stack with scripts
                    pod_root_dir = os.path.join(dev_root_path, pod_repository.split("/")[-1], pod["path"])
                    compose_file_name = os.path.join(pod_root_dir, f"docker-compose-{pod_name}.yml")
                    pod_pre_start_command = pod.get("pre_start_command")
@@ -16,7 +16,7 @@
 import os
 from typing import List, Any
 from stack_orchestrator.deploy.deploy_types import DeployCommandContext, VolumeMapping
-from stack_orchestrator.util import get_parsed_stack_config, get_yaml, get_pod_list, resolve_compose_file
+from stack_orchestrator.util import get_parsed_stack_config, get_yaml, get_compose_file_dir, get_pod_list
 from stack_orchestrator.opts import opts


@@ -27,7 +27,7 @@ def _container_image_from_service(stack: str, service: str):
    pods = get_pod_list(parsed_stack)
    yaml = get_yaml()
    for pod in pods:
-        pod_file_path = resolve_compose_file(stack, pod)
+        pod_file_path = os.path.join(get_compose_file_dir(), f"docker-compose-{pod}.yml")
        parsed_pod_file = yaml.load(open(pod_file_path, "r"))
        if "services" in parsed_pod_file:
            services = parsed_pod_file["services"]
@@ -26,8 +26,7 @@ from stack_orchestrator import constants
 from stack_orchestrator.opts import opts
 from stack_orchestrator.util import (get_stack_file_path, get_parsed_deployment_spec, get_parsed_stack_config,
                                     global_options, get_yaml, get_pod_list, get_pod_file_path, pod_has_scripts,
-                                     get_pod_script_paths, get_plugin_code_paths, error_exit, env_var_map_from_file,
-                                     resolve_config_dir)
+                                     get_pod_script_paths, get_plugin_code_paths, error_exit, env_var_map_from_file)
 from stack_orchestrator.deploy.spec import Spec
 from stack_orchestrator.deploy.deploy_types import LaconicStackSetupCommand
 from stack_orchestrator.deploy.deployer_factory import getDeployerConfigGenerator
@@ -44,7 +43,7 @@ def _get_ports(stack):
    pods = get_pod_list(parsed_stack)
    yaml = get_yaml()
    for pod in pods:
-        pod_file_path = get_pod_file_path(stack, parsed_stack, pod)
+        pod_file_path = get_pod_file_path(parsed_stack, pod)
        parsed_pod_file = yaml.load(open(pod_file_path, "r"))
        if "services" in parsed_pod_file:
            for svc_name, svc in parsed_pod_file["services"].items():
@@ -80,7 +79,7 @@ def _get_named_volumes(stack):
        return ret

    for pod in pods:
-        pod_file_path = get_pod_file_path(stack, parsed_stack, pod)
+        pod_file_path = get_pod_file_path(parsed_stack, pod)
        parsed_pod_file = yaml.load(open(pod_file_path, "r"))
        if "volumes" in parsed_pod_file:
            volumes = parsed_pod_file["volumes"]
@@ -481,9 +480,10 @@ def create_operation(deployment_command_context, spec_file, deployment_dir, netw
    os.mkdir(destination_compose_dir)
    destination_pods_dir = deployment_dir_path.joinpath("pods")
    os.mkdir(destination_pods_dir)
+    data_dir = Path(__file__).absolute().parent.parent.joinpath("data")
    yaml = get_yaml()
    for pod in pods:
-        pod_file_path = get_pod_file_path(stack_name, parsed_stack, pod)
+        pod_file_path = get_pod_file_path(parsed_stack, pod)
        parsed_pod_file = yaml.load(open(pod_file_path, "r"))
        extra_config_dirs = _find_extra_config_dirs(parsed_pod_file, pod)
        destination_pod_dir = destination_pods_dir.joinpath(pod)
@@ -497,7 +497,7 @@ def create_operation(deployment_command_context, spec_file, deployment_dir, netw
        config_dirs = {pod}
        config_dirs = config_dirs.union(extra_config_dirs)
        for config_dir in config_dirs:
-            source_config_dir = resolve_config_dir(stack_name, config_dir)
+            source_config_dir = data_dir.joinpath("config", config_dir)
            if os.path.exists(source_config_dir):
                destination_config_dir = deployment_dir_path.joinpath("config", config_dir)
                # If the same config dir appears in multiple pods, it may already have been copied
@@ -17,7 +17,6 @@ import click

 from stack_orchestrator.command_types import CommandOptions
 from stack_orchestrator.repos import setup_repositories
-from stack_orchestrator.repos import fetch_stack
 from stack_orchestrator.build import build_containers, fetch_containers
 from stack_orchestrator.build import build_npms
 from stack_orchestrator.build import build_webapp
@@ -51,7 +50,6 @@ def cli(ctx, stack, quiet, verbose, dry_run, local_stack, debug, continue_on_err
    ctx.obj = command_options


-cli.add_command(fetch_stack.command, "fetch-stack")
 cli.add_command(setup_repositories.command, "setup-repositories")
 cli.add_command(build_containers.command, "build-containers")
 cli.add_command(fetch_containers.command, "fetch-containers")
@@ -1,45 +0,0 @@
-# Copyright © 2022, 2023 Vulcanize
-
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
-
-# env vars:
-# CERC_REPO_BASE_DIR defaults to ~/cerc
-
-
-import click
-import os
-
-from decouple import config
-from git import exc
-
-from stack_orchestrator.opts import opts
-from stack_orchestrator.repos.setup_repositories import process_repo
-from stack_orchestrator.util import error_exit
-
-
-@click.command()
-@click.argument('stack-locator')
-@click.option('--git-ssh', is_flag=True, default=False)
-@click.option('--check-only', is_flag=True, default=False)
-@click.option('--pull', is_flag=True, default=False)
-@click.pass_context
-def command(ctx, stack_locator, git_ssh, check_only, pull):
-    '''optionally resolve then git clone a repository containing one or more stack definitions'''
-    dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc"))
-    if not opts.o.quiet:
-        print(f"Dev Root is: {dev_root_path}")
-    try:
-        process_repo(pull, check_only, git_ssh, dev_root_path, None, stack_locator)
-    except exc.GitCommandError as error:
-        error_exit(f"\n******* git command returned error exit status:\n{error}")
@@ -26,7 +26,6 @@ import importlib.resources
 from pathlib import Path
 import yaml
 from stack_orchestrator.constants import stack_file_name
-from stack_orchestrator.opts import opts
 from stack_orchestrator.util import include_exclude_check, stack_is_external, error_exit, warn_exit


@@ -88,8 +87,8 @@ def _get_repo_current_branch_or_tag(full_filesystem_repo_path):


 # TODO: fix the messy arg list here
-def process_repo(pull, check_only, git_ssh, dev_root_path, branches_array, fully_qualified_repo):
-    if opts.o.verbose:
+def process_repo(verbose, quiet, dry_run, pull, check_only, git_ssh, dev_root_path, branches_array, fully_qualified_repo):
+    if verbose:
        print(f"Processing repo: {fully_qualified_repo}")
    repo_host, repo_path, repo_branch = host_and_path_for_repo(fully_qualified_repo)
    git_ssh_prefix = f"git@{repo_host}:"
@@ -101,7 +100,7 @@ def process_repo(pull, check_only, git_ssh, dev_root_path, branches_array, fully
    (current_repo_branch_or_tag, is_branch) = _get_repo_current_branch_or_tag(
        full_filesystem_repo_path
        ) if is_present else (None, None)
-    if not opts.o.quiet:
+    if not quiet:
        present_text = f"already exists active {'branch' if is_branch else 'tag'}: {current_repo_branch_or_tag}" if is_present \
            else 'Needs to be fetched'
        print(f"Checking: {full_filesystem_repo_path}: {present_text}")
@@ -112,25 +111,25 @@ def process_repo(pull, check_only, git_ssh, dev_root_path, branches_array, fully
            sys.exit(1)
        else:
            if pull:
-                if opts.o.verbose:
+                if verbose:
                    print(f"Running git pull for {full_filesystem_repo_path}")
                if not check_only:
                    if is_branch:
                        git_repo = git.Repo(full_filesystem_repo_path)
                        origin = git_repo.remotes.origin
-                        origin.pull(progress=None if opts.o.quiet else GitProgress())
+                        origin.pull(progress=None if quiet else GitProgress())
                    else:
                        print("skipping pull because this repo checked out a tag")
                else:
                    print("(git pull skipped)")
    if not is_present:
        # Clone
-        if opts.o.verbose:
+        if verbose:
            print(f'Running git clone for {full_github_repo_path} into {full_filesystem_repo_path}')
-        if not opts.o.dry_run:
+        if not dry_run:
            git.Repo.clone_from(full_github_repo_path,
                                full_filesystem_repo_path,
-                                progress=None if opts.o.quiet else GitProgress())
+                                progress=None if quiet else GitProgress())
        else:
            print("(git clone skipped)")
    # Checkout the requested branch, if one was specified
@@ -151,13 +150,13 @@ def process_repo(pull, check_only, git_ssh, dev_root_path, branches_array, fully
            current_repo_branch_or_tag and (
                current_repo_branch_or_tag != branch_to_checkout)
                ):
-            if not opts.o.quiet:
+            if not quiet:
                print(f"switching to branch {branch_to_checkout} in repo {repo_path}")
            git_repo = git.Repo(full_filesystem_repo_path)
            # git checkout works for both branches and tags
            git_repo.git.checkout(branch_to_checkout)
        else:
-            if opts.o.verbose:
+            if verbose:
                print(f"repo {repo_path} is already on branch/tag {branch_to_checkout}")


@@ -183,18 +182,36 @@ def parse_branches(branches_string):
@click.option('--check-only', is_flag=True, default=False)
@click.option('--pull', is_flag=True, default=False)
@click.option("--branches", help="override branches for repositories")
+@click.option('--branches-file', help="checkout branches specified in this file")
@click.pass_context
-def command(ctx, include, exclude, git_ssh, check_only, pull, branches):
+def command(ctx, include, exclude, git_ssh, check_only, pull, branches, branches_file):
    '''git clone the set of repositories required to build the complete system from source'''

-    quiet = opts.o.quiet
-    verbose = opts.o.verbose
-    stack = opts.o.stack
+    quiet = ctx.obj.quiet
+    verbose = ctx.obj.verbose
+    dry_run = ctx.obj.dry_run
+    stack = ctx.obj.stack

    branches_array = []

+    # TODO: branches file needs to be re-worked in the context of stacks
+    if branches_file:
+        if branches:
+            print("Error: can't specify both --branches and --branches-file")
+            sys.exit(1)
+        else:
+            if verbose:
+                print(f"loading branches from: {branches_file}")
+            with open(branches_file) as branches_file_open:
+                branches_array = branches_file_open.read().splitlines()
+
+    print(f"branches: {branches}")
    if branches:
-        branches_array = parse_branches(branches)
+        if branches_file:
+            print("Error: can't specify both --branches and --branches-file")
+            sys.exit(1)
+        else:
+            branches_array = parse_branches(branches)

    if branches_array and verbose:
        print(f"Branches are: {branches_array}")
@@ -254,6 +271,7 @@ def command(ctx, include, exclude, git_ssh, check_only, pull, branches):

    for repo in repos:
        try:
-            process_repo(pull, check_only, git_ssh, dev_root_path, branches_array, repo)
+            process_repo(verbose, quiet, dry_run, pull, check_only, git_ssh, dev_root_path, branches_array, repo)
        except git.exc.GitCommandError as error:
-            error_exit(f"\n******* git command returned error exit status:\n{error}")
+            print(f"\n******* git command returned error exit status:\n{error}")
+            sys.exit(1)
@@ -94,38 +94,10 @@ def get_plugin_code_paths(stack) -> List[Path]:
    return list(result)


-# # Find a config directory, looking first in any external stack
-# and if not found there, internally
-def resolve_config_dir(stack, config_dir_name: str):
-    if stack_is_external(stack):
-        # First try looking in the external stack for the compose file
-        config_base = Path(stack).parent.parent.joinpath("config")
-        proposed_dir = config_base.joinpath(config_dir_name)
-        if proposed_dir.exists():
-            return proposed_dir
-        # If we don't find it fall through to the internal case
-    config_base = get_internal_config_dir()
-    return config_base.joinpath(config_dir_name)
-
-
-# Find a compose file, looking first in any external stack
-# and if not found there, internally
-def resolve_compose_file(stack, pod_name: str):
-    if stack_is_external(stack):
-        # First try looking in the external stack for the compose file
-        compose_base = Path(stack).parent.parent.joinpath("compose")
-        proposed_file = compose_base.joinpath(f"docker-compose-{pod_name}.yml")
-        if proposed_file.exists():
-            return proposed_file
-        # If we don't find it fall through to the internal case
-    compose_base = get_internal_compose_file_dir()
-    return compose_base.joinpath(f"docker-compose-{pod_name}.yml")
-
-
-def get_pod_file_path(stack, parsed_stack, pod_name: str):
+def get_pod_file_path(parsed_stack, pod_name: str):
    pods = parsed_stack["pods"]
    if type(pods[0]) is str:
-        result = resolve_compose_file(stack, pod_name)
+        result = os.path.join(get_compose_file_dir(), f"docker-compose-{pod_name}.yml")
    else:
        for pod in pods:
            if pod["name"] == pod_name:
@@ -159,7 +131,7 @@ def pod_has_scripts(parsed_stack, pod_name: str):
    return result


-def get_internal_compose_file_dir():
+def get_compose_file_dir():
    # TODO: refactor to use common code with deploy command
    # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
    data_dir = Path(__file__).absolute().parent.joinpath("data")
@@ -167,7 +139,7 @@ def get_internal_compose_file_dir():
    return source_compose_dir


-def get_internal_config_dir():
+def get_config_file_dir():
    # TODO: refactor to use common code with deploy command
    data_dir = Path(__file__).absolute().parent.joinpath("data")
    source_config_dir = data_dir.joinpath("config")
@@ -1,185 +0,0 @@
-#!/usr/bin/env bash
-set -e
-if [ -n "$CERC_SCRIPT_DEBUG" ]; then
-  set -x
-fi
-# Dump environment variables for debugging
-echo "Environment variables:"
-env
-
-if [ "$1" == "from-path" ]; then
-    TEST_TARGET_SO="laconic-so"
-else
-    TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 )
-fi
-
-delete_cluster_exit () {
-    $TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes
-    exit 1
-}
-
-# Test basic stack-orchestrator deploy
-echo "Running stack-orchestrator external stack deploy test"
-# Set a non-default repo dir
-export CERC_REPO_BASE_DIR=~/stack-orchestrator-test/repo-base-dir
-echo "Testing this package: $TEST_TARGET_SO"
-echo "Test version command"
-reported_version_string=$( $TEST_TARGET_SO version )
-echo "Version reported is: ${reported_version_string}"
-echo "Cloning repositories into: $CERC_REPO_BASE_DIR"
-rm -rf $CERC_REPO_BASE_DIR
-mkdir -p $CERC_REPO_BASE_DIR
-# Clone the external test stack
-$TEST_TARGET_SO fetch-stack git.vdb.to/cerc-io/test-external-stack
-stack_name="$CERC_REPO_BASE_DIR/test-external-stack/stack-orchestrator/stacks/test-external-stack"
-TEST_TARGET_SO_STACK="$TEST_TARGET_SO --stack ${stack_name}"
-# Test bringing the test container up and down
-# with and without volume removal
-$TEST_TARGET_SO_STACK setup-repositories
-$TEST_TARGET_SO_STACK build-containers
-# Test deploy command execution
-$TEST_TARGET_SO_STACK deploy setup $CERC_REPO_BASE_DIR
-# Check that we now have the expected output directory
-container_output_dir=$CERC_REPO_BASE_DIR/container-output-dir
-if [ ! -d "$container_output_dir" ]; then
-    echo "deploy setup test: output directory not present"
-    echo "deploy setup test: FAILED"
-    exit 1
-fi
-if [ ! -f "$container_output_dir/output-file" ]; then
-    echo "deploy setup test: output file not present"
-    echo "deploy setup test: FAILED"
-    exit 1
-fi
-output_file_content=$(<$container_output_dir/output-file)
-if [ ! "$output_file_content" == "output-data"  ]; then
-    echo "deploy setup test: output file contents not correct"
-    echo "deploy setup test: FAILED"
-    exit 1
-fi
-# Check that we now have the expected output file
-$TEST_TARGET_SO_STACK deploy up
-# Test deploy port command
-deploy_port_output=$( $TEST_TARGET_SO_STACK deploy port test 80 )
-if [[ "$deploy_port_output" =~ ^0.0.0.0:[1-9][0-9]* ]]; then
-    echo "Deploy port test: passed"
-else
-    echo "Deploy port test: FAILED"
-    exit 1
-fi
-$TEST_TARGET_SO_STACK deploy down
-# The next time we bring the container up the volume will be old (from the previous run above)
-$TEST_TARGET_SO_STACK deploy up
-log_output_1=$( $TEST_TARGET_SO_STACK deploy logs )
-if [[ "$log_output_1" == *"filesystem is old"* ]]; then
-    echo "Retain volumes test: passed"
-else
-    echo "Retain volumes test: FAILED"
-    exit 1
-fi
-$TEST_TARGET_SO_STACK deploy down --delete-volumes
-# Now when we bring the container up the volume will be new again
-$TEST_TARGET_SO_STACK deploy up
-log_output_2=$( $TEST_TARGET_SO_STACK deploy logs )
-if [[ "$log_output_2" == *"filesystem is fresh"* ]]; then
-    echo "Delete volumes test: passed"
-else
-    echo "Delete volumes test: FAILED"
-    exit 1
-fi
-$TEST_TARGET_SO_STACK deploy down --delete-volumes
-# Basic test of creating a deployment
-test_deployment_dir=$CERC_REPO_BASE_DIR/test-deployment-dir
-test_deployment_spec=$CERC_REPO_BASE_DIR/test-deployment-spec.yml
-$TEST_TARGET_SO_STACK deploy init --output $test_deployment_spec --config CERC_TEST_PARAM_1=PASSED,CERC_TEST_PARAM_3=FAST
-# Check the file now exists
-if [ ! -f "$test_deployment_spec" ]; then
-    echo "deploy init test: spec file not present"
-    echo "deploy init test: FAILED"
-    exit 1
-fi
-echo "deploy init test: passed"
-$TEST_TARGET_SO_STACK deploy create --spec-file $test_deployment_spec --deployment-dir $test_deployment_dir
-# Check the deployment dir exists
-if [ ! -d "$test_deployment_dir" ]; then
-    echo "deploy create test: deployment directory not present"
-    echo "deploy create test: FAILED"
-    exit 1
-fi
-echo "deploy create test: passed"
-# Check the file writted by the create command in the stack now exists
-if [ ! -f "$test_deployment_dir/create-file" ]; then
-    echo "deploy create test: create output file not present"
-    echo "deploy create test: FAILED"
-    exit 1
-fi
-# And has the right content
-create_file_content=$(<$test_deployment_dir/create-file)
-if [ ! "$create_file_content" == "create-command-output-data"  ]; then
-    echo "deploy create test: create output file contents not correct"
-    echo "deploy create test: FAILED"
-    exit 1
-fi
-
-# Add a config file to be picked up by the ConfigMap before starting.
-echo "dbfc7a4d-44a7-416d-b5f3-29842cc47650" > $test_deployment_dir/data/test-config/test_config
-
-echo "deploy create output file test: passed"
-# Try to start the deployment
-$TEST_TARGET_SO deployment --dir $test_deployment_dir start
-# Check logs command works
-log_output_3=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
-if [[ "$log_output_3" == *"filesystem is fresh"* ]]; then
-    echo "deployment logs test: passed"
-else
-    echo "deployment logs test: FAILED"
-    exit 1
-fi
-# Check the config variable CERC_TEST_PARAM_1 was passed correctly
-if [[ "$log_output_3" == *"Test-param-1: PASSED"* ]]; then
-    echo "deployment config test: passed"
-else
-    echo "deployment config test: FAILED"
-    exit 1
-fi
-# Check the config variable CERC_TEST_PARAM_2 was passed correctly from the compose file
-if [[ "$log_output_3" == *"Test-param-2: CERC_TEST_PARAM_2_VALUE"* ]]; then
-    echo "deployment compose config test: passed"
-else
-    echo "deployment compose config test: FAILED"
-    exit 1
-fi
-# Check the config variable CERC_TEST_PARAM_3 was passed correctly
-if [[ "$log_output_3" == *"Test-param-3: FAST"* ]]; then
-    echo "deployment config test: passed"
-else
-    echo "deployment config test: FAILED"
-    exit 1
-fi
-
-# Check that the ConfigMap is mounted and contains the expected content.
-log_output_4=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
-if [[ "$log_output_4" == *"/config/test_config:"* ]] && [[ "$log_output_4" == *"dbfc7a4d-44a7-416d-b5f3-29842cc47650"* ]]; then
-    echo "deployment ConfigMap test: passed"
-else
-    echo "deployment ConfigMap test: FAILED"
-    delete_cluster_exit
-fi
-
-# Stop then start again and check the volume was preserved
-$TEST_TARGET_SO deployment --dir $test_deployment_dir stop
-# Sleep a bit just in case
-# sleep for longer to check if that's why the subsequent create cluster fails
-sleep 20
-$TEST_TARGET_SO deployment --dir $test_deployment_dir start
-log_output_5=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
-if [[ "$log_output_5" == *"filesystem is old"* ]]; then
-    echo "Retain volumes test: passed"
-else
-    echo "Retain volumes test: FAILED"
-    delete_cluster_exit
-fi
-
-# Stop and clean up
-$TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes
-echo "Test passed"