Merge branch 'main' into dboreham/laconicd-k8s

.gitea/workflows/lint.yml

@@ -1,21 +0,0 @@
-name: Lint Checks
-
-on:
-  pull_request:
-    branches: '*'
-  push:
-    branches: '*'
-
-jobs:
-  test:
-    name: "Run linter"
-    runs-on: ubuntu-latest
-    steps:
-      - name: "Clone project repository"
-        uses: actions/checkout@v3
-      - name: "Install Python"
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.8'
-      - name : "Run flake8"
-        uses: py-actions/flake8@v2

README.md

@@ -29,10 +29,10 @@ chmod +x ~/.docker/cli-plugins/docker-compose
 Next decide on a directory where you would like to put the stack-orchestrator program. Typically this would be 
 a "user" binary directory such as `~/bin` or perhaps `/usr/local/laconic` or possibly just the current working directory.
 
-Now, having selected that directory, download the latest release from [this page](https://git.vdb.to/cerc-io/stack-orchestrator/tags) into it (we're using `~/bin` below for concreteness but edit to suit if you selected a different directory). Also be sure that the destination directory exists and is writable:
+Now, having selected that directory, download the latest release from [this page](https://github.com/cerc-io/stack-orchestrator/tags) into it (we're using `~/bin` below for concreteness but edit to suit if you selected a different directory). Also be sure that the destination directory exists and is writable:
 
 ```bash
-curl -L -o ~/bin/laconic-so https://git.vdb.to/cerc-io/stack-orchestrator/releases/download/latest/laconic-so
+curl -L -o ~/bin/laconic-so https://github.com/cerc-io/stack-orchestrator/releases/latest/download/laconic-so
 ```
 
 Give it execute permissions:
@@ -52,7 +52,7 @@ Version: 1.1.0-7a607c2-202304260513
 Save the distribution url to `~/.laconic-so/config.yml`:
 ```bash
 mkdir ~/.laconic-so
-echo "distribution-url: https://git.vdb.to/cerc-io/stack-orchestrator/releases/download/latest/laconic-so" >  ~/.laconic-so/config.yml
+echo "distribution-url: https://github.com/cerc-io/stack-orchestrator/releases/latest/download/laconic-so" >  ~/.laconic-so/config.yml
 ```
 
 ### Update

docs/CONTRIBUTING.md

@@ -26,7 +26,7 @@ In addition to the pre-requisites listed in the [README](/README.md), the follow
 
 1. Clone this repository:
    ```
-   $ git clone https://git.vdb.to/cerc-io/stack-orchestrator.git
+   $ git clone https://github.com/cerc-io/stack-orchestrator.git
    ```
 
 2. Enter the project directory:

docs/adding-a-new-stack.md

@@ -1,10 +1,10 @@
 # Adding a new stack
 
-See [this PR](https://git.vdb.to/cerc-io/stack-orchestrator/pull/434) for an example of how to currently add a minimal stack to stack orchestrator. The [reth stack](https://git.vdb.to/cerc-io/stack-orchestrator/pull/435) is another good example.
+See [this PR](https://github.com/cerc-io/stack-orchestrator/pull/434) for an example of how to currently add a minimal stack to stack orchestrator. The [reth stack](https://github.com/cerc-io/stack-orchestrator/pull/435) is another good example.
 
 For external developers, we recommend forking this repo and adding your stack directly to your fork. This initially requires running in "developer mode" as described [here](/docs/CONTRIBUTING.md). Check out the [Namada stack](https://github.com/vknowable/stack-orchestrator/blob/main/app/data/stacks/public-namada/digitalocean_quickstart.md) from Knowable to see how that is done.
 
-Core to the feature completeness of stack orchestrator is to [decouple the tool functionality from payload](https://git.vdb.to/cerc-io/stack-orchestrator/issues/315) which will no longer require forking to add a stack.
+Core to the feature completeness of stack orchestrator is to [decouple the tool functionality from payload](https://github.com/cerc-io/stack-orchestrator/issues/315) which will no longer require forking to add a stack.
 
 ## Example
 

docs/spec.md

@@ -1,6 +1,6 @@
 # Specification
 
-Note: this page is out of date (but still useful) - it will no longer be useful once stacks are [decoupled from the tool functionality](https://git.vdb.to/cerc-io/stack-orchestrator/issues/315).
+Note: this page is out of date (but still useful) - it will no longer be useful once stacks are [decoupled from the tool functionality](https://github.com/cerc-io/stack-orchestrator/issues/315).
 
 ## Implementation
 

requirements.txt

@@ -10,4 +10,3 @@ pydantic==1.10.9
 tomli==2.0.1
 validators==0.22.0
 kubernetes>=28.1.0
-humanfriendly>=10.0

scripts/cloud-init-dev-mode-install.yaml

@@ -41,4 +41,4 @@ runcmd:
   - apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
   - systemctl enable docker
   - systemctl start docker
-  - git clone https://git.vdb.to/cerc-io/stack-orchestrator.git /home/ubuntu/stack-orchestrator
+  - git clone https://github.com/cerc-io/stack-orchestrator.git /home/ubuntu/stack-orchestrator

scripts/cloud-init-user-mode-install.yaml

@@ -31,5 +31,5 @@ runcmd:
   - apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
   - systemctl enable docker
   - systemctl start docker
-  - curl -L -o /usr/local/bin/laconic-so https://git.vdb.to/cerc-io/stack-orchestrator/releases/download/latest/laconic-so
+  - curl -L -o /usr/local/bin/laconic-so https://github.com/cerc-io/stack-orchestrator/releases/latest/download/laconic-so
   - chmod +x /usr/local/bin/laconic-so

scripts/quick-install-linux.sh

@@ -137,7 +137,7 @@ fi
 echo "**************************************************************************************"
 echo "Installing laconic-so"
 # install latest `laconic-so`
-distribution_url=https://git.vdb.to/cerc-io/stack-orchestrator/releases/download/latest/laconic-so
+distribution_url=https://github.com/cerc-io/stack-orchestrator/releases/latest/download/laconic-so
 install_filename=${install_dir}/laconic-so
 mkdir -p  ${install_dir}
 curl -L -o ${install_filename} ${distribution_url}

setup.py

@@ -13,7 +13,7 @@ setup(
     description='Orchestrates deployment of the Laconic stack',
     long_description=long_description,
     long_description_content_type="text/markdown",
-    url='https://git.vdb.to/cerc-io/stack-orchestrator',
+    url='https://github.com/cerc-io/stack-orchestrator',
     py_modules=['stack_orchestrator'],
     packages=find_packages(),
     install_requires=[requirements],

stack_orchestrator/data/compose/docker-compose-fixturenet-laconicd.yml

@@ -7,9 +7,7 @@ services:
       # The cosmos-sdk node's database directory:
       - laconicd-data:/root/.laconicd
       # TODO: look at folding these scripts into the container
-      - ../config/fixturenet-laconicd/create-fixturenet.sh:/docker-entrypoint-scripts.d/create-fixturenet.sh
-      - ../config/fixturenet-laconicd/export-mykey.sh:/docker-entrypoint-scripts.d/export-mykey.sh
-      - ../config/fixturenet-laconicd/export-myaddress.sh:/docker-entrypoint-scripts.d/export-myaddress.sh
+      - laconicd-scripts:/docker-entrypoint-scripts.d:ro
     # TODO: determine which of the ports below is really needed
     ports:
       - "6060"
@@ -24,7 +22,9 @@ services:
   cli:
     image: cerc/laconic-registry-cli:local
     volumes:
-      - ../config/fixturenet-laconicd/registry-cli-config-template.yml:/registry-cli-config-template.yml
+      - cli-config:/registry-cli-config
 
 volumes:
   laconicd-data:
+  laconicd-scripts:
+  cli-config:

stack_orchestrator/data/compose/docker-compose-mainnet-laconicd.yml

@@ -9,9 +9,7 @@ services:
       - laconicd-config:/root/.laconicd/config
       - laconicd-keyring:/root/.laconicd/keyring-test
       # TODO: look at folding these scripts into the container
-      - ../config/mainnet-laconicd/scripts/run-laconicd.sh:/opt/run-laconicd.sh
-      - ../config/mainnet-laconicd/scripts/export-mykey.sh:/docker-entrypoint-scripts.d/export-mykey.sh
-      - ../config/mainnet-laconicd/scripts/export-myaddress.sh:/docker-entrypoint-scripts.d/export-myaddress.sh
+      - laconicd-scripts:/docker-entrypoint-scripts.d:ro
     # TODO: determine which of the ports below is really needed
     ports:
       - "6060"
@@ -26,9 +24,11 @@ services:
   cli:
     image: cerc/laconic-registry-cli:local
     volumes:
-      - ../config/mainnet-laconicd/registry-cli-config-template.yml:/registry-cli-config-template.yml
+      - cli-config:/registry-cli-config
 
 volumes:
   laconicd-data:
   laconicd-config:
+  laconicd-scripts:
   laconicd-keyring:
+  cli-config:

stack_orchestrator/data/compose/docker-compose-mars.yml

@@ -1,20 +0,0 @@
-version: "3.2"
-
-services:
-  mars:
-    image: cerc/mars:local
-    restart: always
-    ports:
-      - "3000:3000"
-    environment:
-      - URL_OSMOSIS_GQL=https://osmosis-node.marsprotocol.io/GGSFGSFGFG34/osmosis-hive-front/graphql
-      - URL_OSMOSIS_REST=https://lcd-osmosis.blockapsis.com
-      - URL_OSMOSIS_RPC=https://rpc-osmosis.blockapsis.com
-      - URL_NEUTRON_GQL=https://neutron.rpc.p2p.world/qgrnU6PsQZA8F9S5Fb8Fn3tV3kXmMBl2M9bcc9jWLjQy8p/hive/graphql
-      - URL_NEUTRON_REST=https://rest-kralum.neutron-1.neutron.org
-      - URL_NEUTRON_RPC=https://rpc-kralum.neutron-1.neutron.org
-      - URL_NEUTRON_TEST_GQL=https://testnet-neutron-gql.marsprotocol.io/graphql
-      - URL_NEUTRON_TEST_REST=https://rest-palvus.pion-1.ntrn.tech
-      - URL_NEUTRON_TEST_RPC=https://rpc-palvus.pion-1.ntrn.tech
-      - WALLET_CONNECT_ID=0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x
-

stack_orchestrator/data/compose/docker-compose-ping-pub.yml

@@ -1,8 +0,0 @@
-version: "3.2"
-
-services:
-  ping-pub:
-    image: cerc/ping-pub:local
-    restart: always
-    ports:
-      - "5173:5173"

stack_orchestrator/data/compose/docker-compose-test.yml

@@ -5,7 +5,6 @@ services:
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
       CERC_TEST_PARAM_1: ${CERC_TEST_PARAM_1:-FAILED}
-      CERC_TEST_PARAM_2: "CERC_TEST_PARAM_2_VALUE"
     volumes:
       - test-data:/data
       - test-config:/config:ro

stack_orchestrator/data/config/fixturenet-laconic-loaded/create-fixturenet.sh

@@ -0,0 +1,125 @@
+#!/bin/bash
+
+# TODO: this file is now an unmodified copy of cerc-io/laconicd/init.sh
+# so we should have a mechanism to bundle it inside the container rather than link from here
+# at deploy time.
+
+KEY="mykey"
+CHAINID="laconic_9000-1"
+MONIKER="localtestnet"
+KEYRING="test"
+KEYALGO="eth_secp256k1"
+LOGLEVEL="info"
+# trace evm
+TRACE="--trace"
+# TRACE=""
+
+if [ "$1" == "clean" ] || [ ! -d "$HOME/.laconicd/data/blockstore.db" ]; then
+  # validate dependencies are installed
+  command -v jq > /dev/null 2>&1 || { echo >&2 "jq not installed. More info: https://stedolan.github.io/jq/download/"; exit 1; }
+
+  # remove existing daemon and client
+  rm -rf $HOME/.laconicd/*
+  rm -rf $HOME/.laconic/*
+
+  if [ -n "`which make`" ]; then
+    make install
+  fi
+
+  laconicd config keyring-backend $KEYRING
+  laconicd config chain-id $CHAINID
+
+  # if $KEY exists it should be deleted
+  laconicd keys add $KEY --keyring-backend $KEYRING --algo $KEYALGO
+
+  # Set moniker and chain-id for Ethermint (Moniker can be anything, chain-id must be an integer)
+  laconicd init $MONIKER --chain-id $CHAINID
+
+  # Change parameter token denominations to aphoton
+  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["staking"]["params"]["bond_denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["crisis"]["constant_fee"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["gov"]["deposit_params"]["min_deposit"][0]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["mint"]["params"]["mint_denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+  # Custom modules
+  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["record_rent"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_rent"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_commit_fee"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_reveal_fee"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_minimum_bid"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+
+  if [[ "$TEST_REGISTRY_EXPIRY" == "true" ]]; then
+    echo "Setting timers for expiry tests."
+
+    cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["record_rent_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+    cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_grace_period"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+    cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_rent_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+  fi
+
+  if [[ "$TEST_AUCTION_ENABLED" == "true" ]]; then
+    echo "Enabling auction and setting timers."
+
+    cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_enabled"]=true' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+    cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_rent_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+    cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_grace_period"]="300s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+    cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_commits_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+    cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_reveals_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+  fi
+
+  # increase block time (?)
+  cat $HOME/.laconicd/config/genesis.json | jq '.consensus_params["block"]["time_iota_ms"]="1000"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+
+  # Set gas limit in genesis
+  cat $HOME/.laconicd/config/genesis.json | jq '.consensus_params["block"]["max_gas"]="10000000"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+
+  # disable produce empty block
+  if [[ "$OSTYPE" == "darwin"* ]]; then
+      sed -i '' 's/create_empty_blocks = true/create_empty_blocks = false/g' $HOME/.laconicd/config/config.toml
+    else
+      sed -i 's/create_empty_blocks = true/create_empty_blocks = false/g' $HOME/.laconicd/config/config.toml
+  fi
+
+  if [[ $1 == "pending" ]]; then
+    if [[ "$OSTYPE" == "darwin"* ]]; then
+        sed -i '' 's/create_empty_blocks_interval = "0s"/create_empty_blocks_interval = "30s"/g' $HOME/.laconicd/config/config.toml
+        sed -i '' 's/timeout_propose = "3s"/timeout_propose = "30s"/g' $HOME/.laconicd/config/config.toml
+        sed -i '' 's/timeout_propose_delta = "500ms"/timeout_propose_delta = "5s"/g' $HOME/.laconicd/config/config.toml
+        sed -i '' 's/timeout_prevote = "1s"/timeout_prevote = "10s"/g' $HOME/.laconicd/config/config.toml
+        sed -i '' 's/timeout_prevote_delta = "500ms"/timeout_prevote_delta = "5s"/g' $HOME/.laconicd/config/config.toml
+        sed -i '' 's/timeout_precommit = "1s"/timeout_precommit = "10s"/g' $HOME/.laconicd/config/config.toml
+        sed -i '' 's/timeout_precommit_delta = "500ms"/timeout_precommit_delta = "5s"/g' $HOME/.laconicd/config/config.toml
+        sed -i '' 's/timeout_commit = "5s"/timeout_commit = "150s"/g' $HOME/.laconicd/config/config.toml
+        sed -i '' 's/timeout_broadcast_tx_commit = "10s"/timeout_broadcast_tx_commit = "150s"/g' $HOME/.laconicd/config/config.toml
+    else
+        sed -i 's/create_empty_blocks_interval = "0s"/create_empty_blocks_interval = "30s"/g' $HOME/.laconicd/config/config.toml
+        sed -i 's/timeout_propose = "3s"/timeout_propose = "30s"/g' $HOME/.laconicd/config/config.toml
+        sed -i 's/timeout_propose_delta = "500ms"/timeout_propose_delta = "5s"/g' $HOME/.laconicd/config/config.toml
+        sed -i 's/timeout_prevote = "1s"/timeout_prevote = "10s"/g' $HOME/.laconicd/config/config.toml
+        sed -i 's/timeout_prevote_delta = "500ms"/timeout_prevote_delta = "5s"/g' $HOME/.laconicd/config/config.toml
+        sed -i 's/timeout_precommit = "1s"/timeout_precommit = "10s"/g' $HOME/.laconicd/config/config.toml
+        sed -i 's/timeout_precommit_delta = "500ms"/timeout_precommit_delta = "5s"/g' $HOME/.laconicd/config/config.toml
+        sed -i 's/timeout_commit = "5s"/timeout_commit = "150s"/g' $HOME/.laconicd/config/config.toml
+        sed -i 's/timeout_broadcast_tx_commit = "10s"/timeout_broadcast_tx_commit = "150s"/g' $HOME/.laconicd/config/config.toml
+    fi
+  fi
+
+  # Allocate genesis accounts (cosmos formatted addresses)
+  laconicd add-genesis-account $KEY 100000000000000000000000000aphoton --keyring-backend $KEYRING
+
+  # Sign genesis transaction
+  laconicd gentx $KEY 1000000000000000000000aphoton --keyring-backend $KEYRING --chain-id $CHAINID
+
+  # Collect genesis tx
+  laconicd collect-gentxs
+
+  # Run this to ensure everything worked and that the genesis file is setup correctly
+  laconicd validate-genesis
+
+  if [[ $1 == "pending" ]]; then
+    echo "pending mode is on, please wait for the first block committed."
+  fi
+else
+  echo "Using existing database at $HOME/.laconicd.  To replace, run '`basename $0` clean'"
+fi
+
+# Start the node (remove the --pruning=nothing flag if historical queries are not needed)
+laconicd start --pruning=nothing --evm.tracer=json $TRACE --log_level $LOGLEVEL --minimum-gas-prices=0.0001aphoton --json-rpc.api eth,txpool,personal,net,debug,web3,miner --api.enable --gql-server --gql-playground

stack_orchestrator/data/config/fixturenet-laconic-loaded/export-myaddress.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+laconicd keys show mykey | grep address | cut -d ' ' -f 3

stack_orchestrator/data/config/fixturenet-laconic-loaded/export-mykey.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+echo y | laconicd keys export mykey --unarmored-hex --unsafe

stack_orchestrator/data/config/fixturenet-laconic-loaded/registry-cli-config-template.yml

@@ -0,0 +1,9 @@
+services:
+  cns:
+    restEndpoint: 'http://laconicd:1317'
+    gqlEndpoint: 'http://laconicd:9473/api'
+    userKey: REPLACE_WITH_MYKEY
+    bondId:
+    chainId: laconic_9000-1
+    gas: 350000
+    fees: 200000aphoton

stack_orchestrator/data/container-build/cerc-laconic-registry-cli/import-key.sh

@@ -1,2 +1,2 @@
 #!/bin/sh
-sed 's/REPLACE_WITH_MYKEY/'${1}'/' registry-cli-config-template.yml > config.yml
+sed 's/REPLACE_WITH_MYKEY/'${1}'/' /registry-cli-config/registry-cli-config-template.yml > config.yml

stack_orchestrator/data/container-build/cerc-mars/build.sh

@@ -1,4 +0,0 @@
-#!/usr/bin/env bash
-# Build the mars image
-source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
-docker build -t cerc/mars:local -f ${CERC_REPO_BASE_DIR}/mars-interface/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/mars-interface

stack_orchestrator/data/container-build/cerc-ping-pub/build.sh

@@ -1,5 +0,0 @@
-#!/usr/bin/env bash
-# Build the ping pub image
-source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
-
-docker build -t cerc/ping-pub:local ${build_command_args} -f $CERC_REPO_BASE_DIR/explorer/Dockerfile $CERC_REPO_BASE_DIR/explorer

stack_orchestrator/data/container-build/cerc-test-container/run.sh

@@ -17,9 +17,6 @@ fi
 if [ -n "$CERC_TEST_PARAM_1" ]; then
   echo "Test-param-1: ${CERC_TEST_PARAM_1}"
 fi
-if [ -n "$CERC_TEST_PARAM_2" ]; then
-  echo "Test-param-2: ${CERC_TEST_PARAM_2}"
-fi
 
 if [ -d "/config" ]; then
   echo "/config: EXISTS"

stack_orchestrator/data/stacks/fixturenet-eth/README.md

@@ -1,6 +1,6 @@
 # fixturenet-eth
 
-Instructions for deploying a local a geth + lighthouse blockchain "fixturenet" for development and testing purposes using laconic-stack-orchestrator (the installation of which is covered [here](https://git.vdb.to/cerc-io/stack-orchestrator)):
+Instructions for deploying a local a geth + lighthouse blockchain "fixturenet" for development and testing purposes using laconic-stack-orchestrator (the installation of which is covered [here](https://github.com/cerc-io/stack-orchestrator)):
 
 ## Clone required repositories
 

stack_orchestrator/data/stacks/fixturenet-laconic-loaded/README.md

@@ -7,11 +7,11 @@ Instructions for deploying a local Laconic blockchain "fixturenet" for developme
 **Note:** For building some NPMs, access to the @lirewine repositories is required. If you don't have access, see [this tutorial](/docs/laconicd-fixturenet.md) to run this stack
 
 ## 1. Install Laconic Stack Orchestrator
-Installation is covered in detail [here](https://git.vdb.to/cerc-io/stack-orchestrator#user-mode) but if you're on Linux and already have docker installed it should be as simple as:
+Installation is covered in detail [here](https://github.com/cerc-io/stack-orchestrator#user-mode) but if you're on Linux and already have docker installed it should be as simple as:
 ```
 $ mkdir my-working-dir
 $ cd my-working-dir
-$ curl -L -o ./laconic-so https://git.vdb.to/cerc-io/stack-orchestrator/releases/download/latest/laconic-so
+$ curl -L -o ./laconic-so https://github.com/cerc-io/stack-orchestrator/releases/latest/download/laconic-so
 $ chmod +x ./laconic-so
 $ export PATH=$PATH:$(pwd)  # Or move laconic-so to ~/bin or your favorite on-path directory
 ```

stack_orchestrator/data/stacks/fixturenet-laconic-loaded/deploy/commands.py

@@ -0,0 +1,72 @@
+# Copyright © 2022-2024 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+from stack_orchestrator.util import get_yaml, get_config_file_dir
+from stack_orchestrator.deploy.deploy_types import DeployCommandContext
+from stack_orchestrator.deploy.deployment_context import DeploymentContext
+from pathlib import Path
+from shutil import copy
+
+default_spec_file_content = ""
+
+
+def _stack_config_dir(context: DeploymentContext) -> Path:
+    return get_config_file_dir().joinpath(context.stack.name)
+
+
+def _deployment_volume_dir(context: DeploymentContext, volume_name: str) -> Path:
+    return context.deployment_dir.joinpath("data", volume_name)
+
+
+def _copy_laconicd_scripts(context: DeploymentContext):
+    scripts = [
+        "create-fixturenet.sh",
+        "export-mykey.sh",
+        "export-myaddress.sh"
+    ]
+    destination_dir = _deployment_volume_dir(context, "laconicd-scripts")
+    for script in scripts:
+        # ../config/fixturenet-laconicd/
+        source_file = _stack_config_dir(context).joinpath(script)
+        copy(source_file, destination_dir)
+
+
+def _copy_cli_template(context: DeploymentContext):
+    destination_dir = _deployment_volume_dir(context, "cli-config")
+    # ../config/fixturenet-laconicd/
+    source_file = _stack_config_dir(context).joinpath("registry-cli-config-template.yml")
+    copy(source_file, destination_dir)
+
+
+def setup(command_context: DeployCommandContext, parameters, extra_args):
+    pass
+
+
+def init(command_context: DeployCommandContext):
+    yaml = get_yaml()
+    return yaml.load(default_spec_file_content)
+
+
+def create(context: DeploymentContext, extra_args):
+    _copy_laconicd_scripts(context)
+    _copy_cli_template(context)
+
+
+def get_state(command_context: DeployCommandContext):
+    pass
+
+
+def change_state(command_context: DeployCommandContext):
+    pass

stack_orchestrator/data/stacks/fixturenet-laconicd/README.md

@@ -3,11 +3,11 @@
 Instructions for deploying a local Laconic blockchain "fixturenet" for development and testing purposes using laconic-stack-orchestrator.
 
 ## 1. Install Laconic Stack Orchestrator
-Installation is covered in detail [here](https://git.vdb.to/cerc-io/stack-orchestrator#user-mode) but if you're on Linux and already have docker installed it should be as simple as:
+Installation is covered in detail [here](https://github.com/cerc-io/stack-orchestrator#user-mode) but if you're on Linux and already have docker installed it should be as simple as:
 ```
 $ mkdir my-working-dir
 $ cd my-working-dir
-$ curl -L -o ./laconic-so https://git.vdb.to/cerc-io/stack-orchestrator/releases/download/latest/laconic-so
+$ curl -L -o ./laconic-so https://github.com/cerc-io/stack-orchestrator/releases/latest/download/laconic-so
 $ chmod +x ./laconic-so
 $ export PATH=$PATH:$(pwd)  # Or move laconic-so to ~/bin or your favorite on-path directory
 ```

stack_orchestrator/data/stacks/mars/README.md

@@ -1,16 +0,0 @@
-# mars
-
-On a fresh Digital Ocean droplet with Ubuntu:
-
-```
-git clone https://github.com/cerc-io/stack-orchestrator
-cd stack-orchestrator
-./scripts/quick-install-linux.sh
-```
-Read and follow the instructions output from the above output to complete installation, then:
-
-```
-laconic-so --stack mars setup-repositories
-laconic-so --stack mars build-containers
-laconic-so --stack mars deploy up
-```

stack_orchestrator/data/stacks/mars/stack.yml

@@ -1,8 +0,0 @@
-version: "0.1"
-name: mars
-repos:
-  - github.com/cerc-io/mars-interface
-containers:
-  - cerc/mars
-pods:
-  - mars

stack_orchestrator/data/stacks/mobymask/README.md

@@ -4,7 +4,7 @@ The MobyMask watcher is a Laconic Network component that provides efficient acce
 
 ## Deploy the MobyMask Watcher
 
-The instructions below show how to deploy a MobyMask watcher using laconic-stack-orchestrator (the installation of which is covered [here](https://git.vdb.to/cerc-io/stack-orchestrator#install)).
+The instructions below show how to deploy a MobyMask watcher using laconic-stack-orchestrator (the installation of which is covered [here](https://github.com/cerc-io/stack-orchestrator#install)).
 
 This deployment expects that ipld-eth-server's endpoints are available on the local machine at http://ipld-eth-server.example.com:8083/graphql and http://ipld-eth-server.example.com:8082. More advanced configurations are supported by modifying the watcher's [config file](../../config/watcher-mobymask/mobymask-watcher.toml).
 

stack_orchestrator/data/stacks/ping-pub/README.md

@@ -1,10 +0,0 @@
-# ping-pub
-Experimental block explorer for laconic
-
-```
-laconic-so --stack ping-pub setup-repositories
-laconic-so --stack ping-pub build-containers
-laconic-so --stack ping-pub deploy init --output ping-pub-spec.yml --map-ports-to-host localhost-same
-laconic-so --stack ping-pub deploy create --spec-file ping-pub-spec.yml --deployment-dir pp-deployment
-laconic-so deployment --dir pp-deployment start
-```

stack_orchestrator/data/stacks/ping-pub/stack.yml

@@ -1,9 +0,0 @@
-version: "0.1"
-name: ping-pub
-repos:
-  # fork, but only for config & Dockerfile reasonsb
-  - github.com/LaconicNetwork/explorer@laconic
-containers:
-  - cerc/ping-pub
-pods:
-  - ping-pub

stack_orchestrator/data/stacks/webapp-deployer-backend/stack.yml

@@ -2,10 +2,10 @@ version: "1.0"
 name: webapp-deployer-backend
 description: "Deployer for webapps"
 repos:
-  - git.vdb.to/telackey/webapp-deployment-status-api
+  - git.vdb.to:telackey/webapp-deployment-status-api
 containers:
   - cerc/webapp-deployer-backend
 pods:
   - name: webapp-deployer-backend
-    repository: git.vdb.to/telackey/webapp-deployment-status-api
+    repository: git.vdb.to:telackey/webapp-deployment-status-api
     path: ./

stack_orchestrator/deploy/k8s/cluster_info.py

@@ -22,41 +22,12 @@ from stack_orchestrator.opts import opts
 from stack_orchestrator.util import env_var_map_from_file
 from stack_orchestrator.deploy.k8s.helpers import named_volumes_from_pod_files, volume_mounts_for_service, volumes_for_pod_files
 from stack_orchestrator.deploy.k8s.helpers import get_node_pv_mount_path
-from stack_orchestrator.deploy.k8s.helpers import envs_from_environment_variables_map, envs_from_compose_file, merge_envs
+from stack_orchestrator.deploy.k8s.helpers import envs_from_environment_variables_map
 from stack_orchestrator.deploy.deploy_util import parsed_pod_files_map_from_file_names, images_for_deployment
 from stack_orchestrator.deploy.deploy_types import DeployEnvVars
-from stack_orchestrator.deploy.spec import Spec, Resources, ResourceLimits
+from stack_orchestrator.deploy.spec import Spec
 from stack_orchestrator.deploy.images import remote_tag_for_image
 
-DEFAULT_VOLUME_RESOURCES = Resources({
-    "reservations": {"storage": "2Gi"}
-})
-
-DEFAULT_CONTAINER_RESOURCES = Resources({
-    "reservations": {"cpus": "0.1", "memory": "200M"},
-    "limits": {"cpus": "1.0", "memory": "2000M"},
-})
-
-
-def to_k8s_resource_requirements(resources: Resources) -> client.V1ResourceRequirements:
-    def to_dict(limits: ResourceLimits):
-        if not limits:
-            return None
-
-        ret = {}
-        if limits.cpus:
-            ret["cpu"] = str(limits.cpus)
-        if limits.memory:
-            ret["memory"] = f"{int(limits.memory / (1000 * 1000))}M"
-        if limits.storage:
-            ret["storage"] = f"{int(limits.storage / (1000 * 1000))}M"
-        return ret
-
-    return client.V1ResourceRequirements(
-        requests=to_dict(resources.reservations),
-        limits=to_dict(resources.limits)
-    )
-
 
 class ClusterInfo:
     parsed_pod_yaml_map: Any
@@ -164,13 +135,9 @@ class ClusterInfo:
         result = []
         spec_volumes = self.spec.get_volumes()
         named_volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map)
-        resources = self.spec.get_volume_resources()
-        if not resources:
-            resources = DEFAULT_VOLUME_RESOURCES
         if opts.o.debug:
             print(f"Spec Volumes: {spec_volumes}")
             print(f"Named Volumes: {named_volumes}")
-            print(f"Resources: {resources}")
         for volume_name in spec_volumes:
             if volume_name not in named_volumes:
                 if opts.o.debug:
@@ -179,7 +146,9 @@ class ClusterInfo:
             spec = client.V1PersistentVolumeClaimSpec(
                 access_modes=["ReadWriteOnce"],
                 storage_class_name="manual",
-                resources=to_k8s_resource_requirements(resources),
+                resources=client.V1ResourceRequirements(
+                    requests={"storage": "2Gi"}
+                ),
                 volume_name=f"{self.app_name}-{volume_name}"
             )
             pvc = client.V1PersistentVolumeClaim(
@@ -223,9 +192,6 @@ class ClusterInfo:
         result = []
         spec_volumes = self.spec.get_volumes()
         named_volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map)
-        resources = self.spec.get_volume_resources()
-        if not resources:
-            resources = DEFAULT_VOLUME_RESOURCES
         for volume_name in spec_volumes:
             if volume_name not in named_volumes:
                 if opts.o.debug:
@@ -234,7 +200,7 @@ class ClusterInfo:
             spec = client.V1PersistentVolumeSpec(
                 storage_class_name="manual",
                 access_modes=["ReadWriteOnce"],
-                capacity=to_k8s_resource_requirements(resources).requests,
+                capacity={"storage": "2Gi"},
                 host_path=client.V1HostPathVolumeSource(path=get_node_pv_mount_path(volume_name))
             )
             pv = client.V1PersistentVolume(
@@ -248,9 +214,6 @@ class ClusterInfo:
     # TODO: put things like image pull policy into an object-scope struct
     def get_deployment(self, image_pull_policy: str = None):
         containers = []
-        resources = self.spec.get_container_resources()
-        if not resources:
-            resources = DEFAULT_CONTAINER_RESOURCES
         for pod_name in self.parsed_pod_yaml_map:
             pod = self.parsed_pod_yaml_map[pod_name]
             services = pod["services"]
@@ -263,13 +226,6 @@ class ClusterInfo:
                     if opts.o.debug:
                         print(f"image: {image}")
                         print(f"service port: {port}")
-                merged_envs = merge_envs(
-                    envs_from_compose_file(
-                        service_info["environment"]), self.environment_variables.map
-                        ) if "environment" in service_info else self.environment_variables.map
-                envs = envs_from_environment_variables_map(merged_envs)
-                if opts.o.debug:
-                    print(f"Merged envs: {envs}")
                 # Re-write the image tag for remote deployment
                 image_to_use = remote_tag_for_image(
                     image, self.spec.get_image_registry()) if self.spec.get_image_registry() is not None else image
@@ -278,10 +234,13 @@ class ClusterInfo:
                     name=container_name,
                     image=image_to_use,
                     image_pull_policy=image_pull_policy,
-                    env=envs,
+                    env=envs_from_environment_variables_map(self.environment_variables.map),
                     ports=[client.V1ContainerPort(container_port=port)],
                     volume_mounts=volume_mounts,
-                    resources=to_k8s_resource_requirements(resources),
+                    resources=client.V1ResourceRequirements(
+                        requests={"cpu": "100m", "memory": "200Mi"},
+                        limits={"cpu": "1000m", "memory": "2000Mi"},
+                    ),
                 )
                 containers.append(container)
         volumes = volumes_for_pod_files(self.parsed_pod_yaml_map, self.spec, self.app_name)

stack_orchestrator/deploy/k8s/helpers.py

@@ -17,7 +17,6 @@ from kubernetes import client
 import os
 from pathlib import Path
 import subprocess
-import re
 from typing import Set, Mapping, List
 
 from stack_orchestrator.opts import opts
@@ -215,33 +214,6 @@ def _generate_kind_port_mappings(parsed_pod_files):
     )
 
 
-# Note: this makes any duplicate definition in b overwrite a
-def merge_envs(a: Mapping[str, str], b: Mapping[str, str]) -> Mapping[str, str]:
-    result = {**a, **b}
-    return result
-
-
-def _expand_shell_vars(raw_val: str) -> str:
-    # could be: <string> or ${<env-var-name>} or ${<env-var-name>:-<default-value>}
-    # TODO: implement support for variable substitution and default values
-    # if raw_val is like ${<something>} print a warning and substitute an empty string
-    # otherwise return raw_val
-    match = re.search(r"^\$\{(.*)\}$", raw_val)
-    if match:
-        print(f"WARNING: found unimplemented environment variable substitution: {raw_val}")
-    else:
-        return raw_val
-
-
-# TODO: handle the case where the same env var is defined in multiple places
-def envs_from_compose_file(compose_file_envs: Mapping[str, str]) -> Mapping[str, str]:
-    result = {}
-    for env_var, env_val in compose_file_envs.items():
-        expanded_env_val = _expand_shell_vars(env_val)
-        result.update({env_var: expanded_env_val})
-    return result
-
-
 def envs_from_environment_variables_map(map: Mapping[str, str]) -> List[client.V1EnvVar]:
     result = []
     for env_var, env_val in map.items():

stack_orchestrator/deploy/spec.py

@@ -13,60 +13,12 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
-import typing
-import humanfriendly
-
 from pathlib import Path
-
+import typing
 from stack_orchestrator.util import get_yaml
 from stack_orchestrator import constants
 
 
-class ResourceLimits:
-    cpus: float = None
-    memory: int = None
-    storage: int = None
-
-    def __init__(self, obj={}):
-        if "cpus" in obj:
-            self.cpus = float(obj["cpus"])
-        if "memory" in obj:
-            self.memory = humanfriendly.parse_size(obj["memory"])
-        if "storage" in obj:
-            self.storage = humanfriendly.parse_size(obj["storage"])
-
-    def __len__(self):
-        return len(self.__dict__)
-
-    def __iter__(self):
-        for k in self.__dict__:
-            yield k, self.__dict__[k]
-
-    def __repr__(self):
-        return str(self.__dict__)
-
-
-class Resources:
-    limits: ResourceLimits = None
-    reservations: ResourceLimits = None
-
-    def __init__(self, obj={}):
-        if "reservations" in obj:
-            self.reservations = ResourceLimits(obj["reservations"])
-        if "limits" in obj:
-            self.limits = ResourceLimits(obj["limits"])
-
-    def __len__(self):
-        return len(self.__dict__)
-
-    def __iter__(self):
-        for k in self.__dict__:
-            yield k, self.__dict__[k]
-
-    def __repr__(self):
-        return str(self.__dict__)
-
-
 class Spec:
 
     obj: typing.Any
@@ -95,12 +47,6 @@ class Spec:
                 if self.obj and "configmaps" in self.obj
                 else {})
 
-    def get_container_resources(self):
-        return Resources(self.obj.get("resources", {}).get("containers", {}))
-
-    def get_volume_resources(self):
-        return Resources(self.obj.get("resources", {}).get("volumes", {}))
-
     def get_http_proxy(self):
         return (self.obj[constants.network_key][constants.http_proxy_key]
                 if self.obj and constants.network_key in self.obj

stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py

@@ -19,8 +19,6 @@ import shlex
 import shutil
 import sys
 import tempfile
-import time
-import uuid
 
 import click
 
@@ -29,7 +27,7 @@ from stack_orchestrator.deploy.webapp.util import (LaconicRegistryClient,
                                                    build_container_image, push_container_image,
                                                    file_hash, deploy_to_k8s, publish_deployment,
                                                    hostname_for_deployment_request, generate_hostname_for_app,
-                                                   match_owner, skip_by_tag)
+                                                   match_owner)
 
 
 def process_app_deployment_request(
@@ -41,19 +39,8 @@ def process_app_deployment_request(
     dns_suffix,
     deployment_parent_dir,
     kube_config,
-    image_registry,
-    log_parent_dir
+    image_registry
 ):
-    run_id = f"{app_deployment_request.id}-{str(time.time()).split('.')[0]}-{str(uuid.uuid4()).split('-')[0]}"
-    log_file = None
-    if log_parent_dir:
-        log_dir = os.path.join(log_parent_dir, app_deployment_request.id)
-        if not os.path.exists(log_dir):
-            os.mkdir(log_dir)
-        log_file_path = os.path.join(log_dir, f"{run_id}.log")
-        print(f"Directing build logs to: {log_file_path}")
-        log_file = open(log_file_path, "wt")
-
     # 1. look up application
     app = laconic.get_record(app_deployment_request.attributes.application, require=True)
 
@@ -115,10 +102,8 @@ def process_app_deployment_request(
     needs_k8s_deploy = False
     # 6. build container (if needed)
     if not deployment_record or deployment_record.attributes.application != app.id:
-        # TODO: pull from request
-        extra_build_args = []
-        build_container_image(app, deployment_container_tag, extra_build_args, log_file)
-        push_container_image(deployment_dir, log_file)
+        build_container_image(app, deployment_container_tag)
+        push_container_image(deployment_dir)
         needs_k8s_deploy = True
 
     # 7. update config (if needed)
@@ -131,7 +116,6 @@ def process_app_deployment_request(
         deploy_to_k8s(
             deployment_record,
             deployment_dir,
-            log_file
         )
 
     publish_deployment(
@@ -178,14 +162,10 @@ def dump_known_requests(filename, requests, status="SEEN"):
 @click.option("--record-namespace-dns", help="eg, crn://laconic/dns")
 @click.option("--record-namespace-deployments", help="eg, crn://laconic/deployments")
 @click.option("--dry-run", help="Don't do anything, just report what would be done.", is_flag=True)
-@click.option("--include-tags", help="Only include requests with matching tags (comma-separated).", default="")
-@click.option("--exclude-tags", help="Exclude requests with matching tags (comma-separated).", default="")
-@click.option("--log-dir", help="Output build/deployment logs to directory.", default=None)
 @click.pass_context
-def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_dir,  # noqa: C901
+def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_dir,
             request_id, discover, state_file, only_update_state,
-            dns_suffix, record_namespace_dns, record_namespace_deployments, dry_run,
-            include_tags, exclude_tags, log_dir):
+            dns_suffix, record_namespace_dns, record_namespace_deployments, dry_run):
     if request_id and discover:
         print("Cannot specify both --request-id and --discover", file=sys.stderr)
         sys.exit(2)
@@ -203,10 +183,6 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
             print("--dns-suffix, --record-namespace-dns, and --record-namespace-deployments are all required", file=sys.stderr)
             sys.exit(2)
 
-    # Split CSV and clean up values.
-    include_tags = [tag.strip() for tag in include_tags.split(",") if tag]
-    exclude_tags = [tag.strip() for tag in exclude_tags.split(",") if tag]
-
     laconic = LaconicRegistryClient(laconic_config)
 
     # Find deployment requests.
@@ -228,7 +204,6 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
     requests.sort(key=lambda r: r.createTime)
     requests.reverse()
     requests_by_name = {}
-    skipped_by_name = {}
     for r in requests:
         # TODO: Do this _after_ filtering deployments and cancellations to minimize round trips.
         app = laconic.get_record(r.attributes.application)
@@ -241,20 +216,17 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
             requested_name = generate_hostname_for_app(app)
             print("Generating name %s for request %s." % (requested_name, r.id))
 
-        if requested_name in skipped_by_name or requested_name in requests_by_name:
-            print("Ignoring request %s, it has been superseded." % r.id)
-            continue
-
-        if skip_by_tag(r, include_tags, exclude_tags):
-            print("Skipping request %s, filtered by tag (include %s, exclude %s, present %s)" % (r.id,
-                                                                                                 include_tags,
-                                                                                                 exclude_tags,
-                                                                                                 r.attributes.tags))
-            skipped_by_name[requested_name] = r
-            continue
-
-        print("Found request %s to run application %s on %s." % (r.id, r.attributes.application, requested_name))
-        requests_by_name[requested_name] = r
+        if requested_name not in requests_by_name:
+            print(
+                "Found request %s to run application %s on %s."
+                % (r.id, r.attributes.application, requested_name)
+            )
+            requests_by_name[requested_name] = r
+        else:
+            print(
+                "Ignoring request %s, it is superseded by %s."
+                % (r.id, requests_by_name[requested_name].id)
+            )
 
     # Find deployments.
     deployments = laconic.app_deployments()
@@ -301,8 +273,7 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
                     dns_suffix,
                     os.path.abspath(deployment_parent_dir),
                     kube_config,
-                    image_registry,
-                    log_dir
+                    image_registry
                 )
                 status = "DEPLOYED"
             finally:

stack_orchestrator/deploy/webapp/undeploy_webapp_from_registry.py

@@ -20,7 +20,7 @@ import sys
 
 import click
 
-from stack_orchestrator.deploy.webapp.util import LaconicRegistryClient, match_owner, skip_by_tag
+from stack_orchestrator.deploy.webapp.util import LaconicRegistryClient, match_owner
 
 
 def process_app_removal_request(ctx,
@@ -107,12 +107,10 @@ def dump_known_requests(filename, requests):
 @click.option("--delete-names/--preserve-names", help="Delete all names associated with removed deployments.", default=True)
 @click.option("--delete-volumes/--preserve-volumes", default=True, help="delete data volumes")
 @click.option("--dry-run", help="Don't do anything, just report what would be done.", is_flag=True)
-@click.option("--include-tags", help="Only include requests with matching tags (comma-separated).", default="")
-@click.option("--exclude-tags", help="Exclude requests with matching tags (comma-separated).", default="")
 @click.pass_context
 def command(ctx, laconic_config, deployment_parent_dir,
             request_id, discover, state_file, only_update_state,
-            delete_names, delete_volumes, dry_run, include_tags, exclude_tags):
+            delete_names, delete_volumes, dry_run):
     if request_id and discover:
         print("Cannot specify both --request-id and --discover", file=sys.stderr)
         sys.exit(2)
@@ -125,10 +123,6 @@ def command(ctx, laconic_config, deployment_parent_dir,
         print("--only-update-state requires --state-file", file=sys.stderr)
         sys.exit(2)
 
-    # Split CSV and clean up values.
-    include_tags = [tag.strip() for tag in include_tags.split(",") if tag]
-    exclude_tags = [tag.strip() for tag in exclude_tags.split(",") if tag]
-
     laconic = LaconicRegistryClient(laconic_config)
 
     # Find deployment removal requests.
@@ -161,22 +155,10 @@ def command(ctx, laconic_config, deployment_parent_dir,
             # TODO: should we handle CRNs?
             removals_by_deployment[r.attributes.deployment] = r
 
-    one_per_deployment = {}
+    requests_to_execute = []
     for r in requests:
         if not r.attributes.deployment:
             print(f"Skipping removal request {r.id} since it was a cancellation.")
-        elif r.attributes.deployment in one_per_deployment:
-            print(f"Skipping removal request {r.id} since it was superseded.")
-        else:
-            one_per_deployment[r.attributes.deployment] = r
-
-    requests_to_execute = []
-    for r in one_per_deployment.values():
-        if skip_by_tag(r, include_tags, exclude_tags):
-            print("Skipping removal request %s, filtered by tag (include %s, exclude %s, present %s)" % (r.id,
-                                                                                                         include_tags,
-                                                                                                         exclude_tags,
-                                                                                                         r.attributes.tags))
         elif r.id in removals_by_request:
             print(f"Found satisfied request for {r.id} at {removals_by_request[r.id].id}")
         elif r.attributes.deployment in removals_by_deployment:

stack_orchestrator/deploy/webapp/util.py

@@ -212,7 +212,7 @@ def determine_base_container(clone_dir, app_type="webapp"):
     return base_container
 
 
-def build_container_image(app_record, tag, extra_build_args=[], log_file=None):
+def build_container_image(app_record, tag, extra_build_args=[]):
     tmpdir = tempfile.mkdtemp()
 
     try:
@@ -227,10 +227,10 @@ def build_container_image(app_record, tag, extra_build_args=[], log_file=None):
             git_env = dict(os.environ.copy())
             # Never prompt
             git_env["GIT_TERMINAL_PROMPT"] = "0"
-            subprocess.check_call(["git", "clone", repo, clone_dir], env=git_env, stdout=log_file, stderr=log_file)
-            subprocess.check_call(["git", "checkout", ref], cwd=clone_dir, env=git_env, stdout=log_file, stderr=log_file)
+            subprocess.check_call(["git", "clone", repo, clone_dir], env=git_env)
+            subprocess.check_call(["git", "checkout", ref], cwd=clone_dir, env=git_env)
         else:
-            result = subprocess.run(["git", "clone", "--depth", "1", repo, clone_dir], stdout=log_file, stderr=log_file)
+            result = subprocess.run(["git", "clone", "--depth", "1", repo, clone_dir])
             result.check_returncode()
 
         base_container = determine_base_container(clone_dir, app_record.attributes.app_type)
@@ -246,27 +246,25 @@ def build_container_image(app_record, tag, extra_build_args=[], log_file=None):
             build_command.append("--extra-build-args")
             build_command.append(" ".join(extra_build_args))
 
-        result = subprocess.run(build_command, stdout=log_file, stderr=log_file)
+        result = subprocess.run(build_command)
         result.check_returncode()
     finally:
         cmd("rm", "-rf", tmpdir)
 
 
-def push_container_image(deployment_dir, log_file=None):
+def push_container_image(deployment_dir):
     print("Pushing image ...")
-    result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, "push-images"],
-                            stdout=log_file, stderr=log_file)
+    result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, "push-images"])
     result.check_returncode()
 
 
-def deploy_to_k8s(deploy_record, deployment_dir, log_file=None):
+def deploy_to_k8s(deploy_record, deployment_dir):
     if not deploy_record:
         command = "up"
     else:
         command = "update"
 
-    result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, command],
-                            stdout=log_file, stderr=log_file)
+    result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, command])
     result.check_returncode()
 
 
@@ -351,17 +349,3 @@ def generate_hostname_for_app(app):
     else:
         m.update(app.attributes.repository.encode())
     return "%s-%s" % (last_part, m.hexdigest()[0:10])
-
-
-def skip_by_tag(r, include_tags, exclude_tags):
-    for tag in exclude_tags:
-        if r.attributes.tags and tag in r.attributes.tags:
-            return True
-
-    if include_tags:
-        for tag in include_tags:
-            if r.attributes.tags and tag in r.attributes.tags:
-                return False
-        return True
-
-    return False

tests/deploy/run-deploy-test.sh

@@ -6,12 +6,6 @@ fi
 # Dump environment variables for debugging
 echo "Environment variables:"
 env
-
-delete_cluster_exit () {
-    $TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes
-    exit 1
-}
-
 # Test basic stack-orchestrator deploy
 echo "Running stack-orchestrator deploy test"
 # Bit of a hack, test the most recent package
@@ -112,10 +106,6 @@ if [ ! "$create_file_content" == "create-command-output-data"  ]; then
     echo "deploy create test: FAILED"
     exit 1
 fi
-
-# Add a config file to be picked up by the ConfigMap before starting.
-echo "dbfc7a4d-44a7-416d-b5f3-29842cc47650" > $test_deployment_dir/data/test-config/test_config
-
 echo "deploy create output file test: passed"
 # Try to start the deployment
 $TEST_TARGET_SO deployment --dir $test_deployment_dir start
@@ -134,37 +124,6 @@ else
     echo "deployment config test: FAILED"
     exit 1
 fi
-# Check the config variable CERC_TEST_PARAM_2 was passed correctly from the compose file
-if [[ "$log_output_3" == *"Test-param-2: CERC_TEST_PARAM_2_VALUE"* ]]; then
-    echo "deployment compose config test: passed"
-else
-    echo "deployment compose config test: FAILED"
-    exit 1
-fi
-
-# Check that the ConfigMap is mounted and contains the expected content.
-log_output_4=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
-if [[ "$log_output_4" == *"/config/test_config:"* ]] && [[ "$log_output_4" == *"dbfc7a4d-44a7-416d-b5f3-29842cc47650"* ]]; then
-    echo "deployment ConfigMap test: passed"
-else
-    echo "deployment ConfigMap test: FAILED"
-    delete_cluster_exit
-fi
-
-# Stop then start again and check the volume was preserved
-$TEST_TARGET_SO deployment --dir $test_deployment_dir stop
-# Sleep a bit just in case
-# sleep for longer to check if that's why the subsequent create cluster fails
-sleep 20
-$TEST_TARGET_SO deployment --dir $test_deployment_dir start
-log_output_5=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
-if [[ "$log_output_5" == *"Filesystem is old"* ]]; then
-    echo "Retain volumes test: passed"
-else
-    echo "Retain volumes test: FAILED"
-    delete_cluster_exit
-fi
-
 # Stop and clean up
 $TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes
 echo "Test passed"

tests/k8s-deploy/run-deploy-test.sh

@@ -114,7 +114,6 @@ else
     echo "deployment logs test: FAILED"
     delete_cluster_exit
 fi
-
 # Check the config variable CERC_TEST_PARAM_1 was passed correctly
 if [[ "$log_output_3" == *"Test-param-1: PASSED"* ]]; then
     echo "deployment config test: passed"
@@ -123,14 +122,6 @@ else
     delete_cluster_exit
 fi
 
-# Check the config variable CERC_TEST_PARAM_2 was passed correctly from the compose file
-if [[ "$log_output_3" == *"Test-param-2: CERC_TEST_PARAM_2_VALUE"* ]]; then
-    echo "deployment compose config test: passed"
-else
-    echo "deployment compose config test: FAILED"
-    exit 1
-fi
-
 # Check that the ConfigMap is mounted and contains the expected content.
 log_output_4=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
 if [[ "$log_output_4" == *"/config/test_config:"* ]] && [[ "$log_output_4" == *"dbfc7a4d-44a7-416d-b5f3-29842cc47650"* ]]; then