Rename app -> stack_orchestrator (#625)

add fixturenet-laconicd test

.gitea/workflows/fixturenet-laconicd-test.yml

@@ -0,0 +1,55 @@
+name: Fixturenet-Laconicd-Test
+
+on:
+  push:
+    branches: '*'
+    paths:
+      - '!**'
+      - '.gitea/workflows/triggers/fixturenet-laconicd-test'
+
+# Needed until we can incorporate docker startup into the executor container
+env:
+  DOCKER_HOST: unix:///var/run/dind.sock
+
+
+jobs:
+  test:
+    name: "Run an Laconicd fixturenet test"
+    runs-on: ubuntu-latest
+    steps:
+      - name: 'Update'
+        run: apt-get update
+      - name: 'Setup jq'
+        run: apt-get install jq -y
+      - name: 'Check jq'
+        run: |
+          which jq
+          jq --version
+      - name: "Clone project repository"
+        uses: actions/checkout@v3
+      # At present the stock setup-python action fails on Linux/aarch64
+      # Conditional steps below workaroud this by using deadsnakes for that case only
+      - name: "Install Python for ARM on Linux"
+        if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
+        uses: deadsnakes/action@v3.0.1
+        with:
+          python-version: '3.8'
+      - name: "Install Python cases other than ARM on Linux"
+        if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+      - name: "Print Python version"
+        run: python3 --version
+      - name: "Install shiv"
+        run: pip install shiv
+      - name: "Generate build version file"
+        run: ./scripts/create_build_tag_file.sh
+      - name: "Build local shiv package"
+        run: ./scripts/build_shiv_package.sh
+      - name: Start dockerd # Also needed until we can incorporate into the executor
+        run: |
+          dockerd -H $DOCKER_HOST --userland-proxy=false &
+          sleep 5
+      - name: "Run fixturenet-laconicd tests"
+        run: ./tests/fixturenet-laconicd/run-test.sh

.gitea/workflows/triggers/fixturenet-eth-test

@@ -1,2 +1,2 @@
 Change this file to trigger running the fixturenet-eth-test CI job
-trigger
+

.gitea/workflows/triggers/fixturenet-laconicd-test

@@ -0,0 +1,2 @@
+Change this file to trigger running the fixturenet-laconicd-test CI job
+

.github/workflows/fixturenet-eth.yml

@@ -1,10 +1,11 @@
 name: Fixturenet-Eth Test
 
 on:
-  pull_request:
-    branches: '*'
   push:
     branches: '*'
+    paths:
+      - '!**'
+      - '.github/workflows/triggers/fixturenet-eth-test'
 
 jobs:
   test:

.github/workflows/fixturenet-laconicd.yml

@@ -0,0 +1,30 @@
+name: Fixturenet-Laconicd Test
+
+on:
+  push:
+    branches: '*'
+    paths:
+      - '!**'
+      - '.github/workflows/triggers/fixturenet-laconicd-test'
+
+jobs:
+  test:
+    name: "Run fixturenet-laconicd test suite"
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Clone project repository"
+        uses: actions/checkout@v3
+      - name: "Install Python"
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+      - name: "Print Python version"
+        run: python3 --version
+      - name: "Install shiv"
+        run: pip install shiv
+      - name: "Generate build version file"
+        run: ./scripts/create_build_tag_file.sh
+      - name: "Build local shiv package"
+        run: ./scripts/build_shiv_package.sh
+      - name: "Run fixturenet-laconicd tests"
+        run: ./tests/fixturenet-laconicd/run-test.sh

.github/workflows/triggers/fixturenet-eth-test

@@ -0,0 +1,2 @@
+Change this file to trigger running the fixturenet-eth-test CI job
+

.github/workflows/triggers/fixturenet-laconicd-test

@@ -0,0 +1,3 @@
+Change this file to trigger running the fixturenet-laconicd-test CI job
+
+trigger

.gitignore

@@ -6,5 +6,5 @@ laconic_stack_orchestrator.egg-info
 __pycache__
 *~
 package
-app/data/build_tag.txt
-build
+stack_orchestrator/data/build_tag.txt
+/build

README.md

@@ -64,12 +64,12 @@ laconic-so update
 
 ## Usage
 
-The various [stacks](/app/data/stacks) each contain instructions for running different stacks based on your use case. For example:
+The various [stacks](/stack_orchestrator/data/stacks) each contain instructions for running different stacks based on your use case. For example:
 
-- [self-hosted Gitea](/app/data/stacks/build-support)
-- [an Optimism Fixturenet](/app/data/stacks/fixturenet-optimism)
-- [laconicd with console and CLI](app/data/stacks/fixturenet-laconic-loaded)
-- [kubo (IPFS)](app/data/stacks/kubo)
+- [self-hosted Gitea](/stack_orchestrator/data/stacks/build-support)
+- [an Optimism Fixturenet](/stack_orchestrator/data/stacks/fixturenet-optimism)
+- [laconicd with console and CLI](stack_orchestrator/data/stacks/fixturenet-laconic-loaded)
+- [kubo (IPFS)](stack_orchestrator/data/stacks/kubo)
 
 ## Contributing
 

app/data/container-build/cerc-laconic-registry-cli/create-demo-records.sh

@@ -1,29 +0,0 @@
-#!/usr/bin/env bash
-# Create some demo/test records in the registry
-set -e
-if [ -n "$CERC_SCRIPT_DEBUG" ]; then
-  set -x
-fi
-registry_command="laconic cns"
-record_1_filename=demo-record-1.yml
-cat <<EOF > ${record_1_filename}
-record:
-  type: WebsiteRegistrationRecord
-  url: 'https://cerc.io'
-  repo_registration_record_cid: QmSnuWmxptJZdLJpKRarxBMS2Ju2oANVrgbr2xWbie9b2D
-  build_artifact_cid: QmP8jTG1m9GSDJLCbeWhVSVgEzCPPwXRdCRuJtQ5Tz9Kc9
-  tls_cert_cid: QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR
-  version: 1.0.23
-EOF
-# Check we have funds
-funds_response=$(${registry_command} account get --address $(cat my-address.txt))
-funds_balance=$(echo ${funds_response} | jq -r .[0].balance[0].quantity)
-echo "Balance is: ${funds_balance}"
-# Create a bond
-bond_create_result=$(${registry_command} bond create --type aphoton --quantity 1000000000)
-bond_id=$(echo ${bond_create_result} | jq -r .bondId)
-echo "Created bond with id: ${bond_id}"
-# Publish a demo record
-publish_response=$(${registry_command} record publish --filename ${record_1_filename} --bond-id ${bond_id})
-published_record_id=$(echo ${publish_response} | jq -r .id)
-echo "Published ${record_1_filename} with id: ${published_record_id}"

docs/adding-a-new-stack.md

@@ -8,7 +8,7 @@ Core to the feature completeness of stack orchestrator is to [decouple the tool
 
 ## Example
 
-- in `app/data/stacks/my-new-stack/stack.yml` add:
+- in `stack_orchestrator/data/stacks/my-new-stack/stack.yml` add:
 
 ```yaml
 version: "0.1"
@@ -21,7 +21,7 @@ pods:
   - my-new-stack
 ```
 
-- in `app/data/container-build/cerc-my-new-stack/build.sh` add:
+- in `stack_orchestrator/data/container-build/cerc-my-new-stack/build.sh` add:
 
 ```yaml
 #!/usr/bin/env bash
@@ -30,7 +30,7 @@ source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
 docker build -t cerc/my-new-stack:local -f ${CERC_REPO_BASE_DIR}/my-new-stack/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/my-new-stack
 ```
 
-- in `app/data/compose/docker-compose-my-new-stack.yml` add:
+- in `stack_orchestrator/data/compose/docker-compose-my-new-stack.yml` add:
 
 ```yaml
 version: "3.2"
@@ -43,20 +43,20 @@ services:
       - "0.0.0.0:3000:3000"
 ```
 
-- in `app/data/repository-list.txt` add:
+- in `stack_orchestrator/data/repository-list.txt` add:
 
 ```bash
 github.com/my-org/my-new-stack
 ```
 whereby that repository contains your source code and a `Dockerfile`, and matches the `repos:` field in the `stack.yml`.
 
-- in `app/data/container-image-list.txt` add:
+- in `stack_orchestrator/data/container-image-list.txt` add:
 
 ```bash
 cerc/my-new-stack
 ```
 
-- in `app/data/pod-list.txt` add:
+- in `stack_orchestrator/data/pod-list.txt` add:
 
 ```bash
 my-new-stack

docs/laconicd-with-console.md

@@ -2,10 +2,10 @@
 
 The following tutorial explains the steps to run a laconicd fixturenet with CLI and web console that displays records in the registry. It is designed as an introduction to Stack Orchestrator and to showcase one component of the Laconic Stack. Prior to Stack Orchestrator, the following 4 repositories had to be cloned and setup manually:
 
-- https://github.com/cerc-io/laconicd
-- https://github.com/cerc-io/laconic-sdk
-- https://github.com/cerc-io/laconic-registry-cli
-- https://github.com/cerc-io/laconic-console
+- https://git.vdb.to/cerc-io/laconicd
+- https://git.vdb.to/cerc-io/laconic-sdk
+- https://git.vdb.to/cerc-io/laconic-registry-cli
+- https://git.vdb.to/cerc-io/laconic-console
 
 Now, with Stack Orchestrator, it is a few quick commands. Additionally, the `docker` and `docker compose` integration on the back-end allows the stack to easily persist, facilitating workflows.
 
@@ -52,7 +52,7 @@ laconic-so version
 1. Get the repositories
 
 ```
-laconic-so --stack fixturenet-laconic-loaded setup-repositories --include github.com/cerc-io/laconicd,github.com/cerc-io/laconic-sdk,github.com/cerc-io/laconic-registry-cli,github.com/cerc-io/laconic-console
+laconic-so --stack fixturenet-laconic-loaded setup-repositories --include git.vdb.to/cerc-io/laconicd,git.vdb.to/cerc-io/laconic-sdk,git.vdb.to/cerc-io/laconic-registry-cli,git.vdb.to/cerc-io/laconic-console
 ```
 
 2. Set this environment variable to the Laconic self-hosted Gitea instance:
@@ -212,7 +212,7 @@ record:
 
 3. Try out additional CLI commands
 
-- these are documented [here](https://github.com/cerc-io/laconic-registry-cli#readme) and updates are forthcoming
+- these are documented [here](https://git.vdb.to/cerc-io/laconic-registry-cli#readme) and updates are forthcoming
 - e.g,:
 
 ```

requirements.txt

@@ -8,3 +8,4 @@ ruamel.yaml>=0.17.32
 pydantic==1.10.9
 tomli==2.0.1
 validators==0.22.0
+kubernetes>=28.1.0

scripts/create_build_tag_file.sh

@@ -1,6 +1,6 @@
-build_tag_file_name=./app/data/build_tag.txt
+build_tag_file_name=./stack_orchestrator/data/build_tag.txt
 echo "# This file should be re-generated running: scripts/create_build_tag_file.sh script" > $build_tag_file_name
-product_version_string=$( tail -1 ./app/data/version.txt )
+product_version_string=$( tail -1 ./stack_orchestrator/data/version.txt )
 commit_string=$( git rev-parse --short HEAD )
 timestamp_string=$(date +'%Y%m%d%H%M')
 build_tag_string=${product_version_string}-${commit_string}-${timestamp_string}

setup.py

@@ -14,7 +14,7 @@ setup(
     long_description=long_description,
     long_description_content_type="text/markdown",
     url='https://github.com/cerc-io/stack-orchestrator',
-    py_modules=['cli', 'app'],
+    py_modules=['stack_orchestrator'],
     packages=find_packages(),
     install_requires=[requirements],
     python_requires='>=3.7',
@@ -25,6 +25,6 @@ setup(
         "Operating System :: OS Independent",
     ],
     entry_points={
-        'console_scripts': ['laconic-so=cli:cli'],
+        'console_scripts': ['laconic-so=stack_orchestrator.main:cli'],
     }
 )

stack_orchestrator/base.py

@@ -15,7 +15,7 @@
 
 import os
 from abc import ABC, abstractmethod
-from app.deploy import get_stack_status
+from stack_orchestrator.deploy.deploy import get_stack_status
 from decouple import config
 
 

stack_orchestrator/build/build_containers.py

@@ -27,8 +27,8 @@ import subprocess
 import click
 import importlib.resources
 from pathlib import Path
-from app.util import include_exclude_check, get_parsed_stack_config
-from app.base import get_npm_registry_url
+from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config
+from stack_orchestrator.base import get_npm_registry_url
 
 # TODO: find a place for this
 #    epilog="Config provided either in .env or settings.ini or env vars: CERC_REPO_BASE_DIR (defaults to ~/cerc)"
@@ -52,7 +52,7 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args):
     continue_on_error = ctx.obj.continue_on_error
 
     # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
-    container_build_dir = Path(__file__).absolute().parent.joinpath("data", "container-build")
+    container_build_dir = Path(__file__).absolute().parent.parent.joinpath("data", "container-build")
 
     if local_stack:
         dev_root_path = os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")]
@@ -67,7 +67,7 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args):
         print('Dev root directory doesn\'t exist, creating')
 
     # See: https://stackoverflow.com/a/20885799/1701505
-    from app import data
+    from stack_orchestrator import data
     with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file:
         all_containers = container_list_file.read().splitlines()
 

stack_orchestrator/build/build_npms.py

@@ -25,8 +25,8 @@ from decouple import config
 import click
 import importlib.resources
 from python_on_whales import docker, DockerException
-from app.base import get_stack
-from app.util import include_exclude_check, get_parsed_stack_config
+from stack_orchestrator.base import get_stack
+from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config
 
 builder_js_image_name = "cerc/builder-js:local"
 
@@ -83,7 +83,7 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args):
         os.makedirs(build_root_path)
 
     # See: https://stackoverflow.com/a/20885799/1701505
-    from app import data
+    from stack_orchestrator import data
     with importlib.resources.open_text(data, "npm-package-list.txt") as package_list_file:
         all_packages = package_list_file.read().splitlines()
 

stack_orchestrator/data/compose/docker-compose-mainnet-eth-ipld-eth-db.yml

@@ -0,0 +1,29 @@
+version: "3.2"
+
+services:
+  migrations:
+    restart: on-failure
+    depends_on:
+      ipld-eth-db:
+        condition: service_healthy
+    image: cerc/ipld-eth-db:local
+    env_file:
+      - ../config/mainnet-eth-ipld-eth-db/db.env
+
+  ipld-eth-db:
+    image: timescale/timescaledb:2.8.1-pg14
+    restart: always
+    env_file:
+      - ../config/mainnet-eth-ipld-eth-db/db.env
+    volumes:
+      - mainnet_eth_ipld_eth_db:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "5432"]
+      interval: 30s
+      timeout: 10s
+      retries: 10
+      start_period: 3s
+    ports:
+      - "5432"
+volumes:
+  mainnet_eth_ipld_eth_db:

stack_orchestrator/data/compose/docker-compose-mainnet-eth-ipld-eth-server.yml

@@ -0,0 +1,24 @@
+version: "3.7"
+services:
+  ipld-eth-server:
+    restart: always
+    depends_on:
+      ipld-eth-db:
+        condition: service_healthy
+    image: cerc/ipld-eth-server:local
+    env_file:
+      - ../config/mainnet-eth-ipld-eth-db/db.env
+      - ../config/mainnet-eth-ipld-eth-server/srv.env
+    volumes:
+      - ../config/mainnet-eth-ipld-eth-server/config.toml:/app/config.toml:ro
+    ports:
+      - "8081"
+      - "8082"
+      - "8090"
+      - "40001"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "8081"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s

stack_orchestrator/data/compose/docker-compose-mainnet-eth-keycloak.yml

@@ -6,7 +6,7 @@ services:
     env_file:
       - ../config/mainnet-eth-keycloak/keycloak.env
     healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "5432"]
+      test: ["CMD", "nc", "-v", "localhost", "35432"]
       interval: 30s
       timeout: 10s
       retries: 10
@@ -14,7 +14,7 @@ services:
     volumes:
       - mainnet_eth_keycloak_db:/var/lib/postgresql/data
     ports:
-      - 5432
+      - 35432
 
   keycloak:
     image: cerc/keycloak:local

stack_orchestrator/data/compose/docker-compose-mainnet-eth-plugeth.yml

@@ -0,0 +1,72 @@
+
+services:
+
+  mainnet-eth-geth-1:
+    restart: always
+    hostname: mainnet-eth-geth-1
+    cap_add:
+      - SYS_PTRACE
+    image: cerc/plugeth-with-plugins:local
+    entrypoint: /bin/sh
+    command: -c "/opt/run-geth.sh"
+    env_file:
+      - ../config/mainnet-eth-ipld-eth-db/db.env
+      - ../config/mainnet-eth-plugeth/geth.env
+    volumes:
+      - mainnet_eth_plugeth_geth_1_data:/data
+      - mainnet_eth_plugeth_config_data:/etc/mainnet-eth
+      - ../config/mainnet-eth-plugeth/scripts/run-geth.sh:/opt/run-geth.sh
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "8545"]
+      interval: 30s
+      timeout: 10s
+      retries: 10
+      start_period: 3s
+    ports:
+      # http api
+      - "8545"
+      # ws api
+      - "8546"
+      # ws el
+      - "8551"
+      # p2p
+      - "30303"
+      - "30303/udp"
+      # debugging
+      - "40000"
+      # metrics
+      - "6060"
+
+  mainnet-eth-lighthouse-1:
+    restart: always
+    hostname: mainnet-eth-lighthouse-1
+    healthcheck:
+      test: ["CMD", "wget", "--tries=1", "--connect-timeout=1", "--quiet", "-O", "-", "http://localhost:5052/eth/v2/beacon/blocks/head"]
+      interval: 30s
+      timeout: 10s
+      retries: 10
+      start_period: 30s
+    environment:
+      LIGHTHOUSE_EXECUTION_ENDPOINT: "http://mainnet-eth-geth-1:8551"
+    env_file:
+      - ../config/mainnet-eth-plugeth/lighthouse.env
+    image: cerc/lighthouse:local
+    entrypoint: /bin/sh
+    command: -c "/opt/run-lighthouse.sh"
+    volumes:
+      - mainnet_eth_plugeth_lighthouse_1_data:/data
+      - mainnet_eth_plugeth_config_data:/etc/mainnet-eth
+      - ../config/mainnet-eth-plugeth/scripts/run-lighthouse.sh:/opt/run-lighthouse.sh
+    ports:
+      # api
+      - "5052"
+      # metrics
+      - "5054"
+      # p2p
+      - "9000"
+      - "9000/udp"
+
+volumes:
+  mainnet_eth_plugeth_config_data:
+  mainnet_eth_plugeth_geth_1_data:
+  mainnet_eth_plugeth_lighthouse_1_data:

stack_orchestrator/data/config/fixturenet-laconicd/registry-cli-config-template.yml

@@ -5,5 +5,5 @@ services:
     userKey: REPLACE_WITH_MYKEY
     bondId:
     chainId: laconic_9000-1
-    gas: 250000
+    gas: 350000
     fees: 200000aphoton