Turn off long run time test on push/pr unless explicitly triggered (#606)

* increase gas

* create six demo records

* print record

* use moon

* typos

* use git.vdb.t

* use the right moon

* ok

.gitea/workflows/triggers/fixturenet-eth-test

@@ -1,2 +1,2 @@
 Change this file to trigger running the fixturenet-eth-test CI job
-trigger
+

.github/workflows/fixturenet-eth.yml

@@ -0,0 +1,30 @@
+name: Fixturenet-Eth Test
+
+on:
+  push:
+    branches: '*'
+    paths:
+      - '!**'
+      - '.github/workflows/triggers/fixturenet-eth-test'
+
+jobs:
+  test:
+    name: "Run fixturenet-eth test suite"
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Clone project repository"
+        uses: actions/checkout@v3
+      - name: "Install Python"
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+      - name: "Print Python version"
+        run: python3 --version
+      - name: "Install shiv"
+        run: pip install shiv
+      - name: "Generate build version file"
+        run: ./scripts/create_build_tag_file.sh
+      - name: "Build local shiv package"
+        run: ./scripts/build_shiv_package.sh
+      - name: "Run fixturenet-eth tests"
+        run: ./tests/fixturenet-eth/run-test.sh

.github/workflows/triggers/fixturenet-eth-test

@@ -0,0 +1,2 @@
+Change this file to trigger running the fixturenet-eth-test CI job
+

.gitignore

@@ -7,4 +7,4 @@ __pycache__
 *~
 package
 app/data/build_tag.txt
-build
+/build

app/base.py

@@ -15,7 +15,7 @@
 
 import os
 from abc import ABC, abstractmethod
-from app.deploy import get_stack_status
+from app.deploy.deploy import get_stack_status
 from decouple import config
 
 

app/build/build_containers.py

@@ -52,7 +52,7 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args):
     continue_on_error = ctx.obj.continue_on_error
 
     # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
-    container_build_dir = Path(__file__).absolute().parent.joinpath("data", "container-build")
+    container_build_dir = Path(__file__).absolute().parent.parent.joinpath("data", "container-build")
 
     if local_stack:
         dev_root_path = os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")]

app/data/config/fixturenet-laconicd/registry-cli-config-template.yml

@@ -5,5 +5,5 @@ services:
     userKey: REPLACE_WITH_MYKEY
     bondId:
     chainId: laconic_9000-1
-    gas: 250000
+    gas: 350000
     fees: 200000aphoton

app/data/container-build/cerc-laconic-registry-cli/Dockerfile

@@ -54,8 +54,10 @@ RUN yarn global add @cerc-io/laconic-registry-cli
 
 # Add scripts
 RUN mkdir /scripts
+RUN mkdir /scripts/demo-records
 ENV PATH="${PATH}:/scripts"
 COPY ./create-demo-records.sh /scripts
+COPY ./demo-records /scripts/demo-records
 COPY ./import-key.sh /scripts
 COPY ./import-address.sh /scripts
 

app/data/container-build/cerc-laconic-registry-cli/create-demo-records.sh

@@ -4,26 +4,26 @@ set -e
 if [ -n "$CERC_SCRIPT_DEBUG" ]; then
   set -x
 fi
+
 registry_command="laconic cns"
-record_1_filename=demo-record-1.yml
-cat <<EOF > ${record_1_filename}
-record:
-  type: WebsiteRegistrationRecord
-  url: 'https://cerc.io'
-  repo_registration_record_cid: QmSnuWmxptJZdLJpKRarxBMS2Ju2oANVrgbr2xWbie9b2D
-  build_artifact_cid: QmP8jTG1m9GSDJLCbeWhVSVgEzCPPwXRdCRuJtQ5Tz9Kc9
-  tls_cert_cid: QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR
-  version: 1.0.23
-EOF
+demo_records_dir="scripts/demo-records"
+
 # Check we have funds
 funds_response=$(${registry_command} account get --address $(cat my-address.txt))
 funds_balance=$(echo ${funds_response} | jq -r .[0].balance[0].quantity)
 echo "Balance is: ${funds_balance}"
+
 # Create a bond
 bond_create_result=$(${registry_command} bond create --type aphoton --quantity 1000000000)
 bond_id=$(echo ${bond_create_result} | jq -r .bondId)
 echo "Created bond with id: ${bond_id}"
-# Publish a demo record
-publish_response=$(${registry_command} record publish --filename ${record_1_filename} --bond-id ${bond_id})
-published_record_id=$(echo ${publish_response} | jq -r .id)
-echo "Published ${record_1_filename} with id: ${published_record_id}"
+
+## Publish the demo records
+if [ -d $demo_records_dir ]; then
+  for demo_record in "${demo_records_dir}"/*; do
+    publish_response=$(${registry_command} record publish --filename ${demo_record} --bond-id ${bond_id})
+    published_record_id=$(echo ${publish_response} | jq -r .id)
+    echo "Published ${demo_record} with id: ${published_record_id}"
+    cat ${demo_record}
+  done
+fi

app/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-1.yml

@@ -0,0 +1,7 @@
+record:
+  type: WebsiteRegistrationRecord
+  url: 'https://hello-urbit.laconic.com'
+  repo_registration_record_cid: QmTZQ8ZJS6mALjEM2wY71msFno6zzxFftVCiZELj9xREPx
+  build_artifact_cid: '~lostex-rabdur-labtul-moltev/hello-urbit'
+  tls_cert_cid: QmR1acEmQt7Tjmhp9cFtymie2eFcrHURQKt9kGto1TQTW1
+  version: 0.2.4

app/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-2.yml

@@ -0,0 +1,7 @@
+record:
+  type: WebsiteRegistrationRecord
+  url: 'https://dev-hello-urbit.laconic.com'
+  repo_registration_record_cid: QmPbKkK7gKojnGAgKAE6bK5qBY8BcYp2YDB1e4b28NxvVv
+  build_artifact_cid: '~lostex-rabdur-labtul-moltev/hello-urbit-dev'
+  tls_cert_cid: QmWbCv4k9Q9P6XptHNJN6EupDyKVLSLRV6W6T2sAYkfBxr
+  version: 0.2.5

app/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-3.yml

@@ -0,0 +1,7 @@
+record:
+  type: WebsiteRegistrationRecord
+  url: 'https://app.osmosis.zone'
+  repo_registration_record_cid: QmSnuWmxptJZdLJpKRarxBMS2Ju2oANVrgbr2xWbie9b2D
+  build_artifact_cid: '~larryx-woldyr/osmosis:latest'
+  tls_cert_cid: QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR
+  version: 1.3.5

app/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-4.yml

@@ -0,0 +1,7 @@
+record:
+  type: WebsiteRegistrationRecord
+  url: 'https://dev.osmosis.zone'
+  repo_registration_record_cid: QmPbKkK7gKojnGAgKAE6bK5qBY8BcYp2YDB1e4b28NxvVv
+  build_artifact_cid: '~larryx-woldyr/osmosis:dev'
+  tls_cert_cid: QmWbCv4k9Q9P6XptHNJN6EupDyKVLSLRV6W6T2sAYkfBxr
+  version: 1.3.6

app/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-5.yml

@@ -0,0 +1,7 @@
+record:
+  type: WebsiteRegistrationRecord
+  url: 'https://pwa-1-1-3.iglootools.xyz'
+  repo_registration_record_cid: QmSnuWmxptJZdLJpKRarxBMS2Ju2oANVrgbr2xWbie9b2D
+  build_artifact_cid: https://ipfs.io/ipfs/QmP8jTG1m9GSDJLCbeWhVSVgEzCPPwXRdCRuJtQ5Tz9Kc9
+  tls_cert_cid: QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR
+  version: 1.1.3

app/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-6.yml

@@ -0,0 +1,7 @@
+record:
+  type: WebsiteRegistrationRecord
+  url: 'https://pwa-1-2-3.iglootools.xyz'
+  repo_registration_record_cid: QmVjYQ59z3fuKfYUV1aXhDR82JAKxdqB5JMPfUsHkJUkJq
+  build_artifact_cid: https://ipfs.io/ipfs/QmN2yevgjC92gzGi2BY6Q2yBWmR6JNWQ9yscoPPyy1V2Hv
+  tls_cert_cid: QmTAnmDMSnzqAN3yyXtFD4KpyATkeze62ZkbmXHjJ8xQW8
+  version: 1.2.3

app/data/stacks/mainnet-laconic/deploy/commands.py

@@ -14,9 +14,9 @@
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
 from app.util import get_yaml
-from app.deploy_types import DeployCommandContext, LaconicStackSetupCommand, DeploymentContext
-from app.stack_state import State
-from app.deploy_util import VolumeMapping, run_container_command
+from app.deploy.deploy_types import DeployCommandContext, LaconicStackSetupCommand, DeploymentContext
+from app.deploy.stack_state import State
+from app.deploy.deploy_util import VolumeMapping, run_container_command
 from app.command_types import CommandOptions
 from enum import Enum
 from pathlib import Path

app/data/stacks/test/deploy/commands.py

@@ -14,9 +14,9 @@
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
 from app.util import get_yaml
-from app.deploy_types import DeployCommandContext
-from app.stack_state import State
-from app.deploy_util import VolumeMapping, run_container_command
+from app.deploy.deploy_types import DeployCommandContext
+from app.deploy.stack_state import State
+from app.deploy.deploy_util import VolumeMapping, run_container_command
 from pathlib import Path
 
 default_spec_file_content = """config:

app/deploy/compose/deploy_docker.py

@@ -0,0 +1,67 @@
+# Copyright © 2022, 2023 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+from python_on_whales import DockerClient, DockerException
+from app.deploy.deployer import Deployer, DeployerException
+
+
+class DockerDeployer(Deployer):
+    name: str = "compose"
+
+    def __init__(self, compose_files, compose_project_name, compose_env_file) -> None:
+        self.docker = DockerClient(compose_files=compose_files, compose_project_name=compose_project_name,
+                                   compose_env_file=compose_env_file)
+
+    def up(self, detach, services):
+        try:
+            return self.docker.compose.up(detach=detach, services=services)
+        except DockerException as e:
+            raise DeployerException(e)
+
+    def down(self, timeout, volumes):
+        try:
+            return self.docker.compose.down(timeout=timeout, volumes=volumes)
+        except DockerException as e:
+            raise DeployerException(e)
+
+    def ps(self):
+        try:
+            return self.docker.compose.ps()
+        except DockerException as e:
+            raise DeployerException(e)
+
+    def port(self, service, private_port):
+        try:
+            return self.docker.compose.port(service=service, private_port=private_port)
+        except DockerException as e:
+            raise DeployerException(e)
+
+    def execute(self, service, command, envs):
+        try:
+            return self.docker.compose.execute(service=service, command=command, envs=envs)
+        except DockerException as e:
+            raise DeployerException(e)
+
+    def logs(self, services, tail, follow, stream):
+        try:
+            return self.docker.compose.logs(services=services, tail=tail, follow=follow, stream=stream)
+        except DockerException as e:
+            raise DeployerException(e)
+
+    def run(self, image, command, user, volumes, entrypoint=None):
+        try:
+            return self.docker.run(image=image, command=command, user=user, volumes=volumes, entrypoint=entrypoint)
+        except DockerException as e:
+            raise DeployerException(e)

app/deploy/deploy.py

@@ -13,7 +13,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
-# Deploys the system components using docker-compose
+# Deploys the system components using a deployer (either docker-compose or k8s)
 
 import hashlib
 import copy
@@ -22,14 +22,15 @@ import sys
 from dataclasses import dataclass
 from importlib import resources
 import subprocess
-from python_on_whales import DockerClient, DockerException
 import click
 from pathlib import Path
 from app.util import include_exclude_check, get_parsed_stack_config, global_options2, get_dev_root_path
-from app.deploy_types import ClusterContext, DeployCommandContext
-from app.deployment_create import create as deployment_create
-from app.deployment_create import init as deployment_init
-from app.deployment_create import setup as deployment_setup
+from app.deploy.deployer import Deployer, DeployerException
+from app.deploy.deployer_factory import getDeployer
+from app.deploy.deploy_types import ClusterContext, DeployCommandContext
+from app.deploy.deployment_create import create as deployment_create
+from app.deploy.deployment_create import init as deployment_init
+from app.deploy.deployment_create import setup as deployment_setup
 
 
 @click.group()
@@ -37,8 +38,9 @@ from app.deployment_create import setup as deployment_setup
 @click.option("--exclude", help="don\'t start these components")
 @click.option("--env-file", help="env file to be used")
 @click.option("--cluster", help="specify a non-default cluster name")
+@click.option("--deploy-to", help="cluster system to deploy to (compose or k8s)")
 @click.pass_context
-def command(ctx, include, exclude, env_file, cluster):
+def command(ctx, include, exclude, env_file, cluster, deploy_to):
     '''deploy a stack'''
 
     # Although in theory for some subcommands (e.g. deploy create) the stack can be inferred,
@@ -50,16 +52,20 @@ def command(ctx, include, exclude, env_file, cluster):
 
     if ctx.parent.obj.debug:
         print(f"ctx.parent.obj: {ctx.parent.obj}")
-    ctx.obj = create_deploy_context(global_options2(ctx), stack, include, exclude, cluster, env_file)
+
+    if deploy_to is None:
+        deploy_to = "compose"
+
+    ctx.obj = create_deploy_context(global_options2(ctx), stack, include, exclude, cluster, env_file, deploy_to)
     # Subcommand is executed now, by the magic of click
 
 
-def create_deploy_context(global_context, stack, include, exclude, cluster, env_file):
+def create_deploy_context(global_context, stack, include, exclude, cluster, env_file, deployer):
     cluster_context = _make_cluster_context(global_context, stack, include, exclude, cluster, env_file)
     # See: https://gabrieldemarmiesse.github.io/python-on-whales/sub-commands/compose/
-    docker = DockerClient(compose_files=cluster_context.compose_files, compose_project_name=cluster_context.cluster,
-                          compose_env_file=cluster_context.env_file)
-    return DeployCommandContext(stack, cluster_context, docker)
+    deployer = getDeployer(deployer, compose_files=cluster_context.compose_files, compose_project_name=cluster_context.cluster,
+                           compose_env_file=cluster_context.env_file)
+    return DeployCommandContext(stack, cluster_context, deployer)
 
 
 def up_operation(ctx, services_list, stay_attached=False):
@@ -74,10 +80,10 @@ def up_operation(ctx, services_list, stay_attached=False):
             print(f"Running compose up with container_exec_env: {container_exec_env}, extra_args: {services_list}")
         for pre_start_command in cluster_context.pre_start_commands:
             _run_command(global_context, cluster_context.cluster, pre_start_command)
-        deploy_context.docker.compose.up(detach=not stay_attached, services=services_list)
+        deploy_context.deployer.up(detach=not stay_attached, services=services_list)
         for post_start_command in cluster_context.post_start_commands:
             _run_command(global_context, cluster_context.cluster, post_start_command)
-        _orchestrate_cluster_config(global_context, cluster_context.config, deploy_context.docker, container_exec_env)
+        _orchestrate_cluster_config(global_context, cluster_context.config, deploy_context.deployer, container_exec_env)
 
 
 def down_operation(ctx, delete_volumes, extra_args_list):
@@ -89,7 +95,7 @@ def down_operation(ctx, delete_volumes, extra_args_list):
         if extra_args_list:
             timeout_arg = extra_args_list[0]
         # Specify shutdown timeout (default 10s) to give services enough time to shutdown gracefully
-        ctx.obj.docker.compose.down(timeout=timeout_arg, volumes=delete_volumes)
+        ctx.obj.deployer.down(timeout=timeout_arg, volumes=delete_volumes)
 
 
 def ps_operation(ctx):
@@ -97,7 +103,7 @@ def ps_operation(ctx):
     if not global_context.dry_run:
         if global_context.verbose:
             print("Running compose ps")
-        container_list = ctx.obj.docker.compose.ps()
+        container_list = ctx.obj.deployer.ps()
         if len(container_list) > 0:
             print("Running containers:")
             for container in container_list:
@@ -128,7 +134,7 @@ def port_operation(ctx, extra_args):
         exposed_port = extra_args_list[1]
         if global_context.verbose:
             print(f"Running compose port {service_name} {exposed_port}")
-        mapped_port_data = ctx.obj.docker.compose.port(service_name, exposed_port)
+        mapped_port_data = ctx.obj.deployer.port(service_name, exposed_port)
         print(f"{mapped_port_data[0]}:{mapped_port_data[1]}")
 
 
@@ -145,8 +151,8 @@ def exec_operation(ctx, extra_args):
         if global_context.verbose:
             print(f"Running compose exec {service_name} {command_to_exec}")
         try:
-            ctx.obj.docker.compose.execute(service_name, command_to_exec, envs=container_exec_env)
-        except DockerException:
+            ctx.obj.deployer.execute(service_name, command_to_exec, envs=container_exec_env)
+        except DeployerException:
             print("container command returned error exit status")
 
 
@@ -157,7 +163,7 @@ def logs_operation(ctx, tail: int, follow: bool, extra_args: str):
         if global_context.verbose:
             print("Running compose logs")
         services_list = extra_args_list if extra_args_list is not None else []
-        logs_stream = ctx.obj.docker.compose.logs(services=services_list, tail=tail, follow=follow, stream=True)
+        logs_stream = ctx.obj.deployer.logs(services=services_list, tail=tail, follow=follow, stream=True)
         for stream_type, stream_content in logs_stream:
             print(stream_content.decode("utf-8"), end="")
 
@@ -214,11 +220,11 @@ def get_stack_status(ctx, stack):
     ctx_copy.stack = stack
 
     cluster_context = _make_cluster_context(ctx_copy, stack, None, None, None, None)
-    docker = DockerClient(compose_files=cluster_context.compose_files, compose_project_name=cluster_context.cluster)
+    deployer = Deployer(compose_files=cluster_context.compose_files, compose_project_name=cluster_context.cluster)
     # TODO: refactor to avoid duplicating this code above
     if ctx.verbose:
         print("Running compose ps")
-    container_list = docker.compose.ps()
+    container_list = deployer.ps()
     if len(container_list) > 0:
         if ctx.debug:
             print(f"Container list from compose ps: {container_list}")
@@ -252,7 +258,7 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file):
         deployment = True
     else:
         # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
-        compose_dir = Path(__file__).absolute().parent.joinpath("data", "compose")
+        compose_dir = Path(__file__).absolute().parent.parent.joinpath("data", "compose")
 
     if cluster is None:
         # Create default unique, stable cluster name from confile file path and stack name if provided
@@ -359,7 +365,7 @@ def _run_command(ctx, cluster_name, command):
         sys.exit(1)
 
 
-def _orchestrate_cluster_config(ctx, cluster_config, docker, container_exec_env):
+def _orchestrate_cluster_config(ctx, cluster_config, deployer, container_exec_env):
 
     @dataclass
     class ConfigDirective:
@@ -390,13 +396,13 @@ def _orchestrate_cluster_config(ctx, cluster_config, docker, container_exec_env)
                     # TODO: fix the script paths so they're consistent between containers
                     source_value = None
                     try:
-                        source_value = docker.compose.execute(pd.source_container,
-                                                              ["sh", "-c",
-                                                                  "sh /docker-entrypoint-scripts.d/export-"
-                                                                  f"{pd.source_variable}.sh"],
-                                                              tty=False,
-                                                              envs=container_exec_env)
-                    except DockerException as error:
+                        source_value = deployer.execute(pd.source_container,
+                                                        ["sh", "-c",
+                                                         "sh /docker-entrypoint-scripts.d/export-"
+                                                         f"{pd.source_variable}.sh"],
+                                                        tty=False,
+                                                        envs=container_exec_env)
+                    except DeployerException as error:
                         if ctx.debug:
                             print(f"Docker exception reading config source: {error}")
                         # If the script executed failed for some reason, we get:
@@ -411,12 +417,12 @@ def _orchestrate_cluster_config(ctx, cluster_config, docker, container_exec_env)
                     if source_value:
                         if ctx.debug:
                             print(f"fetched source value: {source_value}")
-                        destination_output = docker.compose.execute(pd.destination_container,
-                                                                    ["sh", "-c",
-                                                                        f"sh /scripts/import-{pd.destination_variable}.sh"
-                                                                        f" {source_value}"],
-                                                                    tty=False,
-                                                                    envs=container_exec_env)
+                        destination_output = deployer.execute(pd.destination_container,
+                                                              ["sh", "-c",
+                                                               f"sh /scripts/import-{pd.destination_variable}.sh"
+                                                               f" {source_value}"],
+                                                              tty=False,
+                                                              envs=container_exec_env)
                         waiting_for_data = False
                     if ctx.debug:
                         print(f"destination output: {destination_output}")

app/deploy/deploy_types.py

@@ -16,8 +16,8 @@
 from typing import List
 from dataclasses import dataclass
 from pathlib import Path
-from python_on_whales import DockerClient
 from app.command_types import CommandOptions
+from app.deploy.deployer import Deployer
 
 
 @dataclass
@@ -35,7 +35,7 @@ class ClusterContext:
 class DeployCommandContext:
     stack: str
     cluster_context: ClusterContext
-    docker: DockerClient
+    deployer: Deployer
 
 
 @dataclass

app/deploy/deploy_util.py

@@ -15,7 +15,7 @@
 
 import os
 from typing import List
-from app.deploy_types import DeployCommandContext, VolumeMapping
+from app.deploy.deploy_types import DeployCommandContext, VolumeMapping
 from app.util import get_parsed_stack_config, get_yaml, get_compose_file_dir, get_pod_list
 
 
@@ -47,12 +47,12 @@ def _volumes_to_docker(mounts: List[VolumeMapping]):
 
 
 def run_container_command(ctx: DeployCommandContext, service: str, command: str, mounts: List[VolumeMapping]):
-    docker = ctx.docker
+    deployer = ctx.deployer
     container_image = _container_image_from_service(ctx.stack, service)
     docker_volumes = _volumes_to_docker(mounts)
     if ctx.cluster_context.options.debug:
         print(f"Running this command in {service} container: {command}")
-    docker_output = docker.run(
+    docker_output = deployer.run(
         container_image,
         ["-c", command], entrypoint="sh",
         user=f"{os.getuid()}:{os.getgid()}",

app/deploy/deployer.py

@@ -0,0 +1,52 @@
+# Copyright © 2023 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+from abc import ABC, abstractmethod
+
+
+class Deployer(ABC):
+
+    @abstractmethod
+    def up(self, detach, services):
+        pass
+
+    @abstractmethod
+    def down(self, timeout, volumes):
+        pass
+
+    @abstractmethod
+    def ps(self):
+        pass
+
+    @abstractmethod
+    def port(self, service, private_port):
+        pass
+
+    @abstractmethod
+    def execute(self, service_name, command, envs):
+        pass
+
+    @abstractmethod
+    def logs(self, services, tail, follow, stream):
+        pass
+
+    @abstractmethod
+    def run(self, image, command, user, volumes, entrypoint):
+        pass
+
+
+class DeployerException(Exception):
+    def __init__(self, *args: object) -> None:
+        super().__init__(*args)

app/deploy/deployer_factory.py

@@ -0,0 +1,26 @@
+# Copyright © 2023 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+from app.deploy.k8s.deploy_k8s import K8sDeployer
+from app.deploy.compose.deploy_docker import DockerDeployer
+
+
+def getDeployer(type, compose_files, compose_project_name, compose_env_file):
+    if type == "compose" or type is None:
+        return DockerDeployer(compose_files, compose_project_name, compose_env_file)
+    elif type == "k8s":
+        return K8sDeployer(compose_files, compose_project_name, compose_env_file)
+    else:
+        print(f"ERROR: deploy-to {type} is not valid")

app/deploy/deployment.py

@@ -14,20 +14,25 @@
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
 import click
-from dataclasses import dataclass
 from pathlib import Path
 import sys
-from app.deploy import up_operation, down_operation, ps_operation, port_operation
-from app.deploy import exec_operation, logs_operation, create_deploy_context
+from app.deploy.deploy import up_operation, down_operation, ps_operation, port_operation
+from app.deploy.deploy import exec_operation, logs_operation, create_deploy_context
+from app.deploy.stack import Stack
+from app.deploy.spec import Spec
 
 
-@dataclass
 class DeploymentContext:
     dir: Path
+    spec: Spec
+    stack: Stack
 
     def get_stack_file(self):
         return self.dir.joinpath("stack.yml")
 
+    def get_spec_file(self):
+        return self.dir.joinpath("spec.yml")
+
     def get_env_file(self):
         return self.dir.joinpath("config.env")
 
@@ -35,12 +40,19 @@ class DeploymentContext:
     def get_cluster_name(self):
         return None
 
+    def init(self, dir):
+        self.dir = dir
+        self.stack = Stack()
+        self.stack.init_from_file(self.get_stack_file())
+        self.spec = Spec()
+        self.spec.init_from_file(self.get_spec_file())
+
 
 @click.group()
 @click.option("--dir", required=True, help="path to deployment directory")
 @click.pass_context
 def command(ctx, dir):
-    '''create a deployment'''
+    '''manage a deployment'''
 
     # Check that --stack wasn't supplied
     if ctx.parent.obj.stack:
@@ -55,14 +67,18 @@ def command(ctx, dir):
         print(f"Error: supplied deployment directory path {dir} exists but is a file not a directory")
         sys.exit(1)
     # Store the deployment context for subcommands
-    ctx.obj = DeploymentContext(dir_path)
+    deployment_context = DeploymentContext()
+    deployment_context.init(dir_path)
+    ctx.obj = deployment_context
 
 
 def make_deploy_context(ctx):
-    stack_file_path = ctx.obj.get_stack_file()
-    env_file = ctx.obj.get_env_file()
-    cluster_name = ctx.obj.get_cluster_name()
-    return create_deploy_context(ctx.parent.parent.obj, stack_file_path, None, None, cluster_name, env_file)
+    context: DeploymentContext = ctx.obj
+    stack_file_path = context.get_stack_file()
+    env_file = context.get_env_file()
+    cluster_name = context.get_cluster_name()
+    return create_deploy_context(ctx.parent.parent.obj, stack_file_path, None, None, cluster_name, env_file,
+                                 context.spec.obj["deploy-to"])
 
 
 @command.command()

app/deploy/deployment_create.py

@@ -23,7 +23,7 @@ from shutil import copy, copyfile, copytree
 import sys
 from app.util import (get_stack_file_path, get_parsed_deployment_spec, get_parsed_stack_config, global_options, get_yaml,
                       get_pod_list, get_pod_file_path, pod_has_scripts, get_pod_script_paths, get_plugin_code_path)
-from app.deploy_types import DeploymentContext, DeployCommandContext, LaconicStackSetupCommand
+from app.deploy.deploy_types import DeploymentContext, DeployCommandContext, LaconicStackSetupCommand
 
 
 def _make_default_deployment_dir():
@@ -249,7 +249,7 @@ def init(ctx, config, output, map_ports_to_host):
     stack = global_options(ctx).stack
     debug = global_options(ctx).debug
     default_spec_file_content = call_stack_deploy_init(ctx.obj)
-    spec_file_content = {"stack": stack}
+    spec_file_content = {"stack": stack, "deploy-to": ctx.obj.deployer.name}
     if default_spec_file_content:
         spec_file_content.update(default_spec_file_content)
     config_variables = _parse_config_variables(config)
@@ -315,7 +315,7 @@ def create(ctx, spec_file, deployment_dir, network_dir, initial_peers):
         sys.exit(1)
     os.mkdir(deployment_dir)
     # Copy spec file and the stack file into the deployment dir
-    copyfile(spec_file, os.path.join(deployment_dir, os.path.basename(spec_file)))
+    copyfile(spec_file, os.path.join(deployment_dir, "spec.yml"))
     copyfile(stack_file, os.path.join(deployment_dir, os.path.basename(stack_file)))
     # Copy any config varibles from the spec file into an env file suitable for compose
     _write_config_file(spec_file, os.path.join(deployment_dir, "config.env"))
@@ -325,7 +325,7 @@ def create(ctx, spec_file, deployment_dir, network_dir, initial_peers):
     os.mkdir(destination_compose_dir)
     destination_pods_dir = os.path.join(deployment_dir, "pods")
     os.mkdir(destination_pods_dir)
-    data_dir = Path(__file__).absolute().parent.joinpath("data")
+    data_dir = Path(__file__).absolute().parent.parent.joinpath("data")
     yaml = get_yaml()
     for pod in pods:
         pod_file_path = get_pod_file_path(parsed_stack, pod)

app/deploy/k8s/deploy_k8s.py

@@ -0,0 +1,46 @@
+# Copyright © 2023 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+from kubernetes import client, config
+from app.deploy.deployer import Deployer
+
+
+class K8sDeployer(Deployer):
+    name: str = "k8s"
+
+    def __init__(self, compose_files, compose_project_name, compose_env_file) -> None:
+        config.load_kube_config()
+        self.client = client.CoreV1Api()
+
+    def up(self, detach, services):
+        pass
+
+    def down(self, timeout, volumes):
+        pass
+
+    def ps(self):
+        pass
+
+    def port(self, service, private_port):
+        pass
+
+    def execute(self, service_name, command, envs):
+        pass
+
+    def logs(self, services, tail, follow, stream):
+        pass
+
+    def run(self, image, command, user, volumes, entrypoint=None):
+        pass

app/deploy/spec.py

@@ -0,0 +1,30 @@
+# Copyright © 2022, 2023 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+from pathlib import Path
+import typing
+from app.util import get_yaml
+
+
+class Spec:
+
+    obj: typing.Any
+
+    def __init__(self) -> None:
+        pass
+
+    def init_from_file(self, file_path: Path):
+        with file_path:
+            self.obj = get_yaml().load(open(file_path, "r"))

app/deploy/stack.py

@@ -0,0 +1,30 @@
+# Copyright © 2023 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+from pathlib import Path
+import typing
+from app.util import get_yaml
+
+
+class Stack:
+
+    obj: typing.Any
+
+    def __init__(self) -> None:
+        pass
+
+    def init_from_file(self, file_path: Path):
+        with file_path:
+            self.obj = get_yaml().load(open(file_path, "r"))

app/repos/setup_repositories.py

@@ -240,7 +240,7 @@ def command(ctx, include, exclude, git_ssh, check_only, pull, branches, branches
     if stack:
         # In order to be compatible with Python 3.8 we need to use this hack to get the path:
         # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
-        stack_file_path = Path(__file__).absolute().parent.joinpath("data", "stacks", stack, "stack.yml")
+        stack_file_path = Path(__file__).absolute().parent.parent.joinpath("data", "stacks", stack, "stack.yml")
         with stack_file_path:
             stack_config = yaml.safe_load(open(stack_file_path, "r"))
             # TODO: syntax check the input here

cli.py

@@ -16,12 +16,12 @@
 import click
 
 from app.command_types import CommandOptions
-from app import setup_repositories
-from app import build_containers
-from app import build_npms
-from app import deploy
+from app.repos import setup_repositories
+from app.build import build_containers
+from app.build import build_npms
+from app.deploy import deploy
 from app import version
-from app import deployment
+from app.deploy import deployment
 from app import update
 
 CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])

docs/laconicd-with-console.md

@@ -2,10 +2,10 @@
 
 The following tutorial explains the steps to run a laconicd fixturenet with CLI and web console that displays records in the registry. It is designed as an introduction to Stack Orchestrator and to showcase one component of the Laconic Stack. Prior to Stack Orchestrator, the following 4 repositories had to be cloned and setup manually:
 
-- https://github.com/cerc-io/laconicd
-- https://github.com/cerc-io/laconic-sdk
-- https://github.com/cerc-io/laconic-registry-cli
-- https://github.com/cerc-io/laconic-console
+- https://git.vdb.to/cerc-io/laconicd
+- https://git.vdb.to/cerc-io/laconic-sdk
+- https://git.vdb.to/cerc-io/laconic-registry-cli
+- https://git.vdb.to/cerc-io/laconic-console
 
 Now, with Stack Orchestrator, it is a few quick commands. Additionally, the `docker` and `docker compose` integration on the back-end allows the stack to easily persist, facilitating workflows.
 
@@ -52,7 +52,7 @@ laconic-so version
 1. Get the repositories
 
 ```
-laconic-so --stack fixturenet-laconic-loaded setup-repositories --include github.com/cerc-io/laconicd,github.com/cerc-io/laconic-sdk,github.com/cerc-io/laconic-registry-cli,github.com/cerc-io/laconic-console
+laconic-so --stack fixturenet-laconic-loaded setup-repositories --include git.vdb.to/cerc-io/laconicd,git.vdb.to/cerc-io/laconic-sdk,git.vdb.to/cerc-io/laconic-registry-cli,git.vdb.to/cerc-io/laconic-console
 ```
 
 2. Set this environment variable to the Laconic self-hosted Gitea instance:
@@ -212,7 +212,7 @@ record:
 
 3. Try out additional CLI commands
 
-- these are documented [here](https://github.com/cerc-io/laconic-registry-cli#readme) and updates are forthcoming
+- these are documented [here](https://git.vdb.to/cerc-io/laconic-registry-cli#readme) and updates are forthcoming
 - e.g,:
 
 ```

requirements.txt

@@ -8,3 +8,4 @@ ruamel.yaml>=0.17.32
 pydantic==1.10.9
 tomli==2.0.1
 validators==0.22.0
+kubernetes>=28.1.0

tests/fixturenet-eth/run-test.sh

@@ -24,13 +24,45 @@ echo "$(date +"%Y-%m-%d %T"): Stack started"
 $TEST_TARGET_SO --stack fixturenet-eth deploy ps
 # echo "$(date +"%Y-%m-%d %T"): Getting stack status"
 # $TEST_TARGET_SO --stack fixturenet-eth deploy exec fixturenet-eth-bootnode-lighthouse /scripts/status-internal.sh
-echo "$(date +"%Y-%m-%d %T"): Getting initial block number"
-initial_block_number=$($TEST_TARGET_SO --stack fixturenet-eth deploy exec foundry "cast block-number")
-# Check that the block number increases some time later
-sleep 120
-echo "$(date +"%Y-%m-%d %T"): Getting subsequent block number"
-subsequent_block_number=$($TEST_TARGET_SO  --stack fixturenet-eth deploy exec foundry "cast block-number")
+
+timeout=900 # 15 minutes
+echo "$(date +"%Y-%m-%d %T"): Getting initial block number. Timeout set to $timeout seconds"
+start_time=$(date +%s)
+elapsed_time=0
+initial_block_number=0
+while [ "$initial_block_number" -eq 0 ]  && [ $elapsed_time -lt $timeout ]; do
+  sleep 10
+  echo "$(date +"%Y-%m-%d %T"): Waiting for initial block..."
+  initial_block_number=$($TEST_TARGET_SO --stack fixturenet-eth deploy exec foundry "cast block-number")
+  current_time=$(date +%s)
+  elapsed_time=$((current_time - start_time))
+done
+
+subsequent_block_number=$initial_block_number
+
+# if initial block was 0 after timeout, assume chain did not start successfully and skip finding subsequent block
+if [[ $initial_block_number -gt 0 ]]; then
+  timeout=300
+  echo "$(date +"%Y-%m-%d %T"): Getting subsequent block number. Timeout set to $timeout seconds"
+  start_time=$(date +%s)
+  elapsed_time=0
+  # wait for 5 blocks or timeout
+  while [ "$subsequent_block_number" -le $((initial_block_number + 5)) ]  && [ $elapsed_time -lt $timeout ]; do
+    sleep 10
+    echo "$(date +"%Y-%m-%d %T"): Waiting for five blocks or $timeout seconds..."
+    subsequent_block_number=$($TEST_TARGET_SO --stack fixturenet-eth deploy exec foundry "cast block-number")
+    current_time=$(date +%s)
+    elapsed_time=$((current_time - start_time))
+  done
+fi
+
+# will return 0 if either of the above loops timed out
 block_number_difference=$((subsequent_block_number - initial_block_number))
+
+echo "$(date +"%Y-%m-%d %T"): Results of block height queries:"
+echo "Initial block height: $initial_block_number"
+echo "Subsequent block height: $subsequent_block_number"
+
 # Block height difference should be between 1 and some small number
 if [[ $block_number_difference -gt 1 && $block_number_difference -lt 100 ]]; then
   echo "Test passed"
@@ -41,7 +73,7 @@ else
   $TEST_TARGET_SO --stack fixturenet-eth deploy logs
   test_result=1
 fi
-$TEST_TARGET_SO --stack fixturenet-eth deploy down
+$TEST_TARGET_SO --stack fixturenet-eth deploy down --delete-volumes
 echo "$(date +"%Y-%m-%d %T"): Removing cloned repositories"
 rm -rf $CERC_REPO_BASE_DIR
 echo "$(date +"%Y-%m-%d %T"): Test finished"