Turn off long run time test on push/pr unless explicitly triggered (#606)

.github/workflows/fixturenet-eth.yml

@@ -1,10 +1,11 @@
 name: Fixturenet-Eth Test
 
 on:
-  pull_request:
-    branches: '*'
   push:
     branches: '*'
+    paths:
+      - '!**'
+      - '.github/workflows/triggers/fixturenet-eth-test'
 
 jobs:
   test:

.github/workflows/triggers/fixturenet-eth-test

@@ -0,0 +1,2 @@
+Change this file to trigger running the fixturenet-eth-test CI job
+

.gitignore

@@ -7,4 +7,4 @@ __pycache__
 *~
 package
 app/data/build_tag.txt
-build
+/build

app/base.py

@@ -15,7 +15,7 @@
 
 import os
 from abc import ABC, abstractmethod
-from app.deploy import get_stack_status
+from app.deploy.deploy import get_stack_status
 from decouple import config
 
 

app/build/build_containers.py

@@ -52,7 +52,7 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args):
     continue_on_error = ctx.obj.continue_on_error
 
     # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
-    container_build_dir = Path(__file__).absolute().parent.joinpath("data", "container-build")
+    container_build_dir = Path(__file__).absolute().parent.parent.joinpath("data", "container-build")
 
     if local_stack:
         dev_root_path = os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")]

app/data/stacks/mainnet-laconic/deploy/commands.py

@@ -14,9 +14,9 @@
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
 from app.util import get_yaml
-from app.deploy_types import DeployCommandContext, LaconicStackSetupCommand, DeploymentContext
-from app.stack_state import State
-from app.deploy_util import VolumeMapping, run_container_command
+from app.deploy.deploy_types import DeployCommandContext, LaconicStackSetupCommand, DeploymentContext
+from app.deploy.stack_state import State
+from app.deploy.deploy_util import VolumeMapping, run_container_command
 from app.command_types import CommandOptions
 from enum import Enum
 from pathlib import Path

app/data/stacks/test/deploy/commands.py

@@ -14,9 +14,9 @@
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
 from app.util import get_yaml
-from app.deploy_types import DeployCommandContext
-from app.stack_state import State
-from app.deploy_util import VolumeMapping, run_container_command
+from app.deploy.deploy_types import DeployCommandContext
+from app.deploy.stack_state import State
+from app.deploy.deploy_util import VolumeMapping, run_container_command
 from pathlib import Path
 
 default_spec_file_content = """config:

app/deploy/compose/deploy_docker.py

@@ -14,7 +14,7 @@
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
 from python_on_whales import DockerClient, DockerException
-from app.deployer import Deployer, DeployerException
+from app.deploy.deployer import Deployer, DeployerException
 
 
 class DockerDeployer(Deployer):
@@ -24,37 +24,37 @@ class DockerDeployer(Deployer):
         self.docker = DockerClient(compose_files=compose_files, compose_project_name=compose_project_name,
                                    compose_env_file=compose_env_file)
 
-    def compose_up(self, detach, services):
+    def up(self, detach, services):
         try:
             return self.docker.compose.up(detach=detach, services=services)
         except DockerException as e:
             raise DeployerException(e)
 
-    def compose_down(self, timeout, volumes):
+    def down(self, timeout, volumes):
         try:
             return self.docker.compose.down(timeout=timeout, volumes=volumes)
         except DockerException as e:
             raise DeployerException(e)
 
-    def compose_ps(self):
+    def ps(self):
         try:
             return self.docker.compose.ps()
         except DockerException as e:
             raise DeployerException(e)
 
-    def compose_port(self, service, private_port):
+    def port(self, service, private_port):
         try:
             return self.docker.compose.port(service=service, private_port=private_port)
         except DockerException as e:
             raise DeployerException(e)
 
-    def compose_execute(self, service, command, envs):
+    def execute(self, service, command, envs):
         try:
             return self.docker.compose.execute(service=service, command=command, envs=envs)
         except DockerException as e:
             raise DeployerException(e)
 
-    def compose_logs(self, services, tail, follow, stream):
+    def logs(self, services, tail, follow, stream):
         try:
             return self.docker.compose.logs(services=services, tail=tail, follow=follow, stream=stream)
         except DockerException as e:

app/deploy/deploy.py

@@ -25,12 +25,12 @@ import subprocess
 import click
 from pathlib import Path
 from app.util import include_exclude_check, get_parsed_stack_config, global_options2, get_dev_root_path
-from app.deployer import Deployer, DeployerException
-from app.deployer_factory import getDeployer
-from app.deploy_types import ClusterContext, DeployCommandContext
-from app.deployment_create import create as deployment_create
-from app.deployment_create import init as deployment_init
-from app.deployment_create import setup as deployment_setup
+from app.deploy.deployer import Deployer, DeployerException
+from app.deploy.deployer_factory import getDeployer
+from app.deploy.deploy_types import ClusterContext, DeployCommandContext
+from app.deploy.deployment_create import create as deployment_create
+from app.deploy.deployment_create import init as deployment_init
+from app.deploy.deployment_create import setup as deployment_setup
 
 
 @click.group()
@@ -80,7 +80,7 @@ def up_operation(ctx, services_list, stay_attached=False):
             print(f"Running compose up with container_exec_env: {container_exec_env}, extra_args: {services_list}")
         for pre_start_command in cluster_context.pre_start_commands:
             _run_command(global_context, cluster_context.cluster, pre_start_command)
-        deploy_context.deployer.compose_up(detach=not stay_attached, services=services_list)
+        deploy_context.deployer.up(detach=not stay_attached, services=services_list)
         for post_start_command in cluster_context.post_start_commands:
             _run_command(global_context, cluster_context.cluster, post_start_command)
         _orchestrate_cluster_config(global_context, cluster_context.config, deploy_context.deployer, container_exec_env)
@@ -95,7 +95,7 @@ def down_operation(ctx, delete_volumes, extra_args_list):
         if extra_args_list:
             timeout_arg = extra_args_list[0]
         # Specify shutdown timeout (default 10s) to give services enough time to shutdown gracefully
-        ctx.obj.deployer.compose_down(timeout=timeout_arg, volumes=delete_volumes)
+        ctx.obj.deployer.down(timeout=timeout_arg, volumes=delete_volumes)
 
 
 def ps_operation(ctx):
@@ -103,7 +103,7 @@ def ps_operation(ctx):
     if not global_context.dry_run:
         if global_context.verbose:
             print("Running compose ps")
-        container_list = ctx.obj.deployer.compose_ps()
+        container_list = ctx.obj.deployer.ps()
         if len(container_list) > 0:
             print("Running containers:")
             for container in container_list:
@@ -134,7 +134,7 @@ def port_operation(ctx, extra_args):
         exposed_port = extra_args_list[1]
         if global_context.verbose:
             print(f"Running compose port {service_name} {exposed_port}")
-        mapped_port_data = ctx.obj.deployer.compose_port(service_name, exposed_port)
+        mapped_port_data = ctx.obj.deployer.port(service_name, exposed_port)
         print(f"{mapped_port_data[0]}:{mapped_port_data[1]}")
 
 
@@ -151,7 +151,7 @@ def exec_operation(ctx, extra_args):
         if global_context.verbose:
             print(f"Running compose exec {service_name} {command_to_exec}")
         try:
-            ctx.obj.deployer.compose_execute(service_name, command_to_exec, envs=container_exec_env)
+            ctx.obj.deployer.execute(service_name, command_to_exec, envs=container_exec_env)
         except DeployerException:
             print("container command returned error exit status")
 
@@ -163,7 +163,7 @@ def logs_operation(ctx, tail: int, follow: bool, extra_args: str):
         if global_context.verbose:
             print("Running compose logs")
         services_list = extra_args_list if extra_args_list is not None else []
-        logs_stream = ctx.obj.deployer.compose_logs(services=services_list, tail=tail, follow=follow, stream=True)
+        logs_stream = ctx.obj.deployer.logs(services=services_list, tail=tail, follow=follow, stream=True)
         for stream_type, stream_content in logs_stream:
             print(stream_content.decode("utf-8"), end="")
 
@@ -224,7 +224,7 @@ def get_stack_status(ctx, stack):
     # TODO: refactor to avoid duplicating this code above
     if ctx.verbose:
         print("Running compose ps")
-    container_list = deployer.compose_ps()
+    container_list = deployer.ps()
     if len(container_list) > 0:
         if ctx.debug:
             print(f"Container list from compose ps: {container_list}")
@@ -258,7 +258,7 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file):
         deployment = True
     else:
         # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
-        compose_dir = Path(__file__).absolute().parent.joinpath("data", "compose")
+        compose_dir = Path(__file__).absolute().parent.parent.joinpath("data", "compose")
 
     if cluster is None:
         # Create default unique, stable cluster name from confile file path and stack name if provided
@@ -396,12 +396,12 @@ def _orchestrate_cluster_config(ctx, cluster_config, deployer, container_exec_en
                     # TODO: fix the script paths so they're consistent between containers
                     source_value = None
                     try:
-                        source_value = deployer.compose_execute(pd.source_container,
-                                                                ["sh", "-c",
-                                                                 "sh /docker-entrypoint-scripts.d/export-"
-                                                                 f"{pd.source_variable}.sh"],
-                                                                tty=False,
-                                                                envs=container_exec_env)
+                        source_value = deployer.execute(pd.source_container,
+                                                        ["sh", "-c",
+                                                         "sh /docker-entrypoint-scripts.d/export-"
+                                                         f"{pd.source_variable}.sh"],
+                                                        tty=False,
+                                                        envs=container_exec_env)
                     except DeployerException as error:
                         if ctx.debug:
                             print(f"Docker exception reading config source: {error}")
@@ -417,12 +417,12 @@ def _orchestrate_cluster_config(ctx, cluster_config, deployer, container_exec_en
                     if source_value:
                         if ctx.debug:
                             print(f"fetched source value: {source_value}")
-                        destination_output = deployer.compose_execute(pd.destination_container,
-                                                                      ["sh", "-c",
-                                                                       f"sh /scripts/import-{pd.destination_variable}.sh"
-                                                                       f" {source_value}"],
-                                                                      tty=False,
-                                                                      envs=container_exec_env)
+                        destination_output = deployer.execute(pd.destination_container,
+                                                              ["sh", "-c",
+                                                               f"sh /scripts/import-{pd.destination_variable}.sh"
+                                                               f" {source_value}"],
+                                                              tty=False,
+                                                              envs=container_exec_env)
                         waiting_for_data = False
                     if ctx.debug:
                         print(f"destination output: {destination_output}")

app/deploy/deploy_types.py

@@ -17,7 +17,7 @@ from typing import List
 from dataclasses import dataclass
 from pathlib import Path
 from app.command_types import CommandOptions
-from app.deployer import Deployer
+from app.deploy.deployer import Deployer
 
 
 @dataclass

app/deploy/deploy_util.py

@@ -15,7 +15,7 @@
 
 import os
 from typing import List
-from app.deploy_types import DeployCommandContext, VolumeMapping
+from app.deploy.deploy_types import DeployCommandContext, VolumeMapping
 from app.util import get_parsed_stack_config, get_yaml, get_compose_file_dir, get_pod_list
 
 

app/deploy/deployer.py

@@ -19,27 +19,27 @@ from abc import ABC, abstractmethod
 class Deployer(ABC):
 
     @abstractmethod
-    def compose_up(self, detach, services):
+    def up(self, detach, services):
         pass
 
     @abstractmethod
-    def compose_down(self, timeout, volumes):
+    def down(self, timeout, volumes):
         pass
 
     @abstractmethod
-    def compose_ps(self):
+    def ps(self):
         pass
 
     @abstractmethod
-    def compose_port(self, service, private_port):
+    def port(self, service, private_port):
         pass
 
     @abstractmethod
-    def compose_execute(self, service_name, command, envs):
+    def execute(self, service_name, command, envs):
         pass
 
     @abstractmethod
-    def compose_logs(self, services, tail, follow, stream):
+    def logs(self, services, tail, follow, stream):
         pass
 
     @abstractmethod

app/deploy/deployer_factory.py

@@ -13,8 +13,8 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
-from app.deploy_k8s import K8sDeployer
-from app.deploy_docker import DockerDeployer
+from app.deploy.k8s.deploy_k8s import K8sDeployer
+from app.deploy.compose.deploy_docker import DockerDeployer
 
 
 def getDeployer(type, compose_files, compose_project_name, compose_env_file):

app/deploy/deployment.py

@@ -16,10 +16,10 @@
 import click
 from pathlib import Path
 import sys
-from app.deploy import up_operation, down_operation, ps_operation, port_operation
-from app.deploy import exec_operation, logs_operation, create_deploy_context
-from app.stack import Stack
-from app.spec import Spec
+from app.deploy.deploy import up_operation, down_operation, ps_operation, port_operation
+from app.deploy.deploy import exec_operation, logs_operation, create_deploy_context
+from app.deploy.stack import Stack
+from app.deploy.spec import Spec
 
 
 class DeploymentContext:

app/deploy/deployment_create.py

@@ -23,7 +23,7 @@ from shutil import copy, copyfile, copytree
 import sys
 from app.util import (get_stack_file_path, get_parsed_deployment_spec, get_parsed_stack_config, global_options, get_yaml,
                       get_pod_list, get_pod_file_path, pod_has_scripts, get_pod_script_paths, get_plugin_code_path)
-from app.deploy_types import DeploymentContext, DeployCommandContext, LaconicStackSetupCommand
+from app.deploy.deploy_types import DeploymentContext, DeployCommandContext, LaconicStackSetupCommand
 
 
 def _make_default_deployment_dir():
@@ -325,7 +325,7 @@ def create(ctx, spec_file, deployment_dir, network_dir, initial_peers):
     os.mkdir(destination_compose_dir)
     destination_pods_dir = os.path.join(deployment_dir, "pods")
     os.mkdir(destination_pods_dir)
-    data_dir = Path(__file__).absolute().parent.joinpath("data")
+    data_dir = Path(__file__).absolute().parent.parent.joinpath("data")
     yaml = get_yaml()
     for pod in pods:
         pod_file_path = get_pod_file_path(parsed_stack, pod)

app/deploy/k8s/deploy_k8s.py

@@ -14,7 +14,7 @@
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
 from kubernetes import client, config
-from app.deployer import Deployer
+from app.deploy.deployer import Deployer
 
 
 class K8sDeployer(Deployer):
@@ -24,22 +24,22 @@ class K8sDeployer(Deployer):
         config.load_kube_config()
         self.client = client.CoreV1Api()
 
-    def compose_up(self, detach, services):
+    def up(self, detach, services):
         pass
 
-    def compose_down(self, timeout, volumes):
+    def down(self, timeout, volumes):
         pass
 
-    def compose_ps(self):
+    def ps(self):
         pass
 
-    def compose_port(self, service, private_port):
+    def port(self, service, private_port):
         pass
 
-    def compose_execute(self, service_name, command, envs):
+    def execute(self, service_name, command, envs):
         pass
 
-    def compose_logs(self, services, tail, follow, stream):
+    def logs(self, services, tail, follow, stream):
         pass
 
     def run(self, image, command, user, volumes, entrypoint=None):

app/repos/setup_repositories.py

@@ -240,7 +240,7 @@ def command(ctx, include, exclude, git_ssh, check_only, pull, branches, branches
     if stack:
         # In order to be compatible with Python 3.8 we need to use this hack to get the path:
         # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
-        stack_file_path = Path(__file__).absolute().parent.joinpath("data", "stacks", stack, "stack.yml")
+        stack_file_path = Path(__file__).absolute().parent.parent.joinpath("data", "stacks", stack, "stack.yml")
         with stack_file_path:
             stack_config = yaml.safe_load(open(stack_file_path, "r"))
             # TODO: syntax check the input here

cli.py

@@ -16,12 +16,12 @@
 import click
 
 from app.command_types import CommandOptions
-from app import setup_repositories
-from app import build_containers
-from app import build_npms
-from app import deploy
+from app.repos import setup_repositories
+from app.build import build_containers
+from app.build import build_npms
+from app.deploy import deploy
 from app import version
-from app import deployment
+from app.deploy import deployment
 from app import update
 
 CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])