forked from cerc-io/stack-orchestrator
Fixes for external stack deployment (#851)
Fixes - stack path resolution for `build` - external stack path resolution for deployments - "extra" config detection - `deployment ports` command - `version` command in dist or source install (without build_tag.txt) - `setup-repos`, so it won't die when an existing repo is not at a branch or exact tag Used in cerc-io/fixturenet-eth-stacks#14 Reviewed-on: cerc-io/stack-orchestrator#851 Reviewed-by: David Boreham <dboreham@noreply.git.vdb.to>
This commit is contained in:
parent
a2d6201be9
commit
36d4969b2d
4
setup.py
4
setup.py
@ -4,9 +4,11 @@ with open("README.md", "r", encoding="utf-8") as fh:
|
||||
long_description = fh.read()
|
||||
with open("requirements.txt", "r", encoding="utf-8") as fh:
|
||||
requirements = fh.read()
|
||||
with open("stack_orchestrator/data/version.txt", "r", encoding="utf-8") as fh:
|
||||
version = fh.readlines()[-1].strip(" \n")
|
||||
setup(
|
||||
name='laconic-stack-orchestrator',
|
||||
version='1.0.12',
|
||||
version=version,
|
||||
author='Cerc',
|
||||
author_email='info@cerc.io',
|
||||
license='GNU Affero General Public License',
|
||||
|
||||
@ -21,11 +21,6 @@ from stack_orchestrator.util import get_parsed_stack_config, warn_exit
|
||||
|
||||
def get_containers_in_scope(stack: str):
|
||||
|
||||
# See: https://stackoverflow.com/a/20885799/1701505
|
||||
from stack_orchestrator import data
|
||||
with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file:
|
||||
all_containers = container_list_file.read().splitlines()
|
||||
|
||||
containers_in_scope = []
|
||||
if stack:
|
||||
stack_config = get_parsed_stack_config(stack)
|
||||
@ -33,11 +28,14 @@ def get_containers_in_scope(stack: str):
|
||||
warn_exit(f"stack {stack} does not define any containers")
|
||||
containers_in_scope = stack_config['containers']
|
||||
else:
|
||||
containers_in_scope = all_containers
|
||||
# See: https://stackoverflow.com/a/20885799/1701505
|
||||
from stack_orchestrator import data
|
||||
with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file:
|
||||
containers_in_scope = container_list_file.read().splitlines()
|
||||
|
||||
if opts.o.verbose:
|
||||
print(f'Containers: {containers_in_scope}')
|
||||
if stack:
|
||||
print(f"Stack: {stack}")
|
||||
|
||||
return containers_in_scope
|
||||
return containers_in_scope
|
||||
|
||||
@ -26,8 +26,15 @@ import click
|
||||
from pathlib import Path
|
||||
from stack_orchestrator import constants
|
||||
from stack_orchestrator.opts import opts
|
||||
from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config, global_options2, get_dev_root_path
|
||||
from stack_orchestrator.util import resolve_compose_file
|
||||
from stack_orchestrator.util import (
|
||||
get_stack_path,
|
||||
include_exclude_check,
|
||||
get_parsed_stack_config,
|
||||
global_options2,
|
||||
get_dev_root_path,
|
||||
stack_is_in_deployment,
|
||||
resolve_compose_file,
|
||||
)
|
||||
from stack_orchestrator.deploy.deployer import Deployer, DeployerException
|
||||
from stack_orchestrator.deploy.deployer_factory import getDeployer
|
||||
from stack_orchestrator.deploy.deploy_types import ClusterContext, DeployCommandContext
|
||||
@ -60,6 +67,7 @@ def command(ctx, include, exclude, env_file, cluster, deploy_to):
|
||||
if deploy_to is None:
|
||||
deploy_to = "compose"
|
||||
|
||||
stack = get_stack_path(stack)
|
||||
ctx.obj = create_deploy_context(global_options2(ctx), None, stack, include, exclude, cluster, env_file, deploy_to)
|
||||
# Subcommand is executed now, by the magic of click
|
||||
|
||||
@ -274,16 +282,12 @@ def _make_default_cluster_name(deployment, compose_dir, stack, include, exclude)
|
||||
|
||||
# stack has to be either PathLike pointing to a stack yml file, or a string with the name of a known stack
|
||||
def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file):
|
||||
|
||||
dev_root_path = get_dev_root_path(ctx)
|
||||
|
||||
# TODO: huge hack, fix this
|
||||
# If the caller passed a path for the stack file, then we know that we can get the compose files
|
||||
# from the same directory
|
||||
deployment = False
|
||||
if isinstance(stack, os.PathLike):
|
||||
compose_dir = stack.parent.joinpath("compose")
|
||||
deployment = True
|
||||
# TODO: hack, this should be encapsulated by the deployment context.
|
||||
deployment = stack_is_in_deployment(stack)
|
||||
if deployment:
|
||||
compose_dir = stack.joinpath("compose")
|
||||
else:
|
||||
# See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
|
||||
compose_dir = Path(__file__).absolute().parent.parent.joinpath("data", "compose")
|
||||
|
||||
@ -50,15 +50,15 @@ def command(ctx, dir):
|
||||
|
||||
def make_deploy_context(ctx) -> DeployCommandContext:
|
||||
context: DeploymentContext = ctx.obj
|
||||
stack_file_path = context.get_stack_file()
|
||||
env_file = context.get_env_file()
|
||||
cluster_name = context.get_cluster_id()
|
||||
if constants.deploy_to_key in context.spec.obj:
|
||||
deployment_type = context.spec.obj[constants.deploy_to_key]
|
||||
else:
|
||||
deployment_type = constants.compose_deploy_type
|
||||
return create_deploy_context(ctx.parent.parent.obj, context, stack_file_path, None, None, cluster_name, env_file,
|
||||
deployment_type)
|
||||
stack = context.deployment_dir
|
||||
return create_deploy_context(ctx.parent.parent.obj, context, stack, None, None,
|
||||
cluster_name, env_file, deployment_type)
|
||||
|
||||
|
||||
@command.command()
|
||||
@ -123,6 +123,7 @@ def push_images(ctx):
|
||||
@click.argument('extra_args', nargs=-1) # help: command: port <service1> <service2>
|
||||
@click.pass_context
|
||||
def port(ctx, extra_args):
|
||||
ctx.obj = make_deploy_context(ctx)
|
||||
port_operation(ctx, extra_args)
|
||||
|
||||
|
||||
|
||||
@ -24,7 +24,7 @@ from secrets import token_hex
|
||||
import sys
|
||||
from stack_orchestrator import constants
|
||||
from stack_orchestrator.opts import opts
|
||||
from stack_orchestrator.util import (get_stack_file_path, get_parsed_deployment_spec, get_parsed_stack_config,
|
||||
from stack_orchestrator.util import (get_stack_path, get_parsed_deployment_spec, get_parsed_stack_config,
|
||||
global_options, get_yaml, get_pod_list, get_pod_file_path, pod_has_scripts,
|
||||
get_pod_script_paths, get_plugin_code_paths, error_exit, env_var_map_from_file,
|
||||
resolve_config_dir)
|
||||
@ -238,6 +238,11 @@ def _find_extra_config_dirs(parsed_pod_file, pod):
|
||||
config_dir = host_path.split("/")[2]
|
||||
if config_dir != pod:
|
||||
config_dirs.add(config_dir)
|
||||
for env_file in service_info.get("env_file", []):
|
||||
if env_file.startswith("../config"):
|
||||
config_dir = env_file.split("/")[2]
|
||||
if config_dir != pod:
|
||||
config_dirs.add(config_dir)
|
||||
return config_dirs
|
||||
|
||||
|
||||
@ -454,7 +459,7 @@ def create_operation(deployment_command_context, spec_file, deployment_dir, netw
|
||||
_check_volume_definitions(parsed_spec)
|
||||
stack_name = parsed_spec["stack"]
|
||||
deployment_type = parsed_spec[constants.deploy_to_key]
|
||||
stack_file = get_stack_file_path(stack_name)
|
||||
stack_file = get_stack_path(stack_name).joinpath(constants.stack_file_name)
|
||||
parsed_stack = get_parsed_stack_config(stack_name)
|
||||
if opts.o.debug:
|
||||
print(f"parsed spec: {parsed_spec}")
|
||||
@ -467,7 +472,7 @@ def create_operation(deployment_command_context, spec_file, deployment_dir, netw
|
||||
os.mkdir(deployment_dir_path)
|
||||
# Copy spec file and the stack file into the deployment dir
|
||||
copyfile(spec_file, deployment_dir_path.joinpath(constants.spec_file_name))
|
||||
copyfile(stack_file, deployment_dir_path.joinpath(os.path.basename(stack_file)))
|
||||
copyfile(stack_file, deployment_dir_path.joinpath(constants.stack_file_name))
|
||||
_create_deployment_file(deployment_dir_path)
|
||||
# Copy any config varibles from the spec file into an env file suitable for compose
|
||||
_write_config_file(spec_file, deployment_dir_path.joinpath(constants.config_file_name))
|
||||
|
||||
@ -20,14 +20,12 @@ import os
|
||||
import sys
|
||||
from decouple import config
|
||||
import git
|
||||
from git.exc import GitCommandError
|
||||
from tqdm import tqdm
|
||||
import click
|
||||
import importlib.resources
|
||||
from pathlib import Path
|
||||
import yaml
|
||||
from stack_orchestrator.constants import stack_file_name
|
||||
from stack_orchestrator.opts import opts
|
||||
from stack_orchestrator.util import include_exclude_check, stack_is_external, error_exit, warn_exit
|
||||
from stack_orchestrator.util import get_parsed_stack_config, include_exclude_check, error_exit, warn_exit
|
||||
|
||||
|
||||
class GitProgress(git.RemoteProgress):
|
||||
@ -81,9 +79,13 @@ def _get_repo_current_branch_or_tag(full_filesystem_repo_path):
|
||||
except TypeError:
|
||||
# This means that the current ref is not a branch, so possibly a tag
|
||||
# Let's try to get the tag
|
||||
current_repo_branch_or_tag = git.Repo(full_filesystem_repo_path).git.describe("--tags", "--exact-match")
|
||||
# Note that git is assymetric -- the tag you told it to check out may not be the one
|
||||
# you get back here (if there are multiple tags associated with the same commit)
|
||||
try:
|
||||
current_repo_branch_or_tag = git.Repo(full_filesystem_repo_path).git.describe("--tags", "--exact-match")
|
||||
# Note that git is asymmetric -- the tag you told it to check out may not be the one
|
||||
# you get back here (if there are multiple tags associated with the same commit)
|
||||
except GitCommandError:
|
||||
# If there is no matching branch or tag checked out, just use the current SHA
|
||||
current_repo_branch_or_tag = git.Repo(full_filesystem_repo_path).commit("HEAD").hexsha
|
||||
return current_repo_branch_or_tag, is_branch
|
||||
|
||||
|
||||
@ -102,7 +104,7 @@ def process_repo(pull, check_only, git_ssh, dev_root_path, branches_array, fully
|
||||
full_filesystem_repo_path
|
||||
) if is_present else (None, None)
|
||||
if not opts.o.quiet:
|
||||
present_text = f"already exists active {'branch' if is_branch else 'tag'}: {current_repo_branch_or_tag}" if is_present \
|
||||
present_text = f"already exists active {'branch' if is_branch else 'ref'}: {current_repo_branch_or_tag}" if is_present \
|
||||
else 'Needs to be fetched'
|
||||
print(f"Checking: {full_filesystem_repo_path}: {present_text}")
|
||||
# Quick check that it's actually a repo
|
||||
@ -120,7 +122,7 @@ def process_repo(pull, check_only, git_ssh, dev_root_path, branches_array, fully
|
||||
origin = git_repo.remotes.origin
|
||||
origin.pull(progress=None if opts.o.quiet else GitProgress())
|
||||
else:
|
||||
print("skipping pull because this repo checked out a tag")
|
||||
print("skipping pull because this repo is not on a branch")
|
||||
else:
|
||||
print("(git pull skipped)")
|
||||
if not is_present:
|
||||
@ -222,20 +224,10 @@ def command(ctx, include, exclude, git_ssh, check_only, pull, branches):
|
||||
|
||||
repos_in_scope = []
|
||||
if stack:
|
||||
if stack_is_external(stack):
|
||||
stack_file_path = Path(stack).joinpath(stack_file_name)
|
||||
else:
|
||||
# In order to be compatible with Python 3.8 we need to use this hack to get the path:
|
||||
# See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
|
||||
stack_file_path = Path(__file__).absolute().parent.parent.joinpath("data", "stacks", stack, stack_file_name)
|
||||
if not stack_file_path.exists():
|
||||
error_exit(f"stack {stack} does not exist")
|
||||
with stack_file_path:
|
||||
stack_config = yaml.safe_load(open(stack_file_path, "r"))
|
||||
if "repos" not in stack_config or stack_config["repos"] is None:
|
||||
warn_exit(f"stack {stack} does not define any repositories")
|
||||
else:
|
||||
repos_in_scope = stack_config["repos"]
|
||||
stack_config = get_parsed_stack_config(stack)
|
||||
if "repos" not in stack_config or stack_config["repos"] is None:
|
||||
warn_exit(f"stack {stack} does not define any repositories")
|
||||
repos_in_scope = stack_config["repos"]
|
||||
else:
|
||||
repos_in_scope = all_repos
|
||||
|
||||
|
||||
@ -20,6 +20,7 @@ import ruamel.yaml
|
||||
from pathlib import Path
|
||||
from dotenv import dotenv_values
|
||||
from typing import Mapping, Set, List
|
||||
from stack_orchestrator.constants import stack_file_name, deployment_file_name
|
||||
|
||||
|
||||
def include_exclude_check(s, include, exclude):
|
||||
@ -33,11 +34,14 @@ def include_exclude_check(s, include, exclude):
|
||||
return s not in exclude_list
|
||||
|
||||
|
||||
def get_stack_file_path(stack):
|
||||
# In order to be compatible with Python 3.8 we need to use this hack to get the path:
|
||||
# See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
|
||||
stack_file_path = Path(__file__).absolute().parent.joinpath("data", "stacks", stack, "stack.yml")
|
||||
return stack_file_path
|
||||
def get_stack_path(stack):
|
||||
if stack_is_external(stack):
|
||||
stack_path = Path(stack)
|
||||
else:
|
||||
# In order to be compatible with Python 3.8 we need to use this hack to get the path:
|
||||
# See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
|
||||
stack_path = Path(__file__).absolute().parent.joinpath("data", "stacks", stack)
|
||||
return stack_path
|
||||
|
||||
|
||||
def get_dev_root_path(ctx):
|
||||
@ -52,21 +56,14 @@ def get_dev_root_path(ctx):
|
||||
|
||||
# Caller can pass either the name of a stack, or a path to a stack file
|
||||
def get_parsed_stack_config(stack):
|
||||
stack_file_path = stack if isinstance(stack, os.PathLike) else get_stack_file_path(stack)
|
||||
try:
|
||||
with stack_file_path:
|
||||
stack_config = get_yaml().load(open(stack_file_path, "r"))
|
||||
return stack_config
|
||||
except FileNotFoundError as error:
|
||||
# We try here to generate a useful diagnostic error
|
||||
# First check if the stack directory is present
|
||||
stack_directory = stack_file_path.parent
|
||||
if os.path.exists(stack_directory):
|
||||
print(f"Error: stack.yml file is missing from stack: {stack}")
|
||||
else:
|
||||
print(f"Error: stack: {stack} does not exist")
|
||||
print(f"Exiting, error: {error}")
|
||||
sys.exit(1)
|
||||
stack_file_path = get_stack_path(stack).joinpath(stack_file_name)
|
||||
if stack_file_path.exists():
|
||||
return get_yaml().load(open(stack_file_path, "r"))
|
||||
# We try here to generate a useful diagnostic error
|
||||
# First check if the stack directory is present
|
||||
if stack_file_path.parent.exists():
|
||||
error_exit(f"stack.yml file is missing from stack: {stack}")
|
||||
error_exit(f"stack {stack} does not exist")
|
||||
|
||||
|
||||
def get_pod_list(parsed_stack):
|
||||
@ -87,7 +84,7 @@ def get_plugin_code_paths(stack) -> List[Path]:
|
||||
result: Set[Path] = set()
|
||||
for pod in pods:
|
||||
if type(pod) is str:
|
||||
result.add(get_stack_file_path(stack).parent)
|
||||
result.add(get_stack_path(stack))
|
||||
else:
|
||||
pod_root_dir = os.path.join(get_dev_root_path(None), pod["repository"].split("/")[-1], pod["path"])
|
||||
result.add(Path(os.path.join(pod_root_dir, "stack")))
|
||||
@ -199,6 +196,10 @@ def stack_is_external(stack: str):
|
||||
return Path(stack).exists() if stack is not None else False
|
||||
|
||||
|
||||
def stack_is_in_deployment(stack: Path):
|
||||
return stack.joinpath(deployment_file_name).exists()
|
||||
|
||||
|
||||
def get_yaml():
|
||||
# See: https://stackoverflow.com/a/45701840/1701505
|
||||
yaml = ruamel.yaml.YAML()
|
||||
|
||||
@ -14,7 +14,7 @@
|
||||
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
|
||||
|
||||
import click
|
||||
import importlib.resources
|
||||
from importlib import resources, metadata
|
||||
|
||||
|
||||
@click.command()
|
||||
@ -24,8 +24,11 @@ def command(ctx):
|
||||
|
||||
# See: https://stackoverflow.com/a/20885799/1701505
|
||||
from stack_orchestrator import data
|
||||
with importlib.resources.open_text(data, "build_tag.txt") as version_file:
|
||||
# TODO: code better version that skips comment lines
|
||||
version_string = version_file.read().splitlines()[1]
|
||||
if resources.is_resource(data, "build_tag.txt"):
|
||||
with resources.open_text(data, "build_tag.txt") as version_file:
|
||||
# TODO: code better version that skips comment lines
|
||||
version_string = version_file.read().splitlines()[1]
|
||||
else:
|
||||
version_string = metadata.version("laconic-stack-orchestrator") + "-unknown"
|
||||
|
||||
print(f"Version: {version_string}")
|
||||
print(version_string)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user