cerc-io/stack-orchestrator
17
1
Fork 1
Code Issues 110 Pull Requests 12 Actions Packages Projects Releases 473 Wiki Activity

Some lint goodness

Browse Source
This commit is contained in:
David Boreham 2022-10-03 19:37:18 -06:00
parent ce09fce83e
commit 1050e25607
7 changed files with 52 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
app/build_containers.py
View File

@ -22,7 +22,6 @@
import os import os
import sys import sys
import argparse
from decouple import config from decouple import config
import subprocess import subprocess
import click import click
@ -31,6 +30,7 @@ from .util import include_exclude_check
# TODO: find a place for this # TODO: find a place for this
# epilog="Config provided either in .env or settings.ini or env vars: CERC_REPO_BASE_DIR (defaults to ~/cerc)" # epilog="Config provided either in .env or settings.ini or env vars: CERC_REPO_BASE_DIR (defaults to ~/cerc)"
@click.command() @click.command()
@click.option('--include', help="only build these containers") @click.option('--include', help="only build these containers")
@click.option('--exclude', help="don\'t build these containers") @click.option('--exclude', help="don\'t build these containers")
@ -44,7 +44,7 @@ def command(ctx, include, exclude):
local_stack = ctx.obj.local_stack local_stack = ctx.obj.local_stack
if local_stack: if local_stack:
dev_root_path = default=os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")] dev_root_path = default = os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")]
print(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}') print(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}')
else: else:
dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc")) dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc"))
@ -53,7 +53,7 @@ def command(ctx, include, exclude):
print(f'Dev Root is: {dev_root_path}') print(f'Dev Root is: {dev_root_path}')
if not os.path.isdir(dev_root_path): if not os.path.isdir(dev_root_path):
print(f'Dev root directory doesn\'t exist, creating') print('Dev root directory doesn\'t exist, creating')
with open("container-image-list.txt") as container_list_file: with open("container-image-list.txt") as container_list_file:
containers = container_list_file.read().splitlines() containers = container_list_file.read().splitlines()
@ -64,7 +64,7 @@ def command(ctx, include, exclude):
def process_container(container): def process_container(container):
if not quiet: if not quiet:
print(f"Building: {container}") print(f"Building: {container}")
build_script_filename = os.path.join("container-build",container.replace("/","-"),"build.sh") build_script_filename = os.path.join("container-build", container.replace("/", "-"), "build.sh")
if verbose: if verbose:
print(f"Script: {build_script_filename}") print(f"Script: {build_script_filename}")
if not os.path.exists(build_script_filename): if not os.path.exists(build_script_filename):
@ -72,7 +72,7 @@ def command(ctx, include, exclude):
sys.exit(1) sys.exit(1)
if not dry_run: if not dry_run:
# We need to export CERC_REPO_BASE_DIR # We need to export CERC_REPO_BASE_DIR
build_result = subprocess.run(build_script_filename, shell=True, env={'CERC_REPO_BASE_DIR':dev_root_path}) build_result = subprocess.run(build_script_filename, shell=True, env={'CERC_REPO_BASE_DIR': dev_root_path})
# TODO: check result in build_result.returncode # TODO: check result in build_result.returncode
print(f"Result is: {build_result}") print(f"Result is: {build_result}")
else: else:

10
app/deploy_system.py
View File

@ -16,18 +16,18 @@
# Deploys the system components using docker-compose # Deploys the system components using docker-compose
import os import os
import argparse
from decouple import config
from python_on_whales import DockerClient from python_on_whales import DockerClient
import click import click
from .util import include_exclude_check from .util import include_exclude_check
@click.command() @click.command()
@click.option('--include', help="only start these components") @click.option("--include", help="only start these components")
@click.option('--exclude', help="don\'t start these components") @click.option("--exclude", help="don\'t start these components")
@click.option("--score", help="read list of components from file")
@click.argument('command') # help: command: up|down|ps @click.argument('command') # help: command: up|down|ps
@click.pass_context @click.pass_context
def command(ctx, include, exclude, command): def command(ctx, include, exclude, score, command):
'''deploy a stack''' '''deploy a stack'''
# TODO: implement option exclusion and command value constraint lost with the move from argparse to click # TODO: implement option exclusion and command value constraint lost with the move from argparse to click

26
app/setup_repositories.py
View File

@ -18,22 +18,23 @@
import os import os
import sys import sys
import argparse
from decouple import config from decouple import config
import git import git
from tqdm import tqdm from tqdm import tqdm
import click import click
class GitProgress(git.RemoteProgress): class GitProgress(git.RemoteProgress):
def __init__(self): def __init__(self):
super().__init__() super().__init__()
self.pbar = tqdm(unit = 'B', ascii = True, unit_scale = True) self.pbar = tqdm(unit='B', ascii=True, unit_scale=True)
def update(self, op_code, cur_count, max_count=None, message=''): def update(self, op_code, cur_count, max_count=None, message=''):
self.pbar.total = max_count self.pbar.total = max_count
self.pbar.n = cur_count self.pbar.n = cur_count
self.pbar.refresh() self.pbar.refresh()
def is_git_repo(path): def is_git_repo(path):
try: try:
_ = git.Repo(path).git_dir _ = git.Repo(path).git_dir
@ -42,10 +43,11 @@ def is_git_repo(path):
return False return False
# TODO: find a place for this in the context of click # TODO: find a place for this in the context of click
#parser = argparse.ArgumentParser( # parser = argparse.ArgumentParser(
# epilog="Config provided either in .env or settings.ini or env vars: CERC_REPO_BASE_DIR (defaults to ~/cerc)" # epilog="Config provided either in .env or settings.ini or env vars: CERC_REPO_BASE_DIR (defaults to ~/cerc)"
# ) # )
@click.command() @click.command()
@click.option('--check-only', is_flag=True, default=False) @click.option('--check-only', is_flag=True, default=False)
@click.option('--pull', is_flag=True, default=False) @click.option('--pull', is_flag=True, default=False)
@ -71,7 +73,7 @@ def command(ctx, check_only, pull, branches_file):
local_stack = ctx.obj.local_stack local_stack = ctx.obj.local_stack
if local_stack: if local_stack:
dev_root_path = default=os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")] dev_root_path = default = os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")]
print(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}') print(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}')
else: else:
dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc")) dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc"))
@ -81,14 +83,14 @@ def command(ctx, check_only, pull, branches_file):
if not os.path.isdir(dev_root_path): if not os.path.isdir(dev_root_path):
if not quiet: if not quiet:
print(f'Dev root directory doesn\'t exist, creating') print('Dev root directory doesn\'t exist, creating')
os.makedirs(dev_root_path) os.makedirs(dev_root_path)
with open("repository-list.txt") as repository_list_file: with open("repository-list.txt") as repository_list_file:
repos = repository_list_file.read().splitlines() repos = repository_list_file.read().splitlines()
if verbose: if verbose:
print (f'Repos: {repos}') print(f'Repos: {repos}')
def process_repo(repo): def process_repo(repo):
full_github_repo_path = f'git@github.com:{repo}' full_github_repo_path = f'git@github.com:{repo}'
@ -96,7 +98,8 @@ def command(ctx, check_only, pull, branches_file):
full_filesystem_repo_path = os.path.join(dev_root_path, repoName) full_filesystem_repo_path = os.path.join(dev_root_path, repoName)
is_present = os.path.isdir(full_filesystem_repo_path) is_present = os.path.isdir(full_filesystem_repo_path)
if not quiet: if not quiet:
present_text = f'already exists active branch: {git.Repo(full_filesystem_repo_path).active_branch}' if is_present else 'Needs to be fetched' present_text = f'already exists active branch: {git.Repo(full_filesystem_repo_path).active_branch}' if is_present \
else 'Needs to be fetched'
print(f'Checking: {full_filesystem_repo_path}: {present_text}') print(f'Checking: {full_filesystem_repo_path}: {present_text}')
# Quick check that it's actually a repo # Quick check that it's actually a repo
if is_present: if is_present:
@ -110,7 +113,7 @@ def command(ctx, check_only, pull, branches_file):
if not check_only: if not check_only:
git_repo = git.Repo(full_filesystem_repo_path) git_repo = git.Repo(full_filesystem_repo_path)
origin = git_repo.remotes.origin origin = git_repo.remotes.origin
origin.pull(progress = None if quiet else GitProgress()) origin.pull(progress=None if quiet else GitProgress())
else: else:
print("(git pull skipped)") print("(git pull skipped)")
if not is_present: if not is_present:
@ -118,8 +121,9 @@ def command(ctx, check_only, pull, branches_file):
if verbose: if verbose:
print(f'Running git clone for {full_github_repo_path} into {full_filesystem_repo_path}') print(f'Running git clone for {full_github_repo_path} into {full_filesystem_repo_path}')
if not dry_run: if not dry_run:
git.Repo.clone_from(full_github_repo_path, full_filesystem_repo_path, git.Repo.clone_from(full_github_repo_path,
progress = None if quiet else GitProgress()) full_filesystem_repo_path,
progress=None if quiet else GitProgress())
else: else:
print("(git clone skipped)") print("(git clone skipped)")
# Checkout the requested branch, if one was specified # Checkout the requested branch, if one was specified
@ -135,7 +139,5 @@ def command(ctx, check_only, pull, branches_file):
git_repo = git.Repo(full_filesystem_repo_path) git_repo = git.Repo(full_filesystem_repo_path)
git_repo.git.checkout(branch_to_checkout) git_repo.git.checkout(branch_to_checkout)
for repo in repos: for repo in repos:
process_repo(repo) process_repo(repo)

6
app/util.py
View File

@ -14,11 +14,11 @@
# along with this program. If not, see <http:#www.gnu.org/licenses/>. # along with this program. If not, see <http:#www.gnu.org/licenses/>.
def include_exclude_check(s, include, exclude): def include_exclude_check(s, include, exclude):
if include == None and exclude == None: if include is None and exclude is None:
return True return True
if include != None: if include is not None:
include_list = include.split(",") include_list = include.split(",")
return s in include_list return s in include_list
if exclude != None: if exclude is not None:
exclude_list = exclude.split(",") exclude_list = exclude.split(",")
return s not in exclude_list return s not in exclude_list

9
cli.py
View File

@ -21,6 +21,7 @@ from app import deploy_system
CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
class Options(object): class Options(object):
def __init__(self, quiet, verbose, dry_run, local_stack): def __init__(self, quiet, verbose, dry_run, local_stack):
self.quiet = quiet self.quiet = quiet
@ -33,13 +34,13 @@ class Options(object):
@click.option('--verbose', is_flag=True, default=False) @click.option('--verbose', is_flag=True, default=False)
@click.option('--dry-run', is_flag=True, default=False) @click.option('--dry-run', is_flag=True, default=False)
@click.option('--local_stack', is_flag=True, default=False) @click.option('--local_stack', is_flag=True, default=False)
# See: https://click.palletsprojects.com/en/8.1.x/complex/#building-a-git-clone # See: https://click.palletsprojects.com/en/8.1.x/complex/#building-a-git-clone
@click.pass_context @click.pass_context
def cli(ctx, quiet, verbose, dry_run, local_stack): def cli(ctx, quiet, verbose, dry_run, local_stack):
"""Laconic Stack Orchestrator""" """Laconic Stack Orchestrator"""
ctx.obj = Options(quiet, verbose, dry_run, local_stack) ctx.obj = Options(quiet, verbose, dry_run, local_stack)
cli.add_command(setup_repositories.command,"setup-repositories")
cli.add_command(build_containers.command,"build-containers") cli.add_command(setup_repositories.command, "setup-repositories")
cli.add_command(deploy_system.command,"deploy-system") cli.add_command(build_containers.command, "build-containers")
cli.add_command(deploy_system.command, "deploy-system")

26
setup.py
View File

@ -5,24 +5,24 @@ with open("README.md", "r", encoding="utf-8") as fh:
with open("requirements.txt", "r", encoding="utf-8") as fh: with open("requirements.txt", "r", encoding="utf-8") as fh:
requirements = fh.read() requirements = fh.read()
setup( setup(
name = 'laconic-stack-orchestrator', name='laconic-stack-orchestrator',
version = '0.0.5', version='0.0.5',
author = 'Cerc', author='Cerc',
author_email = 'info@cerc.io', author_email='info@cerc.io',
license = 'GNU Affero General Public License', license='GNU Affero General Public License',
description = 'Orchestrates deployment of the Laconic stack', description='Orchestrates deployment of the Laconic stack',
long_description = long_description, long_description=long_description,
long_description_content_type = "text/markdown", long_description_content_type="text/markdown",
url = 'https://github.com/cerc-io/stack-orchestrator', url='https://github.com/cerc-io/stack-orchestrator',
py_modules = ['cli', 'app'], py_modules=['cli', 'app'],
packages = find_packages(), packages=find_packages(),
install_requires = [requirements], install_requires=[requirements],
python_requires='>=3.7', python_requires='>=3.7',
classifiers=[ classifiers=[
"Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.8",
"Operating System :: OS Independent", "Operating System :: OS Independent",
], ],
entry_points = { entry_points={
'console_scripts': ['laconic-so=cli:cli'], 'console_scripts': ['laconic-so=cli:cli'],
} }
) )

6
tox.ini Normal file
View File

@ -0,0 +1,6 @@
[flake8]
extend-ignore = E203
exclude = .git,__pycache__,docs/source/conf.py,old,build,dist,venv
max-complexity = 10
max-line-length = 132