cerc-io/stack-orchestrator
17
1
Fork 3
Code Issues 113 Pull Requests 18 Actions Packages Projects Releases 480 Wiki Activity

Beginnings of click support

Browse Source
This commit is contained in:
David Boreham 2022-08-23 21:27:42 -06:00
parent 8e75a1cedb
commit fbf9d17a64
4 changed files with 124 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
app/build_containers.py
View File

@ -25,6 +25,7 @@ import sys
import argparse import argparse
from decouple import config from decouple import config
import subprocess import subprocess
import click
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="build the set of containers required for a complete stack", description="build the set of containers required for a complete stack",
@ -37,40 +38,42 @@ parser.add_argument("--dry-run", action="store_true", help="don\'t do anything,
args = parser.parse_args() args = parser.parse_args()
verbose = args.verbose @click.command()
quiet = args.quiet def command():
verbose = args.verbose
quiet = args.quiet
dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc")) dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc"))
if not args.quiet: if not args.quiet:
print(f'Dev Root is: {dev_root_path}') print(f'Dev Root is: {dev_root_path}')
if not os.path.isdir(dev_root_path): if not os.path.isdir(dev_root_path):
print(f'Dev root directory doesn\'t exist, creating') print(f'Dev root directory doesn\'t exist, creating')
with open("container-image-list.txt") as container_list_file: with open("container-image-list.txt") as container_list_file:
containers = container_list_file.read().splitlines() containers = container_list_file.read().splitlines()
if verbose:
print(f'Containers: {containers}')
def process_container(container):
if not quiet:
print(f"Building: {container}")
build_script_filename = os.path.join("container-build",container.replace("/","-"),"build.sh")
if verbose: if verbose:
print(f"Script: {build_script_filename}") print(f'Containers: {containers}')
if not os.path.exists(build_script_filename):
print(f"Error, script: {build_script_filename} doesn't exist")
sys.exit(1)
if not args.dry_run:
# We need to export CERC_REPO_BASE_DIR
build_result = subprocess.run(build_script_filename, shell=True, env={'CERC_REPO_BASE_DIR':dev_root_path})
# TODO: check result in build_result.returncode
print(f"Result is: {build_result}")
for container in containers: def process_container(container):
process_container(container) if not quiet:
print(f"Building: {container}")
build_script_filename = os.path.join("container-build",container.replace("/","-"),"build.sh")
if verbose:
print(f"Script: {build_script_filename}")
if not os.path.exists(build_script_filename):
print(f"Error, script: {build_script_filename} doesn't exist")
sys.exit(1)
if not args.dry_run:
# We need to export CERC_REPO_BASE_DIR
build_result = subprocess.run(build_script_filename, shell=True, env={'CERC_REPO_BASE_DIR':dev_root_path})
# TODO: check result in build_result.returncode
print(f"Result is: {build_result}")
for container in containers:
process_container(container)

58
app/deploy_system.py
View File

@ -19,6 +19,7 @@ import os
import argparse import argparse
from decouple import config from decouple import config
from python_on_whales import DockerClient from python_on_whales import DockerClient
import click
def include_exclude_check(s, args): def include_exclude_check(s, args):
if args.include == None and args.exclude == None: if args.include == None and args.exclude == None:
@ -47,38 +48,39 @@ args = parser.parse_args()
verbose = args.verbose verbose = args.verbose
quiet = args.quiet quiet = args.quiet
print(args) @click.command()
def command():
with open("cluster-list.txt") as cluster_list_file: with open("cluster-list.txt") as cluster_list_file:
clusters = cluster_list_file.read().splitlines() clusters = cluster_list_file.read().splitlines()
if verbose: if verbose:
print(f'Cluster components: {clusters}') print(f'Cluster components: {clusters}')
# Construct a docker compose command suitable for our purpose # Construct a docker compose command suitable for our purpose
compose_files = [] compose_files = []
for cluster in clusters: for cluster in clusters:
if include_exclude_check(cluster, args): if include_exclude_check(cluster, args):
compose_file_name = os.path.join("compose", f"docker-compose-{cluster}.yml") compose_file_name = os.path.join("compose", f"docker-compose-{cluster}.yml")
compose_files.append(compose_file_name) compose_files.append(compose_file_name)
else: else:
if not quiet: if not quiet:
print(f"Excluding: {cluster}") print(f"Excluding: {cluster}")
if verbose: if verbose:
print(f"files: {compose_files}") print(f"files: {compose_files}")
# See: https://gabrieldemarmiesse.github.io/python-on-whales/sub-commands/compose/ # See: https://gabrieldemarmiesse.github.io/python-on-whales/sub-commands/compose/
docker = DockerClient(compose_files=compose_files) docker = DockerClient(compose_files=compose_files)
command = args.command[0] command = args.command[0]
if not args.dry_run: if not args.dry_run:
if command == "up": if command == "up":
if verbose: if verbose:
print("Running compose up") print("Running compose up")
docker.compose.up(detach=True) docker.compose.up(detach=True)
elif command == "down": elif command == "down":
if verbose: if verbose:
print("Running compose down") print("Running compose down")
docker.compose.down() docker.compose.down()

98
app/setup_repositories.py
View File

@ -22,6 +22,7 @@ import argparse
from decouple import config from decouple import config
import git import git
from tqdm import tqdm from tqdm import tqdm
import click
class GitProgress(git.RemoteProgress): class GitProgress(git.RemoteProgress):
def __init__(self): def __init__(self):
@ -52,58 +53,61 @@ parser.add_argument("--pull", action="store_true", help="pull from remote in alr
args = parser.parse_args() args = parser.parse_args()
verbose = args.verbose @click.command()
quiet = args.quiet def command():
dev_root_path = os.path.expanduser(config("DEV_ROOT", default="~/cerc")) verbose = args.verbose
quiet = args.quiet
if not args.quiet: dev_root_path = os.path.expanduser(config("DEV_ROOT", default="~/cerc"))
print(f'Dev Root is: {dev_root_path}')
if not os.path.isdir(dev_root_path): if not args.quiet:
if not quiet: print(f'Dev Root is: {dev_root_path}')
print(f'Dev root directory doesn\'t exist, creating')
os.makedirs(dev_root_path)
with open("repository-list.txt") as repository_list_file: if not os.path.isdir(dev_root_path):
repos = repository_list_file.read().splitlines() if not quiet:
print(f'Dev root directory doesn\'t exist, creating')
os.makedirs(dev_root_path)
if verbose: with open("repository-list.txt") as repository_list_file:
print (f'Repos: {repos}') repos = repository_list_file.read().splitlines()
def process_repo(repo): if verbose:
full_github_repo_path = f'git@github.com:{repo}' print (f'Repos: {repos}')
repoName = repo.split("/")[-1]
full_filesystem_repo_path = os.path.join(dev_root_path, repoName) def process_repo(repo):
is_present = os.path.isdir(full_filesystem_repo_path) full_github_repo_path = f'git@github.com:{repo}'
if not quiet: repoName = repo.split("/")[-1]
present_text = f'already exists active branch: {git.Repo(full_filesystem_repo_path).active_branch}' if is_present else 'Needs to be fetched' full_filesystem_repo_path = os.path.join(dev_root_path, repoName)
print(f'Checking: {full_filesystem_repo_path}: {present_text}') is_present = os.path.isdir(full_filesystem_repo_path)
# Quick check that it's actually a repo if not quiet:
if is_present: present_text = f'already exists active branch: {git.Repo(full_filesystem_repo_path).active_branch}' if is_present else 'Needs to be fetched'
if not is_git_repo(full_filesystem_repo_path): print(f'Checking: {full_filesystem_repo_path}: {present_text}')
print(f'Error: {full_filesystem_repo_path} does not contain a valid git repository') # Quick check that it's actually a repo
sys.exit(1) if is_present:
else: if not is_git_repo(full_filesystem_repo_path):
if args.pull: print(f'Error: {full_filesystem_repo_path} does not contain a valid git repository')
if verbose: sys.exit(1)
print(f'Running git pull for {full_filesystem_repo_path}') else:
if not args.check_only: if args.pull:
repo = git.Repo(full_filesystem_repo_path) if verbose:
origin = repo.remotes.origin print(f'Running git pull for {full_filesystem_repo_path}')
origin.pull(progress = None if quiet else GitProgress()) if not args.check_only:
else: repo = git.Repo(full_filesystem_repo_path)
print("(git pull skipped)") origin = repo.remotes.origin
if not is_present: origin.pull(progress = None if quiet else GitProgress())
# Clone else:
if verbose: print("(git pull skipped)")
print(f'Running git clone for {full_github_repo_path} into {full_filesystem_repo_path}') if not is_present:
if not args.check_only: # Clone
git.Repo.clone_from(full_github_repo_path, full_filesystem_repo_path, if verbose:
progress = None if quiet else GitProgress()) print(f'Running git clone for {full_github_repo_path} into {full_filesystem_repo_path}')
else: if not args.check_only:
print("(git clone skipped)") git.Repo.clone_from(full_github_repo_path, full_filesystem_repo_path,
progress = None if quiet else GitProgress())
else:
print("(git clone skipped)")
for repo in repos: for repo in repos:
process_repo(repo) process_repo(repo)

13
orchestrator.py
View File

@ -13,3 +13,16 @@
# You should have received a copy of the GNU Affero General Public License # You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http:#www.gnu.org/licenses/>. # along with this program. If not, see <http:#www.gnu.org/licenses/>.
import click
from .app import setup_repositories
from .app import build_containers
from .app import deploy_system
@click.group()
def main_command_group():
pass
main_command_group.add_command(setup_repositories.command)
main_command_group.add_command(build_containers.command)
main_command_group.add_command(deploy_system.command)