Initial version of pip packaging

2022-08-23 11:32:55 -06:00
parent a14b803ff7
commit cebdb45559
10 changed files with 804 additions and 0 deletions
@@ -0,0 +1,61 @@
+# Builds or pulls containers for the system components
+
+# env vars:
+# VULCANIZE_REPO_BASE_DIR defaults to ~/vulcanize
+
+# TODO: display the available list of containers; allow re-build of either all or specific containers
+
+import os
+import sys
+import argparse
+from decouple import config
+import subprocess
+
+parser = argparse.ArgumentParser(
+    description="build the set of containers required for a complete stack",
+    epilog="Config provided either in .env or settings.ini or env vars: VULCANIZE_REPO_BASE_DIR (defaults to ~/vulcanize)"
+    )
+parser.add_argument("--verbose", action="store_true", help="increase output verbosity")
+parser.add_argument("--quiet", action="store_true", help="don\'t print informational output")
+parser.add_argument("--check-only", action="store_true", help="looks at what\'s already there and checks if it looks good")
+parser.add_argument("--dry-run", action="store_true", help="don\'t do anything, just print the commands that would be executed")
+
+args = parser.parse_args()
+
+verbose = args.verbose
+quiet = args.quiet
+
+dev_root_path = os.path.expanduser(config("VULCANIZE_REPO_BASE_DIR", default="~/vulcanize"))
+
+if not args.quiet:
+    print(f'Dev Root is: {dev_root_path}')
+
+if not os.path.isdir(dev_root_path):
+    print(f'Dev root directory doesn\'t exist, creating')
+
+with open("container-image-list.txt") as container_list_file:
+    containers = container_list_file.read().splitlines()
+
+if verbose:
+    print(f'Containers: {containers}')
+
+def process_container(container):
+    if not quiet:
+        print(f"Building: {container}")
+    build_script_filename = os.path.join("container-build",container.replace("/","-"),"build.sh")
+    if verbose:
+        print(f"Script: {build_script_filename}")
+    if not os.path.exists(build_script_filename):
+        print(f"Error, script: {build_script_filename} doesn't exist")
+        sys.exit(1)
+    if not args.dry_run:
+        # We need to export VULCANIZE_REPO_BASE_DIR
+        build_result = subprocess.run(build_script_filename, shell=True, env={'VULCANIZE_REPO_BASE_DIR':dev_root_path})
+        # TODO: check result in build_result.returncode
+        print(f"Result is: {build_result}")
+
+for container in containers:
+    process_container(container)
+
+
+
@@ -0,0 +1,69 @@
+# Deploys the system components using docker-compose
+
+import os
+import argparse
+from decouple import config
+from python_on_whales import DockerClient
+
+def include_exclude_check(s, args):
+    if args.include == None and args.exclude == None:
+        return True
+    if args.include != None:
+        include_list = args.include.split(",")
+        return s in include_list
+    if args.exclude != None:
+        exclude_list = args.exclude.split(",")
+        return s not in exclude_list
+
+parser = argparse.ArgumentParser(
+    description="deploy the complete stack"
+    )
+parser.add_argument("command", type=str, nargs=1, choices=['up', 'down', 'ps'], help="command: up|down|ps")
+parser.add_argument("--verbose", action="store_true", help="increase output verbosity")
+parser.add_argument("--quiet", action="store_true", help="don\'t print informational output")
+parser.add_argument("--check-only", action="store_true", help="looks at what\'s already there and checks if it looks good")
+parser.add_argument("--dry-run", action="store_true", help="don\'t do anything, just print the commands that would be executed")
+group = parser.add_mutually_exclusive_group()
+group.add_argument("--exclude", type=str, help="don\'t start these components")
+group.add_argument("--include", type=str, help="only start these components")
+
+args = parser.parse_args()
+
+verbose = args.verbose
+quiet = args.quiet
+
+print(args)
+
+with open("cluster-list.txt") as cluster_list_file:
+    clusters = cluster_list_file.read().splitlines()
+
+if verbose:
+    print(f'Cluster components: {clusters}')
+
+# Construct a docker compose command suitable for our purpose
+
+compose_files = []
+for cluster in clusters:
+    if include_exclude_check(cluster, args):
+        compose_file_name = os.path.join("compose", f"docker-compose-{cluster}.yml")
+        compose_files.append(compose_file_name)
+    else:
+        if not quiet:
+            print(f"Excluding: {cluster}")
+
+if verbose:
+    print(f"files: {compose_files}")
+
+# See: https://gabrieldemarmiesse.github.io/python-on-whales/sub-commands/compose/
+docker = DockerClient(compose_files=compose_files)
+
+command = args.command[0]
+if not args.dry_run:
+    if command == "up":
+        if verbose:
+            print("Running compose up")
+        docker.compose.up(detach=True)
+    elif command == "down":
+        if verbose:
+            print("Running compose down")
+        docker.compose.down()
@@ -0,0 +1,94 @@
+# env vars:
+# VULCANIZE_REPO_BASE_DIR defaults to ~/vulcanize
+
+import os
+import sys
+import argparse
+from decouple import config
+import git
+from tqdm import tqdm
+
+class GitProgress(git.RemoteProgress):
+    def __init__(self):
+        super().__init__()
+        self.pbar = tqdm(unit = 'B', ascii = True, unit_scale = True)
+
+    def update(self, op_code, cur_count, max_count=None, message=''):
+        self.pbar.total = max_count
+        self.pbar.n = cur_count
+        self.pbar.refresh()
+
+def is_git_repo(path):
+    try:
+        _ = git.Repo(path).git_dir
+        return True
+    except git.exc.InvalidGitRepositoryError:
+        return False
+
+parser = argparse.ArgumentParser(
+    description="git clone the set of repositories required to build the complete system from source",
+    epilog="Config provided either in .env or settings.ini or env vars: VULCANIZE_REPO_BASE_DIR (defaults to ~/vulcanize)"
+    )
+parser.add_argument("--verbose", action="store_true", help="increase output verbosity")
+parser.add_argument("--quiet", action="store_true", help="don\'t print informational output")
+parser.add_argument("--check-only", action="store_true", help="looks at what\'s already there and checks if it looks good")
+parser.add_argument("--dry-run", action="store_true", help="don\'t do anything, just print the commands that would be executed")
+parser.add_argument("--pull", action="store_true", help="pull from remote in already existing repositories")
+
+args = parser.parse_args()
+
+verbose = args.verbose
+quiet = args.quiet
+
+dev_root_path = os.path.expanduser(config("DEV_ROOT", default="~/vulcanize"))
+
+if not args.quiet:
+    print(f'Dev Root is: {dev_root_path}')
+
+if not os.path.isdir(dev_root_path):
+    if not quiet:
+        print(f'Dev root directory doesn\'t exist, creating')
+    os.makedirs(dev_root_path)
+
+with open("repository-list.txt") as repository_list_file:
+    repos = repository_list_file.read().splitlines()
+
+if verbose:
+    print (f'Repos: {repos}')
+
+def process_repo(repo):
+    full_github_repo_path = f'git@github.com:{repo}'
+    repoName = repo.split("/")[-1]
+    full_filesystem_repo_path = os.path.join(dev_root_path, repoName)
+    is_present = os.path.isdir(full_filesystem_repo_path)
+    if not quiet:
+        present_text = f'already exists active branch: {git.Repo(full_filesystem_repo_path).active_branch}' if is_present else 'Needs to be fetched'
+        print(f'Checking: {full_filesystem_repo_path}: {present_text}')
+    # Quick check that it's actually a repo
+    if is_present:
+        if not is_git_repo(full_filesystem_repo_path):
+            print(f'Error: {full_filesystem_repo_path} does not contain a valid git repository')
+            sys.exit(1)
+        else:
+            if args.pull:
+                if verbose:
+                  print(f'Running git pull for {full_filesystem_repo_path}')
+                if not args.check_only:
+                  repo = git.Repo(full_filesystem_repo_path)
+                  origin = repo.remotes.origin
+                  origin.pull(progress = None if quiet else GitProgress())
+                else:
+                    print("(git pull skipped)")
+    if not is_present:
+        # Clone
+        if verbose:
+            print(f'Running git clone for {full_github_repo_path} into {full_filesystem_repo_path}')
+        if not args.check_only:
+            git.Repo.clone_from(full_github_repo_path, full_filesystem_repo_path, 
+            progress = None if quiet else GitProgress())
+        else:
+            print("(git clone skipped)")
+
+
+for repo in repos:
+    process_repo(repo)