From 5a69bf471b9e489f0d569f924a04db5534599674 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 15 Nov 2022 11:34:13 -0700 Subject: [PATCH 1/2] Move data files into subdirectory --- app/build_containers.py | 2 +- app/deploy_system.py | 2 +- app/setup_repositories.py | 2 +- container-image-list.txt => data/container-image-list.txt | 0 pod-list.txt => data/pod-list.txt | 0 repository-list.txt => data/repository-list.txt | 0 6 files changed, 3 insertions(+), 3 deletions(-) rename container-image-list.txt => data/container-image-list.txt (100%) rename pod-list.txt => data/pod-list.txt (100%) rename repository-list.txt => data/repository-list.txt (100%) diff --git a/app/build_containers.py b/app/build_containers.py index 817dc0e3..7a660b8a 100644 --- a/app/build_containers.py +++ b/app/build_containers.py @@ -55,7 +55,7 @@ def command(ctx, include, exclude): if not os.path.isdir(dev_root_path): print('Dev root directory doesn\'t exist, creating') - with open("container-image-list.txt") as container_list_file: + with open("data/container-image-list.txt") as container_list_file: containers = container_list_file.read().splitlines() if verbose: diff --git a/app/deploy_system.py b/app/deploy_system.py index ad8614ff..56f00604 100644 --- a/app/deploy_system.py +++ b/app/deploy_system.py @@ -48,7 +48,7 @@ def command(ctx, include, exclude, cluster, command, services): if verbose: print(f"Using cluster name: {cluster}") - with open("pod-list.txt") as pod_list_file: + with open("data/pod-list.txt") as pod_list_file: pods = pod_list_file.read().splitlines() if verbose: diff --git a/app/setup_repositories.py b/app/setup_repositories.py index 5110b36d..ff8e8f84 100644 --- a/app/setup_repositories.py +++ b/app/setup_repositories.py @@ -89,7 +89,7 @@ def command(ctx, include, exclude, check_only, pull, branches_file): print('Dev root directory doesn\'t exist, creating') os.makedirs(dev_root_path) - with open("repository-list.txt") as repository_list_file: + with open("data/repository-list.txt") as repository_list_file: all_repos = repository_list_file.read().splitlines() if verbose: diff --git a/container-image-list.txt b/data/container-image-list.txt similarity index 100% rename from container-image-list.txt rename to data/container-image-list.txt diff --git a/pod-list.txt b/data/pod-list.txt similarity index 100% rename from pod-list.txt rename to data/pod-list.txt diff --git a/repository-list.txt b/data/repository-list.txt similarity index 100% rename from repository-list.txt rename to data/repository-list.txt From 1a5844af7aee2c8cd80caa60a7c464f1f07527d9 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 15 Nov 2022 22:02:44 -0700 Subject: [PATCH 2/2] Fix shiv package --- app/build_containers.py | 5 +++-- {data => app/data}/container-image-list.txt | 0 {data => app/data}/pod-list.txt | 0 {data => app/data}/repository-list.txt | 0 app/deploy_system.py | 7 ++++--- app/setup_repositories.py | 21 +++++++++++---------- setup.py | 2 ++ 7 files changed, 20 insertions(+), 15 deletions(-) rename {data => app/data}/container-image-list.txt (100%) rename {data => app/data}/pod-list.txt (100%) rename {data => app/data}/repository-list.txt (100%) diff --git a/app/build_containers.py b/app/build_containers.py index 7a660b8a..3621e5da 100644 --- a/app/build_containers.py +++ b/app/build_containers.py @@ -25,6 +25,7 @@ import sys from decouple import config import subprocess import click +import pkg_resources from .util import include_exclude_check # TODO: find a place for this @@ -55,8 +56,8 @@ def command(ctx, include, exclude): if not os.path.isdir(dev_root_path): print('Dev root directory doesn\'t exist, creating') - with open("data/container-image-list.txt") as container_list_file: - containers = container_list_file.read().splitlines() + with pkg_resources.resource_stream(__name__, "data/container-image-list.txt") as container_list_file: + containers = container_list_file.read().decode().splitlines() if verbose: print(f'Containers: {containers}') diff --git a/data/container-image-list.txt b/app/data/container-image-list.txt similarity index 100% rename from data/container-image-list.txt rename to app/data/container-image-list.txt diff --git a/data/pod-list.txt b/app/data/pod-list.txt similarity index 100% rename from data/pod-list.txt rename to app/data/pod-list.txt diff --git a/data/repository-list.txt b/app/data/repository-list.txt similarity index 100% rename from data/repository-list.txt rename to app/data/repository-list.txt diff --git a/app/deploy_system.py b/app/deploy_system.py index 56f00604..9cef3f65 100644 --- a/app/deploy_system.py +++ b/app/deploy_system.py @@ -20,6 +20,7 @@ import os import sys from python_on_whales import DockerClient import click +import pkg_resources from .util import include_exclude_check @@ -48,11 +49,11 @@ def command(ctx, include, exclude, cluster, command, services): if verbose: print(f"Using cluster name: {cluster}") - with open("data/pod-list.txt") as pod_list_file: - pods = pod_list_file.read().splitlines() + with pkg_resources.resource_stream(__name__, "data/pod-list.txt") as pod_list_file: + pods = pod_list_file.read().decode().splitlines() if verbose: - print(f'Pods: {pods}') + print(f"Pods: {pods}") # Construct a docker compose command suitable for our purpose diff --git a/app/setup_repositories.py b/app/setup_repositories.py index ff8e8f84..e7693147 100644 --- a/app/setup_repositories.py +++ b/app/setup_repositories.py @@ -22,6 +22,7 @@ from decouple import config import git from tqdm import tqdm import click +import pkg_resources from .util import include_exclude_check @@ -77,23 +78,23 @@ def command(ctx, include, exclude, check_only, pull, branches_file): if local_stack: dev_root_path = os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")] - print(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}') + print(f"Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}") else: dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc")) if not quiet: - print(f'Dev Root is: {dev_root_path}') + print(f"Dev Root is: {dev_root_path}") if not os.path.isdir(dev_root_path): if not quiet: print('Dev root directory doesn\'t exist, creating') os.makedirs(dev_root_path) - with open("data/repository-list.txt") as repository_list_file: - all_repos = repository_list_file.read().splitlines() + with pkg_resources.resource_stream(__name__, "data/repository-list.txt") as repository_list_file: + all_repos = repository_list_file.read().decode().splitlines() if verbose: - print(f'Repos: {all_repos}') + print(f"Repos: {all_repos}") repos = [] for repo in all_repos: @@ -104,23 +105,23 @@ def command(ctx, include, exclude, check_only, pull, branches_file): print(f"Excluding: {repo}") def process_repo(repo): - full_github_repo_path = f'git@github.com:{repo}' + full_github_repo_path = f"git@github.com:{repo}" repoName = repo.split("/")[-1] full_filesystem_repo_path = os.path.join(dev_root_path, repoName) is_present = os.path.isdir(full_filesystem_repo_path) if not quiet: - present_text = f'already exists active branch: {git.Repo(full_filesystem_repo_path).active_branch}' if is_present \ + present_text = f"already exists active branch: {git.Repo(full_filesystem_repo_path).active_branch}" if is_present \ else 'Needs to be fetched' - print(f'Checking: {full_filesystem_repo_path}: {present_text}') + print(f"Checking: {full_filesystem_repo_path}: {present_text}") # Quick check that it's actually a repo if is_present: if not is_git_repo(full_filesystem_repo_path): - print(f'Error: {full_filesystem_repo_path} does not contain a valid git repository') + print(f"Error: {full_filesystem_repo_path} does not contain a valid git repository") sys.exit(1) else: if pull: if verbose: - print(f'Running git pull for {full_filesystem_repo_path}') + print(f"Running git pull for {full_filesystem_repo_path}") if not check_only: git_repo = git.Repo(full_filesystem_repo_path) origin = git_repo.remotes.origin diff --git a/setup.py b/setup.py index 88e10155..17d2ae35 100644 --- a/setup.py +++ b/setup.py @@ -18,6 +18,8 @@ setup( packages=find_packages(), install_requires=[requirements], python_requires='>=3.7', + include_package_data=True, + package_data={'': ['data/*.txt']}, classifiers=[ "Programming Language :: Python :: 3.8", "Operating System :: OS Independent",