From 6d620ba9c2969d363acfd197b8e8ccfe23a9ea52 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Thu, 25 May 2023 09:49:26 +0800 Subject: [PATCH] git branch in stack and on command line (#409) * Support @branch notation in stack.yml * Refactor and support branches directive --- app/data/stacks/test/stack.yml | 2 +- app/setup_repositories.py | 190 +++++++++++++++++++++------------ 2 files changed, 121 insertions(+), 71 deletions(-) diff --git a/app/data/stacks/test/stack.yml b/app/data/stacks/test/stack.yml index b564ccaa..a62da193 100644 --- a/app/data/stacks/test/stack.yml +++ b/app/data/stacks/test/stack.yml @@ -3,7 +3,7 @@ name: test description: "A test stack" repos: - github.com/cerc-io/laconicd - - git.vdb.to/cerc-io/test-project + - git.vdb.to/cerc-io/test-project@test-branch containers: - cerc/test-container pods: diff --git a/app/setup_repositories.py b/app/setup_repositories.py index 770af6d9..f6ee1e02 100644 --- a/app/setup_repositories.py +++ b/app/setup_repositories.py @@ -52,15 +52,112 @@ def is_git_repo(path): # ) +def branch_strip(s): + return s.split('@')[0] + + +def host_and_path_for_repo(fully_qualified_repo): + repo_branch_split = fully_qualified_repo.split("@") + repo_branch = repo_branch_split[-1] if len(repo_branch_split) > 1 else None + repo_host_split = repo_branch_split[0].split("/") + # Legacy unqualified repo means github + if len(repo_host_split) == 2: + return "github.com", "/".join(repo_host_split), repo_branch + else: + if len(repo_host_split) == 3: + # First part is the host + return repo_host_split[0], "/".join(repo_host_split[1:]), repo_branch + + +# TODO: fix the messy arg list here +def process_repo(verbose, quiet, dry_run, pull, check_only, git_ssh, dev_root_path, branches_array, fully_qualified_repo): + repo_host, repo_path, repo_branch = host_and_path_for_repo(fully_qualified_repo) + git_ssh_prefix = f"git@{repo_host}:" + git_http_prefix = f"https://{repo_host}/" + full_github_repo_path = f"{git_ssh_prefix if git_ssh else git_http_prefix}{repo_path}" + repoName = repo_path.split("/")[-1] + full_filesystem_repo_path = os.path.join(dev_root_path, repoName) + is_present = os.path.isdir(full_filesystem_repo_path) + current_repo_branch = git.Repo(full_filesystem_repo_path).active_branch.name if is_present else None + if not quiet: + present_text = f"already exists active branch: {current_repo_branch}" if is_present \ + else 'Needs to be fetched' + print(f"Checking: {full_filesystem_repo_path}: {present_text}") + # Quick check that it's actually a repo + if is_present: + if not is_git_repo(full_filesystem_repo_path): + print(f"Error: {full_filesystem_repo_path} does not contain a valid git repository") + sys.exit(1) + else: + if pull: + if verbose: + print(f"Running git pull for {full_filesystem_repo_path}") + if not check_only: + git_repo = git.Repo(full_filesystem_repo_path) + origin = git_repo.remotes.origin + origin.pull(progress=None if quiet else GitProgress()) + else: + print("(git pull skipped)") + if not is_present: + # Clone + if verbose: + print(f'Running git clone for {full_github_repo_path} into {full_filesystem_repo_path}') + if not dry_run: + git.Repo.clone_from(full_github_repo_path, + full_filesystem_repo_path, + progress=None if quiet else GitProgress()) + else: + print("(git clone skipped)") + # Checkout the requested branch, if one was specified + branch_to_checkout = None + if branches_array: + # Find the current repo in the branches list + print("Checking") + for repo_branch in branches_array: + repo_branch_tuple = repo_branch.split(" ") + if repo_branch_tuple[0] == branch_strip(fully_qualified_repo): + # checkout specified branch + branch_to_checkout = repo_branch_tuple[1] + else: + branch_to_checkout = repo_branch + + print(f"branch_to_checkout: {branch_to_checkout}") + if branch_to_checkout: + if current_repo_branch and (current_repo_branch != branch_to_checkout): + if not quiet: + print(f"switching to branch {branch_to_checkout} in repo {repo_path}") + git_repo = git.Repo(full_filesystem_repo_path) + git_repo.git.checkout(branch_to_checkout) + else: + if verbose: + print(f"repo {repo_path} is already switched to branch {branch_to_checkout}") + + +def parse_branches(branches_string): + if branches_string: + result_array = [] + branches_directives = branches_string.split(",") + for branch_directive in branches_directives: + split_directive = branch_directive.split("@") + if len(split_directive) != 2: + print(f"Error: branch specified is not valid: {branch_directive}") + sys.exit(1) + result_array.append(f"{split_directive[0]} {split_directive[1]}") + return result_array + else: + return None + + @click.command() @click.option("--include", help="only clone these repositories") @click.option("--exclude", help="don\'t clone these repositories") @click.option('--git-ssh', is_flag=True, default=False) @click.option('--check-only', is_flag=True, default=False) @click.option('--pull', is_flag=True, default=False) +@click.option("--branches", help="override branches for repositories") @click.option('--branches-file', help="checkout branches specified in this file") @click.pass_context -def command(ctx, include, exclude, git_ssh, check_only, pull, branches_file): +def command(ctx, include, exclude, git_ssh, check_only, pull, branches, branches_file): '''git clone the set of repositories required to build the complete system from source''' quiet = ctx.obj.quiet @@ -68,16 +165,29 @@ def command(ctx, include, exclude, git_ssh, check_only, pull, branches_file): dry_run = ctx.obj.dry_run stack = ctx.obj.stack - branches = [] + branches_array = [] # TODO: branches file needs to be re-worked in the context of stacks if branches_file: - if verbose: - print(f"loading branches from: {branches_file}") - with open(branches_file) as branches_file_open: - branches = branches_file_open.read().splitlines() - if verbose: - print(f"Branches are: {branches}") + if branches: + print("Error: can't specify both --branches and --branches-file") + sys.exit(1) + else: + if verbose: + print(f"loading branches from: {branches_file}") + with open(branches_file) as branches_file_open: + branches_array = branches_file_open.read().splitlines() + + print(f"branches: {branches}") + if branches: + if branches_file: + print("Error: can't specify both --branches and --branches-file") + sys.exit(1) + else: + branches_array = parse_branches(branches) + + if branches_array and verbose: + print(f"Branches are: {branches_array}") local_stack = ctx.obj.local_stack @@ -119,75 +229,15 @@ def command(ctx, include, exclude, git_ssh, check_only, pull, branches_file): repos = [] for repo in repos_in_scope: - if include_exclude_check(repo, include, exclude): + if include_exclude_check(branch_strip(repo), include, exclude): repos.append(repo) else: if verbose: print(f"Excluding: {repo}") - def host_and_path_for_repo(fully_qualified_repo): - repo_split = fully_qualified_repo.split("/") - # Legacy unqualified repo means github - if len(repo_split) == 2: - return "github.com", "/".join(repo_split) - else: - if len(repo_split) == 3: - # First part is the host - return repo_split[0], "/".join(repo_split[1:]) - - def process_repo(fully_qualified_repo): - repo_host, repo_path = host_and_path_for_repo(fully_qualified_repo) - git_ssh_prefix = f"git@{repo_host}:" - git_http_prefix = f"https://{repo_host}/" - full_github_repo_path = f"{git_ssh_prefix if git_ssh else git_http_prefix}{repo_path}" - repoName = repo_path.split("/")[-1] - full_filesystem_repo_path = os.path.join(dev_root_path, repoName) - is_present = os.path.isdir(full_filesystem_repo_path) - if not quiet: - present_text = f"already exists active branch: {git.Repo(full_filesystem_repo_path).active_branch}" if is_present \ - else 'Needs to be fetched' - print(f"Checking: {full_filesystem_repo_path}: {present_text}") - # Quick check that it's actually a repo - if is_present: - if not is_git_repo(full_filesystem_repo_path): - print(f"Error: {full_filesystem_repo_path} does not contain a valid git repository") - sys.exit(1) - else: - if pull: - if verbose: - print(f"Running git pull for {full_filesystem_repo_path}") - if not check_only: - git_repo = git.Repo(full_filesystem_repo_path) - origin = git_repo.remotes.origin - origin.pull(progress=None if quiet else GitProgress()) - else: - print("(git pull skipped)") - if not is_present: - # Clone - if verbose: - print(f'Running git clone for {full_github_repo_path} into {full_filesystem_repo_path}') - if not dry_run: - git.Repo.clone_from(full_github_repo_path, - full_filesystem_repo_path, - progress=None if quiet else GitProgress()) - else: - print("(git clone skipped)") - # Checkout the requested branch, if one was specified - if branches: - # Find the current repo in the branches list - for repo_branch in branches: - repo_branch_tuple = repo_branch.split(" ") - if repo_branch_tuple[0] == repo: - # checkout specified branch - branch_to_checkout = repo_branch_tuple[1] - if verbose: - print(f"checking out branch {branch_to_checkout} in repo {repo}") - git_repo = git.Repo(full_filesystem_repo_path) - git_repo.git.checkout(branch_to_checkout) - for repo in repos: try: - process_repo(repo) + process_repo(verbose, quiet, dry_run, pull, check_only, git_ssh, dev_root_path, branches_array, repo) except git.exc.GitCommandError as error: print(f"\n******* git command returned error exit status:\n{error}") sys.exit(1)