diff --git a/github_scripts/get_git_sources.py b/github_scripts/get_git_sources.py index 1cde083..f467ed8 100644 --- a/github_scripts/get_git_sources.py +++ b/github_scripts/get_git_sources.py @@ -87,6 +87,48 @@ def datetime_str() -> str: return datetime.now().strftime("%Y-%m-%d %H:%M:%S") +def clone_and_merge( + dependency: str, opts: list | dict, loc: Path, use_mirrors: bool, mirror_loc: Path +) -> None: + """ + Wrapper script for calling get_source and merge_source for a single dependency + + dependency: name of the dependency + opts: dict or list of dicts for a dependency in the dependencies file + loc: path to location to clone to + use_mirrors: bool, use local git mirrors if true + mirror_loc: path to local git mirrors + """ + + if not isinstance(opts, list): + opts = [opts] + + for i, values in enumerate(opts): + if values["ref"] is None: + values["ref"] = "" + + # Clone the first provided source + if i == 0: + get_source( + values["source"], + values["ref"], + loc, + dependency, + use_mirrors, + mirror_loc, + ) + # For all other sources, attempt to merge into the first + else: + merge_source( + values["source"], + values["ref"], + loc, + dependency, + use_mirrors, + mirror_loc, + ) + + def get_source( source: str, ref: str, @@ -104,7 +146,10 @@ def get_source( logger.info( f"[{datetime_str()}] Cloning {repo} from {mirror_loc} at ref {ref}" ) - mirror_loc = Path(mirror_loc) / "MetOffice" / repo + mirror_repo = repo + if "jules-internal" in source: + mirror_repo = "jules-internal" + mirror_loc = Path(mirror_loc) / "MetOffice" / mirror_repo clone_repo_mirror(source, ref, mirror_loc, dest) else: logger.info(f"[{datetime_str()}] Cloning {repo} from {source} at ref {ref}") @@ -304,8 +349,12 @@ def sync_repo(repo_source: str, repo_ref: str, loc: Path) -> None: loc.mkdir(parents=True) exclude_dirs = [] - host, path = repo_source.split(":", 1) - result = run_command(f"ssh {host} git -C {path} status --ignored -s") + try: + host, path = repo_source.split(":", 1) + result = run_command(f"ssh {host} git -C {path} status --ignored -s") + except ValueError: + # In case the path does not contain `host:` - see if it can be accessed locally + result = run_command(f"git -C {repo_source} status --ignored -s") for ignore_file in result.stdout.split("\n"): ignore_file = ignore_file.strip() if not ignore_file.startswith("!!"): diff --git a/github_scripts/merge_sources.py b/github_scripts/merge_sources.py index fafde37..87d7464 100755 --- a/github_scripts/merge_sources.py +++ b/github_scripts/merge_sources.py @@ -12,7 +12,7 @@ import os import yaml from pathlib import Path -from get_git_sources import get_source, merge_source, set_https, validate_dependencies +from get_git_sources import clone_and_merge, set_https, validate_dependencies import logging @@ -80,37 +80,9 @@ def main(): if args.tokens: dependencies = set_https(dependencies) - for dependency, opts in dependencies.items(): + for dependency, sources in dependencies.items(): dest = args.path / dependency - - if not isinstance(opts, list): - opts = [opts] - - for i, values in enumerate(opts): - if values["ref"] is None: - values["ref"] = "" - - # Clone the first provided source - if i == 0: - get_source( - values["source"], - values["ref"], - dest, - dependency, - args.mirrors, - args.mirror_loc, - ) - continue - - # For all other sources, attempt to merge into the first - merge_source( - values["source"], - values["ref"], - dest, - dependency, - args.mirrors, - args.mirror_loc, - ) + clone_and_merge(dependency, sources, dest, args.mirrors, args.mirror_loc) if __name__ == "__main__": diff --git a/github_scripts/rose_stem_extract_source.py b/github_scripts/rose_stem_extract_source.py index b3c5a86..d7bbf55 100755 --- a/github_scripts/rose_stem_extract_source.py +++ b/github_scripts/rose_stem_extract_source.py @@ -14,7 +14,7 @@ import os from pathlib import Path from ast import literal_eval -from get_git_sources import get_source, merge_source, set_https, validate_dependencies +from get_git_sources import clone_and_merge, set_https, validate_dependencies import logging @@ -43,36 +43,9 @@ def main() -> None: use_mirrors = os.environ.get("USE_MIRRORS", "false").lower() == "true" mirror_loc = Path(os.getenv("GIT_MIRROR_LOC", "")) - for dependency, opts in dependencies.items(): + for dependency, sources in dependencies.items(): loc = clone_loc / dependency - - if not isinstance(opts, list): - opts = [opts] - - for i, values in enumerate(opts): - if values["ref"] is None: - values["ref"] = "" - - # Clone the first provided source - if i == 0: - get_source( - values["source"], - values["ref"], - loc, - dependency, - use_mirrors, - mirror_loc, - ) - continue - # For all other sources, attempt to merge into the first - merge_source( - values["source"], - values["ref"], - loc, - dependency, - use_mirrors, - mirror_loc, - ) + clone_and_merge(dependency, sources, loc, use_mirrors, mirror_loc) if __name__ == "__main__":