Skip to content

further tidies to git extraction #177

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
t00sa merged 5 commits into from
Feb 3, 2026
+58 −64
Merged

further tidies to git extraction #177

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
1b19623
further tidies to git extraction
james-bruten-mo Feb 3, 2026
68b7f27
copy over rsync changes
james-bruten-mo Feb 3, 2026
8bb9575
delete unused line
james-bruten-mo Feb 3, 2026
531ffdc
bug
james-bruten-mo Feb 3, 2026
5979471
fix for jules internal
james-bruten-mo Feb 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 52 additions & 3 deletions github_scripts/get_git_sources.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,48 @@ def datetime_str() -> str:
return datetime.now().strftime("%Y-%m-%d %H:%M:%S")


def clone_and_merge(
dependency: str, opts: list | dict, loc: Path, use_mirrors: bool, mirror_loc: Path
) -> None:
"""
Wrapper script for calling get_source and merge_source for a single dependency

dependency: name of the dependency
opts: dict or list of dicts for a dependency in the dependencies file
loc: path to location to clone to
use_mirrors: bool, use local git mirrors if true
mirror_loc: path to local git mirrors
"""

if not isinstance(opts, list):
opts = [opts]

for i, values in enumerate(opts):
if values["ref"] is None:
values["ref"] = ""

# Clone the first provided source
if i == 0:
get_source(
values["source"],
values["ref"],
loc,
dependency,
use_mirrors,
mirror_loc,
)
# For all other sources, attempt to merge into the first
else:
merge_source(
values["source"],
values["ref"],
loc,
dependency,
use_mirrors,
mirror_loc,
)


def get_source(
source: str,
ref: str,
Expand All @@ -104,7 +146,10 @@ def get_source(
logger.info(
f"[{datetime_str()}] Cloning {repo} from {mirror_loc} at ref {ref}"
)
mirror_loc = Path(mirror_loc) / "MetOffice" / repo
mirror_repo = repo
if "jules-internal" in source:
mirror_repo = "jules-internal"
mirror_loc = Path(mirror_loc) / "MetOffice" / mirror_repo
clone_repo_mirror(source, ref, mirror_loc, dest)
else:
logger.info(f"[{datetime_str()}] Cloning {repo} from {source} at ref {ref}")
Expand Down Expand Up @@ -304,8 +349,12 @@ def sync_repo(repo_source: str, repo_ref: str, loc: Path) -> None:
loc.mkdir(parents=True)

exclude_dirs = []
host, path = repo_source.split(":", 1)
result = run_command(f"ssh {host} git -C {path} status --ignored -s")
try:
host, path = repo_source.split(":", 1)
result = run_command(f"ssh {host} git -C {path} status --ignored -s")
except ValueError:
# In case the path does not contain `host:` - see if it can be accessed locally
result = run_command(f"git -C {repo_source} status --ignored -s")
for ignore_file in result.stdout.split("\n"):
ignore_file = ignore_file.strip()
if not ignore_file.startswith("!!"):
Expand Down
34 changes: 3 additions & 31 deletions github_scripts/merge_sources.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
import os
import yaml
from pathlib import Path
from get_git_sources import get_source, merge_source, set_https, validate_dependencies
from get_git_sources import clone_and_merge, set_https, validate_dependencies
import logging


Expand Down Expand Up @@ -80,37 +80,9 @@ def main():
if args.tokens:
dependencies = set_https(dependencies)

for dependency, opts in dependencies.items():
for dependency, sources in dependencies.items():
dest = args.path / dependency

if not isinstance(opts, list):
opts = [opts]

for i, values in enumerate(opts):
if values["ref"] is None:
values["ref"] = ""

# Clone the first provided source
if i == 0:
get_source(
values["source"],
values["ref"],
dest,
dependency,
args.mirrors,
args.mirror_loc,
)
continue

# For all other sources, attempt to merge into the first
merge_source(
values["source"],
values["ref"],
dest,
dependency,
args.mirrors,
args.mirror_loc,
)
clone_and_merge(dependency, sources, dest, args.mirrors, args.mirror_loc)


if __name__ == "__main__":
Expand Down
33 changes: 3 additions & 30 deletions github_scripts/rose_stem_extract_source.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
import os
from pathlib import Path
from ast import literal_eval
from get_git_sources import get_source, merge_source, set_https, validate_dependencies
from get_git_sources import clone_and_merge, set_https, validate_dependencies
import logging


Expand Down Expand Up @@ -43,36 +43,9 @@ def main() -> None:
use_mirrors = os.environ.get("USE_MIRRORS", "false").lower() == "true"
mirror_loc = Path(os.getenv("GIT_MIRROR_LOC", ""))

for dependency, opts in dependencies.items():
for dependency, sources in dependencies.items():
loc = clone_loc / dependency

if not isinstance(opts, list):
opts = [opts]

for i, values in enumerate(opts):
if values["ref"] is None:
values["ref"] = ""

# Clone the first provided source
if i == 0:
get_source(
values["source"],
values["ref"],
loc,
dependency,
use_mirrors,
mirror_loc,
)
continue
# For all other sources, attempt to merge into the first
merge_source(
values["source"],
values["ref"],
loc,
dependency,
use_mirrors,
mirror_loc,
)
clone_and_merge(dependency, sources, loc, use_mirrors, mirror_loc)


if __name__ == "__main__":
Expand Down