From 774e3d0f5e2a3482846685f44e6bac8e54e730dc Mon Sep 17 00:00:00 2001 From: MattWellie Date: Thu, 22 Jan 2026 13:44:50 +1000 Subject: [PATCH 1/5] fix(pyproject.toml): remove rule-ignores --- cpg_flow_test/jobs/build_pyramid.py | 1 - cpg_flow_test/jobs/cumulative_calc.py | 1 - cpg_flow_test/jobs/filter_evens.py | 2 - cpg_flow_test/jobs/say_hi.py | 1 - cpg_flow_test/stages.py | 8 +- pyproject.toml | 124 -------------------------- 6 files changed, 4 insertions(+), 133 deletions(-) diff --git a/cpg_flow_test/jobs/build_pyramid.py b/cpg_flow_test/jobs/build_pyramid.py index 0b9fc48..f044210 100644 --- a/cpg_flow_test/jobs/build_pyramid.py +++ b/cpg_flow_test/jobs/build_pyramid.py @@ -2,7 +2,6 @@ from loguru import logger -from hailtop.batch import Batch from hailtop.batch.job import Job from cpg_flow.targets.sequencing_group import SequencingGroup diff --git a/cpg_flow_test/jobs/cumulative_calc.py b/cpg_flow_test/jobs/cumulative_calc.py index accef0f..9aae150 100644 --- a/cpg_flow_test/jobs/cumulative_calc.py +++ b/cpg_flow_test/jobs/cumulative_calc.py @@ -1,6 +1,5 @@ from loguru import logger -from hailtop.batch import Batch from hailtop.batch.job import Job from cpg_flow.targets.sequencing_group import SequencingGroup diff --git a/cpg_flow_test/jobs/filter_evens.py b/cpg_flow_test/jobs/filter_evens.py index a512b08..8ebc998 100644 --- a/cpg_flow_test/jobs/filter_evens.py +++ b/cpg_flow_test/jobs/filter_evens.py @@ -1,5 +1,3 @@ -from typing import Any - from loguru import logger from hailtop.batch.job import Job diff --git a/cpg_flow_test/jobs/say_hi.py b/cpg_flow_test/jobs/say_hi.py index e5f1324..f16b559 100644 --- a/cpg_flow_test/jobs/say_hi.py +++ b/cpg_flow_test/jobs/say_hi.py @@ -1,6 +1,5 @@ from loguru import logger -from hailtop.batch import Batch from hailtop.batch.job import Job from cpg_flow.targets.sequencing_group import SequencingGroup diff --git a/cpg_flow_test/stages.py b/cpg_flow_test/stages.py index 5ffadfa..483f170 100644 --- a/cpg_flow_test/stages.py +++ b/cpg_flow_test/stages.py @@ -15,7 +15,7 @@ --- ### Task: Prime Pyramid -Write a program that builds a "Prime Pyramid" based on a given input number \( N \). The pyramid is built in four steps: +Write a program that builds a "Prime Pyramid" based on a given input number `N`. The pyramid is built in four steps: #### Step 1: **Generate Prime Numbers** Write a function to generate the first `N` prime numbers. i.e. if `N=5``, the output would be `[2, 3, 5, 7, 11]`. @@ -60,7 +60,7 @@ def expected_outputs(self, sequencing_group: SequencingGroup) -> dict[str, Path] 'primes': sequencing_group.dataset.prefix() / WORKFLOW_FOLDER / f'{sequencing_group.id}_primes.txt', } - def queue_jobs(self, sequencing_group: SequencingGroup, inputs: StageInput) -> StageOutput: + def queue_jobs(self, sequencing_group: SequencingGroup, _inputs: StageInput) -> StageOutput: # Print out alignment input for this sequencing group logger.info('-----ALIGNMENT INPUT-----') logger.info(sequencing_group.alignment_input) @@ -75,13 +75,13 @@ def queue_jobs(self, sequencing_group: SequencingGroup, inputs: StageInput) -> S ) # Generate first N primes - primes_output_path = str(self.expected_outputs(sequencing_group).get('primes', '')) job_primes = first_n_primes.first_n_primes_job( sequencing_group, outputs['id_sum'], self.get_job_attrs(sequencing_group), outputs['primes'], ) + # set a dependency job_primes.depends_on(job_id_sum) @@ -122,7 +122,7 @@ def expected_outputs(self, sequencing_group: SequencingGroup) -> dict[str, Path] 'hello': sequencing_group.dataset.prefix() / WORKFLOW_FOLDER / f'{sequencing_group.id}_cumulative.txt', } - def queue_jobs(self, sequencing_group: SequencingGroup, inputs: StageInput) -> StageOutput | None: + def queue_jobs(self, sequencing_group: SequencingGroup, _inputs: StageInput) -> StageOutput | None: outputs = self.expected_outputs(sequencing_group) return self.make_outputs( sequencing_group, diff --git a/pyproject.toml b/pyproject.toml index faf94e4..1b56348 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,131 +81,7 @@ select = [ fixable = ["ALL"] ignore = [ - "ANN204", # Missing type annotation for special method `__init__` "Q000", # Single quotes found but double quotes preferred - "S101", # Use of assert detected - "SLF001", # Private member accessed: `_preemptible` - - "ARG001", # Unused function argument - "ARG002", # Unused method argument - - "PLR2004", # Magic value used - - "ANN001", - "ANN202", - "C408", - "TID252", - "RET504", - "ERA001", - "UP032", - "RUF100", - "ISC001", - "PIE804", - "F401", - "C901", - "W605", - "RET505", - "ANN003", - "RUF013", - "UP031", - "RUF010", - "B006", - "ANN002", - "B023", - "EXE001", - "G001", - "SIM108", - "RUF005", - "G002", - "PD901", - "N999", - "SIM118", - "SIM102", - "PLW2901", - "S603", - "ARG005", - "PGH003", - "B904", - "N802", - "ISC003", - "ANN205", - "S607", - "RUF015", - "E701", - "N818", - "PIE790", - "N803", - "A002", - "RUF012", - "W291", - "S113", - "S311", - "N806", - "PLR5501", - "F403", - "SIM115", - "B007", - "F841", - "C405", - "C419", - "SIM300", - "PD011", - "UP015", - "S602", - "Q002", - "ISC002", - "COM819", - "C416", - "DTZ005", - "G003", - "S608", - "PIE808", - "B008", - "S108", - "E402", - "S605", - "F821", - "RET507", - "RET503", - "UP030", - "UP026", - "PLR1714", - "C403", - "PLR1711", - "PIE810", - "DTZ011", - "S105", - "BLE001", - "C401", - "C400", - "PLR0402", - "SIM201", - "RET506", - "C417", - "PD010", - "PLW1510", - "A001", - "W292", - "PYI024", - "Q003", - "S301", - "RET501", - "PD003", - "SIM117", - "RUF002", - "SIM105", - "E713", - "S324", - "S310", - "Q001", - "UP020", - "S506", - "N805", - "E712", - "E401", - "SIM212", - "DTZ002", - "UP007", ] [tool.ruff.lint.isort] From 134853bfca10a115ab370120e292a573a48fd2a8 Mon Sep 17 00:00:00 2001 From: MattWellie Date: Thu, 22 Jan 2026 13:47:32 +1000 Subject: [PATCH 2/5] fix(pyproject.toml): higher cpg-utils --- pyproject.toml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1b56348..0049185 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,9 +7,8 @@ requires-python = ">=3.10,<3.12" dependencies = [ "analysis-runner>=3.2.2", "cpg-flow~=1.3", - "cpg-utils>=5.1.1", + "cpg-utils>=5.5.0", "hail~=0.2.137", - "loguru>=0.7.3", ] [project.optional-dependencies] @@ -78,15 +77,16 @@ select = [ "YTT", ] -fixable = ["ALL"] - -ignore = [ - "Q000", # Single quotes found but double quotes preferred -] - [tool.ruff.lint.isort] section-order = ["future", "standard-library", "third-party", "hail", "cpg", "first-party", "local-folder"] [tool.ruff.lint.isort.sections] cpg = ["metamist", "cpg_flow", "cpg_utils"] hail = ["hail", "hailtop"] + + +fixable = ["ALL"] + +ignore = [ + "Q000", # Single quotes found but double quotes preferred +] From f2c44493ffe74a62ec05d6e2f1b0b3356d1ac244 Mon Sep 17 00:00:00 2001 From: MattWellie Date: Wed, 28 Jan 2026 11:39:34 +1000 Subject: [PATCH 3/5] fix(pyproject.toml): reposition linting rule block --- pyproject.toml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0049185..8b9e975 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,16 +77,15 @@ select = [ "YTT", ] +fixable = ["ALL"] + +ignore = [ + "Q000", # Single quotes found but double quotes preferred +] + [tool.ruff.lint.isort] section-order = ["future", "standard-library", "third-party", "hail", "cpg", "first-party", "local-folder"] [tool.ruff.lint.isort.sections] cpg = ["metamist", "cpg_flow", "cpg_utils"] hail = ["hail", "hailtop"] - - -fixable = ["ALL"] - -ignore = [ - "Q000", # Single quotes found but double quotes preferred -] From ebac9e04c6bbf723a1f7a3b3d218ab90e61b366a Mon Sep 17 00:00:00 2001 From: MattWellie Date: Thu, 29 Jan 2026 10:01:15 +1000 Subject: [PATCH 4/5] fix(jobs): give each job a specific image --- cpg_flow_test/configs/default_config.toml | 3 +++ cpg_flow_test/jobs/build_pyramid.py | 4 +++- cpg_flow_test/jobs/cumulative_calc.py | 3 ++- cpg_flow_test/jobs/filter_evens.py | 5 ++++- cpg_flow_test/jobs/first_n_primes.py | 3 ++- cpg_flow_test/jobs/iterative_digit_sum.py | 3 ++- cpg_flow_test/jobs/say_hi.py | 3 ++- 7 files changed, 18 insertions(+), 6 deletions(-) diff --git a/cpg_flow_test/configs/default_config.toml b/cpg_flow_test/configs/default_config.toml index 6139525..68a350d 100644 --- a/cpg_flow_test/configs/default_config.toml +++ b/cpg_flow_test/configs/default_config.toml @@ -107,3 +107,6 @@ sequencing_type = 'genome' [resource_overrides] # Override the default resources for a stage. + +[images] +ubuntu = 'australia-southeast1-docker.pkg.dev/cpg-common/images/sv/ubuntu1804:latest' \ No newline at end of file diff --git a/cpg_flow_test/jobs/build_pyramid.py b/cpg_flow_test/jobs/build_pyramid.py index f044210..bf218b6 100644 --- a/cpg_flow_test/jobs/build_pyramid.py +++ b/cpg_flow_test/jobs/build_pyramid.py @@ -5,7 +5,7 @@ from hailtop.batch.job import Job from cpg_flow.targets.sequencing_group import SequencingGroup -from cpg_utils import Path, hail_batch +from cpg_utils import Path, config, hail_batch def build_pyramid_job( @@ -22,6 +22,7 @@ def build_pyramid_job( sg_output_files: list[Path] = [] for sg in sequencing_groups: job = b.new_bash_job(name=title + ': ' + sg.id, attributes=job_attrs | {'sequencing_group': sg.id}) + job.image(config.config_retrieve(['images', 'ubuntu'])) no_evens_input_file = b.read_input(input_files[sg.id]['no_evens']) @@ -53,6 +54,7 @@ def build_pyramid_job( # Merge the no evens lists for all sequencing groups into a single file job = b.new_bash_job(name=title, attributes=job_attrs | {'tool': 'cat'}) + job.image(config.config_retrieve(['images', 'ubuntu'])) job.depends_on(*sg_jobs) inputs = ' '.join([b.read_input(f) for f in sg_output_files]) job.command(f'cat {inputs} >> {job.pyramid}') diff --git a/cpg_flow_test/jobs/cumulative_calc.py b/cpg_flow_test/jobs/cumulative_calc.py index 9aae150..df1deb3 100644 --- a/cpg_flow_test/jobs/cumulative_calc.py +++ b/cpg_flow_test/jobs/cumulative_calc.py @@ -3,7 +3,7 @@ from hailtop.batch.job import Job from cpg_flow.targets.sequencing_group import SequencingGroup -from cpg_utils import Path, hail_batch +from cpg_utils import Path, config, hail_batch def cumulative_calc_job( @@ -14,6 +14,7 @@ def cumulative_calc_job( ) -> list[Job]: b = hail_batch.get_batch() job = b.new_job(name=f'Cumulative Calc: {sequencing_group.id}', attributes=job_attrs) + job.image(config.config_retrieve(['images', 'ubuntu'])) primes_path = b.read_input(input_file_path) cmd = f""" diff --git a/cpg_flow_test/jobs/filter_evens.py b/cpg_flow_test/jobs/filter_evens.py index 8ebc998..651b692 100644 --- a/cpg_flow_test/jobs/filter_evens.py +++ b/cpg_flow_test/jobs/filter_evens.py @@ -3,7 +3,7 @@ from hailtop.batch.job import Job from cpg_flow.targets.sequencing_group import SequencingGroup -from cpg_utils import Path, hail_batch +from cpg_utils import Path, config, hail_batch def filter_evens_job( @@ -20,6 +20,7 @@ def filter_evens_job( sg_output_files = [] for sg in sequencing_groups: job = b.new_bash_job(name=f'{title}: {sg.id}', attributes=job_attrs) + job.image(config.config_retrieve(['images', 'ubuntu'])) input_file_path = input_files[sg.id]['cumulative'] sg_output_files.append(sg_outputs[sg.id]) @@ -39,6 +40,8 @@ def filter_evens_job( # Merge the no evens lists for all sequencing groups into a single file job = b.new_bash_job(name=title, attributes=job_attrs) + job.image(config.config_retrieve(['images', 'ubuntu'])) + job.depends_on(*sg_jobs) inputs = ' '.join([b.read_input(f) for f in sg_output_files]) job.command(f'cat {inputs} >> {job.no_evens_file}') diff --git a/cpg_flow_test/jobs/first_n_primes.py b/cpg_flow_test/jobs/first_n_primes.py index 2516e7b..f0965c6 100644 --- a/cpg_flow_test/jobs/first_n_primes.py +++ b/cpg_flow_test/jobs/first_n_primes.py @@ -3,7 +3,7 @@ from hailtop.batch.job import Job from cpg_flow.targets.sequencing_group import SequencingGroup -from cpg_utils import Path, hail_batch +from cpg_utils import Path, config, hail_batch def first_n_primes_job( @@ -14,6 +14,7 @@ def first_n_primes_job( ) -> Job: b = hail_batch.get_batch() job = b.new_job(name=f'First N Primes: {sequencing_group.id}', attributes=job_attrs) + job.image(config.config_retrieve(['images', 'ubuntu'])) id_sum_path = b.read_input(input_file_path) job.command(f""" diff --git a/cpg_flow_test/jobs/iterative_digit_sum.py b/cpg_flow_test/jobs/iterative_digit_sum.py index 34be3dc..3929c74 100644 --- a/cpg_flow_test/jobs/iterative_digit_sum.py +++ b/cpg_flow_test/jobs/iterative_digit_sum.py @@ -3,7 +3,7 @@ from hailtop.batch.job import Job from cpg_flow.targets.sequencing_group import SequencingGroup -from cpg_utils import Path, hail_batch +from cpg_utils import Path, config, hail_batch def iterative_digit_sum_job( @@ -13,6 +13,7 @@ def iterative_digit_sum_job( ) -> Job: b = hail_batch.get_batch() job = b.new_job(name=f'Iterative Digit Sum: {sequencing_group.id}', attributes=job_attrs) + job.image(config.config_retrieve(['images', 'ubuntu'])) job.command(f"""\ #!/bin/bash diff --git a/cpg_flow_test/jobs/say_hi.py b/cpg_flow_test/jobs/say_hi.py index f16b559..1e170e1 100644 --- a/cpg_flow_test/jobs/say_hi.py +++ b/cpg_flow_test/jobs/say_hi.py @@ -3,7 +3,7 @@ from hailtop.batch.job import Job from cpg_flow.targets.sequencing_group import SequencingGroup -from cpg_utils import Path, hail_batch +from cpg_utils import Path, config, hail_batch def say_hi_job( @@ -13,6 +13,7 @@ def say_hi_job( ) -> Job: b = hail_batch.get_batch() job = b.new_job(name=f'Say Hi: {sequencing_group.id}', attributes=job_attrs) + job.image(config.config_retrieve(['images', 'ubuntu'])) job.command(f""" echo "This is a hello from sequencing_group {sequencing_group.id}" > {job.sayhi} From acd86c4309aa38a06b8c129e0743c3d3c71e5ab1 Mon Sep 17 00:00:00 2001 From: MattWellie Date: Thu, 29 Jan 2026 10:01:48 +1000 Subject: [PATCH 5/5] fix(jobs): lint --- cpg_flow_test/configs/default_config.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpg_flow_test/configs/default_config.toml b/cpg_flow_test/configs/default_config.toml index 68a350d..04f4629 100644 --- a/cpg_flow_test/configs/default_config.toml +++ b/cpg_flow_test/configs/default_config.toml @@ -109,4 +109,4 @@ sequencing_type = 'genome' # Override the default resources for a stage. [images] -ubuntu = 'australia-southeast1-docker.pkg.dev/cpg-common/images/sv/ubuntu1804:latest' \ No newline at end of file +ubuntu = 'australia-southeast1-docker.pkg.dev/cpg-common/images/sv/ubuntu1804:latest'