From 2e9f6bd1dd67ac30159cca045f0462dc2ff9d326 Mon Sep 17 00:00:00 2001 From: Anna Scholtz Date: Fri, 6 Feb 2026 09:53:23 -0800 Subject: [PATCH] chore(bigquery-etl): [DENG-8804] Use data-artifacts bigquery-etl docker image --- dags/bqetl_artifact_deployment.py | 2 +- dags/bqetl_backfill.py | 2 +- dags/bqetl_backfill_complete.py | 2 +- dags/bqetl_backfill_initiate.py | 2 +- dags/bqetl_dryrun.py | 2 +- dags/microsoft_store.py | 2 +- dags/shredder.py | 2 +- dags/shredder_backfill.py | 2 +- dags/socorro_import.py | 2 +- utils/gcp.py | 2 +- utils/glam_subdags/generate_query.py | 4 ++-- 11 files changed, 12 insertions(+), 12 deletions(-) diff --git a/dags/bqetl_artifact_deployment.py b/dags/bqetl_artifact_deployment.py index 9d735f638..25679bb46 100644 --- a/dags/bqetl_artifact_deployment.py +++ b/dags/bqetl_artifact_deployment.py @@ -114,7 +114,7 @@ def should_run_deployment(dag_id: str, generate_sql: bool) -> bool: tags=tags, params=params, ) as dag: - docker_image = "gcr.io/moz-fx-data-airflow-prod-88e0/bigquery-etl:latest" + docker_image = "us-docker.pkg.dev/moz-fx-data-artifacts-prod/bigquery-etl/bigquery-etl:latest" skip_if_queued_runs_exist = ShortCircuitOperator( task_id="skip_if_queued_runs_exist", diff --git a/dags/bqetl_backfill.py b/dags/bqetl_backfill.py index effe0c462..7387b8075 100644 --- a/dags/bqetl_backfill.py +++ b/dags/bqetl_backfill.py @@ -191,7 +191,7 @@ def generate_backfill_command(**context): reattach_on_restart=True, task_id="bqetl_backfill", arguments=generate_backfill_command(), - image="gcr.io/moz-fx-data-airflow-prod-88e0/bigquery-etl:latest", + image="us-docker.pkg.dev/moz-fx-data-artifacts-prod/bigquery-etl/bigquery-etl:latest", gcp_conn_id="google_cloud_airflow_gke", ) diff --git a/dags/bqetl_backfill_complete.py b/dags/bqetl_backfill_complete.py index 22c11966e..2bf29a478 100644 --- a/dags/bqetl_backfill_complete.py +++ b/dags/bqetl_backfill_complete.py @@ -18,7 +18,7 @@ "slack_conn_id": SLACK_CONNECTION_ID, "channel": AUTOMATION_SLACK_CHANNEL, } -DOCKER_IMAGE = "gcr.io/moz-fx-data-airflow-prod-88e0/bigquery-etl:latest" +DOCKER_IMAGE = "us-docker.pkg.dev/moz-fx-data-artifacts-prod/bigquery-etl/bigquery-etl:latest" tags = [Tag.ImpactTier.tier_3] diff --git a/dags/bqetl_backfill_initiate.py b/dags/bqetl_backfill_initiate.py index 3eaec5e41..42de0cea9 100644 --- a/dags/bqetl_backfill_initiate.py +++ b/dags/bqetl_backfill_initiate.py @@ -19,7 +19,7 @@ "slack_conn_id": SLACK_CONNECTION_ID, "channel": AUTOMATION_SLACK_CHANNEL, } -DOCKER_IMAGE = "gcr.io/moz-fx-data-airflow-prod-88e0/bigquery-etl:latest" +DOCKER_IMAGE = "us-docker.pkg.dev/moz-fx-data-artifacts-prod/bigquery-etl/bigquery-etl:latest" tags = [Tag.ImpactTier.tier_3] diff --git a/dags/bqetl_dryrun.py b/dags/bqetl_dryrun.py index 863ce0f61..50a5af923 100644 --- a/dags/bqetl_dryrun.py +++ b/dags/bqetl_dryrun.py @@ -58,7 +58,7 @@ def should_dryrun(dag_id: str) -> bool: doc_md=__doc__, tags=tags, ) as dag: - docker_image = "gcr.io/moz-fx-data-airflow-prod-88e0/bigquery-etl:latest" + docker_image = "us-docker.pkg.dev/moz-fx-data-artifacts-prod/bigquery-etl/bigquery-etl:latest" skip_if_queued_runs_exist = ShortCircuitOperator( task_id="skip_if_queued_runs_exist", diff --git a/dags/microsoft_store.py b/dags/microsoft_store.py index d237dd0fd..e55d40ff5 100644 --- a/dags/microsoft_store.py +++ b/dags/microsoft_store.py @@ -83,7 +83,7 @@ f"sql/moz-fx-data-shared-prod/microsoft_derived/{table}_v1/query.py", "--date={{ macros.ds_add(ds, -3) }}", ], - image="gcr.io/moz-fx-data-airflow-prod-88e0/bigquery-etl:latest", + image="us-docker.pkg.dev/moz-fx-data-artifacts-prod/bigquery-etl/bigquery-etl:latest", owner="mhirose@mozilla.com", email=["mhirose@mozilla.com", "telemetry-alerts@mozilla.com"], ) diff --git a/dags/shredder.py b/dags/shredder.py index c549cd44d..7836eb461 100644 --- a/dags/shredder.py +++ b/dags/shredder.py @@ -65,7 +65,7 @@ doc_md=docs, tags=tags, ) -docker_image = "gcr.io/moz-fx-data-airflow-prod-88e0/bigquery-etl:latest" +docker_image = "us-docker.pkg.dev/moz-fx-data-artifacts-prod/bigquery-etl/bigquery-etl:latest" base_command = [ "script/shredder_delete", "--state-table=moz-fx-data-shredder.shredder_state.shredder_state", diff --git a/dags/shredder_backfill.py b/dags/shredder_backfill.py index 9ea7d8220..c83deb3d4 100644 --- a/dags/shredder_backfill.py +++ b/dags/shredder_backfill.py @@ -98,7 +98,7 @@ def base_backfill_operator(dry_run): ], # target_tables will be rendered as a python list arguments="{{ params.target_tables }}", - image="gcr.io/moz-fx-data-airflow-prod-88e0/bigquery-etl:latest", + image="us-docker.pkg.dev/moz-fx-data-artifacts-prod/bigquery-etl/bigquery-etl:latest", on_finish_action=OnFinishAction.DELETE_POD.value, reattach_on_restart=True, ) diff --git a/dags/socorro_import.py b/dags/socorro_import.py index fbce76f81..ad22fc3e5 100644 --- a/dags/socorro_import.py +++ b/dags/socorro_import.py @@ -120,7 +120,7 @@ task_id="remove_socorro_crash_bq_table_partition", gcp_conn_id=bq_gcp_conn_id, name="remove_socorro_crash_bq_table_partition", - image="gcr.io/moz-fx-data-airflow-prod-88e0/bigquery-etl:latest", + image="us-docker.pkg.dev/moz-fx-data-artifacts-prod/bigquery-etl/bigquery-etl:latest", arguments=["bq", "rm", "-f", "--table", table_name], ) diff --git a/utils/gcp.py b/utils/gcp.py index b76658a72..1771b5c29 100644 --- a/utils/gcp.py +++ b/utils/gcp.py @@ -19,7 +19,7 @@ GCP_PROJECT_ID = "moz-fx-data-airflow-gke-prod" DATAPROC_PROJECT_ID = "airflow-dataproc" -BIGQUERY_ETL_DOCKER_IMAGE = "gcr.io/moz-fx-data-airflow-prod-88e0/bigquery-etl:latest" +BIGQUERY_ETL_DOCKER_IMAGE = "us-docker.pkg.dev/moz-fx-data-artifacts-prod/bigquery-etl/bigquery-etl:latest" def export_to_parquet( diff --git a/utils/glam_subdags/generate_query.py b/utils/glam_subdags/generate_query.py index 3b748bbbd..2c79b62ae 100644 --- a/utils/glam_subdags/generate_query.py +++ b/utils/glam_subdags/generate_query.py @@ -7,7 +7,7 @@ def generate_and_run_glean_queries( destination_project_id, destination_dataset_id="glam_etl", source_project_id="moz-fx-data-shared-prod", - docker_image="gcr.io/moz-fx-data-airflow-prod-88e0/bigquery-etl:latest", + docker_image="us-docker.pkg.dev/moz-fx-data-artifacts-prod/bigquery-etl/bigquery-etl:latest", env_vars=None, **kwargs, ): @@ -53,7 +53,7 @@ def generate_and_run_glean_task( destination_project_id, destination_dataset_id="glam_etl", source_project_id="moz-fx-data-shared-prod", - docker_image="gcr.io/moz-fx-data-airflow-prod-88e0/bigquery-etl:latest", + docker_image="us-docker.pkg.dev/moz-fx-data-artifacts-prod/bigquery-etl/bigquery-etl:latest", env_vars=None, min_sample_id = 0, max_sample_id = 99,