From e3d0beaf76f8147b15617137db8259a5ce84cb5e Mon Sep 17 00:00:00 2001 From: Corban Cloud Date: Mon, 9 Dec 2024 15:31:28 -0800 Subject: [PATCH 1/3] Update bucket ref to match v2 infra bucket name --- glam/api/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/glam/api/constants.py b/glam/api/constants.py index eac0bdf14..10b9314d4 100644 --- a/glam/api/constants.py +++ b/glam/api/constants.py @@ -45,4 +45,4 @@ PROCESS_NAMES = dict(PROCESS_CHOICES) PROCESS_IDS = {v: k for k, v in PROCESS_NAMES.items()} -GCS_BUCKET = "moz-fx-data-glam-prod-fca7-etl-data" +GCS_BUCKET = "moz-fx-glam-prod-etl-data" From 1123f18a25a0bc998f9f73aa1ce8163db08fe927 Mon Sep 17 00:00:00 2001 From: Corban Cloud Date: Mon, 9 Dec 2024 18:03:32 -0800 Subject: [PATCH 2/3] SWAG: Add CLI arg to change GCP project the BQ Job is run in for migrations --- glam/api/constants.py | 4 ++++ glam/api/management/commands/import_desktop_aggs.py | 8 +++++++- glam/api/management/commands/import_revisions.py | 12 +++++++++++- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/glam/api/constants.py b/glam/api/constants.py index 10b9314d4..88449a5c7 100644 --- a/glam/api/constants.py +++ b/glam/api/constants.py @@ -46,3 +46,7 @@ PROCESS_IDS = {v: k for k, v in PROCESS_NAMES.items()} GCS_BUCKET = "moz-fx-glam-prod-etl-data" + +GCP_PROD_PROJECT = "moz-fx-glam-prod" +GCP_NON_PROD_PROJECT = "moz-fx-glam-nonprod" +GCP_PROJECT_CHOICES = list(GCP_NON_PROD_PROJECT, GCP_PROD_PROJECT) \ No newline at end of file diff --git a/glam/api/management/commands/import_desktop_aggs.py b/glam/api/management/commands/import_desktop_aggs.py index 62b618214..4c70ee3ae 100644 --- a/glam/api/management/commands/import_desktop_aggs.py +++ b/glam/api/management/commands/import_desktop_aggs.py @@ -31,6 +31,7 @@ def log(channel, message): class Command(BaseCommand): help = "Imports aggregation data" + bq_job_projects = list(GCP_NON_PROD_PROJECT, GCP_PROD_PROJECT) def add_arguments(self, parser): parser.add_argument( @@ -42,6 +43,11 @@ def add_arguments(self, parser): help="The bucket location for the exported aggregates", default=constants.GCS_BUCKET, ) + parser.add_argument( + "--bq_job_project", + choices=self.bq_job_project, + required=True + ) def handle(self, bucket, *args, **options): @@ -49,7 +55,7 @@ def handle(self, bucket, *args, **options): model = apps.get_model(CHANNEL_TO_MODEL[channel]) self.gcs_client = storage.Client() - self.bq_client = bigquery.Client() + self.bq_client = bigquery.Client( project=options["bq_job_project"] ) blobs = self.gcs_client.list_blobs(bucket) blobs = list( diff --git a/glam/api/management/commands/import_revisions.py b/glam/api/management/commands/import_revisions.py index fee411f7c..601d5073e 100644 --- a/glam/api/management/commands/import_revisions.py +++ b/glam/api/management/commands/import_revisions.py @@ -23,6 +23,7 @@ class Command(BaseCommand): help = "Imports builds SHA revisions" channel_choices = list(constants.CHANNEL_IDS.keys()) + ["all"] + bq_job_projects = list(GCP_NON_PROD_PROJECT, GCP_PROD_PROJECT) def add_arguments(self, parser): parser.add_argument( @@ -32,13 +33,22 @@ def add_arguments(self, parser): required=False, ) + parser.add_argument( + "--bq_job_project", + choices=self.bq_job_project, + required=True + ) + def handle(self, *args, **options): - self.bq_client = bigquery.Client() + channels = ( [options["channel"]] if options["channel"] != "all" else ["nightly", "beta", "release"] ) + + self.bq_client = bigquery.Client( project=options["bq_job_project"] ) + for channel in channels: self.import_revisions(channel) From af4d3dfe6973c2c41bd704b3fc96b691cd445a23 Mon Sep 17 00:00:00 2001 From: Corban Cloud Date: Tue, 10 Dec 2024 14:14:22 -0800 Subject: [PATCH 3/3] Revert "SWAG: Add CLI arg to change GCP project the BQ Job is run in for migrations" This reverts commit 1123f18a25a0bc998f9f73aa1ce8163db08fe927. --- glam/api/constants.py | 4 ---- glam/api/management/commands/import_desktop_aggs.py | 8 +------- glam/api/management/commands/import_revisions.py | 12 +----------- 3 files changed, 2 insertions(+), 22 deletions(-) diff --git a/glam/api/constants.py b/glam/api/constants.py index 88449a5c7..10b9314d4 100644 --- a/glam/api/constants.py +++ b/glam/api/constants.py @@ -46,7 +46,3 @@ PROCESS_IDS = {v: k for k, v in PROCESS_NAMES.items()} GCS_BUCKET = "moz-fx-glam-prod-etl-data" - -GCP_PROD_PROJECT = "moz-fx-glam-prod" -GCP_NON_PROD_PROJECT = "moz-fx-glam-nonprod" -GCP_PROJECT_CHOICES = list(GCP_NON_PROD_PROJECT, GCP_PROD_PROJECT) \ No newline at end of file diff --git a/glam/api/management/commands/import_desktop_aggs.py b/glam/api/management/commands/import_desktop_aggs.py index 4c70ee3ae..62b618214 100644 --- a/glam/api/management/commands/import_desktop_aggs.py +++ b/glam/api/management/commands/import_desktop_aggs.py @@ -31,7 +31,6 @@ def log(channel, message): class Command(BaseCommand): help = "Imports aggregation data" - bq_job_projects = list(GCP_NON_PROD_PROJECT, GCP_PROD_PROJECT) def add_arguments(self, parser): parser.add_argument( @@ -43,11 +42,6 @@ def add_arguments(self, parser): help="The bucket location for the exported aggregates", default=constants.GCS_BUCKET, ) - parser.add_argument( - "--bq_job_project", - choices=self.bq_job_project, - required=True - ) def handle(self, bucket, *args, **options): @@ -55,7 +49,7 @@ def handle(self, bucket, *args, **options): model = apps.get_model(CHANNEL_TO_MODEL[channel]) self.gcs_client = storage.Client() - self.bq_client = bigquery.Client( project=options["bq_job_project"] ) + self.bq_client = bigquery.Client() blobs = self.gcs_client.list_blobs(bucket) blobs = list( diff --git a/glam/api/management/commands/import_revisions.py b/glam/api/management/commands/import_revisions.py index 601d5073e..fee411f7c 100644 --- a/glam/api/management/commands/import_revisions.py +++ b/glam/api/management/commands/import_revisions.py @@ -23,7 +23,6 @@ class Command(BaseCommand): help = "Imports builds SHA revisions" channel_choices = list(constants.CHANNEL_IDS.keys()) + ["all"] - bq_job_projects = list(GCP_NON_PROD_PROJECT, GCP_PROD_PROJECT) def add_arguments(self, parser): parser.add_argument( @@ -33,22 +32,13 @@ def add_arguments(self, parser): required=False, ) - parser.add_argument( - "--bq_job_project", - choices=self.bq_job_project, - required=True - ) - def handle(self, *args, **options): - + self.bq_client = bigquery.Client() channels = ( [options["channel"]] if options["channel"] != "all" else ["nightly", "beta", "release"] ) - - self.bq_client = bigquery.Client( project=options["bq_job_project"] ) - for channel in channels: self.import_revisions(channel)