From cf1f16144243d2f1b4b6d6c663249df0fedc0687 Mon Sep 17 00:00:00 2001 From: Florent Rivoire Date: Tue, 5 Aug 2025 18:56:26 +0200 Subject: [PATCH 1/7] Add --user-account option --- msstats.py | 85 +++++++++++++++++++++++++++++++++--------------------- 1 file changed, 52 insertions(+), 33 deletions(-) diff --git a/msstats.py b/msstats.py index c64034a..2af80c2 100644 --- a/msstats.py +++ b/msstats.py @@ -582,20 +582,18 @@ def processMetricPoint(metricPoint): return processedMetricPoint -def process_google_service_account( - service_account, project_id, duration=604800, step=60 -): - if not project_id: - try: - f = open(service_account, "r") - data = json.loads(f.read()) - f.close() - project_id = data["project_id"] - if not project_id: - raise Exception("Invalid json file") - except: - print(f"Error: Could not read service account file {service_account}") - return +def get_project_from_service_account_and_authenticate(service_account): + try: + f = open(service_account, "r") + data = json.loads(f.read()) + f.close() + project_id = data["project_id"] + if not project_id: + raise Exception("Invalid json file") + return project_id + except: + print(f"Error: Could not read service account file {service_account}") + return # Set the value GOOGLE_APPLICATION_CREDENTIALS variable os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = service_account @@ -604,6 +602,11 @@ def process_google_service_account( service_account, ) + +def process_google_project(project_id, duration=604800, step=60): + if not project_id: + raise Exception("Missing project_id") + client = monitoring_v3.MetricServiceClient() project_name = f"projects/{project_id}" @@ -787,7 +790,7 @@ def process_google_service_account( if not metric_points: print(f"Warning: No data for project {project_id} - Excel file will be empty") - return project_id, metric_points + return metric_points def create_workbooks(outDir, projects): @@ -838,6 +841,13 @@ def main(): help="The Google Cloud Project ID containing MemoryStore instances.", metavar="PROJECT_ID", ) + parser.add_option( + "--user-account", + dest="use_user_account", + action='store_true', + default=False, + help="Connect to GCP using the (default) auth of the machine, thus not using any service-accounts", + ) parser.add_option( "--step", @@ -859,26 +869,35 @@ def main(): if not os.path.isdir(options.outDir): os.makedirs(options.outDir) - # Scan for .json files in order to find the service account files - path_to_json = "." - service_accounts = [ - os.path.abspath(os.path.join(path_to_json, pos_json)) - for pos_json in os.listdir(path_to_json) - if pos_json.endswith(".json") - ] - - if not service_accounts: - print("Error: No service account JSON files found in current directory") - exit(1) - projects = {} - # For each service account found try to fetch the clusters metrics using the - # google cloud monitoring api metrics - for service_account in service_accounts: - project_id, stats = process_google_service_account( - service_account, options.project_id, options.duration, options.step + if(options.use_user_account): + # We don't use the service-account + projects[options.project_id] = process_google_project( + options.project_id, options.duration, options.step ) - projects[project_id] = stats + else: + # Scan for .json files in order to find the service account files + path_to_json = "." + service_accounts = [ + os.path.abspath(os.path.join(path_to_json, pos_json)) + for pos_json in os.listdir(path_to_json) + if pos_json.endswith(".json") + ] + + if not service_accounts: + print("Error: No service account JSON files found in current directory") + exit(1) + + # For each service account found try to fetch the clusters metrics using the + # google cloud monitoring api metrics + for service_account in service_accounts: + project_id = get_project_from_service_account_and_authenticate(service_account) + if options.project_id and options.project_id != project_id: + # skip this project, since only one project has been requested + continue + projects[project_id] = process_google_project( + project_id, options.duration, options.step + ) if not projects: print("Error: No projects were successfully processed") From c2f0604cad739e6bda9c315ea78d689d7d6adacf Mon Sep 17 00:00:00 2001 From: Florent Rivoire Date: Tue, 5 Aug 2025 19:08:50 +0200 Subject: [PATCH 2/7] Update README --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index f95fbb6..cebb7ca 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ pip install -r requirements.txt ``` Copy your service account .json files in the root directory of the project: +(skip this step if you want to use the user-account of your machine, cf you already use `gcloud` on the machine) ``` cp path/to/service_account.json . @@ -45,7 +46,11 @@ cp path/to/service_account.json . Execute ``` +# To use the copied service-account: python msstats.py + +# or, to use the `gcloud` user: +python3 msstats.py --user-account ``` This generates a file named .xlsx. You need to get that file and send it to Redis. From ee388f7e556ac197d54a921959cd50d6df5d1791 Mon Sep 17 00:00:00 2001 From: Florent Rivoire Date: Tue, 5 Aug 2025 19:10:26 +0200 Subject: [PATCH 3/7] add a log line --- msstats.py | 1 + 1 file changed, 1 insertion(+) diff --git a/msstats.py b/msstats.py index 2af80c2..c591aa3 100644 --- a/msstats.py +++ b/msstats.py @@ -891,6 +891,7 @@ def main(): # For each service account found try to fetch the clusters metrics using the # google cloud monitoring api metrics for service_account in service_accounts: + print(f"Loading service-account: {service_account}") project_id = get_project_from_service_account_and_authenticate(service_account) if options.project_id and options.project_id != project_id: # skip this project, since only one project has been requested From de797d79e9f4cbbbea22cea776199897b7d8fa4f Mon Sep 17 00:00:00 2001 From: Florent Rivoire Date: Wed, 6 Aug 2025 09:23:33 +0200 Subject: [PATCH 4/7] fix tests --- test_msstats.py | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/test_msstats.py b/test_msstats.py index b282e9e..f6eac5d 100644 --- a/test_msstats.py +++ b/test_msstats.py @@ -11,7 +11,8 @@ get_all_commands, processNodeStats, processMetricPoint, - process_google_service_account, + get_project_from_service_account_and_authenticate, + process_google_project, create_workbooks, ) @@ -289,7 +290,7 @@ def tearDown(self): os.rmdir(self.temp_dir) @patch("msstats.monitoring_v3.MetricServiceClient") - def test_process_google_service_account_with_mock_data(self, mock_client_class): + def test_process_google_project_with_mock_data(self, mock_client_class): """Test processing service account with mocked Google Cloud responses""" # Mock the monitoring client mock_client = MagicMock() @@ -341,15 +342,13 @@ def test_process_google_service_account_with_mock_data(self, mock_client_class): ] # Test the function - project_id, stats = process_google_service_account( - self.service_account_file, + stats = process_google_project( self.test_project_id, duration=3600, # 1 hour step=60, ) # Assertions - self.assertEqual(project_id, self.test_project_id) self.assertIsInstance(stats, dict) self.assertIn("test-redis", stats) @@ -441,15 +440,9 @@ def test_service_account_file_parsing(self, mock_client_class): mock_client_class.return_value = mock_client mock_client.list_time_series.return_value = [] - # Test with explicit project_id - project_id, _ = process_google_service_account( - self.service_account_file, "explicit-project-id" - ) - self.assertEqual(project_id, "explicit-project-id") - - # Test without explicit project_id (should read from file) - project_id, _ = process_google_service_account( - self.service_account_file, "" # Empty project_id should read from file + # It should read from file: + project_id = get_project_from_service_account_and_authenticate( + self.service_account_file ) self.assertEqual(project_id, self.test_project_id) @@ -461,7 +454,7 @@ def test_invalid_service_account_file(self): f.write("invalid json content") # Should return None for invalid files - result = process_google_service_account(invalid_file, "") + result = get_project_from_service_account_and_authenticate(invalid_file) self.assertIsNone(result) # Clean up @@ -472,7 +465,7 @@ def test_missing_service_account_file(self): nonexistent_file = os.path.join(self.temp_dir, "nonexistent.json") # Should return None for missing files - result = process_google_service_account(nonexistent_file, "") + result = get_project_from_service_account_and_authenticate(nonexistent_file) self.assertIsNone(result) From 014561476e42fd452a37bcd1c8f907c525f3134d Mon Sep 17 00:00:00 2001 From: Florent Rivoire Date: Wed, 6 Aug 2025 17:38:43 +0200 Subject: [PATCH 5/7] black reformatting --- msstats.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/msstats.py b/msstats.py index c591aa3..dee7e49 100644 --- a/msstats.py +++ b/msstats.py @@ -844,7 +844,7 @@ def main(): parser.add_option( "--user-account", dest="use_user_account", - action='store_true', + action="store_true", default=False, help="Connect to GCP using the (default) auth of the machine, thus not using any service-accounts", ) @@ -870,7 +870,7 @@ def main(): os.makedirs(options.outDir) projects = {} - if(options.use_user_account): + if options.use_user_account: # We don't use the service-account projects[options.project_id] = process_google_project( options.project_id, options.duration, options.step @@ -892,7 +892,9 @@ def main(): # google cloud monitoring api metrics for service_account in service_accounts: print(f"Loading service-account: {service_account}") - project_id = get_project_from_service_account_and_authenticate(service_account) + project_id = get_project_from_service_account_and_authenticate( + service_account + ) if options.project_id and options.project_id != project_id: # skip this project, since only one project has been requested continue From 58e6276c0ae5160980fb034529f49a9d64b512d7 Mon Sep 17 00:00:00 2001 From: Florent Rivoire Date: Wed, 6 Aug 2025 17:39:35 +0200 Subject: [PATCH 6/7] Update README.md Co-authored-by: Andrew Gingrich --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cebb7ca..ca105ea 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ pip install -r requirements.txt ``` Copy your service account .json files in the root directory of the project: -(skip this step if you want to use the user-account of your machine, cf you already use `gcloud` on the machine) +(skip this step if you want to use the user-account of your machine, if you already use `gcloud` on the machine) ``` cp path/to/service_account.json . From e2b1e89672a0761491df3ce443ab863aa72adb22 Mon Sep 17 00:00:00 2001 From: Florent Rivoire Date: Wed, 6 Aug 2025 17:40:58 +0200 Subject: [PATCH 7/7] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ca105ea..561ee02 100644 --- a/README.md +++ b/README.md @@ -49,8 +49,8 @@ Execute # To use the copied service-account: python msstats.py -# or, to use the `gcloud` user: -python3 msstats.py --user-account +# or, to use the `gcloud` user (then, you _need_ to give a precise project): +python3 msstats.py --user-account --project-id my-gcp-project ``` This generates a file named .xlsx. You need to get that file and send it to Redis.