diff --git a/README.md b/README.md index f95fbb6..561ee02 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ pip install -r requirements.txt ``` Copy your service account .json files in the root directory of the project: +(skip this step if you want to use the user-account of your machine, if you already use `gcloud` on the machine) ``` cp path/to/service_account.json . @@ -45,7 +46,11 @@ cp path/to/service_account.json . Execute ``` +# To use the copied service-account: python msstats.py + +# or, to use the `gcloud` user (then, you _need_ to give a precise project): +python3 msstats.py --user-account --project-id my-gcp-project ``` This generates a file named .xlsx. You need to get that file and send it to Redis. diff --git a/msstats.py b/msstats.py index c64034a..dee7e49 100644 --- a/msstats.py +++ b/msstats.py @@ -582,20 +582,18 @@ def processMetricPoint(metricPoint): return processedMetricPoint -def process_google_service_account( - service_account, project_id, duration=604800, step=60 -): - if not project_id: - try: - f = open(service_account, "r") - data = json.loads(f.read()) - f.close() - project_id = data["project_id"] - if not project_id: - raise Exception("Invalid json file") - except: - print(f"Error: Could not read service account file {service_account}") - return +def get_project_from_service_account_and_authenticate(service_account): + try: + f = open(service_account, "r") + data = json.loads(f.read()) + f.close() + project_id = data["project_id"] + if not project_id: + raise Exception("Invalid json file") + return project_id + except: + print(f"Error: Could not read service account file {service_account}") + return # Set the value GOOGLE_APPLICATION_CREDENTIALS variable os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = service_account @@ -604,6 +602,11 @@ def process_google_service_account( service_account, ) + +def process_google_project(project_id, duration=604800, step=60): + if not project_id: + raise Exception("Missing project_id") + client = monitoring_v3.MetricServiceClient() project_name = f"projects/{project_id}" @@ -787,7 +790,7 @@ def process_google_service_account( if not metric_points: print(f"Warning: No data for project {project_id} - Excel file will be empty") - return project_id, metric_points + return metric_points def create_workbooks(outDir, projects): @@ -838,6 +841,13 @@ def main(): help="The Google Cloud Project ID containing MemoryStore instances.", metavar="PROJECT_ID", ) + parser.add_option( + "--user-account", + dest="use_user_account", + action="store_true", + default=False, + help="Connect to GCP using the (default) auth of the machine, thus not using any service-accounts", + ) parser.add_option( "--step", @@ -859,26 +869,38 @@ def main(): if not os.path.isdir(options.outDir): os.makedirs(options.outDir) - # Scan for .json files in order to find the service account files - path_to_json = "." - service_accounts = [ - os.path.abspath(os.path.join(path_to_json, pos_json)) - for pos_json in os.listdir(path_to_json) - if pos_json.endswith(".json") - ] - - if not service_accounts: - print("Error: No service account JSON files found in current directory") - exit(1) - projects = {} - # For each service account found try to fetch the clusters metrics using the - # google cloud monitoring api metrics - for service_account in service_accounts: - project_id, stats = process_google_service_account( - service_account, options.project_id, options.duration, options.step + if options.use_user_account: + # We don't use the service-account + projects[options.project_id] = process_google_project( + options.project_id, options.duration, options.step ) - projects[project_id] = stats + else: + # Scan for .json files in order to find the service account files + path_to_json = "." + service_accounts = [ + os.path.abspath(os.path.join(path_to_json, pos_json)) + for pos_json in os.listdir(path_to_json) + if pos_json.endswith(".json") + ] + + if not service_accounts: + print("Error: No service account JSON files found in current directory") + exit(1) + + # For each service account found try to fetch the clusters metrics using the + # google cloud monitoring api metrics + for service_account in service_accounts: + print(f"Loading service-account: {service_account}") + project_id = get_project_from_service_account_and_authenticate( + service_account + ) + if options.project_id and options.project_id != project_id: + # skip this project, since only one project has been requested + continue + projects[project_id] = process_google_project( + project_id, options.duration, options.step + ) if not projects: print("Error: No projects were successfully processed") diff --git a/test_msstats.py b/test_msstats.py index b282e9e..f6eac5d 100644 --- a/test_msstats.py +++ b/test_msstats.py @@ -11,7 +11,8 @@ get_all_commands, processNodeStats, processMetricPoint, - process_google_service_account, + get_project_from_service_account_and_authenticate, + process_google_project, create_workbooks, ) @@ -289,7 +290,7 @@ def tearDown(self): os.rmdir(self.temp_dir) @patch("msstats.monitoring_v3.MetricServiceClient") - def test_process_google_service_account_with_mock_data(self, mock_client_class): + def test_process_google_project_with_mock_data(self, mock_client_class): """Test processing service account with mocked Google Cloud responses""" # Mock the monitoring client mock_client = MagicMock() @@ -341,15 +342,13 @@ def test_process_google_service_account_with_mock_data(self, mock_client_class): ] # Test the function - project_id, stats = process_google_service_account( - self.service_account_file, + stats = process_google_project( self.test_project_id, duration=3600, # 1 hour step=60, ) # Assertions - self.assertEqual(project_id, self.test_project_id) self.assertIsInstance(stats, dict) self.assertIn("test-redis", stats) @@ -441,15 +440,9 @@ def test_service_account_file_parsing(self, mock_client_class): mock_client_class.return_value = mock_client mock_client.list_time_series.return_value = [] - # Test with explicit project_id - project_id, _ = process_google_service_account( - self.service_account_file, "explicit-project-id" - ) - self.assertEqual(project_id, "explicit-project-id") - - # Test without explicit project_id (should read from file) - project_id, _ = process_google_service_account( - self.service_account_file, "" # Empty project_id should read from file + # It should read from file: + project_id = get_project_from_service_account_and_authenticate( + self.service_account_file ) self.assertEqual(project_id, self.test_project_id) @@ -461,7 +454,7 @@ def test_invalid_service_account_file(self): f.write("invalid json content") # Should return None for invalid files - result = process_google_service_account(invalid_file, "") + result = get_project_from_service_account_and_authenticate(invalid_file) self.assertIsNone(result) # Clean up @@ -472,7 +465,7 @@ def test_missing_service_account_file(self): nonexistent_file = os.path.join(self.temp_dir, "nonexistent.json") # Should return None for missing files - result = process_google_service_account(nonexistent_file, "") + result = get_project_from_service_account_and_authenticate(nonexistent_file) self.assertIsNone(result)