[Logging] Fix and standardize timestamp formats across Cloudwatch log configurations.

- Rename timestamp format keys to use consistent naming convention (iso8610, default_seconds)
- Update CloudWatch agent config to use iso8610 format for JSON event logs (clustermgtd, slurm_resume)
- Consolidate Slurm log timestamp formats (slurmd, slurmctld, slurmdbd) to use iso8610
- Update SSSD log timestamp format from default to default_seconds for consistency
- Change DCV authenticator log format from bracket_default to default
- Add millisecond precision to PS4 prompt in generate_ssh_key.sh for better debug logging
- Add millisecond precision to pcluster_dcv_connect.sh log timestamps for improved log accuracy
- Improves log parsing consistency and debugging capabilities across all services

Author: gmarciani

CHANGELOG.md

@@ -6,6 +6,9 @@ This file is used to list changes made in each version of the AWS ParallelCluste
 3.15.0
 ------
 
+**CHANGES**
+- Fix and standardize timestamp formats in CloudWatch log configurations to facilitate maintainability and debugging.
+
 3.14.1
 ------
 

cookbooks/aws-parallelcluster-environment/files/cloudwatch/cloudwatch_agent_config.json

@@ -2,10 +2,9 @@
   "timestamp_formats": {
     "month_first": "%b %-d %H:%M:%S",
     "default": "%Y-%m-%d %H:%M:%S,%f",
-    "bracket_default": "[%Y-%m-%d %H:%M:%S]",
-    "slurm": "%Y-%m-%dT%H:%M:%S.%f",
-    "chef": "[%Y-%m-%dT%H:%M:%S",
-    "json": ""
+    "default_seconds": "%Y-%m-%d %H:%M:%S",
+    "iso8610": "%Y-%m-%dT%H:%M:%S.%f",
+    "chef": "[%Y-%m-%dT%H:%M:%S"
   },
   "log_configs": [
     {
@@ -100,7 +99,7 @@
       "feature_conditions": []
     },
     {
-      "timestamp_format_key": "json",
+      "timestamp_format_key": "iso8610",
       "file_path": "/var/log/parallelcluster/bootstrap_error_msg",
       "log_stream_name": "bootstrap_error_msg",
       "schedulers": [
@@ -113,7 +112,7 @@
       "feature_conditions": []
     },
     {
-      "timestamp_format_key": "default",
+      "timestamp_format_key": "default_seconds",
       "file_path": "/var/log/cloud-init.log",
       "log_stream_name": "cloud-init",
       "schedulers": [
@@ -130,7 +129,7 @@
       "feature_conditions": []
     },
     {
-      "timestamp_format_key": "default",
+      "timestamp_format_key": "default_seconds",
       "file_path": "/var/log/cloud-init-output.log",
       "log_stream_name": "cloud-init-output",
       "schedulers": [
@@ -174,7 +173,7 @@
       "feature_conditions": []
     },
     {
-      "timestamp_format_key": "json",
+      "timestamp_format_key": "iso8610",
       "file_path": "/var/log/parallelcluster/clustermgtd.events",
       "log_stream_name": "clustermgtd_events",
       "schedulers": [
@@ -187,7 +186,7 @@
       "feature_conditions": []
     },
     {
-      "timestamp_format_key": "json",
+      "timestamp_format_key": "iso8610",
       "file_path": "/var/log/parallelcluster/slurm_resume.events",
       "log_stream_name": "slurm_resume_events",
       "schedulers": [
@@ -265,7 +264,7 @@
       "feature_conditions": []
     },
     {
-      "timestamp_format_key": "slurm",
+      "timestamp_format_key": "iso8610",
       "file_path": "/var/log/slurmd.log",
       "log_stream_name": "slurmd",
       "schedulers": [
@@ -278,7 +277,7 @@
       "feature_conditions": []
     },
     {
-      "timestamp_format_key": "slurm",
+      "timestamp_format_key": "iso8610",
       "file_path": "/var/log/slurmctld.log",
       "log_stream_name": "slurmctld",
       "schedulers": [
@@ -291,7 +290,7 @@
       "feature_conditions": []
     },
     {
-      "timestamp_format_key": "slurm",
+      "timestamp_format_key": "iso8610",
       "file_path": "/var/log/slurmdbd.log",
       "log_stream_name": "slurmdbd",
       "schedulers": [
@@ -325,7 +324,7 @@
       ]
     },
     {
-      "timestamp_format_key": "default",
+      "timestamp_format_key": "default_seconds",
       "file_path": "/var/log/sssd/sssd.log",
       "log_stream_name": "sssd",
       "schedulers": [
@@ -346,7 +345,7 @@
       ]
     },
     {
-      "timestamp_format_key": "default",
+      "timestamp_format_key": "default_seconds",
       "file_path": "/var/log/sssd/sssd_default.log",
       "log_stream_name": "sssd_domain_default",
       "schedulers": [
@@ -389,7 +388,7 @@
       ]
     },
     {
-      "timestamp_format_key": "bracket_default",
+      "timestamp_format_key": "default",
       "file_path": "/var/log/parallelcluster/pcluster_dcv_connect.log",
       "log_stream_name": "dcv-ext-authenticator",
       "schedulers": [
@@ -522,7 +521,7 @@
       "feature_conditions": []
     },
     {
-      "timestamp_format_key": "json",
+      "timestamp_format_key": "iso8610",
       "file_path": "/var/log/parallelcluster/slurm_health_check.events",
       "log_stream_name": "slurm_health_check_events",
       "schedulers": [

cookbooks/aws-parallelcluster-environment/templates/directory_service/generate_ssh_key.sh.erb

@@ -1,4 +1,5 @@
 #!/bin/bash
+PS4="$(date '+%Y-%m-%d %H:%M:%S.%3N') "
 set -ex
 env
 

cookbooks/aws-parallelcluster-platform/files/dcv/pcluster_dcv_connect.sh

@@ -82,7 +82,7 @@ _log() {
     fi
 
     # append log
-    log_time=$(date "+%Y-%m-%d %H:%M:%S")
+    log_time=$(date "+%Y-%m-%d %H:%M:%S,%3N")
     echo "[${log_time}]: ${text}" >> "${LOG_FILE_PATH}"
 }