Adding Functionality For Custom Metrics Testing

.github/workflows/python-ec2-default-test.yml

@@ -32,6 +32,14 @@ on:
         required: false
         type: string
         default: 'github'
+      custom-metrics-enabled:
+        required: false
+        type: boolean
+        default: false
+      custom-metrics-config:
+        required: false
+        type: string
+        default: '{}'
     outputs:
       job-started:
         value: ${{ jobs.python-ec2-default.outputs.job-started }}
@@ -49,6 +57,8 @@ env:
   CPU_ARCHITECTURE: ${{ inputs.cpu-architecture }}
   ADOT_WHEEL_NAME: ${{ inputs.staging-wheel-name }}
   OTEL_SOURCE: ${{ inputs.otel-source }}
+  CUSTOM_METRICS_ENABLED: ${{ inputs.custom-metrics-enabled }}
+  CUSTOM_METRICS_CONFIG: ${{ inputs.custom-metrics-config }}
   SAMPLE_APP_ZIP: s3://aws-appsignals-sample-app-prod-${{ inputs.aws-region }}/python-sample-app.zip
   E2E_TEST_ACCOUNT_ID: ${{ secrets.APPLICATION_SIGNALS_E2E_TEST_ACCOUNT_ID }}
   E2E_TEST_ROLE_NAME: ${{ secrets.APPLICATION_SIGNALS_E2E_TEST_ROLE_NAME }}
@@ -171,6 +181,8 @@ jobs:
               -var="get_adot_wheel_command=${{ env.GET_ADOT_WHEEL_COMMAND }}" \
               -var="language_version=${{ env.PYTHON_VERSION }}" \
               -var="cpu_architecture=${{ env.CPU_ARCHITECTURE }}" \
+              -var="custom_metrics_enabled=${{ env.CUSTOM_METRICS_ENABLED }}" \
+              -var="custom_metrics_config=${{ env.CUSTOM_METRICS_CONFIG }}" \
             || deployment_failed=$?
 
             if [ $deployment_failed -eq 1 ]; then

sample-apps/python/django_frontend_service/ec2-requirements.txt

@@ -6,3 +6,4 @@ requests~=2.25.1
 schedule~=1.2.1
 opentelemetry-sdk==1.33.1
 opentelemetry-api==1.33.1
+opentelemetry-exporter-otlp-proto-http==1.22.0

sample-apps/python/django_frontend_service/frontend_service_app/views.py

@@ -11,11 +11,78 @@
 import requests
 import schedule
 from django.http import HttpResponse, JsonResponse
-from opentelemetry import trace
+from opentelemetry import trace, metrics
 from opentelemetry.trace.span import format_trace_id
+from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
+from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter
+from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter as HTTPMetricExporter
+from opentelemetry.sdk.metrics.export import ConsoleMetricsExporter
+from opentelemetry.sdk.resources import Resource
 
 logger = logging.getLogger(__name__)
 
+# Initialize custom OTEL metrics export pipeline - OTLP approach (OTEL/Span export 1) Agent based
+custom_resource = Resource.create({
+        "service.name": os.getenv("OTEL_SERVICE_NAME", "python-sample-application"),
+        "deployment.environment.name": os.getenv("DEPLOYMENT_ENV", "test"),
+        #"test" being over-rided by python-sample-application-${var.test_id} in main.tf
+        })
+custom_otlp_exporter = OTLPMetricExporter(
+    endpoint="http://localhost:4317",
+    insecure=True
+)
+custom_otlp_reader = PeriodicExportingMetricReader(
+    exporter=custom_otlp_exporter,
+    export_interval_millis=5000
+)
+
+# Initialize Console exporter - Direct output approach (OTEL export 2) Custom export pipeline
+custom_console_exporter = ConsoleMetricsExporter()
+custom_console_reader = PeriodicExportingMetricReader(
+    exporter=custom_console_exporter,
+    export_interval_millis=5000
+)
+
+# Custom Export Pipeline - HTTP Direct
+resource = Resource.create({
+    "service.name": os.getenv("OTEL_SERVICE_NAME", "python-sample-application"),
+    "deployment.environment.name": os.getenv("DEPLOYMENT_ENV", "test")
+    })#"test" being over-rided by python-sample-application-${var.test_id} in main.tf
+
+# OtlpHttpMetricExporter.builder().setEndpoint().build()
+metricExporter = HTTPMetricExporter(
+    endpoint="http://localhost:4318/v1/metrics"
+)
+
+# PeriodicMetricReader.builder(metricExporter).setInterval(Duration.ofSeconds(10)).build()
+metricReader = PeriodicExportingMetricReader(
+    exporter=metricExporter,
+    export_interval_millis=5000
+)
+
+# SdkMeterProvider.builder().setResource(resource).registerMetricReader(metricReader).build()
+meterProvider = MeterProvider(
+    resource=resource,
+    metric_readers=[metricReader]
+)
+
+# meterProvider.get("myMeter")
+meter = meterProvider.get_meter("myMeter")
+
+
+# Create meter provider with both exporters
+custom_meter_provider = MeterProvider(
+    resource=custom_resource,
+    metric_readers=[custom_otlp_reader, custom_console_reader]
+)
+
+# Initialize span metrics using custom meter provider
+custom_meter = custom_meter_provider.get_meter("custom-metrics")
+custom_request_counter = custom_meter.create_counter("custom_requests_total", description="Total requests")
+http_counter = meter.create_counter("custom_requests_total", description="Total requests")
+# custom_response_time_histogram = custom_meter.create_histogram("custom_response_time", description="Response time")
+
 should_send_local_root_client_call = False
 lock = threading.Lock()
 def run_local_root_client_call_recurring_service():
@@ -50,6 +117,13 @@ def healthcheck(request):
     return HttpResponse("healthcheck")
 
 def aws_sdk_call(request):
+    # Setup Span Attributes And Initialize Counter/Histogram To Recieve Custom Metrics
+    # start_time = time.time() #start histogram
+    custom_request_counter.add(1, {"operation.type": "aws_sdk_call"})  # Agent-based export
+    http_counter.add(1, {"operation.type": "aws_sdk_call"})  # Custom export pipeline
+    # duration = time.time() - start_time #end histogram
+    # custom_response_time_histogram.record(duration, {"operation.type": "aws_sdk_call"}) #record histogram
+
     bucket_name = "e2e-test-bucket-name"
 
     # Add a unique test ID to bucketname to associate buckets to specific test runs

terraform/dotnet/ec2/default/amazon-cloudwatch-custom-agent.json

@@ -0,0 +1,22 @@
+{
+  "traces": {
+    "traces_collected": {
+      "application_signals": {}
+    }
+  },
+  "logs": {
+    "metrics_collected": {
+      "application_signals": {}
+    }
+  },
+  "metrics": {
+    "namespace": "CWAgent",
+    "metrics_collected": {
+      "application_signals": {},
+      "otlp": {
+        "grpc_endpoint": "0.0.0.0:4317",
+        "http_endpoint": "0.0.0.0:4318"
+      }
+    }
+  }
+}

terraform/java/ec2/default/amazon-cloudwatch-custom-agent.json

@@ -0,0 +1,22 @@
+{
+  "traces": {
+    "traces_collected": {
+      "application_signals": {}
+    }
+  },
+  "logs": {
+    "metrics_collected": {
+      "application_signals": {}
+    }
+  },
+  "metrics": {
+    "namespace": "CWAgent",
+    "metrics_collected": {
+      "application_signals": {},
+      "otlp": {
+        "grpc_endpoint": "0.0.0.0:4317",
+        "http_endpoint": "0.0.0.0:4318"
+      }
+    }
+  }
+}

terraform/node/ec2/default/amazon-cloudwatch-custom-agent.json

@@ -0,0 +1,22 @@
+{
+  "traces": {
+    "traces_collected": {
+      "application_signals": {}
+    }
+  },
+  "logs": {
+    "metrics_collected": {
+      "application_signals": {}
+    }
+  },
+  "metrics": {
+    "namespace": "CWAgent",
+    "metrics_collected": {
+      "application_signals": {},
+      "otlp": {
+        "grpc_endpoint": "0.0.0.0:4317",
+        "http_endpoint": "0.0.0.0:4318"
+      }
+    }
+  }
+}

terraform/python/ec2/default/amazon-cloudwatch-custom-agent.json

@@ -0,0 +1,22 @@
+{
+  "traces": {
+    "traces_collected": {
+      "application_signals": {}
+    }
+  },
+  "logs": {
+    "metrics_collected": {
+      "application_signals": {}
+    }
+  },
+  "metrics": {
+    "namespace": "CWAgent",
+    "metrics_collected": {
+      "application_signals": {},
+      "otlp": {
+        "grpc_endpoint": "0.0.0.0:4317",
+        "http_endpoint": "0.0.0.0:4318"
+      }
+    }
+  }
+}

terraform/python/ec2/default/main.tf

@@ -145,7 +145,7 @@ resource "null_resource" "main_service_setup" {
       sudo yum install ec2-instance-connect -y
 
       # Copy in CW Agent configuration
-      agent_config='${replace(replace(file("./amazon-cloudwatch-agent.json"), "/\\s+/", ""), "$REGION", var.aws_region)}'
+      agent_config='${replace(replace(file(var.custom_metrics_enabled ? "./amazon-cloudwatch-custom-agent.json" : "./amazon-cloudwatch-agent.json"), "/\\s+/", ""), "$REGION", var.aws_region)}'
       echo $agent_config > amazon-cloudwatch-agent.json
 
       # Get and run CW agent rpm
@@ -170,7 +170,7 @@ resource "null_resource" "main_service_setup" {
       export DJANGO_SETTINGS_MODULE="django_frontend_service.settings"
       export OTEL_PYTHON_DISTRO="aws_distro"
       export OTEL_PYTHON_CONFIGURATOR="aws_configurator"
-      export OTEL_METRICS_EXPORTER=none
+      export OTEL_METRICS_EXPORTER=otlp
       export OTEL_TRACES_EXPORTER=otlp
       export OTEL_AWS_APPLICATION_SIGNALS_ENABLED=true
       export OTEL_AWS_APPLICATION_SIGNALS_EXPORTER_ENDPOINT=http://localhost:4315
@@ -179,6 +179,11 @@ resource "null_resource" "main_service_setup" {
       export OTEL_EXPORTER_OTLP_METRICS_PROTOCOL=grpc
       export OTEL_SERVICE_NAME=python-sample-application-${var.test_id}
       export OTEL_TRACES_SAMPLER=always_on
+      export OTEL_RESOURCE_ATTRIBUTES="service.name=python-sample-application-${var.test_id}",deployment.environment.name=${var.test_id}"
+      export OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=http://localhost:4317
+      export AWS_REGION='${var.aws_region}'
+      export CUSTOM_METRICS_CONFIG='${var.custom_metrics_config}'
+      export CUSTOM_METRICS_ENABLED='${var.custom_metrics_enabled}'
       python${var.language_version} manage.py migrate
       nohup opentelemetry-instrument python${var.language_version} manage.py runserver 0.0.0.0:8000 --noreload &
 
@@ -275,7 +280,7 @@ resource "null_resource" "remote_service_setup" {
       sudo yum install ec2-instance-connect -y
 
       # Copy in CW Agent configuration
-      agent_config='${replace(replace(file("./amazon-cloudwatch-agent.json"), "/\\s+/", ""), "$REGION", var.aws_region)}'
+      agent_config='${replace(replace(file(var.custom_metrics_enabled ? "./amazon-cloudwatch-custom-agent.json" : "./amazon-cloudwatch-agent.json"), "/\\s+/", ""), "$REGION", var.aws_region)}'
       echo $agent_config > amazon-cloudwatch-agent.json
 
       # Get and run CW agent rpm

terraform/python/ec2/default/variables.tf

@@ -47,4 +47,17 @@ variable "language_version" {
 
 variable "cpu_architecture" {
   default = "x86_64"
+}
+#Adding Custom Metrics Variables
+
+variable "custom_metrics_enabled" {
+  description = "Enable custom OTEL metrics in the sample application"
+  type        = bool
+  default     = false
+}
+
+variable "custom_metrics_config" {
+  description = "JSON configuration for custom metrics"
+  type        = string
+  default     = "amazon-cloudwatch-custom-agent.json"
 }

validator/src/main/java/com/amazon/aoc/fileconfigs/PredefinedExpectedTemplate.java

@@ -210,6 +210,14 @@ public enum PredefinedExpectedTemplate implements FileConfig {
   PYTHON_EC2_DEFAULT_CLIENT_CALL_METRIC("/expected-data-template/python/ec2/default/client-call-metric.mustache"),
   PYTHON_EC2_DEFAULT_CLIENT_CALL_TRACE("/expected-data-template/python/ec2/default/client-call-trace.mustache"),
 
+  /** Python EC2 Default Custom Metrics Test Case Validations */
+  PYTHON_EC2_DEFAULT_AWS_SDK_CALL_CUSTOM_METRIC("/expected-data-template/python/ec2/default/aws-sdk-call-custom-metric.mustache"),
+  PYTHON_EC2_DEFAULT_AWS_OTEL_CUSTOM_METRICS("/expected-data-template/python/ec2/default/aws-otel-custom-metrics.mustache"),
+  /**PYTHON_EC2_DEFAULT_OUTGOING_HTTP_CALL_CUSTOM_METRIC("/expected-data-template/python/ec2/default/outgoing-http-call-custom-metric.mustache"),
+    PYTHON_EC2_DEFAULT_REMOTE_SERVICE_CUSTOM_METRIC("/expected-data-template/python/ec2/default/remote-service-custom-metric.mustache"),
+    PYTHON_EC2_DEFAULT_CLIENT_CALL_CUSTOM_METRIC("/expected-data-template/python/ec2/default/client-call-custom-metric.mustache"),
+    PYTHON_EC2_DEFAULT_AWS_OTEL_CUSTOM_METRICS("/expected-data-template/python/ec2/default/aws-otel-custom-metrics.mustache"), */
+
   /** Python EC2 Asg Test Case Validations */
   PYTHON_EC2_ASG_OUTGOING_HTTP_CALL_LOG("/expected-data-template/python/ec2/asg/outgoing-http-call-log.mustache"),
   PYTHON_EC2_ASG_OUTGOING_HTTP_CALL_METRIC("/expected-data-template/python/ec2/asg/outgoing-http-call-metric.mustache"),

validator/src/main/resources/expected-data-template/python/ec2/default/aws-otel-custom-metrics.mustache

@@ -0,0 +1,26 @@
+# OpenTelemetry Custom Metrics Validation Templates - AWS SDK Call Only
+-
+  metricName: custom_requests_total
+  namespace: {{customMetricNamespace}}
+  dimensions:
+    -
+      name: operation.type
+      value: aws_sdk_call
+    -
+      name: Service
+      value: {{serviceName}}
+    -
+      name: Environment
+      value: {{testingId}}
+
+# Code for adding in Custom Histogram Metrics
+# -
+#   metricName: custom_response_time
+#   namespace: {{customMetricNamespace}}
+#   dimensions:
+#     -
+#       name: operation.type
+#       value: aws_sdk_call
+#     -
+#       name: Service
+#       value: {{serviceName}}