From d4aa755f956d1a6e4f2f364c991bf00792d3a2f1 Mon Sep 17 00:00:00 2001 From: Michi Date: Wed, 4 Jul 2018 20:58:23 +0200 Subject: [PATCH 01/51] adopted new extension framework rewrote to Azure Bindings Changed filter strategy --- pom.xml | 92 +++++---- .../appdynamics/monitors/azure/AzureAuth.java | 49 +++-- .../monitors/azure/AzureMonitor.java | 154 +++------------ .../monitors/azure/AzureMonitorTask.java | 171 +++++++--------- .../monitors/azure/MetricPrinter.java | 54 ----- .../monitors/azure/ServiceFabricTask.java | 92 +++++---- .../appdynamics/monitors/azure/Utilities.java | 119 ----------- .../azure/config/AuthenticationResults.java | 16 -- .../monitors/azure/config/Globals.java | 50 ----- .../monitors/azure/metrics/AzureMetrics.java | 185 ++++++++++++++++++ .../monitors/azure/utils/Constants.java | 17 ++ .../monitors/azure/utils/Utilities.java | 49 +++++ src/main/resources/conf/config.yml | 32 +-- .../monitors/azure/AzureMonitorTest.java | 134 +++---------- src/test/resources/json/metric.json | 24 --- .../resources/json/metricDefinitions.json | 21 -- src/test/resources/json/resources.json | 14 -- 17 files changed, 541 insertions(+), 732 deletions(-) delete mode 100644 src/main/java/com/appdynamics/monitors/azure/MetricPrinter.java delete mode 100644 src/main/java/com/appdynamics/monitors/azure/Utilities.java delete mode 100644 src/main/java/com/appdynamics/monitors/azure/config/AuthenticationResults.java delete mode 100644 src/main/java/com/appdynamics/monitors/azure/config/Globals.java create mode 100644 src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java create mode 100644 src/main/java/com/appdynamics/monitors/azure/utils/Constants.java create mode 100644 src/main/java/com/appdynamics/monitors/azure/utils/Utilities.java delete mode 100644 src/test/resources/json/metric.json delete mode 100644 src/test/resources/json/metricDefinitions.json delete mode 100644 src/test/resources/json/resources.json diff --git a/pom.xml b/pom.xml index 2028ec2..cc4d9a4 100644 --- a/pom.xml +++ b/pom.xml @@ -6,25 +6,46 @@ com.appdynamics.monitors azure-monitoring-extension - 1.1.9.3 + 2.0.0 + jar + http://maven.apache.org UTF-8 + yyyy-MM-dd HH:mm:ss + ${project.build.directory}/AzureMonitor + ${basedir}/lib - com.fasterxml.jackson.core - jackson-databind - 2.9.2 - - - org.yaml - snakeyaml - 1.19 + com.microsoft.azure + azure + 1.12.0 com.appdynamics appd-exts-commons - 1.6.6 + 2.1.0 + + + org.apache.httpcomponents + httpclient + 4.3.5 + + + commons-io + commons-io + 2.4 + + + commons-codec + commons-codec + 1.6 + + + org.mockito + mockito-all + 1.9.5 + test com.appdynamics @@ -33,41 +54,40 @@ provided - com.microsoft.azure - adal4j - 1.3.0 - - - commons-io - commons-io - 2.6 - test + log4j + log4j + 1.2.17 + provided commons-logging commons-logging - 1.2 - test + 1.1.3 + provided org.slf4j - slf4j-simple - 1.7.25 + jcl-over-slf4j + 1.7.12 + + + org.eclipse.jetty + jetty-webapp + 8.1.10.v20130312 test junit junit 4.12 - test - org.mockito - mockito-core - 2.11.0 - test + org.testng + testng + RELEASE + ${project.artifactId} @@ -76,8 +96,8 @@ maven-compiler-plugin 2.3.2 - 1.6 - 1.6 + 1.7 + 1.7 @@ -105,12 +125,11 @@ - - + + - AzureMonitor v${project.version} Build Date ${maven.build.timestamp} + Starter Monitor v${project.version} Build Date ${maven.build.timestamp} + com.appdynamics.extensions.workbench.WorkbenchServerLauncher @@ -177,7 +196,4 @@ https://github.com/Appdynamics/maven-repo/raw/master/releases - - scm:git:https://github.com/michaelenglert/azure-monitoring-extension - \ No newline at end of file diff --git a/src/main/java/com/appdynamics/monitors/azure/AzureAuth.java b/src/main/java/com/appdynamics/monitors/azure/AzureAuth.java index 7784fea..bd65300 100644 --- a/src/main/java/com/appdynamics/monitors/azure/AzureAuth.java +++ b/src/main/java/com/appdynamics/monitors/azure/AzureAuth.java @@ -8,13 +8,16 @@ package com.appdynamics.monitors.azure; -import com.appdynamics.monitors.azure.config.AuthenticationResults; -import com.appdynamics.monitors.azure.config.Globals; +import com.appdynamics.monitors.azure.utils.Constants; +import com.appdynamics.monitors.azure.utils.Utilities; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.base.Strings; import com.microsoft.aad.adal4j.AuthenticationContext; import com.microsoft.aad.adal4j.AuthenticationResult; import com.microsoft.aad.adal4j.ClientCredential; +import com.microsoft.azure.AzureEnvironment; +import com.microsoft.azure.credentials.ApplicationTokenCredentials; +import com.microsoft.azure.management.Azure; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.net.MalformedURLException; @@ -28,16 +31,16 @@ class AzureAuth { private static final Logger logger = LoggerFactory.getLogger(AzureAuth.class); - public static void getAzureAuth (Map config) { - String keyvaultClientSecretUrl = (String) config.get(Globals.keyvaultClientSecretUrl); - String keyvaultClientId = (String) config.get(Globals.keyvaultClientId); - String keyvaultClientKey = (String) config.get(Globals.keyvaultClientKey); - String clientId = (String) config.get(Globals.clientId); - String clientKey = (String) config.get(Globals.clientKey); - String tenantId = (String) config.get(Globals.tenantId); - String encryptionKey = (String) config.get(Globals.encryptionKey); - String encryptedClientKey = (String) config.get(Globals.encryptedClientKey); - String encryptedKeyvaultClientKey = (String) config.get(Globals.encryptedKeyvaultClientKey); + static void getAzureAuth(Map subscription) { + String keyvaultClientSecretUrl = (String) subscription.get("keyvaultClientSecretUrl"); + String keyvaultClientId = (String) subscription.get("keyvaultClientId"); + String keyvaultClientKey = (String) subscription.get("keyvaultClientKey"); + String clientId = (String) subscription.get("clientId"); + String clientKey = (String) subscription.get("clientKey"); + String tenantId = (String) subscription.get("tenantId"); + String encryptionKey = (String) subscription.get("encryption-key"); + String encryptedClientKey = (String) subscription.get("encryptedClientKey"); + String encryptedKeyvaultClientKey = (String) subscription.get("encryptedKeyvaultClientKey"); if (!Strings.isNullOrEmpty(encryptionKey) && !Strings.isNullOrEmpty(encryptedClientKey)) { clientKey = Utilities.getDecryptedKey(encryptedClientKey, encryptionKey); @@ -48,26 +51,32 @@ public static void getAzureAuth (Map config) { } if (!Strings.isNullOrEmpty(keyvaultClientId) && !Strings.isNullOrEmpty(keyvaultClientKey) && !Strings.isNullOrEmpty(keyvaultClientSecretUrl)){ - URL keyvaultUrl = Utilities.getUrl(keyvaultClientSecretUrl + "?" + Globals.azureApiVersion + - "=" + config.get("keyvault-api-version")); - AuthenticationResults.azureKeyVaultAuth = getAuthenticationResult(Globals.azureKeyvaultEndpoint,keyvaultClientId,keyvaultClientKey, tenantId); - JsonNode keyVaultResponse = AzureRestOperation.doGet(AuthenticationResults.azureKeyVaultAuth, keyvaultUrl); + URL keyvaultUrl = Utilities.getUrl(keyvaultClientSecretUrl + "?" + "api-version" + + "=" + subscription.get("keyvault-api-version")); + AuthenticationResult azureKeyVaultAuth = getAuthenticationResult(keyvaultClientId, keyvaultClientKey, tenantId); + JsonNode keyVaultResponse = AzureRestOperation.doGet(azureKeyVaultAuth, keyvaultUrl); assert keyVaultResponse != null; clientKey = keyVaultResponse.get("value").textValue(); } - AuthenticationResults.azureMonitorAuth = getAuthenticationResult(Globals.azureEndpoint + "/", clientId, clientKey, tenantId); + + ApplicationTokenCredentials applicationTokenCredentials = new ApplicationTokenCredentials( + clientId, + tenantId, + clientKey, + AzureEnvironment.AZURE); + Constants.azureMonitorAuth = Azure.authenticate(applicationTokenCredentials); } - private static AuthenticationResult getAuthenticationResult(String endpoint, String Id, String Key, String tenantId){ + private static AuthenticationResult getAuthenticationResult(String Id, String Key, String tenantId){ ExecutorService service = Executors.newSingleThreadExecutor(); AuthenticationResult result = null; - String authority = Globals.azureAuthEndpoint + tenantId; + String authority = "https://login.microsoftonline.com/" + tenantId; try { AuthenticationContext context; context = new AuthenticationContext(authority, false, service); ClientCredential cred = new ClientCredential(Id, Key); - Future future = context.acquireToken(endpoint, cred, null); + Future future = context.acquireToken("https://vault.azure.net", cred, null); result = future.get(); } catch (MalformedURLException e) { logger.error("Not a valid Azure authentication Authority {}", authority, e); diff --git a/src/main/java/com/appdynamics/monitors/azure/AzureMonitor.java b/src/main/java/com/appdynamics/monitors/azure/AzureMonitor.java index 78156b4..27d98d2 100644 --- a/src/main/java/com/appdynamics/monitors/azure/AzureMonitor.java +++ b/src/main/java/com/appdynamics/monitors/azure/AzureMonitor.java @@ -8,139 +8,43 @@ package com.appdynamics.monitors.azure; -import com.appdynamics.extensions.conf.MonitorConfiguration; -import com.appdynamics.extensions.util.MetricWriteHelper; -import com.appdynamics.extensions.util.MetricWriteHelperFactory; -import com.appdynamics.extensions.conf.MonitorConfiguration.ConfItem; -import com.appdynamics.monitors.azure.config.AuthenticationResults; -import com.appdynamics.monitors.azure.config.Globals; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.singularity.ee.agent.systemagent.api.AManagedMonitor; -import com.singularity.ee.agent.systemagent.api.TaskExecutionContext; -import com.singularity.ee.agent.systemagent.api.TaskOutput; -import com.singularity.ee.agent.systemagent.api.exception.TaskExecutionException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.net.URL; -import java.util.ArrayList; +import com.appdynamics.extensions.ABaseMonitor; +import com.appdynamics.extensions.TasksExecutionServiceProvider; +import com.appdynamics.extensions.util.AssertUtils; +import com.appdynamics.monitors.azure.utils.Constants; +import java.util.List; import java.util.Map; +import java.util.concurrent.CountDownLatch; -@SuppressWarnings("WeakerAccess") -public class AzureMonitor extends AManagedMonitor { - private static final Logger logger = LoggerFactory.getLogger(AzureMonitor.class); - private MonitorConfiguration configuration; +@SuppressWarnings({"WeakerAccess", "unchecked"}) +public class AzureMonitor extends ABaseMonitor { - public AzureMonitor() { logger.info(String.format("Using Azure Monitor Version [%s]", getImplementationVersion())); } - - private void initialize(Map argsMap) { - if (configuration == null) { - MetricWriteHelper metricWriteHelper = MetricWriteHelperFactory.create(this); - MonitorConfiguration conf = new MonitorConfiguration(Globals.defaultMetricPrefix, - new TaskRunnable(), metricWriteHelper); - final String configFilePath = argsMap.get(Globals.configFile); - conf.setConfigYml(configFilePath); - conf.setMetricWriter(MetricWriteHelperFactory.create(this)); - conf.checkIfInitialized(ConfItem.CONFIG_YML, - ConfItem.EXECUTOR_SERVICE, - ConfItem.METRIC_PREFIX, - ConfItem.METRIC_WRITE_HELPER); - this.configuration = conf; - } + @Override + protected String getDefaultMetricPrefix() { + return Constants.DEFAULT_METRIC_PREFIX; } - public TaskOutput execute(Map map, TaskExecutionContext taskExecutionContext) throws TaskExecutionException { - try{ - if(map != null){ - if (logger.isDebugEnabled()) {logger.debug("The raw arguments are {}", map);} - initialize(map); - configuration.executeTask(); - return new TaskOutput("Azure Monitor Metric Upload Complete"); - } - } - catch(Exception e) { - logger.error("Failed to execute the Azure monitoring task", e); - } - throw new TaskExecutionException("Azure monitoring task completed with failures."); - + @Override + public String getMonitorName() { + return "Azure Monitoring Extension"; } - private class TaskRunnable implements Runnable { - public void run () { - Map config = configuration.getConfigYml(); - if (config != null ) { - AzureAuth.getAzureAuth(config); - JsonNode filtersJson = Utilities.getFiltersJson((ArrayList) config.get(Globals.azureApiFilter)); - String filterUrl = Utilities.getFilterUrl(filtersJson); - Map resourceFilter = Utilities.getResourceFilter(filtersJson); - URL resourcesUrl = Utilities.getUrl(Globals.azureEndpoint + - Globals.azureApiSubscriptions + - config.get(Globals.subscriptionId) + - Globals.azureApiResources + - "?" + Globals.azureApiVersion + - "=" + config.get(Globals.azureApiVersion) + - filterUrl); - JsonNode resourcesResponse = AzureRestOperation.doGet(AuthenticationResults.azureMonitorAuth,resourcesUrl); - if (logger.isDebugEnabled()) { - logger.debug("Get Resources REST API Request: " + resourcesUrl.toString()); - logger.debug("Get Resources Response JSON: " + Utilities.prettifyJson(resourcesResponse)); - } - assert resourcesResponse != null; - ArrayNode resourceElements = (ArrayNode) resourcesResponse.get("value"); - for(JsonNode resourceNode:resourceElements){ - if (resourceNode.get("id").asText().contains("Microsoft.ServiceFabric/clusters")){ - JsonNode serviceFabricResponse = AzureRestOperation.doGet(AuthenticationResults.azureMonitorAuth, Utilities.getUrl( - Globals.azureEndpoint + - resourceNode.get("id").asText() + - "?" + Globals.azureApiVersion + - "=" + config.get(Globals.serviceFabricResourceApiVersion))); - assert serviceFabricResponse != null; - ServiceFabricTask fabricTask = new ServiceFabricTask( - configuration, - serviceFabricResponse, - serviceFabricResponse.get("name").asText()); - configuration.getExecutorService().execute(fabricTask); - } - URL metricDefinitions = Utilities.getUrl( - Globals.azureEndpoint + - resourceNode.get("id").asText() + - Globals.azureApiMetricDefinitions + - "?" + Globals.azureApiVersion + - "=" + config.get(Globals.azureMonitorApiVersion)); - JsonNode metricDefinitionResponse = AzureRestOperation.doGet(AuthenticationResults.azureMonitorAuth,metricDefinitions); - if (logger.isDebugEnabled()) { - logger.debug("Get Metric Definitions REST API Request: " - + metricDefinitions.toString()); - logger.debug("Get Metric Definitions Response JSON: " - + Utilities.prettifyJson(metricDefinitionResponse)); - } - assert metricDefinitionResponse != null; - ArrayNode metricDefinitionElements = (ArrayNode) metricDefinitionResponse.get("value"); - for(JsonNode metricDefinitionNode:metricDefinitionElements){ - if (metricDefinitionNode.get("isDimensionRequired").asText().equals("true")){ - logger.info("Dimensions are currently not supported. Skipping " - + metricDefinitionNode.get("id").asText()); - } - else if (Utilities.checkResourceFilter(metricDefinitionNode,resourceFilter)){ - logger.info("Ignoring Metric " + - metricDefinitionNode.get("name").get("value").asText() + - " for Resource " + metricDefinitionNode.get("resourceId")); - } - else { - AzureMonitorTask monitorTask = new AzureMonitorTask( - configuration, - resourceNode, - AuthenticationResults.azureMonitorAuth, - metricDefinitionNode.get("name").get("value").asText()); - configuration.getExecutorService().execute(monitorTask); - } - } - } - logger.info("Finished gathering Metrics"); - } - else { logger.error("The config.yml is not loaded due to previous errors.The task will not run"); } + @Override + protected void doRun(TasksExecutionServiceProvider tasksExecutionServiceProvider) { + List> subscriptions = (List>)getContextConfiguration().getConfigYml().get("subscriptions"); + AssertUtils.assertNotNull(subscriptions, "The 'subscriptions' section in config.yml is not initialised"); + for (Map subscription : subscriptions) { + AzureMonitorTask task = new AzureMonitorTask(getContextConfiguration(), tasksExecutionServiceProvider.getMetricWriteHelper(), subscription, new CountDownLatch(getTaskCount())); + tasksExecutionServiceProvider.submit(subscription.get("subscriptionId").toString(),task); } } - private static String getImplementationVersion() { return AzureMonitor.class.getPackage().getImplementationTitle(); } + @Override + protected int getTaskCount() { + List> subscriptions = (List>)getContextConfiguration().getConfigYml().get("subscriptions"); + List> serviceFabrics = (List>)getContextConfiguration().getConfigYml().get("serviceFabrics"); + AssertUtils.assertNotNull(subscriptions, "The 'subscriptions' section in config.yml is not initialised"); + AssertUtils.assertNotNull(serviceFabrics, "The 'serviceFabrics' section in config.yml is not initialised"); + return subscriptions.size() + serviceFabrics.size(); + } } diff --git a/src/main/java/com/appdynamics/monitors/azure/AzureMonitorTask.java b/src/main/java/com/appdynamics/monitors/azure/AzureMonitorTask.java index c5955c5..3443314 100644 --- a/src/main/java/com/appdynamics/monitors/azure/AzureMonitorTask.java +++ b/src/main/java/com/appdynamics/monitors/azure/AzureMonitorTask.java @@ -8,120 +8,99 @@ package com.appdynamics.monitors.azure; -import com.appdynamics.extensions.conf.MonitorConfiguration; -import com.appdynamics.monitors.azure.config.Globals; -import com.fasterxml.jackson.databind.JsonNode; -import com.microsoft.aad.adal4j.AuthenticationResult; -import org.slf4j.Logger; +import com.appdynamics.extensions.AMonitorTaskRunnable; +import com.appdynamics.extensions.MetricWriteHelper; +import com.appdynamics.extensions.conf.MonitorContextConfiguration; +import com.appdynamics.extensions.util.AssertUtils; +import com.appdynamics.monitors.azure.metrics.AzureMetrics; +import com.appdynamics.monitors.azure.utils.Constants; +import com.microsoft.azure.PagedList; +import com.microsoft.azure.management.Azure; +import com.microsoft.azure.management.resources.GenericResource; import org.slf4j.LoggerFactory; -import java.io.UnsupportedEncodingException; -import java.math.BigDecimal; -import java.net.URL; -import java.net.URLEncoder; -import java.text.SimpleDateFormat; -import java.util.Calendar; -import java.util.TimeZone; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; -class AzureMonitorTask implements Runnable { - private static final Logger logger = LoggerFactory.getLogger(AzureMonitorTask.class); - private final MonitorConfiguration configuration; - private final JsonNode node; - private final AuthenticationResult azureAuth; - private final String metric; +@SuppressWarnings("unchecked") +class AzureMonitorTask implements AMonitorTaskRunnable{ + private static final org.slf4j.Logger logger = LoggerFactory.getLogger(AzureMonitorTask.class); + private final MonitorContextConfiguration configuration; + private final MetricWriteHelper metricWriteHelper; + private final Map subscription; + private final List> resourceGroupFilters; + private final CountDownLatch countDownLatch; - AzureMonitorTask(MonitorConfiguration configuration, JsonNode node, AuthenticationResult azureAuth, String metric) { - this.configuration = configuration; - this.node = node; - this.azureAuth = azureAuth; - this.metric = metric; + AzureMonitorTask(MonitorContextConfiguration configuration, MetricWriteHelper metricWriteHelper, Map subscription, CountDownLatch countDownLatch) { + this.configuration = configuration; + this.metricWriteHelper = metricWriteHelper; + this.subscription = subscription; + this.countDownLatch = countDownLatch; + resourceGroupFilters = (List>) subscription.get("resourceGroups"); + AssertUtils.assertNotNull(resourceGroupFilters, "The 'resourceGroupFilters' section in config.yml is either null or empty"); } + @Override + public void onTaskComplete() { + logger.info("Task Complete"); + } + + @Override public void run() { try { runTask(); - } catch (Exception e) { - configuration.getMetricWriter().registerError(e.getMessage(), e); - logger.error("Error while running the task", e); + } + catch(Exception e){ + logger.error(e.getMessage()); + } + finally { + countDownLatch.countDown(); } } + private void runTask(){ + AzureAuth.getAzureAuth(subscription); + Azure azure = Constants.azureMonitorAuth.withSubscription(subscription.get("subscriptionId").toString()); + String subscriptionName; + PagedList resources = azure.genericResources().list(); + CountDownLatch countDownLatchAzure = new CountDownLatch(resources.size()); - private void runTask() { - TimeZone utc = TimeZone.getTimeZone("UTC"); - Calendar endTime = Calendar.getInstance(); - Calendar startTime = Calendar.getInstance(); - startTime.add(Calendar.MINUTE, Globals.timeOffset); - SimpleDateFormat dateFormatter = new SimpleDateFormat(Globals.azureApiTimeFormat); - dateFormatter.setTimeZone(utc); - if (logger.isDebugEnabled()) { - logger.debug("JSON Node: " + Utilities.prettifyJson(node)); + if (subscription.containsKey("subscriptionName")){ + subscriptionName = subscription.get("subscriptionName").toString(); } - URL url = null; - try { - url = Utilities.getUrl( - Globals.azureEndpoint + - node.get("id").asText() + - Globals.azureApiMetrics + - "?" + Globals.azureApiVersion + - "=" + configuration.getConfigYml().get(Globals.azureMonitorApiVersion) + - "&" + Globals.azureApiTimeSpan + - "=" + dateFormatter.format(startTime.getTime()) + - "/" + dateFormatter.format(endTime.getTime()) + - "&" + Globals.metric + - "=" + URLEncoder.encode(metric,Globals.urlEncoding)); - } catch (UnsupportedEncodingException e) { - logger.error("Failed to encode Metric {} with {}", metric, Globals.urlEncoding, e); + else { + subscriptionName = subscription.get("subscriptionId").toString(); } - if (logger.isDebugEnabled()) { - assert url != null; - logger.debug("Get Metrics REST API Request: " + url.toString());} - assert url != null; - extractMetrics(AzureRestOperation.doGet(azureAuth,url)); - } - private void extractMetrics(JsonNode json){ - if (logger.isDebugEnabled()) {logger.debug("Get Metrics Response JSON: " + Utilities.prettifyJson(json));} - JsonNode jsonValue = json.get("value"); - for (JsonNode currentValueNode:jsonValue){ - String metricId = extractMetridId(currentValueNode.get("id").asText()); - String metricNameValue = currentValueNode.get("name").get("value").asText(); - String metricUnit = currentValueNode.get("unit").asText(); - String metricType = null; - BigDecimal metricValue = null; - if(currentValueNode.get("timeseries").has(0)){ - JsonNode jsonData = currentValueNode.get("timeseries").get(0).get("data"); - for (JsonNode currentDataNode:jsonData){ - if (currentDataNode.has("average")){ - metricType = "average"; metricValue = currentDataNode.get("average").decimalValue(); - } - else if (currentDataNode.has("total")){ - metricType = "total"; metricValue = currentDataNode.get("total").decimalValue(); - } - else if (currentDataNode.has("last")){ - metricType = "last"; metricValue = currentDataNode.get("last").decimalValue(); - } - else if (currentDataNode.has("maximum")){ - metricType = "maximum"; metricValue = currentDataNode.get("maximum").decimalValue(); + for (Map resourceGroupFilter : resourceGroupFilters) { + List> resourceTypeFilters = (List>) resourceGroupFilter.get("resourceTypes"); + for (Map resourceTypeFilter : resourceTypeFilters) { + List> resourceFilters = (List>) resourceTypeFilter.get("resources"); + for (Map resourceFilter : resourceFilters) { + String currentResourceGroupFilter = resourceGroupFilter.get("resourceGroup").toString(); + for (GenericResource resource : resources) { + String currentResourceFilter = resourceFilter.get("resource").toString(); + String currentResourceTypeFilter = resourceTypeFilter.get("resourceType").toString(); + AzureMetrics azureMetricsTask = new AzureMetrics( + resourceFilter, + currentResourceGroupFilter, + currentResourceFilter, + currentResourceTypeFilter, + resource, + subscriptionName, + countDownLatchAzure, + metricWriteHelper, + azure, + configuration.getMetricPrefix()); + configuration.getContext().getExecutorService().execute("AzureMetrics", azureMetricsTask); } } - if (metricId != null && - metricNameValue != null && - metricType != null && - metricUnit != null && - metricValue != null){ - MetricPrinter metricPrinter = new MetricPrinter(configuration.getMetricWriter()); - metricPrinter.reportMetric(configuration.getMetricPrefix() + - metricId + - metricNameValue, metricValue); - } } } - } - private static String extractMetridId(String fullId){ - String metricId; - String[] metricIdSegments = fullId.split("resourceGroups")[1].split("providers/Microsoft\\."); - metricId = metricIdSegments[0]+metricIdSegments[1]; - metricId = metricId.replace("/","|"); - return metricId; + try{ + countDownLatchAzure.await(); + } catch (InterruptedException e) { + e.printStackTrace(); + } } } \ No newline at end of file diff --git a/src/main/java/com/appdynamics/monitors/azure/MetricPrinter.java b/src/main/java/com/appdynamics/monitors/azure/MetricPrinter.java deleted file mode 100644 index 1a6fcfd..0000000 --- a/src/main/java/com/appdynamics/monitors/azure/MetricPrinter.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2017. AppDynamics LLC and its affiliates. - * All Rights Reserved. - * This is unpublished proprietary source code of AppDynamics LLC and its affiliates. - * The copyright notice above does not evidence any actual or intended publication of such source code. - * - */ - -package com.appdynamics.monitors.azure; - -import com.appdynamics.extensions.util.MetricWriteHelper; -import com.singularity.ee.agent.systemagent.api.MetricWriter; -import org.slf4j.LoggerFactory; -import java.math.BigDecimal; -import java.math.RoundingMode; - -@SuppressWarnings("SameParameterValue") -class MetricPrinter { - private static final org.slf4j.Logger logger = LoggerFactory.getLogger(MetricPrinter.class); - private final MetricWriteHelper metricWriter; - - MetricPrinter(MetricWriteHelper metricWriter){ - this.metricWriter = metricWriter; - } - - void reportMetric(String metricName, BigDecimal metricValue) { - if(metricValue == null){ - return; - } - printMetric(metricName, - metricValue, - MetricWriter.METRIC_AGGREGATION_TYPE_AVERAGE, - MetricWriter.METRIC_TIME_ROLLUP_TYPE_AVERAGE, - MetricWriter.METRIC_CLUSTER_ROLLUP_TYPE_INDIVIDUAL); - } - - private void printMetric(String metricPath, BigDecimal metricValue, String aggType, String timeRollupType, String clusterRollupType) { - - try{ - String metricValStr = toBigIntString(metricValue); - if(metricValStr != null) { - metricWriter.printMetric(metricPath,metricValStr,aggType,timeRollupType,clusterRollupType); - logger.debug("Sending [{}|{}|{}] metric= {},value={}", aggType, timeRollupType, clusterRollupType, metricPath, metricValStr); - } - } - catch (Exception e){ - logger.error("Error reporting metric {} with value {}",metricPath,metricValue,e); - } - } - - private String toBigIntString(BigDecimal metricValue) { - return metricValue.setScale(0, RoundingMode.HALF_UP).toBigInteger().toString(); - } -} diff --git a/src/main/java/com/appdynamics/monitors/azure/ServiceFabricTask.java b/src/main/java/com/appdynamics/monitors/azure/ServiceFabricTask.java index 3395fbf..41286ca 100644 --- a/src/main/java/com/appdynamics/monitors/azure/ServiceFabricTask.java +++ b/src/main/java/com/appdynamics/monitors/azure/ServiceFabricTask.java @@ -8,89 +8,107 @@ package com.appdynamics.monitors.azure; -import com.appdynamics.extensions.conf.MonitorConfiguration; -import com.appdynamics.monitors.azure.config.Globals; +import com.appdynamics.extensions.AMonitorTaskRunnable; +import com.appdynamics.extensions.MetricWriteHelper; +import com.appdynamics.extensions.conf.MonitorContextConfiguration; +import com.appdynamics.extensions.metrics.Metric; +import com.appdynamics.monitors.azure.utils.Constants; import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.Lists; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.math.BigDecimal; import java.net.URL; import java.util.List; import java.util.Map; +import java.util.Objects; -class ServiceFabricTask implements Runnable { +class ServiceFabricTask implements AMonitorTaskRunnable { private static final Logger logger = LoggerFactory.getLogger(ServiceFabricTask.class); - private final MonitorConfiguration configuration; - private final JsonNode node; - private final String metric; + private final MonitorContextConfiguration configuration; + private final MetricWriteHelper metricWriteHelper; + private final Map serviceFabric; + private final String managementEndpoint; + private final String metricName; + private List finalMetricList; - ServiceFabricTask(MonitorConfiguration configuration, JsonNode node, String metric) { + private ServiceFabricTask(MonitorContextConfiguration configuration, MetricWriteHelper metricWriteHelper, Map serviceFabric, String managementEndpoint, String metricName) { this.configuration = configuration; - this.node = node; - this.metric = metric; + this.metricWriteHelper = metricWriteHelper; + this.serviceFabric = serviceFabric; + this.managementEndpoint = managementEndpoint; + this.metricName = metricName; } + @Override + public void onTaskComplete() { + logger.info("Task Complete"); + } + + @Override public void run() { try { runTask(); } catch (Exception e) { - configuration.getMetricWriter().registerError(e.getMessage(), e); + metricWriteHelper.registerError(e.getMessage(), e); logger.error("Error while running the task", e); } } private void runTask() throws IOException { - String serviceFabricBody = configuration.getConfigYml().get(Globals.serviceFabricBody).toString(); - String serviceFabricCert = configuration.getConfigYml().get(Globals.serviceFabricCert).toString(); - String serviceFabricPassphrase = configuration.getConfigYml().get(Globals.serviceFabricPassphrase).toString(); + String serviceFabricBody = serviceFabric.get("serviceFabricBody").toString(); + String serviceFabricCert = serviceFabric.get("serviceFabricCert").toString(); + String serviceFabricPassphrase = serviceFabric.get("serviceFabricPassphrase").toString(); + finalMetricList = Lists.newArrayList(); - if (logger.isDebugEnabled()) {logger.debug("JSON Node: " + Utilities.prettifyJson(node));} - URL url = new URL(node.get("properties").get("managementEndpoint").asText() + Globals.serviceFabricGetClusterHealthChunk + - "?" + Globals.azureApiVersion + "=" + configuration.getConfigYml().get(Globals.serviceFabricApiVersion)); + URL url = new URL(managementEndpoint + "/$/GetClusterHealthChunk" + + "?api-version=" + serviceFabric.get("serviceFabricApiVersion")); if (logger.isDebugEnabled()) {logger.debug("Get Metrics REST API Request: " + url.toString());} if (url.toString().matches("https://.*") && !serviceFabricCert.isEmpty()){ - //logger.info("Skipping Service Fabric Cluster {} because the Authentication Method is currently not supported", - // node.get("properties").get("managementEndpoint").asText()); extractMetrics(AzureRestOperation.doSecurePost(url, serviceFabricBody, serviceFabricCert, serviceFabricPassphrase)); } else { - extractMetrics(AzureRestOperation.doPost(url, serviceFabricBody)); + extractMetrics(Objects.requireNonNull(AzureRestOperation.doPost(url, serviceFabricBody))); } } private void extractMetrics(JsonNode json){ - if (logger.isDebugEnabled()) {logger.debug("Get Metrics Response JSON: " + Utilities.prettifyJson(json));} - List healtStateList = (List) configuration.getConfigYml().get(Globals.serviceFabricHealthStates); + List healtStateList = (List) serviceFabric.get("serviceFabricHealthStates"); + Metric metric; Map healtStates = (Map) healtStateList.get(0); - String metricName = configuration.getMetricPrefix() + "|" + metric + "|"; - MetricPrinter metricPrinter = new MetricPrinter(configuration.getMetricWriter()); - metricPrinter.reportMetric(metricName + - "ClusterHealth", - BigDecimal.valueOf((Integer) healtStates.get(json.get("HealthState").asText()))); - + String metricPath = configuration.getMetricPrefix() + Constants.METRIC_SEPARATOR + metricName + Constants.METRIC_SEPARATOR; + metric = new Metric("ClusterHealth", healtStates.get(json.get("HealthState").asText()).toString(), metricPath); + finalMetricList.add(metric); JsonNode nodes = json.get("NodeHealthStateChunks").get("Items"); for (JsonNode node:nodes){ - metricPrinter.reportMetric(metricName + + metric = new Metric(node.get("NodeName").asText(), healtStates.get(node.get("HealthState").asText()).toString(), metricPath + "NodeHealth" + - "|" + node.get("NodeName").asText(), - BigDecimal.valueOf((Integer) healtStates.get(node.get("HealthState").asText()))); + Constants.METRIC_SEPARATOR); + finalMetricList.add(metric); } JsonNode apps = json.get("ApplicationHealthStateChunks").get("Items"); for (JsonNode app:apps){ if (app.get("ApplicationTypeName").asText().isEmpty()){ - metricPrinter.reportMetric(metricName + + metric = new Metric("System", healtStates.get(app.get("HealthState").asText()).toString(), metricPath + "ApplicationHealth" + - "|" + "System", - BigDecimal.valueOf((Integer) healtStates.get(app.get("HealthState").asText()))); + Constants.METRIC_SEPARATOR); + finalMetricList.add(metric); } else { - metricPrinter.reportMetric(metricName + + metric = new Metric(app.get("ApplicationTypeName").asText(), healtStates.get(app.get("HealthState").asText()).toString(), metricPath + "ApplicationHealth" + - "|" + app.get("ApplicationTypeName").asText(), - BigDecimal.valueOf((Integer) healtStates.get(app.get("HealthState").asText()))); + Constants.METRIC_SEPARATOR); + finalMetricList.add(metric); } } + if (finalMetricList.isEmpty()){ + if (logger.isDebugEnabled()) { + logger.debug("Metric List is empty"); + } + } + else { + metricWriteHelper.transformAndPrintMetrics(finalMetricList); + } } } \ No newline at end of file diff --git a/src/main/java/com/appdynamics/monitors/azure/Utilities.java b/src/main/java/com/appdynamics/monitors/azure/Utilities.java deleted file mode 100644 index 33f7bd2..0000000 --- a/src/main/java/com/appdynamics/monitors/azure/Utilities.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Copyright 2017. AppDynamics LLC and its affiliates. - * All Rights Reserved. - * This is unpublished proprietary source code of AppDynamics LLC and its affiliates. - * The copyright notice above does not evidence any actual or intended publication of such source code. - * - */ - -package com.appdynamics.monitors.azure; - -import com.appdynamics.extensions.crypto.CryptoUtil; -import com.appdynamics.monitors.azure.config.Globals; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.Maps; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.net.MalformedURLException; -import java.net.URL; -import java.net.URLEncoder; -import java.util.*; - -class Utilities { - private static final Logger logger = LoggerFactory.getLogger(Utilities.class); - - static JsonNode getFiltersJson(ArrayList filters){ - ObjectMapper objectMapper = new ObjectMapper(); - String jsonInString; - JsonNode filtersJson = null; - try { - jsonInString = objectMapper.writeValueAsString(filters); - filtersJson = objectMapper.readTree(jsonInString); - } catch (JsonProcessingException e) { - e.printStackTrace(); - } catch (IOException e) { - e.printStackTrace(); - } - return filtersJson; - } - - static String getFilterUrl(JsonNode filtersJson){ - StringBuilder filterUrl = null; - Iterator iter = filtersJson.iterator(); - JsonNode currentValueNode; - if (!filtersJson.isNull()) { - filterUrl = new StringBuilder("&$" + Globals.azureApiFilter + "="); - while (iter.hasNext()) { - currentValueNode = iter.next(); - try { - filterUrl.append(URLEncoder.encode(Globals.filterBy + - Globals.filterComOp + "'" + - currentValueNode.get(Globals.filterBy).asText() + - "'", Globals.urlEncoding)); - if (iter.hasNext()) { - filterUrl.append(URLEncoder.encode(Globals.filterLogOp, Globals.urlEncoding)); - } - } catch (UnsupportedEncodingException e) { - e.printStackTrace(); - } - - } - } - return filterUrl != null ? filterUrl.toString() : null; - } - - static Map getResourceFilter(JsonNode filtersJson) { - Map resourceFilter = new HashMap(); - for (JsonNode currentValueNode:filtersJson){ - if(currentValueNode.has("exclude")){ - if (!currentValueNode.get("exclude").asText().isEmpty()){ - resourceFilter.put(currentValueNode.get(Globals.filterBy).asText(),currentValueNode.get("exclude").asText()); - } - } - } - return resourceFilter; - } - - static Boolean checkResourceFilter(JsonNode jsonNode, Map resourceFilter){ - for(Map.Entry entry : resourceFilter.entrySet()){ - if(jsonNode.get("resourceId").asText().contains(entry.getKey())){ - if (jsonNode.get("name").get("value").asText().matches(entry.getValue())){ - return true; - } - } - } - return false; - } - - static String getDecryptedKey(String encryptedKey, String encryptionKey){ - java.util.Map cryptoMap = Maps.newHashMap(); - cryptoMap.put(Globals.passwordEncrypted, encryptedKey); - cryptoMap.put(Globals.encryptionKey, encryptionKey); - return CryptoUtil.getPassword(cryptoMap); - } - - static URL getUrl(String input){ - URL url = null; - try { - url = new URL(input); - } catch (MalformedURLException e) { - logger.error("Error forming our from String {}", input, e); - } - return url; - } - - static String prettifyJson(JsonNode json) { - ObjectMapper mapper = new ObjectMapper(); - try { - return mapper.writerWithDefaultPrettyPrinter().writeValueAsString(json); - } catch (JsonProcessingException e) { - logger.error("Can not process JSON {}", json.asText(), e); - } - return null; - } -} diff --git a/src/main/java/com/appdynamics/monitors/azure/config/AuthenticationResults.java b/src/main/java/com/appdynamics/monitors/azure/config/AuthenticationResults.java deleted file mode 100644 index 98692e6..0000000 --- a/src/main/java/com/appdynamics/monitors/azure/config/AuthenticationResults.java +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright 2017. AppDynamics LLC and its affiliates. - * All Rights Reserved. - * This is unpublished proprietary source code of AppDynamics LLC and its affiliates. - * The copyright notice above does not evidence any actual or intended publication of such source code. - * - */ - -package com.appdynamics.monitors.azure.config; - -import com.microsoft.aad.adal4j.AuthenticationResult; - -public class AuthenticationResults { - public static AuthenticationResult azureMonitorAuth = null; - public static AuthenticationResult azureKeyVaultAuth = null; -} diff --git a/src/main/java/com/appdynamics/monitors/azure/config/Globals.java b/src/main/java/com/appdynamics/monitors/azure/config/Globals.java deleted file mode 100644 index 73c5298..0000000 --- a/src/main/java/com/appdynamics/monitors/azure/config/Globals.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright 2017. AppDynamics LLC and its affiliates. - * All Rights Reserved. - * This is unpublished proprietary source code of AppDynamics LLC and its affiliates. - * The copyright notice above does not evidence any actual or intended publication of such source code. - * - */ - -package com.appdynamics.monitors.azure.config; - -public class Globals { - public static final String azureAuthEndpoint = "https://login.microsoftonline.com/"; - public static final String azureKeyvaultEndpoint = "https://vault.azure.net"; - public static final String azureEndpoint = "https://management.azure.com"; - public static final String azureApiVersion = "api-version"; - public static final String azureMonitorApiVersion = "monitor-api-version"; - public static final String defaultMetricPrefix = "Custom Metrics|AzureMonitor|"; - public static final String azureApiMetrics = "/providers/microsoft.insights/metrics"; - public static final String azureApiMetricDefinitions = "/providers/microsoft.insights/metricDefinitions"; - public static final String azureApiSubscriptions = "/subscriptions/"; - public static final String azureApiResources = "/resources"; - public static final String azureApiFilter = "filter"; - public static final String azureApiTimeSpan = "timespan"; - public static final String azureApiTimeFormat = "yyyy-MM-dd'T'HH:mm:ss'Z'"; - public static final String filterBy = "resourceType"; - public static final String filterLogOp = " or "; - public static final String filterComOp = " eq "; - public static final String urlEncoding = "UTF-8"; - public static final String configFile = "config-file"; - public static final String clientId = "clientId"; - public static final String tenantId = "tenantId"; - public static final String metric = "metric"; - public static final String clientKey = "clientKey"; - public static final String keyvaultClientId = "keyvaultClientId"; - public static final String keyvaultClientKey = "keyvaultClientKey"; - public static final String keyvaultClientSecretUrl = "keyvaultClientSecretUrl"; - public static final String subscriptionId = "subscriptionId"; - public static final String encryptionKey = "encryption-key"; - public static final String encryptedClientKey = "encryptedClientKey"; - public static final String encryptedKeyvaultClientKey = "encryptedKeyvaultClientKey"; - public static final String passwordEncrypted = "password-encrypted"; - public static final int timeOffset = -2; - public static final String serviceFabricGetClusterHealthChunk = "/$/GetClusterHealthChunk"; - public static final String serviceFabricBody = "serviceFabricBody"; - public static final String serviceFabricCert = "serviceFabricCert"; - public static final String serviceFabricPassphrase = "serviceFabricPassphrase"; - public static final String serviceFabricApiVersion = "serviceFabricApiVersion"; - public static final String serviceFabricResourceApiVersion = "serviceFabricResourceApiVersion"; - public static final String serviceFabricHealthStates = "serviceFabricHealthStates"; -} diff --git a/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java b/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java new file mode 100644 index 0000000..3ec1ece --- /dev/null +++ b/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java @@ -0,0 +1,185 @@ +/* + * Copyright 2017. AppDynamics LLC and its affiliates. + * All Rights Reserved. + * This is unpublished proprietary source code of AppDynamics LLC and its affiliates. + * The copyright notice above does not evidence any actual or intended publication of such source code. + * + */ + +package com.appdynamics.monitors.azure.metrics; + +import com.appdynamics.extensions.AMonitorTaskRunnable; +import com.appdynamics.extensions.MetricWriteHelper; +import com.appdynamics.extensions.metrics.Metric; +import com.appdynamics.monitors.azure.utils.Constants; +import com.google.common.collect.Lists; +import com.microsoft.azure.management.Azure; +import com.microsoft.azure.management.monitor.MetricCollection; +import com.microsoft.azure.management.monitor.MetricDefinition; +import com.microsoft.azure.management.resources.GenericResource; +import org.joda.time.DateTime; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; + +@SuppressWarnings("unchecked") +public class AzureMetrics implements AMonitorTaskRunnable { + private static final org.slf4j.Logger logger = LoggerFactory.getLogger(AzureMetrics.class); + private final Map resourceFilter; + private final String currentResourceGroupFilter; + private final String currentResourceFilter; + private final String currentResourceTypeFilter; + private final GenericResource resource; + private final String subscriptionName; + private final CountDownLatch countDownLatch; + private final MetricWriteHelper metricWriteHelper; + private final Azure azure; + private final String metricPrefix; + + public AzureMetrics(Map resourceFilter, + String currentResourceGroupFilter, + String currentResourceFilter, + String currentResourceTypeFilter, + GenericResource resource, + String subscriptionName, + CountDownLatch countDownLatch, + MetricWriteHelper metricWriteHelper, + Azure azure, + String metricPrefix) { + this.resourceFilter = resourceFilter; + this.currentResourceGroupFilter = currentResourceGroupFilter; + this.currentResourceFilter = currentResourceFilter; + this.currentResourceTypeFilter = currentResourceTypeFilter; + this.resource = resource; + this.subscriptionName = subscriptionName; + this.countDownLatch = countDownLatch; + this.metricWriteHelper = metricWriteHelper; + this.azure = azure; + this.metricPrefix = metricPrefix; + } + + @Override + public void onTaskComplete() { + logger.info("Task Complete"); + } + + @Override + public void run() { + try { + runTask(); + } + catch(Exception e){ + logger.error(e.getMessage()); + } + finally { + countDownLatch.countDown(); + } + } + private void runTask(){ + List finalMetricList = Lists.newArrayList(); + DateTime recordDateTime = DateTime.now(); + if (resource.name().matches(currentResourceFilter) && + resource.resourceType().matches(currentResourceTypeFilter) && + resource.resourceGroupName().matches(currentResourceGroupFilter)) { + if (logger.isDebugEnabled()) { + logger.debug("Working on Resource {} of Type {} in Group {} because of Resource Filter {} of Type Filter {} in Group Filter {}", + resource.name(), + resource.resourceType(), + resource.resourceGroupName(), + currentResourceTypeFilter, + currentResourceTypeFilter, + currentResourceGroupFilter); + } + List resourceMetrics = azure.metricDefinitions().listByResource(resource.id()); + List> metricFilters = (List>) resourceFilter.get("metrics"); + + for (Map metricFilter : metricFilters) { + for (MetricDefinition resourceMetric : resourceMetrics) { + String currentMetricFilter = metricFilter.get("metric"); + if (resourceMetric.isDimensionRequired()) { + if (logger.isDebugEnabled()) { + logger.debug("Resource Metric {} needs Dimensions. This is currently not supported", + resourceMetric.id()); + } + } else if (resourceMetric.name().value().matches(currentMetricFilter)) { + MetricCollection metricCollection = resourceMetric.defineQuery().startingFrom(recordDateTime.minusMinutes(2)).endsBefore(recordDateTime).execute(); + String metricAggregation = resourceMetric.primaryAggregationType().toString(); + String metricName = resourceMetric.name().value(); + String metricValue = null; + String metricPath = metricPrefix + + Constants.METRIC_SEPARATOR + + subscriptionName + + Constants.METRIC_SEPARATOR + + resource.resourceGroupName() + + Constants.METRIC_SEPARATOR + + resource.resourceType() + + Constants.METRIC_SEPARATOR + + resource.name() + + Constants.METRIC_SEPARATOR; + switch (metricAggregation) { + case "Count": + metricValue = String.valueOf(metricCollection.metrics().get(0).timeseries().get(0).data().get(0).count()); + break; + case "Average": + metricValue = String.valueOf(metricCollection.metrics().get(0).timeseries().get(0).data().get(0).average()); + break; + case "Total": + metricValue = String.valueOf(metricCollection.metrics().get(0).timeseries().get(0).data().get(0).total()); + break; + case "Maximum": + metricValue = String.valueOf(metricCollection.metrics().get(0).timeseries().get(0).data().get(0).maximum()); + break; + default: + logger.info("Not Reporting Metric {} for Resource {} as the aggregation type is not supported", + metricName, + resource.name()); + break; + } + assert metricValue != null; + if (metricValue.isEmpty() || metricValue.equals("0.0") || metricValue.equals("null")) { + logger.debug("Ignoring Metric {} for Resource {} as it is null or empty", + metricName, + resource.name(), + metricValue); + } else { + Metric metric = new Metric(metricName, metricValue, metricPath + metricName); + if (logger.isDebugEnabled()) { + logger.debug("Reporting Metric {} for Resource {} with value {}", + metricName, + resource.name(), + metricValue); + } + finalMetricList.add(metric); + } + } else { + if (logger.isDebugEnabled()) { + logger.debug("Not Reporting Metric {} for Resource {} as it is filtered by {}", + resourceMetric.name().value(), + resource.name(), + currentMetricFilter); + } + } + } + } + } else { + if (logger.isDebugEnabled()) { + logger.debug("Skipping Resource {} of Type {} in Group {} because of Resource Filter {} of Type Filter {} in Group Filter {}", + resource.name(), + resource.resourceType(), + resource.resourceGroupName(), + currentResourceTypeFilter, + currentResourceTypeFilter, + currentResourceGroupFilter); + } + } + if (finalMetricList.isEmpty()) { + if (logger.isDebugEnabled()) { + logger.debug("Metric List is empty"); + } + } else { + metricWriteHelper.transformAndPrintMetrics(finalMetricList); + } + } +} diff --git a/src/main/java/com/appdynamics/monitors/azure/utils/Constants.java b/src/main/java/com/appdynamics/monitors/azure/utils/Constants.java new file mode 100644 index 0000000..4c5b338 --- /dev/null +++ b/src/main/java/com/appdynamics/monitors/azure/utils/Constants.java @@ -0,0 +1,17 @@ +/* + * Copyright 2017. AppDynamics LLC and its affiliates. + * All Rights Reserved. + * This is unpublished proprietary source code of AppDynamics LLC and its affiliates. + * The copyright notice above does not evidence any actual or intended publication of such source code. + * + */ + +package com.appdynamics.monitors.azure.utils; + +import com.microsoft.azure.management.Azure; + +public class Constants { + public static final String DEFAULT_METRIC_PREFIX = "Custom Metrics|AzureMonitor|"; + public static final String METRIC_SEPARATOR = "|"; + public static Azure.Authenticated azureMonitorAuth = null; +} diff --git a/src/main/java/com/appdynamics/monitors/azure/utils/Utilities.java b/src/main/java/com/appdynamics/monitors/azure/utils/Utilities.java new file mode 100644 index 0000000..5c575e6 --- /dev/null +++ b/src/main/java/com/appdynamics/monitors/azure/utils/Utilities.java @@ -0,0 +1,49 @@ +/* + * Copyright 2017. AppDynamics LLC and its affiliates. + * All Rights Reserved. + * This is unpublished proprietary source code of AppDynamics LLC and its affiliates. + * The copyright notice above does not evidence any actual or intended publication of such source code. + * + */ + +package com.appdynamics.monitors.azure.utils; + +import com.appdynamics.extensions.crypto.CryptoUtil; +import com.google.common.collect.Maps; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.net.MalformedURLException; +import java.net.URL; + +public class Utilities { + private static final Logger logger = LoggerFactory.getLogger(Utilities.class); + + public static String getDecryptedKey(String encryptedKey, String encryptionKey){ + java.util.Map cryptoMap = Maps.newHashMap(); + cryptoMap.put("password-encrypted", encryptedKey); + cryptoMap.put("encryption-key", encryptionKey); + return CryptoUtil.getPassword(cryptoMap); + } + + public static URL getUrl(String input){ + URL url = null; + try { + url = new URL(input); + } catch (MalformedURLException e) { + logger.error("Error forming our from String {}", input, e); + } + return url; + } + +// --Commented out by Inspection START (7/4/18 6:16 PM): +// static String prettifyJson(JsonNode json) { +// ObjectMapper mapper = new ObjectMapper(); +// try { +// return mapper.writerWithDefaultPrettyPrinter().writeValueAsString(json); +// } catch (JsonProcessingException e) { +// logger.error("Can not process JSON {}", json.asText(), e); +// } +// return null; +// } +// --Commented out by Inspection STOP (7/4/18 6:16 PM) +} diff --git a/src/main/resources/conf/config.yml b/src/main/resources/conf/config.yml index c6d4a9e..ff7bf18 100644 --- a/src/main/resources/conf/config.yml +++ b/src/main/resources/conf/config.yml @@ -12,24 +12,26 @@ metricPrefix: "Custom Metrics|AzureMonitor|" #encryptedClientKey: "" #encryptedKeyvaultClientKey: "" +# Subscription ID obtained from the Azure Portal +# Tenant ID obtained from the Azure Portal +# Client ID obtained from the Azure Portal +# Client Key for the upper ID obtained from the Azure Portal # Keyvault Client ID obtained from the Azure Portal -keyvaultClientId: "" # Keyvault Key for the upper ID obtained from the Azure Portal -keyvaultClientKey: "" # Keyvault Client Secret Url. From this URL the clientKey will be obtained -keyvaultClientSecretUrl: "" -# Client ID obtained from the Azure Portal -clientId: "" -# Client Key for the upper ID obtained from the Azure Portal -clientKey: "" -# Tenant ID obtained from the Azure Portal -tenantId: "" -# Subscription ID obtained from the Azure Portal -subscriptionId: "" - -api-version: "2017-05-10" -monitor-api-version: "2017-05-01-preview" -keyvault-api-version: "2016-10-01" +subscriptions: + - subscriptionId: "" + tenantId: "" + clientId: "" + clientKey: "" + keyvaultClientId: "" + keyvaultClientKey: "" + keyvaultClientSecretUrl: "" + api-version: "2017-05-10" + monitor-api-version: "2017-05-01-preview" + keyvault-api-version: "2016-10-01" + resourceTypes: + - resourceType: "Microsoft.AnalysisServices/servers" # Include Filter - These Resource Types will be monitored: # The exclude Element will ignore by matching the regular expression against the Metric Name from the Metric Definition Response diff --git a/src/test/java/com/appdynamics/monitors/azure/AzureMonitorTest.java b/src/test/java/com/appdynamics/monitors/azure/AzureMonitorTest.java index b3f94d5..8f34439 100644 --- a/src/test/java/com/appdynamics/monitors/azure/AzureMonitorTest.java +++ b/src/test/java/com/appdynamics/monitors/azure/AzureMonitorTest.java @@ -1,37 +1,25 @@ package com.appdynamics.monitors.azure; -import com.appdynamics.extensions.conf.MonitorConfiguration; -import com.appdynamics.extensions.util.MetricWriteHelper; -import com.appdynamics.monitors.azure.config.AuthenticationResults; -import com.appdynamics.monitors.azure.config.Globals; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.singularity.ee.agent.systemagent.api.TaskOutput; +import com.appdynamics.extensions.*; +import com.appdynamics.extensions.conf.MonitorContextConfiguration; +import com.appdynamics.extensions.util.AssertUtils; +import com.appdynamics.monitors.azure.utils.Constants; import com.singularity.ee.agent.systemagent.api.exception.TaskExecutionException; import org.junit.Test; -import org.mockito.Mockito; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.*; -import java.math.BigDecimal; -import java.net.URL; -import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.CountDownLatch; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; public class AzureMonitorTest { - private static final Logger logger = LoggerFactory.getLogger(AzureMonitorTest.class); + // --Commented out by Inspection (7/4/18 6:16 PM):private static final Logger logger = LoggerFactory.getLogger(AzureMonitorTest.class); @Test public void testAzureMonitor(){ - Map taskArgs = new HashMap(); + Map taskArgs = new HashMap<>(); taskArgs.put("config-file", "src/test/resources/conf/integration-test-config.yml"); try { testAzureMonitorRun(taskArgs); @@ -42,7 +30,7 @@ public void testAzureMonitor(){ @Test public void testAzureMonitorWithEncryption(){ - Map taskArgs = new HashMap(); + Map taskArgs = new HashMap<>(); taskArgs.put("config-file", "src/test/resources/conf/integration-test-encrypted-config.yml"); try { testAzureMonitorRun(taskArgs); @@ -96,92 +84,32 @@ public void testResourceFilters(){ } } - @Test - public void testExtractMetrics() throws Exception { - ObjectMapper mapper = new ObjectMapper(); - JsonNode json = mapper.readTree(new FileReader("src/test/resources/json/metric.json")); - - JsonNode jsonValue = json.get("value"); - for (JsonNode currentValueNode:jsonValue){ - String metricNameValue = currentValueNode.get("name").get("value").asText(); - String metricUnit = currentValueNode.get("unit").asText(); - String metricType = null; - BigDecimal metricValue = null; - if(currentValueNode.get("timeseries").has(0)){ - JsonNode jsonData = currentValueNode.get("timeseries").get(0).get("data"); - for (JsonNode currentDataNode:jsonData){ - if (currentDataNode.has("average")){ metricType = "average"; metricValue = currentDataNode.get("average").decimalValue(); } - else if (currentDataNode.has("total")){ metricType = "total"; metricValue = currentDataNode.get("total").decimalValue(); } - else if (currentDataNode.has("last")){ metricType = "last"; metricValue = currentDataNode.get("last").decimalValue(); } - else if (currentDataNode.has("maximum")){ metricType = "maximum"; metricValue = currentDataNode.get("maximum").decimalValue(); } - } - } - assertNotNull(metricValue); - assertNotNull(metricType); - assertNotNull(metricUnit); - assertNotNull(metricNameValue); - } - } - - private void testAzureMonitorRun(Map taskArgs) throws TaskExecutionException { - TaskOutput result = new AzureMonitor().execute(taskArgs, null); - assertTrue(result.getStatusMessage().contains("Metric Upload Complete")); + new AzureMonitor().execute(taskArgs, null); } - @SuppressWarnings("ConstantConditions") - private void testAzureMonitorTaskRun(String configYml) throws Exception { - MetricWriteHelper writer = Mockito.mock(MetricWriteHelper.class); - Runnable runner = Mockito.mock(Runnable.class); - MonitorConfiguration conf = new MonitorConfiguration(Globals.defaultMetricPrefix, runner, writer); - conf.setConfigYml(configYml); - Mockito.doAnswer(new Answer() { - public Object answer(InvocationOnMock invocationOnMock) { Object[] args = invocationOnMock.getArguments(); - System.out.println(args[0] + "=" + args[1]); - return null; - } - }).when(writer).printMetric(Mockito.anyString(), Mockito.any(BigDecimal.class), Mockito.anyString()); - conf.setMetricWriter(writer); - - AzureAuth.getAzureAuth(conf.getConfigYml()); - JsonNode filtersJson = Utilities.getFiltersJson((ArrayList) conf.getConfigYml().get(Globals.azureApiFilter)); - String filterUrl = Utilities.getFilterUrl(filtersJson); - Map resourceFilter = Utilities.getResourceFilter(filtersJson); - URL url = Utilities.getUrl(Globals.azureEndpoint + Globals.azureApiSubscriptions + conf.getConfigYml().get(Globals.subscriptionId) + Globals.azureApiResources + - "?" + Globals.azureApiVersion + "=" + conf.getConfigYml().get(Globals.azureApiVersion) + - filterUrl); - ArrayNode resourceElements = (ArrayNode) AzureRestOperation.doGet(AuthenticationResults.azureMonitorAuth,url).get("value"); - Utilities.prettifyJson(resourceElements); - for(JsonNode resourceNode:resourceElements){ - URL metricDefinitions = Utilities.getUrl(Globals.azureEndpoint + resourceNode.get("id").asText() + Globals.azureApiMetricDefinitions + "?" + Globals.azureApiVersion + "=" + conf.getConfigYml().get(Globals.azureMonitorApiVersion)); - JsonNode metricDefinitionResponse = AzureRestOperation.doGet(AuthenticationResults.azureMonitorAuth,metricDefinitions); - assert metricDefinitionResponse != null; - ArrayNode metricDefinitionElements = (ArrayNode) metricDefinitionResponse.get("value"); - for(JsonNode metricDefinitionNode:metricDefinitionElements){ - if (metricDefinitionNode.get("isDimensionRequired").asText().equals("true")){ - logger.info("Dimensions are currently not supported. Skipping " - + metricDefinitionNode.get("id").asText()); - } - else if (Utilities.checkResourceFilter(metricDefinitionNode,resourceFilter)){ - logger.info("Ignoring Metric " + - metricDefinitionNode.get("name").get("value").asText() + - " for Resource " + metricDefinitionNode.get("resourceId")); - } - else { - AzureMonitorTask monitorTask = new AzureMonitorTask( - conf, - resourceNode, - AuthenticationResults.azureMonitorAuth, - metricDefinitionNode.get("name").get("value").asText()); - conf.getExecutorService().execute(monitorTask); - } + @SuppressWarnings({"ConstantConditions", "unchecked"}) + private void testAzureMonitorTaskRun(String configYml) { + AMonitorJob aMonitorJob = mock(AMonitorJob.class); + MetricWriteHelper metricWriteHelper = mock(MetricWriteHelper.class); + CountDownLatch countDownLatch = new CountDownLatch(1); + MonitorContextConfiguration monitorContextConfiguration = new MonitorContextConfiguration( + "AzureMonitor", + Constants.DEFAULT_METRIC_PREFIX, + new File(System.getProperty("user.dir")), + aMonitorJob); + monitorContextConfiguration.setConfigYml(configYml); + List> subscriptions = (List>)monitorContextConfiguration.getConfigYml().get("subscriptions"); + AssertUtils.assertNotNull(subscriptions, "The 'subscriptions' section in config.yml is not initialised"); + for (Map subscription : subscriptions) { + AzureMonitorTask task = new AzureMonitorTask(monitorContextConfiguration, metricWriteHelper, subscription, countDownLatch); + monitorContextConfiguration.getContext().getExecutorService().execute("Azure Monitor", task); + try { + countDownLatch.await(); + } catch (InterruptedException e) { + e.printStackTrace(); } } - conf.getExecutorService().awaitTermination(2, TimeUnit.SECONDS); - } - @Test(expected = TaskExecutionException.class) - public void testAzureMonitorTaskExcecutionException() throws Exception { - new AzureMonitor().execute(null, null); } } diff --git a/src/test/resources/json/metric.json b/src/test/resources/json/metric.json deleted file mode 100644 index 745761b..0000000 --- a/src/test/resources/json/metric.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "cost" : 0, - "timespan" : "2018-01-18T06:57:48Z/2018-01-18T06:59:48Z", - "interval" : "PT1M", - "value" : [ { - "id" : "/subscriptions/subscriptionId/resourceGroups/resourceGroup/providers/Microsoft.Compute/virtualMachines/resourceName/providers/Microsoft.Insights/metrics/Percentage CPU", - "type" : "Microsoft.Insights/metrics", - "name" : { - "value" : "Percentage CPU", - "localizedValue" : "Percentage CPU" - }, - "unit" : "Percent", - "timeseries" : [ { - "metadatavalues" : [ ], - "data" : [ { - "timeStamp" : "2018-01-18T06:57:00Z", - "average" : 1.05 - }, { - "timeStamp" : "2018-01-18T06:58:00Z", - "average" : 0.985 - } ] - } ] - } ] -} \ No newline at end of file diff --git a/src/test/resources/json/metricDefinitions.json b/src/test/resources/json/metricDefinitions.json deleted file mode 100644 index 9a401f7..0000000 --- a/src/test/resources/json/metricDefinitions.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "value" : [ { - "id" : "/subscriptions/subscriptionId/resourceGroups/resourceGroup/providers/Microsoft.Compute/virtualMachines/resourceName/providers/microsoft.insights/metricdefinitions/Percentage CPU", - "resourceId" : "/subscriptions/subscriptionId/resourceGroups/resourceGroup/providers/Microsoft.Compute/virtualMachines/resourceName", - "name" : { - "value" : "Percentage CPU", - "localizedValue" : "Percentage CPU" - }, - "isDimensionRequired" : false, - "unit" : "Percent", - "primaryAggregationType" : "Average", - "metricAvailabilities" : [ { - "timeGrain" : "PT1M", - "retention" : "P93D" - }, { - "timeGrain" : "PT1H", - "retention" : "P93D" - } ] - } - ] -} \ No newline at end of file diff --git a/src/test/resources/json/resources.json b/src/test/resources/json/resources.json deleted file mode 100644 index 0fac449..0000000 --- a/src/test/resources/json/resources.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "value": [ - { - "id": "/subscriptions/subscriptionId/resourceGroups/resourceGroup/providers/Microsoft.Web/sites/resourceName", - "name": "resourceName", - "type": "Microsoft.Web/sites", - "kind": "app", - "location": "location", - "tags": { - "displayName": "WebApp" - } - } - ] -} \ No newline at end of file From acf3928408c30f64e61f1a33ff7e3feb4d893c06 Mon Sep 17 00:00:00 2001 From: Michi Date: Wed, 4 Jul 2018 21:06:07 +0200 Subject: [PATCH 02/51] adopted new extension framework rewrote to Azure Bindings Changed filter strategy --- README.MD | 54 ++++++++--------- src/main/resources/conf/config.yml | 93 +++++++----------------------- 2 files changed, 50 insertions(+), 97 deletions(-) diff --git a/README.MD b/README.MD index d4dd7c1..a54cf72 100644 --- a/README.MD +++ b/README.MD @@ -12,7 +12,7 @@ Monitor Metrics provided by the Azure Monitor API and let them report into the A ## Installation -Either [Download the Extension from the Github release](https://github.com/michaelenglert/azure-monitoring-extension/releases/tag/v1.1) or Build from Source. +Either [Download the Extension from the Github release](https://github.com/michaelenglert/azure-monitoring-extension/releases) or Build from Source. 1. Deploy the `AzureMonitor-.zip` file into the `/monitors` directory. @@ -20,17 +20,11 @@ Either [Download the Extension from the Github release](https://github.com/micha 2. Set up `config.yml`. At minimum this is: ``` - # Client ID obtained from the Azure Portal - clientId: "" - - # Client Key for the upper ID obtained from the Azure Portal - clientKey: "" - - # Tenant ID obtained from the Azure Portal - tenantId: "" - - # Subscription ID obtained from the Azure Portal - subscriptionId: "" +subscriptions: + - subscriptionId: "" + tenantId: "" + clientId: "" + clientKey: "" ``` Details for the Setup can be found in the [Azure - Resource Manager - Howto - Control Access - Create Service Principal - Azure Portal](https://docs.microsoft.com/en-us/azure/azure-resource-manager/resource-group-create-service-principal-portal) @@ -78,30 +72,36 @@ Metrics available for Azure are maintained within the [Azure Monitor Documentati ## Excluding Metrics for Resources -You can exlcude certain Metrics by adding an exclude within a filter Resource Element. To do this the following configuration in `config.yml` is required: +You can exlcude certain Metrics by adding an include within a filter Resource Element. To do this the following configuration in `config.yml` is required: ``` - filter: - - resourceType: "Microsoft.Storage/storageAccounts" - exclude: ".*" +subscriptions: + - subscriptionId: "" + tenantId: "" + clientId: "" + clientKey: "" + resourceGroups: # You can Filter with regex on any of these Types. + - resourceGroup: ".*" # If you have multiple entries per Type be sure they do not overlap --> This will produce duplicate Metrics + resourceTypes: + - resourceType: ".*" + resources: + - resource: ".*" + metrics: + - metric: ".*" ``` -A regular expression is needed. Any match will be ignored for that specific resource. +A regular expression is needed. Any match will be included for that specific type. ## Integration with Azure Key Vault If you do not want to store the Client Key from the Service Principal that is used to query Resources, Metric Definitions and Metrics you can obtain the Key from an Azure Key Vault Secret. To achieve this the following configuration in `config.yml` is required: ``` - # Keyvault Client ID obtained from the Azure Portal +subscriptions: + - subscriptionId: "" + tenantId: "" + clientId: "" keyvaultClientId: "" - - # Keyvault Key for the upper ID obtained from the Azure Portal keyvaultClientKey: "" - - # Keyvault Client Secret Url. From this URL the clientKey will be obtained keyvaultClientSecretUrl: "" - - # Client ID obtained from the Azure Portal - clientId: "" ``` The **Key Vault Client Key** can also be encrypted just as the normal **Client Key**. @@ -110,7 +110,9 @@ The **Key Vault Client Key** can also be encrypted just as the normal **Client K Currently the Extension support Certificate based Authentication to gather the Health Status of your Service Fabric Clusters. To use this those properties in `config.yml` have to be used: ``` - serviceFabricCert: 'monitors/AzureMonitor/your-cert.pfx' +serviceFabrics: + - serviceFabric: "ServiceFabric" + serviceFabricCert: 'src/test/resources/cert/integration-test-sf.pfx' serviceFabricPassphrase: '' ``` diff --git a/src/main/resources/conf/config.yml b/src/main/resources/conf/config.yml index ff7bf18..6012b29 100644 --- a/src/main/resources/conf/config.yml +++ b/src/main/resources/conf/config.yml @@ -21,81 +21,32 @@ metricPrefix: "Custom Metrics|AzureMonitor|" # Keyvault Client Secret Url. From this URL the clientKey will be obtained subscriptions: - subscriptionId: "" + subscriptionName: "" # If Empty the subscriptionId will be used for the Metric Path tenantId: "" clientId: "" clientKey: "" keyvaultClientId: "" keyvaultClientKey: "" keyvaultClientSecretUrl: "" - api-version: "2017-05-10" - monitor-api-version: "2017-05-01-preview" - keyvault-api-version: "2016-10-01" - resourceTypes: - - resourceType: "Microsoft.AnalysisServices/servers" + resourceGroups: # You can Filter with regex on any of these Types. + - resourceGroup: ".*" # If you have multiple entries per Type be sure they do not overlap --> This will produce duplicate Metrics + resourceTypes: + - resourceType: ".*" + resources: + - resource: ".*" + metrics: + - metric: ".*" -# Include Filter - These Resource Types will be monitored: -# The exclude Element will ignore by matching the regular expression against the Metric Name from the Metric Definition Response -filter: - - resourceType: "Microsoft.AnalysisServices/servers" -# exclude: ".*" - - resourceType: "Microsoft.ApiManagement/service" - - resourceType: "Microsoft.Automation/automationAccounts" - - resourceType: "Microsoft.Batch/batchAccounts" - - resourceType: "Microsoft.Cache/redis" - - resourceType: "Microsoft.ClassicCompute/virtualMachines" - - resourceType: "Microsoft.CognitiveServices/accounts" - - resourceType: "Microsoft.Compute/virtualMachines" - - resourceType: "Microsoft.Compute/virtualMachineScaleSets" - - resourceType: "Microsoft.Compute/virtualMachineScaleSets/virtualMachines" - - resourceType: "Microsoft.CustomerInsights/hubs" - - resourceType: "Microsoft.DataLakeAnalytics/accounts" - - resourceType: "Microsoft.DataLakeStore/accounts" - - resourceType: "Microsoft.DBforMySQL/servers" - - resourceType: "Microsoft.DBforPostgreSQL/servers" - - resourceType: "Microsoft.Devices/IotHubs" - - resourceType: "Microsoft.Devices/provisioningServices" - - resourceType: "Microsoft.DocumentDB/databaseAccounts" - - resourceType: "Microsoft.EventHub/namespaces" - - resourceType: "Microsoft.Insights/AutoscaleSettings" - - resourceType: "Microsoft.Logic/workflows" -# - resourceType: "Microsoft.Network/loadBalancers" -- Produced Errors during Test - - resourceType: "Microsoft.Network/publicIPAddresses" - - resourceType: "Microsoft.Network/applicationGateways" - - resourceType: "Microsoft.Network/virtualNetworkGateways" - - resourceType: "Microsoft.Network/expressRouteCircuits" - - resourceType: "Microsoft.Network/trafficManagerProfiles" - - resourceType: "Microsoft.NotificationHubs/Namespaces/NotificationHubs" - - resourceType: "Microsoft.Search/searchServices" - - resourceType: "Microsoft.ServiceBus/namespaces" - - resourceType: "Microsoft.Sql/servers/databases" - - resourceType: "Microsoft.Sql/servers/elasticPools" - - resourceType: "Microsoft.Sql/servers" - - resourceType: "Microsoft.Storage/storageAccounts" - - resourceType: "Microsoft.Storage/storageAccounts/blobServices" - - resourceType: "Microsoft.Storage/storageAccounts/tableServices" - - resourceType: "Microsoft.Storage/storageAccounts/queueServices" - - resourceType: "Microsoft.Storage/storageAccounts/fileServices" - - resourceType: "Microsoft.StreamAnalytics/streamingjobs" - - resourceType: "Microsoft.Web/serverfarms" - - resourceType: "Microsoft.Web/sites" - - resourceType: "Microsoft.Web/sites/slots" - - resourceType: "Microsoft.Web/hostingEnvironments/multiRolePools" - - resourceType: "Microsoft.Web/hostingEnvironments/workerPools" - - resourceType: "Microsoft.ServiceFabric/clusters" - -serviceFabricApiVersion: "3.0" -serviceFabricResourceApiVersion: "2016-09-01" -serviceFabricBody: '{"ApplicationFilters":[{"HealthStateFilter":65535}],"NodeFilters":[{"HealthStateFilter":65535}]}' - -# Certificate Authentication will be used if the Service Fabric Management Endpoint is https://... -serviceFabricCert: 'monitors/AzureMonitor/your-cert.pfx' -serviceFabricPassphrase: '' - -# Service Fabric Health States. These States will be translated to numbers according to the table below -# The Defaults are derived from here https://docs.microsoft.com/en-us/rest/api/servicefabric/sfclient-model-healthinformation -serviceFabricHealthStates: - - Invalid: 0 - Ok: 1 - Warning: 2 - Error: 3 - Unknown: 65535 \ No newline at end of file +serviceFabrics: + - serviceFabric: "ServiceFabric" + serviceFabricApiVersion: "3.0" + serviceFabricResourceApiVersion: "2016-09-01" + serviceFabricBody: '{"ApplicationFilters":[{"HealthStateFilter":65535}],"NodeFilters":[{"HealthStateFilter":65535}]}' + serviceFabricCert: 'src/test/resources/cert/integration-test-sf.pfx' + serviceFabricPassphrase: '' + serviceFabricHealthStates: + - Invalid: 0 + Ok: 1 + Warning: 2 + Error: 3 + Unknown: 65535 \ No newline at end of file From 3b593242ad355335bda55b88febc70d1576b2c12 Mon Sep 17 00:00:00 2001 From: Michi Date: Wed, 4 Jul 2018 21:08:04 +0200 Subject: [PATCH 03/51] format adjustments --- README.MD | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/README.MD b/README.MD index a54cf72..8126b1a 100644 --- a/README.MD +++ b/README.MD @@ -74,18 +74,18 @@ Metrics available for Azure are maintained within the [Azure Monitor Documentati You can exlcude certain Metrics by adding an include within a filter Resource Element. To do this the following configuration in `config.yml` is required: ``` -subscriptions: - - subscriptionId: "" - tenantId: "" - clientId: "" - clientKey: "" - resourceGroups: # You can Filter with regex on any of these Types. - - resourceGroup: ".*" # If you have multiple entries per Type be sure they do not overlap --> This will produce duplicate Metrics - resourceTypes: - - resourceType: ".*" - resources: - - resource: ".*" - metrics: + subscriptions: + - subscriptionId: "" + tenantId: "" + clientId: "" + clientKey: "" + resourceGroups: # You can Filter with regex on any of these Types. + - resourceGroup: ".*" # If you have multiple entries per Type be sure they do not overlap --> This will produce duplicate Metrics + resourceTypes: + - resourceType: ".*" + resources: + - resource: ".*" + metrics: - metric: ".*" ``` A regular expression is needed. Any match will be included for that specific type. @@ -95,13 +95,13 @@ A regular expression is needed. Any match will be included for that specific typ If you do not want to store the Client Key from the Service Principal that is used to query Resources, Metric Definitions and Metrics you can obtain the Key from an Azure Key Vault Secret. To achieve this the following configuration in `config.yml` is required: ``` -subscriptions: - - subscriptionId: "" - tenantId: "" - clientId: "" - keyvaultClientId: "" - keyvaultClientKey: "" - keyvaultClientSecretUrl: "" + subscriptions: + - subscriptionId: "" + tenantId: "" + clientId: "" + keyvaultClientId: "" + keyvaultClientKey: "" + keyvaultClientSecretUrl: "" ``` The **Key Vault Client Key** can also be encrypted just as the normal **Client Key**. @@ -110,10 +110,10 @@ The **Key Vault Client Key** can also be encrypted just as the normal **Client K Currently the Extension support Certificate based Authentication to gather the Health Status of your Service Fabric Clusters. To use this those properties in `config.yml` have to be used: ``` -serviceFabrics: - - serviceFabric: "ServiceFabric" - serviceFabricCert: 'src/test/resources/cert/integration-test-sf.pfx' - serviceFabricPassphrase: '' + serviceFabrics: + - serviceFabric: "ServiceFabric" + serviceFabricCert: 'src/test/resources/cert/integration-test-sf.pfx' + serviceFabricPassphrase: '' ``` ## Password Encryption Support From 53abc508b659ccc9ec6bcc5129d1c053ac5c2a2f Mon Sep 17 00:00:00 2001 From: Michi Date: Wed, 4 Jul 2018 21:13:35 +0200 Subject: [PATCH 04/51] readme change --- README.MD | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.MD b/README.MD index 8126b1a..5916b11 100644 --- a/README.MD +++ b/README.MD @@ -80,8 +80,8 @@ You can exlcude certain Metrics by adding an include within a filter Resource El clientId: "" clientKey: "" resourceGroups: # You can Filter with regex on any of these Types. - - resourceGroup: ".*" # If you have multiple entries per Type be sure they do not overlap --> This will produce duplicate Metrics - resourceTypes: + - resourceGroup: ".*" # If you have multiple entries per Type be sure they do not overlap + resourceTypes: # --> This will produce duplicate Metrics!!!! - resourceType: ".*" resources: - resource: ".*" From 65feb8da6c18c774fb9de360a2c0200f99176bd0 Mon Sep 17 00:00:00 2001 From: Michi Date: Wed, 4 Jul 2018 21:18:12 +0200 Subject: [PATCH 05/51] removed unused code --- pom.xml | 8 ++++++++ .../appdynamics/monitors/azure/utils/Utilities.java | 12 ------------ src/main/resources/conf/monitor.xml | 8 ++++++++ .../appdynamics/monitors/azure/AzureMonitorTest.java | 8 ++++++++ 4 files changed, 24 insertions(+), 12 deletions(-) diff --git a/pom.xml b/pom.xml index cc4d9a4..5e4d630 100644 --- a/pom.xml +++ b/pom.xml @@ -1,4 +1,12 @@ + + diff --git a/src/main/java/com/appdynamics/monitors/azure/utils/Utilities.java b/src/main/java/com/appdynamics/monitors/azure/utils/Utilities.java index 5c575e6..92bfeb1 100644 --- a/src/main/java/com/appdynamics/monitors/azure/utils/Utilities.java +++ b/src/main/java/com/appdynamics/monitors/azure/utils/Utilities.java @@ -34,16 +34,4 @@ public static URL getUrl(String input){ } return url; } - -// --Commented out by Inspection START (7/4/18 6:16 PM): -// static String prettifyJson(JsonNode json) { -// ObjectMapper mapper = new ObjectMapper(); -// try { -// return mapper.writerWithDefaultPrettyPrinter().writeValueAsString(json); -// } catch (JsonProcessingException e) { -// logger.error("Can not process JSON {}", json.asText(), e); -// } -// return null; -// } -// --Commented out by Inspection STOP (7/4/18 6:16 PM) } diff --git a/src/main/resources/conf/monitor.xml b/src/main/resources/conf/monitor.xml index 1405c09..e360152 100644 --- a/src/main/resources/conf/monitor.xml +++ b/src/main/resources/conf/monitor.xml @@ -1,3 +1,11 @@ + + AzureMonitor diff --git a/src/test/java/com/appdynamics/monitors/azure/AzureMonitorTest.java b/src/test/java/com/appdynamics/monitors/azure/AzureMonitorTest.java index 8f34439..a758bd1 100644 --- a/src/test/java/com/appdynamics/monitors/azure/AzureMonitorTest.java +++ b/src/test/java/com/appdynamics/monitors/azure/AzureMonitorTest.java @@ -1,3 +1,11 @@ +/* + * Copyright 2017. AppDynamics LLC and its affiliates. + * All Rights Reserved. + * This is unpublished proprietary source code of AppDynamics LLC and its affiliates. + * The copyright notice above does not evidence any actual or intended publication of such source code. + * + */ + package com.appdynamics.monitors.azure; import com.appdynamics.extensions.*; From c7cf7809b204e32fdd0e843ae2651e3c1c99a4a8 Mon Sep 17 00:00:00 2001 From: Michi Date: Wed, 4 Jul 2018 21:21:12 +0200 Subject: [PATCH 06/51] copyright update --- pom.xml | 2 +- src/main/java/com/appdynamics/monitors/azure/AzureAuth.java | 2 +- src/main/java/com/appdynamics/monitors/azure/AzureMonitor.java | 2 +- .../java/com/appdynamics/monitors/azure/AzureMonitorTask.java | 2 +- .../java/com/appdynamics/monitors/azure/AzureRestOperation.java | 2 +- .../java/com/appdynamics/monitors/azure/ServiceFabricTask.java | 2 +- .../com/appdynamics/monitors/azure/metrics/AzureMetrics.java | 2 +- .../java/com/appdynamics/monitors/azure/utils/Constants.java | 2 +- .../java/com/appdynamics/monitors/azure/utils/Utilities.java | 2 +- src/main/resources/conf/monitor.xml | 2 +- .../java/com/appdynamics/monitors/azure/AzureMonitorTest.java | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/pom.xml b/pom.xml index 5e4d630..526bf21 100644 --- a/pom.xml +++ b/pom.xml @@ -1,6 +1,6 @@ + + org.apache.commons + commons-collections4 + 4.0 + commons-io commons-io From 8a7f8c38780ff96983caa7f44414899534032dd6 Mon Sep 17 00:00:00 2001 From: Isi Ramirez Date: Thu, 9 Aug 2018 11:12:56 +0200 Subject: [PATCH 18/51] Add azureAuthResult to Constants --- .../java/com/appdynamics/monitors/azure/AzureAuth.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/appdynamics/monitors/azure/AzureAuth.java b/src/main/java/com/appdynamics/monitors/azure/AzureAuth.java index 6bd233c..84e5a81 100644 --- a/src/main/java/com/appdynamics/monitors/azure/AzureAuth.java +++ b/src/main/java/com/appdynamics/monitors/azure/AzureAuth.java @@ -53,7 +53,7 @@ static void getAzureAuth(Map subscription) { if (!Strings.isNullOrEmpty(keyvaultClientId) && !Strings.isNullOrEmpty(keyvaultClientKey) && !Strings.isNullOrEmpty(keyvaultClientSecretUrl)){ URL keyvaultUrl = Utilities.getUrl(keyvaultClientSecretUrl + "?" + "api-version" + "=" + subscription.get("keyvault-api-version")); - AuthenticationResult azureKeyVaultAuth = getAuthenticationResult(keyvaultClientId, keyvaultClientKey, tenantId); + AuthenticationResult azureKeyVaultAuth = getAuthenticationResult(keyvaultClientId, keyvaultClientKey, tenantId, Constants.AZURE_VAULT_URL); JsonNode keyVaultResponse = AzureRestOperation.doGet(azureKeyVaultAuth, keyvaultUrl); assert keyVaultResponse != null; clientKey = keyVaultResponse.get("value").textValue(); @@ -69,9 +69,12 @@ static void getAzureAuth(Map subscription) { } else { Constants.azureMonitorAuth = Azure.authenticate(applicationTokenCredentials); } + AuthenticationResult azureAuthResult = getAuthenticationResult(clientId, clientKey, tenantId, Constants.AZURE_MANAGEMENT_URL); + Constants.azureAuthResult = azureAuthResult; + logger.debug("Bearer {}", azureAuthResult.getAccessToken()); } - private static AuthenticationResult getAuthenticationResult(String Id, String Key, String tenantId){ + private static AuthenticationResult getAuthenticationResult(String Id, String Key, String tenantId, String resourceUrl){ ExecutorService service = Executors.newSingleThreadExecutor(); AuthenticationResult result = null; String authority = "https://login.microsoftonline.com/" + tenantId; @@ -80,7 +83,7 @@ private static AuthenticationResult getAuthenticationResult(String Id, String Ke AuthenticationContext context; context = new AuthenticationContext(authority, false, service); ClientCredential cred = new ClientCredential(Id, Key); - Future future = context.acquireToken("https://vault.azure.net", cred, null); + Future future = context.acquireToken(resourceUrl, cred, null); result = future.get(); } catch (MalformedURLException e) { logger.error("Not a valid Azure authentication Authority {}", authority, e); From 3f4c03e980177554ad4565640bae59bfb088c844 Mon Sep 17 00:00:00 2001 From: Isi Ramirez Date: Thu, 9 Aug 2018 11:25:02 +0200 Subject: [PATCH 19/51] Make doGet method public --- .../com/appdynamics/monitors/azure/AzureRestOperation.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/appdynamics/monitors/azure/AzureRestOperation.java b/src/main/java/com/appdynamics/monitors/azure/AzureRestOperation.java index bdef06e..c8b905c 100644 --- a/src/main/java/com/appdynamics/monitors/azure/AzureRestOperation.java +++ b/src/main/java/com/appdynamics/monitors/azure/AzureRestOperation.java @@ -19,11 +19,12 @@ import java.net.URL; import java.security.*; -class AzureRestOperation { +public class AzureRestOperation { private static final Logger logger = LoggerFactory.getLogger(AzureRestOperation.class); - static JsonNode doGet(AuthenticationResult azureAuth, URL url) { + public static JsonNode doGet(AuthenticationResult azureAuth, URL url) { try { + logger.debug("--> GET " + url); ObjectMapper objectMapper = new ObjectMapper(); String response = ""; HttpURLConnection conn; From 607e79dbaa50f24fc03738db3dbc4343c0e35367 Mon Sep 17 00:00:00 2001 From: Isi Ramirez Date: Thu, 9 Aug 2018 11:35:27 +0200 Subject: [PATCH 20/51] Add direct call to api and parse the results --- .../monitors/azure/AzureMonitorTask.java | 2 +- .../monitors/azure/metrics/AzureMetrics.java | 239 ++++++++++++------ .../monitors/azure/utils/Constants.java | 6 + 3 files changed, 162 insertions(+), 85 deletions(-) diff --git a/src/main/java/com/appdynamics/monitors/azure/AzureMonitorTask.java b/src/main/java/com/appdynamics/monitors/azure/AzureMonitorTask.java index 0a7d17c..0fab9e3 100644 --- a/src/main/java/com/appdynamics/monitors/azure/AzureMonitorTask.java +++ b/src/main/java/com/appdynamics/monitors/azure/AzureMonitorTask.java @@ -92,7 +92,7 @@ private void runTask(){ currentResourceFilter, currentResourceTypeFilter, resource, - subscriptionName, + subscription, countDownLatchAzure, metricWriteHelper, azure, diff --git a/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java b/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java index 6df21d8..94a0ce6 100644 --- a/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java +++ b/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java @@ -8,22 +8,37 @@ package com.appdynamics.monitors.azure.metrics; +import java.io.UnsupportedEncodingException; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.text.SimpleDateFormat; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.TimeZone; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.collections4.ListUtils; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; +import org.slf4j.LoggerFactory; + import com.appdynamics.extensions.AMonitorTaskRunnable; import com.appdynamics.extensions.MetricWriteHelper; import com.appdynamics.extensions.metrics.Metric; +import com.appdynamics.monitors.azure.AzureRestOperation; import com.appdynamics.monitors.azure.utils.Constants; import com.google.common.collect.Lists; import com.microsoft.azure.management.Azure; import com.microsoft.azure.management.monitor.MetricCollection; import com.microsoft.azure.management.monitor.MetricDefinition; import com.microsoft.azure.management.resources.GenericResource; -import org.joda.time.DateTime; -import org.slf4j.LoggerFactory; - -import java.util.List; -import java.util.Map; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.atomic.AtomicInteger; +import com.fasterxml.jackson.databind.JsonNode; @SuppressWarnings("unchecked") public class AzureMetrics implements AMonitorTaskRunnable { @@ -37,22 +52,24 @@ public class AzureMetrics implements AMonitorTaskRunnable { private final String currentResourceFilter; private final String currentResourceTypeFilter; private final GenericResource resource; - private final String subscriptionName; + private final Map subscription; private final CountDownLatch countDownLatch; private final MetricWriteHelper metricWriteHelper; private final Azure azure; private final String metricPrefix; - private List finalMetricList = Lists.newArrayList(); + private List finalMetricList = Lists.newArrayList(); + + private DateTime recordDateTime = DateTime.now(DateTimeZone.UTC); - private DateTime recordDateTime = DateTime.now(); + private String subscriptionName; public AzureMetrics(Map resourceFilter, String currentResourceGroupFilter, String currentResourceFilter, String currentResourceTypeFilter, GenericResource resource, - String subscriptionName, + Map subscription, CountDownLatch countDownLatch, MetricWriteHelper metricWriteHelper, Azure azure, @@ -62,7 +79,7 @@ public AzureMetrics(Map resourceFilter, this.currentResourceFilter = currentResourceFilter; this.currentResourceTypeFilter = currentResourceTypeFilter; this.resource = resource; - this.subscriptionName = subscriptionName; + this.subscription = subscription; this.countDownLatch = countDownLatch; this.metricWriteHelper = metricWriteHelper; this.azure = azure; @@ -76,6 +93,7 @@ public void onTaskComplete() { @Override public void run() { + try { runTask(); } @@ -86,7 +104,14 @@ public void run() { countDownLatch.countDown(); } } + private void runTask(){ + if (subscription.containsKey("subscriptionName")){ + subscriptionName = subscription.get("subscriptionName").toString(); + } + else { + subscriptionName = subscription.get("subscriptionId").toString(); + } if (resource.name().matches(currentResourceFilter) && resource.resourceType().matches(currentResourceTypeFilter) && resource.resourceGroupName().matches(currentResourceGroupFilter)) { @@ -101,9 +126,16 @@ private void runTask(){ } azureMetricDefinitionsCallCount.incrementAndGet(); List resourceMetrics = azure.metricDefinitions().listByResource(resource.id()); - logger.info("fos: " + resourceMetrics.toString() + " " + resourceFilter.toString()); List> metricFilters = (List>) resourceFilter.get("metrics"); - getResourceMetrics(resourceMetrics, metricFilters); + try { + getResourceMetrics(resourceMetrics, metricFilters); + } catch (MalformedURLException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (UnsupportedEncodingException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } } else { if (logger.isDebugEnabled()) { logger.debug("Skipping Resource {} of Type {} in Group {} because of Resource Filter {} of Type Filter {} in Group Filter {}", @@ -126,79 +158,118 @@ private void runTask(){ logger.debug("azureMetricsCallCount: " + azureMetricsCallCount); } - private void getResourceMetrics(List resourceMetrics, List> metricFilters) { + private void getResourceMetrics(List resourceMetrics, List> metricFilters) throws MalformedURLException, UnsupportedEncodingException { + List filteredMetrics = Lists.newArrayList(); + List filteredMetricsNames = Lists.newArrayList(); + for (Map metricFilter : metricFilters) { for (MetricDefinition resourceMetric : resourceMetrics) { - String currentMetricFilter = metricFilter.get("metric"); - if (resourceMetric.isDimensionRequired()) { - if (logger.isDebugEnabled()) { - logger.debug("Resource Metric {} needs Dimensions. This is currently not supported", - resourceMetric.id()); - } - } else if (resourceMetric.name().value().matches(currentMetricFilter)) { - azureMetricsCallCount.incrementAndGet(); - MetricCollection metricCollection = resourceMetric.defineQuery().startingFrom(recordDateTime.minusMinutes(2)).endsBefore(recordDateTime).execute(); - addMetrics(resourceMetric, metricCollection); - } else { - if (logger.isDebugEnabled()) { - logger.debug("Not Reporting Metric {} for Resource {} as it is filtered by {}", - resourceMetric.name().value(), - resource.name(), - currentMetricFilter); - } - } - } + String currentMetricFilter = metricFilter.get("metric"); + if (resourceMetric.isDimensionRequired()) { + if (logger.isDebugEnabled()) { + logger.debug("Resource Metric {} needs Dimensions. This is currently not supported", + resourceMetric.id()); + } + } else if (resourceMetric.name().value().matches(currentMetricFilter)) { + filteredMetrics.add(resourceMetric); + filteredMetricsNames.add(URLEncoder.encode(resourceMetric.name().value(), "UTF-8")); + } else { + if (logger.isDebugEnabled()) { + logger.debug("Not Reporting Metric {} for Resource {} as it is filtered by {}", + resourceMetric.name().value(), + resource.name(), + currentMetricFilter); + } + } + } + if (!filteredMetrics.isEmpty()) { + List> filteredMetricsChunks = ListUtils.partition(filteredMetrics, Constants.AZURE_METRICS_CHUNK_SIZE); + List> filteredMetricsNamesChunks = ListUtils.partition(filteredMetricsNames, Constants.AZURE_METRICS_CHUNK_SIZE); + Iterator> filteredMetricsIterator = filteredMetricsChunks.iterator(); + Iterator> filteredMetricsNamesIterator = filteredMetricsNamesChunks.iterator(); + + String apiEndpointBase = filteredMetrics.get(0).id().substring(0,StringUtils.ordinalIndexOf(filteredMetrics.get(0).id(), "/", 11)); + + while (filteredMetricsIterator.hasNext() && filteredMetricsNamesIterator.hasNext()) { + List filteredMetricsChunk = filteredMetricsIterator.next(); + List filteredMetricsNamesChunk = filteredMetricsNamesIterator.next(); + URL apiEndpointFull = new URL(Constants.AZURE_MANAGEMENT_URL + + apiEndpointBase + + "/metrics?timespan=" + recordDateTime.minusMinutes(2).toDateTimeISO() + "/" + recordDateTime.toDateTimeISO() + + "&metricnames=" + StringUtils.join(filteredMetricsNamesChunk, ',') + + "&api-version=" + subscription.get("api-version")); + azureMetricsCallCount.incrementAndGet(); + JsonNode apiResponse = AzureRestOperation.doGet(Constants.azureAuthResult, apiEndpointFull); + if (apiResponse != null) { + logger.debug("API response: " + apiResponse.toString()); + addMetrics(filteredMetricsChunk, apiResponse); + } + } + } } } - private void addMetrics(MetricDefinition resourceMetric, MetricCollection metricCollection) { - String metricAggregation = resourceMetric.primaryAggregationType().toString(); - String metricName = resourceMetric.name().value(); - String metricValue = null; - String metricPath = metricPrefix + - Constants.METRIC_SEPARATOR + - subscriptionName + - Constants.METRIC_SEPARATOR + - resource.resourceGroupName() + - Constants.METRIC_SEPARATOR + - resource.resourceType() + - Constants.METRIC_SEPARATOR + - resource.name() + - Constants.METRIC_SEPARATOR; - switch (metricAggregation) { - case "Count": - metricValue = String.valueOf(metricCollection.metrics().get(0).timeseries().get(0).data().get(0).count()); - break; - case "Average": - metricValue = String.valueOf(metricCollection.metrics().get(0).timeseries().get(0).data().get(0).average()); - break; - case "Total": - metricValue = String.valueOf(metricCollection.metrics().get(0).timeseries().get(0).data().get(0).total()); - break; - case "Maximum": - metricValue = String.valueOf(metricCollection.metrics().get(0).timeseries().get(0).data().get(0).maximum()); - break; - default: - logger.info("Not Reporting Metric {} for Resource {} as the aggregation type is not supported", - metricName, - resource.name()); - break; - } - assert metricValue != null; - if (metricValue.isEmpty() || metricValue.equals("0.0") || metricValue.equals("null")) { - logger.debug("Ignoring Metric {} for Resource {} as it is null or empty", - metricName, - resource.name(), - metricValue); - } else { - Metric metric = new Metric(metricName, metricValue, metricPath + metricName); - if (logger.isDebugEnabled()) { - logger.debug("Reporting Metric {} for Resource {} with value {}", - metricName, - resource.name(), - metricValue); - } - finalMetricList.add(metric); - } - } + private void addMetrics(List filteredMetricsChunk, JsonNode responseMetrics) { + + JsonNode responseMetricsValues = responseMetrics.get("value"); + Iterator responseMetricsIterator = responseMetricsValues.elements(); + Iterator filteredMetricsIterator = filteredMetricsChunk.iterator(); + while (responseMetricsIterator.hasNext() && filteredMetricsIterator.hasNext()) { + MetricDefinition resourceMetric = filteredMetricsIterator.next(); + JsonNode responseMetric = responseMetricsIterator.next(); + logger.debug("Response metric: {}", responseMetric.toString()); + + String metricAggregation = resourceMetric.primaryAggregationType().toString(); + String metricName = resourceMetric.name().value(); + String metricValue = null; + String metricPath = metricPrefix + + Constants.METRIC_SEPARATOR + + subscriptionName + + Constants.METRIC_SEPARATOR + + resource.resourceGroupName() + + Constants.METRIC_SEPARATOR + + resource.resourceType() + + Constants.METRIC_SEPARATOR + + resource.name() + + Constants.METRIC_SEPARATOR; + logger.debug("Metric name / aggregation / path: {} / {} / {}", metricName, metricAggregation, metricPath); + + JsonNode data = responseMetric.findPath("timeseries").findPath("data"); + switch (metricAggregation) { + case "Count": + metricValue = (data.path(0).path("count").isMissingNode()) ? null : data.get(0).get("count").toString(); + break; + case "Average": + metricValue = (data.path(0).path("average").isMissingNode()) ? null : data.get(0).get("average").toString(); + break; + case "Total": + metricValue = (data.path(0).path("total").isMissingNode()) ? null : data.get(0).get("total").toString(); + break; + case "Maximum": + metricValue = (data.path(0).path("maximum").isMissingNode()) ? null : data.get(0).get("maximum").toString(); + break; + default: + logger.info("Not Reporting Metric {} for Resource {} as the aggregation type is not supported", + metricName, + resource.name()); + break; + } + + if (metricValue == null) { + logger.debug("Ignoring Metric {} for Resource {} as it is null or empty", + metricName, + resource.name(), + metricValue); + } else { + Metric metric = new Metric(metricName, metricValue, metricPath + metricName); + if (logger.isDebugEnabled()) { + logger.debug("Reporting Metric {} for Resource {} with value {}", + metricName, + resource.name(), + metricValue); + } + finalMetricList.add(metric); + } + } + } } diff --git a/src/main/java/com/appdynamics/monitors/azure/utils/Constants.java b/src/main/java/com/appdynamics/monitors/azure/utils/Constants.java index 87d43d7..851d151 100644 --- a/src/main/java/com/appdynamics/monitors/azure/utils/Constants.java +++ b/src/main/java/com/appdynamics/monitors/azure/utils/Constants.java @@ -8,11 +8,17 @@ package com.appdynamics.monitors.azure.utils; +import com.microsoft.aad.adal4j.AuthenticationResult; +import com.microsoft.azure.credentials.ApplicationTokenCredentials; import com.microsoft.azure.management.Azure; public class Constants { public static final String DEFAULT_METRIC_PREFIX = "Custom Metrics|AzureMonitor|"; public static final String METRIC_SEPARATOR = "|"; public static final String TEST_CONFIG_FILE = "src/test/resources/conf/test-config.yml"; + public static final String AZURE_MANAGEMENT_URL = "https://management.azure.com"; + public static final String AZURE_VAULT_URL = "https://vault.azure.net"; + public static final int AZURE_METRICS_CHUNK_SIZE = 20; public static Azure.Authenticated azureMonitorAuth = null; + public static AuthenticationResult azureAuthResult; } From 934c5bbff49cbbad4cdf2febe9299f4d33930bff Mon Sep 17 00:00:00 2001 From: Isi Ramirez Date: Thu, 9 Aug 2018 14:25:46 +0200 Subject: [PATCH 21/51] Fixing resource type method and make it case-insensitive --- .../appdynamics/monitors/azure/metrics/AzureMetrics.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java b/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java index 94a0ce6..ec7fd1a 100644 --- a/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java +++ b/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java @@ -112,9 +112,12 @@ private void runTask(){ else { subscriptionName = subscription.get("subscriptionId").toString(); } + logger.debug("Resource name ({}): {} {}", resource.name().matches(currentResourceFilter), resource.name(), currentResourceFilter); + logger.debug("Resource type ({}): {} {}", resource.type().matches(currentResourceTypeFilter), resource.type(), currentResourceTypeFilter); + logger.debug("Resource group ({}): {} {}", resource.resourceGroupName().matches("(?i:" + currentResourceGroupFilter + ")"), resource.resourceGroupName(), currentResourceGroupFilter); if (resource.name().matches(currentResourceFilter) && - resource.resourceType().matches(currentResourceTypeFilter) && - resource.resourceGroupName().matches(currentResourceGroupFilter)) { + resource.type().matches(currentResourceTypeFilter) && + resource.resourceGroupName().matches("(?i:" + currentResourceGroupFilter + ")")) { if (logger.isDebugEnabled()) { logger.debug("Working on Resource {} of Type {} in Group {} because of Resource Filter {} of Type Filter {} in Group Filter {}", resource.name(), From 40ae111e9ea2ea013459583224f880d4f856b817 Mon Sep 17 00:00:00 2001 From: Isi Ramirez Date: Thu, 9 Aug 2018 22:08:53 +0200 Subject: [PATCH 22/51] Check if debug is enabled --- .../monitors/azure/metrics/AzureMetrics.java | 44 ++++++++++++------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java b/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java index ec7fd1a..981d0b0 100644 --- a/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java +++ b/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java @@ -112,9 +112,11 @@ private void runTask(){ else { subscriptionName = subscription.get("subscriptionId").toString(); } - logger.debug("Resource name ({}): {} {}", resource.name().matches(currentResourceFilter), resource.name(), currentResourceFilter); - logger.debug("Resource type ({}): {} {}", resource.type().matches(currentResourceTypeFilter), resource.type(), currentResourceTypeFilter); - logger.debug("Resource group ({}): {} {}", resource.resourceGroupName().matches("(?i:" + currentResourceGroupFilter + ")"), resource.resourceGroupName(), currentResourceGroupFilter); + if (logger.isDebugEnabled()) { + logger.debug("Resource name ({}): {} {}", resource.name().matches(currentResourceFilter), resource.name(), currentResourceFilter); + logger.debug("Resource type ({}): {} {}", resource.type().matches(currentResourceTypeFilter), resource.type(), currentResourceTypeFilter); + logger.debug("Resource group ({}): {} {}", resource.resourceGroupName().matches("(?i:" + currentResourceGroupFilter + ")"), resource.resourceGroupName(), currentResourceGroupFilter); + } if (resource.name().matches(currentResourceFilter) && resource.type().matches(currentResourceTypeFilter) && resource.resourceGroupName().matches("(?i:" + currentResourceGroupFilter + ")")) { @@ -157,8 +159,10 @@ private void runTask(){ } else { metricWriteHelper.transformAndPrintMetrics(finalMetricList); } - logger.debug("azureMetricDefinitionsCallCount {}: {}", resource.id(), azureMetricDefinitionsCallCount); - logger.debug("azureMetricsCallCount: " + azureMetricsCallCount); + if (logger.isDebugEnabled()) { + logger.debug("azureMetricDefinitionsCallCount {}: {}", resource.id(), azureMetricDefinitionsCallCount); + logger.debug("azureMetricsCallCount: " + azureMetricsCallCount); + } } private void getResourceMetrics(List resourceMetrics, List> metricFilters) throws MalformedURLException, UnsupportedEncodingException { @@ -204,7 +208,9 @@ private void getResourceMetrics(List resourceMetrics, List filteredMetricsChunk, JsonNode re while (responseMetricsIterator.hasNext() && filteredMetricsIterator.hasNext()) { MetricDefinition resourceMetric = filteredMetricsIterator.next(); JsonNode responseMetric = responseMetricsIterator.next(); - logger.debug("Response metric: {}", responseMetric.toString()); + if (logger.isDebugEnabled()) { + logger.debug("Response metric: {}", responseMetric.toString()); + } String metricAggregation = resourceMetric.primaryAggregationType().toString(); String metricName = resourceMetric.name().value(); @@ -235,7 +243,9 @@ private void addMetrics(List filteredMetricsChunk, JsonNode re Constants.METRIC_SEPARATOR + resource.name() + Constants.METRIC_SEPARATOR; - logger.debug("Metric name / aggregation / path: {} / {} / {}", metricName, metricAggregation, metricPath); + if (logger.isDebugEnabled()) { + logger.debug("Metric name / aggregation / path: {} / {} / {}", metricName, metricAggregation, metricPath); + } JsonNode data = responseMetric.findPath("timeseries").findPath("data"); switch (metricAggregation) { @@ -252,17 +262,21 @@ private void addMetrics(List filteredMetricsChunk, JsonNode re metricValue = (data.path(0).path("maximum").isMissingNode()) ? null : data.get(0).get("maximum").toString(); break; default: - logger.info("Not Reporting Metric {} for Resource {} as the aggregation type is not supported", - metricName, - resource.name()); + if (logger.isDebugEnabled()) { + logger.info("Not Reporting Metric {} for Resource {} as the aggregation type is not supported", + metricName, + resource.name()); + } break; } if (metricValue == null) { - logger.debug("Ignoring Metric {} for Resource {} as it is null or empty", - metricName, - resource.name(), - metricValue); + if (logger.isDebugEnabled()) { + logger.debug("Ignoring Metric {} for Resource {} as it is null or empty", + metricName, + resource.name(), + metricValue); + } } else { Metric metric = new Metric(metricName, metricValue, metricPath + metricName); if (logger.isDebugEnabled()) { From 2cc2ba1d8fd23ad34a72ee05c8833a63bcfd81ed Mon Sep 17 00:00:00 2001 From: Isi Ramirez Date: Tue, 14 Aug 2018 14:18:50 +0200 Subject: [PATCH 23/51] Add basic dimensions support --- .../monitors/azure/metrics/AzureMetrics.java | 245 +++++++++++------- 1 file changed, 147 insertions(+), 98 deletions(-) diff --git a/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java b/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java index 981d0b0..481e2ef 100644 --- a/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java +++ b/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java @@ -46,23 +46,19 @@ public class AzureMetrics implements AMonitorTaskRunnable { public final AtomicInteger azureMetricsCallCount = new AtomicInteger(0); public final AtomicInteger azureMetricDefinitionsCallCount = new AtomicInteger(0); - private final Map resourceFilter; private final String currentResourceGroupFilter; private final String currentResourceFilter; private final String currentResourceTypeFilter; private final GenericResource resource; private final Map subscription; + private final String subscriptionName; private final CountDownLatch countDownLatch; private final MetricWriteHelper metricWriteHelper; private final Azure azure; private final String metricPrefix; - - private List finalMetricList = Lists.newArrayList(); - - private DateTime recordDateTime = DateTime.now(DateTimeZone.UTC); - - private String subscriptionName; + private final List finalMetricList = Lists.newArrayList(); + private final DateTime recordDateTime = DateTime.now(DateTimeZone.UTC); public AzureMetrics(Map resourceFilter, String currentResourceGroupFilter, @@ -84,6 +80,12 @@ public AzureMetrics(Map resourceFilter, this.metricWriteHelper = metricWriteHelper; this.azure = azure; this.metricPrefix = metricPrefix; + if (subscription.containsKey("subscriptionName")){ + subscriptionName = subscription.get("subscriptionName").toString(); + } + else { + subscriptionName = subscription.get("subscriptionId").toString(); + } } @Override @@ -93,7 +95,6 @@ public void onTaskComplete() { @Override public void run() { - try { runTask(); } @@ -106,12 +107,6 @@ public void run() { } private void runTask(){ - if (subscription.containsKey("subscriptionName")){ - subscriptionName = subscription.get("subscriptionName").toString(); - } - else { - subscriptionName = subscription.get("subscriptionId").toString(); - } if (logger.isDebugEnabled()) { logger.debug("Resource name ({}): {} {}", resource.name().matches(currentResourceFilter), resource.name(), currentResourceFilter); logger.debug("Resource type ({}): {} {}", resource.type().matches(currentResourceTypeFilter), resource.type(), currentResourceTypeFilter); @@ -131,9 +126,8 @@ private void runTask(){ } azureMetricDefinitionsCallCount.incrementAndGet(); List resourceMetrics = azure.metricDefinitions().listByResource(resource.id()); - List> metricFilters = (List>) resourceFilter.get("metrics"); try { - getResourceMetrics(resourceMetrics, metricFilters); + generateMetrics(resourceMetrics); } catch (MalformedURLException e) { // TODO Auto-generated catch block e.printStackTrace(); @@ -165,21 +159,40 @@ private void runTask(){ } } - private void getResourceMetrics(List resourceMetrics, List> metricFilters) throws MalformedURLException, UnsupportedEncodingException { + private void generateMetrics(List resourceMetrics) throws MalformedURLException, UnsupportedEncodingException { + List> metricFilters = (List>) resourceFilter.get("metrics"); + List filteredMetrics = Lists.newArrayList(); List filteredMetricsNames = Lists.newArrayList(); - for (Map metricFilter : metricFilters) { + for (Map metricFilter : metricFilters) { for (MetricDefinition resourceMetric : resourceMetrics) { - String currentMetricFilter = metricFilter.get("metric"); - if (resourceMetric.isDimensionRequired()) { - if (logger.isDebugEnabled()) { - logger.debug("Resource Metric {} needs Dimensions. This is currently not supported", - resourceMetric.id()); + String currentMetricFilter = metricFilter.get("metric").toString(); + if (logger.isDebugEnabled()) { + logger.debug("resourceMetric name ({})", resourceMetric.name().value()); + logger.debug("currentMetricFilter ({})", currentMetricFilter); + logger.debug("match ({})", resourceMetric.name().value().matches(currentMetricFilter), resourceMetric.name().value(), currentMetricFilter); + } + + String apiEndpointBase = resourceMetric.id().substring(0,StringUtils.ordinalIndexOf(resourceMetric.id(), "/", 11)); + Object filterObject = metricFilter.get("filter"); + String filter = filterObject == null ? "" : filterObject.toString(); + + if (resourceMetric.name().value().matches(currentMetricFilter)) { + if (!filter.isEmpty()) { + JsonNode apiResponse = getAzureMetrics(apiEndpointBase, resourceMetric.name().value(), filter); + if (apiResponse != null) { + if (logger.isDebugEnabled()) { + logger.debug("API response: " + apiResponse.toString()); + } + addMetric(resourceMetric, apiResponse, metricFilter); + + return; + } + } else { + filteredMetrics.add(resourceMetric); + filteredMetricsNames.add(URLEncoder.encode(resourceMetric.name().value(), "UTF-8")); } - } else if (resourceMetric.name().value().matches(currentMetricFilter)) { - filteredMetrics.add(resourceMetric); - filteredMetricsNames.add(URLEncoder.encode(resourceMetric.name().value(), "UTF-8")); } else { if (logger.isDebugEnabled()) { logger.debug("Not Reporting Metric {} for Resource {} as it is filtered by {}", @@ -189,35 +202,55 @@ private void getResourceMetrics(List resourceMetrics, List> filteredMetricsChunks = ListUtils.partition(filteredMetrics, Constants.AZURE_METRICS_CHUNK_SIZE); - List> filteredMetricsNamesChunks = ListUtils.partition(filteredMetricsNames, Constants.AZURE_METRICS_CHUNK_SIZE); - Iterator> filteredMetricsIterator = filteredMetricsChunks.iterator(); - Iterator> filteredMetricsNamesIterator = filteredMetricsNamesChunks.iterator(); + } + logger.debug("filteredMetrics size ({})", filteredMetrics.size()); + consumeAndImportAzureMetrics(filteredMetrics, filteredMetricsNames); + } - String apiEndpointBase = filteredMetrics.get(0).id().substring(0,StringUtils.ordinalIndexOf(filteredMetrics.get(0).id(), "/", 11)); + private void consumeAndImportAzureMetrics(List filteredMetrics, List filteredMetricsNames) throws MalformedURLException, UnsupportedEncodingException { + if (!filteredMetrics.isEmpty()) { + List> filteredMetricsChunks = ListUtils.partition(filteredMetrics, Constants.AZURE_METRICS_CHUNK_SIZE); + List> filteredMetricsNamesChunks = ListUtils.partition(filteredMetricsNames, Constants.AZURE_METRICS_CHUNK_SIZE); + Iterator> filteredMetricsIterator = filteredMetricsChunks.iterator(); + Iterator> filteredMetricsNamesIterator = filteredMetricsNamesChunks.iterator(); - while (filteredMetricsIterator.hasNext() && filteredMetricsNamesIterator.hasNext()) { - List filteredMetricsChunk = filteredMetricsIterator.next(); - List filteredMetricsNamesChunk = filteredMetricsNamesIterator.next(); - URL apiEndpointFull = new URL(Constants.AZURE_MANAGEMENT_URL - + apiEndpointBase - + "/metrics?timespan=" + recordDateTime.minusMinutes(2).toDateTimeISO() + "/" + recordDateTime.toDateTimeISO() - + "&metricnames=" + StringUtils.join(filteredMetricsNamesChunk, ',') - + "&api-version=" + subscription.get("api-version")); - azureMetricsCallCount.incrementAndGet(); - JsonNode apiResponse = AzureRestOperation.doGet(Constants.azureAuthResult, apiEndpointFull); - if (apiResponse != null) { - if (logger.isDebugEnabled()) { - logger.debug("API response: " + apiResponse.toString()); - } - addMetrics(filteredMetricsChunk, apiResponse); + String apiEndpointBase = filteredMetrics.get(0).id().substring(0,StringUtils.ordinalIndexOf(filteredMetrics.get(0).id(), "/", 11)); + + while (filteredMetricsIterator.hasNext() && filteredMetricsNamesIterator.hasNext()) { + List filteredMetricsChunk = filteredMetricsIterator.next(); + List filteredMetricsNamesChunk = filteredMetricsNamesIterator.next(); + JsonNode apiResponse = getAzureMetrics(apiEndpointBase, StringUtils.join(filteredMetricsNamesChunk, ',')); + if (apiResponse != null) { + if (logger.isDebugEnabled()) { + logger.debug("API response: " + apiResponse.toString()); } + addMetrics(filteredMetricsChunk, apiResponse); } } } } + private JsonNode getAzureMetrics(String apiEndpointBase, String metricNames) throws MalformedURLException, UnsupportedEncodingException { + return getAzureMetrics(apiEndpointBase, metricNames, null); + } + + private JsonNode getAzureMetrics(String apiEndpointBase, String metricNames, + String filter) throws MalformedURLException, UnsupportedEncodingException { + String url = Constants.AZURE_MANAGEMENT_URL + + apiEndpointBase + + "/metrics?timespan=" + recordDateTime.minusMinutes(2).toDateTimeISO() + "/" + recordDateTime.toDateTimeISO() + + "&metricnames=" + metricNames + + "&api-version=" + subscription.get("api-version"); + if (filter != null) { + url += "&$filter=" + URLEncoder.encode(filter, "UTF-8"); + } + URL apiEndpointFull = new URL(url); + azureMetricsCallCount.incrementAndGet(); + JsonNode apiResponse = AzureRestOperation.doGet(Constants.azureAuthResult, apiEndpointFull); + + return apiResponse; + } + private void addMetrics(List filteredMetricsChunk, JsonNode responseMetrics) { JsonNode responseMetricsValues = responseMetrics.get("value"); @@ -229,64 +262,80 @@ private void addMetrics(List filteredMetricsChunk, JsonNode re if (logger.isDebugEnabled()) { logger.debug("Response metric: {}", responseMetric.toString()); } + addMetric(resourceMetric, responseMetric); + } + } - String metricAggregation = resourceMetric.primaryAggregationType().toString(); - String metricName = resourceMetric.name().value(); - String metricValue = null; - String metricPath = metricPrefix + - Constants.METRIC_SEPARATOR + - subscriptionName + - Constants.METRIC_SEPARATOR + - resource.resourceGroupName() + - Constants.METRIC_SEPARATOR + - resource.resourceType() + - Constants.METRIC_SEPARATOR + - resource.name() + - Constants.METRIC_SEPARATOR; - if (logger.isDebugEnabled()) { - logger.debug("Metric name / aggregation / path: {} / {} / {}", metricName, metricAggregation, metricPath); - } + private void addMetric(MetricDefinition resourceMetric, JsonNode responseMetric) { + addMetric(resourceMetric, responseMetric, null); + } - JsonNode data = responseMetric.findPath("timeseries").findPath("data"); - switch (metricAggregation) { - case "Count": - metricValue = (data.path(0).path("count").isMissingNode()) ? null : data.get(0).get("count").toString(); - break; - case "Average": - metricValue = (data.path(0).path("average").isMissingNode()) ? null : data.get(0).get("average").toString(); - break; - case "Total": - metricValue = (data.path(0).path("total").isMissingNode()) ? null : data.get(0).get("total").toString(); - break; - case "Maximum": - metricValue = (data.path(0).path("maximum").isMissingNode()) ? null : data.get(0).get("maximum").toString(); - break; - default: - if (logger.isDebugEnabled()) { - logger.info("Not Reporting Metric {} for Resource {} as the aggregation type is not supported", - metricName, - resource.name()); - } - break; - } + private void addMetric(MetricDefinition resourceMetric, JsonNode responseMetric, Map metricFilter) { + String metricAggregation = resourceMetric.primaryAggregationType().toString(); + String metricName = resourceMetric.name().value(); + String metricValue = null; + String metricPath = metricPrefix + + Constants.METRIC_SEPARATOR + + subscriptionName + + Constants.METRIC_SEPARATOR + + resource.resourceGroupName() + + Constants.METRIC_SEPARATOR + + resource.resourceType() + + Constants.METRIC_SEPARATOR + + resource.name() + + Constants.METRIC_SEPARATOR; + if (logger.isDebugEnabled()) { + logger.debug("Metric name / aggregation / path: {} / {} / {}", metricName, metricAggregation, metricPath); + } - if (metricValue == null) { + JsonNode data = responseMetric.findPath("timeseries").findPath("data"); + switch (metricAggregation) { + case "Count": + metricValue = (data.path(0).path("count").isMissingNode()) ? null : data.get(0).get("count").toString(); + break; + case "Average": + metricValue = (data.path(0).path("average").isMissingNode()) ? null : data.get(0).get("average").toString(); + break; + case "Total": + metricValue = (data.path(0).path("total").isMissingNode()) ? null : data.get(0).get("total").toString(); + break; + case "Maximum": + metricValue = (data.path(0).path("maximum").isMissingNode()) ? null : data.get(0).get("maximum").toString(); + break; + default: if (logger.isDebugEnabled()) { - logger.debug("Ignoring Metric {} for Resource {} as it is null or empty", + logger.info("Not Reporting Metric {} for Resource {} as the aggregation type is not supported", metricName, - resource.name(), - metricValue); + resource.name()); } - } else { - Metric metric = new Metric(metricName, metricValue, metricPath + metricName); - if (logger.isDebugEnabled()) { - logger.debug("Reporting Metric {} for Resource {} with value {}", - metricName, - resource.name(), - metricValue); - } - finalMetricList.add(metric); + break; + } + + if (metricValue == null) { + if (logger.isDebugEnabled()) { + logger.debug("Ignoring Metric {} for Resource {} as it is null or empty", + metricName, + resource.name(), + metricValue); + } + } else { + String subpath = ""; + Object aliasObject = null; + Object subpathObject = null; + if (metricFilter != null) { + aliasObject = metricFilter.get("alias"); + subpathObject = metricFilter.get("subpath"); + } + metricName = aliasObject == null ? metricName : aliasObject.toString(); + subpath = subpathObject == null ? "" : subpathObject.toString() + "|"; + Metric metric = new Metric(metricName, metricValue, metricPath + subpath + metricName); + if (logger.isDebugEnabled()) { + logger.debug("Reporting Metric {} for Resource {} with value {}", + metricName, + resource.name(), + metricValue); } + finalMetricList.add(metric); } } } From 87aa69a1370b0d30f373fc1556524ac076eb4150 Mon Sep 17 00:00:00 2001 From: Isi Ramirez Date: Wed, 15 Aug 2018 09:32:40 +0200 Subject: [PATCH 24/51] Check timeseries size in API response --- .../monitors/azure/metrics/AzureMetrics.java | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java b/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java index 481e2ef..865e193 100644 --- a/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java +++ b/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java @@ -185,7 +185,12 @@ private void generateMetrics(List resourceMetrics) throws Malf if (logger.isDebugEnabled()) { logger.debug("API response: " + apiResponse.toString()); } - addMetric(resourceMetric, apiResponse, metricFilter); + try { + addMetric(resourceMetric, apiResponse, metricFilter); + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } return; } @@ -267,10 +272,15 @@ private void addMetrics(List filteredMetricsChunk, JsonNode re } private void addMetric(MetricDefinition resourceMetric, JsonNode responseMetric) { - addMetric(resourceMetric, responseMetric, null); + try { + addMetric(resourceMetric, responseMetric, null); + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } } - private void addMetric(MetricDefinition resourceMetric, JsonNode responseMetric, Map metricFilter) { + private void addMetric(MetricDefinition resourceMetric, JsonNode responseMetric, Map metricFilter) throws Exception { String metricAggregation = resourceMetric.primaryAggregationType().toString(); String metricName = resourceMetric.name().value(); String metricValue = null; @@ -288,7 +298,12 @@ private void addMetric(MetricDefinition resourceMetric, JsonNode responseMetric, logger.debug("Metric name / aggregation / path: {} / {} / {}", metricName, metricAggregation, metricPath); } - JsonNode data = responseMetric.findPath("timeseries").findPath("data"); + JsonNode timeseries = responseMetric.findPath("timeseries"); + logger.debug("timeseries.size():" + timeseries.size()); + if (timeseries.size() > 1) { + throw new Exception("Multiple timeseries not supported"); + } + JsonNode data = timeseries.findPath("data"); switch (metricAggregation) { case "Count": metricValue = (data.path(0).path("count").isMissingNode()) ? null : data.get(0).get("count").toString(); From 4f91bbc1ecf139f92dc7f6384a49b5bf33c9b80f Mon Sep 17 00:00:00 2001 From: Isi Ramirez Date: Wed, 15 Aug 2018 16:18:12 +0200 Subject: [PATCH 25/51] Move countdownlatch --- .../monitors/azure/AzureMonitorTask.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/appdynamics/monitors/azure/AzureMonitorTask.java b/src/main/java/com/appdynamics/monitors/azure/AzureMonitorTask.java index 0fab9e3..ff0b1bc 100644 --- a/src/main/java/com/appdynamics/monitors/azure/AzureMonitorTask.java +++ b/src/main/java/com/appdynamics/monitors/azure/AzureMonitorTask.java @@ -68,8 +68,6 @@ private void runTask(){ String subscriptionName; this.azureResourcesCallCount.incrementAndGet(); PagedList resources = azure.genericResources().list(); - CountDownLatch countDownLatchAzure = new CountDownLatch(resources.size()); - if (subscription.containsKey("subscriptionName")){ subscriptionName = subscription.get("subscriptionName").toString(); } @@ -83,6 +81,7 @@ private void runTask(){ List> resourceFilters = (List>) resourceTypeFilter.get("resources"); for (Map resourceFilter : resourceFilters) { String currentResourceGroupFilter = resourceGroupFilter.get("resourceGroup").toString(); + CountDownLatch countDownLatchAzure = new CountDownLatch(resources.size()); for (GenericResource resource : resources) { String currentResourceFilter = resourceFilter.get("resource").toString(); String currentResourceTypeFilter = resourceTypeFilter.get("resourceType").toString(); @@ -97,16 +96,16 @@ private void runTask(){ metricWriteHelper, azure, configuration.getMetricPrefix()); + configuration.getContext().getExecutorService().execute("AzureMetrics", azureMetricsTask); } + try{ + countDownLatchAzure.await(); + } catch (InterruptedException e) { + e.printStackTrace(); + } } } } - - try{ - countDownLatchAzure.await(); - } catch (InterruptedException e) { - e.printStackTrace(); - } } } \ No newline at end of file From b0927269629f8b0dd3401af885272d7a9faacd4f Mon Sep 17 00:00:00 2001 From: Isi Ramirez Date: Thu, 16 Aug 2018 10:55:16 +0200 Subject: [PATCH 26/51] Add aggregation and roll up support --- .../monitors/azure/AzureMonitorTask.java | 2 +- .../monitors/azure/metrics/AzureMetrics.java | 169 +++++++++++------- .../monitors/azure/utils/Constants.java | 2 +- 3 files changed, 103 insertions(+), 70 deletions(-) diff --git a/src/main/java/com/appdynamics/monitors/azure/AzureMonitorTask.java b/src/main/java/com/appdynamics/monitors/azure/AzureMonitorTask.java index ff0b1bc..ad6bb59 100644 --- a/src/main/java/com/appdynamics/monitors/azure/AzureMonitorTask.java +++ b/src/main/java/com/appdynamics/monitors/azure/AzureMonitorTask.java @@ -47,7 +47,7 @@ class AzureMonitorTask implements AMonitorTaskRunnable{ @Override public void onTaskComplete() { logger.info("Task Complete"); - logger.debug("azureResourcesCallCount: " + this.azureResourcesCallCount); +// logger.debug("azureResourcesCallCount: " + this.azureResourcesCallCount); } @Override diff --git a/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java b/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java index 865e193..49e0b96 100644 --- a/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java +++ b/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java @@ -38,12 +38,12 @@ import com.microsoft.azure.management.monitor.MetricCollection; import com.microsoft.azure.management.monitor.MetricDefinition; import com.microsoft.azure.management.resources.GenericResource; +import com.singularity.ee.agent.systemagent.api.MetricWriter; import com.fasterxml.jackson.databind.JsonNode; @SuppressWarnings("unchecked") public class AzureMetrics implements AMonitorTaskRunnable { private static final org.slf4j.Logger logger = LoggerFactory.getLogger(AzureMetrics.class); - public final AtomicInteger azureMetricsCallCount = new AtomicInteger(0); public final AtomicInteger azureMetricDefinitionsCallCount = new AtomicInteger(0); private final Map resourceFilter; @@ -107,11 +107,11 @@ public void run() { } private void runTask(){ - if (logger.isDebugEnabled()) { - logger.debug("Resource name ({}): {} {}", resource.name().matches(currentResourceFilter), resource.name(), currentResourceFilter); - logger.debug("Resource type ({}): {} {}", resource.type().matches(currentResourceTypeFilter), resource.type(), currentResourceTypeFilter); - logger.debug("Resource group ({}): {} {}", resource.resourceGroupName().matches("(?i:" + currentResourceGroupFilter + ")"), resource.resourceGroupName(), currentResourceGroupFilter); - } +// if (logger.isDebugEnabled()) { +// logger.debug("Resource name ({}): {} {}", resource.name().matches(currentResourceFilter), resource.name(), currentResourceFilter); +// logger.debug("Resource type ({}): {} {}", resource.type().matches(currentResourceTypeFilter), resource.type(), currentResourceTypeFilter); +// logger.debug("Resource group ({}): {} {}", resource.resourceGroupName().matches("(?i:" + currentResourceGroupFilter + ")"), resource.resourceGroupName(), currentResourceGroupFilter); +// } if (resource.name().matches(currentResourceFilter) && resource.type().matches(currentResourceTypeFilter) && resource.resourceGroupName().matches("(?i:" + currentResourceGroupFilter + ")")) { @@ -136,27 +136,27 @@ private void runTask(){ e.printStackTrace(); } } else { - if (logger.isDebugEnabled()) { - logger.debug("Skipping Resource {} of Type {} in Group {} because of Resource Filter {} of Type Filter {} in Group Filter {}", - resource.name(), - resource.resourceType(), - resource.resourceGroupName(), - currentResourceTypeFilter, - currentResourceTypeFilter, - currentResourceGroupFilter); - } +// if (logger.isDebugEnabled()) { +// logger.debug("Skipping Resource {} of Type {} in Group {} because of Resource Filter {} of Type Filter {} in Group Filter {}", +// resource.name(), +// resource.resourceType(), +// resource.resourceGroupName(), +// currentResourceTypeFilter, +// currentResourceTypeFilter, +// currentResourceGroupFilter); +// } } if (finalMetricList.isEmpty()) { - if (logger.isDebugEnabled()) { - logger.debug("Metric List is empty"); - } +// if (logger.isDebugEnabled()) { +// logger.debug("Metric List is empty"); +// } } else { metricWriteHelper.transformAndPrintMetrics(finalMetricList); } - if (logger.isDebugEnabled()) { - logger.debug("azureMetricDefinitionsCallCount {}: {}", resource.id(), azureMetricDefinitionsCallCount); - logger.debug("azureMetricsCallCount: " + azureMetricsCallCount); - } +// if (logger.isDebugEnabled()) { +// logger.debug("azureMetricDefinitionsCallCount {}: {}", resource.id(), azureMetricDefinitionsCallCount); +// logger.debug("azureMetricsCallCount: " + azureMetricsCallCount); +// } } private void generateMetrics(List resourceMetrics) throws MalformedURLException, UnsupportedEncodingException { @@ -168,22 +168,27 @@ private void generateMetrics(List resourceMetrics) throws Malf for (Map metricFilter : metricFilters) { for (MetricDefinition resourceMetric : resourceMetrics) { String currentMetricFilter = metricFilter.get("metric").toString(); - if (logger.isDebugEnabled()) { - logger.debug("resourceMetric name ({})", resourceMetric.name().value()); - logger.debug("currentMetricFilter ({})", currentMetricFilter); - logger.debug("match ({})", resourceMetric.name().value().matches(currentMetricFilter), resourceMetric.name().value(), currentMetricFilter); - } +// if (logger.isDebugEnabled()) { +// logger.debug("resourceMetric name ({})", resourceMetric.name().value()); +// logger.debug("currentMetricFilter ({})", currentMetricFilter); +// logger.debug("match ({})", resourceMetric.name().value().matches(currentMetricFilter), resourceMetric.name().value(), currentMetricFilter); +// } String apiEndpointBase = resourceMetric.id().substring(0,StringUtils.ordinalIndexOf(resourceMetric.id(), "/", 11)); Object filterObject = metricFilter.get("filter"); - String filter = filterObject == null ? "" : filterObject.toString(); - + String filter = filterObject == null ? null : filterObject.toString(); + Object aggregatorObject = metricFilter.get("aggregator"); + String aggregator = aggregatorObject == null ? "" : aggregatorObject.toString(); + Object timeRollUpObject = metricFilter.get("timeRollUp"); + String timeRollUp = timeRollUpObject == null ? "" : timeRollUpObject.toString(); + Object clusterRollUpObject = metricFilter.get("clusterRollUp"); + String clusterRollUp = clusterRollUpObject == null ? "" : clusterRollUpObject.toString(); if (resourceMetric.name().value().matches(currentMetricFilter)) { - if (!filter.isEmpty()) { - JsonNode apiResponse = getAzureMetrics(apiEndpointBase, resourceMetric.name().value(), filter); + if (( filter != null && !filter.isEmpty() )|| !aggregator.isEmpty() || !timeRollUp.isEmpty() || !clusterRollUp.isEmpty() ) { + JsonNode apiResponse = getAzureMetrics(apiEndpointBase, URLEncoder.encode(resourceMetric.name().value(), "UTF-8"), filter); if (apiResponse != null) { if (logger.isDebugEnabled()) { - logger.debug("API response: " + apiResponse.toString()); + logger.debug("generateMetrics: API response: " + apiResponse.toString()); } try { addMetric(resourceMetric, apiResponse, metricFilter); @@ -199,16 +204,16 @@ private void generateMetrics(List resourceMetrics) throws Malf filteredMetricsNames.add(URLEncoder.encode(resourceMetric.name().value(), "UTF-8")); } } else { - if (logger.isDebugEnabled()) { - logger.debug("Not Reporting Metric {} for Resource {} as it is filtered by {}", - resourceMetric.name().value(), - resource.name(), - currentMetricFilter); - } +// if (logger.isDebugEnabled()) { +// logger.debug("Not Reporting Metric {} for Resource {} as it is filtered by {}", +// resourceMetric.name().value(), +// resource.name(), +// currentMetricFilter); +// } } } } - logger.debug("filteredMetrics size ({})", filteredMetrics.size()); +// logger.debug("filteredMetrics size ({})", filteredMetrics.size()); consumeAndImportAzureMetrics(filteredMetrics, filteredMetricsNames); } @@ -227,7 +232,7 @@ private void consumeAndImportAzureMetrics(List filteredMetrics JsonNode apiResponse = getAzureMetrics(apiEndpointBase, StringUtils.join(filteredMetricsNamesChunk, ',')); if (apiResponse != null) { if (logger.isDebugEnabled()) { - logger.debug("API response: " + apiResponse.toString()); + logger.debug("consumeAndImportAzureMetrics: API response: " + apiResponse.toString()); } addMetrics(filteredMetricsChunk, apiResponse); } @@ -264,9 +269,9 @@ private void addMetrics(List filteredMetricsChunk, JsonNode re while (responseMetricsIterator.hasNext() && filteredMetricsIterator.hasNext()) { MetricDefinition resourceMetric = filteredMetricsIterator.next(); JsonNode responseMetric = responseMetricsIterator.next(); - if (logger.isDebugEnabled()) { - logger.debug("Response metric: {}", responseMetric.toString()); - } +// if (logger.isDebugEnabled()) { +// logger.debug("Response metric: {}", responseMetric.toString()); +// } addMetric(resourceMetric, responseMetric); } } @@ -281,9 +286,7 @@ private void addMetric(MetricDefinition resourceMetric, JsonNode responseMetric) } private void addMetric(MetricDefinition resourceMetric, JsonNode responseMetric, Map metricFilter) throws Exception { - String metricAggregation = resourceMetric.primaryAggregationType().toString(); - String metricName = resourceMetric.name().value(); - String metricValue = null; + String azureMetricValue = null; String metricPath = metricPrefix + Constants.METRIC_SEPARATOR + subscriptionName + @@ -294,61 +297,91 @@ private void addMetric(MetricDefinition resourceMetric, JsonNode responseMetric, Constants.METRIC_SEPARATOR + resource.name() + Constants.METRIC_SEPARATOR; - if (logger.isDebugEnabled()) { - logger.debug("Metric name / aggregation / path: {} / {} / {}", metricName, metricAggregation, metricPath); - } +// if (logger.isDebugEnabled()) { +// logger.debug("Metric name / aggregation / path: {} / {} / {}", metricName, metricAggregation, metricPath); +// } JsonNode timeseries = responseMetric.findPath("timeseries"); - logger.debug("timeseries.size():" + timeseries.size()); +// logger.debug("timeseries.size():" + timeseries.size()); if (timeseries.size() > 1) { throw new Exception("Multiple timeseries not supported"); } + String azureMetricName = resourceMetric.name().value(); + String azureMetricAggregation = resourceMetric.primaryAggregationType().toString(); + String appdAggregationType = null; + String appdTimeRollUpType = null; + String appdClusterRollUpType = null; JsonNode data = timeseries.findPath("data"); - switch (metricAggregation) { - case "Count": - metricValue = (data.path(0).path("count").isMissingNode()) ? null : data.get(0).get("count").toString(); - break; + switch (azureMetricAggregation) { case "Average": - metricValue = (data.path(0).path("average").isMissingNode()) ? null : data.get(0).get("average").toString(); + azureMetricValue = (data.path(0).path("average").isMissingNode()) ? null : data.get(0).get("average").toString(); + appdAggregationType = MetricWriter.METRIC_AGGREGATION_TYPE_AVERAGE; + appdTimeRollUpType = MetricWriter.METRIC_TIME_ROLLUP_TYPE_AVERAGE; + appdClusterRollUpType = MetricWriter.METRIC_CLUSTER_ROLLUP_TYPE_INDIVIDUAL; break; case "Total": - metricValue = (data.path(0).path("total").isMissingNode()) ? null : data.get(0).get("total").toString(); + azureMetricValue = (data.path(0).path("total").isMissingNode()) ? null : data.get(0).get("total").toString(); + appdAggregationType = MetricWriter.METRIC_AGGREGATION_TYPE_AVERAGE; + appdTimeRollUpType = MetricWriter.METRIC_TIME_ROLLUP_TYPE_SUM; + appdClusterRollUpType = MetricWriter.METRIC_CLUSTER_ROLLUP_TYPE_COLLECTIVE; break; case "Maximum": - metricValue = (data.path(0).path("maximum").isMissingNode()) ? null : data.get(0).get("maximum").toString(); + azureMetricValue = (data.path(0).path("maximum").isMissingNode()) ? null : data.get(0).get("maximum").toString(); + appdTimeRollUpType = MetricWriter.METRIC_TIME_ROLLUP_TYPE_AVERAGE; + appdTimeRollUpType = MetricWriter.METRIC_TIME_ROLLUP_TYPE_AVERAGE; + appdClusterRollUpType = MetricWriter.METRIC_CLUSTER_ROLLUP_TYPE_INDIVIDUAL; + break; + case "Mininum": + azureMetricValue = (data.path(0).path("minimum").isMissingNode()) ? null : data.get(0).get("minimum").toString(); + appdTimeRollUpType = MetricWriter.METRIC_TIME_ROLLUP_TYPE_AVERAGE; + appdTimeRollUpType = MetricWriter.METRIC_TIME_ROLLUP_TYPE_AVERAGE; + appdClusterRollUpType = MetricWriter.METRIC_CLUSTER_ROLLUP_TYPE_INDIVIDUAL; break; default: if (logger.isDebugEnabled()) { logger.info("Not Reporting Metric {} for Resource {} as the aggregation type is not supported", - metricName, + azureMetricName, resource.name()); } break; } - - if (metricValue == null) { + if (azureMetricValue == null) { if (logger.isDebugEnabled()) { logger.debug("Ignoring Metric {} for Resource {} as it is null or empty", - metricName, + azureMetricName, resource.name(), - metricValue); + azureMetricValue); } } else { - String subpath = ""; - Object aliasObject = null; Object subpathObject = null; + Object aliasObject = null; + Object aggregatorObject = null; + Object timeRollUpObject = null; + Object clusterRollUpObject = null; if (metricFilter != null) { aliasObject = metricFilter.get("alias"); subpathObject = metricFilter.get("subpath"); + aggregatorObject = metricFilter.get("aggregator"); + timeRollUpObject = metricFilter.get("timeRollUp"); + clusterRollUpObject = metricFilter.get("clusterRollUp"); } - metricName = aliasObject == null ? metricName : aliasObject.toString(); + azureMetricName = aliasObject == null ? azureMetricName : aliasObject.toString(); + String subpath = ""; subpath = subpathObject == null ? "" : subpathObject.toString() + "|"; - Metric metric = new Metric(metricName, metricValue, metricPath + subpath + metricName); + appdAggregationType = aggregatorObject == null ? appdAggregationType : MetricWriter.class.getDeclaredField(aggregatorObject.toString()).get(null).toString(); + appdTimeRollUpType = timeRollUpObject == null ? appdTimeRollUpType : MetricWriter.class.getDeclaredField(timeRollUpObject.toString()).get(null).toString(); + appdClusterRollUpType = clusterRollUpObject == null ? appdClusterRollUpType : MetricWriter.class.getDeclaredField(clusterRollUpObject.toString()).get(null).toString(); + Metric metric = new Metric(azureMetricName, + azureMetricValue, + metricPath + subpath + azureMetricName, + appdAggregationType, + appdTimeRollUpType, + appdClusterRollUpType); if (logger.isDebugEnabled()) { logger.debug("Reporting Metric {} for Resource {} with value {}", - metricName, + azureMetricName, resource.name(), - metricValue); + azureMetricValue); } finalMetricList.add(metric); } diff --git a/src/main/java/com/appdynamics/monitors/azure/utils/Constants.java b/src/main/java/com/appdynamics/monitors/azure/utils/Constants.java index 851d151..d45f118 100644 --- a/src/main/java/com/appdynamics/monitors/azure/utils/Constants.java +++ b/src/main/java/com/appdynamics/monitors/azure/utils/Constants.java @@ -9,8 +9,8 @@ package com.appdynamics.monitors.azure.utils; import com.microsoft.aad.adal4j.AuthenticationResult; -import com.microsoft.azure.credentials.ApplicationTokenCredentials; import com.microsoft.azure.management.Azure; +import com.singularity.ee.agent.systemagent.api.MetricWriter; public class Constants { public static final String DEFAULT_METRIC_PREFIX = "Custom Metrics|AzureMonitor|"; From 33eb95e8681a3feeebd6e4d686cb936b6d25d460 Mon Sep 17 00:00:00 2001 From: Isi Ramirez Date: Thu, 16 Aug 2018 11:17:30 +0200 Subject: [PATCH 27/51] Fix warnings --- .../com/appdynamics/monitors/azure/AzureMonitorTask.java | 8 -------- .../appdynamics/monitors/azure/metrics/AzureMetrics.java | 5 ----- 2 files changed, 13 deletions(-) diff --git a/src/main/java/com/appdynamics/monitors/azure/AzureMonitorTask.java b/src/main/java/com/appdynamics/monitors/azure/AzureMonitorTask.java index ad6bb59..a809ba2 100644 --- a/src/main/java/com/appdynamics/monitors/azure/AzureMonitorTask.java +++ b/src/main/java/com/appdynamics/monitors/azure/AzureMonitorTask.java @@ -65,16 +65,8 @@ public void run() { private void runTask(){ AzureAuth.getAzureAuth(subscription); Azure azure = Constants.azureMonitorAuth.withSubscription(subscription.get("subscriptionId").toString()); - String subscriptionName; this.azureResourcesCallCount.incrementAndGet(); PagedList resources = azure.genericResources().list(); - if (subscription.containsKey("subscriptionName")){ - subscriptionName = subscription.get("subscriptionName").toString(); - } - else { - subscriptionName = subscription.get("subscriptionId").toString(); - } - for (Map resourceGroupFilter : resourceGroupFilters) { List> resourceTypeFilters = (List>) resourceGroupFilter.get("resourceTypes"); for (Map resourceTypeFilter : resourceTypeFilters) { diff --git a/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java b/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java index 49e0b96..0159088 100644 --- a/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java +++ b/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java @@ -12,13 +12,9 @@ import java.net.MalformedURLException; import java.net.URL; import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; -import java.text.SimpleDateFormat; import java.util.Iterator; import java.util.List; -import java.util.Locale; import java.util.Map; -import java.util.TimeZone; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicInteger; @@ -35,7 +31,6 @@ import com.appdynamics.monitors.azure.utils.Constants; import com.google.common.collect.Lists; import com.microsoft.azure.management.Azure; -import com.microsoft.azure.management.monitor.MetricCollection; import com.microsoft.azure.management.monitor.MetricDefinition; import com.microsoft.azure.management.resources.GenericResource; import com.singularity.ee.agent.systemagent.api.MetricWriter; From 513a23b26634d49c3d94dfcedb3664e5c8191ab4 Mon Sep 17 00:00:00 2001 From: Isi Ramirez Date: Thu, 16 Aug 2018 14:58:50 +0200 Subject: [PATCH 28/51] Release notes --- CHANGES.MD | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGES.MD b/CHANGES.MD index b80243f..e5c81f7 100644 --- a/CHANGES.MD +++ b/CHANGES.MD @@ -1,3 +1,9 @@ +### Version 1.10.0.0 (Alpha) + +* Add basic dimensions support +* Add CLI support +* Get multiple metrics with a simple API call + ### Version 1.9.9.0 (Alpha) * Changed to latest Extension Framework From d1b00604ba186cae22e6f1570f2fbecfcd9e9aeb Mon Sep 17 00:00:00 2001 From: Isi Ramirez Date: Mon, 20 Aug 2018 13:53:53 +0200 Subject: [PATCH 29/51] Add config when adding non-dimensional metrics --- .../monitors/azure/metrics/AzureMetrics.java | 83 ++++++++++--------- 1 file changed, 46 insertions(+), 37 deletions(-) diff --git a/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java b/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java index 0159088..3263446 100644 --- a/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java +++ b/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java @@ -155,30 +155,31 @@ private void runTask(){ } private void generateMetrics(List resourceMetrics) throws MalformedURLException, UnsupportedEncodingException { - List> metricFilters = (List>) resourceFilter.get("metrics"); + List> metricConfigs = (List>) resourceFilter.get("metrics"); List filteredMetrics = Lists.newArrayList(); List filteredMetricsNames = Lists.newArrayList(); + List> filteredMetricsConfig = Lists.newArrayList(); - for (Map metricFilter : metricFilters) { + for (Map metricConfig : metricConfigs) { for (MetricDefinition resourceMetric : resourceMetrics) { - String currentMetricFilter = metricFilter.get("metric").toString(); + String currentMetricConfig = metricConfig.get("metric").toString(); // if (logger.isDebugEnabled()) { // logger.debug("resourceMetric name ({})", resourceMetric.name().value()); -// logger.debug("currentMetricFilter ({})", currentMetricFilter); -// logger.debug("match ({})", resourceMetric.name().value().matches(currentMetricFilter), resourceMetric.name().value(), currentMetricFilter); +// logger.debug("currentMetricConfig ({})", currentMetricConfig); +// logger.debug("match ({})", resourceMetric.name().value().matches(currentMetricConfig), resourceMetric.name().value(), currentMetricConfig); // } String apiEndpointBase = resourceMetric.id().substring(0,StringUtils.ordinalIndexOf(resourceMetric.id(), "/", 11)); - Object filterObject = metricFilter.get("filter"); + Object filterObject = metricConfig.get("filter"); String filter = filterObject == null ? null : filterObject.toString(); - Object aggregatorObject = metricFilter.get("aggregator"); + Object aggregatorObject = metricConfig.get("aggregator"); String aggregator = aggregatorObject == null ? "" : aggregatorObject.toString(); - Object timeRollUpObject = metricFilter.get("timeRollUp"); + Object timeRollUpObject = metricConfig.get("timeRollUp"); String timeRollUp = timeRollUpObject == null ? "" : timeRollUpObject.toString(); - Object clusterRollUpObject = metricFilter.get("clusterRollUp"); + Object clusterRollUpObject = metricConfig.get("clusterRollUp"); String clusterRollUp = clusterRollUpObject == null ? "" : clusterRollUpObject.toString(); - if (resourceMetric.name().value().matches(currentMetricFilter)) { + if (resourceMetric.name().value().matches(currentMetricConfig)) { if (( filter != null && !filter.isEmpty() )|| !aggregator.isEmpty() || !timeRollUp.isEmpty() || !clusterRollUp.isEmpty() ) { JsonNode apiResponse = getAzureMetrics(apiEndpointBase, URLEncoder.encode(resourceMetric.name().value(), "UTF-8"), filter); if (apiResponse != null) { @@ -186,7 +187,7 @@ private void generateMetrics(List resourceMetrics) throws Malf logger.debug("generateMetrics: API response: " + apiResponse.toString()); } try { - addMetric(resourceMetric, apiResponse, metricFilter); + addMetric(resourceMetric, apiResponse, metricConfig); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); @@ -197,39 +198,43 @@ private void generateMetrics(List resourceMetrics) throws Malf } else { filteredMetrics.add(resourceMetric); filteredMetricsNames.add(URLEncoder.encode(resourceMetric.name().value(), "UTF-8")); + filteredMetricsConfig.add(metricConfig); } } else { // if (logger.isDebugEnabled()) { // logger.debug("Not Reporting Metric {} for Resource {} as it is filtered by {}", // resourceMetric.name().value(), // resource.name(), -// currentMetricFilter); +// currentMetricConfig); // } } } } // logger.debug("filteredMetrics size ({})", filteredMetrics.size()); - consumeAndImportAzureMetrics(filteredMetrics, filteredMetricsNames); + consumeAndImportAzureMetrics(filteredMetrics, filteredMetricsNames, filteredMetricsConfig); } - private void consumeAndImportAzureMetrics(List filteredMetrics, List filteredMetricsNames) throws MalformedURLException, UnsupportedEncodingException { + private void consumeAndImportAzureMetrics(List filteredMetrics, List filteredMetricsNames, List> filteredMetricsConfig) throws MalformedURLException, UnsupportedEncodingException { if (!filteredMetrics.isEmpty()) { List> filteredMetricsChunks = ListUtils.partition(filteredMetrics, Constants.AZURE_METRICS_CHUNK_SIZE); List> filteredMetricsNamesChunks = ListUtils.partition(filteredMetricsNames, Constants.AZURE_METRICS_CHUNK_SIZE); + List>> filteredMetricsConfigChunks = ListUtils.partition(filteredMetricsConfig, Constants.AZURE_METRICS_CHUNK_SIZE); Iterator> filteredMetricsIterator = filteredMetricsChunks.iterator(); Iterator> filteredMetricsNamesIterator = filteredMetricsNamesChunks.iterator(); + Iterator>> filteredMetricsConfigIterator = filteredMetricsConfigChunks.iterator(); String apiEndpointBase = filteredMetrics.get(0).id().substring(0,StringUtils.ordinalIndexOf(filteredMetrics.get(0).id(), "/", 11)); - while (filteredMetricsIterator.hasNext() && filteredMetricsNamesIterator.hasNext()) { + while (filteredMetricsIterator.hasNext() && filteredMetricsNamesIterator.hasNext() && filteredMetricsConfigIterator.hasNext()) { List filteredMetricsChunk = filteredMetricsIterator.next(); List filteredMetricsNamesChunk = filteredMetricsNamesIterator.next(); + List> filteredMetricsConfigChunk = filteredMetricsConfigIterator.next(); JsonNode apiResponse = getAzureMetrics(apiEndpointBase, StringUtils.join(filteredMetricsNamesChunk, ',')); if (apiResponse != null) { if (logger.isDebugEnabled()) { logger.debug("consumeAndImportAzureMetrics: API response: " + apiResponse.toString()); } - addMetrics(filteredMetricsChunk, apiResponse); + addMetrics(filteredMetricsChunk, filteredMetricsConfigChunk, apiResponse); } } } @@ -256,31 +261,29 @@ private JsonNode getAzureMetrics(String apiEndpointBase, String metricNames, return apiResponse; } - private void addMetrics(List filteredMetricsChunk, JsonNode responseMetrics) { + private void addMetrics(List filteredMetricsChunk, List> filteredMetricsConfigChunk, JsonNode responseMetrics) { JsonNode responseMetricsValues = responseMetrics.get("value"); Iterator responseMetricsIterator = responseMetricsValues.elements(); Iterator filteredMetricsIterator = filteredMetricsChunk.iterator(); - while (responseMetricsIterator.hasNext() && filteredMetricsIterator.hasNext()) { + Iterator> filteredMetricsConfigIterator = filteredMetricsConfigChunk.iterator(); + while (responseMetricsIterator.hasNext() && filteredMetricsConfigIterator.hasNext() && filteredMetricsIterator.hasNext()) { MetricDefinition resourceMetric = filteredMetricsIterator.next(); + Map resourceMetricConfig = filteredMetricsConfigIterator.next(); JsonNode responseMetric = responseMetricsIterator.next(); // if (logger.isDebugEnabled()) { // logger.debug("Response metric: {}", responseMetric.toString()); // } - addMetric(resourceMetric, responseMetric); - } - } - - private void addMetric(MetricDefinition resourceMetric, JsonNode responseMetric) { - try { - addMetric(resourceMetric, responseMetric, null); - } catch (Exception e) { - // TODO Auto-generated catch block - e.printStackTrace(); + try { + addMetric(resourceMetric, responseMetric, resourceMetricConfig); + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } } } - private void addMetric(MetricDefinition resourceMetric, JsonNode responseMetric, Map metricFilter) throws Exception { + private void addMetric(MetricDefinition resourceMetric, JsonNode responseMetric, Map metricConfig) throws Exception { String azureMetricValue = null; String metricPath = metricPrefix + Constants.METRIC_SEPARATOR + @@ -314,6 +317,12 @@ private void addMetric(MetricDefinition resourceMetric, JsonNode responseMetric, appdTimeRollUpType = MetricWriter.METRIC_TIME_ROLLUP_TYPE_AVERAGE; appdClusterRollUpType = MetricWriter.METRIC_CLUSTER_ROLLUP_TYPE_INDIVIDUAL; break; + case "Count": + azureMetricValue = (data.path(0).path("total").isMissingNode()) ? null : data.get(0).get("count").toString(); + appdAggregationType = MetricWriter.METRIC_AGGREGATION_TYPE_AVERAGE; + appdTimeRollUpType = MetricWriter.METRIC_TIME_ROLLUP_TYPE_SUM; + appdClusterRollUpType = MetricWriter.METRIC_CLUSTER_ROLLUP_TYPE_COLLECTIVE; + break; case "Total": azureMetricValue = (data.path(0).path("total").isMissingNode()) ? null : data.get(0).get("total").toString(); appdAggregationType = MetricWriter.METRIC_AGGREGATION_TYPE_AVERAGE; @@ -322,13 +331,13 @@ private void addMetric(MetricDefinition resourceMetric, JsonNode responseMetric, break; case "Maximum": azureMetricValue = (data.path(0).path("maximum").isMissingNode()) ? null : data.get(0).get("maximum").toString(); - appdTimeRollUpType = MetricWriter.METRIC_TIME_ROLLUP_TYPE_AVERAGE; + appdAggregationType = MetricWriter.METRIC_AGGREGATION_TYPE_AVERAGE; appdTimeRollUpType = MetricWriter.METRIC_TIME_ROLLUP_TYPE_AVERAGE; appdClusterRollUpType = MetricWriter.METRIC_CLUSTER_ROLLUP_TYPE_INDIVIDUAL; break; case "Mininum": azureMetricValue = (data.path(0).path("minimum").isMissingNode()) ? null : data.get(0).get("minimum").toString(); - appdTimeRollUpType = MetricWriter.METRIC_TIME_ROLLUP_TYPE_AVERAGE; + appdAggregationType = MetricWriter.METRIC_AGGREGATION_TYPE_AVERAGE; appdTimeRollUpType = MetricWriter.METRIC_TIME_ROLLUP_TYPE_AVERAGE; appdClusterRollUpType = MetricWriter.METRIC_CLUSTER_ROLLUP_TYPE_INDIVIDUAL; break; @@ -353,12 +362,12 @@ private void addMetric(MetricDefinition resourceMetric, JsonNode responseMetric, Object aggregatorObject = null; Object timeRollUpObject = null; Object clusterRollUpObject = null; - if (metricFilter != null) { - aliasObject = metricFilter.get("alias"); - subpathObject = metricFilter.get("subpath"); - aggregatorObject = metricFilter.get("aggregator"); - timeRollUpObject = metricFilter.get("timeRollUp"); - clusterRollUpObject = metricFilter.get("clusterRollUp"); + if (metricConfig != null) { + aliasObject = metricConfig.get("alias"); + subpathObject = metricConfig.get("subpath"); + aggregatorObject = metricConfig.get("aggregator"); + timeRollUpObject = metricConfig.get("timeRollUp"); + clusterRollUpObject = metricConfig.get("clusterRollUp"); } azureMetricName = aliasObject == null ? azureMetricName : aliasObject.toString(); String subpath = ""; From 5a6ff7ae2c624a47be95be3edbc7ca21bc1dd1e1 Mon Sep 17 00:00:00 2001 From: Isi Ramirez Date: Mon, 20 Aug 2018 16:25:37 +0200 Subject: [PATCH 30/51] Add await to main countdownlatch --- .../java/com/appdynamics/monitors/azure/AzureMonitor.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/appdynamics/monitors/azure/AzureMonitor.java b/src/main/java/com/appdynamics/monitors/azure/AzureMonitor.java index 57743b0..134f979 100644 --- a/src/main/java/com/appdynamics/monitors/azure/AzureMonitor.java +++ b/src/main/java/com/appdynamics/monitors/azure/AzureMonitor.java @@ -48,10 +48,16 @@ public void onComplete() { protected void doRun(TasksExecutionServiceProvider tasksExecutionServiceProvider) { List> subscriptions = (List>)getContextConfiguration().getConfigYml().get("subscriptions"); AssertUtils.assertNotNull(subscriptions, "The 'subscriptions' section in config.yml is not initialised"); + CountDownLatch countDownLatch = new CountDownLatch(getTaskCount()); for (Map subscription : subscriptions) { - AzureMonitorTask task = new AzureMonitorTask(getContextConfiguration(), tasksExecutionServiceProvider.getMetricWriteHelper(), subscription, new CountDownLatch(getTaskCount())); + AzureMonitorTask task = new AzureMonitorTask(getContextConfiguration(), tasksExecutionServiceProvider.getMetricWriteHelper(), subscription, countDownLatch); tasksExecutionServiceProvider.submit(subscription.get("subscriptionId").toString(),task); } + try{ + countDownLatch.await(); + } catch (InterruptedException e) { + e.printStackTrace(); + } } @Override From 67e2dbce564b1f3051be358cbecd6af7d419dfd0 Mon Sep 17 00:00:00 2001 From: Isi Ramirez Date: Mon, 20 Aug 2018 17:13:57 +0200 Subject: [PATCH 31/51] Move total time from monitor to tasks --- .../java/com/appdynamics/monitors/azure/AzureMonitor.java | 7 ++----- .../com/appdynamics/monitors/azure/AzureMonitorTask.java | 7 ++++++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/appdynamics/monitors/azure/AzureMonitor.java b/src/main/java/com/appdynamics/monitors/azure/AzureMonitor.java index 134f979..3f7a3cd 100644 --- a/src/main/java/com/appdynamics/monitors/azure/AzureMonitor.java +++ b/src/main/java/com/appdynamics/monitors/azure/AzureMonitor.java @@ -25,8 +25,6 @@ public class AzureMonitor extends ABaseMonitor { private static final org.slf4j.Logger logger = LoggerFactory.getLogger(AzureMonitorTask.class); - private static long startTime = System.currentTimeMillis(); - @Override protected String getDefaultMetricPrefix() { return Constants.DEFAULT_METRIC_PREFIX; @@ -39,9 +37,8 @@ public String getMonitorName() { @Override public void onComplete() { - long finishTime = System.currentTimeMillis(); - long totalTime = finishTime - startTime; - logger.debug("Total time: " + (totalTime / 1000.0f) + " ms"); + logger.info("Monitor Completed"); + System.exit(0); } @Override diff --git a/src/main/java/com/appdynamics/monitors/azure/AzureMonitorTask.java b/src/main/java/com/appdynamics/monitors/azure/AzureMonitorTask.java index a809ba2..d75aa5b 100644 --- a/src/main/java/com/appdynamics/monitors/azure/AzureMonitorTask.java +++ b/src/main/java/com/appdynamics/monitors/azure/AzureMonitorTask.java @@ -29,6 +29,8 @@ class AzureMonitorTask implements AMonitorTaskRunnable{ private final AtomicInteger azureResourcesCallCount = new AtomicInteger(0); + private static long startTime = System.currentTimeMillis(); + private final MonitorContextConfiguration configuration; private final MetricWriteHelper metricWriteHelper; private final Map subscription; @@ -46,7 +48,10 @@ class AzureMonitorTask implements AMonitorTaskRunnable{ @Override public void onTaskComplete() { - logger.info("Task Complete"); + logger.info("Task Completed for subscription {}", subscription.get("subscriptionId").toString()); + long finishTime = System.currentTimeMillis(); + long totalTime = finishTime - startTime; + logger.debug("Total time: " + (totalTime / 1000.0f) + " ms"); // logger.debug("azureResourcesCallCount: " + this.azureResourcesCallCount); } From e9dc255908cc67ca94e9fb86202b13727cb290ce Mon Sep 17 00:00:00 2001 From: Isi Ramirez Date: Mon, 20 Aug 2018 17:16:28 +0200 Subject: [PATCH 32/51] Add MSI credentials to AzureAuth --- .../appdynamics/monitors/azure/AzureAuth.java | 37 +++++++++++++------ 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/appdynamics/monitors/azure/AzureAuth.java b/src/main/java/com/appdynamics/monitors/azure/AzureAuth.java index 84e5a81..aa93f42 100644 --- a/src/main/java/com/appdynamics/monitors/azure/AzureAuth.java +++ b/src/main/java/com/appdynamics/monitors/azure/AzureAuth.java @@ -17,6 +17,7 @@ import com.microsoft.aad.adal4j.ClientCredential; import com.microsoft.azure.AzureEnvironment; import com.microsoft.azure.credentials.ApplicationTokenCredentials; +import com.microsoft.azure.credentials.MSICredentials; import com.microsoft.azure.management.Azure; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,6 +33,7 @@ class AzureAuth { private static final Logger logger = LoggerFactory.getLogger(AzureAuth.class); static void getAzureAuth(Map subscription) { + Boolean useMSI = subscription.get("useMSI") == null ? false : Boolean.valueOf(subscription.get("useMSI").toString()); String keyvaultClientSecretUrl = (String) subscription.get("keyvaultClientSecretUrl"); String keyvaultClientId = (String) subscription.get("keyvaultClientId"); String keyvaultClientKey = (String) subscription.get("keyvaultClientKey"); @@ -59,19 +61,32 @@ static void getAzureAuth(Map subscription) { clientKey = keyVaultResponse.get("value").textValue(); } - ApplicationTokenCredentials applicationTokenCredentials = new ApplicationTokenCredentials( - clientId, - tenantId, - clientKey, - AzureEnvironment.AZURE); - if (logger.isDebugEnabled()) { - Constants.azureMonitorAuth = Azure.configure().withLogLevel(com.microsoft.rest.LogLevel.BASIC).authenticate(applicationTokenCredentials); + if (useMSI) { + MSICredentials credentials = new MSICredentials(AzureEnvironment.AZURE); + if (logger.isDebugEnabled()) { + Constants.azureMonitorAuth = Azure.configure() + .withLogLevel(com.microsoft.rest.LogLevel.BASIC) + .authenticate(credentials); + } else { + Constants.azureMonitorAuth = Azure.authenticate(credentials); + } } else { - Constants.azureMonitorAuth = Azure.authenticate(applicationTokenCredentials); + ApplicationTokenCredentials applicationTokenCredentials = new ApplicationTokenCredentials( + clientId, + tenantId, + clientKey, + AzureEnvironment.AZURE); + if (logger.isDebugEnabled()) { + Constants.azureMonitorAuth = Azure.configure() + .withLogLevel(com.microsoft.rest.LogLevel.BASIC) + .authenticate(applicationTokenCredentials); + } else { + Constants.azureMonitorAuth = Azure.authenticate(applicationTokenCredentials); + } + AuthenticationResult azureAuthResult = getAuthenticationResult(clientId, clientKey, tenantId, Constants.AZURE_MANAGEMENT_URL); + Constants.azureAuthResult = azureAuthResult; + logger.debug("Bearer {}", azureAuthResult.getAccessToken()); } - AuthenticationResult azureAuthResult = getAuthenticationResult(clientId, clientKey, tenantId, Constants.AZURE_MANAGEMENT_URL); - Constants.azureAuthResult = azureAuthResult; - logger.debug("Bearer {}", azureAuthResult.getAccessToken()); } private static AuthenticationResult getAuthenticationResult(String Id, String Key, String tenantId, String resourceUrl){ From c296b0bf5685af5725faa99e6d32278c414b6875 Mon Sep 17 00:00:00 2001 From: Isi Ramirez Date: Tue, 21 Aug 2018 09:53:13 +0200 Subject: [PATCH 33/51] Store access token in Constants --- .../java/com/appdynamics/monitors/azure/AzureAuth.java | 7 +++---- .../com/appdynamics/monitors/azure/AzureRestOperation.java | 4 ++-- .../appdynamics/monitors/azure/metrics/AzureMetrics.java | 2 +- .../com/appdynamics/monitors/azure/utils/Constants.java | 2 +- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/appdynamics/monitors/azure/AzureAuth.java b/src/main/java/com/appdynamics/monitors/azure/AzureAuth.java index aa93f42..60f5845 100644 --- a/src/main/java/com/appdynamics/monitors/azure/AzureAuth.java +++ b/src/main/java/com/appdynamics/monitors/azure/AzureAuth.java @@ -56,7 +56,7 @@ static void getAzureAuth(Map subscription) { URL keyvaultUrl = Utilities.getUrl(keyvaultClientSecretUrl + "?" + "api-version" + "=" + subscription.get("keyvault-api-version")); AuthenticationResult azureKeyVaultAuth = getAuthenticationResult(keyvaultClientId, keyvaultClientKey, tenantId, Constants.AZURE_VAULT_URL); - JsonNode keyVaultResponse = AzureRestOperation.doGet(azureKeyVaultAuth, keyvaultUrl); + JsonNode keyVaultResponse = AzureRestOperation.doGet(azureKeyVaultAuth.getAccessToken(), keyvaultUrl); assert keyVaultResponse != null; clientKey = keyVaultResponse.get("value").textValue(); } @@ -84,7 +84,7 @@ static void getAzureAuth(Map subscription) { Constants.azureMonitorAuth = Azure.authenticate(applicationTokenCredentials); } AuthenticationResult azureAuthResult = getAuthenticationResult(clientId, clientKey, tenantId, Constants.AZURE_MANAGEMENT_URL); - Constants.azureAuthResult = azureAuthResult; + Constants.accessToken = azureAuthResult.getAccessToken(); logger.debug("Bearer {}", azureAuthResult.getAccessToken()); } } @@ -95,8 +95,7 @@ private static AuthenticationResult getAuthenticationResult(String Id, String Ke String authority = "https://login.microsoftonline.com/" + tenantId; try { - AuthenticationContext context; - context = new AuthenticationContext(authority, false, service); + AuthenticationContext context = new AuthenticationContext(authority, false, service); ClientCredential cred = new ClientCredential(Id, Key); Future future = context.acquireToken(resourceUrl, cred, null); result = future.get(); diff --git a/src/main/java/com/appdynamics/monitors/azure/AzureRestOperation.java b/src/main/java/com/appdynamics/monitors/azure/AzureRestOperation.java index c8b905c..68f8a2a 100644 --- a/src/main/java/com/appdynamics/monitors/azure/AzureRestOperation.java +++ b/src/main/java/com/appdynamics/monitors/azure/AzureRestOperation.java @@ -22,7 +22,7 @@ public class AzureRestOperation { private static final Logger logger = LoggerFactory.getLogger(AzureRestOperation.class); - public static JsonNode doGet(AuthenticationResult azureAuth, URL url) { + public static JsonNode doGet(String accessToken, URL url) { try { logger.debug("--> GET " + url); ObjectMapper objectMapper = new ObjectMapper(); @@ -31,7 +31,7 @@ public static JsonNode doGet(AuthenticationResult azureAuth, URL url) { conn = (HttpURLConnection) url.openConnection(); conn.setDoOutput(true); conn.setRequestMethod("GET"); - conn.setRequestProperty("Authorization", "Bearer " + azureAuth.getAccessToken()); + conn.setRequestProperty("Authorization", "Bearer " + accessToken); conn.setRequestProperty("Content-Type", "application/json"); BufferedReader br = new BufferedReader(new InputStreamReader( (conn.getInputStream()))); diff --git a/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java b/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java index 3263446..6e8f9e6 100644 --- a/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java +++ b/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java @@ -256,7 +256,7 @@ private JsonNode getAzureMetrics(String apiEndpointBase, String metricNames, } URL apiEndpointFull = new URL(url); azureMetricsCallCount.incrementAndGet(); - JsonNode apiResponse = AzureRestOperation.doGet(Constants.azureAuthResult, apiEndpointFull); + JsonNode apiResponse = AzureRestOperation.doGet(Constants.accessToken, apiEndpointFull); return apiResponse; } diff --git a/src/main/java/com/appdynamics/monitors/azure/utils/Constants.java b/src/main/java/com/appdynamics/monitors/azure/utils/Constants.java index d45f118..0b3d2e1 100644 --- a/src/main/java/com/appdynamics/monitors/azure/utils/Constants.java +++ b/src/main/java/com/appdynamics/monitors/azure/utils/Constants.java @@ -20,5 +20,5 @@ public class Constants { public static final String AZURE_VAULT_URL = "https://vault.azure.net"; public static final int AZURE_METRICS_CHUNK_SIZE = 20; public static Azure.Authenticated azureMonitorAuth = null; - public static AuthenticationResult azureAuthResult; + public static String accessToken; } From e5623decdfbf770a37503e3f49f230e7fc385836 Mon Sep 17 00:00:00 2001 From: Isi Ramirez Date: Tue, 21 Aug 2018 10:59:42 +0200 Subject: [PATCH 34/51] Refactor doGet to allow msi token api calls --- .../appdynamics/monitors/azure/AzureAuth.java | 61 +++++++++++++------ .../monitors/azure/AzureRestOperation.java | 16 ++++- .../monitors/azure/metrics/AzureMetrics.java | 2 +- .../monitors/azure/utils/Constants.java | 3 +- 4 files changed, 59 insertions(+), 23 deletions(-) diff --git a/src/main/java/com/appdynamics/monitors/azure/AzureAuth.java b/src/main/java/com/appdynamics/monitors/azure/AzureAuth.java index 60f5845..1978d93 100644 --- a/src/main/java/com/appdynamics/monitors/azure/AzureAuth.java +++ b/src/main/java/com/appdynamics/monitors/azure/AzureAuth.java @@ -21,8 +21,12 @@ import com.microsoft.azure.management.Azure; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + +import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URL; +import java.net.URLEncoder; +import java.util.HashMap; import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -44,23 +48,6 @@ static void getAzureAuth(Map subscription) { String encryptedClientKey = (String) subscription.get("encryptedClientKey"); String encryptedKeyvaultClientKey = (String) subscription.get("encryptedKeyvaultClientKey"); - if (!Strings.isNullOrEmpty(encryptionKey) && !Strings.isNullOrEmpty(encryptedClientKey)) { - clientKey = Utilities.getDecryptedKey(encryptedClientKey, encryptionKey); - } - - if (!Strings.isNullOrEmpty(encryptionKey) && !Strings.isNullOrEmpty(encryptedKeyvaultClientKey)) { - keyvaultClientKey = Utilities.getDecryptedKey(encryptedKeyvaultClientKey, encryptionKey); - } - - if (!Strings.isNullOrEmpty(keyvaultClientId) && !Strings.isNullOrEmpty(keyvaultClientKey) && !Strings.isNullOrEmpty(keyvaultClientSecretUrl)){ - URL keyvaultUrl = Utilities.getUrl(keyvaultClientSecretUrl + "?" + "api-version" + - "=" + subscription.get("keyvault-api-version")); - AuthenticationResult azureKeyVaultAuth = getAuthenticationResult(keyvaultClientId, keyvaultClientKey, tenantId, Constants.AZURE_VAULT_URL); - JsonNode keyVaultResponse = AzureRestOperation.doGet(azureKeyVaultAuth.getAccessToken(), keyvaultUrl); - assert keyVaultResponse != null; - clientKey = keyVaultResponse.get("value").textValue(); - } - if (useMSI) { MSICredentials credentials = new MSICredentials(AzureEnvironment.AZURE); if (logger.isDebugEnabled()) { @@ -70,7 +57,29 @@ static void getAzureAuth(Map subscription) { } else { Constants.azureMonitorAuth = Azure.authenticate(credentials); } + + Constants.accessToken = getMSIAccessToken(subscription); } else { + + if (!Strings.isNullOrEmpty(encryptionKey) && !Strings.isNullOrEmpty(encryptedClientKey)) { + clientKey = Utilities.getDecryptedKey(encryptedClientKey, encryptionKey); + } + + if (!Strings.isNullOrEmpty(encryptionKey) && !Strings.isNullOrEmpty(encryptedKeyvaultClientKey)) { + keyvaultClientKey = Utilities.getDecryptedKey(encryptedKeyvaultClientKey, encryptionKey); + } + + if (!Strings.isNullOrEmpty(keyvaultClientId) && !Strings.isNullOrEmpty(keyvaultClientKey) && !Strings.isNullOrEmpty(keyvaultClientSecretUrl)){ + URL keyvaultUrl = Utilities.getUrl(keyvaultClientSecretUrl + "?" + "api-version" + + "=" + subscription.get("keyvault-api-version")); + AuthenticationResult azureKeyVaultAuth = getAuthenticationResult(keyvaultClientId, keyvaultClientKey, tenantId, Constants.AZURE_VAULT_URL); + Constants.accessToken = azureKeyVaultAuth.getAccessToken(); + logger.debug("Bearer {}", azureKeyVaultAuth.getAccessToken()); + JsonNode keyVaultResponse = AzureRestOperation.doGet(keyvaultUrl); + assert keyVaultResponse != null; + clientKey = keyVaultResponse.get("value").textValue(); + } + ApplicationTokenCredentials applicationTokenCredentials = new ApplicationTokenCredentials( clientId, tenantId, @@ -89,6 +98,24 @@ static void getAzureAuth(Map subscription) { } } + private static String getMSIAccessToken(Map subscription) { + String resource = null; + try { + resource = URLEncoder.encode("https://management.azure.com/", "UTF-8"); + } catch (UnsupportedEncodingException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + URL url = Utilities.getUrl(Constants.AZURE_MSI_TOKEN_ENDPOINT + + "?api-version=" + subscription.get("msi-token-api-version") + + "&resource=" + resource); + Map headers = new HashMap<>(); + headers.put("Metadata", "true"); + JsonNode keyVaultResponse = AzureRestOperation.doGet(url, headers); + + return keyVaultResponse.get("access_token").textValue(); + } + private static AuthenticationResult getAuthenticationResult(String Id, String Key, String tenantId, String resourceUrl){ ExecutorService service = Executors.newSingleThreadExecutor(); AuthenticationResult result = null; diff --git a/src/main/java/com/appdynamics/monitors/azure/AzureRestOperation.java b/src/main/java/com/appdynamics/monitors/azure/AzureRestOperation.java index 68f8a2a..8a1a7da 100644 --- a/src/main/java/com/appdynamics/monitors/azure/AzureRestOperation.java +++ b/src/main/java/com/appdynamics/monitors/azure/AzureRestOperation.java @@ -8,9 +8,9 @@ package com.appdynamics.monitors.azure; +import com.appdynamics.monitors.azure.utils.Constants; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.microsoft.aad.adal4j.AuthenticationResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.net.ssl.*; @@ -18,11 +18,19 @@ import java.net.HttpURLConnection; import java.net.URL; import java.security.*; +import java.util.HashMap; +import java.util.Map; public class AzureRestOperation { private static final Logger logger = LoggerFactory.getLogger(AzureRestOperation.class); - public static JsonNode doGet(String accessToken, URL url) { + public static JsonNode doGet(URL url) { + Map headers = new HashMap<>(); + headers.put("Authorization", "Bearer " + Constants.accessToken); + return doGet(url, headers); + } + + public static JsonNode doGet(URL url, Map headers) { try { logger.debug("--> GET " + url); ObjectMapper objectMapper = new ObjectMapper(); @@ -31,7 +39,9 @@ public static JsonNode doGet(String accessToken, URL url) { conn = (HttpURLConnection) url.openConnection(); conn.setDoOutput(true); conn.setRequestMethod("GET"); - conn.setRequestProperty("Authorization", "Bearer " + accessToken); + for (Map.Entry header : headers.entrySet()) { + conn.setRequestProperty(header.getKey(), header.getValue()); + } conn.setRequestProperty("Content-Type", "application/json"); BufferedReader br = new BufferedReader(new InputStreamReader( (conn.getInputStream()))); diff --git a/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java b/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java index 6e8f9e6..3598711 100644 --- a/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java +++ b/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java @@ -256,7 +256,7 @@ private JsonNode getAzureMetrics(String apiEndpointBase, String metricNames, } URL apiEndpointFull = new URL(url); azureMetricsCallCount.incrementAndGet(); - JsonNode apiResponse = AzureRestOperation.doGet(Constants.accessToken, apiEndpointFull); + JsonNode apiResponse = AzureRestOperation.doGet(apiEndpointFull); return apiResponse; } diff --git a/src/main/java/com/appdynamics/monitors/azure/utils/Constants.java b/src/main/java/com/appdynamics/monitors/azure/utils/Constants.java index 0b3d2e1..68294b0 100644 --- a/src/main/java/com/appdynamics/monitors/azure/utils/Constants.java +++ b/src/main/java/com/appdynamics/monitors/azure/utils/Constants.java @@ -8,9 +8,7 @@ package com.appdynamics.monitors.azure.utils; -import com.microsoft.aad.adal4j.AuthenticationResult; import com.microsoft.azure.management.Azure; -import com.singularity.ee.agent.systemagent.api.MetricWriter; public class Constants { public static final String DEFAULT_METRIC_PREFIX = "Custom Metrics|AzureMonitor|"; @@ -18,6 +16,7 @@ public class Constants { public static final String TEST_CONFIG_FILE = "src/test/resources/conf/test-config.yml"; public static final String AZURE_MANAGEMENT_URL = "https://management.azure.com"; public static final String AZURE_VAULT_URL = "https://vault.azure.net"; + public static final String AZURE_MSI_TOKEN_ENDPOINT = "http://169.254.169.254/metadata/identity/oauth2/token"; public static final int AZURE_METRICS_CHUNK_SIZE = 20; public static Azure.Authenticated azureMonitorAuth = null; public static String accessToken; From e9f98f56414a67a50e8d7b732c63d40796dd4961 Mon Sep 17 00:00:00 2001 From: Isi Ramirez Date: Tue, 21 Aug 2018 11:41:21 +0200 Subject: [PATCH 35/51] Refactor AzureAuth --- .../appdynamics/monitors/azure/AzureAuth.java | 124 ++++++++++-------- .../monitors/azure/AzureMonitor.java | 2 +- 2 files changed, 70 insertions(+), 56 deletions(-) diff --git a/src/main/java/com/appdynamics/monitors/azure/AzureAuth.java b/src/main/java/com/appdynamics/monitors/azure/AzureAuth.java index 1978d93..28b4156 100644 --- a/src/main/java/com/appdynamics/monitors/azure/AzureAuth.java +++ b/src/main/java/com/appdynamics/monitors/azure/AzureAuth.java @@ -8,6 +8,20 @@ package com.appdynamics.monitors.azure; +import java.io.UnsupportedEncodingException; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLEncoder; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import com.appdynamics.monitors.azure.utils.Constants; import com.appdynamics.monitors.azure.utils.Utilities; import com.fasterxml.jackson.databind.JsonNode; @@ -19,39 +33,17 @@ import com.microsoft.azure.credentials.ApplicationTokenCredentials; import com.microsoft.azure.credentials.MSICredentials; import com.microsoft.azure.management.Azure; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.UnsupportedEncodingException; -import java.net.MalformedURLException; -import java.net.URL; -import java.net.URLEncoder; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; class AzureAuth { private static final Logger logger = LoggerFactory.getLogger(AzureAuth.class); static void getAzureAuth(Map subscription) { Boolean useMSI = subscription.get("useMSI") == null ? false : Boolean.valueOf(subscription.get("useMSI").toString()); - String keyvaultClientSecretUrl = (String) subscription.get("keyvaultClientSecretUrl"); - String keyvaultClientId = (String) subscription.get("keyvaultClientId"); - String keyvaultClientKey = (String) subscription.get("keyvaultClientKey"); - String clientId = (String) subscription.get("clientId"); - String clientKey = (String) subscription.get("clientKey"); - String tenantId = (String) subscription.get("tenantId"); - String encryptionKey = (String) subscription.get("encryption-key"); - String encryptedClientKey = (String) subscription.get("encryptedClientKey"); - String encryptedKeyvaultClientKey = (String) subscription.get("encryptedKeyvaultClientKey"); - if (useMSI) { MSICredentials credentials = new MSICredentials(AzureEnvironment.AZURE); if (logger.isDebugEnabled()) { - Constants.azureMonitorAuth = Azure.configure() + Constants.azureMonitorAuth = Azure + .configure() .withLogLevel(com.microsoft.rest.LogLevel.BASIC) .authenticate(credentials); } else { @@ -59,43 +51,65 @@ static void getAzureAuth(Map subscription) { } Constants.accessToken = getMSIAccessToken(subscription); + logger.debug("MSI: Bearer {}", Constants.accessToken); + } else { + authorizeWithClientCredentials(subscription); + } + } - if (!Strings.isNullOrEmpty(encryptionKey) && !Strings.isNullOrEmpty(encryptedClientKey)) { - clientKey = Utilities.getDecryptedKey(encryptedClientKey, encryptionKey); - } + private static void authorizeWithClientCredentials(Map subscription) { + String clientId = (String) subscription.get("clientId"); + String clientKey = (String) subscription.get("clientKey"); + String tenantId = (String) subscription.get("tenantId"); - if (!Strings.isNullOrEmpty(encryptionKey) && !Strings.isNullOrEmpty(encryptedKeyvaultClientKey)) { - keyvaultClientKey = Utilities.getDecryptedKey(encryptedKeyvaultClientKey, encryptionKey); - } + String encryptedOrKeyVaultClientKey= getEncryptedOrKeyVaultClientKey(subscription); + if (encryptedOrKeyVaultClientKey != null) clientKey = encryptedOrKeyVaultClientKey; - if (!Strings.isNullOrEmpty(keyvaultClientId) && !Strings.isNullOrEmpty(keyvaultClientKey) && !Strings.isNullOrEmpty(keyvaultClientSecretUrl)){ - URL keyvaultUrl = Utilities.getUrl(keyvaultClientSecretUrl + "?" + "api-version" + - "=" + subscription.get("keyvault-api-version")); - AuthenticationResult azureKeyVaultAuth = getAuthenticationResult(keyvaultClientId, keyvaultClientKey, tenantId, Constants.AZURE_VAULT_URL); - Constants.accessToken = azureKeyVaultAuth.getAccessToken(); - logger.debug("Bearer {}", azureKeyVaultAuth.getAccessToken()); - JsonNode keyVaultResponse = AzureRestOperation.doGet(keyvaultUrl); - assert keyVaultResponse != null; - clientKey = keyVaultResponse.get("value").textValue(); - } + ApplicationTokenCredentials applicationTokenCredentials = new ApplicationTokenCredentials( + clientId, + tenantId, + clientKey, + AzureEnvironment.AZURE); + if (logger.isDebugEnabled()) { + Constants.azureMonitorAuth = Azure.configure() + .withLogLevel(com.microsoft.rest.LogLevel.BASIC) + .authenticate(applicationTokenCredentials); + } else { + Constants.azureMonitorAuth = Azure.authenticate(applicationTokenCredentials); + } + AuthenticationResult azureAuthResult = getAuthenticationResult(clientId, clientKey, tenantId, Constants.AZURE_MANAGEMENT_URL); + Constants.accessToken = azureAuthResult.getAccessToken(); + logger.debug("Bearer {}", azureAuthResult.getAccessToken()); + } - ApplicationTokenCredentials applicationTokenCredentials = new ApplicationTokenCredentials( - clientId, - tenantId, - clientKey, - AzureEnvironment.AZURE); - if (logger.isDebugEnabled()) { - Constants.azureMonitorAuth = Azure.configure() - .withLogLevel(com.microsoft.rest.LogLevel.BASIC) - .authenticate(applicationTokenCredentials); - } else { - Constants.azureMonitorAuth = Azure.authenticate(applicationTokenCredentials); - } - AuthenticationResult azureAuthResult = getAuthenticationResult(clientId, clientKey, tenantId, Constants.AZURE_MANAGEMENT_URL); - Constants.accessToken = azureAuthResult.getAccessToken(); - logger.debug("Bearer {}", azureAuthResult.getAccessToken()); + private static String getEncryptedOrKeyVaultClientKey(Map subscription) { + String clientKey = null; + String tenantId = (String) subscription.get("tenantId"); + String encryptionKey = (String) subscription.get("encryption-key"); + String encryptedClientKey = (String) subscription.get("encryptedClientKey"); + String encryptedKeyvaultClientKey = (String) subscription.get("encryptedKeyvaultClientKey"); + if (!Strings.isNullOrEmpty(encryptionKey) && !Strings.isNullOrEmpty(encryptedClientKey)) { + clientKey = Utilities.getDecryptedKey(encryptedClientKey, encryptionKey); + } + String keyvaultClientSecretUrl = (String) subscription.get("keyvaultClientSecretUrl"); + String keyvaultClientId = (String) subscription.get("keyvaultClientId"); + String keyvaultClientKey = (String) subscription.get("keyvaultClientKey"); + if (!Strings.isNullOrEmpty(encryptionKey) && !Strings.isNullOrEmpty(encryptedKeyvaultClientKey)) { + keyvaultClientKey = Utilities.getDecryptedKey(encryptedKeyvaultClientKey, encryptionKey); + } + + if (!Strings.isNullOrEmpty(keyvaultClientId) && !Strings.isNullOrEmpty(keyvaultClientKey) && !Strings.isNullOrEmpty(keyvaultClientSecretUrl)){ + URL keyvaultUrl = Utilities.getUrl(keyvaultClientSecretUrl + "?" + "api-version" + + "=" + subscription.get("keyvault-api-version")); + AuthenticationResult azureKeyVaultAuth = getAuthenticationResult(keyvaultClientId, keyvaultClientKey, tenantId, Constants.AZURE_VAULT_URL); + Constants.accessToken = azureKeyVaultAuth.getAccessToken(); + logger.debug("Bearer {}", azureKeyVaultAuth.getAccessToken()); + JsonNode keyVaultResponse = AzureRestOperation.doGet(keyvaultUrl); + assert keyVaultResponse != null; + clientKey = keyVaultResponse.get("value").textValue(); } + return clientKey; } private static String getMSIAccessToken(Map subscription) { diff --git a/src/main/java/com/appdynamics/monitors/azure/AzureMonitor.java b/src/main/java/com/appdynamics/monitors/azure/AzureMonitor.java index 3f7a3cd..948d40a 100644 --- a/src/main/java/com/appdynamics/monitors/azure/AzureMonitor.java +++ b/src/main/java/com/appdynamics/monitors/azure/AzureMonitor.java @@ -12,6 +12,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import org.slf4j.LoggerFactory; @@ -38,7 +39,6 @@ public String getMonitorName() { @Override public void onComplete() { logger.info("Monitor Completed"); - System.exit(0); } @Override From fa00d9f8737eb7f5bde0ecbae069953903b16ec8 Mon Sep 17 00:00:00 2001 From: Isi Ramirez Date: Thu, 23 Aug 2018 10:26:04 +0200 Subject: [PATCH 36/51] Add timeout to monitor --- src/main/java/com/appdynamics/monitors/azure/AzureMonitor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/appdynamics/monitors/azure/AzureMonitor.java b/src/main/java/com/appdynamics/monitors/azure/AzureMonitor.java index 948d40a..27a6b17 100644 --- a/src/main/java/com/appdynamics/monitors/azure/AzureMonitor.java +++ b/src/main/java/com/appdynamics/monitors/azure/AzureMonitor.java @@ -51,7 +51,7 @@ protected void doRun(TasksExecutionServiceProvider tasksExecutionServiceProvider tasksExecutionServiceProvider.submit(subscription.get("subscriptionId").toString(),task); } try{ - countDownLatch.await(); + countDownLatch.await(45, TimeUnit.SECONDS); } catch (InterruptedException e) { e.printStackTrace(); } From 99133ae9e8270023cabb1ad5c9b08b767faa0164 Mon Sep 17 00:00:00 2001 From: Isi Ramirez Date: Thu, 23 Aug 2018 10:32:32 +0200 Subject: [PATCH 37/51] Remove unnecessary return that was causing the normal metrics not to be sent when using dimensions as well --- .../com/appdynamics/monitors/azure/metrics/AzureMetrics.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java b/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java index 3598711..a1e2845 100644 --- a/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java +++ b/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java @@ -192,8 +192,6 @@ private void generateMetrics(List resourceMetrics) throws Malf // TODO Auto-generated catch block e.printStackTrace(); } - - return; } } else { filteredMetrics.add(resourceMetric); From 4f9d2bc38de45d4f8f8a201f2728301ee6947c47 Mon Sep 17 00:00:00 2001 From: Isi Ramirez Date: Thu, 23 Aug 2018 15:30:38 +0200 Subject: [PATCH 38/51] Change config filename to config.yml --- .../java/com/appdynamics/monitors/azure/utils/Constants.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/appdynamics/monitors/azure/utils/Constants.java b/src/main/java/com/appdynamics/monitors/azure/utils/Constants.java index 68294b0..8ec37d3 100644 --- a/src/main/java/com/appdynamics/monitors/azure/utils/Constants.java +++ b/src/main/java/com/appdynamics/monitors/azure/utils/Constants.java @@ -13,7 +13,7 @@ public class Constants { public static final String DEFAULT_METRIC_PREFIX = "Custom Metrics|AzureMonitor|"; public static final String METRIC_SEPARATOR = "|"; - public static final String TEST_CONFIG_FILE = "src/test/resources/conf/test-config.yml"; + public static final String TEST_CONFIG_FILE = "src/test/resources/conf/config.yml"; public static final String AZURE_MANAGEMENT_URL = "https://management.azure.com"; public static final String AZURE_VAULT_URL = "https://vault.azure.net"; public static final String AZURE_MSI_TOKEN_ENDPOINT = "http://169.254.169.254/metadata/identity/oauth2/token"; From dded1dafcf4085e368adb8a11c735a98e079c3a4 Mon Sep 17 00:00:00 2001 From: Isi Ramirez Date: Thu, 23 Aug 2018 17:27:38 +0200 Subject: [PATCH 39/51] Configure mvn to launch tests --- pom.xml | 36 ++++++++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index 4300d92..416efd9 100644 --- a/pom.xml +++ b/pom.xml @@ -39,12 +39,12 @@ httpclient 4.3.5 - - - org.apache.commons - commons-collections4 - 4.0 - + + + org.apache.commons + commons-collections4 + 4.0 + commons-io commons-io @@ -196,6 +196,30 @@ ${project.artifactId}-${project.version} + + maven-surefire-plugin + 2.21.0 + + + **/Test*.java + **/*Test.java + **/*Tests.java + **/*TestCase.java + + + + + org.junit.platform + junit-platform-surefire-provider + 1.2.0 + + + org.junit.vintage + junit-vintage-engine + 5.2.0 + + + From bd2cce57ee8c869c8a5163bedbc98fa6d4d0d005 Mon Sep 17 00:00:00 2001 From: Isi Ramirez Date: Mon, 27 Aug 2018 19:32:42 +0200 Subject: [PATCH 40/51] Refactor auth class --- .../appdynamics/monitors/azure/AzureAuth.java | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/appdynamics/monitors/azure/AzureAuth.java b/src/main/java/com/appdynamics/monitors/azure/AzureAuth.java index 28b4156..ce61b65 100644 --- a/src/main/java/com/appdynamics/monitors/azure/AzureAuth.java +++ b/src/main/java/com/appdynamics/monitors/azure/AzureAuth.java @@ -33,21 +33,23 @@ import com.microsoft.azure.credentials.ApplicationTokenCredentials; import com.microsoft.azure.credentials.MSICredentials; import com.microsoft.azure.management.Azure; +import com.microsoft.azure.management.Azure.Authenticated; -class AzureAuth { +public class AzureAuth { private static final Logger logger = LoggerFactory.getLogger(AzureAuth.class); + private static Authenticated azureMonitorAuth; - static void getAzureAuth(Map subscription) { + public static void authorizeAzure(Map subscription) { Boolean useMSI = subscription.get("useMSI") == null ? false : Boolean.valueOf(subscription.get("useMSI").toString()); if (useMSI) { MSICredentials credentials = new MSICredentials(AzureEnvironment.AZURE); if (logger.isDebugEnabled()) { - Constants.azureMonitorAuth = Azure + azureMonitorAuth = Azure .configure() .withLogLevel(com.microsoft.rest.LogLevel.BASIC) .authenticate(credentials); } else { - Constants.azureMonitorAuth = Azure.authenticate(credentials); + azureMonitorAuth = Azure.authenticate(credentials); } Constants.accessToken = getMSIAccessToken(subscription); @@ -72,11 +74,11 @@ private static void authorizeWithClientCredentials(Map subscription) clientKey, AzureEnvironment.AZURE); if (logger.isDebugEnabled()) { - Constants.azureMonitorAuth = Azure.configure() + azureMonitorAuth = Azure.configure() .withLogLevel(com.microsoft.rest.LogLevel.BASIC) .authenticate(applicationTokenCredentials); } else { - Constants.azureMonitorAuth = Azure.authenticate(applicationTokenCredentials); + azureMonitorAuth = Azure.authenticate(applicationTokenCredentials); } AuthenticationResult azureAuthResult = getAuthenticationResult(clientId, clientKey, tenantId, Constants.AZURE_MANAGEMENT_URL); Constants.accessToken = azureAuthResult.getAccessToken(); @@ -150,4 +152,8 @@ private static AuthenticationResult getAuthenticationResult(String Id, String Ke service.shutdown(); return result; } + + public static Authenticated getAzureMonitorAuth() { + return azureMonitorAuth; + } } From fa9b3046d7fa132d96b2cd599ee3119f50143f3f Mon Sep 17 00:00:00 2001 From: Isi Ramirez Date: Mon, 27 Aug 2018 19:34:40 +0200 Subject: [PATCH 41/51] Refactor monitor and monitor task --- .../monitors/azure/AzureMonitor.java | 4 +- .../monitors/azure/AzureMonitorTask.java | 41 +++++++++---------- 2 files changed, 22 insertions(+), 23 deletions(-) diff --git a/src/main/java/com/appdynamics/monitors/azure/AzureMonitor.java b/src/main/java/com/appdynamics/monitors/azure/AzureMonitor.java index 27a6b17..2f7b4ff 100644 --- a/src/main/java/com/appdynamics/monitors/azure/AzureMonitor.java +++ b/src/main/java/com/appdynamics/monitors/azure/AzureMonitor.java @@ -19,6 +19,7 @@ import com.appdynamics.extensions.ABaseMonitor; import com.appdynamics.extensions.TasksExecutionServiceProvider; import com.appdynamics.extensions.util.AssertUtils; +import com.appdynamics.monitors.azure.utils.AzureAPIWrapper; import com.appdynamics.monitors.azure.utils.Constants; import com.singularity.ee.agent.systemagent.api.exception.TaskExecutionException; @@ -47,7 +48,8 @@ protected void doRun(TasksExecutionServiceProvider tasksExecutionServiceProvider AssertUtils.assertNotNull(subscriptions, "The 'subscriptions' section in config.yml is not initialised"); CountDownLatch countDownLatch = new CountDownLatch(getTaskCount()); for (Map subscription : subscriptions) { - AzureMonitorTask task = new AzureMonitorTask(getContextConfiguration(), tasksExecutionServiceProvider.getMetricWriteHelper(), subscription, countDownLatch); + AzureAPIWrapper azure = new AzureAPIWrapper(subscription); + AzureMonitorTask task = new AzureMonitorTask(getContextConfiguration(), tasksExecutionServiceProvider.getMetricWriteHelper(), subscription, countDownLatch, azure); tasksExecutionServiceProvider.submit(subscription.get("subscriptionId").toString(),task); } try{ diff --git a/src/main/java/com/appdynamics/monitors/azure/AzureMonitorTask.java b/src/main/java/com/appdynamics/monitors/azure/AzureMonitorTask.java index d75aa5b..8eaacdf 100644 --- a/src/main/java/com/appdynamics/monitors/azure/AzureMonitorTask.java +++ b/src/main/java/com/appdynamics/monitors/azure/AzureMonitorTask.java @@ -11,24 +11,19 @@ import com.appdynamics.extensions.AMonitorTaskRunnable; import com.appdynamics.extensions.MetricWriteHelper; import com.appdynamics.extensions.conf.MonitorContextConfiguration; -import com.appdynamics.extensions.util.AssertUtils; import com.appdynamics.monitors.azure.metrics.AzureMetrics; -import com.appdynamics.monitors.azure.utils.Constants; -import com.microsoft.azure.PagedList; -import com.microsoft.azure.management.Azure; -import com.microsoft.azure.management.resources.GenericResource; +import com.appdynamics.monitors.azure.utils.AzureAPIWrapper; +import com.appdynamics.monitors.azure.utils.Resource; + import org.slf4j.LoggerFactory; import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.atomic.AtomicInteger; @SuppressWarnings("unchecked") class AzureMonitorTask implements AMonitorTaskRunnable{ private static final org.slf4j.Logger logger = LoggerFactory.getLogger(AzureMonitorTask.class); - private final AtomicInteger azureResourcesCallCount = new AtomicInteger(0); - private static long startTime = System.currentTimeMillis(); private final MonitorContextConfiguration configuration; @@ -37,13 +32,16 @@ class AzureMonitorTask implements AMonitorTaskRunnable{ private final List> resourceGroupFilters; private final CountDownLatch countDownLatch; - AzureMonitorTask(MonitorContextConfiguration configuration, MetricWriteHelper metricWriteHelper, Map subscription, CountDownLatch countDownLatch) { - this.configuration = configuration; - this.metricWriteHelper = metricWriteHelper; - this.subscription = subscription; - this.countDownLatch = countDownLatch; - resourceGroupFilters = (List>) subscription.get("resourceGroups"); - AssertUtils.assertNotNull(resourceGroupFilters, "The 'resourceGroupFilters' section in config.yml is either null or empty"); + private AzureAPIWrapper azure; + + AzureMonitorTask(MonitorContextConfiguration configuration, MetricWriteHelper metricWriteHelper, Map subscription, CountDownLatch countDownLatch, AzureAPIWrapper azure) { + this.configuration = configuration; + this.metricWriteHelper = metricWriteHelper; + this.subscription = subscription; + this.countDownLatch = countDownLatch; + this.resourceGroupFilters = (List>) subscription.get("resourceGroups"); + this.azure = azure; +// AssertUtils.assertNotNull(resourceGroupFilters, "The 'resourceGroupFilters' section in config.yml is either null or empty"); } @Override @@ -67,11 +65,10 @@ public void run() { countDownLatch.countDown(); } } - private void runTask(){ - AzureAuth.getAzureAuth(subscription); - Azure azure = Constants.azureMonitorAuth.withSubscription(subscription.get("subscriptionId").toString()); - this.azureResourcesCallCount.incrementAndGet(); - PagedList resources = azure.genericResources().list(); + private void runTask() throws Exception{ + azure.authorize(); + List resources = azure.getResources(); + if (resources == null || resources.size() == 0) throw new Exception("Resources list is null or empty"); for (Map resourceGroupFilter : resourceGroupFilters) { List> resourceTypeFilters = (List>) resourceGroupFilter.get("resourceTypes"); for (Map resourceTypeFilter : resourceTypeFilters) { @@ -79,10 +76,11 @@ private void runTask(){ for (Map resourceFilter : resourceFilters) { String currentResourceGroupFilter = resourceGroupFilter.get("resourceGroup").toString(); CountDownLatch countDownLatchAzure = new CountDownLatch(resources.size()); - for (GenericResource resource : resources) { + for (Resource resource : resources) { String currentResourceFilter = resourceFilter.get("resource").toString(); String currentResourceTypeFilter = resourceTypeFilter.get("resourceType").toString(); AzureMetrics azureMetricsTask = new AzureMetrics( + azure, resourceFilter, currentResourceGroupFilter, currentResourceFilter, @@ -91,7 +89,6 @@ private void runTask(){ subscription, countDownLatchAzure, metricWriteHelper, - azure, configuration.getMetricPrefix()); configuration.getContext().getExecutorService().execute("AzureMetrics", azureMetricsTask); From ca06492e19ebbcae1e0583673f733f1467a02909 Mon Sep 17 00:00:00 2001 From: Isi Ramirez Date: Mon, 27 Aug 2018 19:35:55 +0200 Subject: [PATCH 42/51] Refactor azure metrics --- .../monitors/azure/AzureRestOperation.java | 3 + .../monitors/azure/metrics/AzureMetrics.java | 145 +++++++----------- .../monitors/azure/utils/Constants.java | 4 +- 3 files changed, 58 insertions(+), 94 deletions(-) diff --git a/src/main/java/com/appdynamics/monitors/azure/AzureRestOperation.java b/src/main/java/com/appdynamics/monitors/azure/AzureRestOperation.java index 8a1a7da..09c3f27 100644 --- a/src/main/java/com/appdynamics/monitors/azure/AzureRestOperation.java +++ b/src/main/java/com/appdynamics/monitors/azure/AzureRestOperation.java @@ -48,6 +48,9 @@ public static JsonNode doGet(URL url, Map headers) { //noinspection StatementWithEmptyBody for (String line; (line = br.readLine()) != null; response += line); conn.disconnect(); + if (logger.isDebugEnabled()) { + logger.debug("API response: " + response); + } return objectMapper.readTree(response); } catch (IOException e) { logger.error("Error while processing GET on URL {}", url, e); diff --git a/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java b/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java index a1e2845..334b375 100644 --- a/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java +++ b/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java @@ -10,70 +10,60 @@ import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; -import java.net.URL; import java.net.URLEncoder; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.lang3.StringUtils; import org.apache.commons.collections4.ListUtils; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; import org.slf4j.LoggerFactory; import com.appdynamics.extensions.AMonitorTaskRunnable; import com.appdynamics.extensions.MetricWriteHelper; import com.appdynamics.extensions.metrics.Metric; -import com.appdynamics.monitors.azure.AzureRestOperation; import com.appdynamics.monitors.azure.utils.Constants; +import com.appdynamics.monitors.azure.utils.Resource; +import com.appdynamics.monitors.azure.utils.MetricDefinition; +import com.appdynamics.monitors.azure.utils.AzureAPIWrapper; import com.google.common.collect.Lists; -import com.microsoft.azure.management.Azure; -import com.microsoft.azure.management.monitor.MetricDefinition; -import com.microsoft.azure.management.resources.GenericResource; import com.singularity.ee.agent.systemagent.api.MetricWriter; import com.fasterxml.jackson.databind.JsonNode; @SuppressWarnings("unchecked") public class AzureMetrics implements AMonitorTaskRunnable { private static final org.slf4j.Logger logger = LoggerFactory.getLogger(AzureMetrics.class); - public final AtomicInteger azureMetricsCallCount = new AtomicInteger(0); - public final AtomicInteger azureMetricDefinitionsCallCount = new AtomicInteger(0); private final Map resourceFilter; private final String currentResourceGroupFilter; private final String currentResourceFilter; private final String currentResourceTypeFilter; - private final GenericResource resource; - private final Map subscription; + private final Resource resource; private final String subscriptionName; private final CountDownLatch countDownLatch; private final MetricWriteHelper metricWriteHelper; - private final Azure azure; private final String metricPrefix; private final List finalMetricList = Lists.newArrayList(); - private final DateTime recordDateTime = DateTime.now(DateTimeZone.UTC); + private AzureAPIWrapper azure; - public AzureMetrics(Map resourceFilter, + public AzureMetrics(AzureAPIWrapper azure, + Map resourceFilter, String currentResourceGroupFilter, String currentResourceFilter, String currentResourceTypeFilter, - GenericResource resource, + Resource resource, Map subscription, CountDownLatch countDownLatch, MetricWriteHelper metricWriteHelper, - Azure azure, String metricPrefix) { + this.azure = azure; this.resourceFilter = resourceFilter; this.currentResourceGroupFilter = currentResourceGroupFilter; this.currentResourceFilter = currentResourceFilter; this.currentResourceTypeFilter = currentResourceTypeFilter; this.resource = resource; - this.subscription = subscription; this.countDownLatch = countDownLatch; this.metricWriteHelper = metricWriteHelper; - this.azure = azure; this.metricPrefix = metricPrefix; if (subscription.containsKey("subscriptionName")){ subscriptionName = subscription.get("subscriptionName").toString(); @@ -107,22 +97,21 @@ private void runTask(){ // logger.debug("Resource type ({}): {} {}", resource.type().matches(currentResourceTypeFilter), resource.type(), currentResourceTypeFilter); // logger.debug("Resource group ({}): {} {}", resource.resourceGroupName().matches("(?i:" + currentResourceGroupFilter + ")"), resource.resourceGroupName(), currentResourceGroupFilter); // } - if (resource.name().matches(currentResourceFilter) && - resource.type().matches(currentResourceTypeFilter) && - resource.resourceGroupName().matches("(?i:" + currentResourceGroupFilter + ")")) { + if (resource.getName().matches(currentResourceFilter) && + resource.getType().matches(currentResourceTypeFilter) && + resource.getResourceGroupName().matches("(?i:" + currentResourceGroupFilter + ")")) { if (logger.isDebugEnabled()) { logger.debug("Working on Resource {} of Type {} in Group {} because of Resource Filter {} of Type Filter {} in Group Filter {}", - resource.name(), - resource.resourceType(), - resource.resourceGroupName(), + resource.getName(), + resource.getResourceType(), + resource.getResourceGroupName(), currentResourceTypeFilter, currentResourceTypeFilter, currentResourceGroupFilter); } - azureMetricDefinitionsCallCount.incrementAndGet(); - List resourceMetrics = azure.metricDefinitions().listByResource(resource.id()); + List metricDefinitions = azure.getMetricDefinitions(resource.getId()); try { - generateMetrics(resourceMetrics); + generateMetrics(metricDefinitions); } catch (MalformedURLException e) { // TODO Auto-generated catch block e.printStackTrace(); @@ -154,7 +143,7 @@ private void runTask(){ // } } - private void generateMetrics(List resourceMetrics) throws MalformedURLException, UnsupportedEncodingException { + private void generateMetrics(List metricDefinitions) throws MalformedURLException, UnsupportedEncodingException { List> metricConfigs = (List>) resourceFilter.get("metrics"); List filteredMetrics = Lists.newArrayList(); @@ -162,40 +151,28 @@ private void generateMetrics(List resourceMetrics) throws Malf List> filteredMetricsConfig = Lists.newArrayList(); for (Map metricConfig : metricConfigs) { - for (MetricDefinition resourceMetric : resourceMetrics) { + for (MetricDefinition metricDefinition : metricDefinitions) { String currentMetricConfig = metricConfig.get("metric").toString(); // if (logger.isDebugEnabled()) { // logger.debug("resourceMetric name ({})", resourceMetric.name().value()); // logger.debug("currentMetricConfig ({})", currentMetricConfig); // logger.debug("match ({})", resourceMetric.name().value().matches(currentMetricConfig), resourceMetric.name().value(), currentMetricConfig); // } - - String apiEndpointBase = resourceMetric.id().substring(0,StringUtils.ordinalIndexOf(resourceMetric.id(), "/", 11)); - Object filterObject = metricConfig.get("filter"); - String filter = filterObject == null ? null : filterObject.toString(); - Object aggregatorObject = metricConfig.get("aggregator"); - String aggregator = aggregatorObject == null ? "" : aggregatorObject.toString(); - Object timeRollUpObject = metricConfig.get("timeRollUp"); - String timeRollUp = timeRollUpObject == null ? "" : timeRollUpObject.toString(); - Object clusterRollUpObject = metricConfig.get("clusterRollUp"); - String clusterRollUp = clusterRollUpObject == null ? "" : clusterRollUpObject.toString(); - if (resourceMetric.name().value().matches(currentMetricConfig)) { - if (( filter != null && !filter.isEmpty() )|| !aggregator.isEmpty() || !timeRollUp.isEmpty() || !clusterRollUp.isEmpty() ) { - JsonNode apiResponse = getAzureMetrics(apiEndpointBase, URLEncoder.encode(resourceMetric.name().value(), "UTF-8"), filter); - if (apiResponse != null) { - if (logger.isDebugEnabled()) { - logger.debug("generateMetrics: API response: " + apiResponse.toString()); - } - try { - addMetric(resourceMetric, apiResponse, metricConfig); - } catch (Exception e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } + if (metricDefinition.getName().matches(currentMetricConfig)) { + Object filterObject = metricConfig.get("filter"); + String filter = filterObject == null ? null : filterObject.toString(); + boolean hasQualifiers = hasQualifiers(metricConfig); + if (( filter != null && !filter.isEmpty() ) || hasQualifiers ) { + JsonNode apiResponse = azure.getMetrics(metricDefinition, URLEncoder.encode(metricDefinition.getName(), "UTF-8"), filter); + try { + addMetric(metricDefinition, apiResponse, metricConfig); + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); } } else { - filteredMetrics.add(resourceMetric); - filteredMetricsNames.add(URLEncoder.encode(resourceMetric.name().value(), "UTF-8")); + filteredMetrics.add(metricDefinition); + filteredMetricsNames.add(URLEncoder.encode(metricDefinition.getName(), "UTF-8")); filteredMetricsConfig.add(metricConfig); } } else { @@ -212,6 +189,17 @@ private void generateMetrics(List resourceMetrics) throws Malf consumeAndImportAzureMetrics(filteredMetrics, filteredMetricsNames, filteredMetricsConfig); } + public boolean hasQualifiers(Map metricConfig) { + Object aggregatorObject = metricConfig.get("aggregator"); + String aggregator = aggregatorObject == null ? "" : aggregatorObject.toString(); + Object timeRollUpObject = metricConfig.get("timeRollUp"); + String timeRollUp = timeRollUpObject == null ? "" : timeRollUpObject.toString(); + Object clusterRollUpObject = metricConfig.get("clusterRollUp"); + String clusterRollUp = clusterRollUpObject == null ? "" : clusterRollUpObject.toString(); + boolean hasQualifiers = !aggregator.isEmpty() || !timeRollUp.isEmpty() || !clusterRollUp.isEmpty(); + return hasQualifiers; + } + private void consumeAndImportAzureMetrics(List filteredMetrics, List filteredMetricsNames, List> filteredMetricsConfig) throws MalformedURLException, UnsupportedEncodingException { if (!filteredMetrics.isEmpty()) { List> filteredMetricsChunks = ListUtils.partition(filteredMetrics, Constants.AZURE_METRICS_CHUNK_SIZE); @@ -221,44 +209,18 @@ private void consumeAndImportAzureMetrics(List filteredMetrics Iterator> filteredMetricsNamesIterator = filteredMetricsNamesChunks.iterator(); Iterator>> filteredMetricsConfigIterator = filteredMetricsConfigChunks.iterator(); - String apiEndpointBase = filteredMetrics.get(0).id().substring(0,StringUtils.ordinalIndexOf(filteredMetrics.get(0).id(), "/", 11)); - while (filteredMetricsIterator.hasNext() && filteredMetricsNamesIterator.hasNext() && filteredMetricsConfigIterator.hasNext()) { List filteredMetricsChunk = filteredMetricsIterator.next(); List filteredMetricsNamesChunk = filteredMetricsNamesIterator.next(); List> filteredMetricsConfigChunk = filteredMetricsConfigIterator.next(); - JsonNode apiResponse = getAzureMetrics(apiEndpointBase, StringUtils.join(filteredMetricsNamesChunk, ',')); + JsonNode apiResponse = azure.getMetrics(filteredMetrics.get(0), StringUtils.join(filteredMetricsNamesChunk, ',')); if (apiResponse != null) { - if (logger.isDebugEnabled()) { - logger.debug("consumeAndImportAzureMetrics: API response: " + apiResponse.toString()); - } addMetrics(filteredMetricsChunk, filteredMetricsConfigChunk, apiResponse); } } } } - private JsonNode getAzureMetrics(String apiEndpointBase, String metricNames) throws MalformedURLException, UnsupportedEncodingException { - return getAzureMetrics(apiEndpointBase, metricNames, null); - } - - private JsonNode getAzureMetrics(String apiEndpointBase, String metricNames, - String filter) throws MalformedURLException, UnsupportedEncodingException { - String url = Constants.AZURE_MANAGEMENT_URL - + apiEndpointBase - + "/metrics?timespan=" + recordDateTime.minusMinutes(2).toDateTimeISO() + "/" + recordDateTime.toDateTimeISO() - + "&metricnames=" + metricNames - + "&api-version=" + subscription.get("api-version"); - if (filter != null) { - url += "&$filter=" + URLEncoder.encode(filter, "UTF-8"); - } - URL apiEndpointFull = new URL(url); - azureMetricsCallCount.incrementAndGet(); - JsonNode apiResponse = AzureRestOperation.doGet(apiEndpointFull); - - return apiResponse; - } - private void addMetrics(List filteredMetricsChunk, List> filteredMetricsConfigChunk, JsonNode responseMetrics) { JsonNode responseMetricsValues = responseMetrics.get("value"); @@ -282,16 +244,17 @@ private void addMetrics(List filteredMetricsChunk, List metricConfig) throws Exception { + if (responseMetric == null) return; String azureMetricValue = null; String metricPath = metricPrefix + Constants.METRIC_SEPARATOR + subscriptionName + Constants.METRIC_SEPARATOR + - resource.resourceGroupName() + + resource.getResourceGroupName() + Constants.METRIC_SEPARATOR + - resource.resourceType() + + resource.getResourceType() + Constants.METRIC_SEPARATOR + - resource.name() + + resource.getName() + Constants.METRIC_SEPARATOR; // if (logger.isDebugEnabled()) { // logger.debug("Metric name / aggregation / path: {} / {} / {}", metricName, metricAggregation, metricPath); @@ -302,8 +265,8 @@ private void addMetric(MetricDefinition resourceMetric, JsonNode responseMetric, if (timeseries.size() > 1) { throw new Exception("Multiple timeseries not supported"); } - String azureMetricName = resourceMetric.name().value(); - String azureMetricAggregation = resourceMetric.primaryAggregationType().toString(); + String azureMetricName = resourceMetric.getName(); + String azureMetricAggregation = resourceMetric.getPrimaryAggregationType().toString(); String appdAggregationType = null; String appdTimeRollUpType = null; String appdClusterRollUpType = null; @@ -316,7 +279,7 @@ private void addMetric(MetricDefinition resourceMetric, JsonNode responseMetric, appdClusterRollUpType = MetricWriter.METRIC_CLUSTER_ROLLUP_TYPE_INDIVIDUAL; break; case "Count": - azureMetricValue = (data.path(0).path("total").isMissingNode()) ? null : data.get(0).get("count").toString(); + azureMetricValue = (data.path(0).path("count").isMissingNode()) ? null : data.get(0).get("count").toString(); appdAggregationType = MetricWriter.METRIC_AGGREGATION_TYPE_AVERAGE; appdTimeRollUpType = MetricWriter.METRIC_TIME_ROLLUP_TYPE_SUM; appdClusterRollUpType = MetricWriter.METRIC_CLUSTER_ROLLUP_TYPE_COLLECTIVE; @@ -343,7 +306,7 @@ private void addMetric(MetricDefinition resourceMetric, JsonNode responseMetric, if (logger.isDebugEnabled()) { logger.info("Not Reporting Metric {} for Resource {} as the aggregation type is not supported", azureMetricName, - resource.name()); + resource.getName()); } break; } @@ -351,7 +314,7 @@ private void addMetric(MetricDefinition resourceMetric, JsonNode responseMetric, if (logger.isDebugEnabled()) { logger.debug("Ignoring Metric {} for Resource {} as it is null or empty", azureMetricName, - resource.name(), + resource.getName(), azureMetricValue); } } else { @@ -382,7 +345,7 @@ private void addMetric(MetricDefinition resourceMetric, JsonNode responseMetric, if (logger.isDebugEnabled()) { logger.debug("Reporting Metric {} for Resource {} with value {}", azureMetricName, - resource.name(), + resource.getName(), azureMetricValue); } finalMetricList.add(metric); diff --git a/src/main/java/com/appdynamics/monitors/azure/utils/Constants.java b/src/main/java/com/appdynamics/monitors/azure/utils/Constants.java index 8ec37d3..364ca27 100644 --- a/src/main/java/com/appdynamics/monitors/azure/utils/Constants.java +++ b/src/main/java/com/appdynamics/monitors/azure/utils/Constants.java @@ -8,8 +8,6 @@ package com.appdynamics.monitors.azure.utils; -import com.microsoft.azure.management.Azure; - public class Constants { public static final String DEFAULT_METRIC_PREFIX = "Custom Metrics|AzureMonitor|"; public static final String METRIC_SEPARATOR = "|"; @@ -18,6 +16,6 @@ public class Constants { public static final String AZURE_VAULT_URL = "https://vault.azure.net"; public static final String AZURE_MSI_TOKEN_ENDPOINT = "http://169.254.169.254/metadata/identity/oauth2/token"; public static final int AZURE_METRICS_CHUNK_SIZE = 20; - public static Azure.Authenticated azureMonitorAuth = null; + public static final int AZURE_METRICS_API_ENDPOINT_LAST_SLASH_POS = 11; public static String accessToken; } From 982f870226e30c5de946882c317c9e84ddd5c0ef Mon Sep 17 00:00:00 2001 From: Isi Ramirez Date: Mon, 27 Aug 2018 19:38:55 +0200 Subject: [PATCH 43/51] Add test with metrics and without dimensions (mocked both api and controller calls) --- .../monitors/azure/AzureMonitorTest.java | 210 ++++++++++++------ 1 file changed, 144 insertions(+), 66 deletions(-) diff --git a/src/test/java/com/appdynamics/monitors/azure/AzureMonitorTest.java b/src/test/java/com/appdynamics/monitors/azure/AzureMonitorTest.java index b9034c9..091ffb3 100644 --- a/src/test/java/com/appdynamics/monitors/azure/AzureMonitorTest.java +++ b/src/test/java/com/appdynamics/monitors/azure/AzureMonitorTest.java @@ -10,96 +10,124 @@ import com.appdynamics.extensions.*; import com.appdynamics.extensions.conf.MonitorContextConfiguration; +import com.appdynamics.extensions.metrics.Metric; import com.appdynamics.extensions.util.AssertUtils; +import com.appdynamics.monitors.azure.utils.AzureAPIWrapper; import com.appdynamics.monitors.azure.utils.Constants; +import com.appdynamics.monitors.azure.utils.MetricDefinition; +import com.appdynamics.monitors.azure.utils.Resource; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Lists; import com.singularity.ee.agent.systemagent.api.exception.TaskExecutionException; import org.junit.Test; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.*; +import java.net.MalformedURLException; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; -import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.*; public class AzureMonitorTest { - // --Commented out by Inspection (7/4/18 6:16 PM):private static final Logger logger = LoggerFactory.getLogger(AzureMonitorTest.class); - - @Test - public void testAzureMonitor(){ - Map taskArgs = new HashMap<>(); - taskArgs.put("config-file", "src/test/resources/conf/integration-test-config.yml"); - try { - testAzureMonitorRun(taskArgs); - } catch (TaskExecutionException e) { - e.printStackTrace(); - } - } - - @Test - public void testAzureMonitorWithEncryption(){ - Map taskArgs = new HashMap<>(); - taskArgs.put("config-file", "src/test/resources/conf/integration-test-encrypted-config.yml"); - try { - testAzureMonitorRun(taskArgs); - } catch (TaskExecutionException e) { - e.printStackTrace(); - } - } - - @Test - public void testAzureMonitorTask(){ - try { - testAzureMonitorTaskRun("src/test/resources/conf/integration-test-config.yml"); - } catch (Exception e) { - e.printStackTrace(); - } - } - - @Test - public void testAzureMonitorTaskWithEncryption(){ - try { - testAzureMonitorTaskRun("src/test/resources/conf/integration-test-encrypted-config.yml"); - } catch (Exception e) { - e.printStackTrace(); - } - } + // --Commented out by Inspection (7/4/18 6:16 PM): + private static final Logger logger = LoggerFactory.getLogger(AzureMonitorTest.class); - @Test - public void testAzureMonitorTaskWithKeyvault(){ - try { - testAzureMonitorTaskRun("src/test/resources/conf/integration-test-keyvault-config.yml"); - } catch (Exception e) { - e.printStackTrace(); - } - } +// @Test +// public void testAzureMonitor(){ +// Map taskArgs = new HashMap<>(); +// taskArgs.put("config-file", "src/test/resources/conf/integration-test-config.yml"); +// try { +// testAzureMonitorRun(taskArgs); +// } catch (TaskExecutionException e) { +// e.printStackTrace(); +// } +// } - @Test - public void testAzureMonitorTaskWithKeyvaultWithEncryption(){ - try { - testAzureMonitorTaskRun("src/test/resources/conf/integration-test-keyvault-encrypted-config.yml"); - } catch (Exception e) { - e.printStackTrace(); - } - } +// @Test +// public void testAzureMonitorWithEncryption(){ +// Map taskArgs = new HashMap<>(); +// taskArgs.put("config-file", "src/test/resources/conf/integration-test-encrypted-config.yml"); +// try { +// testAzureMonitorRun(taskArgs); +// } catch (TaskExecutionException e) { +// e.printStackTrace(); +// } +// } @Test - public void testResourceFilters(){ + public void testAzureMonitorTaskWithMetricsWithoutDimensionsRun(){ try { - testAzureMonitorTaskRun("src/test/resources/conf/integration-test-resourcefilter-config.yml"); + testAzureMonitorTaskWithMetricsWithoutDimensionsRun("src/test/resources/conf/integration-test-config.yml"); } catch (Exception e) { e.printStackTrace(); } } - - private void testAzureMonitorRun(Map taskArgs) throws TaskExecutionException { - new AzureMonitor().execute(taskArgs, null); - } +// +// @Test +// public void testAzureMonitorTaskWithEncryption(){ +// try { +// testAzureMonitorTaskRun("src/test/resources/conf/integration-test-encrypted-config.yml"); +// } catch (Exception e) { +// e.printStackTrace(); +// } +// } +// +// @Test +// public void testAzureMonitorTaskWithKeyvault(){ +// try { +// testAzureMonitorTaskRun("src/test/resources/conf/integration-test-keyvault-config.yml"); +// } catch (Exception e) { +// e.printStackTrace(); +// } +// } +// +// @Test +// public void testAzureMonitorTaskWithKeyvaultWithEncryption(){ +// try { +// testAzureMonitorTaskRun("src/test/resources/conf/integration-test-keyvault-encrypted-config.yml"); +// } catch (Exception e) { +// e.printStackTrace(); +// } +// } +// +// @Test +// public void testResourceFilters(){ +// try { +// testAzureMonitorTaskRun("src/test/resources/conf/integration-test-resourcefilter-config.yml"); +// } catch (Exception e) { +// e.printStackTrace(); +// } +// } +// +// private void testAzureMonitorRun(Map taskArgs) throws TaskExecutionException { +// new AzureMonitor().execute(taskArgs, null); +// } @SuppressWarnings({"ConstantConditions", "unchecked"}) - private void testAzureMonitorTaskRun(String configYml) { + private void testAzureMonitorTaskWithMetricsWithoutDimensionsRun(String configYml) { AMonitorJob aMonitorJob = mock(AMonitorJob.class); MetricWriteHelper metricWriteHelper = mock(MetricWriteHelper.class); + doAnswer(new Answer() { + public Object answer(InvocationOnMock invocation) { + List metrics = (List) invocation.getArguments()[0]; + Object mock = invocation.getMock(); + for (Object metric : metrics) { + logger.debug("Printing metric " + metric.toString()); + } + return null; + } + }) + .when(metricWriteHelper) + .transformAndPrintMetrics(Mockito.any(List.class)); + CountDownLatch countDownLatch = new CountDownLatch(1); MonitorContextConfiguration monitorContextConfiguration = new MonitorContextConfiguration( "AzureMonitor", @@ -110,7 +138,8 @@ private void testAzureMonitorTaskRun(String configYml) { List> subscriptions = (List>)monitorContextConfiguration.getConfigYml().get("subscriptions"); AssertUtils.assertNotNull(subscriptions, "The 'subscriptions' section in config.yml is not initialised"); for (Map subscription : subscriptions) { - AzureMonitorTask task = new AzureMonitorTask(monitorContextConfiguration, metricWriteHelper, subscription, countDownLatch); + AzureAPIWrapper azure = mockAzureAPIWrapperWithMetricsAndWithoutDimensions(subscription); + AzureMonitorTask task = new AzureMonitorTask(monitorContextConfiguration, metricWriteHelper, subscription, countDownLatch, azure); monitorContextConfiguration.getContext().getExecutorService().execute("Azure Monitor", task); try { countDownLatch.await(); @@ -118,6 +147,55 @@ private void testAzureMonitorTaskRun(String configYml) { e.printStackTrace(); } } + } + + private AzureAPIWrapper mockAzureAPIWrapperWithMetricsAndWithoutDimensions(Map subscription) { + AzureAPIWrapper azure = mock(AzureAPIWrapper.class); + + List resources = Lists.newArrayList(); + Resource resource1 = new Resource(); + resource1.setId("/subscriptions/39eee2d9-0ea1-4e3c-93bc-3619f821acb1/resourceGroups/AppDTest/providers/Microsoft.ApiManagement/service/CoBaApiManagement1"); + resource1.setName("CoBaApiManagement1"); + resource1.setResourceGroupName("AppDTest"); + resource1.setResourceType("service"); + resource1.setType("Microsoft.ApiManagement/service"); + resources.add(resource1); + when(azure.getResources()).thenReturn(resources); + + List metricDefinitions = Lists.newArrayList(); + MetricDefinition metricDefinition1 = new MetricDefinition(); + metricDefinition1.setId("/subscriptions/39eee2d9-0ea1-4e3c-93bc-3619f821acb1/resourceGroups/AppDTest/providers/Microsoft.ApiManagement/service/CoBaApiManagement1/providers/microsoft.insights/metricdefinitions/SuccessfulRequests"); + metricDefinition1.setName("SuccessfulRequests"); + metricDefinition1.setPrimaryAggregationType("Total"); + metricDefinitions.add(metricDefinition1); + MetricDefinition metricDefinition2 = new MetricDefinition(); + metricDefinition2.setId("/subscriptions/39eee2d9-0ea1-4e3c-93bc-3619f821acb1/resourceGroups/AppDTest/providers/Microsoft.ApiManagement/service/CoBaApiManagement1/providers/microsoft.insights/metricdefinitions/FailedRequests"); + metricDefinition2.setName("FailedRequests"); + metricDefinition2.setPrimaryAggregationType("Total"); + metricDefinitions.add(metricDefinition2); + MetricDefinition metricDefinition3 = new MetricDefinition(); + metricDefinition3.setId("/subscriptions/39eee2d9-0ea1-4e3c-93bc-3619f821acb1/resourceGroups/AppDTest/providers/Microsoft.ApiManagement/service/CoBaApiManagement1/providers/microsoft.insights/metricdefinitions/UnauthorizedRequests"); + metricDefinition3.setName("UnauthorizedRequests"); + metricDefinition3.setPrimaryAggregationType("Total"); + metricDefinitions.add(metricDefinition3); + when(azure.getMetricDefinitions(resource1.getId())).thenReturn(metricDefinitions); + + ObjectMapper objectMapper = new ObjectMapper(); + String apiResponse = "{\"cost\":0,\"timespan\":\"2018-08-27T15:58:01Z/2018-08-27T16:00:01Z\",\"interval\":\"PT1M\",\"value\":[{\"id\":\"/subscriptions/39eee2d9-0ea1-4e3c-93bc-3619f821acb1/resourceGroups/AppDTest/providers/Microsoft.ApiManagement/service/CoBaApiManagement1/providers/Microsoft.Insights/metrics/SuccessfulRequests\",\"type\":\"Microsoft.Insights/metrics\",\"name\":{\"value\":\"SuccessfulRequests\",\"localizedValue\":\"Successful Gateway Requests\"},\"unit\":\"Count\",\"timeseries\":[{\"metadatavalues\":[],\"data\":[{\"timeStamp\":\"2018-08-27T15:58:00Z\",\"total\":0.0},{\"timeStamp\":\"2018-08-27T15:59:00Z\",\"total\":0.0}]}]},{\"id\":\"/subscriptions/39eee2d9-0ea1-4e3c-93bc-3619f821acb1/resourceGroups/AppDTest/providers/Microsoft.ApiManagement/service/CoBaApiManagement1/providers/Microsoft.Insights/metrics/FailedRequests\",\"type\":\"Microsoft.Insights/metrics\",\"name\":{\"value\":\"FailedRequests\",\"localizedValue\":\"Failed Gateway Requests\"},\"unit\":\"Count\",\"timeseries\":[{\"metadatavalues\":[],\"data\":[{\"timeStamp\":\"2018-08-27T15:58:00Z\",\"total\":0.0},{\"timeStamp\":\"2018-08-27T15:59:00Z\",\"total\":0.0}]}]},{\"id\":\"/subscriptions/39eee2d9-0ea1-4e3c-93bc-3619f821acb1/resourceGroups/AppDTest/providers/Microsoft.ApiManagement/service/CoBaApiManagement1/providers/Microsoft.Insights/metrics/UnauthorizedRequests\",\"type\":\"Microsoft.Insights/metrics\",\"name\":{\"value\":\"UnauthorizedRequests\",\"localizedValue\":\"Unauthorized Gateway Requests\"},\"unit\":\"Count\",\"timeseries\":[{\"metadatavalues\":[],\"data\":[{\"timeStamp\":\"2018-08-27T15:58:00Z\",\"total\":0.0},{\"timeStamp\":\"2018-08-27T15:59:00Z\",\"total\":0.0}]}]},{\"id\":\"/subscriptions/39eee2d9-0ea1-4e3c-93bc-3619f821acb1/resourceGroups/AppDTest/providers/Microsoft.ApiManagement/service/CoBaApiManagement1/providers/Microsoft.Insights/metrics/Duration\",\"type\":\"Microsoft.Insights/metrics\",\"name\":{\"value\":\"Duration\",\"localizedValue\":\"Overall Duration of Gateway Requests\"},\"unit\":\"MilliSeconds\",\"timeseries\":[{\"metadatavalues\":[],\"data\":[{\"timeStamp\":\"2018-08-27T15:58:00Z\",\"total\":0.0},{\"timeStamp\":\"2018-08-27T15:59:00Z\",\"total\":0.0}]}]}],\"namespace\":\"Microsoft.ApiManagement/service\",\"resourceregion\":\"northeurope\"}"; + JsonNode metrics = null; + try { + metrics = objectMapper.readTree(apiResponse); + } catch (IOException e1) { + // TODO Auto-generated catch block + e1.printStackTrace(); + } + try { + when(azure.getMetrics(metricDefinition1, "SuccessfulRequests,FailedRequests,UnauthorizedRequests")).thenReturn(metrics); + } catch (MalformedURLException | UnsupportedEncodingException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + return azure; } } From 393e0ab9f9778bff45e438439c752c8bb03770c6 Mon Sep 17 00:00:00 2001 From: Isi Ramirez Date: Tue, 28 Aug 2018 12:24:41 +0200 Subject: [PATCH 44/51] Add test with qualifiers - and some refactor --- .../monitors/azure/AzureMonitorTest.java | 297 ++++++++++-------- 1 file changed, 164 insertions(+), 133 deletions(-) diff --git a/src/test/java/com/appdynamics/monitors/azure/AzureMonitorTest.java b/src/test/java/com/appdynamics/monitors/azure/AzureMonitorTest.java index 091ffb3..05cb90e 100644 --- a/src/test/java/com/appdynamics/monitors/azure/AzureMonitorTest.java +++ b/src/test/java/com/appdynamics/monitors/azure/AzureMonitorTest.java @@ -19,8 +19,11 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Lists; -import com.singularity.ee.agent.systemagent.api.exception.TaskExecutionException; + import org.junit.Test; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; +import org.junit.runner.RunWith; import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; @@ -29,168 +32,175 @@ import java.io.*; import java.net.MalformedURLException; -import java.util.HashMap; +import java.util.Arrays; +import java.util.Collection; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; +import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.*; +@RunWith(Parameterized.class) public class AzureMonitorTest { - // --Commented out by Inspection (7/4/18 6:16 PM): private static final Logger logger = LoggerFactory.getLogger(AzureMonitorTest.class); + private static MonitorContextConfiguration monitorContextConfiguration; + private static MetricWriteHelper metricWriteHelper; + private static Map subscription; + private static String metricPrefix; + private List resources; + private List metricDefinitions; + private String metricsApiResponse; + private String metricNames; + private String filter; + private String expectedOutput; -// @Test -// public void testAzureMonitor(){ -// Map taskArgs = new HashMap<>(); -// taskArgs.put("config-file", "src/test/resources/conf/integration-test-config.yml"); -// try { -// testAzureMonitorRun(taskArgs); -// } catch (TaskExecutionException e) { -// e.printStackTrace(); -// } -// } - -// @Test -// public void testAzureMonitorWithEncryption(){ -// Map taskArgs = new HashMap<>(); -// taskArgs.put("config-file", "src/test/resources/conf/integration-test-encrypted-config.yml"); -// try { -// testAzureMonitorRun(taskArgs); -// } catch (TaskExecutionException e) { -// e.printStackTrace(); -// } -// } - - @Test - public void testAzureMonitorTaskWithMetricsWithoutDimensionsRun(){ - try { - testAzureMonitorTaskWithMetricsWithoutDimensionsRun("src/test/resources/conf/integration-test-config.yml"); - } catch (Exception e) { - e.printStackTrace(); - } + public AzureMonitorTest(List resources, List metricDefinitions, String metricsApiResponse, String metricNames, String filter, String expectedOutput) { + this.resources = resources; + this.expectedOutput = expectedOutput; + this.metricDefinitions = metricDefinitions; + this.metricsApiResponse = metricsApiResponse; + this.metricNames = metricNames; + this.filter = filter; } -// -// @Test -// public void testAzureMonitorTaskWithEncryption(){ -// try { -// testAzureMonitorTaskRun("src/test/resources/conf/integration-test-encrypted-config.yml"); -// } catch (Exception e) { -// e.printStackTrace(); -// } -// } -// -// @Test -// public void testAzureMonitorTaskWithKeyvault(){ -// try { -// testAzureMonitorTaskRun("src/test/resources/conf/integration-test-keyvault-config.yml"); -// } catch (Exception e) { -// e.printStackTrace(); -// } -// } -// -// @Test -// public void testAzureMonitorTaskWithKeyvaultWithEncryption(){ -// try { -// testAzureMonitorTaskRun("src/test/resources/conf/integration-test-keyvault-encrypted-config.yml"); -// } catch (Exception e) { -// e.printStackTrace(); -// } -// } -// -// @Test -// public void testResourceFilters(){ -// try { -// testAzureMonitorTaskRun("src/test/resources/conf/integration-test-resourcefilter-config.yml"); -// } catch (Exception e) { -// e.printStackTrace(); -// } -// } -// -// private void testAzureMonitorRun(Map taskArgs) throws TaskExecutionException { -// new AzureMonitor().execute(taskArgs, null); -// } - @SuppressWarnings({"ConstantConditions", "unchecked"}) - private void testAzureMonitorTaskWithMetricsWithoutDimensionsRun(String configYml) { + @Parameters + public static Collection data() { AMonitorJob aMonitorJob = mock(AMonitorJob.class); - MetricWriteHelper metricWriteHelper = mock(MetricWriteHelper.class); - doAnswer(new Answer() { - public Object answer(InvocationOnMock invocation) { - List metrics = (List) invocation.getArguments()[0]; - Object mock = invocation.getMock(); - for (Object metric : metrics) { - logger.debug("Printing metric " + metric.toString()); - } - return null; - } - }) - .when(metricWriteHelper) - .transformAndPrintMetrics(Mockito.any(List.class)); - - CountDownLatch countDownLatch = new CountDownLatch(1); - MonitorContextConfiguration monitorContextConfiguration = new MonitorContextConfiguration( + monitorContextConfiguration = new MonitorContextConfiguration( "AzureMonitor", Constants.DEFAULT_METRIC_PREFIX, new File(System.getProperty("user.dir")), aMonitorJob); + metricWriteHelper = mock(MetricWriteHelper.class); + metricPrefix = "Custom Metrics|AzureMonitor|"; + + String configYml = "src/test/resources/conf/integration-test-config.yml"; monitorContextConfiguration.setConfigYml(configYml); - List> subscriptions = (List>)monitorContextConfiguration.getConfigYml().get("subscriptions"); + List> subscriptions = (List>) monitorContextConfiguration.getConfigYml().get("subscriptions"); AssertUtils.assertNotNull(subscriptions, "The 'subscriptions' section in config.yml is not initialised"); - for (Map subscription : subscriptions) { - AzureAPIWrapper azure = mockAzureAPIWrapperWithMetricsAndWithoutDimensions(subscription); - AzureMonitorTask task = new AzureMonitorTask(monitorContextConfiguration, metricWriteHelper, subscription, countDownLatch, azure); - monitorContextConfiguration.getContext().getExecutorService().execute("Azure Monitor", task); - try { - countDownLatch.await(); - } catch (InterruptedException e) { - e.printStackTrace(); - } + + Iterator> iter = subscriptions.iterator(); + subscription = iter.next(); + String subscriptionName = ""; + if (subscription.containsKey("subscriptionName")){ + subscriptionName = subscription.get("subscriptionName").toString(); } + else { + subscriptionName = subscription.get("subscriptionId").toString(); + } + + List resourcesWithMetrics = Lists.newArrayList(); + Resource resourceWithMetrics = new Resource(); + resourceWithMetrics.setId("/subscriptions/1a2b3c4b-e5f6-g7h8-a123-1a23bcde456f/resourceGroups/AppDTest/providers/Microsoft.ApiManagement/service/TestApiManagement1"); + resourceWithMetrics.setName("TestApiManagement1"); + resourceWithMetrics.setResourceGroupName("AppDTest"); + resourceWithMetrics.setResourceType("service"); + resourceWithMetrics.setType("Microsoft.ApiManagement/service"); + resourcesWithMetrics.add(resourceWithMetrics); + + List resourcesWithQualifier = Lists.newArrayList(); + Resource resourceWithQualifier = new Resource(); + resourceWithQualifier.setId("/subscriptions/1a2b3c4b-e5f6-g7h8-a123-1a23bcde456f/resourceGroups/AppDTest/providers/Microsoft.ApiManagement/service/TestApiManagement2"); + resourceWithQualifier.setName("TestApiManagement2"); + resourceWithQualifier.setResourceGroupName("AppDTest"); + resourceWithQualifier.setResourceType("service"); + resourceWithQualifier.setType("Microsoft.ApiManagement/service"); + resourcesWithQualifier.add(resourceWithQualifier); + + List resourcesWithDimensions = Lists.newArrayList(); + Resource resourceWithDimensions = new Resource(); + resourceWithDimensions.setId("/subscriptions/1a2b3c4b-e5f6-g7h8-a123-1a23bcde456f/resourceGroups/AppDTest/providers/Microsoft.EventHub/namespaces/TestHub1"); + resourceWithDimensions.setName("TestHub1"); + resourceWithDimensions.setResourceGroupName("AppDTest"); + resourceWithDimensions.setResourceType("namespaces"); + resourceWithDimensions.setType("Microsoft.EventHub/namespaces"); + resourcesWithDimensions.add(resourceWithDimensions); + + List metricDefinitionsWithMetrics = Lists.newArrayList(); + MetricDefinition metricDefinitionSuccessfulRequests = new MetricDefinition(); + metricDefinitionSuccessfulRequests.setId("/subscriptions/1a2b3c4b-e5f6-g7h8-a123-1a23bcde456f/resourceGroups/AppDTest/providers/Microsoft.ApiManagement/service/TestApiManagement1/providers/microsoft.insights/metricdefinitions/SuccessfulRequests"); + metricDefinitionSuccessfulRequests.setName("SuccessfulRequests"); + metricDefinitionSuccessfulRequests.setPrimaryAggregationType("Total"); + metricDefinitionsWithMetrics.add(metricDefinitionSuccessfulRequests); + MetricDefinition metricDefinitionFailedRequests = new MetricDefinition(); + metricDefinitionFailedRequests.setId("/subscriptions/1a2b3c4b-e5f6-g7h8-a123-1a23bcde456f/resourceGroups/AppDTest/providers/Microsoft.ApiManagement/service/TestApiManagement1/providers/microsoft.insights/metricdefinitions/FailedRequests"); + metricDefinitionFailedRequests.setName("FailedRequests"); + metricDefinitionFailedRequests.setPrimaryAggregationType("Total"); + metricDefinitionsWithMetrics.add(metricDefinitionFailedRequests); + MetricDefinition metricDefinitionUnauthorizedRequests = new MetricDefinition(); + metricDefinitionUnauthorizedRequests.setId("/subscriptions/1a2b3c4b-e5f6-g7h8-a123-1a23bcde456f/resourceGroups/AppDTest/providers/Microsoft.ApiManagement/service/TestApiManagement1/providers/microsoft.insights/metricdefinitions/UnauthorizedRequests"); + metricDefinitionUnauthorizedRequests.setName("UnauthorizedRequests"); + metricDefinitionUnauthorizedRequests.setPrimaryAggregationType("Total"); + metricDefinitionsWithMetrics.add(metricDefinitionUnauthorizedRequests); + List metricDefinitionsWithQualifier = Lists.newArrayList(); + MetricDefinition metricDefinitionUnauthorizedRequestsWithQualifier = new MetricDefinition(); + metricDefinitionUnauthorizedRequestsWithQualifier.setId("/subscriptions/1a2b3c4b-e5f6-g7h8-a123-1a23bcde456f/resourceGroups/AppDTest/providers/Microsoft.ApiManagement/service/TestApiManagement2/providers/microsoft.insights/metricdefinitions/UnauthorizedRequests"); + metricDefinitionUnauthorizedRequestsWithQualifier.setName("UnauthorizedRequests"); + metricDefinitionUnauthorizedRequestsWithQualifier.setPrimaryAggregationType("Total"); + metricDefinitionsWithQualifier.add(metricDefinitionUnauthorizedRequestsWithQualifier); + + List metricDefinitionsWithDimensions = Lists.newArrayList(); + MetricDefinition metricDefinitionThrottledRequests = new MetricDefinition(); + metricDefinitionThrottledRequests.setId("/subscriptions/1a2b3c4b-e5f6-g7h8-a123-1a23bcde456f/resourceGroups/AppDTest/providers/Microsoft.EventHub/namespaces/TestHub1/providers/microsoft.insights/metricdefinitions/ThrottledRequests"); + metricDefinitionThrottledRequests.setName("ThrottledRequests"); + metricDefinitionThrottledRequests.setPrimaryAggregationType("Total"); + metricDefinitionsWithDimensions.add(metricDefinitionThrottledRequests); + + String apiResponseWithMetrics = "{\"cost\":0,\"timespan\":\"2018-08-28T09:02:19Z/2018-08-28T09:04:19Z\",\"interval\":\"PT1M\",\"value\":[{\"id\":\"/subscriptions/1a2b3c4b-e5f6-g7h8-a123-1a23bcde456f/resourceGroups/AppDTest/providers/Microsoft.ApiManagement/service/TestApiManagement1/providers/Microsoft.Insights/metrics/SuccessfulRequests\",\"type\":\"Microsoft.Insights/metrics\",\"name\":{\"value\":\"SuccessfulRequests\",\"localizedValue\":\"Successful Gateway Requests\"},\"unit\":\"Count\",\"timeseries\":[{\"metadatavalues\":[],\"data\":[{\"timeStamp\":\"2018-08-28T09:02:00Z\",\"total\":0.0},{\"timeStamp\":\"2018-08-28T09:03:00Z\",\"total\":0.0}]}]},{\"id\":\"/subscriptions/1a2b3c4b-e5f6-g7h8-a123-1a23bcde456f/resourceGroups/AppDTest/providers/Microsoft.ApiManagement/service/TestApiManagement1/providers/Microsoft.Insights/metrics/FailedRequests\",\"type\":\"Microsoft.Insights/metrics\",\"name\":{\"value\":\"FailedRequests\",\"localizedValue\":\"Failed Gateway Requests\"},\"unit\":\"Count\",\"timeseries\":[{\"metadatavalues\":[],\"data\":[{\"timeStamp\":\"2018-08-28T09:02:00Z\",\"total\":0.0},{\"timeStamp\":\"2018-08-28T09:03:00Z\",\"total\":0.0}]}]},{\"id\":\"/subscriptions/1a2b3c4b-e5f6-g7h8-a123-1a23bcde456f/resourceGroups/AppDTest/providers/Microsoft.ApiManagement/service/TestApiManagement1/providers/Microsoft.Insights/metrics/UnauthorizedRequests\",\"type\":\"Microsoft.Insights/metrics\",\"name\":{\"value\":\"UnauthorizedRequests\",\"localizedValue\":\"Unauthorized Gateway Requests\"},\"unit\":\"Count\",\"timeseries\":[{\"metadatavalues\":[],\"data\":[{\"timeStamp\":\"2018-08-28T09:02:00Z\",\"total\":0.0},{\"timeStamp\":\"2018-08-28T09:03:00Z\",\"total\":0.0}]}]}],\"namespace\":\"Microsoft.ApiManagement/service\",\"resourceregion\":\"northeurope\"}"; + String apiResponseWithQualifier = "{\"cost\":0,\"timespan\":\"2018-08-28T09:04:38Z/2018-08-28T09:06:38Z\",\"interval\":\"PT1M\",\"value\":[{\"id\":\"/subscriptions/1a2b3c4b-e5f6-g7h8-a123-1a23bcde456f/resourceGroups/AppDTest/providers/Microsoft.ApiManagement/service/TestApiManagement2/providers/Microsoft.Insights/metrics/UnauthorizedRequests\",\"type\":\"Microsoft.Insights/metrics\",\"name\":{\"value\":\"UnauthorizedRequests\",\"localizedValue\":\"Unauthorized Gateway Requests\"},\"unit\":\"Count\",\"timeseries\":[{\"metadatavalues\":[],\"data\":[{\"timeStamp\":\"2018-08-28T09:04:00Z\",\"total\":0.0},{\"timeStamp\":\"2018-08-28T09:05:00Z\",\"total\":0.0}]}]}],\"namespace\":\"Microsoft.ApiManagement/service\",\"resourceregion\":\"northeurope\"}"; + String apiResponseWithDimensions = "{\"cost\":0,\"timespan\":\"2018-08-28T11:53:19Z/2018-08-28T11:55:19Z\",\"interval\":\"PT1M\",\"value\":[{\"id\":\"/subscriptions/1a2b3c4b-e5f6-g7h8-a123-1a23bcde456f/resourceGroups/AppDTest/providers/Microsoft.EventHub/namespaces/TestHub1/providers/Microsoft.Insights/metrics/ThrottledRequests\",\"type\":\"Microsoft.Insights/metrics\",\"name\":{\"value\":\"ThrottledRequests\",\"localizedValue\":\"Throttled Requests. (Preview)\"},\"unit\":\"Count\",\"timeseries\":[{\"metadatavalues\":[{\"name\":{\"value\":\"entityname\",\"localizedValue\":\"entityname\"},\"value\":\"cooltopica\"}],\"data\":[{\"timeStamp\":\"2018-08-28T11:53:00Z\",\"total\":0.0},{\"timeStamp\":\"2018-08-28T11:54:00Z\",\"total\":0.0}]}]}],\"namespace\":\"Microsoft.EventHub/namespaces\",\"resourceregion\":\"northeurope\"}"; + + String expectedOutputWithMetrics = "[AVERAGE/SUM/COLLECTIVE] [" + metricPrefix + subscriptionName + "|AppDTest|service|TestApiManagement1|SuccessfulRequests]=[0.0]]" + + "[AVERAGE/SUM/COLLECTIVE] [" + metricPrefix + subscriptionName + "|AppDTest|service|TestApiManagement1|FailedRequests]=[0.0]]" + + "[AVERAGE/SUM/COLLECTIVE] [" + metricPrefix + subscriptionName + "|AppDTest|service|TestApiManagement1|UnauthorizedRequests]=[0.0]]"; + + String expectedOutputWithQualifier = "[AVERAGE/AVERAGE/COLLECTIVE] [" + metricPrefix + subscriptionName + "|AppDTest|service|TestApiManagement2|UnauthorizedRequests]=[0.0]]"; + + String expectedOutputWithDimensions = "[AVERAGE/SUM/COLLECTIVE] [" + metricPrefix + subscriptionName + "|AppDTest|namespaces|TestHub1|coolTopicA|Throttled Requests]=[0.0]]"; + + return Arrays.asList(new Object[][] { + {resourcesWithMetrics, metricDefinitionsWithMetrics, apiResponseWithMetrics, "SuccessfulRequests,FailedRequests,UnauthorizedRequests", null, expectedOutputWithMetrics}, + {resourcesWithQualifier, metricDefinitionsWithQualifier, apiResponseWithQualifier, "UnauthorizedRequests", null, expectedOutputWithQualifier}, + {resourcesWithDimensions, metricDefinitionsWithDimensions, apiResponseWithDimensions, "ThrottledRequests", "EntityName eq 'cooltopica'", expectedOutputWithDimensions} + }); } - private AzureAPIWrapper mockAzureAPIWrapperWithMetricsAndWithoutDimensions(Map subscription) { - AzureAPIWrapper azure = mock(AzureAPIWrapper.class); + @Test + @SuppressWarnings({"ConstantConditions", "unchecked"}) + public void testAzureMonitorTask() { - List resources = Lists.newArrayList(); - Resource resource1 = new Resource(); - resource1.setId("/subscriptions/39eee2d9-0ea1-4e3c-93bc-3619f821acb1/resourceGroups/AppDTest/providers/Microsoft.ApiManagement/service/CoBaApiManagement1"); - resource1.setName("CoBaApiManagement1"); - resource1.setResourceGroupName("AppDTest"); - resource1.setResourceType("service"); - resource1.setType("Microsoft.ApiManagement/service"); - resources.add(resource1); - when(azure.getResources()).thenReturn(resources); - - List metricDefinitions = Lists.newArrayList(); - MetricDefinition metricDefinition1 = new MetricDefinition(); - metricDefinition1.setId("/subscriptions/39eee2d9-0ea1-4e3c-93bc-3619f821acb1/resourceGroups/AppDTest/providers/Microsoft.ApiManagement/service/CoBaApiManagement1/providers/microsoft.insights/metricdefinitions/SuccessfulRequests"); - metricDefinition1.setName("SuccessfulRequests"); - metricDefinition1.setPrimaryAggregationType("Total"); - metricDefinitions.add(metricDefinition1); - MetricDefinition metricDefinition2 = new MetricDefinition(); - metricDefinition2.setId("/subscriptions/39eee2d9-0ea1-4e3c-93bc-3619f821acb1/resourceGroups/AppDTest/providers/Microsoft.ApiManagement/service/CoBaApiManagement1/providers/microsoft.insights/metricdefinitions/FailedRequests"); - metricDefinition2.setName("FailedRequests"); - metricDefinition2.setPrimaryAggregationType("Total"); - metricDefinitions.add(metricDefinition2); - MetricDefinition metricDefinition3 = new MetricDefinition(); - metricDefinition3.setId("/subscriptions/39eee2d9-0ea1-4e3c-93bc-3619f821acb1/resourceGroups/AppDTest/providers/Microsoft.ApiManagement/service/CoBaApiManagement1/providers/microsoft.insights/metricdefinitions/UnauthorizedRequests"); - metricDefinition3.setName("UnauthorizedRequests"); - metricDefinition3.setPrimaryAggregationType("Total"); - metricDefinitions.add(metricDefinition3); - when(azure.getMetricDefinitions(resource1.getId())).thenReturn(metricDefinitions); + AzureAPIWrapper azure = mockAzureAPIWrapper(subscription, metricWriteHelper); + mockExtensionOutput(); + + CountDownLatch countDownLatch = new CountDownLatch(1); + AzureMonitorTask task = new AzureMonitorTask(monitorContextConfiguration, metricWriteHelper, subscription, countDownLatch, azure); + monitorContextConfiguration.getContext().getExecutorService().execute("Azure Monitor", task); + try { + countDownLatch.await(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + private AzureAPIWrapper mockAzureAPIWrapper(Map subscription, MetricWriteHelper metricWriteHelper) { + AzureAPIWrapper azure = mock(AzureAPIWrapper.class); + + when(azure.getResources()).thenReturn(resources); + when(azure.getMetricDefinitions(resources.get(0).getId())).thenReturn(metricDefinitions); - ObjectMapper objectMapper = new ObjectMapper(); - String apiResponse = "{\"cost\":0,\"timespan\":\"2018-08-27T15:58:01Z/2018-08-27T16:00:01Z\",\"interval\":\"PT1M\",\"value\":[{\"id\":\"/subscriptions/39eee2d9-0ea1-4e3c-93bc-3619f821acb1/resourceGroups/AppDTest/providers/Microsoft.ApiManagement/service/CoBaApiManagement1/providers/Microsoft.Insights/metrics/SuccessfulRequests\",\"type\":\"Microsoft.Insights/metrics\",\"name\":{\"value\":\"SuccessfulRequests\",\"localizedValue\":\"Successful Gateway Requests\"},\"unit\":\"Count\",\"timeseries\":[{\"metadatavalues\":[],\"data\":[{\"timeStamp\":\"2018-08-27T15:58:00Z\",\"total\":0.0},{\"timeStamp\":\"2018-08-27T15:59:00Z\",\"total\":0.0}]}]},{\"id\":\"/subscriptions/39eee2d9-0ea1-4e3c-93bc-3619f821acb1/resourceGroups/AppDTest/providers/Microsoft.ApiManagement/service/CoBaApiManagement1/providers/Microsoft.Insights/metrics/FailedRequests\",\"type\":\"Microsoft.Insights/metrics\",\"name\":{\"value\":\"FailedRequests\",\"localizedValue\":\"Failed Gateway Requests\"},\"unit\":\"Count\",\"timeseries\":[{\"metadatavalues\":[],\"data\":[{\"timeStamp\":\"2018-08-27T15:58:00Z\",\"total\":0.0},{\"timeStamp\":\"2018-08-27T15:59:00Z\",\"total\":0.0}]}]},{\"id\":\"/subscriptions/39eee2d9-0ea1-4e3c-93bc-3619f821acb1/resourceGroups/AppDTest/providers/Microsoft.ApiManagement/service/CoBaApiManagement1/providers/Microsoft.Insights/metrics/UnauthorizedRequests\",\"type\":\"Microsoft.Insights/metrics\",\"name\":{\"value\":\"UnauthorizedRequests\",\"localizedValue\":\"Unauthorized Gateway Requests\"},\"unit\":\"Count\",\"timeseries\":[{\"metadatavalues\":[],\"data\":[{\"timeStamp\":\"2018-08-27T15:58:00Z\",\"total\":0.0},{\"timeStamp\":\"2018-08-27T15:59:00Z\",\"total\":0.0}]}]},{\"id\":\"/subscriptions/39eee2d9-0ea1-4e3c-93bc-3619f821acb1/resourceGroups/AppDTest/providers/Microsoft.ApiManagement/service/CoBaApiManagement1/providers/Microsoft.Insights/metrics/Duration\",\"type\":\"Microsoft.Insights/metrics\",\"name\":{\"value\":\"Duration\",\"localizedValue\":\"Overall Duration of Gateway Requests\"},\"unit\":\"MilliSeconds\",\"timeseries\":[{\"metadatavalues\":[],\"data\":[{\"timeStamp\":\"2018-08-27T15:58:00Z\",\"total\":0.0},{\"timeStamp\":\"2018-08-27T15:59:00Z\",\"total\":0.0}]}]}],\"namespace\":\"Microsoft.ApiManagement/service\",\"resourceregion\":\"northeurope\"}"; JsonNode metrics = null; + ObjectMapper objectMapper = new ObjectMapper(); try { - metrics = objectMapper.readTree(apiResponse); + metrics = objectMapper.readTree(metricsApiResponse); } catch (IOException e1) { // TODO Auto-generated catch block e1.printStackTrace(); } try { - when(azure.getMetrics(metricDefinition1, "SuccessfulRequests,FailedRequests,UnauthorizedRequests")).thenReturn(metrics); + when(azure.getMetrics(metricDefinitions.get(0), metricNames)).thenReturn(metrics); + when(azure.getMetrics(metricDefinitions.get(0), metricNames, filter)).thenReturn(metrics); } catch (MalformedURLException | UnsupportedEncodingException e) { // TODO Auto-generated catch block e.printStackTrace(); @@ -198,4 +208,25 @@ private AzureAPIWrapper mockAzureAPIWrapperWithMetricsAndWithoutDimensions(Map metrics = (List) invocation.getArguments()[0]; + Object mock = invocation.getMock(); + String realOutput = ""; + String loggerOutput = "\n"; + for (Object metric : metrics) { + realOutput += metric.toString(); + loggerOutput += "Printing metric " + metric.toString() + "\n"; + } + assertEquals(expectedOutput, realOutput); + logger.debug(loggerOutput); + + return null; + } + }) + .when(metricWriteHelper) + .transformAndPrintMetrics(Mockito.any(List.class)); + } } From 05070f39fd9f6cea018c7d66b2fc99945bd535c2 Mon Sep 17 00:00:00 2001 From: Isi Ramirez Date: Wed, 29 Aug 2018 15:27:50 +0200 Subject: [PATCH 45/51] Split Azure metrics calls into chucks --- .../monitors/azure/AzureMonitor.java | 4 +- .../monitors/azure/AzureMonitorTask.java | 54 +++++++++++-------- 2 files changed, 33 insertions(+), 25 deletions(-) diff --git a/src/main/java/com/appdynamics/monitors/azure/AzureMonitor.java b/src/main/java/com/appdynamics/monitors/azure/AzureMonitor.java index 2f7b4ff..3fcff34 100644 --- a/src/main/java/com/appdynamics/monitors/azure/AzureMonitor.java +++ b/src/main/java/com/appdynamics/monitors/azure/AzureMonitor.java @@ -50,10 +50,10 @@ protected void doRun(TasksExecutionServiceProvider tasksExecutionServiceProvider for (Map subscription : subscriptions) { AzureAPIWrapper azure = new AzureAPIWrapper(subscription); AzureMonitorTask task = new AzureMonitorTask(getContextConfiguration(), tasksExecutionServiceProvider.getMetricWriteHelper(), subscription, countDownLatch, azure); - tasksExecutionServiceProvider.submit(subscription.get("subscriptionId").toString(),task); + tasksExecutionServiceProvider.submit(subscription.get("subscriptionId").toString(), task); } try{ - countDownLatch.await(45, TimeUnit.SECONDS); + countDownLatch.await(Constants.MONITOR_COUNTDOWN_LATCH_TIMEOUT, TimeUnit.SECONDS); } catch (InterruptedException e) { e.printStackTrace(); } diff --git a/src/main/java/com/appdynamics/monitors/azure/AzureMonitorTask.java b/src/main/java/com/appdynamics/monitors/azure/AzureMonitorTask.java index 8eaacdf..8ad487d 100644 --- a/src/main/java/com/appdynamics/monitors/azure/AzureMonitorTask.java +++ b/src/main/java/com/appdynamics/monitors/azure/AzureMonitorTask.java @@ -13,18 +13,22 @@ import com.appdynamics.extensions.conf.MonitorContextConfiguration; import com.appdynamics.monitors.azure.metrics.AzureMetrics; import com.appdynamics.monitors.azure.utils.AzureAPIWrapper; +import com.appdynamics.monitors.azure.utils.Constants; +import com.appdynamics.monitors.azure.utils.MetricDefinition; import com.appdynamics.monitors.azure.utils.Resource; +import org.apache.commons.collections4.ListUtils; import org.slf4j.LoggerFactory; import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; @SuppressWarnings("unchecked") class AzureMonitorTask implements AMonitorTaskRunnable{ private static final org.slf4j.Logger logger = LoggerFactory.getLogger(AzureMonitorTask.class); - private static long startTime = System.currentTimeMillis(); + private static long startTime; private final MonitorContextConfiguration configuration; private final MetricWriteHelper metricWriteHelper; @@ -56,6 +60,7 @@ public void onTaskComplete() { @Override public void run() { try { + startTime = System.currentTimeMillis(); runTask(); } catch(Exception e){ @@ -75,28 +80,31 @@ private void runTask() throws Exception{ List> resourceFilters = (List>) resourceTypeFilter.get("resources"); for (Map resourceFilter : resourceFilters) { String currentResourceGroupFilter = resourceGroupFilter.get("resourceGroup").toString(); - CountDownLatch countDownLatchAzure = new CountDownLatch(resources.size()); - for (Resource resource : resources) { - String currentResourceFilter = resourceFilter.get("resource").toString(); - String currentResourceTypeFilter = resourceTypeFilter.get("resourceType").toString(); - AzureMetrics azureMetricsTask = new AzureMetrics( - azure, - resourceFilter, - currentResourceGroupFilter, - currentResourceFilter, - currentResourceTypeFilter, - resource, - subscription, - countDownLatchAzure, - metricWriteHelper, - configuration.getMetricPrefix()); - - configuration.getContext().getExecutorService().execute("AzureMetrics", azureMetricsTask); - } - try{ - countDownLatchAzure.await(); - } catch (InterruptedException e) { - e.printStackTrace(); + List> resourcesChunks = ListUtils.partition(resources, (int) (.75 * (int) configuration.getConfigYml().get("numberOfThreads"))); + for (List resourcesChunk : resourcesChunks) { + CountDownLatch countDownLatchAzure = new CountDownLatch(resourcesChunk.size()); + for (Resource resource : resourcesChunk) { + String currentResourceFilter = resourceFilter.get("resource").toString(); + String currentResourceTypeFilter = resourceTypeFilter.get("resourceType").toString(); + AzureMetrics azureMetricsTask = new AzureMetrics( + azure, + resourceFilter, + currentResourceGroupFilter, + currentResourceFilter, + currentResourceTypeFilter, + resource, + subscription, + countDownLatchAzure, + metricWriteHelper, + configuration.getMetricPrefix()); + + configuration.getContext().getExecutorService().submit(resource.getName(), azureMetricsTask); + } + try{ + countDownLatchAzure.await(Constants.TASKS_COUNTDOWN_LATCH_TIMEOUT, TimeUnit.SECONDS); + } catch (InterruptedException e) { + e.printStackTrace(); + } } } } From 6f01252d6cae0e36d45f60931332d028175c6082 Mon Sep 17 00:00:00 2001 From: Isi Ramirez Date: Wed, 29 Aug 2018 15:29:49 +0200 Subject: [PATCH 46/51] Change azure metrics to runnable --- .../appdynamics/monitors/azure/AzureRestOperation.java | 2 ++ .../appdynamics/monitors/azure/metrics/AzureMetrics.java | 9 ++------- .../com/appdynamics/monitors/azure/utils/Constants.java | 2 ++ 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/appdynamics/monitors/azure/AzureRestOperation.java b/src/main/java/com/appdynamics/monitors/azure/AzureRestOperation.java index 09c3f27..22119db 100644 --- a/src/main/java/com/appdynamics/monitors/azure/AzureRestOperation.java +++ b/src/main/java/com/appdynamics/monitors/azure/AzureRestOperation.java @@ -11,6 +11,7 @@ import com.appdynamics.monitors.azure.utils.Constants; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.net.ssl.*; @@ -19,6 +20,7 @@ import java.net.URL; import java.security.*; import java.util.HashMap; +import java.util.List; import java.util.Map; public class AzureRestOperation { diff --git a/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java b/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java index 334b375..72eff0b 100644 --- a/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java +++ b/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java @@ -20,7 +20,6 @@ import org.apache.commons.collections4.ListUtils; import org.slf4j.LoggerFactory; -import com.appdynamics.extensions.AMonitorTaskRunnable; import com.appdynamics.extensions.MetricWriteHelper; import com.appdynamics.extensions.metrics.Metric; import com.appdynamics.monitors.azure.utils.Constants; @@ -32,7 +31,8 @@ import com.fasterxml.jackson.databind.JsonNode; @SuppressWarnings("unchecked") -public class AzureMetrics implements AMonitorTaskRunnable { +public class AzureMetrics implements Runnable { + private static final org.slf4j.Logger logger = LoggerFactory.getLogger(AzureMetrics.class); private final Map resourceFilter; private final String currentResourceGroupFilter; @@ -73,11 +73,6 @@ public AzureMetrics(AzureAPIWrapper azure, } } - @Override - public void onTaskComplete() { - logger.info("Task Complete"); - } - @Override public void run() { try { diff --git a/src/main/java/com/appdynamics/monitors/azure/utils/Constants.java b/src/main/java/com/appdynamics/monitors/azure/utils/Constants.java index 364ca27..79f4e5a 100644 --- a/src/main/java/com/appdynamics/monitors/azure/utils/Constants.java +++ b/src/main/java/com/appdynamics/monitors/azure/utils/Constants.java @@ -17,5 +17,7 @@ public class Constants { public static final String AZURE_MSI_TOKEN_ENDPOINT = "http://169.254.169.254/metadata/identity/oauth2/token"; public static final int AZURE_METRICS_CHUNK_SIZE = 20; public static final int AZURE_METRICS_API_ENDPOINT_LAST_SLASH_POS = 11; + public static final long MONITOR_COUNTDOWN_LATCH_TIMEOUT = 45; + public static final long TASKS_COUNTDOWN_LATCH_TIMEOUT = 45; public static String accessToken; } From ec96cae8bb58e775c4c18958e1ac2e33a9e23e89 Mon Sep 17 00:00:00 2001 From: Isi Ramirez Date: Thu, 30 Aug 2018 21:33:13 +0200 Subject: [PATCH 47/51] Close connection and input stream in finally clause --- .../monitors/azure/AzureMonitorTask.java | 1 - .../monitors/azure/AzureRestOperation.java | 18 +++++++++++++++--- .../monitors/azure/utils/Constants.java | 2 ++ 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/appdynamics/monitors/azure/AzureMonitorTask.java b/src/main/java/com/appdynamics/monitors/azure/AzureMonitorTask.java index 8ad487d..1c7af52 100644 --- a/src/main/java/com/appdynamics/monitors/azure/AzureMonitorTask.java +++ b/src/main/java/com/appdynamics/monitors/azure/AzureMonitorTask.java @@ -14,7 +14,6 @@ import com.appdynamics.monitors.azure.metrics.AzureMetrics; import com.appdynamics.monitors.azure.utils.AzureAPIWrapper; import com.appdynamics.monitors.azure.utils.Constants; -import com.appdynamics.monitors.azure.utils.MetricDefinition; import com.appdynamics.monitors.azure.utils.Resource; import org.apache.commons.collections4.ListUtils; diff --git a/src/main/java/com/appdynamics/monitors/azure/AzureRestOperation.java b/src/main/java/com/appdynamics/monitors/azure/AzureRestOperation.java index 22119db..febd872 100644 --- a/src/main/java/com/appdynamics/monitors/azure/AzureRestOperation.java +++ b/src/main/java/com/appdynamics/monitors/azure/AzureRestOperation.java @@ -33,23 +33,25 @@ public static JsonNode doGet(URL url) { } public static JsonNode doGet(URL url, Map headers) { + HttpURLConnection conn = null; + BufferedReader br = null; try { logger.debug("--> GET " + url); ObjectMapper objectMapper = new ObjectMapper(); String response = ""; - HttpURLConnection conn; conn = (HttpURLConnection) url.openConnection(); + conn.setConnectTimeout(Constants.AZURE_CONNECTION_TIMEOUT); + conn.setReadTimeout(Constants.AZURE_READ_TIMEOUT); conn.setDoOutput(true); conn.setRequestMethod("GET"); for (Map.Entry header : headers.entrySet()) { conn.setRequestProperty(header.getKey(), header.getValue()); } conn.setRequestProperty("Content-Type", "application/json"); - BufferedReader br = new BufferedReader(new InputStreamReader( + br = new BufferedReader(new InputStreamReader( (conn.getInputStream()))); //noinspection StatementWithEmptyBody for (String line; (line = br.readLine()) != null; response += line); - conn.disconnect(); if (logger.isDebugEnabled()) { logger.debug("API response: " + response); } @@ -57,6 +59,16 @@ public static JsonNode doGet(URL url, Map headers) { } catch (IOException e) { logger.error("Error while processing GET on URL {}", url, e); return null; + } finally { + if (br != null) { + try { + br.close(); + } catch (IOException e) { + } + } + if (conn != null) { + conn.disconnect(); + } } } diff --git a/src/main/java/com/appdynamics/monitors/azure/utils/Constants.java b/src/main/java/com/appdynamics/monitors/azure/utils/Constants.java index 79f4e5a..da7ff56 100644 --- a/src/main/java/com/appdynamics/monitors/azure/utils/Constants.java +++ b/src/main/java/com/appdynamics/monitors/azure/utils/Constants.java @@ -17,6 +17,8 @@ public class Constants { public static final String AZURE_MSI_TOKEN_ENDPOINT = "http://169.254.169.254/metadata/identity/oauth2/token"; public static final int AZURE_METRICS_CHUNK_SIZE = 20; public static final int AZURE_METRICS_API_ENDPOINT_LAST_SLASH_POS = 11; + public static final int AZURE_CONNECTION_TIMEOUT = 10000; + public static final int AZURE_READ_TIMEOUT = 10000; public static final long MONITOR_COUNTDOWN_LATCH_TIMEOUT = 45; public static final long TASKS_COUNTDOWN_LATCH_TIMEOUT = 45; public static String accessToken; From a5856902af408b8cf87a73e38c9e2a7f1d2a2bf7 Mon Sep 17 00:00:00 2001 From: Isi Ramirez Date: Thu, 30 Aug 2018 21:47:20 +0200 Subject: [PATCH 48/51] Add release notes --- CHANGES.MD | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGES.MD b/CHANGES.MD index e5c81f7..1166b7b 100644 --- a/CHANGES.MD +++ b/CHANGES.MD @@ -1,3 +1,9 @@ +### Version 1.11.0.0 (Alpha) + +* Add managed service identity support +* Add tests (mock both api and controller calls) +* Bug fixes (memory leaks) + ### Version 1.10.0.0 (Alpha) * Add basic dimensions support From 265e5bee3be4844cc449ce4be794a50dcc2cc688 Mon Sep 17 00:00:00 2001 From: Isi Ramirez Date: Thu, 30 Aug 2018 22:47:38 +0200 Subject: [PATCH 49/51] Update README file --- README.MD | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) diff --git a/README.MD b/README.MD index 5916b11..71ea48d 100644 --- a/README.MD +++ b/README.MD @@ -36,6 +36,14 @@ subscriptions: 2. Run `mvn -DskipTests clean install` 3. The `AzureMonitor-.zip` file can be found in the `target` directory +## CLI support + +It is possible to test the extension without the machine agent, and without sending real metrics to the controller by using the command line. Both the machine agent and the extension jar files must be present in the class path. + +Usage: java -Dorg.slf4j.simpleLogger.defaultLogLevel=DEBUG -cp ./machine-agent-x.x.xx.jar:./target/azure-monitoring-extension.jar com.appdynamics.monitors.azure.AzureMonitor + +NOTE: Depending on JRE version, you might need to add this parameter "--add-modules java.xml.bind" + ## Directory Structure @@ -66,13 +74,62 @@ subscriptions:
+## Authentication using MSI (Managed Service Identity) + +Managed Service Identity is a feature of Azure Active Directory that simplifies the way that credentials are handled. More info here: https://docs.microsoft.com/en-us/azure/active-directory/managed-service-identity/overview + +To make use of this feature on a subscription, set the parameter useMSI to true in the config.yml file. + ``` +subscriptions: + - subscriptionId: "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" + useMSI: "true" + ``` + +## Dimensions + +In case you need more granularity for the metrics, it is possible to define filters to use Azure dimensions. For more info, check https://docs.microsoft.com/en-us/azure/monitoring-and-diagnostics/monitoring-rest-api-walkthrough#retrieve-metric-definitions-multi-dimensional-api + + ``` + metrics: + - metric: "ThrottledRequests" + filter: "EntityName eq 'myTopic'" + alias: "Throttled Requests" + subpath: "myTopic" + ``` + +As a result, this would be the path of the metric that would be sent to the controller (note the use of the alias parameter to change the name of the metric in the path): + +Custom Metrics|AzureMonitor|xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx|resourceGroup1|namespaces|TestEventHub1|myTopic|Throttled Requests + ## Metrics Metrics available for Azure are maintained within the [Azure Monitor Documentation - Reference - Metrics Supported](https://docs.microsoft.com/en-us/azure/monitoring-and-diagnostics/monitoring-supported-metrics) +## Metrics alias + +The same way as with dimensions, it is possible to use the alias parameter to change the name of any simple metric in the path. + + ``` + metrics: + - metric: "ThrottledRequests" + alias: "Throttled Requests" + ``` +And this would be the path of the metric sent to the controller: + +Custom Metrics|AzureMonitor|xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx|resourceGroup1|namespaces|TestEventHub1|Throttled Requests + +## Metric time rollup + +The time-rollup qualifier specifies how the Controller rolls up the values when it converts from one-minute granularity tables to 10-minute granularity and 60-minute granularity tables over time. For more info, check https://docs.appdynamics.com/display/PRO45/Build+a+Monitoring+Extension+Using+Java#BuildaMonitoringExtensionUsingJava-MetricProcessingQualifier + + ``` + - metric: "UnauthorizedRequests" + timeRollUp: "METRIC_TIME_ROLLUP_TYPE_AVERAGE" + ``` + ## Excluding Metrics for Resources -You can exlcude certain Metrics by adding an include within a filter Resource Element. To do this the following configuration in `config.yml` is required: +You can exclude certain Metrics by adding an include within a filter Resource Element. To do this the following configuration in `config.yml` is required: ``` subscriptions: - subscriptionId: "" From 6726f5bfb51c2ebd88a9c5bcb53f3500165067bd Mon Sep 17 00:00:00 2001 From: Isi Ramirez Date: Thu, 30 Aug 2018 23:00:12 +0200 Subject: [PATCH 50/51] Fix README style --- README.MD | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.MD b/README.MD index 71ea48d..d5d1a37 100644 --- a/README.MD +++ b/README.MD @@ -40,9 +40,9 @@ subscriptions: It is possible to test the extension without the machine agent, and without sending real metrics to the controller by using the command line. Both the machine agent and the extension jar files must be present in the class path. -Usage: java -Dorg.slf4j.simpleLogger.defaultLogLevel=DEBUG -cp ./machine-agent-x.x.xx.jar:./target/azure-monitoring-extension.jar com.appdynamics.monitors.azure.AzureMonitor +Usage: `java -Dorg.slf4j.simpleLogger.defaultLogLevel=DEBUG -cp ./machine-agent-x.x.xx.jar:./target/azure-monitoring-extension.jar com.appdynamics.monitors.azure.AzureMonitor` -NOTE: Depending on JRE version, you might need to add this parameter "--add-modules java.xml.bind" +NOTE: Depending on JRE version, you might need to add this parameter `--add-modules java.xml.bind` ## Directory Structure @@ -76,7 +76,7 @@ NOTE: Depending on JRE version, you might need to add this parameter "--add-modu ## Authentication using MSI (Managed Service Identity) -Managed Service Identity is a feature of Azure Active Directory that simplifies the way that credentials are handled. More info here: https://docs.microsoft.com/en-us/azure/active-directory/managed-service-identity/overview +Managed Service Identity is a feature of Azure Active Directory that simplifies the way that credentials are handled. More info [here](https://docs.microsoft.com/en-us/azure/active-directory/managed-service-identity/overview) To make use of this feature on a subscription, set the parameter useMSI to true in the config.yml file. ``` @@ -87,7 +87,7 @@ subscriptions: ## Dimensions -In case you need more granularity for the metrics, it is possible to define filters to use Azure dimensions. For more info, check https://docs.microsoft.com/en-us/azure/monitoring-and-diagnostics/monitoring-rest-api-walkthrough#retrieve-metric-definitions-multi-dimensional-api +In case you need more granularity for the metrics, it is possible to define filters to use Azure dimensions. For more info, check [here](https://docs.microsoft.com/en-us/azure/monitoring-and-diagnostics/monitoring-rest-api-walkthrough#retrieve-metric-definitions-multi-dimensional-api) ``` metrics: @@ -99,7 +99,7 @@ In case you need more granularity for the metrics, it is possible to define filt As a result, this would be the path of the metric that would be sent to the controller (note the use of the alias parameter to change the name of the metric in the path): -Custom Metrics|AzureMonitor|xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx|resourceGroup1|namespaces|TestEventHub1|myTopic|Throttled Requests +`Custom Metrics|AzureMonitor|xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx|resourceGroup1|namespaces|TestEventHub1|myTopic|Throttled Requests` ## Metrics @@ -116,11 +116,11 @@ The same way as with dimensions, it is possible to use the alias parameter to ch ``` And this would be the path of the metric sent to the controller: -Custom Metrics|AzureMonitor|xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx|resourceGroup1|namespaces|TestEventHub1|Throttled Requests +`Custom Metrics|AzureMonitor|xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx|resourceGroup1|namespaces|TestEventHub1|Throttled Requests` -## Metric time rollup +## Metric time-rollup -The time-rollup qualifier specifies how the Controller rolls up the values when it converts from one-minute granularity tables to 10-minute granularity and 60-minute granularity tables over time. For more info, check https://docs.appdynamics.com/display/PRO45/Build+a+Monitoring+Extension+Using+Java#BuildaMonitoringExtensionUsingJava-MetricProcessingQualifier +The time-rollup qualifier specifies how the Controller rolls up the values when it converts from one-minute granularity tables to 10-minute granularity and 60-minute granularity tables over time. For more info, check [here](https://docs.appdynamics.com/display/PRO45/Build+a+Monitoring+Extension+Using+Java#BuildaMonitoringExtensionUsingJava-MetricProcessingQualifier) ``` - metric: "UnauthorizedRequests" From e5c588931fbabce93adabd5eec4fa0bf4bbaae0c Mon Sep 17 00:00:00 2001 From: Isi Ramirez Date: Mon, 10 Dec 2018 08:57:01 +0100 Subject: [PATCH 51/51] Add utils missing classes --- .../monitors/azure/utils/AzureAPIWrapper.java | 105 ++++++++++++++++++ .../azure/utils/MetricDefinition.java | 26 +++++ .../monitors/azure/utils/Resource.java | 51 +++++++++ 3 files changed, 182 insertions(+) create mode 100644 src/main/java/com/appdynamics/monitors/azure/utils/AzureAPIWrapper.java create mode 100644 src/main/java/com/appdynamics/monitors/azure/utils/MetricDefinition.java create mode 100644 src/main/java/com/appdynamics/monitors/azure/utils/Resource.java diff --git a/src/main/java/com/appdynamics/monitors/azure/utils/AzureAPIWrapper.java b/src/main/java/com/appdynamics/monitors/azure/utils/AzureAPIWrapper.java new file mode 100644 index 0000000..cc50ebf --- /dev/null +++ b/src/main/java/com/appdynamics/monitors/azure/utils/AzureAPIWrapper.java @@ -0,0 +1,105 @@ +package com.appdynamics.monitors.azure.utils; + +import java.io.UnsupportedEncodingException; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLEncoder; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.commons.lang3.StringUtils; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; + +import com.appdynamics.monitors.azure.AzureAuth; +import com.appdynamics.monitors.azure.AzureRestOperation; +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.Lists; +import com.microsoft.azure.PagedList; +import com.microsoft.azure.management.Azure; +import com.microsoft.azure.management.monitor.MetricDefinition; +import com.microsoft.azure.management.resources.GenericResource; + +public class AzureAPIWrapper { + private Azure azure; + private AtomicInteger azureResourcesCallCount = new AtomicInteger(0); + private AtomicInteger azureMetricDefinitionsCallCount = new AtomicInteger(0); +// private AtomicInteger azureMetricsCallCount = new AtomicInteger(0); + private Map subscription; + + public AzureAPIWrapper(Map subscription) { + this.subscription = subscription; + } + + public void authorize() { + AzureAuth.authorizeAzure(subscription); + azure = AzureAuth.getAzureMonitorAuth().withSubscription(subscription.get("subscriptionId").toString()); + } + + public List getResources() { + azureResourcesCallCount.incrementAndGet(); + PagedList resources = retrieveResources(); + List list = Lists.newArrayList();; + + for(GenericResource resource : resources) { + Resource wrappedResource = new Resource(); + wrappedResource.setName(resource.name()); + wrappedResource.setType(resource.type()); + wrappedResource.setResourceGroupName(resource.resourceGroupName()); + wrappedResource.setResourceType(resource.resourceType()); + wrappedResource.setId(resource.id()); + list.add(wrappedResource); + } + + return list; + } + + private PagedList retrieveResources() { + // TODO Auto-generated method stub + return azure.genericResources().list(); + } + + public List getMetricDefinitions(String resourceId) { + azureMetricDefinitionsCallCount.incrementAndGet(); + List resourceMetrics = azure.metricDefinitions().listByResource(resourceId); + + List list = Lists.newArrayList();; + + for(MetricDefinition resourceMetric : resourceMetrics) { + com.appdynamics.monitors.azure.utils.MetricDefinition wrappedMetric = new com.appdynamics.monitors.azure.utils.MetricDefinition(); + wrappedMetric.setName(resourceMetric.name().value().toString()); + wrappedMetric.setPrimaryAggregationType(resourceMetric.primaryAggregationType().toString()); + wrappedMetric.setId(resourceMetric.id()); + list.add(wrappedMetric); + } + + return list; + } + + public JsonNode getMetrics(com.appdynamics.monitors.azure.utils.MetricDefinition metricDefinition, String metricNames) throws MalformedURLException, UnsupportedEncodingException { + return getMetrics(metricDefinition, metricNames, null); + } + + public JsonNode getMetrics(com.appdynamics.monitors.azure.utils.MetricDefinition metricDefinition, String metricNames, String filter) throws MalformedURLException, UnsupportedEncodingException { + DateTime recordDateTime = DateTime.now(DateTimeZone.UTC); + Integer ordinalIndexOfLastSlash = StringUtils.ordinalIndexOf(metricDefinition.getId(), "/", Constants.AZURE_METRICS_API_ENDPOINT_LAST_SLASH_POS); + if (ordinalIndexOfLastSlash < 0) { + throw new MalformedURLException("Invalid metrics API endpoint"); + } + String apiEndpointBase = metricDefinition.getId().substring(0,ordinalIndexOfLastSlash); + String url = Constants.AZURE_MANAGEMENT_URL + + apiEndpointBase + + "/metrics?timespan=" + recordDateTime.minusMinutes(2).toDateTimeISO() + "/" + recordDateTime.toDateTimeISO() + + "&metricnames=" + metricNames + + "&api-version=" + subscription.get("api-version"); + if (filter != null) { + url += "&$filter=" + URLEncoder.encode(filter, "UTF-8"); + } + URL apiEndpointFull = new URL(url); +// azureMetricsCallCount.incrementAndGet(); + JsonNode apiResponse = AzureRestOperation.doGet(apiEndpointFull); + + return apiResponse; + } +} diff --git a/src/main/java/com/appdynamics/monitors/azure/utils/MetricDefinition.java b/src/main/java/com/appdynamics/monitors/azure/utils/MetricDefinition.java new file mode 100644 index 0000000..c2bb2d3 --- /dev/null +++ b/src/main/java/com/appdynamics/monitors/azure/utils/MetricDefinition.java @@ -0,0 +1,26 @@ +package com.appdynamics.monitors.azure.utils; + +public class MetricDefinition { + private String id; + private String name; + private String primaryAggregationType; + + public String getId() { + return id; + } + public void setId(String id) { + this.id = id; + } + public String getName() { + return name; + } + public void setName(String name) { + this.name = name; + } + public String getPrimaryAggregationType() { + return primaryAggregationType; + } + public void setPrimaryAggregationType(String primaryAggregationType) { + this.primaryAggregationType = primaryAggregationType; + } +} diff --git a/src/main/java/com/appdynamics/monitors/azure/utils/Resource.java b/src/main/java/com/appdynamics/monitors/azure/utils/Resource.java new file mode 100644 index 0000000..8179048 --- /dev/null +++ b/src/main/java/com/appdynamics/monitors/azure/utils/Resource.java @@ -0,0 +1,51 @@ +package com.appdynamics.monitors.azure.utils; + +public class Resource { + + private String name; + private String type; + private String resourceGroupName; + private Object resourceType; + private String id; + + public String getName() { + return name; + } + + public String getType() { + return type; + } + + public String getResourceGroupName() { + return resourceGroupName; + } + + public Object getResourceType() { + return resourceType; + } + + public String getId() { + return id; + } + + public void setName(String name) { + this.name = name; + } + + public void setType(String type) { + this.type = type; + } + + public void setResourceGroupName(String resourceGroupName) { + this.resourceGroupName = resourceGroupName; + } + + public void setResourceType(String resourceType) { + this.resourceType = resourceType; + } + + public void setId(String id) { + this.id = id; + } + +}