diff --git a/CHANGES.MD b/CHANGES.MD index 446e1d1..1166b7b 100644 --- a/CHANGES.MD +++ b/CHANGES.MD @@ -1,3 +1,38 @@ +### Version 1.11.0.0 (Alpha) + +* Add managed service identity support +* Add tests (mock both api and controller calls) +* Bug fixes (memory leaks) + +### Version 1.10.0.0 (Alpha) + +* Add basic dimensions support +* Add CLI support +* Get multiple metrics with a simple API call + +### Version 1.9.9.0 (Alpha) + +* Changed to latest Extension Framework +* Usage of Azure Java SDK Bindings +* Change of Filter Strategy +* Change of Config Structure + +### Version 1.2.3 + +* Changed Exclude Filter Strategy + +### Version 1.2.2 + +* Fixed config.yml change bug + +### Version 1.2.1 + +* Error handling adjustments + +### Version 1.2 + +* Released v1.2 + ### Version 1.1.9.3 (Beta) * Added Service Fabric Certificate Authentication diff --git a/README.MD b/README.MD index d4dd7c1..d5d1a37 100644 --- a/README.MD +++ b/README.MD @@ -12,7 +12,7 @@ Monitor Metrics provided by the Azure Monitor API and let them report into the A ## Installation -Either [Download the Extension from the Github release](https://github.com/michaelenglert/azure-monitoring-extension/releases/tag/v1.1) or Build from Source. +Either [Download the Extension from the Github release](https://github.com/michaelenglert/azure-monitoring-extension/releases) or Build from Source. 1. Deploy the `AzureMonitor-.zip` file into the `/monitors` directory. @@ -20,17 +20,11 @@ Either [Download the Extension from the Github release](https://github.com/micha 2. Set up `config.yml`. At minimum this is: ``` - # Client ID obtained from the Azure Portal - clientId: "" - - # Client Key for the upper ID obtained from the Azure Portal - clientKey: "" - - # Tenant ID obtained from the Azure Portal - tenantId: "" - - # Subscription ID obtained from the Azure Portal - subscriptionId: "" +subscriptions: + - subscriptionId: "" + tenantId: "" + clientId: "" + clientKey: "" ``` Details for the Setup can be found in the [Azure - Resource Manager - Howto - Control Access - Create Service Principal - Azure Portal](https://docs.microsoft.com/en-us/azure/azure-resource-manager/resource-group-create-service-principal-portal) @@ -42,6 +36,14 @@ Either [Download the Extension from the Github release](https://github.com/micha 2. Run `mvn -DskipTests clean install` 3. The `AzureMonitor-.zip` file can be found in the `target` directory +## CLI support + +It is possible to test the extension without the machine agent, and without sending real metrics to the controller by using the command line. Both the machine agent and the extension jar files must be present in the class path. + +Usage: `java -Dorg.slf4j.simpleLogger.defaultLogLevel=DEBUG -cp ./machine-agent-x.x.xx.jar:./target/azure-monitoring-extension.jar com.appdynamics.monitors.azure.AzureMonitor` + +NOTE: Depending on JRE version, you might need to add this parameter `--add-modules java.xml.bind` + ## Directory Structure @@ -72,36 +74,91 @@ Either [Download the Extension from the Github release](https://github.com/micha
+## Authentication using MSI (Managed Service Identity) + +Managed Service Identity is a feature of Azure Active Directory that simplifies the way that credentials are handled. More info [here](https://docs.microsoft.com/en-us/azure/active-directory/managed-service-identity/overview) + +To make use of this feature on a subscription, set the parameter useMSI to true in the config.yml file. + ``` +subscriptions: + - subscriptionId: "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" + useMSI: "true" + ``` + +## Dimensions + +In case you need more granularity for the metrics, it is possible to define filters to use Azure dimensions. For more info, check [here](https://docs.microsoft.com/en-us/azure/monitoring-and-diagnostics/monitoring-rest-api-walkthrough#retrieve-metric-definitions-multi-dimensional-api) + + ``` + metrics: + - metric: "ThrottledRequests" + filter: "EntityName eq 'myTopic'" + alias: "Throttled Requests" + subpath: "myTopic" + ``` + +As a result, this would be the path of the metric that would be sent to the controller (note the use of the alias parameter to change the name of the metric in the path): + +`Custom Metrics|AzureMonitor|xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx|resourceGroup1|namespaces|TestEventHub1|myTopic|Throttled Requests` + ## Metrics Metrics available for Azure are maintained within the [Azure Monitor Documentation - Reference - Metrics Supported](https://docs.microsoft.com/en-us/azure/monitoring-and-diagnostics/monitoring-supported-metrics) +## Metrics alias + +The same way as with dimensions, it is possible to use the alias parameter to change the name of any simple metric in the path. + + ``` + metrics: + - metric: "ThrottledRequests" + alias: "Throttled Requests" + ``` +And this would be the path of the metric sent to the controller: + +`Custom Metrics|AzureMonitor|xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx|resourceGroup1|namespaces|TestEventHub1|Throttled Requests` + +## Metric time-rollup + +The time-rollup qualifier specifies how the Controller rolls up the values when it converts from one-minute granularity tables to 10-minute granularity and 60-minute granularity tables over time. For more info, check [here](https://docs.appdynamics.com/display/PRO45/Build+a+Monitoring+Extension+Using+Java#BuildaMonitoringExtensionUsingJava-MetricProcessingQualifier) + + ``` + - metric: "UnauthorizedRequests" + timeRollUp: "METRIC_TIME_ROLLUP_TYPE_AVERAGE" + ``` + ## Excluding Metrics for Resources -You can exlcude certain Metrics by adding an exclude within a filter Resource Element. To do this the following configuration in `config.yml` is required: +You can exclude certain Metrics by adding an include within a filter Resource Element. To do this the following configuration in `config.yml` is required: ``` - filter: - - resourceType: "Microsoft.Storage/storageAccounts" - exclude: ".*" + subscriptions: + - subscriptionId: "" + tenantId: "" + clientId: "" + clientKey: "" + resourceGroups: # You can Filter with regex on any of these Types. + - resourceGroup: ".*" # If you have multiple entries per Type be sure they do not overlap + resourceTypes: # --> This will produce duplicate Metrics!!!! + - resourceType: ".*" + resources: + - resource: ".*" + metrics: + - metric: ".*" ``` -A regular expression is needed. Any match will be ignored for that specific resource. +A regular expression is needed. Any match will be included for that specific type. ## Integration with Azure Key Vault If you do not want to store the Client Key from the Service Principal that is used to query Resources, Metric Definitions and Metrics you can obtain the Key from an Azure Key Vault Secret. To achieve this the following configuration in `config.yml` is required: ``` - # Keyvault Client ID obtained from the Azure Portal - keyvaultClientId: "" - - # Keyvault Key for the upper ID obtained from the Azure Portal - keyvaultClientKey: "" - - # Keyvault Client Secret Url. From this URL the clientKey will be obtained - keyvaultClientSecretUrl: "" - - # Client ID obtained from the Azure Portal - clientId: "" + subscriptions: + - subscriptionId: "" + tenantId: "" + clientId: "" + keyvaultClientId: "" + keyvaultClientKey: "" + keyvaultClientSecretUrl: "" ``` The **Key Vault Client Key** can also be encrypted just as the normal **Client Key**. @@ -110,8 +167,10 @@ The **Key Vault Client Key** can also be encrypted just as the normal **Client K Currently the Extension support Certificate based Authentication to gather the Health Status of your Service Fabric Clusters. To use this those properties in `config.yml` have to be used: ``` - serviceFabricCert: 'monitors/AzureMonitor/your-cert.pfx' - serviceFabricPassphrase: '' + serviceFabrics: + - serviceFabric: "ServiceFabric" + serviceFabricCert: 'src/test/resources/cert/integration-test-sf.pfx' + serviceFabricPassphrase: '' ``` ## Password Encryption Support diff --git a/pom.xml b/pom.xml index 2028ec2..416efd9 100644 --- a/pom.xml +++ b/pom.xml @@ -1,4 +1,12 @@ + + @@ -6,68 +14,94 @@ com.appdynamics.monitors azure-monitoring-extension - 1.1.9.3 + 1.9.9.0 + jar + http://maven.apache.org UTF-8 + yyyy-MM-dd HH:mm:ss + ${project.build.directory}/AzureMonitor + ${basedir}/lib - com.fasterxml.jackson.core - jackson-databind - 2.9.2 - - - org.yaml - snakeyaml - 1.19 + com.microsoft.azure + azure + 1.12.0 com.appdynamics appd-exts-commons - 1.6.6 + 2.1.0 - com.appdynamics - machine-agent - 3.7.11 - provided + org.apache.httpcomponents + httpclient + 4.3.5 + - com.microsoft.azure - adal4j - 1.3.0 + org.apache.commons + commons-collections4 + 4.0 commons-io commons-io - 2.6 + 2.4 + + + commons-codec + commons-codec + 1.6 + + + org.mockito + mockito-all + 1.9.5 test + + com.appdynamics + machine-agent + 3.7.11 + provided + + + log4j + log4j + 1.2.17 + provided + commons-logging commons-logging - 1.2 - test + 1.1.3 + provided org.slf4j - slf4j-simple - 1.7.25 + jcl-over-slf4j + 1.7.12 + + + org.eclipse.jetty + jetty-webapp + 8.1.10.v20130312 test junit junit 4.12 - test - org.mockito - mockito-core - 2.11.0 - test + org.testng + testng + RELEASE + ${project.artifactId} @@ -76,8 +110,8 @@ maven-compiler-plugin 2.3.2 - 1.6 - 1.6 + 1.7 + 1.7 @@ -105,12 +139,11 @@ - - + + - AzureMonitor v${project.version} Build Date ${maven.build.timestamp} + Starter Monitor v${project.version} Build Date ${maven.build.timestamp} + com.appdynamics.extensions.workbench.WorkbenchServerLauncher @@ -163,6 +196,30 @@ ${project.artifactId}-${project.version} + + maven-surefire-plugin + 2.21.0 + + + **/Test*.java + **/*Test.java + **/*Tests.java + **/*TestCase.java + + + + + org.junit.platform + junit-platform-surefire-provider + 1.2.0 + + + org.junit.vintage + junit-vintage-engine + 5.2.0 + + + @@ -177,7 +234,4 @@ https://github.com/Appdynamics/maven-repo/raw/master/releases - - scm:git:https://github.com/michaelenglert/azure-monitoring-extension - \ No newline at end of file diff --git a/src/main/java/com/appdynamics/monitors/azure/AzureAuth.java b/src/main/java/com/appdynamics/monitors/azure/AzureAuth.java index 7784fea..ce61b65 100644 --- a/src/main/java/com/appdynamics/monitors/azure/AzureAuth.java +++ b/src/main/java/com/appdynamics/monitors/azure/AzureAuth.java @@ -1,5 +1,5 @@ /* - * Copyright 2017. AppDynamics LLC and its affiliates. + * Copyright 2018. AppDynamics LLC and its affiliates. * All Rights Reserved. * This is unpublished proprietary source code of AppDynamics LLC and its affiliates. * The copyright notice above does not evidence any actual or intended publication of such source code. @@ -8,66 +8,139 @@ package com.appdynamics.monitors.azure; -import com.appdynamics.monitors.azure.config.AuthenticationResults; -import com.appdynamics.monitors.azure.config.Globals; -import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.base.Strings; -import com.microsoft.aad.adal4j.AuthenticationContext; -import com.microsoft.aad.adal4j.AuthenticationResult; -import com.microsoft.aad.adal4j.ClientCredential; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URL; +import java.net.URLEncoder; +import java.util.HashMap; import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; -class AzureAuth { +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.appdynamics.monitors.azure.utils.Constants; +import com.appdynamics.monitors.azure.utils.Utilities; +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.base.Strings; +import com.microsoft.aad.adal4j.AuthenticationContext; +import com.microsoft.aad.adal4j.AuthenticationResult; +import com.microsoft.aad.adal4j.ClientCredential; +import com.microsoft.azure.AzureEnvironment; +import com.microsoft.azure.credentials.ApplicationTokenCredentials; +import com.microsoft.azure.credentials.MSICredentials; +import com.microsoft.azure.management.Azure; +import com.microsoft.azure.management.Azure.Authenticated; + +public class AzureAuth { private static final Logger logger = LoggerFactory.getLogger(AzureAuth.class); + private static Authenticated azureMonitorAuth; + + public static void authorizeAzure(Map subscription) { + Boolean useMSI = subscription.get("useMSI") == null ? false : Boolean.valueOf(subscription.get("useMSI").toString()); + if (useMSI) { + MSICredentials credentials = new MSICredentials(AzureEnvironment.AZURE); + if (logger.isDebugEnabled()) { + azureMonitorAuth = Azure + .configure() + .withLogLevel(com.microsoft.rest.LogLevel.BASIC) + .authenticate(credentials); + } else { + azureMonitorAuth = Azure.authenticate(credentials); + } - public static void getAzureAuth (Map config) { - String keyvaultClientSecretUrl = (String) config.get(Globals.keyvaultClientSecretUrl); - String keyvaultClientId = (String) config.get(Globals.keyvaultClientId); - String keyvaultClientKey = (String) config.get(Globals.keyvaultClientKey); - String clientId = (String) config.get(Globals.clientId); - String clientKey = (String) config.get(Globals.clientKey); - String tenantId = (String) config.get(Globals.tenantId); - String encryptionKey = (String) config.get(Globals.encryptionKey); - String encryptedClientKey = (String) config.get(Globals.encryptedClientKey); - String encryptedKeyvaultClientKey = (String) config.get(Globals.encryptedKeyvaultClientKey); + Constants.accessToken = getMSIAccessToken(subscription); + logger.debug("MSI: Bearer {}", Constants.accessToken); + } else { + authorizeWithClientCredentials(subscription); + } + } + + private static void authorizeWithClientCredentials(Map subscription) { + String clientId = (String) subscription.get("clientId"); + String clientKey = (String) subscription.get("clientKey"); + String tenantId = (String) subscription.get("tenantId"); + + String encryptedOrKeyVaultClientKey= getEncryptedOrKeyVaultClientKey(subscription); + if (encryptedOrKeyVaultClientKey != null) clientKey = encryptedOrKeyVaultClientKey; + + ApplicationTokenCredentials applicationTokenCredentials = new ApplicationTokenCredentials( + clientId, + tenantId, + clientKey, + AzureEnvironment.AZURE); + if (logger.isDebugEnabled()) { + azureMonitorAuth = Azure.configure() + .withLogLevel(com.microsoft.rest.LogLevel.BASIC) + .authenticate(applicationTokenCredentials); + } else { + azureMonitorAuth = Azure.authenticate(applicationTokenCredentials); + } + AuthenticationResult azureAuthResult = getAuthenticationResult(clientId, clientKey, tenantId, Constants.AZURE_MANAGEMENT_URL); + Constants.accessToken = azureAuthResult.getAccessToken(); + logger.debug("Bearer {}", azureAuthResult.getAccessToken()); + } + + private static String getEncryptedOrKeyVaultClientKey(Map subscription) { + String clientKey = null; + String tenantId = (String) subscription.get("tenantId"); + String encryptionKey = (String) subscription.get("encryption-key"); + String encryptedClientKey = (String) subscription.get("encryptedClientKey"); + String encryptedKeyvaultClientKey = (String) subscription.get("encryptedKeyvaultClientKey"); if (!Strings.isNullOrEmpty(encryptionKey) && !Strings.isNullOrEmpty(encryptedClientKey)) { clientKey = Utilities.getDecryptedKey(encryptedClientKey, encryptionKey); } - + String keyvaultClientSecretUrl = (String) subscription.get("keyvaultClientSecretUrl"); + String keyvaultClientId = (String) subscription.get("keyvaultClientId"); + String keyvaultClientKey = (String) subscription.get("keyvaultClientKey"); if (!Strings.isNullOrEmpty(encryptionKey) && !Strings.isNullOrEmpty(encryptedKeyvaultClientKey)) { keyvaultClientKey = Utilities.getDecryptedKey(encryptedKeyvaultClientKey, encryptionKey); } if (!Strings.isNullOrEmpty(keyvaultClientId) && !Strings.isNullOrEmpty(keyvaultClientKey) && !Strings.isNullOrEmpty(keyvaultClientSecretUrl)){ - URL keyvaultUrl = Utilities.getUrl(keyvaultClientSecretUrl + "?" + Globals.azureApiVersion + - "=" + config.get("keyvault-api-version")); - AuthenticationResults.azureKeyVaultAuth = getAuthenticationResult(Globals.azureKeyvaultEndpoint,keyvaultClientId,keyvaultClientKey, tenantId); - JsonNode keyVaultResponse = AzureRestOperation.doGet(AuthenticationResults.azureKeyVaultAuth, keyvaultUrl); + URL keyvaultUrl = Utilities.getUrl(keyvaultClientSecretUrl + "?" + "api-version" + + "=" + subscription.get("keyvault-api-version")); + AuthenticationResult azureKeyVaultAuth = getAuthenticationResult(keyvaultClientId, keyvaultClientKey, tenantId, Constants.AZURE_VAULT_URL); + Constants.accessToken = azureKeyVaultAuth.getAccessToken(); + logger.debug("Bearer {}", azureKeyVaultAuth.getAccessToken()); + JsonNode keyVaultResponse = AzureRestOperation.doGet(keyvaultUrl); assert keyVaultResponse != null; clientKey = keyVaultResponse.get("value").textValue(); } - AuthenticationResults.azureMonitorAuth = getAuthenticationResult(Globals.azureEndpoint + "/", clientId, clientKey, tenantId); + return clientKey; + } + + private static String getMSIAccessToken(Map subscription) { + String resource = null; + try { + resource = URLEncoder.encode("https://management.azure.com/", "UTF-8"); + } catch (UnsupportedEncodingException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + URL url = Utilities.getUrl(Constants.AZURE_MSI_TOKEN_ENDPOINT + + "?api-version=" + subscription.get("msi-token-api-version") + + "&resource=" + resource); + Map headers = new HashMap<>(); + headers.put("Metadata", "true"); + JsonNode keyVaultResponse = AzureRestOperation.doGet(url, headers); + + return keyVaultResponse.get("access_token").textValue(); } - private static AuthenticationResult getAuthenticationResult(String endpoint, String Id, String Key, String tenantId){ + private static AuthenticationResult getAuthenticationResult(String Id, String Key, String tenantId, String resourceUrl){ ExecutorService service = Executors.newSingleThreadExecutor(); AuthenticationResult result = null; - String authority = Globals.azureAuthEndpoint + tenantId; + String authority = "https://login.microsoftonline.com/" + tenantId; try { - AuthenticationContext context; - context = new AuthenticationContext(authority, false, service); + AuthenticationContext context = new AuthenticationContext(authority, false, service); ClientCredential cred = new ClientCredential(Id, Key); - Future future = context.acquireToken(endpoint, cred, null); + Future future = context.acquireToken(resourceUrl, cred, null); result = future.get(); } catch (MalformedURLException e) { logger.error("Not a valid Azure authentication Authority {}", authority, e); @@ -79,4 +152,8 @@ private static AuthenticationResult getAuthenticationResult(String endpoint, Str service.shutdown(); return result; } + + public static Authenticated getAzureMonitorAuth() { + return azureMonitorAuth; + } } diff --git a/src/main/java/com/appdynamics/monitors/azure/AzureMonitor.java b/src/main/java/com/appdynamics/monitors/azure/AzureMonitor.java index 78156b4..3fcff34 100644 --- a/src/main/java/com/appdynamics/monitors/azure/AzureMonitor.java +++ b/src/main/java/com/appdynamics/monitors/azure/AzureMonitor.java @@ -1,5 +1,5 @@ /* - * Copyright 2017. AppDynamics LLC and its affiliates. + * Copyright 2018. AppDynamics LLC and its affiliates. * All Rights Reserved. * This is unpublished proprietary source code of AppDynamics LLC and its affiliates. * The copyright notice above does not evidence any actual or intended publication of such source code. @@ -8,139 +8,73 @@ package com.appdynamics.monitors.azure; -import com.appdynamics.extensions.conf.MonitorConfiguration; -import com.appdynamics.extensions.util.MetricWriteHelper; -import com.appdynamics.extensions.util.MetricWriteHelperFactory; -import com.appdynamics.extensions.conf.MonitorConfiguration.ConfItem; -import com.appdynamics.monitors.azure.config.AuthenticationResults; -import com.appdynamics.monitors.azure.config.Globals; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.singularity.ee.agent.systemagent.api.AManagedMonitor; -import com.singularity.ee.agent.systemagent.api.TaskExecutionContext; -import com.singularity.ee.agent.systemagent.api.TaskOutput; -import com.singularity.ee.agent.systemagent.api.exception.TaskExecutionException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.net.URL; -import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; -@SuppressWarnings("WeakerAccess") -public class AzureMonitor extends AManagedMonitor { - private static final Logger logger = LoggerFactory.getLogger(AzureMonitor.class); - private MonitorConfiguration configuration; +import org.slf4j.LoggerFactory; - public AzureMonitor() { logger.info(String.format("Using Azure Monitor Version [%s]", getImplementationVersion())); } +import com.appdynamics.extensions.ABaseMonitor; +import com.appdynamics.extensions.TasksExecutionServiceProvider; +import com.appdynamics.extensions.util.AssertUtils; +import com.appdynamics.monitors.azure.utils.AzureAPIWrapper; +import com.appdynamics.monitors.azure.utils.Constants; +import com.singularity.ee.agent.systemagent.api.exception.TaskExecutionException; - private void initialize(Map argsMap) { - if (configuration == null) { - MetricWriteHelper metricWriteHelper = MetricWriteHelperFactory.create(this); - MonitorConfiguration conf = new MonitorConfiguration(Globals.defaultMetricPrefix, - new TaskRunnable(), metricWriteHelper); - final String configFilePath = argsMap.get(Globals.configFile); - conf.setConfigYml(configFilePath); - conf.setMetricWriter(MetricWriteHelperFactory.create(this)); - conf.checkIfInitialized(ConfItem.CONFIG_YML, - ConfItem.EXECUTOR_SERVICE, - ConfItem.METRIC_PREFIX, - ConfItem.METRIC_WRITE_HELPER); - this.configuration = conf; - } +@SuppressWarnings({"WeakerAccess", "unchecked"}) +public class AzureMonitor extends ABaseMonitor { + private static final org.slf4j.Logger logger = LoggerFactory.getLogger(AzureMonitorTask.class); + + @Override + protected String getDefaultMetricPrefix() { + return Constants.DEFAULT_METRIC_PREFIX; } - public TaskOutput execute(Map map, TaskExecutionContext taskExecutionContext) throws TaskExecutionException { - try{ - if(map != null){ - if (logger.isDebugEnabled()) {logger.debug("The raw arguments are {}", map);} - initialize(map); - configuration.executeTask(); - return new TaskOutput("Azure Monitor Metric Upload Complete"); - } - } - catch(Exception e) { - logger.error("Failed to execute the Azure monitoring task", e); - } - throw new TaskExecutionException("Azure monitoring task completed with failures."); + @Override + public String getMonitorName() { + return "Azure Monitoring Extension"; + } + @Override + public void onComplete() { + logger.info("Monitor Completed"); } - private class TaskRunnable implements Runnable { - public void run () { - Map config = configuration.getConfigYml(); - if (config != null ) { - AzureAuth.getAzureAuth(config); - JsonNode filtersJson = Utilities.getFiltersJson((ArrayList) config.get(Globals.azureApiFilter)); - String filterUrl = Utilities.getFilterUrl(filtersJson); - Map resourceFilter = Utilities.getResourceFilter(filtersJson); - URL resourcesUrl = Utilities.getUrl(Globals.azureEndpoint + - Globals.azureApiSubscriptions + - config.get(Globals.subscriptionId) + - Globals.azureApiResources + - "?" + Globals.azureApiVersion + - "=" + config.get(Globals.azureApiVersion) + - filterUrl); - JsonNode resourcesResponse = AzureRestOperation.doGet(AuthenticationResults.azureMonitorAuth,resourcesUrl); - if (logger.isDebugEnabled()) { - logger.debug("Get Resources REST API Request: " + resourcesUrl.toString()); - logger.debug("Get Resources Response JSON: " + Utilities.prettifyJson(resourcesResponse)); - } - assert resourcesResponse != null; - ArrayNode resourceElements = (ArrayNode) resourcesResponse.get("value"); - for(JsonNode resourceNode:resourceElements){ - if (resourceNode.get("id").asText().contains("Microsoft.ServiceFabric/clusters")){ - JsonNode serviceFabricResponse = AzureRestOperation.doGet(AuthenticationResults.azureMonitorAuth, Utilities.getUrl( - Globals.azureEndpoint + - resourceNode.get("id").asText() + - "?" + Globals.azureApiVersion + - "=" + config.get(Globals.serviceFabricResourceApiVersion))); - assert serviceFabricResponse != null; - ServiceFabricTask fabricTask = new ServiceFabricTask( - configuration, - serviceFabricResponse, - serviceFabricResponse.get("name").asText()); - configuration.getExecutorService().execute(fabricTask); - } - URL metricDefinitions = Utilities.getUrl( - Globals.azureEndpoint + - resourceNode.get("id").asText() + - Globals.azureApiMetricDefinitions + - "?" + Globals.azureApiVersion + - "=" + config.get(Globals.azureMonitorApiVersion)); - JsonNode metricDefinitionResponse = AzureRestOperation.doGet(AuthenticationResults.azureMonitorAuth,metricDefinitions); - if (logger.isDebugEnabled()) { - logger.debug("Get Metric Definitions REST API Request: " - + metricDefinitions.toString()); - logger.debug("Get Metric Definitions Response JSON: " - + Utilities.prettifyJson(metricDefinitionResponse)); - } - assert metricDefinitionResponse != null; - ArrayNode metricDefinitionElements = (ArrayNode) metricDefinitionResponse.get("value"); - for(JsonNode metricDefinitionNode:metricDefinitionElements){ - if (metricDefinitionNode.get("isDimensionRequired").asText().equals("true")){ - logger.info("Dimensions are currently not supported. Skipping " - + metricDefinitionNode.get("id").asText()); - } - else if (Utilities.checkResourceFilter(metricDefinitionNode,resourceFilter)){ - logger.info("Ignoring Metric " + - metricDefinitionNode.get("name").get("value").asText() + - " for Resource " + metricDefinitionNode.get("resourceId")); - } - else { - AzureMonitorTask monitorTask = new AzureMonitorTask( - configuration, - resourceNode, - AuthenticationResults.azureMonitorAuth, - metricDefinitionNode.get("name").get("value").asText()); - configuration.getExecutorService().execute(monitorTask); - } - } - } - logger.info("Finished gathering Metrics"); - } - else { logger.error("The config.yml is not loaded due to previous errors.The task will not run"); } + @Override + protected void doRun(TasksExecutionServiceProvider tasksExecutionServiceProvider) { + List> subscriptions = (List>)getContextConfiguration().getConfigYml().get("subscriptions"); + AssertUtils.assertNotNull(subscriptions, "The 'subscriptions' section in config.yml is not initialised"); + CountDownLatch countDownLatch = new CountDownLatch(getTaskCount()); + for (Map subscription : subscriptions) { + AzureAPIWrapper azure = new AzureAPIWrapper(subscription); + AzureMonitorTask task = new AzureMonitorTask(getContextConfiguration(), tasksExecutionServiceProvider.getMetricWriteHelper(), subscription, countDownLatch, azure); + tasksExecutionServiceProvider.submit(subscription.get("subscriptionId").toString(), task); + } + try{ + countDownLatch.await(Constants.MONITOR_COUNTDOWN_LATCH_TIMEOUT, TimeUnit.SECONDS); + } catch (InterruptedException e) { + e.printStackTrace(); } } - private static String getImplementationVersion() { return AzureMonitor.class.getPackage().getImplementationTitle(); } + @Override + protected int getTaskCount() { + List> subscriptions = (List>)getContextConfiguration().getConfigYml().get("subscriptions"); + // List> serviceFabrics = (List>)getContextConfiguration().getConfigYml().get("serviceFabrics"); + AssertUtils.assertNotNull(subscriptions, "The 'subscriptions' section in config.yml is not initialised"); + // AssertUtils.assertNotNull(serviceFabrics, "The 'serviceFabrics' section in config.yml is not initialised"); +// return subscriptions.size() + serviceFabrics.size(); + return subscriptions.size(); + } + + public static void main(String[] args) throws TaskExecutionException { + + AzureMonitor monitor = new AzureMonitor(); + + Map taskArgs = new HashMap(); + taskArgs.put("config-file",Constants.TEST_CONFIG_FILE); + monitor.execute(taskArgs, null); + } } diff --git a/src/main/java/com/appdynamics/monitors/azure/AzureMonitorTask.java b/src/main/java/com/appdynamics/monitors/azure/AzureMonitorTask.java index c5955c5..1c7af52 100644 --- a/src/main/java/com/appdynamics/monitors/azure/AzureMonitorTask.java +++ b/src/main/java/com/appdynamics/monitors/azure/AzureMonitorTask.java @@ -1,5 +1,5 @@ /* - * Copyright 2017. AppDynamics LLC and its affiliates. + * Copyright 2018. AppDynamics LLC and its affiliates. * All Rights Reserved. * This is unpublished proprietary source code of AppDynamics LLC and its affiliates. * The copyright notice above does not evidence any actual or intended publication of such source code. @@ -8,120 +8,105 @@ package com.appdynamics.monitors.azure; -import com.appdynamics.extensions.conf.MonitorConfiguration; -import com.appdynamics.monitors.azure.config.Globals; -import com.fasterxml.jackson.databind.JsonNode; -import com.microsoft.aad.adal4j.AuthenticationResult; -import org.slf4j.Logger; +import com.appdynamics.extensions.AMonitorTaskRunnable; +import com.appdynamics.extensions.MetricWriteHelper; +import com.appdynamics.extensions.conf.MonitorContextConfiguration; +import com.appdynamics.monitors.azure.metrics.AzureMetrics; +import com.appdynamics.monitors.azure.utils.AzureAPIWrapper; +import com.appdynamics.monitors.azure.utils.Constants; +import com.appdynamics.monitors.azure.utils.Resource; + +import org.apache.commons.collections4.ListUtils; import org.slf4j.LoggerFactory; -import java.io.UnsupportedEncodingException; -import java.math.BigDecimal; -import java.net.URL; -import java.net.URLEncoder; -import java.text.SimpleDateFormat; -import java.util.Calendar; -import java.util.TimeZone; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +@SuppressWarnings("unchecked") +class AzureMonitorTask implements AMonitorTaskRunnable{ + private static final org.slf4j.Logger logger = LoggerFactory.getLogger(AzureMonitorTask.class); + + private static long startTime; + + private final MonitorContextConfiguration configuration; + private final MetricWriteHelper metricWriteHelper; + private final Map subscription; + private final List> resourceGroupFilters; + private final CountDownLatch countDownLatch; -class AzureMonitorTask implements Runnable { - private static final Logger logger = LoggerFactory.getLogger(AzureMonitorTask.class); - private final MonitorConfiguration configuration; - private final JsonNode node; - private final AuthenticationResult azureAuth; - private final String metric; + private AzureAPIWrapper azure; - AzureMonitorTask(MonitorConfiguration configuration, JsonNode node, AuthenticationResult azureAuth, String metric) { + AzureMonitorTask(MonitorContextConfiguration configuration, MetricWriteHelper metricWriteHelper, Map subscription, CountDownLatch countDownLatch, AzureAPIWrapper azure) { this.configuration = configuration; - this.node = node; - this.azureAuth = azureAuth; - this.metric = metric; + this.metricWriteHelper = metricWriteHelper; + this.subscription = subscription; + this.countDownLatch = countDownLatch; + this.resourceGroupFilters = (List>) subscription.get("resourceGroups"); + this.azure = azure; +// AssertUtils.assertNotNull(resourceGroupFilters, "The 'resourceGroupFilters' section in config.yml is either null or empty"); } + @Override + public void onTaskComplete() { + logger.info("Task Completed for subscription {}", subscription.get("subscriptionId").toString()); + long finishTime = System.currentTimeMillis(); + long totalTime = finishTime - startTime; + logger.debug("Total time: " + (totalTime / 1000.0f) + " ms"); +// logger.debug("azureResourcesCallCount: " + this.azureResourcesCallCount); + } + + @Override public void run() { try { + startTime = System.currentTimeMillis(); runTask(); - } catch (Exception e) { - configuration.getMetricWriter().registerError(e.getMessage(), e); - logger.error("Error while running the task", e); } - } - - private void runTask() { - TimeZone utc = TimeZone.getTimeZone("UTC"); - Calendar endTime = Calendar.getInstance(); - Calendar startTime = Calendar.getInstance(); - startTime.add(Calendar.MINUTE, Globals.timeOffset); - SimpleDateFormat dateFormatter = new SimpleDateFormat(Globals.azureApiTimeFormat); - dateFormatter.setTimeZone(utc); - if (logger.isDebugEnabled()) { - logger.debug("JSON Node: " + Utilities.prettifyJson(node)); + catch(Exception e){ + logger.error(e.getMessage()); } - URL url = null; - try { - url = Utilities.getUrl( - Globals.azureEndpoint + - node.get("id").asText() + - Globals.azureApiMetrics + - "?" + Globals.azureApiVersion + - "=" + configuration.getConfigYml().get(Globals.azureMonitorApiVersion) + - "&" + Globals.azureApiTimeSpan + - "=" + dateFormatter.format(startTime.getTime()) + - "/" + dateFormatter.format(endTime.getTime()) + - "&" + Globals.metric + - "=" + URLEncoder.encode(metric,Globals.urlEncoding)); - } catch (UnsupportedEncodingException e) { - logger.error("Failed to encode Metric {} with {}", metric, Globals.urlEncoding, e); + finally { + countDownLatch.countDown(); } - if (logger.isDebugEnabled()) { - assert url != null; - logger.debug("Get Metrics REST API Request: " + url.toString());} - assert url != null; - extractMetrics(AzureRestOperation.doGet(azureAuth,url)); } - - private void extractMetrics(JsonNode json){ - if (logger.isDebugEnabled()) {logger.debug("Get Metrics Response JSON: " + Utilities.prettifyJson(json));} - JsonNode jsonValue = json.get("value"); - for (JsonNode currentValueNode:jsonValue){ - String metricId = extractMetridId(currentValueNode.get("id").asText()); - String metricNameValue = currentValueNode.get("name").get("value").asText(); - String metricUnit = currentValueNode.get("unit").asText(); - String metricType = null; - BigDecimal metricValue = null; - if(currentValueNode.get("timeseries").has(0)){ - JsonNode jsonData = currentValueNode.get("timeseries").get(0).get("data"); - for (JsonNode currentDataNode:jsonData){ - if (currentDataNode.has("average")){ - metricType = "average"; metricValue = currentDataNode.get("average").decimalValue(); - } - else if (currentDataNode.has("total")){ - metricType = "total"; metricValue = currentDataNode.get("total").decimalValue(); - } - else if (currentDataNode.has("last")){ - metricType = "last"; metricValue = currentDataNode.get("last").decimalValue(); + private void runTask() throws Exception{ + azure.authorize(); + List resources = azure.getResources(); + if (resources == null || resources.size() == 0) throw new Exception("Resources list is null or empty"); + for (Map resourceGroupFilter : resourceGroupFilters) { + List> resourceTypeFilters = (List>) resourceGroupFilter.get("resourceTypes"); + for (Map resourceTypeFilter : resourceTypeFilters) { + List> resourceFilters = (List>) resourceTypeFilter.get("resources"); + for (Map resourceFilter : resourceFilters) { + String currentResourceGroupFilter = resourceGroupFilter.get("resourceGroup").toString(); + List> resourcesChunks = ListUtils.partition(resources, (int) (.75 * (int) configuration.getConfigYml().get("numberOfThreads"))); + for (List resourcesChunk : resourcesChunks) { + CountDownLatch countDownLatchAzure = new CountDownLatch(resourcesChunk.size()); + for (Resource resource : resourcesChunk) { + String currentResourceFilter = resourceFilter.get("resource").toString(); + String currentResourceTypeFilter = resourceTypeFilter.get("resourceType").toString(); + AzureMetrics azureMetricsTask = new AzureMetrics( + azure, + resourceFilter, + currentResourceGroupFilter, + currentResourceFilter, + currentResourceTypeFilter, + resource, + subscription, + countDownLatchAzure, + metricWriteHelper, + configuration.getMetricPrefix()); + + configuration.getContext().getExecutorService().submit(resource.getName(), azureMetricsTask); + } + try{ + countDownLatchAzure.await(Constants.TASKS_COUNTDOWN_LATCH_TIMEOUT, TimeUnit.SECONDS); + } catch (InterruptedException e) { + e.printStackTrace(); + } } - else if (currentDataNode.has("maximum")){ - metricType = "maximum"; metricValue = currentDataNode.get("maximum").decimalValue(); - } - } - if (metricId != null && - metricNameValue != null && - metricType != null && - metricUnit != null && - metricValue != null){ - MetricPrinter metricPrinter = new MetricPrinter(configuration.getMetricWriter()); - metricPrinter.reportMetric(configuration.getMetricPrefix() + - metricId + - metricNameValue, metricValue); } } } } - - private static String extractMetridId(String fullId){ - String metricId; - String[] metricIdSegments = fullId.split("resourceGroups")[1].split("providers/Microsoft\\."); - metricId = metricIdSegments[0]+metricIdSegments[1]; - metricId = metricId.replace("/","|"); - return metricId; - } } \ No newline at end of file diff --git a/src/main/java/com/appdynamics/monitors/azure/AzureRestOperation.java b/src/main/java/com/appdynamics/monitors/azure/AzureRestOperation.java index 1ace35d..febd872 100644 --- a/src/main/java/com/appdynamics/monitors/azure/AzureRestOperation.java +++ b/src/main/java/com/appdynamics/monitors/azure/AzureRestOperation.java @@ -1,5 +1,5 @@ /* - * Copyright 2017. AppDynamics LLC and its affiliates. + * Copyright 2018. AppDynamics LLC and its affiliates. * All Rights Reserved. * This is unpublished proprietary source code of AppDynamics LLC and its affiliates. * The copyright notice above does not evidence any actual or intended publication of such source code. @@ -8,9 +8,10 @@ package com.appdynamics.monitors.azure; +import com.appdynamics.monitors.azure.utils.Constants; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.microsoft.aad.adal4j.AuthenticationResult; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.net.ssl.*; @@ -18,29 +19,56 @@ import java.net.HttpURLConnection; import java.net.URL; import java.security.*; +import java.util.HashMap; +import java.util.List; +import java.util.Map; -class AzureRestOperation { +public class AzureRestOperation { private static final Logger logger = LoggerFactory.getLogger(AzureRestOperation.class); - static JsonNode doGet(AuthenticationResult azureAuth, URL url) { + public static JsonNode doGet(URL url) { + Map headers = new HashMap<>(); + headers.put("Authorization", "Bearer " + Constants.accessToken); + return doGet(url, headers); + } + + public static JsonNode doGet(URL url, Map headers) { + HttpURLConnection conn = null; + BufferedReader br = null; try { + logger.debug("--> GET " + url); ObjectMapper objectMapper = new ObjectMapper(); String response = ""; - HttpURLConnection conn; conn = (HttpURLConnection) url.openConnection(); + conn.setConnectTimeout(Constants.AZURE_CONNECTION_TIMEOUT); + conn.setReadTimeout(Constants.AZURE_READ_TIMEOUT); conn.setDoOutput(true); conn.setRequestMethod("GET"); - conn.setRequestProperty("Authorization", "Bearer " + azureAuth.getAccessToken()); + for (Map.Entry header : headers.entrySet()) { + conn.setRequestProperty(header.getKey(), header.getValue()); + } conn.setRequestProperty("Content-Type", "application/json"); - BufferedReader br = new BufferedReader(new InputStreamReader( + br = new BufferedReader(new InputStreamReader( (conn.getInputStream()))); //noinspection StatementWithEmptyBody for (String line; (line = br.readLine()) != null; response += line); - conn.disconnect(); + if (logger.isDebugEnabled()) { + logger.debug("API response: " + response); + } return objectMapper.readTree(response); } catch (IOException e) { logger.error("Error while processing GET on URL {}", url, e); return null; + } finally { + if (br != null) { + try { + br.close(); + } catch (IOException e) { + } + } + if (conn != null) { + conn.disconnect(); + } } } diff --git a/src/main/java/com/appdynamics/monitors/azure/MetricPrinter.java b/src/main/java/com/appdynamics/monitors/azure/MetricPrinter.java deleted file mode 100644 index 1a6fcfd..0000000 --- a/src/main/java/com/appdynamics/monitors/azure/MetricPrinter.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2017. AppDynamics LLC and its affiliates. - * All Rights Reserved. - * This is unpublished proprietary source code of AppDynamics LLC and its affiliates. - * The copyright notice above does not evidence any actual or intended publication of such source code. - * - */ - -package com.appdynamics.monitors.azure; - -import com.appdynamics.extensions.util.MetricWriteHelper; -import com.singularity.ee.agent.systemagent.api.MetricWriter; -import org.slf4j.LoggerFactory; -import java.math.BigDecimal; -import java.math.RoundingMode; - -@SuppressWarnings("SameParameterValue") -class MetricPrinter { - private static final org.slf4j.Logger logger = LoggerFactory.getLogger(MetricPrinter.class); - private final MetricWriteHelper metricWriter; - - MetricPrinter(MetricWriteHelper metricWriter){ - this.metricWriter = metricWriter; - } - - void reportMetric(String metricName, BigDecimal metricValue) { - if(metricValue == null){ - return; - } - printMetric(metricName, - metricValue, - MetricWriter.METRIC_AGGREGATION_TYPE_AVERAGE, - MetricWriter.METRIC_TIME_ROLLUP_TYPE_AVERAGE, - MetricWriter.METRIC_CLUSTER_ROLLUP_TYPE_INDIVIDUAL); - } - - private void printMetric(String metricPath, BigDecimal metricValue, String aggType, String timeRollupType, String clusterRollupType) { - - try{ - String metricValStr = toBigIntString(metricValue); - if(metricValStr != null) { - metricWriter.printMetric(metricPath,metricValStr,aggType,timeRollupType,clusterRollupType); - logger.debug("Sending [{}|{}|{}] metric= {},value={}", aggType, timeRollupType, clusterRollupType, metricPath, metricValStr); - } - } - catch (Exception e){ - logger.error("Error reporting metric {} with value {}",metricPath,metricValue,e); - } - } - - private String toBigIntString(BigDecimal metricValue) { - return metricValue.setScale(0, RoundingMode.HALF_UP).toBigInteger().toString(); - } -} diff --git a/src/main/java/com/appdynamics/monitors/azure/ServiceFabricTask.java b/src/main/java/com/appdynamics/monitors/azure/ServiceFabricTask.java index 3395fbf..fdcbfd6 100644 --- a/src/main/java/com/appdynamics/monitors/azure/ServiceFabricTask.java +++ b/src/main/java/com/appdynamics/monitors/azure/ServiceFabricTask.java @@ -1,5 +1,5 @@ /* - * Copyright 2017. AppDynamics LLC and its affiliates. + * Copyright 2018. AppDynamics LLC and its affiliates. * All Rights Reserved. * This is unpublished proprietary source code of AppDynamics LLC and its affiliates. * The copyright notice above does not evidence any actual or intended publication of such source code. @@ -8,89 +8,107 @@ package com.appdynamics.monitors.azure; -import com.appdynamics.extensions.conf.MonitorConfiguration; -import com.appdynamics.monitors.azure.config.Globals; +import com.appdynamics.extensions.AMonitorTaskRunnable; +import com.appdynamics.extensions.MetricWriteHelper; +import com.appdynamics.extensions.conf.MonitorContextConfiguration; +import com.appdynamics.extensions.metrics.Metric; +import com.appdynamics.monitors.azure.utils.Constants; import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.Lists; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.math.BigDecimal; import java.net.URL; import java.util.List; import java.util.Map; +import java.util.Objects; -class ServiceFabricTask implements Runnable { +class ServiceFabricTask implements AMonitorTaskRunnable { private static final Logger logger = LoggerFactory.getLogger(ServiceFabricTask.class); - private final MonitorConfiguration configuration; - private final JsonNode node; - private final String metric; + private final MonitorContextConfiguration configuration; + private final MetricWriteHelper metricWriteHelper; + private final Map serviceFabric; + private final String managementEndpoint; + private final String metricName; + private List finalMetricList; - ServiceFabricTask(MonitorConfiguration configuration, JsonNode node, String metric) { + private ServiceFabricTask(MonitorContextConfiguration configuration, MetricWriteHelper metricWriteHelper, Map serviceFabric, String managementEndpoint, String metricName) { this.configuration = configuration; - this.node = node; - this.metric = metric; + this.metricWriteHelper = metricWriteHelper; + this.serviceFabric = serviceFabric; + this.managementEndpoint = managementEndpoint; + this.metricName = metricName; } + @Override + public void onTaskComplete() { + logger.info("Task Complete"); + } + + @Override public void run() { try { runTask(); } catch (Exception e) { - configuration.getMetricWriter().registerError(e.getMessage(), e); + metricWriteHelper.registerError(e.getMessage(), e); logger.error("Error while running the task", e); } } private void runTask() throws IOException { - String serviceFabricBody = configuration.getConfigYml().get(Globals.serviceFabricBody).toString(); - String serviceFabricCert = configuration.getConfigYml().get(Globals.serviceFabricCert).toString(); - String serviceFabricPassphrase = configuration.getConfigYml().get(Globals.serviceFabricPassphrase).toString(); + String serviceFabricBody = serviceFabric.get("serviceFabricBody").toString(); + String serviceFabricCert = serviceFabric.get("serviceFabricCert").toString(); + String serviceFabricPassphrase = serviceFabric.get("serviceFabricPassphrase").toString(); + finalMetricList = Lists.newArrayList(); - if (logger.isDebugEnabled()) {logger.debug("JSON Node: " + Utilities.prettifyJson(node));} - URL url = new URL(node.get("properties").get("managementEndpoint").asText() + Globals.serviceFabricGetClusterHealthChunk + - "?" + Globals.azureApiVersion + "=" + configuration.getConfigYml().get(Globals.serviceFabricApiVersion)); + URL url = new URL(managementEndpoint + "/$/GetClusterHealthChunk" + + "?api-version=" + serviceFabric.get("serviceFabricApiVersion")); if (logger.isDebugEnabled()) {logger.debug("Get Metrics REST API Request: " + url.toString());} if (url.toString().matches("https://.*") && !serviceFabricCert.isEmpty()){ - //logger.info("Skipping Service Fabric Cluster {} because the Authentication Method is currently not supported", - // node.get("properties").get("managementEndpoint").asText()); extractMetrics(AzureRestOperation.doSecurePost(url, serviceFabricBody, serviceFabricCert, serviceFabricPassphrase)); } else { - extractMetrics(AzureRestOperation.doPost(url, serviceFabricBody)); + extractMetrics(Objects.requireNonNull(AzureRestOperation.doPost(url, serviceFabricBody))); } } private void extractMetrics(JsonNode json){ - if (logger.isDebugEnabled()) {logger.debug("Get Metrics Response JSON: " + Utilities.prettifyJson(json));} - List healtStateList = (List) configuration.getConfigYml().get(Globals.serviceFabricHealthStates); + List healtStateList = (List) serviceFabric.get("serviceFabricHealthStates"); + Metric metric; Map healtStates = (Map) healtStateList.get(0); - String metricName = configuration.getMetricPrefix() + "|" + metric + "|"; - MetricPrinter metricPrinter = new MetricPrinter(configuration.getMetricWriter()); - metricPrinter.reportMetric(metricName + - "ClusterHealth", - BigDecimal.valueOf((Integer) healtStates.get(json.get("HealthState").asText()))); - + String metricPath = configuration.getMetricPrefix() + Constants.METRIC_SEPARATOR + metricName + Constants.METRIC_SEPARATOR; + metric = new Metric("ClusterHealth", healtStates.get(json.get("HealthState").asText()).toString(), metricPath); + finalMetricList.add(metric); JsonNode nodes = json.get("NodeHealthStateChunks").get("Items"); for (JsonNode node:nodes){ - metricPrinter.reportMetric(metricName + + metric = new Metric(node.get("NodeName").asText(), healtStates.get(node.get("HealthState").asText()).toString(), metricPath + "NodeHealth" + - "|" + node.get("NodeName").asText(), - BigDecimal.valueOf((Integer) healtStates.get(node.get("HealthState").asText()))); + Constants.METRIC_SEPARATOR); + finalMetricList.add(metric); } JsonNode apps = json.get("ApplicationHealthStateChunks").get("Items"); for (JsonNode app:apps){ if (app.get("ApplicationTypeName").asText().isEmpty()){ - metricPrinter.reportMetric(metricName + + metric = new Metric("System", healtStates.get(app.get("HealthState").asText()).toString(), metricPath + "ApplicationHealth" + - "|" + "System", - BigDecimal.valueOf((Integer) healtStates.get(app.get("HealthState").asText()))); + Constants.METRIC_SEPARATOR); + finalMetricList.add(metric); } else { - metricPrinter.reportMetric(metricName + + metric = new Metric(app.get("ApplicationTypeName").asText(), healtStates.get(app.get("HealthState").asText()).toString(), metricPath + "ApplicationHealth" + - "|" + app.get("ApplicationTypeName").asText(), - BigDecimal.valueOf((Integer) healtStates.get(app.get("HealthState").asText()))); + Constants.METRIC_SEPARATOR); + finalMetricList.add(metric); } } + if (finalMetricList.isEmpty()){ + if (logger.isDebugEnabled()) { + logger.debug("Metric List is empty"); + } + } + else { + metricWriteHelper.transformAndPrintMetrics(finalMetricList); + } } } \ No newline at end of file diff --git a/src/main/java/com/appdynamics/monitors/azure/Utilities.java b/src/main/java/com/appdynamics/monitors/azure/Utilities.java deleted file mode 100644 index 33f7bd2..0000000 --- a/src/main/java/com/appdynamics/monitors/azure/Utilities.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Copyright 2017. AppDynamics LLC and its affiliates. - * All Rights Reserved. - * This is unpublished proprietary source code of AppDynamics LLC and its affiliates. - * The copyright notice above does not evidence any actual or intended publication of such source code. - * - */ - -package com.appdynamics.monitors.azure; - -import com.appdynamics.extensions.crypto.CryptoUtil; -import com.appdynamics.monitors.azure.config.Globals; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.Maps; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.net.MalformedURLException; -import java.net.URL; -import java.net.URLEncoder; -import java.util.*; - -class Utilities { - private static final Logger logger = LoggerFactory.getLogger(Utilities.class); - - static JsonNode getFiltersJson(ArrayList filters){ - ObjectMapper objectMapper = new ObjectMapper(); - String jsonInString; - JsonNode filtersJson = null; - try { - jsonInString = objectMapper.writeValueAsString(filters); - filtersJson = objectMapper.readTree(jsonInString); - } catch (JsonProcessingException e) { - e.printStackTrace(); - } catch (IOException e) { - e.printStackTrace(); - } - return filtersJson; - } - - static String getFilterUrl(JsonNode filtersJson){ - StringBuilder filterUrl = null; - Iterator iter = filtersJson.iterator(); - JsonNode currentValueNode; - if (!filtersJson.isNull()) { - filterUrl = new StringBuilder("&$" + Globals.azureApiFilter + "="); - while (iter.hasNext()) { - currentValueNode = iter.next(); - try { - filterUrl.append(URLEncoder.encode(Globals.filterBy + - Globals.filterComOp + "'" + - currentValueNode.get(Globals.filterBy).asText() + - "'", Globals.urlEncoding)); - if (iter.hasNext()) { - filterUrl.append(URLEncoder.encode(Globals.filterLogOp, Globals.urlEncoding)); - } - } catch (UnsupportedEncodingException e) { - e.printStackTrace(); - } - - } - } - return filterUrl != null ? filterUrl.toString() : null; - } - - static Map getResourceFilter(JsonNode filtersJson) { - Map resourceFilter = new HashMap(); - for (JsonNode currentValueNode:filtersJson){ - if(currentValueNode.has("exclude")){ - if (!currentValueNode.get("exclude").asText().isEmpty()){ - resourceFilter.put(currentValueNode.get(Globals.filterBy).asText(),currentValueNode.get("exclude").asText()); - } - } - } - return resourceFilter; - } - - static Boolean checkResourceFilter(JsonNode jsonNode, Map resourceFilter){ - for(Map.Entry entry : resourceFilter.entrySet()){ - if(jsonNode.get("resourceId").asText().contains(entry.getKey())){ - if (jsonNode.get("name").get("value").asText().matches(entry.getValue())){ - return true; - } - } - } - return false; - } - - static String getDecryptedKey(String encryptedKey, String encryptionKey){ - java.util.Map cryptoMap = Maps.newHashMap(); - cryptoMap.put(Globals.passwordEncrypted, encryptedKey); - cryptoMap.put(Globals.encryptionKey, encryptionKey); - return CryptoUtil.getPassword(cryptoMap); - } - - static URL getUrl(String input){ - URL url = null; - try { - url = new URL(input); - } catch (MalformedURLException e) { - logger.error("Error forming our from String {}", input, e); - } - return url; - } - - static String prettifyJson(JsonNode json) { - ObjectMapper mapper = new ObjectMapper(); - try { - return mapper.writerWithDefaultPrettyPrinter().writeValueAsString(json); - } catch (JsonProcessingException e) { - logger.error("Can not process JSON {}", json.asText(), e); - } - return null; - } -} diff --git a/src/main/java/com/appdynamics/monitors/azure/config/AuthenticationResults.java b/src/main/java/com/appdynamics/monitors/azure/config/AuthenticationResults.java deleted file mode 100644 index 98692e6..0000000 --- a/src/main/java/com/appdynamics/monitors/azure/config/AuthenticationResults.java +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright 2017. AppDynamics LLC and its affiliates. - * All Rights Reserved. - * This is unpublished proprietary source code of AppDynamics LLC and its affiliates. - * The copyright notice above does not evidence any actual or intended publication of such source code. - * - */ - -package com.appdynamics.monitors.azure.config; - -import com.microsoft.aad.adal4j.AuthenticationResult; - -public class AuthenticationResults { - public static AuthenticationResult azureMonitorAuth = null; - public static AuthenticationResult azureKeyVaultAuth = null; -} diff --git a/src/main/java/com/appdynamics/monitors/azure/config/Globals.java b/src/main/java/com/appdynamics/monitors/azure/config/Globals.java deleted file mode 100644 index 73c5298..0000000 --- a/src/main/java/com/appdynamics/monitors/azure/config/Globals.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright 2017. AppDynamics LLC and its affiliates. - * All Rights Reserved. - * This is unpublished proprietary source code of AppDynamics LLC and its affiliates. - * The copyright notice above does not evidence any actual or intended publication of such source code. - * - */ - -package com.appdynamics.monitors.azure.config; - -public class Globals { - public static final String azureAuthEndpoint = "https://login.microsoftonline.com/"; - public static final String azureKeyvaultEndpoint = "https://vault.azure.net"; - public static final String azureEndpoint = "https://management.azure.com"; - public static final String azureApiVersion = "api-version"; - public static final String azureMonitorApiVersion = "monitor-api-version"; - public static final String defaultMetricPrefix = "Custom Metrics|AzureMonitor|"; - public static final String azureApiMetrics = "/providers/microsoft.insights/metrics"; - public static final String azureApiMetricDefinitions = "/providers/microsoft.insights/metricDefinitions"; - public static final String azureApiSubscriptions = "/subscriptions/"; - public static final String azureApiResources = "/resources"; - public static final String azureApiFilter = "filter"; - public static final String azureApiTimeSpan = "timespan"; - public static final String azureApiTimeFormat = "yyyy-MM-dd'T'HH:mm:ss'Z'"; - public static final String filterBy = "resourceType"; - public static final String filterLogOp = " or "; - public static final String filterComOp = " eq "; - public static final String urlEncoding = "UTF-8"; - public static final String configFile = "config-file"; - public static final String clientId = "clientId"; - public static final String tenantId = "tenantId"; - public static final String metric = "metric"; - public static final String clientKey = "clientKey"; - public static final String keyvaultClientId = "keyvaultClientId"; - public static final String keyvaultClientKey = "keyvaultClientKey"; - public static final String keyvaultClientSecretUrl = "keyvaultClientSecretUrl"; - public static final String subscriptionId = "subscriptionId"; - public static final String encryptionKey = "encryption-key"; - public static final String encryptedClientKey = "encryptedClientKey"; - public static final String encryptedKeyvaultClientKey = "encryptedKeyvaultClientKey"; - public static final String passwordEncrypted = "password-encrypted"; - public static final int timeOffset = -2; - public static final String serviceFabricGetClusterHealthChunk = "/$/GetClusterHealthChunk"; - public static final String serviceFabricBody = "serviceFabricBody"; - public static final String serviceFabricCert = "serviceFabricCert"; - public static final String serviceFabricPassphrase = "serviceFabricPassphrase"; - public static final String serviceFabricApiVersion = "serviceFabricApiVersion"; - public static final String serviceFabricResourceApiVersion = "serviceFabricResourceApiVersion"; - public static final String serviceFabricHealthStates = "serviceFabricHealthStates"; -} diff --git a/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java b/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java new file mode 100644 index 0000000..72eff0b --- /dev/null +++ b/src/main/java/com/appdynamics/monitors/azure/metrics/AzureMetrics.java @@ -0,0 +1,349 @@ +/* + * Copyright 2018. AppDynamics LLC and its affiliates. + * All Rights Reserved. + * This is unpublished proprietary source code of AppDynamics LLC and its affiliates. + * The copyright notice above does not evidence any actual or intended publication of such source code. + * + */ + +package com.appdynamics.monitors.azure.metrics; + +import java.io.UnsupportedEncodingException; +import java.net.MalformedURLException; +import java.net.URLEncoder; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.collections4.ListUtils; +import org.slf4j.LoggerFactory; + +import com.appdynamics.extensions.MetricWriteHelper; +import com.appdynamics.extensions.metrics.Metric; +import com.appdynamics.monitors.azure.utils.Constants; +import com.appdynamics.monitors.azure.utils.Resource; +import com.appdynamics.monitors.azure.utils.MetricDefinition; +import com.appdynamics.monitors.azure.utils.AzureAPIWrapper; +import com.google.common.collect.Lists; +import com.singularity.ee.agent.systemagent.api.MetricWriter; +import com.fasterxml.jackson.databind.JsonNode; + +@SuppressWarnings("unchecked") +public class AzureMetrics implements Runnable { + + private static final org.slf4j.Logger logger = LoggerFactory.getLogger(AzureMetrics.class); + private final Map resourceFilter; + private final String currentResourceGroupFilter; + private final String currentResourceFilter; + private final String currentResourceTypeFilter; + private final Resource resource; + private final String subscriptionName; + private final CountDownLatch countDownLatch; + private final MetricWriteHelper metricWriteHelper; + private final String metricPrefix; + private final List finalMetricList = Lists.newArrayList(); + private AzureAPIWrapper azure; + + public AzureMetrics(AzureAPIWrapper azure, + Map resourceFilter, + String currentResourceGroupFilter, + String currentResourceFilter, + String currentResourceTypeFilter, + Resource resource, + Map subscription, + CountDownLatch countDownLatch, + MetricWriteHelper metricWriteHelper, + String metricPrefix) { + this.azure = azure; + this.resourceFilter = resourceFilter; + this.currentResourceGroupFilter = currentResourceGroupFilter; + this.currentResourceFilter = currentResourceFilter; + this.currentResourceTypeFilter = currentResourceTypeFilter; + this.resource = resource; + this.countDownLatch = countDownLatch; + this.metricWriteHelper = metricWriteHelper; + this.metricPrefix = metricPrefix; + if (subscription.containsKey("subscriptionName")){ + subscriptionName = subscription.get("subscriptionName").toString(); + } + else { + subscriptionName = subscription.get("subscriptionId").toString(); + } + } + + @Override + public void run() { + try { + runTask(); + } + catch(Exception e){ + logger.error(e.getMessage()); + } + finally { + countDownLatch.countDown(); + } + } + + private void runTask(){ +// if (logger.isDebugEnabled()) { +// logger.debug("Resource name ({}): {} {}", resource.name().matches(currentResourceFilter), resource.name(), currentResourceFilter); +// logger.debug("Resource type ({}): {} {}", resource.type().matches(currentResourceTypeFilter), resource.type(), currentResourceTypeFilter); +// logger.debug("Resource group ({}): {} {}", resource.resourceGroupName().matches("(?i:" + currentResourceGroupFilter + ")"), resource.resourceGroupName(), currentResourceGroupFilter); +// } + if (resource.getName().matches(currentResourceFilter) && + resource.getType().matches(currentResourceTypeFilter) && + resource.getResourceGroupName().matches("(?i:" + currentResourceGroupFilter + ")")) { + if (logger.isDebugEnabled()) { + logger.debug("Working on Resource {} of Type {} in Group {} because of Resource Filter {} of Type Filter {} in Group Filter {}", + resource.getName(), + resource.getResourceType(), + resource.getResourceGroupName(), + currentResourceTypeFilter, + currentResourceTypeFilter, + currentResourceGroupFilter); + } + List metricDefinitions = azure.getMetricDefinitions(resource.getId()); + try { + generateMetrics(metricDefinitions); + } catch (MalformedURLException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (UnsupportedEncodingException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } else { +// if (logger.isDebugEnabled()) { +// logger.debug("Skipping Resource {} of Type {} in Group {} because of Resource Filter {} of Type Filter {} in Group Filter {}", +// resource.name(), +// resource.resourceType(), +// resource.resourceGroupName(), +// currentResourceTypeFilter, +// currentResourceTypeFilter, +// currentResourceGroupFilter); +// } + } + if (finalMetricList.isEmpty()) { +// if (logger.isDebugEnabled()) { +// logger.debug("Metric List is empty"); +// } + } else { + metricWriteHelper.transformAndPrintMetrics(finalMetricList); + } +// if (logger.isDebugEnabled()) { +// logger.debug("azureMetricDefinitionsCallCount {}: {}", resource.id(), azureMetricDefinitionsCallCount); +// logger.debug("azureMetricsCallCount: " + azureMetricsCallCount); +// } + } + + private void generateMetrics(List metricDefinitions) throws MalformedURLException, UnsupportedEncodingException { + List> metricConfigs = (List>) resourceFilter.get("metrics"); + + List filteredMetrics = Lists.newArrayList(); + List filteredMetricsNames = Lists.newArrayList(); + List> filteredMetricsConfig = Lists.newArrayList(); + + for (Map metricConfig : metricConfigs) { + for (MetricDefinition metricDefinition : metricDefinitions) { + String currentMetricConfig = metricConfig.get("metric").toString(); +// if (logger.isDebugEnabled()) { +// logger.debug("resourceMetric name ({})", resourceMetric.name().value()); +// logger.debug("currentMetricConfig ({})", currentMetricConfig); +// logger.debug("match ({})", resourceMetric.name().value().matches(currentMetricConfig), resourceMetric.name().value(), currentMetricConfig); +// } + if (metricDefinition.getName().matches(currentMetricConfig)) { + Object filterObject = metricConfig.get("filter"); + String filter = filterObject == null ? null : filterObject.toString(); + boolean hasQualifiers = hasQualifiers(metricConfig); + if (( filter != null && !filter.isEmpty() ) || hasQualifiers ) { + JsonNode apiResponse = azure.getMetrics(metricDefinition, URLEncoder.encode(metricDefinition.getName(), "UTF-8"), filter); + try { + addMetric(metricDefinition, apiResponse, metricConfig); + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } else { + filteredMetrics.add(metricDefinition); + filteredMetricsNames.add(URLEncoder.encode(metricDefinition.getName(), "UTF-8")); + filteredMetricsConfig.add(metricConfig); + } + } else { +// if (logger.isDebugEnabled()) { +// logger.debug("Not Reporting Metric {} for Resource {} as it is filtered by {}", +// resourceMetric.name().value(), +// resource.name(), +// currentMetricConfig); +// } + } + } + } +// logger.debug("filteredMetrics size ({})", filteredMetrics.size()); + consumeAndImportAzureMetrics(filteredMetrics, filteredMetricsNames, filteredMetricsConfig); + } + + public boolean hasQualifiers(Map metricConfig) { + Object aggregatorObject = metricConfig.get("aggregator"); + String aggregator = aggregatorObject == null ? "" : aggregatorObject.toString(); + Object timeRollUpObject = metricConfig.get("timeRollUp"); + String timeRollUp = timeRollUpObject == null ? "" : timeRollUpObject.toString(); + Object clusterRollUpObject = metricConfig.get("clusterRollUp"); + String clusterRollUp = clusterRollUpObject == null ? "" : clusterRollUpObject.toString(); + boolean hasQualifiers = !aggregator.isEmpty() || !timeRollUp.isEmpty() || !clusterRollUp.isEmpty(); + return hasQualifiers; + } + + private void consumeAndImportAzureMetrics(List filteredMetrics, List filteredMetricsNames, List> filteredMetricsConfig) throws MalformedURLException, UnsupportedEncodingException { + if (!filteredMetrics.isEmpty()) { + List> filteredMetricsChunks = ListUtils.partition(filteredMetrics, Constants.AZURE_METRICS_CHUNK_SIZE); + List> filteredMetricsNamesChunks = ListUtils.partition(filteredMetricsNames, Constants.AZURE_METRICS_CHUNK_SIZE); + List>> filteredMetricsConfigChunks = ListUtils.partition(filteredMetricsConfig, Constants.AZURE_METRICS_CHUNK_SIZE); + Iterator> filteredMetricsIterator = filteredMetricsChunks.iterator(); + Iterator> filteredMetricsNamesIterator = filteredMetricsNamesChunks.iterator(); + Iterator>> filteredMetricsConfigIterator = filteredMetricsConfigChunks.iterator(); + + while (filteredMetricsIterator.hasNext() && filteredMetricsNamesIterator.hasNext() && filteredMetricsConfigIterator.hasNext()) { + List filteredMetricsChunk = filteredMetricsIterator.next(); + List filteredMetricsNamesChunk = filteredMetricsNamesIterator.next(); + List> filteredMetricsConfigChunk = filteredMetricsConfigIterator.next(); + JsonNode apiResponse = azure.getMetrics(filteredMetrics.get(0), StringUtils.join(filteredMetricsNamesChunk, ',')); + if (apiResponse != null) { + addMetrics(filteredMetricsChunk, filteredMetricsConfigChunk, apiResponse); + } + } + } + } + + private void addMetrics(List filteredMetricsChunk, List> filteredMetricsConfigChunk, JsonNode responseMetrics) { + + JsonNode responseMetricsValues = responseMetrics.get("value"); + Iterator responseMetricsIterator = responseMetricsValues.elements(); + Iterator filteredMetricsIterator = filteredMetricsChunk.iterator(); + Iterator> filteredMetricsConfigIterator = filteredMetricsConfigChunk.iterator(); + while (responseMetricsIterator.hasNext() && filteredMetricsConfigIterator.hasNext() && filteredMetricsIterator.hasNext()) { + MetricDefinition resourceMetric = filteredMetricsIterator.next(); + Map resourceMetricConfig = filteredMetricsConfigIterator.next(); + JsonNode responseMetric = responseMetricsIterator.next(); +// if (logger.isDebugEnabled()) { +// logger.debug("Response metric: {}", responseMetric.toString()); +// } + try { + addMetric(resourceMetric, responseMetric, resourceMetricConfig); + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + } + + private void addMetric(MetricDefinition resourceMetric, JsonNode responseMetric, Map metricConfig) throws Exception { + if (responseMetric == null) return; + String azureMetricValue = null; + String metricPath = metricPrefix + + Constants.METRIC_SEPARATOR + + subscriptionName + + Constants.METRIC_SEPARATOR + + resource.getResourceGroupName() + + Constants.METRIC_SEPARATOR + + resource.getResourceType() + + Constants.METRIC_SEPARATOR + + resource.getName() + + Constants.METRIC_SEPARATOR; +// if (logger.isDebugEnabled()) { +// logger.debug("Metric name / aggregation / path: {} / {} / {}", metricName, metricAggregation, metricPath); +// } + + JsonNode timeseries = responseMetric.findPath("timeseries"); +// logger.debug("timeseries.size():" + timeseries.size()); + if (timeseries.size() > 1) { + throw new Exception("Multiple timeseries not supported"); + } + String azureMetricName = resourceMetric.getName(); + String azureMetricAggregation = resourceMetric.getPrimaryAggregationType().toString(); + String appdAggregationType = null; + String appdTimeRollUpType = null; + String appdClusterRollUpType = null; + JsonNode data = timeseries.findPath("data"); + switch (azureMetricAggregation) { + case "Average": + azureMetricValue = (data.path(0).path("average").isMissingNode()) ? null : data.get(0).get("average").toString(); + appdAggregationType = MetricWriter.METRIC_AGGREGATION_TYPE_AVERAGE; + appdTimeRollUpType = MetricWriter.METRIC_TIME_ROLLUP_TYPE_AVERAGE; + appdClusterRollUpType = MetricWriter.METRIC_CLUSTER_ROLLUP_TYPE_INDIVIDUAL; + break; + case "Count": + azureMetricValue = (data.path(0).path("count").isMissingNode()) ? null : data.get(0).get("count").toString(); + appdAggregationType = MetricWriter.METRIC_AGGREGATION_TYPE_AVERAGE; + appdTimeRollUpType = MetricWriter.METRIC_TIME_ROLLUP_TYPE_SUM; + appdClusterRollUpType = MetricWriter.METRIC_CLUSTER_ROLLUP_TYPE_COLLECTIVE; + break; + case "Total": + azureMetricValue = (data.path(0).path("total").isMissingNode()) ? null : data.get(0).get("total").toString(); + appdAggregationType = MetricWriter.METRIC_AGGREGATION_TYPE_AVERAGE; + appdTimeRollUpType = MetricWriter.METRIC_TIME_ROLLUP_TYPE_SUM; + appdClusterRollUpType = MetricWriter.METRIC_CLUSTER_ROLLUP_TYPE_COLLECTIVE; + break; + case "Maximum": + azureMetricValue = (data.path(0).path("maximum").isMissingNode()) ? null : data.get(0).get("maximum").toString(); + appdAggregationType = MetricWriter.METRIC_AGGREGATION_TYPE_AVERAGE; + appdTimeRollUpType = MetricWriter.METRIC_TIME_ROLLUP_TYPE_AVERAGE; + appdClusterRollUpType = MetricWriter.METRIC_CLUSTER_ROLLUP_TYPE_INDIVIDUAL; + break; + case "Mininum": + azureMetricValue = (data.path(0).path("minimum").isMissingNode()) ? null : data.get(0).get("minimum").toString(); + appdAggregationType = MetricWriter.METRIC_AGGREGATION_TYPE_AVERAGE; + appdTimeRollUpType = MetricWriter.METRIC_TIME_ROLLUP_TYPE_AVERAGE; + appdClusterRollUpType = MetricWriter.METRIC_CLUSTER_ROLLUP_TYPE_INDIVIDUAL; + break; + default: + if (logger.isDebugEnabled()) { + logger.info("Not Reporting Metric {} for Resource {} as the aggregation type is not supported", + azureMetricName, + resource.getName()); + } + break; + } + if (azureMetricValue == null) { + if (logger.isDebugEnabled()) { + logger.debug("Ignoring Metric {} for Resource {} as it is null or empty", + azureMetricName, + resource.getName(), + azureMetricValue); + } + } else { + Object subpathObject = null; + Object aliasObject = null; + Object aggregatorObject = null; + Object timeRollUpObject = null; + Object clusterRollUpObject = null; + if (metricConfig != null) { + aliasObject = metricConfig.get("alias"); + subpathObject = metricConfig.get("subpath"); + aggregatorObject = metricConfig.get("aggregator"); + timeRollUpObject = metricConfig.get("timeRollUp"); + clusterRollUpObject = metricConfig.get("clusterRollUp"); + } + azureMetricName = aliasObject == null ? azureMetricName : aliasObject.toString(); + String subpath = ""; + subpath = subpathObject == null ? "" : subpathObject.toString() + "|"; + appdAggregationType = aggregatorObject == null ? appdAggregationType : MetricWriter.class.getDeclaredField(aggregatorObject.toString()).get(null).toString(); + appdTimeRollUpType = timeRollUpObject == null ? appdTimeRollUpType : MetricWriter.class.getDeclaredField(timeRollUpObject.toString()).get(null).toString(); + appdClusterRollUpType = clusterRollUpObject == null ? appdClusterRollUpType : MetricWriter.class.getDeclaredField(clusterRollUpObject.toString()).get(null).toString(); + Metric metric = new Metric(azureMetricName, + azureMetricValue, + metricPath + subpath + azureMetricName, + appdAggregationType, + appdTimeRollUpType, + appdClusterRollUpType); + if (logger.isDebugEnabled()) { + logger.debug("Reporting Metric {} for Resource {} with value {}", + azureMetricName, + resource.getName(), + azureMetricValue); + } + finalMetricList.add(metric); + } + } +} diff --git a/src/main/java/com/appdynamics/monitors/azure/utils/AzureAPIWrapper.java b/src/main/java/com/appdynamics/monitors/azure/utils/AzureAPIWrapper.java new file mode 100644 index 0000000..cc50ebf --- /dev/null +++ b/src/main/java/com/appdynamics/monitors/azure/utils/AzureAPIWrapper.java @@ -0,0 +1,105 @@ +package com.appdynamics.monitors.azure.utils; + +import java.io.UnsupportedEncodingException; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLEncoder; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.commons.lang3.StringUtils; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; + +import com.appdynamics.monitors.azure.AzureAuth; +import com.appdynamics.monitors.azure.AzureRestOperation; +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.Lists; +import com.microsoft.azure.PagedList; +import com.microsoft.azure.management.Azure; +import com.microsoft.azure.management.monitor.MetricDefinition; +import com.microsoft.azure.management.resources.GenericResource; + +public class AzureAPIWrapper { + private Azure azure; + private AtomicInteger azureResourcesCallCount = new AtomicInteger(0); + private AtomicInteger azureMetricDefinitionsCallCount = new AtomicInteger(0); +// private AtomicInteger azureMetricsCallCount = new AtomicInteger(0); + private Map subscription; + + public AzureAPIWrapper(Map subscription) { + this.subscription = subscription; + } + + public void authorize() { + AzureAuth.authorizeAzure(subscription); + azure = AzureAuth.getAzureMonitorAuth().withSubscription(subscription.get("subscriptionId").toString()); + } + + public List getResources() { + azureResourcesCallCount.incrementAndGet(); + PagedList resources = retrieveResources(); + List list = Lists.newArrayList();; + + for(GenericResource resource : resources) { + Resource wrappedResource = new Resource(); + wrappedResource.setName(resource.name()); + wrappedResource.setType(resource.type()); + wrappedResource.setResourceGroupName(resource.resourceGroupName()); + wrappedResource.setResourceType(resource.resourceType()); + wrappedResource.setId(resource.id()); + list.add(wrappedResource); + } + + return list; + } + + private PagedList retrieveResources() { + // TODO Auto-generated method stub + return azure.genericResources().list(); + } + + public List getMetricDefinitions(String resourceId) { + azureMetricDefinitionsCallCount.incrementAndGet(); + List resourceMetrics = azure.metricDefinitions().listByResource(resourceId); + + List list = Lists.newArrayList();; + + for(MetricDefinition resourceMetric : resourceMetrics) { + com.appdynamics.monitors.azure.utils.MetricDefinition wrappedMetric = new com.appdynamics.monitors.azure.utils.MetricDefinition(); + wrappedMetric.setName(resourceMetric.name().value().toString()); + wrappedMetric.setPrimaryAggregationType(resourceMetric.primaryAggregationType().toString()); + wrappedMetric.setId(resourceMetric.id()); + list.add(wrappedMetric); + } + + return list; + } + + public JsonNode getMetrics(com.appdynamics.monitors.azure.utils.MetricDefinition metricDefinition, String metricNames) throws MalformedURLException, UnsupportedEncodingException { + return getMetrics(metricDefinition, metricNames, null); + } + + public JsonNode getMetrics(com.appdynamics.monitors.azure.utils.MetricDefinition metricDefinition, String metricNames, String filter) throws MalformedURLException, UnsupportedEncodingException { + DateTime recordDateTime = DateTime.now(DateTimeZone.UTC); + Integer ordinalIndexOfLastSlash = StringUtils.ordinalIndexOf(metricDefinition.getId(), "/", Constants.AZURE_METRICS_API_ENDPOINT_LAST_SLASH_POS); + if (ordinalIndexOfLastSlash < 0) { + throw new MalformedURLException("Invalid metrics API endpoint"); + } + String apiEndpointBase = metricDefinition.getId().substring(0,ordinalIndexOfLastSlash); + String url = Constants.AZURE_MANAGEMENT_URL + + apiEndpointBase + + "/metrics?timespan=" + recordDateTime.minusMinutes(2).toDateTimeISO() + "/" + recordDateTime.toDateTimeISO() + + "&metricnames=" + metricNames + + "&api-version=" + subscription.get("api-version"); + if (filter != null) { + url += "&$filter=" + URLEncoder.encode(filter, "UTF-8"); + } + URL apiEndpointFull = new URL(url); +// azureMetricsCallCount.incrementAndGet(); + JsonNode apiResponse = AzureRestOperation.doGet(apiEndpointFull); + + return apiResponse; + } +} diff --git a/src/main/java/com/appdynamics/monitors/azure/utils/Constants.java b/src/main/java/com/appdynamics/monitors/azure/utils/Constants.java new file mode 100644 index 0000000..da7ff56 --- /dev/null +++ b/src/main/java/com/appdynamics/monitors/azure/utils/Constants.java @@ -0,0 +1,25 @@ +/* + * Copyright 2018. AppDynamics LLC and its affiliates. + * All Rights Reserved. + * This is unpublished proprietary source code of AppDynamics LLC and its affiliates. + * The copyright notice above does not evidence any actual or intended publication of such source code. + * + */ + +package com.appdynamics.monitors.azure.utils; + +public class Constants { + public static final String DEFAULT_METRIC_PREFIX = "Custom Metrics|AzureMonitor|"; + public static final String METRIC_SEPARATOR = "|"; + public static final String TEST_CONFIG_FILE = "src/test/resources/conf/config.yml"; + public static final String AZURE_MANAGEMENT_URL = "https://management.azure.com"; + public static final String AZURE_VAULT_URL = "https://vault.azure.net"; + public static final String AZURE_MSI_TOKEN_ENDPOINT = "http://169.254.169.254/metadata/identity/oauth2/token"; + public static final int AZURE_METRICS_CHUNK_SIZE = 20; + public static final int AZURE_METRICS_API_ENDPOINT_LAST_SLASH_POS = 11; + public static final int AZURE_CONNECTION_TIMEOUT = 10000; + public static final int AZURE_READ_TIMEOUT = 10000; + public static final long MONITOR_COUNTDOWN_LATCH_TIMEOUT = 45; + public static final long TASKS_COUNTDOWN_LATCH_TIMEOUT = 45; + public static String accessToken; +} diff --git a/src/main/java/com/appdynamics/monitors/azure/utils/MetricDefinition.java b/src/main/java/com/appdynamics/monitors/azure/utils/MetricDefinition.java new file mode 100644 index 0000000..c2bb2d3 --- /dev/null +++ b/src/main/java/com/appdynamics/monitors/azure/utils/MetricDefinition.java @@ -0,0 +1,26 @@ +package com.appdynamics.monitors.azure.utils; + +public class MetricDefinition { + private String id; + private String name; + private String primaryAggregationType; + + public String getId() { + return id; + } + public void setId(String id) { + this.id = id; + } + public String getName() { + return name; + } + public void setName(String name) { + this.name = name; + } + public String getPrimaryAggregationType() { + return primaryAggregationType; + } + public void setPrimaryAggregationType(String primaryAggregationType) { + this.primaryAggregationType = primaryAggregationType; + } +} diff --git a/src/main/java/com/appdynamics/monitors/azure/utils/Resource.java b/src/main/java/com/appdynamics/monitors/azure/utils/Resource.java new file mode 100644 index 0000000..8179048 --- /dev/null +++ b/src/main/java/com/appdynamics/monitors/azure/utils/Resource.java @@ -0,0 +1,51 @@ +package com.appdynamics.monitors.azure.utils; + +public class Resource { + + private String name; + private String type; + private String resourceGroupName; + private Object resourceType; + private String id; + + public String getName() { + return name; + } + + public String getType() { + return type; + } + + public String getResourceGroupName() { + return resourceGroupName; + } + + public Object getResourceType() { + return resourceType; + } + + public String getId() { + return id; + } + + public void setName(String name) { + this.name = name; + } + + public void setType(String type) { + this.type = type; + } + + public void setResourceGroupName(String resourceGroupName) { + this.resourceGroupName = resourceGroupName; + } + + public void setResourceType(String resourceType) { + this.resourceType = resourceType; + } + + public void setId(String id) { + this.id = id; + } + +} diff --git a/src/main/java/com/appdynamics/monitors/azure/utils/Utilities.java b/src/main/java/com/appdynamics/monitors/azure/utils/Utilities.java new file mode 100644 index 0000000..501808d --- /dev/null +++ b/src/main/java/com/appdynamics/monitors/azure/utils/Utilities.java @@ -0,0 +1,37 @@ +/* + * Copyright 2018. AppDynamics LLC and its affiliates. + * All Rights Reserved. + * This is unpublished proprietary source code of AppDynamics LLC and its affiliates. + * The copyright notice above does not evidence any actual or intended publication of such source code. + * + */ + +package com.appdynamics.monitors.azure.utils; + +import com.appdynamics.extensions.crypto.CryptoUtil; +import com.google.common.collect.Maps; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.net.MalformedURLException; +import java.net.URL; + +public class Utilities { + private static final Logger logger = LoggerFactory.getLogger(Utilities.class); + + public static String getDecryptedKey(String encryptedKey, String encryptionKey){ + java.util.Map cryptoMap = Maps.newHashMap(); + cryptoMap.put("password-encrypted", encryptedKey); + cryptoMap.put("encryption-key", encryptionKey); + return CryptoUtil.getPassword(cryptoMap); + } + + public static URL getUrl(String input){ + URL url = null; + try { + url = new URL(input); + } catch (MalformedURLException e) { + logger.error("Error forming our from String {}", input, e); + } + return url; + } +} diff --git a/src/main/resources/conf/config.yml b/src/main/resources/conf/config.yml index c6d4a9e..6012b29 100644 --- a/src/main/resources/conf/config.yml +++ b/src/main/resources/conf/config.yml @@ -12,88 +12,41 @@ metricPrefix: "Custom Metrics|AzureMonitor|" #encryptedClientKey: "" #encryptedKeyvaultClientKey: "" +# Subscription ID obtained from the Azure Portal +# Tenant ID obtained from the Azure Portal +# Client ID obtained from the Azure Portal +# Client Key for the upper ID obtained from the Azure Portal # Keyvault Client ID obtained from the Azure Portal -keyvaultClientId: "" # Keyvault Key for the upper ID obtained from the Azure Portal -keyvaultClientKey: "" # Keyvault Client Secret Url. From this URL the clientKey will be obtained -keyvaultClientSecretUrl: "" -# Client ID obtained from the Azure Portal -clientId: "" -# Client Key for the upper ID obtained from the Azure Portal -clientKey: "" -# Tenant ID obtained from the Azure Portal -tenantId: "" -# Subscription ID obtained from the Azure Portal -subscriptionId: "" - -api-version: "2017-05-10" -monitor-api-version: "2017-05-01-preview" -keyvault-api-version: "2016-10-01" - -# Include Filter - These Resource Types will be monitored: -# The exclude Element will ignore by matching the regular expression against the Metric Name from the Metric Definition Response -filter: - - resourceType: "Microsoft.AnalysisServices/servers" -# exclude: ".*" - - resourceType: "Microsoft.ApiManagement/service" - - resourceType: "Microsoft.Automation/automationAccounts" - - resourceType: "Microsoft.Batch/batchAccounts" - - resourceType: "Microsoft.Cache/redis" - - resourceType: "Microsoft.ClassicCompute/virtualMachines" - - resourceType: "Microsoft.CognitiveServices/accounts" - - resourceType: "Microsoft.Compute/virtualMachines" - - resourceType: "Microsoft.Compute/virtualMachineScaleSets" - - resourceType: "Microsoft.Compute/virtualMachineScaleSets/virtualMachines" - - resourceType: "Microsoft.CustomerInsights/hubs" - - resourceType: "Microsoft.DataLakeAnalytics/accounts" - - resourceType: "Microsoft.DataLakeStore/accounts" - - resourceType: "Microsoft.DBforMySQL/servers" - - resourceType: "Microsoft.DBforPostgreSQL/servers" - - resourceType: "Microsoft.Devices/IotHubs" - - resourceType: "Microsoft.Devices/provisioningServices" - - resourceType: "Microsoft.DocumentDB/databaseAccounts" - - resourceType: "Microsoft.EventHub/namespaces" - - resourceType: "Microsoft.Insights/AutoscaleSettings" - - resourceType: "Microsoft.Logic/workflows" -# - resourceType: "Microsoft.Network/loadBalancers" -- Produced Errors during Test - - resourceType: "Microsoft.Network/publicIPAddresses" - - resourceType: "Microsoft.Network/applicationGateways" - - resourceType: "Microsoft.Network/virtualNetworkGateways" - - resourceType: "Microsoft.Network/expressRouteCircuits" - - resourceType: "Microsoft.Network/trafficManagerProfiles" - - resourceType: "Microsoft.NotificationHubs/Namespaces/NotificationHubs" - - resourceType: "Microsoft.Search/searchServices" - - resourceType: "Microsoft.ServiceBus/namespaces" - - resourceType: "Microsoft.Sql/servers/databases" - - resourceType: "Microsoft.Sql/servers/elasticPools" - - resourceType: "Microsoft.Sql/servers" - - resourceType: "Microsoft.Storage/storageAccounts" - - resourceType: "Microsoft.Storage/storageAccounts/blobServices" - - resourceType: "Microsoft.Storage/storageAccounts/tableServices" - - resourceType: "Microsoft.Storage/storageAccounts/queueServices" - - resourceType: "Microsoft.Storage/storageAccounts/fileServices" - - resourceType: "Microsoft.StreamAnalytics/streamingjobs" - - resourceType: "Microsoft.Web/serverfarms" - - resourceType: "Microsoft.Web/sites" - - resourceType: "Microsoft.Web/sites/slots" - - resourceType: "Microsoft.Web/hostingEnvironments/multiRolePools" - - resourceType: "Microsoft.Web/hostingEnvironments/workerPools" - - resourceType: "Microsoft.ServiceFabric/clusters" - -serviceFabricApiVersion: "3.0" -serviceFabricResourceApiVersion: "2016-09-01" -serviceFabricBody: '{"ApplicationFilters":[{"HealthStateFilter":65535}],"NodeFilters":[{"HealthStateFilter":65535}]}' - -# Certificate Authentication will be used if the Service Fabric Management Endpoint is https://... -serviceFabricCert: 'monitors/AzureMonitor/your-cert.pfx' -serviceFabricPassphrase: '' - -# Service Fabric Health States. These States will be translated to numbers according to the table below -# The Defaults are derived from here https://docs.microsoft.com/en-us/rest/api/servicefabric/sfclient-model-healthinformation -serviceFabricHealthStates: - - Invalid: 0 - Ok: 1 - Warning: 2 - Error: 3 - Unknown: 65535 \ No newline at end of file +subscriptions: + - subscriptionId: "" + subscriptionName: "" # If Empty the subscriptionId will be used for the Metric Path + tenantId: "" + clientId: "" + clientKey: "" + keyvaultClientId: "" + keyvaultClientKey: "" + keyvaultClientSecretUrl: "" + resourceGroups: # You can Filter with regex on any of these Types. + - resourceGroup: ".*" # If you have multiple entries per Type be sure they do not overlap --> This will produce duplicate Metrics + resourceTypes: + - resourceType: ".*" + resources: + - resource: ".*" + metrics: + - metric: ".*" + +serviceFabrics: + - serviceFabric: "ServiceFabric" + serviceFabricApiVersion: "3.0" + serviceFabricResourceApiVersion: "2016-09-01" + serviceFabricBody: '{"ApplicationFilters":[{"HealthStateFilter":65535}],"NodeFilters":[{"HealthStateFilter":65535}]}' + serviceFabricCert: 'src/test/resources/cert/integration-test-sf.pfx' + serviceFabricPassphrase: '' + serviceFabricHealthStates: + - Invalid: 0 + Ok: 1 + Warning: 2 + Error: 3 + Unknown: 65535 \ No newline at end of file diff --git a/src/main/resources/conf/monitor.xml b/src/main/resources/conf/monitor.xml index 1405c09..8c5605c 100644 --- a/src/main/resources/conf/monitor.xml +++ b/src/main/resources/conf/monitor.xml @@ -1,3 +1,11 @@ + + AzureMonitor diff --git a/src/test/java/com/appdynamics/monitors/azure/AzureMonitorTest.java b/src/test/java/com/appdynamics/monitors/azure/AzureMonitorTest.java index b3f94d5..05cb90e 100644 --- a/src/test/java/com/appdynamics/monitors/azure/AzureMonitorTest.java +++ b/src/test/java/com/appdynamics/monitors/azure/AzureMonitorTest.java @@ -1,187 +1,232 @@ +/* + * Copyright 2018. AppDynamics LLC and its affiliates. + * All Rights Reserved. + * This is unpublished proprietary source code of AppDynamics LLC and its affiliates. + * The copyright notice above does not evidence any actual or intended publication of such source code. + * + */ + package com.appdynamics.monitors.azure; -import com.appdynamics.extensions.conf.MonitorConfiguration; -import com.appdynamics.extensions.util.MetricWriteHelper; -import com.appdynamics.monitors.azure.config.AuthenticationResults; -import com.appdynamics.monitors.azure.config.Globals; +import com.appdynamics.extensions.*; +import com.appdynamics.extensions.conf.MonitorContextConfiguration; +import com.appdynamics.extensions.metrics.Metric; +import com.appdynamics.extensions.util.AssertUtils; +import com.appdynamics.monitors.azure.utils.AzureAPIWrapper; +import com.appdynamics.monitors.azure.utils.Constants; +import com.appdynamics.monitors.azure.utils.MetricDefinition; +import com.appdynamics.monitors.azure.utils.Resource; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.singularity.ee.agent.systemagent.api.TaskOutput; -import com.singularity.ee.agent.systemagent.api.exception.TaskExecutionException; +import com.google.common.collect.Lists; + import org.junit.Test; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; +import org.junit.runner.RunWith; import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import java.io.*; -import java.math.BigDecimal; -import java.net.URL; -import java.util.ArrayList; -import java.util.HashMap; +import java.net.MalformedURLException; +import java.util.Arrays; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; import java.util.Map; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.CountDownLatch; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.*; +@RunWith(Parameterized.class) public class AzureMonitorTest { private static final Logger logger = LoggerFactory.getLogger(AzureMonitorTest.class); - - @Test - public void testAzureMonitor(){ - Map taskArgs = new HashMap(); - taskArgs.put("config-file", "src/test/resources/conf/integration-test-config.yml"); - try { - testAzureMonitorRun(taskArgs); - } catch (TaskExecutionException e) { - e.printStackTrace(); - } + private static MonitorContextConfiguration monitorContextConfiguration; + private static MetricWriteHelper metricWriteHelper; + private static Map subscription; + private static String metricPrefix; + private List resources; + private List metricDefinitions; + private String metricsApiResponse; + private String metricNames; + private String filter; + private String expectedOutput; + + public AzureMonitorTest(List resources, List metricDefinitions, String metricsApiResponse, String metricNames, String filter, String expectedOutput) { + this.resources = resources; + this.expectedOutput = expectedOutput; + this.metricDefinitions = metricDefinitions; + this.metricsApiResponse = metricsApiResponse; + this.metricNames = metricNames; + this.filter = filter; } - @Test - public void testAzureMonitorWithEncryption(){ - Map taskArgs = new HashMap(); - taskArgs.put("config-file", "src/test/resources/conf/integration-test-encrypted-config.yml"); - try { - testAzureMonitorRun(taskArgs); - } catch (TaskExecutionException e) { - e.printStackTrace(); + @Parameters + public static Collection data() { + AMonitorJob aMonitorJob = mock(AMonitorJob.class); + monitorContextConfiguration = new MonitorContextConfiguration( + "AzureMonitor", + Constants.DEFAULT_METRIC_PREFIX, + new File(System.getProperty("user.dir")), + aMonitorJob); + metricWriteHelper = mock(MetricWriteHelper.class); + metricPrefix = "Custom Metrics|AzureMonitor|"; + + String configYml = "src/test/resources/conf/integration-test-config.yml"; + monitorContextConfiguration.setConfigYml(configYml); + List> subscriptions = (List>) monitorContextConfiguration.getConfigYml().get("subscriptions"); + AssertUtils.assertNotNull(subscriptions, "The 'subscriptions' section in config.yml is not initialised"); + + Iterator> iter = subscriptions.iterator(); + subscription = iter.next(); + String subscriptionName = ""; + if (subscription.containsKey("subscriptionName")){ + subscriptionName = subscription.get("subscriptionName").toString(); } - } - - @Test - public void testAzureMonitorTask(){ - try { - testAzureMonitorTaskRun("src/test/resources/conf/integration-test-config.yml"); - } catch (Exception e) { - e.printStackTrace(); + else { + subscriptionName = subscription.get("subscriptionId").toString(); } + + List resourcesWithMetrics = Lists.newArrayList(); + Resource resourceWithMetrics = new Resource(); + resourceWithMetrics.setId("/subscriptions/1a2b3c4b-e5f6-g7h8-a123-1a23bcde456f/resourceGroups/AppDTest/providers/Microsoft.ApiManagement/service/TestApiManagement1"); + resourceWithMetrics.setName("TestApiManagement1"); + resourceWithMetrics.setResourceGroupName("AppDTest"); + resourceWithMetrics.setResourceType("service"); + resourceWithMetrics.setType("Microsoft.ApiManagement/service"); + resourcesWithMetrics.add(resourceWithMetrics); + + List resourcesWithQualifier = Lists.newArrayList(); + Resource resourceWithQualifier = new Resource(); + resourceWithQualifier.setId("/subscriptions/1a2b3c4b-e5f6-g7h8-a123-1a23bcde456f/resourceGroups/AppDTest/providers/Microsoft.ApiManagement/service/TestApiManagement2"); + resourceWithQualifier.setName("TestApiManagement2"); + resourceWithQualifier.setResourceGroupName("AppDTest"); + resourceWithQualifier.setResourceType("service"); + resourceWithQualifier.setType("Microsoft.ApiManagement/service"); + resourcesWithQualifier.add(resourceWithQualifier); + + List resourcesWithDimensions = Lists.newArrayList(); + Resource resourceWithDimensions = new Resource(); + resourceWithDimensions.setId("/subscriptions/1a2b3c4b-e5f6-g7h8-a123-1a23bcde456f/resourceGroups/AppDTest/providers/Microsoft.EventHub/namespaces/TestHub1"); + resourceWithDimensions.setName("TestHub1"); + resourceWithDimensions.setResourceGroupName("AppDTest"); + resourceWithDimensions.setResourceType("namespaces"); + resourceWithDimensions.setType("Microsoft.EventHub/namespaces"); + resourcesWithDimensions.add(resourceWithDimensions); + + List metricDefinitionsWithMetrics = Lists.newArrayList(); + MetricDefinition metricDefinitionSuccessfulRequests = new MetricDefinition(); + metricDefinitionSuccessfulRequests.setId("/subscriptions/1a2b3c4b-e5f6-g7h8-a123-1a23bcde456f/resourceGroups/AppDTest/providers/Microsoft.ApiManagement/service/TestApiManagement1/providers/microsoft.insights/metricdefinitions/SuccessfulRequests"); + metricDefinitionSuccessfulRequests.setName("SuccessfulRequests"); + metricDefinitionSuccessfulRequests.setPrimaryAggregationType("Total"); + metricDefinitionsWithMetrics.add(metricDefinitionSuccessfulRequests); + MetricDefinition metricDefinitionFailedRequests = new MetricDefinition(); + metricDefinitionFailedRequests.setId("/subscriptions/1a2b3c4b-e5f6-g7h8-a123-1a23bcde456f/resourceGroups/AppDTest/providers/Microsoft.ApiManagement/service/TestApiManagement1/providers/microsoft.insights/metricdefinitions/FailedRequests"); + metricDefinitionFailedRequests.setName("FailedRequests"); + metricDefinitionFailedRequests.setPrimaryAggregationType("Total"); + metricDefinitionsWithMetrics.add(metricDefinitionFailedRequests); + MetricDefinition metricDefinitionUnauthorizedRequests = new MetricDefinition(); + metricDefinitionUnauthorizedRequests.setId("/subscriptions/1a2b3c4b-e5f6-g7h8-a123-1a23bcde456f/resourceGroups/AppDTest/providers/Microsoft.ApiManagement/service/TestApiManagement1/providers/microsoft.insights/metricdefinitions/UnauthorizedRequests"); + metricDefinitionUnauthorizedRequests.setName("UnauthorizedRequests"); + metricDefinitionUnauthorizedRequests.setPrimaryAggregationType("Total"); + metricDefinitionsWithMetrics.add(metricDefinitionUnauthorizedRequests); + List metricDefinitionsWithQualifier = Lists.newArrayList(); + MetricDefinition metricDefinitionUnauthorizedRequestsWithQualifier = new MetricDefinition(); + metricDefinitionUnauthorizedRequestsWithQualifier.setId("/subscriptions/1a2b3c4b-e5f6-g7h8-a123-1a23bcde456f/resourceGroups/AppDTest/providers/Microsoft.ApiManagement/service/TestApiManagement2/providers/microsoft.insights/metricdefinitions/UnauthorizedRequests"); + metricDefinitionUnauthorizedRequestsWithQualifier.setName("UnauthorizedRequests"); + metricDefinitionUnauthorizedRequestsWithQualifier.setPrimaryAggregationType("Total"); + metricDefinitionsWithQualifier.add(metricDefinitionUnauthorizedRequestsWithQualifier); + + List metricDefinitionsWithDimensions = Lists.newArrayList(); + MetricDefinition metricDefinitionThrottledRequests = new MetricDefinition(); + metricDefinitionThrottledRequests.setId("/subscriptions/1a2b3c4b-e5f6-g7h8-a123-1a23bcde456f/resourceGroups/AppDTest/providers/Microsoft.EventHub/namespaces/TestHub1/providers/microsoft.insights/metricdefinitions/ThrottledRequests"); + metricDefinitionThrottledRequests.setName("ThrottledRequests"); + metricDefinitionThrottledRequests.setPrimaryAggregationType("Total"); + metricDefinitionsWithDimensions.add(metricDefinitionThrottledRequests); + + String apiResponseWithMetrics = "{\"cost\":0,\"timespan\":\"2018-08-28T09:02:19Z/2018-08-28T09:04:19Z\",\"interval\":\"PT1M\",\"value\":[{\"id\":\"/subscriptions/1a2b3c4b-e5f6-g7h8-a123-1a23bcde456f/resourceGroups/AppDTest/providers/Microsoft.ApiManagement/service/TestApiManagement1/providers/Microsoft.Insights/metrics/SuccessfulRequests\",\"type\":\"Microsoft.Insights/metrics\",\"name\":{\"value\":\"SuccessfulRequests\",\"localizedValue\":\"Successful Gateway Requests\"},\"unit\":\"Count\",\"timeseries\":[{\"metadatavalues\":[],\"data\":[{\"timeStamp\":\"2018-08-28T09:02:00Z\",\"total\":0.0},{\"timeStamp\":\"2018-08-28T09:03:00Z\",\"total\":0.0}]}]},{\"id\":\"/subscriptions/1a2b3c4b-e5f6-g7h8-a123-1a23bcde456f/resourceGroups/AppDTest/providers/Microsoft.ApiManagement/service/TestApiManagement1/providers/Microsoft.Insights/metrics/FailedRequests\",\"type\":\"Microsoft.Insights/metrics\",\"name\":{\"value\":\"FailedRequests\",\"localizedValue\":\"Failed Gateway Requests\"},\"unit\":\"Count\",\"timeseries\":[{\"metadatavalues\":[],\"data\":[{\"timeStamp\":\"2018-08-28T09:02:00Z\",\"total\":0.0},{\"timeStamp\":\"2018-08-28T09:03:00Z\",\"total\":0.0}]}]},{\"id\":\"/subscriptions/1a2b3c4b-e5f6-g7h8-a123-1a23bcde456f/resourceGroups/AppDTest/providers/Microsoft.ApiManagement/service/TestApiManagement1/providers/Microsoft.Insights/metrics/UnauthorizedRequests\",\"type\":\"Microsoft.Insights/metrics\",\"name\":{\"value\":\"UnauthorizedRequests\",\"localizedValue\":\"Unauthorized Gateway Requests\"},\"unit\":\"Count\",\"timeseries\":[{\"metadatavalues\":[],\"data\":[{\"timeStamp\":\"2018-08-28T09:02:00Z\",\"total\":0.0},{\"timeStamp\":\"2018-08-28T09:03:00Z\",\"total\":0.0}]}]}],\"namespace\":\"Microsoft.ApiManagement/service\",\"resourceregion\":\"northeurope\"}"; + String apiResponseWithQualifier = "{\"cost\":0,\"timespan\":\"2018-08-28T09:04:38Z/2018-08-28T09:06:38Z\",\"interval\":\"PT1M\",\"value\":[{\"id\":\"/subscriptions/1a2b3c4b-e5f6-g7h8-a123-1a23bcde456f/resourceGroups/AppDTest/providers/Microsoft.ApiManagement/service/TestApiManagement2/providers/Microsoft.Insights/metrics/UnauthorizedRequests\",\"type\":\"Microsoft.Insights/metrics\",\"name\":{\"value\":\"UnauthorizedRequests\",\"localizedValue\":\"Unauthorized Gateway Requests\"},\"unit\":\"Count\",\"timeseries\":[{\"metadatavalues\":[],\"data\":[{\"timeStamp\":\"2018-08-28T09:04:00Z\",\"total\":0.0},{\"timeStamp\":\"2018-08-28T09:05:00Z\",\"total\":0.0}]}]}],\"namespace\":\"Microsoft.ApiManagement/service\",\"resourceregion\":\"northeurope\"}"; + String apiResponseWithDimensions = "{\"cost\":0,\"timespan\":\"2018-08-28T11:53:19Z/2018-08-28T11:55:19Z\",\"interval\":\"PT1M\",\"value\":[{\"id\":\"/subscriptions/1a2b3c4b-e5f6-g7h8-a123-1a23bcde456f/resourceGroups/AppDTest/providers/Microsoft.EventHub/namespaces/TestHub1/providers/Microsoft.Insights/metrics/ThrottledRequests\",\"type\":\"Microsoft.Insights/metrics\",\"name\":{\"value\":\"ThrottledRequests\",\"localizedValue\":\"Throttled Requests. (Preview)\"},\"unit\":\"Count\",\"timeseries\":[{\"metadatavalues\":[{\"name\":{\"value\":\"entityname\",\"localizedValue\":\"entityname\"},\"value\":\"cooltopica\"}],\"data\":[{\"timeStamp\":\"2018-08-28T11:53:00Z\",\"total\":0.0},{\"timeStamp\":\"2018-08-28T11:54:00Z\",\"total\":0.0}]}]}],\"namespace\":\"Microsoft.EventHub/namespaces\",\"resourceregion\":\"northeurope\"}"; + + String expectedOutputWithMetrics = "[AVERAGE/SUM/COLLECTIVE] [" + metricPrefix + subscriptionName + "|AppDTest|service|TestApiManagement1|SuccessfulRequests]=[0.0]]" + + "[AVERAGE/SUM/COLLECTIVE] [" + metricPrefix + subscriptionName + "|AppDTest|service|TestApiManagement1|FailedRequests]=[0.0]]" + + "[AVERAGE/SUM/COLLECTIVE] [" + metricPrefix + subscriptionName + "|AppDTest|service|TestApiManagement1|UnauthorizedRequests]=[0.0]]"; + + String expectedOutputWithQualifier = "[AVERAGE/AVERAGE/COLLECTIVE] [" + metricPrefix + subscriptionName + "|AppDTest|service|TestApiManagement2|UnauthorizedRequests]=[0.0]]"; + + String expectedOutputWithDimensions = "[AVERAGE/SUM/COLLECTIVE] [" + metricPrefix + subscriptionName + "|AppDTest|namespaces|TestHub1|coolTopicA|Throttled Requests]=[0.0]]"; + + return Arrays.asList(new Object[][] { + {resourcesWithMetrics, metricDefinitionsWithMetrics, apiResponseWithMetrics, "SuccessfulRequests,FailedRequests,UnauthorizedRequests", null, expectedOutputWithMetrics}, + {resourcesWithQualifier, metricDefinitionsWithQualifier, apiResponseWithQualifier, "UnauthorizedRequests", null, expectedOutputWithQualifier}, + {resourcesWithDimensions, metricDefinitionsWithDimensions, apiResponseWithDimensions, "ThrottledRequests", "EntityName eq 'cooltopica'", expectedOutputWithDimensions} + }); } @Test - public void testAzureMonitorTaskWithEncryption(){ - try { - testAzureMonitorTaskRun("src/test/resources/conf/integration-test-encrypted-config.yml"); - } catch (Exception e) { - e.printStackTrace(); - } - } + @SuppressWarnings({"ConstantConditions", "unchecked"}) + public void testAzureMonitorTask() { - @Test - public void testAzureMonitorTaskWithKeyvault(){ + AzureAPIWrapper azure = mockAzureAPIWrapper(subscription, metricWriteHelper); + mockExtensionOutput(); + + CountDownLatch countDownLatch = new CountDownLatch(1); + AzureMonitorTask task = new AzureMonitorTask(monitorContextConfiguration, metricWriteHelper, subscription, countDownLatch, azure); + monitorContextConfiguration.getContext().getExecutorService().execute("Azure Monitor", task); try { - testAzureMonitorTaskRun("src/test/resources/conf/integration-test-keyvault-config.yml"); - } catch (Exception e) { + countDownLatch.await(); + } catch (InterruptedException e) { e.printStackTrace(); } } - @Test - public void testAzureMonitorTaskWithKeyvaultWithEncryption(){ + private AzureAPIWrapper mockAzureAPIWrapper(Map subscription, MetricWriteHelper metricWriteHelper) { + AzureAPIWrapper azure = mock(AzureAPIWrapper.class); + + when(azure.getResources()).thenReturn(resources); + when(azure.getMetricDefinitions(resources.get(0).getId())).thenReturn(metricDefinitions); + + JsonNode metrics = null; + ObjectMapper objectMapper = new ObjectMapper(); try { - testAzureMonitorTaskRun("src/test/resources/conf/integration-test-keyvault-encrypted-config.yml"); - } catch (Exception e) { - e.printStackTrace(); + metrics = objectMapper.readTree(metricsApiResponse); + } catch (IOException e1) { + // TODO Auto-generated catch block + e1.printStackTrace(); } - } - - @Test - public void testResourceFilters(){ try { - testAzureMonitorTaskRun("src/test/resources/conf/integration-test-resourcefilter-config.yml"); - } catch (Exception e) { + when(azure.getMetrics(metricDefinitions.get(0), metricNames)).thenReturn(metrics); + when(azure.getMetrics(metricDefinitions.get(0), metricNames, filter)).thenReturn(metrics); + } catch (MalformedURLException | UnsupportedEncodingException e) { + // TODO Auto-generated catch block e.printStackTrace(); } - } - - @Test - public void testExtractMetrics() throws Exception { - ObjectMapper mapper = new ObjectMapper(); - JsonNode json = mapper.readTree(new FileReader("src/test/resources/json/metric.json")); - - JsonNode jsonValue = json.get("value"); - for (JsonNode currentValueNode:jsonValue){ - String metricNameValue = currentValueNode.get("name").get("value").asText(); - String metricUnit = currentValueNode.get("unit").asText(); - String metricType = null; - BigDecimal metricValue = null; - if(currentValueNode.get("timeseries").has(0)){ - JsonNode jsonData = currentValueNode.get("timeseries").get(0).get("data"); - for (JsonNode currentDataNode:jsonData){ - if (currentDataNode.has("average")){ metricType = "average"; metricValue = currentDataNode.get("average").decimalValue(); } - else if (currentDataNode.has("total")){ metricType = "total"; metricValue = currentDataNode.get("total").decimalValue(); } - else if (currentDataNode.has("last")){ metricType = "last"; metricValue = currentDataNode.get("last").decimalValue(); } - else if (currentDataNode.has("maximum")){ metricType = "maximum"; metricValue = currentDataNode.get("maximum").decimalValue(); } - } - } - assertNotNull(metricValue); - assertNotNull(metricType); - assertNotNull(metricUnit); - assertNotNull(metricNameValue); - } - } - - private void testAzureMonitorRun(Map taskArgs) throws TaskExecutionException { - TaskOutput result = new AzureMonitor().execute(taskArgs, null); - assertTrue(result.getStatusMessage().contains("Metric Upload Complete")); + return azure; } - @SuppressWarnings("ConstantConditions") - private void testAzureMonitorTaskRun(String configYml) throws Exception { - MetricWriteHelper writer = Mockito.mock(MetricWriteHelper.class); - Runnable runner = Mockito.mock(Runnable.class); - MonitorConfiguration conf = new MonitorConfiguration(Globals.defaultMetricPrefix, runner, writer); - conf.setConfigYml(configYml); - Mockito.doAnswer(new Answer() { - public Object answer(InvocationOnMock invocationOnMock) { Object[] args = invocationOnMock.getArguments(); - System.out.println(args[0] + "=" + args[1]); - return null; - } - }).when(writer).printMetric(Mockito.anyString(), Mockito.any(BigDecimal.class), Mockito.anyString()); - conf.setMetricWriter(writer); - - AzureAuth.getAzureAuth(conf.getConfigYml()); - JsonNode filtersJson = Utilities.getFiltersJson((ArrayList) conf.getConfigYml().get(Globals.azureApiFilter)); - String filterUrl = Utilities.getFilterUrl(filtersJson); - Map resourceFilter = Utilities.getResourceFilter(filtersJson); - URL url = Utilities.getUrl(Globals.azureEndpoint + Globals.azureApiSubscriptions + conf.getConfigYml().get(Globals.subscriptionId) + Globals.azureApiResources + - "?" + Globals.azureApiVersion + "=" + conf.getConfigYml().get(Globals.azureApiVersion) + - filterUrl); - ArrayNode resourceElements = (ArrayNode) AzureRestOperation.doGet(AuthenticationResults.azureMonitorAuth,url).get("value"); - Utilities.prettifyJson(resourceElements); - for(JsonNode resourceNode:resourceElements){ - URL metricDefinitions = Utilities.getUrl(Globals.azureEndpoint + resourceNode.get("id").asText() + Globals.azureApiMetricDefinitions + "?" + Globals.azureApiVersion + "=" + conf.getConfigYml().get(Globals.azureMonitorApiVersion)); - JsonNode metricDefinitionResponse = AzureRestOperation.doGet(AuthenticationResults.azureMonitorAuth,metricDefinitions); - assert metricDefinitionResponse != null; - ArrayNode metricDefinitionElements = (ArrayNode) metricDefinitionResponse.get("value"); - for(JsonNode metricDefinitionNode:metricDefinitionElements){ - if (metricDefinitionNode.get("isDimensionRequired").asText().equals("true")){ - logger.info("Dimensions are currently not supported. Skipping " - + metricDefinitionNode.get("id").asText()); + public void mockExtensionOutput() { + doAnswer(new Answer() { + public Object answer(InvocationOnMock invocation) { + List metrics = (List) invocation.getArguments()[0]; + Object mock = invocation.getMock(); + String realOutput = ""; + String loggerOutput = "\n"; + for (Object metric : metrics) { + realOutput += metric.toString(); + loggerOutput += "Printing metric " + metric.toString() + "\n"; } - else if (Utilities.checkResourceFilter(metricDefinitionNode,resourceFilter)){ - logger.info("Ignoring Metric " + - metricDefinitionNode.get("name").get("value").asText() + - " for Resource " + metricDefinitionNode.get("resourceId")); - } - else { - AzureMonitorTask monitorTask = new AzureMonitorTask( - conf, - resourceNode, - AuthenticationResults.azureMonitorAuth, - metricDefinitionNode.get("name").get("value").asText()); - conf.getExecutorService().execute(monitorTask); - } - } - } - conf.getExecutorService().awaitTermination(2, TimeUnit.SECONDS); - } - - @Test(expected = TaskExecutionException.class) - public void testAzureMonitorTaskExcecutionException() throws Exception { - new AzureMonitor().execute(null, null); + assertEquals(expectedOutput, realOutput); + logger.debug(loggerOutput); + + return null; + } + }) + .when(metricWriteHelper) + .transformAndPrintMetrics(Mockito.any(List.class)); } } diff --git a/src/test/resources/json/metric.json b/src/test/resources/json/metric.json deleted file mode 100644 index 745761b..0000000 --- a/src/test/resources/json/metric.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "cost" : 0, - "timespan" : "2018-01-18T06:57:48Z/2018-01-18T06:59:48Z", - "interval" : "PT1M", - "value" : [ { - "id" : "/subscriptions/subscriptionId/resourceGroups/resourceGroup/providers/Microsoft.Compute/virtualMachines/resourceName/providers/Microsoft.Insights/metrics/Percentage CPU", - "type" : "Microsoft.Insights/metrics", - "name" : { - "value" : "Percentage CPU", - "localizedValue" : "Percentage CPU" - }, - "unit" : "Percent", - "timeseries" : [ { - "metadatavalues" : [ ], - "data" : [ { - "timeStamp" : "2018-01-18T06:57:00Z", - "average" : 1.05 - }, { - "timeStamp" : "2018-01-18T06:58:00Z", - "average" : 0.985 - } ] - } ] - } ] -} \ No newline at end of file diff --git a/src/test/resources/json/metricDefinitions.json b/src/test/resources/json/metricDefinitions.json deleted file mode 100644 index 9a401f7..0000000 --- a/src/test/resources/json/metricDefinitions.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "value" : [ { - "id" : "/subscriptions/subscriptionId/resourceGroups/resourceGroup/providers/Microsoft.Compute/virtualMachines/resourceName/providers/microsoft.insights/metricdefinitions/Percentage CPU", - "resourceId" : "/subscriptions/subscriptionId/resourceGroups/resourceGroup/providers/Microsoft.Compute/virtualMachines/resourceName", - "name" : { - "value" : "Percentage CPU", - "localizedValue" : "Percentage CPU" - }, - "isDimensionRequired" : false, - "unit" : "Percent", - "primaryAggregationType" : "Average", - "metricAvailabilities" : [ { - "timeGrain" : "PT1M", - "retention" : "P93D" - }, { - "timeGrain" : "PT1H", - "retention" : "P93D" - } ] - } - ] -} \ No newline at end of file diff --git a/src/test/resources/json/resources.json b/src/test/resources/json/resources.json deleted file mode 100644 index 0fac449..0000000 --- a/src/test/resources/json/resources.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "value": [ - { - "id": "/subscriptions/subscriptionId/resourceGroups/resourceGroup/providers/Microsoft.Web/sites/resourceName", - "name": "resourceName", - "type": "Microsoft.Web/sites", - "kind": "app", - "location": "location", - "tags": { - "displayName": "WebApp" - } - } - ] -} \ No newline at end of file