[skip ci] Error-Handling for Logstash processing issues

Author: Chris Wiechmann

elk-traffic-monitor-api/flows/trafficMonitorApi-search.json

@@ -134,7 +134,7 @@
 				{
 					"name": "code",
 					"type": "string",
-					"value": "\"var result = {};\\n  result.processId = \\\"\\\";\\n  result.data = [];\\n  data.hits.map(function(entry) {\\n    var dataObject = {};\\n    var _source = entry._source;\\n    dataObject.correlationId = _source.correlationId;\\n    dataObject.timestamp = Date.parse(_source['@timestamp']);\\n    if(_source.transactionSummary && _source.transactionSummary.serviceContext) {\\n      dataObject.serviceName = _source.transactionSummary.serviceContext.service;\\n      dataObject.operation = _source.transactionSummary.serviceContext.method;\\n    }\\n    processInitialTransactionElement(dataObject, _source);\\n    result.data.push(dataObject);\\n  });\\n  \\n  function processInitialTransactionElement(dataObject, _source) {\\n    if(_source.transactionElements && _source.transactionElements.leg0) {\\n      dataObject.statustext = _source.transactionElements.leg0.protocolInfo.http.statusText;\\n      dataObject.method = _source.transactionElements.leg0.protocolInfo.http.method;\\n      dataObject.status = _source.transactionElements.leg0.protocolInfo.http.status;\\n      dataObject.wafStatus = _source.transactionElements.leg0.protocolInfo.http.wafStatus;\\n      dataObject.subject = _source.transactionElements.leg0.protocolInfo.http.authSubjectId;\\n      dataObject.localPort = _source.transactionElements.leg0.protocolInfo.http.localPort;\\n      dataObject.uri = _source.transactionElements.leg0.protocolInfo.http.uri;\\n      dataObject.duration = _source.transactionElements.leg0.duration;\\n      dataObject.type = getProtocolType(_source.transactionElements.leg0.protocolInfo);\\n      dataObject.finalStatus = _source.transactionElements.leg0.finalStatus;\\n    }\\n  }\\n  \\n  function getProtocolType(protocolInfo) {\\n    if(protocolInfo.hasOwnProperty('http')) {\\n       return 'http';\\n    }\\n  }\\n  \\n  return result;\"",
+					"value": "\"var result = {};\\n  result.processId = \\\"\\\";\\n  result.data = [];\\n  data.hits.map(function(entry) {\\n    var dataObject = {};\\n    var _source = entry._source;\\n    dataObject.correlationId = _source.correlationId;\\n    dataObject.timestamp = Date.parse(_source['@timestamp']);\\n    if(_source.transactionSummary && _source.transactionSummary.serviceContext) {\\n      dataObject.serviceName = _source.transactionSummary.serviceContext.service;\\n      dataObject.operation = _source.transactionSummary.serviceContext.method;\\n    }\\n    processInitialTransactionElement(dataObject, _source);\\n    result.data.push(dataObject);\\n  });\\n  \\n  function processInitialTransactionElement(dataObject, _source) {\\n    if(_source.transactionElements && _source.transactionElements.leg0) {\\n      dataObject.statustext = _source.transactionElements.leg0.protocolInfo.http.statusText;\\n      dataObject.method = _source.transactionElements.leg0.protocolInfo.http.method;\\n      dataObject.status = _source.transactionElements.leg0.protocolInfo.http.status;\\n      dataObject.wafStatus = _source.transactionElements.leg0.protocolInfo.http.wafStatus;\\n      dataObject.subject = _source.transactionElements.leg0.protocolInfo.http.authSubjectId;\\n      dataObject.localPort = _source.transactionElements.leg0.protocolInfo.http.localPort;\\n      dataObject.uri = _source.transactionElements.leg0.protocolInfo.http.uri;\\n      dataObject.vhost = _source.transactionElements.leg0.protocolInfo.http.vhost;\\n      dataObject.duration = _source.transactionElements.leg0.duration;\\n      dataObject.type = getProtocolType(_source.transactionElements.leg0.protocolInfo);\\n      dataObject.finalStatus = _source.transactionElements.leg0.finalStatus;\\n    }\\n  }\\n  \\n  function getProtocolType(protocolInfo) {\\n    if(protocolInfo.hasOwnProperty('http')) {\\n       return 'http';\\n    }\\n  }\\n  \\n  return result;\"",
 					"metaName": "code",
 					"metaDescription": "A JavaScript function body. Supports `await` and returning promises."
 				}
@@ -321,7 +321,7 @@
 				{
 					"name": "code",
 					"type": "string",
-					"value": "\"let index = 'apigw-traffic-details*';\\n  if(data.config.testElasticIndex) {\\n    index = data.config.testElasticIndex;\\n  }\\n  return index;\"",
+					"value": "\"// Overview screen (Traffic-Details table) \\n  // should also include the apigw-traffic-errors documents\\n  let index = 'apigw-traffic-*';\\n  if(data.config.testElasticIndex) {\\n    index = data.config.testElasticIndex;\\n  }\\n  return index;\"",
 					"metaName": "code",
 					"metaDescription": "A JavaScript function body. Supports `await` and returning promises."
 				}

elk-traffic-monitor-api/test/asAdmin/test_search_endpoint_AsAdmin.js

@@ -581,6 +581,26 @@ describe('Endpoints', function () {
 				expect(body.data[0].uri).to.equals('/petstore/v2/pet/findByStatus');
 			});
 		});
+		it.only('[Endpoint-0023] Should include the V-Host value', () => {
+			const auth = {
+				user: server.apibuilder.config.apikey || 'test',
+				password: ''
+			};
+			return requestAsync({
+				method: 'GET',
+				uri: `http://localhost:${server.apibuilder.port}/api/elk/v1/api/router/service/instance-1/ops/search?field=correlationId&value=7a240f5f0e21555d2d343482`,
+				headers: {
+					'cookie': 'VIDUSR=Search-0022-DAVID-1597468226-Z+qdRW4rGZnwzQ==', 
+					'csrf-token': '04F9F07E59F588CDE469FC367A12ED3A4B845FDA9A9AE2D9A77686823067CDDC'
+				},
+				auth: auth,
+				json: true
+			}).then(({ response, body }) => {
+				expect(response.statusCode).to.equal(200);
+				expect(body).to.be.an('Object');
+				expect(body.data[0].vhost).to.equal('api.customer.com:443', 'V-Host is not part of the result');
+			});
+		});
 	});
 });
 

elk-traffic-monitor-api/test/documents/basic/search_test_documents.js

@@ -76,6 +76,7 @@ module.exports = [
                         "bytesReceived": 747,
                         "localPort": "8065",
                         "method": "GET",
+                        "vhost": "api.customer.com:443",
                         "statusText": "Not Found",
                         "remotePort": "59641",
                         "bytesSent": 834,

logstash/pipeline/pipeline.conf

@@ -80,6 +80,8 @@ filter {
           clone {
             clones => ['errorEvent']
           }
+          # Fill the URI based on the given path to have it shown in Traffic-Monitor
+          mutate { add_field => { "[transactionElements][leg0][protocolInfo][http][uri]" => "%{[transactionSummary][path]}" } }
           mutate { replace => { "[transactionElements][leg0][protocolInfo][http][status]" => "XXX" } }
           mutate { replace => { "[transactionElements][leg0][protocolInfo][http][statusText]" => "ERROR" } }
           mutate { replace => { "[transactionElements][leg0][protocolInfo][http][authSubjectId]" => "ID: %{[correlationId]}" } }
@@ -127,7 +129,7 @@ output {
   } else if [type] == 'errorEvent' {
     elasticsearch {
       hosts => "elasticsearch1:9200"
-      index => "apigw-traffic-details-err-%{+YYYY}"
+      index => "apigw-traffic-errors-%{+YYYY}"
       template => "${HOME}/config/traffic_details_index_template.json"
       template_overwrite => true
       document_id => "%{correlationId}"