Adjusting index-names

Author: Chris Wiechmann

elk-traffic-monitor-api/flows/trafficMonitorApi-circuitpath.json

@@ -142,19 +142,19 @@
 			"name": "Define Index",
 			"method": "Execute",
 			"parameters": [
-				{
-					"name": "code",
-					"type": "string",
-					"value": "\"let index = 'logstash-openlog';\\n  if(data.config.testElasticIndex) {\\n    index = data.config.testElasticIndex;\\n  }\\n  return index;\"",
-					"metaName": "code",
-					"metaDescription": "A JavaScript function body. Supports `await` and returning promises"
-				},
 				{
 					"name": "data",
 					"type": "jsonpath",
 					"value": "$",
 					"metaName": "data",
 					"metaDescription": "The value to apply as the `data` argument to the JavaScript code. For objects and arrays, `data` is passed by reference."
+				},
+				{
+					"name": "code",
+					"type": "string",
+					"value": "\"let index = 'apigw-traffic-*';\\n  if(data.config.testElasticIndex) {\\n    index = data.config.testElasticIndex;\\n  }\\n  return index;\"",
+					"metaName": "code",
+					"metaDescription": "A JavaScript function body. Supports `await` and returning promises"
 				}
 			],
 			"outputs": {

elk-traffic-monitor-api/flows/trafficMonitorApi-getinfo.json

@@ -47,7 +47,7 @@
 				{
 					"name": "code",
 					"type": "string",
-					"value": "\"let index = 'logstash-openlog';\\n  //let index = 'getinfo_test_1688';\\n  \\n  if(data.config.testElasticIndex) {\\n    index = data.config.testElasticIndex;\\n  }\\n  \\n  return index;\"",
+					"value": "\"let index = 'apigw-traffic-*';\\n  //let index = 'getinfo_test_1688';\\n  \\n  if(data.config.testElasticIndex) {\\n    index = data.config.testElasticIndex;\\n  }\\n  \\n  return index;\"",
 					"metaName": "code",
 					"metaDescription": "A JavaScript function body. Supports `await` and returning promises"
 				}

elk-traffic-monitor-api/flows/trafficMonitorApi-search.json

@@ -313,7 +313,7 @@
 				{
 					"name": "code",
 					"type": "string",
-					"value": "\"let index = 'logstash-openlog';\\n  if(data.config.testElasticIndex) {\\n    index = data.config.testElasticIndex;\\n  }\\n  return index;\"",
+					"value": "\"let index = 'apigw-traffic-*';\\n  if(data.config.testElasticIndex) {\\n    index = data.config.testElasticIndex;\\n  }\\n  return index;\"",
 					"metaName": "code",
 					"metaDescription": "A JavaScript function body. Supports `await` and returning promises"
 				}

elk-traffic-monitor-api/flows/trafficMonitorApi-trace.json

@@ -283,19 +283,19 @@
 			"name": "Define Index",
 			"method": "Execute",
 			"parameters": [
-				{
-					"name": "code",
-					"type": "string",
-					"value": "\"let index = 'logstash-trace';\\n  if(data.config.testElasticIndex) {\\n    index = data.config.testElasticIndex;\\n  }\\n  return index;\"",
-					"metaName": "code",
-					"metaDescription": "A JavaScript function body. Supports `await` and returning promises"
-				},
 				{
 					"name": "data",
 					"type": "jsonpath",
 					"value": "$",
 					"metaName": "data",
 					"metaDescription": "The value to apply as the `data` argument to the JavaScript code. For objects and arrays, `data` is passed by reference."
+				},
+				{
+					"name": "code",
+					"type": "string",
+					"value": "\"let index = 'apigw-traffic-trace-*';\\n  if(data.config.testElasticIndex) {\\n    index = data.config.testElasticIndex;\\n  }\\n  return index;\"",
+					"metaName": "code",
+					"metaDescription": "A JavaScript function body. Supports `await` and returning promises"
 				}
 			],
 			"outputs": {

logstash/pipeline/pipeline.conf

@@ -58,10 +58,6 @@ filter {
     grok {
       match => { "message" => "%{LOGLEVEL:level}%{SPACE}(?<loggedDate>\d{2}\/.{3}\/\d{4}:\d{2}:\d{2}:\d{2}\.\d{3})%{SPACE}\[%{WORD:fluff}\:%{WORD:correlationId}\]\s*%{GREEDYDATA:body}" }
     }
-    # Add trace-messages not related to a specific error to a generic trace index
-    if([correlationId] == "000000000000000000000000") {
-      mutate { add_field => { "[@metadata][indexkey]" => "general_trace" } }
-    } 
     date {
       # LoggedDate is provided like so 13/Jul/2020:15:26:35.108
       match => [ "loggedDate", "dd/MMM/yyyy:HH:mm:ss.SSS" ]
@@ -80,12 +76,17 @@ output {
   if [correlationId] == "000000000000000000000000" {
     elasticsearch {
       hosts => "elasticsearch1:9200"
-      index => "openlog-general-%{+YYYY.MM.dd}"
+      index => "apigw-trace-messages-%{+YYYY.MM.dd}"
+    }
+  } else if [logtype] == "trace" {
+    elasticsearch {
+      hosts => "elasticsearch1:9200"
+      index => "apigw-traffic-trace-%{+YYYY.MM.dd}"
     }
   } else {
     elasticsearch {
       hosts => "elasticsearch1:9200"
-      index => "apigw-%{logtype}-%{+YYYY.MM.dd}"
+      index => "apigw-traffic-%{+YYYY.MM.dd}"
       #template => "${HOME}/config/openlog_index_template.json"
       template_overwrite => true
       document_id => "%{correlationId}"