[skip ci] Adding memcached

Author: Chris Wiechmann

docker-compose.yml

@@ -88,6 +88,15 @@ services:
     depends_on:
       - elasticsearch1
       - elk-traffic-monitor-api
+      - memcached
+    networks:
+      - elastic
+
+  # Memcached is used by the Logstash pipeline to cache API-Lookup information that are used to get the API-Organization
+  memcached:
+    image: memcached:1.6.6-alpine
+    links:
+      - elasticsearch1
     networks:
       - elastic
 

logstash/pipeline/pipeline.conf

@@ -64,54 +64,56 @@ filter {
        remove_field => "[transactionSummary][serviceContexts]"
        rename => ["[transactionSummary][serviceContext][org]", "[transactionSummary][serviceContext][appOrg]" ]
       }
-      # Future plan is to perform side-calls only if not yet already cached
-      #mutate {
-      #  add_field => { "apiCacheKey" => "%{[transactionSummary][serviceContext][service]}---%{[transactionSummary][path]}" }
-      #}
-      #memcached {
-      #  namespace => "api_details"
-      #  ttl => 300
-      #  get => { "apiCacheKey" => "[transactionSummary][serviceContext][serviceOrg]" }
-      #  remove_field => [ "apiCacheKey" ]
-      #}
-      #if ![transactionSummary][serviceContext][serviceOrg] {
-      # Perform a side-call to load organization details for the API
-      if([transactionSummary][serviceContext]) {
-        http {
-          url => "${API_BUILDER_URL}/api/elk/v1/api/lookup/api"
-          query => {
-            "apiName" => "%{[transactionSummary][serviceContext][service]}"
-            "apiPath" => "%{[transactionSummary][path]}"
-          }
-          target_body => "apiDetails"
-          add_field => { 
-            "[transactionSummary][serviceContext][apiOrg]" => "%{[apiDetails][organizationName]}"
-            "[transactionSummary][serviceContext][apiVersion]" => "%{[apiDetails][version]}" 
-            "[transactionSummary][serviceContext][apiDeprecated]" => "%{[apiDetails][deprecated]}" 
-            "[transactionSummary][serviceContext][apiState]" => "%{[apiDetails][state]}"
-          }
-          remove_field => [ "apiDetails", "headers" ]
-        }
-        # If the API-Lookup failed - Clone the event which is send to an Error index and shown in Traffic-Monitor
-        if("_httprequestfailure" in [tags]) {
-          clone {
-            clones => ['errorEvent']
+      # Create a key for the API
+      mutate {
+        add_field => { "apiCacheKey" => "%{[transactionSummary][serviceContext][service]}---%{[transactionSummary][path]}" }
+      }
+      memcached {
+        hosts => ["memcached"]
+        namespace => "api_details"
+        ttl => 300
+        get => { "apiCacheKey" => "[transactionSummary][serviceContext][apiOrg]" }
+        remove_field => [ "apiCacheKey" ]
+      }
+      # If the apiOrg is already set, the information has been loaded before from memcached 
+      if ![transactionSummary][serviceContext][apiOrg] {
+        # Perform a side-call to load organization details for the API
+        if([transactionSummary][serviceContext]) {
+          http {
+            url => "${API_BUILDER_URL}/api/elk/v1/api/lookup/api"
+            query => {
+              "apiName" => "%{[transactionSummary][serviceContext][service]}"
+              "apiPath" => "%{[transactionSummary][path]}"
+            }
+            target_body => "apiDetails"
+            add_field => { 
+              "[transactionSummary][serviceContext][apiOrg]" => "%{[apiDetails][organizationName]}"
+              "[transactionSummary][serviceContext][apiVersion]" => "%{[apiDetails][version]}" 
+              "[transactionSummary][serviceContext][apiDeprecated]" => "%{[apiDetails][deprecated]}" 
+              "[transactionSummary][serviceContext][apiState]" => "%{[apiDetails][state]}"
+            }
+            remove_field => [ "apiDetails", "headers" ]
           }
-          if [type] == 'errorEvent' {
-            # Fill the URI based on the given path to have it shown in Traffic-Monitor
-            mutate { add_field => { "[transactionElements][leg0][protocolInfo][http][uri]" => "%{[transactionSummary][path]}" } }
-            mutate { replace => { "[transactionElements][leg0][protocolInfo][http][vhost]" => "Logstash Error" } }
-            mutate { replace => { "[transactionElements][leg0][protocolInfo][http][method]" => "LOGSTASH" } }
-            mutate { replace => { "[transactionElements][leg0][protocolInfo][http][status]" => "900" } }
-            mutate { replace => { "[transactionElements][leg0][protocolInfo][http][statusText]" => "ERROR" } }
-            mutate { replace => { "[transactionElements][leg0][protocolInfo][http][authSubjectId]" => "ID: %{[correlationId]}" } }
-            mutate { replace => { "[transactionSummary][serviceContext][method]" => "check the logs" } }
-            # Make sure, the Correlation-ID is unique to avoid an update of the original document
-            mutate { replace => { "[correlationId]" => "%{[correlationId]}-Error" } }
+          # If the API-Lookup failed - Clone the event which is send to an Error index and shown in Traffic-Monitor
+          if("_httprequestfailure" in [tags]) {
+            clone {
+              clones => ['errorEvent']
+            }
+            if [type] == 'errorEvent' {
+              # Fill the URI based on the given path to have it shown in Traffic-Monitor
+              mutate { add_field => { "[transactionElements][leg0][protocolInfo][http][uri]" => "%{[transactionSummary][path]}" } }
+              mutate { replace => { "[transactionElements][leg0][protocolInfo][http][vhost]" => "Logstash Error" } }
+              mutate { replace => { "[transactionElements][leg0][protocolInfo][http][method]" => "LOGSTASH" } }
+              mutate { replace => { "[transactionElements][leg0][protocolInfo][http][status]" => "900" } }
+              mutate { replace => { "[transactionElements][leg0][protocolInfo][http][statusText]" => "ERROR" } }
+              mutate { replace => { "[transactionElements][leg0][protocolInfo][http][authSubjectId]" => "ID: %{[correlationId]}" } }
+              mutate { replace => { "[transactionSummary][serviceContext][method]" => "check the logs" } }
+              # Make sure, the Correlation-ID is unique to avoid an update of the original document
+              mutate { replace => { "[correlationId]" => "%{[correlationId]}-Error" } }
+            }
           }
         }
       }
-      #}
     }
   } else if([logtype] == "trace") {
     if [message] =~ /^#/ {