Added support to memcache API-Details instead of perform HTTP-Lookup

with every request

Author: Chris Wiechmann

logstash/pipeline/pipeline.conf

@@ -64,55 +64,71 @@ filter {
        remove_field => "[transactionSummary][serviceContexts]"
        rename => ["[transactionSummary][serviceContext][org]", "[transactionSummary][serviceContext][appOrg]" ]
       }
-      # Create a key for the API
-      mutate {
-        add_field => { "apiCacheKey" => "%{[transactionSummary][serviceContext][service]}---%{[transactionSummary][path]}" }
-      }
-      memcached {
-        hosts => ["memcached"]
-        namespace => "api_details"
-        ttl => 300
-        get => { "apiCacheKey" => "[transactionSummary][serviceContext][apiOrg]" }
-        remove_field => [ "apiCacheKey" ]
-      }
-      # If the apiOrg is already set, the information has been loaded before from memcached 
-      if ![transactionSummary][serviceContext][apiOrg] {
-        # Perform a side-call to load organization details for the API
-        if([transactionSummary][serviceContext]) {
+      # If we have a ServiceConext we have to enrich it with the API-Details
+      if([transactionSummary][serviceContext]) {
+        # Create a key for the API
+        mutate {
+          add_field => { "apiCacheKey" => "%{[transactionSummary][path]}" }
+        }
+        # Lookup the cache with the created API-Key (API-Name---API-Path)
+        memcached {
+          hosts => ["memcached"]
+          namespace => "api_details"
+          get => { "%{apiCacheKey}" => "[apiDetails]" }
+        }
+        # If we have nothing in the cache, perform the Lookup via the HTTP-Builder API
+        if !([apiDetails]) {
           http {
             url => "${API_BUILDER_URL}/api/elk/v1/api/lookup/api"
             query => {
               "apiName" => "%{[transactionSummary][serviceContext][service]}"
               "apiPath" => "%{[transactionSummary][path]}"
             }
             target_body => "apiDetails"
+            add_field => { "updateAPICache" => "true" }
+          }
+        }
+        # At this point we should have the apiDetails either from the cache or looked up via HTTP
+        if([apiDetails]) {
+          mutate {
             add_field => { 
               "[transactionSummary][serviceContext][apiOrg]" => "%{[apiDetails][organizationName]}"
               "[transactionSummary][serviceContext][apiVersion]" => "%{[apiDetails][version]}" 
               "[transactionSummary][serviceContext][apiDeprecated]" => "%{[apiDetails][deprecated]}" 
               "[transactionSummary][serviceContext][apiState]" => "%{[apiDetails][state]}"
             }
-            remove_field => [ "apiDetails", "headers" ]
           }
-          # If the API-Lookup failed - Clone the event which is send to an Error index and shown in Traffic-Monitor
-          if("_httprequestfailure" in [tags]) {
-            clone {
-              clones => ['errorEvent']
-            }
-            if [type] == 'errorEvent' {
-              # Fill the URI based on the given path to have it shown in Traffic-Monitor
-              mutate { add_field => { "[transactionElements][leg0][protocolInfo][http][uri]" => "%{[transactionSummary][path]}" } }
-              mutate { replace => { "[transactionElements][leg0][protocolInfo][http][vhost]" => "Logstash Error" } }
-              mutate { replace => { "[transactionElements][leg0][protocolInfo][http][method]" => "LOGSTASH" } }
-              mutate { replace => { "[transactionElements][leg0][protocolInfo][http][status]" => "900" } }
-              mutate { replace => { "[transactionElements][leg0][protocolInfo][http][statusText]" => "ERROR" } }
-              mutate { replace => { "[transactionElements][leg0][protocolInfo][http][authSubjectId]" => "ID: %{[correlationId]}" } }
-              mutate { replace => { "[transactionSummary][serviceContext][method]" => "check the logs" } }
-              # Make sure, the Correlation-ID is unique to avoid an update of the original document
-              mutate { replace => { "[correlationId]" => "%{[correlationId]}-Error" } }
+          # If the API has been looked up add it to the cache
+          if([updateAPICache]=="true") {
+            memcached {
+              hosts => ["memcached"]
+              namespace => "api_details"
+              ttl => 3600
+              set => { "[apiDetails]" => "%{apiCacheKey}" }
             }
           }
         }
+        mutate {
+          remove_field => [ "apiDetails", "headers", "updateAPICache", "apiCacheKey" ]
+        }
+        # If the API-Lookup failed - Clone the event which is send to an Error index and shown in Traffic-Monitor
+        if("_httprequestfailure" in [tags]) {
+          clone {
+            clones => ['errorEvent']
+          }
+          if [type] == 'errorEvent' {
+            # Fill the URI based on the given path to have it shown in Traffic-Monitor
+            mutate { add_field => { "[transactionElements][leg0][protocolInfo][http][uri]" => "%{[transactionSummary][path]}" } }
+            mutate { replace => { "[transactionElements][leg0][protocolInfo][http][vhost]" => "Logstash Error" } }
+            mutate { replace => { "[transactionElements][leg0][protocolInfo][http][method]" => "LOGSTASH" } }
+            mutate { replace => { "[transactionElements][leg0][protocolInfo][http][status]" => "900" } }
+            mutate { replace => { "[transactionElements][leg0][protocolInfo][http][statusText]" => "ERROR" } }
+            mutate { replace => { "[transactionElements][leg0][protocolInfo][http][authSubjectId]" => "ID: %{[correlationId]}" } }
+            mutate { replace => { "[transactionSummary][serviceContext][method]" => "check the logs" } }
+            # Make sure, the Correlation-ID is unique to avoid an update of the original document
+            mutate { replace => { "[correlationId]" => "%{[correlationId]}-Error" } }
+          }
+        }
       }
     }
   } else if([logtype] == "trace") {