Added new parameter IGNORE_API_LOOKUP_ENABLED

Author: Chris Wiechmann

CHANGELOG.md

@@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 ## [4.4.0] 2022-03-31
 ### Added
 - Now the parameter: `CACHE_API_PATHS` is also used to cache the API-Ignore lookup more efficient
+- New parameter: `APPLICATION_LOOKUP_ENABLED` to disable application lookup for API-Gateway Traffic Monitoring improvement
+- New parameter: `IGNORE_API_LOOKUP_ENABLED` to disable the API-Ignore lookup, when not used anyway
 
 ## [4.4.0] 2022-03-29
 ### Changed

UPDATE.md

@@ -37,6 +37,7 @@ On the other hand, the API builder Docker image, as a central component of the s
 
 | Ver   | API-Builder                        | Logstash                           | Memcached                          | Filebeat      | ANM-Config      | Dashboards      | Params          |Elastic-Config      | ELK-Ver.                                 | Notes      |
 | :---  | :---:                              | :---:                              | :---:                              | :---:         | :---:           | :---:           | :---:           |:---:               | :---:                                    | :---       |
+| 4.5.0 | -                                  | [X](#api-builderlogstashmemcached) | -                                  | -             | -               | -               | [X](#parameters)|-                   | [7.17.1](#update-elastic-stack-version)  |            |
 | 4.4.0 | [X](#api-builderlogstashmemcached) | [X](#api-builderlogstashmemcached) | -                                  | -             | -               | -               | [X](#parameters)|-                   | [7.17.1](#update-elastic-stack-version)  |            |
 | 4.3.0 | [X](#api-builderlogstashmemcached) | [X](#api-builderlogstashmemcached) | -                                  | -             | -               | [X](#dashboards)| [X](#parameters)|[X](#elastic-config)| [7.17.1](#update-elastic-stack-version)  |            |
 | 4.2.0 | [X](#api-builderlogstashmemcached) | -                                  | -                                  | -             | -               | [X](#dashboards)| [X](#parameters)|-                   | [7.17.0](#update-elastic-stack-version)  |            |

docker-compose.yml

@@ -31,6 +31,8 @@ services:
       - xpack.geoip.download.endpoint=${GEOIP_DOWNLOAD_ENDPOINT}
       - EVENTLOG_CUSTOM_ATTR=${EVENTLOG_CUSTOM_ATTR}
       - CACHE_API_PATHS=${CACHE_API_PATHS}
+      - APPLICATION_LOOKUP_ENABLED=${APPLICATION_LOOKUP_ENABLED:-true}
+      - IGNORE_API_LOOKUP_ENABLED=${IGNORE_API_LOOKUP_ENABLED:-true}
     ports:
       - 5044:5044
     volumes:

env-sample

@@ -430,6 +430,30 @@ ELASTIC_VERSION=7.17.1
 # Defaults to all levels are forwarded.
 # DROP_TRACE_MESSAGE_LEVELS=DEBUG,DATA
 
+# ----------------------------------------------------------------------------------------------
+# If an API is called by a registered API-Manager application and no user is authenticated, then 
+# this application ID appears in the Subject column in the traffic monitor.
+# Now, by default, the solution tries to convert the unhelpful application IDs passed in the 
+# authentication.subject.id attribute to the actual application name. This is done whenever the 
+# authentication subject ID is a UUID. 
+# However, if other UUIDs are used in your configuration in the authentication subject 
+# (for example, customer IDs), then the lookup is actually always incorrect and an unnecessary 
+# number of API requests are made. In this case, you should disable the function, since you have 
+# no use for it anyway.
+# By the way, this has nothing to do with the application details, which are passed separately.
+# Used-By: Logstash
+# Defaults to true
+# APPLICATION_LOOKUP_ENABLED=false
+
+# ----------------------------------------------------------------------------------------------
+# There is a possibility to exclude APIs from indexing in Elasticsearch, i.e. to ignore them. 
+# If you do not want to make use of this, i.e. do not want to ignore APIs, then you can 
+# completely deactivate the necessary Loookup, which is executed as part of the OpenTraffic 
+# pipeline, in order to further reduce the ingest latency, for example.
+# Used-By: Logstash
+# Defaults to true
+# IGNORE_API_LOOKUP_ENABLED=false
+
 # ----------------------------------------------------------------------------------------------
 # Disables the setup flows in API-Builder that are used to configure Elasticsearch.
 # If you run more than one API builder, you can set this parameter to true, as it is not

helm/templates/elasticApimLogstash/logstash-config.yaml

@@ -38,6 +38,8 @@ data:
   LOOKUP_CACHE_TTL: {{ default "" .Values.logstash.lookupCacheTTL | quote }}
   {{- end }}
   CACHE_API_PATHS: {{ default "" .Values.logstash.cacheAPIPaths | quote }}
+  APPLICATION_LOOKUP_ENABLED: {{ default "true" .Values.logstash.applicationLookupEnabled | quote }}
+  IGNORE_API_LOOKUP_ENABLED: {{ default "true" .Values.logstash.ignoreApiLookupEnabled | quote }}
   GEOIP_ENABLED: {{ default "true" .Values.logstash.geoip.enabled | quote }}
   GEOIP_CACHE_SIZE: {{ default "1000" .Values.logstash.geoip.cacheSize | quote }}
   GEOIP_CUSTOM_ATTRIBUTE: {{ default "true" .Values.logstash.geoip.customAttribute | quote }}

helm/values.yaml

@@ -274,6 +274,12 @@ logstash:
   # Check the env-sample parameter: CACHE_API_PATHS for more details.
   # cacheAPIPaths: "/api/v2/petstore, /api/v1/user, ..."
 
+  # Disable application lookup. For more details please check parameter APPLICATION_LOOKUP_ENABLED
+  # applicationLookupEnabled: "false"
+
+  # Disable the lookup if an API should be ignored. Makes sense if you dont ignore any API
+  # ignoreApiLookupEnabled: "false"
+
   # Injects the environment variables from the ConfigMaps and Secrets into the 
   # Logstash container. Specify your own ConfigMaps or Secrets if you don't
   # provide Configuration and Secrets as part of this values.yaml.

logstash/pipelines/OpenTrafficPipeline.conf

@@ -25,6 +25,8 @@ filter {
     add_field => { "[@metadata][ignoreField][policyName]" => "" } # and policyName used for ingoreLookup
     add_field => { "[@metadata][apiCacheKeyPrefix]" => "" } # Create a default for the cacheKeyPrefix
     add_field => { "[@metadata][cacheAPIPaths]" => "${CACHE_API_PATHS:''}" } 
+    add_field => { "[@metadata][applicationLookupEnabled]" => "${APPLICATION_LOOKUP_ENABLED:'true'}" } 
+    add_field => { "[@metadata][ignoreApiLookupEnabled]" => "${IGNORE_API_LOOKUP_ENABLED:'true'}" } 
   }
 
   # Check if the document should be ignored/dropped (configured based on the apiPath or PolicyName)
@@ -70,54 +72,56 @@ filter {
       '
     }
   }
-  # Create a cache key for the event, either on API-Path or Policy-Name
-  mutate {
-    add_field => { "[@metadata][ignoreLookupCacheKey]" => "%{[@metadata][apiCacheKeyPrefix]}###%{[@metadata][ignoreField][policyName]}" }
-  }
-  # Lookup the cache is the document should be ignored
-  memcached {
-    hosts => "${MEMCACHED}"
-    namespace => "ignoredAPIs"
-    get => { "%{[@metadata][ignoreLookupCacheKey]}" => "[isIgnoreAPI]" }
-  }
-  # If we don't know yet, consult the API-Builder API
-  if !([isIgnoreAPI]) {
-    http {
-      id => "Ignore API check"
-      url => "${API_BUILDER_URL}/api/elk/v1/api/lookup/api/ignore"
-      query => {
-        "policyName" => "%{[@metadata][ignoreField][policyName]}"
-        "apiPath" => "%{[@metadata][ignoreField][apiPath]}"
-        "groupId" => "%{[processInfo][groupId]}"
-        "region" => "%{[processInfo][gatewayRegion]}"
-        "correlationId" => "%{[correlationId]}"
-      }
-      automatic_retries => 1 # 1 retry only, as this is not super important and should not block the entire processing
-      cacert => "${API_BUILDER_SSL_CERT}"
-      target_body => "isIgnoreAPI"
-      add_field => { "[@metadata][updateIsIgnoreCache]" => "true" }
-    }
-  }
-  # Drop the document if it should be ignored
-  if ([isIgnoreAPI][ignore]) {
-    ruby {
-      code => 'logger.debug("Drop API/event as it should be ignored: ", "isIgnoreAPI" => event.get("[isIgnoreAPI]"), "ignoreLookupCacheKey" => event.get("[@metadata][ignoreLookupCacheKey]") );'
-    }
-    drop {}
-  }
-  # If the PolicyName or API-Path has been looked up right now add it to the cache
-  if([@metadata][updateIsIgnoreCache]=="true") {
-    ruby {
-      code => 'logger.debug("Adding ignore status to cache: ", "isIgnoreAPI" => event.get("[isIgnoreAPI]"), "ignoreLookupCacheKey" => event.get("[@metadata][ignoreLookupCacheKey]") );'
+  if([@metadata][ignoreApiLookupEnabled]) {
+    # Create a cache key for the event, either on API-Path or Policy-Name
+    mutate {
+      add_field => { "[@metadata][ignoreLookupCacheKey]" => "%{[@metadata][apiCacheKeyPrefix]}###%{[@metadata][ignoreField][policyName]}" }
     }
+    # Lookup the cache is the document should be ignored
     memcached {
       hosts => "${MEMCACHED}"
       namespace => "ignoredAPIs"
-      ttl => "${LOOKUP_CACHE_TTL:600}"
-      set => { "[isIgnoreAPI]" => "%{[@metadata][ignoreLookupCacheKey]}" }
+      get => { "%{[@metadata][ignoreLookupCacheKey]}" => "[isIgnoreAPI]" }
+    }
+    # If we don't know yet, consult the API-Builder API
+    if !([isIgnoreAPI]) {
+      http {
+        id => "Ignore API check"
+        url => "${API_BUILDER_URL}/api/elk/v1/api/lookup/api/ignore"
+        query => {
+          "policyName" => "%{[@metadata][ignoreField][policyName]}"
+          "apiPath" => "%{[@metadata][ignoreField][apiPath]}"
+          "groupId" => "%{[processInfo][groupId]}"
+          "region" => "%{[processInfo][gatewayRegion]}"
+          "correlationId" => "%{[correlationId]}"
+        }
+        automatic_retries => 1 # 1 retry only, as this is not super important and should not block the entire processing
+        cacert => "${API_BUILDER_SSL_CERT}"
+        target_body => "isIgnoreAPI"
+        add_field => { "[@metadata][updateIsIgnoreCache]" => "true" }
+      }
+    }
+    # Drop the document if it should be ignored
+    if ([isIgnoreAPI][ignore]) {
+      ruby {
+        code => 'logger.debug("Drop API/event as it should be ignored: ", "isIgnoreAPI" => event.get("[isIgnoreAPI]"), "ignoreLookupCacheKey" => event.get("[@metadata][ignoreLookupCacheKey]") );'
+      }
+      drop {}
+    }
+    # If the PolicyName or API-Path has been looked up right now add it to the cache
+    if([@metadata][updateIsIgnoreCache]=="true") {
+      ruby {
+        code => 'logger.info("Adding ignore status to cache: ", "ignoreLookupCacheKey" => event.get("[@metadata][ignoreLookupCacheKey]"), "isIgnoreAPI" => event.get("[isIgnoreAPI]") );'
+      }
+      memcached {
+        hosts => "${MEMCACHED}"
+        namespace => "ignoredAPIs"
+        ttl => "${LOOKUP_CACHE_TTL:600}"
+        set => { "[isIgnoreAPI]" => "%{[@metadata][ignoreLookupCacheKey]}" }
+      }
     }
+    mutate { remove_field => "[isIgnoreAPI]" }
   }
-  mutate { remove_field => "[isIgnoreAPI]" }
 
   # Check, if event is a Scheduled-Policy - Most of the properties are NULL
   if([correlationId]!="000000000000000000000000" and ![circuitPath] and ![transactionElement] and ![transactionSummary][path] and ![transactionSummary][protocol] and ![transactionSummary][protocolSrc]) {
@@ -348,7 +352,7 @@ filter {
             }
           }
           # Try to translate the applicationId (e.g. 180b1f32-d72f-40f4-949a-fc3f3f7dec2c) into a meaningful application-name
-          if ([http]) { # Translation only supported for HTTP-Requests
+          if ([http] and [@metadata][applicationLookupEnabled]) { # Translation only supported for HTTP-Requests
             grok {
               match => { "[http][authSubjectId]" => "^.{8}-.{4}-.{4}-.{4}-.{12}$" }
               tag_on_failure => ["_authNSubjectNoUUID"]