Refactor datadog_checks_base and dev

Author: AAraKKe

datadog_checks_base/datadog_checks/base/checks/base.py

@@ -147,12 +147,12 @@ class except the `check` method but sometimes it might be useful for a Check to
     # a mapping type, then each key will be considered a `name` and will be sent with its (str) value.
     METADATA_TRANSFORMERS = None
 
-    FIRST_CAP_RE = re.compile(br'(.)([A-Z][a-z]+)')
-    ALL_CAP_RE = re.compile(br'([a-z0-9])([A-Z])')
-    METRIC_REPLACEMENT = re.compile(br'([^a-zA-Z0-9_.]+)|(^[^a-zA-Z]+)')
-    TAG_REPLACEMENT = re.compile(br'[,\+\*\-/()\[\]{}\s]')
-    MULTIPLE_UNDERSCORE_CLEANUP = re.compile(br'__+')
-    DOT_UNDERSCORE_CLEANUP = re.compile(br'_*\._*')
+    FIRST_CAP_RE = re.compile(rb'(.)([A-Z][a-z]+)')
+    ALL_CAP_RE = re.compile(rb'([a-z0-9])([A-Z])')
+    METRIC_REPLACEMENT = re.compile(rb'([^a-zA-Z0-9_.]+)|(^[^a-zA-Z]+)')
+    TAG_REPLACEMENT = re.compile(rb'[,\+\*\-/()\[\]{}\s]')
+    MULTIPLE_UNDERSCORE_CLEANUP = re.compile(rb'__+')
+    DOT_UNDERSCORE_CLEANUP = re.compile(rb'_*\._*')
 
     # allows to set a limit on the number of metric name and tags combination
     # this check can send per run. This is useful for checks that have an unbounded
@@ -1146,10 +1146,10 @@ def convert_to_underscore_separated(self, name):
         And substitute illegal metric characters
         """
         name = ensure_bytes(name)
-        metric_name = self.FIRST_CAP_RE.sub(br'\1_\2', name)
-        metric_name = self.ALL_CAP_RE.sub(br'\1_\2', metric_name).lower()
-        metric_name = self.METRIC_REPLACEMENT.sub(br'_', metric_name)
-        return self.DOT_UNDERSCORE_CLEANUP.sub(br'.', metric_name).strip(b'_')
+        metric_name = self.FIRST_CAP_RE.sub(rb'\1_\2', name)
+        metric_name = self.ALL_CAP_RE.sub(rb'\1_\2', metric_name).lower()
+        metric_name = self.METRIC_REPLACEMENT.sub(rb'_', metric_name)
+        return self.DOT_UNDERSCORE_CLEANUP.sub(rb'.', metric_name).strip(b'_')
 
     def warning(self, warning_message, *args, **kwargs):
         # type: (str, *Any, **Any) -> None
@@ -1243,10 +1243,10 @@ def normalize(self, metric, prefix=None, fix_case=False):
             if prefix is not None:
                 prefix = self.convert_to_underscore_separated(prefix)
         else:
-            name = self.METRIC_REPLACEMENT.sub(br'_', metric)
-            name = self.DOT_UNDERSCORE_CLEANUP.sub(br'.', name).strip(b'_')
+            name = self.METRIC_REPLACEMENT.sub(rb'_', metric)
+            name = self.DOT_UNDERSCORE_CLEANUP.sub(rb'.', name).strip(b'_')
 
-        name = self.MULTIPLE_UNDERSCORE_CLEANUP.sub(br'_', name)
+        name = self.MULTIPLE_UNDERSCORE_CLEANUP.sub(rb'_', name)
 
         if prefix is not None:
             name = ensure_bytes(prefix) + b"." + name
@@ -1262,9 +1262,9 @@ def normalize_tag(self, tag):
         """
         if isinstance(tag, str):
             tag = tag.encode('utf-8', 'ignore')
-        tag = self.TAG_REPLACEMENT.sub(br'_', tag)
-        tag = self.MULTIPLE_UNDERSCORE_CLEANUP.sub(br'_', tag)
-        tag = self.DOT_UNDERSCORE_CLEANUP.sub(br'.', tag).strip(b'_')
+        tag = self.TAG_REPLACEMENT.sub(rb'_', tag)
+        tag = self.MULTIPLE_UNDERSCORE_CLEANUP.sub(rb'_', tag)
+        tag = self.DOT_UNDERSCORE_CLEANUP.sub(rb'.', tag).strip(b'_')
         return to_native_string(tag)
 
     def check(self, instance):

datadog_checks_base/datadog_checks/base/checks/network.py

@@ -33,7 +33,7 @@ def check(self, instance):
         try:
             statuses = self._check(instance)
         except Exception:
-            self.log.exception(u"Failed to run instance '%s'.", instance.get('name', u""))
+            self.log.exception("Failed to run instance '%s'.", instance.get('name', ""))
         else:
             if isinstance(statuses, tuple):
                 # Assume the check only returns one service check

datadog_checks_base/datadog_checks/base/checks/prometheus/mixins.py

@@ -561,15 +561,15 @@ def poll(self, endpoint, pFormat=PrometheusFormat.PROTOBUF, headers=None, instan
             headers['Accept-Encoding'] = 'gzip'
         if pFormat == PrometheusFormat.PROTOBUF:
             headers['accept'] = (
-                'application/vnd.google.protobuf; ' 'proto=io.prometheus.client.MetricFamily; ' 'encoding=delimited'
+                'application/vnd.google.protobuf; proto=io.prometheus.client.MetricFamily; encoding=delimited'
             )
         handler = self.get_http_handler(endpoint, instance)
         if (
             endpoint.startswith('https')
             and not handler.ignore_tls_warning
             and not is_affirmative(handler.options.get('ssl_verify', True))
         ):
-            self.log.debug(u'An unverified HTTPS request is being made to %s', endpoint)
+            self.log.debug('An unverified HTTPS request is being made to %s', endpoint)
 
         try:
             response = handler.get(endpoint, extra_headers=headers, stream=False)

datadog_checks_base/datadog_checks/base/checks/win/wmi/base.py

@@ -68,14 +68,14 @@ def _format_tag_query(self, sampler, wmi_obj, tag_query):
             target_property = tag_query[3]
         except IndexError:
             self.log.error(
-                u"Wrong `tag_queries` parameter format. " "Please refer to the configuration file for more information."
+                "Wrong `tag_queries` parameter format. Please refer to the configuration file for more information."
             )
             raise
         except TypeError:
             wmi_property = tag_query[0]
             wmi_class = sampler.class_name
             self.log.error(
-                u"Incorrect 'link source property' in `tag_queries` parameter: `%s` is not a property of `%s`",
+                "Incorrect 'link source property' in `tag_queries` parameter: `%s` is not a property of `%s`",
                 wmi_property,
                 wmi_class,
             )
@@ -94,7 +94,7 @@ def _raise_on_invalid_tag_query_result(self, sampler, wmi_obj, tag_query):
                 message = "multiple results returned (one expected)"
 
             self.log.warning(
-                u"Failed to extract a tag from `tag_queries` parameter: %s. wmi_object=%s - query=%s",
+                "Failed to extract a tag from `tag_queries` parameter: %s. wmi_object=%s - query=%s",
                 message,
                 wmi_obj,
                 tag_query,
@@ -103,7 +103,7 @@ def _raise_on_invalid_tag_query_result(self, sampler, wmi_obj, tag_query):
 
         if sampler[0][target_property] is None:
             self.log.error(
-                u"Incorrect 'target property' in `tag_queries` parameter: `%s` is empty or is not a property of `%s`",
+                "Incorrect 'target property' in `tag_queries` parameter: `%s` is empty or is not a property of `%s`",
                 target_property,
                 target_class,
             )
@@ -116,7 +116,7 @@ def _get_tag_query_tag(self, sampler, wmi_obj, tag_query):
 
         Returns: tag or TagQueryUniquenessFailure exception.
         """
-        self.log.debug(u"`tag_queries` parameter found. wmi_object=%s - query=%s", wmi_obj, tag_query)
+        self.log.debug("`tag_queries` parameter found. wmi_object=%s - query=%s", wmi_obj, tag_query)
 
         # Extract query information
         target_class, target_property, filters = self._format_tag_query(sampler, wmi_obj, tag_query)
@@ -134,7 +134,7 @@ def _get_tag_query_tag(self, sampler, wmi_obj, tag_query):
 
         tag = "{tag_name}:{tag_value}".format(tag_name=target_property.lower(), tag_value="_".join(link_value.split()))
 
-        self.log.debug(u"Extracted `tag_queries` tag: '%s'", tag)
+        self.log.debug("Extracted `tag_queries` tag: '%s'", tag)
         return tag
 
     def _extract_metrics(self, wmi_sampler, tag_by, tag_queries, constant_tags):
@@ -154,7 +154,7 @@ def _extract_metrics(self, wmi_sampler, tag_by, tag_queries, constant_tags):
         """
         if len(wmi_sampler) > 1 and not tag_by:
             raise MissingTagBy(
-                u"WMI query returned multiple rows but no `tag_by` value was given."
+                "WMI query returned multiple rows but no `tag_by` value was given."
                 " class={wmi_class} - properties={wmi_properties} - filters={filters}".format(
                     wmi_class=wmi_sampler.class_name,
                     wmi_properties=wmi_sampler.property_names,
@@ -209,11 +209,11 @@ def _extract_metrics(self, wmi_sampler, tag_by, tag_queries, constant_tags):
                     extracted_metrics.append(WMIMetric(wmi_property, float(wmi_value), tags))
                 except ValueError:
                     self.log.warning(
-                        u"When extracting metrics with WMI, found a non digit value for property '%s'.", wmi_property
+                        "When extracting metrics with WMI, found a non digit value for property '%s'.", wmi_property
                     )
                     continue
                 except TypeError:
-                    self.log.warning(u"When extracting metrics with WMI, found a missing property '%s'", wmi_property)
+                    self.log.warning("When extracting metrics with WMI, found a missing property '%s'", wmi_property)
                     continue
         return extracted_metrics
 
@@ -241,7 +241,7 @@ def _submit_metrics(self, metrics, metric_name_and_type_by_property):
             try:
                 func = getattr(self, metric_type.lower())
             except AttributeError:
-                raise Exception(u"Invalid metric type: {0}".format(metric_type))
+                raise Exception("Invalid metric type: {0}".format(metric_type))
 
             func(metric_name, metric.value, metric.tags)
 
@@ -270,7 +270,7 @@ def get_running_wmi_sampler(self, properties, filters, **kwargs):
             username=self.username,
             password=self.password,
             tag_by=tag_by,
-            **kwargs
+            **kwargs,
         )
 
     def _get_running_wmi_sampler(self, instance_key, wmi_class, properties, tag_by="", **kwargs):

datadog_checks_base/datadog_checks/base/checks/win/wmi/sampler.py

@@ -23,6 +23,7 @@
 Original discussion thread: https://github.com/DataDog/dd-agent/issues/1952
 Credits to @TheCloudlessSky (https://github.com/TheCloudlessSky)
 """
+
 from copy import deepcopy
 from threading import Event, Thread
 
@@ -227,7 +228,7 @@ def provider(self, value):
                 result = parsed_value
 
         if result is None:
-            self.logger.error(u"Invalid '%s' WMI Provider Architecture. The parameter is ignored.", value)
+            self.logger.error("Invalid '%s' WMI Provider Architecture. The parameter is ignored.", value)
 
         self._provider = result or ProviderArchitecture.DEFAULT
 
@@ -278,7 +279,7 @@ def __len__(self):
         """
         # No data is returned while sampling
         if self._sampling:
-            raise TypeError(u"Sampling `WMISampler` object has no len()")
+            raise TypeError("Sampling `WMISampler` object has no len()")
 
         return len(self._current_sample)
 
@@ -288,7 +289,7 @@ def __iter__(self):
         """
         # No data is returned while sampling
         if self._sampling:
-            raise TypeError(u"Sampling `WMISampler` object is not iterable")
+            raise TypeError("Sampling `WMISampler` object is not iterable")
 
         if self.is_raw_perf_class:
             # Format required
@@ -334,7 +335,7 @@ def _get_property_calculator(self, counter_type):
             calculator = get_calculator(counter_type)
         except UndefinedCalculator:
             self.logger.warning(
-                u"Undefined WMI calculator for counter_type %s. Values are reported as RAW.", counter_type
+                "Undefined WMI calculator for counter_type %s. Values are reported as RAW.", counter_type
             )
 
         return calculator
@@ -364,7 +365,7 @@ def get_connection(self):
         Create a new WMI connection
         """
         self.logger.debug(
-            u"Connecting to WMI server (host=%s, namespace=%s, provider=%s, username=%s).",
+            "Connecting to WMI server (host=%s, namespace=%s, provider=%s, username=%s).",
             self.host,
             self.namespace,
             self.provider,
@@ -550,7 +551,7 @@ def _query(self):  # pylint: disable=E0202
             wql = "Select {property_names} from {class_name}{filters}".format(
                 property_names=formated_property_names, class_name=self.class_name, filters=self.formatted_filters
             )
-            self.logger.debug(u"Querying WMI: %s", wql)
+            self.logger.debug("Querying WMI: %s", wql)
         except Exception as e:
             self.logger.error(str(e))
             return []
@@ -575,7 +576,7 @@ def _query(self):  # pylint: disable=E0202
             results = self._parse_results(raw_results, includes_qualifiers=includes_qualifiers)
 
         except pywintypes.com_error:
-            self.logger.warning(u"Failed to execute WMI query (%s)", wql, exc_info=True)
+            self.logger.warning("Failed to execute WMI query (%s)", wql, exc_info=True)
             results = []
 
         return results
@@ -615,7 +616,6 @@ def _parse_results(self, raw_results, includes_qualifiers):
                 )
 
                 if should_get_qualifier_type:
-
                     # Can't index into "Qualifiers_" for keys that don't exist
                     # without getting an exception.
                     qualifiers = dict((q.Name, q.Value) for q in wmi_property.Qualifiers_)
@@ -628,14 +628,14 @@ def _parse_results(self, raw_results, includes_qualifiers):
                         self._property_counter_types[wmi_property.Name] = counter_type
 
                         self.logger.debug(
-                            u"Caching property qualifier CounterType: %s.%s = %s",
+                            "Caching property qualifier CounterType: %s.%s = %s",
                             self.class_name,
                             wmi_property.Name,
                             counter_type,
                         )
                     else:
                         self.logger.debug(
-                            u"CounterType qualifier not found for %s.%s", self.class_name, wmi_property.Name
+                            "CounterType qualifier not found for %s.%s", self.class_name, wmi_property.Name
                         )
 
                 try:

datadog_checks_base/datadog_checks/base/checks/windows/perf_counters/connection.py

@@ -77,7 +77,7 @@ def __init__(self, config):
             # https://docs.microsoft.com/en-us/windows/win32/api/winnetwk/ns-winnetwk-netresourcea
             # https://mhammond.github.io/pywin32/PyNETRESOURCE.html
             self.network_resource = win32wnet.NETRESOURCE()
-            self.network_resource.lpRemoteName = fr'\\{server}'
+            self.network_resource.lpRemoteName = rf'\\{server}'
 
         self.__query_handle = None
 

datadog_checks_base/datadog_checks/base/stubs/datadog_agent.py

@@ -67,10 +67,10 @@ def assert_external_tags(self, hostname, external_tags, match_tags_order=False):
                     external_tags = {k: sorted(v) for (k, v) in external_tags.items()}
                     tags = {k: sorted(v) for (k, v) in tags.items()}
 
-                assert (
-                    external_tags == tags
-                ), 'Expected {} external tags for hostname {}, found {}. Submitted external tags: {}'.format(
-                    external_tags, hostname, tags, repr(self._external_tags)
+                assert external_tags == tags, (
+                    'Expected {} external tags for hostname {}, found {}. Submitted external tags: {}'.format(
+                        external_tags, hostname, tags, repr(self._external_tags)
+                    )
                 )
                 return
 

datadog_checks_base/datadog_checks/base/utils/db/query.py

@@ -214,9 +214,7 @@ def compile(
             elif extra_type not in extra_transformers and extra_type not in submission_transformers:
                 raise ValueError('unknown type `{}` for extra {} of {}'.format(extra_type, extra_name, query_name))
 
-            transformer_factory = extra_transformers.get(
-                extra_type, submission_transformers.get(extra_type)
-            )  # type: TransformerFactory
+            transformer_factory = extra_transformers.get(extra_type, submission_transformers.get(extra_type))  # type: TransformerFactory
 
             extra_source = extra.get('source')
             if extra_type in submission_transformers:

datadog_checks_base/datadog_checks/base/utils/db/transform.py

@@ -85,9 +85,7 @@ def get_monotonic_gauge(transformers, column_name, **modifiers):
     Send the result as both a `gauge` suffixed by `.total` and a `monotonic_count` suffixed by `.count`.
     """
     gauge = transformers['gauge'](transformers, '{}.total'.format(column_name), **modifiers)  # type: Callable
-    monotonic_count = transformers['monotonic_count'](
-        transformers, '{}.count'.format(column_name), **modifiers
-    )  # type: Callable
+    monotonic_count = transformers['monotonic_count'](transformers, '{}.count'.format(column_name), **modifiers)  # type: Callable
 
     def monotonic_gauge(_, value, **kwargs):
         # type: (List, str, Dict[str, Any]) -> None

datadog_checks_base/datadog_checks/base/utils/http.py

@@ -378,7 +378,7 @@ def options_method(self, url, **options):
 
     def _request(self, method, url, options):
         if self.log_requests:
-            self.logger.debug(u'Sending %s request to %s', method.upper(), url)
+            self.logger.debug('Sending %s request to %s', method.upper(), url)
 
         if self.no_proxy_uris and should_bypass_proxy(url, self.no_proxy_uris):
             options.setdefault('proxies', PROXY_SETTINGS_DISABLED)
@@ -390,7 +390,7 @@ def _request(self, method, url, options):
         new_options = self.populate_options(options)
 
         if url.startswith('https') and not self.ignore_tls_warning and not new_options['verify']:
-            self.logger.debug(u'An unverified HTTPS request is being made to %s', url)
+            self.logger.debug('An unverified HTTPS request is being made to %s', url)
 
         extra_headers = options.pop('extra_headers', None)
         if extra_headers is not None:
@@ -416,7 +416,7 @@ def _request(self, method, url, options):
                     response = self.make_request_aia_chasing(request_method, method, url, new_options, persist)
                     response.raise_for_status()
                 except Exception as e:
-                    self.logger.debug(u'Renewing auth token, as an error occurred: %s', e)
+                    self.logger.debug('Renewing auth token, as an error occurred: %s', e)
                     self.handle_auth_token(method=method, url=url, default_options=self.options, error=str(e))
                     response = self.make_request_aia_chasing(request_method, method, url, new_options, persist)
             else: