t

functionally working

iprep parsing

fixing tests

Revert "Add new iprep sources"

This reverts commit bdd94ac

There is no need to add a different type of repo for iprep based repos.  We can just parse each as if it is either.

fixing integrations

pulling variables from yaml instead of hard coding.

refactoring logic to ignore hidden "." files

removing unneeded code

iprep merge start

creating category class and tests

creating separate file_parser

separating out iprep from rule

removing iprep from rule

fixed iprep mapping bug

moving wite_merg logic to main

get iprep configs in the right dir

fix get_iprep_dir

adding more tests & clean up

making python2 compatible

removing python2 incompatible callable typing

fixing requirements

file_parser refactor

py2 compatibility

modifying rules to match new fileparser format

testing docker change

adding pip2 py2-ipaddress installs for all configs

adding python3 install to all distros

using pip from python

installing pip

undoing all test changes

removing use of ipaddress module for max compatability

Author: michaelschem

suricata/update/category.py

@@ -0,0 +1,157 @@
+# Copyright (C) 2017-2019 Open Information Security Foundation
+# Copyright (c) 2020 Michael Schem
+#
+# You can copy, redistribute or modify this Program under the terms of
+# the GNU General Public License version 2 as published by the Free
+# Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# version 2 along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+""" Module for parsing categories.txt files.
+
+Parsing is done using regular expressions and the job of the module is
+to do its best at parsing out fields of interest from the categories file
+rather than perform a sanity check.
+
+"""
+
+from __future__ import print_function
+
+from suricata.update import fileparser
+
+import io
+import re
+import logging
+
+logger = logging.getLogger(__name__)
+
+# Compile a re pattern for basic iprep directive matching
+category_pattern = re.compile(r"^(?P<enabled>#)*[\s#]*"
+                              r"(?P<id>\d+),"
+                              r"(?P<short_name>\w+),"
+                              r"(?P<description>.*$)")
+
+class Category(dict):
+    """ Class representing an iprep category
+
+    The category class also acts like a dictionary.
+
+    Dictionary fields:
+
+    - **enabled**: True if the category is enabled (uncommented), false is
+      disabled (commented)
+    - **id**: The maximum value for the category id is hard coded at 60
+      currently (Suricata 5.0.3).
+    - **short_name**: The shortname that refers to the category.
+    - **description**: A description of the category.
+
+    :param enabled: Optional parameter to set the enabled state of the category
+
+    """
+
+    def __init__(self, enabled=None):
+        dict.__init__(self)
+        self["enabled"] = enabled
+        self["id"] = None
+        self["short_name"] = None
+        self["description"] = None
+
+    def __getattr__(self, name):
+        return self[name]
+
+    @property
+    def id(self):
+        """ The ID of the category.
+
+        :returns: An int ID of the category
+        :rtype: int
+        """
+        return int(self["id"])
+
+    @property
+    def idstr(self):
+        """Return the gid and sid of the rule as a string formatted like:
+        '[id]'"""
+        return "[%s]" % str(self.id)
+
+    def __str__(self):
+        """ The string representation of the category.
+
+        If the category is disabled it will be returned as commented out.
+        """
+        return self.format()
+
+    def format(self):
+        return "{0}{1},{2},{3}".format(u"" if self["enabled"] else u"# ",
+                                       self['id'],
+                                       self['short_name'],
+                                       self['description'])
+
+
+def parse(buf, group=None):
+    """ Parse a single Iprep category from a string buffer.
+
+    :param buf: A string buffer containing a single Iprep category.
+
+    :returns: An instance of a :py:class:`.Category` representing the parsed Iprep category
+    """
+
+    if type(buf) == type(b""):
+        buf = buf.decode("utf-8")
+    buf = buf.strip()
+
+    m = category_pattern.match(buf)
+    if not m:
+        return None
+
+    if m.group("enabled") == "#":
+        enabled = False
+    else:
+        enabled = True
+
+    # header = m.group("header").strip()
+
+    category = Category(enabled=enabled)
+
+    category["id"] = int(m.group("id").strip())
+
+    if not 0 < category["id"] < 60:
+        logging.error("Category id of {0}, not valid. Id is required to be between 0 and 60.".format(category["id"]))
+        return None
+
+    category["short_name"] = m.group("short_name").strip()
+
+    category["description"] = m.group("description").strip()
+
+    return category
+
+
+def parse_fileobj(fileobj, group=None):
+    """ Parse multiple ipreps from a file like object.
+
+    Note: At this time ipreps must exist on one line.
+
+    :param fileobj: A file like object to parse rules from.
+
+    :returns: A list of :py:class:`.Iprep` instances, one for each rule parsed
+    """
+    return fileparser.parse_fileobj(fileobj, parse, group)
+
+def parse_file(filename, group=None):
+    """ Parse multiple ipreps from the provided filename.
+
+    :param filename: Name of file to parse ipreps from
+
+    :returns: A list of :py:class:`.Iprep` instances, one for each iprep parsed
+    """
+    with io.open(filename, encoding="utf-8") as fileobj:
+        return parse_fileobj(fileobj, group)
+

suricata/update/commands/addsource.py

@@ -37,8 +37,6 @@ def register(parser):
                         help="Additional HTTP header to add to requests")
     parser.add_argument("--no-checksum", action="store_false",
                         help="Skips downloading the checksum URL")
-    parser.add_argument("--iprep", action="store_true",
-                        help="Identifies source as an IPRep Source")
     parser.set_defaults(func=add_source)
 
 
@@ -69,8 +67,6 @@ def add_source():
 
     header = args.http_header if args.http_header else None
 
-    is_iprep = args.iprep
-
     source_config = sources.SourceConfiguration(
-        name, header=header, url=url, checksum=checksum, is_iprep=is_iprep)
+        name, header=header, url=url, checksum=checksum)
     sources.save_source_config(source_config)

suricata/update/config.py

@@ -49,6 +49,7 @@
 DROP_CONF_KEY = "drop-conf"
 LOCAL_CONF_KEY = "local"
 OUTPUT_KEY = "output"
+IPREP_OUTPUT_KEY = "iprep"
 DIST_RULE_DIRECTORY_KEY = "dist-rule-directory"
 
 if has_defaults:
@@ -83,6 +84,7 @@
     # The default file patterns to ignore.
     "ignore": [
         "*deleted.rules",
+        ".*"
     ],
 }
 
@@ -136,6 +138,12 @@ def get_output_dir():
         return _config[OUTPUT_KEY]
     return os.path.join(get_state_dir(), "rules")
 
+def get_iprep_dir():
+    """Get the rule output directory."""
+    if IPREP_OUTPUT_KEY in _config:
+        return _config[IPREP_OUTPUT_KEY]
+    return os.path.join(get_state_dir(), "iprep")
+
 def args():
     """Return sthe parsed argument object."""
     return _args

suricata/update/fileparser.py

@@ -0,0 +1,77 @@
+# Copyright (C) 2017-2019 Open Information Security Foundation
+# Copyright (c) 2020 Michael Schem
+#
+# You can copy, redistribute or modify this Program under the terms of
+# the GNU General Public License version 2 as published by the Free
+# Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# version 2 along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+""" Module for files with either rules, ipreps, or category files.
+
+Parse funcitons
+
+"""
+
+from __future__ import print_function
+
+import sys
+import re
+import logging
+import io
+
+
+logger = logging.getLogger(__name__)
+
+
+def parse_fileobj(fileobj, parse, group=None):
+    """ Parse multiple line based items from a file like object.
+
+    Note: At this point items must exist on one line
+
+    :param fileobj: A file like object to parse items from.
+    :param parse: A function used to parse items.
+
+    :returns: A list of :py:class:`.Rule`, :py:class:`.Iprep`, or :py:class:`.Category`
+    instances depending on what parsing function is passed in.
+    """
+    items = []
+    buf = ""
+    for line in fileobj:
+        try:
+            if type(line) == type(b""):
+                line = line.decode()
+        except:
+            pass
+        if line.rstrip().endswith("\\"):
+            buf = "%s%s " % (buf, line.rstrip()[0:-1])
+            continue
+        buf = buf + line
+        try:
+            item = parse(buf, group)
+            if item:
+                items.append(item)
+        except Exception as err:
+            logger.error("Failed to parse: %s: %s", buf.rstrip(), err)
+        buf = ""
+    return items
+
+
+def parse_file(filename, group=None):
+    """ Parse multiple rules from the provided filename.
+
+    :param filename: Name of file to parse ipreps from
+
+    :returns: A list of .rules files or :py:class:`.Iprep` instances
+    for .list files, one for each rule parsed
+    """
+    with io.open(filename, encoding="utf-8") as fileobj:
+        return parse_fileobj(fileobj, group)