Mime type reload v2.4

.github/workflows/builds.yml

@@ -827,7 +827,7 @@ jobs:
           path: prep
       - run: tar xf prep/suricata-update.tar.gz
       - run: ./autogen.sh
-      - run: RUSTC_WRAPPER="$(pwd)/scripts/rustc.py" ./configure --enable-warnings --disable-shared
+      - run: RUSTC_WRAPPER="$(pwd)/scripts/rustc.py" ./configure --enable-warnings --disable-shared --enable-mimetype
         env:
           CC: "clang"
           RUSTFLAGS: "-Cinstrument-coverage"
@@ -837,6 +837,8 @@ jobs:
           CC: "clang"
           RUSTFLAGS: "-Cinstrument-coverage"
           CFLAGS: "-fprofile-instr-generate -fcoverage-mapping -O0"
+      - name: install mimetype files
+        run: make upgrade-data
       - name: Extracting suricata-verify
         run: tar xf prep/suricata-verify.tar.gz
       - run: rm -r prep
@@ -1328,14 +1330,153 @@ jobs:
       - run: tar xf prep/suricata-update.tar.gz
       - run: tar xf prep/suricata-verify.tar.gz
       - run: ./autogen.sh
-      - run: ./configure --enable-unittests --enable-coccinelle
+      - run: ./configure --enable-unittests --enable-coccinelle --enable-mimetype
       - run: make -j ${{ env.CPUS }}
       - run: CONCURRENCY_LEVEL=${{ env.CPUS }} make check
+      - run: make upgrade-data
       - run: python3 ./suricata-verify/run.py -q --debug-failed
       - run: make install
       - run: make install-headers
       - run: make install-library
 
+  ubuntu-24-04-sys-mime:
+    name: Ubuntu 24.04 (mime sys data)
+    runs-on: ubuntu-latest
+    container: ubuntu:24.04
+    needs: [prepare-deps]
+    steps:
+      - name: Cache ~/.cargo
+        uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57
+        with:
+          path: ~/.cargo/registry
+          key: cargo-registry
+
+      - name: Determine number of CPUs
+        run: echo CPUS=$(nproc --all) >> $GITHUB_ENV
+
+      - name: Install dependencies
+        run: |
+          apt update
+          apt -y install \
+                autoconf \
+                automake \
+                build-essential \
+                cargo \
+                cbindgen \
+                clang-14 \
+                coccinelle \
+                dpdk-dev \
+                git \
+                hwloc \
+                libhwloc-dev \
+                jq \
+                libcap-ng-dev \
+                libevent-dev \
+                libevent-pthreads-2.1-7 \
+                libhiredis-dev \
+                libhyperscan-dev \
+                libjansson-dev \
+                libmagic-dev \
+                libnet1-dev \
+                libnuma-dev \
+                libpcap-dev \
+                libpcre2-dev \
+                libpython3.12 \
+                libtool \
+                libyaml-dev \
+                llvm-14-dev \
+                make \
+                parallel \
+                python-is-python3 \
+                python3-yaml \
+                rustc \
+                shared-mime-info \
+                software-properties-common \
+                zlib1g \
+                zlib1g-dev
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+      - run: git config --global --add safe.directory /__w/suricata/suricata
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093
+        with:
+          name: prep
+          path: prep
+      - run: tar xf prep/suricata-update.tar.gz
+      - run: tar xf prep/suricata-verify.tar.gz
+      - run: ./autogen.sh
+      - run: ./configure --enable-unittests --enable-coccinelle --enable-mimetype
+      - run: make -j ${{ env.CPUS }}
+      - run: CONCURRENCY_LEVEL=${{ env.CPUS }} make check
+      - run: python3 ./suricata-verify/run.py -q --debug-failed
+
+
+  ubuntu-24-04-gpl-mime:
+    name: Ubuntu 24.04 (gpl data)
+    runs-on: ubuntu-latest
+    container: ubuntu:24.04
+    needs: [prepare-deps]
+    steps:
+      - name: Cache ~/.cargo
+        uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57
+        with:
+          path: ~/.cargo/registry
+          key: cargo-registry
+
+      - name: Determine number of CPUs
+        run: echo CPUS=$(nproc --all) >> $GITHUB_ENV
+
+      - name: Install dependencies
+        run: |
+          apt update
+          apt -y install \
+                autoconf \
+                automake \
+                build-essential \
+                cargo \
+                cbindgen \
+                clang-14 \
+                coccinelle \
+                dpdk-dev \
+                git \
+                hwloc \
+                libhwloc-dev \
+                jq \
+                libcap-ng-dev \
+                libevent-dev \
+                libevent-pthreads-2.1-7 \
+                libhiredis-dev \
+                libhyperscan-dev \
+                libjansson-dev \
+                libmagic-dev \
+                libnet1-dev \
+                libnuma-dev \
+                libpcap-dev \
+                libpcre2-dev \
+                libpython3.12 \
+                libtool \
+                libyaml-dev \
+                llvm-14-dev \
+                make \
+                parallel \
+                python-is-python3 \
+                python3-yaml \
+                rustc \
+                software-properties-common \
+                zlib1g \
+                zlib1g-dev
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+      - run: git config --global --add safe.directory /__w/suricata/suricata
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093
+        with:
+          name: prep
+          path: prep
+      - run: tar xf prep/suricata-update.tar.gz
+      - run: tar xf prep/suricata-verify.tar.gz
+      - run: ./autogen.sh
+      - run: ./configure --enable-unittests --enable-coccinelle --enable-mimetype --enable-bundled-gpl-mimetype
+      - run: make -j ${{ env.CPUS }}
+      - run: CONCURRENCY_LEVEL=${{ env.CPUS }} make check
+      - run: python3 ./suricata-verify/run.py -q --debug-failed
+
   ubuntu-24-04-cov-ut:
     name: Ubuntu 24.04 (unittests coverage)
     runs-on: ubuntu-latest

.github/workflows/rust.yml

@@ -68,7 +68,7 @@ jobs:
       - run: git config --global --add safe.directory /__w/suricata/suricata
       - run: ./scripts/bundle.sh
       - run: ./autogen.sh
-      - run: ./configure --enable-warnings
+      - run: ./configure --enable-warnings --enable-mimetype --enable-bundled-gpl-mimetype
       - name: Checking bindgen output
         working-directory: rust
         run: |

Makefile.am

@@ -39,8 +39,28 @@ install-conf:
 	install -d "$(DESTDIR)$(e_rundir)"
 	install -m 770 -d "$(DESTDIR)$(e_localstatedir)"
 	install -m 770 -d "$(DESTDIR)$(e_datadir)"
+if INSTALL_MIMETYPE_FILES
+	install -m 770 -d "$(DESTDIR)$(e_datadir)/mimetype"
+	@test -e "$(DESTDIR)$(e_datadir)/mimetype/magic" || \
+		install -m 600 "$(top_srcdir)/etc/mimetype/magic" "$(DESTDIR)$(e_datadir)/mimetype"
+	@test -e "$(DESTDIR)$(e_datadir)/mimetype/aliases" || \
+		install -m 600 "$(top_srcdir)/etc/mimetype/aliases" "$(DESTDIR)$(e_datadir)/mimetype"
+	@test -e "$(DESTDIR)$(e_datadir)/mimetype/subclasses" || \
+		install -m 600 "$(top_srcdir)/etc/mimetype/subclasses" "$(DESTDIR)$(e_datadir)/mimetype"
+endif
+
 	install -m 770 -d "$(DESTDIR)$(e_sghcachedir)"
 
+upgrade-data:
+if INSTALL_MIMETYPE_FILES
+	install -d "$(DESTDIR)$(e_datadir)/mimetype"
+	install -m 600 "$(top_srcdir)/etc/mimetype/magic" "$(DESTDIR)$(e_datadir)/mimetype"
+	install -m 600 "$(top_srcdir)/etc/mimetype/aliases" "$(DESTDIR)$(e_datadir)/mimetype"
+	install -m 600 "$(top_srcdir)/etc/mimetype/subclasses" "$(DESTDIR)$(e_datadir)/mimetype"
+else
+	@echo "Not installing Mime type files as mimetype support not enabled or gpl data are embedded in the binary."
+endif
+
 install-rules:
 if INSTALL_SURICATA_UPDATE
 	LD_LIBRARY_PATH=$(libdir) $(DESTDIR)$(bindir)/suricata-update \

configure.ac

@@ -1765,6 +1765,34 @@
         fi
     fi
 
+  # mimetype
+    AC_ARG_ENABLE(mimetype,
+           AS_HELP_STRING([--enable-mimetype], [Enable mimetype support [default=no]]),
+                        [enable_mimetype="$enableval"],[enable_mimetype=no])
+    install_mimetype_files="no"
+    AS_IF([test "x$enable_mimetype" = "xyes"], [
+        install_mimetype_files="yes"
+        AC_DEFINE([HAVE_MIMETYPE],[1],(Mimetype support enabled))
+        AC_ARG_ENABLE(bundled-gpl-mimetype,
+               AS_HELP_STRING([--enable-bundled-gpl-mimetype], [Embed GPL data in mimetype support [default=no]]),
+                            [enable_gpl_mimetype="$enableval"],[enable_gpl_mimetype=no])
+        AS_IF([test "x$enable_gpl_mimetype" = "xyes"], [
+            WITH_GPL_DATA="\"with-gpl-data\""
+            AC_SUBST(WITH_GPL_DATA)
+            AC_DEFINE([HAVE_GPL_MIMETYPE],[1],[Embedded GPL data for Mimetype support enabled])
+            install_mimetype_files="no"
+        ])
+        AM_CONDITIONAL([HAVE_MIMETYPE], [true])
+        ],
+        [
+        AM_CONDITIONAL([HAVE_MIMETYPE], [false])
+        ]
+    )
+    if test "x$enable_gpl_mimetype" != "xyes"; then
+        enable_gpl_mimetype="no"
+    fi
+    AM_CONDITIONAL([INSTALL_MIMETYPE_FILES], [test "x$install_mimetype_files" = "xyes"])
+
     # Napatech - Using the 3GD API
     AC_ARG_ENABLE(napatech,
                 AS_HELP_STRING([--enable-napatech],[Enable Napatech Devices]),
@@ -2578,6 +2606,8 @@ SURICATA_BUILD_CONF="Suricata Configuration:
   Detection enabled:                       ${enable_detection}
 
   Libmagic support:                        ${enable_magic}
+  mimetype support:                        ${enable_mimetype}
+  GPL Mimetype DB inclusion:               ${enable_gpl_mimetype}
   libjansson support:                      ${enable_jansson}
   hiredis support:                         ${enable_hiredis}
   hiredis async with libevent:             ${enable_hiredis_async}

doc/userguide/output/eve/eve-json-output.rst

@@ -315,6 +315,65 @@ YAML::
             # Default: all.
             #types: [a, aaaa, cname, mx, ns, ptr, txt]
 
+
+.. _output-eve-fileinfo:
+
+Fileinfo
+~~~~~~~~
+
+Fileinfo records are logged when files are seen on network traffic. The log
+contains information about the file, such as its name, size, and
+hashes.
+
+The type of the file can also be determined by doing an analysis of the beginning of the content
+of the file. This is done by using libmagic and/or libmimetype. Magic is slower than mimetype, but 
+it can provide more information about the file type. At the same time, magic output is less predictive than
+mimetype one that follows a standard.
+
+If you want to log the file type, you can enable either or both of these options.
+
+YAML::
+
+        - files:
+            force-magic: no   # force logging magic on all logged files
+            force-mimetype: no # force logging mime type on all logged files
+            # force logging of checksums, available hash functions are md5,
+            # sha1 and sha256
+            #force-hash: [md5]
+
+The identification by magic is using a file available on the system. The file can be set by using a
+dedicated configuration option:
+
+YAML::
+
+  magic-file: /usr/share/file/magic
+
+The mimetype analysis is using by default Suricata provided MIME type files if the bundled GPL
+data is not included at build time and if these mime type files have been installed on the system.
+This can be done by using ``make install-conf`` for initial installation or by using
+``make upgrade-data`` for updating the files.
+
+If Suricata MIME types files are not available, then the system MIME type files will be used.
+In that case, the used files have the same syntax as the one
+used by magic but they are auto discovered using predefined path and paths
+built upon XDG_DATA_DIRS and XDG_DATA_HOME variables.
+
+If bundled GPL data is included then no files on disk are used and the results of the
+identification are only dependent of the version of the `tree_magic_mini Rust crate <https://github.com/mbrubeck/tree_magic/>`_
+that has been used to build Suricata.
+
+If ever customer MIME type files are needed, then the ``mimetype-dir`` option can be used to
+specify a directory where these MIME type files are located. This directory will be used instead
+of the default one.
+
+YAML::
+
+  mimetype-dir: /etc/suricata/mime
+
+Alternatively, the ``TREE_MAGIC_DIR`` environment variable can be set to
+point to a directory where the MIME type files are located. This will override the setting
+in the configuration file.
+
 TLS
 ~~~
 

doc/userguide/rules/file-keywords.rst

@@ -127,6 +127,24 @@ here: https://redmine.openinfosecfoundation.org/issues/437
 
 ``file.magic`` supports multiple buffer matching, see :doc:`multi-buffer-matching`.
 
+file.mimetype
+-------------
+
+Sticky buffer that matches on the MIME type guessed from the binary content of a file.
+
+Example::
+
+  file.mimetype; content:"application/vnd.microsoft.portable-executable";
+
+``file.mimetype`` supports multiple buffer matching, see :doc:`multi-buffer-matching`.
+
+The MIME type analysis is faster than the magic analysis and the identification is also
+more reproducible across different Suricata versions and operating systems. Being a
+standard, this is also improving correlation with other tools.
+
+See :ref:`output-eve-fileinfo` for more information on how to set up MIME type
+identification database.
+
 filestore
 ---------