diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index e35078835e57..fdd7751ba3d5 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -827,7 +827,7 @@ jobs: path: prep - run: tar xf prep/suricata-update.tar.gz - run: ./autogen.sh - - run: RUSTC_WRAPPER="$(pwd)/scripts/rustc.py" ./configure --enable-warnings --disable-shared + - run: RUSTC_WRAPPER="$(pwd)/scripts/rustc.py" ./configure --enable-warnings --disable-shared --enable-mimetype env: CC: "clang" RUSTFLAGS: "-Cinstrument-coverage" @@ -837,6 +837,8 @@ jobs: CC: "clang" RUSTFLAGS: "-Cinstrument-coverage" CFLAGS: "-fprofile-instr-generate -fcoverage-mapping -O0" + - name: install mimetype files + run: make upgrade-data - name: Extracting suricata-verify run: tar xf prep/suricata-verify.tar.gz - run: rm -r prep @@ -1328,14 +1330,153 @@ jobs: - run: tar xf prep/suricata-update.tar.gz - run: tar xf prep/suricata-verify.tar.gz - run: ./autogen.sh - - run: ./configure --enable-unittests --enable-coccinelle + - run: ./configure --enable-unittests --enable-coccinelle --enable-mimetype - run: make -j ${{ env.CPUS }} - run: CONCURRENCY_LEVEL=${{ env.CPUS }} make check + - run: make upgrade-data - run: python3 ./suricata-verify/run.py -q --debug-failed - run: make install - run: make install-headers - run: make install-library + ubuntu-24-04-sys-mime: + name: Ubuntu 24.04 (mime sys data) + runs-on: ubuntu-latest + container: ubuntu:24.04 + needs: [prepare-deps] + steps: + - name: Cache ~/.cargo + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 + with: + path: ~/.cargo/registry + key: cargo-registry + + - name: Determine number of CPUs + run: echo CPUS=$(nproc --all) >> $GITHUB_ENV + + - name: Install dependencies + run: | + apt update + apt -y install \ + autoconf \ + automake \ + build-essential \ + cargo \ + cbindgen \ + clang-14 \ + coccinelle \ + dpdk-dev \ + git \ + hwloc \ + libhwloc-dev \ + jq \ + libcap-ng-dev \ + libevent-dev \ + libevent-pthreads-2.1-7 \ + libhiredis-dev \ + libhyperscan-dev \ + libjansson-dev \ + libmagic-dev \ + libnet1-dev \ + libnuma-dev \ + libpcap-dev \ + libpcre2-dev \ + libpython3.12 \ + libtool \ + libyaml-dev \ + llvm-14-dev \ + make \ + parallel \ + python-is-python3 \ + python3-yaml \ + rustc \ + shared-mime-info \ + software-properties-common \ + zlib1g \ + zlib1g-dev + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + - run: git config --global --add safe.directory /__w/suricata/suricata + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + name: prep + path: prep + - run: tar xf prep/suricata-update.tar.gz + - run: tar xf prep/suricata-verify.tar.gz + - run: ./autogen.sh + - run: ./configure --enable-unittests --enable-coccinelle --enable-mimetype + - run: make -j ${{ env.CPUS }} + - run: CONCURRENCY_LEVEL=${{ env.CPUS }} make check + - run: python3 ./suricata-verify/run.py -q --debug-failed + + + ubuntu-24-04-gpl-mime: + name: Ubuntu 24.04 (gpl data) + runs-on: ubuntu-latest + container: ubuntu:24.04 + needs: [prepare-deps] + steps: + - name: Cache ~/.cargo + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 + with: + path: ~/.cargo/registry + key: cargo-registry + + - name: Determine number of CPUs + run: echo CPUS=$(nproc --all) >> $GITHUB_ENV + + - name: Install dependencies + run: | + apt update + apt -y install \ + autoconf \ + automake \ + build-essential \ + cargo \ + cbindgen \ + clang-14 \ + coccinelle \ + dpdk-dev \ + git \ + hwloc \ + libhwloc-dev \ + jq \ + libcap-ng-dev \ + libevent-dev \ + libevent-pthreads-2.1-7 \ + libhiredis-dev \ + libhyperscan-dev \ + libjansson-dev \ + libmagic-dev \ + libnet1-dev \ + libnuma-dev \ + libpcap-dev \ + libpcre2-dev \ + libpython3.12 \ + libtool \ + libyaml-dev \ + llvm-14-dev \ + make \ + parallel \ + python-is-python3 \ + python3-yaml \ + rustc \ + software-properties-common \ + zlib1g \ + zlib1g-dev + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + - run: git config --global --add safe.directory /__w/suricata/suricata + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + name: prep + path: prep + - run: tar xf prep/suricata-update.tar.gz + - run: tar xf prep/suricata-verify.tar.gz + - run: ./autogen.sh + - run: ./configure --enable-unittests --enable-coccinelle --enable-mimetype --enable-bundled-gpl-mimetype + - run: make -j ${{ env.CPUS }} + - run: CONCURRENCY_LEVEL=${{ env.CPUS }} make check + - run: python3 ./suricata-verify/run.py -q --debug-failed + ubuntu-24-04-cov-ut: name: Ubuntu 24.04 (unittests coverage) runs-on: ubuntu-latest diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 134d23517e8e..cdb2677f5318 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -68,7 +68,7 @@ jobs: - run: git config --global --add safe.directory /__w/suricata/suricata - run: ./scripts/bundle.sh - run: ./autogen.sh - - run: ./configure --enable-warnings + - run: ./configure --enable-warnings --enable-mimetype --enable-bundled-gpl-mimetype - name: Checking bindgen output working-directory: rust run: | diff --git a/Makefile.am b/Makefile.am index 7e8c37e9fa5f..21181dd82c8f 100644 --- a/Makefile.am +++ b/Makefile.am @@ -39,8 +39,28 @@ install-conf: install -d "$(DESTDIR)$(e_rundir)" install -m 770 -d "$(DESTDIR)$(e_localstatedir)" install -m 770 -d "$(DESTDIR)$(e_datadir)" +if INSTALL_MIMETYPE_FILES + install -m 770 -d "$(DESTDIR)$(e_datadir)/mimetype" + @test -e "$(DESTDIR)$(e_datadir)/mimetype/magic" || \ + install -m 600 "$(top_srcdir)/etc/mimetype/magic" "$(DESTDIR)$(e_datadir)/mimetype" + @test -e "$(DESTDIR)$(e_datadir)/mimetype/aliases" || \ + install -m 600 "$(top_srcdir)/etc/mimetype/aliases" "$(DESTDIR)$(e_datadir)/mimetype" + @test -e "$(DESTDIR)$(e_datadir)/mimetype/subclasses" || \ + install -m 600 "$(top_srcdir)/etc/mimetype/subclasses" "$(DESTDIR)$(e_datadir)/mimetype" +endif + install -m 770 -d "$(DESTDIR)$(e_sghcachedir)" +upgrade-data: +if INSTALL_MIMETYPE_FILES + install -d "$(DESTDIR)$(e_datadir)/mimetype" + install -m 600 "$(top_srcdir)/etc/mimetype/magic" "$(DESTDIR)$(e_datadir)/mimetype" + install -m 600 "$(top_srcdir)/etc/mimetype/aliases" "$(DESTDIR)$(e_datadir)/mimetype" + install -m 600 "$(top_srcdir)/etc/mimetype/subclasses" "$(DESTDIR)$(e_datadir)/mimetype" +else + @echo "Not installing Mime type files as mimetype support not enabled or gpl data are embedded in the binary." +endif + install-rules: if INSTALL_SURICATA_UPDATE LD_LIBRARY_PATH=$(libdir) $(DESTDIR)$(bindir)/suricata-update \ diff --git a/configure.ac b/configure.ac index afe93bd8fa7e..d36616098340 100644 --- a/configure.ac +++ b/configure.ac @@ -1765,6 +1765,34 @@ fi fi + # mimetype + AC_ARG_ENABLE(mimetype, + AS_HELP_STRING([--enable-mimetype], [Enable mimetype support [default=no]]), + [enable_mimetype="$enableval"],[enable_mimetype=no]) + install_mimetype_files="no" + AS_IF([test "x$enable_mimetype" = "xyes"], [ + install_mimetype_files="yes" + AC_DEFINE([HAVE_MIMETYPE],[1],(Mimetype support enabled)) + AC_ARG_ENABLE(bundled-gpl-mimetype, + AS_HELP_STRING([--enable-bundled-gpl-mimetype], [Embed GPL data in mimetype support [default=no]]), + [enable_gpl_mimetype="$enableval"],[enable_gpl_mimetype=no]) + AS_IF([test "x$enable_gpl_mimetype" = "xyes"], [ + WITH_GPL_DATA="\"with-gpl-data\"" + AC_SUBST(WITH_GPL_DATA) + AC_DEFINE([HAVE_GPL_MIMETYPE],[1],[Embedded GPL data for Mimetype support enabled]) + install_mimetype_files="no" + ]) + AM_CONDITIONAL([HAVE_MIMETYPE], [true]) + ], + [ + AM_CONDITIONAL([HAVE_MIMETYPE], [false]) + ] + ) + if test "x$enable_gpl_mimetype" != "xyes"; then + enable_gpl_mimetype="no" + fi + AM_CONDITIONAL([INSTALL_MIMETYPE_FILES], [test "x$install_mimetype_files" = "xyes"]) + # Napatech - Using the 3GD API AC_ARG_ENABLE(napatech, AS_HELP_STRING([--enable-napatech],[Enable Napatech Devices]), @@ -2578,6 +2606,8 @@ SURICATA_BUILD_CONF="Suricata Configuration: Detection enabled: ${enable_detection} Libmagic support: ${enable_magic} + mimetype support: ${enable_mimetype} + GPL Mimetype DB inclusion: ${enable_gpl_mimetype} libjansson support: ${enable_jansson} hiredis support: ${enable_hiredis} hiredis async with libevent: ${enable_hiredis_async} diff --git a/doc/userguide/output/eve/eve-json-output.rst b/doc/userguide/output/eve/eve-json-output.rst index a6c88d1e640a..0d3ca4bdca82 100644 --- a/doc/userguide/output/eve/eve-json-output.rst +++ b/doc/userguide/output/eve/eve-json-output.rst @@ -315,6 +315,65 @@ YAML:: # Default: all. #types: [a, aaaa, cname, mx, ns, ptr, txt] + +.. _output-eve-fileinfo: + +Fileinfo +~~~~~~~~ + +Fileinfo records are logged when files are seen on network traffic. The log +contains information about the file, such as its name, size, and +hashes. + +The type of the file can also be determined by doing an analysis of the beginning of the content +of the file. This is done by using libmagic and/or libmimetype. Magic is slower than mimetype, but +it can provide more information about the file type. At the same time, magic output is less predictive than +mimetype one that follows a standard. + +If you want to log the file type, you can enable either or both of these options. + +YAML:: + + - files: + force-magic: no # force logging magic on all logged files + force-mimetype: no # force logging mime type on all logged files + # force logging of checksums, available hash functions are md5, + # sha1 and sha256 + #force-hash: [md5] + +The identification by magic is using a file available on the system. The file can be set by using a +dedicated configuration option: + +YAML:: + + magic-file: /usr/share/file/magic + +The mimetype analysis is using by default Suricata provided MIME type files if the bundled GPL +data is not included at build time and if these mime type files have been installed on the system. +This can be done by using ``make install-conf`` for initial installation or by using +``make upgrade-data`` for updating the files. + +If Suricata MIME types files are not available, then the system MIME type files will be used. +In that case, the used files have the same syntax as the one +used by magic but they are auto discovered using predefined path and paths +built upon XDG_DATA_DIRS and XDG_DATA_HOME variables. + +If bundled GPL data is included then no files on disk are used and the results of the +identification are only dependent of the version of the `tree_magic_mini Rust crate `_ +that has been used to build Suricata. + +If ever customer MIME type files are needed, then the ``mimetype-dir`` option can be used to +specify a directory where these MIME type files are located. This directory will be used instead +of the default one. + +YAML:: + + mimetype-dir: /etc/suricata/mime + +Alternatively, the ``TREE_MAGIC_DIR`` environment variable can be set to +point to a directory where the MIME type files are located. This will override the setting +in the configuration file. + TLS ~~~ diff --git a/doc/userguide/rules/file-keywords.rst b/doc/userguide/rules/file-keywords.rst index 91637576852a..e7aac16c8c35 100644 --- a/doc/userguide/rules/file-keywords.rst +++ b/doc/userguide/rules/file-keywords.rst @@ -127,6 +127,24 @@ here: https://redmine.openinfosecfoundation.org/issues/437 ``file.magic`` supports multiple buffer matching, see :doc:`multi-buffer-matching`. +file.mimetype +------------- + +Sticky buffer that matches on the MIME type guessed from the binary content of a file. + +Example:: + + file.mimetype; content:"application/vnd.microsoft.portable-executable"; + +``file.mimetype`` supports multiple buffer matching, see :doc:`multi-buffer-matching`. + +The MIME type analysis is faster than the magic analysis and the identification is also +more reproducible across different Suricata versions and operating systems. Being a +standard, this is also improving correlation with other tools. + +See :ref:`output-eve-fileinfo` for more information on how to set up MIME type +identification database. + filestore --------- diff --git a/etc/mimetype/aliases b/etc/mimetype/aliases new file mode 100644 index 000000000000..db19cd14df02 --- /dev/null +++ b/etc/mimetype/aliases @@ -0,0 +1,328 @@ +application/acrobat application/pdf +application/bat application/x-bat +application/bzip2 application/x-bzip2 +application/cdr application/vnd.corel-draw +application/coreldraw application/vnd.corel-draw +application/dbase application/vnd.dbf +application/dbf application/vnd.dbf +application/docbook+xml application/x-docbook+xml +application/emf image/emf +application/font-woff font/woff +application/futuresplash application/vnd.adobe.flash.movie +application/gpx application/gpx+xml +application/ico image/vnd.microsoft.icon +application/ics text/calendar +application/java application/x-java +application/java-byte-code application/x-java +application/java-vm application/x-java +application/javascript text/javascript +application/lotus123 application/vnd.lotus-1-2-3 +application/m3u audio/x-mpegurl +application/mdb application/vnd.ms-access +application/ms-tnef application/vnd.ms-tnef +application/msaccess application/vnd.ms-access +application/msexcel application/vnd.ms-excel +application/mspowerpoint application/vnd.ms-powerpoint +application/nappdf application/pdf +application/pcap application/vnd.tcpdump.pcap +application/pgp application/pgp-encrypted +application/photoshop image/vnd.adobe.photoshop +application/pkcs12 application/x-pkcs12 +application/pls audio/x-scpls +application/powerpoint application/vnd.ms-powerpoint +application/smil application/smil+xml +application/stuffit application/x-stuffit +application/tga image/x-tga +application/vnd.adobe.illustrator application/illustrator +application/vnd.geo+json application/geo+json +application/vnd.haansoft-hwp application/x-hwp +application/vnd.haansoft-hwt application/x-hwt +application/vnd.ms-3mfdocument model/3mf +application/vnd.ms-word application/msword +application/vnd.msaccess application/vnd.ms-access +application/vnd.oasis.docbook+xml application/x-docbook+xml +application/vnd.rn-realmedia-vbr application/vnd.rn-realmedia +application/vnd.sdp application/sdp +application/vnd.stardivision.writer-global application/vnd.stardivision.writer +application/vnd.sun.xml.base application/vnd.oasis.opendocument.database +application/vnd.truedoc application/font-tdpfr +application/vnd.xdgapp application/vnd.flatpak +application/vnd.youtube.yt video/vnd.youtube.yt +application/wk1 application/vnd.lotus-1-2-3 +application/wmf image/wmf +application/wordperfect application/vnd.wordperfect +application/wwf application/x-wwf +application/x-123 application/vnd.lotus-1-2-3 +application/x-annodex application/annodex +application/x-bzip application/x-bzip2 +application/x-bzip-compressed-tar application/x-bzip2-compressed-tar +application/x-cbr application/vnd.comicbook-rar +application/x-cbz application/vnd.comicbook+zip +application/x-cd-image application/vnd.efi.iso +application/x-cdr application/vnd.corel-draw +application/x-chess-pgn application/vnd.chess-pgn +application/x-chm application/vnd.ms-htmlhelp +application/x-coreldraw application/vnd.corel-draw +application/x-dbase application/vnd.dbf +application/x-dbf application/vnd.dbf +application/x-deb application/vnd.debian.binary-package +application/x-debian-package application/vnd.debian.binary-package +application/x-emf image/emf +application/x-fd-file application/x-raw-floppy-disk-image +application/x-fictionbook application/x-fictionbook+xml +application/x-flash-video video/x-flv +application/x-font-otf font/otf +application/x-font-ttf font/ttf +application/x-frame application/vnd.framemaker +application/x-gamecube-iso-image application/x-gamecube-rom +application/x-gedcom text/vnd.familysearch.gedcom +application/x-gerber application/vnd.gerber +application/x-gettext text/x-gettext-translation +application/x-gnome-app-info application/x-desktop +application/x-gpx application/gpx+xml +application/x-gpx+xml application/gpx+xml +application/x-gtar application/x-tar +application/x-gzip application/gzip +application/x-hfe-file application/x-hfe-floppy-image +application/x-iso9660-image application/vnd.efi.iso +application/x-iwork-keynote-sffkey application/vnd.apple.keynote +application/x-iwork-numbers-sffnumbers application/vnd.apple.numbers +application/x-iwork-pages-sffpages application/vnd.apple.pages +application/x-jar application/java-archive +application/x-java-archive application/java-archive +application/x-java-class application/x-java +application/x-java-vm application/x-java +application/x-javascript text/javascript +application/x-kexiproject-sqlite application/x-kexiproject-sqlite3 +application/x-linguist text/vnd.trolltech.linguist +application/x-lotus123 application/vnd.lotus-1-2-3 +application/x-lzh-compressed application/x-lha +application/x-mathematica application/mathematica +application/x-mdb application/vnd.ms-access +application/x-mobi8-ebook application/vnd.amazon.mobi8-ebook +application/x-mobipocket-subscription-magazine application/x-mobipocket-subscription +application/x-ms-asx audio/x-ms-asx +application/x-ms-dos-executable application/x-msdownload +application/x-msaccess application/vnd.ms-access +application/x-msexcel application/vnd.ms-excel +application/x-msmetafile image/wmf +application/x-mspowerpoint application/vnd.ms-powerpoint +application/x-msword application/msword +application/x-netscape-bookmarks application/x-mozilla-bookmarks +application/x-ogg application/ogg +application/x-palm-database application/vnd.palm +application/x-pcap application/vnd.tcpdump.pcap +application/x-pdf application/pdf +application/x-photoshop image/vnd.adobe.photoshop +application/x-pkcs12 application/pkcs12 +application/x-pkcs7-certificates application/pkcs7-mime +application/x-quicktimeplayer application/x-quicktime-media-link +application/x-rar application/vnd.rar +application/x-rar-compressed application/vnd.rar +application/x-raw-disk-image application/vnd.efi.img +application/x-redhat-package-manager application/x-rpm +application/x-reject text/x-reject +application/x-rnc application/relax-ng-compact-syntax +application/x-sap-file application/x-thomson-sap-image +application/x-sdp application/sdp +application/x-shockwave-flash application/vnd.adobe.flash.movie +application/x-sit application/x-stuffit +application/x-sitx application/x-stuffitx +application/x-smaf application/vnd.smaf +application/x-snes-rom application/vnd.nintendo.snes.rom +application/x-spss-savefile application/x-spss-sav +application/x-sqlite3 application/vnd.sqlite3 +application/x-srt application/x-subrip +application/x-targa image/x-tga +application/x-tex text/x-tex +application/x-tga image/x-tga +application/x-trig application/trig +application/x-troff text/troff +application/x-virtualbox-ova application/ovf +application/x-virtualbox-vdi application/x-vdi-disk +application/x-virtualbox-vhd application/x-vhd-disk +application/x-virtualbox-vhdx application/x-vhdx-disk +application/x-virtualbox-vmdk application/x-vmdk-disk +application/x-vnd.kde.kexi application/x-kexiproject-sqlite3 +application/x-wbfs application/x-wii-rom +application/x-wia application/x-wii-rom +application/x-wii-iso-image application/x-wii-rom +application/x-win-lnk application/x-ms-shortcut +application/x-wmf image/wmf +application/x-wordperfect application/vnd.wordperfect +application/x-x509-ca-cert application/pkix-cert +application/x-x509-user-cert application/pkix-cert +application/x-xliff application/xliff+xml +application/x-xspf+xml application/xspf+xml +application/x-yaml application/yaml +application/x-zip application/zip +application/x-zip-compressed application/zip +application/xps application/vnd.ms-xpsdocument +audio/3gpp video/3gpp +audio/3gpp-encrypted video/3gpp +audio/3gpp2 video/3gpp2 +audio/amr-encrypted audio/AMR +audio/amr-wb-encrypted audio/AMR-WB +audio/dff audio/x-dff +audio/dsd audio/x-dsf +audio/dsf audio/x-dsf +audio/iMelody text/x-iMelody +audio/m3u audio/x-mpegurl +audio/m4a audio/mp4 +audio/mp3 audio/mpeg +audio/mpegurl audio/x-mpegurl +audio/scpls audio/x-scpls +audio/tta audio/x-tta +audio/vnd.audible audio/x-pn-audibleaudio +audio/vnd.m-realaudio audio/vnd.rn-realaudio +audio/vnd.nokia.mobile-xmf audio/mobile-xmf +audio/vorbis audio/x-vorbis+ogg +audio/wav audio/vnd.wave +audio/wma audio/x-ms-wma +audio/x-aac audio/aac +audio/x-aiffc audio/x-aifc +audio/x-annodex audio/annodex +audio/x-dsd audio/x-dsf +audio/x-dts audio/vnd.dts +audio/x-dtshd audio/vnd.dts.hd +audio/x-flac audio/flac +audio/x-iMelody text/x-iMelody +audio/x-m3u audio/x-mpegurl +audio/x-m4a audio/mp4 +audio/x-midi audio/midi +audio/x-mp2 audio/mp2 +audio/x-mp3 audio/mpeg +audio/x-mp3-playlist audio/x-mpegurl +audio/x-mpeg audio/mpeg +audio/x-mpg audio/mpeg +audio/x-ogg audio/ogg +audio/x-oggflac audio/x-flac+ogg +audio/x-pn-realaudio audio/vnd.rn-realaudio +audio/x-rn-3gpp-amr video/3gpp +audio/x-rn-3gpp-amr-encrypted video/3gpp +audio/x-rn-3gpp-amr-wb video/3gpp +audio/x-rn-3gpp-amr-wb-encrypted video/3gpp +audio/x-shorten application/x-shorten +audio/x-vorbis audio/x-vorbis+ogg +audio/x-wav audio/vnd.wave +audio/xmf audio/x-xmf +flv-application/octet-stream video/x-flv +image/avif-sequence image/avif +image/cdr application/vnd.corel-draw +image/fax-g3 image/g3fax +image/fits application/fits +image/heic image/heif +image/heic-sequence image/heif +image/heif-sequence image/heif +image/ico image/vnd.microsoft.icon +image/icon image/vnd.microsoft.icon +image/jpeg2000 image/jp2 +image/jpeg2000-image image/jp2 +image/pdf application/pdf +image/photoshop image/vnd.adobe.photoshop +image/pjpeg image/jpeg +image/psd image/vnd.adobe.photoshop +image/targa image/x-tga +image/tga image/x-tga +image/vnd.mozilla.apng image/apng +image/vnd.ms-photo image/jxr +image/x-MS-bmp image/bmp +image/x-bmp image/bmp +image/x-cdr application/vnd.corel-draw +image/x-djvu image/vnd.djvu +image/x-emf image/emf +image/x-fits application/fits +image/x-icb image/x-tga +image/x-ico image/vnd.microsoft.icon +image/x-icon image/vnd.microsoft.icon +image/x-iff image/x-ilbm +image/x-jpeg2000-image image/jp2 +image/x-panasonic-raw image/x-panasonic-rw +image/x-panasonic-raw2 image/x-panasonic-rw2 +image/x-pcx image/vnd.zbrush.pcx +image/x-photoshop image/vnd.adobe.photoshop +image/x-psd image/vnd.adobe.photoshop +image/x-targa image/x-tga +image/x-win-metafile image/wmf +image/x-wmf image/wmf +image/x-xpm image/x-xpixmap +image/x.djvu image/vnd.djvu +model/x.stl-ascii model/stl +model/x.stl-binary model/stl +text/crystal text/x-crystal +text/directory text/vcard +text/ecmascript application/ecmascript +text/gedcom text/vnd.familysearch.gedcom +text/google-video-pointer text/x-google-video-pointer +text/ico image/vnd.microsoft.icon +text/jscript text/javascript +text/mathml application/mathml+xml +text/rdf application/rdf+xml +text/rss application/rss+xml +text/rtf application/rtf +text/vbs text/vbscript +text/vnd.qt.linguist text/vnd.trolltech.linguist +text/x-c text/x-csrc +text/x-comma-separated-values text/csv +text/x-csv text/csv +text/x-dart application/vnd.dart +text/x-diff text/x-patch +text/x-dtd application/xml-dtd +text/x-fish application/x-fishscript +text/x-lyx application/x-lyx +text/x-markdown text/markdown +text/x-nu application/x-nuscript +text/x-octave text/x-matlab +text/x-opml text/x-opml+xml +text/x-perl application/x-perl +text/x-po text/x-gettext-translation +text/x-pot text/x-gettext-translation-template +text/x-sh application/x-shellscript +text/x-sql application/sql +text/x-tcl text/tcl +text/x-troff text/troff +text/x-vcalendar text/calendar +text/x-vcard text/vcard +text/x-yaml application/yaml +text/xml application/xml +text/xml-external-parsed-entity application/xml-external-parsed-entity +text/yaml application/yaml +video/3gp video/3gpp +video/3gpp-encrypted video/3gpp +video/avi video/vnd.avi +video/divx video/vnd.avi +video/fli video/x-flic +video/flv video/x-flv +video/mp4v-es video/mp4 +video/mpeg-system video/mpeg +video/msvideo video/vnd.avi +video/vivo video/vnd.vivo +video/vnd.divx video/vnd.avi +video/x-annodex video/annodex +video/x-avi video/vnd.avi +video/x-fli video/x-flic +video/x-m4v video/mp4 +video/x-mpeg video/mpeg +video/x-mpeg-system video/mpeg +video/x-mpeg2 video/mpeg +video/x-mpegurl video/vnd.mpegurl +video/x-ms-asf application/vnd.ms-asf +video/x-ms-asf-plugin application/vnd.ms-asf +video/x-ms-wax audio/x-ms-asx +video/x-ms-wm application/vnd.ms-asf +video/x-ms-wmx audio/x-ms-asx +video/x-ms-wvx audio/x-ms-asx +video/x-msvideo video/vnd.avi +video/x-ogg video/ogg +video/x-ogm video/x-ogm+ogg +video/x-real-video video/vnd.rn-realvideo +video/x-theora video/x-theora+ogg +x-directory/normal inode/directory +zz-application/zz-winassoc-123 application/vnd.lotus-1-2-3 +zz-application/zz-winassoc-cab application/vnd.ms-cab-compressed +zz-application/zz-winassoc-cdr application/vnd.corel-draw +zz-application/zz-winassoc-doc application/msword +zz-application/zz-winassoc-hlp application/winhlp +zz-application/zz-winassoc-mdb application/vnd.ms-access +zz-application/zz-winassoc-uu text/x-uuencode +zz-application/zz-winassoc-xls application/vnd.ms-excel diff --git a/etc/mimetype/magic b/etc/mimetype/magic new file mode 100644 index 000000000000..598dfe75fe35 Binary files /dev/null and b/etc/mimetype/magic differ diff --git a/etc/mimetype/subclasses b/etc/mimetype/subclasses new file mode 100644 index 000000000000..d9b74e8f39b7 --- /dev/null +++ b/etc/mimetype/subclasses @@ -0,0 +1,516 @@ +model/vrml text/plain +text/x-kaitai-struct application/yaml +application/msix application/zip +application/vnd.ms-officetheme application/zip +text/x-gradle text/x-groovy +application/x-lyx text/plain +application/vnd.oasis.opendocument.spreadsheet-template application/zip +application/xhtml+xml application/xml +application/vnd.openxmlformats-officedocument.wordprocessingml.document application/zip +text/x-credits text/plain +application/x-raw-floppy-disk-image application/vnd.efi.img +application/vnd.oasis.opendocument.text-web application/zip +text/x-gettext-translation text/plain +application/metalink+xml application/xml +text/x-uil text/plain +application/gpx+xml application/xml +text/x-basic text/plain +application/vnd.oasis.opendocument.chart application/zip +application/vnd.sun.xml.draw.template application/zip +video/annodex application/annodex +application/pkcs12+pem application/x-pem-file +application/x-netshow-channel application/vnd.ms-asf +application/xliff+xml application/xml +application/vnd.visio application/x-ole-storage +application/x-modrinth-modpack+zip application/zip +application/vnd.oasis.opendocument.text application/zip +application/vnd.oasis.opendocument.chart-template application/zip +text/vnd.wap.wml application/xml +application/owl+xml application/xml +application/vnd.sun.xml.writer.global application/zip +application/x-nuscript application/x-executable +application/x-nuscript text/plain +text/x-vb text/plain +application/x-kindle-application application/x-java-archive +application/x-audacity-project text/xml +application/vnd.ms-excel.addin.macroEnabled.12 application/vnd.openxmlformats-officedocument.spreadsheetml.sheet +application/vnd.ms-excel.sheet.binary.macroEnabled.12 application/vnd.openxmlformats-officedocument.spreadsheetml.sheet +application/x-wais-source text/plain +image/x-sony-arw image/x-dcraw +image/x-sony-arw image/tiff +application/vnd.ms-visio.template.macroEnabled.main+xml application/zip +image/x-portable-bitmap image/x-portable-anymap +application/x-designer application/xml +image/x-sony-srf image/x-dcraw +image/x-sony-srf image/tiff +text/x-cobol text/plain +audio/x-vorbis+ogg audio/ogg +application/x-magicpoint text/plain +application/x-awk application/x-executable +application/x-awk text/plain +application/vnd.apple.pages application/zip +application/x-openvpn-profile text/plain +text/vnd.senx.warpscript text/plain +application/x-bzip3-compressed-tar application/x-bzip3 +application/x-bat text/plain +application/x-fictionbook+xml application/xml +text/x-mrml application/xml +application/x-csh application/x-shellscript +application/x-csh text/plain +application/java-archive application/zip +text/markdown text/plain +image/apng image/png +application/x-tarz application/x-compress +text/tab-separated-values text/plain +text/x-dsl text/plain +application/x-aportisdoc application/vnd.palm +audio/x-opus+ogg audio/ogg +application/x-apple-systemprofiler+xml application/xml +text/x-patch text/plain +application/x-tzo application/x-lzop +application/vnd.openxmlformats-officedocument.presentationml.slide application/zip +application/epub+zip application/zip +text/x-xslfo application/xml +application/x-ipynb+json application/json +text/x-scala text/plain +application/vnd.ms-powerpoint.slide.macroEnabled.12 application/vnd.openxmlformats-officedocument.presentationml.slide +text/x-verilog text/plain +application/msword application/x-ole-storage +text/x-authors text/plain +text/x-subviewer text/plain +image/openraster application/zip +application/vnd.oasis.opendocument.presentation-flat-xml application/xml +application/vnd.oasis.opendocument.presentation-flat-xml application/xml +text/x-csharp text/x-csrc +application/x-cdrdao-toc text/plain +application/x-wwf application/pdf +audio/vnd.wave application/x-riff +text/x-vhdl text/plain +application/vnd.oasis.opendocument.formula application/zip +application/vnd.ms-xpsdocument application/zip +text/x-fortran text/plain +text/html text/plain +application/x-dosexec application/x-msdownload +application/vnd.gerber text/plain +text/x-opml+xml application/xml +application/ovf application/x-tar +application/x-compressed-tar application/gzip +text/x-genie text/plain +application/appx application/zip +application/vnd.ms-powerpoint.slideshow.macroEnabled.12 application/vnd.openxmlformats-officedocument.presentationml.slideshow +text/x-ldif text/plain +image/x-kodak-kdc image/x-dcraw +image/x-kodak-kdc image/tiff +application/vnd.oasis.opendocument.formula-template application/zip +text/x-blueprint text/plain +text/vnd.trolltech.linguist application/xml +application/x-perl application/x-executable +application/x-perl text/plain +video/x-ms-wmv application/vnd.ms-asf +text/x-uri text/plain +application/x-gzpostscript application/gzip +application/x-cbt application/x-tar +text/x-matlab text/plain +video/x-ogm+ogg video/ogg +application/xslt+xml application/xml +text/x-moc text/plain +text/cache-manifest text/plain +text/x-uuencode text/plain +application/pkcs7-mime+pem application/x-pem-file +text/x-java text/x-csrc +text/x-iptables text/plain +message/partial text/plain +application/schema+json application/json +text/x-install text/plain +text/x-todo-txt text/plain +text/x-systemd-unit text/plain +application/x-desktop text/plain +text/vnd.graphviz text/plain +text/xmcd text/plain +application/postscript text/plain +video/vnd.mpegurl text/plain +application/x-windows-themepack application/vnd.ms-cab-compressed +text/x-makefile text/plain +application/yaml text/plain +text/x-mof text/x-csrc +application/x-zip-compressed-fb2 application/zip +application/x-qtiplot text/plain +application/vnd.ms-visio.stencil.macroEnabled.main+xml application/zip +application/vnd.ms-powerpoint.template.macroEnabled.12 application/vnd.openxmlformats-officedocument.presentationml.template +chemical/x-pdb text/plain +text/x-crystal text/plain +text/vnd.rn-realtext text/plain +application/gml+xml application/xml +text/richtext text/plain +application/ecmascript text/javascript +application/mac-binhex40 text/plain +application/x-ms-ne-executable application/x-msdownload +text/x-elixir text/plain +image/x-canon-crw image/x-dcraw +application/vnd.efi.iso application/vnd.efi.img +text/vnd.familysearch.gedcom text/plain +application/x-audacity-project+sqlite3 application/vdn.sqlite3 +application/vnd.oasis.opendocument.database application/zip +text/css text/plain +x-content/win32-software x-content/software +text/x-nimscript text/x-nim +text/x-gherkin text/plain +image/x-adobe-dng image/x-dcraw +image/x-adobe-dng image/tiff +application/vnd.chess-pgn text/plain +image/x-bzeps application/x-bzip2 +application/x-troff-man text/plain +text/rfc822-headers text/plain +text/x-scons text/x-python +text/csv text/plain +text/x-groovy text/x-csrc +application/x-java-jnlp-file application/xml +image/x-panasonic-rw image/x-dcraw +audio/x-minipsf audio/x-psf +audio/x-aifc application/x-iff +application/vnd.openxmlformats-officedocument.spreadsheetml.template application/zip +application/vnd.ms-excel application/x-ole-storage +application/vnd.openxmlformats-officedocument.spreadsheetml.sheet application/zip +application/trig text/plain +text/x-maven+xml application/xml +text/vbscript text/plain +application/x-ufraw application/xml +application/vnd.ms-excel.template.macroEnabled.12 application/vnd.openxmlformats-officedocument.spreadsheetml.template +application/vnd.sun.xml.writer.template application/zip +application/x-go-sgf text/plain +application/sparql-results+xml application/xml +text/x-ocaml text/plain +audio/x-aiff application/x-iff +inode/mount-point inode/directory +application/x-m4 text/plain +text/x-objc++src text/x-c++src +text/x-objc++src text/x-objcsrc +application/vnd.ms-visio.drawing.macroEnabled.main+xml application/zip +message/rfc822 text/plain +application/vnd.cups-ppd text/plain +text/x-google-video-pointer text/plain +application/relax-ng-compact-syntax text/plain +image/x-pentax-pef image/x-dcraw +image/x-pentax-pef image/tiff +application/vnd.ms-word.template.macroEnabled.12 application/vnd.openxmlformats-officedocument.wordprocessingml.template +application/x-lzip-compressed-tar application/x-lzip +application/x-glade application/xml +audio/x-matroska application/x-matroska +text/x-troff-me text/plain +application/x-docbook+xml application/xml +application/mathematica text/plain +image/svg+xml application/xml +text/vcard text/plain +application/pkcs8+pem application/x-pem-file +image/x-ilbm application/x-iff +video/x-mjpeg image/jpeg +application/its+xml application/xml +application/vnd.apple.mpegurl text/plain +application/x-mswinurl text/plain +image/x-gzeps application/gzip +application/x-bzdvi application/x-bzip2 +text/x-scss text/plain +application/rdf+xml application/xml +application/jrd+json application/json +text/troff text/plain +application/pkcs10+pem application/x-pem-file +application/vnd.comicbook-rar application/vnd.rar +application/sql text/plain +audio/x-ms-wma application/vnd.ms-asf +application/vnd.android.package-archive application/java-archive +application/x-pyspread-bz-spreadsheet application/x-bzip2 +application/x-zstd-compressed-tar application/zstd +application/vnd.sun.xml.writer application/zip +image/x-portable-pixmap image/x-portable-anymap +text/x-emacs-lisp text/plain +application/x-theme application/x-desktop +application/vnd.openxmlformats-officedocument.presentationml.slideshow application/zip +image/x-fuji-raf image/x-dcraw +image/x-nikon-nrw image/x-dcraw +image/x-nikon-nrw image/tiff +application/vnd.sun.xml.impress application/zip +text/x-reject text/plain +text/x-bibtex text/plain +application/vnd.oasis.opendocument.text-flat-xml application/xml +application/vnd.oasis.opendocument.text-flat-xml application/xml +application/x-subrip text/plain +text/x-dcl text/plain +text/x-sagemath text/x-python +application/x-pem-key application/x-pem-file +application/x-markaby application/x-ruby +application/x-msi application/x-ole-storage +text/x-troff-mm text/troff +application/vnd.oasis.opendocument.graphics-flat-xml application/xml +application/vnd.oasis.opendocument.graphics-flat-xml application/xml +text/x-rpm-spec text/plain +application/vnd.openxmlformats-officedocument.wordprocessingml.template application/zip +text/x-idl text/plain +application/vnd.oasis.opendocument.spreadsheet application/zip +application/oxps application/zip +application/x-godot-shader text/plain +application/x-tiled-tmx application/xml +application/json5 text/javascript +application/mathml+xml application/xml +text/x-rst text/plain +application/x-gzdvi application/gzip +audio/x-m4b audio/mp4 +text/x-ms-regedit text/plain +text/x-mup text/plain +application/x-kexiproject-sqlite2 application/x-sqlite2 +audio/x-flac+ogg audio/ogg +application/x-gnuplot text/plain +application/x-bzip2-compressed-tar application/x-bzip2 +application/x-kexiproject-sqlite3 application/vnd.sqlite3 +text/x-ocl text/plain +audio/ogg application/ogg +text/x-readme text/plain +text/x-pascal text/plain +image/svg+xml-compressed application/gzip +application/x-font-type1 application/postscript +application/json application/json5 +text/x-erlang text/plain +text/x-cmake text/plain +text/x-log text/plain +text/x-troff-ms text/plain +text/x-changelog text/plain +application/vnd.openofficeorg.extension application/zip +text/x-qml text/plain +application/x-pagemaker application/x-ole-storage +text/x-haskell text/plain +text/x-typst text/plain +application/vnd.google-earth.kmz application/zip +image/x-sony-sr2 image/x-dcraw +image/x-sony-sr2 image/tiff +text/x-common-lisp text/plain +image/x-canon-cr2 image/x-dcraw +image/x-canon-cr2 image/tiff +application/vnd.ms-word.document.macroEnabled.12 application/vnd.openxmlformats-officedocument.wordprocessingml.document +text/x-twig text/plain +audio/vnd.dts.hd audio/vnd.dts +application/vnd.oasis.opendocument.spreadsheet-flat-xml application/xml +application/vnd.oasis.opendocument.spreadsheet-flat-xml application/xml +image/x-canon-cr3 image/x-dcraw +application/vnd.ms-visio.template.main+xml application/zip +text/vnd.wap.wmlscript text/plain +application/x-pem-file text/plain +application/x-dia-diagram application/xml +application/x-gz-font-linux-psf application/gzip +application/x-cb7 application/x-7z-compressed +video/ogg application/ogg +application/x-shared-library-la text/plain +text/julia text/plain +text/vtt text/plain +application/x-cpio-compressed application/gzip +application/x-mobipocket-subscription application/x-mobipocket-ebook +application/vnd.sun.xml.math application/zip +text/x-microdvd text/plain +application/metalink4+xml application/xml +application/vnd.oasis.opendocument.presentation application/zip +text/org text/plain +text/x-lua application/x-executable +text/x-lua text/plain +text/sgml text/plain +application/xspf+xml application/xml +application/x-lz4-compressed-tar application/x-lz4 +text/x-copying text/plain +application/x-lzma-compressed-tar application/x-lzma +application/x-nzb application/xml +application/vnd.oasis.opendocument.text-master application/zip +application/x-tiled-tsx application/xml +application/vnd.dart text/plain +application/toml text/plain +text/x-eiffel text/plain +application/vnd.ms-excel.sheet.macroEnabled.12 application/vnd.openxmlformats-officedocument.spreadsheetml.sheet +image/x-olympus-orf image/x-dcraw +text/x-iMelody text/plain +application/atom+xml application/xml +text/x-xmi application/xml +application/vnd.coffeescript text/plain +text/x-literate-haskell text/plain +audio/annodex application/annodex +image/x-nikon-nef image/x-dcraw +image/x-nikon-nef image/tiff +application/x-shellscript application/x-executable +application/x-shellscript text/plain +application/vnd.ms-visio.stencil.main+xml application/zip +text/htmlh text/plain +application/x-raw-disk-image-xz-compressed application/x-xz +application/vnd.apple.numbers application/zip +video/x-matroska-3d application/x-matroska +application/x-msdownload application/x-executable +application/xml text/plain +application/vnd.oasis.opendocument.image application/zip +text/x-setext text/plain +text/vbscript.encode application/x-executable +text/vbscript.encode text/plain +application/xml-external-parsed-entity application/xml +model/gltf+json application/json +audio/x-m4r video/mp4 +image/x-sigma-x3f image/x-dcraw +application/vnd.oasis.opendocument.presentation-template application/zip +application/vnd.ms-works application/x-ole-storage +audio/webm video/webm +text/javascript application/x-executable +text/javascript text/plain +application/vnd.microsoft.windows.thumbnail-cache application/x-ole-storage +application/msword-template application/msword +application/x-godot-project text/plain +text/x-mpl2 text/plain +application/x-dia-shape application/xml +application/rtf text/plain +text/x-ooc text/x-csrc +application/x-excellon text/plain +application/x-mobipocket-ebook application/x-palm-database +application/x-mobipocket-ebook application/vnd.palm +application/vnd.amazon.mobi8-ebook application/x-mobipocket-ebook +model/3mf application/zip +text/x-dbus-service text/plain +application/x-source-rpm application/x-rpm +application/x-font-ttx application/xml +text/x-c++src text/x-csrc +application/vnd.appimage application/x-executable +application/vnd.appimage application/vnd.squashfs +application/x-profile text/plain +application/vnd.snap application/vnd.squashfs +application/mbox text/plain +image/x-eps application/postscript +application/pkix-crl+pem application/x-pem-file +application/x-abiword application/xml +application/vnd.ms-powerpoint.presentation.macroEnabled.12 application/vnd.openxmlformats-officedocument.presentationml.presentation +text/calendar text/plain +text/x-gettext-translation-template text/plain +application/x-powershell text/plain +model/mtl text/plain +video/x-theora+ogg video/ogg +application/x-ruby application/x-executable +application/x-ruby text/plain +application/rss+xml application/xml +application/vnd.comicbook+zip application/zip +application/x-quicktime-media-link video/quicktime +application/x-mimearchive multipart/related +application/vnd.mozilla.xul+xml application/xml +application/x-asp text/plain +application/x-xzpdf application/x-xz +message/delivery-status text/plain +audio/x-speex+ogg audio/ogg +application/ld+json application/json +application/x-bzpdf application/x-bzip2 +application/vnd.ms-powerpoint application/x-ole-storage +application/x-ica text/plain +application/vnd.oasis.opendocument.graphics application/zip +text/x-kotlin text/plain +image/x-portable-graymap image/x-portable-anymap +application/x-it87 text/plain +application/vnd.ms-publisher application/x-ole-storage +text/x-csrc text/plain +application/raml+yaml application/yaml +application/appxbundle application/zip +model/obj text/plain +image/vnd.djvu+multipage image/vnd.djvu +text/x-vala text/x-csrc +application/x-bzip1-compressed-tar application/x-bzip1 +text/x-opencl-src text/x-csrc +application/x-fishscript application/x-executable +application/x-fishscript text/plain +text/x.gcode text/plain +application/vnd.google-earth.kml+xml application/xml +audio/x-mpegurl text/plain +application/json-patch+json application/json +text/x-python3 text/x-python +application/x-xbel application/xml +application/x-ccmx text/plain +application/x-cue text/plain +text/x-lilypond text/plain +text/x-c++hdr text/x-chdr +application/x-nautilus-link text/plain +application/x-php text/plain +application/vnd.microsoft.portable-executable application/x-msdownload +application/msixbundle application/zip +text/enriched text/plain +x-content/unix-software x-content/software +application/pkix-cert+pem application/x-pem-file +application/sdp text/plain +text/x-texinfo text/plain +text/x-scheme text/plain +application/x-gerber-job application/json +text/x-modelica text/plain +application/x-gzpdf application/gzip +text/x-devicetree-source text/plain +application/xml-dtd text/plain +text/x-svsrc text/x-verilog +application/vnd.sun.xml.draw application/zip +text/x-adasrc text/plain +application/geo+json application/json +application/vnd.ms-visio.drawing.main+xml application/zip +text/x-apt-sources-list text/plain +video/vnd.youtube.yt application/zip +application/vnd.openxmlformats-officedocument.presentationml.template application/zip +application/x-lrzip-compressed-tar application/x-lrzip +application/vnd.apple.pkpass application/zip +text/x-nfo text/x-readme +image/x-kodak-dcr image/x-dcraw +image/x-kodak-dcr image/tiff +application/x-iso9660-appimage application/x-executable +application/x-iso9660-appimage application/vnd.efi.iso +text/x-go text/plain +application/smil+xml application/xml +application/x-mobi8-ebook application/x-palm-database +text/x-chdr text/x-csrc +text/x-objcsrc text/x-csrc +model/iges text/plain +text/rust text/plain +application/appinstaller application/xml +application/vnd.flatpak.ref text/plain +application/x-xpinstall application/zip +text/vnd.sun.j2me.app-descriptor text/plain +application/x-spkac+base64 text/plain +application/pkcs7-signature text/plain +video/x-javafx video/x-flv +application/x-eris-link+cbor application/cbor +application/vnd.sun.xml.calc application/zip +application/vnd.sun.xml.impress.template application/zip +text/x-nim text/plain +application/vnd.flatpak.repo text/plain +application/vnd.oasis.opendocument.text-template application/zip +application/vnd.openxmlformats-officedocument.presentationml.presentation application/zip +text/x-txt2tags text/plain +image/x-minolta-mrw image/x-dcraw +text/jscript.encode application/x-executable +image/x-kodak-k25 image/x-dcraw +image/x-kodak-k25 image/tiff +application/x-lzpdf application/x-lzip +text/x-sass text/plain +application/x-cisco-vpn-settings text/plain +text/x-mpsub text/plain +application/vnd.oasis.opendocument.graphics-template application/zip +application/vnd.sun.xml.calc.template application/zip +text/x-dsrc text/x-csrc +image/x-tiff-multipage image/tiff +audio/x-psflib audio/x-psf +message/news text/plain +image/x-panasonic-rw2 image/x-dcraw +application/x-mozilla-bookmarks text/html +message/disposition-notification text/plain +font/otf font/ttf +video/3gpp video/mp4 +text/turtle text/plain +application/vnd.apple.keynote application/zip +text/x-svhdr text/x-verilog +text/tcl text/plain +application/x-fluid text/plain +text/x-python application/x-executable +text/x-python text/plain +application/x-gtk-builder application/xml +text/spreadsheet text/plain +text/csv-schema text/plain +text/x-component application/xml +application/x-sami text/plain +text/x-meson text/plain +application/x-bzpostscript application/x-bzip2 +application/x-gd-rom-cue text/plain +application/x-gdscript text/plain +text/x-ssa text/plain +video/x-matroska application/x-matroska +application/x-xz-compressed-tar application/x-xz +text/x-tex text/plain +application/vnd.squashfs application/vnd.efi.img diff --git a/etc/schema.json b/etc/schema.json index 7e847f8a6e32..aaba287192b0 100644 --- a/etc/schema.json +++ b/etc/schema.json @@ -1729,6 +1729,10 @@ "md5": { "type": "string" }, + "mimetype": { + "type": "string", + "description": "The MIME type of the file (e.g., application/pdf, image/png, etc.)" + }, "sha1": { "type": "string" }, @@ -1788,6 +1792,10 @@ "md5": { "type": "string" }, + "mimetype": { + "type": "string", + "description": "The MIME type of the file (e.g., application/pdf, image/png, etc.)" + }, "sha1": { "type": "string" }, diff --git a/rust/Cargo.lock.in b/rust/Cargo.lock.in index 933fbe72e502..a68cc4f08ae2 100644 --- a/rust/Cargo.lock.in +++ b/rust/Cargo.lock.in @@ -480,6 +480,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + [[package]] name = "flate2" version = "1.0.35" @@ -1046,6 +1052,16 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "petgraph" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" +dependencies = [ + "fixedbitset", + "indexmap", +] + [[package]] name = "phf" version = "0.10.1" @@ -1552,6 +1568,7 @@ dependencies = [ "test-case", "time", "tls-parser", + "tree_magic_mini", "uuid", "widestring", "x509-parser", @@ -1861,6 +1878,25 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "tree_magic_db" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd30f22e7532ed0d3d846e24841a132c8dcb779f5b497bda82d904aa04755375" + +[[package]] +name = "tree_magic_mini" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f943391d896cdfe8eec03a04d7110332d445be7df856db382dd96a730667562c" +dependencies = [ + "memchr", + "nom", + "once_cell", + "petgraph", + "tree_magic_db", +] + [[package]] name = "typenum" version = "1.18.0" diff --git a/rust/Cargo.toml.in b/rust/Cargo.toml.in index 4d656df7f6b9..af516c21b492 100644 --- a/rust/Cargo.toml.in +++ b/rust/Cargo.toml.in @@ -35,6 +35,7 @@ debug = [] debug-validate = [] ja3 = [] ja4 = [] +filemimetype = ["dep:tree_magic_mini"] [dependencies] nom7 = { version="7.1", package="nom" } @@ -58,6 +59,8 @@ lru = "~0.12.5" der-parser = { version = "~9.0.0", default-features = false } kerberos-parser = { version = "~0.8.0", default-features = false } +tree_magic_mini = { version = "~3.2.0", features = [@WITH_GPL_DATA@], optional = true } + sawp-modbus = "~0.13.1" sawp-pop3 = "~0.13.1" sawp = "~0.13.1" diff --git a/rust/Makefile.am b/rust/Makefile.am index 49104786ae7e..b0b87f674d4f 100644 --- a/rust/Makefile.am +++ b/rust/Makefile.am @@ -39,6 +39,10 @@ if DEBUG RUST_FEATURES += debug endif +if HAVE_MIMETYPE +RUST_FEATURES += filemimetype +endif + if DEBUG_VALIDATION RUST_FEATURES += debug-validate endif diff --git a/rust/src/filemimetype.rs b/rust/src/filemimetype.rs new file mode 100644 index 000000000000..29cca7e00f29 --- /dev/null +++ b/rust/src/filemimetype.rs @@ -0,0 +1,26 @@ +/* Copyright (C) 2025 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use crate::common::rust_string_to_c; +use std::os::raw::c_char; + +#[no_mangle] +pub unsafe extern "C" fn SCGetMimetype(input: *const u8, len: u32) -> * mut c_char { + let slice: &[u8] = std::slice::from_raw_parts(input as *mut u8, len as usize); + let result = tree_magic_mini::from_u8(slice); + rust_string_to_c(result.to_string()) +} diff --git a/rust/src/lib.rs b/rust/src/lib.rs index bb7fe6e9fdf6..29f30cb5abe1 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -90,6 +90,8 @@ pub mod applayer; pub mod frames; pub mod filecontainer; pub mod filetracker; +#[cfg(feature = "filemimetype")] +pub mod filemimetype; pub mod kerberos; pub mod detect; pub mod utils; diff --git a/src/Makefile.am b/src/Makefile.am index 5a83ab290d1a..78408afbf0f9 100755 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -160,6 +160,7 @@ noinst_HEADERS = \ detect-fast-pattern.h \ detect-file-data.h \ detect-file-hash-common.h \ + detect-file-mimetype.h \ detect-filemagic.h \ detect-filemd5.h \ detect-filename.h \ @@ -565,6 +566,7 @@ noinst_HEADERS = \ util-memcmp.h \ util-memcpy.h \ util-memrchr.h \ + util-mimetype.h \ util-misc.h \ util-mpm-ac-ks.h \ util-mpm-ac-queue.h \ @@ -761,6 +763,7 @@ libsuricata_c_a_SOURCES = \ detect-fast-pattern.c \ detect-file-data.c \ detect-file-hash-common.c \ + detect-file-mimetype.c \ detect-filemagic.c \ detect-filemd5.c \ detect-filename.c \ @@ -1147,6 +1150,7 @@ libsuricata_c_a_SOURCES = \ util-mem.c \ util-memcmp.c \ util-memrchr.c \ + util-mimetype.c \ util-misc.c \ util-mpm-ac-ks-small.c \ util-mpm-ac-ks.c \ diff --git a/src/app-layer-smtp.c b/src/app-layer-smtp.c index a7b76ec44290..59806a9eb942 100644 --- a/src/app-layer-smtp.c +++ b/src/app-layer-smtp.c @@ -36,6 +36,7 @@ #include "app-layer-smtp.h" #include "util-enum.h" +#include "util-file.h" #include "util-mpm.h" #include "util-debug.h" #include "util-byte.h" @@ -1193,7 +1194,7 @@ static int SMTPProcessRequest( } else if (smtp_config.raw_extraction) { if (FileOpenFileWithId(&tx->files_ts, &smtp_config.sbcfg, state->file_track_id++, (uint8_t *)rawmsgname, strlen(rawmsgname), NULL, 0, - FILE_NOMD5 | FILE_NOMAGIC) == 0) { + FILE_NOMD5 | FILE_NOMAGIC | FILE_NOMIMETYPE) == 0) { SMTPNewFile(tx, tx->files_ts.tail); } } else if (smtp_config.decode_mime) { diff --git a/src/detect-engine-build.c b/src/detect-engine-build.c index f356e5beff8a..033d6073808e 100644 --- a/src/detect-engine-build.c +++ b/src/detect-engine-build.c @@ -128,6 +128,25 @@ int SignatureIsFilemagicInspecting(const Signature *s) return 0; } +/** + * \brief Check if a signature contains the file.mimetype keyword. + * + * \param s signature + * + * \retval 0 no + * \retval 1 yes + */ +int SignatureIsFileMimetypeInspecting(const Signature *s) +{ + if (s == NULL) + return 0; + + if (s->file_flags & FILE_SIG_NEED_MIMETYPE) + return 1; + + return 0; +} + /** * \brief Check if a signature contains the filemd5 keyword. * diff --git a/src/detect-engine-build.h b/src/detect-engine-build.h index bb83682c700b..a497dbd05f9b 100644 --- a/src/detect-engine-build.h +++ b/src/detect-engine-build.h @@ -23,6 +23,7 @@ void PacketCreateMask(Packet *p, SignatureMask *mask, AppProto alproto, int SignatureIsFilestoring(const Signature *); int SignatureIsFilemagicInspecting(const Signature *); +int SignatureIsFileMimetypeInspecting(const Signature *); int SignatureIsFileMd5Inspecting(const Signature *); int SignatureIsFileSha1Inspecting(const Signature *s); int SignatureIsFileSha256Inspecting(const Signature *s); diff --git a/src/detect-engine-file.c b/src/detect-engine-file.c index 26601ce8a96b..ce2cd47a9293 100644 --- a/src/detect-engine-file.c +++ b/src/detect-engine-file.c @@ -92,6 +92,12 @@ static uint8_t DetectFileInspect(DetectEngineThreadCtx *det_ctx, Flow *f, const continue; } + if ((s->file_flags & FILE_SIG_NEED_MIMETYPE) && file_size == 0) { + SCLogDebug("sig needs file content, but we don't have any"); + r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH; + continue; + } + if ((s->file_flags & FILE_SIG_NEED_FILECONTENT) && file_size == 0) { SCLogDebug("sig needs file content, but we don't have any"); r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH; diff --git a/src/detect-engine-register.c b/src/detect-engine-register.c index 81da9fa1ee0f..674d6451a7ce 100644 --- a/src/detect-engine-register.c +++ b/src/detect-engine-register.c @@ -137,6 +137,7 @@ #include "detect-filesha1.h" #include "detect-filesha256.h" #include "detect-filesize.h" +#include "detect-file-mimetype.h" #include "detect-dataset.h" #include "detect-datarep.h" #include "detect-dsize.h" @@ -565,6 +566,7 @@ void SigTableSetup(void) DetectFileSha1Register(); DetectFileSha256Register(); DetectFilesizeRegister(); + DetectFileMimetypeRegister(); DetectHttpUARegister(); DetectHttpHHRegister(); diff --git a/src/detect-engine-register.h b/src/detect-engine-register.h index 0e5e52242c76..3edaa3b10de5 100644 --- a/src/detect-engine-register.h +++ b/src/detect-engine-register.h @@ -235,6 +235,7 @@ enum DetectKeywordId { DETECT_FILESTORE_POSTMATCH, DETECT_FILEMAGIC, DETECT_FILE_MAGIC, + DETECT_FILE_MIMETYPE, DETECT_FILEMD5, DETECT_FILESHA1, DETECT_FILESHA256, diff --git a/src/detect-engine-siggroup.c b/src/detect-engine-siggroup.c index a852e241e07a..c6d5995e4c2e 100644 --- a/src/detect-engine-siggroup.c +++ b/src/detect-engine-siggroup.c @@ -597,6 +597,9 @@ void SigGroupHeadSetupFiles(const DetectEngineCtx *de_ctx, SigGroupHead *sgh) sgh->flags |= SIG_GROUP_HEAD_HAVEFILEMAGIC; } #endif + if (SignatureIsFileMimetypeInspecting(s)) { + sgh->flags |= SIG_GROUP_HEAD_HAVEFILEMIMETYPE; + } if (SignatureIsFilestoring(s)) { // should be insured by caller that we do not overflow DEBUG_VALIDATE_BUG_ON(sgh->filestore_cnt == UINT16_MAX); diff --git a/src/detect-file-mimetype.c b/src/detect-file-mimetype.c new file mode 100644 index 000000000000..be45ab667dcb --- /dev/null +++ b/src/detect-file-mimetype.c @@ -0,0 +1,232 @@ +/* Copyright (C) 2025 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Eric Leblond + * + */ + +#include "suricata-common.h" +#include "detect-file-mimetype.h" +#include "detect-engine.h" +#include "detect-engine-buffer.h" +#include "detect-engine-mpm.h" +#include "detect-engine-prefilter.h" +#include "detect-engine-content-inspection.h" +#include "detect-parse.h" +#include "detect-file-data.h" +#include "rust.h" +#include "util-mimetype.h" +#include "util-profiling.h" +#include "app-layer-parser.h" + +#include "conf.h" + +#ifndef HAVE_MIMETYPE + +static int DetectFileMimetypeSetupNoSupport(DetectEngineCtx *de_ctx, Signature *s, const char *str) +{ + SCLogError("no mimetype support built in, needed for file.mimetype keyword"); + return -1; +} + +/** + * \brief Registration function for keyword: filemagic + */ +void DetectFileMimetypeRegister(void) +{ + sigmatch_table[DETECT_FILE_MIMETYPE].name = "file.mimetype"; + sigmatch_table[DETECT_FILE_MIMETYPE].desc = "sticky buffer to match on file mime type"; + sigmatch_table[DETECT_FILE_MIMETYPE].url = "/rules/file-keywords.html#file_mimetype"; + sigmatch_table[DETECT_FILE_MIMETYPE].Setup = DetectFileMimetypeSetupNoSupport; + sigmatch_table[DETECT_FILE_MIMETYPE].flags = SIGMATCH_NOOPT | SIGMATCH_INFO_STICKY_BUFFER; +} + +#else /* HAVE_MIMETYPE */ + +static int g_file_match_list_id = 0; + +static int DetectFileMimetypeSetup(DetectEngineCtx *de_ctx, Signature *s, const char *str); +static int g_file_mimetype_buffer_id = 0; + +static int PrefilterMpmFileMimetypeRegister(DetectEngineCtx *de_ctx, SigGroupHead *sgh, + MpmCtx *mpm_ctx, const DetectBufferMpmRegistry *mpm_reg, int list_id); +static unsigned char DetectEngineInspectFileMimetype(DetectEngineCtx *de_ctx, + DetectEngineThreadCtx *det_ctx, const DetectEngineAppInspectionEngine *engine, + const Signature *s, Flow *f, uint8_t flags, void *alstate, void *txv, uint64_t tx_id); + +void DetectFileMimetypeRegister(void) +{ + sigmatch_table[DETECT_FILE_MIMETYPE].name = "file.mime_type"; + sigmatch_table[DETECT_FILE_MIMETYPE].desc = "sticky buffer to match on file mime type"; + sigmatch_table[DETECT_FILE_MIMETYPE].url = "/rules/file-keywords.html#file_mimetype"; + sigmatch_table[DETECT_FILE_MIMETYPE].Setup = DetectFileMimetypeSetup; + sigmatch_table[DETECT_FILE_MIMETYPE].flags = SIGMATCH_NOOPT | SIGMATCH_INFO_STICKY_BUFFER; + + filehandler_table[DETECT_FILE_MIMETYPE].name = "file.mime_type", + filehandler_table[DETECT_FILE_MIMETYPE].priority = 2; + filehandler_table[DETECT_FILE_MIMETYPE].PrefilterFn = PrefilterMpmFileMimetypeRegister; + filehandler_table[DETECT_FILE_MIMETYPE].Callback = DetectEngineInspectFileMimetype; + + g_file_match_list_id = DetectBufferTypeRegister("files"); + + DetectBufferTypeSetDescriptionByName("file.mime_type", "file mime_type"); + DetectBufferTypeSupportsMultiInstance("file.mime_type"); + + g_file_mimetype_buffer_id = DetectBufferTypeGetByName("file.mime_type"); + + SCLogDebug("registering file mime type rule option"); +} + +static int DetectFileMimetypeSetup(DetectEngineCtx *de_ctx, Signature *s, const char *str) +{ + if (SCDetectBufferSetActiveList(de_ctx, s, g_file_mimetype_buffer_id) < 0) + return -1; + s->file_flags |= (FILE_SIG_NEED_FILE | FILE_SIG_NEED_MIMETYPE); + return 0; +} + +static InspectionBuffer *FileMimetypeGetDataCallback(DetectEngineThreadCtx *det_ctx, + const DetectEngineTransforms *transforms, Flow *f, uint8_t flow_flags, File *cur_file, + int list_id, int local_file_id, bool first) +{ + SCEnter(); + + InspectionBuffer *buffer = InspectionBufferMultipleForListGet(det_ctx, list_id, local_file_id); + if (buffer == NULL) + return NULL; + if (!first && buffer->inspect != NULL) + return buffer; + + if (cur_file->mimetype == NULL) + FileMimetypeLookup(cur_file); + if (cur_file->mimetype == NULL) + return NULL; + + const uint8_t *data = (uint8_t *)cur_file->mimetype; + uint32_t data_len = (uint32_t)strlen(cur_file->mimetype); + + InspectionBufferSetupMulti(det_ctx, buffer, transforms, data, data_len); + + SCReturnPtr(buffer, "InspectionBuffer"); +} + +typedef struct PrefilterMpmFileMimetype { + int list_id; + const MpmCtx *mpm_ctx; + const DetectEngineTransforms *transforms; +} PrefilterMpmFileMimetype; + +/** \brief Filemimetype Filemimetype Mpm prefilter callback + * + * \param det_ctx detection engine thread ctx + * \param pectx inspection context + * \param p packet to inspect + * \param f flow to inspect + * \param txv tx to inspect + * \param idx transaction id + * \param flags STREAM_* flags including direction + */ +static void PrefilterTxFileMimetype(DetectEngineThreadCtx *det_ctx, const void *pectx, Packet *p, + Flow *f, void *txv, const uint64_t idx, const AppLayerTxData *txd, const uint8_t flags) +{ + SCEnter(); + + const PrefilterMpmFileMimetype *ctx = (const PrefilterMpmFileMimetype *)pectx; + const MpmCtx *mpm_ctx = ctx->mpm_ctx; + const int list_id = ctx->list_id; + + AppLayerGetFileState files = AppLayerParserGetTxFiles(f, txv, flags); + FileContainer *ffc = files.fc; + if (ffc != NULL) { + int local_file_id = 0; + for (File *file = ffc->head; file != NULL; file = file->next) { + InspectionBuffer *buffer = FileMimetypeGetDataCallback( + det_ctx, ctx->transforms, f, flags, file, list_id, local_file_id, txv); + if (buffer == NULL) + continue; + + if (buffer->inspect_len >= mpm_ctx->minlen) { + (void)mpm_table[mpm_ctx->mpm_type].Search(mpm_ctx, &det_ctx->mtc, &det_ctx->pmq, + buffer->inspect, buffer->inspect_len); + PREFILTER_PROFILING_ADD_BYTES(det_ctx, buffer->inspect_len); + } + local_file_id++; + } + } +} + +static void PrefilterMpmFileMimetypeFree(void *ptr) +{ + SCFree(ptr); +} + +static int PrefilterMpmFileMimetypeRegister(DetectEngineCtx *de_ctx, SigGroupHead *sgh, + MpmCtx *mpm_ctx, const DetectBufferMpmRegistry *mpm_reg, int list_id) +{ + PrefilterMpmFileMimetype *pectx = SCCalloc(1, sizeof(*pectx)); + if (pectx == NULL) + return -1; + pectx->list_id = list_id; + pectx->mpm_ctx = mpm_ctx; + pectx->transforms = &mpm_reg->transforms; + + return PrefilterAppendTxEngine(de_ctx, sgh, PrefilterTxFileMimetype, mpm_reg->app_v2.alproto, + mpm_reg->app_v2.tx_min_progress, pectx, PrefilterMpmFileMimetypeFree, mpm_reg->pname); +} + +static unsigned char DetectEngineInspectFileMimetype(DetectEngineCtx *de_ctx, + DetectEngineThreadCtx *det_ctx, const DetectEngineAppInspectionEngine *engine, + const Signature *s, Flow *f, uint8_t flags, void *alstate, void *txv, uint64_t tx_id) +{ + const DetectEngineTransforms *transforms = NULL; + if (!engine->mpm) { + transforms = engine->v2.transforms; + } + + AppLayerGetFileState files = AppLayerParserGetTxFiles(f, txv, flags); + FileContainer *ffc = files.fc; + if (ffc == NULL) { + return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES; + } + + uint8_t r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH; + int local_file_id = 0; + for (File *file = ffc->head; file != NULL; file = file->next) { + InspectionBuffer *buffer = FileMimetypeGetDataCallback( + det_ctx, transforms, f, flags, file, engine->sm_list, local_file_id, txv); + if (buffer == NULL) { + local_file_id++; + continue; + } + + const bool match = DetectEngineContentInspection(de_ctx, det_ctx, s, engine->smd, NULL, f, + (uint8_t *)buffer->inspect, buffer->inspect_len, buffer->inspect_offset, + DETECT_CI_FLAGS_SINGLE, DETECT_ENGINE_CONTENT_INSPECTION_MODE_STATE); + if (match) { + return DETECT_ENGINE_INSPECT_SIG_MATCH; + } else { + r = DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES; + } + local_file_id++; + } + return r; +} + +#endif /* HAVE_MIMETYPE */ diff --git a/src/detect-file-mimetype.h b/src/detect-file-mimetype.h new file mode 100644 index 000000000000..6badb38060be --- /dev/null +++ b/src/detect-file-mimetype.h @@ -0,0 +1,31 @@ +/* Copyright (C) 2025 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Eric Leblond + * + */ + +#ifndef SURICATA_DETECT_FILE_MIMETYPE_H +#define SURICATA_DETECT_FILE_MIMETYPE_H + +/* prototypes */ +void DetectFileMimetypeRegister(void); + +#endif /* SURICATA_DETECT_FILE_MIMETYPE_H */ diff --git a/src/detect.c b/src/detect.c index a86d99973acc..79fb38ead23d 100644 --- a/src/detect.c +++ b/src/detect.c @@ -374,6 +374,10 @@ DetectPostInspectFileFlagsUpdate(Flow *f, const SigGroupHead *sgh, uint8_t direc flow_file_flags |= (FLOWFILE_NO_MAGIC_TS|FLOWFILE_NO_MAGIC_TC); } #endif + if (!(sgh->flags & SIG_GROUP_HEAD_HAVEFILEMIMETYPE)) { + SCLogDebug("requesting disabling mimetype for flow"); + flow_file_flags |= (FLOWFILE_NO_MIMETYPE_TS | FLOWFILE_NO_MIMETYPE_TC); + } if (!(sgh->flags & SIG_GROUP_HEAD_HAVEFILEMD5)) { SCLogDebug("requesting disabling md5 for flow"); flow_file_flags |= (FLOWFILE_NO_MD5_TS|FLOWFILE_NO_MD5_TC); diff --git a/src/detect.h b/src/detect.h index 6f092b533312..a7b6bb5a29fa 100644 --- a/src/detect.h +++ b/src/detect.h @@ -317,14 +317,15 @@ typedef struct DetectPort_ { // vacancy 1x #define SIG_MASK_REQUIRE_ENGINE_EVENT BIT_U8(7) -#define FILE_SIG_NEED_FILE 0x01 -#define FILE_SIG_NEED_FILENAME 0x02 -#define FILE_SIG_NEED_MAGIC 0x04 /**< need the start of the file */ -#define FILE_SIG_NEED_FILECONTENT 0x08 -#define FILE_SIG_NEED_MD5 0x10 -#define FILE_SIG_NEED_SHA1 0x20 -#define FILE_SIG_NEED_SHA256 0x40 -#define FILE_SIG_NEED_SIZE 0x80 +#define FILE_SIG_NEED_FILE BIT_U16(0) +#define FILE_SIG_NEED_FILENAME BIT_U16(1) +#define FILE_SIG_NEED_MAGIC BIT_U16(2) /**< need the start of the file */ +#define FILE_SIG_NEED_FILECONTENT BIT_U16(3) +#define FILE_SIG_NEED_MD5 BIT_U16(4) +#define FILE_SIG_NEED_SHA1 BIT_U16(5) +#define FILE_SIG_NEED_SHA256 BIT_U16(6) +#define FILE_SIG_NEED_SIZE BIT_U16(7) +#define FILE_SIG_NEED_MIMETYPE BIT_U16(8) /* Detection Engine flags */ #define DE_QUIET 0x01 /**< DE is quiet (esp for unittests) */ @@ -681,7 +682,7 @@ typedef struct Signature_ { /** inline -- action */ uint8_t action; - uint8_t file_flags; + uint16_t file_flags; /** addresses, ports and proto this sig matches on */ DetectProto proto; @@ -1491,6 +1492,7 @@ enum { // vacancy #define SIG_GROUP_HEAD_HAVEFILESHA1 BIT_U16(4) #define SIG_GROUP_HEAD_HAVEFILESHA256 BIT_U16(5) +#define SIG_GROUP_HEAD_HAVEFILEMIMETYPE BIT_U16(6) enum MpmBuiltinBuffers { MPMB_TCP_PKT_TS, diff --git a/src/flow.c b/src/flow.c index 70777a1ecdd6..295bd7b3df30 100644 --- a/src/flow.c +++ b/src/flow.c @@ -222,6 +222,7 @@ static inline void FlowSwapFileFlags(Flow *f) SWAP_FLAGS(f->file_flags, FLOWFILE_NO_MD5_TS, FLOWFILE_NO_MD5_TC); SWAP_FLAGS(f->file_flags, FLOWFILE_NO_SHA1_TS, FLOWFILE_NO_SHA1_TC); SWAP_FLAGS(f->file_flags, FLOWFILE_NO_SHA256_TS, FLOWFILE_NO_SHA256_TC); + SWAP_FLAGS(f->file_flags, FLOWFILE_NO_MIMETYPE_TS, FLOWFILE_NO_MIMETYPE_TC); } static inline void TcpStreamFlowSwap(Flow *f) diff --git a/src/flow.h b/src/flow.h index 1a72fce49f49..b94841a3c75b 100644 --- a/src/flow.h +++ b/src/flow.h @@ -151,12 +151,17 @@ typedef struct AppLayerParserState_ AppLayerParserState; #define FLOWFILE_STORE_TS BIT_U16(12) #define FLOWFILE_STORE_TC BIT_U16(13) +/** no mime type tracking of files in this flow */ +#define FLOWFILE_NO_MIMETYPE_TS BIT_U16(14) +#define FLOWFILE_NO_MIMETYPE_TC BIT_U16(15) + #define FLOWFILE_NONE_TS \ (FLOWFILE_NO_MAGIC_TS | FLOWFILE_NO_STORE_TS | FLOWFILE_NO_MD5_TS | FLOWFILE_NO_SHA1_TS | \ - FLOWFILE_NO_SHA256_TS) + FLOWFILE_NO_SHA256_TS | FLOWFILE_NO_MIMETYPE_TS) #define FLOWFILE_NONE_TC \ (FLOWFILE_NO_MAGIC_TC | FLOWFILE_NO_STORE_TC | FLOWFILE_NO_MD5_TC | FLOWFILE_NO_SHA1_TC | \ - FLOWFILE_NO_SHA256_TC) + FLOWFILE_NO_SHA256_TC | FLOWFILE_NO_MIMETYPE_TC) + #define FLOWFILE_NONE (FLOWFILE_NONE_TS|FLOWFILE_NONE_TC) #define FLOW_IS_IPV4(f) \ diff --git a/src/output-file.c b/src/output-file.c index e468f14d25c0..5b8a313495de 100644 --- a/src/output-file.c +++ b/src/output-file.c @@ -33,6 +33,7 @@ #include "detect-filemagic.h" #include "util-file.h" #include "util-magic.h" +#include "util-mimetype.h" #include "util-profiling.h" #include "util-validate.h" @@ -131,6 +132,11 @@ void OutputFileLogFfc(ThreadVars *tv, OutputFileLoggerThreadData *op_thread_data if (FileForceMagic() && ff->magic == NULL) { FilemagicThreadLookup(&op_thread_data->magic_ctx, ff); } +#endif +#ifdef HAVE_MIMETYPE + if (FileForceMimetype() && ff->mimetype == NULL) { + FileMimetypeLookup(ff); + } #endif const OutputFileLogger *logger = list; const OutputLoggerThreadStore *store = op_thread_data->store; diff --git a/src/output-filedata.c b/src/output-filedata.c index 1e07902fdcd5..37ed310d5be9 100644 --- a/src/output-filedata.c +++ b/src/output-filedata.c @@ -33,6 +33,7 @@ #include "util-validate.h" #include "util-magic.h" #include "util-path.h" +#include "util-mimetype.h" bool g_filedata_logger_enabled = false; @@ -136,6 +137,12 @@ void OutputFiledataLogFfc(ThreadVars *tv, OutputFiledataLoggerThreadData *td, Pa FilemagicThreadLookup(&td->magic_ctx, ff); } #endif +#ifdef HAVE_MIMETYPE + if (FileForceMimetype() && ff->mimetype == NULL) { + FileMimetypeLookup(ff); + } +#endif + if (ff->flags & FILE_STORED) { continue; } diff --git a/src/output-filestore.c b/src/output-filestore.c index b177ea2bf7f4..c41f19dfa51b 100644 --- a/src/output-filestore.c +++ b/src/output-filestore.c @@ -446,6 +446,12 @@ static OutputInitResult OutputFilestoreLogInitCtx(SCConfNode *conf) SCLogConfig("Filestore (v2) forcing magic lookup for stored files"); } + const char *force_mimetype = SCConfNodeLookupChildValue(conf, "force-mimetype"); + if (force_mimetype != NULL && SCConfValIsTrue(force_mimetype)) { + FileForceMimetypeEnable(); + SCLogConfig("forcing mimetype lookup for logged files"); + } + FileForceHashParseCfg(conf); /* The new filestore requires SHA256. */ diff --git a/src/output-json-alert.c b/src/output-json-alert.c index 184370a65edc..3a5274aaece9 100644 --- a/src/output-json-alert.c +++ b/src/output-json-alert.c @@ -65,6 +65,7 @@ #include "util-buffer.h" #include "util-reference-config.h" #include "util-validate.h" +#include "util-mimetype.h" #include "action-globals.h" @@ -487,6 +488,10 @@ static void AlertAddFiles(const Packet *p, SCJsonBuilder *jb, const uint64_t tx_ isopen = true; SCJbOpenArray(jb, "files"); } +#ifdef HAVE_MIMETYPE + if (FileForceMimetype() && file->mimetype == NULL) + FileMimetypeLookup(file); +#endif SCJbStartObject(jb); EveFileInfo(jb, file, tx_id, file->flags); SCJbClose(jb); diff --git a/src/output-json-file.c b/src/output-json-file.c index 9c046b3c7d18..f3ba44367362 100644 --- a/src/output-json-file.c +++ b/src/output-json-file.c @@ -327,6 +327,12 @@ static OutputInitResult OutputFileLogInitSub(SCConfNode *conf, OutputCtx *parent SCLogConfig("forcing magic lookup for logged files"); } + const char *force_mimetype = SCConfNodeLookupChildValue(conf, "force-mimetype"); + if (force_mimetype != NULL && SCConfValIsTrue(force_mimetype)) { + FileForceMimetypeEnable(); + SCLogConfig("forcing mimetype lookup for logged files"); + } + FileForceHashParseCfg(conf); } diff --git a/src/output-json.c b/src/output-json.c index 1e04e13c49dd..3d2072d20bf2 100644 --- a/src/output-json.c +++ b/src/output-json.c @@ -137,7 +137,12 @@ void EveFileInfo(SCJsonBuilder *jb, const File *ff, const uint64_t tx_id, const if (ff->magic) SCJbSetString(jb, "magic", (char *)ff->magic); #endif + + if (ff->mimetype) + SCJbSetString(jb, "mimetype", (char *)ff->mimetype); + SCJbSetBool(jb, "gaps", ff->flags & FILE_HAS_GAPS); + switch (ff->state) { case FILE_STATE_CLOSED: JB_SET_STRING(jb, "state", "CLOSED"); diff --git a/src/suricata.c b/src/suricata.c index f7b62c9dd2cc..5df51c0b30d5 100644 --- a/src/suricata.c +++ b/src/suricata.c @@ -128,6 +128,7 @@ #include "util-landlock.h" #include "util-macset.h" #include "util-flow-rate.h" +#include "util-mimetype.h" #include "util-misc.h" #include "util-mpm-hs.h" #include "util-path.h" @@ -807,6 +808,9 @@ static void PrintBuildInfo(void) strlcat(features, "HAVE_JA4 ", sizeof(features)); #endif strlcat(features, "HAVE_LIBJANSSON ", sizeof(features)); +#ifdef HAVE_MIMETYPE + strlcat(features, "HAVE_MIMETYPE ", sizeof(features)); +#endif #ifdef PROFILING strlcat(features, "PROFILING ", sizeof(features)); #endif @@ -3033,6 +3037,11 @@ void SuricataInit(void) /* Ignore recursion level when comparing flows. */ g_recurlvl_mask = 0x00; } + +#ifdef HAVE_MIMETYPE + FileMimetypeSetup(); +#endif /* HAVE_MIMETYPE */ + SetupUserMode(&suricata); InitRunAs(&suricata); diff --git a/src/util-file.c b/src/util-file.c index d90b566b6e3c..915e77d3f85f 100644 --- a/src/util-file.c +++ b/src/util-file.c @@ -55,6 +55,11 @@ static int g_file_force_filestore = 0; */ static int g_file_force_magic = 0; +/** \brief switch to force mimetype checks on all files + * regardless of the rules. + */ +static int g_file_force_mimetype = 0; + /** \brief switch to force md5 calculation on all files * regardless of the rules. */ @@ -101,6 +106,12 @@ void FileForceMagicEnable(void) g_file_flow_mask |= (FLOWFILE_NO_MAGIC_TS|FLOWFILE_NO_MAGIC_TC); } +void FileForceMimetypeEnable(void) +{ + g_file_force_mimetype = 1; + g_file_flow_mask |= (FLOWFILE_NO_MIMETYPE_TS | FLOWFILE_NO_MIMETYPE_TC); +} + void FileForceMd5Enable(void) { g_file_force_md5 = 1; @@ -143,6 +154,11 @@ int FileForceMagic(void) return g_file_force_magic; } +int FileForceMimetype(void) +{ + return g_file_force_mimetype; +} + int FileForceMd5(void) { return g_file_force_md5; @@ -239,6 +255,10 @@ uint16_t FileFlowFlagsToFlags(const uint16_t flow_file_flags, uint8_t direction) if (flow_file_flags & FLOWFILE_NO_SHA256_TS) { flags |= FILE_NOSHA256; } + + if (flow_file_flags & FLOWFILE_NO_MIMETYPE_TS) { + flags |= FILE_NOMIMETYPE; + } } else { if ((flow_file_flags & (FLOWFILE_NO_STORE_TC | FLOWFILE_STORE_TC)) == FLOWFILE_NO_STORE_TC) { @@ -262,6 +282,10 @@ uint16_t FileFlowFlagsToFlags(const uint16_t flow_file_flags, uint8_t direction) if (flow_file_flags & FLOWFILE_NO_SHA256_TC) { flags |= FILE_NOSHA256; } + + if (flow_file_flags & FLOWFILE_NO_MIMETYPE_TC) { + flags |= FILE_NOMIMETYPE; + } } DEBUG_VALIDATE_BUG_ON((flags & (FILE_STORE | FILE_NOSTORE)) == (FILE_STORE | FILE_NOSTORE)); @@ -367,6 +391,15 @@ static int FilePruneFile(File *file, const StreamingBufferConfig *cfg) SCLogDebug("file->flags & FILE_NOMAGIC == true"); } #endif + if (!(file->flags & FILE_NOMIMETYPE)) { + /* need mimetype but haven't set it yet, bail out */ + if (file->mimetype == NULL) + SCReturnInt(0); + else + SCLogDebug("file->mimetype %s", file->mimetype); + } else { + SCLogDebug("file->flags & FILE_NOMIMETYPE == true"); + } uint64_t left_edge = FileDataSize(file); if (file->flags & FILE_STORE) { left_edge = MIN(left_edge,file->content_stored); @@ -578,6 +611,8 @@ static void FileFree(File *ff, const StreamingBufferConfig *sbcfg) if (ff->magic != NULL) SCFree(ff->magic); #endif + if (ff->mimetype != NULL) + SCRustCStringFree(ff->mimetype); if (ff->sb != NULL) { StreamingBufferFree(ff->sb, sbcfg); } @@ -912,6 +947,10 @@ static File *FileOpenFile(FileContainer *ffc, const StreamingBufferConfig *sbcfg SCLogDebug("not doing magic for this file"); ff->flags |= FILE_NOMAGIC; } + if (flags & FILE_NOMIMETYPE) { + SCLogDebug("not doing mimetype for this file"); + ff->flags |= FILE_NOMIMETYPE; + } if (flags & FILE_NOMD5) { SCLogDebug("not doing md5 for this file"); ff->flags |= FILE_NOMD5; diff --git a/src/util-file.h b/src/util-file.h index 3e42efda650d..e6cd7ac4f8fd 100644 --- a/src/util-file.h +++ b/src/util-file.h @@ -56,6 +56,7 @@ typedef struct SCMd5 SCMd5; #define FILE_STORED BIT_U16(11) #define FILE_NOTRACK BIT_U16(12) /**< track size of file */ #define FILE_USE_DETECT BIT_U16(13) /**< use content_inspected tracker */ +#define FILE_NOMIMETYPE BIT_U16(14) #define FILE_HAS_GAPS BIT_U16(15) // to be used instead of PATH_MAX which depends on the OS @@ -89,6 +90,7 @@ typedef struct File_ { #ifdef HAVE_MAGIC char *magic; #endif + char *mimetype; struct File_ *next; SCMd5 *md5_ctx; uint8_t md5[SC_MD5_LEN]; @@ -221,6 +223,9 @@ uint32_t FileReassemblyDepth(void); void FileForceMagicEnable(void); int FileForceMagic(void); +void FileForceMimetypeEnable(void); +int FileForceMimetype(void); + void FileForceMd5Enable(void); int FileForceMd5(void); diff --git a/src/util-mimetype.c b/src/util-mimetype.c new file mode 100644 index 000000000000..96c22c5c9374 --- /dev/null +++ b/src/util-mimetype.c @@ -0,0 +1,84 @@ +/* Copyright (C) 2025 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Eric Leblond + * + */ + +#include "suricata-common.h" +#include "util-conf.h" +#include "util-file.h" +#include "util-mimetype.h" +#include "rust.h" + +#ifdef HAVE_MIMETYPE + +#define FILE_MIMETYPE_MIN_SIZE 512 + +int FileMimetypeLookup(File *file) +{ + if (file == NULL || FileDataSize(file) == 0) { + SCReturnInt(-1); + } + + const uint8_t *data = NULL; + uint32_t data_len = 0; + uint64_t offset = 0; + + StreamingBufferGetData(file->sb, &data, &data_len, &offset); + if (offset == 0) { + if (FileDataSize(file) >= FILE_MIMETYPE_MIN_SIZE) { + file->mimetype = SCGetMimetype(data, data_len); + } else if (file->state >= FILE_STATE_CLOSED) { + file->mimetype = SCGetMimetype(data, data_len); + } + } + SCReturnInt(0); +} + +void FileMimetypeSetup(void) +{ +#ifndef HAVE_GPL_MIMETYPE + /* create MIME type directory environment variable early so that tree_magic_mini implicit + * initialization can use it */ + const char *mimetype_dir; + if (SCConfGet("mimetype-dir", &mimetype_dir) == 1) { + /* only set the mimetype dir environment variable if it is set in the config */ + SCLogConfig("Using mimetype directory: %s", mimetype_dir); + setenv("TREE_MAGIC_DIR", mimetype_dir, 0); + } else { + /* if not set, try to use default if available */ + const char *data_dir = ConfigGetDataDirectory(); + struct stat st; + char mime_path[PATH_MAX]; + snprintf(mime_path, sizeof(mime_path), "%s/mimetype/", data_dir); // TODO WINDOWS + if (stat(data_dir, &st) != 0) { + SCLogConfig("Using mimetype information from system"); + return; + } + SCLogConfig("Using default mimetype directory: %s", mime_path); + setenv("TREE_MAGIC_DIR", mime_path, 0); + } +#else + SCLogConfig("Using embedded mimetype information"); +#endif /* not HAVE_GPL_MIMETYPE */ +} + +#endif /* HAVE_MIMETYPE */ diff --git a/src/util-mimetype.h b/src/util-mimetype.h new file mode 100644 index 000000000000..806110608e32 --- /dev/null +++ b/src/util-mimetype.h @@ -0,0 +1,32 @@ +/* Copyright (C) 2025 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Eric Leblond + * + */ + +#ifndef SURICATA_UTIL_MIMETYPE_H +#define SURICATA_UTIL_MIMETYPE_H + +int FileMimetypeLookup(File *file); + +void FileMimetypeSetup(void); + +#endif /* SURICATA_UTIL_MIMETYPE_H */ diff --git a/suricata.yaml.in b/suricata.yaml.in index 9b96a59ea3d5..b98a10f62399 100644 --- a/suricata.yaml.in +++ b/suricata.yaml.in @@ -292,6 +292,7 @@ outputs: #custom: [subject, issuer, session_resumed, serial, fingerprint, sni, version, not_before, not_after, certificate, chain, ja3, ja3s, ja4, subjectaltname, client, client_certificate, client_chain, client_alpns, server_alpns, client_handshake, server_handshake] - files: force-magic: no # force logging magic on all logged files + force-mimetype: no # force logging mime type on all logged files # force logging of checksums, available hash functions are md5, # sha1 and sha256 #force-hash: [md5] @@ -1403,6 +1404,11 @@ unix-command: #magic-file: /usr/share/file/magic @e_magic_file_comment@magic-file: @e_magic_file@ +# Directory where the mimetype files are stored. +# Default path will be picked if variable is not set so use this +# variable only to implement custom mimetype files. +#mimetype-dir: @e_datadir@/mimetype/ + # GeoIP2 database file. Specify path and filename of GeoIP2 database # if using rules with "geoip" rule option. #geoip-database: /usr/local/share/GeoLite2/GeoLite2-Country.mmdb