Merge pull request #19 from theseus-rs/correct-absolute-eof-logic

fix: update internal signature to require all byte sequences match

Author: brianheineman

export/apache_httpd/src/main.rs

@@ -4,7 +4,7 @@
 #![deny(clippy::unwrap_used)]
 
 use anyhow::Result;
-use file_type::pronom::{DocumentIdentifier, ExternalSignature, FileFormat};
+use file_type::pronom::{DocumentIdentifier, ExternalSignature, FileFormat, SignatureType};
 use file_type::FileType;
 use jiff::civil::Date;
 use jiff::tz::TimeZone;
@@ -128,7 +128,7 @@ fn process_mime_types(mime_types: HashMap<String, Vec<String>>) -> Result<Vec<Fi
             .iter()
             .enumerate()
             .map(|(index, extension)| {
-                ExternalSignature::new(index, extension.as_str(), "File extension")
+                ExternalSignature::new(index, extension.as_str(), SignatureType::FileExtension)
             })
             .collect::<Vec<ExternalSignature>>();
 

export/linguist/src/main.rs

@@ -4,7 +4,7 @@
 #![deny(clippy::unwrap_used)]
 
 use anyhow::Result;
-use file_type::pronom::{DocumentIdentifier, ExternalSignature, FileFormat};
+use file_type::pronom::{DocumentIdentifier, ExternalSignature, FileFormat, SignatureType};
 use file_type::FileType;
 use jiff::civil::Date;
 use jiff::tz::TimeZone;
@@ -149,7 +149,7 @@ fn process_languages(languages: Vec<Language>) -> Vec<FileFormat> {
             .iter()
             .enumerate()
             .map(|(index, extension)| {
-                ExternalSignature::new(index, extension.as_str(), "File extension")
+                ExternalSignature::new(index, extension.as_str(), SignatureType::FileExtension)
             })
             .collect::<Vec<ExternalSignature>>();
 

file_type/src/pronom/byte_sequence.rs

@@ -149,7 +149,10 @@ impl ByteSequence {
             }
             PositionType::AbsoluteFromEOF => {
                 let offset = self.offset.unwrap_or_default();
-                let offset = usize::checked_sub(bytes.len(), offset).unwrap_or(0);
+                let value = self.regex.to_string();
+                let regex_len = value.len() / 2;
+                let offset = usize::checked_add(regex_len, offset).unwrap_or(regex_len);
+                let offset = usize::checked_sub(bytes.len(), offset).unwrap_or(bytes.len());
                 self.regex.is_match_at(bytes, offset)
             }
             PositionType::Variable => {

file_type/src/pronom/external_signature.rs

@@ -1,5 +1,13 @@
 use serde::{Deserialize, Serialize};
 
+/// The type of signature.
+#[derive(Clone, Debug, Default, Deserialize, Serialize)]
+pub enum SignatureType {
+    #[default]
+    #[serde(rename = "File extension")]
+    FileExtension,
+}
+
 /// An external signature.
 #[derive(Clone, Debug, Default, Deserialize, Serialize)]
 #[serde(default, rename_all = "PascalCase")]
@@ -8,16 +16,16 @@ pub struct ExternalSignature {
     id: usize,
     signature: String,
     #[serde(rename = "SignatureType")]
-    r#type: String,
+    r#type: SignatureType,
 }
 
 impl ExternalSignature {
     /// Create a new external signature.
-    pub fn new<S: AsRef<str>>(id: usize, signature: S, r#type: S) -> Self {
+    pub fn new<S: AsRef<str>>(id: usize, signature: S, r#type: SignatureType) -> Self {
         Self {
             id,
             signature: signature.as_ref().to_string(),
-            r#type: r#type.as_ref().to_string(),
+            r#type,
         }
     }
 
@@ -35,7 +43,7 @@ impl ExternalSignature {
 
     /// Get the type of the signature.
     #[must_use]
-    pub fn r#type(&self) -> &str {
+    pub fn r#type(&self) -> &SignatureType {
         &self.r#type
     }
 }
@@ -65,16 +73,22 @@ mod test {
         let external_signature: ExternalSignature = from_str(xml.as_str())?;
 
         assert_eq!(external_signature.id(), 2421);
+        assert!(matches!(
+            external_signature.r#type(),
+            SignatureType::FileExtension
+        ));
         assert_eq!(external_signature.signature(), "json");
-        assert_eq!(external_signature.r#type(), "File extension");
         Ok(())
     }
 
     #[test]
     fn test_new() {
-        let external_signature = ExternalSignature::new(2421, "json", "File extension");
+        let external_signature = ExternalSignature::new(2421, "json", SignatureType::FileExtension);
         assert_eq!(external_signature.id(), 2421);
+        assert!(matches!(
+            external_signature.r#type(),
+            SignatureType::FileExtension
+        ));
         assert_eq!(external_signature.signature(), "json");
-        assert_eq!(external_signature.r#type(), "File extension");
     }
 }

file_type/src/pronom/file_format.rs

@@ -360,6 +360,7 @@ impl FileFormat {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::pronom::external_signature::SignatureType;
     use crate::pronom::{Author, ByteSequence, Endianness, PositionType};
     use indoc::indoc;
     use quick_xml::de::from_str;
@@ -538,7 +539,7 @@ mod tests {
             ],
             vec![],
             vec![
-                ExternalSignature::new(761, "png", "File extension"),
+                ExternalSignature::new(761, "png", SignatureType::FileExtension),
             ],
             vec![],
             vec![],

file_type/src/pronom/internal_signature.rs

@@ -59,9 +59,10 @@ impl InternalSignature {
     /// Check if this internal signature is a match for the given bytes
     #[must_use]
     pub fn is_match(&self, bytes: &[u8]) -> bool {
+        // All byte sequences must match in order for the internal signature to match
         self.byte_sequences
             .iter()
-            .any(|byte_sequence| byte_sequence.is_match(bytes))
+            .all(|byte_sequence| byte_sequence.is_match(bytes))
     }
 }
 

file_type/src/pronom/mod.rs

@@ -16,7 +16,7 @@ pub use byte_sequence::{ByteSequence, Endianness, PositionType};
 pub use compression_type::{CompressionType, Lossiness};
 pub use document::Document;
 pub use document_identifier::DocumentIdentifier;
-pub use external_signature::ExternalSignature;
+pub use external_signature::{ExternalSignature, SignatureType};
 pub use file_format::FileFormat;
 pub use internal_signature::InternalSignature;
 pub use publisher::Publisher;

file_type/tests/classify_pronom_files.rs

@@ -12,9 +12,32 @@ fn data_dir() -> PathBuf {
         .join("pronom")
 }
 
+async fn test_file(file_name: &str) -> Result<(String, &FileType)> {
+    let data_dir = data_dir();
+    let path = data_dir.join(file_name);
+    let file_name = path
+        .file_name()
+        .expect("file name")
+        .to_string_lossy()
+        .to_string();
+    let file_name = file_name.split('.').next().expect("split").to_string();
+    let id = if file_name.starts_with("x-fmt-") {
+        let parts: Vec<&str> = file_name.split('-').collect();
+        format!("{}-{}/{}", parts[0], parts[1], parts[2])
+    } else {
+        let parts: Vec<&str> = file_name.split('-').collect();
+        format!("{}/{}", parts[0], parts[1])
+    };
+
+    let file_type = FileType::try_from_file(path).await?;
+    Ok((id, file_type))
+}
+
 #[tokio::test]
 async fn test_file_classification() -> Result<()> {
     let data_dir = data_dir();
+    let mut passed_tests = 0;
+    let mut failed_tests = 0;
 
     for entry in WalkDir::new(data_dir) {
         let entry = entry?;
@@ -23,32 +46,34 @@ async fn test_file_classification() -> Result<()> {
             continue;
         }
 
-        let full_filename = path
+        let file_name = path
             .file_name()
             .expect("file name")
             .to_string_lossy()
             .to_string();
-        let filename = full_filename.split('.').next().expect("split").to_string();
-        let id = if filename.starts_with("x-fmt-") {
-            let parts: Vec<&str> = filename.split('-').collect();
-            format!("{}-{}/{}", parts[0], parts[1], parts[2])
-        } else {
-            let parts: Vec<&str> = filename.split('-').collect();
-            format!("{}/{}", parts[0], parts[1])
-        };
+        let (id, file_type) = test_file(&file_name).await?;
 
-        let file_type = FileType::try_from_file(path).await?;
         if file_type.id() == id {
-            println!("file_type.id()={}, id={}", file_type.id(), id);
+            assert_eq!(file_type.id(), id);
+            passed_tests += 1;
         } else {
             eprintln!(
-                "[ERROR] file_type.id()={}, id={}: {full_filename}",
+                "[ERROR] file_type.id()={}, id={}: {file_name}",
                 file_type.id(),
                 id
             );
+            failed_tests += 1;
         }
-        // assert_eq!(file_type.id(), id);
     }
 
+    println!("Passed: {passed_tests}");
+    println!("Failed: {failed_tests}");
     Ok(())
 }
+
+// #[tokio::test]
+// async fn test_single_file_classification() -> Result<()> {
+//     let (id, file_type) = test_file("fmt-708-signature-id-831.wav").await?;
+//     assert_eq!(file_type.id(), id);
+//     Ok(())
+// }