Merge remote-tracking branch 'origin/main' into word-count-classifier

Author: LeonardPuettmannKern

classifiers/__init__.py

@@ -11,6 +11,7 @@
 
 from .reference_quality import (
     word_count_classifier,
+    special_character_classifier,
     chunked_sentence_complexity,
 )
 
@@ -63,6 +64,8 @@
     bert_sentiment_german,
     word_count_classifier,
     chunked_sentence_complexity,
+    special_character_classifier,
+    chunked_sentence_complexity
 ]:
     module_name = module.__name__.split(".")[-1]
     model_name = (

classifiers/reference_quality/chunked_sentence_complexity/config.py

@@ -18,6 +18,15 @@ def get_config():
             "text_analysis"
         ],  # first entry should be parent directory
         # bricks integrator information
+        cognition_init_mapping={
+            "very easy": "null",
+            "easy": "null",
+            "fairly easy": "null",
+            "standard": "null",
+            "fairly difficult": "Needs fix",
+            "difficult": "Needs fix",
+            "very difficult": "Needs fix",
+        },
         integrator_inputs={
             "name": "chunked_sentence_complexity",
             "refineryDataType": RefineryDataType.TEXT.value,

classifiers/reference_quality/special_character_classifier/README.md

@@ -0,0 +1 @@
+The purpose of this brick is to identify if there are any unusual characters in the given text. This function can be useful for text preprocessing tasks, especially for checking reference material in RAG (Retrieval Augmented Generation) use cases where you want to filter out text that contains unusual or unexpected characters.

classifiers/reference_quality/special_character_classifier/__init__.py

@@ -0,0 +1,37 @@
+import unicodedata
+from typing import Optional, List, Tuple
+from pydantic import BaseModel
+
+INPUT_EXAMPLE = {
+    "text": "Super funny haha 😀.",
+    "allowedRange": None
+}
+
+ALLOWED_RANGE = set(range(32, 127)).union( # Basic Latin
+    set(range(160, 255)), # Latin-1 Supplement
+    set(range(256, 384)),  # Latin Extended-A
+    set(range(384, 592)),  # Latin Extended-B
+    set(range(8192, 8303)),  # General Punctuation
+    set(range(8352, 8399)),  # Currency Symbols
+    set([ord("\t"), ord("\n"), ord("\r")])# common stop chars
+)
+
+class SpecialCharacterClassifierModel(BaseModel):
+    text: str
+    allowedRange: Optional[List[int]] = None
+
+    class Config:
+        schema_extra = {"example": INPUT_EXAMPLE}
+
+
+def special_character_classifier(req: SpecialCharacterClassifierModel):
+    """Checks if a string contains special characters"""
+    text = req.text
+    allowed_range = req.allowedRange
+    if allowed_range is None:
+        allowed_range = ALLOWED_RANGE
+
+    for char in text:
+        if ord(char) not in allowed_range and unicodedata.category(char) != "Zs":
+            return {"contains_special_char": True}
+    return {"contains_special_char": False}

classifiers/reference_quality/special_character_classifier/code_snippet_common.md

@@ -0,0 +1,41 @@
+```python
+import unicodedata
+from typing import List, Tuple
+
+DEFAULT_ALLOWED_RANGE = set(range(32, 127)).union( # Basic Latin
+    set(range(160, 255)), # Latin-1 Supplement
+    set(range(256, 384)),  # Latin Extended-A
+    set(range(384, 592)),  # Latin Extended-B
+    set(range(8192, 8303)),  # General Punctuation
+    set(range(8352, 8399)),  # Currency Symbols
+    set([ord("\t"), ord("\n"), ord("\r")])# common stop chars
+)
+
+
+def special_character_classifier(text: str, allowed_range: List[int] = None) -> str:
+    """
+    @param text: Text to detect special characters in
+    @param allowed_ranges: whitelist of hexcodes
+    @return: boolean if text contains special characters
+    """
+    
+    if allowed_range is None:
+        allowed_range= DEFAULT_ALLOWED_RANGE
+    
+    for char in text:
+        if ord(char) not in allowed_range and unicodedata.category(char) != "Zs":
+            return True
+    return False
+
+
+# ↑ necessary bricks function 
+# -----------------------------------------------------------------------------------------
+# ↓ example implementation 
+
+def example_integration():
+    texts = ["This contains a special char 你好.", "Such a clean text, wow!", "This is a greek letter: α", "Super funny 😀", "Rainbows are very nice."]
+    for text in texts:
+        print(f"\"{text}\" -> {special_character_classifier(text)}")
+
+example_integration()
+```

classifiers/reference_quality/special_character_classifier/code_snippet_refinery.md

@@ -0,0 +1,27 @@
+```python
+import unicodedata
+from typing import Optional, List, Tuple
+
+ATTRIBUTE: str = "text" # only text attributes
+LABEL: str = "has_special_character"
+ALLOWED_RANGE: List[int] = None # list of integers that represent Unicode code points
+
+def special_character_classifier(record):
+    text = record[ATTRIBUTE].text    
+
+    allowed = ALLOWED_RANGE
+    if not allowed:
+        allowed = default_allowed_values
+    for char in text:
+        if ord(char) not in allowed and unicodedata.category(char) != "Zs":
+            return LABEL
+
+default_allowed_values = set(range(32, 127)).union( # Basic Latin
+    set(range(160, 255)), # Latin-1 Supplement
+    set(range(256, 384)),  # Latin Extended-A
+    set(range(384, 592)),  # Latin Extended-B
+    set(range(8192, 8303)),  # General Punctuation
+    set(range(8352, 8399)),  # Currency Symbols
+    set([ord("\t"), ord("\n"), ord("\r")])# common stop chars
+)
+```

classifiers/reference_quality/special_character_classifier/config.py

@@ -0,0 +1,52 @@
+from util.configs import build_classifier_function_config
+from util.enums import State, BricksVariableType, RefineryDataType, SelectionType
+from . import special_character_classifier, INPUT_EXAMPLE
+
+
+def get_config():
+    return build_classifier_function_config(
+        # strapi information
+        function=special_character_classifier,
+        input_example=INPUT_EXAMPLE,
+        issue_id=345,
+        tabler_icon="LanguageKatakana",
+        min_refinery_version="1.7.0",
+        state=State.PUBLIC.value,
+        type="python_function",
+        available_for=["refinery", "common"],
+        part_of_group=[
+            "reference_quality",
+        ],  # first entry should be parent directory
+        # bricks integrator information
+        cognition_init_mapping = {
+            "@@LABEL@@": "Needs fix",
+        },
+        integrator_inputs={
+            "name": "special_character_classifier",
+            "refineryDataType": RefineryDataType.TEXT.value,
+            "variables": {
+                "ATTRIBUTE": {
+                    "selectionType": SelectionType.CHOICE.value,
+                    "addInfo": [
+                        BricksVariableType.ATTRIBUTE.value,
+                        BricksVariableType.GENERIC_STRING.value
+                    ]
+                },
+                "ALLOWED_RANGE": {
+                    "selectionType": SelectionType.LIST.value,
+                    "optional": "true",
+                    "addInfo": [
+                        BricksVariableType.GENERIC_INT.value
+                    ]
+                },
+                "LABEL": {
+                    "selectionType": SelectionType.CHOICE.value,
+                    "defaultValue": "has_special_character",
+                    "addInfo": [
+                        BricksVariableType.LABEL.value,
+                        BricksVariableType.GENERIC_STRING.value
+                    ]
+                }
+            }
+        }      
+    )

classifiers/text_analysis/language_detection/__init__.py

@@ -17,8 +17,6 @@ def language_detection(request: LanguageDetectionModel):
     """Detects the language of a given text."""
 
     text = request.text
-    try:
-        language = detect(text)
-        return {"language": language}
-    except LangDetectException:
-        return "No language detected."
+    if not text or not text.strip():
+        return {"language": "unknown"}
+    return {"language": detect(text)}

classifiers/text_analysis/language_detection/code_snippet_common.md

@@ -6,6 +6,8 @@ def language_detection(text:str)->str:
     @param text: text to check
     @return: language iso code. Full list here https://github.com/Mimino666/langdetect#languages
     """
+    if not text or not text.strip():
+        return "unknown"
     return detect(text)
 
 # ↑ necessary bricks function 

classifiers/text_analysis/language_detection/code_snippet_refinery.md

@@ -5,5 +5,7 @@ ATTRIBUTE: str = "text" #only text attributes
 
 def language_detection(record):
     text = record[ATTRIBUTE].text # SpaCy document, hence we need to call .text to get the string
-    return detect(text) # e.g. "en"
+    if not text or not text.strip():
+        return "unknown"
+    return detect(text) 
 ```