Merge remote-tracking branch 'origin/main' into location-extraction

Author: LeonardPuettmannKern

classifiers/__init__.py

@@ -9,9 +9,20 @@
 
 from .lookup_lists import lookup_list
 
+from .reference_complexity import (
+    chunked_sentence_complexity,
+
+from .question_type import (
+    question_type_classifier
+)
+
+from .communication_style import (
+    communication_style_classifier
+)
+
 from .reference_quality import (
+    word_count_classifier,
     special_character_classifier,
-    chunked_sentence_complexity,
 )
 
 from .dates_and_times import (
@@ -61,8 +72,12 @@
     workday_classifier,
     deberta_review_classifier, 
     bert_sentiment_german,
+    word_count_classifier,
+    chunked_sentence_complexity,
     special_character_classifier,
-    chunked_sentence_complexity
+    chunked_sentence_complexity,
+    question_type_classifier,
+    communication_style_classifier
 ]:
     module_name = module.__name__.split(".")[-1]
     model_name = (

classifiers/communication_style/communication_style_classifier/README.md

@@ -0,0 +1 @@
+Uses an `intfloat/multilingual-e5-small` model, which was finetuned on english and german examples of different question types. The model is hosted on Kern AIs own infrastructure and is meant to be used to classify text sequences by the labels `action-seeking`, `fact-oriented`, `information-seeking` or `self-revealing`.

classifiers/communication_style/communication_style_classifier/__init__.py

@@ -0,0 +1,27 @@
+from pydantic import BaseModel
+import requests
+
+INPUT_EXAMPLE = {
+    "text": "Change the number in row 2 and 3.",
+    "model_name": "KernAI/multilingual-e5-communication-style",
+}
+
+
+class CommunicationStyleClassifierModel(BaseModel):
+    text: str
+    model_name: str
+
+    class Config:
+        schema_extra = {"example": INPUT_EXAMPLE}
+
+
+def communication_style_classifier(req: CommunicationStyleClassifierModel):
+    """Uses custom E5 model to classify communication style of a text"""
+    payload = {
+        "model_name": req.model_name,
+        "text": req.text
+    }      
+    response = requests.post("https://free.api.kern.ai/inference", json=payload)
+    if response.ok:
+        return {"communication_style": response.json()["label"]}
+    return response.raise_for_status()

classifiers/communication_style/communication_style_classifier/code_snippet_common.md

@@ -0,0 +1,34 @@
+```python
+import requests
+
+def communication_style_classifier(text: str, model_name: str, request_url: str = "https://free.api.kern.ai/inference") -> str:
+    """
+    @param text: text with a user query you want to classify
+    @param model_name: Name of a model provided by Kern AI
+    @param request_url: URL to the API endpoint of Kern AI
+    @return: returns either 'action-seeking', 'fact-oriented', 'information-seeking' or 'self-revealing'.
+    """
+    payload = {
+        "model_name": model_name,
+        "text": text
+    }      
+    response = requests.post(request_url, json=payload)
+    if response.ok:
+        return response.json()["label"]
+    return response.raise_for_status()
+
+
+# ↑ necessary bricks function 
+# -----------------------------------------------------------------------------------------
+# ↓ example implementation 
+
+
+model_name = "KernAI/multilingual-e5-communication-style"
+
+def example_integration():
+    texts = ["Change the number in row 2 and 3.", "Can you show me some data from the references?", "I am super happy today."]
+    for text in texts:
+        print(f"the communication style of \"{text}\" is \"{communication_style_classifier(text, model_name=model_name)}\"")
+
+example_integration()
+```

classifiers/communication_style/communication_style_classifier/code_snippet_refinery.md

@@ -0,0 +1,16 @@
+```python
+import requests
+
+ATTRIBUTE: str = "text" # only text attributes
+MODEL_NAME: str = "KernAI/multilingual-e5-communication-style"
+REQUEST_URL: str = "https://free.api.kern.ai/inference"
+
+def communication_style_classifier(record):
+    payload = {
+        "model_name": MODEL_NAME,
+        "text": record[ATTRIBUTE].text
+    }      
+    response = requests.post(REQUEST_URL, json=payload)
+    if response.ok:
+        return response.json()["label"]
+```

classifiers/communication_style/communication_style_classifier/config.py

@@ -0,0 +1,48 @@
+from util.configs import build_classifier_function_config
+from util.enums import State, RefineryDataType, BricksVariableType, SelectionType
+from . import communication_style_classifier, INPUT_EXAMPLE
+
+
+def get_config():
+    return build_classifier_function_config(
+        function=communication_style_classifier,
+        input_example=INPUT_EXAMPLE,
+        issue_id=343,
+        tabler_icon="CircleDotted",
+        min_refinery_version="1.7.0",
+        state=State.PUBLIC.value,
+        type="python_function",
+        available_for=["refinery", "common"],
+        part_of_group=[
+            "communication_style"
+        ],  # first entry should be parent directory
+        # bricks integrator information
+        integrator_inputs={
+            "name": "communication_style_classifier",
+            "refineryDataType": RefineryDataType.TEXT.value,
+            "outputs": ["action-seeking", "fact-oriented", "information-seeking", "self-revealing"],
+            "variables": {
+                "ATTRIBUTE": {
+                    "selectionType": SelectionType.CHOICE.value,
+                    "addInfo": [
+                        BricksVariableType.ATTRIBUTE.value,
+                        BricksVariableType.GENERIC_STRING.value
+                    ]
+                },
+                "MODEL_NAME": {
+                    "selectionType": SelectionType.STRING.value,
+                    "defaultValue": "KernAI/multilingual-e5-communication-style",
+                    "addInfo": [
+                        BricksVariableType.GENERIC_STRING.value
+                    ]
+                },
+                "REQUEST_URL": {
+                    "selectionType": SelectionType.STRING.value,
+                    "defaultValue": "https://free.api.kern.ai/inference",
+                    "addInfo": [
+                        BricksVariableType.GENERIC_STRING.value
+                    ]
+                }
+            }
+        }
+    )

classifiers/question_type/question_type_classifier/README.md

@@ -0,0 +1 @@
+Uses an `intfloat/multilingual-e5-small` model, which was finetuned on english and german examples of different question types. The model is hosted on Kern AIs own infrastructure and is meant to be used to classify text sequences by the labels `keyword-question`, `statement-question` or `interrogative-question`.

classifiers/question_type/question_type_classifier/__init__.py

@@ -0,0 +1,27 @@
+from pydantic import BaseModel
+import requests
+
+INPUT_EXAMPLE = {
+    "text": "Sushi restaurants Barcelona",
+    "model_name": "KernAI/multilingual-e5-question-type",
+}
+
+
+class QuestionTypeClassifierModel(BaseModel):
+    text: str
+    model_name: str
+
+    class Config:
+        schema_extra = {"example": INPUT_EXAMPLE}
+
+
+def question_type_classifier(req: QuestionTypeClassifierModel):
+    """Uses custom E5 model to classify the question type of a text"""
+    payload = {
+        "model_name": req.model_name,
+        "text": req.text
+    }      
+    response = requests.post("https://free.api.kern.ai/inference", json=payload)
+    if response.ok:
+        return {"question_type": response.json()["label"]}
+    return response.raise_for_status()

classifiers/question_type/question_type_classifier/code_snippet_common.md

@@ -0,0 +1,34 @@
+```python
+import requests
+
+def question_type_classifier(text: str, model_name: str, request_url: str = "https://free.api.kern.ai/inference") -> str:
+    """
+    @param text: text with a user query you want to classify
+    @param model_name: Name of a model provided by Kern AI
+    @param request_url: URL to the API endpoint of Kern AI
+    @return: returns either 'keyword-question', 'interrogative-question' or 'statement-question' 
+    """
+    payload = {
+        "model_name": model_name,
+        "text": text
+    }      
+    response = requests.post(request_url, json=payload)
+    if response.ok:
+        return response.json()["label"]
+    return response.raise_for_status()
+
+
+# ↑ necessary bricks function 
+# -----------------------------------------------------------------------------------------
+# ↓ example implementation 
+
+
+model_name = "KernAI/multilingual-e5-question-type"
+
+def example_integration():
+    texts = ["Travel documents Germany", "Give me documents related to travel insurance.", "What is the content of these documents about?"]
+    for text in texts:
+        print(f"the question type of \"{text}\" is \"{question_type_classifier(text, model_name=model_name)}\"")
+
+example_integration()
+```

classifiers/question_type/question_type_classifier/code_snippet_refinery.md

@@ -0,0 +1,16 @@
+```python
+import requests
+
+ATTRIBUTE: str = "text" # only text attributes
+MODEL_NAME: str = "KernAI/multilingual-e5-question-type"
+REQUEST_URL: str = "https://free.api.kern.ai/inference"
+
+def question_type_classifier(record):
+    payload = {
+        "model_name": MODEL_NAME,
+        "text": record[ATTRIBUTE].text
+    }      
+    response = requests.post(REQUEST_URL, json=payload)
+    if response.ok:
+        return response.json()["label"]
+```

classifiers/question_type/question_type_classifier/config.py

@@ -0,0 +1,48 @@
+from util.configs import build_classifier_function_config
+from util.enums import State, RefineryDataType, BricksVariableType, SelectionType
+from . import question_type_classifier, INPUT_EXAMPLE
+
+
+def get_config():
+    return build_classifier_function_config(
+        function=question_type_classifier,
+        input_example=INPUT_EXAMPLE,
+        issue_id=344,
+        tabler_icon="ZoomQuestion",
+        min_refinery_version="1.7.0",
+        state=State.PUBLIC.value,
+        type="python_function",
+        available_for=["refinery", "common"],
+        part_of_group=[
+            "question_type"
+        ],  # first entry should be parent directory
+        # bricks integrator information
+        integrator_inputs={
+            "name": "question_type_classifier",
+            "refineryDataType": RefineryDataType.TEXT.value,
+            "outputs": ["keyword-question", "statement-question", "interrogative-question"],
+            "variables": {
+                "ATTRIBUTE": {
+                    "selectionType": SelectionType.CHOICE.value,
+                    "addInfo": [
+                        BricksVariableType.ATTRIBUTE.value,
+                        BricksVariableType.GENERIC_STRING.value
+                    ]
+                },
+                "MODEL_NAME": {
+                    "selectionType": SelectionType.STRING.value,
+                    "defaultValue": "KernAI/multilingual-e5-question-type",
+                    "addInfo": [
+                        BricksVariableType.GENERIC_STRING.value
+                    ]
+                },
+                "REQUEST_URL": {
+                    "selectionType": SelectionType.STRING.value,
+                    "defaultValue": "https://free.api.kern.ai/inference",
+                    "addInfo": [
+                        BricksVariableType.GENERIC_STRING.value
+                    ]
+                }
+            }
+        }
+    )

classifiers/reference_quality/chunked_sentence_complexity/README.md renamed to classifiers/reference_complexity/chunked_sentence_complexity/README.md

@@ -14,18 +14,18 @@ def get_config():
         type="python_function",
         available_for=["refinery", "common"],
         part_of_group=[
-            "reference_quality",
+            "reference_complexity",
             "text_analysis"
         ],  # first entry should be parent directory
         # bricks integrator information
         cognition_init_mapping={
-            "very easy": "null",
-            "easy": "null",
-            "fairly easy": "null",
-            "standard": "null",
-            "fairly difficult": "Needs fix",
-            "difficult": "Needs fix",
-            "very difficult": "Needs fix",
+            "very easy": "Low",
+            "easy": "Low",
+            "fairly easy": "Low",
+            "standard": "Medium",
+            "fairly difficult": "Medium",
+            "difficult": "High",
+            "very difficult": "High",
         },
         integrator_inputs={
             "name": "chunked_sentence_complexity",

classifiers/reference_quality/chunked_sentence_complexity/__init__.py renamed to classifiers/reference_complexity/chunked_sentence_complexity/__init__.py

@@ -0,0 +1,2 @@
+This simple brick counts the amount of words in a string by splitting it. If there are less than five words found, it returns "short". If there are less than 20, it returns "medium" 
+and with more than 20, it returns "long". This brick can be used to check the quality of references for RAG (Retrieval Augmented Generation) use cases.

classifiers/reference_quality/chunked_sentence_complexity/code_snippet_common.md renamed to classifiers/reference_complexity/chunked_sentence_complexity/code_snippet_common.md

@@ -0,0 +1,22 @@
+from pydantic import BaseModel
+
+INPUT_EXAMPLE = {"text": "This is too short!"}
+
+
+class WordCountClassifierModel(BaseModel):
+    text: str
+
+    class Config:
+        schema_extra = {"example": INPUT_EXAMPLE}
+
+
+def word_count_classifier(req: WordCountClassifierModel):
+    """Checks the length of a string by counting the number of words in it"""
+    words = req.text.split()
+    length = len(words)
+    if length < 5:
+        return {"text_length": "short"}
+    elif length < 20:
+        return {"text_length": "medium"}
+    else:
+        return {"text_length": "long"}

classifiers/reference_quality/chunked_sentence_complexity/code_snippet_refinery.md renamed to classifiers/reference_complexity/chunked_sentence_complexity/code_snippet_refinery.md

@@ -0,0 +1,26 @@
+```python
+def word_count_classifier(text: str) -> str:
+    """
+    @param text: text to check the length of.
+    @return: either 'short', 'medium' or 'long' depending on the counted words.
+    """
+    words = text.split()
+    length = len(words)
+    if length < 5:
+          return "short"
+    elif length < 20:
+          return "medium"
+    else:
+          return "long"
+
+# ↑ necessary bricks function 
+# -----------------------------------------------------------------------------------------
+# ↓ example implementation 
+
+def example_integration():
+    texts = ["This is short.", "This is a text with medium length.", "This is a longer text with many more words. There is even a second sentence with extra words. Splendid, what a joyful day!"]
+    for text in texts:
+        print(f"\"{text}\" is -> {word_count_classifier(text)}")
+
+example_integration()
+```