Merge remote-tracking branch 'origin/main' into communication-style-classifier

Author: LeonardPuettmannKern

classifiers/__init__.py

@@ -9,8 +9,8 @@
 
 from .lookup_lists import lookup_list
 
-from .communication_style import (
-    communication_style_classifier
+from .question_type import (
+    question_type_classifier
 )
 
 from .reference_quality import (
@@ -67,7 +67,7 @@
     bert_sentiment_german,
     special_character_classifier,
     chunked_sentence_complexity,
-    communication_style_classifier
+    question_type_classifier
 ]:
     module_name = module.__name__.split(".")[-1]
     model_name = (

classifiers/question_type/question_type_classifier/README.md

@@ -0,0 +1 @@
+Uses an `intfloat/multilingual-e5-small` model, which was finetuned on english and german examples of different question types. The model is hosted on Kern AIs own infrastructure and is meant to be used to classify text sequences by the labels `keyword-question`, `statement-question` or `interrogative-question`.

classifiers/question_type/question_type_classifier/__init__.py

@@ -0,0 +1,27 @@
+from pydantic import BaseModel
+import requests
+
+INPUT_EXAMPLE = {
+    "text": "Sushi restaurants Barcelona",
+    "model_name": "KernAI/multilingual-e5-question-type",
+}
+
+
+class QuestionTypeClassifierModel(BaseModel):
+    text: str
+    model_name: str
+
+    class Config:
+        schema_extra = {"example": INPUT_EXAMPLE}
+
+
+def question_type_classifier(req: QuestionTypeClassifierModel):
+    """Uses custom E5 model to classify the question type of a text"""
+    payload = {
+        "model_name": req.model_name,
+        "text": req.text
+    }      
+    response = requests.post("https://free.api.kern.ai/inference", json=payload)
+    if response.ok:
+        return {"question_type": response.json()["label"]}
+    return response.raise_for_status()

classifiers/question_type/question_type_classifier/code_snippet_common.md

@@ -0,0 +1,34 @@
+```python
+import requests
+
+def question_type_classifier(text: str, model_name: str, request_url: str = "https://free.api.kern.ai/inference") -> str:
+    """
+    @param text: text with a user query you want to classify
+    @param model_name: Name of a model provided by Kern AI
+    @param request_url: URL to the API endpoint of Kern AI
+    @return: returns either 'keyword-question', 'interrogative-question' or 'statement-question' 
+    """
+    payload = {
+        "model_name": model_name,
+        "text": text
+    }      
+    response = requests.post(request_url, json=payload)
+    if response.ok:
+        return response.json()["label"]
+    return response.raise_for_status()
+
+
+# ↑ necessary bricks function 
+# -----------------------------------------------------------------------------------------
+# ↓ example implementation 
+
+
+model_name = "KernAI/multilingual-e5-question-type"
+
+def example_integration():
+    texts = ["Travel documents Germany", "Give me documents related to travel insurance.", "What is the content of these documents about?"]
+    for text in texts:
+        print(f"the question type of \"{text}\" is \"{question_type_classifier(text, model_name=model_name)}\"")
+
+example_integration()
+```

classifiers/question_type/question_type_classifier/code_snippet_refinery.md

@@ -0,0 +1,16 @@
+```python
+import requests
+
+ATTRIBUTE: str = "text" # only text attributes
+MODEL_NAME: str = "KernAI/multilingual-e5-question-type"
+REQUEST_URL: str = "https://free.api.kern.ai/inference"
+
+def question_type_classifier(record):
+    payload = {
+        "model_name": MODEL_NAME,
+        "text": record[ATTRIBUTE].text
+    }      
+    response = requests.post(REQUEST_URL, json=payload)
+    if response.ok:
+        return response.json()["label"]
+```

classifiers/question_type/question_type_classifier/config.py

@@ -0,0 +1,48 @@
+from util.configs import build_classifier_function_config
+from util.enums import State, RefineryDataType, BricksVariableType, SelectionType
+from . import question_type_classifier, INPUT_EXAMPLE
+
+
+def get_config():
+    return build_classifier_function_config(
+        function=question_type_classifier,
+        input_example=INPUT_EXAMPLE,
+        issue_id=344,
+        tabler_icon="ZoomQuestion",
+        min_refinery_version="1.7.0",
+        state=State.PUBLIC.value,
+        type="python_function",
+        available_for=["refinery", "common"],
+        part_of_group=[
+            "question_type"
+        ],  # first entry should be parent directory
+        # bricks integrator information
+        integrator_inputs={
+            "name": "question_type_classifier",
+            "refineryDataType": RefineryDataType.TEXT.value,
+            "outputs": ["keyword-question", "statement-question", "interrogative-question"],
+            "variables": {
+                "ATTRIBUTE": {
+                    "selectionType": SelectionType.CHOICE.value,
+                    "addInfo": [
+                        BricksVariableType.ATTRIBUTE.value,
+                        BricksVariableType.GENERIC_STRING.value
+                    ]
+                },
+                "MODEL_NAME": {
+                    "selectionType": SelectionType.STRING.value,
+                    "defaultValue": "KernAI/multilingual-e5-question-type",
+                    "addInfo": [
+                        BricksVariableType.GENERIC_STRING.value
+                    ]
+                },
+                "REQUEST_URL": {
+                    "selectionType": SelectionType.STRING.value,
+                    "defaultValue": "https://free.api.kern.ai/inference",
+                    "addInfo": [
+                        BricksVariableType.GENERIC_STRING.value
+                    ]
+                }
+            }
+        }
+    )