solves merge conflicts

Author: FelixKirschKern

classifiers/__init__.py

@@ -1,21 +1,27 @@
 from fastapi import APIRouter
 
 from .llm import (
-    gpt_classifier, 
+    gpt_classifier,
     deberta_review_classifier,
     bert_sentiment_german,
-    distilbert_stock_news_classifier
+    distilbert_stock_news_classifier,
 )
 
 from .lookup_lists import lookup_list
 
 from .reference_complexity import (
-    maximum_sentence_complexity
+    maximum_sentence_complexity,
+    tiktoken_length_classifier,
+    chunked_sentence_complexity,
 )
 
+from .question_type import question_type_classifier
+
+from .communication_style import communication_style_classifier
+
 from .reference_quality import (
+    word_count_classifier,
     special_character_classifier,
-    chunked_sentence_complexity,
 )
 
 from .dates_and_times import (
@@ -31,10 +37,6 @@
     cosine_similarity,
 )
 
-from .spelling import (
-    spelling_check,
-)
-
 from .text_analysis import (
     emotionality_detection,
     language_detection,
@@ -63,11 +65,15 @@
     textblob_subjectivity,
     distilbert_stock_news_classifier,
     workday_classifier,
-    deberta_review_classifier, 
+    deberta_review_classifier,
     bert_sentiment_german,
+    tiktoken_length_classifier,
+    word_count_classifier,
     special_character_classifier,
     chunked_sentence_complexity,
-    maximum_sentence_complexity
+    maximum_sentence_complexity,
+    question_type_classifier,
+    communication_style_classifier,
 ]:
     module_name = module.__name__.split(".")[-1]
     model_name = (

classifiers/communication_style/communication_style_classifier/README.md

@@ -0,0 +1 @@
+Uses an `intfloat/multilingual-e5-small` model, which was finetuned on english and german examples of different question types. The model is hosted on Kern AIs own infrastructure and is meant to be used to classify text sequences by the labels `Action-seeking`, `Fact-oriented`, `Information-seeking` or `Self-revealing`.

classifiers/communication_style/communication_style_classifier/__init__.py

@@ -0,0 +1,27 @@
+from pydantic import BaseModel
+import requests
+
+INPUT_EXAMPLE = {
+    "text": "Change the number in row 2 and 3.",
+    "model_name": "KernAI/multilingual-e5-communication-style",
+}
+
+
+class CommunicationStyleClassifierModel(BaseModel):
+    text: str
+    model_name: str
+
+    class Config:
+        schema_extra = {"example": INPUT_EXAMPLE}
+
+
+def communication_style_classifier(req: CommunicationStyleClassifierModel):
+    """Uses custom E5 model to classify communication style of a text"""
+    payload = {
+        "model_name": req.model_name,
+        "text": req.text
+    }      
+    response = requests.post("https://free.api.kern.ai/inference", json=payload)
+    if response.ok:
+        return {"communication_style": response.json()["label"]}
+    return response.raise_for_status()

classifiers/communication_style/communication_style_classifier/code_snippet_common.md

@@ -0,0 +1,34 @@
+```python
+import requests
+
+def communication_style_classifier(text: str, model_name: str, request_url: str = "https://free.api.kern.ai/inference") -> str:
+    """
+    @param text: text with a user query you want to classify
+    @param model_name: Name of a model provided by Kern AI
+    @param request_url: URL to the API endpoint of Kern AI
+    @return: returns either 'Action-seeking', 'Fact-oriented', 'Information-seeking' or 'Self-revealing'.
+    """
+    payload = {
+        "model_name": model_name,
+        "text": text
+    }      
+    response = requests.post(request_url, json=payload)
+    if response.ok:
+        return response.json()["label"]
+    return response.raise_for_status()
+
+
+# ↑ necessary bricks function 
+# -----------------------------------------------------------------------------------------
+# ↓ example implementation 
+
+
+model_name = "KernAI/multilingual-e5-communication-style"
+
+def example_integration():
+    texts = ["Change the number in row 2 and 3.", "Can you show me some data from the references?", "I am super happy today."]
+    for text in texts:
+        print(f"the communication style of \"{text}\" is \"{communication_style_classifier(text, model_name=model_name)}\"")
+
+example_integration()
+```

classifiers/communication_style/communication_style_classifier/code_snippet_refinery.md

@@ -0,0 +1,16 @@
+```python
+import requests
+
+ATTRIBUTE: str = "text" # only text attributes
+MODEL_NAME: str = "KernAI/multilingual-e5-communication-style"
+REQUEST_URL: str = "https://free.api.kern.ai/inference"
+
+def communication_style_classifier(record):
+    payload = {
+        "model_name": MODEL_NAME,
+        "text": record[ATTRIBUTE].text
+    }      
+    response = requests.post(REQUEST_URL, json=payload)
+    if response.ok:
+        return response.json()["label"]
+```

classifiers/communication_style/communication_style_classifier/config.py

@@ -0,0 +1,48 @@
+from util.configs import build_classifier_function_config
+from util.enums import State, RefineryDataType, BricksVariableType, SelectionType
+from . import communication_style_classifier, INPUT_EXAMPLE
+
+
+def get_config():
+    return build_classifier_function_config(
+        function=communication_style_classifier,
+        input_example=INPUT_EXAMPLE,
+        issue_id=343,
+        tabler_icon="CircleDotted",
+        min_refinery_version="1.7.0",
+        state=State.PUBLIC.value,
+        type="python_function",
+        available_for=["refinery", "common"],
+        part_of_group=[
+            "communication_style"
+        ],  # first entry should be parent directory
+        # bricks integrator information
+        integrator_inputs={
+            "name": "communication_style_classifier",
+            "refineryDataType": RefineryDataType.TEXT.value,
+            "outputs": ["Action-seeking", "Fact-oriented", "Information-seeking", "Self-revealing"],
+            "variables": {
+                "ATTRIBUTE": {
+                    "selectionType": SelectionType.CHOICE.value,
+                    "addInfo": [
+                        BricksVariableType.ATTRIBUTE.value,
+                        BricksVariableType.GENERIC_STRING.value
+                    ]
+                },
+                "MODEL_NAME": {
+                    "selectionType": SelectionType.STRING.value,
+                    "defaultValue": "KernAI/multilingual-e5-communication-style",
+                    "addInfo": [
+                        BricksVariableType.GENERIC_STRING.value
+                    ]
+                },
+                "REQUEST_URL": {
+                    "selectionType": SelectionType.STRING.value,
+                    "defaultValue": "https://free.api.kern.ai/inference",
+                    "addInfo": [
+                        BricksVariableType.GENERIC_STRING.value
+                    ]
+                }
+            }
+        }
+    )

classifiers/question_type/question_type_classifier/README.md

@@ -0,0 +1 @@
+Uses an `intfloat/multilingual-e5-small` model, which was finetuned on english and german examples of different question types. The model is hosted on Kern AIs own infrastructure and is meant to be used to classify text sequences by the labels `Keyword-question`, `Statement-question` or `Interrogative-question`.

classifiers/question_type/question_type_classifier/__init__.py

@@ -0,0 +1,27 @@
+from pydantic import BaseModel
+import requests
+
+INPUT_EXAMPLE = {
+    "text": "Sushi restaurants Barcelona",
+    "model_name": "KernAI/multilingual-e5-question-type",
+}
+
+
+class QuestionTypeClassifierModel(BaseModel):
+    text: str
+    model_name: str
+
+    class Config:
+        schema_extra = {"example": INPUT_EXAMPLE}
+
+
+def question_type_classifier(req: QuestionTypeClassifierModel):
+    """Uses custom E5 model to classify the question type of a text"""
+    payload = {
+        "model_name": req.model_name,
+        "text": req.text
+    }      
+    response = requests.post("https://free.api.kern.ai/inference", json=payload)
+    if response.ok:
+        return {"question_type": response.json()["label"]}
+    return response.raise_for_status()

classifiers/question_type/question_type_classifier/code_snippet_common.md

@@ -0,0 +1,34 @@
+```python
+import requests
+
+def question_type_classifier(text: str, model_name: str, request_url: str = "https://free.api.kern.ai/inference") -> str:
+    """
+    @param text: text with a user question you want to classify
+    @param model_name: Name of a model provided by Kern AI
+    @param request_url: URL to the API endpoint of Kern AI
+    @return: returns either 'Keyword-question', 'Interrogative-question' or 'Statement-question' 
+    """
+    payload = {
+        "model_name": model_name,
+        "text": text
+    }      
+    response = requests.post(request_url, json=payload)
+    if response.ok:
+        return response.json()["label"]
+    return response.raise_for_status()
+
+
+# ↑ necessary bricks function 
+# -----------------------------------------------------------------------------------------
+# ↓ example implementation 
+
+
+model_name = "KernAI/multilingual-e5-question-type"
+
+def example_integration():
+    texts = ["Travel documents Germany", "Give me documents related to travel insurance.", "What is the content of these documents about?"]
+    for text in texts:
+        print(f"the question type of \"{text}\" is \"{question_type_classifier(text, model_name=model_name)}\"")
+
+example_integration()
+```

classifiers/question_type/question_type_classifier/code_snippet_refinery.md

@@ -0,0 +1,16 @@
+```python
+import requests
+
+ATTRIBUTE: str = "text" # only text attributes
+MODEL_NAME: str = "KernAI/multilingual-e5-question-type"
+REQUEST_URL: str = "https://free.api.kern.ai/inference"
+
+def question_type_classifier(record):
+    payload = {
+        "model_name": MODEL_NAME,
+        "text": record[ATTRIBUTE].text
+    }      
+    response = requests.post(REQUEST_URL, json=payload)
+    if response.ok:
+        return response.json()["label"]
+```