Merge pull request #388 from code-kern-ai/regex-changes

Updates regex expressions

Author: JWittmeyer

extractors/numbers/ip_extraction/__init__.py

@@ -22,7 +22,6 @@ def ip_extraction(request: IpExtractionModel):
     nlp = SpacySingleton.get_nlp(request.spacyTokenizer)
     doc = nlp(text)
     regex = re.compile(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b")
-    regex.findall(text)
 
     ip_addresses = []
     for match in regex.finditer(text):

extractors/numbers/isbn_extraction/__init__.py

@@ -28,5 +28,5 @@ def isbn_extraction(request: IsbnExtractionModel):
     for match in regex.finditer(text):
         start, end = match.span()
         span = doc.char_span(start, end, alignment_mode="expand")
-        isbn.append([span.start, span.end, span.text])
+        isbn.append(["isbn", span.start, span.end])
     return {"isbn": isbn}

extractors/numbers/percentage_extraction/__init__.py

@@ -1,4 +1,3 @@
-
 from pydantic import BaseModel
 from extractors.util.spacy import SpacySingleton
 import re
@@ -11,39 +10,22 @@
 
 class PercentageExtractionModel(BaseModel):
     text: str
-    regex: str = r"(-?\d+(?:[.,]\d*)?|-?[.,]\d+)%"
     spacyTokenizer: str = "en_core_web_sm"
-    yourLabel: str = "percentage"
 
     class Config:
         schema_extra = {"example": INPUT_EXAMPLE}
 
 
 def percentage_extraction(request: PercentageExtractionModel):
-    """Extracts percentages from a given text."""
-    nlp = SpacySingleton.get_nlp(request.spacyTokenizer)
-    doc = nlp(request.text)
-
-    matches = []
-
-    def regex_search(pattern, string):
-        """
-        some helper function to easily iterate over regex matches
-        """
-        prev_end = 0
-        while True:
-            match = re.search(pattern, string)
-            if not match:
-                break
+    """Extracts the Percentages from a text"""
 
-            start, end = match.span()
-            yield start + prev_end, end + prev_end
-
-            prev_end += end
-            string = string[end:]
-
-    for start, end in regex_search(request.regex, request.text):
+    text = request.text
+    nlp = SpacySingleton.get_nlp(request.spacyTokenizer)
+    doc = nlp(text)
+    regex = re.compile(r"(-?\d+(?:[.,]\d*)?|-?[.,]\d+)\s*%")
+    percentages = []
+    for match in regex.finditer(text):
+        start, end = match.span()
         span = doc.char_span(start, end, alignment_mode="expand")
-        matches.append([request.yourLabel, span.start, span.end])
-
-    return {f"{request.yourLabel}s": matches}
+        percentages.append(["percentage", span.start, span.end])
+    return {"percentages": percentages}

extractors/numbers/percentage_extraction/code_snippet_common.md

@@ -3,26 +3,20 @@ import re
 import spacy
 from typing import List, Tuple
 
-def percentage_extraction(text: str, extraction_keyword: str, regex_pattern: str) -> List[Tuple[str, int]]:
-
-    def regex_search(pattern, string):
-        prev_end = 0
-        while True:
-            match = re.search(pattern, string)
-            if not match:
-                break
-
-            start_, end_ = match.span()
-            yield start_ + prev_end, end_ + prev_end
-
-            prev_end += end_
-            string = string[end_:]
-
+def percentage_extraction(text: str, extraction_keyword:str) -> List[Tuple[str, int, int]]:
+    """
+    @param text: the input text
+    @param extraction_keyword: the label that is assigned to extracted words
+    @return: positions of extracted percentages
+    """
     nlp = spacy.load("en_core_web_sm")
     doc = nlp(text)
 
-    percentage_positions = []  
-    for start, end in regex_search(regex_pattern, text):
+    regex = re.compile(r"(-?\d+(?:[.,]\d*)?|-?[.,]\d+)\s*%")
+    
+    percentage_positions = []
+    for match in regex.finditer(text):
+        start, end = match.span()
         span = doc.char_span(start, end, alignment_mode="expand")
         percentage_positions.append((extraction_keyword, span.start, span.end))
     return percentage_positions
@@ -33,14 +27,15 @@ def percentage_extraction(text: str, extraction_keyword: str, regex_pattern: str
 
 def example_integration():
     texts = ["percentages 110% are found -.5% at 42,13% positions 1, 5 and 8", "Apple stock fell today."]
-    regex_pattern = r"(-?\d+(?:[.,]\d*)?|-?[.,]\d+)%"
     extraction_keyword = "percentage"
     for text in texts:
-        found = percentage_extraction(text, regex_pattern, extraction_keyword)
+        found = percentage_extraction(text, extraction_keyword)
         if found:
             print(f"text: \"{text}\" has {extraction_keyword} -> \"{found}\"")
         else:
             print(f"text: \"{text}\" doesn't have {extraction_keyword}")
 
 example_integration()
+
+
 ```

extractors/numbers/percentage_extraction/code_snippet_refinery.md

@@ -2,25 +2,14 @@
 import re
 
 ATTRIBUTE: str = "text" # only text attributes
-REGEX: str = r"(-?\d+(?:[.,]\d*)?|-?[.,]\d+)%" # this will capture all percentages
-LABEL: str = "percentage" # Choose any available label here
+LABEL: str = "percentage"
 
 def percentage_extraction(record):
+    regex = re.compile(r"(-?\d+(?:[.,]\d*)?|-?[.,]\d+)\s*%")
+    text = record[ATTRIBUTE].text # SpaCy doc, hence we need to use .text to get the string
 
-    def regex_search(pattern, string):
-        prev_end = 0
-        while True:
-            match = re.search(pattern, string)
-            if not match:
-                break
-
-            start_, end_ = match.span()
-            yield start_ + prev_end, end_ + prev_end
-
-            prev_end += end_
-            string = string[end_:]
-            
-    for start, end in regex_search(REGEX, record[ATTRIBUTE].text):
+    for match in regex.finditer(text):
+        start, end = match.span()
         span = record[ATTRIBUTE].char_span(start, end, alignment_mode="expand")
         yield LABEL, span.start, span.end
 ```

extractors/numbers/percentage_extraction/config.py

@@ -31,20 +31,13 @@ def get_config():
                 },
                 "LABEL": {
                     "selectionType": SelectionType.CHOICE.value,
-                    "defaultValue": "isbn",
+                    "defaultValue": "percentage",
                     "optional": "false",
                     "addInfo": [
                         BricksVariableType.LABEL.value,
                         BricksVariableType.GENERIC_STRING.value,
                     ],
                 },
-                "REGEX": {
-                    "selectionType": SelectionType.STRING.value,
-                    "defaultValue": "(-?\d+(?:[.,]\d*)?|-?[.,]\d+)%",
-                    "description": "Choose any regex here",
-                    "optional": "false",
-                    "addInfo": [BricksVariableType.REGEX.value],
-                },
             },
         },
     )

extractors/paths/url_extraction/__init__.py

@@ -22,7 +22,7 @@ def url_extraction(request: UrlExtractionModel):
     nlp = SpacySingleton.get_nlp(request.spacyTokenizer)
     doc = nlp(text)
 
-    regex_pattern = re.compile(r"(?:(?:https?|ftp):\/\/)?[\w/\-?=%.]+\.[\w/\-&?=%.]+")
+    regex_pattern = re.compile(r"(?:(?:(?:https?|ftp):\/\/){1})?[\w\-\/?=%.]{3,}\.[\/\w\-&?=%.]{2,}")
     regex_pattern.findall(text)
 
     urls = []

extractors/paths/url_extraction/code_snippet_common.md

@@ -12,7 +12,7 @@ def url_extraction(text: str, extraction_keyword: str) -> List[Tuple[str, int]]:
     npl = spacy.load("en_core_web_sm")
     doc = npl(text)
 
-    regex_pattern = re.compile(r"(?:(?:https?|ftp):\/\/)?[\w/\-?=%.]+\.[\w/\-&?=%.]+")  
+    regex_pattern = re.compile(r"(?:(?:(?:https?|ftp):\/\/){1})?[\w\-\/?=%.]{3,}\.[\/\w\-&?=%.]{2,}")  
 
     url_positions = []   
     for match in regex_pattern.finditer(text):

extractors/paths/url_extraction/code_snippet_refinery.md

@@ -6,7 +6,7 @@ LABEL: str = "url"
 
 def url_extraction(record):
     text = record[ATTRIBUTE].text # SpaCy doc, hence we need to use .text to get the string.
-    regex_pattern = re.compile(r"(?:(?:https?|ftp):\/\/)?[\w/\-?=%.]+\.[\w/\-&?=%.]+")
+    regex_pattern = re.compile(r"(?:(?:(?:https?|ftp):\/\/){1})?[\w\-\/?=%.]{3,}\.[\/\w\-&?=%.]{2,}")
 
     for match in regex_pattern.finditer(text):
         start, end = match.span()

extractors/words/goodbye_extraction/__init__.py

@@ -25,12 +25,12 @@ def goodbye_extraction(request: GoodbyeExtractionModel):
     text = request.text
     nlp = SpacySingleton.get_nlp(request.spacyTokenizer)
     doc = nlp(text)
-    regex = re.compile(r"((?:((?i)good)(?:[ ])?)?((?i)bye)|(?i)Ciao|(?:((?i)see you)(?:[ ]?)((?i)tomorrow|later|soon)?))")
+    regex = re.compile(r"((?:(good)(?:[ ])?)?(bye)|Ciao|(?:(see you)(?:[ ]?)(tomorrow|later|soon)?))", re.IGNORECASE)
 
     farewell = []
     for match in regex.finditer(text):
         start, end = match.span()
         span = doc.char_span(start, end, alignment_mode="expand")
-        farewell.append(["span", span.start, span.end])
+        farewell.append(["farewellWords", span.start, span.end])
 
     return {"farewellWords": farewell}