Updates regex expressions

Author: JWittmeyer

extractors/numbers/percentage_extraction/__init__.py

@@ -8,42 +8,25 @@
     "spacyTokenizer": "en_core_web_sm",
 }
 
-
 class PercentageExtractionModel(BaseModel):
     text: str
-    regex: str = r"(-?\d+(?:[.,]\d*)?|-?[.,]\d+)%"
     spacyTokenizer: str = "en_core_web_sm"
-    yourLabel: str = "percentage"
 
     class Config:
         schema_extra = {"example": INPUT_EXAMPLE}
 
 
 def percentage_extraction(request: PercentageExtractionModel):
-    """Extracts percentages from a given text."""
-    nlp = SpacySingleton.get_nlp(request.spacyTokenizer)
-    doc = nlp(request.text)
-
-    matches = []
-
-    def regex_search(pattern, string):
-        """
-        some helper function to easily iterate over regex matches
-        """
-        prev_end = 0
-        while True:
-            match = re.search(pattern, string)
-            if not match:
-                break
+    """Extracts the Percentages from a text"""
 
-            start, end = match.span()
-            yield start + prev_end, end + prev_end
-
-            prev_end += end
-            string = string[end:]
+    text = request.text
+    nlp = SpacySingleton.get_nlp(request.spacyTokenizer)
+    doc = nlp(text)
+    regex = re.compile(r"(?:[\d-]{17}|[\d-]{13})")
 
-    for start, end in regex_search(request.regex, request.text):
+    p = []
+    for match in regex.finditer(text):
+        start, end = match.span()
         span = doc.char_span(start, end, alignment_mode="expand")
-        matches.append([request.yourLabel, span.start, span.end])
-
-    return {f"{request.yourLabel}s": matches}
+        p.append([span.start, span.end, span.text])
+    return {"percentages": p}

extractors/numbers/percentage_extraction/code_snippet_common.md

@@ -3,44 +3,39 @@ import re
 import spacy
 from typing import List, Tuple
 
-def percentage_extraction(text: str, extraction_keyword: str, regex_pattern: str) -> List[Tuple[str, int]]:
-
-    def regex_search(pattern, string):
-        prev_end = 0
-        while True:
-            match = re.search(pattern, string)
-            if not match:
-                break
-
-            start_, end_ = match.span()
-            yield start_ + prev_end, end_ + prev_end
-
-            prev_end += end_
-            string = string[end_:]
-
+def percentage_extraction(text: str, extraction_keyword:str) -> List[Tuple[str, int, int]]:
+    """
+    @param text: the input text
+    @param extraction_keyword: the label that is assigned to extracted words
+    @return: positions of extracted percentages
+    """
     nlp = spacy.load("en_core_web_sm")
     doc = nlp(text)
 
-    percentage_positions = []  
-    for start, end in regex_search(regex_pattern, text):
+    regex = re.compile(r"(-?\d+(?:[.,]\d*)?|-?[.,]\d+)%")
+    
+    isbn_positions = []
+    for match in regex.finditer(text):
+        start, end = match.span()
         span = doc.char_span(start, end, alignment_mode="expand")
-        percentage_positions.append((extraction_keyword, span.start, span.end))
-    return percentage_positions
+        isbn_positions.append((extraction_keyword, span.start, span.end))
+    return isbn_positions
 
 # ↑ necessary bricks function 
 # -----------------------------------------------------------------------------------------
 # ↓ example implementation
 
 def example_integration():
     texts = ["percentages 110% are found -.5% at 42,13% positions 1, 5 and 8", "Apple stock fell today."]
-    regex_pattern = r"(-?\d+(?:[.,]\d*)?|-?[.,]\d+)%"
     extraction_keyword = "percentage"
     for text in texts:
-        found = percentage_extraction(text, regex_pattern, extraction_keyword)
+        found = percentage_extraction(text, extraction_keyword)
         if found:
             print(f"text: \"{text}\" has {extraction_keyword} -> \"{found}\"")
         else:
             print(f"text: \"{text}\" doesn't have {extraction_keyword}")
 
 example_integration()
+
+
 ```

extractors/numbers/percentage_extraction/code_snippet_refinery.md

@@ -2,25 +2,14 @@
 import re
 
 ATTRIBUTE: str = "text" # only text attributes
-REGEX: str = r"(-?\d+(?:[.,]\d*)?|-?[.,]\d+)%" # this will capture all percentages
-LABEL: str = "percentage" # Choose any available label here
+LABEL: str = "percentage"
 
 def percentage_extraction(record):
+    regex = re.compile(r"(-?\d+(?:[.,]\d*)?|-?[.,]\d+)%")
+    text = record[ATTRIBUTE].text # SpaCy doc, hence we need to use .text to get the string
 
-    def regex_search(pattern, string):
-        prev_end = 0
-        while True:
-            match = re.search(pattern, string)
-            if not match:
-                break
-
-            start_, end_ = match.span()
-            yield start_ + prev_end, end_ + prev_end
-
-            prev_end += end_
-            string = string[end_:]
-            
-    for start, end in regex_search(REGEX, record[ATTRIBUTE].text):
+    for match in regex.finditer(text):
+        start, end = match.span()
         span = record[ATTRIBUTE].char_span(start, end, alignment_mode="expand")
         yield LABEL, span.start, span.end
 ```

extractors/paths/url_extraction/__init__.py

@@ -22,7 +22,7 @@ def url_extraction(request: UrlExtractionModel):
     nlp = SpacySingleton.get_nlp(request.spacyTokenizer)
     doc = nlp(text)
 
-    regex_pattern = re.compile(r"(?:(?:https?|ftp):\/\/)?[\w/\-?=%.]+\.[\w/\-&?=%.]+")
+    regex_pattern = re.compile(r"(?:(?:(?:https?|ftp):\/\/){1})?[\w\-\/?=%.]{3,}\.[\/\w\-&?=%.]{2,}")
     regex_pattern.findall(text)
 
     urls = []

extractors/paths/url_extraction/code_snippet_common.md

@@ -12,7 +12,7 @@ def url_extraction(text: str, extraction_keyword: str) -> List[Tuple[str, int]]:
     npl = spacy.load("en_core_web_sm")
     doc = npl(text)
 
-    regex_pattern = re.compile(r"(?:(?:https?|ftp):\/\/)?[\w/\-?=%.]+\.[\w/\-&?=%.]+")  
+    regex_pattern = re.compile(r"(?:(?:(?:https?|ftp):\/\/){1})?[\w\-\/?=%.]{3,}\.[\/\w\-&?=%.]{2,}")  
 
     url_positions = []   
     for match in regex_pattern.finditer(text):

extractors/paths/url_extraction/code_snippet_refinery.md

@@ -6,7 +6,7 @@ LABEL: str = "url"
 
 def url_extraction(record):
     text = record[ATTRIBUTE].text # SpaCy doc, hence we need to use .text to get the string.
-    regex_pattern = re.compile(r"(?:(?:https?|ftp):\/\/)?[\w/\-?=%.]+\.[\w/\-&?=%.]+")
+    regex_pattern = re.compile(r"(?:(?:(?:https?|ftp):\/\/){1})?[\w\-\/?=%.]{3,}\.[\/\w\-&?=%.]{2,}")
 
     for match in regex_pattern.finditer(text):
         start, end = match.span()

extractors/words/goodbye_extraction/__init__.py

@@ -25,7 +25,7 @@ def goodbye_extraction(request: GoodbyeExtractionModel):
     text = request.text
     nlp = SpacySingleton.get_nlp(request.spacyTokenizer)
     doc = nlp(text)
-    regex = re.compile(r"((?:((?i)good)(?:[ ])?)?((?i)bye)|(?i)Ciao|(?:((?i)see you)(?:[ ]?)((?i)tomorrow|later|soon)?))")
+    regex = re.compile(r"((?:(good)(?:[ ])?)?(bye)|Ciao|(?:(see you)(?:[ ]?)(tomorrow|later|soon)?))", re.IGNORECASE)
 
     farewell = []
     for match in regex.finditer(text):

extractors/words/goodbye_extraction/code_snippet_common.md

@@ -12,7 +12,7 @@ def goodbye_extraction(text: str, extraction_keyword: str) -> List[Tuple[str,int
     nlp = spacy.load("en_core_web_sm")
     doc = nlp(text)
 
-    regex = re.compile(r"((?:((?i)good)(?:[ ])?)?((?i)bye)|(?i)Ciao|(?:((?i)see you)(?:[ ]?)((?i)tomorrow|later|soon)?))")
+    regex = re.compile(r"((?:(good)(?:[ ])?)?(bye)|Ciao|(?:(see you)(?:[ ]?)(tomorrow|later|soon)?))", re.IGNORECASE)
 
     goodbye_positions = []
     for match in regex.finditer(text):

extractors/words/goodbye_extraction/code_snippet_refinery.md

@@ -5,7 +5,7 @@ ATTRIBUTE: str = "text" # only text attributes
 LABEL: str = "goodbye"
 
 def goodbye_extraction(record):
-    regex = re.compile(r"((?:((?i)good)(?:[ ])?)?((?i)bye)|(?i)Ciao|(?:((?i)see you)(?:[ ]?)((?i)tomorrow|later|soon)?))")
+    regex = re.compile(r"((?:(good)(?:[ ])?)?(bye)|Ciao|(?:(see you)(?:[ ]?)(tomorrow|later|soon)?))", re.IGNORECASE)
     text = record[ATTRIBUTE].text # SpaCy doc, hence we need to use .text to get the string.
 
     for match in regex.finditer(text):