implement loading sentences

Author: phueb

aonewsela/__init__.py

@@ -1,3 +1,3 @@
 
 
-__version__ = '1.0.0'
+__version__ = '1.1.0'

aonewsela/dataset.py

@@ -35,7 +35,20 @@ def load_tokens(self) -> List[str]:
         return res
 
     def load_sentences(self) -> List[str]:
-        raise NotImplementedError
+
+        exceptions = {'u.s.', 'u.n.', 'st.', 'dr.', 'd.c.', 'jan.', 'feb.'}
+
+        sentences = []
+        tokens_in_sentence = []
+        for token in self.load_tokens():
+            if (token.endswith('.') or token.endswith('!') or token.endswith('?')) and token not in exceptions:
+                sentence = ' '.join(tokens_in_sentence) + ' ' + token
+                sentences.append(sentence)
+                tokens_in_sentence = []
+            else:
+                tokens_in_sentence.append(token)
+
+        return sentences
 
     def load_text(self) -> str:
         return ' '.join(self.load_tokens())

aonewsela/pipeline.py

@@ -46,12 +46,28 @@ def path_to_version(p: Path):
         res = []
         for path_article in sorted(article_paths, key=lambda p: path_to_version(p), reverse=True):
 
-            text = path_article.read_text(encoding='utf-8').replace('\n', ' ').lower()
+            # TODO many article start with a location followed by a dash - remove this
+
+            # filter article sub-headings
+            lines_filtered = []
+            for line in path_article.open(encoding='utf-8').readlines():
+
+                if line.startswith('##'):
+                    continue
+
+                if 'http' in line:  # TODO only exclude the sentence or link, not the entire line
+                    continue
+
+                if '<img' in line:
+                    continue
+
+                lines_filtered.append(line.rstrip('\n'))
 
             if not self.params.punctuation:
                 raise NotImplementedError
 
-            res.append(Transcript(text, path_to_version(path_article)))
+            text = ' '.join(lines_filtered)
+            res.append(Transcript(text.lower(), path_to_version(path_article)))
 
         pbar.update()