add setup.py

Author: phueb

aonewsela/__init__.py

@@ -0,0 +1,3 @@
+
+
+__version__ = '1.0.0'

aonewsela/configs.py

@@ -4,4 +4,5 @@
 class Dirs:
     src = Path(__file__).parent
     root = src.parent
-    ludwig_data = '/media/ludwig_data'  # path to location of shared drive, mounted on local machine
+    mnt = Path('/media')
+    ludwig_data = mnt / 'ludwig_data'  # path to location of shared drive, mounted on local machine

aonewsela/pipeline.py

@@ -16,29 +16,46 @@ def load_age_ordered_articles(self,
                                   path_ludwig_data: Optional[Path] = None,
                                   ) -> List[Transcript]:
 
+        """
+        notes:
+         age-ordering articles requires loading highest version first.
+          (higher version -> higher simplification -> smaller grade level -> younger students)
+        """
+
+        def path_to_version(p: Path):
+            return int(p.stem.split('.')[-1])
+
         if path_ludwig_data is None:
             path_ludwig_data = configs.Dirs.ludwig_data
 
-        print('Preparing AONewsela articles...')
-        pbar = pyprind.ProgBar(6, stream=1)
+
+        path_data = path_ludwig_data / 'AONewsela' / 'newsela_article_corpus_2016-01-29'
+        print(f'Looking for articles in {path_data}')
+        if not path_data.exists():
+            raise FileNotFoundError(f'Did not find {path_data}.'
+                                    f' Do you have access to the UIUC Language Learning Lab shared drive?')
+
+        # search directory once, to save time
+        article_paths = [p for p in path_data.rglob('*.en.*.txt')]
+        if not article_paths:
+            raise RuntimeError(f'Did not find any articles in {article_paths}')
+
+        print(f'Preparing {len(article_paths)} AONewsela articles...')
+        pbar = pyprind.ProgBar(len(article_paths), stream=1)
 
         res = []
-        for version in reversed(range(6)):  # simple first means highest version first
+        for path_article in sorted(article_paths, key=lambda p: path_to_version(p), reverse=True):
 
-            articles_path = path_ludwig_data / 'AONewsela' / 'newsela_article_corpus_2016-01-29'
-            print(f'Looking for articles in {articles_path}')
-            if not articles_path.exists():
-                raise FileNotFoundError(f'Did not find {articles_path}.'
-                                        f' Do you have access to the UIUC Language Learning Lab shared drive?')
+            text = path_article.read_text(encoding='utf-8').replace('\n', ' ').lower()
 
-            for path in articles_path.glob(f'*.en.{version}.txt'):
-                text = path.read_text(encoding='utf-8').replace('\n', ' ').lower()
+            if not self.params.punctuation:
+                raise NotImplementedError
 
-                if not self.params.punctuation:
-                    raise NotImplementedError
+            res.append(Transcript(text, path_to_version(path_article)))
 
-                res.append(Transcript(text, version))
+        pbar.update()
 
-            pbar.update()
+        print(f'Found {len(res)} articles', flush=True)
+        assert len(res) == len(article_paths)
 
         return res

requirements.txt

@@ -0,0 +1,18 @@
+from setuptools import setup
+
+from aonewsela import __name__, __version__
+
+setup(
+    name=__name__,
+    version=__version__,
+    packages=[__name__],
+    include_package_data=True,
+    install_requires=[
+        'pyprind',
+    ],
+    url='https://github.com/UIUCLearningLanguageLab/AOCHILDES',
+    license='',
+    author='Philip Huebner',
+    author_email='info@philhuebner.com',
+    description='Retrieve text from the American-English CHILDES database'
+)