Add compatibility with python versions and create tests for it

Author: anamatoso

.github/workflows/python-package.yml

@@ -0,0 +1,41 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: Python package
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+  
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip setuptools wheel
+        python -m pip install flake8 pytest
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+    - name: Lint with flake8
+      run: |
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+    - name: Test with pytest
+      run: |
+        PYTHONPATH=. pytest
+

README.md

@@ -1,3 +1,8 @@
+[![Python 3.8](https://img.shields.io/badge/python-3.8-blue.svg)](https://www.python.org/downloads/release/python-360/)
+[![Python 3.9](https://img.shields.io/badge/python-3.9-blue.svg)](https://www.python.org/downloads/release/python-360/)
+[![Python 3.10](https://img.shields.io/badge/python-3.10-blue.svg)](https://www.python.org/downloads/release/python-360/)
+[![Python 3.11](https://img.shields.io/badge/python-3.11-blue.svg)](https://www.python.org/downloads/release/python-360/)
+
 arXivCollector
 ======
 

arxivcollector.py

@@ -68,8 +68,8 @@ def find_data(self, soup: BeautifulSoup, keyword) -> str:
         for p in soup.select('p'):
             if p.getText(strip=True).startswith(keyword):
                 temp = p.getText(strip=True).split(';')
-                sub = temp[0].strip().removeprefix('Submitted')
-                ann = temp[-1].strip().removeprefix('originally announced')
+                sub = temp[0].strip()[len('Submitted'):]
+                ann = temp[-1].strip()[len('originally announced'):]
                 # Convert sub to a datetime object
                 sub = datetime.datetime.strptime(sub, "%d %B, %Y")
                 break
@@ -88,7 +88,11 @@ def parse_html(self,response:httpx.Response):
             temp_authors = li.select('p.authors>a')
             authors = ' AND '.join([', '.join(j.getText(strip=True).split()[::-1]) for j in temp_authors])
 
-            Abstract = self.extract_text(li,'span.abstract-full').removesuffix('△ Less')
+            abstract_text = self.extract_text(li, 'span.abstract-full')
+            if abstract_text:
+                Abstract = abstract_text[:-len('△ Less')]
+            else:
+                Abstract = ''
 
             extracted_text = self.extract_text(li,'p.comments > span:nth-of-type(2)')
             note = extracted_text if extracted_text else ""

tests/test_generate_csv.py

@@ -0,0 +1,15 @@
+from pathlib import Path
+
+from example import create
+
+
+def test_create_csv():
+    url = 'https://arxiv.org/search/advanced?advanced=&terms-0-operator=AND&terms-0-term=stochastic+parrot&terms-0-field=title&classification-physics_archives=all&classification-include_cross_list=include&date-filter_by=all_dates&date-year=&date-from_date=&date-to_date=&date-date_type=submitted_date&abstracts=show&size=50&order=-announced_date_first'
+    create(url, "output", "csv")
+    create(url, "output", "bibtex")
+
+    # Check if file exists
+    filenamecsv = "output.csv"
+    filenamebib = "output.bib"
+    assert Path(filenamecsv).exists(), "csv file was not created."
+    assert Path(filenamebib).exists(), "bibtex file was not created."