Skip to content

Update and rename crawler.yml to crawl.yml #1

Update and rename crawler.yml to crawl.yml

Update and rename crawler.yml to crawl.yml #1

Workflow file for this run

name: Web Crawler
on:
push:
branches:
- main
jobs:
crawl_and_commit:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.x'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install requests beautifulsoup4
- name: Run web crawling script
run: python crawl.py
- name: Commit crawled files
uses: stefanzweifel/git-auto-commit-action@v5
with:
commit_message: 'Add new crawled data'
branch: main
# Add the output directory to the git commit
file_pattern: 'crawled_output/*'