From 8ead7af7b452f52893fadc5f5f7fbf98ea7bf428 Mon Sep 17 00:00:00 2001 From: Anas Iqbal Date: Sun, 13 Jul 2025 17:07:15 +0530 Subject: [PATCH 1/4] Add workflow for syncing course content and utility script for downloading course content --- .github/workflows/sync_content.yml | 55 ++++++++ sync-course.json | 9 ++ utils/download_and_sync_courses.py | 209 +++++++++++++++++++++++++++++ 3 files changed, 273 insertions(+) create mode 100644 .github/workflows/sync_content.yml create mode 100644 sync-course.json create mode 100644 utils/download_and_sync_courses.py diff --git a/.github/workflows/sync_content.yml b/.github/workflows/sync_content.yml new file mode 100644 index 0000000..50ffb6e --- /dev/null +++ b/.github/workflows/sync_content.yml @@ -0,0 +1,55 @@ +name: Sync Course Content on PR + +on: + push: + branches: [sync-course] + pull_request: + branches: [sync-course] + +env: + OMEGAUP_API_TOKEN: ${{ secrets.OMEGAUP_API_TOKEN }} + GIT_USERNAME: ${{ github.actor }} + +jobs: + sync-content: + runs-on: ubuntu-22.04 + + steps: + - name: Checkout repository with PR branch + uses: actions/checkout@v4 + with: + ref: ${{ github.head_ref }} + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.8' + + - name: Set up pipenv + run: | + python -m pip install --upgrade pip + pip install pipenv==2023.11.15 + + - name: Install Python dependencies with pipenv + run: | + cd utils + pipenv install + + - name: Run course download script + working-directory: utils + run: pipenv run python3 download_and_sync_courses.py + + - name: Commit and push changes + id: commit + run: | + git config --global user.name "github-actions" + git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add . + # Only commit if there are changes + if git diff --staged --quiet; then + echo "No changes to commit" + else + git commit -m "📝 Auto-sync: Downloaded latest OmegaUp content" + git push + fi diff --git a/sync-course.json b/sync-course.json new file mode 100644 index 0000000..0787379 --- /dev/null +++ b/sync-course.json @@ -0,0 +1,9 @@ +{ + "requests":[ + { + "omegaup_usernmae": "", + "github_username": "", + "requests_number": 0 + } + ] +} \ No newline at end of file diff --git a/utils/download_and_sync_courses.py b/utils/download_and_sync_courses.py new file mode 100644 index 0000000..fe766ff --- /dev/null +++ b/utils/download_and_sync_courses.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 + +import argparse +import json +import logging +import os +import zipfile +from typing import Dict, Any +from urllib.parse import urlparse, urljoin +import omegaup.api +import shutil +import http.client +import ssl + +# Create SSL context that skips certificate verification +context = ssl._create_unverified_context() + +logging.basicConfig(level=logging.INFO) +LOG = logging.getLogger(__name__) + +API_CLIENT = None +BASE_URL = None + +# 👇 Add your course aliases here +COURSE_ALIASES = [ + "omi-public-course" +] + +BASE_COURSE_FOLDER = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "Courses")) + + +if os.path.exists(BASE_COURSE_FOLDER): + LOG.warning("Delete existing course folder to avoid conflicts") + shutil.rmtree(BASE_COURSE_FOLDER) + + + + +def handle_input(): + global BASE_URL, API_TOKEN + parser = argparse.ArgumentParser(description="Download and extract problems from multiple course assignments") + parser.add_argument("--url", default="https://omegaup.com", help="omegaUp base URL") + parser.add_argument("--api-token", type=str, default=os.environ.get("OMEGAUP_API_TOKEN"), required=("OMEGAUP_API_TOKEN" not in os.environ)) + args = parser.parse_args() + BASE_URL = args.url + return args.api_token + + + +def get_json(endpoint: str, params: Dict[str, str]) -> Dict[str, Any]: + return API_CLIENT.query(endpoint, params) + + +def sanitize_filename(name: str) -> str: + return "".join(c for c in name if c.isalnum() or c in " -_").strip() + + +def get_course_details(course_alias: str, course_base_folder: str) -> Dict[str, Any]: + details = get_json("/api/course/details/", {"alias": course_alias}) + course_folder = os.path.join(course_base_folder, course_alias) + os.makedirs(course_folder, exist_ok=True) + + # Save course_settings.json + course_settings_path = os.path.join(course_folder, "course_settings.json") + with open(course_settings_path, "w", encoding="utf-8") as f: + json.dump(details, f, indent=2, ensure_ascii=False) + + return details + + +def get_assignments(course_alias: str): + return get_json("/api/course/listAssignments/", {"course_alias": course_alias})["assignments"] + + +def get_assignment_details(course_alias: str, assignment_alias: str): + return get_json("/api/course/assignmentDetails/", { + "course": course_alias, + "assignment": assignment_alias + }) + + + + + +def download_and_unzip(problem_alias: str, assignment_folder: str): + try: + download_url = urljoin(BASE_URL, f"/api/problem/download/problem_alias/{problem_alias}/") + parsed_url = urlparse(download_url) + conn = http.client.HTTPSConnection(parsed_url.hostname, context=context) + + headers = {'Authorization': f'token {API_CLIENT.api_token}'} + path = parsed_url.path + + conn.request("GET", path, headers=headers) + response = conn.getresponse() + + if response.status == 404: + LOG.warning(f"⚠️ Problem '{problem_alias}' not found or access denied (404).") + return + elif response.status != 200: + LOG.error(f"❌ Failed to download '{problem_alias}'. HTTP status: {response.status}") + return + + problem_folder = os.path.join(assignment_folder, sanitize_filename(problem_alias)) + os.makedirs(problem_folder, exist_ok=True) + + zip_path = os.path.join(problem_folder, f"{problem_alias}.zip") + with open(zip_path, "wb") as f: + while True: + chunk = response.read(8192) + if not chunk: + break + f.write(chunk) + + try: + with zipfile.ZipFile(zip_path, "r") as zip_ref: + zip_ref.extractall(problem_folder) + os.remove(zip_path) + LOG.info(f"✅ Extracted: {problem_alias} → {problem_folder}") + except zipfile.BadZipFile: + LOG.error(f"❌ Failed to unzip: {zip_path}") + return + + settings_path = os.path.join(problem_folder, "settings.json") + if os.path.exists(settings_path): + try: + with open(settings_path, "r+", encoding="utf-8") as f: + settings = json.load(f) + settings["alias"] = problem_alias + settings["title"] = problem_alias + f.seek(0) + json.dump(settings, f, indent=2, ensure_ascii=False) + f.truncate() + LOG.info(f"🛠️ Updated settings.json with alias: {problem_alias}") + except Exception as e: + LOG.warning(f"⚠️ Failed to update settings.json for '{problem_alias}': {e}") + else: + LOG.warning(f"⚠️ No settings.json found for '{problem_alias}'") + + except Exception as e: + LOG.error(f"❌ Failed to download '{problem_alias}': {e}") + + + + +def main(): + global API_CLIENT + api_token = handle_input() + API_CLIENT = omegaup.api.Client(api_token=api_token, url=BASE_URL) + + os.makedirs(BASE_COURSE_FOLDER, exist_ok=True) + all_problems = [] + + for course_alias in COURSE_ALIASES: + LOG.info(f"📘 Starting course: {course_alias}") + try: + course_details = get_course_details(course_alias, BASE_COURSE_FOLDER) + assignments = get_assignments(course_alias) + + if not assignments: + LOG.warning(f"No assignments found in {course_alias}.") + continue + + course_folder = os.path.join(BASE_COURSE_FOLDER, course_alias) + + for assignment in assignments: + assignment_alias = assignment["alias"] + assignment_name = assignment["name"] + LOG.info(f"📂 Processing assignment: {assignment_name} ({assignment_alias})") + + try: + details = get_assignment_details(course_alias, assignment_alias) + assignment_folder = os.path.join(course_folder, assignment_alias) + os.makedirs(assignment_folder, exist_ok=True) + + assignment_settings_path = os.path.join(assignment_folder, "assignment_settings.json") + with open(assignment_settings_path, "w", encoding="utf-8") as f: + json.dump(details, f, indent=2, ensure_ascii=False) + + problems = details.get("problems", []) + + for problem in problems: + try: + download_and_unzip(problem["alias"], assignment_folder) + rel_path = os.path.join( + "Courses", course_alias, assignment_alias, sanitize_filename(problem["alias"]) + ) + LOG.info(f"📂 Added problem path: {rel_path}") + all_problems.append({"path": rel_path}) + except Exception as e: + LOG.error(f"❌ Error while processing problem '{problem['alias']}': {e}") + + except Exception as e: + LOG.error(f"❌ Failed to process assignment '{assignment_alias}': {e}") + + except Exception as e: + LOG.error(f"❌ Failed to process course '{course_alias}': {e}") + + # ✅ Write problems.json + problems_json_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "problems.json")) + with open(problems_json_path, "w", encoding="utf-8") as f: + LOG.info(f"Writing problems.json to {problems_json_path}") + json.dump({"problems": all_problems}, f, indent=2, ensure_ascii=False) + LOG.info("📝 Created problems.json with all problem paths.") + + + +if __name__ == "__main__": + main() From 2ea0b78372bd755ba74aa2cd9b737e6e1567f0ef Mon Sep 17 00:00:00 2001 From: Anas Iqbal Date: Wed, 16 Jul 2025 15:13:42 +0530 Subject: [PATCH 2/4] Refactor course folder handling to ensure clean state before downloads --- utils/download_and_sync_courses.py | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/utils/download_and_sync_courses.py b/utils/download_and_sync_courses.py index fe766ff..818704d 100644 --- a/utils/download_and_sync_courses.py +++ b/utils/download_and_sync_courses.py @@ -29,13 +29,6 @@ BASE_COURSE_FOLDER = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "Courses")) -if os.path.exists(BASE_COURSE_FOLDER): - LOG.warning("Delete existing course folder to avoid conflicts") - shutil.rmtree(BASE_COURSE_FOLDER) - - - - def handle_input(): global BASE_URL, API_TOKEN parser = argparse.ArgumentParser(description="Download and extract problems from multiple course assignments") @@ -46,7 +39,6 @@ def handle_input(): return args.api_token - def get_json(endpoint: str, params: Dict[str, str]) -> Dict[str, Any]: return API_CLIENT.query(endpoint, params) @@ -79,9 +71,6 @@ def get_assignment_details(course_alias: str, assignment_alias: str): }) - - - def download_and_unzip(problem_alias: str, assignment_folder: str): try: download_url = urljoin(BASE_URL, f"/api/problem/download/problem_alias/{problem_alias}/") @@ -141,13 +130,15 @@ def download_and_unzip(problem_alias: str, assignment_folder: str): LOG.error(f"❌ Failed to download '{problem_alias}': {e}") - - def main(): global API_CLIENT api_token = handle_input() API_CLIENT = omegaup.api.Client(api_token=api_token, url=BASE_URL) + if os.path.exists(BASE_COURSE_FOLDER): + LOG.warning("Delete existing course folder to avoid conflicts") + shutil.rmtree(BASE_COURSE_FOLDER) + os.makedirs(BASE_COURSE_FOLDER, exist_ok=True) all_problems = [] @@ -204,6 +195,5 @@ def main(): LOG.info("📝 Created problems.json with all problem paths.") - if __name__ == "__main__": main() From 3c26df7f4cba00eeeeed414ec529bfd3c0603221 Mon Sep 17 00:00:00 2001 From: Anas Iqbal Date: Wed, 16 Jul 2025 22:52:22 +0530 Subject: [PATCH 3/4] Update course alias to "Curso-de-Python-FutureLabs" --- utils/download_and_sync_courses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/download_and_sync_courses.py b/utils/download_and_sync_courses.py index 818704d..cecd03a 100644 --- a/utils/download_and_sync_courses.py +++ b/utils/download_and_sync_courses.py @@ -23,7 +23,7 @@ # 👇 Add your course aliases here COURSE_ALIASES = [ - "omi-public-course" + "Curso-de-Python-FutureLabs" ] BASE_COURSE_FOLDER = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "Courses")) From 8697022dcd88bb787aa79932892e746f01caa918 Mon Sep 17 00:00:00 2001 From: Anas Iqbal Date: Wed, 16 Jul 2025 23:42:52 +0530 Subject: [PATCH 4/4] Update GitHub Actions checkout action to version 4 --- .github/workflows/continuous-integration.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/continuous-integration.yaml b/.github/workflows/continuous-integration.yaml index b549f94..9837238 100644 --- a/.github/workflows/continuous-integration.yaml +++ b/.github/workflows/continuous-integration.yaml @@ -51,7 +51,7 @@ jobs: with: python-version: '3.8' - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: ref: ${{ env.GITHUB_CURRENT_COMMIT }} fetch-depth: 0