Optimize the bootstrapper and tests (#2)

* Changes for the upcoming release

* optimize cloning sequence

* fix tests

* add pre-commit checks

* update pre-commit.yaml

* Update .github/workflows/tests.yaml

* Optimized folder discovery and creation

* Make the logger customizable

Author: egeakman

.github/workflows/pre-commit.yaml

@@ -0,0 +1,17 @@
+name: pre-commit checks
+
+on: [push, pull_request, workflow_dispatch]
+
+permissions:
+  contents: read
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.x"
+      - uses: pre-commit/action@v3.0.0

.github/workflows/tests.yaml

@@ -1,12 +1,6 @@
 name: Tests
-on:
-  workflow_dispatch:
-  push:
-    branches:
-      - "**"
-  pull_request:
-    branches:
-      - "**"
+
+on: [push, pull_request, workflow_dispatch]
 
 jobs:
   test:
@@ -15,11 +9,12 @@ jobs:
     strategy:
       matrix:
         os: [macos-latest, windows-latest, ubuntu-latest]
-        python-version: ["3.10", "3.11", "3.12-dev"]
+        python-version: ["3.11", "3.12"]
+        build-version: ["3.11", "3.12"]
 
     steps:
       - name: Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Python
         uses: actions/setup-python@v4
@@ -41,8 +36,8 @@ jobs:
       - name: Install the package
         run: pip install .
 
-      - name: Run the package
-        run: bootstrapper tr -b 3.12
+      - name: Run the bootstrapper
+        run: bootstrapper tr -b ${{ matrix.build-version }}
 
       - name: See results (non-Windows)
         if: matrix.os != 'windows-latest'

.pre-commit-config.yaml

@@ -9,11 +9,12 @@ repos:
     rev: 23.10.1
     hooks:
       - id: black
-        name: black (python)
 
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.5.0
     hooks:
+      - id: check-case-conflict
+      - id: check-merge-conflict
       - id: check-toml
       - id: check-yaml
       - id: end-of-file-fixer

bootstrapper/bootstrapper.py

@@ -21,17 +21,16 @@ def emit(self, record):
         self.flush()
 
 
-logger = logging.getLogger()
-logger.setLevel(logging.INFO)
-handler = _NoNewLine()
-handler.setFormatter(logging.Formatter("%(message)s"))
-logger.addHandler(handler)
-
-
 class Bootstrapper:
-    def __init__(self, language: str, branch: str = "3.12") -> None:
+    def __init__(
+        self,
+        language: str,
+        branch: str = "3.12",
+        logger: logging.Logger = logging.getLogger(),
+    ) -> None:
         self.language = language
         self.branch = branch
+        self.logger = logger
         self.translation_repo = f"python-docs-{self.language}"
         self.cpython_repo = f"{self.translation_repo}/venv/cpython"
         self.readme_url = "https://raw.githubusercontent.com/egeakman/python-docs-bootstrapper/master/bootstrapper/data/README.md"
@@ -43,36 +42,36 @@ def _request(self, url: str) -> str:
             return response.read().decode()
 
     def create_dirs(self) -> None:
-        logger.info("Creating directories...")
+        self.logger.info("Creating directories...")
         os.makedirs(self.translation_repo, exist_ok=True)
         os.makedirs(self.cpython_repo, exist_ok=True)
-        print("✅")
+        self.logger.info("✅\n")
 
     def setup_cpython_repo(self) -> None:
         if not os.path.exists(f"{self.cpython_repo}/.git") and not os.path.isdir(
             f"{self.cpython_repo}/.git"
         ):
-            logger.info("Cloning CPython repo...")
+            self.logger.info("Cloning CPython repo...")
             subprocess.run(
                 [
                     "git",
                     "clone",
                     "https://github.com/python/cpython.git",
                     self.cpython_repo,
+                    f"--branch={self.branch}",
                     "-q",
                 ],
                 check=True,
             )
-            print("✅")
+            self.logger.info("✅\n")
 
-        subprocess.run(
-            ["git", "-C", self.cpython_repo, "checkout", self.branch, "-q"], check=True
-        )
+        self.logger.info("Updating CPython repo...")
         subprocess.run(
             ["git", "-C", self.cpython_repo, "pull", "--ff-only", "-q"], check=True
         )
+        self.logger.info("✅\n")
 
-        logger.info("Building gettext files...")
+        self.logger.info("Building gettext files...")
         subprocess.run(
             [
                 "sphinx-build",
@@ -85,21 +84,18 @@ def setup_cpython_repo(self) -> None:
             cwd=self.cpython_repo,
             check=True,
         )
-        print("✅")
+        self.logger.info("✅\n")
 
     def setup_translation_repo(self) -> None:
-        logger.info("Initializing translation repo...")
+        self.logger.info("Initializing translation repo...")
         subprocess.run(["git", "init", "-q"], cwd=self.translation_repo, check=True)
         subprocess.run(
             ["git", "branch", "-m", self.branch], cwd=self.translation_repo, check=True
         )
-        print("✅")
-
-        logger.info("Copying gettext files...")
-        files = glob.glob(f"{self.cpython_repo}/pot/**/*.pot") + glob.glob(
-            f"{self.cpython_repo}/pot/*.pot"
-        )
+        self.logger.info("✅\n")
 
+        self.logger.info("Copying gettext files...")
+        files = glob.glob(f"{self.cpython_repo}/pot/**/*.pot", recursive=True)
         files = [path.replace("\\", "/") for path in files]
 
         for file in files:
@@ -108,69 +104,62 @@ def setup_translation_repo(self) -> None:
                     ".pot", ".po"
                 )
             )
-
-            if len(file.split("/")) > 5:
-                os.makedirs("/".join(dest_path.split("/")[:2]), exist_ok=True)
-
+            os.makedirs(os.path.dirname(dest_path), exist_ok=True)
             shutil.copyfile(file, dest_path)
             files[files.index(file)] = dest_path
-        print("✅")
+        self.logger.info("✅\n")
 
-        logger.info("Cleaning up gettext files...")
+        self.logger.info("Cleaning up gettext files...")
         for file in files:
             with open(file, "r", encoding="utf-8") as f:
                 contents = f.read()
                 contents = re.sub("^#: .*Doc/", "#: ", contents, flags=re.M)
             with open(file, "w", encoding="utf-8") as f:
                 f.write(contents)
-        print("✅")
+        self.logger.info("✅\n")
 
     def create_readme(self) -> None:
-        logger.info("Creating README.md...")
+        self.logger.info("Creating README.md...")
         try:
             readme = self._request(self.readme_url)
         except (urllib.error.HTTPError, urllib.error.URLError):
-            logger.warning(
+            self.logger.warning(
                 "\n ⚠️ Failed to fetch README.md from GitHub, using local copy..."
             )
             readme = Path(f"{os.path.dirname(__file__)}/data/README.md").read_text(
                 encoding="utf-8"
             )
-
         readme = readme.replace("{{translation.language}}", self.language)
-
         with open(f"{self.translation_repo}/README.md", "w", encoding="utf-8") as f:
             f.write(readme)
-        print("✅")
+        self.logger.info("✅\n")
 
     def create_gitignore(self) -> None:
-        logger.info("Creating .gitignore...")
+        self.logger.info("Creating .gitignore...")
         try:
             gitignore = self._request(self.gitignore_url)
         except (urllib.error.HTTPError, urllib.error.URLError):
-            logger.warning(
+            self.logger.warning(
                 "\n ⚠️ Failed to fetch .gitignore from GitHub, using local copy..."
             )
             gitignore = Path(f"{os.path.dirname(__file__)}/data/.gitignore").read_text(
                 encoding="utf-8"
             )
-
         with open(f"{self.translation_repo}/.gitignore", "w", encoding="utf-8") as f:
             f.write(gitignore)
-        print("✅")
+        self.logger.info("✅\n")
 
     def create_makefile(self) -> None:
-        logging.info("Creating .makefile...")
+        logging.info("Creating Makefile...")
         try:
             makefile = self._request(self.makefile_url)
         except (urllib.error.HTTPError, urllib.error.URLError):
-            logger.warning(
+            self.logger.warning(
                 "\n ⚠️ Failed to fetch Makefile from GitHub, using local copy..."
             )
             makefile = Path(f"{os.path.dirname(__file__)}/data/Makefile").read_text(
                 encoding="utf-8"
             )
-
         head = (
             subprocess.run(
                 ["git", "-C", self.cpython_repo, "rev-parse", "HEAD"],
@@ -180,14 +169,12 @@ def create_makefile(self) -> None:
             .stdout.strip()
             .decode()
         )
-
         makefile = makefile.replace("{{translation.language}}", self.language)
         makefile = makefile.replace("{{translation.branch}}", self.branch)
         makefile = makefile.replace("{{translation.head}}", head)
-
         with open(f"{self.translation_repo}/Makefile", "w", encoding="utf-8") as f:
             f.write(makefile)
-        print("✅")
+        self.logger.info("✅\n")
 
     def run(self) -> None:
         try:
@@ -197,9 +184,11 @@ def run(self) -> None:
             self.create_readme()
             self.create_gitignore()
             self.create_makefile()
-            logger.info(f"🎉 Done bootstrapping the {self.language} translation ✅\n")
+            self.logger.info(
+                f"🎉 Done bootstrapping the {self.language} translation ✅\n"
+            )
         except Exception as e:
-            logger.critical(
+            self.logger.critical(
                 f"❌ Bootstrapping of the {self.language} translation failed: {e}\n"
             )
             sys.exit(1)
@@ -218,6 +207,11 @@ def main() -> None:
         "-b", "--branch", type=str, default="3.12", help="CPython branch (e.g. 3.12)"
     )
     args = parser.parse_args()
+    logger = logging.getLogger()
+    logger.setLevel(logging.INFO)
+    handler = _NoNewLine()
+    handler.setFormatter(logging.Formatter("%(message)s"))
+    logger.addHandler(handler)
     Bootstrapper(args.language.lower().replace("_", "-"), args.branch).run()
 
 

bootstrapper/data/Makefile

@@ -10,9 +10,7 @@
 # - make clean # To remove build artifacts
 # - make fuzzy  # To find fuzzy strings
 #
-# Modes are: autobuild-stable, autobuild-dev, and autobuild-html,
-# documented in gen/src/3.6/Doc/Makefile as we're only delegating the
-# real work to the Python Doc Makefile.
+# Modes are: autobuild-stable, autobuild-dev, and autobuild-html
 
 # Configuration
 

bootstrapper/data/README.md

@@ -2,7 +2,7 @@
 
 Hello! This is the repository of the {{translation.language}} translation of the Python documentation.
 
-You can refer to the following resources throughout this journey:
+**You can refer to the following resources throughout this journey:**
 
 - [DevGuide Translating Section](https://devguide.python.org/documentation/translating/) (add your translation to the list!)
 - [PEP 545](https://www.python.org/dev/peps/pep-0545/) (the PEP that describes the translation process)
@@ -14,4 +14,4 @@ Python's documentation is maintained using a global network of volunteers. By po
 
 You signify acceptance of this agreement by submitting your work to the PSF for inclusion in the documentation.
 
-**Do not forget to replace this file with your own README that describes your translation and community!**
+***Do not forget to replace this file with your own README that describes your translation and community, while keeping the above agreement!***

pyproject.toml

@@ -39,9 +39,9 @@ dependencies = [
   "sphinx==4.5",
 ]
 [project.urls]
-Releases = "https://github.com/egeakman/python-docs-bootstrapper/releases"
 Homepage = "https://github.com/egeakman/python-docs-bootstrapper"
 Issues = "https://github.com/egeakman/python-docs-bootstrapper/issues"
+Releases = "https://github.com/egeakman/python-docs-bootstrapper/releases"
 [project.scripts]
 bootstrapper = "bootstrapper.bootstrapper:main"