Build: complete cleanup (#12430)

`clean_build` is not called when the task is terminated and there could
be some lingering files around. Instead of only deleting the files for
the current build, we delete the whole `settings.DOCROOT` path when the
build starts.

Author: humitos

readthedocs/projects/models.py

@@ -1975,6 +1975,7 @@ def add_features(sender, **kwargs):
     # Build related features
     SCALE_IN_PROTECTION = "scale_in_prtection"
     USE_S3_SCOPED_CREDENTIALS_ON_BUILDERS = "use_s3_scoped_credentials_on_builders"
+    BUILD_FULL_CLEAN = "build_full_clean"
     DONT_CLEAN_BUILD = "dont_clean_build"
     BUILD_HEALTHCHECK = "build_healthcheck"
     BUILD_NO_ACKS_LATE = "build_no_acks_late"
@@ -2037,6 +2038,10 @@ def add_features(sender, **kwargs):
                 "Build: Don't clean the build directory. Only for Enterprise users with dedicated builders."
             ),
         ),
+        (
+            BUILD_FULL_CLEAN,
+            _("Build: Clean all build directories to avoid leftovers from other projects."),
+        ),
         (
             BUILD_HEALTHCHECK,
             _("Build: Use background cURL healthcheck."),

readthedocs/projects/tasks/builds.py

@@ -81,11 +81,11 @@
 @dataclass(slots=True)
 class TaskData:
     """
-    Object to store all data related to a Celery task excecution.
+    Object to store all data related to a Celery task execution.
 
-    We use this object from inside the task to store data while we are runnig
+    We use this object from inside the task to store data while we are running
     the task. This is to avoid using `self.` inside the task due to its
-    limitations: it's instanciated once and that instance is re-used for all
+    limitations: it's instantiated once and that instance is re-used for all
     the tasks ran. This could produce sharing instance state between two
     different and unrelated tasks.
 
@@ -418,9 +418,12 @@ def before_start(self, task_id, args, kwargs):
                 protected_from_scale_in=True,
             )
 
-        # Clean the build paths completely to avoid conflicts with previous run
-        # (e.g. cleanup task failed for some reason)
-        clean_build(self.data.version)
+        if self.data.project.has_feature(Feature.BUILD_FULL_CLEAN):
+            # Clean DOCROOT path completely to avoid conflicts other projects
+            clean_build()
+        else:
+            # Clean the build paths for this version to avoid conflicts with previous run
+            clean_build(self.data.version)
 
         # NOTE: this is never called. I didn't find anything in the logs, so we
         # can probably remove it
@@ -558,7 +561,7 @@ def get_valid_artifact_types(self):
 
         TODO: remove the limitation of only 1 file.
         Add support for multiple PDF files in the output directory and
-        grab them by using glob syntaxt between other files that could be garbage.
+        grab them by using glob syntax between other files that could be garbage.
         """
         valid_artifacts = []
         for artifact_type in ARTIFACT_TYPES:
@@ -789,7 +792,7 @@ def update_build(self, state=None):
             log.exception("Error while updating the build object.", state=state)
 
     def execute(self):
-        # Clonning
+        # Cloning
         self.update_build(state=BUILD_STATE_CLONING)
 
         # TODO: remove the ``create_vcs_environment`` hack. Ideally, this should be

readthedocs/projects/tasks/utils.py

@@ -25,10 +25,10 @@
 log = structlog.get_logger(__name__)
 
 
-def clean_build(version):
+def clean_build(version=None):
     """Clean the files used in the build of the given version."""
 
-    if version.project.has_feature(
+    if version and version.project.has_feature(
         Feature.DONT_CLEAN_BUILD,
     ):
         log.info(
@@ -38,15 +38,23 @@ def clean_build(version):
         )
         return
 
-    del_dirs = [
-        os.path.join(version.project.doc_path, dir_, version.slug)
-        for dir_ in ("checkouts", "envs", "conda", "artifacts")
-    ]
-    del_dirs.append(os.path.join(version.project.doc_path, ".cache"))
+    if version:
+        del_dirs = [
+            os.path.join(version.project.doc_path, dir_, version.slug)
+            for dir_ in ("checkouts", "envs", "conda", "artifacts")
+        ]
+        del_dirs.append(os.path.join(version.project.doc_path, ".cache"))
+
+        log.info("Removing directories.", directories=del_dirs)
+        for path in del_dirs:
+            safe_rmtree(path, ignore_errors=True)
 
-    log.info("Removing directories.", directories=del_dirs)
-    for path in del_dirs:
-        safe_rmtree(path, ignore_errors=True)
+    # Clean up DOCROOT (e.g. `user_builds/`) completely
+    else:
+        log.info("Removing DOCROOT directory.", docroot=settings.DOCROOT)
+        safe_rmtree(settings.DOCROOT, ignore_errors=True)
+        os.makedirs(settings.DOCROOT)
+        return
 
 
 @app.task(queue="web")