Build: better support for empty default branch (#12462)

The basic idea is to have the value of latest in sync with the value
from the default branch, which we already do. But, in case the default
branch is left empty, we were falling back to master, which is just a
guess of the default, so instead of guessing the value, we just don't
update it if we don't know its value. When cloning the repo, we have
access to the default branch, so that's when we update latest for those
projects.

Note this is only relevant for projects that aren't linked to a remote
repository, since for those we know the default branch. So, when we try
to build the latest version of a project that doesn't have a default
branch, we:

- Omit the ref from the fetch command.
- Use the default branch from the repository for the checkout (we could
skip this step, but since we have a new feature that keeps the files
from a previous clone, we need to do a checkout to make sure we are
building from the given branch/commit).
- Update the identifier from latest to the default branch we detected
from the cloned repository.

There's still the problem about latest's identifier being out of sync
when the default branch changes, but that's fixed after a new build, and
this isn't a new problem (now users aren't blocked at least), and
changing the default branch of a repo isn't something users do that
often, and this only affects projects that aren't linked to a
repository, so I think we are okay with that for now.

This replaces #10927
Closes #12231

Author: stsewd

readthedocs/api/v2/views/integrations.py

@@ -21,6 +21,7 @@
 
 from readthedocs.builds.constants import BRANCH
 from readthedocs.builds.constants import LATEST
+from readthedocs.builds.constants import LATEST_VERBOSE_NAME
 from readthedocs.builds.constants import TAG
 from readthedocs.core.signals import webhook_bitbucket
 from readthedocs.core.signals import webhook_github
@@ -334,7 +335,7 @@ def get_closed_external_version_response(self, project):
 
     def update_default_branch(self, default_branch):
         """
-        Update the `Version.identifer` for `latest` with the VCS's `default_branch`.
+        Update the `Version.identifier` for `latest` with the VCS's `default_branch`.
 
         The VCS's `default_branch` is the branch cloned when there is no specific branch specified
         (e.g. `git clone <URL>`).
@@ -356,7 +357,9 @@ def update_default_branch(self, default_branch):
             # Always check for the machine attribute, since latest can be user created.
             # RTD doesn't manage those.
             self.project.versions.filter(slug=LATEST, machine=True).update(
-                identifier=default_branch
+                identifier=default_branch,
+                verbose_name=LATEST_VERBOSE_NAME,
+                type=BRANCH,
             )
 
 

readthedocs/builds/models.py

@@ -222,6 +222,10 @@ def is_public(self):
     def is_external(self):
         return self.type == EXTERNAL
 
+    @property
+    def is_machine_latest(self):
+        return self.machine and self.slug == LATEST
+
     @property
     def explicit_name(self):
         """

readthedocs/doc_builder/director.py

@@ -232,9 +232,26 @@ def checkout(self):
                         dismissable=True,
                     )
 
-        identifier = self.data.build_commit or self.data.version.identifier
-        log.info("Checking out.", identifier=identifier)
-        self.vcs_repository.checkout(identifier)
+        # Get the default branch of the repository if the project doesn't
+        # have an explicit default branch set and we are building latest.
+        # The identifier from latest will be updated with this value
+        # if the build succeeds.
+        is_latest_without_default_branch = (
+            self.data.version.is_machine_latest and not self.data.project.default_branch
+        )
+        if is_latest_without_default_branch:
+            self.data.default_branch = self.data.build_director.vcs_repository.get_default_branch()
+            log.info(
+                "Default branch for the repository detected.",
+                default_branch=self.data.default_branch,
+            )
+
+        # We can skip the checkout step since we just cloned the repository,
+        # and the default branch is already checked out.
+        if not is_latest_without_default_branch:
+            identifier = self.data.build_commit or self.data.version.identifier
+            log.info("Checking out.", identifier=identifier)
+            self.vcs_repository.checkout(identifier)
 
         # The director is responsible for understanding which config file to use for a build.
         # In order to reproduce a build 1:1, we may use readthedocs_yaml_path defined by the build

readthedocs/projects/models.py

@@ -1201,15 +1201,21 @@ def get_original_latest_version(self):
         When latest is machine created, it's basically an alias
         for the default branch/tag (like main/master),
 
-        Returns None if the current default version doesn't point to a valid version.
+        Returns None if latest doesn't point to a valid version,
+        or if isn't managed by RTD (machine=False).
         """
-        default_version_name = self.get_default_branch()
+        # For latest, the identifier is the name of the branch/tag.
+        latest_version_identifier = (
+            self.versions.filter(slug=LATEST, machine=True)
+            .values_list("identifier", flat=True)
+            .first()
+        )
+        if not latest_version_identifier:
+            return None
         return (
             self.versions(manager=INTERNAL)
             .exclude(slug=LATEST)
-            .filter(
-                verbose_name=default_version_name,
-            )
+            .filter(verbose_name=latest_version_identifier)
             .first()
         )
 
@@ -1227,8 +1233,22 @@ def update_latest_version(self):
             return
 
         # default_branch can be a tag or a branch name!
-        default_version_name = self.get_default_branch()
-        original_latest = self.get_original_latest_version()
+        default_version_name = self.get_default_branch(fallback_to_vcs=False)
+        # If the default_branch is not set, it means that the user
+        # wants to use the default branch of the respository, but
+        # we don't know what that is here, `latest` will be updated
+        # on the next build.
+        if not default_version_name:
+            return
+
+        # Search for a branch or tag with the name of the default branch,
+        # so we can sync latest with it.
+        original_latest = (
+            self.versions(manager=INTERNAL)
+            .exclude(slug=LATEST)
+            .filter(verbose_name=default_version_name)
+            .first()
+        )
         latest.verbose_name = LATEST_VERBOSE_NAME
         latest.type = original_latest.type if original_latest else BRANCH
         # For latest, the identifier is the name of the branch/tag.
@@ -1328,14 +1348,22 @@ def get_default_version(self):
             return self.default_version
         return LATEST
 
-    def get_default_branch(self):
-        """Get the version representing 'latest'."""
+    def get_default_branch(self, fallback_to_vcs=True):
+        """
+        Get the name of the branch or tag that the user wants to use as 'latest'.
+
+        In case the user explicitly set a default branch, we use that,
+        otherwise we try to get it from the remote repository.
+        """
         if self.default_branch:
             return self.default_branch
 
         if self.remote_repository and self.remote_repository.default_branch:
             return self.remote_repository.default_branch
 
+        if not fallback_to_vcs:
+            return None
+
         vcs_class = self.vcs_class()
         if vcs_class:
             return vcs_class.fallback_branch

readthedocs/projects/tasks/builds.py

@@ -27,6 +27,7 @@
 from readthedocs.builds import tasks as build_tasks
 from readthedocs.builds.constants import ARTIFACT_TYPES
 from readthedocs.builds.constants import ARTIFACT_TYPES_WITHOUT_MULTIPLE_FILES_SUPPORT
+from readthedocs.builds.constants import BRANCH
 from readthedocs.builds.constants import BUILD_FINAL_STATES
 from readthedocs.builds.constants import BUILD_STATE_BUILDING
 from readthedocs.builds.constants import BUILD_STATE_CANCELLED
@@ -117,6 +118,10 @@ class TaskData:
     config: BuildConfigV2 = None
     project: APIProject = None
     version: APIVersion = None
+    # Default branch for the repository.
+    # Only set when building the latest version, and the project
+    # doesn't have an explicit default branch.
+    default_branch: str | None = None
 
     # Dictionary returned from the API.
     build: dict = field(default_factory=dict)
@@ -665,18 +670,22 @@ def on_success(self, retval, task_id, args, kwargs):
         # NOTE: we are updating the db version instance *only* when
         # TODO: remove this condition and *always* update the DB Version instance
         if "html" in valid_artifacts:
+            data = {
+                "built": True,
+                "documentation_type": self.data.version.documentation_type,
+                "has_pdf": "pdf" in valid_artifacts,
+                "has_epub": "epub" in valid_artifacts,
+                "has_htmlzip": "htmlzip" in valid_artifacts,
+                "build_data": self.data.version.build_data,
+                "addons": self.data.version.addons,
+            }
+            # Update the latest version to point to the current VCS default branch
+            # if the project doesn't have an explicit default branch set.
+            if self.data.default_branch:
+                data["identifier"] = self.data.default_branch
+                data["type"] = BRANCH
             try:
-                self.data.api_client.version(self.data.version.pk).patch(
-                    {
-                        "built": True,
-                        "documentation_type": self.data.version.documentation_type,
-                        "has_pdf": "pdf" in valid_artifacts,
-                        "has_epub": "epub" in valid_artifacts,
-                        "has_htmlzip": "htmlzip" in valid_artifacts,
-                        "build_data": self.data.version.build_data,
-                        "addons": self.data.version.addons,
-                    }
-                )
+                self.data.api_client.version(self.data.version.pk).patch(data)
             except HttpClientError:
                 # NOTE: I think we should fail the build if we cannot update
                 # the version at this point. Otherwise, we will have inconsistent data