Give up on add all occurrence counts in one query for now

Author: mihow

ami/main/models.py

@@ -2165,46 +2165,16 @@ def with_children(self):
     def with_occurrence_counts(self) -> models.QuerySet:
         """
         Count the number of occurrences for a taxon and all occurrences of the taxon's children.
-        """
-        qs = self.get_queryset()
 
-        # Subquery to get all child taxa IDs
-        import json
+        @TODO Try a recursive CTE in a raw SQL query,
+        or count the occurrences in a separate query and attach them to the Taxon objects.
+        """
 
-        child_taxa = qs.filter(
-            parents_json__contains=models.Value(
-                json.dumps([{"id": models.OuterRef("pk")}]), output_field=models.JSONField()
-            )
-        ).values("pk")
-
-        # Debugging: Log the SQL for child_taxa
-        logger.info(f"SQL for child_taxa: {child_taxa.query}")
-
-        # Debugging: Count and log the number of child taxa
-        child_count = child_taxa.count()
-        logger.info(f"Number of child taxa found: {child_count}")
-
-        # Debugging: Log a sample of child taxa
-        sample_children = list(child_taxa[:5])
-        logger.info(f"Sample of child taxa: {sample_children}")
-
-        # Annotate with occurrence counts
-        qs = qs.annotate(
-            occurrences_count_bulk=models.Count(
-                "occurrences",
-                filter=models.Q(
-                    models.Q(occurrences__determination_id=models.F("pk"))
-                    | models.Q(occurrences__determination_id__in=models.Subquery(child_taxa))
-                ),
-                distinct=True,
-            )
+        raise NotImplementedError(
+            "Occurrence counts can not be calculated in a subquery with the current JSONField schema. "
+            "Fetch them per taxon."
         )
 
-        # Debugging: Log the final SQL query
-        logger.info(f"Final SQL query: {qs.query}")
-
-        return qs
-
 
 class TaxonParent(pydantic.BaseModel):
     """

ami/main/tests.py

@@ -569,15 +569,16 @@ def _count_occurrences_recursive(taxon):
     def test_recursive_occurrence_count_from_manager(self):
         from ami.main.models import Taxon
 
-        taxa_with_counts = Taxon.objects.with_occurrence_counts()
-        for taxon in taxa_with_counts:
-            occurrence_count_total = taxon.occurrences_count_recursive()
-            self.assertEqual(occurrence_count_total, taxon.occurrences_count)
-
-        for taxon in taxa_with_counts:
-            occurrence_count_direct = taxon.occurrences.count()
-            occurrence_count_total = taxon.occurrences_count_recursive()
-            self.assertEqual(occurrence_count_total, occurrence_count_direct)
+        with self.assertRaises(NotImplementedError):
+            taxa_with_counts = Taxon.objects.with_occurrence_counts()
+            for taxon in taxa_with_counts:
+                occurrence_count_total = taxon.occurrences_count_recursive()
+                self.assertEqual(occurrence_count_total, taxon.occurrences_count)
+
+            for taxon in taxa_with_counts:
+                occurrence_count_direct = taxon.occurrences.count()
+                occurrence_count_total = taxon.occurrences_count_recursive()
+                self.assertEqual(occurrence_count_total, occurrence_count_direct)
 
 
 class TestIdentification(APITestCase):