✨(backend) Index deleted documents

Add SEARCH_INDEXER_COUNTDOWN as configurable setting.
Make the search backend creation simplier (only 'get_document_indexer' now).
Allow indexation of deleted documents.

Signed-off-by: Fabre Florian <ffabre@hybird.org>

Author: joehybird

src/backend/core/api/viewsets.py

@@ -50,7 +50,7 @@
     YdocConverter,
 )
 from core.services.search_indexers import (
-    default_document_indexer,
+    get_document_indexer,
     get_visited_document_ids_of,
 )
 from core.tasks.mail import send_ask_for_access_mail
@@ -1076,21 +1076,30 @@ def duplicate(self, request, *args, **kwargs):
     def search(self, request, *args, **kwargs):
         """
         Returns a DRF response containing the filtered, annotated and ordered document list.
-        The filtering allows full text search through the opensearch indexation app "find".
+
+        Applies filtering based on request parameter 'q' from `FindDocumentSerializer`.
+        Depending of the configuration it can be:
+         - A fulltext search through the opensearch indexation app "find" if the backend is
+           enabled (see SEARCH_BACKEND_CLASS)
+         - A filtering by the model field 'title'.
+
+        The ordering is always by the most recent first.
         """
         access_token = request.session.get("oidc_access_token")
         user = request.user
 
         serializer = serializers.FindDocumentSerializer(data=request.query_params)
         serializer.is_valid(raise_exception=True)
 
-        indexer = default_document_indexer()
+        indexer = get_document_indexer()
+        text = serializer.validated_data["q"]
 
+        # The indexer is not configured, so we fallback on a simple filter on the
+        # model field 'title'.
         if not indexer:
+            # As the 'list' view we get a prefiltered queryset (deleted docs are excluded)
             queryset = self.get_queryset()
-            filterset = DocumentFilter(
-                {"title": serializer.validated_data.get("q", "")}, queryset=queryset
-            )
+            filterset = DocumentFilter({"title": text}, queryset=queryset)
 
             if not filterset.is_valid():
                 raise drf.exceptions.ValidationError(filterset.errors)
@@ -1105,15 +1114,17 @@ def search(self, request, *args, **kwargs):
             )
 
         queryset = models.Document.objects.all()
+
+        # Retrieve the documents ids from Find.
         results = indexer.search(
-            text=serializer.validated_data.get("q", ""),
+            text=text,
             token=access_token,
             visited=get_visited_document_ids_of(queryset, user),
             page=serializer.validated_data.get("page", 1),
             page_size=serializer.validated_data.get("page_size", 20),
         )
 
-        queryset = queryset.filter(pk__in=results)
+        queryset = queryset.filter(pk__in=results).order_by("-updated_at")
 
         return self.get_response_for_queryset(
             queryset,

src/backend/core/management/commands/index.py

@@ -5,9 +5,9 @@
 import logging
 import time
 
-from django.core.management.base import BaseCommand
+from django.core.management.base import BaseCommand, CommandError
 
-from ...services.search_indexers import FindDocumentIndexer
+from core.services.search_indexers import get_document_indexer
 
 logger = logging.getLogger("docs.search.bootstrap_search")
 
@@ -19,9 +19,18 @@ class Command(BaseCommand):
 
     def handle(self, *args, **options):
         """Launch and log search index generation."""
+        indexer = get_document_indexer()
+
+        if not indexer:
+            raise CommandError("The indexer is not enabled or properly configured.")
+
         logger.info("Starting to regenerate Find index...")
         start = time.perf_counter()
-        count = FindDocumentIndexer().index()
+
+        try:
+            count = indexer.index()
+        except Exception as err:
+            raise CommandError("Unable to regenerate index") from err
 
         duration = time.perf_counter() - start
         logger.info(

src/backend/core/services/search_indexers.py

@@ -19,37 +19,24 @@
 
 
 @cache
-def default_document_indexer():
-    """Returns default indexer service is enabled and properly configured."""
+def get_document_indexer():
+    """Returns an instance of indexer service if enabled and properly configured."""
+    classpath = settings.SEARCH_INDEXER_CLASS
 
     # For this usecase an empty indexer class is not an issue but a feature.
-    if not getattr(settings, "SEARCH_INDEXER_CLASS", None):
+    if not classpath:
         logger.info("Document indexer is not configured (see SEARCH_INDEXER_CLASS)")
         return None
 
     try:
-        return get_document_indexer_class()()
+        indexer_class = import_string(settings.SEARCH_INDEXER_CLASS)
+        return indexer_class()
+    except ImportError as err:
+        logger.error("SEARCH_INDEXER_CLASS setting is not valid : %s", err)
     except ImproperlyConfigured as err:
         logger.error("Document indexer is not properly configured : %s", err)
-        return None
 
-
-@cache
-def get_document_indexer_class():
-    """Return the indexer backend class based on the settings."""
-    classpath = settings.SEARCH_INDEXER_CLASS
-
-    if not classpath:
-        raise ImproperlyConfigured(
-            "SEARCH_INDEXER_CLASS must be set in Django settings."
-        )
-
-    try:
-        return import_string(settings.SEARCH_INDEXER_CLASS)
-    except ImportError as err:
-        raise ImproperlyConfigured(
-            f"SEARCH_INDEXER_CLASS setting is not valid : {err}"
-        ) from err
+    return None
 
 
 def get_batch_accesses_by_users_and_teams(paths):
@@ -100,9 +87,11 @@ def get_visited_document_ids_of(queryset, user):
             ancestors_deleted_at__isnull=True,
         )
         .filter(pk__in=Subquery(qs.values("document_id")))
+        .order_by("pk")
+        .distinct("pk")
     )
 
-    return list({str(id) for id in docs.values_list("pk", flat=True)})
+    return [str(id) for id in docs.values_list("pk", flat=True)]
 
 
 class BaseDocumentIndexer(ABC):

src/backend/core/signals.py

@@ -9,7 +9,6 @@
 from django.dispatch import receiver
 
 from . import models
-from .services.search_indexers import default_document_indexer
 from .tasks.find import trigger_document_indexer
 
 
@@ -20,14 +19,13 @@ def document_post_save(sender, instance, **kwargs):  # pylint: disable=unused-ar
     Note : Within the transaction we can have an empty content and a serialization
     error.
     """
-    if default_document_indexer() is not None:
-        transaction.on_commit(partial(trigger_document_indexer, instance))
+    transaction.on_commit(partial(trigger_document_indexer, instance))
 
 
 @receiver(signals.post_save, sender=models.DocumentAccess)
 def document_access_post_save(sender, instance, created, **kwargs):  # pylint: disable=unused-argument
     """
     Asynchronous call to the document indexer at the end of the transaction.
     """
-    if not created and default_document_indexer() is not None:
+    if not created:
         transaction.on_commit(partial(trigger_document_indexer, instance.document))

src/backend/core/tasks/find.py

@@ -10,22 +10,21 @@
 logger = getLogger(__file__)
 
 
-def document_indexer_debounce_key(document_id):
-    """Returns debounce cache key"""
-    return f"doc-indexer-debounce-{document_id}"
-
-
-def incr_counter(key):
+def indexer_debounce_lock(document_id):
     """Increase or reset counter"""
+    key = f"doc-indexer-debounce-{document_id}"
+
     try:
         return cache.incr(key)
     except ValueError:
         cache.set(key, 1)
         return 1
 
 
-def decr_counter(key):
+def indexer_debounce_release(document_id):
     """Decrease or reset counter"""
+    key = f"doc-indexer-debounce-{document_id}"
+
     try:
         return cache.decr(key)
     except ValueError:
@@ -36,24 +35,26 @@ def decr_counter(key):
 @app.task
 def document_indexer_task(document_id):
     """Celery Task : Sends indexation query for a document."""
-    key = document_indexer_debounce_key(document_id)
+    # Prevents some circular imports
+    # pylint: disable=import-outside-toplevel
+    from core import models  # noqa : PLC0415
+    from core.services.search_indexers import (  # noqa : PLC0415
+        get_batch_accesses_by_users_and_teams,
+        get_document_indexer,
+    )
 
     # check if the counter : if still up, skip the task. only the last one
     # within the countdown delay will do the query.
-    if decr_counter(key) > 0:
+    if indexer_debounce_release(document_id) > 0:
         logger.info("Skip document %s indexation", document_id)
         return
 
-    # Prevents some circular imports
-    # pylint: disable=import-outside-toplevel
-    from core import models  # noqa: PLC0415
-    from core.services.search_indexers import (  # noqa: PLC0415
-        get_batch_accesses_by_users_and_teams,
-        get_document_indexer_class,
-    )
+    indexer = get_document_indexer()
+
+    if indexer is None:
+        return
 
     doc = models.Document.objects.get(pk=document_id)
-    indexer = get_document_indexer_class()()
     accesses = get_batch_accesses_by_users_and_teams((doc.path,))
 
     data = indexer.serialize_document(document=doc, accesses=accesses)
@@ -69,11 +70,11 @@ def trigger_document_indexer(document):
     Args:
         document (Document): The document instance.
     """
-    if document.deleted_at or document.ancestors_deleted_at:
-        return
+    countdown = settings.SEARCH_INDEXER_COUNTDOWN
 
-    key = document_indexer_debounce_key(document.pk)
-    countdown = getattr(settings, "SEARCH_INDEXER_COUNTDOWN", 1)
+    # DO NOT create a task if indexation if disabled
+    if not settings.SEARCH_INDEXER_CLASS:
+        return
 
     logger.info(
         "Add task for document %s indexation in %.2f seconds",
@@ -83,6 +84,6 @@ def trigger_document_indexer(document):
 
     # Each time this method is called during the countdown, we increment the
     # counter and each task decrease it, so the index be run only once.
-    incr_counter(key)
+    indexer_debounce_lock(document.pk)
 
     document_indexer_task.apply_async(args=[document.pk], countdown=countdown)

src/backend/core/tests/commands/test_index.py

@@ -5,7 +5,7 @@
 from operator import itemgetter
 from unittest import mock
 
-from django.core.management import call_command
+from django.core.management import CommandError, call_command
 from django.db import transaction
 
 import pytest
@@ -51,3 +51,15 @@ def test_index():
             ],
             key=itemgetter("id"),
         )
+
+
+@pytest.mark.django_db
+@pytest.mark.usefixtures("indexer_settings")
+def test_index_improperly_configured(indexer_settings):
+    """The command should raise an exception if the indexer is not configured"""
+    indexer_settings.SEARCH_INDEXER_CLASS = None
+
+    with pytest.raises(CommandError) as err:
+        call_command("index")
+
+    assert str(err.value) == "The indexer is not enabled or properly configured."

src/backend/core/tests/conftest.py

@@ -34,12 +34,10 @@ def indexer_settings_fixture(settings):
 
     # pylint: disable-next=import-outside-toplevel
     from core.services.search_indexers import (  # noqa: PLC0415
-        default_document_indexer,
-        get_document_indexer_class,
+        get_document_indexer,
     )
 
-    default_document_indexer.cache_clear()
-    get_document_indexer_class.cache_clear()
+    get_document_indexer.cache_clear()
 
     settings.SEARCH_INDEXER_CLASS = "core.services.search_indexers.FindDocumentIndexer"
     settings.SEARCH_INDEXER_SECRET = "ThisIsAKeyForTest"
@@ -51,5 +49,4 @@ def indexer_settings_fixture(settings):
     yield settings
 
     # clear cache to prevent issues with other tests
-    default_document_indexer.cache_clear()
-    get_document_indexer_class.cache_clear()
+    get_document_indexer.cache_clear()

src/backend/core/tests/documents/test_api_documents_search.py

@@ -10,7 +10,7 @@
 from rest_framework.test import APIClient
 
 from core import factories, models
-from core.services.search_indexers import default_document_indexer
+from core.services.search_indexers import get_document_indexer
 
 fake = Faker()
 pytestmark = pytest.mark.django_db
@@ -54,7 +54,7 @@ def test_api_documents_search_endpoint_is_none(indexer_settings):
     """
     indexer_settings.SEARCH_INDEXER_QUERY_URL = None
 
-    assert default_document_indexer() is None
+    assert get_document_indexer() is None
 
     user = factories.UserFactory()
     document = factories.DocumentFactory(title="alpha")
@@ -130,7 +130,7 @@ def test_api_documents_search_format(indexer_settings):
     """Validate the format of documents as returned by the search view."""
     indexer_settings.SEARCH_INDEXER_QUERY_URL = "http://find/api/v1.0/search"
 
-    assert default_document_indexer() is not None
+    assert get_document_indexer() is not None
 
     user = factories.UserFactory()
 
@@ -193,7 +193,7 @@ def test_api_documents_search_pagination(indexer_settings):
     """Documents should be ordered by descending "updated_at" by default"""
     indexer_settings.SEARCH_INDEXER_QUERY_URL = "http://find/api/v1.0/search"
 
-    assert default_document_indexer() is not None
+    assert get_document_indexer() is not None
 
     user = factories.UserFactory()