Add error handling w retty when opening index

.github/workflows/ci.yml

@@ -275,8 +275,12 @@ jobs:
   tests-frontend-e2e:
     name: "Frontend E2E Tests (Node ${{ matrix.node-version }} - ${{ matrix.shard-index }}/${{ matrix.shard-count }})"
     runs-on: ubuntu-24.04
+    container: mcr.microsoft.com/playwright:v1.57.0-noble
     needs:
       - install-frontend-dependencies
+    env:
+      PLAYWRIGHT_BROWSERS_PATH: /ms-playwright
+      PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
     strategy:
       fail-fast: false
       matrix:
@@ -305,19 +309,8 @@ jobs:
           key: ${{ runner.os }}-frontenddeps-${{ hashFiles('src-ui/pnpm-lock.yaml') }}
       - name: Re-link Angular cli
         run: cd src-ui && pnpm link @angular/cli
-      - name: Cache Playwright browsers
-        uses: actions/cache@v4
-        with:
-          path: ~/.cache/ms-playwright
-          key: ${{ runner.os }}-playwright-${{ hashFiles('src-ui/pnpm-lock.yaml') }}
-          restore-keys: |
-            ${{ runner.os }}-playwright-
-      - name: Install Playwright system dependencies
-        run: npx playwright install-deps
       - name: Install dependencies
         run: cd src-ui && pnpm install --no-frozen-lockfile
-      - name: Install Playwright
-        run: cd src-ui && pnpm exec playwright install
       - name: Run Playwright e2e tests
         run: cd src-ui && pnpm exec playwright test --shard ${{ matrix.shard-index }}/${{ matrix.shard-count }}
   frontend-bundle-analysis:

pyproject.toml

@@ -45,7 +45,7 @@ dependencies = [
   "drf-writable-nested~=0.7.1",
   "filelock~=3.20.0",
   "flower~=2.0.1",
-  "gotenberg-client~=0.13.1",
+  "gotenberg-client~=0.12.0",
   "httpx-oauth~=0.16",
   "imap-tools~=1.11.0",
   "inotifyrecursive~=0.3",

src/documents/index.py

@@ -10,6 +10,7 @@ from datetime import time
 from datetime import timedelta
 from datetime import timezone
 from shutil import rmtree
+from time import sleep
 from typing import TYPE_CHECKING
 from typing import Literal
 
@@ -32,6 +33,7 @@ from whoosh.highlight import HtmlFormatter
 from whoosh.idsets import BitSet
 from whoosh.idsets import DocIdSet
 from whoosh.index import FileIndex
+from whoosh.index import LockError
 from whoosh.index import create_in
 from whoosh.index import exists_in
 from whoosh.index import open_dir
@@ -97,11 +99,33 @@ def get_schema() -> Schema:
 
 
 def open_index(*, recreate=False) -> FileIndex:
-    try:
-        if exists_in(settings.INDEX_DIR) and not recreate:
-            return open_dir(settings.INDEX_DIR, schema=get_schema())
-    except Exception:
-        logger.exception("Error while opening the index, recreating.")
+    transient_exceptions = (FileNotFoundError, LockError)
+    max_retries = 3
+    retry_delay = 0.1
+
+    for attempt in range(max_retries + 1):
+        try:
+            if exists_in(settings.INDEX_DIR) and not recreate:
+                return open_dir(settings.INDEX_DIR, schema=get_schema())
+            break
+        except transient_exceptions as exc:
+            is_last_attempt = attempt == max_retries or recreate
+            if is_last_attempt:
+                logger.exception(
+                    "Error while opening the index after retries, recreating.",
+                )
+                break
+
+            logger.warning(
+                "Transient error while opening the index (attempt %s/%s): %s. Retrying.",
+                attempt + 1,
+                max_retries + 1,
+                exc,
+            )
+            sleep(retry_delay)
+        except Exception:
+            logger.exception("Error while opening the index, recreating.")
+            break
 
     # create_in doesn't handle corrupted indexes very well, remove the directory entirely first
     if settings.INDEX_DIR.is_dir():

src/documents/tests/test_index.py

@@ -1,6 +1,7 @@
 from datetime import datetime
 from unittest import mock
 
+from django.conf import settings
 from django.contrib.auth.models import User
 from django.test import SimpleTestCase
 from django.test import TestCase
@@ -251,3 +252,31 @@ class TestRewriteNaturalDateKeywords(SimpleTestCase):
         result = self._rewrite_with_now("added:today", fixed_now)
         # Should convert to UTC properly
         self.assertIn("added:[20250719", result)
+
+
+class TestIndexResilience(DirectoriesMixin, SimpleTestCase):
+    def test_transient_missing_segment_does_not_force_recreate(self):
+        file_marker = settings.INDEX_DIR / "file_marker.txt"
+        file_marker.write_text("keep")
+        expected_index = object()
+
+        with (
+            mock.patch("documents.index.exists_in", return_value=True),
+            mock.patch(
+                "documents.index.open_dir",
+                side_effect=[FileNotFoundError("missing"), expected_index],
+            ) as mock_open_dir,
+            mock.patch(
+                "documents.index.create_in",
+            ) as mock_create_in,
+            mock.patch(
+                "documents.index.rmtree",
+            ) as mock_rmtree,
+        ):
+            ix = index.open_index()
+
+        self.assertIs(ix, expected_index)
+        self.assertGreaterEqual(mock_open_dir.call_count, 2)
+        mock_rmtree.assert_not_called()
+        mock_create_in.assert_not_called()
+        self.assertEqual(file_marker.read_text(), "keep")

uv.lock

@@ -1073,15 +1073,15 @@ wheels = [
 
 [[package]]
 name = "gotenberg-client"
-version = "0.13.1"
+version = "0.12.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "httpx", extra = ["http2"], marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "typing-extensions", marker = "(python_full_version < '3.11' and sys_platform == 'darwin') or (python_full_version < '3.11' and sys_platform == 'linux')" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/e4/6c/aaadd6657ca42fbd148b1c00604b98c1ead5a22552f4e5365ce5f0632430/gotenberg_client-0.13.1.tar.gz", hash = "sha256:cdd6bbb535cd739b87446cd1b4f6347ed7f9af6a0d4b19baf7c064b75528ee54", size = 1211143, upload-time = "2025-12-04T20:45:24.151Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/61/6d/07ea213c146bbe91dffebff2d8f4dc61e7076d3dd34d4fd1467f9163e752/gotenberg_client-0.12.0.tar.gz", hash = "sha256:1ab50878024469fc003c414ee9810ceeb00d4d7d7c36bd2fb75318fbff139e9b", size = 1210884, upload-time = "2025-10-15T15:32:37.669Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/79/f6/7a6e6785295332d2538f729ae19516cef712273a5ab8b90d015f08e37a45/gotenberg_client-0.13.1-py3-none-any.whl", hash = "sha256:613f7083a5e8a81699dd8d715c97e5806a424ac48920aad25d7c11b600cdfaf3", size = 51058, upload-time = "2025-12-04T20:45:22.603Z" },
+    { url = "https://files.pythonhosted.org/packages/12/39/fcb24ff053b1be7e5124f56c3d358706a23a328f685c6db33bc9dbc5472d/gotenberg_client-0.12.0-py3-none-any.whl", hash = "sha256:a540b35ac518e902c2860a88fbe448c15fe5a56fe8ec8604e6a2c8c2228fd0cb", size = 51051, upload-time = "2025-10-15T15:32:36.32Z" },
 ]
 
 [[package]]
@@ -2282,7 +2282,7 @@ requires-dist = [
     { name = "drf-writable-nested", specifier = "~=0.7.1" },
     { name = "filelock", specifier = "~=3.20.0" },
     { name = "flower", specifier = "~=2.0.1" },
-    { name = "gotenberg-client", specifier = "~=0.13.1" },
+    { name = "gotenberg-client", specifier = "~=0.12.0" },
     { name = "granian", extras = ["uvloop"], marker = "extra == 'webserver'", specifier = "~=2.5.1" },
     { name = "httpx-oauth", specifier = "~=0.16" },
     { name = "imap-tools", specifier = "~=1.11.0" },