Performance: add setting to enable DB connection pooling for PostgreSQL (#10354)

---------

Co-authored-by: Trenton H <797416+stumpylog@users.noreply.github.com>

src/documents/signals/handlers.py

@@ -12,11 +12,13 @@ from celery.signals import before_task_publish
 from celery.signals import task_failure
 from celery.signals import task_postrun
 from celery.signals import task_prerun
+from celery.signals import worker_process_init
 from django.conf import settings
 from django.contrib.auth.models import Group
 from django.contrib.auth.models import User
 from django.db import DatabaseError
 from django.db import close_old_connections
+from django.db import connections
 from django.db import models
 from django.db.models import Q
 from django.dispatch import receiver
@@ -1439,3 +1441,18 @@ def task_failure_handler(
             task_instance.save()
     except Exception:  # pragma: no cover
         logger.exception("Updating PaperlessTask failed")
+
+
+@worker_process_init.connect
+def close_connection_pool_on_worker_init(**kwargs):
+    """
+    Close the DB connection pool for each Celery child process after it starts.
+
+    This is necessary because the parent process parse the Django configuration,
+    initializes connection pools then forks.
+
+    Closing these pools after forking ensures child processes have a valid connection.
+    """
+    for conn in connections.all(initialized_only=True):
+        if conn.alias == "default" and hasattr(conn, "pool") and conn.pool:
+            conn.close_pool()

src/paperless/settings.py

@@ -703,6 +703,9 @@ def _parse_db_settings() -> dict:
         # Leave room for future extensibility
         if os.getenv("PAPERLESS_DBENGINE") == "mariadb":
             engine = "django.db.backends.mysql"
+            # Contrary to Postgres, Django does not natively support connection pooling for MariaDB.
+            # However, since MariaDB uses threads instead of forks, establishing connections is significantly faster
+            # compared to PostgreSQL, so the lack of pooling is not an issue
             options = {
                 "read_default_file": "/etc/mysql/my.cnf",
                 "charset": "utf8mb4",
@@ -722,6 +725,15 @@ def _parse_db_settings() -> dict:
                 "sslcert": os.getenv("PAPERLESS_DBSSLCERT", None),
                 "sslkey": os.getenv("PAPERLESS_DBSSLKEY", None),
             }
+            if int(os.getenv("PAPERLESS_DB_POOLSIZE", 0)) > 0:
+                options.update(
+                    {
+                        "pool": {
+                            "min_size": 1,
+                            "max_size": int(os.getenv("PAPERLESS_DB_POOLSIZE")),
+                        },
+                    },
+                )
 
         databases["default"]["ENGINE"] = engine
         databases["default"]["OPTIONS"].update(options)