Merge pull request #3576 from paperless-ngx/fix/issue-3569

Fix use of `PAPERLESS_DB_TIMEOUT` for all db types

docs/configuration.md

@@ -136,11 +136,11 @@ changed here.
 
     Defaults to unset, using the documented path in the home directory.
 
-`PAPERLESS_DB_TIMEOUT=<float>`
+`PAPERLESS_DB_TIMEOUT=<int>`
 
 : Amount of time for a database connection to wait for the database to
-unlock. Mostly applicable for an sqlite based installation, consider
+unlock. Mostly applicable for sqlite based installation. Consider changing
-changing to postgresql if you need to increase this.
+to postgresql if you are having concurrency problems with sqlite.
 
     Defaults to unset, keeping the Django defaults.
 

src/paperless/settings.py

@@ -476,69 +476,82 @@ CSRF_COOKIE_NAME = f"{COOKIE_PREFIX}csrftoken"
 SESSION_COOKIE_NAME = f"{COOKIE_PREFIX}sessionid"
 LANGUAGE_COOKIE_NAME = f"{COOKIE_PREFIX}django_language"
 
+
 ###############################################################################
 # Database                                                                    #
 ###############################################################################
-
+def _parse_db_settings() -> Dict:
-DATABASES = {
+    databases = {
-    "default": {
+        "default": {
-        "ENGINE": "django.db.backends.sqlite3",
+            "ENGINE": "django.db.backends.sqlite3",
-        "NAME": os.path.join(DATA_DIR, "db.sqlite3"),
+            "NAME": os.path.join(DATA_DIR, "db.sqlite3"),
-        "OPTIONS": {},
+            "OPTIONS": {},
-    },
+        },
-}
-
-if os.getenv("PAPERLESS_DBHOST"):
-    # Have sqlite available as a second option for management commands
-    # This is important when migrating to/from sqlite
-    DATABASES["sqlite"] = DATABASES["default"].copy()
-
-    DATABASES["default"] = {
-        "HOST": os.getenv("PAPERLESS_DBHOST"),
-        "NAME": os.getenv("PAPERLESS_DBNAME", "paperless"),
-        "USER": os.getenv("PAPERLESS_DBUSER", "paperless"),
-        "PASSWORD": os.getenv("PAPERLESS_DBPASS", "paperless"),
-        "OPTIONS": {},
     }
-    if os.getenv("PAPERLESS_DBPORT"):
+    if os.getenv("PAPERLESS_DBHOST"):
-        DATABASES["default"]["PORT"] = os.getenv("PAPERLESS_DBPORT")
+        # Have sqlite available as a second option for management commands
+        # This is important when migrating to/from sqlite
+        databases["sqlite"] = databases["default"].copy()
 
-    # Leave room for future extensibility
+        databases["default"] = {
-    if os.getenv("PAPERLESS_DBENGINE") == "mariadb":
+            "HOST": os.getenv("PAPERLESS_DBHOST"),
-        engine = "django.db.backends.mysql"
+            "NAME": os.getenv("PAPERLESS_DBNAME", "paperless"),
-        options = {
+            "USER": os.getenv("PAPERLESS_DBUSER", "paperless"),
-            "read_default_file": "/etc/mysql/my.cnf",
+            "PASSWORD": os.getenv("PAPERLESS_DBPASS", "paperless"),
-            "charset": "utf8mb4",
+            "OPTIONS": {},
-            "ssl": {
-                "ssl_mode": os.getenv("PAPERLESS_DBSSLMODE", "PREFERRED"),
-                "ca": os.getenv("PAPERLESS_DBSSLROOTCERT", None),
-                "cert": os.getenv("PAPERLESS_DBSSLCERT", None),
-                "key": os.getenv("PAPERLESS_DBSSLKEY", None),
-            },
         }
+        if os.getenv("PAPERLESS_DBPORT"):
+            databases["default"]["PORT"] = os.getenv("PAPERLESS_DBPORT")
 
-        # Silence Django error on old MariaDB versions.
+        # Leave room for future extensibility
-        # VARCHAR can support > 255 in modern versions
+        if os.getenv("PAPERLESS_DBENGINE") == "mariadb":
-        # https://docs.djangoproject.com/en/4.1/ref/checks/#database
+            engine = "django.db.backends.mysql"
-        # https://mariadb.com/kb/en/innodb-system-variables/#innodb_large_prefix
+            options = {
-        SILENCED_SYSTEM_CHECKS = ["mysql.W003"]
+                "read_default_file": "/etc/mysql/my.cnf",
+                "charset": "utf8mb4",
+                "ssl": {
+                    "ssl_mode": os.getenv("PAPERLESS_DBSSLMODE", "PREFERRED"),
+                    "ca": os.getenv("PAPERLESS_DBSSLROOTCERT", None),
+                    "cert": os.getenv("PAPERLESS_DBSSLCERT", None),
+                    "key": os.getenv("PAPERLESS_DBSSLKEY", None),
+                },
+            }
 
-    else:  # Default to PostgresDB
+        else:  # Default to PostgresDB
-        engine = "django.db.backends.postgresql_psycopg2"
+            engine = "django.db.backends.postgresql_psycopg2"
-        options = {
+            options = {
-            "sslmode": os.getenv("PAPERLESS_DBSSLMODE", "prefer"),
+                "sslmode": os.getenv("PAPERLESS_DBSSLMODE", "prefer"),
-            "sslrootcert": os.getenv("PAPERLESS_DBSSLROOTCERT", None),
+                "sslrootcert": os.getenv("PAPERLESS_DBSSLROOTCERT", None),
-            "sslcert": os.getenv("PAPERLESS_DBSSLCERT", None),
+                "sslcert": os.getenv("PAPERLESS_DBSSLCERT", None),
-            "sslkey": os.getenv("PAPERLESS_DBSSLKEY", None),
+                "sslkey": os.getenv("PAPERLESS_DBSSLKEY", None),
-        }
+            }
 
-    DATABASES["default"]["ENGINE"] = engine
+        databases["default"]["ENGINE"] = engine
-    DATABASES["default"]["OPTIONS"].update(options)
+        databases["default"]["OPTIONS"].update(options)
 
-if os.getenv("PAPERLESS_DB_TIMEOUT") is not None:
+    if os.getenv("PAPERLESS_DB_TIMEOUT") is not None:
-    DATABASES["default"]["OPTIONS"].update(
+        if databases["default"]["ENGINE"] == "django.db.backends.sqlite3":
-        {"timeout": float(os.getenv("PAPERLESS_DB_TIMEOUT"))},
+            databases["default"]["OPTIONS"].update(
-    )
+                {"timeout": int(os.getenv("PAPERLESS_DB_TIMEOUT"))},
+            )
+        else:
+            databases["default"]["OPTIONS"].update(
+                {"connect_timeout": int(os.getenv("PAPERLESS_DB_TIMEOUT"))},
+            )
+            databases["sqlite"]["OPTIONS"].update(
+                {"timeout": int(os.getenv("PAPERLESS_DB_TIMEOUT"))},
+            )
+    return databases
+
+
+DATABASES = _parse_db_settings()
+
+if os.getenv("PAPERLESS_DBENGINE") == "mariadb":
+    # Silence Django error on old MariaDB versions.
+    # VARCHAR can support > 255 in modern versions
+    # https://docs.djangoproject.com/en/4.1/ref/checks/#database
+    # https://mariadb.com/kb/en/innodb-system-variables/#innodb_large_prefix
+    SILENCED_SYSTEM_CHECKS = ["mysql.W003"]
 
 DEFAULT_AUTO_FIELD = "django.db.models.AutoField"
 

src/paperless/tests/test_settings.py

@@ -6,6 +6,7 @@ from unittest import mock
 from celery.schedules import crontab
 
 from paperless.settings import _parse_beat_schedule
+from paperless.settings import _parse_db_settings
 from paperless.settings import _parse_ignore_dates
 from paperless.settings import _parse_redis_url
 from paperless.settings import default_threads_per_worker
@@ -291,3 +292,60 @@ class TestCeleryScheduleParsing(TestCase):
             {},
             schedule,
         )
+
+
+class TestDBSettings(TestCase):
+    def test_db_timeout_with_sqlite(self):
+        """
+        GIVEN:
+            - PAPERLESS_DB_TIMEOUT is set
+        WHEN:
+            - Settings are parsed
+        THEN:
+            - PAPERLESS_DB_TIMEOUT set for sqlite
+        """
+        with mock.patch.dict(
+            os.environ,
+            {
+                "PAPERLESS_DB_TIMEOUT": "10",
+            },
+        ):
+            databases = _parse_db_settings()
+
+            self.assertDictEqual(
+                {
+                    "timeout": 10.0,
+                },
+                databases["default"]["OPTIONS"],
+            )
+
+    def test_db_timeout_with_not_sqlite(self):
+        """
+        GIVEN:
+            - PAPERLESS_DB_TIMEOUT is set but db is not sqlite
+        WHEN:
+            - Settings are parsed
+        THEN:
+            - PAPERLESS_DB_TIMEOUT set correctly in non-sqlite db & for fallback sqlite db
+        """
+        with mock.patch.dict(
+            os.environ,
+            {
+                "PAPERLESS_DBHOST": "127.0.0.1",
+                "PAPERLESS_DB_TIMEOUT": "10",
+            },
+        ):
+            databases = _parse_db_settings()
+
+            self.assertDictContainsSubset(
+                {
+                    "connect_timeout": 10.0,
+                },
+                databases["default"]["OPTIONS"],
+            )
+            self.assertDictEqual(
+                {
+                    "timeout": 10.0,
+                },
+                databases["sqlite"]["OPTIONS"],
+            )