From 3d85dc11279a37ef6c8ab229e9e084aa106fad66 Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Sun, 11 Jun 2023 13:20:40 -0700 Subject: [PATCH 1/2] Fix use of `PAPERLESS_DB_TIMEOUT` for all db types --- docs/configuration.md | 4 +- src/paperless/settings.py | 119 +++++++++++++++------------ src/paperless/tests/test_settings.py | 58 +++++++++++++ 3 files changed, 126 insertions(+), 55 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 35dc86ffb..da446401c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -139,8 +139,8 @@ changed here. `PAPERLESS_DB_TIMEOUT=` : Amount of time for a database connection to wait for the database to -unlock. Mostly applicable for an sqlite based installation, consider -changing to postgresql if you need to increase this. +unlock. Mostly applicable for sqlite based installation. Consider changing +to postgresql if you are having concurrency problems with sqlite. Defaults to unset, keeping the Django defaults. diff --git a/src/paperless/settings.py b/src/paperless/settings.py index f48f208f2..ad386e410 100644 --- a/src/paperless/settings.py +++ b/src/paperless/settings.py @@ -476,69 +476,82 @@ CSRF_COOKIE_NAME = f"{COOKIE_PREFIX}csrftoken" SESSION_COOKIE_NAME = f"{COOKIE_PREFIX}sessionid" LANGUAGE_COOKIE_NAME = f"{COOKIE_PREFIX}django_language" + ############################################################################### # Database # ############################################################################### - -DATABASES = { - "default": { - "ENGINE": "django.db.backends.sqlite3", - "NAME": os.path.join(DATA_DIR, "db.sqlite3"), - "OPTIONS": {}, - }, -} - -if os.getenv("PAPERLESS_DBHOST"): - # Have sqlite available as a second option for management commands - # This is important when migrating to/from sqlite - DATABASES["sqlite"] = DATABASES["default"].copy() - - DATABASES["default"] = { - "HOST": os.getenv("PAPERLESS_DBHOST"), - "NAME": os.getenv("PAPERLESS_DBNAME", "paperless"), - "USER": os.getenv("PAPERLESS_DBUSER", "paperless"), - "PASSWORD": os.getenv("PAPERLESS_DBPASS", "paperless"), - "OPTIONS": {}, +def _parse_db_settings() -> Dict: + databases = { + "default": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": os.path.join(DATA_DIR, "db.sqlite3"), + "OPTIONS": {}, + }, } - if os.getenv("PAPERLESS_DBPORT"): - DATABASES["default"]["PORT"] = os.getenv("PAPERLESS_DBPORT") + if os.getenv("PAPERLESS_DBHOST"): + # Have sqlite available as a second option for management commands + # This is important when migrating to/from sqlite + databases["sqlite"] = databases["default"].copy() - # Leave room for future extensibility - if os.getenv("PAPERLESS_DBENGINE") == "mariadb": - engine = "django.db.backends.mysql" - options = { - "read_default_file": "/etc/mysql/my.cnf", - "charset": "utf8mb4", - "ssl": { - "ssl_mode": os.getenv("PAPERLESS_DBSSLMODE", "PREFERRED"), - "ca": os.getenv("PAPERLESS_DBSSLROOTCERT", None), - "cert": os.getenv("PAPERLESS_DBSSLCERT", None), - "key": os.getenv("PAPERLESS_DBSSLKEY", None), - }, + databases["default"] = { + "HOST": os.getenv("PAPERLESS_DBHOST"), + "NAME": os.getenv("PAPERLESS_DBNAME", "paperless"), + "USER": os.getenv("PAPERLESS_DBUSER", "paperless"), + "PASSWORD": os.getenv("PAPERLESS_DBPASS", "paperless"), + "OPTIONS": {}, } + if os.getenv("PAPERLESS_DBPORT"): + databases["default"]["PORT"] = os.getenv("PAPERLESS_DBPORT") - # Silence Django error on old MariaDB versions. - # VARCHAR can support > 255 in modern versions - # https://docs.djangoproject.com/en/4.1/ref/checks/#database - # https://mariadb.com/kb/en/innodb-system-variables/#innodb_large_prefix - SILENCED_SYSTEM_CHECKS = ["mysql.W003"] + # Leave room for future extensibility + if os.getenv("PAPERLESS_DBENGINE") == "mariadb": + engine = "django.db.backends.mysql" + options = { + "read_default_file": "/etc/mysql/my.cnf", + "charset": "utf8mb4", + "ssl": { + "ssl_mode": os.getenv("PAPERLESS_DBSSLMODE", "PREFERRED"), + "ca": os.getenv("PAPERLESS_DBSSLROOTCERT", None), + "cert": os.getenv("PAPERLESS_DBSSLCERT", None), + "key": os.getenv("PAPERLESS_DBSSLKEY", None), + }, + } - else: # Default to PostgresDB - engine = "django.db.backends.postgresql_psycopg2" - options = { - "sslmode": os.getenv("PAPERLESS_DBSSLMODE", "prefer"), - "sslrootcert": os.getenv("PAPERLESS_DBSSLROOTCERT", None), - "sslcert": os.getenv("PAPERLESS_DBSSLCERT", None), - "sslkey": os.getenv("PAPERLESS_DBSSLKEY", None), - } + else: # Default to PostgresDB + engine = "django.db.backends.postgresql_psycopg2" + options = { + "sslmode": os.getenv("PAPERLESS_DBSSLMODE", "prefer"), + "sslrootcert": os.getenv("PAPERLESS_DBSSLROOTCERT", None), + "sslcert": os.getenv("PAPERLESS_DBSSLCERT", None), + "sslkey": os.getenv("PAPERLESS_DBSSLKEY", None), + } - DATABASES["default"]["ENGINE"] = engine - DATABASES["default"]["OPTIONS"].update(options) + databases["default"]["ENGINE"] = engine + databases["default"]["OPTIONS"].update(options) -if os.getenv("PAPERLESS_DB_TIMEOUT") is not None: - DATABASES["default"]["OPTIONS"].update( - {"timeout": float(os.getenv("PAPERLESS_DB_TIMEOUT"))}, - ) + if os.getenv("PAPERLESS_DB_TIMEOUT") is not None: + if databases["default"]["ENGINE"] == "django.db.backends.sqlite3": + databases["default"]["OPTIONS"].update( + {"timeout": float(os.getenv("PAPERLESS_DB_TIMEOUT"))}, + ) + else: + databases["default"]["OPTIONS"].update( + {"connect_timeout": float(os.getenv("PAPERLESS_DB_TIMEOUT"))}, + ) + databases["sqlite"]["OPTIONS"].update( + {"timeout": float(os.getenv("PAPERLESS_DB_TIMEOUT"))}, + ) + return databases + + +DATABASES = _parse_db_settings() + +if os.getenv("PAPERLESS_DBENGINE") == "mariadb": + # Silence Django error on old MariaDB versions. + # VARCHAR can support > 255 in modern versions + # https://docs.djangoproject.com/en/4.1/ref/checks/#database + # https://mariadb.com/kb/en/innodb-system-variables/#innodb_large_prefix + SILENCED_SYSTEM_CHECKS = ["mysql.W003"] DEFAULT_AUTO_FIELD = "django.db.models.AutoField" diff --git a/src/paperless/tests/test_settings.py b/src/paperless/tests/test_settings.py index 6b69765dd..41abb0a50 100644 --- a/src/paperless/tests/test_settings.py +++ b/src/paperless/tests/test_settings.py @@ -6,6 +6,7 @@ from unittest import mock from celery.schedules import crontab from paperless.settings import _parse_beat_schedule +from paperless.settings import _parse_db_settings from paperless.settings import _parse_ignore_dates from paperless.settings import _parse_redis_url from paperless.settings import default_threads_per_worker @@ -291,3 +292,60 @@ class TestCeleryScheduleParsing(TestCase): {}, schedule, ) + + +class TestDBSettings(TestCase): + def test_db_timeout_with_sqlite(self): + """ + GIVEN: + - PAPERLESS_DB_TIMEOUT is set + WHEN: + - Settings are parsed + THEN: + - PAPERLESS_DB_TIMEOUT set for sqlite + """ + with mock.patch.dict( + os.environ, + { + "PAPERLESS_DB_TIMEOUT": "10", + }, + ): + databases = _parse_db_settings() + + self.assertDictEqual( + { + "timeout": 10.0, + }, + databases["default"]["OPTIONS"], + ) + + def test_db_timeout_with_not_sqlite(self): + """ + GIVEN: + - PAPERLESS_DB_TIMEOUT is set but db is not sqlite + WHEN: + - Settings are parsed + THEN: + - PAPERLESS_DB_TIMEOUT set correctly in non-sqlite db & for fallback sqlite db + """ + with mock.patch.dict( + os.environ, + { + "PAPERLESS_DBHOST": "127.0.0.1", + "PAPERLESS_DB_TIMEOUT": "10", + }, + ): + databases = _parse_db_settings() + + self.assertDictContainsSubset( + { + "connect_timeout": 10.0, + }, + databases["default"]["OPTIONS"], + ) + self.assertDictEqual( + { + "timeout": 10.0, + }, + databases["sqlite"]["OPTIONS"], + ) From dd6ae1328100ba7339b995a64963711161230de3 Mon Sep 17 00:00:00 2001 From: Trenton H <797416+stumpylog@users.noreply.github.com> Date: Mon, 12 Jun 2023 08:45:57 -0700 Subject: [PATCH 2/2] Changes the type of the connection timeout to be an int, not a float --- docs/configuration.md | 2 +- src/paperless/settings.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index da446401c..d3874256f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -136,7 +136,7 @@ changed here. Defaults to unset, using the documented path in the home directory. -`PAPERLESS_DB_TIMEOUT=` +`PAPERLESS_DB_TIMEOUT=` : Amount of time for a database connection to wait for the database to unlock. Mostly applicable for sqlite based installation. Consider changing diff --git a/src/paperless/settings.py b/src/paperless/settings.py index ad386e410..c18fd5717 100644 --- a/src/paperless/settings.py +++ b/src/paperless/settings.py @@ -532,14 +532,14 @@ def _parse_db_settings() -> Dict: if os.getenv("PAPERLESS_DB_TIMEOUT") is not None: if databases["default"]["ENGINE"] == "django.db.backends.sqlite3": databases["default"]["OPTIONS"].update( - {"timeout": float(os.getenv("PAPERLESS_DB_TIMEOUT"))}, + {"timeout": int(os.getenv("PAPERLESS_DB_TIMEOUT"))}, ) else: databases["default"]["OPTIONS"].update( - {"connect_timeout": float(os.getenv("PAPERLESS_DB_TIMEOUT"))}, + {"connect_timeout": int(os.getenv("PAPERLESS_DB_TIMEOUT"))}, ) databases["sqlite"]["OPTIONS"].update( - {"timeout": float(os.getenv("PAPERLESS_DB_TIMEOUT"))}, + {"timeout": int(os.getenv("PAPERLESS_DB_TIMEOUT"))}, ) return databases