diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d0e070e15..e042683de 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,6 +15,7 @@ env: DEFAULT_UV_VERSION: "0.8.x" # This is the default version of Python to use in most steps which aren't specific DEFAULT_PYTHON_VERSION: "3.11" + NLTK_DATA: "/usr/share/nltk_data" jobs: pre-commit: # We want to run on external PRs, but not on our own internal PRs as they'll be run @@ -121,8 +122,11 @@ jobs: - name: List installed Python dependencies run: | uv pip list + - name: Install or update NLTK dependencies + run: uv run python -m nltk.downloader punkt punkt_tab snowball_data stopwords -d ${{ env.NLTK_DATA }} - name: Tests env: + NLTK_DATA: ${{ env.NLTK_DATA }} PAPERLESS_CI_TEST: 1 # Enable paperless_mail testing against real server PAPERLESS_MAIL_TEST_HOST: ${{ secrets.TEST_MAIL_HOST }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 64a183186..2c50e81a6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,7 +31,7 @@ repos: rev: v2.4.1 hooks: - id: codespell - exclude: "(^src-ui/src/locale/)|(^src-ui/pnpm-lock.yaml)|(^src-ui/e2e/)|(^src/paperless_mail/tests/samples/)" + exclude: "(^src-ui/src/locale/)|(^src-ui/pnpm-lock.yaml)|(^src-ui/e2e/)|(^src/paperless_mail/tests/samples/)|(^src/documents/tests/samples/)" exclude_types: - pofile - json diff --git a/docs/configuration.md b/docs/configuration.md index 92500d26c..9b7041962 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -159,6 +159,23 @@ Available options are `postgresql` and `mariadb`. Defaults to unset, which uses Django’s built-in defaults. +#### [`PAPERLESS_DB_POOLSIZE=`](#PAPERLESS_DB_POOLSIZE) {#PAPERLESS_DB_POOLSIZE} + +: Defines the maximum number of database connections to keep in the pool. + + Only applies to PostgreSQL. This setting is ignored for other database engines. + + The value must be greater than or equal to 1 to be used. + Defaults to unset, which disables connection pooling. + + !!! note + + A small pool is typically sufficient — for example, a size of 4. + Make sure your PostgreSQL server's max_connections setting is large enough to handle: + ```(Paperless workers + Celery workers) × pool size + safety margin``` + For example, with 4 Paperless workers and 2 Celery workers, and a pool size of 4: + (4 + 2) × 4 + 10 = 34 connections required. + #### [`PAPERLESS_DB_READ_CACHE_ENABLED=`](#PAPERLESS_DB_READ_CACHE_ENABLED) {#PAPERLESS_DB_READ_CACHE_ENABLED} : Caches the database read query results into Redis. This can significantly improve application response times by caching database queries, at the cost of slightly increased memory usage. diff --git a/docs/usage.md b/docs/usage.md index 5c377a466..a2686ab7d 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -30,6 +30,9 @@ Each document has data fields that you can assign to them: - A _document type_ is used to demarcate the type of a document such as letter, bank statement, invoice, contract, etc. It is used to identify what a document is about. +- The document _storage path_ is the location where the document files + are stored. See [Storage Paths](advanced_usage.md#storage-paths) for + more information. - The _date added_ of a document is the date the document was scanned into paperless. You cannot and should not change this date. - The _date created_ of a document is the date the document was diff --git a/pyproject.toml b/pyproject.toml index 26f192231..4b7aaa68d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,6 +53,7 @@ dependencies = [ "ocrmypdf~=16.10.0", "pathvalidate~=3.3.1", "pdf2image~=1.17.0", + "psycopg-pool", "python-dateutil~=2.9.0", "python-dotenv~=1.1.0", "python-gnupg~=0.5.4", @@ -63,7 +64,7 @@ dependencies = [ "redis[hiredis]~=5.2.1", "scikit-learn~=1.7.0", "setproctitle~=1.3.4", - "tika-client~=0.9.0", + "tika-client~=0.10.0", "tqdm~=4.67.1", "watchdog~=6.0", "whitenoise~=6.9", @@ -75,9 +76,10 @@ optional-dependencies.mariadb = [ "mysqlclient~=2.2.7", ] optional-dependencies.postgres = [ - "psycopg[c]==3.2.9", + "psycopg[c,pool]==3.2.9", # Direct dependency for proper resolution of the pre-built wheels "psycopg-c==3.2.9", + "psycopg-pool==3.2.6", ] optional-dependencies.webserver = [ "granian[uvloop]~=2.4.1", @@ -203,15 +205,9 @@ lint.per-file-ignores."docker/wait-for-redis.py" = [ "INP001", "T201", ] -lint.per-file-ignores."src/documents/file_handling.py" = [ - "PTH", -] # TODO Enable & remove lint.per-file-ignores."src/documents/management/commands/document_consumer.py" = [ "PTH", ] # TODO Enable & remove -lint.per-file-ignores."src/documents/management/commands/document_exporter.py" = [ - "PTH", -] # TODO Enable & remove lint.per-file-ignores."src/documents/migrations/1012_fix_archive_files.py" = [ "PTH", ] # TODO Enable & remove @@ -221,9 +217,6 @@ lint.per-file-ignores."src/documents/models.py" = [ lint.per-file-ignores."src/documents/parsers.py" = [ "PTH", ] # TODO Enable & remove -lint.per-file-ignores."src/documents/signals/handlers.py" = [ - "PTH", -] # TODO Enable & remove lint.per-file-ignores."src/paperless_tesseract/tests/test_parser.py" = [ "RUF001", ] diff --git a/src-ui/messages.xlf b/src-ui/messages.xlf index 7b26a3e7a..5bf7c66e1 100644 --- a/src-ui/messages.xlf +++ b/src-ui/messages.xlf @@ -332,19 +332,19 @@ src/app/components/manage/management-list/management-list.component.html - 102 + 103 src/app/components/manage/management-list/management-list.component.html - 102 + 103 src/app/components/manage/management-list/management-list.component.html - 102 + 103 src/app/components/manage/management-list/management-list.component.html - 102 + 103 @@ -545,7 +545,7 @@ src/app/components/admin/settings/settings.component.html - 361 + 362 src/app/components/common/edit-dialog/correspondent-edit-dialog/correspondent-edit-dialog.component.html @@ -605,7 +605,7 @@ src/app/components/manage/saved-views/saved-views.component.html - 73 + 74 @@ -763,19 +763,19 @@ src/app/components/manage/management-list/management-list.component.html - 51 + 52 src/app/components/manage/management-list/management-list.component.html - 51 + 52 src/app/components/manage/management-list/management-list.component.html - 51 + 52 src/app/components/manage/management-list/management-list.component.html - 51 + 52 src/app/components/manage/saved-views/saved-views.component.html @@ -1225,19 +1225,19 @@ src/app/components/manage/management-list/management-list.component.html - 6 + 7 src/app/components/manage/management-list/management-list.component.html - 6 + 7 src/app/components/manage/management-list/management-list.component.html - 6 + 7 src/app/components/manage/management-list/management-list.component.html - 6 + 7 @@ -1432,7 +1432,7 @@ Cancel src/app/components/admin/settings/settings.component.html - 362 + 361 src/app/components/common/confirm-dialog/confirm-dialog.component.ts @@ -1500,7 +1500,7 @@ src/app/components/manage/saved-views/saved-views.component.html - 74 + 73 @@ -1598,19 +1598,19 @@ src/app/components/manage/management-list/management-list.component.html - 3 + 4 src/app/components/manage/management-list/management-list.component.html - 3 + 4 src/app/components/manage/management-list/management-list.component.html - 3 + 4 src/app/components/manage/management-list/management-list.component.html - 3 + 4 @@ -1696,35 +1696,35 @@ src/app/components/manage/management-list/management-list.component.html - 20 + 21 src/app/components/manage/management-list/management-list.component.html - 20 + 21 src/app/components/manage/management-list/management-list.component.html - 20 + 21 src/app/components/manage/management-list/management-list.component.html - 20 + 21 src/app/components/manage/management-list/management-list.component.html - 37 + 38 src/app/components/manage/management-list/management-list.component.html - 37 + 38 src/app/components/manage/management-list/management-list.component.html - 37 + 38 src/app/components/manage/management-list/management-list.component.html - 37 + 38 src/app/components/manage/workflows/workflows.component.html @@ -1816,19 +1816,19 @@ src/app/components/manage/management-list/management-list.component.html - 43 + 44 src/app/components/manage/management-list/management-list.component.html - 43 + 44 src/app/components/manage/management-list/management-list.component.html - 43 + 44 src/app/components/manage/management-list/management-list.component.html - 43 + 44 src/app/components/manage/saved-views/saved-views.component.html @@ -2121,51 +2121,51 @@ src/app/components/manage/management-list/management-list.component.html - 9 + 10 src/app/components/manage/management-list/management-list.component.html - 9 + 10 src/app/components/manage/management-list/management-list.component.html - 9 + 10 src/app/components/manage/management-list/management-list.component.html - 9 + 10 src/app/components/manage/management-list/management-list.component.html - 84 + 85 src/app/components/manage/management-list/management-list.component.html - 84 + 85 src/app/components/manage/management-list/management-list.component.html - 84 + 85 src/app/components/manage/management-list/management-list.component.html - 84 + 85 src/app/components/manage/management-list/management-list.component.html - 96 + 97 src/app/components/manage/management-list/management-list.component.html - 96 + 97 src/app/components/manage/management-list/management-list.component.html - 96 + 97 src/app/components/manage/management-list/management-list.component.html - 96 + 97 src/app/components/manage/management-list/management-list.component.ts @@ -2440,35 +2440,35 @@ src/app/components/manage/management-list/management-list.component.html - 83 + 84 src/app/components/manage/management-list/management-list.component.html - 83 + 84 src/app/components/manage/management-list/management-list.component.html - 83 + 84 src/app/components/manage/management-list/management-list.component.html - 83 + 84 src/app/components/manage/management-list/management-list.component.html - 93 + 94 src/app/components/manage/management-list/management-list.component.html - 93 + 94 src/app/components/manage/management-list/management-list.component.html - 93 + 94 src/app/components/manage/management-list/management-list.component.html - 93 + 94 src/app/components/manage/workflows/workflows.component.html @@ -5227,19 +5227,19 @@ src/app/components/manage/management-list/management-list.component.html - 12 + 13 src/app/components/manage/management-list/management-list.component.html - 12 + 13 src/app/components/manage/management-list/management-list.component.html - 12 + 13 src/app/components/manage/management-list/management-list.component.html - 12 + 13 @@ -8333,19 +8333,19 @@ src/app/components/manage/management-list/management-list.component.html - 86 + 87 src/app/components/manage/management-list/management-list.component.html - 86 + 87 src/app/components/manage/management-list/management-list.component.html - 86 + 87 src/app/components/manage/management-list/management-list.component.html - 86 + 87 @@ -8672,76 +8672,76 @@ Filter by: src/app/components/manage/management-list/management-list.component.html - 19 + 20 src/app/components/manage/management-list/management-list.component.html - 19 + 20 src/app/components/manage/management-list/management-list.component.html - 19 + 20 src/app/components/manage/management-list/management-list.component.html - 19 + 20 Matching src/app/components/manage/management-list/management-list.component.html - 38 + 39 src/app/components/manage/management-list/management-list.component.html - 38 + 39 src/app/components/manage/management-list/management-list.component.html - 38 + 39 src/app/components/manage/management-list/management-list.component.html - 38 + 39 Document count src/app/components/manage/management-list/management-list.component.html - 39 + 40 src/app/components/manage/management-list/management-list.component.html - 39 + 40 src/app/components/manage/management-list/management-list.component.html - 39 + 40 src/app/components/manage/management-list/management-list.component.html - 39 + 40 {VAR_PLURAL, plural, =1 {One } other { total }} src/app/components/manage/management-list/management-list.component.html - 118 + 119 src/app/components/manage/management-list/management-list.component.html - 118 + 119 src/app/components/manage/management-list/management-list.component.html - 118 + 119 src/app/components/manage/management-list/management-list.component.html - 118 + 119 diff --git a/src-ui/src/app/components/admin/config/config.component.html b/src-ui/src/app/components/admin/config/config.component.html index 0f74339fb..e1d7340a6 100644 --- a/src-ui/src/app/components/admin/config/config.component.html +++ b/src-ui/src/app/components/admin/config/config.component.html @@ -50,7 +50,7 @@