mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2025-08-16 00:36:22 +00:00
Compare commits
6 Commits
feature-re
...
dependabot
Author | SHA1 | Date | |
---|---|---|---|
![]() |
8f4e97edd9 | ||
![]() |
42bdbc1b2d | ||
![]() |
2f529a9500 | ||
![]() |
ee6b700243 | ||
![]() |
b1a84c65ed | ||
![]() |
edb8c06e2a |
@@ -1,3 +0,0 @@
|
|||||||
[codespell]
|
|
||||||
write-changes = True
|
|
||||||
ignore-words-list = criterias,afterall,valeu,ureue,equest,ure,assertIn
|
|
@@ -31,7 +31,6 @@ repos:
|
|||||||
rev: v2.4.1
|
rev: v2.4.1
|
||||||
hooks:
|
hooks:
|
||||||
- id: codespell
|
- id: codespell
|
||||||
exclude: "(^src-ui/src/locale/)|(^src-ui/pnpm-lock.yaml)|(^src-ui/e2e/)|(^src/paperless_mail/tests/samples/)|(^src/documents/tests/samples/)"
|
|
||||||
exclude_types:
|
exclude_types:
|
||||||
- pofile
|
- pofile
|
||||||
- json
|
- json
|
||||||
|
@@ -32,7 +32,7 @@ RUN set -eux \
|
|||||||
# Purpose: Installs s6-overlay and rootfs
|
# Purpose: Installs s6-overlay and rootfs
|
||||||
# Comments:
|
# Comments:
|
||||||
# - Don't leave anything extra in here either
|
# - Don't leave anything extra in here either
|
||||||
FROM ghcr.io/astral-sh/uv:0.8.4-python3.12-bookworm-slim AS s6-overlay-base
|
FROM ghcr.io/astral-sh/uv:0.8.8-python3.12-bookworm-slim AS s6-overlay-base
|
||||||
|
|
||||||
WORKDIR /usr/src/s6
|
WORKDIR /usr/src/s6
|
||||||
|
|
||||||
|
@@ -179,10 +179,14 @@ following:
|
|||||||
|
|
||||||
### Database Upgrades
|
### Database Upgrades
|
||||||
|
|
||||||
In general, paperless does not require a specific version of PostgreSQL or MariaDB and it is
|
Paperless-ngx is compatible with Django-supported versions of PostgreSQL and MariaDB and it is generally
|
||||||
safe to update them to newer versions. However, you should always take a backup and follow
|
safe to update them to newer versions. However, you should always take a backup and follow
|
||||||
the instructions from your database's documentation for how to upgrade between major versions.
|
the instructions from your database's documentation for how to upgrade between major versions.
|
||||||
|
|
||||||
|
!!! note
|
||||||
|
|
||||||
|
As of Paperless-ngx v2.18, the minimum supported version of PostgreSQL is 13.
|
||||||
|
|
||||||
For PostgreSQL, refer to [Upgrading a PostgreSQL Cluster](https://www.postgresql.org/docs/current/upgrading.html).
|
For PostgreSQL, refer to [Upgrading a PostgreSQL Cluster](https://www.postgresql.org/docs/current/upgrading.html).
|
||||||
|
|
||||||
For MariaDB, refer to [Upgrading MariaDB](https://mariadb.com/kb/en/upgrading/)
|
For MariaDB, refer to [Upgrading MariaDB](https://mariadb.com/kb/en/upgrading/)
|
||||||
|
@@ -434,6 +434,136 @@ provided. The template is provided as a string, potentially multiline, and rende
|
|||||||
In addition, the entire Document instance is available to be utilized in a more advanced way, as well as some variables which only make sense to be accessed
|
In addition, the entire Document instance is available to be utilized in a more advanced way, as well as some variables which only make sense to be accessed
|
||||||
with more complex logic.
|
with more complex logic.
|
||||||
|
|
||||||
|
#### Custom Jinja2 Filters
|
||||||
|
|
||||||
|
##### Custom Field Access
|
||||||
|
|
||||||
|
The `get_cf_value` filter retrieves a value from custom field data with optional default fallback.
|
||||||
|
|
||||||
|
###### Syntax
|
||||||
|
|
||||||
|
```jinja2
|
||||||
|
{{ custom_fields | get_cf_value('field_name') }}
|
||||||
|
{{ custom_fields | get_cf_value('field_name', 'default_value') }}
|
||||||
|
```
|
||||||
|
|
||||||
|
###### Parameters
|
||||||
|
|
||||||
|
- `custom_fields`: This _must_ be the provided custom field data
|
||||||
|
- `name` (str): Name of the custom field to retrieve
|
||||||
|
- `default` (str, optional): Default value to return if field is not found or has no value
|
||||||
|
|
||||||
|
###### Returns
|
||||||
|
|
||||||
|
- `str | None`: The field value, default value, or `None` if neither exists
|
||||||
|
|
||||||
|
###### Examples
|
||||||
|
|
||||||
|
```jinja2
|
||||||
|
<!-- Basic usage -->
|
||||||
|
{{ custom_fields | get_cf_value('department') }}
|
||||||
|
|
||||||
|
<!-- With default value -->
|
||||||
|
{{ custom_fields | get_cf_value('phone', 'Not provided') }}
|
||||||
|
```
|
||||||
|
|
||||||
|
##### Datetime Formatting
|
||||||
|
|
||||||
|
The `format_datetime`filter formats a datetime string or datetime object using Python's strftime formatting.
|
||||||
|
|
||||||
|
###### Syntax
|
||||||
|
|
||||||
|
```jinja2
|
||||||
|
{{ datetime_value | format_datetime('%Y-%m-%d %H:%M:%S') }}
|
||||||
|
```
|
||||||
|
|
||||||
|
###### Parameters
|
||||||
|
|
||||||
|
- `value` (str | datetime): Date/time value to format (strings will be parsed automatically)
|
||||||
|
- `format` (str): Python strftime format string
|
||||||
|
|
||||||
|
###### Returns
|
||||||
|
|
||||||
|
- `str`: Formatted datetime string
|
||||||
|
|
||||||
|
###### Examples
|
||||||
|
|
||||||
|
```jinja2
|
||||||
|
<!-- Format datetime object -->
|
||||||
|
{{ created_at | format_datetime('%B %d, %Y at %I:%M %p') }}
|
||||||
|
<!-- Output: "January 15, 2024 at 02:30 PM" -->
|
||||||
|
|
||||||
|
<!-- Format datetime string -->
|
||||||
|
{{ "2024-01-15T14:30:00" | format_datetime('%m/%d/%Y') }}
|
||||||
|
<!-- Output: "01/15/2024" -->
|
||||||
|
|
||||||
|
<!-- Custom formatting -->
|
||||||
|
{{ timestamp | format_datetime('%A, %B %d, %Y') }}
|
||||||
|
<!-- Output: "Monday, January 15, 2024" -->
|
||||||
|
```
|
||||||
|
|
||||||
|
See the [strftime format code documentation](https://docs.python.org/3.13/library/datetime.html#strftime-and-strptime-format-codes)
|
||||||
|
for the possible codes and their meanings.
|
||||||
|
|
||||||
|
##### Date Localization
|
||||||
|
|
||||||
|
The `localize_date` filter formats a date or datetime object into a localized string using Babel internationalization.
|
||||||
|
This takes into account the provided locale for translation.
|
||||||
|
|
||||||
|
###### Syntax
|
||||||
|
|
||||||
|
```jinja2
|
||||||
|
{{ date_value | localize_date('medium', 'en_US') }}
|
||||||
|
{{ datetime_value | localize_date('short', 'fr_FR') }}
|
||||||
|
```
|
||||||
|
|
||||||
|
###### Parameters
|
||||||
|
|
||||||
|
- `value` (date | datetime): Date or datetime object to format (datetime should be timezone-aware)
|
||||||
|
- `format` (str): Format type - either a Babel preset ('short', 'medium', 'long', 'full') or custom pattern
|
||||||
|
- `locale` (str): Locale code for localization (e.g., 'en_US', 'fr_FR', 'de_DE')
|
||||||
|
|
||||||
|
###### Returns
|
||||||
|
|
||||||
|
- `str`: Localized, formatted date string
|
||||||
|
|
||||||
|
###### Examples
|
||||||
|
|
||||||
|
```jinja2
|
||||||
|
<!-- Preset formats -->
|
||||||
|
{{ created_date | localize_date('short', 'en_US') }}
|
||||||
|
<!-- Output: "1/15/24" -->
|
||||||
|
|
||||||
|
{{ created_date | localize_date('medium', 'en_US') }}
|
||||||
|
<!-- Output: "Jan 15, 2024" -->
|
||||||
|
|
||||||
|
{{ created_date | localize_date('long', 'en_US') }}
|
||||||
|
<!-- Output: "January 15, 2024" -->
|
||||||
|
|
||||||
|
{{ created_date | localize_date('full', 'en_US') }}
|
||||||
|
<!-- Output: "Monday, January 15, 2024" -->
|
||||||
|
|
||||||
|
<!-- Different locales -->
|
||||||
|
{{ created_date | localize_date('medium', 'fr_FR') }}
|
||||||
|
<!-- Output: "15 janv. 2024" -->
|
||||||
|
|
||||||
|
{{ created_date | localize_date('medium', 'de_DE') }}
|
||||||
|
<!-- Output: "15.01.2024" -->
|
||||||
|
|
||||||
|
<!-- Custom patterns -->
|
||||||
|
{{ created_date | localize_date('dd/MM/yyyy', 'en_GB') }}
|
||||||
|
<!-- Output: "15/01/2024" -->
|
||||||
|
```
|
||||||
|
|
||||||
|
See the [supported format codes](https://unicode.org/reports/tr35/tr35-dates.html#Date_Format_Patterns) for more options.
|
||||||
|
|
||||||
|
### Format Presets
|
||||||
|
|
||||||
|
- **short**: Abbreviated format (e.g., "1/15/24")
|
||||||
|
- **medium**: Medium-length format (e.g., "Jan 15, 2024")
|
||||||
|
- **long**: Long format with full month name (e.g., "January 15, 2024")
|
||||||
|
- **full**: Full format including day of week (e.g., "Monday, January 15, 2024")
|
||||||
|
|
||||||
#### Additional Variables
|
#### Additional Variables
|
||||||
|
|
||||||
- `{{ tag_name_list }}`: A list of tag names applied to the document, ordered by the tag name. Note this is a list, not a single string
|
- `{{ tag_name_list }}`: A list of tag names applied to the document, ordered by the tag name. Note this is a list, not a single string
|
||||||
|
@@ -1800,23 +1800,3 @@ password. All of these options come from their similarly-named [Django settings]
|
|||||||
#### [`PAPERLESS_EMAIL_USE_SSL=<bool>`](#PAPERLESS_EMAIL_USE_SSL) {#PAPERLESS_EMAIL_USE_SSL}
|
#### [`PAPERLESS_EMAIL_USE_SSL=<bool>`](#PAPERLESS_EMAIL_USE_SSL) {#PAPERLESS_EMAIL_USE_SSL}
|
||||||
|
|
||||||
: Defaults to false.
|
: Defaults to false.
|
||||||
|
|
||||||
## Remote OCR
|
|
||||||
|
|
||||||
#### [`PAPERLESS_REMOTE_OCR_ENGINE=<str>`](#PAPERLESS_REMOTE_OCR_ENGINE) {#PAPERLESS_REMOTE_OCR_ENGINE}
|
|
||||||
|
|
||||||
: The remote OCR engine to use. Currently only Azure AI is supported as "azureai".
|
|
||||||
|
|
||||||
Defaults to None, which disables remote OCR.
|
|
||||||
|
|
||||||
#### [`PAPERLESS_REMOTE_OCR_API_KEY=<str>`](#PAPERLESS_REMOTE_OCR_API_KEY) {#PAPERLESS_REMOTE_OCR_API_KEY}
|
|
||||||
|
|
||||||
: The API key to use for the remote OCR engine.
|
|
||||||
|
|
||||||
Defaults to None.
|
|
||||||
|
|
||||||
#### [`PAPERLESS_REMOTE_OCR_ENDPOINT=<str>`](#PAPERLESS_REMOTE_OCR_ENDPOINT) {#PAPERLESS_REMOTE_OCR_ENDPOINT}
|
|
||||||
|
|
||||||
: The endpoint to use for the remote OCR engine. This is required for Azure AI.
|
|
||||||
|
|
||||||
Defaults to None.
|
|
||||||
|
@@ -25,10 +25,9 @@ physical documents into a searchable online archive so you can keep, well, _less
|
|||||||
## Features
|
## Features
|
||||||
|
|
||||||
- **Organize and index** your scanned documents with tags, correspondents, types, and more.
|
- **Organize and index** your scanned documents with tags, correspondents, types, and more.
|
||||||
- _Your_ data is stored locally on _your_ server and is never transmitted or shared in any way, unless you explicitly choose to do so.
|
- _Your_ data is stored locally on _your_ server and is never transmitted or shared in any way.
|
||||||
- Performs **OCR** on your documents, adding searchable and selectable text, even to documents scanned with only images.
|
- Performs **OCR** on your documents, adding searchable and selectable text, even to documents scanned with only images.
|
||||||
- Utilizes the open-source Tesseract engine to recognize more than 100 languages.
|
- Utilizes the open-source Tesseract engine to recognize more than 100 languages.
|
||||||
- _New!_ Supports remote OCR with Azure AI (opt-in).
|
|
||||||
- Documents are saved as PDF/A format which is designed for long term storage, alongside the unaltered originals.
|
- Documents are saved as PDF/A format which is designed for long term storage, alongside the unaltered originals.
|
||||||
- Uses machine-learning to automatically add tags, correspondents and document types to your documents.
|
- Uses machine-learning to automatically add tags, correspondents and document types to your documents.
|
||||||
- Supports PDF documents, images, plain text files, Office documents (Word, Excel, PowerPoint, and LibreOffice equivalents)[^1] and more.
|
- Supports PDF documents, images, plain text files, Office documents (Word, Excel, PowerPoint, and LibreOffice equivalents)[^1] and more.
|
||||||
|
@@ -850,18 +850,6 @@ how regularly you intend to scan documents and use paperless.
|
|||||||
performed the task associated with the document, move it to the
|
performed the task associated with the document, move it to the
|
||||||
inbox.
|
inbox.
|
||||||
|
|
||||||
## Remote OCR
|
|
||||||
|
|
||||||
!!! important
|
|
||||||
|
|
||||||
This feature is disabled by default and will always remain strictly "opt-in".
|
|
||||||
|
|
||||||
Paperless-ngx supports performing OCR on documents using remote services. At the moment, this is limited to
|
|
||||||
[Microsoft's Azure "Document Intelligence" service](https://azure.microsoft.com/en-us/products/ai-services/ai-document-intelligence).
|
|
||||||
This is of course a paid service (with a free tier) which requires an Azure account and subscription. Azure AI is not affiliated with
|
|
||||||
Paperless-ngx in any way. When enabled, Paperless-ngx will automatically send appropriate documents to Azure for OCR processing, bypassing
|
|
||||||
the local OCR engine. See the [configuration](configuration.md#PAPERLESS_REMOTE_OCR_ENGINE) options for more details.
|
|
||||||
|
|
||||||
## Architecture
|
## Architecture
|
||||||
|
|
||||||
Paperless-ngx consists of the following components:
|
Paperless-ngx consists of the following components:
|
||||||
|
@@ -15,7 +15,7 @@ classifiers = [
|
|||||||
# This will allow testing to not install a webserver, mysql, etc
|
# This will allow testing to not install a webserver, mysql, etc
|
||||||
|
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"azure-ai-documentintelligence>=1.0.2",
|
"babel>=2.17",
|
||||||
"bleach~=6.2.0",
|
"bleach~=6.2.0",
|
||||||
"celery[redis]~=5.5.1",
|
"celery[redis]~=5.5.1",
|
||||||
"channels~=4.2",
|
"channels~=4.2",
|
||||||
@@ -24,22 +24,22 @@ dependencies = [
|
|||||||
"dateparser~=1.2",
|
"dateparser~=1.2",
|
||||||
# WARNING: django does not use semver.
|
# WARNING: django does not use semver.
|
||||||
# Only patch versions are guaranteed to not introduce breaking changes.
|
# Only patch versions are guaranteed to not introduce breaking changes.
|
||||||
"django~=5.1.7",
|
"django~=5.2.5",
|
||||||
"django-allauth[socialaccount,mfa]~=65.4.0",
|
"django-allauth[socialaccount,mfa]~=65.4.0",
|
||||||
"django-auditlog~=3.1.2",
|
"django-auditlog~=3.2.1",
|
||||||
"django-cachalot~=2.8.0",
|
"django-cachalot~=2.8.0",
|
||||||
"django-celery-results~=2.6.0",
|
"django-celery-results~=2.6.0",
|
||||||
"django-compression-middleware~=0.5.0",
|
"django-compression-middleware~=0.5.0",
|
||||||
"django-cors-headers~=4.7.0",
|
"django-cors-headers~=4.7.0",
|
||||||
"django-extensions~=4.1",
|
"django-extensions~=4.1",
|
||||||
"django-filter~=25.1",
|
"django-filter~=25.1",
|
||||||
"django-guardian~=2.4.0",
|
"django-guardian~=3.0.3",
|
||||||
"django-multiselectfield~=0.1.13",
|
"django-multiselectfield~=1.0.1",
|
||||||
"django-soft-delete~=1.0.18",
|
"django-soft-delete~=1.0.18",
|
||||||
"djangorestframework~=3.15",
|
"djangorestframework~=3.16",
|
||||||
"djangorestframework-guardian~=0.3.0",
|
"djangorestframework-guardian~=0.4.0",
|
||||||
"drf-spectacular~=0.28",
|
"drf-spectacular~=0.28",
|
||||||
"drf-spectacular-sidecar~=2025.4.1",
|
"drf-spectacular-sidecar~=2025.8.1",
|
||||||
"drf-writable-nested~=0.7.1",
|
"drf-writable-nested~=0.7.1",
|
||||||
"filelock~=3.18.0",
|
"filelock~=3.18.0",
|
||||||
"flower~=2.0.1",
|
"flower~=2.0.1",
|
||||||
@@ -104,7 +104,7 @@ testing = [
|
|||||||
"imagehash",
|
"imagehash",
|
||||||
"pytest~=8.4.1",
|
"pytest~=8.4.1",
|
||||||
"pytest-cov~=6.2.1",
|
"pytest-cov~=6.2.1",
|
||||||
"pytest-django~=4.10.0",
|
"pytest-django~=4.11.1",
|
||||||
"pytest-env",
|
"pytest-env",
|
||||||
"pytest-httpx",
|
"pytest-httpx",
|
||||||
"pytest-mock",
|
"pytest-mock",
|
||||||
@@ -114,7 +114,7 @@ testing = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
lint = [
|
lint = [
|
||||||
"pre-commit~=4.2.0",
|
"pre-commit~=4.3.0",
|
||||||
"pre-commit-uv~=4.1.3",
|
"pre-commit-uv~=4.1.3",
|
||||||
"ruff~=0.12.2",
|
"ruff~=0.12.2",
|
||||||
]
|
]
|
||||||
@@ -222,6 +222,11 @@ lint.per-file-ignores."src/paperless_tesseract/tests/test_parser.py" = [
|
|||||||
]
|
]
|
||||||
lint.isort.force-single-line = true
|
lint.isort.force-single-line = true
|
||||||
|
|
||||||
|
[tool.codespell]
|
||||||
|
write-changes = true
|
||||||
|
ignore-words-list = "criterias,afterall,valeu,ureue,equest,ure,assertIn,Oktober"
|
||||||
|
skip = "src-ui/src/locale/*,src-ui/pnpm-lock.yaml,src-ui/e2e/*,src/paperless_mail/tests/samples/*,src/documents/tests/samples/*,*.po,*.json"
|
||||||
|
|
||||||
[tool.pytest.ini_options]
|
[tool.pytest.ini_options]
|
||||||
minversion = "8.0"
|
minversion = "8.0"
|
||||||
pythonpath = [
|
pythonpath = [
|
||||||
@@ -234,7 +239,6 @@ testpaths = [
|
|||||||
"src/paperless_tesseract/tests/",
|
"src/paperless_tesseract/tests/",
|
||||||
"src/paperless_tika/tests",
|
"src/paperless_tika/tests",
|
||||||
"src/paperless_text/tests/",
|
"src/paperless_text/tests/",
|
||||||
"src/paperless_remote/tests/",
|
|
||||||
]
|
]
|
||||||
addopts = [
|
addopts = [
|
||||||
"--pythonwarnings=all",
|
"--pythonwarnings=all",
|
||||||
|
@@ -125,14 +125,14 @@ class Command(MultiProcessMixin, ProgressBarMixin, BaseCommand):
|
|||||||
messages.append(
|
messages.append(
|
||||||
self.style.NOTICE(
|
self.style.NOTICE(
|
||||||
f"Document {result.doc_one_pk} fuzzy match"
|
f"Document {result.doc_one_pk} fuzzy match"
|
||||||
f" to {result.doc_two_pk} (confidence {result.ratio:.3f})",
|
f" to {result.doc_two_pk} (confidence {result.ratio:.3f})\n",
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
maybe_delete_ids.append(result.doc_two_pk)
|
maybe_delete_ids.append(result.doc_two_pk)
|
||||||
|
|
||||||
if len(messages) == 0:
|
if len(messages) == 0:
|
||||||
messages.append(
|
messages.append(
|
||||||
self.style.SUCCESS("No matches found"),
|
self.style.SUCCESS("No matches found\n"),
|
||||||
)
|
)
|
||||||
self.stdout.writelines(
|
self.stdout.writelines(
|
||||||
messages,
|
messages,
|
||||||
|
@@ -2089,6 +2089,24 @@ class WorkflowTriggerSerializer(serializers.ModelSerializer):
|
|||||||
|
|
||||||
return attrs
|
return attrs
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def normalize_workflow_trigger_sources(trigger):
|
||||||
|
"""
|
||||||
|
Convert sources to strings to handle django-multiselectfield v1.0 changes
|
||||||
|
"""
|
||||||
|
if trigger and "sources" in trigger:
|
||||||
|
trigger["sources"] = [
|
||||||
|
str(s.value if hasattr(s, "value") else s) for s in trigger["sources"]
|
||||||
|
]
|
||||||
|
|
||||||
|
def create(self, validated_data):
|
||||||
|
WorkflowTriggerSerializer.normalize_workflow_trigger_sources(validated_data)
|
||||||
|
return super().create(validated_data)
|
||||||
|
|
||||||
|
def update(self, instance, validated_data):
|
||||||
|
WorkflowTriggerSerializer.normalize_workflow_trigger_sources(validated_data)
|
||||||
|
return super().update(instance, validated_data)
|
||||||
|
|
||||||
|
|
||||||
class WorkflowActionEmailSerializer(serializers.ModelSerializer):
|
class WorkflowActionEmailSerializer(serializers.ModelSerializer):
|
||||||
id = serializers.IntegerField(allow_null=True, required=False)
|
id = serializers.IntegerField(allow_null=True, required=False)
|
||||||
@@ -2253,6 +2271,8 @@ class WorkflowSerializer(serializers.ModelSerializer):
|
|||||||
if triggers is not None and triggers is not serializers.empty:
|
if triggers is not None and triggers is not serializers.empty:
|
||||||
for trigger in triggers:
|
for trigger in triggers:
|
||||||
filter_has_tags = trigger.pop("filter_has_tags", None)
|
filter_has_tags = trigger.pop("filter_has_tags", None)
|
||||||
|
# Convert sources to strings to handle django-multiselectfield v1.0 changes
|
||||||
|
WorkflowTriggerSerializer.normalize_workflow_trigger_sources(trigger)
|
||||||
trigger_instance, _ = WorkflowTrigger.objects.update_or_create(
|
trigger_instance, _ = WorkflowTrigger.objects.update_or_create(
|
||||||
id=trigger.get("id"),
|
id=trigger.get("id"),
|
||||||
defaults=trigger,
|
defaults=trigger,
|
||||||
|
@@ -2,10 +2,13 @@ import logging
|
|||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
from collections.abc import Iterable
|
from collections.abc import Iterable
|
||||||
|
from datetime import date
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from pathlib import PurePath
|
from pathlib import PurePath
|
||||||
|
|
||||||
import pathvalidate
|
import pathvalidate
|
||||||
|
from babel import Locale
|
||||||
|
from babel import dates
|
||||||
from django.utils import timezone
|
from django.utils import timezone
|
||||||
from django.utils.dateparse import parse_date
|
from django.utils.dateparse import parse_date
|
||||||
from django.utils.text import slugify as django_slugify
|
from django.utils.text import slugify as django_slugify
|
||||||
@@ -90,19 +93,51 @@ def get_cf_value(
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
_template_environment.filters["get_cf_value"] = get_cf_value
|
|
||||||
|
|
||||||
|
|
||||||
def format_datetime(value: str | datetime, format: str) -> str:
|
def format_datetime(value: str | datetime, format: str) -> str:
|
||||||
if isinstance(value, str):
|
if isinstance(value, str):
|
||||||
value = parse_date(value)
|
value = parse_date(value)
|
||||||
return value.strftime(format=format)
|
return value.strftime(format=format)
|
||||||
|
|
||||||
|
|
||||||
|
def localize_date(value: date | datetime, format: str, locale: str) -> str:
|
||||||
|
"""
|
||||||
|
Format a date or datetime object into a localized string using Babel.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
value (date | datetime): The date or datetime to format. If a datetime
|
||||||
|
is provided, it should be timezone-aware (e.g., UTC from a Django DB object).
|
||||||
|
format (str): The format to use. Can be one of Babel's preset formats
|
||||||
|
('short', 'medium', 'long', 'full') or a custom pattern string.
|
||||||
|
locale (str): The locale code (e.g., 'en_US', 'fr_FR') to use for
|
||||||
|
localization.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
str: The localized, formatted date string.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
TypeError: If `value` is not a date or datetime instance.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
Locale.parse(locale)
|
||||||
|
except Exception as e:
|
||||||
|
raise ValueError(f"Invalid locale identifier: {locale}") from e
|
||||||
|
|
||||||
|
if isinstance(value, datetime):
|
||||||
|
return dates.format_datetime(value, format=format, locale=locale)
|
||||||
|
elif isinstance(value, date):
|
||||||
|
return dates.format_date(value, format=format, locale=locale)
|
||||||
|
else:
|
||||||
|
raise TypeError(f"Unsupported type {type(value)} for localize_date")
|
||||||
|
|
||||||
|
|
||||||
|
_template_environment.filters["get_cf_value"] = get_cf_value
|
||||||
|
|
||||||
_template_environment.filters["datetime"] = format_datetime
|
_template_environment.filters["datetime"] = format_datetime
|
||||||
|
|
||||||
_template_environment.filters["slugify"] = django_slugify
|
_template_environment.filters["slugify"] = django_slugify
|
||||||
|
|
||||||
|
_template_environment.filters["localize_date"] = localize_date
|
||||||
|
|
||||||
|
|
||||||
def create_dummy_document():
|
def create_dummy_document():
|
||||||
"""
|
"""
|
||||||
|
@@ -4,6 +4,7 @@ import tempfile
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from unittest import mock
|
from unittest import mock
|
||||||
|
|
||||||
|
import pytest
|
||||||
from auditlog.context import disable_auditlog
|
from auditlog.context import disable_auditlog
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
from django.contrib.auth.models import User
|
from django.contrib.auth.models import User
|
||||||
@@ -22,6 +23,8 @@ from documents.models import Document
|
|||||||
from documents.models import DocumentType
|
from documents.models import DocumentType
|
||||||
from documents.models import StoragePath
|
from documents.models import StoragePath
|
||||||
from documents.tasks import empty_trash
|
from documents.tasks import empty_trash
|
||||||
|
from documents.templating.filepath import localize_date
|
||||||
|
from documents.tests.factories import DocumentFactory
|
||||||
from documents.tests.utils import DirectoriesMixin
|
from documents.tests.utils import DirectoriesMixin
|
||||||
from documents.tests.utils import FileSystemAssertsMixin
|
from documents.tests.utils import FileSystemAssertsMixin
|
||||||
|
|
||||||
@@ -1586,3 +1589,196 @@ class TestFilenameGeneration(DirectoriesMixin, TestCase):
|
|||||||
generate_filename(doc),
|
generate_filename(doc),
|
||||||
Path("brussels-belgium/some-title-with-special-characters.pdf"),
|
Path("brussels-belgium/some-title-with-special-characters.pdf"),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestDateLocalization:
|
||||||
|
"""
|
||||||
|
Groups all tests related to the `localize_date` function.
|
||||||
|
"""
|
||||||
|
|
||||||
|
TEST_DATE = datetime.date(2023, 10, 26)
|
||||||
|
|
||||||
|
TEST_DATETIME = datetime.datetime(
|
||||||
|
2023,
|
||||||
|
10,
|
||||||
|
26,
|
||||||
|
14,
|
||||||
|
30,
|
||||||
|
5,
|
||||||
|
tzinfo=datetime.timezone.utc,
|
||||||
|
)
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"value, format_style, locale_str, expected_output",
|
||||||
|
[
|
||||||
|
pytest.param(
|
||||||
|
TEST_DATE,
|
||||||
|
"EEEE, MMM d, yyyy",
|
||||||
|
"en_US",
|
||||||
|
"Thursday, Oct 26, 2023",
|
||||||
|
id="date-en_US-custom",
|
||||||
|
),
|
||||||
|
pytest.param(
|
||||||
|
TEST_DATE,
|
||||||
|
"dd.MM.yyyy",
|
||||||
|
"de_DE",
|
||||||
|
"26.10.2023",
|
||||||
|
id="date-de_DE-custom",
|
||||||
|
),
|
||||||
|
# German weekday and month name translation
|
||||||
|
pytest.param(
|
||||||
|
TEST_DATE,
|
||||||
|
"EEEE",
|
||||||
|
"de_DE",
|
||||||
|
"Donnerstag",
|
||||||
|
id="weekday-de_DE",
|
||||||
|
),
|
||||||
|
pytest.param(
|
||||||
|
TEST_DATE,
|
||||||
|
"MMMM",
|
||||||
|
"de_DE",
|
||||||
|
"Oktober",
|
||||||
|
id="month-de_DE",
|
||||||
|
),
|
||||||
|
# French weekday and month name translation
|
||||||
|
pytest.param(
|
||||||
|
TEST_DATE,
|
||||||
|
"EEEE",
|
||||||
|
"fr_FR",
|
||||||
|
"jeudi",
|
||||||
|
id="weekday-fr_FR",
|
||||||
|
),
|
||||||
|
pytest.param(
|
||||||
|
TEST_DATE,
|
||||||
|
"MMMM",
|
||||||
|
"fr_FR",
|
||||||
|
"octobre",
|
||||||
|
id="month-fr_FR",
|
||||||
|
),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_localize_date_with_date_objects(
|
||||||
|
self,
|
||||||
|
value: datetime.date,
|
||||||
|
format_style: str,
|
||||||
|
locale_str: str,
|
||||||
|
expected_output: str,
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
Tests `localize_date` with `date` objects across different locales and formats.
|
||||||
|
"""
|
||||||
|
assert localize_date(value, format_style, locale_str) == expected_output
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"value, format_style, locale_str, expected_output",
|
||||||
|
[
|
||||||
|
pytest.param(
|
||||||
|
TEST_DATETIME,
|
||||||
|
"yyyy.MM.dd G 'at' HH:mm:ss zzz",
|
||||||
|
"en_US",
|
||||||
|
"2023.10.26 AD at 14:30:05 UTC",
|
||||||
|
id="datetime-en_US-custom",
|
||||||
|
),
|
||||||
|
pytest.param(
|
||||||
|
TEST_DATETIME,
|
||||||
|
"dd.MM.yyyy",
|
||||||
|
"fr_FR",
|
||||||
|
"26.10.2023",
|
||||||
|
id="date-fr_FR-custom",
|
||||||
|
),
|
||||||
|
# Spanish weekday and month translation
|
||||||
|
pytest.param(
|
||||||
|
TEST_DATETIME,
|
||||||
|
"EEEE",
|
||||||
|
"es_ES",
|
||||||
|
"jueves",
|
||||||
|
id="weekday-es_ES",
|
||||||
|
),
|
||||||
|
pytest.param(
|
||||||
|
TEST_DATETIME,
|
||||||
|
"MMMM",
|
||||||
|
"es_ES",
|
||||||
|
"octubre",
|
||||||
|
id="month-es_ES",
|
||||||
|
),
|
||||||
|
# Italian weekday and month translation
|
||||||
|
pytest.param(
|
||||||
|
TEST_DATETIME,
|
||||||
|
"EEEE",
|
||||||
|
"it_IT",
|
||||||
|
"giovedì",
|
||||||
|
id="weekday-it_IT",
|
||||||
|
),
|
||||||
|
pytest.param(
|
||||||
|
TEST_DATETIME,
|
||||||
|
"MMMM",
|
||||||
|
"it_IT",
|
||||||
|
"ottobre",
|
||||||
|
id="month-it_IT",
|
||||||
|
),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_localize_date_with_datetime_objects(
|
||||||
|
self,
|
||||||
|
value: datetime.datetime,
|
||||||
|
format_style: str,
|
||||||
|
locale_str: str,
|
||||||
|
expected_output: str,
|
||||||
|
):
|
||||||
|
# To handle the non-breaking space in French and other locales
|
||||||
|
result = localize_date(value, format_style, locale_str)
|
||||||
|
assert result.replace("\u202f", " ") == expected_output.replace("\u202f", " ")
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"invalid_value",
|
||||||
|
[
|
||||||
|
"2023-10-26",
|
||||||
|
1698330605,
|
||||||
|
None,
|
||||||
|
[],
|
||||||
|
{},
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_localize_date_raises_type_error_for_invalid_input(self, invalid_value):
|
||||||
|
with pytest.raises(TypeError) as excinfo:
|
||||||
|
localize_date(invalid_value, "medium", "en_US")
|
||||||
|
|
||||||
|
assert f"Unsupported type {type(invalid_value)}" in str(excinfo.value)
|
||||||
|
|
||||||
|
def test_localize_date_raises_error_for_invalid_locale(self):
|
||||||
|
with pytest.raises(ValueError) as excinfo:
|
||||||
|
localize_date(self.TEST_DATE, "medium", "invalid_locale_code")
|
||||||
|
|
||||||
|
assert "Invalid locale identifier" in str(excinfo.value)
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"filename_format,expected_filename",
|
||||||
|
[
|
||||||
|
pytest.param(
|
||||||
|
"{{title}}_{{ document.created | localize_date('MMMM', 'es_ES')}}",
|
||||||
|
"My Document_octubre.pdf",
|
||||||
|
id="spanish_month_name",
|
||||||
|
),
|
||||||
|
pytest.param(
|
||||||
|
"{{title}}_{{ document.created | localize_date('EEEE', 'fr_FR')}}",
|
||||||
|
"My Document_jeudi.pdf",
|
||||||
|
id="french_day_of_week",
|
||||||
|
),
|
||||||
|
pytest.param(
|
||||||
|
"{{title}}_{{ document.created | localize_date('dd/MM/yyyy', 'en_GB')}}",
|
||||||
|
"My Document_26/10/2023.pdf",
|
||||||
|
id="uk_date_format",
|
||||||
|
),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_localize_date_path_building(self, filename_format, expected_filename):
|
||||||
|
document = DocumentFactory.create(
|
||||||
|
title="My Document",
|
||||||
|
mime_type="application/pdf",
|
||||||
|
storage_type=Document.STORAGE_TYPE_UNENCRYPTED,
|
||||||
|
created=self.TEST_DATE, # 2023-10-26 (which is a Thursday)
|
||||||
|
)
|
||||||
|
with override_settings(FILENAME_FORMAT=filename_format):
|
||||||
|
filename = generate_filename(document)
|
||||||
|
assert filename == Path(expected_filename)
|
||||||
|
@@ -123,7 +123,7 @@ class TestExportImport(
|
|||||||
|
|
||||||
self.trigger = WorkflowTrigger.objects.create(
|
self.trigger = WorkflowTrigger.objects.create(
|
||||||
type=WorkflowTrigger.WorkflowTriggerType.CONSUMPTION,
|
type=WorkflowTrigger.WorkflowTriggerType.CONSUMPTION,
|
||||||
sources=[1],
|
sources=[str(WorkflowTrigger.DocumentSourceChoices.CONSUME_FOLDER.value)],
|
||||||
filter_filename="*",
|
filter_filename="*",
|
||||||
)
|
)
|
||||||
self.action = WorkflowAction.objects.create(assign_title="new title")
|
self.action = WorkflowAction.objects.create(assign_title="new title")
|
||||||
|
@@ -87,7 +87,7 @@ class TestFuzzyMatchCommand(TestCase):
|
|||||||
filename="other_test.pdf",
|
filename="other_test.pdf",
|
||||||
)
|
)
|
||||||
stdout, _ = self.call_command()
|
stdout, _ = self.call_command()
|
||||||
self.assertEqual(stdout, "No matches found\n")
|
self.assertIn("No matches found", stdout)
|
||||||
|
|
||||||
def test_with_matches(self):
|
def test_with_matches(self):
|
||||||
"""
|
"""
|
||||||
@@ -116,7 +116,7 @@ class TestFuzzyMatchCommand(TestCase):
|
|||||||
filename="other_test.pdf",
|
filename="other_test.pdf",
|
||||||
)
|
)
|
||||||
stdout, _ = self.call_command("--processes", "1")
|
stdout, _ = self.call_command("--processes", "1")
|
||||||
self.assertRegex(stdout, self.MSG_REGEX + "\n")
|
self.assertRegex(stdout, self.MSG_REGEX)
|
||||||
|
|
||||||
def test_with_3_matches(self):
|
def test_with_3_matches(self):
|
||||||
"""
|
"""
|
||||||
@@ -152,11 +152,10 @@ class TestFuzzyMatchCommand(TestCase):
|
|||||||
filename="final_test.pdf",
|
filename="final_test.pdf",
|
||||||
)
|
)
|
||||||
stdout, _ = self.call_command()
|
stdout, _ = self.call_command()
|
||||||
lines = [x.strip() for x in stdout.split("\n") if len(x.strip())]
|
lines = [x.strip() for x in stdout.splitlines() if x.strip()]
|
||||||
self.assertEqual(len(lines), 3)
|
self.assertEqual(len(lines), 3)
|
||||||
self.assertRegex(lines[0], self.MSG_REGEX)
|
for line in lines:
|
||||||
self.assertRegex(lines[1], self.MSG_REGEX)
|
self.assertRegex(line, self.MSG_REGEX)
|
||||||
self.assertRegex(lines[2], self.MSG_REGEX)
|
|
||||||
|
|
||||||
def test_document_deletion(self):
|
def test_document_deletion(self):
|
||||||
"""
|
"""
|
||||||
@@ -197,14 +196,12 @@ class TestFuzzyMatchCommand(TestCase):
|
|||||||
|
|
||||||
stdout, _ = self.call_command("--delete")
|
stdout, _ = self.call_command("--delete")
|
||||||
|
|
||||||
lines = [x.strip() for x in stdout.split("\n") if len(x.strip())]
|
self.assertIn(
|
||||||
self.assertEqual(len(lines), 3)
|
|
||||||
self.assertEqual(
|
|
||||||
lines[0],
|
|
||||||
"The command is configured to delete documents. Use with caution",
|
"The command is configured to delete documents. Use with caution",
|
||||||
|
stdout,
|
||||||
)
|
)
|
||||||
self.assertRegex(lines[1], self.MSG_REGEX)
|
self.assertRegex(stdout, self.MSG_REGEX)
|
||||||
self.assertEqual(lines[2], "Deleting 1 documents based on ratio matches")
|
self.assertIn("Deleting 1 documents based on ratio matches", stdout)
|
||||||
|
|
||||||
self.assertEqual(Document.objects.count(), 2)
|
self.assertEqual(Document.objects.count(), 2)
|
||||||
self.assertIsNotNone(Document.objects.get(pk=1))
|
self.assertIsNotNone(Document.objects.get(pk=1))
|
||||||
|
@@ -104,7 +104,7 @@ class TestReverseMigrateWorkflow(TestMigrations):
|
|||||||
|
|
||||||
trigger = WorkflowTrigger.objects.create(
|
trigger = WorkflowTrigger.objects.create(
|
||||||
type=0,
|
type=0,
|
||||||
sources=[DocumentSource.ConsumeFolder],
|
sources=[str(DocumentSource.ConsumeFolder)],
|
||||||
filter_path="*/path/*",
|
filter_path="*/path/*",
|
||||||
filter_filename="*file*",
|
filter_filename="*file*",
|
||||||
)
|
)
|
||||||
|
@@ -54,7 +54,7 @@ class HttpRemoteUserMiddleware(PersistentRemoteUserMiddleware):
|
|||||||
|
|
||||||
header = settings.HTTP_REMOTE_USER_HEADER_NAME
|
header = settings.HTTP_REMOTE_USER_HEADER_NAME
|
||||||
|
|
||||||
def process_request(self, request: HttpRequest) -> None:
|
def __call__(self, request: HttpRequest) -> None:
|
||||||
# If remote user auth is enabled only for the frontend, not the API,
|
# If remote user auth is enabled only for the frontend, not the API,
|
||||||
# then we need dont want to authenticate the user for API requests.
|
# then we need dont want to authenticate the user for API requests.
|
||||||
if (
|
if (
|
||||||
@@ -62,8 +62,8 @@ class HttpRemoteUserMiddleware(PersistentRemoteUserMiddleware):
|
|||||||
and "paperless.auth.PaperlessRemoteUserAuthentication"
|
and "paperless.auth.PaperlessRemoteUserAuthentication"
|
||||||
not in settings.REST_FRAMEWORK["DEFAULT_AUTHENTICATION_CLASSES"]
|
not in settings.REST_FRAMEWORK["DEFAULT_AUTHENTICATION_CLASSES"]
|
||||||
):
|
):
|
||||||
return
|
return self.get_response(request)
|
||||||
return super().process_request(request)
|
return super().__call__(request)
|
||||||
|
|
||||||
|
|
||||||
class PaperlessRemoteUserAuthentication(authentication.RemoteUserAuthentication):
|
class PaperlessRemoteUserAuthentication(authentication.RemoteUserAuthentication):
|
||||||
|
@@ -214,31 +214,3 @@ def audit_log_check(app_configs, **kwargs):
|
|||||||
)
|
)
|
||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
@register()
|
|
||||||
def check_postgres_version(app_configs, **kwargs):
|
|
||||||
"""
|
|
||||||
Django 5.2 removed PostgreSQL 13 support and thus it will be removed in
|
|
||||||
a future Paperless-ngx version. This check can be removed eventually.
|
|
||||||
See https://docs.djangoproject.com/en/5.2/releases/5.2/#dropped-support-for-postgresql-13
|
|
||||||
"""
|
|
||||||
db_conn = connections["default"]
|
|
||||||
result = []
|
|
||||||
if db_conn.vendor == "postgresql":
|
|
||||||
try:
|
|
||||||
with db_conn.cursor() as cursor:
|
|
||||||
cursor.execute("SHOW server_version;")
|
|
||||||
version = cursor.fetchone()[0]
|
|
||||||
if version.startswith("13"):
|
|
||||||
return [
|
|
||||||
Warning(
|
|
||||||
"PostgreSQL 13 is deprecated and will not be supported in a future Paperless-ngx release.",
|
|
||||||
hint="Upgrade to PostgreSQL 14 or newer.",
|
|
||||||
),
|
|
||||||
]
|
|
||||||
except Exception: # pragma: no cover
|
|
||||||
# Don't block checks on version query failure
|
|
||||||
pass
|
|
||||||
|
|
||||||
return result
|
|
||||||
|
@@ -324,7 +324,6 @@ INSTALLED_APPS = [
|
|||||||
"paperless_tesseract.apps.PaperlessTesseractConfig",
|
"paperless_tesseract.apps.PaperlessTesseractConfig",
|
||||||
"paperless_text.apps.PaperlessTextConfig",
|
"paperless_text.apps.PaperlessTextConfig",
|
||||||
"paperless_mail.apps.PaperlessMailConfig",
|
"paperless_mail.apps.PaperlessMailConfig",
|
||||||
"paperless_remote.apps.PaperlessRemoteParserConfig",
|
|
||||||
"django.contrib.admin",
|
"django.contrib.admin",
|
||||||
"rest_framework",
|
"rest_framework",
|
||||||
"rest_framework.authtoken",
|
"rest_framework.authtoken",
|
||||||
@@ -1444,10 +1443,3 @@ WEBHOOKS_ALLOW_INTERNAL_REQUESTS = __get_boolean(
|
|||||||
"PAPERLESS_WEBHOOKS_ALLOW_INTERNAL_REQUESTS",
|
"PAPERLESS_WEBHOOKS_ALLOW_INTERNAL_REQUESTS",
|
||||||
"true",
|
"true",
|
||||||
)
|
)
|
||||||
|
|
||||||
###############################################################################
|
|
||||||
# Remote Parser #
|
|
||||||
###############################################################################
|
|
||||||
REMOTE_OCR_ENGINE = os.getenv("PAPERLESS_REMOTE_OCR_ENGINE")
|
|
||||||
REMOTE_OCR_API_KEY = os.getenv("PAPERLESS_REMOTE_OCR_API_KEY")
|
|
||||||
REMOTE_OCR_ENDPOINT = os.getenv("PAPERLESS_REMOTE_OCR_ENDPOINT")
|
|
||||||
|
@@ -9,7 +9,6 @@ from documents.tests.utils import DirectoriesMixin
|
|||||||
from documents.tests.utils import FileSystemAssertsMixin
|
from documents.tests.utils import FileSystemAssertsMixin
|
||||||
from paperless.checks import audit_log_check
|
from paperless.checks import audit_log_check
|
||||||
from paperless.checks import binaries_check
|
from paperless.checks import binaries_check
|
||||||
from paperless.checks import check_postgres_version
|
|
||||||
from paperless.checks import debug_mode_check
|
from paperless.checks import debug_mode_check
|
||||||
from paperless.checks import paths_check
|
from paperless.checks import paths_check
|
||||||
from paperless.checks import settings_values_check
|
from paperless.checks import settings_values_check
|
||||||
@@ -263,39 +262,3 @@ class TestAuditLogChecks(TestCase):
|
|||||||
("auditlog table was found but audit log is disabled."),
|
("auditlog table was found but audit log is disabled."),
|
||||||
msg.msg,
|
msg.msg,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class TestPostgresVersionCheck(TestCase):
|
|
||||||
@mock.patch("paperless.checks.connections")
|
|
||||||
def test_postgres_13_warns(self, mock_connections):
|
|
||||||
mock_connection = mock.MagicMock()
|
|
||||||
mock_connection.vendor = "postgresql"
|
|
||||||
mock_cursor = mock.MagicMock()
|
|
||||||
mock_cursor.__enter__.return_value.fetchone.return_value = ["13.11"]
|
|
||||||
mock_connection.cursor.return_value = mock_cursor
|
|
||||||
mock_connections.__getitem__.return_value = mock_connection
|
|
||||||
|
|
||||||
warnings = check_postgres_version(None)
|
|
||||||
self.assertEqual(len(warnings), 1)
|
|
||||||
self.assertIn("PostgreSQL 13 is deprecated", warnings[0].msg)
|
|
||||||
|
|
||||||
@mock.patch("paperless.checks.connections")
|
|
||||||
def test_postgres_14_passes(self, mock_connections):
|
|
||||||
mock_connection = mock.MagicMock()
|
|
||||||
mock_connection.vendor = "postgresql"
|
|
||||||
mock_cursor = mock.MagicMock()
|
|
||||||
mock_cursor.__enter__.return_value.fetchone.return_value = ["14.10"]
|
|
||||||
mock_connection.cursor.return_value = mock_cursor
|
|
||||||
mock_connections.__getitem__.return_value = mock_connection
|
|
||||||
|
|
||||||
warnings = check_postgres_version(None)
|
|
||||||
self.assertEqual(warnings, [])
|
|
||||||
|
|
||||||
@mock.patch("paperless.checks.connections")
|
|
||||||
def test_non_postgres_skipped(self, mock_connections):
|
|
||||||
mock_connection = mock.MagicMock()
|
|
||||||
mock_connection.vendor = "sqlite"
|
|
||||||
mock_connections.__getitem__.return_value = mock_connection
|
|
||||||
|
|
||||||
warnings = check_postgres_version(None)
|
|
||||||
self.assertEqual(warnings, [])
|
|
||||||
|
@@ -1,6 +1,7 @@
|
|||||||
import os
|
import os
|
||||||
from unittest import mock
|
from unittest import mock
|
||||||
|
|
||||||
|
from django.conf import settings
|
||||||
from django.contrib.auth.models import User
|
from django.contrib.auth.models import User
|
||||||
from django.test import override_settings
|
from django.test import override_settings
|
||||||
from rest_framework import status
|
from rest_framework import status
|
||||||
@@ -91,6 +92,7 @@ class TestRemoteUser(DirectoriesMixin, APITestCase):
|
|||||||
|
|
||||||
@override_settings(
|
@override_settings(
|
||||||
REST_FRAMEWORK={
|
REST_FRAMEWORK={
|
||||||
|
**settings.REST_FRAMEWORK,
|
||||||
"DEFAULT_AUTHENTICATION_CLASSES": [
|
"DEFAULT_AUTHENTICATION_CLASSES": [
|
||||||
"rest_framework.authentication.BasicAuthentication",
|
"rest_framework.authentication.BasicAuthentication",
|
||||||
"rest_framework.authentication.TokenAuthentication",
|
"rest_framework.authentication.TokenAuthentication",
|
||||||
|
@@ -1,4 +0,0 @@
|
|||||||
# this is here so that django finds the checks.
|
|
||||||
from paperless_remote.checks import check_remote_parser_configured
|
|
||||||
|
|
||||||
__all__ = ["check_remote_parser_configured"]
|
|
@@ -1,14 +0,0 @@
|
|||||||
from django.apps import AppConfig
|
|
||||||
|
|
||||||
from paperless_remote.signals import remote_consumer_declaration
|
|
||||||
|
|
||||||
|
|
||||||
class PaperlessRemoteParserConfig(AppConfig):
|
|
||||||
name = "paperless_remote"
|
|
||||||
|
|
||||||
def ready(self):
|
|
||||||
from documents.signals import document_consumer_declaration
|
|
||||||
|
|
||||||
document_consumer_declaration.connect(remote_consumer_declaration)
|
|
||||||
|
|
||||||
AppConfig.ready(self)
|
|
@@ -1,15 +0,0 @@
|
|||||||
from django.conf import settings
|
|
||||||
from django.core.checks import Error
|
|
||||||
from django.core.checks import register
|
|
||||||
|
|
||||||
|
|
||||||
@register()
|
|
||||||
def check_remote_parser_configured(app_configs, **kwargs):
|
|
||||||
if settings.REMOTE_OCR_ENGINE == "azureai" and not settings.REMOTE_OCR_ENDPOINT:
|
|
||||||
return [
|
|
||||||
Error(
|
|
||||||
"Azure AI remote parser requires endpoint to be configured.",
|
|
||||||
),
|
|
||||||
]
|
|
||||||
|
|
||||||
return []
|
|
@@ -1,113 +0,0 @@
|
|||||||
from pathlib import Path
|
|
||||||
|
|
||||||
from django.conf import settings
|
|
||||||
|
|
||||||
from paperless_tesseract.parsers import RasterisedDocumentParser
|
|
||||||
|
|
||||||
|
|
||||||
class RemoteEngineConfig:
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
engine: str,
|
|
||||||
api_key: str | None = None,
|
|
||||||
endpoint: str | None = None,
|
|
||||||
):
|
|
||||||
self.engine = engine
|
|
||||||
self.api_key = api_key
|
|
||||||
self.endpoint = endpoint
|
|
||||||
|
|
||||||
def engine_is_valid(self):
|
|
||||||
valid = self.engine in ["azureai"] and self.api_key is not None
|
|
||||||
if self.engine == "azureai":
|
|
||||||
valid = valid and self.endpoint is not None
|
|
||||||
return valid
|
|
||||||
|
|
||||||
|
|
||||||
class RemoteDocumentParser(RasterisedDocumentParser):
|
|
||||||
"""
|
|
||||||
This parser uses a remote OCR engine to parse documents. Currently, it supports Azure AI Vision
|
|
||||||
as this is the only service that provides a remote OCR API with text-embedded PDF output.
|
|
||||||
"""
|
|
||||||
|
|
||||||
logging_name = "paperless.parsing.remote"
|
|
||||||
|
|
||||||
def get_settings(self) -> RemoteEngineConfig:
|
|
||||||
"""
|
|
||||||
Returns the configuration for the remote OCR engine, loaded from Django settings.
|
|
||||||
"""
|
|
||||||
return RemoteEngineConfig(
|
|
||||||
engine=settings.REMOTE_OCR_ENGINE,
|
|
||||||
api_key=settings.REMOTE_OCR_API_KEY,
|
|
||||||
endpoint=settings.REMOTE_OCR_ENDPOINT,
|
|
||||||
)
|
|
||||||
|
|
||||||
def supported_mime_types(self):
|
|
||||||
if self.settings.engine_is_valid():
|
|
||||||
return {
|
|
||||||
"application/pdf": ".pdf",
|
|
||||||
"image/png": ".png",
|
|
||||||
"image/jpeg": ".jpg",
|
|
||||||
"image/tiff": ".tiff",
|
|
||||||
"image/bmp": ".bmp",
|
|
||||||
"image/gif": ".gif",
|
|
||||||
"image/webp": ".webp",
|
|
||||||
}
|
|
||||||
else:
|
|
||||||
return {}
|
|
||||||
|
|
||||||
def azure_ai_vision_parse(
|
|
||||||
self,
|
|
||||||
file: Path,
|
|
||||||
) -> str | None:
|
|
||||||
"""
|
|
||||||
Uses Azure AI Vision to parse the document and return the text content.
|
|
||||||
It requests a searchable PDF output with embedded text.
|
|
||||||
The PDF is saved to the archive_path attribute.
|
|
||||||
Returns the text content extracted from the document.
|
|
||||||
If the parsing fails, it returns None.
|
|
||||||
"""
|
|
||||||
from azure.ai.documentintelligence import DocumentIntelligenceClient
|
|
||||||
from azure.ai.documentintelligence.models import AnalyzeDocumentRequest
|
|
||||||
from azure.ai.documentintelligence.models import AnalyzeOutputOption
|
|
||||||
from azure.ai.documentintelligence.models import DocumentContentFormat
|
|
||||||
from azure.core.credentials import AzureKeyCredential
|
|
||||||
|
|
||||||
client = DocumentIntelligenceClient(
|
|
||||||
endpoint=self.settings.endpoint,
|
|
||||||
credential=AzureKeyCredential(self.settings.api_key),
|
|
||||||
)
|
|
||||||
|
|
||||||
with file.open("rb") as f:
|
|
||||||
analyze_request = AnalyzeDocumentRequest(bytes_source=f.read())
|
|
||||||
poller = client.begin_analyze_document(
|
|
||||||
model_id="prebuilt-read",
|
|
||||||
body=analyze_request,
|
|
||||||
output_content_format=DocumentContentFormat.TEXT,
|
|
||||||
output=[AnalyzeOutputOption.PDF], # request searchable PDF output
|
|
||||||
content_type="application/json",
|
|
||||||
)
|
|
||||||
|
|
||||||
poller.wait()
|
|
||||||
result_id = poller.details["operation_id"]
|
|
||||||
result = poller.result()
|
|
||||||
|
|
||||||
# Download the PDF with embedded text
|
|
||||||
self.archive_path = Path(self.tempdir) / "archive.pdf"
|
|
||||||
with self.archive_path.open("wb") as f:
|
|
||||||
for chunk in client.get_analyze_result_pdf(
|
|
||||||
model_id="prebuilt-read",
|
|
||||||
result_id=result_id,
|
|
||||||
):
|
|
||||||
f.write(chunk)
|
|
||||||
|
|
||||||
return result.content
|
|
||||||
|
|
||||||
def parse(self, document_path: Path, mime_type, file_name=None):
|
|
||||||
if not self.settings.engine_is_valid():
|
|
||||||
self.log.warning(
|
|
||||||
"No valid remote parser engine is configured, content will be empty.",
|
|
||||||
)
|
|
||||||
self.text = ""
|
|
||||||
return
|
|
||||||
elif self.settings.engine == "azureai":
|
|
||||||
self.text = self.azure_ai_vision_parse(document_path)
|
|
@@ -1,18 +0,0 @@
|
|||||||
def get_parser(*args, **kwargs):
|
|
||||||
from paperless_remote.parsers import RemoteDocumentParser
|
|
||||||
|
|
||||||
return RemoteDocumentParser(*args, **kwargs)
|
|
||||||
|
|
||||||
|
|
||||||
def get_supported_mime_types():
|
|
||||||
from paperless_remote.parsers import RemoteDocumentParser
|
|
||||||
|
|
||||||
return RemoteDocumentParser(None).supported_mime_types()
|
|
||||||
|
|
||||||
|
|
||||||
def remote_consumer_declaration(sender, **kwargs):
|
|
||||||
return {
|
|
||||||
"parser": get_parser,
|
|
||||||
"weight": 5,
|
|
||||||
"mime_types": get_supported_mime_types(),
|
|
||||||
}
|
|
Binary file not shown.
@@ -1,29 +0,0 @@
|
|||||||
from django.test import TestCase
|
|
||||||
from django.test import override_settings
|
|
||||||
|
|
||||||
from paperless_remote import check_remote_parser_configured
|
|
||||||
|
|
||||||
|
|
||||||
class TestChecks(TestCase):
|
|
||||||
@override_settings(REMOTE_OCR_ENGINE=None)
|
|
||||||
def test_no_engine(self):
|
|
||||||
msgs = check_remote_parser_configured(None)
|
|
||||||
self.assertEqual(len(msgs), 0)
|
|
||||||
|
|
||||||
@override_settings(REMOTE_OCR_ENGINE="azureai")
|
|
||||||
@override_settings(REMOTE_OCR_API_KEY="somekey")
|
|
||||||
@override_settings(REMOTE_OCR_ENDPOINT=None)
|
|
||||||
def test_azure_no_endpoint(self):
|
|
||||||
msgs = check_remote_parser_configured(None)
|
|
||||||
self.assertEqual(len(msgs), 1)
|
|
||||||
self.assertTrue(
|
|
||||||
msgs[0].msg.startswith(
|
|
||||||
"Azure AI remote parser requires endpoint to be configured.",
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
@override_settings(REMOTE_OCR_ENGINE="something")
|
|
||||||
@override_settings(REMOTE_OCR_API_KEY="somekey")
|
|
||||||
def test_valid_configuration(self):
|
|
||||||
msgs = check_remote_parser_configured(None)
|
|
||||||
self.assertEqual(len(msgs), 0)
|
|
@@ -1,101 +0,0 @@
|
|||||||
import uuid
|
|
||||||
from pathlib import Path
|
|
||||||
from unittest import mock
|
|
||||||
|
|
||||||
from django.test import TestCase
|
|
||||||
from django.test import override_settings
|
|
||||||
|
|
||||||
from documents.tests.utils import DirectoriesMixin
|
|
||||||
from documents.tests.utils import FileSystemAssertsMixin
|
|
||||||
from paperless_remote.parsers import RemoteDocumentParser
|
|
||||||
from paperless_remote.signals import get_parser
|
|
||||||
|
|
||||||
|
|
||||||
class TestParser(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
|
|
||||||
SAMPLE_FILES = Path(__file__).resolve().parent / "samples"
|
|
||||||
|
|
||||||
def assertContainsStrings(self, content, strings):
|
|
||||||
# Asserts that all strings appear in content, in the given order.
|
|
||||||
indices = []
|
|
||||||
for s in strings:
|
|
||||||
if s in content:
|
|
||||||
indices.append(content.index(s))
|
|
||||||
else:
|
|
||||||
self.fail(f"'{s}' is not in '{content}'")
|
|
||||||
self.assertListEqual(indices, sorted(indices))
|
|
||||||
|
|
||||||
@mock.patch("paperless_tesseract.parsers.run_subprocess")
|
|
||||||
@mock.patch("azure.ai.documentintelligence.DocumentIntelligenceClient")
|
|
||||||
def test_get_text_with_azure(self, mock_client_cls, mock_subprocess):
|
|
||||||
# Arrange mock Azure client
|
|
||||||
mock_client = mock.Mock()
|
|
||||||
mock_client_cls.return_value = mock_client
|
|
||||||
|
|
||||||
# Simulate poller result and its `.details`
|
|
||||||
mock_poller = mock.Mock()
|
|
||||||
mock_poller.wait.return_value = None
|
|
||||||
mock_poller.details = {"operation_id": "fake-op-id"}
|
|
||||||
mock_client.begin_analyze_document.return_value = mock_poller
|
|
||||||
mock_poller.result.return_value.content = "This is a test document."
|
|
||||||
|
|
||||||
# Return dummy PDF bytes
|
|
||||||
mock_client.get_analyze_result_pdf.return_value = [
|
|
||||||
b"%PDF-",
|
|
||||||
b"1.7 ",
|
|
||||||
b"FAKEPDF",
|
|
||||||
]
|
|
||||||
|
|
||||||
# Simulate pdftotext by writing dummy text to sidecar file
|
|
||||||
def fake_run(cmd, *args, **kwargs):
|
|
||||||
with Path(cmd[-1]).open("w", encoding="utf-8") as f:
|
|
||||||
f.write("This is a test document.")
|
|
||||||
|
|
||||||
mock_subprocess.side_effect = fake_run
|
|
||||||
|
|
||||||
with override_settings(
|
|
||||||
REMOTE_OCR_ENGINE="azureai",
|
|
||||||
REMOTE_OCR_API_KEY="somekey",
|
|
||||||
REMOTE_OCR_ENDPOINT="https://endpoint.cognitiveservices.azure.com",
|
|
||||||
):
|
|
||||||
parser = get_parser(uuid.uuid4())
|
|
||||||
parser.parse(
|
|
||||||
self.SAMPLE_FILES / "simple-digital.pdf",
|
|
||||||
"application/pdf",
|
|
||||||
)
|
|
||||||
|
|
||||||
self.assertContainsStrings(
|
|
||||||
parser.text.strip(),
|
|
||||||
["This is a test document."],
|
|
||||||
)
|
|
||||||
|
|
||||||
@override_settings(
|
|
||||||
REMOTE_OCR_ENGINE="azureai",
|
|
||||||
REMOTE_OCR_API_KEY="key",
|
|
||||||
REMOTE_OCR_ENDPOINT="https://endpoint.cognitiveservices.azure.com",
|
|
||||||
)
|
|
||||||
def test_supported_mime_types_valid_config(self):
|
|
||||||
parser = RemoteDocumentParser(uuid.uuid4())
|
|
||||||
expected_types = {
|
|
||||||
"application/pdf": ".pdf",
|
|
||||||
"image/png": ".png",
|
|
||||||
"image/jpeg": ".jpg",
|
|
||||||
"image/tiff": ".tiff",
|
|
||||||
"image/bmp": ".bmp",
|
|
||||||
"image/gif": ".gif",
|
|
||||||
"image/webp": ".webp",
|
|
||||||
}
|
|
||||||
self.assertEqual(parser.supported_mime_types(), expected_types)
|
|
||||||
|
|
||||||
def test_supported_mime_types_invalid_config(self):
|
|
||||||
parser = get_parser(uuid.uuid4())
|
|
||||||
self.assertEqual(parser.supported_mime_types(), {})
|
|
||||||
|
|
||||||
@override_settings(
|
|
||||||
REMOTE_OCR_ENGINE=None,
|
|
||||||
REMOTE_OCR_API_KEY=None,
|
|
||||||
REMOTE_OCR_ENDPOINT=None,
|
|
||||||
)
|
|
||||||
def test_parse_with_invalid_config(self):
|
|
||||||
parser = get_parser(uuid.uuid4())
|
|
||||||
parser.parse(self.SAMPLE_FILES / "simple-digital.pdf", "application/pdf")
|
|
||||||
self.assertEqual(parser.text, "")
|
|
Reference in New Issue
Block a user