paradizelost/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2025-09-16 21:55:37 -05:00
Code Issues Packages Projects Releases Wiki Activity

Enhancement: jinja template support for workflow title assignment (#10700)

Browse Source 
---------

Co-authored-by: Trenton Holmes <797416+stumpylog@users.noreply.github.com>
Co-authored-by: shamoon <4887959+shamoon@users.noreply.github.com>
This commit is contained in:
sidey79
2025-09-11 15:56:16 +02:00
committed by GitHub
parent 84942a4e69
commit 9e11e7fd05
12 changed files with 825 additions and 300 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
src/documents/tests/test_consumer.py
View File

@@ -304,22 +304,6 @@ class TestConsumer(
self.assertEqual(document.title, "Override Title")
self._assert_first_last_send_progress()
def testOverrideTitleInvalidPlaceholders(self):
with self.assertLogs("paperless.consumer", level="ERROR") as cm:
with self.get_consumer(
self.get_test_file(),
DocumentMetadataOverrides(title="Override {correspondent]"),
) as consumer:
consumer.run()
document = Document.objects.first()
self.assertIsNotNone(document)
self.assertEqual(document.title, "sample")
expected_str = "Error occurred parsing title override 'Override {correspondent]', falling back to original"
self.assertIn(expected_str, cm.output[0])
def testOverrideCorrespondent(self):
c = Correspondent.objects.create(name="test")
@@ -437,7 +421,7 @@ class TestConsumer(
DocumentMetadataOverrides(
correspondent_id=c.pk,
document_type_id=dt.pk,
title="{correspondent}{document_type} {added_month}-{added_year_short}",
title="{{correspondent}}{{document_type}} {{added_month}}-{{added_year_short}}",
),
) as consumer:
consumer.run()

156
src/documents/tests/test_file_handling.py
View File

@@ -23,7 +23,6 @@ from documents.models import Document
from documents.models import DocumentType
from documents.models import StoragePath
from documents.tasks import empty_trash
from documents.templating.filepath import localize_date
from documents.tests.factories import DocumentFactory
from documents.tests.utils import DirectoriesMixin
from documents.tests.utils import FileSystemAssertsMixin
@@ -1591,166 +1590,13 @@ class TestFilenameGeneration(DirectoriesMixin, TestCase):
)
class TestDateLocalization:
class TestPathDateLocalization:
"""
Groups all tests related to the `localize_date` function.
"""
TEST_DATE = datetime.date(2023, 10, 26)
TEST_DATETIME = datetime.datetime(
2023,
10,
26,
14,
30,
5,
tzinfo=datetime.timezone.utc,
)
@pytest.mark.parametrize(
"value, format_style, locale_str, expected_output",
[
pytest.param(
TEST_DATE,
"EEEE, MMM d, yyyy",
"en_US",
"Thursday, Oct 26, 2023",
id="date-en_US-custom",
),
pytest.param(
TEST_DATE,
"dd.MM.yyyy",
"de_DE",
"26.10.2023",
id="date-de_DE-custom",
),
# German weekday and month name translation
pytest.param(
TEST_DATE,
"EEEE",
"de_DE",
"Donnerstag",
id="weekday-de_DE",
),
pytest.param(
TEST_DATE,
"MMMM",
"de_DE",
"Oktober",
id="month-de_DE",
),
# French weekday and month name translation
pytest.param(
TEST_DATE,
"EEEE",
"fr_FR",
"jeudi",
id="weekday-fr_FR",
),
pytest.param(
TEST_DATE,
"MMMM",
"fr_FR",
"octobre",
id="month-fr_FR",
),
],
)
def test_localize_date_with_date_objects(
self,
value: datetime.date,
format_style: str,
locale_str: str,
expected_output: str,
):
"""
Tests `localize_date` with `date` objects across different locales and formats.
"""
assert localize_date(value, format_style, locale_str) == expected_output
@pytest.mark.parametrize(
"value, format_style, locale_str, expected_output",
[
pytest.param(
TEST_DATETIME,
"yyyy.MM.dd G 'at' HH:mm:ss zzz",
"en_US",
"2023.10.26 AD at 14:30:05 UTC",
id="datetime-en_US-custom",
),
pytest.param(
TEST_DATETIME,
"dd.MM.yyyy",
"fr_FR",
"26.10.2023",
id="date-fr_FR-custom",
),
# Spanish weekday and month translation
pytest.param(
TEST_DATETIME,
"EEEE",
"es_ES",
"jueves",
id="weekday-es_ES",
),
pytest.param(
TEST_DATETIME,
"MMMM",
"es_ES",
"octubre",
id="month-es_ES",
),
# Italian weekday and month translation
pytest.param(
TEST_DATETIME,
"EEEE",
"it_IT",
"giovedì",
id="weekday-it_IT",
),
pytest.param(
TEST_DATETIME,
"MMMM",
"it_IT",
"ottobre",
id="month-it_IT",
),
],
)
def test_localize_date_with_datetime_objects(
self,
value: datetime.datetime,
format_style: str,
locale_str: str,
expected_output: str,
):
# To handle the non-breaking space in French and other locales
result = localize_date(value, format_style, locale_str)
assert result.replace("\u202f", " ") == expected_output.replace("\u202f", " ")
@pytest.mark.parametrize(
"invalid_value",
[
"2023-10-26",
1698330605,
None,
[],
{},
],
)
def test_localize_date_raises_type_error_for_invalid_input(self, invalid_value):
with pytest.raises(TypeError) as excinfo:
localize_date(invalid_value, "medium", "en_US")
assert f"Unsupported type {type(invalid_value)}" in str(excinfo.value)
def test_localize_date_raises_error_for_invalid_locale(self):
with pytest.raises(ValueError) as excinfo:
localize_date(self.TEST_DATE, "medium", "invalid_locale_code")
assert "Invalid locale identifier" in str(excinfo.value)
@pytest.mark.django_db
@pytest.mark.parametrize(
"filename_format,expected_filename",

296
src/documents/tests/test_filters.py Normal file
View File

@@ -0,0 +1,296 @@
import datetime
from typing import Any
from typing import Literal
import pytest
from documents.templating.filters import localize_date
class TestDateLocalization:
"""
Groups all tests related to the `localize_date` function.
"""
TEST_DATE = datetime.date(2023, 10, 26)
TEST_DATETIME = datetime.datetime(
2023,
10,
26,
14,
30,
5,
tzinfo=datetime.timezone.utc,
)
TEST_DATETIME_STRING: str = "2023-10-26T14:30:05+00:00"
TEST_DATE_STRING: str = "2023-10-26"
@pytest.mark.parametrize(
"value, format_style, locale_str, expected_output",
[
pytest.param(
TEST_DATE,
"EEEE, MMM d, yyyy",
"en_US",
"Thursday, Oct 26, 2023",
id="date-en_US-custom",
),
pytest.param(
TEST_DATE,
"dd.MM.yyyy",
"de_DE",
"26.10.2023",
id="date-de_DE-custom",
),
# German weekday and month name translation
pytest.param(
TEST_DATE,
"EEEE",
"de_DE",
"Donnerstag",
id="weekday-de_DE",
),
pytest.param(
TEST_DATE,
"MMMM",
"de_DE",
"Oktober",
id="month-de_DE",
),
# French weekday and month name translation
pytest.param(
TEST_DATE,
"EEEE",
"fr_FR",
"jeudi",
id="weekday-fr_FR",
),
pytest.param(
TEST_DATE,
"MMMM",
"fr_FR",
"octobre",
id="month-fr_FR",
),
],
)
def test_localize_date_with_date_objects(
self,
value: datetime.date,
format_style: str,
locale_str: str,
expected_output: str,
):
"""
Tests `localize_date` with `date` objects across different locales and formats.
"""
assert localize_date(value, format_style, locale_str) == expected_output
@pytest.mark.parametrize(
"value, format_style, locale_str, expected_output",
[
pytest.param(
TEST_DATETIME,
"yyyy.MM.dd G 'at' HH:mm:ss zzz",
"en_US",
"2023.10.26 AD at 14:30:05 UTC",
id="datetime-en_US-custom",
),
pytest.param(
TEST_DATETIME,
"dd.MM.yyyy",
"fr_FR",
"26.10.2023",
id="date-fr_FR-custom",
),
# Spanish weekday and month translation
pytest.param(
TEST_DATETIME,
"EEEE",
"es_ES",
"jueves",
id="weekday-es_ES",
),
pytest.param(
TEST_DATETIME,
"MMMM",
"es_ES",
"octubre",
id="month-es_ES",
),
# Italian weekday and month translation
pytest.param(
TEST_DATETIME,
"EEEE",
"it_IT",
"giovedì",
id="weekday-it_IT",
),
pytest.param(
TEST_DATETIME,
"MMMM",
"it_IT",
"ottobre",
id="month-it_IT",
),
],
)
def test_localize_date_with_datetime_objects(
self,
value: datetime.datetime,
format_style: str,
locale_str: str,
expected_output: str,
):
# To handle the non-breaking space in French and other locales
result = localize_date(value, format_style, locale_str)
assert result.replace("\u202f", " ") == expected_output.replace("\u202f", " ")
@pytest.mark.parametrize(
"invalid_value",
[
1698330605,
None,
[],
{},
],
)
def test_localize_date_raises_type_error_for_invalid_input(
self,
invalid_value: None | list[object] | dict[Any, Any] | Literal[1698330605],
):
with pytest.raises(TypeError) as excinfo:
localize_date(invalid_value, "medium", "en_US")
assert f"Unsupported type {type(invalid_value)}" in str(excinfo.value)
def test_localize_date_raises_error_for_invalid_locale(self):
with pytest.raises(ValueError) as excinfo:
localize_date(self.TEST_DATE, "medium", "invalid_locale_code")
assert "Invalid locale identifier" in str(excinfo.value)
@pytest.mark.parametrize(
"value, format_style, locale_str, expected_output",
[
pytest.param(
TEST_DATETIME_STRING,
"EEEE, MMM d, yyyy",
"en_US",
"Thursday, Oct 26, 2023",
id="date-en_US-custom",
),
pytest.param(
TEST_DATETIME_STRING,
"dd.MM.yyyy",
"de_DE",
"26.10.2023",
id="date-de_DE-custom",
),
# German weekday and month name translation
pytest.param(
TEST_DATETIME_STRING,
"EEEE",
"de_DE",
"Donnerstag",
id="weekday-de_DE",
),
pytest.param(
TEST_DATETIME_STRING,
"MMMM",
"de_DE",
"Oktober",
id="month-de_DE",
),
# French weekday and month name translation
pytest.param(
TEST_DATETIME_STRING,
"EEEE",
"fr_FR",
"jeudi",
id="weekday-fr_FR",
),
pytest.param(
TEST_DATETIME_STRING,
"MMMM",
"fr_FR",
"octobre",
id="month-fr_FR",
),
],
)
def test_localize_date_with_datetime_string(
self,
value: str,
format_style: str,
locale_str: str,
expected_output: str,
):
"""
Tests `localize_date` with `date` string across different locales and formats.
"""
assert localize_date(value, format_style, locale_str) == expected_output
@pytest.mark.parametrize(
"value, format_style, locale_str, expected_output",
[
pytest.param(
TEST_DATE_STRING,
"EEEE, MMM d, yyyy",
"en_US",
"Thursday, Oct 26, 2023",
id="date-en_US-custom",
),
pytest.param(
TEST_DATE_STRING,
"dd.MM.yyyy",
"de_DE",
"26.10.2023",
id="date-de_DE-custom",
),
# German weekday and month name translation
pytest.param(
TEST_DATE_STRING,
"EEEE",
"de_DE",
"Donnerstag",
id="weekday-de_DE",
),
pytest.param(
TEST_DATE_STRING,
"MMMM",
"de_DE",
"Oktober",
id="month-de_DE",
),
# French weekday and month name translation
pytest.param(
TEST_DATE_STRING,
"EEEE",
"fr_FR",
"jeudi",
id="weekday-fr_FR",
),
pytest.param(
TEST_DATE_STRING,
"MMMM",
"fr_FR",
"octobre",
id="month-fr_FR",
),
],
)
def test_localize_date_with_date_string(
self,
value: str,
format_style: str,
locale_str: str,
expected_output: str,
):
"""
Tests `localize_date` with `date` string across different locales and formats.
"""
assert localize_date(value, format_style, locale_str) == expected_output

302
src/documents/tests/test_workflows.py
View File

@@ -1,6 +1,8 @@
import datetime
import shutil
import socket
from datetime import timedelta
from pathlib import Path
from typing import TYPE_CHECKING
from unittest import mock
@@ -15,6 +17,7 @@ from guardian.shortcuts import get_users_with_perms
from httpx import HTTPError
from httpx import HTTPStatusError
from pytest_httpx import HTTPXMock
from rest_framework.test import APIClient
from rest_framework.test import APITestCase
from documents.signals.handlers import run_workflows
@@ -22,7 +25,7 @@ from documents.signals.handlers import send_webhook
if TYPE_CHECKING:
from django.db.models import QuerySet
from pytest_django.fixtures import SettingsWrapper
from documents import tasks
from documents.data_models import ConsumableDocument
@@ -122,7 +125,7 @@ class TestWorkflows(
filter_path=f"*/{self.dirs.scratch_dir.parts[-1]}/*",
)
action = WorkflowAction.objects.create(
assign_title="Doc from {correspondent}",
assign_title="Doc from {{correspondent}}",
assign_correspondent=self.c,
assign_document_type=self.dt,
assign_storage_path=self.sp,
@@ -241,7 +244,7 @@ class TestWorkflows(
)
action = WorkflowAction.objects.create(
assign_title="Doc from {correspondent}",
assign_title="Doc from {{correspondent}}",
assign_correspondent=self.c,
assign_document_type=self.dt,
assign_storage_path=self.sp,
@@ -892,7 +895,7 @@ class TestWorkflows(
filter_filename="*sample*",
)
action = WorkflowAction.objects.create(
assign_title="Doc created in {created_year}",
assign_title="Doc created in {{created_year}}",
assign_correspondent=self.c2,
assign_document_type=self.dt,
assign_storage_path=self.sp,
@@ -1155,7 +1158,7 @@ class TestWorkflows(
WHEN:
- File that matches is added
THEN:
- Title is not updated, error is output
- Title is updated but the placeholder isn't replaced
"""
trigger = WorkflowTrigger.objects.create(
type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED,
@@ -1181,15 +1184,12 @@ class TestWorkflows(
created=created,
)
with self.assertLogs("paperless.handlers", level="ERROR") as cm:
document_consumption_finished.send(
sender=self.__class__,
document=doc,
)
expected_str = f"Error occurred parsing title assignment '{action.assign_title}', falling back to original"
self.assertIn(expected_str, cm.output[0])
document_consumption_finished.send(
sender=self.__class__,
document=doc,
)
self.assertEqual(doc.title, "sample test")
self.assertEqual(doc.title, "Doc {created_year]")
def test_document_updated_workflow(self):
trigger = WorkflowTrigger.objects.create(
@@ -1223,6 +1223,45 @@ class TestWorkflows(
self.assertEqual(doc.custom_fields.all().count(), 1)
def test_document_consumption_workflow_month_placeholder_addded(self):
trigger = WorkflowTrigger.objects.create(
type=WorkflowTrigger.WorkflowTriggerType.CONSUMPTION,
sources=f"{DocumentSource.ApiUpload}",
filter_filename="simple*",
)
action = WorkflowAction.objects.create(
assign_title="Doc added in {{added_month_name_short}}",
)
w = Workflow.objects.create(
name="Workflow 1",
order=0,
)
w.triggers.add(trigger)
w.actions.add(action)
w.save()
superuser = User.objects.create_superuser("superuser")
self.client.force_authenticate(user=superuser)
test_file = shutil.copy(
self.SAMPLE_DIR / "simple.pdf",
self.dirs.scratch_dir / "simple.pdf",
)
with mock.patch("documents.tasks.ProgressManager", DummyProgressManager):
tasks.consume_file(
ConsumableDocument(
source=DocumentSource.ApiUpload,
original_file=test_file,
),
None,
)
document = Document.objects.first()
self.assertRegex(
document.title,
r"Doc added in \w{3,}",
) # Match any 3-letter month name
def test_document_updated_workflow_existing_custom_field(self):
"""
GIVEN:
@@ -2035,7 +2074,7 @@ class TestWorkflows(
filter_filename="*simple*",
)
action = WorkflowAction.objects.create(
assign_title="Doc from {correspondent}",
assign_title="Doc from {{correspondent}}",
assign_correspondent=self.c,
assign_document_type=self.dt,
assign_storage_path=self.sp,
@@ -2614,7 +2653,7 @@ class TestWorkflows(
)
webhook_action = WorkflowActionWebhook.objects.create(
use_params=False,
body="Test message: {doc_url}",
body="Test message: {{doc_url}}",
url="http://paperless-ngx.com",
include_document=False,
)
@@ -2673,7 +2712,7 @@ class TestWorkflows(
)
webhook_action = WorkflowActionWebhook.objects.create(
use_params=False,
body="Test message: {doc_url}",
body="Test message: {{doc_url}}",
url="http://paperless-ngx.com",
include_document=True,
)
@@ -3130,3 +3169,234 @@ class TestWebhookSecurity:
req = httpx_mock.get_request()
assert req.headers["Host"] == "paperless-ngx.com"
assert "evil.test" not in req.headers.get("Host", "")
@pytest.mark.django_db
class TestDateWorkflowLocalization(
SampleDirMixin,
):
"""Test cases for workflows that use date localization in templates."""
TEST_DATETIME = datetime.datetime(
2023,
6,
26,
14,
30,
5,
tzinfo=datetime.timezone.utc,
)
@pytest.mark.parametrize(
"title_template,expected_title",
[
pytest.param(
"Created at {{ created | localize_date('MMMM', 'es_ES') }}",
"Created at junio",
id="spanish_month",
),
pytest.param(
"Created at {{ created | localize_date('MMMM', 'de_DE') }}",
"Created at Juni", # codespell:ignore
id="german_month",
),
pytest.param(
"Created at {{ created | localize_date('dd/MM/yyyy', 'en_GB') }}",
"Created at 26/06/2023",
id="british_date_format",
),
],
)
def test_document_added_workflow_localization(
self,
title_template: str,
expected_title: str,
):
"""
GIVEN:
- Document added workflow with title template using localize_date filter
WHEN:
- Document is consumed
THEN:
- Document title is set with localized date
"""
trigger = WorkflowTrigger.objects.create(
type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED,
filter_filename="*sample*",
)
action = WorkflowAction.objects.create(
assign_title=title_template,
)
workflow = Workflow.objects.create(
name="Workflow 1",
order=0,
)
workflow.triggers.add(trigger)
workflow.actions.add(action)
workflow.save()
doc = Document.objects.create(
title="sample test",
correspondent=None,
original_filename="sample.pdf",
created=self.TEST_DATETIME,
)
document_consumption_finished.send(
sender=self.__class__,
document=doc,
)
doc.refresh_from_db()
assert doc.title == expected_title
@pytest.mark.parametrize(
"title_template,expected_title",
[
pytest.param(
"Created at {{ created | localize_date('MMMM', 'es_ES') }}",
"Created at junio",
id="spanish_month",
),
pytest.param(
"Created at {{ created | localize_date('MMMM', 'de_DE') }}",
"Created at Juni", # codespell:ignore
id="german_month",
),
pytest.param(
"Created at {{ created | localize_date('dd/MM/yyyy', 'en_GB') }}",
"Created at 26/06/2023",
id="british_date_format",
),
],
)
def test_document_updated_workflow_localization(
self,
title_template: str,
expected_title: str,
):
"""
GIVEN:
- Document updated workflow with title template using localize_date filter
WHEN:
- Document is updated via API
THEN:
- Document title is set with localized date
"""
# Setup test data
dt = DocumentType.objects.create(name="DocType Name")
c = Correspondent.objects.create(name="Correspondent Name")
client = APIClient()
superuser = User.objects.create_superuser("superuser")
client.force_authenticate(user=superuser)
trigger = WorkflowTrigger.objects.create(
type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_UPDATED,
filter_has_document_type=dt,
)
doc = Document.objects.create(
title="sample test",
correspondent=c,
original_filename="sample.pdf",
created=self.TEST_DATETIME,
)
action = WorkflowAction.objects.create(
assign_title=title_template,
)
workflow = Workflow.objects.create(
name="Workflow 1",
order=0,
)
workflow.triggers.add(trigger)
workflow.actions.add(action)
workflow.save()
client.patch(
f"/api/documents/{doc.id}/",
{"document_type": dt.id},
format="json",
)
doc.refresh_from_db()
assert doc.title == expected_title
@pytest.mark.parametrize(
"title_template,expected_title",
[
pytest.param(
"Added at {{ added | localize_date('MMMM', 'es_ES') }}",
"Added at junio",
id="spanish_month",
),
pytest.param(
"Added at {{ added | localize_date('MMMM', 'de_DE') }}",
"Added at Juni", # codespell:ignore
id="german_month",
),
pytest.param(
"Added at {{ added | localize_date('dd/MM/yyyy', 'en_GB') }}",
"Added at 26/06/2023",
id="british_date_format",
),
],
)
def test_document_consumption_workflow_localization(
self,
tmp_path: Path,
settings: SettingsWrapper,
title_template: str,
expected_title: str,
):
trigger = WorkflowTrigger.objects.create(
type=WorkflowTrigger.WorkflowTriggerType.CONSUMPTION,
sources=f"{DocumentSource.ApiUpload}",
filter_filename="simple*",
)
test_file = shutil.copy(
self.SAMPLE_DIR / "simple.pdf",
tmp_path / "simple.pdf",
)
action = WorkflowAction.objects.create(
assign_title=title_template,
)
w = Workflow.objects.create(
name="Workflow 1",
order=0,
)
w.triggers.add(trigger)
w.actions.add(action)
w.save()
settings.SCRATCH_DIR = tmp_path / "scratch"
(tmp_path / "scratch").mkdir(parents=True, exist_ok=True)
# Temporarily override "now" for the environment so templates using
# added/created placeholders behave as if it's a different system date.
with (
mock.patch(
"documents.tasks.ProgressManager",
DummyProgressManager,
),
mock.patch(
"django.utils.timezone.now",
return_value=self.TEST_DATETIME,
),
):
tasks.consume_file(
ConsumableDocument(
source=DocumentSource.ApiUpload,
original_file=test_file,
),
None,
)
document = Document.objects.first()
assert document.title == expected_title