paradizelost/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2025-09-30 01:32:43 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev' into feature-ai

Browse Source
This commit is contained in:
shamoon
2025-08-17 07:49:01 -07:00
committed by GitHub
parent 867c7d9e62 521fd1c957
commit 4c719948d9
123 changed files with 42275 additions and 39408 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/documents/management/commands/document_fuzzy_match.py
View File

@@ -125,14 +125,14 @@ class Command(MultiProcessMixin, ProgressBarMixin, BaseCommand):
messages.append(
self.style.NOTICE(
f"Document {result.doc_one_pk} fuzzy match"
f" to {result.doc_two_pk} (confidence {result.ratio:.3f})",
f" to {result.doc_two_pk} (confidence {result.ratio:.3f})\n",
),
)
maybe_delete_ids.append(result.doc_two_pk)
if len(messages) == 0:
messages.append(
self.style.SUCCESS("No matches found"),
self.style.SUCCESS("No matches found\n"),
)
self.stdout.writelines(
messages,

20
src/documents/serialisers.py
View File

@@ -2089,6 +2089,24 @@ class WorkflowTriggerSerializer(serializers.ModelSerializer):
return attrs
@staticmethod
def normalize_workflow_trigger_sources(trigger):
"""
Convert sources to strings to handle django-multiselectfield v1.0 changes
"""
if trigger and "sources" in trigger:
trigger["sources"] = [
str(s.value if hasattr(s, "value") else s) for s in trigger["sources"]
]
def create(self, validated_data):
WorkflowTriggerSerializer.normalize_workflow_trigger_sources(validated_data)
return super().create(validated_data)
def update(self, instance, validated_data):
WorkflowTriggerSerializer.normalize_workflow_trigger_sources(validated_data)
return super().update(instance, validated_data)
class WorkflowActionEmailSerializer(serializers.ModelSerializer):
id = serializers.IntegerField(allow_null=True, required=False)
@@ -2253,6 +2271,8 @@ class WorkflowSerializer(serializers.ModelSerializer):
if triggers is not None and triggers is not serializers.empty:
for trigger in triggers:
filter_has_tags = trigger.pop("filter_has_tags", None)
# Convert sources to strings to handle django-multiselectfield v1.0 changes
WorkflowTriggerSerializer.normalize_workflow_trigger_sources(trigger)
trigger_instance, _ = WorkflowTrigger.objects.update_or_create(
id=trigger.get("id"),
defaults=trigger,

41
src/documents/templating/filepath.py
View File

@@ -2,10 +2,13 @@ import logging
import os
import re
from collections.abc import Iterable
from datetime import date
from datetime import datetime
from pathlib import PurePath
import pathvalidate
from babel import Locale
from babel import dates
from django.utils import timezone
from django.utils.dateparse import parse_date
from django.utils.text import slugify as django_slugify
@@ -90,19 +93,51 @@ def get_cf_value(
return None
_template_environment.filters["get_cf_value"] = get_cf_value
def format_datetime(value: str | datetime, format: str) -> str:
if isinstance(value, str):
value = parse_date(value)
return value.strftime(format=format)
def localize_date(value: date | datetime, format: str, locale: str) -> str:
"""
Format a date or datetime object into a localized string using Babel.
Args:
value (date | datetime): The date or datetime to format. If a datetime
is provided, it should be timezone-aware (e.g., UTC from a Django DB object).
format (str): The format to use. Can be one of Babel's preset formats
('short', 'medium', 'long', 'full') or a custom pattern string.
locale (str): The locale code (e.g., 'en_US', 'fr_FR') to use for
localization.
Returns:
str: The localized, formatted date string.
Raises:
TypeError: If `value` is not a date or datetime instance.
"""
try:
Locale.parse(locale)
except Exception as e:
raise ValueError(f"Invalid locale identifier: {locale}") from e
if isinstance(value, datetime):
return dates.format_datetime(value, format=format, locale=locale)
elif isinstance(value, date):
return dates.format_date(value, format=format, locale=locale)
else:
raise TypeError(f"Unsupported type {type(value)} for localize_date")
_template_environment.filters["get_cf_value"] = get_cf_value
_template_environment.filters["datetime"] = format_datetime
_template_environment.filters["slugify"] = django_slugify
_template_environment.filters["localize_date"] = localize_date
def create_dummy_document():
"""

4
src/documents/tests/samples/malicious.svg Normal file
View File

@@ -0,0 +1,4 @@
<svg xmlns="http://www.w3.org/2000/svg" width="100" height="100">
<text x="10" y="20">Hello</text>
<script>alert('XSS')</script>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 140 B

70
src/documents/tests/test_api_app_config.py
View File

@@ -3,6 +3,7 @@ from pathlib import Path
from unittest.mock import patch
from django.contrib.auth.models import User
from django.core.files.uploadedfile import SimpleUploadedFile
from rest_framework import status
from rest_framework.test import APITestCase
@@ -157,25 +158,66 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
THEN:
- old app_logo file is deleted
"""
with (Path(__file__).parent / "samples" / "simple.jpg").open("rb") as f:
self.client.patch(
f"{self.ENDPOINT}1/",
{
"app_logo": f,
},
)
admin = User.objects.create_superuser(username="admin")
self.client.force_login(user=admin)
response = self.client.get("/logo/")
self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
self.client.patch(
f"{self.ENDPOINT}1/",
{
"app_logo": SimpleUploadedFile(
name="simple.jpg",
content=(
Path(__file__).parent / "samples" / "simple.jpg"
).read_bytes(),
content_type="image/jpeg",
),
},
)
# Logo exists at /logo/simple.jpg
response = self.client.get("/logo/simple.jpg")
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertIn("image/jpeg", response["Content-Type"])
config = ApplicationConfiguration.objects.first()
old_logo = config.app_logo
self.assertTrue(Path(old_logo.path).exists())
with (Path(__file__).parent / "samples" / "simple.png").open("rb") as f:
self.client.patch(
f"{self.ENDPOINT}1/",
{
"app_logo": f,
},
)
self.client.patch(
f"{self.ENDPOINT}1/",
{
"app_logo": SimpleUploadedFile(
name="simple.png",
content=(
Path(__file__).parent / "samples" / "simple.png"
).read_bytes(),
content_type="image/png",
),
},
)
self.assertFalse(Path(old_logo.path).exists())
def test_api_rejects_malicious_svg_logo(self):
"""
GIVEN:
- An SVG logo containing a <script> tag
WHEN:
- Uploaded via PATCH to app config
THEN:
- SVG is rejected with 400
"""
path = Path(__file__).parent / "samples" / "malicious.svg"
with path.open("rb") as f:
response = self.client.patch(
f"{self.ENDPOINT}1/",
{"app_logo": f},
format="multipart",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn("disallowed", str(response.data).lower())
def test_create_not_allowed(self):
"""
GIVEN:

196
src/documents/tests/test_file_handling.py
View File

@@ -4,6 +4,7 @@ import tempfile
from pathlib import Path
from unittest import mock
import pytest
from auditlog.context import disable_auditlog
from django.conf import settings
from django.contrib.auth.models import User
@@ -22,6 +23,8 @@ from documents.models import Document
from documents.models import DocumentType
from documents.models import StoragePath
from documents.tasks import empty_trash
from documents.templating.filepath import localize_date
from documents.tests.factories import DocumentFactory
from documents.tests.utils import DirectoriesMixin
from documents.tests.utils import FileSystemAssertsMixin
@@ -1586,3 +1589,196 @@ class TestFilenameGeneration(DirectoriesMixin, TestCase):
generate_filename(doc),
Path("brussels-belgium/some-title-with-special-characters.pdf"),
)
class TestDateLocalization:
"""
Groups all tests related to the `localize_date` function.
"""
TEST_DATE = datetime.date(2023, 10, 26)
TEST_DATETIME = datetime.datetime(
2023,
10,
26,
14,
30,
5,
tzinfo=datetime.timezone.utc,
)
@pytest.mark.parametrize(
"value, format_style, locale_str, expected_output",
[
pytest.param(
TEST_DATE,
"EEEE, MMM d, yyyy",
"en_US",
"Thursday, Oct 26, 2023",
id="date-en_US-custom",
),
pytest.param(
TEST_DATE,
"dd.MM.yyyy",
"de_DE",
"26.10.2023",
id="date-de_DE-custom",
),
# German weekday and month name translation
pytest.param(
TEST_DATE,
"EEEE",
"de_DE",
"Donnerstag",
id="weekday-de_DE",
),
pytest.param(
TEST_DATE,
"MMMM",
"de_DE",
"Oktober",
id="month-de_DE",
),
# French weekday and month name translation
pytest.param(
TEST_DATE,
"EEEE",
"fr_FR",
"jeudi",
id="weekday-fr_FR",
),
pytest.param(
TEST_DATE,
"MMMM",
"fr_FR",
"octobre",
id="month-fr_FR",
),
],
)
def test_localize_date_with_date_objects(
self,
value: datetime.date,
format_style: str,
locale_str: str,
expected_output: str,
):
"""
Tests `localize_date` with `date` objects across different locales and formats.
"""
assert localize_date(value, format_style, locale_str) == expected_output
@pytest.mark.parametrize(
"value, format_style, locale_str, expected_output",
[
pytest.param(
TEST_DATETIME,
"yyyy.MM.dd G 'at' HH:mm:ss zzz",
"en_US",
"2023.10.26 AD at 14:30:05 UTC",
id="datetime-en_US-custom",
),
pytest.param(
TEST_DATETIME,
"dd.MM.yyyy",
"fr_FR",
"26.10.2023",
id="date-fr_FR-custom",
),
# Spanish weekday and month translation
pytest.param(
TEST_DATETIME,
"EEEE",
"es_ES",
"jueves",
id="weekday-es_ES",
),
pytest.param(
TEST_DATETIME,
"MMMM",
"es_ES",
"octubre",
id="month-es_ES",
),
# Italian weekday and month translation
pytest.param(
TEST_DATETIME,
"EEEE",
"it_IT",
"giovedì",
id="weekday-it_IT",
),
pytest.param(
TEST_DATETIME,
"MMMM",
"it_IT",
"ottobre",
id="month-it_IT",
),
],
)
def test_localize_date_with_datetime_objects(
self,
value: datetime.datetime,
format_style: str,
locale_str: str,
expected_output: str,
):
# To handle the non-breaking space in French and other locales
result = localize_date(value, format_style, locale_str)
assert result.replace("\u202f", " ") == expected_output.replace("\u202f", " ")
@pytest.mark.parametrize(
"invalid_value",
[
"2023-10-26",
1698330605,
None,
[],
{},
],
)
def test_localize_date_raises_type_error_for_invalid_input(self, invalid_value):
with pytest.raises(TypeError) as excinfo:
localize_date(invalid_value, "medium", "en_US")
assert f"Unsupported type {type(invalid_value)}" in str(excinfo.value)
def test_localize_date_raises_error_for_invalid_locale(self):
with pytest.raises(ValueError) as excinfo:
localize_date(self.TEST_DATE, "medium", "invalid_locale_code")
assert "Invalid locale identifier" in str(excinfo.value)
@pytest.mark.django_db
@pytest.mark.parametrize(
"filename_format,expected_filename",
[
pytest.param(
"{{title}}_{{ document.created | localize_date('MMMM', 'es_ES')}}",
"My Document_octubre.pdf",
id="spanish_month_name",
),
pytest.param(
"{{title}}_{{ document.created | localize_date('EEEE', 'fr_FR')}}",
"My Document_jeudi.pdf",
id="french_day_of_week",
),
pytest.param(
"{{title}}_{{ document.created | localize_date('dd/MM/yyyy', 'en_GB')}}",
"My Document_26/10/2023.pdf",
id="uk_date_format",
),
],
)
def test_localize_date_path_building(self, filename_format, expected_filename):
document = DocumentFactory.create(
title="My Document",
mime_type="application/pdf",
storage_type=Document.STORAGE_TYPE_UNENCRYPTED,
created=self.TEST_DATE, # 2023-10-26 (which is a Thursday)
)
with override_settings(FILENAME_FORMAT=filename_format):
filename = generate_filename(document)
assert filename == Path(expected_filename)

2
src/documents/tests/test_management_exporter.py
View File

@@ -123,7 +123,7 @@ class TestExportImport(
self.trigger = WorkflowTrigger.objects.create(
type=WorkflowTrigger.WorkflowTriggerType.CONSUMPTION,
sources=[1],
sources=[str(WorkflowTrigger.DocumentSourceChoices.CONSUME_FOLDER.value)],
filter_filename="*",
)
self.action = WorkflowAction.objects.create(assign_title="new title")

21
src/documents/tests/test_management_fuzzy.py
View File

@@ -87,7 +87,7 @@ class TestFuzzyMatchCommand(TestCase):
filename="other_test.pdf",
)
stdout, _ = self.call_command()
self.assertEqual(stdout, "No matches found\n")
self.assertIn("No matches found", stdout)
def test_with_matches(self):
"""
@@ -116,7 +116,7 @@ class TestFuzzyMatchCommand(TestCase):
filename="other_test.pdf",
)
stdout, _ = self.call_command("--processes", "1")
self.assertRegex(stdout, self.MSG_REGEX + "\n")
self.assertRegex(stdout, self.MSG_REGEX)
def test_with_3_matches(self):
"""
@@ -152,11 +152,10 @@ class TestFuzzyMatchCommand(TestCase):
filename="final_test.pdf",
)
stdout, _ = self.call_command()
lines = [x.strip() for x in stdout.split("\n") if len(x.strip())]
lines = [x.strip() for x in stdout.splitlines() if x.strip()]
self.assertEqual(len(lines), 3)
self.assertRegex(lines[0], self.MSG_REGEX)
self.assertRegex(lines[1], self.MSG_REGEX)
self.assertRegex(lines[2], self.MSG_REGEX)
for line in lines:
self.assertRegex(line, self.MSG_REGEX)
def test_document_deletion(self):
"""
@@ -197,14 +196,12 @@ class TestFuzzyMatchCommand(TestCase):
stdout, _ = self.call_command("--delete")
lines = [x.strip() for x in stdout.split("\n") if len(x.strip())]
self.assertEqual(len(lines), 3)
self.assertEqual(
lines[0],
self.assertIn(
"The command is configured to delete documents. Use with caution",
stdout,
)
self.assertRegex(lines[1], self.MSG_REGEX)
self.assertEqual(lines[2], "Deleting 1 documents based on ratio matches")
self.assertRegex(stdout, self.MSG_REGEX)
self.assertIn("Deleting 1 documents based on ratio matches", stdout)
self.assertEqual(Document.objects.count(), 2)
self.assertIsNotNone(Document.objects.get(pk=1))

2
src/documents/tests/test_migration_workflows.py
View File

@@ -104,7 +104,7 @@ class TestReverseMigrateWorkflow(TestMigrations):
trigger = WorkflowTrigger.objects.create(
type=0,
sources=[DocumentSource.ConsumeFolder],
sources=[str(DocumentSource.ConsumeFolder)],
filter_path="*/path/*",
filter_filename="*file*",
)

25
src/documents/views.py
View File

@@ -14,6 +14,7 @@ from urllib.parse import quote
from urllib.parse import urlparse
import httpx
import magic
import pathvalidate
from celery import states
from django.conf import settings
@@ -34,6 +35,7 @@ from django.db.models import When
from django.db.models.functions import Length
from django.db.models.functions import Lower
from django.db.models.manager import Manager
from django.http import FileResponse
from django.http import Http404
from django.http import HttpResponse
from django.http import HttpResponseBadRequest
@@ -180,6 +182,7 @@ from paperless.celery import app as celery_app
from paperless.config import AIConfig
from paperless.config import GeneralConfig
from paperless.db import GnuPG
from paperless.models import ApplicationConfiguration
from paperless.serialisers import GroupSerializer
from paperless.serialisers import UserSerializer
from paperless.views import StandardPagination
@@ -3109,3 +3112,25 @@ class TrashView(ListModelMixin, PassUserMixin):
doc_ids = [doc.id for doc in docs]
empty_trash(doc_ids=doc_ids)
return Response({"result": "OK", "doc_ids": doc_ids})
def serve_logo(request, filename=None):
"""
Serves the configured logo file with Content-Disposition: attachment.
Prevents inline execution of SVGs. See GHSA-6p53-hqqw-8j62
"""
config = ApplicationConfiguration.objects.first()
app_logo = config.app_logo
if not app_logo:
raise Http404("No logo configured")
path = app_logo.path
content_type = magic.from_file(path, mime=True) or "application/octet-stream"
return FileResponse(
app_logo.open("rb"),
content_type=content_type,
filename=app_logo.name,
as_attachment=True,
)