paradizelost/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2025-11-23 23:49:08 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev' into feature-remote-ocr-2

Browse Source
This commit is contained in:
shamoon
2025-11-13 15:45:08 -08:00
committed by GitHub
parent 0537e87cb5 005ef4fce6
commit ef834ae808
13 changed files with 275 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
docs/changelog.md
View File

@@ -1,5 +1,19 @@
# Changelog
## paperless-ngx 2.19.5
### Bug Fixes
- Fix: ensure custom field query propagation, change detection [@shamoon](https://github.com/shamoon) ([#11291](https://github.com/paperless-ngx/paperless-ngx/pull/11291))
### Dependencies
- docker(deps): Bump astral-sh/uv from 0.9.4-python3.12-bookworm-slim to 0.9.7-python3.12-bookworm-slim @[dependabot[bot]](https://github.com/apps/dependabot) ([#11283](https://github.com/paperless-ngx/paperless-ngx/pull/11283))
### All App Changes
- Fix: ensure custom field query propagation, change detection [@shamoon](https://github.com/shamoon) ([#11291](https://github.com/paperless-ngx/paperless-ngx/pull/11291))
## paperless-ngx 2.19.4
### Bug Fixes

24
src-ui/messages.xlf
View File

@@ -2557,7 +2557,7 @@
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts</context>
<context context-type="linenumber">195</context>
<context context-type="linenumber">196</context>
</context-group>
</trans-unit>
<trans-unit id="2753185112875184719" datatype="html">
@@ -6044,71 +6044,71 @@
<source>Profile updated successfully</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts</context>
<context context-type="linenumber">192</context>
<context context-type="linenumber">193</context>
</context-group>
</trans-unit>
<trans-unit id="3417726855410304962" datatype="html">
<source>Error saving profile</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts</context>
<context context-type="linenumber">206</context>
<context context-type="linenumber">207</context>
</context-group>
</trans-unit>
<trans-unit id="154249228726292516" datatype="html">
<source>Error generating auth token</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts</context>
<context context-type="linenumber">223</context>
<context context-type="linenumber">225</context>
</context-group>
</trans-unit>
<trans-unit id="4153637646944982460" datatype="html">
<source>Error disconnecting social account</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts</context>
<context context-type="linenumber">248</context>
<context context-type="linenumber">250</context>
</context-group>
</trans-unit>
<trans-unit id="5939111172212776886" datatype="html">
<source>Error fetching TOTP settings</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts</context>
<context context-type="linenumber">267</context>
<context context-type="linenumber">269</context>
</context-group>
</trans-unit>
<trans-unit id="1030314492414713260" datatype="html">
<source>TOTP activated successfully</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts</context>
<context context-type="linenumber">288</context>
<context context-type="linenumber">290</context>
</context-group>
</trans-unit>
<trans-unit id="3755006064892435830" datatype="html">
<source>Error activating TOTP</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts</context>
<context context-type="linenumber">290</context>
<context context-type="linenumber">292</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts</context>
<context context-type="linenumber">296</context>
<context context-type="linenumber">298</context>
</context-group>
</trans-unit>
<trans-unit id="5919827473541889422" datatype="html">
<source>TOTP deactivated successfully</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts</context>
<context context-type="linenumber">312</context>
<context context-type="linenumber">314</context>
</context-group>
</trans-unit>
<trans-unit id="6214722303383624015" datatype="html">
<source>Error deactivating TOTP</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts</context>
<context context-type="linenumber">314</context>
<context context-type="linenumber">316</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts</context>
<context context-type="linenumber">319</context>
<context context-type="linenumber">321</context>
</context-group>
</trans-unit>
<trans-unit id="6617773613987957957" datatype="html">

2
src-ui/src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts
View File

@@ -183,6 +183,7 @@ export class ProfileEditDialogComponent
this.newPassword && this.currentPassword !== this.newPassword
const profile = Object.assign({}, this.form.value)
delete profile.totp_code
this.error = null
this.networkActive = true
this.profileService
.update(profile)
@@ -204,6 +205,7 @@ export class ProfileEditDialogComponent
},
error: (error) => {
this.toastService.showError($localize`Error saving profile`, error)
this.error = error?.error
this.networkActive = false
},
})

21
src/documents/file_handling.py
View File

@@ -99,16 +99,7 @@ def generate_unique_filename(doc, *, archive_filename=False) -> Path:
return new_filename
def generate_filename(
doc: Document,
*,
counter=0,
append_gpg=True,
archive_filename=False,
) -> Path:
base_path: Path | None = None
def format_filename(document: Document, template_str: str) -> str | None:
def format_filename(document: Document, template_str: str) -> str | None:
rendered_filename = validate_filepath_template_and_render(
template_str,
document,
@@ -130,6 +121,16 @@ def generate_filename(
return rendered_filename
def generate_filename(
doc: Document,
*,
counter=0,
append_gpg=True,
archive_filename=False,
) -> Path:
base_path: Path | None = None
# Determine the source of the format string
if doc.storage_path is not None:
filename_format = doc.storage_path.path

31
src/documents/templating/filepath.py
View File

@@ -52,6 +52,33 @@ class FilePathTemplate(Template):
return clean_filepath(original_render)
class PlaceholderString(str):
"""
String subclass used as a sentinel for empty metadata values inside templates.
- Renders as \"-none-\" to preserve existing filename cleaning logic.
- Compares equal to either \"-none-\" or \"none\" so templates can check for either.
- Evaluates to False so {% if correspondent %} behaves intuitively.
"""
def __new__(cls, value: str = "-none-"):
return super().__new__(cls, value)
def __bool__(self) -> bool:
return False
def __eq__(self, other) -> bool:
if isinstance(other, str) and other == "none":
other = "-none-"
return super().__eq__(other)
def __ne__(self, other) -> bool:
return not self.__eq__(other)
NO_VALUE_PLACEHOLDER = PlaceholderString("-none-")
_template_environment.undefined = _LogStrictUndefined
_template_environment.filters["get_cf_value"] = get_cf_value
@@ -128,7 +155,7 @@ def get_added_date_context(document: Document) -> dict[str, str]:
def get_basic_metadata_context(
document: Document,
*,
no_value_default: str,
no_value_default: str = NO_VALUE_PLACEHOLDER,
) -> dict[str, str]:
"""
Given a Document, constructs some basic information about it. If certain values are not set,
@@ -266,7 +293,7 @@ def validate_filepath_template_and_render(
# Build the context dictionary
context = (
{"document": document}
| get_basic_metadata_context(document, no_value_default="-none-")
| get_basic_metadata_context(document, no_value_default=NO_VALUE_PLACEHOLDER)
| get_creation_date_context(document)
| get_added_date_context(document)
| get_tags_context(tags_list)

40
src/documents/tests/test_api_objects.py
View File

@@ -4,6 +4,7 @@ from unittest import mock
from django.contrib.auth.models import Permission
from django.contrib.auth.models import User
from django.test import override_settings
from rest_framework import status
from rest_framework.test import APITestCase
@@ -334,6 +335,45 @@ class TestApiStoragePaths(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data, "path/Something")
def test_test_storage_path_respects_none_placeholder_setting(self):
"""
GIVEN:
- A storage path template referencing an empty field
WHEN:
- Testing the template before and after enabling remove-none
THEN:
- The preview shows "none" by default and drops the placeholder when configured
"""
document = Document.objects.create(
mime_type="application/pdf",
storage_path=self.sp1,
title="Something",
checksum="123",
)
payload = json.dumps(
{
"document": document.id,
"path": "folder/{{ correspondent }}/{{ title }}",
},
)
response = self.client.post(
f"{self.ENDPOINT}test/",
payload,
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data, "folder/none/Something")
with override_settings(FILENAME_FORMAT_REMOVE_NONE=True):
response = self.client.post(
f"{self.ENDPOINT}test/",
payload,
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data, "folder/Something")
class TestBulkEditObjects(APITestCase):
# See test_api_permissions.py for bulk tests on permissions

4
src/documents/tests/test_api_permissions.py
View File

@@ -648,7 +648,7 @@ class TestApiUser(DirectoriesMixin, APITestCase):
user1 = {
"username": "testuser",
"password": "test",
"password": "areallysupersecretpassword235",
"first_name": "Test",
"last_name": "User",
}
@@ -730,7 +730,7 @@ class TestApiUser(DirectoriesMixin, APITestCase):
f"{self.ENDPOINT}{user1.pk}/",
data={
"first_name": "Updated Name 2",
"password": "123xyz",
"password": "newreallystrongpassword456",
},
)

59
src/documents/tests/test_api_profile.py
View File

@@ -192,6 +192,65 @@ class TestApiProfile(DirectoriesMixin, APITestCase):
self.assertEqual(user.first_name, user_data["first_name"])
self.assertEqual(user.last_name, user_data["last_name"])
def test_update_profile_invalid_password_returns_field_error(self):
"""
GIVEN:
- Configured user
WHEN:
- API call is made to update profile with weak password
THEN:
- Profile update fails with password field error
"""
user_data = {
"email": "new@email.com",
"password": "short", # shorter than default validator threshold
"first_name": "new first name",
"last_name": "new last name",
}
response = self.client.patch(self.ENDPOINT, user_data)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn("password", response.data)
self.assertIsInstance(response.data["password"], list)
self.assertTrue(
any(
"too short" in message.lower() for message in response.data["password"]
),
)
def test_update_profile_placeholder_password_skips_validation(self):
"""
GIVEN:
- Configured user with existing password
WHEN:
- API call is made with the obfuscated placeholder password value
THEN:
- Profile is updated without changing the password or running validators
"""
original_password = "orig-pass-12345"
self.user.set_password(original_password)
self.user.save()
user_data = {
"email": "new@email.com",
"password": "*" * 12, # matches obfuscated value from serializer
"first_name": "new first name",
"last_name": "new last name",
}
response = self.client.patch(self.ENDPOINT, user_data)
self.assertEqual(response.status_code, status.HTTP_200_OK)
user = User.objects.get(username=self.user.username)
self.assertTrue(user.check_password(original_password))
self.assertEqual(user.email, user_data["email"])
self.assertEqual(user.first_name, user_data["first_name"])
self.assertEqual(user.last_name, user_data["last_name"])
def test_update_auth_token(self):
"""
GIVEN:

41
src/documents/tests/test_file_handling.py
View File

@@ -1078,6 +1078,47 @@ class TestFilenameGeneration(DirectoriesMixin, TestCase):
Path("SomeImportantNone/2020-07-25.pdf"),
)
@override_settings(
FILENAME_FORMAT=(
"{% if correspondent == 'none' %}none/{% endif %}"
"{% if correspondent == '-none-' %}dash/{% endif %}"
"{% if not correspondent %}false/{% endif %}"
"{% if correspondent != 'none' %}notnoneyes/{% else %}notnoneno/{% endif %}"
"{{ correspondent or 'missing' }}/{{ title }}"
),
)
def test_placeholder_matches_none_variants_and_false(self):
"""
GIVEN:
- Templates that compare against 'none', '-none-' and rely on truthiness
WHEN:
- A document has or lacks a correspondent
THEN:
- Empty placeholders behave like both strings and evaluate False
"""
doc_without_correspondent = Document.objects.create(
title="does not matter",
mime_type="application/pdf",
checksum="abc",
)
doc_with_correspondent = Document.objects.create(
title="does not matter",
mime_type="application/pdf",
checksum="def",
correspondent=Correspondent.objects.create(name="Acme"),
)
self.assertEqual(
generate_filename(doc_without_correspondent),
Path(
"none/dash/false/notnoneno/missing/does not matter.pdf",
),
)
self.assertEqual(
generate_filename(doc_with_correspondent),
Path("notnoneyes/Acme/does not matter.pdf"),
)
@override_settings(
FILENAME_FORMAT="{created_year_short}/{created_month_name_short}/{created_month_name}/{title}",
)

24
src/documents/views.py
View File

@@ -23,6 +23,7 @@ from django.conf import settings
from django.contrib.auth.models import Group
from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
from django.core.cache import cache
from django.db import connections
from django.db.migrations.loader import MigrationLoader
from django.db.migrations.recorder import MigrationRecorder
@@ -51,7 +52,6 @@ from django.utils.timezone import make_aware
from django.utils.translation import get_language
from django.views import View
from django.views.decorators.cache import cache_control
from django.views.decorators.cache import cache_page
from django.views.decorators.http import condition
from django.views.decorators.http import last_modified
from django.views.generic import TemplateView
@@ -108,6 +108,7 @@ from documents.conditionals import thumbnail_last_modified
from documents.data_models import ConsumableDocument
from documents.data_models import DocumentMetadataOverrides
from documents.data_models import DocumentSource
from documents.file_handling import format_filename
from documents.filters import CorrespondentFilterSet
from documents.filters import CustomFieldFilterSet
from documents.filters import DocumentFilterSet
@@ -183,7 +184,6 @@ from documents.tasks import index_optimize
from documents.tasks import sanity_check
from documents.tasks import train_classifier
from documents.tasks import update_document_parent_tags
from documents.templating.filepath import validate_filepath_template_and_render
from documents.utils import get_boolean
from paperless import version
from paperless.celery import app as celery_app
@@ -2336,7 +2336,7 @@ class StoragePathViewSet(ModelViewSet, PermissionsAwareDocumentCountMixin):
document = serializer.validated_data.get("document")
path = serializer.validated_data.get("path")
result = validate_filepath_template_and_render(path, document)
result = format_filename(document, path)
return Response(result)
@@ -2426,7 +2426,6 @@ class UiSettingsView(GenericAPIView):
)
@method_decorator(cache_page(60 * 15), name="dispatch")
@extend_schema_view(
get=extend_schema(
description="Get the current version of the Paperless-NGX server",
@@ -2436,17 +2435,18 @@ class UiSettingsView(GenericAPIView):
),
)
class RemoteVersionView(GenericAPIView):
cache_key = "remote_version_view_latest_release"
def get(self, request, format=None):
remote_version = "0.0.0"
is_greater_than_current = False
current_version = packaging_version.parse(version.__full_version_str__)
remote_version = cache.get(self.cache_key)
if remote_version is None:
try:
resp = httpx.get(
"https://api.github.com/repos/paperless-ngx/paperless-ngx/releases/latest",
headers={"Accept": "application/json"},
)
resp.raise_for_status()
try:
data = resp.json()
remote_version = data["tag_name"]
# Some early tags used ngx-x.y.z
@@ -2456,11 +2456,13 @@ class RemoteVersionView(GenericAPIView):
except httpx.HTTPError as e:
logger.debug(f"An error occurred checking for available updates: {e}")
if remote_version:
cache.set(self.cache_key, remote_version, 60 * 15)
else:
remote_version = "0.0.0"
is_greater_than_current = (
packaging_version.parse(
remote_version,
)
> current_version
packaging_version.parse(remote_version) > current_version
)
return Response(

39
src/paperless/serialisers.py
View File

@@ -9,6 +9,7 @@ from allauth.socialaccount.models import SocialApp
from django.contrib.auth.models import Group
from django.contrib.auth.models import Permission
from django.contrib.auth.models import User
from django.contrib.auth.password_validation import validate_password
from rest_framework import serializers
from rest_framework.authtoken.serializers import AuthTokenSerializer
@@ -19,6 +20,23 @@ from paperless_mail.serialisers import ObfuscatedPasswordField
logger = logging.getLogger("paperless.settings")
class PasswordValidationMixin:
def _has_real_password(self, value: str | None) -> bool:
return bool(value) and value.replace("*", "") != ""
def validate_password(self, value: str) -> str:
if not self._has_real_password(value):
return value
request = self.context.get("request") if hasattr(self, "context") else None
user = self.instance or (
request.user if request and hasattr(request, "user") else None
)
validate_password(value, user) # raise ValidationError if invalid
return value
class PaperlessAuthTokenSerializer(AuthTokenSerializer):
code = serializers.CharField(
label="MFA Code",
@@ -49,7 +67,7 @@ class PaperlessAuthTokenSerializer(AuthTokenSerializer):
return attrs
class UserSerializer(serializers.ModelSerializer):
class UserSerializer(PasswordValidationMixin, serializers.ModelSerializer):
password = ObfuscatedPasswordField(required=False)
user_permissions = serializers.SlugRelatedField(
many=True,
@@ -87,11 +105,11 @@ class UserSerializer(serializers.ModelSerializer):
return obj.get_group_permissions()
def update(self, instance, validated_data):
if "password" in validated_data:
if len(validated_data.get("password").replace("*", "")) > 0:
instance.set_password(validated_data.get("password"))
password = validated_data.pop("password", None)
if self._has_real_password(password):
instance.set_password(password)
instance.save()
validated_data.pop("password")
super().update(instance, validated_data)
return instance
@@ -102,12 +120,7 @@ class UserSerializer(serializers.ModelSerializer):
user_permissions = None
if "user_permissions" in validated_data:
user_permissions = validated_data.pop("user_permissions")
password = None
if (
"password" in validated_data
and len(validated_data.get("password").replace("*", "")) > 0
):
password = validated_data.pop("password")
password = validated_data.pop("password", None)
user = User.objects.create(**validated_data)
# set groups
if groups:
@@ -116,7 +129,7 @@ class UserSerializer(serializers.ModelSerializer):
if user_permissions:
user.user_permissions.set(user_permissions)
# set password
if password:
if self._has_real_password(password):
user.set_password(password)
user.save()
return user
@@ -156,7 +169,7 @@ class SocialAccountSerializer(serializers.ModelSerializer):
return "Unknown App"
class ProfileSerializer(serializers.ModelSerializer):
class ProfileSerializer(PasswordValidationMixin, serializers.ModelSerializer):
email = serializers.EmailField(allow_blank=True, required=False)
password = ObfuscatedPasswordField(required=False, allow_null=False)
auth_token = serializers.SlugRelatedField(read_only=True, slug_field="key")

6
src/paperless/views.py
View File

@@ -197,10 +197,10 @@ class ProfileView(GenericAPIView):
serializer.is_valid(raise_exception=True)
user = self.request.user if hasattr(self.request, "user") else None
if len(serializer.validated_data.get("password").replace("*", "")) > 0:
user.set_password(serializer.validated_data.get("password"))
password = serializer.validated_data.pop("password", None)
if password and password.replace("*", ""):
user.set_password(password)
user.save()
serializer.validated_data.pop("password")
for key, value in serializer.validated_data.items():
setattr(user, key, value)

3
src/paperless_mail/oauth.py
View File

@@ -103,6 +103,9 @@ class PaperlessMailOAuth2Manager:
refresh_token=account.refresh_token,
),
)
if "refresh_token" in result:
# Outlook returns a new refresh token on refresh, Gmail does not
account.refresh_token = result["refresh_token"]
account.password = result["access_token"]
account.expiration = timezone.now() + timedelta(
seconds=result["expires_in"],