paradizelost/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2025-11-21 04:36:53 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev' into feature-ai

Browse Source
This commit is contained in:
shamoon
2025-11-13 09:34:17 -08:00
parent c4a85186bd 005ef4fce6
commit 9043563df3
13 changed files with 275 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
docs/changelog.md
View File

@@ -1,5 +1,19 @@
# Changelog
## paperless-ngx 2.19.5
### Bug Fixes
- Fix: ensure custom field query propagation, change detection [@shamoon](https://github.com/shamoon) ([#11291](https://github.com/paperless-ngx/paperless-ngx/pull/11291))
### Dependencies
- docker(deps): Bump astral-sh/uv from 0.9.4-python3.12-bookworm-slim to 0.9.7-python3.12-bookworm-slim @[dependabot[bot]](https://github.com/apps/dependabot) ([#11283](https://github.com/paperless-ngx/paperless-ngx/pull/11283))
### All App Changes
- Fix: ensure custom field query propagation, change detection [@shamoon](https://github.com/shamoon) ([#11291](https://github.com/paperless-ngx/paperless-ngx/pull/11291))
## paperless-ngx 2.19.4
### Bug Fixes

24
src-ui/messages.xlf
View File

@@ -2557,7 +2557,7 @@
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts</context>
<context context-type="linenumber">195</context>
<context context-type="linenumber">196</context>
</context-group>
</trans-unit>
<trans-unit id="2753185112875184719" datatype="html">
@@ -6044,71 +6044,71 @@
<source>Profile updated successfully</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts</context>
<context context-type="linenumber">192</context>
<context context-type="linenumber">193</context>
</context-group>
</trans-unit>
<trans-unit id="3417726855410304962" datatype="html">
<source>Error saving profile</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts</context>
<context context-type="linenumber">206</context>
<context context-type="linenumber">207</context>
</context-group>
</trans-unit>
<trans-unit id="154249228726292516" datatype="html">
<source>Error generating auth token</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts</context>
<context context-type="linenumber">223</context>
<context context-type="linenumber">225</context>
</context-group>
</trans-unit>
<trans-unit id="4153637646944982460" datatype="html">
<source>Error disconnecting social account</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts</context>
<context context-type="linenumber">248</context>
<context context-type="linenumber">250</context>
</context-group>
</trans-unit>
<trans-unit id="5939111172212776886" datatype="html">
<source>Error fetching TOTP settings</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts</context>
<context context-type="linenumber">267</context>
<context context-type="linenumber">269</context>
</context-group>
</trans-unit>
<trans-unit id="1030314492414713260" datatype="html">
<source>TOTP activated successfully</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts</context>
<context context-type="linenumber">288</context>
<context context-type="linenumber">290</context>
</context-group>
</trans-unit>
<trans-unit id="3755006064892435830" datatype="html">
<source>Error activating TOTP</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts</context>
<context context-type="linenumber">290</context>
<context context-type="linenumber">292</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts</context>
<context context-type="linenumber">296</context>
<context context-type="linenumber">298</context>
</context-group>
</trans-unit>
<trans-unit id="5919827473541889422" datatype="html">
<source>TOTP deactivated successfully</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts</context>
<context context-type="linenumber">312</context>
<context context-type="linenumber">314</context>
</context-group>
</trans-unit>
<trans-unit id="6214722303383624015" datatype="html">
<source>Error deactivating TOTP</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts</context>
<context context-type="linenumber">314</context>
<context context-type="linenumber">316</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts</context>
<context context-type="linenumber">319</context>
<context context-type="linenumber">321</context>
</context-group>
</trans-unit>
<trans-unit id="6617773613987957957" datatype="html">

2
src-ui/src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts
View File

@@ -183,6 +183,7 @@ export class ProfileEditDialogComponent
this.newPassword && this.currentPassword !== this.newPassword
const profile = Object.assign({}, this.form.value)
delete profile.totp_code
this.error = null
this.networkActive = true
this.profileService
.update(profile)
@@ -204,6 +205,7 @@ export class ProfileEditDialogComponent
},
error: (error) => {
this.toastService.showError($localize`Error saving profile`, error)
this.error = error?.error
this.networkActive = false
},
})

45
src/documents/file_handling.py
View File

@@ -99,6 +99,29 @@ def generate_unique_filename(doc, *, archive_filename=False) -> Path:
return new_filename
def format_filename(document: Document, template_str: str) -> str | None:
rendered_filename = validate_filepath_template_and_render(
template_str,
document,
)
if rendered_filename is None:
return None
# Apply this setting. It could become a filter in the future (or users could use |default)
if settings.FILENAME_FORMAT_REMOVE_NONE:
rendered_filename = rendered_filename.replace("/-none-/", "/")
rendered_filename = rendered_filename.replace(" -none-", "")
rendered_filename = rendered_filename.replace("-none-", "")
rendered_filename = rendered_filename.strip(os.sep)
rendered_filename = rendered_filename.replace(
"-none-",
"none",
) # backward compatibility
return rendered_filename
def generate_filename(
doc: Document,
*,
@@ -108,28 +131,6 @@ def generate_filename(
) -> Path:
base_path: Path | None = None
def format_filename(document: Document, template_str: str) -> str | None:
rendered_filename = validate_filepath_template_and_render(
template_str,
document,
)
if rendered_filename is None:
return None
# Apply this setting. It could become a filter in the future (or users could use |default)
if settings.FILENAME_FORMAT_REMOVE_NONE:
rendered_filename = rendered_filename.replace("/-none-/", "/")
rendered_filename = rendered_filename.replace(" -none-", "")
rendered_filename = rendered_filename.replace("-none-", "")
rendered_filename = rendered_filename.strip(os.sep)
rendered_filename = rendered_filename.replace(
"-none-",
"none",
) # backward compatibility
return rendered_filename
# Determine the source of the format string
if doc.storage_path is not None:
filename_format = doc.storage_path.path

31
src/documents/templating/filepath.py
View File

@@ -52,6 +52,33 @@ class FilePathTemplate(Template):
return clean_filepath(original_render)
class PlaceholderString(str):
"""
String subclass used as a sentinel for empty metadata values inside templates.
- Renders as \"-none-\" to preserve existing filename cleaning logic.
- Compares equal to either \"-none-\" or \"none\" so templates can check for either.
- Evaluates to False so {% if correspondent %} behaves intuitively.
"""
def __new__(cls, value: str = "-none-"):
return super().__new__(cls, value)
def __bool__(self) -> bool:
return False
def __eq__(self, other) -> bool:
if isinstance(other, str) and other == "none":
other = "-none-"
return super().__eq__(other)
def __ne__(self, other) -> bool:
return not self.__eq__(other)
NO_VALUE_PLACEHOLDER = PlaceholderString("-none-")
_template_environment.undefined = _LogStrictUndefined
_template_environment.filters["get_cf_value"] = get_cf_value
@@ -128,7 +155,7 @@ def get_added_date_context(document: Document) -> dict[str, str]:
def get_basic_metadata_context(
document: Document,
*,
no_value_default: str,
no_value_default: str = NO_VALUE_PLACEHOLDER,
) -> dict[str, str]:
"""
Given a Document, constructs some basic information about it. If certain values are not set,
@@ -266,7 +293,7 @@ def validate_filepath_template_and_render(
# Build the context dictionary
context = (
{"document": document}
| get_basic_metadata_context(document, no_value_default="-none-")
| get_basic_metadata_context(document, no_value_default=NO_VALUE_PLACEHOLDER)
| get_creation_date_context(document)
| get_added_date_context(document)
| get_tags_context(tags_list)

40
src/documents/tests/test_api_objects.py
View File

@@ -4,6 +4,7 @@ from unittest import mock
from django.contrib.auth.models import Permission
from django.contrib.auth.models import User
from django.test import override_settings
from rest_framework import status
from rest_framework.test import APITestCase
@@ -334,6 +335,45 @@ class TestApiStoragePaths(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data, "path/Something")
def test_test_storage_path_respects_none_placeholder_setting(self):
"""
GIVEN:
- A storage path template referencing an empty field
WHEN:
- Testing the template before and after enabling remove-none
THEN:
- The preview shows "none" by default and drops the placeholder when configured
"""
document = Document.objects.create(
mime_type="application/pdf",
storage_path=self.sp1,
title="Something",
checksum="123",
)
payload = json.dumps(
{
"document": document.id,
"path": "folder/{{ correspondent }}/{{ title }}",
},
)
response = self.client.post(
f"{self.ENDPOINT}test/",
payload,
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data, "folder/none/Something")
with override_settings(FILENAME_FORMAT_REMOVE_NONE=True):
response = self.client.post(
f"{self.ENDPOINT}test/",
payload,
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data, "folder/Something")
class TestBulkEditObjects(APITestCase):
# See test_api_permissions.py for bulk tests on permissions

4
src/documents/tests/test_api_permissions.py
View File

@@ -648,7 +648,7 @@ class TestApiUser(DirectoriesMixin, APITestCase):
user1 = {
"username": "testuser",
"password": "test",
"password": "areallysupersecretpassword235",
"first_name": "Test",
"last_name": "User",
}
@@ -730,7 +730,7 @@ class TestApiUser(DirectoriesMixin, APITestCase):
f"{self.ENDPOINT}{user1.pk}/",
data={
"first_name": "Updated Name 2",
"password": "123xyz",
"password": "newreallystrongpassword456",
},
)

59
src/documents/tests/test_api_profile.py
View File

@@ -192,6 +192,65 @@ class TestApiProfile(DirectoriesMixin, APITestCase):
self.assertEqual(user.first_name, user_data["first_name"])
self.assertEqual(user.last_name, user_data["last_name"])
def test_update_profile_invalid_password_returns_field_error(self):
"""
GIVEN:
- Configured user
WHEN:
- API call is made to update profile with weak password
THEN:
- Profile update fails with password field error
"""
user_data = {
"email": "new@email.com",
"password": "short", # shorter than default validator threshold
"first_name": "new first name",
"last_name": "new last name",
}
response = self.client.patch(self.ENDPOINT, user_data)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn("password", response.data)
self.assertIsInstance(response.data["password"], list)
self.assertTrue(
any(
"too short" in message.lower() for message in response.data["password"]
),
)
def test_update_profile_placeholder_password_skips_validation(self):
"""
GIVEN:
- Configured user with existing password
WHEN:
- API call is made with the obfuscated placeholder password value
THEN:
- Profile is updated without changing the password or running validators
"""
original_password = "orig-pass-12345"
self.user.set_password(original_password)
self.user.save()
user_data = {
"email": "new@email.com",
"password": "*" * 12, # matches obfuscated value from serializer
"first_name": "new first name",
"last_name": "new last name",
}
response = self.client.patch(self.ENDPOINT, user_data)
self.assertEqual(response.status_code, status.HTTP_200_OK)
user = User.objects.get(username=self.user.username)
self.assertTrue(user.check_password(original_password))
self.assertEqual(user.email, user_data["email"])
self.assertEqual(user.first_name, user_data["first_name"])
self.assertEqual(user.last_name, user_data["last_name"])
def test_update_auth_token(self):
"""
GIVEN:

41
src/documents/tests/test_file_handling.py
View File

@@ -1078,6 +1078,47 @@ class TestFilenameGeneration(DirectoriesMixin, TestCase):
Path("SomeImportantNone/2020-07-25.pdf"),
)
@override_settings(
FILENAME_FORMAT=(
"{% if correspondent == 'none' %}none/{% endif %}"
"{% if correspondent == '-none-' %}dash/{% endif %}"
"{% if not correspondent %}false/{% endif %}"
"{% if correspondent != 'none' %}notnoneyes/{% else %}notnoneno/{% endif %}"
"{{ correspondent or 'missing' }}/{{ title }}"
),
)
def test_placeholder_matches_none_variants_and_false(self):
"""
GIVEN:
- Templates that compare against 'none', '-none-' and rely on truthiness
WHEN:
- A document has or lacks a correspondent
THEN:
- Empty placeholders behave like both strings and evaluate False
"""
doc_without_correspondent = Document.objects.create(
title="does not matter",
mime_type="application/pdf",
checksum="abc",
)
doc_with_correspondent = Document.objects.create(
title="does not matter",
mime_type="application/pdf",
checksum="def",
correspondent=Correspondent.objects.create(name="Acme"),
)
self.assertEqual(
generate_filename(doc_without_correspondent),
Path(
"none/dash/false/notnoneno/missing/does not matter.pdf",
),
)
self.assertEqual(
generate_filename(doc_with_correspondent),
Path("notnoneyes/Acme/does not matter.pdf"),
)
@override_settings(
FILENAME_FORMAT="{created_year_short}/{created_month_name_short}/{created_month_name}/{title}",
)

38
src/documents/views.py
View File

@@ -25,6 +25,7 @@ from django.contrib.auth.decorators import login_required
from django.contrib.auth.models import Group
from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
from django.core.cache import cache
from django.db import connections
from django.db.migrations.loader import MigrationLoader
from django.db.migrations.recorder import MigrationRecorder
@@ -54,7 +55,6 @@ from django.utils.timezone import make_aware
from django.utils.translation import get_language
from django.views import View
from django.views.decorators.cache import cache_control
from django.views.decorators.cache import cache_page
from django.views.decorators.csrf import ensure_csrf_cookie
from django.views.decorators.http import condition
from django.views.decorators.http import last_modified
@@ -114,6 +114,7 @@ from documents.conditionals import thumbnail_last_modified
from documents.data_models import ConsumableDocument
from documents.data_models import DocumentMetadataOverrides
from documents.data_models import DocumentSource
from documents.file_handling import format_filename
from documents.filters import CorrespondentFilterSet
from documents.filters import CustomFieldFilterSet
from documents.filters import DocumentFilterSet
@@ -189,7 +190,6 @@ from documents.tasks import index_optimize
from documents.tasks import sanity_check
from documents.tasks import train_classifier
from documents.tasks import update_document_parent_tags
from documents.templating.filepath import validate_filepath_template_and_render
from documents.utils import get_boolean
from paperless import version
from paperless.celery import app as celery_app
@@ -2463,7 +2463,7 @@ class StoragePathViewSet(ModelViewSet, PermissionsAwareDocumentCountMixin):
document = serializer.validated_data.get("document")
path = serializer.validated_data.get("path")
result = validate_filepath_template_and_render(path, document)
result = format_filename(document, path)
return Response(result)
@@ -2557,7 +2557,6 @@ class UiSettingsView(GenericAPIView):
)
@method_decorator(cache_page(60 * 15), name="dispatch")
@extend_schema_view(
get=extend_schema(
description="Get the current version of the Paperless-NGX server",
@@ -2567,31 +2566,34 @@ class UiSettingsView(GenericAPIView):
),
)
class RemoteVersionView(GenericAPIView):
cache_key = "remote_version_view_latest_release"
def get(self, request, format=None):
remote_version = "0.0.0"
is_greater_than_current = False
current_version = packaging_version.parse(version.__full_version_str__)
try:
resp = httpx.get(
"https://api.github.com/repos/paperless-ngx/paperless-ngx/releases/latest",
headers={"Accept": "application/json"},
)
resp.raise_for_status()
remote_version = cache.get(self.cache_key)
if remote_version is None:
try:
resp = httpx.get(
"https://api.github.com/repos/paperless-ngx/paperless-ngx/releases/latest",
headers={"Accept": "application/json"},
)
resp.raise_for_status()
data = resp.json()
remote_version = data["tag_name"]
# Some early tags used ngx-x.y.z
remote_version = remote_version.removeprefix("ngx-")
except ValueError as e:
logger.debug(f"An error occurred parsing remote version json: {e}")
except httpx.HTTPError as e:
logger.debug(f"An error occurred checking for available updates: {e}")
except httpx.HTTPError as e:
logger.debug(f"An error occurred checking for available updates: {e}")
if remote_version:
cache.set(self.cache_key, remote_version, 60 * 15)
else:
remote_version = "0.0.0"
is_greater_than_current = (
packaging_version.parse(
remote_version,
)
> current_version
packaging_version.parse(remote_version) > current_version
)
return Response(

41
src/paperless/serialisers.py
View File

@@ -9,6 +9,7 @@ from allauth.socialaccount.models import SocialApp
from django.contrib.auth.models import Group
from django.contrib.auth.models import Permission
from django.contrib.auth.models import User
from django.contrib.auth.password_validation import validate_password
from rest_framework import serializers
from rest_framework.authtoken.serializers import AuthTokenSerializer
@@ -19,6 +20,23 @@ from paperless_mail.serialisers import ObfuscatedPasswordField
logger = logging.getLogger("paperless.settings")
class PasswordValidationMixin:
def _has_real_password(self, value: str | None) -> bool:
return bool(value) and value.replace("*", "") != ""
def validate_password(self, value: str) -> str:
if not self._has_real_password(value):
return value
request = self.context.get("request") if hasattr(self, "context") else None
user = self.instance or (
request.user if request and hasattr(request, "user") else None
)
validate_password(value, user) # raise ValidationError if invalid
return value
class PaperlessAuthTokenSerializer(AuthTokenSerializer):
code = serializers.CharField(
label="MFA Code",
@@ -49,7 +67,7 @@ class PaperlessAuthTokenSerializer(AuthTokenSerializer):
return attrs
class UserSerializer(serializers.ModelSerializer):
class UserSerializer(PasswordValidationMixin, serializers.ModelSerializer):
password = ObfuscatedPasswordField(required=False)
user_permissions = serializers.SlugRelatedField(
many=True,
@@ -87,11 +105,11 @@ class UserSerializer(serializers.ModelSerializer):
return obj.get_group_permissions()
def update(self, instance, validated_data):
if "password" in validated_data:
if len(validated_data.get("password").replace("*", "")) > 0:
instance.set_password(validated_data.get("password"))
instance.save()
validated_data.pop("password")
password = validated_data.pop("password", None)
if self._has_real_password(password):
instance.set_password(password)
instance.save()
super().update(instance, validated_data)
return instance
@@ -102,12 +120,7 @@ class UserSerializer(serializers.ModelSerializer):
user_permissions = None
if "user_permissions" in validated_data:
user_permissions = validated_data.pop("user_permissions")
password = None
if (
"password" in validated_data
and len(validated_data.get("password").replace("*", "")) > 0
):
password = validated_data.pop("password")
password = validated_data.pop("password", None)
user = User.objects.create(**validated_data)
# set groups
if groups:
@@ -116,7 +129,7 @@ class UserSerializer(serializers.ModelSerializer):
if user_permissions:
user.user_permissions.set(user_permissions)
# set password
if password:
if self._has_real_password(password):
user.set_password(password)
user.save()
return user
@@ -156,7 +169,7 @@ class SocialAccountSerializer(serializers.ModelSerializer):
return "Unknown App"
class ProfileSerializer(serializers.ModelSerializer):
class ProfileSerializer(PasswordValidationMixin, serializers.ModelSerializer):
email = serializers.EmailField(allow_blank=True, required=False)
password = ObfuscatedPasswordField(required=False, allow_null=False)
auth_token = serializers.SlugRelatedField(read_only=True, slug_field="key")

6
src/paperless/views.py
View File

@@ -199,10 +199,10 @@ class ProfileView(GenericAPIView):
serializer.is_valid(raise_exception=True)
user = self.request.user if hasattr(self.request, "user") else None
if len(serializer.validated_data.get("password").replace("*", "")) > 0:
user.set_password(serializer.validated_data.get("password"))
password = serializer.validated_data.pop("password", None)
if password and password.replace("*", ""):
user.set_password(password)
user.save()
serializer.validated_data.pop("password")
for key, value in serializer.validated_data.items():
setattr(user, key, value)

3
src/paperless_mail/oauth.py
View File

@@ -103,6 +103,9 @@ class PaperlessMailOAuth2Manager:
refresh_token=account.refresh_token,
),
)
if "refresh_token" in result:
# Outlook returns a new refresh token on refresh, Gmail does not
account.refresh_token = result["refresh_token"]
account.password = result["access_token"]
account.expiration = timezone.now() + timedelta(
seconds=result["expires_in"],