paradizelost/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2025-09-30 01:32:43 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev' into feature-ai

Browse Source
This commit is contained in:
shamoon
2025-09-17 16:50:33 -07:00
parent a569b0574d 6231211f9b
commit a78a2349bb
59 changed files with 1872 additions and 674 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
src/documents/admin.py
View File

@@ -1,6 +1,7 @@
from django.conf import settings
from django.contrib import admin
from guardian.admin import GuardedModelAdmin
from treenode.admin import TreeNodeModelAdmin
from documents.models import Correspondent
from documents.models import CustomField
@@ -14,6 +15,7 @@ from documents.models import SavedViewFilterRule
from documents.models import ShareLink
from documents.models import StoragePath
from documents.models import Tag
from documents.tasks import update_document_parent_tags
if settings.AUDIT_LOG_ENABLED:
from auditlog.admin import LogEntryAdmin
@@ -26,12 +28,25 @@ class CorrespondentAdmin(GuardedModelAdmin):
list_editable = ("match", "matching_algorithm")
class TagAdmin(GuardedModelAdmin):
class TagAdmin(GuardedModelAdmin, TreeNodeModelAdmin):
list_display = ("name", "color", "match", "matching_algorithm")
list_filter = ("matching_algorithm",)
list_editable = ("color", "match", "matching_algorithm")
search_fields = ("color", "name")
def save_model(self, request, obj, form, change):
old_parent = None
if change and obj.pk:
tag = Tag.objects.get(pk=obj.pk)
old_parent = tag.get_parent() if tag else None
super().save_model(request, obj, form, change)
# sync parent tags on documents if changed
new_parent = obj.get_parent()
if new_parent and old_parent != new_parent:
update_document_parent_tags(obj, new_parent)
class DocumentTypeAdmin(GuardedModelAdmin):
list_display = ("name", "match", "matching_algorithm")

103
src/documents/bulk_edit.py
View File

@@ -1,7 +1,6 @@
from __future__ import annotations
import hashlib
import itertools
import logging
import tempfile
from pathlib import Path
@@ -13,6 +12,7 @@ from celery import chord
from celery import group
from celery import shared_task
from django.conf import settings
from django.db import transaction
from django.db.models import Q
from django.utils import timezone
@@ -25,6 +25,7 @@ from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import DocumentType
from documents.models import StoragePath
from documents.models import Tag
from documents.permissions import set_permissions_for_object
from documents.plugins.helpers import DocumentsStatusManager
from documents.tasks import bulk_update_documents
@@ -96,31 +97,45 @@ def set_document_type(doc_ids: list[int], document_type: DocumentType) -> Litera
def add_tag(doc_ids: list[int], tag: int) -> Literal["OK"]:
qs = Document.objects.filter(Q(id__in=doc_ids) & ~Q(tags__id=tag)).only("pk")
affected_docs = list(qs.values_list("pk", flat=True))
tag_obj = Tag.objects.get(pk=tag)
tags_to_add = [tag_obj, *tag_obj.get_ancestors()]
DocumentTagRelationship = Document.tags.through
to_create = []
affected_docs: set[int] = set()
DocumentTagRelationship.objects.bulk_create(
[DocumentTagRelationship(document_id=doc, tag_id=tag) for doc in affected_docs],
)
for t in tags_to_add:
qs = Document.objects.filter(Q(id__in=doc_ids) & ~Q(tags__id=t.id)).only("pk")
doc_ids_missing_tag = list(qs.values_list("pk", flat=True))
affected_docs.update(doc_ids_missing_tag)
to_create.extend(
DocumentTagRelationship(document_id=doc, tag_id=t.id)
for doc in doc_ids_missing_tag
)
bulk_update_documents.delay(document_ids=affected_docs)
if to_create:
DocumentTagRelationship.objects.bulk_create(to_create)
if affected_docs:
bulk_update_documents.delay(document_ids=list(affected_docs))
return "OK"
def remove_tag(doc_ids: list[int], tag: int) -> Literal["OK"]:
qs = Document.objects.filter(Q(id__in=doc_ids) & Q(tags__id=tag)).only("pk")
affected_docs = list(qs.values_list("pk", flat=True))
tag_obj = Tag.objects.get(pk=tag)
tag_ids = [tag_obj.id, *tag_obj.get_descendants_pks()]
DocumentTagRelationship = Document.tags.through
qs = DocumentTagRelationship.objects.filter(
document_id__in=doc_ids,
tag_id__in=tag_ids,
)
affected_docs = list(qs.values_list("document_id", flat=True).distinct())
qs.delete()
DocumentTagRelationship.objects.filter(
Q(document_id__in=affected_docs) & Q(tag_id=tag),
).delete()
bulk_update_documents.delay(document_ids=affected_docs)
if affected_docs:
bulk_update_documents.delay(document_ids=affected_docs)
return "OK"
@@ -132,23 +147,57 @@ def modify_tags(
) -> Literal["OK"]:
qs = Document.objects.filter(id__in=doc_ids).only("pk")
affected_docs = list(qs.values_list("pk", flat=True))
DocumentTagRelationship = Document.tags.through
DocumentTagRelationship.objects.filter(
document_id__in=affected_docs,
tag_id__in=remove_tags,
).delete()
# add with all ancestors
expanded_add_tags: set[int] = set()
add_tag_objects = Tag.objects.filter(pk__in=add_tags)
for t in add_tag_objects:
expanded_add_tags.add(int(t.id))
expanded_add_tags.update(int(pk) for pk in t.get_ancestors_pks())
DocumentTagRelationship.objects.bulk_create(
[
DocumentTagRelationship(document_id=doc, tag_id=tag)
for (doc, tag) in itertools.product(affected_docs, add_tags)
],
ignore_conflicts=True,
)
# remove with all descendants
expanded_remove_tags: set[int] = set()
remove_tag_objects = Tag.objects.filter(pk__in=remove_tags)
for t in remove_tag_objects:
expanded_remove_tags.add(int(t.id))
expanded_remove_tags.update(int(pk) for pk in t.get_descendants_pks())
bulk_update_documents.delay(document_ids=affected_docs)
try:
with transaction.atomic():
if expanded_remove_tags:
DocumentTagRelationship.objects.filter(
document_id__in=affected_docs,
tag_id__in=expanded_remove_tags,
).delete()
to_create = []
if expanded_add_tags:
existing_pairs = set(
DocumentTagRelationship.objects.filter(
document_id__in=affected_docs,
tag_id__in=expanded_add_tags,
).values_list("document_id", "tag_id"),
)
to_create = [
DocumentTagRelationship(document_id=doc, tag_id=tag)
for doc in affected_docs
for tag in expanded_add_tags
if (doc, tag) not in existing_pairs
]
if to_create:
DocumentTagRelationship.objects.bulk_create(
to_create,
ignore_conflicts=True,
)
if affected_docs:
bulk_update_documents.delay(document_ids=affected_docs)
except Exception as e:
logger.error(f"Error modifying tags: {e}")
return "ERROR"
return "OK"

2
src/documents/consumer.py
View File

@@ -689,7 +689,7 @@ class ConsumerPlugin(
if self.metadata.tag_ids:
for tag_id in self.metadata.tag_ids:
document.tags.add(Tag.objects.get(pk=tag_id))
document.add_nested_tags([Tag.objects.get(pk=tag_id)])
if self.metadata.storage_path_id:
document.storage_path = StoragePath.objects.get(

2
src/documents/migrations/0003_sender.py
View File

@@ -17,7 +17,7 @@ def move_sender_strings_to_sender_model(apps, schema_editor):
if document.sender:
(
DOCUMENT_SENDER_MAP[document.pk],
created,
_,
) = sender_model.objects.get_or_create(
name=document.sender,
defaults={"slug": slugify(document.sender)},

159
src/documents/migrations/1071_tag_tn_ancestors_count_tag_tn_ancestors_pks_and_more.py Normal file
View File

@@ -0,0 +1,159 @@
# Generated by Django 5.2.6 on 2025-09-12 18:42
import django.core.validators
import django.db.models.deletion
from django.db import migrations
from django.db import models
class Migration(migrations.Migration):
dependencies = [
("documents", "1070_customfieldinstance_value_long_text_and_more"),
]
operations = [
migrations.AddField(
model_name="tag",
name="tn_ancestors_count",
field=models.PositiveIntegerField(
default=0,
editable=False,
verbose_name="Ancestors count",
),
),
migrations.AddField(
model_name="tag",
name="tn_ancestors_pks",
field=models.TextField(
blank=True,
default="",
editable=False,
verbose_name="Ancestors pks",
),
),
migrations.AddField(
model_name="tag",
name="tn_children_count",
field=models.PositiveIntegerField(
default=0,
editable=False,
verbose_name="Children count",
),
),
migrations.AddField(
model_name="tag",
name="tn_children_pks",
field=models.TextField(
blank=True,
default="",
editable=False,
verbose_name="Children pks",
),
),
migrations.AddField(
model_name="tag",
name="tn_depth",
field=models.PositiveIntegerField(
default=0,
editable=False,
validators=[
django.core.validators.MinValueValidator(0),
django.core.validators.MaxValueValidator(10),
],
verbose_name="Depth",
),
),
migrations.AddField(
model_name="tag",
name="tn_descendants_count",
field=models.PositiveIntegerField(
default=0,
editable=False,
verbose_name="Descendants count",
),
),
migrations.AddField(
model_name="tag",
name="tn_descendants_pks",
field=models.TextField(
blank=True,
default="",
editable=False,
verbose_name="Descendants pks",
),
),
migrations.AddField(
model_name="tag",
name="tn_index",
field=models.PositiveIntegerField(
default=0,
editable=False,
verbose_name="Index",
),
),
migrations.AddField(
model_name="tag",
name="tn_level",
field=models.PositiveIntegerField(
default=1,
editable=False,
validators=[
django.core.validators.MinValueValidator(1),
django.core.validators.MaxValueValidator(10),
],
verbose_name="Level",
),
),
migrations.AddField(
model_name="tag",
name="tn_order",
field=models.PositiveIntegerField(
default=0,
editable=False,
verbose_name="Order",
),
),
migrations.AddField(
model_name="tag",
name="tn_parent",
field=models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.CASCADE,
related_name="tn_children",
to="documents.tag",
verbose_name="Parent",
),
),
migrations.AddField(
model_name="tag",
name="tn_priority",
field=models.PositiveIntegerField(
default=0,
validators=[
django.core.validators.MinValueValidator(0),
django.core.validators.MaxValueValidator(9999999999),
],
verbose_name="Priority",
),
),
migrations.AddField(
model_name="tag",
name="tn_siblings_count",
field=models.PositiveIntegerField(
default=0,
editable=False,
verbose_name="Siblings count",
),
),
migrations.AddField(
model_name="tag",
name="tn_siblings_pks",
field=models.TextField(
blank=True,
default="",
editable=False,
verbose_name="Siblings pks",
),
),
]

37
src/documents/models.py
View File

@@ -7,12 +7,14 @@ from celery import states
from django.conf import settings
from django.contrib.auth.models import Group
from django.contrib.auth.models import User
from django.core.exceptions import ValidationError
from django.core.validators import MaxValueValidator
from django.core.validators import MinValueValidator
from django.db import models
from django.utils import timezone
from django.utils.translation import gettext_lazy as _
from multiselectfield import MultiSelectField
from treenode.models import TreeNodeModel
if settings.AUDIT_LOG_ENABLED:
from auditlog.registry import auditlog
@@ -96,8 +98,10 @@ class Correspondent(MatchingModel):
verbose_name_plural = _("correspondents")
class Tag(MatchingModel):
class Tag(MatchingModel, TreeNodeModel):
color = models.CharField(_("color"), max_length=7, default="#a6cee3")
# Maximum allowed nesting depth for tags (root = 1, max depth = 5)
MAX_NESTING_DEPTH: Final[int] = 5
is_inbox_tag = models.BooleanField(
_("is inbox tag"),
@@ -108,10 +112,30 @@ class Tag(MatchingModel):
),
)
class Meta(MatchingModel.Meta):
class Meta(MatchingModel.Meta, TreeNodeModel.Meta):
verbose_name = _("tag")
verbose_name_plural = _("tags")
def clean(self):
# Prevent self-parenting and assigning a descendant as parent
parent = self.get_parent()
if parent == self:
raise ValidationError({"parent": _("Cannot set itself as parent.")})
if parent and self.pk is not None and self.is_ancestor_of(parent):
raise ValidationError({"parent": _("Cannot set parent to a descendant.")})
# Enforce maximum nesting depth
new_parent_depth = 0
if parent:
new_parent_depth = parent.get_ancestors_count() + 1
height = 0 if self.pk is None else self.get_depth()
deepest_new_depth = (new_parent_depth + 1) + height
if deepest_new_depth > self.MAX_NESTING_DEPTH:
raise ValidationError(_("Maximum nesting depth exceeded."))
return super().clean()
class DocumentType(MatchingModel):
class Meta(MatchingModel.Meta):
@@ -398,6 +422,15 @@ class Document(SoftDeleteModel, ModelWithOwner):
def created_date(self):
return self.created
def add_nested_tags(self, tags):
tag_ids = set()
for tag in tags:
tag_ids.add(tag.id)
tag_ids.update(tag.get_ancestors_pks())
tags_to_add = self.tags.model.objects.filter(id__in=tag_ids)
self.tags.add(*tags_to_add)
class SavedView(ModelWithOwner):
class DisplayMode(models.TextChoices):

141
src/documents/serialisers.py
View File

@@ -13,6 +13,7 @@ from django.conf import settings
from django.contrib.auth.models import Group
from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
from django.core.exceptions import ValidationError
from django.core.validators import DecimalValidator
from django.core.validators import MaxLengthValidator
from django.core.validators import RegexValidator
@@ -540,6 +541,32 @@ class TagSerializer(MatchingModelSerializer, OwnedObjectSerializer):
text_color = serializers.SerializerMethodField()
# map to treenode's tn_parent
parent = serializers.PrimaryKeyRelatedField(
queryset=Tag.objects.all(),
allow_null=True,
required=False,
source="tn_parent",
)
@extend_schema_field(
field=serializers.ListSerializer(
child=serializers.PrimaryKeyRelatedField(
queryset=Tag.objects.all(),
),
),
)
def get_children(self, obj):
serializer = TagSerializer(
obj.get_children(),
many=True,
context=self.context,
)
return serializer.data
# children as nested Tag objects
children = serializers.SerializerMethodField()
class Meta:
model = Tag
fields = (
@@ -557,6 +584,8 @@ class TagSerializer(MatchingModelSerializer, OwnedObjectSerializer):
"permissions",
"user_can_change",
"set_permissions",
"parent",
"children",
)
def validate_color(self, color):
@@ -565,6 +594,36 @@ class TagSerializer(MatchingModelSerializer, OwnedObjectSerializer):
raise serializers.ValidationError(_("Invalid color."))
return color
def validate(self, attrs):
# Validate when changing parent
parent = attrs.get(
"tn_parent",
self.instance.get_parent() if self.instance else None,
)
if self.instance:
# Temporarily set parent on the instance if updating and use model clean()
original_parent = self.instance.get_parent()
try:
# Temporarily set tn_parent in-memory to validate clean()
self.instance.tn_parent = parent
self.instance.clean()
except ValidationError as e:
logger.debug("Tag parent validation failed: %s", e)
raise e
finally:
self.instance.tn_parent = original_parent
else:
# For new instances, create a transient Tag and validate
temp = Tag(tn_parent=parent)
try:
temp.clean()
except ValidationError as e:
logger.debug("Tag parent validation failed: %s", e)
raise serializers.ValidationError({"parent": _("Invalid parent tag.")})
return super().validate(attrs)
class CorrespondentField(serializers.PrimaryKeyRelatedField):
def get_queryset(self):
@@ -1028,6 +1087,28 @@ class DocumentSerializer(
custom_field_instance.field,
doc_id,
)
if "tags" in validated_data:
# Respect tag hierarchy on updates:
# - Adding a child adds its ancestors
# - Removing a parent removes all its descendants
prev_tags = set(instance.tags.all())
requested_tags = set(validated_data["tags"])
# Tags being removed in this update and all descendants
removed_tags = prev_tags - requested_tags
blocked_tags = set(removed_tags)
for t in removed_tags:
blocked_tags.update(t.get_descendants())
# Add all parent tags
final_tags = set(requested_tags)
for t in requested_tags:
final_tags.update(t.get_ancestors())
# Drop removed parents and their descendants
final_tags.difference_update(blocked_tags)
validated_data["tags"] = list(final_tags)
if validated_data.get("remove_inbox_tags"):
tag_ids_being_added = (
[
@@ -1668,9 +1749,8 @@ class PostDocumentSerializer(serializers.Serializer):
max_value=Document.ARCHIVE_SERIAL_NUMBER_MAX,
)
custom_fields = serializers.PrimaryKeyRelatedField(
many=True,
queryset=CustomField.objects.all(),
# Accept either a list of custom field ids or a dict mapping id -> value
custom_fields = serializers.JSONField(
label="Custom fields",
write_only=True,
required=False,
@@ -1727,11 +1807,60 @@ class PostDocumentSerializer(serializers.Serializer):
return None
def validate_custom_fields(self, custom_fields):
if custom_fields:
return [custom_field.id for custom_field in custom_fields]
else:
if not custom_fields:
return None
# Normalize single values to a list
if isinstance(custom_fields, int):
custom_fields = [custom_fields]
if isinstance(custom_fields, dict):
custom_field_serializer = CustomFieldInstanceSerializer()
normalized = {}
for field_id, value in custom_fields.items():
try:
field_id_int = int(field_id)
except (TypeError, ValueError):
raise serializers.ValidationError(
_("Custom field id must be an integer: %(id)s")
% {"id": field_id},
)
try:
field = CustomField.objects.get(id=field_id_int)
except CustomField.DoesNotExist:
raise serializers.ValidationError(
_("Custom field with id %(id)s does not exist")
% {"id": field_id_int},
)
custom_field_serializer.validate(
{
"field": field,
"value": value,
},
)
normalized[field_id_int] = value
return normalized
elif isinstance(custom_fields, list):
try:
ids = [int(i) for i in custom_fields]
except (TypeError, ValueError):
raise serializers.ValidationError(
_(
"Custom fields must be a list of integers or an object mapping ids to values.",
),
)
if CustomField.objects.filter(id__in=ids).count() != len(set(ids)):
raise serializers.ValidationError(
_("Some custom fields don't exist or were specified twice."),
)
return ids
raise serializers.ValidationError(
_(
"Custom fields must be a list of integers or an object mapping ids to values.",
),
)
# custom_fields_w_values handled via validate_custom_fields
def validate_created(self, created):
# support datetime format for created for backwards compatibility
if isinstance(created, datetime):

30
src/documents/signals/handlers.py
View File

@@ -73,7 +73,7 @@ def add_inbox_tags(sender, document: Document, logging_group=None, **kwargs):
else:
tags = Tag.objects.all()
inbox_tags = tags.filter(is_inbox_tag=True)
document.tags.add(*inbox_tags)
document.add_nested_tags(inbox_tags)
def _suggestion_printer(
@@ -262,7 +262,7 @@ def set_tags(
extra={"group": logging_group},
)
document.tags.add(*relevant_tags)
document.add_nested_tags(relevant_tags)
def set_storage_path(
@@ -778,14 +778,17 @@ def run_workflows(
def assignment_action():
if action.assign_tags.exists():
tag_ids_to_add: set[int] = set()
for tag in action.assign_tags.all():
tag_ids_to_add.add(tag.pk)
tag_ids_to_add.update(int(pk) for pk in tag.get_ancestors_pks())
if not use_overrides:
doc_tag_ids.extend(action.assign_tags.values_list("pk", flat=True))
doc_tag_ids[:] = list(set(doc_tag_ids) | tag_ids_to_add)
else:
if overrides.tag_ids is None:
overrides.tag_ids = []
overrides.tag_ids.extend(
action.assign_tags.values_list("pk", flat=True),
)
overrides.tag_ids = list(set(overrides.tag_ids) | tag_ids_to_add)
if action.assign_correspondent:
if not use_overrides:
@@ -928,14 +931,17 @@ def run_workflows(
else:
overrides.tag_ids = None
else:
tag_ids_to_remove: set[int] = set()
for tag in action.remove_tags.all():
tag_ids_to_remove.add(tag.pk)
tag_ids_to_remove.update(int(pk) for pk in tag.get_descendants_pks())
if not use_overrides:
for tag in action.remove_tags.filter(
pk__in=document.tags.values_list("pk", flat=True),
):
doc_tag_ids.remove(tag.pk)
doc_tag_ids[:] = [t for t in doc_tag_ids if t not in tag_ids_to_remove]
elif overrides.tag_ids:
for tag in action.remove_tags.filter(pk__in=overrides.tag_ids):
overrides.tag_ids.remove(tag.pk)
overrides.tag_ids = [
t for t in overrides.tag_ids if t not in tag_ids_to_remove
]
if not use_overrides and (
action.remove_all_correspondents

48
src/documents/tasks.py
View File

@@ -532,6 +532,54 @@ def check_scheduled_workflows():
)
def update_document_parent_tags(tag: Tag, new_parent: Tag) -> None:
"""
When a tag's parent changes, ensure all documents containing the tag also have
the parent tag (and its ancestors) applied.
"""
doc_tag_relationship = Document.tags.through
doc_ids: list[int] = list(
Document.objects.filter(tags=tag).values_list("pk", flat=True),
)
if not doc_ids:
return
parent_ids = [new_parent.id, *new_parent.get_ancestors_pks()]
parent_ids = list(dict.fromkeys(parent_ids))
existing_pairs = set(
doc_tag_relationship.objects.filter(
document_id__in=doc_ids,
tag_id__in=parent_ids,
).values_list("document_id", "tag_id"),
)
to_create: list = []
affected: set[int] = set()
for doc_id in doc_ids:
for parent_id in parent_ids:
if (doc_id, parent_id) in existing_pairs:
continue
to_create.append(
doc_tag_relationship(document_id=doc_id, tag_id=parent_id),
)
affected.add(doc_id)
if to_create:
doc_tag_relationship.objects.bulk_create(
to_create,
ignore_conflicts=True,
)
if affected:
bulk_update_documents.delay(document_ids=list(affected))
@shared_task
def llmindex_index(
*,

21
src/documents/tests/test_admin.py
View File

@@ -1,4 +1,5 @@
import types
from unittest.mock import patch
from django.contrib.admin.sites import AdminSite
from django.contrib.auth.models import User
@@ -7,7 +8,9 @@ from django.utils import timezone
from documents import index
from documents.admin import DocumentAdmin
from documents.admin import TagAdmin
from documents.models import Document
from documents.models import Tag
from documents.tests.utils import DirectoriesMixin
from paperless.admin import PaperlessUserAdmin
@@ -70,6 +73,24 @@ class TestDocumentAdmin(DirectoriesMixin, TestCase):
self.assertEqual(self.doc_admin.created_(doc), "2020-04-12")
class TestTagAdmin(DirectoriesMixin, TestCase):
def setUp(self) -> None:
super().setUp()
self.tag_admin = TagAdmin(model=Tag, admin_site=AdminSite())
@patch("documents.tasks.bulk_update_documents")
def test_parent_tags_get_added(self, mock_bulk_update):
document = Document.objects.create(title="test")
parent = Tag.objects.create(name="parent")
child = Tag.objects.create(name="child")
document.tags.add(child)
child.tn_parent = parent
self.tag_admin.save_model(None, child, None, change=True)
document.refresh_from_db()
self.assertIn(parent, document.tags.all())
class TestPaperlessAdmin(DirectoriesMixin, TestCase):
def setUp(self) -> None:
super().setUp()

4
src/documents/tests/test_api_bulk_edit.py
View File

@@ -839,7 +839,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_200_OK)
m.assert_called()
args, kwargs = m.call_args
_, kwargs = m.call_args
self.assertEqual(kwargs["merge"], False)
response = self.client.post(
@@ -857,7 +857,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_200_OK)
m.assert_called()
args, kwargs = m.call_args
_, kwargs = m.call_args
self.assertEqual(kwargs["merge"], True)
@mock.patch("documents.serialisers.bulk_edit.set_storage_path")

85
src/documents/tests/test_api_documents.py
View File

@@ -1,4 +1,5 @@
import datetime
import json
import shutil
import tempfile
import uuid
@@ -1528,7 +1529,7 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
input_doc, overrides = self.get_last_consume_delay_call_args()
new_overrides, msg = run_workflows(
new_overrides, _ = run_workflows(
trigger_type=WorkflowTrigger.WorkflowTriggerType.CONSUMPTION,
document=input_doc,
logging_group=None,
@@ -1537,6 +1538,86 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
overrides.update(new_overrides)
self.assertEqual(overrides.custom_fields, {cf.id: None, cf2.id: 123})
def test_upload_with_custom_field_values(self):
"""
GIVEN: A document with a source file
WHEN: Upload the document with custom fields and values
THEN: Metadata is set correctly
"""
self.consume_file_mock.return_value = celery.result.AsyncResult(
id=str(uuid.uuid4()),
)
cf_string = CustomField.objects.create(
name="stringfield",
data_type=CustomField.FieldDataType.STRING,
)
cf_int = CustomField.objects.create(
name="intfield",
data_type=CustomField.FieldDataType.INT,
)
with (Path(__file__).parent / "samples" / "simple.pdf").open("rb") as f:
response = self.client.post(
"/api/documents/post_document/",
{
"document": f,
"custom_fields": json.dumps(
{
str(cf_string.id): "a string",
str(cf_int.id): 123,
},
),
},
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.consume_file_mock.assert_called_once()
input_doc, overrides = self.get_last_consume_delay_call_args()
self.assertEqual(input_doc.original_file.name, "simple.pdf")
self.assertEqual(overrides.filename, "simple.pdf")
self.assertEqual(
overrides.custom_fields,
{cf_string.id: "a string", cf_int.id: 123},
)
def test_upload_with_custom_fields_errors(self):
"""
GIVEN: A document with a source file
WHEN: Upload the document with invalid custom fields payloads
THEN: The upload is rejected
"""
self.consume_file_mock.return_value = celery.result.AsyncResult(
id=str(uuid.uuid4()),
)
error_payloads = [
# Non-integer key in mapping
{"custom_fields": json.dumps({"abc": "a string"})},
# List with non-integer entry
{"custom_fields": json.dumps(["abc"])},
# Nonexistent id in mapping
{"custom_fields": json.dumps({99999999: "a string"})},
# Nonexistent id in list
{"custom_fields": json.dumps([99999999])},
# Invalid type (JSON string, not list/dict/int)
{"custom_fields": json.dumps("not-a-supported-structure")},
]
for payload in error_payloads:
with (Path(__file__).parent / "samples" / "simple.pdf").open("rb") as f:
data = {"document": f, **payload}
response = self.client.post(
"/api/documents/post_document/",
data,
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.consume_file_mock.assert_not_called()
def test_upload_with_webui_source(self):
"""
GIVEN: A document with a source file
@@ -1557,7 +1638,7 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
self.consume_file_mock.assert_called_once()
input_doc, overrides = self.get_last_consume_delay_call_args()
input_doc, _ = self.get_last_consume_delay_call_args()
self.assertEqual(input_doc.source, WorkflowTrigger.DocumentSourceChoices.WEB_UI)

22
src/documents/tests/test_bulk_edit.py
View File

@@ -74,7 +74,7 @@ class TestBulkEdit(DirectoriesMixin, TestCase):
)
self.assertEqual(Document.objects.filter(correspondent=self.c2).count(), 3)
self.async_task.assert_called_once()
args, kwargs = self.async_task.call_args
_, kwargs = self.async_task.call_args
self.assertCountEqual(kwargs["document_ids"], [self.doc1.id, self.doc2.id])
def test_unset_correspondent(self):
@@ -82,7 +82,7 @@ class TestBulkEdit(DirectoriesMixin, TestCase):
bulk_edit.set_correspondent([self.doc1.id, self.doc2.id, self.doc3.id], None)
self.assertEqual(Document.objects.filter(correspondent=self.c2).count(), 0)
self.async_task.assert_called_once()
args, kwargs = self.async_task.call_args
_, kwargs = self.async_task.call_args
self.assertCountEqual(kwargs["document_ids"], [self.doc2.id, self.doc3.id])
def test_set_document_type(self):
@@ -93,7 +93,7 @@ class TestBulkEdit(DirectoriesMixin, TestCase):
)
self.assertEqual(Document.objects.filter(document_type=self.dt2).count(), 3)
self.async_task.assert_called_once()
args, kwargs = self.async_task.call_args
_, kwargs = self.async_task.call_args
self.assertCountEqual(kwargs["document_ids"], [self.doc1.id, self.doc2.id])
def test_unset_document_type(self):
@@ -101,7 +101,7 @@ class TestBulkEdit(DirectoriesMixin, TestCase):
bulk_edit.set_document_type([self.doc1.id, self.doc2.id, self.doc3.id], None)
self.assertEqual(Document.objects.filter(document_type=self.dt2).count(), 0)
self.async_task.assert_called_once()
args, kwargs = self.async_task.call_args
_, kwargs = self.async_task.call_args
self.assertCountEqual(kwargs["document_ids"], [self.doc2.id, self.doc3.id])
def test_set_document_storage_path(self):
@@ -123,7 +123,7 @@ class TestBulkEdit(DirectoriesMixin, TestCase):
self.assertEqual(Document.objects.filter(storage_path=None).count(), 4)
self.async_task.assert_called_once()
args, kwargs = self.async_task.call_args
_, kwargs = self.async_task.call_args
self.assertCountEqual(kwargs["document_ids"], [self.doc1.id])
@@ -154,7 +154,7 @@ class TestBulkEdit(DirectoriesMixin, TestCase):
self.assertEqual(Document.objects.filter(storage_path=None).count(), 5)
self.async_task.assert_called()
args, kwargs = self.async_task.call_args
_, kwargs = self.async_task.call_args
self.assertCountEqual(kwargs["document_ids"], [self.doc1.id])
@@ -166,7 +166,7 @@ class TestBulkEdit(DirectoriesMixin, TestCase):
)
self.assertEqual(Document.objects.filter(tags__id=self.t1.id).count(), 4)
self.async_task.assert_called_once()
args, kwargs = self.async_task.call_args
_, kwargs = self.async_task.call_args
self.assertCountEqual(kwargs["document_ids"], [self.doc1.id, self.doc3.id])
def test_remove_tag(self):
@@ -174,7 +174,7 @@ class TestBulkEdit(DirectoriesMixin, TestCase):
bulk_edit.remove_tag([self.doc1.id, self.doc3.id, self.doc4.id], self.t1.id)
self.assertEqual(Document.objects.filter(tags__id=self.t1.id).count(), 1)
self.async_task.assert_called_once()
args, kwargs = self.async_task.call_args
_, kwargs = self.async_task.call_args
self.assertCountEqual(kwargs["document_ids"], [self.doc4.id])
def test_modify_tags(self):
@@ -191,7 +191,7 @@ class TestBulkEdit(DirectoriesMixin, TestCase):
self.assertCountEqual(list(self.doc3.tags.all()), [self.t2, tag_unrelated])
self.async_task.assert_called_once()
args, kwargs = self.async_task.call_args
_, kwargs = self.async_task.call_args
# TODO: doc3 should not be affected, but the query for that is rather complicated
self.assertCountEqual(kwargs["document_ids"], [self.doc2.id, self.doc3.id])
@@ -248,7 +248,7 @@ class TestBulkEdit(DirectoriesMixin, TestCase):
)
self.async_task.assert_called_once()
args, kwargs = self.async_task.call_args
_, kwargs = self.async_task.call_args
self.assertCountEqual(kwargs["document_ids"], [self.doc1.id, self.doc2.id])
def test_modify_custom_fields_with_values(self):
@@ -325,7 +325,7 @@ class TestBulkEdit(DirectoriesMixin, TestCase):
)
self.async_task.assert_called_once()
args, kwargs = self.async_task.call_args
_, kwargs = self.async_task.call_args
self.assertCountEqual(kwargs["document_ids"], [self.doc1.id, self.doc2.id])
# removal of document link cf, should also remove symmetric link

16
src/documents/tests/test_management_retagger.py
View File

@@ -123,14 +123,14 @@ class TestRetagger(DirectoriesMixin, TestCase):
def test_add_type(self):
call_command("document_retagger", "--document_type")
d_first, d_second, d_unrelated, d_auto = self.get_updated_docs()
d_first, d_second, _, _ = self.get_updated_docs()
self.assertEqual(d_first.document_type, self.doctype_first)
self.assertEqual(d_second.document_type, self.doctype_second)
def test_add_correspondent(self):
call_command("document_retagger", "--correspondent")
d_first, d_second, d_unrelated, d_auto = self.get_updated_docs()
d_first, d_second, _, _ = self.get_updated_docs()
self.assertEqual(d_first.correspondent, self.correspondent_first)
self.assertEqual(d_second.correspondent, self.correspondent_second)
@@ -160,7 +160,7 @@ class TestRetagger(DirectoriesMixin, TestCase):
def test_add_tags_suggest(self):
call_command("document_retagger", "--tags", "--suggest")
d_first, d_second, d_unrelated, d_auto = self.get_updated_docs()
d_first, d_second, _, d_auto = self.get_updated_docs()
self.assertEqual(d_first.tags.count(), 0)
self.assertEqual(d_second.tags.count(), 0)
@@ -168,14 +168,14 @@ class TestRetagger(DirectoriesMixin, TestCase):
def test_add_type_suggest(self):
call_command("document_retagger", "--document_type", "--suggest")
d_first, d_second, d_unrelated, d_auto = self.get_updated_docs()
d_first, d_second, _, _ = self.get_updated_docs()
self.assertIsNone(d_first.document_type)
self.assertIsNone(d_second.document_type)
def test_add_correspondent_suggest(self):
call_command("document_retagger", "--correspondent", "--suggest")
d_first, d_second, d_unrelated, d_auto = self.get_updated_docs()
d_first, d_second, _, _ = self.get_updated_docs()
self.assertIsNone(d_first.correspondent)
self.assertIsNone(d_second.correspondent)
@@ -187,7 +187,7 @@ class TestRetagger(DirectoriesMixin, TestCase):
"--suggest",
"--base-url=http://localhost",
)
d_first, d_second, d_unrelated, d_auto = self.get_updated_docs()
d_first, d_second, _, d_auto = self.get_updated_docs()
self.assertEqual(d_first.tags.count(), 0)
self.assertEqual(d_second.tags.count(), 0)
@@ -200,7 +200,7 @@ class TestRetagger(DirectoriesMixin, TestCase):
"--suggest",
"--base-url=http://localhost",
)
d_first, d_second, d_unrelated, d_auto = self.get_updated_docs()
d_first, d_second, _, _ = self.get_updated_docs()
self.assertIsNone(d_first.document_type)
self.assertIsNone(d_second.document_type)
@@ -212,7 +212,7 @@ class TestRetagger(DirectoriesMixin, TestCase):
"--suggest",
"--base-url=http://localhost",
)
d_first, d_second, d_unrelated, d_auto = self.get_updated_docs()
d_first, d_second, _, _ = self.get_updated_docs()
self.assertIsNone(d_first.correspondent)
self.assertIsNone(d_second.correspondent)

3
src/documents/tests/test_migration_archive_files.py
View File

@@ -4,6 +4,7 @@ import shutil
from pathlib import Path
from unittest import mock
import pytest
from django.conf import settings
from django.test import override_settings
@@ -281,6 +282,7 @@ class TestMigrateArchiveFilesErrors(DirectoriesMixin, TestMigrations):
migrate_to = "1012_fix_archive_files"
auto_migrate = False
@pytest.mark.skip(reason="Fails with migration tearDown util. Needs investigation.")
def test_archive_missing(self):
Document = self.apps.get_model("documents", "Document")
@@ -300,6 +302,7 @@ class TestMigrateArchiveFilesErrors(DirectoriesMixin, TestMigrations):
self.performMigration,
)
@pytest.mark.skip(reason="Fails with migration tearDown util. Needs investigation.")
def test_parser_missing(self):
Document = self.apps.get_model("documents", "Document")

205
src/documents/tests/test_tag_hierarchy.py Normal file
View File

@@ -0,0 +1,205 @@
from unittest import mock
from django.contrib.auth.models import User
from rest_framework.test import APITestCase
from documents import bulk_edit
from documents.models import Document
from documents.models import Tag
from documents.models import Workflow
from documents.models import WorkflowAction
from documents.models import WorkflowTrigger
from documents.signals.handlers import run_workflows
class TestTagHierarchy(APITestCase):
def setUp(self):
self.user = User.objects.create_superuser(username="admin")
self.client.force_authenticate(user=self.user)
self.parent = Tag.objects.create(name="Parent")
self.child = Tag.objects.create(name="Child", tn_parent=self.parent)
patcher = mock.patch("documents.bulk_edit.bulk_update_documents.delay")
self.async_task = patcher.start()
self.addCleanup(patcher.stop)
self.document = Document.objects.create(
title="doc",
content="",
checksum="1",
mime_type="application/pdf",
)
def test_document_api_add_child_adds_parent(self):
self.client.patch(
f"/api/documents/{self.document.pk}/",
{"tags": [self.child.pk]},
format="json",
)
self.document.refresh_from_db()
tags = set(self.document.tags.values_list("pk", flat=True))
assert tags == {self.parent.pk, self.child.pk}
def test_document_api_remove_parent_removes_children(self):
self.document.add_nested_tags([self.parent, self.child])
self.client.patch(
f"/api/documents/{self.document.pk}/",
{"tags": [self.child.pk]},
format="json",
)
self.document.refresh_from_db()
assert self.document.tags.count() == 0
def test_document_api_remove_parent_removes_child(self):
self.document.add_nested_tags([self.child])
self.client.patch(
f"/api/documents/{self.document.pk}/",
{"tags": []},
format="json",
)
self.document.refresh_from_db()
assert self.document.tags.count() == 0
def test_bulk_edit_respects_hierarchy(self):
bulk_edit.add_tag([self.document.pk], self.child.pk)
self.document.refresh_from_db()
tags = set(self.document.tags.values_list("pk", flat=True))
assert tags == {self.parent.pk, self.child.pk}
bulk_edit.remove_tag([self.document.pk], self.parent.pk)
self.document.refresh_from_db()
assert self.document.tags.count() == 0
bulk_edit.modify_tags([self.document.pk], [self.child.pk], [])
self.document.refresh_from_db()
tags = set(self.document.tags.values_list("pk", flat=True))
assert tags == {self.parent.pk, self.child.pk}
bulk_edit.modify_tags([self.document.pk], [], [self.parent.pk])
self.document.refresh_from_db()
assert self.document.tags.count() == 0
def test_workflow_actions(self):
workflow = Workflow.objects.create(name="wf", order=0)
trigger = WorkflowTrigger.objects.create(
type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED,
)
assign_action = WorkflowAction.objects.create()
assign_action.assign_tags.add(self.child)
workflow.triggers.add(trigger)
workflow.actions.add(assign_action)
run_workflows(trigger.type, self.document)
self.document.refresh_from_db()
tags = set(self.document.tags.values_list("pk", flat=True))
assert tags == {self.parent.pk, self.child.pk}
# removal
removal_action = WorkflowAction.objects.create(
type=WorkflowAction.WorkflowActionType.REMOVAL,
)
removal_action.remove_tags.add(self.parent)
workflow.actions.clear()
workflow.actions.add(removal_action)
run_workflows(trigger.type, self.document)
self.document.refresh_from_db()
assert self.document.tags.count() == 0
def test_tag_view_parent_update_adds_parent_to_docs(self):
orphan = Tag.objects.create(name="Orphan")
self.document.tags.add(orphan)
self.client.patch(
f"/api/tags/{orphan.pk}/",
{"parent": self.parent.pk},
format="json",
)
self.document.refresh_from_db()
tags = set(self.document.tags.values_list("pk", flat=True))
assert tags == {self.parent.pk, orphan.pk}
def test_cannot_set_parent_to_self(self):
tag = Tag.objects.create(name="Selfie")
resp = self.client.patch(
f"/api/tags/{tag.pk}/",
{"parent": tag.pk},
format="json",
)
assert resp.status_code == 400
assert "Cannot set itself as parent" in str(resp.data["parent"])
def test_cannot_set_parent_to_descendant(self):
a = Tag.objects.create(name="A")
b = Tag.objects.create(name="B", tn_parent=a)
c = Tag.objects.create(name="C", tn_parent=b)
# Attempt to set A's parent to C (descendant) should fail
resp = self.client.patch(
f"/api/tags/{a.pk}/",
{"parent": c.pk},
format="json",
)
assert resp.status_code == 400
assert "Cannot set parent to a descendant" in str(resp.data["parent"])
def test_max_depth_on_create(self):
a = Tag.objects.create(name="A1")
b = Tag.objects.create(name="B1", tn_parent=a)
c = Tag.objects.create(name="C1", tn_parent=b)
d = Tag.objects.create(name="D1", tn_parent=c)
# Creating E under D yields depth 5: allowed
resp_ok = self.client.post(
"/api/tags/",
{"name": "E1", "parent": d.pk},
format="json",
)
assert resp_ok.status_code in (200, 201)
e_id = (
resp_ok.data["id"] if resp_ok.status_code == 201 else resp_ok.data.get("id")
)
assert e_id is not None
# Creating F under E would yield depth 6: rejected
resp_fail = self.client.post(
"/api/tags/",
{"name": "F1", "parent": e_id},
format="json",
)
assert resp_fail.status_code == 400
assert "parent" in resp_fail.data
assert "Invalid" in str(resp_fail.data["parent"])
def test_max_depth_on_move_subtree(self):
a = Tag.objects.create(name="A2")
b = Tag.objects.create(name="B2", tn_parent=a)
c = Tag.objects.create(name="C2", tn_parent=b)
d = Tag.objects.create(name="D2", tn_parent=c)
x = Tag.objects.create(name="X2")
y = Tag.objects.create(name="Y2", tn_parent=x)
assert y.parent_pk == x.pk
# Moving X under D would make deepest node Y exceed depth 5 -> reject
resp_fail = self.client.patch(
f"/api/tags/{x.pk}/",
{"parent": d.pk},
format="json",
)
assert resp_fail.status_code == 400
assert "Maximum nesting depth exceeded" in str(
resp_fail.data["non_field_errors"],
)
# Moving X under C (depth 3) should be allowed (deepest becomes 5)
resp_ok = self.client.patch(
f"/api/tags/{x.pk}/",
{"parent": c.pk},
format="json",
)
assert resp_ok.status_code in (200, 202)
x.refresh_from_db()
assert x.parent_pk == c.id

13
src/documents/tests/utils.py
View File

@@ -327,6 +327,19 @@ class TestMigrations(TransactionTestCase):
def setUpBeforeMigration(self, apps):
pass
def tearDown(self):
"""
Ensure the database schema is restored to the latest migration after
each migration test, so subsequent tests run against HEAD.
"""
try:
executor = MigrationExecutor(connection)
executor.loader.build_graph()
targets = executor.loader.graph.leaf_nodes()
executor.migrate(targets)
finally:
super().tearDown()
class SampleDirMixin:
SAMPLE_DIR = Path(__file__).parent / "samples"

20
src/documents/views.py
View File

@@ -175,6 +175,7 @@ from documents.tasks import empty_trash
from documents.tasks import index_optimize
from documents.tasks import sanity_check
from documents.tasks import train_classifier
from documents.tasks import update_document_parent_tags
from documents.templating.filepath import validate_filepath_template_and_render
from documents.utils import get_boolean
from paperless import version
@@ -356,6 +357,13 @@ class TagViewSet(ModelViewSet, PermissionsAwareDocumentCountMixin):
filterset_class = TagFilterSet
ordering_fields = ("color", "name", "matching_algorithm", "match", "document_count")
def perform_update(self, serializer):
old_parent = self.get_object().get_parent()
tag = serializer.save()
new_parent = tag.get_parent()
if new_parent and old_parent != new_parent:
update_document_parent_tags(tag, new_parent)
@extend_schema_view(**generate_object_with_permissions_schema(DocumentTypeSerializer))
class DocumentTypeViewSet(ModelViewSet, PermissionsAwareDocumentCountMixin):
@@ -1624,7 +1632,7 @@ class PostDocumentView(GenericAPIView):
title = serializer.validated_data.get("title")
created = serializer.validated_data.get("created")
archive_serial_number = serializer.validated_data.get("archive_serial_number")
custom_field_ids = serializer.validated_data.get("custom_fields")
cf = serializer.validated_data.get("custom_fields")
from_webui = serializer.validated_data.get("from_webui")
t = int(mktime(datetime.now().timetuple()))
@@ -1643,6 +1651,11 @@ class PostDocumentView(GenericAPIView):
source=DocumentSource.WebUI if from_webui else DocumentSource.ApiUpload,
original_file=temp_file_path,
)
custom_fields = None
if isinstance(cf, dict) and cf:
custom_fields = cf
elif isinstance(cf, list) and cf:
custom_fields = dict.fromkeys(cf, None)
input_doc_overrides = DocumentMetadataOverrides(
filename=doc_name,
title=title,
@@ -1653,10 +1666,7 @@ class PostDocumentView(GenericAPIView):
created=created,
asn=archive_serial_number,
owner_id=request.user.id,
# TODO: set values
custom_fields={cf_id: None for cf_id in custom_field_ids}
if custom_field_ids
else None,
custom_fields=custom_fields,
)
async_task = consume_file.delay(