paradizelost/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2025-07-28 18:24:38 -05:00
Code Issues Packages Projects Releases Wiki Activity

Feature: custom fields filtering & bulk editing (#6484)

Browse Source
This commit is contained in:
shamoon
2024-04-26 15:10:03 -07:00
committed by GitHub
parent bd4476d484
commit 63e1f9f5d3
28 changed files with 1563 additions and 323 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
src/documents/bulk_edit.py
View File

@@ -12,6 +12,7 @@ from documents.data_models import ConsumableDocument
from documents.data_models import DocumentMetadataOverrides
from documents.data_models import DocumentSource
from documents.models import Correspondent
from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import DocumentType
from documents.models import StoragePath
@@ -120,6 +121,30 @@ def modify_tags(doc_ids, add_tags, remove_tags):
return "OK"
def modify_custom_fields(doc_ids, add_custom_fields, remove_custom_fields):
qs = Document.objects.filter(id__in=doc_ids)
affected_docs = [doc.id for doc in qs]
fields_to_add = []
for field in add_custom_fields:
for doc_id in affected_docs:
fields_to_add.append(
CustomFieldInstance(
document_id=doc_id,
field_id=field,
),
)
CustomFieldInstance.objects.bulk_create(fields_to_add)
CustomFieldInstance.objects.filter(
document_id__in=affected_docs,
field_id__in=remove_custom_fields,
).delete()
bulk_update_documents.delay(document_ids=affected_docs)
return "OK"
def delete(doc_ids):
Document.objects.filter(id__in=doc_ids).delete()

19
src/documents/filters.py
View File

@@ -199,6 +199,25 @@ class DocumentFilterSet(FilterSet):
custom_fields__icontains = CustomFieldsFilter()
custom_fields__id__all = ObjectFilter(field_name="custom_fields__field")
custom_fields__id__none = ObjectFilter(
field_name="custom_fields__field",
exclude=True,
)
custom_fields__id__in = ObjectFilter(
field_name="custom_fields__field",
in_list=True,
)
has_custom_fields = BooleanFilter(
label="Has custom field",
field_name="custom_fields",
lookup_expr="isnull",
exclude=True,
)
shared_by__id = SharedByUser()
class Meta:

18
src/documents/index.py
View File

@@ -70,6 +70,8 @@ def get_schema():
num_notes=NUMERIC(sortable=True, signed=False),
custom_fields=TEXT(),
custom_field_count=NUMERIC(sortable=True, signed=False),
has_custom_fields=BOOLEAN(),
custom_fields_id=KEYWORD(commas=True),
owner=TEXT(),
owner_id=NUMERIC(),
has_owner=BOOLEAN(),
@@ -125,6 +127,9 @@ def update_document(writer: AsyncWriter, doc: Document):
custom_fields = ",".join(
[str(c) for c in CustomFieldInstance.objects.filter(document=doc)],
)
custom_fields_ids = ",".join(
[str(f.field.id) for f in CustomFieldInstance.objects.filter(document=doc)],
)
asn = doc.archive_serial_number
if asn is not None and (
asn < Document.ARCHIVE_SERIAL_NUMBER_MIN
@@ -166,6 +171,8 @@ def update_document(writer: AsyncWriter, doc: Document):
num_notes=len(notes),
custom_fields=custom_fields,
custom_field_count=len(doc.custom_fields.all()),
has_custom_fields=len(custom_fields) > 0,
custom_fields_id=custom_fields_ids if custom_fields_ids else None,
owner=doc.owner.username if doc.owner else None,
owner_id=doc.owner.id if doc.owner else None,
has_owner=doc.owner is not None,
@@ -206,7 +213,10 @@ class DelayedQuery:
"created": ("created", ["date__lt", "date__gt"]),
"checksum": ("checksum", ["icontains", "istartswith"]),
"original_filename": ("original_filename", ["icontains", "istartswith"]),
"custom_fields": ("custom_fields", ["icontains", "istartswith"]),
"custom_fields": (
"custom_fields",
["icontains", "istartswith", "id__all", "id__in", "id__none"],
),
}
def _get_query(self):
@@ -220,6 +230,12 @@ class DelayedQuery:
criterias.append(query.Term("has_tag", self.evalBoolean(value)))
continue
if key == "has_custom_fields":
criterias.append(
query.Term("has_custom_fields", self.evalBoolean(value)),
)
continue
# Don't process query params without a filter
if "__" not in key:
continue

65
src/documents/migrations/1048_alter_savedviewfilterrule_rule_type.py Normal file
View File

@@ -0,0 +1,65 @@
# Generated by Django 4.2.11 on 2024-04-24 04:58
from django.db import migrations
from django.db import models
class Migration(migrations.Migration):
dependencies = [
("documents", "1047_savedview_display_mode_and_more"),
]
operations = [
migrations.AlterField(
model_name="savedviewfilterrule",
name="rule_type",
field=models.PositiveIntegerField(
choices=[
(0, "title contains"),
(1, "content contains"),
(2, "ASN is"),
(3, "correspondent is"),
(4, "document type is"),
(5, "is in inbox"),
(6, "has tag"),
(7, "has any tag"),
(8, "created before"),
(9, "created after"),
(10, "created year is"),
(11, "created month is"),
(12, "created day is"),
(13, "added before"),
(14, "added after"),
(15, "modified before"),
(16, "modified after"),
(17, "does not have tag"),
(18, "does not have ASN"),
(19, "title or content contains"),
(20, "fulltext query"),
(21, "more like this"),
(22, "has tags in"),
(23, "ASN greater than"),
(24, "ASN less than"),
(25, "storage path is"),
(26, "has correspondent in"),
(27, "does not have correspondent in"),
(28, "has document type in"),
(29, "does not have document type in"),
(30, "has storage path in"),
(31, "does not have storage path in"),
(32, "owner is"),
(33, "has owner in"),
(34, "does not have owner"),
(35, "does not have owner in"),
(36, "has custom field value"),
(37, "is shared by me"),
(38, "has custom fields"),
(39, "has custom field in"),
(40, "does not have custom field in"),
(41, "does not have custom field"),
],
verbose_name="rule type",
),
),
]

4
src/documents/models.py
View File

@@ -500,6 +500,10 @@ class SavedViewFilterRule(models.Model):
(35, _("does not have owner in")),
(36, _("has custom field value")),
(37, _("is shared by me")),
(38, _("has custom fields")),
(39, _("has custom field in")),
(40, _("does not have custom field in")),
(41, _("does not have custom field")),
]
saved_view = models.ForeignKey(

33
src/documents/serialisers.py
View File

@@ -905,6 +905,7 @@ class BulkEditSerializer(
"add_tag",
"remove_tag",
"modify_tags",
"modify_custom_fields",
"delete",
"redo_ocr",
"set_permissions",
@@ -929,6 +930,17 @@ class BulkEditSerializer(
f"Some tags in {name} don't exist or were specified twice.",
)
def _validate_custom_field_id_list(self, custom_fields, name="custom_fields"):
if not isinstance(custom_fields, list):
raise serializers.ValidationError(f"{name} must be a list")
if not all(isinstance(i, int) for i in custom_fields):
raise serializers.ValidationError(f"{name} must be a list of integers")
count = CustomField.objects.filter(id__in=custom_fields).count()
if not count == len(custom_fields):
raise serializers.ValidationError(
f"Some custom fields in {name} don't exist or were specified twice.",
)
def validate_method(self, method):
if method == "set_correspondent":
return bulk_edit.set_correspondent
@@ -942,6 +954,8 @@ class BulkEditSerializer(
return bulk_edit.remove_tag
elif method == "modify_tags":
return bulk_edit.modify_tags
elif method == "modify_custom_fields":
return bulk_edit.modify_custom_fields
elif method == "delete":
return bulk_edit.delete
elif method == "redo_ocr":
@@ -1017,6 +1031,23 @@ class BulkEditSerializer(
else:
raise serializers.ValidationError("remove_tags not specified")
def _validate_parameters_modify_custom_fields(self, parameters):
if "add_custom_fields" in parameters:
self._validate_custom_field_id_list(
parameters["add_custom_fields"],
"add_custom_fields",
)
else:
raise serializers.ValidationError("add_custom_fields not specified")
if "remove_custom_fields" in parameters:
self._validate_custom_field_id_list(
parameters["remove_custom_fields"],
"remove_custom_fields",
)
else:
raise serializers.ValidationError("remove_custom_fields not specified")
def _validate_owner(self, owner):
ownerUser = User.objects.get(pk=owner)
if ownerUser is None:
@@ -1079,6 +1110,8 @@ class BulkEditSerializer(
self._validate_parameters_modify_tags(parameters)
elif method == bulk_edit.set_storage_path:
self._validate_storage_path(parameters)
elif method == bulk_edit.modify_custom_fields:
self._validate_parameters_modify_custom_fields(parameters)
elif method == bulk_edit.set_permissions:
self._validate_parameters_set_permissions(parameters)
elif method == bulk_edit.rotate:

132
src/documents/tests/test_api_bulk_edit.py
View File

@@ -7,6 +7,7 @@ from rest_framework import status
from rest_framework.test import APITestCase
from documents.models import Correspondent
from documents.models import CustomField
from documents.models import Document
from documents.models import DocumentType
from documents.models import StoragePath
@@ -49,6 +50,8 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.doc3.tags.add(self.t2)
self.doc4.tags.add(self.t1, self.t2)
self.sp1 = StoragePath.objects.create(name="sp1", path="Something/{checksum}")
self.cf1 = CustomField.objects.create(name="cf1", data_type="text")
self.cf2 = CustomField.objects.create(name="cf2", data_type="text")
@mock.patch("documents.serialisers.bulk_edit.set_correspondent")
def test_api_set_correspondent(self, m):
@@ -222,6 +225,135 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
m.assert_not_called()
@mock.patch("documents.serialisers.bulk_edit.modify_custom_fields")
def test_api_modify_custom_fields(self, m):
m.return_value = "OK"
response = self.client.post(
"/api/documents/bulk_edit/",
json.dumps(
{
"documents": [self.doc1.id, self.doc3.id],
"method": "modify_custom_fields",
"parameters": {
"add_custom_fields": [self.cf1.id],
"remove_custom_fields": [self.cf2.id],
},
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
m.assert_called_once()
args, kwargs = m.call_args
self.assertListEqual(args[0], [self.doc1.id, self.doc3.id])
self.assertEqual(kwargs["add_custom_fields"], [self.cf1.id])
self.assertEqual(kwargs["remove_custom_fields"], [self.cf2.id])
@mock.patch("documents.serialisers.bulk_edit.modify_custom_fields")
def test_api_modify_custom_fields_invalid_params(self, m):
"""
GIVEN:
- API data to modify custom fields is malformed
WHEN:
- API to edit custom fields is called
THEN:
- API returns HTTP 400
- modify_custom_fields is not called
"""
m.return_value = "OK"
# Missing add_custom_fields
response = self.client.post(
"/api/documents/bulk_edit/",
json.dumps(
{
"documents": [self.doc1.id, self.doc3.id],
"method": "modify_custom_fields",
"parameters": {
"add_custom_fields": [self.cf1.id],
},
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
m.assert_not_called()
# Missing remove_custom_fields
response = self.client.post(
"/api/documents/bulk_edit/",
json.dumps(
{
"documents": [self.doc1.id, self.doc3.id],
"method": "modify_custom_fields",
"parameters": {
"remove_custom_fields": [self.cf1.id],
},
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
m.assert_not_called()
# Not a list
response = self.client.post(
"/api/documents/bulk_edit/",
json.dumps(
{
"documents": [self.doc1.id, self.doc3.id],
"method": "modify_custom_fields",
"parameters": {
"add_custom_fields": self.cf1.id,
"remove_custom_fields": self.cf2.id,
},
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
m.assert_not_called()
# Not a list of integers
# Missing remove_custom_fields
response = self.client.post(
"/api/documents/bulk_edit/",
json.dumps(
{
"documents": [self.doc1.id, self.doc3.id],
"method": "modify_custom_fields",
"parameters": {
"add_custom_fields": ["foo"],
"remove_custom_fields": ["bar"],
},
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
m.assert_not_called()
# Custom field ID not found
# Missing remove_custom_fields
response = self.client.post(
"/api/documents/bulk_edit/",
json.dumps(
{
"documents": [self.doc1.id, self.doc3.id],
"method": "modify_custom_fields",
"parameters": {
"add_custom_fields": [self.cf1.id],
"remove_custom_fields": [99],
},
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
m.assert_not_called()
@mock.patch("documents.serialisers.bulk_edit.delete")
def test_api_delete(self, m):
m.return_value = "OK"

28
src/documents/tests/test_api_search.py
View File

@@ -920,6 +920,34 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
),
)
self.assertIn(
d4.id,
search_query(
"&has_custom_fields=1",
),
)
self.assertIn(
d4.id,
search_query(
"&custom_fields__id__in=" + str(cf1.id),
),
)
self.assertIn(
d4.id,
search_query(
"&custom_fields__id__all=" + str(cf1.id),
),
)
self.assertNotIn(
d4.id,
search_query(
"&custom_fields__id__none=" + str(cf1.id),
),
)
def test_search_filtering_respect_owner(self):
"""
GIVEN:

49
src/documents/tests/test_bulk_edit.py
View File

@@ -11,6 +11,8 @@ from guardian.shortcuts import get_users_with_perms
from documents import bulk_edit
from documents.models import Correspondent
from documents.models import CustomField
from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import DocumentType
from documents.models import StoragePath
@@ -186,6 +188,53 @@ class TestBulkEdit(DirectoriesMixin, TestCase):
# TODO: doc3 should not be affected, but the query for that is rather complicated
self.assertCountEqual(kwargs["document_ids"], [self.doc2.id, self.doc3.id])
def test_modify_custom_fields(self):
cf = CustomField.objects.create(
name="cf1",
data_type=CustomField.FieldDataType.STRING,
)
cf2 = CustomField.objects.create(
name="cf2",
data_type=CustomField.FieldDataType.INT,
)
cf3 = CustomField.objects.create(
name="cf3",
data_type=CustomField.FieldDataType.STRING,
)
CustomFieldInstance.objects.create(
document=self.doc1,
field=cf,
)
CustomFieldInstance.objects.create(
document=self.doc2,
field=cf,
)
CustomFieldInstance.objects.create(
document=self.doc2,
field=cf3,
)
bulk_edit.modify_custom_fields(
[self.doc1.id, self.doc2.id],
add_custom_fields=[cf2.id],
remove_custom_fields=[cf.id],
)
self.doc1.refresh_from_db()
self.doc2.refresh_from_db()
self.assertEqual(
self.doc1.custom_fields.count(),
1,
)
self.assertEqual(
self.doc2.custom_fields.count(),
2,
)
self.async_task.assert_called_once()
args, kwargs = self.async_task.call_args
self.assertCountEqual(kwargs["document_ids"], [self.doc1.id, self.doc2.id])
def test_delete(self):
self.assertEqual(Document.objects.count(), 5)
bulk_edit.delete([self.doc1.id, self.doc2.id])

16
src/documents/views.py
View File

@@ -1065,6 +1065,18 @@ class SelectionDataView(GenericAPIView):
),
)
custom_fields = CustomField.objects.annotate(
document_count=Count(
Case(
When(
fields__document__id__in=ids,
then=1,
),
output_field=IntegerField(),
),
),
)
r = Response(
{
"selected_correspondents": [
@@ -1081,6 +1093,10 @@ class SelectionDataView(GenericAPIView):
{"id": t.id, "document_count": t.document_count}
for t in storage_paths
],
"selected_custom_fields": [
{"id": t.id, "document_count": t.document_count}
for t in custom_fields
],
},
)