mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2025-07-28 18:24:38 -05:00
Enhancement: custom field sorting (#8494)
This commit is contained in:
shamoon
@@ -6,10 +6,17 @@ from collections.abc import Callable
|
||||
from contextlib import contextmanager
|
||||
|
||||
from django.contrib.contenttypes.models import ContentType
|
||||
from django.db.models import Case
|
||||
from django.db.models import CharField
|
||||
from django.db.models import Count
|
||||
from django.db.models import Exists
|
||||
from django.db.models import IntegerField
|
||||
from django.db.models import OuterRef
|
||||
from django.db.models import Q
|
||||
from django.db.models import Subquery
|
||||
from django.db.models import Sum
|
||||
from django.db.models import Value
|
||||
from django.db.models import When
|
||||
from django.db.models.functions import Cast
|
||||
from django.utils.translation import gettext_lazy as _
|
||||
from django_filters.rest_framework import BooleanFilter
|
||||
@@ -18,6 +25,7 @@ from django_filters.rest_framework import FilterSet
|
||||
from guardian.utils import get_group_obj_perms_model
|
||||
from guardian.utils import get_user_obj_perms_model
|
||||
from rest_framework import serializers
|
||||
from rest_framework.filters import OrderingFilter
|
||||
from rest_framework_guardian.filters import ObjectPermissionsFilter
|
||||
|
||||
from documents.models import Correspondent
|
||||
@@ -760,3 +768,141 @@ class ObjectOwnedPermissionsFilter(ObjectPermissionsFilter):
|
||||
objects_owned = queryset.filter(owner=request.user)
|
||||
objects_unowned = queryset.filter(owner__isnull=True)
|
||||
return objects_owned | objects_unowned
|
||||
|
||||
|
||||
class DocumentsOrderingFilter(OrderingFilter):
|
||||
field_name = "ordering"
|
||||
prefix = "custom_field_"
|
||||
|
||||
def filter_queryset(self, request, queryset, view):
|
||||
param = request.query_params.get("ordering")
|
||||
if param and self.prefix in param:
|
||||
custom_field_id = int(param.split(self.prefix)[1])
|
||||
try:
|
||||
field = CustomField.objects.get(pk=custom_field_id)
|
||||
except CustomField.DoesNotExist:
|
||||
raise serializers.ValidationError(
|
||||
{self.prefix + str(custom_field_id): [_("Custom field not found")]},
|
||||
)
|
||||
|
||||
annotation = None
|
||||
match field.data_type:
|
||||
case CustomField.FieldDataType.STRING:
|
||||
annotation = Subquery(
|
||||
CustomFieldInstance.objects.filter(
|
||||
document_id=OuterRef("id"),
|
||||
field_id=custom_field_id,
|
||||
).values("value_text")[:1],
|
||||
)
|
||||
case CustomField.FieldDataType.INT:
|
||||
annotation = Subquery(
|
||||
CustomFieldInstance.objects.filter(
|
||||
document_id=OuterRef("id"),
|
||||
field_id=custom_field_id,
|
||||
).values("value_int")[:1],
|
||||
)
|
||||
case CustomField.FieldDataType.FLOAT:
|
||||
annotation = Subquery(
|
||||
CustomFieldInstance.objects.filter(
|
||||
document_id=OuterRef("id"),
|
||||
field_id=custom_field_id,
|
||||
).values("value_float")[:1],
|
||||
)
|
||||
case CustomField.FieldDataType.DATE:
|
||||
annotation = Subquery(
|
||||
CustomFieldInstance.objects.filter(
|
||||
document_id=OuterRef("id"),
|
||||
field_id=custom_field_id,
|
||||
).values("value_date")[:1],
|
||||
)
|
||||
case CustomField.FieldDataType.MONETARY:
|
||||
annotation = Subquery(
|
||||
CustomFieldInstance.objects.filter(
|
||||
document_id=OuterRef("id"),
|
||||
field_id=custom_field_id,
|
||||
).values("value_monetary_amount")[:1],
|
||||
)
|
||||
case CustomField.FieldDataType.SELECT:
|
||||
# Select options are a little more complicated since the value is the id of the option, not
|
||||
# the label. Additionally, to support sqlite we can't use StringAgg, so we need to create a
|
||||
# case statement for each option, setting the value to the index of the option in a list
|
||||
# sorted by label, and then summing the results to give a single value for the annotation
|
||||
|
||||
select_options = sorted(
|
||||
field.extra_data.get("select_options", []),
|
||||
key=lambda x: x.get("label"),
|
||||
)
|
||||
whens = [
|
||||
When(
|
||||
custom_fields__field_id=custom_field_id,
|
||||
custom_fields__value_select=option.get("id"),
|
||||
then=Value(idx, output_field=IntegerField()),
|
||||
)
|
||||
for idx, option in enumerate(select_options)
|
||||
]
|
||||
whens.append(
|
||||
When(
|
||||
custom_fields__field_id=custom_field_id,
|
||||
custom_fields__value_select__isnull=True,
|
||||
then=Value(
|
||||
len(select_options),
|
||||
output_field=IntegerField(),
|
||||
),
|
||||
),
|
||||
)
|
||||
annotation = Sum(
|
||||
Case(
|
||||
*whens,
|
||||
default=Value(0),
|
||||
output_field=IntegerField(),
|
||||
),
|
||||
)
|
||||
case CustomField.FieldDataType.DOCUMENTLINK:
|
||||
annotation = Subquery(
|
||||
CustomFieldInstance.objects.filter(
|
||||
document_id=OuterRef("id"),
|
||||
field_id=custom_field_id,
|
||||
).values("value_document_ids")[:1],
|
||||
)
|
||||
case CustomField.FieldDataType.URL:
|
||||
annotation = Subquery(
|
||||
CustomFieldInstance.objects.filter(
|
||||
document_id=OuterRef("id"),
|
||||
field_id=custom_field_id,
|
||||
).values("value_url")[:1],
|
||||
)
|
||||
case CustomField.FieldDataType.BOOL:
|
||||
annotation = Subquery(
|
||||
CustomFieldInstance.objects.filter(
|
||||
document_id=OuterRef("id"),
|
||||
field_id=custom_field_id,
|
||||
).values("value_bool")[:1],
|
||||
)
|
||||
|
||||
if not annotation:
|
||||
# Only happens if a new data type is added and not handled here
|
||||
raise ValueError("Invalid custom field data type")
|
||||
|
||||
queryset = (
|
||||
queryset.annotate(
|
||||
# We need to annotate the queryset with the custom field value
|
||||
custom_field_value=annotation,
|
||||
# We also need to annotate the queryset with a boolean for sorting whether the field exists
|
||||
has_field=Exists(
|
||||
CustomFieldInstance.objects.filter(
|
||||
document_id=OuterRef("id"),
|
||||
field_id=custom_field_id,
|
||||
),
|
||||
),
|
||||
)
|
||||
.order_by(
|
||||
"-has_field",
|
||||
param.replace(
|
||||
self.prefix + str(custom_field_id),
|
||||
"custom_field_value",
|
||||
),
|
||||
)
|
||||
.distinct()
|
||||
)
|
||||
|
||||
return super().filter_queryset(request, queryset, view)
|
||||
|
||||
@@ -5,6 +5,7 @@ import tempfile
|
||||
import uuid
|
||||
import zoneinfo
|
||||
from binascii import hexlify
|
||||
from datetime import date
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
@@ -2762,3 +2763,184 @@ class TestDocumentApiV2(DirectoriesMixin, APITestCase):
|
||||
self.client.get(f"/api/tags/{t.id}/", format="json").data["text_color"],
|
||||
"#000000",
|
||||
)
|
||||
|
||||
|
||||
class TestDocumentApiCustomFieldsSorting(DirectoriesMixin, APITestCase):
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
|
||||
self.user = User.objects.create_superuser(username="temp_admin")
|
||||
self.client.force_authenticate(user=self.user)
|
||||
|
||||
self.doc1 = Document.objects.create(
|
||||
title="none1",
|
||||
checksum="A",
|
||||
mime_type="application/pdf",
|
||||
)
|
||||
self.doc2 = Document.objects.create(
|
||||
title="none2",
|
||||
checksum="B",
|
||||
mime_type="application/pdf",
|
||||
)
|
||||
self.doc3 = Document.objects.create(
|
||||
title="none3",
|
||||
checksum="C",
|
||||
mime_type="application/pdf",
|
||||
)
|
||||
|
||||
cache.clear()
|
||||
|
||||
def test_document_custom_fields_sorting(self):
|
||||
"""
|
||||
GIVEN:
|
||||
- Documents with custom fields
|
||||
WHEN:
|
||||
- API request for document filtering with custom field sorting
|
||||
THEN:
|
||||
- Documents are sorted by custom field values
|
||||
"""
|
||||
values = {
|
||||
CustomField.FieldDataType.STRING: {
|
||||
"values": ["foo", "bar", "baz"],
|
||||
"field_name": CustomFieldInstance.TYPE_TO_DATA_STORE_NAME_MAP[
|
||||
CustomField.FieldDataType.STRING
|
||||
],
|
||||
},
|
||||
CustomField.FieldDataType.INT: {
|
||||
"values": [3, 1, 2],
|
||||
"field_name": CustomFieldInstance.TYPE_TO_DATA_STORE_NAME_MAP[
|
||||
CustomField.FieldDataType.INT
|
||||
],
|
||||
},
|
||||
CustomField.FieldDataType.FLOAT: {
|
||||
"values": [3.3, 1.1, 2.2],
|
||||
"field_name": CustomFieldInstance.TYPE_TO_DATA_STORE_NAME_MAP[
|
||||
CustomField.FieldDataType.FLOAT
|
||||
],
|
||||
},
|
||||
CustomField.FieldDataType.BOOL: {
|
||||
"values": [True, False, False],
|
||||
"field_name": CustomFieldInstance.TYPE_TO_DATA_STORE_NAME_MAP[
|
||||
CustomField.FieldDataType.BOOL
|
||||
],
|
||||
},
|
||||
CustomField.FieldDataType.DATE: {
|
||||
"values": [date(2021, 1, 3), date(2021, 1, 1), date(2021, 1, 2)],
|
||||
"field_name": CustomFieldInstance.TYPE_TO_DATA_STORE_NAME_MAP[
|
||||
CustomField.FieldDataType.DATE
|
||||
],
|
||||
},
|
||||
CustomField.FieldDataType.URL: {
|
||||
"values": [
|
||||
"http://example.org",
|
||||
"http://example.com",
|
||||
"http://example.net",
|
||||
],
|
||||
"field_name": CustomFieldInstance.TYPE_TO_DATA_STORE_NAME_MAP[
|
||||
CustomField.FieldDataType.URL
|
||||
],
|
||||
},
|
||||
CustomField.FieldDataType.MONETARY: {
|
||||
"values": ["USD789.00", "USD123.00", "USD456.00"],
|
||||
"field_name": CustomFieldInstance.TYPE_TO_DATA_STORE_NAME_MAP[
|
||||
CustomField.FieldDataType.MONETARY
|
||||
],
|
||||
},
|
||||
CustomField.FieldDataType.DOCUMENTLINK: {
|
||||
"values": [self.doc3.pk, self.doc1.pk, self.doc2.pk],
|
||||
"field_name": CustomFieldInstance.TYPE_TO_DATA_STORE_NAME_MAP[
|
||||
CustomField.FieldDataType.DOCUMENTLINK
|
||||
],
|
||||
},
|
||||
CustomField.FieldDataType.SELECT: {
|
||||
"values": ["ghi-789", "abc-123", "def-456"],
|
||||
"field_name": CustomFieldInstance.TYPE_TO_DATA_STORE_NAME_MAP[
|
||||
CustomField.FieldDataType.SELECT
|
||||
],
|
||||
"extra_data": {
|
||||
"select_options": [
|
||||
{"label": "Option 1", "id": "abc-123"},
|
||||
{"label": "Option 2", "id": "def-456"},
|
||||
{"label": "Option 3", "id": "ghi-789"},
|
||||
],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for data_type, data in values.items():
|
||||
CustomField.objects.all().delete()
|
||||
CustomFieldInstance.objects.all().delete()
|
||||
custom_field = CustomField.objects.create(
|
||||
name=f"custom field {data_type}",
|
||||
data_type=data_type,
|
||||
extra_data=data.get("extra_data", {}),
|
||||
)
|
||||
for i, value in enumerate(data["values"]):
|
||||
CustomFieldInstance.objects.create(
|
||||
document=[self.doc1, self.doc2, self.doc3][i],
|
||||
field=custom_field,
|
||||
**{data["field_name"]: value},
|
||||
)
|
||||
response = self.client.get(
|
||||
f"/api/documents/?ordering=custom_field_{custom_field.pk}",
|
||||
)
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
results = response.data["results"]
|
||||
self.assertEqual(len(results), 3)
|
||||
self.assertEqual(
|
||||
[results[0]["id"], results[1]["id"], results[2]["id"]],
|
||||
[self.doc2.id, self.doc3.id, self.doc1.id],
|
||||
)
|
||||
|
||||
response = self.client.get(
|
||||
f"/api/documents/?ordering=-custom_field_{custom_field.pk}",
|
||||
)
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
results = response.data["results"]
|
||||
self.assertEqual(len(results), 3)
|
||||
if data_type == CustomField.FieldDataType.BOOL:
|
||||
# just check the first one for bools, as the rest are the same
|
||||
self.assertEqual(
|
||||
[results[0]["id"]],
|
||||
[self.doc1.id],
|
||||
)
|
||||
else:
|
||||
self.assertEqual(
|
||||
[results[0]["id"], results[1]["id"], results[2]["id"]],
|
||||
[self.doc1.id, self.doc3.id, self.doc2.id],
|
||||
)
|
||||
|
||||
def test_document_custom_fields_sorting_invalid(self):
|
||||
"""
|
||||
GIVEN:
|
||||
- Documents with custom fields
|
||||
WHEN:
|
||||
- API request for document filtering with invalid custom field sorting
|
||||
THEN:
|
||||
- 400 is returned
|
||||
"""
|
||||
|
||||
response = self.client.get(
|
||||
"/api/documents/?ordering=custom_field_999",
|
||||
)
|
||||
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
|
||||
|
||||
def test_document_custom_fields_sorting_invalid_data_type(self):
|
||||
"""
|
||||
GIVEN:
|
||||
- Documents with custom fields
|
||||
WHEN:
|
||||
- API request for document filtering with a custom field sorting with a new (unhandled) data type
|
||||
THEN:
|
||||
- Error is raised
|
||||
"""
|
||||
|
||||
custom_field = CustomField.objects.create(
|
||||
name="custom field",
|
||||
data_type="foo",
|
||||
)
|
||||
|
||||
with self.assertRaises(ValueError):
|
||||
self.client.get(
|
||||
f"/api/documents/?ordering=custom_field_{custom_field.pk}",
|
||||
)
|
||||
|
||||
@@ -96,6 +96,7 @@ from documents.data_models import DocumentSource
|
||||
from documents.filters import CorrespondentFilterSet
|
||||
from documents.filters import CustomFieldFilterSet
|
||||
from documents.filters import DocumentFilterSet
|
||||
from documents.filters import DocumentsOrderingFilter
|
||||
from documents.filters import DocumentTypeFilterSet
|
||||
from documents.filters import ObjectOwnedOrGrantedPermissionsFilter
|
||||
from documents.filters import ObjectOwnedPermissionsFilter
|
||||
@@ -350,7 +351,7 @@ class DocumentViewSet(
|
||||
filter_backends = (
|
||||
DjangoFilterBackend,
|
||||
SearchFilter,
|
||||
OrderingFilter,
|
||||
DocumentsOrderingFilter,
|
||||
ObjectOwnedOrGrantedPermissionsFilter,
|
||||
)
|
||||
filterset_class = DocumentFilterSet
|
||||
@@ -367,6 +368,7 @@ class DocumentViewSet(
|
||||
"num_notes",
|
||||
"owner",
|
||||
"page_count",
|
||||
"custom_field_",
|
||||
)
|
||||
|
||||
def get_queryset(self):
|
||||
|
||||
Reference in New Issue
Block a user