Feature: Implement custom fields for documents (#4502)

Adds custom fields of certain data types, attachable to documents and searchable

Co-Authored-By: Trenton H <797416+stumpylog@users.noreply.github.com>

src/documents/tests/test_api.py

@@ -35,6 +35,8 @@ from documents import index
 from documents.data_models import DocumentSource
 from documents.models import ConsumptionTemplate
 from documents.models import Correspondent
+from documents.models import CustomField
+from documents.models import CustomFieldInstance
 from documents.models import Document
 from documents.models import DocumentType
 from documents.models import MatchingModel
@@ -347,11 +349,36 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
         tag_2 = Tag.objects.create(name="t2")
         tag_3 = Tag.objects.create(name="t3")
 
+        cf1 = CustomField.objects.create(
+            name="stringfield",
+            data_type=CustomField.FieldDataType.STRING,
+        )
+        cf2 = CustomField.objects.create(
+            name="numberfield",
+            data_type=CustomField.FieldDataType.INT,
+        )
+
         doc1.tags.add(tag_inbox)
         doc2.tags.add(tag_2)
         doc3.tags.add(tag_2)
         doc3.tags.add(tag_3)
 
+        cf1_d1 = CustomFieldInstance.objects.create(
+            document=doc1,
+            field=cf1,
+            value_text="foobard1",
+        )
+        CustomFieldInstance.objects.create(
+            document=doc1,
+            field=cf2,
+            value_int=999,
+        )
+        cf1_d3 = CustomFieldInstance.objects.create(
+            document=doc3,
+            field=cf1,
+            value_text="foobard3",
+        )
+
         response = self.client.get("/api/documents/?is_in_inbox=true")
         self.assertEqual(response.status_code, status.HTTP_200_OK)
         results = response.data["results"]
@@ -423,6 +450,31 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
         results = response.data["results"]
         self.assertEqual(len(results), 0)
 
+        # custom field name
+        response = self.client.get(
+            f"/api/documents/?custom_fields__icontains={cf1.name}",
+        )
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        results = response.data["results"]
+        self.assertEqual(len(results), 2)
+
+        # custom field value
+        response = self.client.get(
+            f"/api/documents/?custom_fields__icontains={cf1_d1.value}",
+        )
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        results = response.data["results"]
+        self.assertEqual(len(results), 1)
+        self.assertEqual(results[0]["id"], doc1.id)
+
+        response = self.client.get(
+            f"/api/documents/?custom_fields__icontains={cf1_d3.value}",
+        )
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        results = response.data["results"]
+        self.assertEqual(len(results), 1)
+        self.assertEqual(results[0]["id"], doc3.id)
+
     def test_document_checksum_filter(self):
         Document.objects.create(
             title="none1",
@@ -1146,6 +1198,14 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
         dt2 = DocumentType.objects.create(name="type2")
         sp = StoragePath.objects.create(name="path")
         sp2 = StoragePath.objects.create(name="path2")
+        cf1 = CustomField.objects.create(
+            name="string field",
+            data_type=CustomField.FieldDataType.STRING,
+        )
+        cf2 = CustomField.objects.create(
+            name="number field",
+            data_type=CustomField.FieldDataType.INT,
+        )
 
         d1 = Document.objects.create(checksum="1", correspondent=c, content="test")
         d2 = Document.objects.create(checksum="2", document_type=dt, content="test")
@@ -1176,6 +1236,22 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
             content="test",
         )
 
+        cf1_d1 = CustomFieldInstance.objects.create(
+            document=d1,
+            field=cf1,
+            value_text="foobard1",
+        )
+        cf2_d1 = CustomFieldInstance.objects.create(
+            document=d1,
+            field=cf2,
+            value_int=999,
+        )
+        cf1_d4 = CustomFieldInstance.objects.create(
+            document=d4,
+            field=cf1,
+            value_text="foobard4",
+        )
+
         with AsyncWriter(index.open_index()) as writer:
             for doc in Document.objects.all():
                 index.update_document(writer, doc)
@@ -1304,6 +1380,7 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
                 + datetime.datetime(2020, 1, 2).strftime("%Y-%m-%d"),
             ),
         )
+
         self.assertIn(
             d5.id,
             search_query(
@@ -1322,6 +1399,27 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
             [d4.id, d5.id],
         )
 
+        self.assertIn(
+            d1.id,
+            search_query(
+                "&custom_fields__icontains=" + cf1_d1.value,
+            ),
+        )
+
+        self.assertIn(
+            d1.id,
+            search_query(
+                "&custom_fields__icontains=" + str(cf2_d1.value),
+            ),
+        )
+
+        self.assertIn(
+            d4.id,
+            search_query(
+                "&custom_fields__icontains=" + cf1_d4.value,
+            ),
+        )
+
     def test_search_filtering_respect_owner(self):
         """
         GIVEN:
@@ -2421,7 +2519,7 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
             f"/api/documents/{doc.pk}/notes/",
             format="json",
         )
-        self.assertEqual(resp.content, b"Insufficient permissions to view")
+        self.assertEqual(resp.content, b"Insufficient permissions to view notes")
         self.assertEqual(resp.status_code, status.HTTP_403_FORBIDDEN)
 
         assign_perm("view_document", user1, doc)
@@ -2430,7 +2528,7 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
             f"/api/documents/{doc.pk}/notes/",
             data={"note": "this is a posted note"},
         )
-        self.assertEqual(resp.content, b"Insufficient permissions to create")
+        self.assertEqual(resp.content, b"Insufficient permissions to create notes")
         self.assertEqual(resp.status_code, status.HTTP_403_FORBIDDEN)
 
         note = Note.objects.create(
@@ -2444,7 +2542,7 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
             format="json",
         )
 
-        self.assertEqual(response.content, b"Insufficient permissions to delete")
+        self.assertEqual(response.content, b"Insufficient permissions to delete notes")
         self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
 
     def test_delete_note(self):
@@ -2694,7 +2792,7 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
             f"/api/documents/{doc.pk}/share_links/",
             format="json",
         )
-        self.assertEqual(resp.content, b"Insufficient permissions")
+        self.assertEqual(resp.content, b"Insufficient permissions to add share link")
         self.assertEqual(resp.status_code, status.HTTP_403_FORBIDDEN)
 
         assign_perm("change_document", user1, doc)

src/documents/tests/test_api_custom_fields.py

@@ -0,0 +1,384 @@
+from datetime import date
+
+from django.contrib.auth.models import User
+from rest_framework import status
+from rest_framework.test import APITestCase
+
+from documents.models import CustomField
+from documents.models import CustomFieldInstance
+from documents.models import Document
+from documents.tests.utils import DirectoriesMixin
+
+
+class TestCustomField(DirectoriesMixin, APITestCase):
+    ENDPOINT = "/api/custom_fields/"
+
+    def setUp(self):
+        self.user = User.objects.create_superuser(username="temp_admin")
+        self.client.force_authenticate(user=self.user)
+        return super().setUp()
+
+    def test_create_custom_field(self):
+        """
+        GIVEN:
+            - Each of the supported data types is created
+        WHEN:
+            - API request to create custom metadata is made
+        THEN:
+            - the field is created
+            - the field returns the correct fields
+        """
+        for field_type, name in [
+            ("string", "Custom Text"),
+            ("url", "Wikipedia Link"),
+            ("date", "Invoiced Date"),
+            ("integer", "Invoice #"),
+            ("boolean", "Is Active"),
+            ("float", "Total Paid"),
+        ]:
+            resp = self.client.post(
+                self.ENDPOINT,
+                data={
+                    "data_type": field_type,
+                    "name": name,
+                },
+            )
+            self.assertEqual(resp.status_code, status.HTTP_201_CREATED)
+
+            data = resp.json()
+
+            self.assertEqual(len(data), 3)
+            self.assertEqual(data["name"], name)
+            self.assertEqual(data["data_type"], field_type)
+
+    def test_create_custom_field_instance(self):
+        """
+        GIVEN:
+            - Field of each data type is created
+        WHEN:
+            - API request to create custom metadata instance with each data type
+        THEN:
+            - the field instance is created
+            - the field returns the correct fields and values
+            - the field is attached to the given document
+        """
+        doc = Document.objects.create(
+            title="WOW",
+            content="the content",
+            checksum="123",
+            mime_type="application/pdf",
+        )
+        custom_field_string = CustomField.objects.create(
+            name="Test Custom Field String",
+            data_type=CustomField.FieldDataType.STRING,
+        )
+        custom_field_date = CustomField.objects.create(
+            name="Test Custom Field Date",
+            data_type=CustomField.FieldDataType.DATE,
+        )
+        custom_field_int = CustomField.objects.create(
+            name="Test Custom Field Int",
+            data_type=CustomField.FieldDataType.INT,
+        )
+        custom_field_boolean = CustomField.objects.create(
+            name="Test Custom Field Boolean",
+            data_type=CustomField.FieldDataType.BOOL,
+        )
+        custom_field_url = CustomField.objects.create(
+            name="Test Custom Field Url",
+            data_type=CustomField.FieldDataType.URL,
+        )
+        custom_field_float = CustomField.objects.create(
+            name="Test Custom Field Float",
+            data_type=CustomField.FieldDataType.FLOAT,
+        )
+        custom_field_monetary = CustomField.objects.create(
+            name="Test Custom Field Monetary",
+            data_type=CustomField.FieldDataType.MONETARY,
+        )
+
+        date_value = date.today()
+
+        resp = self.client.patch(
+            f"/api/documents/{doc.id}/",
+            data={
+                "custom_fields": [
+                    {
+                        "field": custom_field_string.id,
+                        "value": "test value",
+                    },
+                    {
+                        "field": custom_field_date.id,
+                        "value": date_value.isoformat(),
+                    },
+                    {
+                        "field": custom_field_int.id,
+                        "value": 3,
+                    },
+                    {
+                        "field": custom_field_boolean.id,
+                        "value": True,
+                    },
+                    {
+                        "field": custom_field_url.id,
+                        "value": "https://example.com",
+                    },
+                    {
+                        "field": custom_field_float.id,
+                        "value": 12.3456,
+                    },
+                    {
+                        "field": custom_field_monetary.id,
+                        "value": 11.10,
+                    },
+                ],
+            },
+            format="json",
+        )
+
+        self.assertEqual(resp.status_code, status.HTTP_200_OK)
+
+        resp_data = resp.json()["custom_fields"]
+
+        self.assertCountEqual(
+            resp_data,
+            [
+                {"field": custom_field_string.id, "value": "test value"},
+                {"field": custom_field_date.id, "value": date_value.isoformat()},
+                {"field": custom_field_int.id, "value": 3},
+                {"field": custom_field_boolean.id, "value": True},
+                {"field": custom_field_url.id, "value": "https://example.com"},
+                {"field": custom_field_float.id, "value": 12.3456},
+                {"field": custom_field_monetary.id, "value": 11.10},
+            ],
+        )
+
+        doc.refresh_from_db()
+        self.assertEqual(len(doc.custom_fields.all()), 7)
+
+    def test_change_custom_field_instance_value(self):
+        """
+        GIVEN:
+            - Custom field instance is created and attached to document
+        WHEN:
+            - API request to create change the value of the custom field
+        THEN:
+            - the field instance is updated
+            - the field returns the correct fields and values
+        """
+        doc = Document.objects.create(
+            title="WOW",
+            content="the content",
+            checksum="123",
+            mime_type="application/pdf",
+        )
+        custom_field_string = CustomField.objects.create(
+            name="Test Custom Field String",
+            data_type=CustomField.FieldDataType.STRING,
+        )
+
+        self.assertEqual(CustomFieldInstance.objects.count(), 0)
+
+        # Create
+        resp = self.client.patch(
+            f"/api/documents/{doc.id}/",
+            data={
+                "custom_fields": [
+                    {
+                        "field": custom_field_string.id,
+                        "value": "test value",
+                    },
+                ],
+            },
+            format="json",
+        )
+
+        self.assertEqual(resp.status_code, status.HTTP_200_OK)
+        self.assertEqual(CustomFieldInstance.objects.count(), 1)
+        self.assertEqual(doc.custom_fields.first().value, "test value")
+
+        # Update
+        resp = self.client.patch(
+            f"/api/documents/{doc.id}/",
+            data={
+                "custom_fields": [
+                    {
+                        "field": custom_field_string.id,
+                        "value": "a new test value",
+                    },
+                ],
+            },
+            format="json",
+        )
+        self.assertEqual(resp.status_code, status.HTTP_200_OK)
+        self.assertEqual(CustomFieldInstance.objects.count(), 1)
+        self.assertEqual(doc.custom_fields.first().value, "a new test value")
+
+    def test_delete_custom_field_instance(self):
+        """
+        GIVEN:
+            - Multiple custom field instances are created and attached to document
+        WHEN:
+            - API request to remove a field
+        THEN:
+            - the field instance is removed
+            - the other field remains unchanged
+            - the field returns the correct fields and values
+        """
+        doc = Document.objects.create(
+            title="WOW",
+            content="the content",
+            checksum="123",
+            mime_type="application/pdf",
+        )
+        custom_field_string = CustomField.objects.create(
+            name="Test Custom Field String",
+            data_type=CustomField.FieldDataType.STRING,
+        )
+        custom_field_date = CustomField.objects.create(
+            name="Test Custom Field Date",
+            data_type=CustomField.FieldDataType.DATE,
+        )
+
+        date_value = date.today()
+
+        resp = self.client.patch(
+            f"/api/documents/{doc.id}/",
+            data={
+                "custom_fields": [
+                    {
+                        "field": custom_field_string.id,
+                        "value": "a new test value",
+                    },
+                    {
+                        "field": custom_field_date.id,
+                        "value": date_value.isoformat(),
+                    },
+                ],
+            },
+            format="json",
+        )
+        self.assertEqual(resp.status_code, status.HTTP_200_OK)
+        self.assertEqual(CustomFieldInstance.objects.count(), 2)
+        self.assertEqual(len(doc.custom_fields.all()), 2)
+
+        resp = self.client.patch(
+            f"/api/documents/{doc.id}/",
+            data={
+                "custom_fields": [
+                    {
+                        "field": custom_field_date.id,
+                        "value": date_value.isoformat(),
+                    },
+                ],
+            },
+            format="json",
+        )
+        self.assertEqual(resp.status_code, status.HTTP_200_OK)
+        self.assertEqual(CustomFieldInstance.objects.count(), 1)
+        self.assertEqual(Document.objects.count(), 1)
+        self.assertEqual(len(doc.custom_fields.all()), 1)
+        self.assertEqual(doc.custom_fields.first().value, date_value)
+
+    def test_custom_field_validation(self):
+        """
+        GIVEN:
+            - Document exists with no fields
+        WHEN:
+            - API request to remove a field
+            - API request is not valid
+        THEN:
+            - HTTP 400 is returned
+            - No field created
+            - No field attached to the document
+        """
+        doc = Document.objects.create(
+            title="WOW",
+            content="the content",
+            checksum="123",
+            mime_type="application/pdf",
+        )
+        custom_field_string = CustomField.objects.create(
+            name="Test Custom Field String",
+            data_type=CustomField.FieldDataType.STRING,
+        )
+
+        resp = self.client.patch(
+            f"/api/documents/{doc.id}/",
+            data={
+                "custom_fields": [
+                    {
+                        "field": custom_field_string.id,
+                        # Whoops, spelling
+                        "valeu": "a new test value",
+                    },
+                ],
+            },
+            format="json",
+        )
+        from pprint import pprint
+
+        pprint(resp.json())
+        self.assertEqual(resp.status_code, status.HTTP_400_BAD_REQUEST)
+        self.assertEqual(CustomFieldInstance.objects.count(), 0)
+        self.assertEqual(len(doc.custom_fields.all()), 0)
+
+    def test_custom_field_value_validation(self):
+        """
+        GIVEN:
+            - Document & custom field exist
+        WHEN:
+            - API request to set a field value
+        THEN:
+            - HTTP 400 is returned
+            - No field instance is created or attached to the document
+        """
+        doc = Document.objects.create(
+            title="WOW",
+            content="the content",
+            checksum="123",
+            mime_type="application/pdf",
+        )
+        custom_field_url = CustomField.objects.create(
+            name="Test Custom Field URL",
+            data_type=CustomField.FieldDataType.URL,
+        )
+        custom_field_int = CustomField.objects.create(
+            name="Test Custom Field INT",
+            data_type=CustomField.FieldDataType.INT,
+        )
+
+        resp = self.client.patch(
+            f"/api/documents/{doc.id}/",
+            data={
+                "custom_fields": [
+                    {
+                        "field": custom_field_url.id,
+                        "value": "not a url",
+                    },
+                ],
+            },
+            format="json",
+        )
+
+        self.assertEqual(resp.status_code, status.HTTP_400_BAD_REQUEST)
+        self.assertEqual(CustomFieldInstance.objects.count(), 0)
+        self.assertEqual(len(doc.custom_fields.all()), 0)
+
+        self.assertRaises(
+            Exception,
+            self.client.patch,
+            f"/api/documents/{doc.id}/",
+            data={
+                "custom_fields": [
+                    {
+                        "field": custom_field_int.id,
+                        "value": "not an int",
+                    },
+                ],
+            },
+            format="json",
+        )
+
+        self.assertEqual(CustomFieldInstance.objects.count(), 0)
+        self.assertEqual(len(doc.custom_fields.all()), 0)

src/documents/tests/test_management_exporter.py

@@ -153,7 +153,7 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
 
         manifest = self._do_export(use_filename_format=use_filename_format)
 
-        self.assertEqual(len(manifest), 159)
+        self.assertEqual(len(manifest), 169)
 
         # dont include consumer or AnonymousUser users
         self.assertEqual(
@@ -247,7 +247,7 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
             self.assertEqual(Document.objects.get(id=self.d4.id).title, "wow_dec")
             self.assertEqual(GroupObjectPermission.objects.count(), 1)
             self.assertEqual(UserObjectPermission.objects.count(), 1)
-            self.assertEqual(Permission.objects.count(), 116)
+            self.assertEqual(Permission.objects.count(), 124)
             messages = check_sanity()
             # everything is alright after the test
             self.assertEqual(len(messages), 0)
@@ -676,15 +676,15 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
             os.path.join(self.dirs.media_dir, "documents"),
         )
 
-        self.assertEqual(ContentType.objects.count(), 29)
-        self.assertEqual(Permission.objects.count(), 116)
+        self.assertEqual(ContentType.objects.count(), 31)
+        self.assertEqual(Permission.objects.count(), 124)
 
         manifest = self._do_export()
 
         with paperless_environment():
             self.assertEqual(
                 len(list(filter(lambda e: e["model"] == "auth.permission", manifest))),
-                116,
+                124,
             )
             # add 1 more to db to show objects are not re-created by import
             Permission.objects.create(
@@ -692,7 +692,7 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
                 codename="test_perm",
                 content_type_id=1,
             )
-            self.assertEqual(Permission.objects.count(), 117)
+            self.assertEqual(Permission.objects.count(), 125)
 
             # will cause an import error
             self.user.delete()
@@ -701,5 +701,5 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
             with self.assertRaises(IntegrityError):
                 call_command("document_importer", "--no-progress-bar", self.target)
 
-            self.assertEqual(ContentType.objects.count(), 29)
-            self.assertEqual(Permission.objects.count(), 117)
+            self.assertEqual(ContentType.objects.count(), 31)
+            self.assertEqual(Permission.objects.count(), 125)