Enhancement: support assigning custom fields via consumption templates (#4727)

src/documents/consumer.py

@@ -29,6 +29,8 @@ from documents.loggers import LoggingMixin
 from documents.matching import document_matches_template
 from documents.models import ConsumptionTemplate
 from documents.models import Correspondent
+from documents.models import CustomField
+from documents.models import CustomFieldInstance
 from documents.models import Document
 from documents.models import DocumentType
 from documents.models import FileInfo
@@ -124,6 +126,7 @@ class Consumer(LoggingMixin):
         self.override_asn = None
         self.task_id = None
         self.override_owner_id = None
+        self.override_custom_field_ids = None
 
         self.channel_layer = get_channel_layer()
 
@@ -333,6 +336,7 @@ class Consumer(LoggingMixin):
         override_view_groups=None,
         override_change_users=None,
         override_change_groups=None,
+        override_custom_field_ids=None,
     ) -> Document:
         """
         Return the document object if it was successfully created.
@@ -353,6 +357,7 @@ class Consumer(LoggingMixin):
         self.override_view_groups = override_view_groups
         self.override_change_users = override_change_users
         self.override_change_groups = override_change_groups
+        self.override_custom_field_ids = override_custom_field_ids
 
         self._send_progress(
             0,
@@ -644,6 +649,11 @@ class Consumer(LoggingMixin):
                     template_overrides.change_groups = [
                         group.pk for group in template.assign_change_groups.all()
                     ]
+                if template.assign_custom_fields is not None:
+                    template_overrides.custom_field_ids = [
+                        field.pk for field in template.assign_custom_fields.all()
+                    ]
+
                 overrides.update(template_overrides)
         return overrides
 
@@ -782,6 +792,14 @@ class Consumer(LoggingMixin):
             }
             set_permissions_for_object(permissions=permissions, object=document)
 
+        if self.override_custom_field_ids:
+            for field_id in self.override_custom_field_ids:
+                field = CustomField.objects.get(pk=field_id)
+                CustomFieldInstance.objects.create(
+                    field=field,
+                    document=document,
+                )  # adds to document
+
     def _write(self, storage_type, source, target):
         with open(source, "rb") as read_file, open(target, "wb") as write_file:
             write_file.write(read_file.read())

src/documents/data_models.py

@@ -28,6 +28,7 @@ class DocumentMetadataOverrides:
     view_groups: Optional[list[int]] = None
     change_users: Optional[list[int]] = None
     change_groups: Optional[list[int]] = None
+    custom_field_ids: Optional[list[int]] = None
 
     def update(self, other: "DocumentMetadataOverrides") -> "DocumentMetadataOverrides":
         """
@@ -74,6 +75,12 @@ class DocumentMetadataOverrides:
             self.change_groups = other.change_groups
         elif other.change_groups is not None:
             self.change_groups.extend(other.change_groups)
+
+        if self.custom_field_ids is None:
+            self.custom_field_ids = other.custom_field_ids
+        elif other.custom_field_ids is not None:
+            self.custom_field_ids.extend(other.custom_field_ids)
+
         return self
 
 

src/documents/migrations/1042_consumptiontemplate_assign_custom_fields.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.2.7 on 2023-11-30 17:44
+
+from django.db import migrations
+from django.db import models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("documents", "1041_alter_consumptiontemplate_sources"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="consumptiontemplate",
+            name="assign_custom_fields",
+            field=models.ManyToManyField(
+                blank=True,
+                related_name="+",
+                to="documents.customfield",
+                verbose_name="assign these custom fields",
+            ),
+        ),
+    ]

src/documents/models.py

@@ -743,140 +743,6 @@ class ShareLink(models.Model):
         return f"Share Link for {self.document.title}"
 
 
-class ConsumptionTemplate(models.Model):
-    class DocumentSourceChoices(models.IntegerChoices):
-        CONSUME_FOLDER = DocumentSource.ConsumeFolder.value, _("Consume Folder")
-        API_UPLOAD = DocumentSource.ApiUpload.value, _("Api Upload")
-        MAIL_FETCH = DocumentSource.MailFetch.value, _("Mail Fetch")
-
-    name = models.CharField(_("name"), max_length=256, unique=True)
-
-    order = models.IntegerField(_("order"), default=0)
-
-    sources = MultiSelectField(
-        max_length=5,
-        choices=DocumentSourceChoices.choices,
-        default=f"{DocumentSource.ConsumeFolder},{DocumentSource.ApiUpload},{DocumentSource.MailFetch}",
-    )
-
-    filter_path = models.CharField(
-        _("filter path"),
-        max_length=256,
-        null=True,
-        blank=True,
-        help_text=_(
-            "Only consume documents with a path that matches "
-            "this if specified. Wildcards specified as * are "
-            "allowed. Case insensitive.",
-        ),
-    )
-
-    filter_filename = models.CharField(
-        _("filter filename"),
-        max_length=256,
-        null=True,
-        blank=True,
-        help_text=_(
-            "Only consume documents which entirely match this "
-            "filename if specified. Wildcards such as *.pdf or "
-            "*invoice* are allowed. Case insensitive.",
-        ),
-    )
-
-    filter_mailrule = models.ForeignKey(
-        "paperless_mail.MailRule",
-        null=True,
-        blank=True,
-        on_delete=models.SET_NULL,
-        verbose_name=_("filter documents from this mail rule"),
-    )
-
-    assign_title = models.CharField(
-        _("assign title"),
-        max_length=256,
-        null=True,
-        blank=True,
-        help_text=_(
-            "Assign a document title, can include some placeholders, "
-            "see documentation.",
-        ),
-    )
-
-    assign_tags = models.ManyToManyField(
-        Tag,
-        blank=True,
-        verbose_name=_("assign this tag"),
-    )
-
-    assign_document_type = models.ForeignKey(
-        DocumentType,
-        null=True,
-        blank=True,
-        on_delete=models.SET_NULL,
-        verbose_name=_("assign this document type"),
-    )
-
-    assign_correspondent = models.ForeignKey(
-        Correspondent,
-        null=True,
-        blank=True,
-        on_delete=models.SET_NULL,
-        verbose_name=_("assign this correspondent"),
-    )
-
-    assign_storage_path = models.ForeignKey(
-        StoragePath,
-        null=True,
-        blank=True,
-        on_delete=models.SET_NULL,
-        verbose_name=_("assign this storage path"),
-    )
-
-    assign_owner = models.ForeignKey(
-        User,
-        null=True,
-        blank=True,
-        on_delete=models.SET_NULL,
-        related_name="+",
-        verbose_name=_("assign this owner"),
-    )
-
-    assign_view_users = models.ManyToManyField(
-        User,
-        blank=True,
-        related_name="+",
-        verbose_name=_("grant view permissions to these users"),
-    )
-
-    assign_view_groups = models.ManyToManyField(
-        Group,
-        blank=True,
-        related_name="+",
-        verbose_name=_("grant view permissions to these groups"),
-    )
-
-    assign_change_users = models.ManyToManyField(
-        User,
-        blank=True,
-        related_name="+",
-        verbose_name=_("grant change permissions to these users"),
-    )
-
-    assign_change_groups = models.ManyToManyField(
-        Group,
-        blank=True,
-        related_name="+",
-        verbose_name=_("grant change permissions to these groups"),
-    )
-
-    class Meta:
-        verbose_name = _("consumption template")
-        verbose_name_plural = _("consumption templates")
-
-    def __str__(self):
-        return f"{self.name}"
-
-
 class CustomField(models.Model):
     """
     Defines the name and type of a custom field
@@ -1013,3 +879,144 @@ if settings.AUDIT_LOG_ENABLED:
     auditlog.register(Note)
     auditlog.register(CustomField)
     auditlog.register(CustomFieldInstance)
+
+
+class ConsumptionTemplate(models.Model):
+    class DocumentSourceChoices(models.IntegerChoices):
+        CONSUME_FOLDER = DocumentSource.ConsumeFolder.value, _("Consume Folder")
+        API_UPLOAD = DocumentSource.ApiUpload.value, _("Api Upload")
+        MAIL_FETCH = DocumentSource.MailFetch.value, _("Mail Fetch")
+
+    name = models.CharField(_("name"), max_length=256, unique=True)
+
+    order = models.IntegerField(_("order"), default=0)
+
+    sources = MultiSelectField(
+        max_length=5,
+        choices=DocumentSourceChoices.choices,
+        default=f"{DocumentSource.ConsumeFolder},{DocumentSource.ApiUpload},{DocumentSource.MailFetch}",
+    )
+
+    filter_path = models.CharField(
+        _("filter path"),
+        max_length=256,
+        null=True,
+        blank=True,
+        help_text=_(
+            "Only consume documents with a path that matches "
+            "this if specified. Wildcards specified as * are "
+            "allowed. Case insensitive.",
+        ),
+    )
+
+    filter_filename = models.CharField(
+        _("filter filename"),
+        max_length=256,
+        null=True,
+        blank=True,
+        help_text=_(
+            "Only consume documents which entirely match this "
+            "filename if specified. Wildcards such as *.pdf or "
+            "*invoice* are allowed. Case insensitive.",
+        ),
+    )
+
+    filter_mailrule = models.ForeignKey(
+        "paperless_mail.MailRule",
+        null=True,
+        blank=True,
+        on_delete=models.SET_NULL,
+        verbose_name=_("filter documents from this mail rule"),
+    )
+
+    assign_title = models.CharField(
+        _("assign title"),
+        max_length=256,
+        null=True,
+        blank=True,
+        help_text=_(
+            "Assign a document title, can include some placeholders, "
+            "see documentation.",
+        ),
+    )
+
+    assign_tags = models.ManyToManyField(
+        Tag,
+        blank=True,
+        verbose_name=_("assign this tag"),
+    )
+
+    assign_document_type = models.ForeignKey(
+        DocumentType,
+        null=True,
+        blank=True,
+        on_delete=models.SET_NULL,
+        verbose_name=_("assign this document type"),
+    )
+
+    assign_correspondent = models.ForeignKey(
+        Correspondent,
+        null=True,
+        blank=True,
+        on_delete=models.SET_NULL,
+        verbose_name=_("assign this correspondent"),
+    )
+
+    assign_storage_path = models.ForeignKey(
+        StoragePath,
+        null=True,
+        blank=True,
+        on_delete=models.SET_NULL,
+        verbose_name=_("assign this storage path"),
+    )
+
+    assign_owner = models.ForeignKey(
+        User,
+        null=True,
+        blank=True,
+        on_delete=models.SET_NULL,
+        related_name="+",
+        verbose_name=_("assign this owner"),
+    )
+
+    assign_view_users = models.ManyToManyField(
+        User,
+        blank=True,
+        related_name="+",
+        verbose_name=_("grant view permissions to these users"),
+    )
+
+    assign_view_groups = models.ManyToManyField(
+        Group,
+        blank=True,
+        related_name="+",
+        verbose_name=_("grant view permissions to these groups"),
+    )
+
+    assign_change_users = models.ManyToManyField(
+        User,
+        blank=True,
+        related_name="+",
+        verbose_name=_("grant change permissions to these users"),
+    )
+
+    assign_change_groups = models.ManyToManyField(
+        Group,
+        blank=True,
+        related_name="+",
+        verbose_name=_("grant change permissions to these groups"),
+    )
+
+    assign_custom_fields = models.ManyToManyField(
+        CustomField,
+        blank=True,
+        related_name="+",
+        verbose_name=_("assign these custom fields"),
+    )
+
+    class Meta:
+        verbose_name = _("consumption template")
+        verbose_name_plural = _("consumption templates")
+
+    def __str__(self):
+        return f"{self.name}"

src/documents/serialisers.py

@@ -429,7 +429,7 @@ class ReadWriteSerializerMethodField(serializers.SerializerMethodField):
 
 class CustomFieldInstanceSerializer(serializers.ModelSerializer):
     field = serializers.PrimaryKeyRelatedField(queryset=CustomField.objects.all())
-    value = ReadWriteSerializerMethodField()
+    value = ReadWriteSerializerMethodField(allow_null=True)
 
     def create(self, validated_data):
         type_to_data_store_name_map = {
@@ -1166,6 +1166,7 @@ class ConsumptionTemplateSerializer(serializers.ModelSerializer):
             "assign_view_groups",
             "assign_change_users",
             "assign_change_groups",
+            "assign_custom_fields",
         ]
 
     def validate(self, attrs):

src/documents/tasks.py

@@ -179,6 +179,7 @@ def consume_file(
         override_view_groups=overrides.view_groups,
         override_change_users=overrides.change_users,
         override_change_groups=overrides.change_groups,
+        override_custom_field_ids=overrides.custom_field_ids,
         task_id=self.request.id,
     )
 

src/documents/tests/test_api.py

@@ -5649,6 +5649,11 @@ class TestApiConsumptionTemplates(DirectoriesMixin, APITestCase):
         self.t2 = Tag.objects.create(name="t2")
         self.t3 = Tag.objects.create(name="t3")
         self.sp = StoragePath.objects.create(path="/test/")
+        self.cf1 = CustomField.objects.create(name="Custom Field 1", data_type="string")
+        self.cf2 = CustomField.objects.create(
+            name="Custom Field 2",
+            data_type="integer",
+        )
 
         self.ct = ConsumptionTemplate.objects.create(
             name="Template 1",
@@ -5669,6 +5674,8 @@ class TestApiConsumptionTemplates(DirectoriesMixin, APITestCase):
         self.ct.assign_view_groups.add(self.group1.pk)
         self.ct.assign_change_users.add(self.user3.pk)
         self.ct.assign_change_groups.add(self.group1.pk)
+        self.ct.assign_custom_fields.add(self.cf1.pk)
+        self.ct.assign_custom_fields.add(self.cf2.pk)
         self.ct.save()
 
     def test_api_get_consumption_template(self):

src/documents/tests/test_consumer.py

@@ -22,6 +22,7 @@ from documents.consumer import Consumer
 from documents.consumer import ConsumerError
 from documents.consumer import ConsumerFilePhase
 from documents.models import Correspondent
+from documents.models import CustomField
 from documents.models import Document
 from documents.models import DocumentType
 from documents.models import FileInfo
@@ -458,6 +459,29 @@ class TestConsumer(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
         self.assertIn(t3, document.tags.all())
         self._assert_first_last_send_progress()
 
+    def testOverrideCustomFields(self):
+        cf1 = CustomField.objects.create(name="Custom Field 1", data_type="string")
+        cf2 = CustomField.objects.create(
+            name="Custom Field 2",
+            data_type="integer",
+        )
+        cf3 = CustomField.objects.create(
+            name="Custom Field 3",
+            data_type="url",
+        )
+        document = self.consumer.try_consume_file(
+            self.get_test_file(),
+            override_custom_field_ids=[cf1.id, cf3.id],
+        )
+
+        fields_used = [
+            field_instance.field for field_instance in document.custom_fields.all()
+        ]
+        self.assertIn(cf1, fields_used)
+        self.assertNotIn(cf2, fields_used)
+        self.assertIn(cf3, fields_used)
+        self._assert_first_last_send_progress()
+
     def testOverrideAsn(self):
         document = self.consumer.try_consume_file(
             self.get_test_file(),

src/documents/tests/test_consumption_templates.py

@@ -11,6 +11,7 @@ from documents.data_models import ConsumableDocument
 from documents.data_models import DocumentSource
 from documents.models import ConsumptionTemplate
 from documents.models import Correspondent
+from documents.models import CustomField
 from documents.models import DocumentType
 from documents.models import StoragePath
 from documents.models import Tag
@@ -32,6 +33,11 @@ class TestConsumptionTemplates(DirectoriesMixin, FileSystemAssertsMixin, TestCas
         self.t2 = Tag.objects.create(name="t2")
         self.t3 = Tag.objects.create(name="t3")
         self.sp = StoragePath.objects.create(path="/test/")
+        self.cf1 = CustomField.objects.create(name="Custom Field 1", data_type="string")
+        self.cf2 = CustomField.objects.create(
+            name="Custom Field 2",
+            data_type="integer",
+        )
 
         self.user2 = User.objects.create(username="user2")
         self.user3 = User.objects.create(username="user3")
@@ -95,6 +101,8 @@ class TestConsumptionTemplates(DirectoriesMixin, FileSystemAssertsMixin, TestCas
         ct.assign_view_groups.add(self.group1.pk)
         ct.assign_change_users.add(self.user3.pk)
         ct.assign_change_groups.add(self.group1.pk)
+        ct.assign_custom_fields.add(self.cf1.pk)
+        ct.assign_custom_fields.add(self.cf2.pk)
         ct.save()
 
         self.assertEqual(ct.__str__(), "Template 1")
@@ -128,6 +136,10 @@ class TestConsumptionTemplates(DirectoriesMixin, FileSystemAssertsMixin, TestCas
                     overrides["override_title"],
                     "Doc from {correspondent}",
                 )
+                self.assertEqual(
+                    overrides["override_custom_field_ids"],
+                    [self.cf1.pk, self.cf2.pk],
+                )
 
         info = cm.output[0]
         expected_str = f"Document matched template {ct}"