Enhancement: support assigning custom fields via consumption templates (#4727)

docs/usage.md

@@ -283,6 +283,7 @@ Consumption templates can assign:
 - Tags, correspondent, document types
 - Document owner
 - View and / or edit permissions to users or groups
+- Custom fields. Note that no value for the field will be set
 
 ### Consumption template permissions
 

src-ui/src/app/components/common/edit-dialog/consumption-template-edit-dialog/consumption-template-edit-dialog.component.html

@@ -35,6 +35,7 @@
               <pngx-input-select i18n-title title="Assign document type" [items]="documentTypes" [allowNull]="true" formControlName="assign_document_type"></pngx-input-select>
               <pngx-input-select i18n-title title="Assign correspondent" [items]="correspondents" [allowNull]="true" formControlName="assign_correspondent"></pngx-input-select>
               <pngx-input-select i18n-title title="Assign storage path" [items]="storagePaths" [allowNull]="true" formControlName="assign_storage_path"></pngx-input-select>
+              <pngx-input-select i18n-title title="Assign custom fields" multiple="true" [items]="customFields" [allowNull]="true" formControlName="assign_custom_fields"></pngx-input-select>
           </div>
           <div class="col">
             <pngx-input-select i18n-title title="Assign owner" [items]="users" bindLabel="username" formControlName="assign_owner" [allowNull]="true"></pngx-input-select>

src-ui/src/app/components/common/edit-dialog/consumption-template-edit-dialog/consumption-template-edit-dialog.component.spec.ts

@@ -20,6 +20,7 @@ import { TagsComponent } from '../../input/tags/tags.component'
 import { TextComponent } from '../../input/text/text.component'
 import { EditDialogMode } from '../edit-dialog.component'
 import { ConsumptionTemplateEditDialogComponent } from './consumption-template-edit-dialog.component'
+import { CustomFieldsService } from 'src/app/services/rest/custom-fields.service'
 
 describe('ConsumptionTemplateEditDialogComponent', () => {
   let component: ConsumptionTemplateEditDialogComponent
@@ -93,6 +94,15 @@ describe('ConsumptionTemplateEditDialogComponent', () => {
               }),
           },
         },
+        {
+          provide: CustomFieldsService,
+          useValue: {
+            listAll: () =>
+              of({
+                results: [],
+              }),
+          },
+        },
       ],
       imports: [
         HttpClientTestingModule,

src-ui/src/app/components/common/edit-dialog/consumption-template-edit-dialog/consumption-template-edit-dialog.component.ts

@@ -18,6 +18,8 @@ import { SettingsService } from 'src/app/services/settings.service'
 import { EditDialogComponent } from '../edit-dialog.component'
 import { MailRuleService } from 'src/app/services/rest/mail-rule.service'
 import { PaperlessMailRule } from 'src/app/data/paperless-mail-rule'
+import { CustomFieldsService } from 'src/app/services/rest/custom-fields.service'
+import { PaperlessCustomField } from 'src/app/data/paperless-custom-field'
 
 export const DOCUMENT_SOURCE_OPTIONS = [
   {
@@ -45,6 +47,7 @@ export class ConsumptionTemplateEditDialogComponent extends EditDialogComponent<
   documentTypes: PaperlessDocumentType[]
   storagePaths: PaperlessStoragePath[]
   mailRules: PaperlessMailRule[]
+  customFields: PaperlessCustomField[]
 
   constructor(
     service: ConsumptionTemplateService,
@@ -54,7 +57,8 @@ export class ConsumptionTemplateEditDialogComponent extends EditDialogComponent<
     storagePathService: StoragePathService,
     mailRuleService: MailRuleService,
     userService: UserService,
-    settingsService: SettingsService
+    settingsService: SettingsService,
+    customFieldsService: CustomFieldsService
   ) {
     super(service, activeModal, userService, settingsService)
 
@@ -77,6 +81,11 @@ export class ConsumptionTemplateEditDialogComponent extends EditDialogComponent<
       .listAll()
       .pipe(first())
       .subscribe((result) => (this.mailRules = result.results))
+
+    customFieldsService
+      .listAll()
+      .pipe(first())
+      .subscribe((result) => (this.customFields = result.results))
   }
 
   getCreateTitle() {
@@ -106,6 +115,7 @@ export class ConsumptionTemplateEditDialogComponent extends EditDialogComponent<
       assign_view_groups: new FormControl([]),
       assign_change_users: new FormControl([]),
       assign_change_groups: new FormControl([]),
+      assign_custom_fields: new FormControl([]),
     })
   }
 

src-ui/src/app/data/paperless-consumption-template.ts

@@ -38,4 +38,6 @@ export interface PaperlessConsumptionTemplate extends ObjectWithId {
   assign_change_users?: number[] // [PaperlessUser.id]
 
   assign_change_groups?: number[] // [PaperlessGroup.id]
+
+  assign_custom_fields?: number[] // [PaperlessCustomField.id]
 }

src/documents/consumer.py

@@ -29,6 +29,8 @@ from documents.loggers import LoggingMixin
 from documents.matching import document_matches_template
 from documents.models import ConsumptionTemplate
 from documents.models import Correspondent
+from documents.models import CustomField
+from documents.models import CustomFieldInstance
 from documents.models import Document
 from documents.models import DocumentType
 from documents.models import FileInfo
@@ -124,6 +126,7 @@ class Consumer(LoggingMixin):
         self.override_asn = None
         self.task_id = None
         self.override_owner_id = None
+        self.override_custom_field_ids = None
 
         self.channel_layer = get_channel_layer()
 
@@ -333,6 +336,7 @@ class Consumer(LoggingMixin):
         override_view_groups=None,
         override_change_users=None,
         override_change_groups=None,
+        override_custom_field_ids=None,
     ) -> Document:
         """
         Return the document object if it was successfully created.
@@ -353,6 +357,7 @@ class Consumer(LoggingMixin):
         self.override_view_groups = override_view_groups
         self.override_change_users = override_change_users
         self.override_change_groups = override_change_groups
+        self.override_custom_field_ids = override_custom_field_ids
 
         self._send_progress(
             0,
@@ -644,6 +649,11 @@ class Consumer(LoggingMixin):
                     template_overrides.change_groups = [
                         group.pk for group in template.assign_change_groups.all()
                     ]
+                if template.assign_custom_fields is not None:
+                    template_overrides.custom_field_ids = [
+                        field.pk for field in template.assign_custom_fields.all()
+                    ]
+
                 overrides.update(template_overrides)
         return overrides
 
@@ -782,6 +792,14 @@ class Consumer(LoggingMixin):
             }
             set_permissions_for_object(permissions=permissions, object=document)
 
+        if self.override_custom_field_ids:
+            for field_id in self.override_custom_field_ids:
+                field = CustomField.objects.get(pk=field_id)
+                CustomFieldInstance.objects.create(
+                    field=field,
+                    document=document,
+                )  # adds to document
+
     def _write(self, storage_type, source, target):
         with open(source, "rb") as read_file, open(target, "wb") as write_file:
             write_file.write(read_file.read())

src/documents/data_models.py

@@ -28,6 +28,7 @@ class DocumentMetadataOverrides:
     view_groups: Optional[list[int]] = None
     change_users: Optional[list[int]] = None
     change_groups: Optional[list[int]] = None
+    custom_field_ids: Optional[list[int]] = None
 
     def update(self, other: "DocumentMetadataOverrides") -> "DocumentMetadataOverrides":
         """
@@ -74,6 +75,12 @@ class DocumentMetadataOverrides:
             self.change_groups = other.change_groups
         elif other.change_groups is not None:
             self.change_groups.extend(other.change_groups)
+
+        if self.custom_field_ids is None:
+            self.custom_field_ids = other.custom_field_ids
+        elif other.custom_field_ids is not None:
+            self.custom_field_ids.extend(other.custom_field_ids)
+
         return self
 
 

src/documents/migrations/1042_consumptiontemplate_assign_custom_fields.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.2.7 on 2023-11-30 17:44
+
+from django.db import migrations
+from django.db import models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("documents", "1041_alter_consumptiontemplate_sources"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="consumptiontemplate",
+            name="assign_custom_fields",
+            field=models.ManyToManyField(
+                blank=True,
+                related_name="+",
+                to="documents.customfield",
+                verbose_name="assign these custom fields",
+            ),
+        ),
+    ]

src/documents/models.py

@@ -743,140 +743,6 @@ class ShareLink(models.Model):
         return f"Share Link for {self.document.title}"
 
 
-class ConsumptionTemplate(models.Model):
-    class DocumentSourceChoices(models.IntegerChoices):
-        CONSUME_FOLDER = DocumentSource.ConsumeFolder.value, _("Consume Folder")
-        API_UPLOAD = DocumentSource.ApiUpload.value, _("Api Upload")
-        MAIL_FETCH = DocumentSource.MailFetch.value, _("Mail Fetch")
-
-    name = models.CharField(_("name"), max_length=256, unique=True)
-
-    order = models.IntegerField(_("order"), default=0)
-
-    sources = MultiSelectField(
-        max_length=5,
-        choices=DocumentSourceChoices.choices,
-        default=f"{DocumentSource.ConsumeFolder},{DocumentSource.ApiUpload},{DocumentSource.MailFetch}",
-    )
-
-    filter_path = models.CharField(
-        _("filter path"),
-        max_length=256,
-        null=True,
-        blank=True,
-        help_text=_(
-            "Only consume documents with a path that matches "
-            "this if specified. Wildcards specified as * are "
-            "allowed. Case insensitive.",
-        ),
-    )
-
-    filter_filename = models.CharField(
-        _("filter filename"),
-        max_length=256,
-        null=True,
-        blank=True,
-        help_text=_(
-            "Only consume documents which entirely match this "
-            "filename if specified. Wildcards such as *.pdf or "
-            "*invoice* are allowed. Case insensitive.",
-        ),
-    )
-
-    filter_mailrule = models.ForeignKey(
-        "paperless_mail.MailRule",
-        null=True,
-        blank=True,
-        on_delete=models.SET_NULL,
-        verbose_name=_("filter documents from this mail rule"),
-    )
-
-    assign_title = models.CharField(
-        _("assign title"),
-        max_length=256,
-        null=True,
-        blank=True,
-        help_text=_(
-            "Assign a document title, can include some placeholders, "
-            "see documentation.",
-        ),
-    )
-
-    assign_tags = models.ManyToManyField(
-        Tag,
-        blank=True,
-        verbose_name=_("assign this tag"),
-    )
-
-    assign_document_type = models.ForeignKey(
-        DocumentType,
-        null=True,
-        blank=True,
-        on_delete=models.SET_NULL,
-        verbose_name=_("assign this document type"),
-    )
-
-    assign_correspondent = models.ForeignKey(
-        Correspondent,
-        null=True,
-        blank=True,
-        on_delete=models.SET_NULL,
-        verbose_name=_("assign this correspondent"),
-    )
-
-    assign_storage_path = models.ForeignKey(
-        StoragePath,
-        null=True,
-        blank=True,
-        on_delete=models.SET_NULL,
-        verbose_name=_("assign this storage path"),
-    )
-
-    assign_owner = models.ForeignKey(
-        User,
-        null=True,
-        blank=True,
-        on_delete=models.SET_NULL,
-        related_name="+",
-        verbose_name=_("assign this owner"),
-    )
-
-    assign_view_users = models.ManyToManyField(
-        User,
-        blank=True,
-        related_name="+",
-        verbose_name=_("grant view permissions to these users"),
-    )
-
-    assign_view_groups = models.ManyToManyField(
-        Group,
-        blank=True,
-        related_name="+",
-        verbose_name=_("grant view permissions to these groups"),
-    )
-
-    assign_change_users = models.ManyToManyField(
-        User,
-        blank=True,
-        related_name="+",
-        verbose_name=_("grant change permissions to these users"),
-    )
-
-    assign_change_groups = models.ManyToManyField(
-        Group,
-        blank=True,
-        related_name="+",
-        verbose_name=_("grant change permissions to these groups"),
-    )
-
-    class Meta:
-        verbose_name = _("consumption template")
-        verbose_name_plural = _("consumption templates")
-
-    def __str__(self):
-        return f"{self.name}"
-
-
 class CustomField(models.Model):
     """
     Defines the name and type of a custom field
@@ -1013,3 +879,144 @@ if settings.AUDIT_LOG_ENABLED:
     auditlog.register(Note)
     auditlog.register(CustomField)
     auditlog.register(CustomFieldInstance)
+
+
+class ConsumptionTemplate(models.Model):
+    class DocumentSourceChoices(models.IntegerChoices):
+        CONSUME_FOLDER = DocumentSource.ConsumeFolder.value, _("Consume Folder")
+        API_UPLOAD = DocumentSource.ApiUpload.value, _("Api Upload")
+        MAIL_FETCH = DocumentSource.MailFetch.value, _("Mail Fetch")
+
+    name = models.CharField(_("name"), max_length=256, unique=True)
+
+    order = models.IntegerField(_("order"), default=0)
+
+    sources = MultiSelectField(
+        max_length=5,
+        choices=DocumentSourceChoices.choices,
+        default=f"{DocumentSource.ConsumeFolder},{DocumentSource.ApiUpload},{DocumentSource.MailFetch}",
+    )
+
+    filter_path = models.CharField(
+        _("filter path"),
+        max_length=256,
+        null=True,
+        blank=True,
+        help_text=_(
+            "Only consume documents with a path that matches "
+            "this if specified. Wildcards specified as * are "
+            "allowed. Case insensitive.",
+        ),
+    )
+
+    filter_filename = models.CharField(
+        _("filter filename"),
+        max_length=256,
+        null=True,
+        blank=True,
+        help_text=_(
+            "Only consume documents which entirely match this "
+            "filename if specified. Wildcards such as *.pdf or "
+            "*invoice* are allowed. Case insensitive.",
+        ),
+    )
+
+    filter_mailrule = models.ForeignKey(
+        "paperless_mail.MailRule",
+        null=True,
+        blank=True,
+        on_delete=models.SET_NULL,
+        verbose_name=_("filter documents from this mail rule"),
+    )
+
+    assign_title = models.CharField(
+        _("assign title"),
+        max_length=256,
+        null=True,
+        blank=True,
+        help_text=_(
+            "Assign a document title, can include some placeholders, "
+            "see documentation.",
+        ),
+    )
+
+    assign_tags = models.ManyToManyField(
+        Tag,
+        blank=True,
+        verbose_name=_("assign this tag"),
+    )
+
+    assign_document_type = models.ForeignKey(
+        DocumentType,
+        null=True,
+        blank=True,
+        on_delete=models.SET_NULL,
+        verbose_name=_("assign this document type"),
+    )
+
+    assign_correspondent = models.ForeignKey(
+        Correspondent,
+        null=True,
+        blank=True,
+        on_delete=models.SET_NULL,
+        verbose_name=_("assign this correspondent"),
+    )
+
+    assign_storage_path = models.ForeignKey(
+        StoragePath,
+        null=True,
+        blank=True,
+        on_delete=models.SET_NULL,
+        verbose_name=_("assign this storage path"),
+    )
+
+    assign_owner = models.ForeignKey(
+        User,
+        null=True,
+        blank=True,
+        on_delete=models.SET_NULL,
+        related_name="+",
+        verbose_name=_("assign this owner"),
+    )
+
+    assign_view_users = models.ManyToManyField(
+        User,
+        blank=True,
+        related_name="+",
+        verbose_name=_("grant view permissions to these users"),
+    )
+
+    assign_view_groups = models.ManyToManyField(
+        Group,
+        blank=True,
+        related_name="+",
+        verbose_name=_("grant view permissions to these groups"),
+    )
+
+    assign_change_users = models.ManyToManyField(
+        User,
+        blank=True,
+        related_name="+",
+        verbose_name=_("grant change permissions to these users"),
+    )
+
+    assign_change_groups = models.ManyToManyField(
+        Group,
+        blank=True,
+        related_name="+",
+        verbose_name=_("grant change permissions to these groups"),
+    )
+
+    assign_custom_fields = models.ManyToManyField(
+        CustomField,
+        blank=True,
+        related_name="+",
+        verbose_name=_("assign these custom fields"),
+    )
+
+    class Meta:
+        verbose_name = _("consumption template")
+        verbose_name_plural = _("consumption templates")
+
+    def __str__(self):
+        return f"{self.name}"

src/documents/serialisers.py

@@ -429,7 +429,7 @@ class ReadWriteSerializerMethodField(serializers.SerializerMethodField):
 
 class CustomFieldInstanceSerializer(serializers.ModelSerializer):
     field = serializers.PrimaryKeyRelatedField(queryset=CustomField.objects.all())
-    value = ReadWriteSerializerMethodField()
+    value = ReadWriteSerializerMethodField(allow_null=True)
 
     def create(self, validated_data):
         type_to_data_store_name_map = {
@@ -1166,6 +1166,7 @@ class ConsumptionTemplateSerializer(serializers.ModelSerializer):
             "assign_view_groups",
             "assign_change_users",
             "assign_change_groups",
+            "assign_custom_fields",
         ]
 
     def validate(self, attrs):

src/documents/tasks.py

@@ -179,6 +179,7 @@ def consume_file(
         override_view_groups=overrides.view_groups,
         override_change_users=overrides.change_users,
         override_change_groups=overrides.change_groups,
+        override_custom_field_ids=overrides.custom_field_ids,
         task_id=self.request.id,
     )
 

src/documents/tests/test_api.py

@@ -5649,6 +5649,11 @@ class TestApiConsumptionTemplates(DirectoriesMixin, APITestCase):
         self.t2 = Tag.objects.create(name="t2")
         self.t3 = Tag.objects.create(name="t3")
         self.sp = StoragePath.objects.create(path="/test/")
+        self.cf1 = CustomField.objects.create(name="Custom Field 1", data_type="string")
+        self.cf2 = CustomField.objects.create(
+            name="Custom Field 2",
+            data_type="integer",
+        )
 
         self.ct = ConsumptionTemplate.objects.create(
             name="Template 1",
@@ -5669,6 +5674,8 @@ class TestApiConsumptionTemplates(DirectoriesMixin, APITestCase):
         self.ct.assign_view_groups.add(self.group1.pk)
         self.ct.assign_change_users.add(self.user3.pk)
         self.ct.assign_change_groups.add(self.group1.pk)
+        self.ct.assign_custom_fields.add(self.cf1.pk)
+        self.ct.assign_custom_fields.add(self.cf2.pk)
         self.ct.save()
 
     def test_api_get_consumption_template(self):

src/documents/tests/test_consumer.py

@@ -22,6 +22,7 @@ from documents.consumer import Consumer
 from documents.consumer import ConsumerError
 from documents.consumer import ConsumerFilePhase
 from documents.models import Correspondent
+from documents.models import CustomField
 from documents.models import Document
 from documents.models import DocumentType
 from documents.models import FileInfo
@@ -458,6 +459,29 @@ class TestConsumer(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
         self.assertIn(t3, document.tags.all())
         self._assert_first_last_send_progress()
 
+    def testOverrideCustomFields(self):
+        cf1 = CustomField.objects.create(name="Custom Field 1", data_type="string")
+        cf2 = CustomField.objects.create(
+            name="Custom Field 2",
+            data_type="integer",
+        )
+        cf3 = CustomField.objects.create(
+            name="Custom Field 3",
+            data_type="url",
+        )
+        document = self.consumer.try_consume_file(
+            self.get_test_file(),
+            override_custom_field_ids=[cf1.id, cf3.id],
+        )
+
+        fields_used = [
+            field_instance.field for field_instance in document.custom_fields.all()
+        ]
+        self.assertIn(cf1, fields_used)
+        self.assertNotIn(cf2, fields_used)
+        self.assertIn(cf3, fields_used)
+        self._assert_first_last_send_progress()
+
     def testOverrideAsn(self):
         document = self.consumer.try_consume_file(
             self.get_test_file(),

src/documents/tests/test_consumption_templates.py

@@ -11,6 +11,7 @@ from documents.data_models import ConsumableDocument
 from documents.data_models import DocumentSource
 from documents.models import ConsumptionTemplate
 from documents.models import Correspondent
+from documents.models import CustomField
 from documents.models import DocumentType
 from documents.models import StoragePath
 from documents.models import Tag
@@ -32,6 +33,11 @@ class TestConsumptionTemplates(DirectoriesMixin, FileSystemAssertsMixin, TestCas
         self.t2 = Tag.objects.create(name="t2")
         self.t3 = Tag.objects.create(name="t3")
         self.sp = StoragePath.objects.create(path="/test/")
+        self.cf1 = CustomField.objects.create(name="Custom Field 1", data_type="string")
+        self.cf2 = CustomField.objects.create(
+            name="Custom Field 2",
+            data_type="integer",
+        )
 
         self.user2 = User.objects.create(username="user2")
         self.user3 = User.objects.create(username="user3")
@@ -95,6 +101,8 @@ class TestConsumptionTemplates(DirectoriesMixin, FileSystemAssertsMixin, TestCas
         ct.assign_view_groups.add(self.group1.pk)
         ct.assign_change_users.add(self.user3.pk)
         ct.assign_change_groups.add(self.group1.pk)
+        ct.assign_custom_fields.add(self.cf1.pk)
+        ct.assign_custom_fields.add(self.cf2.pk)
         ct.save()
 
         self.assertEqual(ct.__str__(), "Template 1")
@@ -128,6 +136,10 @@ class TestConsumptionTemplates(DirectoriesMixin, FileSystemAssertsMixin, TestCas
                     overrides["override_title"],
                     "Doc from {correspondent}",
                 )
+                self.assertEqual(
+                    overrides["override_custom_field_ids"],
+                    [self.cf1.pk, self.cf2.pk],
+                )
 
         info = cm.output[0]
         expected_str = f"Document matched template {ct}"