paradizelost/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2025-10-30 03:56:23 -05:00
Code Issues Packages Projects Releases Wiki Activity

Enhancement: support assigning custom fields via consumption templates (#4727)

Browse Source
This commit is contained in:
shamoon
2023-12-03 15:35:30 -08:00
committed by GitHub
parent 28dbe96dc3
commit 084c31b37a
14 changed files with 260 additions and 136 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
docs/usage.md
View File

@@ -283,6 +283,7 @@ Consumption templates can assign:
- Tags, correspondent, document types
- Document owner
- View and / or edit permissions to users or groups
- Custom fields. Note that no value for the field will be set
### Consumption template permissions

1
src-ui/src/app/components/common/edit-dialog/consumption-template-edit-dialog/consumption-template-edit-dialog.component.html
View File

@@ -35,6 +35,7 @@
<pngx-input-select i18n-title title="Assign document type" [items]="documentTypes" [allowNull]="true" formControlName="assign_document_type"></pngx-input-select>
<pngx-input-select i18n-title title="Assign correspondent" [items]="correspondents" [allowNull]="true" formControlName="assign_correspondent"></pngx-input-select>
<pngx-input-select i18n-title title="Assign storage path" [items]="storagePaths" [allowNull]="true" formControlName="assign_storage_path"></pngx-input-select>
<pngx-input-select i18n-title title="Assign custom fields" multiple="true" [items]="customFields" [allowNull]="true" formControlName="assign_custom_fields"></pngx-input-select>
</div>
<div class="col">
<pngx-input-select i18n-title title="Assign owner" [items]="users" bindLabel="username" formControlName="assign_owner" [allowNull]="true"></pngx-input-select>

10
src-ui/src/app/components/common/edit-dialog/consumption-template-edit-dialog/consumption-template-edit-dialog.component.spec.ts
View File

@@ -20,6 +20,7 @@ import { TagsComponent } from '../../input/tags/tags.component'
import { TextComponent } from '../../input/text/text.component'
import { EditDialogMode } from '../edit-dialog.component'
import { ConsumptionTemplateEditDialogComponent } from './consumption-template-edit-dialog.component'
import { CustomFieldsService } from 'src/app/services/rest/custom-fields.service'
describe('ConsumptionTemplateEditDialogComponent', () => {
let component: ConsumptionTemplateEditDialogComponent
@@ -93,6 +94,15 @@ describe('ConsumptionTemplateEditDialogComponent', () => {
}),
},
},
{
provide: CustomFieldsService,
useValue: {
listAll: () =>
of({
results: [],
}),
},
},
],
imports: [
HttpClientTestingModule,

12
src-ui/src/app/components/common/edit-dialog/consumption-template-edit-dialog/consumption-template-edit-dialog.component.ts
View File

@@ -18,6 +18,8 @@ import { SettingsService } from 'src/app/services/settings.service'
import { EditDialogComponent } from '../edit-dialog.component'
import { MailRuleService } from 'src/app/services/rest/mail-rule.service'
import { PaperlessMailRule } from 'src/app/data/paperless-mail-rule'
import { CustomFieldsService } from 'src/app/services/rest/custom-fields.service'
import { PaperlessCustomField } from 'src/app/data/paperless-custom-field'
export const DOCUMENT_SOURCE_OPTIONS = [
{
@@ -45,6 +47,7 @@ export class ConsumptionTemplateEditDialogComponent extends EditDialogComponent<
documentTypes: PaperlessDocumentType[]
storagePaths: PaperlessStoragePath[]
mailRules: PaperlessMailRule[]
customFields: PaperlessCustomField[]
constructor(
service: ConsumptionTemplateService,
@@ -54,7 +57,8 @@ export class ConsumptionTemplateEditDialogComponent extends EditDialogComponent<
storagePathService: StoragePathService,
mailRuleService: MailRuleService,
userService: UserService,
settingsService: SettingsService
settingsService: SettingsService,
customFieldsService: CustomFieldsService
) {
super(service, activeModal, userService, settingsService)
@@ -77,6 +81,11 @@ export class ConsumptionTemplateEditDialogComponent extends EditDialogComponent<
.listAll()
.pipe(first())
.subscribe((result) => (this.mailRules = result.results))
customFieldsService
.listAll()
.pipe(first())
.subscribe((result) => (this.customFields = result.results))
}
getCreateTitle() {
@@ -106,6 +115,7 @@ export class ConsumptionTemplateEditDialogComponent extends EditDialogComponent<
assign_view_groups: new FormControl([]),
assign_change_users: new FormControl([]),
assign_change_groups: new FormControl([]),
assign_custom_fields: new FormControl([]),
})
}

2
src-ui/src/app/data/paperless-consumption-template.ts
View File

@@ -38,4 +38,6 @@ export interface PaperlessConsumptionTemplate extends ObjectWithId {
assign_change_users?: number[] // [PaperlessUser.id]
assign_change_groups?: number[] // [PaperlessGroup.id]
assign_custom_fields?: number[] // [PaperlessCustomField.id]
}

18
src/documents/consumer.py
View File

@@ -29,6 +29,8 @@ from documents.loggers import LoggingMixin
from documents.matching import document_matches_template
from documents.models import ConsumptionTemplate
from documents.models import Correspondent
from documents.models import CustomField
from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import DocumentType
from documents.models import FileInfo
@@ -124,6 +126,7 @@ class Consumer(LoggingMixin):
self.override_asn = None
self.task_id = None
self.override_owner_id = None
self.override_custom_field_ids = None
self.channel_layer = get_channel_layer()
@@ -333,6 +336,7 @@ class Consumer(LoggingMixin):
override_view_groups=None,
override_change_users=None,
override_change_groups=None,
override_custom_field_ids=None,
) -> Document:
"""
Return the document object if it was successfully created.
@@ -353,6 +357,7 @@ class Consumer(LoggingMixin):
self.override_view_groups = override_view_groups
self.override_change_users = override_change_users
self.override_change_groups = override_change_groups
self.override_custom_field_ids = override_custom_field_ids
self._send_progress(
0,
@@ -644,6 +649,11 @@ class Consumer(LoggingMixin):
template_overrides.change_groups = [
group.pk for group in template.assign_change_groups.all()
]
if template.assign_custom_fields is not None:
template_overrides.custom_field_ids = [
field.pk for field in template.assign_custom_fields.all()
]
overrides.update(template_overrides)
return overrides
@@ -782,6 +792,14 @@ class Consumer(LoggingMixin):
}
set_permissions_for_object(permissions=permissions, object=document)
if self.override_custom_field_ids:
for field_id in self.override_custom_field_ids:
field = CustomField.objects.get(pk=field_id)
CustomFieldInstance.objects.create(
field=field,
document=document,
) # adds to document
def _write(self, storage_type, source, target):
with open(source, "rb") as read_file, open(target, "wb") as write_file:
write_file.write(read_file.read())

7
src/documents/data_models.py
View File

@@ -28,6 +28,7 @@ class DocumentMetadataOverrides:
view_groups: Optional[list[int]] = None
change_users: Optional[list[int]] = None
change_groups: Optional[list[int]] = None
custom_field_ids: Optional[list[int]] = None
def update(self, other: "DocumentMetadataOverrides") -> "DocumentMetadataOverrides":
"""
@@ -74,6 +75,12 @@ class DocumentMetadataOverrides:
self.change_groups = other.change_groups
elif other.change_groups is not None:
self.change_groups.extend(other.change_groups)
if self.custom_field_ids is None:
self.custom_field_ids = other.custom_field_ids
elif other.custom_field_ids is not None:
self.custom_field_ids.extend(other.custom_field_ids)
return self

23
src/documents/migrations/1042_consumptiontemplate_assign_custom_fields.py Normal file
View File

@@ -0,0 +1,23 @@
# Generated by Django 4.2.7 on 2023-11-30 17:44
from django.db import migrations
from django.db import models
class Migration(migrations.Migration):
dependencies = [
("documents", "1041_alter_consumptiontemplate_sources"),
]
operations = [
migrations.AddField(
model_name="consumptiontemplate",
name="assign_custom_fields",
field=models.ManyToManyField(
blank=True,
related_name="+",
to="documents.customfield",
verbose_name="assign these custom fields",
),
),
]

275
src/documents/models.py
View File

@@ -743,140 +743,6 @@ class ShareLink(models.Model):
return f"Share Link for {self.document.title}"
class ConsumptionTemplate(models.Model):
class DocumentSourceChoices(models.IntegerChoices):
CONSUME_FOLDER = DocumentSource.ConsumeFolder.value, _("Consume Folder")
API_UPLOAD = DocumentSource.ApiUpload.value, _("Api Upload")
MAIL_FETCH = DocumentSource.MailFetch.value, _("Mail Fetch")
name = models.CharField(_("name"), max_length=256, unique=True)
order = models.IntegerField(_("order"), default=0)
sources = MultiSelectField(
max_length=5,
choices=DocumentSourceChoices.choices,
default=f"{DocumentSource.ConsumeFolder},{DocumentSource.ApiUpload},{DocumentSource.MailFetch}",
)
filter_path = models.CharField(
_("filter path"),
max_length=256,
null=True,
blank=True,
help_text=_(
"Only consume documents with a path that matches "
"this if specified. Wildcards specified as * are "
"allowed. Case insensitive.",
),
)
filter_filename = models.CharField(
_("filter filename"),
max_length=256,
null=True,
blank=True,
help_text=_(
"Only consume documents which entirely match this "
"filename if specified. Wildcards such as *.pdf or "
"*invoice* are allowed. Case insensitive.",
),
)
filter_mailrule = models.ForeignKey(
"paperless_mail.MailRule",
null=True,
blank=True,
on_delete=models.SET_NULL,
verbose_name=_("filter documents from this mail rule"),
)
assign_title = models.CharField(
_("assign title"),
max_length=256,
null=True,
blank=True,
help_text=_(
"Assign a document title, can include some placeholders, "
"see documentation.",
),
)
assign_tags = models.ManyToManyField(
Tag,
blank=True,
verbose_name=_("assign this tag"),
)
assign_document_type = models.ForeignKey(
DocumentType,
null=True,
blank=True,
on_delete=models.SET_NULL,
verbose_name=_("assign this document type"),
)
assign_correspondent = models.ForeignKey(
Correspondent,
null=True,
blank=True,
on_delete=models.SET_NULL,
verbose_name=_("assign this correspondent"),
)
assign_storage_path = models.ForeignKey(
StoragePath,
null=True,
blank=True,
on_delete=models.SET_NULL,
verbose_name=_("assign this storage path"),
)
assign_owner = models.ForeignKey(
User,
null=True,
blank=True,
on_delete=models.SET_NULL,
related_name="+",
verbose_name=_("assign this owner"),
)
assign_view_users = models.ManyToManyField(
User,
blank=True,
related_name="+",
verbose_name=_("grant view permissions to these users"),
)
assign_view_groups = models.ManyToManyField(
Group,
blank=True,
related_name="+",
verbose_name=_("grant view permissions to these groups"),
)
assign_change_users = models.ManyToManyField(
User,
blank=True,
related_name="+",
verbose_name=_("grant change permissions to these users"),
)
assign_change_groups = models.ManyToManyField(
Group,
blank=True,
related_name="+",
verbose_name=_("grant change permissions to these groups"),
)
class Meta:
verbose_name = _("consumption template")
verbose_name_plural = _("consumption templates")
def __str__(self):
return f"{self.name}"
class CustomField(models.Model):
"""
Defines the name and type of a custom field
@@ -1013,3 +879,144 @@ if settings.AUDIT_LOG_ENABLED:
auditlog.register(Note)
auditlog.register(CustomField)
auditlog.register(CustomFieldInstance)
class ConsumptionTemplate(models.Model):
class DocumentSourceChoices(models.IntegerChoices):
CONSUME_FOLDER = DocumentSource.ConsumeFolder.value, _("Consume Folder")
API_UPLOAD = DocumentSource.ApiUpload.value, _("Api Upload")
MAIL_FETCH = DocumentSource.MailFetch.value, _("Mail Fetch")
name = models.CharField(_("name"), max_length=256, unique=True)
order = models.IntegerField(_("order"), default=0)
sources = MultiSelectField(
max_length=5,
choices=DocumentSourceChoices.choices,
default=f"{DocumentSource.ConsumeFolder},{DocumentSource.ApiUpload},{DocumentSource.MailFetch}",
)
filter_path = models.CharField(
_("filter path"),
max_length=256,
null=True,
blank=True,
help_text=_(
"Only consume documents with a path that matches "
"this if specified. Wildcards specified as * are "
"allowed. Case insensitive.",
),
)
filter_filename = models.CharField(
_("filter filename"),
max_length=256,
null=True,
blank=True,
help_text=_(
"Only consume documents which entirely match this "
"filename if specified. Wildcards such as *.pdf or "
"*invoice* are allowed. Case insensitive.",
),
)
filter_mailrule = models.ForeignKey(
"paperless_mail.MailRule",
null=True,
blank=True,
on_delete=models.SET_NULL,
verbose_name=_("filter documents from this mail rule"),
)
assign_title = models.CharField(
_("assign title"),
max_length=256,
null=True,
blank=True,
help_text=_(
"Assign a document title, can include some placeholders, "
"see documentation.",
),
)
assign_tags = models.ManyToManyField(
Tag,
blank=True,
verbose_name=_("assign this tag"),
)
assign_document_type = models.ForeignKey(
DocumentType,
null=True,
blank=True,
on_delete=models.SET_NULL,
verbose_name=_("assign this document type"),
)
assign_correspondent = models.ForeignKey(
Correspondent,
null=True,
blank=True,
on_delete=models.SET_NULL,
verbose_name=_("assign this correspondent"),
)
assign_storage_path = models.ForeignKey(
StoragePath,
null=True,
blank=True,
on_delete=models.SET_NULL,
verbose_name=_("assign this storage path"),
)
assign_owner = models.ForeignKey(
User,
null=True,
blank=True,
on_delete=models.SET_NULL,
related_name="+",
verbose_name=_("assign this owner"),
)
assign_view_users = models.ManyToManyField(
User,
blank=True,
related_name="+",
verbose_name=_("grant view permissions to these users"),
)
assign_view_groups = models.ManyToManyField(
Group,
blank=True,
related_name="+",
verbose_name=_("grant view permissions to these groups"),
)
assign_change_users = models.ManyToManyField(
User,
blank=True,
related_name="+",
verbose_name=_("grant change permissions to these users"),
)
assign_change_groups = models.ManyToManyField(
Group,
blank=True,
related_name="+",
verbose_name=_("grant change permissions to these groups"),
)
assign_custom_fields = models.ManyToManyField(
CustomField,
blank=True,
related_name="+",
verbose_name=_("assign these custom fields"),
)
class Meta:
verbose_name = _("consumption template")
verbose_name_plural = _("consumption templates")
def __str__(self):
return f"{self.name}"

3
src/documents/serialisers.py
View File

@@ -429,7 +429,7 @@ class ReadWriteSerializerMethodField(serializers.SerializerMethodField):
class CustomFieldInstanceSerializer(serializers.ModelSerializer):
field = serializers.PrimaryKeyRelatedField(queryset=CustomField.objects.all())
value = ReadWriteSerializerMethodField()
value = ReadWriteSerializerMethodField(allow_null=True)
def create(self, validated_data):
type_to_data_store_name_map = {
@@ -1166,6 +1166,7 @@ class ConsumptionTemplateSerializer(serializers.ModelSerializer):
"assign_view_groups",
"assign_change_users",
"assign_change_groups",
"assign_custom_fields",
]
def validate(self, attrs):

1
src/documents/tasks.py
View File

@@ -179,6 +179,7 @@ def consume_file(
override_view_groups=overrides.view_groups,
override_change_users=overrides.change_users,
override_change_groups=overrides.change_groups,
override_custom_field_ids=overrides.custom_field_ids,
task_id=self.request.id,
)

7
src/documents/tests/test_api.py
View File

@@ -5649,6 +5649,11 @@ class TestApiConsumptionTemplates(DirectoriesMixin, APITestCase):
self.t2 = Tag.objects.create(name="t2")
self.t3 = Tag.objects.create(name="t3")
self.sp = StoragePath.objects.create(path="/test/")
self.cf1 = CustomField.objects.create(name="Custom Field 1", data_type="string")
self.cf2 = CustomField.objects.create(
name="Custom Field 2",
data_type="integer",
)
self.ct = ConsumptionTemplate.objects.create(
name="Template 1",
@@ -5669,6 +5674,8 @@ class TestApiConsumptionTemplates(DirectoriesMixin, APITestCase):
self.ct.assign_view_groups.add(self.group1.pk)
self.ct.assign_change_users.add(self.user3.pk)
self.ct.assign_change_groups.add(self.group1.pk)
self.ct.assign_custom_fields.add(self.cf1.pk)
self.ct.assign_custom_fields.add(self.cf2.pk)
self.ct.save()
def test_api_get_consumption_template(self):

24
src/documents/tests/test_consumer.py
View File

@@ -22,6 +22,7 @@ from documents.consumer import Consumer
from documents.consumer import ConsumerError
from documents.consumer import ConsumerFilePhase
from documents.models import Correspondent
from documents.models import CustomField
from documents.models import Document
from documents.models import DocumentType
from documents.models import FileInfo
@@ -458,6 +459,29 @@ class TestConsumer(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
self.assertIn(t3, document.tags.all())
self._assert_first_last_send_progress()
def testOverrideCustomFields(self):
cf1 = CustomField.objects.create(name="Custom Field 1", data_type="string")
cf2 = CustomField.objects.create(
name="Custom Field 2",
data_type="integer",
)
cf3 = CustomField.objects.create(
name="Custom Field 3",
data_type="url",
)
document = self.consumer.try_consume_file(
self.get_test_file(),
override_custom_field_ids=[cf1.id, cf3.id],
)
fields_used = [
field_instance.field for field_instance in document.custom_fields.all()
]
self.assertIn(cf1, fields_used)
self.assertNotIn(cf2, fields_used)
self.assertIn(cf3, fields_used)
self._assert_first_last_send_progress()
def testOverrideAsn(self):
document = self.consumer.try_consume_file(
self.get_test_file(),

12
src/documents/tests/test_consumption_templates.py
View File

@@ -11,6 +11,7 @@ from documents.data_models import ConsumableDocument
from documents.data_models import DocumentSource
from documents.models import ConsumptionTemplate
from documents.models import Correspondent
from documents.models import CustomField
from documents.models import DocumentType
from documents.models import StoragePath
from documents.models import Tag
@@ -32,6 +33,11 @@ class TestConsumptionTemplates(DirectoriesMixin, FileSystemAssertsMixin, TestCas
self.t2 = Tag.objects.create(name="t2")
self.t3 = Tag.objects.create(name="t3")
self.sp = StoragePath.objects.create(path="/test/")
self.cf1 = CustomField.objects.create(name="Custom Field 1", data_type="string")
self.cf2 = CustomField.objects.create(
name="Custom Field 2",
data_type="integer",
)
self.user2 = User.objects.create(username="user2")
self.user3 = User.objects.create(username="user3")
@@ -95,6 +101,8 @@ class TestConsumptionTemplates(DirectoriesMixin, FileSystemAssertsMixin, TestCas
ct.assign_view_groups.add(self.group1.pk)
ct.assign_change_users.add(self.user3.pk)
ct.assign_change_groups.add(self.group1.pk)
ct.assign_custom_fields.add(self.cf1.pk)
ct.assign_custom_fields.add(self.cf2.pk)
ct.save()
self.assertEqual(ct.__str__(), "Template 1")
@@ -128,6 +136,10 @@ class TestConsumptionTemplates(DirectoriesMixin, FileSystemAssertsMixin, TestCas
overrides["override_title"],
"Doc from {correspondent}",
)
self.assertEqual(
overrides["override_custom_field_ids"],
[self.cf1.pk, self.cf2.pk],
)
info = cm.output[0]
expected_str = f"Document matched template {ct}"