paradizelost/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2025-07-28 18:24:38 -05:00
Code Issues Packages Projects Releases Wiki Activity

Enhancement: support assigning custom field values in workflows (#9272)

Browse Source
This commit is contained in:
shamoon
2025-03-05 12:30:19 -08:00
committed by GitHub
parent 89e5c08a1f
commit edc7181843
20 changed files with 605 additions and 175 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
src/documents/consumer.py
View File

@@ -806,13 +806,19 @@ class ConsumerPlugin(
}
set_permissions_for_object(permissions=permissions, object=document)
if self.metadata.custom_field_ids:
for field_id in self.metadata.custom_field_ids:
field = CustomField.objects.get(pk=field_id)
CustomFieldInstance.objects.create(
field=field,
document=document,
) # adds to document
if self.metadata.custom_fields:
for field in CustomField.objects.filter(
id__in=self.metadata.custom_fields.keys(),
).distinct():
value_field_name = CustomFieldInstance.get_value_field_name(
data_type=field.data_type,
)
args = {
"field": field,
"document": document,
value_field_name: self.metadata.custom_fields.get(field.id, None),
}
CustomFieldInstance.objects.create(**args) # adds to document
def _write(self, storage_type, source, target):
with (

18
src/documents/data_models.py
View File

@@ -29,7 +29,7 @@ class DocumentMetadataOverrides:
view_groups: list[int] | None = None
change_users: list[int] | None = None
change_groups: list[int] | None = None
custom_field_ids: list[int] | None = None
custom_fields: dict | None = None
def update(self, other: "DocumentMetadataOverrides") -> "DocumentMetadataOverrides":
"""
@@ -81,11 +81,10 @@ class DocumentMetadataOverrides:
self.change_groups.extend(other.change_groups)
self.change_groups = list(set(self.change_groups))
if self.custom_field_ids is None:
self.custom_field_ids = other.custom_field_ids
elif other.custom_field_ids is not None:
self.custom_field_ids.extend(other.custom_field_ids)
self.custom_field_ids = list(set(self.custom_field_ids))
if self.custom_fields is None:
self.custom_fields = other.custom_fields
elif other.custom_fields is not None:
self.custom_fields.update(other.custom_fields)
return self
@@ -114,9 +113,10 @@ class DocumentMetadataOverrides:
only_with_perms_in=["change_document"],
).values_list("id", flat=True),
)
overrides.custom_field_ids = list(
doc.custom_fields.values_list("field", flat=True),
)
overrides.custom_fields = {
custom_field.id: custom_field.value
for custom_field in doc.custom_fields.all()
}
groups_with_perms = get_groups_with_perms(
doc,

24
src/documents/migrations/1065_workflowaction_assign_custom_fields_values.py Normal file
View File

@@ -0,0 +1,24 @@
# Generated by Django 5.1.6 on 2025-03-01 18:10
from django.db import migrations
from django.db import models
class Migration(migrations.Migration):
dependencies = [
("documents", "1064_delete_log"),
]
operations = [
migrations.AddField(
model_name="workflowaction",
name="assign_custom_fields_values",
field=models.JSONField(
blank=True,
help_text="Optional values to assign to the custom fields.",
null=True,
verbose_name="custom field values",
default=dict,
),
),
]

10
src/documents/models.py
View File

@@ -1271,6 +1271,16 @@ class WorkflowAction(models.Model):
verbose_name=_("assign these custom fields"),
)
assign_custom_fields_values = models.JSONField(
_("custom field values"),
null=True,
blank=True,
help_text=_(
"Optional values to assign to the custom fields.",
),
default=dict,
)
remove_tags = models.ManyToManyField(
Tag,
blank=True,

1
src/documents/serialisers.py
View File

@@ -2018,6 +2018,7 @@ class WorkflowActionSerializer(serializers.ModelSerializer):
"assign_change_users",
"assign_change_groups",
"assign_custom_fields",
"assign_custom_fields_values",
"remove_all_tags",
"remove_tags",
"remove_all_correspondents",

45
src/documents/signals/handlers.py
View File

@@ -770,23 +770,40 @@ def run_workflows(
if action.assign_custom_fields.exists():
if not use_overrides:
for field in action.assign_custom_fields.all():
if not CustomFieldInstance.objects.filter(
value_field_name = CustomFieldInstance.get_value_field_name(
data_type=field.data_type,
)
args = {
value_field_name: action.assign_custom_fields_values.get(
str(field.pk),
None,
),
}
# for some reason update_or_create doesn't work here
instance = CustomFieldInstance.objects.filter(
field=field,
document=document,
).exists():
# can be triggered on existing docs, so only add the field if it doesn't already exist
).first()
if instance:
setattr(instance, value_field_name, args[value_field_name])
instance.save()
else:
CustomFieldInstance.objects.create(
**args,
field=field,
document=document,
)
else:
overrides.custom_field_ids = list(
set(
(overrides.custom_field_ids or [])
+ list(
action.assign_custom_fields.values_list("pk", flat=True),
),
),
if overrides.custom_fields is None:
overrides.custom_fields = {}
overrides.custom_fields.update(
{
field.pk: action.assign_custom_fields_values.get(
str(field.pk),
None,
)
for field in action.assign_custom_fields.all()
},
)
def removal_action():
@@ -944,18 +961,18 @@ def run_workflows(
if not use_overrides:
CustomFieldInstance.objects.filter(document=document).delete()
else:
overrides.custom_field_ids = None
overrides.custom_fields = None
elif action.remove_custom_fields.exists():
if not use_overrides:
CustomFieldInstance.objects.filter(
field__in=action.remove_custom_fields.all(),
document=document,
).delete()
elif overrides.custom_field_ids:
elif overrides.custom_fields:
for field in action.remove_custom_fields.filter(
pk__in=overrides.custom_field_ids,
pk__in=overrides.custom_fields.keys(),
):
overrides.custom_field_ids.remove(field.pk)
overrides.custom_fields.pop(field.pk, None)
def email_action():
if not settings.EMAIL_ENABLED:

68
src/documents/tests/test_api_documents.py
View File

@@ -28,6 +28,7 @@ from documents.caching import CACHE_50_MINUTES
from documents.caching import CLASSIFIER_HASH_KEY
from documents.caching import CLASSIFIER_MODIFIED_KEY
from documents.caching import CLASSIFIER_VERSION_KEY
from documents.data_models import DocumentSource
from documents.models import Correspondent
from documents.models import CustomField
from documents.models import CustomFieldInstance
@@ -39,7 +40,10 @@ from documents.models import SavedView
from documents.models import ShareLink
from documents.models import StoragePath
from documents.models import Tag
from documents.models import Workflow
from documents.models import WorkflowAction
from documents.models import WorkflowTrigger
from documents.signals.handlers import run_workflows
from documents.tests.utils import DirectoriesMixin
from documents.tests.utils import DocumentConsumeDelayMixin
@@ -1362,7 +1366,69 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
self.assertEqual(input_doc.original_file.name, "simple.pdf")
self.assertEqual(overrides.filename, "simple.pdf")
self.assertEqual(overrides.custom_field_ids, [custom_field.id])
self.assertEqual(overrides.custom_fields, {custom_field.id: None})
def test_upload_with_custom_fields_and_workflow(self):
"""
GIVEN: A document with a source file
WHEN: Upload the document with custom fields and a workflow
THEN: Metadata is set correctly, mimicking what happens in the real consumer plugin
"""
self.consume_file_mock.return_value = celery.result.AsyncResult(
id=str(uuid.uuid4()),
)
cf = CustomField.objects.create(
name="stringfield",
data_type=CustomField.FieldDataType.STRING,
)
cf2 = CustomField.objects.create(
name="intfield",
data_type=CustomField.FieldDataType.INT,
)
trigger1 = WorkflowTrigger.objects.create(
type=WorkflowTrigger.WorkflowTriggerType.CONSUMPTION,
sources=f"{DocumentSource.ApiUpload},{DocumentSource.ConsumeFolder},{DocumentSource.MailFetch}",
)
action1 = WorkflowAction.objects.create(
assign_title="Doc title",
)
action1.assign_custom_fields.add(cf2)
action1.assign_custom_fields_values = {cf2.id: 123}
action1.save()
w1 = Workflow.objects.create(
name="Workflow 1",
order=0,
)
w1.triggers.add(trigger1)
w1.actions.add(action1)
w1.save()
with (Path(__file__).parent / "samples" / "simple.pdf").open("rb") as f:
response = self.client.post(
"/api/documents/post_document/",
{
"document": f,
"custom_fields": [cf.id],
},
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.consume_file_mock.assert_called_once()
input_doc, overrides = self.get_last_consume_delay_call_args()
new_overrides, msg = run_workflows(
trigger_type=WorkflowTrigger.WorkflowTriggerType.CONSUMPTION,
document=input_doc,
logging_group=None,
overrides=overrides,
)
overrides.update(new_overrides)
self.assertEqual(overrides.custom_fields, {cf.id: None, cf2.id: 123})
def test_upload_with_webui_source(self):
"""

9
src/documents/tests/test_consumer.py
View File

@@ -408,7 +408,9 @@ class TestConsumer(
with self.get_consumer(
self.get_test_file(),
DocumentMetadataOverrides(custom_field_ids=[cf1.id, cf3.id]),
DocumentMetadataOverrides(
custom_fields={cf1.id: "value1", cf3.id: "http://example.com"},
),
) as consumer:
consumer.run()
@@ -420,6 +422,11 @@ class TestConsumer(
self.assertIn(cf1, fields_used)
self.assertNotIn(cf2, fields_used)
self.assertIn(cf3, fields_used)
self.assertEqual(document.custom_fields.get(field=cf1).value, "value1")
self.assertEqual(
document.custom_fields.get(field=cf3).value,
"http://example.com",
)
self._assert_first_last_send_progress()
def testOverrideAsn(self):

16
src/documents/tests/test_workflows.py
View File

@@ -133,6 +133,9 @@ class TestWorkflows(
action.assign_change_groups.add(self.group1.pk)
action.assign_custom_fields.add(self.cf1.pk)
action.assign_custom_fields.add(self.cf2.pk)
action.assign_custom_fields_values = {
self.cf2.pk: 42,
}
action.save()
w = Workflow.objects.create(
name="Workflow 1",
@@ -209,6 +212,10 @@ class TestWorkflows(
list(document.custom_fields.all().values_list("field", flat=True)),
[self.cf1.pk, self.cf2.pk],
)
self.assertEqual(
document.custom_fields.get(field=self.cf2.pk).value,
42,
)
info = cm.output[0]
expected_str = f"Document matched {trigger} from {w}"
@@ -1215,11 +1222,11 @@ class TestWorkflows(
def test_document_updated_workflow_existing_custom_field(self):
"""
GIVEN:
- Existing workflow with UPDATED trigger and action that adds a custom field
- Existing workflow with UPDATED trigger and action that assigns a custom field with a value
WHEN:
- Document is updated that already contains the field
THEN:
- Document update succeeds without trying to re-create the field
- Document update succeeds and updates the field
"""
trigger = WorkflowTrigger.objects.create(
type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_UPDATED,
@@ -1227,6 +1234,8 @@ class TestWorkflows(
)
action = WorkflowAction.objects.create()
action.assign_custom_fields.add(self.cf1)
action.assign_custom_fields_values = {self.cf1.pk: "new value"}
action.save()
w = Workflow.objects.create(
name="Workflow 1",
order=0,
@@ -1251,6 +1260,9 @@ class TestWorkflows(
format="json",
)
doc.refresh_from_db()
self.assertEqual(doc.custom_fields.get(field=self.cf1).value, "new value")
def test_document_updated_workflow_merge_permissions(self):
"""
GIVEN:

5
src/documents/views.py
View File

@@ -1471,7 +1471,10 @@ class PostDocumentView(GenericAPIView):
created=created,
asn=archive_serial_number,
owner_id=request.user.id,
custom_field_ids=custom_field_ids,
# TODO: set values
custom_fields={cf_id: None for cf_id in custom_field_ids}
if custom_field_ids
else None,
)
async_task = consume_file.delay(

170
src/locale/en_US/LC_MESSAGES/django.po
View File

@@ -2,7 +2,7 @@ msgid ""
msgstr ""
"Project-Id-Version: paperless-ngx\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2025-02-25 11:07-0800\n"
"POT-Creation-Date: 2025-03-01 21:03-0800\n"
"PO-Revision-Date: 2022-02-17 04:17\n"
"Last-Translator: \n"
"Language-Team: English\n"
@@ -21,39 +21,39 @@ msgstr ""
msgid "Documents"
msgstr ""
#: documents/filters.py:370
#: documents/filters.py:375
msgid "Value must be valid JSON."
msgstr ""
#: documents/filters.py:389
#: documents/filters.py:394
msgid "Invalid custom field query expression"
msgstr ""
#: documents/filters.py:399
#: documents/filters.py:404
msgid "Invalid expression list. Must be nonempty."
msgstr ""
#: documents/filters.py:420
#: documents/filters.py:425
msgid "Invalid logical operator {op!r}"
msgstr ""
#: documents/filters.py:434
#: documents/filters.py:439
msgid "Maximum number of query conditions exceeded."
msgstr ""
#: documents/filters.py:499
#: documents/filters.py:504
msgid "{name!r} is not a valid custom field."
msgstr ""
#: documents/filters.py:536
#: documents/filters.py:541
msgid "{data_type} does not support query expr {expr!r}."
msgstr ""
#: documents/filters.py:644
#: documents/filters.py:649
msgid "Maximum nesting depth exceeded."
msgstr ""
#: documents/filters.py:829
#: documents/filters.py:834
msgid "Custom field not found"
msgstr ""
@@ -89,7 +89,7 @@ msgstr ""
msgid "Automatic"
msgstr ""
#: documents/models.py:67 documents/models.py:433 documents/models.py:1526
#: documents/models.py:67 documents/models.py:433 documents/models.py:1536
#: paperless_mail/models.py:23 paperless_mail/models.py:143
msgid "name"
msgstr ""
@@ -256,7 +256,7 @@ msgid "The position of this document in your physical document archive."
msgstr ""
#: documents/models.py:295 documents/models.py:761 documents/models.py:815
#: documents/models.py:1569
#: documents/models.py:1579
msgid "document"
msgstr ""
@@ -1088,141 +1088,149 @@ msgstr ""
msgid "assign these custom fields"
msgstr ""
#: documents/models.py:1398
#: documents/models.py:1395
msgid "custom field values"
msgstr ""
#: documents/models.py:1399
msgid "Optional values to assign to the custom fields."
msgstr ""
#: documents/models.py:1408
msgid "remove these tag(s)"
msgstr ""
#: documents/models.py:1403
#: documents/models.py:1413
msgid "remove all tags"
msgstr ""
#: documents/models.py:1410
#: documents/models.py:1420
msgid "remove these document type(s)"
msgstr ""
#: documents/models.py:1415
#: documents/models.py:1425
msgid "remove all document types"
msgstr ""
#: documents/models.py:1422
#: documents/models.py:1432
msgid "remove these correspondent(s)"
msgstr ""
#: documents/models.py:1427
#: documents/models.py:1437
msgid "remove all correspondents"
msgstr ""
#: documents/models.py:1434
#: documents/models.py:1444
msgid "remove these storage path(s)"
msgstr ""
#: documents/models.py:1439
#: documents/models.py:1449
msgid "remove all storage paths"
msgstr ""
#: documents/models.py:1446
#: documents/models.py:1456
msgid "remove these owner(s)"
msgstr ""
#: documents/models.py:1451
#: documents/models.py:1461
msgid "remove all owners"
msgstr ""
#: documents/models.py:1458
#: documents/models.py:1468
msgid "remove view permissions for these users"
msgstr ""
#: documents/models.py:1465
#: documents/models.py:1475
msgid "remove view permissions for these groups"
msgstr ""
#: documents/models.py:1472
#: documents/models.py:1482
msgid "remove change permissions for these users"
msgstr ""
#: documents/models.py:1479
#: documents/models.py:1489
msgid "remove change permissions for these groups"
msgstr ""
#: documents/models.py:1484
#: documents/models.py:1494
msgid "remove all permissions"
msgstr ""
#: documents/models.py:1491
#: documents/models.py:1501
msgid "remove these custom fields"
msgstr ""
#: documents/models.py:1496
#: documents/models.py:1506
msgid "remove all custom fields"
msgstr ""
#: documents/models.py:1505
#: documents/models.py:1515
msgid "email"
msgstr ""
#: documents/models.py:1514
#: documents/models.py:1524
msgid "webhook"
msgstr ""
#: documents/models.py:1518
#: documents/models.py:1528
msgid "workflow action"
msgstr ""
#: documents/models.py:1519
#: documents/models.py:1529
msgid "workflow actions"
msgstr ""
#: documents/models.py:1528 paperless_mail/models.py:145
#: documents/models.py:1538 paperless_mail/models.py:145
msgid "order"
msgstr ""
#: documents/models.py:1534
#: documents/models.py:1544
msgid "triggers"
msgstr ""
#: documents/models.py:1541
#: documents/models.py:1551
msgid "actions"
msgstr ""
#: documents/models.py:1544 paperless_mail/models.py:154
#: documents/models.py:1554 paperless_mail/models.py:154
msgid "enabled"
msgstr ""
#: documents/models.py:1555
#: documents/models.py:1565
msgid "workflow"
msgstr ""
#: documents/models.py:1559
#: documents/models.py:1569
msgid "workflow trigger type"
msgstr ""
#: documents/models.py:1573
#: documents/models.py:1583
msgid "date run"
msgstr ""
#: documents/models.py:1579
#: documents/models.py:1589
msgid "workflow run"
msgstr ""
#: documents/models.py:1580
#: documents/models.py:1590
msgid "workflow runs"
msgstr ""
#: documents/serialisers.py:128
#: documents/serialisers.py:134
#, python-format
msgid "Invalid regular expression: %(error)s"
msgstr ""
#: documents/serialisers.py:554
#: documents/serialisers.py:560
msgid "Invalid color."
msgstr ""
#: documents/serialisers.py:1570
#: documents/serialisers.py:1576
#, python-format
msgid "File type %(type)s not supported"
msgstr ""
#: documents/serialisers.py:1659
#: documents/serialisers.py:1665
msgid "Invalid variable detected."
msgstr ""
@@ -1463,7 +1471,7 @@ msgstr ""
msgid "Unable to parse URI {value}"
msgstr ""
#: paperless/apps.py:10
#: paperless/apps.py:11
msgid "Paperless"
msgstr ""
@@ -1611,139 +1619,139 @@ msgstr ""
msgid "paperless application settings"
msgstr ""
#: paperless/settings.py:721
#: paperless/settings.py:724
msgid "English (US)"
msgstr ""
#: paperless/settings.py:722
#: paperless/settings.py:725
msgid "Arabic"
msgstr ""
#: paperless/settings.py:723
#: paperless/settings.py:726
msgid "Afrikaans"
msgstr ""
#: paperless/settings.py:724
#: paperless/settings.py:727
msgid "Belarusian"
msgstr ""
#: paperless/settings.py:725
#: paperless/settings.py:728
msgid "Bulgarian"
msgstr ""
#: paperless/settings.py:726
#: paperless/settings.py:729
msgid "Catalan"
msgstr ""
#: paperless/settings.py:727
#: paperless/settings.py:730
msgid "Czech"
msgstr ""
#: paperless/settings.py:728
#: paperless/settings.py:731
msgid "Danish"
msgstr ""
#: paperless/settings.py:729
#: paperless/settings.py:732
msgid "German"
msgstr ""
#: paperless/settings.py:730
#: paperless/settings.py:733
msgid "Greek"
msgstr ""
#: paperless/settings.py:731
#: paperless/settings.py:734
msgid "English (GB)"
msgstr ""
#: paperless/settings.py:732
#: paperless/settings.py:735
msgid "Spanish"
msgstr ""
#: paperless/settings.py:733
#: paperless/settings.py:736
msgid "Finnish"
msgstr ""
#: paperless/settings.py:734
#: paperless/settings.py:737
msgid "French"
msgstr ""
#: paperless/settings.py:735
#: paperless/settings.py:738
msgid "Hungarian"
msgstr ""
#: paperless/settings.py:736
#: paperless/settings.py:739
msgid "Italian"
msgstr ""
#: paperless/settings.py:737
#: paperless/settings.py:740
msgid "Japanese"
msgstr ""
#: paperless/settings.py:738
#: paperless/settings.py:741
msgid "Korean"
msgstr ""
#: paperless/settings.py:739
#: paperless/settings.py:742
msgid "Luxembourgish"
msgstr ""
#: paperless/settings.py:740
#: paperless/settings.py:743
msgid "Norwegian"
msgstr ""
#: paperless/settings.py:741
#: paperless/settings.py:744
msgid "Dutch"
msgstr ""
#: paperless/settings.py:742
#: paperless/settings.py:745
msgid "Polish"
msgstr ""
#: paperless/settings.py:743
#: paperless/settings.py:746
msgid "Portuguese (Brazil)"
msgstr ""
#: paperless/settings.py:744
#: paperless/settings.py:747
msgid "Portuguese"
msgstr ""
#: paperless/settings.py:745
#: paperless/settings.py:748
msgid "Romanian"
msgstr ""
#: paperless/settings.py:746
#: paperless/settings.py:749
msgid "Russian"
msgstr ""
#: paperless/settings.py:747
#: paperless/settings.py:750
msgid "Slovak"
msgstr ""
#: paperless/settings.py:748
#: paperless/settings.py:751
msgid "Slovenian"
msgstr ""
#: paperless/settings.py:749
#: paperless/settings.py:752
msgid "Serbian"
msgstr ""
#: paperless/settings.py:750
#: paperless/settings.py:753
msgid "Swedish"
msgstr ""
#: paperless/settings.py:751
#: paperless/settings.py:754
msgid "Turkish"
msgstr ""
#: paperless/settings.py:752
#: paperless/settings.py:755
msgid "Ukrainian"
msgstr ""
#: paperless/settings.py:753
#: paperless/settings.py:756
msgid "Chinese Simplified"
msgstr ""
#: paperless/settings.py:754
#: paperless/settings.py:757
msgid "Chinese Traditional"
msgstr ""