paradizelost/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2025-07-30 18:27:45 -05:00
Code Issues Packages Projects Releases Wiki Activity

Enhancement: support assigning custom field values in workflows (#9272)

Browse Source
This commit is contained in:
shamoon
2025-03-05 12:30:19 -08:00
committed by GitHub
parent 89e5c08a1f
commit edc7181843
20 changed files with 605 additions and 175 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
src/documents/consumer.py
View File

@@ -806,13 +806,19 @@ class ConsumerPlugin(
}
set_permissions_for_object(permissions=permissions, object=document)
if self.metadata.custom_field_ids:
for field_id in self.metadata.custom_field_ids:
field = CustomField.objects.get(pk=field_id)
CustomFieldInstance.objects.create(
field=field,
document=document,
) # adds to document
if self.metadata.custom_fields:
for field in CustomField.objects.filter(
id__in=self.metadata.custom_fields.keys(),
).distinct():
value_field_name = CustomFieldInstance.get_value_field_name(
data_type=field.data_type,
)
args = {
"field": field,
"document": document,
value_field_name: self.metadata.custom_fields.get(field.id, None),
}
CustomFieldInstance.objects.create(**args) # adds to document
def _write(self, storage_type, source, target):
with (

18
src/documents/data_models.py
View File

@@ -29,7 +29,7 @@ class DocumentMetadataOverrides:
view_groups: list[int] | None = None
change_users: list[int] | None = None
change_groups: list[int] | None = None
custom_field_ids: list[int] | None = None
custom_fields: dict | None = None
def update(self, other: "DocumentMetadataOverrides") -> "DocumentMetadataOverrides":
"""
@@ -81,11 +81,10 @@ class DocumentMetadataOverrides:
self.change_groups.extend(other.change_groups)
self.change_groups = list(set(self.change_groups))
if self.custom_field_ids is None:
self.custom_field_ids = other.custom_field_ids
elif other.custom_field_ids is not None:
self.custom_field_ids.extend(other.custom_field_ids)
self.custom_field_ids = list(set(self.custom_field_ids))
if self.custom_fields is None:
self.custom_fields = other.custom_fields
elif other.custom_fields is not None:
self.custom_fields.update(other.custom_fields)
return self
@@ -114,9 +113,10 @@ class DocumentMetadataOverrides:
only_with_perms_in=["change_document"],
).values_list("id", flat=True),
)
overrides.custom_field_ids = list(
doc.custom_fields.values_list("field", flat=True),
)
overrides.custom_fields = {
custom_field.id: custom_field.value
for custom_field in doc.custom_fields.all()
}
groups_with_perms = get_groups_with_perms(
doc,

24
src/documents/migrations/1065_workflowaction_assign_custom_fields_values.py Normal file
View File

@@ -0,0 +1,24 @@
# Generated by Django 5.1.6 on 2025-03-01 18:10
from django.db import migrations
from django.db import models
class Migration(migrations.Migration):
dependencies = [
("documents", "1064_delete_log"),
]
operations = [
migrations.AddField(
model_name="workflowaction",
name="assign_custom_fields_values",
field=models.JSONField(
blank=True,
help_text="Optional values to assign to the custom fields.",
null=True,
verbose_name="custom field values",
default=dict,
),
),
]

10
src/documents/models.py
View File

@@ -1271,6 +1271,16 @@ class WorkflowAction(models.Model):
verbose_name=_("assign these custom fields"),
)
assign_custom_fields_values = models.JSONField(
_("custom field values"),
null=True,
blank=True,
help_text=_(
"Optional values to assign to the custom fields.",
),
default=dict,
)
remove_tags = models.ManyToManyField(
Tag,
blank=True,

1
src/documents/serialisers.py
View File

@@ -2018,6 +2018,7 @@ class WorkflowActionSerializer(serializers.ModelSerializer):
"assign_change_users",
"assign_change_groups",
"assign_custom_fields",
"assign_custom_fields_values",
"remove_all_tags",
"remove_tags",
"remove_all_correspondents",

45
src/documents/signals/handlers.py
View File

@@ -770,23 +770,40 @@ def run_workflows(
if action.assign_custom_fields.exists():
if not use_overrides:
for field in action.assign_custom_fields.all():
if not CustomFieldInstance.objects.filter(
value_field_name = CustomFieldInstance.get_value_field_name(
data_type=field.data_type,
)
args = {
value_field_name: action.assign_custom_fields_values.get(
str(field.pk),
None,
),
}
# for some reason update_or_create doesn't work here
instance = CustomFieldInstance.objects.filter(
field=field,
document=document,
).exists():
# can be triggered on existing docs, so only add the field if it doesn't already exist
).first()
if instance:
setattr(instance, value_field_name, args[value_field_name])
instance.save()
else:
CustomFieldInstance.objects.create(
**args,
field=field,
document=document,
)
else:
overrides.custom_field_ids = list(
set(
(overrides.custom_field_ids or [])
+ list(
action.assign_custom_fields.values_list("pk", flat=True),
),
),
if overrides.custom_fields is None:
overrides.custom_fields = {}
overrides.custom_fields.update(
{
field.pk: action.assign_custom_fields_values.get(
str(field.pk),
None,
)
for field in action.assign_custom_fields.all()
},
)
def removal_action():
@@ -944,18 +961,18 @@ def run_workflows(
if not use_overrides:
CustomFieldInstance.objects.filter(document=document).delete()
else:
overrides.custom_field_ids = None
overrides.custom_fields = None
elif action.remove_custom_fields.exists():
if not use_overrides:
CustomFieldInstance.objects.filter(
field__in=action.remove_custom_fields.all(),
document=document,
).delete()
elif overrides.custom_field_ids:
elif overrides.custom_fields:
for field in action.remove_custom_fields.filter(
pk__in=overrides.custom_field_ids,
pk__in=overrides.custom_fields.keys(),
):
overrides.custom_field_ids.remove(field.pk)
overrides.custom_fields.pop(field.pk, None)
def email_action():
if not settings.EMAIL_ENABLED:

68
src/documents/tests/test_api_documents.py
View File

@@ -28,6 +28,7 @@ from documents.caching import CACHE_50_MINUTES
from documents.caching import CLASSIFIER_HASH_KEY
from documents.caching import CLASSIFIER_MODIFIED_KEY
from documents.caching import CLASSIFIER_VERSION_KEY
from documents.data_models import DocumentSource
from documents.models import Correspondent
from documents.models import CustomField
from documents.models import CustomFieldInstance
@@ -39,7 +40,10 @@ from documents.models import SavedView
from documents.models import ShareLink
from documents.models import StoragePath
from documents.models import Tag
from documents.models import Workflow
from documents.models import WorkflowAction
from documents.models import WorkflowTrigger
from documents.signals.handlers import run_workflows
from documents.tests.utils import DirectoriesMixin
from documents.tests.utils import DocumentConsumeDelayMixin
@@ -1362,7 +1366,69 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
self.assertEqual(input_doc.original_file.name, "simple.pdf")
self.assertEqual(overrides.filename, "simple.pdf")
self.assertEqual(overrides.custom_field_ids, [custom_field.id])
self.assertEqual(overrides.custom_fields, {custom_field.id: None})
def test_upload_with_custom_fields_and_workflow(self):
"""
GIVEN: A document with a source file
WHEN: Upload the document with custom fields and a workflow
THEN: Metadata is set correctly, mimicking what happens in the real consumer plugin
"""
self.consume_file_mock.return_value = celery.result.AsyncResult(
id=str(uuid.uuid4()),
)
cf = CustomField.objects.create(
name="stringfield",
data_type=CustomField.FieldDataType.STRING,
)
cf2 = CustomField.objects.create(
name="intfield",
data_type=CustomField.FieldDataType.INT,
)
trigger1 = WorkflowTrigger.objects.create(
type=WorkflowTrigger.WorkflowTriggerType.CONSUMPTION,
sources=f"{DocumentSource.ApiUpload},{DocumentSource.ConsumeFolder},{DocumentSource.MailFetch}",
)
action1 = WorkflowAction.objects.create(
assign_title="Doc title",
)
action1.assign_custom_fields.add(cf2)
action1.assign_custom_fields_values = {cf2.id: 123}
action1.save()
w1 = Workflow.objects.create(
name="Workflow 1",
order=0,
)
w1.triggers.add(trigger1)
w1.actions.add(action1)
w1.save()
with (Path(__file__).parent / "samples" / "simple.pdf").open("rb") as f:
response = self.client.post(
"/api/documents/post_document/",
{
"document": f,
"custom_fields": [cf.id],
},
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.consume_file_mock.assert_called_once()
input_doc, overrides = self.get_last_consume_delay_call_args()
new_overrides, msg = run_workflows(
trigger_type=WorkflowTrigger.WorkflowTriggerType.CONSUMPTION,
document=input_doc,
logging_group=None,
overrides=overrides,
)
overrides.update(new_overrides)
self.assertEqual(overrides.custom_fields, {cf.id: None, cf2.id: 123})
def test_upload_with_webui_source(self):
"""

9
src/documents/tests/test_consumer.py
View File

@@ -408,7 +408,9 @@ class TestConsumer(
with self.get_consumer(
self.get_test_file(),
DocumentMetadataOverrides(custom_field_ids=[cf1.id, cf3.id]),
DocumentMetadataOverrides(
custom_fields={cf1.id: "value1", cf3.id: "http://example.com"},
),
) as consumer:
consumer.run()
@@ -420,6 +422,11 @@ class TestConsumer(
self.assertIn(cf1, fields_used)
self.assertNotIn(cf2, fields_used)
self.assertIn(cf3, fields_used)
self.assertEqual(document.custom_fields.get(field=cf1).value, "value1")
self.assertEqual(
document.custom_fields.get(field=cf3).value,
"http://example.com",
)
self._assert_first_last_send_progress()
def testOverrideAsn(self):

16
src/documents/tests/test_workflows.py
View File

@@ -133,6 +133,9 @@ class TestWorkflows(
action.assign_change_groups.add(self.group1.pk)
action.assign_custom_fields.add(self.cf1.pk)
action.assign_custom_fields.add(self.cf2.pk)
action.assign_custom_fields_values = {
self.cf2.pk: 42,
}
action.save()
w = Workflow.objects.create(
name="Workflow 1",
@@ -209,6 +212,10 @@ class TestWorkflows(
list(document.custom_fields.all().values_list("field", flat=True)),
[self.cf1.pk, self.cf2.pk],
)
self.assertEqual(
document.custom_fields.get(field=self.cf2.pk).value,
42,
)
info = cm.output[0]
expected_str = f"Document matched {trigger} from {w}"
@@ -1215,11 +1222,11 @@ class TestWorkflows(
def test_document_updated_workflow_existing_custom_field(self):
"""
GIVEN:
- Existing workflow with UPDATED trigger and action that adds a custom field
- Existing workflow with UPDATED trigger and action that assigns a custom field with a value
WHEN:
- Document is updated that already contains the field
THEN:
- Document update succeeds without trying to re-create the field
- Document update succeeds and updates the field
"""
trigger = WorkflowTrigger.objects.create(
type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_UPDATED,
@@ -1227,6 +1234,8 @@ class TestWorkflows(
)
action = WorkflowAction.objects.create()
action.assign_custom_fields.add(self.cf1)
action.assign_custom_fields_values = {self.cf1.pk: "new value"}
action.save()
w = Workflow.objects.create(
name="Workflow 1",
order=0,
@@ -1251,6 +1260,9 @@ class TestWorkflows(
format="json",
)
doc.refresh_from_db()
self.assertEqual(doc.custom_fields.get(field=self.cf1).value, "new value")
def test_document_updated_workflow_merge_permissions(self):
"""
GIVEN:

5
src/documents/views.py
View File

@@ -1471,7 +1471,10 @@ class PostDocumentView(GenericAPIView):
created=created,
asn=archive_serial_number,
owner_id=request.user.id,
custom_field_ids=custom_field_ids,
# TODO: set values
custom_fields={cf_id: None for cf_id in custom_field_ids}
if custom_field_ids
else None,
)
async_task = consume_file.delay(