mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2025-10-10 02:16:12 -05:00
Support CF queries!
This commit is contained in:
@@ -6,8 +6,11 @@ from fnmatch import fnmatch
|
||||
from fnmatch import translate as fnmatch_translate
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from rest_framework import serializers
|
||||
|
||||
from documents.data_models import ConsumableDocument
|
||||
from documents.data_models import DocumentSource
|
||||
from documents.filters import CustomFieldQueryParser
|
||||
from documents.models import Correspondent
|
||||
from documents.models import Document
|
||||
from documents.models import DocumentType
|
||||
@@ -475,6 +478,25 @@ def existing_document_matches_workflow(
|
||||
)
|
||||
trigger_matched = False
|
||||
|
||||
if trigger_matched and trigger.filter_custom_field_query:
|
||||
parser = CustomFieldQueryParser("filter_custom_field_query")
|
||||
try:
|
||||
custom_field_q, annotations = parser.parse(
|
||||
trigger.filter_custom_field_query,
|
||||
)
|
||||
except serializers.ValidationError:
|
||||
reason = "Invalid custom field query configuration"
|
||||
trigger_matched = False
|
||||
else:
|
||||
qs = (
|
||||
Document.objects.filter(id=document.id)
|
||||
.annotate(**annotations)
|
||||
.filter(custom_field_q)
|
||||
)
|
||||
if not qs.exists():
|
||||
reason = "Document custom fields do not match the configured custom field query"
|
||||
trigger_matched = False
|
||||
|
||||
# Document original_filename vs trigger filename
|
||||
if (
|
||||
trigger.filter_filename is not None
|
||||
@@ -549,6 +571,17 @@ def prefilter_documents_by_workflowtrigger(
|
||||
storage_path__in=trigger.filter_has_not_storage_paths.all(),
|
||||
)
|
||||
|
||||
if trigger.filter_custom_field_query:
|
||||
parser = CustomFieldQueryParser("filter_custom_field_query")
|
||||
try:
|
||||
custom_field_q, annotations = parser.parse(
|
||||
trigger.filter_custom_field_query,
|
||||
)
|
||||
except serializers.ValidationError:
|
||||
return documents.none()
|
||||
|
||||
documents = documents.annotate(**annotations).filter(custom_field_q)
|
||||
|
||||
if trigger.filter_filename is not None and len(trigger.filter_filename) > 0:
|
||||
# the true fnmatch will actually run later so we just want a loose filter here
|
||||
regex = fnmatch_translate(trigger.filter_filename).lstrip("^").rstrip("$")
|
||||
|
@@ -1,4 +1,4 @@
|
||||
# Generated by Django 5.2.6 on 2025-10-07 16:22
|
||||
# Generated by Django 5.2.6 on 2025-10-07 18:52
|
||||
|
||||
from django.db import migrations
|
||||
from django.db import models
|
||||
@@ -10,6 +10,16 @@ class Migration(migrations.Migration):
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name="workflowtrigger",
|
||||
name="filter_custom_field_query",
|
||||
field=models.TextField(
|
||||
blank=True,
|
||||
help_text="JSON-encoded custom field query expression.",
|
||||
null=True,
|
||||
verbose_name="filter custom field query",
|
||||
),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="workflowtrigger",
|
||||
name="filter_has_all_tags",
|
@@ -1124,6 +1124,13 @@ class WorkflowTrigger(models.Model):
|
||||
verbose_name=_("does not have these storage path(s)"),
|
||||
)
|
||||
|
||||
filter_custom_field_query = models.TextField(
|
||||
_("filter custom field query"),
|
||||
null=True,
|
||||
blank=True,
|
||||
help_text=_("JSON-encoded custom field query expression."),
|
||||
)
|
||||
|
||||
schedule_offset_days = models.IntegerField(
|
||||
_("schedule offset days"),
|
||||
default=0,
|
||||
|
@@ -43,6 +43,7 @@ if settings.AUDIT_LOG_ENABLED:
|
||||
|
||||
from documents import bulk_edit
|
||||
from documents.data_models import DocumentSource
|
||||
from documents.filters import CustomFieldQueryParser
|
||||
from documents.models import Correspondent
|
||||
from documents.models import CustomField
|
||||
from documents.models import CustomFieldInstance
|
||||
@@ -2196,6 +2197,7 @@ class WorkflowTriggerSerializer(serializers.ModelSerializer):
|
||||
"filter_has_tags",
|
||||
"filter_has_all_tags",
|
||||
"filter_has_not_tags",
|
||||
"filter_custom_field_query",
|
||||
"filter_has_not_correspondents",
|
||||
"filter_has_not_document_types",
|
||||
"filter_has_not_storage_paths",
|
||||
@@ -2224,6 +2226,20 @@ class WorkflowTriggerSerializer(serializers.ModelSerializer):
|
||||
):
|
||||
attrs["filter_path"] = None
|
||||
|
||||
if (
|
||||
"filter_custom_field_query" in attrs
|
||||
and attrs["filter_custom_field_query"] is not None
|
||||
and len(attrs["filter_custom_field_query"]) == 0
|
||||
):
|
||||
attrs["filter_custom_field_query"] = None
|
||||
|
||||
if (
|
||||
"filter_custom_field_query" in attrs
|
||||
and attrs["filter_custom_field_query"] is not None
|
||||
):
|
||||
parser = CustomFieldQueryParser("filter_custom_field_query")
|
||||
parser.parse(attrs["filter_custom_field_query"])
|
||||
|
||||
trigger_type = attrs.get("type", getattr(self.instance, "type", None))
|
||||
if (
|
||||
trigger_type == WorkflowTrigger.WorkflowTriggerType.CONSUMPTION
|
||||
|
@@ -189,6 +189,12 @@ class TestApiWorkflows(DirectoriesMixin, APITestCase):
|
||||
"filter_has_not_correspondents": [self.c2.id],
|
||||
"filter_has_not_document_types": [self.dt2.id],
|
||||
"filter_has_not_storage_paths": [self.sp2.id],
|
||||
"filter_custom_field_query": json.dumps(
|
||||
[
|
||||
"AND",
|
||||
[[self.cf1.id, "exact", "value"]],
|
||||
],
|
||||
),
|
||||
"filter_has_document_type": self.dt.id,
|
||||
"filter_has_correspondent": self.c.id,
|
||||
"filter_has_storage_path": self.sp.id,
|
||||
@@ -254,6 +260,10 @@ class TestApiWorkflows(DirectoriesMixin, APITestCase):
|
||||
set(trigger.filter_has_not_storage_paths.values_list("id", flat=True)),
|
||||
{self.sp2.id},
|
||||
)
|
||||
self.assertEqual(
|
||||
trigger.filter_custom_field_query,
|
||||
json.dumps(["AND", [[self.cf1.id, "exact", "value"]]]),
|
||||
)
|
||||
|
||||
def test_api_create_invalid_workflow_trigger(self):
|
||||
"""
|
||||
@@ -412,6 +422,9 @@ class TestApiWorkflows(DirectoriesMixin, APITestCase):
|
||||
"filter_has_not_correspondents": [self.c2.id],
|
||||
"filter_has_not_document_types": [self.dt2.id],
|
||||
"filter_has_not_storage_paths": [self.sp2.id],
|
||||
"filter_custom_field_query": json.dumps(
|
||||
["AND", [[self.cf1.id, "exact", "value"]]],
|
||||
),
|
||||
"filter_has_correspondent": self.c.id,
|
||||
"filter_has_document_type": self.dt.id,
|
||||
},
|
||||
@@ -449,6 +462,10 @@ class TestApiWorkflows(DirectoriesMixin, APITestCase):
|
||||
workflow.triggers.first().filter_has_not_storage_paths.first(),
|
||||
self.sp2,
|
||||
)
|
||||
self.assertEqual(
|
||||
workflow.triggers.first().filter_custom_field_query,
|
||||
json.dumps(["AND", [[self.cf1.id, "exact", "value"]]]),
|
||||
)
|
||||
self.assertEqual(workflow.actions.first().assign_title, "Action New Title")
|
||||
|
||||
def test_api_update_workflow_no_trigger_actions(self):
|
||||
|
@@ -1,4 +1,5 @@
|
||||
import datetime
|
||||
import json
|
||||
import shutil
|
||||
import socket
|
||||
from datetime import timedelta
|
||||
@@ -31,6 +32,7 @@ from documents import tasks
|
||||
from documents.data_models import ConsumableDocument
|
||||
from documents.data_models import DocumentSource
|
||||
from documents.matching import document_matches_workflow
|
||||
from documents.matching import existing_document_matches_workflow
|
||||
from documents.matching import prefilter_documents_by_workflowtrigger
|
||||
from documents.models import Correspondent
|
||||
from documents.models import CustomField
|
||||
@@ -1267,6 +1269,114 @@ class TestWorkflows(
|
||||
)
|
||||
self.assertIn(expected_str, cm.output[1])
|
||||
|
||||
def test_document_added_custom_field_query_no_match(self):
|
||||
trigger = WorkflowTrigger.objects.create(
|
||||
type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED,
|
||||
filter_custom_field_query=json.dumps(
|
||||
[
|
||||
"AND",
|
||||
[[self.cf1.id, "exact", "expected"]],
|
||||
],
|
||||
),
|
||||
)
|
||||
action = WorkflowAction.objects.create(
|
||||
assign_title="Doc assign owner",
|
||||
assign_owner=self.user2,
|
||||
)
|
||||
workflow = Workflow.objects.create(name="Workflow 1", order=0)
|
||||
workflow.triggers.add(trigger)
|
||||
workflow.actions.add(action)
|
||||
workflow.save()
|
||||
|
||||
doc = Document.objects.create(
|
||||
title="sample test",
|
||||
correspondent=self.c,
|
||||
original_filename="sample.pdf",
|
||||
)
|
||||
CustomFieldInstance.objects.create(
|
||||
document=doc,
|
||||
field=self.cf1,
|
||||
value_text="other",
|
||||
)
|
||||
|
||||
with self.assertLogs("paperless.matching", level="DEBUG") as cm:
|
||||
document_consumption_finished.send(
|
||||
sender=self.__class__,
|
||||
document=doc,
|
||||
)
|
||||
expected_str = f"Document did not match {workflow}"
|
||||
self.assertIn(expected_str, cm.output[0])
|
||||
self.assertIn(
|
||||
"Document custom fields do not match the configured custom field query",
|
||||
cm.output[1],
|
||||
)
|
||||
|
||||
def test_document_added_custom_field_query_match(self):
|
||||
trigger = WorkflowTrigger.objects.create(
|
||||
type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED,
|
||||
filter_custom_field_query=json.dumps(
|
||||
[
|
||||
"AND",
|
||||
[[self.cf1.id, "exact", "expected"]],
|
||||
],
|
||||
),
|
||||
)
|
||||
doc = Document.objects.create(
|
||||
title="sample test",
|
||||
correspondent=self.c,
|
||||
original_filename="sample.pdf",
|
||||
)
|
||||
CustomFieldInstance.objects.create(
|
||||
document=doc,
|
||||
field=self.cf1,
|
||||
value_text="expected",
|
||||
)
|
||||
|
||||
matched, reason = existing_document_matches_workflow(doc, trigger)
|
||||
self.assertTrue(matched)
|
||||
self.assertEqual(reason, "")
|
||||
|
||||
def test_prefilter_documents_custom_field_query(self):
|
||||
trigger = WorkflowTrigger.objects.create(
|
||||
type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED,
|
||||
filter_custom_field_query=json.dumps(
|
||||
[
|
||||
"AND",
|
||||
[[self.cf1.id, "exact", "match"]],
|
||||
],
|
||||
),
|
||||
)
|
||||
doc1 = Document.objects.create(
|
||||
title="doc 1",
|
||||
correspondent=self.c,
|
||||
original_filename="doc1.pdf",
|
||||
checksum="checksum1",
|
||||
)
|
||||
CustomFieldInstance.objects.create(
|
||||
document=doc1,
|
||||
field=self.cf1,
|
||||
value_text="match",
|
||||
)
|
||||
|
||||
doc2 = Document.objects.create(
|
||||
title="doc 2",
|
||||
correspondent=self.c,
|
||||
original_filename="doc2.pdf",
|
||||
checksum="checksum2",
|
||||
)
|
||||
CustomFieldInstance.objects.create(
|
||||
document=doc2,
|
||||
field=self.cf1,
|
||||
value_text="different",
|
||||
)
|
||||
|
||||
filtered = prefilter_documents_by_workflowtrigger(
|
||||
Document.objects.all(),
|
||||
trigger,
|
||||
)
|
||||
self.assertIn(doc1, filtered)
|
||||
self.assertNotIn(doc2, filtered)
|
||||
|
||||
def test_document_added_no_match_doctype(self):
|
||||
trigger = WorkflowTrigger.objects.create(
|
||||
type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED,
|
||||
|
Reference in New Issue
Block a user