Support CF queries!

This commit is contained in:
shamoon
2025-10-07 11:53:47 -07:00
parent 3d9cf696a7
commit 88fcc5f339
12 changed files with 457 additions and 56 deletions

View File

@@ -6,8 +6,11 @@ from fnmatch import fnmatch
from fnmatch import translate as fnmatch_translate
from typing import TYPE_CHECKING
from rest_framework import serializers
from documents.data_models import ConsumableDocument
from documents.data_models import DocumentSource
from documents.filters import CustomFieldQueryParser
from documents.models import Correspondent
from documents.models import Document
from documents.models import DocumentType
@@ -475,6 +478,25 @@ def existing_document_matches_workflow(
)
trigger_matched = False
if trigger_matched and trigger.filter_custom_field_query:
parser = CustomFieldQueryParser("filter_custom_field_query")
try:
custom_field_q, annotations = parser.parse(
trigger.filter_custom_field_query,
)
except serializers.ValidationError:
reason = "Invalid custom field query configuration"
trigger_matched = False
else:
qs = (
Document.objects.filter(id=document.id)
.annotate(**annotations)
.filter(custom_field_q)
)
if not qs.exists():
reason = "Document custom fields do not match the configured custom field query"
trigger_matched = False
# Document original_filename vs trigger filename
if (
trigger.filter_filename is not None
@@ -549,6 +571,17 @@ def prefilter_documents_by_workflowtrigger(
storage_path__in=trigger.filter_has_not_storage_paths.all(),
)
if trigger.filter_custom_field_query:
parser = CustomFieldQueryParser("filter_custom_field_query")
try:
custom_field_q, annotations = parser.parse(
trigger.filter_custom_field_query,
)
except serializers.ValidationError:
return documents.none()
documents = documents.annotate(**annotations).filter(custom_field_q)
if trigger.filter_filename is not None and len(trigger.filter_filename) > 0:
# the true fnmatch will actually run later so we just want a loose filter here
regex = fnmatch_translate(trigger.filter_filename).lstrip("^").rstrip("$")

View File

@@ -1,4 +1,4 @@
# Generated by Django 5.2.6 on 2025-10-07 16:22
# Generated by Django 5.2.6 on 2025-10-07 18:52
from django.db import migrations
from django.db import models
@@ -10,6 +10,16 @@ class Migration(migrations.Migration):
]
operations = [
migrations.AddField(
model_name="workflowtrigger",
name="filter_custom_field_query",
field=models.TextField(
blank=True,
help_text="JSON-encoded custom field query expression.",
null=True,
verbose_name="filter custom field query",
),
),
migrations.AddField(
model_name="workflowtrigger",
name="filter_has_all_tags",

View File

@@ -1124,6 +1124,13 @@ class WorkflowTrigger(models.Model):
verbose_name=_("does not have these storage path(s)"),
)
filter_custom_field_query = models.TextField(
_("filter custom field query"),
null=True,
blank=True,
help_text=_("JSON-encoded custom field query expression."),
)
schedule_offset_days = models.IntegerField(
_("schedule offset days"),
default=0,

View File

@@ -43,6 +43,7 @@ if settings.AUDIT_LOG_ENABLED:
from documents import bulk_edit
from documents.data_models import DocumentSource
from documents.filters import CustomFieldQueryParser
from documents.models import Correspondent
from documents.models import CustomField
from documents.models import CustomFieldInstance
@@ -2196,6 +2197,7 @@ class WorkflowTriggerSerializer(serializers.ModelSerializer):
"filter_has_tags",
"filter_has_all_tags",
"filter_has_not_tags",
"filter_custom_field_query",
"filter_has_not_correspondents",
"filter_has_not_document_types",
"filter_has_not_storage_paths",
@@ -2224,6 +2226,20 @@ class WorkflowTriggerSerializer(serializers.ModelSerializer):
):
attrs["filter_path"] = None
if (
"filter_custom_field_query" in attrs
and attrs["filter_custom_field_query"] is not None
and len(attrs["filter_custom_field_query"]) == 0
):
attrs["filter_custom_field_query"] = None
if (
"filter_custom_field_query" in attrs
and attrs["filter_custom_field_query"] is not None
):
parser = CustomFieldQueryParser("filter_custom_field_query")
parser.parse(attrs["filter_custom_field_query"])
trigger_type = attrs.get("type", getattr(self.instance, "type", None))
if (
trigger_type == WorkflowTrigger.WorkflowTriggerType.CONSUMPTION

View File

@@ -189,6 +189,12 @@ class TestApiWorkflows(DirectoriesMixin, APITestCase):
"filter_has_not_correspondents": [self.c2.id],
"filter_has_not_document_types": [self.dt2.id],
"filter_has_not_storage_paths": [self.sp2.id],
"filter_custom_field_query": json.dumps(
[
"AND",
[[self.cf1.id, "exact", "value"]],
],
),
"filter_has_document_type": self.dt.id,
"filter_has_correspondent": self.c.id,
"filter_has_storage_path": self.sp.id,
@@ -254,6 +260,10 @@ class TestApiWorkflows(DirectoriesMixin, APITestCase):
set(trigger.filter_has_not_storage_paths.values_list("id", flat=True)),
{self.sp2.id},
)
self.assertEqual(
trigger.filter_custom_field_query,
json.dumps(["AND", [[self.cf1.id, "exact", "value"]]]),
)
def test_api_create_invalid_workflow_trigger(self):
"""
@@ -412,6 +422,9 @@ class TestApiWorkflows(DirectoriesMixin, APITestCase):
"filter_has_not_correspondents": [self.c2.id],
"filter_has_not_document_types": [self.dt2.id],
"filter_has_not_storage_paths": [self.sp2.id],
"filter_custom_field_query": json.dumps(
["AND", [[self.cf1.id, "exact", "value"]]],
),
"filter_has_correspondent": self.c.id,
"filter_has_document_type": self.dt.id,
},
@@ -449,6 +462,10 @@ class TestApiWorkflows(DirectoriesMixin, APITestCase):
workflow.triggers.first().filter_has_not_storage_paths.first(),
self.sp2,
)
self.assertEqual(
workflow.triggers.first().filter_custom_field_query,
json.dumps(["AND", [[self.cf1.id, "exact", "value"]]]),
)
self.assertEqual(workflow.actions.first().assign_title, "Action New Title")
def test_api_update_workflow_no_trigger_actions(self):

View File

@@ -1,4 +1,5 @@
import datetime
import json
import shutil
import socket
from datetime import timedelta
@@ -31,6 +32,7 @@ from documents import tasks
from documents.data_models import ConsumableDocument
from documents.data_models import DocumentSource
from documents.matching import document_matches_workflow
from documents.matching import existing_document_matches_workflow
from documents.matching import prefilter_documents_by_workflowtrigger
from documents.models import Correspondent
from documents.models import CustomField
@@ -1267,6 +1269,114 @@ class TestWorkflows(
)
self.assertIn(expected_str, cm.output[1])
def test_document_added_custom_field_query_no_match(self):
trigger = WorkflowTrigger.objects.create(
type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED,
filter_custom_field_query=json.dumps(
[
"AND",
[[self.cf1.id, "exact", "expected"]],
],
),
)
action = WorkflowAction.objects.create(
assign_title="Doc assign owner",
assign_owner=self.user2,
)
workflow = Workflow.objects.create(name="Workflow 1", order=0)
workflow.triggers.add(trigger)
workflow.actions.add(action)
workflow.save()
doc = Document.objects.create(
title="sample test",
correspondent=self.c,
original_filename="sample.pdf",
)
CustomFieldInstance.objects.create(
document=doc,
field=self.cf1,
value_text="other",
)
with self.assertLogs("paperless.matching", level="DEBUG") as cm:
document_consumption_finished.send(
sender=self.__class__,
document=doc,
)
expected_str = f"Document did not match {workflow}"
self.assertIn(expected_str, cm.output[0])
self.assertIn(
"Document custom fields do not match the configured custom field query",
cm.output[1],
)
def test_document_added_custom_field_query_match(self):
trigger = WorkflowTrigger.objects.create(
type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED,
filter_custom_field_query=json.dumps(
[
"AND",
[[self.cf1.id, "exact", "expected"]],
],
),
)
doc = Document.objects.create(
title="sample test",
correspondent=self.c,
original_filename="sample.pdf",
)
CustomFieldInstance.objects.create(
document=doc,
field=self.cf1,
value_text="expected",
)
matched, reason = existing_document_matches_workflow(doc, trigger)
self.assertTrue(matched)
self.assertEqual(reason, "")
def test_prefilter_documents_custom_field_query(self):
trigger = WorkflowTrigger.objects.create(
type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED,
filter_custom_field_query=json.dumps(
[
"AND",
[[self.cf1.id, "exact", "match"]],
],
),
)
doc1 = Document.objects.create(
title="doc 1",
correspondent=self.c,
original_filename="doc1.pdf",
checksum="checksum1",
)
CustomFieldInstance.objects.create(
document=doc1,
field=self.cf1,
value_text="match",
)
doc2 = Document.objects.create(
title="doc 2",
correspondent=self.c,
original_filename="doc2.pdf",
checksum="checksum2",
)
CustomFieldInstance.objects.create(
document=doc2,
field=self.cf1,
value_text="different",
)
filtered = prefilter_documents_by_workflowtrigger(
Document.objects.all(),
trigger,
)
self.assertIn(doc1, filtered)
self.assertNotIn(doc2, filtered)
def test_document_added_no_match_doctype(self):
trigger = WorkflowTrigger.objects.create(
type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED,