Add negation for other things, universal query builder

This commit is contained in:
shamoon
2025-10-07 09:42:00 -07:00
parent 1fed785c7d
commit 4965480958
10 changed files with 797 additions and 143 deletions

View File

@@ -404,34 +404,76 @@ def existing_document_matches_workflow(
trigger_matched = False
# Document correspondent vs trigger has_correspondent
if (
trigger.filter_has_correspondent is not None
and document.correspondent != trigger.filter_has_correspondent
):
reason = (
f"Document correspondent {document.correspondent} does not match {trigger.filter_has_correspondent}",
)
trigger_matched = False
if trigger_matched:
if (
trigger.filter_has_correspondent is not None
and document.correspondent != trigger.filter_has_correspondent
):
reason = (
f"Document correspondent {document.correspondent} does not match {trigger.filter_has_correspondent}",
)
trigger_matched = False
if (
trigger.filter_has_not_correspondents.all().count() > 0
and document.correspondent
and trigger.filter_has_not_correspondents.filter(
id=document.correspondent_id,
).exists()
):
reason = (
f"Document correspondent {document.correspondent} is excluded by"
f" {trigger.filter_has_not_correspondents.all()}",
)
trigger_matched = False
# Document document_type vs trigger has_document_type
if (
trigger.filter_has_document_type is not None
and document.document_type != trigger.filter_has_document_type
):
reason = (
f"Document doc type {document.document_type} does not match {trigger.filter_has_document_type}",
)
trigger_matched = False
if trigger_matched:
if (
trigger.filter_has_document_type is not None
and document.document_type != trigger.filter_has_document_type
):
reason = (
f"Document doc type {document.document_type} does not match {trigger.filter_has_document_type}",
)
trigger_matched = False
if (
trigger.filter_has_not_document_types.all().count() > 0
and document.document_type
and trigger.filter_has_not_document_types.filter(
id=document.document_type_id,
).exists()
):
reason = (
f"Document doc type {document.document_type} is excluded by"
f" {trigger.filter_has_not_document_types.all()}",
)
trigger_matched = False
# Document storage_path vs trigger has_storage_path
if (
trigger.filter_has_storage_path is not None
and document.storage_path != trigger.filter_has_storage_path
):
reason = (
f"Document storage path {document.storage_path} does not match {trigger.filter_has_storage_path}",
)
trigger_matched = False
if trigger_matched:
if (
trigger.filter_has_storage_path is not None
and document.storage_path != trigger.filter_has_storage_path
):
reason = (
f"Document storage path {document.storage_path} does not match {trigger.filter_has_storage_path}",
)
trigger_matched = False
if (
trigger.filter_has_not_storage_paths.all().count() > 0
and document.storage_path
and trigger.filter_has_not_storage_paths.filter(
id=document.storage_path_id,
).exists()
):
reason = (
f"Document storage path {document.storage_path} is excluded by"
f" {trigger.filter_has_not_storage_paths.all()}",
)
trigger_matched = False
# Document original_filename vs trigger filename
if (
@@ -482,16 +524,31 @@ def prefilter_documents_by_workflowtrigger(
correspondent=trigger.filter_has_correspondent,
)
if trigger.filter_has_not_correspondents.all().count() > 0:
documents = documents.exclude(
correspondent__in=trigger.filter_has_not_correspondents.all(),
)
if trigger.filter_has_document_type is not None:
documents = documents.filter(
document_type=trigger.filter_has_document_type,
)
if trigger.filter_has_not_document_types.all().count() > 0:
documents = documents.exclude(
document_type__in=trigger.filter_has_not_document_types.all(),
)
if trigger.filter_has_storage_path is not None:
documents = documents.filter(
storage_path=trigger.filter_has_storage_path,
)
if trigger.filter_has_not_storage_paths.all().count() > 0:
documents = documents.exclude(
storage_path__in=trigger.filter_has_not_storage_paths.all(),
)
if trigger.filter_filename is not None and len(trigger.filter_filename) > 0:
# the true fnmatch will actually run later so we just want a loose filter here
regex = fnmatch_translate(trigger.filter_filename).lstrip("^").rstrip("$")

View File

@@ -1,4 +1,4 @@
# Generated by Django 5.2.6 on 2025-10-07 16:09
# Generated by Django 5.2.6 on 2025-10-07 16:22
from django.db import migrations
from django.db import models
@@ -20,6 +20,36 @@ class Migration(migrations.Migration):
verbose_name="has all of these tag(s)",
),
),
migrations.AddField(
model_name="workflowtrigger",
name="filter_has_not_correspondents",
field=models.ManyToManyField(
blank=True,
related_name="workflowtriggers_has_not_correspondent",
to="documents.correspondent",
verbose_name="does not have these correspondent(s)",
),
),
migrations.AddField(
model_name="workflowtrigger",
name="filter_has_not_document_types",
field=models.ManyToManyField(
blank=True,
related_name="workflowtriggers_has_not_document_type",
to="documents.documenttype",
verbose_name="does not have these document type(s)",
),
),
migrations.AddField(
model_name="workflowtrigger",
name="filter_has_not_storage_paths",
field=models.ManyToManyField(
blank=True,
related_name="workflowtriggers_has_not_storage_path",
to="documents.storagepath",
verbose_name="does not have these storage path(s)",
),
),
migrations.AddField(
model_name="workflowtrigger",
name="filter_has_not_tags",

View File

@@ -1087,6 +1087,13 @@ class WorkflowTrigger(models.Model):
verbose_name=_("has this document type"),
)
filter_has_not_document_types = models.ManyToManyField(
DocumentType,
blank=True,
related_name="workflowtriggers_has_not_document_type",
verbose_name=_("does not have these document type(s)"),
)
filter_has_correspondent = models.ForeignKey(
Correspondent,
null=True,
@@ -1095,6 +1102,13 @@ class WorkflowTrigger(models.Model):
verbose_name=_("has this correspondent"),
)
filter_has_not_correspondents = models.ManyToManyField(
Correspondent,
blank=True,
related_name="workflowtriggers_has_not_correspondent",
verbose_name=_("does not have these correspondent(s)"),
)
filter_has_storage_path = models.ForeignKey(
StoragePath,
null=True,
@@ -1103,6 +1117,13 @@ class WorkflowTrigger(models.Model):
verbose_name=_("has this storage path"),
)
filter_has_not_storage_paths = models.ManyToManyField(
StoragePath,
blank=True,
related_name="workflowtriggers_has_not_storage_path",
verbose_name=_("does not have these storage path(s)"),
)
schedule_offset_days = models.IntegerField(
_("schedule offset days"),
default=0,

View File

@@ -2196,6 +2196,9 @@ class WorkflowTriggerSerializer(serializers.ModelSerializer):
"filter_has_tags",
"filter_has_all_tags",
"filter_has_not_tags",
"filter_has_not_correspondents",
"filter_has_not_document_types",
"filter_has_not_storage_paths",
"filter_has_correspondent",
"filter_has_document_type",
"filter_has_storage_path",
@@ -2418,6 +2421,18 @@ class WorkflowSerializer(serializers.ModelSerializer):
filter_has_tags = trigger.pop("filter_has_tags", None)
filter_has_all_tags = trigger.pop("filter_has_all_tags", None)
filter_has_not_tags = trigger.pop("filter_has_not_tags", None)
filter_has_not_correspondents = trigger.pop(
"filter_has_not_correspondents",
None,
)
filter_has_not_document_types = trigger.pop(
"filter_has_not_document_types",
None,
)
filter_has_not_storage_paths = trigger.pop(
"filter_has_not_storage_paths",
None,
)
# Convert sources to strings to handle django-multiselectfield v1.0 changes
WorkflowTriggerSerializer.normalize_workflow_trigger_sources(trigger)
trigger_instance, _ = WorkflowTrigger.objects.update_or_create(
@@ -2430,6 +2445,18 @@ class WorkflowSerializer(serializers.ModelSerializer):
trigger_instance.filter_has_all_tags.set(filter_has_all_tags)
if filter_has_not_tags is not None:
trigger_instance.filter_has_not_tags.set(filter_has_not_tags)
if filter_has_not_correspondents is not None:
trigger_instance.filter_has_not_correspondents.set(
filter_has_not_correspondents,
)
if filter_has_not_document_types is not None:
trigger_instance.filter_has_not_document_types.set(
filter_has_not_document_types,
)
if filter_has_not_storage_paths is not None:
trigger_instance.filter_has_not_storage_paths.set(
filter_has_not_storage_paths,
)
set_triggers.append(trigger_instance)
if actions is not None and actions is not serializers.empty:

View File

@@ -186,6 +186,9 @@ class TestApiWorkflows(DirectoriesMixin, APITestCase):
"filter_has_tags": [self.t1.id],
"filter_has_all_tags": [self.t2.id],
"filter_has_not_tags": [self.t3.id],
"filter_has_not_correspondents": [self.c2.id],
"filter_has_not_document_types": [self.dt2.id],
"filter_has_not_storage_paths": [self.sp2.id],
"filter_has_document_type": self.dt.id,
"filter_has_correspondent": self.c.id,
"filter_has_storage_path": self.sp.id,
@@ -239,6 +242,18 @@ class TestApiWorkflows(DirectoriesMixin, APITestCase):
set(trigger.filter_has_not_tags.values_list("id", flat=True)),
{self.t3.id},
)
self.assertSetEqual(
set(trigger.filter_has_not_correspondents.values_list("id", flat=True)),
{self.c2.id},
)
self.assertSetEqual(
set(trigger.filter_has_not_document_types.values_list("id", flat=True)),
{self.dt2.id},
)
self.assertSetEqual(
set(trigger.filter_has_not_storage_paths.values_list("id", flat=True)),
{self.sp2.id},
)
def test_api_create_invalid_workflow_trigger(self):
"""
@@ -394,6 +409,9 @@ class TestApiWorkflows(DirectoriesMixin, APITestCase):
"filter_has_tags": [self.t1.id],
"filter_has_all_tags": [self.t2.id],
"filter_has_not_tags": [self.t3.id],
"filter_has_not_correspondents": [self.c2.id],
"filter_has_not_document_types": [self.dt2.id],
"filter_has_not_storage_paths": [self.sp2.id],
"filter_has_correspondent": self.c.id,
"filter_has_document_type": self.dt.id,
},
@@ -419,6 +437,18 @@ class TestApiWorkflows(DirectoriesMixin, APITestCase):
workflow.triggers.first().filter_has_not_tags.first(),
self.t3,
)
self.assertEqual(
workflow.triggers.first().filter_has_not_correspondents.first(),
self.c2,
)
self.assertEqual(
workflow.triggers.first().filter_has_not_document_types.first(),
self.dt2,
)
self.assertEqual(
workflow.triggers.first().filter_has_not_storage_paths.first(),
self.sp2,
)
self.assertEqual(workflow.actions.first().assign_title, "Action New Title")
def test_api_update_workflow_no_trigger_actions(self):

View File

@@ -1159,6 +1159,114 @@ class TestWorkflows(
)
self.assertIn(expected_str, cm.output[1])
def test_document_added_excluded_correspondent(self):
trigger = WorkflowTrigger.objects.create(
type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED,
)
trigger.filter_has_not_correspondents.set([self.c])
action = WorkflowAction.objects.create(
assign_title="Doc assign owner",
assign_owner=self.user2,
)
w = Workflow.objects.create(
name="Workflow 1",
order=0,
)
w.triggers.add(trigger)
w.actions.add(action)
w.save()
doc = Document.objects.create(
title="sample test",
correspondent=self.c,
original_filename="sample.pdf",
)
with self.assertLogs("paperless.matching", level="DEBUG") as cm:
document_consumption_finished.send(
sender=self.__class__,
document=doc,
)
expected_str = f"Document did not match {w}"
self.assertIn(expected_str, cm.output[0])
expected_str = (
f"Document correspondent {doc.correspondent} is excluded by"
f" {trigger.filter_has_not_correspondents.all()}"
)
self.assertIn(expected_str, cm.output[1])
def test_document_added_excluded_document_types(self):
trigger = WorkflowTrigger.objects.create(
type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED,
)
trigger.filter_has_not_document_types.set([self.dt])
action = WorkflowAction.objects.create(
assign_title="Doc assign owner",
assign_owner=self.user2,
)
w = Workflow.objects.create(
name="Workflow 1",
order=0,
)
w.triggers.add(trigger)
w.actions.add(action)
w.save()
doc = Document.objects.create(
title="sample test",
document_type=self.dt,
original_filename="sample.pdf",
)
with self.assertLogs("paperless.matching", level="DEBUG") as cm:
document_consumption_finished.send(
sender=self.__class__,
document=doc,
)
expected_str = f"Document did not match {w}"
self.assertIn(expected_str, cm.output[0])
expected_str = (
f"Document doc type {doc.document_type} is excluded by"
f" {trigger.filter_has_not_document_types.all()}"
)
self.assertIn(expected_str, cm.output[1])
def test_document_added_excluded_storage_paths(self):
trigger = WorkflowTrigger.objects.create(
type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED,
)
trigger.filter_has_not_storage_paths.set([self.sp])
action = WorkflowAction.objects.create(
assign_title="Doc assign owner",
assign_owner=self.user2,
)
w = Workflow.objects.create(
name="Workflow 1",
order=0,
)
w.triggers.add(trigger)
w.actions.add(action)
w.save()
doc = Document.objects.create(
title="sample test",
storage_path=self.sp,
original_filename="sample.pdf",
)
with self.assertLogs("paperless.matching", level="DEBUG") as cm:
document_consumption_finished.send(
sender=self.__class__,
document=doc,
)
expected_str = f"Document did not match {w}"
self.assertIn(expected_str, cm.output[0])
expected_str = (
f"Document storage path {doc.storage_path} is excluded by"
f" {trigger.filter_has_not_storage_paths.all()}"
)
self.assertIn(expected_str, cm.output[1])
def test_document_added_no_match_doctype(self):
trigger = WorkflowTrigger.objects.create(
type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED,