From 344cc70cd564ee96e5910eb27122a14c286acd6c Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Sun, 11 May 2025 13:04:46 -0700 Subject: [PATCH] Enhancement: support negative offset in scheduled workflows (#9746) --- docs/usage.md | 3 +- src-ui/messages.xlf | 147 +++++++++--------- .../workflow-edit-dialog.component.html | 10 +- ...er_workflowtrigger_schedule_offset_days.py | 22 +++ src/documents/models.py | 2 +- src/documents/tasks.py | 76 ++++++--- src/documents/tests/test_workflows.py | 64 +++++++- 7 files changed, 227 insertions(+), 97 deletions(-) create mode 100644 src/documents/migrations/1066_alter_workflowtrigger_schedule_offset_days.py diff --git a/docs/usage.md b/docs/usage.md index bce0e2013..94fe8b9a1 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -406,7 +406,8 @@ Currently, there are three events that correspond to workflow trigger 'types': 3. **Document Updated**: when a document is updated. Similar to 'added' events, triggers can include filtering by content matching, tags, doc type, or correspondent. 4. **Scheduled**: a scheduled trigger that can be used to run workflows at a specific time. The date used can be either the document - added, created, updated date or you can specify a (date) custom field. You can also specify a day offset from the date. + added, created, updated date or you can specify a (date) custom field. You can also specify a day offset from the date (positive + offsets will trigger before the date, negative offsets will trigger after). The following flow diagram illustrates the three document trigger types: diff --git a/src-ui/messages.xlf b/src-ui/messages.xlf index f38ea0d3e..3a021b39b 100644 --- a/src-ui/messages.xlf +++ b/src-ui/messages.xlf @@ -1284,19 +1284,19 @@ src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 201 + 209 src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 220 + 228 src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 287 + 295 src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 306 + 314 src/app/components/common/input/permissions/permissions-form/permissions-form.component.html @@ -1319,19 +1319,19 @@ src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 209 + 217 src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 228 + 236 src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 295 + 303 src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 314 + 322 src/app/components/common/input/permissions/permissions-form/permissions-form.component.html @@ -1357,11 +1357,11 @@ src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 234 + 242 src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 320 + 328 src/app/components/common/input/permissions/permissions-form/permissions-form.component.html @@ -3691,7 +3691,7 @@ src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 163 + 171 @@ -4115,7 +4115,7 @@ src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 188 + 196 @@ -4133,7 +4133,7 @@ src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 189 + 197 @@ -4645,381 +4645,388 @@ Offset days src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 126 + 128 + + + + Positive values will trigger the workflow before the date, negative values after. + + src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html + 132 Relative to src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 129 + 137 Custom field src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 133 + 141 Custom field to use for date. src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 133 + 141 Recurring src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 139 + 147 Trigger is recurring. src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 139 + 147 Recurring interval days src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 143 + 151 Repeat the trigger every n days. src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 143 + 151 Trigger for documents that match all filters specified below. src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 148 + 156 Filter filename src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 151 + 159 Apply to documents that match this filename. Wildcards such as *.pdf or *invoice* are allowed. Case insensitive. src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 151 + 159 Filter sources src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 153 + 161 Filter path src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 154 + 162 Apply to documents that match this path. Wildcards specified as * are allowed. Case-normalized.</a> src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 154 + 162 Filter mail rule src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 155 + 163 Apply to documents consumed via this mail rule. src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 155 + 163 Content matching algorithm src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 158 + 166 Content matching pattern src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 160 + 168 Has any of tags src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 169 + 177 Has correspondent src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 170 + 178 Has document type src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 171 + 179 Action type src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 181 + 189 Assign title src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 186 + 194 Can include some placeholders, see <a target='_blank' href='https://docs.paperless-ngx.com/usage/#workflows'>documentation</a>. src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 186 + 194 Assign tags src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 187 + 195 Assign storage path src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 190 + 198 Assign custom fields src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 191 + 199 Assign owner src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 195 + 203 Assign view permissions src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 197 + 205 Assign edit permissions src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 216 + 224 Remove tags src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 243 + 251 Remove all src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 244 + 252 src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 250 + 258 src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 256 + 264 src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 262 + 270 src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 268 + 276 src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 275 + 283 src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 281 + 289 Remove correspondents src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 249 + 257 Remove document types src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 255 + 263 Remove storage paths src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 261 + 269 Remove custom fields src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 267 + 275 Remove owners src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 274 + 282 Remove permissions src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 280 + 288 View permissions src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 283 + 291 Edit permissions src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 302 + 310 Email subject src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 330 + 338 Email body src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 331 + 339 Email recipients src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 332 + 340 Attach document src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 333 + 341 Webhook url src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 341 + 349 Use parameters for webhook body src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 343 + 351 Send webhook payload as JSON src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 344 + 352 Webhook params src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 347 + 355 Webhook body src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 349 + 357 Webhook headers src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 351 + 359 Include document src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 352 + 360 diff --git a/src-ui/src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html b/src-ui/src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html index 4e5d5ba9b..c9869bfcb 100644 --- a/src-ui/src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html +++ b/src-ui/src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html @@ -123,7 +123,15 @@

Set scheduled trigger offset and which date field to use.

- +
diff --git a/src/documents/migrations/1066_alter_workflowtrigger_schedule_offset_days.py b/src/documents/migrations/1066_alter_workflowtrigger_schedule_offset_days.py new file mode 100644 index 000000000..eaf23ad64 --- /dev/null +++ b/src/documents/migrations/1066_alter_workflowtrigger_schedule_offset_days.py @@ -0,0 +1,22 @@ +# Generated by Django 5.1.7 on 2025-04-15 19:18 + +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + dependencies = [ + ("documents", "1065_workflowaction_assign_custom_fields_values"), + ] + + operations = [ + migrations.AlterField( + model_name="workflowtrigger", + name="schedule_offset_days", + field=models.IntegerField( + default=0, + help_text="The number of days to offset the schedule trigger by.", + verbose_name="schedule offset days", + ), + ), + ] diff --git a/src/documents/models.py b/src/documents/models.py index 4b3f97e50..74090700c 100644 --- a/src/documents/models.py +++ b/src/documents/models.py @@ -1019,7 +1019,7 @@ class WorkflowTrigger(models.Model): verbose_name=_("has this correspondent"), ) - schedule_offset_days = models.PositiveIntegerField( + schedule_offset_days = models.IntegerField( _("schedule offset days"), default=0, help_text=_( diff --git a/src/documents/tasks.py b/src/documents/tasks.py index 857ace928..13c104185 100644 --- a/src/documents/tasks.py +++ b/src/documents/tasks.py @@ -1,8 +1,8 @@ +import datetime import hashlib import logging import shutil import uuid -from datetime import timedelta from pathlib import Path from tempfile import TemporaryDirectory @@ -357,7 +357,7 @@ def empty_trash(doc_ids=None): if doc_ids is not None else Document.deleted_objects.filter( deleted_at__lt=timezone.localtime(timezone.now()) - - timedelta( + - datetime.timedelta( days=settings.EMPTY_TRASH_DELAY, ), ) @@ -397,6 +397,7 @@ def check_scheduled_workflows(): ) if scheduled_workflows.count() > 0: logger.debug(f"Checking {len(scheduled_workflows)} scheduled workflows") + now = timezone.now() for workflow in scheduled_workflows: schedule_triggers = workflow.triggers.filter( type=WorkflowTrigger.WorkflowTriggerType.SCHEDULED, @@ -404,31 +405,60 @@ def check_scheduled_workflows(): trigger: WorkflowTrigger for trigger in schedule_triggers: documents = Document.objects.none() - offset_td = timedelta(days=trigger.schedule_offset_days) + offset_td = datetime.timedelta(days=-trigger.schedule_offset_days) + threshold = now - offset_td logger.debug( - f"Checking trigger {trigger} with offset {offset_td} against field: {trigger.schedule_date_field}", + f"Trigger {trigger.id}: checking if (date + {offset_td}) <= now ({now})", ) + match trigger.schedule_date_field: case WorkflowTrigger.ScheduleDateField.ADDED: - documents = Document.objects.filter( - added__lt=timezone.now() - offset_td, - ) + documents = Document.objects.filter(added__lte=threshold) + case WorkflowTrigger.ScheduleDateField.CREATED: - documents = Document.objects.filter( - created__lt=timezone.now() - offset_td, - ) + documents = Document.objects.filter(created__lte=threshold) + case WorkflowTrigger.ScheduleDateField.MODIFIED: - documents = Document.objects.filter( - modified__lt=timezone.now() - offset_td, - ) + documents = Document.objects.filter(modified__lte=threshold) + case WorkflowTrigger.ScheduleDateField.CUSTOM_FIELD: - cf_instances = CustomFieldInstance.objects.filter( - field=trigger.schedule_date_custom_field, - value_date__lt=timezone.now() - offset_td, - ) - documents = Document.objects.filter( - id__in=cf_instances.values_list("document", flat=True), + # cap earliest date to avoid massive scans + earliest_date = now - datetime.timedelta(days=365) + if offset_td.days < -365: + logger.warning( + f"Trigger {trigger.id} has large negative offset ({offset_td.days}), " + f"limiting earliest scan date to {earliest_date}", + ) + + cf_filter_kwargs = { + "field": trigger.schedule_date_custom_field, + "value_date__isnull": False, + "value_date__lte": threshold, + "value_date__gte": earliest_date, + } + + recent_cf_instances = CustomFieldInstance.objects.filter( + **cf_filter_kwargs, ) + + matched_ids = [ + cfi.document_id + for cfi in recent_cf_instances + if cfi.value_date + and ( + timezone.make_aware( + datetime.datetime.combine( + cfi.value_date, + datetime.time.min, + ), + ) + + offset_td + <= now + ) + ] + + documents = Document.objects.filter(id__in=matched_ids) + if documents.count() > 0: logger.debug( f"Found {documents.count()} documents for trigger {trigger}", @@ -440,18 +470,18 @@ def check_scheduled_workflows(): workflow=workflow, ).order_by("-run_at") if not trigger.schedule_is_recurring and workflow_runs.exists(): - # schedule is non-recurring and the workflow has already been run logger.debug( f"Skipping document {document} for non-recurring workflow {workflow} as it has already been run", ) continue - elif ( + + if ( trigger.schedule_is_recurring and workflow_runs.exists() and ( workflow_runs.last().run_at - > timezone.now() - - timedelta( + > now + - datetime.timedelta( days=trigger.schedule_recurring_interval_days, ) ) diff --git a/src/documents/tests/test_workflows.py b/src/documents/tests/test_workflows.py index 1c0c4449c..17464b973 100644 --- a/src/documents/tests/test_workflows.py +++ b/src/documents/tests/test_workflows.py @@ -1336,6 +1336,8 @@ class TestWorkflows( GIVEN: - Existing workflow with SCHEDULED trigger against the created field and action that assigns owner - Existing doc that matches the trigger + - Workflow set to trigger at (now - offset) = now - 1 day + - Document created date is 2 days ago → trigger condition met WHEN: - Scheduled workflows are checked THEN: @@ -1359,7 +1361,7 @@ class TestWorkflows( w.save() now = timezone.localtime(timezone.now()) - created = now - timedelta(weeks=520) + created = now - timedelta(days=2) doc = Document.objects.create( title="sample test", correspondent=self.c, @@ -1377,6 +1379,8 @@ class TestWorkflows( GIVEN: - Existing workflow with SCHEDULED trigger against the added field and action that assigns owner - Existing doc that matches the trigger + - Workflow set to trigger at (now - offset) = now - 1 day + - Document added date is 365 days ago WHEN: - Scheduled workflows are checked THEN: @@ -1418,6 +1422,8 @@ class TestWorkflows( GIVEN: - Existing workflow with SCHEDULED trigger against the modified field and action that assigns owner - Existing doc that matches the trigger + - Workflow set to trigger at (now - offset) = now - 1 day + - Document modified date is mocked as sufficiently in the past WHEN: - Scheduled workflows are checked THEN: @@ -1458,6 +1464,8 @@ class TestWorkflows( GIVEN: - Existing workflow with SCHEDULED trigger against a custom field and action that assigns owner - Existing doc that matches the trigger + - Workflow set to trigger at (now - offset) = now - 1 day + - Custom field date is 2 days ago WHEN: - Scheduled workflows are checked THEN: @@ -1502,6 +1510,7 @@ class TestWorkflows( GIVEN: - Existing workflow with SCHEDULED trigger - Existing doc that has already had the workflow run + - Document created 2 days ago, workflow offset = 1 day → trigger time = yesterday WHEN: - Scheduled workflows are checked THEN: @@ -1552,6 +1561,7 @@ class TestWorkflows( GIVEN: - Existing workflow with SCHEDULED trigger and recurring interval of 7 days - Workflow run date is 6 days ago + - Document created 40 days ago, offset = 30 → trigger time = 10 days ago WHEN: - Scheduled workflows are checked THEN: @@ -1600,6 +1610,58 @@ class TestWorkflows( doc.refresh_from_db() self.assertIsNone(doc.owner) + def test_workflow_scheduled_trigger_negative_offset(self): + """ + GIVEN: + - Existing workflow with SCHEDULED trigger and negative offset of -7 days (so 7 days after date) + - Custom field date initially set to 5 days ago → trigger time = 2 days in future + - Then updated to 8 days ago → trigger time = 1 day ago + WHEN: + - Scheduled workflows are checked for document with custom field date 8 days in the past + THEN: + - Workflow runs and document owner is updated + """ + trigger = WorkflowTrigger.objects.create( + type=WorkflowTrigger.WorkflowTriggerType.SCHEDULED, + schedule_offset_days=-7, + schedule_date_field=WorkflowTrigger.ScheduleDateField.CUSTOM_FIELD, + schedule_date_custom_field=self.cf1, + ) + action = WorkflowAction.objects.create( + assign_title="Doc assign owner", + assign_owner=self.user2, + ) + w = Workflow.objects.create( + name="Workflow 1", + order=0, + ) + w.triggers.add(trigger) + w.actions.add(action) + w.save() + + doc = Document.objects.create( + title="sample test", + correspondent=self.c, + original_filename="sample.pdf", + ) + cfi = CustomFieldInstance.objects.create( + document=doc, + field=self.cf1, + value_date=timezone.now() - timedelta(days=5), + ) + + tasks.check_scheduled_workflows() + + doc.refresh_from_db() + self.assertIsNone(doc.owner) # has not triggered yet + + cfi.value_date = timezone.now() - timedelta(days=8) + cfi.save() + + tasks.check_scheduled_workflows() + doc.refresh_from_db() + self.assertEqual(doc.owner, self.user2) + def test_workflow_enabled_disabled(self): trigger = WorkflowTrigger.objects.create( type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED,