From 2c05247d8446f5f04ccb53a2f53614dece1f3801 Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Wed, 23 Oct 2024 21:43:17 -0700 Subject: [PATCH] Refactor --- Pipfile | 1 - Pipfile.lock | 11 +---- src-ui/messages.xlf | 16 +++---- .../workflow-edit-dialog.component.html | 10 ++--- .../workflow-edit-dialog.component.ts | 30 ++++++------- src-ui/src/app/data/workflow-trigger.ts | 8 ++-- ...er_schedule_date_custom_field_and_more.py} | 45 ++++++++++++------- src/documents/models.py | 32 +++++++------ src/documents/serialisers.py | 22 ++------- src/documents/signals/handlers.py | 1 + src/documents/tasks.py | 34 +++++++------- src/documents/tests/test_workflows.py | 12 ++--- 12 files changed, 106 insertions(+), 116 deletions(-) rename src/documents/migrations/{1056_workflowtrigger_schedule_delay_and_more.py => 1056_workflowtrigger_schedule_date_custom_field_and_more.py} (76%) diff --git a/Pipfile b/Pipfile index b31749495..d3f3bf4b3 100644 --- a/Pipfile +++ b/Pipfile @@ -26,7 +26,6 @@ celery = {extras = ["redis"], version = "*"} channels = "~=4.1" channels-redis = "*" concurrent-log-handler = "*" -duration-parser = "*" filelock = "*" flower = "*" gotenberg-client = "*" diff --git a/Pipfile.lock b/Pipfile.lock index df97a0bc0..4880bbb74 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "3b615d0801cbb9ede6809ff8ecdd1ce1166bfefbc9e87b6fbfd26518b3425ec9" + "sha256": "584249cbeaf29659c975000b5e02b12e45d768d795e4a8ac36118e73bd7c0b8a" }, "pipfile-spec": 6, "requires": {}, @@ -576,15 +576,6 @@ "markers": "python_version >= '3.7'", "version": "==0.7.0" }, - "duration-parser": { - "hashes": [ - "sha256:2d5c465aeccd467f5c981fa78d69edd13459d6cc8ce3c751e12cbe40742163f3", - "sha256:aecbb05af545f688f3f6277ab7720e538a8ab834e22c443e2a912f6c7ab6ec5c" - ], - "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==1.0.1" - }, "exceptiongroup": { "hashes": [ "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b", diff --git a/src-ui/messages.xlf b/src-ui/messages.xlf index 675c53328..5cf4b0898 100644 --- a/src-ui/messages.xlf +++ b/src-ui/messages.xlf @@ -4380,22 +4380,22 @@ 121 - - Set scheduled trigger delay and which field to use. + + Set scheduled trigger offset and which field to use. src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html 123 - - Delay + + Offset src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html 126 - - Delay time string such as '3months', '1y'. + + Offset in days. Use 0 for immediate. src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html 126 @@ -4429,8 +4429,8 @@ 134 - - Custom field to use for delay. + + Custom field to use for date. src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html 134 diff --git a/src-ui/src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html b/src-ui/src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html index 99e25e42f..93cf7d8c2 100644 --- a/src-ui/src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html +++ b/src-ui/src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html @@ -120,18 +120,18 @@ @if (formGroup.get('type').value === WorkflowTriggerType.Scheduled) { -

Set scheduled trigger delay and which field to use.

+

Set scheduled trigger offset and which field to use.

- +
- +
- @if (formGroup.get('schedule_delay_field').value === 'custom_field') { + @if (formGroup.get('schedule_date_field').value === 'custom_field') {
- +
}
diff --git a/src-ui/src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts b/src-ui/src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts index 09296250d..d0170ce13 100644 --- a/src-ui/src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts +++ b/src-ui/src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts @@ -19,7 +19,7 @@ import { CustomFieldsService } from 'src/app/services/rest/custom-fields.service import { CustomField, CustomFieldDataType } from 'src/app/data/custom-field' import { DocumentSource, - ScheduleDelayField, + ScheduleDateField, WorkflowTrigger, WorkflowTriggerType, } from 'src/app/data/workflow-trigger' @@ -49,21 +49,21 @@ export const DOCUMENT_SOURCE_OPTIONS = [ }, ] -export const SCHEDULE_DELAY_FIELD_OPTIONS = [ +export const SCHEDULE_DATE_FIELD_OPTIONS = [ { - id: ScheduleDelayField.Added, + id: ScheduleDateField.Added, name: $localize`Added`, }, { - id: ScheduleDelayField.Created, + id: ScheduleDateField.Created, name: $localize`Created`, }, { - id: ScheduleDelayField.Modified, + id: ScheduleDateField.Modified, name: $localize`Modified`, }, { - id: ScheduleDelayField.CustomField, + id: ScheduleDateField.CustomField, name: $localize`Custom Field`, }, ] @@ -338,11 +338,11 @@ export class WorkflowEditDialogComponent filter_has_document_type: new FormControl( trigger.filter_has_document_type ), - schedule_delay: new FormControl(trigger.schedule_delay), + schedule_offset_days: new FormControl(trigger.schedule_offset_days), schedule_is_recurring: new FormControl(trigger.schedule_is_recurring), - schedule_delay_field: new FormControl(trigger.schedule_delay_field), - schedule_delay_custom_field: new FormControl( - trigger.schedule_delay_custom_field + schedule_date_field: new FormControl(trigger.schedule_date_field), + schedule_date_custom_field: new FormControl( + trigger.schedule_date_custom_field ), }), { emitEvent } @@ -418,8 +418,8 @@ export class WorkflowEditDialogComponent return WORKFLOW_TYPE_OPTIONS } - get scheduleDelayFieldOptions() { - return SCHEDULE_DELAY_FIELD_OPTIONS + get scheduleDateFieldOptions() { + return SCHEDULE_DATE_FIELD_OPTIONS } get dateCustomFields() { @@ -448,10 +448,10 @@ export class WorkflowEditDialogComponent matching_algorithm: MATCH_NONE, match: '', is_insensitive: true, - schedule_delay: null, + schedule_offset_days: null, schedule_is_recurring: false, - schedule_delay_field: ScheduleDelayField.Added, - schedule_delay_custom_field: null, + schedule_date_field: ScheduleDateField.Added, + schedule_date_custom_field: null, } this.object.triggers.push(trigger) this.createTriggerField(trigger) diff --git a/src-ui/src/app/data/workflow-trigger.ts b/src-ui/src/app/data/workflow-trigger.ts index ed160e024..c782d0903 100644 --- a/src-ui/src/app/data/workflow-trigger.ts +++ b/src-ui/src/app/data/workflow-trigger.ts @@ -13,7 +13,7 @@ export enum WorkflowTriggerType { Scheduled = 4, } -export enum ScheduleDelayField { +export enum ScheduleDateField { Added = 'added', Created = 'created', Modified = 'modified', @@ -43,11 +43,11 @@ export interface WorkflowTrigger extends ObjectWithId { filter_has_document_type?: number // DocumentType.id - schedule_delay?: string + schedule_offset_days?: number schedule_is_recurring?: boolean - schedule_delay_field?: ScheduleDelayField + schedule_date_field?: ScheduleDateField - schedule_delay_custom_field?: number // CustomField.id + schedule_date_custom_field?: number // CustomField.id } diff --git a/src/documents/migrations/1056_workflowtrigger_schedule_delay_and_more.py b/src/documents/migrations/1056_workflowtrigger_schedule_date_custom_field_and_more.py similarity index 76% rename from src/documents/migrations/1056_workflowtrigger_schedule_delay_and_more.py rename to src/documents/migrations/1056_workflowtrigger_schedule_date_custom_field_and_more.py index 8a4b949b9..618bb7483 100644 --- a/src/documents/migrations/1056_workflowtrigger_schedule_delay_and_more.py +++ b/src/documents/migrations/1056_workflowtrigger_schedule_date_custom_field_and_more.py @@ -1,4 +1,4 @@ -# Generated by Django 5.1.1 on 2024-10-24 04:03 +# Generated by Django 5.1.1 on 2024-10-24 04:15 import django.db.models.deletion import django.utils.timezone @@ -14,29 +14,18 @@ class Migration(migrations.Migration): operations = [ migrations.AddField( model_name="workflowtrigger", - name="schedule_delay", - field=models.CharField( - blank=True, - help_text="The delay before the scheduled trigger is activated.", - max_length=256, - null=True, - verbose_name="schedule delay", - ), - ), - migrations.AddField( - model_name="workflowtrigger", - name="schedule_delay_custom_field", + name="schedule_date_custom_field", field=models.ForeignKey( blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to="documents.customfield", - verbose_name="schedule delay custom field", + verbose_name="schedule date custom field", ), ), migrations.AddField( model_name="workflowtrigger", - name="schedule_delay_field", + name="schedule_date_field", field=models.CharField( choices=[ ("added", "Added"), @@ -45,9 +34,9 @@ class Migration(migrations.Migration): ("custom_field", "Custom Field"), ], default="added", - help_text="The field to use for the delay.", + help_text="The field to check for a schedule trigger.", max_length=20, - verbose_name="schedule delay field", + verbose_name="schedule date field", ), ), migrations.AddField( @@ -59,6 +48,15 @@ class Migration(migrations.Migration): verbose_name="schedule is recurring", ), ), + migrations.AddField( + model_name="workflowtrigger", + name="schedule_offset_days", + field=models.PositiveIntegerField( + default=0, + help_text="The number of days to offset the schedule trigger by.", + verbose_name="schedule offset days", + ), + ), migrations.AlterField( model_name="workflowtrigger", name="type", @@ -85,6 +83,19 @@ class Migration(migrations.Migration): verbose_name="ID", ), ), + ( + "type", + models.PositiveIntegerField( + choices=[ + (1, "Consumption Started"), + (2, "Document Added"), + (3, "Document Updated"), + (4, "Scheduled"), + ], + null=True, + verbose_name="workflow trigger type", + ), + ), ( "run_at", models.DateTimeField( diff --git a/src/documents/models.py b/src/documents/models.py index 798075d8f..80f3baefe 100644 --- a/src/documents/models.py +++ b/src/documents/models.py @@ -1023,7 +1023,7 @@ class WorkflowTrigger(models.Model): API_UPLOAD = DocumentSource.ApiUpload.value, _("Api Upload") MAIL_FETCH = DocumentSource.MailFetch.value, _("Mail Fetch") - class ScheduleDelayField(models.TextChoices): + class ScheduleDateField(models.TextChoices): ADDED = "added", _("Added") CREATED = "created", _("Created") MODIFIED = "modified", _("Modified") @@ -1105,13 +1105,11 @@ class WorkflowTrigger(models.Model): verbose_name=_("has this correspondent"), ) - schedule_delay = models.CharField( - _("schedule delay"), - max_length=256, - null=True, - blank=True, + schedule_offset_days = models.PositiveIntegerField( + _("schedule offset days"), + default=0, help_text=_( - "The delay before the scheduled trigger is activated.", + "The number of days to offset the schedule trigger by.", ), ) @@ -1123,22 +1121,22 @@ class WorkflowTrigger(models.Model): ), ) - schedule_delay_field = models.CharField( - _("schedule delay field"), + schedule_date_field = models.CharField( + _("schedule date field"), max_length=20, - choices=ScheduleDelayField.choices, - default=ScheduleDelayField.ADDED, + choices=ScheduleDateField.choices, + default=ScheduleDateField.ADDED, help_text=_( - "The field to use for the delay.", + "The field to check for a schedule trigger.", ), ) - schedule_delay_custom_field = models.ForeignKey( + schedule_date_custom_field = models.ForeignKey( CustomField, null=True, blank=True, on_delete=models.SET_NULL, - verbose_name=_("schedule delay custom field"), + verbose_name=_("schedule date custom field"), ) class Meta: @@ -1401,6 +1399,12 @@ class WorkflowRun(models.Model): verbose_name=_("workflow"), ) + type = models.PositiveIntegerField( + _("workflow trigger type"), + choices=WorkflowTrigger.WorkflowTriggerType.choices, + null=True, + ) + document = models.ForeignKey( Document, null=True, diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py index 764d0a8aa..04e2867be 100644 --- a/src/documents/serialisers.py +++ b/src/documents/serialisers.py @@ -21,7 +21,6 @@ from django.utils.crypto import get_random_string from django.utils.text import slugify from django.utils.translation import gettext as _ from drf_writable_nested.serializers import NestedUpdateMixin -from duration_parser import parse_timedelta from guardian.core import ObjectPermissionChecker from guardian.shortcuts import get_users_with_perms from guardian.utils import get_group_obj_perms_model @@ -1753,22 +1752,12 @@ class WorkflowTriggerSerializer(serializers.ModelSerializer): "filter_has_tags", "filter_has_correspondent", "filter_has_document_type", - "schedule_delay", + "schedule_offset_days", "schedule_is_recurring", - "schedule_delay_field", - "schedule_delay_custom_field", + "schedule_date_field", + "schedule_date_custom_field", ] - def validate_schedule_delay(self, value): - if value is not None: - try: - parse_timedelta(value) - except Exception as e: - raise serializers.ValidationError( - f"Invalid schedule delay format: {e}", - ) - return value - def validate(self, attrs): # Empty strings treated as None to avoid unexpected behavior if ( @@ -1794,11 +1783,6 @@ class WorkflowTriggerSerializer(serializers.ModelSerializer): "File name, path or mail rule filter are required", ) - if attrs["type"] == WorkflowTrigger.WorkflowTriggerType.SCHEDULED and ( - "schedule_delay" not in attrs or attrs["schedule_delay"] is None - ): - raise serializers.ValidationError("Schedule delay is required") - return attrs diff --git a/src/documents/signals/handlers.py b/src/documents/signals/handlers.py index 0a8439085..1a6219761 100644 --- a/src/documents/signals/handlers.py +++ b/src/documents/signals/handlers.py @@ -918,6 +918,7 @@ def run_workflows( WorkflowRun.objects.create( workflow=workflow, + type=trigger_type, document=document if not use_overrides else None, ) diff --git a/src/documents/tasks.py b/src/documents/tasks.py index 5ed23b1af..a907fb6e8 100644 --- a/src/documents/tasks.py +++ b/src/documents/tasks.py @@ -14,7 +14,6 @@ from django.db import models from django.db import transaction from django.db.models.signals import post_save from django.utils import timezone -from duration_parser import parse_timedelta from filelock import FileLock from whoosh.writing import AsyncWriter @@ -342,38 +341,38 @@ def check_scheduled_workflows(): trigger: WorkflowTrigger for trigger in schedule_triggers: documents = Document.objects.none() - delay_td = parse_timedelta(trigger.schedule_delay) + offset_td = timedelta(days=trigger.schedule_offset_days) logger.debug( - f"Checking trigger {trigger} with delay {delay_td} against field: {trigger.schedule_delay_field}", + f"Checking trigger {trigger} with offset {offset_td} against field: {trigger.schedule_date_field}", ) if ( - trigger.schedule_delay_field - == WorkflowTrigger.ScheduleDelayField.ADDED + trigger.schedule_date_field + == WorkflowTrigger.ScheduleDateField.ADDED ): documents = Document.objects.filter( - added__lt=timezone.now() - delay_td, + added__lt=timezone.now() - offset_td, ) elif ( - trigger.schedule_delay_field - == WorkflowTrigger.ScheduleDelayField.CREATED + trigger.schedule_date_field + == WorkflowTrigger.ScheduleDateField.CREATED ): documents = Document.objects.filter( - created__lt=timezone.now() - delay_td, + created__lt=timezone.now() - offset_td, ) elif ( - trigger.schedule_delay_field - == WorkflowTrigger.ScheduleDelayField.MODIFIED + trigger.schedule_date_field + == WorkflowTrigger.ScheduleDateField.MODIFIED ): documents = Document.objects.filter( - modified__lt=timezone.now() - delay_td, + modified__lt=timezone.now() - offset_td, ) elif ( - trigger.schedule_delay_field - == WorkflowTrigger.ScheduleDelayField.CUSTOM_FIELD + trigger.schedule_date_field + == WorkflowTrigger.ScheduleDateField.CUSTOM_FIELD ): cf_instances = CustomFieldInstance.objects.filter( - field=trigger.schedule_delay_custom_field, - value_date__lt=timezone.now() - delay_td, + field=trigger.schedule_date_custom_field, + value_date__lt=timezone.now() - offset_td, ) documents = Document.objects.filter( id__in=cf_instances.values_list("document", flat=True), @@ -385,6 +384,7 @@ def check_scheduled_workflows(): for document in documents: workflow_runs = WorkflowRun.objects.filter( document=document, + type=WorkflowTrigger.WorkflowTriggerType.SCHEDULED, workflow=workflow, ) if not trigger.schedule_is_recurring and workflow_runs.exists(): @@ -396,7 +396,7 @@ def check_scheduled_workflows(): elif ( trigger.schedule_is_recurring and workflow_runs.exists() - and workflow_runs.last().run_at > timezone.now() - delay_td + and workflow_runs.last().run_at > timezone.now() - offset_td ): # schedule is recurring but the last run was within the delay logger.debug( diff --git a/src/documents/tests/test_workflows.py b/src/documents/tests/test_workflows.py index 9786252cb..42dc34325 100644 --- a/src/documents/tests/test_workflows.py +++ b/src/documents/tests/test_workflows.py @@ -1318,8 +1318,8 @@ class TestWorkflows(DirectoriesMixin, FileSystemAssertsMixin, APITestCase): """ trigger = WorkflowTrigger.objects.create( type=WorkflowTrigger.WorkflowTriggerType.SCHEDULED, - schedule_delay="1 day", - schedule_delay_field="created", + schedule_offset_days=1, + schedule_date_field="created", ) action = WorkflowAction.objects.create( assign_title="Doc assign owner", @@ -1359,8 +1359,8 @@ class TestWorkflows(DirectoriesMixin, FileSystemAssertsMixin, APITestCase): """ trigger = WorkflowTrigger.objects.create( type=WorkflowTrigger.WorkflowTriggerType.SCHEDULED, - schedule_delay="1 day", - schedule_delay_field=WorkflowTrigger.ScheduleDelayField.ADDED, + schedule_offset_days=1, + schedule_date_field=WorkflowTrigger.ScheduleDateField.ADDED, ) action = WorkflowAction.objects.create( assign_title="Doc assign owner", @@ -1402,8 +1402,8 @@ class TestWorkflows(DirectoriesMixin, FileSystemAssertsMixin, APITestCase): mock_filter.return_value = Document.objects.all() trigger = WorkflowTrigger.objects.create( type=WorkflowTrigger.WorkflowTriggerType.SCHEDULED, - schedule_delay="1 day", - schedule_delay_field=WorkflowTrigger.ScheduleDelayField.MODIFIED, + schedule_offset_days=1, + schedule_date_field=WorkflowTrigger.ScheduleDateField.MODIFIED, ) action = WorkflowAction.objects.create( assign_title="Doc assign owner",