From f6c004183eee674f2e38ae5dd8e262bbde6cd578 Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Mon, 13 Oct 2025 15:23:56 -0700 Subject: [PATCH] Feature: Advanced Workflow Trigger Filters (#11029) --- docs/usage.md | 19 +- ...ustom-fields-query-dropdown.component.html | 52 +- .../custom-fields-query-dropdown.component.ts | 17 +- .../workflow-edit-dialog.component.html | 100 ++- .../workflow-edit-dialog.component.scss | 4 + .../workflow-edit-dialog.component.spec.ts | 566 ++++++++++++++- .../workflow-edit-dialog.component.ts | 668 +++++++++++++++++- .../common/input/select/select.component.html | 118 ++-- .../common/input/tags/tags.component.html | 8 +- src-ui/src/app/data/workflow-trigger.ts | 12 + src/documents/matching.py | 230 ++++-- ...gger_filter_custom_field_query_and_more.py | 73 ++ src/documents/models.py | 42 ++ src/documents/serialisers.py | 51 ++ src/documents/tests/test_api_workflows.py | 73 ++ src/documents/tests/test_workflows.py | 405 ++++++++++- 16 files changed, 2267 insertions(+), 171 deletions(-) create mode 100644 src/documents/migrations/1072_workflowtrigger_filter_custom_field_query_and_more.py diff --git a/docs/usage.md b/docs/usage.md index 8bec8b059..d902f6814 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -462,15 +462,24 @@ flowchart TD Workflows allow you to filter by: - Source, e.g. documents uploaded via consume folder, API (& the web UI) and mail fetch -- File name, including wildcards e.g. \*.pdf will apply to all pdfs +- File name, including wildcards e.g. \*.pdf will apply to all pdfs. - File path, including wildcards. Note that enabling `PAPERLESS_CONSUMER_RECURSIVE` would allow, for example, automatically assigning documents to different owners based on the upload directory. - Mail rule. Choosing this option will force 'mail fetch' to be the workflow source. - Content matching (`Added`, `Updated` and `Scheduled` triggers only). Filter document content using the matching settings. -- Tags (`Added`, `Updated` and `Scheduled` triggers only). Filter for documents with any of the specified tags -- Document type (`Added`, `Updated` and `Scheduled` triggers only). Filter documents with this doc type -- Correspondent (`Added`, `Updated` and `Scheduled` triggers only). Filter documents with this correspondent -- Storage path (`Added`, `Updated` and `Scheduled` triggers only). Filter documents with this storage path + +There are also 'advanced' filters available for `Added`, `Updated` and `Scheduled` triggers: + +- Any Tags: Filter for documents with any of the specified tags. +- All Tags: Filter for documents with all of the specified tags. +- No Tags: Filter for documents with none of the specified tags. +- Document type: Filter documents with this document type. +- Not Document types: Filter documents without any of these document types. +- Correspondent: Filter documents with this correspondent. +- Not Correspondents: Filter documents without any of these correspondents. +- Storage path: Filter documents with this storage path. +- Not Storage paths: Filter documents without any of these storage paths. +- Custom field query: Filter documents with a custom field query (the same as used for the document list filters). ### Workflow Actions diff --git a/src-ui/src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.html b/src-ui/src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.html index 57aff1bd9..a8973e702 100644 --- a/src-ui/src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.html +++ b/src-ui/src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.html @@ -1,28 +1,36 @@ -
- -
-
- @for (element of selectionModel.queries; track element.id; let i = $index) { -
- @switch (element.type) { - @case (CustomFieldQueryComponentType.Atom) { - - } - @case (CustomFieldQueryComponentType.Expression) { - - } - } -
+@if (useDropdown) { +
+ +
+
-
+} @else { + +} + + +
+ @for (element of queries; track element.id; let i = $index) { +
+ @switch (element.type) { + @case (CustomFieldQueryComponentType.Atom) { + + } + @case (CustomFieldQueryComponentType.Expression) { + + } + } +
+ } +
+ @if (getCustomFieldByID(atom.field)?.data_type === CustomFieldDataType.Date) { diff --git a/src-ui/src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.ts b/src-ui/src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.ts index ef56d6ac5..fc4e8ef19 100644 --- a/src-ui/src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.ts +++ b/src-ui/src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.ts @@ -120,6 +120,12 @@ export class CustomFieldQueriesModel { }) } + addInitialAtom() { + this.addAtom( + new CustomFieldQueryAtom([null, CustomFieldQueryOperator.Exists, 'true']) + ) + } + private findElement( queryElement: CustomFieldQueryElement, elements: any[] @@ -206,6 +212,9 @@ export class CustomFieldsQueryDropdownComponent extends LoadingComponentWithPerm @Input() applyOnClose = false + @Input() + useDropdown: boolean = true + get name(): string { return this.title ? this.title.replace(/\s/g, '_').toLowerCase() : null } @@ -258,13 +267,7 @@ export class CustomFieldsQueryDropdownComponent extends LoadingComponentWithPerm public onOpenChange(open: boolean) { if (open) { if (this.selectionModel.queries.length === 0) { - this.selectionModel.addAtom( - new CustomFieldQueryAtom([ - null, - CustomFieldQueryOperator.Exists, - 'true', - ]) - ) + this.selectionModel.addInitialAtom() } if ( this.selectionModel.queries.length === 1 && diff --git a/src-ui/src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html b/src-ui/src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html index 7163ba289..61daa1fa2 100644 --- a/src-ui/src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html +++ b/src-ui/src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html @@ -156,31 +156,97 @@

Trigger for documents that match all filters specified below.

- + @if (formGroup.get('type').value === WorkflowTriggerType.Consumption) { - - - + + + } @if (formGroup.get('type').value === WorkflowTriggerType.DocumentAdded || formGroup.get('type').value === WorkflowTriggerType.DocumentUpdated || formGroup.get('type').value === WorkflowTriggerType.Scheduled) { - - @if (patternRequired) { - + + @if (matchingPatternRequired(formGroup)) { + } - @if (patternRequired) { - + @if (matchingPatternRequired(formGroup)) { + } }
- @if (formGroup.get('type').value === WorkflowTriggerType.DocumentAdded || formGroup.get('type').value === WorkflowTriggerType.DocumentUpdated || formGroup.get('type').value === WorkflowTriggerType.Scheduled) { -
- - - - -
- }
+ @if (formGroup.get('type').value === WorkflowTriggerType.DocumentAdded || formGroup.get('type').value === WorkflowTriggerType.DocumentUpdated || formGroup.get('type').value === WorkflowTriggerType.Scheduled) { +
+
+
+
+ + +
+
    + @if (getFiltersFormArray(formGroup).length === 0) { +

    No advanced workflow filters defined.

    + } + @for (filter of getFiltersFormArray(formGroup).controls; track filter; let filterIndex = $index) { +
  • +
    +
    + +
    +
    + @if (isTagsFilter(filter.get('type').value)) { + + } @else if ( + isCustomFieldQueryFilter(filter.get('type').value) + ) { + + @if (!isCustomFieldQueryValid(filter)) { +
    + Complete the custom field query configuration. +
    + } + } @else { + + } +
    + +
    +
    • + } +
+
+
+
+ }
diff --git a/src-ui/src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.scss b/src-ui/src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.scss index 6cfcf86b4..d026a5b2b 100644 --- a/src-ui/src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.scss +++ b/src-ui/src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.scss @@ -7,3 +7,7 @@ .accordion-button { font-size: 1rem; } + +:host ::ng-deep .filters .paperless-input-select.mb-3 { + margin-bottom: 0 !important; +} diff --git a/src-ui/src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.spec.ts b/src-ui/src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.spec.ts index 930164dce..0736e2215 100644 --- a/src-ui/src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.spec.ts +++ b/src-ui/src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.spec.ts @@ -11,8 +11,14 @@ import { import { NgbActiveModal, NgbModule } from '@ng-bootstrap/ng-bootstrap' import { NgSelectModule } from '@ng-select/ng-select' import { of } from 'rxjs' +import { CustomFieldQueriesModel } from 'src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component' import { CustomFieldDataType } from 'src/app/data/custom-field' -import { MATCHING_ALGORITHMS, MATCH_AUTO } from 'src/app/data/matching-model' +import { CustomFieldQueryLogicalOperator } from 'src/app/data/custom-field-query' +import { + MATCHING_ALGORITHMS, + MATCH_AUTO, + MATCH_NONE, +} from 'src/app/data/matching-model' import { Workflow } from 'src/app/data/workflow' import { WorkflowAction, @@ -31,6 +37,7 @@ import { DocumentTypeService } from 'src/app/services/rest/document-type.service import { MailRuleService } from 'src/app/services/rest/mail-rule.service' import { StoragePathService } from 'src/app/services/rest/storage-path.service' import { SettingsService } from 'src/app/services/settings.service' +import { CustomFieldQueryExpression } from 'src/app/utils/custom-field-query-element' import { ConfirmButtonComponent } from '../../confirm-button/confirm-button.component' import { NumberComponent } from '../../input/number/number.component' import { PermissionsGroupComponent } from '../../input/permissions/permissions-group/permissions-group.component' @@ -43,6 +50,7 @@ import { EditDialogMode } from '../edit-dialog.component' import { DOCUMENT_SOURCE_OPTIONS, SCHEDULE_DATE_FIELD_OPTIONS, + TriggerFilterType, WORKFLOW_ACTION_OPTIONS, WORKFLOW_TYPE_OPTIONS, WorkflowEditDialogComponent, @@ -375,6 +383,562 @@ describe('WorkflowEditDialogComponent', () => { expect(component.objectForm.get('actions').value[0].webhook).toBeNull() }) + it('should require matching pattern when algorithm is not none', () => { + const triggerGroup = new FormGroup({ + matching_algorithm: new FormControl(MATCH_AUTO), + match: new FormControl(''), + }) + expect(component.matchingPatternRequired(triggerGroup)).toBe(true) + triggerGroup.get('matching_algorithm').setValue(MATCHING_ALGORITHMS[0].id) + expect(component.matchingPatternRequired(triggerGroup)).toBe(true) + triggerGroup.get('matching_algorithm').setValue(MATCH_NONE) + expect(component.matchingPatternRequired(triggerGroup)).toBe(false) + }) + + it('should map filter builder values into trigger filters on save', () => { + component.object = undefined + component.addTrigger() + const triggerGroup = component.triggerFields.at(0) + component.addFilter(triggerGroup as FormGroup) + component.addFilter(triggerGroup as FormGroup) + component.addFilter(triggerGroup as FormGroup) + + const filters = component.getFiltersFormArray(triggerGroup as FormGroup) + expect(filters.length).toBe(3) + + filters.at(0).get('values').setValue([1]) + filters.at(1).get('values').setValue([2, 3]) + filters.at(2).get('values').setValue([4]) + + const addFilterOfType = (type: TriggerFilterType) => { + const newFilter = component.addFilter(triggerGroup as FormGroup) + newFilter.get('type').setValue(type) + return newFilter + } + + const correspondentIs = addFilterOfType(TriggerFilterType.CorrespondentIs) + correspondentIs.get('values').setValue(1) + + const correspondentNot = addFilterOfType(TriggerFilterType.CorrespondentNot) + correspondentNot.get('values').setValue([1]) + + const documentTypeIs = addFilterOfType(TriggerFilterType.DocumentTypeIs) + documentTypeIs.get('values').setValue(1) + + const documentTypeNot = addFilterOfType(TriggerFilterType.DocumentTypeNot) + documentTypeNot.get('values').setValue([1]) + + const storagePathIs = addFilterOfType(TriggerFilterType.StoragePathIs) + storagePathIs.get('values').setValue(1) + + const storagePathNot = addFilterOfType(TriggerFilterType.StoragePathNot) + storagePathNot.get('values').setValue([1]) + + const customFieldFilter = addFilterOfType( + TriggerFilterType.CustomFieldQuery + ) + const customFieldQuery = JSON.stringify(['AND', [[1, 'exact', 'test']]]) + customFieldFilter.get('values').setValue(customFieldQuery) + + const formValues = component['getFormValues']() + + expect(formValues.triggers[0].filter_has_tags).toEqual([1]) + expect(formValues.triggers[0].filter_has_all_tags).toEqual([2, 3]) + expect(formValues.triggers[0].filter_has_not_tags).toEqual([4]) + expect(formValues.triggers[0].filter_has_correspondent).toEqual(1) + expect(formValues.triggers[0].filter_has_not_correspondents).toEqual([1]) + expect(formValues.triggers[0].filter_has_document_type).toEqual(1) + expect(formValues.triggers[0].filter_has_not_document_types).toEqual([1]) + expect(formValues.triggers[0].filter_has_storage_path).toEqual(1) + expect(formValues.triggers[0].filter_has_not_storage_paths).toEqual([1]) + expect(formValues.triggers[0].filter_custom_field_query).toEqual( + customFieldQuery + ) + expect(formValues.triggers[0].filters).toBeUndefined() + }) + + it('should ignore empty and null filter values when mapping filters', () => { + component.object = undefined + component.addTrigger() + const triggerGroup = component.triggerFields.at(0) as FormGroup + + const tagsFilter = component.addFilter(triggerGroup) + tagsFilter.get('type').setValue(TriggerFilterType.TagsAny) + tagsFilter.get('values').setValue([]) + + const correspondentFilter = component.addFilter(triggerGroup) + correspondentFilter.get('type').setValue(TriggerFilterType.CorrespondentIs) + correspondentFilter.get('values').setValue(null) + + const formValues = component['getFormValues']() + + expect(formValues.triggers[0].filter_has_tags).toEqual([]) + expect(formValues.triggers[0].filter_has_correspondent).toBeNull() + }) + + it('should derive single select filters from array values', () => { + component.object = undefined + component.addTrigger() + const triggerGroup = component.triggerFields.at(0) as FormGroup + + const addFilterOfType = (type: TriggerFilterType, value: any) => { + const filter = component.addFilter(triggerGroup) + filter.get('type').setValue(type) + filter.get('values').setValue(value) + } + + addFilterOfType(TriggerFilterType.CorrespondentIs, [5]) + addFilterOfType(TriggerFilterType.DocumentTypeIs, [6]) + addFilterOfType(TriggerFilterType.StoragePathIs, [7]) + + const formValues = component['getFormValues']() + + expect(formValues.triggers[0].filter_has_correspondent).toEqual(5) + expect(formValues.triggers[0].filter_has_document_type).toEqual(6) + expect(formValues.triggers[0].filter_has_storage_path).toEqual(7) + }) + + it('should convert multi-value filter values when aggregating filters', () => { + component.object = undefined + component.addTrigger() + const triggerGroup = component.triggerFields.at(0) as FormGroup + + const setFilter = (type: TriggerFilterType, value: number): void => { + const filter = component.addFilter(triggerGroup) as FormGroup + filter.get('type').setValue(type) + filter.get('values').setValue(value) + } + + setFilter(TriggerFilterType.TagsAll, 11) + setFilter(TriggerFilterType.TagsNone, 12) + setFilter(TriggerFilterType.CorrespondentNot, 13) + setFilter(TriggerFilterType.DocumentTypeNot, 14) + setFilter(TriggerFilterType.StoragePathNot, 15) + + const formValues = component['getFormValues']() + + expect(formValues.triggers[0].filter_has_all_tags).toEqual([11]) + expect(formValues.triggers[0].filter_has_not_tags).toEqual([12]) + expect(formValues.triggers[0].filter_has_not_correspondents).toEqual([13]) + expect(formValues.triggers[0].filter_has_not_document_types).toEqual([14]) + expect(formValues.triggers[0].filter_has_not_storage_paths).toEqual([15]) + }) + + it('should reuse filter type options and update disabled state', () => { + component.object = undefined + component.addTrigger() + const triggerGroup = component.triggerFields.at(0) as FormGroup + component.addFilter(triggerGroup) + + const optionsFirst = component.getFilterTypeOptions(triggerGroup, 0) + const optionsSecond = component.getFilterTypeOptions(triggerGroup, 0) + expect(optionsFirst).toBe(optionsSecond) + + // to force disabled flag + component.addFilter(triggerGroup) + const filterArray = component.getFiltersFormArray(triggerGroup) + const firstFilter = filterArray.at(0) + firstFilter.get('type').setValue(TriggerFilterType.CorrespondentIs) + + component.addFilter(triggerGroup) + const updatedFilters = component.getFiltersFormArray(triggerGroup) + const secondFilter = updatedFilters.at(1) + const options = component.getFilterTypeOptions(triggerGroup, 1) + const correspondentIsOption = options.find( + (option) => option.id === TriggerFilterType.CorrespondentIs + ) + expect(correspondentIsOption.disabled).toBe(true) + + firstFilter.get('type').setValue(TriggerFilterType.DocumentTypeNot) + secondFilter.get('type').setValue(TriggerFilterType.TagsAll) + const postChangeOptions = component.getFilterTypeOptions(triggerGroup, 1) + const correspondentOptionAfter = postChangeOptions.find( + (option) => option.id === TriggerFilterType.CorrespondentIs + ) + expect(correspondentOptionAfter.disabled).toBe(false) + }) + + it('should keep multi-entry filter options enabled and allow duplicates', () => { + component.object = undefined + component.addTrigger() + const triggerGroup = component.triggerFields.at(0) as FormGroup + + component.filterDefinitions = [ + { + id: TriggerFilterType.TagsAny, + name: 'Any tags', + inputType: 'tags', + allowMultipleEntries: true, + allowMultipleValues: true, + } as any, + { + id: TriggerFilterType.CorrespondentIs, + name: 'Correspondent is', + inputType: 'select', + allowMultipleEntries: false, + allowMultipleValues: false, + selectItems: 'correspondents', + } as any, + ] + + const firstFilter = component.addFilter(triggerGroup) + firstFilter.get('type').setValue(TriggerFilterType.TagsAny) + + const secondFilter = component.addFilter(triggerGroup) + expect(secondFilter).not.toBeNull() + + const options = component.getFilterTypeOptions(triggerGroup, 1) + const multiEntryOption = options.find( + (option) => option.id === TriggerFilterType.TagsAny + ) + + expect(multiEntryOption.disabled).toBe(false) + expect(component.canAddFilter(triggerGroup)).toBe(true) + }) + + it('should return null when no filter definitions remain available', () => { + component.object = undefined + component.addTrigger() + const triggerGroup = component.triggerFields.at(0) as FormGroup + + component.filterDefinitions = [ + { + id: TriggerFilterType.TagsAny, + name: 'Any tags', + inputType: 'tags', + allowMultipleEntries: false, + allowMultipleValues: true, + } as any, + { + id: TriggerFilterType.CorrespondentIs, + name: 'Correspondent is', + inputType: 'select', + allowMultipleEntries: false, + allowMultipleValues: false, + selectItems: 'correspondents', + } as any, + ] + + const firstFilter = component.addFilter(triggerGroup) + firstFilter.get('type').setValue(TriggerFilterType.TagsAny) + const secondFilter = component.addFilter(triggerGroup) + secondFilter.get('type').setValue(TriggerFilterType.CorrespondentIs) + + expect(component.canAddFilter(triggerGroup)).toBe(false) + expect(component.addFilter(triggerGroup)).toBeNull() + }) + + it('should skip filter definitions without handlers when building form array', () => { + const originalDefinitions = component.filterDefinitions + component.filterDefinitions = [ + { + id: 999, + name: 'Unsupported', + inputType: 'text', + allowMultipleEntries: false, + allowMultipleValues: false, + } as any, + ] + + const trigger = { + filter_has_tags: [], + filter_has_all_tags: [], + filter_has_not_tags: [], + filter_has_not_correspondents: [], + filter_has_not_document_types: [], + filter_has_not_storage_paths: [], + filter_has_correspondent: null, + filter_has_document_type: null, + filter_has_storage_path: null, + filter_custom_field_query: null, + } as any + + const filters = component['buildFiltersFormArray'](trigger) + expect(filters.length).toBe(0) + + component.filterDefinitions = originalDefinitions + }) + + it('should return null when adding filter for unknown trigger form group', () => { + expect(component.addFilter(new FormGroup({}) as any)).toBeNull() + }) + + it('should ignore remove filter calls for unknown trigger form group', () => { + expect(() => + component.removeFilter(new FormGroup({}) as any, 0) + ).not.toThrow() + }) + + it('should teardown custom field query model when removing a custom field filter', () => { + component.object = undefined + component.addTrigger() + const triggerGroup = component.triggerFields.at(0) as FormGroup + + component.addFilter(triggerGroup) + const filters = component.getFiltersFormArray(triggerGroup) + const filterGroup = filters.at(0) as FormGroup + filterGroup.get('type').setValue(TriggerFilterType.CustomFieldQuery) + + const model = component.getCustomFieldQueryModel(filterGroup) + expect(model).toBeDefined() + expect( + component['getStoredCustomFieldQueryModel'](filterGroup as any) + ).toBe(model) + + component.removeFilter(triggerGroup, 0) + expect( + component['getStoredCustomFieldQueryModel'](filterGroup as any) + ).toBeNull() + }) + + it('should return readable filter names', () => { + expect(component.getFilterName(TriggerFilterType.TagsAny)).toBe( + 'Has any of these tags' + ) + expect(component.getFilterName(999 as any)).toBe('') + }) + + it('should build filter form array from existing trigger filters', () => { + const trigger = workflow.triggers[0] + trigger.filter_has_tags = [1] + trigger.filter_has_all_tags = [2, 3] + trigger.filter_has_not_tags = [4] + trigger.filter_has_correspondent = 5 as any + trigger.filter_has_not_correspondents = [6] as any + trigger.filter_has_document_type = 7 as any + trigger.filter_has_not_document_types = [8] as any + trigger.filter_has_storage_path = 9 as any + trigger.filter_has_not_storage_paths = [10] as any + trigger.filter_custom_field_query = JSON.stringify([ + 'AND', + [[1, 'exact', 'value']], + ]) as any + + component.object = workflow + component.ngOnInit() + const triggerGroup = component.triggerFields.at(0) as FormGroup + const filters = component.getFiltersFormArray(triggerGroup) + expect(filters.length).toBe(10) + const customFieldFilter = filters.at(9) as FormGroup + expect(customFieldFilter.get('type').value).toBe( + TriggerFilterType.CustomFieldQuery + ) + const model = component.getCustomFieldQueryModel(customFieldFilter) + expect(model.isValid()).toBe(true) + }) + + it('should expose select metadata helpers', () => { + expect(component.isSelectMultiple(TriggerFilterType.CorrespondentNot)).toBe( + true + ) + expect(component.isSelectMultiple(TriggerFilterType.CorrespondentIs)).toBe( + false + ) + + component.correspondents = [{ id: 1, name: 'C1' } as any] + component.documentTypes = [{ id: 2, name: 'DT' } as any] + component.storagePaths = [{ id: 3, name: 'SP' } as any] + + expect( + component.getFilterSelectItems(TriggerFilterType.CorrespondentIs) + ).toEqual(component.correspondents) + expect( + component.getFilterSelectItems(TriggerFilterType.DocumentTypeIs) + ).toEqual(component.documentTypes) + expect( + component.getFilterSelectItems(TriggerFilterType.StoragePathIs) + ).toEqual(component.storagePaths) + expect(component.getFilterSelectItems(TriggerFilterType.TagsAll)).toEqual( + [] + ) + + expect( + component.isCustomFieldQueryFilter(TriggerFilterType.CustomFieldQuery) + ).toBe(true) + }) + + it('should return empty select items when definition is missing', () => { + const originalDefinitions = component.filterDefinitions + component.filterDefinitions = [] + + expect( + component.getFilterSelectItems(TriggerFilterType.CorrespondentIs) + ).toEqual([]) + + component.filterDefinitions = originalDefinitions + }) + + it('should return empty select items when definition has unknown source', () => { + const originalDefinitions = component.filterDefinitions + component.filterDefinitions = [ + { + id: TriggerFilterType.CorrespondentIs, + name: 'Correspondent is', + inputType: 'select', + allowMultipleEntries: false, + allowMultipleValues: false, + selectItems: 'unknown', + } as any, + ] + + expect( + component.getFilterSelectItems(TriggerFilterType.CorrespondentIs) + ).toEqual([]) + + component.filterDefinitions = originalDefinitions + }) + + it('should handle custom field query selection change and validation states', () => { + const formGroup = new FormGroup({ + values: new FormControl(null), + }) + const model = new CustomFieldQueriesModel() + + const changeSpy = jest.spyOn( + component as any, + 'onCustomFieldQueryModelChanged' + ) + + component.onCustomFieldQuerySelectionChange(formGroup, model) + expect(changeSpy).toHaveBeenCalledWith(formGroup, model) + + expect(component.isCustomFieldQueryValid(formGroup)).toBe(true) + component['setCustomFieldQueryModel'](formGroup as any, model as any) + + const validSpy = jest.spyOn(model, 'isValid').mockReturnValue(false) + const emptySpy = jest.spyOn(model, 'isEmpty').mockReturnValue(false) + expect(component.isCustomFieldQueryValid(formGroup)).toBe(false) + expect(validSpy).toHaveBeenCalled() + + validSpy.mockReturnValue(true) + emptySpy.mockReturnValue(true) + expect(component.isCustomFieldQueryValid(formGroup)).toBe(true) + + emptySpy.mockReturnValue(false) + expect(component.isCustomFieldQueryValid(formGroup)).toBe(true) + + component['clearCustomFieldQueryModel'](formGroup as any) + }) + + it('should recover from invalid custom field query json and update control on changes', () => { + const filterGroup = new FormGroup({ + values: new FormControl('not-json'), + }) + + component['ensureCustomFieldQueryModel'](filterGroup, 'not-json') + + const model = component['getStoredCustomFieldQueryModel']( + filterGroup as any + ) + expect(model).toBeDefined() + expect(model.queries.length).toBeGreaterThan(0) + + const valuesControl = filterGroup.get('values') + expect(valuesControl.value).toBeNull() + + const expression = new CustomFieldQueryExpression([ + CustomFieldQueryLogicalOperator.And, + [[1, 'exact', 'value']], + ]) + model.queries = [expression] + + jest.spyOn(model, 'isValid').mockReturnValue(true) + jest.spyOn(model, 'isEmpty').mockReturnValue(false) + + model.changed.next(model) + + expect(valuesControl.value).toEqual(JSON.stringify(expression.serialize())) + + component['clearCustomFieldQueryModel'](filterGroup as any) + }) + + it('should handle custom field query model change edge cases', () => { + const groupWithoutControl = new FormGroup({}) + const dummyModel = { + isValid: jest.fn().mockReturnValue(true), + isEmpty: jest.fn().mockReturnValue(false), + } + + expect(() => + component['onCustomFieldQueryModelChanged']( + groupWithoutControl as any, + dummyModel as any + ) + ).not.toThrow() + + const groupWithControl = new FormGroup({ + values: new FormControl('initial'), + }) + const emptyModel = { + isValid: jest.fn().mockReturnValue(true), + isEmpty: jest.fn().mockReturnValue(true), + } + + component['onCustomFieldQueryModelChanged']( + groupWithControl as any, + emptyModel as any + ) + + expect(groupWithControl.get('values').value).toBeNull() + }) + + it('should normalize filter values for single and multi selects', () => { + expect( + component['normalizeFilterValue'](TriggerFilterType.TagsAny) + ).toEqual([]) + expect( + component['normalizeFilterValue'](TriggerFilterType.TagsAny, 5) + ).toEqual([5]) + expect( + component['normalizeFilterValue'](TriggerFilterType.TagsAny, [5, 6]) + ).toEqual([5, 6]) + expect( + component['normalizeFilterValue'](TriggerFilterType.CorrespondentIs, [7]) + ).toEqual(7) + expect( + component['normalizeFilterValue'](TriggerFilterType.CorrespondentIs, 8) + ).toEqual(8) + const customFieldJson = JSON.stringify(['AND', [[1, 'exact', 'test']]]) + expect( + component['normalizeFilterValue']( + TriggerFilterType.CustomFieldQuery, + customFieldJson + ) + ).toEqual(customFieldJson) + + const customFieldObject = ['AND', [[1, 'exact', 'other']]] + expect( + component['normalizeFilterValue']( + TriggerFilterType.CustomFieldQuery, + customFieldObject + ) + ).toEqual(JSON.stringify(customFieldObject)) + + expect( + component['normalizeFilterValue']( + TriggerFilterType.CustomFieldQuery, + false + ) + ).toBeNull() + }) + + it('should add and remove filter form groups', () => { + component['changeDetector'] = { detectChanges: jest.fn() } as any + component.object = undefined + component.addTrigger() + const triggerGroup = component.triggerFields.at(0) as FormGroup + + component.addFilter(triggerGroup) + + component.removeFilter(triggerGroup, 0) + expect(component.getFiltersFormArray(triggerGroup).length).toBe(0) + + component.addFilter(triggerGroup) + const filterArrayAfterAdd = component.getFiltersFormArray(triggerGroup) + filterArrayAfterAdd.at(0).get('type').setValue(TriggerFilterType.TagsAll) + expect(component.getFiltersFormArray(triggerGroup).length).toBe(1) + }) + it('should remove selected custom field from the form group', () => { const formGroup = new FormGroup({ assign_custom_fields: new FormControl([1, 2, 3]), diff --git a/src-ui/src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts b/src-ui/src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts index ec27d6c59..f6d9e60f5 100644 --- a/src-ui/src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts +++ b/src-ui/src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts @@ -6,6 +6,7 @@ import { import { NgTemplateOutlet } from '@angular/common' import { Component, OnInit, inject } from '@angular/core' import { + AbstractControl, FormArray, FormControl, FormGroup, @@ -14,7 +15,7 @@ import { } from '@angular/forms' import { NgbAccordionModule } from '@ng-bootstrap/ng-bootstrap' import { NgxBootstrapIconsModule } from 'ngx-bootstrap-icons' -import { first } from 'rxjs' +import { Subscription, first, takeUntil } from 'rxjs' import { Correspondent } from 'src/app/data/correspondent' import { CustomField, CustomFieldDataType } from 'src/app/data/custom-field' import { DocumentType } from 'src/app/data/document-type' @@ -45,7 +46,12 @@ import { StoragePathService } from 'src/app/services/rest/storage-path.service' import { UserService } from 'src/app/services/rest/user.service' import { WorkflowService } from 'src/app/services/rest/workflow.service' import { SettingsService } from 'src/app/services/settings.service' +import { CustomFieldQueryExpression } from 'src/app/utils/custom-field-query-element' import { ConfirmButtonComponent } from '../../confirm-button/confirm-button.component' +import { + CustomFieldQueriesModel, + CustomFieldsQueryDropdownComponent, +} from '../../custom-fields-query-dropdown/custom-fields-query-dropdown.component' import { CheckComponent } from '../../input/check/check.component' import { CustomFieldsValuesComponent } from '../../input/custom-fields-values/custom-fields-values.component' import { EntriesComponent } from '../../input/entries/entries.component' @@ -135,10 +141,235 @@ export const WORKFLOW_ACTION_OPTIONS = [ }, ] +export enum TriggerFilterType { + TagsAny = 'tags_any', + TagsAll = 'tags_all', + TagsNone = 'tags_none', + CorrespondentIs = 'correspondent_is', + CorrespondentNot = 'correspondent_not', + DocumentTypeIs = 'document_type_is', + DocumentTypeNot = 'document_type_not', + StoragePathIs = 'storage_path_is', + StoragePathNot = 'storage_path_not', + CustomFieldQuery = 'custom_field_query', +} + +interface TriggerFilterDefinition { + id: TriggerFilterType + name: string + inputType: 'tags' | 'select' | 'customFieldQuery' + allowMultipleEntries: boolean + allowMultipleValues: boolean + selectItems?: 'correspondents' | 'documentTypes' | 'storagePaths' + disabled?: boolean +} + +type TriggerFilterOption = TriggerFilterDefinition & { + disabled?: boolean +} + +type TriggerFilterAggregate = { + filter_has_tags: number[] + filter_has_all_tags: number[] + filter_has_not_tags: number[] + filter_has_not_correspondents: number[] + filter_has_not_document_types: number[] + filter_has_not_storage_paths: number[] + filter_has_correspondent: number | null + filter_has_document_type: number | null + filter_has_storage_path: number | null + filter_custom_field_query: string | null +} + +interface FilterHandler { + apply: (aggregate: TriggerFilterAggregate, values: any) => void + extract: (trigger: WorkflowTrigger) => any + hasValue: (value: any) => boolean +} + +const CUSTOM_FIELD_QUERY_MODEL_KEY = Symbol('customFieldQueryModel') +const CUSTOM_FIELD_QUERY_SUBSCRIPTION_KEY = Symbol( + 'customFieldQuerySubscription' +) + +type CustomFieldFilterGroup = FormGroup & { + [CUSTOM_FIELD_QUERY_MODEL_KEY]?: CustomFieldQueriesModel + [CUSTOM_FIELD_QUERY_SUBSCRIPTION_KEY]?: Subscription +} + +const TRIGGER_FILTER_DEFINITIONS: TriggerFilterDefinition[] = [ + { + id: TriggerFilterType.TagsAny, + name: $localize`Has any of these tags`, + inputType: 'tags', + allowMultipleEntries: false, + allowMultipleValues: true, + }, + { + id: TriggerFilterType.TagsAll, + name: $localize`Has all of these tags`, + inputType: 'tags', + allowMultipleEntries: false, + allowMultipleValues: true, + }, + { + id: TriggerFilterType.TagsNone, + name: $localize`Does not have these tags`, + inputType: 'tags', + allowMultipleEntries: false, + allowMultipleValues: true, + }, + { + id: TriggerFilterType.CorrespondentIs, + name: $localize`Has correspondent`, + inputType: 'select', + allowMultipleEntries: false, + allowMultipleValues: false, + selectItems: 'correspondents', + }, + { + id: TriggerFilterType.CorrespondentNot, + name: $localize`Does not have correspondents`, + inputType: 'select', + allowMultipleEntries: false, + allowMultipleValues: true, + selectItems: 'correspondents', + }, + { + id: TriggerFilterType.DocumentTypeIs, + name: $localize`Has document type`, + inputType: 'select', + allowMultipleEntries: false, + allowMultipleValues: false, + selectItems: 'documentTypes', + }, + { + id: TriggerFilterType.DocumentTypeNot, + name: $localize`Does not have document types`, + inputType: 'select', + allowMultipleEntries: false, + allowMultipleValues: true, + selectItems: 'documentTypes', + }, + { + id: TriggerFilterType.StoragePathIs, + name: $localize`Has storage path`, + inputType: 'select', + allowMultipleEntries: false, + allowMultipleValues: false, + selectItems: 'storagePaths', + }, + { + id: TriggerFilterType.StoragePathNot, + name: $localize`Does not have storage paths`, + inputType: 'select', + allowMultipleEntries: false, + allowMultipleValues: true, + selectItems: 'storagePaths', + }, + { + id: TriggerFilterType.CustomFieldQuery, + name: $localize`Matches custom field query`, + inputType: 'customFieldQuery', + allowMultipleEntries: false, + allowMultipleValues: false, + }, +] + const TRIGGER_MATCHING_ALGORITHMS = MATCHING_ALGORITHMS.filter( (a) => a.id !== MATCH_AUTO ) +const FILTER_HANDLERS: Record = { + [TriggerFilterType.TagsAny]: { + apply: (aggregate, values) => { + aggregate.filter_has_tags = Array.isArray(values) ? [...values] : [values] + }, + extract: (trigger) => trigger.filter_has_tags, + hasValue: (value) => Array.isArray(value) && value.length > 0, + }, + [TriggerFilterType.TagsAll]: { + apply: (aggregate, values) => { + aggregate.filter_has_all_tags = Array.isArray(values) + ? [...values] + : [values] + }, + extract: (trigger) => trigger.filter_has_all_tags, + hasValue: (value) => Array.isArray(value) && value.length > 0, + }, + [TriggerFilterType.TagsNone]: { + apply: (aggregate, values) => { + aggregate.filter_has_not_tags = Array.isArray(values) + ? [...values] + : [values] + }, + extract: (trigger) => trigger.filter_has_not_tags, + hasValue: (value) => Array.isArray(value) && value.length > 0, + }, + [TriggerFilterType.CorrespondentIs]: { + apply: (aggregate, values) => { + aggregate.filter_has_correspondent = Array.isArray(values) + ? (values[0] ?? null) + : values + }, + extract: (trigger) => trigger.filter_has_correspondent, + hasValue: (value) => value !== null && value !== undefined, + }, + [TriggerFilterType.CorrespondentNot]: { + apply: (aggregate, values) => { + aggregate.filter_has_not_correspondents = Array.isArray(values) + ? [...values] + : [values] + }, + extract: (trigger) => trigger.filter_has_not_correspondents, + hasValue: (value) => Array.isArray(value) && value.length > 0, + }, + [TriggerFilterType.DocumentTypeIs]: { + apply: (aggregate, values) => { + aggregate.filter_has_document_type = Array.isArray(values) + ? (values[0] ?? null) + : values + }, + extract: (trigger) => trigger.filter_has_document_type, + hasValue: (value) => value !== null && value !== undefined, + }, + [TriggerFilterType.DocumentTypeNot]: { + apply: (aggregate, values) => { + aggregate.filter_has_not_document_types = Array.isArray(values) + ? [...values] + : [values] + }, + extract: (trigger) => trigger.filter_has_not_document_types, + hasValue: (value) => Array.isArray(value) && value.length > 0, + }, + [TriggerFilterType.StoragePathIs]: { + apply: (aggregate, values) => { + aggregate.filter_has_storage_path = Array.isArray(values) + ? (values[0] ?? null) + : values + }, + extract: (trigger) => trigger.filter_has_storage_path, + hasValue: (value) => value !== null && value !== undefined, + }, + [TriggerFilterType.StoragePathNot]: { + apply: (aggregate, values) => { + aggregate.filter_has_not_storage_paths = Array.isArray(values) + ? [...values] + : [values] + }, + extract: (trigger) => trigger.filter_has_not_storage_paths, + hasValue: (value) => Array.isArray(value) && value.length > 0, + }, + [TriggerFilterType.CustomFieldQuery]: { + apply: (aggregate, values) => { + aggregate.filter_custom_field_query = values as string + }, + extract: (trigger) => trigger.filter_custom_field_query, + hasValue: (value) => + typeof value === 'string' && value !== null && value.trim().length > 0, + }, +} + @Component({ selector: 'pngx-workflow-edit-dialog', templateUrl: './workflow-edit-dialog.component.html', @@ -153,6 +384,7 @@ const TRIGGER_MATCHING_ALGORITHMS = MATCHING_ALGORITHMS.filter( TextAreaComponent, TagsComponent, CustomFieldsValuesComponent, + CustomFieldsQueryDropdownComponent, PermissionsGroupComponent, PermissionsUserComponent, ConfirmButtonComponent, @@ -170,6 +402,8 @@ export class WorkflowEditDialogComponent { public WorkflowTriggerType = WorkflowTriggerType public WorkflowActionType = WorkflowActionType + public TriggerFilterType = TriggerFilterType + public filterDefinitions = TRIGGER_FILTER_DEFINITIONS private correspondentService: CorrespondentService private documentTypeService: DocumentTypeService @@ -189,6 +423,11 @@ export class WorkflowEditDialogComponent private allowedActionTypes = [] + private readonly triggerFilterOptionsMap = new WeakMap< + FormArray, + TriggerFilterOption[] + >() + constructor() { super() this.service = inject(WorkflowService) @@ -390,6 +629,416 @@ export class WorkflowEditDialogComponent return this.objectForm.get('actions') as FormArray } + protected override getFormValues(): any { + const formValues = super.getFormValues() + + if (formValues?.triggers?.length) { + formValues.triggers = formValues.triggers.map( + (trigger: any, index: number) => { + const triggerFormGroup = this.triggerFields.at(index) as FormGroup + const filters = this.getFiltersFormArray(triggerFormGroup) + + const aggregate: TriggerFilterAggregate = { + filter_has_tags: [], + filter_has_all_tags: [], + filter_has_not_tags: [], + filter_has_not_correspondents: [], + filter_has_not_document_types: [], + filter_has_not_storage_paths: [], + filter_has_correspondent: null, + filter_has_document_type: null, + filter_has_storage_path: null, + filter_custom_field_query: null, + } + + for (const control of filters.controls) { + const type = control.get('type').value as TriggerFilterType + const values = control.get('values').value + + if (values === null || values === undefined) { + continue + } + + if (Array.isArray(values) && values.length === 0) { + continue + } + + const handler = FILTER_HANDLERS[type] + handler?.apply(aggregate, values) + } + + trigger.filter_has_tags = aggregate.filter_has_tags + trigger.filter_has_all_tags = aggregate.filter_has_all_tags + trigger.filter_has_not_tags = aggregate.filter_has_not_tags + trigger.filter_has_not_correspondents = + aggregate.filter_has_not_correspondents + trigger.filter_has_not_document_types = + aggregate.filter_has_not_document_types + trigger.filter_has_not_storage_paths = + aggregate.filter_has_not_storage_paths + trigger.filter_has_correspondent = + aggregate.filter_has_correspondent ?? null + trigger.filter_has_document_type = + aggregate.filter_has_document_type ?? null + trigger.filter_has_storage_path = + aggregate.filter_has_storage_path ?? null + trigger.filter_custom_field_query = + aggregate.filter_custom_field_query ?? null + + delete trigger.filters + + return trigger + } + ) + } + + return formValues + } + + public matchingPatternRequired(formGroup: FormGroup): boolean { + return formGroup.get('matching_algorithm').value !== MATCH_NONE + } + + private createFilterFormGroup( + type: TriggerFilterType, + initialValue?: any + ): FormGroup { + const group = new FormGroup({ + type: new FormControl(type), + values: new FormControl(this.normalizeFilterValue(type, initialValue)), + }) + + group.get('type').valueChanges.subscribe((newType: TriggerFilterType) => { + if (newType === TriggerFilterType.CustomFieldQuery) { + this.ensureCustomFieldQueryModel(group) + } else { + this.clearCustomFieldQueryModel(group) + group.get('values').setValue(this.getDefaultFilterValue(newType), { + emitEvent: false, + }) + } + }) + + if (type === TriggerFilterType.CustomFieldQuery) { + this.ensureCustomFieldQueryModel(group, initialValue) + } + + return group + } + + private buildFiltersFormArray(trigger: WorkflowTrigger): FormArray { + const filters = new FormArray([]) + + for (const definition of this.filterDefinitions) { + const handler = FILTER_HANDLERS[definition.id] + if (!handler) { + continue + } + + const value = handler.extract(trigger) + if (!handler.hasValue(value)) { + continue + } + + filters.push(this.createFilterFormGroup(definition.id, value)) + } + + return filters + } + + getFiltersFormArray(formGroup: FormGroup): FormArray { + return formGroup.get('filters') as FormArray + } + + getFilterTypeOptions(formGroup: FormGroup, filterIndex: number) { + const filters = this.getFiltersFormArray(formGroup) + const options = this.getFilterTypeOptionsForArray(filters) + const currentType = filters.at(filterIndex).get('type') + .value as TriggerFilterType + const usedTypes = new Set( + filters.controls.map( + (control) => control.get('type').value as TriggerFilterType + ) + ) + + for (const option of options) { + if (option.allowMultipleEntries) { + option.disabled = false + continue + } + + option.disabled = usedTypes.has(option.id) && option.id !== currentType + } + + return options + } + + canAddFilter(formGroup: FormGroup): boolean { + const filters = this.getFiltersFormArray(formGroup) + const usedTypes = new Set( + filters.controls.map( + (control) => control.get('type').value as TriggerFilterType + ) + ) + + return this.filterDefinitions.some((definition) => { + if (definition.allowMultipleEntries) { + return true + } + return !usedTypes.has(definition.id) + }) + } + + addFilter(triggerFormGroup: FormGroup): FormGroup | null { + const triggerIndex = this.triggerFields.controls.indexOf(triggerFormGroup) + if (triggerIndex === -1) { + return null + } + + const filters = this.getFiltersFormArray(triggerFormGroup) + + const availableDefinition = this.filterDefinitions.find((definition) => { + if (definition.allowMultipleEntries) { + return true + } + return !filters.controls.some( + (control) => control.get('type').value === definition.id + ) + }) + + if (!availableDefinition) { + return null + } + + filters.push(this.createFilterFormGroup(availableDefinition.id)) + triggerFormGroup.markAsDirty() + triggerFormGroup.markAsTouched() + + return filters.at(-1) as FormGroup + } + + removeFilter(triggerFormGroup: FormGroup, filterIndex: number) { + const triggerIndex = this.triggerFields.controls.indexOf(triggerFormGroup) + if (triggerIndex === -1) { + return + } + + const filters = this.getFiltersFormArray(triggerFormGroup) + const filterGroup = filters.at(filterIndex) as FormGroup + if (filterGroup?.get('type').value === TriggerFilterType.CustomFieldQuery) { + this.clearCustomFieldQueryModel(filterGroup) + } + filters.removeAt(filterIndex) + triggerFormGroup.markAsDirty() + triggerFormGroup.markAsTouched() + } + + getFilterDefinition( + type: TriggerFilterType + ): TriggerFilterDefinition | undefined { + return this.filterDefinitions.find((definition) => definition.id === type) + } + + getFilterName(type: TriggerFilterType): string { + return this.getFilterDefinition(type)?.name ?? '' + } + + isTagsFilter(type: TriggerFilterType): boolean { + return this.getFilterDefinition(type)?.inputType === 'tags' + } + + isCustomFieldQueryFilter(type: TriggerFilterType): boolean { + return this.getFilterDefinition(type)?.inputType === 'customFieldQuery' + } + + isMultiValueFilter(type: TriggerFilterType): boolean { + switch (type) { + case TriggerFilterType.TagsAny: + case TriggerFilterType.TagsAll: + case TriggerFilterType.TagsNone: + case TriggerFilterType.CorrespondentNot: + case TriggerFilterType.DocumentTypeNot: + case TriggerFilterType.StoragePathNot: + return true + default: + return false + } + } + + isSelectMultiple(type: TriggerFilterType): boolean { + return !this.isTagsFilter(type) && this.isMultiValueFilter(type) + } + + getFilterSelectItems(type: TriggerFilterType) { + const definition = this.getFilterDefinition(type) + if (!definition || definition.inputType !== 'select') { + return [] + } + + switch (definition.selectItems) { + case 'correspondents': + return this.correspondents + case 'documentTypes': + return this.documentTypes + case 'storagePaths': + return this.storagePaths + default: + return [] + } + } + + getCustomFieldQueryModel(control: AbstractControl): CustomFieldQueriesModel { + return this.ensureCustomFieldQueryModel(control as FormGroup) + } + + onCustomFieldQuerySelectionChange( + control: AbstractControl, + model: CustomFieldQueriesModel + ) { + this.onCustomFieldQueryModelChanged(control as FormGroup, model) + } + + isCustomFieldQueryValid(control: AbstractControl): boolean { + const model = this.getStoredCustomFieldQueryModel(control as FormGroup) + if (!model) { + return true + } + + return model.isEmpty() || model.isValid() + } + + private getFilterTypeOptionsForArray( + filters: FormArray + ): TriggerFilterOption[] { + let cached = this.triggerFilterOptionsMap.get(filters) + if (!cached) { + cached = this.filterDefinitions.map((definition) => ({ + ...definition, + disabled: false, + })) + this.triggerFilterOptionsMap.set(filters, cached) + } + return cached + } + + private ensureCustomFieldQueryModel( + filterGroup: FormGroup, + initialValue?: any + ): CustomFieldQueriesModel { + const existingModel = this.getStoredCustomFieldQueryModel(filterGroup) + if (existingModel) { + return existingModel + } + + const model = new CustomFieldQueriesModel() + this.setCustomFieldQueryModel(filterGroup, model) + + const rawValue = + typeof initialValue === 'string' + ? initialValue + : (filterGroup.get('values').value as string) + + if (rawValue) { + try { + const parsed = JSON.parse(rawValue) + const expression = new CustomFieldQueryExpression(parsed) + model.queries = [expression] + } catch { + model.clear(false) + model.addInitialAtom() + } + } + + const subscription = model.changed + .pipe(takeUntil(this.unsubscribeNotifier)) + .subscribe(() => { + this.onCustomFieldQueryModelChanged(filterGroup, model) + }) + filterGroup[CUSTOM_FIELD_QUERY_SUBSCRIPTION_KEY]?.unsubscribe() + filterGroup[CUSTOM_FIELD_QUERY_SUBSCRIPTION_KEY] = subscription + + this.onCustomFieldQueryModelChanged(filterGroup, model) + + return model + } + + private clearCustomFieldQueryModel(filterGroup: FormGroup) { + const group = filterGroup as CustomFieldFilterGroup + group[CUSTOM_FIELD_QUERY_SUBSCRIPTION_KEY]?.unsubscribe() + delete group[CUSTOM_FIELD_QUERY_SUBSCRIPTION_KEY] + delete group[CUSTOM_FIELD_QUERY_MODEL_KEY] + } + + private getStoredCustomFieldQueryModel( + filterGroup: FormGroup + ): CustomFieldQueriesModel | null { + return ( + (filterGroup as CustomFieldFilterGroup)[CUSTOM_FIELD_QUERY_MODEL_KEY] ?? + null + ) + } + + private setCustomFieldQueryModel( + filterGroup: FormGroup, + model: CustomFieldQueriesModel + ) { + const group = filterGroup as CustomFieldFilterGroup + group[CUSTOM_FIELD_QUERY_MODEL_KEY] = model + } + + private onCustomFieldQueryModelChanged( + filterGroup: FormGroup, + model: CustomFieldQueriesModel + ) { + const control = filterGroup.get('values') + if (!control) { + return + } + + if (!model.isValid()) { + control.setValue(null, { emitEvent: false }) + return + } + + if (model.isEmpty()) { + control.setValue(null, { emitEvent: false }) + return + } + + const serialized = JSON.stringify(model.queries[0].serialize()) + control.setValue(serialized, { emitEvent: false }) + } + + private getDefaultFilterValue(type: TriggerFilterType) { + if (type === TriggerFilterType.CustomFieldQuery) { + return null + } + return this.isMultiValueFilter(type) ? [] : null + } + + private normalizeFilterValue(type: TriggerFilterType, value?: any) { + if (value === undefined || value === null) { + return this.getDefaultFilterValue(type) + } + + if (type === TriggerFilterType.CustomFieldQuery) { + if (typeof value === 'string') { + return value + } + return value ? JSON.stringify(value) : null + } + + if (this.isMultiValueFilter(type)) { + return Array.isArray(value) ? [...value] : [value] + } + + if (Array.isArray(value)) { + return value.length > 0 ? value[0] : null + } + + return value + } + private createTriggerField( trigger: WorkflowTrigger, emitEvent: boolean = false @@ -405,16 +1054,7 @@ export class WorkflowEditDialogComponent matching_algorithm: new FormControl(trigger.matching_algorithm), match: new FormControl(trigger.match), is_insensitive: new FormControl(trigger.is_insensitive), - filter_has_tags: new FormControl(trigger.filter_has_tags), - filter_has_correspondent: new FormControl( - trigger.filter_has_correspondent - ), - filter_has_document_type: new FormControl( - trigger.filter_has_document_type - ), - filter_has_storage_path: new FormControl( - trigger.filter_has_storage_path - ), + filters: this.buildFiltersFormArray(trigger), schedule_offset_days: new FormControl(trigger.schedule_offset_days), schedule_is_recurring: new FormControl(trigger.schedule_is_recurring), schedule_recurring_interval_days: new FormControl( @@ -537,6 +1177,12 @@ export class WorkflowEditDialogComponent filter_path: null, filter_mailrule: null, filter_has_tags: [], + filter_has_all_tags: [], + filter_has_not_tags: [], + filter_has_not_correspondents: [], + filter_has_not_document_types: [], + filter_has_not_storage_paths: [], + filter_custom_field_query: null, filter_has_correspondent: null, filter_has_document_type: null, filter_has_storage_path: null, diff --git a/src-ui/src/app/components/common/input/select/select.component.html b/src-ui/src/app/components/common/input/select/select.component.html index ef7be3b62..eb351cbe6 100644 --- a/src-ui/src/app/components/common/input/select/select.component.html +++ b/src-ui/src/app/components/common/input/select/select.component.html @@ -1,66 +1,68 @@
-
- @if (title) { - - } - @if (removable) { - + } +
+ } +
+
+ + + {{item[bindLabel]}} + + + @if (allowCreateNew && !hideAddButton) { + + } + @if (showFilter) { + }
-
-
- - - {{item[bindLabel]}} - - - @if (allowCreateNew && !hideAddButton) { - - } - @if (showFilter) { - - } -
-
- {{error}} -
- @if (hint) { - {{hint}} - } - @if (getSuggestions().length > 0) { - - Suggestions:  - @for (s of getSuggestions(); track s) { - {{s.name}}  - } - - } +
+ {{error}}
+ @if (hint) { + {{hint}} + } + @if (getSuggestions().length > 0) { + + Suggestions:  + @for (s of getSuggestions(); track s) { + {{s.name}}  + } + + }
+
diff --git a/src-ui/src/app/components/common/input/tags/tags.component.html b/src-ui/src/app/components/common/input/tags/tags.component.html index 6dcd74b4b..f04863f40 100644 --- a/src-ui/src/app/components/common/input/tags/tags.component.html +++ b/src-ui/src/app/components/common/input/tags/tags.component.html @@ -1,8 +1,10 @@
-
- -
+ @if (title) { +
+ +
+ }
tuple[bool, str]: +) -> tuple[bool, str | None]: """ Returns True if the Document matches all filters from the workflow trigger, False otherwise. Includes a reason if doesn't match """ - trigger_matched = True - reason = "" - + # Check content matching algorithm if trigger.matching_algorithm > MatchingModel.MATCH_NONE and not matches( trigger, document, ): - reason = ( + return ( + False, f"Document content matching settings for algorithm '{trigger.matching_algorithm}' did not match", ) - trigger_matched = False - # Document tags vs trigger has_tags - if ( - trigger.filter_has_tags.all().count() > 0 - and document.tags.filter( - id__in=trigger.filter_has_tags.all().values_list("id"), - ).count() - == 0 - ): - reason = ( - f"Document tags {document.tags.all()} do not include" - f" {trigger.filter_has_tags.all()}", - ) - trigger_matched = False + # Check if any tag filters exist to determine if we need to load document tags + trigger_has_tags_qs = trigger.filter_has_tags.all() + trigger_has_all_tags_qs = trigger.filter_has_all_tags.all() + trigger_has_not_tags_qs = trigger.filter_has_not_tags.all() + + has_tags_filter = trigger_has_tags_qs.exists() + has_all_tags_filter = trigger_has_all_tags_qs.exists() + has_not_tags_filter = trigger_has_not_tags_qs.exists() + + # Load document tags once if any tag filters exist + document_tag_ids = None + if has_tags_filter or has_all_tags_filter or has_not_tags_filter: + document_tag_ids = set(document.tags.values_list("id", flat=True)) + + # Document tags vs trigger has_tags (any of) + if has_tags_filter: + trigger_has_tag_ids = set(trigger_has_tags_qs.values_list("id", flat=True)) + if not (document_tag_ids & trigger_has_tag_ids): + # For error message, load the actual tag objects + return ( + False, + f"Document tags {list(document.tags.all())} do not include {list(trigger_has_tags_qs)}", + ) + + # Document tags vs trigger has_all_tags (all of) + if has_all_tags_filter: + required_tag_ids = set(trigger_has_all_tags_qs.values_list("id", flat=True)) + if not required_tag_ids.issubset(document_tag_ids): + return ( + False, + f"Document tags {list(document.tags.all())} do not contain all of {list(trigger_has_all_tags_qs)}", + ) + + # Document tags vs trigger has_not_tags (none of) + if has_not_tags_filter: + excluded_tag_ids = set(trigger_has_not_tags_qs.values_list("id", flat=True)) + if document_tag_ids & excluded_tag_ids: + return ( + False, + f"Document tags {list(document.tags.all())} include excluded tags {list(trigger_has_not_tags_qs)}", + ) # Document correspondent vs trigger has_correspondent if ( - trigger.filter_has_correspondent is not None - and document.correspondent != trigger.filter_has_correspondent + trigger.filter_has_correspondent_id is not None + and document.correspondent_id != trigger.filter_has_correspondent_id ): - reason = ( + return ( + False, f"Document correspondent {document.correspondent} does not match {trigger.filter_has_correspondent}", ) - trigger_matched = False + + if ( + document.correspondent_id + and trigger.filter_has_not_correspondents.filter( + id=document.correspondent_id, + ).exists() + ): + return ( + False, + f"Document correspondent {document.correspondent} is excluded by {list(trigger.filter_has_not_correspondents.all())}", + ) # Document document_type vs trigger has_document_type if ( - trigger.filter_has_document_type is not None - and document.document_type != trigger.filter_has_document_type + trigger.filter_has_document_type_id is not None + and document.document_type_id != trigger.filter_has_document_type_id ): - reason = ( + return ( + False, f"Document doc type {document.document_type} does not match {trigger.filter_has_document_type}", ) - trigger_matched = False + + if ( + document.document_type_id + and trigger.filter_has_not_document_types.filter( + id=document.document_type_id, + ).exists() + ): + return ( + False, + f"Document doc type {document.document_type} is excluded by {list(trigger.filter_has_not_document_types.all())}", + ) # Document storage_path vs trigger has_storage_path if ( - trigger.filter_has_storage_path is not None - and document.storage_path != trigger.filter_has_storage_path + trigger.filter_has_storage_path_id is not None + and document.storage_path_id != trigger.filter_has_storage_path_id ): - reason = ( + return ( + False, f"Document storage path {document.storage_path} does not match {trigger.filter_has_storage_path}", ) - trigger_matched = False + + if ( + document.storage_path_id + and trigger.filter_has_not_storage_paths.filter( + id=document.storage_path_id, + ).exists() + ): + return ( + False, + f"Document storage path {document.storage_path} is excluded by {list(trigger.filter_has_not_storage_paths.all())}", + ) + + # Custom field query check + if trigger.filter_custom_field_query: + parser = CustomFieldQueryParser("filter_custom_field_query") + try: + custom_field_q, annotations = parser.parse( + trigger.filter_custom_field_query, + ) + except serializers.ValidationError: + return (False, "Invalid custom field query configuration") + + qs = ( + Document.objects.filter(id=document.id) + .annotate(**annotations) + .filter(custom_field_q) + ) + if not qs.exists(): + return ( + False, + "Document custom fields do not match the configured custom field query", + ) # Document original_filename vs trigger filename if ( @@ -414,13 +497,12 @@ def existing_document_matches_workflow( trigger.filter_filename.lower(), ) ): - reason = ( - f"Document filename {document.original_filename} does not match" - f" {trigger.filter_filename.lower()}", + return ( + False, + f"Document filename {document.original_filename} does not match {trigger.filter_filename.lower()}", ) - trigger_matched = False - return (trigger_matched, reason) + return (True, None) def prefilter_documents_by_workflowtrigger( @@ -433,31 +515,66 @@ def prefilter_documents_by_workflowtrigger( document_matches_workflow in run_workflows """ - if trigger.filter_has_tags.all().count() > 0: - documents = documents.filter( - tags__in=trigger.filter_has_tags.all(), - ).distinct() + # Filter for documents that have AT LEAST ONE of the specified tags. + if trigger.filter_has_tags.exists(): + documents = documents.filter(tags__in=trigger.filter_has_tags.all()).distinct() + + # Filter for documents that have ALL of the specified tags. + if trigger.filter_has_all_tags.exists(): + for tag in trigger.filter_has_all_tags.all(): + documents = documents.filter(tags=tag) + # Multiple JOINs can create duplicate results. + documents = documents.distinct() + + # Exclude documents that have ANY of the specified tags. + if trigger.filter_has_not_tags.exists(): + documents = documents.exclude(tags__in=trigger.filter_has_not_tags.all()) + + # Correspondent, DocumentType, etc. filtering if trigger.filter_has_correspondent is not None: documents = documents.filter( correspondent=trigger.filter_has_correspondent, ) + if trigger.filter_has_not_correspondents.exists(): + documents = documents.exclude( + correspondent__in=trigger.filter_has_not_correspondents.all(), + ) if trigger.filter_has_document_type is not None: documents = documents.filter( document_type=trigger.filter_has_document_type, ) + if trigger.filter_has_not_document_types.exists(): + documents = documents.exclude( + document_type__in=trigger.filter_has_not_document_types.all(), + ) if trigger.filter_has_storage_path is not None: documents = documents.filter( storage_path=trigger.filter_has_storage_path, ) + if trigger.filter_has_not_storage_paths.exists(): + documents = documents.exclude( + storage_path__in=trigger.filter_has_not_storage_paths.all(), + ) - if trigger.filter_filename is not None and len(trigger.filter_filename) > 0: - # the true fnmatch will actually run later so we just want a loose filter here + # Custom Field & Filename Filtering + + if trigger.filter_custom_field_query: + parser = CustomFieldQueryParser("filter_custom_field_query") + try: + custom_field_q, annotations = parser.parse( + trigger.filter_custom_field_query, + ) + except serializers.ValidationError: + return documents.none() + + documents = documents.annotate(**annotations).filter(custom_field_q) + + if trigger.filter_filename: regex = fnmatch_translate(trigger.filter_filename).lstrip("^").rstrip("$") - regex = f"(?i){regex}" - documents = documents.filter(original_filename__regex=regex) + documents = documents.filter(original_filename__iregex=regex) return documents @@ -472,13 +589,34 @@ def document_matches_workflow( settings from the workflow trigger, False otherwise """ + triggers_queryset = ( + workflow.triggers.filter( + type=trigger_type, + ) + .select_related( + "filter_mailrule", + "filter_has_document_type", + "filter_has_correspondent", + "filter_has_storage_path", + "schedule_date_custom_field", + ) + .prefetch_related( + "filter_has_tags", + "filter_has_all_tags", + "filter_has_not_tags", + "filter_has_not_document_types", + "filter_has_not_correspondents", + "filter_has_not_storage_paths", + ) + ) + trigger_matched = True - if workflow.triggers.filter(type=trigger_type).count() == 0: + if not triggers_queryset.exists(): trigger_matched = False logger.info(f"Document did not match {workflow}") logger.debug(f"No matching triggers with type {trigger_type} found") else: - for trigger in workflow.triggers.filter(type=trigger_type): + for trigger in triggers_queryset: if trigger_type == WorkflowTrigger.WorkflowTriggerType.CONSUMPTION: trigger_matched, reason = consumable_document_matches_workflow( document, diff --git a/src/documents/migrations/1072_workflowtrigger_filter_custom_field_query_and_more.py b/src/documents/migrations/1072_workflowtrigger_filter_custom_field_query_and_more.py new file mode 100644 index 000000000..1a22f6b4f --- /dev/null +++ b/src/documents/migrations/1072_workflowtrigger_filter_custom_field_query_and_more.py @@ -0,0 +1,73 @@ +# Generated by Django 5.2.6 on 2025-10-07 18:52 + +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + dependencies = [ + ("documents", "1071_tag_tn_ancestors_count_tag_tn_ancestors_pks_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="workflowtrigger", + name="filter_custom_field_query", + field=models.TextField( + blank=True, + help_text="JSON-encoded custom field query expression.", + null=True, + verbose_name="filter custom field query", + ), + ), + migrations.AddField( + model_name="workflowtrigger", + name="filter_has_all_tags", + field=models.ManyToManyField( + blank=True, + related_name="workflowtriggers_has_all", + to="documents.tag", + verbose_name="has all of these tag(s)", + ), + ), + migrations.AddField( + model_name="workflowtrigger", + name="filter_has_not_correspondents", + field=models.ManyToManyField( + blank=True, + related_name="workflowtriggers_has_not_correspondent", + to="documents.correspondent", + verbose_name="does not have these correspondent(s)", + ), + ), + migrations.AddField( + model_name="workflowtrigger", + name="filter_has_not_document_types", + field=models.ManyToManyField( + blank=True, + related_name="workflowtriggers_has_not_document_type", + to="documents.documenttype", + verbose_name="does not have these document type(s)", + ), + ), + migrations.AddField( + model_name="workflowtrigger", + name="filter_has_not_storage_paths", + field=models.ManyToManyField( + blank=True, + related_name="workflowtriggers_has_not_storage_path", + to="documents.storagepath", + verbose_name="does not have these storage path(s)", + ), + ), + migrations.AddField( + model_name="workflowtrigger", + name="filter_has_not_tags", + field=models.ManyToManyField( + blank=True, + related_name="workflowtriggers_has_not", + to="documents.tag", + verbose_name="does not have these tag(s)", + ), + ), + ] diff --git a/src/documents/models.py b/src/documents/models.py index 8d542cd8c..ea8662023 100644 --- a/src/documents/models.py +++ b/src/documents/models.py @@ -1065,6 +1065,20 @@ class WorkflowTrigger(models.Model): verbose_name=_("has these tag(s)"), ) + filter_has_all_tags = models.ManyToManyField( + Tag, + blank=True, + related_name="workflowtriggers_has_all", + verbose_name=_("has all of these tag(s)"), + ) + + filter_has_not_tags = models.ManyToManyField( + Tag, + blank=True, + related_name="workflowtriggers_has_not", + verbose_name=_("does not have these tag(s)"), + ) + filter_has_document_type = models.ForeignKey( DocumentType, null=True, @@ -1073,6 +1087,13 @@ class WorkflowTrigger(models.Model): verbose_name=_("has this document type"), ) + filter_has_not_document_types = models.ManyToManyField( + DocumentType, + blank=True, + related_name="workflowtriggers_has_not_document_type", + verbose_name=_("does not have these document type(s)"), + ) + filter_has_correspondent = models.ForeignKey( Correspondent, null=True, @@ -1081,6 +1102,13 @@ class WorkflowTrigger(models.Model): verbose_name=_("has this correspondent"), ) + filter_has_not_correspondents = models.ManyToManyField( + Correspondent, + blank=True, + related_name="workflowtriggers_has_not_correspondent", + verbose_name=_("does not have these correspondent(s)"), + ) + filter_has_storage_path = models.ForeignKey( StoragePath, null=True, @@ -1089,6 +1117,20 @@ class WorkflowTrigger(models.Model): verbose_name=_("has this storage path"), ) + filter_has_not_storage_paths = models.ManyToManyField( + StoragePath, + blank=True, + related_name="workflowtriggers_has_not_storage_path", + verbose_name=_("does not have these storage path(s)"), + ) + + filter_custom_field_query = models.TextField( + _("filter custom field query"), + null=True, + blank=True, + help_text=_("JSON-encoded custom field query expression."), + ) + schedule_offset_days = models.IntegerField( _("schedule offset days"), default=0, diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py index 0633684bb..da9bef1ea 100644 --- a/src/documents/serialisers.py +++ b/src/documents/serialisers.py @@ -44,6 +44,7 @@ if settings.AUDIT_LOG_ENABLED: from documents import bulk_edit from documents.data_models import DocumentSource +from documents.filters import CustomFieldQueryParser from documents.models import Correspondent from documents.models import CustomField from documents.models import CustomFieldInstance @@ -2240,6 +2241,12 @@ class WorkflowTriggerSerializer(serializers.ModelSerializer): "match", "is_insensitive", "filter_has_tags", + "filter_has_all_tags", + "filter_has_not_tags", + "filter_custom_field_query", + "filter_has_not_correspondents", + "filter_has_not_document_types", + "filter_has_not_storage_paths", "filter_has_correspondent", "filter_has_document_type", "filter_has_storage_path", @@ -2265,6 +2272,20 @@ class WorkflowTriggerSerializer(serializers.ModelSerializer): ): attrs["filter_path"] = None + if ( + "filter_custom_field_query" in attrs + and attrs["filter_custom_field_query"] is not None + and len(attrs["filter_custom_field_query"]) == 0 + ): + attrs["filter_custom_field_query"] = None + + if ( + "filter_custom_field_query" in attrs + and attrs["filter_custom_field_query"] is not None + ): + parser = CustomFieldQueryParser("filter_custom_field_query") + parser.parse(attrs["filter_custom_field_query"]) + trigger_type = attrs.get("type", getattr(self.instance, "type", None)) if ( trigger_type == WorkflowTrigger.WorkflowTriggerType.CONSUMPTION @@ -2460,6 +2481,20 @@ class WorkflowSerializer(serializers.ModelSerializer): if triggers is not None and triggers is not serializers.empty: for trigger in triggers: filter_has_tags = trigger.pop("filter_has_tags", None) + filter_has_all_tags = trigger.pop("filter_has_all_tags", None) + filter_has_not_tags = trigger.pop("filter_has_not_tags", None) + filter_has_not_correspondents = trigger.pop( + "filter_has_not_correspondents", + None, + ) + filter_has_not_document_types = trigger.pop( + "filter_has_not_document_types", + None, + ) + filter_has_not_storage_paths = trigger.pop( + "filter_has_not_storage_paths", + None, + ) # Convert sources to strings to handle django-multiselectfield v1.0 changes WorkflowTriggerSerializer.normalize_workflow_trigger_sources(trigger) trigger_instance, _ = WorkflowTrigger.objects.update_or_create( @@ -2468,6 +2503,22 @@ class WorkflowSerializer(serializers.ModelSerializer): ) if filter_has_tags is not None: trigger_instance.filter_has_tags.set(filter_has_tags) + if filter_has_all_tags is not None: + trigger_instance.filter_has_all_tags.set(filter_has_all_tags) + if filter_has_not_tags is not None: + trigger_instance.filter_has_not_tags.set(filter_has_not_tags) + if filter_has_not_correspondents is not None: + trigger_instance.filter_has_not_correspondents.set( + filter_has_not_correspondents, + ) + if filter_has_not_document_types is not None: + trigger_instance.filter_has_not_document_types.set( + filter_has_not_document_types, + ) + if filter_has_not_storage_paths is not None: + trigger_instance.filter_has_not_storage_paths.set( + filter_has_not_storage_paths, + ) set_triggers.append(trigger_instance) if actions is not None and actions is not serializers.empty: diff --git a/src/documents/tests/test_api_workflows.py b/src/documents/tests/test_api_workflows.py index 305467048..9efdb8451 100644 --- a/src/documents/tests/test_api_workflows.py +++ b/src/documents/tests/test_api_workflows.py @@ -184,6 +184,17 @@ class TestApiWorkflows(DirectoriesMixin, APITestCase): "filter_filename": "*", "filter_path": "*/samples/*", "filter_has_tags": [self.t1.id], + "filter_has_all_tags": [self.t2.id], + "filter_has_not_tags": [self.t3.id], + "filter_has_not_correspondents": [self.c2.id], + "filter_has_not_document_types": [self.dt2.id], + "filter_has_not_storage_paths": [self.sp2.id], + "filter_custom_field_query": json.dumps( + [ + "AND", + [[self.cf1.id, "exact", "value"]], + ], + ), "filter_has_document_type": self.dt.id, "filter_has_correspondent": self.c.id, "filter_has_storage_path": self.sp.id, @@ -223,6 +234,36 @@ class TestApiWorkflows(DirectoriesMixin, APITestCase): ) self.assertEqual(response.status_code, status.HTTP_201_CREATED) self.assertEqual(Workflow.objects.count(), 2) + workflow = Workflow.objects.get(name="Workflow 2") + trigger = workflow.triggers.first() + self.assertSetEqual( + set(trigger.filter_has_tags.values_list("id", flat=True)), + {self.t1.id}, + ) + self.assertSetEqual( + set(trigger.filter_has_all_tags.values_list("id", flat=True)), + {self.t2.id}, + ) + self.assertSetEqual( + set(trigger.filter_has_not_tags.values_list("id", flat=True)), + {self.t3.id}, + ) + self.assertSetEqual( + set(trigger.filter_has_not_correspondents.values_list("id", flat=True)), + {self.c2.id}, + ) + self.assertSetEqual( + set(trigger.filter_has_not_document_types.values_list("id", flat=True)), + {self.dt2.id}, + ) + self.assertSetEqual( + set(trigger.filter_has_not_storage_paths.values_list("id", flat=True)), + {self.sp2.id}, + ) + self.assertEqual( + trigger.filter_custom_field_query, + json.dumps(["AND", [[self.cf1.id, "exact", "value"]]]), + ) def test_api_create_invalid_workflow_trigger(self): """ @@ -376,6 +417,14 @@ class TestApiWorkflows(DirectoriesMixin, APITestCase): { "type": WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED, "filter_has_tags": [self.t1.id], + "filter_has_all_tags": [self.t2.id], + "filter_has_not_tags": [self.t3.id], + "filter_has_not_correspondents": [self.c2.id], + "filter_has_not_document_types": [self.dt2.id], + "filter_has_not_storage_paths": [self.sp2.id], + "filter_custom_field_query": json.dumps( + ["AND", [[self.cf1.id, "exact", "value"]]], + ), "filter_has_correspondent": self.c.id, "filter_has_document_type": self.dt.id, }, @@ -393,6 +442,30 @@ class TestApiWorkflows(DirectoriesMixin, APITestCase): workflow = Workflow.objects.get(id=response.data["id"]) self.assertEqual(workflow.name, "Workflow Updated") self.assertEqual(workflow.triggers.first().filter_has_tags.first(), self.t1) + self.assertEqual( + workflow.triggers.first().filter_has_all_tags.first(), + self.t2, + ) + self.assertEqual( + workflow.triggers.first().filter_has_not_tags.first(), + self.t3, + ) + self.assertEqual( + workflow.triggers.first().filter_has_not_correspondents.first(), + self.c2, + ) + self.assertEqual( + workflow.triggers.first().filter_has_not_document_types.first(), + self.dt2, + ) + self.assertEqual( + workflow.triggers.first().filter_has_not_storage_paths.first(), + self.sp2, + ) + self.assertEqual( + workflow.triggers.first().filter_custom_field_query, + json.dumps(["AND", [[self.cf1.id, "exact", "value"]]]), + ) self.assertEqual(workflow.actions.first().assign_title, "Action New Title") def test_api_update_workflow_no_trigger_actions(self): diff --git a/src/documents/tests/test_workflows.py b/src/documents/tests/test_workflows.py index 7652d22b5..a6da01578 100644 --- a/src/documents/tests/test_workflows.py +++ b/src/documents/tests/test_workflows.py @@ -1,4 +1,5 @@ import datetime +import json import shutil import socket from datetime import timedelta @@ -31,6 +32,7 @@ from documents import tasks from documents.data_models import ConsumableDocument from documents.data_models import DocumentSource from documents.matching import document_matches_workflow +from documents.matching import existing_document_matches_workflow from documents.matching import prefilter_documents_by_workflowtrigger from documents.models import Correspondent from documents.models import CustomField @@ -46,6 +48,7 @@ from documents.models import WorkflowActionEmail from documents.models import WorkflowActionWebhook from documents.models import WorkflowRun from documents.models import WorkflowTrigger +from documents.serialisers import WorkflowTriggerSerializer from documents.signals import document_consumption_finished from documents.tests.utils import DirectoriesMixin from documents.tests.utils import DummyProgressManager @@ -1080,9 +1083,409 @@ class TestWorkflows( ) expected_str = f"Document did not match {w}" self.assertIn(expected_str, cm.output[0]) - expected_str = f"Document tags {doc.tags.all()} do not include {trigger.filter_has_tags.all()}" + expected_str = f"Document tags {list(doc.tags.all())} do not include {list(trigger.filter_has_tags.all())}" self.assertIn(expected_str, cm.output[1]) + def test_document_added_no_match_all_tags(self): + trigger = WorkflowTrigger.objects.create( + type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED, + ) + trigger.filter_has_all_tags.set([self.t1, self.t2]) + action = WorkflowAction.objects.create( + assign_title="Doc assign owner", + assign_owner=self.user2, + ) + w = Workflow.objects.create( + name="Workflow 1", + order=0, + ) + w.triggers.add(trigger) + w.actions.add(action) + w.save() + + doc = Document.objects.create( + title="sample test", + correspondent=self.c, + original_filename="sample.pdf", + ) + doc.tags.set([self.t1]) + doc.save() + + with self.assertLogs("paperless.matching", level="DEBUG") as cm: + document_consumption_finished.send( + sender=self.__class__, + document=doc, + ) + expected_str = f"Document did not match {w}" + self.assertIn(expected_str, cm.output[0]) + expected_str = ( + f"Document tags {list(doc.tags.all())} do not contain all of" + f" {list(trigger.filter_has_all_tags.all())}" + ) + self.assertIn(expected_str, cm.output[1]) + + def test_document_added_excluded_tags(self): + trigger = WorkflowTrigger.objects.create( + type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED, + ) + trigger.filter_has_not_tags.set([self.t3]) + action = WorkflowAction.objects.create( + assign_title="Doc assign owner", + assign_owner=self.user2, + ) + w = Workflow.objects.create( + name="Workflow 1", + order=0, + ) + w.triggers.add(trigger) + w.actions.add(action) + w.save() + + doc = Document.objects.create( + title="sample test", + correspondent=self.c, + original_filename="sample.pdf", + ) + doc.tags.set([self.t3]) + doc.save() + + with self.assertLogs("paperless.matching", level="DEBUG") as cm: + document_consumption_finished.send( + sender=self.__class__, + document=doc, + ) + expected_str = f"Document did not match {w}" + self.assertIn(expected_str, cm.output[0]) + expected_str = ( + f"Document tags {list(doc.tags.all())} include excluded tags" + f" {list(trigger.filter_has_not_tags.all())}" + ) + self.assertIn(expected_str, cm.output[1]) + + def test_document_added_excluded_correspondent(self): + trigger = WorkflowTrigger.objects.create( + type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED, + ) + trigger.filter_has_not_correspondents.set([self.c]) + action = WorkflowAction.objects.create( + assign_title="Doc assign owner", + assign_owner=self.user2, + ) + w = Workflow.objects.create( + name="Workflow 1", + order=0, + ) + w.triggers.add(trigger) + w.actions.add(action) + w.save() + + doc = Document.objects.create( + title="sample test", + correspondent=self.c, + original_filename="sample.pdf", + ) + + with self.assertLogs("paperless.matching", level="DEBUG") as cm: + document_consumption_finished.send( + sender=self.__class__, + document=doc, + ) + expected_str = f"Document did not match {w}" + self.assertIn(expected_str, cm.output[0]) + expected_str = ( + f"Document correspondent {doc.correspondent} is excluded by" + f" {list(trigger.filter_has_not_correspondents.all())}" + ) + self.assertIn(expected_str, cm.output[1]) + + def test_document_added_excluded_document_types(self): + trigger = WorkflowTrigger.objects.create( + type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED, + ) + trigger.filter_has_not_document_types.set([self.dt]) + action = WorkflowAction.objects.create( + assign_title="Doc assign owner", + assign_owner=self.user2, + ) + w = Workflow.objects.create( + name="Workflow 1", + order=0, + ) + w.triggers.add(trigger) + w.actions.add(action) + w.save() + + doc = Document.objects.create( + title="sample test", + document_type=self.dt, + original_filename="sample.pdf", + ) + + with self.assertLogs("paperless.matching", level="DEBUG") as cm: + document_consumption_finished.send( + sender=self.__class__, + document=doc, + ) + expected_str = f"Document did not match {w}" + self.assertIn(expected_str, cm.output[0]) + expected_str = ( + f"Document doc type {doc.document_type} is excluded by" + f" {list(trigger.filter_has_not_document_types.all())}" + ) + self.assertIn(expected_str, cm.output[1]) + + def test_document_added_excluded_storage_paths(self): + trigger = WorkflowTrigger.objects.create( + type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED, + ) + trigger.filter_has_not_storage_paths.set([self.sp]) + action = WorkflowAction.objects.create( + assign_title="Doc assign owner", + assign_owner=self.user2, + ) + w = Workflow.objects.create( + name="Workflow 1", + order=0, + ) + w.triggers.add(trigger) + w.actions.add(action) + w.save() + + doc = Document.objects.create( + title="sample test", + storage_path=self.sp, + original_filename="sample.pdf", + ) + + with self.assertLogs("paperless.matching", level="DEBUG") as cm: + document_consumption_finished.send( + sender=self.__class__, + document=doc, + ) + expected_str = f"Document did not match {w}" + self.assertIn(expected_str, cm.output[0]) + expected_str = ( + f"Document storage path {doc.storage_path} is excluded by" + f" {list(trigger.filter_has_not_storage_paths.all())}" + ) + self.assertIn(expected_str, cm.output[1]) + + def test_document_added_custom_field_query_no_match(self): + trigger = WorkflowTrigger.objects.create( + type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED, + filter_custom_field_query=json.dumps( + [ + "AND", + [[self.cf1.id, "exact", "expected"]], + ], + ), + ) + action = WorkflowAction.objects.create( + assign_title="Doc assign owner", + assign_owner=self.user2, + ) + workflow = Workflow.objects.create(name="Workflow 1", order=0) + workflow.triggers.add(trigger) + workflow.actions.add(action) + workflow.save() + + doc = Document.objects.create( + title="sample test", + correspondent=self.c, + original_filename="sample.pdf", + ) + CustomFieldInstance.objects.create( + document=doc, + field=self.cf1, + value_text="other", + ) + + with self.assertLogs("paperless.matching", level="DEBUG") as cm: + document_consumption_finished.send( + sender=self.__class__, + document=doc, + ) + expected_str = f"Document did not match {workflow}" + self.assertIn(expected_str, cm.output[0]) + self.assertIn( + "Document custom fields do not match the configured custom field query", + cm.output[1], + ) + + def test_document_added_custom_field_query_match(self): + trigger = WorkflowTrigger.objects.create( + type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED, + filter_custom_field_query=json.dumps( + [ + "AND", + [[self.cf1.id, "exact", "expected"]], + ], + ), + ) + doc = Document.objects.create( + title="sample test", + correspondent=self.c, + original_filename="sample.pdf", + ) + CustomFieldInstance.objects.create( + document=doc, + field=self.cf1, + value_text="expected", + ) + + matched, reason = existing_document_matches_workflow(doc, trigger) + self.assertTrue(matched) + self.assertIsNone(reason) + + def test_prefilter_documents_custom_field_query(self): + trigger = WorkflowTrigger.objects.create( + type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED, + filter_custom_field_query=json.dumps( + [ + "AND", + [[self.cf1.id, "exact", "match"]], + ], + ), + ) + doc1 = Document.objects.create( + title="doc 1", + correspondent=self.c, + original_filename="doc1.pdf", + checksum="checksum1", + ) + CustomFieldInstance.objects.create( + document=doc1, + field=self.cf1, + value_text="match", + ) + + doc2 = Document.objects.create( + title="doc 2", + correspondent=self.c, + original_filename="doc2.pdf", + checksum="checksum2", + ) + CustomFieldInstance.objects.create( + document=doc2, + field=self.cf1, + value_text="different", + ) + + filtered = prefilter_documents_by_workflowtrigger( + Document.objects.all(), + trigger, + ) + self.assertIn(doc1, filtered) + self.assertNotIn(doc2, filtered) + + def test_consumption_trigger_requires_filter_configuration(self): + serializer = WorkflowTriggerSerializer( + data={ + "type": WorkflowTrigger.WorkflowTriggerType.CONSUMPTION, + }, + ) + + self.assertFalse(serializer.is_valid()) + errors = serializer.errors.get("non_field_errors", []) + self.assertIn( + "File name, path or mail rule filter are required", + [str(error) for error in errors], + ) + + def test_workflow_trigger_serializer_clears_empty_custom_field_query(self): + serializer = WorkflowTriggerSerializer( + data={ + "type": WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED, + "filter_custom_field_query": "", + }, + ) + + self.assertTrue(serializer.is_valid(), serializer.errors) + self.assertIsNone(serializer.validated_data.get("filter_custom_field_query")) + + def test_existing_document_invalid_custom_field_query_configuration(self): + trigger = WorkflowTrigger.objects.create( + type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED, + filter_custom_field_query="{ not json", + ) + + document = Document.objects.create( + title="doc invalid query", + original_filename="invalid.pdf", + checksum="checksum-invalid-query", + ) + + matched, reason = existing_document_matches_workflow(document, trigger) + self.assertFalse(matched) + self.assertEqual(reason, "Invalid custom field query configuration") + + def test_prefilter_documents_returns_none_for_invalid_custom_field_query(self): + trigger = WorkflowTrigger.objects.create( + type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED, + filter_custom_field_query="{ not json", + ) + + Document.objects.create( + title="doc", + original_filename="doc.pdf", + checksum="checksum-prefilter-invalid", + ) + + filtered = prefilter_documents_by_workflowtrigger( + Document.objects.all(), + trigger, + ) + + self.assertEqual(list(filtered), []) + + def test_prefilter_documents_applies_all_filters(self): + other_document_type = DocumentType.objects.create(name="Other Type") + other_storage_path = StoragePath.objects.create( + name="Blocked path", + path="/blocked/", + ) + + trigger = WorkflowTrigger.objects.create( + type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED, + filter_has_correspondent=self.c, + filter_has_document_type=self.dt, + filter_has_storage_path=self.sp, + ) + trigger.filter_has_tags.set([self.t1]) + trigger.filter_has_all_tags.set([self.t1, self.t2]) + trigger.filter_has_not_tags.set([self.t3]) + trigger.filter_has_not_correspondents.set([self.c2]) + trigger.filter_has_not_document_types.set([other_document_type]) + trigger.filter_has_not_storage_paths.set([other_storage_path]) + + allowed_document = Document.objects.create( + title="allowed", + correspondent=self.c, + document_type=self.dt, + storage_path=self.sp, + original_filename="allow.pdf", + checksum="checksum-prefilter-allowed", + ) + allowed_document.tags.set([self.t1, self.t2]) + + blocked_document = Document.objects.create( + title="blocked", + correspondent=self.c2, + document_type=other_document_type, + storage_path=other_storage_path, + original_filename="block.pdf", + checksum="checksum-prefilter-blocked", + ) + blocked_document.tags.set([self.t1, self.t3]) + + filtered = prefilter_documents_by_workflowtrigger( + Document.objects.all(), + trigger, + ) + + self.assertIn(allowed_document, filtered) + self.assertNotIn(blocked_document, filtered) + def test_document_added_no_match_doctype(self): trigger = WorkflowTrigger.objects.create( type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED,