Compare commits

..

2 Commits

Author SHA1 Message Date
shamoon
0b265aadc6 Add error handling w retty when opening index 2025-12-17 09:14:26 -08:00
shamoon
84511d07cd Add test to mock issue 2025-12-17 09:07:22 -08:00
11 changed files with 60 additions and 403 deletions

View File

@@ -416,9 +416,6 @@ describe('WorkflowEditDialogComponent', () => {
return newFilter return newFilter
} }
const correspondentAny = addFilterOfType(TriggerFilterType.CorrespondentAny)
correspondentAny.get('values').setValue([11])
const correspondentIs = addFilterOfType(TriggerFilterType.CorrespondentIs) const correspondentIs = addFilterOfType(TriggerFilterType.CorrespondentIs)
correspondentIs.get('values').setValue(1) correspondentIs.get('values').setValue(1)
@@ -428,18 +425,12 @@ describe('WorkflowEditDialogComponent', () => {
const documentTypeIs = addFilterOfType(TriggerFilterType.DocumentTypeIs) const documentTypeIs = addFilterOfType(TriggerFilterType.DocumentTypeIs)
documentTypeIs.get('values').setValue(1) documentTypeIs.get('values').setValue(1)
const documentTypeAny = addFilterOfType(TriggerFilterType.DocumentTypeAny)
documentTypeAny.get('values').setValue([12])
const documentTypeNot = addFilterOfType(TriggerFilterType.DocumentTypeNot) const documentTypeNot = addFilterOfType(TriggerFilterType.DocumentTypeNot)
documentTypeNot.get('values').setValue([1]) documentTypeNot.get('values').setValue([1])
const storagePathIs = addFilterOfType(TriggerFilterType.StoragePathIs) const storagePathIs = addFilterOfType(TriggerFilterType.StoragePathIs)
storagePathIs.get('values').setValue(1) storagePathIs.get('values').setValue(1)
const storagePathAny = addFilterOfType(TriggerFilterType.StoragePathAny)
storagePathAny.get('values').setValue([13])
const storagePathNot = addFilterOfType(TriggerFilterType.StoragePathNot) const storagePathNot = addFilterOfType(TriggerFilterType.StoragePathNot)
storagePathNot.get('values').setValue([1]) storagePathNot.get('values').setValue([1])
@@ -454,13 +445,10 @@ describe('WorkflowEditDialogComponent', () => {
expect(formValues.triggers[0].filter_has_tags).toEqual([1]) expect(formValues.triggers[0].filter_has_tags).toEqual([1])
expect(formValues.triggers[0].filter_has_all_tags).toEqual([2, 3]) expect(formValues.triggers[0].filter_has_all_tags).toEqual([2, 3])
expect(formValues.triggers[0].filter_has_not_tags).toEqual([4]) expect(formValues.triggers[0].filter_has_not_tags).toEqual([4])
expect(formValues.triggers[0].filter_has_any_correspondents).toEqual([11])
expect(formValues.triggers[0].filter_has_correspondent).toEqual(1) expect(formValues.triggers[0].filter_has_correspondent).toEqual(1)
expect(formValues.triggers[0].filter_has_not_correspondents).toEqual([1]) expect(formValues.triggers[0].filter_has_not_correspondents).toEqual([1])
expect(formValues.triggers[0].filter_has_any_document_types).toEqual([12])
expect(formValues.triggers[0].filter_has_document_type).toEqual(1) expect(formValues.triggers[0].filter_has_document_type).toEqual(1)
expect(formValues.triggers[0].filter_has_not_document_types).toEqual([1]) expect(formValues.triggers[0].filter_has_not_document_types).toEqual([1])
expect(formValues.triggers[0].filter_has_any_storage_paths).toEqual([13])
expect(formValues.triggers[0].filter_has_storage_path).toEqual(1) expect(formValues.triggers[0].filter_has_storage_path).toEqual(1)
expect(formValues.triggers[0].filter_has_not_storage_paths).toEqual([1]) expect(formValues.triggers[0].filter_has_not_storage_paths).toEqual([1])
expect(formValues.triggers[0].filter_custom_field_query).toEqual( expect(formValues.triggers[0].filter_custom_field_query).toEqual(
@@ -523,22 +511,16 @@ describe('WorkflowEditDialogComponent', () => {
setFilter(TriggerFilterType.TagsAll, 11) setFilter(TriggerFilterType.TagsAll, 11)
setFilter(TriggerFilterType.TagsNone, 12) setFilter(TriggerFilterType.TagsNone, 12)
setFilter(TriggerFilterType.CorrespondentAny, 16)
setFilter(TriggerFilterType.CorrespondentNot, 13) setFilter(TriggerFilterType.CorrespondentNot, 13)
setFilter(TriggerFilterType.DocumentTypeAny, 17)
setFilter(TriggerFilterType.DocumentTypeNot, 14) setFilter(TriggerFilterType.DocumentTypeNot, 14)
setFilter(TriggerFilterType.StoragePathAny, 18)
setFilter(TriggerFilterType.StoragePathNot, 15) setFilter(TriggerFilterType.StoragePathNot, 15)
const formValues = component['getFormValues']() const formValues = component['getFormValues']()
expect(formValues.triggers[0].filter_has_all_tags).toEqual([11]) expect(formValues.triggers[0].filter_has_all_tags).toEqual([11])
expect(formValues.triggers[0].filter_has_not_tags).toEqual([12]) expect(formValues.triggers[0].filter_has_not_tags).toEqual([12])
expect(formValues.triggers[0].filter_has_any_correspondents).toEqual([16])
expect(formValues.triggers[0].filter_has_not_correspondents).toEqual([13]) expect(formValues.triggers[0].filter_has_not_correspondents).toEqual([13])
expect(formValues.triggers[0].filter_has_any_document_types).toEqual([17])
expect(formValues.triggers[0].filter_has_not_document_types).toEqual([14]) expect(formValues.triggers[0].filter_has_not_document_types).toEqual([14])
expect(formValues.triggers[0].filter_has_any_storage_paths).toEqual([18])
expect(formValues.triggers[0].filter_has_not_storage_paths).toEqual([15]) expect(formValues.triggers[0].filter_has_not_storage_paths).toEqual([15])
}) })
@@ -662,11 +644,8 @@ describe('WorkflowEditDialogComponent', () => {
filter_has_tags: [], filter_has_tags: [],
filter_has_all_tags: [], filter_has_all_tags: [],
filter_has_not_tags: [], filter_has_not_tags: [],
filter_has_any_correspondents: [],
filter_has_not_correspondents: [], filter_has_not_correspondents: [],
filter_has_any_document_types: [],
filter_has_not_document_types: [], filter_has_not_document_types: [],
filter_has_any_storage_paths: [],
filter_has_not_storage_paths: [], filter_has_not_storage_paths: [],
filter_has_correspondent: null, filter_has_correspondent: null,
filter_has_document_type: null, filter_has_document_type: null,
@@ -724,14 +703,11 @@ describe('WorkflowEditDialogComponent', () => {
trigger.filter_has_tags = [1] trigger.filter_has_tags = [1]
trigger.filter_has_all_tags = [2, 3] trigger.filter_has_all_tags = [2, 3]
trigger.filter_has_not_tags = [4] trigger.filter_has_not_tags = [4]
trigger.filter_has_any_correspondents = [10] as any
trigger.filter_has_correspondent = 5 as any trigger.filter_has_correspondent = 5 as any
trigger.filter_has_not_correspondents = [6] as any trigger.filter_has_not_correspondents = [6] as any
trigger.filter_has_document_type = 7 as any trigger.filter_has_document_type = 7 as any
trigger.filter_has_any_document_types = [11] as any
trigger.filter_has_not_document_types = [8] as any trigger.filter_has_not_document_types = [8] as any
trigger.filter_has_storage_path = 9 as any trigger.filter_has_storage_path = 9 as any
trigger.filter_has_any_storage_paths = [12] as any
trigger.filter_has_not_storage_paths = [10] as any trigger.filter_has_not_storage_paths = [10] as any
trigger.filter_custom_field_query = JSON.stringify([ trigger.filter_custom_field_query = JSON.stringify([
'AND', 'AND',
@@ -742,8 +718,8 @@ describe('WorkflowEditDialogComponent', () => {
component.ngOnInit() component.ngOnInit()
const triggerGroup = component.triggerFields.at(0) as FormGroup const triggerGroup = component.triggerFields.at(0) as FormGroup
const filters = component.getFiltersFormArray(triggerGroup) const filters = component.getFiltersFormArray(triggerGroup)
expect(filters.length).toBe(13) expect(filters.length).toBe(10)
const customFieldFilter = filters.at(12) as FormGroup const customFieldFilter = filters.at(9) as FormGroup
expect(customFieldFilter.get('type').value).toBe( expect(customFieldFilter.get('type').value).toBe(
TriggerFilterType.CustomFieldQuery TriggerFilterType.CustomFieldQuery
) )
@@ -752,27 +728,12 @@ describe('WorkflowEditDialogComponent', () => {
}) })
it('should expose select metadata helpers', () => { it('should expose select metadata helpers', () => {
expect(component.isSelectMultiple(TriggerFilterType.CorrespondentAny)).toBe(
true
)
expect(component.isSelectMultiple(TriggerFilterType.CorrespondentNot)).toBe( expect(component.isSelectMultiple(TriggerFilterType.CorrespondentNot)).toBe(
true true
) )
expect(component.isSelectMultiple(TriggerFilterType.CorrespondentIs)).toBe( expect(component.isSelectMultiple(TriggerFilterType.CorrespondentIs)).toBe(
false false
) )
expect(component.isSelectMultiple(TriggerFilterType.DocumentTypeAny)).toBe(
true
)
expect(component.isSelectMultiple(TriggerFilterType.DocumentTypeIs)).toBe(
false
)
expect(component.isSelectMultiple(TriggerFilterType.StoragePathAny)).toBe(
true
)
expect(component.isSelectMultiple(TriggerFilterType.StoragePathIs)).toBe(
false
)
component.correspondents = [{ id: 1, name: 'C1' } as any] component.correspondents = [{ id: 1, name: 'C1' } as any]
component.documentTypes = [{ id: 2, name: 'DT' } as any] component.documentTypes = [{ id: 2, name: 'DT' } as any]
@@ -784,15 +745,9 @@ describe('WorkflowEditDialogComponent', () => {
expect( expect(
component.getFilterSelectItems(TriggerFilterType.DocumentTypeIs) component.getFilterSelectItems(TriggerFilterType.DocumentTypeIs)
).toEqual(component.documentTypes) ).toEqual(component.documentTypes)
expect(
component.getFilterSelectItems(TriggerFilterType.DocumentTypeAny)
).toEqual(component.documentTypes)
expect( expect(
component.getFilterSelectItems(TriggerFilterType.StoragePathIs) component.getFilterSelectItems(TriggerFilterType.StoragePathIs)
).toEqual(component.storagePaths) ).toEqual(component.storagePaths)
expect(
component.getFilterSelectItems(TriggerFilterType.StoragePathAny)
).toEqual(component.storagePaths)
expect(component.getFilterSelectItems(TriggerFilterType.TagsAll)).toEqual( expect(component.getFilterSelectItems(TriggerFilterType.TagsAll)).toEqual(
[] []
) )

View File

@@ -145,13 +145,10 @@ export enum TriggerFilterType {
TagsAny = 'tags_any', TagsAny = 'tags_any',
TagsAll = 'tags_all', TagsAll = 'tags_all',
TagsNone = 'tags_none', TagsNone = 'tags_none',
CorrespondentAny = 'correspondent_any',
CorrespondentIs = 'correspondent_is', CorrespondentIs = 'correspondent_is',
CorrespondentNot = 'correspondent_not', CorrespondentNot = 'correspondent_not',
DocumentTypeAny = 'document_type_any',
DocumentTypeIs = 'document_type_is', DocumentTypeIs = 'document_type_is',
DocumentTypeNot = 'document_type_not', DocumentTypeNot = 'document_type_not',
StoragePathAny = 'storage_path_any',
StoragePathIs = 'storage_path_is', StoragePathIs = 'storage_path_is',
StoragePathNot = 'storage_path_not', StoragePathNot = 'storage_path_not',
CustomFieldQuery = 'custom_field_query', CustomFieldQuery = 'custom_field_query',
@@ -175,11 +172,8 @@ type TriggerFilterAggregate = {
filter_has_tags: number[] filter_has_tags: number[]
filter_has_all_tags: number[] filter_has_all_tags: number[]
filter_has_not_tags: number[] filter_has_not_tags: number[]
filter_has_any_correspondents: number[]
filter_has_not_correspondents: number[] filter_has_not_correspondents: number[]
filter_has_any_document_types: number[]
filter_has_not_document_types: number[] filter_has_not_document_types: number[]
filter_has_any_storage_paths: number[]
filter_has_not_storage_paths: number[] filter_has_not_storage_paths: number[]
filter_has_correspondent: number | null filter_has_correspondent: number | null
filter_has_document_type: number | null filter_has_document_type: number | null
@@ -225,14 +219,6 @@ const TRIGGER_FILTER_DEFINITIONS: TriggerFilterDefinition[] = [
allowMultipleEntries: false, allowMultipleEntries: false,
allowMultipleValues: true, allowMultipleValues: true,
}, },
{
id: TriggerFilterType.CorrespondentAny,
name: $localize`Has any of these correspondents`,
inputType: 'select',
allowMultipleEntries: false,
allowMultipleValues: true,
selectItems: 'correspondents',
},
{ {
id: TriggerFilterType.CorrespondentIs, id: TriggerFilterType.CorrespondentIs,
name: $localize`Has correspondent`, name: $localize`Has correspondent`,
@@ -257,14 +243,6 @@ const TRIGGER_FILTER_DEFINITIONS: TriggerFilterDefinition[] = [
allowMultipleValues: false, allowMultipleValues: false,
selectItems: 'documentTypes', selectItems: 'documentTypes',
}, },
{
id: TriggerFilterType.DocumentTypeAny,
name: $localize`Has any of these document types`,
inputType: 'select',
allowMultipleEntries: false,
allowMultipleValues: true,
selectItems: 'documentTypes',
},
{ {
id: TriggerFilterType.DocumentTypeNot, id: TriggerFilterType.DocumentTypeNot,
name: $localize`Does not have document types`, name: $localize`Does not have document types`,
@@ -281,14 +259,6 @@ const TRIGGER_FILTER_DEFINITIONS: TriggerFilterDefinition[] = [
allowMultipleValues: false, allowMultipleValues: false,
selectItems: 'storagePaths', selectItems: 'storagePaths',
}, },
{
id: TriggerFilterType.StoragePathAny,
name: $localize`Has any of these storage paths`,
inputType: 'select',
allowMultipleEntries: false,
allowMultipleValues: true,
selectItems: 'storagePaths',
},
{ {
id: TriggerFilterType.StoragePathNot, id: TriggerFilterType.StoragePathNot,
name: $localize`Does not have storage paths`, name: $localize`Does not have storage paths`,
@@ -336,15 +306,6 @@ const FILTER_HANDLERS: Record<TriggerFilterType, FilterHandler> = {
extract: (trigger) => trigger.filter_has_not_tags, extract: (trigger) => trigger.filter_has_not_tags,
hasValue: (value) => Array.isArray(value) && value.length > 0, hasValue: (value) => Array.isArray(value) && value.length > 0,
}, },
[TriggerFilterType.CorrespondentAny]: {
apply: (aggregate, values) => {
aggregate.filter_has_any_correspondents = Array.isArray(values)
? [...values]
: [values]
},
extract: (trigger) => trigger.filter_has_any_correspondents,
hasValue: (value) => Array.isArray(value) && value.length > 0,
},
[TriggerFilterType.CorrespondentIs]: { [TriggerFilterType.CorrespondentIs]: {
apply: (aggregate, values) => { apply: (aggregate, values) => {
aggregate.filter_has_correspondent = Array.isArray(values) aggregate.filter_has_correspondent = Array.isArray(values)
@@ -372,15 +333,6 @@ const FILTER_HANDLERS: Record<TriggerFilterType, FilterHandler> = {
extract: (trigger) => trigger.filter_has_document_type, extract: (trigger) => trigger.filter_has_document_type,
hasValue: (value) => value !== null && value !== undefined, hasValue: (value) => value !== null && value !== undefined,
}, },
[TriggerFilterType.DocumentTypeAny]: {
apply: (aggregate, values) => {
aggregate.filter_has_any_document_types = Array.isArray(values)
? [...values]
: [values]
},
extract: (trigger) => trigger.filter_has_any_document_types,
hasValue: (value) => Array.isArray(value) && value.length > 0,
},
[TriggerFilterType.DocumentTypeNot]: { [TriggerFilterType.DocumentTypeNot]: {
apply: (aggregate, values) => { apply: (aggregate, values) => {
aggregate.filter_has_not_document_types = Array.isArray(values) aggregate.filter_has_not_document_types = Array.isArray(values)
@@ -399,15 +351,6 @@ const FILTER_HANDLERS: Record<TriggerFilterType, FilterHandler> = {
extract: (trigger) => trigger.filter_has_storage_path, extract: (trigger) => trigger.filter_has_storage_path,
hasValue: (value) => value !== null && value !== undefined, hasValue: (value) => value !== null && value !== undefined,
}, },
[TriggerFilterType.StoragePathAny]: {
apply: (aggregate, values) => {
aggregate.filter_has_any_storage_paths = Array.isArray(values)
? [...values]
: [values]
},
extract: (trigger) => trigger.filter_has_any_storage_paths,
hasValue: (value) => Array.isArray(value) && value.length > 0,
},
[TriggerFilterType.StoragePathNot]: { [TriggerFilterType.StoragePathNot]: {
apply: (aggregate, values) => { apply: (aggregate, values) => {
aggregate.filter_has_not_storage_paths = Array.isArray(values) aggregate.filter_has_not_storage_paths = Array.isArray(values)
@@ -699,11 +642,8 @@ export class WorkflowEditDialogComponent
filter_has_tags: [], filter_has_tags: [],
filter_has_all_tags: [], filter_has_all_tags: [],
filter_has_not_tags: [], filter_has_not_tags: [],
filter_has_any_correspondents: [],
filter_has_not_correspondents: [], filter_has_not_correspondents: [],
filter_has_any_document_types: [],
filter_has_not_document_types: [], filter_has_not_document_types: [],
filter_has_any_storage_paths: [],
filter_has_not_storage_paths: [], filter_has_not_storage_paths: [],
filter_has_correspondent: null, filter_has_correspondent: null,
filter_has_document_type: null, filter_has_document_type: null,
@@ -730,16 +670,10 @@ export class WorkflowEditDialogComponent
trigger.filter_has_tags = aggregate.filter_has_tags trigger.filter_has_tags = aggregate.filter_has_tags
trigger.filter_has_all_tags = aggregate.filter_has_all_tags trigger.filter_has_all_tags = aggregate.filter_has_all_tags
trigger.filter_has_not_tags = aggregate.filter_has_not_tags trigger.filter_has_not_tags = aggregate.filter_has_not_tags
trigger.filter_has_any_correspondents =
aggregate.filter_has_any_correspondents
trigger.filter_has_not_correspondents = trigger.filter_has_not_correspondents =
aggregate.filter_has_not_correspondents aggregate.filter_has_not_correspondents
trigger.filter_has_any_document_types =
aggregate.filter_has_any_document_types
trigger.filter_has_not_document_types = trigger.filter_has_not_document_types =
aggregate.filter_has_not_document_types aggregate.filter_has_not_document_types
trigger.filter_has_any_storage_paths =
aggregate.filter_has_any_storage_paths
trigger.filter_has_not_storage_paths = trigger.filter_has_not_storage_paths =
aggregate.filter_has_not_storage_paths aggregate.filter_has_not_storage_paths
trigger.filter_has_correspondent = trigger.filter_has_correspondent =
@@ -922,11 +856,8 @@ export class WorkflowEditDialogComponent
case TriggerFilterType.TagsAny: case TriggerFilterType.TagsAny:
case TriggerFilterType.TagsAll: case TriggerFilterType.TagsAll:
case TriggerFilterType.TagsNone: case TriggerFilterType.TagsNone:
case TriggerFilterType.CorrespondentAny:
case TriggerFilterType.CorrespondentNot: case TriggerFilterType.CorrespondentNot:
case TriggerFilterType.DocumentTypeAny:
case TriggerFilterType.DocumentTypeNot: case TriggerFilterType.DocumentTypeNot:
case TriggerFilterType.StoragePathAny:
case TriggerFilterType.StoragePathNot: case TriggerFilterType.StoragePathNot:
return true return true
default: default:
@@ -1248,11 +1179,8 @@ export class WorkflowEditDialogComponent
filter_has_tags: [], filter_has_tags: [],
filter_has_all_tags: [], filter_has_all_tags: [],
filter_has_not_tags: [], filter_has_not_tags: [],
filter_has_any_correspondents: [],
filter_has_not_correspondents: [], filter_has_not_correspondents: [],
filter_has_any_document_types: [],
filter_has_not_document_types: [], filter_has_not_document_types: [],
filter_has_any_storage_paths: [],
filter_has_not_storage_paths: [], filter_has_not_storage_paths: [],
filter_custom_field_query: null, filter_custom_field_query: null,
filter_has_correspondent: null, filter_has_correspondent: null,

View File

@@ -44,16 +44,10 @@ export interface WorkflowTrigger extends ObjectWithId {
filter_has_not_tags?: number[] // Tag.id[] filter_has_not_tags?: number[] // Tag.id[]
filter_has_any_correspondents?: number[] // Correspondent.id[]
filter_has_not_correspondents?: number[] // Correspondent.id[] filter_has_not_correspondents?: number[] // Correspondent.id[]
filter_has_any_document_types?: number[] // DocumentType.id[]
filter_has_not_document_types?: number[] // DocumentType.id[] filter_has_not_document_types?: number[] // DocumentType.id[]
filter_has_any_storage_paths?: number[] // StoragePath.id[]
filter_has_not_storage_paths?: number[] // StoragePath.id[] filter_has_not_storage_paths?: number[] // StoragePath.id[]
filter_custom_field_query?: string filter_custom_field_query?: string

View File

@@ -10,6 +10,7 @@ from datetime import time
from datetime import timedelta from datetime import timedelta
from datetime import timezone from datetime import timezone
from shutil import rmtree from shutil import rmtree
from time import sleep
from typing import TYPE_CHECKING from typing import TYPE_CHECKING
from typing import Literal from typing import Literal
@@ -32,6 +33,7 @@ from whoosh.highlight import HtmlFormatter
from whoosh.idsets import BitSet from whoosh.idsets import BitSet
from whoosh.idsets import DocIdSet from whoosh.idsets import DocIdSet
from whoosh.index import FileIndex from whoosh.index import FileIndex
from whoosh.index import LockError
from whoosh.index import create_in from whoosh.index import create_in
from whoosh.index import exists_in from whoosh.index import exists_in
from whoosh.index import open_dir from whoosh.index import open_dir
@@ -97,11 +99,33 @@ def get_schema() -> Schema:
def open_index(*, recreate=False) -> FileIndex: def open_index(*, recreate=False) -> FileIndex:
transient_exceptions = (FileNotFoundError, LockError)
max_retries = 3
retry_delay = 0.1
for attempt in range(max_retries + 1):
try: try:
if exists_in(settings.INDEX_DIR) and not recreate: if exists_in(settings.INDEX_DIR) and not recreate:
return open_dir(settings.INDEX_DIR, schema=get_schema()) return open_dir(settings.INDEX_DIR, schema=get_schema())
break
except transient_exceptions as exc:
is_last_attempt = attempt == max_retries or recreate
if is_last_attempt:
logger.exception(
"Error while opening the index after retries, recreating.",
)
break
logger.warning(
"Transient error while opening the index (attempt %s/%s): %s. Retrying.",
attempt + 1,
max_retries + 1,
exc,
)
sleep(retry_delay)
except Exception: except Exception:
logger.exception("Error while opening the index, recreating.") logger.exception("Error while opening the index, recreating.")
break
# create_in doesn't handle corrupted indexes very well, remove the directory entirely first # create_in doesn't handle corrupted indexes very well, remove the directory entirely first
if settings.INDEX_DIR.is_dir(): if settings.INDEX_DIR.is_dir():

View File

@@ -403,18 +403,6 @@ def existing_document_matches_workflow(
f"Document tags {list(document.tags.all())} include excluded tags {list(trigger_has_not_tags_qs)}", f"Document tags {list(document.tags.all())} include excluded tags {list(trigger_has_not_tags_qs)}",
) )
allowed_correspondent_ids = set(
trigger.filter_has_any_correspondents.values_list("id", flat=True),
)
if (
allowed_correspondent_ids
and document.correspondent_id not in allowed_correspondent_ids
):
return (
False,
f"Document correspondent {document.correspondent} is not one of {list(trigger.filter_has_any_correspondents.all())}",
)
# Document correspondent vs trigger has_correspondent # Document correspondent vs trigger has_correspondent
if ( if (
trigger.filter_has_correspondent_id is not None trigger.filter_has_correspondent_id is not None
@@ -436,17 +424,6 @@ def existing_document_matches_workflow(
f"Document correspondent {document.correspondent} is excluded by {list(trigger.filter_has_not_correspondents.all())}", f"Document correspondent {document.correspondent} is excluded by {list(trigger.filter_has_not_correspondents.all())}",
) )
allowed_document_type_ids = set(
trigger.filter_has_any_document_types.values_list("id", flat=True),
)
if allowed_document_type_ids and (
document.document_type_id not in allowed_document_type_ids
):
return (
False,
f"Document doc type {document.document_type} is not one of {list(trigger.filter_has_any_document_types.all())}",
)
# Document document_type vs trigger has_document_type # Document document_type vs trigger has_document_type
if ( if (
trigger.filter_has_document_type_id is not None trigger.filter_has_document_type_id is not None
@@ -468,17 +445,6 @@ def existing_document_matches_workflow(
f"Document doc type {document.document_type} is excluded by {list(trigger.filter_has_not_document_types.all())}", f"Document doc type {document.document_type} is excluded by {list(trigger.filter_has_not_document_types.all())}",
) )
allowed_storage_path_ids = set(
trigger.filter_has_any_storage_paths.values_list("id", flat=True),
)
if allowed_storage_path_ids and (
document.storage_path_id not in allowed_storage_path_ids
):
return (
False,
f"Document storage path {document.storage_path} is not one of {list(trigger.filter_has_any_storage_paths.all())}",
)
# Document storage_path vs trigger has_storage_path # Document storage_path vs trigger has_storage_path
if ( if (
trigger.filter_has_storage_path_id is not None trigger.filter_has_storage_path_id is not None
@@ -566,10 +532,6 @@ def prefilter_documents_by_workflowtrigger(
# Correspondent, DocumentType, etc. filtering # Correspondent, DocumentType, etc. filtering
if trigger.filter_has_any_correspondents.exists():
documents = documents.filter(
correspondent__in=trigger.filter_has_any_correspondents.all(),
)
if trigger.filter_has_correspondent is not None: if trigger.filter_has_correspondent is not None:
documents = documents.filter( documents = documents.filter(
correspondent=trigger.filter_has_correspondent, correspondent=trigger.filter_has_correspondent,
@@ -579,10 +541,6 @@ def prefilter_documents_by_workflowtrigger(
correspondent__in=trigger.filter_has_not_correspondents.all(), correspondent__in=trigger.filter_has_not_correspondents.all(),
) )
if trigger.filter_has_any_document_types.exists():
documents = documents.filter(
document_type__in=trigger.filter_has_any_document_types.all(),
)
if trigger.filter_has_document_type is not None: if trigger.filter_has_document_type is not None:
documents = documents.filter( documents = documents.filter(
document_type=trigger.filter_has_document_type, document_type=trigger.filter_has_document_type,
@@ -592,10 +550,6 @@ def prefilter_documents_by_workflowtrigger(
document_type__in=trigger.filter_has_not_document_types.all(), document_type__in=trigger.filter_has_not_document_types.all(),
) )
if trigger.filter_has_any_storage_paths.exists():
documents = documents.filter(
storage_path__in=trigger.filter_has_any_storage_paths.all(),
)
if trigger.filter_has_storage_path is not None: if trigger.filter_has_storage_path is not None:
documents = documents.filter( documents = documents.filter(
storage_path=trigger.filter_has_storage_path, storage_path=trigger.filter_has_storage_path,
@@ -650,11 +604,8 @@ def document_matches_workflow(
"filter_has_tags", "filter_has_tags",
"filter_has_all_tags", "filter_has_all_tags",
"filter_has_not_tags", "filter_has_not_tags",
"filter_has_any_document_types",
"filter_has_not_document_types", "filter_has_not_document_types",
"filter_has_any_correspondents",
"filter_has_not_correspondents", "filter_has_not_correspondents",
"filter_has_any_storage_paths",
"filter_has_not_storage_paths", "filter_has_not_storage_paths",
) )
) )

View File

@@ -1,43 +0,0 @@
# Generated by Django 5.2.7 on 2025-12-17 22:25
from django.db import migrations
from django.db import models
class Migration(migrations.Migration):
dependencies = [
("documents", "1074_workflowrun_deleted_at_workflowrun_restored_at_and_more"),
]
operations = [
migrations.AddField(
model_name="workflowtrigger",
name="filter_has_any_correspondents",
field=models.ManyToManyField(
blank=True,
related_name="workflowtriggers_has_any_correspondent",
to="documents.correspondent",
verbose_name="has one of these correspondents",
),
),
migrations.AddField(
model_name="workflowtrigger",
name="filter_has_any_document_types",
field=models.ManyToManyField(
blank=True,
related_name="workflowtriggers_has_any_document_type",
to="documents.documenttype",
verbose_name="has one of these document types",
),
),
migrations.AddField(
model_name="workflowtrigger",
name="filter_has_any_storage_paths",
field=models.ManyToManyField(
blank=True,
related_name="workflowtriggers_has_any_storage_path",
to="documents.storagepath",
verbose_name="has one of these storage paths",
),
),
]

View File

@@ -1087,13 +1087,6 @@ class WorkflowTrigger(models.Model):
verbose_name=_("has this document type"), verbose_name=_("has this document type"),
) )
filter_has_any_document_types = models.ManyToManyField(
DocumentType,
blank=True,
related_name="workflowtriggers_has_any_document_type",
verbose_name=_("has one of these document types"),
)
filter_has_not_document_types = models.ManyToManyField( filter_has_not_document_types = models.ManyToManyField(
DocumentType, DocumentType,
blank=True, blank=True,
@@ -1116,13 +1109,6 @@ class WorkflowTrigger(models.Model):
verbose_name=_("does not have these correspondent(s)"), verbose_name=_("does not have these correspondent(s)"),
) )
filter_has_any_correspondents = models.ManyToManyField(
Correspondent,
blank=True,
related_name="workflowtriggers_has_any_correspondent",
verbose_name=_("has one of these correspondents"),
)
filter_has_storage_path = models.ForeignKey( filter_has_storage_path = models.ForeignKey(
StoragePath, StoragePath,
null=True, null=True,
@@ -1131,13 +1117,6 @@ class WorkflowTrigger(models.Model):
verbose_name=_("has this storage path"), verbose_name=_("has this storage path"),
) )
filter_has_any_storage_paths = models.ManyToManyField(
StoragePath,
blank=True,
related_name="workflowtriggers_has_any_storage_path",
verbose_name=_("has one of these storage paths"),
)
filter_has_not_storage_paths = models.ManyToManyField( filter_has_not_storage_paths = models.ManyToManyField(
StoragePath, StoragePath,
blank=True, blank=True,

View File

@@ -2275,11 +2275,8 @@ class WorkflowTriggerSerializer(serializers.ModelSerializer):
"filter_has_all_tags", "filter_has_all_tags",
"filter_has_not_tags", "filter_has_not_tags",
"filter_custom_field_query", "filter_custom_field_query",
"filter_has_any_correspondents",
"filter_has_not_correspondents", "filter_has_not_correspondents",
"filter_has_any_document_types",
"filter_has_not_document_types", "filter_has_not_document_types",
"filter_has_any_storage_paths",
"filter_has_not_storage_paths", "filter_has_not_storage_paths",
"filter_has_correspondent", "filter_has_correspondent",
"filter_has_document_type", "filter_has_document_type",
@@ -2517,26 +2514,14 @@ class WorkflowSerializer(serializers.ModelSerializer):
filter_has_tags = trigger.pop("filter_has_tags", None) filter_has_tags = trigger.pop("filter_has_tags", None)
filter_has_all_tags = trigger.pop("filter_has_all_tags", None) filter_has_all_tags = trigger.pop("filter_has_all_tags", None)
filter_has_not_tags = trigger.pop("filter_has_not_tags", None) filter_has_not_tags = trigger.pop("filter_has_not_tags", None)
filter_has_any_correspondents = trigger.pop(
"filter_has_any_correspondents",
None,
)
filter_has_not_correspondents = trigger.pop( filter_has_not_correspondents = trigger.pop(
"filter_has_not_correspondents", "filter_has_not_correspondents",
None, None,
) )
filter_has_any_document_types = trigger.pop(
"filter_has_any_document_types",
None,
)
filter_has_not_document_types = trigger.pop( filter_has_not_document_types = trigger.pop(
"filter_has_not_document_types", "filter_has_not_document_types",
None, None,
) )
filter_has_any_storage_paths = trigger.pop(
"filter_has_any_storage_paths",
None,
)
filter_has_not_storage_paths = trigger.pop( filter_has_not_storage_paths = trigger.pop(
"filter_has_not_storage_paths", "filter_has_not_storage_paths",
None, None,
@@ -2553,26 +2538,14 @@ class WorkflowSerializer(serializers.ModelSerializer):
trigger_instance.filter_has_all_tags.set(filter_has_all_tags) trigger_instance.filter_has_all_tags.set(filter_has_all_tags)
if filter_has_not_tags is not None: if filter_has_not_tags is not None:
trigger_instance.filter_has_not_tags.set(filter_has_not_tags) trigger_instance.filter_has_not_tags.set(filter_has_not_tags)
if filter_has_any_correspondents is not None:
trigger_instance.filter_has_any_correspondents.set(
filter_has_any_correspondents,
)
if filter_has_not_correspondents is not None: if filter_has_not_correspondents is not None:
trigger_instance.filter_has_not_correspondents.set( trigger_instance.filter_has_not_correspondents.set(
filter_has_not_correspondents, filter_has_not_correspondents,
) )
if filter_has_any_document_types is not None:
trigger_instance.filter_has_any_document_types.set(
filter_has_any_document_types,
)
if filter_has_not_document_types is not None: if filter_has_not_document_types is not None:
trigger_instance.filter_has_not_document_types.set( trigger_instance.filter_has_not_document_types.set(
filter_has_not_document_types, filter_has_not_document_types,
) )
if filter_has_any_storage_paths is not None:
trigger_instance.filter_has_any_storage_paths.set(
filter_has_any_storage_paths,
)
if filter_has_not_storage_paths is not None: if filter_has_not_storage_paths is not None:
trigger_instance.filter_has_not_storage_paths.set( trigger_instance.filter_has_not_storage_paths.set(
filter_has_not_storage_paths, filter_has_not_storage_paths,

View File

@@ -186,11 +186,8 @@ class TestApiWorkflows(DirectoriesMixin, APITestCase):
"filter_has_tags": [self.t1.id], "filter_has_tags": [self.t1.id],
"filter_has_all_tags": [self.t2.id], "filter_has_all_tags": [self.t2.id],
"filter_has_not_tags": [self.t3.id], "filter_has_not_tags": [self.t3.id],
"filter_has_any_correspondents": [self.c.id],
"filter_has_not_correspondents": [self.c2.id], "filter_has_not_correspondents": [self.c2.id],
"filter_has_any_document_types": [self.dt.id],
"filter_has_not_document_types": [self.dt2.id], "filter_has_not_document_types": [self.dt2.id],
"filter_has_any_storage_paths": [self.sp.id],
"filter_has_not_storage_paths": [self.sp2.id], "filter_has_not_storage_paths": [self.sp2.id],
"filter_custom_field_query": json.dumps( "filter_custom_field_query": json.dumps(
[ [
@@ -251,26 +248,14 @@ class TestApiWorkflows(DirectoriesMixin, APITestCase):
set(trigger.filter_has_not_tags.values_list("id", flat=True)), set(trigger.filter_has_not_tags.values_list("id", flat=True)),
{self.t3.id}, {self.t3.id},
) )
self.assertSetEqual(
set(trigger.filter_has_any_correspondents.values_list("id", flat=True)),
{self.c.id},
)
self.assertSetEqual( self.assertSetEqual(
set(trigger.filter_has_not_correspondents.values_list("id", flat=True)), set(trigger.filter_has_not_correspondents.values_list("id", flat=True)),
{self.c2.id}, {self.c2.id},
) )
self.assertSetEqual(
set(trigger.filter_has_any_document_types.values_list("id", flat=True)),
{self.dt.id},
)
self.assertSetEqual( self.assertSetEqual(
set(trigger.filter_has_not_document_types.values_list("id", flat=True)), set(trigger.filter_has_not_document_types.values_list("id", flat=True)),
{self.dt2.id}, {self.dt2.id},
) )
self.assertSetEqual(
set(trigger.filter_has_any_storage_paths.values_list("id", flat=True)),
{self.sp.id},
)
self.assertSetEqual( self.assertSetEqual(
set(trigger.filter_has_not_storage_paths.values_list("id", flat=True)), set(trigger.filter_has_not_storage_paths.values_list("id", flat=True)),
{self.sp2.id}, {self.sp2.id},
@@ -434,11 +419,8 @@ class TestApiWorkflows(DirectoriesMixin, APITestCase):
"filter_has_tags": [self.t1.id], "filter_has_tags": [self.t1.id],
"filter_has_all_tags": [self.t2.id], "filter_has_all_tags": [self.t2.id],
"filter_has_not_tags": [self.t3.id], "filter_has_not_tags": [self.t3.id],
"filter_has_any_correspondents": [self.c.id],
"filter_has_not_correspondents": [self.c2.id], "filter_has_not_correspondents": [self.c2.id],
"filter_has_any_document_types": [self.dt.id],
"filter_has_not_document_types": [self.dt2.id], "filter_has_not_document_types": [self.dt2.id],
"filter_has_any_storage_paths": [self.sp.id],
"filter_has_not_storage_paths": [self.sp2.id], "filter_has_not_storage_paths": [self.sp2.id],
"filter_custom_field_query": json.dumps( "filter_custom_field_query": json.dumps(
["AND", [[self.cf1.id, "exact", "value"]]], ["AND", [[self.cf1.id, "exact", "value"]]],
@@ -468,26 +450,14 @@ class TestApiWorkflows(DirectoriesMixin, APITestCase):
workflow.triggers.first().filter_has_not_tags.first(), workflow.triggers.first().filter_has_not_tags.first(),
self.t3, self.t3,
) )
self.assertEqual(
workflow.triggers.first().filter_has_any_correspondents.first(),
self.c,
)
self.assertEqual( self.assertEqual(
workflow.triggers.first().filter_has_not_correspondents.first(), workflow.triggers.first().filter_has_not_correspondents.first(),
self.c2, self.c2,
) )
self.assertEqual(
workflow.triggers.first().filter_has_any_document_types.first(),
self.dt,
)
self.assertEqual( self.assertEqual(
workflow.triggers.first().filter_has_not_document_types.first(), workflow.triggers.first().filter_has_not_document_types.first(),
self.dt2, self.dt2,
) )
self.assertEqual(
workflow.triggers.first().filter_has_any_storage_paths.first(),
self.sp,
)
self.assertEqual( self.assertEqual(
workflow.triggers.first().filter_has_not_storage_paths.first(), workflow.triggers.first().filter_has_not_storage_paths.first(),
self.sp2, self.sp2,

View File

@@ -1,6 +1,7 @@
from datetime import datetime from datetime import datetime
from unittest import mock from unittest import mock
from django.conf import settings
from django.contrib.auth.models import User from django.contrib.auth.models import User
from django.test import SimpleTestCase from django.test import SimpleTestCase
from django.test import TestCase from django.test import TestCase
@@ -251,3 +252,31 @@ class TestRewriteNaturalDateKeywords(SimpleTestCase):
result = self._rewrite_with_now("added:today", fixed_now) result = self._rewrite_with_now("added:today", fixed_now)
# Should convert to UTC properly # Should convert to UTC properly
self.assertIn("added:[20250719", result) self.assertIn("added:[20250719", result)
class TestIndexResilience(DirectoriesMixin, SimpleTestCase):
def test_transient_missing_segment_does_not_force_recreate(self):
file_marker = settings.INDEX_DIR / "file_marker.txt"
file_marker.write_text("keep")
expected_index = object()
with (
mock.patch("documents.index.exists_in", return_value=True),
mock.patch(
"documents.index.open_dir",
side_effect=[FileNotFoundError("missing"), expected_index],
) as mock_open_dir,
mock.patch(
"documents.index.create_in",
) as mock_create_in,
mock.patch(
"documents.index.rmtree",
) as mock_rmtree,
):
ix = index.open_index()
self.assertIs(ix, expected_index)
self.assertGreaterEqual(mock_open_dir.call_count, 2)
mock_rmtree.assert_not_called()
mock_create_in.assert_not_called()
self.assertEqual(file_marker.read_text(), "keep")

View File

@@ -1276,76 +1276,6 @@ class TestWorkflows(
) )
self.assertIn(expected_str, cm.output[1]) self.assertIn(expected_str, cm.output[1])
def test_document_added_any_filters(self):
trigger = WorkflowTrigger.objects.create(
type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED,
)
trigger.filter_has_any_correspondents.set([self.c])
trigger.filter_has_any_document_types.set([self.dt])
trigger.filter_has_any_storage_paths.set([self.sp])
matching_doc = Document.objects.create(
title="sample test",
correspondent=self.c,
document_type=self.dt,
storage_path=self.sp,
original_filename="sample.pdf",
checksum="checksum-any-match",
)
matched, reason = existing_document_matches_workflow(matching_doc, trigger)
self.assertTrue(matched)
self.assertIsNone(reason)
wrong_correspondent = Document.objects.create(
title="wrong correspondent",
correspondent=self.c2,
document_type=self.dt,
storage_path=self.sp,
original_filename="sample2.pdf",
)
matched, reason = existing_document_matches_workflow(
wrong_correspondent,
trigger,
)
self.assertFalse(matched)
self.assertIn("correspondent", reason)
other_document_type = DocumentType.objects.create(name="Other")
wrong_document_type = Document.objects.create(
title="wrong doc type",
correspondent=self.c,
document_type=other_document_type,
storage_path=self.sp,
original_filename="sample3.pdf",
checksum="checksum-wrong-doc-type",
)
matched, reason = existing_document_matches_workflow(
wrong_document_type,
trigger,
)
self.assertFalse(matched)
self.assertIn("doc type", reason)
other_storage_path = StoragePath.objects.create(
name="Other path",
path="/other/",
)
wrong_storage_path = Document.objects.create(
title="wrong storage",
correspondent=self.c,
document_type=self.dt,
storage_path=other_storage_path,
original_filename="sample4.pdf",
checksum="checksum-wrong-storage-path",
)
matched, reason = existing_document_matches_workflow(
wrong_storage_path,
trigger,
)
self.assertFalse(matched)
self.assertIn("storage path", reason)
def test_document_added_custom_field_query_no_match(self): def test_document_added_custom_field_query_no_match(self):
trigger = WorkflowTrigger.objects.create( trigger = WorkflowTrigger.objects.create(
type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED, type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED,
@@ -1454,39 +1384,6 @@ class TestWorkflows(
self.assertIn(doc1, filtered) self.assertIn(doc1, filtered)
self.assertNotIn(doc2, filtered) self.assertNotIn(doc2, filtered)
def test_prefilter_documents_any_filters(self):
trigger = WorkflowTrigger.objects.create(
type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED,
)
trigger.filter_has_any_correspondents.set([self.c])
trigger.filter_has_any_document_types.set([self.dt])
trigger.filter_has_any_storage_paths.set([self.sp])
allowed_document = Document.objects.create(
title="allowed",
correspondent=self.c,
document_type=self.dt,
storage_path=self.sp,
original_filename="doc-allowed.pdf",
checksum="checksum-any-allowed",
)
blocked_document = Document.objects.create(
title="blocked",
correspondent=self.c2,
document_type=self.dt,
storage_path=self.sp,
original_filename="doc-blocked.pdf",
checksum="checksum-any-blocked",
)
filtered = prefilter_documents_by_workflowtrigger(
Document.objects.all(),
trigger,
)
self.assertIn(allowed_document, filtered)
self.assertNotIn(blocked_document, filtered)
def test_consumption_trigger_requires_filter_configuration(self): def test_consumption_trigger_requires_filter_configuration(self):
serializer = WorkflowTriggerSerializer( serializer = WorkflowTriggerSerializer(
data={ data={