Compare commits

..

1 Commits

Author SHA1 Message Date
shamoon
5e1fd1dafa Enhancement: Add 'any of' workflow trigger filters 2026-01-09 20:44:40 -08:00
15 changed files with 434 additions and 333 deletions

View File

@@ -414,6 +414,9 @@ describe('WorkflowEditDialogComponent', () => {
return newFilter return newFilter
} }
const correspondentAny = addFilterOfType(TriggerFilterType.CorrespondentAny)
correspondentAny.get('values').setValue([11])
const correspondentIs = addFilterOfType(TriggerFilterType.CorrespondentIs) const correspondentIs = addFilterOfType(TriggerFilterType.CorrespondentIs)
correspondentIs.get('values').setValue(1) correspondentIs.get('values').setValue(1)
@@ -423,12 +426,18 @@ describe('WorkflowEditDialogComponent', () => {
const documentTypeIs = addFilterOfType(TriggerFilterType.DocumentTypeIs) const documentTypeIs = addFilterOfType(TriggerFilterType.DocumentTypeIs)
documentTypeIs.get('values').setValue(1) documentTypeIs.get('values').setValue(1)
const documentTypeAny = addFilterOfType(TriggerFilterType.DocumentTypeAny)
documentTypeAny.get('values').setValue([12])
const documentTypeNot = addFilterOfType(TriggerFilterType.DocumentTypeNot) const documentTypeNot = addFilterOfType(TriggerFilterType.DocumentTypeNot)
documentTypeNot.get('values').setValue([1]) documentTypeNot.get('values').setValue([1])
const storagePathIs = addFilterOfType(TriggerFilterType.StoragePathIs) const storagePathIs = addFilterOfType(TriggerFilterType.StoragePathIs)
storagePathIs.get('values').setValue(1) storagePathIs.get('values').setValue(1)
const storagePathAny = addFilterOfType(TriggerFilterType.StoragePathAny)
storagePathAny.get('values').setValue([13])
const storagePathNot = addFilterOfType(TriggerFilterType.StoragePathNot) const storagePathNot = addFilterOfType(TriggerFilterType.StoragePathNot)
storagePathNot.get('values').setValue([1]) storagePathNot.get('values').setValue([1])
@@ -443,10 +452,13 @@ describe('WorkflowEditDialogComponent', () => {
expect(formValues.triggers[0].filter_has_tags).toEqual([1]) expect(formValues.triggers[0].filter_has_tags).toEqual([1])
expect(formValues.triggers[0].filter_has_all_tags).toEqual([2, 3]) expect(formValues.triggers[0].filter_has_all_tags).toEqual([2, 3])
expect(formValues.triggers[0].filter_has_not_tags).toEqual([4]) expect(formValues.triggers[0].filter_has_not_tags).toEqual([4])
expect(formValues.triggers[0].filter_has_any_correspondents).toEqual([11])
expect(formValues.triggers[0].filter_has_correspondent).toEqual(1) expect(formValues.triggers[0].filter_has_correspondent).toEqual(1)
expect(formValues.triggers[0].filter_has_not_correspondents).toEqual([1]) expect(formValues.triggers[0].filter_has_not_correspondents).toEqual([1])
expect(formValues.triggers[0].filter_has_any_document_types).toEqual([12])
expect(formValues.triggers[0].filter_has_document_type).toEqual(1) expect(formValues.triggers[0].filter_has_document_type).toEqual(1)
expect(formValues.triggers[0].filter_has_not_document_types).toEqual([1]) expect(formValues.triggers[0].filter_has_not_document_types).toEqual([1])
expect(formValues.triggers[0].filter_has_any_storage_paths).toEqual([13])
expect(formValues.triggers[0].filter_has_storage_path).toEqual(1) expect(formValues.triggers[0].filter_has_storage_path).toEqual(1)
expect(formValues.triggers[0].filter_has_not_storage_paths).toEqual([1]) expect(formValues.triggers[0].filter_has_not_storage_paths).toEqual([1])
expect(formValues.triggers[0].filter_custom_field_query).toEqual( expect(formValues.triggers[0].filter_custom_field_query).toEqual(
@@ -509,16 +521,22 @@ describe('WorkflowEditDialogComponent', () => {
setFilter(TriggerFilterType.TagsAll, 11) setFilter(TriggerFilterType.TagsAll, 11)
setFilter(TriggerFilterType.TagsNone, 12) setFilter(TriggerFilterType.TagsNone, 12)
setFilter(TriggerFilterType.CorrespondentAny, 16)
setFilter(TriggerFilterType.CorrespondentNot, 13) setFilter(TriggerFilterType.CorrespondentNot, 13)
setFilter(TriggerFilterType.DocumentTypeAny, 17)
setFilter(TriggerFilterType.DocumentTypeNot, 14) setFilter(TriggerFilterType.DocumentTypeNot, 14)
setFilter(TriggerFilterType.StoragePathAny, 18)
setFilter(TriggerFilterType.StoragePathNot, 15) setFilter(TriggerFilterType.StoragePathNot, 15)
const formValues = component['getFormValues']() const formValues = component['getFormValues']()
expect(formValues.triggers[0].filter_has_all_tags).toEqual([11]) expect(formValues.triggers[0].filter_has_all_tags).toEqual([11])
expect(formValues.triggers[0].filter_has_not_tags).toEqual([12]) expect(formValues.triggers[0].filter_has_not_tags).toEqual([12])
expect(formValues.triggers[0].filter_has_any_correspondents).toEqual([16])
expect(formValues.triggers[0].filter_has_not_correspondents).toEqual([13]) expect(formValues.triggers[0].filter_has_not_correspondents).toEqual([13])
expect(formValues.triggers[0].filter_has_any_document_types).toEqual([17])
expect(formValues.triggers[0].filter_has_not_document_types).toEqual([14]) expect(formValues.triggers[0].filter_has_not_document_types).toEqual([14])
expect(formValues.triggers[0].filter_has_any_storage_paths).toEqual([18])
expect(formValues.triggers[0].filter_has_not_storage_paths).toEqual([15]) expect(formValues.triggers[0].filter_has_not_storage_paths).toEqual([15])
}) })
@@ -642,8 +660,11 @@ describe('WorkflowEditDialogComponent', () => {
filter_has_tags: [], filter_has_tags: [],
filter_has_all_tags: [], filter_has_all_tags: [],
filter_has_not_tags: [], filter_has_not_tags: [],
filter_has_any_correspondents: [],
filter_has_not_correspondents: [], filter_has_not_correspondents: [],
filter_has_any_document_types: [],
filter_has_not_document_types: [], filter_has_not_document_types: [],
filter_has_any_storage_paths: [],
filter_has_not_storage_paths: [], filter_has_not_storage_paths: [],
filter_has_correspondent: null, filter_has_correspondent: null,
filter_has_document_type: null, filter_has_document_type: null,
@@ -701,11 +722,14 @@ describe('WorkflowEditDialogComponent', () => {
trigger.filter_has_tags = [1] trigger.filter_has_tags = [1]
trigger.filter_has_all_tags = [2, 3] trigger.filter_has_all_tags = [2, 3]
trigger.filter_has_not_tags = [4] trigger.filter_has_not_tags = [4]
trigger.filter_has_any_correspondents = [10] as any
trigger.filter_has_correspondent = 5 as any trigger.filter_has_correspondent = 5 as any
trigger.filter_has_not_correspondents = [6] as any trigger.filter_has_not_correspondents = [6] as any
trigger.filter_has_document_type = 7 as any trigger.filter_has_document_type = 7 as any
trigger.filter_has_any_document_types = [11] as any
trigger.filter_has_not_document_types = [8] as any trigger.filter_has_not_document_types = [8] as any
trigger.filter_has_storage_path = 9 as any trigger.filter_has_storage_path = 9 as any
trigger.filter_has_any_storage_paths = [12] as any
trigger.filter_has_not_storage_paths = [10] as any trigger.filter_has_not_storage_paths = [10] as any
trigger.filter_custom_field_query = JSON.stringify([ trigger.filter_custom_field_query = JSON.stringify([
'AND', 'AND',
@@ -716,8 +740,8 @@ describe('WorkflowEditDialogComponent', () => {
component.ngOnInit() component.ngOnInit()
const triggerGroup = component.triggerFields.at(0) as FormGroup const triggerGroup = component.triggerFields.at(0) as FormGroup
const filters = component.getFiltersFormArray(triggerGroup) const filters = component.getFiltersFormArray(triggerGroup)
expect(filters.length).toBe(10) expect(filters.length).toBe(13)
const customFieldFilter = filters.at(9) as FormGroup const customFieldFilter = filters.at(12) as FormGroup
expect(customFieldFilter.get('type').value).toBe( expect(customFieldFilter.get('type').value).toBe(
TriggerFilterType.CustomFieldQuery TriggerFilterType.CustomFieldQuery
) )
@@ -726,12 +750,27 @@ describe('WorkflowEditDialogComponent', () => {
}) })
it('should expose select metadata helpers', () => { it('should expose select metadata helpers', () => {
expect(component.isSelectMultiple(TriggerFilterType.CorrespondentAny)).toBe(
true
)
expect(component.isSelectMultiple(TriggerFilterType.CorrespondentNot)).toBe( expect(component.isSelectMultiple(TriggerFilterType.CorrespondentNot)).toBe(
true true
) )
expect(component.isSelectMultiple(TriggerFilterType.CorrespondentIs)).toBe( expect(component.isSelectMultiple(TriggerFilterType.CorrespondentIs)).toBe(
false false
) )
expect(component.isSelectMultiple(TriggerFilterType.DocumentTypeAny)).toBe(
true
)
expect(component.isSelectMultiple(TriggerFilterType.DocumentTypeIs)).toBe(
false
)
expect(component.isSelectMultiple(TriggerFilterType.StoragePathAny)).toBe(
true
)
expect(component.isSelectMultiple(TriggerFilterType.StoragePathIs)).toBe(
false
)
component.correspondents = [{ id: 1, name: 'C1' } as any] component.correspondents = [{ id: 1, name: 'C1' } as any]
component.documentTypes = [{ id: 2, name: 'DT' } as any] component.documentTypes = [{ id: 2, name: 'DT' } as any]
@@ -743,9 +782,15 @@ describe('WorkflowEditDialogComponent', () => {
expect( expect(
component.getFilterSelectItems(TriggerFilterType.DocumentTypeIs) component.getFilterSelectItems(TriggerFilterType.DocumentTypeIs)
).toEqual(component.documentTypes) ).toEqual(component.documentTypes)
expect(
component.getFilterSelectItems(TriggerFilterType.DocumentTypeAny)
).toEqual(component.documentTypes)
expect( expect(
component.getFilterSelectItems(TriggerFilterType.StoragePathIs) component.getFilterSelectItems(TriggerFilterType.StoragePathIs)
).toEqual(component.storagePaths) ).toEqual(component.storagePaths)
expect(
component.getFilterSelectItems(TriggerFilterType.StoragePathAny)
).toEqual(component.storagePaths)
expect(component.getFilterSelectItems(TriggerFilterType.TagsAll)).toEqual( expect(component.getFilterSelectItems(TriggerFilterType.TagsAll)).toEqual(
[] []
) )

View File

@@ -145,10 +145,13 @@ export enum TriggerFilterType {
TagsAny = 'tags_any', TagsAny = 'tags_any',
TagsAll = 'tags_all', TagsAll = 'tags_all',
TagsNone = 'tags_none', TagsNone = 'tags_none',
CorrespondentAny = 'correspondent_any',
CorrespondentIs = 'correspondent_is', CorrespondentIs = 'correspondent_is',
CorrespondentNot = 'correspondent_not', CorrespondentNot = 'correspondent_not',
DocumentTypeAny = 'document_type_any',
DocumentTypeIs = 'document_type_is', DocumentTypeIs = 'document_type_is',
DocumentTypeNot = 'document_type_not', DocumentTypeNot = 'document_type_not',
StoragePathAny = 'storage_path_any',
StoragePathIs = 'storage_path_is', StoragePathIs = 'storage_path_is',
StoragePathNot = 'storage_path_not', StoragePathNot = 'storage_path_not',
CustomFieldQuery = 'custom_field_query', CustomFieldQuery = 'custom_field_query',
@@ -172,8 +175,11 @@ type TriggerFilterAggregate = {
filter_has_tags: number[] filter_has_tags: number[]
filter_has_all_tags: number[] filter_has_all_tags: number[]
filter_has_not_tags: number[] filter_has_not_tags: number[]
filter_has_any_correspondents: number[]
filter_has_not_correspondents: number[] filter_has_not_correspondents: number[]
filter_has_any_document_types: number[]
filter_has_not_document_types: number[] filter_has_not_document_types: number[]
filter_has_any_storage_paths: number[]
filter_has_not_storage_paths: number[] filter_has_not_storage_paths: number[]
filter_has_correspondent: number | null filter_has_correspondent: number | null
filter_has_document_type: number | null filter_has_document_type: number | null
@@ -219,6 +225,14 @@ const TRIGGER_FILTER_DEFINITIONS: TriggerFilterDefinition[] = [
allowMultipleEntries: false, allowMultipleEntries: false,
allowMultipleValues: true, allowMultipleValues: true,
}, },
{
id: TriggerFilterType.CorrespondentAny,
name: $localize`Has any of these correspondents`,
inputType: 'select',
allowMultipleEntries: false,
allowMultipleValues: true,
selectItems: 'correspondents',
},
{ {
id: TriggerFilterType.CorrespondentIs, id: TriggerFilterType.CorrespondentIs,
name: $localize`Has correspondent`, name: $localize`Has correspondent`,
@@ -243,6 +257,14 @@ const TRIGGER_FILTER_DEFINITIONS: TriggerFilterDefinition[] = [
allowMultipleValues: false, allowMultipleValues: false,
selectItems: 'documentTypes', selectItems: 'documentTypes',
}, },
{
id: TriggerFilterType.DocumentTypeAny,
name: $localize`Has any of these document types`,
inputType: 'select',
allowMultipleEntries: false,
allowMultipleValues: true,
selectItems: 'documentTypes',
},
{ {
id: TriggerFilterType.DocumentTypeNot, id: TriggerFilterType.DocumentTypeNot,
name: $localize`Does not have document types`, name: $localize`Does not have document types`,
@@ -259,6 +281,14 @@ const TRIGGER_FILTER_DEFINITIONS: TriggerFilterDefinition[] = [
allowMultipleValues: false, allowMultipleValues: false,
selectItems: 'storagePaths', selectItems: 'storagePaths',
}, },
{
id: TriggerFilterType.StoragePathAny,
name: $localize`Has any of these storage paths`,
inputType: 'select',
allowMultipleEntries: false,
allowMultipleValues: true,
selectItems: 'storagePaths',
},
{ {
id: TriggerFilterType.StoragePathNot, id: TriggerFilterType.StoragePathNot,
name: $localize`Does not have storage paths`, name: $localize`Does not have storage paths`,
@@ -306,6 +336,15 @@ const FILTER_HANDLERS: Record<TriggerFilterType, FilterHandler> = {
extract: (trigger) => trigger.filter_has_not_tags, extract: (trigger) => trigger.filter_has_not_tags,
hasValue: (value) => Array.isArray(value) && value.length > 0, hasValue: (value) => Array.isArray(value) && value.length > 0,
}, },
[TriggerFilterType.CorrespondentAny]: {
apply: (aggregate, values) => {
aggregate.filter_has_any_correspondents = Array.isArray(values)
? [...values]
: [values]
},
extract: (trigger) => trigger.filter_has_any_correspondents,
hasValue: (value) => Array.isArray(value) && value.length > 0,
},
[TriggerFilterType.CorrespondentIs]: { [TriggerFilterType.CorrespondentIs]: {
apply: (aggregate, values) => { apply: (aggregate, values) => {
aggregate.filter_has_correspondent = Array.isArray(values) aggregate.filter_has_correspondent = Array.isArray(values)
@@ -333,6 +372,15 @@ const FILTER_HANDLERS: Record<TriggerFilterType, FilterHandler> = {
extract: (trigger) => trigger.filter_has_document_type, extract: (trigger) => trigger.filter_has_document_type,
hasValue: (value) => value !== null && value !== undefined, hasValue: (value) => value !== null && value !== undefined,
}, },
[TriggerFilterType.DocumentTypeAny]: {
apply: (aggregate, values) => {
aggregate.filter_has_any_document_types = Array.isArray(values)
? [...values]
: [values]
},
extract: (trigger) => trigger.filter_has_any_document_types,
hasValue: (value) => Array.isArray(value) && value.length > 0,
},
[TriggerFilterType.DocumentTypeNot]: { [TriggerFilterType.DocumentTypeNot]: {
apply: (aggregate, values) => { apply: (aggregate, values) => {
aggregate.filter_has_not_document_types = Array.isArray(values) aggregate.filter_has_not_document_types = Array.isArray(values)
@@ -351,6 +399,15 @@ const FILTER_HANDLERS: Record<TriggerFilterType, FilterHandler> = {
extract: (trigger) => trigger.filter_has_storage_path, extract: (trigger) => trigger.filter_has_storage_path,
hasValue: (value) => value !== null && value !== undefined, hasValue: (value) => value !== null && value !== undefined,
}, },
[TriggerFilterType.StoragePathAny]: {
apply: (aggregate, values) => {
aggregate.filter_has_any_storage_paths = Array.isArray(values)
? [...values]
: [values]
},
extract: (trigger) => trigger.filter_has_any_storage_paths,
hasValue: (value) => Array.isArray(value) && value.length > 0,
},
[TriggerFilterType.StoragePathNot]: { [TriggerFilterType.StoragePathNot]: {
apply: (aggregate, values) => { apply: (aggregate, values) => {
aggregate.filter_has_not_storage_paths = Array.isArray(values) aggregate.filter_has_not_storage_paths = Array.isArray(values)
@@ -642,8 +699,11 @@ export class WorkflowEditDialogComponent
filter_has_tags: [], filter_has_tags: [],
filter_has_all_tags: [], filter_has_all_tags: [],
filter_has_not_tags: [], filter_has_not_tags: [],
filter_has_any_correspondents: [],
filter_has_not_correspondents: [], filter_has_not_correspondents: [],
filter_has_any_document_types: [],
filter_has_not_document_types: [], filter_has_not_document_types: [],
filter_has_any_storage_paths: [],
filter_has_not_storage_paths: [], filter_has_not_storage_paths: [],
filter_has_correspondent: null, filter_has_correspondent: null,
filter_has_document_type: null, filter_has_document_type: null,
@@ -670,10 +730,16 @@ export class WorkflowEditDialogComponent
trigger.filter_has_tags = aggregate.filter_has_tags trigger.filter_has_tags = aggregate.filter_has_tags
trigger.filter_has_all_tags = aggregate.filter_has_all_tags trigger.filter_has_all_tags = aggregate.filter_has_all_tags
trigger.filter_has_not_tags = aggregate.filter_has_not_tags trigger.filter_has_not_tags = aggregate.filter_has_not_tags
trigger.filter_has_any_correspondents =
aggregate.filter_has_any_correspondents
trigger.filter_has_not_correspondents = trigger.filter_has_not_correspondents =
aggregate.filter_has_not_correspondents aggregate.filter_has_not_correspondents
trigger.filter_has_any_document_types =
aggregate.filter_has_any_document_types
trigger.filter_has_not_document_types = trigger.filter_has_not_document_types =
aggregate.filter_has_not_document_types aggregate.filter_has_not_document_types
trigger.filter_has_any_storage_paths =
aggregate.filter_has_any_storage_paths
trigger.filter_has_not_storage_paths = trigger.filter_has_not_storage_paths =
aggregate.filter_has_not_storage_paths aggregate.filter_has_not_storage_paths
trigger.filter_has_correspondent = trigger.filter_has_correspondent =
@@ -856,8 +922,11 @@ export class WorkflowEditDialogComponent
case TriggerFilterType.TagsAny: case TriggerFilterType.TagsAny:
case TriggerFilterType.TagsAll: case TriggerFilterType.TagsAll:
case TriggerFilterType.TagsNone: case TriggerFilterType.TagsNone:
case TriggerFilterType.CorrespondentAny:
case TriggerFilterType.CorrespondentNot: case TriggerFilterType.CorrespondentNot:
case TriggerFilterType.DocumentTypeAny:
case TriggerFilterType.DocumentTypeNot: case TriggerFilterType.DocumentTypeNot:
case TriggerFilterType.StoragePathAny:
case TriggerFilterType.StoragePathNot: case TriggerFilterType.StoragePathNot:
return true return true
default: default:
@@ -1179,8 +1248,11 @@ export class WorkflowEditDialogComponent
filter_has_tags: [], filter_has_tags: [],
filter_has_all_tags: [], filter_has_all_tags: [],
filter_has_not_tags: [], filter_has_not_tags: [],
filter_has_any_correspondents: [],
filter_has_not_correspondents: [], filter_has_not_correspondents: [],
filter_has_any_document_types: [],
filter_has_not_document_types: [], filter_has_not_document_types: [],
filter_has_any_storage_paths: [],
filter_has_not_storage_paths: [], filter_has_not_storage_paths: [],
filter_custom_field_query: null, filter_custom_field_query: null,
filter_has_correspondent: null, filter_has_correspondent: null,

View File

@@ -44,10 +44,16 @@ export interface WorkflowTrigger extends ObjectWithId {
filter_has_not_tags?: number[] // Tag.id[] filter_has_not_tags?: number[] // Tag.id[]
filter_has_any_correspondents?: number[] // Correspondent.id[]
filter_has_not_correspondents?: number[] // Correspondent.id[] filter_has_not_correspondents?: number[] // Correspondent.id[]
filter_has_any_document_types?: number[] // DocumentType.id[]
filter_has_not_document_types?: number[] // DocumentType.id[] filter_has_not_document_types?: number[] // DocumentType.id[]
filter_has_any_storage_paths?: number[] // StoragePath.id[]
filter_has_not_storage_paths?: number[] // StoragePath.id[] filter_has_not_storage_paths?: number[] // StoragePath.id[]
filter_custom_field_query?: string filter_custom_field_query?: string

View File

@@ -16,7 +16,6 @@ from pikepdf import Pdf
from documents.converters import convert_from_tiff_to_pdf from documents.converters import convert_from_tiff_to_pdf
from documents.data_models import ConsumableDocument from documents.data_models import ConsumableDocument
from documents.data_models import DocumentMetadataOverrides from documents.data_models import DocumentMetadataOverrides
from documents.models import Document
from documents.models import Tag from documents.models import Tag
from documents.plugins.base import ConsumeTaskPlugin from documents.plugins.base import ConsumeTaskPlugin
from documents.plugins.base import StopConsumeTaskError from documents.plugins.base import StopConsumeTaskError
@@ -116,24 +115,6 @@ class BarcodePlugin(ConsumeTaskPlugin):
self._tiff_conversion_done = False self._tiff_conversion_done = False
self.barcodes: list[Barcode] = [] self.barcodes: list[Barcode] = []
def _apply_detected_asn(self, detected_asn: int) -> None:
"""
Apply a detected ASN to metadata if allowed.
"""
if (
self.metadata.skip_asn_if_exists
and Document.global_objects.filter(
archive_serial_number=detected_asn,
).exists()
):
logger.info(
f"Found ASN in barcode {detected_asn} but skipping because it already exists.",
)
return
logger.info(f"Found ASN in barcode: {detected_asn}")
self.metadata.asn = detected_asn
def run(self) -> None: def run(self) -> None:
# Some operations may use PIL, override pixel setting if needed # Some operations may use PIL, override pixel setting if needed
maybe_override_pixel_limit() maybe_override_pixel_limit()
@@ -205,8 +186,13 @@ class BarcodePlugin(ConsumeTaskPlugin):
# Update/overwrite an ASN if possible # Update/overwrite an ASN if possible
# After splitting, as otherwise each split document gets the same ASN # After splitting, as otherwise each split document gets the same ASN
if self.settings.barcode_enable_asn and (located_asn := self.asn) is not None: if (
self._apply_detected_asn(located_asn) self.settings.barcode_enable_asn
and not self.metadata.skip_asn
and (located_asn := self.asn) is not None
):
logger.info(f"Found ASN in barcode: {located_asn}")
self.metadata.asn = located_asn
def cleanup(self) -> None: def cleanup(self) -> None:
self.temp_dir.cleanup() self.temp_dir.cleanup()

View File

@@ -7,6 +7,7 @@ from pathlib import Path
from typing import TYPE_CHECKING from typing import TYPE_CHECKING
from typing import Literal from typing import Literal
from celery import chain
from celery import chord from celery import chord
from celery import group from celery import group
from celery import shared_task from celery import shared_task
@@ -37,42 +38,6 @@ if TYPE_CHECKING:
logger: logging.Logger = logging.getLogger("paperless.bulk_edit") logger: logging.Logger = logging.getLogger("paperless.bulk_edit")
@shared_task(bind=True)
def restore_archive_serial_numbers_task(
self,
backup: dict[int, int],
*args,
**kwargs,
) -> None:
restore_archive_serial_numbers(backup)
def release_archive_serial_numbers(doc_ids: list[int]) -> dict[int, int]:
"""
Clears ASNs on documents that are about to be replaced so new documents
can be assigned ASNs without uniqueness collisions. Returns a backup map
of doc_id -> previous ASN for potential restoration.
"""
qs = Document.objects.filter(
id__in=doc_ids,
archive_serial_number__isnull=False,
).only("pk", "archive_serial_number")
backup = dict(qs.values_list("pk", "archive_serial_number"))
qs.update(archive_serial_number=None)
logger.info(f"Released archive serial numbers for documents {list(backup.keys())}")
return backup
def restore_archive_serial_numbers(backup: dict[int, int]) -> None:
"""
Restores ASNs using the provided backup map, intended for
rollback when replacement consumption fails.
"""
for doc_id, asn in backup.items():
Document.objects.filter(pk=doc_id).update(archive_serial_number=asn)
logger.info(f"Restored archive serial numbers for documents {list(backup.keys())}")
def set_correspondent( def set_correspondent(
doc_ids: list[int], doc_ids: list[int],
correspondent: Correspondent, correspondent: Correspondent,
@@ -421,7 +386,6 @@ def merge(
merged_pdf = pikepdf.new() merged_pdf = pikepdf.new()
version: str = merged_pdf.pdf_version version: str = merged_pdf.pdf_version
handoff_asn: int | None = None
# use doc_ids to preserve order # use doc_ids to preserve order
for doc_id in doc_ids: for doc_id in doc_ids:
doc = qs.get(id=doc_id) doc = qs.get(id=doc_id)
@@ -437,8 +401,6 @@ def merge(
version = max(version, pdf.pdf_version) version = max(version, pdf.pdf_version)
merged_pdf.pages.extend(pdf.pages) merged_pdf.pages.extend(pdf.pages)
affected_docs.append(doc.id) affected_docs.append(doc.id)
if handoff_asn is None and doc.archive_serial_number is not None:
handoff_asn = doc.archive_serial_number
except Exception as e: except Exception as e:
logger.exception( logger.exception(
f"Error merging document {doc.id}, it will not be included in the merge: {e}", f"Error merging document {doc.id}, it will not be included in the merge: {e}",
@@ -464,8 +426,6 @@ def merge(
DocumentMetadataOverrides.from_document(metadata_document) DocumentMetadataOverrides.from_document(metadata_document)
) )
overrides.title = metadata_document.title + " (merged)" overrides.title = metadata_document.title + " (merged)"
if metadata_document.archive_serial_number is not None:
handoff_asn = metadata_document.archive_serial_number
else: else:
overrides = DocumentMetadataOverrides() overrides = DocumentMetadataOverrides()
else: else:
@@ -473,11 +433,8 @@ def merge(
if user is not None: if user is not None:
overrides.owner_id = user.id overrides.owner_id = user.id
if not delete_originals: # Avoid copying or detecting ASN from merged PDFs to prevent collision
overrides.skip_asn_if_exists = True overrides.skip_asn = True
if delete_originals and handoff_asn is not None:
overrides.asn = handoff_asn
logger.info("Adding merged document to the task queue.") logger.info("Adding merged document to the task queue.")
@@ -490,20 +447,12 @@ def merge(
) )
if delete_originals: if delete_originals:
backup = release_archive_serial_numbers(affected_docs)
logger.info( logger.info(
"Queueing removal of original documents after consumption of merged document", "Queueing removal of original documents after consumption of merged document",
) )
try: chain(consume_task, delete.si(affected_docs)).delay()
consume_task.apply_async( else:
link=[delete.si(affected_docs)], consume_task.delay()
link_error=[restore_archive_serial_numbers_task.s(backup)],
)
except Exception:
restore_archive_serial_numbers(backup)
raise
else:
consume_task.delay()
return "OK" return "OK"
@@ -545,8 +494,6 @@ def split(
overrides.title = f"{doc.title} (split {idx + 1})" overrides.title = f"{doc.title} (split {idx + 1})"
if user is not None: if user is not None:
overrides.owner_id = user.id overrides.owner_id = user.id
if not delete_originals:
overrides.skip_asn_if_exists = True
logger.info( logger.info(
f"Adding split document with pages {split_doc} to the task queue.", f"Adding split document with pages {split_doc} to the task queue.",
) )
@@ -561,20 +508,10 @@ def split(
) )
if delete_originals: if delete_originals:
backup = release_archive_serial_numbers([doc.id])
logger.info( logger.info(
"Queueing removal of original document after consumption of the split documents", "Queueing removal of original document after consumption of the split documents",
) )
try: chord(header=consume_tasks, body=delete.si([doc.id])).delay()
chord(
header=consume_tasks,
body=delete.si([doc.id]),
).apply_async(
link_error=[restore_archive_serial_numbers_task.s(backup)],
)
except Exception:
restore_archive_serial_numbers(backup)
raise
else: else:
group(consume_tasks).delay() group(consume_tasks).delay()
@@ -677,10 +614,7 @@ def edit_pdf(
) )
if user is not None: if user is not None:
overrides.owner_id = user.id overrides.owner_id = user.id
if not delete_original:
overrides.skip_asn_if_exists = True
if delete_original and len(pdf_docs) == 1:
overrides.asn = doc.archive_serial_number
for idx, pdf in enumerate(pdf_docs, start=1): for idx, pdf in enumerate(pdf_docs, start=1):
filepath: Path = ( filepath: Path = (
Path(tempfile.mkdtemp(dir=settings.SCRATCH_DIR)) Path(tempfile.mkdtemp(dir=settings.SCRATCH_DIR))
@@ -699,17 +633,7 @@ def edit_pdf(
) )
if delete_original: if delete_original:
backup = release_archive_serial_numbers([doc.id]) chord(header=consume_tasks, body=delete.si([doc.id])).delay()
try:
chord(
header=consume_tasks,
body=delete.si([doc.id]),
).apply_async(
link_error=[restore_archive_serial_numbers_task.s(backup)],
)
except Exception:
restore_archive_serial_numbers(backup)
raise
else: else:
group(consume_tasks).delay() group(consume_tasks).delay()

View File

@@ -696,7 +696,7 @@ class ConsumerPlugin(
pk=self.metadata.storage_path_id, pk=self.metadata.storage_path_id,
) )
if self.metadata.asn is not None: if self.metadata.asn is not None and not self.metadata.skip_asn:
document.archive_serial_number = self.metadata.asn document.archive_serial_number = self.metadata.asn
if self.metadata.owner_id: if self.metadata.owner_id:
@@ -812,8 +812,8 @@ class ConsumerPreflightPlugin(
""" """
Check that if override_asn is given, it is unique and within a valid range Check that if override_asn is given, it is unique and within a valid range
""" """
if self.metadata.asn is None: if self.metadata.skip_asn or self.metadata.asn is None:
# if ASN is None # if skip is set or ASN is None
return return
# Validate the range is above zero and less than uint32_t max # Validate the range is above zero and less than uint32_t max
# otherwise, Whoosh can't handle it in the index # otherwise, Whoosh can't handle it in the index

View File

@@ -30,7 +30,7 @@ class DocumentMetadataOverrides:
change_users: list[int] | None = None change_users: list[int] | None = None
change_groups: list[int] | None = None change_groups: list[int] | None = None
custom_fields: dict | None = None custom_fields: dict | None = None
skip_asn_if_exists: bool = False skip_asn: bool = False
def update(self, other: "DocumentMetadataOverrides") -> "DocumentMetadataOverrides": def update(self, other: "DocumentMetadataOverrides") -> "DocumentMetadataOverrides":
""" """
@@ -50,8 +50,8 @@ class DocumentMetadataOverrides:
self.storage_path_id = other.storage_path_id self.storage_path_id = other.storage_path_id
if other.owner_id is not None: if other.owner_id is not None:
self.owner_id = other.owner_id self.owner_id = other.owner_id
if other.skip_asn_if_exists: if other.skip_asn:
self.skip_asn_if_exists = True self.skip_asn = True
# merge # merge
if self.tag_ids is None: if self.tag_ids is None:

View File

@@ -403,6 +403,18 @@ def existing_document_matches_workflow(
f"Document tags {list(document.tags.all())} include excluded tags {list(trigger_has_not_tags_qs)}", f"Document tags {list(document.tags.all())} include excluded tags {list(trigger_has_not_tags_qs)}",
) )
allowed_correspondent_ids = set(
trigger.filter_has_any_correspondents.values_list("id", flat=True),
)
if (
allowed_correspondent_ids
and document.correspondent_id not in allowed_correspondent_ids
):
return (
False,
f"Document correspondent {document.correspondent} is not one of {list(trigger.filter_has_any_correspondents.all())}",
)
# Document correspondent vs trigger has_correspondent # Document correspondent vs trigger has_correspondent
if ( if (
trigger.filter_has_correspondent_id is not None trigger.filter_has_correspondent_id is not None
@@ -424,6 +436,17 @@ def existing_document_matches_workflow(
f"Document correspondent {document.correspondent} is excluded by {list(trigger.filter_has_not_correspondents.all())}", f"Document correspondent {document.correspondent} is excluded by {list(trigger.filter_has_not_correspondents.all())}",
) )
allowed_document_type_ids = set(
trigger.filter_has_any_document_types.values_list("id", flat=True),
)
if allowed_document_type_ids and (
document.document_type_id not in allowed_document_type_ids
):
return (
False,
f"Document doc type {document.document_type} is not one of {list(trigger.filter_has_any_document_types.all())}",
)
# Document document_type vs trigger has_document_type # Document document_type vs trigger has_document_type
if ( if (
trigger.filter_has_document_type_id is not None trigger.filter_has_document_type_id is not None
@@ -445,6 +468,17 @@ def existing_document_matches_workflow(
f"Document doc type {document.document_type} is excluded by {list(trigger.filter_has_not_document_types.all())}", f"Document doc type {document.document_type} is excluded by {list(trigger.filter_has_not_document_types.all())}",
) )
allowed_storage_path_ids = set(
trigger.filter_has_any_storage_paths.values_list("id", flat=True),
)
if allowed_storage_path_ids and (
document.storage_path_id not in allowed_storage_path_ids
):
return (
False,
f"Document storage path {document.storage_path} is not one of {list(trigger.filter_has_any_storage_paths.all())}",
)
# Document storage_path vs trigger has_storage_path # Document storage_path vs trigger has_storage_path
if ( if (
trigger.filter_has_storage_path_id is not None trigger.filter_has_storage_path_id is not None
@@ -532,6 +566,10 @@ def prefilter_documents_by_workflowtrigger(
# Correspondent, DocumentType, etc. filtering # Correspondent, DocumentType, etc. filtering
if trigger.filter_has_any_correspondents.exists():
documents = documents.filter(
correspondent__in=trigger.filter_has_any_correspondents.all(),
)
if trigger.filter_has_correspondent is not None: if trigger.filter_has_correspondent is not None:
documents = documents.filter( documents = documents.filter(
correspondent=trigger.filter_has_correspondent, correspondent=trigger.filter_has_correspondent,
@@ -541,6 +579,10 @@ def prefilter_documents_by_workflowtrigger(
correspondent__in=trigger.filter_has_not_correspondents.all(), correspondent__in=trigger.filter_has_not_correspondents.all(),
) )
if trigger.filter_has_any_document_types.exists():
documents = documents.filter(
document_type__in=trigger.filter_has_any_document_types.all(),
)
if trigger.filter_has_document_type is not None: if trigger.filter_has_document_type is not None:
documents = documents.filter( documents = documents.filter(
document_type=trigger.filter_has_document_type, document_type=trigger.filter_has_document_type,
@@ -550,6 +592,10 @@ def prefilter_documents_by_workflowtrigger(
document_type__in=trigger.filter_has_not_document_types.all(), document_type__in=trigger.filter_has_not_document_types.all(),
) )
if trigger.filter_has_any_storage_paths.exists():
documents = documents.filter(
storage_path__in=trigger.filter_has_any_storage_paths.all(),
)
if trigger.filter_has_storage_path is not None: if trigger.filter_has_storage_path is not None:
documents = documents.filter( documents = documents.filter(
storage_path=trigger.filter_has_storage_path, storage_path=trigger.filter_has_storage_path,
@@ -604,8 +650,11 @@ def document_matches_workflow(
"filter_has_tags", "filter_has_tags",
"filter_has_all_tags", "filter_has_all_tags",
"filter_has_not_tags", "filter_has_not_tags",
"filter_has_any_document_types",
"filter_has_not_document_types", "filter_has_not_document_types",
"filter_has_any_correspondents",
"filter_has_not_correspondents", "filter_has_not_correspondents",
"filter_has_any_storage_paths",
"filter_has_not_storage_paths", "filter_has_not_storage_paths",
) )
) )

View File

@@ -0,0 +1,43 @@
# Generated by Django 5.2.7 on 2025-12-17 22:25
from django.db import migrations
from django.db import models
class Migration(migrations.Migration):
dependencies = [
("documents", "1074_workflowrun_deleted_at_workflowrun_restored_at_and_more"),
]
operations = [
migrations.AddField(
model_name="workflowtrigger",
name="filter_has_any_correspondents",
field=models.ManyToManyField(
blank=True,
related_name="workflowtriggers_has_any_correspondent",
to="documents.correspondent",
verbose_name="has one of these correspondents",
),
),
migrations.AddField(
model_name="workflowtrigger",
name="filter_has_any_document_types",
field=models.ManyToManyField(
blank=True,
related_name="workflowtriggers_has_any_document_type",
to="documents.documenttype",
verbose_name="has one of these document types",
),
),
migrations.AddField(
model_name="workflowtrigger",
name="filter_has_any_storage_paths",
field=models.ManyToManyField(
blank=True,
related_name="workflowtriggers_has_any_storage_path",
to="documents.storagepath",
verbose_name="has one of these storage paths",
),
),
]

View File

@@ -1087,6 +1087,13 @@ class WorkflowTrigger(models.Model):
verbose_name=_("has this document type"), verbose_name=_("has this document type"),
) )
filter_has_any_document_types = models.ManyToManyField(
DocumentType,
blank=True,
related_name="workflowtriggers_has_any_document_type",
verbose_name=_("has one of these document types"),
)
filter_has_not_document_types = models.ManyToManyField( filter_has_not_document_types = models.ManyToManyField(
DocumentType, DocumentType,
blank=True, blank=True,
@@ -1109,6 +1116,13 @@ class WorkflowTrigger(models.Model):
verbose_name=_("does not have these correspondent(s)"), verbose_name=_("does not have these correspondent(s)"),
) )
filter_has_any_correspondents = models.ManyToManyField(
Correspondent,
blank=True,
related_name="workflowtriggers_has_any_correspondent",
verbose_name=_("has one of these correspondents"),
)
filter_has_storage_path = models.ForeignKey( filter_has_storage_path = models.ForeignKey(
StoragePath, StoragePath,
null=True, null=True,
@@ -1117,6 +1131,13 @@ class WorkflowTrigger(models.Model):
verbose_name=_("has this storage path"), verbose_name=_("has this storage path"),
) )
filter_has_any_storage_paths = models.ManyToManyField(
StoragePath,
blank=True,
related_name="workflowtriggers_has_any_storage_path",
verbose_name=_("has one of these storage paths"),
)
filter_has_not_storage_paths = models.ManyToManyField( filter_has_not_storage_paths = models.ManyToManyField(
StoragePath, StoragePath,
blank=True, blank=True,

View File

@@ -2295,8 +2295,11 @@ class WorkflowTriggerSerializer(serializers.ModelSerializer):
"filter_has_all_tags", "filter_has_all_tags",
"filter_has_not_tags", "filter_has_not_tags",
"filter_custom_field_query", "filter_custom_field_query",
"filter_has_any_correspondents",
"filter_has_not_correspondents", "filter_has_not_correspondents",
"filter_has_any_document_types",
"filter_has_not_document_types", "filter_has_not_document_types",
"filter_has_any_storage_paths",
"filter_has_not_storage_paths", "filter_has_not_storage_paths",
"filter_has_correspondent", "filter_has_correspondent",
"filter_has_document_type", "filter_has_document_type",
@@ -2534,14 +2537,26 @@ class WorkflowSerializer(serializers.ModelSerializer):
filter_has_tags = trigger.pop("filter_has_tags", None) filter_has_tags = trigger.pop("filter_has_tags", None)
filter_has_all_tags = trigger.pop("filter_has_all_tags", None) filter_has_all_tags = trigger.pop("filter_has_all_tags", None)
filter_has_not_tags = trigger.pop("filter_has_not_tags", None) filter_has_not_tags = trigger.pop("filter_has_not_tags", None)
filter_has_any_correspondents = trigger.pop(
"filter_has_any_correspondents",
None,
)
filter_has_not_correspondents = trigger.pop( filter_has_not_correspondents = trigger.pop(
"filter_has_not_correspondents", "filter_has_not_correspondents",
None, None,
) )
filter_has_any_document_types = trigger.pop(
"filter_has_any_document_types",
None,
)
filter_has_not_document_types = trigger.pop( filter_has_not_document_types = trigger.pop(
"filter_has_not_document_types", "filter_has_not_document_types",
None, None,
) )
filter_has_any_storage_paths = trigger.pop(
"filter_has_any_storage_paths",
None,
)
filter_has_not_storage_paths = trigger.pop( filter_has_not_storage_paths = trigger.pop(
"filter_has_not_storage_paths", "filter_has_not_storage_paths",
None, None,
@@ -2558,14 +2573,26 @@ class WorkflowSerializer(serializers.ModelSerializer):
trigger_instance.filter_has_all_tags.set(filter_has_all_tags) trigger_instance.filter_has_all_tags.set(filter_has_all_tags)
if filter_has_not_tags is not None: if filter_has_not_tags is not None:
trigger_instance.filter_has_not_tags.set(filter_has_not_tags) trigger_instance.filter_has_not_tags.set(filter_has_not_tags)
if filter_has_any_correspondents is not None:
trigger_instance.filter_has_any_correspondents.set(
filter_has_any_correspondents,
)
if filter_has_not_correspondents is not None: if filter_has_not_correspondents is not None:
trigger_instance.filter_has_not_correspondents.set( trigger_instance.filter_has_not_correspondents.set(
filter_has_not_correspondents, filter_has_not_correspondents,
) )
if filter_has_any_document_types is not None:
trigger_instance.filter_has_any_document_types.set(
filter_has_any_document_types,
)
if filter_has_not_document_types is not None: if filter_has_not_document_types is not None:
trigger_instance.filter_has_not_document_types.set( trigger_instance.filter_has_not_document_types.set(
filter_has_not_document_types, filter_has_not_document_types,
) )
if filter_has_any_storage_paths is not None:
trigger_instance.filter_has_any_storage_paths.set(
filter_has_any_storage_paths,
)
if filter_has_not_storage_paths is not None: if filter_has_not_storage_paths is not None:
trigger_instance.filter_has_not_storage_paths.set( trigger_instance.filter_has_not_storage_paths.set(
filter_has_not_storage_paths, filter_has_not_storage_paths,

View File

@@ -186,8 +186,11 @@ class TestApiWorkflows(DirectoriesMixin, APITestCase):
"filter_has_tags": [self.t1.id], "filter_has_tags": [self.t1.id],
"filter_has_all_tags": [self.t2.id], "filter_has_all_tags": [self.t2.id],
"filter_has_not_tags": [self.t3.id], "filter_has_not_tags": [self.t3.id],
"filter_has_any_correspondents": [self.c.id],
"filter_has_not_correspondents": [self.c2.id], "filter_has_not_correspondents": [self.c2.id],
"filter_has_any_document_types": [self.dt.id],
"filter_has_not_document_types": [self.dt2.id], "filter_has_not_document_types": [self.dt2.id],
"filter_has_any_storage_paths": [self.sp.id],
"filter_has_not_storage_paths": [self.sp2.id], "filter_has_not_storage_paths": [self.sp2.id],
"filter_custom_field_query": json.dumps( "filter_custom_field_query": json.dumps(
[ [
@@ -248,14 +251,26 @@ class TestApiWorkflows(DirectoriesMixin, APITestCase):
set(trigger.filter_has_not_tags.values_list("id", flat=True)), set(trigger.filter_has_not_tags.values_list("id", flat=True)),
{self.t3.id}, {self.t3.id},
) )
self.assertSetEqual(
set(trigger.filter_has_any_correspondents.values_list("id", flat=True)),
{self.c.id},
)
self.assertSetEqual( self.assertSetEqual(
set(trigger.filter_has_not_correspondents.values_list("id", flat=True)), set(trigger.filter_has_not_correspondents.values_list("id", flat=True)),
{self.c2.id}, {self.c2.id},
) )
self.assertSetEqual(
set(trigger.filter_has_any_document_types.values_list("id", flat=True)),
{self.dt.id},
)
self.assertSetEqual( self.assertSetEqual(
set(trigger.filter_has_not_document_types.values_list("id", flat=True)), set(trigger.filter_has_not_document_types.values_list("id", flat=True)),
{self.dt2.id}, {self.dt2.id},
) )
self.assertSetEqual(
set(trigger.filter_has_any_storage_paths.values_list("id", flat=True)),
{self.sp.id},
)
self.assertSetEqual( self.assertSetEqual(
set(trigger.filter_has_not_storage_paths.values_list("id", flat=True)), set(trigger.filter_has_not_storage_paths.values_list("id", flat=True)),
{self.sp2.id}, {self.sp2.id},
@@ -419,8 +434,11 @@ class TestApiWorkflows(DirectoriesMixin, APITestCase):
"filter_has_tags": [self.t1.id], "filter_has_tags": [self.t1.id],
"filter_has_all_tags": [self.t2.id], "filter_has_all_tags": [self.t2.id],
"filter_has_not_tags": [self.t3.id], "filter_has_not_tags": [self.t3.id],
"filter_has_any_correspondents": [self.c.id],
"filter_has_not_correspondents": [self.c2.id], "filter_has_not_correspondents": [self.c2.id],
"filter_has_any_document_types": [self.dt.id],
"filter_has_not_document_types": [self.dt2.id], "filter_has_not_document_types": [self.dt2.id],
"filter_has_any_storage_paths": [self.sp.id],
"filter_has_not_storage_paths": [self.sp2.id], "filter_has_not_storage_paths": [self.sp2.id],
"filter_custom_field_query": json.dumps( "filter_custom_field_query": json.dumps(
["AND", [[self.cf1.id, "exact", "value"]]], ["AND", [[self.cf1.id, "exact", "value"]]],
@@ -450,14 +468,26 @@ class TestApiWorkflows(DirectoriesMixin, APITestCase):
workflow.triggers.first().filter_has_not_tags.first(), workflow.triggers.first().filter_has_not_tags.first(),
self.t3, self.t3,
) )
self.assertEqual(
workflow.triggers.first().filter_has_any_correspondents.first(),
self.c,
)
self.assertEqual( self.assertEqual(
workflow.triggers.first().filter_has_not_correspondents.first(), workflow.triggers.first().filter_has_not_correspondents.first(),
self.c2, self.c2,
) )
self.assertEqual(
workflow.triggers.first().filter_has_any_document_types.first(),
self.dt,
)
self.assertEqual( self.assertEqual(
workflow.triggers.first().filter_has_not_document_types.first(), workflow.triggers.first().filter_has_not_document_types.first(),
self.dt2, self.dt2,
) )
self.assertEqual(
workflow.triggers.first().filter_has_any_storage_paths.first(),
self.sp,
)
self.assertEqual( self.assertEqual(
workflow.triggers.first().filter_has_not_storage_paths.first(), workflow.triggers.first().filter_has_not_storage_paths.first(),
self.sp2, self.sp2,

View File

@@ -603,21 +603,23 @@ class TestPDFActions(DirectoriesMixin, TestCase):
expected_filename, expected_filename,
) )
self.assertEqual(consume_file_args[1].title, None) self.assertEqual(consume_file_args[1].title, None)
# No metadata_document_id, delete_originals False, so ASN should be None self.assertTrue(consume_file_args[1].skip_asn)
self.assertIsNone(consume_file_args[1].asn)
# With metadata_document_id overrides # With metadata_document_id overrides
result = bulk_edit.merge(doc_ids, metadata_document_id=metadata_document_id) result = bulk_edit.merge(doc_ids, metadata_document_id=metadata_document_id)
consume_file_args, _ = mock_consume_file.call_args consume_file_args, _ = mock_consume_file.call_args
self.assertEqual(consume_file_args[1].title, "B (merged)") self.assertEqual(consume_file_args[1].title, "B (merged)")
self.assertEqual(consume_file_args[1].created, self.doc2.created) self.assertEqual(consume_file_args[1].created, self.doc2.created)
self.assertTrue(consume_file_args[1].skip_asn)
self.assertEqual(result, "OK") self.assertEqual(result, "OK")
@mock.patch("documents.bulk_edit.delete.si") @mock.patch("documents.bulk_edit.delete.si")
@mock.patch("documents.tasks.consume_file.s") @mock.patch("documents.tasks.consume_file.s")
@mock.patch("documents.bulk_edit.chain")
def test_merge_and_delete_originals( def test_merge_and_delete_originals(
self, self,
mock_chain,
mock_consume_file, mock_consume_file,
mock_delete_documents, mock_delete_documents,
): ):
@@ -631,12 +633,6 @@ class TestPDFActions(DirectoriesMixin, TestCase):
- Document deletion task should be called - Document deletion task should be called
""" """
doc_ids = [self.doc1.id, self.doc2.id, self.doc3.id] doc_ids = [self.doc1.id, self.doc2.id, self.doc3.id]
self.doc1.archive_serial_number = 101
self.doc2.archive_serial_number = 102
self.doc3.archive_serial_number = 103
self.doc1.save()
self.doc2.save()
self.doc3.save()
result = bulk_edit.merge(doc_ids, delete_originals=True) result = bulk_edit.merge(doc_ids, delete_originals=True)
self.assertEqual(result, "OK") self.assertEqual(result, "OK")
@@ -647,8 +643,7 @@ class TestPDFActions(DirectoriesMixin, TestCase):
mock_consume_file.assert_called() mock_consume_file.assert_called()
mock_delete_documents.assert_called() mock_delete_documents.assert_called()
consume_sig = mock_consume_file.return_value mock_chain.assert_called_once()
consume_sig.apply_async.assert_called_once()
consume_file_args, _ = mock_consume_file.call_args consume_file_args, _ = mock_consume_file.call_args
self.assertEqual( self.assertEqual(
@@ -656,7 +651,7 @@ class TestPDFActions(DirectoriesMixin, TestCase):
expected_filename, expected_filename,
) )
self.assertEqual(consume_file_args[1].title, None) self.assertEqual(consume_file_args[1].title, None)
self.assertEqual(consume_file_args[1].asn, 101) self.assertTrue(consume_file_args[1].skip_asn)
delete_documents_args, _ = mock_delete_documents.call_args delete_documents_args, _ = mock_delete_documents.call_args
self.assertEqual( self.assertEqual(
@@ -664,92 +659,6 @@ class TestPDFActions(DirectoriesMixin, TestCase):
doc_ids, doc_ids,
) )
self.doc1.refresh_from_db()
self.doc2.refresh_from_db()
self.doc3.refresh_from_db()
self.assertIsNone(self.doc1.archive_serial_number)
self.assertIsNone(self.doc2.archive_serial_number)
self.assertIsNone(self.doc3.archive_serial_number)
@mock.patch("documents.bulk_edit.delete.si")
@mock.patch("documents.tasks.consume_file.s")
def test_merge_and_delete_originals_restore_on_failure(
self,
mock_consume_file,
mock_delete_documents,
):
"""
GIVEN:
- Existing documents
WHEN:
- Merge action with deleting documents is called with 1 document
- Error occurs when queuing consume file task
THEN:
- Archive serial numbers are restored
"""
doc_ids = [self.doc1.id]
self.doc1.archive_serial_number = 111
self.doc1.save()
sig = mock.Mock()
sig.apply_async.side_effect = Exception("boom")
mock_consume_file.return_value = sig
with self.assertRaises(Exception):
bulk_edit.merge(doc_ids, delete_originals=True)
self.doc1.refresh_from_db()
self.assertEqual(self.doc1.archive_serial_number, 111)
@mock.patch("documents.bulk_edit.delete.si")
@mock.patch("documents.tasks.consume_file.s")
def test_merge_and_delete_originals_metadata_handoff(
self,
mock_consume_file,
mock_delete_documents,
):
"""
GIVEN:
- Existing documents with ASNs
WHEN:
- Merge with delete_originals=True and metadata_document_id set
THEN:
- Handoff ASN uses metadata document ASN
"""
doc_ids = [self.doc1.id, self.doc2.id]
self.doc1.archive_serial_number = 101
self.doc2.archive_serial_number = 202
self.doc1.save()
self.doc2.save()
result = bulk_edit.merge(
doc_ids,
metadata_document_id=self.doc2.id,
delete_originals=True,
)
self.assertEqual(result, "OK")
consume_file_args, _ = mock_consume_file.call_args
self.assertEqual(consume_file_args[1].asn, 202)
def test_restore_archive_serial_numbers_task(self):
"""
GIVEN:
- Existing document with no archive serial number
WHEN:
- Restore archive serial number task is called with backup data
THEN:
- Document archive serial number is restored
"""
self.doc1.archive_serial_number = 444
self.doc1.save()
Document.objects.filter(pk=self.doc1.id).update(archive_serial_number=None)
backup = {self.doc1.id: 444}
bulk_edit.restore_archive_serial_numbers_task(backup)
self.doc1.refresh_from_db()
self.assertEqual(self.doc1.archive_serial_number, 444)
@mock.patch("documents.tasks.consume_file.s") @mock.patch("documents.tasks.consume_file.s")
def test_merge_with_archive_fallback(self, mock_consume_file): def test_merge_with_archive_fallback(self, mock_consume_file):
""" """
@@ -818,7 +727,6 @@ class TestPDFActions(DirectoriesMixin, TestCase):
self.assertEqual(mock_consume_file.call_count, 2) self.assertEqual(mock_consume_file.call_count, 2)
consume_file_args, _ = mock_consume_file.call_args consume_file_args, _ = mock_consume_file.call_args
self.assertEqual(consume_file_args[1].title, "B (split 2)") self.assertEqual(consume_file_args[1].title, "B (split 2)")
self.assertIsNone(consume_file_args[1].asn)
self.assertEqual(result, "OK") self.assertEqual(result, "OK")
@@ -843,8 +751,6 @@ class TestPDFActions(DirectoriesMixin, TestCase):
""" """
doc_ids = [self.doc2.id] doc_ids = [self.doc2.id]
pages = [[1, 2], [3]] pages = [[1, 2], [3]]
self.doc2.archive_serial_number = 200
self.doc2.save()
result = bulk_edit.split(doc_ids, pages, delete_originals=True) result = bulk_edit.split(doc_ids, pages, delete_originals=True)
self.assertEqual(result, "OK") self.assertEqual(result, "OK")
@@ -862,42 +768,6 @@ class TestPDFActions(DirectoriesMixin, TestCase):
doc_ids, doc_ids,
) )
self.doc2.refresh_from_db()
self.assertIsNone(self.doc2.archive_serial_number)
@mock.patch("documents.bulk_edit.delete.si")
@mock.patch("documents.tasks.consume_file.s")
@mock.patch("documents.bulk_edit.chord")
def test_split_restore_on_failure(
self,
mock_chord,
mock_consume_file,
mock_delete_documents,
):
"""
GIVEN:
- Existing documents
WHEN:
- Split action with deleting documents is called with 1 document and 2 page groups
- Error occurs when queuing chord task
THEN:
- Archive serial numbers are restored
"""
doc_ids = [self.doc2.id]
pages = [[1, 2]]
self.doc2.archive_serial_number = 222
self.doc2.save()
sig = mock.Mock()
sig.apply_async.side_effect = Exception("boom")
mock_chord.return_value = sig
result = bulk_edit.split(doc_ids, pages, delete_originals=True)
self.assertEqual(result, "OK")
self.doc2.refresh_from_db()
self.assertEqual(self.doc2.archive_serial_number, 222)
@mock.patch("documents.tasks.consume_file.delay") @mock.patch("documents.tasks.consume_file.delay")
@mock.patch("pikepdf.Pdf.save") @mock.patch("pikepdf.Pdf.save")
def test_split_with_errors(self, mock_save_pdf, mock_consume_file): def test_split_with_errors(self, mock_save_pdf, mock_consume_file):
@@ -1098,49 +968,10 @@ class TestPDFActions(DirectoriesMixin, TestCase):
mock_chord.return_value.delay.return_value = None mock_chord.return_value.delay.return_value = None
doc_ids = [self.doc2.id] doc_ids = [self.doc2.id]
operations = [{"page": 1}, {"page": 2}] operations = [{"page": 1}, {"page": 2}]
self.doc2.archive_serial_number = 250
self.doc2.save()
result = bulk_edit.edit_pdf(doc_ids, operations, delete_original=True) result = bulk_edit.edit_pdf(doc_ids, operations, delete_original=True)
self.assertEqual(result, "OK") self.assertEqual(result, "OK")
mock_chord.assert_called_once() mock_chord.assert_called_once()
consume_file_args, _ = mock_consume_file.call_args
self.assertEqual(consume_file_args[1].asn, 250)
self.doc2.refresh_from_db()
self.assertIsNone(self.doc2.archive_serial_number)
@mock.patch("documents.bulk_edit.delete.si")
@mock.patch("documents.tasks.consume_file.s")
@mock.patch("documents.bulk_edit.chord")
def test_edit_pdf_restore_on_failure(
self,
mock_chord,
mock_consume_file,
mock_delete_documents,
):
"""
GIVEN:
- Existing document
WHEN:
- edit_pdf is called with delete_original=True
- Error occurs when queuing chord task
THEN:
- Archive serial numbers are restored
"""
doc_ids = [self.doc2.id]
operations = [{"page": 1}]
self.doc2.archive_serial_number = 333
self.doc2.save()
sig = mock.Mock()
sig.apply_async.side_effect = Exception("boom")
mock_chord.return_value = sig
with self.assertRaises(Exception):
bulk_edit.edit_pdf(doc_ids, operations, delete_original=True)
self.doc2.refresh_from_db()
self.assertEqual(self.doc2.archive_serial_number, 333)
@mock.patch("documents.tasks.update_document_content_maybe_archive_file.delay") @mock.patch("documents.tasks.update_document_content_maybe_archive_file.delay")
def test_edit_pdf_with_update_document(self, mock_update_document): def test_edit_pdf_with_update_document(self, mock_update_document):

View File

@@ -14,7 +14,6 @@ from django.test import override_settings
from django.utils import timezone from django.utils import timezone
from guardian.core import ObjectPermissionChecker from guardian.core import ObjectPermissionChecker
from documents.barcodes import BarcodePlugin
from documents.consumer import ConsumerError from documents.consumer import ConsumerError
from documents.data_models import DocumentMetadataOverrides from documents.data_models import DocumentMetadataOverrides
from documents.data_models import DocumentSource from documents.data_models import DocumentSource
@@ -413,6 +412,14 @@ class TestConsumer(
self.assertEqual(document.archive_serial_number, 123) self.assertEqual(document.archive_serial_number, 123)
self._assert_first_last_send_progress() self._assert_first_last_send_progress()
def testMetadataOverridesSkipAsnPropagation(self):
overrides = DocumentMetadataOverrides()
incoming = DocumentMetadataOverrides(skip_asn=True)
overrides.update(incoming)
self.assertTrue(overrides.skip_asn)
def testOverrideTitlePlaceholders(self): def testOverrideTitlePlaceholders(self):
c = Correspondent.objects.create(name="Correspondent Name") c = Correspondent.objects.create(name="Correspondent Name")
dt = DocumentType.objects.create(name="DocType Name") dt = DocumentType.objects.create(name="DocType Name")
@@ -1233,46 +1240,3 @@ class PostConsumeTestCase(DirectoriesMixin, GetConsumerMixin, TestCase):
r"sample\.pdf: Error while executing post-consume script: Command '\[.*\]' returned non-zero exit status \d+\.", r"sample\.pdf: Error while executing post-consume script: Command '\[.*\]' returned non-zero exit status \d+\.",
): ):
consumer.run_post_consume_script(doc) consumer.run_post_consume_script(doc)
class TestMetadataOverrides(TestCase):
def test_update_skip_asn_if_exists(self):
base = DocumentMetadataOverrides()
incoming = DocumentMetadataOverrides(skip_asn_if_exists=True)
base.update(incoming)
self.assertTrue(base.skip_asn_if_exists)
class TestBarcodeApplyDetectedASN(TestCase):
"""
GIVEN:
- Existing Documents with ASN 123
WHEN:
- A BarcodePlugin which detected an ASN
THEN:
- If skip_asn_if_exists is set, and ASN exists, do not set ASN
- If skip_asn_if_exists is set, and ASN does not exist, set ASN
"""
def test_apply_detected_asn_skips_existing_when_flag_set(self):
doc = Document.objects.create(
checksum="X1",
title="D1",
archive_serial_number=123,
)
metadata = DocumentMetadataOverrides(skip_asn_if_exists=True)
plugin = BarcodePlugin(
input_doc=mock.Mock(),
metadata=metadata,
status_mgr=mock.Mock(),
base_tmp_dir=tempfile.gettempdir(),
task_id="test-task",
)
plugin._apply_detected_asn(123)
self.assertIsNone(plugin.metadata.asn)
doc.hard_delete()
plugin._apply_detected_asn(123)
self.assertEqual(plugin.metadata.asn, 123)

View File

@@ -1276,6 +1276,76 @@ class TestWorkflows(
) )
self.assertIn(expected_str, cm.output[1]) self.assertIn(expected_str, cm.output[1])
def test_document_added_any_filters(self):
trigger = WorkflowTrigger.objects.create(
type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED,
)
trigger.filter_has_any_correspondents.set([self.c])
trigger.filter_has_any_document_types.set([self.dt])
trigger.filter_has_any_storage_paths.set([self.sp])
matching_doc = Document.objects.create(
title="sample test",
correspondent=self.c,
document_type=self.dt,
storage_path=self.sp,
original_filename="sample.pdf",
checksum="checksum-any-match",
)
matched, reason = existing_document_matches_workflow(matching_doc, trigger)
self.assertTrue(matched)
self.assertIsNone(reason)
wrong_correspondent = Document.objects.create(
title="wrong correspondent",
correspondent=self.c2,
document_type=self.dt,
storage_path=self.sp,
original_filename="sample2.pdf",
)
matched, reason = existing_document_matches_workflow(
wrong_correspondent,
trigger,
)
self.assertFalse(matched)
self.assertIn("correspondent", reason)
other_document_type = DocumentType.objects.create(name="Other")
wrong_document_type = Document.objects.create(
title="wrong doc type",
correspondent=self.c,
document_type=other_document_type,
storage_path=self.sp,
original_filename="sample3.pdf",
checksum="checksum-wrong-doc-type",
)
matched, reason = existing_document_matches_workflow(
wrong_document_type,
trigger,
)
self.assertFalse(matched)
self.assertIn("doc type", reason)
other_storage_path = StoragePath.objects.create(
name="Other path",
path="/other/",
)
wrong_storage_path = Document.objects.create(
title="wrong storage",
correspondent=self.c,
document_type=self.dt,
storage_path=other_storage_path,
original_filename="sample4.pdf",
checksum="checksum-wrong-storage-path",
)
matched, reason = existing_document_matches_workflow(
wrong_storage_path,
trigger,
)
self.assertFalse(matched)
self.assertIn("storage path", reason)
def test_document_added_custom_field_query_no_match(self): def test_document_added_custom_field_query_no_match(self):
trigger = WorkflowTrigger.objects.create( trigger = WorkflowTrigger.objects.create(
type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED, type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED,
@@ -1384,6 +1454,39 @@ class TestWorkflows(
self.assertIn(doc1, filtered) self.assertIn(doc1, filtered)
self.assertNotIn(doc2, filtered) self.assertNotIn(doc2, filtered)
def test_prefilter_documents_any_filters(self):
trigger = WorkflowTrigger.objects.create(
type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED,
)
trigger.filter_has_any_correspondents.set([self.c])
trigger.filter_has_any_document_types.set([self.dt])
trigger.filter_has_any_storage_paths.set([self.sp])
allowed_document = Document.objects.create(
title="allowed",
correspondent=self.c,
document_type=self.dt,
storage_path=self.sp,
original_filename="doc-allowed.pdf",
checksum="checksum-any-allowed",
)
blocked_document = Document.objects.create(
title="blocked",
correspondent=self.c2,
document_type=self.dt,
storage_path=self.sp,
original_filename="doc-blocked.pdf",
checksum="checksum-any-blocked",
)
filtered = prefilter_documents_by_workflowtrigger(
Document.objects.all(),
trigger,
)
self.assertIn(allowed_document, filtered)
self.assertNotIn(blocked_document, filtered)
def test_consumption_trigger_requires_filter_configuration(self): def test_consumption_trigger_requires_filter_configuration(self):
serializer = WorkflowTriggerSerializer( serializer = WorkflowTriggerSerializer(
data={ data={