Compare commits

..

7 Commits

Author SHA1 Message Date
shamoon
bd7752e4d6 Happy Sonar 2026-01-29 11:47:21 -08:00
shamoon
fbcaaf2e9b Also in the tests 2026-01-29 11:45:14 -08:00
shamoon
2229479499 Refactor to separate AsnCheckPlugin 2026-01-29 11:32:13 -08:00
shamoon
7a044763dd Merge branch 'dev' into fix-11679 2026-01-29 11:25:07 -08:00
shamoon
f23433bd57 Merge branch 'dev' into fix-11679 2026-01-09 20:45:38 -08:00
shamoon
c968505f64 Fix: run pre-flight ASN check after barcode detection 2025-12-30 10:02:32 -08:00
shamoon
fbb5864757 Add test to reproduce asn in trash error 2025-12-30 10:02:31 -08:00
15 changed files with 80 additions and 479 deletions

View File

@@ -430,24 +430,6 @@
</div>
</div>
}
@case (WorkflowActionType.PasswordRemoval) {
<div class="row">
<div class="col">
<p class="small" i18n>
One password per line. The workflow will try them in order until one succeeds.
</p>
<pngx-input-textarea
i18n-title
title="Passwords"
formControlName="passwords"
rows="4"
[error]="error?.actions?.[i]?.passwords"
hint="Passwords are stored in plain text. Use with caution."
i18n-hint
></pngx-input-textarea>
</div>
</div>
}
}
</div>
</ng-template>

View File

@@ -3,7 +3,6 @@ import { provideHttpClient, withInterceptorsFromDi } from '@angular/common/http'
import { provideHttpClientTesting } from '@angular/common/http/testing'
import { ComponentFixture, TestBed } from '@angular/core/testing'
import {
FormArray,
FormControl,
FormGroup,
FormsModule,
@@ -995,32 +994,4 @@ describe('WorkflowEditDialogComponent', () => {
component.removeSelectedCustomField(3, formGroup)
expect(formGroup.get('assign_custom_fields').value).toEqual([])
})
it('should handle parsing of passwords from array to string and back on save', () => {
const passwordAction: WorkflowAction = {
id: 1,
type: WorkflowActionType.PasswordRemoval,
passwords: ['pass1', 'pass2'],
}
component.object = {
name: 'Workflow with Passwords',
id: 1,
order: 1,
enabled: true,
triggers: [],
actions: [passwordAction],
}
component.ngOnInit()
const formActions = component.objectForm.get('actions') as FormArray
expect(formActions.value[0].passwords).toBe('pass1\npass2')
formActions.at(0).get('passwords').setValue('pass1\npass2\npass3')
component.save()
expect(component.objectForm.get('actions').value[0].passwords).toEqual([
'pass1',
'pass2',
'pass3',
])
})
})

View File

@@ -139,10 +139,6 @@ export const WORKFLOW_ACTION_OPTIONS = [
id: WorkflowActionType.Webhook,
name: $localize`Webhook`,
},
{
id: WorkflowActionType.PasswordRemoval,
name: $localize`Password removal`,
},
]
export enum TriggerFilterType {
@@ -1206,25 +1202,11 @@ export class WorkflowEditDialogComponent
headers: new FormControl(action.webhook?.headers),
include_document: new FormControl(!!action.webhook?.include_document),
}),
passwords: new FormControl(
this.formatPasswords(action.passwords ?? [])
),
}),
{ emitEvent }
)
}
private formatPasswords(passwords: string[] = []): string {
return passwords.join('\n')
}
private parsePasswords(value: string = ''): string[] {
return value
.split(/[\n,]+/)
.map((entry) => entry.trim())
.filter((entry) => entry.length > 0)
}
private updateAllTriggerActionFields(emitEvent: boolean = false) {
this.triggerFields.clear({ emitEvent: false })
this.object?.triggers.forEach((trigger) => {
@@ -1349,7 +1331,6 @@ export class WorkflowEditDialogComponent
headers: null,
include_document: false,
},
passwords: [],
}
this.object.actions.push(action)
this.createActionField(action)
@@ -1386,7 +1367,6 @@ export class WorkflowEditDialogComponent
if (action.type !== WorkflowActionType.Email) {
action.email = null
}
action.passwords = this.parsePasswords(action.passwords as any)
})
super.save()
}

View File

@@ -5,7 +5,6 @@ export enum WorkflowActionType {
Removal = 2,
Email = 3,
Webhook = 4,
PasswordRemoval = 5,
}
export interface WorkflowActionEmail extends ObjectWithId {
@@ -98,6 +97,4 @@ export interface WorkflowAction extends ObjectWithId {
email?: WorkflowActionEmail
webhook?: WorkflowActionWebhook
passwords?: string[]
}

View File

@@ -5,6 +5,7 @@ import tempfile
from enum import Enum
from pathlib import Path
from typing import TYPE_CHECKING
from typing import Final
import magic
from django.conf import settings
@@ -49,6 +50,8 @@ from documents.utils import copy_file_with_basic_stats
from documents.utils import run_subprocess
from paperless_mail.parsers import MailDocumentParser
LOGGING_NAME: Final[str] = "paperless.consumer"
class WorkflowTriggerPlugin(
NoCleanupPluginMixin,
@@ -156,7 +159,7 @@ class ConsumerPlugin(
ConsumerPluginMixin,
ConsumeTaskPlugin,
):
logging_name = "paperless.consumer"
logging_name = LOGGING_NAME
def run_pre_consume_script(self):
"""
@@ -753,7 +756,7 @@ class ConsumerPreflightPlugin(
ConsumeTaskPlugin,
):
NAME: str = "ConsumerPreflightPlugin"
logging_name = "paperless.consumer"
logging_name = LOGGING_NAME
def pre_check_file_exists(self):
"""
@@ -828,6 +831,32 @@ class ConsumerPreflightPlugin(
settings.ORIGINALS_DIR.mkdir(parents=True, exist_ok=True)
settings.ARCHIVE_DIR.mkdir(parents=True, exist_ok=True)
def run(self) -> None:
self._send_progress(
0,
100,
ProgressStatusOptions.STARTED,
ConsumerStatusShortMessage.NEW_FILE,
)
# Make sure that preconditions for consuming the file are met.
self.pre_check_file_exists()
self.pre_check_duplicate()
self.pre_check_directories()
class AsnCheckPlugin(
NoCleanupPluginMixin,
NoSetupPluginMixin,
AlwaysRunPluginMixin,
LoggingMixin,
ConsumerPluginMixin,
ConsumeTaskPlugin,
):
NAME: str = "AsnCheckPlugin"
logging_name = LOGGING_NAME
def pre_check_asn_value(self):
"""
Check that if override_asn is given, it is unique and within a valid range
@@ -865,16 +894,4 @@ class ConsumerPreflightPlugin(
)
def run(self) -> None:
self._send_progress(
0,
100,
ProgressStatusOptions.STARTED,
ConsumerStatusShortMessage.NEW_FILE,
)
# Make sure that preconditions for consuming the file are met.
self.pre_check_file_exists()
self.pre_check_duplicate()
self.pre_check_directories()
self.pre_check_asn_value()

View File

@@ -1,38 +0,0 @@
# Generated by Django 5.2.7 on 2025-12-29 03:56
from django.db import migrations
from django.db import models
class Migration(migrations.Migration):
dependencies = [
("documents", "0008_sharelinkbundle"),
]
operations = [
migrations.AddField(
model_name="workflowaction",
name="passwords",
field=models.JSONField(
blank=True,
help_text="Passwords to try when removing PDF protection. Separate with commas or new lines.",
null=True,
verbose_name="passwords",
),
),
migrations.AlterField(
model_name="workflowaction",
name="type",
field=models.PositiveIntegerField(
choices=[
(1, "Assignment"),
(2, "Removal"),
(3, "Email"),
(4, "Webhook"),
(5, "Password removal"),
],
default=1,
verbose_name="Workflow Action Type",
),
),
]

View File

@@ -1405,10 +1405,6 @@ class WorkflowAction(models.Model):
4,
_("Webhook"),
)
PASSWORD_REMOVAL = (
5,
_("Password removal"),
)
type = models.PositiveIntegerField(
_("Workflow Action Type"),
@@ -1638,15 +1634,6 @@ class WorkflowAction(models.Model):
verbose_name=_("webhook"),
)
passwords = models.JSONField(
_("passwords"),
null=True,
blank=True,
help_text=_(
"Passwords to try when removing PDF protection. Separate with commas or new lines.",
),
)
class Meta:
verbose_name = _("workflow action")
verbose_name_plural = _("workflow actions")

View File

@@ -2613,7 +2613,6 @@ class WorkflowActionSerializer(serializers.ModelSerializer):
"remove_change_groups",
"email",
"webhook",
"passwords",
]
def validate(self, attrs):
@@ -2670,23 +2669,6 @@ class WorkflowActionSerializer(serializers.ModelSerializer):
"Webhook data is required for webhook actions",
)
if (
"type" in attrs
and attrs["type"] == WorkflowAction.WorkflowActionType.PASSWORD_REMOVAL
):
passwords = attrs.get("passwords")
# ensure passwords is a non-empty list of non-empty strings
if (
passwords is None
or not isinstance(passwords, list)
or len(passwords) == 0
or any(not isinstance(pw, str) for pw in passwords)
or any(len(pw.strip()) == 0 for pw in passwords)
):
raise serializers.ValidationError(
"Passwords are required for password removal actions",
)
return attrs

View File

@@ -48,7 +48,6 @@ from documents.permissions import get_objects_for_user_owner_aware
from documents.templating.utils import convert_format_str_to_template_format
from documents.workflows.actions import build_workflow_action_context
from documents.workflows.actions import execute_email_action
from documents.workflows.actions import execute_password_removal_action
from documents.workflows.actions import execute_webhook_action
from documents.workflows.mutations import apply_assignment_to_document
from documents.workflows.mutations import apply_assignment_to_overrides
@@ -823,8 +822,6 @@ def run_workflows(
logging_group,
original_file,
)
elif action.type == WorkflowAction.WorkflowActionType.PASSWORD_REMOVAL:
execute_password_removal_action(action, document, logging_group)
if not use_overrides:
# limit title to 128 characters

View File

@@ -29,6 +29,7 @@ from documents.bulk_download import OriginalsOnlyStrategy
from documents.caching import clear_document_caches
from documents.classifier import DocumentClassifier
from documents.classifier import load_classifier
from documents.consumer import AsnCheckPlugin
from documents.consumer import ConsumerPlugin
from documents.consumer import ConsumerPreflightPlugin
from documents.consumer import WorkflowTriggerPlugin
@@ -157,8 +158,10 @@ def consume_file(
plugins: list[type[ConsumeTaskPlugin]] = [
ConsumerPreflightPlugin,
AsnCheckPlugin,
CollatePlugin,
BarcodePlugin,
AsnCheckPlugin, # Re-run ASN check after barcode reading
WorkflowTriggerPlugin,
ConsumerPlugin,
]

View File

@@ -838,61 +838,3 @@ class TestApiWorkflows(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.action.refresh_from_db()
self.assertEqual(self.action.assign_title, "Patched Title")
def test_password_action_passwords_field(self):
"""
GIVEN:
- Nothing
WHEN:
- A workflow password removal action is created with passwords set
THEN:
- The passwords field is correctly stored and retrieved
"""
passwords = ["password1", "password2", "password3"]
response = self.client.post(
"/api/workflow_actions/",
json.dumps(
{
"type": WorkflowAction.WorkflowActionType.PASSWORD_REMOVAL,
"passwords": passwords,
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
self.assertEqual(response.data["passwords"], passwords)
def test_password_action_invalid_passwords_field(self):
"""
GIVEN:
- Nothing
WHEN:
- A workflow password removal action is created with invalid passwords field
THEN:
- The required validation error is raised
"""
for payload in [
{"type": WorkflowAction.WorkflowActionType.PASSWORD_REMOVAL},
{
"type": WorkflowAction.WorkflowActionType.PASSWORD_REMOVAL,
"passwords": "",
},
{
"type": WorkflowAction.WorkflowActionType.PASSWORD_REMOVAL,
"passwords": [],
},
{
"type": WorkflowAction.WorkflowActionType.PASSWORD_REMOVAL,
"passwords": ["", "password2"],
},
]:
response = self.client.post(
"/api/workflow_actions/",
json.dumps(payload),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn(
"Passwords are required",
str(response.data["non_field_errors"][0]),
)

View File

@@ -11,6 +11,7 @@ from django.test import override_settings
from documents import tasks
from documents.barcodes import BarcodePlugin
from documents.consumer import ConsumerError
from documents.data_models import ConsumableDocument
from documents.data_models import DocumentMetadataOverrides
from documents.data_models import DocumentSource
@@ -93,6 +94,41 @@ class TestBarcode(
self.assertDictEqual(separator_page_numbers, {1: False})
@override_settings(CONSUMER_ENABLE_ASN_BARCODE=True)
def test_asn_barcode_duplicate_in_trash_fails(self):
"""
GIVEN:
- A document with ASN barcode 123 is in the trash
WHEN:
- A file with the same barcode ASN is consumed
THEN:
- The ASN check is re-run and consumption fails
"""
test_file = self.BARCODE_SAMPLE_DIR / "barcode-39-asn-123.pdf"
first_doc = Document.objects.create(
title="First ASN 123",
content="",
checksum="asn123first",
mime_type="application/pdf",
archive_serial_number=123,
)
first_doc.delete()
dupe_asn = settings.SCRATCH_DIR / "barcode-39-asn-123-second.pdf"
shutil.copy(test_file, dupe_asn)
with mock.patch("documents.tasks.ProgressManager", DummyProgressManager):
with self.assertRaisesRegex(ConsumerError, r"ASN 123.*trash"):
tasks.consume_file(
ConsumableDocument(
source=DocumentSource.ConsumeFolder,
original_file=dupe_asn,
),
None,
)
@override_settings(
CONSUMER_BARCODE_TIFF_SUPPORT=True,
)

View File

@@ -2,7 +2,6 @@ import datetime
import json
import shutil
import socket
import tempfile
from datetime import timedelta
from pathlib import Path
from typing import TYPE_CHECKING
@@ -61,7 +60,6 @@ from documents.tests.utils import DirectoriesMixin
from documents.tests.utils import DummyProgressManager
from documents.tests.utils import FileSystemAssertsMixin
from documents.tests.utils import SampleDirMixin
from documents.workflows.actions import execute_password_removal_action
from paperless_mail.models import MailAccount
from paperless_mail.models import MailRule
@@ -3718,196 +3716,6 @@ class TestWorkflows(
mock_post.assert_called_once()
@mock.patch("documents.bulk_edit.remove_password")
def test_password_removal_action_attempts_multiple_passwords(
self,
mock_remove_password,
):
"""
GIVEN:
- Workflow password removal action
- Multiple passwords provided
WHEN:
- Document updated triggering the workflow
THEN:
- Password removal is attempted until one succeeds
"""
doc = Document.objects.create(
title="Protected",
checksum="pw-checksum",
)
trigger = WorkflowTrigger.objects.create(
type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_UPDATED,
)
action = WorkflowAction.objects.create(
type=WorkflowAction.WorkflowActionType.PASSWORD_REMOVAL,
passwords="wrong, right\n extra ",
)
workflow = Workflow.objects.create(name="Password workflow")
workflow.triggers.add(trigger)
workflow.actions.add(action)
mock_remove_password.side_effect = [
ValueError("wrong password"),
"OK",
]
run_workflows(trigger.type, doc)
assert mock_remove_password.call_count == 2
mock_remove_password.assert_has_calls(
[
mock.call(
[doc.id],
password="wrong",
update_document=True,
user=doc.owner,
),
mock.call(
[doc.id],
password="right",
update_document=True,
user=doc.owner,
),
],
)
@mock.patch("documents.bulk_edit.remove_password")
def test_password_removal_action_fails_without_correct_password(
self,
mock_remove_password,
):
"""
GIVEN:
- Workflow password removal action
- No correct password provided
WHEN:
- Document updated triggering the workflow
THEN:
- Password removal is attempted for all passwords and fails
"""
doc = Document.objects.create(
title="Protected",
checksum="pw-checksum-2",
)
trigger = WorkflowTrigger.objects.create(
type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_UPDATED,
)
action = WorkflowAction.objects.create(
type=WorkflowAction.WorkflowActionType.PASSWORD_REMOVAL,
passwords=" \n , ",
)
workflow = Workflow.objects.create(name="Password workflow missing passwords")
workflow.triggers.add(trigger)
workflow.actions.add(action)
run_workflows(trigger.type, doc)
mock_remove_password.assert_not_called()
@mock.patch("documents.bulk_edit.remove_password")
def test_password_removal_action_skips_without_passwords(
self,
mock_remove_password,
):
"""
GIVEN:
- Workflow password removal action with no passwords
WHEN:
- Workflow is run
THEN:
- Password removal is not attempted
"""
doc = Document.objects.create(
title="Protected",
checksum="pw-checksum-2",
)
trigger = WorkflowTrigger.objects.create(
type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_UPDATED,
)
action = WorkflowAction.objects.create(
type=WorkflowAction.WorkflowActionType.PASSWORD_REMOVAL,
passwords="",
)
workflow = Workflow.objects.create(name="Password workflow missing passwords")
workflow.triggers.add(trigger)
workflow.actions.add(action)
run_workflows(trigger.type, doc)
mock_remove_password.assert_not_called()
@mock.patch("documents.bulk_edit.remove_password")
def test_password_removal_consumable_document_deferred(
self,
mock_remove_password,
):
"""
GIVEN:
- Workflow password removal action
- Simulated consumption trigger (a ConsumableDocument is used)
WHEN:
- Document consumption is finished
THEN:
- Password removal is attempted
"""
action = WorkflowAction.objects.create(
type=WorkflowAction.WorkflowActionType.PASSWORD_REMOVAL,
passwords="first, second",
)
temp_dir = Path(tempfile.mkdtemp())
original_file = temp_dir / "file.pdf"
original_file.write_bytes(b"pdf content")
consumable = ConsumableDocument(
source=DocumentSource.ApiUpload,
original_file=original_file,
)
execute_password_removal_action(action, consumable, logging_group=None)
mock_remove_password.assert_not_called()
mock_remove_password.side_effect = [
ValueError("bad password"),
"OK",
]
doc = Document.objects.create(
checksum="pw-checksum-consumed",
title="Protected",
)
document_consumption_finished.send(
sender=self.__class__,
document=doc,
)
assert mock_remove_password.call_count == 2
mock_remove_password.assert_has_calls(
[
mock.call(
[doc.id],
password="first",
update_document=True,
user=doc.owner,
),
mock.call(
[doc.id],
password="second",
update_document=True,
user=doc.owner,
),
],
)
# ensure handler disconnected after first run
document_consumption_finished.send(
sender=self.__class__,
document=doc,
)
assert mock_remove_password.call_count == 2
class TestWebhookSend:
def test_send_webhook_data_or_json(

View File

@@ -20,6 +20,7 @@ from django.db.migrations.executor import MigrationExecutor
from django.test import TransactionTestCase
from django.test import override_settings
from documents.consumer import AsnCheckPlugin
from documents.consumer import ConsumerPlugin
from documents.consumer import ConsumerPreflightPlugin
from documents.data_models import ConsumableDocument
@@ -371,6 +372,14 @@ class GetConsumerMixin:
"task-id",
)
preflight_plugin.setup()
asncheck_plugin = AsnCheckPlugin(
doc,
overrides or DocumentMetadataOverrides(),
self.status, # type: ignore
self.dirs.scratch_dir,
"task-id",
)
asncheck_plugin.setup()
reader = ConsumerPlugin(
doc,
overrides or DocumentMetadataOverrides(),
@@ -381,6 +390,7 @@ class GetConsumerMixin:
reader.setup()
try:
preflight_plugin.run()
asncheck_plugin.run()
yield reader
finally:
reader.cleanup()

View File

@@ -1,5 +1,4 @@
import logging
import re
from pathlib import Path
from django.conf import settings
@@ -15,7 +14,6 @@ from documents.models import Document
from documents.models import DocumentType
from documents.models import WorkflowAction
from documents.models import WorkflowTrigger
from documents.signals import document_consumption_finished
from documents.templating.workflows import parse_w_workflow_placeholders
from documents.workflows.webhooks import send_webhook
@@ -267,74 +265,3 @@ def execute_webhook_action(
f"Error occurred sending webhook: {e}",
extra={"group": logging_group},
)
def execute_password_removal_action(
action: WorkflowAction,
document: Document | ConsumableDocument,
logging_group,
) -> None:
"""
Try to remove a password from a document using the configured list.
"""
passwords = action.passwords
if not passwords:
logger.warning(
"Password removal action %s has no passwords configured",
action.pk,
extra={"group": logging_group},
)
return
passwords = [
password.strip()
for password in re.split(r"[,\n]", passwords)
if password.strip()
]
if isinstance(document, ConsumableDocument):
# hook the consumption-finished signal to attempt password removal later
def handler(sender, **kwargs):
consumed_document: Document = kwargs.get("document")
if consumed_document is not None:
execute_password_removal_action(
action,
consumed_document,
logging_group,
)
document_consumption_finished.disconnect(handler)
document_consumption_finished.connect(handler, weak=False)
return
# import here to avoid circular dependency
from documents.bulk_edit import remove_password
for password in passwords:
try:
remove_password(
[document.id],
password=password,
update_document=True,
user=document.owner,
)
logger.info(
"Removed password from document %s using workflow action %s",
document.pk,
action.pk,
extra={"group": logging_group},
)
return
except ValueError as e:
logger.warning(
"Password removal failed for document %s with supplied password: %s",
document.pk,
e,
extra={"group": logging_group},
)
logger.error(
"Password removal failed for document %s after trying all provided passwords",
document.pk,
extra={"group": logging_group},
)