paradizelost/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2025-04-02 13:45:10 -05:00
Code Issues Packages Projects Releases Wiki Activity

Enhancement: Allow excluding mail attachments by name (#4691)

Browse Source 
* Adds new filtering to exclude attachments from processing

* Frontend use include / exclude mail rule filename filters

---------

Co-authored-by: shamoon <4887959+shamoon@users.noreply.github.com>
This commit is contained in:
Trenton H 2023-12-02 08:26:19 -08:00 committed by GitHub
parent 1b69b89d2d
commit 6e371ac5ac
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 189 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
src-ui/messages.xlf
View File

@ -1315,7 +1315,7 @@
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html</context>
<context context-type="linenumber">42</context>
<context context-type="linenumber">43</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component.html</context>
@ -2520,7 +2520,7 @@
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html</context>
<context context-type="linenumber">32</context>
<context context-type="linenumber">33</context>
</context-group>
</trans-unit>
<trans-unit id="4875491778188965469" datatype="html">
@ -2531,7 +2531,7 @@
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html</context>
<context context-type="linenumber">34</context>
<context context-type="linenumber">35</context>
</context-group>
</trans-unit>
<trans-unit id="7198346314713788799" datatype="html">
@ -2570,7 +2570,7 @@
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html</context>
<context context-type="linenumber">40</context>
<context context-type="linenumber">41</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/toasts/toasts.component.html</context>
@ -2605,7 +2605,7 @@
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html</context>
<context context-type="linenumber">41</context>
<context context-type="linenumber">42</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component.html</context>
@ -3016,8 +3016,8 @@
<context context-type="linenumber">23</context>
</context-group>
</trans-unit>
<trans-unit id="5031687746498952417" datatype="html">
<source>Filter attachment filename</source>
<trans-unit id="4603548543464136402" datatype="html">
<source>Filter attachment filename includes</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html</context>
<context context-type="linenumber">24</context>
@ -3030,53 +3030,67 @@
<context context-type="linenumber">24</context>
</context-group>
</trans-unit>
<trans-unit id="6869675473865305593" datatype="html">
<source>Filter attachment filename excluding</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html</context>
<context context-type="linenumber">25</context>
</context-group>
</trans-unit>
<trans-unit id="6774472763442688477" datatype="html">
<source>Do not consume documents which entirely match this filename if specified. Wildcards such as *.pdf or *invoice* are allowed. Case insensitive.</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html</context>
<context context-type="linenumber">25</context>
</context-group>
</trans-unit>
<trans-unit id="9216117865911519658" datatype="html">
<source>Action</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html</context>
<context context-type="linenumber">27</context>
<context context-type="linenumber">28</context>
</context-group>
</trans-unit>
<trans-unit id="4274038999388817994" datatype="html">
<source>Action is only performed when documents are consumed from the mail. Mails without attachments remain entirely untouched.</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html</context>
<context context-type="linenumber">27</context>
<context context-type="linenumber">28</context>
</context-group>
</trans-unit>
<trans-unit id="1261794314435932203" datatype="html">
<source>Action parameter</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html</context>
<context context-type="linenumber">28</context>
<context context-type="linenumber">29</context>
</context-group>
</trans-unit>
<trans-unit id="5512171567357420308" datatype="html">
<source>Assignments specified here will supersede any consumption templates.</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html</context>
<context context-type="linenumber">29</context>
<context context-type="linenumber">30</context>
</context-group>
</trans-unit>
<trans-unit id="6093797930511670257" datatype="html">
<source>Assign title from</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html</context>
<context context-type="linenumber">30</context>
<context context-type="linenumber">31</context>
</context-group>
</trans-unit>
<trans-unit id="4754802869258527587" datatype="html">
<source>Assign correspondent from</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html</context>
<context context-type="linenumber">33</context>
<context context-type="linenumber">34</context>
</context-group>
</trans-unit>
<trans-unit id="5232720756589450549" datatype="html">
<source>Assign owner from rule</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html</context>
<context context-type="linenumber">35</context>
<context context-type="linenumber">36</context>
</context-group>
</trans-unit>
<trans-unit id="6886003843406464884" datatype="html">

3
src-ui/src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html
View File

@ -21,7 +21,8 @@
<pngx-input-text i18n-title title="Filter to" formControlName="filter_to" [error]="error?.filter_to"></pngx-input-text>
<pngx-input-text i18n-title title="Filter subject" formControlName="filter_subject" [error]="error?.filter_subject"></pngx-input-text>
<pngx-input-text i18n-title title="Filter body" formControlName="filter_body" [error]="error?.filter_body"></pngx-input-text>
<pngx-input-text i18n-title title="Filter attachment filename" formControlName="filter_attachment_filename" i18n-hint hint="Only consume documents which entirely match this filename if specified. Wildcards such as *.pdf or *invoice* are allowed. Case insensitive." [error]="error?.filter_attachment_filename"></pngx-input-text>
<pngx-input-text i18n-title title="Filter attachment filename includes" formControlName="filter_attachment_filename_include" i18n-hint hint="Only consume documents which entirely match this filename if specified. Wildcards such as *.pdf or *invoice* are allowed. Case insensitive." [error]="error?.filter_attachment_filename_include"></pngx-input-text>
<pngx-input-text i18n-title title="Filter attachment filename excluding" formControlName="filter_attachment_filename_exclude" i18n-hint hint="Do not consume documents which entirely match this filename if specified. Wildcards such as *.pdf or *invoice* are allowed. Case insensitive." [error]="error?.filter_attachment_filename_exclude"></pngx-input-text>
</div>
<div class="col-md-4">
<pngx-input-select i18n-title title="Action" [items]="actionOptions" formControlName="action" i18n-hint hint="Action is only performed when documents are consumed from the mail. Mails without attachments remain entirely untouched."></pngx-input-select>

3
src-ui/src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.ts
View File

@ -158,7 +158,8 @@ export class MailRuleEditDialogComponent extends EditDialogComponent<PaperlessMa
filter_to: new FormControl(null),
filter_subject: new FormControl(null),
filter_body: new FormControl(null),
filter_attachment_filename: new FormControl(null),
filter_attachment_filename_include: new FormControl(null),
filter_attachment_filename_exclude: new FormControl(null),
maximum_age: new FormControl(null),
attachment_type: new FormControl(MailFilterAttachmentType.Attachments),
consumption_scope: new FormControl(MailRuleConsumptionScope.Attachments),

4
src-ui/src/app/data/paperless-mail-rule.ts
View File

@ -49,7 +49,9 @@ export interface PaperlessMailRule extends ObjectWithPermissions {
filter_body: string
filter_attachment_filename: string
filter_attachment_filename_include: string
filter_attachment_filename_exclude: string
maximum_age: number

9
src-ui/src/app/services/rest/mail-rule.service.spec.ts
View File

@ -23,7 +23,8 @@ const mail_rules = [
filter_to: null,
filter_subject: null,
filter_body: null,
filter_attachment_filename: null,
filter_attachment_filename_include: null,
filter_attachment_filename_exclude: null,
maximum_age: 30,
attachment_type: MailFilterAttachmentType.Everything,
action: MailAction.MarkRead,
@ -40,7 +41,8 @@ const mail_rules = [
filter_to: null,
filter_subject: null,
filter_body: null,
filter_attachment_filename: null,
filter_attachment_filename_include: null,
filter_attachment_filename_exclude: null,
maximum_age: 30,
attachment_type: MailFilterAttachmentType.Everything,
action: MailAction.Delete,
@ -57,7 +59,8 @@ const mail_rules = [
filter_to: null,
filter_subject: null,
filter_body: null,
filter_attachment_filename: null,
filter_attachment_filename_include: null,
filter_attachment_filename_exclude: null,
maximum_age: 30,
attachment_type: MailFilterAttachmentType.Everything,
action: MailAction.Flag,

2
src/documents/tests/test_api.py
View File

@ -5769,7 +5769,7 @@ class TestApiConsumptionTemplates(DirectoriesMixin, APITestCase):
filter_to="someone@somewhere.com",
filter_subject="subject",
filter_body="body",
filter_attachment_filename="file.pdf",
filter_attachment_filename_include="file.pdf",
maximum_age=30,
action=MailRule.MailAction.MARK_READ,
assign_title_from=MailRule.TitleSource.FROM_SUBJECT,

2
src/documents/tests/test_consumption_templates.py
View File

@ -54,7 +54,7 @@ class TestConsumptionTemplates(DirectoriesMixin, FileSystemAssertsMixin, TestCas
filter_to="someone@somewhere.com",
filter_subject="subject",
filter_body="body",
filter_attachment_filename="file.pdf",
filter_attachment_filename_include="file.pdf",
maximum_age=30,
action=MailRule.MailAction.MARK_READ,
assign_title_from=MailRule.TitleSource.NONE,

3
src/paperless_mail/admin.py
View File

@ -68,7 +68,8 @@ class MailRuleAdmin(GuardedModelAdmin):
"filter_to",
"filter_subject",
"filter_body",
"filter_attachment_filename",
"filter_attachment_filename_include",
"filter_attachment_filename_exclude",
"maximum_age",
"consumption_scope",
"attachment_type",

21
src/paperless_mail/mail.py
View File

@ -668,12 +668,29 @@ class MailAccountHandler(LoggingMixin):
)
continue
if rule.filter_attachment_filename and not fnmatch(
if rule.filter_attachment_filename_include and not fnmatch(
att.filename.lower(),
rule.filter_attachment_filename.lower(),
rule.filter_attachment_filename_include.lower(),
):
# Force the filename and pattern to the lowercase
# as this is system dependent otherwise
self.log.debug(
f"Rule {rule}: "
f"Skipping attachment {att.filename} "
f"does not match pattern {rule.filter_attachment_filename_include}",
)
continue
elif rule.filter_attachment_filename_exclude and fnmatch(
att.filename.lower(),
rule.filter_attachment_filename_exclude.lower(),
):
# Force the filename and pattern to the lowercase
# as this is system dependent otherwise
self.log.debug(
f"Rule {rule}: "
f"Skipping attachment {att.filename} "
f"does match pattern {rule.filter_attachment_filename_exclude}",
)
continue
correspondent = self._get_correspondent(message, rule)

29
src/paperless_mail/migrations/0023_remove_mailrule_filter_attachment_filename_and_more.py Normal file
View File

@ -0,0 +1,29 @@
# Generated by Django 4.2.7 on 2023-11-28 17:47
from django.db import migrations
from django.db import models
class Migration(migrations.Migration):
dependencies = [
("paperless_mail", "0022_mailrule_assign_owner_from_rule_and_more"),
]
operations = [
migrations.RenameField(
model_name="mailrule",
old_name="filter_attachment_filename",
new_name="filter_attachment_filename_include",
),
migrations.AddField(
model_name="mailrule",
name="filter_attachment_filename_exclude",
field=models.CharField(
blank=True,
help_text="Do not consume documents which entirely match this filename if specified. Wildcards such as *.pdf or *invoice* are allowed. Case insensitive.",
max_length=256,
null=True,
verbose_name="filter attachment filename exclusive",
),
),
]

16
src/paperless_mail/models.py
View File

@ -139,8 +139,8 @@ class MailRule(document_models.ModelWithOwner):
blank=True,
)
filter_attachment_filename = models.CharField(
_("filter attachment filename"),
filter_attachment_filename_include = models.CharField(
_("filter attachment filename inclusive"),
max_length=256,
null=True,
blank=True,
@ -151,6 +151,18 @@ class MailRule(document_models.ModelWithOwner):
),
)
filter_attachment_filename_exclude = models.CharField(
_("filter attachment filename exclusive"),
max_length=256,
null=True,
blank=True,
help_text=_(
"Do not consume documents which entirely match this "
"filename if specified. Wildcards such as *.pdf or "
"*invoice* are allowed. Case insensitive.",
),
)
maximum_age = models.PositiveIntegerField(
_("maximum age"),
default=30,

3
src/paperless_mail/serialisers.py
View File

@ -79,7 +79,8 @@ class MailRuleSerializer(OwnedObjectSerializer):
"filter_to",
"filter_subject",
"filter_body",
"filter_attachment_filename",
"filter_attachment_filename_include",
"filter_attachment_filename_exclude",
"maximum_age",
"action",
"action_parameter",

16
src/paperless_mail/tests/test_api.py
View File

@ -377,7 +377,7 @@ class TestAPIMailRules(DirectoriesMixin, APITestCase):
filter_to="someone@somewhere.com",
filter_subject="subject",
filter_body="body",
filter_attachment_filename="file.pdf",
filter_attachment_filename_include="file.pdf",
maximum_age=30,
action=MailRule.MailAction.MARK_READ,
assign_title_from=MailRule.TitleSource.FROM_SUBJECT,
@ -400,8 +400,8 @@ class TestAPIMailRules(DirectoriesMixin, APITestCase):
self.assertEqual(returned_rule1["filter_subject"], rule1.filter_subject)
self.assertEqual(returned_rule1["filter_body"], rule1.filter_body)
self.assertEqual(
returned_rule1["filter_attachment_filename"],
rule1.filter_attachment_filename,
returned_rule1["filter_attachment_filename_include"],
rule1.filter_attachment_filename_include,
)
self.assertEqual(returned_rule1["maximum_age"], rule1.maximum_age)
self.assertEqual(returned_rule1["action"], rule1.action)
@ -453,7 +453,7 @@ class TestAPIMailRules(DirectoriesMixin, APITestCase):
"filter_to": "aperson@aplace.com",
"filter_subject": "subject",
"filter_body": "body",
"filter_attachment_filename": "file.pdf",
"filter_attachment_filename_include": "file.pdf",
"maximum_age": 30,
"action": MailRule.MailAction.MARK_READ,
"assign_title_from": MailRule.TitleSource.FROM_SUBJECT,
@ -488,8 +488,8 @@ class TestAPIMailRules(DirectoriesMixin, APITestCase):
self.assertEqual(returned_rule1["filter_subject"], rule1["filter_subject"])
self.assertEqual(returned_rule1["filter_body"], rule1["filter_body"])
self.assertEqual(
returned_rule1["filter_attachment_filename"],
rule1["filter_attachment_filename"],
returned_rule1["filter_attachment_filename_include"],
rule1["filter_attachment_filename_include"],
)
self.assertEqual(returned_rule1["maximum_age"], rule1["maximum_age"])
self.assertEqual(returned_rule1["action"], rule1["action"])
@ -545,7 +545,7 @@ class TestAPIMailRules(DirectoriesMixin, APITestCase):
filter_from="from@example.com",
filter_subject="subject",
filter_body="body",
filter_attachment_filename="file.pdf",
filter_attachment_filename_include="file.pdf",
maximum_age=30,
action=MailRule.MailAction.MARK_READ,
assign_title_from=MailRule.TitleSource.FROM_SUBJECT,
@ -589,7 +589,7 @@ class TestAPIMailRules(DirectoriesMixin, APITestCase):
filter_from="from@example.com",
filter_subject="subject",
filter_body="body",
filter_attachment_filename="file.pdf",
filter_attachment_filename_include="file.pdf",
maximum_age=30,
action=MailRule.MailAction.MARK_READ,
assign_title_from=MailRule.TitleSource.FROM_SUBJECT,

81
src/paperless_mail/tests/test_mail.py
View File

@ -526,6 +526,16 @@ class TestMail(
)
def test_filename_filter(self):
"""
GIVEN:
- Email with multiple similar named attachments
- Rule with inclusive and exclusive filters
WHEN:
- Mail action filtering is checked
THEN:
- Mail action should not be performed for files excluded
- Mail action should be performed for files included
"""
message = self.create_message(
attachments=[
_AttachmentDef(filename="f1.pdf"),
@ -537,15 +547,67 @@ class TestMail(
],
)
@dataclasses.dataclass(frozen=True)
class FilterTestCase:
name: str
include_pattern: Optional[str]
exclude_pattern: Optional[str]
expected_matches: list[str]
tests = [
("*.pdf", ["f1.pdf", "f2.pdf", "f3.pdf", "file.PDf", "f1.Pdf"]),
("f1.pdf", ["f1.pdf", "f1.Pdf"]),
("*", ["f1.pdf", "f2.pdf", "f3.pdf", "f2.png", "file.PDf", "f1.Pdf"]),
("*.png", ["f2.png"]),
FilterTestCase(
"PDF Wildcard",
include_pattern="*.pdf",
exclude_pattern=None,
expected_matches=["f1.pdf", "f2.pdf", "f3.pdf", "file.PDf", "f1.Pdf"],
),
FilterTestCase(
"F1 PDF Only",
include_pattern="f1.pdf",
exclude_pattern=None,
expected_matches=["f1.pdf", "f1.Pdf"],
),
FilterTestCase(
"All Files",
include_pattern="*",
exclude_pattern=None,
expected_matches=[
"f1.pdf",
"f2.pdf",
"f3.pdf",
"f2.png",
"file.PDf",
"f1.Pdf",
],
),
FilterTestCase(
"PNG Only",
include_pattern="*.png",
exclude_pattern=None,
expected_matches=["f2.png"],
),
FilterTestCase(
"PDF Files without f1",
include_pattern="*.pdf",
exclude_pattern="f1*",
expected_matches=["f2.pdf", "f3.pdf", "file.PDf"],
),
FilterTestCase(
"All Files, no PNG",
include_pattern="*",
exclude_pattern="*.png",
expected_matches=[
"f1.pdf",
"f2.pdf",
"f3.pdf",
"file.PDf",
"f1.Pdf",
],
),
]
for pattern, matches in tests:
with self.subTest(msg=pattern):
for test_case in tests:
with self.subTest(msg=test_case.name):
self._queue_consumption_tasks_mock.reset_mock()
account = MailAccount(name=str(uuid.uuid4()))
account.save()
@ -553,14 +615,15 @@ class TestMail(
name=str(uuid.uuid4()),
assign_title_from=MailRule.TitleSource.FROM_FILENAME,
account=account,
filter_attachment_filename=pattern,
filter_attachment_filename_include=test_case.include_pattern,
filter_attachment_filename_exclude=test_case.exclude_pattern,
)
rule.save()
self.mail_account_handler._handle_message(message, rule)
self.assert_queue_consumption_tasks_call_args(
[
[{"override_filename": m} for m in matches],
[{"override_filename": m} for m in test_case.expected_matches],
],
)
@ -593,7 +656,7 @@ class TestMail(
name=str(uuid.uuid4()),
assign_title_from=MailRule.TitleSource.FROM_FILENAME,
account=account,
filter_attachment_filename="*.pdf",
filter_attachment_filename_include="*.pdf",
attachment_type=MailRule.AttachmentProcessing.EVERYTHING,
action=MailRule.MailAction.DELETE,
)