Enhancement: Allow excluding mail attachments by name (#4691)

* Adds new filtering to exclude attachments from processing * Frontend use include / exclude mail rule filename filters --------- Co-authored-by: shamoon <4887959+shamoon@users.noreply.github.com>
2025-04-25 10:49:30 -05:00 · 2023-12-02 08:26:19 -08:00 · 2023-12-02 08:26:19 -08:00 · 6e371ac5ac
commit 6e371ac5ac
parent 1b69b89d2d
14 changed files with 189 additions and 45 deletions
--- a/src-ui/messages.xlf
+++ b/src-ui/messages.xlf
@ -1315,7 +1315,7 @@
        </context-group>
        <context-group purpose="location">
          <context context-type="sourcefile">src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html</context>
-          <context context-type="linenumber">42</context>
+          <context context-type="linenumber">43</context>
        </context-group>
        <context-group purpose="location">
          <context context-type="sourcefile">src/app/components/common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component.html</context>
@ -2520,7 +2520,7 @@
        </context-group>
        <context-group purpose="location">
          <context context-type="sourcefile">src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html</context>
-          <context context-type="linenumber">32</context>
+          <context context-type="linenumber">33</context>
        </context-group>
      </trans-unit>
      <trans-unit id="4875491778188965469" datatype="html">
@ -2531,7 +2531,7 @@
        </context-group>
        <context-group purpose="location">
          <context context-type="sourcefile">src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html</context>
-          <context context-type="linenumber">34</context>
+          <context context-type="linenumber">35</context>
        </context-group>
      </trans-unit>
      <trans-unit id="7198346314713788799" datatype="html">
@ -2570,7 +2570,7 @@
        </context-group>
        <context-group purpose="location">
          <context context-type="sourcefile">src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html</context>
-          <context context-type="linenumber">40</context>
+          <context context-type="linenumber">41</context>
        </context-group>
        <context-group purpose="location">
          <context context-type="sourcefile">src/app/components/common/toasts/toasts.component.html</context>
@ -2605,7 +2605,7 @@
        </context-group>
        <context-group purpose="location">
          <context context-type="sourcefile">src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html</context>
-          <context context-type="linenumber">41</context>
+          <context context-type="linenumber">42</context>
        </context-group>
        <context-group purpose="location">
          <context context-type="sourcefile">src/app/components/common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component.html</context>
@ -3016,8 +3016,8 @@
          <context context-type="linenumber">23</context>
        </context-group>
      </trans-unit>
-      <trans-unit id="5031687746498952417" datatype="html">
-        <source>Filter attachment filename</source>
+      <trans-unit id="4603548543464136402" datatype="html">
+        <source>Filter attachment filename includes</source>
        <context-group purpose="location">
          <context context-type="sourcefile">src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html</context>
          <context context-type="linenumber">24</context>
@ -3030,53 +3030,67 @@
          <context context-type="linenumber">24</context>
        </context-group>
      </trans-unit>
+      <trans-unit id="6869675473865305593" datatype="html">
+        <source>Filter attachment filename excluding</source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html</context>
+          <context context-type="linenumber">25</context>
+        </context-group>
+      </trans-unit>
+      <trans-unit id="6774472763442688477" datatype="html">
+        <source>Do not consume documents which entirely match this filename if specified. Wildcards such as *.pdf or *invoice* are allowed. Case insensitive.</source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html</context>
+          <context context-type="linenumber">25</context>
+        </context-group>
+      </trans-unit>
      <trans-unit id="9216117865911519658" datatype="html">
        <source>Action</source>
        <context-group purpose="location">
          <context context-type="sourcefile">src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html</context>
-          <context context-type="linenumber">27</context>
+          <context context-type="linenumber">28</context>
        </context-group>
      </trans-unit>
      <trans-unit id="4274038999388817994" datatype="html">
        <source>Action is only performed when documents are consumed from the mail. Mails without attachments remain entirely untouched.</source>
        <context-group purpose="location">
          <context context-type="sourcefile">src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html</context>
-          <context context-type="linenumber">27</context>
+          <context context-type="linenumber">28</context>
        </context-group>
      </trans-unit>
      <trans-unit id="1261794314435932203" datatype="html">
        <source>Action parameter</source>
        <context-group purpose="location">
          <context context-type="sourcefile">src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html</context>
-          <context context-type="linenumber">28</context>
+          <context context-type="linenumber">29</context>
        </context-group>
      </trans-unit>
      <trans-unit id="5512171567357420308" datatype="html">
        <source>Assignments specified here will supersede any consumption templates.</source>
        <context-group purpose="location">
          <context context-type="sourcefile">src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html</context>
-          <context context-type="linenumber">29</context>
+          <context context-type="linenumber">30</context>
        </context-group>
      </trans-unit>
      <trans-unit id="6093797930511670257" datatype="html">
        <source>Assign title from</source>
        <context-group purpose="location">
          <context context-type="sourcefile">src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html</context>
-          <context context-type="linenumber">30</context>
+          <context context-type="linenumber">31</context>
        </context-group>
      </trans-unit>
      <trans-unit id="4754802869258527587" datatype="html">
        <source>Assign correspondent from</source>
        <context-group purpose="location">
          <context context-type="sourcefile">src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html</context>
-          <context context-type="linenumber">33</context>
+          <context context-type="linenumber">34</context>
        </context-group>
      </trans-unit>
      <trans-unit id="5232720756589450549" datatype="html">
        <source>Assign owner from rule</source>
        <context-group purpose="location">
          <context context-type="sourcefile">src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html</context>
-          <context context-type="linenumber">35</context>
+          <context context-type="linenumber">36</context>
        </context-group>
      </trans-unit>
      <trans-unit id="6886003843406464884" datatype="html">
--- a/src-ui/src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html
+++ b/src-ui/src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html
@ -21,7 +21,8 @@
        <pngx-input-text i18n-title title="Filter to" formControlName="filter_to" [error]="error?.filter_to"></pngx-input-text>
        <pngx-input-text i18n-title title="Filter subject" formControlName="filter_subject" [error]="error?.filter_subject"></pngx-input-text>
        <pngx-input-text i18n-title title="Filter body" formControlName="filter_body" [error]="error?.filter_body"></pngx-input-text>
-        <pngx-input-text i18n-title title="Filter attachment filename" formControlName="filter_attachment_filename" i18n-hint hint="Only consume documents which entirely match this filename if specified. Wildcards such as *.pdf or *invoice* are allowed. Case insensitive." [error]="error?.filter_attachment_filename"></pngx-input-text>
+        <pngx-input-text i18n-title title="Filter attachment filename includes" formControlName="filter_attachment_filename_include" i18n-hint hint="Only consume documents which entirely match this filename if specified. Wildcards such as *.pdf or *invoice* are allowed. Case insensitive." [error]="error?.filter_attachment_filename_include"></pngx-input-text>
+        <pngx-input-text i18n-title title="Filter attachment filename excluding" formControlName="filter_attachment_filename_exclude" i18n-hint hint="Do not consume documents which entirely match this filename if specified. Wildcards such as *.pdf or *invoice* are allowed. Case insensitive." [error]="error?.filter_attachment_filename_exclude"></pngx-input-text>
      </div>
      <div class="col-md-4">
        <pngx-input-select i18n-title title="Action" [items]="actionOptions" formControlName="action" i18n-hint hint="Action is only performed when documents are consumed from the mail. Mails without attachments remain entirely untouched."></pngx-input-select>
--- a/src-ui/src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.ts
+++ b/src-ui/src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.ts
@ -158,7 +158,8 @@ export class MailRuleEditDialogComponent extends EditDialogComponent<PaperlessMa
      filter_to: new FormControl(null),
      filter_subject: new FormControl(null),
      filter_body: new FormControl(null),
-      filter_attachment_filename: new FormControl(null),
+      filter_attachment_filename_include: new FormControl(null),
+      filter_attachment_filename_exclude: new FormControl(null),
      maximum_age: new FormControl(null),
      attachment_type: new FormControl(MailFilterAttachmentType.Attachments),
      consumption_scope: new FormControl(MailRuleConsumptionScope.Attachments),
--- a/src-ui/src/app/data/paperless-mail-rule.ts
+++ b/src-ui/src/app/data/paperless-mail-rule.ts
@ -49,7 +49,9 @@ export interface PaperlessMailRule extends ObjectWithPermissions {

  filter_body: string

-  filter_attachment_filename: string
+  filter_attachment_filename_include: string
+
+  filter_attachment_filename_exclude: string

  maximum_age: number

--- a/src-ui/src/app/services/rest/mail-rule.service.spec.ts
+++ b/src-ui/src/app/services/rest/mail-rule.service.spec.ts
@ -23,7 +23,8 @@ const mail_rules = [
    filter_to: null,
    filter_subject: null,
    filter_body: null,
-    filter_attachment_filename: null,
+    filter_attachment_filename_include: null,
+    filter_attachment_filename_exclude: null,
    maximum_age: 30,
    attachment_type: MailFilterAttachmentType.Everything,
    action: MailAction.MarkRead,
@ -40,7 +41,8 @@ const mail_rules = [
    filter_to: null,
    filter_subject: null,
    filter_body: null,
-    filter_attachment_filename: null,
+    filter_attachment_filename_include: null,
+    filter_attachment_filename_exclude: null,
    maximum_age: 30,
    attachment_type: MailFilterAttachmentType.Everything,
    action: MailAction.Delete,
@ -57,7 +59,8 @@ const mail_rules = [
    filter_to: null,
    filter_subject: null,
    filter_body: null,
-    filter_attachment_filename: null,
+    filter_attachment_filename_include: null,
+    filter_attachment_filename_exclude: null,
    maximum_age: 30,
    attachment_type: MailFilterAttachmentType.Everything,
    action: MailAction.Flag,
--- a/src/documents/tests/test_api.py
+++ b/src/documents/tests/test_api.py
@ -5769,7 +5769,7 @@ class TestApiConsumptionTemplates(DirectoriesMixin, APITestCase):
            filter_to="someone@somewhere.com",
            filter_subject="subject",
            filter_body="body",
-            filter_attachment_filename="file.pdf",
+            filter_attachment_filename_include="file.pdf",
            maximum_age=30,
            action=MailRule.MailAction.MARK_READ,
            assign_title_from=MailRule.TitleSource.FROM_SUBJECT,
--- a/src/documents/tests/test_consumption_templates.py
+++ b/src/documents/tests/test_consumption_templates.py
@ -54,7 +54,7 @@ class TestConsumptionTemplates(DirectoriesMixin, FileSystemAssertsMixin, TestCas
            filter_to="someone@somewhere.com",
            filter_subject="subject",
            filter_body="body",
-            filter_attachment_filename="file.pdf",
+            filter_attachment_filename_include="file.pdf",
            maximum_age=30,
            action=MailRule.MailAction.MARK_READ,
            assign_title_from=MailRule.TitleSource.NONE,
--- a/src/paperless_mail/admin.py
+++ b/src/paperless_mail/admin.py
@ -68,7 +68,8 @@ class MailRuleAdmin(GuardedModelAdmin):
                    "filter_to",
                    "filter_subject",
                    "filter_body",
-                    "filter_attachment_filename",
+                    "filter_attachment_filename_include",
+                    "filter_attachment_filename_exclude",
                    "maximum_age",
                    "consumption_scope",
                    "attachment_type",
--- a/src/paperless_mail/mail.py
+++ b/src/paperless_mail/mail.py
@ -668,12 +668,29 @@ class MailAccountHandler(LoggingMixin):
                )
                continue

-            if rule.filter_attachment_filename and not fnmatch(
+            if rule.filter_attachment_filename_include and not fnmatch(
                att.filename.lower(),
-                rule.filter_attachment_filename.lower(),
+                rule.filter_attachment_filename_include.lower(),
            ):
                # Force the filename and pattern to the lowercase
                # as this is system dependent otherwise
+                self.log.debug(
+                    f"Rule {rule}: "
+                    f"Skipping attachment {att.filename} "
+                    f"does not match pattern {rule.filter_attachment_filename_include}",
+                )
+                continue
+            elif rule.filter_attachment_filename_exclude and fnmatch(
+                att.filename.lower(),
+                rule.filter_attachment_filename_exclude.lower(),
+            ):
+                # Force the filename and pattern to the lowercase
+                # as this is system dependent otherwise
+                self.log.debug(
+                    f"Rule {rule}: "
+                    f"Skipping attachment {att.filename} "
+                    f"does match pattern {rule.filter_attachment_filename_exclude}",
+                )
                continue

            correspondent = self._get_correspondent(message, rule)
--- a/src/paperless_mail/migrations/0023_remove_mailrule_filter_attachment_filename_and_more.py
+++ b/src/paperless_mail/migrations/0023_remove_mailrule_filter_attachment_filename_and_more.py
@ -0,0 +1,29 @@
+# Generated by Django 4.2.7 on 2023-11-28 17:47
+
+from django.db import migrations
+from django.db import models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("paperless_mail", "0022_mailrule_assign_owner_from_rule_and_more"),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name="mailrule",
+            old_name="filter_attachment_filename",
+            new_name="filter_attachment_filename_include",
+        ),
+        migrations.AddField(
+            model_name="mailrule",
+            name="filter_attachment_filename_exclude",
+            field=models.CharField(
+                blank=True,
+                help_text="Do not consume documents which entirely match this filename if specified. Wildcards such as *.pdf or *invoice* are allowed. Case insensitive.",
+                max_length=256,
+                null=True,
+                verbose_name="filter attachment filename exclusive",
+            ),
+        ),
+    ]
--- a/src/paperless_mail/models.py
+++ b/src/paperless_mail/models.py
@ -139,8 +139,8 @@ class MailRule(document_models.ModelWithOwner):
        blank=True,
    )

-    filter_attachment_filename = models.CharField(
-        _("filter attachment filename"),
+    filter_attachment_filename_include = models.CharField(
+        _("filter attachment filename inclusive"),
        max_length=256,
        null=True,
        blank=True,
@ -151,6 +151,18 @@ class MailRule(document_models.ModelWithOwner):
        ),
    )

+    filter_attachment_filename_exclude = models.CharField(
+        _("filter attachment filename exclusive"),
+        max_length=256,
+        null=True,
+        blank=True,
+        help_text=_(
+            "Do not consume documents which entirely match this "
+            "filename if specified. Wildcards such as *.pdf or "
+            "*invoice* are allowed. Case insensitive.",
+        ),
+    )
+
    maximum_age = models.PositiveIntegerField(
        _("maximum age"),
        default=30,
--- a/src/paperless_mail/serialisers.py
+++ b/src/paperless_mail/serialisers.py
@ -79,7 +79,8 @@ class MailRuleSerializer(OwnedObjectSerializer):
            "filter_to",
            "filter_subject",
            "filter_body",
-            "filter_attachment_filename",
+            "filter_attachment_filename_include",
+            "filter_attachment_filename_exclude",
            "maximum_age",
            "action",
            "action_parameter",
--- a/src/paperless_mail/tests/test_api.py
+++ b/src/paperless_mail/tests/test_api.py
@ -377,7 +377,7 @@ class TestAPIMailRules(DirectoriesMixin, APITestCase):
            filter_to="someone@somewhere.com",
            filter_subject="subject",
            filter_body="body",
-            filter_attachment_filename="file.pdf",
+            filter_attachment_filename_include="file.pdf",
            maximum_age=30,
            action=MailRule.MailAction.MARK_READ,
            assign_title_from=MailRule.TitleSource.FROM_SUBJECT,
@ -400,8 +400,8 @@ class TestAPIMailRules(DirectoriesMixin, APITestCase):
        self.assertEqual(returned_rule1["filter_subject"], rule1.filter_subject)
        self.assertEqual(returned_rule1["filter_body"], rule1.filter_body)
        self.assertEqual(
-            returned_rule1["filter_attachment_filename"],
-            rule1.filter_attachment_filename,
+            returned_rule1["filter_attachment_filename_include"],
+            rule1.filter_attachment_filename_include,
        )
        self.assertEqual(returned_rule1["maximum_age"], rule1.maximum_age)
        self.assertEqual(returned_rule1["action"], rule1.action)
@ -453,7 +453,7 @@ class TestAPIMailRules(DirectoriesMixin, APITestCase):
            "filter_to": "aperson@aplace.com",
            "filter_subject": "subject",
            "filter_body": "body",
-            "filter_attachment_filename": "file.pdf",
+            "filter_attachment_filename_include": "file.pdf",
            "maximum_age": 30,
            "action": MailRule.MailAction.MARK_READ,
            "assign_title_from": MailRule.TitleSource.FROM_SUBJECT,
@ -488,8 +488,8 @@ class TestAPIMailRules(DirectoriesMixin, APITestCase):
        self.assertEqual(returned_rule1["filter_subject"], rule1["filter_subject"])
        self.assertEqual(returned_rule1["filter_body"], rule1["filter_body"])
        self.assertEqual(
-            returned_rule1["filter_attachment_filename"],
-            rule1["filter_attachment_filename"],
+            returned_rule1["filter_attachment_filename_include"],
+            rule1["filter_attachment_filename_include"],
        )
        self.assertEqual(returned_rule1["maximum_age"], rule1["maximum_age"])
        self.assertEqual(returned_rule1["action"], rule1["action"])
@ -545,7 +545,7 @@ class TestAPIMailRules(DirectoriesMixin, APITestCase):
            filter_from="from@example.com",
            filter_subject="subject",
            filter_body="body",
-            filter_attachment_filename="file.pdf",
+            filter_attachment_filename_include="file.pdf",
            maximum_age=30,
            action=MailRule.MailAction.MARK_READ,
            assign_title_from=MailRule.TitleSource.FROM_SUBJECT,
@ -589,7 +589,7 @@ class TestAPIMailRules(DirectoriesMixin, APITestCase):
            filter_from="from@example.com",
            filter_subject="subject",
            filter_body="body",
-            filter_attachment_filename="file.pdf",
+            filter_attachment_filename_include="file.pdf",
            maximum_age=30,
            action=MailRule.MailAction.MARK_READ,
            assign_title_from=MailRule.TitleSource.FROM_SUBJECT,
--- a/src/paperless_mail/tests/test_mail.py
+++ b/src/paperless_mail/tests/test_mail.py
@ -526,6 +526,16 @@ class TestMail(
        )

    def test_filename_filter(self):
+        """
+        GIVEN:
+            - Email with multiple similar named attachments
+            - Rule with inclusive and exclusive filters
+        WHEN:
+            - Mail action filtering is checked
+        THEN:
+            - Mail action should not be performed for files excluded
+            - Mail action should be performed for files included
+        """
        message = self.create_message(
            attachments=[
                _AttachmentDef(filename="f1.pdf"),
@ -537,15 +547,67 @@ class TestMail(
            ],
        )

+        @dataclasses.dataclass(frozen=True)
+        class FilterTestCase:
+            name: str
+            include_pattern: Optional[str]
+            exclude_pattern: Optional[str]
+            expected_matches: list[str]
+
        tests = [
-            ("*.pdf", ["f1.pdf", "f2.pdf", "f3.pdf", "file.PDf", "f1.Pdf"]),
-            ("f1.pdf", ["f1.pdf", "f1.Pdf"]),
-            ("*", ["f1.pdf", "f2.pdf", "f3.pdf", "f2.png", "file.PDf", "f1.Pdf"]),
-            ("*.png", ["f2.png"]),
+            FilterTestCase(
+                "PDF Wildcard",
+                include_pattern="*.pdf",
+                exclude_pattern=None,
+                expected_matches=["f1.pdf", "f2.pdf", "f3.pdf", "file.PDf", "f1.Pdf"],
+            ),
+            FilterTestCase(
+                "F1 PDF Only",
+                include_pattern="f1.pdf",
+                exclude_pattern=None,
+                expected_matches=["f1.pdf", "f1.Pdf"],
+            ),
+            FilterTestCase(
+                "All Files",
+                include_pattern="*",
+                exclude_pattern=None,
+                expected_matches=[
+                    "f1.pdf",
+                    "f2.pdf",
+                    "f3.pdf",
+                    "f2.png",
+                    "file.PDf",
+                    "f1.Pdf",
+                ],
+            ),
+            FilterTestCase(
+                "PNG Only",
+                include_pattern="*.png",
+                exclude_pattern=None,
+                expected_matches=["f2.png"],
+            ),
+            FilterTestCase(
+                "PDF Files without f1",
+                include_pattern="*.pdf",
+                exclude_pattern="f1*",
+                expected_matches=["f2.pdf", "f3.pdf", "file.PDf"],
+            ),
+            FilterTestCase(
+                "All Files, no PNG",
+                include_pattern="*",
+                exclude_pattern="*.png",
+                expected_matches=[
+                    "f1.pdf",
+                    "f2.pdf",
+                    "f3.pdf",
+                    "file.PDf",
+                    "f1.Pdf",
+                ],
+            ),
        ]

-        for pattern, matches in tests:
-            with self.subTest(msg=pattern):
+        for test_case in tests:
+            with self.subTest(msg=test_case.name):
                self._queue_consumption_tasks_mock.reset_mock()
                account = MailAccount(name=str(uuid.uuid4()))
                account.save()
@ -553,14 +615,15 @@ class TestMail(
                    name=str(uuid.uuid4()),
                    assign_title_from=MailRule.TitleSource.FROM_FILENAME,
                    account=account,
-                    filter_attachment_filename=pattern,
+                    filter_attachment_filename_include=test_case.include_pattern,
+                    filter_attachment_filename_exclude=test_case.exclude_pattern,
                )
                rule.save()

                self.mail_account_handler._handle_message(message, rule)
                self.assert_queue_consumption_tasks_call_args(
                    [
-                        [{"override_filename": m} for m in matches],
+                        [{"override_filename": m} for m in test_case.expected_matches],
                    ],
                )

@ -593,7 +656,7 @@ class TestMail(
            name=str(uuid.uuid4()),
            assign_title_from=MailRule.TitleSource.FROM_FILENAME,
            account=account,
-            filter_attachment_filename="*.pdf",
+            filter_attachment_filename_include="*.pdf",
            attachment_type=MailRule.AttachmentProcessing.EVERYTHING,
            action=MailRule.MailAction.DELETE,
        )