Enhancement: Allow excluding mail attachments by name (#4691)

* Adds new filtering to exclude attachments from processing

* Frontend use include / exclude mail rule filename filters

---------

Co-authored-by: shamoon <4887959+shamoon@users.noreply.github.com>

src-ui/messages.xlf

@@ -1315,7 +1315,7 @@
         </context-group>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html</context>
-          <context context-type="linenumber">42</context>
+          <context context-type="linenumber">43</context>
         </context-group>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component.html</context>
@@ -2520,7 +2520,7 @@
         </context-group>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html</context>
-          <context context-type="linenumber">32</context>
+          <context context-type="linenumber">33</context>
         </context-group>
       </trans-unit>
       <trans-unit id="4875491778188965469" datatype="html">
@@ -2531,7 +2531,7 @@
         </context-group>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html</context>
-          <context context-type="linenumber">34</context>
+          <context context-type="linenumber">35</context>
         </context-group>
       </trans-unit>
       <trans-unit id="7198346314713788799" datatype="html">
@@ -2570,7 +2570,7 @@
         </context-group>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html</context>
-          <context context-type="linenumber">40</context>
+          <context context-type="linenumber">41</context>
         </context-group>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/common/toasts/toasts.component.html</context>
@@ -2605,7 +2605,7 @@
         </context-group>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html</context>
-          <context context-type="linenumber">41</context>
+          <context context-type="linenumber">42</context>
         </context-group>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component.html</context>
@@ -3016,8 +3016,8 @@
           <context context-type="linenumber">23</context>
         </context-group>
       </trans-unit>
-      <trans-unit id="5031687746498952417" datatype="html">
+      <trans-unit id="4603548543464136402" datatype="html">
-        <source>Filter attachment filename</source>
+        <source>Filter attachment filename includes</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html</context>
           <context context-type="linenumber">24</context>
@@ -3030,53 +3030,67 @@
           <context context-type="linenumber">24</context>
         </context-group>
       </trans-unit>
+      <trans-unit id="6869675473865305593" datatype="html">
+        <source>Filter attachment filename excluding</source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html</context>
+          <context context-type="linenumber">25</context>
+        </context-group>
+      </trans-unit>
+      <trans-unit id="6774472763442688477" datatype="html">
+        <source>Do not consume documents which entirely match this filename if specified. Wildcards such as *.pdf or *invoice* are allowed. Case insensitive.</source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html</context>
+          <context context-type="linenumber">25</context>
+        </context-group>
+      </trans-unit>
       <trans-unit id="9216117865911519658" datatype="html">
         <source>Action</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html</context>
-          <context context-type="linenumber">27</context>
+          <context context-type="linenumber">28</context>
         </context-group>
       </trans-unit>
       <trans-unit id="4274038999388817994" datatype="html">
         <source>Action is only performed when documents are consumed from the mail. Mails without attachments remain entirely untouched.</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html</context>
-          <context context-type="linenumber">27</context>
+          <context context-type="linenumber">28</context>
         </context-group>
       </trans-unit>
       <trans-unit id="1261794314435932203" datatype="html">
         <source>Action parameter</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html</context>
-          <context context-type="linenumber">28</context>
+          <context context-type="linenumber">29</context>
         </context-group>
       </trans-unit>
       <trans-unit id="5512171567357420308" datatype="html">
         <source>Assignments specified here will supersede any consumption templates.</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html</context>
-          <context context-type="linenumber">29</context>
+          <context context-type="linenumber">30</context>
         </context-group>
       </trans-unit>
       <trans-unit id="6093797930511670257" datatype="html">
         <source>Assign title from</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html</context>
-          <context context-type="linenumber">30</context>
+          <context context-type="linenumber">31</context>
         </context-group>
       </trans-unit>
       <trans-unit id="4754802869258527587" datatype="html">
         <source>Assign correspondent from</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html</context>
-          <context context-type="linenumber">33</context>
+          <context context-type="linenumber">34</context>
         </context-group>
       </trans-unit>
       <trans-unit id="5232720756589450549" datatype="html">
         <source>Assign owner from rule</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html</context>
-          <context context-type="linenumber">35</context>
+          <context context-type="linenumber">36</context>
         </context-group>
       </trans-unit>
       <trans-unit id="6886003843406464884" datatype="html">

src-ui/src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html

@@ -21,7 +21,8 @@
         <pngx-input-text i18n-title title="Filter to" formControlName="filter_to" [error]="error?.filter_to"></pngx-input-text>
         <pngx-input-text i18n-title title="Filter subject" formControlName="filter_subject" [error]="error?.filter_subject"></pngx-input-text>
         <pngx-input-text i18n-title title="Filter body" formControlName="filter_body" [error]="error?.filter_body"></pngx-input-text>
-        <pngx-input-text i18n-title title="Filter attachment filename" formControlName="filter_attachment_filename" i18n-hint hint="Only consume documents which entirely match this filename if specified. Wildcards such as *.pdf or *invoice* are allowed. Case insensitive." [error]="error?.filter_attachment_filename"></pngx-input-text>
+        <pngx-input-text i18n-title title="Filter attachment filename includes" formControlName="filter_attachment_filename_include" i18n-hint hint="Only consume documents which entirely match this filename if specified. Wildcards such as *.pdf or *invoice* are allowed. Case insensitive." [error]="error?.filter_attachment_filename_include"></pngx-input-text>
+        <pngx-input-text i18n-title title="Filter attachment filename excluding" formControlName="filter_attachment_filename_exclude" i18n-hint hint="Do not consume documents which entirely match this filename if specified. Wildcards such as *.pdf or *invoice* are allowed. Case insensitive." [error]="error?.filter_attachment_filename_exclude"></pngx-input-text>
       </div>
       <div class="col-md-4">
         <pngx-input-select i18n-title title="Action" [items]="actionOptions" formControlName="action" i18n-hint hint="Action is only performed when documents are consumed from the mail. Mails without attachments remain entirely untouched."></pngx-input-select>

src-ui/src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.ts

@@ -158,7 +158,8 @@ export class MailRuleEditDialogComponent extends EditDialogComponent<PaperlessMa
       filter_to: new FormControl(null),
       filter_subject: new FormControl(null),
       filter_body: new FormControl(null),
-      filter_attachment_filename: new FormControl(null),
+      filter_attachment_filename_include: new FormControl(null),
+      filter_attachment_filename_exclude: new FormControl(null),
       maximum_age: new FormControl(null),
       attachment_type: new FormControl(MailFilterAttachmentType.Attachments),
       consumption_scope: new FormControl(MailRuleConsumptionScope.Attachments),

src-ui/src/app/data/paperless-mail-rule.ts

@@ -49,7 +49,9 @@ export interface PaperlessMailRule extends ObjectWithPermissions {
 
   filter_body: string
 
-  filter_attachment_filename: string
+  filter_attachment_filename_include: string
+
+  filter_attachment_filename_exclude: string
 
   maximum_age: number
 

src-ui/src/app/services/rest/mail-rule.service.spec.ts

@@ -23,7 +23,8 @@ const mail_rules = [
     filter_to: null,
     filter_subject: null,
     filter_body: null,
-    filter_attachment_filename: null,
+    filter_attachment_filename_include: null,
+    filter_attachment_filename_exclude: null,
     maximum_age: 30,
     attachment_type: MailFilterAttachmentType.Everything,
     action: MailAction.MarkRead,
@@ -40,7 +41,8 @@ const mail_rules = [
     filter_to: null,
     filter_subject: null,
     filter_body: null,
-    filter_attachment_filename: null,
+    filter_attachment_filename_include: null,
+    filter_attachment_filename_exclude: null,
     maximum_age: 30,
     attachment_type: MailFilterAttachmentType.Everything,
     action: MailAction.Delete,
@@ -57,7 +59,8 @@ const mail_rules = [
     filter_to: null,
     filter_subject: null,
     filter_body: null,
-    filter_attachment_filename: null,
+    filter_attachment_filename_include: null,
+    filter_attachment_filename_exclude: null,
     maximum_age: 30,
     attachment_type: MailFilterAttachmentType.Everything,
     action: MailAction.Flag,

src/documents/tests/test_api.py

@@ -5769,7 +5769,7 @@ class TestApiConsumptionTemplates(DirectoriesMixin, APITestCase):
             filter_to="someone@somewhere.com",
             filter_subject="subject",
             filter_body="body",
-            filter_attachment_filename="file.pdf",
+            filter_attachment_filename_include="file.pdf",
             maximum_age=30,
             action=MailRule.MailAction.MARK_READ,
             assign_title_from=MailRule.TitleSource.FROM_SUBJECT,

src/documents/tests/test_consumption_templates.py

@@ -54,7 +54,7 @@ class TestConsumptionTemplates(DirectoriesMixin, FileSystemAssertsMixin, TestCas
             filter_to="someone@somewhere.com",
             filter_subject="subject",
             filter_body="body",
-            filter_attachment_filename="file.pdf",
+            filter_attachment_filename_include="file.pdf",
             maximum_age=30,
             action=MailRule.MailAction.MARK_READ,
             assign_title_from=MailRule.TitleSource.NONE,

src/paperless_mail/admin.py

@@ -68,7 +68,8 @@ class MailRuleAdmin(GuardedModelAdmin):
                     "filter_to",
                     "filter_subject",
                     "filter_body",
-                    "filter_attachment_filename",
+                    "filter_attachment_filename_include",
+                    "filter_attachment_filename_exclude",
                     "maximum_age",
                     "consumption_scope",
                     "attachment_type",

src/paperless_mail/mail.py

@@ -668,12 +668,29 @@ class MailAccountHandler(LoggingMixin):
                 )
                 continue
 
-            if rule.filter_attachment_filename and not fnmatch(
+            if rule.filter_attachment_filename_include and not fnmatch(
                 att.filename.lower(),
-                rule.filter_attachment_filename.lower(),
+                rule.filter_attachment_filename_include.lower(),
             ):
                 # Force the filename and pattern to the lowercase
                 # as this is system dependent otherwise
+                self.log.debug(
+                    f"Rule {rule}: "
+                    f"Skipping attachment {att.filename} "
+                    f"does not match pattern {rule.filter_attachment_filename_include}",
+                )
+                continue
+            elif rule.filter_attachment_filename_exclude and fnmatch(
+                att.filename.lower(),
+                rule.filter_attachment_filename_exclude.lower(),
+            ):
+                # Force the filename and pattern to the lowercase
+                # as this is system dependent otherwise
+                self.log.debug(
+                    f"Rule {rule}: "
+                    f"Skipping attachment {att.filename} "
+                    f"does match pattern {rule.filter_attachment_filename_exclude}",
+                )
                 continue
 
             correspondent = self._get_correspondent(message, rule)

src/paperless_mail/migrations/0023_remove_mailrule_filter_attachment_filename_and_more.py

@@ -0,0 +1,29 @@
+# Generated by Django 4.2.7 on 2023-11-28 17:47
+
+from django.db import migrations
+from django.db import models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("paperless_mail", "0022_mailrule_assign_owner_from_rule_and_more"),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name="mailrule",
+            old_name="filter_attachment_filename",
+            new_name="filter_attachment_filename_include",
+        ),
+        migrations.AddField(
+            model_name="mailrule",
+            name="filter_attachment_filename_exclude",
+            field=models.CharField(
+                blank=True,
+                help_text="Do not consume documents which entirely match this filename if specified. Wildcards such as *.pdf or *invoice* are allowed. Case insensitive.",
+                max_length=256,
+                null=True,
+                verbose_name="filter attachment filename exclusive",
+            ),
+        ),
+    ]

src/paperless_mail/models.py

@@ -139,8 +139,8 @@ class MailRule(document_models.ModelWithOwner):
         blank=True,
     )
 
-    filter_attachment_filename = models.CharField(
+    filter_attachment_filename_include = models.CharField(
-        _("filter attachment filename"),
+        _("filter attachment filename inclusive"),
         max_length=256,
         null=True,
         blank=True,
@@ -151,6 +151,18 @@ class MailRule(document_models.ModelWithOwner):
         ),
     )
 
+    filter_attachment_filename_exclude = models.CharField(
+        _("filter attachment filename exclusive"),
+        max_length=256,
+        null=True,
+        blank=True,
+        help_text=_(
+            "Do not consume documents which entirely match this "
+            "filename if specified. Wildcards such as *.pdf or "
+            "*invoice* are allowed. Case insensitive.",
+        ),
+    )
+
     maximum_age = models.PositiveIntegerField(
         _("maximum age"),
         default=30,

src/paperless_mail/serialisers.py

@@ -79,7 +79,8 @@ class MailRuleSerializer(OwnedObjectSerializer):
             "filter_to",
             "filter_subject",
             "filter_body",
-            "filter_attachment_filename",
+            "filter_attachment_filename_include",
+            "filter_attachment_filename_exclude",
             "maximum_age",
             "action",
             "action_parameter",

src/paperless_mail/tests/test_api.py

@@ -377,7 +377,7 @@ class TestAPIMailRules(DirectoriesMixin, APITestCase):
             filter_to="someone@somewhere.com",
             filter_subject="subject",
             filter_body="body",
-            filter_attachment_filename="file.pdf",
+            filter_attachment_filename_include="file.pdf",
             maximum_age=30,
             action=MailRule.MailAction.MARK_READ,
             assign_title_from=MailRule.TitleSource.FROM_SUBJECT,
@@ -400,8 +400,8 @@ class TestAPIMailRules(DirectoriesMixin, APITestCase):
         self.assertEqual(returned_rule1["filter_subject"], rule1.filter_subject)
         self.assertEqual(returned_rule1["filter_body"], rule1.filter_body)
         self.assertEqual(
-            returned_rule1["filter_attachment_filename"],
+            returned_rule1["filter_attachment_filename_include"],
-            rule1.filter_attachment_filename,
+            rule1.filter_attachment_filename_include,
         )
         self.assertEqual(returned_rule1["maximum_age"], rule1.maximum_age)
         self.assertEqual(returned_rule1["action"], rule1.action)
@@ -453,7 +453,7 @@ class TestAPIMailRules(DirectoriesMixin, APITestCase):
             "filter_to": "aperson@aplace.com",
             "filter_subject": "subject",
             "filter_body": "body",
-            "filter_attachment_filename": "file.pdf",
+            "filter_attachment_filename_include": "file.pdf",
             "maximum_age": 30,
             "action": MailRule.MailAction.MARK_READ,
             "assign_title_from": MailRule.TitleSource.FROM_SUBJECT,
@@ -488,8 +488,8 @@ class TestAPIMailRules(DirectoriesMixin, APITestCase):
         self.assertEqual(returned_rule1["filter_subject"], rule1["filter_subject"])
         self.assertEqual(returned_rule1["filter_body"], rule1["filter_body"])
         self.assertEqual(
-            returned_rule1["filter_attachment_filename"],
+            returned_rule1["filter_attachment_filename_include"],
-            rule1["filter_attachment_filename"],
+            rule1["filter_attachment_filename_include"],
         )
         self.assertEqual(returned_rule1["maximum_age"], rule1["maximum_age"])
         self.assertEqual(returned_rule1["action"], rule1["action"])
@@ -545,7 +545,7 @@ class TestAPIMailRules(DirectoriesMixin, APITestCase):
             filter_from="from@example.com",
             filter_subject="subject",
             filter_body="body",
-            filter_attachment_filename="file.pdf",
+            filter_attachment_filename_include="file.pdf",
             maximum_age=30,
             action=MailRule.MailAction.MARK_READ,
             assign_title_from=MailRule.TitleSource.FROM_SUBJECT,
@@ -589,7 +589,7 @@ class TestAPIMailRules(DirectoriesMixin, APITestCase):
             filter_from="from@example.com",
             filter_subject="subject",
             filter_body="body",
-            filter_attachment_filename="file.pdf",
+            filter_attachment_filename_include="file.pdf",
             maximum_age=30,
             action=MailRule.MailAction.MARK_READ,
             assign_title_from=MailRule.TitleSource.FROM_SUBJECT,

src/paperless_mail/tests/test_mail.py

@@ -526,6 +526,16 @@ class TestMail(
         )
 
     def test_filename_filter(self):
+        """
+        GIVEN:
+            - Email with multiple similar named attachments
+            - Rule with inclusive and exclusive filters
+        WHEN:
+            - Mail action filtering is checked
+        THEN:
+            - Mail action should not be performed for files excluded
+            - Mail action should be performed for files included
+        """
         message = self.create_message(
             attachments=[
                 _AttachmentDef(filename="f1.pdf"),
@@ -537,15 +547,67 @@ class TestMail(
             ],
         )
 
+        @dataclasses.dataclass(frozen=True)
+        class FilterTestCase:
+            name: str
+            include_pattern: Optional[str]
+            exclude_pattern: Optional[str]
+            expected_matches: list[str]
+
         tests = [
-            ("*.pdf", ["f1.pdf", "f2.pdf", "f3.pdf", "file.PDf", "f1.Pdf"]),
+            FilterTestCase(
-            ("f1.pdf", ["f1.pdf", "f1.Pdf"]),
+                "PDF Wildcard",
-            ("*", ["f1.pdf", "f2.pdf", "f3.pdf", "f2.png", "file.PDf", "f1.Pdf"]),
+                include_pattern="*.pdf",
-            ("*.png", ["f2.png"]),
+                exclude_pattern=None,
+                expected_matches=["f1.pdf", "f2.pdf", "f3.pdf", "file.PDf", "f1.Pdf"],
+            ),
+            FilterTestCase(
+                "F1 PDF Only",
+                include_pattern="f1.pdf",
+                exclude_pattern=None,
+                expected_matches=["f1.pdf", "f1.Pdf"],
+            ),
+            FilterTestCase(
+                "All Files",
+                include_pattern="*",
+                exclude_pattern=None,
+                expected_matches=[
+                    "f1.pdf",
+                    "f2.pdf",
+                    "f3.pdf",
+                    "f2.png",
+                    "file.PDf",
+                    "f1.Pdf",
+                ],
+            ),
+            FilterTestCase(
+                "PNG Only",
+                include_pattern="*.png",
+                exclude_pattern=None,
+                expected_matches=["f2.png"],
+            ),
+            FilterTestCase(
+                "PDF Files without f1",
+                include_pattern="*.pdf",
+                exclude_pattern="f1*",
+                expected_matches=["f2.pdf", "f3.pdf", "file.PDf"],
+            ),
+            FilterTestCase(
+                "All Files, no PNG",
+                include_pattern="*",
+                exclude_pattern="*.png",
+                expected_matches=[
+                    "f1.pdf",
+                    "f2.pdf",
+                    "f3.pdf",
+                    "file.PDf",
+                    "f1.Pdf",
+                ],
+            ),
         ]
 
-        for pattern, matches in tests:
+        for test_case in tests:
-            with self.subTest(msg=pattern):
+            with self.subTest(msg=test_case.name):
                 self._queue_consumption_tasks_mock.reset_mock()
                 account = MailAccount(name=str(uuid.uuid4()))
                 account.save()
@@ -553,14 +615,15 @@ class TestMail(
                     name=str(uuid.uuid4()),
                     assign_title_from=MailRule.TitleSource.FROM_FILENAME,
                     account=account,
-                    filter_attachment_filename=pattern,
+                    filter_attachment_filename_include=test_case.include_pattern,
+                    filter_attachment_filename_exclude=test_case.exclude_pattern,
                 )
                 rule.save()
 
                 self.mail_account_handler._handle_message(message, rule)
                 self.assert_queue_consumption_tasks_call_args(
                     [
-                        [{"override_filename": m} for m in matches],
+                        [{"override_filename": m} for m in test_case.expected_matches],
                     ],
                 )
 
@@ -593,7 +656,7 @@ class TestMail(
             name=str(uuid.uuid4()),
             assign_title_from=MailRule.TitleSource.FROM_FILENAME,
             account=account,
-            filter_attachment_filename="*.pdf",
+            filter_attachment_filename_include="*.pdf",
             attachment_type=MailRule.AttachmentProcessing.EVERYTHING,
             action=MailRule.MailAction.DELETE,
         )