Enhancement: prevent duplicate mail processing across rules (#12159)

.mypy-baseline.txt

@@ -2118,7 +2118,6 @@ src/paperless_mail/mail.py:0: error: Function is missing a return type annotatio
 src/paperless_mail/mail.py:0: error: Function is missing a return type annotation  [no-untyped-def]
 src/paperless_mail/mail.py:0: error: Function is missing a return type annotation  [no-untyped-def]
 src/paperless_mail/mail.py:0: error: Function is missing a return type annotation  [no-untyped-def]
-src/paperless_mail/mail.py:0: error: Function is missing a return type annotation  [no-untyped-def]
 src/paperless_mail/mail.py:0: error: Function is missing a type annotation for one or more arguments  [no-untyped-def]
 src/paperless_mail/mail.py:0: error: Function is missing a type annotation for one or more arguments  [no-untyped-def]
 src/paperless_mail/mail.py:0: error: Function is missing a type annotation for one or more arguments  [no-untyped-def]

src/paperless_mail/mail.py

@@ -536,6 +536,7 @@ class MailAccountHandler(LoggingMixin):
         self.log.debug(f"Processing mail account {account}")
 
         total_processed_files = 0
+        consumed_messages: set[tuple[str, str | None]] = set()
         try:
             with get_mailbox(
                 account.imap_server,
@@ -574,6 +575,7 @@ class MailAccountHandler(LoggingMixin):
                             M,
                             rule,
                             supports_gmail_labels=supports_gmail_labels,
+                            consumed_messages=consumed_messages,
                         )
                         if total_processed_files > 0 and rule.stop_processing:
                             self.log.debug(
@@ -605,7 +607,8 @@ class MailAccountHandler(LoggingMixin):
         rule: MailRule,
         *,
         supports_gmail_labels: bool,
-    ):
+        consumed_messages: set[tuple[str, str | None]],
+    ) -> int:
         folders = [rule.folder]
         # In case of MOVE, make sure also the destination exists
         if rule.action == MailRule.MailAction.MOVE:
@@ -652,11 +655,26 @@ class MailAccountHandler(LoggingMixin):
 
         mails_processed = 0
         total_processed_files = 0
+        rule_seen_messages: set[tuple[str, str | None]] = set()
 
         for message in messages:
             if TYPE_CHECKING:
                 assert isinstance(message, MailMessage)
 
+            message_key = (rule.folder, message.uid)
+            if message_key in rule_seen_messages:
+                self.log.debug(
+                    f"Skipping duplicate fetched mail '{message.uid}' subject '{message.subject}' from '{message.from_}'.",
+                )
+                continue
+            rule_seen_messages.add(message_key)
+
+            if message_key in consumed_messages:
+                self.log.debug(
+                    f"Skipping mail '{message.uid}' subject '{message.subject}' from '{message.from_}', already queued by a previous rule in this run.",
+                )
+                continue
+
             if ProcessedMail.objects.filter(
                 rule=rule,
                 uid=message.uid,
@@ -669,6 +687,8 @@ class MailAccountHandler(LoggingMixin):
 
             try:
                 processed_files = self._handle_message(message, rule)
+                if processed_files > 0:
+                    consumed_messages.add(message_key)
 
                 total_processed_files += processed_files
                 mails_processed += 1

src/paperless_mail/tests/test_mail.py

@@ -863,6 +863,82 @@ class TestMail(
 
         self.assertEqual(len(self.mailMocker.bogus_mailbox.messages), 0)
 
+    def test_handle_mail_account_overlapping_rules_only_first_consumes(self) -> None:
+        """
+        GIVEN:
+            - Multiple rules that match the same mail
+        WHEN:
+            - Mail account is processed
+        THEN:
+            - Only the first rule should be applied
+        """
+        account = MailAccount.objects.create(
+            name="test",
+            imap_server="",
+            username="admin",
+            password="secret",
+        )
+
+        first_rule = MailRule.objects.create(
+            name="testrule-first",
+            account=account,
+            action=MailRule.MailAction.DELETE,
+            filter_subject="Claim",
+            order=1,
+        )
+        _ = MailRule.objects.create(
+            name="testrule-second",
+            account=account,
+            action=MailRule.MailAction.DELETE,
+            filter_subject="Claim",
+            order=2,
+        )
+
+        self.mail_account_handler.handle_mail_account(account)
+        self.mailMocker.apply_mail_actions()
+
+        self.assertEqual(self.mailMocker._queue_consumption_tasks_mock.call_count, 1)
+        queued_rule = self.mailMocker._queue_consumption_tasks_mock.call_args.kwargs[
+            "rule"
+        ]
+        self.assertEqual(queued_rule.id, first_rule.id)
+
+    def test_handle_mail_account_skip_duplicate_uids_from_fetch(self) -> None:
+        """
+        GIVEN:
+            - Multiple mails with the same UID returned from the mailbox fetch method
+        WHEN:
+            - Mail account is processed
+        THEN:
+            - Only one of the mails should be processed, to avoid duplicate processing due to fetch issues
+        """
+        account = MailAccount.objects.create(
+            name="test",
+            imap_server="",
+            username="admin",
+            password="secret",
+        )
+        _ = MailRule.objects.create(
+            name="testrule",
+            account=account,
+            action=MailRule.MailAction.DELETE,
+            filter_subject="Duplicated mail",
+        )
+
+        duplicated_message = self.mailMocker.messageBuilder.create_message(
+            subject="Duplicated mail",
+        )
+        self.mailMocker.bogus_mailbox.messages = [
+            duplicated_message,
+            duplicated_message,
+        ]
+        self.mailMocker.bogus_mailbox.updateClient()
+
+        self.mail_account_handler.handle_mail_account(account)
+        self.mailMocker.apply_mail_actions()
+
+        self.assertEqual(self.mailMocker._queue_consumption_tasks_mock.call_count, 1)
+
     @pytest.mark.flaky(reruns=4)
     def test_handle_mail_account_flag(self) -> None:
         account = MailAccount.objects.create(