diff --git a/.mypy-baseline.txt b/.mypy-baseline.txt index 68d1cdd5f..12e169efd 100644 --- a/.mypy-baseline.txt +++ b/.mypy-baseline.txt @@ -2118,7 +2118,6 @@ src/paperless_mail/mail.py:0: error: Function is missing a return type annotatio src/paperless_mail/mail.py:0: error: Function is missing a return type annotation [no-untyped-def] src/paperless_mail/mail.py:0: error: Function is missing a return type annotation [no-untyped-def] src/paperless_mail/mail.py:0: error: Function is missing a return type annotation [no-untyped-def] -src/paperless_mail/mail.py:0: error: Function is missing a return type annotation [no-untyped-def] src/paperless_mail/mail.py:0: error: Function is missing a type annotation for one or more arguments [no-untyped-def] src/paperless_mail/mail.py:0: error: Function is missing a type annotation for one or more arguments [no-untyped-def] src/paperless_mail/mail.py:0: error: Function is missing a type annotation for one or more arguments [no-untyped-def] diff --git a/src/paperless_mail/mail.py b/src/paperless_mail/mail.py index f6a46a292..56eaefaad 100644 --- a/src/paperless_mail/mail.py +++ b/src/paperless_mail/mail.py @@ -536,6 +536,7 @@ class MailAccountHandler(LoggingMixin): self.log.debug(f"Processing mail account {account}") total_processed_files = 0 + consumed_messages: set[tuple[str, str | None]] = set() try: with get_mailbox( account.imap_server, @@ -574,6 +575,7 @@ class MailAccountHandler(LoggingMixin): M, rule, supports_gmail_labels=supports_gmail_labels, + consumed_messages=consumed_messages, ) if total_processed_files > 0 and rule.stop_processing: self.log.debug( @@ -605,7 +607,8 @@ class MailAccountHandler(LoggingMixin): rule: MailRule, *, supports_gmail_labels: bool, - ): + consumed_messages: set[tuple[str, str | None]], + ) -> int: folders = [rule.folder] # In case of MOVE, make sure also the destination exists if rule.action == MailRule.MailAction.MOVE: @@ -652,11 +655,26 @@ class MailAccountHandler(LoggingMixin): mails_processed = 0 total_processed_files = 0 + rule_seen_messages: set[tuple[str, str | None]] = set() for message in messages: if TYPE_CHECKING: assert isinstance(message, MailMessage) + message_key = (rule.folder, message.uid) + if message_key in rule_seen_messages: + self.log.debug( + f"Skipping duplicate fetched mail '{message.uid}' subject '{message.subject}' from '{message.from_}'.", + ) + continue + rule_seen_messages.add(message_key) + + if message_key in consumed_messages: + self.log.debug( + f"Skipping mail '{message.uid}' subject '{message.subject}' from '{message.from_}', already queued by a previous rule in this run.", + ) + continue + if ProcessedMail.objects.filter( rule=rule, uid=message.uid, @@ -669,6 +687,8 @@ class MailAccountHandler(LoggingMixin): try: processed_files = self._handle_message(message, rule) + if processed_files > 0: + consumed_messages.add(message_key) total_processed_files += processed_files mails_processed += 1 diff --git a/src/paperless_mail/tests/test_mail.py b/src/paperless_mail/tests/test_mail.py index b37d09ed4..162a764f2 100644 --- a/src/paperless_mail/tests/test_mail.py +++ b/src/paperless_mail/tests/test_mail.py @@ -863,6 +863,82 @@ class TestMail( self.assertEqual(len(self.mailMocker.bogus_mailbox.messages), 0) + def test_handle_mail_account_overlapping_rules_only_first_consumes(self) -> None: + """ + GIVEN: + - Multiple rules that match the same mail + WHEN: + - Mail account is processed + THEN: + - Only the first rule should be applied + """ + account = MailAccount.objects.create( + name="test", + imap_server="", + username="admin", + password="secret", + ) + + first_rule = MailRule.objects.create( + name="testrule-first", + account=account, + action=MailRule.MailAction.DELETE, + filter_subject="Claim", + order=1, + ) + _ = MailRule.objects.create( + name="testrule-second", + account=account, + action=MailRule.MailAction.DELETE, + filter_subject="Claim", + order=2, + ) + + self.mail_account_handler.handle_mail_account(account) + self.mailMocker.apply_mail_actions() + + self.assertEqual(self.mailMocker._queue_consumption_tasks_mock.call_count, 1) + queued_rule = self.mailMocker._queue_consumption_tasks_mock.call_args.kwargs[ + "rule" + ] + self.assertEqual(queued_rule.id, first_rule.id) + + def test_handle_mail_account_skip_duplicate_uids_from_fetch(self) -> None: + """ + GIVEN: + - Multiple mails with the same UID returned from the mailbox fetch method + WHEN: + - Mail account is processed + THEN: + - Only one of the mails should be processed, to avoid duplicate processing due to fetch issues + """ + account = MailAccount.objects.create( + name="test", + imap_server="", + username="admin", + password="secret", + ) + _ = MailRule.objects.create( + name="testrule", + account=account, + action=MailRule.MailAction.DELETE, + filter_subject="Duplicated mail", + ) + + duplicated_message = self.mailMocker.messageBuilder.create_message( + subject="Duplicated mail", + ) + self.mailMocker.bogus_mailbox.messages = [ + duplicated_message, + duplicated_message, + ] + self.mailMocker.bogus_mailbox.updateClient() + + self.mail_account_handler.handle_mail_account(account) + self.mailMocker.apply_mail_actions() + + self.assertEqual(self.mailMocker._queue_consumption_tasks_mock.call_count, 1) + @pytest.mark.flaky(reruns=4) def test_handle_mail_account_flag(self) -> None: account = MailAccount.objects.create(