Merge remote-tracking branch 'paperless/dev' into feature-consume-eml

src/paperless_mail/admin.py

@@ -83,7 +83,7 @@ class MailRuleAdmin(admin.ModelAdmin):
                 ),
                 "fields": (
                     "assign_title_from",
-                    "assign_tag",
+                    "assign_tags",
                     "assign_document_type",
                     "assign_correspondent_from",
                     "assign_correspondent",

src/paperless_mail/mail.py

@@ -3,6 +3,7 @@ import tempfile
 from datetime import date
 from datetime import timedelta
 from fnmatch import fnmatch
+from imaplib import IMAP4
 
 import magic
 import pathvalidate
@@ -145,7 +146,7 @@ class MailAccountHandler(LoggingMixin):
 
         else:
             raise NotImplementedError(
-                "Unknwown correspondent selector",
+                "Unknown correspondent selector",
             )  # pragma: nocover
 
     def handle_mail_account(self, account):
@@ -161,8 +162,34 @@ class MailAccountHandler(LoggingMixin):
                 account.imap_port,
                 account.imap_security,
             ) as M:
+
                 try:
                     M.login(account.username, account.password)
+
+                except UnicodeEncodeError:
+                    self.log("debug", "Falling back to AUTH=PLAIN")
+                    try:
+                        # rfc2595 section 6 - PLAIN SASL mechanism
+                        client: IMAP4 = M.client
+                        encoded = (
+                            b"\0"
+                            + account.username.encode("utf8")
+                            + b"\0"
+                            + account.password.encode("utf8")
+                        )
+                        # Assumption is the server supports AUTH=PLAIN capability
+                        # Could check the list with client.capability(), but then what?
+                        # We're failing anyway then
+                        client.authenticate("PLAIN", lambda x: encoded)
+
+                        # Need to transition out of AUTH state to SELECTED
+                        M.folder.set("INBOX")
+                    except Exception:
+                        self.log(
+                            "error",
+                            "Unable to authenticate with mail server using AUTH=PLAIN",
+                        )
+                        raise MailError(f"Error while authenticating account {account}")
                 except Exception as e:
                     self.log(
                         "error",
@@ -199,13 +226,28 @@ class MailAccountHandler(LoggingMixin):
 
         return total_processed_files
 
-    def handle_mail_rule(self, M, rule: MailRule):
+    def handle_mail_rule(self, M: MailBox, rule: MailRule):
 
         self.log("debug", f"Rule {rule}: Selecting folder {rule.folder}")
 
         try:
             M.folder.set(rule.folder)
         except MailboxFolderSelectError:
+
+            self.log(
+                "error",
+                f"Unable to access folder {rule.folder}, attempting folder listing",
+            )
+            try:
+                for folder_info in M.folder.list():
+                    self.log("info", f"Located folder: {folder_info.name}")
+            except Exception as e:
+                self.log(
+                    "error",
+                    "Exception during folder listing, unable to provide list folders: "
+                    + str(e),
+                )
+
             raise MailError(
                 f"Rule {rule}: Folder {rule.folder} "
                 f"does not exist in account {rule.account}",
@@ -284,7 +326,7 @@ class MailAccountHandler(LoggingMixin):
         )
 
         correspondent = self.get_correspondent(message, rule)
-        tag = rule.assign_tag
+        tag_ids = [tag.id for tag in rule.assign_tags.all()]
         doc_type = rule.assign_document_type
 
         processed_attachments = 0
@@ -317,7 +359,7 @@ class MailAccountHandler(LoggingMixin):
                 override_title=message.subject,
                 override_correspondent_id=correspondent.id if correspondent else None,
                 override_document_type_id=doc_type.id if doc_type else None,
-                override_tag_ids=[tag.id] if tag else None,
+                override_tag_ids=tag_ids,
                 task_name=message.subject[:100],
             )
             processed_attachments += 1
@@ -384,7 +426,7 @@ class MailAccountHandler(LoggingMixin):
                         if correspondent
                         else None,
                         override_document_type_id=doc_type.id if doc_type else None,
-                        override_tag_ids=[tag.id] if tag else None,
+                        override_tag_ids=tag_ids,
                         task_name=att.filename[:100],
                     )
 

src/paperless_mail/migrations/0009_mailrule_assign_tags.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.2.12 on 2022-03-11 15:00
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("paperless_mail", "0008_auto_20210516_0940"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="mailrule",
+            name="assign_tags",
+            field=models.ManyToManyField(
+                blank=True,
+                related_name="mail_rules_multi",
+                to="documents.Tag",
+                verbose_name="assign this tag",
+            ),
+        ),
+    ]

src/paperless_mail/migrations/0010_auto_20220311_1602.py

@@ -0,0 +1,40 @@
+# Generated by Django 3.2.12 on 2022-03-11 15:02
+
+from django.db import migrations
+
+
+def migrate_tag_to_tags(apps, schema_editor):
+    # Manual data migration, see
+    # https://docs.djangoproject.com/en/3.2/topics/migrations/#data-migrations
+    #
+    # Copy the assign_tag property to the new assign_tags set if it exists.
+    MailRule = apps.get_model("paperless_mail", "MailRule")
+    for mail_rule in MailRule.objects.all():
+        if mail_rule.assign_tag:
+            mail_rule.assign_tags.add(mail_rule.assign_tag)
+            mail_rule.save()
+
+
+def migrate_tags_to_tag(apps, schema_editor):
+    # Manual data migration, see
+    # https://docs.djangoproject.com/en/3.2/topics/migrations/#data-migrations
+    #
+    # Copy the unique value in the assign_tags set to the old assign_tag property.
+    # Do nothing if the tag is not unique.
+    MailRule = apps.get_model("paperless_mail", "MailRule")
+    for mail_rule in MailRule.objects.all():
+        tags = mail_rule.assign_tags.all()
+        if len(tags) == 1:
+            mail_rule.assign_tag = tags[0]
+            mail_rule.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("paperless_mail", "0009_mailrule_assign_tags"),
+    ]
+
+    operations = [
+        migrations.RunPython(migrate_tag_to_tags, migrate_tags_to_tag),
+    ]

src/paperless_mail/migrations/0011_remove_mailrule_assign_tag.py

@@ -0,0 +1,17 @@
+# Generated by Django 3.2.12 on 2022-03-11 15:18
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("paperless_mail", "0010_auto_20220311_1602"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="mailrule",
+            name="assign_tag",
+        ),
+    ]

src/paperless_mail/migrations/0012_alter_mailrule_assign_tags.py

@@ -0,0 +1,20 @@
+# Generated by Django 3.2.12 on 2022-03-11 16:21
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("paperless_mail", "0011_remove_mailrule_assign_tag"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="mailrule",
+            name="assign_tags",
+            field=models.ManyToManyField(
+                blank=True, to="documents.Tag", verbose_name="assign this tag"
+            ),
+        ),
+    ]

src/paperless_mail/migrations/0013_merge_20220412_1051.py

@@ -0,0 +1,13 @@
+# Generated by Django 4.0.4 on 2022-04-12 08:51
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("paperless_mail", "0009_alter_mailrule_action_alter_mailrule_folder"),
+        ("paperless_mail", "0012_alter_mailrule_assign_tags"),
+    ]
+
+    operations = []

src/paperless_mail/migrations/0014_alter_mailrule_action.py

@@ -0,0 +1,27 @@
+# Generated by Django 4.0.4 on 2022-04-18 22:57
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("paperless_mail", "0013_merge_20220412_1051"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="mailrule",
+            name="action",
+            field=models.PositiveIntegerField(
+                choices=[
+                    (1, "Delete"),
+                    (2, "Move to specified folder"),
+                    (3, "Mark as read, don't process read mails"),
+                    (4, "Flag the mail, don't process flagged mails"),
+                ],
+                default=3,
+                verbose_name="action",
+            ),
+        ),
+    ]

src/paperless_mail/models.py

@@ -183,11 +183,9 @@ class MailRule(models.Model):
         default=TitleSource.FROM_SUBJECT,
     )
 
-    assign_tag = models.ForeignKey(
+    assign_tags = models.ManyToManyField(
         document_models.Tag,
-        null=True,
         blank=True,
-        on_delete=models.SET_NULL,
         verbose_name=_("assign this tag"),
     )
 

src/paperless_mail/tests/test_mail.py

@@ -15,7 +15,9 @@ from django.test import TestCase
 from documents.models import Correspondent
 from documents.tests.utils import DirectoriesMixin
 from imap_tools import EmailAddress
+from imap_tools import FolderInfo
 from imap_tools import MailboxFolderSelectError
+from imap_tools import MailboxLoginError
 from imap_tools import MailMessage
 from imap_tools import MailMessageFlags
 from paperless_mail import tasks
@@ -43,6 +45,14 @@ class BogusFolderManager:
         self.current_folder = new_folder
 
 
+class BogusClient(object):
+    def authenticate(self, mechanism, authobject):
+        # authobject must be a callable object
+        auth_bytes = authobject(None)
+        if auth_bytes != b"\x00admin\x00w57\xc3\xa4\xc3\xb6\xc3\xbcw4b6huwb6nhu":
+            raise MailboxLoginError("BAD", "OK")
+
+
 class BogusMailBox(ContextManager):
     def __enter__(self):
         return self
@@ -53,12 +63,15 @@ class BogusMailBox(ContextManager):
     def __init__(self):
         self.messages: List[MailMessage] = []
         self.messages_spam: List[MailMessage] = []
+        self.folder = BogusFolderManager()
+        self.client = BogusClient()
 
     def login(self, username, password):
-        if not (username == "admin" and password == "secret"):
-            raise Exception()
-
-    folder = BogusFolderManager()
+        # This will raise a UnicodeEncodeError if the password is not ASCII only
+        password.encode("ascii")
+        # Otherwise, check for correct values
+        if username != "admin" or password not in {"secret"}:
+            raise MailboxLoginError("BAD", "OK")
 
     def fetch(self, criteria, mark_seen, charset=""):
         msg = self.messages
@@ -228,7 +241,6 @@ class TestMail(DirectoriesMixin, TestCase):
         message.from_values = EmailAddress(
             "Someone!",
             "someone@somewhere.com",
-            "Someone! <someone@somewhere.com>",
         )
 
         message2 = namedtuple("MailMessage", [])
@@ -236,7 +248,6 @@ class TestMail(DirectoriesMixin, TestCase):
         message2.from_values = EmailAddress(
             "",
             "fake@localhost.com",
-            "",
         )
 
         me_localhost = Correspondent.objects.create(name=message2.from_)
@@ -308,10 +319,12 @@ class TestMail(DirectoriesMixin, TestCase):
         )
 
         account = MailAccount()
+        account.save()
         rule = MailRule(
             assign_title_from=MailRule.TitleSource.FROM_FILENAME,
             account=account,
         )
+        rule.save()
 
         result = self.mail_account_handler.handle_message(message, rule)
 
@@ -355,10 +368,12 @@ class TestMail(DirectoriesMixin, TestCase):
         )
 
         account = MailAccount()
+        account.save()
         rule = MailRule(
             assign_title_from=MailRule.TitleSource.FROM_FILENAME,
             account=account,
         )
+        rule.save()
 
         result = self.mail_account_handler.handle_message(message, rule)
 
@@ -381,10 +396,12 @@ class TestMail(DirectoriesMixin, TestCase):
         )
 
         account = MailAccount()
+        account.save()
         rule = MailRule(
             assign_title_from=MailRule.TitleSource.FROM_FILENAME,
             account=account,
         )
+        rule.save()
 
         result = self.mail_account_handler.handle_message(message, rule)
 
@@ -406,11 +423,13 @@ class TestMail(DirectoriesMixin, TestCase):
         )
 
         account = MailAccount()
+        account.save()
         rule = MailRule(
             assign_title_from=MailRule.TitleSource.FROM_FILENAME,
             account=account,
             attachment_type=MailRule.AttachmentProcessing.EVERYTHING,
         )
+        rule.save()
 
         result = self.mail_account_handler.handle_message(message, rule)
 
@@ -440,12 +459,15 @@ class TestMail(DirectoriesMixin, TestCase):
         for (pattern, matches) in tests:
             matches.sort()
             self.async_task.reset_mock()
-            account = MailAccount()
+            account = MailAccount(name=str(uuid.uuid4()))
+            account.save()
             rule = MailRule(
+                name=str(uuid.uuid4()),
                 assign_title_from=MailRule.TitleSource.FROM_FILENAME,
                 account=account,
                 filter_attachment_filename=pattern,
             )
+            rule.save()
 
             result = self.mail_account_handler.handle_message(message, rule)
 
@@ -621,6 +643,72 @@ class TestMail(DirectoriesMixin, TestCase):
         self.assertEqual(len(self.bogus_mailbox.messages), 2)
         self.assertEqual(len(self.bogus_mailbox.messages_spam), 1)
 
+    def test_error_folder_set(self):
+        """
+        GIVEN:
+            - Mail rule with non-existent folder
+        THEN:
+            - Should call list to output all folders in the account
+            - Should not process any messages
+        """
+        account = MailAccount.objects.create(
+            name="test2",
+            imap_server="",
+            username="admin",
+            password="secret",
+        )
+        _ = MailRule.objects.create(
+            name="testrule",
+            account=account,
+            action=MailRule.MailAction.MOVE,
+            action_parameter="spam",
+            filter_subject="Claim",
+            order=1,
+            folder="uuuhhhh",  # Invalid folder name
+        )
+
+        self.bogus_mailbox.folder.list = mock.Mock(
+            return_value=[FolderInfo("SomeFoldername", "|", ())],
+        )
+
+        self.mail_account_handler.handle_mail_account(account)
+
+        self.bogus_mailbox.folder.list.assert_called_once()
+        self.assertEqual(self.async_task.call_count, 0)
+
+    def test_error_folder_set_error_listing(self):
+        """
+        GIVEN:
+            - Mail rule with non-existent folder
+            - Mail account folder listing raises exception
+        THEN:
+            - Should not process any messages
+        """
+        account = MailAccount.objects.create(
+            name="test2",
+            imap_server="",
+            username="admin",
+            password="secret",
+        )
+        _ = MailRule.objects.create(
+            name="testrule",
+            account=account,
+            action=MailRule.MailAction.MOVE,
+            action_parameter="spam",
+            filter_subject="Claim",
+            order=1,
+            folder="uuuhhhh",  # Invalid folder name
+        )
+
+        self.bogus_mailbox.folder.list = mock.Mock(
+            side_effect=MailboxFolderSelectError(None, "uhm"),
+        )
+
+        self.mail_account_handler.handle_mail_account(account)
+
+        self.bogus_mailbox.folder.list.assert_called_once()
+        self.assertEqual(self.async_task.call_count, 0)
+
     @mock.patch("paperless_mail.mail.MailAccountHandler.get_correspondent")
     def test_error_skip_mail(self, m):
         def get_correspondent_fake(message, rule):
@@ -744,6 +832,66 @@ class TestMail(DirectoriesMixin, TestCase):
         self.assertEqual(len(self.bogus_mailbox.messages), 2)
         self.assertEqual(self.async_task.call_count, 5)
 
+    def test_auth_plain_fallback(self):
+        """
+        GIVEN:
+            - Mail account with password containing non-ASCII characters
+        THEN:
+            - Should still authenticate to the mail account
+        """
+        account = MailAccount.objects.create(
+            name="test",
+            imap_server="",
+            username="admin",
+            # Note the non-ascii characters here
+            password="w57äöüw4b6huwb6nhu",
+        )
+
+        _ = MailRule.objects.create(
+            name="testrule",
+            account=account,
+            action=MailRule.MailAction.MARK_READ,
+        )
+
+        self.assertEqual(len(self.bogus_mailbox.messages), 3)
+        self.assertEqual(self.async_task.call_count, 0)
+        self.assertEqual(len(self.bogus_mailbox.fetch("UNSEEN", False)), 2)
+
+        self.mail_account_handler.handle_mail_account(account)
+
+        self.assertEqual(self.async_task.call_count, 2)
+        self.assertEqual(len(self.bogus_mailbox.fetch("UNSEEN", False)), 0)
+        self.assertEqual(len(self.bogus_mailbox.messages), 3)
+
+    def test_auth_plain_fallback_fails_still(self):
+        """
+        GIVEN:
+            - Mail account with password containing non-ASCII characters
+            - Incorrect password value
+        THEN:
+            - Should raise a MailError for the account
+        """
+        account = MailAccount.objects.create(
+            name="test",
+            imap_server="",
+            username="admin",
+            # Note the non-ascii characters here
+            # Passes the check in login, not in authenticate
+            password="réception",
+        )
+
+        _ = MailRule.objects.create(
+            name="testrule",
+            account=account,
+            action=MailRule.MailAction.MARK_READ,
+        )
+
+        self.assertRaises(
+            MailError,
+            self.mail_account_handler.handle_mail_account,
+            account,
+        )
+
 
 class TestManagementCommand(TestCase):
     @mock.patch(