Merge remote-tracking branch 'paperless/dev' into feature-consume-eml

src/documents/tasks.py

@@ -22,6 +22,8 @@ from documents.models import Tag
 from documents.sanity_checker import SanityCheckFailedException
 from pdf2image import convert_from_path
 from pikepdf import Pdf
+from PIL import Image
+from PIL import ImageSequence
 from pyzbar import pyzbar
 from whoosh.writing import AsyncWriter
 
@@ -93,9 +95,41 @@ def barcode_reader(image) -> List[str]:
     return barcodes
 
 
+def convert_from_tiff_to_pdf(filepath: str) -> str:
+    """
+    converts a given TIFF image file to pdf into a temp. directory.
+    Returns the new pdf file.
+    """
+    file_name = os.path.splitext(os.path.basename(filepath))[0]
+    file_extension = os.path.splitext(os.path.basename(filepath))[1].lower()
+    tempdir = tempfile.mkdtemp(prefix="paperless-", dir=settings.SCRATCH_DIR)
+    # use old file name with pdf extension
+    if file_extension == ".tif" or file_extension == ".tiff":
+        newpath = os.path.join(tempdir, file_name + ".pdf")
+    else:
+        logger.warning(f"Cannot convert from {str(file_extension)} to pdf.")
+        return None
+    with Image.open(filepath) as image:
+        images = []
+        for i, page in enumerate(ImageSequence.Iterator(image)):
+            page = page.convert("RGB")
+            images.append(page)
+        try:
+            if len(images) == 1:
+                images[0].save(newpath)
+            else:
+                images[0].save(newpath, save_all=True, append_images=images[1:])
+        except OSError as e:
+            logger.warning(
+                f"Could not save the file as pdf. Error: {str(e)}",
+            )
+            return None
+    return newpath
+
+
 def scan_file_for_separating_barcodes(filepath: str) -> List[int]:
     """
-    Scan the provided file for page separating barcodes
+    Scan the provided pdf file for page separating barcodes
     Returns a list of pagenumbers, which separate the file
     """
     separator_page_numbers = []
@@ -112,7 +146,7 @@ def scan_file_for_separating_barcodes(filepath: str) -> List[int]:
 
 def separate_pages(filepath: str, pages_to_split_on: List[int]) -> List[str]:
     """
-    Separate the provided file on the pages_to_split_on.
+    Separate the provided pdf file on the pages_to_split_on.
     The pages which are defined by page_numbers will be removed.
     Returns a list of (temporary) filepaths to consume.
     These will need to be deleted later.
@@ -195,42 +229,70 @@ def consume_file(
     if settings.CONSUMER_ENABLE_BARCODES:
         separators = []
         document_list = []
-        separators = scan_file_for_separating_barcodes(path)
-        if separators:
-            logger.debug(f"Pages with separators found in: {str(path)}")
-            document_list = separate_pages(path, separators)
-        if document_list:
-            for n, document in enumerate(document_list):
-                # save to consumption dir
-                # rename it to the original filename  with number prefix
-                if override_filename:
-                    newname = f"{str(n)}_" + override_filename
-                else:
-                    newname = None
-                save_to_dir(document, newname=newname)
-            # if we got here, the document was successfully split
-            # and can safely be deleted
-            logger.debug("Deleting file {}".format(path))
-            os.unlink(path)
-            # notify the sender, otherwise the progress bar
-            # in the UI stays stuck
-            payload = {
-                "filename": override_filename,
-                "task_id": task_id,
-                "current_progress": 100,
-                "max_progress": 100,
-                "status": "SUCCESS",
-                "message": "finished",
-            }
-            try:
-                async_to_sync(get_channel_layer().group_send)(
-                    "status_updates",
-                    {"type": "status_update", "data": payload},
+        converted_tiff = None
+        if settings.CONSUMER_BARCODE_TIFF_SUPPORT:
+            supported_extensions = [".pdf", ".tiff", ".tif"]
+        else:
+            supported_extensions = [".pdf"]
+        file_extension = os.path.splitext(os.path.basename(path))[1].lower()
+        if file_extension not in supported_extensions:
+            # if not supported, skip this routine
+            logger.warning(
+                f"Unsupported file format for barcode reader: {str(file_extension)}",
+            )
+        else:
+            if file_extension in {".tif", ".tiff"}:
+                file_to_process = convert_from_tiff_to_pdf(path)
+            else:
+                file_to_process = path
+
+            separators = scan_file_for_separating_barcodes(file_to_process)
+
+            if separators:
+                logger.debug(
+                    f"Pages with separators found in: {str(path)}",
                 )
-            except OSError as e:
-                logger.warning("OSError. It could be, the broker cannot be reached.")
-                logger.warning(str(e))
-            return "File successfully split"
+                document_list = separate_pages(file_to_process, separators)
+
+            if document_list:
+                for n, document in enumerate(document_list):
+                    # save to consumption dir
+                    # rename it to the original filename  with number prefix
+                    if override_filename:
+                        newname = f"{str(n)}_" + override_filename
+                    else:
+                        newname = None
+                    save_to_dir(document, newname=newname)
+                # if we got here, the document was successfully split
+                # and can safely be deleted
+                if converted_tiff:
+                    logger.debug("Deleting file {}".format(file_to_process))
+                    os.unlink(file_to_process)
+                logger.debug("Deleting file {}".format(path))
+                os.unlink(path)
+                # notify the sender, otherwise the progress bar
+                # in the UI stays stuck
+                payload = {
+                    "filename": override_filename,
+                    "task_id": task_id,
+                    "current_progress": 100,
+                    "max_progress": 100,
+                    "status": "SUCCESS",
+                    "message": "finished",
+                }
+                try:
+                    async_to_sync(get_channel_layer().group_send)(
+                        "status_updates",
+                        {"type": "status_update", "data": payload},
+                    )
+                except OSError as e:
+                    logger.warning(
+                        "OSError. It could be, the broker cannot be reached.",
+                    )
+                    logger.warning(str(e))
+                # consuming stops here, since the original document with
+                # the barcodes has been split and will be consumed separately
+                return "File successfully split"
 
     # continue with consumption if no barcode was found
     document = Consumer().try_consume_file(

src/documents/tests/test_tasks.py

@@ -204,6 +204,29 @@ class TestTasks(DirectoriesMixin, TestCase):
         img = Image.open(test_file)
         self.assertEqual(tasks.barcode_reader(img), ["CUSTOM BARCODE"])
 
+    def test_convert_from_tiff_to_pdf(self):
+        test_file = os.path.join(
+            os.path.dirname(__file__),
+            "samples",
+            "simple.tiff",
+        )
+        dst = os.path.join(settings.SCRATCH_DIR, "simple.tiff")
+        shutil.copy(test_file, dst)
+        target_file = tasks.convert_from_tiff_to_pdf(dst)
+        file_extension = os.path.splitext(os.path.basename(target_file))[1]
+        self.assertTrue(os.path.isfile(target_file))
+        self.assertEqual(file_extension, ".pdf")
+
+    def test_convert_error_from_pdf_to_pdf(self):
+        test_file = os.path.join(
+            os.path.dirname(__file__),
+            "samples",
+            "simple.pdf",
+        )
+        dst = os.path.join(settings.SCRATCH_DIR, "simple.pdf")
+        shutil.copy(test_file, dst)
+        self.assertIsNone(tasks.convert_from_tiff_to_pdf(dst))
+
     def test_scan_file_for_separating_barcodes(self):
         test_file = os.path.join(
             os.path.dirname(__file__),
@@ -400,11 +423,64 @@ class TestTasks(DirectoriesMixin, TestCase):
             "barcodes",
             "patch-code-t-middle.pdf",
         )
-        dst = os.path.join(settings.SCRATCH_DIR, "patch-code-t-middle.pd")
+        dst = os.path.join(settings.SCRATCH_DIR, "patch-code-t-middle.pdf")
         shutil.copy(test_file, dst)
 
         self.assertEqual(tasks.consume_file(dst), "File successfully split")
 
+    @override_settings(
+        CONSUMER_ENABLE_BARCODES=True,
+        CONSUMER_BARCODE_TIFF_SUPPORT=True,
+    )
+    def test_consume_barcode_tiff_file(self):
+        test_file = os.path.join(
+            os.path.dirname(__file__),
+            "samples",
+            "barcodes",
+            "patch-code-t-middle.tiff",
+        )
+        dst = os.path.join(settings.SCRATCH_DIR, "patch-code-t-middle.tiff")
+        shutil.copy(test_file, dst)
+
+        self.assertEqual(tasks.consume_file(dst), "File successfully split")
+
+    @override_settings(
+        CONSUMER_ENABLE_BARCODES=True,
+        CONSUMER_BARCODE_TIFF_SUPPORT=True,
+    )
+    @mock.patch("documents.consumer.Consumer.try_consume_file")
+    def test_consume_barcode_unsupported_jpg_file(self, m):
+        """
+        This test assumes barcode and TIFF support are enabled and
+        the user uploads an unsupported image file (e.g. jpg)
+
+        The function shouldn't try to scan for separating barcodes
+        and continue archiving the file as is.
+        """
+        test_file = os.path.join(
+            os.path.dirname(__file__),
+            "samples",
+            "simple.jpg",
+        )
+        dst = os.path.join(settings.SCRATCH_DIR, "simple.jpg")
+        shutil.copy(test_file, dst)
+        with self.assertLogs("paperless.tasks", level="WARNING") as cm:
+            self.assertIn("Success", tasks.consume_file(dst))
+        self.assertEqual(
+            cm.output,
+            [
+                "WARNING:paperless.tasks:Unsupported file format for barcode reader: .jpg",
+            ],
+        )
+        m.assert_called_once()
+
+        args, kwargs = m.call_args
+        self.assertIsNone(kwargs["override_filename"])
+        self.assertIsNone(kwargs["override_title"])
+        self.assertIsNone(kwargs["override_correspondent_id"])
+        self.assertIsNone(kwargs["override_document_type_id"])
+        self.assertIsNone(kwargs["override_tag_ids"])
+
     @mock.patch("documents.tasks.sanity_checker.check_sanity")
     def test_sanity_check_success(self, m):
         m.return_value = SanityCheckMessages()

src/locale/pl_PL/LC_MESSAGES/django.po

@@ -3,7 +3,7 @@ msgstr ""
 "Project-Id-Version: paperless-ngx\n"
 "Report-Msgid-Bugs-To: \n"
 "POT-Creation-Date: 2022-03-02 11:20-0800\n"
-"PO-Revision-Date: 2022-03-14 23:41\n"
+"PO-Revision-Date: 2022-04-12 15:26\n"
 "Last-Translator: \n"
 "Language-Team: Polish\n"
 "Language: pl_PL\n"
@@ -60,7 +60,7 @@ msgstr "algorytm dopasowania"
 
 #: documents/models.py:48
 msgid "is insensitive"
-msgstr "bez rozróżniania wielkości liter"
+msgstr "bez rozróżniania wielkości znaków"
 
 #: documents/models.py:61 documents/models.py:104
 msgid "correspondent"
@@ -240,7 +240,7 @@ msgstr "użytkownik"
 
 #: documents/models.py:317
 msgid "show on dashboard"
-msgstr "pokaż na pulpicie"
+msgstr "pokaż na stronie głównej"
 
 #: documents/models.py:320
 msgid "show in sidebar"
@@ -638,7 +638,7 @@ msgstr "konto"
 
 #: paperless_mail/models.py:119
 msgid "folder"
-msgstr "katalog"
+msgstr "folder"
 
 #: paperless_mail/models.py:122
 msgid "Subfolders must be separated by dots."

src/paperless/settings.py

@@ -251,7 +251,8 @@ if _paperless_url:
     if _allowed_hosts:
         ALLOWED_HOSTS.append(_paperless_uri.hostname)
     else:
-        ALLOWED_HOSTS = [_paperless_uri.hostname]
+        # always allow localhost. Necessary e.g. for healthcheck in docker.
+        ALLOWED_HOSTS = [_paperless_uri.hostname] + ["localhost"]
 
 # The secret key has a default that should be fine so long as you're hosting
 # Paperless on a closed network.  However, if you're putting this anywhere
@@ -502,6 +503,10 @@ CONSUMER_ENABLE_BARCODES = __get_boolean(
     "PAPERLESS_CONSUMER_ENABLE_BARCODES",
 )
 
+CONSUMER_BARCODE_TIFF_SUPPORT = __get_boolean(
+    "PAPERLESS_CONSUMER_BARCODE_TIFF_SUPPORT",
+)
+
 CONSUMER_BARCODE_STRING = os.getenv("PAPERLESS_CONSUMER_BARCODE_STRING", "PATCHT")
 
 OPTIMIZE_THUMBNAILS = __get_boolean("PAPERLESS_OPTIMIZE_THUMBNAILS", "true")

src/paperless/version.py

@@ -1 +1 @@
-__version__ = (1, 6, 0)
+__version__ = (1, 7, 0)

src/paperless_mail/admin.py

@@ -83,7 +83,7 @@ class MailRuleAdmin(admin.ModelAdmin):
                 ),
                 "fields": (
                     "assign_title_from",
-                    "assign_tag",
+                    "assign_tags",
                     "assign_document_type",
                     "assign_correspondent_from",
                     "assign_correspondent",

src/paperless_mail/mail.py

@@ -3,6 +3,7 @@ import tempfile
 from datetime import date
 from datetime import timedelta
 from fnmatch import fnmatch
+from imaplib import IMAP4
 
 import magic
 import pathvalidate
@@ -145,7 +146,7 @@ class MailAccountHandler(LoggingMixin):
 
         else:
             raise NotImplementedError(
-                "Unknwown correspondent selector",
+                "Unknown correspondent selector",
             )  # pragma: nocover
 
     def handle_mail_account(self, account):
@@ -161,8 +162,34 @@ class MailAccountHandler(LoggingMixin):
                 account.imap_port,
                 account.imap_security,
             ) as M:
+
                 try:
                     M.login(account.username, account.password)
+
+                except UnicodeEncodeError:
+                    self.log("debug", "Falling back to AUTH=PLAIN")
+                    try:
+                        # rfc2595 section 6 - PLAIN SASL mechanism
+                        client: IMAP4 = M.client
+                        encoded = (
+                            b"\0"
+                            + account.username.encode("utf8")
+                            + b"\0"
+                            + account.password.encode("utf8")
+                        )
+                        # Assumption is the server supports AUTH=PLAIN capability
+                        # Could check the list with client.capability(), but then what?
+                        # We're failing anyway then
+                        client.authenticate("PLAIN", lambda x: encoded)
+
+                        # Need to transition out of AUTH state to SELECTED
+                        M.folder.set("INBOX")
+                    except Exception:
+                        self.log(
+                            "error",
+                            "Unable to authenticate with mail server using AUTH=PLAIN",
+                        )
+                        raise MailError(f"Error while authenticating account {account}")
                 except Exception as e:
                     self.log(
                         "error",
@@ -199,13 +226,28 @@ class MailAccountHandler(LoggingMixin):
 
         return total_processed_files
 
-    def handle_mail_rule(self, M, rule: MailRule):
+    def handle_mail_rule(self, M: MailBox, rule: MailRule):
 
         self.log("debug", f"Rule {rule}: Selecting folder {rule.folder}")
 
         try:
             M.folder.set(rule.folder)
         except MailboxFolderSelectError:
+
+            self.log(
+                "error",
+                f"Unable to access folder {rule.folder}, attempting folder listing",
+            )
+            try:
+                for folder_info in M.folder.list():
+                    self.log("info", f"Located folder: {folder_info.name}")
+            except Exception as e:
+                self.log(
+                    "error",
+                    "Exception during folder listing, unable to provide list folders: "
+                    + str(e),
+                )
+
             raise MailError(
                 f"Rule {rule}: Folder {rule.folder} "
                 f"does not exist in account {rule.account}",
@@ -284,7 +326,7 @@ class MailAccountHandler(LoggingMixin):
         )
 
         correspondent = self.get_correspondent(message, rule)
-        tag = rule.assign_tag
+        tag_ids = [tag.id for tag in rule.assign_tags.all()]
         doc_type = rule.assign_document_type
 
         processed_attachments = 0
@@ -317,7 +359,7 @@ class MailAccountHandler(LoggingMixin):
                 override_title=message.subject,
                 override_correspondent_id=correspondent.id if correspondent else None,
                 override_document_type_id=doc_type.id if doc_type else None,
-                override_tag_ids=[tag.id] if tag else None,
+                override_tag_ids=tag_ids,
                 task_name=message.subject[:100],
             )
             processed_attachments += 1
@@ -384,7 +426,7 @@ class MailAccountHandler(LoggingMixin):
                         if correspondent
                         else None,
                         override_document_type_id=doc_type.id if doc_type else None,
-                        override_tag_ids=[tag.id] if tag else None,
+                        override_tag_ids=tag_ids,
                         task_name=att.filename[:100],
                     )
 

src/paperless_mail/migrations/0009_mailrule_assign_tags.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.2.12 on 2022-03-11 15:00
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("paperless_mail", "0008_auto_20210516_0940"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="mailrule",
+            name="assign_tags",
+            field=models.ManyToManyField(
+                blank=True,
+                related_name="mail_rules_multi",
+                to="documents.Tag",
+                verbose_name="assign this tag",
+            ),
+        ),
+    ]

src/paperless_mail/migrations/0010_auto_20220311_1602.py

@@ -0,0 +1,40 @@
+# Generated by Django 3.2.12 on 2022-03-11 15:02
+
+from django.db import migrations
+
+
+def migrate_tag_to_tags(apps, schema_editor):
+    # Manual data migration, see
+    # https://docs.djangoproject.com/en/3.2/topics/migrations/#data-migrations
+    #
+    # Copy the assign_tag property to the new assign_tags set if it exists.
+    MailRule = apps.get_model("paperless_mail", "MailRule")
+    for mail_rule in MailRule.objects.all():
+        if mail_rule.assign_tag:
+            mail_rule.assign_tags.add(mail_rule.assign_tag)
+            mail_rule.save()
+
+
+def migrate_tags_to_tag(apps, schema_editor):
+    # Manual data migration, see
+    # https://docs.djangoproject.com/en/3.2/topics/migrations/#data-migrations
+    #
+    # Copy the unique value in the assign_tags set to the old assign_tag property.
+    # Do nothing if the tag is not unique.
+    MailRule = apps.get_model("paperless_mail", "MailRule")
+    for mail_rule in MailRule.objects.all():
+        tags = mail_rule.assign_tags.all()
+        if len(tags) == 1:
+            mail_rule.assign_tag = tags[0]
+            mail_rule.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("paperless_mail", "0009_mailrule_assign_tags"),
+    ]
+
+    operations = [
+        migrations.RunPython(migrate_tag_to_tags, migrate_tags_to_tag),
+    ]

src/paperless_mail/migrations/0011_remove_mailrule_assign_tag.py

@@ -0,0 +1,17 @@
+# Generated by Django 3.2.12 on 2022-03-11 15:18
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("paperless_mail", "0010_auto_20220311_1602"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="mailrule",
+            name="assign_tag",
+        ),
+    ]

src/paperless_mail/migrations/0012_alter_mailrule_assign_tags.py

@@ -0,0 +1,20 @@
+# Generated by Django 3.2.12 on 2022-03-11 16:21
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("paperless_mail", "0011_remove_mailrule_assign_tag"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="mailrule",
+            name="assign_tags",
+            field=models.ManyToManyField(
+                blank=True, to="documents.Tag", verbose_name="assign this tag"
+            ),
+        ),
+    ]

src/paperless_mail/migrations/0013_merge_20220412_1051.py

@@ -0,0 +1,13 @@
+# Generated by Django 4.0.4 on 2022-04-12 08:51
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("paperless_mail", "0009_alter_mailrule_action_alter_mailrule_folder"),
+        ("paperless_mail", "0012_alter_mailrule_assign_tags"),
+    ]
+
+    operations = []

src/paperless_mail/migrations/0014_alter_mailrule_action.py

@@ -0,0 +1,27 @@
+# Generated by Django 4.0.4 on 2022-04-18 22:57
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("paperless_mail", "0013_merge_20220412_1051"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="mailrule",
+            name="action",
+            field=models.PositiveIntegerField(
+                choices=[
+                    (1, "Delete"),
+                    (2, "Move to specified folder"),
+                    (3, "Mark as read, don't process read mails"),
+                    (4, "Flag the mail, don't process flagged mails"),
+                ],
+                default=3,
+                verbose_name="action",
+            ),
+        ),
+    ]

src/paperless_mail/models.py

@@ -183,11 +183,9 @@ class MailRule(models.Model):
         default=TitleSource.FROM_SUBJECT,
     )
 
-    assign_tag = models.ForeignKey(
+    assign_tags = models.ManyToManyField(
         document_models.Tag,
-        null=True,
         blank=True,
-        on_delete=models.SET_NULL,
         verbose_name=_("assign this tag"),
     )
 

src/paperless_mail/tests/test_mail.py

@@ -15,7 +15,9 @@ from django.test import TestCase
 from documents.models import Correspondent
 from documents.tests.utils import DirectoriesMixin
 from imap_tools import EmailAddress
+from imap_tools import FolderInfo
 from imap_tools import MailboxFolderSelectError
+from imap_tools import MailboxLoginError
 from imap_tools import MailMessage
 from imap_tools import MailMessageFlags
 from paperless_mail import tasks
@@ -43,6 +45,14 @@ class BogusFolderManager:
         self.current_folder = new_folder
 
 
+class BogusClient(object):
+    def authenticate(self, mechanism, authobject):
+        # authobject must be a callable object
+        auth_bytes = authobject(None)
+        if auth_bytes != b"\x00admin\x00w57\xc3\xa4\xc3\xb6\xc3\xbcw4b6huwb6nhu":
+            raise MailboxLoginError("BAD", "OK")
+
+
 class BogusMailBox(ContextManager):
     def __enter__(self):
         return self
@@ -53,12 +63,15 @@ class BogusMailBox(ContextManager):
     def __init__(self):
         self.messages: List[MailMessage] = []
         self.messages_spam: List[MailMessage] = []
+        self.folder = BogusFolderManager()
+        self.client = BogusClient()
 
     def login(self, username, password):
-        if not (username == "admin" and password == "secret"):
-            raise Exception()
-
-    folder = BogusFolderManager()
+        # This will raise a UnicodeEncodeError if the password is not ASCII only
+        password.encode("ascii")
+        # Otherwise, check for correct values
+        if username != "admin" or password not in {"secret"}:
+            raise MailboxLoginError("BAD", "OK")
 
     def fetch(self, criteria, mark_seen, charset=""):
         msg = self.messages
@@ -228,7 +241,6 @@ class TestMail(DirectoriesMixin, TestCase):
         message.from_values = EmailAddress(
             "Someone!",
             "someone@somewhere.com",
-            "Someone! <someone@somewhere.com>",
         )
 
         message2 = namedtuple("MailMessage", [])
@@ -236,7 +248,6 @@ class TestMail(DirectoriesMixin, TestCase):
         message2.from_values = EmailAddress(
             "",
             "fake@localhost.com",
-            "",
         )
 
         me_localhost = Correspondent.objects.create(name=message2.from_)
@@ -308,10 +319,12 @@ class TestMail(DirectoriesMixin, TestCase):
         )
 
         account = MailAccount()
+        account.save()
         rule = MailRule(
             assign_title_from=MailRule.TitleSource.FROM_FILENAME,
             account=account,
         )
+        rule.save()
 
         result = self.mail_account_handler.handle_message(message, rule)
 
@@ -355,10 +368,12 @@ class TestMail(DirectoriesMixin, TestCase):
         )
 
         account = MailAccount()
+        account.save()
         rule = MailRule(
             assign_title_from=MailRule.TitleSource.FROM_FILENAME,
             account=account,
         )
+        rule.save()
 
         result = self.mail_account_handler.handle_message(message, rule)
 
@@ -381,10 +396,12 @@ class TestMail(DirectoriesMixin, TestCase):
         )
 
         account = MailAccount()
+        account.save()
         rule = MailRule(
             assign_title_from=MailRule.TitleSource.FROM_FILENAME,
             account=account,
         )
+        rule.save()
 
         result = self.mail_account_handler.handle_message(message, rule)
 
@@ -406,11 +423,13 @@ class TestMail(DirectoriesMixin, TestCase):
         )
 
         account = MailAccount()
+        account.save()
         rule = MailRule(
             assign_title_from=MailRule.TitleSource.FROM_FILENAME,
             account=account,
             attachment_type=MailRule.AttachmentProcessing.EVERYTHING,
         )
+        rule.save()
 
         result = self.mail_account_handler.handle_message(message, rule)
 
@@ -440,12 +459,15 @@ class TestMail(DirectoriesMixin, TestCase):
         for (pattern, matches) in tests:
             matches.sort()
             self.async_task.reset_mock()
-            account = MailAccount()
+            account = MailAccount(name=str(uuid.uuid4()))
+            account.save()
             rule = MailRule(
+                name=str(uuid.uuid4()),
                 assign_title_from=MailRule.TitleSource.FROM_FILENAME,
                 account=account,
                 filter_attachment_filename=pattern,
             )
+            rule.save()
 
             result = self.mail_account_handler.handle_message(message, rule)
 
@@ -621,6 +643,72 @@ class TestMail(DirectoriesMixin, TestCase):
         self.assertEqual(len(self.bogus_mailbox.messages), 2)
         self.assertEqual(len(self.bogus_mailbox.messages_spam), 1)
 
+    def test_error_folder_set(self):
+        """
+        GIVEN:
+            - Mail rule with non-existent folder
+        THEN:
+            - Should call list to output all folders in the account
+            - Should not process any messages
+        """
+        account = MailAccount.objects.create(
+            name="test2",
+            imap_server="",
+            username="admin",
+            password="secret",
+        )
+        _ = MailRule.objects.create(
+            name="testrule",
+            account=account,
+            action=MailRule.MailAction.MOVE,
+            action_parameter="spam",
+            filter_subject="Claim",
+            order=1,
+            folder="uuuhhhh",  # Invalid folder name
+        )
+
+        self.bogus_mailbox.folder.list = mock.Mock(
+            return_value=[FolderInfo("SomeFoldername", "|", ())],
+        )
+
+        self.mail_account_handler.handle_mail_account(account)
+
+        self.bogus_mailbox.folder.list.assert_called_once()
+        self.assertEqual(self.async_task.call_count, 0)
+
+    def test_error_folder_set_error_listing(self):
+        """
+        GIVEN:
+            - Mail rule with non-existent folder
+            - Mail account folder listing raises exception
+        THEN:
+            - Should not process any messages
+        """
+        account = MailAccount.objects.create(
+            name="test2",
+            imap_server="",
+            username="admin",
+            password="secret",
+        )
+        _ = MailRule.objects.create(
+            name="testrule",
+            account=account,
+            action=MailRule.MailAction.MOVE,
+            action_parameter="spam",
+            filter_subject="Claim",
+            order=1,
+            folder="uuuhhhh",  # Invalid folder name
+        )
+
+        self.bogus_mailbox.folder.list = mock.Mock(
+            side_effect=MailboxFolderSelectError(None, "uhm"),
+        )
+
+        self.mail_account_handler.handle_mail_account(account)
+
+        self.bogus_mailbox.folder.list.assert_called_once()
+        self.assertEqual(self.async_task.call_count, 0)
+
     @mock.patch("paperless_mail.mail.MailAccountHandler.get_correspondent")
     def test_error_skip_mail(self, m):
         def get_correspondent_fake(message, rule):
@@ -744,6 +832,66 @@ class TestMail(DirectoriesMixin, TestCase):
         self.assertEqual(len(self.bogus_mailbox.messages), 2)
         self.assertEqual(self.async_task.call_count, 5)
 
+    def test_auth_plain_fallback(self):
+        """
+        GIVEN:
+            - Mail account with password containing non-ASCII characters
+        THEN:
+            - Should still authenticate to the mail account
+        """
+        account = MailAccount.objects.create(
+            name="test",
+            imap_server="",
+            username="admin",
+            # Note the non-ascii characters here
+            password="w57äöüw4b6huwb6nhu",
+        )
+
+        _ = MailRule.objects.create(
+            name="testrule",
+            account=account,
+            action=MailRule.MailAction.MARK_READ,
+        )
+
+        self.assertEqual(len(self.bogus_mailbox.messages), 3)
+        self.assertEqual(self.async_task.call_count, 0)
+        self.assertEqual(len(self.bogus_mailbox.fetch("UNSEEN", False)), 2)
+
+        self.mail_account_handler.handle_mail_account(account)
+
+        self.assertEqual(self.async_task.call_count, 2)
+        self.assertEqual(len(self.bogus_mailbox.fetch("UNSEEN", False)), 0)
+        self.assertEqual(len(self.bogus_mailbox.messages), 3)
+
+    def test_auth_plain_fallback_fails_still(self):
+        """
+        GIVEN:
+            - Mail account with password containing non-ASCII characters
+            - Incorrect password value
+        THEN:
+            - Should raise a MailError for the account
+        """
+        account = MailAccount.objects.create(
+            name="test",
+            imap_server="",
+            username="admin",
+            # Note the non-ascii characters here
+            # Passes the check in login, not in authenticate
+            password="réception",
+        )
+
+        _ = MailRule.objects.create(
+            name="testrule",
+            account=account,
+            action=MailRule.MailAction.MARK_READ,
+        )
+
+        self.assertRaises(
+            MailError,
+            self.mail_account_handler.handle_mail_account,
+            account,
+        )
+
 
 class TestManagementCommand(TestCase):
     @mock.patch(