add support for configuring mail server character set per server. fixes #548

src/locale/en_US/LC_MESSAGES/django.po

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2021-05-04 15:00+0000\n"
+"POT-Creation-Date: 2021-05-16 09:38+0000\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
@@ -46,7 +46,7 @@ msgid "Automatic"
 msgstr ""
 
 #: documents/models.py:41 documents/models.py:350 paperless_mail/models.py:25
-#: paperless_mail/models.py:109
+#: paperless_mail/models.py:117
 msgid "name"
 msgstr ""
 
@@ -376,127 +376,135 @@ msgstr ""
 msgid "File type %(type)s not supported"
 msgstr ""
 
-#: documents/templates/index.html:21
+#: documents/templates/index.html:22
 msgid "Paperless-ng is loading..."
 msgstr ""
 
-#: documents/templates/registration/logged_out.html:13
+#: documents/templates/registration/logged_out.html:14
 msgid "Paperless-ng signed out"
 msgstr ""
 
-#: documents/templates/registration/logged_out.html:41
+#: documents/templates/registration/logged_out.html:45
 msgid "You have been successfully logged out. Bye!"
 msgstr ""
 
-#: documents/templates/registration/logged_out.html:42
+#: documents/templates/registration/logged_out.html:46
 msgid "Sign in again"
 msgstr ""
 
-#: documents/templates/registration/login.html:13
+#: documents/templates/registration/login.html:15
 msgid "Paperless-ng sign in"
 msgstr ""
 
-#: documents/templates/registration/login.html:42
+#: documents/templates/registration/login.html:47
 msgid "Please sign in."
 msgstr ""
 
-#: documents/templates/registration/login.html:45
+#: documents/templates/registration/login.html:50
 msgid "Your username and password didn't match. Please try again."
 msgstr ""
 
-#: documents/templates/registration/login.html:48
+#: documents/templates/registration/login.html:53
 msgid "Username"
 msgstr ""
 
-#: documents/templates/registration/login.html:49
+#: documents/templates/registration/login.html:54
 msgid "Password"
 msgstr ""
 
-#: documents/templates/registration/login.html:54
+#: documents/templates/registration/login.html:59
 msgid "Sign in"
 msgstr ""
 
-#: paperless/settings.py:300
+#: paperless/settings.py:303
 msgid "English (US)"
 msgstr ""
 
-#: paperless/settings.py:301
+#: paperless/settings.py:304
 msgid "English (GB)"
 msgstr ""
 
-#: paperless/settings.py:302
+#: paperless/settings.py:305
 msgid "German"
 msgstr ""
 
-#: paperless/settings.py:303
+#: paperless/settings.py:306
 msgid "Dutch"
 msgstr ""
 
-#: paperless/settings.py:304
+#: paperless/settings.py:307
 msgid "French"
 msgstr ""
 
-#: paperless/settings.py:305
+#: paperless/settings.py:308
 msgid "Portuguese (Brazil)"
 msgstr ""
 
-#: paperless/settings.py:306
+#: paperless/settings.py:309
 msgid "Portuguese"
 msgstr ""
 
-#: paperless/settings.py:307
+#: paperless/settings.py:310
 msgid "Italian"
 msgstr ""
 
-#: paperless/settings.py:308
+#: paperless/settings.py:311
 msgid "Romanian"
 msgstr ""
 
-#: paperless/settings.py:309
+#: paperless/settings.py:312
 msgid "Russian"
 msgstr ""
 
-#: paperless/settings.py:310
+#: paperless/settings.py:313
 msgid "Spanish"
 msgstr ""
 
-#: paperless/settings.py:311
+#: paperless/settings.py:314
 msgid "Polish"
 msgstr ""
 
-#: paperless/settings.py:312
+#: paperless/settings.py:315
 msgid "Swedish"
 msgstr ""
 
-#: paperless/urls.py:113
+#: paperless/urls.py:120
 msgid "Paperless-ng administration"
 msgstr ""
 
-#: paperless_mail/admin.py:25
+#: paperless_mail/admin.py:15
+msgid "Authentication"
+msgstr ""
+
+#: paperless_mail/admin.py:18
+msgid "Advanced settings"
+msgstr ""
+
+#: paperless_mail/admin.py:37
 msgid "Filter"
 msgstr ""
 
-#: paperless_mail/admin.py:27
+#: paperless_mail/admin.py:39
 msgid ""
 "Paperless will only process mails that match ALL of the filters given below."
 msgstr ""
 
-#: paperless_mail/admin.py:37
+#: paperless_mail/admin.py:49
 msgid "Actions"
 msgstr ""
 
-#: paperless_mail/admin.py:39
+#: paperless_mail/admin.py:51
 msgid ""
 "The action applied to the mail. This action is only performed when documents "
 "were consumed from the mail. Mails without attachments will remain entirely "
 "untouched."
 msgstr ""
 
-#: paperless_mail/admin.py:46
+#: paperless_mail/admin.py:58
 msgid "Metadata"
 msgstr ""
 
-#: paperless_mail/admin.py:48
+#: paperless_mail/admin.py:60
 msgid ""
 "Assign metadata to documents consumed from this rule automatically. If you "
 "do not assign tags, types or correspondents here, paperless will still "
@@ -553,144 +561,158 @@ msgstr ""
 msgid "password"
 msgstr ""
 
-#: paperless_mail/models.py:60
+#: paperless_mail/models.py:54
-msgid "mail rule"
+msgid "character set"
 msgstr ""
 
-#: paperless_mail/models.py:61
+#: paperless_mail/models.py:57
-msgid "mail rules"
+msgid ""
-msgstr ""
+"The character set to use when communicating with the mail server, such as "
-
+"'UTF-8' or 'US-ASCII'."
-#: paperless_mail/models.py:67
-msgid "Only process attachments."
 msgstr ""
 
 #: paperless_mail/models.py:68
+msgid "mail rule"
+msgstr ""
+
+#: paperless_mail/models.py:69
+msgid "mail rules"
+msgstr ""
+
+#: paperless_mail/models.py:75
+msgid "Only process attachments."
+msgstr ""
+
+#: paperless_mail/models.py:76
 msgid "Process all files, including 'inline' attachments."
 msgstr ""
 
-#: paperless_mail/models.py:78
+#: paperless_mail/models.py:86
 msgid "Mark as read, don't process read mails"
 msgstr ""
 
-#: paperless_mail/models.py:79
+#: paperless_mail/models.py:87
 msgid "Flag the mail, don't process flagged mails"
 msgstr ""
 
-#: paperless_mail/models.py:80
+#: paperless_mail/models.py:88
 msgid "Move to specified folder"
 msgstr ""
 
-#: paperless_mail/models.py:81
+#: paperless_mail/models.py:89
 msgid "Delete"
 msgstr ""
 
-#: paperless_mail/models.py:88
+#: paperless_mail/models.py:96
 msgid "Use subject as title"
 msgstr ""
 
-#: paperless_mail/models.py:89
+#: paperless_mail/models.py:97
 msgid "Use attachment filename as title"
 msgstr ""
 
-#: paperless_mail/models.py:99
+#: paperless_mail/models.py:107
 msgid "Do not assign a correspondent"
 msgstr ""
 
-#: paperless_mail/models.py:101
+#: paperless_mail/models.py:109
 msgid "Use mail address"
 msgstr ""
 
-#: paperless_mail/models.py:103
+#: paperless_mail/models.py:111
 msgid "Use name (or mail address if not available)"
 msgstr ""
 
-#: paperless_mail/models.py:105
+#: paperless_mail/models.py:113
 msgid "Use correspondent selected below"
 msgstr ""
 
-#: paperless_mail/models.py:113
+#: paperless_mail/models.py:121
 msgid "order"
 msgstr ""
 
-#: paperless_mail/models.py:120
+#: paperless_mail/models.py:128
 msgid "account"
 msgstr ""
 
-#: paperless_mail/models.py:124
+#: paperless_mail/models.py:132
 msgid "folder"
 msgstr ""
 
-#: paperless_mail/models.py:128
-msgid "filter from"
-msgstr ""
-
-#: paperless_mail/models.py:131
-msgid "filter subject"
-msgstr ""
-
 #: paperless_mail/models.py:134
-msgid "filter body"
+msgid "Subfolders must be separated by dots."
 msgstr ""
 
 #: paperless_mail/models.py:138
+msgid "filter from"
+msgstr ""
+
+#: paperless_mail/models.py:141
+msgid "filter subject"
+msgstr ""
+
+#: paperless_mail/models.py:144
+msgid "filter body"
+msgstr ""
+
+#: paperless_mail/models.py:148
 msgid "filter attachment filename"
 msgstr ""
 
-#: paperless_mail/models.py:140
+#: paperless_mail/models.py:150
 msgid ""
 "Only consume documents which entirely match this filename if specified. "
 "Wildcards such as *.pdf or *invoice* are allowed. Case insensitive."
 msgstr ""
 
-#: paperless_mail/models.py:146
+#: paperless_mail/models.py:156
 msgid "maximum age"
 msgstr ""
 
-#: paperless_mail/models.py:148
+#: paperless_mail/models.py:158
 msgid "Specified in days."
 msgstr ""
 
-#: paperless_mail/models.py:151
+#: paperless_mail/models.py:161
 msgid "attachment type"
 msgstr ""
 
-#: paperless_mail/models.py:154
+#: paperless_mail/models.py:164
 msgid ""
 "Inline attachments include embedded images, so it's best to combine this "
 "option with a filename filter."
 msgstr ""
 
-#: paperless_mail/models.py:159
+#: paperless_mail/models.py:169
 msgid "action"
 msgstr ""
 
-#: paperless_mail/models.py:165
+#: paperless_mail/models.py:175
 msgid "action parameter"
 msgstr ""
 
-#: paperless_mail/models.py:167
+#: paperless_mail/models.py:177
 msgid ""
 "Additional parameter for the action selected above, i.e., the target folder "
-"of the move to folder action."
+"of the move to folder action. Subfolders must be separated by dots."
 msgstr ""
 
-#: paperless_mail/models.py:173
+#: paperless_mail/models.py:184
 msgid "assign title from"
 msgstr ""
 
-#: paperless_mail/models.py:183
+#: paperless_mail/models.py:194
 msgid "assign this tag"
 msgstr ""
 
-#: paperless_mail/models.py:191
+#: paperless_mail/models.py:202
 msgid "assign this document type"
 msgstr ""
 
-#: paperless_mail/models.py:195
+#: paperless_mail/models.py:206
 msgid "assign correspondent from"
 msgstr ""
 
-#: paperless_mail/models.py:205
+#: paperless_mail/models.py:216
 msgid "assign this correspondent"
 msgstr ""

src/paperless_mail/admin.py

@@ -8,6 +8,18 @@ class MailAccountAdmin(admin.ModelAdmin):
 
     list_display = ("name", "imap_server", "username")
 
+    fieldsets = [
+        (None, {
+            'fields': ['name', 'imap_server', 'imap_port']
+        }),
+        (_("Authentication"), {
+            'fields': ['imap_security', 'username', 'password']
+        }),
+        (_("Advanced settings"), {
+            'fields': ['character_set']
+        })
+    ]
+
 
 class MailRuleAdmin(admin.ModelAdmin):
 

src/paperless_mail/mail.py

@@ -200,8 +200,10 @@ class MailAccountHandler(LoggingMixin):
             f"{str(AND(**criterias))}")
 
         try:
-            messages = M.fetch(criteria=AND(**criterias),
+            messages = M.fetch(
-                               mark_seen=False, charset='UTF-8')
+                criteria=AND(**criterias),
+                mark_seen=False,
+                charset=rule.account.character_set)
         except Exception:
             raise MailError(
                 f"Rule {rule}: Error while fetching folder {rule.folder}")

src/paperless_mail/migrations/0008_auto_20210516_0940.py

@@ -0,0 +1,28 @@
+# Generated by Django 3.2.3 on 2021-05-16 09:40
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('paperless_mail', '0007_auto_20210106_0138'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='mailaccount',
+            name='character_set',
+            field=models.CharField(default='UTF-8', help_text="The character set to use when communicating with the mail server, such as 'UTF-8' or 'US-ASCII'.", max_length=256, verbose_name='character set'),
+        ),
+        migrations.AlterField(
+            model_name='mailrule',
+            name='action_parameter',
+            field=models.CharField(blank=True, help_text='Additional parameter for the action selected above, i.e., the target folder of the move to folder action. Subfolders must be separated by dots.', max_length=256, null=True, verbose_name='action parameter'),
+        ),
+        migrations.AlterField(
+            model_name='mailrule',
+            name='folder',
+            field=models.CharField(default='INBOX', help_text='Subfolders must be separated by dots.', max_length=256, verbose_name='folder'),
+        ),
+    ]

src/paperless_mail/models.py

@@ -50,6 +50,14 @@ class MailAccount(models.Model):
         _("password"),
         max_length=256)
 
+    character_set = models.CharField(
+        _("character set"),
+        max_length=256,
+        default="UTF-8",
+        help_text=_("The character set to use when communicating with the "
+                    "mail server, such as 'UTF-8' or 'US-ASCII'.")
+    )
+
     def __str__(self):
         return self.name
 
@@ -122,7 +130,9 @@ class MailRule(models.Model):
 
     folder = models.CharField(
         _("folder"),
-        default='INBOX', max_length=256)
+        default='INBOX', max_length=256,
+        help_text=_("Subfolders must be separated by dots.")
+    )
 
     filter_from = models.CharField(
         _("filter from"),
@@ -166,7 +176,8 @@ class MailRule(models.Model):
         max_length=256, blank=True, null=True,
         help_text=_("Additional parameter for the action selected above, "
                     "i.e., "
-                    "the target folder of the move to folder action.")
+                    "the target folder of the move to folder action. "
+                    "Subfolders must be separated by dots.")
     )
 
     assign_title_from = models.PositiveIntegerField(