first implementation of the mail rework

src/paperless_mail/admin.py

@@ -0,0 +1,27 @@
+from django.contrib import admin
+from django import forms
+
+from paperless_mail.models import MailAccount, MailRule
+
+
+class MailAccountForm(forms.ModelForm):
+
+    password = forms.CharField(widget=forms.PasswordInput)
+
+    class Meta:
+        fields = '__all__'
+        model = MailAccount
+
+
+class MailAccountAdmin(admin.ModelAdmin):
+
+    list_display = ("name", "imap_server", "username")
+
+
+class MailRuleAdmin(admin.ModelAdmin):
+
+    list_display = ("name", "account", "folder", "action")
+
+
+admin.site.register(MailAccount, MailAccountAdmin)
+admin.site.register(MailRule, MailRuleAdmin)

src/paperless_mail/apps.py

@@ -0,0 +1,7 @@
+from django.apps import AppConfig
+
+
+class PaperlessMailConfig(AppConfig):
+    name = 'paperless_mail'
+
+    verbose_name = 'Paperless Mail'

src/paperless_mail/mail.py

@@ -0,0 +1,149 @@
+import email
+from datetime import datetime, timedelta, date
+from email.parser import BytesParser
+
+from django.utils.text import slugify
+from imap_tools import MailBox, MailBoxUnencrypted, AND, MailMessageFlags
+
+from documents.models import Correspondent
+from paperless_mail.models import MailAccount, MailRule
+
+
+class BaseMailAction:
+
+    def get_criteria(self):
+        return {}
+
+    def post_consume(self, M, message_uids, parameter):
+        pass
+
+
+class DeleteMailAction(BaseMailAction):
+
+    def post_consume(self, M, message_uids, parameter):
+        M.delete(message_uids)
+
+
+class MarkReadMailAction(BaseMailAction):
+
+    def get_criteria(self):
+        return {'seen': False}
+
+    def post_consume(self, M, message_uids, parameter):
+        M.seen(message_uids, True)
+
+
+class MoveMailAction(BaseMailAction):
+
+    def post_consume(self, M, message_uids, parameter):
+        M.move(message_uids, parameter)
+
+
+class FlagMailAction(BaseMailAction):
+
+    def get_criteria(self):
+        return {'flagged': False}
+
+    def post_consume(self, M, message_uids, parameter):
+        M.flag(message_uids, [MailMessageFlags.FLAGGED], True)
+
+
+def get_rule_action(action):
+    if action == MailRule.ACTION_FLAG:
+        return FlagMailAction()
+    elif action == MailRule.ACTION_DELETE:
+        return DeleteMailAction()
+    elif action == MailRule.ACTION_MOVE:
+        return MoveMailAction()
+    elif action == MailRule.ACTION_MARK_READ:
+        return MarkReadMailAction()
+    else:
+        raise ValueError("Unknown action.")
+
+
+def handle_mail_account(account):
+
+    if account.imap_security == MailAccount.IMAP_SECURITY_NONE:
+        mailbox = MailBoxUnencrypted(account.imap_server, account.imap_port)
+    elif account.imap_security == MailAccount.IMAP_SECURITY_STARTTLS:
+        mailbox = MailBox(account.imap_server, account.imap_port, starttls=True)
+    elif account.imap_security == MailAccount.IMAP_SECURITY_SSL:
+        mailbox = MailBox(account.imap_server, account.imap_port)
+    else:
+        raise ValueError("Unknown IMAP security")
+
+    with mailbox.login(account.username, account.password) as M:
+
+        for rule in account.rules.all():
+
+            M.folder.set(rule.folder)
+
+            maximum_age = date.today() - timedelta(days=rule.maximum_age)
+            criterias = {
+                "date_gte": maximum_age
+            }
+            if rule.filter_from:
+                criterias["from_"] = rule.filter_from
+            if rule.filter_subject:
+                criterias["subject"] = rule.filter_subject
+            if rule.filter_body:
+                criterias["body"] = rule.filter_body
+
+            action = get_rule_action(rule.action)
+            criterias = {**criterias, **action.get_criteria()}
+
+            messages = M.fetch(criteria=AND(**criterias), mark_seen=False)
+
+            post_consume_messages = []
+
+            for message in messages:
+                result = handle_message(M, message, rule)
+                if result:
+                    post_consume_messages.append(message.uid)
+
+            action.post_consume(M, post_consume_messages, rule.action_parameter)
+
+
+def handle_message(M, message, rule):
+    if not message.attachments:
+        return False
+
+    if rule.assign_correspondent_from == MailRule.CORRESPONDENT_FROM_NOTHING:
+        correspondent = None
+    elif rule.assign_correspondent_from == MailRule.CORRESPONDENT_FROM_EMAIL:
+        corerspondent_name = message.from_
+        correspondent = Correspondent.objects.get_or_create(
+            name=corerspondent_name, defaults={
+                "slug": slugify(corerspondent_name)
+            })[0]
+    elif rule.assign_correspondent_from == MailRule.CORRESPONDENT_FROM_NAME:
+        corerspondent_name = message.from_values.name if message.from_values and message.from_values.name else message.from_
+        correspondent = Correspondent.objects.get_or_create(
+            name=corerspondent_name, defaults={
+                "slug": slugify(corerspondent_name)
+            })[0]
+    elif rule.assign_correspondent_from == MailRule.CORRESPONDENT_FROM_CUSTOM:
+        correspondent = rule.assign_correspondent
+    else:
+        raise ValueError("Unknwown correspondent selector")
+
+    tag = rule.assign_tag
+
+    doc_type = rule.assign_document_type
+
+    for att in message.attachments:
+
+        if rule.assign_title_from == MailRule.TITLE_FROM_SUBJECT:
+            title = message.subject
+        elif rule.assign_title_from == MailRule.TITLE_FROM_FILENAME:
+            title = att.filename
+        else:
+            raise ValueError("Unknown title selector.")
+
+        if att.content_type == 'application/pdf':
+            print("This is where I would consume the file with name {} and I would "
+                  "give it the title '{}', correspondent '{}', tag '{}', and doc type"
+                  "'{}'."
+                  .format(att.filename, title, correspondent, tag, doc_type))
+
+    return True

src/paperless_mail/management/commands/mail_fetcher.py

@@ -0,0 +1,13 @@
+from django.core.management.base import BaseCommand
+
+from paperless_mail import mail, tasks
+
+
+class Command(BaseCommand):
+
+    help = """
+    """.replace("    ", "")
+
+    def handle(self, *args, **options):
+
+        tasks.process_mail_accounts()

src/paperless_mail/migrations/0001_initial.py

@@ -0,0 +1,48 @@
+# Generated by Django 3.1.3 on 2020-11-15 22:54
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('documents', '1002_auto_20201111_1105'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='MailAccount',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=256, unique=True)),
+                ('imap_server', models.CharField(max_length=256)),
+                ('imap_port', models.IntegerField(blank=True, null=True)),
+                ('imap_security', models.PositiveIntegerField(choices=[(1, 'No encryption'), (2, 'Use SSL'), (3, 'Use STARTTLS')], default=2)),
+                ('username', models.CharField(max_length=256)),
+                ('password', models.CharField(max_length=256)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='MailRule',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=256)),
+                ('folder', models.CharField(default='INBOX', max_length=256)),
+                ('filter_from', models.CharField(blank=True, max_length=256, null=True)),
+                ('filter_subject', models.CharField(blank=True, max_length=256, null=True)),
+                ('filter_body', models.CharField(blank=True, max_length=256, null=True)),
+                ('maximum_age', models.PositiveIntegerField(default=30)),
+                ('action', models.PositiveIntegerField(choices=[(1, 'Delete'), (2, 'Move to specified folder'), (3, "Mark as read, don't process read mails"), (4, "Flag the mail, don't process flagged mails")], default=3, help_text='The action applied to the mail. This action is only performed when documents were consumed from the mail. Mails without attachments will remain entirely untouched.')),
+                ('action_parameter', models.CharField(blank=True, help_text='Additional parameter for the action selected above, i.e., the target folder of the move to folder action.', max_length=256, null=True)),
+                ('assign_title_from', models.PositiveIntegerField(choices=[(1, 'Use subject as title'), (2, 'Use attachment filename as title')], default=1)),
+                ('assign_correspondent_from', models.PositiveIntegerField(choices=[(1, 'Do not assign a correspondent'), (2, 'Use mail address'), (3, 'Use name (or mail address if not available)'), (4, 'Use correspondent selected below')], default=1)),
+                ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='rules', to='paperless_mail.mailaccount')),
+                ('assign_correspondent', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='documents.correspondent')),
+                ('assign_document_type', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='documents.documenttype')),
+                ('assign_tag', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='documents.tag')),
+            ],
+        ),
+    ]

src/paperless_mail/models.py

@@ -0,0 +1,137 @@
+from django.db import models
+
+# Create your models here.
+from django.db import models
+
+import documents.models as document_models
+
+
+class MailAccount(models.Model):
+
+    IMAP_SECURITY_NONE = 1
+    IMAP_SECURITY_SSL = 2
+    IMAP_SECURITY_STARTTLS = 3
+
+    IMAP_SECURITY_OPTIONS = (
+        (IMAP_SECURITY_NONE, "No encryption"),
+        (IMAP_SECURITY_SSL, "Use SSL"),
+        (IMAP_SECURITY_STARTTLS, "Use STARTTLS"),
+    )
+
+    name = models.CharField(max_length=256, unique=True)
+
+    imap_server = models.CharField(max_length=256)
+
+    imap_port = models.IntegerField(blank=True, null=True)
+
+    imap_security = models.PositiveIntegerField(
+        choices=IMAP_SECURITY_OPTIONS,
+        default=IMAP_SECURITY_SSL
+    )
+
+    username = models.CharField(max_length=256)
+
+    password = models.CharField(max_length=256)
+
+    def __str__(self):
+        return self.name
+
+
+class MailRule(models.Model):
+
+    ACTION_DELETE = 1
+    ACTION_MOVE = 2
+    ACTION_MARK_READ = 3
+    ACTION_FLAG = 4
+
+    ACTIONS = (
+        (ACTION_DELETE, "Delete"),
+        (ACTION_MOVE, "Move to specified folder"),
+        (ACTION_MARK_READ, "Mark as read, don't process read mails"),
+        (ACTION_FLAG, "Flag the mail, don't process flagged mails")
+    )
+
+    TITLE_FROM_SUBJECT = 1
+    TITLE_FROM_FILENAME = 2
+
+    TITLE_SELECTOR = (
+        (TITLE_FROM_SUBJECT, "Use subject as title"),
+        (TITLE_FROM_FILENAME, "Use attachment filename as title")
+    )
+
+    CORRESPONDENT_FROM_NOTHING = 1
+    CORRESPONDENT_FROM_EMAIL = 2
+    CORRESPONDENT_FROM_NAME = 3
+    CORRESPONDENT_FROM_CUSTOM = 4
+
+    CORRESPONDENT_SELECTOR = (
+        (CORRESPONDENT_FROM_NOTHING, "Do not assign a correspondent"),
+        (CORRESPONDENT_FROM_EMAIL, "Use mail address"),
+        (CORRESPONDENT_FROM_NAME, "Use name (or mail address if not available)"),
+        (CORRESPONDENT_FROM_CUSTOM, "Use correspondent selected below")
+    )
+
+    name = models.CharField(max_length=256)
+
+    account = models.ForeignKey(
+        MailAccount,
+        related_name="rules",
+        on_delete=models.CASCADE
+    )
+
+    folder = models.CharField(default='INBOX', max_length=256)
+
+    filter_from = models.CharField(max_length=256, null=True, blank=True)
+    filter_subject = models.CharField(max_length=256, null=True, blank=True)
+    filter_body = models.CharField(max_length=256, null=True, blank=True)
+
+    maximum_age = models.PositiveIntegerField(default=30)
+
+    action = models.PositiveIntegerField(
+        choices=ACTIONS,
+        default=ACTION_MARK_READ,
+        help_text="The action applied to the mail. This action is only "
+                  "performed when documents were consumed from the mail. "
+                  "Mails without attachments will remain entirely "
+                  "untouched."
+    )
+
+    action_parameter = models.CharField(
+        max_length=256, blank=True, null=True,
+        help_text="Additional parameter for the action selected above, i.e., "
+                  "the target folder of the move to folder action."
+    )
+
+    assign_title_from = models.PositiveIntegerField(
+        choices=TITLE_SELECTOR,
+        default=TITLE_FROM_SUBJECT
+    )
+
+    assign_tag = models.ForeignKey(
+        document_models.Tag,
+        null=True,
+        blank=True,
+        on_delete=models.SET_NULL
+    )
+
+    assign_document_type = models.ForeignKey(
+        document_models.DocumentType,
+        null=True,
+        blank=True,
+        on_delete=models.SET_NULL
+    )
+
+    assign_correspondent_from = models.PositiveIntegerField(
+        choices=CORRESPONDENT_SELECTOR,
+        default=CORRESPONDENT_FROM_NOTHING
+    )
+
+    assign_correspondent = models.ForeignKey(
+        document_models.Correspondent,
+        null=True,
+        blank=True,
+        on_delete=models.SET_NULL
+    )
+
+    def __str__(self):
+        return self.name

src/paperless_mail/tasks.py

@@ -0,0 +1,18 @@
+import logging
+
+from paperless_mail import mail
+from paperless_mail.models import MailAccount
+
+
+def process_mail_accounts():
+    for account in MailAccount.objects.all():
+        mail.handle_mail_account(account)
+
+
+def process_mail_account(name):
+    account = MailAccount.objects.find(name=name)
+    if account:
+        mail.handle_mail_account(account)
+    else:
+        logging.error("Unknown mail acccount: {}".format(name))
+

src/paperless_mail/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

src/paperless_mail/views.py

@@ -0,0 +1,3 @@
+from django.shortcuts import render
+
+# Create your views here.