From 402228405984b4d2941aafdef062c091461bcf9c Mon Sep 17 00:00:00 2001 From: jonasc Date: Fri, 11 Mar 2022 16:15:42 +0100 Subject: [PATCH 1/6] Allow setting more than one tag in mail rules The three migrations do the following to preserve existing data in assign_tag: 1. Add the new many-to-many field assign_tags. 2. Copy existing data from the assign_tag field to the assign_tags. 3. Delete the existing assign_tag field. --- src/paperless_mail/admin.py | 2 +- src/paperless_mail/mail.py | 4 +-- .../migrations/0009_mailrule_assign_tags.py | 23 ++++++++++++++++ .../migrations/0010_auto_20220311_1602.py | 26 +++++++++++++++++++ .../0011_remove_mailrule_assign_tag.py | 17 ++++++++++++ src/paperless_mail/models.py | 5 ++-- 6 files changed, 71 insertions(+), 6 deletions(-) create mode 100644 src/paperless_mail/migrations/0009_mailrule_assign_tags.py create mode 100644 src/paperless_mail/migrations/0010_auto_20220311_1602.py create mode 100644 src/paperless_mail/migrations/0011_remove_mailrule_assign_tag.py diff --git a/src/paperless_mail/admin.py b/src/paperless_mail/admin.py index b56bc0727..3b488b153 100644 --- a/src/paperless_mail/admin.py +++ b/src/paperless_mail/admin.py @@ -82,7 +82,7 @@ class MailRuleAdmin(admin.ModelAdmin): ), "fields": ( "assign_title_from", - "assign_tag", + "assign_tags", "assign_document_type", "assign_correspondent_from", "assign_correspondent", diff --git a/src/paperless_mail/mail.py b/src/paperless_mail/mail.py index 5df29e2b0..3295f88ca 100644 --- a/src/paperless_mail/mail.py +++ b/src/paperless_mail/mail.py @@ -265,7 +265,7 @@ class MailAccountHandler(LoggingMixin): ) correspondent = self.get_correspondent(message, rule) - tag = rule.assign_tag + tag_ids = [tag.id for tag in rule.assign_tags.all()] doc_type = rule.assign_document_type processed_attachments = 0 @@ -328,7 +328,7 @@ class MailAccountHandler(LoggingMixin): if correspondent else None, override_document_type_id=doc_type.id if doc_type else None, - override_tag_ids=[tag.id] if tag else None, + override_tag_ids=tag_ids, task_name=att.filename[:100], ) diff --git a/src/paperless_mail/migrations/0009_mailrule_assign_tags.py b/src/paperless_mail/migrations/0009_mailrule_assign_tags.py new file mode 100644 index 000000000..fe2447e62 --- /dev/null +++ b/src/paperless_mail/migrations/0009_mailrule_assign_tags.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.12 on 2022-03-11 15:00 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("paperless_mail", "0008_auto_20210516_0940"), + ] + + operations = [ + migrations.AddField( + model_name="mailrule", + name="assign_tags", + field=models.ManyToManyField( + blank=True, + related_name="mail_rules_multi", + to="documents.Tag", + verbose_name="assign this tag", + ), + ), + ] diff --git a/src/paperless_mail/migrations/0010_auto_20220311_1602.py b/src/paperless_mail/migrations/0010_auto_20220311_1602.py new file mode 100644 index 000000000..bf0481302 --- /dev/null +++ b/src/paperless_mail/migrations/0010_auto_20220311_1602.py @@ -0,0 +1,26 @@ +# Generated by Django 3.2.12 on 2022-03-11 15:02 + +from django.db import migrations + + +def migrate_tag(apps, schema_editor): + # Manual data migration, see + # https://docs.djangoproject.com/en/3.2/topics/migrations/#data-migrations + # + # Copy the assign_tag property to the new assign_tags set if it exists. + MailRule = apps.get_model("paperless_mail", "MailRule") + for mail_rule in MailRule.objects.all(): + if mail_rule.assign_tag: + mail_rule.assign_tags.add(mail_rule.assign_tag) + mail_rule.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ("paperless_mail", "0009_mailrule_assign_tags"), + ] + + operations = [ + migrations.RunPython(migrate_tag), + ] diff --git a/src/paperless_mail/migrations/0011_remove_mailrule_assign_tag.py b/src/paperless_mail/migrations/0011_remove_mailrule_assign_tag.py new file mode 100644 index 000000000..ce3c24932 --- /dev/null +++ b/src/paperless_mail/migrations/0011_remove_mailrule_assign_tag.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2.12 on 2022-03-11 15:18 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("paperless_mail", "0010_auto_20220311_1602"), + ] + + operations = [ + migrations.RemoveField( + model_name="mailrule", + name="assign_tag", + ), + ] diff --git a/src/paperless_mail/models.py b/src/paperless_mail/models.py index 2c7b9fb6d..4f2038d4f 100644 --- a/src/paperless_mail/models.py +++ b/src/paperless_mail/models.py @@ -169,11 +169,10 @@ class MailRule(models.Model): default=TitleSource.FROM_SUBJECT, ) - assign_tag = models.ForeignKey( + assign_tags = models.ManyToManyField( document_models.Tag, - null=True, blank=True, - on_delete=models.SET_NULL, + related_name="mail_rules_multi", verbose_name=_("assign this tag"), ) From 37f7ef41f2a56aca8b88b3a43536453167ae53d2 Mon Sep 17 00:00:00 2001 From: jonasc Date: Fri, 11 Mar 2022 17:00:27 +0100 Subject: [PATCH 2/6] Save MailRule and MailAccount objects in tests This fixes the errors in the tests that the new many-to-many assign_tags property cannot be accessed due to MailRule not having an id. In one case it it necessary to give additional names to both objects as several are created. --- src/paperless_mail/tests/test_mail.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/paperless_mail/tests/test_mail.py b/src/paperless_mail/tests/test_mail.py index 9335bcd75..036caf086 100644 --- a/src/paperless_mail/tests/test_mail.py +++ b/src/paperless_mail/tests/test_mail.py @@ -308,10 +308,12 @@ class TestMail(DirectoriesMixin, TestCase): ) account = MailAccount() + account.save() rule = MailRule( assign_title_from=MailRule.TitleSource.FROM_FILENAME, account=account, ) + rule.save() result = self.mail_account_handler.handle_message(message, rule) @@ -355,10 +357,12 @@ class TestMail(DirectoriesMixin, TestCase): ) account = MailAccount() + account.save() rule = MailRule( assign_title_from=MailRule.TitleSource.FROM_FILENAME, account=account, ) + rule.save() result = self.mail_account_handler.handle_message(message, rule) @@ -381,10 +385,12 @@ class TestMail(DirectoriesMixin, TestCase): ) account = MailAccount() + account.save() rule = MailRule( assign_title_from=MailRule.TitleSource.FROM_FILENAME, account=account, ) + rule.save() result = self.mail_account_handler.handle_message(message, rule) @@ -406,11 +412,13 @@ class TestMail(DirectoriesMixin, TestCase): ) account = MailAccount() + account.save() rule = MailRule( assign_title_from=MailRule.TitleSource.FROM_FILENAME, account=account, attachment_type=MailRule.AttachmentProcessing.EVERYTHING, ) + rule.save() result = self.mail_account_handler.handle_message(message, rule) @@ -440,12 +448,15 @@ class TestMail(DirectoriesMixin, TestCase): for (pattern, matches) in tests: matches.sort() self.async_task.reset_mock() - account = MailAccount() + account = MailAccount(name=str(uuid.uuid4())) + account.save() rule = MailRule( + name=str(uuid.uuid4()), assign_title_from=MailRule.TitleSource.FROM_FILENAME, account=account, filter_attachment_filename=pattern, ) + rule.save() result = self.mail_account_handler.handle_message(message, rule) From 5ff304324db584942287c44d0dd08dbf63492620 Mon Sep 17 00:00:00 2001 From: jonasc Date: Fri, 11 Mar 2022 17:02:59 +0100 Subject: [PATCH 3/6] Add backwards data migration The documents.tests.test_migration_mime_type test suite failes if no backwards migration is provided. This simple backwards migration sets the old assign_tag field with a tag if exactly one is set in assign_tags. --- .../migrations/0010_auto_20220311_1602.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/paperless_mail/migrations/0010_auto_20220311_1602.py b/src/paperless_mail/migrations/0010_auto_20220311_1602.py index bf0481302..77a00d3c8 100644 --- a/src/paperless_mail/migrations/0010_auto_20220311_1602.py +++ b/src/paperless_mail/migrations/0010_auto_20220311_1602.py @@ -3,7 +3,7 @@ from django.db import migrations -def migrate_tag(apps, schema_editor): +def migrate_tag_to_tags(apps, schema_editor): # Manual data migration, see # https://docs.djangoproject.com/en/3.2/topics/migrations/#data-migrations # @@ -15,6 +15,20 @@ def migrate_tag(apps, schema_editor): mail_rule.save() +def migrate_tags_to_tag(apps, schema_editor): + # Manual data migration, see + # https://docs.djangoproject.com/en/3.2/topics/migrations/#data-migrations + # + # Copy the unique value in the assign_tags set to the old assign_tag property. + # Do nothing if the tag is not unique. + MailRule = apps.get_model("paperless_mail", "MailRule") + for mail_rule in MailRule.objects.all(): + tags = mail_rule.assign_tags.all() + if len(tags) == 1: + mail_rule.assign_tag = tags[0] + mail_rule.save() + + class Migration(migrations.Migration): dependencies = [ @@ -22,5 +36,5 @@ class Migration(migrations.Migration): ] operations = [ - migrations.RunPython(migrate_tag), + migrations.RunPython(migrate_tag_to_tags, migrate_tags_to_tag), ] From 0de1230a1a59ced5276cd9a6e7918c7346cb9ee7 Mon Sep 17 00:00:00 2001 From: jonasc Date: Fri, 11 Mar 2022 17:26:42 +0100 Subject: [PATCH 4/6] Reset related_name (was set to prevent duplicate) --- .../0012_alter_mailrule_assign_tags.py | 20 +++++++++++++++++++ src/paperless_mail/models.py | 1 - 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 src/paperless_mail/migrations/0012_alter_mailrule_assign_tags.py diff --git a/src/paperless_mail/migrations/0012_alter_mailrule_assign_tags.py b/src/paperless_mail/migrations/0012_alter_mailrule_assign_tags.py new file mode 100644 index 000000000..811c7d90a --- /dev/null +++ b/src/paperless_mail/migrations/0012_alter_mailrule_assign_tags.py @@ -0,0 +1,20 @@ +# Generated by Django 3.2.12 on 2022-03-11 16:21 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("paperless_mail", "0011_remove_mailrule_assign_tag"), + ] + + operations = [ + migrations.AlterField( + model_name="mailrule", + name="assign_tags", + field=models.ManyToManyField( + blank=True, to="documents.Tag", verbose_name="assign this tag" + ), + ), + ] diff --git a/src/paperless_mail/models.py b/src/paperless_mail/models.py index 4f2038d4f..5ebb89fa8 100644 --- a/src/paperless_mail/models.py +++ b/src/paperless_mail/models.py @@ -172,7 +172,6 @@ class MailRule(models.Model): assign_tags = models.ManyToManyField( document_models.Tag, blank=True, - related_name="mail_rules_multi", verbose_name=_("assign this tag"), ) From 1ce19f544432ebce269690620fa5dde2547ac94a Mon Sep 17 00:00:00 2001 From: jonasc Date: Tue, 12 Apr 2022 10:51:19 +0200 Subject: [PATCH 5/6] merge migrations --- .../migrations/0013_merge_20220412_1051.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 src/paperless_mail/migrations/0013_merge_20220412_1051.py diff --git a/src/paperless_mail/migrations/0013_merge_20220412_1051.py b/src/paperless_mail/migrations/0013_merge_20220412_1051.py new file mode 100644 index 000000000..e8a64bf24 --- /dev/null +++ b/src/paperless_mail/migrations/0013_merge_20220412_1051.py @@ -0,0 +1,14 @@ +# Generated by Django 4.0.4 on 2022-04-12 08:51 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('paperless_mail', '0009_alter_mailrule_action_alter_mailrule_folder'), + ('paperless_mail', '0012_alter_mailrule_assign_tags'), + ] + + operations = [ + ] From 834ad1ef8472ebc23f747fc08727f872c44e2492 Mon Sep 17 00:00:00 2001 From: jonasc Date: Tue, 12 Apr 2022 15:06:38 +0200 Subject: [PATCH 6/6] reformat migration with black --- src/paperless_mail/migrations/0013_merge_20220412_1051.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/paperless_mail/migrations/0013_merge_20220412_1051.py b/src/paperless_mail/migrations/0013_merge_20220412_1051.py index e8a64bf24..2fdb17ff5 100644 --- a/src/paperless_mail/migrations/0013_merge_20220412_1051.py +++ b/src/paperless_mail/migrations/0013_merge_20220412_1051.py @@ -6,9 +6,8 @@ from django.db import migrations class Migration(migrations.Migration): dependencies = [ - ('paperless_mail', '0009_alter_mailrule_action_alter_mailrule_folder'), - ('paperless_mail', '0012_alter_mailrule_assign_tags'), + ("paperless_mail", "0009_alter_mailrule_action_alter_mailrule_folder"), + ("paperless_mail", "0012_alter_mailrule_assign_tags"), ] - operations = [ - ] + operations = []