paradizelost/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2025-07-28 18:24:38 -05:00
Code Issues Packages Projects Releases Wiki Activity

Feature: consumption templates (#4196)

Browse Source 
* Initial implementation of consumption templates

* Frontend implementation of consumption templates

Testing

* Support consumption template source

* order templates, automatically add permissions

* Support title assignment in consumption templates

* Refactoring, filters to and, show sources on list

Show sources on template list, update some translation strings

Make filters and

minor testing

* Update strings

* Only update django-multiselectfield

* Basic docs, document some methods

* Improve testing coverage, template multi-assignment merges
This commit is contained in:
shamoon
2023-09-22 16:53:13 -07:00
committed by GitHub
parent 86d223fd93
commit 9712ac109d
51 changed files with 3250 additions and 444 deletions
Download Patch File Download Diff File Expand all files Collapse all files

131
src/documents/consumer.py
View File

@@ -20,6 +20,10 @@ from django.utils import timezone
from filelock import FileLock
from rest_framework.reverse import reverse
from documents.data_models import ConsumableDocument
from documents.data_models import DocumentMetadataOverrides
from documents.matching import document_matches_template
from documents.permissions import set_permissions_for_object
from documents.utils import copy_basic_file_stats
from documents.utils import copy_file_with_basic_stats
@@ -27,10 +31,12 @@ from .classifier import load_classifier
from .file_handling import create_source_path_directory
from .file_handling import generate_unique_filename
from .loggers import LoggingMixin
from .models import ConsumptionTemplate
from .models import Correspondent
from .models import Document
from .models import DocumentType
from .models import FileInfo
from .models import StoragePath
from .models import Tag
from .parsers import DocumentParser
from .parsers import ParseError
@@ -319,10 +325,15 @@ class Consumer(LoggingMixin):
override_correspondent_id=None,
override_document_type_id=None,
override_tag_ids=None,
override_storage_path_id=None,
task_id=None,
override_created=None,
override_asn=None,
override_owner_id=None,
override_view_users=None,
override_view_groups=None,
override_change_users=None,
override_change_groups=None,
) -> Document:
"""
Return the document object if it was successfully created.
@@ -334,10 +345,15 @@ class Consumer(LoggingMixin):
self.override_correspondent_id = override_correspondent_id
self.override_document_type_id = override_document_type_id
self.override_tag_ids = override_tag_ids
self.override_storage_path_id = override_storage_path_id
self.task_id = task_id or str(uuid.uuid4())
self.override_created = override_created
self.override_asn = override_asn
self.override_owner_id = override_owner_id
self.override_view_users = override_view_users
self.override_view_groups = override_view_groups
self.override_change_users = override_change_users
self.override_change_groups = override_change_groups
self._send_progress(
0,
@@ -578,6 +594,92 @@ class Consumer(LoggingMixin):
return document
def get_template_overrides(
self,
input_doc: ConsumableDocument,
) -> DocumentMetadataOverrides:
"""
Match consumption templates to a document based on source and
file name filters, path filters or mail rule filter if specified
"""
overrides = DocumentMetadataOverrides()
for template in ConsumptionTemplate.objects.all().order_by("order"):
template_overrides = DocumentMetadataOverrides()
if document_matches_template(input_doc, template):
if template.assign_title is not None:
template_overrides.title = template.assign_title
if template.assign_tags is not None:
template_overrides.tag_ids = [
tag.pk for tag in template.assign_tags.all()
]
if template.assign_correspondent is not None:
template_overrides.correspondent_id = (
template.assign_correspondent.pk
)
if template.assign_document_type is not None:
template_overrides.document_type_id = (
template.assign_document_type.pk
)
if template.assign_storage_path is not None:
template_overrides.storage_path_id = template.assign_storage_path.pk
if template.assign_owner is not None:
template_overrides.owner_id = template.assign_owner.pk
if template.assign_view_users is not None:
template_overrides.view_users = [
user.pk for user in template.assign_view_users.all()
]
if template.assign_view_groups is not None:
template_overrides.view_groups = [
group.pk for group in template.assign_view_groups.all()
]
if template.assign_change_users is not None:
template_overrides.change_users = [
user.pk for user in template.assign_change_users.all()
]
if template.assign_change_groups is not None:
template_overrides.change_groups = [
group.pk for group in template.assign_change_groups.all()
]
overrides.update(template_overrides)
return overrides
def _parse_title_placeholders(self, title: str) -> str:
"""
Consumption template title placeholders can only include items that are
assigned as part of this template (since auto-matching hasnt happened yet)
"""
local_added = timezone.localtime(timezone.now())
correspondent_name = (
Correspondent.objects.get(pk=self.override_correspondent_id).name
if self.override_correspondent_id is not None
else None
)
doc_type_name = (
DocumentType.objects.get(pk=self.override_document_type_id).name
if self.override_correspondent_id is not None
else None
)
owner_username = (
User.objects.get(pk=self.override_owner_id).username
if self.override_owner_id is not None
else None
)
return title.format(
correspondent=correspondent_name,
document_type=doc_type_name,
added=local_added.isoformat(),
added_year=local_added.strftime("%Y"),
added_year_short=local_added.strftime("%y"),
added_month=local_added.strftime("%m"),
added_month_name=local_added.strftime("%B"),
added_month_name_short=local_added.strftime("%b"),
added_day=local_added.strftime("%d"),
owner_username=owner_username,
).strip()
def _store(
self,
text: str,
@@ -612,7 +714,11 @@ class Consumer(LoggingMixin):
with open(self.path, "rb") as f:
document = Document.objects.create(
title=(self.override_title or file_info.title)[:127],
title=(
self._parse_title_placeholders(self.override_title)
if self.override_title is not None
else file_info.title
)[:127],
content=text,
mime_type=mime_type,
checksum=hashlib.md5(f.read()).hexdigest(),
@@ -643,6 +749,11 @@ class Consumer(LoggingMixin):
for tag_id in self.override_tag_ids:
document.tags.add(Tag.objects.get(pk=tag_id))
if self.override_storage_path_id:
document.storage_path = StoragePath.objects.get(
pk=self.override_storage_path_id,
)
if self.override_asn:
document.archive_serial_number = self.override_asn
@@ -651,6 +762,24 @@ class Consumer(LoggingMixin):
pk=self.override_owner_id,
)
if (
self.override_view_users is not None
or self.override_view_groups is not None
or self.override_change_users is not None
or self.override_change_users is not None
):
permissions = {
"view": {
"users": self.override_view_users or [],
"groups": self.override_view_groups or [],
},
"change": {
"users": self.override_change_users or [],
"groups": self.override_change_groups or [],
},
}
set_permissions_for_object(permissions=permissions, object=document)
def _write(self, storage_type, source, target):
with open(source, "rb") as read_file, open(target, "wb") as write_file:
write_file.write(read_file.read())

62
src/documents/data_models.py
View File

@@ -1,6 +1,6 @@
import dataclasses
import datetime
import enum
from enum import IntEnum
from pathlib import Path
from typing import Optional
@@ -20,19 +20,70 @@ class DocumentMetadataOverrides:
correspondent_id: Optional[int] = None
document_type_id: Optional[int] = None
tag_ids: Optional[list[int]] = None
storage_path_id: Optional[int] = None
created: Optional[datetime.datetime] = None
asn: Optional[int] = None
owner_id: Optional[int] = None
view_users: Optional[list[int]] = None
view_groups: Optional[list[int]] = None
change_users: Optional[list[int]] = None
change_groups: Optional[list[int]] = None
def update(self, other: "DocumentMetadataOverrides") -> "DocumentMetadataOverrides":
"""
Merges two DocumentMetadataOverrides objects such that object B's overrides
are only applied if the property is empty in object A or merged if multiple
are accepted.
The update is an in-place modification of self
"""
# only if empty
if self.title is None:
self.title = other.title
if self.correspondent_id is None:
self.correspondent_id = other.correspondent_id
if self.document_type_id is None:
self.document_type_id = other.document_type_id
if self.storage_path_id is None:
self.storage_path_id = other.storage_path_id
if self.owner_id is None:
self.owner_id = other.owner_id
# merge
# TODO: Handle the case where other is also None
if self.tag_ids is None:
self.tag_ids = other.tag_ids
else:
self.tag_ids.extend(other.tag_ids)
if self.view_users is None:
self.view_users = other.view_users
else:
self.view_users.extend(other.view_users)
if self.view_groups is None:
self.view_groups = other.view_groups
else:
self.view_groups.extend(other.view_groups)
if self.change_users is None:
self.change_users = other.change_users
else:
self.change_users.extend(other.change_users)
if self.change_groups is None:
self.change_groups = other.change_groups
else:
self.change_groups = [
*self.change_groups,
*other.change_groups,
]
return self
class DocumentSource(enum.IntEnum):
class DocumentSource(IntEnum):
"""
The source of an incoming document. May have other uses in the future
"""
ConsumeFolder = enum.auto()
ApiUpload = enum.auto()
MailFetch = enum.auto()
ConsumeFolder = 1
ApiUpload = 2
MailFetch = 3
@dataclasses.dataclass
@@ -44,6 +95,7 @@ class ConsumableDocument:
source: DocumentSource
original_file: Path
mailrule_id: Optional[int] = None
mime_type: str = dataclasses.field(init=False, default=None)
def __post_init__(self):

68
src/documents/matching.py
View File

@@ -1,7 +1,11 @@
import logging
import re
from fnmatch import fnmatch
from documents.classifier import DocumentClassifier
from documents.data_models import ConsumableDocument
from documents.data_models import DocumentSource
from documents.models import ConsumptionTemplate
from documents.models import Correspondent
from documents.models import Document
from documents.models import DocumentType
@@ -231,3 +235,67 @@ def _split_match(matching_model):
re.escape(normspace(" ", (t[0] or t[1]).strip())).replace(r"\ ", r"\s+")
for t in findterms(matching_model.match)
]
def document_matches_template(
document: ConsumableDocument,
template: ConsumptionTemplate,
) -> bool:
"""
Returns True if the incoming document matches all filters and
settings from the template, False otherwise
"""
def log_match_failure(reason: str):
logger.info(f"Document did not match template {template.name}")
logger.debug(reason)
# Document source vs template source
if document.source not in [int(x) for x in list(template.sources)]:
log_match_failure(
f"Document source {document.source.name} not in"
f" {[DocumentSource(int(x)).name for x in template.sources]}",
)
return False
# Document mail rule vs template mail rule
if (
document.mailrule_id is not None
and template.filter_mailrule is not None
and document.mailrule_id != template.filter_mailrule.pk
):
log_match_failure(
f"Document mail rule {document.mailrule_id}"
f" != {template.filter_mailrule.pk}",
)
return False
# Document filename vs template filename
if (
template.filter_filename is not None
and len(template.filter_filename) > 0
and not fnmatch(
document.original_file.name.lower(),
template.filter_filename.lower(),
)
):
log_match_failure(
f"Document filename {document.original_file.name} does not match"
f" {template.filter_filename.lower()}",
)
return False
# Document path vs template path
if (
template.filter_path is not None
and len(template.filter_path) > 0
and not document.original_file.match(template.filter_path)
):
log_match_failure(
f"Document path {document.original_file}"
f" does not match {template.filter_path}",
)
return False
logger.info(f"Document matched template {template.name}")
return True

219
src/documents/migrations/1039_consumptiontemplate.py Normal file
View File

@@ -0,0 +1,219 @@
# Generated by Django 4.1.11 on 2023-09-16 18:04
import django.db.models.deletion
import multiselectfield.db.fields
from django.conf import settings
from django.contrib.auth.management import create_permissions
from django.contrib.auth.models import Group
from django.contrib.auth.models import Permission
from django.contrib.auth.models import User
from django.db import migrations
from django.db import models
from django.db.models import Q
def add_consumptiontemplate_permissions(apps, schema_editor):
# create permissions without waiting for post_migrate signal
for app_config in apps.get_app_configs():
app_config.models_module = True
create_permissions(app_config, apps=apps, verbosity=0)
app_config.models_module = None
add_permission = Permission.objects.get(codename="add_document")
consumptiontemplate_permissions = Permission.objects.filter(
codename__contains="consumptiontemplate",
)
for user in User.objects.filter(Q(user_permissions=add_permission)).distinct():
user.user_permissions.add(*consumptiontemplate_permissions)
for group in Group.objects.filter(Q(permissions=add_permission)).distinct():
group.permissions.add(*consumptiontemplate_permissions)
def remove_consumptiontemplate_permissions(apps, schema_editor):
consumptiontemplate_permissions = Permission.objects.filter(
codename__contains="consumptiontemplate",
)
for user in User.objects.all():
user.user_permissions.remove(*consumptiontemplate_permissions)
for group in Group.objects.all():
group.permissions.remove(*consumptiontemplate_permissions)
class Migration(migrations.Migration):
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
("auth", "0012_alter_user_first_name_max_length"),
("documents", "1038_sharelink"),
("paperless_mail", "0021_alter_mailaccount_password"),
]
operations = [
migrations.CreateModel(
name="ConsumptionTemplate",
fields=[
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
(
"name",
models.CharField(max_length=256, unique=True, verbose_name="name"),
),
("order", models.IntegerField(default=0, verbose_name="order")),
(
"sources",
multiselectfield.db.fields.MultiSelectField(
choices=[
(1, "Consume Folder"),
(2, "Api Upload"),
(3, "Mail Fetch"),
],
default="1,2,3",
max_length=3,
),
),
(
"filter_path",
models.CharField(
blank=True,
help_text="Only consume documents with a path that matches this if specified. Wildcards specified as * are allowed. Case insensitive.",
max_length=256,
null=True,
verbose_name="filter path",
),
),
(
"filter_filename",
models.CharField(
blank=True,
help_text="Only consume documents which entirely match this filename if specified. Wildcards such as *.pdf or *invoice* are allowed. Case insensitive.",
max_length=256,
null=True,
verbose_name="filter filename",
),
),
(
"filter_mailrule",
models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.SET_NULL,
to="paperless_mail.mailrule",
verbose_name="filter documents from this mail rule",
),
),
(
"assign_change_groups",
models.ManyToManyField(
blank=True,
related_name="+",
to="auth.group",
verbose_name="grant change permissions to these groups",
),
),
(
"assign_change_users",
models.ManyToManyField(
blank=True,
related_name="+",
to=settings.AUTH_USER_MODEL,
verbose_name="grant change permissions to these users",
),
),
(
"assign_correspondent",
models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.SET_NULL,
to="documents.correspondent",
verbose_name="assign this correspondent",
),
),
(
"assign_document_type",
models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.SET_NULL,
to="documents.documenttype",
verbose_name="assign this document type",
),
),
(
"assign_owner",
models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.SET_NULL,
related_name="+",
to=settings.AUTH_USER_MODEL,
verbose_name="assign this owner",
),
),
(
"assign_storage_path",
models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.SET_NULL,
to="documents.storagepath",
verbose_name="assign this storage path",
),
),
(
"assign_tags",
models.ManyToManyField(
blank=True,
to="documents.tag",
verbose_name="assign this tag",
),
),
(
"assign_title",
models.CharField(
blank=True,
help_text="Assign a document title, can include some placeholders, see documentation.",
max_length=256,
null=True,
verbose_name="assign title",
),
),
(
"assign_view_groups",
models.ManyToManyField(
blank=True,
related_name="+",
to="auth.group",
verbose_name="grant view permissions to these groups",
),
),
(
"assign_view_users",
models.ManyToManyField(
blank=True,
related_name="+",
to=settings.AUTH_USER_MODEL,
verbose_name="grant view permissions to these users",
),
),
],
options={
"verbose_name": "consumption template",
"verbose_name_plural": "consumption templates",
},
),
migrations.RunPython(
add_consumptiontemplate_permissions,
remove_consumptiontemplate_permissions,
),
]

143
src/documents/models.py
View File

@@ -11,18 +11,18 @@ import dateutil.parser
import pathvalidate
from celery import states
from django.conf import settings
from django.contrib.auth.models import Group
from django.contrib.auth.models import User
from django.core.validators import MaxValueValidator
from django.core.validators import MinValueValidator
from django.db import models
from django.utils import timezone
from django.utils.translation import gettext_lazy as _
from multiselectfield import MultiSelectField
from documents.data_models import DocumentSource
from documents.parsers import get_default_file_extension
ALL_STATES = sorted(states.ALL_STATES)
TASK_STATE_CHOICES = sorted(zip(ALL_STATES, ALL_STATES))
class ModelWithOwner(models.Model):
owner = models.ForeignKey(
@@ -572,6 +572,9 @@ class UiSettings(models.Model):
class PaperlessTask(models.Model):
ALL_STATES = sorted(states.ALL_STATES)
TASK_STATE_CHOICES = sorted(zip(ALL_STATES, ALL_STATES))
task_id = models.CharField(
max_length=255,
unique=True,
@@ -735,3 +738,137 @@ class ShareLink(models.Model):
def __str__(self):
return f"Share Link for {self.document.title}"
class ConsumptionTemplate(models.Model):
class DocumentSourceChoices(models.IntegerChoices):
CONSUME_FOLDER = DocumentSource.ConsumeFolder.value, _("Consume Folder")
API_UPLOAD = DocumentSource.ApiUpload.value, _("Api Upload")
MAIL_FETCH = DocumentSource.MailFetch.value, _("Mail Fetch")
name = models.CharField(_("name"), max_length=256, unique=True)
order = models.IntegerField(_("order"), default=0)
sources = MultiSelectField(
max_length=3,
choices=DocumentSourceChoices.choices,
default=f"{DocumentSource.ConsumeFolder},{DocumentSource.ApiUpload},{DocumentSource.MailFetch}",
)
filter_path = models.CharField(
_("filter path"),
max_length=256,
null=True,
blank=True,
help_text=_(
"Only consume documents with a path that matches "
"this if specified. Wildcards specified as * are "
"allowed. Case insensitive.",
),
)
filter_filename = models.CharField(
_("filter filename"),
max_length=256,
null=True,
blank=True,
help_text=_(
"Only consume documents which entirely match this "
"filename if specified. Wildcards such as *.pdf or "
"*invoice* are allowed. Case insensitive.",
),
)
filter_mailrule = models.ForeignKey(
"paperless_mail.MailRule",
null=True,
blank=True,
on_delete=models.SET_NULL,
verbose_name=_("filter documents from this mail rule"),
)
assign_title = models.CharField(
_("assign title"),
max_length=256,
null=True,
blank=True,
help_text=_(
"Assign a document title, can include some placeholders, "
"see documentation.",
),
)
assign_tags = models.ManyToManyField(
Tag,
blank=True,
verbose_name=_("assign this tag"),
)
assign_document_type = models.ForeignKey(
DocumentType,
null=True,
blank=True,
on_delete=models.SET_NULL,
verbose_name=_("assign this document type"),
)
assign_correspondent = models.ForeignKey(
Correspondent,
null=True,
blank=True,
on_delete=models.SET_NULL,
verbose_name=_("assign this correspondent"),
)
assign_storage_path = models.ForeignKey(
StoragePath,
null=True,
blank=True,
on_delete=models.SET_NULL,
verbose_name=_("assign this storage path"),
)
assign_owner = models.ForeignKey(
User,
null=True,
blank=True,
on_delete=models.SET_NULL,
related_name="+",
verbose_name=_("assign this owner"),
)
assign_view_users = models.ManyToManyField(
User,
blank=True,
related_name="+",
verbose_name=_("grant view permissions to these users"),
)
assign_view_groups = models.ManyToManyField(
Group,
blank=True,
related_name="+",
verbose_name=_("grant view permissions to these groups"),
)
assign_change_users = models.ManyToManyField(
User,
blank=True,
related_name="+",
verbose_name=_("grant change permissions to these users"),
)
assign_change_groups = models.ManyToManyField(
Group,
blank=True,
related_name="+",
verbose_name=_("grant change permissions to these groups"),
)
class Meta:
verbose_name = _("consumption template")
verbose_name_plural = _("consumption templates")
def __str__(self):
return f"{self.name}"

56
src/documents/serialisers.py
View File

@@ -13,9 +13,12 @@ from django.utils.text import slugify
from django.utils.translation import gettext as _
from guardian.core import ObjectPermissionChecker
from guardian.shortcuts import get_users_with_perms
from rest_framework import fields
from rest_framework import serializers
from rest_framework.fields import SerializerMethodField
from documents.data_models import DocumentSource
from documents.models import ConsumptionTemplate
from documents.permissions import get_groups_with_only_permission
from documents.permissions import set_permissions_for_object
@@ -1035,3 +1038,56 @@ class BulkEditObjectPermissionsSerializer(serializers.Serializer, SetPermissions
self._validate_permissions(permissions)
return attrs
class ConsumptionTemplateSerializer(serializers.ModelSerializer):
order = serializers.IntegerField(required=False)
sources = fields.MultipleChoiceField(
choices=ConsumptionTemplate.DocumentSourceChoices.choices,
allow_empty=False,
default={
DocumentSource.ConsumeFolder,
DocumentSource.ApiUpload,
DocumentSource.MailFetch,
},
)
assign_correspondent = CorrespondentField(allow_null=True, required=False)
assign_tags = TagsField(many=True, allow_null=True, required=False)
assign_document_type = DocumentTypeField(allow_null=True, required=False)
assign_storage_path = StoragePathField(allow_null=True, required=False)
class Meta:
model = ConsumptionTemplate
fields = [
"id",
"name",
"order",
"sources",
"filter_path",
"filter_filename",
"filter_mailrule",
"assign_title",
"assign_tags",
"assign_correspondent",
"assign_document_type",
"assign_storage_path",
"assign_owner",
"assign_view_users",
"assign_view_groups",
"assign_change_users",
"assign_change_groups",
]
def validate(self, attrs):
if ("filter_mailrule") in attrs and attrs["filter_mailrule"] is not None:
attrs["sources"] = {DocumentSource.MailFetch.value}
if (
("filter_mailrule" not in attrs)
and ("filter_filename" not in attrs or len(attrs["filter_filename"]) == 0)
and ("filter_path" not in attrs or len(attrs["filter_path"]) == 0)
):
raise serializers.ValidationError(
"File name, path or mail rule filter are required",
)
return attrs

11
src/documents/tasks.py
View File

@@ -153,6 +153,12 @@ def consume_file(
overrides.asn = reader.asn
logger.info(f"Found ASN in barcode: {overrides.asn}")
template_overrides = Consumer().get_template_overrides(
input_doc=input_doc,
)
overrides.update(template_overrides)
# continue with consumption if no barcode was found
document = Consumer().try_consume_file(
input_doc.original_file,
@@ -161,9 +167,14 @@ def consume_file(
override_correspondent_id=overrides.correspondent_id,
override_document_type_id=overrides.document_type_id,
override_tag_ids=overrides.tag_ids,
override_storage_path_id=overrides.storage_path_id,
override_created=overrides.created,
override_asn=overrides.asn,
override_owner_id=overrides.owner_id,
override_view_users=overrides.view_users,
override_view_groups=overrides.view_groups,
override_change_users=overrides.change_users,
override_change_groups=overrides.change_groups,
task_id=self.request.id,
)

169
src/documents/tests/test_api.py
View File

@@ -32,6 +32,8 @@ from whoosh.writing import AsyncWriter
from documents import bulk_edit
from documents import index
from documents.data_models import DocumentSource
from documents.models import ConsumptionTemplate
from documents.models import Correspondent
from documents.models import Document
from documents.models import DocumentType
@@ -45,6 +47,8 @@ from documents.models import Tag
from documents.tests.utils import DirectoriesMixin
from documents.tests.utils import DocumentConsumeDelayMixin
from paperless import version
from paperless_mail.models import MailAccount
from paperless_mail.models import MailRule
class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
@@ -5313,3 +5317,168 @@ class TestBulkEditObjectPermissions(APITestCase):
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
class TestApiConsumptionTemplates(DirectoriesMixin, APITestCase):
ENDPOINT = "/api/consumption_templates/"
def setUp(self) -> None:
super().setUp()
user = User.objects.create_superuser(username="temp_admin")
self.client.force_authenticate(user=user)
self.user2 = User.objects.create(username="user2")
self.user3 = User.objects.create(username="user3")
self.group1 = Group.objects.create(name="group1")
self.c = Correspondent.objects.create(name="Correspondent Name")
self.c2 = Correspondent.objects.create(name="Correspondent Name 2")
self.dt = DocumentType.objects.create(name="DocType Name")
self.t1 = Tag.objects.create(name="t1")
self.t2 = Tag.objects.create(name="t2")
self.t3 = Tag.objects.create(name="t3")
self.sp = StoragePath.objects.create(path="/test/")
self.ct = ConsumptionTemplate.objects.create(
name="Template 1",
order=0,
sources=f"{int(DocumentSource.ApiUpload)},{int(DocumentSource.ConsumeFolder)},{int(DocumentSource.MailFetch)}",
filter_filename="*simple*",
filter_path="*/samples/*",
assign_title="Doc from {correspondent}",
assign_correspondent=self.c,
assign_document_type=self.dt,
assign_storage_path=self.sp,
assign_owner=self.user2,
)
self.ct.assign_tags.add(self.t1)
self.ct.assign_tags.add(self.t2)
self.ct.assign_tags.add(self.t3)
self.ct.assign_view_users.add(self.user3.pk)
self.ct.assign_view_groups.add(self.group1.pk)
self.ct.assign_change_users.add(self.user3.pk)
self.ct.assign_change_groups.add(self.group1.pk)
self.ct.save()
def test_api_get_consumption_template(self):
"""
GIVEN:
- API request to get all consumption template
WHEN:
- API is called
THEN:
- Existing consumption templates are returned
"""
response = self.client.get(self.ENDPOINT, format="json")
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data["count"], 1)
resp_consumption_template = response.data["results"][0]
self.assertEqual(resp_consumption_template["id"], self.ct.id)
self.assertEqual(
resp_consumption_template["assign_correspondent"],
self.ct.assign_correspondent.pk,
)
def test_api_create_consumption_template(self):
"""
GIVEN:
- API request to create a consumption template
WHEN:
- API is called
THEN:
- Correct HTTP response
- New template is created
"""
response = self.client.post(
self.ENDPOINT,
json.dumps(
{
"name": "Template 2",
"order": 1,
"sources": [DocumentSource.ApiUpload],
"filter_filename": "*test*",
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
self.assertEqual(ConsumptionTemplate.objects.count(), 2)
def test_api_create_invalid_consumption_template(self):
"""
GIVEN:
- API request to create a consumption template
- Neither file name nor path filter are specified
WHEN:
- API is called
THEN:
- Correct HTTP 400 response
- No template is created
"""
response = self.client.post(
self.ENDPOINT,
json.dumps(
{
"name": "Template 2",
"order": 1,
"sources": [DocumentSource.ApiUpload],
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertEqual(StoragePath.objects.count(), 1)
def test_api_create_consumption_template_with_mailrule(self):
"""
GIVEN:
- API request to create a consumption template with a mail rule but no MailFetch source
WHEN:
- API is called
THEN:
- Correct HTTP response
- New template is created with MailFetch as source
"""
account1 = MailAccount.objects.create(
name="Email1",
username="username1",
password="password1",
imap_server="server.example.com",
imap_port=443,
imap_security=MailAccount.ImapSecurity.SSL,
character_set="UTF-8",
)
rule1 = MailRule.objects.create(
name="Rule1",
account=account1,
folder="INBOX",
filter_from="from@example.com",
filter_to="someone@somewhere.com",
filter_subject="subject",
filter_body="body",
filter_attachment_filename="file.pdf",
maximum_age=30,
action=MailRule.MailAction.MARK_READ,
assign_title_from=MailRule.TitleSource.FROM_SUBJECT,
assign_correspondent_from=MailRule.CorrespondentSource.FROM_NOTHING,
order=0,
attachment_type=MailRule.AttachmentProcessing.ATTACHMENTS_ONLY,
)
response = self.client.post(
self.ENDPOINT,
json.dumps(
{
"name": "Template 2",
"order": 1,
"sources": [DocumentSource.ApiUpload],
"filter_mailrule": rule1.pk,
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
self.assertEqual(ConsumptionTemplate.objects.count(), 2)
ct = ConsumptionTemplate.objects.get(name="Template 2")
self.assertEqual(ct.sources, [int(DocumentSource.MailFetch).__str__()])

59
src/documents/tests/test_consumer.py
View File

@@ -11,9 +11,12 @@ from unittest.mock import MagicMock
from dateutil import tz
from django.conf import settings
from django.contrib.auth.models import Group
from django.contrib.auth.models import User
from django.test import TestCase
from django.test import override_settings
from django.utils import timezone
from guardian.core import ObjectPermissionChecker
from documents.consumer import Consumer
from documents.consumer import ConsumerError
@@ -22,6 +25,7 @@ from documents.models import Correspondent
from documents.models import Document
from documents.models import DocumentType
from documents.models import FileInfo
from documents.models import StoragePath
from documents.models import Tag
from documents.parsers import DocumentParser
from documents.parsers import ParseError
@@ -431,6 +435,16 @@ class TestConsumer(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
self.assertEqual(document.document_type.id, dt.id)
self._assert_first_last_send_progress()
def testOverrideStoragePath(self):
sp = StoragePath.objects.create(name="test")
document = self.consumer.try_consume_file(
self.get_test_file(),
override_storage_path_id=sp.pk,
)
self.assertEqual(document.storage_path.id, sp.id)
self._assert_first_last_send_progress()
def testOverrideTags(self):
t1 = Tag.objects.create(name="t1")
t2 = Tag.objects.create(name="t2")
@@ -445,6 +459,51 @@ class TestConsumer(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
self.assertIn(t3, document.tags.all())
self._assert_first_last_send_progress()
def testOverrideAsn(self):
document = self.consumer.try_consume_file(
self.get_test_file(),
override_asn=123,
)
self.assertEqual(document.archive_serial_number, 123)
self._assert_first_last_send_progress()
def testOverrideTitlePlaceholders(self):
c = Correspondent.objects.create(name="Correspondent Name")
dt = DocumentType.objects.create(name="DocType Name")
document = self.consumer.try_consume_file(
self.get_test_file(),
override_correspondent_id=c.pk,
override_document_type_id=dt.pk,
override_title="{correspondent}{document_type} {added_month}-{added_year_short}",
)
now = timezone.now()
self.assertEqual(document.title, f"{c.name}{dt.name} {now.strftime('%m-%y')}")
self._assert_first_last_send_progress()
def testOverrideOwner(self):
testuser = User.objects.create(username="testuser")
document = self.consumer.try_consume_file(
self.get_test_file(),
override_owner_id=testuser.pk,
)
self.assertEqual(document.owner, testuser)
self._assert_first_last_send_progress()
def testOverridePermissions(self):
testuser = User.objects.create(username="testuser")
testgroup = Group.objects.create(name="testgroup")
document = self.consumer.try_consume_file(
self.get_test_file(),
override_view_users=[testuser.pk],
override_view_groups=[testgroup.pk],
)
user_checker = ObjectPermissionChecker(testuser)
self.assertTrue(user_checker.has_perm("view_document", document))
group_checker = ObjectPermissionChecker(testgroup)
self.assertTrue(group_checker.has_perm("view_document", document))
self._assert_first_last_send_progress()
def testNotAFile(self):
self.assertRaisesMessage(
ConsumerError,

476
src/documents/tests/test_consumption_templates.py Normal file
View File

@@ -0,0 +1,476 @@
from pathlib import Path
from unittest import TestCase
from unittest import mock
import pytest
from django.contrib.auth.models import Group
from django.contrib.auth.models import User
from documents import tasks
from documents.data_models import ConsumableDocument
from documents.data_models import DocumentSource
from documents.models import ConsumptionTemplate
from documents.models import Correspondent
from documents.models import DocumentType
from documents.models import StoragePath
from documents.models import Tag
from documents.tests.utils import DirectoriesMixin
from documents.tests.utils import FileSystemAssertsMixin
from paperless_mail.models import MailAccount
from paperless_mail.models import MailRule
@pytest.mark.django_db
class TestConsumptionTemplates(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
SAMPLE_DIR = Path(__file__).parent / "samples"
def setUp(self) -> None:
self.c = Correspondent.objects.create(name="Correspondent Name")
self.c2 = Correspondent.objects.create(name="Correspondent Name 2")
self.dt = DocumentType.objects.create(name="DocType Name")
self.t1 = Tag.objects.create(name="t1")
self.t2 = Tag.objects.create(name="t2")
self.t3 = Tag.objects.create(name="t3")
self.sp = StoragePath.objects.create(path="/test/")
self.user2 = User.objects.create(username="user2")
self.user3 = User.objects.create(username="user3")
self.group1 = Group.objects.create(name="group1")
account1 = MailAccount.objects.create(
name="Email1",
username="username1",
password="password1",
imap_server="server.example.com",
imap_port=443,
imap_security=MailAccount.ImapSecurity.SSL,
character_set="UTF-8",
)
self.rule1 = MailRule.objects.create(
name="Rule1",
account=account1,
folder="INBOX",
filter_from="from@example.com",
filter_to="someone@somewhere.com",
filter_subject="subject",
filter_body="body",
filter_attachment_filename="file.pdf",
maximum_age=30,
action=MailRule.MailAction.MARK_READ,
assign_title_from=MailRule.TitleSource.NONE,
assign_correspondent_from=MailRule.CorrespondentSource.FROM_NOTHING,
order=0,
attachment_type=MailRule.AttachmentProcessing.ATTACHMENTS_ONLY,
assign_owner_from_rule=False,
)
return super().setUp()
@mock.patch("documents.consumer.Consumer.try_consume_file")
def test_consumption_template_match(self, m):
"""
GIVEN:
- Existing consumption template
WHEN:
- File that matches is consumed
THEN:
- Template overrides are applied
"""
ct = ConsumptionTemplate.objects.create(
name="Template 1",
order=0,
sources=f"{DocumentSource.ApiUpload},{DocumentSource.ConsumeFolder},{DocumentSource.MailFetch}",
filter_filename="*simple*",
filter_path="*/samples/*",
assign_title="Doc from {correspondent}",
assign_correspondent=self.c,
assign_document_type=self.dt,
assign_storage_path=self.sp,
assign_owner=self.user2,
)
ct.assign_tags.add(self.t1)
ct.assign_tags.add(self.t2)
ct.assign_tags.add(self.t3)
ct.assign_view_users.add(self.user3.pk)
ct.assign_view_groups.add(self.group1.pk)
ct.assign_change_users.add(self.user3.pk)
ct.assign_change_groups.add(self.group1.pk)
ct.save()
self.assertEqual(ct.__str__(), "Template 1")
test_file = self.SAMPLE_DIR / "simple.pdf"
with mock.patch("documents.tasks.async_to_sync"):
with self.assertLogs("paperless.matching", level="INFO") as cm:
tasks.consume_file(
ConsumableDocument(
source=DocumentSource.ConsumeFolder,
original_file=test_file,
),
None,
)
m.assert_called_once()
_, overrides = m.call_args
self.assertEqual(overrides["override_correspondent_id"], self.c.pk)
self.assertEqual(overrides["override_document_type_id"], self.dt.pk)
self.assertEqual(
overrides["override_tag_ids"],
[self.t1.pk, self.t2.pk, self.t3.pk],
)
self.assertEqual(overrides["override_storage_path_id"], self.sp.pk)
self.assertEqual(overrides["override_owner_id"], self.user2.pk)
self.assertEqual(overrides["override_view_users"], [self.user3.pk])
self.assertEqual(overrides["override_view_groups"], [self.group1.pk])
self.assertEqual(overrides["override_change_users"], [self.user3.pk])
self.assertEqual(overrides["override_change_groups"], [self.group1.pk])
self.assertEqual(
overrides["override_title"],
"Doc from {correspondent}",
)
info = cm.output[0]
expected_str = f"Document matched template {ct}"
self.assertIn(expected_str, info)
@mock.patch("documents.consumer.Consumer.try_consume_file")
def test_consumption_template_match_mailrule(self, m):
"""
GIVEN:
- Existing consumption template
WHEN:
- File that matches is consumed via mail rule
THEN:
- Template overrides are applied
"""
ct = ConsumptionTemplate.objects.create(
name="Template 1",
order=0,
sources=f"{DocumentSource.ApiUpload},{DocumentSource.ConsumeFolder},{DocumentSource.MailFetch}",
filter_mailrule=self.rule1,
assign_title="Doc from {correspondent}",
assign_correspondent=self.c,
assign_document_type=self.dt,
assign_storage_path=self.sp,
assign_owner=self.user2,
)
ct.assign_tags.add(self.t1)
ct.assign_tags.add(self.t2)
ct.assign_tags.add(self.t3)
ct.assign_view_users.add(self.user3.pk)
ct.assign_view_groups.add(self.group1.pk)
ct.assign_change_users.add(self.user3.pk)
ct.assign_change_groups.add(self.group1.pk)
ct.save()
self.assertEqual(ct.__str__(), "Template 1")
test_file = self.SAMPLE_DIR / "simple.pdf"
with mock.patch("documents.tasks.async_to_sync"):
with self.assertLogs("paperless.matching", level="INFO") as cm:
tasks.consume_file(
ConsumableDocument(
source=DocumentSource.ConsumeFolder,
original_file=test_file,
mailrule_id=self.rule1.pk,
),
None,
)
m.assert_called_once()
_, overrides = m.call_args
self.assertEqual(overrides["override_correspondent_id"], self.c.pk)
self.assertEqual(overrides["override_document_type_id"], self.dt.pk)
self.assertEqual(
overrides["override_tag_ids"],
[self.t1.pk, self.t2.pk, self.t3.pk],
)
self.assertEqual(overrides["override_storage_path_id"], self.sp.pk)
self.assertEqual(overrides["override_owner_id"], self.user2.pk)
self.assertEqual(overrides["override_view_users"], [self.user3.pk])
self.assertEqual(overrides["override_view_groups"], [self.group1.pk])
self.assertEqual(overrides["override_change_users"], [self.user3.pk])
self.assertEqual(overrides["override_change_groups"], [self.group1.pk])
self.assertEqual(
overrides["override_title"],
"Doc from {correspondent}",
)
info = cm.output[0]
expected_str = f"Document matched template {ct}"
self.assertIn(expected_str, info)
@mock.patch("documents.consumer.Consumer.try_consume_file")
def test_consumption_template_match_multiple(self, m):
"""
GIVEN:
- Multiple existing consumption template
WHEN:
- File that matches is consumed
THEN:
- Template overrides are applied with subsequent templates only overwriting empty values
or merging if multiple
"""
ct1 = ConsumptionTemplate.objects.create(
name="Template 1",
order=0,
sources=f"{DocumentSource.ApiUpload},{DocumentSource.ConsumeFolder},{DocumentSource.MailFetch}",
filter_path="*/samples/*",
assign_title="Doc from {correspondent}",
assign_correspondent=self.c,
assign_document_type=self.dt,
)
ct1.assign_tags.add(self.t1)
ct1.assign_tags.add(self.t2)
ct1.assign_view_users.add(self.user2)
ct1.save()
ct2 = ConsumptionTemplate.objects.create(
name="Template 2",
order=0,
sources=f"{DocumentSource.ApiUpload},{DocumentSource.ConsumeFolder},{DocumentSource.MailFetch}",
filter_filename="*simple*",
assign_title="Doc from {correspondent}",
assign_correspondent=self.c2,
assign_storage_path=self.sp,
)
ct2.assign_tags.add(self.t3)
ct1.assign_view_users.add(self.user3)
ct2.save()
test_file = self.SAMPLE_DIR / "simple.pdf"
with mock.patch("documents.tasks.async_to_sync"):
with self.assertLogs("paperless.matching", level="INFO") as cm:
tasks.consume_file(
ConsumableDocument(
source=DocumentSource.ConsumeFolder,
original_file=test_file,
),
None,
)
m.assert_called_once()
_, overrides = m.call_args
# template 1
self.assertEqual(overrides["override_correspondent_id"], self.c.pk)
self.assertEqual(overrides["override_document_type_id"], self.dt.pk)
# template 2
self.assertEqual(overrides["override_storage_path_id"], self.sp.pk)
# template 1 & 2
self.assertEqual(
overrides["override_tag_ids"],
[self.t1.pk, self.t2.pk, self.t3.pk],
)
self.assertEqual(
overrides["override_view_users"],
[self.user2.pk, self.user3.pk],
)
expected_str = f"Document matched template {ct1}"
self.assertIn(expected_str, cm.output[0])
expected_str = f"Document matched template {ct2}"
self.assertIn(expected_str, cm.output[1])
@mock.patch("documents.consumer.Consumer.try_consume_file")
def test_consumption_template_no_match_filename(self, m):
"""
GIVEN:
- Existing consumption template
WHEN:
- File that does not match on filename is consumed
THEN:
- Template overrides are not applied
"""
ct = ConsumptionTemplate.objects.create(
name="Template 1",
order=0,
sources=f"{DocumentSource.ApiUpload},{DocumentSource.ConsumeFolder},{DocumentSource.MailFetch}",
filter_filename="*foobar*",
filter_path=None,
assign_title="Doc from {correspondent}",
assign_correspondent=self.c,
assign_document_type=self.dt,
assign_storage_path=self.sp,
assign_owner=self.user2,
)
test_file = self.SAMPLE_DIR / "simple.pdf"
with mock.patch("documents.tasks.async_to_sync"):
with self.assertLogs("paperless.matching", level="DEBUG") as cm:
tasks.consume_file(
ConsumableDocument(
source=DocumentSource.ConsumeFolder,
original_file=test_file,
),
None,
)
m.assert_called_once()
_, overrides = m.call_args
self.assertIsNone(overrides["override_correspondent_id"])
self.assertIsNone(overrides["override_document_type_id"])
self.assertIsNone(overrides["override_tag_ids"])
self.assertIsNone(overrides["override_storage_path_id"])
self.assertIsNone(overrides["override_owner_id"])
self.assertIsNone(overrides["override_view_users"])
self.assertIsNone(overrides["override_view_groups"])
self.assertIsNone(overrides["override_change_users"])
self.assertIsNone(overrides["override_change_groups"])
self.assertIsNone(overrides["override_title"])
expected_str = f"Document did not match template {ct}"
self.assertIn(expected_str, cm.output[0])
expected_str = f"Document filename {test_file.name} does not match"
self.assertIn(expected_str, cm.output[1])
@mock.patch("documents.consumer.Consumer.try_consume_file")
def test_consumption_template_no_match_path(self, m):
"""
GIVEN:
- Existing consumption template
WHEN:
- File that does not match on path is consumed
THEN:
- Template overrides are not applied
"""
ct = ConsumptionTemplate.objects.create(
name="Template 1",
order=0,
sources=f"{DocumentSource.ApiUpload},{DocumentSource.ConsumeFolder},{DocumentSource.MailFetch}",
filter_path="*foo/bar*",
assign_title="Doc from {correspondent}",
assign_correspondent=self.c,
assign_document_type=self.dt,
assign_storage_path=self.sp,
assign_owner=self.user2,
)
test_file = self.SAMPLE_DIR / "simple.pdf"
with mock.patch("documents.tasks.async_to_sync"):
with self.assertLogs("paperless.matching", level="DEBUG") as cm:
tasks.consume_file(
ConsumableDocument(
source=DocumentSource.ConsumeFolder,
original_file=test_file,
),
None,
)
m.assert_called_once()
_, overrides = m.call_args
self.assertIsNone(overrides["override_correspondent_id"])
self.assertIsNone(overrides["override_document_type_id"])
self.assertIsNone(overrides["override_tag_ids"])
self.assertIsNone(overrides["override_storage_path_id"])
self.assertIsNone(overrides["override_owner_id"])
self.assertIsNone(overrides["override_view_users"])
self.assertIsNone(overrides["override_view_groups"])
self.assertIsNone(overrides["override_change_users"])
self.assertIsNone(overrides["override_change_groups"])
self.assertIsNone(overrides["override_title"])
expected_str = f"Document did not match template {ct}"
self.assertIn(expected_str, cm.output[0])
expected_str = f"Document path {test_file} does not match"
self.assertIn(expected_str, cm.output[1])
@mock.patch("documents.consumer.Consumer.try_consume_file")
def test_consumption_template_no_match_mail_rule(self, m):
"""
GIVEN:
- Existing consumption template
WHEN:
- File that does not match on source is consumed
THEN:
- Template overrides are not applied
"""
ct = ConsumptionTemplate.objects.create(
name="Template 1",
order=0,
sources=f"{DocumentSource.ApiUpload},{DocumentSource.ConsumeFolder},{DocumentSource.MailFetch}",
filter_mailrule=self.rule1,
assign_title="Doc from {correspondent}",
assign_correspondent=self.c,
assign_document_type=self.dt,
assign_storage_path=self.sp,
assign_owner=self.user2,
)
test_file = self.SAMPLE_DIR / "simple.pdf"
with mock.patch("documents.tasks.async_to_sync"):
with self.assertLogs("paperless.matching", level="DEBUG") as cm:
tasks.consume_file(
ConsumableDocument(
source=DocumentSource.ConsumeFolder,
original_file=test_file,
mailrule_id=99,
),
None,
)
m.assert_called_once()
_, overrides = m.call_args
self.assertIsNone(overrides["override_correspondent_id"])
self.assertIsNone(overrides["override_document_type_id"])
self.assertIsNone(overrides["override_tag_ids"])
self.assertIsNone(overrides["override_storage_path_id"])
self.assertIsNone(overrides["override_owner_id"])
self.assertIsNone(overrides["override_view_users"])
self.assertIsNone(overrides["override_view_groups"])
self.assertIsNone(overrides["override_change_users"])
self.assertIsNone(overrides["override_change_groups"])
self.assertIsNone(overrides["override_title"])
expected_str = f"Document did not match template {ct}"
self.assertIn(expected_str, cm.output[0])
expected_str = "Document mail rule 99 !="
self.assertIn(expected_str, cm.output[1])
@mock.patch("documents.consumer.Consumer.try_consume_file")
def test_consumption_template_no_match_source(self, m):
"""
GIVEN:
- Existing consumption template
WHEN:
- File that does not match on source is consumed
THEN:
- Template overrides are not applied
"""
ct = ConsumptionTemplate.objects.create(
name="Template 1",
order=0,
sources=f"{DocumentSource.ConsumeFolder},{DocumentSource.MailFetch}",
filter_path="*",
assign_title="Doc from {correspondent}",
assign_correspondent=self.c,
assign_document_type=self.dt,
assign_storage_path=self.sp,
assign_owner=self.user2,
)
test_file = self.SAMPLE_DIR / "simple.pdf"
with mock.patch("documents.tasks.async_to_sync"):
with self.assertLogs("paperless.matching", level="DEBUG") as cm:
tasks.consume_file(
ConsumableDocument(
source=DocumentSource.ApiUpload,
original_file=test_file,
),
None,
)
m.assert_called_once()
_, overrides = m.call_args
self.assertIsNone(overrides["override_correspondent_id"])
self.assertIsNone(overrides["override_document_type_id"])
self.assertIsNone(overrides["override_tag_ids"])
self.assertIsNone(overrides["override_storage_path_id"])
self.assertIsNone(overrides["override_owner_id"])
self.assertIsNone(overrides["override_view_users"])
self.assertIsNone(overrides["override_view_groups"])
self.assertIsNone(overrides["override_change_users"])
self.assertIsNone(overrides["override_change_groups"])
self.assertIsNone(overrides["override_title"])
expected_str = f"Document did not match template {ct}"
self.assertIn(expected_str, cm.output[0])
expected_str = f"Document source {DocumentSource.ApiUpload.name} not in ['{DocumentSource.ConsumeFolder.name}', '{DocumentSource.MailFetch.name}']"
self.assertIn(expected_str, cm.output[1])

16
src/documents/tests/test_management_exporter.py
View File

@@ -153,7 +153,7 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
manifest = self._do_export(use_filename_format=use_filename_format)
self.assertEqual(len(manifest), 154)
self.assertEqual(len(manifest), 159)
# dont include consumer or AnonymousUser users
self.assertEqual(
@@ -247,7 +247,7 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
self.assertEqual(Document.objects.get(id=self.d4.id).title, "wow_dec")
self.assertEqual(GroupObjectPermission.objects.count(), 1)
self.assertEqual(UserObjectPermission.objects.count(), 1)
self.assertEqual(Permission.objects.count(), 112)
self.assertEqual(Permission.objects.count(), 116)
messages = check_sanity()
# everything is alright after the test
self.assertEqual(len(messages), 0)
@@ -676,15 +676,15 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
os.path.join(self.dirs.media_dir, "documents"),
)
self.assertEqual(ContentType.objects.count(), 28)
self.assertEqual(Permission.objects.count(), 112)
self.assertEqual(ContentType.objects.count(), 29)
self.assertEqual(Permission.objects.count(), 116)
manifest = self._do_export()
with paperless_environment():
self.assertEqual(
len(list(filter(lambda e: e["model"] == "auth.permission", manifest))),
112,
116,
)
# add 1 more to db to show objects are not re-created by import
Permission.objects.create(
@@ -692,7 +692,7 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
codename="test_perm",
content_type_id=1,
)
self.assertEqual(Permission.objects.count(), 113)
self.assertEqual(Permission.objects.count(), 117)
# will cause an import error
self.user.delete()
@@ -701,5 +701,5 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
with self.assertRaises(IntegrityError):
call_command("document_importer", "--no-progress-bar", self.target)
self.assertEqual(ContentType.objects.count(), 28)
self.assertEqual(Permission.objects.count(), 113)
self.assertEqual(ContentType.objects.count(), 29)
self.assertEqual(Permission.objects.count(), 117)

43
src/documents/tests/test_migration_consumption_templates.py Normal file
View File

@@ -0,0 +1,43 @@
from django.contrib.auth import get_user_model
from documents.tests.utils import TestMigrations
class TestMigrateConsumptionTemplate(TestMigrations):
migrate_from = "1038_sharelink"
migrate_to = "1039_consumptiontemplate"
def setUpBeforeMigration(self, apps):
User = get_user_model()
Group = apps.get_model("auth.Group")
self.Permission = apps.get_model("auth", "Permission")
self.user = User.objects.create(username="user1")
self.group = Group.objects.create(name="group1")
permission = self.Permission.objects.get(codename="add_document")
self.user.user_permissions.add(permission.id)
self.group.permissions.add(permission.id)
def test_users_with_add_documents_get_add_consumptiontemplate(self):
permission = self.Permission.objects.get(codename="add_consumptiontemplate")
self.assertTrue(self.user.has_perm(f"documents.{permission.codename}"))
self.assertTrue(permission in self.group.permissions.all())
class TestReverseMigrateConsumptionTemplate(TestMigrations):
migrate_from = "1039_consumptiontemplate"
migrate_to = "1038_sharelink"
def setUpBeforeMigration(self, apps):
User = get_user_model()
Group = apps.get_model("auth.Group")
self.Permission = apps.get_model("auth", "Permission")
self.user = User.objects.create(username="user1")
self.group = Group.objects.create(name="group1")
permission = self.Permission.objects.get(codename="add_consumptiontemplate")
self.user.user_permissions.add(permission.id)
self.group.permissions.add(permission.id)
def test_remove_consumptiontemplate_permissions(self):
permission = self.Permission.objects.get(codename="add_consumptiontemplate")
self.assertFalse(self.user.has_perm(f"documents.{permission.codename}"))
self.assertFalse(permission in self.group.permissions.all())

13
src/documents/views.py
View File

@@ -86,6 +86,7 @@ from .matching import match_correspondents
from .matching import match_document_types
from .matching import match_storage_paths
from .matching import match_tags
from .models import ConsumptionTemplate
from .models import Correspondent
from .models import Document
from .models import DocumentType
@@ -101,6 +102,7 @@ from .serialisers import AcknowledgeTasksViewSerializer
from .serialisers import BulkDownloadSerializer
from .serialisers import BulkEditObjectPermissionsSerializer
from .serialisers import BulkEditSerializer
from .serialisers import ConsumptionTemplateSerializer
from .serialisers import CorrespondentSerializer
from .serialisers import DocumentListSerializer
from .serialisers import DocumentSerializer
@@ -1248,3 +1250,14 @@ class BulkEditObjectPermissionsView(GenericAPIView, PassUserMixin):
return HttpResponseBadRequest(
"Error performing bulk permissions edit, check logs for more detail.",
)
class ConsumptionTemplateViewSet(ModelViewSet):
permission_classes = (IsAuthenticated, PaperlessObjectPermissions)
serializer_class = ConsumptionTemplateSerializer
pagination_class = StandardPagination
model = ConsumptionTemplate
queryset = ConsumptionTemplate.objects.all().order_by("order")