paradizelost/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2025-10-30 03:56:23 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/dev' into feature-multiple-barcode-scanners

Browse Source
This commit is contained in:
Trenton H
2023-03-29 09:40:10 -07:00
parent 0778c2808b 45c9518a6d
commit 66929a9088
104 changed files with 2615 additions and 1098 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
src/documents/admin.py
View File

@@ -4,6 +4,7 @@ from guardian.admin import GuardedModelAdmin
from .models import Correspondent
from .models import Document
from .models import DocumentType
from .models import Note
from .models import PaperlessTask
from .models import SavedView
from .models import SavedViewFilterRule
@@ -131,6 +132,13 @@ class TaskAdmin(admin.ModelAdmin):
)
class NotesAdmin(GuardedModelAdmin):
list_display = ("user", "created", "note", "document")
list_filter = ("created", "user")
list_display_links = ("created",)
admin.site.register(Correspondent, CorrespondentAdmin)
admin.site.register(Tag, TagAdmin)
admin.site.register(DocumentType, DocumentTypeAdmin)
@@ -138,3 +146,4 @@ admin.site.register(Document, DocumentAdmin)
admin.site.register(SavedView, SavedViewAdmin)
admin.site.register(StoragePath, StoragePathAdmin)
admin.site.register(PaperlessTask, TaskAdmin)
admin.site.register(Note, NotesAdmin)

27
src/documents/filters.py
View File

@@ -36,29 +36,30 @@ class DocumentTypeFilterSet(FilterSet):
fields = {"name": CHAR_KWARGS}
class TagsFilter(Filter):
def __init__(self, exclude=False, in_list=False):
class ObjectFilter(Filter):
def __init__(self, exclude=False, in_list=False, field_name=""):
super().__init__()
self.exclude = exclude
self.in_list = in_list
self.field_name = field_name
def filter(self, qs, value):
if not value:
return qs
try:
tag_ids = [int(x) for x in value.split(",")]
object_ids = [int(x) for x in value.split(",")]
except ValueError:
return qs
if self.in_list:
qs = qs.filter(tags__id__in=tag_ids).distinct()
qs = qs.filter(**{f"{self.field_name}__id__in": object_ids}).distinct()
else:
for tag_id in tag_ids:
for obj_id in object_ids:
if self.exclude:
qs = qs.exclude(tags__id=tag_id)
qs = qs.exclude(**{f"{self.field_name}__id": obj_id})
else:
qs = qs.filter(tags__id=tag_id)
qs = qs.filter(**{f"{self.field_name}__id": obj_id})
return qs
@@ -90,11 +91,17 @@ class DocumentFilterSet(FilterSet):
exclude=True,
)
tags__id__all = TagsFilter()
tags__id__all = ObjectFilter(field_name="tags")
tags__id__none = TagsFilter(exclude=True)
tags__id__none = ObjectFilter(field_name="tags", exclude=True)
tags__id__in = TagsFilter(in_list=True)
tags__id__in = ObjectFilter(field_name="tags", in_list=True)
correspondent__id__none = ObjectFilter(field_name="correspondent", exclude=True)
document_type__id__none = ObjectFilter(field_name="document_type", exclude=True)
storage_path__id__none = ObjectFilter(field_name="storage_path", exclude=True)
is_in_inbox = InboxFilter()

10
src/documents/index.py
View File

@@ -6,8 +6,8 @@ from contextlib import contextmanager
from dateutil.parser import isoparse
from django.conf import settings
from django.utils import timezone
from documents.models import Comment
from documents.models import Document
from documents.models import Note
from guardian.shortcuts import get_users_with_perms
from whoosh import classify
from whoosh import highlight
@@ -52,7 +52,7 @@ def get_schema():
path=TEXT(sortable=True),
path_id=NUMERIC(),
has_path=BOOLEAN(),
comments=TEXT(),
notes=TEXT(),
owner=TEXT(),
owner_id=NUMERIC(),
has_owner=BOOLEAN(),
@@ -98,7 +98,7 @@ def open_index_searcher():
def update_document(writer: AsyncWriter, doc: Document):
tags = ",".join([t.name for t in doc.tags.all()])
tags_ids = ",".join([str(t.id) for t in doc.tags.all()])
comments = ",".join([str(c.comment) for c in Comment.objects.filter(document=doc)])
notes = ",".join([str(c.note) for c in Note.objects.filter(document=doc)])
asn = doc.archive_serial_number
if asn is not None and (
asn < Document.ARCHIVE_SERIAL_NUMBER_MIN
@@ -136,7 +136,7 @@ def update_document(writer: AsyncWriter, doc: Document):
path=doc.storage_path.name if doc.storage_path else None,
path_id=doc.storage_path.id if doc.storage_path else None,
has_path=doc.storage_path is not None,
comments=comments,
notes=notes,
owner=doc.owner.username if doc.owner else None,
owner_id=doc.owner.id if doc.owner else None,
has_owner=doc.owner is not None,
@@ -293,7 +293,7 @@ class DelayedFullTextQuery(DelayedQuery):
def _get_query(self):
q_str = self.query_params["query"]
qp = MultifieldParser(
["content", "title", "correspondent", "tag", "type", "comments"],
["content", "title", "correspondent", "tag", "type", "notes"],
self.searcher.ixreader.schema,
)
qp.add_plugin(DateParserPlugin(basedate=timezone.now()))

33
src/documents/management/commands/document_consumer.py
View File

@@ -1,10 +1,10 @@
import logging
import os
from concurrent.futures import ThreadPoolExecutor
from fnmatch import filter
from pathlib import Path
from pathlib import PurePath
from threading import Event
from threading import Thread
from time import monotonic
from time import sleep
from typing import Final
@@ -168,11 +168,15 @@ def _consume_wait_unmodified(file: str) -> None:
class Handler(FileSystemEventHandler):
def __init__(self, pool: ThreadPoolExecutor) -> None:
super().__init__()
self._pool = pool
def on_created(self, event):
Thread(target=_consume_wait_unmodified, args=(event.src_path,)).start()
self._pool.submit(_consume_wait_unmodified, event.src_path)
def on_moved(self, event):
Thread(target=_consume_wait_unmodified, args=(event.dest_path,)).start()
self._pool.submit(_consume_wait_unmodified, event.dest_path)
class Command(BaseCommand):
@@ -246,17 +250,18 @@ class Command(BaseCommand):
timeout = self.testing_timeout_s
logger.debug(f"Configuring timeout to {timeout}s")
observer = PollingObserver(timeout=settings.CONSUMER_POLLING)
observer.schedule(Handler(), directory, recursive=recursive)
observer.start()
try:
while observer.is_alive():
observer.join(timeout)
if self.stop_flag.is_set():
observer.stop()
except KeyboardInterrupt:
observer.stop()
observer.join()
with ThreadPoolExecutor(max_workers=4) as pool:
observer = PollingObserver(timeout=settings.CONSUMER_POLLING)
observer.schedule(Handler(pool), directory, recursive=recursive)
observer.start()
try:
while observer.is_alive():
observer.join(timeout)
if self.stop_flag.is_set():
observer.stop()
except KeyboardInterrupt:
observer.stop()
observer.join()
def handle_inotify(self, directory, recursive, is_testing: bool):
logger.info(f"Using inotify to watch directory for changes: {directory}")

12
src/documents/management/commands/document_exporter.py
View File

@@ -17,10 +17,10 @@ from django.core.management.base import BaseCommand
from django.core.management.base import CommandError
from django.db import transaction
from django.utils import timezone
from documents.models import Comment
from documents.models import Correspondent
from documents.models import Document
from documents.models import DocumentType
from documents.models import Note
from documents.models import SavedView
from documents.models import SavedViewFilterRule
from documents.models import StoragePath
@@ -206,7 +206,7 @@ class Command(BaseCommand):
self.files_in_export_dir.add(x.resolve())
# 2. Create manifest, containing all correspondents, types, tags, storage paths
# comments, documents and ui_settings
# note, documents and ui_settings
with transaction.atomic():
manifest = json.loads(
serializers.serialize("json", Correspondent.objects.all()),
@@ -222,11 +222,11 @@ class Command(BaseCommand):
serializers.serialize("json", StoragePath.objects.all()),
)
comments = json.loads(
serializers.serialize("json", Comment.objects.all()),
notes = json.loads(
serializers.serialize("json", Note.objects.all()),
)
if not self.split_manifest:
manifest += comments
manifest += notes
documents = Document.objects.order_by("id")
document_map = {d.pk: d for d in documents}
@@ -359,7 +359,7 @@ class Command(BaseCommand):
content += list(
filter(
lambda d: d["fields"]["document"] == document_dict["pk"],
comments,
notes,
),
)
manifest_name.write_text(json.dumps(content, indent=2))

54
src/documents/migrations/1034_alter_savedviewfilterrule_rule_type.py Normal file
View File

@@ -0,0 +1,54 @@
# Generated by Django 4.1.5 on 2023-03-15 07:10
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("documents", "1033_alter_documenttype_options_alter_tag_options_and_more"),
]
operations = [
migrations.AlterField(
model_name="savedviewfilterrule",
name="rule_type",
field=models.PositiveIntegerField(
choices=[
(0, "title contains"),
(1, "content contains"),
(2, "ASN is"),
(3, "correspondent is"),
(4, "document type is"),
(5, "is in inbox"),
(6, "has tag"),
(7, "has any tag"),
(8, "created before"),
(9, "created after"),
(10, "created year is"),
(11, "created month is"),
(12, "created day is"),
(13, "added before"),
(14, "added after"),
(15, "modified before"),
(16, "modified after"),
(17, "does not have tag"),
(18, "does not have ASN"),
(19, "title or content contains"),
(20, "fulltext query"),
(21, "more like this"),
(22, "has tags in"),
(23, "ASN greater than"),
(24, "ASN less than"),
(25, "storage path is"),
(26, "has correspondent in"),
(27, "does not have correspondent in"),
(28, "has document type in"),
(29, "does not have document type in"),
(30, "has storage path in"),
(31, "does not have storage path in"),
],
verbose_name="rule type",
),
),
]

61
src/documents/migrations/1035_rename_comment_note.py Normal file
View File

@@ -0,0 +1,61 @@
# Generated by Django 4.1.5 on 2023-03-17 22:15
from django.conf import settings
from django.db import migrations
from django.db import models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
("documents", "1034_alter_savedviewfilterrule_rule_type"),
]
operations = [
migrations.RenameModel(
old_name="Comment",
new_name="Note",
),
migrations.RenameField(model_name="note", old_name="comment", new_name="note"),
migrations.AlterModelOptions(
name="note",
options={
"ordering": ("created",),
"verbose_name": "note",
"verbose_name_plural": "notes",
},
),
migrations.AlterField(
model_name="note",
name="document",
field=models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.CASCADE,
related_name="notes",
to="documents.document",
verbose_name="document",
),
),
migrations.AlterField(
model_name="note",
name="note",
field=models.TextField(
blank=True, help_text="Note for the document", verbose_name="content"
),
),
migrations.AlterField(
model_name="note",
name="user",
field=models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.SET_NULL,
related_name="notes",
to=settings.AUTH_USER_MODEL,
verbose_name="user",
),
),
]

22
src/documents/models.py
View File

@@ -447,6 +447,12 @@ class SavedViewFilterRule(models.Model):
(23, _("ASN greater than")),
(24, _("ASN less than")),
(25, _("storage path is")),
(26, _("has correspondent in")),
(27, _("does not have correspondent in")),
(28, _("has document type in")),
(29, _("does not have document type in")),
(30, _("has storage path in")),
(31, _("does not have storage path in")),
]
saved_view = models.ForeignKey(
@@ -629,11 +635,11 @@ class PaperlessTask(models.Model):
)
class Comment(models.Model):
comment = models.TextField(
class Note(models.Model):
note = models.TextField(
_("content"),
blank=True,
help_text=_("Comment for the document"),
help_text=_("Note for the document"),
)
created = models.DateTimeField(
@@ -646,7 +652,7 @@ class Comment(models.Model):
Document,
blank=True,
null=True,
related_name="documents",
related_name="notes",
on_delete=models.CASCADE,
verbose_name=_("document"),
)
@@ -655,15 +661,15 @@ class Comment(models.Model):
User,
blank=True,
null=True,
related_name="users",
related_name="notes",
on_delete=models.SET_NULL,
verbose_name=_("user"),
)
class Meta:
ordering = ("created",)
verbose_name = _("comment")
verbose_name_plural = _("comments")
verbose_name = _("note")
verbose_name_plural = _("notes")
def __str__(self):
return self.content
return self.note

4
src/documents/serialisers.py
View File

@@ -78,10 +78,11 @@ class MatchingModelSerializer(serializers.ModelSerializer):
if hasattr(self, "user")
else None
)
pk = self.instance.pk if hasattr(self.instance, "pk") else None
if ("name" in data or "owner" in data) and self.Meta.model.objects.filter(
name=name,
owner=owner,
).exists():
).exclude(pk=pk).exists():
raise serializers.ValidationError(
{"error": "Object violates owner / name unique constraint"},
)
@@ -442,6 +443,7 @@ class DocumentSerializer(OwnedObjectSerializer, DynamicFieldsModelSerializer):
"owner",
"permissions",
"set_permissions",
"notes",
)

102
src/documents/tests/test_api.py
View File

@@ -38,7 +38,7 @@ from documents.models import PaperlessTask
from documents.models import SavedView
from documents.models import StoragePath
from documents.models import Tag
from documents.models import Comment
from documents.models import Note
from documents.tests.utils import DirectoriesMixin
from paperless import version
from rest_framework.test import APITestCase
@@ -1039,9 +1039,24 @@ class TestDocumentApi(DirectoriesMixin, APITestCase):
def test_statistics(self):
doc1 = Document.objects.create(title="none1", checksum="A")
doc2 = Document.objects.create(title="none2", checksum="B")
doc3 = Document.objects.create(title="none3", checksum="C")
doc1 = Document.objects.create(
title="none1",
checksum="A",
mime_type="application/pdf",
content="abc",
)
doc2 = Document.objects.create(
title="none2",
checksum="B",
mime_type="application/pdf",
content="123",
)
doc3 = Document.objects.create(
title="none3",
checksum="C",
mime_type="text/plain",
content="hello",
)
tag_inbox = Tag.objects.create(name="t1", is_inbox_tag=True)
@@ -1051,6 +1066,16 @@ class TestDocumentApi(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data["documents_total"], 3)
self.assertEqual(response.data["documents_inbox"], 1)
self.assertEqual(response.data["inbox_tag"], tag_inbox.pk)
self.assertEqual(
response.data["document_file_type_counts"][0]["mime_type_count"],
2,
)
self.assertEqual(
response.data["document_file_type_counts"][1]["mime_type_count"],
1,
)
self.assertEqual(response.data["character_count"], 11)
def test_statistics_no_inbox_tag(self):
Document.objects.create(title="none1", checksum="A")
@@ -1058,6 +1083,7 @@ class TestDocumentApi(DirectoriesMixin, APITestCase):
response = self.client.get("/api/statistics/")
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data["documents_inbox"], None)
self.assertEqual(response.data["inbox_tag"], None)
@mock.patch("documents.views.consume_file.delay")
def test_upload(self, m):
@@ -1717,28 +1743,28 @@ class TestDocumentApi(DirectoriesMixin, APITestCase):
1,
)
def test_get_existing_comments(self):
def test_get_existing_notes(self):
"""
GIVEN:
- A document with a single comment
- A document with a single note
WHEN:
- API reuqest for document comments is made
- API reuqest for document notes is made
THEN:
- The associated comment is returned
- The associated note is returned
"""
doc = Document.objects.create(
title="test",
mime_type="application/pdf",
content="this is a document which will have comments!",
content="this is a document which will have notes!",
)
comment = Comment.objects.create(
comment="This is a comment.",
note = Note.objects.create(
note="This is a note.",
document=doc,
user=self.user,
)
response = self.client.get(
f"/api/documents/{doc.pk}/comments/",
f"/api/documents/{doc.pk}/notes/",
format="json",
)
@@ -1754,39 +1780,39 @@ class TestDocumentApi(DirectoriesMixin, APITestCase):
self.assertDictEqual(
resp_data,
{
"id": comment.id,
"comment": comment.comment,
"id": note.id,
"note": note.note,
"user": {
"id": comment.user.id,
"username": comment.user.username,
"first_name": comment.user.first_name,
"last_name": comment.user.last_name,
"id": note.user.id,
"username": note.user.username,
"first_name": note.user.first_name,
"last_name": note.user.last_name,
},
},
)
def test_create_comment(self):
def test_create_note(self):
"""
GIVEN:
- Existing document
WHEN:
- API request is made to add a comment
- API request is made to add a note
THEN:
- Comment is created and associated with document
- note is created and associated with document
"""
doc = Document.objects.create(
title="test",
mime_type="application/pdf",
content="this is a document which will have comments added",
content="this is a document which will have notes added",
)
resp = self.client.post(
f"/api/documents/{doc.pk}/comments/",
data={"comment": "this is a posted comment"},
f"/api/documents/{doc.pk}/notes/",
data={"note": "this is a posted note"},
)
self.assertEqual(resp.status_code, status.HTTP_200_OK)
response = self.client.get(
f"/api/documents/{doc.pk}/comments/",
f"/api/documents/{doc.pk}/notes/",
format="json",
)
@@ -1798,48 +1824,48 @@ class TestDocumentApi(DirectoriesMixin, APITestCase):
resp_data = resp_data[0]
self.assertEqual(resp_data["comment"], "this is a posted comment")
self.assertEqual(resp_data["note"], "this is a posted note")
def test_delete_comment(self):
def test_delete_note(self):
"""
GIVEN:
- Existing document
WHEN:
- API request is made to add a comment
- API request is made to add a note
THEN:
- Comment is created and associated with document
- note is created and associated with document
"""
doc = Document.objects.create(
title="test",
mime_type="application/pdf",
content="this is a document which will have comments!",
content="this is a document which will have notes!",
)
comment = Comment.objects.create(
comment="This is a comment.",
note = Note.objects.create(
note="This is a note.",
document=doc,
user=self.user,
)
response = self.client.delete(
f"/api/documents/{doc.pk}/comments/?id={comment.pk}",
f"/api/documents/{doc.pk}/notes/?id={note.pk}",
format="json",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(len(Comment.objects.all()), 0)
self.assertEqual(len(Note.objects.all()), 0)
def test_get_comments_no_doc(self):
def test_get_notes_no_doc(self):
"""
GIVEN:
- A request to get comments from a non-existent document
- A request to get notes from a non-existent document
WHEN:
- API request for document comments is made
- API request for document notes is made
THEN:
- HTTP status.HTTP_404_NOT_FOUND is returned
"""
response = self.client.get(
"/api/documents/500/comments/",
"/api/documents/500/notes/",
format="json",
)
self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)

10
src/documents/tests/test_management_exporter.py
View File

@@ -13,10 +13,10 @@ from django.test import override_settings
from django.test import TestCase
from django.utils import timezone
from documents.management.commands import document_exporter
from documents.models import Comment
from documents.models import Correspondent
from documents.models import Document
from documents.models import DocumentType
from documents.models import Note
from documents.models import StoragePath
from documents.models import Tag
from documents.models import User
@@ -66,8 +66,8 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
storage_type=Document.STORAGE_TYPE_GPG,
)
self.comment = Comment.objects.create(
comment="This is a comment. amaze.",
self.note = Note.objects.create(
note="This is a note. amaze.",
document=self.d1,
user=self.user,
)
@@ -199,8 +199,8 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
checksum = hashlib.md5(f.read()).hexdigest()
self.assertEqual(checksum, element["fields"]["archive_checksum"])
elif element["model"] == "documents.comment":
self.assertEqual(element["fields"]["comment"], self.comment.comment)
elif element["model"] == "documents.note":
self.assertEqual(element["fields"]["note"], self.note.note)
self.assertEqual(element["fields"]["document"], self.d1.id)
self.assertEqual(element["fields"]["user"], self.user.id)

77
src/documents/views.py
View File

@@ -20,7 +20,9 @@ from django.db.models import Case
from django.db.models import Count
from django.db.models import IntegerField
from django.db.models import Max
from django.db.models import Sum
from django.db.models import When
from django.db.models.functions import Length
from django.db.models.functions import Lower
from django.http import Http404
from django.http import HttpResponse
@@ -72,10 +74,10 @@ from .matching import match_correspondents
from .matching import match_document_types
from .matching import match_storage_paths
from .matching import match_tags
from .models import Comment
from .models import Correspondent
from .models import Document
from .models import DocumentType
from .models import Note
from .models import PaperlessTask
from .models import SavedView
from .models import StoragePath
@@ -186,6 +188,7 @@ class TagViewSet(ModelViewSet, PassUserMixin):
)
def get_serializer_class(self, *args, **kwargs):
print(self.request.version)
if int(self.request.version) == 1:
return TagSerializerVersion1
else:
@@ -230,7 +233,7 @@ class DocumentViewSet(
GenericViewSet,
):
model = Document
queryset = Document.objects.all()
queryset = Document.objects.annotate(num_notes=Count("notes"))
serializer_class = DocumentSerializer
pagination_class = StandardPagination
permission_classes = (IsAuthenticated, PaperlessObjectPermissions)
@@ -251,10 +254,11 @@ class DocumentViewSet(
"modified",
"added",
"archive_serial_number",
"num_notes",
)
def get_queryset(self):
return Document.objects.distinct()
return Document.objects.distinct().annotate(num_notes=Count("notes"))
def get_serializer(self, *args, **kwargs):
super().get_serializer(*args, **kwargs)
@@ -441,11 +445,11 @@ class DocumentViewSet(
except (FileNotFoundError, Document.DoesNotExist):
raise Http404()
def getComments(self, doc):
def getNotes(self, doc):
return [
{
"id": c.id,
"comment": c.comment,
"note": c.note,
"created": c.created,
"user": {
"id": c.user.id,
@@ -454,11 +458,11 @@ class DocumentViewSet(
"last_name": c.user.last_name,
},
}
for c in Comment.objects.filter(document=doc).order_by("-created")
for c in Note.objects.filter(document=doc).order_by("-created")
]
@action(methods=["get", "post", "delete"], detail=True)
def comments(self, request, pk=None):
def notes(self, request, pk=None):
try:
doc = Document.objects.get(pk=pk)
except Document.DoesNotExist:
@@ -468,17 +472,17 @@ class DocumentViewSet(
if request.method == "GET":
try:
return Response(self.getComments(doc))
return Response(self.getNotes(doc))
except Exception as e:
logger.warning(f"An error occurred retrieving comments: {str(e)}")
logger.warning(f"An error occurred retrieving notes: {str(e)}")
return Response(
{"error": "Error retreiving comments, check logs for more detail."},
{"error": "Error retreiving notes, check logs for more detail."},
)
elif request.method == "POST":
try:
c = Comment.objects.create(
c = Note.objects.create(
document=doc,
comment=request.data["comment"],
note=request.data["note"],
user=currentUser,
)
c.save()
@@ -487,23 +491,23 @@ class DocumentViewSet(
index.add_or_update_document(self.get_object())
return Response(self.getComments(doc))
return Response(self.getNotes(doc))
except Exception as e:
logger.warning(f"An error occurred saving comment: {str(e)}")
logger.warning(f"An error occurred saving note: {str(e)}")
return Response(
{
"error": "Error saving comment, check logs for more detail.",
"error": "Error saving note, check logs for more detail.",
},
)
elif request.method == "DELETE":
comment = Comment.objects.get(id=int(request.GET.get("id")))
comment.delete()
note = Note.objects.get(id=int(request.GET.get("id")))
note.delete()
from documents import index
index.add_or_update_document(self.get_object())
return Response(self.getComments(doc))
return Response(self.getNotes(doc))
return Response(
{
@@ -515,14 +519,14 @@ class DocumentViewSet(
class SearchResultSerializer(DocumentSerializer, PassUserMixin):
def to_representation(self, instance):
doc = Document.objects.get(id=instance["id"])
comments = ",".join(
[str(c.comment) for c in Comment.objects.filter(document=instance["id"])],
notes = ",".join(
[str(c.note) for c in Note.objects.filter(document=instance["id"])],
)
r = super().to_representation(doc)
r["__search_hit__"] = {
"score": instance.score,
"highlights": instance.highlights("content", text=doc.content),
"comment_highlights": instance.highlights("comments", text=comments)
"note_highlights": instance.highlights("notes", text=notes)
if doc
else None,
"rank": instance.rank,
@@ -794,17 +798,38 @@ class StatisticsView(APIView):
def get(self, request, format=None):
documents_total = Document.objects.all().count()
if Tag.objects.filter(is_inbox_tag=True).exists():
documents_inbox = (
Document.objects.filter(tags__is_inbox_tag=True).distinct().count()
inbox_tag = Tag.objects.filter(is_inbox_tag=True)
documents_inbox = (
Document.objects.filter(tags__is_inbox_tag=True).distinct().count()
if inbox_tag.exists()
else None
)
document_file_type_counts = (
Document.objects.values("mime_type")
.annotate(mime_type_count=Count("mime_type"))
.order_by("-mime_type_count")
if documents_total > 0
else 0
)
character_count = (
Document.objects.annotate(
characters=Length("content"),
)
else:
documents_inbox = None
.aggregate(Sum("characters"))
.get("characters__sum")
)
return Response(
{
"documents_total": documents_total,
"documents_inbox": documents_inbox,
"inbox_tag": inbox_tag.first().pk if inbox_tag.exists() else None,
"document_file_type_counts": document_file_type_counts,
"character_count": character_count,
},
)

18
src/paperless/settings.py
View File

@@ -358,7 +358,7 @@ TEMPLATES = [
CHANNEL_LAYERS = {
"default": {
"BACKEND": "channels_redis.core.RedisChannelLayer",
"BACKEND": "channels_redis.pubsub.RedisPubSubChannelLayer",
"CONFIG": {
"hosts": [_CHANNELS_REDIS_URL],
"capacity": 2000, # default 100
@@ -509,7 +509,12 @@ if os.getenv("PAPERLESS_DBHOST"):
else: # Default to PostgresDB
engine = "django.db.backends.postgresql_psycopg2"
options = {"sslmode": os.getenv("PAPERLESS_DBSSLMODE", "prefer")}
options = {
"sslmode": os.getenv("PAPERLESS_DBSSLMODE", "prefer"),
"sslrootcert": os.getenv("PAPERLESS_DBSSLROOTCERT", None),
"sslcert": os.getenv("PAPERLESS_DBSSLCERT", None),
"sslkey": os.getenv("PAPERLESS_DBSSLKEY", None),
}
DATABASES["default"]["ENGINE"] = engine
DATABASES["default"]["OPTIONS"].update(options)
@@ -606,11 +611,20 @@ LOGGING = {
"maxBytes": LOGROTATE_MAX_SIZE,
"backupCount": LOGROTATE_MAX_BACKUPS,
},
"file_celery": {
"class": "concurrent_log_handler.ConcurrentRotatingFileHandler",
"formatter": "verbose",
"filename": os.path.join(LOGGING_DIR, "celery.log"),
"maxBytes": LOGROTATE_MAX_SIZE,
"backupCount": LOGROTATE_MAX_BACKUPS,
},
},
"root": {"handlers": ["console"]},
"loggers": {
"paperless": {"handlers": ["file_paperless"], "level": "DEBUG"},
"paperless_mail": {"handlers": ["file_mail"], "level": "DEBUG"},
"celery": {"handlers": ["file_celery"], "level": "DEBUG"},
"kombu": {"handlers": ["file_celery"], "level": "DEBUG"},
},
}

6
src/paperless/urls.py
View File

@@ -29,6 +29,7 @@ from paperless.consumers import StatusConsumer
from paperless.views import FaviconView
from paperless.views import GroupViewSet
from paperless.views import UserViewSet
from paperless_mail.views import MailAccountTestView
from paperless_mail.views import MailAccountViewSet
from paperless_mail.views import MailRuleViewSet
from rest_framework.authtoken import views
@@ -102,6 +103,11 @@ urlpatterns = [
AcknowledgeTasksView.as_view(),
name="acknowledge_tasks",
),
re_path(
r"^mail_accounts/test/",
MailAccountTestView.as_view(),
name="mail_accounts_test",
),
path("token/", views.obtain_auth_token),
]
+ api_router.urls,

25
src/paperless_mail/mail.py
View File

@@ -202,20 +202,21 @@ def mailbox_login(mailbox: MailBox, account: MailAccount):
try:
mailbox.login(account.username, account.password)
if account.is_token:
mailbox.xoauth2(account.username, account.password)
else:
try:
_ = account.password.encode("ascii")
use_ascii_login = True
except UnicodeEncodeError:
use_ascii_login = False
except UnicodeEncodeError:
logger.debug("Falling back to AUTH=PLAIN")
if use_ascii_login:
mailbox.login(account.username, account.password)
else:
logger.debug("Falling back to AUTH=PLAIN")
mailbox.login_utf8(account.username, account.password)
try:
mailbox.login_utf8(account.username, account.password)
except Exception as e:
logger.error(
"Unable to authenticate with mail server using AUTH=PLAIN",
)
raise MailError(
f"Error while authenticating account {account}",
) from e
except Exception as e:
logger.error(
f"Error while authenticating account {account}: {e}",

19
src/paperless_mail/migrations/0020_mailaccount_is_token.py Normal file
View File

@@ -0,0 +1,19 @@
# Generated by Django 4.1.7 on 2023-03-22 17:51
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("paperless_mail", "0019_mailrule_filter_to"),
]
operations = [
migrations.AddField(
model_name="mailaccount",
name="is_token",
field=models.BooleanField(
default=False, verbose_name="Is token authentication"
),
),
]

2
src/paperless_mail/models.py
View File

@@ -38,6 +38,8 @@ class MailAccount(document_models.ModelWithOwner):
password = models.CharField(_("password"), max_length=256)
is_token = models.BooleanField(_("Is token authentication"), default=False)
character_set = models.CharField(
_("character set"),
max_length=256,

1
src/paperless_mail/serialisers.py
View File

@@ -34,6 +34,7 @@ class MailAccountSerializer(OwnedObjectSerializer):
"username",
"password",
"character_set",
"is_token",
]
def update(self, instance, validated_data):

99
src/paperless_mail/tests/test_api.py
View File

@@ -1,3 +1,6 @@
import json
from unittest import mock
from django.contrib.auth.models import User
from documents.models import Correspondent
from documents.models import DocumentType
@@ -5,6 +8,7 @@ from documents.models import Tag
from documents.tests.utils import DirectoriesMixin
from paperless_mail.models import MailAccount
from paperless_mail.models import MailRule
from paperless_mail.tests.test_mail import BogusMailBox
from rest_framework import status
from rest_framework.test import APITestCase
@@ -13,6 +17,13 @@ class TestAPIMailAccounts(DirectoriesMixin, APITestCase):
ENDPOINT = "/api/mail_accounts/"
def setUp(self):
self.bogus_mailbox = BogusMailBox()
patcher = mock.patch("paperless_mail.mail.MailBox")
m = patcher.start()
m.return_value = self.bogus_mailbox
self.addCleanup(patcher.stop)
super().setUp()
self.user = User.objects.create_superuser(username="temp_admin")
@@ -166,6 +177,94 @@ class TestAPIMailAccounts(DirectoriesMixin, APITestCase):
self.assertEqual(returned_account2.name, "Updated Name 2")
self.assertEqual(returned_account2.password, "123xyz")
def test_mail_account_test_fail(self):
"""
GIVEN:
- Errnoeous mail account details
WHEN:
- API call is made to test account
THEN:
- API returns 400 bad request
"""
response = self.client.post(
f"{self.ENDPOINT}test/",
json.dumps(
{
"imap_server": "server.example.com",
"imap_port": 443,
"imap_security": MailAccount.ImapSecurity.SSL,
"username": "admin",
"password": "notcorrect",
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
def test_mail_account_test_success(self):
"""
GIVEN:
- Working mail account details
WHEN:
- API call is made to test account
THEN:
- API returns success
"""
response = self.client.post(
f"{self.ENDPOINT}test/",
json.dumps(
{
"imap_server": "server.example.com",
"imap_port": 443,
"imap_security": MailAccount.ImapSecurity.SSL,
"username": "admin",
"password": "secret",
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data["success"], True)
def test_mail_account_test_existing(self):
"""
GIVEN:
- Testing server details for an existing account with obfuscated password (***)
WHEN:
- API call is made to test account
THEN:
- API returns success
"""
account = MailAccount.objects.create(
name="Email1",
username="admin",
password="secret",
imap_server="server.example.com",
imap_port=443,
imap_security=MailAccount.ImapSecurity.SSL,
character_set="UTF-8",
)
response = self.client.post(
f"{self.ENDPOINT}test/",
json.dumps(
{
"id": account.pk,
"imap_server": "server.example.com",
"imap_port": 443,
"imap_security": MailAccount.ImapSecurity.SSL,
"username": "admin",
"password": "******",
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data["success"], True)
class TestAPIMailRules(DirectoriesMixin, APITestCase):
ENDPOINT = "/api/mail_rules/"

53
src/paperless_mail/tests/test_mail.py
View File

@@ -83,6 +83,8 @@ class BogusMailBox(ContextManager):
ASCII_PASSWORD: str = "secret"
# Note the non-ascii characters here
UTF_PASSWORD: str = "w57äöüw4b6huwb6nhu"
# A dummy access token
ACCESS_TOKEN = "ea7e075cd3acf2c54c48e600398d5d5a"
def __init__(self):
self.messages: List[MailMessage] = []
@@ -112,6 +114,10 @@ class BogusMailBox(ContextManager):
if username != self.USERNAME or password != self.UTF_PASSWORD:
raise MailboxLoginError("BAD", "OK")
def xoauth2(self, username: str, access_token: str):
if username != self.USERNAME or access_token != self.ACCESS_TOKEN:
raise MailboxLoginError("BAD", "OK")
def fetch(self, criteria, mark_seen, charset=""):
msg = self.messages
@@ -737,6 +743,14 @@ class TestMail(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
self.assertEqual(len(self.bogus_mailbox.messages), 3)
def test_error_login(self):
"""
GIVEN:
- Account configured with incorrect password
WHEN:
- Account tried to login
THEN:
- MailError with correct message raised
"""
account = MailAccount.objects.create(
name="test",
imap_server="",
@@ -1007,6 +1021,8 @@ class TestMail(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
"""
GIVEN:
- Mail account with password containing non-ASCII characters
WHEN:
- Mail account is handled
THEN:
- Should still authenticate to the mail account
"""
@@ -1040,6 +1056,8 @@ class TestMail(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
GIVEN:
- Mail account with password containing non-ASCII characters
- Incorrect password value
WHEN:
- Mail account is handled
THEN:
- Should raise a MailError for the account
"""
@@ -1064,6 +1082,41 @@ class TestMail(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
account,
)
def test_auth_with_valid_token(self):
"""
GIVEN:
- Mail account configured with access token
WHEN:
- Mail account is handled
THEN:
- Should still authenticate to the mail account
"""
account = MailAccount.objects.create(
name="test",
imap_server="",
username=BogusMailBox.USERNAME,
# Note the non-ascii characters here
password=BogusMailBox.ACCESS_TOKEN,
is_token=True,
)
_ = MailRule.objects.create(
name="testrule",
account=account,
action=MailRule.MailAction.MARK_READ,
)
self.assertEqual(len(self.bogus_mailbox.messages), 3)
self.assertEqual(self._queue_consumption_tasks_mock.call_count, 0)
self.assertEqual(len(self.bogus_mailbox.fetch("UNSEEN", False)), 2)
self.mail_account_handler.handle_mail_account(account)
self.apply_mail_actions()
self.assertEqual(self._queue_consumption_tasks_mock.call_count, 2)
self.assertEqual(len(self.bogus_mailbox.fetch("UNSEEN", False)), 0)
self.assertEqual(len(self.bogus_mailbox.messages), 3)
def assert_queue_consumption_tasks_call_args(self, expected_call_args: List):
"""
Verifies that queue_consumption_tasks has been called with the expected arguments.

47
src/paperless_mail/views.py
View File

@@ -1,10 +1,19 @@
import datetime
import logging
from django.http import HttpResponseBadRequest
from documents.views import PassUserMixin
from paperless.views import StandardPagination
from paperless_mail.mail import get_mailbox
from paperless_mail.mail import mailbox_login
from paperless_mail.mail import MailError
from paperless_mail.models import MailAccount
from paperless_mail.models import MailRule
from paperless_mail.serialisers import MailAccountSerializer
from paperless_mail.serialisers import MailRuleSerializer
from rest_framework.generics import GenericAPIView
from rest_framework.permissions import IsAuthenticated
from rest_framework.response import Response
from rest_framework.viewsets import ModelViewSet
@@ -24,3 +33,41 @@ class MailRuleViewSet(ModelViewSet, PassUserMixin):
serializer_class = MailRuleSerializer
pagination_class = StandardPagination
permission_classes = (IsAuthenticated,)
class MailAccountTestView(GenericAPIView):
permission_classes = (IsAuthenticated,)
serializer_class = MailAccountSerializer
def post(self, request, *args, **kwargs):
logger = logging.getLogger("paperless_mail")
request.data["name"] = datetime.datetime.now().isoformat()
serializer = self.get_serializer(data=request.data)
serializer.is_valid(raise_exception=True)
# account exists, use the password from there instead of ***
if (
len(serializer.validated_data.get("password").replace("*", "")) == 0
and request.data["id"] is not None
):
serializer.validated_data["password"] = MailAccount.objects.get(
pk=request.data["id"],
).password
account = MailAccount(**serializer.validated_data)
with get_mailbox(
account.imap_server,
account.imap_port,
account.imap_security,
) as M:
try:
mailbox_login(M, account)
return Response({"success": True})
except MailError as e:
logger.error(
f"Mail account {account} test failed: {e}",
exc_info=False,
)
return HttpResponseBadRequest("Unable to connect to server")