paradizelost/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2025-07-28 18:24:38 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev' into feature-websockets-status

Browse Source
This commit is contained in:
jonaswinkler
2021-01-23 22:22:17 +01:00
parent d153672f0d 227f7b6946
commit 05d69c0882
163 changed files with 5843 additions and 2520 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/documents/consumer.py
View File

@@ -91,6 +91,11 @@ class Consumer(LoggingMixin):
if not settings.PRE_CONSUME_SCRIPT:
return
if not os.path.isfile(settings.PRE_CONSUME_SCRIPT):
raise ConsumerError(
f"Configured pre-consume script "
f"{settings.PRE_CONSUME_SCRIPT} does not exist.")
try:
Popen((settings.PRE_CONSUME_SCRIPT, self.path)).wait()
except Exception as e:
@@ -102,6 +107,11 @@ class Consumer(LoggingMixin):
if not settings.POST_CONSUME_SCRIPT:
return
if not os.path.isfile(settings.POST_CONSUME_SCRIPT):
raise ConsumerError(
f"Configured post-consume script "
f"{settings.POST_CONSUME_SCRIPT} does not exist.")
try:
Popen((
settings.POST_CONSUME_SCRIPT,

4
src/documents/file_handling.py
View File

@@ -91,7 +91,7 @@ def generate_unique_filename(doc, root):
return new_filename
def generate_filename(doc, counter=0):
def generate_filename(doc, counter=0, append_gpg=True):
path = ""
try:
@@ -151,7 +151,7 @@ def generate_filename(doc, counter=0):
filename = f"{doc.pk:07}{counter_str}{doc.file_type}"
# Append .gpg for encrypted files
if doc.storage_type == doc.STORAGE_TYPE_GPG:
if append_gpg and doc.storage_type == doc.STORAGE_TYPE_GPG:
filename += ".gpg"
return filename

7
src/documents/management/commands/document_archiver.py
View File

@@ -11,6 +11,7 @@ from django import db
from django.conf import settings
from django.core.management.base import BaseCommand
from django.db import transaction
from filelock import FileLock
from whoosh.writing import AsyncWriter
from documents.models import Document
@@ -47,8 +48,10 @@ def handle_document(document_id):
archive_checksum=checksum,
content=parser.get_text()
)
create_source_path_directory(document.archive_path)
shutil.move(parser.get_archive_path(), document.archive_path)
with FileLock(settings.MEDIA_LOCK):
create_source_path_directory(document.archive_path)
shutil.move(parser.get_archive_path(),
document.archive_path)
with AsyncWriter(index.open_index()) as writer:
index.update_document(writer, document)

3
src/documents/management/commands/document_consumer.py
View File

@@ -5,7 +5,6 @@ from time import sleep
from django.conf import settings
from django.core.management.base import BaseCommand, CommandError
from django.utils.text import slugify
from django_q.tasks import async_task
from watchdog.events import FileSystemEventHandler
from watchdog.observers.polling import PollingObserver
@@ -46,7 +45,7 @@ def _consume(filepath):
return
if not is_file_ext_supported(os.path.splitext(filepath)[1]):
logger.debug(
logger.warning(
f"Not consuming file {filepath}: Unknown file extension.")
return

173
src/documents/management/commands/document_exporter.py
View File

@@ -1,15 +1,21 @@
import hashlib
import json
import os
import shutil
import time
import tqdm
from django.conf import settings
from django.core import serializers
from django.core.management.base import BaseCommand, CommandError
from django.db import transaction
from filelock import FileLock
from documents.models import Document, Correspondent, Tag, DocumentType
from documents.settings import EXPORTER_FILE_NAME, EXPORTER_THUMBNAIL_NAME, \
EXPORTER_ARCHIVE_NAME
from paperless.db import GnuPG
from ...file_handling import generate_filename, delete_empty_directories
from ...mixins import Renderable
@@ -24,13 +30,47 @@ class Command(Renderable, BaseCommand):
def add_arguments(self, parser):
parser.add_argument("target")
parser.add_argument(
"-c", "--compare-checksums",
default=False,
action="store_true",
help="Compare file checksums when determining whether to export "
"a file or not. If not specified, file size and time "
"modified is used instead."
)
parser.add_argument(
"-f", "--use-filename-format",
default=False,
action="store_true",
help="Use PAPERLESS_FILENAME_FORMAT for storing files in the "
"export directory, if configured."
)
parser.add_argument(
"-d", "--delete",
default=False,
action="store_true",
help="After exporting, delete files in the export directory that "
"do not belong to the current export, such as files from "
"deleted documents."
)
def __init__(self, *args, **kwargs):
BaseCommand.__init__(self, *args, **kwargs)
self.target = None
self.files_in_export_dir = []
self.exported_files = []
self.compare_checksums = False
self.use_filename_format = False
self.delete = False
def handle(self, *args, **options):
self.target = options["target"]
self.compare_checksums = options['compare_checksums']
self.use_filename_format = options['use_filename_format']
self.delete = options['delete']
if not os.path.exists(self.target):
raise CommandError("That path doesn't exist")
@@ -38,83 +78,148 @@ class Command(Renderable, BaseCommand):
if not os.access(self.target, os.W_OK):
raise CommandError("That path doesn't appear to be writable")
if os.listdir(self.target):
raise CommandError("That directory is not empty.")
self.dump()
with FileLock(settings.MEDIA_LOCK):
self.dump()
def dump(self):
# 1. Take a snapshot of what files exist in the current export folder
for root, dirs, files in os.walk(self.target):
self.files_in_export_dir.extend(
map(lambda f: os.path.abspath(os.path.join(root, f)), files)
)
documents = Document.objects.all()
document_map = {d.pk: d for d in documents}
manifest = json.loads(serializers.serialize("json", documents))
# 2. Create manifest, containing all correspondents, types, tags and
# documents
with transaction.atomic():
manifest = json.loads(
serializers.serialize("json", Correspondent.objects.all()))
for index, document_dict in enumerate(manifest):
manifest += json.loads(serializers.serialize(
"json", Tag.objects.all()))
# Force output to unencrypted as that will be the current state.
# The importer will make the decision to encrypt or not.
manifest[index]["fields"]["storage_type"] = Document.STORAGE_TYPE_UNENCRYPTED # NOQA: E501
manifest += json.loads(serializers.serialize(
"json", DocumentType.objects.all()))
documents = Document.objects.order_by("id")
document_map = {d.pk: d for d in documents}
document_manifest = json.loads(
serializers.serialize("json", documents))
manifest += document_manifest
# 3. Export files from each document
for index, document_dict in tqdm.tqdm(enumerate(document_manifest),
total=len(document_manifest)):
# 3.1. store files unencrypted
document_dict["fields"]["storage_type"] = Document.STORAGE_TYPE_UNENCRYPTED # NOQA: E501
document = document_map[document_dict["pk"]]
print(f"Exporting: {document}")
# 3.2. generate a unique filename
filename_counter = 0
while True:
original_name = document.get_public_filename(
counter=filename_counter)
original_target = os.path.join(self.target, original_name)
if self.use_filename_format:
base_name = generate_filename(
document, counter=filename_counter,
append_gpg=False)
else:
base_name = document.get_public_filename(
counter=filename_counter)
if not os.path.exists(original_target):
if base_name not in self.exported_files:
self.exported_files.append(base_name)
break
else:
filename_counter += 1
thumbnail_name = original_name + "-thumbnail.png"
thumbnail_target = os.path.join(self.target, thumbnail_name)
# 3.3. write filenames into manifest
original_name = base_name
original_target = os.path.join(self.target, original_name)
document_dict[EXPORTER_FILE_NAME] = original_name
thumbnail_name = base_name + "-thumbnail.png"
thumbnail_target = os.path.join(self.target, thumbnail_name)
document_dict[EXPORTER_THUMBNAIL_NAME] = thumbnail_name
if os.path.exists(document.archive_path):
archive_name = document.get_public_filename(
archive=True, counter=filename_counter, suffix="_archive")
archive_name = base_name + "-archive.pdf"
archive_target = os.path.join(self.target, archive_name)
document_dict[EXPORTER_ARCHIVE_NAME] = archive_name
else:
archive_target = None
# 3.4. write files to target folder
t = int(time.mktime(document.created.timetuple()))
if document.storage_type == Document.STORAGE_TYPE_GPG:
os.makedirs(os.path.dirname(original_target), exist_ok=True)
with open(original_target, "wb") as f:
f.write(GnuPG.decrypted(document.source_file))
os.utime(original_target, times=(t, t))
os.makedirs(os.path.dirname(thumbnail_target), exist_ok=True)
with open(thumbnail_target, "wb") as f:
f.write(GnuPG.decrypted(document.thumbnail_file))
os.utime(thumbnail_target, times=(t, t))
if archive_target:
os.makedirs(os.path.dirname(archive_target), exist_ok=True)
with open(archive_target, "wb") as f:
f.write(GnuPG.decrypted(document.archive_path))
os.utime(archive_target, times=(t, t))
else:
self.check_and_copy(document.source_path,
document.checksum,
original_target)
shutil.copy(document.source_path, original_target)
shutil.copy(document.thumbnail_path, thumbnail_target)
self.check_and_copy(document.thumbnail_path,
None,
thumbnail_target)
if archive_target:
shutil.copy(document.archive_path, archive_target)
self.check_and_copy(document.archive_path,
document.archive_checksum,
archive_target)
manifest += json.loads(
serializers.serialize("json", Correspondent.objects.all()))
# 4. write manifest to target forlder
manifest_path = os.path.abspath(
os.path.join(self.target, "manifest.json"))
manifest += json.loads(serializers.serialize(
"json", Tag.objects.all()))
manifest += json.loads(serializers.serialize(
"json", DocumentType.objects.all()))
with open(os.path.join(self.target, "manifest.json"), "w") as f:
with open(manifest_path, "w") as f:
json.dump(manifest, f, indent=2)
if self.delete:
# 5. Remove files which we did not explicitly export in this run
if manifest_path in self.files_in_export_dir:
self.files_in_export_dir.remove(manifest_path)
for f in self.files_in_export_dir:
os.remove(f)
delete_empty_directories(os.path.abspath(os.path.dirname(f)),
os.path.abspath(self.target))
def check_and_copy(self, source, source_checksum, target):
if os.path.abspath(target) in self.files_in_export_dir:
self.files_in_export_dir.remove(os.path.abspath(target))
perform_copy = False
if os.path.exists(target):
source_stat = os.stat(source)
target_stat = os.stat(target)
if self.compare_checksums and source_checksum:
with open(target, "rb") as f:
target_checksum = hashlib.md5(f.read()).hexdigest()
perform_copy = target_checksum != source_checksum
elif source_stat.st_mtime != target_stat.st_mtime:
perform_copy = True
elif source_stat.st_size != target_stat.st_size:
perform_copy = True
else:
# Copy if it does not exist
perform_copy = True
if perform_copy:
os.makedirs(os.path.dirname(target), exist_ok=True)
shutil.copy2(source, target)

6
src/documents/management/commands/document_importer.py
View File

@@ -148,10 +148,10 @@ class Command(Renderable, BaseCommand):
create_source_path_directory(document.source_path)
shutil.copy(document_path, document.source_path)
shutil.copy(thumbnail_path, document.thumbnail_path)
shutil.copy2(document_path, document.source_path)
shutil.copy2(thumbnail_path, document.thumbnail_path)
if archive_path:
create_source_path_directory(document.archive_path)
shutil.copy(archive_path, document.archive_path)
shutil.copy2(archive_path, document.archive_path)
document.save()

10
src/documents/management/commands/document_thumbnails.py
View File

@@ -13,8 +13,14 @@ from ...parsers import get_parser_class_for_mime_type
def _process_document(doc_in):
document = Document.objects.get(id=doc_in)
parser = get_parser_class_for_mime_type(document.mime_type)(
logging_group=None)
parser_class = get_parser_class_for_mime_type(document.mime_type)
if parser_class:
parser = parser_class(logging_group=None)
else:
print(f"{document} No parser for mime type {document.mime_type}")
return
try:
thumb = parser.get_optimised_thumbnail(
document.source_path, document.mime_type)

83
src/documents/matching.py
View File

@@ -1,3 +1,4 @@
import logging
import re
from fuzzywuzzy import fuzz
@@ -5,49 +6,59 @@ from fuzzywuzzy import fuzz
from documents.models import MatchingModel, Correspondent, DocumentType, Tag
def match_correspondents(document_content, classifier):
logger = logging.getLogger(__name__)
def log_reason(matching_model, document, reason):
class_name = type(matching_model).__name__
logger.debug(
f"Assigning {class_name} {matching_model.name} to document "
f"{document} because {reason}")
def match_correspondents(document, classifier):
if classifier:
pred_id = classifier.predict_correspondent(document_content)
pred_id = classifier.predict_correspondent(document.content)
else:
pred_id = None
correspondents = Correspondent.objects.all()
return list(filter(
lambda o: matches(o, document_content) or o.pk == pred_id,
lambda o: matches(o, document) or o.pk == pred_id,
correspondents))
def match_document_types(document_content, classifier):
def match_document_types(document, classifier):
if classifier:
pred_id = classifier.predict_document_type(document_content)
pred_id = classifier.predict_document_type(document.content)
else:
pred_id = None
document_types = DocumentType.objects.all()
return list(filter(
lambda o: matches(o, document_content) or o.pk == pred_id,
lambda o: matches(o, document) or o.pk == pred_id,
document_types))
def match_tags(document_content, classifier):
def match_tags(document, classifier):
if classifier:
predicted_tag_ids = classifier.predict_tags(document_content)
predicted_tag_ids = classifier.predict_tags(document.content)
else:
predicted_tag_ids = []
tags = Tag.objects.all()
return list(filter(
lambda o: matches(o, document_content) or o.pk in predicted_tag_ids,
lambda o: matches(o, document) or o.pk in predicted_tag_ids,
tags))
def matches(matching_model, document_content):
def matches(matching_model, document):
search_kwargs = {}
document_content = document_content.lower()
document_content = document.content.lower()
# Check that match is not empty
if matching_model.match.strip() == "":
@@ -62,26 +73,54 @@ def matches(matching_model, document_content):
rf"\b{word}\b", document_content, **search_kwargs)
if not search_result:
return False
log_reason(
matching_model, document,
f"it contains all of these words: {matching_model.match}"
)
return True
elif matching_model.matching_algorithm == MatchingModel.MATCH_ANY:
for word in _split_match(matching_model):
if re.search(rf"\b{word}\b", document_content, **search_kwargs):
log_reason(
matching_model, document,
f"it contains this word: {word}"
)
return True
return False
elif matching_model.matching_algorithm == MatchingModel.MATCH_LITERAL:
return bool(re.search(
result = bool(re.search(
rf"\b{matching_model.match}\b",
document_content,
**search_kwargs
))
if result:
log_reason(
matching_model, document,
f"it contains this string: \"{matching_model.match}\""
)
return result
elif matching_model.matching_algorithm == MatchingModel.MATCH_REGEX:
return bool(re.search(
re.compile(matching_model.match, **search_kwargs),
document_content
))
try:
match = re.search(
re.compile(matching_model.match, **search_kwargs),
document_content
)
except re.error:
logger.error(
f"Error while processing regular expression "
f"{matching_model.match}"
)
return False
if match:
log_reason(
matching_model, document,
f"the string {match.group()} matches the regular expression "
f"{matching_model.match}"
)
return bool(match)
elif matching_model.matching_algorithm == MatchingModel.MATCH_FUZZY:
match = re.sub(r'[^\w\s]', '', matching_model.match)
@@ -89,8 +128,16 @@ def matches(matching_model, document_content):
if matching_model.is_insensitive:
match = match.lower()
text = text.lower()
return fuzz.partial_ratio(match, text) >= 90
if fuzz.partial_ratio(match, text) >= 90:
# TODO: make this better
log_reason(
matching_model, document,
f"parts of the document content somehow match the string "
f"{matching_model.match}"
)
return True
else:
return False
elif matching_model.matching_algorithm == MatchingModel.MATCH_AUTO:
# this is done elsewhere.

12
src/documents/models.py
View File

@@ -12,6 +12,7 @@ from django.conf import settings
from django.contrib.auth.models import User
from django.db import models
from django.utils import timezone
from django.utils.timezone import is_aware
from django.utils.translation import gettext_lazy as _
@@ -62,12 +63,6 @@ class MatchingModel(models.Model):
def __str__(self):
return self.name
def save(self, *args, **kwargs):
self.match = self.match.lower()
models.Model.save(self, *args, **kwargs)
class Correspondent(MatchingModel):
@@ -233,7 +228,10 @@ class Document(models.Model):
verbose_name_plural = _("documents")
def __str__(self):
created = datetime.date.isoformat(self.created)
if is_aware(self.created):
created = timezone.localdate(self.created).isoformat()
else:
created = datetime.date.isoformat(self.created)
if self.correspondent and self.title:
return f"{created} {self.correspondent} {self.title}"
else:

15
src/documents/parsers.py
View File

@@ -210,6 +210,13 @@ def parse_date(filename, text):
}
)
def __filter(date):
if date and date.year > 1900 and \
date <= timezone.now() and \
date.date() not in settings.IGNORE_DATES:
return date
return None
date = None
# if filename date parsing is enabled, search there first:
@@ -223,7 +230,8 @@ def parse_date(filename, text):
# Skip all matches that do not parse to a proper date
continue
if date and date.year > 1900 and date <= timezone.now():
date = __filter(date)
if date is not None:
return date
# Iterate through all regex matches in text and try to parse the date
@@ -236,10 +244,9 @@ def parse_date(filename, text):
# Skip all matches that do not parse to a proper date
continue
if date and date.year > 1900 and date <= timezone.now():
date = __filter(date)
if date is not None:
break
else:
date = None
return date

7
src/documents/serialisers.py
View File

@@ -382,13 +382,6 @@ class PostDocumentSerializer(serializers.Serializer):
return document.name, document_data
def validate_title(self, title):
if title:
return title
else:
# do not return empty strings.
return None
def validate_correspondent(self, correspondent):
if correspondent:
return correspondent.id

6
src/documents/signals/handlers.py
View File

@@ -38,7 +38,7 @@ def set_correspondent(sender,
if document.correspondent and not replace:
return
potential_correspondents = matching.match_correspondents(document.content,
potential_correspondents = matching.match_correspondents(document,
classifier)
potential_count = len(potential_correspondents)
@@ -81,7 +81,7 @@ def set_document_type(sender,
if document.document_type and not replace:
return
potential_document_type = matching.match_document_types(document.content,
potential_document_type = matching.match_document_types(document,
classifier)
potential_count = len(potential_document_type)
@@ -130,7 +130,7 @@ def set_tags(sender,
current_tags = set(document.tags.all())
matched_tags = matching.match_tags(document.content, classifier)
matched_tags = matching.match_tags(document, classifier)
relevant_tags = set(matched_tags) - current_tags

3
src/documents/templates/index.html
View File

@@ -1,6 +1,7 @@
<!doctype html>
{% load static %}
{% load i18n %}
<html lang="en">
<head>
@@ -16,7 +17,7 @@
<link rel="stylesheet" href="{% static styles_css %}">
</head>
<body>
<app-root>Loading...</app-root>
<app-root>{% translate "Paperless-ng is loading..." %}</app-root>
<script src="{% static runtime_js %}" defer></script>
<script src="{% static polyfills_js %}" defer></script>
<script src="{% static main_js %}" defer></script>

9
src/documents/templates/registration/logged_out.html
View File

@@ -1,6 +1,7 @@
<!doctype html>
{% load static %}
{% load i18n %}
<html lang="en">
<head>
@@ -9,7 +10,7 @@
<meta name="description" content="">
<meta name="author" content="Mark Otto, Jacob Thornton, and Bootstrap contributors">
<meta name="generator" content="Jekyll v4.1.1">
<title>Paperless Sign In</title>
<title>{% translate "Paperless-ng signed out" %}</title>
<!-- Bootstrap core CSS -->
<link href="{% static 'bootstrap.min.css' %}" rel="stylesheet">
@@ -36,9 +37,9 @@
<body class="text-center">
<div class="form-signin">
<img class="mb-4" src="{% static 'frontend/assets/logo.svg' %}" alt="" width="300">
<p>You have been successfully logged out. Bye!</p>
<a href="/">Sign in again</a>
<img class="mb-4" src="{% static 'frontend/en-US/assets/logo.svg' %}" alt="" width="300">
<p>{% translate "You have been successfully logged out. Bye!" %}</p>
<a href="/">{% translate "Sign in again" %}</a>
</div>
</body>
</html>

21
src/documents/templates/registration/login.html
View File

@@ -1,6 +1,7 @@
<!doctype html>
{% load static %}
{% load i18n %}
<html lang="en">
<head>
@@ -9,7 +10,7 @@
<meta name="description" content="">
<meta name="author" content="Mark Otto, Jacob Thornton, and Bootstrap contributors">
<meta name="generator" content="Jekyll v4.1.1">
<title>Paperless Sign In</title>
<title>{% translate "Paperless-ng sign in" %}</title>
<!-- Bootstrap core CSS -->
<link href="{% static 'bootstrap.min.css' %}" rel="stylesheet">
@@ -37,18 +38,20 @@
<body class="text-center">
<form class="form-signin" method="post">
{% csrf_token %}
<img class="mb-4" src="{% static 'frontend/assets/logo.svg' %}" alt="" width="300">
<p>Please sign in.</p>
<img class="mb-4" src="{% static 'frontend/en-US/assets/logo.svg' %}" alt="" width="300">
<p>{% translate "Please sign in." %}</p>
{% if form.errors %}
<div class="alert alert-danger" role="alert">
Your username and password didn't match. Please try again.
{% translate "Your username and password didn't match. Please try again." %}
</div>
{% endif %}
<label for="inputUsername" class="sr-only">Username</label>
<input type="text" name="username" id="inputUsername" class="form-control" placeholder="Username" required autofocus>
<label for="inputPassword" class="sr-only">Password</label>
<input type="password" name="password" id="inputPassword" class="form-control" placeholder="Password" required>
<button class="btn btn-lg btn-primary btn-block" type="submit">Sign in</button>
{% translate "Username" as i18n_username %}
{% translate "Password" as i18n_password %}
<label for="inputUsername" class="sr-only">{{ i18n_username }}</label>
<input type="text" name="username" id="inputUsername" class="form-control" placeholder="{{ i18n_username }}" required autofocus>
<label for="inputPassword" class="sr-only">{{ i18n_password }}</label>
<input type="password" name="password" id="inputPassword" class="form-control" placeholder="{{ i18n_password }}" required>
<button class="btn btn-lg btn-primary btn-block" type="submit">{% translate "Sign in" %}</button>
</form>
</body>
</html>

BIN
src/documents/tests/samples/documents/originals/0000002.pdf Normal file
View File

Binary file not shown.

BIN
src/documents/tests/samples/documents/originals/0000002.pdf.gpg
View File

Binary file not shown.

BIN
src/documents/tests/samples/documents/originals/0000003.pdf Normal file
View File

Binary file not shown.

BIN
src/documents/tests/samples/documents/originals/0000004.pdf.gpg Normal file
View File

Binary file not shown.

BIN
src/documents/tests/samples/documents/thumbnails/0000002.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 7.7 KiB

BIN
src/documents/tests/samples/documents/thumbnails/0000003.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 7.7 KiB

0
src/documents/tests/samples/documents/thumbnails/0000002.png.gpg → src/documents/tests/samples/documents/thumbnails/0000004.png.gpg
View File

6
src/documents/tests/test_admin.py
View File

@@ -5,12 +5,14 @@ from django.test import TestCase
from django.utils import timezone
from documents.admin import DocumentAdmin
from documents.models import Document, Tag
from documents.models import Document
from documents.tests.utils import DirectoriesMixin
class TestDocumentAdmin(TestCase):
class TestDocumentAdmin(DirectoriesMixin, TestCase):
def setUp(self) -> None:
super(TestDocumentAdmin, self).setUp()
self.doc_admin = DocumentAdmin(model=Document, admin_site=AdminSite())
@mock.patch("documents.admin.index.add_or_update_document")

72
src/documents/tests/test_api.py
View File

@@ -114,8 +114,6 @@ class TestDocumentApi(DirectoriesMixin, APITestCase):
results = response.data['results']
self.assertEqual(len(results[0]), 0)
def test_document_actions(self):
_, filename = tempfile.mkstemp(dir=self.dirs.originals_dir)
@@ -230,6 +228,12 @@ class TestDocumentApi(DirectoriesMixin, APITestCase):
self.assertEqual(len(results), 2)
self.assertCountEqual([results[0]['id'], results[1]['id']], [doc1.id, doc3.id])
response = self.client.get("/api/documents/?tags__id__in={},{}".format(tag_2.id, tag_3.id))
self.assertEqual(response.status_code, 200)
results = response.data['results']
self.assertEqual(len(results), 2)
self.assertCountEqual([results[0]['id'], results[1]['id']], [doc2.id, doc3.id])
response = self.client.get("/api/documents/?tags__id__all={},{}".format(tag_2.id, tag_3.id))
self.assertEqual(response.status_code, 200)
results = response.data['results']
@@ -455,6 +459,23 @@ class TestDocumentApi(DirectoriesMixin, APITestCase):
self.assertIsNone(kwargs['override_document_type_id'])
self.assertIsNone(kwargs['override_tag_ids'])
@mock.patch("documents.views.async_task")
def test_upload_empty_metadata(self, m):
with open(os.path.join(os.path.dirname(__file__), "samples", "simple.pdf"), "rb") as f:
response = self.client.post("/api/documents/post_document/", {"document": f, "title": "", "correspondent": "", "document_type": ""})
self.assertEqual(response.status_code, 200)
m.assert_called_once()
args, kwargs = m.call_args
self.assertEqual(kwargs['override_filename'], "simple.pdf")
self.assertIsNone(kwargs['override_title'])
self.assertIsNone(kwargs['override_correspondent_id'])
self.assertIsNone(kwargs['override_document_type_id'])
self.assertIsNone(kwargs['override_tag_ids'])
@mock.patch("documents.views.async_task")
def test_upload_invalid_form(self, m):
@@ -908,6 +929,14 @@ class TestBulkEdit(DirectoriesMixin, APITestCase):
doc2 = Document.objects.get(id=self.doc2.id)
self.assertEqual(doc2.correspondent, self.c1)
def test_api_no_correspondent(self):
response = self.client.post("/api/documents/bulk_edit/", json.dumps({
"documents": [self.doc2.id],
"method": "set_correspondent",
"parameters": {}
}), content_type='application/json')
self.assertEqual(response.status_code, 400)
def test_api_invalid_document_type(self):
self.assertEqual(self.doc2.document_type, self.dt1)
response = self.client.post("/api/documents/bulk_edit/", json.dumps({
@@ -920,6 +949,14 @@ class TestBulkEdit(DirectoriesMixin, APITestCase):
doc2 = Document.objects.get(id=self.doc2.id)
self.assertEqual(doc2.document_type, self.dt1)
def test_api_no_document_type(self):
response = self.client.post("/api/documents/bulk_edit/", json.dumps({
"documents": [self.doc2.id],
"method": "set_document_type",
"parameters": {}
}), content_type='application/json')
self.assertEqual(response.status_code, 400)
def test_api_add_invalid_tag(self):
self.assertEqual(list(self.doc2.tags.all()), [self.t1])
response = self.client.post("/api/documents/bulk_edit/", json.dumps({
@@ -931,6 +968,14 @@ class TestBulkEdit(DirectoriesMixin, APITestCase):
self.assertEqual(list(self.doc2.tags.all()), [self.t1])
def test_api_add_tag_no_tag(self):
response = self.client.post("/api/documents/bulk_edit/", json.dumps({
"documents": [self.doc2.id],
"method": "add_tag",
"parameters": {}
}), content_type='application/json')
self.assertEqual(response.status_code, 400)
def test_api_delete_invalid_tag(self):
self.assertEqual(list(self.doc2.tags.all()), [self.t1])
response = self.client.post("/api/documents/bulk_edit/", json.dumps({
@@ -942,6 +987,14 @@ class TestBulkEdit(DirectoriesMixin, APITestCase):
self.assertEqual(list(self.doc2.tags.all()), [self.t1])
def test_api_delete_tag_no_tag(self):
response = self.client.post("/api/documents/bulk_edit/", json.dumps({
"documents": [self.doc2.id],
"method": "remove_tag",
"parameters": {}
}), content_type='application/json')
self.assertEqual(response.status_code, 400)
def test_api_modify_invalid_tags(self):
self.assertEqual(list(self.doc2.tags.all()), [self.t1])
response = self.client.post("/api/documents/bulk_edit/", json.dumps({
@@ -951,6 +1004,21 @@ class TestBulkEdit(DirectoriesMixin, APITestCase):
}), content_type='application/json')
self.assertEqual(response.status_code, 400)
def test_api_modify_tags_no_tags(self):
response = self.client.post("/api/documents/bulk_edit/", json.dumps({
"documents": [self.doc2.id],
"method": "modify_tags",
"parameters": {"remove_tags": [1123123]}
}), content_type='application/json')
self.assertEqual(response.status_code, 400)
response = self.client.post("/api/documents/bulk_edit/", json.dumps({
"documents": [self.doc2.id],
"method": "modify_tags",
"parameters": {'add_tags': [self.t2.id, 1657]}
}), content_type='application/json')
self.assertEqual(response.status_code, 400)
def test_api_selection_data_empty(self):
response = self.client.post("/api/documents/selection_data/", json.dumps({
"documents": []

91
src/documents/tests/test_consumer.py
View File

@@ -468,6 +468,42 @@ class TestConsumer(DirectoriesMixin, TestCase):
self.assertTrue(os.path.isfile(dst))
class PreConsumeTestCase(TestCase):
@mock.patch("documents.consumer.Popen")
@override_settings(PRE_CONSUME_SCRIPT=None)
def test_no_pre_consume_script(self, m):
c = Consumer()
c.path = "path-to-file"
c.run_pre_consume_script()
m.assert_not_called()
@mock.patch("documents.consumer.Popen")
@override_settings(PRE_CONSUME_SCRIPT="does-not-exist")
def test_pre_consume_script_not_found(self, m):
c = Consumer()
c.path = "path-to-file"
self.assertRaises(ConsumerError, c.run_pre_consume_script)
@mock.patch("documents.consumer.Popen")
def test_pre_consume_script(self, m):
with tempfile.NamedTemporaryFile() as script:
with override_settings(PRE_CONSUME_SCRIPT=script.name):
c = Consumer()
c.path = "path-to-file"
c.run_pre_consume_script()
m.assert_called_once()
args, kwargs = m.call_args
command = args[0]
self.assertEqual(command[0], script.name)
self.assertEqual(command[1], "path-to-file")
class PostConsumeTestCase(TestCase):
@mock.patch("documents.consumer.Popen")
@@ -483,36 +519,45 @@ class PostConsumeTestCase(TestCase):
m.assert_not_called()
@mock.patch("documents.consumer.Popen")
@override_settings(POST_CONSUME_SCRIPT="script")
def test_post_consume_script_simple(self, m):
@override_settings(POST_CONSUME_SCRIPT="does-not-exist")
def test_post_consume_script_not_found(self):
doc = Document.objects.create(title="Test", mime_type="application/pdf")
Consumer().run_post_consume_script(doc)
m.assert_called_once()
self.assertRaises(ConsumerError, Consumer().run_post_consume_script, doc)
@mock.patch("documents.consumer.Popen")
def test_post_consume_script_simple(self, m):
with tempfile.NamedTemporaryFile() as script:
with override_settings(POST_CONSUME_SCRIPT=script.name):
doc = Document.objects.create(title="Test", mime_type="application/pdf")
Consumer().run_post_consume_script(doc)
m.assert_called_once()
@mock.patch("documents.consumer.Popen")
@override_settings(POST_CONSUME_SCRIPT="script")
def test_post_consume_script_with_correspondent(self, m):
c = Correspondent.objects.create(name="my_bank")
doc = Document.objects.create(title="Test", mime_type="application/pdf", correspondent=c)
tag1 = Tag.objects.create(name="a")
tag2 = Tag.objects.create(name="b")
doc.tags.add(tag1)
doc.tags.add(tag2)
with tempfile.NamedTemporaryFile() as script:
with override_settings(POST_CONSUME_SCRIPT=script.name):
c = Correspondent.objects.create(name="my_bank")
doc = Document.objects.create(title="Test", mime_type="application/pdf", correspondent=c)
tag1 = Tag.objects.create(name="a")
tag2 = Tag.objects.create(name="b")
doc.tags.add(tag1)
doc.tags.add(tag2)
Consumer().run_post_consume_script(doc)
Consumer().run_post_consume_script(doc)
m.assert_called_once()
m.assert_called_once()
args, kwargs = m.call_args
args, kwargs = m.call_args
command = args[0]
command = args[0]
self.assertEqual(command[0], "script")
self.assertEqual(command[1], str(doc.pk))
self.assertEqual(command[5], f"/api/documents/{doc.pk}/download/")
self.assertEqual(command[6], f"/api/documents/{doc.pk}/thumb/")
self.assertEqual(command[7], "my_bank")
self.assertCountEqual(command[8].split(","), ["a", "b"])
self.assertEqual(command[0], script.name)
self.assertEqual(command[1], str(doc.pk))
self.assertEqual(command[5], f"/api/documents/{doc.pk}/download/")
self.assertEqual(command[6], f"/api/documents/{doc.pk}/thumb/")
self.assertEqual(command[7], "my_bank")
self.assertCountEqual(command[8].split(","), ["a", "b"])

15
src/documents/tests/test_date_parsing.py
View File

@@ -138,3 +138,18 @@ class TestDate(TestCase):
@override_settings(FILENAME_DATE_ORDER="YMD")
def test_filename_date_parse_invalid(self, *args):
self.assertIsNone(parse_date("/tmp/20 408000l 2475 - test.pdf", "No date in here"))
@override_settings(IGNORE_DATES=(datetime.date(2019, 11, 3), datetime.date(2020, 1, 17)))
def test_ignored_dates(self, *args):
text = (
"lorem ipsum 110319, 20200117 and lorem 13.02.2018 lorem "
"ipsum"
)
date = parse_date("", text)
self.assertEqual(
date,
datetime.datetime(
2018, 2, 13, 0, 0,
tzinfo=tz.gettz(settings.TIME_ZONE)
)
)

10
src/documents/tests/test_document_model.py
View File

@@ -1,10 +1,10 @@
import shutil
import tempfile
from datetime import datetime
from pathlib import Path
from unittest import mock
from django.test import TestCase, override_settings
from django.utils import timezone
from ..models import Document, Correspondent
@@ -47,20 +47,20 @@ class TestDocument(TestCase):
def test_file_name(self):
doc = Document(mime_type="application/pdf", title="test", created=datetime(2020, 12, 25))
doc = Document(mime_type="application/pdf", title="test", created=timezone.datetime(2020, 12, 25))
self.assertEqual(doc.get_public_filename(), "2020-12-25 test.pdf")
def test_file_name_jpg(self):
doc = Document(mime_type="image/jpeg", title="test", created=datetime(2020, 12, 25))
doc = Document(mime_type="image/jpeg", title="test", created=timezone.datetime(2020, 12, 25))
self.assertEqual(doc.get_public_filename(), "2020-12-25 test.jpg")
def test_file_name_unknown(self):
doc = Document(mime_type="application/zip", title="test", created=datetime(2020, 12, 25))
doc = Document(mime_type="application/zip", title="test", created=timezone.datetime(2020, 12, 25))
self.assertEqual(doc.get_public_filename(), "2020-12-25 test.zip")
def test_file_name_invalid_type(self):
doc = Document(mime_type="image/jpegasd", title="test", created=datetime(2020, 12, 25))
doc = Document(mime_type="image/jpegasd", title="test", created=timezone.datetime(2020, 12, 25))
self.assertEqual(doc.get_public_filename(), "2020-12-25 test")

10
src/documents/tests/test_management.py
View File

@@ -70,18 +70,18 @@ class TestDecryptDocuments(TestCase):
PASSPHRASE="test"
).enable()
doc = Document.objects.create(checksum="9c9691e51741c1f4f41a20896af31770", title="wow", filename="0000002.pdf.gpg", mime_type="application/pdf", storage_type=Document.STORAGE_TYPE_GPG)
doc = Document.objects.create(checksum="82186aaa94f0b98697d704b90fd1c072", title="wow", filename="0000004.pdf.gpg", mime_type="application/pdf", storage_type=Document.STORAGE_TYPE_GPG)
shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "originals", "0000002.pdf.gpg"), os.path.join(originals_dir, "0000002.pdf.gpg"))
shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "thumbnails", f"0000002.png.gpg"), os.path.join(thumb_dir, f"{doc.id:07}.png.gpg"))
shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "originals", "0000004.pdf.gpg"), os.path.join(originals_dir, "0000004.pdf.gpg"))
shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "thumbnails", f"0000004.png.gpg"), os.path.join(thumb_dir, f"{doc.id:07}.png.gpg"))
call_command('decrypt_documents')
doc.refresh_from_db()
self.assertEqual(doc.storage_type, Document.STORAGE_TYPE_UNENCRYPTED)
self.assertEqual(doc.filename, "0000002.pdf")
self.assertTrue(os.path.isfile(os.path.join(originals_dir, "0000002.pdf")))
self.assertEqual(doc.filename, "0000004.pdf")
self.assertTrue(os.path.isfile(os.path.join(originals_dir, "0000004.pdf")))
self.assertTrue(os.path.isfile(doc.source_path))
self.assertTrue(os.path.isfile(os.path.join(thumb_dir, f"{doc.id:07}.png")))
self.assertTrue(os.path.isfile(doc.thumbnail_path))

196
src/documents/tests/test_management_exporter.py
View File

@@ -3,6 +3,8 @@ import json
import os
import shutil
import tempfile
from pathlib import Path
from unittest import mock
from django.core.management import call_command
from django.test import TestCase, override_settings
@@ -10,54 +12,87 @@ from django.test import TestCase, override_settings
from documents.management.commands import document_exporter
from documents.models import Document, Tag, DocumentType, Correspondent
from documents.sanity_checker import check_sanity
from documents.settings import EXPORTER_FILE_NAME
from documents.tests.utils import DirectoriesMixin, paperless_environment
class TestExportImport(DirectoriesMixin, TestCase):
def setUp(self) -> None:
self.target = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, self.target)
self.d1 = Document.objects.create(content="Content", checksum="42995833e01aea9b3edee44bbfdd7ce1", archive_checksum="62acb0bcbfbcaa62ca6ad3668e4e404b", title="wow1", filename="0000001.pdf", mime_type="application/pdf")
self.d2 = Document.objects.create(content="Content", checksum="9c9691e51741c1f4f41a20896af31770", title="wow2", filename="0000002.pdf", mime_type="application/pdf")
self.d3 = Document.objects.create(content="Content", checksum="d38d7ed02e988e072caf924e0f3fcb76", title="wow2", filename="0000003.pdf", mime_type="application/pdf")
self.d4 = Document.objects.create(content="Content", checksum="82186aaa94f0b98697d704b90fd1c072", title="wow_dec", filename="0000004.pdf.gpg", mime_type="application/pdf", storage_type=Document.STORAGE_TYPE_GPG)
self.t1 = Tag.objects.create(name="t")
self.dt1 = DocumentType.objects.create(name="dt")
self.c1 = Correspondent.objects.create(name="c")
self.d1.tags.add(self.t1)
self.d1.correspondent = self.c1
self.d1.document_type = self.dt1
self.d1.save()
super(TestExportImport, self).setUp()
def _get_document_from_manifest(self, manifest, id):
f = list(filter(lambda d: d['model'] == "documents.document" and d['pk'] == id, manifest))
if len(f) == 1:
return f[0]
else:
raise ValueError(f"document with id {id} does not exist in manifest")
@override_settings(
PASSPHRASE="test"
)
def test_exporter(self):
def _do_export(self, use_filename_format=False, compare_checksums=False, delete=False):
args = ['document_exporter', self.target]
if use_filename_format:
args += ["--use-filename-format"]
if compare_checksums:
args += ["--compare-checksums"]
if delete:
args += ["--delete"]
call_command(*args)
with open(os.path.join(self.target, "manifest.json")) as f:
manifest = json.load(f)
return manifest
def test_exporter(self, use_filename_format=False):
shutil.rmtree(os.path.join(self.dirs.media_dir, "documents"))
shutil.copytree(os.path.join(os.path.dirname(__file__), "samples", "documents"), os.path.join(self.dirs.media_dir, "documents"))
file = os.path.join(self.dirs.originals_dir, "0000001.pdf")
manifest = self._do_export(use_filename_format=use_filename_format)
d1 = Document.objects.create(content="Content", checksum="42995833e01aea9b3edee44bbfdd7ce1", archive_checksum="62acb0bcbfbcaa62ca6ad3668e4e404b", title="wow", filename="0000001.pdf", mime_type="application/pdf")
d2 = Document.objects.create(content="Content", checksum="9c9691e51741c1f4f41a20896af31770", title="wow", filename="0000002.pdf.gpg", mime_type="application/pdf", storage_type=Document.STORAGE_TYPE_GPG)
t1 = Tag.objects.create(name="t")
dt1 = DocumentType.objects.create(name="dt")
c1 = Correspondent.objects.create(name="c")
self.assertEqual(len(manifest), 7)
self.assertEqual(len(list(filter(lambda e: e['model'] == 'documents.document', manifest))), 4)
d1.tags.add(t1)
d1.correspondents = c1
d1.document_type = dt1
d1.save()
d2.save()
self.assertTrue(os.path.exists(os.path.join(self.target, "manifest.json")))
target = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, target)
call_command('document_exporter', target)
with open(os.path.join(target, "manifest.json")) as f:
manifest = json.load(f)
self.assertEqual(len(manifest), 5)
self.assertEqual(self._get_document_from_manifest(manifest, self.d1.id)['fields']['title'], "wow1")
self.assertEqual(self._get_document_from_manifest(manifest, self.d2.id)['fields']['title'], "wow2")
self.assertEqual(self._get_document_from_manifest(manifest, self.d3.id)['fields']['title'], "wow2")
self.assertEqual(self._get_document_from_manifest(manifest, self.d4.id)['fields']['title'], "wow_dec")
for element in manifest:
if element['model'] == 'documents.document':
fname = os.path.join(target, element[document_exporter.EXPORTER_FILE_NAME])
fname = os.path.join(self.target, element[document_exporter.EXPORTER_FILE_NAME])
self.assertTrue(os.path.exists(fname))
self.assertTrue(os.path.exists(os.path.join(target, element[document_exporter.EXPORTER_THUMBNAIL_NAME])))
self.assertTrue(os.path.exists(os.path.join(self.target, element[document_exporter.EXPORTER_THUMBNAIL_NAME])))
with open(fname, "rb") as f:
checksum = hashlib.md5(f.read()).hexdigest()
self.assertEqual(checksum, element['fields']['checksum'])
self.assertEqual(element['fields']['storage_type'], Document.STORAGE_TYPE_UNENCRYPTED)
if document_exporter.EXPORTER_ARCHIVE_NAME in element:
fname = os.path.join(target, element[document_exporter.EXPORTER_ARCHIVE_NAME])
fname = os.path.join(self.target, element[document_exporter.EXPORTER_ARCHIVE_NAME])
self.assertTrue(os.path.exists(fname))
with open(fname, "rb") as f:
@@ -65,24 +100,123 @@ class TestExportImport(DirectoriesMixin, TestCase):
self.assertEqual(checksum, element['fields']['archive_checksum'])
with paperless_environment() as dirs:
self.assertEqual(Document.objects.count(), 2)
self.assertEqual(Document.objects.count(), 4)
Document.objects.all().delete()
Correspondent.objects.all().delete()
DocumentType.objects.all().delete()
Tag.objects.all().delete()
self.assertEqual(Document.objects.count(), 0)
call_command('document_importer', target)
self.assertEqual(Document.objects.count(), 2)
call_command('document_importer', self.target)
self.assertEqual(Document.objects.count(), 4)
self.assertEqual(Tag.objects.count(), 1)
self.assertEqual(Correspondent.objects.count(), 1)
self.assertEqual(DocumentType.objects.count(), 1)
self.assertEqual(Document.objects.get(id=self.d1.id).title, "wow1")
self.assertEqual(Document.objects.get(id=self.d2.id).title, "wow2")
self.assertEqual(Document.objects.get(id=self.d3.id).title, "wow2")
self.assertEqual(Document.objects.get(id=self.d4.id).title, "wow_dec")
messages = check_sanity()
# everything is alright after the test
self.assertEqual(len(messages), 0, str([str(m) for m in messages]))
@override_settings(
PAPERLESS_FILENAME_FORMAT="{title}"
)
def test_exporter_with_filename_format(self):
self.test_exporter()
shutil.rmtree(os.path.join(self.dirs.media_dir, "documents"))
shutil.copytree(os.path.join(os.path.dirname(__file__), "samples", "documents"), os.path.join(self.dirs.media_dir, "documents"))
with override_settings(PAPERLESS_FILENAME_FORMAT="{created_year}/{correspondent}/{title}"):
self.test_exporter(use_filename_format=True)
def test_update_export_changed_time(self):
shutil.rmtree(os.path.join(self.dirs.media_dir, "documents"))
shutil.copytree(os.path.join(os.path.dirname(__file__), "samples", "documents"), os.path.join(self.dirs.media_dir, "documents"))
self._do_export()
self.assertTrue(os.path.exists(os.path.join(self.target, "manifest.json")))
st_mtime_1 = os.stat(os.path.join(self.target, "manifest.json")).st_mtime
with mock.patch("documents.management.commands.document_exporter.shutil.copy2") as m:
self._do_export()
m.assert_not_called()
self.assertTrue(os.path.exists(os.path.join(self.target, "manifest.json")))
st_mtime_2 = os.stat(os.path.join(self.target, "manifest.json")).st_mtime
Path(self.d1.source_path).touch()
with mock.patch("documents.management.commands.document_exporter.shutil.copy2") as m:
self._do_export()
self.assertEqual(m.call_count, 1)
st_mtime_3 = os.stat(os.path.join(self.target, "manifest.json")).st_mtime
self.assertTrue(os.path.exists(os.path.join(self.target, "manifest.json")))
self.assertNotEqual(st_mtime_1, st_mtime_2)
self.assertNotEqual(st_mtime_2, st_mtime_3)
def test_update_export_changed_checksum(self):
shutil.rmtree(os.path.join(self.dirs.media_dir, "documents"))
shutil.copytree(os.path.join(os.path.dirname(__file__), "samples", "documents"), os.path.join(self.dirs.media_dir, "documents"))
self._do_export()
self.assertTrue(os.path.exists(os.path.join(self.target, "manifest.json")))
with mock.patch("documents.management.commands.document_exporter.shutil.copy2") as m:
self._do_export()
m.assert_not_called()
self.assertTrue(os.path.exists(os.path.join(self.target, "manifest.json")))
self.d2.checksum = "asdfasdgf3"
self.d2.save()
with mock.patch("documents.management.commands.document_exporter.shutil.copy2") as m:
self._do_export(compare_checksums=True)
self.assertEqual(m.call_count, 1)
self.assertTrue(os.path.exists(os.path.join(self.target, "manifest.json")))
def test_update_export_deleted_document(self):
shutil.rmtree(os.path.join(self.dirs.media_dir, "documents"))
shutil.copytree(os.path.join(os.path.dirname(__file__), "samples", "documents"), os.path.join(self.dirs.media_dir, "documents"))
manifest = self._do_export()
self.assertTrue(len(manifest), 7)
doc_from_manifest = self._get_document_from_manifest(manifest, self.d3.id)
self.assertTrue(os.path.isfile(os.path.join(self.target, doc_from_manifest[EXPORTER_FILE_NAME])))
self.d3.delete()
manifest = self._do_export()
self.assertRaises(ValueError, self._get_document_from_manifest, manifest, self.d3.id)
self.assertTrue(os.path.isfile(os.path.join(self.target, doc_from_manifest[EXPORTER_FILE_NAME])))
manifest = self._do_export(delete=True)
self.assertFalse(os.path.isfile(os.path.join(self.target, doc_from_manifest[EXPORTER_FILE_NAME])))
self.assertTrue(len(manifest), 6)
@override_settings(PAPERLESS_FILENAME_FORMAT="{title}/{correspondent}")
def test_update_export_changed_location(self):
shutil.rmtree(os.path.join(self.dirs.media_dir, "documents"))
shutil.copytree(os.path.join(os.path.dirname(__file__), "samples", "documents"), os.path.join(self.dirs.media_dir, "documents"))
m = self._do_export(use_filename_format=True)
self.assertTrue(os.path.isfile(os.path.join(self.target, "wow1", "c.pdf")))
self.assertTrue(os.path.exists(os.path.join(self.target, "manifest.json")))
self.d1.title = "new_title"
self.d1.save()
self._do_export(use_filename_format=True, delete=True)
self.assertFalse(os.path.isfile(os.path.join(self.target, "wow1", "c.pdf")))
self.assertFalse(os.path.isdir(os.path.join(self.target, "wow1")))
self.assertTrue(os.path.isfile(os.path.join(self.target, "new_title", "c.pdf")))
self.assertTrue(os.path.exists(os.path.join(self.target, "manifest.json")))
self.assertTrue(os.path.isfile(os.path.join(self.target, "wow2", "none.pdf")))
self.assertTrue(os.path.isfile(os.path.join(self.target, "wow2", "none_01.pdf")))
def test_export_missing_files(self):

52
src/documents/tests/test_management_thumbnails.py Normal file
View File

@@ -0,0 +1,52 @@
import os
import shutil
from unittest import mock
from django.core.management import call_command
from django.test import TestCase
from documents.management.commands.document_thumbnails import _process_document
from documents.models import Document, Tag, Correspondent, DocumentType
from documents.tests.utils import DirectoriesMixin
class TestMakeThumbnails(DirectoriesMixin, TestCase):
def make_models(self):
self.d1 = Document.objects.create(checksum="A", title="A", content="first document", mime_type="application/pdf", filename="test.pdf")
shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "simple.pdf"), self.d1.source_path)
self.d2 = Document.objects.create(checksum="Ass", title="A", content="first document", mime_type="application/pdf", filename="test2.pdf")
shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "simple.pdf"), self.d2.source_path)
def setUp(self) -> None:
super(TestMakeThumbnails, self).setUp()
self.make_models()
def test_process_document(self):
self.assertFalse(os.path.isfile(self.d1.thumbnail_path))
_process_document(self.d1.id)
self.assertTrue(os.path.isfile(self.d1.thumbnail_path))
@mock.patch("documents.management.commands.document_thumbnails.shutil.move")
def test_process_document_invalid_mime_type(self, m):
self.d1.mime_type = "asdasdasd"
self.d1.save()
_process_document(self.d1.id)
m.assert_not_called()
def test_command(self):
self.assertFalse(os.path.isfile(self.d1.thumbnail_path))
self.assertFalse(os.path.isfile(self.d2.thumbnail_path))
call_command('document_thumbnails')
self.assertTrue(os.path.isfile(self.d1.thumbnail_path))
self.assertTrue(os.path.isfile(self.d2.thumbnail_path))
def test_command_documentid(self):
self.assertFalse(os.path.isfile(self.d1.thumbnail_path))
self.assertFalse(os.path.isfile(self.d2.thumbnail_path))
call_command('document_thumbnails', '-d', f"{self.d1.id}")
self.assertTrue(os.path.isfile(self.d1.thumbnail_path))
self.assertFalse(os.path.isfile(self.d2.thumbnail_path))

18
src/documents/tests/test_matchables.py
View File

@@ -21,13 +21,15 @@ class TestMatching(TestCase):
matching_algorithm=getattr(klass, algorithm)
)
for string in true:
doc = Document(content=string)
self.assertTrue(
matching.matches(instance, string),
matching.matches(instance, doc),
'"%s" should match "%s" but it does not' % (text, string)
)
for string in false:
doc = Document(content=string)
self.assertFalse(
matching.matches(instance, string),
matching.matches(instance, doc),
'"%s" should not match "%s" but it does' % (text, string)
)
@@ -169,7 +171,7 @@ class TestMatching(TestCase):
def test_match_regex(self):
self._test_matching(
r"alpha\w+gamma",
"alpha\w+gamma",
"MATCH_REGEX",
(
"I have alpha_and_gamma in me",
@@ -187,6 +189,16 @@ class TestMatching(TestCase):
)
)
def test_tach_invalid_regex(self):
self._test_matching(
"[[",
"MATCH_REGEX",
[],
[
"Don't match this"
]
)
def test_match_fuzzy(self):
self._test_matching(

2
src/documents/tests/test_migrations.py
View File

@@ -98,7 +98,7 @@ class TestMigrateMimeType(DirectoriesMixin, TestMigrations):
doc2 = Document.objects.create(checksum="B", file_type="pdf", storage_type=STORAGE_TYPE_GPG)
self.doc2_id = doc2.id
shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "originals", "0000002.pdf.gpg"), source_path_before(doc2))
shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "originals", "0000004.pdf.gpg"), source_path_before(doc2))
def testMimeTypesMigrated(self):
Document = self.apps.get_model('documents', 'Document')

1
src/documents/tests/test_parsers.py
View File

@@ -120,3 +120,4 @@ class TestParserAvailability(TestCase):
self.assertTrue(is_file_ext_supported('.pdf'))
self.assertFalse(is_file_ext_supported('.hsdfh'))
self.assertFalse(is_file_ext_supported(''))

34
src/documents/tests/test_settings.py Normal file
View File

@@ -0,0 +1,34 @@
import logging
from unittest import mock
from django.test import TestCase
from paperless.settings import default_task_workers, default_threads_per_worker
class TestSettings(TestCase):
@mock.patch("paperless.settings.multiprocessing.cpu_count")
def test_single_core(self, cpu_count):
cpu_count.return_value = 1
default_workers = default_task_workers()
default_threads = default_threads_per_worker(default_workers)
self.assertEqual(default_workers, 1)
self.assertEqual(default_threads, 1)
def test_workers_threads(self):
for i in range(2, 64):
with mock.patch("paperless.settings.multiprocessing.cpu_count") as cpu_count:
cpu_count.return_value = i
default_workers = default_task_workers()
default_threads = default_threads_per_worker(default_workers)
self.assertTrue(default_workers >= 1)
self.assertTrue(default_threads >= 1)
self.assertTrue(default_workers * default_threads < i, f"{i}")

30
src/documents/tests/test_views.py Normal file
View File

@@ -0,0 +1,30 @@
from django.conf import settings
from django.contrib.auth.models import User
from django.test import TestCase
class TestViews(TestCase):
def setUp(self) -> None:
self.user = User.objects.create_user("testuser")
def test_login_redirect(self):
response = self.client.get('/')
self.assertEqual(response.status_code, 302)
self.assertEqual(response.url, "/accounts/login/?next=/")
def test_index(self):
self.client.force_login(self.user)
for (language_given, language_actual) in [("", "en-US"), ("en-US", "en-US"), ("de", "de"), ("en", "en-US"), ("en-us", "en-US"), ("fr", "fr"), ("jp", "en-US")]:
if language_given:
self.client.cookies.load({settings.LANGUAGE_COOKIE_NAME: language_given})
elif settings.LANGUAGE_COOKIE_NAME in self.client.cookies.keys():
self.client.cookies.pop(settings.LANGUAGE_COOKIE_NAME)
response = self.client.get('/', )
self.assertEqual(response.status_code, 200)
self.assertEqual(response.context_data['webmanifest'], f"frontend/{language_actual}/manifest.webmanifest")
self.assertEqual(response.context_data['styles_css'], f"frontend/{language_actual}/styles.css")
self.assertEqual(response.context_data['runtime_js'], f"frontend/{language_actual}/runtime.js")
self.assertEqual(response.context_data['polyfills_js'], f"frontend/{language_actual}/polyfills.js")
self.assertEqual(response.context_data['main_js'], f"frontend/{language_actual}/main.js")

21
src/documents/views.py
View File

@@ -1,3 +1,4 @@
import logging
import os
import tempfile
from datetime import datetime
@@ -79,7 +80,7 @@ class IndexView(TemplateView):
context['runtime_js'] = f"frontend/{self.get_language()}/runtime.js"
context['polyfills_js'] = f"frontend/{self.get_language()}/polyfills.js" # NOQA: E501
context['main_js'] = f"frontend/{self.get_language()}/main.js"
context['manifest'] = f"frontend/{self.get_language()}/manifest.webmanifest" # NOQA: E501
context['webmanifest'] = f"frontend/{self.get_language()}/manifest.webmanifest" # NOQA: E501
return context
@@ -158,6 +159,9 @@ class DocumentViewSet(RetrieveModelMixin,
"added",
"archive_serial_number")
def get_queryset(self):
return Document.objects.distinct()
def get_serializer(self, *args, **kwargs):
fields_param = self.request.query_params.get('fields', None)
if fields_param:
@@ -458,12 +462,21 @@ class SearchView(APIView):
self.ix = index.open_index()
def add_infos_to_hit(self, r):
doc = Document.objects.get(id=r['id'])
try:
doc = Document.objects.get(id=r['id'])
except Document.DoesNotExist:
logging.getLogger(__name__).warning(
f"Search index returned a non-existing document: "
f"id: {r['id']}, title: {r['title']}. "
f"Search index needs reindex."
)
doc = None
return {'id': r['id'],
'highlights': r.highlights("content", text=doc.content),
'highlights': r.highlights("content", text=doc.content) if doc else None, # NOQA: E501
'score': r.score,
'rank': r.rank,
'document': DocumentSerializer(doc).data,
'document': DocumentSerializer(doc).data if doc else None,
'title': r['title']
}

161
src/locale/de/LC_MESSAGES/django.po
View File

@@ -4,16 +4,16 @@
# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
#
# Translators:
# Jonas Winkler <dev@jpwinkler.de>, 2021
# Jonas Winkler, 2021
#
#, fuzzy
msgid ""
msgstr ""
"Project-Id-Version: PACKAGE VERSION\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2021-01-02 00:26+0000\n"
"POT-Creation-Date: 2021-01-10 21:41+0000\n"
"PO-Revision-Date: 2020-12-30 19:27+0000\n"
"Last-Translator: Jonas Winkler <dev@jpwinkler.de>, 2021\n"
"Last-Translator: Jonas Winkler, 2021\n"
"Language-Team: German (https://www.transifex.com/paperless/teams/115905/de/)\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
@@ -50,7 +50,7 @@ msgid "Automatic"
msgstr "Automatisch"
#: documents/models.py:41 documents/models.py:354 paperless_mail/models.py:25
#: paperless_mail/models.py:100
#: paperless_mail/models.py:109
msgid "name"
msgstr "Name"
@@ -346,34 +346,84 @@ msgstr "Filterregel"
msgid "filter rules"
msgstr "Filterregeln"
#: paperless/settings.py:254
#: documents/templates/index.html:20
msgid "Paperless-ng is loading..."
msgstr "Paperless-ng wird geladen..."
#: documents/templates/registration/logged_out.html:13
msgid "Paperless-ng signed out"
msgstr "Paperless-ng abgemeldet"
#: documents/templates/registration/logged_out.html:41
msgid "You have been successfully logged out. Bye!"
msgstr "Sie wurden erfolgreich abgemeldet. Auf Wiedersehen!"
#: documents/templates/registration/logged_out.html:42
msgid "Sign in again"
msgstr "Erneut anmelden"
#: documents/templates/registration/login.html:13
msgid "Paperless-ng sign in"
msgstr "Paperless-ng Anmeldung"
#: documents/templates/registration/login.html:42
msgid "Please sign in."
msgstr "Bitte melden Sie sich an."
#: documents/templates/registration/login.html:45
msgid "Your username and password didn't match. Please try again."
msgstr ""
"Ihr Benutzername und Passwort stimmen nicht überein. Bitte versuchen Sie es "
"erneut."
#: documents/templates/registration/login.html:48
msgid "Username"
msgstr "Benutzername"
#: documents/templates/registration/login.html:49
msgid "Password"
msgstr "Passwort"
#: documents/templates/registration/login.html:54
msgid "Sign in"
msgstr "Anmelden"
#: paperless/settings.py:268
msgid "English"
msgstr "Englisch"
#: paperless/settings.py:255
#: paperless/settings.py:269
msgid "German"
msgstr "Deutsch"
#: paperless/settings.py:270
msgid "Dutch"
msgstr "Niederländisch"
#: paperless/settings.py:271
msgid "French"
msgstr "Französisch"
#: paperless/urls.py:108
msgid "Paperless-ng administration"
msgstr "Paperless-ng Administration"
#: paperless_mail/admin.py:24
#: paperless_mail/admin.py:25
msgid "Filter"
msgstr "Filter"
#: paperless_mail/admin.py:26
#: paperless_mail/admin.py:27
msgid ""
"Paperless will only process mails that match ALL of the filters given below."
msgstr ""
"Paperless wird nur E-Mails verarbeiten, für die alle der hier angegebenen "
"Filter zutreffen."
#: paperless_mail/admin.py:34
#: paperless_mail/admin.py:37
msgid "Actions"
msgstr "Aktionen"
#: paperless_mail/admin.py:36
#: paperless_mail/admin.py:39
msgid ""
"The action applied to the mail. This action is only performed when documents"
" were consumed from the mail. Mails without attachments will remain entirely"
@@ -383,11 +433,11 @@ msgstr ""
"auf E-Mails angewendet, aus denen Anhänge verarbeitet wurden. E-Mails ohne "
"Anhänge werden vollständig ignoriert."
#: paperless_mail/admin.py:43
#: paperless_mail/admin.py:46
msgid "Metadata"
msgstr "Metadaten"
#: paperless_mail/admin.py:45
#: paperless_mail/admin.py:48
msgid ""
"Assign metadata to documents consumed from this rule automatically. If you "
"do not assign tags, types or correspondents here, paperless will still "
@@ -448,7 +498,7 @@ msgstr "Benutzername"
#: paperless_mail/models.py:50
msgid "password"
msgstr "Password"
msgstr "Passwort"
#: paperless_mail/models.py:60
msgid "mail rule"
@@ -458,87 +508,120 @@ msgstr "E-Mail-Regel"
msgid "mail rules"
msgstr "E-Mail-Regeln"
#: paperless_mail/models.py:69
#: paperless_mail/models.py:67
msgid "Only process attachments."
msgstr "Nur Anhänge verarbeiten."
#: paperless_mail/models.py:68
msgid "Process all files, including 'inline' attachments."
msgstr "Alle Dateien verarbeiten, auch 'inline'-Anhänge."
#: paperless_mail/models.py:78
msgid "Mark as read, don't process read mails"
msgstr "Als gelesen markieren, gelesene E-Mails nicht verarbeiten"
#: paperless_mail/models.py:70
#: paperless_mail/models.py:79
msgid "Flag the mail, don't process flagged mails"
msgstr "Als wichtig markieren, markierte E-Mails nicht verarbeiten"
#: paperless_mail/models.py:71
#: paperless_mail/models.py:80
msgid "Move to specified folder"
msgstr "In angegebenen Ordner verschieben"
#: paperless_mail/models.py:72
#: paperless_mail/models.py:81
msgid "Delete"
msgstr "Löschen"
#: paperless_mail/models.py:79
#: paperless_mail/models.py:88
msgid "Use subject as title"
msgstr "Betreff als Titel verwenden"
#: paperless_mail/models.py:80
#: paperless_mail/models.py:89
msgid "Use attachment filename as title"
msgstr "Dateiname des Anhangs als Titel verwenden"
#: paperless_mail/models.py:90
#: paperless_mail/models.py:99
msgid "Do not assign a correspondent"
msgstr "Keinen Korrespondenten zuweisen"
#: paperless_mail/models.py:92
#: paperless_mail/models.py:101
msgid "Use mail address"
msgstr "E-Mail-Adresse benutzen"
#: paperless_mail/models.py:94
#: paperless_mail/models.py:103
msgid "Use name (or mail address if not available)"
msgstr "Absendername benutzen (oder E-Mail-Adressen, wenn nicht verfügbar)"
#: paperless_mail/models.py:96
#: paperless_mail/models.py:105
msgid "Use correspondent selected below"
msgstr "Nachfolgend ausgewählten Korrespondent verwenden"
#: paperless_mail/models.py:104
#: paperless_mail/models.py:113
msgid "order"
msgstr "Reihenfolge"
#: paperless_mail/models.py:111
#: paperless_mail/models.py:120
msgid "account"
msgstr "Konto"
#: paperless_mail/models.py:115
#: paperless_mail/models.py:124
msgid "folder"
msgstr "Ordner"
#: paperless_mail/models.py:119
#: paperless_mail/models.py:128
msgid "filter from"
msgstr "Absender filtern"
#: paperless_mail/models.py:122
#: paperless_mail/models.py:131
msgid "filter subject"
msgstr "Betreff filtern"
#: paperless_mail/models.py:125
#: paperless_mail/models.py:134
msgid "filter body"
msgstr "Nachrichteninhalt filtern"
#: paperless_mail/models.py:129
#: paperless_mail/models.py:138
msgid "filter attachment filename"
msgstr "Anhang-Dateiname filtern"
#: paperless_mail/models.py:140
msgid ""
"Only consume documents which entirely match this filename if specified. "
"Wildcards such as *.pdf or *invoice* are allowed. Case insensitive."
msgstr ""
"Wenn angegeben werden nur Dateien verarbeitet, die diesem Dateinamen exakt "
"entsprechen. Platzhalter wie *.pdf oder *rechnung* sind erlaubt. Groß- und "
"Kleinschreibung ist irrelevant."
#: paperless_mail/models.py:146
msgid "maximum age"
msgstr "Maximales Alter"
#: paperless_mail/models.py:131
#: paperless_mail/models.py:148
msgid "Specified in days."
msgstr "Angegeben in Tagen."
#: paperless_mail/models.py:134
#: paperless_mail/models.py:151
msgid "attachment type"
msgstr "Dateianhangstyp"
#: paperless_mail/models.py:154
msgid ""
"Inline attachments include embedded images, so it's best to combine this "
"option with a filename filter."
msgstr ""
"'Inline'-Anhänge schließen eingebettete Bilder mit ein, daher sollte diese "
"Einstellung mit einem Dateinamenfilter kombiniert werden."
#: paperless_mail/models.py:159
msgid "action"
msgstr "Aktion"
#: paperless_mail/models.py:140
#: paperless_mail/models.py:165
msgid "action parameter"
msgstr "Parameter für Aktion"
#: paperless_mail/models.py:142
#: paperless_mail/models.py:167
msgid ""
"Additional parameter for the action selected above, i.e., the target folder "
"of the move to folder action."
@@ -546,22 +629,22 @@ msgstr ""
"Zusätzlicher Parameter für die oben ausgewählte Aktion, zum Beispiel der "
"Zielordner für die Aktion \"In angegebenen Ordner verschieben\""
#: paperless_mail/models.py:148
#: paperless_mail/models.py:173
msgid "assign title from"
msgstr "Titel zuweisen von"
#: paperless_mail/models.py:158
#: paperless_mail/models.py:183
msgid "assign this tag"
msgstr "Dieses Tag zuweisen"
#: paperless_mail/models.py:166
#: paperless_mail/models.py:191
msgid "assign this document type"
msgstr "Diesen Dokumenttyp zuweisen"
#: paperless_mail/models.py:170
#: paperless_mail/models.py:195
msgid "assign correspondent from"
msgstr "Korrespondent zuweisen von"
#: paperless_mail/models.py:180
#: paperless_mail/models.py:205
msgid "assign this correspondent"
msgstr "Diesen Korrespondent zuweisen"

160
src/locale/en-us/LC_MESSAGES/django.po
View File

@@ -8,7 +8,7 @@ msgid ""
msgstr ""
"Project-Id-Version: PACKAGE VERSION\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2021-01-02 00:26+0000\n"
"POT-Creation-Date: 2021-01-10 21:41+0000\n"
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
"Language-Team: LANGUAGE <LL@li.org>\n"
@@ -46,7 +46,7 @@ msgid "Automatic"
msgstr ""
#: documents/models.py:41 documents/models.py:354 paperless_mail/models.py:25
#: paperless_mail/models.py:100
#: paperless_mail/models.py:109
msgid "name"
msgstr ""
@@ -338,43 +338,91 @@ msgstr ""
msgid "filter rules"
msgstr ""
#: paperless/settings.py:254
#: documents/templates/index.html:20
msgid "Paperless-ng is loading..."
msgstr ""
#: documents/templates/registration/logged_out.html:13
msgid "Paperless-ng signed out"
msgstr ""
#: documents/templates/registration/logged_out.html:41
msgid "You have been successfully logged out. Bye!"
msgstr ""
#: documents/templates/registration/logged_out.html:42
msgid "Sign in again"
msgstr ""
#: documents/templates/registration/login.html:13
msgid "Paperless-ng sign in"
msgstr ""
#: documents/templates/registration/login.html:42
msgid "Please sign in."
msgstr ""
#: documents/templates/registration/login.html:45
msgid "Your username and password didn't match. Please try again."
msgstr ""
#: documents/templates/registration/login.html:48
msgid "Username"
msgstr ""
#: documents/templates/registration/login.html:49
msgid "Password"
msgstr ""
#: documents/templates/registration/login.html:54
msgid "Sign in"
msgstr ""
#: paperless/settings.py:268
msgid "English"
msgstr ""
#: paperless/settings.py:255
#: paperless/settings.py:269
msgid "German"
msgstr ""
#: paperless/settings.py:270
msgid "Dutch"
msgstr ""
#: paperless/settings.py:271
msgid "French"
msgstr ""
#: paperless/urls.py:108
msgid "Paperless-ng administration"
msgstr ""
#: paperless_mail/admin.py:24
#: paperless_mail/admin.py:25
msgid "Filter"
msgstr ""
#: paperless_mail/admin.py:26
#: paperless_mail/admin.py:27
msgid ""
"Paperless will only process mails that match ALL of the filters given below."
msgstr ""
#: paperless_mail/admin.py:34
#: paperless_mail/admin.py:37
msgid "Actions"
msgstr ""
#: paperless_mail/admin.py:36
#: paperless_mail/admin.py:39
msgid ""
"The action applied to the mail. This action is only performed when documents "
"were consumed from the mail. Mails without attachments will remain entirely "
"untouched."
msgstr ""
#: paperless_mail/admin.py:43
#: paperless_mail/admin.py:46
msgid "Metadata"
msgstr ""
#: paperless_mail/admin.py:45
#: paperless_mail/admin.py:48
msgid ""
"Assign metadata to documents consumed from this rule automatically. If you "
"do not assign tags, types or correspondents here, paperless will still "
@@ -439,108 +487,136 @@ msgstr ""
msgid "mail rules"
msgstr ""
#: paperless_mail/models.py:69
#: paperless_mail/models.py:67
msgid "Only process attachments."
msgstr ""
#: paperless_mail/models.py:68
msgid "Process all files, including 'inline' attachments."
msgstr ""
#: paperless_mail/models.py:78
msgid "Mark as read, don't process read mails"
msgstr ""
#: paperless_mail/models.py:70
#: paperless_mail/models.py:79
msgid "Flag the mail, don't process flagged mails"
msgstr ""
#: paperless_mail/models.py:71
#: paperless_mail/models.py:80
msgid "Move to specified folder"
msgstr ""
#: paperless_mail/models.py:72
#: paperless_mail/models.py:81
msgid "Delete"
msgstr ""
#: paperless_mail/models.py:79
#: paperless_mail/models.py:88
msgid "Use subject as title"
msgstr ""
#: paperless_mail/models.py:80
#: paperless_mail/models.py:89
msgid "Use attachment filename as title"
msgstr ""
#: paperless_mail/models.py:90
#: paperless_mail/models.py:99
msgid "Do not assign a correspondent"
msgstr ""
#: paperless_mail/models.py:92
#: paperless_mail/models.py:101
msgid "Use mail address"
msgstr ""
#: paperless_mail/models.py:94
#: paperless_mail/models.py:103
msgid "Use name (or mail address if not available)"
msgstr ""
#: paperless_mail/models.py:96
#: paperless_mail/models.py:105
msgid "Use correspondent selected below"
msgstr ""
#: paperless_mail/models.py:104
#: paperless_mail/models.py:113
msgid "order"
msgstr ""
#: paperless_mail/models.py:111
#: paperless_mail/models.py:120
msgid "account"
msgstr ""
#: paperless_mail/models.py:115
#: paperless_mail/models.py:124
msgid "folder"
msgstr ""
#: paperless_mail/models.py:119
#: paperless_mail/models.py:128
msgid "filter from"
msgstr ""
#: paperless_mail/models.py:122
#: paperless_mail/models.py:131
msgid "filter subject"
msgstr ""
#: paperless_mail/models.py:125
#: paperless_mail/models.py:134
msgid "filter body"
msgstr ""
#: paperless_mail/models.py:129
msgid "maximum age"
msgstr ""
#: paperless_mail/models.py:131
msgid "Specified in days."
msgstr ""
#: paperless_mail/models.py:134
msgid "action"
#: paperless_mail/models.py:138
msgid "filter attachment filename"
msgstr ""
#: paperless_mail/models.py:140
msgid ""
"Only consume documents which entirely match this filename if specified. "
"Wildcards such as *.pdf or *invoice* are allowed. Case insensitive."
msgstr ""
#: paperless_mail/models.py:146
msgid "maximum age"
msgstr ""
#: paperless_mail/models.py:148
msgid "Specified in days."
msgstr ""
#: paperless_mail/models.py:151
msgid "attachment type"
msgstr ""
#: paperless_mail/models.py:154
msgid ""
"Inline attachments include embedded images, so it's best to combine this "
"option with a filename filter."
msgstr ""
#: paperless_mail/models.py:159
msgid "action"
msgstr ""
#: paperless_mail/models.py:165
msgid "action parameter"
msgstr ""
#: paperless_mail/models.py:142
#: paperless_mail/models.py:167
msgid ""
"Additional parameter for the action selected above, i.e., the target folder "
"of the move to folder action."
msgstr ""
#: paperless_mail/models.py:148
#: paperless_mail/models.py:173
msgid "assign title from"
msgstr ""
#: paperless_mail/models.py:158
#: paperless_mail/models.py:183
msgid "assign this tag"
msgstr ""
#: paperless_mail/models.py:166
#: paperless_mail/models.py:191
msgid "assign this document type"
msgstr ""
#: paperless_mail/models.py:170
#: paperless_mail/models.py:195
msgid "assign correspondent from"
msgstr ""
#: paperless_mail/models.py:180
#: paperless_mail/models.py:205
msgid "assign this correspondent"
msgstr ""

163
src/locale/fr/LC_MESSAGES/django.po
View File

@@ -4,7 +4,7 @@
# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
#
# Translators:
# Jonas Winkler <dev@jpwinkler.de>, 2020
# Jonas Winkler, 2020
# Philmo67, 2021
#
#, fuzzy
@@ -12,7 +12,7 @@ msgid ""
msgstr ""
"Project-Id-Version: PACKAGE VERSION\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2021-01-02 00:26+0000\n"
"POT-Creation-Date: 2021-01-10 21:41+0000\n"
"PO-Revision-Date: 2020-12-30 19:27+0000\n"
"Last-Translator: Philmo67, 2021\n"
"Language-Team: French (https://www.transifex.com/paperless/teams/115905/fr/)\n"
@@ -51,7 +51,7 @@ msgid "Automatic"
msgstr "Automatique"
#: documents/models.py:41 documents/models.py:354 paperless_mail/models.py:25
#: paperless_mail/models.py:100
#: paperless_mail/models.py:109
msgid "name"
msgstr "nom"
@@ -348,34 +348,84 @@ msgstr "règle de filtrage"
msgid "filter rules"
msgstr "règles de filtrage"
#: paperless/settings.py:254
#: documents/templates/index.html:20
msgid "Paperless-ng is loading..."
msgstr "Paperless-ng est en cours de chargement..."
#: documents/templates/registration/logged_out.html:13
msgid "Paperless-ng signed out"
msgstr "Déconnecté de Paperless-ng"
#: documents/templates/registration/logged_out.html:41
msgid "You have been successfully logged out. Bye!"
msgstr "Vous avez été déconnecté avec succès. Au revoir !"
#: documents/templates/registration/logged_out.html:42
msgid "Sign in again"
msgstr "Se reconnecter"
#: documents/templates/registration/login.html:13
msgid "Paperless-ng sign in"
msgstr "Connexion à Paperless-ng"
#: documents/templates/registration/login.html:42
msgid "Please sign in."
msgstr "Veuillez vous connecter."
#: documents/templates/registration/login.html:45
msgid "Your username and password didn't match. Please try again."
msgstr ""
"Votre nom d'utilisateur et votre mot de passe ne correspondent pas. Veuillez"
" réessayer."
#: documents/templates/registration/login.html:48
msgid "Username"
msgstr "Nom d'utilisateur"
#: documents/templates/registration/login.html:49
msgid "Password"
msgstr "Mot de passe"
#: documents/templates/registration/login.html:54
msgid "Sign in"
msgstr "S'identifier"
#: paperless/settings.py:268
msgid "English"
msgstr "Anglais"
#: paperless/settings.py:255
#: paperless/settings.py:269
msgid "German"
msgstr "Allemand"
#: paperless/settings.py:270
msgid "Dutch"
msgstr "Néerlandais"
#: paperless/settings.py:271
msgid "French"
msgstr "Français"
#: paperless/urls.py:108
msgid "Paperless-ng administration"
msgstr "Administration de Paperless-ng"
#: paperless_mail/admin.py:24
#: paperless_mail/admin.py:25
msgid "Filter"
msgstr "Filtrage"
#: paperless_mail/admin.py:26
#: paperless_mail/admin.py:27
msgid ""
"Paperless will only process mails that match ALL of the filters given below."
msgstr ""
"Paperless-ng ne traitera que les courriers qui correspondent à TOUS les "
"filtres ci-dessous."
#: paperless_mail/admin.py:34
#: paperless_mail/admin.py:37
msgid "Actions"
msgstr "Actions"
#: paperless_mail/admin.py:36
#: paperless_mail/admin.py:39
msgid ""
"The action applied to the mail. This action is only performed when documents"
" were consumed from the mail. Mails without attachments will remain entirely"
@@ -385,20 +435,20 @@ msgstr ""
"documents ont été traités depuis des courriels. Les courriels sans pièces "
"jointes demeurent totalement inchangés."
#: paperless_mail/admin.py:43
#: paperless_mail/admin.py:46
msgid "Metadata"
msgstr "Métadonnées"
#: paperless_mail/admin.py:45
#: paperless_mail/admin.py:48
msgid ""
"Assign metadata to documents consumed from this rule automatically. If you "
"do not assign tags, types or correspondents here, paperless will still "
"process all matching rules that you have defined."
msgstr ""
"Affecter automatiquement des métadonnées aux documents traités à partir de "
"cette règle. Si vous n'affectez pas d'étiquettes, de types ou de "
"correspondants ici, Paperless-ng traitera quand même toutes les règles de "
"rapprochement que vous avez définies."
"cette règle. Si vous n'affectez pas d'étiquette, de type ou de correspondant"
" ici, Paperless-ng appliquera toutes les autres règles de rapprochement que "
"vous avez définies."
#: paperless_mail/apps.py:9
msgid "Paperless mail"
@@ -460,87 +510,120 @@ msgstr "règle de courriel"
msgid "mail rules"
msgstr "règles de courriel"
#: paperless_mail/models.py:69
#: paperless_mail/models.py:67
msgid "Only process attachments."
msgstr "Ne traiter que les pièces jointes."
#: paperless_mail/models.py:68
msgid "Process all files, including 'inline' attachments."
msgstr "Traiter tous les fichiers, y compris les pièces jointes \"en ligne\"."
#: paperless_mail/models.py:78
msgid "Mark as read, don't process read mails"
msgstr "Marquer comme lu, ne pas traiter les courriels lus"
#: paperless_mail/models.py:70
#: paperless_mail/models.py:79
msgid "Flag the mail, don't process flagged mails"
msgstr "Marquer le courriel, ne pas traiter les courriels marqués"
#: paperless_mail/models.py:71
#: paperless_mail/models.py:80
msgid "Move to specified folder"
msgstr "Déplacer vers le dossier spécifié"
#: paperless_mail/models.py:72
#: paperless_mail/models.py:81
msgid "Delete"
msgstr "Supprimer"
#: paperless_mail/models.py:79
#: paperless_mail/models.py:88
msgid "Use subject as title"
msgstr "Utiliser le sujet en tant que titre"
#: paperless_mail/models.py:80
#: paperless_mail/models.py:89
msgid "Use attachment filename as title"
msgstr "Utiliser le nom de la pièce jointe en tant que titre"
#: paperless_mail/models.py:90
#: paperless_mail/models.py:99
msgid "Do not assign a correspondent"
msgstr "Ne pas affecter de correspondant"
#: paperless_mail/models.py:92
#: paperless_mail/models.py:101
msgid "Use mail address"
msgstr "Utiliser l'adresse électronique"
#: paperless_mail/models.py:94
#: paperless_mail/models.py:103
msgid "Use name (or mail address if not available)"
msgstr "Utiliser le nom (ou l'adresse électronique s'il n'est pas disponible)"
#: paperless_mail/models.py:96
#: paperless_mail/models.py:105
msgid "Use correspondent selected below"
msgstr "Utiliser le correspondant sélectionné ci-dessous"
#: paperless_mail/models.py:104
#: paperless_mail/models.py:113
msgid "order"
msgstr "ordre"
#: paperless_mail/models.py:111
#: paperless_mail/models.py:120
msgid "account"
msgstr "compte"
#: paperless_mail/models.py:115
#: paperless_mail/models.py:124
msgid "folder"
msgstr "répertoire"
#: paperless_mail/models.py:119
#: paperless_mail/models.py:128
msgid "filter from"
msgstr "filtrer l'expéditeur"
#: paperless_mail/models.py:122
#: paperless_mail/models.py:131
msgid "filter subject"
msgstr "filtrer le sujet"
#: paperless_mail/models.py:125
#: paperless_mail/models.py:134
msgid "filter body"
msgstr "filtrer le corps du message"
#: paperless_mail/models.py:129
#: paperless_mail/models.py:138
msgid "filter attachment filename"
msgstr "filtrer le nom de fichier de la pièce jointe"
#: paperless_mail/models.py:140
msgid ""
"Only consume documents which entirely match this filename if specified. "
"Wildcards such as *.pdf or *invoice* are allowed. Case insensitive."
msgstr ""
"Ne traiter que les documents correspondant intégralement à ce nom de fichier"
" s'il est spécifié. Les jokers tels que *.pdf ou *facture* sont autorisés. "
"La casse n'est pas prise en compte."
#: paperless_mail/models.py:146
msgid "maximum age"
msgstr "âge maximum"
#: paperless_mail/models.py:131
#: paperless_mail/models.py:148
msgid "Specified in days."
msgstr "En jours."
#: paperless_mail/models.py:134
#: paperless_mail/models.py:151
msgid "attachment type"
msgstr "type de pièce jointe"
#: paperless_mail/models.py:154
msgid ""
"Inline attachments include embedded images, so it's best to combine this "
"option with a filename filter."
msgstr ""
"Les pièces jointes en ligne comprennent les images intégrées, il est donc "
"préférable de combiner cette option avec un filtre de nom de fichier."
#: paperless_mail/models.py:159
msgid "action"
msgstr "action"
#: paperless_mail/models.py:140
#: paperless_mail/models.py:165
msgid "action parameter"
msgstr "paramètre d'action"
#: paperless_mail/models.py:142
#: paperless_mail/models.py:167
msgid ""
"Additional parameter for the action selected above, i.e., the target folder "
"of the move to folder action."
@@ -548,22 +631,22 @@ msgstr ""
"Paramètre supplémentaire pour l'action sélectionnée ci-dessus, par exemple "
"le dossier cible de l'action de déplacement vers un dossier."
#: paperless_mail/models.py:148
#: paperless_mail/models.py:173
msgid "assign title from"
msgstr "affecter le titre depuis"
#: paperless_mail/models.py:158
#: paperless_mail/models.py:183
msgid "assign this tag"
msgstr "affecter cette étiquette"
#: paperless_mail/models.py:166
#: paperless_mail/models.py:191
msgid "assign this document type"
msgstr "affecter ce type de document"
#: paperless_mail/models.py:170
#: paperless_mail/models.py:195
msgid "assign correspondent from"
msgstr "affecter le correspondant depuis"
#: paperless_mail/models.py:180
#: paperless_mail/models.py:205
msgid "assign this correspondent"
msgstr "affecter ce correspondant"

171
src/locale/nl_NL/LC_MESSAGES/django.po
View File

@@ -4,17 +4,17 @@
# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
#
# Translators:
# Ben Zweekhorst <bzweekhorst@gmail.com>, 2021
# J V <bugs.github@dwarfy.be>, 2021
# Ben <bzweekhorst@gmail.com>, 2021
# Jo Vandeginste <jo.vandeginste@gmail.com>, 2021
#
#, fuzzy
msgid ""
msgstr ""
"Project-Id-Version: PACKAGE VERSION\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2021-01-02 00:26+0000\n"
"POT-Creation-Date: 2021-01-10 21:41+0000\n"
"PO-Revision-Date: 2020-12-30 19:27+0000\n"
"Last-Translator: J V <bugs.github@dwarfy.be>, 2021\n"
"Last-Translator: Jo Vandeginste <jo.vandeginste@gmail.com>, 2021\n"
"Language-Team: Dutch (Netherlands) (https://www.transifex.com/paperless/teams/115905/nl_NL/)\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
@@ -44,14 +44,14 @@ msgstr "Reguliere expressie"
#: documents/models.py:36
msgid "Fuzzy word"
msgstr "Vaag woord"
msgstr "Gelijkaardig woord"
#: documents/models.py:37
msgid "Automatic"
msgstr "Automatisch"
#: documents/models.py:41 documents/models.py:354 paperless_mail/models.py:25
#: paperless_mail/models.py:100
#: paperless_mail/models.py:109
msgid "name"
msgstr "naam"
@@ -93,11 +93,11 @@ msgstr ""
#: documents/models.py:114
msgid "tag"
msgstr "tag"
msgstr "etiket"
#: documents/models.py:115 documents/models.py:171
msgid "tags"
msgstr "tags"
msgstr "etiketten"
#: documents/models.py:121 documents/models.py:153
msgid "document type"
@@ -285,11 +285,11 @@ msgstr "zit in \"Postvak in\""
#: documents/models.py:380
msgid "has tag"
msgstr "heeft tag"
msgstr "heeft etiket"
#: documents/models.py:381
msgid "has any tag"
msgstr "heeft elke tag"
msgstr "heeft één van de etiketten"
#: documents/models.py:382
msgid "created before"
@@ -329,7 +329,7 @@ msgstr "gewijzigd na"
#: documents/models.py:391
msgid "does not have tag"
msgstr "heeft geen tag"
msgstr "heeft geen etiket"
#: documents/models.py:402
msgid "rule type"
@@ -347,33 +347,81 @@ msgstr "filterregel"
msgid "filter rules"
msgstr "filterregels"
#: paperless/settings.py:254
#: documents/templates/index.html:20
msgid "Paperless-ng is loading..."
msgstr "Paperless-ng is aan het laden..."
#: documents/templates/registration/logged_out.html:13
msgid "Paperless-ng signed out"
msgstr "Paperless-ng - afmelden"
#: documents/templates/registration/logged_out.html:41
msgid "You have been successfully logged out. Bye!"
msgstr "Je bent nu afgemeld. Tot later!"
#: documents/templates/registration/logged_out.html:42
msgid "Sign in again"
msgstr "Meld je opnieuw aan"
#: documents/templates/registration/login.html:13
msgid "Paperless-ng sign in"
msgstr "Paperless-ng - aanmelden"
#: documents/templates/registration/login.html:42
msgid "Please sign in."
msgstr "Gelieve aan te melden."
#: documents/templates/registration/login.html:45
msgid "Your username and password didn't match. Please try again."
msgstr "Je gebruikersnaam en wachtwoord komen niet overeen. Probeer opnieuw."
#: documents/templates/registration/login.html:48
msgid "Username"
msgstr "Gebruikersnaam"
#: documents/templates/registration/login.html:49
msgid "Password"
msgstr "Wachtwoord"
#: documents/templates/registration/login.html:54
msgid "Sign in"
msgstr "Aanmelden"
#: paperless/settings.py:268
msgid "English"
msgstr "Engels"
#: paperless/settings.py:255
#: paperless/settings.py:269
msgid "German"
msgstr "Duits"
#: paperless/settings.py:270
msgid "Dutch"
msgstr "Nederlands"
#: paperless/settings.py:271
msgid "French"
msgstr "Frans"
#: paperless/urls.py:108
msgid "Paperless-ng administration"
msgstr "Paperless-ng administratie"
#: paperless_mail/admin.py:24
#: paperless_mail/admin.py:25
msgid "Filter"
msgstr "Filter"
#: paperless_mail/admin.py:26
#: paperless_mail/admin.py:27
msgid ""
"Paperless will only process mails that match ALL of the filters given below."
msgstr ""
"Paperless verwerkt alleen e-mails die voldoen aan ALLE onderstaande filters."
#: paperless_mail/admin.py:34
#: paperless_mail/admin.py:37
msgid "Actions"
msgstr "Acties"
#: paperless_mail/admin.py:36
#: paperless_mail/admin.py:39
msgid ""
"The action applied to the mail. This action is only performed when documents"
" were consumed from the mail. Mails without attachments will remain entirely"
@@ -383,11 +431,11 @@ msgstr ""
"wanneer documenten verwerkt werden uit de mail. Mails zonder bijlage blijven"
" onaangeroerd."
#: paperless_mail/admin.py:43
#: paperless_mail/admin.py:46
msgid "Metadata"
msgstr "Metadata"
#: paperless_mail/admin.py:45
#: paperless_mail/admin.py:48
msgid ""
"Assign metadata to documents consumed from this rule automatically. If you "
"do not assign tags, types or correspondents here, paperless will still "
@@ -457,87 +505,120 @@ msgstr "email-regel"
msgid "mail rules"
msgstr "email-regels"
#: paperless_mail/models.py:69
#: paperless_mail/models.py:67
msgid "Only process attachments."
msgstr "Alleen bijlagen verwerken"
#: paperless_mail/models.py:68
msgid "Process all files, including 'inline' attachments."
msgstr "Verwerk alle bestanden, inclusief 'inline' bijlagen."
#: paperless_mail/models.py:78
msgid "Mark as read, don't process read mails"
msgstr "Markeer als gelezen, verwerk geen gelezen mails"
#: paperless_mail/models.py:70
#: paperless_mail/models.py:79
msgid "Flag the mail, don't process flagged mails"
msgstr "Markeer de mail, verwerk geen mails met markering"
#: paperless_mail/models.py:71
#: paperless_mail/models.py:80
msgid "Move to specified folder"
msgstr "Verplaats naar gegeven map"
#: paperless_mail/models.py:72
#: paperless_mail/models.py:81
msgid "Delete"
msgstr "Verwijder"
#: paperless_mail/models.py:79
#: paperless_mail/models.py:88
msgid "Use subject as title"
msgstr "Gebruik onderwerp als titel"
#: paperless_mail/models.py:80
#: paperless_mail/models.py:89
msgid "Use attachment filename as title"
msgstr "Gebruik naam van bijlage als titel"
#: paperless_mail/models.py:90
#: paperless_mail/models.py:99
msgid "Do not assign a correspondent"
msgstr "Wijs geen correspondent toe"
#: paperless_mail/models.py:92
#: paperless_mail/models.py:101
msgid "Use mail address"
msgstr "Gebruik het email-adres"
#: paperless_mail/models.py:94
#: paperless_mail/models.py:103
msgid "Use name (or mail address if not available)"
msgstr "Gebruik de naam, en anders het email-adres"
#: paperless_mail/models.py:96
#: paperless_mail/models.py:105
msgid "Use correspondent selected below"
msgstr "Gebruik de hieronder aangeduide correspondent"
#: paperless_mail/models.py:104
#: paperless_mail/models.py:113
msgid "order"
msgstr "volgorde"
#: paperless_mail/models.py:111
#: paperless_mail/models.py:120
msgid "account"
msgstr "account"
#: paperless_mail/models.py:115
#: paperless_mail/models.py:124
msgid "folder"
msgstr "map"
#: paperless_mail/models.py:119
#: paperless_mail/models.py:128
msgid "filter from"
msgstr "filter afzender"
#: paperless_mail/models.py:122
#: paperless_mail/models.py:131
msgid "filter subject"
msgstr "filter onderwerp"
#: paperless_mail/models.py:125
#: paperless_mail/models.py:134
msgid "filter body"
msgstr "filter inhoud"
#: paperless_mail/models.py:129
#: paperless_mail/models.py:138
msgid "filter attachment filename"
msgstr "Filter bestandsnaam van bijlage"
#: paperless_mail/models.py:140
msgid ""
"Only consume documents which entirely match this filename if specified. "
"Wildcards such as *.pdf or *invoice* are allowed. Case insensitive."
msgstr ""
"Alleen documenten verwerken die volledig overeenkomen, indien aangegeven. Je"
" kunt jokertekens gebruiken, zoals *.pdf of *factuur*. Dit is niet "
"hoofdlettergevoelig."
#: paperless_mail/models.py:146
msgid "maximum age"
msgstr "Maximale leeftijd"
#: paperless_mail/models.py:131
#: paperless_mail/models.py:148
msgid "Specified in days."
msgstr "Aangegeven in dagen"
#: paperless_mail/models.py:134
#: paperless_mail/models.py:151
msgid "attachment type"
msgstr "Type bijlage"
#: paperless_mail/models.py:154
msgid ""
"Inline attachments include embedded images, so it's best to combine this "
"option with a filename filter."
msgstr ""
"\"Inline\" bijlagen bevatten vaak ook afbeeldingen. In dit geval valt het "
"aan te raden om ook een filter voor de bestandsnaam op te geven."
#: paperless_mail/models.py:159
msgid "action"
msgstr "actie"
#: paperless_mail/models.py:140
#: paperless_mail/models.py:165
msgid "action parameter"
msgstr "actie parameters"
#: paperless_mail/models.py:142
#: paperless_mail/models.py:167
msgid ""
"Additional parameter for the action selected above, i.e., the target folder "
"of the move to folder action."
@@ -545,22 +626,22 @@ msgstr ""
"Extra parameters voor de hierboven gekozen actie, met andere woorden: de "
"bestemmingsmap voor de verplaats-actie."
#: paperless_mail/models.py:148
#: paperless_mail/models.py:173
msgid "assign title from"
msgstr "wijs titel toe van"
#: paperless_mail/models.py:158
#: paperless_mail/models.py:183
msgid "assign this tag"
msgstr "wijs dit etiket toe"
#: paperless_mail/models.py:166
#: paperless_mail/models.py:191
msgid "assign this document type"
msgstr "wijs dit documenttype toe"
#: paperless_mail/models.py:170
#: paperless_mail/models.py:195
msgid "assign correspondent from"
msgstr "wijs correspondent toe van"
#: paperless_mail/models.py:180
#: paperless_mail/models.py:205
msgid "assign this correspondent"
msgstr "wijs deze correspondent toe"

9
src/paperless/auth.py
View File

@@ -2,6 +2,7 @@ from django.conf import settings
from django.contrib.auth.models import User
from django.utils.deprecation import MiddlewareMixin
from rest_framework import authentication
from django.contrib.auth.middleware import RemoteUserMiddleware
class AutoLoginMiddleware(MiddlewareMixin):
@@ -26,3 +27,11 @@ class AngularApiAuthenticationOverride(authentication.BaseAuthentication):
return (user, None)
else:
return None
class HttpRemoteUserMiddleware(RemoteUserMiddleware):
""" This class allows authentication via HTTP_REMOTE_USER which is set for
example by certain SSO applications.
"""
header = 'HTTP_REMOTE_USER'

47
src/paperless/settings.py
View File

@@ -4,6 +4,7 @@ import multiprocessing
import os
import re
import dateparser
from dotenv import load_dotenv
from django.utils.translation import gettext_lazy as _
@@ -89,7 +90,6 @@ INSTALLED_APPS = [
"documents.apps.DocumentsConfig",
"paperless_tesseract.apps.PaperlessTesseractConfig",
"paperless_text.apps.PaperlessTextConfig",
"paperless_tika.apps.PaperlessTikaConfig",
"paperless_mail.apps.PaperlessMailConfig",
"django.contrib.admin",
@@ -177,6 +177,25 @@ if AUTO_LOGIN_USERNAME:
# regular login in case the provided user does not exist.
MIDDLEWARE.insert(_index+1, 'paperless.auth.AutoLoginMiddleware')
ENABLE_HTTP_REMOTE_USER = __get_boolean("PAPERLESS_ENABLE_HTTP_REMOTE_USER")
if ENABLE_HTTP_REMOTE_USER:
MIDDLEWARE.append(
'paperless.auth.HttpRemoteUserMiddleware'
)
AUTHENTICATION_BACKENDS = [
'django.contrib.auth.backends.RemoteUserBackend',
'django.contrib.auth.backends.ModelBackend'
]
REST_FRAMEWORK['DEFAULT_AUTHENTICATION_CLASSES'].append(
'rest_framework.authentication.RemoteUserAuthentication'
)
# X-Frame options for embedded PDF display:
if DEBUG:
X_FRAME_OPTIONS = 'ANY'
else:
X_FRAME_OPTIONS = 'SAMEORIGIN'
# We allow CORS from localhost:8080
CORS_ALLOWED_ORIGINS = tuple(os.getenv("PAPERLESS_CORS_ALLOWED_HOSTS", "http://localhost:8000").split(","))
@@ -252,6 +271,7 @@ if os.getenv("PAPERLESS_DBHOST"):
"NAME": os.getenv("PAPERLESS_DBNAME", "paperless"),
"USER": os.getenv("PAPERLESS_DBUSER", "paperless"),
"PASSWORD": os.getenv("PAPERLESS_DBPASS", "paperless"),
'OPTIONS': {'sslmode': os.getenv("PAPERLESS_DBSSLMODE", "prefer")},
}
if os.getenv("PAPERLESS_DBPORT"):
DATABASES["default"]["PORT"] = os.getenv("PAPERLESS_DBPORT")
@@ -306,7 +326,7 @@ LOGGING = {
"class": "documents.loggers.PaperlessHandler",
},
"console": {
"level": "INFO",
"level": "DEBUG" if DEBUG else "INFO",
"class": "logging.StreamHandler",
"formatter": "verbose",
}
@@ -343,10 +363,13 @@ LOGGING = {
# Favors threads per worker on smaller systems and never exceeds cpu_count()
# in total.
def default_task_workers():
# always leave one core open
available_cores = max(multiprocessing.cpu_count() - 1, 1)
try:
return max(
math.floor(math.sqrt(multiprocessing.cpu_count())),
math.floor(math.sqrt(available_cores)),
1
)
except NotImplementedError:
@@ -363,17 +386,19 @@ Q_CLUSTER = {
}
def default_threads_per_worker():
def default_threads_per_worker(task_workers):
# always leave one core open
available_cores = max(multiprocessing.cpu_count() - 1, 1)
try:
return max(
math.floor(multiprocessing.cpu_count() / TASK_WORKERS),
math.floor(available_cores / task_workers),
1
)
except NotImplementedError:
return 1
THREADS_PER_WORKER = os.getenv("PAPERLESS_THREADS_PER_WORKER", default_threads_per_worker())
THREADS_PER_WORKER = os.getenv("PAPERLESS_THREADS_PER_WORKER", default_threads_per_worker(TASK_WORKERS))
###############################################################################
# Paperless Specific Settings #
@@ -458,3 +483,13 @@ PAPERLESS_TIKA_ENDPOINT = os.getenv("PAPERLESS_TIKA_ENDPOINT", "http://localhost
PAPERLESS_TIKA_GOTENBERG_ENDPOINT = os.getenv(
"PAPERLESS_TIKA_GOTENBERG_ENDPOINT", "http://localhost:3000"
)
if PAPERLESS_TIKA_ENABLED:
INSTALLED_APPS.append("paperless_tika.apps.PaperlessTikaConfig")
# List dates that should be ignored when trying to parse date from document text
IGNORE_DATES = set()
for s in os.getenv("PAPERLESS_IGNORE_DATES", "").split(","):
d = dateparser.parse(s)
if d:
IGNORE_DATES.add(d.date())

2
src/paperless/version.py
View File

@@ -1 +1 @@
__version__ = (0, 9, 11)
__version__ = (1, 0, 0)

5
src/paperless_mail/admin.py
View File

@@ -12,6 +12,7 @@ class MailAccountAdmin(admin.ModelAdmin):
class MailRuleAdmin(admin.ModelAdmin):
radio_fields = {
"attachment_type": admin.VERTICAL,
"action": admin.VERTICAL,
"assign_title_from": admin.VERTICAL,
"assign_correspondent_from": admin.VERTICAL
@@ -29,7 +30,9 @@ class MailRuleAdmin(admin.ModelAdmin):
('filter_from',
'filter_subject',
'filter_body',
'maximum_age')
'filter_attachment_filename',
'maximum_age',
'attachment_type')
}),
(_("Actions"), {
'description':

9
src/paperless_mail/mail.py
View File

@@ -1,6 +1,7 @@
import os
import tempfile
from datetime import timedelta, date
from fnmatch import fnmatch
import magic
import pathvalidate
@@ -198,7 +199,7 @@ class MailAccountHandler(LoggingMixin):
try:
messages = M.fetch(criteria=AND(**criterias),
mark_seen=False)
mark_seen=False, charset='UTF-8')
except Exception:
raise MailError(
f"Rule {rule}: Error while fetching folder {rule.folder}")
@@ -263,7 +264,7 @@ class MailAccountHandler(LoggingMixin):
for att in message.attachments:
if not att.content_disposition == "attachment":
if not att.content_disposition == "attachment" and rule.attachment_type == MailRule.ATTACHMENT_TYPE_ATTACHMENTS_ONLY: # NOQA: E501
self.log(
'debug',
f"Rule {rule}: "
@@ -271,6 +272,10 @@ class MailAccountHandler(LoggingMixin):
f"with content disposition {att.content_disposition}")
continue
if rule.filter_attachment_filename:
if not fnmatch(att.filename, rule.filter_attachment_filename):
continue
title = self.get_title(message, att, rule)
# don't trust the content type of the attachment. Could be

23
src/paperless_mail/migrations/0007_auto_20210106_0138.py Normal file
View File

@@ -0,0 +1,23 @@
# Generated by Django 3.1.5 on 2021-01-06 01:38
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('paperless_mail', '0006_auto_20210101_2340'),
]
operations = [
migrations.AddField(
model_name='mailrule',
name='attachment_type',
field=models.PositiveIntegerField(choices=[(1, 'Only process attachments.'), (2, "Process all files, including 'inline' attachments.")], default=1, help_text="Inline attachments include embedded images, so it's best to combine this option with a filename filter.", verbose_name='attachment type'),
),
migrations.AddField(
model_name='mailrule',
name='filter_attachment_filename',
field=models.CharField(blank=True, help_text='Only consume documents which entirely match this filename if specified. Wildcards such as *.pdf or *invoice* are allowed. Case insensitive.', max_length=256, null=True, verbose_name='filter attachment filename'),
),
]

25
src/paperless_mail/models.py
View File

@@ -60,6 +60,15 @@ class MailRule(models.Model):
verbose_name = _("mail rule")
verbose_name_plural = _("mail rules")
ATTACHMENT_TYPE_ATTACHMENTS_ONLY = 1
ATTACHMENT_TYPE_EVERYTHING = 2
ATTACHMENT_TYPES = (
(ATTACHMENT_TYPE_ATTACHMENTS_ONLY, _("Only process attachments.")),
(ATTACHMENT_TYPE_EVERYTHING, _("Process all files, including 'inline' "
"attachments."))
)
ACTION_DELETE = 1
ACTION_MOVE = 2
ACTION_MARK_READ = 3
@@ -125,11 +134,27 @@ class MailRule(models.Model):
_("filter body"),
max_length=256, null=True, blank=True)
filter_attachment_filename = models.CharField(
_("filter attachment filename"),
max_length=256, null=True, blank=True,
help_text=_("Only consume documents which entirely match this "
"filename if specified. Wildcards such as *.pdf or "
"*invoice* are allowed. Case insensitive.")
)
maximum_age = models.PositiveIntegerField(
_("maximum age"),
default=30,
help_text=_("Specified in days."))
attachment_type = models.PositiveIntegerField(
_("attachment type"),
choices=ATTACHMENT_TYPES,
default=ATTACHMENT_TYPE_ATTACHMENTS_ONLY,
help_text=_("Inline attachments include embedded images, so it's best "
"to combine this option with a filename filter.")
)
action = models.PositiveIntegerField(
_("action"),
choices=ACTIONS,

55
src/paperless_mail/tests/test_mail.py
View File

@@ -1,3 +1,4 @@
import os
import uuid
from collections import namedtuple
from typing import ContextManager
@@ -9,6 +10,7 @@ from django.test import TestCase
from imap_tools import MailMessageFlags, MailboxFolderSelectError
from documents.models import Correspondent
from documents.tests.utils import DirectoriesMixin
from paperless_mail import tasks
from paperless_mail.mail import MailError, MailAccountHandler
from paperless_mail.models import MailRule, MailAccount
@@ -41,7 +43,7 @@ class BogusMailBox(ContextManager):
folder = BogusFolderManager()
def fetch(self, criteria, mark_seen):
def fetch(self, criteria, mark_seen, charset=""):
msg = self.messages
criteria = str(criteria).strip('()').split(" ")
@@ -130,7 +132,7 @@ def fake_magic_from_buffer(buffer, mime=False):
@mock.patch('paperless_mail.mail.magic.from_buffer', fake_magic_from_buffer)
class TestMail(TestCase):
class TestMail(DirectoriesMixin, TestCase):
def setUp(self):
patcher = mock.patch('paperless_mail.mail.MailBox')
@@ -146,6 +148,7 @@ class TestMail(TestCase):
self.reset_bogus_mailbox()
self.mail_account_handler = MailAccountHandler()
super(TestMail, self).setUp()
def reset_bogus_mailbox(self):
self.bogus_mailbox.messages = []
@@ -220,9 +223,13 @@ class TestMail(TestCase):
args1, kwargs1 = self.async_task.call_args_list[0]
args2, kwargs2 = self.async_task.call_args_list[1]
self.assertTrue(os.path.isfile(kwargs1['path']), kwargs1['path'])
self.assertEqual(kwargs1['override_title'], "file_0")
self.assertEqual(kwargs1['override_filename'], "file_0.pdf")
self.assertTrue(os.path.isfile(kwargs2['path']), kwargs1['path'])
self.assertEqual(kwargs2['override_title'], "file_1")
self.assertEqual(kwargs2['override_filename'], "file_1.pdf")
@@ -253,6 +260,7 @@ class TestMail(TestCase):
self.assertEqual(self.async_task.call_count, 1)
args, kwargs = self.async_task.call_args
self.assertTrue(os.path.isfile(kwargs['path']), kwargs['path'])
self.assertEqual(kwargs['override_filename'], "f1.pdf")
def test_handle_disposition(self):
@@ -273,6 +281,49 @@ class TestMail(TestCase):
args, kwargs = self.async_task.call_args
self.assertEqual(kwargs['override_filename'], "f2.pdf")
def test_handle_inline_files(self):
message = create_message()
message.attachments = [
create_attachment(filename="f1.pdf", content_disposition='inline'),
create_attachment(filename="f2.pdf", content_disposition='attachment')
]
account = MailAccount()
rule = MailRule(assign_title_from=MailRule.TITLE_FROM_FILENAME, account=account, attachment_type=MailRule.ATTACHMENT_TYPE_EVERYTHING)
result = self.mail_account_handler.handle_message(message, rule)
self.assertEqual(result, 2)
self.assertEqual(self.async_task.call_count, 2)
def test_filename_filter(self):
message = create_message()
message.attachments = [
create_attachment(filename="f1.pdf"),
create_attachment(filename="f2.pdf"),
create_attachment(filename="f3.pdf"),
create_attachment(filename="f2.png"),
]
tests = [
("*.pdf", ["f1.pdf", "f2.pdf", "f3.pdf"]),
("f1.pdf", ["f1.pdf"]),
("f1", []),
("*", ["f1.pdf", "f2.pdf", "f3.pdf", "f2.png"]),
("*.png", ["f2.png"]),
]
for (pattern, matches) in tests:
self.async_task.reset_mock()
account = MailAccount()
rule = MailRule(assign_title_from=MailRule.TITLE_FROM_FILENAME, account=account, filter_attachment_filename=pattern)
result = self.mail_account_handler.handle_message(message, rule)
self.assertEqual(result, len(matches))
filenames = [a[1]['override_filename'] for a in self.async_task.call_args_list]
self.assertCountEqual(filenames, matches)
def test_handle_mail_account_mark_read(self):
account = MailAccount.objects.create(name="test", imap_server="", username="admin", password="secret")