paradizelost/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2025-09-12 21:35:40 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev' into feature-created-date

Browse Source
This commit is contained in:
shamoon
2022-05-26 14:28:33 -07:00
committed by GitHub
parent 88b7f8ac1e a4927477fb
commit 9eee37bc68
118 changed files with 5276 additions and 3044 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/documents/admin.py
View File

@@ -5,6 +5,7 @@ from .models import Document
from .models import DocumentType
from .models import SavedView
from .models import SavedViewFilterRule
from .models import StoragePath
from .models import Tag
@@ -100,8 +101,19 @@ class SavedViewAdmin(admin.ModelAdmin):
inlines = [RuleInline]
class StoragePathInline(admin.TabularInline):
model = StoragePath
class StoragePathAdmin(admin.ModelAdmin):
list_display = ("name", "path", "match", "matching_algorithm")
list_filter = ("path", "matching_algorithm")
list_editable = ("path", "match", "matching_algorithm")
admin.site.register(Correspondent, CorrespondentAdmin)
admin.site.register(Tag, TagAdmin)
admin.site.register(DocumentType, DocumentTypeAdmin)
admin.site.register(Document, DocumentAdmin)
admin.site.register(SavedView, SavedViewAdmin)
admin.site.register(StoragePath, StoragePathAdmin)

2
src/documents/apps.py
View File

@@ -16,6 +16,7 @@ class DocumentsConfig(AppConfig):
set_correspondent,
set_document_type,
set_tags,
set_storage_path,
add_to_index,
)
@@ -23,6 +24,7 @@ class DocumentsConfig(AppConfig):
document_consumption_finished.connect(set_correspondent)
document_consumption_finished.connect(set_document_type)
document_consumption_finished.connect(set_tags)
document_consumption_finished.connect(set_storage_path)
document_consumption_finished.connect(set_log_entry)
document_consumption_finished.connect(add_to_index)

19
src/documents/bulk_edit.py
View File

@@ -5,6 +5,7 @@ from django_q.tasks import async_task
from documents.models import Correspondent
from documents.models import Document
from documents.models import DocumentType
from documents.models import StoragePath
def set_correspondent(doc_ids, correspondent):
@@ -20,6 +21,24 @@ def set_correspondent(doc_ids, correspondent):
return "OK"
def set_storage_path(doc_ids, storage_path):
if storage_path:
storage_path = StoragePath.objects.get(id=storage_path)
qs = Document.objects.filter(
Q(id__in=doc_ids) & ~Q(storage_path=storage_path),
)
affected_docs = [doc.id for doc in qs]
qs.update(storage_path=storage_path)
async_task(
"documents.tasks.bulk_update_documents",
document_ids=affected_docs,
)
return "OK"
def set_document_type(doc_ids, document_type):
if document_type:
document_type = DocumentType.objects.get(id=document_type)

45
src/documents/classifier.py
View File

@@ -59,8 +59,8 @@ def load_classifier():
class DocumentClassifier:
# v7 - Updated scikit-learn package version
FORMAT_VERSION = 7
# v8 - Added storage path classifier
FORMAT_VERSION = 8
def __init__(self):
# hash of the training data. used to prevent re-training when the
@@ -72,6 +72,7 @@ class DocumentClassifier:
self.tags_classifier = None
self.correspondent_classifier = None
self.document_type_classifier = None
self.storage_path_classifier = None
def load(self):
with open(settings.MODEL_FILE, "rb") as f:
@@ -90,6 +91,7 @@ class DocumentClassifier:
self.tags_classifier = pickle.load(f)
self.correspondent_classifier = pickle.load(f)
self.document_type_classifier = pickle.load(f)
self.storage_path_classifier = pickle.load(f)
except Exception:
raise ClassifierModelCorruptError()
@@ -107,6 +109,7 @@ class DocumentClassifier:
pickle.dump(self.tags_classifier, f)
pickle.dump(self.correspondent_classifier, f)
pickle.dump(self.document_type_classifier, f)
pickle.dump(self.storage_path_classifier, f)
if os.path.isfile(target_file):
os.unlink(target_file)
@@ -118,6 +121,7 @@ class DocumentClassifier:
labels_tags = list()
labels_correspondent = list()
labels_document_type = list()
labels_storage_path = list()
# Step 1: Extract and preprocess training data from the database.
logger.debug("Gathering data from database...")
@@ -153,6 +157,13 @@ class DocumentClassifier:
m.update(tag.to_bytes(4, "little", signed=True))
labels_tags.append(tags)
y = -1
sd = doc.storage_path
if sd and sd.matching_algorithm == MatchingModel.MATCH_AUTO:
y = sd.pk
m.update(y.to_bytes(4, "little", signed=True))
labels_storage_path.append(y)
if not data:
raise ValueError("No training data available.")
@@ -172,14 +183,16 @@ class DocumentClassifier:
# it usually is.
num_correspondents = len(set(labels_correspondent) | {-1}) - 1
num_document_types = len(set(labels_document_type) | {-1}) - 1
num_storage_paths = len(set(labels_storage_path) | {-1}) - 1
logger.debug(
"{} documents, {} tag(s), {} correspondent(s), "
"{} document type(s).".format(
"{} document type(s). {} storage path(es)".format(
len(data),
num_tags,
num_correspondents,
num_document_types,
num_storage_paths,
),
)
@@ -242,6 +255,21 @@ class DocumentClassifier:
"classifier.",
)
if num_storage_paths > 0:
logger.debug(
"Training storage paths classifier...",
)
self.storage_path_classifier = MLPClassifier(tol=0.01)
self.storage_path_classifier.fit(
data_vectorized,
labels_storage_path,
)
else:
self.storage_path_classifier = None
logger.debug(
"There are no storage paths. Not training storage path classifier.",
)
self.data_hash = new_data_hash
return True
@@ -288,3 +316,14 @@ class DocumentClassifier:
return []
else:
return []
def predict_storage_path(self, content):
if self.storage_path_classifier:
X = self.data_vectorizer.transform([preprocess_content(content)])
storage_path_id = self.storage_path_classifier.predict(X)
if storage_path_id != -1:
return storage_path_id
else:
return None
else:
return None

36
src/documents/file_handling.py
View File

@@ -128,13 +128,26 @@ def generate_unique_filename(doc, archive_filename=False):
def generate_filename(doc, counter=0, append_gpg=True, archive_filename=False):
path = ""
filename_format = settings.FILENAME_FORMAT
try:
if settings.PAPERLESS_FILENAME_FORMAT is not None:
tags = defaultdictNoStr(lambda: slugify(None), many_to_dictionary(doc.tags))
if doc.storage_path is not None:
logger.debug(
f"Document has storage_path {doc.storage_path.id} "
f"({doc.storage_path.path}) set",
)
filename_format = doc.storage_path.path
if filename_format is not None:
tags = defaultdictNoStr(
lambda: slugify(None),
many_to_dictionary(doc.tags),
)
tag_list = pathvalidate.sanitize_filename(
",".join(sorted(tag.name for tag in doc.tags.all())),
",".join(
sorted(tag.name for tag in doc.tags.all()),
),
replacement_text="-",
)
@@ -144,7 +157,7 @@ def generate_filename(doc, counter=0, append_gpg=True, archive_filename=False):
replacement_text="-",
)
else:
correspondent = "none"
correspondent = "-none-"
if doc.document_type:
document_type = pathvalidate.sanitize_filename(
@@ -152,18 +165,18 @@ def generate_filename(doc, counter=0, append_gpg=True, archive_filename=False):
replacement_text="-",
)
else:
document_type = "none"
document_type = "-none-"
if doc.archive_serial_number:
asn = str(doc.archive_serial_number)
else:
asn = "none"
asn = "-none-"
# Convert UTC database date to localized date
local_added = timezone.localdate(doc.added)
local_created = timezone.localdate(doc.created)
path = settings.PAPERLESS_FILENAME_FORMAT.format(
path = filename_format.format(
title=pathvalidate.sanitize_filename(doc.title, replacement_text="-"),
correspondent=correspondent,
document_type=document_type,
@@ -180,12 +193,17 @@ def generate_filename(doc, counter=0, append_gpg=True, archive_filename=False):
tag_list=tag_list,
).strip()
if settings.FILENAME_FORMAT_REMOVE_NONE:
path = path.replace("-none-/", "") # remove empty directories
path = path.replace(" -none-", "") # remove when spaced, with space
path = path.replace("-none-", "") # remove rest of the occurences
path = path.replace("-none-", "none") # backward compatibility
path = path.strip(os.sep)
except (ValueError, KeyError, IndexError):
logger.warning(
f"Invalid PAPERLESS_FILENAME_FORMAT: "
f"{settings.PAPERLESS_FILENAME_FORMAT}, falling back to default",
f"Invalid filename_format '{filename_format}', falling back to default",
)
counter_str = f"_{counter:02}" if counter else ""

13
src/documents/filters.py
View File

@@ -7,6 +7,7 @@ from .models import Correspondent
from .models import Document
from .models import DocumentType
from .models import Log
from .models import StoragePath
from .models import Tag
CHAR_KWARGS = ["istartswith", "iendswith", "icontains", "iexact"]
@@ -114,6 +115,9 @@ class DocumentFilterSet(FilterSet):
"document_type": ["isnull"],
"document_type__id": ID_KWARGS,
"document_type__name": CHAR_KWARGS,
"storage_path": ["isnull"],
"storage_path__id": ID_KWARGS,
"storage_path__name": CHAR_KWARGS,
}
@@ -121,3 +125,12 @@ class LogFilterSet(FilterSet):
class Meta:
model = Log
fields = {"level": INT_KWARGS, "created": DATE_KWARGS, "group": ID_KWARGS}
class StoragePathFilterSet(FilterSet):
class Meta:
model = StoragePath
fields = {
"name": CHAR_KWARGS,
"path": CHAR_KWARGS,
}

11
src/documents/index.py
View File

@@ -46,6 +46,9 @@ def get_schema():
created=DATETIME(sortable=True),
modified=DATETIME(sortable=True),
added=DATETIME(sortable=True),
path=TEXT(sortable=True),
path_id=NUMERIC(),
has_path=BOOLEAN(),
)
@@ -104,6 +107,9 @@ def update_document(writer, doc):
added=doc.added,
asn=doc.archive_serial_number,
modified=doc.modified,
path=doc.storage_path.name if doc.storage_path else None,
path_id=doc.storage_path.id if doc.storage_path else None,
has_path=doc.storage_path is not None,
)
@@ -157,6 +163,11 @@ class DelayedQuery:
criterias.append(query.DateRange("added", start=isoparse(v), end=None))
elif k == "added__date__lt":
criterias.append(query.DateRange("added", start=None, end=isoparse(v)))
elif k == "storage_path__id":
criterias.append(query.Term("path_id", v))
elif k == "storage_path__isnull":
criterias.append(query.Term("has_path", v == "false"))
if len(criterias) > 0:
return query.And(criterias)
else:

2
src/documents/management/commands/document_archiver.py
View File

@@ -152,4 +152,4 @@ class Command(BaseCommand):
),
)
except KeyboardInterrupt:
self.stdout.write(self.style.NOTICE(("Aborting...")))
self.stdout.write(self.style.NOTICE("Aborting..."))

9
src/documents/management/commands/document_exporter.py
View File

@@ -18,6 +18,7 @@ from documents.models import DocumentType
from documents.models import SavedView
from documents.models import SavedViewFilterRule
from documents.models import Tag
from documents.models import UiSettings
from documents.settings import EXPORTER_ARCHIVE_NAME
from documents.settings import EXPORTER_FILE_NAME
from documents.settings import EXPORTER_THUMBNAIL_NAME
@@ -112,8 +113,8 @@ class Command(BaseCommand):
map(lambda f: os.path.abspath(os.path.join(root, f)), files),
)
# 2. Create manifest, containing all correspondents, types, tags and
# documents
# 2. Create manifest, containing all correspondents, types, tags,
# documents and ui_settings
with transaction.atomic():
manifest = json.loads(
serializers.serialize("json", Correspondent.objects.all()),
@@ -150,6 +151,10 @@ class Command(BaseCommand):
manifest += json.loads(serializers.serialize("json", User.objects.all()))
manifest += json.loads(
serializers.serialize("json", UiSettings.objects.all()),
)
# 3. Export files from each document
for index, document_dict in tqdm.tqdm(
enumerate(document_manifest),

17
src/documents/matching.py
View File

@@ -4,6 +4,7 @@ import re
from documents.models import Correspondent
from documents.models import DocumentType
from documents.models import MatchingModel
from documents.models import StoragePath
from documents.models import Tag
@@ -57,6 +58,22 @@ def match_tags(document, classifier):
)
def match_storage_paths(document, classifier):
if classifier:
pred_id = classifier.predict_storage_path(document.content)
else:
pred_id = None
storage_paths = StoragePath.objects.all()
return list(
filter(
lambda o: matches(o, document) or o.pk == pred_id,
storage_paths,
),
)
def matches(matching_model, document):
search_kwargs = {}

6
src/documents/migrations/1012_fix_archive_files.py
View File

@@ -83,7 +83,7 @@ def generate_filename(doc, counter=0, append_gpg=True, archive_filename=False):
path = ""
try:
if settings.PAPERLESS_FILENAME_FORMAT is not None:
if settings.FILENAME_FORMAT is not None:
tags = defaultdictNoStr(lambda: slugify(None), many_to_dictionary(doc.tags))
tag_list = pathvalidate.sanitize_filename(
@@ -105,7 +105,7 @@ def generate_filename(doc, counter=0, append_gpg=True, archive_filename=False):
else:
document_type = "none"
path = settings.PAPERLESS_FILENAME_FORMAT.format(
path = settings.FILENAME_FORMAT.format(
title=pathvalidate.sanitize_filename(doc.title, replacement_text="-"),
correspondent=correspondent,
document_type=document_type,
@@ -128,7 +128,7 @@ def generate_filename(doc, counter=0, append_gpg=True, archive_filename=False):
except (ValueError, KeyError, IndexError):
logger.warning(
f"Invalid PAPERLESS_FILENAME_FORMAT: "
f"{settings.PAPERLESS_FILENAME_FORMAT}, falling back to default"
f"{settings.FILENAME_FORMAT}, falling back to default"
)
counter_str = f"_{counter:02}" if counter else ""

73
src/documents/migrations/1019_storagepath_document_storage_path.py Normal file
View File

@@ -0,0 +1,73 @@
# Generated by Django 4.0.4 on 2022-05-02 15:56
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
("documents", "1018_alter_savedviewfilterrule_value"),
]
operations = [
migrations.CreateModel(
name="StoragePath",
fields=[
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
(
"name",
models.CharField(max_length=128, unique=True, verbose_name="name"),
),
(
"match",
models.CharField(blank=True, max_length=256, verbose_name="match"),
),
(
"matching_algorithm",
models.PositiveIntegerField(
choices=[
(1, "Any word"),
(2, "All words"),
(3, "Exact match"),
(4, "Regular expression"),
(5, "Fuzzy word"),
(6, "Automatic"),
],
default=1,
verbose_name="matching algorithm",
),
),
(
"is_insensitive",
models.BooleanField(default=True, verbose_name="is insensitive"),
),
("path", models.CharField(max_length=512, verbose_name="path")),
],
options={
"verbose_name": "storage path",
"verbose_name_plural": "storage paths",
"ordering": ("name",),
},
),
migrations.AddField(
model_name="document",
name="storage_path",
field=models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.SET_NULL,
related_name="documents",
to="documents.storagepath",
verbose_name="storage path",
),
),
]

39
src/documents/migrations/1019_uisettings.py Normal file
View File

@@ -0,0 +1,39 @@
# Generated by Django 4.0.4 on 2022-05-07 05:10
from django.conf import settings
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
("documents", "1018_alter_savedviewfilterrule_value"),
]
operations = [
migrations.CreateModel(
name="UiSettings",
fields=[
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("settings", models.JSONField(null=True)),
(
"user",
models.OneToOneField(
on_delete=django.db.models.deletion.CASCADE,
related_name="ui_settings",
to=settings.AUTH_USER_MODEL,
),
),
],
),
]

13
src/documents/migrations/1020_merge_20220518_1839.py Normal file
View File

@@ -0,0 +1,13 @@
# Generated by Django 4.0.4 on 2022-05-18 18:39
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("documents", "1019_storagepath_document_storage_path"),
("documents", "1019_uisettings"),
]
operations = []

35
src/documents/models.py
View File

@@ -83,6 +83,18 @@ class DocumentType(MatchingModel):
verbose_name_plural = _("document types")
class StoragePath(MatchingModel):
path = models.CharField(
_("path"),
max_length=512,
)
class Meta:
ordering = ("name",)
verbose_name = _("storage path")
verbose_name_plural = _("storage paths")
class Document(models.Model):
STORAGE_TYPE_UNENCRYPTED = "unencrypted"
@@ -101,6 +113,15 @@ class Document(models.Model):
verbose_name=_("correspondent"),
)
storage_path = models.ForeignKey(
StoragePath,
blank=True,
null=True,
related_name="documents",
on_delete=models.SET_NULL,
verbose_name=_("storage path"),
)
title = models.CharField(_("title"), max_length=128, blank=True, db_index=True)
document_type = models.ForeignKey(
@@ -469,3 +490,17 @@ class FileInfo:
cls._mangle_property(properties, "created")
cls._mangle_property(properties, "title")
return cls(**properties)
# Extending User Model Using a One-To-One Link
class UiSettings(models.Model):
user = models.OneToOneField(
User,
on_delete=models.CASCADE,
related_name="ui_settings",
)
settings = models.JSONField(null=True)
def __str__(self):
return self.user.username

90
src/documents/serialisers.py
View File

@@ -20,7 +20,9 @@ from .models import DocumentType
from .models import MatchingModel
from .models import SavedView
from .models import SavedViewFilterRule
from .models import StoragePath
from .models import Tag
from .models import UiSettings
from .parsers import is_mime_type_supported
@@ -204,11 +206,17 @@ class DocumentTypeField(serializers.PrimaryKeyRelatedField):
return DocumentType.objects.all()
class StoragePathField(serializers.PrimaryKeyRelatedField):
def get_queryset(self):
return StoragePath.objects.all()
class DocumentSerializer(DynamicFieldsModelSerializer):
correspondent = CorrespondentField(allow_null=True)
tags = TagsField(many=True)
document_type = DocumentTypeField(allow_null=True)
storage_path = StoragePathField(allow_null=True)
original_file_name = SerializerMethodField()
archived_file_name = SerializerMethodField()
@@ -242,6 +250,7 @@ class DocumentSerializer(DynamicFieldsModelSerializer):
"id",
"correspondent",
"document_type",
"storage_path",
"title",
"content",
"tags",
@@ -329,6 +338,7 @@ class BulkEditSerializer(DocumentListSerializer):
choices=[
"set_correspondent",
"set_document_type",
"set_storage_path",
"add_tag",
"remove_tag",
"modify_tags",
@@ -356,6 +366,8 @@ class BulkEditSerializer(DocumentListSerializer):
return bulk_edit.set_correspondent
elif method == "set_document_type":
return bulk_edit.set_document_type
elif method == "set_storage_path":
return bulk_edit.set_storage_path
elif method == "add_tag":
return bulk_edit.add_tag
elif method == "remove_tag":
@@ -402,6 +414,20 @@ class BulkEditSerializer(DocumentListSerializer):
else:
raise serializers.ValidationError("correspondent not specified")
def _validate_storage_path(self, parameters):
if "storage_path" in parameters:
storage_path_id = parameters["storage_path"]
if storage_path_id is None:
return
try:
StoragePath.objects.get(id=storage_path_id)
except StoragePath.DoesNotExist:
raise serializers.ValidationError(
"Storage path does not exist",
)
else:
raise serializers.ValidationError("storage path not specified")
def _validate_parameters_modify_tags(self, parameters):
if "add_tags" in parameters:
self._validate_tag_id_list(parameters["add_tags"], "add_tags")
@@ -426,6 +452,8 @@ class BulkEditSerializer(DocumentListSerializer):
self._validate_parameters_tags(parameters)
elif method == bulk_edit.modify_tags:
self._validate_parameters_modify_tags(parameters)
elif method == bulk_edit.set_storage_path:
self._validate_storage_path(parameters)
return attrs
@@ -525,3 +553,65 @@ class BulkDownloadSerializer(DocumentListSerializer):
"bzip2": zipfile.ZIP_BZIP2,
"lzma": zipfile.ZIP_LZMA,
}[compression]
class StoragePathSerializer(MatchingModelSerializer):
document_count = serializers.IntegerField(read_only=True)
class Meta:
model = StoragePath
fields = (
"id",
"slug",
"name",
"path",
"match",
"matching_algorithm",
"is_insensitive",
"document_count",
)
def validate_path(self, path):
try:
path.format(
title="title",
correspondent="correspondent",
document_type="document_type",
created="created",
created_year="created_year",
created_month="created_month",
created_day="created_day",
added="added",
added_year="added_year",
added_month="added_month",
added_day="added_day",
asn="asn",
tags="tags",
tag_list="tag_list",
)
except (KeyError):
raise serializers.ValidationError(_("Invalid variable detected."))
return path
class UiSettingsViewSerializer(serializers.ModelSerializer):
class Meta:
model = UiSettings
depth = 1
fields = [
"id",
"settings",
]
def update(self, instance, validated_data):
super().update(instance, validated_data)
return instance
def create(self, validated_data):
ui_settings = UiSettings.objects.update_or_create(
user=validated_data.get("user"),
defaults={"settings": validated_data.get("settings", None)},
)
return ui_settings

70
src/documents/signals/handlers.py
View File

@@ -230,6 +230,76 @@ def set_tags(
document.tags.add(*relevant_tags)
def set_storage_path(
sender,
document=None,
logging_group=None,
classifier=None,
replace=False,
use_first=True,
suggest=False,
base_url=None,
color=False,
**kwargs,
):
if document.storage_path and not replace:
return
potential_storage_path = matching.match_storage_paths(
document,
classifier,
)
potential_count = len(potential_storage_path)
if potential_storage_path:
selected = potential_storage_path[0]
else:
selected = None
if potential_count > 1:
if use_first:
logger.info(
f"Detected {potential_count} potential storage paths, "
f"so we've opted for {selected}",
extra={"group": logging_group},
)
else:
logger.info(
f"Detected {potential_count} potential storage paths, "
f"not assigning any storage directory",
extra={"group": logging_group},
)
return
if selected or replace:
if suggest:
if base_url:
print(
termcolors.colorize(str(document), fg="green")
if color
else str(document),
)
print(f"{base_url}/documents/{document.pk}")
else:
print(
(
termcolors.colorize(str(document), fg="green")
if color
else str(document)
)
+ f" [{document.pk}]",
)
print(f"Sugest storage directory {selected}")
else:
logger.info(
f"Assigning storage path {selected} to {document}",
extra={"group": logging_group},
)
document.storage_path = selected
document.save(update_fields=("storage_path",))
@receiver(models.signals.post_delete, sender=Document)
def cleanup_document_deletion(sender, instance, using, **kwargs):
with FileLock(settings.MEDIA_LOCK):

2
src/documents/tasks.py
View File

@@ -19,6 +19,7 @@ from documents.consumer import ConsumerError
from documents.models import Correspondent
from documents.models import Document
from documents.models import DocumentType
from documents.models import StoragePath
from documents.models import Tag
from documents.sanity_checker import SanityCheckFailedException
from pdf2image import convert_from_path
@@ -53,6 +54,7 @@ def train_classifier():
not Tag.objects.filter(matching_algorithm=Tag.MATCH_AUTO).exists()
and not DocumentType.objects.filter(matching_algorithm=Tag.MATCH_AUTO).exists()
and not Correspondent.objects.filter(matching_algorithm=Tag.MATCH_AUTO).exists()
and not StoragePath.objects.filter(matching_algorithm=Tag.MATCH_AUTO).exists()
):
return

2
src/documents/templates/index.html
View File

@@ -9,8 +9,6 @@
<title>Paperless-ngx</title>
<base href="{% url 'base' %}">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="username" content="{{username}}">
<meta name="full_name" content="{{full_name}}">
<meta name="cookie_prefix" content="{{cookie_prefix}}">
<meta name="robots" content="noindex,nofollow">
<link rel="icon" type="image/x-icon" href="favicon.ico">

BIN
src/documents/tests/data/model.pickle
View File

Binary file not shown.

162
src/documents/tests/test_api.py
View File

@@ -26,7 +26,10 @@ from documents.models import Document
from documents.models import DocumentType
from documents.models import MatchingModel
from documents.models import SavedView
from documents.models import StoragePath
from documents.models import Tag
from documents.models import UiSettings
from documents.models import StoragePath
from documents.tests.utils import DirectoriesMixin
from paperless import version
from rest_framework.test import APITestCase
@@ -38,7 +41,7 @@ class TestDocumentApi(DirectoriesMixin, APITestCase):
super().setUp()
self.user = User.objects.create_superuser(username="temp_admin")
self.client.force_login(user=self.user)
self.client.force_authenticate(user=self.user)
def testDocuments(self):
@@ -98,6 +101,7 @@ class TestDocumentApi(DirectoriesMixin, APITestCase):
c = Correspondent.objects.create(name="c", pk=41)
dt = DocumentType.objects.create(name="dt", pk=63)
tag = Tag.objects.create(name="t", pk=85)
storage_path = StoragePath.objects.create(name="sp", pk=77, path="p")
doc = Document.objects.create(
title="WOW",
content="the content",
@@ -105,6 +109,7 @@ class TestDocumentApi(DirectoriesMixin, APITestCase):
document_type=dt,
checksum="123",
mime_type="application/pdf",
storage_path=storage_path,
)
response = self.client.get("/api/documents/", format="json")
@@ -191,7 +196,7 @@ class TestDocumentApi(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, 200)
self.assertEqual(response.content, content_thumbnail)
@override_settings(PAPERLESS_FILENAME_FORMAT="")
@override_settings(FILENAME_FORMAT="")
def test_download_with_archive(self):
content = b"This is a test"
@@ -579,10 +584,12 @@ class TestDocumentApi(DirectoriesMixin, APITestCase):
t2 = Tag.objects.create(name="tag2")
c = Correspondent.objects.create(name="correspondent")
dt = DocumentType.objects.create(name="type")
sp = StoragePath.objects.create(name="path")
d1 = Document.objects.create(checksum="1", correspondent=c, content="test")
d2 = Document.objects.create(checksum="2", document_type=dt, content="test")
d3 = Document.objects.create(checksum="3", content="test")
d3.tags.add(t)
d3.tags.add(t2)
d4 = Document.objects.create(
@@ -597,6 +604,7 @@ class TestDocumentApi(DirectoriesMixin, APITestCase):
content="test",
)
d6 = Document.objects.create(checksum="6", content="test2")
d7 = Document.objects.create(checksum="7", storage_path=sp, content="test")
with AsyncWriter(index.open_index()) as writer:
for doc in Document.objects.all():
@@ -607,18 +615,30 @@ class TestDocumentApi(DirectoriesMixin, APITestCase):
self.assertEqual(r.status_code, 200)
return [hit["id"] for hit in r.data["results"]]
self.assertCountEqual(search_query(""), [d1.id, d2.id, d3.id, d4.id, d5.id])
self.assertCountEqual(
search_query(""),
[d1.id, d2.id, d3.id, d4.id, d5.id, d7.id],
)
self.assertCountEqual(search_query("&is_tagged=true"), [d3.id, d4.id])
self.assertCountEqual(search_query("&is_tagged=false"), [d1.id, d2.id, d5.id])
self.assertCountEqual(
search_query("&is_tagged=false"),
[d1.id, d2.id, d5.id, d7.id],
)
self.assertCountEqual(search_query("&correspondent__id=" + str(c.id)), [d1.id])
self.assertCountEqual(search_query("&document_type__id=" + str(dt.id)), [d2.id])
self.assertCountEqual(search_query("&storage_path__id=" + str(sp.id)), [d7.id])
self.assertCountEqual(
search_query("&storage_path__isnull"),
[d1.id, d2.id, d3.id, d4.id, d5.id],
)
self.assertCountEqual(
search_query("&correspondent__isnull"),
[d2.id, d3.id, d4.id, d5.id],
[d2.id, d3.id, d4.id, d5.id, d7.id],
)
self.assertCountEqual(
search_query("&document_type__isnull"),
[d1.id, d3.id, d4.id, d5.id],
[d1.id, d3.id, d4.id, d5.id, d7.id],
)
self.assertCountEqual(
search_query("&tags__id__all=" + str(t.id) + "," + str(t2.id)),
@@ -1079,35 +1099,49 @@ class TestDocumentApi(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, 200)
self.assertEqual(
response.data,
{"correspondents": [], "tags": [], "document_types": []},
{
"correspondents": [],
"tags": [],
"document_types": [],
"storage_paths": [],
},
)
def test_get_suggestions_invalid_doc(self):
response = self.client.get(f"/api/documents/34676/suggestions/")
self.assertEqual(response.status_code, 404)
@mock.patch("documents.views.match_correspondents")
@mock.patch("documents.views.match_tags")
@mock.patch("documents.views.match_storage_paths")
@mock.patch("documents.views.match_document_types")
@mock.patch("documents.views.match_tags")
@mock.patch("documents.views.match_correspondents")
def test_get_suggestions(
self,
match_document_types,
match_tags,
match_correspondents,
match_tags,
match_document_types,
match_storage_paths,
):
doc = Document.objects.create(
title="test",
mime_type="application/pdf",
content="this is an invoice!",
)
match_correspondents.return_value = [Correspondent(id=88), Correspondent(id=2)]
match_tags.return_value = [Tag(id=56), Tag(id=123)]
match_document_types.return_value = [DocumentType(id=23)]
match_correspondents.return_value = [Correspondent(id=88), Correspondent(id=2)]
match_storage_paths.return_value = [StoragePath(id=99), StoragePath(id=77)]
response = self.client.get(f"/api/documents/{doc.pk}/suggestions/")
self.assertEqual(
response.data,
{"correspondents": [88, 2], "tags": [56, 123], "document_types": [23]},
{
"correspondents": [88, 2],
"tags": [56, 123],
"document_types": [23],
"storage_paths": [99, 77],
},
)
def test_saved_views(self):
@@ -1142,7 +1176,7 @@ class TestDocumentApi(DirectoriesMixin, APITestCase):
self.assertEqual(self.client.get(f"/api/saved_views/{v1.id}/").status_code, 404)
self.client.force_login(user=u1)
self.client.force_authenticate(user=u1)
response = self.client.get("/api/saved_views/")
self.assertEqual(response.status_code, 200)
@@ -1150,7 +1184,7 @@ class TestDocumentApi(DirectoriesMixin, APITestCase):
self.assertEqual(self.client.get(f"/api/saved_views/{v1.id}/").status_code, 200)
self.client.force_login(user=u2)
self.client.force_authenticate(user=u2)
response = self.client.get("/api/saved_views/")
self.assertEqual(response.status_code, 200)
@@ -1324,7 +1358,7 @@ class TestDocumentApiV2(DirectoriesMixin, APITestCase):
self.user = User.objects.create_superuser(username="temp_admin")
self.client.force_login(user=self.user)
self.client.force_authenticate(user=self.user)
self.client.defaults["HTTP_ACCEPT"] = "application/json; version=2"
def test_tag_validate_color(self):
@@ -1398,13 +1432,48 @@ class TestDocumentApiV2(DirectoriesMixin, APITestCase):
"#000000",
)
def test_ui_settings(self):
test_user = User.objects.create_superuser(username="test")
self.client.force_authenticate(user=test_user)
response = self.client.get("/api/ui_settings/", format="json")
self.assertEqual(response.status_code, 200)
self.assertDictEqual(
response.data["settings"],
{},
)
settings = {
"settings": {
"dark_mode": {
"enabled": True,
},
},
}
response = self.client.post(
"/api/ui_settings/",
json.dumps(settings),
content_type="application/json",
)
self.assertEqual(response.status_code, 200)
response = self.client.get("/api/ui_settings/", format="json")
self.assertEqual(response.status_code, 200)
self.assertDictEqual(
response.data["settings"],
settings["settings"],
)
class TestBulkEdit(DirectoriesMixin, APITestCase):
def setUp(self):
super().setUp()
user = User.objects.create_superuser(username="temp_admin")
self.client.force_login(user=user)
self.client.force_authenticate(user=user)
patcher = mock.patch("documents.bulk_edit.async_task")
self.async_task = patcher.start()
@@ -1433,6 +1502,7 @@ class TestBulkEdit(DirectoriesMixin, APITestCase):
self.doc2.tags.add(self.t1)
self.doc3.tags.add(self.t2)
self.doc4.tags.add(self.t1, self.t2)
self.sp1 = StoragePath.objects.create(name="sp1", path="Something/{checksum}")
def test_set_correspondent(self):
self.assertEqual(Document.objects.filter(correspondent=self.c2).count(), 1)
@@ -1472,6 +1542,60 @@ class TestBulkEdit(DirectoriesMixin, APITestCase):
args, kwargs = self.async_task.call_args
self.assertCountEqual(kwargs["document_ids"], [self.doc2.id, self.doc3.id])
def test_set_document_storage_path(self):
"""
GIVEN:
- 5 documents without defined storage path
WHEN:
- Bulk edit called to add storage path to 1 document
THEN:
- Single document storage path update
"""
self.assertEqual(Document.objects.filter(storage_path=None).count(), 5)
bulk_edit.set_storage_path(
[self.doc1.id],
self.sp1.id,
)
self.assertEqual(Document.objects.filter(storage_path=None).count(), 4)
self.async_task.assert_called_once()
args, kwargs = self.async_task.call_args
self.assertCountEqual(kwargs["document_ids"], [self.doc1.id])
def test_unset_document_storage_path(self):
"""
GIVEN:
- 4 documents without defined storage path
- 1 document with a defined storage
WHEN:
- Bulk edit called to remove storage path from 1 document
THEN:
- Single document storage path removed
"""
self.assertEqual(Document.objects.filter(storage_path=None).count(), 5)
bulk_edit.set_storage_path(
[self.doc1.id],
self.sp1.id,
)
self.assertEqual(Document.objects.filter(storage_path=None).count(), 4)
bulk_edit.set_storage_path(
[self.doc1.id],
None,
)
self.assertEqual(Document.objects.filter(storage_path=None).count(), 5)
self.async_task.assert_called()
args, kwargs = self.async_task.call_args
self.assertCountEqual(kwargs["document_ids"], [self.doc1.id])
def test_add_tag(self):
self.assertEqual(Document.objects.filter(tags__id=self.t1.id).count(), 2)
bulk_edit.add_tag(
@@ -1925,7 +2049,7 @@ class TestBulkDownload(DirectoriesMixin, APITestCase):
super().setUp()
user = User.objects.create_superuser(username="temp_admin")
self.client.force_login(user=user)
self.client.force_authenticate(user=user)
self.doc1 = Document.objects.create(title="unrelated", checksum="A")
self.doc2 = Document.objects.create(
@@ -2126,7 +2250,7 @@ class TestApiAuth(APITestCase):
def test_api_version_with_auth(self):
user = User.objects.create_superuser(username="test")
self.client.force_login(user)
self.client.force_authenticate(user)
response = self.client.get("/api/")
self.assertIn("X-Api-Version", response)
self.assertIn("X-Version", response)

62
src/documents/tests/test_classifier.py
View File

@@ -13,6 +13,7 @@ from documents.classifier import load_classifier
from documents.models import Correspondent
from documents.models import Document
from documents.models import DocumentType
from documents.models import StoragePath
from documents.models import Tag
from documents.tests.utils import DirectoriesMixin
@@ -56,6 +57,16 @@ class TestClassifier(DirectoriesMixin, TestCase):
name="dt2",
matching_algorithm=DocumentType.MATCH_AUTO,
)
self.sp1 = StoragePath.objects.create(
name="sp1",
path="path1",
matching_algorithm=DocumentType.MATCH_AUTO,
)
self.sp2 = StoragePath.objects.create(
name="sp2",
path="path2",
matching_algorithm=DocumentType.MATCH_AUTO,
)
self.doc1 = Document.objects.create(
title="doc1",
@@ -64,12 +75,14 @@ class TestClassifier(DirectoriesMixin, TestCase):
checksum="A",
document_type=self.dt,
)
self.doc2 = Document.objects.create(
title="doc1",
content="this is another document, but from c2",
correspondent=self.c2,
checksum="B",
)
self.doc_inbox = Document.objects.create(
title="doc235",
content="aa",
@@ -81,6 +94,8 @@ class TestClassifier(DirectoriesMixin, TestCase):
self.doc2.tags.add(self.t3)
self.doc_inbox.tags.add(self.t2)
self.doc1.storage_path = self.sp1
def testNoTrainingData(self):
try:
self.classifier.train()
@@ -177,6 +192,14 @@ class TestClassifier(DirectoriesMixin, TestCase):
new_classifier.load()
self.assertFalse(new_classifier.train())
# @override_settings(
# MODEL_FILE=os.path.join(os.path.dirname(__file__), "data", "model.pickle"),
# )
# def test_create_test_load_and_classify(self):
# self.generate_test_data()
# self.classifier.train()
# self.classifier.save()
@override_settings(
MODEL_FILE=os.path.join(os.path.dirname(__file__), "data", "model.pickle"),
)
@@ -263,6 +286,45 @@ class TestClassifier(DirectoriesMixin, TestCase):
self.assertEqual(self.classifier.predict_document_type(doc1.content), dt.pk)
self.assertIsNone(self.classifier.predict_document_type(doc2.content))
def test_one_path_predict(self):
sp = StoragePath.objects.create(
name="sp",
matching_algorithm=StoragePath.MATCH_AUTO,
)
doc1 = Document.objects.create(
title="doc1",
content="this is a document from c1",
checksum="A",
storage_path=sp,
)
self.classifier.train()
self.assertEqual(self.classifier.predict_storage_path(doc1.content), sp.pk)
def test_one_path_predict_manydocs(self):
sp = StoragePath.objects.create(
name="sp",
matching_algorithm=StoragePath.MATCH_AUTO,
)
doc1 = Document.objects.create(
title="doc1",
content="this is a document from c1",
checksum="A",
storage_path=sp,
)
doc2 = Document.objects.create(
title="doc1",
content="this is a document from c2",
checksum="B",
)
self.classifier.train()
self.assertEqual(self.classifier.predict_storage_path(doc1.content), sp.pk)
self.assertIsNone(self.classifier.predict_storage_path(doc2.content))
def test_one_tag_predict(self):
t1 = Tag.objects.create(name="t1", matching_algorithm=Tag.MATCH_AUTO, pk=12)

12
src/documents/tests/test_consumer.py
View File

@@ -320,7 +320,7 @@ class TestConsumer(DirectoriesMixin, TestCase):
shutil.copy(src, dst)
return dst
@override_settings(PAPERLESS_FILENAME_FORMAT=None, TIME_ZONE="America/Chicago")
@override_settings(FILENAME_FORMAT=None, TIME_ZONE="America/Chicago")
def testNormalOperation(self):
filename = self.get_test_file()
@@ -351,7 +351,7 @@ class TestConsumer(DirectoriesMixin, TestCase):
self.assertEqual(document.created.tzinfo, zoneinfo.ZoneInfo("America/Chicago"))
@override_settings(PAPERLESS_FILENAME_FORMAT=None)
@override_settings(FILENAME_FORMAT=None)
def testDeleteMacFiles(self):
# https://github.com/jonaswinkler/paperless-ng/discussions/1037
@@ -518,7 +518,7 @@ class TestConsumer(DirectoriesMixin, TestCase):
# Database empty
self.assertEqual(len(Document.objects.all()), 0)
@override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{title}")
@override_settings(FILENAME_FORMAT="{correspondent}/{title}")
def testFilenameHandling(self):
filename = self.get_test_file()
@@ -530,7 +530,7 @@ class TestConsumer(DirectoriesMixin, TestCase):
self._assert_first_last_send_progress()
@override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{title}")
@override_settings(FILENAME_FORMAT="{correspondent}/{title}")
@mock.patch("documents.signals.handlers.generate_unique_filename")
def testFilenameHandlingUnstableFormat(self, m):
@@ -612,7 +612,7 @@ class TestConsumer(DirectoriesMixin, TestCase):
self._assert_first_last_send_progress(last_status="FAILED")
@override_settings(PAPERLESS_FILENAME_FORMAT="{title}")
@override_settings(FILENAME_FORMAT="{title}")
@mock.patch("documents.parsers.document_consumer_declaration.send")
def test_similar_filenames(self, m):
shutil.copy(
@@ -660,7 +660,7 @@ class TestConsumer(DirectoriesMixin, TestCase):
@mock.patch("documents.consumer.magic.from_file", fake_magic_from_file)
class TestConsumerCreatedDate(DirectoriesMixin, TestCase):
def setUp(self):
super(TestConsumerCreatedDate, self).setUp()
super().setUp()
# this prevents websocket message reports during testing.
patcher = mock.patch("documents.consumer.Consumer._send_progress")

216
src/documents/tests/test_file_handling.py
View File

@@ -20,12 +20,12 @@ from ..file_handling import generate_unique_filename
from ..models import Correspondent
from ..models import Document
from ..models import DocumentType
from ..models import Tag
from ..models import StoragePath
from .utils import DirectoriesMixin
class TestFileHandling(DirectoriesMixin, TestCase):
@override_settings(PAPERLESS_FILENAME_FORMAT="")
@override_settings(FILENAME_FORMAT="")
def test_generate_source_filename(self):
document = Document()
document.mime_type = "application/pdf"
@@ -40,7 +40,7 @@ class TestFileHandling(DirectoriesMixin, TestCase):
f"{document.pk:07d}.pdf.gpg",
)
@override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{correspondent}")
@override_settings(FILENAME_FORMAT="{correspondent}/{correspondent}")
def test_file_renaming(self):
document = Document()
document.mime_type = "application/pdf"
@@ -82,7 +82,7 @@ class TestFileHandling(DirectoriesMixin, TestCase):
True,
)
@override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{correspondent}")
@override_settings(FILENAME_FORMAT="{correspondent}/{correspondent}")
def test_file_renaming_missing_permissions(self):
document = Document()
document.mime_type = "application/pdf"
@@ -117,7 +117,7 @@ class TestFileHandling(DirectoriesMixin, TestCase):
os.chmod(settings.ORIGINALS_DIR + "/none", 0o777)
@override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{correspondent}")
@override_settings(FILENAME_FORMAT="{correspondent}/{correspondent}")
def test_file_renaming_database_error(self):
document1 = Document.objects.create(
@@ -156,7 +156,7 @@ class TestFileHandling(DirectoriesMixin, TestCase):
)
self.assertEqual(document.filename, "none/none.pdf")
@override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{correspondent}")
@override_settings(FILENAME_FORMAT="{correspondent}/{correspondent}")
def test_document_delete(self):
document = Document()
document.mime_type = "application/pdf"
@@ -180,7 +180,7 @@ class TestFileHandling(DirectoriesMixin, TestCase):
self.assertEqual(os.path.isdir(settings.ORIGINALS_DIR + "/none"), False)
@override_settings(
PAPERLESS_FILENAME_FORMAT="{correspondent}/{correspondent}",
FILENAME_FORMAT="{correspondent}/{correspondent}",
TRASH_DIR=tempfile.mkdtemp(),
)
def test_document_delete_trash(self):
@@ -218,7 +218,7 @@ class TestFileHandling(DirectoriesMixin, TestCase):
document.delete()
self.assertEqual(os.path.isfile(settings.TRASH_DIR + "/none_01.pdf"), True)
@override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{correspondent}")
@override_settings(FILENAME_FORMAT="{correspondent}/{correspondent}")
def test_document_delete_nofile(self):
document = Document()
document.mime_type = "application/pdf"
@@ -227,7 +227,7 @@ class TestFileHandling(DirectoriesMixin, TestCase):
document.delete()
@override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{correspondent}")
@override_settings(FILENAME_FORMAT="{correspondent}/{correspondent}")
def test_directory_not_empty(self):
document = Document()
document.mime_type = "application/pdf"
@@ -253,7 +253,7 @@ class TestFileHandling(DirectoriesMixin, TestCase):
self.assertEqual(os.path.isdir(settings.ORIGINALS_DIR + "/none"), True)
self.assertTrue(os.path.isfile(important_file))
@override_settings(PAPERLESS_FILENAME_FORMAT="{document_type} - {title}")
@override_settings(FILENAME_FORMAT="{document_type} - {title}")
def test_document_type(self):
dt = DocumentType.objects.create(name="my_doc_type")
d = Document.objects.create(title="the_doc", mime_type="application/pdf")
@@ -264,7 +264,7 @@ class TestFileHandling(DirectoriesMixin, TestCase):
self.assertEqual(generate_filename(d), "my_doc_type - the_doc.pdf")
@override_settings(PAPERLESS_FILENAME_FORMAT="{asn} - {title}")
@override_settings(FILENAME_FORMAT="{asn} - {title}")
def test_asn(self):
d1 = Document.objects.create(
title="the_doc",
@@ -281,7 +281,7 @@ class TestFileHandling(DirectoriesMixin, TestCase):
self.assertEqual(generate_filename(d1), "652 - the_doc.pdf")
self.assertEqual(generate_filename(d2), "none - the_doc.pdf")
@override_settings(PAPERLESS_FILENAME_FORMAT="{tags[type]}")
@override_settings(FILENAME_FORMAT="{tags[type]}")
def test_tags_with_underscore(self):
document = Document()
document.mime_type = "application/pdf"
@@ -296,7 +296,7 @@ class TestFileHandling(DirectoriesMixin, TestCase):
# Ensure that filename is properly generated
self.assertEqual(generate_filename(document), "demo.pdf")
@override_settings(PAPERLESS_FILENAME_FORMAT="{tags[type]}")
@override_settings(FILENAME_FORMAT="{tags[type]}")
def test_tags_with_dash(self):
document = Document()
document.mime_type = "application/pdf"
@@ -311,7 +311,7 @@ class TestFileHandling(DirectoriesMixin, TestCase):
# Ensure that filename is properly generated
self.assertEqual(generate_filename(document), "demo.pdf")
@override_settings(PAPERLESS_FILENAME_FORMAT="{tags[type]}")
@override_settings(FILENAME_FORMAT="{tags[type]}")
def test_tags_malformed(self):
document = Document()
document.mime_type = "application/pdf"
@@ -326,7 +326,7 @@ class TestFileHandling(DirectoriesMixin, TestCase):
# Ensure that filename is properly generated
self.assertEqual(generate_filename(document), "none.pdf")
@override_settings(PAPERLESS_FILENAME_FORMAT="{tags[0]}")
@override_settings(FILENAME_FORMAT="{tags[0]}")
def test_tags_all(self):
document = Document()
document.mime_type = "application/pdf"
@@ -340,7 +340,7 @@ class TestFileHandling(DirectoriesMixin, TestCase):
# Ensure that filename is properly generated
self.assertEqual(generate_filename(document), "demo.pdf")
@override_settings(PAPERLESS_FILENAME_FORMAT="{tags[1]}")
@override_settings(FILENAME_FORMAT="{tags[1]}")
def test_tags_out_of_bounds(self):
document = Document()
document.mime_type = "application/pdf"
@@ -354,7 +354,7 @@ class TestFileHandling(DirectoriesMixin, TestCase):
# Ensure that filename is properly generated
self.assertEqual(generate_filename(document), "none.pdf")
@override_settings(PAPERLESS_FILENAME_FORMAT="{tags}")
@override_settings(FILENAME_FORMAT="{tags}")
def test_tags_without_args(self):
document = Document()
document.mime_type = "application/pdf"
@@ -363,7 +363,7 @@ class TestFileHandling(DirectoriesMixin, TestCase):
self.assertEqual(generate_filename(document), f"{document.pk:07}.pdf")
@override_settings(PAPERLESS_FILENAME_FORMAT="{title} {tag_list}")
@override_settings(FILENAME_FORMAT="{title} {tag_list}")
def test_tag_list(self):
doc = Document.objects.create(title="doc1", mime_type="application/pdf")
doc.tags.create(name="tag2")
@@ -379,7 +379,7 @@ class TestFileHandling(DirectoriesMixin, TestCase):
self.assertEqual(generate_filename(doc), "doc2.pdf")
@override_settings(PAPERLESS_FILENAME_FORMAT="//etc/something/{title}")
@override_settings(FILENAME_FORMAT="//etc/something/{title}")
def test_filename_relative(self):
doc = Document.objects.create(title="doc1", mime_type="application/pdf")
doc.filename = generate_filename(doc)
@@ -391,7 +391,7 @@ class TestFileHandling(DirectoriesMixin, TestCase):
)
@override_settings(
PAPERLESS_FILENAME_FORMAT="{created_year}-{created_month}-{created_day}",
FILENAME_FORMAT="{created_year}-{created_month}-{created_day}",
)
def test_created_year_month_day(self):
d1 = timezone.make_aware(datetime.datetime(2020, 3, 6, 1, 1, 1))
@@ -408,7 +408,7 @@ class TestFileHandling(DirectoriesMixin, TestCase):
self.assertEqual(generate_filename(doc1), "2020-11-16.pdf")
@override_settings(
PAPERLESS_FILENAME_FORMAT="{added_year}-{added_month}-{added_day}",
FILENAME_FORMAT="{added_year}-{added_month}-{added_day}",
)
def test_added_year_month_day(self):
d1 = timezone.make_aware(datetime.datetime(232, 1, 9, 1, 1, 1))
@@ -425,7 +425,7 @@ class TestFileHandling(DirectoriesMixin, TestCase):
self.assertEqual(generate_filename(doc1), "2020-11-16.pdf")
@override_settings(
PAPERLESS_FILENAME_FORMAT="{correspondent}/{correspondent}/{correspondent}",
FILENAME_FORMAT="{correspondent}/{correspondent}/{correspondent}",
)
def test_nested_directory_cleanup(self):
document = Document()
@@ -453,7 +453,7 @@ class TestFileHandling(DirectoriesMixin, TestCase):
self.assertEqual(os.path.isdir(settings.ORIGINALS_DIR + "/none"), False)
self.assertEqual(os.path.isdir(settings.ORIGINALS_DIR), True)
@override_settings(PAPERLESS_FILENAME_FORMAT=None)
@override_settings(FILENAME_FORMAT=None)
def test_format_none(self):
document = Document()
document.pk = 1
@@ -479,7 +479,7 @@ class TestFileHandling(DirectoriesMixin, TestCase):
self.assertEqual(os.path.isfile(os.path.join(tmp, "notempty", "file")), True)
self.assertEqual(os.path.isdir(os.path.join(tmp, "notempty", "empty")), False)
@override_settings(PAPERLESS_FILENAME_FORMAT="{created/[title]")
@override_settings(FILENAME_FORMAT="{created/[title]")
def test_invalid_format(self):
document = Document()
document.pk = 1
@@ -488,7 +488,7 @@ class TestFileHandling(DirectoriesMixin, TestCase):
self.assertEqual(generate_filename(document), "0000001.pdf")
@override_settings(PAPERLESS_FILENAME_FORMAT="{created__year}")
@override_settings(FILENAME_FORMAT="{created__year}")
def test_invalid_format_key(self):
document = Document()
document.pk = 1
@@ -497,7 +497,7 @@ class TestFileHandling(DirectoriesMixin, TestCase):
self.assertEqual(generate_filename(document), "0000001.pdf")
@override_settings(PAPERLESS_FILENAME_FORMAT="{title}")
@override_settings(FILENAME_FORMAT="{title}")
def test_duplicates(self):
document = Document.objects.create(
mime_type="application/pdf",
@@ -548,7 +548,7 @@ class TestFileHandling(DirectoriesMixin, TestCase):
self.assertTrue(os.path.isfile(document.source_path))
self.assertEqual(document2.filename, "qwe.pdf")
@override_settings(PAPERLESS_FILENAME_FORMAT="{title}")
@override_settings(FILENAME_FORMAT="{title}")
@mock.patch("documents.signals.handlers.Document.objects.filter")
def test_no_update_without_change(self, m):
doc = Document.objects.create(
@@ -568,7 +568,7 @@ class TestFileHandling(DirectoriesMixin, TestCase):
class TestFileHandlingWithArchive(DirectoriesMixin, TestCase):
@override_settings(PAPERLESS_FILENAME_FORMAT=None)
@override_settings(FILENAME_FORMAT=None)
def test_create_no_format(self):
original = os.path.join(settings.ORIGINALS_DIR, "0000001.pdf")
archive = os.path.join(settings.ARCHIVE_DIR, "0000001.pdf")
@@ -587,7 +587,7 @@ class TestFileHandlingWithArchive(DirectoriesMixin, TestCase):
self.assertTrue(os.path.isfile(doc.source_path))
self.assertTrue(os.path.isfile(doc.archive_path))
@override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{title}")
@override_settings(FILENAME_FORMAT="{correspondent}/{title}")
def test_create_with_format(self):
original = os.path.join(settings.ORIGINALS_DIR, "0000001.pdf")
archive = os.path.join(settings.ARCHIVE_DIR, "0000001.pdf")
@@ -615,7 +615,7 @@ class TestFileHandlingWithArchive(DirectoriesMixin, TestCase):
os.path.join(settings.ARCHIVE_DIR, "none", "my_doc.pdf"),
)
@override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{title}")
@override_settings(FILENAME_FORMAT="{correspondent}/{title}")
def test_move_archive_gone(self):
original = os.path.join(settings.ORIGINALS_DIR, "0000001.pdf")
archive = os.path.join(settings.ARCHIVE_DIR, "0000001.pdf")
@@ -634,7 +634,7 @@ class TestFileHandlingWithArchive(DirectoriesMixin, TestCase):
self.assertTrue(os.path.isfile(doc.source_path))
self.assertFalse(os.path.isfile(doc.archive_path))
@override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{title}")
@override_settings(FILENAME_FORMAT="{correspondent}/{title}")
def test_move_archive_exists(self):
original = os.path.join(settings.ORIGINALS_DIR, "0000001.pdf")
archive = os.path.join(settings.ARCHIVE_DIR, "0000001.pdf")
@@ -659,7 +659,7 @@ class TestFileHandlingWithArchive(DirectoriesMixin, TestCase):
self.assertTrue(os.path.isfile(existing_archive_file))
self.assertEqual(doc.archive_filename, "none/my_doc_01.pdf")
@override_settings(PAPERLESS_FILENAME_FORMAT="{title}")
@override_settings(FILENAME_FORMAT="{title}")
def test_move_original_only(self):
original = os.path.join(settings.ORIGINALS_DIR, "document_01.pdf")
archive = os.path.join(settings.ARCHIVE_DIR, "document.pdf")
@@ -681,7 +681,7 @@ class TestFileHandlingWithArchive(DirectoriesMixin, TestCase):
self.assertTrue(os.path.isfile(doc.source_path))
self.assertTrue(os.path.isfile(doc.archive_path))
@override_settings(PAPERLESS_FILENAME_FORMAT="{title}")
@override_settings(FILENAME_FORMAT="{title}")
def test_move_archive_only(self):
original = os.path.join(settings.ORIGINALS_DIR, "document.pdf")
archive = os.path.join(settings.ARCHIVE_DIR, "document_01.pdf")
@@ -703,7 +703,7 @@ class TestFileHandlingWithArchive(DirectoriesMixin, TestCase):
self.assertTrue(os.path.isfile(doc.source_path))
self.assertTrue(os.path.isfile(doc.archive_path))
@override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{title}")
@override_settings(FILENAME_FORMAT="{correspondent}/{title}")
@mock.patch("documents.signals.handlers.os.rename")
def test_move_archive_error(self, m):
def fake_rename(src, dst):
@@ -734,7 +734,7 @@ class TestFileHandlingWithArchive(DirectoriesMixin, TestCase):
self.assertTrue(os.path.isfile(doc.source_path))
self.assertTrue(os.path.isfile(doc.archive_path))
@override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{title}")
@override_settings(FILENAME_FORMAT="{correspondent}/{title}")
def test_move_file_gone(self):
original = os.path.join(settings.ORIGINALS_DIR, "0000001.pdf")
archive = os.path.join(settings.ARCHIVE_DIR, "0000001.pdf")
@@ -754,7 +754,7 @@ class TestFileHandlingWithArchive(DirectoriesMixin, TestCase):
self.assertFalse(os.path.isfile(doc.source_path))
self.assertTrue(os.path.isfile(doc.archive_path))
@override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{title}")
@override_settings(FILENAME_FORMAT="{correspondent}/{title}")
@mock.patch("documents.signals.handlers.os.rename")
def test_move_file_error(self, m):
def fake_rename(src, dst):
@@ -785,7 +785,7 @@ class TestFileHandlingWithArchive(DirectoriesMixin, TestCase):
self.assertTrue(os.path.isfile(doc.source_path))
self.assertTrue(os.path.isfile(doc.archive_path))
@override_settings(PAPERLESS_FILENAME_FORMAT="")
@override_settings(FILENAME_FORMAT="")
def test_archive_deleted(self):
original = os.path.join(settings.ORIGINALS_DIR, "0000001.pdf")
archive = os.path.join(settings.ARCHIVE_DIR, "0000001.pdf")
@@ -812,7 +812,7 @@ class TestFileHandlingWithArchive(DirectoriesMixin, TestCase):
self.assertFalse(os.path.isfile(doc.source_path))
self.assertFalse(os.path.isfile(doc.archive_path))
@override_settings(PAPERLESS_FILENAME_FORMAT="{title}")
@override_settings(FILENAME_FORMAT="{title}")
def test_archive_deleted2(self):
original = os.path.join(settings.ORIGINALS_DIR, "document.png")
original2 = os.path.join(settings.ORIGINALS_DIR, "0000001.pdf")
@@ -846,7 +846,7 @@ class TestFileHandlingWithArchive(DirectoriesMixin, TestCase):
self.assertTrue(os.path.isfile(doc1.archive_path))
self.assertFalse(os.path.isfile(doc2.source_path))
@override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{title}")
@override_settings(FILENAME_FORMAT="{correspondent}/{title}")
def test_database_error(self):
original = os.path.join(settings.ORIGINALS_DIR, "0000001.pdf")
@@ -872,7 +872,7 @@ class TestFileHandlingWithArchive(DirectoriesMixin, TestCase):
class TestFilenameGeneration(TestCase):
@override_settings(PAPERLESS_FILENAME_FORMAT="{title}")
@override_settings(FILENAME_FORMAT="{title}")
def test_invalid_characters(self):
doc = Document.objects.create(
@@ -891,7 +891,7 @@ class TestFilenameGeneration(TestCase):
)
self.assertEqual(generate_filename(doc), "my-invalid-..-title-yay.pdf")
@override_settings(PAPERLESS_FILENAME_FORMAT="{created}")
@override_settings(FILENAME_FORMAT="{created}")
def test_date(self):
doc = Document.objects.create(
title="does not matter",
@@ -902,6 +902,140 @@ class TestFilenameGeneration(TestCase):
)
self.assertEqual(generate_filename(doc), "2020-05-21.pdf")
def test_dynamic_path(self):
"""
GIVEN:
- A document with a defined storage path
WHEN:
- the filename is generated for the document
THEN:
- the generated filename uses the defined storage path for the document
"""
doc = Document.objects.create(
title="does not matter",
created=timezone.make_aware(datetime.datetime(2020, 6, 25, 7, 36, 51, 153)),
mime_type="application/pdf",
pk=2,
checksum="2",
storage_path=StoragePath.objects.create(path="TestFolder/{created}"),
)
self.assertEqual(generate_filename(doc), "TestFolder/2020-06-25.pdf")
def test_dynamic_path_with_none(self):
"""
GIVEN:
- A document with a defined storage path
- The defined storage path uses an undefined field for the document
WHEN:
- the filename is generated for the document
THEN:
- the generated filename uses the defined storage path for the document
- the generated filename includes "none" in the place undefined field
"""
doc = Document.objects.create(
title="does not matter",
created=timezone.make_aware(datetime.datetime(2020, 6, 25, 7, 36, 51, 153)),
mime_type="application/pdf",
pk=2,
checksum="2",
storage_path=StoragePath.objects.create(path="{asn} - {created}"),
)
self.assertEqual(generate_filename(doc), "none - 2020-06-25.pdf")
@override_settings(
FILENAME_FORMAT_REMOVE_NONE=True,
)
def test_dynamic_path_remove_none(self):
"""
GIVEN:
- A document with a defined storage path
- The defined storage path uses an undefined field for the document
- The setting for removing undefined fields is enabled
WHEN:
- the filename is generated for the document
THEN:
- the generated filename uses the defined storage path for the document
- the generated filename does not include "none" in the place undefined field
"""
doc = Document.objects.create(
title="does not matter",
created=timezone.make_aware(datetime.datetime(2020, 6, 25, 7, 36, 51, 153)),
mime_type="application/pdf",
pk=2,
checksum="2",
storage_path=StoragePath.objects.create(path="TestFolder/{asn}/{created}"),
)
self.assertEqual(generate_filename(doc), "TestFolder/2020-06-25.pdf")
def test_multiple_doc_paths(self):
"""
GIVEN:
- Two documents, each with different storage paths
WHEN:
- the filename is generated for the documents
THEN:
- Each document generated filename uses its storage path
"""
doc_a = Document.objects.create(
title="does not matter",
created=timezone.make_aware(datetime.datetime(2020, 6, 25, 7, 36, 51, 153)),
mime_type="application/pdf",
pk=2,
checksum="2",
archive_serial_number=4,
storage_path=StoragePath.objects.create(
name="sp1",
path="ThisIsAFolder/{asn}/{created}",
),
)
doc_b = Document.objects.create(
title="does not matter",
created=timezone.make_aware(datetime.datetime(2020, 7, 25, 7, 36, 51, 153)),
mime_type="application/pdf",
pk=5,
checksum="abcde",
storage_path=StoragePath.objects.create(
name="sp2",
path="SomeImportantNone/{created}",
),
)
self.assertEqual(generate_filename(doc_a), "ThisIsAFolder/4/2020-06-25.pdf")
self.assertEqual(generate_filename(doc_b), "SomeImportantNone/2020-07-25.pdf")
def test_no_path_fallback(self):
"""
GIVEN:
- Two documents, one with defined storage path, the other not
WHEN:
- the filename is generated for the documents
THEN:
- Document with defined path uses its format
- Document without defined path uses the default path
"""
doc_a = Document.objects.create(
title="does not matter",
created=timezone.make_aware(datetime.datetime(2020, 6, 25, 7, 36, 51, 153)),
mime_type="application/pdf",
pk=2,
checksum="2",
archive_serial_number=4,
)
doc_b = Document.objects.create(
title="does not matter",
created=timezone.make_aware(datetime.datetime(2020, 7, 25, 7, 36, 51, 153)),
mime_type="application/pdf",
pk=5,
checksum="abcde",
storage_path=StoragePath.objects.create(
name="sp2",
path="SomeImportantNone/{created}",
),
)
self.assertEqual(generate_filename(doc_a), "0000002.pdf")
self.assertEqual(generate_filename(doc_b), "SomeImportantNone/2020-07-25.pdf")
def run():
doc = Document.objects.create(

10
src/documents/tests/test_management.py
View File

@@ -18,7 +18,7 @@ from documents.tests.utils import DirectoriesMixin
sample_file = os.path.join(os.path.dirname(__file__), "samples", "simple.pdf")
@override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{title}")
@override_settings(FILENAME_FORMAT="{correspondent}/{title}")
class TestArchiver(DirectoriesMixin, TestCase):
def make_models(self):
return Document.objects.create(
@@ -72,7 +72,7 @@ class TestArchiver(DirectoriesMixin, TestCase):
self.assertIsNone(doc.archive_filename)
self.assertTrue(os.path.isfile(doc.source_path))
@override_settings(PAPERLESS_FILENAME_FORMAT="{title}")
@override_settings(FILENAME_FORMAT="{title}")
def test_naming_priorities(self):
doc1 = Document.objects.create(
checksum="A",
@@ -109,7 +109,7 @@ class TestDecryptDocuments(TestCase):
ORIGINALS_DIR=os.path.join(os.path.dirname(__file__), "samples", "originals"),
THUMBNAIL_DIR=os.path.join(os.path.dirname(__file__), "samples", "thumb"),
PASSPHRASE="test",
PAPERLESS_FILENAME_FORMAT=None,
FILENAME_FORMAT=None,
)
@mock.patch("documents.management.commands.decrypt_documents.input")
def test_decrypt(self, m):
@@ -184,7 +184,7 @@ class TestMakeIndex(TestCase):
class TestRenamer(DirectoriesMixin, TestCase):
@override_settings(PAPERLESS_FILENAME_FORMAT="")
@override_settings(FILENAME_FORMAT="")
def test_rename(self):
doc = Document.objects.create(title="test", mime_type="image/jpeg")
doc.filename = generate_filename(doc)
@@ -194,7 +194,7 @@ class TestRenamer(DirectoriesMixin, TestCase):
Path(doc.source_path).touch()
Path(doc.archive_path).touch()
with override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{title}"):
with override_settings(FILENAME_FORMAT="{correspondent}/{title}"):
call_command("document_renamer")
doc2 = Document.objects.get(id=doc.id)

4
src/documents/tests/test_management_exporter.py
View File

@@ -200,7 +200,7 @@ class TestExportImport(DirectoriesMixin, TestCase):
)
with override_settings(
PAPERLESS_FILENAME_FORMAT="{created_year}/{correspondent}/{title}",
FILENAME_FORMAT="{created_year}/{correspondent}/{title}",
):
self.test_exporter(use_filename_format=True)
@@ -309,7 +309,7 @@ class TestExportImport(DirectoriesMixin, TestCase):
self.assertTrue(len(manifest), 6)
@override_settings(PAPERLESS_FILENAME_FORMAT="{title}/{correspondent}")
@override_settings(FILENAME_FORMAT="{title}/{correspondent}")
def test_update_export_changed_location(self):
shutil.rmtree(os.path.join(self.dirs.media_dir, "documents"))
shutil.copytree(

12
src/documents/tests/test_migration_archive_files.py
View File

@@ -111,7 +111,7 @@ simple_png = os.path.join(os.path.dirname(__file__), "samples", "simple-noalpha.
simple_png2 = os.path.join(os.path.dirname(__file__), "examples", "no-text.png")
@override_settings(PAPERLESS_FILENAME_FORMAT="")
@override_settings(FILENAME_FORMAT="")
class TestMigrateArchiveFiles(DirectoriesMixin, TestMigrations):
migrate_from = "1011_auto_20210101_2340"
@@ -240,7 +240,7 @@ class TestMigrateArchiveFiles(DirectoriesMixin, TestMigrations):
)
@override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{title}")
@override_settings(FILENAME_FORMAT="{correspondent}/{title}")
class TestMigrateArchiveFilesWithFilenameFormat(TestMigrateArchiveFiles):
def test_filenames(self):
Document = self.apps.get_model("documents", "Document")
@@ -279,7 +279,7 @@ def fake_parse_wrapper(parser, path, mime_type, file_name):
parser.text = "the text"
@override_settings(PAPERLESS_FILENAME_FORMAT="")
@override_settings(FILENAME_FORMAT="")
class TestMigrateArchiveFilesErrors(DirectoriesMixin, TestMigrations):
migrate_from = "1011_auto_20210101_2340"
@@ -447,7 +447,7 @@ class TestMigrateArchiveFilesErrors(DirectoriesMixin, TestMigrations):
self.assertIsNone(doc2.archive_filename)
@override_settings(PAPERLESS_FILENAME_FORMAT="")
@override_settings(FILENAME_FORMAT="")
class TestMigrateArchiveFilesBackwards(DirectoriesMixin, TestMigrations):
migrate_from = "1012_fix_archive_files"
@@ -505,14 +505,14 @@ class TestMigrateArchiveFilesBackwards(DirectoriesMixin, TestMigrations):
)
@override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{title}")
@override_settings(FILENAME_FORMAT="{correspondent}/{title}")
class TestMigrateArchiveFilesBackwardsWithFilenameFormat(
TestMigrateArchiveFilesBackwards,
):
pass
@override_settings(PAPERLESS_FILENAME_FORMAT="")
@override_settings(FILENAME_FORMAT="")
class TestMigrateArchiveFilesBackwardsErrors(DirectoriesMixin, TestMigrations):
migrate_from = "1012_fix_archive_files"

94
src/documents/views.py
View File

@@ -11,6 +11,7 @@ from unicodedata import normalize
from urllib.parse import quote
from django.conf import settings
from django.contrib.auth.models import User
from django.db.models import Case
from django.db.models import Count
from django.db.models import IntegerField
@@ -54,14 +55,17 @@ from .classifier import load_classifier
from .filters import CorrespondentFilterSet
from .filters import DocumentFilterSet
from .filters import DocumentTypeFilterSet
from .filters import StoragePathFilterSet
from .filters import TagFilterSet
from .matching import match_correspondents
from .matching import match_document_types
from .matching import match_storage_paths
from .matching import match_tags
from .models import Correspondent
from .models import Document
from .models import DocumentType
from .models import SavedView
from .models import StoragePath
from .models import Tag
from .parsers import get_parser_class_for_mime_type
from .serialisers import BulkDownloadSerializer
@@ -72,8 +76,10 @@ from .serialisers import DocumentSerializer
from .serialisers import DocumentTypeSerializer
from .serialisers import PostDocumentSerializer
from .serialisers import SavedViewSerializer
from .serialisers import StoragePathSerializer
from .serialisers import TagSerializer
from .serialisers import TagSerializerVersion1
from .serialisers import UiSettingsViewSerializer
logger = logging.getLogger("paperless.api")
@@ -81,12 +87,18 @@ logger = logging.getLogger("paperless.api")
class IndexView(TemplateView):
template_name = "index.html"
def get_language(self):
def get_frontend_language(self):
if hasattr(
self.request.user,
"ui_settings",
) and self.request.user.ui_settings.settings.get("language"):
lang = self.request.user.ui_settings.settings.get("language")
else:
lang = get_language()
# This is here for the following reason:
# Django identifies languages in the form "en-us"
# However, angular generates locales as "en-US".
# this translates between these two forms.
lang = get_language()
if "-" in lang:
first = lang[: lang.index("-")]
second = lang[lang.index("-") + 1 :]
@@ -99,16 +111,18 @@ class IndexView(TemplateView):
context["cookie_prefix"] = settings.COOKIE_PREFIX
context["username"] = self.request.user.username
context["full_name"] = self.request.user.get_full_name()
context["styles_css"] = f"frontend/{self.get_language()}/styles.css"
context["runtime_js"] = f"frontend/{self.get_language()}/runtime.js"
context["polyfills_js"] = f"frontend/{self.get_language()}/polyfills.js"
context["main_js"] = f"frontend/{self.get_language()}/main.js"
context["styles_css"] = f"frontend/{self.get_frontend_language()}/styles.css"
context["runtime_js"] = f"frontend/{self.get_frontend_language()}/runtime.js"
context[
"polyfills_js"
] = f"frontend/{self.get_frontend_language()}/polyfills.js"
context["main_js"] = f"frontend/{self.get_frontend_language()}/main.js"
context[
"webmanifest"
] = f"frontend/{self.get_language()}/manifest.webmanifest" # noqa: E501
] = f"frontend/{self.get_frontend_language()}/manifest.webmanifest" # noqa: E501
context[
"apple_touch_icon"
] = f"frontend/{self.get_language()}/apple-touch-icon.png" # noqa: E501
] = f"frontend/{self.get_frontend_language()}/apple-touch-icon.png" # noqa: E501
return context
@@ -325,6 +339,7 @@ class DocumentViewSet(
"document_types": [
dt.id for dt in match_document_types(doc, classifier)
],
"storage_paths": [dt.id for dt in match_storage_paths(doc, classifier)],
},
)
@@ -567,6 +582,12 @@ class SelectionDataView(GenericAPIView):
),
)
storage_paths = StoragePath.objects.annotate(
document_count=Count(
Case(When(documents__id__in=ids, then=1), output_field=IntegerField()),
),
)
r = Response(
{
"selected_correspondents": [
@@ -579,6 +600,10 @@ class SelectionDataView(GenericAPIView):
"selected_document_types": [
{"id": t.id, "document_count": t.document_count} for t in types
],
"selected_storage_paths": [
{"id": t.id, "document_count": t.document_count}
for t in storage_paths
],
},
)
@@ -717,3 +742,56 @@ class RemoteVersionView(GenericAPIView):
"feature_is_set": feature_is_set,
},
)
class StoragePathViewSet(ModelViewSet):
model = DocumentType
queryset = StoragePath.objects.annotate(document_count=Count("documents")).order_by(
Lower("name"),
)
serializer_class = StoragePathSerializer
pagination_class = StandardPagination
permission_classes = (IsAuthenticated,)
filter_backends = (DjangoFilterBackend, OrderingFilter)
filterset_class = StoragePathFilterSet
ordering_fields = ("name", "path", "matching_algorithm", "match", "document_count")
class UiSettingsView(GenericAPIView):
permission_classes = (IsAuthenticated,)
serializer_class = UiSettingsViewSerializer
def get(self, request, format=None):
serializer = self.get_serializer(data=request.data)
serializer.is_valid(raise_exception=True)
user = User.objects.get(pk=request.user.id)
displayname = user.username
if user.first_name or user.last_name:
displayname = " ".join([user.first_name, user.last_name])
settings = {}
if hasattr(user, "ui_settings"):
settings = user.ui_settings.settings
return Response(
{
"user_id": user.id,
"username": user.username,
"display_name": displayname,
"settings": settings,
},
)
def post(self, request, format=None):
serializer = self.get_serializer(data=request.data)
serializer.is_valid(raise_exception=True)
serializer.save(user=self.request.user)
return Response(
{
"success": True,
},
)

382
src/locale/en_US/LC_MESSAGES/django.po
View File

@@ -2,7 +2,7 @@ msgid ""
msgstr ""
"Project-Id-Version: paperless-ngx\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2022-03-02 11:20-0800\n"
"POT-Creation-Date: 2022-05-19 15:24-0700\n"
"PO-Revision-Date: 2022-02-17 04:17\n"
"Last-Translator: \n"
"Language-Team: English\n"
@@ -17,373 +17,397 @@ msgstr ""
"X-Crowdin-File: /dev/src/locale/en_US/LC_MESSAGES/django.po\n"
"X-Crowdin-File-ID: 14\n"
#: documents/apps.py:10
#: documents/apps.py:9
msgid "Documents"
msgstr ""
#: documents/models.py:32
#: documents/models.py:27
msgid "Any word"
msgstr ""
#: documents/models.py:33
#: documents/models.py:28
msgid "All words"
msgstr ""
#: documents/models.py:34
#: documents/models.py:29
msgid "Exact match"
msgstr ""
#: documents/models.py:35
#: documents/models.py:30
msgid "Regular expression"
msgstr ""
#: documents/models.py:36
#: documents/models.py:31
msgid "Fuzzy word"
msgstr ""
#: documents/models.py:37
#: documents/models.py:32
msgid "Automatic"
msgstr ""
#: documents/models.py:40 documents/models.py:314 paperless_mail/models.py:23
#: paperless_mail/models.py:107
#: documents/models.py:35 documents/models.py:343 paperless_mail/models.py:16
#: paperless_mail/models.py:79
msgid "name"
msgstr ""
#: documents/models.py:42
#: documents/models.py:37
msgid "match"
msgstr ""
#: documents/models.py:45
#: documents/models.py:40
msgid "matching algorithm"
msgstr ""
#: documents/models.py:48
#: documents/models.py:45
msgid "is insensitive"
msgstr ""
#: documents/models.py:61 documents/models.py:104
#: documents/models.py:58 documents/models.py:113
msgid "correspondent"
msgstr ""
#: documents/models.py:62
#: documents/models.py:59
msgid "correspondents"
msgstr ""
#: documents/models.py:67
#: documents/models.py:64
msgid "color"
msgstr ""
#: documents/models.py:70
#: documents/models.py:67
msgid "is inbox tag"
msgstr ""
#: documents/models.py:73
#: documents/models.py:70
msgid ""
"Marks this tag as an inbox tag: All newly consumed documents will be tagged "
"with inbox tags."
msgstr ""
#: documents/models.py:79
#: documents/models.py:76
msgid "tag"
msgstr ""
#: documents/models.py:80 documents/models.py:130
#: documents/models.py:77 documents/models.py:151
msgid "tags"
msgstr ""
#: documents/models.py:85 documents/models.py:115
#: documents/models.py:82 documents/models.py:133
msgid "document type"
msgstr ""
#: documents/models.py:86
#: documents/models.py:83
msgid "document types"
msgstr ""
#: documents/models.py:94
msgid "Unencrypted"
#: documents/models.py:88
msgid "path"
msgstr ""
#: documents/models.py:94 documents/models.py:122
msgid "storage path"
msgstr ""
#: documents/models.py:95
msgid "storage paths"
msgstr ""
#: documents/models.py:103
msgid "Unencrypted"
msgstr ""
#: documents/models.py:104
msgid "Encrypted with GNU Privacy Guard"
msgstr ""
#: documents/models.py:107
#: documents/models.py:125
msgid "title"
msgstr ""
#: documents/models.py:119
#: documents/models.py:137
msgid "content"
msgstr ""
#: documents/models.py:122
#: documents/models.py:140
msgid ""
"The raw, text-only data of the document. This field is primarily used for "
"searching."
msgstr ""
#: documents/models.py:127
#: documents/models.py:145
msgid "mime type"
msgstr ""
#: documents/models.py:134
#: documents/models.py:155
msgid "checksum"
msgstr ""
#: documents/models.py:138
#: documents/models.py:159
msgid "The checksum of the original document."
msgstr ""
#: documents/models.py:142
#: documents/models.py:163
msgid "archive checksum"
msgstr ""
#: documents/models.py:147
#: documents/models.py:168
msgid "The checksum of the archived document."
msgstr ""
#: documents/models.py:150 documents/models.py:295
#: documents/models.py:171 documents/models.py:324
msgid "created"
msgstr ""
#: documents/models.py:153
#: documents/models.py:174
msgid "modified"
msgstr ""
#: documents/models.py:157
#: documents/models.py:181
msgid "storage type"
msgstr ""
#: documents/models.py:165
#: documents/models.py:189
msgid "added"
msgstr ""
#: documents/models.py:169
#: documents/models.py:196
msgid "filename"
msgstr ""
#: documents/models.py:175
#: documents/models.py:202
msgid "Current filename in storage"
msgstr ""
#: documents/models.py:179
#: documents/models.py:206
msgid "archive filename"
msgstr ""
#: documents/models.py:185
#: documents/models.py:212
msgid "Current archive filename in storage"
msgstr ""
#: documents/models.py:189
#: documents/models.py:216
msgid "archive serial number"
msgstr ""
#: documents/models.py:195
#: documents/models.py:222
msgid "The position of this document in your physical document archive."
msgstr ""
#: documents/models.py:201
#: documents/models.py:228
msgid "document"
msgstr ""
#: documents/models.py:202
#: documents/models.py:229
msgid "documents"
msgstr ""
#: documents/models.py:280
#: documents/models.py:307
msgid "debug"
msgstr ""
#: documents/models.py:281
#: documents/models.py:308
msgid "information"
msgstr ""
#: documents/models.py:282
#: documents/models.py:309
msgid "warning"
msgstr ""
#: documents/models.py:283
#: documents/models.py:310
msgid "error"
msgstr ""
#: documents/models.py:284
#: documents/models.py:311
msgid "critical"
msgstr ""
#: documents/models.py:287
#: documents/models.py:314
msgid "group"
msgstr ""
#: documents/models.py:289
#: documents/models.py:316
msgid "message"
msgstr ""
#: documents/models.py:292
#: documents/models.py:319
msgid "level"
msgstr ""
#: documents/models.py:299
#: documents/models.py:328
msgid "log"
msgstr ""
#: documents/models.py:300
#: documents/models.py:329
msgid "logs"
msgstr ""
#: documents/models.py:310 documents/models.py:360
#: documents/models.py:339 documents/models.py:392
msgid "saved view"
msgstr ""
#: documents/models.py:311
#: documents/models.py:340
msgid "saved views"
msgstr ""
#: documents/models.py:313
#: documents/models.py:342
msgid "user"
msgstr ""
#: documents/models.py:317
#: documents/models.py:346
msgid "show on dashboard"
msgstr ""
#: documents/models.py:320
#: documents/models.py:349
msgid "show in sidebar"
msgstr ""
#: documents/models.py:324
#: documents/models.py:353
msgid "sort field"
msgstr ""
#: documents/models.py:326
#: documents/models.py:358
msgid "sort reverse"
msgstr ""
#: documents/models.py:331
#: documents/models.py:363
msgid "title contains"
msgstr ""
#: documents/models.py:332
#: documents/models.py:364
msgid "content contains"
msgstr ""
#: documents/models.py:333
#: documents/models.py:365
msgid "ASN is"
msgstr ""
#: documents/models.py:334
#: documents/models.py:366
msgid "correspondent is"
msgstr ""
#: documents/models.py:335
#: documents/models.py:367
msgid "document type is"
msgstr ""
#: documents/models.py:336
#: documents/models.py:368
msgid "is in inbox"
msgstr ""
#: documents/models.py:337
#: documents/models.py:369
msgid "has tag"
msgstr ""
#: documents/models.py:338
#: documents/models.py:370
msgid "has any tag"
msgstr ""
#: documents/models.py:339
#: documents/models.py:371
msgid "created before"
msgstr ""
#: documents/models.py:340
#: documents/models.py:372
msgid "created after"
msgstr ""
#: documents/models.py:341
#: documents/models.py:373
msgid "created year is"
msgstr ""
#: documents/models.py:342
#: documents/models.py:374
msgid "created month is"
msgstr ""
#: documents/models.py:343
#: documents/models.py:375
msgid "created day is"
msgstr ""
#: documents/models.py:344
#: documents/models.py:376
msgid "added before"
msgstr ""
#: documents/models.py:345
#: documents/models.py:377
msgid "added after"
msgstr ""
#: documents/models.py:346
#: documents/models.py:378
msgid "modified before"
msgstr ""
#: documents/models.py:347
#: documents/models.py:379
msgid "modified after"
msgstr ""
#: documents/models.py:348
#: documents/models.py:380
msgid "does not have tag"
msgstr ""
#: documents/models.py:349
#: documents/models.py:381
msgid "does not have ASN"
msgstr ""
#: documents/models.py:350
#: documents/models.py:382
msgid "title or content contains"
msgstr ""
#: documents/models.py:351
#: documents/models.py:383
msgid "fulltext query"
msgstr ""
#: documents/models.py:352
#: documents/models.py:384
msgid "more like this"
msgstr ""
#: documents/models.py:353
#: documents/models.py:385
msgid "has tags in"
msgstr ""
#: documents/models.py:363
#: documents/models.py:395
msgid "rule type"
msgstr ""
#: documents/models.py:365
#: documents/models.py:397
msgid "value"
msgstr ""
#: documents/models.py:368
#: documents/models.py:400
msgid "filter rule"
msgstr ""
#: documents/models.py:369
#: documents/models.py:401
msgid "filter rules"
msgstr ""
#: documents/serialisers.py:64
#: documents/serialisers.py:63
#, python-format
msgid "Invalid regular expression: %(error)s"
msgstr ""
#: documents/serialisers.py:185
#: documents/serialisers.py:184
msgid "Invalid color."
msgstr ""
#: documents/serialisers.py:459
#: documents/serialisers.py:491
#, python-format
msgid "File type %(type)s not supported"
msgstr ""
#: documents/templates/index.html:22
#: documents/serialisers.py:574
msgid "Invalid variable detected."
msgstr ""
#: documents/templates/index.html:78
msgid "Paperless-ngx is loading..."
msgstr ""
#: documents/templates/index.html:79
msgid "Still here?! Hmm, something might be wrong."
msgstr ""
#: documents/templates/index.html:79
msgid "Here's a link to the docs."
msgstr ""
#: documents/templates/registration/logged_out.html:14
msgid "Paperless-ngx signed out"
msgstr ""
@@ -420,71 +444,91 @@ msgstr ""
msgid "Sign in"
msgstr ""
#: paperless/settings.py:299
#: paperless/settings.py:338
msgid "English (US)"
msgstr ""
#: paperless/settings.py:300
#: paperless/settings.py:339
msgid "Belarusian"
msgstr ""
#: paperless/settings.py:340
msgid "Czech"
msgstr ""
#: paperless/settings.py:301
#: paperless/settings.py:341
msgid "Danish"
msgstr ""
#: paperless/settings.py:302
#: paperless/settings.py:342
msgid "German"
msgstr ""
#: paperless/settings.py:303
#: paperless/settings.py:343
msgid "English (GB)"
msgstr ""
#: paperless/settings.py:304
#: paperless/settings.py:344
msgid "Spanish"
msgstr ""
#: paperless/settings.py:305
#: paperless/settings.py:345
msgid "French"
msgstr ""
#: paperless/settings.py:306
#: paperless/settings.py:346
msgid "Italian"
msgstr ""
#: paperless/settings.py:307
#: paperless/settings.py:347
msgid "Luxembourgish"
msgstr ""
#: paperless/settings.py:308
#: paperless/settings.py:348
msgid "Dutch"
msgstr ""
#: paperless/settings.py:309
#: paperless/settings.py:349
msgid "Polish"
msgstr ""
#: paperless/settings.py:310
#: paperless/settings.py:350
msgid "Portuguese (Brazil)"
msgstr ""
#: paperless/settings.py:311
#: paperless/settings.py:351
msgid "Portuguese"
msgstr ""
#: paperless/settings.py:312
#: paperless/settings.py:352
msgid "Romanian"
msgstr ""
#: paperless/settings.py:313
#: paperless/settings.py:353
msgid "Russian"
msgstr ""
#: paperless/settings.py:314
#: paperless/settings.py:354
msgid "Slovenian"
msgstr ""
#: paperless/settings.py:355
msgid "Serbian"
msgstr ""
#: paperless/settings.py:356
msgid "Swedish"
msgstr ""
#: paperless/urls.py:139
#: paperless/settings.py:357
msgid "Turkish"
msgstr ""
#: paperless/settings.py:358
msgid "Chinese Simplified"
msgstr ""
#: paperless/urls.py:153
msgid "Paperless-ngx administration"
msgstr ""
@@ -527,208 +571,210 @@ msgid ""
"process all matching rules that you have defined."
msgstr ""
#: paperless_mail/apps.py:9
#: paperless_mail/apps.py:8
msgid "Paperless mail"
msgstr ""
#: paperless_mail/models.py:10
#: paperless_mail/models.py:8
msgid "mail account"
msgstr ""
#: paperless_mail/models.py:11
#: paperless_mail/models.py:9
msgid "mail accounts"
msgstr ""
#: paperless_mail/models.py:18
#: paperless_mail/models.py:12
msgid "No encryption"
msgstr ""
#: paperless_mail/models.py:19
#: paperless_mail/models.py:13
msgid "Use SSL"
msgstr ""
#: paperless_mail/models.py:20
#: paperless_mail/models.py:14
msgid "Use STARTTLS"
msgstr ""
#: paperless_mail/models.py:25
#: paperless_mail/models.py:18
msgid "IMAP server"
msgstr ""
#: paperless_mail/models.py:28
#: paperless_mail/models.py:21
msgid "IMAP port"
msgstr ""
#: paperless_mail/models.py:32
#: paperless_mail/models.py:25
msgid ""
"This is usually 143 for unencrypted and STARTTLS connections, and 993 for "
"SSL connections."
msgstr ""
#: paperless_mail/models.py:38
#: paperless_mail/models.py:31
msgid "IMAP security"
msgstr ""
#: paperless_mail/models.py:41
#: paperless_mail/models.py:36
msgid "username"
msgstr ""
#: paperless_mail/models.py:43
#: paperless_mail/models.py:38
msgid "password"
msgstr ""
#: paperless_mail/models.py:46
#: paperless_mail/models.py:41
msgid "character set"
msgstr ""
#: paperless_mail/models.py:50
#: paperless_mail/models.py:45
msgid ""
"The character set to use when communicating with the mail server, such as "
"'UTF-8' or 'US-ASCII'."
msgstr ""
#: paperless_mail/models.py:61
#: paperless_mail/models.py:56
msgid "mail rule"
msgstr ""
#: paperless_mail/models.py:62
#: paperless_mail/models.py:57
msgid "mail rules"
msgstr ""
#: paperless_mail/models.py:68
#: paperless_mail/models.py:60
msgid "Only process attachments."
msgstr ""
#: paperless_mail/models.py:71
#: paperless_mail/models.py:61
msgid "Process all files, including 'inline' attachments."
msgstr ""
#: paperless_mail/models.py:81
msgid "Mark as read, don't process read mails"
msgstr ""
#: paperless_mail/models.py:82
msgid "Flag the mail, don't process flagged mails"
msgstr ""
#: paperless_mail/models.py:83
msgid "Move to specified folder"
msgstr ""
#: paperless_mail/models.py:84
#: paperless_mail/models.py:64
msgid "Delete"
msgstr ""
#: paperless_mail/models.py:91
#: paperless_mail/models.py:65
msgid "Move to specified folder"
msgstr ""
#: paperless_mail/models.py:66
msgid "Mark as read, don't process read mails"
msgstr ""
#: paperless_mail/models.py:67
msgid "Flag the mail, don't process flagged mails"
msgstr ""
#: paperless_mail/models.py:70
msgid "Use subject as title"
msgstr ""
#: paperless_mail/models.py:92
#: paperless_mail/models.py:71
msgid "Use attachment filename as title"
msgstr ""
#: paperless_mail/models.py:101
#: paperless_mail/models.py:74
msgid "Do not assign a correspondent"
msgstr ""
#: paperless_mail/models.py:102
#: paperless_mail/models.py:75
msgid "Use mail address"
msgstr ""
#: paperless_mail/models.py:103
#: paperless_mail/models.py:76
msgid "Use name (or mail address if not available)"
msgstr ""
#: paperless_mail/models.py:104
#: paperless_mail/models.py:77
msgid "Use correspondent selected below"
msgstr ""
#: paperless_mail/models.py:109
#: paperless_mail/models.py:81
msgid "order"
msgstr ""
#: paperless_mail/models.py:115
#: paperless_mail/models.py:87
msgid "account"
msgstr ""
#: paperless_mail/models.py:119
#: paperless_mail/models.py:91
msgid "folder"
msgstr ""
#: paperless_mail/models.py:122
msgid "Subfolders must be separated by dots."
#: paperless_mail/models.py:95
msgid ""
"Subfolders must be separated by a delimiter, often a dot ('.') or slash "
"('/'), but it varies by mail server."
msgstr ""
#: paperless_mail/models.py:126
#: paperless_mail/models.py:101
msgid "filter from"
msgstr ""
#: paperless_mail/models.py:129
#: paperless_mail/models.py:107
msgid "filter subject"
msgstr ""
#: paperless_mail/models.py:132
#: paperless_mail/models.py:113
msgid "filter body"
msgstr ""
#: paperless_mail/models.py:136
#: paperless_mail/models.py:120
msgid "filter attachment filename"
msgstr ""
#: paperless_mail/models.py:141
#: paperless_mail/models.py:125
msgid ""
"Only consume documents which entirely match this filename if specified. "
"Wildcards such as *.pdf or *invoice* are allowed. Case insensitive."
msgstr ""
#: paperless_mail/models.py:148
#: paperless_mail/models.py:132
msgid "maximum age"
msgstr ""
#: paperless_mail/models.py:148
#: paperless_mail/models.py:134
msgid "Specified in days."
msgstr ""
#: paperless_mail/models.py:152
#: paperless_mail/models.py:138
msgid "attachment type"
msgstr ""
#: paperless_mail/models.py:156
#: paperless_mail/models.py:142
msgid ""
"Inline attachments include embedded images, so it's best to combine this "
"option with a filename filter."
msgstr ""
#: paperless_mail/models.py:162
#: paperless_mail/models.py:148
msgid "action"
msgstr ""
#: paperless_mail/models.py:168
#: paperless_mail/models.py:154
msgid "action parameter"
msgstr ""
#: paperless_mail/models.py:173
#: paperless_mail/models.py:159
msgid ""
"Additional parameter for the action selected above, i.e., the target folder "
"of the move to folder action. Subfolders must be separated by dots."
msgstr ""
#: paperless_mail/models.py:181
#: paperless_mail/models.py:167
msgid "assign title from"
msgstr ""
#: paperless_mail/models.py:189
#: paperless_mail/models.py:175
msgid "assign this tag"
msgstr ""
#: paperless_mail/models.py:197
#: paperless_mail/models.py:183
msgid "assign this document type"
msgstr ""
#: paperless_mail/models.py:201
#: paperless_mail/models.py:187
msgid "assign correspondent from"
msgstr ""
#: paperless_mail/models.py:211
#: paperless_mail/models.py:197
msgid "assign this correspondent"
msgstr ""

19
src/paperless/settings.py
View File

@@ -5,6 +5,7 @@ import multiprocessing
import os
import re
from typing import Final
from typing import Optional
from typing import Set
from urllib.parse import urlparse
@@ -551,10 +552,9 @@ OCR_ROTATE_PAGES_THRESHOLD = float(
os.getenv("PAPERLESS_OCR_ROTATE_PAGES_THRESHOLD", 12.0),
)
OCR_MAX_IMAGE_PIXELS = os.environ.get(
"PAPERLESS_OCR_MAX_IMAGE_PIXELS",
256000000,
)
OCR_MAX_IMAGE_PIXELS: Optional[int] = None
if os.environ.get("PAPERLESS_OCR_MAX_IMAGE_PIXELS") is not None:
OCR_MAX_IMAGE_PIXELS: int = int(os.environ.get("PAPERLESS_OCR_MAX_IMAGE_PIXELS"))
OCR_USER_ARGS = os.getenv("PAPERLESS_OCR_USER_ARGS", "{}")
@@ -597,15 +597,22 @@ FILENAME_PARSE_TRANSFORMS = []
for t in json.loads(os.getenv("PAPERLESS_FILENAME_PARSE_TRANSFORMS", "[]")):
FILENAME_PARSE_TRANSFORMS.append((re.compile(t["pattern"]), t["repl"]))
# TODO: this should not have a prefix.
# Specify the filename format for out files
PAPERLESS_FILENAME_FORMAT = os.getenv("PAPERLESS_FILENAME_FORMAT")
FILENAME_FORMAT = os.getenv("PAPERLESS_FILENAME_FORMAT")
# If this is enabled, variables in filename format will resolve to empty-string instead of 'none'.
# Directories with 'empty names' are omitted, too.
FILENAME_FORMAT_REMOVE_NONE = __get_boolean(
"PAPERLESS_FILENAME_FORMAT_REMOVE_NONE",
"NO",
)
THUMBNAIL_FONT_NAME = os.getenv(
"PAPERLESS_THUMBNAIL_FONT_NAME",
"/usr/share/fonts/liberation/LiberationSerif-Regular.ttf",
)
# TODO: this should not have a prefix.
# Tika settings
PAPERLESS_TIKA_ENABLED = __get_boolean("PAPERLESS_TIKA_ENABLED", "NO")
PAPERLESS_TIKA_ENDPOINT = os.getenv("PAPERLESS_TIKA_ENDPOINT", "http://localhost:9998")

8
src/paperless/urls.py
View File

@@ -19,7 +19,9 @@ from documents.views import SavedViewViewSet
from documents.views import SearchAutoCompleteView
from documents.views import SelectionDataView
from documents.views import StatisticsView
from documents.views import StoragePathViewSet
from documents.views import TagViewSet
from documents.views import UiSettingsView
from documents.views import UnifiedSearchViewSet
from paperless.consumers import StatusConsumer
from paperless.views import FaviconView
@@ -33,6 +35,7 @@ api_router.register(r"documents", UnifiedSearchViewSet)
api_router.register(r"logs", LogViewSet, basename="logs")
api_router.register(r"tags", TagViewSet)
api_router.register(r"saved_views", SavedViewViewSet)
api_router.register(r"storage_paths", StoragePathViewSet)
urlpatterns = [
@@ -78,6 +81,11 @@ urlpatterns = [
RemoteVersionView.as_view(),
name="remoteversion",
),
re_path(
r"^ui_settings/",
UiSettingsView.as_view(),
name="ui_settings",
),
path("token/", views.obtain_auth_token),
]
+ api_router.urls,

20
src/paperless_tesseract/parsers.py
View File

@@ -8,8 +8,6 @@ from documents.parsers import make_thumbnail_from_pdf
from documents.parsers import ParseError
from PIL import Image
Image.MAX_IMAGE_PIXELS = settings.OCR_MAX_IMAGE_PIXELS
class NoTextFoundException(Exception):
pass
@@ -225,6 +223,24 @@ class RasterisedDocumentParser(DocumentParser):
f"they will not be used. Error: {e}",
)
if settings.OCR_MAX_IMAGE_PIXELS is not None:
# Convert pixels to mega-pixels and provide to ocrmypdf
max_pixels_mpixels = settings.OCR_MAX_IMAGE_PIXELS / 1_000_000.0
if max_pixels_mpixels > 0:
self.log(
"debug",
f"Calculated {max_pixels_mpixels} megapixels for OCR",
)
ocrmypdf_args["max_image_mpixels"] = max_pixels_mpixels
else:
self.log(
"warning",
"There is an issue with PAPERLESS_OCR_MAX_IMAGE_PIXELS, "
"this value must be at least 1 megapixel if set",
)
return ocrmypdf_args
def parse(self, document_path, mime_type, file_name=None):

4
src/paperless_text/parsers.py
View File

@@ -6,8 +6,6 @@ from PIL import Image
from PIL import ImageDraw
from PIL import ImageFont
Image.MAX_IMAGE_PIXELS = settings.OCR_MAX_IMAGE_PIXELS
class TextDocumentParser(DocumentParser):
"""
@@ -28,7 +26,7 @@ class TextDocumentParser(DocumentParser):
font = ImageFont.truetype(
font=settings.THUMBNAIL_FONT_NAME,
size=20,
layout_engine=ImageFont.LAYOUT_BASIC,
layout_engine=ImageFont.Layout.BASIC,
)
draw.text((5, 5), read_text(), font=font, fill="black")