paradizelost/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2025-07-28 18:24:38 -05:00
Code Issues Packages Projects Releases Wiki Activity

Feature: Enhanced templating for filename format (#7836)

Browse Source 
Co-authored-by: shamoon <4887959+shamoon@users.noreply.github.com>
This commit is contained in:
Trenton H
2024-10-06 12:54:01 -07:00
committed by GitHub
parent e49ed58f1a
commit 7c11a37150
29 changed files with 1299 additions and 615 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
src/documents/checks.py
View File

@@ -2,12 +2,14 @@ import textwrap
from django.conf import settings
from django.core.checks import Error
from django.core.checks import Warning
from django.core.checks import register
from django.core.exceptions import FieldError
from django.db.utils import OperationalError
from django.db.utils import ProgrammingError
from documents.signals import document_consumer_declaration
from documents.templating.utils import convert_format_str_to_template_format
@register()
@@ -69,3 +71,19 @@ def parser_check(app_configs, **kwargs):
]
else:
return []
@register()
def filename_format_check(app_configs, **kwargs):
if settings.FILENAME_FORMAT:
converted_format = convert_format_str_to_template_format(
settings.FILENAME_FORMAT,
)
if converted_format != settings.FILENAME_FORMAT:
return [
Warning(
f"Filename format {settings.FILENAME_FORMAT} is using the old style, please update to use double curly brackets",
hint=converted_format,
),
]
return []

176
src/documents/file_handling.py
View File

@@ -1,21 +1,10 @@
import logging
import os
from collections import defaultdict
from pathlib import PurePath
import pathvalidate
from django.conf import settings
from django.template.defaultfilters import slugify
from django.utils import timezone
from documents.models import Document
logger = logging.getLogger("paperless.filehandling")
class defaultdictNoStr(defaultdict):
def __str__(self):
raise ValueError("Don't use {tags} directly.")
from documents.templating.filepath import validate_filepath_template_and_render
from documents.templating.utils import convert_format_str_to_template_format
def create_source_path_directory(source_path):
@@ -54,32 +43,6 @@ def delete_empty_directories(directory, root):
directory = os.path.normpath(os.path.dirname(directory))
def many_to_dictionary(field):
# Converts ManyToManyField to dictionary by assuming, that field
# entries contain an _ or - which will be used as a delimiter
mydictionary = dict()
for index, t in enumerate(field.all()):
# Populate tag names by index
mydictionary[index] = slugify(t.name)
# Find delimiter
delimiter = t.name.find("_")
if delimiter == -1:
delimiter = t.name.find("-")
if delimiter == -1:
continue
key = t.name[:delimiter]
value = t.name[delimiter + 1 :]
mydictionary[slugify(key)] = slugify(value)
return mydictionary
def generate_unique_filename(doc, archive_filename=False):
"""
Generates a unique filename for doc in settings.ORIGINALS_DIR.
@@ -134,116 +97,51 @@ def generate_filename(
archive_filename=False,
):
path = ""
filename_format = settings.FILENAME_FORMAT
try:
if doc.storage_path is not None:
logger.debug(
f"Document has storage_path {doc.storage_path.id} "
f"({doc.storage_path.path}) set",
)
filename_format = doc.storage_path.path
if filename_format is not None:
tags = defaultdictNoStr(
lambda: slugify(None),
many_to_dictionary(doc.tags),
)
tag_list = pathvalidate.sanitize_filename(
",".join(
sorted(tag.name for tag in doc.tags.all()),
),
replacement_text="-",
)
no_value_default = "-none-"
if doc.correspondent:
correspondent = pathvalidate.sanitize_filename(
doc.correspondent.name,
replacement_text="-",
)
else:
correspondent = no_value_default
if doc.document_type:
document_type = pathvalidate.sanitize_filename(
doc.document_type.name,
replacement_text="-",
)
else:
document_type = no_value_default
if doc.archive_serial_number:
asn = str(doc.archive_serial_number)
else:
asn = no_value_default
if doc.owner is not None:
owner_username_str = str(doc.owner.username)
else:
owner_username_str = no_value_default
if doc.original_filename is not None:
# No extension
original_name = PurePath(doc.original_filename).with_suffix("").name
else:
original_name = no_value_default
# Convert UTC database datetime to localized date
local_added = timezone.localdate(doc.added)
local_created = timezone.localdate(doc.created)
path = filename_format.format(
title=pathvalidate.sanitize_filename(doc.title, replacement_text="-"),
correspondent=correspondent,
document_type=document_type,
created=local_created.isoformat(),
created_year=local_created.strftime("%Y"),
created_year_short=local_created.strftime("%y"),
created_month=local_created.strftime("%m"),
created_month_name=local_created.strftime("%B"),
created_month_name_short=local_created.strftime("%b"),
created_day=local_created.strftime("%d"),
added=local_added.isoformat(),
added_year=local_added.strftime("%Y"),
added_year_short=local_added.strftime("%y"),
added_month=local_added.strftime("%m"),
added_month_name=local_added.strftime("%B"),
added_month_name_short=local_added.strftime("%b"),
added_day=local_added.strftime("%d"),
asn=asn,
tags=tags,
tag_list=tag_list,
owner_username=owner_username_str,
original_name=original_name,
doc_pk=f"{doc.pk:07}",
).strip()
if settings.FILENAME_FORMAT_REMOVE_NONE:
path = path.replace("/-none-/", "/") # remove empty directories
path = path.replace(" -none-", "") # remove when spaced, with space
path = path.replace("-none-", "") # remove rest of the occurrences
path = path.replace("-none-", "none") # backward compatibility
path = path.strip(os.sep)
except (ValueError, KeyError, IndexError):
logger.warning(
f"Invalid filename_format '{filename_format}', falling back to default",
def format_filename(document: Document, template_str: str) -> str | None:
rendered_filename = validate_filepath_template_and_render(
template_str,
document,
)
if rendered_filename is None:
return None
# Apply this setting. It could become a filter in the future (or users could use |default)
if settings.FILENAME_FORMAT_REMOVE_NONE:
rendered_filename = rendered_filename.replace("/-none-/", "/")
rendered_filename = rendered_filename.replace(" -none-", "")
rendered_filename = rendered_filename.replace("-none-", "")
rendered_filename = rendered_filename.replace(
"-none-",
"none",
) # backward compatibility
return rendered_filename
# Determine the source of the format string
if doc.storage_path is not None:
filename_format = doc.storage_path.path
elif settings.FILENAME_FORMAT is not None:
# Maybe convert old to new style
filename_format = convert_format_str_to_template_format(
settings.FILENAME_FORMAT,
)
else:
filename_format = None
# If we have one, render it
if filename_format is not None:
path = format_filename(doc, filename_format)
counter_str = f"_{counter:02}" if counter else ""
filetype_str = ".pdf" if archive_filename else doc.file_type
if len(path) > 0:
if path:
filename = f"{path}{counter_str}{filetype_str}"
else:
filename = f"{doc.pk:07}{counter_str}{filetype_str}"
# Append .gpg for encrypted files
if append_gpg and doc.storage_type == doc.STORAGE_TYPE_GPG:
filename += ".gpg"

34
src/documents/migrations/1012_fix_archive_files.py
View File

@@ -4,6 +4,7 @@ import hashlib
import logging
import os
import shutil
from collections import defaultdict
from time import sleep
import pathvalidate
@@ -12,14 +13,41 @@ from django.db import migrations
from django.db import models
from django.template.defaultfilters import slugify
from documents.file_handling import defaultdictNoStr
from documents.file_handling import many_to_dictionary
logger = logging.getLogger("paperless.migrations")
###############################################################################
# This is code copied straight paperless before the change.
###############################################################################
class defaultdictNoStr(defaultdict):
def __str__(self): # pragma: no cover
raise ValueError("Don't use {tags} directly.")
def many_to_dictionary(field): # pragma: no cover
# Converts ManyToManyField to dictionary by assuming, that field
# entries contain an _ or - which will be used as a delimiter
mydictionary = dict()
for index, t in enumerate(field.all()):
# Populate tag names by index
mydictionary[index] = slugify(t.name)
# Find delimiter
delimiter = t.name.find("_")
if delimiter == -1:
delimiter = t.name.find("-")
if delimiter == -1:
continue
key = t.name[:delimiter]
value = t.name[delimiter + 1 :]
mydictionary[slugify(key)] = slugify(value)
return mydictionary
def archive_name_from_filename(filename):

36
src/documents/migrations/1055_alter_storagepath_path.py Normal file
View File

@@ -0,0 +1,36 @@
# Generated by Django 5.1.1 on 2024-10-03 14:47
from django.conf import settings
from django.db import migrations
from django.db import models
from django.db import transaction
from filelock import FileLock
from documents.templating.utils import convert_format_str_to_template_format
def convert_from_format_to_template(apps, schema_editor):
StoragePath = apps.get_model("documents", "StoragePath")
with transaction.atomic(), FileLock(settings.MEDIA_LOCK):
for storage_path in StoragePath.objects.all():
storage_path.path = convert_format_str_to_template_format(storage_path.path)
storage_path.save()
class Migration(migrations.Migration):
dependencies = [
("documents", "1054_customfieldinstance_value_monetary_amount_and_more"),
]
operations = [
migrations.AlterField(
model_name="storagepath",
name="path",
field=models.CharField(max_length=2048, verbose_name="path"),
),
migrations.RunPython(
convert_from_format_to_template,
migrations.RunPython.noop,
),
]

2
src/documents/models.py
View File

@@ -127,7 +127,7 @@ class DocumentType(MatchingModel):
class StoragePath(MatchingModel):
path = models.CharField(
_("path"),
max_length=512,
max_length=2048,
)
class Meta(MatchingModel.Meta):

43
src/documents/serialisers.py
View File

@@ -1,4 +1,5 @@
import datetime
import logging
import math
import re
import zoneinfo
@@ -52,8 +53,12 @@ from documents.models import WorkflowTrigger
from documents.parsers import is_mime_type_supported
from documents.permissions import get_groups_with_only_permission
from documents.permissions import set_permissions_for_object
from documents.templating.filepath import validate_filepath_template_and_render
from documents.templating.utils import convert_format_str_to_template_format
from documents.validators import uri_validator
logger = logging.getLogger("paperless.serializers")
# https://www.django-rest-framework.org/api-guide/serializers/#example
class DynamicFieldsModelSerializer(serializers.ModelSerializer):
@@ -1482,38 +1487,18 @@ class StoragePathSerializer(MatchingModelSerializer, OwnedObjectSerializer):
"set_permissions",
)
def validate_path(self, path):
try:
path.format(
title="title",
correspondent="correspondent",
document_type="document_type",
created="created",
created_year="created_year",
created_year_short="created_year_short",
created_month="created_month",
created_month_name="created_month_name",
created_month_name_short="created_month_name_short",
created_day="created_day",
added="added",
added_year="added_year",
added_year_short="added_year_short",
added_month="added_month",
added_month_name="added_month_name",
added_month_name_short="added_month_name_short",
added_day="added_day",
asn="asn",
tags="tags",
tag_list="tag_list",
owner_username="someone",
original_name="testfile",
doc_pk="doc_pk",
def validate_path(self, path: str):
converted_path = convert_format_str_to_template_format(path)
if converted_path != path:
logger.warning(
f"Storage path {path} is not using the new style format, consider updating",
)
result = validate_filepath_template_and_render(converted_path)
except KeyError as err:
raise serializers.ValidationError(_("Invalid variable detected.")) from err
if result is None:
raise serializers.ValidationError(_("Invalid variable detected."))
return path
return converted_path
def update(self, instance, validated_data):
"""

0
src/documents/templating/__init__.py Normal file
View File

333
src/documents/templating/filepath.py Normal file
View File

@@ -0,0 +1,333 @@
import logging
import os
import re
from collections.abc import Iterable
from datetime import datetime
from pathlib import PurePath
import pathvalidate
from django.utils import timezone
from django.utils.dateparse import parse_date
from jinja2 import StrictUndefined
from jinja2 import Template
from jinja2 import TemplateSyntaxError
from jinja2 import UndefinedError
from jinja2 import make_logging_undefined
from jinja2.sandbox import SandboxedEnvironment
from jinja2.sandbox import SecurityError
from documents.models import Correspondent
from documents.models import CustomField
from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import DocumentType
from documents.models import StoragePath
from documents.models import Tag
logger = logging.getLogger("paperless.templating")
_LogStrictUndefined = make_logging_undefined(logger, StrictUndefined)
class FilePathEnvironment(SandboxedEnvironment):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.undefined_tracker = None
def is_safe_callable(self, obj):
# Block access to .save() and .delete() methods
if callable(obj) and getattr(obj, "__name__", None) in (
"save",
"delete",
"update",
):
return False
# Call the parent method for other cases
return super().is_safe_callable(obj)
_template_environment = FilePathEnvironment(
trim_blocks=True,
lstrip_blocks=True,
keep_trailing_newline=False,
autoescape=False,
extensions=["jinja2.ext.loopcontrols"],
undefined=_LogStrictUndefined,
)
class FilePathTemplate(Template):
def render(self, *args, **kwargs) -> str:
def clean_filepath(value: str) -> str:
"""
Clean up a filepath by:
1. Removing newlines and carriage returns
2. Removing extra spaces before and after forward slashes
3. Preserving spaces in other parts of the path
"""
value = value.replace("\n", "").replace("\r", "")
value = re.sub(r"\s*/\s*", "/", value)
# We remove trailing and leading separators, as these are always relative paths, not absolute, even if the user
# tries
return value.strip().strip(os.sep)
original_render = super().render(*args, **kwargs)
return clean_filepath(original_render)
def get_cf_value(
custom_field_data: dict[str, dict[str, str]],
name: str,
default: str | None = None,
) -> str | None:
if name in custom_field_data:
return custom_field_data[name]["value"]
elif default is not None:
return default
return None
_template_environment.filters["get_cf_value"] = get_cf_value
def format_datetime(value: str | datetime, format: str) -> str:
if isinstance(value, str):
value = parse_date(value)
return value.strftime(format=format)
_template_environment.filters["datetime"] = format_datetime
def create_dummy_document():
"""
Create a dummy Document instance with all possible fields filled
"""
# Populate the document with representative values for every field
dummy_doc = Document(
pk=1,
title="Sample Title",
correspondent=Correspondent(name="Sample Correspondent"),
storage_path=StoragePath(path="/dummy/path"),
document_type=DocumentType(name="Sample Type"),
content="This is some sample document content.",
mime_type="application/pdf",
checksum="dummychecksum12345678901234567890123456789012",
archive_checksum="dummyarchivechecksum123456789012345678901234",
page_count=5,
created=timezone.now(),
modified=timezone.now(),
storage_type=Document.STORAGE_TYPE_UNENCRYPTED,
added=timezone.now(),
filename="/dummy/filename.pdf",
archive_filename="/dummy/archive_filename.pdf",
original_filename="original_file.pdf",
archive_serial_number=12345,
)
return dummy_doc
def get_creation_date_context(document: Document) -> dict[str, str]:
"""
Given a Document, localizes the creation date and builds a context dictionary with some common, shorthand
formatted values from it
"""
local_created = timezone.localdate(document.created)
return {
"created": local_created.isoformat(),
"created_year": local_created.strftime("%Y"),
"created_year_short": local_created.strftime("%y"),
"created_month": local_created.strftime("%m"),
"created_month_name": local_created.strftime("%B"),
"created_month_name_short": local_created.strftime("%b"),
"created_day": local_created.strftime("%d"),
}
def get_added_date_context(document: Document) -> dict[str, str]:
"""
Given a Document, localizes the added date and builds a context dictionary with some common, shorthand
formatted values from it
"""
local_added = timezone.localdate(document.added)
return {
"added": local_added.isoformat(),
"added_year": local_added.strftime("%Y"),
"added_year_short": local_added.strftime("%y"),
"added_month": local_added.strftime("%m"),
"added_month_name": local_added.strftime("%B"),
"added_month_name_short": local_added.strftime("%b"),
"added_day": local_added.strftime("%d"),
}
def get_basic_metadata_context(
document: Document,
*,
no_value_default: str,
) -> dict[str, str]:
"""
Given a Document, constructs some basic information about it. If certain values are not set,
they will be replaced with the no_value_default.
Regardless of set or not, the values will be sanitized
"""
return {
"title": pathvalidate.sanitize_filename(
document.title,
replacement_text="-",
),
"correspondent": pathvalidate.sanitize_filename(
document.correspondent.name,
replacement_text="-",
)
if document.correspondent
else no_value_default,
"document_type": pathvalidate.sanitize_filename(
document.document_type.name,
replacement_text="-",
)
if document.document_type
else no_value_default,
"asn": str(document.archive_serial_number)
if document.archive_serial_number
else no_value_default,
"owner_username": document.owner.username
if document.owner
else no_value_default,
"original_name": PurePath(document.original_filename).with_suffix("").name
if document.original_filename
else no_value_default,
"doc_pk": f"{document.pk:07}",
}
def get_tags_context(tags: Iterable[Tag]) -> dict[str, str | list[str]]:
"""
Given an Iterable of tags, constructs some context from them for usage
"""
return {
"tag_list": pathvalidate.sanitize_filename(
",".join(
sorted(tag.name for tag in tags),
),
replacement_text="-",
),
# Assumed to be ordered, but a template could loop through to find what they want
"tag_name_list": [x.name for x in tags],
}
def get_custom_fields_context(
custom_fields: Iterable[CustomFieldInstance],
) -> dict[str, dict[str, dict[str, str]]]:
"""
Given an Iterable of CustomFieldInstance, builds a dictionary mapping the field name
to its type and value
"""
field_data = {"custom_fields": {}}
for field_instance in custom_fields:
type_ = pathvalidate.sanitize_filename(
field_instance.field.data_type,
replacement_text="-",
)
# String types need to be sanitized
if field_instance.field.data_type in {
CustomField.FieldDataType.DOCUMENTLINK,
CustomField.FieldDataType.MONETARY,
CustomField.FieldDataType.STRING,
CustomField.FieldDataType.URL,
}:
value = pathvalidate.sanitize_filename(
field_instance.value,
replacement_text="-",
)
elif (
field_instance.field.data_type == CustomField.FieldDataType.SELECT
and field_instance.field.extra_data["select_options"] is not None
):
options = field_instance.field.extra_data["select_options"]
value = pathvalidate.sanitize_filename(
options[int(field_instance.value)],
replacement_text="-",
)
else:
value = field_instance.value
field_data["custom_fields"][
pathvalidate.sanitize_filename(
field_instance.field.name,
replacement_text="-",
)
] = {
"type": type_,
"value": value,
}
return field_data
def validate_filepath_template_and_render(
template_string: str,
document: Document | None = None,
) -> str | None:
"""
Renders the given template string using either the given Document or using a dummy Document and data
Returns None if the string is not valid or an error occurred, otherwise
"""
# Create the dummy document object with all fields filled in for validation purposes
if document is None:
document = create_dummy_document()
tags_list = [Tag(name="Test Tag 1"), Tag(name="Another Test Tag")]
custom_fields = [
CustomFieldInstance(
field=CustomField(
name="Text Custom Field",
data_type=CustomField.FieldDataType.STRING,
),
value_text="Some String Text",
),
]
else:
# or use the real document information
tags_list = document.tags.order_by("name").all()
custom_fields = document.custom_fields.all()
# Build the context dictionary
context = (
{"document": document}
| get_basic_metadata_context(document, no_value_default="-none-")
| get_creation_date_context(document)
| get_added_date_context(document)
| get_tags_context(tags_list)
| get_custom_fields_context(custom_fields)
)
# Try rendering the template
try:
# We load the custom tag used to remove spaces and newlines from the final string around the user string
template = _template_environment.from_string(
template_string,
template_class=FilePathTemplate,
)
rendered_template = template.render(context)
# We're good!
return rendered_template
except UndefinedError:
# The undefined class logs this already for us
pass
except TemplateSyntaxError as e:
logger.warning(f"Template syntax error in filename generation: {e}")
except SecurityError as e:
logger.warning(f"Template attempted restricted operation: {e}")
except Exception as e:
logger.warning(f"Unknown error in filename generation: {e}")
logger.warning(
f"Invalid filename_format '{template_string}', falling back to default",
)
return None

24
src/documents/templating/utils.py Normal file
View File

@@ -0,0 +1,24 @@
import re
def convert_format_str_to_template_format(old_format: str) -> str:
"""
Converts old Python string format (with {}) to Jinja2 template style (with {{ }}),
while ignoring existing {{ ... }} placeholders.
:param old_format: The old style format string (e.g., "{title} by {author}")
:return: Converted string in Django Template style (e.g., "{{ title }} by {{ author }}")
"""
# Step 1: Match placeholders with single curly braces but not those with double braces
pattern = r"(?<!\{)\{(\w*)\}(?!\})" # Matches {var} but not {{var}}
# Step 2: Replace the placeholders with {{ var }} or {{ }}
def replace_with_django(match):
variable = match.group(1) # The variable inside the braces
return f"{{{{ {variable} }}}}" # Convert to {{ variable }}
# Apply the substitution
converted_format = re.sub(pattern, replace_with_django, old_format)
return converted_format

2
src/documents/tests/test_api_objects.py
View File

@@ -239,7 +239,7 @@ class TestApiStoragePaths(DirectoriesMixin, APITestCase):
"/{created_year_short}/{created_month}/{created_month_name}"
"/{created_month_name_short}/{created_day}/{added}/{added_year}"
"/{added_year_short}/{added_month}/{added_month_name}"
"/{added_month_name_short}/{added_day}/{asn}/{tags}"
"/{added_month_name_short}/{added_day}/{asn}"
"/{tag_list}/{owner_username}/{original_name}/{doc_pk}/",
},
),

16
src/documents/tests/test_checks.py
View File

@@ -2,10 +2,12 @@ import textwrap
from unittest import mock
from django.core.checks import Error
from django.core.checks import Warning
from django.test import TestCase
from django.test import override_settings
from documents.checks import changed_password_check
from documents.checks import filename_format_check
from documents.checks import parser_check
from documents.models import Document
from documents.tests.factories import DocumentFactory
@@ -73,3 +75,17 @@ class TestDocumentChecks(TestCase):
),
],
)
def test_filename_format_check(self):
self.assertEqual(filename_format_check(None), [])
with override_settings(FILENAME_FORMAT="{created}/{title}"):
self.assertEqual(
filename_format_check(None),
[
Warning(
"Filename format {created}/{title} is using the old style, please update to use double curly brackets",
hint="{{ created }}/{{ title }}",
),
],
)

394
src/documents/tests/test_file_handling.py
View File

@@ -1,4 +1,5 @@
import datetime
import logging
import os
import tempfile
from pathlib import Path
@@ -16,6 +17,8 @@ from documents.file_handling import create_source_path_directory
from documents.file_handling import delete_empty_directories
from documents.file_handling import generate_filename
from documents.models import Correspondent
from documents.models import CustomField
from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import DocumentType
from documents.models import StoragePath
@@ -290,88 +293,6 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
self.assertEqual(generate_filename(d1), "652 - the_doc.pdf")
self.assertEqual(generate_filename(d2), "none - the_doc.pdf")
@override_settings(FILENAME_FORMAT="{tags[type]}")
def test_tags_with_underscore(self):
document = Document()
document.mime_type = "application/pdf"
document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED
document.save()
# Add tag to document
document.tags.create(name="type_demo")
document.tags.create(name="foo_bar")
document.save()
# Ensure that filename is properly generated
self.assertEqual(generate_filename(document), "demo.pdf")
@override_settings(FILENAME_FORMAT="{tags[type]}")
def test_tags_with_dash(self):
document = Document()
document.mime_type = "application/pdf"
document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED
document.save()
# Add tag to document
document.tags.create(name="type-demo")
document.tags.create(name="foo-bar")
document.save()
# Ensure that filename is properly generated
self.assertEqual(generate_filename(document), "demo.pdf")
@override_settings(FILENAME_FORMAT="{tags[type]}")
def test_tags_malformed(self):
document = Document()
document.mime_type = "application/pdf"
document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED
document.save()
# Add tag to document
document.tags.create(name="type:demo")
document.tags.create(name="foo:bar")
document.save()
# Ensure that filename is properly generated
self.assertEqual(generate_filename(document), "none.pdf")
@override_settings(FILENAME_FORMAT="{tags[0]}")
def test_tags_all(self):
document = Document()
document.mime_type = "application/pdf"
document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED
document.save()
# Add tag to document
document.tags.create(name="demo")
document.save()
# Ensure that filename is properly generated
self.assertEqual(generate_filename(document), "demo.pdf")
@override_settings(FILENAME_FORMAT="{tags[1]}")
def test_tags_out_of_bounds(self):
document = Document()
document.mime_type = "application/pdf"
document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED
document.save()
# Add tag to document
document.tags.create(name="demo")
document.save()
# Ensure that filename is properly generated
self.assertEqual(generate_filename(document), "none.pdf")
@override_settings(FILENAME_FORMAT="{tags}")
def test_tags_without_args(self):
document = Document()
document.mime_type = "application/pdf"
document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED
document.save()
self.assertEqual(generate_filename(document), f"{document.pk:07}.pdf")
@override_settings(FILENAME_FORMAT="{title} {tag_list}")
def test_tag_list(self):
doc = Document.objects.create(title="doc1", mime_type="application/pdf")
@@ -501,7 +422,7 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
self.assertIsFile(os.path.join(tmp, "notempty", "file"))
self.assertIsNotDir(os.path.join(tmp, "notempty", "empty"))
@override_settings(FILENAME_FORMAT="{created/[title]")
@override_settings(FILENAME_FORMAT="{% if x is None %}/{title]")
def test_invalid_format(self):
document = Document()
document.pk = 1
@@ -957,7 +878,7 @@ class TestFilenameGeneration(DirectoriesMixin, TestCase):
mime_type="application/pdf",
pk=2,
checksum="2",
storage_path=StoragePath.objects.create(path="TestFolder/{created}"),
storage_path=StoragePath.objects.create(path="TestFolder/{{created}}"),
)
self.assertEqual(generate_filename(doc), "TestFolder/2020-06-25.pdf")
@@ -978,7 +899,7 @@ class TestFilenameGeneration(DirectoriesMixin, TestCase):
mime_type="application/pdf",
pk=2,
checksum="2",
storage_path=StoragePath.objects.create(path="{asn} - {created}"),
storage_path=StoragePath.objects.create(path="{{asn}} - {{created}}"),
)
self.assertEqual(generate_filename(doc), "none - 2020-06-25.pdf")
@@ -1003,7 +924,9 @@ class TestFilenameGeneration(DirectoriesMixin, TestCase):
mime_type="application/pdf",
pk=2,
checksum="2",
storage_path=StoragePath.objects.create(path="TestFolder/{asn}/{created}"),
storage_path=StoragePath.objects.create(
path="TestFolder/{{asn}}/{{created}}",
),
)
self.assertEqual(generate_filename(doc), "TestFolder/2020-06-25.pdf")
@@ -1025,7 +948,7 @@ class TestFilenameGeneration(DirectoriesMixin, TestCase):
archive_serial_number=4,
storage_path=StoragePath.objects.create(
name="sp1",
path="ThisIsAFolder/{asn}/{created}",
path="ThisIsAFolder/{{asn}}/{{created}}",
),
)
doc_b = Document.objects.create(
@@ -1036,7 +959,7 @@ class TestFilenameGeneration(DirectoriesMixin, TestCase):
checksum="abcde",
storage_path=StoragePath.objects.create(
name="sp2",
path="SomeImportantNone/{created}",
path="SomeImportantNone/{{created}}",
),
)
@@ -1072,7 +995,7 @@ class TestFilenameGeneration(DirectoriesMixin, TestCase):
checksum="abcde",
storage_path=StoragePath.objects.create(
name="sp2",
path="SomeImportantNone/{created}",
path="SomeImportantNone/{{created}}",
),
)
@@ -1221,3 +1144,296 @@ class TestFilenameGeneration(DirectoriesMixin, TestCase):
# Ensure that filename is properly generated
document.filename = generate_filename(document)
self.assertEqual(document.filename, "XX/doc1.pdf")
def test_complex_template_strings(self):
"""
GIVEN:
- Storage paths with complex conditionals and logic
WHEN:
- Filepath for a document with this storage path is called
THEN:
- The filepath is rendered without error
- The filepath is rendered as a single line string
"""
sp = StoragePath.objects.create(
name="sp1",
path="""
somepath/
{% if document.checksum == '2' %}
some where/{{created}}
{% else %}
{{added}}
{% endif %}
/{{ title }}
""",
)
doc_a = Document.objects.create(
title="Does Matter",
created=timezone.make_aware(datetime.datetime(2020, 6, 25, 7, 36, 51, 153)),
added=timezone.make_aware(datetime.datetime(2024, 10, 1, 7, 36, 51, 153)),
mime_type="application/pdf",
pk=2,
checksum="2",
archive_serial_number=25,
storage_path=sp,
)
self.assertEqual(
generate_filename(doc_a),
"somepath/some where/2020-06-25/Does Matter.pdf",
)
doc_a.checksum = "5"
self.assertEqual(
generate_filename(doc_a),
"somepath/2024-10-01/Does Matter.pdf",
)
sp.path = "{{ document.title|lower }}{{ document.archive_serial_number - 2 }}"
sp.save()
self.assertEqual(generate_filename(doc_a), "does matter23.pdf")
sp.path = """
somepath/
{% if document.archive_serial_number >= 0 and document.archive_serial_number <= 200 %}
asn-000-200/{{title}}
{% elif document.archive_serial_number >= 201 and document.archive_serial_number <= 400 %}
asn-201-400
{% if document.archive_serial_number >= 201 and document.archive_serial_number < 300 %}
/asn-2xx
{% elif document.archive_serial_number >= 300 and document.archive_serial_number < 400 %}
/asn-3xx
{% endif %}
{% endif %}
/{{ title }}
"""
sp.save()
self.assertEqual(
generate_filename(doc_a),
"somepath/asn-000-200/Does Matter/Does Matter.pdf",
)
doc_a.archive_serial_number = 301
doc_a.save()
self.assertEqual(
generate_filename(doc_a),
"somepath/asn-201-400/asn-3xx/Does Matter.pdf",
)
@override_settings(
FILENAME_FORMAT="{{creation_date}}/{{ title_name_str }}",
)
def test_template_with_undefined_var(self):
"""
GIVEN:
- Filename format with one or more undefined variables
WHEN:
- Filepath for a document with this format is called
THEN:
- The first undefined variable is logged
- The default format is used
"""
doc_a = Document.objects.create(
title="Does Matter",
created=timezone.make_aware(datetime.datetime(2020, 6, 25, 7, 36, 51, 153)),
added=timezone.make_aware(datetime.datetime(2024, 10, 1, 7, 36, 51, 153)),
mime_type="application/pdf",
pk=2,
checksum="2",
archive_serial_number=25,
)
with self.assertLogs(level=logging.WARNING) as capture:
self.assertEqual(
generate_filename(doc_a),
"0000002.pdf",
)
self.assertEqual(len(capture.output), 1)
self.assertEqual(
capture.output[0],
"WARNING:paperless.templating:Template variable warning: 'creation_date' is undefined",
)
@override_settings(
FILENAME_FORMAT="{{created}}/{{ document.save() }}",
)
def test_template_with_security(self):
"""
GIVEN:
- Filename format with one or more undefined variables
WHEN:
- Filepath for a document with this format is called
THEN:
- The first undefined variable is logged
- The default format is used
"""
doc_a = Document.objects.create(
title="Does Matter",
created=timezone.make_aware(datetime.datetime(2020, 6, 25, 7, 36, 51, 153)),
added=timezone.make_aware(datetime.datetime(2024, 10, 1, 7, 36, 51, 153)),
mime_type="application/pdf",
pk=2,
checksum="2",
archive_serial_number=25,
)
with self.assertLogs(level=logging.WARNING) as capture:
self.assertEqual(
generate_filename(doc_a),
"0000002.pdf",
)
self.assertEqual(len(capture.output), 1)
self.assertEqual(
capture.output[0],
"WARNING:paperless.templating:Template attempted restricted operation: <bound method Model.save of <Document: 2020-06-25 Does Matter>> is not safely callable",
)
def test_template_with_custom_fields(self):
"""
GIVEN:
- Filename format which accesses custom field data
WHEN:
- Filepath for a document with this format is called
THEN:
- The custom field data is rendered
- If the field name is not defined, the default value is rendered, if any
"""
doc_a = Document.objects.create(
title="Some Title",
created=timezone.make_aware(datetime.datetime(2020, 6, 25, 7, 36, 51, 153)),
added=timezone.make_aware(datetime.datetime(2024, 10, 1, 7, 36, 51, 153)),
mime_type="application/pdf",
pk=2,
checksum="2",
archive_serial_number=25,
)
cf = CustomField.objects.create(
name="Invoice",
data_type=CustomField.FieldDataType.INT,
)
cf2 = CustomField.objects.create(
name="Select Field",
data_type=CustomField.FieldDataType.SELECT,
extra_data={"select_options": ["ChoiceOne", "ChoiceTwo"]},
)
CustomFieldInstance.objects.create(
document=doc_a,
field=cf2,
value_select=0,
)
cfi = CustomFieldInstance.objects.create(
document=doc_a,
field=cf,
value_int=1234,
)
with override_settings(
FILENAME_FORMAT="""
{% if "Invoice" in custom_fields %}
invoices/{{ custom_fields | get_cf_value('Invoice') }}
{% else %}
not-invoices/{{ title }}
{% endif %}
""",
):
self.assertEqual(
generate_filename(doc_a),
"invoices/1234.pdf",
)
with override_settings(
FILENAME_FORMAT="""
{% if "Select Field" in custom_fields %}
{{ title }}_{{ custom_fields | get_cf_value('Select Field') }}
{% else %}
{{ title }}
{% endif %}
""",
):
self.assertEqual(
generate_filename(doc_a),
"Some Title_ChoiceOne.pdf",
)
cf.name = "Invoice Number"
cfi.value_int = 4567
cfi.save()
cf.save()
with override_settings(
FILENAME_FORMAT="invoices/{{ custom_fields | get_cf_value('Invoice Number') }}",
):
self.assertEqual(
generate_filename(doc_a),
"invoices/4567.pdf",
)
with override_settings(
FILENAME_FORMAT="invoices/{{ custom_fields | get_cf_value('Ince Number', 0) }}",
):
self.assertEqual(
generate_filename(doc_a),
"invoices/0.pdf",
)
def test_datetime_filter(self):
"""
GIVEN:
- Filename format with datetime filter
WHEN:
- Filepath for a document with this format is called
THEN:
- The datetime filter is rendered
"""
doc_a = Document.objects.create(
title="Some Title",
created=timezone.make_aware(datetime.datetime(2020, 6, 25, 7, 36, 51, 153)),
added=timezone.make_aware(datetime.datetime(2024, 10, 1, 7, 36, 51, 153)),
mime_type="application/pdf",
pk=2,
checksum="2",
archive_serial_number=25,
)
CustomField.objects.create(
name="Invoice Date",
data_type=CustomField.FieldDataType.DATE,
)
CustomFieldInstance.objects.create(
document=doc_a,
field=CustomField.objects.get(name="Invoice Date"),
value_date=timezone.make_aware(
datetime.datetime(2024, 10, 1, 7, 36, 51, 153),
),
)
with override_settings(
FILENAME_FORMAT="{{ created | datetime('%Y') }}/{{ title }}",
):
self.assertEqual(
generate_filename(doc_a),
"2020/Some Title.pdf",
)
with override_settings(
FILENAME_FORMAT="{{ created | datetime('%Y-%m-%d') }}/{{ title }}",
):
self.assertEqual(
generate_filename(doc_a),
"2020-06-25/Some Title.pdf",
)
with override_settings(
FILENAME_FORMAT="{{ custom_fields | get_cf_value('Invoice Date') | datetime('%Y-%m-%d') }}/{{ title }}",
):
self.assertEqual(
generate_filename(doc_a),
"2024-10-01/Some Title.pdf",
)

30
src/documents/tests/test_migration_storage_path_template.py Normal file
View File

@@ -0,0 +1,30 @@
from documents.models import StoragePath
from documents.tests.utils import TestMigrations
class TestMigrateStoragePathToTemplate(TestMigrations):
migrate_from = "1054_customfieldinstance_value_monetary_amount_and_more"
migrate_to = "1055_alter_storagepath_path"
def setUpBeforeMigration(self, apps):
self.old_format = StoragePath.objects.create(
name="sp1",
path="Something/{title}",
)
self.new_format = StoragePath.objects.create(
name="sp2",
path="{{asn}}/{{title}}",
)
self.no_formatting = StoragePath.objects.create(
name="sp3",
path="Some/Fixed/Path",
)
def test_migrate_old_to_new_storage_path(self):
self.old_format.refresh_from_db()
self.new_format.refresh_from_db()
self.no_formatting.refresh_from_db()
self.assertEqual(self.old_format.path, "Something/{{ title }}")
self.assertEqual(self.new_format.path, "{{asn}}/{{title}}")
self.assertEqual(self.no_formatting.path, "Some/Fixed/Path")