Feature: Enhanced templating for filename format (#7836)

Co-authored-by: shamoon <4887959+shamoon@users.noreply.github.com>

src/documents/tests/test_api_objects.py

@@ -239,7 +239,7 @@ class TestApiStoragePaths(DirectoriesMixin, APITestCase):
                     "/{created_year_short}/{created_month}/{created_month_name}"
                     "/{created_month_name_short}/{created_day}/{added}/{added_year}"
                     "/{added_year_short}/{added_month}/{added_month_name}"
-                    "/{added_month_name_short}/{added_day}/{asn}/{tags}"
+                    "/{added_month_name_short}/{added_day}/{asn}"
                     "/{tag_list}/{owner_username}/{original_name}/{doc_pk}/",
                 },
             ),

src/documents/tests/test_checks.py

@@ -2,10 +2,12 @@ import textwrap
 from unittest import mock
 
 from django.core.checks import Error
+from django.core.checks import Warning
 from django.test import TestCase
 from django.test import override_settings
 
 from documents.checks import changed_password_check
+from documents.checks import filename_format_check
 from documents.checks import parser_check
 from documents.models import Document
 from documents.tests.factories import DocumentFactory
@@ -73,3 +75,17 @@ class TestDocumentChecks(TestCase):
                     ),
                 ],
             )
+
+    def test_filename_format_check(self):
+        self.assertEqual(filename_format_check(None), [])
+
+        with override_settings(FILENAME_FORMAT="{created}/{title}"):
+            self.assertEqual(
+                filename_format_check(None),
+                [
+                    Warning(
+                        "Filename format {created}/{title} is using the old style, please update to use double curly brackets",
+                        hint="{{ created }}/{{ title }}",
+                    ),
+                ],
+            )

src/documents/tests/test_file_handling.py

@@ -1,4 +1,5 @@
 import datetime
+import logging
 import os
 import tempfile
 from pathlib import Path
@@ -16,6 +17,8 @@ from documents.file_handling import create_source_path_directory
 from documents.file_handling import delete_empty_directories
 from documents.file_handling import generate_filename
 from documents.models import Correspondent
+from documents.models import CustomField
+from documents.models import CustomFieldInstance
 from documents.models import Document
 from documents.models import DocumentType
 from documents.models import StoragePath
@@ -290,88 +293,6 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
         self.assertEqual(generate_filename(d1), "652 - the_doc.pdf")
         self.assertEqual(generate_filename(d2), "none - the_doc.pdf")
 
-    @override_settings(FILENAME_FORMAT="{tags[type]}")
-    def test_tags_with_underscore(self):
-        document = Document()
-        document.mime_type = "application/pdf"
-        document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED
-        document.save()
-
-        # Add tag to document
-        document.tags.create(name="type_demo")
-        document.tags.create(name="foo_bar")
-        document.save()
-
-        # Ensure that filename is properly generated
-        self.assertEqual(generate_filename(document), "demo.pdf")
-
-    @override_settings(FILENAME_FORMAT="{tags[type]}")
-    def test_tags_with_dash(self):
-        document = Document()
-        document.mime_type = "application/pdf"
-        document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED
-        document.save()
-
-        # Add tag to document
-        document.tags.create(name="type-demo")
-        document.tags.create(name="foo-bar")
-        document.save()
-
-        # Ensure that filename is properly generated
-        self.assertEqual(generate_filename(document), "demo.pdf")
-
-    @override_settings(FILENAME_FORMAT="{tags[type]}")
-    def test_tags_malformed(self):
-        document = Document()
-        document.mime_type = "application/pdf"
-        document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED
-        document.save()
-
-        # Add tag to document
-        document.tags.create(name="type:demo")
-        document.tags.create(name="foo:bar")
-        document.save()
-
-        # Ensure that filename is properly generated
-        self.assertEqual(generate_filename(document), "none.pdf")
-
-    @override_settings(FILENAME_FORMAT="{tags[0]}")
-    def test_tags_all(self):
-        document = Document()
-        document.mime_type = "application/pdf"
-        document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED
-        document.save()
-
-        # Add tag to document
-        document.tags.create(name="demo")
-        document.save()
-
-        # Ensure that filename is properly generated
-        self.assertEqual(generate_filename(document), "demo.pdf")
-
-    @override_settings(FILENAME_FORMAT="{tags[1]}")
-    def test_tags_out_of_bounds(self):
-        document = Document()
-        document.mime_type = "application/pdf"
-        document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED
-        document.save()
-
-        # Add tag to document
-        document.tags.create(name="demo")
-        document.save()
-
-        # Ensure that filename is properly generated
-        self.assertEqual(generate_filename(document), "none.pdf")
-
-    @override_settings(FILENAME_FORMAT="{tags}")
-    def test_tags_without_args(self):
-        document = Document()
-        document.mime_type = "application/pdf"
-        document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED
-        document.save()
-
-        self.assertEqual(generate_filename(document), f"{document.pk:07}.pdf")
-
     @override_settings(FILENAME_FORMAT="{title} {tag_list}")
     def test_tag_list(self):
         doc = Document.objects.create(title="doc1", mime_type="application/pdf")
@@ -501,7 +422,7 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
         self.assertIsFile(os.path.join(tmp, "notempty", "file"))
         self.assertIsNotDir(os.path.join(tmp, "notempty", "empty"))
 
-    @override_settings(FILENAME_FORMAT="{created/[title]")
+    @override_settings(FILENAME_FORMAT="{% if x is None %}/{title]")
     def test_invalid_format(self):
         document = Document()
         document.pk = 1
@@ -957,7 +878,7 @@ class TestFilenameGeneration(DirectoriesMixin, TestCase):
             mime_type="application/pdf",
             pk=2,
             checksum="2",
-            storage_path=StoragePath.objects.create(path="TestFolder/{created}"),
+            storage_path=StoragePath.objects.create(path="TestFolder/{{created}}"),
         )
         self.assertEqual(generate_filename(doc), "TestFolder/2020-06-25.pdf")
 
@@ -978,7 +899,7 @@ class TestFilenameGeneration(DirectoriesMixin, TestCase):
             mime_type="application/pdf",
             pk=2,
             checksum="2",
-            storage_path=StoragePath.objects.create(path="{asn} - {created}"),
+            storage_path=StoragePath.objects.create(path="{{asn}} - {{created}}"),
         )
         self.assertEqual(generate_filename(doc), "none - 2020-06-25.pdf")
 
@@ -1003,7 +924,9 @@ class TestFilenameGeneration(DirectoriesMixin, TestCase):
             mime_type="application/pdf",
             pk=2,
             checksum="2",
-            storage_path=StoragePath.objects.create(path="TestFolder/{asn}/{created}"),
+            storage_path=StoragePath.objects.create(
+                path="TestFolder/{{asn}}/{{created}}",
+            ),
         )
         self.assertEqual(generate_filename(doc), "TestFolder/2020-06-25.pdf")
 
@@ -1025,7 +948,7 @@ class TestFilenameGeneration(DirectoriesMixin, TestCase):
             archive_serial_number=4,
             storage_path=StoragePath.objects.create(
                 name="sp1",
-                path="ThisIsAFolder/{asn}/{created}",
+                path="ThisIsAFolder/{{asn}}/{{created}}",
             ),
         )
         doc_b = Document.objects.create(
@@ -1036,7 +959,7 @@ class TestFilenameGeneration(DirectoriesMixin, TestCase):
             checksum="abcde",
             storage_path=StoragePath.objects.create(
                 name="sp2",
-                path="SomeImportantNone/{created}",
+                path="SomeImportantNone/{{created}}",
             ),
         )
 
@@ -1072,7 +995,7 @@ class TestFilenameGeneration(DirectoriesMixin, TestCase):
             checksum="abcde",
             storage_path=StoragePath.objects.create(
                 name="sp2",
-                path="SomeImportantNone/{created}",
+                path="SomeImportantNone/{{created}}",
             ),
         )
 
@@ -1221,3 +1144,296 @@ class TestFilenameGeneration(DirectoriesMixin, TestCase):
         # Ensure that filename is properly generated
         document.filename = generate_filename(document)
         self.assertEqual(document.filename, "XX/doc1.pdf")
+
+    def test_complex_template_strings(self):
+        """
+        GIVEN:
+            - Storage paths with complex conditionals and logic
+        WHEN:
+            - Filepath for a document with this storage path is called
+        THEN:
+            - The filepath is rendered without error
+            - The filepath is rendered as a single line string
+        """
+        sp = StoragePath.objects.create(
+            name="sp1",
+            path="""
+                 somepath/
+                 {% if document.checksum == '2' %}
+                   some where/{{created}}
+                 {% else %}
+                   {{added}}
+                 {% endif %}
+                 /{{ title }}
+                 """,
+        )
+
+        doc_a = Document.objects.create(
+            title="Does Matter",
+            created=timezone.make_aware(datetime.datetime(2020, 6, 25, 7, 36, 51, 153)),
+            added=timezone.make_aware(datetime.datetime(2024, 10, 1, 7, 36, 51, 153)),
+            mime_type="application/pdf",
+            pk=2,
+            checksum="2",
+            archive_serial_number=25,
+            storage_path=sp,
+        )
+
+        self.assertEqual(
+            generate_filename(doc_a),
+            "somepath/some where/2020-06-25/Does Matter.pdf",
+        )
+        doc_a.checksum = "5"
+
+        self.assertEqual(
+            generate_filename(doc_a),
+            "somepath/2024-10-01/Does Matter.pdf",
+        )
+
+        sp.path = "{{ document.title|lower }}{{ document.archive_serial_number - 2 }}"
+        sp.save()
+
+        self.assertEqual(generate_filename(doc_a), "does matter23.pdf")
+
+        sp.path = """
+                 somepath/
+                 {% if document.archive_serial_number >= 0 and document.archive_serial_number <= 200 %}
+                   asn-000-200/{{title}}
+                 {% elif document.archive_serial_number >= 201 and document.archive_serial_number <= 400 %}
+                   asn-201-400
+                   {% if document.archive_serial_number >= 201 and document.archive_serial_number < 300 %}
+                     /asn-2xx
+                   {% elif document.archive_serial_number >= 300 and document.archive_serial_number < 400 %}
+                     /asn-3xx
+                   {% endif %}
+                 {% endif %}
+                 /{{ title }}
+                 """
+        sp.save()
+        self.assertEqual(
+            generate_filename(doc_a),
+            "somepath/asn-000-200/Does Matter/Does Matter.pdf",
+        )
+        doc_a.archive_serial_number = 301
+        doc_a.save()
+        self.assertEqual(
+            generate_filename(doc_a),
+            "somepath/asn-201-400/asn-3xx/Does Matter.pdf",
+        )
+
+    @override_settings(
+        FILENAME_FORMAT="{{creation_date}}/{{ title_name_str }}",
+    )
+    def test_template_with_undefined_var(self):
+        """
+        GIVEN:
+            - Filename format with one or more undefined variables
+        WHEN:
+            - Filepath for a document with this format is called
+        THEN:
+            - The first undefined variable is logged
+            - The default format is used
+        """
+        doc_a = Document.objects.create(
+            title="Does Matter",
+            created=timezone.make_aware(datetime.datetime(2020, 6, 25, 7, 36, 51, 153)),
+            added=timezone.make_aware(datetime.datetime(2024, 10, 1, 7, 36, 51, 153)),
+            mime_type="application/pdf",
+            pk=2,
+            checksum="2",
+            archive_serial_number=25,
+        )
+
+        with self.assertLogs(level=logging.WARNING) as capture:
+            self.assertEqual(
+                generate_filename(doc_a),
+                "0000002.pdf",
+            )
+
+            self.assertEqual(len(capture.output), 1)
+            self.assertEqual(
+                capture.output[0],
+                "WARNING:paperless.templating:Template variable warning: 'creation_date' is undefined",
+            )
+
+    @override_settings(
+        FILENAME_FORMAT="{{created}}/{{ document.save() }}",
+    )
+    def test_template_with_security(self):
+        """
+        GIVEN:
+            - Filename format with one or more undefined variables
+        WHEN:
+            - Filepath for a document with this format is called
+        THEN:
+            - The first undefined variable is logged
+            - The default format is used
+        """
+        doc_a = Document.objects.create(
+            title="Does Matter",
+            created=timezone.make_aware(datetime.datetime(2020, 6, 25, 7, 36, 51, 153)),
+            added=timezone.make_aware(datetime.datetime(2024, 10, 1, 7, 36, 51, 153)),
+            mime_type="application/pdf",
+            pk=2,
+            checksum="2",
+            archive_serial_number=25,
+        )
+
+        with self.assertLogs(level=logging.WARNING) as capture:
+            self.assertEqual(
+                generate_filename(doc_a),
+                "0000002.pdf",
+            )
+
+            self.assertEqual(len(capture.output), 1)
+            self.assertEqual(
+                capture.output[0],
+                "WARNING:paperless.templating:Template attempted restricted operation: <bound method Model.save of <Document: 2020-06-25 Does Matter>> is not safely callable",
+            )
+
+    def test_template_with_custom_fields(self):
+        """
+        GIVEN:
+            - Filename format which accesses custom field data
+        WHEN:
+            - Filepath for a document with this format is called
+        THEN:
+            - The custom field data is rendered
+            - If the field name is not defined, the default value is rendered, if any
+        """
+        doc_a = Document.objects.create(
+            title="Some Title",
+            created=timezone.make_aware(datetime.datetime(2020, 6, 25, 7, 36, 51, 153)),
+            added=timezone.make_aware(datetime.datetime(2024, 10, 1, 7, 36, 51, 153)),
+            mime_type="application/pdf",
+            pk=2,
+            checksum="2",
+            archive_serial_number=25,
+        )
+
+        cf = CustomField.objects.create(
+            name="Invoice",
+            data_type=CustomField.FieldDataType.INT,
+        )
+
+        cf2 = CustomField.objects.create(
+            name="Select Field",
+            data_type=CustomField.FieldDataType.SELECT,
+            extra_data={"select_options": ["ChoiceOne", "ChoiceTwo"]},
+        )
+
+        CustomFieldInstance.objects.create(
+            document=doc_a,
+            field=cf2,
+            value_select=0,
+        )
+
+        cfi = CustomFieldInstance.objects.create(
+            document=doc_a,
+            field=cf,
+            value_int=1234,
+        )
+
+        with override_settings(
+            FILENAME_FORMAT="""
+                 {% if "Invoice" in custom_fields %}
+                   invoices/{{ custom_fields | get_cf_value('Invoice') }}
+                 {% else %}
+                   not-invoices/{{ title }}
+                 {% endif %}
+                 """,
+        ):
+            self.assertEqual(
+                generate_filename(doc_a),
+                "invoices/1234.pdf",
+            )
+
+        with override_settings(
+            FILENAME_FORMAT="""
+                 {% if "Select Field" in custom_fields %}
+                   {{ title }}_{{ custom_fields | get_cf_value('Select Field') }}
+                 {% else %}
+                   {{ title }}
+                 {% endif %}
+                 """,
+        ):
+            self.assertEqual(
+                generate_filename(doc_a),
+                "Some Title_ChoiceOne.pdf",
+            )
+
+        cf.name = "Invoice Number"
+        cfi.value_int = 4567
+        cfi.save()
+        cf.save()
+
+        with override_settings(
+            FILENAME_FORMAT="invoices/{{ custom_fields | get_cf_value('Invoice Number') }}",
+        ):
+            self.assertEqual(
+                generate_filename(doc_a),
+                "invoices/4567.pdf",
+            )
+
+        with override_settings(
+            FILENAME_FORMAT="invoices/{{ custom_fields | get_cf_value('Ince Number', 0) }}",
+        ):
+            self.assertEqual(
+                generate_filename(doc_a),
+                "invoices/0.pdf",
+            )
+
+    def test_datetime_filter(self):
+        """
+        GIVEN:
+            - Filename format with datetime filter
+        WHEN:
+            - Filepath for a document with this format is called
+        THEN:
+            - The datetime filter is rendered
+        """
+        doc_a = Document.objects.create(
+            title="Some Title",
+            created=timezone.make_aware(datetime.datetime(2020, 6, 25, 7, 36, 51, 153)),
+            added=timezone.make_aware(datetime.datetime(2024, 10, 1, 7, 36, 51, 153)),
+            mime_type="application/pdf",
+            pk=2,
+            checksum="2",
+            archive_serial_number=25,
+        )
+
+        CustomField.objects.create(
+            name="Invoice Date",
+            data_type=CustomField.FieldDataType.DATE,
+        )
+        CustomFieldInstance.objects.create(
+            document=doc_a,
+            field=CustomField.objects.get(name="Invoice Date"),
+            value_date=timezone.make_aware(
+                datetime.datetime(2024, 10, 1, 7, 36, 51, 153),
+            ),
+        )
+
+        with override_settings(
+            FILENAME_FORMAT="{{ created | datetime('%Y') }}/{{ title }}",
+        ):
+            self.assertEqual(
+                generate_filename(doc_a),
+                "2020/Some Title.pdf",
+            )
+
+        with override_settings(
+            FILENAME_FORMAT="{{ created | datetime('%Y-%m-%d') }}/{{ title }}",
+        ):
+            self.assertEqual(
+                generate_filename(doc_a),
+                "2020-06-25/Some Title.pdf",
+            )
+
+        with override_settings(
+            FILENAME_FORMAT="{{ custom_fields | get_cf_value('Invoice Date') | datetime('%Y-%m-%d') }}/{{ title }}",
+        ):
+            self.assertEqual(
+                generate_filename(doc_a),
+                "2024-10-01/Some Title.pdf",
+            )

src/documents/tests/test_migration_storage_path_template.py

@@ -0,0 +1,30 @@
+from documents.models import StoragePath
+from documents.tests.utils import TestMigrations
+
+
+class TestMigrateStoragePathToTemplate(TestMigrations):
+    migrate_from = "1054_customfieldinstance_value_monetary_amount_and_more"
+    migrate_to = "1055_alter_storagepath_path"
+
+    def setUpBeforeMigration(self, apps):
+        self.old_format = StoragePath.objects.create(
+            name="sp1",
+            path="Something/{title}",
+        )
+        self.new_format = StoragePath.objects.create(
+            name="sp2",
+            path="{{asn}}/{{title}}",
+        )
+        self.no_formatting = StoragePath.objects.create(
+            name="sp3",
+            path="Some/Fixed/Path",
+        )
+
+    def test_migrate_old_to_new_storage_path(self):
+        self.old_format.refresh_from_db()
+        self.new_format.refresh_from_db()
+        self.no_formatting.refresh_from_db()
+
+        self.assertEqual(self.old_format.path, "Something/{{ title }}")
+        self.assertEqual(self.new_format.path, "{{asn}}/{{title}}")
+        self.assertEqual(self.no_formatting.path, "Some/Fixed/Path")