From 69ef26dab04d51e7e102dcb33cd98ddc6ad975fd Mon Sep 17 00:00:00 2001 From: Markus <46903097+mskg@users.noreply.github.com> Date: Thu, 19 May 2022 23:42:25 +0200 Subject: [PATCH] Feature: Dynamic document storage pathes (#916) * Added devcontainer * Add feature storage pathes * Exclude tests and add versioning * Check escaping * Check escaping * Check quoting * Echo * Escape * Escape : * Double escape \ * Escaping * Remove if * Escape colon * Missing \ * Esacpe : * Escape all * test * Remove sed * Fix exclude * Remove SED command * Add LD_LIBRARY_PATH * Adjusted to v1.7 * Updated test-cases * Remove devcontainer * Removed internal build-file * Run pre-commit * Corrected flak8 error * Adjusted to v1.7 * Updated test-cases * Corrected flak8 error * Adjusted to new plural translations * Small adjustments due to code-review backend * Adjusted line-break * Removed PAPERLESS prefix from settings variables * Corrected style change due to search+replace * First documentation draft * Revert changes to Pipfile * Add sphinx-autobuild with keep-outdated * Revert merge error that results in wrong storage path is evaluated * Adjust styles of generated files ... * Adds additional testing to cover dynamic storage path functionality * Remove unnecessary condition * Add hint to edit storage path dialog * Correct spelling of pathes to paths * Minor documentation tweaks * Minor typo * improving wrapping of filter editor buttons with new storage path button * Update .gitignore * Fix select border radius in non input-groups * Better storage path edit hint * Add note to edit storage path dialog re document_renamer * Add note to bulk edit storage path re document_renamer * Rename FILTER_STORAGE_DIRECTORY to PATH * Fix broken filter rule parsing * Show default storage if unspecified * Remove note re storage path on bulk edit * Add basic validation of filename variables Co-authored-by: Markus Kling Co-authored-by: Trenton Holmes Co-authored-by: Michael Shamoon <4887959+shamoon@users.noreply.github.com> Co-authored-by: Quinn Casey --- .gitignore | 3 + Pipfile | 1 + Pipfile.lock | 77 ++++++- docs/Makefile | 4 + docs/_static/js/darkmode.js | 52 ++--- docs/advanced_usage.rst | 124 +++++++--- docs/configuration.rst | 8 + paperless.conf.example | 1 + src-ui/cypress.json | 2 +- src-ui/src/app/app-routing.module.ts | 2 + src-ui/src/app/app.module.ts | 4 + .../app-frame/app-frame.component.html | 7 + .../confirm-dialog.component.html | 2 +- .../storage-path-edit-dialog.component.html | 24 ++ .../storage-path-edit-dialog.component.scss | 0 .../storage-path-edit-dialog.component.ts | 50 ++++ .../common/input/select/select.component.html | 3 +- .../common/input/select/select.component.ts | 3 + .../common/input/text/text.component.html | 2 +- .../common/toasts/toasts.component.scss | 2 +- .../document-detail.component.html | 2 + .../document-detail.component.ts | 34 +++ .../bulk-editor/bulk-editor.component.html | 9 + .../bulk-editor/bulk-editor.component.ts | 57 ++++- .../document-card-large.component.html | 7 + .../document-card-large.component.ts | 3 + .../document-card-small.component.html | 7 + .../document-card-small.component.ts | 3 + .../document-list.component.html | 15 +- .../document-list/document-list.component.ts | 7 + .../filter-editor.component.html | 83 ++++--- .../filter-editor/filter-editor.component.ts | 36 ++- .../storage-path-list.component.ts | 48 ++++ src-ui/src/app/data/filter-rule-type.ts | 10 +- .../data/paperless-document-suggestions.ts | 2 + src-ui/src/app/data/paperless-document.ts | 5 + src-ui/src/app/data/paperless-storage-path.ts | 5 + .../src/app/services/rest/document.service.ts | 8 +- .../app/services/rest/storage-path.service.ts | 13 ++ src-ui/src/styles.scss | 24 +- src/documents/admin.py | 12 + src/documents/apps.py | 2 + src/documents/bulk_edit.py | 19 ++ src/documents/classifier.py | 45 +++- src/documents/file_handling.py | 36 ++- src/documents/filters.py | 13 ++ src/documents/index.py | 11 + .../management/commands/document_archiver.py | 2 +- src/documents/matching.py | 17 ++ .../migrations/1012_fix_archive_files.py | 6 +- .../1019_storagepath_document_storage_path.py | 73 ++++++ .../migrations/1020_merge_20220518_1839.py | 13 ++ src/documents/models.py | 21 ++ src/documents/serialisers.py | 68 ++++++ src/documents/signals/handlers.py | 70 ++++++ src/documents/tasks.py | 2 + src/documents/tests/data/model.pickle | Bin 156607 -> 155972 bytes src/documents/tests/test_api.py | 112 ++++++++- src/documents/tests/test_classifier.py | 62 +++++ src/documents/tests/test_consumer.py | 12 +- src/documents/tests/test_file_handling.py | 216 ++++++++++++++---- src/documents/tests/test_management.py | 10 +- .../tests/test_management_exporter.py | 4 +- .../tests/test_migration_archive_files.py | 12 +- src/documents/views.py | 30 +++ src/paperless/settings.py | 11 +- src/paperless/urls.py | 2 + 67 files changed, 1427 insertions(+), 203 deletions(-) create mode 100644 src-ui/src/app/components/common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component.html create mode 100644 src-ui/src/app/components/common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component.scss create mode 100644 src-ui/src/app/components/common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component.ts create mode 100644 src-ui/src/app/components/manage/storage-path-list/storage-path-list.component.ts create mode 100644 src-ui/src/app/data/paperless-storage-path.ts create mode 100644 src-ui/src/app/services/rest/storage-path.service.ts create mode 100644 src/documents/migrations/1019_storagepath_document_storage_path.py create mode 100644 src/documents/migrations/1020_merge_20220518_1839.py diff --git a/.gitignore b/.gitignore index bb0027b7f..2ba27f9b5 100644 --- a/.gitignore +++ b/.gitignore @@ -89,3 +89,6 @@ scripts/nuke # this is where the compiled frontend is moved to. /src/documents/static/frontend/ + +# mac os +.DS_Store diff --git a/Pipfile b/Pipfile index 6c2168b7d..7113f1cfd 100644 --- a/Pipfile +++ b/Pipfile @@ -69,4 +69,5 @@ sphinx_rtd_theme = "*" tox = "*" black = "*" pre-commit = "*" +sphinx-autobuild = "*" myst-parser = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 6215e7d0c..272cc4269 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "edaf53125fd5a0dc3aff5b75e188523ef3b7bc29bda792ee78ee67506e0b831d" + "sha256": "818f3513df4a757e6302baf5a17ce61e85c7d69a7666e7d49e7e50e78e064ae3" }, "pipfile-spec": 6, "requires": {}, @@ -466,7 +466,7 @@ "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff", "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d" ], - "markers": "python_version >= '3'", + "markers": "python_version >= '3.5'", "version": "==3.3" }, "imap-tools": { @@ -1587,6 +1587,14 @@ "markers": "python_version >= '3.7'", "version": "==8.1.3" }, + "colorama": { + "hashes": [ + "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b", + "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==0.4.4" + }, "coverage": { "extras": [ "toml" @@ -1711,7 +1719,7 @@ "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff", "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d" ], - "markers": "python_version >= '3'", + "markers": "python_version >= '3.5'", "version": "==3.3" }, "imagesize": { @@ -1745,6 +1753,12 @@ "markers": "python_version >= '3.7'", "version": "==3.1.2" }, + "livereload": { + "hashes": [ + "sha256:776f2f865e59fde56490a56bcc6773b6917366bce0c267c60ee8aaf1a0959869" + ], + "version": "==2.6.3" + }, "markdown-it-py": { "hashes": [ "sha256:93de681e5c021a432c63147656fe21790bc01231e0cd2da73626f1aa3ac0fe27", @@ -2046,6 +2060,14 @@ "index": "pypi", "version": "==4.5.0" }, + "sphinx-autobuild": { + "hashes": [ + "sha256:8fe8cbfdb75db04475232f05187c776f46f6e9e04cacf1e49ce81bdac649ccac", + "sha256:de1ca3b66e271d2b5b5140c35034c89e47f263f2cd5db302c9217065f7443f05" + ], + "index": "pypi", + "version": "==2021.3.14" + }, "sphinx-rtd-theme": { "hashes": [ "sha256:4d35a56f4508cfee4c4fb604373ede6feae2a306731d533f409ef5c3496fdbd8", @@ -2121,9 +2143,56 @@ "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f" ], - "markers": "python_version < '3.11'", + "markers": "python_version >= '3.7'", "version": "==2.0.1" }, + "tornado": { + "hashes": [ + "sha256:0a00ff4561e2929a2c37ce706cb8233b7907e0cdc22eab98888aca5dd3775feb", + "sha256:0d321a39c36e5f2c4ff12b4ed58d41390460f798422c4504e09eb5678e09998c", + "sha256:1e8225a1070cd8eec59a996c43229fe8f95689cb16e552d130b9793cb570a288", + "sha256:20241b3cb4f425e971cb0a8e4ffc9b0a861530ae3c52f2b0434e6c1b57e9fd95", + "sha256:25ad220258349a12ae87ede08a7b04aca51237721f63b1808d39bdb4b2164558", + "sha256:33892118b165401f291070100d6d09359ca74addda679b60390b09f8ef325ffe", + "sha256:33c6e81d7bd55b468d2e793517c909b139960b6c790a60b7991b9b6b76fb9791", + "sha256:3447475585bae2e77ecb832fc0300c3695516a47d46cefa0528181a34c5b9d3d", + "sha256:34ca2dac9e4d7afb0bed4677512e36a52f09caa6fded70b4e3e1c89dbd92c326", + "sha256:3e63498f680547ed24d2c71e6497f24bca791aca2fe116dbc2bd0ac7f191691b", + "sha256:548430be2740e327b3fe0201abe471f314741efcb0067ec4f2d7dcfb4825f3e4", + "sha256:6196a5c39286cc37c024cd78834fb9345e464525d8991c21e908cc046d1cc02c", + "sha256:61b32d06ae8a036a6607805e6720ef00a3c98207038444ba7fd3d169cd998910", + "sha256:6286efab1ed6e74b7028327365cf7346b1d777d63ab30e21a0f4d5b275fc17d5", + "sha256:65d98939f1a2e74b58839f8c4dab3b6b3c1ce84972ae712be02845e65391ac7c", + "sha256:66324e4e1beede9ac79e60f88de548da58b1f8ab4b2f1354d8375774f997e6c0", + "sha256:6c77c9937962577a6a76917845d06af6ab9197702a42e1346d8ae2e76b5e3675", + "sha256:70dec29e8ac485dbf57481baee40781c63e381bebea080991893cd297742b8fd", + "sha256:7250a3fa399f08ec9cb3f7b1b987955d17e044f1ade821b32e5f435130250d7f", + "sha256:748290bf9112b581c525e6e6d3820621ff020ed95af6f17fedef416b27ed564c", + "sha256:7da13da6f985aab7f6f28debab00c67ff9cbacd588e8477034c0652ac141feea", + "sha256:8f959b26f2634a091bb42241c3ed8d3cedb506e7c27b8dd5c7b9f745318ddbb6", + "sha256:9de9e5188a782be6b1ce866e8a51bc76a0fbaa0e16613823fc38e4fc2556ad05", + "sha256:a48900ecea1cbb71b8c71c620dee15b62f85f7c14189bdeee54966fbd9a0c5bd", + "sha256:b87936fd2c317b6ee08a5741ea06b9d11a6074ef4cc42e031bc6403f82a32575", + "sha256:c77da1263aa361938476f04c4b6c8916001b90b2c2fdd92d8d535e1af48fba5a", + "sha256:cb5ec8eead331e3bb4ce8066cf06d2dfef1bfb1b2a73082dfe8a161301b76e37", + "sha256:cc0ee35043162abbf717b7df924597ade8e5395e7b66d18270116f8745ceb795", + "sha256:d14d30e7f46a0476efb0deb5b61343b1526f73ebb5ed84f23dc794bdb88f9d9f", + "sha256:d371e811d6b156d82aa5f9a4e08b58debf97c302a35714f6f45e35139c332e32", + "sha256:d3d20ea5782ba63ed13bc2b8c291a053c8d807a8fa927d941bd718468f7b950c", + "sha256:d3f7594930c423fd9f5d1a76bee85a2c36fd8b4b16921cae7e965f22575e9c01", + "sha256:dcef026f608f678c118779cd6591c8af6e9b4155c44e0d1bc0c87c036fb8c8c4", + "sha256:e0791ac58d91ac58f694d8d2957884df8e4e2f6687cdf367ef7eb7497f79eaa2", + "sha256:e385b637ac3acaae8022e7e47dfa7b83d3620e432e3ecb9a3f7f58f150e50921", + "sha256:e519d64089b0876c7b467274468709dadf11e41d65f63bba207e04217f47c085", + "sha256:e7229e60ac41a1202444497ddde70a48d33909e484f96eb0da9baf8dc68541df", + "sha256:ed3ad863b1b40cd1d4bd21e7498329ccaece75db5a5bf58cd3c9f130843e7102", + "sha256:f0ba29bafd8e7e22920567ce0d232c26d4d47c8b5cf4ed7b562b5db39fa199c5", + "sha256:fa2ba70284fa42c2a5ecb35e322e68823288a4251f9ba9cc77be04ae15eada68", + "sha256:fba85b6cd9c39be262fcd23865652920832b61583de2a2ca907dbd8e8a8c81e5" + ], + "markers": "python_version >= '3.5'", + "version": "==6.1" + }, "tox": { "hashes": [ "sha256:0805727eb4d6b049de304977dfc9ce315a1938e6619c3ab9f38682bb04662a5a", diff --git a/docs/Makefile b/docs/Makefile index cf5dbff6a..7890f9828 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -24,6 +24,7 @@ I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . help: @echo "Please use \`make ' where is one of" @echo " html to make standalone HTML files" + @echo " livehtml to preview changes with live reload in your browser" @echo " dirhtml to make HTML files named index.html in directories" @echo " singlehtml to make a single large HTML file" @echo " pickle to make pickle files" @@ -54,6 +55,9 @@ html: @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." +livehtml: + sphinx-autobuild "./" "$(BUILDDIR)" $(O) + dirhtml: $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo diff --git a/docs/_static/js/darkmode.js b/docs/_static/js/darkmode.js index 49cf0eeec..909472587 100644 --- a/docs/_static/js/darkmode.js +++ b/docs/_static/js/darkmode.js @@ -1,47 +1,47 @@ -let toggleButton; -let icon; +let toggleButton +let icon function load() { - "use strict"; + 'use strict' - toggleButton = document.createElement("button"); - toggleButton.setAttribute("title", "Toggle dark mode"); - toggleButton.classList.add("dark-mode-toggle"); - icon = document.createElement("i"); - icon.classList.add("fa", darkModeState ? "fa-sun-o" : "fa-moon-o"); - toggleButton.appendChild(icon); - document.body.prepend(toggleButton); + toggleButton = document.createElement('button') + toggleButton.setAttribute('title', 'Toggle dark mode') + toggleButton.classList.add('dark-mode-toggle') + icon = document.createElement('i') + icon.classList.add('fa', darkModeState ? 'fa-sun-o' : 'fa-moon-o') + toggleButton.appendChild(icon) + document.body.prepend(toggleButton) // Listen for changes in the OS settings // addListener is used because older versions of Safari don't support addEventListener // prefersDarkQuery set in if (prefersDarkQuery) { prefersDarkQuery.addListener(function (evt) { - toggleDarkMode(evt.matches); - }); + toggleDarkMode(evt.matches) + }) } // Initial setting depending on the prefers-color-mode or localstorage // darkModeState should be set in the document to prevent flash - if (darkModeState == undefined) darkModeState = false; - toggleDarkMode(darkModeState); + if (darkModeState == undefined) darkModeState = false + toggleDarkMode(darkModeState) // Toggles the "dark-mode" class on click and sets localStorage state - toggleButton.addEventListener("click", () => { - darkModeState = !darkModeState; + toggleButton.addEventListener('click', () => { + darkModeState = !darkModeState - toggleDarkMode(darkModeState); - localStorage.setItem("dark-mode", darkModeState); - }); + toggleDarkMode(darkModeState) + localStorage.setItem('dark-mode', darkModeState) + }) } function toggleDarkMode(state) { - document.documentElement.classList.toggle("dark-mode", state); - document.documentElement.classList.toggle("light-mode", !state); - icon.classList.remove("fa-sun-o"); - icon.classList.remove("fa-moon-o"); - icon.classList.add(state ? "fa-sun-o" : "fa-moon-o"); - darkModeState = state; + document.documentElement.classList.toggle('dark-mode', state) + document.documentElement.classList.toggle('light-mode', !state) + icon.classList.remove('fa-sun-o') + icon.classList.remove('fa-moon-o') + icon.classList.add(state ? 'fa-sun-o' : 'fa-moon-o') + darkModeState = state } -document.addEventListener("DOMContentLoaded", load); +document.addEventListener('DOMContentLoaded', load) diff --git a/docs/advanced_usage.rst b/docs/advanced_usage.rst index 4dbb32f36..6449c478b 100644 --- a/docs/advanced_usage.rst +++ b/docs/advanced_usage.rst @@ -7,12 +7,12 @@ easier. .. _advanced-matching: -Matching tags, correspondents and document types -################################################ +Matching tags, correspondents, document types, and storage paths +################################################################ -Paperless will compare the matching algorithms defined by every tag and -correspondent already set in your database to see if they apply to the text in -a document. In other words, if you defined a tag called ``Home Utility`` +Paperless will compare the matching algorithms defined by every tag, correspondent, +document type, and storage path in your database to see if they apply to the text +in a document. In other words, if you define a tag called ``Home Utility`` that had a ``match`` property of ``bc hydro`` and a ``matching_algorithm`` of ``literal``, Paperless will automatically tag your newly-consumed document with your ``Home Utility`` tag so long as the text ``bc hydro`` appears in the body @@ -22,10 +22,10 @@ The matching logic is quite powerful. It supports searching the text of your document with different algorithms, and as such, some experimentation may be necessary to get things right. -In order to have a tag, correspondent, or type assigned automatically to newly -consumed documents, assign a match and matching algorithm using the web -interface. These settings define when to assign correspondents, tags, and types -to documents. +In order to have a tag, correspondent, document type, or storage path assigned +automatically to newly consumed documents, assign a match and matching algorithm +using the web interface. These settings define when to assign tags, correspondents, +document types, and storage paths to documents. The following algorithms are available: @@ -37,7 +37,7 @@ The following algorithms are available: * **Literal:** Matches only if the match appears exactly as provided (i.e. preserve ordering) in the PDF. * **Regular expression:** Parses the match as a regular expression and tries to find a match within the document. -* **Fuzzy match:** I dont know. Look at the source. +* **Fuzzy match:** I don't know. Look at the source. * **Auto:** Tries to automatically match new documents. This does not require you to set a match. See the notes below. @@ -47,9 +47,9 @@ defining a match text of ``"Bank of America" BofA`` using the *any* algorithm, will match documents that contain either "Bank of America" or "BofA", but will not match documents containing "Bank of South America". -Then just save your tag/correspondent and run another document through the -consumer. Once complete, you should see the newly-created document, -automatically tagged with the appropriate data. +Then just save your tag, correspondent, document type, or storage path and run +another document through the consumer. Once complete, you should see the +newly-created document, automatically tagged with the appropriate data. .. _advanced-automatic_matching: @@ -58,9 +58,9 @@ Automatic matching ================== Paperless-ngx comes with a new matching algorithm called *Auto*. This matching -algorithm tries to assign tags, correspondents, and document types to your -documents based on how you have already assigned these on existing documents. It -uses a neural network under the hood. +algorithm tries to assign tags, correspondents, document types, and storage paths +to your documents based on how you have already assigned these on existing documents. +It uses a neural network under the hood. If, for example, all your bank statements of your account 123 at the Bank of America are tagged with the tag "bofa_123" and the matching algorithm of this @@ -80,20 +80,21 @@ feature: that the neural network only learns from documents which you have correctly tagged before. * The matching algorithm can only work if there is a correlation between the - tag, correspondent, or document type and the document itself. Your bank - statements usually contain your bank account number and the name of the bank, - so this works reasonably well, However, tags such as "TODO" cannot be - automatically assigned. + tag, correspondent, document type, or storage path and the document itself. + Your bank statements usually contain your bank account number and the name + of the bank, so this works reasonably well, However, tags such as "TODO" + cannot be automatically assigned. * The matching algorithm needs a reasonable number of documents to identify when - to assign tags, correspondents, and types. If one out of a thousand documents - has the correspondent "Very obscure web shop I bought something five years - ago", it will probably not assign this correspondent automatically if you buy - something from them again. The more documents, the better. + to assign tags, correspondents, storage paths, and types. If one out of a + thousand documents has the correspondent "Very obscure web shop I bought + something five years ago", it will probably not assign this correspondent + automatically if you buy something from them again. The more documents, the better. * Paperless also needs a reasonable amount of negative examples to decide when - not to assign a certain tag, correspondent or type. This will usually be the - case as you start filling up paperless with documents. Example: If all your - documents are either from "Webshop" and "Bank", paperless will assign one of - these correspondents to ANY new document, if both are set to automatic matching. + not to assign a certain tag, correspondent, document type, or storage path. This will + usually be the case as you start filling up paperless with documents. + Example: If all your documents are either from "Webshop" and "Bank", paperless + will assign one of these correspondents to ANY new document, if both are set + to automatic matching. Hooking into the consumption process #################################### @@ -268,6 +269,17 @@ If paperless detects that two documents share the same filename, paperless will append ``_01``, ``_02``, etc to the filename. This happens if all the placeholders in a filename evaluate to the same value. +.. hint:: + You can affect how empty placeholders are treated by changing the following setting to + `true`. + + .. code:: + + PAPERLESS_FILENAME_FORMAT_REMOVE_NONE=True + + Doing this results in all empty placeholders resolving to "" instead of "none" as stated above. + Spaces before empty placeholders are removed as well, empty directories are omitted. + .. hint:: Paperless checks the filename of a document whenever it is saved. Therefore, @@ -290,3 +302,59 @@ evaluate to the same value. However, keep in mind that inside docker, if files get stored outside of the predefined volumes, they will be lost after a restart of paperless. + + +Storage paths +############# + +One of the best things in Paperless is that you can not only access the documents via the +web interface, but also via the file system. + +When as single storage layout is not sufficient for your use case, storage paths come to +the rescue. Storage paths allow you to configure more precisely where each document is stored +in the file system. + +- Each storage path is a `PAPERLESS_FILENAME_FORMAT` and follows the rules described above +- Each document is assigned a storage path using the matching algorithms described above, but + can be overwritten at any time + +For example, you could define the following two storage paths: + +1. Normal communications are put into a folder structure sorted by `year/correspondent` +2. Communications with insurance companies are stored in a flat structure with longer file names, + but containing the full date of the correspondence. + +.. code:: + + By Year = {created_year}/{correspondent}/{title} + Insurances = Insurances/{correspondent}/{created_year}-{created_month}-{created_day} {title} + + +If you then map these storage paths to the documents, you might get the following result. +For simplicity, `By Year` defines the same structure as in the previous example above. + +.. code:: text + + 2019/ # By Year + My bank/ + Statement January.pdf + Statement February.pdf + + Insurances/ # Insurances + Healthcare 123/ + 2022-01-01 Statement January.pdf + 2022-02-02 Letter.pdf + 2022-02-03 Letter.pdf + Dental 456/ + 2021-12-01 New Conditions.pdf + + +.. hint:: + + Defining a storage path is optional. If no storage path is defined for a document, the global + `PAPERLESS_FILENAME_FORMAT` is applied. + +.. caution:: + + If you adjust the format of an existing storage path, old documents don't get relocated automatically. + You need to run the :ref:`document renamer ` to adjust their pathes. diff --git a/docs/configuration.rst b/docs/configuration.rst index 3d57236e1..2068a4238 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -111,6 +111,14 @@ PAPERLESS_FILENAME_FORMAT= Default is none, which disables this feature. +PAPERLESS_FILENAME_FORMAT_REMOVE_NONE= + Tells paperless to replace placeholders in `PAPERLESS_FILENAME_FORMAT` that would resolve + to 'none' to be omitted from the resulting filename. This also holds true for directory + names. + See :ref:`advanced-file_name_handling` for details. + + Defaults to `false` which disables this feature. + PAPERLESS_LOGGING_DIR= This is where paperless will store log files. diff --git a/paperless.conf.example b/paperless.conf.example index be5071636..97e907e1f 100644 --- a/paperless.conf.example +++ b/paperless.conf.example @@ -23,6 +23,7 @@ #PAPERLESS_MEDIA_ROOT=../media #PAPERLESS_STATICDIR=../static #PAPERLESS_FILENAME_FORMAT= +#PAPERLESS_FILENAME_FORMAT_REMOVE_NONE= # Security and hosting diff --git a/src-ui/cypress.json b/src-ui/cypress.json index bd321dcbf..3a58fab3f 100644 --- a/src-ui/cypress.json +++ b/src-ui/cypress.json @@ -6,4 +6,4 @@ "pluginsFile": "cypress/plugins/index.ts", "fixturesFolder": "cypress/fixtures", "baseUrl": "http://localhost:4200" -} \ No newline at end of file +} diff --git a/src-ui/src/app/app-routing.module.ts b/src-ui/src/app/app-routing.module.ts index 436d2fad4..3eb13177f 100644 --- a/src-ui/src/app/app-routing.module.ts +++ b/src-ui/src/app/app-routing.module.ts @@ -12,6 +12,7 @@ import { TagListComponent } from './components/manage/tag-list/tag-list.componen import { NotFoundComponent } from './components/not-found/not-found.component' import { DocumentAsnComponent } from './components/document-asn/document-asn.component' import { DirtyFormGuard } from './guards/dirty-form.guard' +import { StoragePathListComponent } from './components/manage/storage-path-list/storage-path-list.component' const routes: Routes = [ { path: '', redirectTo: 'dashboard', pathMatch: 'full' }, @@ -27,6 +28,7 @@ const routes: Routes = [ { path: 'tags', component: TagListComponent }, { path: 'documenttypes', component: DocumentTypeListComponent }, { path: 'correspondents', component: CorrespondentListComponent }, + { path: 'storagepaths', component: StoragePathListComponent }, { path: 'logs', component: LogsComponent }, { path: 'settings', diff --git a/src-ui/src/app/app.module.ts b/src-ui/src/app/app.module.ts index 0c9d628d0..9701aeb18 100644 --- a/src-ui/src/app/app.module.ts +++ b/src-ui/src/app/app.module.ts @@ -87,6 +87,8 @@ import localeSr from '@angular/common/locales/sr' import localeSv from '@angular/common/locales/sv' import localeTr from '@angular/common/locales/tr' import localeZh from '@angular/common/locales/zh' +import { StoragePathListComponent } from './components/manage/storage-path-list/storage-path-list.component' +import { StoragePathEditDialogComponent } from './components/common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component' import { SettingsService } from './services/settings.service' registerLocaleData(localeBe) @@ -125,6 +127,7 @@ function initializeApp(settings: SettingsService) { TagListComponent, DocumentTypeListComponent, CorrespondentListComponent, + StoragePathListComponent, LogsComponent, SettingsComponent, NotFoundComponent, @@ -132,6 +135,7 @@ function initializeApp(settings: SettingsService) { ConfirmDialogComponent, TagEditDialogComponent, DocumentTypeEditDialogComponent, + StoragePathEditDialogComponent, TagComponent, PageHeaderComponent, AppFrameComponent, diff --git a/src-ui/src/app/components/app-frame/app-frame.component.html b/src-ui/src/app/components/app-frame/app-frame.component.html index ce9c8993b..8ec2bf8aa 100644 --- a/src-ui/src/app/components/app-frame/app-frame.component.html +++ b/src-ui/src/app/components/app-frame/app-frame.component.html @@ -134,6 +134,13 @@  Document types +
  • + + + +  Storage paths + +
  • diff --git a/src-ui/src/app/components/common/confirm-dialog/confirm-dialog.component.html b/src-ui/src/app/components/common/confirm-dialog/confirm-dialog.component.html index 19d907c82..92e27e370 100644 --- a/src-ui/src/app/components/common/confirm-dialog/confirm-dialog.component.html +++ b/src-ui/src/app/components/common/confirm-dialog/confirm-dialog.component.html @@ -5,7 +5,7 @@

    {{messageBold}}

    -

    {{message}}

    +

    +
    +
    + +

    + Note that editing a path does not apply changes to stored files until you have run the 'document_renamer' utility. See the documentation. +

    + + + + + + + +
    +
    + + +
    + diff --git a/src-ui/src/app/components/common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component.scss b/src-ui/src/app/components/common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component.scss new file mode 100644 index 000000000..e69de29bb diff --git a/src-ui/src/app/components/common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component.ts b/src-ui/src/app/components/common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component.ts new file mode 100644 index 000000000..2ec0821e6 --- /dev/null +++ b/src-ui/src/app/components/common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component.ts @@ -0,0 +1,50 @@ +import { Component } from '@angular/core' +import { FormControl, FormGroup } from '@angular/forms' +import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap' +import { EditDialogComponent } from 'src/app/components/common/edit-dialog/edit-dialog.component' +import { PaperlessStoragePath } from 'src/app/data/paperless-storage-path' +import { StoragePathService } from 'src/app/services/rest/storage-path.service' +import { ToastService } from 'src/app/services/toast.service' + +@Component({ + selector: 'app-storage-path-edit-dialog', + templateUrl: './storage-path-edit-dialog.component.html', + styleUrls: ['./storage-path-edit-dialog.component.scss'], +}) +export class StoragePathEditDialogComponent extends EditDialogComponent { + constructor( + service: StoragePathService, + activeModal: NgbActiveModal, + toastService: ToastService + ) { + super(service, activeModal, toastService) + } + + get pathHint() { + return ( + $localize`e.g.` + + ' {created_year}-{title} ' + + $localize`or use slashes to add directories e.g.` + + ' {created_year}/{correspondent}/{title}. ' + + $localize`See documentation for full list.` + ) + } + + getCreateTitle() { + return $localize`Create new storage path` + } + + getEditTitle() { + return $localize`Edit storage path` + } + + getForm(): FormGroup { + return new FormGroup({ + name: new FormControl(''), + path: new FormControl(''), + matching_algorithm: new FormControl(1), + match: new FormControl(''), + is_insensitive: new FormControl(true), + }) + } +} diff --git a/src-ui/src/app/components/common/input/select/select.component.html b/src-ui/src/app/components/common/input/select/select.component.html index afb551c46..b111e1656 100644 --- a/src-ui/src/app/components/common/input/select/select.component.html +++ b/src-ui/src/app/components/common/input/select/select.component.html @@ -9,7 +9,8 @@ [items]="items" [addTag]="allowCreateNew && addItemRef" addTagText="Add item" - i18n-addTagText="Used for both types and correspondents" + i18n-addTagText="Used for both types, correspondents, storage paths" + [placeholder]="placeholder" bindLabel="name" bindValue="id" (change)="onChange(value)" diff --git a/src-ui/src/app/components/common/input/select/select.component.ts b/src-ui/src/app/components/common/input/select/select.component.ts index 98aef8d94..d1a3d6e5c 100644 --- a/src-ui/src/app/components/common/input/select/select.component.ts +++ b/src-ui/src/app/components/common/input/select/select.component.ts @@ -41,6 +41,9 @@ export class SelectComponent extends AbstractInputComponent { @Input() suggestions: number[] + @Input() + placeholder: string + @Output() createNew = new EventEmitter() diff --git a/src-ui/src/app/components/common/input/text/text.component.html b/src-ui/src/app/components/common/input/text/text.component.html index 3f4136c11..41f02df64 100644 --- a/src-ui/src/app/components/common/input/text/text.component.html +++ b/src-ui/src/app/components/common/input/text/text.component.html @@ -1,7 +1,7 @@
    - {{hint}} +
    {{error}}
    diff --git a/src-ui/src/app/components/common/toasts/toasts.component.scss b/src-ui/src/app/components/common/toasts/toasts.component.scss index d2aaf20b0..e6dcb7a57 100644 --- a/src-ui/src/app/components/common/toasts/toasts.component.scss +++ b/src-ui/src/app/components/common/toasts/toasts.component.scss @@ -8,4 +8,4 @@ .toast:not(.show) { display: block; // this corrects an ng-bootstrap bug that prevented animations -} \ No newline at end of file +} diff --git a/src-ui/src/app/components/document-detail/document-detail.component.html b/src-ui/src/app/components/document-detail/document-detail.component.html index a4203473f..0f355eb06 100644 --- a/src-ui/src/app/components/document-detail/document-detail.component.html +++ b/src-ui/src/app/components/document-detail/document-detail.component.html @@ -73,6 +73,8 @@ (createNew)="createCorrespondent($event)" [suggestions]="suggestions?.correspondents"> + diff --git a/src-ui/src/app/components/document-detail/document-detail.component.ts b/src-ui/src/app/components/document-detail/document-detail.component.ts index f76e91e4d..6728f746d 100644 --- a/src-ui/src/app/components/document-detail/document-detail.component.ts +++ b/src-ui/src/app/components/document-detail/document-detail.component.ts @@ -33,6 +33,9 @@ import { PaperlessDocumentSuggestions } from 'src/app/data/paperless-document-su import { FILTER_FULLTEXT_MORELIKE } from 'src/app/data/filter-rule-type' import { normalizeDateStr } from 'src/app/utils/date' import { QueryParamsService } from 'src/app/services/query-params.service' +import { StoragePathService } from 'src/app/services/rest/storage-path.service' +import { PaperlessStoragePath } from 'src/app/data/paperless-storage-path' +import { StoragePathEditDialogComponent } from '../common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component' import { SETTINGS_KEYS } from 'src/app/data/paperless-uisettings' @Component({ @@ -66,6 +69,7 @@ export class DocumentDetailComponent correspondents: PaperlessCorrespondent[] documentTypes: PaperlessDocumentType[] + storagePaths: PaperlessStoragePath[] documentForm: FormGroup = new FormGroup({ title: new FormControl(''), @@ -73,6 +77,7 @@ export class DocumentDetailComponent created: new FormControl(), correspondent: new FormControl(), document_type: new FormControl(), + storage_path: new FormControl(), archive_serial_number: new FormControl(), tags: new FormControl([]), }) @@ -115,6 +120,7 @@ export class DocumentDetailComponent private documentTitlePipe: DocumentTitlePipe, private toastService: ToastService, private settings: SettingsService, + private storagePathService: StoragePathService, private queryParamsService: QueryParamsService ) {} @@ -163,11 +169,17 @@ export class DocumentDetailComponent .listAll() .pipe(first()) .subscribe((result) => (this.correspondents = result.results)) + this.documentTypeService .listAll() .pipe(first()) .subscribe((result) => (this.documentTypes = result.results)) + this.storagePathService + .listAll() + .pipe(first()) + .subscribe((result) => (this.storagePaths = result.results)) + this.route.paramMap .pipe( takeUntil(this.unsubscribeNotifier), @@ -230,6 +242,7 @@ export class DocumentDetailComponent created: this.ogDate.toISOString(), correspondent: doc.correspondent, document_type: doc.document_type, + storage_path: doc.storage_path, archive_serial_number: doc.archive_serial_number, tags: [...doc.tags], }) @@ -336,6 +349,27 @@ export class DocumentDetailComponent }) } + createStoragePath(newName: string) { + var modal = this.modalService.open(StoragePathEditDialogComponent, { + backdrop: 'static', + }) + modal.componentInstance.dialogMode = 'create' + if (newName) modal.componentInstance.object = { name: newName } + modal.componentInstance.success + .pipe( + switchMap((newStoragePath) => { + return this.storagePathService + .listAll() + .pipe(map((storagePaths) => ({ newStoragePath, storagePaths }))) + }) + ) + .pipe(takeUntil(this.unsubscribeNotifier)) + .subscribe(({ newStoragePath, documentTypes: storagePaths }) => { + this.storagePaths = storagePaths.results + this.documentForm.get('storage_path').setValue(newStoragePath.id) + }) + } + discard() { this.documentsService .get(this.documentId) diff --git a/src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.html b/src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.html index 631b558d0..8d9389df3 100644 --- a/src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.html +++ b/src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.html @@ -53,6 +53,15 @@ [(selectionModel)]="documentTypeSelectionModel" (apply)="setDocumentTypes($event)"> + +
    diff --git a/src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.ts b/src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.ts index b17af67eb..b9ab00fe6 100644 --- a/src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.ts +++ b/src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.ts @@ -22,6 +22,8 @@ import { MatchingModel } from 'src/app/data/matching-model' import { SettingsService } from 'src/app/services/settings.service' import { ToastService } from 'src/app/services/toast.service' import { saveAs } from 'file-saver' +import { StoragePathService } from 'src/app/services/rest/storage-path.service' +import { PaperlessStoragePath } from 'src/app/data/paperless-storage-path' import { SETTINGS_KEYS } from 'src/app/data/paperless-uisettings' @Component({ @@ -33,10 +35,12 @@ export class BulkEditorComponent { tags: PaperlessTag[] correspondents: PaperlessCorrespondent[] documentTypes: PaperlessDocumentType[] + storagePaths: PaperlessStoragePath[] tagSelectionModel = new FilterableDropdownSelectionModel() correspondentSelectionModel = new FilterableDropdownSelectionModel() documentTypeSelectionModel = new FilterableDropdownSelectionModel() + storagePathsSelectionModel = new FilterableDropdownSelectionModel() awaitingDownload: boolean constructor( @@ -48,7 +52,8 @@ export class BulkEditorComponent { private modalService: NgbModal, private openDocumentService: OpenDocumentsService, private settings: SettingsService, - private toastService: ToastService + private toastService: ToastService, + private storagePathService: StoragePathService ) {} applyOnClose: boolean = this.settings.get( @@ -68,6 +73,9 @@ export class BulkEditorComponent { this.documentTypeService .listAll() .subscribe((result) => (this.documentTypes = result.results)) + this.storagePathService + .listAll() + .subscribe((result) => (this.storagePaths = result.results)) } private executeBulkOperation(modal, method: string, args) { @@ -145,6 +153,17 @@ export class BulkEditorComponent { }) } + openStoragePathDropdown() { + this.documentService + .getSelectionData(Array.from(this.list.selected)) + .subscribe((s) => { + this.applySelectionData( + s.selected_storage_paths, + this.storagePathsSelectionModel + ) + }) + } + private _localizeList(items: MatchingModel[]) { if (items.length == 0) { return '' @@ -299,6 +318,42 @@ export class BulkEditorComponent { } } + setStoragePaths(changedDocumentPaths: ChangedItems) { + if ( + changedDocumentPaths.itemsToAdd.length == 0 && + changedDocumentPaths.itemsToRemove.length == 0 + ) + return + + let storagePath = + changedDocumentPaths.itemsToAdd.length > 0 + ? changedDocumentPaths.itemsToAdd[0] + : null + + if (this.showConfirmationDialogs) { + let modal = this.modalService.open(ConfirmDialogComponent, { + backdrop: 'static', + }) + modal.componentInstance.title = $localize`Confirm storage path assignment` + if (storagePath) { + modal.componentInstance.message = $localize`This operation will assign the storage path "${storagePath.name}" to ${this.list.selected.size} selected document(s).` + } else { + modal.componentInstance.message = $localize`This operation will remove the storage path from ${this.list.selected.size} selected document(s).` + } + modal.componentInstance.btnClass = 'btn-warning' + modal.componentInstance.btnCaption = $localize`Confirm` + modal.componentInstance.confirmClicked.subscribe(() => { + this.executeBulkOperation(modal, 'set_storage_path', { + storage_path: storagePath ? storagePath.id : null, + }) + }) + } else { + this.executeBulkOperation(null, 'set_storage_path', { + storage_path: storagePath ? storagePath.id : null, + }) + } + } + applyDelete() { let modal = this.modalService.open(ConfirmDialogComponent, { backdrop: 'static', diff --git a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html index 5eb4a97dd..cbc507de4 100644 --- a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html +++ b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html @@ -67,6 +67,13 @@ {{(document.document_type$ | async)?.name}} +
    diff --git a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.ts b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.ts index 9b9e20b8c..de6b94032 100644 --- a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.ts +++ b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.ts @@ -52,6 +52,9 @@ export class DocumentCardLargeComponent implements OnInit { @Output() clickDocumentType = new EventEmitter() + @Output() + clickStoragePath = new EventEmitter() + @Output() clickMoreLike = new EventEmitter() diff --git a/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.html b/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.html index 6c68cc26f..339d04e96 100644 --- a/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.html +++ b/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.html @@ -37,6 +37,13 @@ {{(document.document_type$ | async)?.name}} +
    diff --git a/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.ts b/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.ts index 06e2fe967..3f7fc7982 100644 --- a/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.ts +++ b/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.ts @@ -47,6 +47,9 @@ export class DocumentCardSmallComponent implements OnInit { @Output() clickDocumentType = new EventEmitter() + @Output() + clickStoragePath = new EventEmitter() + moreTags: number = null @ViewChild('popover') popover: NgbPopover diff --git a/src-ui/src/app/components/document-list/document-list.component.html b/src-ui/src/app/components/document-list/document-list.component.html index aa43243fc..d0aaf5890 100644 --- a/src-ui/src/app/components/document-list/document-list.component.html +++ b/src-ui/src/app/components/document-list/document-list.component.html @@ -107,7 +107,7 @@
    - +
    @@ -138,6 +138,12 @@ [currentSortReverse]="list.sortReverse" (sort)="onSort($event)" i18n>Document type + Storage path {{(d.document_type$ | async)?.name}} + + + {{(d.storage_path$ | async)?.name}} + + {{d.created | customDate}} @@ -187,7 +198,7 @@
    - +
    diff --git a/src-ui/src/app/components/document-list/document-list.component.ts b/src-ui/src/app/components/document-list/document-list.component.ts index 8cf6c3848..d9355902f 100644 --- a/src-ui/src/app/components/document-list/document-list.component.ts +++ b/src-ui/src/app/components/document-list/document-list.component.ts @@ -265,6 +265,13 @@ export class DocumentListComponent implements OnInit, OnDestroy, AfterViewInit { }) } + clickStoragePath(storagePathID: number) { + this.list.selectNone() + setTimeout(() => { + this.filterEditor.addStoragePath(storagePathID) + }) + } + clickMoreLike(documentID: number) { this.queryParamsService.navigateWithFilterRules([ { rule_type: FILTER_FULLTEXT_MORELIKE, value: documentID.toString() }, diff --git a/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.html b/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.html index 40c62d779..b02d6cc7a 100644 --- a/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.html +++ b/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.html @@ -1,7 +1,7 @@ -
    +
    -
    +
    @@ -18,43 +18,54 @@
    - - - - - +
    + + + + +
    +
    + + +
    -
    -