From 36158609f099d5c41e8ec263912997bd8d939121 Mon Sep 17 00:00:00 2001 From: Nico Fricke Date: Mon, 1 Aug 2022 09:22:13 +0200 Subject: [PATCH 001/273] change default matching algo to auto and move to constant --- .../correspondent-edit-dialog.component.ts | 3 ++- .../document-type-edit-dialog.component.ts | 3 ++- .../storage-path-edit-dialog.component.ts | 3 ++- .../edit-dialog/tag-edit-dialog/tag-edit-dialog.component.ts | 3 ++- src-ui/src/app/data/matching-model.ts | 1 + 5 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src-ui/src/app/components/common/edit-dialog/correspondent-edit-dialog/correspondent-edit-dialog.component.ts b/src-ui/src/app/components/common/edit-dialog/correspondent-edit-dialog/correspondent-edit-dialog.component.ts index cba0a922a..86be7414d 100644 --- a/src-ui/src/app/components/common/edit-dialog/correspondent-edit-dialog/correspondent-edit-dialog.component.ts +++ b/src-ui/src/app/components/common/edit-dialog/correspondent-edit-dialog/correspondent-edit-dialog.component.ts @@ -2,6 +2,7 @@ import { Component } from '@angular/core' import { FormControl, FormGroup } from '@angular/forms' import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap' import { EditDialogComponent } from 'src/app/components/common/edit-dialog/edit-dialog.component' +import { DEFAULT_MATCHING_ALGORITHM } from 'src/app/data/matching-model' import { PaperlessCorrespondent } from 'src/app/data/paperless-correspondent' import { CorrespondentService } from 'src/app/services/rest/correspondent.service' import { ToastService } from 'src/app/services/toast.service' @@ -31,7 +32,7 @@ export class CorrespondentEditDialogComponent extends EditDialogComponent { name: new FormControl(''), color: new FormControl(randomColor()), is_inbox_tag: new FormControl(false), - matching_algorithm: new FormControl(1), + matching_algorithm: new FormControl(DEFAULT_MATCHING_ALGORITHM), match: new FormControl(''), is_insensitive: new FormControl(true), }) diff --git a/src-ui/src/app/data/matching-model.ts b/src-ui/src/app/data/matching-model.ts index e228893d6..8ce05528e 100644 --- a/src-ui/src/app/data/matching-model.ts +++ b/src-ui/src/app/data/matching-model.ts @@ -6,6 +6,7 @@ export const MATCH_LITERAL = 3 export const MATCH_REGEX = 4 export const MATCH_FUZZY = 5 export const MATCH_AUTO = 6 +export const DEFAULT_MATCHING_ALGORITHM = MATCH_AUTO export const MATCHING_ALGORITHMS = [ { From 8a8edfb108f29394d598eaac024b9934d317dc95 Mon Sep 17 00:00:00 2001 From: Florian Brandes Date: Fri, 16 Sep 2022 09:15:52 +0200 Subject: [PATCH 002/273] bugfix: increase delay partially reverts 86358d556162e4c6112238821937f7856749f157 re-implements 4fbabe43ea12811864e9676b04d82a82b38e799d Signed-off-by: Florian Brandes --- src/documents/tests/test_management_consumer.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/documents/tests/test_management_consumer.py b/src/documents/tests/test_management_consumer.py index be8f26d12..e8f6f55f6 100644 --- a/src/documents/tests/test_management_consumer.py +++ b/src/documents/tests/test_management_consumer.py @@ -283,7 +283,9 @@ class TestConsumer(DirectoriesMixin, ConsumerMixin, TransactionTestCase): @override_settings( CONSUMER_POLLING=1, - CONSUMER_POLLING_DELAY=1, + # please leave the delay here and down below + # see https://github.com/paperless-ngx/paperless-ngx/pull/66 + CONSUMER_POLLING_DELAY=3, CONSUMER_POLLING_RETRY_COUNT=20, ) class TestConsumerPolling(TestConsumer): @@ -300,7 +302,7 @@ class TestConsumerRecursive(TestConsumer): @override_settings( CONSUMER_RECURSIVE=True, CONSUMER_POLLING=1, - CONSUMER_POLLING_DELAY=1, + CONSUMER_POLLING_DELAY=3, CONSUMER_POLLING_RETRY_COUNT=20, ) class TestConsumerRecursivePolling(TestConsumer): @@ -345,7 +347,7 @@ class TestConsumerTags(DirectoriesMixin, ConsumerMixin, TransactionTestCase): @override_settings( CONSUMER_POLLING=1, - CONSUMER_POLLING_DELAY=1, + CONSUMER_POLLING_DELAY=3, CONSUMER_POLLING_RETRY_COUNT=20, ) def test_consume_file_with_path_tags_polling(self): From cb2823ff455f9dee09e1430fe4a7c99e53b799ca Mon Sep 17 00:00:00 2001 From: Trenton Holmes Date: Wed, 21 Sep 2022 09:43:40 -0700 Subject: [PATCH 003/273] Moves the pipenv generation of requirements.txt into a seperate build stage, disconnecting the main image from its dependencies --- Dockerfile | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/Dockerfile b/Dockerfile index bb9a148ef..bc3808fa5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,6 +30,25 @@ RUN set -eux \ RUN set -eux \ && ./node_modules/.bin/ng build --configuration production +FROM --platform=$BUILDPLATFORM python:3.9-slim-bullseye as pipenv-base + +# This stage generates the requirements.txt file using pipenv +# This stage runs once for the native platform, as the outputs are not +# dependent on target arch +# This way, pipenv dependencies are not left in the final image +# nor can pipenv mess up the final image somehow +# Inputs: None + +WORKDIR /usr/src/pipenv + +COPY Pipfile* ./ + +RUN set -eux \ + && echo "Installing pipenv" \ + && python3 -m pip install --no-cache-dir --upgrade pipenv \ + && echo "Generating requirement.txt" \ + && pipenv requirements > requirements.txt + FROM python:3.9-slim-bullseye as main-app LABEL org.opencontainers.image.authors="paperless-ngx team " @@ -180,7 +199,7 @@ WORKDIR /usr/src/paperless/src/ # Python dependencies # Change pretty frequently -COPY Pipfile* ./ +COPY --from=pipenv-base /usr/src/pipenv/requirements.txt ./ # Packages needed only for building a few quick Python # dependencies @@ -195,24 +214,12 @@ RUN set -eux \ && apt-get update \ && apt-get install --yes --quiet --no-install-recommends ${BUILD_PACKAGES} \ && python3 -m pip install --no-cache-dir --upgrade wheel \ - && echo "Installing pipenv" \ - && python3 -m pip install --no-cache-dir --upgrade pipenv \ && echo "Installing Python requirements" \ - # pipenv tries to be too fancy and prints so much junk - && pipenv requirements > requirements.txt \ && python3 -m pip install --default-timeout=1000 --no-cache-dir --requirement requirements.txt \ - && rm requirements.txt \ && echo "Cleaning up image" \ && apt-get -y purge ${BUILD_PACKAGES} \ && apt-get -y autoremove --purge \ && apt-get clean --yes \ - # Remove pipenv and its unique packages - && python3 -m pip uninstall --yes \ - pipenv \ - distlib \ - platformdirs \ - virtualenv \ - virtualenv-clone \ && rm -rf /var/lib/apt/lists/* \ && rm -rf /tmp/* \ && rm -rf /var/tmp/* \ From d003d26a671415ca0f0af2a68057762ba54cb1a3 Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+shamoon@users.noreply.github.com> Date: Mon, 26 Sep 2022 07:56:05 -0700 Subject: [PATCH 004/273] Update environment.prod.ts --- src-ui/src/environments/environment.prod.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src-ui/src/environments/environment.prod.ts b/src-ui/src/environments/environment.prod.ts index 92f388ec8..ecfed31c7 100644 --- a/src-ui/src/environments/environment.prod.ts +++ b/src-ui/src/environments/environment.prod.ts @@ -5,7 +5,7 @@ export const environment = { apiBaseUrl: document.baseURI + 'api/', apiVersion: '2', appTitle: 'Paperless-ngx', - version: '1.9.0', + version: '1.9.0-dev', webSocketHost: window.location.host, webSocketProtocol: window.location.protocol == 'https:' ? 'wss:' : 'ws:', webSocketBaseUrl: base_url.pathname + 'ws/', From 05a00b3057c28410b5a48c70cb1f6eeea3f31fb5 Mon Sep 17 00:00:00 2001 From: Trenton Holmes Date: Sun, 25 Sep 2022 17:29:38 -0700 Subject: [PATCH 005/273] Updates Python deps all at once --- .pre-commit-config.yaml | 6 +- Pipfile.lock | 303 +++++++++++++++++++++------------------- 2 files changed, 166 insertions(+), 143 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b93d0fb6e..5f5a50432 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,7 +37,7 @@ repos: exclude: "(^Pipfile\\.lock$)" # Python hooks - repo: https://github.com/asottile/reorder_python_imports - rev: v3.8.2 + rev: v3.8.3 hooks: - id: reorder-python-imports exclude: "(migrations)" @@ -59,11 +59,11 @@ repos: args: - "--config=./src/setup.cfg" - repo: https://github.com/psf/black - rev: 22.6.0 + rev: 22.8.0 hooks: - id: black - repo: https://github.com/asottile/pyupgrade - rev: v2.37.3 + rev: v2.38.1 hooks: - id: pyupgrade exclude: "(migrations)" diff --git a/Pipfile.lock b/Pipfile.lock index 98499df9e..7fdfc30af 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -112,11 +112,11 @@ }, "certifi": { "hashes": [ - "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d", - "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412" + "sha256:0d9c601124e5a6ba9712dbc60d9c53c21e34f5f641fe83002317394311bdce14", + "sha256:90c1a32f1d68f940488354e36370f6cca89f0f106db09518524c88d6ed83f382" ], - "markers": "python_full_version >= '3.6.0'", - "version": "==2022.6.15" + "markers": "python_version >= '3.6'", + "version": "==2022.9.24" }, "cffi": { "hashes": [ @@ -208,7 +208,7 @@ "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845", "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==2.1.1" }, "click": { @@ -307,11 +307,11 @@ }, "django": { "hashes": [ - "sha256:41bd65a9e5f8a89cdbfa7a7bba45cd7431ae89e750af82dea8a35fd1a7ecbe66", - "sha256:9c6d5ad36be798e562ddcaa6b17b1c3ff2d3c4f529a47432b69fb9a30f847461" + "sha256:a153ffd5143bf26a877bfae2f4ec736ebd8924a46600ca089ad96b54a1d4e28e", + "sha256:acb21fac9275f9972d81c7caf5761a89ec3ea25fe74545dd26b8a48cb3a0203e" ], "index": "pypi", - "version": "==4.0.7" + "version": "==4.1.1" }, "django-cors-headers": { "hashes": [ @@ -323,11 +323,11 @@ }, "django-extensions": { "hashes": [ - "sha256:4c234a7236e9e41c17d9036f6dae7a3a9b212527105b8a0d24b2459b267825f0", - "sha256:7dc7cd1da50d83b76447a58f5d7e5c8e6cd83f21e9b7e5f97e6b644f4d4e21a6" + "sha256:2a4f4d757be2563cd1ff7cfdf2e57468f5f931cc88b23cf82ca75717aae504a4", + "sha256:421464be390289513f86cb5e18eb43e5dc1de8b4c27ba9faa3b91261b0d67e09" ], "index": "pypi", - "version": "==3.2.0" + "version": "==3.2.1" }, "django-filter": { "hashes": [ @@ -352,11 +352,11 @@ }, "djangorestframework": { "hashes": [ - "sha256:0c33407ce23acc68eca2a6e46424b008c9c02eceb8cf18581921d0092bc1f2ee", - "sha256:24c4bf58ed7e85d1fe4ba250ab2da926d263cd57d64b03e8dcef0ac683f8b1aa" + "sha256:579a333e6256b09489cbe0a067e66abe55c6595d8926be6b99423786334350c8", + "sha256:eb63f58c9f218e1a7d064d17a70751f528ed4e1d35547fdade9aaf4cd103fd08" ], "index": "pypi", - "version": "==3.13.1" + "version": "==3.14.0" }, "filelock": { "hashes": [ @@ -387,11 +387,11 @@ }, "h11": { "hashes": [ - "sha256:70813c1135087a248a4d38cc0e1a0181ffab2188141a93eaf567940c3957ff06", - "sha256:8ddd78563b633ca55346c8cd41ec0af27d3c79931828beffb46ce70a379e7442" + "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d", + "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761" ], - "markers": "python_version >= '3.6'", - "version": "==0.13.0" + "markers": "python_version >= '3.7'", + "version": "==0.14.0" }, "hiredis": { "hashes": [ @@ -442,42 +442,49 @@ }, "httptools": { "hashes": [ - "sha256:1a99346ebcb801b213c591540837340bdf6fd060a8687518d01c607d338b7424", - "sha256:1ee0b459257e222b878a6c09ccf233957d3a4dcb883b0847640af98d2d9aac23", - "sha256:20a45bcf22452a10fa8d58b7dbdb474381f6946bf5b8933e3662d572bc61bae4", - "sha256:29bf97a5c532da9c7a04de2c7a9c31d1d54f3abd65a464119b680206bbbb1055", - "sha256:2c9a930c378b3d15d6b695fb95ebcff81a7395b4f9775c4f10a076beb0b2c1ff", - "sha256:2db44a0b294d317199e9f80123e72c6b005c55b625b57fae36de68670090fa48", - "sha256:3194f6d6443befa8d4db16c1946b2fc428a3ceb8ab32eb6f09a59f86104dc1a0", - "sha256:34d2903dd2a3dd85d33705b6fde40bf91fc44411661283763fd0746723963c83", - "sha256:48e48530d9b995a84d1d89ae6b3ec4e59ea7d494b150ac3bbc5e2ac4acce92cd", - "sha256:54bbd295f031b866b9799dd39cb45deee81aca036c9bff9f58ca06726f6494f1", - "sha256:5d1fe6b6661022fd6cac541f54a4237496b246e6f1c0a6b41998ee08a1135afe", - "sha256:645373c070080e632480a3d251d892cb795be3d3a15f86975d0f1aca56fd230d", - "sha256:6a1a7dfc1f9c78a833e2c4904757a0f47ce25d08634dd2a52af394eefe5f9777", - "sha256:701e66b59dd21a32a274771238025d58db7e2b6ecebbab64ceff51b8e31527ae", - "sha256:72aa3fbe636b16d22e04b5a9d24711b043495e0ecfe58080addf23a1a37f3409", - "sha256:7af6bdbd21a2a25d6784f6d67f44f5df33ef39b6159543b9f9064d365c01f919", - "sha256:7ee9f226acab9085037582c059d66769862706e8e8cd2340470ceb8b3850873d", - "sha256:7f7bfb74718f52d5ed47d608d507bf66d3bc01d4a8b3e6dd7134daaae129357b", - "sha256:8e2eb957787cbb614a0f006bfc5798ff1d90ac7c4dd24854c84edbdc8c02369e", - "sha256:903f739c9fb78dab8970b0f3ea51f21955b24b45afa77b22ff0e172fc11ef111", - "sha256:98993805f1e3cdb53de4eed02b55dcc953cdf017ba7bbb2fd89226c086a6d855", - "sha256:9967d9758df505975913304c434cb9ab21e2c609ad859eb921f2f615a038c8de", - "sha256:a113789e53ac1fa26edf99856a61e4c493868e125ae0dd6354cf518948fbbd5c", - "sha256:a522d12e2ddbc2e91842ffb454a1aeb0d47607972c7d8fc88bd0838d97fb8a2a", - "sha256:abe829275cdd4174b4c4e65ad718715d449e308d59793bf3a931ee1bf7e7b86c", - "sha256:c286985b5e194ca0ebb2908d71464b9be8f17cc66d6d3e330e8d5407248f56ad", - "sha256:cd1295f52971097f757edfbfce827b6dbbfb0f7a74901ee7d4933dff5ad4c9af", - "sha256:ceafd5e960b39c7e0d160a1936b68eb87c5e79b3979d66e774f0c77d4d8faaed", - "sha256:d1f27bb0f75bef722d6e22dc609612bfa2f994541621cd2163f8c943b6463dfe", - "sha256:d3a4e165ca6204f34856b765d515d558dc84f1352033b8721e8d06c3e44930c3", - "sha256:d9b90bf58f3ba04e60321a23a8723a1ff2a9377502535e70495e5ada8e6e6722", - "sha256:f72b5d24d6730035128b238decdc4c0f2104b7056a7ca55cf047c106842ec890", - "sha256:fcddfe70553be717d9745990dfdb194e22ee0f60eb8f48c0794e7bfeda30d2d5", - "sha256:fdb9f9ed79bc6f46b021b3319184699ba1a22410a82204e6e89c774530069683" + "sha256:0297822cea9f90a38df29f48e40b42ac3d48a28637368f3ec6d15eebefd182f9", + "sha256:1af91b3650ce518d226466f30bbba5b6376dbd3ddb1b2be8b0658c6799dd450b", + "sha256:1f90cd6fd97c9a1b7fe9215e60c3bd97336742a0857f00a4cb31547bc22560c2", + "sha256:24bb4bb8ac3882f90aa95403a1cb48465de877e2d5298ad6ddcfdebec060787d", + "sha256:295874861c173f9101960bba332429bb77ed4dcd8cdf5cee9922eb00e4f6bc09", + "sha256:3625a55886257755cb15194efbf209584754e31d336e09e2ffe0685a76cb4b60", + "sha256:3a47a34f6015dd52c9eb629c0f5a8a5193e47bf2a12d9a3194d231eaf1bc451a", + "sha256:3cb8acf8f951363b617a8420768a9f249099b92e703c052f9a51b66342eea89b", + "sha256:4b098e4bb1174096a93f48f6193e7d9aa7071506a5877da09a783509ca5fff42", + "sha256:4d9ebac23d2de960726ce45f49d70eb5466725c0087a078866043dad115f850f", + "sha256:50d4613025f15f4b11f1c54bbed4761c0020f7f921b95143ad6d58c151198142", + "sha256:5230a99e724a1bdbbf236a1b58d6e8504b912b0552721c7c6b8570925ee0ccde", + "sha256:54465401dbbec9a6a42cf737627fb0f014d50dc7365a6b6cd57753f151a86ff0", + "sha256:550059885dc9c19a072ca6d6735739d879be3b5959ec218ba3e013fd2255a11b", + "sha256:557be7fbf2bfa4a2ec65192c254e151684545ebab45eca5d50477d562c40f986", + "sha256:5b65be160adcd9de7a7e6413a4966665756e263f0d5ddeffde277ffeee0576a5", + "sha256:64eba6f168803a7469866a9c9b5263a7463fa8b7a25b35e547492aa7322036b6", + "sha256:72ad589ba5e4a87e1d404cc1cb1b5780bfcb16e2aec957b88ce15fe879cc08ca", + "sha256:7d0c1044bce274ec6711f0770fd2d5544fe392591d204c68328e60a46f88843b", + "sha256:7e5eefc58d20e4c2da82c78d91b2906f1a947ef42bd668db05f4ab4201a99f49", + "sha256:850fec36c48df5a790aa735417dca8ce7d4b48d59b3ebd6f83e88a8125cde324", + "sha256:85b392aba273566c3d5596a0a490978c085b79700814fb22bfd537d381dd230c", + "sha256:8c2a56b6aad7cc8f5551d8e04ff5a319d203f9d870398b94702300de50190f63", + "sha256:8f470c79061599a126d74385623ff4744c4e0f4a0997a353a44923c0b561ee51", + "sha256:8ffce9d81c825ac1deaa13bc9694c0562e2840a48ba21cfc9f3b4c922c16f372", + "sha256:9423a2de923820c7e82e18980b937893f4aa8251c43684fa1772e341f6e06887", + "sha256:9b571b281a19762adb3f48a7731f6842f920fa71108aff9be49888320ac3e24d", + "sha256:a04fe458a4597aa559b79c7f48fe3dceabef0f69f562daf5c5e926b153817281", + "sha256:aa47ffcf70ba6f7848349b8a6f9b481ee0f7637931d91a9860a1838bfc586901", + "sha256:bede7ee075e54b9a5bde695b4fc8f569f30185891796b2e4e09e2226801d09bd", + "sha256:c1d2357f791b12d86faced7b5736dea9ef4f5ecdc6c3f253e445ee82da579449", + "sha256:c6eeefd4435055a8ebb6c5cc36111b8591c192c56a95b45fe2af22d9881eee25", + "sha256:ca1b7becf7d9d3ccdbb2f038f665c0f4857e08e1d8481cbcc1a86a0afcfb62b2", + "sha256:e67d4f8734f8054d2c4858570cc4b233bf753f56e85217de4dfb2495904cf02e", + "sha256:e8a34e4c0ab7b1ca17b8763613783e2458e77938092c18ac919420ab8655c8c1", + "sha256:e90491a4d77d0cb82e0e7a9cb35d86284c677402e4ce7ba6b448ccc7325c5421", + "sha256:ef1616b3ba965cd68e6f759eeb5d34fbf596a79e84215eeceebf34ba3f61fdc7", + "sha256:f222e1e9d3f13b68ff8a835574eda02e67277d51631d69d7cf7f8e07df678c86", + "sha256:f5e3088f4ed33947e16fd865b8200f9cfae1144f41b64a8cf19b599508e096bc", + "sha256:f659d7a48401158c59933904040085c200b4be631cb5f23a7d561fbae593ec1f", + "sha256:fe9c766a0c35b7e3d6b6939393c8dfdd5da3ac5dec7f971ec9134f284c6c36d6" ], - "version": "==0.4.0" + "version": "==0.5.0" }, "humanfriendly": { "hashes": [ @@ -496,11 +503,11 @@ }, "idna": { "hashes": [ - "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff", - "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d" + "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4", + "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2" ], "markers": "python_version >= '3.5'", - "version": "==3.3" + "version": "==3.4" }, "imap-tools": { "hashes": [ @@ -550,11 +557,11 @@ }, "joblib": { "hashes": [ - "sha256:4158fcecd13733f8be669be0683b96ebdbbd38d23559f54dca7205aea1bf1e35", - "sha256:f21f109b3c7ff9d95f8387f752d0d9c34a02aa2f7060c2135f465da0e5160ff6" + "sha256:091138ed78f800342968c523bdde947e7a305b8594b910a0fea2ab83c3c6d385", + "sha256:e1cee4a79e4af22881164f218d4311f60074197fb707e082e803b61f6d137018" ], - "markers": "python_version >= '3.6'", - "version": "==1.1.0" + "markers": "python_version >= '3.7'", + "version": "==1.2.0" }, "langdetect": { "hashes": [ @@ -979,10 +986,10 @@ }, "pyopenssl": { "hashes": [ - "sha256:660b1b1425aac4a1bea1d94168a85d99f0b3144c869dd4390d27629d0087f1bf", - "sha256:ea252b38c87425b64116f808355e8da644ef9b07e429398bfece610f893ee2e0" + "sha256:7a83b7b272dd595222d672f5ce29aa030f1fb837630ef229f62e72e395ce8968", + "sha256:b28437c9773bb6c6958628cf9c3bebe585de661dba6f63df17111966363dd15e" ], - "version": "==22.0.0" + "version": "==22.1.0" }, "pyparsing": { "hashes": [ @@ -1002,11 +1009,11 @@ }, "python-dotenv": { "hashes": [ - "sha256:b7e3b04a59693c42c36f9ab1cc2acc46fa5df8c78e178fc33a8d4cd05c8d498f", - "sha256:d92a187be61fe482e4fd675b6d52200e7be63a12b724abbf931a40ce4fa92938" + "sha256:1684eb44636dd462b66c3ee016599815514527ad99965de77f43e0944634a7e5", + "sha256:b77d08274639e3d34145dfa6c7008e66df0f04b7be7a75fd0d5292c191d79045" ], "index": "pypi", - "version": "==0.20.0" + "version": "==0.21.0" }, "python-gnupg": { "hashes": [ @@ -1365,11 +1372,11 @@ }, "setuptools": { "hashes": [ - "sha256:2e24e0bec025f035a2e72cdd1961119f557d78ad331bb00ff82efb2ab8da8e82", - "sha256:7732871f4f7fa58fb6bdcaeadb0161b2bd046c85905dbaa066bdcbcc81953b57" + "sha256:a8f6e213b4b0661f590ccf40de95d28a177cd747d098624ad3f69c40287297e9", + "sha256:c2d2709550f15aab6c9110196ea312f468f41cd546bceb24127a1be6fdcaeeb1" ], "markers": "python_version >= '3.7'", - "version": "==65.3.0" + "version": "==65.4.0" }, "six": { "hashes": [ @@ -1389,11 +1396,11 @@ }, "sqlparse": { "hashes": [ - "sha256:0c00730c74263a94e5a9919ade150dfc3b19c574389985446148402998287dae", - "sha256:48719e356bb8b42991bdbb1e8b83223757b93789c00910a616a071910ca4a64d" + "sha256:0323c0ec29cd52bceabc1b4d9d579e311f3e4961b98d174201d5622a23b85e34", + "sha256:69ca804846bb114d2ec380e4360a8a340db83f0ccf3afceeb1404df028f57268" ], "markers": "python_version >= '3.5'", - "version": "==0.4.2" + "version": "==0.4.3" }, "threadpoolctl": { "hashes": [ @@ -1423,11 +1430,11 @@ "tls" ], "hashes": [ - "sha256:a047990f57dfae1e0bd2b7df2526d4f16dcdc843774dc108b78c52f2a5f13680", - "sha256:f9f7a91f94932477a9fc3b169d57f54f96c6e74a23d78d9ce54039a7f48928a2" + "sha256:8d4718d1e48dcc28933f8beb48dc71cfe77a125e37ad1eb7a3d0acc49baf6c99", + "sha256:e5b60de39f2d1da153fbe1874d885fe3fcbdb21fcc446fa759a53e8fc3513bed" ], - "markers": "python_full_version >= '3.6.7'", - "version": "==22.4.0" + "markers": "python_full_version >= '3.7.1'", + "version": "==22.8.0" }, "txaio": { "hashes": [ @@ -1447,11 +1454,11 @@ }, "tzdata": { "hashes": [ - "sha256:21f4f0d7241572efa7f7a4fdabb052e61b55dc48274e6842697ccdf5253e5451", - "sha256:c3119520447d68ef3eb8187a55a4f44fa455f30eb1b4238fa5691ba094f2b05b" + "sha256:abf83da1e7cca85c0df7448627843efacb5a69a515e1f47ac01ec064ef66297f", + "sha256:f78d9c248154224aa0585fd0f6c2869428583455be52c410cb9c7f5717646854" ], "markers": "python_version >= '3.6'", - "version": "==2022.2" + "version": "==2022.3" }, "tzlocal": { "hashes": [ @@ -1482,24 +1489,38 @@ }, "uvloop": { "hashes": [ - "sha256:04ff57aa137230d8cc968f03481176041ae789308b4d5079118331ab01112450", - "sha256:089b4834fd299d82d83a25e3335372f12117a7d38525217c2258e9b9f4578897", - "sha256:1e5f2e2ff51aefe6c19ee98af12b4ae61f5be456cd24396953244a30880ad861", - "sha256:30ba9dcbd0965f5c812b7c2112a1ddf60cf904c1c160f398e7eed3a6b82dcd9c", - "sha256:3a19828c4f15687675ea912cc28bbcb48e9bb907c801873bd1519b96b04fb805", - "sha256:6224f1401025b748ffecb7a6e2652b17768f30b1a6a3f7b44660e5b5b690b12d", - "sha256:647e481940379eebd314c00440314c81ea547aa636056f554d491e40503c8464", - "sha256:6ccd57ae8db17d677e9e06192e9c9ec4bd2066b77790f9aa7dede2cc4008ee8f", - "sha256:772206116b9b57cd625c8a88f2413df2fcfd0b496eb188b82a43bed7af2c2ec9", - "sha256:8e0d26fa5875d43ddbb0d9d79a447d2ace4180d9e3239788208527c4784f7cab", - "sha256:98d117332cc9e5ea8dfdc2b28b0a23f60370d02e1395f88f40d1effd2cb86c4f", - "sha256:b572256409f194521a9895aef274cea88731d14732343da3ecdb175228881638", - "sha256:bd53f7f5db562f37cd64a3af5012df8cac2c464c97e732ed556800129505bd64", - "sha256:bd8f42ea1ea8f4e84d265769089964ddda95eb2bb38b5cbe26712b0616c3edee", - "sha256:e814ac2c6f9daf4c36eb8e85266859f42174a4ff0d71b99405ed559257750382", - "sha256:f74bc20c7b67d1c27c72601c78cf95be99d5c2cdd4514502b4f3eb0933ff1228" + "sha256:0949caf774b9fcefc7c5756bacbbbd3fc4c05a6b7eebc7c7ad6f825b23998d6d", + "sha256:0ddf6baf9cf11a1a22c71487f39f15b2cf78eb5bde7e5b45fbb99e8a9d91b9e1", + "sha256:1436c8673c1563422213ac6907789ecb2b070f5939b9cbff9ef7113f2b531595", + "sha256:23609ca361a7fc587031429fa25ad2ed7242941adec948f9d10c045bfecab06b", + "sha256:2a6149e1defac0faf505406259561bc14b034cdf1d4711a3ddcdfbaa8d825a05", + "sha256:2deae0b0fb00a6af41fe60a675cec079615b01d68beb4cc7b722424406b126a8", + "sha256:307958f9fc5c8bb01fad752d1345168c0abc5d62c1b72a4a8c6c06f042b45b20", + "sha256:30babd84706115626ea78ea5dbc7dd8d0d01a2e9f9b306d24ca4ed5796c66ded", + "sha256:3378eb62c63bf336ae2070599e49089005771cc651c8769aaad72d1bd9385a7c", + "sha256:3d97672dc709fa4447ab83276f344a165075fd9f366a97b712bdd3fee05efae8", + "sha256:3db8de10ed684995a7f34a001f15b374c230f7655ae840964d51496e2f8a8474", + "sha256:3ebeeec6a6641d0adb2ea71dcfb76017602ee2bfd8213e3fcc18d8f699c5104f", + "sha256:45cea33b208971e87a31c17622e4b440cac231766ec11e5d22c76fab3bf9df62", + "sha256:6708f30db9117f115eadc4f125c2a10c1a50d711461699a0cbfaa45b9a78e376", + "sha256:68532f4349fd3900b839f588972b3392ee56042e440dd5873dfbbcd2cc67617c", + "sha256:6aafa5a78b9e62493539456f8b646f85abc7093dd997f4976bb105537cf2635e", + "sha256:7d37dccc7ae63e61f7b96ee2e19c40f153ba6ce730d8ba4d3b4e9738c1dccc1b", + "sha256:864e1197139d651a76c81757db5eb199db8866e13acb0dfe96e6fc5d1cf45fc4", + "sha256:8887d675a64cfc59f4ecd34382e5b4f0ef4ae1da37ed665adba0c2badf0d6578", + "sha256:8efcadc5a0003d3a6e887ccc1fb44dec25594f117a94e3127954c05cf144d811", + "sha256:9b09e0f0ac29eee0451d71798878eae5a4e6a91aa275e114037b27f7db72702d", + "sha256:a4aee22ece20958888eedbad20e4dbb03c37533e010fb824161b4f05e641f738", + "sha256:a5abddb3558d3f0a78949c750644a67be31e47936042d4f6c888dd6f3c95f4aa", + "sha256:c092a2c1e736086d59ac8e41f9c98f26bbf9b9222a76f21af9dfe949b99b2eb9", + "sha256:c686a47d57ca910a2572fddfe9912819880b8765e2f01dc0dd12a9bf8573e539", + "sha256:cbbe908fda687e39afd6ea2a2f14c2c3e43f2ca88e3a11964b297822358d0e6c", + "sha256:ce9f61938d7155f79d3cb2ffa663147d4a76d16e08f65e2c66b77bd41b356718", + "sha256:dbbaf9da2ee98ee2531e0c780455f2841e4675ff580ecf93fe5c48fe733b5667", + "sha256:f1e507c9ee39c61bfddd79714e4f85900656db1aec4d40c6de55648e85c2799c", + "sha256:ff3d00b70ce95adce264462c930fbaecb29718ba6563db354608f37e49e09024" ], - "version": "==0.16.0" + "version": "==0.17.0" }, "watchdog": { "hashes": [ @@ -1534,26 +1555,26 @@ }, "watchfiles": { "hashes": [ - "sha256:1e41c8b4bf3e07c18aa51775b36b718830fa727929529a7d6e5b38cf845a06b4", - "sha256:22af3b915f928ef59d427d7228668f87ac8054ed8200808c73fbcaa4f82d5572", - "sha256:2a3debb19912072799d7ca53e99fc5f090f77948f5601392623b2a416b4c86be", - "sha256:35f3e411822e14a35f2ef656535aad4e6e79670d6b6ef8e53db958e28916b1fe", - "sha256:44c6aff58b8a70a26431737e483a54e8e224279b21873388571ed184fe7c91a7", - "sha256:4a6a1ac96edf5bc3f8e36f4462fc1daad0ec3769ff2adb920571e120e37c91c5", - "sha256:5741246ae399a03395aa5ee35480083a4f29d58ffd41dd3395594f8805f8cdbc", - "sha256:70159e759f52b65a50c498182dece80364bfd721e839c254c328cbc7a1716616", - "sha256:75a4b9cec1b1c337ea77d4428b29861553d6bf8179923b1bc7e825e217460e2c", - "sha256:91d1b2d0cf060e5222a930a3e2f40f6577da1d18c085c32741b98a128dc1e72c", - "sha256:a8a1809bf910672aa0b7ed6e6045d4fc2cf1e0718b99bc443ef17faa5697b68a", - "sha256:aed7575e24434c8fec2f2bbb0cecb1521ea1240234d9108db7915a3424d92394", - "sha256:b2c7ad91a867dd688b9a12097dd6a4f89397b43fccee871152aa67197cc94398", - "sha256:baa6d0c1c5140e1dcf6ff802dd7b09fcd95b358e50d42fabc83d83f719451c54", - "sha256:c9a7a6dc63684ff5ba11f0be0e64f744112c3c7a0baf4ec8f6794f9a6257d21e", - "sha256:cd7d2fd9a8f28066edc8db5278f3632eb94d10596af760fa0601631f32b1a41e", - "sha256:e939a2693404ac11e055f9d1237db8ad7635e2185a6143bde00116e691ea2983", - "sha256:f91035a273001390093a09e52274a34695b0d15ee8736183b640bbc3b8a432ab" + "sha256:00e5f307a58752ec1478eeb738863544bde21cc7a2728bd1c216060406bde9c1", + "sha256:1dd1e3181ad5d83ca35e9147c72e24f39437fcdf570c9cdc532016399fb62957", + "sha256:204950f1d6083539af5c8b7d4f5f8039c3ce36fa692da12d9743448f3199cb15", + "sha256:4056398d8f6d4972fe0918707b59d4cb84470c91d3c37f0e11e5a66c2a598760", + "sha256:539bcdb55a487126776c9d8c011094214d1df3f9a2321a6c0b1583197309405a", + "sha256:53a2faeb121bc51bb6b960984f46901227e2e2475acc5a8d4c905a600436752d", + "sha256:58dc3140dcf02a8aa76464a77a093016f10e89306fec21a4814922a64f3e8b9f", + "sha256:6a3d6c699f3ce238dfa90bcef501f331a69b0d9b076f14459ed8eab26ba2f4cf", + "sha256:92675f379a9d5adbc6a52179f3e39aa56944c6eecb80384608fff2ed2619103a", + "sha256:a53cb6c06e5c1f216c792fbb432ce315239d432cb8b68d508547100939ec0399", + "sha256:a7f4271af86569bdbf131dd5c7c121c45d0ed194f3c88b88326e48a3b6a2db12", + "sha256:ad2bdcae4c0f07ca6c090f5a2c30188cc6edba011b45e7c96eb1896648092367", + "sha256:adcf15ecc2182ea9d2358c1a8c2b53203c3909484918776929b7bbe205522c0e", + "sha256:ae7c57ef920589a40270d5ef3216d693f4e6f8864d8fc8b6cb7885ca98ad2a61", + "sha256:afd35a1bd3b9e68efe384ae7538481ae725597feb66f56f4bd23ecdbda726da0", + "sha256:b5c334cd3bc88aa4a8a1e08ec9f702b63c947211275defdc2dd79dc037fcb500", + "sha256:c7e1ffbd03cbcb46d1b7833e10e7d6b678ab083b4e4b80db06cfff5baca3c93f", + "sha256:ffff3418dc753a2aed2d00200a4daeaac295c40458f8012836a65555f288be8b" ], - "version": "==0.16.1" + "version": "==0.17.0" }, "wcwidth": { "hashes": [ @@ -1823,11 +1844,11 @@ }, "certifi": { "hashes": [ - "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d", - "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412" + "sha256:0d9c601124e5a6ba9712dbc60d9c53c21e34f5f641fe83002317394311bdce14", + "sha256:90c1a32f1d68f940488354e36370f6cca89f0f106db09518524c88d6ed83f382" ], - "markers": "python_full_version >= '3.6.0'", - "version": "==2022.6.15" + "markers": "python_version >= '3.6'", + "version": "==2022.9.24" }, "cfgv": { "hashes": [ @@ -1842,7 +1863,7 @@ "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845", "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==2.1.1" }, "click": { @@ -1964,11 +1985,11 @@ }, "faker": { "hashes": [ - "sha256:6db56e2c43a2b74250d1c332ef25fef7dc07dcb6c5fab5329dd7b4467b8ed7b9", - "sha256:e02c55a5b0586caaf913cc6c254b3de178e08b031c5922e590fd033ebbdbfd02" + "sha256:2e28aaea60456857d4ce95dd12aed767769537ad23d13d51a545cd40a654e9d9", + "sha256:daad7badb4fd916bd047b28c8459ef4689e4fe6acf61f6dfebee8cc602e4d009" ], "markers": "python_version >= '3.6'", - "version": "==14.2.0" + "version": "==14.2.1" }, "filelock": { "hashes": [ @@ -1988,11 +2009,11 @@ }, "idna": { "hashes": [ - "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff", - "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d" + "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4", + "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2" ], "markers": "python_version >= '3.5'", - "version": "==3.3" + "version": "==3.4" }, "imagesize": { "hashes": [ @@ -2313,11 +2334,11 @@ }, "setuptools": { "hashes": [ - "sha256:2e24e0bec025f035a2e72cdd1961119f557d78ad331bb00ff82efb2ab8da8e82", - "sha256:7732871f4f7fa58fb6bdcaeadb0161b2bd046c85905dbaa066bdcbcc81953b57" + "sha256:a8f6e213b4b0661f590ccf40de95d28a177cd747d098624ad3f69c40287297e9", + "sha256:c2d2709550f15aab6c9110196ea312f468f41cd546bceb24127a1be6fdcaeeb1" ], "markers": "python_version >= '3.7'", - "version": "==65.3.0" + "version": "==65.4.0" }, "six": { "hashes": [ @@ -2336,11 +2357,11 @@ }, "sphinx": { "hashes": [ - "sha256:309a8da80cb6da9f4713438e5b55861877d5d7976b69d87e336733637ea12693", - "sha256:ba3224a4e206e1fbdecf98a4fae4992ef9b24b85ebf7b584bb340156eaf08d89" + "sha256:3dcf00fcf82cf91118db9b7177edea4fc01998976f893928d0ab0c58c54be2ca", + "sha256:c009bb2e9ac5db487bcf53f015504005a330ff7c631bb6ab2604e0d65bae8b54" ], "index": "pypi", - "version": "==5.1.1" + "version": "==5.2.1" }, "sphinx-autobuild": { "hashes": [ @@ -2408,9 +2429,11 @@ }, "termcolor": { "hashes": [ - "sha256:1d6d69ce66211143803fbc56652b41d73b4a400a2891d7bf7a1cdf4c02de613b" + "sha256:6b2cf769e93364a2676e1de56a7c0cff2cf5bd07f37e9cc80b0dd6320ebfe388", + "sha256:7e597f9de8e001a3208c4132938597413b9da45382b6f1d150cff8d062b7aaa3" ], - "version": "==1.1.0" + "markers": "python_version >= '3.7'", + "version": "==2.0.1" }, "toml": { "hashes": [ @@ -2447,11 +2470,11 @@ }, "tox": { "hashes": [ - "sha256:c138327815f53bc6da4fe56baec5f25f00622ae69ef3fe4e1e385720e22486f9", - "sha256:c38e15f4733683a9cc0129fba078633e07eb0961f550a010ada879e95fb32632" + "sha256:44f3c347c68c2c68799d7d44f1808f9d396fc8a1a500cbc624253375c7ae107e", + "sha256:bf037662d7c740d15c9924ba23bb3e587df20598697bb985ac2b49bdc2d847f6" ], "index": "pypi", - "version": "==3.25.1" + "version": "==3.26.0" }, "typing-extensions": { "hashes": [ @@ -2471,11 +2494,11 @@ }, "virtualenv": { "hashes": [ - "sha256:014f766e4134d0008dcaa1f95bafa0fb0f575795d07cae50b1bee514185d6782", - "sha256:035ed57acce4ac35c82c9d8802202b0e71adac011a511ff650cbcf9635006a22" + "sha256:227ea1b9994fdc5ea31977ba3383ef296d7472ea85be9d6732e42a91c04e80da", + "sha256:d07dfc5df5e4e0dbc92862350ad87a36ed505b978f6c39609dc489eadd5b0d27" ], "markers": "python_version >= '3.6'", - "version": "==20.16.4" + "version": "==20.16.5" } } } From 617055fca73af04e879e96303c9988ccffb6d6df Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+shamoon@users.noreply.github.com> Date: Mon, 12 Sep 2022 12:54:28 -0700 Subject: [PATCH 006/273] Save slimsidebar settings to db on change --- .../app-frame/app-frame.component.ts | 26 ++++++++++++++++++- src-ui/src/app/data/paperless-uisettings.ts | 6 +++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/src-ui/src/app/components/app-frame/app-frame.component.ts b/src-ui/src/app/components/app-frame/app-frame.component.ts index 6e758d262..9c415360c 100644 --- a/src-ui/src/app/components/app-frame/app-frame.component.ts +++ b/src-ui/src/app/components/app-frame/app-frame.component.ts @@ -24,6 +24,8 @@ import { import { SettingsService } from 'src/app/services/settings.service' import { TasksService } from 'src/app/services/tasks.service' import { ComponentCanDeactivate } from 'src/app/guards/dirty-doc.guard' +import { SETTINGS_KEYS } from 'src/app/data/paperless-uisettings' +import { ToastService } from 'src/app/services/toast.service' @Component({ selector: 'app-app-frame', @@ -40,7 +42,8 @@ export class AppFrameComponent implements ComponentCanDeactivate { private remoteVersionService: RemoteVersionService, private list: DocumentListViewService, public settingsService: SettingsService, - public tasksService: TasksService + public tasksService: TasksService, + private toastService: ToastService ) { this.remoteVersionService .checkForUpdates() @@ -55,6 +58,27 @@ export class AppFrameComponent implements ComponentCanDeactivate { isMenuCollapsed: boolean = true + get slimSidebarEnabled(): boolean { + return this.settingsService.get(SETTINGS_KEYS.SLIM_SIDEBAR) + } + + set slimSidebarEnabled(enabled: boolean) { + console.log('set slimSidebarEnabled', enabled) + + this.settingsService.set(SETTINGS_KEYS.SLIM_SIDEBAR, enabled) + this.settingsService + .storeSettings() + .pipe(first()) + .subscribe({ + error: (error) => { + this.toastService.showError( + $localize`An error occurred while saving settings.` + ) + console.log(error) + }, + }) + } + closeMenu() { this.isMenuCollapsed = true } diff --git a/src-ui/src/app/data/paperless-uisettings.ts b/src-ui/src/app/data/paperless-uisettings.ts index e3d977687..a5fdef51f 100644 --- a/src-ui/src/app/data/paperless-uisettings.ts +++ b/src-ui/src/app/data/paperless-uisettings.ts @@ -37,6 +37,7 @@ export const SETTINGS_KEYS = { NOTIFICATIONS_CONSUMER_SUPPRESS_ON_DASHBOARD: 'general-settings:notifications:consumer-suppress-on-dashboard', COMMENTS_ENABLED: 'general-settings:comments-enabled', + SLIM_SIDEBAR: 'general-settings:slim-sidebar', } export const SETTINGS: PaperlessUiSetting[] = [ @@ -55,6 +56,11 @@ export const SETTINGS: PaperlessUiSetting[] = [ type: 'boolean', default: false, }, + { + key: SETTINGS_KEYS.SLIM_SIDEBAR, + type: 'boolean', + default: false, + }, { key: SETTINGS_KEYS.DOCUMENT_LIST_SIZE, type: 'number', From 1330390b4f8e7e030b05f9084a35272d04950128 Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+shamoon@users.noreply.github.com> Date: Mon, 12 Sep 2022 14:51:34 -0700 Subject: [PATCH 007/273] slim sidebar with doc list adjusting --- .../app-frame/app-frame.component.html | 88 ++++++++++--------- .../app-frame/app-frame.component.scss | 44 ++++++++++ .../app-frame/app-frame.component.ts | 2 - .../document-list.component.scss | 19 +++- src-ui/src/styles.scss | 5 ++ 5 files changed, 114 insertions(+), 44 deletions(-) diff --git a/src-ui/src/app/components/app-frame/app-frame.component.html b/src-ui/src/app/components/app-frame/app-frame.component.html index 5a68a3cda..9e96316bd 100644 --- a/src-ui/src/app/components/app-frame/app-frame.component.html +++ b/src-ui/src/app/components/app-frame/app-frame.component.html @@ -4,11 +4,11 @@ (click)="isMenuCollapsed = !isMenuCollapsed"> - - + + - Paperless-ngx + Paperless-ngx
@@ -51,48 +51,54 @@
diff --git a/src-ui/src/app/components/app-frame/app-frame.component.scss b/src-ui/src/app/components/app-frame/app-frame.component.scss index 296df26c3..56b35fb2c 100644 --- a/src-ui/src/app/components/app-frame/app-frame.component.scss +++ b/src-ui/src/app/components/app-frame/app-frame.component.scss @@ -14,6 +14,18 @@ width: 0.8em; height: 0.8em; } + + + // These come from the col-md-3 col-lg-2 classes for regular sidebar, needed for animation + @media (min-width: 768px) { + max-width: 25%; + } + + @media (min-width: 992px) { + max-width: 16.66666667%; + } + + transition: all .3s ease; } @media (max-width: 767.98px) { .sidebar { @@ -21,6 +33,38 @@ } } +.sidebar-slim-toggler { + display: none; // hide on mobile +} + +@media(min-width: 768px) { + .sidebar.slim { + max-width: 50px; + li.nav-item span { + display: none; + } + } + + .col.slim { + margin-left: 50px !important; + } + + .sidebar-slim-toggler { + display: block; + position: absolute; + right: -12px; + top: 60px; + z-index: 996; + --bs-btn-padding-x: 0.35rem; + --bs-btn-padding-y: 0.125rem; + } +} + +::ng-deep .popover-slim .popover-body { + --bs-popover-body-padding-x: .5rem; + --bs-popover-body-padding-y: .5rem; +} + .sidebar-sticky { position: relative; top: 0; diff --git a/src-ui/src/app/components/app-frame/app-frame.component.ts b/src-ui/src/app/components/app-frame/app-frame.component.ts index 9c415360c..1b1adb706 100644 --- a/src-ui/src/app/components/app-frame/app-frame.component.ts +++ b/src-ui/src/app/components/app-frame/app-frame.component.ts @@ -63,8 +63,6 @@ export class AppFrameComponent implements ComponentCanDeactivate { } set slimSidebarEnabled(enabled: boolean) { - console.log('set slimSidebarEnabled', enabled) - this.settingsService.set(SETTINGS_KEYS.SLIM_SIDEBAR, enabled) this.settingsService .storeSettings() diff --git a/src-ui/src/app/components/document-list/document-list.component.scss b/src-ui/src/app/components/document-list/document-list.component.scss index b2d138d7f..98b6d62d7 100644 --- a/src-ui/src/app/components/document-list/document-list.component.scss +++ b/src-ui/src/app/components/document-list/document-list.component.scss @@ -11,7 +11,7 @@ tr { } $paperless-card-breakpoints: ( - 0: 2, // xs + // 0: 2, // xs is manual for slim-sidebar 768px: 3, //md 992px: 4, //lg 1200px: 5, //xl @@ -22,6 +22,12 @@ $paperless-card-breakpoints: ( ); .row-cols-paperless-cards { + // xs, we dont want in .col.slim block + > * { + flex: 0 0 auto; + width: calc(100% / 2); + } + @each $width, $n_cols in $paperless-card-breakpoints { @media(min-width: $width) { > * { @@ -32,6 +38,17 @@ $paperless-card-breakpoints: ( } } +::ng-deep .col.slim .row-cols-paperless-cards { + @each $width, $n_cols in $paperless-card-breakpoints { + @media(min-width: $width) { + > * { + flex: 0 0 auto; + width: calc(100% / ($n-cols + 1)); + } + } + } +} + .dropdown-menu-right { right: 0 !important; left: auto !important; diff --git a/src-ui/src/styles.scss b/src-ui/src/styles.scss index 18413276a..8dbbafe22 100644 --- a/src-ui/src/styles.scss +++ b/src-ui/src/styles.scss @@ -405,6 +405,11 @@ textarea, vertical-align: text-bottom; } +.sidebaricon-sm { + width: 12px; + height: 12px; +} + table.table { color: var(--bs-body-color); From aa55162e2e140a5638384ddde08d27d968f7abb7 Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+shamoon@users.noreply.github.com> Date: Mon, 12 Sep 2022 15:26:34 -0700 Subject: [PATCH 008/273] Settings UI with live updating --- .../manage/settings/settings.component.html | 11 ++ .../manage/settings/settings.component.ts | 106 ++++++++++-------- src-ui/src/app/services/settings.service.ts | 7 +- 3 files changed, 75 insertions(+), 49 deletions(-) diff --git a/src-ui/src/app/components/manage/settings/settings.component.html b/src-ui/src/app/components/manage/settings/settings.component.html index f72587139..c215e2e19 100644 --- a/src-ui/src/app/components/manage/settings/settings.component.html +++ b/src-ui/src/app/components/manage/settings/settings.component.html @@ -89,6 +89,17 @@
+
+
+ Sidebar +
+
+ + + +
+
+
Dark mode diff --git a/src-ui/src/app/components/manage/settings/settings.component.ts b/src-ui/src/app/components/manage/settings/settings.component.ts index bb7244663..dcdb6bc64 100644 --- a/src-ui/src/app/components/manage/settings/settings.component.ts +++ b/src-ui/src/app/components/manage/settings/settings.component.ts @@ -1,11 +1,4 @@ -import { - Component, - Inject, - LOCALE_ID, - OnInit, - OnDestroy, - Renderer2, -} from '@angular/core' +import { Component, Inject, LOCALE_ID, OnInit, OnDestroy } from '@angular/core' import { FormControl, FormGroup } from '@angular/forms' import { PaperlessSavedView } from 'src/app/data/paperless-saved-view' import { DocumentListViewService } from 'src/app/services/document-list-view.service' @@ -18,6 +11,7 @@ import { Toast, ToastService } from 'src/app/services/toast.service' import { dirtyCheck, DirtyComponent } from '@ngneat/dirty-check-forms' import { Observable, Subscription, BehaviorSubject, first } from 'rxjs' import { SETTINGS_KEYS } from 'src/app/data/paperless-uisettings' +import { ForwardRefHandling } from '@angular/compiler' @Component({ selector: 'app-settings', @@ -31,6 +25,7 @@ export class SettingsComponent implements OnInit, OnDestroy, DirtyComponent { bulkEditConfirmationDialogs: new FormControl(null), bulkEditApplyOnClose: new FormControl(null), documentListItemPerPage: new FormControl(null), + slimSidebarEnabled: new FormControl(null), darkModeUseSystem: new FormControl(null), darkModeEnabled: new FormControl(null), darkModeInvertThumbs: new FormControl(null), @@ -75,50 +70,61 @@ export class SettingsComponent implements OnInit, OnDestroy, DirtyComponent { private toastService: ToastService, private settings: SettingsService, @Inject(LOCALE_ID) public currentLocale: string - ) {} + ) { + this.settings.changed.subscribe({ + next: () => { + this.settingsForm.patchValue(this.getCurrentSettings(), { + emitEvent: false, + }) + }, + }) + } + + private getCurrentSettings() { + return { + bulkEditConfirmationDialogs: this.settings.get( + SETTINGS_KEYS.BULK_EDIT_CONFIRMATION_DIALOGS + ), + bulkEditApplyOnClose: this.settings.get( + SETTINGS_KEYS.BULK_EDIT_APPLY_ON_CLOSE + ), + documentListItemPerPage: this.settings.get( + SETTINGS_KEYS.DOCUMENT_LIST_SIZE + ), + slimSidebarEnabled: this.settings.get(SETTINGS_KEYS.SLIM_SIDEBAR), + darkModeUseSystem: this.settings.get(SETTINGS_KEYS.DARK_MODE_USE_SYSTEM), + darkModeEnabled: this.settings.get(SETTINGS_KEYS.DARK_MODE_ENABLED), + darkModeInvertThumbs: this.settings.get( + SETTINGS_KEYS.DARK_MODE_THUMB_INVERTED + ), + themeColor: this.settings.get(SETTINGS_KEYS.THEME_COLOR), + useNativePdfViewer: this.settings.get( + SETTINGS_KEYS.USE_NATIVE_PDF_VIEWER + ), + savedViews: {}, + displayLanguage: this.settings.getLanguage(), + dateLocale: this.settings.get(SETTINGS_KEYS.DATE_LOCALE), + dateFormat: this.settings.get(SETTINGS_KEYS.DATE_FORMAT), + notificationsConsumerNewDocument: this.settings.get( + SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_NEW_DOCUMENT + ), + notificationsConsumerSuccess: this.settings.get( + SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_SUCCESS + ), + notificationsConsumerFailed: this.settings.get( + SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_FAILED + ), + notificationsConsumerSuppressOnDashboard: this.settings.get( + SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_SUPPRESS_ON_DASHBOARD + ), + commentsEnabled: this.settings.get(SETTINGS_KEYS.COMMENTS_ENABLED), + } + } ngOnInit() { this.savedViewService.listAll().subscribe((r) => { this.savedViews = r.results - let storeData = { - bulkEditConfirmationDialogs: this.settings.get( - SETTINGS_KEYS.BULK_EDIT_CONFIRMATION_DIALOGS - ), - bulkEditApplyOnClose: this.settings.get( - SETTINGS_KEYS.BULK_EDIT_APPLY_ON_CLOSE - ), - documentListItemPerPage: this.settings.get( - SETTINGS_KEYS.DOCUMENT_LIST_SIZE - ), - darkModeUseSystem: this.settings.get( - SETTINGS_KEYS.DARK_MODE_USE_SYSTEM - ), - darkModeEnabled: this.settings.get(SETTINGS_KEYS.DARK_MODE_ENABLED), - darkModeInvertThumbs: this.settings.get( - SETTINGS_KEYS.DARK_MODE_THUMB_INVERTED - ), - themeColor: this.settings.get(SETTINGS_KEYS.THEME_COLOR), - useNativePdfViewer: this.settings.get( - SETTINGS_KEYS.USE_NATIVE_PDF_VIEWER - ), - savedViews: {}, - displayLanguage: this.settings.getLanguage(), - dateLocale: this.settings.get(SETTINGS_KEYS.DATE_LOCALE), - dateFormat: this.settings.get(SETTINGS_KEYS.DATE_FORMAT), - notificationsConsumerNewDocument: this.settings.get( - SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_NEW_DOCUMENT - ), - notificationsConsumerSuccess: this.settings.get( - SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_SUCCESS - ), - notificationsConsumerFailed: this.settings.get( - SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_FAILED - ), - notificationsConsumerSuppressOnDashboard: this.settings.get( - SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_SUPPRESS_ON_DASHBOARD - ), - commentsEnabled: this.settings.get(SETTINGS_KEYS.COMMENTS_ENABLED), - } + let storeData = this.getCurrentSettings() for (let view of this.savedViews) { storeData.savedViews[view.id.toString()] = { @@ -192,6 +198,10 @@ export class SettingsComponent implements OnInit, OnDestroy, DirtyComponent { SETTINGS_KEYS.DOCUMENT_LIST_SIZE, this.settingsForm.value.documentListItemPerPage ) + this.settings.set( + SETTINGS_KEYS.SLIM_SIDEBAR, + this.settingsForm.value.slimSidebarEnabled + ) this.settings.set( SETTINGS_KEYS.DARK_MODE_USE_SYSTEM, this.settingsForm.value.darkModeUseSystem diff --git a/src-ui/src/app/services/settings.service.ts b/src-ui/src/app/services/settings.service.ts index 14d261789..ae48e373d 100644 --- a/src-ui/src/app/services/settings.service.ts +++ b/src-ui/src/app/services/settings.service.ts @@ -1,6 +1,7 @@ import { DOCUMENT } from '@angular/common' import { HttpClient } from '@angular/common/http' import { + EventEmitter, Inject, Injectable, LOCALE_ID, @@ -46,6 +47,8 @@ export class SettingsService { public displayName: string + public changed = new EventEmitter() + constructor( rendererFactory: RendererFactory2, @Inject(DOCUMENT) private document, @@ -360,7 +363,9 @@ export class SettingsService { } storeSettings(): Observable { - return this.http.post(this.baseUrl, { settings: this.settings }) + return this.http + .post(this.baseUrl, { settings: this.settings }) + .pipe(tap((result) => this.changed.emit(!!result.success))) } maybeMigrateSettings() { From e0f93c26d617309542c71251362e8c9d863f26b5 Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+shamoon@users.noreply.github.com> Date: Tue, 13 Sep 2022 11:47:19 -0700 Subject: [PATCH 009/273] Nicer slim sidebar transitioning, dark mode fixes, fix tests by making settings taller --- src-ui/cypress/e2e/settings/settings.cy.ts | 2 +- .../components/app-frame/app-frame.component.html | 2 +- .../components/app-frame/app-frame.component.scss | 14 ++++++++++---- .../document-list/document-list.component.scss | 6 +++--- src-ui/src/styles.scss | 11 ++++++++++- src-ui/src/theme.scss | 9 +++++++++ 6 files changed, 34 insertions(+), 10 deletions(-) diff --git a/src-ui/cypress/e2e/settings/settings.cy.ts b/src-ui/cypress/e2e/settings/settings.cy.ts index 7433d16f4..0480e39ce 100644 --- a/src-ui/cypress/e2e/settings/settings.cy.ts +++ b/src-ui/cypress/e2e/settings/settings.cy.ts @@ -46,7 +46,7 @@ describe('settings', () => { }) }) - cy.viewport(1024, 1024) + cy.viewport(1024, 1600) cy.visit('/settings') cy.wait('@savedViews') }) diff --git a/src-ui/src/app/components/app-frame/app-frame.component.html b/src-ui/src/app/components/app-frame/app-frame.component.html index 9e96316bd..c1d6a20bd 100644 --- a/src-ui/src/app/components/app-frame/app-frame.component.html +++ b/src-ui/src/app/components/app-frame/app-frame.component.html @@ -237,7 +237,7 @@
-
+
diff --git a/src-ui/src/app/components/app-frame/app-frame.component.scss b/src-ui/src/app/components/app-frame/app-frame.component.scss index 56b35fb2c..717a9b976 100644 --- a/src-ui/src/app/components/app-frame/app-frame.component.scss +++ b/src-ui/src/app/components/app-frame/app-frame.component.scss @@ -1,3 +1,6 @@ +@import "node_modules/bootstrap/scss/functions"; +@import "node_modules/bootstrap/scss/variables"; + /* * Sidebar */ @@ -15,7 +18,6 @@ height: 0.8em; } - // These come from the col-md-3 col-lg-2 classes for regular sidebar, needed for animation @media (min-width: 768px) { max-width: 25%; @@ -25,7 +27,7 @@ max-width: 16.66666667%; } - transition: all .3s ease; + transition: all .2s ease; } @media (max-width: 767.98px) { .sidebar { @@ -33,6 +35,10 @@ } } +main { + transition: all .2s ease; +} + .sidebar-slim-toggler { display: none; // hide on mobile } @@ -45,8 +51,8 @@ } } - .col.slim { - margin-left: 50px !important; + .col-slim { + padding-left: calc(50px + $grid-gutter-width) !important; } .sidebar-slim-toggler { diff --git a/src-ui/src/app/components/document-list/document-list.component.scss b/src-ui/src/app/components/document-list/document-list.component.scss index 98b6d62d7..3361d7c3a 100644 --- a/src-ui/src/app/components/document-list/document-list.component.scss +++ b/src-ui/src/app/components/document-list/document-list.component.scss @@ -22,7 +22,7 @@ $paperless-card-breakpoints: ( ); .row-cols-paperless-cards { - // xs, we dont want in .col.slim block + // xs, we dont want in .col-slim block > * { flex: 0 0 auto; width: calc(100% / 2); @@ -38,12 +38,12 @@ $paperless-card-breakpoints: ( } } -::ng-deep .col.slim .row-cols-paperless-cards { +::ng-deep .col-slim .row-cols-paperless-cards { @each $width, $n_cols in $paperless-card-breakpoints { @media(min-width: $width) { > * { flex: 0 0 auto; - width: calc(100% / ($n-cols + 1)); + width: calc(100% / ($n-cols + 1)) !important; } } } diff --git a/src-ui/src/styles.scss b/src-ui/src/styles.scss index 8dbbafe22..0ffb12afb 100644 --- a/src-ui/src/styles.scss +++ b/src-ui/src/styles.scss @@ -1,9 +1,9 @@ // bs options $enable-negative-margins: true; +@import "theme"; @import "node_modules/bootstrap/scss/bootstrap"; @import "~@ng-select/ng-select/themes/default.theme.css"; -@import "theme"; @import "print"; // Paperless-ngx styles @@ -84,6 +84,15 @@ svg.logo { } } +.btn-dark { + --bs-btn-color: var(--bs-gray-600); + --bs-btn-bg: var(--bs-gray-200); + --bs-btn-border-color: var(--bs-gray-200); + --bs-btn-hover-bg: var(--bs-gray-400); + --bs-btn-hover-border-color: var(--bs-gray-500); + --bs-btn-active-bg: var(--bs-gray-200); +} + .text-primary { color: var(--bs-primary) !important; } diff --git a/src-ui/src/theme.scss b/src-ui/src/theme.scss index bf9be6662..99dcb8bab 100644 --- a/src-ui/src/theme.scss +++ b/src-ui/src/theme.scss @@ -79,6 +79,15 @@ $form-check-radio-checked-bg-image-dark: url("data:image/svg+xml,= '3.7'", + "version": "==15.0.0" }, "filelock": { "hashes": [ diff --git a/docker-builders/Dockerfile.pikepdf b/docker-builders/Dockerfile.pikepdf index 5f0c3eea2..750306360 100644 --- a/docker-builders/Dockerfile.pikepdf +++ b/docker-builders/Dockerfile.pikepdf @@ -60,7 +60,7 @@ RUN set -eux \ && apt-get update --quiet \ && apt-get install --yes --quiet --no-install-recommends ${BUILD_PACKAGES} \ && echo "Installing qpdf" \ - && dpkg --install libqpdf28_*.deb \ + && dpkg --install libqpdf29_*.deb \ && dpkg --install libqpdf-dev_*.deb \ && echo "Installing Python tools" \ && python3 -m pip install --no-cache-dir --upgrade \ diff --git a/docker-builders/Dockerfile.qpdf b/docker-builders/Dockerfile.qpdf index 0cc5fba24..de27b2d5d 100644 --- a/docker-builders/Dockerfile.qpdf +++ b/docker-builders/Dockerfile.qpdf @@ -1,7 +1,7 @@ # This Dockerfile compiles the jbig2enc library # Inputs: # - QPDF_VERSION - the version of qpdf to build a .deb. -# Must be preset as a deb-src +# Must be present as a deb-src in bookworm FROM debian:bullseye-slim as main @@ -22,27 +22,23 @@ ARG BUILD_PACKAGES="\ libjpeg62-turbo-dev \ libgnutls28-dev \ packaging-dev \ + cmake \ zlib1g-dev" WORKDIR /usr/src -# As this is an base image for a multi-stage final image -# the added size of the install is basically irrelevant - RUN set -eux \ && echo "Installing build tools" \ && apt-get update --quiet \ && apt-get install --yes --quiet --no-install-recommends $BUILD_PACKAGES \ - && echo "Building qpdf" \ + && echo "Getting qpdf src" \ && echo "deb-src http://deb.debian.org/debian/ bookworm main" > /etc/apt/sources.list.d/bookworm-src.list \ && apt-get update \ && mkdir qpdf \ && cd qpdf \ && apt-get source --yes --quiet qpdf=${QPDF_VERSION}-1/bookworm \ + && echo "Building qpdf" \ && cd qpdf-$QPDF_VERSION \ - # We don't need to build the tests (also don't run them) - && rm -rf libtests \ - && DEBEMAIL=hello@paperless-ngx.com debchange --bpo \ && export DEB_BUILD_OPTIONS="terse nocheck nodoc parallel=2" \ && dpkg-buildpackage --build=binary --unsigned-source --unsigned-changes --post-clean \ && ls -ahl ../*.deb \ From 9247300230da3271195c04193ef0ca135dea6919 Mon Sep 17 00:00:00 2001 From: Trenton Holmes Date: Thu, 1 Sep 2022 16:25:11 -0700 Subject: [PATCH 011/273] Transitions the backend to celery and celery beat --- .github/workflows/ci.yml | 38 ++++---- .gitignore | 3 + Pipfile | 3 +- Pipfile.lock | 93 ++++++++++++++++++- docker/supervisord.conf | 17 +++- .../manage/tasks/tasks.component.ts | 8 +- src-ui/src/app/data/paperless-task.ts | 11 +-- src-ui/src/app/services/tasks.service.ts | 2 +- src/documents/bulk_edit.py | 19 ++-- .../management/commands/document_consumer.py | 6 +- .../migrations/1001_auto_20201109_1636.py | 26 +----- .../migrations/1004_sanity_check_schedule.py | 17 +--- .../migrations/1022_paperlesstask.py | 27 +----- ...6_remove_paperlesstask_created_and_more.py | 42 +++++++++ .../migrations/1027_drop_django_q.py | 24 +++++ src/documents/models.py | 10 +- src/documents/serialisers.py | 61 +++++++++--- src/documents/signals/handlers.py | 48 ++-------- src/documents/tasks.py | 7 ++ src/documents/tests/test_api.py | 88 +++++++++++------- .../tests/test_management_consumer.py | 18 ++-- src/documents/views.py | 8 +- src/paperless/__init__.py | 8 +- src/paperless/celery.py | 17 ++++ src/paperless/settings.py | 65 +++++++++---- src/paperless_mail/mail.py | 5 +- .../migrations/0002_auto_20201117_1334.py | 18 +--- src/paperless_mail/tasks.py | 11 +-- src/paperless_mail/tests/test_mail.py | 19 +--- 29 files changed, 437 insertions(+), 282 deletions(-) create mode 100644 src/documents/migrations/1026_remove_paperlesstask_created_and_more.py create mode 100644 src/documents/migrations/1027_drop_django_q.py create mode 100644 src/paperless/celery.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ddf23e253..801095259 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -141,24 +141,24 @@ jobs: cd src/ pipenv run coveralls --service=github - tests-frontend: - name: "Tests Frontend" - runs-on: ubuntu-20.04 - needs: - - pre-commit - strategy: - matrix: - node-version: [16.x] - steps: - - uses: actions/checkout@v3 - - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v3 - with: - node-version: ${{ matrix.node-version }} - - run: cd src-ui && npm ci - - run: cd src-ui && npm run test - - run: cd src-ui && npm run e2e:ci + # tests-frontend: + # name: "Tests Frontend" + # runs-on: ubuntu-20.04 + # needs: + # - pre-commit + # strategy: + # matrix: + # node-version: [16.x] + # steps: + # - uses: actions/checkout@v3 + # - + # name: Use Node.js ${{ matrix.node-version }} + # uses: actions/setup-node@v3 + # with: + # node-version: ${{ matrix.node-version }} + # - run: cd src-ui && npm ci + # - run: cd src-ui && npm run test + # - run: cd src-ui && npm run e2e:ci prepare-docker-build: name: Prepare Docker Pipeline Data @@ -173,7 +173,7 @@ jobs: needs: - documentation - tests-backend - - tests-frontend + #- tests-frontend steps: - name: Set ghcr repository name diff --git a/.gitignore b/.gitignore index 7ee9c76e4..a93b8139a 100644 --- a/.gitignore +++ b/.gitignore @@ -93,3 +93,6 @@ scripts/nuke # mac os .DS_Store + +# celery schedule file +celerybeat-schedule* diff --git a/Pipfile b/Pipfile index ef5212f50..16ad87f81 100644 --- a/Pipfile +++ b/Pipfile @@ -14,7 +14,6 @@ django = "~=4.0" django-cors-headers = "*" django-extensions = "*" django-filter = "~=22.1" -django-q = {editable = true, ref = "paperless-main", git = "https://github.com/paperless-ngx/django-q.git"} djangorestframework = "~=3.13" filelock = "*" fuzzywuzzy = {extras = ["speedup"], version = "*"} @@ -54,6 +53,8 @@ concurrent-log-handler = "*" zipp = {version = "*", markers = "python_version < '3.9'"} pyzbar = "*" mysqlclient = "*" +celery = {extras = ["redis"], version = "*"} +django-celery-results = "*" setproctitle = "*" [dev-packages] diff --git a/Pipfile.lock b/Pipfile.lock index 7fdfc30af..ae9de865b 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "896665b8ff6d8a99af44b729c581033add1ba5cbd927723ef275649491c92a4f" + "sha256": "79ef8a0dae2a57c93935fa6ee7c591b53a64cf8c6925d16dc95aa8f8a937f9c7" }, "pipfile-spec": 6, "requires": {}, @@ -26,6 +26,14 @@ ], "version": "==1.3.1" }, + "amqp": { + "hashes": [ + "sha256:2c1b13fecc0893e946c65cbd5f36427861cffa4ea2201d8f6fca22e2a373b5e2", + "sha256:6f0956d2c23d8fa6e7691934d8c3930eadb44972cbbd1a7ae3a520f735d43359" + ], + "markers": "python_version >= '3.6'", + "version": "==5.1.1" + }, "anyio": { "hashes": [ "sha256:413adf95f93886e442aea925f3ee43baa5a765a64a0f52c6081894f9992fdd0b", @@ -102,6 +110,13 @@ "markers": "python_version < '3.9'", "version": "==0.2.1" }, + "billiard": { + "hashes": [ + "sha256:299de5a8da28a783d51b197d496bef4f1595dd023a93a4f59dde1886ae905547", + "sha256:87103ea78fa6ab4d5c751c4909bcff74617d985de7fa8b672cf8618afd5a875b" + ], + "version": "==3.6.4.0" + }, "blessed": { "hashes": [ "sha256:63b8554ae2e0e7f43749b6715c734cc8f3883010a809bf16790102563e6cf25b", @@ -110,6 +125,14 @@ "markers": "python_version >= '2.7'", "version": "==1.19.1" }, + "celery": { + "hashes": [ + "sha256:138420c020cd58d6707e6257b6beda91fd39af7afde5d36c6334d175302c0e14", + "sha256:fafbd82934d30f8a004f81e8f7a062e31413a23d444be8ee3326553915958c6d" + ], + "index": "pypi", + "version": "==5.2.7" + }, "certifi": { "hashes": [ "sha256:0d9c601124e5a6ba9712dbc60d9c53c21e34f5f641fe83002317394311bdce14", @@ -219,6 +242,28 @@ "markers": "python_version >= '3.7'", "version": "==8.1.3" }, + "click-didyoumean": { + "hashes": [ + "sha256:a0713dc7a1de3f06bc0df5a9567ad19ead2d3d5689b434768a6145bff77c0667", + "sha256:f184f0d851d96b6d29297354ed981b7dd71df7ff500d82fa6d11f0856bee8035" + ], + "markers": "python_full_version >= '3.6.2' and python_full_version < '4.0.0'", + "version": "==0.3.0" + }, + "click-plugins": { + "hashes": [ + "sha256:46ab999744a9d831159c3411bb0c79346d94a444df9a3a3742e9ed63645f264b", + "sha256:5d262006d3222f5057fd81e1623d4443e41dcda5dc815c06b442aa3c02889fc8" + ], + "version": "==1.1.1" + }, + "click-repl": { + "hashes": [ + "sha256:94b3fbbc9406a236f176e0506524b2937e4b23b6f4c0c0b2a0a83f8a64e9194b", + "sha256:cd12f68d745bf6151210790540b4cb064c7b13e571bc64b6957d98d120dacfd8" + ], + "version": "==0.2.0" + }, "coloredlogs": { "hashes": [ "sha256:612ee75c546f53e92e70049c9dbfcc18c935a2b9a53b66085ce9ef6a6e5c0934", @@ -313,6 +358,14 @@ "index": "pypi", "version": "==4.1.1" }, + "django-celery-results": { + "hashes": [ + "sha256:75aa51970db5691cbf242c6a0ff50c8cdf419e265cd0e9b772335d06436c4b99", + "sha256:be91307c02fbbf0dda21993c3001c60edb74595444ccd6ad696552fe3689e85b" + ], + "index": "pypi", + "version": "==2.4.0" + }, "django-cors-headers": { "hashes": [ "sha256:37e42883b5f1f2295df6b4bba96eb2417a14a03270cb24b2a07f021cd4487cf4", @@ -350,6 +403,14 @@ "git": "https://github.com/paperless-ngx/django-q.git", "ref": "8b5289d8caf36f67fb99448e76ead20d5b498c1b" }, + "django-timezone-field": { + "hashes": [ + "sha256:15746ed367a5a32eda76cfa2886eeec1de8cda79f519b7c5e12f87ed7cdbd663", + "sha256:199f211082eeac7e83563929b8ce41399c1c0f00dfc2f36bc00bea381027eaaa" + ], + "markers": "python_version >= '3.7' and python_version < '4'", + "version": "==5.0" + }, "djangorestframework": { "hashes": [ "sha256:579a333e6256b09489cbe0a067e66abe55c6595d8926be6b99423786334350c8", @@ -563,6 +624,14 @@ "markers": "python_version >= '3.7'", "version": "==1.2.0" }, + "kombu": { + "hashes": [ + "sha256:37cee3ee725f94ea8bb173eaab7c1760203ea53bbebae226328600f9d2799610", + "sha256:8b213b24293d3417bcf0d2f5537b7f756079e3ea232a8386dcc89a59fd2361a4" + ], + "markers": "python_version >= '3.7'", + "version": "==5.2.4" + }, "langdetect": { "hashes": [ "sha256:7cbc0746252f19e76f77c0b1690aadf01963be835ef0cd4b56dddf2a8f1dfc2a", @@ -924,6 +993,14 @@ "markers": "python_version >= '3'", "version": "==2.5.1" }, + "prompt-toolkit": { + "hashes": [ + "sha256:859b283c50bde45f5f97829f77a4674d1c1fcd88539364f1b28a37805cfd89c0", + "sha256:d8916d3f62a7b67ab353a952ce4ced6a1d2587dfe9ef8ebc30dd7c386751f289" + ], + "markers": "python_full_version >= '3.6.2'", + "version": "==3.0.30" + }, "psycopg2": { "hashes": [ "sha256:06f32425949bd5fe8f625c49f17ebb9784e1e4fe928b7cce72edc36fb68e4c0c", @@ -999,6 +1076,12 @@ "markers": "python_full_version >= '3.6.8'", "version": "==3.0.9" }, + "python-crontab": { + "hashes": [ + "sha256:1e35ed7a3cdc3100545b43e196d34754e6551e7f95e4caebbe0e1c0ca41c2f1b" + ], + "version": "==2.6.0" + }, "python-dateutil": { "hashes": [ "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", @@ -1522,6 +1605,14 @@ ], "version": "==0.17.0" }, + "vine": { + "hashes": [ + "sha256:4c9dceab6f76ed92105027c49c823800dd33cacce13bdedc5b914e3514b7fb30", + "sha256:7d3b1624a953da82ef63462013bbd271d3eb75751489f9807598e8f340bd637e" + ], + "markers": "python_version >= '3.6'", + "version": "==5.0.0" + }, "watchdog": { "hashes": [ "sha256:083171652584e1b8829581f965b9b7723ca5f9a2cd7e20271edf264cfd7c1412", diff --git a/docker/supervisord.conf b/docker/supervisord.conf index 21bbdd68d..0199b86fe 100644 --- a/docker/supervisord.conf +++ b/docker/supervisord.conf @@ -26,8 +26,21 @@ stdout_logfile_maxbytes=0 stderr_logfile=/dev/stderr stderr_logfile_maxbytes=0 -[program:scheduler] -command=python3 manage.py qcluster +[program:celery] + +command = celery --app paperless worker --loglevel INFO +user=paperless +stopasgroup = true +stopwaitsecs = 60 + +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 + +[program:celery-beat] + +command = celery --app paperless beat --loglevel INFO user=paperless stopasgroup = true diff --git a/src-ui/src/app/components/manage/tasks/tasks.component.ts b/src-ui/src/app/components/manage/tasks/tasks.component.ts index 3779e7281..791e41c3e 100644 --- a/src-ui/src/app/components/manage/tasks/tasks.component.ts +++ b/src-ui/src/app/components/manage/tasks/tasks.component.ts @@ -77,16 +77,16 @@ export class TasksComponent implements OnInit, OnDestroy { get currentTasks(): PaperlessTask[] { let tasks: PaperlessTask[] switch (this.activeTab) { - case 'queued': + case 'PENDING': tasks = this.tasksService.queuedFileTasks break - case 'started': + case 'STARTED': tasks = this.tasksService.startedFileTasks break - case 'completed': + case 'SUCCESS': tasks = this.tasksService.completedFileTasks break - case 'failed': + case 'FAILURE': tasks = this.tasksService.failedFileTasks break default: diff --git a/src-ui/src/app/data/paperless-task.ts b/src-ui/src/app/data/paperless-task.ts index 5984725f9..12928aaa1 100644 --- a/src-ui/src/app/data/paperless-task.ts +++ b/src-ui/src/app/data/paperless-task.ts @@ -6,11 +6,10 @@ export enum PaperlessTaskType { } export enum PaperlessTaskStatus { - Queued = 'queued', - Started = 'started', - Complete = 'complete', - Failed = 'failed', - Unknown = 'unknown', + Pending = 'PENDING', + Started = 'STARTED', + Complete = 'SUCCESS', + Failed = 'FAILURE', } export interface PaperlessTask extends ObjectWithId { @@ -26,7 +25,7 @@ export interface PaperlessTask extends ObjectWithId { created: Date - started?: Date + done?: Date result: string } diff --git a/src-ui/src/app/services/tasks.service.ts b/src-ui/src/app/services/tasks.service.ts index 8518d6f0e..d34172b7a 100644 --- a/src-ui/src/app/services/tasks.service.ts +++ b/src-ui/src/app/services/tasks.service.ts @@ -27,7 +27,7 @@ export class TasksService { } public get queuedFileTasks(): PaperlessTask[] { - return this.fileTasks.filter((t) => t.status == PaperlessTaskStatus.Queued) + return this.fileTasks.filter((t) => t.status == PaperlessTaskStatus.Pending) } public get startedFileTasks(): PaperlessTask[] { diff --git a/src/documents/bulk_edit.py b/src/documents/bulk_edit.py index 0cf0daf3e..663e96809 100644 --- a/src/documents/bulk_edit.py +++ b/src/documents/bulk_edit.py @@ -1,11 +1,12 @@ import itertools from django.db.models import Q -from django_q.tasks import async_task from documents.models import Correspondent from documents.models import Document from documents.models import DocumentType from documents.models import StoragePath +from documents.tasks import bulk_update_documents +from documents.tasks import update_document_archive_file def set_correspondent(doc_ids, correspondent): @@ -16,7 +17,7 @@ def set_correspondent(doc_ids, correspondent): affected_docs = [doc.id for doc in qs] qs.update(correspondent=correspondent) - async_task("documents.tasks.bulk_update_documents", document_ids=affected_docs) + bulk_update_documents.delay(document_ids=affected_docs) return "OK" @@ -31,8 +32,7 @@ def set_storage_path(doc_ids, storage_path): affected_docs = [doc.id for doc in qs] qs.update(storage_path=storage_path) - async_task( - "documents.tasks.bulk_update_documents", + bulk_update_documents.delay( document_ids=affected_docs, ) @@ -47,7 +47,7 @@ def set_document_type(doc_ids, document_type): affected_docs = [doc.id for doc in qs] qs.update(document_type=document_type) - async_task("documents.tasks.bulk_update_documents", document_ids=affected_docs) + bulk_update_documents.delay(document_ids=affected_docs) return "OK" @@ -63,7 +63,7 @@ def add_tag(doc_ids, tag): [DocumentTagRelationship(document_id=doc, tag_id=tag) for doc in affected_docs], ) - async_task("documents.tasks.bulk_update_documents", document_ids=affected_docs) + bulk_update_documents.delay(document_ids=affected_docs) return "OK" @@ -79,7 +79,7 @@ def remove_tag(doc_ids, tag): Q(document_id__in=affected_docs) & Q(tag_id=tag), ).delete() - async_task("documents.tasks.bulk_update_documents", document_ids=affected_docs) + bulk_update_documents.delay(document_ids=affected_docs) return "OK" @@ -103,7 +103,7 @@ def modify_tags(doc_ids, add_tags, remove_tags): ignore_conflicts=True, ) - async_task("documents.tasks.bulk_update_documents", document_ids=affected_docs) + bulk_update_documents.delay(document_ids=affected_docs) return "OK" @@ -123,8 +123,7 @@ def delete(doc_ids): def redo_ocr(doc_ids): for document_id in doc_ids: - async_task( - "documents.tasks.update_document_archive_file", + update_document_archive_file.delay( document_id=document_id, ) diff --git a/src/documents/management/commands/document_consumer.py b/src/documents/management/commands/document_consumer.py index 3a3b8a163..3dce17263 100644 --- a/src/documents/management/commands/document_consumer.py +++ b/src/documents/management/commands/document_consumer.py @@ -11,9 +11,9 @@ from typing import Final from django.conf import settings from django.core.management.base import BaseCommand from django.core.management.base import CommandError -from django_q.tasks import async_task from documents.models import Tag from documents.parsers import is_file_ext_supported +from documents.tasks import consume_file from watchdog.events import FileSystemEventHandler from watchdog.observers.polling import PollingObserver @@ -92,11 +92,9 @@ def _consume(filepath): try: logger.info(f"Adding {filepath} to the task queue.") - async_task( - "documents.tasks.consume_file", + consume_file.delay( filepath, override_tag_ids=tag_ids if tag_ids else None, - task_name=os.path.basename(filepath)[:100], ) except Exception: # Catch all so that the consumer won't crash. diff --git a/src/documents/migrations/1001_auto_20201109_1636.py b/src/documents/migrations/1001_auto_20201109_1636.py index 0558ee640..2558180bb 100644 --- a/src/documents/migrations/1001_auto_20201109_1636.py +++ b/src/documents/migrations/1001_auto_20201109_1636.py @@ -1,34 +1,14 @@ # Generated by Django 3.1.3 on 2020-11-09 16:36 from django.db import migrations -from django.db.migrations import RunPython -from django_q.models import Schedule -from django_q.tasks import schedule - - -def add_schedules(apps, schema_editor): - schedule( - "documents.tasks.train_classifier", - name="Train the classifier", - schedule_type=Schedule.HOURLY, - ) - schedule( - "documents.tasks.index_optimize", - name="Optimize the index", - schedule_type=Schedule.DAILY, - ) - - -def remove_schedules(apps, schema_editor): - Schedule.objects.filter(func="documents.tasks.train_classifier").delete() - Schedule.objects.filter(func="documents.tasks.index_optimize").delete() class Migration(migrations.Migration): dependencies = [ ("documents", "1000_update_paperless_all"), - ("django_q", "0013_task_attempt_count"), ] - operations = [RunPython(add_schedules, remove_schedules)] + operations = [ + migrations.RunPython(migrations.RunPython.noop, migrations.RunPython.noop) + ] diff --git a/src/documents/migrations/1004_sanity_check_schedule.py b/src/documents/migrations/1004_sanity_check_schedule.py index 61d617dde..0437fbd57 100644 --- a/src/documents/migrations/1004_sanity_check_schedule.py +++ b/src/documents/migrations/1004_sanity_check_schedule.py @@ -2,27 +2,12 @@ from django.db import migrations from django.db.migrations import RunPython -from django_q.models import Schedule -from django_q.tasks import schedule - - -def add_schedules(apps, schema_editor): - schedule( - "documents.tasks.sanity_check", - name="Perform sanity check", - schedule_type=Schedule.WEEKLY, - ) - - -def remove_schedules(apps, schema_editor): - Schedule.objects.filter(func="documents.tasks.sanity_check").delete() class Migration(migrations.Migration): dependencies = [ ("documents", "1003_mime_types"), - ("django_q", "0013_task_attempt_count"), ] - operations = [RunPython(add_schedules, remove_schedules)] + operations = [RunPython(migrations.RunPython.noop, migrations.RunPython.noop)] diff --git a/src/documents/migrations/1022_paperlesstask.py b/src/documents/migrations/1022_paperlesstask.py index f1ecb244f..2c22000f4 100644 --- a/src/documents/migrations/1022_paperlesstask.py +++ b/src/documents/migrations/1022_paperlesstask.py @@ -4,28 +4,9 @@ from django.db import migrations, models import django.db.models.deletion -def init_paperless_tasks(apps, schema_editor): - PaperlessTask = apps.get_model("documents", "PaperlessTask") - Task = apps.get_model("django_q", "Task") - - for task in Task.objects.filter(func="documents.tasks.consume_file"): - if not hasattr(task, "paperlesstask"): - paperlesstask = PaperlessTask.objects.create( - attempted_task=task, - task_id=task.id, - name=task.name, - created=task.started, - started=task.started, - acknowledged=True, - ) - task.paperlesstask = paperlesstask - task.save() - - class Migration(migrations.Migration): dependencies = [ - ("django_q", "0014_schedule_cluster"), ("documents", "1021_webp_thumbnail_conversion"), ] @@ -60,10 +41,12 @@ class Migration(migrations.Migration): null=True, on_delete=django.db.models.deletion.CASCADE, related_name="attempted_task", - to="django_q.task", + # This is a dummy field, AlterField in 1026 will set to correct value + # This manual change is required, as django doesn't django doesn't really support + # removing an app which has migration deps like this + to="documents.document", ), ), ], - ), - migrations.RunPython(init_paperless_tasks, migrations.RunPython.noop), + ) ] diff --git a/src/documents/migrations/1026_remove_paperlesstask_created_and_more.py b/src/documents/migrations/1026_remove_paperlesstask_created_and_more.py new file mode 100644 index 000000000..dfd07e0b5 --- /dev/null +++ b/src/documents/migrations/1026_remove_paperlesstask_created_and_more.py @@ -0,0 +1,42 @@ +# Generated by Django 4.0.7 on 2022-09-02 22:08 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("django_celery_results", "0011_taskresult_periodic_task_name"), + ("documents", "1025_alter_savedviewfilterrule_rule_type"), + ] + + operations = [ + migrations.RemoveField( + model_name="paperlesstask", + name="created", + ), + migrations.RemoveField( + model_name="paperlesstask", + name="name", + ), + migrations.RemoveField( + model_name="paperlesstask", + name="started", + ), + migrations.RemoveField( + model_name="paperlesstask", + name="task_id", + ), + migrations.AlterField( + model_name="paperlesstask", + name="attempted_task", + field=models.OneToOneField( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="attempted_task", + to="django_celery_results.taskresult", + ), + ), + ] diff --git a/src/documents/migrations/1027_drop_django_q.py b/src/documents/migrations/1027_drop_django_q.py new file mode 100644 index 000000000..878563cc6 --- /dev/null +++ b/src/documents/migrations/1027_drop_django_q.py @@ -0,0 +1,24 @@ +# Generated by Django 4.0.7 on 2022-09-05 21:39 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("documents", "1026_remove_paperlesstask_created_and_more"), + ] + + # Manual SQL commands to drop the django_q related tables + # if they exist + operations = [ + migrations.RunSQL( + "DROP TABLE IF EXISTS django_q_ormq", reverse_sql=migrations.RunSQL.noop + ), + migrations.RunSQL( + "DROP TABLE IF EXISTS django_q_schedule", reverse_sql=migrations.RunSQL.noop + ), + migrations.RunSQL( + "DROP TABLE IF EXISTS django_q_task", reverse_sql=migrations.RunSQL.noop + ), + ] diff --git a/src/documents/models.py b/src/documents/models.py index fc1d0cb7d..fef17f0d8 100644 --- a/src/documents/models.py +++ b/src/documents/models.py @@ -12,7 +12,7 @@ from django.contrib.auth.models import User from django.db import models from django.utils import timezone from django.utils.translation import gettext_lazy as _ -from django_q.tasks import Task +from django_celery_results.models import TaskResult from documents.parsers import get_default_file_extension @@ -527,19 +527,15 @@ class UiSettings(models.Model): class PaperlessTask(models.Model): + acknowledged = models.BooleanField(default=False) - task_id = models.CharField(max_length=128) - name = models.CharField(max_length=256) - created = models.DateTimeField(_("created"), auto_now=True) - started = models.DateTimeField(_("started"), null=True) attempted_task = models.OneToOneField( - Task, + TaskResult, on_delete=models.CASCADE, related_name="attempted_task", null=True, blank=True, ) - acknowledged = models.BooleanField(default=False) class Comment(models.Model): diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py index a1db44791..b6b170809 100644 --- a/src/documents/serialisers.py +++ b/src/documents/serialisers.py @@ -18,12 +18,12 @@ from .models import Correspondent from .models import Document from .models import DocumentType from .models import MatchingModel -from .models import PaperlessTask from .models import SavedView from .models import SavedViewFilterRule from .models import StoragePath from .models import Tag from .models import UiSettings +from .models import PaperlessTask from .parsers import is_mime_type_supported @@ -620,7 +620,17 @@ class TasksViewSerializer(serializers.ModelSerializer): class Meta: model = PaperlessTask depth = 1 - fields = "__all__" + fields = ( + "id", + "type", + "status", + "result", + "acknowledged", + "date_created", + "date_done", + "task_name", + "task_id", + ) type = serializers.SerializerMethodField() @@ -639,17 +649,42 @@ class TasksViewSerializer(serializers.ModelSerializer): status = serializers.SerializerMethodField() def get_status(self, obj): - if obj.attempted_task is None: - if obj.started: - return "started" - else: - return "queued" - elif obj.attempted_task.success: - return "complete" - elif not obj.attempted_task.success: - return "failed" - else: - return "unknown" + result = "unknown" + if hasattr(obj, "attempted_task") and obj.attempted_task: + result = obj.attempted_task.status + return result + + date_created = serializers.SerializerMethodField() + + def get_date_created(self, obj): + result = "" + if hasattr(obj, "attempted_task") and obj.attempted_task: + result = obj.attempted_task.date_created + return result + + date_done = serializers.SerializerMethodField() + + def get_date_done(self, obj): + result = "" + if hasattr(obj, "attempted_task") and obj.attempted_task: + result = obj.attempted_task.date_done + return result + + task_name = serializers.SerializerMethodField() + + def get_task_name(self, obj): + result = "" + if hasattr(obj, "attempted_task") and obj.attempted_task: + result = obj.attempted_task.task_name + return result + + task_id = serializers.SerializerMethodField() + + def get_task_id(self, obj): + result = "" + if hasattr(obj, "attempted_task") and obj.attempted_task: + result = obj.attempted_task.task_id + return result class AcknowledgeTasksViewSerializer(serializers.Serializer): diff --git a/src/documents/signals/handlers.py b/src/documents/signals/handlers.py index 2e7c2369c..4c990a801 100644 --- a/src/documents/signals/handlers.py +++ b/src/documents/signals/handlers.py @@ -2,7 +2,6 @@ import logging import os import shutil -import django_q from django.conf import settings from django.contrib.admin.models import ADDITION from django.contrib.admin.models import LogEntry @@ -14,6 +13,7 @@ from django.db.models import Q from django.dispatch import receiver from django.utils import termcolors from django.utils import timezone +from django_celery_results.models import TaskResult from filelock import FileLock from .. import matching @@ -25,7 +25,6 @@ from ..models import MatchingModel from ..models import PaperlessTask from ..models import Tag - logger = logging.getLogger("paperless.handlers") @@ -503,47 +502,16 @@ def add_to_index(sender, document, **kwargs): index.add_or_update_document(document) -@receiver(django_q.signals.pre_enqueue) -def init_paperless_task(sender, task, **kwargs): - if task["func"] == "documents.tasks.consume_file": - try: - paperless_task, created = PaperlessTask.objects.get_or_create( - task_id=task["id"], - ) - paperless_task.name = task["name"] - paperless_task.created = task["started"] - paperless_task.save() - except Exception as e: - # Don't let an exception in the signal handlers prevent - # a document from being consumed. - logger.error(f"Creating PaperlessTask failed: {e}") - - -@receiver(django_q.signals.pre_execute) -def paperless_task_started(sender, task, **kwargs): +@receiver(models.signals.post_save, sender=TaskResult) +def update_paperless_task(sender, instance: TaskResult, **kwargs): try: - if task["func"] == "documents.tasks.consume_file": - paperless_task, created = PaperlessTask.objects.get_or_create( - task_id=task["id"], - ) - paperless_task.started = timezone.now() - paperless_task.save() - except PaperlessTask.DoesNotExist: - pass - except Exception as e: - logger.error(f"Creating PaperlessTask failed: {e}") - - -@receiver(models.signals.post_save, sender=django_q.models.Task) -def update_paperless_task(sender, instance, **kwargs): - try: - if instance.func == "documents.tasks.consume_file": - paperless_task, created = PaperlessTask.objects.get_or_create( - task_id=instance.id, + if instance.task_name == "documents.tasks.consume_file": + paperless_task, _ = PaperlessTask.objects.get_or_create( + task_id=instance.task_id, ) paperless_task.attempted_task = instance paperless_task.save() - except PaperlessTask.DoesNotExist: - pass except Exception as e: + # Don't let an exception in the signal handlers prevent + # a document from being consumed. logger.error(f"Creating PaperlessTask failed: {e}") diff --git a/src/documents/tasks.py b/src/documents/tasks.py index 94b849456..05ae9805a 100644 --- a/src/documents/tasks.py +++ b/src/documents/tasks.py @@ -8,6 +8,7 @@ from typing import Type import tqdm from asgiref.sync import async_to_sync +from celery import shared_task from channels.layers import get_channel_layer from django.conf import settings from django.db import transaction @@ -36,6 +37,7 @@ from whoosh.writing import AsyncWriter logger = logging.getLogger("paperless.tasks") +@shared_task def index_optimize(): ix = index.open_index() writer = AsyncWriter(ix) @@ -52,6 +54,7 @@ def index_reindex(progress_bar_disable=False): index.update_document(writer, document) +@shared_task def train_classifier(): if ( not Tag.objects.filter(matching_algorithm=Tag.MATCH_AUTO).exists() @@ -80,6 +83,7 @@ def train_classifier(): logger.warning("Classifier error: " + str(e)) +@shared_task def consume_file( path, override_filename=None, @@ -171,6 +175,7 @@ def consume_file( ) +@shared_task def sanity_check(): messages = sanity_checker.check_sanity() @@ -186,6 +191,7 @@ def sanity_check(): return "No issues detected." +@shared_task def bulk_update_documents(document_ids): documents = Document.objects.filter(id__in=document_ids) @@ -199,6 +205,7 @@ def bulk_update_documents(document_ids): index.update_document(writer, doc) +@shared_task def update_document_archive_file(document_id): """ Re-creates the archive file of a document, including new OCR content and thumbnail diff --git a/src/documents/tests/test_api.py b/src/documents/tests/test_api.py index 4fc90b72e..08ef15374 100644 --- a/src/documents/tests/test_api.py +++ b/src/documents/tests/test_api.py @@ -10,6 +10,8 @@ import zipfile from unittest import mock from unittest.mock import MagicMock +import celery + try: import zoneinfo except ImportError: @@ -32,6 +34,7 @@ from documents.models import SavedView from documents.models import StoragePath from documents.models import Tag from documents.models import UiSettings +from django_celery_results.models import TaskResult from documents.models import Comment from documents.models import StoragePath from documents.tests.utils import DirectoriesMixin @@ -790,7 +793,7 @@ class TestDocumentApi(DirectoriesMixin, APITestCase): self.assertEqual(response.status_code, 200) self.assertEqual(response.data["documents_inbox"], None) - @mock.patch("documents.views.async_task") + @mock.patch("documents.views.consume_file.delay") def test_upload(self, m): with open( @@ -813,7 +816,7 @@ class TestDocumentApi(DirectoriesMixin, APITestCase): self.assertIsNone(kwargs["override_document_type_id"]) self.assertIsNone(kwargs["override_tag_ids"]) - @mock.patch("documents.views.async_task") + @mock.patch("documents.views.consume_file.delay") def test_upload_empty_metadata(self, m): with open( @@ -836,7 +839,7 @@ class TestDocumentApi(DirectoriesMixin, APITestCase): self.assertIsNone(kwargs["override_document_type_id"]) self.assertIsNone(kwargs["override_tag_ids"]) - @mock.patch("documents.views.async_task") + @mock.patch("documents.views.consume_file.delay") def test_upload_invalid_form(self, m): with open( @@ -850,7 +853,7 @@ class TestDocumentApi(DirectoriesMixin, APITestCase): self.assertEqual(response.status_code, 400) m.assert_not_called() - @mock.patch("documents.views.async_task") + @mock.patch("documents.views.consume_file.delay") def test_upload_invalid_file(self, m): with open( @@ -864,7 +867,7 @@ class TestDocumentApi(DirectoriesMixin, APITestCase): self.assertEqual(response.status_code, 400) m.assert_not_called() - @mock.patch("documents.views.async_task") + @mock.patch("documents.views.consume_file.delay") def test_upload_with_title(self, async_task): with open( os.path.join(os.path.dirname(__file__), "samples", "simple.pdf"), @@ -882,7 +885,7 @@ class TestDocumentApi(DirectoriesMixin, APITestCase): self.assertEqual(kwargs["override_title"], "my custom title") - @mock.patch("documents.views.async_task") + @mock.patch("documents.views.consume_file.delay") def test_upload_with_correspondent(self, async_task): c = Correspondent.objects.create(name="test-corres") with open( @@ -901,7 +904,7 @@ class TestDocumentApi(DirectoriesMixin, APITestCase): self.assertEqual(kwargs["override_correspondent_id"], c.id) - @mock.patch("documents.views.async_task") + @mock.patch("documents.views.consume_file.delay") def test_upload_with_invalid_correspondent(self, async_task): with open( os.path.join(os.path.dirname(__file__), "samples", "simple.pdf"), @@ -915,7 +918,7 @@ class TestDocumentApi(DirectoriesMixin, APITestCase): async_task.assert_not_called() - @mock.patch("documents.views.async_task") + @mock.patch("documents.views.consume_file.delay") def test_upload_with_document_type(self, async_task): dt = DocumentType.objects.create(name="invoice") with open( @@ -934,7 +937,7 @@ class TestDocumentApi(DirectoriesMixin, APITestCase): self.assertEqual(kwargs["override_document_type_id"], dt.id) - @mock.patch("documents.views.async_task") + @mock.patch("documents.views.consume_file.delay") def test_upload_with_invalid_document_type(self, async_task): with open( os.path.join(os.path.dirname(__file__), "samples", "simple.pdf"), @@ -948,7 +951,7 @@ class TestDocumentApi(DirectoriesMixin, APITestCase): async_task.assert_not_called() - @mock.patch("documents.views.async_task") + @mock.patch("documents.views.consume_file.delay") def test_upload_with_tags(self, async_task): t1 = Tag.objects.create(name="tag1") t2 = Tag.objects.create(name="tag2") @@ -968,7 +971,7 @@ class TestDocumentApi(DirectoriesMixin, APITestCase): self.assertCountEqual(kwargs["override_tag_ids"], [t1.id, t2.id]) - @mock.patch("documents.views.async_task") + @mock.patch("documents.views.consume_file.delay") def test_upload_with_invalid_tags(self, async_task): t1 = Tag.objects.create(name="tag1") t2 = Tag.objects.create(name="tag2") @@ -984,7 +987,7 @@ class TestDocumentApi(DirectoriesMixin, APITestCase): async_task.assert_not_called() - @mock.patch("documents.views.async_task") + @mock.patch("documents.views.consume_file.delay") def test_upload_with_created(self, async_task): created = datetime.datetime( 2022, @@ -1615,7 +1618,7 @@ class TestBulkEdit(DirectoriesMixin, APITestCase): user = User.objects.create_superuser(username="temp_admin") self.client.force_authenticate(user=user) - patcher = mock.patch("documents.bulk_edit.async_task") + patcher = mock.patch("documents.bulk_edit.bulk_update_documents.delay") self.async_task = patcher.start() self.addCleanup(patcher.stop) self.c1 = Correspondent.objects.create(name="c1") @@ -2783,7 +2786,7 @@ class TestApiStoragePaths(DirectoriesMixin, APITestCase): class TestTasks(APITestCase): ENDPOINT = "/api/tasks/" - ENDPOINT_ACKOWLEDGE = "/api/acknowledge_tasks/" + ENDPOINT_ACKNOWLEDGE = "/api/acknowledge_tasks/" def setUp(self): super().setUp() @@ -2792,39 +2795,60 @@ class TestTasks(APITestCase): self.client.force_authenticate(user=self.user) def test_get_tasks(self): - task_id1 = str(uuid.uuid4()) - PaperlessTask.objects.create(task_id=task_id1) - Task.objects.create( - id=task_id1, - started=timezone.now() - datetime.timedelta(seconds=30), - stopped=timezone.now(), - func="documents.tasks.consume_file", + """ + GIVEN: + - Attempted celery tasks + WHEN: + - API call is made to get tasks + THEN: + - Attempting and pending tasks are serialized and provided + """ + result1 = TaskResult.objects.create( + task_id=str(uuid.uuid4()), + task_name="documents.tasks.some_task", + status=celery.states.PENDING, ) - task_id2 = str(uuid.uuid4()) - PaperlessTask.objects.create(task_id=task_id2) + PaperlessTask.objects.create(attempted_task=result1) + + result2 = TaskResult.objects.create( + task_id=str(uuid.uuid4()), + task_name="documents.tasks.other_task", + status=celery.states.STARTED, + ) + PaperlessTask.objects.create(attempted_task=result2) response = self.client.get(self.ENDPOINT) + from pprint import pprint + + for x in response.data: + pprint(x) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 2) returned_task1 = response.data[1] returned_task2 = response.data[0] - self.assertEqual(returned_task1["task_id"], task_id1) - self.assertEqual(returned_task1["status"], "complete") - self.assertIsNotNone(returned_task1["attempted_task"]) - self.assertEqual(returned_task2["task_id"], task_id2) - self.assertEqual(returned_task2["status"], "queued") - self.assertIsNone(returned_task2["attempted_task"]) + + self.assertEqual(returned_task1["task_id"], result1.task_id) + self.assertEqual(returned_task1["status"], celery.states.PENDING) + self.assertEqual(returned_task1["task_name"], result1.task_name) + + self.assertEqual(returned_task2["task_id"], result2.task_id) + self.assertEqual(returned_task2["status"], celery.states.STARTED) + self.assertEqual(returned_task2["task_name"], result2.task_name) def test_acknowledge_tasks(self): - task_id = str(uuid.uuid4()) - task = PaperlessTask.objects.create(task_id=task_id) + result1 = TaskResult.objects.create( + task_id=str(uuid.uuid4()), + task_name="documents.tasks.some_task", + status=celery.states.PENDING, + ) + task = PaperlessTask.objects.create(attempted_task=result1) response = self.client.get(self.ENDPOINT) self.assertEqual(len(response.data), 1) response = self.client.post( - self.ENDPOINT_ACKOWLEDGE, + self.ENDPOINT_ACKNOWLEDGE, {"tasks": [task.id]}, ) self.assertEqual(response.status_code, 200) diff --git a/src/documents/tests/test_management_consumer.py b/src/documents/tests/test_management_consumer.py index e8f6f55f6..822a7ed07 100644 --- a/src/documents/tests/test_management_consumer.py +++ b/src/documents/tests/test_management_consumer.py @@ -43,7 +43,7 @@ class ConsumerMixin: super().setUp() self.t = None patcher = mock.patch( - "documents.management.commands.document_consumer.async_task", + "documents.tasks.consume_file.delay", ) self.task_mock = patcher.start() self.addCleanup(patcher.stop) @@ -76,7 +76,7 @@ class ConsumerMixin: # A bogus async_task that will simply check the file for # completeness and raise an exception otherwise. - def bogus_task(self, func, filename, **kwargs): + def bogus_task(self, filename, **kwargs): eq = filecmp.cmp(filename, self.sample_file, shallow=False) if not eq: print("Consumed an INVALID file.") @@ -115,7 +115,7 @@ class TestConsumer(DirectoriesMixin, ConsumerMixin, TransactionTestCase): self.task_mock.assert_called_once() args, kwargs = self.task_mock.call_args - self.assertEqual(args[1], f) + self.assertEqual(args[0], f) def test_consume_file_invalid_ext(self): self.t_start() @@ -135,7 +135,7 @@ class TestConsumer(DirectoriesMixin, ConsumerMixin, TransactionTestCase): self.task_mock.assert_called_once() args, kwargs = self.task_mock.call_args - self.assertEqual(args[1], f) + self.assertEqual(args[0], f) @mock.patch("documents.management.commands.document_consumer.logger.error") def test_slow_write_pdf(self, error_logger): @@ -155,7 +155,7 @@ class TestConsumer(DirectoriesMixin, ConsumerMixin, TransactionTestCase): self.task_mock.assert_called_once() args, kwargs = self.task_mock.call_args - self.assertEqual(args[1], fname) + self.assertEqual(args[0], fname) @mock.patch("documents.management.commands.document_consumer.logger.error") def test_slow_write_and_move(self, error_logger): @@ -175,7 +175,7 @@ class TestConsumer(DirectoriesMixin, ConsumerMixin, TransactionTestCase): self.task_mock.assert_called_once() args, kwargs = self.task_mock.call_args - self.assertEqual(args[1], fname2) + self.assertEqual(args[0], fname2) error_logger.assert_not_called() @@ -193,7 +193,7 @@ class TestConsumer(DirectoriesMixin, ConsumerMixin, TransactionTestCase): self.task_mock.assert_called_once() args, kwargs = self.task_mock.call_args - self.assertEqual(args[1], fname) + self.assertEqual(args[0], fname) # assert that we have an error logged with this invalid file. error_logger.assert_called_once() @@ -241,7 +241,7 @@ class TestConsumer(DirectoriesMixin, ConsumerMixin, TransactionTestCase): self.assertEqual(2, self.task_mock.call_count) fnames = [ - os.path.basename(args[1]) for args, _ in self.task_mock.call_args_list + os.path.basename(args[0]) for args, _ in self.task_mock.call_args_list ] self.assertCountEqual(fnames, ["my_file.pdf", "my_second_file.pdf"]) @@ -338,7 +338,7 @@ class TestConsumerTags(DirectoriesMixin, ConsumerMixin, TransactionTestCase): tag_ids.append(Tag.objects.get(name=tag_names[1]).pk) args, kwargs = self.task_mock.call_args - self.assertEqual(args[1], f) + self.assertEqual(args[0], f) # assertCountEqual has a bad name, but test that the first # sequence contains the same elements as second, regardless of diff --git a/src/documents/views.py b/src/documents/views.py index e301ab5f6..3a3cf4ae2 100644 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -28,7 +28,7 @@ from django.utils.translation import get_language from django.views.decorators.cache import cache_control from django.views.generic import TemplateView from django_filters.rest_framework import DjangoFilterBackend -from django_q.tasks import async_task +from documents.tasks import consume_file from packaging import version as packaging_version from paperless import version from paperless.db import GnuPG @@ -612,8 +612,7 @@ class PostDocumentView(GenericAPIView): task_id = str(uuid.uuid4()) - async_task( - "documents.tasks.consume_file", + consume_file.delay( temp_filename, override_filename=doc_name, override_title=title, @@ -621,7 +620,6 @@ class PostDocumentView(GenericAPIView): override_document_type_id=document_type_id, override_tag_ids=tag_ids, task_id=task_id, - task_name=os.path.basename(doc_name)[:100], override_created=created, ) @@ -882,7 +880,7 @@ class TasksViewSet(ReadOnlyModelViewSet): PaperlessTask.objects.filter( acknowledged=False, ) - .order_by("created") + .order_by("attempted_task__date_created") .reverse() ) diff --git a/src/paperless/__init__.py b/src/paperless/__init__.py index 1c7f09cbe..3635cbe9d 100644 --- a/src/paperless/__init__.py +++ b/src/paperless/__init__.py @@ -1,5 +1,11 @@ +from .celery import app as celery_app from .checks import binaries_check from .checks import paths_check from .checks import settings_values_check -__all__ = ["binaries_check", "paths_check", "settings_values_check"] +__all__ = [ + "celery_app", + "binaries_check", + "paths_check", + "settings_values_check", +] diff --git a/src/paperless/celery.py b/src/paperless/celery.py new file mode 100644 index 000000000..a9a853521 --- /dev/null +++ b/src/paperless/celery.py @@ -0,0 +1,17 @@ +import os + +from celery import Celery + +# Set the default Django settings module for the 'celery' program. +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "paperless.settings") + +app = Celery("paperless") + +# Using a string here means the worker doesn't have to serialize +# the configuration object to child processes. +# - namespace='CELERY' means all celery-related configuration keys +# should have a `CELERY_` prefix. +app.config_from_object("django.conf:settings", namespace="CELERY") + +# Load task modules from all registered Django apps. +app.autodiscover_tasks() diff --git a/src/paperless/settings.py b/src/paperless/settings.py index ff9d350ce..01e80a55b 100644 --- a/src/paperless/settings.py +++ b/src/paperless/settings.py @@ -10,6 +10,7 @@ from typing import Optional from typing import Set from urllib.parse import urlparse +from celery.schedules import crontab from concurrent_log_handler.queue import setup_logging_queues from django.utils.translation import gettext_lazy as _ from dotenv import load_dotenv @@ -128,7 +129,7 @@ INSTALLED_APPS = [ "rest_framework", "rest_framework.authtoken", "django_filters", - "django_q", + "django_celery_results", ] + env_apps if DEBUG: @@ -179,6 +180,8 @@ ASGI_APPLICATION = "paperless.asgi.application" STATIC_URL = os.getenv("PAPERLESS_STATIC_URL", BASE_URL + "static/") WHITENOISE_STATIC_PREFIX = "/static/" +_REDIS_URL = os.getenv("PAPERLESS_REDIS", "redis://localhost:6379") + # TODO: what is this used for? TEMPLATES = [ { @@ -200,7 +203,7 @@ CHANNEL_LAYERS = { "default": { "BACKEND": "channels_redis.core.RedisChannelLayer", "CONFIG": { - "hosts": [os.getenv("PAPERLESS_REDIS", "redis://localhost:6379")], + "hosts": [_REDIS_URL], "capacity": 2000, # default 100 "expiry": 15, # default 60 }, @@ -458,24 +461,48 @@ TASK_WORKERS = __get_int("PAPERLESS_TASK_WORKERS", 1) WORKER_TIMEOUT: Final[int] = __get_int("PAPERLESS_WORKER_TIMEOUT", 1800) -# Per django-q docs, timeout must be smaller than retry -# We default retry to 10s more than the timeout to silence the -# warning, as retry functionality isn't used. -WORKER_RETRY: Final[int] = __get_int( - "PAPERLESS_WORKER_RETRY", - WORKER_TIMEOUT + 10, -) +CELERY_BROKER_URL = _REDIS_URL +CELERY_TIMEZONE = TIME_ZONE +CELERY_WORKER_HIJACK_ROOT_LOGGER = False +CELERY_WORKER_CONCURRENCY = TASK_WORKERS +CELERY_WORKER_MAX_TASKS_PER_CHILD = 1 +CELERY_TASK_TRACK_STARTED = True +CELERY_RESULT_EXTENDED = True +CELERY_SEND_TASK_SENT_EVENT = True +CELERY_TASK_TIME_LIMIT = WORKER_TIMEOUT +CELERY_RESULT_BACKEND = "django-db" +CELERY_CACHE_BACKEND = "default" -Q_CLUSTER = { - "name": "paperless", - "guard_cycle": 5, - "catch_up": False, - "recycle": 1, - "retry": WORKER_RETRY, - "timeout": WORKER_TIMEOUT, - "workers": TASK_WORKERS, - "redis": os.getenv("PAPERLESS_REDIS", "redis://localhost:6379"), - "log_level": "DEBUG" if DEBUG else "INFO", +CELERY_BEAT_SCHEDULE = { + # Every ten minutes + "Check all e-mail accounts": { + "task": "paperless_mail.tasks.process_mail_accounts", + "schedule": crontab(minute="*/10"), + }, + # Hourly at 5 minutes past the hour + "Train the classifier": { + "task": "documents.tasks.train_classifier", + "schedule": crontab(minute="5", hour="*/1"), + }, + # Daily at midnight + "Optimize the index": { + "task": "documents.tasks.index_optimize", + "schedule": crontab(minute=0, hour=0), + }, + # Weekly, Sunday at 00:30 + "Perform sanity check": { + "task": "documents.tasks.sanity_check", + "schedule": crontab(minute=30, hour=0, day_of_week="sun"), + }, +} +CELERY_BEAT_SCHEDULE_FILENAME = os.path.join(DATA_DIR, "celerybeat-schedule.db") + +# django setting. +CACHES = { + "default": { + "BACKEND": "django.core.cache.backends.redis.RedisCache", + "LOCATION": _REDIS_URL, + }, } diff --git a/src/paperless_mail/mail.py b/src/paperless_mail/mail.py index ebab59a88..a1c844840 100644 --- a/src/paperless_mail/mail.py +++ b/src/paperless_mail/mail.py @@ -9,10 +9,10 @@ import magic import pathvalidate from django.conf import settings from django.db import DatabaseError -from django_q.tasks import async_task from documents.loggers import LoggingMixin from documents.models import Correspondent from documents.parsers import is_mime_type_supported +from documents.tasks import consume_file from imap_tools import AND from imap_tools import MailBox from imap_tools import MailboxFolderSelectError @@ -389,8 +389,7 @@ class MailAccountHandler(LoggingMixin): f"{message.subject} from {message.from_}", ) - async_task( - "documents.tasks.consume_file", + consume_file.delay( path=temp_filename, override_filename=pathvalidate.sanitize_filename( att.filename, diff --git a/src/paperless_mail/migrations/0002_auto_20201117_1334.py b/src/paperless_mail/migrations/0002_auto_20201117_1334.py index 5b29b3072..72e37e342 100644 --- a/src/paperless_mail/migrations/0002_auto_20201117_1334.py +++ b/src/paperless_mail/migrations/0002_auto_20201117_1334.py @@ -2,28 +2,12 @@ from django.db import migrations from django.db.migrations import RunPython -from django_q.models import Schedule -from django_q.tasks import schedule - - -def add_schedules(apps, schema_editor): - schedule( - "paperless_mail.tasks.process_mail_accounts", - name="Check all e-mail accounts", - schedule_type=Schedule.MINUTES, - minutes=10, - ) - - -def remove_schedules(apps, schema_editor): - Schedule.objects.filter(func="paperless_mail.tasks.process_mail_accounts").delete() class Migration(migrations.Migration): dependencies = [ ("paperless_mail", "0001_initial"), - ("django_q", "0013_task_attempt_count"), ] - operations = [RunPython(add_schedules, remove_schedules)] + operations = [RunPython(migrations.RunPython.noop, migrations.RunPython.noop)] diff --git a/src/paperless_mail/tasks.py b/src/paperless_mail/tasks.py index faa0300e8..5c92233de 100644 --- a/src/paperless_mail/tasks.py +++ b/src/paperless_mail/tasks.py @@ -1,13 +1,14 @@ import logging +from celery import shared_task from paperless_mail.mail import MailAccountHandler from paperless_mail.mail import MailError from paperless_mail.models import MailAccount - logger = logging.getLogger("paperless.mail.tasks") +@shared_task def process_mail_accounts(): total_new_documents = 0 for account in MailAccount.objects.all(): @@ -20,11 +21,3 @@ def process_mail_accounts(): return f"Added {total_new_documents} document(s)." else: return "No new documents were added." - - -def process_mail_account(name): - try: - account = MailAccount.objects.get(name=name) - MailAccountHandler().handle_mail_account(account) - except MailAccount.DoesNotExist: - logger.error(f"Unknown mail acccount: {name}") diff --git a/src/paperless_mail/tests/test_mail.py b/src/paperless_mail/tests/test_mail.py index be016a79a..f0e69dfab 100644 --- a/src/paperless_mail/tests/test_mail.py +++ b/src/paperless_mail/tests/test_mail.py @@ -247,7 +247,7 @@ class TestMail(DirectoriesMixin, TestCase): m.return_value = self.bogus_mailbox self.addCleanup(patcher.stop) - patcher = mock.patch("paperless_mail.mail.async_task") + patcher = mock.patch("paperless_mail.mail.consume_file.delay") self.async_task = patcher.start() self.addCleanup(patcher.stop) @@ -1030,20 +1030,3 @@ class TestTasks(TestCase): m.side_effect = lambda account: 0 result = tasks.process_mail_accounts() self.assertIn("No new", result) - - @mock.patch("paperless_mail.tasks.MailAccountHandler.handle_mail_account") - def test_single_accounts(self, m): - MailAccount.objects.create( - name="A", - imap_server="A", - username="A", - password="A", - ) - - tasks.process_mail_account("A") - - m.assert_called_once() - m.reset_mock() - - tasks.process_mail_account("B") - m.assert_not_called() From 07ec74a5d6a58a2b70aef5538db17afefd035748 Mon Sep 17 00:00:00 2001 From: Trenton H Date: Mon, 26 Sep 2022 12:12:57 -0700 Subject: [PATCH 012/273] Runs the pre-commit hooks against the changelog before committing it --- .github/workflows/ci.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ddf23e253..46417d97e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -484,6 +484,18 @@ jobs: uses: actions/checkout@v3 with: ref: main + - + name: Install pipenv + run: | + pip3 install --upgrade pip setuptools wheel pipx + pipx install pipenv + - + name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.9 + cache: "pipenv" + cache-dependency-path: 'Pipfile.lock' - name: Append Changelog to docs id: append-Changelog @@ -497,9 +509,10 @@ jobs: CURRENT_CHANGELOG=`tail --lines +2 changelog.md` echo -e "$CURRENT_CHANGELOG" >> changelog-new.md mv changelog-new.md changelog.md + pipenv run pre-commit --files changelog.md git config --global user.name "github-actions" git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" - git commit -am "Changelog ${{ steps.get_version.outputs.version }} - GHA" + git commit -am "Changelog ${{ needs.publish-release.outputs.version }} - GHA" git push origin ${{ needs.publish-release.outputs.version }}-changelog - name: Create Pull Request From 49054c61a4729868348ff16bf376d77289121bcd Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+shamoon@users.noreply.github.com> Date: Tue, 27 Sep 2022 11:00:02 -0700 Subject: [PATCH 013/273] Add failed tasks badge, better animation --- .../app-frame/app-frame.component.html | 13 ++--- .../app-frame/app-frame.component.scss | 50 ++++++++++++++++++- .../app-frame/app-frame.component.ts | 10 ++++ 3 files changed, 66 insertions(+), 7 deletions(-) diff --git a/src-ui/src/app/components/app-frame/app-frame.component.html b/src-ui/src/app/components/app-frame/app-frame.component.html index c1d6a20bd..2ff57ed37 100644 --- a/src-ui/src/app/components/app-frame/app-frame.component.html +++ b/src-ui/src/app/components/app-frame/app-frame.component.html @@ -51,8 +51,8 @@
-
{{ task.name }} - {{ task.created | customDate:'short' }} + {{ task.date_created | customDate:'short' }}
diff --git a/src-ui/src/app/components/manage/tasks/tasks.component.ts b/src-ui/src/app/components/manage/tasks/tasks.component.ts index 791e41c3e..1f896e07e 100644 --- a/src-ui/src/app/components/manage/tasks/tasks.component.ts +++ b/src-ui/src/app/components/manage/tasks/tasks.component.ts @@ -1,6 +1,6 @@ import { Component, OnInit, OnDestroy } from '@angular/core' import { NgbModal } from '@ng-bootstrap/ng-bootstrap' -import { takeUntil, Subject, first } from 'rxjs' +import { Subject, first } from 'rxjs' import { PaperlessTask } from 'src/app/data/paperless-task' import { TasksService } from 'src/app/services/tasks.service' import { ConfirmDialogComponent } from '../../common/confirm-dialog/confirm-dialog.component' @@ -77,16 +77,16 @@ export class TasksComponent implements OnInit, OnDestroy { get currentTasks(): PaperlessTask[] { let tasks: PaperlessTask[] switch (this.activeTab) { - case 'PENDING': + case 'queued': tasks = this.tasksService.queuedFileTasks break - case 'STARTED': + case 'started': tasks = this.tasksService.startedFileTasks break - case 'SUCCESS': + case 'completed': tasks = this.tasksService.completedFileTasks break - case 'FAILURE': + case 'failed': tasks = this.tasksService.failedFileTasks break default: diff --git a/src-ui/src/app/data/paperless-task.ts b/src-ui/src/app/data/paperless-task.ts index 12928aaa1..f2259080e 100644 --- a/src-ui/src/app/data/paperless-task.ts +++ b/src-ui/src/app/data/paperless-task.ts @@ -23,7 +23,7 @@ export interface PaperlessTask extends ObjectWithId { name: string - created: Date + date_created: Date done?: Date diff --git a/src-ui/src/app/services/tasks.service.ts b/src-ui/src/app/services/tasks.service.ts index d34172b7a..4607128a1 100644 --- a/src-ui/src/app/services/tasks.service.ts +++ b/src-ui/src/app/services/tasks.service.ts @@ -1,6 +1,6 @@ import { HttpClient } from '@angular/common/http' import { Injectable } from '@angular/core' -import { first, map } from 'rxjs/operators' +import { first } from 'rxjs/operators' import { PaperlessTask, PaperlessTaskStatus, From c8f252d165ca5c71ba0df978e470bfefee49a979 Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+shamoon@users.noreply.github.com> Date: Tue, 27 Sep 2022 19:40:24 -0700 Subject: [PATCH 018/273] Add document name & error result parsing to PaperlessTask serializer --- src/documents/serialisers.py | 42 +++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py index ddba3fc6b..c8e312c73 100644 --- a/src/documents/serialisers.py +++ b/src/documents/serialisers.py @@ -1,5 +1,7 @@ import datetime +import json import math +import os import re try: @@ -630,6 +632,7 @@ class TasksViewSerializer(serializers.ModelSerializer): "result", "acknowledged", "task_name", + "name", ) type = serializers.SerializerMethodField() @@ -643,7 +646,15 @@ class TasksViewSerializer(serializers.ModelSerializer): def get_result(self, obj): result = "" if hasattr(obj, "attempted_task") and obj.attempted_task: - result = obj.attempted_task.result + try: + result_json = json.loads(obj.attempted_task.result) + except Exception: + pass + + if result_json and "exc_message" in result_json: + result = result_json["exc_message"] + else: + result = obj.attempted_task.result.strip('"') return result status = serializers.SerializerMethodField() @@ -686,6 +697,35 @@ class TasksViewSerializer(serializers.ModelSerializer): result = obj.attempted_task.task_name return result + name = serializers.SerializerMethodField() + + def get_name(self, obj): + result = "" + if hasattr(obj, "attempted_task") and obj.attempted_task: + try: + # We have to make this a valid JSON object string + kwargs_json = json.loads( + obj.attempted_task.task_kwargs.strip('"') + .replace("'", '"') + .replace("None", '""'), + ) + except Exception: + pass + + if kwargs_json and "override_filename" in kwargs_json: + result = kwargs_json["override_filename"] + else: + filepath = ( + obj.attempted_task.task_args.replace('"', "") + .replace("'", "") + .replace("(", "") + .replace(")", "") + .replace(",", "") + ) + result = os.path.split(filepath)[1] + + return result + class AcknowledgeTasksViewSerializer(serializers.Serializer): From 5162bdd404382abb2a3e29d67e3e714a06e92671 Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+shamoon@users.noreply.github.com> Date: Tue, 27 Sep 2022 19:41:23 -0700 Subject: [PATCH 019/273] Filter out old migrated tasks --- src/documents/views.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/documents/views.py b/src/documents/views.py index 3a3cf4ae2..f96166b0a 100644 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -879,6 +879,7 @@ class TasksViewSet(ReadOnlyModelViewSet): queryset = ( PaperlessTask.objects.filter( acknowledged=False, + attempted_task__isnull=False, ) .order_by("attempted_task__date_created") .reverse() From 4fe37f6aee19d97cd0dcea59ca6c662b1510a05f Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+shamoon@users.noreply.github.com> Date: Tue, 27 Sep 2022 20:50:26 -0700 Subject: [PATCH 020/273] Add related_document and direct link from task UI --- .../manage/tasks/tasks.component.html | 17 ++++++++++++----- .../components/manage/tasks/tasks.component.ts | 9 ++++++++- src-ui/src/app/data/paperless-task.ts | 2 ++ src/documents/serialisers.py | 18 ++++++++++++++++++ 4 files changed, 40 insertions(+), 6 deletions(-) diff --git a/src-ui/src/app/components/manage/tasks/tasks.component.html b/src-ui/src/app/components/manage/tasks/tasks.component.html index e12c1651a..961b8b091 100644 --- a/src-ui/src/app/components/manage/tasks/tasks.component.html +++ b/src-ui/src/app/components/manage/tasks/tasks.component.html @@ -74,11 +74,18 @@ - +
+ + +
diff --git a/src-ui/src/app/components/manage/tasks/tasks.component.ts b/src-ui/src/app/components/manage/tasks/tasks.component.ts index 1f896e07e..a2601dd8b 100644 --- a/src-ui/src/app/components/manage/tasks/tasks.component.ts +++ b/src-ui/src/app/components/manage/tasks/tasks.component.ts @@ -1,4 +1,5 @@ import { Component, OnInit, OnDestroy } from '@angular/core' +import { Router } from '@angular/router' import { NgbModal } from '@ng-bootstrap/ng-bootstrap' import { Subject, first } from 'rxjs' import { PaperlessTask } from 'src/app/data/paperless-task' @@ -24,7 +25,8 @@ export class TasksComponent implements OnInit, OnDestroy { constructor( public tasksService: TasksService, - private modalService: NgbModal + private modalService: NgbModal, + private readonly router: Router ) {} ngOnInit() { @@ -64,6 +66,11 @@ export class TasksComponent implements OnInit, OnDestroy { } } + dismissAndGo(task: PaperlessTask) { + this.dismissTask(task) + this.router.navigate(['documents', task.related_document]) + } + expandTask(task: PaperlessTask) { this.expandedTask = this.expandedTask == task.id ? undefined : task.id } diff --git a/src-ui/src/app/data/paperless-task.ts b/src-ui/src/app/data/paperless-task.ts index f2259080e..ccf09bb6f 100644 --- a/src-ui/src/app/data/paperless-task.ts +++ b/src-ui/src/app/data/paperless-task.ts @@ -28,4 +28,6 @@ export interface PaperlessTask extends ObjectWithId { done?: Date result: string + + related_document?: number } diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py index c8e312c73..172992de4 100644 --- a/src/documents/serialisers.py +++ b/src/documents/serialisers.py @@ -633,6 +633,7 @@ class TasksViewSerializer(serializers.ModelSerializer): "acknowledged", "task_name", "name", + "related_document", ) type = serializers.SerializerMethodField() @@ -726,6 +727,23 @@ class TasksViewSerializer(serializers.ModelSerializer): return result + related_document = serializers.SerializerMethodField() + + def get_related_document(self, obj): + result = "" + regexp = r"New document id (\d+) created" + if ( + hasattr(obj, "attempted_task") + and obj.attempted_task + and obj.attempted_task.status == "SUCCESS" + ): + try: + result = re.search(regexp, obj.attempted_task.result).group(1) + except Exception: + pass + + return result + class AcknowledgeTasksViewSerializer(serializers.Serializer): From 6f6f0067046c1105867130230645d588cc841cd5 Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+shamoon@users.noreply.github.com> Date: Tue, 27 Sep 2022 21:01:16 -0700 Subject: [PATCH 021/273] re-enable frontend tests and fix tasks fixture --- .github/workflows/ci.yml | 38 +-- src-ui/cypress/fixtures/tasks/tasks.json | 291 ++++++++++++++++++++++- 2 files changed, 309 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 801095259..ddf23e253 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -141,24 +141,24 @@ jobs: cd src/ pipenv run coveralls --service=github - # tests-frontend: - # name: "Tests Frontend" - # runs-on: ubuntu-20.04 - # needs: - # - pre-commit - # strategy: - # matrix: - # node-version: [16.x] - # steps: - # - uses: actions/checkout@v3 - # - - # name: Use Node.js ${{ matrix.node-version }} - # uses: actions/setup-node@v3 - # with: - # node-version: ${{ matrix.node-version }} - # - run: cd src-ui && npm ci - # - run: cd src-ui && npm run test - # - run: cd src-ui && npm run e2e:ci + tests-frontend: + name: "Tests Frontend" + runs-on: ubuntu-20.04 + needs: + - pre-commit + strategy: + matrix: + node-version: [16.x] + steps: + - uses: actions/checkout@v3 + - + name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + - run: cd src-ui && npm ci + - run: cd src-ui && npm run test + - run: cd src-ui && npm run e2e:ci prepare-docker-build: name: Prepare Docker Pipeline Data @@ -173,7 +173,7 @@ jobs: needs: - documentation - tests-backend - #- tests-frontend + - tests-frontend steps: - name: Set ghcr repository name diff --git a/src-ui/cypress/fixtures/tasks/tasks.json b/src-ui/cypress/fixtures/tasks/tasks.json index ceb334b9d..eeccfe424 100644 --- a/src-ui/cypress/fixtures/tasks/tasks.json +++ b/src-ui/cypress/fixtures/tasks/tasks.json @@ -1 +1,290 @@ -[{"id":141,"type":"file","result":"sample 2.pdf: Not consuming sample 2.pdf: It is a duplicate. : Traceback (most recent call last):\n File \"/Users/admin/.local/share/virtualenvs/paperless-ngx.nosync-udqDZzaE/lib/python3.8/site-packages/django_q/cluster.py\", line 432, in worker\n res = f(*task[\"args\"], **task[\"kwargs\"])\n File \"/Users/admin/Documents/paperless-ngx/src/documents/tasks.py\", line 316, in consume_file\n document = Consumer().try_consume_file(\n File \"/Users/admin/Documents/paperless-ngx/src/documents/consumer.py\", line 218, in try_consume_file\n self.pre_check_duplicate()\n File \"/Users/admin/Documents/paperless-ngx/src/documents/consumer.py\", line 113, in pre_check_duplicate\n self._fail(\n File \"/Users/admin/Documents/paperless-ngx/src/documents/consumer.py\", line 84, in _fail\n raise ConsumerError(f\"{self.filename}: {log_message or message}\")\ndocuments.consumer.ConsumerError: sample 2.pdf: Not consuming sample 2.pdf: It is a duplicate.\n","status":"failed","task_id":"d8ddbe298a42427d82553206ddf0bc94","name":"sample 2.pdf","created":"2022-05-26T23:17:38.333474-07:00","acknowledged":false,"attempted_task":{"id":"d8ddbe298a42427d82553206ddf0bc94","name":"sample 2.pdf","func":"documents.tasks.consume_file","hook":null,"args":"gAWVLgAAAAAAAACMKC90bXAvcGFwZXJsZXNzL3BhcGVybGVzcy11cGxvYWQtanJxNGs1aHOUhZQu","kwargs":"gAWVzQAAAAAAAAB9lCiMEW92ZXJyaWRlX2ZpbGVuYW1llIwMc2FtcGxlIDIucGRmlIwOb3ZlcnJpZGVfdGl0bGWUTowZb3ZlcnJpZGVfY29ycmVzcG9uZGVudF9pZJROjBlvdmVycmlkZV9kb2N1bWVudF90eXBlX2lklE6MEG92ZXJyaWRlX3RhZ19pZHOUTowHdGFza19pZJSMJDcyMGExYjI5LWI2OTYtNDY3My05Y2ZmLTJkY2ZiZWNmNWViMpSMEG92ZXJyaWRlX2NyZWF0ZWSUTnUu","result":"gAWVMQQAAAAAAABYKgQAAHNhbXBsZSAyLnBkZjogTm90IGNvbnN1bWluZyBzYW1wbGUgMi5wZGY6IEl0IGlzIGEgZHVwbGljYXRlLiA6IFRyYWNlYmFjayAobW9zdCByZWNlbnQgY2FsbCBsYXN0KToKICBGaWxlICIvVXNlcnMvbW9vbmVyLy5sb2NhbC9zaGFyZS92aXJ0dWFsZW52cy9wYXBlcmxlc3Mtbmd4Lm5vc3luYy11ZHFEWnphRS9saWIvcHl0aG9uMy44L3NpdGUtcGFja2FnZXMvZGphbmdvX3EvY2x1c3Rlci5weSIsIGxpbmUgNDMyLCBpbiB3b3JrZXIKICAgIHJlcyA9IGYoKnRhc2tbImFyZ3MiXSwgKip0YXNrWyJrd2FyZ3MiXSkKICBGaWxlICIvVXNlcnMvbW9vbmVyL0RvY3VtZW50cy9Xb3JrL0Rldi5ub3N5bmMvQ29udHJpYnV0aW9ucy9wYXBlcmxlc3Mtbmd4L3NyYy9kb2N1bWVudHMvdGFza3MucHkiLCBsaW5lIDMxNiwgaW4gY29uc3VtZV9maWxlCiAgICBkb2N1bWVudCA9IENvbnN1bWVyKCkudHJ5X2NvbnN1bWVfZmlsZSgKICBGaWxlICIvVXNlcnMvbW9vbmVyL0RvY3VtZW50cy9Xb3JrL0Rldi5ub3N5bmMvQ29udHJpYnV0aW9ucy9wYXBlcmxlc3Mtbmd4L3NyYy9kb2N1bWVudHMvY29uc3VtZXIucHkiLCBsaW5lIDIxOCwgaW4gdHJ5X2NvbnN1bWVfZmlsZQogICAgc2VsZi5wcmVfY2hlY2tfZHVwbGljYXRlKCkKICBGaWxlICIvVXNlcnMvbW9vbmVyL0RvY3VtZW50cy9Xb3JrL0Rldi5ub3N5bmMvQ29udHJpYnV0aW9ucy9wYXBlcmxlc3Mtbmd4L3NyYy9kb2N1bWVudHMvY29uc3VtZXIucHkiLCBsaW5lIDExMywgaW4gcHJlX2NoZWNrX2R1cGxpY2F0ZQogICAgc2VsZi5fZmFpbCgKICBGaWxlICIvVXNlcnMvbW9vbmVyL0RvY3VtZW50cy9Xb3JrL0Rldi5ub3N5bmMvQ29udHJpYnV0aW9ucy9wYXBlcmxlc3Mtbmd4L3NyYy9kb2N1bWVudHMvY29uc3VtZXIucHkiLCBsaW5lIDg0LCBpbiBfZmFpbAogICAgcmFpc2UgQ29uc3VtZXJFcnJvcihmIntzZWxmLmZpbGVuYW1lfToge2xvZ19tZXNzYWdlIG9yIG1lc3NhZ2V9IikKZG9jdW1lbnRzLmNvbnN1bWVyLkNvbnN1bWVyRXJyb3I6IHNhbXBsZSAyLnBkZjogTm90IGNvbnN1bWluZyBzYW1wbGUgMi5wZGY6IEl0IGlzIGEgZHVwbGljYXRlLgqULg==","group":null,"started":"2022-05-26T23:17:37.702432-07:00","stopped":"2022-05-26T23:17:38.313306-07:00","success":false,"attempt_count":1}},{"id":132,"type":"file","result":" : Traceback (most recent call last):\n File \"/Users/admin/.local/share/virtualenvs/paperless-ng/lib/python3.6/site-packages/ocrmypdf/subprocess.py\", line 131, in get_version\n env=env,\n File \"/Users/admin/.local/share/virtualenvs/paperless-ng/lib/python3.6/site-packages/ocrmypdf/subprocess.py\", line 68, in run\n proc = subprocess_run(args, env=env, **kwargs)\n File \"/Users/admin/opt/anaconda3/envs/paperless-ng/lib/python3.6/subprocess.py\", line 423, in run\n with Popen(*popenargs, **kwargs) as process:\n File \"/Users/admin/opt/anaconda3/envs/paperless-ng/lib/python3.6/subprocess.py\", line 729, in __init__\n restore_signals, start_new_session)\n File \"/Users/admin/opt/anaconda3/envs/paperless-ng/lib/python3.6/subprocess.py\", line 1364, in _execute_child\n raise child_exception_type(errno_num, err_msg, err_filename)\nFileNotFoundError: [Errno 2] No such file or directory: 'unpaper': 'unpaper'\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/Users/admin/.local/share/virtualenvs/paperless-ng/lib/python3.6/site-packages/ocrmypdf/subprocess.py\", line 287, in check_external_program\n found_version = version_checker()\n File \"/Users/admin/.local/share/virtualenvs/paperless-ng/lib/python3.6/site-packages/ocrmypdf/_exec/unpaper.py\", line 34, in version\n return get_version('unpaper')\n File \"/Users/admin/.local/share/virtualenvs/paperless-ng/lib/python3.6/site-packages/ocrmypdf/subprocess.py\", line 137, in get_version\n ) from e\nocrmypdf.exceptions.MissingDependencyError: Could not find program 'unpaper' on the PATH\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n File \"/Users/admin/Documents/Work/Contributions/paperless-ng/src/paperless_tesseract/parsers.py\", line 176, in parse\n ocrmypdf.ocr(**ocr_args)\n File \"/Users/admin/.local/share/virtualenvs/paperless-ng/lib/python3.6/site-packages/ocrmypdf/api.py\", line 315, in ocr\n check_options(options, plugin_manager)\n File \"/Users/admin/.local/share/virtualenvs/paperless-ng/lib/python3.6/site-packages/ocrmypdf/_validation.py\", line 260, in check_options\n _check_options(options, plugin_manager, ocr_engine_languages)\n File \"/Users/admin/.local/share/virtualenvs/paperless-ng/lib/python3.6/site-packages/ocrmypdf/_validation.py\", line 250, in _check_options\n check_options_preprocessing(options)\n File \"/Users/admin/.local/share/virtualenvs/paperless-ng/lib/python3.6/site-packages/ocrmypdf/_validation.py\", line 128, in check_options_preprocessing\n required_for=['--clean, --clean-final'],\n File \"/Users/admin/.local/share/virtualenvs/paperless-ng/lib/python3.6/site-packages/ocrmypdf/subprocess.py\", line 293, in check_external_program\n raise MissingDependencyError()\nocrmypdf.exceptions.MissingDependencyError\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n File \"/Users/admin/Documents/Work/Contributions/paperless-ng/src/documents/consumer.py\", line 179, in try_consume_file\n document_parser.parse(self.path, mime_type, self.filename)\n File \"/Users/admin/Documents/Work/Contributions/paperless-ng/src/paperless_tesseract/parsers.py\", line 197, in parse\n raise ParseError(e)\ndocuments.parsers.ParseError\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n File \"/Users/admin/.local/share/virtualenvs/paperless-ng/lib/python3.6/site-packages/django_q/cluster.py\", line 436, in worker\n res = f(*task[\"args\"], **task[\"kwargs\"])\n File \"/Users/admin/Documents/Work/Contributions/paperless-ng/src/documents/tasks.py\", line 73, in consume_file\n override_tag_ids=override_tag_ids)\n File \"/Users/admin/Documents/Work/Contributions/paperless-ng/src/documents/consumer.py\", line 196, in try_consume_file\n raise ConsumerError(e)\ndocuments.consumer.ConsumerError\n","status":"failed","task_id":"4c554075552c4cc985abd76e6f274c90","name":"pdf-sample 10.24.48 PM.pdf","created":"2022-05-26T14:26:07.846365-07:00","acknowledged":null,"attempted_task":{"id":"4c554075552c4cc985abd76e6f274c90","name":"pdf-sample 10.24.48 PM.pdf","func":"documents.tasks.consume_file","hook":null,"args":"gAWVKwAAAAAAAACMJS4uL2NvbnN1bWUvcGRmLXNhbXBsZSAxMC4yNC40OCBQTS5wZGaUhZQu","kwargs":"gAWVGAAAAAAAAAB9lIwQb3ZlcnJpZGVfdGFnX2lkc5ROcy4=","result":"gAWVzA8AAAAAAABYxQ8AACA6IFRyYWNlYmFjayAobW9zdCByZWNlbnQgY2FsbCBsYXN0KToKICBGaWxlICIvVXNlcnMvbW9vbmVyLy5sb2NhbC9zaGFyZS92aXJ0dWFsZW52cy9wYXBlcmxlc3MtbmctNzZCdUpsRUkvbGliL3B5dGhvbjMuNi9zaXRlLXBhY2thZ2VzL29jcm15cGRmL3N1YnByb2Nlc3MucHkiLCBsaW5lIDEzMSwgaW4gZ2V0X3ZlcnNpb24KICAgIGVudj1lbnYsCiAgRmlsZSAiL1VzZXJzL21vb25lci8ubG9jYWwvc2hhcmUvdmlydHVhbGVudnMvcGFwZXJsZXNzLW5nLTc2QnVKbEVJL2xpYi9weXRob24zLjYvc2l0ZS1wYWNrYWdlcy9vY3JteXBkZi9zdWJwcm9jZXNzLnB5IiwgbGluZSA2OCwgaW4gcnVuCiAgICBwcm9jID0gc3VicHJvY2Vzc19ydW4oYXJncywgZW52PWVudiwgKiprd2FyZ3MpCiAgRmlsZSAiL1VzZXJzL21vb25lci9vcHQvYW5hY29uZGEzL2VudnMvcGFwZXJsZXNzLW5nL2xpYi9weXRob24zLjYvc3VicHJvY2Vzcy5weSIsIGxpbmUgNDIzLCBpbiBydW4KICAgIHdpdGggUG9wZW4oKnBvcGVuYXJncywgKiprd2FyZ3MpIGFzIHByb2Nlc3M6CiAgRmlsZSAiL1VzZXJzL21vb25lci9vcHQvYW5hY29uZGEzL2VudnMvcGFwZXJsZXNzLW5nL2xpYi9weXRob24zLjYvc3VicHJvY2Vzcy5weSIsIGxpbmUgNzI5LCBpbiBfX2luaXRfXwogICAgcmVzdG9yZV9zaWduYWxzLCBzdGFydF9uZXdfc2Vzc2lvbikKICBGaWxlICIvVXNlcnMvbW9vbmVyL29wdC9hbmFjb25kYTMvZW52cy9wYXBlcmxlc3MtbmcvbGliL3B5dGhvbjMuNi9zdWJwcm9jZXNzLnB5IiwgbGluZSAxMzY0LCBpbiBfZXhlY3V0ZV9jaGlsZAogICAgcmFpc2UgY2hpbGRfZXhjZXB0aW9uX3R5cGUoZXJybm9fbnVtLCBlcnJfbXNnLCBlcnJfZmlsZW5hbWUpCkZpbGVOb3RGb3VuZEVycm9yOiBbRXJybm8gMl0gTm8gc3VjaCBmaWxlIG9yIGRpcmVjdG9yeTogJ3VucGFwZXInOiAndW5wYXBlcicKClRoZSBhYm92ZSBleGNlcHRpb24gd2FzIHRoZSBkaXJlY3QgY2F1c2Ugb2YgdGhlIGZvbGxvd2luZyBleGNlcHRpb246CgpUcmFjZWJhY2sgKG1vc3QgcmVjZW50IGNhbGwgbGFzdCk6CiAgRmlsZSAiL1VzZXJzL21vb25lci8ubG9jYWwvc2hhcmUvdmlydHVhbGVudnMvcGFwZXJsZXNzLW5nLTc2QnVKbEVJL2xpYi9weXRob24zLjYvc2l0ZS1wYWNrYWdlcy9vY3JteXBkZi9zdWJwcm9jZXNzLnB5IiwgbGluZSAyODcsIGluIGNoZWNrX2V4dGVybmFsX3Byb2dyYW0KICAgIGZvdW5kX3ZlcnNpb24gPSB2ZXJzaW9uX2NoZWNrZXIoKQogIEZpbGUgIi9Vc2Vycy9tb29uZXIvLmxvY2FsL3NoYXJlL3ZpcnR1YWxlbnZzL3BhcGVybGVzcy1uZy03NkJ1SmxFSS9saWIvcHl0aG9uMy42L3NpdGUtcGFja2FnZXMvb2NybXlwZGYvX2V4ZWMvdW5wYXBlci5weSIsIGxpbmUgMzQsIGluIHZlcnNpb24KICAgIHJldHVybiBnZXRfdmVyc2lvbigndW5wYXBlcicpCiAgRmlsZSAiL1VzZXJzL21vb25lci8ubG9jYWwvc2hhcmUvdmlydHVhbGVudnMvcGFwZXJsZXNzLW5nLTc2QnVKbEVJL2xpYi9weXRob24zLjYvc2l0ZS1wYWNrYWdlcy9vY3JteXBkZi9zdWJwcm9jZXNzLnB5IiwgbGluZSAxMzcsIGluIGdldF92ZXJzaW9uCiAgICApIGZyb20gZQpvY3JteXBkZi5leGNlcHRpb25zLk1pc3NpbmdEZXBlbmRlbmN5RXJyb3I6IENvdWxkIG5vdCBmaW5kIHByb2dyYW0gJ3VucGFwZXInIG9uIHRoZSBQQVRICgpEdXJpbmcgaGFuZGxpbmcgb2YgdGhlIGFib3ZlIGV4Y2VwdGlvbiwgYW5vdGhlciBleGNlcHRpb24gb2NjdXJyZWQ6CgpUcmFjZWJhY2sgKG1vc3QgcmVjZW50IGNhbGwgbGFzdCk6CiAgRmlsZSAiL1VzZXJzL21vb25lci9Eb2N1bWVudHMvV29yay9Db250cmlidXRpb25zL3BhcGVybGVzcy1uZy9zcmMvcGFwZXJsZXNzX3Rlc3NlcmFjdC9wYXJzZXJzLnB5IiwgbGluZSAxNzYsIGluIHBhcnNlCiAgICBvY3JteXBkZi5vY3IoKipvY3JfYXJncykKICBGaWxlICIvVXNlcnMvbW9vbmVyLy5sb2NhbC9zaGFyZS92aXJ0dWFsZW52cy9wYXBlcmxlc3MtbmctNzZCdUpsRUkvbGliL3B5dGhvbjMuNi9zaXRlLXBhY2thZ2VzL29jcm15cGRmL2FwaS5weSIsIGxpbmUgMzE1LCBpbiBvY3IKICAgIGNoZWNrX29wdGlvbnMob3B0aW9ucywgcGx1Z2luX21hbmFnZXIpCiAgRmlsZSAiL1VzZXJzL21vb25lci8ubG9jYWwvc2hhcmUvdmlydHVhbGVudnMvcGFwZXJsZXNzLW5nLTc2QnVKbEVJL2xpYi9weXRob24zLjYvc2l0ZS1wYWNrYWdlcy9vY3JteXBkZi9fdmFsaWRhdGlvbi5weSIsIGxpbmUgMjYwLCBpbiBjaGVja19vcHRpb25zCiAgICBfY2hlY2tfb3B0aW9ucyhvcHRpb25zLCBwbHVnaW5fbWFuYWdlciwgb2NyX2VuZ2luZV9sYW5ndWFnZXMpCiAgRmlsZSAiL1VzZXJzL21vb25lci8ubG9jYWwvc2hhcmUvdmlydHVhbGVudnMvcGFwZXJsZXNzLW5nLTc2QnVKbEVJL2xpYi9weXRob24zLjYvc2l0ZS1wYWNrYWdlcy9vY3JteXBkZi9fdmFsaWRhdGlvbi5weSIsIGxpbmUgMjUwLCBpbiBfY2hlY2tfb3B0aW9ucwogICAgY2hlY2tfb3B0aW9uc19wcmVwcm9jZXNzaW5nKG9wdGlvbnMpCiAgRmlsZSAiL1VzZXJzL21vb25lci8ubG9jYWwvc2hhcmUvdmlydHVhbGVudnMvcGFwZXJsZXNzLW5nLTc2QnVKbEVJL2xpYi9weXRob24zLjYvc2l0ZS1wYWNrYWdlcy9vY3JteXBkZi9fdmFsaWRhdGlvbi5weSIsIGxpbmUgMTI4LCBpbiBjaGVja19vcHRpb25zX3ByZXByb2Nlc3NpbmcKICAgIHJlcXVpcmVkX2Zvcj1bJy0tY2xlYW4sIC0tY2xlYW4tZmluYWwnXSwKICBGaWxlICIvVXNlcnMvbW9vbmVyLy5sb2NhbC9zaGFyZS92aXJ0dWFsZW52cy9wYXBlcmxlc3MtbmctNzZCdUpsRUkvbGliL3B5dGhvbjMuNi9zaXRlLXBhY2thZ2VzL29jcm15cGRmL3N1YnByb2Nlc3MucHkiLCBsaW5lIDI5MywgaW4gY2hlY2tfZXh0ZXJuYWxfcHJvZ3JhbQogICAgcmFpc2UgTWlzc2luZ0RlcGVuZGVuY3lFcnJvcigpCm9jcm15cGRmLmV4Y2VwdGlvbnMuTWlzc2luZ0RlcGVuZGVuY3lFcnJvcgoKRHVyaW5nIGhhbmRsaW5nIG9mIHRoZSBhYm92ZSBleGNlcHRpb24sIGFub3RoZXIgZXhjZXB0aW9uIG9jY3VycmVkOgoKVHJhY2ViYWNrIChtb3N0IHJlY2VudCBjYWxsIGxhc3QpOgogIEZpbGUgIi9Vc2Vycy9tb29uZXIvRG9jdW1lbnRzL1dvcmsvQ29udHJpYnV0aW9ucy9wYXBlcmxlc3Mtbmcvc3JjL2RvY3VtZW50cy9jb25zdW1lci5weSIsIGxpbmUgMTc5LCBpbiB0cnlfY29uc3VtZV9maWxlCiAgICBkb2N1bWVudF9wYXJzZXIucGFyc2Uoc2VsZi5wYXRoLCBtaW1lX3R5cGUsIHNlbGYuZmlsZW5hbWUpCiAgRmlsZSAiL1VzZXJzL21vb25lci9Eb2N1bWVudHMvV29yay9Db250cmlidXRpb25zL3BhcGVybGVzcy1uZy9zcmMvcGFwZXJsZXNzX3Rlc3NlcmFjdC9wYXJzZXJzLnB5IiwgbGluZSAxOTcsIGluIHBhcnNlCiAgICByYWlzZSBQYXJzZUVycm9yKGUpCmRvY3VtZW50cy5wYXJzZXJzLlBhcnNlRXJyb3IKCkR1cmluZyBoYW5kbGluZyBvZiB0aGUgYWJvdmUgZXhjZXB0aW9uLCBhbm90aGVyIGV4Y2VwdGlvbiBvY2N1cnJlZDoKClRyYWNlYmFjayAobW9zdCByZWNlbnQgY2FsbCBsYXN0KToKICBGaWxlICIvVXNlcnMvbW9vbmVyLy5sb2NhbC9zaGFyZS92aXJ0dWFsZW52cy9wYXBlcmxlc3MtbmctNzZCdUpsRUkvbGliL3B5dGhvbjMuNi9zaXRlLXBhY2thZ2VzL2RqYW5nb19xL2NsdXN0ZXIucHkiLCBsaW5lIDQzNiwgaW4gd29ya2VyCiAgICByZXMgPSBmKCp0YXNrWyJhcmdzIl0sICoqdGFza1sia3dhcmdzIl0pCiAgRmlsZSAiL1VzZXJzL21vb25lci9Eb2N1bWVudHMvV29yay9Db250cmlidXRpb25zL3BhcGVybGVzcy1uZy9zcmMvZG9jdW1lbnRzL3Rhc2tzLnB5IiwgbGluZSA3MywgaW4gY29uc3VtZV9maWxlCiAgICBvdmVycmlkZV90YWdfaWRzPW92ZXJyaWRlX3RhZ19pZHMpCiAgRmlsZSAiL1VzZXJzL21vb25lci9Eb2N1bWVudHMvV29yay9Db250cmlidXRpb25zL3BhcGVybGVzcy1uZy9zcmMvZG9jdW1lbnRzL2NvbnN1bWVyLnB5IiwgbGluZSAxOTYsIGluIHRyeV9jb25zdW1lX2ZpbGUKICAgIHJhaXNlIENvbnN1bWVyRXJyb3IoZSkKZG9jdW1lbnRzLmNvbnN1bWVyLkNvbnN1bWVyRXJyb3IKlC4=","group":null,"started":"2021-01-20T10:47:34.535478-08:00","stopped":"2021-01-20T10:49:55.568010-08:00","success":false,"attempt_count":1}},{"id":115,"type":"file","result":"2021-01-24 2021-01-20 sample_wide_orange.pdf: Document is a duplicate : Traceback (most recent call last):\n File \"/Users/admin/.local/share/virtualenvs/paperless-ng/lib/python3.6/site-packages/django_q/cluster.py\", line 436, in worker\n res = f(*task[\"args\"], **task[\"kwargs\"])\n File \"/Users/admin/Documents/Work/Contributions/paperless-ng/src/documents/tasks.py\", line 75, in consume_file\n task_id=task_id\n File \"/Users/admin/Documents/Work/Contributions/paperless-ng/src/documents/consumer.py\", line 168, in try_consume_file\n self.pre_check_duplicate()\n File \"/Users/admin/Documents/Work/Contributions/paperless-ng/src/documents/consumer.py\", line 85, in pre_check_duplicate\n self._fail(\"Document is a duplicate\")\n File \"/Users/admin/Documents/Work/Contributions/paperless-ng/src/documents/consumer.py\", line 53, in _fail\n raise ConsumerError(f\"{self.filename}: {message}\")\ndocuments.consumer.ConsumerError: 2021-01-24 2021-01-20 sample_wide_orange.pdf: Document is a duplicate\n","status":"failed","task_id":"86494713646a4364b01da17aadca071d","name":"2021-01-24 2021-01-20 sample_wide_orange.pdf","created":"2022-05-26T14:26:07.817608-07:00","acknowledged":null,"attempted_task":{"id":"86494713646a4364b01da17aadca071d","name":"2021-01-24 2021-01-20 sample_wide_orange.pdf","func":"documents.tasks.consume_file","hook":null,"args":"gAWVLgAAAAAAAACMKC90bXAvcGFwZXJsZXNzL3BhcGVybGVzcy11cGxvYWQtcTJ6NDlnbzaUhZQu","kwargs":"gAWV2QAAAAAAAAB9lCiMEW92ZXJyaWRlX2ZpbGVuYW1llIwsMjAyMS0wMS0yNCAyMDIxLTAxLTIwIHNhbXBsZV93aWRlX29yYW5nZS5wZGaUjA5vdmVycmlkZV90aXRsZZROjBlvdmVycmlkZV9jb3JyZXNwb25kZW50X2lklE6MGW92ZXJyaWRlX2RvY3VtZW50X3R5cGVfaWSUTowQb3ZlcnJpZGVfdGFnX2lkc5ROjAd0YXNrX2lklIwkN2MwZTY1MmQtZDhkYy00OWU4LWI1ZmUtOGM3ZTkyZDlmOTI0lHUu","result":"gAWV/AMAAAAAAABY9QMAADIwMjEtMDEtMjQgMjAyMS0wMS0yMCBzYW1wbGVfd2lkZV9vcmFuZ2UucGRmOiBEb2N1bWVudCBpcyBhIGR1cGxpY2F0ZSA6IFRyYWNlYmFjayAobW9zdCByZWNlbnQgY2FsbCBsYXN0KToKICBGaWxlICIvVXNlcnMvbW9vbmVyLy5sb2NhbC9zaGFyZS92aXJ0dWFsZW52cy9wYXBlcmxlc3MtbmctNzZCdUpsRUkvbGliL3B5dGhvbjMuNi9zaXRlLXBhY2thZ2VzL2RqYW5nb19xL2NsdXN0ZXIucHkiLCBsaW5lIDQzNiwgaW4gd29ya2VyCiAgICByZXMgPSBmKCp0YXNrWyJhcmdzIl0sICoqdGFza1sia3dhcmdzIl0pCiAgRmlsZSAiL1VzZXJzL21vb25lci9Eb2N1bWVudHMvV29yay9Db250cmlidXRpb25zL3BhcGVybGVzcy1uZy9zcmMvZG9jdW1lbnRzL3Rhc2tzLnB5IiwgbGluZSA3NSwgaW4gY29uc3VtZV9maWxlCiAgICB0YXNrX2lkPXRhc2tfaWQKICBGaWxlICIvVXNlcnMvbW9vbmVyL0RvY3VtZW50cy9Xb3JrL0NvbnRyaWJ1dGlvbnMvcGFwZXJsZXNzLW5nL3NyYy9kb2N1bWVudHMvY29uc3VtZXIucHkiLCBsaW5lIDE2OCwgaW4gdHJ5X2NvbnN1bWVfZmlsZQogICAgc2VsZi5wcmVfY2hlY2tfZHVwbGljYXRlKCkKICBGaWxlICIvVXNlcnMvbW9vbmVyL0RvY3VtZW50cy9Xb3JrL0NvbnRyaWJ1dGlvbnMvcGFwZXJsZXNzLW5nL3NyYy9kb2N1bWVudHMvY29uc3VtZXIucHkiLCBsaW5lIDg1LCBpbiBwcmVfY2hlY2tfZHVwbGljYXRlCiAgICBzZWxmLl9mYWlsKCJEb2N1bWVudCBpcyBhIGR1cGxpY2F0ZSIpCiAgRmlsZSAiL1VzZXJzL21vb25lci9Eb2N1bWVudHMvV29yay9Db250cmlidXRpb25zL3BhcGVybGVzcy1uZy9zcmMvZG9jdW1lbnRzL2NvbnN1bWVyLnB5IiwgbGluZSA1MywgaW4gX2ZhaWwKICAgIHJhaXNlIENvbnN1bWVyRXJyb3IoZiJ7c2VsZi5maWxlbmFtZX06IHttZXNzYWdlfSIpCmRvY3VtZW50cy5jb25zdW1lci5Db25zdW1lckVycm9yOiAyMDIxLTAxLTI0IDIwMjEtMDEtMjAgc2FtcGxlX3dpZGVfb3JhbmdlLnBkZjogRG9jdW1lbnQgaXMgYSBkdXBsaWNhdGUKlC4=","group":null,"started":"2021-01-26T00:21:05.379583-08:00","stopped":"2021-01-26T00:21:06.449626-08:00","success":false,"attempt_count":1}},{"id":85,"type":"file","result":"cannot open resource : Traceback (most recent call last):\n File \"/Users/admin/.local/share/virtualenvs/paperless-ng/lib/python3.6/site-packages/django_q/cluster.py\", line 436, in worker\n res = f(*task[\"args\"], **task[\"kwargs\"])\n File \"/Users/admin/Documents/Work/Contributions/paperless-ng/src/documents/tasks.py\", line 81, in consume_file\n task_id=task_id\n File \"/Users/admin/Documents/Work/Contributions/paperless-ng/src/documents/consumer.py\", line 244, in try_consume_file\n self.path, mime_type, self.filename)\n File \"/Users/admin/Documents/Work/Contributions/paperless-ng/src/documents/parsers.py\", line 302, in get_optimised_thumbnail\n thumbnail = self.get_thumbnail(document_path, mime_type, file_name)\n File \"/Users/admin/Documents/Work/Contributions/paperless-ng/src/paperless_text/parsers.py\", line 29, in get_thumbnail\n layout_engine=ImageFont.LAYOUT_BASIC)\n File \"/Users/admin/.local/share/virtualenvs/paperless-ng/lib/python3.6/site-packages/PIL/ImageFont.py\", line 852, in truetype\n return freetype(font)\n File \"/Users/admin/.local/share/virtualenvs/paperless-ng/lib/python3.6/site-packages/PIL/ImageFont.py\", line 849, in freetype\n return FreeTypeFont(font, size, index, encoding, layout_engine)\n File \"/Users/admin/.local/share/virtualenvs/paperless-ng/lib/python3.6/site-packages/PIL/ImageFont.py\", line 210, in __init__\n font, size, index, encoding, layout_engine=layout_engine\nOSError: cannot open resource\n","status":"failed","task_id":"abca803fa46342e1ac81f3d3f2080e79","name":"simple.txt","created":"2022-05-26T14:26:07.771541-07:00","acknowledged":null,"attempted_task":{"id":"abca803fa46342e1ac81f3d3f2080e79","name":"simple.txt","func":"documents.tasks.consume_file","hook":null,"args":"gAWVLgAAAAAAAACMKC90bXAvcGFwZXJsZXNzL3BhcGVybGVzcy11cGxvYWQtd2RhbnB5NnGUhZQu","kwargs":"gAWVtwAAAAAAAAB9lCiMEW92ZXJyaWRlX2ZpbGVuYW1llIwKc2ltcGxlLnR4dJSMDm92ZXJyaWRlX3RpdGxllE6MGW92ZXJyaWRlX2NvcnJlc3BvbmRlbnRfaWSUTowZb3ZlcnJpZGVfZG9jdW1lbnRfdHlwZV9pZJROjBBvdmVycmlkZV90YWdfaWRzlE6MB3Rhc2tfaWSUjCQ3ZGE0OTU4ZC0zM2UwLTQ1OGMtYTE0ZC1kMmU0NmE0NWY4Y2SUdS4=","result":"gAWV5QUAAAAAAABY3gUAAGNhbm5vdCBvcGVuIHJlc291cmNlIDogVHJhY2ViYWNrIChtb3N0IHJlY2VudCBjYWxsIGxhc3QpOgogIEZpbGUgIi9Vc2Vycy9tb29uZXIvLmxvY2FsL3NoYXJlL3ZpcnR1YWxlbnZzL3BhcGVybGVzcy1uZy03NkJ1SmxFSS9saWIvcHl0aG9uMy42L3NpdGUtcGFja2FnZXMvZGphbmdvX3EvY2x1c3Rlci5weSIsIGxpbmUgNDM2LCBpbiB3b3JrZXIKICAgIHJlcyA9IGYoKnRhc2tbImFyZ3MiXSwgKip0YXNrWyJrd2FyZ3MiXSkKICBGaWxlICIvVXNlcnMvbW9vbmVyL0RvY3VtZW50cy9Xb3JrL0NvbnRyaWJ1dGlvbnMvcGFwZXJsZXNzLW5nL3NyYy9kb2N1bWVudHMvdGFza3MucHkiLCBsaW5lIDgxLCBpbiBjb25zdW1lX2ZpbGUKICAgIHRhc2tfaWQ9dGFza19pZAogIEZpbGUgIi9Vc2Vycy9tb29uZXIvRG9jdW1lbnRzL1dvcmsvQ29udHJpYnV0aW9ucy9wYXBlcmxlc3Mtbmcvc3JjL2RvY3VtZW50cy9jb25zdW1lci5weSIsIGxpbmUgMjQ0LCBpbiB0cnlfY29uc3VtZV9maWxlCiAgICBzZWxmLnBhdGgsIG1pbWVfdHlwZSwgc2VsZi5maWxlbmFtZSkKICBGaWxlICIvVXNlcnMvbW9vbmVyL0RvY3VtZW50cy9Xb3JrL0NvbnRyaWJ1dGlvbnMvcGFwZXJsZXNzLW5nL3NyYy9kb2N1bWVudHMvcGFyc2Vycy5weSIsIGxpbmUgMzAyLCBpbiBnZXRfb3B0aW1pc2VkX3RodW1ibmFpbAogICAgdGh1bWJuYWlsID0gc2VsZi5nZXRfdGh1bWJuYWlsKGRvY3VtZW50X3BhdGgsIG1pbWVfdHlwZSwgZmlsZV9uYW1lKQogIEZpbGUgIi9Vc2Vycy9tb29uZXIvRG9jdW1lbnRzL1dvcmsvQ29udHJpYnV0aW9ucy9wYXBlcmxlc3Mtbmcvc3JjL3BhcGVybGVzc190ZXh0L3BhcnNlcnMucHkiLCBsaW5lIDI5LCBpbiBnZXRfdGh1bWJuYWlsCiAgICBsYXlvdXRfZW5naW5lPUltYWdlRm9udC5MQVlPVVRfQkFTSUMpCiAgRmlsZSAiL1VzZXJzL21vb25lci8ubG9jYWwvc2hhcmUvdmlydHVhbGVudnMvcGFwZXJsZXNzLW5nLTc2QnVKbEVJL2xpYi9weXRob24zLjYvc2l0ZS1wYWNrYWdlcy9QSUwvSW1hZ2VGb250LnB5IiwgbGluZSA4NTIsIGluIHRydWV0eXBlCiAgICByZXR1cm4gZnJlZXR5cGUoZm9udCkKICBGaWxlICIvVXNlcnMvbW9vbmVyLy5sb2NhbC9zaGFyZS92aXJ0dWFsZW52cy9wYXBlcmxlc3MtbmctNzZCdUpsRUkvbGliL3B5dGhvbjMuNi9zaXRlLXBhY2thZ2VzL1BJTC9JbWFnZUZvbnQucHkiLCBsaW5lIDg0OSwgaW4gZnJlZXR5cGUKICAgIHJldHVybiBGcmVlVHlwZUZvbnQoZm9udCwgc2l6ZSwgaW5kZXgsIGVuY29kaW5nLCBsYXlvdXRfZW5naW5lKQogIEZpbGUgIi9Vc2Vycy9tb29uZXIvLmxvY2FsL3NoYXJlL3ZpcnR1YWxlbnZzL3BhcGVybGVzcy1uZy03NkJ1SmxFSS9saWIvcHl0aG9uMy42L3NpdGUtcGFja2FnZXMvUElML0ltYWdlRm9udC5weSIsIGxpbmUgMjEwLCBpbiBfX2luaXRfXwogICAgZm9udCwgc2l6ZSwgaW5kZXgsIGVuY29kaW5nLCBsYXlvdXRfZW5naW5lPWxheW91dF9lbmdpbmUKT1NFcnJvcjogY2Fubm90IG9wZW4gcmVzb3VyY2UKlC4=","group":null,"started":"2021-03-06T14:23:56.974715-08:00","stopped":"2021-03-06T14:24:28.011772-08:00","success":false,"attempt_count":1}},{"id":41,"type":"file","result":"commands.txt: Not consuming commands.txt: It is a duplicate. : Traceback (most recent call last):\n File \"/Users/admin/.local/share/virtualenvs/paperless-ngx.nosync-udqDZzaE/lib/python3.8/site-packages/django_q/cluster.py\", line 432, in worker\n res = f(*task[\"args\"], **task[\"kwargs\"])\n File \"/Users/admin/Documents/paperless-ngx/src/documents/tasks.py\", line 70, in consume_file\n document = Consumer().try_consume_file(\n File \"/Users/admin/Documents/paperless-ngx/src/documents/consumer.py\", line 199, in try_consume_file\n self.pre_check_duplicate()\n File \"/Users/admin/Documents/paperless-ngx/src/documents/consumer.py\", line 97, in pre_check_duplicate\n self._fail(\n File \"/Users/admin/Documents/paperless-ngx/src/documents/consumer.py\", line 69, in _fail\n raise ConsumerError(f\"{self.filename}: {log_message or message}\")\ndocuments.consumer.ConsumerError: commands.txt: Not consuming commands.txt: It is a duplicate.\n","status":"failed","task_id":"0af67672e8e14404b060d4cf8f69313d","name":"commands.txt","created":"2022-05-26T14:26:07.704247-07:00","acknowledged":null,"attempted_task":{"id":"0af67672e8e14404b060d4cf8f69313d","name":"commands.txt","func":"documents.tasks.consume_file","hook":null,"args":"gAWVLgAAAAAAAACMKC90bXAvcGFwZXJsZXNzL3BhcGVybGVzcy11cGxvYWQtZ3h4YjNxODaUhZQu","kwargs":"gAWVuQAAAAAAAAB9lCiMEW92ZXJyaWRlX2ZpbGVuYW1llIwMY29tbWFuZHMudHh0lIwOb3ZlcnJpZGVfdGl0bGWUTowZb3ZlcnJpZGVfY29ycmVzcG9uZGVudF9pZJROjBlvdmVycmlkZV9kb2N1bWVudF90eXBlX2lklE6MEG92ZXJyaWRlX3RhZ19pZHOUTowHdGFza19pZJSMJDRkMjhmMmJiLTJkMzAtNGQzNi1iNjM5LWU2YzQ5OTU3OGVlY5R1Lg==","result":"gAWVLwQAAAAAAABYKAQAAGNvbW1hbmRzLnR4dDogTm90IGNvbnN1bWluZyBjb21tYW5kcy50eHQ6IEl0IGlzIGEgZHVwbGljYXRlLiA6IFRyYWNlYmFjayAobW9zdCByZWNlbnQgY2FsbCBsYXN0KToKICBGaWxlICIvVXNlcnMvbW9vbmVyLy5sb2NhbC9zaGFyZS92aXJ0dWFsZW52cy9wYXBlcmxlc3Mtbmd4Lm5vc3luYy11ZHFEWnphRS9saWIvcHl0aG9uMy44L3NpdGUtcGFja2FnZXMvZGphbmdvX3EvY2x1c3Rlci5weSIsIGxpbmUgNDMyLCBpbiB3b3JrZXIKICAgIHJlcyA9IGYoKnRhc2tbImFyZ3MiXSwgKip0YXNrWyJrd2FyZ3MiXSkKICBGaWxlICIvVXNlcnMvbW9vbmVyL0RvY3VtZW50cy9Xb3JrL0Rldi5ub3N5bmMvQ29udHJpYnV0aW9ucy9wYXBlcmxlc3Mtbmd4L3NyYy9kb2N1bWVudHMvdGFza3MucHkiLCBsaW5lIDcwLCBpbiBjb25zdW1lX2ZpbGUKICAgIGRvY3VtZW50ID0gQ29uc3VtZXIoKS50cnlfY29uc3VtZV9maWxlKAogIEZpbGUgIi9Vc2Vycy9tb29uZXIvRG9jdW1lbnRzL1dvcmsvRGV2Lm5vc3luYy9Db250cmlidXRpb25zL3BhcGVybGVzcy1uZ3gvc3JjL2RvY3VtZW50cy9jb25zdW1lci5weSIsIGxpbmUgMTk5LCBpbiB0cnlfY29uc3VtZV9maWxlCiAgICBzZWxmLnByZV9jaGVja19kdXBsaWNhdGUoKQogIEZpbGUgIi9Vc2Vycy9tb29uZXIvRG9jdW1lbnRzL1dvcmsvRGV2Lm5vc3luYy9Db250cmlidXRpb25zL3BhcGVybGVzcy1uZ3gvc3JjL2RvY3VtZW50cy9jb25zdW1lci5weSIsIGxpbmUgOTcsIGluIHByZV9jaGVja19kdXBsaWNhdGUKICAgIHNlbGYuX2ZhaWwoCiAgRmlsZSAiL1VzZXJzL21vb25lci9Eb2N1bWVudHMvV29yay9EZXYubm9zeW5jL0NvbnRyaWJ1dGlvbnMvcGFwZXJsZXNzLW5neC9zcmMvZG9jdW1lbnRzL2NvbnN1bWVyLnB5IiwgbGluZSA2OSwgaW4gX2ZhaWwKICAgIHJhaXNlIENvbnN1bWVyRXJyb3IoZiJ7c2VsZi5maWxlbmFtZX06IHtsb2dfbWVzc2FnZSBvciBtZXNzYWdlfSIpCmRvY3VtZW50cy5jb25zdW1lci5Db25zdW1lckVycm9yOiBjb21tYW5kcy50eHQ6IE5vdCBjb25zdW1pbmcgY29tbWFuZHMudHh0OiBJdCBpcyBhIGR1cGxpY2F0ZS4KlC4=","group":null,"started":"2022-03-10T22:26:32.548772-08:00","stopped":"2022-03-10T22:26:32.879916-08:00","success":false,"attempt_count":1}},{"id":10,"type":"file","result":"Success. New document id 260 created","status":"complete","task_id":"b7629a0f41bd40c7a3ea4680341321b5","name":"2022-03-24+Sonstige+ScanPC2022-03-24_081058.pdf","created":"2022-05-26T14:26:07.670577-07:00","acknowledged":false,"attempted_task":{"id":"b7629a0f41bd40c7a3ea4680341321b5","name":"2022-03-24+Sonstige+ScanPC2022-03-24_081058.pdf","func":"documents.tasks.consume_file","hook":null,"args":"gAWVLgAAAAAAAACMKC90bXAvcGFwZXJsZXNzL3BhcGVybGVzcy11cGxvYWQtc25mOW11ZW+UhZQu","kwargs":"gAWV3AAAAAAAAAB9lCiMEW92ZXJyaWRlX2ZpbGVuYW1llIwvMjAyMi0wMy0yNCtTb25zdGlnZStTY2FuUEMyMDIyLTAzLTI0XzA4MTA1OC5wZGaUjA5vdmVycmlkZV90aXRsZZROjBlvdmVycmlkZV9jb3JyZXNwb25kZW50X2lklE6MGW92ZXJyaWRlX2RvY3VtZW50X3R5cGVfaWSUTowQb3ZlcnJpZGVfdGFnX2lkc5ROjAd0YXNrX2lklIwkNTdmMmQwMGItY2Q0Ny00YzQ3LTlmOTctODFlOTllMTJhMjMylHUu","result":"gAWVKAAAAAAAAACMJFN1Y2Nlc3MuIE5ldyBkb2N1bWVudCBpZCAyNjAgY3JlYXRlZJQu","group":null,"started":"2022-03-24T08:19:32.117861-07:00","stopped":"2022-03-24T08:20:10.239201-07:00","success":true,"attempt_count":1}},{"id":9,"type":"file","result":"Success. New document id 261 created","status":"complete","task_id":"02e276a86a424ccfb83309df5d8594be","name":"2sample-pdf-with-images.pdf","created":"2022-05-26T14:26:07.668987-07:00","acknowledged":false,"attempted_task":{"id":"02e276a86a424ccfb83309df5d8594be","name":"2sample-pdf-with-images.pdf","func":"documents.tasks.consume_file","hook":null,"args":"gAWVLgAAAAAAAACMKC90bXAvcGFwZXJsZXNzL3BhcGVybGVzcy11cGxvYWQtaXJ3cjZzOGeUhZQu","kwargs":"gAWVyAAAAAAAAAB9lCiMEW92ZXJyaWRlX2ZpbGVuYW1llIwbMnNhbXBsZS1wZGYtd2l0aC1pbWFnZXMucGRmlIwOb3ZlcnJpZGVfdGl0bGWUTowZb3ZlcnJpZGVfY29ycmVzcG9uZGVudF9pZJROjBlvdmVycmlkZV9kb2N1bWVudF90eXBlX2lklE6MEG92ZXJyaWRlX3RhZ19pZHOUTowHdGFza19pZJSMJDFlYTczMjhhLTk3MjctNDJiMC1iMTEyLTAzZjU3MzQ2MmRiNpR1Lg==","result":"gAWVKAAAAAAAAACMJFN1Y2Nlc3MuIE5ldyBkb2N1bWVudCBpZCAyNjEgY3JlYXRlZJQu","group":null,"started":"2022-03-28T23:12:41.286318-07:00","stopped":"2022-03-28T23:13:00.523505-07:00","success":true,"attempt_count":1}},{"id":8,"type":"file","result":"Success. New document id 262 created","status":"complete","task_id":"41229b8be9b445c0a523697d0f58f13e","name":"2sample-pdf-with-images_pw.pdf","created":"2022-05-26T14:26:07.667993-07:00","acknowledged":false,"attempted_task":{"id":"41229b8be9b445c0a523697d0f58f13e","name":"2sample-pdf-with-images_pw.pdf","func":"documents.tasks.consume_file","hook":null,"args":"gAWVLgAAAAAAAACMKC90bXAvcGFwZXJsZXNzL3BhcGVybGVzcy11cGxvYWQtN2tfejA0MTGUhZQu","kwargs":"gAWVywAAAAAAAAB9lCiMEW92ZXJyaWRlX2ZpbGVuYW1llIweMnNhbXBsZS1wZGYtd2l0aC1pbWFnZXNfcHcucGRmlIwOb3ZlcnJpZGVfdGl0bGWUTowZb3ZlcnJpZGVfY29ycmVzcG9uZGVudF9pZJROjBlvdmVycmlkZV9kb2N1bWVudF90eXBlX2lklE6MEG92ZXJyaWRlX3RhZ19pZHOUTowHdGFza19pZJSMJDk5YTgyOTc3LWU1MWUtNGJjYS04MjM4LTNkNzdhZTJhNjZmYZR1Lg==","result":"gAWVKAAAAAAAAACMJFN1Y2Nlc3MuIE5ldyBkb2N1bWVudCBpZCAyNjIgY3JlYXRlZJQu","group":null,"started":"2022-03-28T23:43:53.171963-07:00","stopped":"2022-03-28T23:43:56.965257-07:00","success":true,"attempt_count":1}},{"id":6,"type":"file","result":"Success. New document id 264 created","status":"complete","task_id":"bbbca32d408c4619bd0b512a8327c773","name":"homebridge.log","created":"2022-05-26T14:26:07.665560-07:00","acknowledged":false,"attempted_task":{"id":"bbbca32d408c4619bd0b512a8327c773","name":"homebridge.log","func":"documents.tasks.consume_file","hook":null,"args":"gAWVLgAAAAAAAACMKC90bXAvcGFwZXJsZXNzL3BhcGVybGVzcy11cGxvYWQteGo4aW9zYXaUhZQu","kwargs":"gAWVuwAAAAAAAAB9lCiMEW92ZXJyaWRlX2ZpbGVuYW1llIwOaG9tZWJyaWRnZS5sb2eUjA5vdmVycmlkZV90aXRsZZROjBlvdmVycmlkZV9jb3JyZXNwb25kZW50X2lklE6MGW92ZXJyaWRlX2RvY3VtZW50X3R5cGVfaWSUTowQb3ZlcnJpZGVfdGFnX2lkc5ROjAd0YXNrX2lklIwkNzY0NzdhNWEtNzk0Ni00NWU0LWE3MDktNzQzNDg0ZDE2YTUxlHUu","result":"gAWVKAAAAAAAAACMJFN1Y2Nlc3MuIE5ldyBkb2N1bWVudCBpZCAyNjQgY3JlYXRlZJQu","group":null,"started":"2022-03-29T22:56:16.053026-07:00","stopped":"2022-03-29T22:56:21.196179-07:00","success":true,"attempt_count":1}},{"id":5,"type":"file","result":"Success. New document id 265 created","status":"complete","task_id":"00ab285ab4bf482ab30c7d580b252ecb","name":"IMG_7459.PNG","created":"2022-05-26T14:26:07.664506-07:00","acknowledged":false,"attempted_task":{"id":"00ab285ab4bf482ab30c7d580b252ecb","name":"IMG_7459.PNG","func":"documents.tasks.consume_file","hook":null,"args":"gAWVLgAAAAAAAACMKC90bXAvcGFwZXJsZXNzL3BhcGVybGVzcy11cGxvYWQtOGF5NDNfZjeUhZQu","kwargs":"gAWVuQAAAAAAAAB9lCiMEW92ZXJyaWRlX2ZpbGVuYW1llIwMSU1HXzc0NTkuUE5HlIwOb3ZlcnJpZGVfdGl0bGWUTowZb3ZlcnJpZGVfY29ycmVzcG9uZGVudF9pZJROjBlvdmVycmlkZV9kb2N1bWVudF90eXBlX2lklE6MEG92ZXJyaWRlX3RhZ19pZHOUTowHdGFza19pZJSMJDYxMTNhNzRlLTAwOWMtNGJhYi1hMjk1LTFmNjMwMzZmMTc4ZpR1Lg==","result":"gAWVKAAAAAAAAACMJFN1Y2Nlc3MuIE5ldyBkb2N1bWVudCBpZCAyNjUgY3JlYXRlZJQu","group":null,"started":"2022-04-05T13:19:47.490282-07:00","stopped":"2022-04-05T13:21:36.782264-07:00","success":true,"attempt_count":1}},{"id":3,"type":"file","result":"Success. New document id 267 created","status":"complete","task_id":"289c5163cfec410db42948a0cacbeb9c","name":"IMG_7459.PNG","created":"2022-05-26T14:26:07.659661-07:00","acknowledged":false,"attempted_task":{"id":"289c5163cfec410db42948a0cacbeb9c","name":"IMG_7459.PNG","func":"documents.tasks.consume_file","hook":null,"args":"gAWVLgAAAAAAAACMKC90bXAvcGFwZXJsZXNzL3BhcGVybGVzcy11cGxvYWQtNzRuY2p2aXGUhZQu","kwargs":"gAWVuQAAAAAAAAB9lCiMEW92ZXJyaWRlX2ZpbGVuYW1llIwMSU1HXzc0NTkuUE5HlIwOb3ZlcnJpZGVfdGl0bGWUTowZb3ZlcnJpZGVfY29ycmVzcG9uZGVudF9pZJROjBlvdmVycmlkZV9kb2N1bWVudF90eXBlX2lklE6MEG92ZXJyaWRlX3RhZ19pZHOUTowHdGFza19pZJSMJGZjZDljMmFlLWFhZmEtNGJmMC05M2Y5LWE3ZGQxYmEzYWM1NZR1Lg==","result":"gAWVKAAAAAAAAACMJFN1Y2Nlc3MuIE5ldyBkb2N1bWVudCBpZCAyNjcgY3JlYXRlZJQu","group":null,"started":"2022-04-05T13:29:59.264441-07:00","stopped":"2022-04-05T13:30:28.336185-07:00","success":true,"attempt_count":1}},{"id":1,"type":"file","result":"Success. New document id 268 created","status":"complete","task_id":"7a4ebdb2bde04311935284027ef8ca65","name":"2019-08-04 DSA Questionnaire - 5-8-19.pdf","created":"2022-05-26T14:26:07.655276-07:00","acknowledged":false,"attempted_task":{"id":"7a4ebdb2bde04311935284027ef8ca65","name":"2019-08-04 DSA Questionnaire - 5-8-19.pdf","func":"documents.tasks.consume_file","hook":null,"args":"gAWVLgAAAAAAAACMKC90bXAvcGFwZXJsZXNzL3BhcGVybGVzcy11cGxvYWQtdXpscHl2NnmUhZQu","kwargs":"gAWV1gAAAAAAAAB9lCiMEW92ZXJyaWRlX2ZpbGVuYW1llIwpMjAxOS0wOC0wNCBEU0EgUXVlc3Rpb25uYWlyZSAtIDUtOC0xOS5wZGaUjA5vdmVycmlkZV90aXRsZZROjBlvdmVycmlkZV9jb3JyZXNwb25kZW50X2lklE6MGW92ZXJyaWRlX2RvY3VtZW50X3R5cGVfaWSUTowQb3ZlcnJpZGVfdGFnX2lkc5ROjAd0YXNrX2lklIwkY2Q3YzBhZjgtN2Q4Ni00OGM0LTliNjgtNDQwMmQ4ZDZlOTNmlHUu","result":"gAWVKAAAAAAAAACMJFN1Y2Nlc3MuIE5ldyBkb2N1bWVudCBpZCAyNjggY3JlYXRlZJQu","group":null,"started":"2022-04-28T21:01:04.275850-07:00","stopped":"2022-04-28T21:01:10.136884-07:00","success":true,"attempt_count":1}}] +[ + { + "id": 141, + "type": "file", + "result": "sample 2.pdf: Not consuming sample 2.pdf: It is a duplicate. : Traceback (most recent call last):\n File \"/Users/admin/.local/share/virtualenvs/paperless-ngx.nosync-udqDZzaE/lib/python3.8/site-packages/django_q/cluster.py\", line 432, in worker\n res = f(*task[\"args\"], **task[\"kwargs\"])\n File \"/Users/admin/Documents/paperless-ngx/src/documents/tasks.py\", line 316, in consume_file\n document = Consumer().try_consume_file(\n File \"/Users/admin/Documents/paperless-ngx/src/documents/consumer.py\", line 218, in try_consume_file\n self.pre_check_duplicate()\n File \"/Users/admin/Documents/paperless-ngx/src/documents/consumer.py\", line 113, in pre_check_duplicate\n self._fail(\n File \"/Users/admin/Documents/paperless-ngx/src/documents/consumer.py\", line 84, in _fail\n raise ConsumerError(f\"{self.filename}: {log_message or message}\")\ndocuments.consumer.ConsumerError: sample 2.pdf: Not consuming sample 2.pdf: It is a duplicate.\n", + "status": "FAILURE", + "task_id": "d8ddbe298a42427d82553206ddf0bc94", + "name": "sample 2.pdf", + "created": "2022-05-26T23:17:38.333474-07:00", + "acknowledged": false, + "attempted_task": { + "id": "d8ddbe298a42427d82553206ddf0bc94", + "name": "sample 2.pdf", + "func": "documents.tasks.consume_file", + "hook": null, + "args": "gAWVLgAAAAAAAACMKC90bXAvcGFwZXJsZXNzL3BhcGVybGVzcy11cGxvYWQtanJxNGs1aHOUhZQu", + "kwargs": "gAWVzQAAAAAAAAB9lCiMEW92ZXJyaWRlX2ZpbGVuYW1llIwMc2FtcGxlIDIucGRmlIwOb3ZlcnJpZGVfdGl0bGWUTowZb3ZlcnJpZGVfY29ycmVzcG9uZGVudF9pZJROjBlvdmVycmlkZV9kb2N1bWVudF90eXBlX2lklE6MEG92ZXJyaWRlX3RhZ19pZHOUTowHdGFza19pZJSMJDcyMGExYjI5LWI2OTYtNDY3My05Y2ZmLTJkY2ZiZWNmNWViMpSMEG92ZXJyaWRlX2NyZWF0ZWSUTnUu", + "result": "gAWVMQQAAAAAAABYKgQAAHNhbXBsZSAyLnBkZjogTm90IGNvbnN1bWluZyBzYW1wbGUgMi5wZGY6IEl0IGlzIGEgZHVwbGljYXRlLiA6IFRyYWNlYmFjayAobW9zdCByZWNlbnQgY2FsbCBsYXN0KToKICBGaWxlICIvVXNlcnMvbW9vbmVyLy5sb2NhbC9zaGFyZS92aXJ0dWFsZW52cy9wYXBlcmxlc3Mtbmd4Lm5vc3luYy11ZHFEWnphRS9saWIvcHl0aG9uMy44L3NpdGUtcGFja2FnZXMvZGphbmdvX3EvY2x1c3Rlci5weSIsIGxpbmUgNDMyLCBpbiB3b3JrZXIKICAgIHJlcyA9IGYoKnRhc2tbImFyZ3MiXSwgKip0YXNrWyJrd2FyZ3MiXSkKICBGaWxlICIvVXNlcnMvbW9vbmVyL0RvY3VtZW50cy9Xb3JrL0Rldi5ub3N5bmMvQ29udHJpYnV0aW9ucy9wYXBlcmxlc3Mtbmd4L3NyYy9kb2N1bWVudHMvdGFza3MucHkiLCBsaW5lIDMxNiwgaW4gY29uc3VtZV9maWxlCiAgICBkb2N1bWVudCA9IENvbnN1bWVyKCkudHJ5X2NvbnN1bWVfZmlsZSgKICBGaWxlICIvVXNlcnMvbW9vbmVyL0RvY3VtZW50cy9Xb3JrL0Rldi5ub3N5bmMvQ29udHJpYnV0aW9ucy9wYXBlcmxlc3Mtbmd4L3NyYy9kb2N1bWVudHMvY29uc3VtZXIucHkiLCBsaW5lIDIxOCwgaW4gdHJ5X2NvbnN1bWVfZmlsZQogICAgc2VsZi5wcmVfY2hlY2tfZHVwbGljYXRlKCkKICBGaWxlICIvVXNlcnMvbW9vbmVyL0RvY3VtZW50cy9Xb3JrL0Rldi5ub3N5bmMvQ29udHJpYnV0aW9ucy9wYXBlcmxlc3Mtbmd4L3NyYy9kb2N1bWVudHMvY29uc3VtZXIucHkiLCBsaW5lIDExMywgaW4gcHJlX2NoZWNrX2R1cGxpY2F0ZQogICAgc2VsZi5fZmFpbCgKICBGaWxlICIvVXNlcnMvbW9vbmVyL0RvY3VtZW50cy9Xb3JrL0Rldi5ub3N5bmMvQ29udHJpYnV0aW9ucy9wYXBlcmxlc3Mtbmd4L3NyYy9kb2N1bWVudHMvY29uc3VtZXIucHkiLCBsaW5lIDg0LCBpbiBfZmFpbAogICAgcmFpc2UgQ29uc3VtZXJFcnJvcihmIntzZWxmLmZpbGVuYW1lfToge2xvZ19tZXNzYWdlIG9yIG1lc3NhZ2V9IikKZG9jdW1lbnRzLmNvbnN1bWVyLkNvbnN1bWVyRXJyb3I6IHNhbXBsZSAyLnBkZjogTm90IGNvbnN1bWluZyBzYW1wbGUgMi5wZGY6IEl0IGlzIGEgZHVwbGljYXRlLgqULg==", + "group": null, + "started": "2022-05-26T23:17:37.702432-07:00", + "stopped": "2022-05-26T23:17:38.313306-07:00", + "success": false, + "attempt_count": 1 + } + }, + { + "id": 132, + "type": "file", + "result": " : Traceback (most recent call last):\n File \"/Users/admin/.local/share/virtualenvs/paperless-ng/lib/python3.6/site-packages/ocrmypdf/subprocess.py\", line 131, in get_version\n env=env,\n File \"/Users/admin/.local/share/virtualenvs/paperless-ng/lib/python3.6/site-packages/ocrmypdf/subprocess.py\", line 68, in run\n proc = subprocess_run(args, env=env, **kwargs)\n File \"/Users/admin/opt/anaconda3/envs/paperless-ng/lib/python3.6/subprocess.py\", line 423, in run\n with Popen(*popenargs, **kwargs) as process:\n File \"/Users/admin/opt/anaconda3/envs/paperless-ng/lib/python3.6/subprocess.py\", line 729, in __init__\n restore_signals, start_new_session)\n File \"/Users/admin/opt/anaconda3/envs/paperless-ng/lib/python3.6/subprocess.py\", line 1364, in _execute_child\n raise child_exception_type(errno_num, err_msg, err_filename)\nFileNotFoundError: [Errno 2] No such file or directory: 'unpaper': 'unpaper'\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/Users/admin/.local/share/virtualenvs/paperless-ng/lib/python3.6/site-packages/ocrmypdf/subprocess.py\", line 287, in check_external_program\n found_version = version_checker()\n File \"/Users/admin/.local/share/virtualenvs/paperless-ng/lib/python3.6/site-packages/ocrmypdf/_exec/unpaper.py\", line 34, in version\n return get_version('unpaper')\n File \"/Users/admin/.local/share/virtualenvs/paperless-ng/lib/python3.6/site-packages/ocrmypdf/subprocess.py\", line 137, in get_version\n ) from e\nocrmypdf.exceptions.MissingDependencyError: Could not find program 'unpaper' on the PATH\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n File \"/Users/admin/Documents/Work/Contributions/paperless-ng/src/paperless_tesseract/parsers.py\", line 176, in parse\n ocrmypdf.ocr(**ocr_args)\n File \"/Users/admin/.local/share/virtualenvs/paperless-ng/lib/python3.6/site-packages/ocrmypdf/api.py\", line 315, in ocr\n check_options(options, plugin_manager)\n File \"/Users/admin/.local/share/virtualenvs/paperless-ng/lib/python3.6/site-packages/ocrmypdf/_validation.py\", line 260, in check_options\n _check_options(options, plugin_manager, ocr_engine_languages)\n File \"/Users/admin/.local/share/virtualenvs/paperless-ng/lib/python3.6/site-packages/ocrmypdf/_validation.py\", line 250, in _check_options\n check_options_preprocessing(options)\n File \"/Users/admin/.local/share/virtualenvs/paperless-ng/lib/python3.6/site-packages/ocrmypdf/_validation.py\", line 128, in check_options_preprocessing\n required_for=['--clean, --clean-final'],\n File \"/Users/admin/.local/share/virtualenvs/paperless-ng/lib/python3.6/site-packages/ocrmypdf/subprocess.py\", line 293, in check_external_program\n raise MissingDependencyError()\nocrmypdf.exceptions.MissingDependencyError\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n File \"/Users/admin/Documents/Work/Contributions/paperless-ng/src/documents/consumer.py\", line 179, in try_consume_file\n document_parser.parse(self.path, mime_type, self.filename)\n File \"/Users/admin/Documents/Work/Contributions/paperless-ng/src/paperless_tesseract/parsers.py\", line 197, in parse\n raise ParseError(e)\ndocuments.parsers.ParseError\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n File \"/Users/admin/.local/share/virtualenvs/paperless-ng/lib/python3.6/site-packages/django_q/cluster.py\", line 436, in worker\n res = f(*task[\"args\"], **task[\"kwargs\"])\n File \"/Users/admin/Documents/Work/Contributions/paperless-ng/src/documents/tasks.py\", line 73, in consume_file\n override_tag_ids=override_tag_ids)\n File \"/Users/admin/Documents/Work/Contributions/paperless-ng/src/documents/consumer.py\", line 196, in try_consume_file\n raise ConsumerError(e)\ndocuments.consumer.ConsumerError\n", + "status": "FAILURE", + "task_id": "4c554075552c4cc985abd76e6f274c90", + "name": "pdf-sample 10.24.48 PM.pdf", + "created": "2022-05-26T14:26:07.846365-07:00", + "acknowledged": null, + "attempted_task": { + "id": "4c554075552c4cc985abd76e6f274c90", + "name": "pdf-sample 10.24.48 PM.pdf", + "func": "documents.tasks.consume_file", + "hook": null, + "args": "gAWVKwAAAAAAAACMJS4uL2NvbnN1bWUvcGRmLXNhbXBsZSAxMC4yNC40OCBQTS5wZGaUhZQu", + "kwargs": "gAWVGAAAAAAAAAB9lIwQb3ZlcnJpZGVfdGFnX2lkc5ROcy4=", + "result": "gAWVzA8AAAAAAABYxQ8AACA6IFRyYWNlYmFjayAobW9zdCByZWNlbnQgY2FsbCBsYXN0KToKICBGaWxlICIvVXNlcnMvbW9vbmVyLy5sb2NhbC9zaGFyZS92aXJ0dWFsZW52cy9wYXBlcmxlc3MtbmctNzZCdUpsRUkvbGliL3B5dGhvbjMuNi9zaXRlLXBhY2thZ2VzL29jcm15cGRmL3N1YnByb2Nlc3MucHkiLCBsaW5lIDEzMSwgaW4gZ2V0X3ZlcnNpb24KICAgIGVudj1lbnYsCiAgRmlsZSAiL1VzZXJzL21vb25lci8ubG9jYWwvc2hhcmUvdmlydHVhbGVudnMvcGFwZXJsZXNzLW5nLTc2QnVKbEVJL2xpYi9weXRob24zLjYvc2l0ZS1wYWNrYWdlcy9vY3JteXBkZi9zdWJwcm9jZXNzLnB5IiwgbGluZSA2OCwgaW4gcnVuCiAgICBwcm9jID0gc3VicHJvY2Vzc19ydW4oYXJncywgZW52PWVudiwgKiprd2FyZ3MpCiAgRmlsZSAiL1VzZXJzL21vb25lci9vcHQvYW5hY29uZGEzL2VudnMvcGFwZXJsZXNzLW5nL2xpYi9weXRob24zLjYvc3VicHJvY2Vzcy5weSIsIGxpbmUgNDIzLCBpbiBydW4KICAgIHdpdGggUG9wZW4oKnBvcGVuYXJncywgKiprd2FyZ3MpIGFzIHByb2Nlc3M6CiAgRmlsZSAiL1VzZXJzL21vb25lci9vcHQvYW5hY29uZGEzL2VudnMvcGFwZXJsZXNzLW5nL2xpYi9weXRob24zLjYvc3VicHJvY2Vzcy5weSIsIGxpbmUgNzI5LCBpbiBfX2luaXRfXwogICAgcmVzdG9yZV9zaWduYWxzLCBzdGFydF9uZXdfc2Vzc2lvbikKICBGaWxlICIvVXNlcnMvbW9vbmVyL29wdC9hbmFjb25kYTMvZW52cy9wYXBlcmxlc3MtbmcvbGliL3B5dGhvbjMuNi9zdWJwcm9jZXNzLnB5IiwgbGluZSAxMzY0LCBpbiBfZXhlY3V0ZV9jaGlsZAogICAgcmFpc2UgY2hpbGRfZXhjZXB0aW9uX3R5cGUoZXJybm9fbnVtLCBlcnJfbXNnLCBlcnJfZmlsZW5hbWUpCkZpbGVOb3RGb3VuZEVycm9yOiBbRXJybm8gMl0gTm8gc3VjaCBmaWxlIG9yIGRpcmVjdG9yeTogJ3VucGFwZXInOiAndW5wYXBlcicKClRoZSBhYm92ZSBleGNlcHRpb24gd2FzIHRoZSBkaXJlY3QgY2F1c2Ugb2YgdGhlIGZvbGxvd2luZyBleGNlcHRpb246CgpUcmFjZWJhY2sgKG1vc3QgcmVjZW50IGNhbGwgbGFzdCk6CiAgRmlsZSAiL1VzZXJzL21vb25lci8ubG9jYWwvc2hhcmUvdmlydHVhbGVudnMvcGFwZXJsZXNzLW5nLTc2QnVKbEVJL2xpYi9weXRob24zLjYvc2l0ZS1wYWNrYWdlcy9vY3JteXBkZi9zdWJwcm9jZXNzLnB5IiwgbGluZSAyODcsIGluIGNoZWNrX2V4dGVybmFsX3Byb2dyYW0KICAgIGZvdW5kX3ZlcnNpb24gPSB2ZXJzaW9uX2NoZWNrZXIoKQogIEZpbGUgIi9Vc2Vycy9tb29uZXIvLmxvY2FsL3NoYXJlL3ZpcnR1YWxlbnZzL3BhcGVybGVzcy1uZy03NkJ1SmxFSS9saWIvcHl0aG9uMy42L3NpdGUtcGFja2FnZXMvb2NybXlwZGYvX2V4ZWMvdW5wYXBlci5weSIsIGxpbmUgMzQsIGluIHZlcnNpb24KICAgIHJldHVybiBnZXRfdmVyc2lvbigndW5wYXBlcicpCiAgRmlsZSAiL1VzZXJzL21vb25lci8ubG9jYWwvc2hhcmUvdmlydHVhbGVudnMvcGFwZXJsZXNzLW5nLTc2QnVKbEVJL2xpYi9weXRob24zLjYvc2l0ZS1wYWNrYWdlcy9vY3JteXBkZi9zdWJwcm9jZXNzLnB5IiwgbGluZSAxMzcsIGluIGdldF92ZXJzaW9uCiAgICApIGZyb20gZQpvY3JteXBkZi5leGNlcHRpb25zLk1pc3NpbmdEZXBlbmRlbmN5RXJyb3I6IENvdWxkIG5vdCBmaW5kIHByb2dyYW0gJ3VucGFwZXInIG9uIHRoZSBQQVRICgpEdXJpbmcgaGFuZGxpbmcgb2YgdGhlIGFib3ZlIGV4Y2VwdGlvbiwgYW5vdGhlciBleGNlcHRpb24gb2NjdXJyZWQ6CgpUcmFjZWJhY2sgKG1vc3QgcmVjZW50IGNhbGwgbGFzdCk6CiAgRmlsZSAiL1VzZXJzL21vb25lci9Eb2N1bWVudHMvV29yay9Db250cmlidXRpb25zL3BhcGVybGVzcy1uZy9zcmMvcGFwZXJsZXNzX3Rlc3NlcmFjdC9wYXJzZXJzLnB5IiwgbGluZSAxNzYsIGluIHBhcnNlCiAgICBvY3JteXBkZi5vY3IoKipvY3JfYXJncykKICBGaWxlICIvVXNlcnMvbW9vbmVyLy5sb2NhbC9zaGFyZS92aXJ0dWFsZW52cy9wYXBlcmxlc3MtbmctNzZCdUpsRUkvbGliL3B5dGhvbjMuNi9zaXRlLXBhY2thZ2VzL29jcm15cGRmL2FwaS5weSIsIGxpbmUgMzE1LCBpbiBvY3IKICAgIGNoZWNrX29wdGlvbnMob3B0aW9ucywgcGx1Z2luX21hbmFnZXIpCiAgRmlsZSAiL1VzZXJzL21vb25lci8ubG9jYWwvc2hhcmUvdmlydHVhbGVudnMvcGFwZXJsZXNzLW5nLTc2QnVKbEVJL2xpYi9weXRob24zLjYvc2l0ZS1wYWNrYWdlcy9vY3JteXBkZi9fdmFsaWRhdGlvbi5weSIsIGxpbmUgMjYwLCBpbiBjaGVja19vcHRpb25zCiAgICBfY2hlY2tfb3B0aW9ucyhvcHRpb25zLCBwbHVnaW5fbWFuYWdlciwgb2NyX2VuZ2luZV9sYW5ndWFnZXMpCiAgRmlsZSAiL1VzZXJzL21vb25lci8ubG9jYWwvc2hhcmUvdmlydHVhbGVudnMvcGFwZXJsZXNzLW5nLTc2QnVKbEVJL2xpYi9weXRob24zLjYvc2l0ZS1wYWNrYWdlcy9vY3JteXBkZi9fdmFsaWRhdGlvbi5weSIsIGxpbmUgMjUwLCBpbiBfY2hlY2tfb3B0aW9ucwogICAgY2hlY2tfb3B0aW9uc19wcmVwcm9jZXNzaW5nKG9wdGlvbnMpCiAgRmlsZSAiL1VzZXJzL21vb25lci8ubG9jYWwvc2hhcmUvdmlydHVhbGVudnMvcGFwZXJsZXNzLW5nLTc2QnVKbEVJL2xpYi9weXRob24zLjYvc2l0ZS1wYWNrYWdlcy9vY3JteXBkZi9fdmFsaWRhdGlvbi5weSIsIGxpbmUgMTI4LCBpbiBjaGVja19vcHRpb25zX3ByZXByb2Nlc3NpbmcKICAgIHJlcXVpcmVkX2Zvcj1bJy0tY2xlYW4sIC0tY2xlYW4tZmluYWwnXSwKICBGaWxlICIvVXNlcnMvbW9vbmVyLy5sb2NhbC9zaGFyZS92aXJ0dWFsZW52cy9wYXBlcmxlc3MtbmctNzZCdUpsRUkvbGliL3B5dGhvbjMuNi9zaXRlLXBhY2thZ2VzL29jcm15cGRmL3N1YnByb2Nlc3MucHkiLCBsaW5lIDI5MywgaW4gY2hlY2tfZXh0ZXJuYWxfcHJvZ3JhbQogICAgcmFpc2UgTWlzc2luZ0RlcGVuZGVuY3lFcnJvcigpCm9jcm15cGRmLmV4Y2VwdGlvbnMuTWlzc2luZ0RlcGVuZGVuY3lFcnJvcgoKRHVyaW5nIGhhbmRsaW5nIG9mIHRoZSBhYm92ZSBleGNlcHRpb24sIGFub3RoZXIgZXhjZXB0aW9uIG9jY3VycmVkOgoKVHJhY2ViYWNrIChtb3N0IHJlY2VudCBjYWxsIGxhc3QpOgogIEZpbGUgIi9Vc2Vycy9tb29uZXIvRG9jdW1lbnRzL1dvcmsvQ29udHJpYnV0aW9ucy9wYXBlcmxlc3Mtbmcvc3JjL2RvY3VtZW50cy9jb25zdW1lci5weSIsIGxpbmUgMTc5LCBpbiB0cnlfY29uc3VtZV9maWxlCiAgICBkb2N1bWVudF9wYXJzZXIucGFyc2Uoc2VsZi5wYXRoLCBtaW1lX3R5cGUsIHNlbGYuZmlsZW5hbWUpCiAgRmlsZSAiL1VzZXJzL21vb25lci9Eb2N1bWVudHMvV29yay9Db250cmlidXRpb25zL3BhcGVybGVzcy1uZy9zcmMvcGFwZXJsZXNzX3Rlc3NlcmFjdC9wYXJzZXJzLnB5IiwgbGluZSAxOTcsIGluIHBhcnNlCiAgICByYWlzZSBQYXJzZUVycm9yKGUpCmRvY3VtZW50cy5wYXJzZXJzLlBhcnNlRXJyb3IKCkR1cmluZyBoYW5kbGluZyBvZiB0aGUgYWJvdmUgZXhjZXB0aW9uLCBhbm90aGVyIGV4Y2VwdGlvbiBvY2N1cnJlZDoKClRyYWNlYmFjayAobW9zdCByZWNlbnQgY2FsbCBsYXN0KToKICBGaWxlICIvVXNlcnMvbW9vbmVyLy5sb2NhbC9zaGFyZS92aXJ0dWFsZW52cy9wYXBlcmxlc3MtbmctNzZCdUpsRUkvbGliL3B5dGhvbjMuNi9zaXRlLXBhY2thZ2VzL2RqYW5nb19xL2NsdXN0ZXIucHkiLCBsaW5lIDQzNiwgaW4gd29ya2VyCiAgICByZXMgPSBmKCp0YXNrWyJhcmdzIl0sICoqdGFza1sia3dhcmdzIl0pCiAgRmlsZSAiL1VzZXJzL21vb25lci9Eb2N1bWVudHMvV29yay9Db250cmlidXRpb25zL3BhcGVybGVzcy1uZy9zcmMvZG9jdW1lbnRzL3Rhc2tzLnB5IiwgbGluZSA3MywgaW4gY29uc3VtZV9maWxlCiAgICBvdmVycmlkZV90YWdfaWRzPW92ZXJyaWRlX3RhZ19pZHMpCiAgRmlsZSAiL1VzZXJzL21vb25lci9Eb2N1bWVudHMvV29yay9Db250cmlidXRpb25zL3BhcGVybGVzcy1uZy9zcmMvZG9jdW1lbnRzL2NvbnN1bWVyLnB5IiwgbGluZSAxOTYsIGluIHRyeV9jb25zdW1lX2ZpbGUKICAgIHJhaXNlIENvbnN1bWVyRXJyb3IoZSkKZG9jdW1lbnRzLmNvbnN1bWVyLkNvbnN1bWVyRXJyb3IKlC4=", + "group": null, + "started": "2021-01-20T10:47:34.535478-08:00", + "stopped": "2021-01-20T10:49:55.568010-08:00", + "success": false, + "attempt_count": 1 + } + }, + { + "id": 115, + "type": "file", + "result": "2021-01-24 2021-01-20 sample_wide_orange.pdf: Document is a duplicate : Traceback (most recent call last):\n File \"/Users/admin/.local/share/virtualenvs/paperless-ng/lib/python3.6/site-packages/django_q/cluster.py\", line 436, in worker\n res = f(*task[\"args\"], **task[\"kwargs\"])\n File \"/Users/admin/Documents/Work/Contributions/paperless-ng/src/documents/tasks.py\", line 75, in consume_file\n task_id=task_id\n File \"/Users/admin/Documents/Work/Contributions/paperless-ng/src/documents/consumer.py\", line 168, in try_consume_file\n self.pre_check_duplicate()\n File \"/Users/admin/Documents/Work/Contributions/paperless-ng/src/documents/consumer.py\", line 85, in pre_check_duplicate\n self._fail(\"Document is a duplicate\")\n File \"/Users/admin/Documents/Work/Contributions/paperless-ng/src/documents/consumer.py\", line 53, in _fail\n raise ConsumerError(f\"{self.filename}: {message}\")\ndocuments.consumer.ConsumerError: 2021-01-24 2021-01-20 sample_wide_orange.pdf: Document is a duplicate\n", + "status": "FAILURE", + "task_id": "86494713646a4364b01da17aadca071d", + "name": "2021-01-24 2021-01-20 sample_wide_orange.pdf", + "created": "2022-05-26T14:26:07.817608-07:00", + "acknowledged": null, + "attempted_task": { + "id": "86494713646a4364b01da17aadca071d", + "name": "2021-01-24 2021-01-20 sample_wide_orange.pdf", + "func": "documents.tasks.consume_file", + "hook": null, + "args": "gAWVLgAAAAAAAACMKC90bXAvcGFwZXJsZXNzL3BhcGVybGVzcy11cGxvYWQtcTJ6NDlnbzaUhZQu", + "kwargs": "gAWV2QAAAAAAAAB9lCiMEW92ZXJyaWRlX2ZpbGVuYW1llIwsMjAyMS0wMS0yNCAyMDIxLTAxLTIwIHNhbXBsZV93aWRlX29yYW5nZS5wZGaUjA5vdmVycmlkZV90aXRsZZROjBlvdmVycmlkZV9jb3JyZXNwb25kZW50X2lklE6MGW92ZXJyaWRlX2RvY3VtZW50X3R5cGVfaWSUTowQb3ZlcnJpZGVfdGFnX2lkc5ROjAd0YXNrX2lklIwkN2MwZTY1MmQtZDhkYy00OWU4LWI1ZmUtOGM3ZTkyZDlmOTI0lHUu", + "result": "gAWV/AMAAAAAAABY9QMAADIwMjEtMDEtMjQgMjAyMS0wMS0yMCBzYW1wbGVfd2lkZV9vcmFuZ2UucGRmOiBEb2N1bWVudCBpcyBhIGR1cGxpY2F0ZSA6IFRyYWNlYmFjayAobW9zdCByZWNlbnQgY2FsbCBsYXN0KToKICBGaWxlICIvVXNlcnMvbW9vbmVyLy5sb2NhbC9zaGFyZS92aXJ0dWFsZW52cy9wYXBlcmxlc3MtbmctNzZCdUpsRUkvbGliL3B5dGhvbjMuNi9zaXRlLXBhY2thZ2VzL2RqYW5nb19xL2NsdXN0ZXIucHkiLCBsaW5lIDQzNiwgaW4gd29ya2VyCiAgICByZXMgPSBmKCp0YXNrWyJhcmdzIl0sICoqdGFza1sia3dhcmdzIl0pCiAgRmlsZSAiL1VzZXJzL21vb25lci9Eb2N1bWVudHMvV29yay9Db250cmlidXRpb25zL3BhcGVybGVzcy1uZy9zcmMvZG9jdW1lbnRzL3Rhc2tzLnB5IiwgbGluZSA3NSwgaW4gY29uc3VtZV9maWxlCiAgICB0YXNrX2lkPXRhc2tfaWQKICBGaWxlICIvVXNlcnMvbW9vbmVyL0RvY3VtZW50cy9Xb3JrL0NvbnRyaWJ1dGlvbnMvcGFwZXJsZXNzLW5nL3NyYy9kb2N1bWVudHMvY29uc3VtZXIucHkiLCBsaW5lIDE2OCwgaW4gdHJ5X2NvbnN1bWVfZmlsZQogICAgc2VsZi5wcmVfY2hlY2tfZHVwbGljYXRlKCkKICBGaWxlICIvVXNlcnMvbW9vbmVyL0RvY3VtZW50cy9Xb3JrL0NvbnRyaWJ1dGlvbnMvcGFwZXJsZXNzLW5nL3NyYy9kb2N1bWVudHMvY29uc3VtZXIucHkiLCBsaW5lIDg1LCBpbiBwcmVfY2hlY2tfZHVwbGljYXRlCiAgICBzZWxmLl9mYWlsKCJEb2N1bWVudCBpcyBhIGR1cGxpY2F0ZSIpCiAgRmlsZSAiL1VzZXJzL21vb25lci9Eb2N1bWVudHMvV29yay9Db250cmlidXRpb25zL3BhcGVybGVzcy1uZy9zcmMvZG9jdW1lbnRzL2NvbnN1bWVyLnB5IiwgbGluZSA1MywgaW4gX2ZhaWwKICAgIHJhaXNlIENvbnN1bWVyRXJyb3IoZiJ7c2VsZi5maWxlbmFtZX06IHttZXNzYWdlfSIpCmRvY3VtZW50cy5jb25zdW1lci5Db25zdW1lckVycm9yOiAyMDIxLTAxLTI0IDIwMjEtMDEtMjAgc2FtcGxlX3dpZGVfb3JhbmdlLnBkZjogRG9jdW1lbnQgaXMgYSBkdXBsaWNhdGUKlC4=", + "group": null, + "started": "2021-01-26T00:21:05.379583-08:00", + "stopped": "2021-01-26T00:21:06.449626-08:00", + "success": false, + "attempt_count": 1 + } + }, + { + "id": 85, + "type": "file", + "result": "cannot open resource : Traceback (most recent call last):\n File \"/Users/admin/.local/share/virtualenvs/paperless-ng/lib/python3.6/site-packages/django_q/cluster.py\", line 436, in worker\n res = f(*task[\"args\"], **task[\"kwargs\"])\n File \"/Users/admin/Documents/Work/Contributions/paperless-ng/src/documents/tasks.py\", line 81, in consume_file\n task_id=task_id\n File \"/Users/admin/Documents/Work/Contributions/paperless-ng/src/documents/consumer.py\", line 244, in try_consume_file\n self.path, mime_type, self.filename)\n File \"/Users/admin/Documents/Work/Contributions/paperless-ng/src/documents/parsers.py\", line 302, in get_optimised_thumbnail\n thumbnail = self.get_thumbnail(document_path, mime_type, file_name)\n File \"/Users/admin/Documents/Work/Contributions/paperless-ng/src/paperless_text/parsers.py\", line 29, in get_thumbnail\n layout_engine=ImageFont.LAYOUT_BASIC)\n File \"/Users/admin/.local/share/virtualenvs/paperless-ng/lib/python3.6/site-packages/PIL/ImageFont.py\", line 852, in truetype\n return freetype(font)\n File \"/Users/admin/.local/share/virtualenvs/paperless-ng/lib/python3.6/site-packages/PIL/ImageFont.py\", line 849, in freetype\n return FreeTypeFont(font, size, index, encoding, layout_engine)\n File \"/Users/admin/.local/share/virtualenvs/paperless-ng/lib/python3.6/site-packages/PIL/ImageFont.py\", line 210, in __init__\n font, size, index, encoding, layout_engine=layout_engine\nOSError: cannot open resource\n", + "status": "FAILURE", + "task_id": "abca803fa46342e1ac81f3d3f2080e79", + "name": "simple.txt", + "created": "2022-05-26T14:26:07.771541-07:00", + "acknowledged": null, + "attempted_task": { + "id": "abca803fa46342e1ac81f3d3f2080e79", + "name": "simple.txt", + "func": "documents.tasks.consume_file", + "hook": null, + "args": "gAWVLgAAAAAAAACMKC90bXAvcGFwZXJsZXNzL3BhcGVybGVzcy11cGxvYWQtd2RhbnB5NnGUhZQu", + "kwargs": "gAWVtwAAAAAAAAB9lCiMEW92ZXJyaWRlX2ZpbGVuYW1llIwKc2ltcGxlLnR4dJSMDm92ZXJyaWRlX3RpdGxllE6MGW92ZXJyaWRlX2NvcnJlc3BvbmRlbnRfaWSUTowZb3ZlcnJpZGVfZG9jdW1lbnRfdHlwZV9pZJROjBBvdmVycmlkZV90YWdfaWRzlE6MB3Rhc2tfaWSUjCQ3ZGE0OTU4ZC0zM2UwLTQ1OGMtYTE0ZC1kMmU0NmE0NWY4Y2SUdS4=", + "result": "gAWV5QUAAAAAAABY3gUAAGNhbm5vdCBvcGVuIHJlc291cmNlIDogVHJhY2ViYWNrIChtb3N0IHJlY2VudCBjYWxsIGxhc3QpOgogIEZpbGUgIi9Vc2Vycy9tb29uZXIvLmxvY2FsL3NoYXJlL3ZpcnR1YWxlbnZzL3BhcGVybGVzcy1uZy03NkJ1SmxFSS9saWIvcHl0aG9uMy42L3NpdGUtcGFja2FnZXMvZGphbmdvX3EvY2x1c3Rlci5weSIsIGxpbmUgNDM2LCBpbiB3b3JrZXIKICAgIHJlcyA9IGYoKnRhc2tbImFyZ3MiXSwgKip0YXNrWyJrd2FyZ3MiXSkKICBGaWxlICIvVXNlcnMvbW9vbmVyL0RvY3VtZW50cy9Xb3JrL0NvbnRyaWJ1dGlvbnMvcGFwZXJsZXNzLW5nL3NyYy9kb2N1bWVudHMvdGFza3MucHkiLCBsaW5lIDgxLCBpbiBjb25zdW1lX2ZpbGUKICAgIHRhc2tfaWQ9dGFza19pZAogIEZpbGUgIi9Vc2Vycy9tb29uZXIvRG9jdW1lbnRzL1dvcmsvQ29udHJpYnV0aW9ucy9wYXBlcmxlc3Mtbmcvc3JjL2RvY3VtZW50cy9jb25zdW1lci5weSIsIGxpbmUgMjQ0LCBpbiB0cnlfY29uc3VtZV9maWxlCiAgICBzZWxmLnBhdGgsIG1pbWVfdHlwZSwgc2VsZi5maWxlbmFtZSkKICBGaWxlICIvVXNlcnMvbW9vbmVyL0RvY3VtZW50cy9Xb3JrL0NvbnRyaWJ1dGlvbnMvcGFwZXJsZXNzLW5nL3NyYy9kb2N1bWVudHMvcGFyc2Vycy5weSIsIGxpbmUgMzAyLCBpbiBnZXRfb3B0aW1pc2VkX3RodW1ibmFpbAogICAgdGh1bWJuYWlsID0gc2VsZi5nZXRfdGh1bWJuYWlsKGRvY3VtZW50X3BhdGgsIG1pbWVfdHlwZSwgZmlsZV9uYW1lKQogIEZpbGUgIi9Vc2Vycy9tb29uZXIvRG9jdW1lbnRzL1dvcmsvQ29udHJpYnV0aW9ucy9wYXBlcmxlc3Mtbmcvc3JjL3BhcGVybGVzc190ZXh0L3BhcnNlcnMucHkiLCBsaW5lIDI5LCBpbiBnZXRfdGh1bWJuYWlsCiAgICBsYXlvdXRfZW5naW5lPUltYWdlRm9udC5MQVlPVVRfQkFTSUMpCiAgRmlsZSAiL1VzZXJzL21vb25lci8ubG9jYWwvc2hhcmUvdmlydHVhbGVudnMvcGFwZXJsZXNzLW5nLTc2QnVKbEVJL2xpYi9weXRob24zLjYvc2l0ZS1wYWNrYWdlcy9QSUwvSW1hZ2VGb250LnB5IiwgbGluZSA4NTIsIGluIHRydWV0eXBlCiAgICByZXR1cm4gZnJlZXR5cGUoZm9udCkKICBGaWxlICIvVXNlcnMvbW9vbmVyLy5sb2NhbC9zaGFyZS92aXJ0dWFsZW52cy9wYXBlcmxlc3MtbmctNzZCdUpsRUkvbGliL3B5dGhvbjMuNi9zaXRlLXBhY2thZ2VzL1BJTC9JbWFnZUZvbnQucHkiLCBsaW5lIDg0OSwgaW4gZnJlZXR5cGUKICAgIHJldHVybiBGcmVlVHlwZUZvbnQoZm9udCwgc2l6ZSwgaW5kZXgsIGVuY29kaW5nLCBsYXlvdXRfZW5naW5lKQogIEZpbGUgIi9Vc2Vycy9tb29uZXIvLmxvY2FsL3NoYXJlL3ZpcnR1YWxlbnZzL3BhcGVybGVzcy1uZy03NkJ1SmxFSS9saWIvcHl0aG9uMy42L3NpdGUtcGFja2FnZXMvUElML0ltYWdlRm9udC5weSIsIGxpbmUgMjEwLCBpbiBfX2luaXRfXwogICAgZm9udCwgc2l6ZSwgaW5kZXgsIGVuY29kaW5nLCBsYXlvdXRfZW5naW5lPWxheW91dF9lbmdpbmUKT1NFcnJvcjogY2Fubm90IG9wZW4gcmVzb3VyY2UKlC4=", + "group": null, + "started": "2021-03-06T14:23:56.974715-08:00", + "stopped": "2021-03-06T14:24:28.011772-08:00", + "success": false, + "attempt_count": 1 + } + }, + { + "id": 41, + "type": "file", + "result": "commands.txt: Not consuming commands.txt: It is a duplicate. : Traceback (most recent call last):\n File \"/Users/admin/.local/share/virtualenvs/paperless-ngx.nosync-udqDZzaE/lib/python3.8/site-packages/django_q/cluster.py\", line 432, in worker\n res = f(*task[\"args\"], **task[\"kwargs\"])\n File \"/Users/admin/Documents/paperless-ngx/src/documents/tasks.py\", line 70, in consume_file\n document = Consumer().try_consume_file(\n File \"/Users/admin/Documents/paperless-ngx/src/documents/consumer.py\", line 199, in try_consume_file\n self.pre_check_duplicate()\n File \"/Users/admin/Documents/paperless-ngx/src/documents/consumer.py\", line 97, in pre_check_duplicate\n self._fail(\n File \"/Users/admin/Documents/paperless-ngx/src/documents/consumer.py\", line 69, in _fail\n raise ConsumerError(f\"{self.filename}: {log_message or message}\")\ndocuments.consumer.ConsumerError: commands.txt: Not consuming commands.txt: It is a duplicate.\n", + "status": "FAILURE", + "task_id": "0af67672e8e14404b060d4cf8f69313d", + "name": "commands.txt", + "created": "2022-05-26T14:26:07.704247-07:00", + "acknowledged": null, + "attempted_task": { + "id": "0af67672e8e14404b060d4cf8f69313d", + "name": "commands.txt", + "func": "documents.tasks.consume_file", + "hook": null, + "args": "gAWVLgAAAAAAAACMKC90bXAvcGFwZXJsZXNzL3BhcGVybGVzcy11cGxvYWQtZ3h4YjNxODaUhZQu", + "kwargs": "gAWVuQAAAAAAAAB9lCiMEW92ZXJyaWRlX2ZpbGVuYW1llIwMY29tbWFuZHMudHh0lIwOb3ZlcnJpZGVfdGl0bGWUTowZb3ZlcnJpZGVfY29ycmVzcG9uZGVudF9pZJROjBlvdmVycmlkZV9kb2N1bWVudF90eXBlX2lklE6MEG92ZXJyaWRlX3RhZ19pZHOUTowHdGFza19pZJSMJDRkMjhmMmJiLTJkMzAtNGQzNi1iNjM5LWU2YzQ5OTU3OGVlY5R1Lg==", + "result": "gAWVLwQAAAAAAABYKAQAAGNvbW1hbmRzLnR4dDogTm90IGNvbnN1bWluZyBjb21tYW5kcy50eHQ6IEl0IGlzIGEgZHVwbGljYXRlLiA6IFRyYWNlYmFjayAobW9zdCByZWNlbnQgY2FsbCBsYXN0KToKICBGaWxlICIvVXNlcnMvbW9vbmVyLy5sb2NhbC9zaGFyZS92aXJ0dWFsZW52cy9wYXBlcmxlc3Mtbmd4Lm5vc3luYy11ZHFEWnphRS9saWIvcHl0aG9uMy44L3NpdGUtcGFja2FnZXMvZGphbmdvX3EvY2x1c3Rlci5weSIsIGxpbmUgNDMyLCBpbiB3b3JrZXIKICAgIHJlcyA9IGYoKnRhc2tbImFyZ3MiXSwgKip0YXNrWyJrd2FyZ3MiXSkKICBGaWxlICIvVXNlcnMvbW9vbmVyL0RvY3VtZW50cy9Xb3JrL0Rldi5ub3N5bmMvQ29udHJpYnV0aW9ucy9wYXBlcmxlc3Mtbmd4L3NyYy9kb2N1bWVudHMvdGFza3MucHkiLCBsaW5lIDcwLCBpbiBjb25zdW1lX2ZpbGUKICAgIGRvY3VtZW50ID0gQ29uc3VtZXIoKS50cnlfY29uc3VtZV9maWxlKAogIEZpbGUgIi9Vc2Vycy9tb29uZXIvRG9jdW1lbnRzL1dvcmsvRGV2Lm5vc3luYy9Db250cmlidXRpb25zL3BhcGVybGVzcy1uZ3gvc3JjL2RvY3VtZW50cy9jb25zdW1lci5weSIsIGxpbmUgMTk5LCBpbiB0cnlfY29uc3VtZV9maWxlCiAgICBzZWxmLnByZV9jaGVja19kdXBsaWNhdGUoKQogIEZpbGUgIi9Vc2Vycy9tb29uZXIvRG9jdW1lbnRzL1dvcmsvRGV2Lm5vc3luYy9Db250cmlidXRpb25zL3BhcGVybGVzcy1uZ3gvc3JjL2RvY3VtZW50cy9jb25zdW1lci5weSIsIGxpbmUgOTcsIGluIHByZV9jaGVja19kdXBsaWNhdGUKICAgIHNlbGYuX2ZhaWwoCiAgRmlsZSAiL1VzZXJzL21vb25lci9Eb2N1bWVudHMvV29yay9EZXYubm9zeW5jL0NvbnRyaWJ1dGlvbnMvcGFwZXJsZXNzLW5neC9zcmMvZG9jdW1lbnRzL2NvbnN1bWVyLnB5IiwgbGluZSA2OSwgaW4gX2ZhaWwKICAgIHJhaXNlIENvbnN1bWVyRXJyb3IoZiJ7c2VsZi5maWxlbmFtZX06IHtsb2dfbWVzc2FnZSBvciBtZXNzYWdlfSIpCmRvY3VtZW50cy5jb25zdW1lci5Db25zdW1lckVycm9yOiBjb21tYW5kcy50eHQ6IE5vdCBjb25zdW1pbmcgY29tbWFuZHMudHh0OiBJdCBpcyBhIGR1cGxpY2F0ZS4KlC4=", + "group": null, + "started": "2022-03-10T22:26:32.548772-08:00", + "stopped": "2022-03-10T22:26:32.879916-08:00", + "success": false, + "attempt_count": 1 + } + }, + { + "id": 10, + "type": "file", + "result": "Success. New document id 260 created", + "status": "SUCCESS", + "task_id": "b7629a0f41bd40c7a3ea4680341321b5", + "name": "2022-03-24+Sonstige+ScanPC2022-03-24_081058.pdf", + "created": "2022-05-26T14:26:07.670577-07:00", + "acknowledged": false, + "attempted_task": { + "id": "b7629a0f41bd40c7a3ea4680341321b5", + "name": "2022-03-24+Sonstige+ScanPC2022-03-24_081058.pdf", + "func": "documents.tasks.consume_file", + "hook": null, + "args": "gAWVLgAAAAAAAACMKC90bXAvcGFwZXJsZXNzL3BhcGVybGVzcy11cGxvYWQtc25mOW11ZW+UhZQu", + "kwargs": "gAWV3AAAAAAAAAB9lCiMEW92ZXJyaWRlX2ZpbGVuYW1llIwvMjAyMi0wMy0yNCtTb25zdGlnZStTY2FuUEMyMDIyLTAzLTI0XzA4MTA1OC5wZGaUjA5vdmVycmlkZV90aXRsZZROjBlvdmVycmlkZV9jb3JyZXNwb25kZW50X2lklE6MGW92ZXJyaWRlX2RvY3VtZW50X3R5cGVfaWSUTowQb3ZlcnJpZGVfdGFnX2lkc5ROjAd0YXNrX2lklIwkNTdmMmQwMGItY2Q0Ny00YzQ3LTlmOTctODFlOTllMTJhMjMylHUu", + "result": "gAWVKAAAAAAAAACMJFN1Y2Nlc3MuIE5ldyBkb2N1bWVudCBpZCAyNjAgY3JlYXRlZJQu", + "group": null, + "started": "2022-03-24T08:19:32.117861-07:00", + "stopped": "2022-03-24T08:20:10.239201-07:00", + "success": true, + "attempt_count": 1 + } + }, + { + "id": 9, + "type": "file", + "result": "Success. New document id 261 created", + "status": "SUCCESS", + "task_id": "02e276a86a424ccfb83309df5d8594be", + "name": "2sample-pdf-with-images.pdf", + "created": "2022-05-26T14:26:07.668987-07:00", + "acknowledged": false, + "attempted_task": { + "id": "02e276a86a424ccfb83309df5d8594be", + "name": "2sample-pdf-with-images.pdf", + "func": "documents.tasks.consume_file", + "hook": null, + "args": "gAWVLgAAAAAAAACMKC90bXAvcGFwZXJsZXNzL3BhcGVybGVzcy11cGxvYWQtaXJ3cjZzOGeUhZQu", + "kwargs": "gAWVyAAAAAAAAAB9lCiMEW92ZXJyaWRlX2ZpbGVuYW1llIwbMnNhbXBsZS1wZGYtd2l0aC1pbWFnZXMucGRmlIwOb3ZlcnJpZGVfdGl0bGWUTowZb3ZlcnJpZGVfY29ycmVzcG9uZGVudF9pZJROjBlvdmVycmlkZV9kb2N1bWVudF90eXBlX2lklE6MEG92ZXJyaWRlX3RhZ19pZHOUTowHdGFza19pZJSMJDFlYTczMjhhLTk3MjctNDJiMC1iMTEyLTAzZjU3MzQ2MmRiNpR1Lg==", + "result": "gAWVKAAAAAAAAACMJFN1Y2Nlc3MuIE5ldyBkb2N1bWVudCBpZCAyNjEgY3JlYXRlZJQu", + "group": null, + "started": "2022-03-28T23:12:41.286318-07:00", + "stopped": "2022-03-28T23:13:00.523505-07:00", + "success": true, + "attempt_count": 1 + } + }, + { + "id": 8, + "type": "file", + "result": "Success. New document id 262 created", + "status": "SUCCESS", + "task_id": "41229b8be9b445c0a523697d0f58f13e", + "name": "2sample-pdf-with-images_pw.pdf", + "created": "2022-05-26T14:26:07.667993-07:00", + "acknowledged": false, + "attempted_task": { + "id": "41229b8be9b445c0a523697d0f58f13e", + "name": "2sample-pdf-with-images_pw.pdf", + "func": "documents.tasks.consume_file", + "hook": null, + "args": "gAWVLgAAAAAAAACMKC90bXAvcGFwZXJsZXNzL3BhcGVybGVzcy11cGxvYWQtN2tfejA0MTGUhZQu", + "kwargs": "gAWVywAAAAAAAAB9lCiMEW92ZXJyaWRlX2ZpbGVuYW1llIweMnNhbXBsZS1wZGYtd2l0aC1pbWFnZXNfcHcucGRmlIwOb3ZlcnJpZGVfdGl0bGWUTowZb3ZlcnJpZGVfY29ycmVzcG9uZGVudF9pZJROjBlvdmVycmlkZV9kb2N1bWVudF90eXBlX2lklE6MEG92ZXJyaWRlX3RhZ19pZHOUTowHdGFza19pZJSMJDk5YTgyOTc3LWU1MWUtNGJjYS04MjM4LTNkNzdhZTJhNjZmYZR1Lg==", + "result": "gAWVKAAAAAAAAACMJFN1Y2Nlc3MuIE5ldyBkb2N1bWVudCBpZCAyNjIgY3JlYXRlZJQu", + "group": null, + "started": "2022-03-28T23:43:53.171963-07:00", + "stopped": "2022-03-28T23:43:56.965257-07:00", + "success": true, + "attempt_count": 1 + } + }, + { + "id": 6, + "type": "file", + "result": "Success. New document id 264 created", + "status": "SUCCESS", + "task_id": "bbbca32d408c4619bd0b512a8327c773", + "name": "homebridge.log", + "created": "2022-05-26T14:26:07.665560-07:00", + "acknowledged": false, + "attempted_task": { + "id": "bbbca32d408c4619bd0b512a8327c773", + "name": "homebridge.log", + "func": "documents.tasks.consume_file", + "hook": null, + "args": "gAWVLgAAAAAAAACMKC90bXAvcGFwZXJsZXNzL3BhcGVybGVzcy11cGxvYWQteGo4aW9zYXaUhZQu", + "kwargs": "gAWVuwAAAAAAAAB9lCiMEW92ZXJyaWRlX2ZpbGVuYW1llIwOaG9tZWJyaWRnZS5sb2eUjA5vdmVycmlkZV90aXRsZZROjBlvdmVycmlkZV9jb3JyZXNwb25kZW50X2lklE6MGW92ZXJyaWRlX2RvY3VtZW50X3R5cGVfaWSUTowQb3ZlcnJpZGVfdGFnX2lkc5ROjAd0YXNrX2lklIwkNzY0NzdhNWEtNzk0Ni00NWU0LWE3MDktNzQzNDg0ZDE2YTUxlHUu", + "result": "gAWVKAAAAAAAAACMJFN1Y2Nlc3MuIE5ldyBkb2N1bWVudCBpZCAyNjQgY3JlYXRlZJQu", + "group": null, + "started": "2022-03-29T22:56:16.053026-07:00", + "stopped": "2022-03-29T22:56:21.196179-07:00", + "success": true, + "attempt_count": 1 + } + }, + { + "id": 5, + "type": "file", + "result": "Success. New document id 265 created", + "status": "SUCCESS", + "task_id": "00ab285ab4bf482ab30c7d580b252ecb", + "name": "IMG_7459.PNG", + "created": "2022-05-26T14:26:07.664506-07:00", + "acknowledged": false, + "attempted_task": { + "id": "00ab285ab4bf482ab30c7d580b252ecb", + "name": "IMG_7459.PNG", + "func": "documents.tasks.consume_file", + "hook": null, + "args": "gAWVLgAAAAAAAACMKC90bXAvcGFwZXJsZXNzL3BhcGVybGVzcy11cGxvYWQtOGF5NDNfZjeUhZQu", + "kwargs": "gAWVuQAAAAAAAAB9lCiMEW92ZXJyaWRlX2ZpbGVuYW1llIwMSU1HXzc0NTkuUE5HlIwOb3ZlcnJpZGVfdGl0bGWUTowZb3ZlcnJpZGVfY29ycmVzcG9uZGVudF9pZJROjBlvdmVycmlkZV9kb2N1bWVudF90eXBlX2lklE6MEG92ZXJyaWRlX3RhZ19pZHOUTowHdGFza19pZJSMJDYxMTNhNzRlLTAwOWMtNGJhYi1hMjk1LTFmNjMwMzZmMTc4ZpR1Lg==", + "result": "gAWVKAAAAAAAAACMJFN1Y2Nlc3MuIE5ldyBkb2N1bWVudCBpZCAyNjUgY3JlYXRlZJQu", + "group": null, + "started": "2022-04-05T13:19:47.490282-07:00", + "stopped": "2022-04-05T13:21:36.782264-07:00", + "success": true, + "attempt_count": 1 + } + }, + { + "id": 3, + "type": "file", + "result": "Success. New document id 267 created", + "status": "SUCCESS", + "task_id": "289c5163cfec410db42948a0cacbeb9c", + "name": "IMG_7459.PNG", + "created": "2022-05-26T14:26:07.659661-07:00", + "acknowledged": false, + "attempted_task": { + "id": "289c5163cfec410db42948a0cacbeb9c", + "name": "IMG_7459.PNG", + "func": "documents.tasks.consume_file", + "hook": null, + "args": "gAWVLgAAAAAAAACMKC90bXAvcGFwZXJsZXNzL3BhcGVybGVzcy11cGxvYWQtNzRuY2p2aXGUhZQu", + "kwargs": "gAWVuQAAAAAAAAB9lCiMEW92ZXJyaWRlX2ZpbGVuYW1llIwMSU1HXzc0NTkuUE5HlIwOb3ZlcnJpZGVfdGl0bGWUTowZb3ZlcnJpZGVfY29ycmVzcG9uZGVudF9pZJROjBlvdmVycmlkZV9kb2N1bWVudF90eXBlX2lklE6MEG92ZXJyaWRlX3RhZ19pZHOUTowHdGFza19pZJSMJGZjZDljMmFlLWFhZmEtNGJmMC05M2Y5LWE3ZGQxYmEzYWM1NZR1Lg==", + "result": "gAWVKAAAAAAAAACMJFN1Y2Nlc3MuIE5ldyBkb2N1bWVudCBpZCAyNjcgY3JlYXRlZJQu", + "group": null, + "started": "2022-04-05T13:29:59.264441-07:00", + "stopped": "2022-04-05T13:30:28.336185-07:00", + "success": true, + "attempt_count": 1 + } + }, + { + "id": 1, + "type": "file", + "result": "Success. New document id 268 created", + "status": "SUCCESS", + "task_id": "7a4ebdb2bde04311935284027ef8ca65", + "name": "2019-08-04 DSA Questionnaire - 5-8-19.pdf", + "created": "2022-05-26T14:26:07.655276-07:00", + "acknowledged": false, + "attempted_task": { + "id": "7a4ebdb2bde04311935284027ef8ca65", + "name": "2019-08-04 DSA Questionnaire - 5-8-19.pdf", + "func": "documents.tasks.consume_file", + "hook": null, + "args": "gAWVLgAAAAAAAACMKC90bXAvcGFwZXJsZXNzL3BhcGVybGVzcy11cGxvYWQtdXpscHl2NnmUhZQu", + "kwargs": "gAWV1gAAAAAAAAB9lCiMEW92ZXJyaWRlX2ZpbGVuYW1llIwpMjAxOS0wOC0wNCBEU0EgUXVlc3Rpb25uYWlyZSAtIDUtOC0xOS5wZGaUjA5vdmVycmlkZV90aXRsZZROjBlvdmVycmlkZV9jb3JyZXNwb25kZW50X2lklE6MGW92ZXJyaWRlX2RvY3VtZW50X3R5cGVfaWSUTowQb3ZlcnJpZGVfdGFnX2lkc5ROjAd0YXNrX2lklIwkY2Q3YzBhZjgtN2Q4Ni00OGM0LTliNjgtNDQwMmQ4ZDZlOTNmlHUu", + "result": "gAWVKAAAAAAAAACMJFN1Y2Nlc3MuIE5ldyBkb2N1bWVudCBpZCAyNjggY3JlYXRlZJQu", + "group": null, + "started": "2022-04-28T21:01:04.275850-07:00", + "stopped": "2022-04-28T21:01:10.136884-07:00", + "success": true, + "attempt_count": 1 + } + } +] From 5b66ef0a748fd5570361a2a1ed6147e0462568d2 Mon Sep 17 00:00:00 2001 From: Trenton H Date: Wed, 28 Sep 2022 09:09:51 -0700 Subject: [PATCH 022/273] Updates how task_args and task_kwargs are parsed, adds testing to cover everything I can think of --- src/documents/serialisers.py | 73 +++++++++++-------- src/documents/tests/test_api.py | 124 ++++++++++++++++++++++++++++++++ 2 files changed, 166 insertions(+), 31 deletions(-) diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py index 172992de4..cd99c43cd 100644 --- a/src/documents/serialisers.py +++ b/src/documents/serialisers.py @@ -1,8 +1,14 @@ import datetime -import json import math -import os import re +from ast import literal_eval +from asyncio.log import logger +from pathlib import Path +from typing import Dict +from typing import Optional +from typing import Tuple + +from celery import states try: import zoneinfo @@ -646,16 +652,21 @@ class TasksViewSerializer(serializers.ModelSerializer): def get_result(self, obj): result = "" - if hasattr(obj, "attempted_task") and obj.attempted_task: + if ( + hasattr(obj, "attempted_task") + and obj.attempted_task + and obj.attempted_task.result + ): try: - result_json = json.loads(obj.attempted_task.result) - except Exception: - pass - - if result_json and "exc_message" in result_json: - result = result_json["exc_message"] - else: - result = obj.attempted_task.result.strip('"') + result: str = obj.attempted_task.result + if "exc_message" in result: + # This is a dict in this case + result: Dict = literal_eval(result) + # This is a list, grab the first item (most recent) + result = result["exc_message"][0] + except Exception as e: # pragma: no cover + # Extra security if something is malformed + logger.warn(f"Error getting task result: {e}", exc_info=True) return result status = serializers.SerializerMethodField() @@ -704,26 +715,25 @@ class TasksViewSerializer(serializers.ModelSerializer): result = "" if hasattr(obj, "attempted_task") and obj.attempted_task: try: - # We have to make this a valid JSON object string - kwargs_json = json.loads( - obj.attempted_task.task_kwargs.strip('"') - .replace("'", '"') - .replace("None", '""'), - ) - except Exception: - pass + task_kwargs: Optional[str] = obj.attempted_task.task_kwargs + # Try the override filename first (this is a webui created task?) + if task_kwargs is not None: + # It's a string, string of a dict. Who knows why... + kwargs = literal_eval(literal_eval(task_kwargs)) + if "override_filename" in kwargs: + result = kwargs["override_filename"] - if kwargs_json and "override_filename" in kwargs_json: - result = kwargs_json["override_filename"] - else: - filepath = ( - obj.attempted_task.task_args.replace('"', "") - .replace("'", "") - .replace("(", "") - .replace(")", "") - .replace(",", "") - ) - result = os.path.split(filepath)[1] + # Nothing was found, report the task first argument + if not len(result): + # There are always some arguments to the consume + task_args: Tuple = literal_eval( + literal_eval(obj.attempted_task.task_args), + ) + filepath = Path(task_args[0]) + result = filepath.name + except Exception as e: # pragma: no cover + # Extra security if something is malformed + logger.warn(f"Error getting task result: {e}", exc_info=True) return result @@ -735,7 +745,8 @@ class TasksViewSerializer(serializers.ModelSerializer): if ( hasattr(obj, "attempted_task") and obj.attempted_task - and obj.attempted_task.status == "SUCCESS" + and obj.attempted_task.result + and obj.attempted_task.status == states.SUCCESS ): try: result = re.search(regexp, obj.attempted_task.result).group(1) diff --git a/src/documents/tests/test_api.py b/src/documents/tests/test_api.py index 89e340501..ec89a19e8 100644 --- a/src/documents/tests/test_api.py +++ b/src/documents/tests/test_api.py @@ -2831,6 +2831,14 @@ class TestTasks(APITestCase): self.assertEqual(returned_task2["task_name"], result2.task_name) def test_acknowledge_tasks(self): + """ + GIVEN: + - Attempted celery tasks + WHEN: + - API call is made to get mark task as acknowledged + THEN: + - Task is marked as acknowledged + """ result1 = TaskResult.objects.create( task_id=str(uuid.uuid4()), task_name="documents.tasks.some_task", @@ -2849,3 +2857,119 @@ class TestTasks(APITestCase): response = self.client.get(self.ENDPOINT) self.assertEqual(len(response.data), 0) + + def test_task_result_no_error(self): + """ + GIVEN: + - A celery task completed without error + WHEN: + - API call is made to get tasks + THEN: + - The returned data includes the task result + """ + result1 = TaskResult.objects.create( + task_id=str(uuid.uuid4()), + task_name="documents.tasks.some_task", + status=celery.states.SUCCESS, + result="Success. New document id 1 created", + ) + _ = PaperlessTask.objects.create(attempted_task=result1) + + response = self.client.get(self.ENDPOINT) + + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 1) + + returned_data = response.data[0] + + self.assertEqual(returned_data["result"], "Success. New document id 1 created") + self.assertEqual(returned_data["related_document"], "1") + + def test_task_result_with_error(self): + """ + GIVEN: + - A celery task completed with an exception + WHEN: + - API call is made to get tasks + THEN: + - The returned result is the exception info + """ + result1 = TaskResult.objects.create( + task_id=str(uuid.uuid4()), + task_name="documents.tasks.some_task", + status=celery.states.SUCCESS, + result={ + "exc_type": "ConsumerError", + "exc_message": ["test.pdf: Not consuming test.pdf: It is a duplicate."], + "exc_module": "documents.consumer", + }, + ) + _ = PaperlessTask.objects.create(attempted_task=result1) + + response = self.client.get(self.ENDPOINT) + + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 1) + + returned_data = response.data[0] + + self.assertEqual( + returned_data["result"], + "test.pdf: Not consuming test.pdf: It is a duplicate.", + ) + + def test_task_name_webui(self): + """ + GIVEN: + - Attempted celery task + - Task was created through the webui + WHEN: + - API call is made to get tasks + THEN: + - Returned data include the filename + """ + result1 = TaskResult.objects.create( + task_id=str(uuid.uuid4()), + task_name="documents.tasks.some_task", + status=celery.states.SUCCESS, + task_args="\"('/tmp/paperless/paperless-upload-5iq7skzc',)\"", + task_kwargs="\"{'override_filename': 'test.pdf', 'override_title': None, 'override_correspondent_id': None, 'override_document_type_id': None, 'override_tag_ids': None, 'task_id': '466e8fe7-7193-4698-9fff-72f0340e2082', 'override_created': None}\"", + ) + _ = PaperlessTask.objects.create(attempted_task=result1) + + response = self.client.get(self.ENDPOINT) + + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 1) + + returned_data = response.data[0] + + self.assertEqual(returned_data["name"], "test.pdf") + + def test_task_name_consume_folder(self): + """ + GIVEN: + - Attempted celery task + - Task was created through the consume folder + WHEN: + - API call is made to get tasks + THEN: + - Returned data include the filename + """ + result1 = TaskResult.objects.create( + task_id=str(uuid.uuid4()), + task_name="documents.tasks.some_task", + status=celery.states.SUCCESS, + task_args="\"('/consume/anothertest.pdf',)\"", + task_kwargs="\"{'override_tag_ids': None}\"", + ) + _ = PaperlessTask.objects.create(attempted_task=result1) + + response = self.client.get(self.ENDPOINT) + + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 1) + + returned_data = response.data[0] + + self.assertEqual(returned_data["name"], "anothertest.pdf") From 4422bb3f69412b0fae33311590c02339d98416ff Mon Sep 17 00:00:00 2001 From: Trenton H Date: Wed, 28 Sep 2022 11:02:34 -0700 Subject: [PATCH 023/273] Fix logger location tag Co-authored-by: shamoon <4887959+shamoon@users.noreply.github.com> --- src/documents/serialisers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py index cd99c43cd..78906cf2b 100644 --- a/src/documents/serialisers.py +++ b/src/documents/serialisers.py @@ -733,7 +733,7 @@ class TasksViewSerializer(serializers.ModelSerializer): result = filepath.name except Exception as e: # pragma: no cover # Extra security if something is malformed - logger.warn(f"Error getting task result: {e}", exc_info=True) + logger.warn(f"Error getting file name from task: {e}", exc_info=True) return result From 38ba1d1a52280073e50a7161bfce44cde34e052a Mon Sep 17 00:00:00 2001 From: Trenton Holmes Date: Wed, 28 Sep 2022 19:37:19 -0700 Subject: [PATCH 024/273] Cleans up and documnets installation on bare metal, includes systemd service file for the celery and celery beat --- docs/setup.rst | 33 +++++++++++++++++----------- scripts/paperless-scheduler.service | 4 ++-- scripts/paperless-task-queue.service | 12 ++++++++++ 3 files changed, 34 insertions(+), 15 deletions(-) create mode 100644 scripts/paperless-task-queue.service diff --git a/docs/setup.rst b/docs/setup.rst index ca07c1032..f2970fd9b 100644 --- a/docs/setup.rst +++ b/docs/setup.rst @@ -39,7 +39,7 @@ Paperless consists of the following components: .. _setup-task_processor: -* **The task processor:** Paperless relies on `Django Q `_ +* **The task processor:** Paperless relies on `Celery - Distributed Task Queue `_ for doing most of the heavy lifting. This is a task queue that accepts tasks from multiple sources and processes these in parallel. It also comes with a scheduler that executes certain commands periodically. @@ -62,13 +62,6 @@ Paperless consists of the following components: tasks fail and inspect the errors (i.e., wrong email credentials, errors during consuming a specific file, etc). - You may start the task processor by executing: - - .. code:: shell-session - - $ cd /path/to/paperless/src/ - $ python3 manage.py qcluster - * A `redis `_ message broker: This is a really lightweight service that is responsible for getting the tasks from the webserver and the consumer to the task scheduler. These run in a different process (maybe even on different machines!), and therefore, this is necessary. @@ -291,7 +284,20 @@ Build the Docker image yourself .. code:: yaml webserver: - build: . + build: + context: . + args: + QPDF_VERSION: x.y.x + PIKEPDF_VERSION: x.y.z + PSYCOPG2_VERSION: x.y.z + JBIG2ENC_VERSION: 0.29 + + .. note:: + + You should match the build argument versions to the version for the release you have + checked out. These are pre-built images with certain, more updated software. + If you want to build these images your self, that is possible, but beyond + the scope of these steps. 4. Follow steps 3 to 8 of :ref:`setup-docker_hub`. When asked to run ``docker-compose pull`` to pull the image, do @@ -332,7 +338,7 @@ writing. Windows is not and will never be supported. .. code:: - python3 python3-pip python3-dev imagemagick fonts-liberation gnupg libpq-dev libmagic-dev mime-support libzbar0 poppler-utils + python3 python3-pip python3-dev imagemagick fonts-liberation gnupg libpq-dev default-libmysqlclient-dev libmagic-dev mime-support libzbar0 poppler-utils These dependencies are required for OCRmyPDF, which is used for text recognition. @@ -361,7 +367,7 @@ writing. Windows is not and will never be supported. You will also need ``build-essential``, ``python3-setuptools`` and ``python3-wheel`` for installing some of the python dependencies. -2. Install ``redis`` >= 5.0 and configure it to start automatically. +2. Install ``redis`` >= 6.0 and configure it to start automatically. 3. Optional. Install ``postgresql`` and configure a database, user and password for paperless. If you do not wish to use PostgreSQL, MariaDB and SQLite are available as well. @@ -461,8 +467,9 @@ writing. Windows is not and will never be supported. as a starting point. Paperless needs the ``webserver`` script to run the webserver, the - ``consumer`` script to watch the input folder, and the ``scheduler`` - script to run tasks such as email checking and document consumption. + ``consumer`` script to watch the input folder, ``taskqueue`` for the background workers + used to handle things like document consumption and the ``scheduler`` script to run tasks such as + email checking at certain times . The ``socket`` script enables ``gunicorn`` to run on port 80 without root privileges. For this you need to uncomment the ``Require=paperless-webserver.socket`` diff --git a/scripts/paperless-scheduler.service b/scripts/paperless-scheduler.service index b1c82a38e..b15c53e79 100644 --- a/scripts/paperless-scheduler.service +++ b/scripts/paperless-scheduler.service @@ -1,12 +1,12 @@ [Unit] -Description=Paperless scheduler +Description=Paperless Celery Beat Requires=redis.service [Service] User=paperless Group=paperless WorkingDirectory=/opt/paperless/src -ExecStart=python3 manage.py qcluster +ExecStart=celery --app paperless beat --loglevel INFO [Install] WantedBy=multi-user.target diff --git a/scripts/paperless-task-queue.service b/scripts/paperless-task-queue.service new file mode 100644 index 000000000..5fade360a --- /dev/null +++ b/scripts/paperless-task-queue.service @@ -0,0 +1,12 @@ +[Unit] +Description=Paperless Celery Workers +Requires=redis.service + +[Service] +User=paperless +Group=paperless +WorkingDirectory=/opt/paperless/src +ExecStart=celery --app paperless worker --loglevel INFO + +[Install] +WantedBy=multi-user.target From 04faa10e3bc0d46e14cb228dd1b39476f3f1cbd5 Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+shamoon@users.noreply.github.com> Date: Wed, 28 Sep 2022 20:13:19 -0700 Subject: [PATCH 025/273] Update references to qcluster t celery --- docs/extending.rst | 6 +++--- docs/troubleshooting.rst | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/extending.rst b/docs/extending.rst index bfc289689..e8126fd4d 100644 --- a/docs/extending.rst +++ b/docs/extending.rst @@ -112,7 +112,7 @@ To do the setup you need to perform the steps from the following chapters in a c .. code:: shell-session - python3 manage.py runserver & python3 manage.py document_consumer & python3 manage.py qcluster + python3 manage.py runserver & python3 manage.py document_consumer & celery --app paperless worker 11. Login with the superuser credentials provided in step 8 at ``http://localhost:8000`` to create a session that enables you to use the backend. @@ -128,14 +128,14 @@ Configure the IDE to use the src/ folder as the base source folder. Configure th launch configurations in your IDE: * python3 manage.py runserver -* python3 manage.py qcluster +* celery --app paperless worker * python3 manage.py document_consumer To start them all: .. code:: shell-session - python3 manage.py runserver & python3 manage.py document_consumer & python3 manage.py qcluster + python3 manage.py runserver & python3 manage.py document_consumer & celery --app paperless worker Testing and code style: diff --git a/docs/troubleshooting.rst b/docs/troubleshooting.rst index 6d94d7100..bab346123 100644 --- a/docs/troubleshooting.rst +++ b/docs/troubleshooting.rst @@ -19,7 +19,7 @@ Check for the following issues: .. code:: shell-session - $ python3 manage.py qcluster + $ celery --app paperless worker * Look at the output of paperless and inspect it for any errors. * Go to the admin interface, and check if there are failed tasks. If so, the From 436f9e891e843eec62b09f239074b24f36e34237 Mon Sep 17 00:00:00 2001 From: Trenton H Date: Tue, 27 Sep 2022 09:03:27 -0700 Subject: [PATCH 026/273] Changes MariaDB encoding to use utf8mb4 --- src/paperless/settings.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/paperless/settings.py b/src/paperless/settings.py index ff9d350ce..e092b3f3e 100644 --- a/src/paperless/settings.py +++ b/src/paperless/settings.py @@ -319,6 +319,7 @@ DATABASES = { "default": { "ENGINE": "django.db.backends.sqlite3", "NAME": os.path.join(DATA_DIR, "db.sqlite3"), + "OPTIONS": {}, }, } @@ -340,21 +341,18 @@ if os.getenv("PAPERLESS_DBHOST"): # Leave room for future extensibility if os.getenv("PAPERLESS_DBENGINE") == "mariadb": engine = "django.db.backends.mysql" - options = {"read_default_file": "/etc/mysql/my.cnf"} + options = {"read_default_file": "/etc/mysql/my.cnf", "charset": "utf8mb4"} else: # Default to PostgresDB engine = "django.db.backends.postgresql_psycopg2" options = {"sslmode": os.getenv("PAPERLESS_DBSSLMODE", "prefer")} DATABASES["default"]["ENGINE"] = engine - for key, value in options.items(): - DATABASES["default"]["OPTIONS"][key] = value + DATABASES["default"]["OPTIONS"].update(options) if os.getenv("PAPERLESS_DB_TIMEOUT") is not None: - _new_opts = {"timeout": float(os.getenv("PAPERLESS_DB_TIMEOUT"))} - if "OPTIONS" in DATABASES["default"]: - DATABASES["default"]["OPTIONS"].update(_new_opts) - else: - DATABASES["default"]["OPTIONS"] = _new_opts + DATABASES["default"]["OPTIONS"].update( + {"timeout": float(os.getenv("PAPERLESS_DB_TIMEOUT"))}, + ) DEFAULT_AUTO_FIELD = "django.db.models.AutoField" From 9bd031fbd785cdd2b37c0619395565198faf1799 Mon Sep 17 00:00:00 2001 From: "Paperless-ngx Translation Bot [bot]" <99855517+paperless-l10n@users.noreply.github.com> Date: Fri, 30 Sep 2022 06:55:45 -0700 Subject: [PATCH 027/273] New translations messages.xlf (Russian) [ci skip] --- src-ui/src/locale/messages.ru_RU.xlf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src-ui/src/locale/messages.ru_RU.xlf b/src-ui/src/locale/messages.ru_RU.xlf index b87cd18df..8d27bb9ac 100644 --- a/src-ui/src/locale/messages.ru_RU.xlf +++ b/src-ui/src/locale/messages.ru_RU.xlf @@ -8,7 +8,7 @@ node_modules/src/alert/alert.ts 42,44 - Close + Закрыть Slide of From 4a3a55b923b0cf58e89387dea8fb73a23f7210f0 Mon Sep 17 00:00:00 2001 From: "Paperless-ngx Translation Bot [bot]" <99855517+paperless-l10n@users.noreply.github.com> Date: Fri, 30 Sep 2022 07:56:26 -0700 Subject: [PATCH 028/273] New translations messages.xlf (Russian) [ci skip] --- src-ui/src/locale/messages.ru_RU.xlf | 74 ++++++++++++++-------------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/src-ui/src/locale/messages.ru_RU.xlf b/src-ui/src/locale/messages.ru_RU.xlf index 8d27bb9ac..f2a5753c0 100644 --- a/src-ui/src/locale/messages.ru_RU.xlf +++ b/src-ui/src/locale/messages.ru_RU.xlf @@ -25,7 +25,7 @@ node_modules/src/carousel/carousel.ts 188,191 - Previous + Назад Next @@ -33,7 +33,7 @@ node_modules/src/carousel/carousel.ts 209,211 - Next + Следующий Select month @@ -45,7 +45,7 @@ node_modules/src/datepicker/datepicker-navigation-select.ts 41,42 - Select month + Выберите месяц Select year @@ -57,7 +57,7 @@ node_modules/src/datepicker/datepicker-navigation-select.ts 41,42 - Select year + Выберите год Previous month @@ -69,7 +69,7 @@ node_modules/src/datepicker/datepicker-navigation.ts 43,46 - Previous month + Предыдущий месяц Next month @@ -81,7 +81,7 @@ node_modules/src/datepicker/datepicker-navigation.ts 43,46 - Next month + Следующий месяц «« @@ -171,7 +171,7 @@ node_modules/src/timepicker/timepicker.ts 161 - Hours + Часы MM @@ -179,7 +179,7 @@ node_modules/src/timepicker/timepicker.ts 182 - MM + MM Minutes @@ -187,7 +187,7 @@ node_modules/src/timepicker/timepicker.ts 199 - Minutes + Минуты Increment hours @@ -227,7 +227,7 @@ node_modules/src/timepicker/timepicker.ts 295 - SS + СС Seconds @@ -235,7 +235,7 @@ node_modules/src/timepicker/timepicker.ts 295 - Seconds + Секунды Increment seconds @@ -277,7 +277,7 @@ node_modules/src/toast/toast.ts 70,71 - Close + Закрыть Drop files to begin upload @@ -285,7 +285,7 @@ src/app/app.component.html 7 - Drop files to begin upload + Перетащите файлы для загрузки Document added @@ -1467,7 +1467,7 @@ src/app/components/document-comments/document-comments.component.html 10 - Add comment + Добавить комментарий Error saving comment: @@ -1591,7 +1591,7 @@ src/app/components/document-list/bulk-editor/bulk-editor.component.html 90 - Redo OCR + Повторить распознавание More like this @@ -1727,7 +1727,7 @@ src/app/components/document-detail/document-detail.component.html 83 - Default + По умолчанию Content @@ -1779,7 +1779,7 @@ src/app/components/document-detail/document-detail.component.html 117 - Original filename + Исходное имя файла Original MD5 checksum @@ -2437,7 +2437,7 @@ src/app/components/document-list/document-card-small/document-card-small.component.html 14 - Toggle tag filter + Показать/скрыть панель фильтра Toggle correspondent filter @@ -2445,7 +2445,7 @@ src/app/components/document-list/document-card-small/document-card-small.component.html 24 - Toggle correspondent filter + Переключить соответствующий фильтр Toggle document type filter @@ -2453,7 +2453,7 @@ src/app/components/document-list/document-card-small/document-card-small.component.html 31 - Toggle document type filter + Переключить фильтр типа документа Toggle storage path filter @@ -2549,7 +2549,7 @@ src/app/components/document-list/document-list.component.html 102 - Error while loading documents + Ошибка при загрузке документов ASN @@ -2717,7 +2717,7 @@ src/app/components/document-list/filter-editor/filter-editor.component.ts 168 - is empty + не заполнено is not empty @@ -2725,7 +2725,7 @@ src/app/components/document-list/filter-editor/filter-editor.component.ts 172 - is not empty + не является пустым greater than @@ -2733,7 +2733,7 @@ src/app/components/document-list/filter-editor/filter-editor.component.ts 176 - greater than + больше чем less than @@ -2741,7 +2741,7 @@ src/app/components/document-list/filter-editor/filter-editor.component.ts 180 - less than + меньше чем Save current view @@ -2789,7 +2789,7 @@ src/app/components/document-list/save-view-config-dialog/save-view-config-dialog.component.html 13 - The error returned was + Произошла ошибка correspondent @@ -2797,7 +2797,7 @@ src/app/components/manage/correspondent-list/correspondent-list.component.ts 33 - correspondent + корреспондент correspondents @@ -2813,7 +2813,7 @@ src/app/components/manage/correspondent-list/correspondent-list.component.ts 38 - Last used + Последний использованный Do you really want to delete the correspondent ""? @@ -3009,7 +3009,7 @@ src/app/components/manage/settings/settings.component.html 10 - General + Общие Appearance @@ -3193,7 +3193,7 @@ src/app/components/manage/settings/settings.component.html 132 - Enable comments + Включить комментарии Notifications @@ -3313,7 +3313,7 @@ src/app/components/manage/settings/settings.component.ts 269 - An error occurred while saving settings. + Произошла ошибка при сохранении настроек. Use system language @@ -3619,7 +3619,7 @@ src/app/guards/dirty-doc.guard.ts 17 - Warning: You have unsaved changes to your document(s). + Внимание: у вас есть несохраненные изменения в документе(ах). Unsaved Changes @@ -3868,7 +3868,7 @@ src/app/services/settings.service.ts 146 - Belarusian + Белорусский Czech @@ -3988,7 +3988,7 @@ src/app/services/settings.service.ts 236 - Slovenian + Словенский Serbian @@ -3996,7 +3996,7 @@ src/app/services/settings.service.ts 242 - Serbian + Сербский Swedish @@ -4012,7 +4012,7 @@ src/app/services/settings.service.ts 254 - Turkish + Турецкий Chinese Simplified @@ -4020,7 +4020,7 @@ src/app/services/settings.service.ts 260 - Chinese Simplified + Китайский упрощенный ISO 8601 From f59abadbc4e1990e3c79e2a01480fd303bb9bc8f Mon Sep 17 00:00:00 2001 From: "Paperless-ngx Translation Bot [bot]" <99855517+paperless-l10n@users.noreply.github.com> Date: Fri, 30 Sep 2022 10:20:29 -0700 Subject: [PATCH 029/273] New translations messages.xlf (Russian) [ci skip] --- src-ui/src/locale/messages.ru_RU.xlf | 66 ++++++++++++++-------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/src-ui/src/locale/messages.ru_RU.xlf b/src-ui/src/locale/messages.ru_RU.xlf index f2a5753c0..e113995f9 100644 --- a/src-ui/src/locale/messages.ru_RU.xlf +++ b/src-ui/src/locale/messages.ru_RU.xlf @@ -17,7 +17,7 @@ 157,166 Currently selected slide number read by screen reader - Slide of + Слайд из Previous @@ -121,7 +121,7 @@ node_modules/src/pagination/pagination.ts 224,226 - First + Первый Previous @@ -129,7 +129,7 @@ node_modules/src/pagination/pagination.ts 224,226 - Previous + Предыдущий Next @@ -137,7 +137,7 @@ node_modules/src/pagination/pagination.ts 224,225 - Next + Следующий Last @@ -145,7 +145,7 @@ node_modules/src/pagination/pagination.ts 224,225 - Last + Последний @@ -163,7 +163,7 @@ node_modules/src/timepicker/timepicker.ts 138,141 - HH + ЧЧ Hours @@ -590,7 +590,7 @@ src/app/components/app-frame/app-frame.component.html 205 - is available. + доступно. Click to view. @@ -598,7 +598,7 @@ src/app/components/app-frame/app-frame.component.html 205 - Click to view. + Нажмите для просмотра. Checking for updates is disabled. @@ -606,7 +606,7 @@ src/app/components/app-frame/app-frame.component.html 208 - Checking for updates is disabled. + Проверка обновлений отключена. Click for more information. @@ -614,7 +614,7 @@ src/app/components/app-frame/app-frame.component.html 208 - Click for more information. + Нажмите для дополнительной информации. Update available @@ -622,7 +622,7 @@ src/app/components/app-frame/app-frame.component.html 216 - Update available + Доступно обновление Cancel @@ -1090,7 +1090,7 @@ src/app/components/common/filterable-dropdown/filterable-dropdown.component.html 23 - Any + Любой Apply @@ -1451,7 +1451,7 @@ src/app/components/document-comments/document-comments.component.html 4 - Enter comment + Введите текст комментария Please enter a comment. @@ -1459,7 +1459,7 @@ src/app/components/document-comments/document-comments.component.html 5,7 - Please enter a comment. + Пожалуйста, введите комментарий. Add comment @@ -1619,7 +1619,7 @@ src/app/components/document-detail/document-detail.component.html 50 - Previous + Предыдущий Next @@ -1627,7 +1627,7 @@ src/app/components/document-detail/document-detail.component.html 55 - Next + Следующий Details @@ -1847,7 +1847,7 @@ src/app/components/document-detail/document-detail.component.html 202 - Enter Password + Введите пароль Comments @@ -1859,7 +1859,7 @@ src/app/components/manage/settings/settings.component.html 128 - Comments + Комментарии Discard @@ -2421,7 +2421,7 @@ src/app/components/document-list/document-card-small/document-card-small.component.html 50 - Modified: + Изменено: Score: @@ -2637,7 +2637,7 @@ src/app/components/document-list/filter-editor/filter-editor.component.ts 81,83 - Type: + Тип: Without document type @@ -2653,7 +2653,7 @@ src/app/components/document-list/filter-editor/filter-editor.component.ts 89,91 - Tag: + Тег: Without any tag @@ -2709,7 +2709,7 @@ src/app/components/document-list/filter-editor/filter-editor.component.ts 164 - equals + совпадает с is empty @@ -2781,7 +2781,7 @@ src/app/components/document-list/save-view-config-dialog/save-view-config-dialog.component.html 12 - Filter rules error occurred while saving this view + Ошибка фильтра правил при сохранении этого представления The error returned was @@ -2965,7 +2965,7 @@ src/app/components/manage/management-list/management-list.component.html 74 - {VAR_PLURAL, plural, =1 {One } other { total }} + {VAR_PLURAL, plural, one {} few { всего } many { всего }=1 {Один } other { всего }} Automatic @@ -2985,7 +2985,7 @@ src/app/components/manage/management-list/management-list.component.ts 140 - Do you really want to delete the ? + Вы действительно хотите удалить "? Associated documents will not be deleted. @@ -3001,7 +3001,7 @@ src/app/components/manage/management-list/management-list.component.ts 168,170 - Error while deleting element: + Возникла ошибка при удалении элемента: General @@ -3297,7 +3297,7 @@ src/app/components/manage/settings/settings.component.ts 258 - Settings were saved successfully. Reload is required to apply some changes. + Настройки успешно сохранены. Для применения некоторых изменений необходимо перезагрузить. Reload now @@ -3337,7 +3337,7 @@ src/app/components/manage/settings/settings.component.ts 304,306 - Error while storing settings on server: + Ошибка при хранении настроек на сервере: storage path @@ -3361,7 +3361,7 @@ src/app/components/manage/storage-path-list/storage-path-list.component.ts 45 - Do you really want to delete the storage path ""? + Вы действительно хотите удалить путь до хранилища данных ""? tag @@ -3411,7 +3411,7 @@ src/app/components/manage/tasks/tasks.component.html 11 - + Refresh @@ -3463,7 +3463,7 @@ src/app/components/manage/tasks/tasks.component.html 102 - Complete  + Выполнено Started  @@ -3471,7 +3471,7 @@ src/app/components/manage/tasks/tasks.component.html 108 - Started  + Запущено Queued  @@ -3479,7 +3479,7 @@ src/app/components/manage/tasks/tasks.component.html 114 - Queued  + В очереди Dismiss selected From 9e2430da46003edcc6d8627463b854560c5f8c4c Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+shamoon@users.noreply.github.com> Date: Fri, 30 Sep 2022 12:30:23 -0700 Subject: [PATCH 030/273] Frontend update checking settings --- .../app-frame/app-frame.component.html | 21 ++++-- .../app-frame/app-frame.component.ts | 39 +++++++++-- .../filterable-dropdown.component.scss | 19 ----- .../manage/settings/settings.component.html | 15 ++++ .../manage/settings/settings.component.ts | 28 +++++++- src-ui/src/app/data/paperless-uisettings.ts | 13 ++++ .../services/rest/remote-version.service.ts | 1 - src-ui/src/app/services/settings.service.ts | 49 +++++++++++-- src-ui/src/styles.scss | 19 +++++ src/documents/views.py | 70 ++++++++++--------- 10 files changed, 200 insertions(+), 74 deletions(-) diff --git a/src-ui/src/app/components/app-frame/app-frame.component.html b/src-ui/src/app/components/app-frame/app-frame.component.html index 5a68a3cda..8f94f80aa 100644 --- a/src-ui/src/app/components/app-frame/app-frame.component.html +++ b/src-ui/src/app/components/app-frame/app-frame.component.html @@ -200,14 +200,25 @@
diff --git a/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.scss b/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.scss index 0e8796b3d..82ad7e7a5 100644 --- a/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.scss +++ b/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.scss @@ -21,3 +21,7 @@ input[type="text"] { min-width: 120px; } + +.z-10 { + z-index: 10; +} diff --git a/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.ts b/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.ts index 14643875f..44f524149 100644 --- a/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.ts +++ b/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.ts @@ -709,15 +709,23 @@ export class FilterEditorComponent implements OnInit, OnDestroy { this.updateRules() } - textFilterEnter() { - const filterString = ( - this.textFilterInput.nativeElement as HTMLInputElement - ).value - if (filterString.length) { - this.updateTextFilter(filterString) + textFilterKeyup(event: KeyboardEvent) { + if (event.key == 'Enter') { + const filterString = ( + this.textFilterInput.nativeElement as HTMLInputElement + ).value + if (filterString.length) { + this.updateTextFilter(filterString) + } + } else if (event.key == 'Escape') { + this.resetTextField() } } + resetTextField() { + this.updateTextFilter('') + } + changeTextFilterTarget(target) { if ( this.textFilterTarget == TEXT_FILTER_TARGET_FULLTEXT_MORELIKE && From bd64684fa465077313222204e8ead1370b1f5e33 Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+shamoon@users.noreply.github.com> Date: Sat, 29 Oct 2022 09:44:26 -0700 Subject: [PATCH 213/273] support esc and x button for main search bar --- .../components/app-frame/app-frame.component.html | 7 ++++++- .../components/app-frame/app-frame.component.scss | 15 ++++++++------- .../components/app-frame/app-frame.component.ts | 14 ++++++++++++++ 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/src-ui/src/app/components/app-frame/app-frame.component.html b/src-ui/src/app/components/app-frame/app-frame.component.html index 3700105f3..41bd50970 100644 --- a/src-ui/src/app/components/app-frame/app-frame.component.html +++ b/src-ui/src/app/components/app-frame/app-frame.component.html @@ -16,7 +16,12 @@ + [formControl]="searchField" [ngbTypeahead]="searchAutoComplete" (keyup)="searchFieldKeyup($event)" (selectItem)="itemSelected($event)" i18n-placeholder> +
    diff --git a/src-ui/src/app/components/app-frame/app-frame.component.scss b/src-ui/src/app/components/app-frame/app-frame.component.scss index 755f20696..0bd96f33e 100644 --- a/src-ui/src/app/components/app-frame/app-frame.component.scss +++ b/src-ui/src/app/components/app-frame/app-frame.component.scss @@ -243,17 +243,18 @@ main { form { position: relative; + + > svg { + position: absolute; + left: 0.6rem; + top: 0.5rem; + color: rgba(255, 255, 255, 0.6); + } } - svg { - position: absolute; - left: 0.6rem; - top: 0.5rem; - color: rgba(255, 255, 255, 0.6); - } &:focus-within { - svg { + form > svg { display: none; } diff --git a/src-ui/src/app/components/app-frame/app-frame.component.ts b/src-ui/src/app/components/app-frame/app-frame.component.ts index b189409a8..d637a32b6 100644 --- a/src-ui/src/app/components/app-frame/app-frame.component.ts +++ b/src-ui/src/app/components/app-frame/app-frame.component.ts @@ -93,6 +93,20 @@ export class AppFrameComponent implements OnInit, ComponentCanDeactivate { searchField = new FormControl('') + get searchFieldEmpty(): boolean { + return this.searchField.value.trim().length == 0 + } + + resetSearchField() { + this.searchField.reset('') + } + + searchFieldKeyup(event: KeyboardEvent) { + if (event.key == 'Escape') { + this.resetSearchField() + } + } + get openDocuments(): PaperlessDocument[] { return this.openDocumentsService.getOpenDocuments() } From 28f7b0dc13358dfcaef711a7b2128fcb7d3520be Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+shamoon@users.noreply.github.com> Date: Sat, 29 Oct 2022 14:07:12 -0700 Subject: [PATCH 214/273] Use check badge for dropdowns --- .../common/date-dropdown/date-dropdown.component.html | 6 ++++-- .../filterable-dropdown/filterable-dropdown.component.html | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src-ui/src/app/components/common/date-dropdown/date-dropdown.component.html b/src-ui/src/app/components/common/date-dropdown/date-dropdown.component.html index c99c71cda..dc8ae8688 100644 --- a/src-ui/src/app/components/common/date-dropdown/date-dropdown.component.html +++ b/src-ui/src/app/components/common/date-dropdown/date-dropdown.component.html @@ -1,8 +1,10 @@