From 9b01c96846e44ca17d17106c0371a02fe8fccd0e Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+shamoon@users.noreply.github.com> Date: Sun, 23 Oct 2022 22:11:24 -0700 Subject: [PATCH 1/6] Grammar error, updated frontend translation https://crowdin.com/translate/paperless-ngx/16/en-it?filter=basic&value=0# --- src-ui/messages.xlf | 8 ++++---- src-ui/src/app/app.component.ts | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src-ui/messages.xlf b/src-ui/messages.xlf index 3b037ff25..8260dff93 100644 --- a/src-ui/messages.xlf +++ b/src-ui/messages.xlf @@ -305,8 +305,8 @@ 122 - - Drag-and-drop documents here to start uploading or place them in the consume folder. You can also drag-and-drop documents anywhere on all other pages of the web app. Once you do, Paperless-ngx will start training it's machine learning algorithms. + + Drag-and-drop documents here to start uploading or place them in the consume folder. You can also drag-and-drop documents anywhere on all other pages of the web app. Once you do, Paperless-ngx will start training its machine learning algorithms. src/app/app.component.ts 129 @@ -2507,14 +2507,14 @@ View "" saved successfully. src/app/components/document-list/document-list.component.ts - 176 + 170 View "" created successfully. src/app/components/document-list/document-list.component.ts - 206 + 210 diff --git a/src-ui/src/app/app.component.ts b/src-ui/src/app/app.component.ts index 4ab48f32c..1d1280eaa 100644 --- a/src-ui/src/app/app.component.ts +++ b/src-ui/src/app/app.component.ts @@ -126,7 +126,7 @@ export class AppComponent implements OnInit, OnDestroy { }, { anchorId: 'tour.upload-widget', - content: $localize`Drag-and-drop documents here to start uploading or place them in the consume folder. You can also drag-and-drop documents anywhere on all other pages of the web app. Once you do, Paperless-ngx will start training it's machine learning algorithms.`, + content: $localize`Drag-and-drop documents here to start uploading or place them in the consume folder. You can also drag-and-drop documents anywhere on all other pages of the web app. Once you do, Paperless-ngx will start training its machine learning algorithms.`, route: '/dashboard', enableBackdrop: true, }, From a72cc5da83e1e3652c17c01c1857268e0ce82f7e Mon Sep 17 00:00:00 2001 From: Trenton H Date: Mon, 17 Oct 2022 12:42:08 -0700 Subject: [PATCH 2/6] Connects up the celery signals to support pending, started and success/failure, without relying on django-celery-results --- src-ui/src/app/data/paperless-task.ts | 2 +- ...e_paperlesstask_attempted_task_and_more.py | 134 ++++++++++++++++++ src/documents/models.py | 83 +++++++++-- src/documents/serialisers.py | 111 +-------------- src/documents/signals/handlers.py | 106 ++++++++++++-- src/documents/tests/test_api.py | 87 ++++++------ src/documents/tests/test_task_signals.py | 126 ++++++++++++++++ src/documents/views.py | 3 +- 8 files changed, 479 insertions(+), 173 deletions(-) create mode 100644 src/documents/migrations/1027_remove_paperlesstask_attempted_task_and_more.py create mode 100644 src/documents/tests/test_task_signals.py diff --git a/src-ui/src/app/data/paperless-task.ts b/src-ui/src/app/data/paperless-task.ts index ccf09bb6f..993eb3f1e 100644 --- a/src-ui/src/app/data/paperless-task.ts +++ b/src-ui/src/app/data/paperless-task.ts @@ -21,7 +21,7 @@ export interface PaperlessTask extends ObjectWithId { task_id: string - name: string + task_file_name: string date_created: Date diff --git a/src/documents/migrations/1027_remove_paperlesstask_attempted_task_and_more.py b/src/documents/migrations/1027_remove_paperlesstask_attempted_task_and_more.py new file mode 100644 index 000000000..fc8ff8ec2 --- /dev/null +++ b/src/documents/migrations/1027_remove_paperlesstask_attempted_task_and_more.py @@ -0,0 +1,134 @@ +# Generated by Django 4.1.2 on 2022-10-17 16:31 + +from django.db import migrations, models +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + ("documents", "1026_transition_to_celery"), + ] + + operations = [ + migrations.RemoveField( + model_name="paperlesstask", + name="attempted_task", + ), + migrations.AddField( + model_name="paperlesstask", + name="date_created", + field=models.DateTimeField( + default=django.utils.timezone.now, + help_text="Datetime field when the task result was created in UTC", + null=True, + verbose_name="Created DateTime", + ), + ), + migrations.AddField( + model_name="paperlesstask", + name="date_done", + field=models.DateTimeField( + default=None, + help_text="Datetime field when the task was completed in UTC", + null=True, + verbose_name="Completed DateTime", + ), + ), + migrations.AddField( + model_name="paperlesstask", + name="date_started", + field=models.DateTimeField( + default=None, + help_text="Datetime field when the task was started in UTC", + null=True, + verbose_name="Started DateTime", + ), + ), + migrations.AddField( + model_name="paperlesstask", + name="result", + field=models.TextField( + default=None, + help_text="The data returned by the task", + null=True, + verbose_name="Result Data", + ), + ), + migrations.AddField( + model_name="paperlesstask", + name="status", + field=models.CharField( + choices=[ + ("FAILURE", "FAILURE"), + ("PENDING", "PENDING"), + ("RECEIVED", "RECEIVED"), + ("RETRY", "RETRY"), + ("REVOKED", "REVOKED"), + ("STARTED", "STARTED"), + ("SUCCESS", "SUCCESS"), + ], + default="PENDING", + help_text="Current state of the task being run", + max_length=30, + verbose_name="Task State", + ), + ), + migrations.AddField( + model_name="paperlesstask", + name="task_args", + field=models.JSONField( + help_text="JSON representation of the positional arguments used with the task", + null=True, + verbose_name="Task Positional Arguments", + ), + ), + migrations.AddField( + model_name="paperlesstask", + name="task_file_name", + field=models.CharField( + help_text="Name of the file which the Task was run for", + max_length=255, + null=True, + verbose_name="Task Name", + ), + ), + migrations.AddField( + model_name="paperlesstask", + name="task_kwargs", + field=models.JSONField( + help_text="JSON representation of the named arguments used with the task", + null=True, + verbose_name="Task Named Arguments", + ), + ), + migrations.AddField( + model_name="paperlesstask", + name="task_name", + field=models.CharField( + help_text="Name of the Task which was run", + max_length=255, + null=True, + verbose_name="Task Name", + ), + ), + migrations.AlterField( + model_name="paperlesstask", + name="acknowledged", + field=models.BooleanField( + default=False, + help_text="If the task is acknowledged via the frontend or API", + verbose_name="Acknowledged", + ), + ), + migrations.AlterField( + model_name="paperlesstask", + name="task_id", + field=models.CharField( + help_text="Celery ID for the Task that was run", + max_length=255, + unique=True, + verbose_name="Task ID", + ), + ), + ] diff --git a/src/documents/models.py b/src/documents/models.py index 5a84c467b..c1b9c88bc 100644 --- a/src/documents/models.py +++ b/src/documents/models.py @@ -7,14 +7,17 @@ from typing import Optional import dateutil.parser import pathvalidate +from celery import states from django.conf import settings from django.contrib.auth.models import User from django.db import models from django.utils import timezone from django.utils.translation import gettext_lazy as _ -from django_celery_results.models import TaskResult from documents.parsers import get_default_file_extension +ALL_STATES = sorted(states.ALL_STATES) +TASK_STATE_CHOICES = sorted(zip(ALL_STATES, ALL_STATES)) + class MatchingModel(models.Model): @@ -527,15 +530,79 @@ class UiSettings(models.Model): class PaperlessTask(models.Model): - task_id = models.CharField(max_length=128) - acknowledged = models.BooleanField(default=False) + task_id = models.CharField( + max_length=255, + unique=True, + verbose_name=_("Task ID"), + help_text=_("Celery ID for the Task that was run"), + ) - attempted_task = models.OneToOneField( - TaskResult, - on_delete=models.CASCADE, - related_name="attempted_task", + acknowledged = models.BooleanField( + default=False, + verbose_name=_("Acknowledged"), + help_text=_("If the task is acknowledged via the frontend or API"), + ) + + task_file_name = models.CharField( null=True, - blank=True, + max_length=255, + verbose_name=_("Task Name"), + help_text=_("Name of the file which the Task was run for"), + ) + + task_name = models.CharField( + null=True, + max_length=255, + verbose_name=_("Task Name"), + help_text=_("Name of the Task which was run"), + ) + + task_args = models.JSONField( + null=True, + verbose_name=_("Task Positional Arguments"), + help_text=_( + "JSON representation of the positional arguments used with the task", + ), + ) + task_kwargs = models.JSONField( + null=True, + verbose_name=_("Task Named Arguments"), + help_text=_( + "JSON representation of the named arguments used with the task", + ), + ) + status = models.CharField( + max_length=30, + default=states.PENDING, + choices=TASK_STATE_CHOICES, + verbose_name=_("Task State"), + help_text=_("Current state of the task being run"), + ) + date_created = models.DateTimeField( + null=True, + default=timezone.now, + verbose_name=_("Created DateTime"), + help_text=_("Datetime field when the task result was created in UTC"), + ) + date_started = models.DateTimeField( + null=True, + default=None, + verbose_name=_("Started DateTime"), + help_text=_("Datetime field when the task was started in UTC"), + ) + date_done = models.DateTimeField( + null=True, + default=None, + verbose_name=_("Completed DateTime"), + help_text=_("Datetime field when the task was completed in UTC"), + ) + result = models.TextField( + null=True, + default=None, + verbose_name=_("Result Data"), + help_text=_( + "The data returned by the task", + ), ) diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py index 5f59d2c17..db282cacd 100644 --- a/src/documents/serialisers.py +++ b/src/documents/serialisers.py @@ -1,12 +1,6 @@ import datetime import math import re -from ast import literal_eval -from asyncio.log import logger -from pathlib import Path -from typing import Dict -from typing import Optional -from typing import Tuple from celery import states @@ -640,14 +634,13 @@ class TasksViewSerializer(serializers.ModelSerializer): fields = ( "id", "task_id", + "task_file_name", "date_created", "date_done", "type", "status", "result", "acknowledged", - "task_name", - "name", "related_document", ) @@ -657,108 +650,14 @@ class TasksViewSerializer(serializers.ModelSerializer): # just file tasks, for now return "file" - result = serializers.SerializerMethodField() - - def get_result(self, obj): - result = "" - if ( - hasattr(obj, "attempted_task") - and obj.attempted_task - and obj.attempted_task.result - ): - try: - result: str = obj.attempted_task.result - if "exc_message" in result: - # This is a dict in this case - result: Dict = literal_eval(result) - # This is a list, grab the first item (most recent) - result = result["exc_message"][0] - except Exception as e: # pragma: no cover - # Extra security if something is malformed - logger.warn(f"Error getting task result: {e}", exc_info=True) - return result - - status = serializers.SerializerMethodField() - - def get_status(self, obj): - result = "unknown" - if hasattr(obj, "attempted_task") and obj.attempted_task: - result = obj.attempted_task.status - return result - - date_created = serializers.SerializerMethodField() - - def get_date_created(self, obj): - result = "" - if hasattr(obj, "attempted_task") and obj.attempted_task: - result = obj.attempted_task.date_created - return result - - date_done = serializers.SerializerMethodField() - - def get_date_done(self, obj): - result = "" - if hasattr(obj, "attempted_task") and obj.attempted_task: - result = obj.attempted_task.date_done - return result - - task_id = serializers.SerializerMethodField() - - def get_task_id(self, obj): - result = "" - if hasattr(obj, "attempted_task") and obj.attempted_task: - result = obj.attempted_task.task_id - return result - - task_name = serializers.SerializerMethodField() - - def get_task_name(self, obj): - result = "" - if hasattr(obj, "attempted_task") and obj.attempted_task: - result = obj.attempted_task.task_name - return result - - name = serializers.SerializerMethodField() - - def get_name(self, obj): - result = "" - if hasattr(obj, "attempted_task") and obj.attempted_task: - try: - task_kwargs: Optional[str] = obj.attempted_task.task_kwargs - # Try the override filename first (this is a webui created task?) - if task_kwargs is not None: - # It's a string, string of a dict. Who knows why... - kwargs = literal_eval(literal_eval(task_kwargs)) - if "override_filename" in kwargs: - result = kwargs["override_filename"] - - # Nothing was found, report the task first argument - if not len(result): - # There are always some arguments to the consume - task_args: Tuple = literal_eval( - literal_eval(obj.attempted_task.task_args), - ) - filepath = Path(task_args[0]) - result = filepath.name - except Exception as e: # pragma: no cover - # Extra security if something is malformed - logger.warning(f"Error getting file name from task: {e}", exc_info=True) - - return result - related_document = serializers.SerializerMethodField() + related_doc_re = re.compile(r"New document id (\d+) created") def get_related_document(self, obj): - result = "" - regexp = r"New document id (\d+) created" - if ( - hasattr(obj, "attempted_task") - and obj.attempted_task - and obj.attempted_task.result - and obj.attempted_task.status == states.SUCCESS - ): + result = None + if obj.status is not None and obj.status == states.SUCCESS: try: - result = re.search(regexp, obj.attempted_task.result).group(1) + result = self.related_doc_re.search(obj.result).group(1) except Exception: pass diff --git a/src/documents/signals/handlers.py b/src/documents/signals/handlers.py index 76ae974c6..1b180626a 100644 --- a/src/documents/signals/handlers.py +++ b/src/documents/signals/handlers.py @@ -1,7 +1,13 @@ import logging import os import shutil +from ast import literal_eval +from pathlib import Path +from celery import states +from celery.signals import before_task_publish +from celery.signals import task_postrun +from celery.signals import task_prerun from django.conf import settings from django.contrib.admin.models import ADDITION from django.contrib.admin.models import LogEntry @@ -13,7 +19,6 @@ from django.db.models import Q from django.dispatch import receiver from django.utils import termcolors from django.utils import timezone -from django_celery_results.models import TaskResult from filelock import FileLock from .. import matching @@ -502,19 +507,94 @@ def add_to_index(sender, document, **kwargs): index.add_or_update_document(document) -@receiver(models.signals.post_save, sender=TaskResult) -def update_paperless_task(sender, instance: TaskResult, **kwargs): +@before_task_publish.connect +def before_task_publish_handler(sender=None, headers=None, body=None, **kwargs): + """ + Creates the PaperlessTask object in a pending state. This is sent before + the task reaches the broker, but + + https://docs.celeryq.dev/en/stable/userguide/signals.html#before-task-publish + + """ + if "task" not in headers or headers["task"] != "documents.tasks.consume_file": + # Assumption: this is only ever a v2 message + return + try: - if instance.task_name == "documents.tasks.consume_file": - paperless_task, _ = PaperlessTask.objects.get_or_create( - task_id=instance.task_id, - ) - paperless_task.name = instance.task_name - paperless_task.created = instance.date_created - paperless_task.completed = instance.date_done - paperless_task.attempted_task = instance - paperless_task.save() - except Exception as e: + task_file_name = "" + if headers["kwargsrepr"] is not None: + task_kwargs = literal_eval(headers["kwargsrepr"]) + if "override_filename" in task_kwargs: + task_file_name = task_kwargs["override_filename"] + else: + task_kwargs = None + + task_args = literal_eval(headers["argsrepr"]) + + # Nothing was found, report the task first argument + if not len(task_file_name): + # There are always some arguments to the consume, first is always filename + filepath = Path(task_args[0]) + task_file_name = filepath.name + + PaperlessTask.objects.create( + task_id=headers["id"], + status=states.PENDING, + task_file_name=task_file_name, + task_name=headers["task"], + task_args=task_args, + task_kwargs=task_kwargs, + result=None, + date_created=timezone.now(), + date_started=None, + date_done=None, + ) + except Exception as e: # pragma: no cover # Don't let an exception in the signal handlers prevent # a document from being consumed. logger.error(f"Creating PaperlessTask failed: {e}") + + +@task_prerun.connect +def task_prerun_handler(sender=None, task_id=None, task=None, **kwargs): + """ + + Updates the PaperlessTask to be started. Sent before the task begins execution + on a worker. + + https://docs.celeryq.dev/en/stable/userguide/signals.html#task-prerun + """ + try: + task_instance = PaperlessTask.objects.filter(task_id=task_id).first() + + if task_instance is not None: + task_instance.status = states.STARTED + task_instance.date_started = timezone.now() + task_instance.save() + except Exception as e: # pragma: no cover + # Don't let an exception in the signal handlers prevent + # a document from being consumed. + logger.error(f"Setting PaperlessTask started failed: {e}") + + +@task_postrun.connect +def task_postrun_handler( + sender=None, task_id=None, task=None, retval=None, state=None, **kwargs +): + """ + Updates the result of the PaperlessTask. + + https://docs.celeryq.dev/en/stable/userguide/signals.html#task-postrun + """ + try: + task_instance = PaperlessTask.objects.filter(task_id=task_id).first() + + if task_instance is not None: + task_instance.status = state + task_instance.result = retval + task_instance.date_done = timezone.now() + task_instance.save() + except Exception as e: # pragma: no cover + # Don't let an exception in the signal handlers prevent + # a document from being consumed. + logger.error(f"Updating PaperlessTask failed: {e}") diff --git a/src/documents/tests/test_api.py b/src/documents/tests/test_api.py index 0a8d72155..d876984bd 100644 --- a/src/documents/tests/test_api.py +++ b/src/documents/tests/test_api.py @@ -32,7 +32,6 @@ from documents.models import PaperlessTask from documents.models import SavedView from documents.models import StoragePath from documents.models import Tag -from django_celery_results.models import TaskResult from documents.models import Comment from documents.models import StoragePath from documents.tests.utils import DirectoriesMixin @@ -2756,19 +2755,16 @@ class TestTasks(APITestCase): THEN: - Attempting and pending tasks are serialized and provided """ - result1 = TaskResult.objects.create( - task_id=str(uuid.uuid4()), - task_name="documents.tasks.some_great_task", - status=celery.states.PENDING, - ) - PaperlessTask.objects.create(attempted_task=result1) - result2 = TaskResult.objects.create( + task1 = PaperlessTask.objects.create( task_id=str(uuid.uuid4()), - task_name="documents.tasks.some_awesome_task", - status=celery.states.STARTED, + task_file_name="task_one.pdf", + ) + + task2 = PaperlessTask.objects.create( + task_id=str(uuid.uuid4()), + task_file_name="task_two.pdf", ) - PaperlessTask.objects.create(attempted_task=result2) response = self.client.get(self.ENDPOINT) @@ -2777,13 +2773,18 @@ class TestTasks(APITestCase): returned_task1 = response.data[1] returned_task2 = response.data[0] - self.assertEqual(returned_task1["task_id"], result1.task_id) - self.assertEqual(returned_task1["status"], celery.states.PENDING) - self.assertEqual(returned_task1["task_name"], result1.task_name) + from pprint import pprint - self.assertEqual(returned_task2["task_id"], result2.task_id) - self.assertEqual(returned_task2["status"], celery.states.STARTED) - self.assertEqual(returned_task2["task_name"], result2.task_name) + pprint(returned_task1) + pprint(returned_task2) + + self.assertEqual(returned_task1["task_id"], task1.task_id) + self.assertEqual(returned_task1["status"], celery.states.PENDING) + self.assertEqual(returned_task1["task_file_name"], task1.task_file_name) + + self.assertEqual(returned_task2["task_id"], task2.task_id) + self.assertEqual(returned_task2["status"], celery.states.PENDING) + self.assertEqual(returned_task2["task_file_name"], task2.task_file_name) def test_acknowledge_tasks(self): """ @@ -2794,12 +2795,10 @@ class TestTasks(APITestCase): THEN: - Task is marked as acknowledged """ - result1 = TaskResult.objects.create( + task = PaperlessTask.objects.create( task_id=str(uuid.uuid4()), - task_name="documents.tasks.some_task", - status=celery.states.PENDING, + task_file_name="task_one.pdf", ) - task = PaperlessTask.objects.create(attempted_task=result1) response = self.client.get(self.ENDPOINT) self.assertEqual(len(response.data), 1) @@ -2822,13 +2821,12 @@ class TestTasks(APITestCase): THEN: - The returned data includes the task result """ - result1 = TaskResult.objects.create( + task = PaperlessTask.objects.create( task_id=str(uuid.uuid4()), - task_name="documents.tasks.some_task", + task_file_name="task_one.pdf", status=celery.states.SUCCESS, result="Success. New document id 1 created", ) - _ = PaperlessTask.objects.create(attempted_task=result1) response = self.client.get(self.ENDPOINT) @@ -2849,17 +2847,12 @@ class TestTasks(APITestCase): THEN: - The returned result is the exception info """ - result1 = TaskResult.objects.create( + task = PaperlessTask.objects.create( task_id=str(uuid.uuid4()), - task_name="documents.tasks.some_task", - status=celery.states.SUCCESS, - result={ - "exc_type": "ConsumerError", - "exc_message": ["test.pdf: Not consuming test.pdf: It is a duplicate."], - "exc_module": "documents.consumer", - }, + task_file_name="task_one.pdf", + status=celery.states.FAILURE, + result="test.pdf: Not consuming test.pdf: It is a duplicate.", ) - _ = PaperlessTask.objects.create(attempted_task=result1) response = self.client.get(self.ENDPOINT) @@ -2883,14 +2876,22 @@ class TestTasks(APITestCase): THEN: - Returned data include the filename """ - result1 = TaskResult.objects.create( + task = PaperlessTask.objects.create( task_id=str(uuid.uuid4()), + task_file_name="test.pdf", task_name="documents.tasks.some_task", status=celery.states.SUCCESS, - task_args="\"('/tmp/paperless/paperless-upload-5iq7skzc',)\"", - task_kwargs="\"{'override_filename': 'test.pdf', 'override_title': None, 'override_correspondent_id': None, 'override_document_type_id': None, 'override_tag_ids': None, 'task_id': '466e8fe7-7193-4698-9fff-72f0340e2082', 'override_created': None}\"", + task_args=("/tmp/paperless/paperless-upload-5iq7skzc",), + task_kwargs={ + "override_filename": "test.pdf", + "override_title": None, + "override_correspondent_id": None, + "override_document_type_id": None, + "override_tag_ids": None, + "task_id": "466e8fe7-7193-4698-9fff-72f0340e2082", + "override_created": None, + }, ) - _ = PaperlessTask.objects.create(attempted_task=result1) response = self.client.get(self.ENDPOINT) @@ -2899,7 +2900,7 @@ class TestTasks(APITestCase): returned_data = response.data[0] - self.assertEqual(returned_data["name"], "test.pdf") + self.assertEqual(returned_data["task_file_name"], "test.pdf") def test_task_name_consume_folder(self): """ @@ -2911,14 +2912,14 @@ class TestTasks(APITestCase): THEN: - Returned data include the filename """ - result1 = TaskResult.objects.create( + task = PaperlessTask.objects.create( task_id=str(uuid.uuid4()), + task_file_name="anothertest.pdf", task_name="documents.tasks.some_task", status=celery.states.SUCCESS, - task_args="\"('/consume/anothertest.pdf',)\"", - task_kwargs="\"{'override_tag_ids': None}\"", + task_args=("/consume/anothertest.pdf",), + task_kwargs={"override_tag_ids": None}, ) - _ = PaperlessTask.objects.create(attempted_task=result1) response = self.client.get(self.ENDPOINT) @@ -2927,4 +2928,4 @@ class TestTasks(APITestCase): returned_data = response.data[0] - self.assertEqual(returned_data["name"], "anothertest.pdf") + self.assertEqual(returned_data["task_file_name"], "anothertest.pdf") diff --git a/src/documents/tests/test_task_signals.py b/src/documents/tests/test_task_signals.py new file mode 100644 index 000000000..8aafc1f12 --- /dev/null +++ b/src/documents/tests/test_task_signals.py @@ -0,0 +1,126 @@ +import celery +from django.test import TestCase +from documents.models import PaperlessTask +from documents.signals.handlers import before_task_publish_handler +from documents.signals.handlers import task_postrun_handler +from documents.signals.handlers import task_prerun_handler +from documents.tests.utils import DirectoriesMixin + + +class TestTaskSignalHandler(DirectoriesMixin, TestCase): + + HEADERS_CONSUME = { + "lang": "py", + "task": "documents.tasks.consume_file", + "id": "52d31e24-9dcc-4c32-9e16-76007e9add5e", + "shadow": None, + "eta": None, + "expires": None, + "group": None, + "group_index": None, + "retries": 0, + "timelimit": [None, None], + "root_id": "52d31e24-9dcc-4c32-9e16-76007e9add5e", + "parent_id": None, + "argsrepr": "('/consume/hello-999.pdf',)", + "kwargsrepr": "{'override_tag_ids': None}", + "origin": "gen260@paperless-ngx-dev-webserver", + "ignore_result": False, + } + + HEADERS_WEB_UI = { + "lang": "py", + "task": "documents.tasks.consume_file", + "id": "6e88a41c-e5f8-4631-9972-68c314512498", + "shadow": None, + "eta": None, + "expires": None, + "group": None, + "group_index": None, + "retries": 0, + "timelimit": [None, None], + "root_id": "6e88a41c-e5f8-4631-9972-68c314512498", + "parent_id": None, + "argsrepr": "('/tmp/paperless/paperless-upload-st9lmbvx',)", + "kwargsrepr": "{'override_filename': 'statement.pdf', 'override_title': None, 'override_correspondent_id': None, 'override_document_type_id': None, 'override_tag_ids': None, 'task_id': 'f5622ca9-3707-4ed0-b418-9680b912572f', 'override_created': None}", + "origin": "gen342@paperless-ngx-dev-webserver", + "ignore_result": False, + } + + def util_call_before_task_publish_handler(self, headers_to_use): + self.assertEqual(PaperlessTask.objects.all().count(), 0) + + before_task_publish_handler(headers=headers_to_use) + + self.assertEqual(PaperlessTask.objects.all().count(), 1) + + def test_before_task_publish_handler_consume(self): + """ + GIVEN: + - A celery task completed with an exception + WHEN: + - API call is made to get tasks + THEN: + - The returned result is the exception info + """ + self.util_call_before_task_publish_handler(headers_to_use=self.HEADERS_CONSUME) + + task = PaperlessTask.objects.get() + self.assertIsNotNone(task) + self.assertEqual(self.HEADERS_CONSUME["id"], task.task_id) + self.assertListEqual(["/consume/hello-999.pdf"], task.task_args) + self.assertDictEqual({"override_tag_ids": None}, task.task_kwargs) + self.assertEqual("hello-999.pdf", task.task_file_name) + self.assertEqual("documents.tasks.consume_file", task.task_name) + self.assertEqual(celery.states.PENDING, task.status) + + def test_before_task_publish_handler_webui(self): + + self.util_call_before_task_publish_handler(headers_to_use=self.HEADERS_WEB_UI) + + task = PaperlessTask.objects.get() + + self.assertIsNotNone(task) + + self.assertEqual(self.HEADERS_WEB_UI["id"], task.task_id) + self.assertListEqual( + ["/tmp/paperless/paperless-upload-st9lmbvx"], + task.task_args, + ) + self.assertDictEqual( + { + "override_filename": "statement.pdf", + "override_title": None, + "override_correspondent_id": None, + "override_document_type_id": None, + "override_tag_ids": None, + "task_id": "f5622ca9-3707-4ed0-b418-9680b912572f", + "override_created": None, + }, + task.task_kwargs, + ) + self.assertEqual("statement.pdf", task.task_file_name) + self.assertEqual("documents.tasks.consume_file", task.task_name) + self.assertEqual(celery.states.PENDING, task.status) + + def test_task_prerun_handler(self): + self.util_call_before_task_publish_handler(headers_to_use=self.HEADERS_CONSUME) + + task_prerun_handler(task_id=self.HEADERS_CONSUME["id"]) + + task = PaperlessTask.objects.get() + + self.assertEqual(celery.states.STARTED, task.status) + + def test_task_postrun_handler(self): + self.util_call_before_task_publish_handler(headers_to_use=self.HEADERS_CONSUME) + + task_postrun_handler( + task_id=self.HEADERS_CONSUME["id"], + retval="Success. New document id 1 created", + state=celery.states.SUCCESS, + ) + + task = PaperlessTask.objects.get() + + self.assertEqual(celery.states.SUCCESS, task.status) diff --git a/src/documents/views.py b/src/documents/views.py index 025ff2f67..10225be6f 100644 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -886,9 +886,8 @@ class TasksViewSet(ReadOnlyModelViewSet): queryset = ( PaperlessTask.objects.filter( acknowledged=False, - attempted_task__isnull=False, ) - .order_by("attempted_task__date_created") + .order_by("date_created") .reverse() ) From 1400dba12cc033da54be256d75c40274716c3a1d Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+shamoon@users.noreply.github.com> Date: Mon, 17 Oct 2022 13:53:44 -0700 Subject: [PATCH 3/6] Fix frontend task name display --- src-ui/src/app/components/manage/tasks/tasks.component.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src-ui/src/app/components/manage/tasks/tasks.component.html b/src-ui/src/app/components/manage/tasks/tasks.component.html index 961b8b091..08c065247 100644 --- a/src-ui/src/app/components/manage/tasks/tasks.component.html +++ b/src-ui/src/app/components/manage/tasks/tasks.component.html @@ -53,7 +53,7 @@ - {{ task.name }} + {{ task.task_file_name }} {{ task.date_created | customDate:'short' }}
Date: Sat, 15 Oct 2022 12:53:01 -0700 Subject: [PATCH 4/6] Integrates an optional starting of Flower into the Docker image --- Dockerfile | 3 + Pipfile | 1 + Pipfile.lock | 255 ++++++++++++++++++++--------------- docker/flower-conditional.sh | 7 + docker/supervisord.conf | 18 ++- docs/advanced_usage.rst | 22 +++ docs/configuration.rst | 8 ++ 7 files changed, 203 insertions(+), 111 deletions(-) create mode 100644 docker/flower-conditional.sh diff --git a/Dockerfile b/Dockerfile index 2eeaba1dc..8b562e73e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -151,6 +151,7 @@ COPY [ \ "docker/paperless_cmd.sh", \ "docker/wait-for-redis.py", \ "docker/management_script.sh", \ + "docker/flower-conditional.sh", \ "docker/install_management_commands.sh", \ "/usr/src/paperless/src/docker/" \ ] @@ -170,6 +171,8 @@ RUN set -eux \ && chmod 755 /sbin/wait-for-redis.py \ && mv paperless_cmd.sh /usr/local/bin/paperless_cmd.sh \ && chmod 755 /usr/local/bin/paperless_cmd.sh \ + && mv flower-conditional.sh /usr/local/bin/flower-conditional.sh \ + && chmod 755 /usr/local/bin/flower-conditional.sh \ && echo "Installing managment commands" \ && chmod +x install_management_commands.sh \ && ./install_management_commands.sh diff --git a/Pipfile b/Pipfile index 0660ef30f..0341ef90a 100644 --- a/Pipfile +++ b/Pipfile @@ -60,6 +60,7 @@ django-celery-results = "*" setproctitle = "*" nltk = "*" pdf2image = "*" +flower = "*" [dev-packages] coveralls = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 30c622545..0a139756c 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "68ff2e4e4ebbed482cc7d646337309ffd51140748b83d0b60d22dec9926f2ccb" + "sha256": "113f88e584af57318d6ece2ea82e1b60aaa9b7dfb8b586459714da33d1f8ffa4" }, "pipfile-spec": 6, "requires": {}, @@ -393,6 +393,14 @@ "index": "pypi", "version": "==3.8.0" }, + "flower": { + "hashes": [ + "sha256:46493c7e8d9ca2167e8a46eb97ae8d280997cb40a81993230124d74f0fe40bac", + "sha256:ae2977cf7343c526cf44def8c7e7173db8dedb8249b91ba4b88cfd18e7a2d486" + ], + "index": "pypi", + "version": "==1.2.0" + }, "fuzzywuzzy": { "extras": [ "speedup" @@ -521,6 +529,14 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==10.0" }, + "humanize": { + "hashes": [ + "sha256:8830ebf2d65d0395c1bd4c79189ad71e023f277c2c7ae00f263124432e6f2ffa", + "sha256:efb2584565cc86b7ea87a977a15066de34cdedaf341b11c851cfcfd2b964779c" + ], + "markers": "python_version >= '3.7'", + "version": "==4.4.0" + }, "hyperlink": { "hashes": [ "sha256:427af957daa58bc909471c6c40f74c5450fa123dd093fc53efd2e91d2705a56b", @@ -608,111 +624,111 @@ }, "levenshtein": { "hashes": [ - "sha256:019ae21de930d6077efa1eac746de4df5234e7c6c11ab10080c0935fc5abbecf", - "sha256:02688fff6d256afdd57da5359144ddab8e054b2ba98ddcf147fe191bdf996e88", - "sha256:0274b87df89d1dda8dce77cf05a9dfab7bd30045a09e0d9435ec8be622e374e6", - "sha256:0323e8dbeec4d63c27111796baa7e8a89b391c32d90e67d78f9404d0c8edeab4", - "sha256:053edbb52fe8b8a1a6698c4fee39590c9e44a602ace807291eb87e3b17f85f48", - "sha256:059027f5dd2aafb916301f46a619c7fe03ff5761cdb2d091cf80bf6dbc24bc29", - "sha256:05f11a4be4f668974238cff21208fbd9f629cab8a68b444b7d4a4cfd8081b1d6", - "sha256:0ab71cc5ea86f6685a7b2235edad65f1f2a4b6341109af259d758973d96eece5", - "sha256:0b439f4fb0b615bc0443cc83eaf5835bd480f680c69ed1be963bdb401b8159f8", - "sha256:0ec50d24a12e50857e94ac9035d3c06fd0827bb477b9ebcd83a2a49dd89e5e23", - "sha256:131fc50d52a52acc367ea8bccb028447b734243d00ba1cfc7d9ff8d0dc37fa38", - "sha256:17b5f1d1a4a5ac536283298c98cafc5632ae3897c8601fb2ec8babc6f47a1be9", - "sha256:183b8da9b870ad171a11a629c43e0587a228aea9d595a969231d59bf530b6c77", - "sha256:18888d50813b9df9b8dc8c1506ec40c783db25f130a6101eb89896b27076f751", - "sha256:25b88277832eb558305c3bb986ad61f19b5cb5a87aced289bce4a1701a92aa31", - "sha256:266cdab48e2242b6c010beb8b7af4164aa87f4ad8d6fbd9f4f531214f8ddb234", - "sha256:281bffb09b2e1620db4e99a9df96e38d939c341c7c43cd5191326fbdb4d42275", - "sha256:28cd002cf5a499e6e9bd69d992ffd501b8473948f3e97d6e075b774df1901e8e", - "sha256:2972c6c6a806e0c788f6ec39510abdb61b3a648fd141a5fa77becd2cc05ff551", - "sha256:2b4027b370cc46c4802ba32a979729209c0407d548723e809f19a50a9df27405", - "sha256:318c924e218be754427ce6bb4c630d9dcb5478eb00a8a3f8a0972086adc763b1", - "sha256:380accae56f8c9df99f34bc7e79d286fee37c3dd06b362c394b08ea96371b7c5", - "sha256:3c7784f9936292c9d3f92fc772d874edc071a16cd883ea0d997e5c4318f6362c", - "sha256:3ebd85fd6253abe89f852fc008294d490eb7a5f66913703148b8d263b048cc90", - "sha256:4126c8fe9d817ac3ab223ee5db41a09d0fa82dbd6bb59d207b6f7313d733f19b", - "sha256:4155f0ab246b6892110960f25989ab91073cd708b974f4732dca4d219a8be3e1", - "sha256:41f16267d8e6d916e06a6a1a0e151f643a6bab1277945a4bd494f359d4185dd2", - "sha256:4522f5d662d3ee55a072fad18e2af5dae480658d4e23b04b455c4b7542ce4327", - "sha256:46c900c807b0614c454ba89271ec6f59212403c54dc68ea493ab1ece2c510618", - "sha256:48291b25a904243f37c9aabbfed3eaba466c9a993f5f5946fe647163b7face07", - "sha256:5038a5e9e106087c117f0a7d6fd9d8a382b228da24bbd085b9f2b5d54ab11c3a", - "sha256:594a26bcf0cb720c16ac6db3fd4b3f411be756f9da7682f2f629089ff15aef18", - "sha256:59706135d3107939effe9f9263bd78c507f4abd7bfb96acc5a7f4176aa0a90d2", - "sha256:5a327d7581696c7a392a8f85cce7e54fa1303f5b79b3b2983abaab309b56cfd6", - "sha256:5eca8a45d38c916783c44e5da06a367b77234efa51d84dda8804654b99efecc9", - "sha256:5fa85f6789178ede5333568cbee5bac5fa9718d5f02406b65545e83368fa8fe9", - "sha256:65097e45ef7a942a9b92999b81d2e91fe80cbd0616215e625af39d2166692018", - "sha256:65cc9938cb9bd8862fc220e0719fd7f9c291d788f0a62bb8840820c46fa5a4d0", - "sha256:6a4c3607e2a0e66337d8ddf95ca7efe9b30ebf944119a4fb86503ea66f777263", - "sha256:72f11a136f148eb1218e7d1492749b8b5594302010db0cebd47423c4ac8c79ee", - "sha256:78b5a71de59e30c697a64c69fc48b032bb99c43b7437091b808a9ba20bb0235c", - "sha256:7b212edc9bf9d0c25cc3117483289b9e1a49a1ed134a02635baa987e9f0d89db", - "sha256:7e0f7045c420abdea249a28384baa846b87bad5c9f42af1957dc50c6e337fa1a", - "sha256:7e83cfec424f546dc3f0cc71896f8cc384a711f4116bc1abb0598302a9af3240", - "sha256:80c55bcc31d21bd07f7d1589e11f2ac1faf3359cf9f93026a1944ee76a40f954", - "sha256:863740d7f45adfd29b95658a680b16113721eaa89857c67e7e9573c61e87bbd8", - "sha256:88484b8c3f71dc9205d0d36da541e2cdcf4bc74474a2ee8d99c2e6411b659b89", - "sha256:8a08810e0bcc606d10cf1c5389c96fc92362244c0cf761358c495c2eb29df3dc", - "sha256:8c0637ae4fcb54d5c7fc9af24d348003b6f9dbaf7a06bf13f769d7b85903af39", - "sha256:8e9e3409338a42e3d4c30c224fdb678364542c77994f089fd6cc8131969eff48", - "sha256:902ea10ba85e014dc5d23a7bbb3ab70722349561e73783dd71571359e8867244", - "sha256:9533db74a2685169380db3db3ab59643453e7c486fffa9bf3ab60b73c4e174be", - "sha256:97f02ff49d1fa21308207a7743bec4fdd7aa90e8dd091539da660fc51e624c4d", - "sha256:9ea9a2a154dc7d8658930fa87cda0e6094235b5e130f037d9894eaf8722119a5", - "sha256:a0440d847b2c9986e4d27e8a59164714e5198530c69a5f9fb2e4620f9136d653", - "sha256:a6d39a27b542a781d691827b955d685d496fb6cccfc6eecc336a78b399032062", - "sha256:a7f4d3c478b1fcf412bf6c82914b02fed33ab359120df9172dda7bc855227461", - "sha256:ad297807bbdffce61b04e5e0c22f3c5d9e1905c1ee186f1f6d029f83bf0f18b8", - "sha256:add6778bb51efb80174937543754d2dfa0f4e504e7302d97896006a642c14f95", - "sha256:ae075ebf7bb5f48b3bd2fc9cd53346e4ff43e2515a4f822914bbc62a3cbd6e7e", - "sha256:b26fb439a7fbb522af63bbd781fbf51ec0c0659134a93f5bc8e9e68641df811e", - "sha256:b2bac59721d246939b21274229b9923aeae3db97b6118da739c658c17e110dd6", - "sha256:b314ad1f0667715e8d1b6197d5336ab579b13e801172721d62331bd40034a30c", - "sha256:b7317035875bd7c4705e2566848b2043b78e18f2f5675ea651f9f7805b5589eb", - "sha256:b8e936e620e5f336a207e08c0da9dace5d4dbcc8e64743ab1acaa77a64bbf060", - "sha256:b906da4e9a7ba4ec33ed2f7238343866932c1a6f84944c804252b2922708d0ee", - "sha256:ba690e4e33c360fcf0b8411ca90f8b9cc595e8deddd6a25a9a75a725b698cd6a", - "sha256:bb14da3d63da994c34cfa47cde469df8013ddf5f575455a22530c8c4a0ed8616", - "sha256:bbc2e1632f4a61fa171ddab3bc8368fb8475e7ce68733ca92fec862fdd8e0f60", - "sha256:bbdd3c896db09993b7879cd35e56da6ed8918d161d6e80f9d9c40d78d34e4784", - "sha256:bcaaa8e542cb7e1962d0a58ce6a25f6b4b6ca2e5ce743155fc1f6eb2fea52574", - "sha256:bee682ab1005aff597946234e47c95fcf0f44d2b1f38075f0aba26bbc4e7545a", - "sha256:bfec6543d60c57e7543d9cbccdd5dfcf562f2c05cd6b814df68108a20794e254", - "sha256:c2e50baf7be8831524a87beec6c1873539519a1948f907dc3d4b9be27ebacb80", - "sha256:c6c79a6138be017d85f3bab1df735669b669a38f9b3ff646a1f179afbacb7b63", - "sha256:c702fb7c8bfd87c9ce9c8bddfc9a5796a492bab35a52b1693adee413721e32f2", - "sha256:c9ba1725826f6571a6e4c1561bb1613711f0058b91927a147dc42c637ba087d9", - "sha256:cf205ac52cb6b45745c0a4891cdb6e709c10ad5b034aa736aff561fc4ce9828c", - "sha256:d0d03fc67499ee90feedfa2add4aaa1c091a7bf333535d847b10fffe390e58fe", - "sha256:d118d63f08fd6ac285cb8166e96c992a6ed0e7a1644e8790c39070b18779e688", - "sha256:d24c09f397c3ce55f20e0250da7ba5b0e5249cb5d21465e71ec15154a3a7e8e0", - "sha256:d41735c7a646dae8612e0552dfc53f45807eeb54364dfb1f0a65ac274bc56b3a", - "sha256:dd1696d91f2a37cece9bd22e507e7be7c37c59ecc61fd15f0d0f31e3b6888957", - "sha256:dfcad9c63a893c95ba1149481b9680ce68dd71211f08df0073ee62700790bc97", - "sha256:e384782608837d9aaf123e413679883091744664a2cd76f0ad0e0a1f12facc57", - "sha256:e5ea0abea338c617b753082f36f64c70ade853d88e91ab5732b301ae8ed16e3f", - "sha256:e6ff81c570413bcc35f1c16850eb66e2493a3259e68efe8672376533d2c82d38", - "sha256:e88951ad2831880405f3f055ab12a6aa72696c20a2815128eeccdc3bf914cd78", - "sha256:e98e16b6ce531b12100c01daac922e8ec5b991832a5f58003f13b7d45ea82dc0", - "sha256:eb0fd32e8e433797499571447d9f975b4744be79c0a3339413868d79517231ed", - "sha256:ee74a73e1f9e16b71f67329e99bb58aa4af9a2c3c4b3a5db9f26e92e7c39e161", - "sha256:f15ec5f825c283a5aa427d78759ab8f84e7b5441d15cfff476b548bce3764666", - "sha256:f296c7fe928ce0e29e313f85c43a5ab80542e096e1163c2605b8cc18aa2aff2b", - "sha256:f32df1b19f773bb41382e8b215955d248c9766e3d6ff5a1dd89709e7d96e4685", - "sha256:f3ed67279a4b317a808ac743d3a915f74187530c5f3d9c859e5d04d475b8c174", - "sha256:f5b972ca514898fb7131671c425a62ca38fdae2a8d6296e4b605ec8202349f8c", - "sha256:f961086c0dbba6c00cbd5c5b5646247efd0d0a4044444bfaa9efc7a6ba5e96a5", - "sha256:f9bd7d7a449667d6f17edd9045ec82a4ed2767afb91743d3d0b18c376a56dfe2", - "sha256:fbac4c8ffadb685189efa92fafdb2f5392e9cbd262eae3818bcdb1bd19acaaf2", - "sha256:fc43c8276d0a7c7b76f31d4f3f80f9eb820673628f1411770a70029c1d5f6a75", - "sha256:fcfded324f0710632e22050a2fd7b56b1cbcb2d21001630bcc26d536f54bffec", - "sha256:ff435abdcbfdf4a070f488830cd53aef77cf8649d0fd8ed76bf27d9566e80e78" + "sha256:01fa4a3a56beaa3f1c606ecc824b87b079e0459a2f7f24d971c99a2a801f45e3", + "sha256:07af31c2e57101389c5ef969db8154769cc86b8b50b2355417516c765a77929e", + "sha256:0bba5ac3cf99541cf26c2241e25d67116c9b8816afe51335bb36c11cbe4729c9", + "sha256:0d53424cbe0c7337a5f046d427988670707f022a57cab9960d157941fea9cf1f", + "sha256:0ec0c1c9c177b0173412743c270c8b0a4df80b2374dec35b8555e27da8ffe3cf", + "sha256:1076fef5396dbd2fb7bd982253de94c6abf929a5227f8d913e2fc74224e05d6f", + "sha256:146b070ab548ec4cc4954001087d7b493241aefa305856d131914fe9527ed97b", + "sha256:164956a22c93c819fa975b65e0c38631fc9e13964ebbf30a1ec228fd5d197e79", + "sha256:1a3050a28500d6b60045438485f66a73b15ac4ac0a42429572d548d7f30d934f", + "sha256:1c372907ffad9553af86b16941de8f7cd2edc19d5aaa62a2547debb6c347ec4c", + "sha256:1fcd7ba7012dac14a30ab28abaa293255860ff8fd82dbd0c5871cb9eccc0cf3b", + "sha256:2222a8b3cd2a13022d5346dcff53d6d93edb6ca59361a0b2ccf1b7f522d986b7", + "sha256:2459cfae2f5c2de5b3462755526039e92e7ae5d2e0be119b90bad6274c47c362", + "sha256:2aec604d623a7b743df4300b6da09276bd6cbd7d312424983fe2fbc2e7772682", + "sha256:2b8d19dcbae5fc223889acd1dc2be5993563d287421c94962e43224d85c09ce8", + "sha256:2d8751e0460d1e3ba5d5ca61b5c5fc31ae6a0f930cdc702a910909e413321661", + "sha256:2ea460ba728b1a5f493c8982996b1ddf56fe79e998a60ea91e8f95c18522ee0f", + "sha256:2f3db10c9bab6a4b185ec5018bf1a31a35abd870cc5c6beef22b8b53a7791b77", + "sha256:2f9e6904fae94e19f651b6f0739124d2ebe9baeec7f87b2be2fb4588fec8e10b", + "sha256:308fea61891a93771984ae500332cc9525cdded91b1509b92befb938e38ac8e1", + "sha256:3124f47cfa2f3dae69652a1b607b9172411e1682ae15f07a42ca1857fc3720d3", + "sha256:3214b44a428ba58f7f32bbed09e674e690ca6a36495b0a25833488fe80182f20", + "sha256:3667b033604a3b911f8312b1060ea1647cd93f64d6b4a6bb2edc866f7bdc39e9", + "sha256:37f7f9e201a1779c96919b9c021291682010138f822a9387f92f97ca209d4841", + "sha256:382bcae76e31ebb927e93f2f9fab590153ab0b04063cdcf2d298ebaf4749c3ec", + "sha256:3e86489dced79a8ba8822ad6103039a27d6d5c96a84cfe9adfea68e3589c40f5", + "sha256:418ae3b359b099367a552f8d5e0647928b440462b45bd9493283031b96095245", + "sha256:4a2947e59b1d6ce07281fab36adbf476a56faf2c3fb7dee7cebbea6c74baa02c", + "sha256:4d1df08e3ea2001da1a26c1f93c0a0f86d1fd5611cb5484f8e326ad6b2bf31e1", + "sha256:4e0abba2719ee5aa7b73e79f7c0b49da17d6f998014c6904062ebe3973c466f6", + "sha256:4ec8380f15e8e30fa0cd2d982563467e5d14e6f33e762b10fc15fe66c68a3b12", + "sha256:4f340188c7dfbf11ec29d6d438684e82e2bf112f355f1a655ec0bddeb04d9c59", + "sha256:4ff7ac6ba172f3dc41175d169a7519b030ecf9f5057357a3c9e9b99286005dde", + "sha256:539886f82d915eae0dbb7332dc07d5200bda2bb01e5ccd43f44bc153214072b3", + "sha256:554583bbd5fd5cbad3cbcaebf276561de18b18e234c4ca78274a285a22059530", + "sha256:5731bdb55c8363978636dad2e1f1c22445891c98bdeb1862b745c6572044d2f3", + "sha256:60d8208067f1b922a595f105fb3e582ed666fff66d62db1f6fe58bccd78175c0", + "sha256:610d58d2319f10517bd081dcdf77c663574a47ba4e658d733fe4210cd6220156", + "sha256:66946463ca573059fd98859c644fba0fd5c9db35b9b170eb2f59e044673970db", + "sha256:6ee5c22d3e53f3056025112e60f1434912ac174701bd2731b7b94555bfab46c6", + "sha256:7102e8bbb5d12271a31af6474ab752712da9e205a3e8dc582032801c65e9aa3c", + "sha256:7323a576dac3e478b172fdfbd3556c10830e702e401508c6897a9e0f3b9de155", + "sha256:73b21c8cdb3ec6b0160695e2c714f37ba6e025b9f77e07d9f68fcc9622119932", + "sha256:75a226cbd704cbf8cd80ae8e864fb30e2ad4575c840bdea9c4b98d326eccf31f", + "sha256:7b739747fed011e6fed5a90aa5f91c5a324e7a27a143dcdae545fc5924bd9c89", + "sha256:7c9e936791fd807419003bb1ff4c4fdc5b66f490954e5b036b27385dcda0992d", + "sha256:8005553379285dd0327a440d8dc8dd70886bdddc7c2e222a2b981f0f9091dd89", + "sha256:8115e075729eeae73454f70746cd6cbb82b1bb97a5dc082d4ad2731865db6045", + "sha256:872ca073b0949ad1c748066ebaa3bccf851ddf4bfc695a41f07a560ede86b201", + "sha256:87524199b2eb943d97f6c95de8a682bbf6ca50fddef403492b058232eb5b26f1", + "sha256:8779cb250db24da46690e3faa3f98b891ffaeba44b1a8d90cfccec71517ab79b", + "sha256:8a2e6e508e20d4f85855ae57c840d59b7ddf794a8a4b428554a2fd543162ba55", + "sha256:8c75ee32d2331439121f454067e054c19fa620be5a5d33482f6368d44c890eaf", + "sha256:8da94efa4147f6f97c9e62b4a6651a4456e0d6986ab12f86e15ebb63a88e0b1a", + "sha256:8e4e9b930fec52651ef6abe7026ca4cb7e7f2888d9f7932e5e358e0992a6dbb8", + "sha256:9022a856f7708cbdeed1e3b62f5023de8cdb73a3196547a45d02d16edb69f215", + "sha256:90e7a7372008e92840c011e733ef9328578e8166daaf9d5c7cfe4cc5a4a6a09e", + "sha256:91287780ebfe1ee28b2d842b8068d88cdfed3af64aea200dca173c2e682739ee", + "sha256:93d94345cfbd2598952e9e9d6445821e84f1338fde55e1a436e4bd002b2d8902", + "sha256:94ca61d6830f205e941249162fa16d9c46ce374eaad9dadf7db9c140fffbc845", + "sha256:9661febc58fed5b3ea4daa4a4c3c226872ef6e5c02cecba9df645070bb141301", + "sha256:996b3a2e7615007f236efb31685c4a8f6b1f3e9209e14cb93c8816973ca85a25", + "sha256:9e424e89e0496aba82643f887893fb124caec2948d970d2f55b3aacfee65531d", + "sha256:a12a8b98d008c5ec50273601e3f24a0d89bf12150042e4e1018a538966bac5ab", + "sha256:a2f2269abade6eeca6e50531d63ebbeb5aff665cc5f6bd8b0c0717349b51a00c", + "sha256:a5ea034050124aca163d3591de4701373ee55ce9392220ed6d2c410323e27db0", + "sha256:a6891f9e81ea0ea25b8878bea0ff283f11eecd3bb79c5111bb73f43a02357b8a", + "sha256:a7b02771b64b5643ce4587850bd9a3c05f45804d8249696712a94e1a2bec4d82", + "sha256:aab9931150c5fcba403edfbc8911ab209bcd7f32f240d74d01259b3067966539", + "sha256:ac05a7ea87b8f2287c0493c889579d814dce1f0a90dd4df62bf74b7d146176d8", + "sha256:ac5382154e8def43b9e4ee54af1cfab9a283fd82282dfc9a6518dc5ccc47186f", + "sha256:acc746deeffab6bc3e5e2b0c2ca1334d9c57f9251c16e7ade7526354fbdaa893", + "sha256:b13431d2511386d6e0d7fd1df155cd16002d659058eed1689885e3b782ab7bbd", + "sha256:b7c5015b576a5a64184e87222a0ff0789ebef1bb5307ccb973de676053854a7e", + "sha256:b8adc09eabaf694b6f191ddf44f6f41cdde010c7b74902293012bde6898b4e38", + "sha256:b8b83e59824b3283c43df1c896bdcd9dab3d9f0b5f83026049a2bc9e8f858878", + "sha256:b8e70d1ad398f06728c2ac54ad371f81ffd1d194453a04a8f2c4de906e74ed2b", + "sha256:b9e641ff0aafd268adecf859d3c8049b9779678d0a02216cc797e75dfb8d3855", + "sha256:bbdb2467362c547193edf6295bf1fc9431337f56a007781e65088ff00fb4353d", + "sha256:bedd9ba56197233d67d497dd8c00cf817149c2050d25095d59beaa89fde3ff4d", + "sha256:c1310716ccdede0f6ee4ccb0df323e0109a62a7c80064f2acb7c1a47b37d21ef", + "sha256:c4ef4bebf76351a2a61ec139db828c7facacb1011d24c09dff6161374fc2141d", + "sha256:c72f54e73cfbbad06e4b437be4e5724cdb2a2bc3f4f39391fc50b5a5db4e74d1", + "sha256:c9bcd8a9f5e40f8c733fe2a054f9d48d4b2822d4e895fea7e2c21972d58a02e5", + "sha256:cd6ecb12402f7f7c30b1e8aea5af294b038efc693c50cb9ed392327456c53384", + "sha256:d2522fbc28f7695fb52fdad4cf4bf7d1871669b235833ba13e2073a14f742a15", + "sha256:d41d3db43e819a6abf1329e7f8facc4f3642dd5de00d82c71b36dea2dc5f7293", + "sha256:d6250983af3fa82b04ad8a835ae87cabfdf368fb1c6e840728f907707afc49a9", + "sha256:d62bccb538c4257643fb509badd989c0cb8ece08b54ee6c3886e343b1f55601a", + "sha256:df606a17401e00a535cb1e503c9989b77852b0e62dd8e6c79336a7b42983e357", + "sha256:df78605f2289c36ac0509c5140efda099ae3e3f4d083285d290f054be7272d9b", + "sha256:e127085af0335e0cd6eff2346a4c2ba8f669bb57638491b3a45e85b63f26535b", + "sha256:e2a512fbf5310ce298e772adcc05ab36be742a0ab115a2e975bd4ef03346789c", + "sha256:e78c897e07a49f227432d245400725ce96515ee4a9d94590c3b9097436fb0db2", + "sha256:eb5fb70a1c24289f3b31081bac917b92f00c5243078c9c0d8a3b0bd9bf1a40a9", + "sha256:ede6e2b96c845057314f5ef1abb7fff9516b33427dda3c1ec0740ae92acb8390", + "sha256:f0381423c30f2536c4c74f82f0063307185add97a80de3e859c81f8e57349bdd", + "sha256:f36263a5dfaf226789c8889d69ffa3d5d5959fa1ccf7526a8460fb492722a483", + "sha256:f36f0d0b4e112211160d2f80a1afc4674815677087bf98c0fdc537d9dd2a9cc0", + "sha256:f6d6f3a41a0923e8bcc2ef5c71cd12558c50b3863243cfa4d55e5b42390bf5e0", + "sha256:f79a24b9f76c0d05cf35b6fc6c24e1e690998feb2d9b69d98537bd6227439a43", + "sha256:fbb32f73e01672372b86e935b1e1b1893c53a61d507e640b20906b4cd19a8ef8" ], "markers": "python_version >= '3.6'", - "version": "==0.20.7" + "version": "==0.20.6" }, "lxml": { "hashes": [ @@ -1086,6 +1102,14 @@ "markers": "python_version >= '3'", "version": "==2.5.1" }, + "prometheus-client": { + "hashes": [ + "sha256:be26aa452490cfcf6da953f9436e95a9f2b4d578ca80094b4458930e5f584ab1", + "sha256:db7c05cbd13a0f79975592d112320f2605a325969b270a94b71dcabc47b931d2" + ], + "markers": "python_version >= '3.6'", + "version": "==0.15.0" + }, "prompt-toolkit": { "hashes": [ "sha256:9696f386133df0fc8ca5af4895afe5d78f5fcfe5258111c2a79a1c3e41ffa96d", @@ -1195,10 +1219,10 @@ }, "python-levenshtein": { "hashes": [ - "sha256:88a58b95e3340a918489dac0c78f731323c0a4d8f5564f839ffea80155574e77", - "sha256:9228af5523f797f0798f045dc4a95ed1f46df72bc2186e52b530a33998a51b37" + "sha256:785b1216912392a610fbb20f27a400ee00849244fd8248f40a52d8f0d63d7e28", + "sha256:edfe6f724bf93ba37e177a0766d8dd93867fdb969f10ee4ceacb0f72022fd50f" ], - "version": "==0.20.7" + "version": "==0.20.6" }, "python-magic": { "hashes": [ @@ -1699,6 +1723,23 @@ "index": "pypi", "version": "==1.24" }, + "tornado": { + "hashes": [ + "sha256:1d54d13ab8414ed44de07efecb97d4ef7c39f7438cf5e976ccd356bebb1b5fca", + "sha256:20f638fd8cc85f3cbae3c732326e96addff0a15e22d80f049e00121651e82e72", + "sha256:5c87076709343557ef8032934ce5f637dbb552efa7b21d08e89ae7619ed0eb23", + "sha256:5f8c52d219d4995388119af7ccaa0bcec289535747620116a58d830e7c25d8a8", + "sha256:6fdfabffd8dfcb6cf887428849d30cf19a3ea34c2c248461e1f7d718ad30b66b", + "sha256:87dcafae3e884462f90c90ecc200defe5e580a7fbbb4365eda7c7c1eb809ebc9", + "sha256:9b630419bde84ec666bfd7ea0a4cb2a8a651c2d5cccdbdd1972a0c859dfc3c13", + "sha256:b8150f721c101abdef99073bf66d3903e292d851bee51910839831caba341a75", + "sha256:ba09ef14ca9893954244fd872798b4ccb2367c165946ce2dd7376aebdde8e3ac", + "sha256:d3a2f5999215a3a06a4fc218026cd84c61b8b2b40ac5296a6db1f1451ef04c1e", + "sha256:e5f923aa6a47e133d1cf87d60700889d7eae68988704e20c75fb2d65677a8e4b" + ], + "markers": "python_version >= '3.7'", + "version": "==6.2" + }, "tqdm": { "hashes": [ "sha256:5f4f682a004951c1b450bc753c710e9280c5746ce6ffedee253ddbcbf54cf1e4", diff --git a/docker/flower-conditional.sh b/docker/flower-conditional.sh new file mode 100644 index 000000000..04319a8e3 --- /dev/null +++ b/docker/flower-conditional.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +echo "Checking if we should start flower..." + +if [[ -n "${PAPERLESS_ENABLE_FLOWER}" ]]; then + celery --app paperless flower +fi diff --git a/docker/supervisord.conf b/docker/supervisord.conf index 0199b86fe..bfb78330a 100644 --- a/docker/supervisord.conf +++ b/docker/supervisord.conf @@ -10,7 +10,7 @@ user=root [program:gunicorn] command=gunicorn -c /usr/src/paperless/gunicorn.conf.py paperless.asgi:application user=paperless - +priority = 1 stdout_logfile=/dev/stdout stdout_logfile_maxbytes=0 stderr_logfile=/dev/stderr @@ -20,7 +20,7 @@ stderr_logfile_maxbytes=0 command=python3 manage.py document_consumer user=paperless stopsignal=INT - +priority = 20 stdout_logfile=/dev/stdout stdout_logfile_maxbytes=0 stderr_logfile=/dev/stderr @@ -32,7 +32,7 @@ command = celery --app paperless worker --loglevel INFO user=paperless stopasgroup = true stopwaitsecs = 60 - +priority = 5 stdout_logfile=/dev/stdout stdout_logfile_maxbytes=0 stderr_logfile=/dev/stderr @@ -43,7 +43,17 @@ stderr_logfile_maxbytes=0 command = celery --app paperless beat --loglevel INFO user=paperless stopasgroup = true - +priority = 10 +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 + +[program:celery-flower] +command = /usr/local/bin/flower-conditional.sh +user = paperless +startsecs = 0 +priority = 40 stdout_logfile=/dev/stdout stdout_logfile_maxbytes=0 stderr_logfile=/dev/stderr diff --git a/docs/advanced_usage.rst b/docs/advanced_usage.rst index 4723ff4b7..844a73817 100644 --- a/docs/advanced_usage.rst +++ b/docs/advanced_usage.rst @@ -364,3 +364,25 @@ For simplicity, `By Year` defines the same structure as in the previous example If you adjust the format of an existing storage path, old documents don't get relocated automatically. You need to run the :ref:`document renamer ` to adjust their pathes. + +.. _advanced-celery-monitoring: + +Celery Monitoring +################# + +The monitoring tool `Flower `_ can be used to view more +detailed information about the health of the celery workers used for asynchronous tasks. This includes details +on currently running, queued and completed tasks, timing and more. Flower can also be used with Prometheus, as it +exports metrics. For details on its capabilities, refer to the Flower documentation. + +To configure Flower further, create a `flowerconfig.py` and place it into the `src/paperless` directory. For +a Docker installation, you can use volumes to accomplish this: + +.. code:: yaml + + services: + # ... + webserver: + # ... + volumes: + - /path/to/my/flowerconfig.py:/usr/src/paperless/src/paperless/flowerconfig.py:ro diff --git a/docs/configuration.rst b/docs/configuration.rst index 402a41761..5a379cb05 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -901,6 +901,14 @@ PAPERLESS_OCR_LANGUAGES= Defaults to none, which does not install any additional languages. +PAPERLESS_ENABLE_FLOWER= + If this environment variable is defined, the Celery monitoring tool + `Flower `_ will + be started by the container. + + You can read more about this in the :ref:`advanced setup ` + documentation. + .. _configuration-update-checking: From 0a19ad4edbc48c15ac5ef917555f6951dfef29df Mon Sep 17 00:00:00 2001 From: Trenton H Date: Fri, 21 Oct 2022 11:10:47 -0700 Subject: [PATCH 5/6] Resolves the conflicts with a lock --- Pipfile.lock | 366 +++++++++++++++++++++++++-------------------------- 1 file changed, 182 insertions(+), 184 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 0a139756c..4493544e2 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "113f88e584af57318d6ece2ea82e1b60aaa9b7dfb8b586459714da33d1f8ffa4" + "sha256": "b1011b40814341fe1bf40ad893db947608e5c273cf77faae429cb1a56e4bb73c" }, "pipfile-spec": 6, "requires": {}, @@ -36,11 +36,11 @@ }, "anyio": { "hashes": [ - "sha256:413adf95f93886e442aea925f3ee43baa5a765a64a0f52c6081894f9992fdd0b", - "sha256:cb29b9c70620506a9a8f87a309591713446953302d7d995344d0d7c6c0c9a7be" + "sha256:25ea0d673ae30af41a0c442f81cf3b38c7e79fdc7b60335a4c14e05eb0947421", + "sha256:fbbe32bd270d2a2ef3ed1c5d45041250284e31fc0a4df4a5a6071842051a51e3" ], "markers": "python_full_version >= '3.6.2'", - "version": "==3.6.1" + "version": "==3.6.2" }, "asgiref": { "hashes": [ @@ -218,7 +218,7 @@ "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845", "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==2.1.1" }, "click": { @@ -234,7 +234,7 @@ "sha256:a0713dc7a1de3f06bc0df5a9567ad19ead2d3d5689b434768a6145bff77c0667", "sha256:f184f0d851d96b6d29297354ed981b7dd71df7ff500d82fa6d11f0856bee8035" ], - "markers": "python_version < '4' and python_full_version >= '3.6.2'", + "markers": "python_full_version >= '3.6.2' and python_full_version < '4.0.0'", "version": "==0.3.0" }, "click-plugins": { @@ -316,11 +316,11 @@ }, "dateparser": { "hashes": [ - "sha256:038196b1f12c7397e38aad3d61588833257f6f552baa63a1499e6987fa8d42d9", - "sha256:9600874312ff28a41f96ec7ccdc73be1d1c44435719da47fea3339d55ff5a628" + "sha256:3821bf191f95b2658c4abd91571c09821ce7a2bc179bf6cefd8b4515c3ccf9ef", + "sha256:d31659dc806a7d88e2b510b2c74f68b525ae531f145c62a57a99bd616b7f90cf" ], "index": "pypi", - "version": "==1.1.1" + "version": "==1.1.2" }, "deprecated": { "hashes": [ @@ -624,111 +624,111 @@ }, "levenshtein": { "hashes": [ - "sha256:01fa4a3a56beaa3f1c606ecc824b87b079e0459a2f7f24d971c99a2a801f45e3", - "sha256:07af31c2e57101389c5ef969db8154769cc86b8b50b2355417516c765a77929e", - "sha256:0bba5ac3cf99541cf26c2241e25d67116c9b8816afe51335bb36c11cbe4729c9", - "sha256:0d53424cbe0c7337a5f046d427988670707f022a57cab9960d157941fea9cf1f", - "sha256:0ec0c1c9c177b0173412743c270c8b0a4df80b2374dec35b8555e27da8ffe3cf", - "sha256:1076fef5396dbd2fb7bd982253de94c6abf929a5227f8d913e2fc74224e05d6f", - "sha256:146b070ab548ec4cc4954001087d7b493241aefa305856d131914fe9527ed97b", - "sha256:164956a22c93c819fa975b65e0c38631fc9e13964ebbf30a1ec228fd5d197e79", - "sha256:1a3050a28500d6b60045438485f66a73b15ac4ac0a42429572d548d7f30d934f", - "sha256:1c372907ffad9553af86b16941de8f7cd2edc19d5aaa62a2547debb6c347ec4c", - "sha256:1fcd7ba7012dac14a30ab28abaa293255860ff8fd82dbd0c5871cb9eccc0cf3b", - "sha256:2222a8b3cd2a13022d5346dcff53d6d93edb6ca59361a0b2ccf1b7f522d986b7", - "sha256:2459cfae2f5c2de5b3462755526039e92e7ae5d2e0be119b90bad6274c47c362", - "sha256:2aec604d623a7b743df4300b6da09276bd6cbd7d312424983fe2fbc2e7772682", - "sha256:2b8d19dcbae5fc223889acd1dc2be5993563d287421c94962e43224d85c09ce8", - "sha256:2d8751e0460d1e3ba5d5ca61b5c5fc31ae6a0f930cdc702a910909e413321661", - "sha256:2ea460ba728b1a5f493c8982996b1ddf56fe79e998a60ea91e8f95c18522ee0f", - "sha256:2f3db10c9bab6a4b185ec5018bf1a31a35abd870cc5c6beef22b8b53a7791b77", - "sha256:2f9e6904fae94e19f651b6f0739124d2ebe9baeec7f87b2be2fb4588fec8e10b", - "sha256:308fea61891a93771984ae500332cc9525cdded91b1509b92befb938e38ac8e1", - "sha256:3124f47cfa2f3dae69652a1b607b9172411e1682ae15f07a42ca1857fc3720d3", - "sha256:3214b44a428ba58f7f32bbed09e674e690ca6a36495b0a25833488fe80182f20", - "sha256:3667b033604a3b911f8312b1060ea1647cd93f64d6b4a6bb2edc866f7bdc39e9", - "sha256:37f7f9e201a1779c96919b9c021291682010138f822a9387f92f97ca209d4841", - "sha256:382bcae76e31ebb927e93f2f9fab590153ab0b04063cdcf2d298ebaf4749c3ec", - "sha256:3e86489dced79a8ba8822ad6103039a27d6d5c96a84cfe9adfea68e3589c40f5", - "sha256:418ae3b359b099367a552f8d5e0647928b440462b45bd9493283031b96095245", - "sha256:4a2947e59b1d6ce07281fab36adbf476a56faf2c3fb7dee7cebbea6c74baa02c", - "sha256:4d1df08e3ea2001da1a26c1f93c0a0f86d1fd5611cb5484f8e326ad6b2bf31e1", - "sha256:4e0abba2719ee5aa7b73e79f7c0b49da17d6f998014c6904062ebe3973c466f6", - "sha256:4ec8380f15e8e30fa0cd2d982563467e5d14e6f33e762b10fc15fe66c68a3b12", - "sha256:4f340188c7dfbf11ec29d6d438684e82e2bf112f355f1a655ec0bddeb04d9c59", - "sha256:4ff7ac6ba172f3dc41175d169a7519b030ecf9f5057357a3c9e9b99286005dde", - "sha256:539886f82d915eae0dbb7332dc07d5200bda2bb01e5ccd43f44bc153214072b3", - "sha256:554583bbd5fd5cbad3cbcaebf276561de18b18e234c4ca78274a285a22059530", - "sha256:5731bdb55c8363978636dad2e1f1c22445891c98bdeb1862b745c6572044d2f3", - "sha256:60d8208067f1b922a595f105fb3e582ed666fff66d62db1f6fe58bccd78175c0", - "sha256:610d58d2319f10517bd081dcdf77c663574a47ba4e658d733fe4210cd6220156", - "sha256:66946463ca573059fd98859c644fba0fd5c9db35b9b170eb2f59e044673970db", - "sha256:6ee5c22d3e53f3056025112e60f1434912ac174701bd2731b7b94555bfab46c6", - "sha256:7102e8bbb5d12271a31af6474ab752712da9e205a3e8dc582032801c65e9aa3c", - "sha256:7323a576dac3e478b172fdfbd3556c10830e702e401508c6897a9e0f3b9de155", - "sha256:73b21c8cdb3ec6b0160695e2c714f37ba6e025b9f77e07d9f68fcc9622119932", - "sha256:75a226cbd704cbf8cd80ae8e864fb30e2ad4575c840bdea9c4b98d326eccf31f", - "sha256:7b739747fed011e6fed5a90aa5f91c5a324e7a27a143dcdae545fc5924bd9c89", - "sha256:7c9e936791fd807419003bb1ff4c4fdc5b66f490954e5b036b27385dcda0992d", - "sha256:8005553379285dd0327a440d8dc8dd70886bdddc7c2e222a2b981f0f9091dd89", - "sha256:8115e075729eeae73454f70746cd6cbb82b1bb97a5dc082d4ad2731865db6045", - "sha256:872ca073b0949ad1c748066ebaa3bccf851ddf4bfc695a41f07a560ede86b201", - "sha256:87524199b2eb943d97f6c95de8a682bbf6ca50fddef403492b058232eb5b26f1", - "sha256:8779cb250db24da46690e3faa3f98b891ffaeba44b1a8d90cfccec71517ab79b", - "sha256:8a2e6e508e20d4f85855ae57c840d59b7ddf794a8a4b428554a2fd543162ba55", - "sha256:8c75ee32d2331439121f454067e054c19fa620be5a5d33482f6368d44c890eaf", - "sha256:8da94efa4147f6f97c9e62b4a6651a4456e0d6986ab12f86e15ebb63a88e0b1a", - "sha256:8e4e9b930fec52651ef6abe7026ca4cb7e7f2888d9f7932e5e358e0992a6dbb8", - "sha256:9022a856f7708cbdeed1e3b62f5023de8cdb73a3196547a45d02d16edb69f215", - "sha256:90e7a7372008e92840c011e733ef9328578e8166daaf9d5c7cfe4cc5a4a6a09e", - "sha256:91287780ebfe1ee28b2d842b8068d88cdfed3af64aea200dca173c2e682739ee", - "sha256:93d94345cfbd2598952e9e9d6445821e84f1338fde55e1a436e4bd002b2d8902", - "sha256:94ca61d6830f205e941249162fa16d9c46ce374eaad9dadf7db9c140fffbc845", - "sha256:9661febc58fed5b3ea4daa4a4c3c226872ef6e5c02cecba9df645070bb141301", - "sha256:996b3a2e7615007f236efb31685c4a8f6b1f3e9209e14cb93c8816973ca85a25", - "sha256:9e424e89e0496aba82643f887893fb124caec2948d970d2f55b3aacfee65531d", - "sha256:a12a8b98d008c5ec50273601e3f24a0d89bf12150042e4e1018a538966bac5ab", - "sha256:a2f2269abade6eeca6e50531d63ebbeb5aff665cc5f6bd8b0c0717349b51a00c", - "sha256:a5ea034050124aca163d3591de4701373ee55ce9392220ed6d2c410323e27db0", - "sha256:a6891f9e81ea0ea25b8878bea0ff283f11eecd3bb79c5111bb73f43a02357b8a", - "sha256:a7b02771b64b5643ce4587850bd9a3c05f45804d8249696712a94e1a2bec4d82", - "sha256:aab9931150c5fcba403edfbc8911ab209bcd7f32f240d74d01259b3067966539", - "sha256:ac05a7ea87b8f2287c0493c889579d814dce1f0a90dd4df62bf74b7d146176d8", - "sha256:ac5382154e8def43b9e4ee54af1cfab9a283fd82282dfc9a6518dc5ccc47186f", - "sha256:acc746deeffab6bc3e5e2b0c2ca1334d9c57f9251c16e7ade7526354fbdaa893", - "sha256:b13431d2511386d6e0d7fd1df155cd16002d659058eed1689885e3b782ab7bbd", - "sha256:b7c5015b576a5a64184e87222a0ff0789ebef1bb5307ccb973de676053854a7e", - "sha256:b8adc09eabaf694b6f191ddf44f6f41cdde010c7b74902293012bde6898b4e38", - "sha256:b8b83e59824b3283c43df1c896bdcd9dab3d9f0b5f83026049a2bc9e8f858878", - "sha256:b8e70d1ad398f06728c2ac54ad371f81ffd1d194453a04a8f2c4de906e74ed2b", - "sha256:b9e641ff0aafd268adecf859d3c8049b9779678d0a02216cc797e75dfb8d3855", - "sha256:bbdb2467362c547193edf6295bf1fc9431337f56a007781e65088ff00fb4353d", - "sha256:bedd9ba56197233d67d497dd8c00cf817149c2050d25095d59beaa89fde3ff4d", - "sha256:c1310716ccdede0f6ee4ccb0df323e0109a62a7c80064f2acb7c1a47b37d21ef", - "sha256:c4ef4bebf76351a2a61ec139db828c7facacb1011d24c09dff6161374fc2141d", - "sha256:c72f54e73cfbbad06e4b437be4e5724cdb2a2bc3f4f39391fc50b5a5db4e74d1", - "sha256:c9bcd8a9f5e40f8c733fe2a054f9d48d4b2822d4e895fea7e2c21972d58a02e5", - "sha256:cd6ecb12402f7f7c30b1e8aea5af294b038efc693c50cb9ed392327456c53384", - "sha256:d2522fbc28f7695fb52fdad4cf4bf7d1871669b235833ba13e2073a14f742a15", - "sha256:d41d3db43e819a6abf1329e7f8facc4f3642dd5de00d82c71b36dea2dc5f7293", - "sha256:d6250983af3fa82b04ad8a835ae87cabfdf368fb1c6e840728f907707afc49a9", - "sha256:d62bccb538c4257643fb509badd989c0cb8ece08b54ee6c3886e343b1f55601a", - "sha256:df606a17401e00a535cb1e503c9989b77852b0e62dd8e6c79336a7b42983e357", - "sha256:df78605f2289c36ac0509c5140efda099ae3e3f4d083285d290f054be7272d9b", - "sha256:e127085af0335e0cd6eff2346a4c2ba8f669bb57638491b3a45e85b63f26535b", - "sha256:e2a512fbf5310ce298e772adcc05ab36be742a0ab115a2e975bd4ef03346789c", - "sha256:e78c897e07a49f227432d245400725ce96515ee4a9d94590c3b9097436fb0db2", - "sha256:eb5fb70a1c24289f3b31081bac917b92f00c5243078c9c0d8a3b0bd9bf1a40a9", - "sha256:ede6e2b96c845057314f5ef1abb7fff9516b33427dda3c1ec0740ae92acb8390", - "sha256:f0381423c30f2536c4c74f82f0063307185add97a80de3e859c81f8e57349bdd", - "sha256:f36263a5dfaf226789c8889d69ffa3d5d5959fa1ccf7526a8460fb492722a483", - "sha256:f36f0d0b4e112211160d2f80a1afc4674815677087bf98c0fdc537d9dd2a9cc0", - "sha256:f6d6f3a41a0923e8bcc2ef5c71cd12558c50b3863243cfa4d55e5b42390bf5e0", - "sha256:f79a24b9f76c0d05cf35b6fc6c24e1e690998feb2d9b69d98537bd6227439a43", - "sha256:fbb32f73e01672372b86e935b1e1b1893c53a61d507e640b20906b4cd19a8ef8" + "sha256:019ae21de930d6077efa1eac746de4df5234e7c6c11ab10080c0935fc5abbecf", + "sha256:02688fff6d256afdd57da5359144ddab8e054b2ba98ddcf147fe191bdf996e88", + "sha256:0274b87df89d1dda8dce77cf05a9dfab7bd30045a09e0d9435ec8be622e374e6", + "sha256:0323e8dbeec4d63c27111796baa7e8a89b391c32d90e67d78f9404d0c8edeab4", + "sha256:053edbb52fe8b8a1a6698c4fee39590c9e44a602ace807291eb87e3b17f85f48", + "sha256:059027f5dd2aafb916301f46a619c7fe03ff5761cdb2d091cf80bf6dbc24bc29", + "sha256:05f11a4be4f668974238cff21208fbd9f629cab8a68b444b7d4a4cfd8081b1d6", + "sha256:0ab71cc5ea86f6685a7b2235edad65f1f2a4b6341109af259d758973d96eece5", + "sha256:0b439f4fb0b615bc0443cc83eaf5835bd480f680c69ed1be963bdb401b8159f8", + "sha256:0ec50d24a12e50857e94ac9035d3c06fd0827bb477b9ebcd83a2a49dd89e5e23", + "sha256:131fc50d52a52acc367ea8bccb028447b734243d00ba1cfc7d9ff8d0dc37fa38", + "sha256:17b5f1d1a4a5ac536283298c98cafc5632ae3897c8601fb2ec8babc6f47a1be9", + "sha256:183b8da9b870ad171a11a629c43e0587a228aea9d595a969231d59bf530b6c77", + "sha256:18888d50813b9df9b8dc8c1506ec40c783db25f130a6101eb89896b27076f751", + "sha256:25b88277832eb558305c3bb986ad61f19b5cb5a87aced289bce4a1701a92aa31", + "sha256:266cdab48e2242b6c010beb8b7af4164aa87f4ad8d6fbd9f4f531214f8ddb234", + "sha256:281bffb09b2e1620db4e99a9df96e38d939c341c7c43cd5191326fbdb4d42275", + "sha256:28cd002cf5a499e6e9bd69d992ffd501b8473948f3e97d6e075b774df1901e8e", + "sha256:2972c6c6a806e0c788f6ec39510abdb61b3a648fd141a5fa77becd2cc05ff551", + "sha256:2b4027b370cc46c4802ba32a979729209c0407d548723e809f19a50a9df27405", + "sha256:318c924e218be754427ce6bb4c630d9dcb5478eb00a8a3f8a0972086adc763b1", + "sha256:380accae56f8c9df99f34bc7e79d286fee37c3dd06b362c394b08ea96371b7c5", + "sha256:3c7784f9936292c9d3f92fc772d874edc071a16cd883ea0d997e5c4318f6362c", + "sha256:3ebd85fd6253abe89f852fc008294d490eb7a5f66913703148b8d263b048cc90", + "sha256:4126c8fe9d817ac3ab223ee5db41a09d0fa82dbd6bb59d207b6f7313d733f19b", + "sha256:4155f0ab246b6892110960f25989ab91073cd708b974f4732dca4d219a8be3e1", + "sha256:41f16267d8e6d916e06a6a1a0e151f643a6bab1277945a4bd494f359d4185dd2", + "sha256:4522f5d662d3ee55a072fad18e2af5dae480658d4e23b04b455c4b7542ce4327", + "sha256:46c900c807b0614c454ba89271ec6f59212403c54dc68ea493ab1ece2c510618", + "sha256:48291b25a904243f37c9aabbfed3eaba466c9a993f5f5946fe647163b7face07", + "sha256:5038a5e9e106087c117f0a7d6fd9d8a382b228da24bbd085b9f2b5d54ab11c3a", + "sha256:594a26bcf0cb720c16ac6db3fd4b3f411be756f9da7682f2f629089ff15aef18", + "sha256:59706135d3107939effe9f9263bd78c507f4abd7bfb96acc5a7f4176aa0a90d2", + "sha256:5a327d7581696c7a392a8f85cce7e54fa1303f5b79b3b2983abaab309b56cfd6", + "sha256:5eca8a45d38c916783c44e5da06a367b77234efa51d84dda8804654b99efecc9", + "sha256:5fa85f6789178ede5333568cbee5bac5fa9718d5f02406b65545e83368fa8fe9", + "sha256:65097e45ef7a942a9b92999b81d2e91fe80cbd0616215e625af39d2166692018", + "sha256:65cc9938cb9bd8862fc220e0719fd7f9c291d788f0a62bb8840820c46fa5a4d0", + "sha256:6a4c3607e2a0e66337d8ddf95ca7efe9b30ebf944119a4fb86503ea66f777263", + "sha256:72f11a136f148eb1218e7d1492749b8b5594302010db0cebd47423c4ac8c79ee", + "sha256:78b5a71de59e30c697a64c69fc48b032bb99c43b7437091b808a9ba20bb0235c", + "sha256:7b212edc9bf9d0c25cc3117483289b9e1a49a1ed134a02635baa987e9f0d89db", + "sha256:7e0f7045c420abdea249a28384baa846b87bad5c9f42af1957dc50c6e337fa1a", + "sha256:7e83cfec424f546dc3f0cc71896f8cc384a711f4116bc1abb0598302a9af3240", + "sha256:80c55bcc31d21bd07f7d1589e11f2ac1faf3359cf9f93026a1944ee76a40f954", + "sha256:863740d7f45adfd29b95658a680b16113721eaa89857c67e7e9573c61e87bbd8", + "sha256:88484b8c3f71dc9205d0d36da541e2cdcf4bc74474a2ee8d99c2e6411b659b89", + "sha256:8a08810e0bcc606d10cf1c5389c96fc92362244c0cf761358c495c2eb29df3dc", + "sha256:8c0637ae4fcb54d5c7fc9af24d348003b6f9dbaf7a06bf13f769d7b85903af39", + "sha256:8e9e3409338a42e3d4c30c224fdb678364542c77994f089fd6cc8131969eff48", + "sha256:902ea10ba85e014dc5d23a7bbb3ab70722349561e73783dd71571359e8867244", + "sha256:9533db74a2685169380db3db3ab59643453e7c486fffa9bf3ab60b73c4e174be", + "sha256:97f02ff49d1fa21308207a7743bec4fdd7aa90e8dd091539da660fc51e624c4d", + "sha256:9ea9a2a154dc7d8658930fa87cda0e6094235b5e130f037d9894eaf8722119a5", + "sha256:a0440d847b2c9986e4d27e8a59164714e5198530c69a5f9fb2e4620f9136d653", + "sha256:a6d39a27b542a781d691827b955d685d496fb6cccfc6eecc336a78b399032062", + "sha256:a7f4d3c478b1fcf412bf6c82914b02fed33ab359120df9172dda7bc855227461", + "sha256:ad297807bbdffce61b04e5e0c22f3c5d9e1905c1ee186f1f6d029f83bf0f18b8", + "sha256:add6778bb51efb80174937543754d2dfa0f4e504e7302d97896006a642c14f95", + "sha256:ae075ebf7bb5f48b3bd2fc9cd53346e4ff43e2515a4f822914bbc62a3cbd6e7e", + "sha256:b26fb439a7fbb522af63bbd781fbf51ec0c0659134a93f5bc8e9e68641df811e", + "sha256:b2bac59721d246939b21274229b9923aeae3db97b6118da739c658c17e110dd6", + "sha256:b314ad1f0667715e8d1b6197d5336ab579b13e801172721d62331bd40034a30c", + "sha256:b7317035875bd7c4705e2566848b2043b78e18f2f5675ea651f9f7805b5589eb", + "sha256:b8e936e620e5f336a207e08c0da9dace5d4dbcc8e64743ab1acaa77a64bbf060", + "sha256:b906da4e9a7ba4ec33ed2f7238343866932c1a6f84944c804252b2922708d0ee", + "sha256:ba690e4e33c360fcf0b8411ca90f8b9cc595e8deddd6a25a9a75a725b698cd6a", + "sha256:bb14da3d63da994c34cfa47cde469df8013ddf5f575455a22530c8c4a0ed8616", + "sha256:bbc2e1632f4a61fa171ddab3bc8368fb8475e7ce68733ca92fec862fdd8e0f60", + "sha256:bbdd3c896db09993b7879cd35e56da6ed8918d161d6e80f9d9c40d78d34e4784", + "sha256:bcaaa8e542cb7e1962d0a58ce6a25f6b4b6ca2e5ce743155fc1f6eb2fea52574", + "sha256:bee682ab1005aff597946234e47c95fcf0f44d2b1f38075f0aba26bbc4e7545a", + "sha256:bfec6543d60c57e7543d9cbccdd5dfcf562f2c05cd6b814df68108a20794e254", + "sha256:c2e50baf7be8831524a87beec6c1873539519a1948f907dc3d4b9be27ebacb80", + "sha256:c6c79a6138be017d85f3bab1df735669b669a38f9b3ff646a1f179afbacb7b63", + "sha256:c702fb7c8bfd87c9ce9c8bddfc9a5796a492bab35a52b1693adee413721e32f2", + "sha256:c9ba1725826f6571a6e4c1561bb1613711f0058b91927a147dc42c637ba087d9", + "sha256:cf205ac52cb6b45745c0a4891cdb6e709c10ad5b034aa736aff561fc4ce9828c", + "sha256:d0d03fc67499ee90feedfa2add4aaa1c091a7bf333535d847b10fffe390e58fe", + "sha256:d118d63f08fd6ac285cb8166e96c992a6ed0e7a1644e8790c39070b18779e688", + "sha256:d24c09f397c3ce55f20e0250da7ba5b0e5249cb5d21465e71ec15154a3a7e8e0", + "sha256:d41735c7a646dae8612e0552dfc53f45807eeb54364dfb1f0a65ac274bc56b3a", + "sha256:dd1696d91f2a37cece9bd22e507e7be7c37c59ecc61fd15f0d0f31e3b6888957", + "sha256:dfcad9c63a893c95ba1149481b9680ce68dd71211f08df0073ee62700790bc97", + "sha256:e384782608837d9aaf123e413679883091744664a2cd76f0ad0e0a1f12facc57", + "sha256:e5ea0abea338c617b753082f36f64c70ade853d88e91ab5732b301ae8ed16e3f", + "sha256:e6ff81c570413bcc35f1c16850eb66e2493a3259e68efe8672376533d2c82d38", + "sha256:e88951ad2831880405f3f055ab12a6aa72696c20a2815128eeccdc3bf914cd78", + "sha256:e98e16b6ce531b12100c01daac922e8ec5b991832a5f58003f13b7d45ea82dc0", + "sha256:eb0fd32e8e433797499571447d9f975b4744be79c0a3339413868d79517231ed", + "sha256:ee74a73e1f9e16b71f67329e99bb58aa4af9a2c3c4b3a5db9f26e92e7c39e161", + "sha256:f15ec5f825c283a5aa427d78759ab8f84e7b5441d15cfff476b548bce3764666", + "sha256:f296c7fe928ce0e29e313f85c43a5ab80542e096e1163c2605b8cc18aa2aff2b", + "sha256:f32df1b19f773bb41382e8b215955d248c9766e3d6ff5a1dd89709e7d96e4685", + "sha256:f3ed67279a4b317a808ac743d3a915f74187530c5f3d9c859e5d04d475b8c174", + "sha256:f5b972ca514898fb7131671c425a62ca38fdae2a8d6296e4b605ec8202349f8c", + "sha256:f961086c0dbba6c00cbd5c5b5646247efd0d0a4044444bfaa9efc7a6ba5e96a5", + "sha256:f9bd7d7a449667d6f17edd9045ec82a4ed2767afb91743d3d0b18c376a56dfe2", + "sha256:fbac4c8ffadb685189efa92fafdb2f5392e9cbd262eae3818bcdb1bd19acaaf2", + "sha256:fc43c8276d0a7c7b76f31d4f3f80f9eb820673628f1411770a70029c1d5f6a75", + "sha256:fcfded324f0710632e22050a2fd7b56b1cbcb2d21001630bcc26d536f54bffec", + "sha256:ff435abdcbfdf4a070f488830cd53aef77cf8649d0fd8ed76bf27d9566e80e78" ], "markers": "python_version >= '3.6'", - "version": "==0.20.6" + "version": "==0.20.7" }, "lxml": { "hashes": [ @@ -980,45 +980,43 @@ }, "pikepdf": { "hashes": [ - "sha256:02a58790137b6e567cd822f590f3da406826bcf362f821c05d5e54e97fb34553", - "sha256:05e374d8223527f07bf27c24b00c7b8fb810f7e0bef199576537e66732cf0757", - "sha256:0681fe83604a6b3a7eb13f64aa2a636110142691e2d9e69b71503502da7ca5ac", - "sha256:0aaf2e5455310b954ca148034d266d6c38ffe330d8990119a5ea6b85c94a3d17", - "sha256:0ac4da28e9093850c526223b29063a18aca96686eff6fb53dcc066c93d7aad5a", - "sha256:188e015b522b304a683bf8e3347d8745245bcc3bb7559e3b18be29bba7aeb939", - "sha256:25681a5b30fbf6fc57af1434b132341a3b621b7d190d6046aa5ea444eaf3a99c", - "sha256:2a553de0a526b45beb4b11a19fe48509fa033e5bfd124966a768c45a3d562235", - "sha256:2c844913e6178e4a012bf31a4c570dbf44dc5dbe744a58b468b18e470b39306c", - "sha256:31637fd4990d2f79e43f407184dc4ce7c9f75af1c7e99e573b1a0edce5386e84", - "sha256:396f2188b7bcb007eea91d3a310114aa85db159981ec7e0a11e6b8083f559933", - "sha256:41755c44bf0187c91bc0bc58b7107e5e6839ebd3c71d4e41970ff6f1738ccfe8", - "sha256:4942530edd535c150dd9924ed42af80217f7ff629256defda9371c3ef7a90816", - "sha256:58501545f38056db362b7997e5ec1a84e6bb258bb1402dc88c00928e9d6c3d89", - "sha256:5aa631972c6ad5620731e5591acd66152af5c94fa0d4f1ad2a502593a07430d8", - "sha256:5d64ea8436aaea551ffa821006c02c4fb3199e24b47bfa1b9acfff97819bbc66", - "sha256:622220c768cc48859be8c317ade07a83d35508a06d9ddf5563d3e85cbf1bf7d4", - "sha256:719140beb3e80d127762dc5497ea5b94b7b8a1ba1782c46ec99d79f49174cffe", - "sha256:71e6d12c2b416c825cc97d1e5041a3f2881a5bc4a454cc6cb7b5096e7bc04bfa", - "sha256:7a6731d19c338ecf98159fa9989eaffbb15b9c25c6eef366a0c5b2a68c299849", - "sha256:836d3adad3445274415880ebe0946a465f5e41e3debbc25ccc59870c2bf3b015", - "sha256:84eddbed6c12b04360eec35a3c9970b1d1321832e18936b1bad0f7165d44bbaa", - "sha256:8534ea179d858ab91d3ae04ed3a9aac3d24241363d47bc0fd60660c468a56adf", - "sha256:90c0bd43745e80ce7ffef8a4848c59e45dbbae6b2feeda3819ad5ce3f8461daa", - "sha256:9e365c7ad614a690c20607039abdf3c5cfbf86e92ac9fdd3764e03b4a7515c23", - "sha256:a2edc608e073bd6d5c183eaecd0be8bcd160dcb1b6acd3019258d7096f6e7b8f", - "sha256:a7c05a430d3af2bab5df231a3f67586e2545a9e436e94f824f95e56a420f9748", - "sha256:ac68c694da7e95420c3d7b46c98dcc659b211c080d4b5088c317b0540d5933d6", - "sha256:ad85ebc064bb44742084be2093db8f340e01ab54f3d23a3be7c270620a43af63", - "sha256:b0fa2ec2fc4bd3ed6e91da06603dc1e486559151e98a21346232c848af73d3f4", - "sha256:b557ae766dbc5f7c1862f5fad6d852a89a58857d8849da1cb37f349d9024ca5f", - "sha256:bc57788c24a57acc64ad634f6ab749c14dfd34db1fff0f4842628a70f260c10a", - "sha256:c9549211f77fcbe3fe7b0fba359400e8c980339e5e011b84d0e12f6cdd87d712", - "sha256:ce1df32bbda7dffbe7ba1f69dd05e14f3fe952db00c5ff67bdddcfb8bfa43132", - "sha256:e3f481da764a5c102a75d89e389524fb87a5d63af7f97ba44ea551f7ab994eb8", - "sha256:edaae73c602e947f9ba98e24eab8548959db91ad189e1a3f9c5a1f091088a5e3" + "sha256:0fb4e028abc84fba49513c1759b78e2948eebabb23ec33e68f5beec9edc3b1e4", + "sha256:169fd0a1d1d15186f0e2ac4d31eba8b7b45a1a0e7c88be5e10358d6edd3387b3", + "sha256:178be221e8bee2282a1a36e32007365a959df7efd58cfd1f7139888aceb0afb8", + "sha256:18fc6bf7c189138c77f976fbef3f8679dd7a26254e869429879a0943dcfd0775", + "sha256:19e25bfa0b7cbb87620dcfaf5e22d235eba675aa0f2bb74acb08ff039cc8a771", + "sha256:215e6850c04ccf89aee37288aa34dfea151c5b8bd1e6fc75cd429e5ace2c8bec", + "sha256:28309830b84e1392949afe5dc8e02d635531b63ab41f0ed6ec05ca8a8cc384ee", + "sha256:28b808dd9cb588c2792b8c3f18e44549e8cf3df637d0a126fc16d7cb9651b704", + "sha256:4b14b7e47cb306719a3ade4a2caedf576912a78e569571a9bdae134679e1ad07", + "sha256:50dbeca36dc3ba04df3754845faad05c4c6fe35e8515a7ebb9ccdcf6d14672d3", + "sha256:55f35903308efc6148e11e7950950da6a188ef73f0c4912f60deb5665c13c3de", + "sha256:5b47df6d084a9f35d66f1166afc01588c51c55afcae1e3792f4a1b900fce95ae", + "sha256:5c656f518fd1eaf9613d39ad1eb437808494038a4c88fae44f40e1a5f57b32ed", + "sha256:6bb2041045aa9f40a47bae436cc24355918e88693db075d7526d194d56c53797", + "sha256:7704956b8a36aeb67b11191b7fc17c6918c9aa73a5d2ae5b102fc7f29eee043a", + "sha256:7a4240a128c26e44cade7a50c12380f38648db254752876fc8c2fd8921532e57", + "sha256:7faf78b668e86cafded3f269bfd602e7af33c1f22c2cad440f3c4a92b76a7254", + "sha256:905e5166954161b760c545bbae2bc4438292f1e487cf7c52991fdaef7d75aac6", + "sha256:93c258b15a0bbd9f061907bda558b289fbe66821fb2ddce19c8f456ed7c61df3", + "sha256:9f0685745f08c1d7d6769e367d14b74817a7fced75c1afca17034f4d6c04c00b", + "sha256:a1fd1968de6c2439008673edbc3e47c6a63858bea5d15baddd0c7e77d35eae0f", + "sha256:ab52590158b073643a8482105e4927356eef63345ae4e9ba59ba8f951193dd54", + "sha256:aeb9a442d384fa37f5e9c6364176b6c661fac9cfd427f7b537ead1a6a660aef1", + "sha256:b8b948febcca7b376c2474c83b80b98ea3bdc76d5dd74872c9fa6fb3b66e315c", + "sha256:c3acd73ca490cd81ca467eb4d24aa6ba63a7e1b6514e86fe1072388607d7f1ba", + "sha256:ddca7062db6fbcf5b7a7d07a031030a219d9eb4a90225c25c53318e2b2dd354d", + "sha256:dfa17513abeb5e9a4de4af5fb098fd0bac4708e961424be9750b16cd00af42c5", + "sha256:e4d7adb67d3c14f05ce3452d8e212c60ded743bb2977f28a28759dfc499ed3a5", + "sha256:e6e548a3a3ec18e790101f984da04f7c968525e1077a072ec8dc4769bbd1fc45", + "sha256:f4e0e86b104354eb1b5195bca51a13fb0c390e218f18544cd39d3ba2d07fc26a", + "sha256:f75ce310d36fe83aa075c78002c1cdcd1e08be594730d4a6f33b0c86670c637e", + "sha256:f80cc66c20b897fc4a4f1f15b8b34a2d20a9a83b421910f0bfacb405af95912a", + "sha256:f85231bcbe8a63feb29e42b2437b8e59d92ad9f9bfea294c345bab23e097a401", + "sha256:ff2adf043a85bca67ffcd3d235e48ce482f00875e3c54931dd9e98b49615868f" ], "index": "pypi", - "version": "==6.2.0" + "version": "==6.2.1" }, "pillow": { "hashes": [ @@ -1096,11 +1094,11 @@ }, "portalocker": { "hashes": [ - "sha256:400bae275366e7b840d4baad0654c6ec5994e07c40c423d78e9e1340279b8352", - "sha256:ae8e9cc2660da04bf41fa1a0eef7e300bb5e4a5869adfb1a6d8551632b559b2b" + "sha256:102ed1f2badd8dec9af3d732ef70e94b215b85ba45a8d7ff3c0003f19b442f4e", + "sha256:964f6830fb42a74b5d32bce99ed37d8308c1d7d44ddf18f3dd89f4680de97b39" ], "markers": "python_version >= '3'", - "version": "==2.5.1" + "version": "==2.6.0" }, "prometheus-client": { "hashes": [ @@ -1219,10 +1217,10 @@ }, "python-levenshtein": { "hashes": [ - "sha256:785b1216912392a610fbb20f27a400ee00849244fd8248f40a52d8f0d63d7e28", - "sha256:edfe6f724bf93ba37e177a0766d8dd93867fdb969f10ee4ceacb0f72022fd50f" + "sha256:88a58b95e3340a918489dac0c78f731323c0a4d8f5564f839ffea80155574e77", + "sha256:9228af5523f797f0798f045dc4a95ed1f46df72bc2186e52b530a33998a51b37" ], - "version": "==0.20.6" + "version": "==0.20.7" }, "python-magic": { "hashes": [ @@ -1234,10 +1232,10 @@ }, "pytz": { "hashes": [ - "sha256:2c0784747071402c6e99f0bafdb7da0fa22645f06554c7ae06bf6358897e9c91", - "sha256:48ce799d83b6f8aab2020e369b627446696619e79645419610b9facd909b3174" + "sha256:335ab46900b1465e714b4fda4963d87363264eb662aab5e65da039c25f1f5b22", + "sha256:c4d88f472f54d615e9cd582a5004d1e5f624854a6a27a6211591c251f22a6914" ], - "version": "==2022.4" + "version": "==2022.5" }, "pytz-deprecation-shim": { "hashes": [ @@ -1804,11 +1802,11 @@ "standard" ], "hashes": [ - "sha256:0abd429ebb41e604ed8d2be6c60530de3408f250e8d2d84967d85ba9e86fe3af", - "sha256:9a66e7c42a2a95222f76ec24a4b754c158261c4696e683b9dadc72b590e0311b" + "sha256:cc277f7e73435748e69e075a721841f7c4a95dba06d12a72fe9874acced16f6f", + "sha256:cf538f3018536edb1f4a826311137ab4944ed741d52aeb98846f52215de57f25" ], "index": "pypi", - "version": "==0.18.3" + "version": "==0.19.0" }, "uvloop": { "hashes": [ @@ -1886,26 +1884,26 @@ }, "watchfiles": { "hashes": [ - "sha256:00e5f307a58752ec1478eeb738863544bde21cc7a2728bd1c216060406bde9c1", - "sha256:1dd1e3181ad5d83ca35e9147c72e24f39437fcdf570c9cdc532016399fb62957", - "sha256:204950f1d6083539af5c8b7d4f5f8039c3ce36fa692da12d9743448f3199cb15", - "sha256:4056398d8f6d4972fe0918707b59d4cb84470c91d3c37f0e11e5a66c2a598760", - "sha256:539bcdb55a487126776c9d8c011094214d1df3f9a2321a6c0b1583197309405a", - "sha256:53a2faeb121bc51bb6b960984f46901227e2e2475acc5a8d4c905a600436752d", - "sha256:58dc3140dcf02a8aa76464a77a093016f10e89306fec21a4814922a64f3e8b9f", - "sha256:6a3d6c699f3ce238dfa90bcef501f331a69b0d9b076f14459ed8eab26ba2f4cf", - "sha256:92675f379a9d5adbc6a52179f3e39aa56944c6eecb80384608fff2ed2619103a", - "sha256:a53cb6c06e5c1f216c792fbb432ce315239d432cb8b68d508547100939ec0399", - "sha256:a7f4271af86569bdbf131dd5c7c121c45d0ed194f3c88b88326e48a3b6a2db12", - "sha256:ad2bdcae4c0f07ca6c090f5a2c30188cc6edba011b45e7c96eb1896648092367", - "sha256:adcf15ecc2182ea9d2358c1a8c2b53203c3909484918776929b7bbe205522c0e", - "sha256:ae7c57ef920589a40270d5ef3216d693f4e6f8864d8fc8b6cb7885ca98ad2a61", - "sha256:afd35a1bd3b9e68efe384ae7538481ae725597feb66f56f4bd23ecdbda726da0", - "sha256:b5c334cd3bc88aa4a8a1e08ec9f702b63c947211275defdc2dd79dc037fcb500", - "sha256:c7e1ffbd03cbcb46d1b7833e10e7d6b678ab083b4e4b80db06cfff5baca3c93f", - "sha256:ffff3418dc753a2aed2d00200a4daeaac295c40458f8012836a65555f288be8b" + "sha256:1686bc4ac40ffde7256b6543b0f9a2cc8b531ae45243786f1d3f1dda2fe39e24", + "sha256:184799818c4fa7dbc6a1e4ca20bcbc6b85e4e0db07ce4554ea2f29b75ccd0cdc", + "sha256:27d64a6ed5e0aebef97c70fa3899a6958d4f7f049effc659e7dc3e81f3170a7b", + "sha256:320bcde0adaa972403ed3b70f784409437325a1a4df2de54ba0672203d8847e5", + "sha256:39b932b044fef6c43e813e0bef908e0edf185bf7b5d8d53246651cb7ac9efe79", + "sha256:4ba6d8c2f957cae3e888bc250bc60ed09fe869b3f55f09d020ed3fecbefb6a4c", + "sha256:4bbc8bfa0f3871b1867af42837a5635a9c1cbb2b68d039754b4750642c34aaee", + "sha256:76a4c4a8e25a2c9a4f7fa3d373bbaf5558c17b97b4cf8411d33de368fe6b68a9", + "sha256:78b1e7c29b92dfc8fc32f15949019232b493767d236c2bff31848df13fdb9e8a", + "sha256:7f39fcdac5d5b9815a0c2ab9005d39854296b11fa15386a9a69c09cbbc5dde2c", + "sha256:8eddc2d19bf6f49aee224072ec0f4f3258125a49f11b5dcff1448e68718a745e", + "sha256:9d3a12e4de5446fb6e286b720d0cb3a080811caf0ef43e556c2db5fe10ef0342", + "sha256:9e611a90482ac14ef3ec234c1604ed921d1b0c68970eba82f1cf0d59a3e4eb76", + "sha256:bbe10d134eef1666451382015e48f092c941a6d4562a98ffa1a288f79a897c46", + "sha256:be87c9b1fe2b02105a9ac6d9df7500a110652bbd97cf46b13964eeaef9a6c89c", + "sha256:cfdbfc4b6797c28dd1a8524581fed00ca333971b4111af8cd42fb7a92dcdc227", + "sha256:d5d799614d4c56d29c5ba56f4f619f967210dc10a0d6965b62d326b9e2f72c9e", + "sha256:fd4215badad1e3d1ad5fb79f21432dd5157e2e7b0765d27a19dc2a28580c6979" ], - "version": "==0.17.0" + "version": "==0.18.0" }, "wcwidth": { "hashes": [ @@ -2178,7 +2176,7 @@ "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845", "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==2.1.1" }, "click": { @@ -2589,10 +2587,10 @@ }, "pytz": { "hashes": [ - "sha256:2c0784747071402c6e99f0bafdb7da0fa22645f06554c7ae06bf6358897e9c91", - "sha256:48ce799d83b6f8aab2020e369b627446696619e79645419610b9facd909b3174" + "sha256:335ab46900b1465e714b4fda4963d87363264eb662aab5e65da039c25f1f5b22", + "sha256:c4d88f472f54d615e9cd582a5004d1e5f624854a6a27a6211591c251f22a6914" ], - "version": "==2022.4" + "version": "==2022.5" }, "pyyaml": { "hashes": [ From f8ce6285df44cc580319c370a9d76149012615b1 Mon Sep 17 00:00:00 2001 From: Trenton H Date: Mon, 24 Oct 2022 08:40:33 -0700 Subject: [PATCH 6/6] Allows using pdf2image instead of pikepdf if desired --- docs/configuration.rst | 11 +++++++++ src/documents/barcodes.py | 22 ++++++++++------- src/documents/tests/test_barcodes.py | 35 ++++++++++++++++++++++++++++ src/paperless/settings.py | 14 ++++++++--- 4 files changed, 70 insertions(+), 12 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 5a379cb05..8a3a25252 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -701,6 +701,17 @@ PAPERLESS_CONSUMER_ENABLE_BARCODES= Defaults to false. +PAPERLESS_CONSUMER_USE_LEGACY_DETECTION= + Enables the legacy method of detecting barcodes. By default, images are + extracted directly from the PDF structure for barcode detection. If this + configuration value is set, images of the whole PDF page will be used instead. + + This is a slower and more memory intensive process, but may be required for + certain files, depending on how it is produced and how images are encoded. + + Defaults to false. + + PAPERLESS_CONSUMER_BARCODE_TIFF_SUPPORT= Whether TIFF image files should be scanned for barcodes. This will automatically convert any TIFF image(s) to pdfs for later diff --git a/src/documents/barcodes.py b/src/documents/barcodes.py index a4be126a5..13e78e181 100644 --- a/src/documents/barcodes.py +++ b/src/documents/barcodes.py @@ -150,16 +150,20 @@ def scan_file_for_separating_barcodes(filepath: str) -> Tuple[Optional[str], Lis if mime_type == "image/tiff": pdf_filepath = convert_from_tiff_to_pdf(filepath) - try: - _pikepdf_barcode_scan(pdf_filepath) - except Exception as e: - - logger.warning( - f"Exception using pikepdf for barcodes, falling back to pdf2image: {e}", - ) - # Reset this incase pikepdf got part way through - separator_page_numbers = [] + if settings.CONSUMER_USE_LEGACY_DETECTION: _pdf2image_barcode_scan(pdf_filepath) + else: + try: + _pikepdf_barcode_scan(pdf_filepath) + except Exception as e: + + logger.warning( + f"Exception using pikepdf for barcodes," + f" falling back to pdf2image: {e}", + ) + # Reset this incase pikepdf got part way through + separator_page_numbers = [] + _pdf2image_barcode_scan(pdf_filepath) else: logger.warning( diff --git a/src/documents/tests/test_barcodes.py b/src/documents/tests/test_barcodes.py index ee8df9f34..1c4ab7cc3 100644 --- a/src/documents/tests/test_barcodes.py +++ b/src/documents/tests/test_barcodes.py @@ -468,6 +468,41 @@ class TestBarcode(DirectoriesMixin, TestCase): self.assertTrue(os.path.isfile(target_file1)) self.assertTrue(os.path.isfile(target_file2)) + @override_settings(CONSUMER_USE_LEGACY_DETECTION=True) + def test_barcode_splitter_legacy_fallback(self): + """ + GIVEN: + - File containing barcode + - Legacy method of detection is enabled + WHEN: + - File is scanned for barcodes + THEN: + - Barcodes are properly detected + """ + test_file = os.path.join( + self.BARCODE_SAMPLE_DIR, + "patch-code-t-middle.pdf", + ) + tempdir = tempfile.mkdtemp(prefix="paperless-", dir=settings.SCRATCH_DIR) + + pdf_file, separator_page_numbers = barcodes.scan_file_for_separating_barcodes( + test_file, + ) + + self.assertEqual(test_file, pdf_file) + self.assertTrue(len(separator_page_numbers) > 0) + + document_list = barcodes.separate_pages(test_file, separator_page_numbers) + self.assertTrue(document_list) + for document in document_list: + barcodes.save_to_dir(document, target_dir=tempdir) + + target_file1 = os.path.join(tempdir, "patch-code-t-middle_document_0.pdf") + target_file2 = os.path.join(tempdir, "patch-code-t-middle_document_1.pdf") + + self.assertTrue(os.path.isfile(target_file1)) + self.assertTrue(os.path.isfile(target_file2)) + @override_settings(CONSUMER_ENABLE_BARCODES=True) def test_consume_barcode_file(self): test_file = os.path.join( diff --git a/src/paperless/settings.py b/src/paperless/settings.py index a262bd501..1fb6ba913 100644 --- a/src/paperless/settings.py +++ b/src/paperless/settings.py @@ -558,15 +558,23 @@ CONSUMER_IGNORE_PATTERNS = list( CONSUMER_SUBDIRS_AS_TAGS = __get_boolean("PAPERLESS_CONSUMER_SUBDIRS_AS_TAGS") -CONSUMER_ENABLE_BARCODES = __get_boolean( +CONSUMER_ENABLE_BARCODES: Final[bool] = __get_boolean( "PAPERLESS_CONSUMER_ENABLE_BARCODES", ) -CONSUMER_BARCODE_TIFF_SUPPORT = __get_boolean( +CONSUMER_BARCODE_TIFF_SUPPORT: Final[bool] = __get_boolean( "PAPERLESS_CONSUMER_BARCODE_TIFF_SUPPORT", ) -CONSUMER_BARCODE_STRING = os.getenv("PAPERLESS_CONSUMER_BARCODE_STRING", "PATCHT") +CONSUMER_USE_LEGACY_DETECTION: Final[bool] = __get_boolean( + "PAPERLESS_CONSUMER_USE_LEGACY_DETECTION", + "NO", +) + +CONSUMER_BARCODE_STRING: Final[str] = os.getenv( + "PAPERLESS_CONSUMER_BARCODE_STRING", + "PATCHT", +) OCR_PAGES = int(os.getenv("PAPERLESS_OCR_PAGES", 0))