diff --git a/src/documents/migrations/1021_paperlesstask.py b/src/documents/migrations/1021_paperlesstask.py new file mode 100644 index 000000000..f827a892a --- /dev/null +++ b/src/documents/migrations/1021_paperlesstask.py @@ -0,0 +1,66 @@ +# Generated by Django 4.0.4 on 2022-05-23 07:14 + +from django.db import migrations, models +import django.db.models.deletion + + +def init_paperless_tasks(apps, schema_editor): + PaperlessTask = apps.get_model("documents", "PaperlessTask") + Task = apps.get_model("django_q", "Task") + + for task in Task.objects.all(): + if not hasattr(task, "paperlesstask"): + paperlesstask = PaperlessTask.objects.create( + task=task, + q_task_id=task.id, + name=task.name, + created=task.started, + acknowledged=False, + ) + task.paperlesstask = paperlesstask + task.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ("django_q", "0014_schedule_cluster"), + ("documents", "1020_merge_20220518_1839"), + ] + + operations = [ + migrations.CreateModel( + name="PaperlessTask", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("q_task_id", models.CharField(max_length=128)), + ("name", models.CharField(max_length=256)), + ( + "created", + models.DateTimeField( + auto_now=True, db_index=True, verbose_name="created" + ), + ), + ("acknowledged", models.BooleanField(default=False)), + ( + "task", + models.OneToOneField( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="task", + to="django_q.task", + ), + ), + ], + ), + migrations.RunPython(init_paperless_tasks, migrations.RunPython.noop), + ] diff --git a/src/documents/models.py b/src/documents/models.py index b85c56037..f7ce9ae95 100644 --- a/src/documents/models.py +++ b/src/documents/models.py @@ -11,6 +11,7 @@ from django.contrib.auth.models import User from django.db import models from django.utils import timezone from django.utils.translation import gettext_lazy as _ +from django_q.tasks import Task from documents.parsers import get_default_file_extension @@ -500,3 +501,18 @@ class UiSettings(models.Model): def __str__(self): return self.user.username + + +class PaperlessTask(models.Model): + + q_task_id = models.CharField(max_length=128) + name = models.CharField(max_length=256) + created = models.DateTimeField(_("created"), auto_now=True, db_index=True) + task = models.OneToOneField( + Task, + on_delete=models.CASCADE, + related_name="task", + null=True, + blank=True, + ) + acknowledged = models.BooleanField(default=False) diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py index 8459cd037..e89c5d686 100644 --- a/src/documents/serialisers.py +++ b/src/documents/serialisers.py @@ -595,3 +595,11 @@ class UiSettingsViewSerializer(serializers.ModelSerializer): defaults={"settings": validated_data.get("settings", None)}, ) return ui_settings + + +class ConsupmtionTasksViewSerializer(serializers.Serializer): + + type = serializers.ChoiceField( + choices=["all", "incomplete", "complete", "failed"], + default="all", + ) diff --git a/src/documents/signals/handlers.py b/src/documents/signals/handlers.py index 34710af78..2e763cee0 100644 --- a/src/documents/signals/handlers.py +++ b/src/documents/signals/handlers.py @@ -13,6 +13,9 @@ from django.db.models import Q from django.dispatch import receiver from django.utils import termcolors from django.utils import timezone +from django_q.signals import post_save +from django_q.signals import pre_enqueue +from django_q.tasks import Task from filelock import FileLock from .. import matching @@ -21,6 +24,7 @@ from ..file_handling import delete_empty_directories from ..file_handling import generate_unique_filename from ..models import Document from ..models import MatchingModel +from ..models import PaperlessTask from ..models import Tag @@ -499,3 +503,20 @@ def add_to_index(sender, document, **kwargs): from documents import index index.add_or_update_document(document) + + +@receiver(pre_enqueue) +def init_paperless_task(sender, task, **kwargs): + if task["func"] == "documents.tasks.consume_file": + paperless_task = PaperlessTask.objects.get_or_create(q_task_id=task["id"]) + paperless_task.name = task["name"] + paperless_task.created = task["started"] + + +@receiver(post_save, sender=Task) +def update_paperless_task(sender, instance, **kwargs): + logger.debug(sender, instance) + papeless_task = PaperlessTask.objects.find(q_task_id=instance.id) + if papeless_task: + papeless_task.task = instance + papeless_task.save() diff --git a/src/documents/tasks.py b/src/documents/tasks.py index 208f74f1d..241ec9766 100644 --- a/src/documents/tasks.py +++ b/src/documents/tasks.py @@ -10,6 +10,7 @@ from asgiref.sync import async_to_sync from channels.layers import get_channel_layer from django.conf import settings from django.db.models.signals import post_save +from django_q.tasks import Task from documents import index from documents import sanity_checker from documents.classifier import DocumentClassifier @@ -359,3 +360,16 @@ def bulk_update_documents(document_ids): with AsyncWriter(ix) as writer: for doc in documents: index.update_document(writer, doc) + + +def create_paperless_task(sender, instance, created, **kwargs): + if created: + Task.objects.create(thing=instance) + + +post_save.connect( + create_paperless_task, + sender=Task, + weak=False, + dispatch_uid="models.create_paperless_task", +) diff --git a/src/documents/views.py b/src/documents/views.py index cdd38180b..d7f8bf10b 100644 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -64,12 +64,14 @@ from .matching import match_tags from .models import Correspondent from .models import Document from .models import DocumentType +from .models import PaperlessTask from .models import SavedView from .models import StoragePath from .models import Tag from .parsers import get_parser_class_for_mime_type from .serialisers import BulkDownloadSerializer from .serialisers import BulkEditSerializer +from .serialisers import ConsupmtionTasksViewSerializer from .serialisers import CorrespondentSerializer from .serialisers import DocumentListSerializer from .serialisers import DocumentSerializer @@ -795,3 +797,50 @@ class UiSettingsView(GenericAPIView): "success": True, }, ) + + +class ConsupmtionTasksView(GenericAPIView): + + permission_classes = (IsAuthenticated,) + serializer_class = ConsupmtionTasksViewSerializer + + def get(self, request, format=None): + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + + consumption_tasks = ( + PaperlessTask.objects.filter( + acknowledged=False, + ) + .order_by("task__started") + .reverse() + ) + incomplete_tasks = consumption_tasks.filter(task=None).values( + "id", + "q_task_id", + "name", + "created", + "acknowledged", + ) + failed_tasks = consumption_tasks.filter(task__success=0).values( + "id", + "q_task_id", + "name", + "created", + "acknowledged", + ) + completed_tasks = consumption_tasks.filter(task__success=1).values( + "id", + "q_task_id", + "name", + "created", + "acknowledged", + ) + return Response( + { + "total": consumption_tasks.count(), + "incomplete": incomplete_tasks, + "failed": failed_tasks, + "completed": completed_tasks, + }, + ) diff --git a/src/paperless/urls.py b/src/paperless/urls.py index 003d79f2d..6cea1b9e4 100644 --- a/src/paperless/urls.py +++ b/src/paperless/urls.py @@ -9,6 +9,7 @@ from django.views.decorators.csrf import csrf_exempt from django.views.generic import RedirectView from documents.views import BulkDownloadView from documents.views import BulkEditView +from documents.views import ConsupmtionTasksView from documents.views import CorrespondentViewSet from documents.views import DocumentTypeViewSet from documents.views import IndexView @@ -86,6 +87,11 @@ urlpatterns = [ UiSettingsView.as_view(), name="ui_settings", ), + re_path( + r"^consumption_tasks/", + ConsupmtionTasksView.as_view(), + name="consumption_tasks", + ), path("token/", views.obtain_auth_token), ] + api_router.urls,