PaperlessTask and consumption_tasks endpoint

src/documents/migrations/1021_paperlesstask.py

@@ -0,0 +1,66 @@
+# Generated by Django 4.0.4 on 2022-05-23 07:14
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+def init_paperless_tasks(apps, schema_editor):
+    PaperlessTask = apps.get_model("documents", "PaperlessTask")
+    Task = apps.get_model("django_q", "Task")
+
+    for task in Task.objects.all():
+        if not hasattr(task, "paperlesstask"):
+            paperlesstask = PaperlessTask.objects.create(
+                task=task,
+                q_task_id=task.id,
+                name=task.name,
+                created=task.started,
+                acknowledged=False,
+            )
+            task.paperlesstask = paperlesstask
+            task.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("django_q", "0014_schedule_cluster"),
+        ("documents", "1020_merge_20220518_1839"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="PaperlessTask",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("q_task_id", models.CharField(max_length=128)),
+                ("name", models.CharField(max_length=256)),
+                (
+                    "created",
+                    models.DateTimeField(
+                        auto_now=True, db_index=True, verbose_name="created"
+                    ),
+                ),
+                ("acknowledged", models.BooleanField(default=False)),
+                (
+                    "task",
+                    models.OneToOneField(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="task",
+                        to="django_q.task",
+                    ),
+                ),
+            ],
+        ),
+        migrations.RunPython(init_paperless_tasks, migrations.RunPython.noop),
+    ]

src/documents/models.py

@@ -11,6 +11,7 @@ from django.contrib.auth.models import User
 from django.db import models
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
+from django_q.tasks import Task
 from documents.parsers import get_default_file_extension
 
 
@@ -500,3 +501,18 @@ class UiSettings(models.Model):
 
     def __str__(self):
         return self.user.username
+
+
+class PaperlessTask(models.Model):
+
+    q_task_id = models.CharField(max_length=128)
+    name = models.CharField(max_length=256)
+    created = models.DateTimeField(_("created"), auto_now=True, db_index=True)
+    task = models.OneToOneField(
+        Task,
+        on_delete=models.CASCADE,
+        related_name="task",
+        null=True,
+        blank=True,
+    )
+    acknowledged = models.BooleanField(default=False)

src/documents/serialisers.py

@@ -595,3 +595,11 @@ class UiSettingsViewSerializer(serializers.ModelSerializer):
             defaults={"settings": validated_data.get("settings", None)},
         )
         return ui_settings
+
+
+class ConsupmtionTasksViewSerializer(serializers.Serializer):
+
+    type = serializers.ChoiceField(
+        choices=["all", "incomplete", "complete", "failed"],
+        default="all",
+    )

src/documents/signals/handlers.py

@@ -13,6 +13,9 @@ from django.db.models import Q
 from django.dispatch import receiver
 from django.utils import termcolors
 from django.utils import timezone
+from django_q.signals import post_save
+from django_q.signals import pre_enqueue
+from django_q.tasks import Task
 from filelock import FileLock
 
 from .. import matching
@@ -21,6 +24,7 @@ from ..file_handling import delete_empty_directories
 from ..file_handling import generate_unique_filename
 from ..models import Document
 from ..models import MatchingModel
+from ..models import PaperlessTask
 from ..models import Tag
 
 
@@ -499,3 +503,20 @@ def add_to_index(sender, document, **kwargs):
     from documents import index
 
     index.add_or_update_document(document)
+
+
+@receiver(pre_enqueue)
+def init_paperless_task(sender, task, **kwargs):
+    if task["func"] == "documents.tasks.consume_file":
+        paperless_task = PaperlessTask.objects.get_or_create(q_task_id=task["id"])
+        paperless_task.name = task["name"]
+        paperless_task.created = task["started"]
+
+
+@receiver(post_save, sender=Task)
+def update_paperless_task(sender, instance, **kwargs):
+    logger.debug(sender, instance)
+    papeless_task = PaperlessTask.objects.find(q_task_id=instance.id)
+    if papeless_task:
+        papeless_task.task = instance
+        papeless_task.save()

src/documents/tasks.py

@@ -10,6 +10,7 @@ from asgiref.sync import async_to_sync
 from channels.layers import get_channel_layer
 from django.conf import settings
 from django.db.models.signals import post_save
+from django_q.tasks import Task
 from documents import index
 from documents import sanity_checker
 from documents.classifier import DocumentClassifier
@@ -359,3 +360,16 @@ def bulk_update_documents(document_ids):
     with AsyncWriter(ix) as writer:
         for doc in documents:
             index.update_document(writer, doc)
+
+
+def create_paperless_task(sender, instance, created, **kwargs):
+    if created:
+        Task.objects.create(thing=instance)
+
+
+post_save.connect(
+    create_paperless_task,
+    sender=Task,
+    weak=False,
+    dispatch_uid="models.create_paperless_task",
+)

src/documents/views.py

@@ -64,12 +64,14 @@ from .matching import match_tags
 from .models import Correspondent
 from .models import Document
 from .models import DocumentType
+from .models import PaperlessTask
 from .models import SavedView
 from .models import StoragePath
 from .models import Tag
 from .parsers import get_parser_class_for_mime_type
 from .serialisers import BulkDownloadSerializer
 from .serialisers import BulkEditSerializer
+from .serialisers import ConsupmtionTasksViewSerializer
 from .serialisers import CorrespondentSerializer
 from .serialisers import DocumentListSerializer
 from .serialisers import DocumentSerializer
@@ -795,3 +797,50 @@ class UiSettingsView(GenericAPIView):
                 "success": True,
             },
         )
+
+
+class ConsupmtionTasksView(GenericAPIView):
+
+    permission_classes = (IsAuthenticated,)
+    serializer_class = ConsupmtionTasksViewSerializer
+
+    def get(self, request, format=None):
+        serializer = self.get_serializer(data=request.data)
+        serializer.is_valid(raise_exception=True)
+
+        consumption_tasks = (
+            PaperlessTask.objects.filter(
+                acknowledged=False,
+            )
+            .order_by("task__started")
+            .reverse()
+        )
+        incomplete_tasks = consumption_tasks.filter(task=None).values(
+            "id",
+            "q_task_id",
+            "name",
+            "created",
+            "acknowledged",
+        )
+        failed_tasks = consumption_tasks.filter(task__success=0).values(
+            "id",
+            "q_task_id",
+            "name",
+            "created",
+            "acknowledged",
+        )
+        completed_tasks = consumption_tasks.filter(task__success=1).values(
+            "id",
+            "q_task_id",
+            "name",
+            "created",
+            "acknowledged",
+        )
+        return Response(
+            {
+                "total": consumption_tasks.count(),
+                "incomplete": incomplete_tasks,
+                "failed": failed_tasks,
+                "completed": completed_tasks,
+            },
+        )

src/paperless/urls.py

@@ -9,6 +9,7 @@ from django.views.decorators.csrf import csrf_exempt
 from django.views.generic import RedirectView
 from documents.views import BulkDownloadView
 from documents.views import BulkEditView
+from documents.views import ConsupmtionTasksView
 from documents.views import CorrespondentViewSet
 from documents.views import DocumentTypeViewSet
 from documents.views import IndexView
@@ -86,6 +87,11 @@ urlpatterns = [
                     UiSettingsView.as_view(),
                     name="ui_settings",
                 ),
+                re_path(
+                    r"^consumption_tasks/",
+                    ConsupmtionTasksView.as_view(),
+                    name="consumption_tasks",
+                ),
                 path("token/", views.obtain_auth_token),
             ]
             + api_router.urls,