Merge pull request #1020 from paperless-ngx/feature-frontend-task-queue

Feature: frontend task queue

src/documents/migrations/1022_paperlesstask.py

@@ -0,0 +1,69 @@
+# Generated by Django 4.0.4 on 2022-05-23 07:14
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+def init_paperless_tasks(apps, schema_editor):
+    PaperlessTask = apps.get_model("documents", "PaperlessTask")
+    Task = apps.get_model("django_q", "Task")
+
+    for task in Task.objects.filter(func="documents.tasks.consume_file"):
+        if not hasattr(task, "paperlesstask"):
+            paperlesstask = PaperlessTask.objects.create(
+                attempted_task=task,
+                task_id=task.id,
+                name=task.name,
+                created=task.started,
+                started=task.started,
+                acknowledged=True,
+            )
+            task.paperlesstask = paperlesstask
+            task.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("django_q", "0014_schedule_cluster"),
+        ("documents", "1021_webp_thumbnail_conversion"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="PaperlessTask",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("task_id", models.CharField(max_length=128)),
+                ("name", models.CharField(max_length=256)),
+                (
+                    "created",
+                    models.DateTimeField(auto_now=True, verbose_name="created"),
+                ),
+                (
+                    "started",
+                    models.DateTimeField(null=True, verbose_name="started"),
+                ),
+                ("acknowledged", models.BooleanField(default=False)),
+                (
+                    "attempted_task",
+                    models.OneToOneField(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="attempted_task",
+                        to="django_q.task",
+                    ),
+                ),
+            ],
+        ),
+        migrations.RunPython(init_paperless_tasks, migrations.RunPython.noop),
+    ]

src/documents/models.py

@@ -12,6 +12,7 @@ from django.contrib.auth.models import User
 from django.db import models
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
+from django_q.tasks import Task
 from documents.parsers import get_default_file_extension
 
 
@@ -510,3 +511,19 @@ class UiSettings(models.Model):
 
     def __str__(self):
         return self.user.username
+
+
+class PaperlessTask(models.Model):
+
+    task_id = models.CharField(max_length=128)
+    name = models.CharField(max_length=256)
+    created = models.DateTimeField(_("created"), auto_now=True)
+    started = models.DateTimeField(_("started"), null=True)
+    attempted_task = models.OneToOneField(
+        Task,
+        on_delete=models.CASCADE,
+        related_name="attempted_task",
+        null=True,
+        blank=True,
+    )
+    acknowledged = models.BooleanField(default=False)

src/documents/serialisers.py

@@ -18,6 +18,7 @@ from .models import Correspondent
 from .models import Document
 from .models import DocumentType
 from .models import MatchingModel
+from .models import PaperlessTask
 from .models import SavedView
 from .models import SavedViewFilterRule
 from .models import StoragePath
@@ -612,3 +613,65 @@ class UiSettingsViewSerializer(serializers.ModelSerializer):
             defaults={"settings": validated_data.get("settings", None)},
         )
         return ui_settings
+
+
+class TasksViewSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = PaperlessTask
+        depth = 1
+        fields = "__all__"
+
+    type = serializers.SerializerMethodField()
+
+    def get_type(self, obj):
+        # just file tasks, for now
+        return "file"
+
+    result = serializers.SerializerMethodField()
+
+    def get_result(self, obj):
+        result = ""
+        if hasattr(obj, "attempted_task") and obj.attempted_task:
+            result = obj.attempted_task.result
+        return result
+
+    status = serializers.SerializerMethodField()
+
+    def get_status(self, obj):
+        if obj.attempted_task is None:
+            if obj.started:
+                return "started"
+            else:
+                return "queued"
+        elif obj.attempted_task.success:
+            return "complete"
+        elif not obj.attempted_task.success:
+            return "failed"
+        else:
+            return "unknown"
+
+
+class AcknowledgeTasksViewSerializer(serializers.Serializer):
+
+    tasks = serializers.ListField(
+        required=True,
+        label="Tasks",
+        write_only=True,
+        child=serializers.IntegerField(),
+    )
+
+    def _validate_task_id_list(self, tasks, name="tasks"):
+        pass
+        if not type(tasks) == list:
+            raise serializers.ValidationError(f"{name} must be a list")
+        if not all([type(i) == int for i in tasks]):
+            raise serializers.ValidationError(f"{name} must be a list of integers")
+        count = PaperlessTask.objects.filter(id__in=tasks).count()
+        if not count == len(tasks):
+            raise serializers.ValidationError(
+                f"Some tasks in {name} don't exist or were specified twice.",
+            )
+
+    def validate_tasks(self, tasks):
+        self._validate_task_id_list(tasks)
+        return tasks

src/documents/signals/handlers.py

@@ -2,6 +2,7 @@ import logging
 import os
 import shutil
 
+import django_q
 from django.conf import settings
 from django.contrib.admin.models import ADDITION
 from django.contrib.admin.models import LogEntry
@@ -21,6 +22,7 @@ from ..file_handling import delete_empty_directories
 from ..file_handling import generate_unique_filename
 from ..models import Document
 from ..models import MatchingModel
+from ..models import PaperlessTask
 from ..models import Tag
 
 
@@ -499,3 +501,36 @@ def add_to_index(sender, document, **kwargs):
     from documents import index
 
     index.add_or_update_document(document)
+
+
+@receiver(django_q.signals.pre_enqueue)
+def init_paperless_task(sender, task, **kwargs):
+    if task["func"] == "documents.tasks.consume_file":
+        paperless_task, created = PaperlessTask.objects.get_or_create(
+            task_id=task["id"],
+        )
+        paperless_task.name = task["name"]
+        paperless_task.created = task["started"]
+        paperless_task.save()
+
+
+@receiver(django_q.signals.pre_execute)
+def paperless_task_started(sender, task, **kwargs):
+    try:
+        if task["func"] == "documents.tasks.consume_file":
+            paperless_task = PaperlessTask.objects.get(task_id=task["id"])
+            paperless_task.started = timezone.now()
+            paperless_task.save()
+    except PaperlessTask.DoesNotExist:
+        pass
+
+
+@receiver(models.signals.post_save, sender=django_q.models.Task)
+def update_paperless_task(sender, instance, **kwargs):
+    try:
+        if instance.func == "documents.tasks.consume_file":
+            paperless_task = PaperlessTask.objects.get(task_id=instance.id)
+            paperless_task.attempted_task = instance
+            paperless_task.save()
+    except PaperlessTask.DoesNotExist:
+        pass

src/documents/tests/test_api.py

@@ -5,6 +5,7 @@ import os
 import shutil
 import tempfile
 import urllib.request
+import uuid
 import zipfile
 from unittest import mock
 from unittest.mock import MagicMock
@@ -19,12 +20,14 @@ from django.conf import settings
 from django.contrib.auth.models import User
 from django.test import override_settings
 from django.utils import timezone
+from django_q.models import Task
 from documents import bulk_edit
 from documents import index
 from documents.models import Correspondent
 from documents.models import Document
 from documents.models import DocumentType
 from documents.models import MatchingModel
+from documents.models import PaperlessTask
 from documents.models import SavedView
 from documents.models import StoragePath
 from documents.models import Tag
@@ -2645,3 +2648,55 @@ class TestApiStoragePaths(DirectoriesMixin, APITestCase):
         )
         self.assertEqual(response.status_code, 400)
         self.assertEqual(StoragePath.objects.count(), 1)
+
+
+class TestTasks(APITestCase):
+    ENDPOINT = "/api/tasks/"
+    ENDPOINT_ACKOWLEDGE = "/api/acknowledge_tasks/"
+
+    def setUp(self):
+        super().setUp()
+
+        self.user = User.objects.create_superuser(username="temp_admin")
+        self.client.force_authenticate(user=self.user)
+
+    def test_get_tasks(self):
+        task_id1 = str(uuid.uuid4())
+        PaperlessTask.objects.create(task_id=task_id1)
+        Task.objects.create(
+            id=task_id1,
+            started=timezone.now() - datetime.timedelta(seconds=30),
+            stopped=timezone.now(),
+            func="documents.tasks.consume_file",
+        )
+        task_id2 = str(uuid.uuid4())
+        PaperlessTask.objects.create(task_id=task_id2)
+
+        response = self.client.get(self.ENDPOINT)
+
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(len(response.data), 2)
+        returned_task1 = response.data[1]
+        returned_task2 = response.data[0]
+        self.assertEqual(returned_task1["task_id"], task_id1)
+        self.assertEqual(returned_task1["status"], "complete")
+        self.assertIsNotNone(returned_task1["attempted_task"])
+        self.assertEqual(returned_task2["task_id"], task_id2)
+        self.assertEqual(returned_task2["status"], "queued")
+        self.assertIsNone(returned_task2["attempted_task"])
+
+    def test_acknowledge_tasks(self):
+        task_id = str(uuid.uuid4())
+        task = PaperlessTask.objects.create(task_id=task_id)
+
+        response = self.client.get(self.ENDPOINT)
+        self.assertEqual(len(response.data), 1)
+
+        response = self.client.post(
+            self.ENDPOINT_ACKOWLEDGE,
+            {"tasks": [task.id]},
+        )
+        self.assertEqual(response.status_code, 200)
+
+        response = self.client.get(self.ENDPOINT)
+        self.assertEqual(len(response.data), 0)

src/documents/views.py

@@ -46,6 +46,7 @@ from rest_framework.response import Response
 from rest_framework.views import APIView
 from rest_framework.viewsets import GenericViewSet
 from rest_framework.viewsets import ModelViewSet
+from rest_framework.viewsets import ReadOnlyModelViewSet
 from rest_framework.viewsets import ViewSet
 
 from .bulk_download import ArchiveOnlyStrategy
@@ -64,10 +65,12 @@ from .matching import match_tags
 from .models import Correspondent
 from .models import Document
 from .models import DocumentType
+from .models import PaperlessTask
 from .models import SavedView
 from .models import StoragePath
 from .models import Tag
 from .parsers import get_parser_class_for_mime_type
+from .serialisers import AcknowledgeTasksViewSerializer
 from .serialisers import BulkDownloadSerializer
 from .serialisers import BulkEditSerializer
 from .serialisers import CorrespondentSerializer
@@ -79,6 +82,7 @@ from .serialisers import SavedViewSerializer
 from .serialisers import StoragePathSerializer
 from .serialisers import TagSerializer
 from .serialisers import TagSerializerVersion1
+from .serialisers import TasksViewSerializer
 from .serialisers import UiSettingsViewSerializer
 
 logger = logging.getLogger("paperless.api")
@@ -796,3 +800,37 @@ class UiSettingsView(GenericAPIView):
                 "success": True,
             },
         )
+
+
+class TasksViewSet(ReadOnlyModelViewSet):
+
+    permission_classes = (IsAuthenticated,)
+    serializer_class = TasksViewSerializer
+
+    queryset = (
+        PaperlessTask.objects.filter(
+            acknowledged=False,
+        )
+        .order_by("created")
+        .reverse()
+    )
+
+
+class AcknowledgeTasksView(GenericAPIView):
+
+    permission_classes = (IsAuthenticated,)
+    serializer_class = AcknowledgeTasksViewSerializer
+
+    def post(self, request, *args, **kwargs):
+        serializer = self.get_serializer(data=request.data)
+        serializer.is_valid(raise_exception=True)
+
+        tasks = serializer.validated_data.get("tasks")
+
+        try:
+            result = PaperlessTask.objects.filter(id__in=tasks).update(
+                acknowledged=True,
+            )
+            return Response({"result": result})
+        except Exception:
+            return HttpResponseBadRequest()

src/paperless/urls.py

@@ -7,6 +7,7 @@ from django.urls import re_path
 from django.utils.translation import gettext_lazy as _
 from django.views.decorators.csrf import csrf_exempt
 from django.views.generic import RedirectView
+from documents.views import AcknowledgeTasksView
 from documents.views import BulkDownloadView
 from documents.views import BulkEditView
 from documents.views import CorrespondentViewSet
@@ -21,6 +22,7 @@ from documents.views import SelectionDataView
 from documents.views import StatisticsView
 from documents.views import StoragePathViewSet
 from documents.views import TagViewSet
+from documents.views import TasksViewSet
 from documents.views import UiSettingsView
 from documents.views import UnifiedSearchViewSet
 from paperless.consumers import StatusConsumer
@@ -36,6 +38,7 @@ api_router.register(r"logs", LogViewSet, basename="logs")
 api_router.register(r"tags", TagViewSet)
 api_router.register(r"saved_views", SavedViewViewSet)
 api_router.register(r"storage_paths", StoragePathViewSet)
+api_router.register(r"tasks", TasksViewSet, basename="tasks")
 
 
 urlpatterns = [
@@ -86,6 +89,11 @@ urlpatterns = [
                     UiSettingsView.as_view(),
                     name="ui_settings",
                 ),
+                re_path(
+                    r"^acknowledge_tasks/",
+                    AcknowledgeTasksView.as_view(),
+                    name="acknowledge_tasks",
+                ),
                 path("token/", views.obtain_auth_token),
             ]
             + api_router.urls,