From 5c980c31becd9a05eb9882665d059e1ae787e027 Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+shamoon@users.noreply.github.com> Date: Mon, 23 May 2022 00:24:52 -0700 Subject: [PATCH 01/26] PaperlessTask and consumption_tasks endpoint --- .../migrations/1021_paperlesstask.py | 66 +++++++++++++++++++ src/documents/models.py | 16 +++++ src/documents/serialisers.py | 8 +++ src/documents/signals/handlers.py | 21 ++++++ src/documents/tasks.py | 14 ++++ src/documents/views.py | 49 ++++++++++++++ src/paperless/urls.py | 6 ++ 7 files changed, 180 insertions(+) create mode 100644 src/documents/migrations/1021_paperlesstask.py diff --git a/src/documents/migrations/1021_paperlesstask.py b/src/documents/migrations/1021_paperlesstask.py new file mode 100644 index 000000000..f827a892a --- /dev/null +++ b/src/documents/migrations/1021_paperlesstask.py @@ -0,0 +1,66 @@ +# Generated by Django 4.0.4 on 2022-05-23 07:14 + +from django.db import migrations, models +import django.db.models.deletion + + +def init_paperless_tasks(apps, schema_editor): + PaperlessTask = apps.get_model("documents", "PaperlessTask") + Task = apps.get_model("django_q", "Task") + + for task in Task.objects.all(): + if not hasattr(task, "paperlesstask"): + paperlesstask = PaperlessTask.objects.create( + task=task, + q_task_id=task.id, + name=task.name, + created=task.started, + acknowledged=False, + ) + task.paperlesstask = paperlesstask + task.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ("django_q", "0014_schedule_cluster"), + ("documents", "1020_merge_20220518_1839"), + ] + + operations = [ + migrations.CreateModel( + name="PaperlessTask", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("q_task_id", models.CharField(max_length=128)), + ("name", models.CharField(max_length=256)), + ( + "created", + models.DateTimeField( + auto_now=True, db_index=True, verbose_name="created" + ), + ), + ("acknowledged", models.BooleanField(default=False)), + ( + "task", + models.OneToOneField( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="task", + to="django_q.task", + ), + ), + ], + ), + migrations.RunPython(init_paperless_tasks, migrations.RunPython.noop), + ] diff --git a/src/documents/models.py b/src/documents/models.py index b85c56037..f7ce9ae95 100644 --- a/src/documents/models.py +++ b/src/documents/models.py @@ -11,6 +11,7 @@ from django.contrib.auth.models import User from django.db import models from django.utils import timezone from django.utils.translation import gettext_lazy as _ +from django_q.tasks import Task from documents.parsers import get_default_file_extension @@ -500,3 +501,18 @@ class UiSettings(models.Model): def __str__(self): return self.user.username + + +class PaperlessTask(models.Model): + + q_task_id = models.CharField(max_length=128) + name = models.CharField(max_length=256) + created = models.DateTimeField(_("created"), auto_now=True, db_index=True) + task = models.OneToOneField( + Task, + on_delete=models.CASCADE, + related_name="task", + null=True, + blank=True, + ) + acknowledged = models.BooleanField(default=False) diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py index 8459cd037..e89c5d686 100644 --- a/src/documents/serialisers.py +++ b/src/documents/serialisers.py @@ -595,3 +595,11 @@ class UiSettingsViewSerializer(serializers.ModelSerializer): defaults={"settings": validated_data.get("settings", None)}, ) return ui_settings + + +class ConsupmtionTasksViewSerializer(serializers.Serializer): + + type = serializers.ChoiceField( + choices=["all", "incomplete", "complete", "failed"], + default="all", + ) diff --git a/src/documents/signals/handlers.py b/src/documents/signals/handlers.py index 34710af78..2e763cee0 100644 --- a/src/documents/signals/handlers.py +++ b/src/documents/signals/handlers.py @@ -13,6 +13,9 @@ from django.db.models import Q from django.dispatch import receiver from django.utils import termcolors from django.utils import timezone +from django_q.signals import post_save +from django_q.signals import pre_enqueue +from django_q.tasks import Task from filelock import FileLock from .. import matching @@ -21,6 +24,7 @@ from ..file_handling import delete_empty_directories from ..file_handling import generate_unique_filename from ..models import Document from ..models import MatchingModel +from ..models import PaperlessTask from ..models import Tag @@ -499,3 +503,20 @@ def add_to_index(sender, document, **kwargs): from documents import index index.add_or_update_document(document) + + +@receiver(pre_enqueue) +def init_paperless_task(sender, task, **kwargs): + if task["func"] == "documents.tasks.consume_file": + paperless_task = PaperlessTask.objects.get_or_create(q_task_id=task["id"]) + paperless_task.name = task["name"] + paperless_task.created = task["started"] + + +@receiver(post_save, sender=Task) +def update_paperless_task(sender, instance, **kwargs): + logger.debug(sender, instance) + papeless_task = PaperlessTask.objects.find(q_task_id=instance.id) + if papeless_task: + papeless_task.task = instance + papeless_task.save() diff --git a/src/documents/tasks.py b/src/documents/tasks.py index 208f74f1d..241ec9766 100644 --- a/src/documents/tasks.py +++ b/src/documents/tasks.py @@ -10,6 +10,7 @@ from asgiref.sync import async_to_sync from channels.layers import get_channel_layer from django.conf import settings from django.db.models.signals import post_save +from django_q.tasks import Task from documents import index from documents import sanity_checker from documents.classifier import DocumentClassifier @@ -359,3 +360,16 @@ def bulk_update_documents(document_ids): with AsyncWriter(ix) as writer: for doc in documents: index.update_document(writer, doc) + + +def create_paperless_task(sender, instance, created, **kwargs): + if created: + Task.objects.create(thing=instance) + + +post_save.connect( + create_paperless_task, + sender=Task, + weak=False, + dispatch_uid="models.create_paperless_task", +) diff --git a/src/documents/views.py b/src/documents/views.py index cdd38180b..d7f8bf10b 100644 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -64,12 +64,14 @@ from .matching import match_tags from .models import Correspondent from .models import Document from .models import DocumentType +from .models import PaperlessTask from .models import SavedView from .models import StoragePath from .models import Tag from .parsers import get_parser_class_for_mime_type from .serialisers import BulkDownloadSerializer from .serialisers import BulkEditSerializer +from .serialisers import ConsupmtionTasksViewSerializer from .serialisers import CorrespondentSerializer from .serialisers import DocumentListSerializer from .serialisers import DocumentSerializer @@ -795,3 +797,50 @@ class UiSettingsView(GenericAPIView): "success": True, }, ) + + +class ConsupmtionTasksView(GenericAPIView): + + permission_classes = (IsAuthenticated,) + serializer_class = ConsupmtionTasksViewSerializer + + def get(self, request, format=None): + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + + consumption_tasks = ( + PaperlessTask.objects.filter( + acknowledged=False, + ) + .order_by("task__started") + .reverse() + ) + incomplete_tasks = consumption_tasks.filter(task=None).values( + "id", + "q_task_id", + "name", + "created", + "acknowledged", + ) + failed_tasks = consumption_tasks.filter(task__success=0).values( + "id", + "q_task_id", + "name", + "created", + "acknowledged", + ) + completed_tasks = consumption_tasks.filter(task__success=1).values( + "id", + "q_task_id", + "name", + "created", + "acknowledged", + ) + return Response( + { + "total": consumption_tasks.count(), + "incomplete": incomplete_tasks, + "failed": failed_tasks, + "completed": completed_tasks, + }, + ) diff --git a/src/paperless/urls.py b/src/paperless/urls.py index 003d79f2d..6cea1b9e4 100644 --- a/src/paperless/urls.py +++ b/src/paperless/urls.py @@ -9,6 +9,7 @@ from django.views.decorators.csrf import csrf_exempt from django.views.generic import RedirectView from documents.views import BulkDownloadView from documents.views import BulkEditView +from documents.views import ConsupmtionTasksView from documents.views import CorrespondentViewSet from documents.views import DocumentTypeViewSet from documents.views import IndexView @@ -86,6 +87,11 @@ urlpatterns = [ UiSettingsView.as_view(), name="ui_settings", ), + re_path( + r"^consumption_tasks/", + ConsupmtionTasksView.as_view(), + name="consumption_tasks", + ), path("token/", views.obtain_auth_token), ] + api_router.urls, From f88e0704550d0da18758e97089fc1dd8f557a613 Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+shamoon@users.noreply.github.com> Date: Mon, 23 May 2022 01:21:06 -0700 Subject: [PATCH 02/26] Toggle functionality for tasks list --- .../manage/tasks/tasks.component.html | 83 +++++++++++++++++ .../manage/tasks/tasks.component.ts | 75 ++++++++++++++++ src-ui/src/app/services/tasks.service.ts | 89 +++++++++++++++++++ 3 files changed, 247 insertions(+) create mode 100644 src-ui/src/app/components/manage/tasks/tasks.component.html create mode 100644 src-ui/src/app/components/manage/tasks/tasks.component.ts create mode 100644 src-ui/src/app/services/tasks.service.ts diff --git a/src-ui/src/app/components/manage/tasks/tasks.component.html b/src-ui/src/app/components/manage/tasks/tasks.component.html new file mode 100644 index 000000000..edbfa126a --- /dev/null +++ b/src-ui/src/app/components/manage/tasks/tasks.component.html @@ -0,0 +1,83 @@ + +
+ + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + +
+ 		NameCreatedActions
+
+ + +
+ 		{{ task.name }}{{ task.created | customDate:'medium' }} + +
+ + + +
diff --git a/src-ui/src/app/components/manage/tasks/tasks.component.ts b/src-ui/src/app/components/manage/tasks/tasks.component.ts new file mode 100644 index 000000000..e0e7c3466 --- /dev/null +++ b/src-ui/src/app/components/manage/tasks/tasks.component.ts @@ -0,0 +1,75 @@ +import { Component, OnInit, OnDestroy } from '@angular/core' +import { takeUntil, Subject } from 'rxjs' +import { PaperlessTask } from 'src/app/data/paperless-task' +import { TasksService } from 'src/app/services/tasks.service' + +@Component({ + selector: 'app-tasks', + templateUrl: './tasks.component.html', + styleUrls: ['./tasks.component.scss'], +}) +export class TasksComponent implements OnInit, OnDestroy { + public activeTab: string + public selectedTasks: Set = new Set() + private unsubscribeNotifer = new Subject() + + get dismissButtonText(): string { + return this.selectedTasks.size > 0 + ? $localize`Dismiss selected` + : $localize`Dismiss all` + } + + constructor(public tasksService: TasksService) {} + + ngOnInit() { + this.tasksService.reload() + } + + ngOnDestroy() { + this.unsubscribeNotifer.next(true) + } + + dismissTask(task: PaperlessTask) { + throw new Error('Not implemented' + task) + } + + dismissMany() { + throw new Error('Not implemented') + } + + toggleSelected(task: PaperlessTask) { + this.selectedTasks.has(task.id) + ? this.selectedTasks.delete(task.id) + : this.selectedTasks.add(task.id) + } + + get currentTasks(): PaperlessTask[] { + let tasks: PaperlessTask[] + switch (this.activeTab) { + case 'incomplete': + tasks = this.tasksService.incomplete + break + case 'completed': + tasks = this.tasksService.completed + break + case 'failed': + tasks = this.tasksService.failed + break + default: + break + } + return tasks + } + + toggleAll(event: PointerEvent) { + if ((event.target as HTMLInputElement).checked) { + this.selectedTasks = new Set(this.currentTasks.map((t) => t.id)) + } else { + this.clearSelection() + } + } + + clearSelection() { + this.selectedTasks = new Set() + } +} diff --git a/src-ui/src/app/services/tasks.service.ts b/src-ui/src/app/services/tasks.service.ts new file mode 100644 index 000000000..3560122a9 --- /dev/null +++ b/src-ui/src/app/services/tasks.service.ts @@ -0,0 +1,89 @@ +import { HttpClient } from '@angular/common/http' +import { Injectable } from '@angular/core' +import { Observable } from 'rxjs' +import { first, map } from 'rxjs/operators' +import { PaperlessTask } from 'src/app/data/paperless-task' +import { environment } from 'src/environments/environment' + +interface TasksAPIResponse { + total: number + incomplete: Array + completed: Array + failed: Array +} + +@Injectable({ + providedIn: 'root', +}) +export class TasksService { + private baseUrl: string = environment.apiBaseUrl + + loading: boolean + + public total: number + + private incompleteTasks: PaperlessTask[] = [] + public get incomplete(): PaperlessTask[] { + return this.incompleteTasks + } + + private completedTasks: PaperlessTask[] = [] + public get completed(): PaperlessTask[] { + return this.completedTasks + } + + private failedTasks: PaperlessTask[] = [] + public get failed(): PaperlessTask[] { + return this.failedTasks + } + + constructor(private http: HttpClient) {} + + public reload() { + this.loading = true + + this.http + .get(`${this.baseUrl}consumption_tasks/`) + .pipe(first()) + .subscribe((r) => { + this.total = r.total + this.incompleteTasks = r.incomplete + this.completedTasks = r.completed + this.failedTasks = r.failed + this.loading = false + return true + }) + } + + // private savedViews: PaperlessSavedView[] = [] + + // get allViews() { + // return this.savedViews + // } + + // get sidebarViews() { + // return this.savedViews.filter((v) => v.show_in_sidebar) + // } + + // get dashboardViews() { + // return this.savedViews.filter((v) => v.show_on_dashboard) + // } + + // create(o: PaperlessSavedView) { + // return super.create(o).pipe(tap(() => this.reload())) + // } + + // update(o: PaperlessSavedView) { + // return super.update(o).pipe(tap(() => this.reload())) + // } + + // patchMany(objects: PaperlessSavedView[]): Observable { + // return combineLatest(objects.map((o) => super.patch(o))).pipe( + // tap(() => this.reload()) + // ) + // } + + // delete(o: PaperlessSavedView) { + // return super.delete(o).pipe(tap(() => this.reload())) + // } +} From 4bbaf5f89c911937d311b8999c7295fcd897b0a9 Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+shamoon@users.noreply.github.com> Date: Mon, 23 May 2022 01:52:46 -0700 Subject: [PATCH 03/26] update post_save signal receiver --- src-ui/src/app/data/paperless-task.ts | 11 ++++++++ .../migrations/1021_paperlesstask.py | 10 ++++---- src/documents/models.py | 6 ++--- src/documents/signals/handlers.py | 25 +++++++++++-------- src/documents/tasks.py | 14 ----------- src/documents/views.py | 12 ++++----- 6 files changed, 39 insertions(+), 39 deletions(-) create mode 100644 src-ui/src/app/data/paperless-task.ts diff --git a/src-ui/src/app/data/paperless-task.ts b/src-ui/src/app/data/paperless-task.ts new file mode 100644 index 000000000..cc864710c --- /dev/null +++ b/src-ui/src/app/data/paperless-task.ts @@ -0,0 +1,11 @@ +import { ObjectWithId } from './object-with-id' + +export interface PaperlessTask extends ObjectWithId { + acknowledged: boolean + + task_id: string + + name: string + + created: Date +} diff --git a/src/documents/migrations/1021_paperlesstask.py b/src/documents/migrations/1021_paperlesstask.py index f827a892a..2c9a0a885 100644 --- a/src/documents/migrations/1021_paperlesstask.py +++ b/src/documents/migrations/1021_paperlesstask.py @@ -11,8 +11,8 @@ def init_paperless_tasks(apps, schema_editor): for task in Task.objects.all(): if not hasattr(task, "paperlesstask"): paperlesstask = PaperlessTask.objects.create( - task=task, - q_task_id=task.id, + attempted_task=task, + task_id=task.id, name=task.name, created=task.started, acknowledged=False, @@ -41,7 +41,7 @@ class Migration(migrations.Migration): verbose_name="ID", ), ), - ("q_task_id", models.CharField(max_length=128)), + ("task_id", models.CharField(max_length=128)), ("name", models.CharField(max_length=256)), ( "created", @@ -51,12 +51,12 @@ class Migration(migrations.Migration): ), ("acknowledged", models.BooleanField(default=False)), ( - "task", + "attempted_task", models.OneToOneField( blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, - related_name="task", + related_name="attempted_task", to="django_q.task", ), ), diff --git a/src/documents/models.py b/src/documents/models.py index f7ce9ae95..63c1c6e33 100644 --- a/src/documents/models.py +++ b/src/documents/models.py @@ -505,13 +505,13 @@ class UiSettings(models.Model): class PaperlessTask(models.Model): - q_task_id = models.CharField(max_length=128) + task_id = models.CharField(max_length=128) name = models.CharField(max_length=256) created = models.DateTimeField(_("created"), auto_now=True, db_index=True) - task = models.OneToOneField( + attempted_task = models.OneToOneField( Task, on_delete=models.CASCADE, - related_name="task", + related_name="attempted_task", null=True, blank=True, ) diff --git a/src/documents/signals/handlers.py b/src/documents/signals/handlers.py index 2e763cee0..2ae78ab80 100644 --- a/src/documents/signals/handlers.py +++ b/src/documents/signals/handlers.py @@ -2,6 +2,7 @@ import logging import os import shutil +import django_q from django.conf import settings from django.contrib.admin.models import ADDITION from django.contrib.admin.models import LogEntry @@ -13,9 +14,6 @@ from django.db.models import Q from django.dispatch import receiver from django.utils import termcolors from django.utils import timezone -from django_q.signals import post_save -from django_q.signals import pre_enqueue -from django_q.tasks import Task from filelock import FileLock from .. import matching @@ -505,18 +503,23 @@ def add_to_index(sender, document, **kwargs): index.add_or_update_document(document) -@receiver(pre_enqueue) +@receiver(django_q.signals.pre_enqueue) def init_paperless_task(sender, task, **kwargs): if task["func"] == "documents.tasks.consume_file": - paperless_task = PaperlessTask.objects.get_or_create(q_task_id=task["id"]) + paperless_task, created = PaperlessTask.objects.get_or_create( + task_id=task["id"], + ) paperless_task.name = task["name"] paperless_task.created = task["started"] + paperless_task.save() -@receiver(post_save, sender=Task) +@receiver(models.signals.post_save, sender=django_q.tasks.Task) def update_paperless_task(sender, instance, **kwargs): - logger.debug(sender, instance) - papeless_task = PaperlessTask.objects.find(q_task_id=instance.id) - if papeless_task: - papeless_task.task = instance - papeless_task.save() + try: + if instance.func == "documents.tasks.consume_file": + paperless_task = PaperlessTask.objects.get(task_id=instance.id) + paperless_task.attempted_task = instance + paperless_task.save() + except PaperlessTask.DoesNotExist: + pass diff --git a/src/documents/tasks.py b/src/documents/tasks.py index 241ec9766..208f74f1d 100644 --- a/src/documents/tasks.py +++ b/src/documents/tasks.py @@ -10,7 +10,6 @@ from asgiref.sync import async_to_sync from channels.layers import get_channel_layer from django.conf import settings from django.db.models.signals import post_save -from django_q.tasks import Task from documents import index from documents import sanity_checker from documents.classifier import DocumentClassifier @@ -360,16 +359,3 @@ def bulk_update_documents(document_ids): with AsyncWriter(ix) as writer: for doc in documents: index.update_document(writer, doc) - - -def create_paperless_task(sender, instance, created, **kwargs): - if created: - Task.objects.create(thing=instance) - - -post_save.connect( - create_paperless_task, - sender=Task, - weak=False, - dispatch_uid="models.create_paperless_task", -) diff --git a/src/documents/views.py b/src/documents/views.py index d7f8bf10b..5514d8d6c 100644 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -812,26 +812,26 @@ class ConsupmtionTasksView(GenericAPIView): PaperlessTask.objects.filter( acknowledged=False, ) - .order_by("task__started") + .order_by("attempted_task__started") .reverse() ) incomplete_tasks = consumption_tasks.filter(task=None).values( "id", - "q_task_id", + "task_id", "name", "created", "acknowledged", ) - failed_tasks = consumption_tasks.filter(task__success=0).values( + failed_tasks = consumption_tasks.filter(attempted_task__success=0).values( "id", - "q_task_id", + "task_id", "name", "created", "acknowledged", ) - completed_tasks = consumption_tasks.filter(task__success=1).values( + completed_tasks = consumption_tasks.filter(attempted_task__success=1).values( "id", - "q_task_id", + "task_id", "name", "created", "acknowledged", From 0a06c291e2d4399210ec6d2d767476f5ec769dc5 Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+shamoon@users.noreply.github.com> Date: Mon, 23 May 2022 10:45:14 -0700 Subject: [PATCH 04/26] acknowledge_tasks endpoint & basic UI Update tasks.service.ts --- .../manage/tasks/tasks.component.html | 2 +- .../manage/tasks/tasks.component.ts | 8 ++-- src-ui/src/app/services/tasks.service.ts | 44 +++++-------------- src/documents/serialisers.py | 29 +++++++++++- src/documents/views.py | 39 ++++++++++++---- src/paperless/urls.py | 14 ++++-- 6 files changed, 86 insertions(+), 50 deletions(-) diff --git a/src-ui/src/app/components/manage/tasks/tasks.component.html b/src-ui/src/app/components/manage/tasks/tasks.component.html index edbfa126a..e0249865d 100644 --- a/src-ui/src/app/components/manage/tasks/tasks.component.html +++ b/src-ui/src/app/components/manage/tasks/tasks.component.html @@ -5,7 +5,7 @@  Clear selection -