From 2ca691d3b8ec2129a6d29f3b9b3203c2288216d1 Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+shamoon@users.noreply.github.com> Date: Sun, 15 May 2022 21:09:42 -0700 Subject: [PATCH 001/173] use created_date --- .../saved-view-widget.component.html | 2 +- .../document-detail.component.html | 2 +- .../document-detail.component.ts | 34 ++----------------- src-ui/src/app/data/paperless-document.ts | 4 +++ src-ui/src/app/pipes/custom-date.pipe.ts | 2 -- src-ui/src/app/utils/date.ts | 5 --- .../app/utils/ngb-iso-date-time-adapter.ts | 23 ++++++++----- src/documents/models.py | 4 +++ src/documents/serialisers.py | 15 ++++++++ 9 files changed, 43 insertions(+), 48 deletions(-) delete mode 100644 src-ui/src/app/utils/date.ts diff --git a/src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.html b/src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.html index 50d064c37..84ee1aabe 100644 --- a/src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.html +++ b/src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.html @@ -12,7 +12,7 @@ - {{doc.created | customDate}} + {{doc.created_date | customDate}} {{doc.title | documentTitle}} diff --git a/src-ui/src/app/components/document-detail/document-detail.component.html b/src-ui/src/app/components/document-detail/document-detail.component.html index a4203473f..ca4fb4275 100644 --- a/src-ui/src/app/components/document-detail/document-detail.component.html +++ b/src-ui/src/app/components/document-detail/document-detail.component.html @@ -68,7 +68,7 @@ - + { + .subscribe(() => { this.error = null - if (this.ogDate) { - try { - let newDate = new Date(normalizeDateStr(changes['created'])) - newDate.setHours( - this.ogDate.getHours(), - this.ogDate.getMinutes(), - this.ogDate.getSeconds(), - this.ogDate.getMilliseconds() - ) - this.documentForm.patchValue( - { created: newDate.toISOString() }, - { emitEvent: false } - ) - } catch (e) { - // catch this before we try to save and simulate an api error - this.error = { created: e.message } - } - } - Object.assign(this.document, this.documentForm.value) }) @@ -223,25 +203,17 @@ export class DocumentDetailComponent }, }) - this.ogDate = new Date(normalizeDateStr(doc.created.toString())) - // Initialize dirtyCheck this.store = new BehaviorSubject({ title: doc.title, content: doc.content, - created: this.ogDate.toISOString(), + created_date: doc.created_date, correspondent: doc.correspondent, document_type: doc.document_type, archive_serial_number: doc.archive_serial_number, tags: [...doc.tags], }) - // start with ISO8601 string - this.documentForm.patchValue( - { created: this.ogDate.toISOString() }, - { emitEvent: false } - ) - this.isDirty$ = dirtyCheck( this.documentForm, this.store.asObservable() diff --git a/src-ui/src/app/data/paperless-document.ts b/src-ui/src/app/data/paperless-document.ts index 92a5aee45..051fa0aaa 100644 --- a/src-ui/src/app/data/paperless-document.ts +++ b/src-ui/src/app/data/paperless-document.ts @@ -32,8 +32,12 @@ export interface PaperlessDocument extends ObjectWithId { checksum?: string + // UTC created?: Date + // localized date + created_date?: Date + modified?: Date added?: Date diff --git a/src-ui/src/app/pipes/custom-date.pipe.ts b/src-ui/src/app/pipes/custom-date.pipe.ts index bd4833d04..bf5ab1457 100644 --- a/src-ui/src/app/pipes/custom-date.pipe.ts +++ b/src-ui/src/app/pipes/custom-date.pipe.ts @@ -1,7 +1,6 @@ import { DatePipe } from '@angular/common' import { Inject, LOCALE_ID, Pipe, PipeTransform } from '@angular/core' import { SettingsService, SETTINGS_KEYS } from '../services/settings.service' -import { normalizeDateStr } from '../utils/date' const FORMAT_TO_ISO_FORMAT = { longDate: 'y-MM-dd', @@ -34,7 +33,6 @@ export class CustomDatePipe implements PipeTransform { this.settings.get(SETTINGS_KEYS.DATE_LOCALE) || this.defaultLocale let f = format || this.settings.get(SETTINGS_KEYS.DATE_FORMAT) - if (typeof value == 'string') value = normalizeDateStr(value) if (l == 'iso-8601') { return this.datePipe.transform(value, FORMAT_TO_ISO_FORMAT[f], timezone) } else { diff --git a/src-ui/src/app/utils/date.ts b/src-ui/src/app/utils/date.ts deleted file mode 100644 index b62ffc939..000000000 --- a/src-ui/src/app/utils/date.ts +++ /dev/null @@ -1,5 +0,0 @@ -// see https://github.com/dateutil/dateutil/issues/878 , JS Date does not -// seem to accept these strings as valid dates so we must normalize offset -export function normalizeDateStr(dateStr: string): string { - return dateStr.replace(/[\+-](\d\d):\d\d:\d\d/gm, `-$1:00`) -} diff --git a/src-ui/src/app/utils/ngb-iso-date-time-adapter.ts b/src-ui/src/app/utils/ngb-iso-date-time-adapter.ts index b9acc38ec..9d0ea4de6 100644 --- a/src-ui/src/app/utils/ngb-iso-date-time-adapter.ts +++ b/src-ui/src/app/utils/ngb-iso-date-time-adapter.ts @@ -5,11 +5,20 @@ import { NgbDateAdapter, NgbDateStruct } from '@ng-bootstrap/ng-bootstrap' export class ISODateTimeAdapter extends NgbDateAdapter { fromModel(value: string | null): NgbDateStruct | null { if (value) { - let date = new Date(value) - return { - day: date.getDate(), - month: date.getMonth() + 1, - year: date.getFullYear(), + if (value.match(/\d\d\d\d\-\d\d\-\d\d/g)) { + const segs = value.split('-') + return { + year: parseInt(segs[0]), + month: parseInt(segs[1]), + day: parseInt(segs[2]), + } + } else { + let date = new Date(value) + return { + day: date.getDate(), + month: date.getMonth() + 1, + year: date.getFullYear(), + } } } else { return null @@ -17,8 +26,6 @@ export class ISODateTimeAdapter extends NgbDateAdapter { } toModel(date: NgbDateStruct | null): string | null { - return date - ? new Date(date.year, date.month - 1, date.day).toISOString() - : null + return date ? [date.year, date.month, date.day].join('-') : null } } diff --git a/src/documents/models.py b/src/documents/models.py index 206df4e8a..cc4cd0613 100644 --- a/src/documents/models.py +++ b/src/documents/models.py @@ -279,6 +279,10 @@ class Document(models.Model): def thumbnail_file(self): return open(self.thumbnail_path, "rb") + @property + def created_date(self): + return timezone.localdate(self.created) + class Log(models.Model): diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py index 1bdcbab9e..564c0aae5 100644 --- a/src/documents/serialisers.py +++ b/src/documents/serialisers.py @@ -1,7 +1,10 @@ +import datetime import math import re import magic +from dateutil import tz +from django.conf import settings from django.utils.text import slugify from django.utils.translation import gettext as _ from rest_framework import serializers @@ -206,6 +209,7 @@ class DocumentSerializer(DynamicFieldsModelSerializer): original_file_name = SerializerMethodField() archived_file_name = SerializerMethodField() + created_date = serializers.DateField() def get_original_file_name(self, obj): return obj.get_public_filename() @@ -216,6 +220,16 @@ class DocumentSerializer(DynamicFieldsModelSerializer): else: return None + def update(self, instance, validated_data): + if "created_date" in validated_data and "created" not in validated_data: + new_datetime = datetime.datetime.combine( + validated_data.get("created_date"), + datetime.time(0, 0, 0, 0, tz.gettz(settings.TIME_ZONE)), + ) + instance.created = new_datetime + instance.save() + return instance + class Meta: model = Document depth = 1 @@ -227,6 +241,7 @@ class DocumentSerializer(DynamicFieldsModelSerializer): "content", "tags", "created", + "created_date", "modified", "added", "archive_serial_number", From 063f6c1d5a4b7d3cc7e7bbac2cafd28db6be6a29 Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+shamoon@users.noreply.github.com> Date: Mon, 16 May 2022 00:01:57 -0700 Subject: [PATCH 002/173] only pass created_date from frontend --- src-ui/src/app/services/rest/document.service.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src-ui/src/app/services/rest/document.service.ts b/src-ui/src/app/services/rest/document.service.ts index 3b242543e..880d119dc 100644 --- a/src-ui/src/app/services/rest/document.service.ts +++ b/src-ui/src/app/services/rest/document.service.ts @@ -121,6 +121,12 @@ export class DocumentService extends AbstractPaperlessService return url } + update(o: PaperlessDocument): Observable { + // we want to only set created_date + o.created = undefined + return super.update(o) + } + uploadDocument(formData) { return this.http.post( this.getResourceUrl(null, 'post_document'), From c398f22e76ac9e096da84638cb3be58ba3f8321f Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+shamoon@users.noreply.github.com> Date: Mon, 16 May 2022 00:13:25 -0700 Subject: [PATCH 003/173] fix docs update endpoint --- src/documents/serialisers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py index 564c0aae5..eee4f0f9c 100644 --- a/src/documents/serialisers.py +++ b/src/documents/serialisers.py @@ -209,7 +209,7 @@ class DocumentSerializer(DynamicFieldsModelSerializer): original_file_name = SerializerMethodField() archived_file_name = SerializerMethodField() - created_date = serializers.DateField() + created_date = serializers.DateField(required=False) def get_original_file_name(self, obj): return obj.get_public_filename() @@ -228,6 +228,8 @@ class DocumentSerializer(DynamicFieldsModelSerializer): ) instance.created = new_datetime instance.save() + validated_data.pop("created_date") + super().update(instance, validated_data) return instance class Meta: From b2911b2eba9e293b7f7e8d36f610cc709285aa73 Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+shamoon@users.noreply.github.com> Date: Mon, 16 May 2022 00:30:21 -0700 Subject: [PATCH 004/173] use created_date in all of frontend --- .../document-card-large/document-card-large.component.html | 2 +- .../document-card-small/document-card-small.component.html | 2 +- .../app/components/document-list/document-list.component.html | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html index 5eb4a97dd..43e5ad2a8 100644 --- a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html +++ b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html @@ -78,7 +78,7 @@ - {{document.created | customDate:'mediumDate'}} + {{document.created_date | customDate:'mediumDate'}}
diff --git a/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.html b/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.html index 6c68cc26f..2697a9425 100644 --- a/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.html +++ b/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.html @@ -51,7 +51,7 @@ - {{document.created | customDate:'mediumDate'}} + {{document.created_date | customDate:'mediumDate'}}
diff --git a/src-ui/src/app/components/document-list/document-list.component.html b/src-ui/src/app/components/document-list/document-list.component.html index aa43243fc..80ccdf01b 100644 --- a/src-ui/src/app/components/document-list/document-list.component.html +++ b/src-ui/src/app/components/document-list/document-list.component.html @@ -177,7 +177,7 @@ - {{d.created | customDate}} + {{d.created_date | customDate}} {{d.added | customDate}} From 94954eeba3e93d10c1423260561b5ef0858f7730 Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+shamoon@users.noreply.github.com> Date: Mon, 16 May 2022 00:31:33 -0700 Subject: [PATCH 005/173] created_date error display --- .../components/document-detail/document-detail.component.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src-ui/src/app/components/document-detail/document-detail.component.html b/src-ui/src/app/components/document-detail/document-detail.component.html index ca4fb4275..2225afa31 100644 --- a/src-ui/src/app/components/document-detail/document-detail.component.html +++ b/src-ui/src/app/components/document-detail/document-detail.component.html @@ -68,7 +68,7 @@ - + Date: Mon, 16 May 2022 00:29:49 -0700 Subject: [PATCH 006/173] Consolidate date adapter --- src-ui/src/app/app.module.ts | 4 +-- src-ui/src/app/utils/ngb-iso-date-adapter.ts | 19 +++++++++--- .../app/utils/ngb-iso-date-time-adapter.ts | 31 ------------------- 3 files changed, 16 insertions(+), 38 deletions(-) delete mode 100644 src-ui/src/app/utils/ngb-iso-date-time-adapter.ts diff --git a/src-ui/src/app/app.module.ts b/src-ui/src/app/app.module.ts index e891d217d..a705c83ec 100644 --- a/src-ui/src/app/app.module.ts +++ b/src-ui/src/app/app.module.ts @@ -61,7 +61,7 @@ import { SafeUrlPipe } from './pipes/safeurl.pipe' import { SafeHtmlPipe } from './pipes/safehtml.pipe' import { CustomDatePipe } from './pipes/custom-date.pipe' import { DateComponent } from './components/common/input/date/date.component' -import { ISODateTimeAdapter } from './utils/ngb-iso-date-time-adapter' +import { ISODateAdapter } from './utils/ngb-iso-date-adapter' import { LocalizedDateParserFormatter } from './utils/ngb-date-parser-formatter' import { ApiVersionInterceptor } from './interceptors/api-version.interceptor' import { ColorSliderModule } from 'ngx-color/slider' @@ -188,7 +188,7 @@ registerLocaleData(localeZh) }, FilterPipe, DocumentTitlePipe, - { provide: NgbDateAdapter, useClass: ISODateTimeAdapter }, + { provide: NgbDateAdapter, useClass: ISODateAdapter }, { provide: NgbDateParserFormatter, useClass: LocalizedDateParserFormatter }, ], bootstrap: [AppComponent], diff --git a/src-ui/src/app/utils/ngb-iso-date-adapter.ts b/src-ui/src/app/utils/ngb-iso-date-adapter.ts index 7aa2214b5..af2fff3f3 100644 --- a/src-ui/src/app/utils/ngb-iso-date-adapter.ts +++ b/src-ui/src/app/utils/ngb-iso-date-adapter.ts @@ -5,11 +5,20 @@ import { NgbDateAdapter, NgbDateStruct } from '@ng-bootstrap/ng-bootstrap' export class ISODateAdapter extends NgbDateAdapter { fromModel(value: string | null): NgbDateStruct | null { if (value) { - let date = new Date(value) - return { - day: date.getDate(), - month: date.getMonth() + 1, - year: date.getFullYear(), + if (value.match(/\d\d\d\d\-\d\d\-\d\d/g)) { + const segs = value.split('-') + return { + year: parseInt(segs[0]), + month: parseInt(segs[1]), + day: parseInt(segs[2]), + } + } else { + let date = new Date(value) + return { + day: date.getDate(), + month: date.getMonth() + 1, + year: date.getFullYear(), + } } } else { return null diff --git a/src-ui/src/app/utils/ngb-iso-date-time-adapter.ts b/src-ui/src/app/utils/ngb-iso-date-time-adapter.ts deleted file mode 100644 index 9d0ea4de6..000000000 --- a/src-ui/src/app/utils/ngb-iso-date-time-adapter.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { Injectable } from '@angular/core' -import { NgbDateAdapter, NgbDateStruct } from '@ng-bootstrap/ng-bootstrap' - -@Injectable() -export class ISODateTimeAdapter extends NgbDateAdapter { - fromModel(value: string | null): NgbDateStruct | null { - if (value) { - if (value.match(/\d\d\d\d\-\d\d\-\d\d/g)) { - const segs = value.split('-') - return { - year: parseInt(segs[0]), - month: parseInt(segs[1]), - day: parseInt(segs[2]), - } - } else { - let date = new Date(value) - return { - day: date.getDate(), - month: date.getMonth() + 1, - year: date.getFullYear(), - } - } - } else { - return null - } - } - - toModel(date: NgbDateStruct | null): string | null { - return date ? [date.year, date.month, date.day].join('-') : null - } -} From 6fbe4404f5e491bf4827111e6f826f3d909572b4 Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Mon, 16 May 2022 11:34:30 -0700 Subject: [PATCH 007/173] Update api.rst --- docs/api.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/api.rst b/docs/api.rst index c23866b60..1e70b19fa 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -31,7 +31,8 @@ The objects served by the document endpoint contain the following fields: * ``tags``: List of IDs of tags assigned to this document, or empty list. * ``document_type``: Document type of this document, or null. * ``correspondent``: Correspondent of this document or null. -* ``created``: The date at which this document was created. +* ``created``: The date time at which this document was created. +* ``created_date``: The date (YYYY-MM-DD) at which this document was created. Optional. If also passed with created, this is ignored. * ``modified``: The date at which this document was last edited in paperless. Read-only. * ``added``: The date at which this document was added to paperless. Read-only. * ``archive_serial_number``: The identifier of this document in a physical document archive. From 88b7f8ac1e2c9c3f157be91b1b9b7c22a40d2a98 Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+shamoon@users.noreply.github.com> Date: Mon, 16 May 2022 21:39:10 -0700 Subject: [PATCH 008/173] Use zoneinfo instead of tz Co-Authored-By: Trenton Holmes <797416+stumpylog@users.noreply.github.com> --- src/documents/serialisers.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py index eee4f0f9c..db55226e2 100644 --- a/src/documents/serialisers.py +++ b/src/documents/serialisers.py @@ -2,8 +2,11 @@ import datetime import math import re +try: + import zoneinfo +except ImportError: + import backports.zoneinfo as zoneinfo import magic -from dateutil import tz from django.conf import settings from django.utils.text import slugify from django.utils.translation import gettext as _ @@ -224,7 +227,7 @@ class DocumentSerializer(DynamicFieldsModelSerializer): if "created_date" in validated_data and "created" not in validated_data: new_datetime = datetime.datetime.combine( validated_data.get("created_date"), - datetime.time(0, 0, 0, 0, tz.gettz(settings.TIME_ZONE)), + datetime.time(0, 0, 0, 0, zoneinfo.ZoneInfo(settings.TIME_ZONE)), ) instance.created = new_datetime instance.save() From e60a7df9a2586cefb7532a3854503c51d946068e Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+shamoon@users.noreply.github.com> Date: Fri, 20 May 2022 15:16:17 -0700 Subject: [PATCH 009/173] Refactor query params service --- .../app-frame/app-frame.component.ts | 5 +- .../saved-view-widget.component.ts | 6 +- .../document-detail.component.ts | 6 +- .../document-list.component.html | 2 +- .../document-list/document-list.component.ts | 49 ++---- .../correspondent-list.component.ts | 6 +- .../document-type-list.component.ts | 6 +- .../management-list.component.ts | 6 +- .../storage-path-list.component.ts | 5 +- .../manage/tag-list/tag-list.component.ts | 6 +- .../services/document-list-view.service.ts | 98 +++++++---- .../src/app/services/query-params.service.ts | 163 ------------------ .../src/app/services/rest/document.service.ts | 4 +- src-ui/src/app/utils/query-params.ts | 100 +++++++++++ 14 files changed, 205 insertions(+), 257 deletions(-) delete mode 100644 src-ui/src/app/services/query-params.service.ts create mode 100644 src-ui/src/app/utils/query-params.ts diff --git a/src-ui/src/app/components/app-frame/app-frame.component.ts b/src-ui/src/app/components/app-frame/app-frame.component.ts index 675bfc920..0d43f17a2 100644 --- a/src-ui/src/app/components/app-frame/app-frame.component.ts +++ b/src-ui/src/app/components/app-frame/app-frame.component.ts @@ -22,7 +22,6 @@ import { RemoteVersionService, AppRemoteVersion, } from 'src/app/services/rest/remote-version.service' -import { QueryParamsService } from 'src/app/services/query-params.service' import { SettingsService } from 'src/app/services/settings.service' @Component({ @@ -38,7 +37,7 @@ export class AppFrameComponent { private searchService: SearchService, public savedViewService: SavedViewService, private remoteVersionService: RemoteVersionService, - private queryParamsService: QueryParamsService, + private list: DocumentListViewService, public settingsService: SettingsService ) { this.remoteVersionService @@ -94,7 +93,7 @@ export class AppFrameComponent { search() { this.closeMenu() - this.queryParamsService.navigateWithFilterRules([ + this.list.quickFilter([ { rule_type: FILTER_FULLTEXT_QUERY, value: (this.searchField.value as string).trim(), diff --git a/src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.ts b/src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.ts index 94e0c4052..17a0d8c89 100644 --- a/src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.ts +++ b/src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.ts @@ -7,8 +7,8 @@ import { ConsumerStatusService } from 'src/app/services/consumer-status.service' import { DocumentService } from 'src/app/services/rest/document.service' import { PaperlessTag } from 'src/app/data/paperless-tag' import { FILTER_HAS_TAGS_ALL } from 'src/app/data/filter-rule-type' -import { QueryParamsService } from 'src/app/services/query-params.service' import { OpenDocumentsService } from 'src/app/services/open-documents.service' +import { DocumentListViewService } from 'src/app/services/document-list-view.service' @Component({ selector: 'app-saved-view-widget', @@ -21,7 +21,7 @@ export class SavedViewWidgetComponent implements OnInit, OnDestroy { constructor( private documentService: DocumentService, private router: Router, - private queryParamsService: QueryParamsService, + private list: DocumentListViewService, private consumerStatusService: ConsumerStatusService, public openDocumentsService: OpenDocumentsService ) {} @@ -73,7 +73,7 @@ export class SavedViewWidgetComponent implements OnInit, OnDestroy { } clickTag(tag: PaperlessTag) { - this.queryParamsService.navigateWithFilterRules([ + this.list.quickFilter([ { rule_type: FILTER_HAS_TAGS_ALL, value: tag.id.toString() }, ]) } diff --git a/src-ui/src/app/components/document-detail/document-detail.component.ts b/src-ui/src/app/components/document-detail/document-detail.component.ts index 6728f746d..c4255e9f7 100644 --- a/src-ui/src/app/components/document-detail/document-detail.component.ts +++ b/src-ui/src/app/components/document-detail/document-detail.component.ts @@ -32,7 +32,6 @@ import { import { PaperlessDocumentSuggestions } from 'src/app/data/paperless-document-suggestions' import { FILTER_FULLTEXT_MORELIKE } from 'src/app/data/filter-rule-type' import { normalizeDateStr } from 'src/app/utils/date' -import { QueryParamsService } from 'src/app/services/query-params.service' import { StoragePathService } from 'src/app/services/rest/storage-path.service' import { PaperlessStoragePath } from 'src/app/data/paperless-storage-path' import { StoragePathEditDialogComponent } from '../common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component' @@ -120,8 +119,7 @@ export class DocumentDetailComponent private documentTitlePipe: DocumentTitlePipe, private toastService: ToastService, private settings: SettingsService, - private storagePathService: StoragePathService, - private queryParamsService: QueryParamsService + private storagePathService: StoragePathService ) {} titleKeyUp(event) { @@ -494,7 +492,7 @@ export class DocumentDetailComponent } moreLike() { - this.queryParamsService.navigateWithFilterRules([ + this.documentListViewService.quickFilter([ { rule_type: FILTER_FULLTEXT_MORELIKE, value: this.documentId.toString(), diff --git a/src-ui/src/app/components/document-list/document-list.component.html b/src-ui/src/app/components/document-list/document-list.component.html index e8cb78995..f812be217 100644 --- a/src-ui/src/app/components/document-list/document-list.component.html +++ b/src-ui/src/app/components/document-list/document-list.component.html @@ -93,7 +93,7 @@ {list.collectionSize, plural, =1 {One document} other {{{list.collectionSize || 0}} documents}} (filtered)

-
diff --git a/src-ui/src/app/components/document-list/document-list.component.ts b/src-ui/src/app/components/document-list/document-list.component.ts index d9355902f..e27f14b14 100644 --- a/src-ui/src/app/components/document-list/document-list.component.ts +++ b/src-ui/src/app/components/document-list/document-list.component.ts @@ -1,5 +1,4 @@ import { - AfterViewInit, Component, OnDestroy, OnInit, @@ -21,7 +20,6 @@ import { import { ConsumerStatusService } from 'src/app/services/consumer-status.service' import { DocumentListViewService } from 'src/app/services/document-list-view.service' import { OpenDocumentsService } from 'src/app/services/open-documents.service' -import { QueryParamsService } from 'src/app/services/query-params.service' import { DOCUMENT_SORT_FIELDS, DOCUMENT_SORT_FIELDS_FULLTEXT, @@ -36,7 +34,7 @@ import { SaveViewConfigDialogComponent } from './save-view-config-dialog/save-vi templateUrl: './document-list.component.html', styleUrls: ['./document-list.component.scss'], }) -export class DocumentListComponent implements OnInit, OnDestroy, AfterViewInit { +export class DocumentListComponent implements OnInit, OnDestroy { constructor( public list: DocumentListViewService, public savedViewService: SavedViewService, @@ -45,7 +43,6 @@ export class DocumentListComponent implements OnInit, OnDestroy, AfterViewInit { private toastService: ToastService, private modalService: NgbModal, private consumerStatusService: ConsumerStatusService, - private queryParamsService: QueryParamsService, public openDocumentsService: OpenDocumentsService ) {} @@ -76,8 +73,6 @@ export class DocumentListComponent implements OnInit, OnDestroy, AfterViewInit { set listSort(reverse: boolean) { this.list.sortReverse = reverse - this.queryParamsService.sortField = this.list.sortField - this.queryParamsService.sortReverse = reverse } get listSort(): boolean { @@ -86,14 +81,14 @@ export class DocumentListComponent implements OnInit, OnDestroy, AfterViewInit { setSortField(field: string) { this.list.sortField = field - this.queryParamsService.sortField = field - this.queryParamsService.sortReverse = this.listSort } onSort(event: SortEvent) { this.list.setSort(event.column, event.reverse) - this.queryParamsService.sortField = event.column - this.queryParamsService.sortReverse = event.reverse + } + + setPage(page: number) { + this.list.currentPage = page } get isBulkEditing(): boolean { @@ -133,7 +128,6 @@ export class DocumentListComponent implements OnInit, OnDestroy, AfterViewInit { } this.list.activateSavedView(view) this.list.reload() - this.queryParamsService.updateFromView(view) this.unmodifiedFilterRules = view.filter_rules }) @@ -148,22 +142,12 @@ export class DocumentListComponent implements OnInit, OnDestroy, AfterViewInit { this.loadViewConfig(parseInt(queryParams.get('view'))) } else { this.list.activateSavedView(null) - this.queryParamsService.parseQueryParams(queryParams) + this.list.loadFromQueryParams(queryParams) this.unmodifiedFilterRules = [] } }) } - ngAfterViewInit(): void { - this.filterEditor.filterRulesChange - .pipe(takeUntil(this.unsubscribeNotifier)) - .subscribe({ - next: (filterRules) => { - this.queryParamsService.updateFilterRules(filterRules) - }, - }) - } - ngOnDestroy() { // unsubscribes all this.unsubscribeNotifier.next(this) @@ -175,9 +159,8 @@ export class DocumentListComponent implements OnInit, OnDestroy, AfterViewInit { .getCached(viewId) .pipe(first()) .subscribe((view) => { - this.list.loadSavedView(view) + this.list.activateSavedView(view) this.list.reload() - this.queryParamsService.updateFromView(view) }) } @@ -246,34 +229,26 @@ export class DocumentListComponent implements OnInit, OnDestroy, AfterViewInit { clickTag(tagID: number) { this.list.selectNone() - setTimeout(() => { - this.filterEditor.addTag(tagID) - }) + this.filterEditor.addTag(tagID) } clickCorrespondent(correspondentID: number) { this.list.selectNone() - setTimeout(() => { - this.filterEditor.addCorrespondent(correspondentID) - }) + this.filterEditor.addCorrespondent(correspondentID) } clickDocumentType(documentTypeID: number) { this.list.selectNone() - setTimeout(() => { - this.filterEditor.addDocumentType(documentTypeID) - }) + this.filterEditor.addDocumentType(documentTypeID) } clickStoragePath(storagePathID: number) { this.list.selectNone() - setTimeout(() => { - this.filterEditor.addStoragePath(storagePathID) - }) + this.filterEditor.addStoragePath(storagePathID) } clickMoreLike(documentID: number) { - this.queryParamsService.navigateWithFilterRules([ + this.list.quickFilter([ { rule_type: FILTER_FULLTEXT_MORELIKE, value: documentID.toString() }, ]) } diff --git a/src-ui/src/app/components/manage/correspondent-list/correspondent-list.component.ts b/src-ui/src/app/components/manage/correspondent-list/correspondent-list.component.ts index 9eb05758c..983c36290 100644 --- a/src-ui/src/app/components/manage/correspondent-list/correspondent-list.component.ts +++ b/src-ui/src/app/components/manage/correspondent-list/correspondent-list.component.ts @@ -3,7 +3,7 @@ import { NgbModal } from '@ng-bootstrap/ng-bootstrap' import { FILTER_CORRESPONDENT } from 'src/app/data/filter-rule-type' import { PaperlessCorrespondent } from 'src/app/data/paperless-correspondent' import { CustomDatePipe } from 'src/app/pipes/custom-date.pipe' -import { QueryParamsService } from 'src/app/services/query-params.service' +import { DocumentListViewService } from 'src/app/services/document-list-view.service' import { CorrespondentService } from 'src/app/services/rest/correspondent.service' import { ToastService } from 'src/app/services/toast.service' import { CorrespondentEditDialogComponent } from '../../common/edit-dialog/correspondent-edit-dialog/correspondent-edit-dialog.component' @@ -20,7 +20,7 @@ export class CorrespondentListComponent extends ManagementListComponent private modalService: NgbModal, private editDialogComponent: any, private toastService: ToastService, - private queryParamsService: QueryParamsService, + private documentListViewService: DocumentListViewService, protected filterRuleType: number, public typeName: string, public typeNamePlural: string, @@ -141,7 +141,7 @@ export abstract class ManagementListComponent } filterDocuments(object: ObjectWithId) { - this.queryParamsService.navigateWithFilterRules([ + this.documentListViewService.quickFilter([ { rule_type: this.filterRuleType, value: object.id.toString() }, ]) } diff --git a/src-ui/src/app/components/manage/storage-path-list/storage-path-list.component.ts b/src-ui/src/app/components/manage/storage-path-list/storage-path-list.component.ts index dc363c4d5..1d7b726a0 100644 --- a/src-ui/src/app/components/manage/storage-path-list/storage-path-list.component.ts +++ b/src-ui/src/app/components/manage/storage-path-list/storage-path-list.component.ts @@ -3,7 +3,6 @@ import { NgbModal } from '@ng-bootstrap/ng-bootstrap' import { FILTER_STORAGE_PATH } from 'src/app/data/filter-rule-type' import { PaperlessStoragePath } from 'src/app/data/paperless-storage-path' import { DocumentListViewService } from 'src/app/services/document-list-view.service' -import { QueryParamsService } from 'src/app/services/query-params.service' import { StoragePathService } from 'src/app/services/rest/storage-path.service' import { ToastService } from 'src/app/services/toast.service' import { StoragePathEditDialogComponent } from '../../common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component' @@ -19,14 +18,14 @@ export class StoragePathListComponent extends ManagementListComponent { tagService: TagService, modalService: NgbModal, toastService: ToastService, - queryParamsService: QueryParamsService + documentListViewService: DocumentListViewService ) { super( tagService, modalService, TagEditDialogComponent, toastService, - queryParamsService, + documentListViewService, FILTER_HAS_TAGS_ALL, $localize`tag`, $localize`tags`, diff --git a/src-ui/src/app/services/document-list-view.service.ts b/src-ui/src/app/services/document-list-view.service.ts index 471fc7944..52a0de296 100644 --- a/src-ui/src/app/services/document-list-view.service.ts +++ b/src-ui/src/app/services/document-list-view.service.ts @@ -10,13 +10,14 @@ import { PaperlessDocument } from '../data/paperless-document' import { PaperlessSavedView } from '../data/paperless-saved-view' import { SETTINGS_KEYS } from '../data/paperless-uisettings' import { DOCUMENT_LIST_SERVICE } from '../data/storage-keys' +import { generateParams, parseQueryParams } from '../utils/query-params' import { DocumentService, DOCUMENT_SORT_FIELDS } from './rest/document.service' import { SettingsService } from './settings.service' /** * Captures the current state of the list view. */ -interface ListViewState { +export interface ListViewState { /** * Title of the document list view. Either "Documents" (localized) or the name of a saved view. */ @@ -32,7 +33,7 @@ interface ListViewState { /** * Total amount of documents with the current filter rules. Used to calculate the number of pages. */ - collectionSize: number + collectionSize?: number /** * Currently selected sort field. @@ -85,6 +86,32 @@ export class DocumentListViewService { return this.activeListViewState.title } + constructor( + private documentService: DocumentService, + private settings: SettingsService, + private router: Router + ) { + let documentListViewConfigJson = localStorage.getItem( + DOCUMENT_LIST_SERVICE.CURRENT_VIEW_CONFIG + ) + if (documentListViewConfigJson) { + try { + let savedState: ListViewState = JSON.parse(documentListViewConfigJson) + // Remove null elements from the restored state + Object.keys(savedState).forEach((k) => { + if (savedState[k] == null) { + delete savedState[k] + } + }) + //only use restored state attributes instead of defaults if they are not null + let newState = Object.assign(this.defaultListViewState(), savedState) + this.listViewStates.set(null, newState) + } catch (e) { + localStorage.removeItem(DOCUMENT_LIST_SERVICE.CURRENT_VIEW_CONFIG) + } + } + } + private defaultListViewState(): ListViewState { return { title: null, @@ -122,20 +149,36 @@ export class DocumentListViewService { if (closeCurrentView) { this._activeSavedViewId = null } + this.activeListViewState.filterRules = cloneFilterRules(view.filter_rules) this.activeListViewState.sortField = view.sort_field this.activeListViewState.sortReverse = view.sort_reverse if (this._activeSavedViewId) { this.activeListViewState.title = view.name } + this.reduceSelectionToFilter() + + if (!this.router.routerState.snapshot.url.includes('/view/')) { + this.router.navigate([], { + queryParams: { view: view.id }, + }) + } } - reload(onFinish?) { + loadFromQueryParams(queryParams) { + let newState: ListViewState = parseQueryParams(queryParams) + this.activeListViewState.filterRules = newState.filterRules + this.activeListViewState.sortField = newState.sortField + this.activeListViewState.sortReverse = newState.sortReverse + this.activeListViewState.currentPage = newState.currentPage + this.reload(null, false) + } + + reload(onFinish?, updateQueryParams: boolean = true) { this.isReloading = true this.error = null let activeListViewState = this.activeListViewState - this.documentService .listFiltered( activeListViewState.currentPage, @@ -149,6 +192,19 @@ export class DocumentListViewService { this.isReloading = false activeListViewState.collectionSize = result.count activeListViewState.documents = result.results + + if (updateQueryParams && !this._activeSavedViewId) { + let base = ['/documents'] + this.router.navigate(base, { + queryParams: generateParams( + activeListViewState.filterRules, + activeListViewState.sortField, + activeListViewState.sortReverse, + activeListViewState.currentPage + ), + }) + } + if (onFinish) { onFinish() } @@ -191,6 +247,7 @@ export class DocumentListViewService { ) { this.activeListViewState.sortField = 'created' } + this._activeSavedViewId = null this.activeListViewState.filterRules = filterRules this.reload() this.reduceSelectionToFilter() @@ -202,6 +259,7 @@ export class DocumentListViewService { } set sortField(field: string) { + this._activeSavedViewId = null this.activeListViewState.sortField = field this.reload() this.saveDocumentListView() @@ -212,6 +270,7 @@ export class DocumentListViewService { } set sortReverse(reverse: boolean) { + this._activeSavedViewId = null this.activeListViewState.sortReverse = reverse this.reload() this.saveDocumentListView() @@ -237,6 +296,8 @@ export class DocumentListViewService { } set currentPage(page: number) { + if (this.activeListViewState.currentPage == page) return + this._activeSavedViewId = null this.activeListViewState.currentPage = page this.reload() this.saveDocumentListView() @@ -273,6 +334,10 @@ export class DocumentListViewService { } } + quickFilter(filterRules: FilterRule[]) { + this.filterRules = filterRules + } + getLastPage(): number { return Math.ceil(this.collectionSize / this.currentPageSize) } @@ -431,29 +496,4 @@ export class DocumentListViewService { documentIndexInCurrentView(documentID: number): number { return this.documents.map((d) => d.id).indexOf(documentID) } - - constructor( - private documentService: DocumentService, - private settings: SettingsService - ) { - let documentListViewConfigJson = localStorage.getItem( - DOCUMENT_LIST_SERVICE.CURRENT_VIEW_CONFIG - ) - if (documentListViewConfigJson) { - try { - let savedState: ListViewState = JSON.parse(documentListViewConfigJson) - // Remove null elements from the restored state - Object.keys(savedState).forEach((k) => { - if (savedState[k] == null) { - delete savedState[k] - } - }) - //only use restored state attributes instead of defaults if they are not null - let newState = Object.assign(this.defaultListViewState(), savedState) - this.listViewStates.set(null, newState) - } catch (e) { - localStorage.removeItem(DOCUMENT_LIST_SERVICE.CURRENT_VIEW_CONFIG) - } - } - } } diff --git a/src-ui/src/app/services/query-params.service.ts b/src-ui/src/app/services/query-params.service.ts deleted file mode 100644 index 888440aae..000000000 --- a/src-ui/src/app/services/query-params.service.ts +++ /dev/null @@ -1,163 +0,0 @@ -import { Injectable } from '@angular/core' -import { ParamMap, Params, Router } from '@angular/router' -import { FilterRule } from '../data/filter-rule' -import { FilterRuleType, FILTER_RULE_TYPES } from '../data/filter-rule-type' -import { PaperlessSavedView } from '../data/paperless-saved-view' -import { DocumentListViewService } from './document-list-view.service' - -const SORT_FIELD_PARAMETER = 'sort' -const SORT_REVERSE_PARAMETER = 'reverse' - -@Injectable({ - providedIn: 'root', -}) -export class QueryParamsService { - constructor(private router: Router, private list: DocumentListViewService) {} - - private filterParams: Params = {} - private sortParams: Params = {} - - updateFilterRules( - filterRules: FilterRule[], - updateQueryParams: boolean = true - ) { - this.filterParams = filterRulesToQueryParams(filterRules) - if (updateQueryParams) this.updateQueryParams() - } - - set sortField(field: string) { - this.sortParams[SORT_FIELD_PARAMETER] = field - this.updateQueryParams() - } - - set sortReverse(reverse: boolean) { - if (!reverse) this.sortParams[SORT_REVERSE_PARAMETER] = undefined - else this.sortParams[SORT_REVERSE_PARAMETER] = reverse - this.updateQueryParams() - } - - get params(): Params { - return { - ...this.sortParams, - ...this.filterParams, - } - } - - private updateQueryParams() { - // if we were on a saved view we navigate 'away' to /documents - let base = [] - if (this.router.routerState.snapshot.url.includes('/view/')) - base = ['/documents'] - - this.router.navigate(base, { - queryParams: this.params, - }) - } - - public parseQueryParams(queryParams: ParamMap) { - let filterRules = filterRulesFromQueryParams(queryParams) - if ( - filterRules.length || - queryParams.has(SORT_FIELD_PARAMETER) || - queryParams.has(SORT_REVERSE_PARAMETER) - ) { - this.list.filterRules = filterRules - this.list.sortField = queryParams.get(SORT_FIELD_PARAMETER) - this.list.sortReverse = - queryParams.has(SORT_REVERSE_PARAMETER) || - (!queryParams.has(SORT_FIELD_PARAMETER) && - !queryParams.has(SORT_REVERSE_PARAMETER)) - this.list.reload() - } else if ( - filterRules.length == 0 && - !queryParams.has(SORT_FIELD_PARAMETER) - ) { - // this is navigating to /documents so we need to update the params from the list - this.updateFilterRules(this.list.filterRules, false) - this.sortParams[SORT_FIELD_PARAMETER] = this.list.sortField - this.sortParams[SORT_REVERSE_PARAMETER] = this.list.sortReverse - this.router.navigate([], { - queryParams: this.params, - replaceUrl: true, - }) - } - } - - updateFromView(view: PaperlessSavedView) { - if (!this.router.routerState.snapshot.url.includes('/view/')) { - // navigation for /documents?view= - this.router.navigate([], { - queryParams: { view: view.id }, - }) - } - // make sure params are up-to-date - this.updateFilterRules(view.filter_rules, false) - this.sortParams[SORT_FIELD_PARAMETER] = this.list.sortField - this.sortParams[SORT_REVERSE_PARAMETER] = this.list.sortReverse - } - - navigateWithFilterRules(filterRules: FilterRule[]) { - this.updateFilterRules(filterRules) - this.router.navigate(['/documents'], { - queryParams: this.params, - }) - } -} - -export function filterRulesToQueryParams(filterRules: FilterRule[]): Object { - if (filterRules) { - let params = {} - for (let rule of filterRules) { - let ruleType = FILTER_RULE_TYPES.find((t) => t.id == rule.rule_type) - if (ruleType.multi) { - params[ruleType.filtervar] = params[ruleType.filtervar] - ? params[ruleType.filtervar] + ',' + rule.value - : rule.value - } else if (ruleType.isnull_filtervar && rule.value == null) { - params[ruleType.isnull_filtervar] = true - } else { - params[ruleType.filtervar] = rule.value - } - } - return params - } else { - return null - } -} - -export function filterRulesFromQueryParams(queryParams: ParamMap) { - const allFilterRuleQueryParams: string[] = FILTER_RULE_TYPES.map( - (rt) => rt.filtervar - ) - .concat(FILTER_RULE_TYPES.map((rt) => rt.isnull_filtervar)) - .filter((rt) => rt !== undefined) - - // transform query params to filter rules - let filterRulesFromQueryParams: FilterRule[] = [] - allFilterRuleQueryParams - .filter((frqp) => queryParams.has(frqp)) - .forEach((filterQueryParamName) => { - const rule_type: FilterRuleType = FILTER_RULE_TYPES.find( - (rt) => - rt.filtervar == filterQueryParamName || - rt.isnull_filtervar == filterQueryParamName - ) - const isNullRuleType = rule_type.isnull_filtervar == filterQueryParamName - const valueURIComponent: string = queryParams.get(filterQueryParamName) - const filterQueryParamValues: string[] = rule_type.multi - ? valueURIComponent.split(',') - : [valueURIComponent] - - filterRulesFromQueryParams = filterRulesFromQueryParams.concat( - // map all values to filter rules - filterQueryParamValues.map((val) => { - return { - rule_type: rule_type.id, - value: isNullRuleType ? null : val, - } - }) - ) - }) - - return filterRulesFromQueryParams -} diff --git a/src-ui/src/app/services/rest/document.service.ts b/src-ui/src/app/services/rest/document.service.ts index 8d5f80c04..190212084 100644 --- a/src-ui/src/app/services/rest/document.service.ts +++ b/src-ui/src/app/services/rest/document.service.ts @@ -11,7 +11,7 @@ import { CorrespondentService } from './correspondent.service' import { DocumentTypeService } from './document-type.service' import { TagService } from './tag.service' import { PaperlessDocumentSuggestions } from 'src/app/data/paperless-document-suggestions' -import { filterRulesToQueryParams } from '../query-params.service' +import { queryParamsFromFilterRules } from '../../utils/query-params' import { StoragePathService } from './storage-path.service' export const DOCUMENT_SORT_FIELDS = [ @@ -91,7 +91,7 @@ export class DocumentService extends AbstractPaperlessService pageSize, sortField, sortReverse, - Object.assign(extraParams, filterRulesToQueryParams(filterRules)) + Object.assign(extraParams, queryParamsFromFilterRules(filterRules)) ).pipe( map((results) => { results.results.forEach((doc) => this.addObservablesToDocument(doc)) diff --git a/src-ui/src/app/utils/query-params.ts b/src-ui/src/app/utils/query-params.ts new file mode 100644 index 000000000..a26093398 --- /dev/null +++ b/src-ui/src/app/utils/query-params.ts @@ -0,0 +1,100 @@ +import { ParamMap, Params } from '@angular/router' +import { FilterRule } from '../data/filter-rule' +import { FilterRuleType, FILTER_RULE_TYPES } from '../data/filter-rule-type' +import { ListViewState } from '../services/document-list-view.service' + +const SORT_FIELD_PARAMETER = 'sort' +const SORT_REVERSE_PARAMETER = 'reverse' +const PAGE_PARAMETER = 'page' + +export function generateParams( + filterRules: FilterRule[], + sortField: string, + sortReverse: boolean, + currentPage: number +): Params { + let params = {} + params[SORT_FIELD_PARAMETER] = sortField + params[SORT_REVERSE_PARAMETER] = sortReverse + params[PAGE_PARAMETER] = isNaN(currentPage) ? 1 : currentPage + return { + ...queryParamsFromFilterRules(filterRules), + ...params, + } +} + +export function parseQueryParams(queryParams: ParamMap): ListViewState { + let filterRules = filterRulesFromQueryParams(queryParams) + let sortField = queryParams.get(SORT_FIELD_PARAMETER) + let sortReverse = + queryParams.has(SORT_REVERSE_PARAMETER) || + (!queryParams.has(SORT_FIELD_PARAMETER) && + !queryParams.has(SORT_REVERSE_PARAMETER)) + let currentPage = queryParams.has(PAGE_PARAMETER) + ? parseInt(queryParams.get(PAGE_PARAMETER)) + : 1 + return { + currentPage: currentPage, + filterRules: filterRules, + sortField: sortField, + sortReverse: sortReverse, + } +} + +export function filterRulesFromQueryParams(queryParams: ParamMap) { + const allFilterRuleQueryParams: string[] = FILTER_RULE_TYPES.map( + (rt) => rt.filtervar + ) + .concat(FILTER_RULE_TYPES.map((rt) => rt.isnull_filtervar)) + .filter((rt) => rt !== undefined) + + // transform query params to filter rules + let filterRulesFromQueryParams: FilterRule[] = [] + allFilterRuleQueryParams + .filter((frqp) => queryParams.has(frqp)) + .forEach((filterQueryParamName) => { + const rule_type: FilterRuleType = FILTER_RULE_TYPES.find( + (rt) => + rt.filtervar == filterQueryParamName || + rt.isnull_filtervar == filterQueryParamName + ) + const isNullRuleType = rule_type.isnull_filtervar == filterQueryParamName + const valueURIComponent: string = queryParams.get(filterQueryParamName) + const filterQueryParamValues: string[] = rule_type.multi + ? valueURIComponent.split(',') + : [valueURIComponent] + + filterRulesFromQueryParams = filterRulesFromQueryParams.concat( + // map all values to filter rules + filterQueryParamValues.map((val) => { + return { + rule_type: rule_type.id, + value: isNullRuleType ? null : val, + } + }) + ) + }) + + return filterRulesFromQueryParams +} + +export function queryParamsFromFilterRules(filterRules: FilterRule[]): Params { + if (filterRules) { + let params = {} + for (let rule of filterRules) { + let ruleType = FILTER_RULE_TYPES.find((t) => t.id == rule.rule_type) + if (ruleType.multi) { + params[ruleType.filtervar] = params[ruleType.filtervar] + ? params[ruleType.filtervar] + ',' + rule.value + : rule.value + } else if (ruleType.isnull_filtervar && rule.value == null) { + params[ruleType.isnull_filtervar] = true + } else { + params[ruleType.filtervar] = rule.value + } + } + return params + } else { + return null + } +} From f6d78a0044dc9c013adcf1ac22fab5012cac4da2 Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+shamoon@users.noreply.github.com> Date: Fri, 20 May 2022 22:14:39 -0700 Subject: [PATCH 010/173] fix documents list without query params --- .../services/document-list-view.service.ts | 21 ++++++++----------- src-ui/src/app/utils/query-params.ts | 2 +- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/src-ui/src/app/services/document-list-view.service.ts b/src-ui/src/app/services/document-list-view.service.ts index 52a0de296..d2fbf4efe 100644 --- a/src-ui/src/app/services/document-list-view.service.ts +++ b/src-ui/src/app/services/document-list-view.service.ts @@ -1,5 +1,5 @@ import { Injectable } from '@angular/core' -import { ActivatedRoute, Params, Router } from '@angular/router' +import { ParamMap, Router } from '@angular/router' import { Observable } from 'rxjs' import { cloneFilterRules, @@ -10,7 +10,7 @@ import { PaperlessDocument } from '../data/paperless-document' import { PaperlessSavedView } from '../data/paperless-saved-view' import { SETTINGS_KEYS } from '../data/paperless-uisettings' import { DOCUMENT_LIST_SERVICE } from '../data/storage-keys' -import { generateParams, parseQueryParams } from '../utils/query-params' +import { generateParams, getStateFromQueryParams } from '../utils/query-params' import { DocumentService, DOCUMENT_SORT_FIELDS } from './rest/document.service' import { SettingsService } from './settings.service' @@ -166,13 +166,17 @@ export class DocumentListViewService { } } - loadFromQueryParams(queryParams) { - let newState: ListViewState = parseQueryParams(queryParams) + loadFromQueryParams(queryParams: ParamMap) { + const paramsEmpty: boolean = queryParams.keys.length == 0 + let newState: ListViewState = this.listViewStates.get(null) + if (!paramsEmpty) newState = getStateFromQueryParams(queryParams) + if (newState == undefined) newState = this.defaultListViewState() // if nothing in local storage + this.activeListViewState.filterRules = newState.filterRules this.activeListViewState.sortField = newState.sortField this.activeListViewState.sortReverse = newState.sortReverse this.activeListViewState.currentPage = newState.currentPage - this.reload(null, false) + this.reload(null, paramsEmpty) // update the params if there arent any } reload(onFinish?, updateQueryParams: boolean = true) { @@ -280,13 +284,6 @@ export class DocumentListViewService { return this.activeListViewState.sortReverse } - get sortParams(): Params { - return { - sortField: this.sortField, - sortReverse: this.sortReverse, - } - } - get collectionSize(): number { return this.activeListViewState.collectionSize } diff --git a/src-ui/src/app/utils/query-params.ts b/src-ui/src/app/utils/query-params.ts index a26093398..5294ad465 100644 --- a/src-ui/src/app/utils/query-params.ts +++ b/src-ui/src/app/utils/query-params.ts @@ -23,7 +23,7 @@ export function generateParams( } } -export function parseQueryParams(queryParams: ParamMap): ListViewState { +export function getStateFromQueryParams(queryParams: ParamMap): ListViewState { let filterRules = filterRulesFromQueryParams(queryParams) let sortField = queryParams.get(SORT_FIELD_PARAMETER) let sortReverse = From 1c83f489d1ab30ae4e426da583ccd884bd13b989 Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+shamoon@users.noreply.github.com> Date: Fri, 20 May 2022 22:23:35 -0700 Subject: [PATCH 011/173] Use 1/0 instead of true/false --- .../filter-editor/filter-editor.component.ts | 5 ++++- src-ui/src/app/utils/query-params.ts | 13 ++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.ts b/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.ts index f52435d49..421cd0693 100644 --- a/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.ts +++ b/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.ts @@ -313,7 +313,10 @@ export class FilterEditorComponent implements OnInit, OnDestroy { break case FILTER_ASN_ISNULL: this.textFilterTarget = TEXT_FILTER_TARGET_ASN - this.textFilterModifier = TEXT_FILTER_MODIFIER_NULL + this.textFilterModifier = + rule.value == 'true' || rule.value == '1' + ? TEXT_FILTER_MODIFIER_NULL + : TEXT_FILTER_MODIFIER_NOTNULL break case FILTER_ASN_GT: this.textFilterTarget = TEXT_FILTER_TARGET_ASN diff --git a/src-ui/src/app/utils/query-params.ts b/src-ui/src/app/utils/query-params.ts index 5294ad465..5c5dae8d5 100644 --- a/src-ui/src/app/utils/query-params.ts +++ b/src-ui/src/app/utils/query-params.ts @@ -15,7 +15,7 @@ export function generateParams( ): Params { let params = {} params[SORT_FIELD_PARAMETER] = sortField - params[SORT_REVERSE_PARAMETER] = sortReverse + params[SORT_REVERSE_PARAMETER] = sortReverse ? 1 : undefined params[PAGE_PARAMETER] = isNaN(currentPage) ? 1 : currentPage return { ...queryParamsFromFilterRules(filterRules), @@ -41,7 +41,9 @@ export function getStateFromQueryParams(queryParams: ParamMap): ListViewState { } } -export function filterRulesFromQueryParams(queryParams: ParamMap) { +export function filterRulesFromQueryParams( + queryParams: ParamMap +): FilterRule[] { const allFilterRuleQueryParams: string[] = FILTER_RULE_TYPES.map( (rt) => rt.filtervar ) @@ -67,6 +69,8 @@ export function filterRulesFromQueryParams(queryParams: ParamMap) { filterRulesFromQueryParams = filterRulesFromQueryParams.concat( // map all values to filter rules filterQueryParamValues.map((val) => { + if (rule_type.datatype == 'boolean') + val = val.replace('1', 'true').replace('0', 'false') return { rule_type: rule_type.id, value: isNullRuleType ? null : val, @@ -88,9 +92,12 @@ export function queryParamsFromFilterRules(filterRules: FilterRule[]): Params { ? params[ruleType.filtervar] + ',' + rule.value : rule.value } else if (ruleType.isnull_filtervar && rule.value == null) { - params[ruleType.isnull_filtervar] = true + params[ruleType.isnull_filtervar] = 1 } else { params[ruleType.filtervar] = rule.value + if (ruleType.datatype == 'boolean') + params[ruleType.filtervar] = + rule.value == 'true' || rule.value == '1' ? 1 : 0 } } return params From 48f9cb09af2fa64a960889430e52234b6047de29 Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+shamoon@users.noreply.github.com> Date: Fri, 20 May 2022 23:32:30 -0700 Subject: [PATCH 012/173] minor refactor query params utils --- .../services/document-list-view.service.ts | 11 +++------ src-ui/src/app/utils/query-params.ts | 24 +++++++------------ 2 files changed, 12 insertions(+), 23 deletions(-) diff --git a/src-ui/src/app/services/document-list-view.service.ts b/src-ui/src/app/services/document-list-view.service.ts index d2fbf4efe..b7ec2708d 100644 --- a/src-ui/src/app/services/document-list-view.service.ts +++ b/src-ui/src/app/services/document-list-view.service.ts @@ -10,7 +10,7 @@ import { PaperlessDocument } from '../data/paperless-document' import { PaperlessSavedView } from '../data/paperless-saved-view' import { SETTINGS_KEYS } from '../data/paperless-uisettings' import { DOCUMENT_LIST_SERVICE } from '../data/storage-keys' -import { generateParams, getStateFromQueryParams } from '../utils/query-params' +import { generateParams, parseParams } from '../utils/query-params' import { DocumentService, DOCUMENT_SORT_FIELDS } from './rest/document.service' import { SettingsService } from './settings.service' @@ -169,7 +169,7 @@ export class DocumentListViewService { loadFromQueryParams(queryParams: ParamMap) { const paramsEmpty: boolean = queryParams.keys.length == 0 let newState: ListViewState = this.listViewStates.get(null) - if (!paramsEmpty) newState = getStateFromQueryParams(queryParams) + if (!paramsEmpty) newState = parseParams(queryParams) if (newState == undefined) newState = this.defaultListViewState() // if nothing in local storage this.activeListViewState.filterRules = newState.filterRules @@ -200,12 +200,7 @@ export class DocumentListViewService { if (updateQueryParams && !this._activeSavedViewId) { let base = ['/documents'] this.router.navigate(base, { - queryParams: generateParams( - activeListViewState.filterRules, - activeListViewState.sortField, - activeListViewState.sortReverse, - activeListViewState.currentPage - ), + queryParams: generateParams(activeListViewState), }) } diff --git a/src-ui/src/app/utils/query-params.ts b/src-ui/src/app/utils/query-params.ts index 5c5dae8d5..6af44e8c9 100644 --- a/src-ui/src/app/utils/query-params.ts +++ b/src-ui/src/app/utils/query-params.ts @@ -7,23 +7,17 @@ const SORT_FIELD_PARAMETER = 'sort' const SORT_REVERSE_PARAMETER = 'reverse' const PAGE_PARAMETER = 'page' -export function generateParams( - filterRules: FilterRule[], - sortField: string, - sortReverse: boolean, - currentPage: number -): Params { - let params = {} - params[SORT_FIELD_PARAMETER] = sortField - params[SORT_REVERSE_PARAMETER] = sortReverse ? 1 : undefined - params[PAGE_PARAMETER] = isNaN(currentPage) ? 1 : currentPage - return { - ...queryParamsFromFilterRules(filterRules), - ...params, - } +export function generateParams(viewState: ListViewState): Params { + let params = queryParamsFromFilterRules(viewState.filterRules) + params[SORT_FIELD_PARAMETER] = viewState.sortField + params[SORT_REVERSE_PARAMETER] = viewState.sortReverse ? 1 : undefined + params[PAGE_PARAMETER] = isNaN(viewState.currentPage) + ? 1 + : viewState.currentPage + return params } -export function getStateFromQueryParams(queryParams: ParamMap): ListViewState { +export function parseParams(queryParams: ParamMap): ListViewState { let filterRules = filterRulesFromQueryParams(queryParams) let sortField = queryParams.get(SORT_FIELD_PARAMETER) let sortReverse = From 73a6e68e037e9f6d6e9001bfc1d56bc1f0707a34 Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+shamoon@users.noreply.github.com> Date: Sat, 21 May 2022 00:07:19 -0700 Subject: [PATCH 013/173] Fix conflicting rule type 22 --- src-ui/src/app/data/filter-rule-type.ts | 28 ++++++++++++++----------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src-ui/src/app/data/filter-rule-type.ts b/src-ui/src/app/data/filter-rule-type.ts index 981169b7e..35e597ab6 100644 --- a/src-ui/src/app/data/filter-rule-type.ts +++ b/src-ui/src/app/data/filter-rule-type.ts @@ -1,34 +1,38 @@ export const FILTER_TITLE = 0 export const FILTER_CONTENT = 1 + export const FILTER_ASN = 2 +export const FILTER_ASN_ISNULL = 18 +export const FILTER_ASN_GT = 23 +export const FILTER_ASN_LT = 24 + export const FILTER_CORRESPONDENT = 3 + export const FILTER_DOCUMENT_TYPE = 4 + export const FILTER_IS_IN_INBOX = 5 export const FILTER_HAS_TAGS_ALL = 6 export const FILTER_HAS_ANY_TAG = 7 +export const FILTER_DOES_NOT_HAVE_TAG = 17 export const FILTER_HAS_TAGS_ANY = 22 + +export const FILTER_STORAGE_PATH = 25 + export const FILTER_CREATED_BEFORE = 8 export const FILTER_CREATED_AFTER = 9 export const FILTER_CREATED_YEAR = 10 export const FILTER_CREATED_MONTH = 11 export const FILTER_CREATED_DAY = 12 + export const FILTER_ADDED_BEFORE = 13 export const FILTER_ADDED_AFTER = 14 + export const FILTER_MODIFIED_BEFORE = 15 export const FILTER_MODIFIED_AFTER = 16 -export const FILTER_DOES_NOT_HAVE_TAG = 17 - -export const FILTER_ASN_ISNULL = 18 -export const FILTER_ASN_GT = 19 -export const FILTER_ASN_LT = 20 - -export const FILTER_TITLE_CONTENT = 21 - -export const FILTER_FULLTEXT_QUERY = 22 -export const FILTER_FULLTEXT_MORELIKE = 23 - -export const FILTER_STORAGE_PATH = 30 +export const FILTER_TITLE_CONTENT = 19 +export const FILTER_FULLTEXT_QUERY = 20 +export const FILTER_FULLTEXT_MORELIKE = 21 export const FILTER_RULE_TYPES: FilterRuleType[] = [ { From 8e2cb6d416276951a55b99c575443fddf9b81fe8 Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+shamoon@users.noreply.github.com> Date: Sun, 22 May 2022 08:59:43 -0700 Subject: [PATCH 014/173] Strip some characters from pasted date input --- .../common/input/date/date.component.html | 2 +- .../components/common/input/date/date.component.ts | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src-ui/src/app/components/common/input/date/date.component.html b/src-ui/src/app/components/common/input/date/date.component.html index ca6ec0b26..e742ead9b 100644 --- a/src-ui/src/app/components/common/input/date/date.component.html +++ b/src-ui/src/app/components/common/input/date/date.component.html @@ -2,7 +2,7 @@
-
-
- -
+
+
+ {{moreTags}}
diff --git a/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.scss b/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.scss index 4d03d0a4d..6ccbba0c2 100644 --- a/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.scss +++ b/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.scss @@ -78,3 +78,11 @@ a { cursor: pointer; } + +.tags { + top: 0; + right: 0; + max-width: 80%; + row-gap: .2rem; + line-height: 1; +} From efcfecca103e30a79153c781adbc74e1034e46f4 Mon Sep 17 00:00:00 2001 From: Trenton Holmes Date: Mon, 23 May 2022 15:53:47 -0700 Subject: [PATCH 023/173] Also output the exception when the Redis ping fails --- docker/wait-for-redis.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docker/wait-for-redis.py b/docker/wait-for-redis.py index 292450352..8ceae1ba9 100755 --- a/docker/wait-for-redis.py +++ b/docker/wait-for-redis.py @@ -26,9 +26,11 @@ if __name__ == "__main__": try: client.ping() break - except Exception: + except Exception as e: print( - f"Redis ping #{attempt} failed, waiting {RETRY_SLEEP_SECONDS}s", + f"Redis ping #{attempt} failed.\n" + f"Error: {str(e)}.\n" + f"Waiting {RETRY_SLEEP_SECONDS}s", flush=True, ) time.sleep(RETRY_SLEEP_SECONDS) From 4b4bfc052f49bb67885421d349905c7b9b12868b Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+shamoon@users.noreply.github.com> Date: Mon, 23 May 2022 20:25:26 -0700 Subject: [PATCH 024/173] Alphabetize tags by default --- src-ui/src/app/services/rest/document.service.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src-ui/src/app/services/rest/document.service.ts b/src-ui/src/app/services/rest/document.service.ts index 8d5f80c04..8b5248b86 100644 --- a/src-ui/src/app/services/rest/document.service.ts +++ b/src-ui/src/app/services/rest/document.service.ts @@ -6,7 +6,7 @@ import { HttpClient, HttpParams } from '@angular/common/http' import { Observable } from 'rxjs' import { Results } from 'src/app/data/results' import { FilterRule } from 'src/app/data/filter-rule' -import { map } from 'rxjs/operators' +import { map, tap } from 'rxjs/operators' import { CorrespondentService } from './correspondent.service' import { DocumentTypeService } from './document-type.service' import { TagService } from './tag.service' @@ -70,7 +70,13 @@ export class DocumentService extends AbstractPaperlessService doc.document_type$ = this.documentTypeService.getCached(doc.document_type) } if (doc.tags) { - doc.tags$ = this.tagService.getCachedMany(doc.tags) + doc.tags$ = this.tagService + .getCachedMany(doc.tags) + .pipe( + tap((tags) => + tags.sort((tagA, tagB) => tagA.name.localeCompare(tagB.name)) + ) + ) } if (doc.storage_path) { doc.storage_path$ = this.storagePathService.getCached(doc.storage_path) From d4e27225867bf1441d5aed555fde8cb41b1bc47c Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+shamoon@users.noreply.github.com> Date: Mon, 23 May 2022 23:07:49 -0700 Subject: [PATCH 025/173] use force_authenticate in api tests --- src/documents/tests/test_api.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/documents/tests/test_api.py b/src/documents/tests/test_api.py index 1f6e012d1..24bdc3a50 100644 --- a/src/documents/tests/test_api.py +++ b/src/documents/tests/test_api.py @@ -41,7 +41,7 @@ class TestDocumentApi(DirectoriesMixin, APITestCase): super().setUp() self.user = User.objects.create_superuser(username="temp_admin") - self.client.force_login(user=self.user) + self.client.force_authenticate(user=self.user) def testDocuments(self): @@ -1176,7 +1176,7 @@ class TestDocumentApi(DirectoriesMixin, APITestCase): self.assertEqual(self.client.get(f"/api/saved_views/{v1.id}/").status_code, 404) - self.client.force_login(user=u1) + self.client.force_authenticate(user=u1) response = self.client.get("/api/saved_views/") self.assertEqual(response.status_code, 200) @@ -1184,7 +1184,7 @@ class TestDocumentApi(DirectoriesMixin, APITestCase): self.assertEqual(self.client.get(f"/api/saved_views/{v1.id}/").status_code, 200) - self.client.force_login(user=u2) + self.client.force_authenticate(user=u2) response = self.client.get("/api/saved_views/") self.assertEqual(response.status_code, 200) @@ -1358,7 +1358,7 @@ class TestDocumentApiV2(DirectoriesMixin, APITestCase): self.user = User.objects.create_superuser(username="temp_admin") - self.client.force_login(user=self.user) + self.client.force_authenticate(user=self.user) self.client.defaults["HTTP_ACCEPT"] = "application/json; version=2" def test_tag_validate_color(self): @@ -1434,7 +1434,7 @@ class TestDocumentApiV2(DirectoriesMixin, APITestCase): def test_ui_settings(self): test_user = User.objects.create_superuser(username="test") - self.client.force_login(user=test_user) + self.client.force_authenticate(user=test_user) response = self.client.get("/api/ui_settings/", format="json") self.assertEqual(response.status_code, 200) @@ -1473,7 +1473,7 @@ class TestBulkEdit(DirectoriesMixin, APITestCase): super().setUp() user = User.objects.create_superuser(username="temp_admin") - self.client.force_login(user=user) + self.client.force_authenticate(user=user) patcher = mock.patch("documents.bulk_edit.async_task") self.async_task = patcher.start() @@ -2049,7 +2049,7 @@ class TestBulkDownload(DirectoriesMixin, APITestCase): super().setUp() user = User.objects.create_superuser(username="temp_admin") - self.client.force_login(user=user) + self.client.force_authenticate(user=user) self.doc1 = Document.objects.create(title="unrelated", checksum="A") self.doc2 = Document.objects.create( @@ -2250,7 +2250,7 @@ class TestApiAuth(APITestCase): def test_api_version_with_auth(self): user = User.objects.create_superuser(username="test") - self.client.force_login(user) + self.client.force_authenticate(user) response = self.client.get("/api/") self.assertIn("X-Api-Version", response) self.assertIn("X-Version", response) From 3ffd2a745b4380df637f96e6ce855373112ddb0d Mon Sep 17 00:00:00 2001 From: tooomm Date: Thu, 19 May 2022 22:05:43 +0200 Subject: [PATCH 026/173] update heading levels for v1.7.0 --- docs/changelog.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 9ba2f9aa0..9d35ee050 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -70,12 +70,12 @@ ## paperless-ngx 1.7.0 -Breaking Changes +### Breaking Changes - `PAPERLESS_URL` is now required when using a reverse proxy. See [\#674](https://github.com/paperless-ngx/paperless-ngx/pull/674). -Features +### Features - Allow setting more than one tag in mail rules [\@jonasc](https://github.com/jonasc) (\#270) @@ -107,7 +107,7 @@ Features - Parse dates when entered without separators [\@GruberViktor](https://github.com/gruberviktor) (\#250). -Bug Fixes +### Bug Fixes - add \"localhost\" to ALLOWED_HOSTS [\@gador](https://github.com/gador) (\#700). @@ -155,7 +155,7 @@ Bug Fixes - Fix: Include excluded items in dropdown count [\@shamoon](https://github.com/shamoon) (\#263). -Translation +### Translation - [\@miku323](https://github.com/miku323) contributed to Slovenian translation. @@ -168,7 +168,7 @@ Translation - [\@Prominence](https://github.com/Prominence) contributed to Belarusian translation. -Documentation +### Documentation - Fix: scanners table [\@qcasey](https://github.com/qcasey) (\#690). - Add [PAPERLESS\_URL]{.title-ref} env variable & CSRF var @@ -180,7 +180,7 @@ Documentation - Fix minor sphinx errors [\@shamoon](https://github.com/shamoon) (\#322). -Maintenance +### Maintenance - Add `PAPERLESS_URL` env variable & CSRF var [\@shamoon](https://github.com/shamoon) (\#674). From ce3f6837e97fd669b21bf9bfb2d5d6f729af9ca5 Mon Sep 17 00:00:00 2001 From: tooomm Date: Thu, 19 May 2022 23:12:40 +0200 Subject: [PATCH 027/173] Link issues, capitalization and minor fixes --- docs/changelog.md | 174 +++++++++++++++++++++++----------------------- 1 file changed, 87 insertions(+), 87 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 9d35ee050..319592b5d 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -17,7 +17,7 @@ ### Bug Fixes - Feature / fix saved view \& sort field query params [\@shamoon](https://github.com/shamoon) ([\#881](https://github.com/paperless-ngx/paperless-ngx/pull/881)) -- mobile friendlier manage pages [\@shamoon](https://github.com/shamoon) ([\#873](https://github.com/paperless-ngx/paperless-ngx/pull/873)) +- Mobile friendlier manage pages [\@shamoon](https://github.com/shamoon) ([\#873](https://github.com/paperless-ngx/paperless-ngx/pull/873)) - Add timeout to healthcheck [\@shamoon](https://github.com/shamoon) ([\#880](https://github.com/paperless-ngx/paperless-ngx/pull/880)) - Always accept yyyy-mm-dd date inputs [\@shamoon](https://github.com/shamoon) ([\#864](https://github.com/paperless-ngx/paperless-ngx/pull/864)) - Fix local Docker image building [\@stumpylog](https://github.com/stumpylog) ([\#849](https://github.com/paperless-ngx/paperless-ngx/pull/849)) @@ -78,129 +78,129 @@ ### Features - Allow setting more than one tag in mail rules - [\@jonasc](https://github.com/jonasc) (\#270) -- global drag\'n\'drop [\@shamoon](https://github.com/shamoon) - (\#283). + [\@jonasc](https://github.com/jonasc) ([\#270](https://github.com/paperless-ngx/paperless-ngx/pull/270)) +- Global drag\'n\'drop [\@shamoon](https://github.com/shamoon) + ([\#283](https://github.com/paperless-ngx/paperless-ngx/pull/283)) - Fix: download buttons should disable while waiting - [\@shamoon](https://github.com/shamoon) (\#630). -- Update checker [\@shamoon](https://github.com/shamoon) (\#591). + [\@shamoon](https://github.com/shamoon) ([\#630](https://github.com/paperless-ngx/paperless-ngx/pull/630)) +- Update checker [\@shamoon](https://github.com/shamoon) ([\#591](https://github.com/paperless-ngx/paperless-ngx/pull/591)) - Show prompt on password-protected pdfs - [\@shamoon](https://github.com/shamoon) (\#564). + [\@shamoon](https://github.com/shamoon) ([\#564](https://github.com/paperless-ngx/paperless-ngx/pull/564)) - Filtering query params aka browser navigation for filtering - [\@shamoon](https://github.com/shamoon) (\#540). + [\@shamoon](https://github.com/shamoon) ([\#540](https://github.com/paperless-ngx/paperless-ngx/pull/540)) - Clickable tags in dashboard widgets - [\@shamoon](https://github.com/shamoon) (\#515). + [\@shamoon](https://github.com/shamoon) ([\#515](https://github.com/paperless-ngx/paperless-ngx/pull/515)) - Add bottom pagination [\@shamoon](https://github.com/shamoon) - (\#372). + ([\#372](https://github.com/paperless-ngx/paperless-ngx/pull/372)) - Feature barcode splitter [\@gador](https://github.com/gador) - (\#532). -- App loading screen [\@shamoon](https://github.com/shamoon) (\#298). + ([\#532](https://github.com/paperless-ngx/paperless-ngx/pull/532)) +- App loading screen [\@shamoon](https://github.com/shamoon) ([\#298](https://github.com/paperless-ngx/paperless-ngx/pull/298)) - Use progress bar for delayed buttons - [\@shamoon](https://github.com/shamoon) (\#415). + [\@shamoon](https://github.com/shamoon) ([\#415](https://github.com/paperless-ngx/paperless-ngx/pull/415)) - Add minimum length for documents text filter - [\@shamoon](https://github.com/shamoon) (\#401). + [\@shamoon](https://github.com/shamoon) ([\#401](https://github.com/paperless-ngx/paperless-ngx/pull/401)) - Added nav buttons in the document detail view - [\@GruberViktor](https://github.com/gruberviktor) (\#273). + [\@GruberViktor](https://github.com/gruberviktor) ([\#273](https://github.com/paperless-ngx/paperless-ngx/pull/273)) - Improve date keyboard input [\@shamoon](https://github.com/shamoon) - (\#253). -- Color theming [\@shamoon](https://github.com/shamoon) (\#243). + ([\#253](https://github.com/paperless-ngx/paperless-ngx/pull/253)) +- Color theming [\@shamoon](https://github.com/shamoon) ([\#243](https://github.com/paperless-ngx/paperless-ngx/pull/243)) - Parse dates when entered without separators - [\@GruberViktor](https://github.com/gruberviktor) (\#250). + [\@GruberViktor](https://github.com/gruberviktor) ([\#250](https://github.com/paperless-ngx/paperless-ngx/pull/250)) ### Bug Fixes -- add \"localhost\" to ALLOWED_HOSTS - [\@gador](https://github.com/gador) (\#700). -- Fix: scanners table [\@qcasey](https://github.com/qcasey) (\#690). +- Add \"localhost\" to ALLOWED_HOSTS + [\@gador](https://github.com/gador) ([\#700](https://github.com/paperless-ngx/paperless-ngx/pull/700)) +- Fix: scanners table [\@qcasey](https://github.com/qcasey) ([\#690](https://github.com/paperless-ngx/paperless-ngx/pull/690)) - Adds wait for file before consuming - [\@stumpylog](https://github.com/stumpylog) (\#483). + [\@stumpylog](https://github.com/stumpylog) ([\#483](https://github.com/paperless-ngx/paperless-ngx/pull/483)) - Fix: frontend document editing erases time data - [\@shamoon](https://github.com/shamoon) (\#654). + [\@shamoon](https://github.com/shamoon) ([\#654](https://github.com/paperless-ngx/paperless-ngx/pull/654)) - Increase length of SavedViewFilterRule - [\@stumpylog](https://github.com/stumpylog) (\#612). + [\@stumpylog](https://github.com/stumpylog) ([\#612](https://github.com/paperless-ngx/paperless-ngx/pull/612)) - Fixes attachment filename matching during mail fetching - [\@stumpylog](https://github.com/stumpylog) (\#680). + [\@stumpylog](https://github.com/stumpylog) ([\#680](https://github.com/paperless-ngx/paperless-ngx/pull/680)) - Add `PAPERLESS_URL` env variable & CSRF var - [\@shamoon](https://github.com/shamoon) (\#674). + [\@shamoon](https://github.com/shamoon) ([\#674](https://github.com/paperless-ngx/paperless-ngx/discussions/674)) - Fix: download buttons should disable while waiting - [\@shamoon](https://github.com/shamoon) (\#630). + [\@shamoon](https://github.com/shamoon) ([\#630](https://github.com/paperless-ngx/paperless-ngx/pull/630)) - Fixes downloaded filename, add more consumer ignore settings - [\@stumpylog](https://github.com/stumpylog) (\#599). + [\@stumpylog](https://github.com/stumpylog) ([\#599](https://github.com/paperless-ngx/paperless-ngx/pull/599)) - FIX BUG: case-sensitive matching was not possible - [\@danielBreitlauch](https://github.com/danielbreitlauch) (\#594). -- uses shutil.move instead of rename - [\@gador](https://github.com/gador) (\#617). + [\@danielBreitlauch](https://github.com/danielbreitlauch) ([\#594](https://github.com/paperless-ngx/paperless-ngx/pull/594)) +- Uses shutil.move instead of rename + [\@gador](https://github.com/gador) ([\#617](https://github.com/paperless-ngx/paperless-ngx/pull/617)) - Fix npm deps 01.02.22 2 [\@shamoon](https://github.com/shamoon) - (\#610). + ([\#610](https://github.com/paperless-ngx/paperless-ngx/discussions/610)) - Fix npm dependencies 01.02.22 - [\@shamoon](https://github.com/shamoon) (\#600). -- fix issue 416: implement PAPERLESS_OCR_MAX_IMAGE_PIXELS - [\@hacker-h](https://github.com/hacker-h) (\#441). -- fix: exclude cypress from build in Dockerfile - [\@FrankStrieter](https://github.com/FrankStrieter) (\#526). + [\@shamoon](https://github.com/shamoon) ([\#600](https://github.com/paperless-ngx/paperless-ngx/pull/600)) +- Fix issue 416: implement `PAPERLESS_OCR_MAX_IMAGE_PIXELS` + [\@hacker-h](https://github.com/hacker-h) ([\#441](https://github.com/paperless-ngx/paperless-ngx/pull/441)) +- Fix: exclude cypress from build in Dockerfile + [\@FrankStrieter](https://github.com/FrankStrieter) ([\#526](https://github.com/paperless-ngx/paperless-ngx/pull/526)) - Corrections to pass pre-commit hooks - [\@schnuffle](https://github.com/schnuffle) (\#454). + [\@schnuffle](https://github.com/schnuffle) ([\#454](https://github.com/paperless-ngx/paperless-ngx/pull/454)) - Fix 311 unable to click checkboxes in document list - [\@shamoon](https://github.com/shamoon) (\#313). + [\@shamoon](https://github.com/shamoon) ([\#313](https://github.com/paperless-ngx/paperless-ngx/pull/313)) - Fix imap tools bug [\@stumpylog](https://github.com/stumpylog) - (\#393). + ([\#393](https://github.com/paperless-ngx/paperless-ngx/pull/393)) - Fix filterable dropdown buttons arent translated - [\@shamoon](https://github.com/shamoon) (\#366). + [\@shamoon](https://github.com/shamoon) ([\#366](https://github.com/paperless-ngx/paperless-ngx/pull/366)) - Fix 224: \"Auto-detected date is day before receipt date\" - [\@a17t](https://github.com/a17t) (\#246). + [\@a17t](https://github.com/a17t) ([\#246](https://github.com/paperless-ngx/paperless-ngx/pull/246)) - Fix minor sphinx errors [\@shamoon](https://github.com/shamoon) - (\#322). + ([\#322](https://github.com/paperless-ngx/paperless-ngx/pull/322)) - Fix page links hidden [\@shamoon](https://github.com/shamoon) - (\#314). + ([\#314](https://github.com/paperless-ngx/paperless-ngx/pull/314)) - Fix: Include excluded items in dropdown count - [\@shamoon](https://github.com/shamoon) (\#263). + [\@shamoon](https://github.com/shamoon) ([\#263](https://github.com/paperless-ngx/paperless-ngx/pull/263)) ### Translation - [\@miku323](https://github.com/miku323) contributed to Slovenian - translation. + translation - [\@FaintGhost](https://github.com/FaintGhost) contributed to Chinese - Simplified translation. + Simplified translation - [\@DarkoBG79](https://github.com/DarkoBG79) contributed to Serbian - translation. + translation - [Kemal Secer](https://crowdin.com/profile/kemal.secer) contributed - to Turkish translation. + to Turkish translation - [\@Prominence](https://github.com/Prominence) contributed to - Belarusian translation. + Belarusian translation ### Documentation -- Fix: scanners table [\@qcasey](https://github.com/qcasey) (\#690). -- Add [PAPERLESS\_URL]{.title-ref} env variable & CSRF var - [\@shamoon](https://github.com/shamoon) (\#674). +- Fix: scanners table [\@qcasey](https://github.com/qcasey) ([\#690](https://github.com/paperless-ngx/paperless-ngx/pull/690)) +- Add `PAPERLESS_URL` env variable & CSRF var + [\@shamoon](https://github.com/shamoon) ([\#674](https://github.com/paperless-ngx/paperless-ngx/pull/674)) - Fixes downloaded filename, add more consumer ignore settings - [\@stumpylog](https://github.com/stumpylog) (\#599). -- fix issue 416: implement `PAPERLESS_OCR_MAX_IMAGE_PIXELS` - [\@hacker-h](https://github.com/hacker-h) (\#441). + [\@stumpylog](https://github.com/stumpylog) ([\#599](https://github.com/paperless-ngx/paperless-ngx/pull/599)) +- Fix issue 416: implement `PAPERLESS_OCR_MAX_IMAGE_PIXELS` + [\@hacker-h](https://github.com/hacker-h) ([\#441](https://github.com/paperless-ngx/paperless-ngx/pull/441)) - Fix minor sphinx errors [\@shamoon](https://github.com/shamoon) - (\#322). + ([\#322](https://github.com/paperless-ngx/paperless-ngx/pull/322)) ### Maintenance - Add `PAPERLESS_URL` env variable & CSRF var - [\@shamoon](https://github.com/shamoon) (\#674). + [\@shamoon](https://github.com/shamoon) ([\#674](https://github.com/paperless-ngx/paperless-ngx/pull/674)) - Chore: Implement release-drafter action for Changelogs - [\@qcasey](https://github.com/qcasey) (\#669). -- Chore: Add CODEOWNERS [\@qcasey](https://github.com/qcasey) (\#667). + [\@qcasey](https://github.com/qcasey) ([\#669](https://github.com/paperless-ngx/paperless-ngx/pull/669)) +- Chore: Add CODEOWNERS [\@qcasey](https://github.com/qcasey) ([\#667](https://github.com/paperless-ngx/paperless-ngx/pull/667)) - Support docker-compose v2 in install - [\@stumpylog](https://github.com/stumpylog) (\#611). + [\@stumpylog](https://github.com/stumpylog) ([\#611](https://github.com/paperless-ngx/paperless-ngx/pull/611)) - Add Belarusian localization [\@shamoon](https://github.com/shamoon) - (\#588). + ([\#588](https://github.com/paperless-ngx/paperless-ngx/pull/588)) - Add Turkish localization [\@shamoon](https://github.com/shamoon) - (\#536). + ([\#536](https://github.com/paperless-ngx/paperless-ngx/pull/536)) - Add Serbian localization [\@shamoon](https://github.com/shamoon) - (\#504). + ([\#504](https://github.com/paperless-ngx/paperless-ngx/pull/504)) - Create PULL_REQUEST_TEMPLATE.md - [\@shamoon](https://github.com/shamoon) (\#304). + [\@shamoon](https://github.com/shamoon) ([\#304](https://github.com/paperless-ngx/paperless-ngx/pull/304)) - Add Chinese localization [\@shamoon](https://github.com/shamoon) - (\#247). + ([\#247](https://github.com/paperless-ngx/paperless-ngx/pull/247)) - Add Slovenian language for frontend - [\@shamoon](https://github.com/shamoon) (\#315). + [\@shamoon](https://github.com/shamoon) ([\#315](https://github.com/paperless-ngx/paperless-ngx/pull/315)) ## paperless-ngx 1.6.0 @@ -216,46 +216,46 @@ include: - Updated Python and Angular dependencies. - Dropped support for Python 3.7. - Dropped support for Ansible playbooks (thanks - [\@slankes](https://github.com/slankes) \#109). If someone would - like to continue supporting them, please see the [ansible + [\@slankes](https://github.com/slankes) [\#109](https://github.com/paperless-ngx/paperless-ngx/pull/109)). If someone would + like to continue supporting them, please see our [ansible repo](https://github.com/paperless-ngx/paperless-ngx-ansible). - Python code is now required to use Black formatting (thanks - [\@kpj](https://github.com/kpj) \#168). + [\@kpj](https://github.com/kpj) [\#168](https://github.com/paperless-ngx/paperless-ngx/pull/168)). - [\@tribut](https://github.com/tribut) added support for a custom SSO - logout redirect (jonaswinkler\#1258). See + logout redirect ([jonaswinkler\#1258](https://github.com/jonaswinkler/paperless-ng/pull/1258)). See `PAPERLESS_LOGOUT_REDIRECT_URL`. - [\@shamoon](https://github.com/shamoon) added a loading indicator - when document list is reloading (jonaswinkler\#1297). + when document list is reloading ([jonaswinkler\#1297](https://github.com/jonaswinkler/paperless-ng/pull/1297)). - [\@shamoon](https://github.com/shamoon) improved the PDF viewer on - mobile (\#2). + mobile ([\#2](https://github.com/paperless-ngx/paperless-ngx/pull/2)). - [\@shamoon](https://github.com/shamoon) added \'any\' / \'all\' and - \'not\' filtering with tags (\#10). + \'not\' filtering with tags ([\#10](https://github.com/paperless-ngx/paperless-ngx/pull/10)). - [\@shamoon](https://github.com/shamoon) added warnings for unsaved - changes, with smart edit buttons (\#13). + changes, with smart edit buttons ([\#13](https://github.com/paperless-ngx/paperless-ngx/pull/13)). - [\@benjaminfrank](https://github.com/benjaminfrank) enabled a - non-root access to port 80 via systemd (\#18). + non-root access to port 80 via systemd ([\#18](https://github.com/paperless-ngx/paperless-ngx/pull/18)). - [\@tribut](https://github.com/tribut) added simple \"delete to - trash\" functionality (\#24). See `PAPERLESS_TRASH_DIR`. + trash\" functionality ([\#24](https://github.com/paperless-ngx/paperless-ngx/pull/24)). See `PAPERLESS_TRASH_DIR`. - [\@amenk](https://github.com/amenk) fixed the search box overlay - menu on mobile (\#32). + menu on mobile ([\#32](https://github.com/paperless-ngx/paperless-ngx/pull/32)). - [\@dblitt](https://github.com/dblitt) updated the login form to not - auto-capitalize usernames (\#36). + auto-capitalize usernames ([\#36](https://github.com/paperless-ngx/paperless-ngx/pull/36)). - [\@evilsidekick293](https://github.com/evilsidekick293) made the - worker timeout configurable (\#37). See `PAPERLESS_WORKER_TIMEOUT`. + worker timeout configurable ([\#37](https://github.com/paperless-ngx/paperless-ngx/pull/37)). See `PAPERLESS_WORKER_TIMEOUT`. - [\@Nicarim](https://github.com/Nicarim) fixed downloads of UTF-8 - formatted documents in Firefox (\#56). + formatted documents in Firefox ([\#56](https://github.com/paperless-ngx/paperless-ngx/pull/56)). - [\@mweimerskirch](https://github.com/mweimerskirch) sorted the - language dropdown by locale (\#78). + language dropdown by locale ([\#78](https://github.com/paperless-ngx/paperless-ngx/issues/78)). - [\@mweimerskirch](https://github.com/mweimerskirch) enabled the - Czech (\#83) and Danish (\#84) translations. + Czech ([\#83](https://github.com/paperless-ngx/paperless-ngx/pull/83)) and Danish ([\#84](https://github.com/paperless-ngx/paperless-ngx/pull/84)) translations. - [\@cschmatzler](https://github.com/cschmatzler) enabled specifying - the webserver port (\#124). See `PAPERLESS_PORT`. + the webserver port ([\#124](https://github.com/paperless-ngx/paperless-ngx/pull/124)). See `PAPERLESS_PORT`. - [\@muellermartin](https://github.com/muellermartin) fixed an error - when uploading transparent PNGs (\#133). + when uploading transparent PNGs ([\#133](https://github.com/paperless-ngx/paperless-ngx/pull/133)). - [\@shamoon](https://github.com/shamoon) created a slick new logo - (\#165). + ([\#165](https://github.com/paperless-ngx/paperless-ngx/pull/165)). - [\@tim-vogel](https://github.com/tim-vogel) fixed exports missing - groups (\#193). + groups ([\#193](https://github.com/paperless-ngx/paperless-ngx/pull/193)). Known issues: From 5a809d7e312da8d4351229d4fe5c960182231c24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20R=C3=BCmpelein?= Date: Wed, 25 May 2022 19:22:50 +0200 Subject: [PATCH 028/173] Add first draft implementation, test broken. --- src/paperless_mail/mail.py | 13 ++++++++ src/paperless_mail/models.py | 1 + src/paperless_mail/tests/test_mail.py | 43 ++++++++++++++++++++++++++- 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/src/paperless_mail/mail.py b/src/paperless_mail/mail.py index 67cc22130..fefcbda5e 100644 --- a/src/paperless_mail/mail.py +++ b/src/paperless_mail/mail.py @@ -62,6 +62,17 @@ class FlagMailAction(BaseMailAction): M.flag(message_uids, [MailMessageFlags.FLAGGED], True) +class TagMailAction(BaseMailAction): + def __init__(self, parameter): + self.keyword = parameter + + def get_criteria(self): + return {"no_keyword": self.keyword} + + def post_consume(self, M: MailBox, message_uids, parameter): + M.flag(message_uids, [self.keyword], True) + + def get_rule_action(rule): if rule.action == MailRule.MailAction.FLAG: return FlagMailAction() @@ -71,6 +82,8 @@ def get_rule_action(rule): return MoveMailAction() elif rule.action == MailRule.MailAction.MARK_READ: return MarkReadMailAction() + elif rule.action == MailRule.MailAction.TAG: + return TagMailAction(rule.action_parameter) else: raise NotImplementedError("Unknown action.") # pragma: nocover diff --git a/src/paperless_mail/models.py b/src/paperless_mail/models.py index 4e90197b7..4c0a1a557 100644 --- a/src/paperless_mail/models.py +++ b/src/paperless_mail/models.py @@ -65,6 +65,7 @@ class MailRule(models.Model): MOVE = 2, _("Move to specified folder") MARK_READ = 3, _("Mark as read, don't process read mails") FLAG = 4, _("Flag the mail, don't process flagged mails") + TAG = 5, _("Tag the mail with specified tag, don't process tagged mails") class TitleSource(models.IntegerChoices): FROM_SUBJECT = 1, _("Use subject as title") diff --git a/src/paperless_mail/tests/test_mail.py b/src/paperless_mail/tests/test_mail.py index 24014b4dc..b275be360 100644 --- a/src/paperless_mail/tests/test_mail.py +++ b/src/paperless_mail/tests/test_mail.py @@ -4,7 +4,7 @@ import os import random import uuid from collections import namedtuple -from typing import ContextManager +from typing import ContextManager, Callable from typing import List from typing import Union from unittest import mock @@ -96,6 +96,18 @@ class BogusMailBox(ContextManager): if "UNFLAGGED" in criteria: msg = filter(lambda m: not m.flagged, msg) + if "NO_KEYWORD" in criteria: + tag: str = criteria[criteria.index("NO_KEYWORD") + 1].strip('"') + print(f"selected tag is {tag}.") + for m in msg: + print(f"Message with id {m.uid} has tags {','.join(m.flags)}") + msg = filter(lambda m: tag not in m.flags, msg) + + if "KEYWORD" in criteria: + tag = criteria[criteria.index("KEYWORD") + 1].strip('"') + print(f"selected tag is {tag}.") + msg = filter(lambda m: tag in m.flags, msg) + return list(msg) def delete(self, uid_list): @@ -130,6 +142,7 @@ def create_message( from_: str = "noone@mail.com", seen: bool = False, flagged: bool = False, + processed: bool = False, ) -> MailMessage: email_msg = email.message.EmailMessage() # TODO: This does NOT set the UID @@ -176,6 +189,10 @@ def create_message( imap_msg.seen = seen imap_msg.flagged = flagged + # Allows to test for both unset tags, and explicitly 'false' ones. + if processed: + imap_msg._raw_flag_data.append("+FLAGS (processed)".encode()) + return imap_msg @@ -225,6 +242,7 @@ class TestMail(DirectoriesMixin, TestCase): body="from my favorite electronic store", seen=False, flagged=True, + processed=True ), ) self.bogus_mailbox.messages.append( @@ -571,6 +589,29 @@ class TestMail(DirectoriesMixin, TestCase): self.assertEqual(len(self.bogus_mailbox.messages), 2) self.assertEqual(len(self.bogus_mailbox.messages_spam), 1) + def test_handle_mail_account_tag(self): + account = MailAccount.objects.create( + name="test", + imap_server="", + username="admin", + password="secret", + ) + + _ = MailRule.objects.create( + name="testrule", + account=account, + action=MailRule.MailAction.TAG, + action_parameter="processed", + ) + + self.assertEqual(len(self.bogus_mailbox.messages), 3) + self.assertEqual(self.async_task.call_count, 0) + self.assertEqual(len(self.bogus_mailbox.fetch("NO_KEYWORD processed", False)), 2) + self.mail_account_handler.handle_mail_account(account) + self.assertEqual(self.async_task.call_count, 2) + self.assertEqual(len(self.bogus_mailbox.fetch("NO_KEYWORD processed", False)), 0) + self.assertEqual(len(self.bogus_mailbox.messages), 3) + def test_error_login(self): account = MailAccount.objects.create( name="test", From 104a68451477845eda2229ac876a76ee7aa1d988 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20R=C3=BCmpelein?= Date: Wed, 25 May 2022 20:47:43 +0200 Subject: [PATCH 029/173] Revert all changes to tests, will need a more structured approach. --- src/paperless_mail/tests/test_mail.py | 43 +-------------------------- 1 file changed, 1 insertion(+), 42 deletions(-) diff --git a/src/paperless_mail/tests/test_mail.py b/src/paperless_mail/tests/test_mail.py index b275be360..24014b4dc 100644 --- a/src/paperless_mail/tests/test_mail.py +++ b/src/paperless_mail/tests/test_mail.py @@ -4,7 +4,7 @@ import os import random import uuid from collections import namedtuple -from typing import ContextManager, Callable +from typing import ContextManager from typing import List from typing import Union from unittest import mock @@ -96,18 +96,6 @@ class BogusMailBox(ContextManager): if "UNFLAGGED" in criteria: msg = filter(lambda m: not m.flagged, msg) - if "NO_KEYWORD" in criteria: - tag: str = criteria[criteria.index("NO_KEYWORD") + 1].strip('"') - print(f"selected tag is {tag}.") - for m in msg: - print(f"Message with id {m.uid} has tags {','.join(m.flags)}") - msg = filter(lambda m: tag not in m.flags, msg) - - if "KEYWORD" in criteria: - tag = criteria[criteria.index("KEYWORD") + 1].strip('"') - print(f"selected tag is {tag}.") - msg = filter(lambda m: tag in m.flags, msg) - return list(msg) def delete(self, uid_list): @@ -142,7 +130,6 @@ def create_message( from_: str = "noone@mail.com", seen: bool = False, flagged: bool = False, - processed: bool = False, ) -> MailMessage: email_msg = email.message.EmailMessage() # TODO: This does NOT set the UID @@ -189,10 +176,6 @@ def create_message( imap_msg.seen = seen imap_msg.flagged = flagged - # Allows to test for both unset tags, and explicitly 'false' ones. - if processed: - imap_msg._raw_flag_data.append("+FLAGS (processed)".encode()) - return imap_msg @@ -242,7 +225,6 @@ class TestMail(DirectoriesMixin, TestCase): body="from my favorite electronic store", seen=False, flagged=True, - processed=True ), ) self.bogus_mailbox.messages.append( @@ -589,29 +571,6 @@ class TestMail(DirectoriesMixin, TestCase): self.assertEqual(len(self.bogus_mailbox.messages), 2) self.assertEqual(len(self.bogus_mailbox.messages_spam), 1) - def test_handle_mail_account_tag(self): - account = MailAccount.objects.create( - name="test", - imap_server="", - username="admin", - password="secret", - ) - - _ = MailRule.objects.create( - name="testrule", - account=account, - action=MailRule.MailAction.TAG, - action_parameter="processed", - ) - - self.assertEqual(len(self.bogus_mailbox.messages), 3) - self.assertEqual(self.async_task.call_count, 0) - self.assertEqual(len(self.bogus_mailbox.fetch("NO_KEYWORD processed", False)), 2) - self.mail_account_handler.handle_mail_account(account) - self.assertEqual(self.async_task.call_count, 2) - self.assertEqual(len(self.bogus_mailbox.fetch("NO_KEYWORD processed", False)), 0) - self.assertEqual(len(self.bogus_mailbox.messages), 3) - def test_error_login(self): account = MailAccount.objects.create( name="test", From e72766a5bf422a439d4ef8cf1856605c21d3d021 Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+shamoon@users.noreply.github.com> Date: Wed, 25 May 2022 14:47:05 -0700 Subject: [PATCH 030/173] fix global overlay color --- src-ui/src/styles.scss | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src-ui/src/styles.scss b/src-ui/src/styles.scss index 7a566fbb9..4b78e9d21 100644 --- a/src-ui/src/styles.scss +++ b/src-ui/src/styles.scss @@ -425,13 +425,17 @@ table.table { right: 0; bottom: 0; left: 0; - background-color: rgba(23, 84, 31, .8); + background-color: hsla(var(--pngx-primary), var(--pngx-primary-lightness), .8); z-index: 1055; // $zindex-modal pointer-events: none !important; user-select: none !important; text-align: center; padding-top: 25%; + h2 { + color: var(--pngx-primary-text-contrast) + } + &.show { opacity: 1 !important; } From 0fa717fe11d40c6a6a5bae719f27ae93f7d1a0d1 Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+shamoon@users.noreply.github.com> Date: Wed, 25 May 2022 16:06:59 -0700 Subject: [PATCH 031/173] Show note on language change and offer reload --- src-ui/messages.xlf | 55 +++++++++++++------ .../common/toasts/toasts.component.html | 2 +- .../manage/settings/settings.component.html | 2 +- .../manage/settings/settings.component.ts | 25 ++++++++- src-ui/src/styles.scss | 4 ++ src-ui/src/theme.scss | 3 +- 6 files changed, 69 insertions(+), 22 deletions(-) diff --git a/src-ui/messages.xlf b/src-ui/messages.xlf index 9ab7e46a2..f4c0d6598 100644 --- a/src-ui/messages.xlf +++ b/src-ui/messages.xlf @@ -1340,7 +1340,7 @@ src/app/components/document-list/document-card-small/document-card-small.component.html - 88 + 86 @@ -1884,7 +1884,7 @@ src/app/components/document-list/document-card-small/document-card-small.component.html - 26 + 24 src/app/components/document-list/document-list.component.html @@ -1899,7 +1899,7 @@ src/app/components/document-list/document-card-small/document-card-small.component.html - 15 + 14 src/app/components/document-list/document-list.component.html @@ -1914,7 +1914,7 @@ src/app/components/document-list/document-card-small/document-card-small.component.html - 72 + 70 src/app/components/manage/management-list/management-list.component.html @@ -1964,7 +1964,7 @@ src/app/components/document-list/document-card-small/document-card-small.component.html - 33 + 31 src/app/components/document-list/document-list.component.html @@ -1979,7 +1979,7 @@ src/app/components/document-list/document-card-small/document-card-small.component.html - 40 + 38 src/app/components/document-list/document-list.component.html @@ -1994,7 +1994,7 @@ src/app/components/document-list/document-card-small/document-card-small.component.html - 50 + 48 @@ -2005,7 +2005,7 @@ src/app/components/document-list/document-card-small/document-card-small.component.html - 51 + 49 @@ -2016,7 +2016,7 @@ src/app/components/document-list/document-card-small/document-card-small.component.html - 52 + 50 @@ -2764,35 +2764,56 @@ Saved view "" deleted. src/app/components/manage/settings/settings.component.ts - 167 + 174 - - Settings saved successfully. + + Settings saved src/app/components/manage/settings/settings.component.ts - 238 + 247 + + + + Settings were saved successfully. + + src/app/components/manage/settings/settings.component.ts + 248 + + + + Settings were saved successfully. Reload is required to apply some changes. + + src/app/components/manage/settings/settings.component.ts + 252 + + + + Reload now + + src/app/components/manage/settings/settings.component.ts + 253 An error occurred while saving settings. src/app/components/manage/settings/settings.component.ts - 242 + 263 Use system language src/app/components/manage/settings/settings.component.ts - 250 + 271 Use date format of display language src/app/components/manage/settings/settings.component.ts - 257 + 278 @@ -2801,7 +2822,7 @@ )"/> src/app/components/manage/settings/settings.component.ts - 277,279 + 298,300 diff --git a/src-ui/src/app/components/common/toasts/toasts.component.html b/src-ui/src/app/components/common/toasts/toasts.component.html index b79208ebf..03715a440 100644 --- a/src-ui/src/app/components/common/toasts/toasts.component.html +++ b/src-ui/src/app/components/common/toasts/toasts.component.html @@ -4,5 +4,5 @@ [class]="toast.classname" (hidden)="toastService.closeToast(toast)">

{{toast.content}}

-

+

diff --git a/src-ui/src/app/components/manage/settings/settings.component.html b/src-ui/src/app/components/manage/settings/settings.component.html index 7e52db59e..002cc4eed 100644 --- a/src-ui/src/app/components/manage/settings/settings.component.html +++ b/src-ui/src/app/components/manage/settings/settings.component.html @@ -22,7 +22,7 @@ - You need to reload the page after applying a new language. + You need to reload the page after applying a new language.
diff --git a/src-ui/src/app/components/manage/settings/settings.component.ts b/src-ui/src/app/components/manage/settings/settings.component.ts index d9877d281..22ecfe9bb 100644 --- a/src-ui/src/app/components/manage/settings/settings.component.ts +++ b/src-ui/src/app/components/manage/settings/settings.component.ts @@ -14,7 +14,7 @@ import { LanguageOption, SettingsService, } from 'src/app/services/settings.service' -import { ToastService } from 'src/app/services/toast.service' +import { Toast, ToastService } from 'src/app/services/toast.service' import { dirtyCheck, DirtyComponent } from '@ngneat/dirty-check-forms' import { Observable, Subscription, BehaviorSubject, first } from 'rxjs' import { SETTINGS_KEYS } from 'src/app/data/paperless-uisettings' @@ -61,6 +61,13 @@ export class SettingsComponent implements OnInit, OnDestroy, DirtyComponent { ) } + get displayLanguageIsDirty(): boolean { + return ( + this.settingsForm.get('displayLanguage').value != + this.store?.getValue()['displayLanguage'] + ) + } + constructor( public savedViewService: SavedViewService, private documentListViewService: DocumentListViewService, @@ -170,6 +177,7 @@ export class SettingsComponent implements OnInit, OnDestroy, DirtyComponent { } private saveLocalSettings() { + const reloadRequired = this.displayLanguageIsDirty // just this one, for now this.settings.set( SETTINGS_KEYS.BULK_EDIT_APPLY_ON_CLOSE, this.settingsForm.value.bulkEditApplyOnClose @@ -235,7 +243,20 @@ export class SettingsComponent implements OnInit, OnDestroy, DirtyComponent { this.store.next(this.settingsForm.value) this.documentListViewService.updatePageSize() this.settings.updateAppearanceSettings() - this.toastService.showInfo($localize`Settings saved successfully.`) + let savedToast: Toast = { + title: $localize`Settings saved`, + content: $localize`Settings were saved successfully.`, + delay: 500000, + } + if (reloadRequired) { + ;(savedToast.content = $localize`Settings were saved successfully. Reload is required to apply some changes.`), + (savedToast.actionName = $localize`Reload now`) + savedToast.action = () => { + location.reload() + } + } + + this.toastService.show(savedToast) }, error: (error) => { this.toastService.showError( diff --git a/src-ui/src/styles.scss b/src-ui/src/styles.scss index 4b78e9d21..e0934b84c 100644 --- a/src-ui/src/styles.scss +++ b/src-ui/src/styles.scss @@ -84,6 +84,10 @@ svg.logo { } } +.text-primary { + color: var(--bs-primary) !important; +} + .btn-outline-primary { border-color: var(--bs-primary) !important; color: var(--bs-primary) !important; diff --git a/src-ui/src/theme.scss b/src-ui/src/theme.scss index 732ac47d9..bf9be6662 100644 --- a/src-ui/src/theme.scss +++ b/src-ui/src/theme.scss @@ -186,7 +186,8 @@ $form-check-radio-checked-bg-image-dark: url("data:image/svg+xml, MailMessage: email_msg = email.message.EmailMessage() # TODO: This does NOT set the UID @@ -175,6 +184,8 @@ def create_message( imap_msg.seen = seen imap_msg.flagged = flagged + if processed: + imap_msg._raw_flag_data.append(f"+FLAGS (processed)".encode()) return imap_msg @@ -189,6 +200,13 @@ def fake_magic_from_buffer(buffer, mime=False): return "Some verbose file description" +def parse_raw_tags_omit_cache(m: MailMessage) -> List[str]: + raw_result: list[bytes] = [] + for rf in m._raw_flag_data: + raw_result.extend(imaplib.ParseFlags(rf)) + return [f.decode().strip() for f in raw_result] + + @mock.patch("paperless_mail.mail.magic.from_buffer", fake_magic_from_buffer) class TestMail(DirectoriesMixin, TestCase): def setUp(self): @@ -217,6 +235,7 @@ class TestMail(DirectoriesMixin, TestCase): body="cables", seen=True, flagged=False, + processed=False, ), ) self.bogus_mailbox.messages.append( @@ -225,6 +244,7 @@ class TestMail(DirectoriesMixin, TestCase): body="from my favorite electronic store", seen=False, flagged=True, + processed=True, ), ) self.bogus_mailbox.messages.append( @@ -571,6 +591,29 @@ class TestMail(DirectoriesMixin, TestCase): self.assertEqual(len(self.bogus_mailbox.messages), 2) self.assertEqual(len(self.bogus_mailbox.messages_spam), 1) + def test_handle_mail_account_tag(self): + account = MailAccount.objects.create( + name="test", + imap_server="", + username="admin", + password="secret", + ) + + _ = MailRule.objects.create( + name="testrule", + account=account, + action=MailRule.MailAction.TAG, + action_parameter="processed", + ) + + self.assertEqual(len(self.bogus_mailbox.messages), 3) + self.assertEqual(self.async_task.call_count, 0) + self.assertEqual(len(self.bogus_mailbox.fetch("UNKEYWORD processed", False)), 2) + self.mail_account_handler.handle_mail_account(account) + self.assertEqual(self.async_task.call_count, 2) + self.assertEqual(len(self.bogus_mailbox.fetch("UNKEYWORD processed", False)), 0) + self.assertEqual(len(self.bogus_mailbox.messages), 3) + def test_error_login(self): account = MailAccount.objects.create( name="test", From 080a23dd8ced7c2bc3d71c993b1e8af048db3cb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20R=C3=BCmpelein?= Date: Thu, 26 May 2022 12:42:29 +0200 Subject: [PATCH 033/173] Update docs. --- docs/usage_overview.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/usage_overview.rst b/docs/usage_overview.rst index d7cfb8250..a321afb93 100644 --- a/docs/usage_overview.rst +++ b/docs/usage_overview.rst @@ -161,6 +161,9 @@ These are as follows: will not consume flagged mails. * **Move to folder:** Moves consumed mails out of the way so that paperless wont consume them again. +* **Add custom Tag:** Adds a custom tag to mails with consumed documents (the IMAP + standard calls these "keywords"). Paperless will not consume mails already tagged. + Not all mail servers support this feature! .. caution:: From 0cc7765f2b9ca8780a3865b7255bb4b3c776852d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20R=C3=BCmpelein?= Date: Thu, 26 May 2022 17:40:11 +0200 Subject: [PATCH 034/173] Revert accidentally included changes. --- docs/changelog.md | 186 +++++++++++++++++++++++----------------------- 1 file changed, 93 insertions(+), 93 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 319592b5d..9ba2f9aa0 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -17,7 +17,7 @@ ### Bug Fixes - Feature / fix saved view \& sort field query params [\@shamoon](https://github.com/shamoon) ([\#881](https://github.com/paperless-ngx/paperless-ngx/pull/881)) -- Mobile friendlier manage pages [\@shamoon](https://github.com/shamoon) ([\#873](https://github.com/paperless-ngx/paperless-ngx/pull/873)) +- mobile friendlier manage pages [\@shamoon](https://github.com/shamoon) ([\#873](https://github.com/paperless-ngx/paperless-ngx/pull/873)) - Add timeout to healthcheck [\@shamoon](https://github.com/shamoon) ([\#880](https://github.com/paperless-ngx/paperless-ngx/pull/880)) - Always accept yyyy-mm-dd date inputs [\@shamoon](https://github.com/shamoon) ([\#864](https://github.com/paperless-ngx/paperless-ngx/pull/864)) - Fix local Docker image building [\@stumpylog](https://github.com/stumpylog) ([\#849](https://github.com/paperless-ngx/paperless-ngx/pull/849)) @@ -70,137 +70,137 @@ ## paperless-ngx 1.7.0 -### Breaking Changes +Breaking Changes - `PAPERLESS_URL` is now required when using a reverse proxy. See [\#674](https://github.com/paperless-ngx/paperless-ngx/pull/674). -### Features +Features - Allow setting more than one tag in mail rules - [\@jonasc](https://github.com/jonasc) ([\#270](https://github.com/paperless-ngx/paperless-ngx/pull/270)) -- Global drag\'n\'drop [\@shamoon](https://github.com/shamoon) - ([\#283](https://github.com/paperless-ngx/paperless-ngx/pull/283)) + [\@jonasc](https://github.com/jonasc) (\#270) +- global drag\'n\'drop [\@shamoon](https://github.com/shamoon) + (\#283). - Fix: download buttons should disable while waiting - [\@shamoon](https://github.com/shamoon) ([\#630](https://github.com/paperless-ngx/paperless-ngx/pull/630)) -- Update checker [\@shamoon](https://github.com/shamoon) ([\#591](https://github.com/paperless-ngx/paperless-ngx/pull/591)) + [\@shamoon](https://github.com/shamoon) (\#630). +- Update checker [\@shamoon](https://github.com/shamoon) (\#591). - Show prompt on password-protected pdfs - [\@shamoon](https://github.com/shamoon) ([\#564](https://github.com/paperless-ngx/paperless-ngx/pull/564)) + [\@shamoon](https://github.com/shamoon) (\#564). - Filtering query params aka browser navigation for filtering - [\@shamoon](https://github.com/shamoon) ([\#540](https://github.com/paperless-ngx/paperless-ngx/pull/540)) + [\@shamoon](https://github.com/shamoon) (\#540). - Clickable tags in dashboard widgets - [\@shamoon](https://github.com/shamoon) ([\#515](https://github.com/paperless-ngx/paperless-ngx/pull/515)) + [\@shamoon](https://github.com/shamoon) (\#515). - Add bottom pagination [\@shamoon](https://github.com/shamoon) - ([\#372](https://github.com/paperless-ngx/paperless-ngx/pull/372)) + (\#372). - Feature barcode splitter [\@gador](https://github.com/gador) - ([\#532](https://github.com/paperless-ngx/paperless-ngx/pull/532)) -- App loading screen [\@shamoon](https://github.com/shamoon) ([\#298](https://github.com/paperless-ngx/paperless-ngx/pull/298)) + (\#532). +- App loading screen [\@shamoon](https://github.com/shamoon) (\#298). - Use progress bar for delayed buttons - [\@shamoon](https://github.com/shamoon) ([\#415](https://github.com/paperless-ngx/paperless-ngx/pull/415)) + [\@shamoon](https://github.com/shamoon) (\#415). - Add minimum length for documents text filter - [\@shamoon](https://github.com/shamoon) ([\#401](https://github.com/paperless-ngx/paperless-ngx/pull/401)) + [\@shamoon](https://github.com/shamoon) (\#401). - Added nav buttons in the document detail view - [\@GruberViktor](https://github.com/gruberviktor) ([\#273](https://github.com/paperless-ngx/paperless-ngx/pull/273)) + [\@GruberViktor](https://github.com/gruberviktor) (\#273). - Improve date keyboard input [\@shamoon](https://github.com/shamoon) - ([\#253](https://github.com/paperless-ngx/paperless-ngx/pull/253)) -- Color theming [\@shamoon](https://github.com/shamoon) ([\#243](https://github.com/paperless-ngx/paperless-ngx/pull/243)) + (\#253). +- Color theming [\@shamoon](https://github.com/shamoon) (\#243). - Parse dates when entered without separators - [\@GruberViktor](https://github.com/gruberviktor) ([\#250](https://github.com/paperless-ngx/paperless-ngx/pull/250)) + [\@GruberViktor](https://github.com/gruberviktor) (\#250). -### Bug Fixes +Bug Fixes -- Add \"localhost\" to ALLOWED_HOSTS - [\@gador](https://github.com/gador) ([\#700](https://github.com/paperless-ngx/paperless-ngx/pull/700)) -- Fix: scanners table [\@qcasey](https://github.com/qcasey) ([\#690](https://github.com/paperless-ngx/paperless-ngx/pull/690)) +- add \"localhost\" to ALLOWED_HOSTS + [\@gador](https://github.com/gador) (\#700). +- Fix: scanners table [\@qcasey](https://github.com/qcasey) (\#690). - Adds wait for file before consuming - [\@stumpylog](https://github.com/stumpylog) ([\#483](https://github.com/paperless-ngx/paperless-ngx/pull/483)) + [\@stumpylog](https://github.com/stumpylog) (\#483). - Fix: frontend document editing erases time data - [\@shamoon](https://github.com/shamoon) ([\#654](https://github.com/paperless-ngx/paperless-ngx/pull/654)) + [\@shamoon](https://github.com/shamoon) (\#654). - Increase length of SavedViewFilterRule - [\@stumpylog](https://github.com/stumpylog) ([\#612](https://github.com/paperless-ngx/paperless-ngx/pull/612)) + [\@stumpylog](https://github.com/stumpylog) (\#612). - Fixes attachment filename matching during mail fetching - [\@stumpylog](https://github.com/stumpylog) ([\#680](https://github.com/paperless-ngx/paperless-ngx/pull/680)) + [\@stumpylog](https://github.com/stumpylog) (\#680). - Add `PAPERLESS_URL` env variable & CSRF var - [\@shamoon](https://github.com/shamoon) ([\#674](https://github.com/paperless-ngx/paperless-ngx/discussions/674)) + [\@shamoon](https://github.com/shamoon) (\#674). - Fix: download buttons should disable while waiting - [\@shamoon](https://github.com/shamoon) ([\#630](https://github.com/paperless-ngx/paperless-ngx/pull/630)) + [\@shamoon](https://github.com/shamoon) (\#630). - Fixes downloaded filename, add more consumer ignore settings - [\@stumpylog](https://github.com/stumpylog) ([\#599](https://github.com/paperless-ngx/paperless-ngx/pull/599)) + [\@stumpylog](https://github.com/stumpylog) (\#599). - FIX BUG: case-sensitive matching was not possible - [\@danielBreitlauch](https://github.com/danielbreitlauch) ([\#594](https://github.com/paperless-ngx/paperless-ngx/pull/594)) -- Uses shutil.move instead of rename - [\@gador](https://github.com/gador) ([\#617](https://github.com/paperless-ngx/paperless-ngx/pull/617)) + [\@danielBreitlauch](https://github.com/danielbreitlauch) (\#594). +- uses shutil.move instead of rename + [\@gador](https://github.com/gador) (\#617). - Fix npm deps 01.02.22 2 [\@shamoon](https://github.com/shamoon) - ([\#610](https://github.com/paperless-ngx/paperless-ngx/discussions/610)) + (\#610). - Fix npm dependencies 01.02.22 - [\@shamoon](https://github.com/shamoon) ([\#600](https://github.com/paperless-ngx/paperless-ngx/pull/600)) -- Fix issue 416: implement `PAPERLESS_OCR_MAX_IMAGE_PIXELS` - [\@hacker-h](https://github.com/hacker-h) ([\#441](https://github.com/paperless-ngx/paperless-ngx/pull/441)) -- Fix: exclude cypress from build in Dockerfile - [\@FrankStrieter](https://github.com/FrankStrieter) ([\#526](https://github.com/paperless-ngx/paperless-ngx/pull/526)) + [\@shamoon](https://github.com/shamoon) (\#600). +- fix issue 416: implement PAPERLESS_OCR_MAX_IMAGE_PIXELS + [\@hacker-h](https://github.com/hacker-h) (\#441). +- fix: exclude cypress from build in Dockerfile + [\@FrankStrieter](https://github.com/FrankStrieter) (\#526). - Corrections to pass pre-commit hooks - [\@schnuffle](https://github.com/schnuffle) ([\#454](https://github.com/paperless-ngx/paperless-ngx/pull/454)) + [\@schnuffle](https://github.com/schnuffle) (\#454). - Fix 311 unable to click checkboxes in document list - [\@shamoon](https://github.com/shamoon) ([\#313](https://github.com/paperless-ngx/paperless-ngx/pull/313)) + [\@shamoon](https://github.com/shamoon) (\#313). - Fix imap tools bug [\@stumpylog](https://github.com/stumpylog) - ([\#393](https://github.com/paperless-ngx/paperless-ngx/pull/393)) + (\#393). - Fix filterable dropdown buttons arent translated - [\@shamoon](https://github.com/shamoon) ([\#366](https://github.com/paperless-ngx/paperless-ngx/pull/366)) + [\@shamoon](https://github.com/shamoon) (\#366). - Fix 224: \"Auto-detected date is day before receipt date\" - [\@a17t](https://github.com/a17t) ([\#246](https://github.com/paperless-ngx/paperless-ngx/pull/246)) + [\@a17t](https://github.com/a17t) (\#246). - Fix minor sphinx errors [\@shamoon](https://github.com/shamoon) - ([\#322](https://github.com/paperless-ngx/paperless-ngx/pull/322)) + (\#322). - Fix page links hidden [\@shamoon](https://github.com/shamoon) - ([\#314](https://github.com/paperless-ngx/paperless-ngx/pull/314)) + (\#314). - Fix: Include excluded items in dropdown count - [\@shamoon](https://github.com/shamoon) ([\#263](https://github.com/paperless-ngx/paperless-ngx/pull/263)) + [\@shamoon](https://github.com/shamoon) (\#263). -### Translation +Translation - [\@miku323](https://github.com/miku323) contributed to Slovenian - translation + translation. - [\@FaintGhost](https://github.com/FaintGhost) contributed to Chinese - Simplified translation + Simplified translation. - [\@DarkoBG79](https://github.com/DarkoBG79) contributed to Serbian - translation + translation. - [Kemal Secer](https://crowdin.com/profile/kemal.secer) contributed - to Turkish translation + to Turkish translation. - [\@Prominence](https://github.com/Prominence) contributed to - Belarusian translation + Belarusian translation. -### Documentation +Documentation -- Fix: scanners table [\@qcasey](https://github.com/qcasey) ([\#690](https://github.com/paperless-ngx/paperless-ngx/pull/690)) -- Add `PAPERLESS_URL` env variable & CSRF var - [\@shamoon](https://github.com/shamoon) ([\#674](https://github.com/paperless-ngx/paperless-ngx/pull/674)) +- Fix: scanners table [\@qcasey](https://github.com/qcasey) (\#690). +- Add [PAPERLESS\_URL]{.title-ref} env variable & CSRF var + [\@shamoon](https://github.com/shamoon) (\#674). - Fixes downloaded filename, add more consumer ignore settings - [\@stumpylog](https://github.com/stumpylog) ([\#599](https://github.com/paperless-ngx/paperless-ngx/pull/599)) -- Fix issue 416: implement `PAPERLESS_OCR_MAX_IMAGE_PIXELS` - [\@hacker-h](https://github.com/hacker-h) ([\#441](https://github.com/paperless-ngx/paperless-ngx/pull/441)) + [\@stumpylog](https://github.com/stumpylog) (\#599). +- fix issue 416: implement `PAPERLESS_OCR_MAX_IMAGE_PIXELS` + [\@hacker-h](https://github.com/hacker-h) (\#441). - Fix minor sphinx errors [\@shamoon](https://github.com/shamoon) - ([\#322](https://github.com/paperless-ngx/paperless-ngx/pull/322)) + (\#322). -### Maintenance +Maintenance - Add `PAPERLESS_URL` env variable & CSRF var - [\@shamoon](https://github.com/shamoon) ([\#674](https://github.com/paperless-ngx/paperless-ngx/pull/674)) + [\@shamoon](https://github.com/shamoon) (\#674). - Chore: Implement release-drafter action for Changelogs - [\@qcasey](https://github.com/qcasey) ([\#669](https://github.com/paperless-ngx/paperless-ngx/pull/669)) -- Chore: Add CODEOWNERS [\@qcasey](https://github.com/qcasey) ([\#667](https://github.com/paperless-ngx/paperless-ngx/pull/667)) + [\@qcasey](https://github.com/qcasey) (\#669). +- Chore: Add CODEOWNERS [\@qcasey](https://github.com/qcasey) (\#667). - Support docker-compose v2 in install - [\@stumpylog](https://github.com/stumpylog) ([\#611](https://github.com/paperless-ngx/paperless-ngx/pull/611)) + [\@stumpylog](https://github.com/stumpylog) (\#611). - Add Belarusian localization [\@shamoon](https://github.com/shamoon) - ([\#588](https://github.com/paperless-ngx/paperless-ngx/pull/588)) + (\#588). - Add Turkish localization [\@shamoon](https://github.com/shamoon) - ([\#536](https://github.com/paperless-ngx/paperless-ngx/pull/536)) + (\#536). - Add Serbian localization [\@shamoon](https://github.com/shamoon) - ([\#504](https://github.com/paperless-ngx/paperless-ngx/pull/504)) + (\#504). - Create PULL_REQUEST_TEMPLATE.md - [\@shamoon](https://github.com/shamoon) ([\#304](https://github.com/paperless-ngx/paperless-ngx/pull/304)) + [\@shamoon](https://github.com/shamoon) (\#304). - Add Chinese localization [\@shamoon](https://github.com/shamoon) - ([\#247](https://github.com/paperless-ngx/paperless-ngx/pull/247)) + (\#247). - Add Slovenian language for frontend - [\@shamoon](https://github.com/shamoon) ([\#315](https://github.com/paperless-ngx/paperless-ngx/pull/315)) + [\@shamoon](https://github.com/shamoon) (\#315). ## paperless-ngx 1.6.0 @@ -216,46 +216,46 @@ include: - Updated Python and Angular dependencies. - Dropped support for Python 3.7. - Dropped support for Ansible playbooks (thanks - [\@slankes](https://github.com/slankes) [\#109](https://github.com/paperless-ngx/paperless-ngx/pull/109)). If someone would - like to continue supporting them, please see our [ansible + [\@slankes](https://github.com/slankes) \#109). If someone would + like to continue supporting them, please see the [ansible repo](https://github.com/paperless-ngx/paperless-ngx-ansible). - Python code is now required to use Black formatting (thanks - [\@kpj](https://github.com/kpj) [\#168](https://github.com/paperless-ngx/paperless-ngx/pull/168)). + [\@kpj](https://github.com/kpj) \#168). - [\@tribut](https://github.com/tribut) added support for a custom SSO - logout redirect ([jonaswinkler\#1258](https://github.com/jonaswinkler/paperless-ng/pull/1258)). See + logout redirect (jonaswinkler\#1258). See `PAPERLESS_LOGOUT_REDIRECT_URL`. - [\@shamoon](https://github.com/shamoon) added a loading indicator - when document list is reloading ([jonaswinkler\#1297](https://github.com/jonaswinkler/paperless-ng/pull/1297)). + when document list is reloading (jonaswinkler\#1297). - [\@shamoon](https://github.com/shamoon) improved the PDF viewer on - mobile ([\#2](https://github.com/paperless-ngx/paperless-ngx/pull/2)). + mobile (\#2). - [\@shamoon](https://github.com/shamoon) added \'any\' / \'all\' and - \'not\' filtering with tags ([\#10](https://github.com/paperless-ngx/paperless-ngx/pull/10)). + \'not\' filtering with tags (\#10). - [\@shamoon](https://github.com/shamoon) added warnings for unsaved - changes, with smart edit buttons ([\#13](https://github.com/paperless-ngx/paperless-ngx/pull/13)). + changes, with smart edit buttons (\#13). - [\@benjaminfrank](https://github.com/benjaminfrank) enabled a - non-root access to port 80 via systemd ([\#18](https://github.com/paperless-ngx/paperless-ngx/pull/18)). + non-root access to port 80 via systemd (\#18). - [\@tribut](https://github.com/tribut) added simple \"delete to - trash\" functionality ([\#24](https://github.com/paperless-ngx/paperless-ngx/pull/24)). See `PAPERLESS_TRASH_DIR`. + trash\" functionality (\#24). See `PAPERLESS_TRASH_DIR`. - [\@amenk](https://github.com/amenk) fixed the search box overlay - menu on mobile ([\#32](https://github.com/paperless-ngx/paperless-ngx/pull/32)). + menu on mobile (\#32). - [\@dblitt](https://github.com/dblitt) updated the login form to not - auto-capitalize usernames ([\#36](https://github.com/paperless-ngx/paperless-ngx/pull/36)). + auto-capitalize usernames (\#36). - [\@evilsidekick293](https://github.com/evilsidekick293) made the - worker timeout configurable ([\#37](https://github.com/paperless-ngx/paperless-ngx/pull/37)). See `PAPERLESS_WORKER_TIMEOUT`. + worker timeout configurable (\#37). See `PAPERLESS_WORKER_TIMEOUT`. - [\@Nicarim](https://github.com/Nicarim) fixed downloads of UTF-8 - formatted documents in Firefox ([\#56](https://github.com/paperless-ngx/paperless-ngx/pull/56)). + formatted documents in Firefox (\#56). - [\@mweimerskirch](https://github.com/mweimerskirch) sorted the - language dropdown by locale ([\#78](https://github.com/paperless-ngx/paperless-ngx/issues/78)). + language dropdown by locale (\#78). - [\@mweimerskirch](https://github.com/mweimerskirch) enabled the - Czech ([\#83](https://github.com/paperless-ngx/paperless-ngx/pull/83)) and Danish ([\#84](https://github.com/paperless-ngx/paperless-ngx/pull/84)) translations. + Czech (\#83) and Danish (\#84) translations. - [\@cschmatzler](https://github.com/cschmatzler) enabled specifying - the webserver port ([\#124](https://github.com/paperless-ngx/paperless-ngx/pull/124)). See `PAPERLESS_PORT`. + the webserver port (\#124). See `PAPERLESS_PORT`. - [\@muellermartin](https://github.com/muellermartin) fixed an error - when uploading transparent PNGs ([\#133](https://github.com/paperless-ngx/paperless-ngx/pull/133)). + when uploading transparent PNGs (\#133). - [\@shamoon](https://github.com/shamoon) created a slick new logo - ([\#165](https://github.com/paperless-ngx/paperless-ngx/pull/165)). + (\#165). - [\@tim-vogel](https://github.com/tim-vogel) fixed exports missing - groups ([\#193](https://github.com/paperless-ngx/paperless-ngx/pull/193)). + groups (\#193). Known issues: From 5c980c31becd9a05eb9882665d059e1ae787e027 Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+shamoon@users.noreply.github.com> Date: Mon, 23 May 2022 00:24:52 -0700 Subject: [PATCH 035/173] PaperlessTask and consumption_tasks endpoint --- .../migrations/1021_paperlesstask.py | 66 +++++++++++++++++++ src/documents/models.py | 16 +++++ src/documents/serialisers.py | 8 +++ src/documents/signals/handlers.py | 21 ++++++ src/documents/tasks.py | 14 ++++ src/documents/views.py | 49 ++++++++++++++ src/paperless/urls.py | 6 ++ 7 files changed, 180 insertions(+) create mode 100644 src/documents/migrations/1021_paperlesstask.py diff --git a/src/documents/migrations/1021_paperlesstask.py b/src/documents/migrations/1021_paperlesstask.py new file mode 100644 index 000000000..f827a892a --- /dev/null +++ b/src/documents/migrations/1021_paperlesstask.py @@ -0,0 +1,66 @@ +# Generated by Django 4.0.4 on 2022-05-23 07:14 + +from django.db import migrations, models +import django.db.models.deletion + + +def init_paperless_tasks(apps, schema_editor): + PaperlessTask = apps.get_model("documents", "PaperlessTask") + Task = apps.get_model("django_q", "Task") + + for task in Task.objects.all(): + if not hasattr(task, "paperlesstask"): + paperlesstask = PaperlessTask.objects.create( + task=task, + q_task_id=task.id, + name=task.name, + created=task.started, + acknowledged=False, + ) + task.paperlesstask = paperlesstask + task.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ("django_q", "0014_schedule_cluster"), + ("documents", "1020_merge_20220518_1839"), + ] + + operations = [ + migrations.CreateModel( + name="PaperlessTask", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("q_task_id", models.CharField(max_length=128)), + ("name", models.CharField(max_length=256)), + ( + "created", + models.DateTimeField( + auto_now=True, db_index=True, verbose_name="created" + ), + ), + ("acknowledged", models.BooleanField(default=False)), + ( + "task", + models.OneToOneField( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="task", + to="django_q.task", + ), + ), + ], + ), + migrations.RunPython(init_paperless_tasks, migrations.RunPython.noop), + ] diff --git a/src/documents/models.py b/src/documents/models.py index b85c56037..f7ce9ae95 100644 --- a/src/documents/models.py +++ b/src/documents/models.py @@ -11,6 +11,7 @@ from django.contrib.auth.models import User from django.db import models from django.utils import timezone from django.utils.translation import gettext_lazy as _ +from django_q.tasks import Task from documents.parsers import get_default_file_extension @@ -500,3 +501,18 @@ class UiSettings(models.Model): def __str__(self): return self.user.username + + +class PaperlessTask(models.Model): + + q_task_id = models.CharField(max_length=128) + name = models.CharField(max_length=256) + created = models.DateTimeField(_("created"), auto_now=True, db_index=True) + task = models.OneToOneField( + Task, + on_delete=models.CASCADE, + related_name="task", + null=True, + blank=True, + ) + acknowledged = models.BooleanField(default=False) diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py index 8459cd037..e89c5d686 100644 --- a/src/documents/serialisers.py +++ b/src/documents/serialisers.py @@ -595,3 +595,11 @@ class UiSettingsViewSerializer(serializers.ModelSerializer): defaults={"settings": validated_data.get("settings", None)}, ) return ui_settings + + +class ConsupmtionTasksViewSerializer(serializers.Serializer): + + type = serializers.ChoiceField( + choices=["all", "incomplete", "complete", "failed"], + default="all", + ) diff --git a/src/documents/signals/handlers.py b/src/documents/signals/handlers.py index 34710af78..2e763cee0 100644 --- a/src/documents/signals/handlers.py +++ b/src/documents/signals/handlers.py @@ -13,6 +13,9 @@ from django.db.models import Q from django.dispatch import receiver from django.utils import termcolors from django.utils import timezone +from django_q.signals import post_save +from django_q.signals import pre_enqueue +from django_q.tasks import Task from filelock import FileLock from .. import matching @@ -21,6 +24,7 @@ from ..file_handling import delete_empty_directories from ..file_handling import generate_unique_filename from ..models import Document from ..models import MatchingModel +from ..models import PaperlessTask from ..models import Tag @@ -499,3 +503,20 @@ def add_to_index(sender, document, **kwargs): from documents import index index.add_or_update_document(document) + + +@receiver(pre_enqueue) +def init_paperless_task(sender, task, **kwargs): + if task["func"] == "documents.tasks.consume_file": + paperless_task = PaperlessTask.objects.get_or_create(q_task_id=task["id"]) + paperless_task.name = task["name"] + paperless_task.created = task["started"] + + +@receiver(post_save, sender=Task) +def update_paperless_task(sender, instance, **kwargs): + logger.debug(sender, instance) + papeless_task = PaperlessTask.objects.find(q_task_id=instance.id) + if papeless_task: + papeless_task.task = instance + papeless_task.save() diff --git a/src/documents/tasks.py b/src/documents/tasks.py index 208f74f1d..241ec9766 100644 --- a/src/documents/tasks.py +++ b/src/documents/tasks.py @@ -10,6 +10,7 @@ from asgiref.sync import async_to_sync from channels.layers import get_channel_layer from django.conf import settings from django.db.models.signals import post_save +from django_q.tasks import Task from documents import index from documents import sanity_checker from documents.classifier import DocumentClassifier @@ -359,3 +360,16 @@ def bulk_update_documents(document_ids): with AsyncWriter(ix) as writer: for doc in documents: index.update_document(writer, doc) + + +def create_paperless_task(sender, instance, created, **kwargs): + if created: + Task.objects.create(thing=instance) + + +post_save.connect( + create_paperless_task, + sender=Task, + weak=False, + dispatch_uid="models.create_paperless_task", +) diff --git a/src/documents/views.py b/src/documents/views.py index cdd38180b..d7f8bf10b 100644 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -64,12 +64,14 @@ from .matching import match_tags from .models import Correspondent from .models import Document from .models import DocumentType +from .models import PaperlessTask from .models import SavedView from .models import StoragePath from .models import Tag from .parsers import get_parser_class_for_mime_type from .serialisers import BulkDownloadSerializer from .serialisers import BulkEditSerializer +from .serialisers import ConsupmtionTasksViewSerializer from .serialisers import CorrespondentSerializer from .serialisers import DocumentListSerializer from .serialisers import DocumentSerializer @@ -795,3 +797,50 @@ class UiSettingsView(GenericAPIView): "success": True, }, ) + + +class ConsupmtionTasksView(GenericAPIView): + + permission_classes = (IsAuthenticated,) + serializer_class = ConsupmtionTasksViewSerializer + + def get(self, request, format=None): + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + + consumption_tasks = ( + PaperlessTask.objects.filter( + acknowledged=False, + ) + .order_by("task__started") + .reverse() + ) + incomplete_tasks = consumption_tasks.filter(task=None).values( + "id", + "q_task_id", + "name", + "created", + "acknowledged", + ) + failed_tasks = consumption_tasks.filter(task__success=0).values( + "id", + "q_task_id", + "name", + "created", + "acknowledged", + ) + completed_tasks = consumption_tasks.filter(task__success=1).values( + "id", + "q_task_id", + "name", + "created", + "acknowledged", + ) + return Response( + { + "total": consumption_tasks.count(), + "incomplete": incomplete_tasks, + "failed": failed_tasks, + "completed": completed_tasks, + }, + ) diff --git a/src/paperless/urls.py b/src/paperless/urls.py index 003d79f2d..6cea1b9e4 100644 --- a/src/paperless/urls.py +++ b/src/paperless/urls.py @@ -9,6 +9,7 @@ from django.views.decorators.csrf import csrf_exempt from django.views.generic import RedirectView from documents.views import BulkDownloadView from documents.views import BulkEditView +from documents.views import ConsupmtionTasksView from documents.views import CorrespondentViewSet from documents.views import DocumentTypeViewSet from documents.views import IndexView @@ -86,6 +87,11 @@ urlpatterns = [ UiSettingsView.as_view(), name="ui_settings", ), + re_path( + r"^consumption_tasks/", + ConsupmtionTasksView.as_view(), + name="consumption_tasks", + ), path("token/", views.obtain_auth_token), ] + api_router.urls, From f88e0704550d0da18758e97089fc1dd8f557a613 Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+shamoon@users.noreply.github.com> Date: Mon, 23 May 2022 01:21:06 -0700 Subject: [PATCH 036/173] Toggle functionality for tasks list --- .../manage/tasks/tasks.component.html | 83 +++++++++++++++++ .../manage/tasks/tasks.component.ts | 75 ++++++++++++++++ src-ui/src/app/services/tasks.service.ts | 89 +++++++++++++++++++ 3 files changed, 247 insertions(+) create mode 100644 src-ui/src/app/components/manage/tasks/tasks.component.html create mode 100644 src-ui/src/app/components/manage/tasks/tasks.component.ts create mode 100644 src-ui/src/app/services/tasks.service.ts diff --git a/src-ui/src/app/components/manage/tasks/tasks.component.html b/src-ui/src/app/components/manage/tasks/tasks.component.html new file mode 100644 index 000000000..edbfa126a --- /dev/null +++ b/src-ui/src/app/components/manage/tasks/tasks.component.html @@ -0,0 +1,83 @@ + +
+ + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + +
+ 		NameCreatedActions
+
+ + +
+ 		{{ task.name }}{{ task.created | customDate:'medium' }} + +
+ + + +
diff --git a/src-ui/src/app/components/manage/tasks/tasks.component.ts b/src-ui/src/app/components/manage/tasks/tasks.component.ts new file mode 100644 index 000000000..e0e7c3466 --- /dev/null +++ b/src-ui/src/app/components/manage/tasks/tasks.component.ts @@ -0,0 +1,75 @@ +import { Component, OnInit, OnDestroy } from '@angular/core' +import { takeUntil, Subject } from 'rxjs' +import { PaperlessTask } from 'src/app/data/paperless-task' +import { TasksService } from 'src/app/services/tasks.service' + +@Component({ + selector: 'app-tasks', + templateUrl: './tasks.component.html', + styleUrls: ['./tasks.component.scss'], +}) +export class TasksComponent implements OnInit, OnDestroy { + public activeTab: string + public selectedTasks: Set = new Set() + private unsubscribeNotifer = new Subject() + + get dismissButtonText(): string { + return this.selectedTasks.size > 0 + ? $localize`Dismiss selected` + : $localize`Dismiss all` + } + + constructor(public tasksService: TasksService) {} + + ngOnInit() { + this.tasksService.reload() + } + + ngOnDestroy() { + this.unsubscribeNotifer.next(true) + } + + dismissTask(task: PaperlessTask) { + throw new Error('Not implemented' + task) + } + + dismissMany() { + throw new Error('Not implemented') + } + + toggleSelected(task: PaperlessTask) { + this.selectedTasks.has(task.id) + ? this.selectedTasks.delete(task.id) + : this.selectedTasks.add(task.id) + } + + get currentTasks(): PaperlessTask[] { + let tasks: PaperlessTask[] + switch (this.activeTab) { + case 'incomplete': + tasks = this.tasksService.incomplete + break + case 'completed': + tasks = this.tasksService.completed + break + case 'failed': + tasks = this.tasksService.failed + break + default: + break + } + return tasks + } + + toggleAll(event: PointerEvent) { + if ((event.target as HTMLInputElement).checked) { + this.selectedTasks = new Set(this.currentTasks.map((t) => t.id)) + } else { + this.clearSelection() + } + } + + clearSelection() { + this.selectedTasks = new Set() + } +} diff --git a/src-ui/src/app/services/tasks.service.ts b/src-ui/src/app/services/tasks.service.ts new file mode 100644 index 000000000..3560122a9 --- /dev/null +++ b/src-ui/src/app/services/tasks.service.ts @@ -0,0 +1,89 @@ +import { HttpClient } from '@angular/common/http' +import { Injectable } from '@angular/core' +import { Observable } from 'rxjs' +import { first, map } from 'rxjs/operators' +import { PaperlessTask } from 'src/app/data/paperless-task' +import { environment } from 'src/environments/environment' + +interface TasksAPIResponse { + total: number + incomplete: Array + completed: Array + failed: Array +} + +@Injectable({ + providedIn: 'root', +}) +export class TasksService { + private baseUrl: string = environment.apiBaseUrl + + loading: boolean + + public total: number + + private incompleteTasks: PaperlessTask[] = [] + public get incomplete(): PaperlessTask[] { + return this.incompleteTasks + } + + private completedTasks: PaperlessTask[] = [] + public get completed(): PaperlessTask[] { + return this.completedTasks + } + + private failedTasks: PaperlessTask[] = [] + public get failed(): PaperlessTask[] { + return this.failedTasks + } + + constructor(private http: HttpClient) {} + + public reload() { + this.loading = true + + this.http + .get(`${this.baseUrl}consumption_tasks/`) + .pipe(first()) + .subscribe((r) => { + this.total = r.total + this.incompleteTasks = r.incomplete + this.completedTasks = r.completed + this.failedTasks = r.failed + this.loading = false + return true + }) + } + + // private savedViews: PaperlessSavedView[] = [] + + // get allViews() { + // return this.savedViews + // } + + // get sidebarViews() { + // return this.savedViews.filter((v) => v.show_in_sidebar) + // } + + // get dashboardViews() { + // return this.savedViews.filter((v) => v.show_on_dashboard) + // } + + // create(o: PaperlessSavedView) { + // return super.create(o).pipe(tap(() => this.reload())) + // } + + // update(o: PaperlessSavedView) { + // return super.update(o).pipe(tap(() => this.reload())) + // } + + // patchMany(objects: PaperlessSavedView[]): Observable { + // return combineLatest(objects.map((o) => super.patch(o))).pipe( + // tap(() => this.reload()) + // ) + // } + + // delete(o: PaperlessSavedView) { + // return super.delete(o).pipe(tap(() => this.reload())) + // } +} From 4bbaf5f89c911937d311b8999c7295fcd897b0a9 Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+shamoon@users.noreply.github.com> Date: Mon, 23 May 2022 01:52:46 -0700 Subject: [PATCH 037/173] update post_save signal receiver --- src-ui/src/app/data/paperless-task.ts | 11 ++++++++ .../migrations/1021_paperlesstask.py | 10 ++++---- src/documents/models.py | 6 ++--- src/documents/signals/handlers.py | 25 +++++++++++-------- src/documents/tasks.py | 14 ----------- src/documents/views.py | 12 ++++----- 6 files changed, 39 insertions(+), 39 deletions(-) create mode 100644 src-ui/src/app/data/paperless-task.ts diff --git a/src-ui/src/app/data/paperless-task.ts b/src-ui/src/app/data/paperless-task.ts new file mode 100644 index 000000000..cc864710c --- /dev/null +++ b/src-ui/src/app/data/paperless-task.ts @@ -0,0 +1,11 @@ +import { ObjectWithId } from './object-with-id' + +export interface PaperlessTask extends ObjectWithId { + acknowledged: boolean + + task_id: string + + name: string + + created: Date +} diff --git a/src/documents/migrations/1021_paperlesstask.py b/src/documents/migrations/1021_paperlesstask.py index f827a892a..2c9a0a885 100644 --- a/src/documents/migrations/1021_paperlesstask.py +++ b/src/documents/migrations/1021_paperlesstask.py @@ -11,8 +11,8 @@ def init_paperless_tasks(apps, schema_editor): for task in Task.objects.all(): if not hasattr(task, "paperlesstask"): paperlesstask = PaperlessTask.objects.create( - task=task, - q_task_id=task.id, + attempted_task=task, + task_id=task.id, name=task.name, created=task.started, acknowledged=False, @@ -41,7 +41,7 @@ class Migration(migrations.Migration): verbose_name="ID", ), ), - ("q_task_id", models.CharField(max_length=128)), + ("task_id", models.CharField(max_length=128)), ("name", models.CharField(max_length=256)), ( "created", @@ -51,12 +51,12 @@ class Migration(migrations.Migration): ), ("acknowledged", models.BooleanField(default=False)), ( - "task", + "attempted_task", models.OneToOneField( blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, - related_name="task", + related_name="attempted_task", to="django_q.task", ), ), diff --git a/src/documents/models.py b/src/documents/models.py index f7ce9ae95..63c1c6e33 100644 --- a/src/documents/models.py +++ b/src/documents/models.py @@ -505,13 +505,13 @@ class UiSettings(models.Model): class PaperlessTask(models.Model): - q_task_id = models.CharField(max_length=128) + task_id = models.CharField(max_length=128) name = models.CharField(max_length=256) created = models.DateTimeField(_("created"), auto_now=True, db_index=True) - task = models.OneToOneField( + attempted_task = models.OneToOneField( Task, on_delete=models.CASCADE, - related_name="task", + related_name="attempted_task", null=True, blank=True, ) diff --git a/src/documents/signals/handlers.py b/src/documents/signals/handlers.py index 2e763cee0..2ae78ab80 100644 --- a/src/documents/signals/handlers.py +++ b/src/documents/signals/handlers.py @@ -2,6 +2,7 @@ import logging import os import shutil +import django_q from django.conf import settings from django.contrib.admin.models import ADDITION from django.contrib.admin.models import LogEntry @@ -13,9 +14,6 @@ from django.db.models import Q from django.dispatch import receiver from django.utils import termcolors from django.utils import timezone -from django_q.signals import post_save -from django_q.signals import pre_enqueue -from django_q.tasks import Task from filelock import FileLock from .. import matching @@ -505,18 +503,23 @@ def add_to_index(sender, document, **kwargs): index.add_or_update_document(document) -@receiver(pre_enqueue) +@receiver(django_q.signals.pre_enqueue) def init_paperless_task(sender, task, **kwargs): if task["func"] == "documents.tasks.consume_file": - paperless_task = PaperlessTask.objects.get_or_create(q_task_id=task["id"]) + paperless_task, created = PaperlessTask.objects.get_or_create( + task_id=task["id"], + ) paperless_task.name = task["name"] paperless_task.created = task["started"] + paperless_task.save() -@receiver(post_save, sender=Task) +@receiver(models.signals.post_save, sender=django_q.tasks.Task) def update_paperless_task(sender, instance, **kwargs): - logger.debug(sender, instance) - papeless_task = PaperlessTask.objects.find(q_task_id=instance.id) - if papeless_task: - papeless_task.task = instance - papeless_task.save() + try: + if instance.func == "documents.tasks.consume_file": + paperless_task = PaperlessTask.objects.get(task_id=instance.id) + paperless_task.attempted_task = instance + paperless_task.save() + except PaperlessTask.DoesNotExist: + pass diff --git a/src/documents/tasks.py b/src/documents/tasks.py index 241ec9766..208f74f1d 100644 --- a/src/documents/tasks.py +++ b/src/documents/tasks.py @@ -10,7 +10,6 @@ from asgiref.sync import async_to_sync from channels.layers import get_channel_layer from django.conf import settings from django.db.models.signals import post_save -from django_q.tasks import Task from documents import index from documents import sanity_checker from documents.classifier import DocumentClassifier @@ -360,16 +359,3 @@ def bulk_update_documents(document_ids): with AsyncWriter(ix) as writer: for doc in documents: index.update_document(writer, doc) - - -def create_paperless_task(sender, instance, created, **kwargs): - if created: - Task.objects.create(thing=instance) - - -post_save.connect( - create_paperless_task, - sender=Task, - weak=False, - dispatch_uid="models.create_paperless_task", -) diff --git a/src/documents/views.py b/src/documents/views.py index d7f8bf10b..5514d8d6c 100644 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -812,26 +812,26 @@ class ConsupmtionTasksView(GenericAPIView): PaperlessTask.objects.filter( acknowledged=False, ) - .order_by("task__started") + .order_by("attempted_task__started") .reverse() ) incomplete_tasks = consumption_tasks.filter(task=None).values( "id", - "q_task_id", + "task_id", "name", "created", "acknowledged", ) - failed_tasks = consumption_tasks.filter(task__success=0).values( + failed_tasks = consumption_tasks.filter(attempted_task__success=0).values( "id", - "q_task_id", + "task_id", "name", "created", "acknowledged", ) - completed_tasks = consumption_tasks.filter(task__success=1).values( + completed_tasks = consumption_tasks.filter(attempted_task__success=1).values( "id", - "q_task_id", + "task_id", "name", "created", "acknowledged", From 0a06c291e2d4399210ec6d2d767476f5ec769dc5 Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+shamoon@users.noreply.github.com> Date: Mon, 23 May 2022 10:45:14 -0700 Subject: [PATCH 038/173] acknowledge_tasks endpoint & basic UI Update tasks.service.ts --- .../manage/tasks/tasks.component.html | 2 +- .../manage/tasks/tasks.component.ts | 8 ++-- src-ui/src/app/services/tasks.service.ts | 44 +++++-------------- src/documents/serialisers.py | 29 +++++++++++- src/documents/views.py | 39 ++++++++++++---- src/paperless/urls.py | 14 ++++-- 6 files changed, 86 insertions(+), 50 deletions(-) diff --git a/src-ui/src/app/components/manage/tasks/tasks.component.html b/src-ui/src/app/components/manage/tasks/tasks.component.html index edbfa126a..e0249865d 100644 --- a/src-ui/src/app/components/manage/tasks/tasks.component.html +++ b/src-ui/src/app/components/manage/tasks/tasks.component.html @@ -5,7 +5,7 @@  Clear selection - - - -
-