From 2d5222673240f659df82998df813d31e2aefdbe8 Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Wed, 26 Feb 2025 14:12:20 -0800 Subject: [PATCH] Enhancement: system status report sanity check, simpler classifier check, styling updates (#9106) --- src-ui/messages.xlf | 88 +++- .../admin/settings/settings.component.spec.ts | 7 + .../admin/settings/settings.component.ts | 5 +- .../admin/tasks/tasks.component.spec.ts | 23 +- .../system-status-dialog.component.html | 235 +++++++--- .../system-status-dialog.component.scss | 3 + .../system-status-dialog.component.spec.ts | 49 ++ .../system-status-dialog.component.ts | 44 +- src-ui/src/app/data/paperless-task.ts | 14 +- src-ui/src/app/data/system-status.ts | 5 + src-ui/src/app/services/tasks.service.spec.ts | 50 +- src-ui/src/app/services/tasks.service.ts | 22 +- src-ui/src/main.ts | 2 + src-ui/src/theme.scss | 22 +- src/documents/classifier.py | 16 - src/documents/filters.py | 16 + .../commands/document_create_classifier.py | 2 +- .../commands/document_sanity_checker.py | 2 +- ...alter_paperlesstask_task_name_and_more.py} | 42 +- src/documents/models.py | 26 +- src/documents/sanity_checker.py | 23 +- src/documents/serialisers.py | 44 +- src/documents/signals/handlers.py | 3 +- src/documents/tasks.py | 38 +- src/documents/tests/test_api_status.py | 147 +++--- src/documents/tests/test_api_tasks.py | 67 ++- src/documents/tests/test_task_signals.py | 2 +- src/documents/tests/test_tasks.py | 13 + src/documents/views.py | 158 ++++--- src/locale/en_US/LC_MESSAGES/django.po | 428 ++++++++++-------- 30 files changed, 1117 insertions(+), 479 deletions(-) rename src/documents/migrations/{1063_alter_workflowactionwebhook_url_and_more.py => 1063_paperlesstask_type_alter_paperlesstask_task_name_and_more.py} (50%) diff --git a/src-ui/messages.xlf b/src-ui/messages.xlf index 12b0a09a6..7a35f94fe 100644 --- a/src-ui/messages.xlf +++ b/src-ui/messages.xlf @@ -1514,7 +1514,7 @@ Error retrieving users src/app/components/admin/settings/settings.component.ts - 217 + 220 src/app/components/admin/users-groups/users-groups.component.ts @@ -1525,7 +1525,7 @@ Error retrieving groups src/app/components/admin/settings/settings.component.ts - 236 + 239 src/app/components/admin/users-groups/users-groups.component.ts @@ -1536,28 +1536,28 @@ Settings were saved successfully. src/app/components/admin/settings/settings.component.ts - 532 + 535 Settings were saved successfully. Reload is required to apply some changes. src/app/components/admin/settings/settings.component.ts - 536 + 539 Reload now src/app/components/admin/settings/settings.component.ts - 537 + 540 An error occurred while saving settings. src/app/components/admin/settings/settings.component.ts - 547 + 550 src/app/components/app-frame/app-frame.component.ts @@ -4099,6 +4099,18 @@ src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html 111 + + src/app/components/common/system-status-dialog/system-status-dialog.component.html + 175 + + + src/app/components/common/system-status-dialog/system-status-dialog.component.html + 209 + + + src/app/components/common/system-status-dialog/system-status-dialog.component.html + 243 + src/app/components/common/toast/toast.component.html 30 @@ -5615,7 +5627,7 @@ src/app/components/common/system-status-dialog/system-status-dialog.component.html - 156 + 261 src/app/components/manage/mail/mail.component.html @@ -6010,77 +6022,113 @@ Migration Status src/app/components/common/system-status-dialog/system-status-dialog.component.html - 56 + 65 Up to date src/app/components/common/system-status-dialog/system-status-dialog.component.html - 59 + 69 Latest Migration src/app/components/common/system-status-dialog/system-status-dialog.component.html - 64 + 74 Pending Migrations src/app/components/common/system-status-dialog/system-status-dialog.component.html - 66 + 76 - - Tasks + + Tasks Queue src/app/components/common/system-status-dialog/system-status-dialog.component.html - 83 + 94 Redis Status src/app/components/common/system-status-dialog/system-status-dialog.component.html - 87 + 98 Celery Status src/app/components/common/system-status-dialog/system-status-dialog.component.html - 96 + 116 + + + + Health + + src/app/components/common/system-status-dialog/system-status-dialog.component.html + 142 Search Index src/app/components/common/system-status-dialog/system-status-dialog.component.html - 105 + 146 + + + + Run Task + + src/app/components/common/system-status-dialog/system-status-dialog.component.html + 166 + + + src/app/components/common/system-status-dialog/system-status-dialog.component.html + 200 + + + src/app/components/common/system-status-dialog/system-status-dialog.component.html + 234 Last Updated src/app/components/common/system-status-dialog/system-status-dialog.component.html - 119 + 173 Classifier src/app/components/common/system-status-dialog/system-status-dialog.component.html - 121 + 178 Last Trained src/app/components/common/system-status-dialog/system-status-dialog.component.html - 139 + 207 + + + + Sanity Checker + + src/app/components/common/system-status-dialog/system-status-dialog.component.html + 212 + + + + Last Run + + src/app/components/common/system-status-dialog/system-status-dialog.component.html + 241 diff --git a/src-ui/src/app/components/admin/settings/settings.component.spec.ts b/src-ui/src/app/components/admin/settings/settings.component.spec.ts index 4f50e7453..c6eeaf896 100644 --- a/src-ui/src/app/components/admin/settings/settings.component.spec.ts +++ b/src-ui/src/app/components/admin/settings/settings.component.spec.ts @@ -303,12 +303,17 @@ describe('SettingsComponent', () => { redis_error: 'Error 61 connecting to localhost:6379. Connection refused.', celery_status: SystemStatusItemStatus.ERROR, + celery_url: 'celery@localhost', + celery_error: 'Error connecting to celery@localhost', index_status: SystemStatusItemStatus.OK, index_last_modified: new Date().toISOString(), index_error: null, classifier_status: SystemStatusItemStatus.OK, classifier_last_trained: new Date().toISOString(), classifier_error: null, + sanity_check_status: SystemStatusItemStatus.ERROR, + sanity_check_last_run: new Date().toISOString(), + sanity_check_error: 'Error running sanity check.', }, } jest.spyOn(systemStatusService, 'get').mockReturnValue(of(status)) @@ -320,6 +325,8 @@ describe('SettingsComponent', () => { component['systemStatus'].database.status = SystemStatusItemStatus.OK component['systemStatus'].tasks.redis_status = SystemStatusItemStatus.OK component['systemStatus'].tasks.celery_status = SystemStatusItemStatus.OK + component['systemStatus'].tasks.sanity_check_status = + SystemStatusItemStatus.OK expect(component.systemStatusHasErrors).toBeFalsy() }) diff --git a/src-ui/src/app/components/admin/settings/settings.component.ts b/src-ui/src/app/components/admin/settings/settings.component.ts index 68f702cfa..8737be160 100644 --- a/src-ui/src/app/components/admin/settings/settings.component.ts +++ b/src-ui/src/app/components/admin/settings/settings.component.ts @@ -164,7 +164,10 @@ export class SettingsComponent this.systemStatus.tasks.redis_status === SystemStatusItemStatus.ERROR || this.systemStatus.tasks.celery_status === SystemStatusItemStatus.ERROR || this.systemStatus.tasks.index_status === SystemStatusItemStatus.ERROR || - this.systemStatus.tasks.classifier_status === SystemStatusItemStatus.ERROR + this.systemStatus.tasks.classifier_status === + SystemStatusItemStatus.ERROR || + this.systemStatus.tasks.sanity_check_status === + SystemStatusItemStatus.ERROR ) } diff --git a/src-ui/src/app/components/admin/tasks/tasks.component.spec.ts b/src-ui/src/app/components/admin/tasks/tasks.component.spec.ts index a0a6f9ea0..8158be7b2 100644 --- a/src-ui/src/app/components/admin/tasks/tasks.component.spec.ts +++ b/src-ui/src/app/components/admin/tasks/tasks.component.spec.ts @@ -19,6 +19,7 @@ import { allIcons, NgxBootstrapIconsModule } from 'ngx-bootstrap-icons' import { routes } from 'src/app/app-routing.module' import { PaperlessTask, + PaperlessTaskName, PaperlessTaskStatus, PaperlessTaskType, } from 'src/app/data/paperless-task' @@ -39,7 +40,8 @@ const tasks: PaperlessTask[] = [ task_file_name: 'test.pdf', date_created: new Date('2023-03-01T10:26:03.093116Z'), date_done: new Date('2023-03-01T10:26:07.223048Z'), - type: PaperlessTaskType.File, + type: PaperlessTaskType.Auto, + task_name: PaperlessTaskName.ConsumeFile, status: PaperlessTaskStatus.Failed, result: 'test.pd: Not consuming test.pdf: It is a duplicate of test (#100)', acknowledged: false, @@ -51,7 +53,8 @@ const tasks: PaperlessTask[] = [ task_file_name: '191092.pdf', date_created: new Date('2023-03-01T09:26:03.093116Z'), date_done: new Date('2023-03-01T09:26:07.223048Z'), - type: PaperlessTaskType.File, + type: PaperlessTaskType.Auto, + task_name: PaperlessTaskName.ConsumeFile, status: PaperlessTaskStatus.Failed, result: '191092.pd: Not consuming 191092.pdf: It is a duplicate of 191092 (#311)', @@ -64,7 +67,8 @@ const tasks: PaperlessTask[] = [ task_file_name: 'Scan Jun 6, 2023 at 3.19 PM.pdf', date_created: new Date('2023-06-06T15:22:05.722323-07:00'), date_done: new Date('2023-06-06T15:22:14.564305-07:00'), - type: PaperlessTaskType.File, + type: PaperlessTaskType.Auto, + task_name: PaperlessTaskName.ConsumeFile, status: PaperlessTaskStatus.Pending, result: null, acknowledged: false, @@ -76,7 +80,8 @@ const tasks: PaperlessTask[] = [ task_file_name: 'paperless-mail-l4dkg8ir', date_created: new Date('2023-06-04T11:24:32.898089-07:00'), date_done: new Date('2023-06-04T11:24:44.678605-07:00'), - type: PaperlessTaskType.File, + type: PaperlessTaskType.Auto, + task_name: PaperlessTaskName.ConsumeFile, status: PaperlessTaskStatus.Complete, result: 'Success. New document id 422 created', acknowledged: false, @@ -88,7 +93,8 @@ const tasks: PaperlessTask[] = [ task_file_name: 'onlinePaymentSummary.pdf', date_created: new Date('2023-06-01T13:49:51.631305-07:00'), date_done: new Date('2023-06-01T13:49:54.190220-07:00'), - type: PaperlessTaskType.File, + type: PaperlessTaskType.Auto, + task_name: PaperlessTaskName.ConsumeFile, status: PaperlessTaskStatus.Complete, result: 'Success. New document id 421 created', acknowledged: false, @@ -100,7 +106,8 @@ const tasks: PaperlessTask[] = [ task_file_name: 'paperless-mail-_rrpmqk6', date_created: new Date('2023-06-07T02:54:35.694916Z'), date_done: null, - type: PaperlessTaskType.File, + type: PaperlessTaskType.Auto, + task_name: PaperlessTaskName.ConsumeFile, status: PaperlessTaskStatus.Started, result: null, acknowledged: false, @@ -155,7 +162,9 @@ describe('TasksComponent', () => { jest.useFakeTimers() fixture.detectChanges() httpTestingController - .expectOne(`${environment.apiBaseUrl}tasks/`) + .expectOne( + `${environment.apiBaseUrl}tasks/?task_name=consume_file&acknowledged=false` + ) .flush(tasks) }) diff --git a/src-ui/src/app/components/common/system-status-dialog/system-status-dialog.component.html b/src-ui/src/app/components/common/system-status-dialog/system-status-dialog.component.html index 2dc934df4..94c4ef22d 100644 --- a/src-ui/src/app/components/common/system-status-dialog/system-status-dialog.component.html +++ b/src-ui/src/app/components/common/system-status-dialog/system-status-dialog.component.html @@ -1,5 +1,5 @@
- +
@@ -11,11 +11,11 @@
} @else { -
+
-
Environment
+
Environment
@@ -38,37 +38,96 @@
-
Database
+
Database
Type
{{status.database.type}}
Status
-
- {{status.database.status}} - @if (status.database.status === 'OK') { - - } @else { - - } +
+ + + @if (status.database.status === 'OK') { + {{status.database.url}} + } @else { + {{status.database.url}}: {{status.database.error}} + } +
Migration Status
-
- @if (status.database.migration_status.unapplied_migrations.length === 0) { - Up to date - } @else { - {{status.database.migration_status.unapplied_migrations.length}} Pending - } - -
Latest Migration:
{{status.database.migration_status.latest_migration}} - @if (status.database.migration_status.unapplied_migrations.length > 0) { -
Pending Migrations:
-
    - @for (migration of status.database.migration_status.unapplied_migrations; track migration) { -
  • {{migration}}
    • - } -
+
+ +
+
+
+
+
+ +
+
+
+
Tasks Queue
+
+
+
+
Redis Status
+
+ + + @if (status.tasks.redis_status === 'OK') { + {{status.tasks.redis_url}} + } @else { + {{status.tasks.redis_url}}: {{status.tasks.redis_error}} + } + +
+
Celery Status
+
+ + + @if (status.tasks.celery_status === 'OK') { + {{status.tasks.celery_url}} + } @else { + {{status.tasks.celery_error}} }
@@ -80,63 +139,109 @@
-
Tasks
+
Health
-
Redis Status
-
- {{status.tasks.redis_status}} - @if (status.tasks.redis_status === 'OK') { - - } @else { - - } -
-
Celery Status
-
- {{status.tasks.celery_status}} - @if (status.tasks.celery_status === 'OK') { - - } @else { - - } -
Search Index
- {{status.tasks.index_status}} - @if (status.tasks.index_status === 'OK') { - @if (isStale(status.tasks.index_last_modified)) { - + + @if (currentUserIsSuperUser) { + @if (isRunning(PaperlessTaskName.IndexOptimize)) { + + } @else { + } - } @else { - }
-
Last Updated:
{{status.tasks.index_last_modified | customDate:'medium'}} + @if (status.tasks.index_status === 'OK') { +
Last Updated:
{{status.tasks.index_last_modified | customDate:'medium'}} + } @else { +
Error:
{{status.tasks.index_error}} + }
Classifier
- {{status.tasks.classifier_status}} - @if (status.tasks.classifier_status === 'OK') { - @if (isStale(status.tasks.classifier_last_trained)) { - + + @if (currentUserIsSuperUser) { + @if (isRunning(PaperlessTaskName.TrainClassifier)) { + + } @else { + + } }
-
Last Trained:
{{status.tasks.classifier_last_trained | customDate:'medium'}} + @if (status.tasks.classifier_status === 'OK') { +
Last Trained:
{{status.tasks.classifier_last_trained | customDate:'medium'}} + } @else { +
Error:
{{status.tasks.classifier_error}} + } + +
Sanity Checker
+
+ + @if (currentUserIsSuperUser) { + @if (isRunning(PaperlessTaskName.SanityCheck)) { + + } @else { + + } + } +
+ + @if (status.tasks.sanity_check_status === 'OK') { +
Last Run:
{{status.tasks.sanity_check_last_run | customDate:'medium'}} + } @else { +
Error:
{{status.tasks.sanity_check_error}} + }
@@ -146,7 +251,7 @@ }
-