From 55a40708a682cf5e573cffb80d720f03dc48e220 Mon Sep 17 00:00:00 2001
From: shamoon <4887959+shamoon@users.noreply.github.com>
Date: Tue, 5 Mar 2024 07:50:04 -0800
Subject: [PATCH] Change: tweaks to system status (#6008)
---
src-ui/messages.xlf | 4 +-
.../system-status-dialog.component.html | 6 +-
.../system-status-dialog.component.ts | 6 +-
src-ui/src/app/data/system-status.ts | 1 +
src/documents/tests/test_api_status.py | 64 +++++++++++++++++--
src/documents/views.py | 50 +++++++++++++--
6 files changed, 114 insertions(+), 17 deletions(-)
diff --git a/src-ui/messages.xlf b/src-ui/messages.xlf
index bd80f5f15..958c5d33b 100644
--- a/src-ui/messages.xlf
+++ b/src-ui/messages.xlf
@@ -4281,7 +4281,7 @@
src/app/components/common/system-status-dialog/system-status-dialog.component.html
- 152
+ 156
@@ -4652,7 +4652,7 @@
Last Trained
src/app/components/common/system-status-dialog/system-status-dialog.component.html
- 135
+ 139
diff --git a/src-ui/src/app/components/common/system-status-dialog/system-status-dialog.component.html b/src-ui/src/app/components/common/system-status-dialog/system-status-dialog.component.html
index 5b60abe34..42ea1d008 100644
--- a/src-ui/src/app/components/common/system-status-dialog/system-status-dialog.component.html
+++ b/src-ui/src/app/components/common/system-status-dialog/system-status-dialog.component.html
@@ -128,7 +128,11 @@
}
} @else {
-
+
}
diff --git a/src-ui/src/app/components/common/system-status-dialog/system-status-dialog.component.ts b/src-ui/src/app/components/common/system-status-dialog/system-status-dialog.component.ts
index ae391c529..83468e711 100644
--- a/src-ui/src/app/components/common/system-status-dialog/system-status-dialog.component.ts
+++ b/src-ui/src/app/components/common/system-status-dialog/system-status-dialog.component.ts
@@ -1,6 +1,9 @@
import { Component, Input } from '@angular/core'
import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap'
-import { SystemStatus } from 'src/app/data/system-status'
+import {
+ SystemStatus,
+ SystemStatusItemStatus,
+} from 'src/app/data/system-status'
import { SystemStatusService } from 'src/app/services/system-status.service'
import { Clipboard } from '@angular/cdk/clipboard'
@@ -10,6 +13,7 @@ import { Clipboard } from '@angular/cdk/clipboard'
styleUrl: './system-status-dialog.component.scss',
})
export class SystemStatusDialogComponent {
+ public SystemStatusItemStatus = SystemStatusItemStatus
public status: SystemStatus
public copied: boolean = false
diff --git a/src-ui/src/app/data/system-status.ts b/src-ui/src/app/data/system-status.ts
index 247535602..a8f4ca621 100644
--- a/src-ui/src/app/data/system-status.ts
+++ b/src-ui/src/app/data/system-status.ts
@@ -6,6 +6,7 @@ export enum InstallType {
export enum SystemStatusItemStatus {
OK = 'OK',
ERROR = 'ERROR',
+ WARNING = 'WARNING',
}
export interface SystemStatus {
diff --git a/src/documents/tests/test_api_status.py b/src/documents/tests/test_api_status.py
index 964995bdc..1f8940ea9 100644
--- a/src/documents/tests/test_api_status.py
+++ b/src/documents/tests/test_api_status.py
@@ -1,4 +1,5 @@
import os
+import tempfile
from pathlib import Path
from unittest import mock
@@ -7,8 +8,11 @@ from django.test import override_settings
from rest_framework import status
from rest_framework.test import APITestCase
+from documents.classifier import ClassifierModelCorruptError
from documents.classifier import DocumentClassifier
from documents.classifier import load_classifier
+from documents.models import Document
+from documents.models import Tag
from paperless import version
@@ -158,7 +162,7 @@ class TestSystemStatus(APITestCase):
WHEN:
- The user requests the system status
THEN:
- - The response contains the correct classifier status
+ - The response contains an OK classifier status
"""
load_classifier()
test_classifier = DocumentClassifier()
@@ -169,18 +173,66 @@ class TestSystemStatus(APITestCase):
self.assertEqual(response.data["tasks"]["classifier_status"], "OK")
self.assertIsNone(response.data["tasks"]["classifier_error"])
- def test_system_status_classifier_error(self):
+ def test_system_status_classifier_warning(self):
"""
GIVEN:
- - The classifier is not found
+ - The classifier does not exist yet
+ - > 0 documents and tags with auto matching exist
WHEN:
- The user requests the system status
THEN:
- - The response contains an error classifier status
+ - The response contains an WARNING classifier status
+ """
+ with override_settings(MODEL_FILE="does_not_exist"):
+ Document.objects.create(
+ title="Test Document",
+ )
+ Tag.objects.create(name="Test Tag", matching_algorithm=Tag.MATCH_AUTO)
+ self.client.force_login(self.user)
+ response = self.client.get(self.ENDPOINT)
+ self.assertEqual(response.status_code, status.HTTP_200_OK)
+ self.assertEqual(response.data["tasks"]["classifier_status"], "WARNING")
+ self.assertIsNotNone(response.data["tasks"]["classifier_error"])
+
+ def test_system_status_classifier_error(self):
+ """
+ GIVEN:
+ - The classifier does exist but is corrupt
+ - > 0 documents and tags with auto matching exist
+ WHEN:
+ - The user requests the system status
+ THEN:
+ - The response contains an ERROR classifier status
+ """
+ does_exist = tempfile.NamedTemporaryFile(
+ dir="/tmp",
+ delete=False,
+ )
+ with override_settings(MODEL_FILE=does_exist):
+ with mock.patch("documents.classifier.load_classifier") as mock_load:
+ mock_load.side_effect = ClassifierModelCorruptError()
+ Document.objects.create(
+ title="Test Document",
+ )
+ Tag.objects.create(name="Test Tag", matching_algorithm=Tag.MATCH_AUTO)
+ self.client.force_login(self.user)
+ response = self.client.get(self.ENDPOINT)
+ self.assertEqual(response.status_code, status.HTTP_200_OK)
+ self.assertEqual(response.data["tasks"]["classifier_status"], "ERROR")
+ self.assertIsNotNone(response.data["tasks"]["classifier_error"])
+
+ def test_system_status_classifier_ok_no_objects(self):
+ """
+ GIVEN:
+ - The classifier does not exist (and should not)
+ - No documents nor objects with auto matching exist
+ WHEN:
+ - The user requests the system status
+ THEN:
+ - The response contains an OK classifier status
"""
with override_settings(MODEL_FILE="does_not_exist"):
self.client.force_login(self.user)
response = self.client.get(self.ENDPOINT)
self.assertEqual(response.status_code, status.HTTP_200_OK)
- self.assertEqual(response.data["tasks"]["classifier_status"], "ERROR")
- self.assertIsNotNone(response.data["tasks"]["classifier_error"])
+ self.assertEqual(response.data["tasks"]["classifier_status"], "OK")
diff --git a/src/documents/views.py b/src/documents/views.py
index bd0b6fa0f..0d2826cf7 100644
--- a/src/documents/views.py
+++ b/src/documents/views.py
@@ -1581,7 +1581,9 @@ class SystemStatusView(GenericAPIView, PassUserMixin):
except Exception as e: # pragma: no cover
applied_migrations = []
db_status = "ERROR"
- logger.exception(f"System status error connecting to database: {e}")
+ logger.exception(
+ f"System status detected a possible problem while connecting to the database: {e}",
+ )
db_error = "Error connecting to database, check logs for more detail."
media_stats = os.statvfs(settings.MEDIA_ROOT)
@@ -1594,7 +1596,9 @@ class SystemStatusView(GenericAPIView, PassUserMixin):
redis_status = "OK"
except Exception as e:
redis_status = "ERROR"
- logger.exception(f"System status error connecting to redis: {e}")
+ logger.exception(
+ f"System status detected a possible problem while connecting to redis: {e}",
+ )
redis_error = "Error connecting to redis, check logs for more detail."
try:
@@ -1615,14 +1619,40 @@ class SystemStatusView(GenericAPIView, PassUserMixin):
except Exception as e:
index_status = "ERROR"
index_error = "Error opening index, check logs for more detail."
- logger.exception(f"System status error opening index: {e}")
+ logger.exception(
+ f"System status detected a possible problem while opening the index: {e}",
+ )
index_last_modified = None
classifier_error = None
+ classifier_status = None
try:
classifier = load_classifier()
if classifier is None:
- raise Exception("Classifier not loaded")
+ # Make sure classifier should exist
+ docs_queryset = Document.objects.exclude(
+ tags__is_inbox_tag=True,
+ )
+ if (
+ docs_queryset.count() > 0
+ and (
+ Tag.objects.filter(matching_algorithm=Tag.MATCH_AUTO).exists()
+ or DocumentType.objects.filter(
+ matching_algorithm=Tag.MATCH_AUTO,
+ ).exists()
+ or Correspondent.objects.filter(
+ matching_algorithm=Tag.MATCH_AUTO,
+ ).exists()
+ or StoragePath.objects.filter(
+ matching_algorithm=Tag.MATCH_AUTO,
+ ).exists()
+ )
+ and not os.path.isfile(settings.MODEL_FILE)
+ ):
+ # if classifier file doesn't exist just classify as a warning
+ classifier_error = "Classifier file does not exist (yet). Re-training may be pending."
+ classifier_status = "WARNING"
+ raise FileNotFoundError(classifier_error)
classifier_status = "OK"
task_result_model = apps.get_model("django_celery_results", "taskresult")
result = (
@@ -1637,10 +1667,16 @@ class SystemStatusView(GenericAPIView, PassUserMixin):
)
classifier_last_trained = result.date_done if result else None
except Exception as e:
- classifier_status = "ERROR"
+ if classifier_status is None:
+ classifier_status = "ERROR"
classifier_last_trained = None
- classifier_error = "Error loading classifier, check logs for more detail."
- logger.exception(f"System status error loading classifier: {e}")
+ if classifier_error is None:
+ classifier_error = (
+ "Unable to load classifier, check logs for more detail."
+ )
+ logger.exception(
+ f"System status detected a possible problem while loading the classifier: {e}",
+ )
return Response(
{