From f421d4558a744cdb11b74e31ecc54856c27f189f Mon Sep 17 00:00:00 2001
From: Jonas Winkler <jonas.winkler@jpwinkler.de>
Date: Wed, 4 Nov 2020 20:09:35 +0100
Subject: [PATCH 1/6] i screwed up the dockerfile

---
 Dockerfile                 | 4 ++--
 docker-compose.yml.example | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 3b58d3ff5..b3a0f7ef3 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,5 +1,3 @@
-LABEL maintainer="Jonas Winkler <dev@jpwinkler.de>"
-
 ###############################################################################
 ### Front end                                                               ###
 ###############################################################################
@@ -81,3 +79,5 @@ RUN sudo -HEu paperless python3 manage.py collectstatic --clear --no-input
 VOLUME ["/usr/src/paperless/data", "/usr/src/paperless/consume", "/usr/src/paperless/export"]
 ENTRYPOINT ["/sbin/docker-entrypoint.sh"]
 CMD ["python3", "manage.py", "--help"]
+
+LABEL maintainer="Jonas Winkler <dev@jpwinkler.de>"
diff --git a/docker-compose.yml.example b/docker-compose.yml.example
index f6312ff50..1130e26a3 100644
--- a/docker-compose.yml.example
+++ b/docker-compose.yml.example
@@ -12,7 +12,7 @@ services:
 
   webserver:
     build: .
-    image: paperless_app
+    image: paperless-ng
     #restart: always
     depends_on:
       - db

From 7d8562b1d4c1a6b76d10560e708a8f13882c486a Mon Sep 17 00:00:00 2001
From: Jonas Winkler <jonas.winkler@jpwinkler.de>
Date: Wed, 4 Nov 2020 21:18:16 +0100
Subject: [PATCH 2/6] layout fixes #13

---
 .../app/components/app-frame/app-frame.component.html  |  8 ++++----
 .../common/page-header/page-header.component.html      | 10 ++++++----
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/src-ui/src/app/components/app-frame/app-frame.component.html b/src-ui/src/app/components/app-frame/app-frame.component.html
index a0350eb78..ad12a9d43 100644
--- a/src-ui/src/app/components/app-frame/app-frame.component.html
+++ b/src-ui/src/app/components/app-frame/app-frame.component.html
@@ -47,8 +47,8 @@
           <span>Saved views</span>
         </h6>
         <ul class="nav flex-column mb-2">
-          <li class="nav-item" *ngFor='let config of viewConfigService.getSideBarConfigs()'>
-            <a class="nav-link" routerLink="view/{{config.id}}" routerLinkActive="active">
+          <li class="nav-item w-100" *ngFor='let config of viewConfigService.getSideBarConfigs()'>
+            <a class="nav-link text-truncate" routerLink="view/{{config.id}}" routerLinkActive="active">
               <svg class="sidebaricon" fill="currentColor">
                 <use xlink:href="assets/bootstrap-icons.svg#funnel"/>
               </svg>
@@ -61,8 +61,8 @@
           <span>Open documents</span>
         </h6>
         <ul class="nav flex-column mb-2">
-          <li class="nav-item" *ngFor='let d of openDocuments'>
-            <a class="nav-link" routerLink="documents/{{d.id}}" routerLinkActive="active">
+          <li class="nav-item w-100" *ngFor='let d of openDocuments'>
+            <a class="nav-link text-truncate" routerLink="documents/{{d.id}}" routerLinkActive="active">
               <svg class="sidebaricon" fill="currentColor">
                 <use xlink:href="assets/bootstrap-icons.svg#file-text"/>
               </svg>
diff --git a/src-ui/src/app/components/common/page-header/page-header.component.html b/src-ui/src/app/components/common/page-header/page-header.component.html
index cff102826..7c84046d8 100644
--- a/src-ui/src/app/components/common/page-header/page-header.component.html
+++ b/src-ui/src/app/components/common/page-header/page-header.component.html
@@ -1,6 +1,8 @@
-<div class="d-flex justify-content-between flex-wrap flex-md-nowrap align-items-center pt-3 pb-2 mb-3 border-bottom">
-  <h1 class="h2">{{title}}</h1>
-  <div class="btn-toolbar mb-2 mb-md-0">
+<div class="row pt-3 pb-2 mb-3 border-bottom align-items-center">
+  <div class="col text-truncate">
+    <h1 class="h2 text-truncate">{{title}}</h1>
+  </div>
+  <div class="btn-toolbar col-auto">
     <ng-content></ng-content>
   </div>
-</div>
\ No newline at end of file
+</div>

From f149d3d10e43f422e3b41b1e2cd4198b6ee53bda Mon Sep 17 00:00:00 2001
From: Jonas Winkler <jonas.winkler@jpwinkler.de>
Date: Thu, 5 Nov 2020 11:23:58 +0100
Subject: [PATCH 3/6] new Dockerfile fixes #21

---
 Dockerfile | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index b3a0f7ef3..015d511d4 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -16,7 +16,7 @@ RUN node_modules/.bin/ng build --prod --output-hashing none --sourceMap=false
 ### Back end                                                                ###
 ###############################################################################
 
-FROM python:3.8-slim
+FROM ubuntu:20.04
 
 WORKDIR /usr/src/paperless/
 
@@ -24,7 +24,7 @@ COPY Pipfile* ./
 
 #Dependencies
 RUN apt-get update \
-  && apt-get -y --no-install-recommends install \
+  && DEBIAN_FRONTEND="noninteractive" apt-get -y --no-install-recommends install \
 		anacron \
 		build-essential \
 		curl \
@@ -35,6 +35,9 @@ RUN apt-get update \
 		libpoppler-cpp-dev \
 		libpq-dev \
 		optipng \
+		python3 \
+		python3-dev \
+		python3-pip \
 		sudo \
 		tesseract-ocr \
 		tesseract-ocr-eng \
@@ -44,10 +47,10 @@ RUN apt-get update \
 		tesseract-ocr-spa \
 		tzdata \
 		unpaper \
-	&& pip install --upgrade pipenv supervisor \
+	&& pip3 install --upgrade pipenv supervisor setuptools \
 	&& pipenv install --system --deploy \
 	&& pipenv --clear \
-	&& apt-get -y purge build-essential \
+	&& apt-get -y purge build-essential python3-pip python3-dev \
 	&& apt-get -y autoremove --purge \
 	&& rm -rf /var/lib/apt/lists/* \
 	&& mkdir /var/log/supervisord /var/run/supervisord

From 9fa5eac9b999c0fe1d073edc32a351724684b71b Mon Sep 17 00:00:00 2001
From: Jonas Winkler <jonas.winkler@jpwinkler.de>
Date: Fri, 6 Nov 2020 14:34:24 +0100
Subject: [PATCH 4/6] layout changed

---
 src-ui/src/app/components/dashboard/dashboard.component.html    | 2 +-
 .../app/components/document-list/document-list.component.html   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src-ui/src/app/components/dashboard/dashboard.component.html b/src-ui/src/app/components/dashboard/dashboard.component.html
index 1894b3e0b..694b431c4 100644
--- a/src-ui/src/app/components/dashboard/dashboard.component.html
+++ b/src-ui/src/app/components/dashboard/dashboard.component.html
@@ -9,7 +9,7 @@
     <ng-container *ngFor="let v of savedDashboardViews">
       <h4>{{v.viewConfig.title}}</h4>
 
-      <table class="table table-striped table-sm">
+      <table class="table table-sm table-hover table-borderless">
         <thead>
         <tr>
           <th>Date created</th>
diff --git a/src-ui/src/app/components/document-list/document-list.component.html b/src-ui/src/app/components/document-list/document-list.component.html
index 1455cabbc..9d275854e 100644
--- a/src-ui/src/app/components/document-list/document-list.component.html
+++ b/src-ui/src/app/components/document-list/document-list.component.html
@@ -81,7 +81,7 @@
   </app-document-card-large>
 </div>
 
-<table class="table table-striped table-sm" *ngIf="displayMode == 'details'">
+<table class="table table-hover table-sm" *ngIf="displayMode == 'details'">
   <thead>
     <th>ASN</th>
     <th>Correspondent</th>

From 296c113b16a6a9fbb5209fef1ee119bbcdf2526a Mon Sep 17 00:00:00 2001
From: Jonas Winkler <jonas.winkler@jpwinkler.de>
Date: Fri, 6 Nov 2020 14:46:06 +0100
Subject: [PATCH 5/6] updated the classifier. Its now much faster and does not
 retrain when data hasnt changed.

---
 src/documents/classifier.py                   | 140 ++++++++++--------
 src/documents/consumer.py                     |   9 +-
 .../commands/document_create_classifier.py    |  26 +++-
 .../management/commands/document_retagger.py  |   9 +-
 4 files changed, 109 insertions(+), 75 deletions(-)

diff --git a/src/documents/classifier.py b/src/documents/classifier.py
index 851a75899..6c90536b0 100755
--- a/src/documents/classifier.py
+++ b/src/documents/classifier.py
@@ -1,61 +1,74 @@
+import hashlib
 import logging
 import os
 import pickle
+import re
+import time
 
 from sklearn.feature_extraction.text import CountVectorizer
 from sklearn.neural_network import MLPClassifier
-from sklearn.preprocessing import MultiLabelBinarizer, LabelBinarizer
+from sklearn.preprocessing import MultiLabelBinarizer
 
 from documents.models import Document, MatchingModel
 from paperless import settings
 
 
+class IncompatibleClassifierVersionError(Exception):
+    pass
+
+
+logger = logging.getLogger(__name__)
+
+
 def preprocess_content(content):
-    content = content.lower()
-    content = content.strip()
-    content = content.replace("\n", " ")
-    content = content.replace("\r", " ")
-    while content.find("  ") > -1:
-        content = content.replace("  ", " ")
+    content = content.lower().strip()
+    content = re.sub(r"\s+", " ", content)
     return content
 
 
 class DocumentClassifier(object):
 
+    FORMAT_VERSION = 5
+
     def __init__(self):
+        # mtime of the model file on disk. used to prevent reloading when nothing has changed.
         self.classifier_version = 0
 
+        # hash of the training data. used to prevent re-training when the training data has not changed.
+        self.data_hash = None
+
         self.data_vectorizer = None
-
         self.tags_binarizer = None
-        self.correspondent_binarizer = None
-        self.document_type_binarizer = None
-
         self.tags_classifier = None
         self.correspondent_classifier = None
         self.document_type_classifier = None
 
     def reload(self):
         if os.path.getmtime(settings.MODEL_FILE) > self.classifier_version:
-            logging.getLogger(__name__).info("Reloading classifier models")
             with open(settings.MODEL_FILE, "rb") as f:
-                self.data_vectorizer = pickle.load(f)
-                self.tags_binarizer = pickle.load(f)
-                self.correspondent_binarizer = pickle.load(f)
-                self.document_type_binarizer = pickle.load(f)
+                schema_version = pickle.load(f)
 
-                self.tags_classifier = pickle.load(f)
-                self.correspondent_classifier = pickle.load(f)
-                self.document_type_classifier = pickle.load(f)
+                if schema_version != self.FORMAT_VERSION:
+                    raise IncompatibleClassifierVersionError("Cannor load classifier, incompatible versions.")
+                else:
+                    if self.classifier_version > 0:
+                        logger.info("Classifier updated on disk, reloading classifier models")
+                    self.data_hash = pickle.load(f)
+                    self.data_vectorizer = pickle.load(f)
+                    self.tags_binarizer = pickle.load(f)
+
+                    self.tags_classifier = pickle.load(f)
+                    self.correspondent_classifier = pickle.load(f)
+                    self.document_type_classifier = pickle.load(f)
             self.classifier_version = os.path.getmtime(settings.MODEL_FILE)
 
     def save_classifier(self):
         with open(settings.MODEL_FILE, "wb") as f:
+            pickle.dump(self.FORMAT_VERSION, f) # Version
+            pickle.dump(self.data_hash, f)
             pickle.dump(self.data_vectorizer, f)
 
             pickle.dump(self.tags_binarizer, f)
-            pickle.dump(self.correspondent_binarizer, f)
-            pickle.dump(self.document_type_binarizer, f)
 
             pickle.dump(self.tags_classifier, f)
             pickle.dump(self.correspondent_classifier, f)
@@ -68,109 +81,121 @@ class DocumentClassifier(object):
         labels_document_type = list()
 
         # Step 1: Extract and preprocess training data from the database.
-        logging.getLogger(__name__).info("Gathering data from database...")
-        for doc in Document.objects.exclude(tags__is_inbox_tag=True):
-            data.append(preprocess_content(doc.content))
+        logging.getLogger(__name__).debug("Gathering data from database...")
+        m = hashlib.sha1()
+        for doc in Document.objects.order_by('pk').exclude(tags__is_inbox_tag=True):
+            preprocessed_content = preprocess_content(doc.content)
+            m.update(preprocessed_content.encode('utf-8'))
+            data.append(preprocessed_content)
 
             y = -1
             if doc.document_type:
                 if doc.document_type.matching_algorithm == MatchingModel.MATCH_AUTO:
                     y = doc.document_type.pk
+            m.update(y.to_bytes(4, 'little', signed=True))
             labels_document_type.append(y)
 
             y = -1
             if doc.correspondent:
                 if doc.correspondent.matching_algorithm == MatchingModel.MATCH_AUTO:
                     y = doc.correspondent.pk
+            m.update(y.to_bytes(4, 'little', signed=True))
             labels_correspondent.append(y)
 
             tags = [tag.pk for tag in doc.tags.filter(
                 matching_algorithm=MatchingModel.MATCH_AUTO
             )]
+            m.update(bytearray(tags))
             labels_tags.append(tags)
 
         if not data:
             raise ValueError("No training data available.")
 
+        new_data_hash = m.digest()
+
+        if self.data_hash and new_data_hash == self.data_hash:
+            return False
+
         labels_tags_unique = set([tag for tags in labels_tags for tag in tags])
-        logging.getLogger(__name__).info(
+
+        num_tags = len(labels_tags_unique)
+        # substract 1 since -1 (null) is also part of the classes.
+        num_correspondents = len(labels_correspondent) - 1
+        num_document_types = len(labels_document_type) - 1
+
+        logging.getLogger(__name__).debug(
             "{} documents, {} tag(s), {} correspondent(s), "
             "{} document type(s).".format(
                 len(data),
-                len(labels_tags_unique),
-                len(set(labels_correspondent)),
-                len(set(labels_document_type))
+                num_tags,
+                num_correspondents,
+                num_document_types
             )
         )
 
         # Step 2: vectorize data
-        logging.getLogger(__name__).info("Vectorizing data...")
+        logging.getLogger(__name__).debug("Vectorizing data...")
         self.data_vectorizer = CountVectorizer(
-            analyzer="char",
-            ngram_range=(3, 5),
-            min_df=0.1
+            analyzer="word",
+            ngram_range=(1,2),
+            min_df=0.01
         )
         data_vectorized = self.data_vectorizer.fit_transform(data)
 
         self.tags_binarizer = MultiLabelBinarizer()
         labels_tags_vectorized = self.tags_binarizer.fit_transform(labels_tags)
 
-        self.correspondent_binarizer = LabelBinarizer()
-        labels_correspondent_vectorized = \
-            self.correspondent_binarizer.fit_transform(labels_correspondent)
-
-        self.document_type_binarizer = LabelBinarizer()
-        labels_document_type_vectorized = \
-            self.document_type_binarizer.fit_transform(labels_document_type)
-
         # Step 3: train the classifiers
-        if len(self.tags_binarizer.classes_) > 0:
-            logging.getLogger(__name__).info("Training tags classifier...")
-            self.tags_classifier = MLPClassifier(verbose=True)
+        if num_tags > 0:
+            logging.getLogger(__name__).debug("Training tags classifier...")
+            self.tags_classifier = MLPClassifier(verbose=True, tol=0.01)
             self.tags_classifier.fit(data_vectorized, labels_tags_vectorized)
         else:
             self.tags_classifier = None
-            logging.getLogger(__name__).info(
+            logging.getLogger(__name__).debug(
                 "There are no tags. Not training tags classifier."
             )
 
-        if len(self.correspondent_binarizer.classes_) > 0:
-            logging.getLogger(__name__).info(
+        if num_correspondents > 0:
+            logging.getLogger(__name__).debug(
                 "Training correspondent classifier..."
             )
-            self.correspondent_classifier = MLPClassifier(verbose=True)
+            self.correspondent_classifier = MLPClassifier(verbose=True, tol=0.01)
             self.correspondent_classifier.fit(
                 data_vectorized,
-                labels_correspondent_vectorized
+                labels_correspondent
             )
         else:
             self.correspondent_classifier = None
-            logging.getLogger(__name__).info(
+            logging.getLogger(__name__).debug(
                 "There are no correspondents. Not training correspondent "
                 "classifier."
             )
 
-        if len(self.document_type_binarizer.classes_) > 0:
-            logging.getLogger(__name__).info(
+        if num_document_types > 0:
+            logging.getLogger(__name__).debug(
                 "Training document type classifier..."
             )
-            self.document_type_classifier = MLPClassifier(verbose=True)
+            self.document_type_classifier = MLPClassifier(verbose=True, tol=0.01)
             self.document_type_classifier.fit(
                 data_vectorized,
-                labels_document_type_vectorized
+                labels_document_type
             )
         else:
             self.document_type_classifier = None
-            logging.getLogger(__name__).info(
+            logging.getLogger(__name__).debug(
                 "There are no document types. Not training document type "
                 "classifier."
             )
 
+        self.data_hash = new_data_hash
+
+        return True
+
     def predict_correspondent(self, content):
         if self.correspondent_classifier:
             X = self.data_vectorizer.transform([preprocess_content(content)])
-            y = self.correspondent_classifier.predict(X)
-            correspondent_id = self.correspondent_binarizer.inverse_transform(y)[0]
+            correspondent_id = self.correspondent_classifier.predict(X)
             if correspondent_id != -1:
                 return correspondent_id
             else:
@@ -181,8 +206,7 @@ class DocumentClassifier(object):
     def predict_document_type(self, content):
         if self.document_type_classifier:
             X = self.data_vectorizer.transform([preprocess_content(content)])
-            y = self.document_type_classifier.predict(X)
-            document_type_id = self.document_type_binarizer.inverse_transform(y)[0]
+            document_type_id = self.document_type_classifier.predict(X)
             if document_type_id != -1:
                 return document_type_id
             else:
diff --git a/src/documents/consumer.py b/src/documents/consumer.py
index 3920f2942..f61d11136 100755
--- a/src/documents/consumer.py
+++ b/src/documents/consumer.py
@@ -10,7 +10,7 @@ from django.db import transaction
 from django.utils import timezone
 
 from paperless.db import GnuPG
-from .classifier import DocumentClassifier
+from .classifier import DocumentClassifier, IncompatibleClassifierVersionError
 from .models import Document, FileInfo
 from .parsers import ParseError, get_parser_class
 from .signals import (
@@ -133,11 +133,8 @@ class Consumer:
             try:
                 self.classifier.reload()
                 classifier = self.classifier
-            except FileNotFoundError:
-                self.log("warning", "Cannot classify documents, classifier "
-                                    "model file was not found. Consider "
-                                    "running python manage.py "
-                                    "document_create_classifier.")
+            except (FileNotFoundError, IncompatibleClassifierVersionError) as e:
+                logging.getLogger(__name__).warning("Cannot classify documents: {}.".format(e))
 
             document_consumption_finished.send(
                 sender=self.__class__,
diff --git a/src/documents/management/commands/document_create_classifier.py b/src/documents/management/commands/document_create_classifier.py
index 9b8f28615..85cb3b446 100755
--- a/src/documents/management/commands/document_create_classifier.py
+++ b/src/documents/management/commands/document_create_classifier.py
@@ -1,7 +1,8 @@
 import logging
 
 from django.core.management.base import BaseCommand
-from documents.classifier import DocumentClassifier
+from documents.classifier import DocumentClassifier, \
+    IncompatibleClassifierVersionError
 from paperless import settings
 from ...mixins import Renderable
 
@@ -18,12 +19,25 @@ class Command(Renderable, BaseCommand):
 
     def handle(self, *args, **options):
         classifier = DocumentClassifier()
+
         try:
-            classifier.train()
-            logging.getLogger(__name__).info(
-                "Saving models to {}...".format(settings.MODEL_FILE)
-            )
-            classifier.save_classifier()
+            # load the classifier, since we might not have to train it again.
+            classifier.reload()
+        except (FileNotFoundError, IncompatibleClassifierVersionError):
+            # This is what we're going to fix here.
+            pass
+
+        try:
+            if classifier.train():
+                logging.getLogger(__name__).info(
+                    "Saving updated classifier model to {}...".format(settings.MODEL_FILE)
+                )
+                classifier.save_classifier()
+            else:
+                logging.getLogger(__name__).debug(
+                    "Training data unchanged."
+                )
+
         except Exception as e:
             logging.getLogger(__name__).error(
                 "Classifier error: " + str(e)
diff --git a/src/documents/management/commands/document_retagger.py b/src/documents/management/commands/document_retagger.py
index 9238bea71..e48b8802c 100755
--- a/src/documents/management/commands/document_retagger.py
+++ b/src/documents/management/commands/document_retagger.py
@@ -2,7 +2,8 @@ import logging
 
 from django.core.management.base import BaseCommand
 
-from documents.classifier import DocumentClassifier
+from documents.classifier import DocumentClassifier, \
+    IncompatibleClassifierVersionError
 from documents.models import Document
 from ...mixins import Renderable
 from ...signals.handlers import set_correspondent, set_document_type, set_tags
@@ -72,10 +73,8 @@ class Command(Renderable, BaseCommand):
         classifier = DocumentClassifier()
         try:
             classifier.reload()
-        except FileNotFoundError:
-            logging.getLogger(__name__).warning("Cannot classify documents, "
-                                              "classifier model file was not "
-                                              "found.")
+        except (FileNotFoundError, IncompatibleClassifierVersionError) as e:
+            logging.getLogger(__name__).warning("Cannot classify documents: {}.".format(e))
             classifier = None
 
         for document in documents:

From e45208bf0186b1e5df464c6450104080fd1124e1 Mon Sep 17 00:00:00 2001
From: Jonas Winkler <jonas.winkler@jpwinkler.de>
Date: Fri, 6 Nov 2020 17:27:21 +0100
Subject: [PATCH 6/6] add correspondent to the index

---
 src/documents/index.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/documents/index.py b/src/documents/index.py
index eb1a9c739..3fa8bd353 100644
--- a/src/documents/index.py
+++ b/src/documents/index.py
@@ -55,7 +55,8 @@ def get_schema():
     return Schema(
         id=NUMERIC(stored=True, unique=True, numtype=int),
         title=TEXT(stored=True),
-        content=TEXT()
+        content=TEXT(),
+        correspondent=TEXT(stored=True)
     )
 
 
@@ -71,7 +72,8 @@ def update_document(writer, doc):
     writer.update_document(
         id=doc.pk,
         title=doc.title,
-        content=doc.content
+        content=doc.content,
+        correspondent=doc.correspondent.name if doc.correspondent else None
     )