Implemented the classifier model, including automatic tagging of new documents

src/documents/classifier.py

@@ -0,0 +1,67 @@
+import pickle
+
+from documents.models import Correspondent, DocumentType, Tag
+from paperless import settings
+
+
+def preprocess_content(content):
+    content = content.lower()
+    content = content.strip()
+    content = content.replace("\n", " ")
+    content = content.replace("\r", " ")
+    while content.find("  ") > -1:
+        content = content.replace("  ", " ")
+    return content
+
+
+class DocumentClassifier(object):
+
+    @staticmethod
+    def load_classifier():
+        clf = DocumentClassifier()
+        clf.reload()
+        return clf
+
+    def reload(self):
+        with open(settings.MODEL_FILE, "rb") as f:
+            self.data_vectorizer = pickle.load(f)
+            self.tags_binarizer = pickle.load(f)
+            self.correspondent_binarizer = pickle.load(f)
+            self.type_binarizer = pickle.load(f)
+
+            self.tags_classifier = pickle.load(f)
+            self.correspondent_classifier = pickle.load(f)
+            self.type_classifier = pickle.load(f)
+
+    def save_classifier(self):
+        with open(settings.MODEL_FILE, "wb") as f:
+            pickle.dump(self.data_vectorizer, f)
+
+            pickle.dump(self.tags_binarizer, f)
+            pickle.dump(self.correspondent_binarizer, f)
+            pickle.dump(self.type_binarizer, f)
+
+            pickle.dump(self.tags_classifier, f)
+            pickle.dump(self.correspondent_classifier, f)
+            pickle.dump(self.type_classifier, f)
+
+    def classify_document(self, document, classify_correspondent=False, classify_type=False, classify_tags=False):
+        X = self.data_vectorizer.transform([preprocess_content(document.content)])
+
+        if classify_correspondent:
+            y_correspondent = self.correspondent_classifier.predict(X)
+            correspondent = self.correspondent_binarizer.inverse_transform(y_correspondent)[0]
+            print("Detected correspondent:", correspondent)
+            document.correspondent = Correspondent.objects.filter(name=correspondent).first()
+
+        if classify_type:
+            y_type = self.type_classifier.predict(X)
+            type = self.type_binarizer.inverse_transform(y_type)[0]
+            print("Detected document type:", type)
+            document.type = DocumentType.objects.filter(name=type).first()
+
+        if classify_tags:
+            y_tags = self.tags_classifier.predict(X)
+            tags = self.tags_binarizer.inverse_transform(y_tags)[0]
+            print("Detected tags:", tags)
+            document.tags.add(*[Tag.objects.filter(name=t).first() for t in tags])