Merge branch 'master' into dev

src/documents/admin.py

@@ -11,6 +11,7 @@ from django.urls import reverse
 from django.utils.html import format_html, format_html_join
 from django.utils.http import urlquote
 from django.utils.safestring import mark_safe
+from djangoql.admin import DjangoQLSearchMixin
 
 from documents.actions import (
     add_tag_to_selected,
@@ -186,7 +187,7 @@ class DocumentTypeAdmin(CommonAdmin):
     document_count.admin_order_field = "document_count"
 
 
-class DocumentAdmin(CommonAdmin):
+class DocumentAdmin(DjangoQLSearchMixin, CommonAdmin):
 
     class Media:
         css = {

src/documents/mail.py

@@ -216,7 +216,11 @@ class MailFetcher(Loggable):
         return r
 
     def _connect(self):
-        self._connection = imaplib.IMAP4_SSL(self._host, self._port)
+        try:
+            self._connection = imaplib.IMAP4_SSL(self._host, self._port)
+        except OSError as e:
+            msg = "Problem connecting to {}: {}".format(self._host, e.strerror)
+            raise MailFetcherError(msg)
 
     def _login(self):
 

src/documents/management/commands/change_storage_type.py

@@ -75,7 +75,8 @@ class Command(BaseCommand):
 
         for document in encrypted_files:
 
-            print(coloured("Decrypting {}".format(document), "green"))
+            print(coloured("Decrypting {}".format(
+                document).encode('utf-8'), "green"))
 
             old_paths = [document.source_path, document.thumbnail_path]
             raw_document = GnuPG.decrypted(document.source_file, passphrase)

src/documents/serialisers.py

@@ -7,14 +7,24 @@ class CorrespondentSerializer(serializers.HyperlinkedModelSerializer):
 
     class Meta:
         model = Correspondent
-        fields = ("id", "slug", "name", "automatic_classification")
+        fields = (
+            "id",
+            "slug",
+            "name",
+            "automatic_classification"
+        )
 
 
 class DocumentTypeSerializer(serializers.HyperlinkedModelSerializer):
 
     class Meta:
         model = DocumentType
-        fields = ("id", "slug", "name", "automatic_classification")
+        fields = (
+            "id",
+            "slug",
+            "name",
+            "automatic_classification"
+        )
 
 
 class TagSerializer(serializers.HyperlinkedModelSerializer):
@@ -22,7 +32,12 @@ class TagSerializer(serializers.HyperlinkedModelSerializer):
     class Meta:
         model = Tag
         fields = (
-            "id", "slug", "name", "colour", "automatic_classification")
+            "id",
+            "slug",
+            "name",
+            "colour",
+            "automatic_classification"
+        )
 
 
 class CorrespondentField(serializers.HyperlinkedRelatedField):

src/documents/views.py

@@ -2,6 +2,7 @@ from django.http import HttpResponse, HttpResponseBadRequest
 from django.views.generic import DetailView, FormView, TemplateView
 from django_filters.rest_framework import DjangoFilterBackend
 from django.conf import settings
+from django.utils import cache
 
 from paperless.db import GnuPG
 from paperless.mixins import SessionOrBasicAuthMixin
@@ -63,10 +64,12 @@ class FetchView(SessionOrBasicAuthMixin, DetailView):
         }
 
         if self.kwargs["kind"] == "thumb":
-            return HttpResponse(
+            response = HttpResponse(
                 self._get_raw_data(self.object.thumbnail_file),
                 content_type=content_types[Document.TYPE_PNG]
             )
+            cache.patch_cache_control(response, max_age=31536000, private=True)
+            return response
 
         response = HttpResponse(
             self._get_raw_data(self.object.source_file),

src/paperless/settings.py

@@ -83,6 +83,7 @@ INSTALLED_APPS = [
     "rest_framework",
     "crispy_forms",
     "django_filters",
+    "djangoql",
 
 ]
 
@@ -270,6 +271,9 @@ CONVERT_TMPDIR = os.getenv("PAPERLESS_CONVERT_TMPDIR")
 CONVERT_MEMORY_LIMIT = os.getenv("PAPERLESS_CONVERT_MEMORY_LIMIT")
 CONVERT_DENSITY = os.getenv("PAPERLESS_CONVERT_DENSITY")
 
+# Ghostscript
+GS_BINARY = os.getenv("PAPERLESS_GS_BINARY", "gs")
+
 # OptiPNG
 OPTIPNG_BINARY = os.getenv("PAPERLESS_OPTIPNG_BINARY", "optipng")
 

src/paperless_tesseract/parsers.py

@@ -29,6 +29,7 @@ class RasterisedDocumentParser(DocumentParser):
     """
 
     CONVERT = settings.CONVERT_BINARY
+    GHOSTSCRIPT = settings.GS_BINARY
     DENSITY = settings.CONVERT_DENSITY if settings.CONVERT_DENSITY else 300
     THREADS = int(settings.OCR_THREADS) if settings.OCR_THREADS else None
     UNPAPER = settings.UNPAPER_BINARY
@@ -47,13 +48,40 @@ class RasterisedDocumentParser(DocumentParser):
         out_path = os.path.join(self.tempdir, "convert.png")
 
         # Run convert to get a decent thumbnail
-        run_convert(
-            self.CONVERT,
-            "-scale", "500x5000",
-            "-alpha", "remove",
-            "{}[0]".format(self.document_path),
-            out_path
-        )
+        try:
+            run_convert(
+                self.CONVERT,
+                "-scale", "500x5000",
+                "-alpha", "remove",
+                "-strip", "-trim",
+                "{}[0]".format(self.document_path),
+                out_path
+            )
+        except ParseError:
+            # if convert fails, fall back to extracting
+            # the first PDF page as a PNG using Ghostscript
+            self.log(
+                "warning",
+                "Thumbnail generation with ImageMagick failed, "
+                "falling back to Ghostscript."
+            )
+            gs_out_path = os.path.join(self.tempdir, "gs_out.png")
+            cmd = [self.GHOSTSCRIPT,
+                   "-q",
+                   "-sDEVICE=pngalpha",
+                   "-o", gs_out_path,
+                   self.document_path]
+            if not subprocess.Popen(cmd).wait() == 0:
+                raise ParseError("Thumbnail (gs) failed at {}".format(cmd))
+            # then run convert on the output from gs
+            run_convert(
+                self.CONVERT,
+                "-scale", "500x5000",
+                "-alpha", "remove",
+                "-strip", "-trim",
+                gs_out_path,
+                out_path
+            )
 
         return out_path
 

src/tox.ini

@@ -17,6 +17,5 @@ deps=pycodestyle
 
 [testenv:doc]
 deps =
-  -r{toxinidir}/../requirements.txt
-  sphinx
+  -r {toxinidir}/../requirements.txt
 commands=sphinx-build -b html ../docs ../docs/_build -W