Merge pull request #190 from danielquinn/feature/new-look

A nicer look for the documents listing

docs/changelog.rst

@@ -1,9 +1,34 @@
 Changelog
 #########
 
+* 0.3.5
+  * A serious facelift for the documents listing page wherein we drop the
+    tabular layout in favour of a tiled interface.
+  * Users can now configure the number of items per page.
+  * Fix for `#171`_: Allow users to specify their own ``SECRET_KEY`` value.
+  * Moved the dotenv loading to the top of settings.py
+  * Fix for `#112`_: Added checks for binaries required for document
+    consumption.
+
+* 0.3.4
+  * Removal of django-suit due to a licensing conflict I bumped into in 0.3.3.
+    Note that you *can* use Django Suit with Paperless, but only in a
+    non-profit situation as their free license prohibits for-profit use.  As a
+    result, I can't bundle Suit with Paperless without conflicting with the
+    GPL.  Further development will be done against the stock Django admin.
+  * I shrunk the thumbnails a little 'cause they were too big for me, even on
+    my high-DPI monitor.
+  * BasicAuth support for document and thumbnail downloads, as well as the Push
+    API thanks to @thomasbrueggemann.  See `#179`_.
+
+* 0.3.3
+  * Thumbnails in the UI and a Django-suit -based face-lift courtesy of @ekw!
+  * Timezone, items per page, and default language are now all configurable,
+    also thanks to @ekw.
+
 * 0.3.2
-  * Fix for #172: defaulting ALLOWED_HOSTS to ``["*"]`` and allowing the user
-    to set her own value via ``PAPERLESS_ALLOWED_HOSTS`` should the need
+  * Fix for `#172`_: defaulting ALLOWED_HOSTS to ``["*"]`` and allowing the
+    user to set her own value via ``PAPERLESS_ALLOWED_HOSTS`` should the need
     arise.
 
 * 0.3.1
@@ -26,7 +51,8 @@ Changelog
     ``paperless.conf``.
   * `#148`_: The database location (sqlite) is now a variable you can set in
     ``paperless.conf``.
-  * `#146`_: Fixed a bug that allowed unauthorised access to the `/fetch` URL.
+  * `#146`_: Fixed a bug that allowed unauthorised access to the ``/fetch``
+    URL.
   * `#131`_: Document files are now automatically removed from disk when
     they're deleted in Paperless.
   * `#121`_: Fixed a bug where Paperless wasn't setting document creation time
@@ -152,8 +178,12 @@ Changelog
 .. _#89: https://github.com/danielquinn/paperless/issues/89
 .. _#94: https://github.com/danielquinn/paperless/issues/94
 .. _#98: https://github.com/danielquinn/paperless/issues/98
+.. _#112: https://github.com/danielquinn/paperless/issues/112
 .. _#121: https://github.com/danielquinn/paperless/issues/121
 .. _#131: https://github.com/danielquinn/paperless/issues/131
 .. _#146: https://github.com/danielquinn/paperless/issues/146
 .. _#148: https://github.com/danielquinn/paperless/pull/148
 .. _#150: https://github.com/danielquinn/paperless/pull/150
+.. _#171: https://github.com/danielquinn/paperless/issues/171
+.. _#172: https://github.com/danielquinn/paperless/issues/172
+.. _#179: https://github.com/danielquinn/paperless/pull/179

paperless.conf.example

@@ -65,8 +65,8 @@ PAPERLESS_SHARED_SECRET=""
 # cases it has proven useful to configure a lesser value.
 # This setting has a high impact on the physical size of tmp page files,
 # the speed of document conversion, and can affect the accuracy of OCR
-# results. Individual results can vary and this setting should be tested 
-# thoroughly against the documents you are importing to see if it has any 
+# results. Individual results can vary and this setting should be tested
+# thoroughly against the documents you are importing to see if it has any
 # impacts either negative or positive. Testing on limited document sets has
 # shown a setting of 200 can cut the size of tmp files by 1/3, and speed up
 # conversion by up to 4x with little impact to OCR accuracy.
@@ -81,13 +81,17 @@ PAPERLESS_SHARED_SECRET=""
 # the web for "MAGICK_TMPDIR".
 #PAPERLESS_CONVERT_TMPDIR=/var/tmp/paperless
 
-# You can specify where you want the SQLite database to be stored instead of 
+# You can specify where you want the SQLite database to be stored instead of
 # the default location
 #PAPERLESS_DBDIR=/path/to/database/file
 
 # Override the default MEDIA_ROOT here.  This is where all files are stored.
 #PAPERLESS_MEDIADIR=/path/to/media
 
+# Override the default STATIC_ROOT here. This is where all static files created
+# using "collectstatic" manager command are stored.
+#PAPERLESS_STATICDIR=""
+
 # The number of seconds that Paperless will wait between checking
 # PAPERLESS_CONSUMPTION_DIR.  If you tend to write documents to this directory
 # very slowly, you may want to use a higher value than the default (10).
@@ -95,8 +99,29 @@ PAPERLESS_SHARED_SECRET=""
 
 # If you're planning on putting Paperless on the open internet, then you
 # really should set this value to the domain name you're using.  Failing to do
-# so leaves you open to XSS attacks.
+# so leaves you open to HTTP host header attacks:
+# https://docs.djangoproject.com/en/1.10/topics/security/#host-headers-virtual-hosting
+#
 # Just remember that this is a comma-separated list, so "example.com" is fine,
 # as is "example.com,www.example.com", but NOT " example.com" or "example.com,"
 #PAPERLESS_ALLOWED_HOSTS="example.com,www.example.com"
 
+# Override the default UTC time zone here
+#PAPERLESS_TIME_ZONE=UTC
+
+# Customize number of list items to show per page
+#PAPERLESS_LIST_PER_PAGE=50
+
+# Customize the default language that tesseract will attempt to use when parsing
+# documents.  It should be a 3-letter language code consistent with ISO 639.
+#PAPERLESS_OCR_LANGUAGE=eng
+
+# The number of items on each page in the web UI.  This value must be a
+# positive integer, but if you don't define one in paperless.conf, a default of
+# 100 will be used.
+#PAPERLESS_LIST_PER_PAGE=100
+
+# The secret key has a default that should be fine so long as you're hosting
+# Paperless on a closed network.  However, if you're putting this anywhere
+# public, you should change the key to something unique and verbose.
+#PAPERLESS_SECRET_KEY="change-me"

requirements.txt

@@ -1,16 +1,17 @@
-Django==1.10.4
+Django==1.10.5
 Pillow>=3.1.1
-django-crispy-forms>=1.6.0
-django-extensions>=1.6.1
+django-crispy-forms>=1.6.1
+django-extensions>=1.7.6
 django-filter>=1.0
-djangorestframework>=3.4.4
+django-flat-responsive>=1.2.0
+djangorestframework>=3.5.3
 filemagic>=1.6
-langdetect>=1.0.5
-pyocr>=0.3.1
-python-dateutil>=2.4.2
-python-dotenv>=0.3.0
-python-gnupg>=0.3.8
-pytz>=2015.7
+langdetect>=1.0.7
+pyocr>=0.4.6
+python-dateutil>=2.6.0
+python-dotenv>=0.6.2
+python-gnupg>=0.3.9
+pytz>=2016.10
 gunicorn==19.6.0
 
 # For the tests

src/documents/admin.py

@@ -1,3 +1,4 @@
+from django.conf import settings
 from django.contrib import admin
 from django.contrib.auth.models import User, Group
 from django.core.urlresolvers import reverse
@@ -31,21 +32,25 @@ class MonthListFilter(admin.SimpleListFilter):
         return queryset.filter(created__year=year, created__month=month)
 
 
-class CorrespondentAdmin(admin.ModelAdmin):
+class CommonAdmin(admin.ModelAdmin):
+    list_per_page = settings.PAPERLESS_LIST_PER_PAGE
+
+
+class CorrespondentAdmin(CommonAdmin):
 
     list_display = ("name", "match", "matching_algorithm")
     list_filter = ("matching_algorithm",)
     list_editable = ("match", "matching_algorithm")
 
 
-class TagAdmin(admin.ModelAdmin):
+class TagAdmin(CommonAdmin):
 
     list_display = ("name", "colour", "match", "matching_algorithm")
     list_filter = ("colour", "matching_algorithm")
     list_editable = ("colour", "match", "matching_algorithm")
 
 
-class DocumentAdmin(admin.ModelAdmin):
+class DocumentAdmin(CommonAdmin):
 
     class Media:
         css = {
@@ -53,13 +58,24 @@ class DocumentAdmin(admin.ModelAdmin):
         }
 
     search_fields = ("correspondent__name", "title", "content")
-    list_display = ("created", "correspondent", "title", "tags_", "document")
+    list_display = ("title", "created", "thumbnail", "correspondent", "tags_")
     list_filter = ("tags", "correspondent", MonthListFilter)
-    list_per_page = 25
+    ordering = ["-created", "correspondent"]
 
     def created_(self, obj):
         return obj.created.date().strftime("%Y-%m-%d")
 
+    def thumbnail(self, obj):
+        png_img = self._html_tag(
+            "img",
+            src="/fetch/thumb/{}".format(obj.id),
+            width=180,
+            alt="thumbnail",
+            title=obj.file_name
+        )
+        return self._html_tag("a", png_img, href=obj.download_url)
+    thumbnail.allow_tags = True
+
     def tags_(self, obj):
         r = ""
         for tag in obj.tags.all():
@@ -108,7 +124,7 @@ class DocumentAdmin(admin.ModelAdmin):
         return "<{} {}/>".format(kind, " ".join(attributes))
 
 
-class LogAdmin(admin.ModelAdmin):
+class LogAdmin(CommonAdmin):
 
     list_display = ("created", "message", "level",)
     list_filter = ("level", "created",)

src/documents/mixins.py

@@ -1,3 +1,8 @@
+from django.contrib.auth.mixins import AccessMixin
+from django.contrib.auth import authenticate, login
+import base64
+
+
 class Renderable(object):
     """
     A handy mixin to make it easier/cleaner to print output based on a
@@ -7,3 +12,46 @@ class Renderable(object):
     def _render(self, text, verbosity):
         if self.verbosity >= verbosity:
             print(text)
+
+
+class SessionOrBasicAuthMixin(AccessMixin):
+    """
+    Session or Basic Authentication mixin for Django.
+    It determines if the requester is already logged in or if they have
+    provided proper http-authorization and returning the view if all goes
+    well, otherwise responding with a 401.
+
+    Base for mixin found here: https://djangosnippets.org/snippets/3073/
+    """
+
+    def dispatch(self, request, *args, **kwargs):
+
+        # check if user is authenticated via the session
+        if request.user.is_authenticated:
+
+            # Already logged in, just return the view.
+            return super(SessionOrBasicAuthMixin, self).dispatch(
+                request, *args, **kwargs
+            )
+
+        # apparently not authenticated via session, maybe via HTTP Basic?
+        if 'HTTP_AUTHORIZATION' in request.META:
+            auth = request.META['HTTP_AUTHORIZATION'].split()
+            if len(auth) == 2:
+                # NOTE: Support for only basic authentication
+                if auth[0].lower() == "basic":
+                    authString = base64.b64decode(auth[1]).decode('utf-8')
+                    uname, passwd = authString.split(':')
+                    user = authenticate(username=uname, password=passwd)
+                    if user is not None:
+                        if user.is_active:
+                            login(request, user)
+                            request.user = user
+                            return super(
+                                SessionOrBasicAuthMixin, self
+                            ).dispatch(
+                                request, *args, **kwargs
+                            )
+
+        # nope, really not authenticated
+        return self.handle_no_permission()

src/documents/templates/admin/change_list_results.html

@@ -0,0 +1,6 @@
+{% load hacks %}
+
+{# See documents.templatetags.hacks.change_list_results for an explanation #}
+
+{% change_list_results %}
+

src/documents/templates/admin/documents/document/change_list_results.html

@@ -0,0 +1,167 @@
+{% load i18n %}
+
+<style>
+  .grid *, .grid *:after, .grid *:before {
+    -webkit-box-sizing: border-box;
+    -moz-box-sizing: border-box;
+    box-sizing: border-box;
+  }
+  .box {
+    width: 12.5%;
+    padding: 1em;
+    float: left;
+    opacity: 0.7;
+    transition: all 0.5s;
+  }
+  .box:hover {
+    opacity: 1;
+    transition: all 0.5s;
+  }
+  .box:last-of-type {
+    padding-right: 0;
+  }
+  .result {
+    border: 1px solid #cccccc;
+    border-radius: 2%;
+    overflow: hidden;
+    height: 300px;
+  }
+  .result .header {
+    padding: 5px;
+    background-color: #79AEC8;
+    height: 6em;
+  }
+  .result .header .checkbox {
+    margin-right: 5px;
+  }
+  .result .header .checkbox{
+    width: 5%;
+    float: left;
+  }
+  .result .header .info {
+    width: 90%;
+    float: left;
+  }
+  .result .header a,
+  .result a.tag {
+    color: #ffffff;
+  }
+  .result .date {
+    padding: 5px;
+  }
+  .result .tags {
+    float: left;
+  }
+  .result .tags a.tag {
+    padding: 2px 5px;
+    border-radius: 2px;
+    display: inline-block;
+    margin: 2px;
+  }
+  .result .date {
+    float: right;
+    color: #cccccc;
+  }
+  .result .image img {
+    width: 100%;
+  }
+
+  .grid {
+    margin-right: 260px;
+  }
+  .grid:after {
+    content: "";
+    display: table;
+    clear: both;
+  }
+
+  @media (max-width: 1600px) {
+    .box {
+      width: 25%
+    }
+  }
+
+  @media (max-width: 991px) {
+    .grid {
+      margin-right: 220px;
+    }
+    .box {
+      width: 50%
+    }
+  }
+
+  @media (max-width: 767px) {
+    .grid {
+      margin-right: 0;
+    }
+  }
+
+  @media (max-width: 500px) {
+    .box {
+      width: 100%
+    }
+  }
+
+</style>
+
+
+{# This is just copypasta from the parent change_list_results.html file #}
+<table id="result_list">
+<thead>
+<tr>
+{% for header in result_headers %}
+<th scope="col" {{ header.class_attrib }}>
+   {% if header.sortable %}
+     {% if header.sort_priority > 0 %}
+       <div class="sortoptions">
+         <a class="sortremove" href="{{ header.url_remove }}" title="{% trans "Remove from sorting" %}"></a>
+         {% if num_sorted_fields > 1 %}<span class="sortpriority" title="{% blocktrans with priority_number=header.sort_priority %}Sorting priority: {{ priority_number }}{% endblocktrans %}">{{ header.sort_priority }}</span>{% endif %}
+         <a href="{{ header.url_toggle }}" class="toggle {% if header.ascending %}ascending{% else %}descending{% endif %}" title="{% trans "Toggle sorting" %}"></a>
+       </div>
+     {% endif %}
+   {% endif %}
+   <div class="text">{% if header.sortable %}<a href="{{ header.url_primary }}">{{ header.text|capfirst }}</a>{% else %}<span>{{ header.text|capfirst }}</span>{% endif %}</div>
+   <div class="clear"></div>
+</th>{% endfor %}
+</tr>
+</thead>
+</table>
+{# /copypasta #}
+
+
+<div class="grid">
+  {% for result in results %}
+    {# 0: Checkbox #}
+    {# 1: Title #}
+    {# 2: Date #}
+    {# 3: Image #}
+    {# 4: Correspondent #}
+    {# 5: Tags #}
+    <div class="box">
+      <div class="result">
+        <div class="header">
+          <div class="checkbox">{{ result.0 }}</div>
+          <div class="info">
+            {{ result.4 }}<br />
+            {{ result.1 }}
+          </div>
+          <div style="clear: both;"></div>
+        </div>
+        <div class="tags">{{ result.5 }}</div>
+        <div class="date">{{ result.2 }}</div>
+        <div style="clear: both;"></div>
+        <div class="image">{{ result.3 }}</div>
+      </div>
+    </div>
+  {% endfor %}
+</div>
+
+
+<script>
+  // We nee to re-build the select-all functionality as the old logic pointed
+  // to a table and we're using divs now.
+  django.jQuery("#action-toggle").on("change", function(){
+    django.jQuery(".grid .box .result .checkbox input")
+      .prop("checked", this.checked);
+  });
+</script>

src/documents/templatetags/hacks.py

@@ -0,0 +1,41 @@
+import os
+
+from django.contrib import admin
+from django.template import Library
+from django.template.loader import get_template
+
+from ..models import Document
+
+
+register = Library()
+
+
+@register.simple_tag(takes_context=True)
+def change_list_results(context):
+    """
+    Django has a lot of places where you can override defaults, but
+    unfortunately, `change_list_results.html` is not one of them.  In fact,
+    it's a downright pain in the ass to override this file on a per-model basis
+    and this is the cleanest way I could come up with.
+
+    Basically all we've done here is defined `change_list_results.html` in an
+    `admin` directory which globally overrides that file for *every* model.
+    That template however simply loads this templatetag which determines
+    whether we're currently looking at a `Document` listing or something else
+    and loads the appropriate file in each case.
+
+    Better work arounds for this are welcome as I hate this myself, but at the
+    moment, it's all I could come up with.
+    """
+
+    path = os.path.join(
+        os.path.dirname(admin.__file__),
+        "templates",
+        "admin",
+        "change_list_results.html"
+    )
+
+    if context["cl"].model == Document:
+        path = "admin/documents/document/change_list_results.html"
+
+    return get_template(path).render(context)

src/documents/views.py

@@ -1,4 +1,3 @@
-from django.contrib.auth.mixins import LoginRequiredMixin
 from django.http import HttpResponse
 from django.views.decorators.csrf import csrf_exempt
 from django.views.generic import DetailView, FormView, TemplateView
@@ -28,6 +27,7 @@ from .serialisers import (
     LogSerializer,
     TagSerializer
 )
+from .mixins import SessionOrBasicAuthMixin
 
 
 class IndexView(TemplateView):
@@ -41,7 +41,7 @@ class IndexView(TemplateView):
         return TemplateView.get_context_data(self, **kwargs)
 
 
-class FetchView(LoginRequiredMixin, DetailView):
+class FetchView(SessionOrBasicAuthMixin, DetailView):
 
     model = Document
 
@@ -74,7 +74,7 @@ class FetchView(LoginRequiredMixin, DetailView):
         return response
 
 
-class PushView(LoginRequiredMixin, FormView):
+class PushView(SessionOrBasicAuthMixin, FormView):
     """
     A crude REST-ish API for creating documents.
     """

src/paperless/__init__.py

@@ -1 +1 @@
-from .checks import paths_check
+from .checks import paths_check, binaries_check

src/paperless/checks.py

@@ -1,10 +1,15 @@
 import os
+import shutil
 
+from django.conf import settings
 from django.core.checks import Error, register, Warning
 
 
 @register()
 def paths_check(app_configs, **kwargs):
+    """
+    Check the various paths for existence, readability and writeability
+    """
 
     check_messages = []
 
@@ -44,4 +49,38 @@ def paths_check(app_configs, **kwargs):
                     writeable_hint.format(directory)
                 ))
 
+    directory = os.getenv("PAPERLESS_STATICDIR")
+    if directory:
+        if not os.path.exists(directory):
+            check_messages.append(Error(
+                exists_message.format("PAPERLESS_STATICDIR"),
+                exists_hint.format(directory)
+            ))
+        if not check_messages:
+            if not os.access(directory, os.W_OK | os.X_OK):
+                check_messages.append(Error(
+                    writeable_message.format("PAPERLESS_STATICDIR"),
+                    writeable_hint.format(directory)
+                ))
+
+    return check_messages
+
+
+@register()
+def binaries_check(app_configs, **kwargs):
+    """
+    Paperless requires the existence of a few binaries, so we do some checks
+    for those here.
+    """
+
+    error = "Paperless can't find {}. Without it, consumption is impossible."
+    hint = "Either it's not in your ${PATH} or it's not installed."
+
+    binaries = (settings.CONVERT_BINARY, settings.UNPAPER_BINARY, "tesseract")
+
+    check_messages = []
+    for binary in binaries:
+        if shutil.which(binary) is None:
+            check_messages.append(Warning(error.format(binary), hint))
+
     return check_messages

src/paperless/settings.py

@@ -14,6 +14,12 @@ import os
 
 from dotenv import load_dotenv
 
+
+# Tap paperless.conf if it's available
+if os.path.exists("/etc/paperless.conf"):
+    load_dotenv("/etc/paperless.conf")
+
+
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
@@ -21,8 +27,14 @@ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 # Quick-start development settings - unsuitable for production
 # See https://docs.djangoproject.com/en/1.9/howto/deployment/checklist/
 
-# SECURITY WARNING: keep the secret key used in production secret!
-SECRET_KEY = 'e11fl1oa-*ytql8p)(06fbj4ukrlo+n7k&q5+$1md7i+mge=ee'
+# The secret key has a default that should be fine so long as you're hosting
+# Paperless on a closed network.  However, if you're putting this anywhere
+# public, you should change the key to something unique and verbose.
+SECRET_KEY = os.getenv(
+    "PAPERLESS_SECRET_KEY",
+    "e11fl1oa-*ytql8p)(06fbj4ukrlo+n7k&q5+$1md7i+mge=ee"
+)
+
 
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = True
@@ -32,29 +44,27 @@ LOGIN_URL = '/admin/login'
 ALLOWED_HOSTS = ["*"]
 
 _allowed_hosts = os.getenv("PAPERLESS_ALLOWED_HOSTS")
-if allowed_hosts:
+if _allowed_hosts:
     ALLOWED_HOSTS = _allowed_hosts.split(",")
 
-# Tap paperless.conf if it's available
-if os.path.exists("/etc/paperless.conf"):
-    load_dotenv("/etc/paperless.conf")
-
 
 # Application definition
 
 INSTALLED_APPS = [
 
-    'django.contrib.admin',
-    'django.contrib.auth',
-    'django.contrib.contenttypes',
-    'django.contrib.sessions',
-    'django.contrib.messages',
-    'django.contrib.staticfiles',
+    "django.contrib.auth",
+    "django.contrib.contenttypes",
+    "django.contrib.sessions",
+    "django.contrib.messages",
+    "django.contrib.staticfiles",
 
     "django_extensions",
 
     "documents.apps.DocumentsConfig",
 
+    "flat_responsive",
+    "django.contrib.admin",
+
     "rest_framework",
     "crispy_forms",
 
@@ -141,7 +151,7 @@ AUTH_PASSWORD_VALIDATORS = [
 
 LANGUAGE_CODE = 'en-us'
 
-TIME_ZONE = 'UTC'
+TIME_ZONE = os.getenv("PAPERLESS_TIME_ZONE", "UTC")
 
 USE_I18N = True
 
@@ -153,7 +163,8 @@ USE_TZ = True
 # Static files (CSS, JavaScript, Images)
 # https://docs.djangoproject.com/en/1.9/howto/static-files/
 
-STATIC_ROOT = os.path.join(BASE_DIR, "..", "static")
+STATIC_ROOT = os.getenv(
+    "PAPERLESS_STATICDIR", os.path.join(BASE_DIR, "..", "static"))
 MEDIA_ROOT = os.getenv(
     "PAPERLESS_MEDIADIR", os.path.join(BASE_DIR, "..", "media"))
 
@@ -187,7 +198,7 @@ LOGGING = {
 
 # The default language that tesseract will attempt to use when parsing
 # documents.  It should be a 3-letter language code consistent with ISO 639.
-OCR_LANGUAGE = "eng"
+OCR_LANGUAGE = os.getenv("PAPERLESS_OCR_LANGUAGE", "eng")
 
 # The amount of threads to use for OCR
 OCR_THREADS = os.getenv("PAPERLESS_OCR_THREADS")
@@ -249,3 +260,8 @@ SHARED_SECRET = os.getenv("PAPERLESS_SHARED_SECRET", "")
 # Trigger a script after every successful document consumption?
 PRE_CONSUME_SCRIPT = os.getenv("PAPERLESS_PRE_CONSUME_SCRIPT")
 POST_CONSUME_SCRIPT = os.getenv("PAPERLESS_POST_CONSUME_SCRIPT")
+
+# The number of items on each page in the web UI.  This value must be a
+# positive integer, but if you don't define one in paperless.conf, a default of
+# 100 will be used.
+PAPERLESS_LIST_PER_PAGE = int(os.getenv("PAPERLESS_LIST_PER_PAGE", 100))

src/paperless/version.py

@@ -1 +1 @@
-__version__ = (0, 3, 2)
+__version__ = (0, 3, 5)