From 7887892e4e7494d02bbad76efaa7bf3a20765228 Mon Sep 17 00:00:00 2001
From: Jonas Winkler <dev@jpwinkler.de>
Date: Thu, 13 Sep 2018 15:19:25 +0200
Subject: [PATCH 01/47] Added a bunch of new features: - Debug mode is now
 configurable in the configuration file. This way, we don't have to edit
 versioned files to disable it on production systems. - Recent correspondents
 filter (enable in configuration file) - Document actions: Edit tags and
 correspondents on multiple documents at once - Replaced month list filter
 with date drilldown - Sortable document count columns on Tag and
 Correspondent admin - Last correspondence column on Correspondent admin -
 Save and edit next functionality for document editing

---
 .gitignore                                    |   2 +
 paperless.conf.example                        |  10 ++
 requirements.txt                              |   3 +
 src/documents/actions.py                      | 108 +++++++++++++
 src/documents/admin.py                        | 142 +++++++++++++-----
 .../admin/documents/document/change_form.html |  18 ++-
 .../documents/document/select_object.html     |  46 ++++++
 src/paperless/settings.py                     |  14 +-
 8 files changed, 301 insertions(+), 42 deletions(-)
 mode change 100644 => 100755 requirements.txt
 create mode 100644 src/documents/actions.py
 create mode 100644 src/documents/templates/admin/documents/document/select_object.html

diff --git a/.gitignore b/.gitignore
index a16f958a3..439d9df4b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -81,3 +81,5 @@ docker-compose.env
 scripts/import-for-development
 scripts/nuke
 
+# Static files collected by the collectstatic command
+static/
diff --git a/paperless.conf.example b/paperless.conf.example
index 15498a26a..05cf81724 100644
--- a/paperless.conf.example
+++ b/paperless.conf.example
@@ -59,6 +59,11 @@ PAPERLESS_EMAIL_SECRET=""
 ####                              Security                                 ####
 ###############################################################################
 
+# Controls whether django's debug mode is enabled. Disable this on production
+# systems. Debug mode is enabled by default.
+PAPERLESS_DEBUG="false"
+
+
 # Paperless can be instructed to attempt to encrypt your PDF files with GPG
 # using the PAPERLESS_PASSPHRASE specified below.  If however you're not
 # concerned about encrypting these files (for example if you have disk
@@ -203,3 +208,8 @@ PAPERLESS_EMAIL_SECRET=""
 # positive integer, but if you don't define one in paperless.conf, a default of
 # 100 will be used.
 #PAPERLESS_LIST_PER_PAGE=100
+
+
+# The number of years for which a correspondent will be included in the recent
+# correspondents filter.
+#PAPERLESS_RECENT_CORRESPONDENT_YEARS=1
diff --git a/requirements.txt b/requirements.txt
old mode 100644
new mode 100755
index 0476efef1..89c7e296f
--- a/requirements.txt
+++ b/requirements.txt
@@ -24,6 +24,7 @@ idna==2.7
 inotify-simple==1.1.8
 langdetect==1.0.7
 more-itertools==4.3.0
+numpy==1.15.1
 pdftotext==2.1.0
 pillow==5.2.0
 pluggy==0.7.1; python_version != '3.1.*'
@@ -45,6 +46,8 @@ pytz==2018.5
 regex==2018.8.29
 requests==2.19.1
 six==1.11.0
+scikit-learn==0.19.2
+scipy==1.1.0
 termcolor==1.1.0
 text-unidecode==1.2
 tzlocal==1.5.1
diff --git a/src/documents/actions.py b/src/documents/actions.py
new file mode 100644
index 000000000..3db5cd314
--- /dev/null
+++ b/src/documents/actions.py
@@ -0,0 +1,108 @@
+from django.contrib import messages
+from django.contrib.admin import helpers
+from django.contrib.admin.utils import model_ngettext
+from django.core.exceptions import PermissionDenied
+from django.template.response import TemplateResponse
+
+from documents.models import Tag, Correspondent
+
+
+def select_action(modeladmin, request, queryset, title, action, modelclass, success_message="", document_action=None, queryset_action=None):
+    opts = modeladmin.model._meta
+    app_label = opts.app_label
+
+    if not modeladmin.has_change_permission(request):
+        raise PermissionDenied
+
+    if request.POST.get('post'):
+        n = queryset.count()
+        selected_object = modelclass.objects.get(id=request.POST.get('obj_id'))
+        if n:
+            for document in queryset:
+                if document_action:
+                    document_action(document, selected_object)
+                document_display = str(document)
+                modeladmin.log_change(request, document, document_display)
+            if queryset_action:
+                queryset_action(queryset, selected_object)
+
+            modeladmin.message_user(request, success_message % {
+                "selected_object": selected_object.name, "count": n, "items": model_ngettext(modeladmin.opts, n)
+            }, messages.SUCCESS)
+
+        # Return None to display the change list page again.
+        return None
+
+    context = dict(
+        modeladmin.admin_site.each_context(request),
+        title=title,
+        queryset=queryset,
+        opts=opts,
+        action_checkbox_name=helpers.ACTION_CHECKBOX_NAME,
+        media=modeladmin.media,
+        action=action,
+        objects=modelclass.objects.all(),
+        itemname=model_ngettext(modelclass, 1)
+    )
+
+    request.current_app = modeladmin.admin_site.name
+
+    return TemplateResponse(request, "admin/%s/%s/select_object.html" % (app_label, opts.model_name), context)
+
+
+def simple_action(modeladmin, request, queryset, success_message="", document_action=None, queryset_action=None):
+    if not modeladmin.has_change_permission(request):
+        raise PermissionDenied
+
+    n = queryset.count()
+    if n:
+        for document in queryset:
+            if document_action:
+                document_action(document)
+            document_display = str(document)
+            modeladmin.log_change(request, document, document_display)
+        if queryset_action:
+            queryset_action(queryset)
+        modeladmin.message_user(request, success_message % {
+            "count": n, "items": model_ngettext(modeladmin.opts, n)
+        }, messages.SUCCESS)
+
+    # Return None to display the change list page again.
+    return None
+
+
+def add_tag_to_selected(modeladmin, request, queryset):
+    return select_action(modeladmin=modeladmin, request=request, queryset=queryset,
+                         title="Add tag to multiple documents",
+                         action="add_tag_to_selected",
+                         modelclass=Tag,
+                         success_message="Successfully added tag %(selected_object)s to %(count)d %(items)s.",
+                         document_action=lambda doc, tag: doc.tags.add(tag))
+add_tag_to_selected.short_description = "Add tag to selected documents"
+
+
+def remove_tag_from_selected(modeladmin, request, queryset):
+    return select_action(modeladmin=modeladmin, request=request, queryset=queryset,
+                         title="Remove tag from multiple documents",
+                         action="remove_tag_from_selected",
+                         modelclass=Tag,
+                         success_message="Successfully removed tag %(selected_object)s from %(count)d %(items)s.",
+                         document_action=lambda doc, tag: doc.tags.remove(tag))
+remove_tag_from_selected.short_description = "Remove tag from selected documents"
+
+
+def set_correspondent_on_selected(modeladmin, request, queryset):
+    return select_action(modeladmin=modeladmin, request=request, queryset=queryset,
+                         title="Set correspondent on multiple documents",
+                         action="set_correspondent_on_selected",
+                         modelclass=Correspondent,
+                         success_message="Successfully set correspondent %(selected_object)s on %(count)d %(items)s.",
+                         queryset_action=lambda qs, correspondent: qs.update(correspondent=correspondent))
+set_correspondent_on_selected.short_description = "Set correspondent on selected documents"
+
+
+def remove_correspondent_from_selected(modeladmin, request, queryset):
+    return simple_action(modeladmin=modeladmin, request=request, queryset=queryset,
+                         success_message="Successfully removed correspondent from %(count)d %(items)s.",
+                         queryset_action=lambda qs: qs.update(correspondent=None))
+remove_correspondent_from_selected.short_description = "Remove correspondent from selected documents"
diff --git a/src/documents/admin.py b/src/documents/admin.py
index 36154b6ba..d545c1c02 100644
--- a/src/documents/admin.py
+++ b/src/documents/admin.py
@@ -1,44 +1,25 @@
-from datetime import datetime
+from datetime import datetime, timedelta
 
 from django.conf import settings
-from django.contrib import admin
+from django.contrib import admin, messages
+from django.contrib.admin.templatetags.admin_urls import add_preserved_filters
 from django.contrib.auth.models import User, Group
+from django.http import HttpResponseRedirect
 try:
     from django.core.urlresolvers import reverse
 except ImportError:
     from django.urls import reverse
 from django.templatetags.static import static
-from django.utils.safestring import mark_safe
 from django.utils.html import format_html, format_html_join
+from django.utils.http import urlquote
+from django.utils.safestring import mark_safe
+from django.db import models
 
+from documents.actions import add_tag_to_selected, remove_tag_from_selected, set_correspondent_on_selected, \
+    remove_correspondent_from_selected
 from .models import Correspondent, Tag, Document, Log
 
 
-class MonthListFilter(admin.SimpleListFilter):
-
-    title = "Month"
-
-    # Parameter for the filter that will be used in the URL query.
-    parameter_name = "month"
-
-    def lookups(self, request, model_admin):
-        r = []
-        for document in Document.objects.all():
-            r.append((
-                document.created.strftime("%Y-%m"),
-                document.created.strftime("%B %Y")
-            ))
-        return sorted(set(r), key=lambda x: x[0], reverse=True)
-
-    def queryset(self, request, queryset):
-
-        if not self.value():
-            return None
-
-        year, month = self.value().split("-")
-        return queryset.filter(created__year=year, created__month=month)
-
-
 class FinancialYearFilter(admin.SimpleListFilter):
 
     title = "Financial Year"
@@ -104,18 +85,43 @@ class FinancialYearFilter(admin.SimpleListFilter):
                                created__lte=self._fy_end(end))
 
 
+class RecentCorrespondentFilter(admin.RelatedFieldListFilter):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.title = "correspondent (recent)"
+
+    def field_choices(self, field, request, model_admin):
+        lookups = []
+        if settings.PAPERLESS_RECENT_CORRESPONDENT_YEARS and settings.PAPERLESS_RECENT_CORRESPONDENT_YEARS > 0:
+            date_limit = datetime.now() - timedelta(days=365*settings.PAPERLESS_RECENT_CORRESPONDENT_YEARS)
+            for c in Correspondent.objects.filter(documents__created__gte=date_limit).distinct():
+                lookups.append((c.id, c.name))
+        return lookups
+
+
 class CommonAdmin(admin.ModelAdmin):
     list_per_page = settings.PAPERLESS_LIST_PER_PAGE
 
 
 class CorrespondentAdmin(CommonAdmin):
 
-    list_display = ("name", "match", "matching_algorithm", "document_count")
+    list_display = ("name", "match", "matching_algorithm", "document_count", "last_correspondence")
     list_filter = ("matching_algorithm",)
     list_editable = ("match", "matching_algorithm")
 
+    def get_queryset(self, request):
+        qs = super(CorrespondentAdmin, self).get_queryset(request)
+        qs = qs.annotate(document_count=models.Count("documents"), last_correspondence=models.Max("documents__created"))
+        return qs
+
     def document_count(self, obj):
-        return obj.documents.count()
+        return obj.document_count
+    document_count.admin_order_field = "document_count"
+
+    def last_correspondence(self, obj):
+        return obj.last_correspondence
+    last_correspondence.admin_order_field = "last_correspondence"
 
 
 class TagAdmin(CommonAdmin):
@@ -125,8 +131,14 @@ class TagAdmin(CommonAdmin):
     list_filter = ("colour", "matching_algorithm")
     list_editable = ("colour", "match", "matching_algorithm")
 
+    def get_queryset(self, request):
+        qs = super(TagAdmin, self).get_queryset(request)
+        qs = qs.annotate(document_count=models.Count("documents"))
+        return qs
+
     def document_count(self, obj):
-        return obj.documents.count()
+        return obj.document_count
+    document_count.admin_order_field = "document_count"
 
 
 class DocumentAdmin(CommonAdmin):
@@ -140,12 +152,18 @@ class DocumentAdmin(CommonAdmin):
     readonly_fields = ("added",)
     list_display = ("title", "created", "added", "thumbnail", "correspondent",
                     "tags_")
-    list_filter = ("tags", "correspondent", FinancialYearFilter,
-                   MonthListFilter)
+    list_filter = ("tags", ('correspondent', RecentCorrespondentFilter), "correspondent", FinancialYearFilter)
+
     filter_horizontal = ("tags",)
 
     ordering = ["-created", "correspondent"]
 
+    actions = [add_tag_to_selected, remove_tag_from_selected, set_correspondent_on_selected, remove_correspondent_from_selected]
+
+    date_hierarchy = 'created'
+
+    document_queue = None
+
     def has_add_permission(self, request):
         return False
 
@@ -153,6 +171,56 @@ class DocumentAdmin(CommonAdmin):
         return obj.created.date().strftime("%Y-%m-%d")
     created_.short_description = "Created"
 
+    def changelist_view(self, request, extra_context=None):
+        response = super().changelist_view(request, extra_context)
+
+        if request.method == 'GET':
+            cl = self.get_changelist_instance(request)
+            self.document_queue = [doc.id for doc in cl.queryset]
+
+        return response
+
+    def change_view(self, request, object_id=None, form_url='', extra_context=None):
+        extra_context = extra_context or {}
+        doc = Document.objects.get(id=object_id)
+        if self.document_queue and object_id and int(object_id) in self.document_queue:
+            # There is a queue of documents
+            current_index = self.document_queue.index(int(object_id))
+            if current_index < len(self.document_queue) - 1:
+                # ... and there are still documents in the queue
+                extra_context['next_object'] = self.document_queue[current_index + 1]
+        return super(DocumentAdmin, self).change_view(
+            request, object_id, form_url, extra_context=extra_context,
+        )
+
+    def response_change(self, request, obj):
+
+        # This is mostly copied from ModelAdmin.response_change()
+        opts = self.model._meta
+        preserved_filters = self.get_preserved_filters(request)
+
+        msg_dict = {
+            'name': opts.verbose_name,
+            'obj': format_html('<a href="{}">{}</a>', urlquote(request.path), obj),
+        }
+        if "_saveandeditnext" in request.POST:
+            msg = format_html(
+                'The {name} "{obj}" was changed successfully. Editing next object.',
+                **msg_dict
+            )
+            self.message_user(request, msg, messages.SUCCESS)
+            redirect_url = reverse('admin:%s_%s_change' %
+                                   (opts.app_label, opts.model_name),
+                                   args=(request.POST['_next_object'],),
+                                   current_app=self.admin_site.name)
+            redirect_url = add_preserved_filters({'preserved_filters': preserved_filters, 'opts': opts}, redirect_url)
+            response = HttpResponseRedirect(redirect_url)
+        else:
+            response = super().response_change(request, obj)
+
+        return response
+
+    @mark_safe
     def thumbnail(self, obj):
         return self._html_tag(
             "a",
@@ -165,8 +233,8 @@ class DocumentAdmin(CommonAdmin):
             ),
             href=obj.download_url
         )
-    thumbnail.allow_tags = True
 
+    @mark_safe
     def tags_(self, obj):
         r = ""
         for tag in obj.tags.all():
@@ -183,10 +251,11 @@ class DocumentAdmin(CommonAdmin):
                     )
                 }
             )
-        return mark_safe(r)
-    tags_.allow_tags = True
+        return r
 
+    @mark_safe
     def document(self, obj):
+        # TODO: is this method even used anymore?
         return self._html_tag(
             "a",
             self._html_tag(
@@ -199,7 +268,6 @@ class DocumentAdmin(CommonAdmin):
             ),
             href=obj.download_url
         )
-    document.allow_tags = True
 
     @staticmethod
     def _html_tag(kind, inside=None, **kwargs):
diff --git a/src/documents/templates/admin/documents/document/change_form.html b/src/documents/templates/admin/documents/document/change_form.html
index 7bd0e483f..88fae955d 100644
--- a/src/documents/templates/admin/documents/document/change_form.html
+++ b/src/documents/templates/admin/documents/document/change_form.html
@@ -1,5 +1,21 @@
 {% extends 'admin/change_form.html' %}
 
+{% block content %}
+
+{{ block.super }}
+
+{% if next_object %}
+	<script type="text/javascript">//<![CDATA[
+		(function($){
+			$('<input type="submit" value="Save and edit next" name="_saveandeditnext" />')
+			.prependTo('div.submit-row');
+			$('<input type="hidden" value="{{next_object}}" name="_next_object" />')
+			.prependTo('div.submit-row');
+		})(django.jQuery);
+	//]]></script>
+{% endif %}
+
+{% endblock content %}
 
 {% block footer %}
 
@@ -10,4 +26,4 @@
 		django.jQuery(".field-created input").first().attr("type", "date")
 	</script>
 
-{% endblock footer %}
\ No newline at end of file
+{% endblock footer %}
diff --git a/src/documents/templates/admin/documents/document/select_object.html b/src/documents/templates/admin/documents/document/select_object.html
new file mode 100644
index 000000000..1439b5c21
--- /dev/null
+++ b/src/documents/templates/admin/documents/document/select_object.html
@@ -0,0 +1,46 @@
+{% extends "admin/base_site.html" %}
+{% load i18n l10n admin_urls static %}
+{% load staticfiles %}
+
+{% block extrahead %}
+{{ block.super }}
+{{ media }}
+<script type="text/javascript" src="{% static 'admin/js/cancel.js' %}"></script>
+
+{% endblock %}
+
+{% block bodyclass %}{{ block.super }} app-{{ opts.app_label }} model-{{ opts.model_name }} delete-confirmation delete-selected-confirmation{% endblock %}
+
+{% block breadcrumbs %}
+<div class="breadcrumbs">
+    <a href="{% url 'admin:index' %}">{% trans 'Home' %}</a>
+    &rsaquo; <a href="{% url 'admin:app_list' app_label=opts.app_label %}">{{ opts.app_config.verbose_name }}</a>
+    &rsaquo; <a href="{% url opts|admin_urlname:'changelist' %}">{{ opts.verbose_name_plural|capfirst }}</a>
+    &rsaquo; {{title}}
+</div>
+{% endblock %}
+
+{% block content %}
+<p>Please select the {{itemname}}.</p>
+<form method="post">{% csrf_token %}
+    <div>
+        {% for obj in queryset %}
+        <input type="hidden" name="{{ action_checkbox_name }}" value="{{ obj.pk|unlocalize }}"/>
+        {% endfor %}
+        <p>
+            <select name="obj_id">
+                {% for obj in objects %}
+                <option value="{{obj.id}}">{{obj.name}}</option>
+                {% endfor %}
+            </select>
+        </p>
+
+        <input type="hidden" name="action" value="{{action}}"/>
+        <input type="hidden" name="post" value="yes"/>
+        <p>
+            <input type="submit" value="{% trans "Confirm" %}" />
+            <a href="#" class="button cancel-link">{% trans "Go back" %}</a>
+        </p>
+    </div>
+</form>
+{% endblock %}
diff --git a/src/paperless/settings.py b/src/paperless/settings.py
index 956b90a7f..e6f3da0cb 100644
--- a/src/paperless/settings.py
+++ b/src/paperless/settings.py
@@ -22,12 +22,12 @@ elif os.path.exists("/usr/local/etc/paperless.conf"):
     load_dotenv("/usr/local/etc/paperless.conf")
 
 
-def __get_boolean(key):
+def __get_boolean(key, default="NO"):
     """
     Return a boolean value based on whatever the user has supplied in the
     environment based on whether the value "looks like" it's True or not.
     """
-    return bool(os.getenv(key, "NO").lower() in ("yes", "y", "1", "t", "true"))
+    return bool(os.getenv(key, default).lower() in ("yes", "y", "1", "t", "true"))
 
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
@@ -47,7 +47,7 @@ SECRET_KEY = os.getenv(
 
 
 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = True
+DEBUG = __get_boolean("PAPERLESS_DEBUG", "YES")
 
 LOGIN_URL = "admin:login"
 
@@ -81,7 +81,7 @@ INSTALLED_APPS = [
 
     "rest_framework",
     "crispy_forms",
-    "django_filters",
+    "django_filters"
 
 ]
 
@@ -292,3 +292,9 @@ FY_END = os.getenv("PAPERLESS_FINANCIAL_YEAR_END")
 
 # Specify the default date order (for autodetected dates)
 DATE_ORDER = os.getenv("PAPERLESS_DATE_ORDER", "DMY")
+
+# Specify for how many years a correspondent is considered recent. Recent
+# correspondents will be shown in a separate "Recent correspondents" filter as
+# well. Set to 0 to disable this filter.
+PAPERLESS_RECENT_CORRESPONDENT_YEARS = int(os.getenv(
+    "PAPERLESS_RECENT_CORRESPONDENT_YEARS", 0))

From 77fc3caad41fd15dc65a4c4779c052f80f7f2b27 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sat, 22 Sep 2018 13:59:50 +0100
Subject: [PATCH 02/47] Clean up release notes

---
 docs/changelog.rst | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 804447855..9396493a7 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -15,7 +15,8 @@ Changelog
 * As his last bit of effort on this release, Joshua also added some code to
   allow you to view the documents inline rather than download them as an
   attachment. `#400`_
-* Finally, `ahyear`_ found a slip in the Docker documentation and patched it. `#401`_
+* Finally, `ahyear`_ found a slip in the Docker documentation and patched it.
+  `#401`_
 
 
 2.2.1
@@ -32,14 +33,14 @@ Changelog
   version of Paperless that supports Django 2.0!  As a result of their hard
   work, you can now also run Paperless on Python 3.7 as well: `#386`_ &
   `#390`_.
-* `Stéphane Brunner`_ added a few lines of code that made tagging interface a lot
-  easier on those of us with lots of different tags: `#391`_.
+* `Stéphane Brunner`_ added a few lines of code that made tagging interface a
+  lot easier on those of us with lots of different tags: `#391`_.
 * `Kilian Koeltzsch`_ noticed a bug in how we capture & automatically create
   tags, so that's fixed now too: `#384`_.
 * `erikarvstedt`_ tweaked the behaviour of the test suite to be better behaved
   for packaging environments: `#383`_.
-* `Lukasz Soluch`_ added CORS support to make building a new Javascript-based front-end
-  cleaner & easier: `#387`_.
+* `Lukasz Soluch`_ added CORS support to make building a new Javascript-based
+  front-end cleaner & easier: `#387`_.
 
 
 2.1.0

From 37c6e68dae9ba25d2375fec2a9b856a213ff91b7 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sat, 22 Sep 2018 14:00:00 +0100
Subject: [PATCH 03/47] Add an example for pdf2pdfocr with the pre-consume hook

---
 docs/consumption.rst | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/docs/consumption.rst b/docs/consumption.rst
index bf62ed0a2..fabaf2641 100644
--- a/docs/consumption.rst
+++ b/docs/consumption.rst
@@ -76,6 +76,29 @@ Pre-consumption script
 
 * Document file name
 
+A simple but common example for this would be creating a simple script like
+this:
+
+.. code:: bash
+    :name: "/usr/local/bin/ocr-pdf"
+
+    #!/usr/bin/env bash
+    pdf2pdfocr.py -i ${1}
+
+.. code:: bash
+    :name: /etc/paperless.conf
+
+    ...
+    PAPERLESS_PRE_CONSUME_SCRIPT="/usr/local/bin/ocr-pdf"
+    ...
+
+This will pass the path to the document about to be consumed to ``/usr/local/bin/ocr-pdf``,
+which will in turn call `pdf2pdfocr.py`_ on your document, which will then
+overwrite the file with an OCR'd version of the file and exit.  At which point,
+the consumption process will begin with the newly modified file.
+
+.. _pdf2pdfocr.py: https://github.com/LeoFCardoso/pdf2pdfocr
+
 
 .. _consumption-director-hook-variables-post:
 

From af9ec47d9b786a7456ca14a109feb90c42f9421f Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sat, 22 Sep 2018 15:27:22 +0100
Subject: [PATCH 04/47] Reduce duplication in docker-compose.env.example

See #404 for more info on where this came from.
---
 docker-compose.env.example | 44 ++++++++++++--------------------------
 1 file changed, 14 insertions(+), 30 deletions(-)

diff --git a/docker-compose.env.example b/docker-compose.env.example
index 3c1664573..51332437d 100644
--- a/docker-compose.env.example
+++ b/docker-compose.env.example
@@ -1,38 +1,22 @@
 # Environment variables to set for Paperless
-# Commented out variables will be replaced by a default within Paperless.
+# Commented out variables will be replaced with a default within Paperless.
+#
+# In addition to what you see here, you can also define any values you find in
+# paperless.conf.example here.  Values like:
+#
+# * PAPERLESS_PASSPHRASE
+# * PAPERLESS_CONSUMPTION_DIR
+# * PAPERLESS_CONSUME_MAIL_HOST
+#
+# ...are all explained in that file but can be defined here, since the Docker
+# installation doesn't make use of paperless.conf.
 
-# Passphrase Paperless uses to encrypt and decrypt your documents, if you want
-# encryption at all.
-# PAPERLESS_PASSPHRASE=CHANGE_ME
 
-# The amount of threads to use for text recognition
-# PAPERLESS_OCR_THREADS=4
-
-# Additional languages to install for text recognition
+# Additional languages to install for text recognition.  Note that this is
+# different from PAPERLESS_OCR_LANGUAGE (default=eng), which defines the
+# default language used when guessing the language from the OCR output.
 # PAPERLESS_OCR_LANGUAGES=deu ita
 
 # You can change the default user and group id to a custom one
 # USERMAP_UID=1000
 # USERMAP_GID=1000
-
-###############################################################################
-####                         Mail Consumption                              ####
-###############################################################################
-
-# These values are required if you want paperless to check a particular email
-# box every 10 minutes and attempt to consume documents from there.  If you
-# don't define a HOST, mail checking will just be disabled.
-# Don't use quotes after = or it will crash your docker
-# PAPERLESS_CONSUME_MAIL_HOST=
-# PAPERLESS_CONSUME_MAIL_PORT=
-# PAPERLESS_CONSUME_MAIL_USER=
-# PAPERLESS_CONSUME_MAIL_PASS=
-
-# Override the default IMAP inbox here. If it's not set, Paperless defaults to
-# INBOX.
-# PAPERLESS_CONSUME_MAIL_INBOX=INBOX
-
-# Any email sent to the target account that does not contain this text will be
-# ignored.  Mail checking won't work without this.
-# PAPERLESS_EMAIL_SECRET=
-

From 0e2d89bb3ffcdd64344ff0caaa940ad735fecec7 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sat, 22 Sep 2018 16:17:18 +0100
Subject: [PATCH 05/47] Make the names of the sample files visible

---
 docs/consumption.rst | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/docs/consumption.rst b/docs/consumption.rst
index fabaf2641..15f6c6393 100644
--- a/docs/consumption.rst
+++ b/docs/consumption.rst
@@ -79,14 +79,16 @@ Pre-consumption script
 A simple but common example for this would be creating a simple script like
 this:
 
+``/usr/local/bin/ocr-pdf``
+
 .. code:: bash
-    :name: "/usr/local/bin/ocr-pdf"
 
     #!/usr/bin/env bash
     pdf2pdfocr.py -i ${1}
 
+``/etc/paperless.conf``
+
 .. code:: bash
-    :name: /etc/paperless.conf
 
     ...
     PAPERLESS_PRE_CONSUME_SCRIPT="/usr/local/bin/ocr-pdf"

From b8fc7e6b1277983a367e6191328626ff95fe5790 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sat, 22 Sep 2018 16:22:03 +0100
Subject: [PATCH 06/47] Add a contribution guide

---
 docs/contributing.rst | 113 ++++++++++++++++++++++++++++++++++++++++++
 docs/index.rst        |   1 +
 2 files changed, 114 insertions(+)
 create mode 100644 docs/contributing.rst

diff --git a/docs/contributing.rst b/docs/contributing.rst
new file mode 100644
index 000000000..4ee6d18d5
--- /dev/null
+++ b/docs/contributing.rst
@@ -0,0 +1,113 @@
+.. _contributing:
+
+Contributing to Paperless
+#########################
+
+Maybe you've been using Paperless for a while and want to add a feature or two,
+or maybe you've come across a bug that you have some ideas how to solve.  The
+beauty of Free software is that you can see what's wrong and help to get it
+fixed for everyone!
+
+
+How to Get Your Changes Rolled Into Paperless
+=============================================
+
+If you've found a bug, but don't know how to fix it, you can always post an
+issue on `GitHub`_ in the hopes that someone will have the time to fix it for
+you.  If however you're the one with the time, pull requests are always
+welcome, you just have to make sure that your code conforms to a few standards:
+
+Pep8
+----
+
+It's the standard for all Python development, so it's `very well documented`_.
+The version is:
+
+* Lines should wrap at 79 characters
+* Use ``snake_case`` for variables, ``CamelCase`` for classes, and ``ALL_CAPS``
+  for constants.
+* Space out your operators: ``stuff + 7`` instead of ``stuff+7``
+* Two empty lines between classes, and functions, but 1 empty line between
+  class methods.
+
+There's more to it than that, but if you follow those, you'll probably be
+alright.  When you submit your pull request, there's a pep8 checker that'll
+look at your code to see if anything is off.  If it finds anything, it'll
+complain at you until you fix it.
+
+
+Additional Style Guides
+-----------------------
+
+Where pep8 is ambiguous, I've tried to be a little more specific.  These rules
+aren't hard-and-fast, but if you can conform to them, I'll appreciate it and
+spend less time trying to conform your PR before merging:
+
+
+Function calls
+..............
+
+If you're calling a function and that necessitates more than one line of code,
+please format it like this:
+
+.. code:: python
+
+    my_function(
+        argument1,
+        kwarg1="x",
+        kwarg2="y"
+        another_really_long_kwarg="some big value"
+        a_kwarg_calling_another_long_function=another_function(
+            another_arg,
+            another_kwarg="kwarg!"
+        )
+    )
+
+This is all in the interest of code uniformity rather than anything else.  If
+we stick to a style, everything is understandable in the same way.
+
+
+Quoting Strings
+...............
+
+pep8 is a little too open-minded on this for my liking.  Python strings should
+be quoted with double quotes (``"``) except in cases where the resulting string
+would require too much escaping of a double quote, in which case, a single
+quoted, or triple-quoted string will do:
+
+.. code:: python
+
+    my_string = "This is my string"
+    problematic_string = 'This is a "string" with "quotes" in it'
+
+In HTML templates, please use double-quotes for tag attributes, and single
+quotes for arguments passed to Django tempalte tags:
+
+.. code:: html
+
+    <div class="stuff">
+        <a href="{% url 'some-url-name' pk='w00t' %}">link this</a>
+    </div>
+
+This is to keep linters happy they look at an HTML file and see an attribute
+closing the ``"`` before it should have been.
+
+--
+
+That's all there is in terms of guidelines, so I hope it's not too daunting.
+
+
+The Code of Conduct
+===================
+
+Paperless has a `code of conduct`_.  It's a lot like the other ones you see out
+there, with a few small changes, but basically it boils down to:
+
+> Don't be an ass, or you might get banned.
+
+I'm proud to say that the CoC has never had to be enforced because everyone has
+been awesome, friendly, and professional.
+
+.. _GitHub: https://github.com/danielquinn/paperless/issues
+.. _very well documented: https://www.python.org/dev/peps/pep-0008/
+.. _code of conduct: https://github.com/danielquinn/paperless/blob/master/CODE_OF_CONDUCT.md
diff --git a/docs/index.rst b/docs/index.rst
index 7710a330c..fd9d57d4e 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -43,5 +43,6 @@ Contents
    customising
    extending
    troubleshooting
+   contributing
    scanners
    changelog

From 8de34a3572da037b6e948cb1c85000dccb84cfd1 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 23 Sep 2018 12:40:46 +0100
Subject: [PATCH 07/47] Remove numpy, scikit-learn, and scipy as they weren't
 being used

---
 requirements.txt | 25 +++++++++++--------------
 1 file changed, 11 insertions(+), 14 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 89c7e296f..b8c26a579 100755
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,10 +1,10 @@
 -i https://pypi.python.org/simple
-apipkg==1.5; python_version != '3.1.*'
-atomicwrites==1.2.1; python_version != '3.1.*'
+apipkg==1.5; python_version != '3.3.*'
+atomicwrites==1.2.1; python_version != '3.3.*'
 attrs==18.2.0
 certifi==2018.8.24
 chardet==3.0.4
-coverage==4.5.1; python_version != '3.1.*'
+coverage==4.5.1; python_version < '4'
 coveralls==1.5.0
 dateparser==0.7.0
 django-cors-headers==2.4.0
@@ -14,9 +14,9 @@ django-filter==2.0.0
 django==2.0.8
 djangorestframework==3.8.2
 docopt==0.6.2
-execnet==1.5.0; python_version != '3.1.*'
+execnet==1.5.0; python_version != '3.3.*'
 factory-boy==2.11.1
-faker==0.9.0
+faker==0.9.0; python_version >= '2.7'
 filemagic==1.6
 fuzzywuzzy==0.15.0
 gunicorn==19.9.0
@@ -24,20 +24,19 @@ idna==2.7
 inotify-simple==1.1.8
 langdetect==1.0.7
 more-itertools==4.3.0
-numpy==1.15.1
 pdftotext==2.1.0
 pillow==5.2.0
-pluggy==0.7.1; python_version != '3.1.*'
-py==1.6.0; python_version != '3.1.*'
+pluggy==0.7.1; python_version != '3.3.*'
+py==1.6.0; python_version != '3.3.*'
 pycodestyle==2.4.0
 pyocr==0.5.3
-pytest-cov==2.5.1
+pytest-cov==2.6.0
 pytest-django==3.4.2
 pytest-env==0.6.2
-pytest-forked==0.2
+pytest-forked==0.2; python_version != '3.3.*'
 pytest-sugar==0.9.1
 pytest-xdist==1.23.0
-pytest==3.7.4
+pytest==3.8.0
 python-dateutil==2.7.3
 python-dotenv==0.9.1
 python-gnupg==0.4.3
@@ -46,9 +45,7 @@ pytz==2018.5
 regex==2018.8.29
 requests==2.19.1
 six==1.11.0
-scikit-learn==0.19.2
-scipy==1.1.0
 termcolor==1.1.0
 text-unidecode==1.2
 tzlocal==1.5.1
-urllib3==1.23; python_version != '3.0.*'
+urllib3==1.23; python_version != '3.3.*'

From 52bfeb2ad08cba87114c189e1e1dc5fb0427da2b Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 23 Sep 2018 12:41:14 +0100
Subject: [PATCH 08/47] Improve the unknown language error message

---
 src/paperless_tesseract/parsers.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/paperless_tesseract/parsers.py b/src/paperless_tesseract/parsers.py
index e3c2ed361..f54461161 100644
--- a/src/paperless_tesseract/parsers.py
+++ b/src/paperless_tesseract/parsers.py
@@ -172,8 +172,8 @@ class RasterisedDocumentParser(DocumentParser):
                 raw_text = self._assemble_ocr_sections(imgs, middle, raw_text)
                 return raw_text
             raise OCRError(
-                "The guessed language is not available in this instance of "
-                "Tesseract."
+                "The guessed language ({}) is not available in this instance "
+                "of Tesseract.".format(guessed_language)
             )
 
     def _ocr(self, imgs, lang):

From 57b9add307fe1ba377468be206528eb488aa3463 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 23 Sep 2018 12:41:28 +0100
Subject: [PATCH 09/47] Conform code to standards

---
 src/documents/actions.py                      |  98 +++++++++----
 src/documents/admin.py                        | 135 ++++++++++++------
 .../documents/document/select_object.html     |  66 +++++----
 src/paperless/settings.py                     |   2 +-
 4 files changed, 198 insertions(+), 103 deletions(-)

diff --git a/src/documents/actions.py b/src/documents/actions.py
index 3db5cd314..cd2698a2c 100644
--- a/src/documents/actions.py
+++ b/src/documents/actions.py
@@ -4,10 +4,13 @@ from django.contrib.admin.utils import model_ngettext
 from django.core.exceptions import PermissionDenied
 from django.template.response import TemplateResponse
 
-from documents.models import Tag, Correspondent
+from documents.models import Correspondent, Tag
 
 
-def select_action(modeladmin, request, queryset, title, action, modelclass, success_message="", document_action=None, queryset_action=None):
+def select_action(
+        modeladmin, request, queryset, title, action, modelclass,
+        success_message="", document_action=None, queryset_action=None):
+
     opts = modeladmin.model._meta
     app_label = opts.app_label
 
@@ -27,7 +30,9 @@ def select_action(modeladmin, request, queryset, title, action, modelclass, succ
                 queryset_action(queryset, selected_object)
 
             modeladmin.message_user(request, success_message % {
-                "selected_object": selected_object.name, "count": n, "items": model_ngettext(modeladmin.opts, n)
+                "selected_object": selected_object.name,
+                "count": n,
+                "items": model_ngettext(modeladmin.opts, n)
             }, messages.SUCCESS)
 
         # Return None to display the change list page again.
@@ -47,10 +52,17 @@ def select_action(modeladmin, request, queryset, title, action, modelclass, succ
 
     request.current_app = modeladmin.admin_site.name
 
-    return TemplateResponse(request, "admin/%s/%s/select_object.html" % (app_label, opts.model_name), context)
+    return TemplateResponse(
+        request,
+        "admin/{}/{}/select_object.html".format(app_label, opts.model_name),
+        context
+    )
 
 
-def simple_action(modeladmin, request, queryset, success_message="", document_action=None, queryset_action=None):
+def simple_action(
+        modeladmin, request, queryset, success_message="",
+        document_action=None, queryset_action=None):
+
     if not modeladmin.has_change_permission(request):
         raise PermissionDenied
 
@@ -72,37 +84,63 @@ def simple_action(modeladmin, request, queryset, success_message="", document_ac
 
 
 def add_tag_to_selected(modeladmin, request, queryset):
-    return select_action(modeladmin=modeladmin, request=request, queryset=queryset,
-                         title="Add tag to multiple documents",
-                         action="add_tag_to_selected",
-                         modelclass=Tag,
-                         success_message="Successfully added tag %(selected_object)s to %(count)d %(items)s.",
-                         document_action=lambda doc, tag: doc.tags.add(tag))
-add_tag_to_selected.short_description = "Add tag to selected documents"
+    return select_action(
+        modeladmin=modeladmin,
+        request=request,
+        queryset=queryset,
+        title="Add tag to multiple documents",
+        action="add_tag_to_selected",
+        modelclass=Tag,
+        success_message="Successfully added tag %(selected_object)s to "
+                        "%(count)d %(items)s.",
+        document_action=lambda doc, tag: doc.tags.add(tag)
+    )
 
 
 def remove_tag_from_selected(modeladmin, request, queryset):
-    return select_action(modeladmin=modeladmin, request=request, queryset=queryset,
-                         title="Remove tag from multiple documents",
-                         action="remove_tag_from_selected",
-                         modelclass=Tag,
-                         success_message="Successfully removed tag %(selected_object)s from %(count)d %(items)s.",
-                         document_action=lambda doc, tag: doc.tags.remove(tag))
-remove_tag_from_selected.short_description = "Remove tag from selected documents"
+    return select_action(
+        modeladmin=modeladmin,
+        request=request,
+        queryset=queryset,
+        title="Remove tag from multiple documents",
+        action="remove_tag_from_selected",
+        modelclass=Tag,
+        success_message="Successfully removed tag %(selected_object)s from "
+                        "%(count)d %(items)s.",
+        document_action=lambda doc, tag: doc.tags.remove(tag)
+    )
 
 
 def set_correspondent_on_selected(modeladmin, request, queryset):
-    return select_action(modeladmin=modeladmin, request=request, queryset=queryset,
-                         title="Set correspondent on multiple documents",
-                         action="set_correspondent_on_selected",
-                         modelclass=Correspondent,
-                         success_message="Successfully set correspondent %(selected_object)s on %(count)d %(items)s.",
-                         queryset_action=lambda qs, correspondent: qs.update(correspondent=correspondent))
-set_correspondent_on_selected.short_description = "Set correspondent on selected documents"
+
+    return select_action(
+        modeladmin=modeladmin,
+        request=request,
+        queryset=queryset,
+        title="Set correspondent on multiple documents",
+        action="set_correspondent_on_selected",
+        modelclass=Correspondent,
+        success_message="Successfully set correspondent %(selected_object)s "
+                        "on %(count)d %(items)s.",
+        queryset_action=lambda qs, corr: qs.update(correspondent=corr)
+    )
 
 
 def remove_correspondent_from_selected(modeladmin, request, queryset):
-    return simple_action(modeladmin=modeladmin, request=request, queryset=queryset,
-                         success_message="Successfully removed correspondent from %(count)d %(items)s.",
-                         queryset_action=lambda qs: qs.update(correspondent=None))
-remove_correspondent_from_selected.short_description = "Remove correspondent from selected documents"
+    return simple_action(
+        modeladmin=modeladmin,
+        request=request,
+        queryset=queryset,
+        success_message="Successfully removed correspondent from %(count)d "
+                        "%(items)s.",
+        queryset_action=lambda qs: qs.update(correspondent=None)
+    )
+
+
+add_tag_to_selected.short_description = "Add tag to selected documents"
+remove_tag_from_selected.short_description = \
+    "Remove tag from selected documents"
+set_correspondent_on_selected.short_description = \
+    "Set correspondent on selected documents"
+remove_correspondent_from_selected.short_description = \
+    "Remove correspondent from selected documents"
diff --git a/src/documents/admin.py b/src/documents/admin.py
index d545c1c02..365a99c1a 100644
--- a/src/documents/admin.py
+++ b/src/documents/admin.py
@@ -3,21 +3,23 @@ from datetime import datetime, timedelta
 from django.conf import settings
 from django.contrib import admin, messages
 from django.contrib.admin.templatetags.admin_urls import add_preserved_filters
-from django.contrib.auth.models import User, Group
+from django.contrib.auth.models import Group, User
+from django.db import models
 from django.http import HttpResponseRedirect
-try:
-    from django.core.urlresolvers import reverse
-except ImportError:
-    from django.urls import reverse
 from django.templatetags.static import static
+from django.urls import reverse
 from django.utils.html import format_html, format_html_join
 from django.utils.http import urlquote
 from django.utils.safestring import mark_safe
-from django.db import models
 
-from documents.actions import add_tag_to_selected, remove_tag_from_selected, set_correspondent_on_selected, \
-    remove_correspondent_from_selected
-from .models import Correspondent, Tag, Document, Log
+from documents.actions import (
+    add_tag_to_selected,
+    remove_correspondent_from_selected,
+    remove_tag_from_selected,
+    set_correspondent_on_selected
+)
+
+from .models import Correspondent, Document, Log, Tag
 
 
 class FinancialYearFilter(admin.SimpleListFilter):
@@ -92,11 +94,18 @@ class RecentCorrespondentFilter(admin.RelatedFieldListFilter):
         self.title = "correspondent (recent)"
 
     def field_choices(self, field, request, model_admin):
+
+        years = settings.PAPERLESS_RECENT_CORRESPONDENT_YEARS
+        days = 365 * years
+
         lookups = []
-        if settings.PAPERLESS_RECENT_CORRESPONDENT_YEARS and settings.PAPERLESS_RECENT_CORRESPONDENT_YEARS > 0:
-            date_limit = datetime.now() - timedelta(days=365*settings.PAPERLESS_RECENT_CORRESPONDENT_YEARS)
-            for c in Correspondent.objects.filter(documents__created__gte=date_limit).distinct():
+        if years and years > 0:
+            correspondents = Correspondent.objects.filter(
+                documents__created__gte=datetime.now() - timedelta(days=days)
+            ).distinct()
+            for c in correspondents:
                 lookups.append((c.id, c.name))
+
         return lookups
 
 
@@ -106,13 +115,22 @@ class CommonAdmin(admin.ModelAdmin):
 
 class CorrespondentAdmin(CommonAdmin):
 
-    list_display = ("name", "match", "matching_algorithm", "document_count", "last_correspondence")
+    list_display = (
+        "name",
+        "match",
+        "matching_algorithm",
+        "document_count",
+        "last_correspondence"
+    )
     list_filter = ("matching_algorithm",)
     list_editable = ("match", "matching_algorithm")
 
     def get_queryset(self, request):
         qs = super(CorrespondentAdmin, self).get_queryset(request)
-        qs = qs.annotate(document_count=models.Count("documents"), last_correspondence=models.Max("documents__created"))
+        qs = qs.annotate(
+            document_count=models.Count("documents"),
+            last_correspondence=models.Max("documents__created")
+        )
         return qs
 
     def document_count(self, obj):
@@ -152,17 +170,29 @@ class DocumentAdmin(CommonAdmin):
     readonly_fields = ("added",)
     list_display = ("title", "created", "added", "thumbnail", "correspondent",
                     "tags_")
-    list_filter = ("tags", ('correspondent', RecentCorrespondentFilter), "correspondent", FinancialYearFilter)
+    list_filter = (
+        "tags",
+        ("correspondent", RecentCorrespondentFilter),
+        "correspondent",
+        FinancialYearFilter
+    )
 
     filter_horizontal = ("tags",)
 
     ordering = ["-created", "correspondent"]
 
-    actions = [add_tag_to_selected, remove_tag_from_selected, set_correspondent_on_selected, remove_correspondent_from_selected]
+    actions = [
+        add_tag_to_selected,
+        remove_tag_from_selected,
+        set_correspondent_on_selected,
+        remove_correspondent_from_selected
+    ]
 
-    date_hierarchy = 'created'
+    date_hierarchy = "created"
 
-    document_queue = None
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.document_queue = []
 
     def has_add_permission(self, request):
         return False
@@ -172,25 +202,38 @@ class DocumentAdmin(CommonAdmin):
     created_.short_description = "Created"
 
     def changelist_view(self, request, extra_context=None):
-        response = super().changelist_view(request, extra_context)
 
-        if request.method == 'GET':
+        response = super().changelist_view(
+            request,
+            extra_context=extra_context
+        )
+
+        if request.method == "GET":
             cl = self.get_changelist_instance(request)
             self.document_queue = [doc.id for doc in cl.queryset]
 
         return response
 
-    def change_view(self, request, object_id=None, form_url='', extra_context=None):
+    def change_view(self, request, object_id=None, form_url='',
+                    extra_context=None):
+
         extra_context = extra_context or {}
-        doc = Document.objects.get(id=object_id)
-        if self.document_queue and object_id and int(object_id) in self.document_queue:
-            # There is a queue of documents
-            current_index = self.document_queue.index(int(object_id))
-            if current_index < len(self.document_queue) - 1:
-                # ... and there are still documents in the queue
-                extra_context['next_object'] = self.document_queue[current_index + 1]
+
+        if self.document_queue and object_id:
+            if int(object_id) in self.document_queue:
+                # There is a queue of documents
+                current_index = self.document_queue.index(int(object_id))
+                if current_index < len(self.document_queue) - 1:
+                    # ... and there are still documents in the queue
+                    extra_context["next_object"] = self.document_queue[
+                        current_index + 1
+                    ]
+
         return super(DocumentAdmin, self).change_view(
-            request, object_id, form_url, extra_context=extra_context,
+            request,
+            object_id,
+            form_url,
+            extra_context=extra_context,
         )
 
     def response_change(self, request, obj):
@@ -200,25 +243,35 @@ class DocumentAdmin(CommonAdmin):
         preserved_filters = self.get_preserved_filters(request)
 
         msg_dict = {
-            'name': opts.verbose_name,
-            'obj': format_html('<a href="{}">{}</a>', urlquote(request.path), obj),
+            "name": opts.verbose_name,
+            "obj": format_html(
+                '<a href="{}">{}</a>',
+                urlquote(request.path),
+                obj
+            ),
         }
         if "_saveandeditnext" in request.POST:
             msg = format_html(
-                'The {name} "{obj}" was changed successfully. Editing next object.',
+                'The {name} "{obj}" was changed successfully. '
+                'Editing next object.',
                 **msg_dict
             )
             self.message_user(request, msg, messages.SUCCESS)
-            redirect_url = reverse('admin:%s_%s_change' %
-                                   (opts.app_label, opts.model_name),
-                                   args=(request.POST['_next_object'],),
-                                   current_app=self.admin_site.name)
-            redirect_url = add_preserved_filters({'preserved_filters': preserved_filters, 'opts': opts}, redirect_url)
-            response = HttpResponseRedirect(redirect_url)
-        else:
-            response = super().response_change(request, obj)
+            redirect_url = reverse(
+                "admin:{}_{}_change".format(opts.app_label, opts.model_name),
+                args=(request.POST["_next_object"],),
+                current_app=self.admin_site.name
+            )
+            redirect_url = add_preserved_filters(
+                {
+                    "preserved_filters": preserved_filters,
+                    "opts": opts
+                },
+                redirect_url
+            )
+            return HttpResponseRedirect(redirect_url)
 
-        return response
+        return super().response_change(request, obj)
 
     @mark_safe
     def thumbnail(self, obj):
diff --git a/src/documents/templates/admin/documents/document/select_object.html b/src/documents/templates/admin/documents/document/select_object.html
index 1439b5c21..775d57b12 100644
--- a/src/documents/templates/admin/documents/document/select_object.html
+++ b/src/documents/templates/admin/documents/document/select_object.html
@@ -1,46 +1,50 @@
 {% extends "admin/base_site.html" %}
+
+
 {% load i18n l10n admin_urls static %}
 {% load staticfiles %}
 
-{% block extrahead %}
-{{ block.super }}
-{{ media }}
-<script type="text/javascript" src="{% static 'admin/js/cancel.js' %}"></script>
 
+{% block extrahead %}
+	{{ block.super }}
+	{{ media }}
+	<script type="text/javascript" src="{% static 'admin/js/cancel.js' %}"></script>
 {% endblock %}
 
+
 {% block bodyclass %}{{ block.super }} app-{{ opts.app_label }} model-{{ opts.model_name }} delete-confirmation delete-selected-confirmation{% endblock %}
 
+
 {% block breadcrumbs %}
-<div class="breadcrumbs">
-    <a href="{% url 'admin:index' %}">{% trans 'Home' %}</a>
-    &rsaquo; <a href="{% url 'admin:app_list' app_label=opts.app_label %}">{{ opts.app_config.verbose_name }}</a>
-    &rsaquo; <a href="{% url opts|admin_urlname:'changelist' %}">{{ opts.verbose_name_plural|capfirst }}</a>
-    &rsaquo; {{title}}
-</div>
+	<div class="breadcrumbs">
+		<a href="{% url 'admin:index' %}">{% trans 'Home' %}</a>
+		&rsaquo; <a href="{% url 'admin:app_list' app_label=opts.app_label %}">{{ opts.app_config.verbose_name }}</a>
+		&rsaquo; <a href="{% url opts|admin_urlname:'changelist' %}">{{ opts.verbose_name_plural|capfirst }}</a>
+		&rsaquo; {{ title }}
+	</div>
 {% endblock %}
 
 {% block content %}
-<p>Please select the {{itemname}}.</p>
-<form method="post">{% csrf_token %}
-    <div>
-        {% for obj in queryset %}
-        <input type="hidden" name="{{ action_checkbox_name }}" value="{{ obj.pk|unlocalize }}"/>
-        {% endfor %}
-        <p>
-            <select name="obj_id">
-                {% for obj in objects %}
-                <option value="{{obj.id}}">{{obj.name}}</option>
-                {% endfor %}
-            </select>
-        </p>
+	<p>Please select the {{itemname}}.</p>
+	<form method="post">{% csrf_token %}
+		<div>
+			{% for obj in queryset %}
+			<input type="hidden" name="{{ action_checkbox_name }}" value="{{ obj.pk|unlocalize }}"/>
+			{% endfor %}
+			<p>
+				<select name="obj_id">
+					{% for obj in objects %}
+					<option value="{{ obj.id }}">{{ obj.name }}</option>
+					{% endfor %}
+				</select>
+			</p>
 
-        <input type="hidden" name="action" value="{{action}}"/>
-        <input type="hidden" name="post" value="yes"/>
-        <p>
-            <input type="submit" value="{% trans "Confirm" %}" />
-            <a href="#" class="button cancel-link">{% trans "Go back" %}</a>
-        </p>
-    </div>
-</form>
+			<input type="hidden" name="action" value="{{ action }}"/>
+			<input type="hidden" name="post" value="yes" />
+			<p>
+				<input type="submit" value="{% trans 'Confirm' %}" />
+				<a href="#" class="button cancel-link">{% trans "Go back" %}</a>
+			</p>
+		</div>
+	</form>
 {% endblock %}
diff --git a/src/paperless/settings.py b/src/paperless/settings.py
index e6f3da0cb..433eabe88 100644
--- a/src/paperless/settings.py
+++ b/src/paperless/settings.py
@@ -81,7 +81,7 @@ INSTALLED_APPS = [
 
     "rest_framework",
     "crispy_forms",
-    "django_filters"
+    "django_filters",
 
 ]
 

From 4634531b76d996b1a406256743887228a4e0db06 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 23 Sep 2018 12:54:39 +0100
Subject: [PATCH 10/47] Add docs for indentation & spacing

---
 docs/contributing.rst | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/docs/contributing.rst b/docs/contributing.rst
index 4ee6d18d5..05f51731c 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -97,6 +97,34 @@ closing the ``"`` before it should have been.
 That's all there is in terms of guidelines, so I hope it's not too daunting.
 
 
+Indentation & Spacing
+.....................
+
+When it comes to indentation:
+
+* For Python, the rule is: follow pep8 and use 4 spaces.
+* For Javascript, CSS, and HTML, please use 1 tab.
+
+Additionally, Django templates making use of block elements like ``{% if %}``,
+``{% for %}``, and ``{% block %}`` etc. should be indented:
+
+Good:
+
+.. code:: html
+
+    {% block stuff %}
+    	<h1>This is the stuff</h1>
+    {% endblock %}
+
+Bad:
+
+.. code:: html
+
+    {% block stuff %}
+    <h1>This is the stuff</h1>
+    {% endblock %}
+
+
 The Code of Conduct
 ===================
 

From 4c54a89fed1a12cd7957356d6b909c8026aa7056 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 23 Sep 2018 12:54:49 +0100
Subject: [PATCH 11/47] Update changelog for new stuff from #405

---
 docs/changelog.rst | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 9396493a7..6fdaee647 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -1,6 +1,14 @@
 Changelog
 #########
 
+2.4.0
+=====
+
+* A new set of actions are now available thanks to `jonaswinkler`_'s very first
+  pull request!  You can now do nifty things like tag documents in bulk, or set
+  correspondents in bulk.  `#405`_
+
+
 2.3.0
 =====
 
@@ -500,8 +508,9 @@ bulk of the work on this big change.
 .. _Kilian Koeltzsch: https://github.com/kiliankoe
 .. _Lukasz Soluch: https://github.com/LukaszSolo
 .. _Joshua Taillon: https://github.com/jat255
-.. _dubit0:  https://github.com/dubit0
-.. _ahyear:  https://github.com/ahyear
+.. _dubit0: https://github.com/dubit0
+.. _ahyear: https://github.com/ahyear
+.. _jonaswinkler: https://github.com/jonaswinkler
 
 .. _#20: https://github.com/danielquinn/paperless/issues/20
 .. _#44: https://github.com/danielquinn/paperless/issues/44
@@ -588,6 +597,7 @@ bulk of the work on this big change.
 .. _#399: https://github.com/danielquinn/paperless/pull/399
 .. _#400: https://github.com/danielquinn/paperless/pull/400
 .. _#401: https://github.com/danielquinn/paperless/pull/401
+.. _#405: https://github.com/danielquinn/paperless/pull/405
 
 .. _pipenv: https://docs.pipenv.org/
 .. _a new home on Docker Hub: https://hub.docker.com/r/danielquinn/paperless/

From fa4fb6fc06aceb1e25c69612c6ca88fa578e2cba Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 23 Sep 2018 12:59:56 +0100
Subject: [PATCH 12/47] Fix typo

---
 docs/contributing.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/contributing.rst b/docs/contributing.rst
index 05f51731c..4678ff3aa 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -21,7 +21,7 @@ Pep8
 ----
 
 It's the standard for all Python development, so it's `very well documented`_.
-The version is:
+The short version is:
 
 * Lines should wrap at 79 characters
 * Use ``snake_case`` for variables, ``CamelCase`` for classes, and ``ALL_CAPS``

From bc33b82978129d3d790aacd9891705d6bd4ed613 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 23 Sep 2018 13:58:40 +0100
Subject: [PATCH 13/47] Tweak the import/export system to handle encryption
 choices better

Now when you export a document, the `storage_type` value is always
`unencrypted` (since that's what it is when it's exported anyway), and
the flag is set by the importing script instead, based on the existence
of a `PAPERLESS_PASSPHRASE` environment variable, indicating that
encryption is enabled.
---
 .../management/commands/document_exporter.py       |  7 ++++++-
 .../management/commands/document_importer.py       | 14 +++++++++++++-
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/src/documents/management/commands/document_exporter.py b/src/documents/management/commands/document_exporter.py
index fce09092c..42a514348 100644
--- a/src/documents/management/commands/document_exporter.py
+++ b/src/documents/management/commands/document_exporter.py
@@ -55,7 +55,12 @@ class Command(Renderable, BaseCommand):
         documents = Document.objects.all()
         document_map = {d.pk: d for d in documents}
         manifest = json.loads(serializers.serialize("json", documents))
-        for document_dict in manifest:
+
+        for index, document_dict in enumerate(manifest):
+
+            # Force output to unencrypted as that will be the current state.
+            # The importer will make the decision to encrypt or not.
+            manifest[index]["fields"]["storage_type"] = Document.STORAGE_TYPE_UNENCRYPTED  # NOQA: E501
 
             document = document_map[document_dict["pk"]]
 
diff --git a/src/documents/management/commands/document_importer.py b/src/documents/management/commands/document_importer.py
index 15401722c..ae5c1853f 100644
--- a/src/documents/management/commands/document_importer.py
+++ b/src/documents/management/commands/document_importer.py
@@ -94,7 +94,7 @@ class Command(Renderable, BaseCommand):
             document_path = os.path.join(self.source, doc_file)
             thumbnail_path = os.path.join(self.source, thumb_file)
 
-            if document.storage_type == Document.STORAGE_TYPE_GPG:
+            if settings.PASSPHRASE:
 
                 with open(document_path, "rb") as unencrypted:
                     with open(document.source_path, "wb") as encrypted:
@@ -112,3 +112,15 @@ class Command(Renderable, BaseCommand):
 
                 shutil.copy(document_path, document.source_path)
                 shutil.copy(thumbnail_path, document.thumbnail_path)
+
+        # Reset the storage type to whatever we've used while importing
+
+        storage_type = Document.STORAGE_TYPE_UNENCRYPTED
+        if settings.PASSPHRASE:
+            storage_type = Document.STORAGE_TYPE_GPG
+
+        Document.objects.filter(
+            pk__in=[r["pk"] for r in self.manifest]
+        ).update(
+            storage_type=storage_type
+        )

From 076ccb3417803c20b04838a6faf0b03af7ba1abf Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 23 Sep 2018 14:00:27 +0100
Subject: [PATCH 14/47] Move the unique key on checksums to migration 15

This shouldn't affect anyone, since this migration is pretty old, but it
allows people using PostgreSQL to actually run Paperless.
---
 src/documents/migrations/0014_document_checksum.py        | 5 -----
 src/documents/migrations/0015_add_insensitive_to_match.py | 5 +++++
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/documents/migrations/0014_document_checksum.py b/src/documents/migrations/0014_document_checksum.py
index bc563cf86..a22348ba4 100644
--- a/src/documents/migrations/0014_document_checksum.py
+++ b/src/documents/migrations/0014_document_checksum.py
@@ -158,9 +158,4 @@ class Migration(migrations.Migration):
             name='modified',
             field=models.DateTimeField(auto_now=True, db_index=True),
         ),
-        migrations.AlterField(
-            model_name='document',
-            name='checksum',
-            field=models.CharField(editable=False, help_text='The checksum of the original document (before it was encrypted).  We use this to prevent duplicate document imports.', max_length=32, unique=True),
-        ),
     ]
diff --git a/src/documents/migrations/0015_add_insensitive_to_match.py b/src/documents/migrations/0015_add_insensitive_to_match.py
index 34a570c6e..30666dea9 100644
--- a/src/documents/migrations/0015_add_insensitive_to_match.py
+++ b/src/documents/migrations/0015_add_insensitive_to_match.py
@@ -12,6 +12,11 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
+        migrations.AlterField(
+            model_name='document',
+            name='checksum',
+            field=models.CharField(editable=False, help_text='The checksum of the original document (before it was encrypted).  We use this to prevent duplicate document imports.', max_length=32, unique=True),
+        ),
         migrations.AddField(
             model_name='correspondent',
             name='is_insensitive',

From 4e8f41150ad7b6acd4250d1f6a518eb8f5fa9482 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 23 Sep 2018 14:01:15 +0100
Subject: [PATCH 15/47] Tweak settings.py to allow for TRUST-based PostgreSQL
 auth

---
 src/paperless/settings.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/paperless/settings.py b/src/paperless/settings.py
index 433eabe88..4e788e56b 100644
--- a/src/paperless/settings.py
+++ b/src/paperless/settings.py
@@ -144,13 +144,14 @@ DATABASES = {
     }
 }
 
-if os.getenv("PAPERLESS_DBUSER") and os.getenv("PAPERLESS_DBPASS"):
+if os.getenv("PAPERLESS_DBUSER"):
     DATABASES["default"] = {
         "ENGINE": "django.db.backends.postgresql_psycopg2",
         "NAME": os.getenv("PAPERLESS_DBNAME", "paperless"),
         "USER": os.getenv("PAPERLESS_DBUSER"),
-        "PASSWORD": os.getenv("PAPERLESS_DBPASS")
     }
+    if os.getenv("PAPERLESS_DBPASS"):
+        DATABASES["default"]["PASSWORD"] = os.getenv("PAPERLESS_DBPASS")
 
 
 # Password validation

From 5ab74df1c0c8640ac90632f65a6abe168524d304 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 23 Sep 2018 14:01:35 +0100
Subject: [PATCH 16/47] Add a tox test for Python 3.7

---
 src/tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/tox.ini b/src/tox.ini
index 98e44e063..ff47136be 100644
--- a/src/tox.ini
+++ b/src/tox.ini
@@ -5,7 +5,7 @@
 
 [tox]
 skipsdist = True
-envlist = py34, py35, py36, pycodestyle, doc
+envlist = py34, py35, py36, py37, pycodestyle, doc
 
 [testenv]
 commands = pytest

From c70dc628455e0cde5805b5e930c0a4411b348432 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 23 Sep 2018 14:03:38 +0100
Subject: [PATCH 17/47] Add note about import/export process changes

---
 docs/changelog.rst | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 6fdaee647..fb728b8d7 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -7,6 +7,11 @@ Changelog
 * A new set of actions are now available thanks to `jonaswinkler`_'s very first
   pull request!  You can now do nifty things like tag documents in bulk, or set
   correspondents in bulk.  `#405`_
+* The import/export system is now a little smarter.  By default, documents are
+  tagged as ``unencrypted``, since exports are by their nature unencrypted.
+  It's now in the import step that we decide the storage type.  This allows you
+  to export from an encrypted system and import into an unencrypted one, or
+  vice-versa.
 
 
 2.3.0

From 8f7b073142c1bc392a384d3844da44d95bca17d7 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 23 Sep 2018 14:05:35 +0100
Subject: [PATCH 18/47] Add note about tweaks to psql connections

---
 docs/changelog.rst | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index fb728b8d7..9db59839e 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -12,7 +12,11 @@ Changelog
   It's now in the import step that we decide the storage type.  This allows you
   to export from an encrypted system and import into an unencrypted one, or
   vice-versa.
-
+* The migration history has been slightly modified to accomodate PostgreSQL
+  users.  Additionally, you can now tell paperless to use PostgreSQL simply by
+  declaring ``PAPERLESS_DBUSER`` in your environment.  This will attempt to
+  connect to your Postgres database without a password unless you also set
+  ``PAPERLESS_DBPASS``.
 
 2.3.0
 =====

From f3ed677f4d2505d29b30374e72cd6d31fcbe2fac Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 23 Sep 2018 15:38:31 +0100
Subject: [PATCH 19/47] Fix implementation of django-filter

---
 docs/changelog.rst       |  6 +++++
 src/documents/filters.py | 50 ++++++++++++++++++----------------------
 2 files changed, 29 insertions(+), 27 deletions(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 9db59839e..6ce2e49a4 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -17,6 +17,10 @@ Changelog
   declaring ``PAPERLESS_DBUSER`` in your environment.  This will attempt to
   connect to your Postgres database without a password unless you also set
   ``PAPERLESS_DBPASS``.
+* A bug was found in the REST API filter system that was the result of an
+  update of django-filter some time ago.  This has now been patched `#412`_.
+  Thanks to `thepill`_ for spotting it!
+
 
 2.3.0
 =====
@@ -520,6 +524,7 @@ bulk of the work on this big change.
 .. _dubit0: https://github.com/dubit0
 .. _ahyear: https://github.com/ahyear
 .. _jonaswinkler: https://github.com/jonaswinkler
+.. _thepill: https://github.com/thepill
 
 .. _#20: https://github.com/danielquinn/paperless/issues/20
 .. _#44: https://github.com/danielquinn/paperless/issues/44
@@ -607,6 +612,7 @@ bulk of the work on this big change.
 .. _#400: https://github.com/danielquinn/paperless/pull/400
 .. _#401: https://github.com/danielquinn/paperless/pull/401
 .. _#405: https://github.com/danielquinn/paperless/pull/405
+.. _#412: https://github.com/danielquinn/paperless/issues/412
 
 .. _pipenv: https://docs.pipenv.org/
 .. _a new home on Docker Hub: https://hub.docker.com/r/danielquinn/paperless/
diff --git a/src/documents/filters.py b/src/documents/filters.py
index 68861d967..d52889666 100644
--- a/src/documents/filters.py
+++ b/src/documents/filters.py
@@ -1,8 +1,14 @@
-from django_filters.rest_framework import CharFilter, FilterSet, BooleanFilter
+from django_filters.rest_framework import CharFilter, FilterSet, BooleanFilter, ModelChoiceFilter
 
 from .models import Correspondent, Document, Tag
 
 
+CHAR_KWARGS = (
+    "startswith", "endswith", "contains",
+    "istartswith", "iendswith", "icontains"
+)
+
+
 class CorrespondentFilterSet(FilterSet):
 
     class Meta:
@@ -31,34 +37,24 @@ class TagFilterSet(FilterSet):
 
 class DocumentFilterSet(FilterSet):
 
-    CHAR_KWARGS = {
-        "lookup_expr": (
-            "startswith",
-            "endswith",
-            "contains",
-            "istartswith",
-            "iendswith",
-            "icontains"
-        )
-    }
-
-    correspondent__name = CharFilter(
-        field_name="correspondent__name", **CHAR_KWARGS)
-    correspondent__slug = CharFilter(
-        field_name="correspondent__slug", **CHAR_KWARGS)
-    tags__name = CharFilter(
-        field_name="tags__name", **CHAR_KWARGS)
-    tags__slug = CharFilter(
-        field_name="tags__slug", **CHAR_KWARGS)
-    tags__empty = BooleanFilter(
-        field_name="tags", lookup_expr="isnull", distinct=True)
+    tags_empty = BooleanFilter(
+        label="Is tagged",
+        field_name="tags",
+        lookup_expr="isnull",
+        exclude=True
+    )
 
     class Meta:
         model = Document
         fields = {
-            "title": [
-                "startswith", "endswith", "contains",
-                "istartswith", "iendswith", "icontains"
-            ],
-            "content": ["contains", "icontains"],
+
+            "title": CHAR_KWARGS,
+            "content": ("contains", "icontains"),
+
+            "correspondent__name": CHAR_KWARGS,
+            "correspondent__slug": CHAR_KWARGS,
+
+            "tags__name": CHAR_KWARGS,
+            "tags__slug": CHAR_KWARGS,
+
         }

From 815f158fc3e35aa13cc26b314f46bd05f353b6a0 Mon Sep 17 00:00:00 2001
From: euri10 <benoit.barthelet@gmail.com>
Date: Mon, 24 Sep 2018 13:30:10 +0200
Subject: [PATCH 20/47] Fix issue where tesseract langages weren't installed
 properly

---
 Dockerfile | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index f02bf3336..968d67da5 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,4 @@
-FROM alpine:3.7
+FROM alpine:3.8
 
 LABEL maintainer="The Paperless Project https://github.com/danielquinn/paperless" \
       contributors="Guy Addadi <addadi@gmail.com>, Pit Kleyersburg <pitkley@googlemail.com>, \
@@ -12,11 +12,10 @@ COPY scripts/docker-entrypoint.sh /sbin/docker-entrypoint.sh
 ENV PAPERLESS_EXPORT_DIR=/export \
     PAPERLESS_CONSUMPTION_DIR=/consume
 
-# Install dependencies
-RUN apk --no-cache --update add \
-        python3 gnupg libmagic bash shadow curl \
+
+RUN apk update --no-cache && apk add python3 gnupg libmagic bash shadow curl \
         sudo poppler tesseract-ocr imagemagick ghostscript unpaper && \
-    apk --no-cache add --virtual .build-dependencies \
+    apk add --virtual .build-dependencies \
         python3-dev poppler-dev gcc g++ musl-dev zlib-dev jpeg-dev && \
 # Install python dependencies
     python3 -m ensurepip && \

From 0a4338143a8804ca77b2295ffe66818a483b238d Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Mon, 1 Oct 2018 20:03:27 +0100
Subject: [PATCH 21/47] Tweak the date guesser to not allow dates prior to 1900
 (#414)

---
 src/paperless_tesseract/parsers.py         | 25 ++++++++++++++++------
 src/paperless_tesseract/tests/test_date.py | 13 +++++++++++
 2 files changed, 31 insertions(+), 7 deletions(-)

diff --git a/src/paperless_tesseract/parsers.py b/src/paperless_tesseract/parsers.py
index f54461161..5305ff053 100644
--- a/src/paperless_tesseract/parsers.py
+++ b/src/paperless_tesseract/parsers.py
@@ -203,6 +203,7 @@ class RasterisedDocumentParser(DocumentParser):
         return text
 
     def get_date(self):
+
         date = None
         datestring = None
 
@@ -217,20 +218,30 @@ class RasterisedDocumentParser(DocumentParser):
 
             try:
                 date = dateparser.parse(
-                           datestring,
-                           settings={'DATE_ORDER': self.DATE_ORDER,
-                                     'PREFER_DAY_OF_MONTH': 'first',
-                                     'RETURN_AS_TIMEZONE_AWARE': True})
+                    datestring,
+                    settings={
+                        "DATE_ORDER": self.DATE_ORDER,
+                        "PREFER_DAY_OF_MONTH": "first",
+                        "RETURN_AS_TIMEZONE_AWARE": True
+                    }
+                )
             except TypeError:
                 # Skip all matches that do not parse to a proper date
                 continue
 
-            if date is not None:
+            if date is not None and date.year > 1900:
                 break
+            else:
+                date = None
 
         if date is not None:
-            self.log("info", "Detected document date " + date.isoformat() +
-                             " based on string " + datestring)
+            self.log(
+                "info",
+                "Detected document date {} based on string {}".format(
+                    date.isoformat(),
+                    datestring
+                )
+            )
         else:
             self.log("info", "Unable to detect date for document")
 
diff --git a/src/paperless_tesseract/tests/test_date.py b/src/paperless_tesseract/tests/test_date.py
index 645cb70ff..e75042ce1 100644
--- a/src/paperless_tesseract/tests/test_date.py
+++ b/src/paperless_tesseract/tests/test_date.py
@@ -384,3 +384,16 @@ class TestDate(TestCase):
             document.get_date(),
             datetime.datetime(2017, 12, 31, 0, 0, tzinfo=tz.tzutc())
         )
+
+    @mock.patch(
+        "paperless_tesseract.parsers.RasterisedDocumentParser.get_text",
+        return_value="01-07-0590 00:00:00"
+    )
+    @mock.patch(
+        "paperless_tesseract.parsers.RasterisedDocumentParser.SCRATCH",
+        SCRATCH
+    )
+    def test_crazy_date(self, *args):
+        document = RasterisedDocumentParser("/dev/null")
+        document.get_text()
+        self.assertIsNone(document.get_date())

From 40901a07347fe4fac24b06e9f182c82df9b35269 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Mon, 1 Oct 2018 20:40:32 +0100
Subject: [PATCH 22/47] Update version number & changelog

---
 docs/changelog.rst       | 14 ++++++++++++++
 src/paperless/version.py |  2 +-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 6ce2e49a4..7daaa9d38 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -1,6 +1,15 @@
 Changelog
 #########
 
+2.4.1
+=====
+
+* An annoying bug in the date capture code was causing some bogus dates to be
+  attached to documents, which in turn busted the UI.  Thanks to `Andrew Peng`_
+  for reporting this. `#414`_.
+* A bug in the Dockerfile meant that Tesseract language files weren't being
+  installed correctly.  `euri10`_ was quick to provide a fix: `#406`_, `#413`_.
+
 2.4.0
 =====
 
@@ -525,6 +534,8 @@ bulk of the work on this big change.
 .. _ahyear: https://github.com/ahyear
 .. _jonaswinkler: https://github.com/jonaswinkler
 .. _thepill: https://github.com/thepill
+.. _Andrew Peng: https://github.com/pengc99
+.. _euri10: https://github.com/euri10
 
 .. _#20: https://github.com/danielquinn/paperless/issues/20
 .. _#44: https://github.com/danielquinn/paperless/issues/44
@@ -612,7 +623,10 @@ bulk of the work on this big change.
 .. _#400: https://github.com/danielquinn/paperless/pull/400
 .. _#401: https://github.com/danielquinn/paperless/pull/401
 .. _#405: https://github.com/danielquinn/paperless/pull/405
+.. _#406: https://github.com/danielquinn/paperless/issues/406
 .. _#412: https://github.com/danielquinn/paperless/issues/412
+.. _#413: https://github.com/danielquinn/paperless/pull/413
+.. _#414: https://github.com/danielquinn/paperless/issues/414
 
 .. _pipenv: https://docs.pipenv.org/
 .. _a new home on Docker Hub: https://hub.docker.com/r/danielquinn/paperless/
diff --git a/src/paperless/version.py b/src/paperless/version.py
index c1b36d9c1..31ee9974c 100644
--- a/src/paperless/version.py
+++ b/src/paperless/version.py
@@ -1 +1 @@
-__version__ = (2, 3, 0)
+__version__ = (2, 4, 1)

From 1e8e9469ee55e4434743cf75366cc7509c49281c Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Mon, 1 Oct 2018 20:40:43 +0100
Subject: [PATCH 23/47] Hopefully fix Travis

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index e2c6acb51..7c0082a24 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -2,7 +2,7 @@ language: python
 
 before_install:
 - sudo apt-get update -qq
-- sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr tesseract-ocr-eng
+- sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr tesseract-ocr-eng tesseract-ocr-cat
 
 sudo: false
 

From 40b9e44bfe1f2f6fd00389c50b175d4a299eb2c7 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 7 Oct 2018 13:12:22 +0100
Subject: [PATCH 24/47] Wrap document consumption in a transaction #262

---
 src/documents/consumer.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/documents/consumer.py b/src/documents/consumer.py
index 28fc28f9e..7dd94ebf1 100644
--- a/src/documents/consumer.py
+++ b/src/documents/consumer.py
@@ -1,3 +1,4 @@
+from django.db import transaction
 import datetime
 import hashlib
 import logging
@@ -111,8 +112,11 @@ class Consumer:
                 if not self.try_consume_file(file):
                     self._ignore.append((file, mtime))
 
+    @transaction.atomic
     def try_consume_file(self, file):
-        "Return True if file was consumed"
+        """
+        Return True if file was consumed
+        """
 
         if not re.match(FileInfo.REGEXES["title"], file):
             return False

From 074609e1fc1f72c6df388b141df9abe7f8a5ec32 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 7 Oct 2018 14:48:49 +0100
Subject: [PATCH 25/47] Consolidate get_date onto the DocumentParser parent
 class

---
 docs/changelog.rst                         |  2 +
 src/documents/parsers.py                   | 53 +++++++++++++++++++++-
 src/paperless_tesseract/parsers.py         | 49 +-------------------
 src/paperless_tesseract/tests/test_date.py | 28 +++++++++++-
 src/paperless_text/parsers.py              | 41 +----------------
 5 files changed, 83 insertions(+), 90 deletions(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 7daaa9d38..aefe65c25 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -9,6 +9,8 @@ Changelog
   for reporting this. `#414`_.
 * A bug in the Dockerfile meant that Tesseract language files weren't being
   installed correctly.  `euri10`_ was quick to provide a fix: `#406`_, `#413`_.
+* The ``get_date()`` functionality of the parsers has been consolidated onto
+  the ``DocumentParser`` class since much of that code was redundant anyway.
 
 2.4.0
 =====
diff --git a/src/documents/parsers.py b/src/documents/parsers.py
index 884f91ae4..29128eaad 100644
--- a/src/documents/parsers.py
+++ b/src/documents/parsers.py
@@ -1,9 +1,12 @@
 import logging
+import os
+import re
 import shutil
 import tempfile
-import re
 
+import dateparser
 from django.conf import settings
+from django.utils import timezone
 
 # This regular expression will try to find dates in the document at
 # hand and will match the following formats:
@@ -32,6 +35,7 @@ class DocumentParser:
     """
 
     SCRATCH = settings.SCRATCH_DIR
+    DATE_ORDER = settings.DATE_ORDER
 
     def __init__(self, path):
         self.document_path = path
@@ -55,7 +59,52 @@ class DocumentParser:
         """
         Returns the date of the document.
         """
-        raise NotImplementedError()
+
+        date = None
+        date_string = None
+
+        try:
+            text = self.get_text()
+        except ParseError:
+            return None
+
+        next_year = timezone.now().year + 5  # Arbitrary 5 year future limit
+
+        # Iterate through all regex matches and try to parse the date
+        for m in re.finditer(DATE_REGEX, text):
+
+            date_string = m.group(0)
+
+            try:
+                date = dateparser.parse(
+                    date_string,
+                    settings={
+                        "DATE_ORDER": self.DATE_ORDER,
+                        "PREFER_DAY_OF_MONTH": "first",
+                        "RETURN_AS_TIMEZONE_AWARE": True
+                    }
+                )
+            except TypeError:
+                # Skip all matches that do not parse to a proper date
+                continue
+
+            if date is not None and next_year > date.year > 1900:
+                break
+            else:
+                date = None
+
+        if date is not None:
+            self.log(
+                "info",
+                "Detected document date {} based on string {}".format(
+                    date.isoformat(),
+                    date_string
+                )
+            )
+        else:
+            self.log("info", "Unable to detect date for document")
+
+        return date
 
     def log(self, level, message):
         getattr(self.logger, level)(message, extra={
diff --git a/src/paperless_tesseract/parsers.py b/src/paperless_tesseract/parsers.py
index 5305ff053..8ba162b9f 100644
--- a/src/paperless_tesseract/parsers.py
+++ b/src/paperless_tesseract/parsers.py
@@ -4,7 +4,6 @@ import re
 import subprocess
 from multiprocessing.pool import Pool
 
-import dateparser
 import langdetect
 import pyocr
 from django.conf import settings
@@ -14,7 +13,7 @@ from pyocr.libtesseract.tesseract_raw import \
 from pyocr.tesseract import TesseractError
 
 import pdftotext
-from documents.parsers import DocumentParser, ParseError, DATE_REGEX
+from documents.parsers import DocumentParser, ParseError
 
 from .languages import ISO639
 
@@ -33,7 +32,6 @@ class RasterisedDocumentParser(DocumentParser):
     DENSITY = settings.CONVERT_DENSITY if settings.CONVERT_DENSITY else 300
     THREADS = int(settings.OCR_THREADS) if settings.OCR_THREADS else None
     UNPAPER = settings.UNPAPER_BINARY
-    DATE_ORDER = settings.DATE_ORDER
     DEFAULT_OCR_LANGUAGE = settings.OCR_LANGUAGE
     OCR_ALWAYS = settings.OCR_ALWAYS
 
@@ -202,51 +200,6 @@ class RasterisedDocumentParser(DocumentParser):
         text += self._ocr(imgs[middle + 1:], self.DEFAULT_OCR_LANGUAGE)
         return text
 
-    def get_date(self):
-
-        date = None
-        datestring = None
-
-        try:
-            text = self.get_text()
-        except ParseError as e:
-            return None
-
-        # Iterate through all regex matches and try to parse the date
-        for m in re.finditer(DATE_REGEX, text):
-            datestring = m.group(0)
-
-            try:
-                date = dateparser.parse(
-                    datestring,
-                    settings={
-                        "DATE_ORDER": self.DATE_ORDER,
-                        "PREFER_DAY_OF_MONTH": "first",
-                        "RETURN_AS_TIMEZONE_AWARE": True
-                    }
-                )
-            except TypeError:
-                # Skip all matches that do not parse to a proper date
-                continue
-
-            if date is not None and date.year > 1900:
-                break
-            else:
-                date = None
-
-        if date is not None:
-            self.log(
-                "info",
-                "Detected document date {} based on string {}".format(
-                    date.isoformat(),
-                    datestring
-                )
-            )
-        else:
-            self.log("info", "Unable to detect date for document")
-
-        return date
-
 
 def run_convert(*args):
 
diff --git a/src/paperless_tesseract/tests/test_date.py b/src/paperless_tesseract/tests/test_date.py
index e75042ce1..15fed1a37 100644
--- a/src/paperless_tesseract/tests/test_date.py
+++ b/src/paperless_tesseract/tests/test_date.py
@@ -393,7 +393,33 @@ class TestDate(TestCase):
         "paperless_tesseract.parsers.RasterisedDocumentParser.SCRATCH",
         SCRATCH
     )
-    def test_crazy_date(self, *args):
+    def test_crazy_date_past(self, *args):
+        document = RasterisedDocumentParser("/dev/null")
+        document.get_text()
+        self.assertIsNone(document.get_date())
+
+    @mock.patch(
+        "paperless_tesseract.parsers.RasterisedDocumentParser.get_text",
+        return_value="01-07-2350 00:00:00"
+    )
+    @mock.patch(
+        "paperless_tesseract.parsers.RasterisedDocumentParser.SCRATCH",
+        SCRATCH
+    )
+    def test_crazy_date_future(self, *args):
+        document = RasterisedDocumentParser("/dev/null")
+        document.get_text()
+        self.assertIsNone(document.get_date())
+
+    @mock.patch(
+        "paperless_tesseract.parsers.RasterisedDocumentParser.get_text",
+        return_value="01-07-0590 00:00:00"
+    )
+    @mock.patch(
+        "paperless_tesseract.parsers.RasterisedDocumentParser.SCRATCH",
+        SCRATCH
+    )
+    def test_crazy_date_past(self, *args):
         document = RasterisedDocumentParser("/dev/null")
         document.get_text()
         self.assertIsNone(document.get_date())
diff --git a/src/paperless_text/parsers.py b/src/paperless_text/parsers.py
index f02ba3ef8..afcfb013c 100644
--- a/src/paperless_text/parsers.py
+++ b/src/paperless_text/parsers.py
@@ -1,11 +1,9 @@
 import os
-import re
 import subprocess
 
-import dateparser
 from django.conf import settings
 
-from documents.parsers import DocumentParser, ParseError, DATE_REGEX
+from documents.parsers import DocumentParser, ParseError
 
 
 class TextDocumentParser(DocumentParser):
@@ -16,7 +14,6 @@ class TextDocumentParser(DocumentParser):
     CONVERT = settings.CONVERT_BINARY
     THREADS = int(settings.OCR_THREADS) if settings.OCR_THREADS else None
     UNPAPER = settings.UNPAPER_BINARY
-    DATE_ORDER = settings.DATE_ORDER
     DEFAULT_OCR_LANGUAGE = settings.OCR_LANGUAGE
     OCR_ALWAYS = settings.OCR_ALWAYS
 
@@ -26,7 +23,7 @@ class TextDocumentParser(DocumentParser):
 
     def get_thumbnail(self):
         """
-        The thumbnail of a txt is just a 500px wide image of the text
+        The thumbnail of a text file is just a 500px wide image of the text
         rendered onto a letter-sized page.
         """
         # The below is heavily cribbed from https://askubuntu.com/a/590951
@@ -84,40 +81,6 @@ class TextDocumentParser(DocumentParser):
 
         return self._text
 
-    def get_date(self):
-        date = None
-        datestring = None
-
-        try:
-            text = self.get_text()
-        except ParseError as e:
-            return None
-
-        # Iterate through all regex matches and try to parse the date
-        for m in re.finditer(DATE_REGEX, text):
-            datestring = m.group(0)
-
-            try:
-                date = dateparser.parse(
-                           datestring,
-                           settings={'DATE_ORDER': self.DATE_ORDER,
-                                     'PREFER_DAY_OF_MONTH': 'first',
-                                     'RETURN_AS_TIMEZONE_AWARE': True})
-            except TypeError:
-                # Skip all matches that do not parse to a proper date
-                continue
-
-            if date is not None:
-                break
-
-        if date is not None:
-            self.log("info", "Detected document date " + date.isoformat() +
-                             " based on string " + datestring)
-        else:
-            self.log("info", "Unable to detect date for document")
-
-        return date
-
 
 def run_command(*args):
     environment = os.environ.copy()

From bc898c199230890e772a5b7b56e3d37d08017034 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 7 Oct 2018 14:56:38 +0100
Subject: [PATCH 26/47] Use optipng to optimise document thumbnails

---
 Dockerfile                         |  2 +-
 docs/changelog.rst                 |  8 +++++-
 paperless.conf.example             | 20 ++++++++++++++
 src/documents/consumer.py          |  2 +-
 src/documents/parsers.py           | 15 +++++++++++
 src/paperless/checks.py            |  7 ++++-
 src/paperless/settings.py          |  3 +++
 src/paperless_tesseract/parsers.py |  7 +++--
 src/paperless_text/parsers.py      | 43 +++++++++++++++++++-----------
 9 files changed, 85 insertions(+), 22 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 968d67da5..55d54cc01 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -14,7 +14,7 @@ ENV PAPERLESS_EXPORT_DIR=/export \
 
 
 RUN apk update --no-cache && apk add python3 gnupg libmagic bash shadow curl \
-        sudo poppler tesseract-ocr imagemagick ghostscript unpaper && \
+        sudo poppler tesseract-ocr imagemagick ghostscript unpaper optipng && \
     apk add --virtual .build-dependencies \
         python3-dev poppler-dev gcc g++ musl-dev zlib-dev jpeg-dev && \
 # Install python dependencies
diff --git a/docs/changelog.rst b/docs/changelog.rst
index aefe65c25..5e548301c 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -1,9 +1,14 @@
 Changelog
 #########
 
-2.4.1
+2.5.0
 =====
 
+* **New dependency**: Paperless now optimises thumbnail generation with
+  `optipng`_, so you'll need to install that somewhere in your PATH or declare
+  its location in ``PAPERLESS_OPTIPNG_BINARY``.  The Docker image has already
+  been updated on the Docker Hub, so you just need to pull the latest one from
+  there if you're a Docker user.
 * An annoying bug in the date capture code was causing some bogus dates to be
   attached to documents, which in turn busted the UI.  Thanks to `Andrew Peng`_
   for reporting this. `#414`_.
@@ -632,3 +637,4 @@ bulk of the work on this big change.
 
 .. _pipenv: https://docs.pipenv.org/
 .. _a new home on Docker Hub: https://hub.docker.com/r/danielquinn/paperless/
+.. _optipng: http://optipng.sourceforge.net/
diff --git a/paperless.conf.example b/paperless.conf.example
index 05cf81724..3604505cb 100644
--- a/paperless.conf.example
+++ b/paperless.conf.example
@@ -213,3 +213,23 @@ PAPERLESS_DEBUG="false"
 # The number of years for which a correspondent will be included in the recent
 # correspondents filter.
 #PAPERLESS_RECENT_CORRESPONDENT_YEARS=1
+
+###############################################################################
+####                     Third-Party Binaries                              ####
+###############################################################################
+
+# There are a few external software packages that Paperless expects to find on
+# your system when it starts up.  Unless you've done something creative with
+# their installation, you probably won't need to edit any of these.  However,
+# if you've installed these programs somewhere where simply typing the name of
+# the program doesn't automatically execute it (ie. the program isn't in your
+# $PATH), then you'll need to specify the literal path for that program here.
+
+# Convert (part of the ImageMagick suite)
+#PAPERLESS_CONVERT_BINARY=/usr/bin/convert
+
+# Unpaper
+#PAPERLESS_UNPAPER_BINARY=/usr/bin/unpaper
+
+# Optipng (for optimising thumbnail sizes)
+#PAPERLESS_OPTIPNG_BINARY=/usr/bin/optipng
diff --git a/src/documents/consumer.py b/src/documents/consumer.py
index 7dd94ebf1..3cb484b2a 100644
--- a/src/documents/consumer.py
+++ b/src/documents/consumer.py
@@ -149,7 +149,7 @@ class Consumer:
         parsed_document = parser_class(doc)
 
         try:
-            thumbnail = parsed_document.get_thumbnail()
+            thumbnail = parsed_document.get_optimised_thumbnail()
             date = parsed_document.get_date()
             document = self._store(
                 parsed_document.get_text(),
diff --git a/src/documents/parsers.py b/src/documents/parsers.py
index 29128eaad..1f60b1479 100644
--- a/src/documents/parsers.py
+++ b/src/documents/parsers.py
@@ -2,6 +2,7 @@ import logging
 import os
 import re
 import shutil
+import subprocess
 import tempfile
 
 import dateparser
@@ -36,6 +37,7 @@ class DocumentParser:
 
     SCRATCH = settings.SCRATCH_DIR
     DATE_ORDER = settings.DATE_ORDER
+    OPTIPNG = settings.OPTIPNG_BINARY
 
     def __init__(self, path):
         self.document_path = path
@@ -49,6 +51,19 @@ class DocumentParser:
         """
         raise NotImplementedError()
 
+    def optimise_thumbnail(self, in_path):
+
+        out_path = os.path.join(self.tempdir, "optipng.png")
+
+        args = (self.OPTIPNG, "-o5", in_path, "-out", out_path)
+        if not subprocess.Popen(args).wait() == 0:
+            raise ParseError("Optipng failed at {}".format(args))
+
+        return out_path
+
+    def get_optimised_thumbnail(self):
+        return self.optimise_thumbnail(self.get_thumbnail())
+
     def get_text(self):
         """
         Returns the text from the document and only the text.
diff --git a/src/paperless/checks.py b/src/paperless/checks.py
index 666425f9c..e8c94362a 100644
--- a/src/paperless/checks.py
+++ b/src/paperless/checks.py
@@ -76,7 +76,12 @@ def binaries_check(app_configs, **kwargs):
     error = "Paperless can't find {}. Without it, consumption is impossible."
     hint = "Either it's not in your ${PATH} or it's not installed."
 
-    binaries = (settings.CONVERT_BINARY, settings.UNPAPER_BINARY, "tesseract")
+    binaries = (
+        settings.CONVERT_BINARY,
+        settings.OPTIPNG_BINARY,
+        settings.UNPAPER_BINARY,
+        "tesseract"
+    )
 
     check_messages = []
     for binary in binaries:
diff --git a/src/paperless/settings.py b/src/paperless/settings.py
index 4e788e56b..fb5a399a8 100644
--- a/src/paperless/settings.py
+++ b/src/paperless/settings.py
@@ -247,6 +247,9 @@ CONVERT_TMPDIR = os.getenv("PAPERLESS_CONVERT_TMPDIR")
 CONVERT_MEMORY_LIMIT = os.getenv("PAPERLESS_CONVERT_MEMORY_LIMIT")
 CONVERT_DENSITY = os.getenv("PAPERLESS_CONVERT_DENSITY")
 
+# OptiPNG
+OPTIPNG_BINARY = os.getenv("PAPERLESS_OPTIPNG_BINARY", "optipng")
+
 # Unpaper
 UNPAPER_BINARY = os.getenv("PAPERLESS_UNPAPER_BINARY", "unpaper")
 
diff --git a/src/paperless_tesseract/parsers.py b/src/paperless_tesseract/parsers.py
index 8ba162b9f..dc5dbd637 100644
--- a/src/paperless_tesseract/parsers.py
+++ b/src/paperless_tesseract/parsers.py
@@ -44,15 +44,18 @@ class RasterisedDocumentParser(DocumentParser):
         The thumbnail of a PDF is just a 500px wide image of the first page.
         """
 
+        out_path = os.path.join(self.tempdir, "convert.png")
+
+        # Run convert to get a decent thumbnail
         run_convert(
             self.CONVERT,
             "-scale", "500x5000",
             "-alpha", "remove",
             "{}[0]".format(self.document_path),
-            os.path.join(self.tempdir, "convert.png")
+            out_path
         )
 
-        return os.path.join(self.tempdir, "convert.png")
+        return out_path
 
     def _is_ocred(self):
 
diff --git a/src/paperless_text/parsers.py b/src/paperless_text/parsers.py
index afcfb013c..3ccb78404 100644
--- a/src/paperless_text/parsers.py
+++ b/src/paperless_text/parsers.py
@@ -32,7 +32,7 @@ class TextDocumentParser(DocumentParser):
         text_color = "black"  # text color
         psize = [500, 647]  # icon size
         n_lines = 50  # number of lines to show
-        output_file = os.path.join(self.tempdir, "convert-txt.png")
+        out_path = os.path.join(self.tempdir, "convert.png")
 
         temp_bg = os.path.join(self.tempdir, "bg.png")
         temp_txlayer = os.path.join(self.tempdir, "tx.png")
@@ -43,9 +43,13 @@ class TextDocumentParser(DocumentParser):
             work_size = ",".join([str(n - 1) for n in psize])
             r = str(round(psize[0] / 10))
             rounded = ",".join([r, r])
-            run_command(self.CONVERT, "-size ", picsize, ' xc:none -draw ',
-                        '"fill ', bg_color, ' roundrectangle 0,0,',
-                        work_size, ",", rounded, '" ', temp_bg)
+            run_command(
+                self.CONVERT,
+                "-size ", picsize,
+                ' xc:none -draw ',
+                '"fill ', bg_color, ' roundrectangle 0,0,', work_size, ",", rounded, '" ',  # NOQA: E501
+                temp_bg
+            )
 
         def read_text():
             with open(self.document_path, 'r') as src:
@@ -54,22 +58,29 @@ class TextDocumentParser(DocumentParser):
                 return text.replace('"', "'")
 
         def create_txlayer():
-            run_command(self.CONVERT,
-                        "-background none",
-                        "-fill",
-                        text_color,
-                        "-pointsize", "12",
-                        "-border 4 -bordercolor none",
-                        "-size ", txsize,
-                        ' caption:"', read_text(), '" ',
-                        temp_txlayer)
+            run_command(
+                self.CONVERT,
+                "-background none",
+                "-fill",
+                text_color,
+                "-pointsize", "12",
+                "-border 4 -bordercolor none",
+                "-size ", txsize,
+                ' caption:"', read_text(), '" ',
+                temp_txlayer
+            )
 
         create_txlayer()
         create_bg()
-        run_command(self.CONVERT, temp_bg, temp_txlayer,
-                    "-background None -layers merge ", output_file)
+        run_command(
+            self.CONVERT,
+            temp_bg,
+            temp_txlayer,
+            "-background None -layers merge ",
+            out_path
+        )
 
-        return output_file
+        return out_path
 
     def get_text(self):
 

From 94f9c9dcea9e072c7f9256a334c4b1e0193727dd Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 7 Oct 2018 14:56:56 +0100
Subject: [PATCH 27/47] Wrap each document consumption in a transaction

---
 docs/changelog.rst | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 5e548301c..598241938 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -14,6 +14,8 @@ Changelog
   for reporting this. `#414`_.
 * A bug in the Dockerfile meant that Tesseract language files weren't being
   installed correctly.  `euri10`_ was quick to provide a fix: `#406`_, `#413`_.
+* Document consumption is now wrapped in a transaction as per an old ticket
+  `#262`_.
 * The ``get_date()`` functionality of the parsers has been consolidated onto
   the ``DocumentParser`` class since much of that code was redundant anyway.
 
@@ -608,6 +610,7 @@ bulk of the work on this big change.
 .. _#322: https://github.com/danielquinn/paperless/pull/322
 .. _#328: https://github.com/danielquinn/paperless/pull/328
 .. _#253: https://github.com/danielquinn/paperless/issues/253
+.. _#262: https://github.com/danielquinn/paperless/issues/262
 .. _#323: https://github.com/danielquinn/paperless/issues/323
 .. _#344: https://github.com/danielquinn/paperless/pull/344
 .. _#351: https://github.com/danielquinn/paperless/pull/351

From 2718945829eaede9992f3bb78c542942fa688a15 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 7 Oct 2018 16:22:52 +0100
Subject: [PATCH 28/47] Fix formatting

---
 src/reminders/models.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/reminders/models.py b/src/reminders/models.py
index 77d872afb..e8ac7020b 100644
--- a/src/reminders/models.py
+++ b/src/reminders/models.py
@@ -4,7 +4,6 @@ from django.db import models
 class Reminder(models.Model):
 
     document = models.ForeignKey(
-        "documents.Document", on_delete=models.PROTECT
-        )
+        "documents.Document", on_delete=models.PROTECT)
     date = models.DateTimeField()
     note = models.TextField(blank=True)

From cbdb902bfc3911c71c149e08ad03a292122c35cf Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 7 Oct 2018 16:23:03 +0100
Subject: [PATCH 29/47] Add migration that should have come in some time ago

---
 .../migrations/0002_auto_20181007_1420.py     | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 src/reminders/migrations/0002_auto_20181007_1420.py

diff --git a/src/reminders/migrations/0002_auto_20181007_1420.py b/src/reminders/migrations/0002_auto_20181007_1420.py
new file mode 100644
index 000000000..324764d2c
--- /dev/null
+++ b/src/reminders/migrations/0002_auto_20181007_1420.py
@@ -0,0 +1,19 @@
+# Generated by Django 2.0.8 on 2018-10-07 14:20
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('reminders', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='reminder',
+            name='document',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='documents.Document'),
+        ),
+    ]

From 081f1022cfef0af52c54aaaf8eca26e44ec03987 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 7 Oct 2018 16:24:05 +0100
Subject: [PATCH 30/47] Rework how slugs are generated/referenced #393

---
 docs/changelog.rst                            | 20 +++++++
 src/documents/admin.py                        |  4 ++
 .../migrations/0022_auto_20181007_1420.py     | 52 +++++++++++++++++++
 src/documents/models.py                       |  9 ++--
 4 files changed, 80 insertions(+), 5 deletions(-)
 create mode 100644 src/documents/migrations/0022_auto_20181007_1420.py

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 598241938..b446e45cf 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -9,13 +9,32 @@ Changelog
   its location in ``PAPERLESS_OPTIPNG_BINARY``.  The Docker image has already
   been updated on the Docker Hub, so you just need to pull the latest one from
   there if you're a Docker user.
+
+* A problem in how we handle slug values on Tags and Correspondents required a
+  few changes to how we handle this field `#393`_:
+
+  1. Slugs are no longer editable.  They're derived from the name of the tag or
+     correspondent at save time, so if you wanna change the slug, you have to
+     change the name, and even then you're restricted to the rules of the
+     ``slugify()`` function.  The slug value is still visible in the admin
+     though.
+  2. I've added a migration to go over all existing tags & correspondents and
+     rewrite the ``.slug`` values to ones conforming to the ``slugify()``
+     rules.
+  3. The consumption process now uses the same rules as ``.save()`` in
+     determining a slug and using that to check for an existing
+     tag/correspondent.
+
 * An annoying bug in the date capture code was causing some bogus dates to be
   attached to documents, which in turn busted the UI.  Thanks to `Andrew Peng`_
   for reporting this. `#414`_.
+
 * A bug in the Dockerfile meant that Tesseract language files weren't being
   installed correctly.  `euri10`_ was quick to provide a fix: `#406`_, `#413`_.
+
 * Document consumption is now wrapped in a transaction as per an old ticket
   `#262`_.
+
 * The ``get_date()`` functionality of the parsers has been consolidated onto
   the ``DocumentParser`` class since much of that code was redundant anyway.
 
@@ -627,6 +646,7 @@ bulk of the work on this big change.
 .. _#391: https://github.com/danielquinn/paperless/pull/391
 .. _#390: https://github.com/danielquinn/paperless/pull/390
 .. _#392: https://github.com/danielquinn/paperless/issues/392
+.. _#393: https://github.com/danielquinn/paperless/issues/393
 .. _#395: https://github.com/danielquinn/paperless/pull/395
 .. _#396: https://github.com/danielquinn/paperless/pull/396
 .. _#399: https://github.com/danielquinn/paperless/pull/399
diff --git a/src/documents/admin.py b/src/documents/admin.py
index 365a99c1a..6dbe7f835 100644
--- a/src/documents/admin.py
+++ b/src/documents/admin.py
@@ -125,6 +125,8 @@ class CorrespondentAdmin(CommonAdmin):
     list_filter = ("matching_algorithm",)
     list_editable = ("match", "matching_algorithm")
 
+    readonly_fields = ("slug",)
+
     def get_queryset(self, request):
         qs = super(CorrespondentAdmin, self).get_queryset(request)
         qs = qs.annotate(
@@ -149,6 +151,8 @@ class TagAdmin(CommonAdmin):
     list_filter = ("colour", "matching_algorithm")
     list_editable = ("colour", "match", "matching_algorithm")
 
+    readonly_fields = ("slug",)
+
     def get_queryset(self, request):
         qs = super(TagAdmin, self).get_queryset(request)
         qs = qs.annotate(document_count=models.Count("documents"))
diff --git a/src/documents/migrations/0022_auto_20181007_1420.py b/src/documents/migrations/0022_auto_20181007_1420.py
new file mode 100644
index 000000000..937695bc8
--- /dev/null
+++ b/src/documents/migrations/0022_auto_20181007_1420.py
@@ -0,0 +1,52 @@
+# Generated by Django 2.0.8 on 2018-10-07 14:20
+
+from django.db import migrations, models
+from django.utils.text import slugify
+
+
+def re_slug_all_the_things(apps, schema_editor):
+    """
+    Rewrite all slug values to make sure they're actually slugs before we brand
+    them as uneditable.
+    """
+
+    Tag = apps.get_model("documents", "Tag")
+    Correspondent = apps.get_model("documents", "Tag")
+
+    for klass in (Tag, Correspondent):
+        for instance in klass.objects.all():
+            klass.objects.filter(
+                pk=instance.pk
+            ).update(
+                slug=slugify(instance.slug)
+            )
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('documents', '0021_document_storage_type'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='tag',
+            options={'ordering': ('name',)},
+        ),
+        migrations.AlterField(
+            model_name='correspondent',
+            name='slug',
+            field=models.SlugField(blank=True, editable=False),
+        ),
+        migrations.AlterField(
+            model_name='document',
+            name='file_type',
+            field=models.CharField(choices=[('pdf', 'PDF'), ('png', 'PNG'), ('jpg', 'JPG'), ('gif', 'GIF'), ('tiff', 'TIFF'), ('txt', 'TXT'), ('csv', 'CSV'), ('md', 'MD')], editable=False, max_length=4),
+        ),
+        migrations.AlterField(
+            model_name='tag',
+            name='slug',
+            field=models.SlugField(blank=True, editable=False),
+        ),
+        migrations.RunPython(re_slug_all_the_things, migrations.RunPython.noop)
+    ]
diff --git a/src/documents/models.py b/src/documents/models.py
index c66bb5b0f..37c1cfdbf 100644
--- a/src/documents/models.py
+++ b/src/documents/models.py
@@ -11,6 +11,7 @@ from django.conf import settings
 from django.db import models
 from django.template.defaultfilters import slugify
 from django.utils import timezone
+from django.utils.text import slugify
 from fuzzywuzzy import fuzz
 
 from .managers import LogManager
@@ -37,7 +38,7 @@ class MatchingModel(models.Model):
     )
 
     name = models.CharField(max_length=128, unique=True)
-    slug = models.SlugField(blank=True)
+    slug = models.SlugField(blank=True, editable=False)
 
     match = models.CharField(max_length=256, blank=True)
     matching_algorithm = models.PositiveIntegerField(
@@ -147,9 +148,7 @@ class MatchingModel(models.Model):
     def save(self, *args, **kwargs):
 
         self.match = self.match.lower()
-
-        if not self.slug:
-            self.slug = slugify(self.name)
+        self.slug = slugify(self.name)
 
         models.Model.save(self, *args, **kwargs)
 
@@ -452,7 +451,7 @@ class FileInfo:
         r = []
         for t in tags.split(","):
             r.append(Tag.objects.get_or_create(
-                slug=t.lower(),
+                slug=slugify(t),
                 defaults={"name": t}
             )[0])
         return tuple(r)

From c570aa1a1006954e1cfd672aae86e4f29d6d142c Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 7 Oct 2018 16:26:05 +0100
Subject: [PATCH 31/47] Add a little more read-only info for documents

---
 src/documents/admin.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/documents/admin.py b/src/documents/admin.py
index 6dbe7f835..5a5a1a50c 100644
--- a/src/documents/admin.py
+++ b/src/documents/admin.py
@@ -171,7 +171,7 @@ class DocumentAdmin(CommonAdmin):
         }
 
     search_fields = ("correspondent__name", "title", "content", "tags__name")
-    readonly_fields = ("added",)
+    readonly_fields = ("added", "file_type", "storage_type",)
     list_display = ("title", "created", "added", "thumbnail", "correspondent",
                     "tags_")
     list_filter = (

From f32a188a2d42783895b9124cfd5bb556c1eca977 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 7 Oct 2018 16:26:23 +0100
Subject: [PATCH 32/47] Rework user hack for "login-free" sessions #394

---
 docs/changelog.rst      |  7 +++++++
 src/paperless/models.py | 19 ++++++++++++-------
 2 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index b446e45cf..db5fb6a15 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -10,6 +10,12 @@ Changelog
   been updated on the Docker Hub, so you just need to pull the latest one from
   there if you're a Docker user.
 
+* "Login free" instances of Paperless were breaking whenever you tried to edit
+  objects in the admin: adding/deleting tags or correspondents, or even fixing
+  spelling.  This was due to the "user hack" we were applying to sessions that
+  weren't using a login, as that hack user didn't have a valid id.  The fix was
+  to attribute the first user id in the system to this hack user.  `#394`_
+
 * A problem in how we handle slug values on Tags and Correspondents required a
   few changes to how we handle this field `#393`_:
 
@@ -648,6 +654,7 @@ bulk of the work on this big change.
 .. _#392: https://github.com/danielquinn/paperless/issues/392
 .. _#393: https://github.com/danielquinn/paperless/issues/393
 .. _#395: https://github.com/danielquinn/paperless/pull/395
+.. _#394: https://github.com/danielquinn/paperless/issues/394
 .. _#396: https://github.com/danielquinn/paperless/pull/396
 .. _#399: https://github.com/danielquinn/paperless/pull/399
 .. _#400: https://github.com/danielquinn/paperless/pull/400
diff --git a/src/paperless/models.py b/src/paperless/models.py
index 4001d3468..e390032db 100644
--- a/src/paperless/models.py
+++ b/src/paperless/models.py
@@ -1,15 +1,20 @@
+from django.contrib.auth.models import User as DjangoUser
+
+
 class User:
     """
-      This is a dummy django User used with our middleware to disable
-      login authentication if that is configured in paperless.conf
+    This is a dummy django User used with our middleware to disable
+    login authentication if that is configured in paperless.conf
     """
+
     is_superuser = True
     is_active = True
     is_staff = True
     is_authenticated = True
 
-    # Must be -1 to avoid colliding with real user ID's (which start at 1)
-    id = -1
+    @property
+    def id(self):
+        return DjangoUser.objects.order_by("pk").first().pk
 
     @property
     def pk(self):
@@ -17,9 +22,9 @@ class User:
 
 
 """
-  NOTE: These are here as a hack instead of being in the User definition
-  above due to the way pycodestyle handles lamdbdas.
-  See https://github.com/PyCQA/pycodestyle/issues/379 for more.
+NOTE: These are here as a hack instead of being in the User definition
+NOTE: above due to the way pycodestyle handles lamdbdas.
+NOTE: See https://github.com/PyCQA/pycodestyle/issues/379 for more.
 """
 
 User.has_module_perms = lambda *_: True

From eae2d241c99927c09ddb33974e503478f59d834e Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 7 Oct 2018 16:28:53 +0100
Subject: [PATCH 33/47] pep8

---
 src/documents/filters.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/documents/filters.py b/src/documents/filters.py
index d52889666..2b7c9cc9f 100644
--- a/src/documents/filters.py
+++ b/src/documents/filters.py
@@ -1,4 +1,4 @@
-from django_filters.rest_framework import CharFilter, FilterSet, BooleanFilter, ModelChoiceFilter
+from django_filters.rest_framework import BooleanFilter, FilterSet
 
 from .models import Correspondent, Document, Tag
 

From ff809d1265baf847e29425c81b64fd7ae7313a3d Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 7 Oct 2018 16:30:36 +0100
Subject: [PATCH 34/47] Version bump: 2.5.0

---
 src/paperless/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/paperless/version.py b/src/paperless/version.py
index 31ee9974c..eb0649352 100644
--- a/src/paperless/version.py
+++ b/src/paperless/version.py
@@ -1 +1 @@
-__version__ = (2, 4, 1)
+__version__ = (2, 5, 0)

From 7022c98aabeef479d76834a0461a4f5690f73b69 Mon Sep 17 00:00:00 2001
From: David Martin <david_martin@fastmail.com>
Date: Mon, 8 Oct 2018 19:12:11 +1100
Subject: [PATCH 35/47] Let unpaper overwrite temporary files.

I'm not sure what the circumstances are, but it looks like unpaper can
attempt to write a temporary file that already exists [0]. This then
fails the consumption. As per daedadu's comment simply letting unpaper
overwrite files fixes this.

[0]
unpaper: error: output file '/tmp/paperless/paperless-pjkrcr4l/convert-0000.unpaper.pnm' already present.
See https://web.archive.org/web/20181008081515/https://github.com/danielquinn/paperless/issues/406#issue-360651630
---
 src/paperless_tesseract/parsers.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/paperless_tesseract/parsers.py b/src/paperless_tesseract/parsers.py
index dc5dbd637..0139738be 100644
--- a/src/paperless_tesseract/parsers.py
+++ b/src/paperless_tesseract/parsers.py
@@ -218,7 +218,8 @@ def run_convert(*args):
 
 def run_unpaper(args):
     unpaper, pnm = args
-    command_args = unpaper, pnm, pnm.replace(".pnm", ".unpaper.pnm")
+    command_args = (unpaper, "--overwrite", pnm,
+                    pnm.replace(".pnm", ".unpaper.pnm"))
     if not subprocess.Popen(command_args).wait() == 0:
         raise ParseError("Unpaper failed at {}".format(command_args))
 

From b0afa37ec14913819614d245e93d947198f40dbe Mon Sep 17 00:00:00 2001
From: David Martin <david_martin@fastmail.com>
Date: Mon, 8 Oct 2018 19:37:05 +1100
Subject: [PATCH 36/47] Mention FORGIVING_OCR config option when language
 detection fails.

It is not obvious that the PAPERLESS_FORGIVING_OCR allows to let
document consumption happen even if no language can be detected.
Mentioning it in the actual error message in the log seems like the best
way to make it clear.
---
 src/paperless_tesseract/parsers.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/paperless_tesseract/parsers.py b/src/paperless_tesseract/parsers.py
index dc5dbd637..ffa2727e5 100644
--- a/src/paperless_tesseract/parsers.py
+++ b/src/paperless_tesseract/parsers.py
@@ -153,7 +153,10 @@ class RasterisedDocumentParser(DocumentParser):
                 )
                 raw_text = self._assemble_ocr_sections(imgs, middle, raw_text)
                 return raw_text
-            raise OCRError("Language detection failed")
+            error_msg = ("Language detection failed. Set "
+                         "PAPERLESS_FORGIVING_OCR in config file to continue "
+                         "anyway.")
+            raise OCRError(error_msg)
 
         if ISO639[guessed_language] == self.DEFAULT_OCR_LANGUAGE:
             raw_text = self._assemble_ocr_sections(imgs, middle, raw_text)

From 8cf32d2a5a4d51b68f39970443d8aece6270d499 Mon Sep 17 00:00:00 2001
From: David Martin <david_martin@fastmail.com>
Date: Mon, 8 Oct 2018 19:38:38 +1100
Subject: [PATCH 37/47] Add PAPERLESS_FORGIVING_OCR option to example config.

It helps having it in the example config as that makes it more clear
that it exists.
---
 paperless.conf.example | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/paperless.conf.example b/paperless.conf.example
index 3604505cb..11e6d905b 100644
--- a/paperless.conf.example
+++ b/paperless.conf.example
@@ -188,6 +188,11 @@ PAPERLESS_DEBUG="false"
 #PAPERLESS_CONSUMER_LOOP_TIME=10
 
 
+# By default Paperless stops consuming a document if no language can be detected.
+# Set to true to consume documents even if the language detection fails.
+#PAPERLESS_FORGIVING_OCR="false"
+
+
 ###############################################################################
 ####                            Interface                                  ####
 ###############################################################################

From 7f3f8fff671086dd4c9015a0fbb8f93a48fd635b Mon Sep 17 00:00:00 2001
From: Erik Arvstedt <erik.arvstedt@gmail.com>
Date: Mon, 8 Oct 2018 10:58:06 +0200
Subject: [PATCH 38/47] requirements.txt: bring back Linux-only restriction for
 inotify-simple

Fixes #418
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index b8c26a579..ac893b8a5 100755
--- a/requirements.txt
+++ b/requirements.txt
@@ -21,7 +21,7 @@ filemagic==1.6
 fuzzywuzzy==0.15.0
 gunicorn==19.9.0
 idna==2.7
-inotify-simple==1.1.8
+inotify-simple==1.1.8; sys_platform == 'linux'
 langdetect==1.0.7
 more-itertools==4.3.0
 pdftotext==2.1.0

From 06f9d4b1ea44fe9ca3c283a6a4afaac200f0eec7 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Mon, 8 Oct 2018 10:38:53 +0100
Subject: [PATCH 39/47] Add pip install to update process

---
 docs/migrating.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/migrating.rst b/docs/migrating.rst
index 45646f058..d4ef40857 100644
--- a/docs/migrating.rst
+++ b/docs/migrating.rst
@@ -82,6 +82,7 @@ rolled in as part of the update:
 
     $ cd /path/to/project
     $ git pull
+    $ pip install -r requirements.txt
     $ cd src
     $ ./manage.py migrate
 

From 7831991ecdfd422131bd8992de50bbbd119a727a Mon Sep 17 00:00:00 2001
From: Sharif Nassar <mrwacky42@users.noreply.github.com>
Date: Sat, 13 Oct 2018 11:31:53 -0700
Subject: [PATCH 40/47] Remove Vagrant docs

* Vagrant does not seem to have any libvirt boxes for Ubuntu any more.
* Vagrant 2 was released a year ago, but vagrant-libvirt only claims
  to support up to Vagrant 1.8.
---
 .gitignore                |  1 -
 Vagrantfile               | 20 -------------
 docs/requirements.rst     |  2 +-
 docs/setup.rst            | 61 +--------------------------------------
 docs/troubleshooting.rst  |  5 ++--
 scripts/vagrant-provision | 31 --------------------
 6 files changed, 4 insertions(+), 116 deletions(-)
 delete mode 100644 Vagrantfile
 delete mode 100644 scripts/vagrant-provision

diff --git a/.gitignore b/.gitignore
index 439d9df4b..cf485e15f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -73,7 +73,6 @@ db.sqlite3
 # Other stuff that doesn't belong
 .virtualenv
 virtualenv
-.vagrant
 docker-compose.yml
 docker-compose.env
 
diff --git a/Vagrantfile b/Vagrantfile
deleted file mode 100644
index 46caa38bc..000000000
--- a/Vagrantfile
+++ /dev/null
@@ -1,20 +0,0 @@
-# -*- mode: ruby -*-
-# vi: set ft=ruby :
-
-VAGRANT_API_VERSION = "2"
-Vagrant.configure(VAGRANT_API_VERSION) do |config|
-  config.vm.box = "ubuntu/trusty64"
-
-  # Provision using shell
-  config.vm.host_name = "dev.paperless"
-  config.vm.synced_folder ".", "/opt/paperless"
-  config.vm.provision "shell", path: "scripts/vagrant-provision"
-
-  # Networking details
-  config.vm.network "private_network", ip: "172.28.128.4"
-
-  config.vm.provider "virtualbox" do |vb|
-    # Customize the amount of memory on the VM:
-    vb.memory = "1024"
-  end
-end
diff --git a/docs/requirements.rst b/docs/requirements.rst
index ee42cb96a..b6cbad213 100644
--- a/docs/requirements.rst
+++ b/docs/requirements.rst
@@ -33,7 +33,7 @@ In addition to the above, there are a number of Python requirements, all of
 which are listed in a file called ``requirements.txt`` in the project root
 directory.
 
-If you're not working on a virtual environment (like Vagrant or Docker), you
+If you're not working on a virtual environment (like Docker), you
 should probably be using a virtualenv, but that's your call.  The reasons why
 you might choose a virtualenv or not aren't really within the scope of this
 document.  Needless to say if you don't know what a virtualenv is, you should
diff --git a/docs/setup.rst b/docs/setup.rst
index 2dcfeb901..d794c663b 100644
--- a/docs/setup.rst
+++ b/docs/setup.rst
@@ -42,18 +42,14 @@ Installation & Configuration
 You can go multiple routes with setting up and running Paperless:
 
  * The `bare metal route`_
- * The `vagrant route`_
  * The `docker route`_
 
 
-The `Vagrant route`_ is quick & easy, but means you're running a VM which comes
-with memory consumption, cpu overhead etc. The `docker route`_ offers the same
-simplicity as Vagrant with lower resource consumption.
+The `docker route`_ is quick & easy.
 
 The `bare metal route`_ is a bit more complicated to setup but makes it easier
 should you want to contribute some code back.
 
-.. _Vagrant route: setup-installation-vagrant_
 .. _docker route: setup-installation-docker_
 .. _bare metal route: setup-installation-bare-metal_
 .. _Docker Machine: https://docs.docker.com/machine/
@@ -267,54 +263,6 @@ Docker Method
    newer ``docker-compose.yml.example`` file
 
 
-.. _setup-installation-vagrant:
-
-Vagrant Method
-++++++++++++++
-
-1. Install `Vagrant`_.  How you do that is really between you and your OS.
-2. Run ``vagrant up``.  An instance will start up for you.  When it's ready and
-   provisioned...
-3. Run ``vagrant ssh`` and once inside your new vagrant box, edit
-   ``/etc/paperless.conf`` and set the values for:
-
-    * ``PAPERLESS_CONSUMPTION_DIR``: This is where your documents will be
-      dumped to be consumed by Paperless.
-    * ``PAPERLESS_PASSPHRASE``: This is the passphrase Paperless uses to
-      encrypt/decrypt the original document.  It's only required if you want
-      your original files to be encrypted, otherwise, just leave it unset.
-    * ``PAPERLESS_EMAIL_SECRET``: this is the "magic word" used when consuming
-      documents from mail or via the API.  If you don't use either, leaving it
-      blank is just fine.
-
-4. Exit the vagrant box and re-enter it with ``vagrant ssh`` again.  This
-   updates the environment to make use of the changes you made to the config
-   file.
-5. Initialise the database with ``/opt/paperless/src/manage.py migrate``.
-6. Still inside your vagrant box, create a user for your Paperless instance
-   with ``/opt/paperless/src/manage.py createsuperuser``. Follow the prompts to
-   create your user.
-7. Start the webserver with
-   ``/opt/paperless/src/manage.py runserver 0.0.0.0:8000``. You should now be
-   able to visit your (empty) `Paperless webserver`_ at ``172.28.128.4:8000``.
-   You can login with the user/pass you created in #6.
-8. In a separate window, run ``vagrant ssh`` again, but this time once inside
-   your vagrant instance, you should start the consumer script with
-   ``/opt/paperless/src/manage.py document_consumer``.
-9. Scan something.  Put it in the ``CONSUMPTION_DIR``.
-10. Wait a few minutes
-11. Visit the document list on your webserver, and it should be there, indexed
-    and downloadable.
-
-.. caution::
-
-    This installation is not secure. Once everything is working head up to
-    `Making things more permanent`_
-
-.. _Vagrant: https://vagrantup.com/
-.. _Paperless server: http://172.28.128.4:8000
-
-
 .. _setup-permanent:
 
 Making Things a Little more Permanent
@@ -513,13 +461,6 @@ second period.
 .. _Upstart: http://upstart.ubuntu.com/
 
 
-Vagrant
-~~~~~~~
-
-You may use the Ubuntu explanation above. Replace
-``(local-filesystems and net-device-up IFACE=eth0)`` with ``vagrant-mounted``.
-
-
 .. _setup-permanent-docker:
 
 Docker
diff --git a/docs/troubleshooting.rst b/docs/troubleshooting.rst
index 268235923..05b314004 100644
--- a/docs/troubleshooting.rst
+++ b/docs/troubleshooting.rst
@@ -14,9 +14,8 @@ FORGIVING_OCR is enabled``, then you might need to install the
 `Tesseract language files <http://packages.ubuntu.com/search?keywords=tesseract-ocr>`_
 marching your document's languages.
 
-As an example, if you are running Paperless from the Vagrant setup provided
-(or from any Ubuntu or Debian box), and your documents are written in Spanish
-you may need to run::
+As an example, if you are running Paperless from any Ubuntu or Debian
+box, and your documents are written in Spanish you may need to run::
 
     apt-get install -y tesseract-ocr-spa
 
diff --git a/scripts/vagrant-provision b/scripts/vagrant-provision
deleted file mode 100644
index 940bf476c..000000000
--- a/scripts/vagrant-provision
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/bin/bash
-
-# Install packages
-apt-get update
-apt-get build-dep -y python-imaging
-apt-get install -y libjpeg8 libjpeg62-dev libfreetype6 libfreetype6-dev
-apt-get install -y build-essential python3-dev python3-pip sqlite3 libsqlite3-dev git
-apt-get install -y tesseract-ocr tesseract-ocr-eng imagemagick unpaper
-
-# Python dependencies
-pip3 install -r /opt/paperless/requirements.txt
-
-# Create the environment file
-cat /opt/paperless/paperless.conf.example | sed -e 's#CONSUMPTION_DIR=""#CONSUMPTION_DIR="/home/vagrant/consumption"#' > /etc/paperless.conf
-chmod 0640 /etc/paperless.conf
-chown root:vagrant /etc/paperless.conf
-
-# Create the consumption directory
-mkdir /home/vagrant/consumption
-chown vagrant:vagrant /home/vagrant/consumption
-
-echo "
-
-
-Now follow the remaining steps in the Vagrant section of the setup
-documentation to complete the process:
-
-http://paperless.readthedocs.org/en/latest/setup.html#setup-installation-vagrant
-
-
-"

From 3f1cb0567bf8f50a9b4f70ba19b9f8d0b189c3f3 Mon Sep 17 00:00:00 2001
From: Dean Perry <dean@deanpcmad.com>
Date: Wed, 31 Oct 2018 12:39:48 +0000
Subject: [PATCH 41/47] added missing ; to nginx config

---
 docs/setup.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/setup.rst b/docs/setup.rst
index d794c663b..bb8ac4018 100644
--- a/docs/setup.rst
+++ b/docs/setup.rst
@@ -346,7 +346,7 @@ instance listening on localhost port 8000.
         location /static {
 
             autoindex on;
-            alias <path-to-paperless-static-directory>
+            alias <path-to-paperless-static-directory>;
 
         }
 
@@ -357,7 +357,7 @@ instance listening on localhost port 8000.
             proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
             proxy_set_header X-Forwarded-Proto $scheme;
 
-            proxy_pass http://127.0.0.1:8000
+            proxy_pass http://127.0.0.1:8000;
         }
     }
 

From f1a4141df5c92699f8aa8b0c53b3ee589d9ea723 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sat, 3 Nov 2018 10:25:51 +0000
Subject: [PATCH 42/47] Allow an infinite number of logs to be deleted.

---
 docs/changelog.rst        | 14 ++++++++++++--
 src/paperless/settings.py | 10 ++++++++++
 2 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index db5fb6a15..d2effb3ac 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -1,6 +1,13 @@
 Changelog
 #########
 
+2.6.0
+=====
+
+* Allow an infinite number of logs to be deleted.  Thanks to `Ulli`_ for noting
+  the problem in `#433`_.
+
+
 2.5.0
 =====
 
@@ -44,6 +51,7 @@ Changelog
 * The ``get_date()`` functionality of the parsers has been consolidated onto
   the ``DocumentParser`` class since much of that code was redundant anyway.
 
+
 2.4.0
 =====
 
@@ -55,13 +63,13 @@ Changelog
   It's now in the import step that we decide the storage type.  This allows you
   to export from an encrypted system and import into an unencrypted one, or
   vice-versa.
-* The migration history has been slightly modified to accomodate PostgreSQL
+* The migration history has been slightly modified to accommodate PostgreSQL
   users.  Additionally, you can now tell paperless to use PostgreSQL simply by
   declaring ``PAPERLESS_DBUSER`` in your environment.  This will attempt to
   connect to your Postgres database without a password unless you also set
   ``PAPERLESS_DBPASS``.
 * A bug was found in the REST API filter system that was the result of an
-  update of django-filter some time ago.  This has now been patched `#412`_.
+  update of django-filter some time ago.  This has now been patched in `#412`_.
   Thanks to `thepill`_ for spotting it!
 
 
@@ -570,6 +578,7 @@ bulk of the work on this big change.
 .. _thepill: https://github.com/thepill
 .. _Andrew Peng: https://github.com/pengc99
 .. _euri10: https://github.com/euri10
+.. _Ulli: https://github.com/Ulli2k
 
 .. _#20: https://github.com/danielquinn/paperless/issues/20
 .. _#44: https://github.com/danielquinn/paperless/issues/44
@@ -664,6 +673,7 @@ bulk of the work on this big change.
 .. _#412: https://github.com/danielquinn/paperless/issues/412
 .. _#413: https://github.com/danielquinn/paperless/pull/413
 .. _#414: https://github.com/danielquinn/paperless/issues/414
+.. _#433: https://github.com/danielquinn/paperless/issues/433
 
 .. _pipenv: https://docs.pipenv.org/
 .. _a new home on Docker Hub: https://hub.docker.com/r/danielquinn/paperless/
diff --git a/src/paperless/settings.py b/src/paperless/settings.py
index fb5a399a8..97226ef44 100644
--- a/src/paperless/settings.py
+++ b/src/paperless/settings.py
@@ -199,6 +199,16 @@ STATIC_URL = os.getenv("PAPERLESS_STATIC_URL", "/static/")
 MEDIA_URL = os.getenv("PAPERLESS_MEDIA_URL", "/media/")
 
 
+# Other
+
+# Disable Django's artificial limit on the number of form fields to submit at
+# once.  This is a protection against overloading the server, but since this is
+# a self-hosted sort of gig, the benefits of being able to mass-delete a tonne
+# of log entries outweight the benefits of such a safeguard.
+
+DATA_UPLOAD_MAX_NUMBER_FIELDS = None
+
+
 # Paperless-specific stuff
 # You shouldn't have to edit any of these values.  Rather, you can set these
 # values in /etc/paperless.conf instead.

From 9264e89bc34cb7367c67bc97fb4552bafdd1deb7 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sat, 3 Nov 2018 11:05:22 +0000
Subject: [PATCH 43/47] Code cleanup

---
 src/documents/admin.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/documents/admin.py b/src/documents/admin.py
index 5a5a1a50c..6c0424a74 100644
--- a/src/documents/admin.py
+++ b/src/documents/admin.py
@@ -61,12 +61,12 @@ class FinancialYearFilter(admin.SimpleListFilter):
 
             # To keep it simple we use the same string for both
             # query parameter and the display.
-            return (query, query)
+            return query, query
 
         else:
             query = "{0}-{0}".format(date.year)
             display = "{}".format(date.year)
-            return (query, display)
+            return query, display
 
     def lookups(self, request, model_admin):
         if not settings.FY_START or not settings.FY_END:
@@ -146,8 +146,8 @@ class CorrespondentAdmin(CommonAdmin):
 
 class TagAdmin(CommonAdmin):
 
-    list_display = ("name", "colour", "match", "matching_algorithm",
-                    "document_count")
+    list_display = (
+        "name", "colour", "match", "matching_algorithm", "document_count")
     list_filter = ("colour", "matching_algorithm")
     list_editable = ("colour", "match", "matching_algorithm")
 

From dd3012b61173fe0511640e128637b67c1cb8c057 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sat, 3 Nov 2018 11:06:55 +0000
Subject: [PATCH 44/47] Fix the correspondent filters #423

---
 docs/changelog.rst     |  6 ++++++
 src/documents/admin.py | 20 +++++++++-----------
 2 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index d2effb3ac..e98436382 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -6,6 +6,9 @@ Changelog
 
 * Allow an infinite number of logs to be deleted.  Thanks to `Ulli`_ for noting
   the problem in `#433`_.
+* Fix the ``RecentCorrespondentsFilter`` correspondents filter that was added
+  in 2.4 to play nice with the defaults.  Thanks to `tsia`_ and `Sblop`_ who
+  pointed this out. `#423`_.
 
 
 2.5.0
@@ -579,6 +582,8 @@ bulk of the work on this big change.
 .. _Andrew Peng: https://github.com/pengc99
 .. _euri10: https://github.com/euri10
 .. _Ulli: https://github.com/Ulli2k
+.. _tsia: https://github.com/tsia
+.. _Sblop:  https://github.com/Sblop
 
 .. _#20: https://github.com/danielquinn/paperless/issues/20
 .. _#44: https://github.com/danielquinn/paperless/issues/44
@@ -673,6 +678,7 @@ bulk of the work on this big change.
 .. _#412: https://github.com/danielquinn/paperless/issues/412
 .. _#413: https://github.com/danielquinn/paperless/pull/413
 .. _#414: https://github.com/danielquinn/paperless/issues/414
+.. _#423: https://github.com/danielquinn/paperless/issues/423
 .. _#433: https://github.com/danielquinn/paperless/issues/433
 
 .. _pipenv: https://docs.pipenv.org/
diff --git a/src/documents/admin.py b/src/documents/admin.py
index 6c0424a74..e61d14815 100644
--- a/src/documents/admin.py
+++ b/src/documents/admin.py
@@ -88,25 +88,24 @@ class FinancialYearFilter(admin.SimpleListFilter):
 
 
 class RecentCorrespondentFilter(admin.RelatedFieldListFilter):
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.title = "correspondent (recent)"
+    """
+    If PAPERLESS_RECENT_CORRESPONDENT_YEARS is set, we limit the available
+    correspondents to documents sent our way over the past ``n`` years.
+    """
 
     def field_choices(self, field, request, model_admin):
 
         years = settings.PAPERLESS_RECENT_CORRESPONDENT_YEARS
-        days = 365 * years
+        correspondents = Correspondent.objects.all()
 
-        lookups = []
         if years and years > 0:
-            correspondents = Correspondent.objects.filter(
+            self.title = "Correspondent (Recent)"
+            days = 365 * years
+            correspondents = correspondents.filter(
                 documents__created__gte=datetime.now() - timedelta(days=days)
             ).distinct()
-            for c in correspondents:
-                lookups.append((c.id, c.name))
 
-        return lookups
+        return [(c.id, c.name) for c in correspondents]
 
 
 class CommonAdmin(admin.ModelAdmin):
@@ -177,7 +176,6 @@ class DocumentAdmin(CommonAdmin):
     list_filter = (
         "tags",
         ("correspondent", RecentCorrespondentFilter),
-        "correspondent",
         FinancialYearFilter
     )
 

From f0de4fdd462b4d280f61d1f7bd14c781a9276378 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sat, 3 Nov 2018 12:49:35 +0000
Subject: [PATCH 45/47] Update dependencies

This includes a security update for requests.
---
 Pipfile            |  11 +-
 Pipfile.lock       | 552 +++++++++++++++++++++++----------------------
 docs/changelog.rst |   2 +
 requirements.txt   |  82 +++----
 4 files changed, 333 insertions(+), 314 deletions(-)

diff --git a/Pipfile b/Pipfile
index b1c30698d..778d15546 100644
--- a/Pipfile
+++ b/Pipfile
@@ -25,6 +25,11 @@ python-dateutil = "*"
 python-dotenv = "*"
 python-gnupg = "*"
 pytz = "*"
+
+[dev-packages]
+ipython = "*"
+sphinx = "*"
+tox = "*"
 pycodestyle = "*"
 pytest = "*"
 pytest-cov = "*"
@@ -32,9 +37,3 @@ pytest-django = "*"
 pytest-sugar = "*"
 pytest-env = "*"
 pytest-xdist = "*"
-
-[dev-packages]
-ipython = "*"
-sphinx = "*"
-tox = "*"
-
diff --git a/Pipfile.lock b/Pipfile.lock
index 71a46d37f..966fbd467 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "6d8bad24aa5d0c102b13b5ae27acba04836cd5a07a4003cb2763de1e0a3406b7"
+            "sha256": "eeeeaf6ecb0cec45a3962eda6647b5263e5ad7939fae29d4b294f59ffe9ca3dd"
         },
         "pipfile-spec": 6,
         "requires": {},
@@ -14,35 +14,12 @@
         ]
     },
     "default": {
-        "apipkg": {
-            "hashes": [
-                "sha256:37228cda29411948b422fae072f57e31d3396d2ee1c9783775980ee9c9990af6",
-                "sha256:58587dd4dc3daefad0487f6d9ae32b4542b185e1c36db6993290e7c41ca2b47c"
-            ],
-            "markers": "python_version != '3.2.*' and python_version >= '2.7' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.3.*'",
-            "version": "==1.5"
-        },
-        "atomicwrites": {
-            "hashes": [
-                "sha256:0312ad34fcad8fac3704d441f7b317e50af620823353ec657a53e981f92920c0",
-                "sha256:ec9ae8adaae229e4f8446952d204a3e4b5fdd2d099f9be3aaf556120135fb3ee"
-            ],
-            "markers": "python_version != '3.2.*' and python_version >= '2.7' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.3.*'",
-            "version": "==1.2.1"
-        },
-        "attrs": {
-            "hashes": [
-                "sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69",
-                "sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb"
-            ],
-            "version": "==18.2.0"
-        },
         "certifi": {
             "hashes": [
-                "sha256:376690d6f16d32f9d1fe8932551d80b23e9d393a8578c5633a2ed39a64861638",
-                "sha256:456048c7e371c089d0a77a5212fb37a2c2dce1e24146e3b7e0261736aaeaa22a"
+                "sha256:339dc09518b07e2fa7eda5450740925974815557727d6bd35d319c1524a04a4c",
+                "sha256:6d58c986d22b038c8c0df30d639f23a3e6d172a05c3583e766f4c0b785c0986a"
             ],
-            "version": "==2018.8.24"
+            "version": "==2018.10.15"
         },
         "chardet": {
             "hashes": [
@@ -55,6 +32,7 @@
             "hashes": [
                 "sha256:03481e81d558d30d230bc12999e3edffe392d244349a90f4ef9b88425fac74ba",
                 "sha256:0b136648de27201056c1869a6c0d4e23f464750fd9a9ba9750b8336a244429ed",
+                "sha256:0bf8cbbd71adfff0ef1f3a1531e6402d13b7b01ac50a79c97ca15f030dba6306",
                 "sha256:10a46017fef60e16694a30627319f38a2b9b52e90182dddb6e37dcdab0f4bf95",
                 "sha256:198626739a79b09fa0a2f06e083ffd12eb55449b5f8bfdbeed1df4910b2ca640",
                 "sha256:23d341cdd4a0371820eb2b0bd6b88f5003a7438bbedb33688cd33b8eae59affd",
@@ -83,18 +61,18 @@
                 "sha256:c1bb572fab8208c400adaf06a8133ac0712179a334c09224fb11393e920abcdd",
                 "sha256:de4418dadaa1c01d497e539210cb6baa015965526ff5afc078c57ca69160108d",
                 "sha256:e05cb4d9aad6233d67e0541caa7e511fa4047ed7750ec2510d466e806e0255d6",
+                "sha256:f05a636b4564104120111800021a92e43397bc12a5c72fed7036be8556e0029e",
                 "sha256:f3f501f345f24383c0000395b26b726e46758b71393267aeae0bd36f8b3ade80"
             ],
-            "markers": "python_version >= '2.6' and python_version != '3.2.*' and python_version != '3.0.*' and python_version != '3.1.*' and python_version < '4'",
             "version": "==4.5.1"
         },
         "coveralls": {
             "hashes": [
-                "sha256:9dee67e78ec17b36c52b778247762851c8e19a893c9a14e921a2fc37f05fac22",
-                "sha256:aec5a1f5e34224b9089664a1b62217732381c7de361b6ed1b3c394d7187b352a"
+                "sha256:ab638e88d38916a6cedbf80a9cd8992d5fa55c77ab755e262e00b36792b7cd6d",
+                "sha256:b2388747e2529fa4c669fb1e3e2756e4e07b6ee56c7d9fce05f35ccccc913aa0"
             ],
             "index": "pypi",
-            "version": "==1.5.0"
+            "version": "==1.5.1"
         },
         "dateparser": {
             "hashes": [
@@ -106,11 +84,11 @@
         },
         "django": {
             "hashes": [
-                "sha256:0c5b65847d00845ee404bbc0b4a85686f15eb3001ffddda3db4e9baa265bf136",
-                "sha256:68aeea369a8130259354b6ba1fa9babe0c5ee6bced505dea4afcd00f765ae38b"
+                "sha256:25df265e1fdb74f7e7305a1de620a84681bcc9c05e84a3ed97e4a1a63024f18d",
+                "sha256:d6d94554abc82ca37e447c3d28958f5ac39bd7d4adaa285543ae97fb1129fd69"
             ],
             "index": "pypi",
-            "version": "==2.0.8"
+            "version": "==2.0.9"
         },
         "django-cors-headers": {
             "hashes": [
@@ -130,11 +108,11 @@
         },
         "django-extensions": {
             "hashes": [
-                "sha256:1f626353a11479014bfe0d77e76d8f866ebca1bb5d595cb57b776230b9e0eb92",
-                "sha256:f21b898598a1628cb73017fb9672e2c5e624133be9764f0eb138e0abf8a62b62"
+                "sha256:30cb6a8c7d6f75a55edf0c0c4491bd98f8264ae1616ce105f9cecac4387edd07",
+                "sha256:4ad86a7a5e84f1c77db030761ae87a600647250c652030a2b71a16e87f3a3d62"
             ],
             "index": "pypi",
-            "version": "==2.1.2"
+            "version": "==2.1.3"
         },
         "django-filter": {
             "hashes": [
@@ -146,11 +124,11 @@
         },
         "djangorestframework": {
             "hashes": [
-                "sha256:b6714c3e4b0f8d524f193c91ecf5f5450092c2145439ac2769711f7eba89a9d9",
-                "sha256:c375e4f95a3a64fccac412e36fb42ba36881e52313ec021ef410b40f67cddca4"
+                "sha256:607865b0bb1598b153793892101d881466bd5a991de12bd6229abb18b1c86136",
+                "sha256:63f76cbe1e7d12b94c357d7e54401103b2e52aef0f7c1650d6c820ad708776e5"
             ],
             "index": "pypi",
-            "version": "==3.8.2"
+            "version": "==3.9.0"
         },
         "docopt": {
             "hashes": [
@@ -158,14 +136,6 @@
             ],
             "version": "==0.6.2"
         },
-        "execnet": {
-            "hashes": [
-                "sha256:a7a84d5fa07a089186a329528f127c9d73b9de57f1a1131b82bb5320ee651f6a",
-                "sha256:fc155a6b553c66c838d1a22dba1dc9f5f505c43285a878c6f74a79c024750b83"
-            ],
-            "markers": "python_version != '3.2.*' and python_version >= '2.7' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.3.*'",
-            "version": "==1.5.0"
-        },
         "factory-boy": {
             "hashes": [
                 "sha256:6f25cc4761ac109efd503f096e2ad99421b1159f01a29dbb917359dcd68e08ca",
@@ -176,11 +146,10 @@
         },
         "faker": {
             "hashes": [
-                "sha256:ea7cfd3aeb1544732d08bd9cfba40c5b78e3a91e17b1a0698ab81bfc5554c628",
-                "sha256:f6d67f04abfb2b4bea7afc7fa6c18cf4c523a67956e455668be9ae42bccc21ad"
+                "sha256:2621643b80a10b91999925cfd20f64d2b36f20bf22136bbdc749bb57d6ffe124",
+                "sha256:5ed822d31bd2d6edf10944d176d30dc9c886afdd381eefb7ba8b7aad86171646"
             ],
-            "markers": "python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.2.*' and python_version >= '2.7'",
-            "version": "==0.9.0"
+            "version": "==0.9.2"
         },
         "filemagic": {
             "hashes": [
@@ -190,12 +159,14 @@
             "version": "==1.6"
         },
         "fuzzywuzzy": {
+            "extras": [
+                "speedup"
+            ],
             "hashes": [
                 "sha256:3759bc6859daa0eecef8c82b45404bdac20c23f23136cf4c18b46b426bbc418f",
                 "sha256:5b36957ccf836e700f4468324fa80ba208990385392e217be077d5cd738ae602"
             ],
             "index": "pypi",
-            "markers": null,
             "version": "==0.15.0"
         },
         "gunicorn": {
@@ -227,80 +198,48 @@
             "index": "pypi",
             "version": "==1.0.7"
         },
-        "more-itertools": {
-            "hashes": [
-                "sha256:c187a73da93e7a8acc0001572aebc7e3c69daf7bf6881a2cea10650bd4420092",
-                "sha256:c476b5d3a34e12d40130bc2f935028b5f636df8f372dc2c1c01dc19681b2039e",
-                "sha256:fcbfeaea0be121980e15bc97b3817b5202ca73d0eae185b4550cbfce2a3ebb3d"
-            ],
-            "version": "==4.3.0"
-        },
         "pdftotext": {
             "hashes": [
-                "sha256:b7312302007e19fc784263a321b41682f01a582af84e14200cef53b3f4e69a50"
+                "sha256:e3ad11efe0aa22cbfc46aa1296b2ea5a52ad208b778288311f2801adef178ccb"
             ],
             "index": "pypi",
-            "version": "==2.1.0"
+            "version": "==2.1.1"
         },
         "pillow": {
             "hashes": [
-                "sha256:00def5b638994f888d1058e4d17c86dec8e1113c3741a0a8a659039aec59a83a",
-                "sha256:026449b64e559226cdb8e6d8c931b5965d8fc90ec18ebbb0baa04c5b36503c72",
-                "sha256:03dbb224ee196ef30ed2156d41b579143e1efeb422974719a5392fc035e4f574",
-                "sha256:03eb0e04f929c102ae24bc436bf1c0c60a4e63b07ebd388e84d8b219df3e6acd",
-                "sha256:1be66b9a89e367e7d20d6cae419794997921fe105090fafd86ef39e20a3baab2",
-                "sha256:1e977a3ed998a599bda5021fb2c2889060617627d3ae228297a529a082a3cd5c",
-                "sha256:22cf3406d135cfcc13ec6228ade774c8461e125c940e80455f500638429be273",
-                "sha256:24adccf1e834f82718c7fc8e3ec1093738da95144b8b1e44c99d5fc7d3e9c554",
-                "sha256:2a3e362c97a5e6a259ee9cd66553292a1f8928a5bdfa3622fdb1501570834612",
-                "sha256:3832e26ecbc9d8a500821e3a1d3765bda99d04ae29ffbb2efba49f5f788dc934",
-                "sha256:4fd1f0c2dc02aaec729d91c92cd85a2df0289d88e9f68d1e8faba750bb9c4786",
-                "sha256:4fda62030f2c515b6e2e673c57caa55cb04026a81968f3128aae10fc28e5cc27",
-                "sha256:5044d75a68b49ce36a813c82d8201384207112d5d81643937fc758c05302f05b",
-                "sha256:522184556921512ec484cb93bd84e0bab915d0ac5a372d49571c241a7f73db62",
-                "sha256:5914cff11f3e920626da48e564be6818831713a3087586302444b9c70e8552d9",
-                "sha256:6661a7908d68c4a133e03dac8178287aa20a99f841ea90beeb98a233ae3fd710",
-                "sha256:79258a8df3e309a54c7ef2ef4a59bb8e28f7e4a8992a3ad17c24b1889ced44f3",
-                "sha256:7d74c20b8f1c3e99d3f781d3b8ff5abfefdd7363d61e23bdeba9992ff32cc4b4",
-                "sha256:81918afeafc16ba5d9d0d4e9445905f21aac969a4ebb6f2bff4b9886da100f4b",
-                "sha256:8194d913ca1f459377c8a4ed8f9b7ad750068b8e0e3f3f9c6963fcc87a84515f",
-                "sha256:84d5d31200b11b3c76fab853b89ac898bf2d05c8b3da07c1fcc23feb06359d6e",
-                "sha256:989981db57abffb52026b114c9a1f114c7142860a6d30a352d28f8cbf186500b",
-                "sha256:a3d7511d3fad1618a82299aab71a5fceee5c015653a77ffea75ced9ef917e71a",
-                "sha256:b3ef168d4d6fd4fa6685aef7c91400f59f7ab1c0da734541f7031699741fb23f",
-                "sha256:c1c5792b6e74bbf2af0f8e892272c2a6c48efa895903211f11b8342e03129fea",
-                "sha256:c5dcb5a56aebb8a8f2585042b2f5c496d7624f0bcfe248f0cc33ceb2fd8d39e7",
-                "sha256:e2bed4a04e2ca1050bb5f00865cf2f83c0b92fd62454d9244f690fcd842e27a4",
-                "sha256:e87a527c06319428007e8c30511e1f0ce035cb7f14bb4793b003ed532c3b9333",
-                "sha256:f63e420180cbe22ff6e32558b612e75f50616fc111c5e095a4631946c782e109",
-                "sha256:f8b3d413c5a8f84b12cd4c5df1d8e211777c9852c6be3ee9c094b626644d3eab"
+                "sha256:00203f406818c3f45d47bb8fe7e67d3feddb8dcbbd45a289a1de7dd789226360",
+                "sha256:0616f800f348664e694dddb0b0c88d26761dd5e9f34e1ed7b7a7d2da14b40cb7",
+                "sha256:1f7908aab90c92ad85af9d2fec5fc79456a89b3adcc26314d2cde0e238bd789e",
+                "sha256:2ea3517cd5779843de8a759c2349a3cd8d3893e03ab47053b66d5ec6f8bc4f93",
+                "sha256:48a9f0538c91fc136b3a576bee0e7cd174773dc9920b310c21dcb5519722e82c",
+                "sha256:5280ebc42641a1283b7b1f2c20e5b936692198b9dd9995527c18b794850be1a8",
+                "sha256:5e34e4b5764af65551647f5cc67cf5198c1d05621781d5173b342e5e55bf023b",
+                "sha256:63b120421ab85cad909792583f83b6ca3584610c2fe70751e23f606a3c2e87f0",
+                "sha256:696b5e0109fe368d0057f484e2e91717b49a03f1e310f857f133a4acec9f91dd",
+                "sha256:870ed021a42b1b02b5fe4a739ea735f671a84128c0a666c705db2cb9abd528eb",
+                "sha256:916da1c19e4012d06a372127d7140dae894806fad67ef44330e5600d77833581",
+                "sha256:9303a289fa0811e1c6abd9ddebfc770556d7c3311cb2b32eff72164ddc49bc64",
+                "sha256:9577888ecc0ad7d06c3746afaba339c94d62b59da16f7a5d1cff9e491f23dace",
+                "sha256:987e1c94a33c93d9b209315bfda9faa54b8edfce6438a1e93ae866ba20de5956",
+                "sha256:99a3bbdbb844f4fb5d6dd59fac836a40749781c1fa63c563bc216c27aef63f60",
+                "sha256:99db8dc3097ceafbcff9cb2bff384b974795edeb11d167d391a02c7bfeeb6e16",
+                "sha256:a5a96cf49eb580756a44ecf12949e52f211e20bffbf5a95760ac14b1e499cd37",
+                "sha256:aa6ca3eb56704cdc0d876fc6047ffd5ee960caad52452fbee0f99908a141a0ae",
+                "sha256:aade5e66795c94e4a2b2624affeea8979648d1b0ae3fcee17e74e2c647fc4a8a",
+                "sha256:b78905860336c1d292409e3df6ad39cc1f1c7f0964e66844bbc2ebfca434d073",
+                "sha256:b92f521cdc4e4a3041cc343625b699f20b0b5f976793fb45681aac1efda565f8",
+                "sha256:bfde84bbd6ae5f782206d454b67b7ee8f7f818c29b99fd02bf022fd33bab14cb",
+                "sha256:c2b62d3df80e694c0e4a0ed47754c9480521e25642251b3ab1dff050a4e60409",
+                "sha256:c5e2be6c263b64f6f7656e23e18a4a9980cffc671442795682e8c4e4f815dd9f",
+                "sha256:c99aa3c63104e0818ec566f8ff3942fb7c7a8f35f9912cb63fd8e12318b214b2",
+                "sha256:dae06620d3978da346375ebf88b9e2dd7d151335ba668c995aea9ed07af7add4",
+                "sha256:db5499d0710823fa4fb88206050d46544e8f0e0136a9a5f5570b026584c8fd74",
+                "sha256:f36baafd82119c4a114b9518202f2a983819101dcc14b26e43fc12cbefdce00e",
+                "sha256:f52b79c8796d81391ab295b04e520bda6feed54d54931708872e8f9ae9db0ea1",
+                "sha256:ff8cff01582fa1a7e533cb97f628531c4014af4b5f38e33cdcfe5eec29b6d888"
             ],
             "index": "pypi",
-            "version": "==5.2.0"
-        },
-        "pluggy": {
-            "hashes": [
-                "sha256:6e3836e39f4d36ae72840833db137f7b7d35105079aee6ec4a62d9f80d594dd1",
-                "sha256:95eb8364a4708392bae89035f45341871286a333f749c3141c20573d2b3876e1"
-            ],
-            "markers": "python_version != '3.2.*' and python_version >= '2.7' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.3.*'",
-            "version": "==0.7.1"
-        },
-        "py": {
-            "hashes": [
-                "sha256:06a30435d058473046be836d3fc4f27167fd84c45b99704f2fb5509ef61f9af1",
-                "sha256:50402e9d1c9005d759426988a492e0edaadb7f4e68bcddfea586bc7432d009c6"
-            ],
-            "markers": "python_version != '3.2.*' and python_version >= '2.7' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.3.*'",
-            "version": "==1.6.0"
-        },
-        "pycodestyle": {
-            "hashes": [
-                "sha256:cbc619d09254895b0d12c2c691e237b2e91e9b2ecf5e84c26b35400f93dcfb83",
-                "sha256:cbfca99bd594a10f674d0cd97a3d802a1fdef635d4361e1a2658de47ed261e3a"
-            ],
-            "index": "pypi",
-            "version": "==2.4.0"
+            "version": "==5.3.0"
         },
         "pyocr": {
             "hashes": [
@@ -309,67 +248,13 @@
             "index": "pypi",
             "version": "==0.5.3"
         },
-        "pytest": {
-            "hashes": [
-                "sha256:453cbbbe5ce6db38717d282b758b917de84802af4288910c12442984bde7b823",
-                "sha256:a8a07f84e680482eb51e244370aaf2caa6301ef265f37c2bdefb3dd3b663f99d"
-            ],
-            "index": "pypi",
-            "version": "==3.8.0"
-        },
-        "pytest-cov": {
-            "hashes": [
-                "sha256:513c425e931a0344944f84ea47f3956be0e416d95acbd897a44970c8d926d5d7",
-                "sha256:e360f048b7dae3f2f2a9a4d067b2dd6b6a015d384d1577c994a43f3f7cbad762"
-            ],
-            "index": "pypi",
-            "version": "==2.6.0"
-        },
-        "pytest-django": {
-            "hashes": [
-                "sha256:2d2e0a618d91c280d463e90bcbea9b4e417609157f611a79685b1c561c4c0836",
-                "sha256:59683def396923b78d7e191a7086a48193f8d5db869ace79acb38f906522bc7b"
-            ],
-            "index": "pypi",
-            "version": "==3.4.2"
-        },
-        "pytest-env": {
-            "hashes": [
-                "sha256:7e94956aef7f2764f3c147d216ce066bf6c42948bb9e293169b1b1c880a580c2"
-            ],
-            "index": "pypi",
-            "version": "==0.6.2"
-        },
-        "pytest-forked": {
-            "hashes": [
-                "sha256:e4500cd0509ec4a26535f7d4112a8cc0f17d3a41c29ffd4eab479d2a55b30805",
-                "sha256:f275cb48a73fc61a6710726348e1da6d68a978f0ec0c54ece5a5fae5977e5a08"
-            ],
-            "markers": "python_version != '3.2.*' and python_version >= '2.7' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.3.*'",
-            "version": "==0.2"
-        },
-        "pytest-sugar": {
-            "hashes": [
-                "sha256:ab8cc42faf121344a4e9b13f39a51257f26f410e416c52ea11078cdd00d98a2c"
-            ],
-            "index": "pypi",
-            "version": "==0.9.1"
-        },
-        "pytest-xdist": {
-            "hashes": [
-                "sha256:0875deac20f6d96597036bdf63970887a6f36d28289c2f6682faf652dfea687b",
-                "sha256:28e25e79698b2662b648319d3971c0f9ae0e6500f88258ccb9b153c31110ba9b"
-            ],
-            "index": "pypi",
-            "version": "==1.23.0"
-        },
         "python-dateutil": {
             "hashes": [
-                "sha256:1adb80e7a782c12e52ef9a8182bebeb73f1d7e24e374397af06fb4956c8dc5c0",
-                "sha256:e27001de32f627c22380a688bcc43ce83504a7bc5da472209b4c70f02829f0b8"
+                "sha256:063df5763652e21de43de7d9e00ccf239f953a832941e37be541614732cdfc93",
+                "sha256:88f9287c0174266bb0d8cedd395cfba9c58e87e5ad86b2ce58859bc11be3cf02"
             ],
             "index": "pypi",
-            "version": "==2.7.3"
+            "version": "==2.7.5"
         },
         "python-dotenv": {
             "hashes": [
@@ -391,42 +276,37 @@
             "hashes": [
                 "sha256:033a11de5e3d19ea25c9302d11224e1a1898fe5abd23c61c7c360c25195e3eb1"
             ],
+            "markers": "extra == 'speedup'",
             "version": "==0.12.0"
         },
         "pytz": {
             "hashes": [
-                "sha256:a061aa0a9e06881eb8b3b2b43f05b9439d6583c206d0a6c340ff72a7b6669053",
-                "sha256:ffb9ef1de172603304d9d2819af6f5ece76f2e85ec10692a524dd876e72bf277"
+                "sha256:31cb35c89bd7d333cd32c5f278fca91b523b0834369e757f4c5641ea252236ca",
+                "sha256:8e0f8568c118d3077b46be7d654cc8167fa916092e28320cde048e54bfc9f1e6"
             ],
             "index": "pypi",
-            "version": "==2018.5"
+            "version": "==2018.7"
         },
         "regex": {
             "hashes": [
-                "sha256:22d7ef8c2df344328a8a3c61edade2ee714e5de9360911d22a9213931c769faa",
-                "sha256:3a699780c6b712c67dc23207b129ccc6a7e1270233f7aadead3ea3f83c893702",
-                "sha256:42f460d349baebd5faec02a0c920988fb0300b24baf898d9c139886565b66b6c",
-                "sha256:43bf3d79940cbdf19adda838d8b26b28b47bec793cda46590b5b25703742f440",
-                "sha256:47d6c7f0588ef33464e00023067c4e7cce68e0d6a686a73c7ee15abfdad503d4",
-                "sha256:5b879f59f25ed9b91bc8693a9a994014b431f224f492519ad0255ce6b54b83e5",
-                "sha256:8ba0093c412900f636b0f826c597a0c3ea0e395344bc99894ddefe88b76c9c7e",
-                "sha256:a4789254a1a0bd7a637036cce0b7ed72d8cc864e93f2e9cfd10ac00ae27bb7b0",
-                "sha256:b73cea07117dca888b0c3671770b501bef19aac9c45c8ffdb5bea2cca2377b0a",
-                "sha256:d3eb59fa3e5b5438438ec97acd9dc86f077428e020b015b43987e35bea68ef4c",
-                "sha256:d51d232b4e2f106deaf286001f563947fee255bc5bd209a696f027e15cf0a1e7",
-                "sha256:d59b03131a8e35061b47a8f186324a95eaf30d5f6ee9cc0637e7b87d29c7c9b5",
-                "sha256:dd705df1b47470388fc4630e4df3cbbe7677e2ab80092a1c660cae630a307b2d",
-                "sha256:e87fffa437a4b00afb17af785da9b01618425d6cd984c677639deb937037d8f2",
-                "sha256:ed40e0474ab5ab228a8d133759d451b31d3ccdebaff698646e54aff82c3de4f8"
+                "sha256:0ef96690c3d2294155b7d44187ca4a151e45c931cb768e106ba464a9fa64c5da",
+                "sha256:251683e01a3bcacd9188acf0d4caf7b29a3b963c843159311825613ae144cddb",
+                "sha256:3fe15a75fe00f04d1ec16713d55cf1e206077c450267a10b33318756fb8b3f99",
+                "sha256:53a962f9dc28cdf403978a142cb1e054479759ad64d312a999f9f042c25b5c9a",
+                "sha256:8bd1da6a93d32336a5e5432886dd8543004f0591c39b83dbfa60705cccdf414d",
+                "sha256:b5423061918f602e9342b54d746ac31c598d328ecaf4ef0618763e960c926fd4",
+                "sha256:d80ebc65b1f7d0403117f59309c16eac24be6a0bc730b593a79f703462858d94",
+                "sha256:fd8419979639b7de7fb964a13bce3ac47e6fe33043b83de0398c3067986e5659",
+                "sha256:ff2f15b2b0b4b58ba8a1de651780a0d3fd54f96ad6b77dceb77695220e5d7b7a"
             ],
-            "version": "==2018.8.29"
+            "version": "==2018.11.2"
         },
         "requests": {
             "hashes": [
-                "sha256:63b52e3c866428a224f97cab011de738c36aec0185aa91cfacd418b5d58911d1",
-                "sha256:ec22d826a36ed72a7358ff3fe56cbd4ba69dd7a6718ffd450ff0e9df7a47ce6a"
+                "sha256:99dcfdaaeb17caf6e526f32b6a7b780461512ab3f1d992187801694cba42770c",
+                "sha256:a84b8c9ab6239b578f22d1c21d51b696dcfe004032bb80ea832398d6909d7279"
             ],
-            "version": "==2.19.1"
+            "version": "==2.20.0"
         },
         "six": {
             "hashes": [
@@ -435,12 +315,6 @@
             ],
             "version": "==1.11.0"
         },
-        "termcolor": {
-            "hashes": [
-                "sha256:1d6d69ce66211143803fbc56652b41d73b4a400a2891d7bf7a1cdf4c02de613b"
-            ],
-            "version": "==1.1.0"
-        },
         "text-unidecode": {
             "hashes": [
                 "sha256:5a1375bb2ba7968740508ae38d92e1f889a0832913cb1c447d5e2046061a396d",
@@ -456,20 +330,40 @@
         },
         "urllib3": {
             "hashes": [
-                "sha256:a68ac5e15e76e7e5dd2b8f94007233e01effe3e50e8daddf69acfd81cb686baf",
-                "sha256:b5725a0bd4ba422ab0e66e89e030c806576753ea3ee08554382c14e685d117b5"
+                "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39",
+                "sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22"
             ],
-            "markers": "python_version >= '2.6' and python_version != '3.2.*' and python_version != '3.0.*' and python_version != '3.1.*' and python_version < '4' and python_version != '3.3.*'",
-            "version": "==1.23"
+            "version": "==1.24.1"
         }
     },
     "develop": {
         "alabaster": {
             "hashes": [
-                "sha256:674bb3bab080f598371f4443c5008cbfeb1a5e622dd312395d2d82af2c54c456",
-                "sha256:b63b1f4dc77c074d386752ec4a8a7517600f6c0db8cd42980cae17ab7b3275d7"
+                "sha256:446438bdcca0e05bd45ea2de1668c1d9b032e1a9154c2c259092d77031ddd359",
+                "sha256:a661d72d58e6ea8a57f7a86e37d86716863ee5e92788398526d58b26a4e4dc02"
             ],
-            "version": "==0.7.11"
+            "version": "==0.7.12"
+        },
+        "apipkg": {
+            "hashes": [
+                "sha256:37228cda29411948b422fae072f57e31d3396d2ee1c9783775980ee9c9990af6",
+                "sha256:58587dd4dc3daefad0487f6d9ae32b4542b185e1c36db6993290e7c41ca2b47c"
+            ],
+            "version": "==1.5"
+        },
+        "atomicwrites": {
+            "hashes": [
+                "sha256:0312ad34fcad8fac3704d441f7b317e50af620823353ec657a53e981f92920c0",
+                "sha256:ec9ae8adaae229e4f8446952d204a3e4b5fdd2d099f9be3aaf556120135fb3ee"
+            ],
+            "version": "==1.2.1"
+        },
+        "attrs": {
+            "hashes": [
+                "sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69",
+                "sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb"
+            ],
+            "version": "==18.2.0"
         },
         "babel": {
             "hashes": [
@@ -487,10 +381,10 @@
         },
         "certifi": {
             "hashes": [
-                "sha256:376690d6f16d32f9d1fe8932551d80b23e9d393a8578c5633a2ed39a64861638",
-                "sha256:456048c7e371c089d0a77a5212fb37a2c2dce1e24146e3b7e0261736aaeaa22a"
+                "sha256:339dc09518b07e2fa7eda5450740925974815557727d6bd35d319c1524a04a4c",
+                "sha256:6d58c986d22b038c8c0df30d639f23a3e6d172a05c3583e766f4c0b785c0986a"
             ],
-            "version": "==2018.8.24"
+            "version": "==2018.10.15"
         },
         "chardet": {
             "hashes": [
@@ -499,6 +393,44 @@
             ],
             "version": "==3.0.4"
         },
+        "coverage": {
+            "hashes": [
+                "sha256:03481e81d558d30d230bc12999e3edffe392d244349a90f4ef9b88425fac74ba",
+                "sha256:0b136648de27201056c1869a6c0d4e23f464750fd9a9ba9750b8336a244429ed",
+                "sha256:0bf8cbbd71adfff0ef1f3a1531e6402d13b7b01ac50a79c97ca15f030dba6306",
+                "sha256:10a46017fef60e16694a30627319f38a2b9b52e90182dddb6e37dcdab0f4bf95",
+                "sha256:198626739a79b09fa0a2f06e083ffd12eb55449b5f8bfdbeed1df4910b2ca640",
+                "sha256:23d341cdd4a0371820eb2b0bd6b88f5003a7438bbedb33688cd33b8eae59affd",
+                "sha256:28b2191e7283f4f3568962e373b47ef7f0392993bb6660d079c62bd50fe9d162",
+                "sha256:2a5b73210bad5279ddb558d9a2bfedc7f4bf6ad7f3c988641d83c40293deaec1",
+                "sha256:2eb564bbf7816a9d68dd3369a510be3327f1c618d2357fa6b1216994c2e3d508",
+                "sha256:337ded681dd2ef9ca04ef5d93cfc87e52e09db2594c296b4a0a3662cb1b41249",
+                "sha256:3a2184c6d797a125dca8367878d3b9a178b6fdd05fdc2d35d758c3006a1cd694",
+                "sha256:3c79a6f7b95751cdebcd9037e4d06f8d5a9b60e4ed0cd231342aa8ad7124882a",
+                "sha256:3d72c20bd105022d29b14a7d628462ebdc61de2f303322c0212a054352f3b287",
+                "sha256:3eb42bf89a6be7deb64116dd1cc4b08171734d721e7a7e57ad64cc4ef29ed2f1",
+                "sha256:4635a184d0bbe537aa185a34193898eee409332a8ccb27eea36f262566585000",
+                "sha256:56e448f051a201c5ebbaa86a5efd0ca90d327204d8b059ab25ad0f35fbfd79f1",
+                "sha256:5a13ea7911ff5e1796b6d5e4fbbf6952381a611209b736d48e675c2756f3f74e",
+                "sha256:69bf008a06b76619d3c3f3b1983f5145c75a305a0fea513aca094cae5c40a8f5",
+                "sha256:6bc583dc18d5979dc0f6cec26a8603129de0304d5ae1f17e57a12834e7235062",
+                "sha256:701cd6093d63e6b8ad7009d8a92425428bc4d6e7ab8d75efbb665c806c1d79ba",
+                "sha256:7608a3dd5d73cb06c531b8925e0ef8d3de31fed2544a7de6c63960a1e73ea4bc",
+                "sha256:76ecd006d1d8f739430ec50cc872889af1f9c1b6b8f48e29941814b09b0fd3cc",
+                "sha256:7aa36d2b844a3e4a4b356708d79fd2c260281a7390d678a10b91ca595ddc9e99",
+                "sha256:7d3f553904b0c5c016d1dad058a7554c7ac4c91a789fca496e7d8347ad040653",
+                "sha256:7e1fe19bd6dce69d9fd159d8e4a80a8f52101380d5d3a4d374b6d3eae0e5de9c",
+                "sha256:8c3cb8c35ec4d9506979b4cf90ee9918bc2e49f84189d9bf5c36c0c1119c6558",
+                "sha256:9d6dd10d49e01571bf6e147d3b505141ffc093a06756c60b053a859cb2128b1f",
+                "sha256:be6cfcd8053d13f5f5eeb284aa8a814220c3da1b0078fa859011c7fffd86dab9",
+                "sha256:c1bb572fab8208c400adaf06a8133ac0712179a334c09224fb11393e920abcdd",
+                "sha256:de4418dadaa1c01d497e539210cb6baa015965526ff5afc078c57ca69160108d",
+                "sha256:e05cb4d9aad6233d67e0541caa7e511fa4047ed7750ec2510d466e806e0255d6",
+                "sha256:f05a636b4564104120111800021a92e43397bc12a5c72fed7036be8556e0029e",
+                "sha256:f3f501f345f24383c0000395b26b726e46758b71393267aeae0bd36f8b3ade80"
+            ],
+            "version": "==4.5.1"
+        },
         "decorator": {
             "hashes": [
                 "sha256:2c51dff8ef3c447388fe5e4453d24a2bf128d3a4c32af3fabef1f01c6851ab82",
@@ -514,6 +446,20 @@
             ],
             "version": "==0.14"
         },
+        "execnet": {
+            "hashes": [
+                "sha256:a7a84d5fa07a089186a329528f127c9d73b9de57f1a1131b82bb5320ee651f6a",
+                "sha256:fc155a6b553c66c838d1a22dba1dc9f5f505c43285a878c6f74a79c024750b83"
+            ],
+            "version": "==1.5.0"
+        },
+        "filelock": {
+            "hashes": [
+                "sha256:b8d5ca5ca1c815e1574aee746650ea7301de63d87935b3463d26368b76e31633",
+                "sha256:d610c1bb404daf85976d7a82eb2ada120f04671007266b708606565dd03b5be6"
+            ],
+            "version": "==3.0.10"
+        },
         "idna": {
             "hashes": [
                 "sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e",
@@ -526,16 +472,15 @@
                 "sha256:3f349de3eb99145973fefb7dbe38554414e5c30abd0c8e4b970a7c9d09f3a1d8",
                 "sha256:f3832918bc3c66617f92e35f5d70729187676313caa60c187eb0f28b8fe5e3b5"
             ],
-            "markers": "python_version != '3.2.*' and python_version >= '2.7' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.3.*'",
             "version": "==1.1.0"
         },
         "ipython": {
             "hashes": [
-                "sha256:007dcd929c14631f83daff35df0147ea51d1af420da303fd078343878bd5fb62",
-                "sha256:b0f2ef9eada4a68ef63ee10b6dde4f35c840035c50fd24265f8052c98947d5a4"
+                "sha256:a5781d6934a3341a1f9acb4ea5acdc7ea0a0855e689dbe755d070ca51e995435",
+                "sha256:b10a7ddd03657c761fc503495bc36471c8158e3fc948573fb9fe82a7029d8efd"
             ],
             "index": "pypi",
-            "version": "==6.5.0"
+            "version": "==7.1.1"
         },
         "ipython-genutils": {
             "hashes": [
@@ -546,10 +491,10 @@
         },
         "jedi": {
             "hashes": [
-                "sha256:b409ed0f6913a701ed474a614a3bb46e6953639033e31f769ca7581da5bd1ec1",
-                "sha256:c254b135fb39ad76e78d4d8f92765ebc9bf92cbc76f49e97ade1d5f5121e1f6f"
+                "sha256:0191c447165f798e6a730285f2eee783fff81b0d3df261945ecb80983b5c3ca7",
+                "sha256:b7493f73a2febe0dc33d51c99b474547f7f6c0b2c8fb2b21f453eef204c12148"
             ],
-            "version": "==0.12.1"
+            "version": "==0.13.1"
         },
         "jinja2": {
             "hashes": [
@@ -564,12 +509,20 @@
             ],
             "version": "==1.0"
         },
+        "more-itertools": {
+            "hashes": [
+                "sha256:c187a73da93e7a8acc0001572aebc7e3c69daf7bf6881a2cea10650bd4420092",
+                "sha256:c476b5d3a34e12d40130bc2f935028b5f636df8f372dc2c1c01dc19681b2039e",
+                "sha256:fcbfeaea0be121980e15bc97b3817b5202ca73d0eae185b4550cbfce2a3ebb3d"
+            ],
+            "version": "==4.3.0"
+        },
         "packaging": {
             "hashes": [
-                "sha256:e9215d2d2535d3ae866c3d6efc77d5b24a0192cce0ff20e42896cc0664f889c0",
-                "sha256:f019b770dd64e585a99714f1fd5e01c7a8f11b45635aa953fd41c689a657375b"
+                "sha256:0886227f54515e592aaa2e5a553332c73962917f2831f1b0f9b9f4380a4b9807",
+                "sha256:f95a1e147590f204328170981833854229bb2912ac3d5f89e2a8ccd2834800c9"
             ],
-            "version": "==17.1"
+            "version": "==18.0"
         },
         "parso": {
             "hashes": [
@@ -588,26 +541,25 @@
         },
         "pickleshare": {
             "hashes": [
-                "sha256:84a9257227dfdd6fe1b4be1319096c20eb85ff1e82c7932f36efccfe1b09737b",
-                "sha256:c9a2541f25aeabc070f12f452e1f2a8eae2abd51e1cd19e8430402bdf4c1d8b5"
+                "sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca",
+                "sha256:9649af414d74d4df115d5d718f82acb59c9d418196b7b4290ed47a12ce62df56"
             ],
-            "version": "==0.7.4"
+            "version": "==0.7.5"
         },
         "pluggy": {
             "hashes": [
-                "sha256:6e3836e39f4d36ae72840833db137f7b7d35105079aee6ec4a62d9f80d594dd1",
-                "sha256:95eb8364a4708392bae89035f45341871286a333f749c3141c20573d2b3876e1"
+                "sha256:447ba94990e8014ee25ec853339faf7b0fc8050cdc3289d4d71f7f410fb90095",
+                "sha256:bde19360a8ec4dfd8a20dcb811780a30998101f078fc7ded6162f0076f50508f"
             ],
-            "markers": "python_version != '3.2.*' and python_version >= '2.7' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.3.*'",
-            "version": "==0.7.1"
+            "version": "==0.8.0"
         },
         "prompt-toolkit": {
             "hashes": [
-                "sha256:1df952620eccb399c53ebb359cc7d9a8d3a9538cb34c5a1344bdbeb29fbcc381",
-                "sha256:3f473ae040ddaa52b52f97f6b4a493cfa9f5920c255a12dc56a7d34397a398a4",
-                "sha256:858588f1983ca497f1cf4ffde01d978a3ea02b01c8a26a8bbc5cd2e66d816917"
+                "sha256:c1d6aff5252ab2ef391c2fe498ed8c088066f66bc64a8d5c095bbf795d9fec34",
+                "sha256:d4c47f79b635a0e70b84fdb97ebd9a274203706b1ee5ed44c10da62755cf3ec9",
+                "sha256:fd17048d8335c1e6d5ee403c3569953ba3eb8555d710bfc548faf0712666ea39"
             ],
-            "version": "==1.0.15"
+            "version": "==2.0.7"
         },
         "ptyprocess": {
             "hashes": [
@@ -618,11 +570,18 @@
         },
         "py": {
             "hashes": [
-                "sha256:06a30435d058473046be836d3fc4f27167fd84c45b99704f2fb5509ef61f9af1",
-                "sha256:50402e9d1c9005d759426988a492e0edaadb7f4e68bcddfea586bc7432d009c6"
+                "sha256:bf92637198836372b520efcba9e020c330123be8ce527e535d185ed4b6f45694",
+                "sha256:e76826342cefe3c3d5f7e8ee4316b80d1dd8a300781612ddbc765c17ba25a6c6"
             ],
-            "markers": "python_version != '3.2.*' and python_version >= '2.7' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.3.*'",
-            "version": "==1.6.0"
+            "version": "==1.7.0"
+        },
+        "pycodestyle": {
+            "hashes": [
+                "sha256:cbc619d09254895b0d12c2c691e237b2e91e9b2ecf5e84c26b35400f93dcfb83",
+                "sha256:cbfca99bd594a10f674d0cd97a3d802a1fdef635d4361e1a2658de47ed261e3a"
+            ],
+            "index": "pypi",
+            "version": "==2.4.0"
         },
         "pygments": {
             "hashes": [
@@ -633,31 +592,78 @@
         },
         "pyparsing": {
             "hashes": [
-                "sha256:0832bcf47acd283788593e7a0f542407bd9550a55a8a8435214a1960e04bcb04",
-                "sha256:fee43f17a9c4087e7ed1605bd6df994c6173c1e977d7ade7b651292fab2bd010"
+                "sha256:40856e74d4987de5d01761a22d1621ae1c7f8774585acae358aa5c5936c6c90b",
+                "sha256:f353aab21fd474459d97b709e527b5571314ee5f067441dc9f88e33eecd96592"
             ],
-            "version": "==2.2.0"
+            "version": "==2.3.0"
+        },
+        "pytest": {
+            "hashes": [
+                "sha256:a9e5e8d7ab9d5b0747f37740276eb362e6a76275d76cebbb52c6049d93b475db",
+                "sha256:bf47e8ed20d03764f963f0070ff1c8fda6e2671fc5dd562a4d3b7148ad60f5ca"
+            ],
+            "index": "pypi",
+            "version": "==3.9.3"
+        },
+        "pytest-cov": {
+            "hashes": [
+                "sha256:513c425e931a0344944f84ea47f3956be0e416d95acbd897a44970c8d926d5d7",
+                "sha256:e360f048b7dae3f2f2a9a4d067b2dd6b6a015d384d1577c994a43f3f7cbad762"
+            ],
+            "index": "pypi",
+            "version": "==2.6.0"
+        },
+        "pytest-django": {
+            "hashes": [
+                "sha256:49e9ffc856bc6a1bec1c26c5c7b7213dff7cc8bc6b64d624c4d143d04aff0bcf",
+                "sha256:b379282feaf89069cb790775ab6bbbd2bd2038a68c7ef9b84a41898e0b551081"
+            ],
+            "index": "pypi",
+            "version": "==3.4.3"
+        },
+        "pytest-env": {
+            "hashes": [
+                "sha256:7e94956aef7f2764f3c147d216ce066bf6c42948bb9e293169b1b1c880a580c2"
+            ],
+            "index": "pypi",
+            "version": "==0.6.2"
+        },
+        "pytest-forked": {
+            "hashes": [
+                "sha256:e4500cd0509ec4a26535f7d4112a8cc0f17d3a41c29ffd4eab479d2a55b30805",
+                "sha256:f275cb48a73fc61a6710726348e1da6d68a978f0ec0c54ece5a5fae5977e5a08"
+            ],
+            "version": "==0.2"
+        },
+        "pytest-sugar": {
+            "hashes": [
+                "sha256:ab8cc42faf121344a4e9b13f39a51257f26f410e416c52ea11078cdd00d98a2c"
+            ],
+            "index": "pypi",
+            "version": "==0.9.1"
+        },
+        "pytest-xdist": {
+            "hashes": [
+                "sha256:3bc9dcb6ff47e607d3c710727cd9996fd7ac1466d405c3b40bb495da99b6b669",
+                "sha256:8e188d13ce6614c7a678179a76f46231199ffdfe6163de031c17e62ffa256917"
+            ],
+            "index": "pypi",
+            "version": "==1.24.0"
         },
         "pytz": {
             "hashes": [
-                "sha256:a061aa0a9e06881eb8b3b2b43f05b9439d6583c206d0a6c340ff72a7b6669053",
-                "sha256:ffb9ef1de172603304d9d2819af6f5ece76f2e85ec10692a524dd876e72bf277"
+                "sha256:31cb35c89bd7d333cd32c5f278fca91b523b0834369e757f4c5641ea252236ca",
+                "sha256:8e0f8568c118d3077b46be7d654cc8167fa916092e28320cde048e54bfc9f1e6"
             ],
             "index": "pypi",
-            "version": "==2018.5"
+            "version": "==2018.7"
         },
         "requests": {
             "hashes": [
-                "sha256:63b52e3c866428a224f97cab011de738c36aec0185aa91cfacd418b5d58911d1",
-                "sha256:ec22d826a36ed72a7358ff3fe56cbd4ba69dd7a6718ffd450ff0e9df7a47ce6a"
+                "sha256:99dcfdaaeb17caf6e526f32b6a7b780461512ab3f1d992187801694cba42770c",
+                "sha256:a84b8c9ab6239b578f22d1c21d51b696dcfe004032bb80ea832398d6909d7279"
             ],
-            "version": "==2.19.1"
-        },
-        "simplegeneric": {
-            "hashes": [
-                "sha256:dc972e06094b9af5b855b3df4a646395e43d1c9d0d39ed345b7393560d0b9173"
-            ],
-            "version": "==0.8.1"
+            "version": "==2.20.0"
         },
         "six": {
             "hashes": [
@@ -675,27 +681,39 @@
         },
         "sphinx": {
             "hashes": [
-                "sha256:217a7705adcb573da5bbe1e0f5cab4fa0bd89fd9342c9159121746f593c2d5a4",
-                "sha256:a602513f385f1d5785ff1ca420d9c7eb1a1b63381733b2f0ea8188a391314a86"
+                "sha256:652eb8c566f18823a022bb4b6dbc868d366df332a11a0226b5bc3a798a479f17",
+                "sha256:d222626d8356de702431e813a05c68a35967e3d66c6cd1c2c89539bb179a7464"
             ],
             "index": "pypi",
-            "version": "==1.7.9"
+            "version": "==1.8.1"
         },
         "sphinxcontrib-websupport": {
             "hashes": [
                 "sha256:68ca7ff70785cbe1e7bccc71a48b5b6d965d79ca50629606c7861a21b206d9dd",
                 "sha256:9de47f375baf1ea07cdb3436ff39d7a9c76042c10a769c52353ec46e4e8fc3b9"
             ],
-            "markers": "python_version != '3.2.*' and python_version >= '2.7' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.3.*'",
             "version": "==1.1.0"
         },
+        "termcolor": {
+            "hashes": [
+                "sha256:1d6d69ce66211143803fbc56652b41d73b4a400a2891d7bf7a1cdf4c02de613b"
+            ],
+            "version": "==1.1.0"
+        },
+        "toml": {
+            "hashes": [
+                "sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c",
+                "sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e"
+            ],
+            "version": "==0.10.0"
+        },
         "tox": {
             "hashes": [
-                "sha256:37cf240781b662fb790710c6998527e65ca6851eace84d1595ee71f7af4e85f7",
-                "sha256:eb61aa5bcce65325538686f09848f04ef679b5cd9b83cc491272099b28739600"
+                "sha256:513e32fdf2f9e2d583c2f248f47ba9886428c949f068ac54a0469cac55df5862",
+                "sha256:75fa30e8329b41b664585f5fb837e23ce1d7e6fa1f7811f2be571c990f9d911b"
             ],
             "index": "pypi",
-            "version": "==3.2.1"
+            "version": "==3.5.3"
         },
         "traitlets": {
             "hashes": [
@@ -706,19 +724,17 @@
         },
         "urllib3": {
             "hashes": [
-                "sha256:a68ac5e15e76e7e5dd2b8f94007233e01effe3e50e8daddf69acfd81cb686baf",
-                "sha256:b5725a0bd4ba422ab0e66e89e030c806576753ea3ee08554382c14e685d117b5"
+                "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39",
+                "sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22"
             ],
-            "markers": "python_version >= '2.6' and python_version != '3.2.*' and python_version != '3.0.*' and python_version != '3.1.*' and python_version < '4' and python_version != '3.3.*'",
-            "version": "==1.23"
+            "version": "==1.24.1"
         },
         "virtualenv": {
             "hashes": [
-                "sha256:2ce32cd126117ce2c539f0134eb89de91a8413a29baac49cbab3eb50e2026669",
-                "sha256:ca07b4c0b54e14a91af9f34d0919790b016923d157afda5efdde55c96718f752"
+                "sha256:686176c23a538ecc56d27ed9d5217abd34644823d6391cbeb232f42bf722baad",
+                "sha256:f899fafcd92e1150f40c8215328be38ff24b519cd95357fa6e78e006c7638208"
             ],
-            "markers": "python_version >= '2.7' and python_version != '3.0.*' and python_version != '3.2.*' and python_version != '3.1.*'",
-            "version": "==16.0.0"
+            "version": "==16.1.0"
         },
         "wcwidth": {
             "hashes": [
diff --git a/docs/changelog.rst b/docs/changelog.rst
index e98436382..059691811 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -9,6 +9,8 @@ Changelog
 * Fix the ``RecentCorrespondentsFilter`` correspondents filter that was added
   in 2.4 to play nice with the defaults.  Thanks to `tsia`_ and `Sblop`_ who
   pointed this out. `#423`_.
+* Updated dependencies to include (among other things) a security patch to
+  requests.
 
 
 2.5.0
diff --git a/requirements.txt b/requirements.txt
index ac893b8a5..61d98a6d7 100755
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,51 +1,53 @@
 -i https://pypi.python.org/simple
-apipkg==1.5; python_version != '3.3.*'
-atomicwrites==1.2.1; python_version != '3.3.*'
+alabaster==0.7.12
+apipkg==1.5
+atomicwrites==1.2.1
 attrs==18.2.0
-certifi==2018.8.24
+babel==2.6.0
+backcall==0.1.0
+certifi==2018.10.15
 chardet==3.0.4
-coverage==4.5.1; python_version < '4'
-coveralls==1.5.0
-dateparser==0.7.0
-django-cors-headers==2.4.0
-django-crispy-forms==1.7.2
-django-extensions==2.1.2
-django-filter==2.0.0
-django==2.0.8
-djangorestframework==3.8.2
-docopt==0.6.2
-execnet==1.5.0; python_version != '3.3.*'
-factory-boy==2.11.1
-faker==0.9.0; python_version >= '2.7'
-filemagic==1.6
-fuzzywuzzy==0.15.0
-gunicorn==19.9.0
+coverage==4.5.1
+decorator==4.3.0
+docutils==0.14
+execnet==1.5.0
+filelock==3.0.10
 idna==2.7
-inotify-simple==1.1.8; sys_platform == 'linux'
-langdetect==1.0.7
+imagesize==1.1.0
+ipython-genutils==0.2.0
+ipython==7.1.1
+jedi==0.13.1
+jinja2==2.10
+markupsafe==1.0
 more-itertools==4.3.0
-pdftotext==2.1.0
-pillow==5.2.0
-pluggy==0.7.1; python_version != '3.3.*'
-py==1.6.0; python_version != '3.3.*'
+packaging==18.0
+parso==0.3.1
+pexpect==4.6.0 ; sys_platform != 'win32'
+pickleshare==0.7.5
+pluggy==0.8.0
+prompt-toolkit==2.0.7
+ptyprocess==0.6.0
+py==1.7.0
 pycodestyle==2.4.0
-pyocr==0.5.3
+pygments==2.2.0
+pyparsing==2.3.0
 pytest-cov==2.6.0
-pytest-django==3.4.2
+pytest-django==3.4.3
 pytest-env==0.6.2
-pytest-forked==0.2; python_version != '3.3.*'
+pytest-forked==0.2
 pytest-sugar==0.9.1
-pytest-xdist==1.23.0
-pytest==3.8.0
-python-dateutil==2.7.3
-python-dotenv==0.9.1
-python-gnupg==0.4.3
-python-levenshtein==0.12.0
-pytz==2018.5
-regex==2018.8.29
-requests==2.19.1
+pytest-xdist==1.24.0
+pytest==3.9.3
+pytz==2018.7
+requests==2.20.0
 six==1.11.0
+snowballstemmer==1.2.1
+sphinx==1.8.1
+sphinxcontrib-websupport==1.1.0
 termcolor==1.1.0
-text-unidecode==1.2
-tzlocal==1.5.1
-urllib3==1.23; python_version != '3.3.*'
+toml==0.10.0
+tox==3.5.3
+traitlets==4.3.2
+urllib3==1.24.1
+virtualenv==16.1.0
+wcwidth==0.1.7

From e67d319d582a73f646f14087d8474d4a8d987e59 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sat, 3 Nov 2018 13:29:22 +0000
Subject: [PATCH 46/47] Fix requiremnts.txt

---
 Pipfile          |   4 +-
 Pipfile.lock     | 561 ++++++++++++++++++++---------------------------
 requirements.txt |  26 +++
 3 files changed, 264 insertions(+), 327 deletions(-)

diff --git a/Pipfile b/Pipfile
index 778d15546..12bae4b66 100644
--- a/Pipfile
+++ b/Pipfile
@@ -25,8 +25,6 @@ python-dateutil = "*"
 python-dotenv = "*"
 python-gnupg = "*"
 pytz = "*"
-
-[dev-packages]
 ipython = "*"
 sphinx = "*"
 tox = "*"
@@ -37,3 +35,5 @@ pytest-django = "*"
 pytest-sugar = "*"
 pytest-env = "*"
 pytest-xdist = "*"
+
+[dev-packages]
diff --git a/Pipfile.lock b/Pipfile.lock
index 966fbd467..96dec448b 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "eeeeaf6ecb0cec45a3962eda6647b5263e5ad7939fae29d4b294f59ffe9ca3dd"
+            "sha256": "3782f7e6b5461c39c8fd0d0048a4622418f247439113bd3cdc91712fd47036f6"
         },
         "pipfile-spec": 6,
         "requires": {},
@@ -14,329 +14,6 @@
         ]
     },
     "default": {
-        "certifi": {
-            "hashes": [
-                "sha256:339dc09518b07e2fa7eda5450740925974815557727d6bd35d319c1524a04a4c",
-                "sha256:6d58c986d22b038c8c0df30d639f23a3e6d172a05c3583e766f4c0b785c0986a"
-            ],
-            "version": "==2018.10.15"
-        },
-        "chardet": {
-            "hashes": [
-                "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
-                "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
-            ],
-            "version": "==3.0.4"
-        },
-        "coverage": {
-            "hashes": [
-                "sha256:03481e81d558d30d230bc12999e3edffe392d244349a90f4ef9b88425fac74ba",
-                "sha256:0b136648de27201056c1869a6c0d4e23f464750fd9a9ba9750b8336a244429ed",
-                "sha256:0bf8cbbd71adfff0ef1f3a1531e6402d13b7b01ac50a79c97ca15f030dba6306",
-                "sha256:10a46017fef60e16694a30627319f38a2b9b52e90182dddb6e37dcdab0f4bf95",
-                "sha256:198626739a79b09fa0a2f06e083ffd12eb55449b5f8bfdbeed1df4910b2ca640",
-                "sha256:23d341cdd4a0371820eb2b0bd6b88f5003a7438bbedb33688cd33b8eae59affd",
-                "sha256:28b2191e7283f4f3568962e373b47ef7f0392993bb6660d079c62bd50fe9d162",
-                "sha256:2a5b73210bad5279ddb558d9a2bfedc7f4bf6ad7f3c988641d83c40293deaec1",
-                "sha256:2eb564bbf7816a9d68dd3369a510be3327f1c618d2357fa6b1216994c2e3d508",
-                "sha256:337ded681dd2ef9ca04ef5d93cfc87e52e09db2594c296b4a0a3662cb1b41249",
-                "sha256:3a2184c6d797a125dca8367878d3b9a178b6fdd05fdc2d35d758c3006a1cd694",
-                "sha256:3c79a6f7b95751cdebcd9037e4d06f8d5a9b60e4ed0cd231342aa8ad7124882a",
-                "sha256:3d72c20bd105022d29b14a7d628462ebdc61de2f303322c0212a054352f3b287",
-                "sha256:3eb42bf89a6be7deb64116dd1cc4b08171734d721e7a7e57ad64cc4ef29ed2f1",
-                "sha256:4635a184d0bbe537aa185a34193898eee409332a8ccb27eea36f262566585000",
-                "sha256:56e448f051a201c5ebbaa86a5efd0ca90d327204d8b059ab25ad0f35fbfd79f1",
-                "sha256:5a13ea7911ff5e1796b6d5e4fbbf6952381a611209b736d48e675c2756f3f74e",
-                "sha256:69bf008a06b76619d3c3f3b1983f5145c75a305a0fea513aca094cae5c40a8f5",
-                "sha256:6bc583dc18d5979dc0f6cec26a8603129de0304d5ae1f17e57a12834e7235062",
-                "sha256:701cd6093d63e6b8ad7009d8a92425428bc4d6e7ab8d75efbb665c806c1d79ba",
-                "sha256:7608a3dd5d73cb06c531b8925e0ef8d3de31fed2544a7de6c63960a1e73ea4bc",
-                "sha256:76ecd006d1d8f739430ec50cc872889af1f9c1b6b8f48e29941814b09b0fd3cc",
-                "sha256:7aa36d2b844a3e4a4b356708d79fd2c260281a7390d678a10b91ca595ddc9e99",
-                "sha256:7d3f553904b0c5c016d1dad058a7554c7ac4c91a789fca496e7d8347ad040653",
-                "sha256:7e1fe19bd6dce69d9fd159d8e4a80a8f52101380d5d3a4d374b6d3eae0e5de9c",
-                "sha256:8c3cb8c35ec4d9506979b4cf90ee9918bc2e49f84189d9bf5c36c0c1119c6558",
-                "sha256:9d6dd10d49e01571bf6e147d3b505141ffc093a06756c60b053a859cb2128b1f",
-                "sha256:be6cfcd8053d13f5f5eeb284aa8a814220c3da1b0078fa859011c7fffd86dab9",
-                "sha256:c1bb572fab8208c400adaf06a8133ac0712179a334c09224fb11393e920abcdd",
-                "sha256:de4418dadaa1c01d497e539210cb6baa015965526ff5afc078c57ca69160108d",
-                "sha256:e05cb4d9aad6233d67e0541caa7e511fa4047ed7750ec2510d466e806e0255d6",
-                "sha256:f05a636b4564104120111800021a92e43397bc12a5c72fed7036be8556e0029e",
-                "sha256:f3f501f345f24383c0000395b26b726e46758b71393267aeae0bd36f8b3ade80"
-            ],
-            "version": "==4.5.1"
-        },
-        "coveralls": {
-            "hashes": [
-                "sha256:ab638e88d38916a6cedbf80a9cd8992d5fa55c77ab755e262e00b36792b7cd6d",
-                "sha256:b2388747e2529fa4c669fb1e3e2756e4e07b6ee56c7d9fce05f35ccccc913aa0"
-            ],
-            "index": "pypi",
-            "version": "==1.5.1"
-        },
-        "dateparser": {
-            "hashes": [
-                "sha256:940828183c937bcec530753211b70f673c0a9aab831e43273489b310538dff86",
-                "sha256:b452ef8b36cd78ae86a50721794bc674aa3994e19b570f7ba92810f4e0a2ae03"
-            ],
-            "index": "pypi",
-            "version": "==0.7.0"
-        },
-        "django": {
-            "hashes": [
-                "sha256:25df265e1fdb74f7e7305a1de620a84681bcc9c05e84a3ed97e4a1a63024f18d",
-                "sha256:d6d94554abc82ca37e447c3d28958f5ac39bd7d4adaa285543ae97fb1129fd69"
-            ],
-            "index": "pypi",
-            "version": "==2.0.9"
-        },
-        "django-cors-headers": {
-            "hashes": [
-                "sha256:5545009c9b233ea7e70da7dbab7cb1c12afa01279895086f98ec243d7eab46fa",
-                "sha256:c4c2ee97139d18541a1be7d96fe337d1694623816d83f53cb7c00da9b94acae1"
-            ],
-            "index": "pypi",
-            "version": "==2.4.0"
-        },
-        "django-crispy-forms": {
-            "hashes": [
-                "sha256:5952bab971110d0b86c278132dae0aa095beee8f723e625c3d3fa28888f1675f",
-                "sha256:705ededc554ad8736157c666681165fe22ead2dec0d5446d65fc9dd976a5a876"
-            ],
-            "index": "pypi",
-            "version": "==1.7.2"
-        },
-        "django-extensions": {
-            "hashes": [
-                "sha256:30cb6a8c7d6f75a55edf0c0c4491bd98f8264ae1616ce105f9cecac4387edd07",
-                "sha256:4ad86a7a5e84f1c77db030761ae87a600647250c652030a2b71a16e87f3a3d62"
-            ],
-            "index": "pypi",
-            "version": "==2.1.3"
-        },
-        "django-filter": {
-            "hashes": [
-                "sha256:6f4e4bc1a11151178520567b50320e5c32f8edb552139d93ea3e30613b886f56",
-                "sha256:86c3925020c27d072cdae7b828aaa5d165c2032a629abbe3c3a1be1edae61c58"
-            ],
-            "index": "pypi",
-            "version": "==2.0.0"
-        },
-        "djangorestframework": {
-            "hashes": [
-                "sha256:607865b0bb1598b153793892101d881466bd5a991de12bd6229abb18b1c86136",
-                "sha256:63f76cbe1e7d12b94c357d7e54401103b2e52aef0f7c1650d6c820ad708776e5"
-            ],
-            "index": "pypi",
-            "version": "==3.9.0"
-        },
-        "docopt": {
-            "hashes": [
-                "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491"
-            ],
-            "version": "==0.6.2"
-        },
-        "factory-boy": {
-            "hashes": [
-                "sha256:6f25cc4761ac109efd503f096e2ad99421b1159f01a29dbb917359dcd68e08ca",
-                "sha256:d552cb872b310ae78bd7429bf318e42e1e903b1a109e899a523293dfa762ea4f"
-            ],
-            "index": "pypi",
-            "version": "==2.11.1"
-        },
-        "faker": {
-            "hashes": [
-                "sha256:2621643b80a10b91999925cfd20f64d2b36f20bf22136bbdc749bb57d6ffe124",
-                "sha256:5ed822d31bd2d6edf10944d176d30dc9c886afdd381eefb7ba8b7aad86171646"
-            ],
-            "version": "==0.9.2"
-        },
-        "filemagic": {
-            "hashes": [
-                "sha256:e684359ef40820fe406f0ebc5bf8a78f89717bdb7fed688af68082d991d6dbf3"
-            ],
-            "index": "pypi",
-            "version": "==1.6"
-        },
-        "fuzzywuzzy": {
-            "extras": [
-                "speedup"
-            ],
-            "hashes": [
-                "sha256:3759bc6859daa0eecef8c82b45404bdac20c23f23136cf4c18b46b426bbc418f",
-                "sha256:5b36957ccf836e700f4468324fa80ba208990385392e217be077d5cd738ae602"
-            ],
-            "index": "pypi",
-            "version": "==0.15.0"
-        },
-        "gunicorn": {
-            "hashes": [
-                "sha256:aa8e0b40b4157b36a5df5e599f45c9c76d6af43845ba3b3b0efe2c70473c2471",
-                "sha256:fa2662097c66f920f53f70621c6c58ca4a3c4d3434205e608e121b5b3b71f4f3"
-            ],
-            "index": "pypi",
-            "version": "==19.9.0"
-        },
-        "idna": {
-            "hashes": [
-                "sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e",
-                "sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16"
-            ],
-            "version": "==2.7"
-        },
-        "inotify-simple": {
-            "hashes": [
-                "sha256:fc2c10dd73278a1027d0663f2db51240af5946390f363a154361406ebdddd8dd"
-            ],
-            "index": "pypi",
-            "version": "==1.1.8"
-        },
-        "langdetect": {
-            "hashes": [
-                "sha256:91a170d5f0ade380db809b3ba67f08e95fe6c6c8641f96d67a51ff7e98a9bf30"
-            ],
-            "index": "pypi",
-            "version": "==1.0.7"
-        },
-        "pdftotext": {
-            "hashes": [
-                "sha256:e3ad11efe0aa22cbfc46aa1296b2ea5a52ad208b778288311f2801adef178ccb"
-            ],
-            "index": "pypi",
-            "version": "==2.1.1"
-        },
-        "pillow": {
-            "hashes": [
-                "sha256:00203f406818c3f45d47bb8fe7e67d3feddb8dcbbd45a289a1de7dd789226360",
-                "sha256:0616f800f348664e694dddb0b0c88d26761dd5e9f34e1ed7b7a7d2da14b40cb7",
-                "sha256:1f7908aab90c92ad85af9d2fec5fc79456a89b3adcc26314d2cde0e238bd789e",
-                "sha256:2ea3517cd5779843de8a759c2349a3cd8d3893e03ab47053b66d5ec6f8bc4f93",
-                "sha256:48a9f0538c91fc136b3a576bee0e7cd174773dc9920b310c21dcb5519722e82c",
-                "sha256:5280ebc42641a1283b7b1f2c20e5b936692198b9dd9995527c18b794850be1a8",
-                "sha256:5e34e4b5764af65551647f5cc67cf5198c1d05621781d5173b342e5e55bf023b",
-                "sha256:63b120421ab85cad909792583f83b6ca3584610c2fe70751e23f606a3c2e87f0",
-                "sha256:696b5e0109fe368d0057f484e2e91717b49a03f1e310f857f133a4acec9f91dd",
-                "sha256:870ed021a42b1b02b5fe4a739ea735f671a84128c0a666c705db2cb9abd528eb",
-                "sha256:916da1c19e4012d06a372127d7140dae894806fad67ef44330e5600d77833581",
-                "sha256:9303a289fa0811e1c6abd9ddebfc770556d7c3311cb2b32eff72164ddc49bc64",
-                "sha256:9577888ecc0ad7d06c3746afaba339c94d62b59da16f7a5d1cff9e491f23dace",
-                "sha256:987e1c94a33c93d9b209315bfda9faa54b8edfce6438a1e93ae866ba20de5956",
-                "sha256:99a3bbdbb844f4fb5d6dd59fac836a40749781c1fa63c563bc216c27aef63f60",
-                "sha256:99db8dc3097ceafbcff9cb2bff384b974795edeb11d167d391a02c7bfeeb6e16",
-                "sha256:a5a96cf49eb580756a44ecf12949e52f211e20bffbf5a95760ac14b1e499cd37",
-                "sha256:aa6ca3eb56704cdc0d876fc6047ffd5ee960caad52452fbee0f99908a141a0ae",
-                "sha256:aade5e66795c94e4a2b2624affeea8979648d1b0ae3fcee17e74e2c647fc4a8a",
-                "sha256:b78905860336c1d292409e3df6ad39cc1f1c7f0964e66844bbc2ebfca434d073",
-                "sha256:b92f521cdc4e4a3041cc343625b699f20b0b5f976793fb45681aac1efda565f8",
-                "sha256:bfde84bbd6ae5f782206d454b67b7ee8f7f818c29b99fd02bf022fd33bab14cb",
-                "sha256:c2b62d3df80e694c0e4a0ed47754c9480521e25642251b3ab1dff050a4e60409",
-                "sha256:c5e2be6c263b64f6f7656e23e18a4a9980cffc671442795682e8c4e4f815dd9f",
-                "sha256:c99aa3c63104e0818ec566f8ff3942fb7c7a8f35f9912cb63fd8e12318b214b2",
-                "sha256:dae06620d3978da346375ebf88b9e2dd7d151335ba668c995aea9ed07af7add4",
-                "sha256:db5499d0710823fa4fb88206050d46544e8f0e0136a9a5f5570b026584c8fd74",
-                "sha256:f36baafd82119c4a114b9518202f2a983819101dcc14b26e43fc12cbefdce00e",
-                "sha256:f52b79c8796d81391ab295b04e520bda6feed54d54931708872e8f9ae9db0ea1",
-                "sha256:ff8cff01582fa1a7e533cb97f628531c4014af4b5f38e33cdcfe5eec29b6d888"
-            ],
-            "index": "pypi",
-            "version": "==5.3.0"
-        },
-        "pyocr": {
-            "hashes": [
-                "sha256:b6ba6263fd92da56627dff6d263d991a2246aacd117d1788f11b93f419ca395f"
-            ],
-            "index": "pypi",
-            "version": "==0.5.3"
-        },
-        "python-dateutil": {
-            "hashes": [
-                "sha256:063df5763652e21de43de7d9e00ccf239f953a832941e37be541614732cdfc93",
-                "sha256:88f9287c0174266bb0d8cedd395cfba9c58e87e5ad86b2ce58859bc11be3cf02"
-            ],
-            "index": "pypi",
-            "version": "==2.7.5"
-        },
-        "python-dotenv": {
-            "hashes": [
-                "sha256:122290a38ece9fe4f162dc7c95cae3357b983505830a154d3c98ef7f6c6cea77",
-                "sha256:4a205787bc829233de2a823aa328e44fd9996fedb954989a21f1fc67c13d7a77"
-            ],
-            "index": "pypi",
-            "version": "==0.9.1"
-        },
-        "python-gnupg": {
-            "hashes": [
-                "sha256:2d158dfc6b54927752b945ebe57e6a0c45da27747fa3b9ae66eccc0d2147ac0d",
-                "sha256:faa69bab58ed0936f0ccf96c99b92369b7a1819305d37dfe5c927d21a437a09d"
-            ],
-            "index": "pypi",
-            "version": "==0.4.3"
-        },
-        "python-levenshtein": {
-            "hashes": [
-                "sha256:033a11de5e3d19ea25c9302d11224e1a1898fe5abd23c61c7c360c25195e3eb1"
-            ],
-            "markers": "extra == 'speedup'",
-            "version": "==0.12.0"
-        },
-        "pytz": {
-            "hashes": [
-                "sha256:31cb35c89bd7d333cd32c5f278fca91b523b0834369e757f4c5641ea252236ca",
-                "sha256:8e0f8568c118d3077b46be7d654cc8167fa916092e28320cde048e54bfc9f1e6"
-            ],
-            "index": "pypi",
-            "version": "==2018.7"
-        },
-        "regex": {
-            "hashes": [
-                "sha256:0ef96690c3d2294155b7d44187ca4a151e45c931cb768e106ba464a9fa64c5da",
-                "sha256:251683e01a3bcacd9188acf0d4caf7b29a3b963c843159311825613ae144cddb",
-                "sha256:3fe15a75fe00f04d1ec16713d55cf1e206077c450267a10b33318756fb8b3f99",
-                "sha256:53a962f9dc28cdf403978a142cb1e054479759ad64d312a999f9f042c25b5c9a",
-                "sha256:8bd1da6a93d32336a5e5432886dd8543004f0591c39b83dbfa60705cccdf414d",
-                "sha256:b5423061918f602e9342b54d746ac31c598d328ecaf4ef0618763e960c926fd4",
-                "sha256:d80ebc65b1f7d0403117f59309c16eac24be6a0bc730b593a79f703462858d94",
-                "sha256:fd8419979639b7de7fb964a13bce3ac47e6fe33043b83de0398c3067986e5659",
-                "sha256:ff2f15b2b0b4b58ba8a1de651780a0d3fd54f96ad6b77dceb77695220e5d7b7a"
-            ],
-            "version": "==2018.11.2"
-        },
-        "requests": {
-            "hashes": [
-                "sha256:99dcfdaaeb17caf6e526f32b6a7b780461512ab3f1d992187801694cba42770c",
-                "sha256:a84b8c9ab6239b578f22d1c21d51b696dcfe004032bb80ea832398d6909d7279"
-            ],
-            "version": "==2.20.0"
-        },
-        "six": {
-            "hashes": [
-                "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9",
-                "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb"
-            ],
-            "version": "==1.11.0"
-        },
-        "text-unidecode": {
-            "hashes": [
-                "sha256:5a1375bb2ba7968740508ae38d92e1f889a0832913cb1c447d5e2046061a396d",
-                "sha256:801e38bd550b943563660a91de8d4b6fa5df60a542be9093f7abf819f86050cc"
-            ],
-            "version": "==1.2"
-        },
-        "tzlocal": {
-            "hashes": [
-                "sha256:4ebeb848845ac898da6519b9b31879cf13b6626f7184c496037b818e238f2c4e"
-            ],
-            "version": "==1.5.1"
-        },
-        "urllib3": {
-            "hashes": [
-                "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39",
-                "sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22"
-            ],
-            "version": "==1.24.1"
-        }
-    },
-    "develop": {
         "alabaster": {
             "hashes": [
                 "sha256:446438bdcca0e05bd45ea2de1668c1d9b032e1a9154c2c259092d77031ddd359",
@@ -431,6 +108,22 @@
             ],
             "version": "==4.5.1"
         },
+        "coveralls": {
+            "hashes": [
+                "sha256:ab638e88d38916a6cedbf80a9cd8992d5fa55c77ab755e262e00b36792b7cd6d",
+                "sha256:b2388747e2529fa4c669fb1e3e2756e4e07b6ee56c7d9fce05f35ccccc913aa0"
+            ],
+            "index": "pypi",
+            "version": "==1.5.1"
+        },
+        "dateparser": {
+            "hashes": [
+                "sha256:940828183c937bcec530753211b70f673c0a9aab831e43273489b310538dff86",
+                "sha256:b452ef8b36cd78ae86a50721794bc674aa3994e19b570f7ba92810f4e0a2ae03"
+            ],
+            "index": "pypi",
+            "version": "==0.7.0"
+        },
         "decorator": {
             "hashes": [
                 "sha256:2c51dff8ef3c447388fe5e4453d24a2bf128d3a4c32af3fabef1f01c6851ab82",
@@ -438,6 +131,60 @@
             ],
             "version": "==4.3.0"
         },
+        "django": {
+            "hashes": [
+                "sha256:25df265e1fdb74f7e7305a1de620a84681bcc9c05e84a3ed97e4a1a63024f18d",
+                "sha256:d6d94554abc82ca37e447c3d28958f5ac39bd7d4adaa285543ae97fb1129fd69"
+            ],
+            "index": "pypi",
+            "version": "==2.0.9"
+        },
+        "django-cors-headers": {
+            "hashes": [
+                "sha256:5545009c9b233ea7e70da7dbab7cb1c12afa01279895086f98ec243d7eab46fa",
+                "sha256:c4c2ee97139d18541a1be7d96fe337d1694623816d83f53cb7c00da9b94acae1"
+            ],
+            "index": "pypi",
+            "version": "==2.4.0"
+        },
+        "django-crispy-forms": {
+            "hashes": [
+                "sha256:5952bab971110d0b86c278132dae0aa095beee8f723e625c3d3fa28888f1675f",
+                "sha256:705ededc554ad8736157c666681165fe22ead2dec0d5446d65fc9dd976a5a876"
+            ],
+            "index": "pypi",
+            "version": "==1.7.2"
+        },
+        "django-extensions": {
+            "hashes": [
+                "sha256:30cb6a8c7d6f75a55edf0c0c4491bd98f8264ae1616ce105f9cecac4387edd07",
+                "sha256:4ad86a7a5e84f1c77db030761ae87a600647250c652030a2b71a16e87f3a3d62"
+            ],
+            "index": "pypi",
+            "version": "==2.1.3"
+        },
+        "django-filter": {
+            "hashes": [
+                "sha256:6f4e4bc1a11151178520567b50320e5c32f8edb552139d93ea3e30613b886f56",
+                "sha256:86c3925020c27d072cdae7b828aaa5d165c2032a629abbe3c3a1be1edae61c58"
+            ],
+            "index": "pypi",
+            "version": "==2.0.0"
+        },
+        "djangorestframework": {
+            "hashes": [
+                "sha256:607865b0bb1598b153793892101d881466bd5a991de12bd6229abb18b1c86136",
+                "sha256:63f76cbe1e7d12b94c357d7e54401103b2e52aef0f7c1650d6c820ad708776e5"
+            ],
+            "index": "pypi",
+            "version": "==3.9.0"
+        },
+        "docopt": {
+            "hashes": [
+                "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491"
+            ],
+            "version": "==0.6.2"
+        },
         "docutils": {
             "hashes": [
                 "sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6",
@@ -453,6 +200,21 @@
             ],
             "version": "==1.5.0"
         },
+        "factory-boy": {
+            "hashes": [
+                "sha256:6f25cc4761ac109efd503f096e2ad99421b1159f01a29dbb917359dcd68e08ca",
+                "sha256:d552cb872b310ae78bd7429bf318e42e1e903b1a109e899a523293dfa762ea4f"
+            ],
+            "index": "pypi",
+            "version": "==2.11.1"
+        },
+        "faker": {
+            "hashes": [
+                "sha256:2621643b80a10b91999925cfd20f64d2b36f20bf22136bbdc749bb57d6ffe124",
+                "sha256:5ed822d31bd2d6edf10944d176d30dc9c886afdd381eefb7ba8b7aad86171646"
+            ],
+            "version": "==0.9.2"
+        },
         "filelock": {
             "hashes": [
                 "sha256:b8d5ca5ca1c815e1574aee746650ea7301de63d87935b3463d26368b76e31633",
@@ -460,6 +222,32 @@
             ],
             "version": "==3.0.10"
         },
+        "filemagic": {
+            "hashes": [
+                "sha256:e684359ef40820fe406f0ebc5bf8a78f89717bdb7fed688af68082d991d6dbf3"
+            ],
+            "index": "pypi",
+            "version": "==1.6"
+        },
+        "fuzzywuzzy": {
+            "extras": [
+                "speedup"
+            ],
+            "hashes": [
+                "sha256:3759bc6859daa0eecef8c82b45404bdac20c23f23136cf4c18b46b426bbc418f",
+                "sha256:5b36957ccf836e700f4468324fa80ba208990385392e217be077d5cd738ae602"
+            ],
+            "index": "pypi",
+            "version": "==0.15.0"
+        },
+        "gunicorn": {
+            "hashes": [
+                "sha256:aa8e0b40b4157b36a5df5e599f45c9c76d6af43845ba3b3b0efe2c70473c2471",
+                "sha256:fa2662097c66f920f53f70621c6c58ca4a3c4d3434205e608e121b5b3b71f4f3"
+            ],
+            "index": "pypi",
+            "version": "==19.9.0"
+        },
         "idna": {
             "hashes": [
                 "sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e",
@@ -474,6 +262,13 @@
             ],
             "version": "==1.1.0"
         },
+        "inotify-simple": {
+            "hashes": [
+                "sha256:fc2c10dd73278a1027d0663f2db51240af5946390f363a154361406ebdddd8dd"
+            ],
+            "index": "pypi",
+            "version": "==1.1.8"
+        },
         "ipython": {
             "hashes": [
                 "sha256:a5781d6934a3341a1f9acb4ea5acdc7ea0a0855e689dbe755d070ca51e995435",
@@ -503,6 +298,13 @@
             ],
             "version": "==2.10"
         },
+        "langdetect": {
+            "hashes": [
+                "sha256:91a170d5f0ade380db809b3ba67f08e95fe6c6c8641f96d67a51ff7e98a9bf30"
+            ],
+            "index": "pypi",
+            "version": "==1.0.7"
+        },
         "markupsafe": {
             "hashes": [
                 "sha256:a6be69091dac236ea9c6bc7d012beab42010fa914c459791d627dad4910eb665"
@@ -531,6 +333,13 @@
             ],
             "version": "==0.3.1"
         },
+        "pdftotext": {
+            "hashes": [
+                "sha256:e3ad11efe0aa22cbfc46aa1296b2ea5a52ad208b778288311f2801adef178ccb"
+            ],
+            "index": "pypi",
+            "version": "==2.1.1"
+        },
         "pexpect": {
             "hashes": [
                 "sha256:2a8e88259839571d1251d278476f3eec5db26deb73a70be5ed5dc5435e418aba",
@@ -546,6 +355,42 @@
             ],
             "version": "==0.7.5"
         },
+        "pillow": {
+            "hashes": [
+                "sha256:00203f406818c3f45d47bb8fe7e67d3feddb8dcbbd45a289a1de7dd789226360",
+                "sha256:0616f800f348664e694dddb0b0c88d26761dd5e9f34e1ed7b7a7d2da14b40cb7",
+                "sha256:1f7908aab90c92ad85af9d2fec5fc79456a89b3adcc26314d2cde0e238bd789e",
+                "sha256:2ea3517cd5779843de8a759c2349a3cd8d3893e03ab47053b66d5ec6f8bc4f93",
+                "sha256:48a9f0538c91fc136b3a576bee0e7cd174773dc9920b310c21dcb5519722e82c",
+                "sha256:5280ebc42641a1283b7b1f2c20e5b936692198b9dd9995527c18b794850be1a8",
+                "sha256:5e34e4b5764af65551647f5cc67cf5198c1d05621781d5173b342e5e55bf023b",
+                "sha256:63b120421ab85cad909792583f83b6ca3584610c2fe70751e23f606a3c2e87f0",
+                "sha256:696b5e0109fe368d0057f484e2e91717b49a03f1e310f857f133a4acec9f91dd",
+                "sha256:870ed021a42b1b02b5fe4a739ea735f671a84128c0a666c705db2cb9abd528eb",
+                "sha256:916da1c19e4012d06a372127d7140dae894806fad67ef44330e5600d77833581",
+                "sha256:9303a289fa0811e1c6abd9ddebfc770556d7c3311cb2b32eff72164ddc49bc64",
+                "sha256:9577888ecc0ad7d06c3746afaba339c94d62b59da16f7a5d1cff9e491f23dace",
+                "sha256:987e1c94a33c93d9b209315bfda9faa54b8edfce6438a1e93ae866ba20de5956",
+                "sha256:99a3bbdbb844f4fb5d6dd59fac836a40749781c1fa63c563bc216c27aef63f60",
+                "sha256:99db8dc3097ceafbcff9cb2bff384b974795edeb11d167d391a02c7bfeeb6e16",
+                "sha256:a5a96cf49eb580756a44ecf12949e52f211e20bffbf5a95760ac14b1e499cd37",
+                "sha256:aa6ca3eb56704cdc0d876fc6047ffd5ee960caad52452fbee0f99908a141a0ae",
+                "sha256:aade5e66795c94e4a2b2624affeea8979648d1b0ae3fcee17e74e2c647fc4a8a",
+                "sha256:b78905860336c1d292409e3df6ad39cc1f1c7f0964e66844bbc2ebfca434d073",
+                "sha256:b92f521cdc4e4a3041cc343625b699f20b0b5f976793fb45681aac1efda565f8",
+                "sha256:bfde84bbd6ae5f782206d454b67b7ee8f7f818c29b99fd02bf022fd33bab14cb",
+                "sha256:c2b62d3df80e694c0e4a0ed47754c9480521e25642251b3ab1dff050a4e60409",
+                "sha256:c5e2be6c263b64f6f7656e23e18a4a9980cffc671442795682e8c4e4f815dd9f",
+                "sha256:c99aa3c63104e0818ec566f8ff3942fb7c7a8f35f9912cb63fd8e12318b214b2",
+                "sha256:dae06620d3978da346375ebf88b9e2dd7d151335ba668c995aea9ed07af7add4",
+                "sha256:db5499d0710823fa4fb88206050d46544e8f0e0136a9a5f5570b026584c8fd74",
+                "sha256:f36baafd82119c4a114b9518202f2a983819101dcc14b26e43fc12cbefdce00e",
+                "sha256:f52b79c8796d81391ab295b04e520bda6feed54d54931708872e8f9ae9db0ea1",
+                "sha256:ff8cff01582fa1a7e533cb97f628531c4014af4b5f38e33cdcfe5eec29b6d888"
+            ],
+            "index": "pypi",
+            "version": "==5.3.0"
+        },
         "pluggy": {
             "hashes": [
                 "sha256:447ba94990e8014ee25ec853339faf7b0fc8050cdc3289d4d71f7f410fb90095",
@@ -590,6 +435,13 @@
             ],
             "version": "==2.2.0"
         },
+        "pyocr": {
+            "hashes": [
+                "sha256:b6ba6263fd92da56627dff6d263d991a2246aacd117d1788f11b93f419ca395f"
+            ],
+            "index": "pypi",
+            "version": "==0.5.3"
+        },
         "pyparsing": {
             "hashes": [
                 "sha256:40856e74d4987de5d01761a22d1621ae1c7f8774585acae358aa5c5936c6c90b",
@@ -650,6 +502,37 @@
             "index": "pypi",
             "version": "==1.24.0"
         },
+        "python-dateutil": {
+            "hashes": [
+                "sha256:063df5763652e21de43de7d9e00ccf239f953a832941e37be541614732cdfc93",
+                "sha256:88f9287c0174266bb0d8cedd395cfba9c58e87e5ad86b2ce58859bc11be3cf02"
+            ],
+            "index": "pypi",
+            "version": "==2.7.5"
+        },
+        "python-dotenv": {
+            "hashes": [
+                "sha256:122290a38ece9fe4f162dc7c95cae3357b983505830a154d3c98ef7f6c6cea77",
+                "sha256:4a205787bc829233de2a823aa328e44fd9996fedb954989a21f1fc67c13d7a77"
+            ],
+            "index": "pypi",
+            "version": "==0.9.1"
+        },
+        "python-gnupg": {
+            "hashes": [
+                "sha256:2d158dfc6b54927752b945ebe57e6a0c45da27747fa3b9ae66eccc0d2147ac0d",
+                "sha256:faa69bab58ed0936f0ccf96c99b92369b7a1819305d37dfe5c927d21a437a09d"
+            ],
+            "index": "pypi",
+            "version": "==0.4.3"
+        },
+        "python-levenshtein": {
+            "hashes": [
+                "sha256:033a11de5e3d19ea25c9302d11224e1a1898fe5abd23c61c7c360c25195e3eb1"
+            ],
+            "markers": "extra == 'speedup'",
+            "version": "==0.12.0"
+        },
         "pytz": {
             "hashes": [
                 "sha256:31cb35c89bd7d333cd32c5f278fca91b523b0834369e757f4c5641ea252236ca",
@@ -658,6 +541,20 @@
             "index": "pypi",
             "version": "==2018.7"
         },
+        "regex": {
+            "hashes": [
+                "sha256:0ef96690c3d2294155b7d44187ca4a151e45c931cb768e106ba464a9fa64c5da",
+                "sha256:251683e01a3bcacd9188acf0d4caf7b29a3b963c843159311825613ae144cddb",
+                "sha256:3fe15a75fe00f04d1ec16713d55cf1e206077c450267a10b33318756fb8b3f99",
+                "sha256:53a962f9dc28cdf403978a142cb1e054479759ad64d312a999f9f042c25b5c9a",
+                "sha256:8bd1da6a93d32336a5e5432886dd8543004f0591c39b83dbfa60705cccdf414d",
+                "sha256:b5423061918f602e9342b54d746ac31c598d328ecaf4ef0618763e960c926fd4",
+                "sha256:d80ebc65b1f7d0403117f59309c16eac24be6a0bc730b593a79f703462858d94",
+                "sha256:fd8419979639b7de7fb964a13bce3ac47e6fe33043b83de0398c3067986e5659",
+                "sha256:ff2f15b2b0b4b58ba8a1de651780a0d3fd54f96ad6b77dceb77695220e5d7b7a"
+            ],
+            "version": "==2018.11.2"
+        },
         "requests": {
             "hashes": [
                 "sha256:99dcfdaaeb17caf6e526f32b6a7b780461512ab3f1d992187801694cba42770c",
@@ -700,6 +597,13 @@
             ],
             "version": "==1.1.0"
         },
+        "text-unidecode": {
+            "hashes": [
+                "sha256:5a1375bb2ba7968740508ae38d92e1f889a0832913cb1c447d5e2046061a396d",
+                "sha256:801e38bd550b943563660a91de8d4b6fa5df60a542be9093f7abf819f86050cc"
+            ],
+            "version": "==1.2"
+        },
         "toml": {
             "hashes": [
                 "sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c",
@@ -722,6 +626,12 @@
             ],
             "version": "==4.3.2"
         },
+        "tzlocal": {
+            "hashes": [
+                "sha256:4ebeb848845ac898da6519b9b31879cf13b6626f7184c496037b818e238f2c4e"
+            ],
+            "version": "==1.5.1"
+        },
         "urllib3": {
             "hashes": [
                 "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39",
@@ -743,5 +653,6 @@
             ],
             "version": "==0.1.7"
         }
-    }
+    },
+    "develop": {}
 }
diff --git a/requirements.txt b/requirements.txt
index 61d98a6d7..a62185562 100755
--- a/requirements.txt
+++ b/requirements.txt
@@ -8,28 +8,47 @@ backcall==0.1.0
 certifi==2018.10.15
 chardet==3.0.4
 coverage==4.5.1
+coveralls==1.5.1
+dateparser==0.7.0
 decorator==4.3.0
+django-cors-headers==2.4.0
+django-crispy-forms==1.7.2
+django-extensions==2.1.3
+django-filter==2.0.0
+django==2.0.9
+djangorestframework==3.9.0
+docopt==0.6.2
 docutils==0.14
 execnet==1.5.0
+factory-boy==2.11.1
+faker==0.9.2
 filelock==3.0.10
+filemagic==1.6
+fuzzywuzzy[speedup]==0.15.0
+gunicorn==19.9.0
 idna==2.7
 imagesize==1.1.0
+inotify-simple==1.1.8
 ipython-genutils==0.2.0
 ipython==7.1.1
 jedi==0.13.1
 jinja2==2.10
+langdetect==1.0.7
 markupsafe==1.0
 more-itertools==4.3.0
 packaging==18.0
 parso==0.3.1
+pdftotext==2.1.1
 pexpect==4.6.0 ; sys_platform != 'win32'
 pickleshare==0.7.5
+pillow==5.3.0
 pluggy==0.8.0
 prompt-toolkit==2.0.7
 ptyprocess==0.6.0
 py==1.7.0
 pycodestyle==2.4.0
 pygments==2.2.0
+pyocr==0.5.3
 pyparsing==2.3.0
 pytest-cov==2.6.0
 pytest-django==3.4.3
@@ -38,16 +57,23 @@ pytest-forked==0.2
 pytest-sugar==0.9.1
 pytest-xdist==1.24.0
 pytest==3.9.3
+python-dateutil==2.7.5
+python-dotenv==0.9.1
+python-gnupg==0.4.3
+python-levenshtein==0.12.0 ; extra == 'speedup'
 pytz==2018.7
+regex==2018.11.2
 requests==2.20.0
 six==1.11.0
 snowballstemmer==1.2.1
 sphinx==1.8.1
 sphinxcontrib-websupport==1.1.0
 termcolor==1.1.0
+text-unidecode==1.2
 toml==0.10.0
 tox==3.5.3
 traitlets==4.3.2
+tzlocal==1.5.1
 urllib3==1.24.1
 virtualenv==16.1.0
 wcwidth==0.1.7

From 8d6825dac0830b12519ae25e4f7f300bd410f002 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sat, 3 Nov 2018 13:42:03 +0000
Subject: [PATCH 47/47] I'm going to have to ditch requirements.txt if it can't
 be reliably generated

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index a62185562..92f4b0ff8 100755
--- a/requirements.txt
+++ b/requirements.txt
@@ -39,7 +39,7 @@ more-itertools==4.3.0
 packaging==18.0
 parso==0.3.1
 pdftotext==2.1.1
-pexpect==4.6.0 ; sys_platform != 'win32'
+pexpect==4.6.0 
 pickleshare==0.7.5
 pillow==5.3.0
 pluggy==0.8.0