From fb6f2e07c97d8b6ad455a269d24f216ac80f5cf9 Mon Sep 17 00:00:00 2001
From: Jonas Winkler <dev@jpwinkler.de>
Date: Thu, 13 Sep 2018 15:19:25 +0200
Subject: [PATCH 01/47] Added a bunch of new features: - Debug mode is now
 configurable in the configuration file. This way, we don't have to edit
 versioned files to disable it on production systems. - Recent correspondents
 filter (enable in configuration file) - Document actions: Edit tags and
 correspondents on multiple documents at once - Replaced month list filter
 with date drilldown - Sortable document count columns on Tag and
 Correspondent admin - Last correspondence column on Correspondent admin -
 Save and edit next functionality for document editing

---
 .gitignore                                    |   2 +
 paperless.conf.example                        |  10 ++
 requirements.txt                              |   3 +
 src/documents/actions.py                      | 108 +++++++++++++
 src/documents/admin.py                        | 142 +++++++++++++-----
 .../admin/documents/document/change_form.html |  18 ++-
 .../documents/document/select_object.html     |  46 ++++++
 src/paperless/settings.py                     |  14 +-
 8 files changed, 301 insertions(+), 42 deletions(-)
 mode change 100644 => 100755 requirements.txt
 create mode 100644 src/documents/actions.py
 create mode 100644 src/documents/templates/admin/documents/document/select_object.html

diff --git a/.gitignore b/.gitignore
index a16f958a3..439d9df4b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -81,3 +81,5 @@ docker-compose.env
 scripts/import-for-development
 scripts/nuke
 
+# Static files collected by the collectstatic command
+static/
diff --git a/paperless.conf.example b/paperless.conf.example
index 15498a26a..05cf81724 100644
--- a/paperless.conf.example
+++ b/paperless.conf.example
@@ -59,6 +59,11 @@ PAPERLESS_EMAIL_SECRET=""
 ####                              Security                                 ####
 ###############################################################################
 
+# Controls whether django's debug mode is enabled. Disable this on production
+# systems. Debug mode is enabled by default.
+PAPERLESS_DEBUG="false"
+
+
 # Paperless can be instructed to attempt to encrypt your PDF files with GPG
 # using the PAPERLESS_PASSPHRASE specified below.  If however you're not
 # concerned about encrypting these files (for example if you have disk
@@ -203,3 +208,8 @@ PAPERLESS_EMAIL_SECRET=""
 # positive integer, but if you don't define one in paperless.conf, a default of
 # 100 will be used.
 #PAPERLESS_LIST_PER_PAGE=100
+
+
+# The number of years for which a correspondent will be included in the recent
+# correspondents filter.
+#PAPERLESS_RECENT_CORRESPONDENT_YEARS=1
diff --git a/requirements.txt b/requirements.txt
old mode 100644
new mode 100755
index 0476efef1..89c7e296f
--- a/requirements.txt
+++ b/requirements.txt
@@ -24,6 +24,7 @@ idna==2.7
 inotify-simple==1.1.8
 langdetect==1.0.7
 more-itertools==4.3.0
+numpy==1.15.1
 pdftotext==2.1.0
 pillow==5.2.0
 pluggy==0.7.1; python_version != '3.1.*'
@@ -45,6 +46,8 @@ pytz==2018.5
 regex==2018.8.29
 requests==2.19.1
 six==1.11.0
+scikit-learn==0.19.2
+scipy==1.1.0
 termcolor==1.1.0
 text-unidecode==1.2
 tzlocal==1.5.1
diff --git a/src/documents/actions.py b/src/documents/actions.py
new file mode 100644
index 000000000..3db5cd314
--- /dev/null
+++ b/src/documents/actions.py
@@ -0,0 +1,108 @@
+from django.contrib import messages
+from django.contrib.admin import helpers
+from django.contrib.admin.utils import model_ngettext
+from django.core.exceptions import PermissionDenied
+from django.template.response import TemplateResponse
+
+from documents.models import Tag, Correspondent
+
+
+def select_action(modeladmin, request, queryset, title, action, modelclass, success_message="", document_action=None, queryset_action=None):
+    opts = modeladmin.model._meta
+    app_label = opts.app_label
+
+    if not modeladmin.has_change_permission(request):
+        raise PermissionDenied
+
+    if request.POST.get('post'):
+        n = queryset.count()
+        selected_object = modelclass.objects.get(id=request.POST.get('obj_id'))
+        if n:
+            for document in queryset:
+                if document_action:
+                    document_action(document, selected_object)
+                document_display = str(document)
+                modeladmin.log_change(request, document, document_display)
+            if queryset_action:
+                queryset_action(queryset, selected_object)
+
+            modeladmin.message_user(request, success_message % {
+                "selected_object": selected_object.name, "count": n, "items": model_ngettext(modeladmin.opts, n)
+            }, messages.SUCCESS)
+
+        # Return None to display the change list page again.
+        return None
+
+    context = dict(
+        modeladmin.admin_site.each_context(request),
+        title=title,
+        queryset=queryset,
+        opts=opts,
+        action_checkbox_name=helpers.ACTION_CHECKBOX_NAME,
+        media=modeladmin.media,
+        action=action,
+        objects=modelclass.objects.all(),
+        itemname=model_ngettext(modelclass, 1)
+    )
+
+    request.current_app = modeladmin.admin_site.name
+
+    return TemplateResponse(request, "admin/%s/%s/select_object.html" % (app_label, opts.model_name), context)
+
+
+def simple_action(modeladmin, request, queryset, success_message="", document_action=None, queryset_action=None):
+    if not modeladmin.has_change_permission(request):
+        raise PermissionDenied
+
+    n = queryset.count()
+    if n:
+        for document in queryset:
+            if document_action:
+                document_action(document)
+            document_display = str(document)
+            modeladmin.log_change(request, document, document_display)
+        if queryset_action:
+            queryset_action(queryset)
+        modeladmin.message_user(request, success_message % {
+            "count": n, "items": model_ngettext(modeladmin.opts, n)
+        }, messages.SUCCESS)
+
+    # Return None to display the change list page again.
+    return None
+
+
+def add_tag_to_selected(modeladmin, request, queryset):
+    return select_action(modeladmin=modeladmin, request=request, queryset=queryset,
+                         title="Add tag to multiple documents",
+                         action="add_tag_to_selected",
+                         modelclass=Tag,
+                         success_message="Successfully added tag %(selected_object)s to %(count)d %(items)s.",
+                         document_action=lambda doc, tag: doc.tags.add(tag))
+add_tag_to_selected.short_description = "Add tag to selected documents"
+
+
+def remove_tag_from_selected(modeladmin, request, queryset):
+    return select_action(modeladmin=modeladmin, request=request, queryset=queryset,
+                         title="Remove tag from multiple documents",
+                         action="remove_tag_from_selected",
+                         modelclass=Tag,
+                         success_message="Successfully removed tag %(selected_object)s from %(count)d %(items)s.",
+                         document_action=lambda doc, tag: doc.tags.remove(tag))
+remove_tag_from_selected.short_description = "Remove tag from selected documents"
+
+
+def set_correspondent_on_selected(modeladmin, request, queryset):
+    return select_action(modeladmin=modeladmin, request=request, queryset=queryset,
+                         title="Set correspondent on multiple documents",
+                         action="set_correspondent_on_selected",
+                         modelclass=Correspondent,
+                         success_message="Successfully set correspondent %(selected_object)s on %(count)d %(items)s.",
+                         queryset_action=lambda qs, correspondent: qs.update(correspondent=correspondent))
+set_correspondent_on_selected.short_description = "Set correspondent on selected documents"
+
+
+def remove_correspondent_from_selected(modeladmin, request, queryset):
+    return simple_action(modeladmin=modeladmin, request=request, queryset=queryset,
+                         success_message="Successfully removed correspondent from %(count)d %(items)s.",
+                         queryset_action=lambda qs: qs.update(correspondent=None))
+remove_correspondent_from_selected.short_description = "Remove correspondent from selected documents"
diff --git a/src/documents/admin.py b/src/documents/admin.py
index 36154b6ba..d545c1c02 100644
--- a/src/documents/admin.py
+++ b/src/documents/admin.py
@@ -1,44 +1,25 @@
-from datetime import datetime
+from datetime import datetime, timedelta
 
 from django.conf import settings
-from django.contrib import admin
+from django.contrib import admin, messages
+from django.contrib.admin.templatetags.admin_urls import add_preserved_filters
 from django.contrib.auth.models import User, Group
+from django.http import HttpResponseRedirect
 try:
     from django.core.urlresolvers import reverse
 except ImportError:
     from django.urls import reverse
 from django.templatetags.static import static
-from django.utils.safestring import mark_safe
 from django.utils.html import format_html, format_html_join
+from django.utils.http import urlquote
+from django.utils.safestring import mark_safe
+from django.db import models
 
+from documents.actions import add_tag_to_selected, remove_tag_from_selected, set_correspondent_on_selected, \
+    remove_correspondent_from_selected
 from .models import Correspondent, Tag, Document, Log
 
 
-class MonthListFilter(admin.SimpleListFilter):
-
-    title = "Month"
-
-    # Parameter for the filter that will be used in the URL query.
-    parameter_name = "month"
-
-    def lookups(self, request, model_admin):
-        r = []
-        for document in Document.objects.all():
-            r.append((
-                document.created.strftime("%Y-%m"),
-                document.created.strftime("%B %Y")
-            ))
-        return sorted(set(r), key=lambda x: x[0], reverse=True)
-
-    def queryset(self, request, queryset):
-
-        if not self.value():
-            return None
-
-        year, month = self.value().split("-")
-        return queryset.filter(created__year=year, created__month=month)
-
-
 class FinancialYearFilter(admin.SimpleListFilter):
 
     title = "Financial Year"
@@ -104,18 +85,43 @@ class FinancialYearFilter(admin.SimpleListFilter):
                                created__lte=self._fy_end(end))
 
 
+class RecentCorrespondentFilter(admin.RelatedFieldListFilter):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.title = "correspondent (recent)"
+
+    def field_choices(self, field, request, model_admin):
+        lookups = []
+        if settings.PAPERLESS_RECENT_CORRESPONDENT_YEARS and settings.PAPERLESS_RECENT_CORRESPONDENT_YEARS > 0:
+            date_limit = datetime.now() - timedelta(days=365*settings.PAPERLESS_RECENT_CORRESPONDENT_YEARS)
+            for c in Correspondent.objects.filter(documents__created__gte=date_limit).distinct():
+                lookups.append((c.id, c.name))
+        return lookups
+
+
 class CommonAdmin(admin.ModelAdmin):
     list_per_page = settings.PAPERLESS_LIST_PER_PAGE
 
 
 class CorrespondentAdmin(CommonAdmin):
 
-    list_display = ("name", "match", "matching_algorithm", "document_count")
+    list_display = ("name", "match", "matching_algorithm", "document_count", "last_correspondence")
     list_filter = ("matching_algorithm",)
     list_editable = ("match", "matching_algorithm")
 
+    def get_queryset(self, request):
+        qs = super(CorrespondentAdmin, self).get_queryset(request)
+        qs = qs.annotate(document_count=models.Count("documents"), last_correspondence=models.Max("documents__created"))
+        return qs
+
     def document_count(self, obj):
-        return obj.documents.count()
+        return obj.document_count
+    document_count.admin_order_field = "document_count"
+
+    def last_correspondence(self, obj):
+        return obj.last_correspondence
+    last_correspondence.admin_order_field = "last_correspondence"
 
 
 class TagAdmin(CommonAdmin):
@@ -125,8 +131,14 @@ class TagAdmin(CommonAdmin):
     list_filter = ("colour", "matching_algorithm")
     list_editable = ("colour", "match", "matching_algorithm")
 
+    def get_queryset(self, request):
+        qs = super(TagAdmin, self).get_queryset(request)
+        qs = qs.annotate(document_count=models.Count("documents"))
+        return qs
+
     def document_count(self, obj):
-        return obj.documents.count()
+        return obj.document_count
+    document_count.admin_order_field = "document_count"
 
 
 class DocumentAdmin(CommonAdmin):
@@ -140,12 +152,18 @@ class DocumentAdmin(CommonAdmin):
     readonly_fields = ("added",)
     list_display = ("title", "created", "added", "thumbnail", "correspondent",
                     "tags_")
-    list_filter = ("tags", "correspondent", FinancialYearFilter,
-                   MonthListFilter)
+    list_filter = ("tags", ('correspondent', RecentCorrespondentFilter), "correspondent", FinancialYearFilter)
+
     filter_horizontal = ("tags",)
 
     ordering = ["-created", "correspondent"]
 
+    actions = [add_tag_to_selected, remove_tag_from_selected, set_correspondent_on_selected, remove_correspondent_from_selected]
+
+    date_hierarchy = 'created'
+
+    document_queue = None
+
     def has_add_permission(self, request):
         return False
 
@@ -153,6 +171,56 @@ class DocumentAdmin(CommonAdmin):
         return obj.created.date().strftime("%Y-%m-%d")
     created_.short_description = "Created"
 
+    def changelist_view(self, request, extra_context=None):
+        response = super().changelist_view(request, extra_context)
+
+        if request.method == 'GET':
+            cl = self.get_changelist_instance(request)
+            self.document_queue = [doc.id for doc in cl.queryset]
+
+        return response
+
+    def change_view(self, request, object_id=None, form_url='', extra_context=None):
+        extra_context = extra_context or {}
+        doc = Document.objects.get(id=object_id)
+        if self.document_queue and object_id and int(object_id) in self.document_queue:
+            # There is a queue of documents
+            current_index = self.document_queue.index(int(object_id))
+            if current_index < len(self.document_queue) - 1:
+                # ... and there are still documents in the queue
+                extra_context['next_object'] = self.document_queue[current_index + 1]
+        return super(DocumentAdmin, self).change_view(
+            request, object_id, form_url, extra_context=extra_context,
+        )
+
+    def response_change(self, request, obj):
+
+        # This is mostly copied from ModelAdmin.response_change()
+        opts = self.model._meta
+        preserved_filters = self.get_preserved_filters(request)
+
+        msg_dict = {
+            'name': opts.verbose_name,
+            'obj': format_html('<a href="{}">{}</a>', urlquote(request.path), obj),
+        }
+        if "_saveandeditnext" in request.POST:
+            msg = format_html(
+                'The {name} "{obj}" was changed successfully. Editing next object.',
+                **msg_dict
+            )
+            self.message_user(request, msg, messages.SUCCESS)
+            redirect_url = reverse('admin:%s_%s_change' %
+                                   (opts.app_label, opts.model_name),
+                                   args=(request.POST['_next_object'],),
+                                   current_app=self.admin_site.name)
+            redirect_url = add_preserved_filters({'preserved_filters': preserved_filters, 'opts': opts}, redirect_url)
+            response = HttpResponseRedirect(redirect_url)
+        else:
+            response = super().response_change(request, obj)
+
+        return response
+
+    @mark_safe
     def thumbnail(self, obj):
         return self._html_tag(
             "a",
@@ -165,8 +233,8 @@ class DocumentAdmin(CommonAdmin):
             ),
             href=obj.download_url
         )
-    thumbnail.allow_tags = True
 
+    @mark_safe
     def tags_(self, obj):
         r = ""
         for tag in obj.tags.all():
@@ -183,10 +251,11 @@ class DocumentAdmin(CommonAdmin):
                     )
                 }
             )
-        return mark_safe(r)
-    tags_.allow_tags = True
+        return r
 
+    @mark_safe
     def document(self, obj):
+        # TODO: is this method even used anymore?
         return self._html_tag(
             "a",
             self._html_tag(
@@ -199,7 +268,6 @@ class DocumentAdmin(CommonAdmin):
             ),
             href=obj.download_url
         )
-    document.allow_tags = True
 
     @staticmethod
     def _html_tag(kind, inside=None, **kwargs):
diff --git a/src/documents/templates/admin/documents/document/change_form.html b/src/documents/templates/admin/documents/document/change_form.html
index 7bd0e483f..88fae955d 100644
--- a/src/documents/templates/admin/documents/document/change_form.html
+++ b/src/documents/templates/admin/documents/document/change_form.html
@@ -1,5 +1,21 @@
 {% extends 'admin/change_form.html' %}
 
+{% block content %}
+
+{{ block.super }}
+
+{% if next_object %}
+	<script type="text/javascript">//<![CDATA[
+		(function($){
+			$('<input type="submit" value="Save and edit next" name="_saveandeditnext" />')
+			.prependTo('div.submit-row');
+			$('<input type="hidden" value="{{next_object}}" name="_next_object" />')
+			.prependTo('div.submit-row');
+		})(django.jQuery);
+	//]]></script>
+{% endif %}
+
+{% endblock content %}
 
 {% block footer %}
 
@@ -10,4 +26,4 @@
 		django.jQuery(".field-created input").first().attr("type", "date")
 	</script>
 
-{% endblock footer %}
\ No newline at end of file
+{% endblock footer %}
diff --git a/src/documents/templates/admin/documents/document/select_object.html b/src/documents/templates/admin/documents/document/select_object.html
new file mode 100644
index 000000000..1439b5c21
--- /dev/null
+++ b/src/documents/templates/admin/documents/document/select_object.html
@@ -0,0 +1,46 @@
+{% extends "admin/base_site.html" %}
+{% load i18n l10n admin_urls static %}
+{% load staticfiles %}
+
+{% block extrahead %}
+{{ block.super }}
+{{ media }}
+<script type="text/javascript" src="{% static 'admin/js/cancel.js' %}"></script>
+
+{% endblock %}
+
+{% block bodyclass %}{{ block.super }} app-{{ opts.app_label }} model-{{ opts.model_name }} delete-confirmation delete-selected-confirmation{% endblock %}
+
+{% block breadcrumbs %}
+<div class="breadcrumbs">
+    <a href="{% url 'admin:index' %}">{% trans 'Home' %}</a>
+    &rsaquo; <a href="{% url 'admin:app_list' app_label=opts.app_label %}">{{ opts.app_config.verbose_name }}</a>
+    &rsaquo; <a href="{% url opts|admin_urlname:'changelist' %}">{{ opts.verbose_name_plural|capfirst }}</a>
+    &rsaquo; {{title}}
+</div>
+{% endblock %}
+
+{% block content %}
+<p>Please select the {{itemname}}.</p>
+<form method="post">{% csrf_token %}
+    <div>
+        {% for obj in queryset %}
+        <input type="hidden" name="{{ action_checkbox_name }}" value="{{ obj.pk|unlocalize }}"/>
+        {% endfor %}
+        <p>
+            <select name="obj_id">
+                {% for obj in objects %}
+                <option value="{{obj.id}}">{{obj.name}}</option>
+                {% endfor %}
+            </select>
+        </p>
+
+        <input type="hidden" name="action" value="{{action}}"/>
+        <input type="hidden" name="post" value="yes"/>
+        <p>
+            <input type="submit" value="{% trans "Confirm" %}" />
+            <a href="#" class="button cancel-link">{% trans "Go back" %}</a>
+        </p>
+    </div>
+</form>
+{% endblock %}
diff --git a/src/paperless/settings.py b/src/paperless/settings.py
index 956b90a7f..e6f3da0cb 100644
--- a/src/paperless/settings.py
+++ b/src/paperless/settings.py
@@ -22,12 +22,12 @@ elif os.path.exists("/usr/local/etc/paperless.conf"):
     load_dotenv("/usr/local/etc/paperless.conf")
 
 
-def __get_boolean(key):
+def __get_boolean(key, default="NO"):
     """
     Return a boolean value based on whatever the user has supplied in the
     environment based on whether the value "looks like" it's True or not.
     """
-    return bool(os.getenv(key, "NO").lower() in ("yes", "y", "1", "t", "true"))
+    return bool(os.getenv(key, default).lower() in ("yes", "y", "1", "t", "true"))
 
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
@@ -47,7 +47,7 @@ SECRET_KEY = os.getenv(
 
 
 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = True
+DEBUG = __get_boolean("PAPERLESS_DEBUG", "YES")
 
 LOGIN_URL = "admin:login"
 
@@ -81,7 +81,7 @@ INSTALLED_APPS = [
 
     "rest_framework",
     "crispy_forms",
-    "django_filters",
+    "django_filters"
 
 ]
 
@@ -292,3 +292,9 @@ FY_END = os.getenv("PAPERLESS_FINANCIAL_YEAR_END")
 
 # Specify the default date order (for autodetected dates)
 DATE_ORDER = os.getenv("PAPERLESS_DATE_ORDER", "DMY")
+
+# Specify for how many years a correspondent is considered recent. Recent
+# correspondents will be shown in a separate "Recent correspondents" filter as
+# well. Set to 0 to disable this filter.
+PAPERLESS_RECENT_CORRESPONDENT_YEARS = int(os.getenv(
+    "PAPERLESS_RECENT_CORRESPONDENT_YEARS", 0))

From cce6b43062b25464a00d09f47fffda83a86e925b Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sat, 22 Sep 2018 13:59:50 +0100
Subject: [PATCH 02/47] Clean up release notes

---
 docs/changelog.rst | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 804447855..9396493a7 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -15,7 +15,8 @@ Changelog
 * As his last bit of effort on this release, Joshua also added some code to
   allow you to view the documents inline rather than download them as an
   attachment. `#400`_
-* Finally, `ahyear`_ found a slip in the Docker documentation and patched it. `#401`_
+* Finally, `ahyear`_ found a slip in the Docker documentation and patched it.
+  `#401`_
 
 
 2.2.1
@@ -32,14 +33,14 @@ Changelog
   version of Paperless that supports Django 2.0!  As a result of their hard
   work, you can now also run Paperless on Python 3.7 as well: `#386`_ &
   `#390`_.
-* `Stéphane Brunner`_ added a few lines of code that made tagging interface a lot
-  easier on those of us with lots of different tags: `#391`_.
+* `Stéphane Brunner`_ added a few lines of code that made tagging interface a
+  lot easier on those of us with lots of different tags: `#391`_.
 * `Kilian Koeltzsch`_ noticed a bug in how we capture & automatically create
   tags, so that's fixed now too: `#384`_.
 * `erikarvstedt`_ tweaked the behaviour of the test suite to be better behaved
   for packaging environments: `#383`_.
-* `Lukasz Soluch`_ added CORS support to make building a new Javascript-based front-end
-  cleaner & easier: `#387`_.
+* `Lukasz Soluch`_ added CORS support to make building a new Javascript-based
+  front-end cleaner & easier: `#387`_.
 
 
 2.1.0

From b4b4d8f25ec274c6c82944f20ca009ac368f364b Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sat, 22 Sep 2018 14:00:00 +0100
Subject: [PATCH 03/47] Add an example for pdf2pdfocr with the pre-consume hook

---
 docs/consumption.rst | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/docs/consumption.rst b/docs/consumption.rst
index bf62ed0a2..fabaf2641 100644
--- a/docs/consumption.rst
+++ b/docs/consumption.rst
@@ -76,6 +76,29 @@ Pre-consumption script
 
 * Document file name
 
+A simple but common example for this would be creating a simple script like
+this:
+
+.. code:: bash
+    :name: "/usr/local/bin/ocr-pdf"
+
+    #!/usr/bin/env bash
+    pdf2pdfocr.py -i ${1}
+
+.. code:: bash
+    :name: /etc/paperless.conf
+
+    ...
+    PAPERLESS_PRE_CONSUME_SCRIPT="/usr/local/bin/ocr-pdf"
+    ...
+
+This will pass the path to the document about to be consumed to ``/usr/local/bin/ocr-pdf``,
+which will in turn call `pdf2pdfocr.py`_ on your document, which will then
+overwrite the file with an OCR'd version of the file and exit.  At which point,
+the consumption process will begin with the newly modified file.
+
+.. _pdf2pdfocr.py: https://github.com/LeoFCardoso/pdf2pdfocr
+
 
 .. _consumption-director-hook-variables-post:
 

From 60ee08adec5491c80d0f0e91a7497892dc9cbb43 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sat, 22 Sep 2018 15:27:22 +0100
Subject: [PATCH 04/47] Reduce duplication in docker-compose.env.example

See #404 for more info on where this came from.
---
 docker-compose.env.example | 44 ++++++++++++--------------------------
 1 file changed, 14 insertions(+), 30 deletions(-)

diff --git a/docker-compose.env.example b/docker-compose.env.example
index 3c1664573..51332437d 100644
--- a/docker-compose.env.example
+++ b/docker-compose.env.example
@@ -1,38 +1,22 @@
 # Environment variables to set for Paperless
-# Commented out variables will be replaced by a default within Paperless.
+# Commented out variables will be replaced with a default within Paperless.
+#
+# In addition to what you see here, you can also define any values you find in
+# paperless.conf.example here.  Values like:
+#
+# * PAPERLESS_PASSPHRASE
+# * PAPERLESS_CONSUMPTION_DIR
+# * PAPERLESS_CONSUME_MAIL_HOST
+#
+# ...are all explained in that file but can be defined here, since the Docker
+# installation doesn't make use of paperless.conf.
 
-# Passphrase Paperless uses to encrypt and decrypt your documents, if you want
-# encryption at all.
-# PAPERLESS_PASSPHRASE=CHANGE_ME
 
-# The amount of threads to use for text recognition
-# PAPERLESS_OCR_THREADS=4
-
-# Additional languages to install for text recognition
+# Additional languages to install for text recognition.  Note that this is
+# different from PAPERLESS_OCR_LANGUAGE (default=eng), which defines the
+# default language used when guessing the language from the OCR output.
 # PAPERLESS_OCR_LANGUAGES=deu ita
 
 # You can change the default user and group id to a custom one
 # USERMAP_UID=1000
 # USERMAP_GID=1000
-
-###############################################################################
-####                         Mail Consumption                              ####
-###############################################################################
-
-# These values are required if you want paperless to check a particular email
-# box every 10 minutes and attempt to consume documents from there.  If you
-# don't define a HOST, mail checking will just be disabled.
-# Don't use quotes after = or it will crash your docker
-# PAPERLESS_CONSUME_MAIL_HOST=
-# PAPERLESS_CONSUME_MAIL_PORT=
-# PAPERLESS_CONSUME_MAIL_USER=
-# PAPERLESS_CONSUME_MAIL_PASS=
-
-# Override the default IMAP inbox here. If it's not set, Paperless defaults to
-# INBOX.
-# PAPERLESS_CONSUME_MAIL_INBOX=INBOX
-
-# Any email sent to the target account that does not contain this text will be
-# ignored.  Mail checking won't work without this.
-# PAPERLESS_EMAIL_SECRET=
-

From 425bbe34efd57db8f3c06b0796d2470f4c3f3bd2 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sat, 22 Sep 2018 16:17:18 +0100
Subject: [PATCH 05/47] Make the names of the sample files visible

---
 docs/consumption.rst | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/docs/consumption.rst b/docs/consumption.rst
index fabaf2641..15f6c6393 100644
--- a/docs/consumption.rst
+++ b/docs/consumption.rst
@@ -79,14 +79,16 @@ Pre-consumption script
 A simple but common example for this would be creating a simple script like
 this:
 
+``/usr/local/bin/ocr-pdf``
+
 .. code:: bash
-    :name: "/usr/local/bin/ocr-pdf"
 
     #!/usr/bin/env bash
     pdf2pdfocr.py -i ${1}
 
+``/etc/paperless.conf``
+
 .. code:: bash
-    :name: /etc/paperless.conf
 
     ...
     PAPERLESS_PRE_CONSUME_SCRIPT="/usr/local/bin/ocr-pdf"

From 9682a6f6fcbe39114b2291848285951c10e953cc Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sat, 22 Sep 2018 16:22:03 +0100
Subject: [PATCH 06/47] Add a contribution guide

---
 docs/contributing.rst | 113 ++++++++++++++++++++++++++++++++++++++++++
 docs/index.rst        |   1 +
 2 files changed, 114 insertions(+)
 create mode 100644 docs/contributing.rst

diff --git a/docs/contributing.rst b/docs/contributing.rst
new file mode 100644
index 000000000..4ee6d18d5
--- /dev/null
+++ b/docs/contributing.rst
@@ -0,0 +1,113 @@
+.. _contributing:
+
+Contributing to Paperless
+#########################
+
+Maybe you've been using Paperless for a while and want to add a feature or two,
+or maybe you've come across a bug that you have some ideas how to solve.  The
+beauty of Free software is that you can see what's wrong and help to get it
+fixed for everyone!
+
+
+How to Get Your Changes Rolled Into Paperless
+=============================================
+
+If you've found a bug, but don't know how to fix it, you can always post an
+issue on `GitHub`_ in the hopes that someone will have the time to fix it for
+you.  If however you're the one with the time, pull requests are always
+welcome, you just have to make sure that your code conforms to a few standards:
+
+Pep8
+----
+
+It's the standard for all Python development, so it's `very well documented`_.
+The version is:
+
+* Lines should wrap at 79 characters
+* Use ``snake_case`` for variables, ``CamelCase`` for classes, and ``ALL_CAPS``
+  for constants.
+* Space out your operators: ``stuff + 7`` instead of ``stuff+7``
+* Two empty lines between classes, and functions, but 1 empty line between
+  class methods.
+
+There's more to it than that, but if you follow those, you'll probably be
+alright.  When you submit your pull request, there's a pep8 checker that'll
+look at your code to see if anything is off.  If it finds anything, it'll
+complain at you until you fix it.
+
+
+Additional Style Guides
+-----------------------
+
+Where pep8 is ambiguous, I've tried to be a little more specific.  These rules
+aren't hard-and-fast, but if you can conform to them, I'll appreciate it and
+spend less time trying to conform your PR before merging:
+
+
+Function calls
+..............
+
+If you're calling a function and that necessitates more than one line of code,
+please format it like this:
+
+.. code:: python
+
+    my_function(
+        argument1,
+        kwarg1="x",
+        kwarg2="y"
+        another_really_long_kwarg="some big value"
+        a_kwarg_calling_another_long_function=another_function(
+            another_arg,
+            another_kwarg="kwarg!"
+        )
+    )
+
+This is all in the interest of code uniformity rather than anything else.  If
+we stick to a style, everything is understandable in the same way.
+
+
+Quoting Strings
+...............
+
+pep8 is a little too open-minded on this for my liking.  Python strings should
+be quoted with double quotes (``"``) except in cases where the resulting string
+would require too much escaping of a double quote, in which case, a single
+quoted, or triple-quoted string will do:
+
+.. code:: python
+
+    my_string = "This is my string"
+    problematic_string = 'This is a "string" with "quotes" in it'
+
+In HTML templates, please use double-quotes for tag attributes, and single
+quotes for arguments passed to Django tempalte tags:
+
+.. code:: html
+
+    <div class="stuff">
+        <a href="{% url 'some-url-name' pk='w00t' %}">link this</a>
+    </div>
+
+This is to keep linters happy they look at an HTML file and see an attribute
+closing the ``"`` before it should have been.
+
+--
+
+That's all there is in terms of guidelines, so I hope it's not too daunting.
+
+
+The Code of Conduct
+===================
+
+Paperless has a `code of conduct`_.  It's a lot like the other ones you see out
+there, with a few small changes, but basically it boils down to:
+
+> Don't be an ass, or you might get banned.
+
+I'm proud to say that the CoC has never had to be enforced because everyone has
+been awesome, friendly, and professional.
+
+.. _GitHub: https://github.com/danielquinn/paperless/issues
+.. _very well documented: https://www.python.org/dev/peps/pep-0008/
+.. _code of conduct: https://github.com/danielquinn/paperless/blob/master/CODE_OF_CONDUCT.md
diff --git a/docs/index.rst b/docs/index.rst
index 7710a330c..fd9d57d4e 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -43,5 +43,6 @@ Contents
    customising
    extending
    troubleshooting
+   contributing
    scanners
    changelog

From b420281be0c9db7f8f81a40eba9752b89a90c20f Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 23 Sep 2018 12:40:46 +0100
Subject: [PATCH 07/47] Remove numpy, scikit-learn, and scipy as they weren't
 being used

---
 requirements.txt | 25 +++++++++++--------------
 1 file changed, 11 insertions(+), 14 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 89c7e296f..b8c26a579 100755
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,10 +1,10 @@
 -i https://pypi.python.org/simple
-apipkg==1.5; python_version != '3.1.*'
-atomicwrites==1.2.1; python_version != '3.1.*'
+apipkg==1.5; python_version != '3.3.*'
+atomicwrites==1.2.1; python_version != '3.3.*'
 attrs==18.2.0
 certifi==2018.8.24
 chardet==3.0.4
-coverage==4.5.1; python_version != '3.1.*'
+coverage==4.5.1; python_version < '4'
 coveralls==1.5.0
 dateparser==0.7.0
 django-cors-headers==2.4.0
@@ -14,9 +14,9 @@ django-filter==2.0.0
 django==2.0.8
 djangorestframework==3.8.2
 docopt==0.6.2
-execnet==1.5.0; python_version != '3.1.*'
+execnet==1.5.0; python_version != '3.3.*'
 factory-boy==2.11.1
-faker==0.9.0
+faker==0.9.0; python_version >= '2.7'
 filemagic==1.6
 fuzzywuzzy==0.15.0
 gunicorn==19.9.0
@@ -24,20 +24,19 @@ idna==2.7
 inotify-simple==1.1.8
 langdetect==1.0.7
 more-itertools==4.3.0
-numpy==1.15.1
 pdftotext==2.1.0
 pillow==5.2.0
-pluggy==0.7.1; python_version != '3.1.*'
-py==1.6.0; python_version != '3.1.*'
+pluggy==0.7.1; python_version != '3.3.*'
+py==1.6.0; python_version != '3.3.*'
 pycodestyle==2.4.0
 pyocr==0.5.3
-pytest-cov==2.5.1
+pytest-cov==2.6.0
 pytest-django==3.4.2
 pytest-env==0.6.2
-pytest-forked==0.2
+pytest-forked==0.2; python_version != '3.3.*'
 pytest-sugar==0.9.1
 pytest-xdist==1.23.0
-pytest==3.7.4
+pytest==3.8.0
 python-dateutil==2.7.3
 python-dotenv==0.9.1
 python-gnupg==0.4.3
@@ -46,9 +45,7 @@ pytz==2018.5
 regex==2018.8.29
 requests==2.19.1
 six==1.11.0
-scikit-learn==0.19.2
-scipy==1.1.0
 termcolor==1.1.0
 text-unidecode==1.2
 tzlocal==1.5.1
-urllib3==1.23; python_version != '3.0.*'
+urllib3==1.23; python_version != '3.3.*'

From 117d7dad0494e33ee4e743f40c5684c7063594c3 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 23 Sep 2018 12:41:14 +0100
Subject: [PATCH 08/47] Improve the unknown language error message

---
 src/paperless_tesseract/parsers.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/paperless_tesseract/parsers.py b/src/paperless_tesseract/parsers.py
index e3c2ed361..f54461161 100644
--- a/src/paperless_tesseract/parsers.py
+++ b/src/paperless_tesseract/parsers.py
@@ -172,8 +172,8 @@ class RasterisedDocumentParser(DocumentParser):
                 raw_text = self._assemble_ocr_sections(imgs, middle, raw_text)
                 return raw_text
             raise OCRError(
-                "The guessed language is not available in this instance of "
-                "Tesseract."
+                "The guessed language ({}) is not available in this instance "
+                "of Tesseract.".format(guessed_language)
             )
 
     def _ocr(self, imgs, lang):

From 4130dd346565b1567ff5902c9340a33570707b66 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 23 Sep 2018 12:41:28 +0100
Subject: [PATCH 09/47] Conform code to standards

---
 src/documents/actions.py                      |  98 +++++++++----
 src/documents/admin.py                        | 135 ++++++++++++------
 .../documents/document/select_object.html     |  66 +++++----
 src/paperless/settings.py                     |   2 +-
 4 files changed, 198 insertions(+), 103 deletions(-)

diff --git a/src/documents/actions.py b/src/documents/actions.py
index 3db5cd314..cd2698a2c 100644
--- a/src/documents/actions.py
+++ b/src/documents/actions.py
@@ -4,10 +4,13 @@ from django.contrib.admin.utils import model_ngettext
 from django.core.exceptions import PermissionDenied
 from django.template.response import TemplateResponse
 
-from documents.models import Tag, Correspondent
+from documents.models import Correspondent, Tag
 
 
-def select_action(modeladmin, request, queryset, title, action, modelclass, success_message="", document_action=None, queryset_action=None):
+def select_action(
+        modeladmin, request, queryset, title, action, modelclass,
+        success_message="", document_action=None, queryset_action=None):
+
     opts = modeladmin.model._meta
     app_label = opts.app_label
 
@@ -27,7 +30,9 @@ def select_action(modeladmin, request, queryset, title, action, modelclass, succ
                 queryset_action(queryset, selected_object)
 
             modeladmin.message_user(request, success_message % {
-                "selected_object": selected_object.name, "count": n, "items": model_ngettext(modeladmin.opts, n)
+                "selected_object": selected_object.name,
+                "count": n,
+                "items": model_ngettext(modeladmin.opts, n)
             }, messages.SUCCESS)
 
         # Return None to display the change list page again.
@@ -47,10 +52,17 @@ def select_action(modeladmin, request, queryset, title, action, modelclass, succ
 
     request.current_app = modeladmin.admin_site.name
 
-    return TemplateResponse(request, "admin/%s/%s/select_object.html" % (app_label, opts.model_name), context)
+    return TemplateResponse(
+        request,
+        "admin/{}/{}/select_object.html".format(app_label, opts.model_name),
+        context
+    )
 
 
-def simple_action(modeladmin, request, queryset, success_message="", document_action=None, queryset_action=None):
+def simple_action(
+        modeladmin, request, queryset, success_message="",
+        document_action=None, queryset_action=None):
+
     if not modeladmin.has_change_permission(request):
         raise PermissionDenied
 
@@ -72,37 +84,63 @@ def simple_action(modeladmin, request, queryset, success_message="", document_ac
 
 
 def add_tag_to_selected(modeladmin, request, queryset):
-    return select_action(modeladmin=modeladmin, request=request, queryset=queryset,
-                         title="Add tag to multiple documents",
-                         action="add_tag_to_selected",
-                         modelclass=Tag,
-                         success_message="Successfully added tag %(selected_object)s to %(count)d %(items)s.",
-                         document_action=lambda doc, tag: doc.tags.add(tag))
-add_tag_to_selected.short_description = "Add tag to selected documents"
+    return select_action(
+        modeladmin=modeladmin,
+        request=request,
+        queryset=queryset,
+        title="Add tag to multiple documents",
+        action="add_tag_to_selected",
+        modelclass=Tag,
+        success_message="Successfully added tag %(selected_object)s to "
+                        "%(count)d %(items)s.",
+        document_action=lambda doc, tag: doc.tags.add(tag)
+    )
 
 
 def remove_tag_from_selected(modeladmin, request, queryset):
-    return select_action(modeladmin=modeladmin, request=request, queryset=queryset,
-                         title="Remove tag from multiple documents",
-                         action="remove_tag_from_selected",
-                         modelclass=Tag,
-                         success_message="Successfully removed tag %(selected_object)s from %(count)d %(items)s.",
-                         document_action=lambda doc, tag: doc.tags.remove(tag))
-remove_tag_from_selected.short_description = "Remove tag from selected documents"
+    return select_action(
+        modeladmin=modeladmin,
+        request=request,
+        queryset=queryset,
+        title="Remove tag from multiple documents",
+        action="remove_tag_from_selected",
+        modelclass=Tag,
+        success_message="Successfully removed tag %(selected_object)s from "
+                        "%(count)d %(items)s.",
+        document_action=lambda doc, tag: doc.tags.remove(tag)
+    )
 
 
 def set_correspondent_on_selected(modeladmin, request, queryset):
-    return select_action(modeladmin=modeladmin, request=request, queryset=queryset,
-                         title="Set correspondent on multiple documents",
-                         action="set_correspondent_on_selected",
-                         modelclass=Correspondent,
-                         success_message="Successfully set correspondent %(selected_object)s on %(count)d %(items)s.",
-                         queryset_action=lambda qs, correspondent: qs.update(correspondent=correspondent))
-set_correspondent_on_selected.short_description = "Set correspondent on selected documents"
+
+    return select_action(
+        modeladmin=modeladmin,
+        request=request,
+        queryset=queryset,
+        title="Set correspondent on multiple documents",
+        action="set_correspondent_on_selected",
+        modelclass=Correspondent,
+        success_message="Successfully set correspondent %(selected_object)s "
+                        "on %(count)d %(items)s.",
+        queryset_action=lambda qs, corr: qs.update(correspondent=corr)
+    )
 
 
 def remove_correspondent_from_selected(modeladmin, request, queryset):
-    return simple_action(modeladmin=modeladmin, request=request, queryset=queryset,
-                         success_message="Successfully removed correspondent from %(count)d %(items)s.",
-                         queryset_action=lambda qs: qs.update(correspondent=None))
-remove_correspondent_from_selected.short_description = "Remove correspondent from selected documents"
+    return simple_action(
+        modeladmin=modeladmin,
+        request=request,
+        queryset=queryset,
+        success_message="Successfully removed correspondent from %(count)d "
+                        "%(items)s.",
+        queryset_action=lambda qs: qs.update(correspondent=None)
+    )
+
+
+add_tag_to_selected.short_description = "Add tag to selected documents"
+remove_tag_from_selected.short_description = \
+    "Remove tag from selected documents"
+set_correspondent_on_selected.short_description = \
+    "Set correspondent on selected documents"
+remove_correspondent_from_selected.short_description = \
+    "Remove correspondent from selected documents"
diff --git a/src/documents/admin.py b/src/documents/admin.py
index d545c1c02..365a99c1a 100644
--- a/src/documents/admin.py
+++ b/src/documents/admin.py
@@ -3,21 +3,23 @@ from datetime import datetime, timedelta
 from django.conf import settings
 from django.contrib import admin, messages
 from django.contrib.admin.templatetags.admin_urls import add_preserved_filters
-from django.contrib.auth.models import User, Group
+from django.contrib.auth.models import Group, User
+from django.db import models
 from django.http import HttpResponseRedirect
-try:
-    from django.core.urlresolvers import reverse
-except ImportError:
-    from django.urls import reverse
 from django.templatetags.static import static
+from django.urls import reverse
 from django.utils.html import format_html, format_html_join
 from django.utils.http import urlquote
 from django.utils.safestring import mark_safe
-from django.db import models
 
-from documents.actions import add_tag_to_selected, remove_tag_from_selected, set_correspondent_on_selected, \
-    remove_correspondent_from_selected
-from .models import Correspondent, Tag, Document, Log
+from documents.actions import (
+    add_tag_to_selected,
+    remove_correspondent_from_selected,
+    remove_tag_from_selected,
+    set_correspondent_on_selected
+)
+
+from .models import Correspondent, Document, Log, Tag
 
 
 class FinancialYearFilter(admin.SimpleListFilter):
@@ -92,11 +94,18 @@ class RecentCorrespondentFilter(admin.RelatedFieldListFilter):
         self.title = "correspondent (recent)"
 
     def field_choices(self, field, request, model_admin):
+
+        years = settings.PAPERLESS_RECENT_CORRESPONDENT_YEARS
+        days = 365 * years
+
         lookups = []
-        if settings.PAPERLESS_RECENT_CORRESPONDENT_YEARS and settings.PAPERLESS_RECENT_CORRESPONDENT_YEARS > 0:
-            date_limit = datetime.now() - timedelta(days=365*settings.PAPERLESS_RECENT_CORRESPONDENT_YEARS)
-            for c in Correspondent.objects.filter(documents__created__gte=date_limit).distinct():
+        if years and years > 0:
+            correspondents = Correspondent.objects.filter(
+                documents__created__gte=datetime.now() - timedelta(days=days)
+            ).distinct()
+            for c in correspondents:
                 lookups.append((c.id, c.name))
+
         return lookups
 
 
@@ -106,13 +115,22 @@ class CommonAdmin(admin.ModelAdmin):
 
 class CorrespondentAdmin(CommonAdmin):
 
-    list_display = ("name", "match", "matching_algorithm", "document_count", "last_correspondence")
+    list_display = (
+        "name",
+        "match",
+        "matching_algorithm",
+        "document_count",
+        "last_correspondence"
+    )
     list_filter = ("matching_algorithm",)
     list_editable = ("match", "matching_algorithm")
 
     def get_queryset(self, request):
         qs = super(CorrespondentAdmin, self).get_queryset(request)
-        qs = qs.annotate(document_count=models.Count("documents"), last_correspondence=models.Max("documents__created"))
+        qs = qs.annotate(
+            document_count=models.Count("documents"),
+            last_correspondence=models.Max("documents__created")
+        )
         return qs
 
     def document_count(self, obj):
@@ -152,17 +170,29 @@ class DocumentAdmin(CommonAdmin):
     readonly_fields = ("added",)
     list_display = ("title", "created", "added", "thumbnail", "correspondent",
                     "tags_")
-    list_filter = ("tags", ('correspondent', RecentCorrespondentFilter), "correspondent", FinancialYearFilter)
+    list_filter = (
+        "tags",
+        ("correspondent", RecentCorrespondentFilter),
+        "correspondent",
+        FinancialYearFilter
+    )
 
     filter_horizontal = ("tags",)
 
     ordering = ["-created", "correspondent"]
 
-    actions = [add_tag_to_selected, remove_tag_from_selected, set_correspondent_on_selected, remove_correspondent_from_selected]
+    actions = [
+        add_tag_to_selected,
+        remove_tag_from_selected,
+        set_correspondent_on_selected,
+        remove_correspondent_from_selected
+    ]
 
-    date_hierarchy = 'created'
+    date_hierarchy = "created"
 
-    document_queue = None
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.document_queue = []
 
     def has_add_permission(self, request):
         return False
@@ -172,25 +202,38 @@ class DocumentAdmin(CommonAdmin):
     created_.short_description = "Created"
 
     def changelist_view(self, request, extra_context=None):
-        response = super().changelist_view(request, extra_context)
 
-        if request.method == 'GET':
+        response = super().changelist_view(
+            request,
+            extra_context=extra_context
+        )
+
+        if request.method == "GET":
             cl = self.get_changelist_instance(request)
             self.document_queue = [doc.id for doc in cl.queryset]
 
         return response
 
-    def change_view(self, request, object_id=None, form_url='', extra_context=None):
+    def change_view(self, request, object_id=None, form_url='',
+                    extra_context=None):
+
         extra_context = extra_context or {}
-        doc = Document.objects.get(id=object_id)
-        if self.document_queue and object_id and int(object_id) in self.document_queue:
-            # There is a queue of documents
-            current_index = self.document_queue.index(int(object_id))
-            if current_index < len(self.document_queue) - 1:
-                # ... and there are still documents in the queue
-                extra_context['next_object'] = self.document_queue[current_index + 1]
+
+        if self.document_queue and object_id:
+            if int(object_id) in self.document_queue:
+                # There is a queue of documents
+                current_index = self.document_queue.index(int(object_id))
+                if current_index < len(self.document_queue) - 1:
+                    # ... and there are still documents in the queue
+                    extra_context["next_object"] = self.document_queue[
+                        current_index + 1
+                    ]
+
         return super(DocumentAdmin, self).change_view(
-            request, object_id, form_url, extra_context=extra_context,
+            request,
+            object_id,
+            form_url,
+            extra_context=extra_context,
         )
 
     def response_change(self, request, obj):
@@ -200,25 +243,35 @@ class DocumentAdmin(CommonAdmin):
         preserved_filters = self.get_preserved_filters(request)
 
         msg_dict = {
-            'name': opts.verbose_name,
-            'obj': format_html('<a href="{}">{}</a>', urlquote(request.path), obj),
+            "name": opts.verbose_name,
+            "obj": format_html(
+                '<a href="{}">{}</a>',
+                urlquote(request.path),
+                obj
+            ),
         }
         if "_saveandeditnext" in request.POST:
             msg = format_html(
-                'The {name} "{obj}" was changed successfully. Editing next object.',
+                'The {name} "{obj}" was changed successfully. '
+                'Editing next object.',
                 **msg_dict
             )
             self.message_user(request, msg, messages.SUCCESS)
-            redirect_url = reverse('admin:%s_%s_change' %
-                                   (opts.app_label, opts.model_name),
-                                   args=(request.POST['_next_object'],),
-                                   current_app=self.admin_site.name)
-            redirect_url = add_preserved_filters({'preserved_filters': preserved_filters, 'opts': opts}, redirect_url)
-            response = HttpResponseRedirect(redirect_url)
-        else:
-            response = super().response_change(request, obj)
+            redirect_url = reverse(
+                "admin:{}_{}_change".format(opts.app_label, opts.model_name),
+                args=(request.POST["_next_object"],),
+                current_app=self.admin_site.name
+            )
+            redirect_url = add_preserved_filters(
+                {
+                    "preserved_filters": preserved_filters,
+                    "opts": opts
+                },
+                redirect_url
+            )
+            return HttpResponseRedirect(redirect_url)
 
-        return response
+        return super().response_change(request, obj)
 
     @mark_safe
     def thumbnail(self, obj):
diff --git a/src/documents/templates/admin/documents/document/select_object.html b/src/documents/templates/admin/documents/document/select_object.html
index 1439b5c21..775d57b12 100644
--- a/src/documents/templates/admin/documents/document/select_object.html
+++ b/src/documents/templates/admin/documents/document/select_object.html
@@ -1,46 +1,50 @@
 {% extends "admin/base_site.html" %}
+
+
 {% load i18n l10n admin_urls static %}
 {% load staticfiles %}
 
-{% block extrahead %}
-{{ block.super }}
-{{ media }}
-<script type="text/javascript" src="{% static 'admin/js/cancel.js' %}"></script>
 
+{% block extrahead %}
+	{{ block.super }}
+	{{ media }}
+	<script type="text/javascript" src="{% static 'admin/js/cancel.js' %}"></script>
 {% endblock %}
 
+
 {% block bodyclass %}{{ block.super }} app-{{ opts.app_label }} model-{{ opts.model_name }} delete-confirmation delete-selected-confirmation{% endblock %}
 
+
 {% block breadcrumbs %}
-<div class="breadcrumbs">
-    <a href="{% url 'admin:index' %}">{% trans 'Home' %}</a>
-    &rsaquo; <a href="{% url 'admin:app_list' app_label=opts.app_label %}">{{ opts.app_config.verbose_name }}</a>
-    &rsaquo; <a href="{% url opts|admin_urlname:'changelist' %}">{{ opts.verbose_name_plural|capfirst }}</a>
-    &rsaquo; {{title}}
-</div>
+	<div class="breadcrumbs">
+		<a href="{% url 'admin:index' %}">{% trans 'Home' %}</a>
+		&rsaquo; <a href="{% url 'admin:app_list' app_label=opts.app_label %}">{{ opts.app_config.verbose_name }}</a>
+		&rsaquo; <a href="{% url opts|admin_urlname:'changelist' %}">{{ opts.verbose_name_plural|capfirst }}</a>
+		&rsaquo; {{ title }}
+	</div>
 {% endblock %}
 
 {% block content %}
-<p>Please select the {{itemname}}.</p>
-<form method="post">{% csrf_token %}
-    <div>
-        {% for obj in queryset %}
-        <input type="hidden" name="{{ action_checkbox_name }}" value="{{ obj.pk|unlocalize }}"/>
-        {% endfor %}
-        <p>
-            <select name="obj_id">
-                {% for obj in objects %}
-                <option value="{{obj.id}}">{{obj.name}}</option>
-                {% endfor %}
-            </select>
-        </p>
+	<p>Please select the {{itemname}}.</p>
+	<form method="post">{% csrf_token %}
+		<div>
+			{% for obj in queryset %}
+			<input type="hidden" name="{{ action_checkbox_name }}" value="{{ obj.pk|unlocalize }}"/>
+			{% endfor %}
+			<p>
+				<select name="obj_id">
+					{% for obj in objects %}
+					<option value="{{ obj.id }}">{{ obj.name }}</option>
+					{% endfor %}
+				</select>
+			</p>
 
-        <input type="hidden" name="action" value="{{action}}"/>
-        <input type="hidden" name="post" value="yes"/>
-        <p>
-            <input type="submit" value="{% trans "Confirm" %}" />
-            <a href="#" class="button cancel-link">{% trans "Go back" %}</a>
-        </p>
-    </div>
-</form>
+			<input type="hidden" name="action" value="{{ action }}"/>
+			<input type="hidden" name="post" value="yes" />
+			<p>
+				<input type="submit" value="{% trans 'Confirm' %}" />
+				<a href="#" class="button cancel-link">{% trans "Go back" %}</a>
+			</p>
+		</div>
+	</form>
 {% endblock %}
diff --git a/src/paperless/settings.py b/src/paperless/settings.py
index e6f3da0cb..433eabe88 100644
--- a/src/paperless/settings.py
+++ b/src/paperless/settings.py
@@ -81,7 +81,7 @@ INSTALLED_APPS = [
 
     "rest_framework",
     "crispy_forms",
-    "django_filters"
+    "django_filters",
 
 ]
 

From 6db788a55034c450bb5340b8b152bf7f9e16a74a Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 23 Sep 2018 12:54:39 +0100
Subject: [PATCH 10/47] Add docs for indentation & spacing

---
 docs/contributing.rst | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/docs/contributing.rst b/docs/contributing.rst
index 4ee6d18d5..05f51731c 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -97,6 +97,34 @@ closing the ``"`` before it should have been.
 That's all there is in terms of guidelines, so I hope it's not too daunting.
 
 
+Indentation & Spacing
+.....................
+
+When it comes to indentation:
+
+* For Python, the rule is: follow pep8 and use 4 spaces.
+* For Javascript, CSS, and HTML, please use 1 tab.
+
+Additionally, Django templates making use of block elements like ``{% if %}``,
+``{% for %}``, and ``{% block %}`` etc. should be indented:
+
+Good:
+
+.. code:: html
+
+    {% block stuff %}
+    	<h1>This is the stuff</h1>
+    {% endblock %}
+
+Bad:
+
+.. code:: html
+
+    {% block stuff %}
+    <h1>This is the stuff</h1>
+    {% endblock %}
+
+
 The Code of Conduct
 ===================
 

From ff111f1bdeb687905c0b15ae26c68f706cbfb93a Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 23 Sep 2018 12:54:49 +0100
Subject: [PATCH 11/47] Update changelog for new stuff from #405

---
 docs/changelog.rst | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 9396493a7..6fdaee647 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -1,6 +1,14 @@
 Changelog
 #########
 
+2.4.0
+=====
+
+* A new set of actions are now available thanks to `jonaswinkler`_'s very first
+  pull request!  You can now do nifty things like tag documents in bulk, or set
+  correspondents in bulk.  `#405`_
+
+
 2.3.0
 =====
 
@@ -500,8 +508,9 @@ bulk of the work on this big change.
 .. _Kilian Koeltzsch: https://github.com/kiliankoe
 .. _Lukasz Soluch: https://github.com/LukaszSolo
 .. _Joshua Taillon: https://github.com/jat255
-.. _dubit0:  https://github.com/dubit0
-.. _ahyear:  https://github.com/ahyear
+.. _dubit0: https://github.com/dubit0
+.. _ahyear: https://github.com/ahyear
+.. _jonaswinkler: https://github.com/jonaswinkler
 
 .. _#20: https://github.com/danielquinn/paperless/issues/20
 .. _#44: https://github.com/danielquinn/paperless/issues/44
@@ -588,6 +597,7 @@ bulk of the work on this big change.
 .. _#399: https://github.com/danielquinn/paperless/pull/399
 .. _#400: https://github.com/danielquinn/paperless/pull/400
 .. _#401: https://github.com/danielquinn/paperless/pull/401
+.. _#405: https://github.com/danielquinn/paperless/pull/405
 
 .. _pipenv: https://docs.pipenv.org/
 .. _a new home on Docker Hub: https://hub.docker.com/r/danielquinn/paperless/

From 79e1e60238e86e772cdfad5f81acc9750b04a31c Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 23 Sep 2018 12:59:56 +0100
Subject: [PATCH 12/47] Fix typo

---
 docs/contributing.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/contributing.rst b/docs/contributing.rst
index 05f51731c..4678ff3aa 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -21,7 +21,7 @@ Pep8
 ----
 
 It's the standard for all Python development, so it's `very well documented`_.
-The version is:
+The short version is:
 
 * Lines should wrap at 79 characters
 * Use ``snake_case`` for variables, ``CamelCase`` for classes, and ``ALL_CAPS``

From 090565d84c6de4381c408e5a1853ed0288a1d42e Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 23 Sep 2018 13:58:40 +0100
Subject: [PATCH 13/47] Tweak the import/export system to handle encryption
 choices better

Now when you export a document, the `storage_type` value is always
`unencrypted` (since that's what it is when it's exported anyway), and
the flag is set by the importing script instead, based on the existence
of a `PAPERLESS_PASSPHRASE` environment variable, indicating that
encryption is enabled.
---
 .../management/commands/document_exporter.py       |  7 ++++++-
 .../management/commands/document_importer.py       | 14 +++++++++++++-
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/src/documents/management/commands/document_exporter.py b/src/documents/management/commands/document_exporter.py
index fce09092c..42a514348 100644
--- a/src/documents/management/commands/document_exporter.py
+++ b/src/documents/management/commands/document_exporter.py
@@ -55,7 +55,12 @@ class Command(Renderable, BaseCommand):
         documents = Document.objects.all()
         document_map = {d.pk: d for d in documents}
         manifest = json.loads(serializers.serialize("json", documents))
-        for document_dict in manifest:
+
+        for index, document_dict in enumerate(manifest):
+
+            # Force output to unencrypted as that will be the current state.
+            # The importer will make the decision to encrypt or not.
+            manifest[index]["fields"]["storage_type"] = Document.STORAGE_TYPE_UNENCRYPTED  # NOQA: E501
 
             document = document_map[document_dict["pk"]]
 
diff --git a/src/documents/management/commands/document_importer.py b/src/documents/management/commands/document_importer.py
index 15401722c..ae5c1853f 100644
--- a/src/documents/management/commands/document_importer.py
+++ b/src/documents/management/commands/document_importer.py
@@ -94,7 +94,7 @@ class Command(Renderable, BaseCommand):
             document_path = os.path.join(self.source, doc_file)
             thumbnail_path = os.path.join(self.source, thumb_file)
 
-            if document.storage_type == Document.STORAGE_TYPE_GPG:
+            if settings.PASSPHRASE:
 
                 with open(document_path, "rb") as unencrypted:
                     with open(document.source_path, "wb") as encrypted:
@@ -112,3 +112,15 @@ class Command(Renderable, BaseCommand):
 
                 shutil.copy(document_path, document.source_path)
                 shutil.copy(thumbnail_path, document.thumbnail_path)
+
+        # Reset the storage type to whatever we've used while importing
+
+        storage_type = Document.STORAGE_TYPE_UNENCRYPTED
+        if settings.PASSPHRASE:
+            storage_type = Document.STORAGE_TYPE_GPG
+
+        Document.objects.filter(
+            pk__in=[r["pk"] for r in self.manifest]
+        ).update(
+            storage_type=storage_type
+        )

From d17497fd5b703d0b21d5dd581a3fe9e1ead2a265 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 23 Sep 2018 14:00:27 +0100
Subject: [PATCH 14/47] Move the unique key on checksums to migration 15

This shouldn't affect anyone, since this migration is pretty old, but it
allows people using PostgreSQL to actually run Paperless.
---
 src/documents/migrations/0014_document_checksum.py        | 5 -----
 src/documents/migrations/0015_add_insensitive_to_match.py | 5 +++++
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/documents/migrations/0014_document_checksum.py b/src/documents/migrations/0014_document_checksum.py
index bc563cf86..a22348ba4 100644
--- a/src/documents/migrations/0014_document_checksum.py
+++ b/src/documents/migrations/0014_document_checksum.py
@@ -158,9 +158,4 @@ class Migration(migrations.Migration):
             name='modified',
             field=models.DateTimeField(auto_now=True, db_index=True),
         ),
-        migrations.AlterField(
-            model_name='document',
-            name='checksum',
-            field=models.CharField(editable=False, help_text='The checksum of the original document (before it was encrypted).  We use this to prevent duplicate document imports.', max_length=32, unique=True),
-        ),
     ]
diff --git a/src/documents/migrations/0015_add_insensitive_to_match.py b/src/documents/migrations/0015_add_insensitive_to_match.py
index 34a570c6e..30666dea9 100644
--- a/src/documents/migrations/0015_add_insensitive_to_match.py
+++ b/src/documents/migrations/0015_add_insensitive_to_match.py
@@ -12,6 +12,11 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
+        migrations.AlterField(
+            model_name='document',
+            name='checksum',
+            field=models.CharField(editable=False, help_text='The checksum of the original document (before it was encrypted).  We use this to prevent duplicate document imports.', max_length=32, unique=True),
+        ),
         migrations.AddField(
             model_name='correspondent',
             name='is_insensitive',

From b20d7eca03b44080530558cab0d406f0a0e3ae0d Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 23 Sep 2018 14:01:15 +0100
Subject: [PATCH 15/47] Tweak settings.py to allow for TRUST-based PostgreSQL
 auth

---
 src/paperless/settings.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/paperless/settings.py b/src/paperless/settings.py
index 433eabe88..4e788e56b 100644
--- a/src/paperless/settings.py
+++ b/src/paperless/settings.py
@@ -144,13 +144,14 @@ DATABASES = {
     }
 }
 
-if os.getenv("PAPERLESS_DBUSER") and os.getenv("PAPERLESS_DBPASS"):
+if os.getenv("PAPERLESS_DBUSER"):
     DATABASES["default"] = {
         "ENGINE": "django.db.backends.postgresql_psycopg2",
         "NAME": os.getenv("PAPERLESS_DBNAME", "paperless"),
         "USER": os.getenv("PAPERLESS_DBUSER"),
-        "PASSWORD": os.getenv("PAPERLESS_DBPASS")
     }
+    if os.getenv("PAPERLESS_DBPASS"):
+        DATABASES["default"]["PASSWORD"] = os.getenv("PAPERLESS_DBPASS")
 
 
 # Password validation

From acf6caca2f51e47baff40f38b9210c79a368b1e8 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 23 Sep 2018 14:01:35 +0100
Subject: [PATCH 16/47] Add a tox test for Python 3.7

---
 src/tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/tox.ini b/src/tox.ini
index 98e44e063..ff47136be 100644
--- a/src/tox.ini
+++ b/src/tox.ini
@@ -5,7 +5,7 @@
 
 [tox]
 skipsdist = True
-envlist = py34, py35, py36, pycodestyle, doc
+envlist = py34, py35, py36, py37, pycodestyle, doc
 
 [testenv]
 commands = pytest

From 8726b0316c7216b682b663016a1aac8f723190d5 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 23 Sep 2018 14:03:38 +0100
Subject: [PATCH 17/47] Add note about import/export process changes

---
 docs/changelog.rst | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 6fdaee647..fb728b8d7 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -7,6 +7,11 @@ Changelog
 * A new set of actions are now available thanks to `jonaswinkler`_'s very first
   pull request!  You can now do nifty things like tag documents in bulk, or set
   correspondents in bulk.  `#405`_
+* The import/export system is now a little smarter.  By default, documents are
+  tagged as ``unencrypted``, since exports are by their nature unencrypted.
+  It's now in the import step that we decide the storage type.  This allows you
+  to export from an encrypted system and import into an unencrypted one, or
+  vice-versa.
 
 
 2.3.0

From 35c5b8e263cd23b72e38725958c0f3cfe45471e5 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 23 Sep 2018 14:05:35 +0100
Subject: [PATCH 18/47] Add note about tweaks to psql connections

---
 docs/changelog.rst | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index fb728b8d7..9db59839e 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -12,7 +12,11 @@ Changelog
   It's now in the import step that we decide the storage type.  This allows you
   to export from an encrypted system and import into an unencrypted one, or
   vice-versa.
-
+* The migration history has been slightly modified to accomodate PostgreSQL
+  users.  Additionally, you can now tell paperless to use PostgreSQL simply by
+  declaring ``PAPERLESS_DBUSER`` in your environment.  This will attempt to
+  connect to your Postgres database without a password unless you also set
+  ``PAPERLESS_DBPASS``.
 
 2.3.0
 =====

From a511d34d694faf907aabdff67b019ae37231d47b Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 23 Sep 2018 15:38:31 +0100
Subject: [PATCH 19/47] Fix implementation of django-filter

---
 docs/changelog.rst       |  6 +++++
 src/documents/filters.py | 50 ++++++++++++++++++----------------------
 2 files changed, 29 insertions(+), 27 deletions(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 9db59839e..6ce2e49a4 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -17,6 +17,10 @@ Changelog
   declaring ``PAPERLESS_DBUSER`` in your environment.  This will attempt to
   connect to your Postgres database without a password unless you also set
   ``PAPERLESS_DBPASS``.
+* A bug was found in the REST API filter system that was the result of an
+  update of django-filter some time ago.  This has now been patched `#412`_.
+  Thanks to `thepill`_ for spotting it!
+
 
 2.3.0
 =====
@@ -520,6 +524,7 @@ bulk of the work on this big change.
 .. _dubit0: https://github.com/dubit0
 .. _ahyear: https://github.com/ahyear
 .. _jonaswinkler: https://github.com/jonaswinkler
+.. _thepill: https://github.com/thepill
 
 .. _#20: https://github.com/danielquinn/paperless/issues/20
 .. _#44: https://github.com/danielquinn/paperless/issues/44
@@ -607,6 +612,7 @@ bulk of the work on this big change.
 .. _#400: https://github.com/danielquinn/paperless/pull/400
 .. _#401: https://github.com/danielquinn/paperless/pull/401
 .. _#405: https://github.com/danielquinn/paperless/pull/405
+.. _#412: https://github.com/danielquinn/paperless/issues/412
 
 .. _pipenv: https://docs.pipenv.org/
 .. _a new home on Docker Hub: https://hub.docker.com/r/danielquinn/paperless/
diff --git a/src/documents/filters.py b/src/documents/filters.py
index 68861d967..d52889666 100644
--- a/src/documents/filters.py
+++ b/src/documents/filters.py
@@ -1,8 +1,14 @@
-from django_filters.rest_framework import CharFilter, FilterSet, BooleanFilter
+from django_filters.rest_framework import CharFilter, FilterSet, BooleanFilter, ModelChoiceFilter
 
 from .models import Correspondent, Document, Tag
 
 
+CHAR_KWARGS = (
+    "startswith", "endswith", "contains",
+    "istartswith", "iendswith", "icontains"
+)
+
+
 class CorrespondentFilterSet(FilterSet):
 
     class Meta:
@@ -31,34 +37,24 @@ class TagFilterSet(FilterSet):
 
 class DocumentFilterSet(FilterSet):
 
-    CHAR_KWARGS = {
-        "lookup_expr": (
-            "startswith",
-            "endswith",
-            "contains",
-            "istartswith",
-            "iendswith",
-            "icontains"
-        )
-    }
-
-    correspondent__name = CharFilter(
-        field_name="correspondent__name", **CHAR_KWARGS)
-    correspondent__slug = CharFilter(
-        field_name="correspondent__slug", **CHAR_KWARGS)
-    tags__name = CharFilter(
-        field_name="tags__name", **CHAR_KWARGS)
-    tags__slug = CharFilter(
-        field_name="tags__slug", **CHAR_KWARGS)
-    tags__empty = BooleanFilter(
-        field_name="tags", lookup_expr="isnull", distinct=True)
+    tags_empty = BooleanFilter(
+        label="Is tagged",
+        field_name="tags",
+        lookup_expr="isnull",
+        exclude=True
+    )
 
     class Meta:
         model = Document
         fields = {
-            "title": [
-                "startswith", "endswith", "contains",
-                "istartswith", "iendswith", "icontains"
-            ],
-            "content": ["contains", "icontains"],
+
+            "title": CHAR_KWARGS,
+            "content": ("contains", "icontains"),
+
+            "correspondent__name": CHAR_KWARGS,
+            "correspondent__slug": CHAR_KWARGS,
+
+            "tags__name": CHAR_KWARGS,
+            "tags__slug": CHAR_KWARGS,
+
         }

From 9dd76f1b878fff4c3b214f1c9224d3364d7e7e96 Mon Sep 17 00:00:00 2001
From: euri10 <benoit.barthelet@gmail.com>
Date: Mon, 24 Sep 2018 13:30:10 +0200
Subject: [PATCH 20/47] Fix issue where tesseract langages weren't installed
 properly

---
 Dockerfile | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index f02bf3336..968d67da5 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,4 @@
-FROM alpine:3.7
+FROM alpine:3.8
 
 LABEL maintainer="The Paperless Project https://github.com/danielquinn/paperless" \
       contributors="Guy Addadi <addadi@gmail.com>, Pit Kleyersburg <pitkley@googlemail.com>, \
@@ -12,11 +12,10 @@ COPY scripts/docker-entrypoint.sh /sbin/docker-entrypoint.sh
 ENV PAPERLESS_EXPORT_DIR=/export \
     PAPERLESS_CONSUMPTION_DIR=/consume
 
-# Install dependencies
-RUN apk --no-cache --update add \
-        python3 gnupg libmagic bash shadow curl \
+
+RUN apk update --no-cache && apk add python3 gnupg libmagic bash shadow curl \
         sudo poppler tesseract-ocr imagemagick ghostscript unpaper && \
-    apk --no-cache add --virtual .build-dependencies \
+    apk add --virtual .build-dependencies \
         python3-dev poppler-dev gcc g++ musl-dev zlib-dev jpeg-dev && \
 # Install python dependencies
     python3 -m ensurepip && \

From 8010d72f189a7a037040eef956e21ca9023bea94 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Mon, 1 Oct 2018 20:03:27 +0100
Subject: [PATCH 21/47] Tweak the date guesser to not allow dates prior to 1900
 (#414)

---
 src/paperless_tesseract/parsers.py         | 25 ++++++++++++++++------
 src/paperless_tesseract/tests/test_date.py | 13 +++++++++++
 2 files changed, 31 insertions(+), 7 deletions(-)

diff --git a/src/paperless_tesseract/parsers.py b/src/paperless_tesseract/parsers.py
index f54461161..5305ff053 100644
--- a/src/paperless_tesseract/parsers.py
+++ b/src/paperless_tesseract/parsers.py
@@ -203,6 +203,7 @@ class RasterisedDocumentParser(DocumentParser):
         return text
 
     def get_date(self):
+
         date = None
         datestring = None
 
@@ -217,20 +218,30 @@ class RasterisedDocumentParser(DocumentParser):
 
             try:
                 date = dateparser.parse(
-                           datestring,
-                           settings={'DATE_ORDER': self.DATE_ORDER,
-                                     'PREFER_DAY_OF_MONTH': 'first',
-                                     'RETURN_AS_TIMEZONE_AWARE': True})
+                    datestring,
+                    settings={
+                        "DATE_ORDER": self.DATE_ORDER,
+                        "PREFER_DAY_OF_MONTH": "first",
+                        "RETURN_AS_TIMEZONE_AWARE": True
+                    }
+                )
             except TypeError:
                 # Skip all matches that do not parse to a proper date
                 continue
 
-            if date is not None:
+            if date is not None and date.year > 1900:
                 break
+            else:
+                date = None
 
         if date is not None:
-            self.log("info", "Detected document date " + date.isoformat() +
-                             " based on string " + datestring)
+            self.log(
+                "info",
+                "Detected document date {} based on string {}".format(
+                    date.isoformat(),
+                    datestring
+                )
+            )
         else:
             self.log("info", "Unable to detect date for document")
 
diff --git a/src/paperless_tesseract/tests/test_date.py b/src/paperless_tesseract/tests/test_date.py
index 645cb70ff..e75042ce1 100644
--- a/src/paperless_tesseract/tests/test_date.py
+++ b/src/paperless_tesseract/tests/test_date.py
@@ -384,3 +384,16 @@ class TestDate(TestCase):
             document.get_date(),
             datetime.datetime(2017, 12, 31, 0, 0, tzinfo=tz.tzutc())
         )
+
+    @mock.patch(
+        "paperless_tesseract.parsers.RasterisedDocumentParser.get_text",
+        return_value="01-07-0590 00:00:00"
+    )
+    @mock.patch(
+        "paperless_tesseract.parsers.RasterisedDocumentParser.SCRATCH",
+        SCRATCH
+    )
+    def test_crazy_date(self, *args):
+        document = RasterisedDocumentParser("/dev/null")
+        document.get_text()
+        self.assertIsNone(document.get_date())

From e4044d0df9b9f1922cbae3f73ea073c1ea2eca49 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Mon, 1 Oct 2018 20:40:32 +0100
Subject: [PATCH 22/47] Update version number & changelog

---
 docs/changelog.rst       | 14 ++++++++++++++
 src/paperless/version.py |  2 +-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 6ce2e49a4..7daaa9d38 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -1,6 +1,15 @@
 Changelog
 #########
 
+2.4.1
+=====
+
+* An annoying bug in the date capture code was causing some bogus dates to be
+  attached to documents, which in turn busted the UI.  Thanks to `Andrew Peng`_
+  for reporting this. `#414`_.
+* A bug in the Dockerfile meant that Tesseract language files weren't being
+  installed correctly.  `euri10`_ was quick to provide a fix: `#406`_, `#413`_.
+
 2.4.0
 =====
 
@@ -525,6 +534,8 @@ bulk of the work on this big change.
 .. _ahyear: https://github.com/ahyear
 .. _jonaswinkler: https://github.com/jonaswinkler
 .. _thepill: https://github.com/thepill
+.. _Andrew Peng: https://github.com/pengc99
+.. _euri10: https://github.com/euri10
 
 .. _#20: https://github.com/danielquinn/paperless/issues/20
 .. _#44: https://github.com/danielquinn/paperless/issues/44
@@ -612,7 +623,10 @@ bulk of the work on this big change.
 .. _#400: https://github.com/danielquinn/paperless/pull/400
 .. _#401: https://github.com/danielquinn/paperless/pull/401
 .. _#405: https://github.com/danielquinn/paperless/pull/405
+.. _#406: https://github.com/danielquinn/paperless/issues/406
 .. _#412: https://github.com/danielquinn/paperless/issues/412
+.. _#413: https://github.com/danielquinn/paperless/pull/413
+.. _#414: https://github.com/danielquinn/paperless/issues/414
 
 .. _pipenv: https://docs.pipenv.org/
 .. _a new home on Docker Hub: https://hub.docker.com/r/danielquinn/paperless/
diff --git a/src/paperless/version.py b/src/paperless/version.py
index c1b36d9c1..31ee9974c 100644
--- a/src/paperless/version.py
+++ b/src/paperless/version.py
@@ -1 +1 @@
-__version__ = (2, 3, 0)
+__version__ = (2, 4, 1)

From b5176d207e2246831473607315176b35757c6754 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Mon, 1 Oct 2018 20:40:43 +0100
Subject: [PATCH 23/47] Hopefully fix Travis

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index e2c6acb51..7c0082a24 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -2,7 +2,7 @@ language: python
 
 before_install:
 - sudo apt-get update -qq
-- sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr tesseract-ocr-eng
+- sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr tesseract-ocr-eng tesseract-ocr-cat
 
 sudo: false
 

From 14bb52b6a45ca49d5c7ed918bbb215978e018367 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 7 Oct 2018 13:12:22 +0100
Subject: [PATCH 24/47] Wrap document consumption in a transaction #262

---
 src/documents/consumer.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/documents/consumer.py b/src/documents/consumer.py
index 28fc28f9e..7dd94ebf1 100644
--- a/src/documents/consumer.py
+++ b/src/documents/consumer.py
@@ -1,3 +1,4 @@
+from django.db import transaction
 import datetime
 import hashlib
 import logging
@@ -111,8 +112,11 @@ class Consumer:
                 if not self.try_consume_file(file):
                     self._ignore.append((file, mtime))
 
+    @transaction.atomic
     def try_consume_file(self, file):
-        "Return True if file was consumed"
+        """
+        Return True if file was consumed
+        """
 
         if not re.match(FileInfo.REGEXES["title"], file):
             return False

From 2a3f766b93d39ee43291c2fccf260ed39fd34437 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 7 Oct 2018 14:48:49 +0100
Subject: [PATCH 25/47] Consolidate get_date onto the DocumentParser parent
 class

---
 docs/changelog.rst                         |  2 +
 src/documents/parsers.py                   | 53 +++++++++++++++++++++-
 src/paperless_tesseract/parsers.py         | 49 +-------------------
 src/paperless_tesseract/tests/test_date.py | 28 +++++++++++-
 src/paperless_text/parsers.py              | 41 +----------------
 5 files changed, 83 insertions(+), 90 deletions(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 7daaa9d38..aefe65c25 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -9,6 +9,8 @@ Changelog
   for reporting this. `#414`_.
 * A bug in the Dockerfile meant that Tesseract language files weren't being
   installed correctly.  `euri10`_ was quick to provide a fix: `#406`_, `#413`_.
+* The ``get_date()`` functionality of the parsers has been consolidated onto
+  the ``DocumentParser`` class since much of that code was redundant anyway.
 
 2.4.0
 =====
diff --git a/src/documents/parsers.py b/src/documents/parsers.py
index 884f91ae4..29128eaad 100644
--- a/src/documents/parsers.py
+++ b/src/documents/parsers.py
@@ -1,9 +1,12 @@
 import logging
+import os
+import re
 import shutil
 import tempfile
-import re
 
+import dateparser
 from django.conf import settings
+from django.utils import timezone
 
 # This regular expression will try to find dates in the document at
 # hand and will match the following formats:
@@ -32,6 +35,7 @@ class DocumentParser:
     """
 
     SCRATCH = settings.SCRATCH_DIR
+    DATE_ORDER = settings.DATE_ORDER
 
     def __init__(self, path):
         self.document_path = path
@@ -55,7 +59,52 @@ class DocumentParser:
         """
         Returns the date of the document.
         """
-        raise NotImplementedError()
+
+        date = None
+        date_string = None
+
+        try:
+            text = self.get_text()
+        except ParseError:
+            return None
+
+        next_year = timezone.now().year + 5  # Arbitrary 5 year future limit
+
+        # Iterate through all regex matches and try to parse the date
+        for m in re.finditer(DATE_REGEX, text):
+
+            date_string = m.group(0)
+
+            try:
+                date = dateparser.parse(
+                    date_string,
+                    settings={
+                        "DATE_ORDER": self.DATE_ORDER,
+                        "PREFER_DAY_OF_MONTH": "first",
+                        "RETURN_AS_TIMEZONE_AWARE": True
+                    }
+                )
+            except TypeError:
+                # Skip all matches that do not parse to a proper date
+                continue
+
+            if date is not None and next_year > date.year > 1900:
+                break
+            else:
+                date = None
+
+        if date is not None:
+            self.log(
+                "info",
+                "Detected document date {} based on string {}".format(
+                    date.isoformat(),
+                    date_string
+                )
+            )
+        else:
+            self.log("info", "Unable to detect date for document")
+
+        return date
 
     def log(self, level, message):
         getattr(self.logger, level)(message, extra={
diff --git a/src/paperless_tesseract/parsers.py b/src/paperless_tesseract/parsers.py
index 5305ff053..8ba162b9f 100644
--- a/src/paperless_tesseract/parsers.py
+++ b/src/paperless_tesseract/parsers.py
@@ -4,7 +4,6 @@ import re
 import subprocess
 from multiprocessing.pool import Pool
 
-import dateparser
 import langdetect
 import pyocr
 from django.conf import settings
@@ -14,7 +13,7 @@ from pyocr.libtesseract.tesseract_raw import \
 from pyocr.tesseract import TesseractError
 
 import pdftotext
-from documents.parsers import DocumentParser, ParseError, DATE_REGEX
+from documents.parsers import DocumentParser, ParseError
 
 from .languages import ISO639
 
@@ -33,7 +32,6 @@ class RasterisedDocumentParser(DocumentParser):
     DENSITY = settings.CONVERT_DENSITY if settings.CONVERT_DENSITY else 300
     THREADS = int(settings.OCR_THREADS) if settings.OCR_THREADS else None
     UNPAPER = settings.UNPAPER_BINARY
-    DATE_ORDER = settings.DATE_ORDER
     DEFAULT_OCR_LANGUAGE = settings.OCR_LANGUAGE
     OCR_ALWAYS = settings.OCR_ALWAYS
 
@@ -202,51 +200,6 @@ class RasterisedDocumentParser(DocumentParser):
         text += self._ocr(imgs[middle + 1:], self.DEFAULT_OCR_LANGUAGE)
         return text
 
-    def get_date(self):
-
-        date = None
-        datestring = None
-
-        try:
-            text = self.get_text()
-        except ParseError as e:
-            return None
-
-        # Iterate through all regex matches and try to parse the date
-        for m in re.finditer(DATE_REGEX, text):
-            datestring = m.group(0)
-
-            try:
-                date = dateparser.parse(
-                    datestring,
-                    settings={
-                        "DATE_ORDER": self.DATE_ORDER,
-                        "PREFER_DAY_OF_MONTH": "first",
-                        "RETURN_AS_TIMEZONE_AWARE": True
-                    }
-                )
-            except TypeError:
-                # Skip all matches that do not parse to a proper date
-                continue
-
-            if date is not None and date.year > 1900:
-                break
-            else:
-                date = None
-
-        if date is not None:
-            self.log(
-                "info",
-                "Detected document date {} based on string {}".format(
-                    date.isoformat(),
-                    datestring
-                )
-            )
-        else:
-            self.log("info", "Unable to detect date for document")
-
-        return date
-
 
 def run_convert(*args):
 
diff --git a/src/paperless_tesseract/tests/test_date.py b/src/paperless_tesseract/tests/test_date.py
index e75042ce1..15fed1a37 100644
--- a/src/paperless_tesseract/tests/test_date.py
+++ b/src/paperless_tesseract/tests/test_date.py
@@ -393,7 +393,33 @@ class TestDate(TestCase):
         "paperless_tesseract.parsers.RasterisedDocumentParser.SCRATCH",
         SCRATCH
     )
-    def test_crazy_date(self, *args):
+    def test_crazy_date_past(self, *args):
+        document = RasterisedDocumentParser("/dev/null")
+        document.get_text()
+        self.assertIsNone(document.get_date())
+
+    @mock.patch(
+        "paperless_tesseract.parsers.RasterisedDocumentParser.get_text",
+        return_value="01-07-2350 00:00:00"
+    )
+    @mock.patch(
+        "paperless_tesseract.parsers.RasterisedDocumentParser.SCRATCH",
+        SCRATCH
+    )
+    def test_crazy_date_future(self, *args):
+        document = RasterisedDocumentParser("/dev/null")
+        document.get_text()
+        self.assertIsNone(document.get_date())
+
+    @mock.patch(
+        "paperless_tesseract.parsers.RasterisedDocumentParser.get_text",
+        return_value="01-07-0590 00:00:00"
+    )
+    @mock.patch(
+        "paperless_tesseract.parsers.RasterisedDocumentParser.SCRATCH",
+        SCRATCH
+    )
+    def test_crazy_date_past(self, *args):
         document = RasterisedDocumentParser("/dev/null")
         document.get_text()
         self.assertIsNone(document.get_date())
diff --git a/src/paperless_text/parsers.py b/src/paperless_text/parsers.py
index f02ba3ef8..afcfb013c 100644
--- a/src/paperless_text/parsers.py
+++ b/src/paperless_text/parsers.py
@@ -1,11 +1,9 @@
 import os
-import re
 import subprocess
 
-import dateparser
 from django.conf import settings
 
-from documents.parsers import DocumentParser, ParseError, DATE_REGEX
+from documents.parsers import DocumentParser, ParseError
 
 
 class TextDocumentParser(DocumentParser):
@@ -16,7 +14,6 @@ class TextDocumentParser(DocumentParser):
     CONVERT = settings.CONVERT_BINARY
     THREADS = int(settings.OCR_THREADS) if settings.OCR_THREADS else None
     UNPAPER = settings.UNPAPER_BINARY
-    DATE_ORDER = settings.DATE_ORDER
     DEFAULT_OCR_LANGUAGE = settings.OCR_LANGUAGE
     OCR_ALWAYS = settings.OCR_ALWAYS
 
@@ -26,7 +23,7 @@ class TextDocumentParser(DocumentParser):
 
     def get_thumbnail(self):
         """
-        The thumbnail of a txt is just a 500px wide image of the text
+        The thumbnail of a text file is just a 500px wide image of the text
         rendered onto a letter-sized page.
         """
         # The below is heavily cribbed from https://askubuntu.com/a/590951
@@ -84,40 +81,6 @@ class TextDocumentParser(DocumentParser):
 
         return self._text
 
-    def get_date(self):
-        date = None
-        datestring = None
-
-        try:
-            text = self.get_text()
-        except ParseError as e:
-            return None
-
-        # Iterate through all regex matches and try to parse the date
-        for m in re.finditer(DATE_REGEX, text):
-            datestring = m.group(0)
-
-            try:
-                date = dateparser.parse(
-                           datestring,
-                           settings={'DATE_ORDER': self.DATE_ORDER,
-                                     'PREFER_DAY_OF_MONTH': 'first',
-                                     'RETURN_AS_TIMEZONE_AWARE': True})
-            except TypeError:
-                # Skip all matches that do not parse to a proper date
-                continue
-
-            if date is not None:
-                break
-
-        if date is not None:
-            self.log("info", "Detected document date " + date.isoformat() +
-                             " based on string " + datestring)
-        else:
-            self.log("info", "Unable to detect date for document")
-
-        return date
-
 
 def run_command(*args):
     environment = os.environ.copy()

From 750ab5bf85300db7903bb24e93dbe55f60f02307 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 7 Oct 2018 14:56:38 +0100
Subject: [PATCH 26/47] Use optipng to optimise document thumbnails

---
 Dockerfile                         |  2 +-
 docs/changelog.rst                 |  8 +++++-
 paperless.conf.example             | 20 ++++++++++++++
 src/documents/consumer.py          |  2 +-
 src/documents/parsers.py           | 15 +++++++++++
 src/paperless/checks.py            |  7 ++++-
 src/paperless/settings.py          |  3 +++
 src/paperless_tesseract/parsers.py |  7 +++--
 src/paperless_text/parsers.py      | 43 +++++++++++++++++++-----------
 9 files changed, 85 insertions(+), 22 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 968d67da5..55d54cc01 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -14,7 +14,7 @@ ENV PAPERLESS_EXPORT_DIR=/export \
 
 
 RUN apk update --no-cache && apk add python3 gnupg libmagic bash shadow curl \
-        sudo poppler tesseract-ocr imagemagick ghostscript unpaper && \
+        sudo poppler tesseract-ocr imagemagick ghostscript unpaper optipng && \
     apk add --virtual .build-dependencies \
         python3-dev poppler-dev gcc g++ musl-dev zlib-dev jpeg-dev && \
 # Install python dependencies
diff --git a/docs/changelog.rst b/docs/changelog.rst
index aefe65c25..5e548301c 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -1,9 +1,14 @@
 Changelog
 #########
 
-2.4.1
+2.5.0
 =====
 
+* **New dependency**: Paperless now optimises thumbnail generation with
+  `optipng`_, so you'll need to install that somewhere in your PATH or declare
+  its location in ``PAPERLESS_OPTIPNG_BINARY``.  The Docker image has already
+  been updated on the Docker Hub, so you just need to pull the latest one from
+  there if you're a Docker user.
 * An annoying bug in the date capture code was causing some bogus dates to be
   attached to documents, which in turn busted the UI.  Thanks to `Andrew Peng`_
   for reporting this. `#414`_.
@@ -632,3 +637,4 @@ bulk of the work on this big change.
 
 .. _pipenv: https://docs.pipenv.org/
 .. _a new home on Docker Hub: https://hub.docker.com/r/danielquinn/paperless/
+.. _optipng: http://optipng.sourceforge.net/
diff --git a/paperless.conf.example b/paperless.conf.example
index 05cf81724..3604505cb 100644
--- a/paperless.conf.example
+++ b/paperless.conf.example
@@ -213,3 +213,23 @@ PAPERLESS_DEBUG="false"
 # The number of years for which a correspondent will be included in the recent
 # correspondents filter.
 #PAPERLESS_RECENT_CORRESPONDENT_YEARS=1
+
+###############################################################################
+####                     Third-Party Binaries                              ####
+###############################################################################
+
+# There are a few external software packages that Paperless expects to find on
+# your system when it starts up.  Unless you've done something creative with
+# their installation, you probably won't need to edit any of these.  However,
+# if you've installed these programs somewhere where simply typing the name of
+# the program doesn't automatically execute it (ie. the program isn't in your
+# $PATH), then you'll need to specify the literal path for that program here.
+
+# Convert (part of the ImageMagick suite)
+#PAPERLESS_CONVERT_BINARY=/usr/bin/convert
+
+# Unpaper
+#PAPERLESS_UNPAPER_BINARY=/usr/bin/unpaper
+
+# Optipng (for optimising thumbnail sizes)
+#PAPERLESS_OPTIPNG_BINARY=/usr/bin/optipng
diff --git a/src/documents/consumer.py b/src/documents/consumer.py
index 7dd94ebf1..3cb484b2a 100644
--- a/src/documents/consumer.py
+++ b/src/documents/consumer.py
@@ -149,7 +149,7 @@ class Consumer:
         parsed_document = parser_class(doc)
 
         try:
-            thumbnail = parsed_document.get_thumbnail()
+            thumbnail = parsed_document.get_optimised_thumbnail()
             date = parsed_document.get_date()
             document = self._store(
                 parsed_document.get_text(),
diff --git a/src/documents/parsers.py b/src/documents/parsers.py
index 29128eaad..1f60b1479 100644
--- a/src/documents/parsers.py
+++ b/src/documents/parsers.py
@@ -2,6 +2,7 @@ import logging
 import os
 import re
 import shutil
+import subprocess
 import tempfile
 
 import dateparser
@@ -36,6 +37,7 @@ class DocumentParser:
 
     SCRATCH = settings.SCRATCH_DIR
     DATE_ORDER = settings.DATE_ORDER
+    OPTIPNG = settings.OPTIPNG_BINARY
 
     def __init__(self, path):
         self.document_path = path
@@ -49,6 +51,19 @@ class DocumentParser:
         """
         raise NotImplementedError()
 
+    def optimise_thumbnail(self, in_path):
+
+        out_path = os.path.join(self.tempdir, "optipng.png")
+
+        args = (self.OPTIPNG, "-o5", in_path, "-out", out_path)
+        if not subprocess.Popen(args).wait() == 0:
+            raise ParseError("Optipng failed at {}".format(args))
+
+        return out_path
+
+    def get_optimised_thumbnail(self):
+        return self.optimise_thumbnail(self.get_thumbnail())
+
     def get_text(self):
         """
         Returns the text from the document and only the text.
diff --git a/src/paperless/checks.py b/src/paperless/checks.py
index 666425f9c..e8c94362a 100644
--- a/src/paperless/checks.py
+++ b/src/paperless/checks.py
@@ -76,7 +76,12 @@ def binaries_check(app_configs, **kwargs):
     error = "Paperless can't find {}. Without it, consumption is impossible."
     hint = "Either it's not in your ${PATH} or it's not installed."
 
-    binaries = (settings.CONVERT_BINARY, settings.UNPAPER_BINARY, "tesseract")
+    binaries = (
+        settings.CONVERT_BINARY,
+        settings.OPTIPNG_BINARY,
+        settings.UNPAPER_BINARY,
+        "tesseract"
+    )
 
     check_messages = []
     for binary in binaries:
diff --git a/src/paperless/settings.py b/src/paperless/settings.py
index 4e788e56b..fb5a399a8 100644
--- a/src/paperless/settings.py
+++ b/src/paperless/settings.py
@@ -247,6 +247,9 @@ CONVERT_TMPDIR = os.getenv("PAPERLESS_CONVERT_TMPDIR")
 CONVERT_MEMORY_LIMIT = os.getenv("PAPERLESS_CONVERT_MEMORY_LIMIT")
 CONVERT_DENSITY = os.getenv("PAPERLESS_CONVERT_DENSITY")
 
+# OptiPNG
+OPTIPNG_BINARY = os.getenv("PAPERLESS_OPTIPNG_BINARY", "optipng")
+
 # Unpaper
 UNPAPER_BINARY = os.getenv("PAPERLESS_UNPAPER_BINARY", "unpaper")
 
diff --git a/src/paperless_tesseract/parsers.py b/src/paperless_tesseract/parsers.py
index 8ba162b9f..dc5dbd637 100644
--- a/src/paperless_tesseract/parsers.py
+++ b/src/paperless_tesseract/parsers.py
@@ -44,15 +44,18 @@ class RasterisedDocumentParser(DocumentParser):
         The thumbnail of a PDF is just a 500px wide image of the first page.
         """
 
+        out_path = os.path.join(self.tempdir, "convert.png")
+
+        # Run convert to get a decent thumbnail
         run_convert(
             self.CONVERT,
             "-scale", "500x5000",
             "-alpha", "remove",
             "{}[0]".format(self.document_path),
-            os.path.join(self.tempdir, "convert.png")
+            out_path
         )
 
-        return os.path.join(self.tempdir, "convert.png")
+        return out_path
 
     def _is_ocred(self):
 
diff --git a/src/paperless_text/parsers.py b/src/paperless_text/parsers.py
index afcfb013c..3ccb78404 100644
--- a/src/paperless_text/parsers.py
+++ b/src/paperless_text/parsers.py
@@ -32,7 +32,7 @@ class TextDocumentParser(DocumentParser):
         text_color = "black"  # text color
         psize = [500, 647]  # icon size
         n_lines = 50  # number of lines to show
-        output_file = os.path.join(self.tempdir, "convert-txt.png")
+        out_path = os.path.join(self.tempdir, "convert.png")
 
         temp_bg = os.path.join(self.tempdir, "bg.png")
         temp_txlayer = os.path.join(self.tempdir, "tx.png")
@@ -43,9 +43,13 @@ class TextDocumentParser(DocumentParser):
             work_size = ",".join([str(n - 1) for n in psize])
             r = str(round(psize[0] / 10))
             rounded = ",".join([r, r])
-            run_command(self.CONVERT, "-size ", picsize, ' xc:none -draw ',
-                        '"fill ', bg_color, ' roundrectangle 0,0,',
-                        work_size, ",", rounded, '" ', temp_bg)
+            run_command(
+                self.CONVERT,
+                "-size ", picsize,
+                ' xc:none -draw ',
+                '"fill ', bg_color, ' roundrectangle 0,0,', work_size, ",", rounded, '" ',  # NOQA: E501
+                temp_bg
+            )
 
         def read_text():
             with open(self.document_path, 'r') as src:
@@ -54,22 +58,29 @@ class TextDocumentParser(DocumentParser):
                 return text.replace('"', "'")
 
         def create_txlayer():
-            run_command(self.CONVERT,
-                        "-background none",
-                        "-fill",
-                        text_color,
-                        "-pointsize", "12",
-                        "-border 4 -bordercolor none",
-                        "-size ", txsize,
-                        ' caption:"', read_text(), '" ',
-                        temp_txlayer)
+            run_command(
+                self.CONVERT,
+                "-background none",
+                "-fill",
+                text_color,
+                "-pointsize", "12",
+                "-border 4 -bordercolor none",
+                "-size ", txsize,
+                ' caption:"', read_text(), '" ',
+                temp_txlayer
+            )
 
         create_txlayer()
         create_bg()
-        run_command(self.CONVERT, temp_bg, temp_txlayer,
-                    "-background None -layers merge ", output_file)
+        run_command(
+            self.CONVERT,
+            temp_bg,
+            temp_txlayer,
+            "-background None -layers merge ",
+            out_path
+        )
 
-        return output_file
+        return out_path
 
     def get_text(self):
 

From 5d32e89c44ac2f1c4a2beb27a2772f871fda1387 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 7 Oct 2018 14:56:56 +0100
Subject: [PATCH 27/47] Wrap each document consumption in a transaction

---
 docs/changelog.rst | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 5e548301c..598241938 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -14,6 +14,8 @@ Changelog
   for reporting this. `#414`_.
 * A bug in the Dockerfile meant that Tesseract language files weren't being
   installed correctly.  `euri10`_ was quick to provide a fix: `#406`_, `#413`_.
+* Document consumption is now wrapped in a transaction as per an old ticket
+  `#262`_.
 * The ``get_date()`` functionality of the parsers has been consolidated onto
   the ``DocumentParser`` class since much of that code was redundant anyway.
 
@@ -608,6 +610,7 @@ bulk of the work on this big change.
 .. _#322: https://github.com/danielquinn/paperless/pull/322
 .. _#328: https://github.com/danielquinn/paperless/pull/328
 .. _#253: https://github.com/danielquinn/paperless/issues/253
+.. _#262: https://github.com/danielquinn/paperless/issues/262
 .. _#323: https://github.com/danielquinn/paperless/issues/323
 .. _#344: https://github.com/danielquinn/paperless/pull/344
 .. _#351: https://github.com/danielquinn/paperless/pull/351

From 65d6599964bb53c88a1f6332ae2faf96ddf45bd0 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 7 Oct 2018 16:22:52 +0100
Subject: [PATCH 28/47] Fix formatting

---
 src/reminders/models.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/reminders/models.py b/src/reminders/models.py
index 77d872afb..e8ac7020b 100644
--- a/src/reminders/models.py
+++ b/src/reminders/models.py
@@ -4,7 +4,6 @@ from django.db import models
 class Reminder(models.Model):
 
     document = models.ForeignKey(
-        "documents.Document", on_delete=models.PROTECT
-        )
+        "documents.Document", on_delete=models.PROTECT)
     date = models.DateTimeField()
     note = models.TextField(blank=True)

From c7f4bfe4f38d09c98dcc1372a229b73c252c7942 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 7 Oct 2018 16:23:03 +0100
Subject: [PATCH 29/47] Add migration that should have come in some time ago

---
 .../migrations/0002_auto_20181007_1420.py     | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 src/reminders/migrations/0002_auto_20181007_1420.py

diff --git a/src/reminders/migrations/0002_auto_20181007_1420.py b/src/reminders/migrations/0002_auto_20181007_1420.py
new file mode 100644
index 000000000..324764d2c
--- /dev/null
+++ b/src/reminders/migrations/0002_auto_20181007_1420.py
@@ -0,0 +1,19 @@
+# Generated by Django 2.0.8 on 2018-10-07 14:20
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('reminders', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='reminder',
+            name='document',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='documents.Document'),
+        ),
+    ]

From 5c3cb1e4ab2bb286659339c721323406c5f7cc04 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 7 Oct 2018 16:24:05 +0100
Subject: [PATCH 30/47] Rework how slugs are generated/referenced #393

---
 docs/changelog.rst                            | 20 +++++++
 src/documents/admin.py                        |  4 ++
 .../migrations/0022_auto_20181007_1420.py     | 52 +++++++++++++++++++
 src/documents/models.py                       |  9 ++--
 4 files changed, 80 insertions(+), 5 deletions(-)
 create mode 100644 src/documents/migrations/0022_auto_20181007_1420.py

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 598241938..b446e45cf 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -9,13 +9,32 @@ Changelog
   its location in ``PAPERLESS_OPTIPNG_BINARY``.  The Docker image has already
   been updated on the Docker Hub, so you just need to pull the latest one from
   there if you're a Docker user.
+
+* A problem in how we handle slug values on Tags and Correspondents required a
+  few changes to how we handle this field `#393`_:
+
+  1. Slugs are no longer editable.  They're derived from the name of the tag or
+     correspondent at save time, so if you wanna change the slug, you have to
+     change the name, and even then you're restricted to the rules of the
+     ``slugify()`` function.  The slug value is still visible in the admin
+     though.
+  2. I've added a migration to go over all existing tags & correspondents and
+     rewrite the ``.slug`` values to ones conforming to the ``slugify()``
+     rules.
+  3. The consumption process now uses the same rules as ``.save()`` in
+     determining a slug and using that to check for an existing
+     tag/correspondent.
+
 * An annoying bug in the date capture code was causing some bogus dates to be
   attached to documents, which in turn busted the UI.  Thanks to `Andrew Peng`_
   for reporting this. `#414`_.
+
 * A bug in the Dockerfile meant that Tesseract language files weren't being
   installed correctly.  `euri10`_ was quick to provide a fix: `#406`_, `#413`_.
+
 * Document consumption is now wrapped in a transaction as per an old ticket
   `#262`_.
+
 * The ``get_date()`` functionality of the parsers has been consolidated onto
   the ``DocumentParser`` class since much of that code was redundant anyway.
 
@@ -627,6 +646,7 @@ bulk of the work on this big change.
 .. _#391: https://github.com/danielquinn/paperless/pull/391
 .. _#390: https://github.com/danielquinn/paperless/pull/390
 .. _#392: https://github.com/danielquinn/paperless/issues/392
+.. _#393: https://github.com/danielquinn/paperless/issues/393
 .. _#395: https://github.com/danielquinn/paperless/pull/395
 .. _#396: https://github.com/danielquinn/paperless/pull/396
 .. _#399: https://github.com/danielquinn/paperless/pull/399
diff --git a/src/documents/admin.py b/src/documents/admin.py
index 365a99c1a..6dbe7f835 100644
--- a/src/documents/admin.py
+++ b/src/documents/admin.py
@@ -125,6 +125,8 @@ class CorrespondentAdmin(CommonAdmin):
     list_filter = ("matching_algorithm",)
     list_editable = ("match", "matching_algorithm")
 
+    readonly_fields = ("slug",)
+
     def get_queryset(self, request):
         qs = super(CorrespondentAdmin, self).get_queryset(request)
         qs = qs.annotate(
@@ -149,6 +151,8 @@ class TagAdmin(CommonAdmin):
     list_filter = ("colour", "matching_algorithm")
     list_editable = ("colour", "match", "matching_algorithm")
 
+    readonly_fields = ("slug",)
+
     def get_queryset(self, request):
         qs = super(TagAdmin, self).get_queryset(request)
         qs = qs.annotate(document_count=models.Count("documents"))
diff --git a/src/documents/migrations/0022_auto_20181007_1420.py b/src/documents/migrations/0022_auto_20181007_1420.py
new file mode 100644
index 000000000..937695bc8
--- /dev/null
+++ b/src/documents/migrations/0022_auto_20181007_1420.py
@@ -0,0 +1,52 @@
+# Generated by Django 2.0.8 on 2018-10-07 14:20
+
+from django.db import migrations, models
+from django.utils.text import slugify
+
+
+def re_slug_all_the_things(apps, schema_editor):
+    """
+    Rewrite all slug values to make sure they're actually slugs before we brand
+    them as uneditable.
+    """
+
+    Tag = apps.get_model("documents", "Tag")
+    Correspondent = apps.get_model("documents", "Tag")
+
+    for klass in (Tag, Correspondent):
+        for instance in klass.objects.all():
+            klass.objects.filter(
+                pk=instance.pk
+            ).update(
+                slug=slugify(instance.slug)
+            )
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('documents', '0021_document_storage_type'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='tag',
+            options={'ordering': ('name',)},
+        ),
+        migrations.AlterField(
+            model_name='correspondent',
+            name='slug',
+            field=models.SlugField(blank=True, editable=False),
+        ),
+        migrations.AlterField(
+            model_name='document',
+            name='file_type',
+            field=models.CharField(choices=[('pdf', 'PDF'), ('png', 'PNG'), ('jpg', 'JPG'), ('gif', 'GIF'), ('tiff', 'TIFF'), ('txt', 'TXT'), ('csv', 'CSV'), ('md', 'MD')], editable=False, max_length=4),
+        ),
+        migrations.AlterField(
+            model_name='tag',
+            name='slug',
+            field=models.SlugField(blank=True, editable=False),
+        ),
+        migrations.RunPython(re_slug_all_the_things, migrations.RunPython.noop)
+    ]
diff --git a/src/documents/models.py b/src/documents/models.py
index c66bb5b0f..37c1cfdbf 100644
--- a/src/documents/models.py
+++ b/src/documents/models.py
@@ -11,6 +11,7 @@ from django.conf import settings
 from django.db import models
 from django.template.defaultfilters import slugify
 from django.utils import timezone
+from django.utils.text import slugify
 from fuzzywuzzy import fuzz
 
 from .managers import LogManager
@@ -37,7 +38,7 @@ class MatchingModel(models.Model):
     )
 
     name = models.CharField(max_length=128, unique=True)
-    slug = models.SlugField(blank=True)
+    slug = models.SlugField(blank=True, editable=False)
 
     match = models.CharField(max_length=256, blank=True)
     matching_algorithm = models.PositiveIntegerField(
@@ -147,9 +148,7 @@ class MatchingModel(models.Model):
     def save(self, *args, **kwargs):
 
         self.match = self.match.lower()
-
-        if not self.slug:
-            self.slug = slugify(self.name)
+        self.slug = slugify(self.name)
 
         models.Model.save(self, *args, **kwargs)
 
@@ -452,7 +451,7 @@ class FileInfo:
         r = []
         for t in tags.split(","):
             r.append(Tag.objects.get_or_create(
-                slug=t.lower(),
+                slug=slugify(t),
                 defaults={"name": t}
             )[0])
         return tuple(r)

From 3f572afb8b611bc4c7f6fafaf52bad5df3d47611 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 7 Oct 2018 16:26:05 +0100
Subject: [PATCH 31/47] Add a little more read-only info for documents

---
 src/documents/admin.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/documents/admin.py b/src/documents/admin.py
index 6dbe7f835..5a5a1a50c 100644
--- a/src/documents/admin.py
+++ b/src/documents/admin.py
@@ -171,7 +171,7 @@ class DocumentAdmin(CommonAdmin):
         }
 
     search_fields = ("correspondent__name", "title", "content", "tags__name")
-    readonly_fields = ("added",)
+    readonly_fields = ("added", "file_type", "storage_type",)
     list_display = ("title", "created", "added", "thumbnail", "correspondent",
                     "tags_")
     list_filter = (

From ce5e8b2658cd0ff2836e609dba861b0f699ae58a Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 7 Oct 2018 16:26:23 +0100
Subject: [PATCH 32/47] Rework user hack for "login-free" sessions #394

---
 docs/changelog.rst      |  7 +++++++
 src/paperless/models.py | 19 ++++++++++++-------
 2 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index b446e45cf..db5fb6a15 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -10,6 +10,12 @@ Changelog
   been updated on the Docker Hub, so you just need to pull the latest one from
   there if you're a Docker user.
 
+* "Login free" instances of Paperless were breaking whenever you tried to edit
+  objects in the admin: adding/deleting tags or correspondents, or even fixing
+  spelling.  This was due to the "user hack" we were applying to sessions that
+  weren't using a login, as that hack user didn't have a valid id.  The fix was
+  to attribute the first user id in the system to this hack user.  `#394`_
+
 * A problem in how we handle slug values on Tags and Correspondents required a
   few changes to how we handle this field `#393`_:
 
@@ -648,6 +654,7 @@ bulk of the work on this big change.
 .. _#392: https://github.com/danielquinn/paperless/issues/392
 .. _#393: https://github.com/danielquinn/paperless/issues/393
 .. _#395: https://github.com/danielquinn/paperless/pull/395
+.. _#394: https://github.com/danielquinn/paperless/issues/394
 .. _#396: https://github.com/danielquinn/paperless/pull/396
 .. _#399: https://github.com/danielquinn/paperless/pull/399
 .. _#400: https://github.com/danielquinn/paperless/pull/400
diff --git a/src/paperless/models.py b/src/paperless/models.py
index 4001d3468..e390032db 100644
--- a/src/paperless/models.py
+++ b/src/paperless/models.py
@@ -1,15 +1,20 @@
+from django.contrib.auth.models import User as DjangoUser
+
+
 class User:
     """
-      This is a dummy django User used with our middleware to disable
-      login authentication if that is configured in paperless.conf
+    This is a dummy django User used with our middleware to disable
+    login authentication if that is configured in paperless.conf
     """
+
     is_superuser = True
     is_active = True
     is_staff = True
     is_authenticated = True
 
-    # Must be -1 to avoid colliding with real user ID's (which start at 1)
-    id = -1
+    @property
+    def id(self):
+        return DjangoUser.objects.order_by("pk").first().pk
 
     @property
     def pk(self):
@@ -17,9 +22,9 @@ class User:
 
 
 """
-  NOTE: These are here as a hack instead of being in the User definition
-  above due to the way pycodestyle handles lamdbdas.
-  See https://github.com/PyCQA/pycodestyle/issues/379 for more.
+NOTE: These are here as a hack instead of being in the User definition
+NOTE: above due to the way pycodestyle handles lamdbdas.
+NOTE: See https://github.com/PyCQA/pycodestyle/issues/379 for more.
 """
 
 User.has_module_perms = lambda *_: True

From 0bb7d27269d10db88fba64cf5263d2d9697b2123 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 7 Oct 2018 16:28:53 +0100
Subject: [PATCH 33/47] pep8

---
 src/documents/filters.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/documents/filters.py b/src/documents/filters.py
index d52889666..2b7c9cc9f 100644
--- a/src/documents/filters.py
+++ b/src/documents/filters.py
@@ -1,4 +1,4 @@
-from django_filters.rest_framework import CharFilter, FilterSet, BooleanFilter, ModelChoiceFilter
+from django_filters.rest_framework import BooleanFilter, FilterSet
 
 from .models import Correspondent, Document, Tag
 

From 2ef2bf873e7a6d4b833ef7044092d42cf970b56c Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sun, 7 Oct 2018 16:30:36 +0100
Subject: [PATCH 34/47] Version bump: 2.5.0

---
 src/paperless/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/paperless/version.py b/src/paperless/version.py
index 31ee9974c..eb0649352 100644
--- a/src/paperless/version.py
+++ b/src/paperless/version.py
@@ -1 +1 @@
-__version__ = (2, 4, 1)
+__version__ = (2, 5, 0)

From f948ee11bed7c6725c390e1f1bdfe283ee98938f Mon Sep 17 00:00:00 2001
From: David Martin <david_martin@fastmail.com>
Date: Mon, 8 Oct 2018 19:12:11 +1100
Subject: [PATCH 35/47] Let unpaper overwrite temporary files.

I'm not sure what the circumstances are, but it looks like unpaper can
attempt to write a temporary file that already exists [0]. This then
fails the consumption. As per daedadu's comment simply letting unpaper
overwrite files fixes this.

[0]
unpaper: error: output file '/tmp/paperless/paperless-pjkrcr4l/convert-0000.unpaper.pnm' already present.
See https://web.archive.org/web/20181008081515/https://github.com/danielquinn/paperless/issues/406#issue-360651630
---
 src/paperless_tesseract/parsers.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/paperless_tesseract/parsers.py b/src/paperless_tesseract/parsers.py
index dc5dbd637..0139738be 100644
--- a/src/paperless_tesseract/parsers.py
+++ b/src/paperless_tesseract/parsers.py
@@ -218,7 +218,8 @@ def run_convert(*args):
 
 def run_unpaper(args):
     unpaper, pnm = args
-    command_args = unpaper, pnm, pnm.replace(".pnm", ".unpaper.pnm")
+    command_args = (unpaper, "--overwrite", pnm,
+                    pnm.replace(".pnm", ".unpaper.pnm"))
     if not subprocess.Popen(command_args).wait() == 0:
         raise ParseError("Unpaper failed at {}".format(command_args))
 

From b350ec48b7f5ef9d1c250961e362aef0a5a4de5c Mon Sep 17 00:00:00 2001
From: David Martin <david_martin@fastmail.com>
Date: Mon, 8 Oct 2018 19:37:05 +1100
Subject: [PATCH 36/47] Mention FORGIVING_OCR config option when language
 detection fails.

It is not obvious that the PAPERLESS_FORGIVING_OCR allows to let
document consumption happen even if no language can be detected.
Mentioning it in the actual error message in the log seems like the best
way to make it clear.
---
 src/paperless_tesseract/parsers.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/paperless_tesseract/parsers.py b/src/paperless_tesseract/parsers.py
index dc5dbd637..ffa2727e5 100644
--- a/src/paperless_tesseract/parsers.py
+++ b/src/paperless_tesseract/parsers.py
@@ -153,7 +153,10 @@ class RasterisedDocumentParser(DocumentParser):
                 )
                 raw_text = self._assemble_ocr_sections(imgs, middle, raw_text)
                 return raw_text
-            raise OCRError("Language detection failed")
+            error_msg = ("Language detection failed. Set "
+                         "PAPERLESS_FORGIVING_OCR in config file to continue "
+                         "anyway.")
+            raise OCRError(error_msg)
 
         if ISO639[guessed_language] == self.DEFAULT_OCR_LANGUAGE:
             raw_text = self._assemble_ocr_sections(imgs, middle, raw_text)

From 818780a1918543100bf4b25c3d558fb6f28e8956 Mon Sep 17 00:00:00 2001
From: David Martin <david_martin@fastmail.com>
Date: Mon, 8 Oct 2018 19:38:38 +1100
Subject: [PATCH 37/47] Add PAPERLESS_FORGIVING_OCR option to example config.

It helps having it in the example config as that makes it more clear
that it exists.
---
 paperless.conf.example | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/paperless.conf.example b/paperless.conf.example
index 3604505cb..11e6d905b 100644
--- a/paperless.conf.example
+++ b/paperless.conf.example
@@ -188,6 +188,11 @@ PAPERLESS_DEBUG="false"
 #PAPERLESS_CONSUMER_LOOP_TIME=10
 
 
+# By default Paperless stops consuming a document if no language can be detected.
+# Set to true to consume documents even if the language detection fails.
+#PAPERLESS_FORGIVING_OCR="false"
+
+
 ###############################################################################
 ####                            Interface                                  ####
 ###############################################################################

From f8cfbb44d275f3500245e238d163126fc7c0fcdc Mon Sep 17 00:00:00 2001
From: Erik Arvstedt <erik.arvstedt@gmail.com>
Date: Mon, 8 Oct 2018 10:58:06 +0200
Subject: [PATCH 38/47] requirements.txt: bring back Linux-only restriction for
 inotify-simple

Fixes #418
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index b8c26a579..ac893b8a5 100755
--- a/requirements.txt
+++ b/requirements.txt
@@ -21,7 +21,7 @@ filemagic==1.6
 fuzzywuzzy==0.15.0
 gunicorn==19.9.0
 idna==2.7
-inotify-simple==1.1.8
+inotify-simple==1.1.8; sys_platform == 'linux'
 langdetect==1.0.7
 more-itertools==4.3.0
 pdftotext==2.1.0

From ec1d5c80ffa1bd005da18b9442683e8f0a0d0129 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Mon, 8 Oct 2018 10:38:53 +0100
Subject: [PATCH 39/47] Add pip install to update process

---
 docs/migrating.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/migrating.rst b/docs/migrating.rst
index 45646f058..d4ef40857 100644
--- a/docs/migrating.rst
+++ b/docs/migrating.rst
@@ -82,6 +82,7 @@ rolled in as part of the update:
 
     $ cd /path/to/project
     $ git pull
+    $ pip install -r requirements.txt
     $ cd src
     $ ./manage.py migrate
 

From 0b377a76d01376e3303c97d7c6a51613ca0f48e8 Mon Sep 17 00:00:00 2001
From: Sharif Nassar <mrwacky42@users.noreply.github.com>
Date: Sat, 13 Oct 2018 11:31:53 -0700
Subject: [PATCH 40/47] Remove Vagrant docs

* Vagrant does not seem to have any libvirt boxes for Ubuntu any more.
* Vagrant 2 was released a year ago, but vagrant-libvirt only claims
  to support up to Vagrant 1.8.
---
 .gitignore                |  1 -
 Vagrantfile               | 20 -------------
 docs/requirements.rst     |  2 +-
 docs/setup.rst            | 61 +--------------------------------------
 docs/troubleshooting.rst  |  5 ++--
 scripts/vagrant-provision | 31 --------------------
 6 files changed, 4 insertions(+), 116 deletions(-)
 delete mode 100644 Vagrantfile
 delete mode 100644 scripts/vagrant-provision

diff --git a/.gitignore b/.gitignore
index 439d9df4b..cf485e15f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -73,7 +73,6 @@ db.sqlite3
 # Other stuff that doesn't belong
 .virtualenv
 virtualenv
-.vagrant
 docker-compose.yml
 docker-compose.env
 
diff --git a/Vagrantfile b/Vagrantfile
deleted file mode 100644
index 46caa38bc..000000000
--- a/Vagrantfile
+++ /dev/null
@@ -1,20 +0,0 @@
-# -*- mode: ruby -*-
-# vi: set ft=ruby :
-
-VAGRANT_API_VERSION = "2"
-Vagrant.configure(VAGRANT_API_VERSION) do |config|
-  config.vm.box = "ubuntu/trusty64"
-
-  # Provision using shell
-  config.vm.host_name = "dev.paperless"
-  config.vm.synced_folder ".", "/opt/paperless"
-  config.vm.provision "shell", path: "scripts/vagrant-provision"
-
-  # Networking details
-  config.vm.network "private_network", ip: "172.28.128.4"
-
-  config.vm.provider "virtualbox" do |vb|
-    # Customize the amount of memory on the VM:
-    vb.memory = "1024"
-  end
-end
diff --git a/docs/requirements.rst b/docs/requirements.rst
index ee42cb96a..b6cbad213 100644
--- a/docs/requirements.rst
+++ b/docs/requirements.rst
@@ -33,7 +33,7 @@ In addition to the above, there are a number of Python requirements, all of
 which are listed in a file called ``requirements.txt`` in the project root
 directory.
 
-If you're not working on a virtual environment (like Vagrant or Docker), you
+If you're not working on a virtual environment (like Docker), you
 should probably be using a virtualenv, but that's your call.  The reasons why
 you might choose a virtualenv or not aren't really within the scope of this
 document.  Needless to say if you don't know what a virtualenv is, you should
diff --git a/docs/setup.rst b/docs/setup.rst
index 2dcfeb901..d794c663b 100644
--- a/docs/setup.rst
+++ b/docs/setup.rst
@@ -42,18 +42,14 @@ Installation & Configuration
 You can go multiple routes with setting up and running Paperless:
 
  * The `bare metal route`_
- * The `vagrant route`_
  * The `docker route`_
 
 
-The `Vagrant route`_ is quick & easy, but means you're running a VM which comes
-with memory consumption, cpu overhead etc. The `docker route`_ offers the same
-simplicity as Vagrant with lower resource consumption.
+The `docker route`_ is quick & easy.
 
 The `bare metal route`_ is a bit more complicated to setup but makes it easier
 should you want to contribute some code back.
 
-.. _Vagrant route: setup-installation-vagrant_
 .. _docker route: setup-installation-docker_
 .. _bare metal route: setup-installation-bare-metal_
 .. _Docker Machine: https://docs.docker.com/machine/
@@ -267,54 +263,6 @@ Docker Method
    newer ``docker-compose.yml.example`` file
 
 
-.. _setup-installation-vagrant:
-
-Vagrant Method
-++++++++++++++
-
-1. Install `Vagrant`_.  How you do that is really between you and your OS.
-2. Run ``vagrant up``.  An instance will start up for you.  When it's ready and
-   provisioned...
-3. Run ``vagrant ssh`` and once inside your new vagrant box, edit
-   ``/etc/paperless.conf`` and set the values for:
-
-    * ``PAPERLESS_CONSUMPTION_DIR``: This is where your documents will be
-      dumped to be consumed by Paperless.
-    * ``PAPERLESS_PASSPHRASE``: This is the passphrase Paperless uses to
-      encrypt/decrypt the original document.  It's only required if you want
-      your original files to be encrypted, otherwise, just leave it unset.
-    * ``PAPERLESS_EMAIL_SECRET``: this is the "magic word" used when consuming
-      documents from mail or via the API.  If you don't use either, leaving it
-      blank is just fine.
-
-4. Exit the vagrant box and re-enter it with ``vagrant ssh`` again.  This
-   updates the environment to make use of the changes you made to the config
-   file.
-5. Initialise the database with ``/opt/paperless/src/manage.py migrate``.
-6. Still inside your vagrant box, create a user for your Paperless instance
-   with ``/opt/paperless/src/manage.py createsuperuser``. Follow the prompts to
-   create your user.
-7. Start the webserver with
-   ``/opt/paperless/src/manage.py runserver 0.0.0.0:8000``. You should now be
-   able to visit your (empty) `Paperless webserver`_ at ``172.28.128.4:8000``.
-   You can login with the user/pass you created in #6.
-8. In a separate window, run ``vagrant ssh`` again, but this time once inside
-   your vagrant instance, you should start the consumer script with
-   ``/opt/paperless/src/manage.py document_consumer``.
-9. Scan something.  Put it in the ``CONSUMPTION_DIR``.
-10. Wait a few minutes
-11. Visit the document list on your webserver, and it should be there, indexed
-    and downloadable.
-
-.. caution::
-
-    This installation is not secure. Once everything is working head up to
-    `Making things more permanent`_
-
-.. _Vagrant: https://vagrantup.com/
-.. _Paperless server: http://172.28.128.4:8000
-
-
 .. _setup-permanent:
 
 Making Things a Little more Permanent
@@ -513,13 +461,6 @@ second period.
 .. _Upstart: http://upstart.ubuntu.com/
 
 
-Vagrant
-~~~~~~~
-
-You may use the Ubuntu explanation above. Replace
-``(local-filesystems and net-device-up IFACE=eth0)`` with ``vagrant-mounted``.
-
-
 .. _setup-permanent-docker:
 
 Docker
diff --git a/docs/troubleshooting.rst b/docs/troubleshooting.rst
index 268235923..05b314004 100644
--- a/docs/troubleshooting.rst
+++ b/docs/troubleshooting.rst
@@ -14,9 +14,8 @@ FORGIVING_OCR is enabled``, then you might need to install the
 `Tesseract language files <http://packages.ubuntu.com/search?keywords=tesseract-ocr>`_
 marching your document's languages.
 
-As an example, if you are running Paperless from the Vagrant setup provided
-(or from any Ubuntu or Debian box), and your documents are written in Spanish
-you may need to run::
+As an example, if you are running Paperless from any Ubuntu or Debian
+box, and your documents are written in Spanish you may need to run::
 
     apt-get install -y tesseract-ocr-spa
 
diff --git a/scripts/vagrant-provision b/scripts/vagrant-provision
deleted file mode 100644
index 940bf476c..000000000
--- a/scripts/vagrant-provision
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/bin/bash
-
-# Install packages
-apt-get update
-apt-get build-dep -y python-imaging
-apt-get install -y libjpeg8 libjpeg62-dev libfreetype6 libfreetype6-dev
-apt-get install -y build-essential python3-dev python3-pip sqlite3 libsqlite3-dev git
-apt-get install -y tesseract-ocr tesseract-ocr-eng imagemagick unpaper
-
-# Python dependencies
-pip3 install -r /opt/paperless/requirements.txt
-
-# Create the environment file
-cat /opt/paperless/paperless.conf.example | sed -e 's#CONSUMPTION_DIR=""#CONSUMPTION_DIR="/home/vagrant/consumption"#' > /etc/paperless.conf
-chmod 0640 /etc/paperless.conf
-chown root:vagrant /etc/paperless.conf
-
-# Create the consumption directory
-mkdir /home/vagrant/consumption
-chown vagrant:vagrant /home/vagrant/consumption
-
-echo "
-
-
-Now follow the remaining steps in the Vagrant section of the setup
-documentation to complete the process:
-
-http://paperless.readthedocs.org/en/latest/setup.html#setup-installation-vagrant
-
-
-"

From 81e488b90d4234984046c040fc4fcfab535a3e79 Mon Sep 17 00:00:00 2001
From: Dean Perry <dean@deanpcmad.com>
Date: Wed, 31 Oct 2018 12:39:48 +0000
Subject: [PATCH 41/47] added missing ; to nginx config

---
 docs/setup.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/setup.rst b/docs/setup.rst
index d794c663b..bb8ac4018 100644
--- a/docs/setup.rst
+++ b/docs/setup.rst
@@ -346,7 +346,7 @@ instance listening on localhost port 8000.
         location /static {
 
             autoindex on;
-            alias <path-to-paperless-static-directory>
+            alias <path-to-paperless-static-directory>;
 
         }
 
@@ -357,7 +357,7 @@ instance listening on localhost port 8000.
             proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
             proxy_set_header X-Forwarded-Proto $scheme;
 
-            proxy_pass http://127.0.0.1:8000
+            proxy_pass http://127.0.0.1:8000;
         }
     }
 

From d825667c9bffe615b753451c5ddbe50d693bc978 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sat, 3 Nov 2018 10:25:51 +0000
Subject: [PATCH 42/47] Allow an infinite number of logs to be deleted.

---
 docs/changelog.rst        | 14 ++++++++++++--
 src/paperless/settings.py | 10 ++++++++++
 2 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index db5fb6a15..d2effb3ac 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -1,6 +1,13 @@
 Changelog
 #########
 
+2.6.0
+=====
+
+* Allow an infinite number of logs to be deleted.  Thanks to `Ulli`_ for noting
+  the problem in `#433`_.
+
+
 2.5.0
 =====
 
@@ -44,6 +51,7 @@ Changelog
 * The ``get_date()`` functionality of the parsers has been consolidated onto
   the ``DocumentParser`` class since much of that code was redundant anyway.
 
+
 2.4.0
 =====
 
@@ -55,13 +63,13 @@ Changelog
   It's now in the import step that we decide the storage type.  This allows you
   to export from an encrypted system and import into an unencrypted one, or
   vice-versa.
-* The migration history has been slightly modified to accomodate PostgreSQL
+* The migration history has been slightly modified to accommodate PostgreSQL
   users.  Additionally, you can now tell paperless to use PostgreSQL simply by
   declaring ``PAPERLESS_DBUSER`` in your environment.  This will attempt to
   connect to your Postgres database without a password unless you also set
   ``PAPERLESS_DBPASS``.
 * A bug was found in the REST API filter system that was the result of an
-  update of django-filter some time ago.  This has now been patched `#412`_.
+  update of django-filter some time ago.  This has now been patched in `#412`_.
   Thanks to `thepill`_ for spotting it!
 
 
@@ -570,6 +578,7 @@ bulk of the work on this big change.
 .. _thepill: https://github.com/thepill
 .. _Andrew Peng: https://github.com/pengc99
 .. _euri10: https://github.com/euri10
+.. _Ulli: https://github.com/Ulli2k
 
 .. _#20: https://github.com/danielquinn/paperless/issues/20
 .. _#44: https://github.com/danielquinn/paperless/issues/44
@@ -664,6 +673,7 @@ bulk of the work on this big change.
 .. _#412: https://github.com/danielquinn/paperless/issues/412
 .. _#413: https://github.com/danielquinn/paperless/pull/413
 .. _#414: https://github.com/danielquinn/paperless/issues/414
+.. _#433: https://github.com/danielquinn/paperless/issues/433
 
 .. _pipenv: https://docs.pipenv.org/
 .. _a new home on Docker Hub: https://hub.docker.com/r/danielquinn/paperless/
diff --git a/src/paperless/settings.py b/src/paperless/settings.py
index fb5a399a8..97226ef44 100644
--- a/src/paperless/settings.py
+++ b/src/paperless/settings.py
@@ -199,6 +199,16 @@ STATIC_URL = os.getenv("PAPERLESS_STATIC_URL", "/static/")
 MEDIA_URL = os.getenv("PAPERLESS_MEDIA_URL", "/media/")
 
 
+# Other
+
+# Disable Django's artificial limit on the number of form fields to submit at
+# once.  This is a protection against overloading the server, but since this is
+# a self-hosted sort of gig, the benefits of being able to mass-delete a tonne
+# of log entries outweight the benefits of such a safeguard.
+
+DATA_UPLOAD_MAX_NUMBER_FIELDS = None
+
+
 # Paperless-specific stuff
 # You shouldn't have to edit any of these values.  Rather, you can set these
 # values in /etc/paperless.conf instead.

From 33abec06631c53379140413de40dedb5450012f6 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sat, 3 Nov 2018 11:05:22 +0000
Subject: [PATCH 43/47] Code cleanup

---
 src/documents/admin.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/documents/admin.py b/src/documents/admin.py
index 5a5a1a50c..6c0424a74 100644
--- a/src/documents/admin.py
+++ b/src/documents/admin.py
@@ -61,12 +61,12 @@ class FinancialYearFilter(admin.SimpleListFilter):
 
             # To keep it simple we use the same string for both
             # query parameter and the display.
-            return (query, query)
+            return query, query
 
         else:
             query = "{0}-{0}".format(date.year)
             display = "{}".format(date.year)
-            return (query, display)
+            return query, display
 
     def lookups(self, request, model_admin):
         if not settings.FY_START or not settings.FY_END:
@@ -146,8 +146,8 @@ class CorrespondentAdmin(CommonAdmin):
 
 class TagAdmin(CommonAdmin):
 
-    list_display = ("name", "colour", "match", "matching_algorithm",
-                    "document_count")
+    list_display = (
+        "name", "colour", "match", "matching_algorithm", "document_count")
     list_filter = ("colour", "matching_algorithm")
     list_editable = ("colour", "match", "matching_algorithm")
 

From eca6250c1b7f07c754688f04a466ba78ca0526a0 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sat, 3 Nov 2018 11:06:55 +0000
Subject: [PATCH 44/47] Fix the correspondent filters #423

---
 docs/changelog.rst     |  6 ++++++
 src/documents/admin.py | 20 +++++++++-----------
 2 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index d2effb3ac..e98436382 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -6,6 +6,9 @@ Changelog
 
 * Allow an infinite number of logs to be deleted.  Thanks to `Ulli`_ for noting
   the problem in `#433`_.
+* Fix the ``RecentCorrespondentsFilter`` correspondents filter that was added
+  in 2.4 to play nice with the defaults.  Thanks to `tsia`_ and `Sblop`_ who
+  pointed this out. `#423`_.
 
 
 2.5.0
@@ -579,6 +582,8 @@ bulk of the work on this big change.
 .. _Andrew Peng: https://github.com/pengc99
 .. _euri10: https://github.com/euri10
 .. _Ulli: https://github.com/Ulli2k
+.. _tsia: https://github.com/tsia
+.. _Sblop:  https://github.com/Sblop
 
 .. _#20: https://github.com/danielquinn/paperless/issues/20
 .. _#44: https://github.com/danielquinn/paperless/issues/44
@@ -673,6 +678,7 @@ bulk of the work on this big change.
 .. _#412: https://github.com/danielquinn/paperless/issues/412
 .. _#413: https://github.com/danielquinn/paperless/pull/413
 .. _#414: https://github.com/danielquinn/paperless/issues/414
+.. _#423: https://github.com/danielquinn/paperless/issues/423
 .. _#433: https://github.com/danielquinn/paperless/issues/433
 
 .. _pipenv: https://docs.pipenv.org/
diff --git a/src/documents/admin.py b/src/documents/admin.py
index 6c0424a74..e61d14815 100644
--- a/src/documents/admin.py
+++ b/src/documents/admin.py
@@ -88,25 +88,24 @@ class FinancialYearFilter(admin.SimpleListFilter):
 
 
 class RecentCorrespondentFilter(admin.RelatedFieldListFilter):
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.title = "correspondent (recent)"
+    """
+    If PAPERLESS_RECENT_CORRESPONDENT_YEARS is set, we limit the available
+    correspondents to documents sent our way over the past ``n`` years.
+    """
 
     def field_choices(self, field, request, model_admin):
 
         years = settings.PAPERLESS_RECENT_CORRESPONDENT_YEARS
-        days = 365 * years
+        correspondents = Correspondent.objects.all()
 
-        lookups = []
         if years and years > 0:
-            correspondents = Correspondent.objects.filter(
+            self.title = "Correspondent (Recent)"
+            days = 365 * years
+            correspondents = correspondents.filter(
                 documents__created__gte=datetime.now() - timedelta(days=days)
             ).distinct()
-            for c in correspondents:
-                lookups.append((c.id, c.name))
 
-        return lookups
+        return [(c.id, c.name) for c in correspondents]
 
 
 class CommonAdmin(admin.ModelAdmin):
@@ -177,7 +176,6 @@ class DocumentAdmin(CommonAdmin):
     list_filter = (
         "tags",
         ("correspondent", RecentCorrespondentFilter),
-        "correspondent",
         FinancialYearFilter
     )
 

From e0acb4a40b3cbebe041c7fabda134a6c3f6ca91a Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sat, 3 Nov 2018 12:49:35 +0000
Subject: [PATCH 45/47] Update dependencies

This includes a security update for requests.
---
 Pipfile            |  11 +-
 Pipfile.lock       | 552 +++++++++++++++++++++++----------------------
 docs/changelog.rst |   2 +
 requirements.txt   |  82 +++----
 4 files changed, 333 insertions(+), 314 deletions(-)

diff --git a/Pipfile b/Pipfile
index b1c30698d..778d15546 100644
--- a/Pipfile
+++ b/Pipfile
@@ -25,6 +25,11 @@ python-dateutil = "*"
 python-dotenv = "*"
 python-gnupg = "*"
 pytz = "*"
+
+[dev-packages]
+ipython = "*"
+sphinx = "*"
+tox = "*"
 pycodestyle = "*"
 pytest = "*"
 pytest-cov = "*"
@@ -32,9 +37,3 @@ pytest-django = "*"
 pytest-sugar = "*"
 pytest-env = "*"
 pytest-xdist = "*"
-
-[dev-packages]
-ipython = "*"
-sphinx = "*"
-tox = "*"
-
diff --git a/Pipfile.lock b/Pipfile.lock
index 71a46d37f..966fbd467 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "6d8bad24aa5d0c102b13b5ae27acba04836cd5a07a4003cb2763de1e0a3406b7"
+            "sha256": "eeeeaf6ecb0cec45a3962eda6647b5263e5ad7939fae29d4b294f59ffe9ca3dd"
         },
         "pipfile-spec": 6,
         "requires": {},
@@ -14,35 +14,12 @@
         ]
     },
     "default": {
-        "apipkg": {
-            "hashes": [
-                "sha256:37228cda29411948b422fae072f57e31d3396d2ee1c9783775980ee9c9990af6",
-                "sha256:58587dd4dc3daefad0487f6d9ae32b4542b185e1c36db6993290e7c41ca2b47c"
-            ],
-            "markers": "python_version != '3.2.*' and python_version >= '2.7' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.3.*'",
-            "version": "==1.5"
-        },
-        "atomicwrites": {
-            "hashes": [
-                "sha256:0312ad34fcad8fac3704d441f7b317e50af620823353ec657a53e981f92920c0",
-                "sha256:ec9ae8adaae229e4f8446952d204a3e4b5fdd2d099f9be3aaf556120135fb3ee"
-            ],
-            "markers": "python_version != '3.2.*' and python_version >= '2.7' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.3.*'",
-            "version": "==1.2.1"
-        },
-        "attrs": {
-            "hashes": [
-                "sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69",
-                "sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb"
-            ],
-            "version": "==18.2.0"
-        },
         "certifi": {
             "hashes": [
-                "sha256:376690d6f16d32f9d1fe8932551d80b23e9d393a8578c5633a2ed39a64861638",
-                "sha256:456048c7e371c089d0a77a5212fb37a2c2dce1e24146e3b7e0261736aaeaa22a"
+                "sha256:339dc09518b07e2fa7eda5450740925974815557727d6bd35d319c1524a04a4c",
+                "sha256:6d58c986d22b038c8c0df30d639f23a3e6d172a05c3583e766f4c0b785c0986a"
             ],
-            "version": "==2018.8.24"
+            "version": "==2018.10.15"
         },
         "chardet": {
             "hashes": [
@@ -55,6 +32,7 @@
             "hashes": [
                 "sha256:03481e81d558d30d230bc12999e3edffe392d244349a90f4ef9b88425fac74ba",
                 "sha256:0b136648de27201056c1869a6c0d4e23f464750fd9a9ba9750b8336a244429ed",
+                "sha256:0bf8cbbd71adfff0ef1f3a1531e6402d13b7b01ac50a79c97ca15f030dba6306",
                 "sha256:10a46017fef60e16694a30627319f38a2b9b52e90182dddb6e37dcdab0f4bf95",
                 "sha256:198626739a79b09fa0a2f06e083ffd12eb55449b5f8bfdbeed1df4910b2ca640",
                 "sha256:23d341cdd4a0371820eb2b0bd6b88f5003a7438bbedb33688cd33b8eae59affd",
@@ -83,18 +61,18 @@
                 "sha256:c1bb572fab8208c400adaf06a8133ac0712179a334c09224fb11393e920abcdd",
                 "sha256:de4418dadaa1c01d497e539210cb6baa015965526ff5afc078c57ca69160108d",
                 "sha256:e05cb4d9aad6233d67e0541caa7e511fa4047ed7750ec2510d466e806e0255d6",
+                "sha256:f05a636b4564104120111800021a92e43397bc12a5c72fed7036be8556e0029e",
                 "sha256:f3f501f345f24383c0000395b26b726e46758b71393267aeae0bd36f8b3ade80"
             ],
-            "markers": "python_version >= '2.6' and python_version != '3.2.*' and python_version != '3.0.*' and python_version != '3.1.*' and python_version < '4'",
             "version": "==4.5.1"
         },
         "coveralls": {
             "hashes": [
-                "sha256:9dee67e78ec17b36c52b778247762851c8e19a893c9a14e921a2fc37f05fac22",
-                "sha256:aec5a1f5e34224b9089664a1b62217732381c7de361b6ed1b3c394d7187b352a"
+                "sha256:ab638e88d38916a6cedbf80a9cd8992d5fa55c77ab755e262e00b36792b7cd6d",
+                "sha256:b2388747e2529fa4c669fb1e3e2756e4e07b6ee56c7d9fce05f35ccccc913aa0"
             ],
             "index": "pypi",
-            "version": "==1.5.0"
+            "version": "==1.5.1"
         },
         "dateparser": {
             "hashes": [
@@ -106,11 +84,11 @@
         },
         "django": {
             "hashes": [
-                "sha256:0c5b65847d00845ee404bbc0b4a85686f15eb3001ffddda3db4e9baa265bf136",
-                "sha256:68aeea369a8130259354b6ba1fa9babe0c5ee6bced505dea4afcd00f765ae38b"
+                "sha256:25df265e1fdb74f7e7305a1de620a84681bcc9c05e84a3ed97e4a1a63024f18d",
+                "sha256:d6d94554abc82ca37e447c3d28958f5ac39bd7d4adaa285543ae97fb1129fd69"
             ],
             "index": "pypi",
-            "version": "==2.0.8"
+            "version": "==2.0.9"
         },
         "django-cors-headers": {
             "hashes": [
@@ -130,11 +108,11 @@
         },
         "django-extensions": {
             "hashes": [
-                "sha256:1f626353a11479014bfe0d77e76d8f866ebca1bb5d595cb57b776230b9e0eb92",
-                "sha256:f21b898598a1628cb73017fb9672e2c5e624133be9764f0eb138e0abf8a62b62"
+                "sha256:30cb6a8c7d6f75a55edf0c0c4491bd98f8264ae1616ce105f9cecac4387edd07",
+                "sha256:4ad86a7a5e84f1c77db030761ae87a600647250c652030a2b71a16e87f3a3d62"
             ],
             "index": "pypi",
-            "version": "==2.1.2"
+            "version": "==2.1.3"
         },
         "django-filter": {
             "hashes": [
@@ -146,11 +124,11 @@
         },
         "djangorestframework": {
             "hashes": [
-                "sha256:b6714c3e4b0f8d524f193c91ecf5f5450092c2145439ac2769711f7eba89a9d9",
-                "sha256:c375e4f95a3a64fccac412e36fb42ba36881e52313ec021ef410b40f67cddca4"
+                "sha256:607865b0bb1598b153793892101d881466bd5a991de12bd6229abb18b1c86136",
+                "sha256:63f76cbe1e7d12b94c357d7e54401103b2e52aef0f7c1650d6c820ad708776e5"
             ],
             "index": "pypi",
-            "version": "==3.8.2"
+            "version": "==3.9.0"
         },
         "docopt": {
             "hashes": [
@@ -158,14 +136,6 @@
             ],
             "version": "==0.6.2"
         },
-        "execnet": {
-            "hashes": [
-                "sha256:a7a84d5fa07a089186a329528f127c9d73b9de57f1a1131b82bb5320ee651f6a",
-                "sha256:fc155a6b553c66c838d1a22dba1dc9f5f505c43285a878c6f74a79c024750b83"
-            ],
-            "markers": "python_version != '3.2.*' and python_version >= '2.7' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.3.*'",
-            "version": "==1.5.0"
-        },
         "factory-boy": {
             "hashes": [
                 "sha256:6f25cc4761ac109efd503f096e2ad99421b1159f01a29dbb917359dcd68e08ca",
@@ -176,11 +146,10 @@
         },
         "faker": {
             "hashes": [
-                "sha256:ea7cfd3aeb1544732d08bd9cfba40c5b78e3a91e17b1a0698ab81bfc5554c628",
-                "sha256:f6d67f04abfb2b4bea7afc7fa6c18cf4c523a67956e455668be9ae42bccc21ad"
+                "sha256:2621643b80a10b91999925cfd20f64d2b36f20bf22136bbdc749bb57d6ffe124",
+                "sha256:5ed822d31bd2d6edf10944d176d30dc9c886afdd381eefb7ba8b7aad86171646"
             ],
-            "markers": "python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.2.*' and python_version >= '2.7'",
-            "version": "==0.9.0"
+            "version": "==0.9.2"
         },
         "filemagic": {
             "hashes": [
@@ -190,12 +159,14 @@
             "version": "==1.6"
         },
         "fuzzywuzzy": {
+            "extras": [
+                "speedup"
+            ],
             "hashes": [
                 "sha256:3759bc6859daa0eecef8c82b45404bdac20c23f23136cf4c18b46b426bbc418f",
                 "sha256:5b36957ccf836e700f4468324fa80ba208990385392e217be077d5cd738ae602"
             ],
             "index": "pypi",
-            "markers": null,
             "version": "==0.15.0"
         },
         "gunicorn": {
@@ -227,80 +198,48 @@
             "index": "pypi",
             "version": "==1.0.7"
         },
-        "more-itertools": {
-            "hashes": [
-                "sha256:c187a73da93e7a8acc0001572aebc7e3c69daf7bf6881a2cea10650bd4420092",
-                "sha256:c476b5d3a34e12d40130bc2f935028b5f636df8f372dc2c1c01dc19681b2039e",
-                "sha256:fcbfeaea0be121980e15bc97b3817b5202ca73d0eae185b4550cbfce2a3ebb3d"
-            ],
-            "version": "==4.3.0"
-        },
         "pdftotext": {
             "hashes": [
-                "sha256:b7312302007e19fc784263a321b41682f01a582af84e14200cef53b3f4e69a50"
+                "sha256:e3ad11efe0aa22cbfc46aa1296b2ea5a52ad208b778288311f2801adef178ccb"
             ],
             "index": "pypi",
-            "version": "==2.1.0"
+            "version": "==2.1.1"
         },
         "pillow": {
             "hashes": [
-                "sha256:00def5b638994f888d1058e4d17c86dec8e1113c3741a0a8a659039aec59a83a",
-                "sha256:026449b64e559226cdb8e6d8c931b5965d8fc90ec18ebbb0baa04c5b36503c72",
-                "sha256:03dbb224ee196ef30ed2156d41b579143e1efeb422974719a5392fc035e4f574",
-                "sha256:03eb0e04f929c102ae24bc436bf1c0c60a4e63b07ebd388e84d8b219df3e6acd",
-                "sha256:1be66b9a89e367e7d20d6cae419794997921fe105090fafd86ef39e20a3baab2",
-                "sha256:1e977a3ed998a599bda5021fb2c2889060617627d3ae228297a529a082a3cd5c",
-                "sha256:22cf3406d135cfcc13ec6228ade774c8461e125c940e80455f500638429be273",
-                "sha256:24adccf1e834f82718c7fc8e3ec1093738da95144b8b1e44c99d5fc7d3e9c554",
-                "sha256:2a3e362c97a5e6a259ee9cd66553292a1f8928a5bdfa3622fdb1501570834612",
-                "sha256:3832e26ecbc9d8a500821e3a1d3765bda99d04ae29ffbb2efba49f5f788dc934",
-                "sha256:4fd1f0c2dc02aaec729d91c92cd85a2df0289d88e9f68d1e8faba750bb9c4786",
-                "sha256:4fda62030f2c515b6e2e673c57caa55cb04026a81968f3128aae10fc28e5cc27",
-                "sha256:5044d75a68b49ce36a813c82d8201384207112d5d81643937fc758c05302f05b",
-                "sha256:522184556921512ec484cb93bd84e0bab915d0ac5a372d49571c241a7f73db62",
-                "sha256:5914cff11f3e920626da48e564be6818831713a3087586302444b9c70e8552d9",
-                "sha256:6661a7908d68c4a133e03dac8178287aa20a99f841ea90beeb98a233ae3fd710",
-                "sha256:79258a8df3e309a54c7ef2ef4a59bb8e28f7e4a8992a3ad17c24b1889ced44f3",
-                "sha256:7d74c20b8f1c3e99d3f781d3b8ff5abfefdd7363d61e23bdeba9992ff32cc4b4",
-                "sha256:81918afeafc16ba5d9d0d4e9445905f21aac969a4ebb6f2bff4b9886da100f4b",
-                "sha256:8194d913ca1f459377c8a4ed8f9b7ad750068b8e0e3f3f9c6963fcc87a84515f",
-                "sha256:84d5d31200b11b3c76fab853b89ac898bf2d05c8b3da07c1fcc23feb06359d6e",
-                "sha256:989981db57abffb52026b114c9a1f114c7142860a6d30a352d28f8cbf186500b",
-                "sha256:a3d7511d3fad1618a82299aab71a5fceee5c015653a77ffea75ced9ef917e71a",
-                "sha256:b3ef168d4d6fd4fa6685aef7c91400f59f7ab1c0da734541f7031699741fb23f",
-                "sha256:c1c5792b6e74bbf2af0f8e892272c2a6c48efa895903211f11b8342e03129fea",
-                "sha256:c5dcb5a56aebb8a8f2585042b2f5c496d7624f0bcfe248f0cc33ceb2fd8d39e7",
-                "sha256:e2bed4a04e2ca1050bb5f00865cf2f83c0b92fd62454d9244f690fcd842e27a4",
-                "sha256:e87a527c06319428007e8c30511e1f0ce035cb7f14bb4793b003ed532c3b9333",
-                "sha256:f63e420180cbe22ff6e32558b612e75f50616fc111c5e095a4631946c782e109",
-                "sha256:f8b3d413c5a8f84b12cd4c5df1d8e211777c9852c6be3ee9c094b626644d3eab"
+                "sha256:00203f406818c3f45d47bb8fe7e67d3feddb8dcbbd45a289a1de7dd789226360",
+                "sha256:0616f800f348664e694dddb0b0c88d26761dd5e9f34e1ed7b7a7d2da14b40cb7",
+                "sha256:1f7908aab90c92ad85af9d2fec5fc79456a89b3adcc26314d2cde0e238bd789e",
+                "sha256:2ea3517cd5779843de8a759c2349a3cd8d3893e03ab47053b66d5ec6f8bc4f93",
+                "sha256:48a9f0538c91fc136b3a576bee0e7cd174773dc9920b310c21dcb5519722e82c",
+                "sha256:5280ebc42641a1283b7b1f2c20e5b936692198b9dd9995527c18b794850be1a8",
+                "sha256:5e34e4b5764af65551647f5cc67cf5198c1d05621781d5173b342e5e55bf023b",
+                "sha256:63b120421ab85cad909792583f83b6ca3584610c2fe70751e23f606a3c2e87f0",
+                "sha256:696b5e0109fe368d0057f484e2e91717b49a03f1e310f857f133a4acec9f91dd",
+                "sha256:870ed021a42b1b02b5fe4a739ea735f671a84128c0a666c705db2cb9abd528eb",
+                "sha256:916da1c19e4012d06a372127d7140dae894806fad67ef44330e5600d77833581",
+                "sha256:9303a289fa0811e1c6abd9ddebfc770556d7c3311cb2b32eff72164ddc49bc64",
+                "sha256:9577888ecc0ad7d06c3746afaba339c94d62b59da16f7a5d1cff9e491f23dace",
+                "sha256:987e1c94a33c93d9b209315bfda9faa54b8edfce6438a1e93ae866ba20de5956",
+                "sha256:99a3bbdbb844f4fb5d6dd59fac836a40749781c1fa63c563bc216c27aef63f60",
+                "sha256:99db8dc3097ceafbcff9cb2bff384b974795edeb11d167d391a02c7bfeeb6e16",
+                "sha256:a5a96cf49eb580756a44ecf12949e52f211e20bffbf5a95760ac14b1e499cd37",
+                "sha256:aa6ca3eb56704cdc0d876fc6047ffd5ee960caad52452fbee0f99908a141a0ae",
+                "sha256:aade5e66795c94e4a2b2624affeea8979648d1b0ae3fcee17e74e2c647fc4a8a",
+                "sha256:b78905860336c1d292409e3df6ad39cc1f1c7f0964e66844bbc2ebfca434d073",
+                "sha256:b92f521cdc4e4a3041cc343625b699f20b0b5f976793fb45681aac1efda565f8",
+                "sha256:bfde84bbd6ae5f782206d454b67b7ee8f7f818c29b99fd02bf022fd33bab14cb",
+                "sha256:c2b62d3df80e694c0e4a0ed47754c9480521e25642251b3ab1dff050a4e60409",
+                "sha256:c5e2be6c263b64f6f7656e23e18a4a9980cffc671442795682e8c4e4f815dd9f",
+                "sha256:c99aa3c63104e0818ec566f8ff3942fb7c7a8f35f9912cb63fd8e12318b214b2",
+                "sha256:dae06620d3978da346375ebf88b9e2dd7d151335ba668c995aea9ed07af7add4",
+                "sha256:db5499d0710823fa4fb88206050d46544e8f0e0136a9a5f5570b026584c8fd74",
+                "sha256:f36baafd82119c4a114b9518202f2a983819101dcc14b26e43fc12cbefdce00e",
+                "sha256:f52b79c8796d81391ab295b04e520bda6feed54d54931708872e8f9ae9db0ea1",
+                "sha256:ff8cff01582fa1a7e533cb97f628531c4014af4b5f38e33cdcfe5eec29b6d888"
             ],
             "index": "pypi",
-            "version": "==5.2.0"
-        },
-        "pluggy": {
-            "hashes": [
-                "sha256:6e3836e39f4d36ae72840833db137f7b7d35105079aee6ec4a62d9f80d594dd1",
-                "sha256:95eb8364a4708392bae89035f45341871286a333f749c3141c20573d2b3876e1"
-            ],
-            "markers": "python_version != '3.2.*' and python_version >= '2.7' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.3.*'",
-            "version": "==0.7.1"
-        },
-        "py": {
-            "hashes": [
-                "sha256:06a30435d058473046be836d3fc4f27167fd84c45b99704f2fb5509ef61f9af1",
-                "sha256:50402e9d1c9005d759426988a492e0edaadb7f4e68bcddfea586bc7432d009c6"
-            ],
-            "markers": "python_version != '3.2.*' and python_version >= '2.7' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.3.*'",
-            "version": "==1.6.0"
-        },
-        "pycodestyle": {
-            "hashes": [
-                "sha256:cbc619d09254895b0d12c2c691e237b2e91e9b2ecf5e84c26b35400f93dcfb83",
-                "sha256:cbfca99bd594a10f674d0cd97a3d802a1fdef635d4361e1a2658de47ed261e3a"
-            ],
-            "index": "pypi",
-            "version": "==2.4.0"
+            "version": "==5.3.0"
         },
         "pyocr": {
             "hashes": [
@@ -309,67 +248,13 @@
             "index": "pypi",
             "version": "==0.5.3"
         },
-        "pytest": {
-            "hashes": [
-                "sha256:453cbbbe5ce6db38717d282b758b917de84802af4288910c12442984bde7b823",
-                "sha256:a8a07f84e680482eb51e244370aaf2caa6301ef265f37c2bdefb3dd3b663f99d"
-            ],
-            "index": "pypi",
-            "version": "==3.8.0"
-        },
-        "pytest-cov": {
-            "hashes": [
-                "sha256:513c425e931a0344944f84ea47f3956be0e416d95acbd897a44970c8d926d5d7",
-                "sha256:e360f048b7dae3f2f2a9a4d067b2dd6b6a015d384d1577c994a43f3f7cbad762"
-            ],
-            "index": "pypi",
-            "version": "==2.6.0"
-        },
-        "pytest-django": {
-            "hashes": [
-                "sha256:2d2e0a618d91c280d463e90bcbea9b4e417609157f611a79685b1c561c4c0836",
-                "sha256:59683def396923b78d7e191a7086a48193f8d5db869ace79acb38f906522bc7b"
-            ],
-            "index": "pypi",
-            "version": "==3.4.2"
-        },
-        "pytest-env": {
-            "hashes": [
-                "sha256:7e94956aef7f2764f3c147d216ce066bf6c42948bb9e293169b1b1c880a580c2"
-            ],
-            "index": "pypi",
-            "version": "==0.6.2"
-        },
-        "pytest-forked": {
-            "hashes": [
-                "sha256:e4500cd0509ec4a26535f7d4112a8cc0f17d3a41c29ffd4eab479d2a55b30805",
-                "sha256:f275cb48a73fc61a6710726348e1da6d68a978f0ec0c54ece5a5fae5977e5a08"
-            ],
-            "markers": "python_version != '3.2.*' and python_version >= '2.7' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.3.*'",
-            "version": "==0.2"
-        },
-        "pytest-sugar": {
-            "hashes": [
-                "sha256:ab8cc42faf121344a4e9b13f39a51257f26f410e416c52ea11078cdd00d98a2c"
-            ],
-            "index": "pypi",
-            "version": "==0.9.1"
-        },
-        "pytest-xdist": {
-            "hashes": [
-                "sha256:0875deac20f6d96597036bdf63970887a6f36d28289c2f6682faf652dfea687b",
-                "sha256:28e25e79698b2662b648319d3971c0f9ae0e6500f88258ccb9b153c31110ba9b"
-            ],
-            "index": "pypi",
-            "version": "==1.23.0"
-        },
         "python-dateutil": {
             "hashes": [
-                "sha256:1adb80e7a782c12e52ef9a8182bebeb73f1d7e24e374397af06fb4956c8dc5c0",
-                "sha256:e27001de32f627c22380a688bcc43ce83504a7bc5da472209b4c70f02829f0b8"
+                "sha256:063df5763652e21de43de7d9e00ccf239f953a832941e37be541614732cdfc93",
+                "sha256:88f9287c0174266bb0d8cedd395cfba9c58e87e5ad86b2ce58859bc11be3cf02"
             ],
             "index": "pypi",
-            "version": "==2.7.3"
+            "version": "==2.7.5"
         },
         "python-dotenv": {
             "hashes": [
@@ -391,42 +276,37 @@
             "hashes": [
                 "sha256:033a11de5e3d19ea25c9302d11224e1a1898fe5abd23c61c7c360c25195e3eb1"
             ],
+            "markers": "extra == 'speedup'",
             "version": "==0.12.0"
         },
         "pytz": {
             "hashes": [
-                "sha256:a061aa0a9e06881eb8b3b2b43f05b9439d6583c206d0a6c340ff72a7b6669053",
-                "sha256:ffb9ef1de172603304d9d2819af6f5ece76f2e85ec10692a524dd876e72bf277"
+                "sha256:31cb35c89bd7d333cd32c5f278fca91b523b0834369e757f4c5641ea252236ca",
+                "sha256:8e0f8568c118d3077b46be7d654cc8167fa916092e28320cde048e54bfc9f1e6"
             ],
             "index": "pypi",
-            "version": "==2018.5"
+            "version": "==2018.7"
         },
         "regex": {
             "hashes": [
-                "sha256:22d7ef8c2df344328a8a3c61edade2ee714e5de9360911d22a9213931c769faa",
-                "sha256:3a699780c6b712c67dc23207b129ccc6a7e1270233f7aadead3ea3f83c893702",
-                "sha256:42f460d349baebd5faec02a0c920988fb0300b24baf898d9c139886565b66b6c",
-                "sha256:43bf3d79940cbdf19adda838d8b26b28b47bec793cda46590b5b25703742f440",
-                "sha256:47d6c7f0588ef33464e00023067c4e7cce68e0d6a686a73c7ee15abfdad503d4",
-                "sha256:5b879f59f25ed9b91bc8693a9a994014b431f224f492519ad0255ce6b54b83e5",
-                "sha256:8ba0093c412900f636b0f826c597a0c3ea0e395344bc99894ddefe88b76c9c7e",
-                "sha256:a4789254a1a0bd7a637036cce0b7ed72d8cc864e93f2e9cfd10ac00ae27bb7b0",
-                "sha256:b73cea07117dca888b0c3671770b501bef19aac9c45c8ffdb5bea2cca2377b0a",
-                "sha256:d3eb59fa3e5b5438438ec97acd9dc86f077428e020b015b43987e35bea68ef4c",
-                "sha256:d51d232b4e2f106deaf286001f563947fee255bc5bd209a696f027e15cf0a1e7",
-                "sha256:d59b03131a8e35061b47a8f186324a95eaf30d5f6ee9cc0637e7b87d29c7c9b5",
-                "sha256:dd705df1b47470388fc4630e4df3cbbe7677e2ab80092a1c660cae630a307b2d",
-                "sha256:e87fffa437a4b00afb17af785da9b01618425d6cd984c677639deb937037d8f2",
-                "sha256:ed40e0474ab5ab228a8d133759d451b31d3ccdebaff698646e54aff82c3de4f8"
+                "sha256:0ef96690c3d2294155b7d44187ca4a151e45c931cb768e106ba464a9fa64c5da",
+                "sha256:251683e01a3bcacd9188acf0d4caf7b29a3b963c843159311825613ae144cddb",
+                "sha256:3fe15a75fe00f04d1ec16713d55cf1e206077c450267a10b33318756fb8b3f99",
+                "sha256:53a962f9dc28cdf403978a142cb1e054479759ad64d312a999f9f042c25b5c9a",
+                "sha256:8bd1da6a93d32336a5e5432886dd8543004f0591c39b83dbfa60705cccdf414d",
+                "sha256:b5423061918f602e9342b54d746ac31c598d328ecaf4ef0618763e960c926fd4",
+                "sha256:d80ebc65b1f7d0403117f59309c16eac24be6a0bc730b593a79f703462858d94",
+                "sha256:fd8419979639b7de7fb964a13bce3ac47e6fe33043b83de0398c3067986e5659",
+                "sha256:ff2f15b2b0b4b58ba8a1de651780a0d3fd54f96ad6b77dceb77695220e5d7b7a"
             ],
-            "version": "==2018.8.29"
+            "version": "==2018.11.2"
         },
         "requests": {
             "hashes": [
-                "sha256:63b52e3c866428a224f97cab011de738c36aec0185aa91cfacd418b5d58911d1",
-                "sha256:ec22d826a36ed72a7358ff3fe56cbd4ba69dd7a6718ffd450ff0e9df7a47ce6a"
+                "sha256:99dcfdaaeb17caf6e526f32b6a7b780461512ab3f1d992187801694cba42770c",
+                "sha256:a84b8c9ab6239b578f22d1c21d51b696dcfe004032bb80ea832398d6909d7279"
             ],
-            "version": "==2.19.1"
+            "version": "==2.20.0"
         },
         "six": {
             "hashes": [
@@ -435,12 +315,6 @@
             ],
             "version": "==1.11.0"
         },
-        "termcolor": {
-            "hashes": [
-                "sha256:1d6d69ce66211143803fbc56652b41d73b4a400a2891d7bf7a1cdf4c02de613b"
-            ],
-            "version": "==1.1.0"
-        },
         "text-unidecode": {
             "hashes": [
                 "sha256:5a1375bb2ba7968740508ae38d92e1f889a0832913cb1c447d5e2046061a396d",
@@ -456,20 +330,40 @@
         },
         "urllib3": {
             "hashes": [
-                "sha256:a68ac5e15e76e7e5dd2b8f94007233e01effe3e50e8daddf69acfd81cb686baf",
-                "sha256:b5725a0bd4ba422ab0e66e89e030c806576753ea3ee08554382c14e685d117b5"
+                "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39",
+                "sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22"
             ],
-            "markers": "python_version >= '2.6' and python_version != '3.2.*' and python_version != '3.0.*' and python_version != '3.1.*' and python_version < '4' and python_version != '3.3.*'",
-            "version": "==1.23"
+            "version": "==1.24.1"
         }
     },
     "develop": {
         "alabaster": {
             "hashes": [
-                "sha256:674bb3bab080f598371f4443c5008cbfeb1a5e622dd312395d2d82af2c54c456",
-                "sha256:b63b1f4dc77c074d386752ec4a8a7517600f6c0db8cd42980cae17ab7b3275d7"
+                "sha256:446438bdcca0e05bd45ea2de1668c1d9b032e1a9154c2c259092d77031ddd359",
+                "sha256:a661d72d58e6ea8a57f7a86e37d86716863ee5e92788398526d58b26a4e4dc02"
             ],
-            "version": "==0.7.11"
+            "version": "==0.7.12"
+        },
+        "apipkg": {
+            "hashes": [
+                "sha256:37228cda29411948b422fae072f57e31d3396d2ee1c9783775980ee9c9990af6",
+                "sha256:58587dd4dc3daefad0487f6d9ae32b4542b185e1c36db6993290e7c41ca2b47c"
+            ],
+            "version": "==1.5"
+        },
+        "atomicwrites": {
+            "hashes": [
+                "sha256:0312ad34fcad8fac3704d441f7b317e50af620823353ec657a53e981f92920c0",
+                "sha256:ec9ae8adaae229e4f8446952d204a3e4b5fdd2d099f9be3aaf556120135fb3ee"
+            ],
+            "version": "==1.2.1"
+        },
+        "attrs": {
+            "hashes": [
+                "sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69",
+                "sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb"
+            ],
+            "version": "==18.2.0"
         },
         "babel": {
             "hashes": [
@@ -487,10 +381,10 @@
         },
         "certifi": {
             "hashes": [
-                "sha256:376690d6f16d32f9d1fe8932551d80b23e9d393a8578c5633a2ed39a64861638",
-                "sha256:456048c7e371c089d0a77a5212fb37a2c2dce1e24146e3b7e0261736aaeaa22a"
+                "sha256:339dc09518b07e2fa7eda5450740925974815557727d6bd35d319c1524a04a4c",
+                "sha256:6d58c986d22b038c8c0df30d639f23a3e6d172a05c3583e766f4c0b785c0986a"
             ],
-            "version": "==2018.8.24"
+            "version": "==2018.10.15"
         },
         "chardet": {
             "hashes": [
@@ -499,6 +393,44 @@
             ],
             "version": "==3.0.4"
         },
+        "coverage": {
+            "hashes": [
+                "sha256:03481e81d558d30d230bc12999e3edffe392d244349a90f4ef9b88425fac74ba",
+                "sha256:0b136648de27201056c1869a6c0d4e23f464750fd9a9ba9750b8336a244429ed",
+                "sha256:0bf8cbbd71adfff0ef1f3a1531e6402d13b7b01ac50a79c97ca15f030dba6306",
+                "sha256:10a46017fef60e16694a30627319f38a2b9b52e90182dddb6e37dcdab0f4bf95",
+                "sha256:198626739a79b09fa0a2f06e083ffd12eb55449b5f8bfdbeed1df4910b2ca640",
+                "sha256:23d341cdd4a0371820eb2b0bd6b88f5003a7438bbedb33688cd33b8eae59affd",
+                "sha256:28b2191e7283f4f3568962e373b47ef7f0392993bb6660d079c62bd50fe9d162",
+                "sha256:2a5b73210bad5279ddb558d9a2bfedc7f4bf6ad7f3c988641d83c40293deaec1",
+                "sha256:2eb564bbf7816a9d68dd3369a510be3327f1c618d2357fa6b1216994c2e3d508",
+                "sha256:337ded681dd2ef9ca04ef5d93cfc87e52e09db2594c296b4a0a3662cb1b41249",
+                "sha256:3a2184c6d797a125dca8367878d3b9a178b6fdd05fdc2d35d758c3006a1cd694",
+                "sha256:3c79a6f7b95751cdebcd9037e4d06f8d5a9b60e4ed0cd231342aa8ad7124882a",
+                "sha256:3d72c20bd105022d29b14a7d628462ebdc61de2f303322c0212a054352f3b287",
+                "sha256:3eb42bf89a6be7deb64116dd1cc4b08171734d721e7a7e57ad64cc4ef29ed2f1",
+                "sha256:4635a184d0bbe537aa185a34193898eee409332a8ccb27eea36f262566585000",
+                "sha256:56e448f051a201c5ebbaa86a5efd0ca90d327204d8b059ab25ad0f35fbfd79f1",
+                "sha256:5a13ea7911ff5e1796b6d5e4fbbf6952381a611209b736d48e675c2756f3f74e",
+                "sha256:69bf008a06b76619d3c3f3b1983f5145c75a305a0fea513aca094cae5c40a8f5",
+                "sha256:6bc583dc18d5979dc0f6cec26a8603129de0304d5ae1f17e57a12834e7235062",
+                "sha256:701cd6093d63e6b8ad7009d8a92425428bc4d6e7ab8d75efbb665c806c1d79ba",
+                "sha256:7608a3dd5d73cb06c531b8925e0ef8d3de31fed2544a7de6c63960a1e73ea4bc",
+                "sha256:76ecd006d1d8f739430ec50cc872889af1f9c1b6b8f48e29941814b09b0fd3cc",
+                "sha256:7aa36d2b844a3e4a4b356708d79fd2c260281a7390d678a10b91ca595ddc9e99",
+                "sha256:7d3f553904b0c5c016d1dad058a7554c7ac4c91a789fca496e7d8347ad040653",
+                "sha256:7e1fe19bd6dce69d9fd159d8e4a80a8f52101380d5d3a4d374b6d3eae0e5de9c",
+                "sha256:8c3cb8c35ec4d9506979b4cf90ee9918bc2e49f84189d9bf5c36c0c1119c6558",
+                "sha256:9d6dd10d49e01571bf6e147d3b505141ffc093a06756c60b053a859cb2128b1f",
+                "sha256:be6cfcd8053d13f5f5eeb284aa8a814220c3da1b0078fa859011c7fffd86dab9",
+                "sha256:c1bb572fab8208c400adaf06a8133ac0712179a334c09224fb11393e920abcdd",
+                "sha256:de4418dadaa1c01d497e539210cb6baa015965526ff5afc078c57ca69160108d",
+                "sha256:e05cb4d9aad6233d67e0541caa7e511fa4047ed7750ec2510d466e806e0255d6",
+                "sha256:f05a636b4564104120111800021a92e43397bc12a5c72fed7036be8556e0029e",
+                "sha256:f3f501f345f24383c0000395b26b726e46758b71393267aeae0bd36f8b3ade80"
+            ],
+            "version": "==4.5.1"
+        },
         "decorator": {
             "hashes": [
                 "sha256:2c51dff8ef3c447388fe5e4453d24a2bf128d3a4c32af3fabef1f01c6851ab82",
@@ -514,6 +446,20 @@
             ],
             "version": "==0.14"
         },
+        "execnet": {
+            "hashes": [
+                "sha256:a7a84d5fa07a089186a329528f127c9d73b9de57f1a1131b82bb5320ee651f6a",
+                "sha256:fc155a6b553c66c838d1a22dba1dc9f5f505c43285a878c6f74a79c024750b83"
+            ],
+            "version": "==1.5.0"
+        },
+        "filelock": {
+            "hashes": [
+                "sha256:b8d5ca5ca1c815e1574aee746650ea7301de63d87935b3463d26368b76e31633",
+                "sha256:d610c1bb404daf85976d7a82eb2ada120f04671007266b708606565dd03b5be6"
+            ],
+            "version": "==3.0.10"
+        },
         "idna": {
             "hashes": [
                 "sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e",
@@ -526,16 +472,15 @@
                 "sha256:3f349de3eb99145973fefb7dbe38554414e5c30abd0c8e4b970a7c9d09f3a1d8",
                 "sha256:f3832918bc3c66617f92e35f5d70729187676313caa60c187eb0f28b8fe5e3b5"
             ],
-            "markers": "python_version != '3.2.*' and python_version >= '2.7' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.3.*'",
             "version": "==1.1.0"
         },
         "ipython": {
             "hashes": [
-                "sha256:007dcd929c14631f83daff35df0147ea51d1af420da303fd078343878bd5fb62",
-                "sha256:b0f2ef9eada4a68ef63ee10b6dde4f35c840035c50fd24265f8052c98947d5a4"
+                "sha256:a5781d6934a3341a1f9acb4ea5acdc7ea0a0855e689dbe755d070ca51e995435",
+                "sha256:b10a7ddd03657c761fc503495bc36471c8158e3fc948573fb9fe82a7029d8efd"
             ],
             "index": "pypi",
-            "version": "==6.5.0"
+            "version": "==7.1.1"
         },
         "ipython-genutils": {
             "hashes": [
@@ -546,10 +491,10 @@
         },
         "jedi": {
             "hashes": [
-                "sha256:b409ed0f6913a701ed474a614a3bb46e6953639033e31f769ca7581da5bd1ec1",
-                "sha256:c254b135fb39ad76e78d4d8f92765ebc9bf92cbc76f49e97ade1d5f5121e1f6f"
+                "sha256:0191c447165f798e6a730285f2eee783fff81b0d3df261945ecb80983b5c3ca7",
+                "sha256:b7493f73a2febe0dc33d51c99b474547f7f6c0b2c8fb2b21f453eef204c12148"
             ],
-            "version": "==0.12.1"
+            "version": "==0.13.1"
         },
         "jinja2": {
             "hashes": [
@@ -564,12 +509,20 @@
             ],
             "version": "==1.0"
         },
+        "more-itertools": {
+            "hashes": [
+                "sha256:c187a73da93e7a8acc0001572aebc7e3c69daf7bf6881a2cea10650bd4420092",
+                "sha256:c476b5d3a34e12d40130bc2f935028b5f636df8f372dc2c1c01dc19681b2039e",
+                "sha256:fcbfeaea0be121980e15bc97b3817b5202ca73d0eae185b4550cbfce2a3ebb3d"
+            ],
+            "version": "==4.3.0"
+        },
         "packaging": {
             "hashes": [
-                "sha256:e9215d2d2535d3ae866c3d6efc77d5b24a0192cce0ff20e42896cc0664f889c0",
-                "sha256:f019b770dd64e585a99714f1fd5e01c7a8f11b45635aa953fd41c689a657375b"
+                "sha256:0886227f54515e592aaa2e5a553332c73962917f2831f1b0f9b9f4380a4b9807",
+                "sha256:f95a1e147590f204328170981833854229bb2912ac3d5f89e2a8ccd2834800c9"
             ],
-            "version": "==17.1"
+            "version": "==18.0"
         },
         "parso": {
             "hashes": [
@@ -588,26 +541,25 @@
         },
         "pickleshare": {
             "hashes": [
-                "sha256:84a9257227dfdd6fe1b4be1319096c20eb85ff1e82c7932f36efccfe1b09737b",
-                "sha256:c9a2541f25aeabc070f12f452e1f2a8eae2abd51e1cd19e8430402bdf4c1d8b5"
+                "sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca",
+                "sha256:9649af414d74d4df115d5d718f82acb59c9d418196b7b4290ed47a12ce62df56"
             ],
-            "version": "==0.7.4"
+            "version": "==0.7.5"
         },
         "pluggy": {
             "hashes": [
-                "sha256:6e3836e39f4d36ae72840833db137f7b7d35105079aee6ec4a62d9f80d594dd1",
-                "sha256:95eb8364a4708392bae89035f45341871286a333f749c3141c20573d2b3876e1"
+                "sha256:447ba94990e8014ee25ec853339faf7b0fc8050cdc3289d4d71f7f410fb90095",
+                "sha256:bde19360a8ec4dfd8a20dcb811780a30998101f078fc7ded6162f0076f50508f"
             ],
-            "markers": "python_version != '3.2.*' and python_version >= '2.7' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.3.*'",
-            "version": "==0.7.1"
+            "version": "==0.8.0"
         },
         "prompt-toolkit": {
             "hashes": [
-                "sha256:1df952620eccb399c53ebb359cc7d9a8d3a9538cb34c5a1344bdbeb29fbcc381",
-                "sha256:3f473ae040ddaa52b52f97f6b4a493cfa9f5920c255a12dc56a7d34397a398a4",
-                "sha256:858588f1983ca497f1cf4ffde01d978a3ea02b01c8a26a8bbc5cd2e66d816917"
+                "sha256:c1d6aff5252ab2ef391c2fe498ed8c088066f66bc64a8d5c095bbf795d9fec34",
+                "sha256:d4c47f79b635a0e70b84fdb97ebd9a274203706b1ee5ed44c10da62755cf3ec9",
+                "sha256:fd17048d8335c1e6d5ee403c3569953ba3eb8555d710bfc548faf0712666ea39"
             ],
-            "version": "==1.0.15"
+            "version": "==2.0.7"
         },
         "ptyprocess": {
             "hashes": [
@@ -618,11 +570,18 @@
         },
         "py": {
             "hashes": [
-                "sha256:06a30435d058473046be836d3fc4f27167fd84c45b99704f2fb5509ef61f9af1",
-                "sha256:50402e9d1c9005d759426988a492e0edaadb7f4e68bcddfea586bc7432d009c6"
+                "sha256:bf92637198836372b520efcba9e020c330123be8ce527e535d185ed4b6f45694",
+                "sha256:e76826342cefe3c3d5f7e8ee4316b80d1dd8a300781612ddbc765c17ba25a6c6"
             ],
-            "markers": "python_version != '3.2.*' and python_version >= '2.7' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.3.*'",
-            "version": "==1.6.0"
+            "version": "==1.7.0"
+        },
+        "pycodestyle": {
+            "hashes": [
+                "sha256:cbc619d09254895b0d12c2c691e237b2e91e9b2ecf5e84c26b35400f93dcfb83",
+                "sha256:cbfca99bd594a10f674d0cd97a3d802a1fdef635d4361e1a2658de47ed261e3a"
+            ],
+            "index": "pypi",
+            "version": "==2.4.0"
         },
         "pygments": {
             "hashes": [
@@ -633,31 +592,78 @@
         },
         "pyparsing": {
             "hashes": [
-                "sha256:0832bcf47acd283788593e7a0f542407bd9550a55a8a8435214a1960e04bcb04",
-                "sha256:fee43f17a9c4087e7ed1605bd6df994c6173c1e977d7ade7b651292fab2bd010"
+                "sha256:40856e74d4987de5d01761a22d1621ae1c7f8774585acae358aa5c5936c6c90b",
+                "sha256:f353aab21fd474459d97b709e527b5571314ee5f067441dc9f88e33eecd96592"
             ],
-            "version": "==2.2.0"
+            "version": "==2.3.0"
+        },
+        "pytest": {
+            "hashes": [
+                "sha256:a9e5e8d7ab9d5b0747f37740276eb362e6a76275d76cebbb52c6049d93b475db",
+                "sha256:bf47e8ed20d03764f963f0070ff1c8fda6e2671fc5dd562a4d3b7148ad60f5ca"
+            ],
+            "index": "pypi",
+            "version": "==3.9.3"
+        },
+        "pytest-cov": {
+            "hashes": [
+                "sha256:513c425e931a0344944f84ea47f3956be0e416d95acbd897a44970c8d926d5d7",
+                "sha256:e360f048b7dae3f2f2a9a4d067b2dd6b6a015d384d1577c994a43f3f7cbad762"
+            ],
+            "index": "pypi",
+            "version": "==2.6.0"
+        },
+        "pytest-django": {
+            "hashes": [
+                "sha256:49e9ffc856bc6a1bec1c26c5c7b7213dff7cc8bc6b64d624c4d143d04aff0bcf",
+                "sha256:b379282feaf89069cb790775ab6bbbd2bd2038a68c7ef9b84a41898e0b551081"
+            ],
+            "index": "pypi",
+            "version": "==3.4.3"
+        },
+        "pytest-env": {
+            "hashes": [
+                "sha256:7e94956aef7f2764f3c147d216ce066bf6c42948bb9e293169b1b1c880a580c2"
+            ],
+            "index": "pypi",
+            "version": "==0.6.2"
+        },
+        "pytest-forked": {
+            "hashes": [
+                "sha256:e4500cd0509ec4a26535f7d4112a8cc0f17d3a41c29ffd4eab479d2a55b30805",
+                "sha256:f275cb48a73fc61a6710726348e1da6d68a978f0ec0c54ece5a5fae5977e5a08"
+            ],
+            "version": "==0.2"
+        },
+        "pytest-sugar": {
+            "hashes": [
+                "sha256:ab8cc42faf121344a4e9b13f39a51257f26f410e416c52ea11078cdd00d98a2c"
+            ],
+            "index": "pypi",
+            "version": "==0.9.1"
+        },
+        "pytest-xdist": {
+            "hashes": [
+                "sha256:3bc9dcb6ff47e607d3c710727cd9996fd7ac1466d405c3b40bb495da99b6b669",
+                "sha256:8e188d13ce6614c7a678179a76f46231199ffdfe6163de031c17e62ffa256917"
+            ],
+            "index": "pypi",
+            "version": "==1.24.0"
         },
         "pytz": {
             "hashes": [
-                "sha256:a061aa0a9e06881eb8b3b2b43f05b9439d6583c206d0a6c340ff72a7b6669053",
-                "sha256:ffb9ef1de172603304d9d2819af6f5ece76f2e85ec10692a524dd876e72bf277"
+                "sha256:31cb35c89bd7d333cd32c5f278fca91b523b0834369e757f4c5641ea252236ca",
+                "sha256:8e0f8568c118d3077b46be7d654cc8167fa916092e28320cde048e54bfc9f1e6"
             ],
             "index": "pypi",
-            "version": "==2018.5"
+            "version": "==2018.7"
         },
         "requests": {
             "hashes": [
-                "sha256:63b52e3c866428a224f97cab011de738c36aec0185aa91cfacd418b5d58911d1",
-                "sha256:ec22d826a36ed72a7358ff3fe56cbd4ba69dd7a6718ffd450ff0e9df7a47ce6a"
+                "sha256:99dcfdaaeb17caf6e526f32b6a7b780461512ab3f1d992187801694cba42770c",
+                "sha256:a84b8c9ab6239b578f22d1c21d51b696dcfe004032bb80ea832398d6909d7279"
             ],
-            "version": "==2.19.1"
-        },
-        "simplegeneric": {
-            "hashes": [
-                "sha256:dc972e06094b9af5b855b3df4a646395e43d1c9d0d39ed345b7393560d0b9173"
-            ],
-            "version": "==0.8.1"
+            "version": "==2.20.0"
         },
         "six": {
             "hashes": [
@@ -675,27 +681,39 @@
         },
         "sphinx": {
             "hashes": [
-                "sha256:217a7705adcb573da5bbe1e0f5cab4fa0bd89fd9342c9159121746f593c2d5a4",
-                "sha256:a602513f385f1d5785ff1ca420d9c7eb1a1b63381733b2f0ea8188a391314a86"
+                "sha256:652eb8c566f18823a022bb4b6dbc868d366df332a11a0226b5bc3a798a479f17",
+                "sha256:d222626d8356de702431e813a05c68a35967e3d66c6cd1c2c89539bb179a7464"
             ],
             "index": "pypi",
-            "version": "==1.7.9"
+            "version": "==1.8.1"
         },
         "sphinxcontrib-websupport": {
             "hashes": [
                 "sha256:68ca7ff70785cbe1e7bccc71a48b5b6d965d79ca50629606c7861a21b206d9dd",
                 "sha256:9de47f375baf1ea07cdb3436ff39d7a9c76042c10a769c52353ec46e4e8fc3b9"
             ],
-            "markers": "python_version != '3.2.*' and python_version >= '2.7' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.3.*'",
             "version": "==1.1.0"
         },
+        "termcolor": {
+            "hashes": [
+                "sha256:1d6d69ce66211143803fbc56652b41d73b4a400a2891d7bf7a1cdf4c02de613b"
+            ],
+            "version": "==1.1.0"
+        },
+        "toml": {
+            "hashes": [
+                "sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c",
+                "sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e"
+            ],
+            "version": "==0.10.0"
+        },
         "tox": {
             "hashes": [
-                "sha256:37cf240781b662fb790710c6998527e65ca6851eace84d1595ee71f7af4e85f7",
-                "sha256:eb61aa5bcce65325538686f09848f04ef679b5cd9b83cc491272099b28739600"
+                "sha256:513e32fdf2f9e2d583c2f248f47ba9886428c949f068ac54a0469cac55df5862",
+                "sha256:75fa30e8329b41b664585f5fb837e23ce1d7e6fa1f7811f2be571c990f9d911b"
             ],
             "index": "pypi",
-            "version": "==3.2.1"
+            "version": "==3.5.3"
         },
         "traitlets": {
             "hashes": [
@@ -706,19 +724,17 @@
         },
         "urllib3": {
             "hashes": [
-                "sha256:a68ac5e15e76e7e5dd2b8f94007233e01effe3e50e8daddf69acfd81cb686baf",
-                "sha256:b5725a0bd4ba422ab0e66e89e030c806576753ea3ee08554382c14e685d117b5"
+                "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39",
+                "sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22"
             ],
-            "markers": "python_version >= '2.6' and python_version != '3.2.*' and python_version != '3.0.*' and python_version != '3.1.*' and python_version < '4' and python_version != '3.3.*'",
-            "version": "==1.23"
+            "version": "==1.24.1"
         },
         "virtualenv": {
             "hashes": [
-                "sha256:2ce32cd126117ce2c539f0134eb89de91a8413a29baac49cbab3eb50e2026669",
-                "sha256:ca07b4c0b54e14a91af9f34d0919790b016923d157afda5efdde55c96718f752"
+                "sha256:686176c23a538ecc56d27ed9d5217abd34644823d6391cbeb232f42bf722baad",
+                "sha256:f899fafcd92e1150f40c8215328be38ff24b519cd95357fa6e78e006c7638208"
             ],
-            "markers": "python_version >= '2.7' and python_version != '3.0.*' and python_version != '3.2.*' and python_version != '3.1.*'",
-            "version": "==16.0.0"
+            "version": "==16.1.0"
         },
         "wcwidth": {
             "hashes": [
diff --git a/docs/changelog.rst b/docs/changelog.rst
index e98436382..059691811 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -9,6 +9,8 @@ Changelog
 * Fix the ``RecentCorrespondentsFilter`` correspondents filter that was added
   in 2.4 to play nice with the defaults.  Thanks to `tsia`_ and `Sblop`_ who
   pointed this out. `#423`_.
+* Updated dependencies to include (among other things) a security patch to
+  requests.
 
 
 2.5.0
diff --git a/requirements.txt b/requirements.txt
index ac893b8a5..61d98a6d7 100755
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,51 +1,53 @@
 -i https://pypi.python.org/simple
-apipkg==1.5; python_version != '3.3.*'
-atomicwrites==1.2.1; python_version != '3.3.*'
+alabaster==0.7.12
+apipkg==1.5
+atomicwrites==1.2.1
 attrs==18.2.0
-certifi==2018.8.24
+babel==2.6.0
+backcall==0.1.0
+certifi==2018.10.15
 chardet==3.0.4
-coverage==4.5.1; python_version < '4'
-coveralls==1.5.0
-dateparser==0.7.0
-django-cors-headers==2.4.0
-django-crispy-forms==1.7.2
-django-extensions==2.1.2
-django-filter==2.0.0
-django==2.0.8
-djangorestframework==3.8.2
-docopt==0.6.2
-execnet==1.5.0; python_version != '3.3.*'
-factory-boy==2.11.1
-faker==0.9.0; python_version >= '2.7'
-filemagic==1.6
-fuzzywuzzy==0.15.0
-gunicorn==19.9.0
+coverage==4.5.1
+decorator==4.3.0
+docutils==0.14
+execnet==1.5.0
+filelock==3.0.10
 idna==2.7
-inotify-simple==1.1.8; sys_platform == 'linux'
-langdetect==1.0.7
+imagesize==1.1.0
+ipython-genutils==0.2.0
+ipython==7.1.1
+jedi==0.13.1
+jinja2==2.10
+markupsafe==1.0
 more-itertools==4.3.0
-pdftotext==2.1.0
-pillow==5.2.0
-pluggy==0.7.1; python_version != '3.3.*'
-py==1.6.0; python_version != '3.3.*'
+packaging==18.0
+parso==0.3.1
+pexpect==4.6.0 ; sys_platform != 'win32'
+pickleshare==0.7.5
+pluggy==0.8.0
+prompt-toolkit==2.0.7
+ptyprocess==0.6.0
+py==1.7.0
 pycodestyle==2.4.0
-pyocr==0.5.3
+pygments==2.2.0
+pyparsing==2.3.0
 pytest-cov==2.6.0
-pytest-django==3.4.2
+pytest-django==3.4.3
 pytest-env==0.6.2
-pytest-forked==0.2; python_version != '3.3.*'
+pytest-forked==0.2
 pytest-sugar==0.9.1
-pytest-xdist==1.23.0
-pytest==3.8.0
-python-dateutil==2.7.3
-python-dotenv==0.9.1
-python-gnupg==0.4.3
-python-levenshtein==0.12.0
-pytz==2018.5
-regex==2018.8.29
-requests==2.19.1
+pytest-xdist==1.24.0
+pytest==3.9.3
+pytz==2018.7
+requests==2.20.0
 six==1.11.0
+snowballstemmer==1.2.1
+sphinx==1.8.1
+sphinxcontrib-websupport==1.1.0
 termcolor==1.1.0
-text-unidecode==1.2
-tzlocal==1.5.1
-urllib3==1.23; python_version != '3.3.*'
+toml==0.10.0
+tox==3.5.3
+traitlets==4.3.2
+urllib3==1.24.1
+virtualenv==16.1.0
+wcwidth==0.1.7

From 9ab50ed09dd60893144d86afadd6279c1970dbf7 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sat, 3 Nov 2018 13:29:22 +0000
Subject: [PATCH 46/47] Fix requiremnts.txt

---
 Pipfile          |   4 +-
 Pipfile.lock     | 561 ++++++++++++++++++++---------------------------
 requirements.txt |  26 +++
 3 files changed, 264 insertions(+), 327 deletions(-)

diff --git a/Pipfile b/Pipfile
index 778d15546..12bae4b66 100644
--- a/Pipfile
+++ b/Pipfile
@@ -25,8 +25,6 @@ python-dateutil = "*"
 python-dotenv = "*"
 python-gnupg = "*"
 pytz = "*"
-
-[dev-packages]
 ipython = "*"
 sphinx = "*"
 tox = "*"
@@ -37,3 +35,5 @@ pytest-django = "*"
 pytest-sugar = "*"
 pytest-env = "*"
 pytest-xdist = "*"
+
+[dev-packages]
diff --git a/Pipfile.lock b/Pipfile.lock
index 966fbd467..96dec448b 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "eeeeaf6ecb0cec45a3962eda6647b5263e5ad7939fae29d4b294f59ffe9ca3dd"
+            "sha256": "3782f7e6b5461c39c8fd0d0048a4622418f247439113bd3cdc91712fd47036f6"
         },
         "pipfile-spec": 6,
         "requires": {},
@@ -14,329 +14,6 @@
         ]
     },
     "default": {
-        "certifi": {
-            "hashes": [
-                "sha256:339dc09518b07e2fa7eda5450740925974815557727d6bd35d319c1524a04a4c",
-                "sha256:6d58c986d22b038c8c0df30d639f23a3e6d172a05c3583e766f4c0b785c0986a"
-            ],
-            "version": "==2018.10.15"
-        },
-        "chardet": {
-            "hashes": [
-                "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
-                "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
-            ],
-            "version": "==3.0.4"
-        },
-        "coverage": {
-            "hashes": [
-                "sha256:03481e81d558d30d230bc12999e3edffe392d244349a90f4ef9b88425fac74ba",
-                "sha256:0b136648de27201056c1869a6c0d4e23f464750fd9a9ba9750b8336a244429ed",
-                "sha256:0bf8cbbd71adfff0ef1f3a1531e6402d13b7b01ac50a79c97ca15f030dba6306",
-                "sha256:10a46017fef60e16694a30627319f38a2b9b52e90182dddb6e37dcdab0f4bf95",
-                "sha256:198626739a79b09fa0a2f06e083ffd12eb55449b5f8bfdbeed1df4910b2ca640",
-                "sha256:23d341cdd4a0371820eb2b0bd6b88f5003a7438bbedb33688cd33b8eae59affd",
-                "sha256:28b2191e7283f4f3568962e373b47ef7f0392993bb6660d079c62bd50fe9d162",
-                "sha256:2a5b73210bad5279ddb558d9a2bfedc7f4bf6ad7f3c988641d83c40293deaec1",
-                "sha256:2eb564bbf7816a9d68dd3369a510be3327f1c618d2357fa6b1216994c2e3d508",
-                "sha256:337ded681dd2ef9ca04ef5d93cfc87e52e09db2594c296b4a0a3662cb1b41249",
-                "sha256:3a2184c6d797a125dca8367878d3b9a178b6fdd05fdc2d35d758c3006a1cd694",
-                "sha256:3c79a6f7b95751cdebcd9037e4d06f8d5a9b60e4ed0cd231342aa8ad7124882a",
-                "sha256:3d72c20bd105022d29b14a7d628462ebdc61de2f303322c0212a054352f3b287",
-                "sha256:3eb42bf89a6be7deb64116dd1cc4b08171734d721e7a7e57ad64cc4ef29ed2f1",
-                "sha256:4635a184d0bbe537aa185a34193898eee409332a8ccb27eea36f262566585000",
-                "sha256:56e448f051a201c5ebbaa86a5efd0ca90d327204d8b059ab25ad0f35fbfd79f1",
-                "sha256:5a13ea7911ff5e1796b6d5e4fbbf6952381a611209b736d48e675c2756f3f74e",
-                "sha256:69bf008a06b76619d3c3f3b1983f5145c75a305a0fea513aca094cae5c40a8f5",
-                "sha256:6bc583dc18d5979dc0f6cec26a8603129de0304d5ae1f17e57a12834e7235062",
-                "sha256:701cd6093d63e6b8ad7009d8a92425428bc4d6e7ab8d75efbb665c806c1d79ba",
-                "sha256:7608a3dd5d73cb06c531b8925e0ef8d3de31fed2544a7de6c63960a1e73ea4bc",
-                "sha256:76ecd006d1d8f739430ec50cc872889af1f9c1b6b8f48e29941814b09b0fd3cc",
-                "sha256:7aa36d2b844a3e4a4b356708d79fd2c260281a7390d678a10b91ca595ddc9e99",
-                "sha256:7d3f553904b0c5c016d1dad058a7554c7ac4c91a789fca496e7d8347ad040653",
-                "sha256:7e1fe19bd6dce69d9fd159d8e4a80a8f52101380d5d3a4d374b6d3eae0e5de9c",
-                "sha256:8c3cb8c35ec4d9506979b4cf90ee9918bc2e49f84189d9bf5c36c0c1119c6558",
-                "sha256:9d6dd10d49e01571bf6e147d3b505141ffc093a06756c60b053a859cb2128b1f",
-                "sha256:be6cfcd8053d13f5f5eeb284aa8a814220c3da1b0078fa859011c7fffd86dab9",
-                "sha256:c1bb572fab8208c400adaf06a8133ac0712179a334c09224fb11393e920abcdd",
-                "sha256:de4418dadaa1c01d497e539210cb6baa015965526ff5afc078c57ca69160108d",
-                "sha256:e05cb4d9aad6233d67e0541caa7e511fa4047ed7750ec2510d466e806e0255d6",
-                "sha256:f05a636b4564104120111800021a92e43397bc12a5c72fed7036be8556e0029e",
-                "sha256:f3f501f345f24383c0000395b26b726e46758b71393267aeae0bd36f8b3ade80"
-            ],
-            "version": "==4.5.1"
-        },
-        "coveralls": {
-            "hashes": [
-                "sha256:ab638e88d38916a6cedbf80a9cd8992d5fa55c77ab755e262e00b36792b7cd6d",
-                "sha256:b2388747e2529fa4c669fb1e3e2756e4e07b6ee56c7d9fce05f35ccccc913aa0"
-            ],
-            "index": "pypi",
-            "version": "==1.5.1"
-        },
-        "dateparser": {
-            "hashes": [
-                "sha256:940828183c937bcec530753211b70f673c0a9aab831e43273489b310538dff86",
-                "sha256:b452ef8b36cd78ae86a50721794bc674aa3994e19b570f7ba92810f4e0a2ae03"
-            ],
-            "index": "pypi",
-            "version": "==0.7.0"
-        },
-        "django": {
-            "hashes": [
-                "sha256:25df265e1fdb74f7e7305a1de620a84681bcc9c05e84a3ed97e4a1a63024f18d",
-                "sha256:d6d94554abc82ca37e447c3d28958f5ac39bd7d4adaa285543ae97fb1129fd69"
-            ],
-            "index": "pypi",
-            "version": "==2.0.9"
-        },
-        "django-cors-headers": {
-            "hashes": [
-                "sha256:5545009c9b233ea7e70da7dbab7cb1c12afa01279895086f98ec243d7eab46fa",
-                "sha256:c4c2ee97139d18541a1be7d96fe337d1694623816d83f53cb7c00da9b94acae1"
-            ],
-            "index": "pypi",
-            "version": "==2.4.0"
-        },
-        "django-crispy-forms": {
-            "hashes": [
-                "sha256:5952bab971110d0b86c278132dae0aa095beee8f723e625c3d3fa28888f1675f",
-                "sha256:705ededc554ad8736157c666681165fe22ead2dec0d5446d65fc9dd976a5a876"
-            ],
-            "index": "pypi",
-            "version": "==1.7.2"
-        },
-        "django-extensions": {
-            "hashes": [
-                "sha256:30cb6a8c7d6f75a55edf0c0c4491bd98f8264ae1616ce105f9cecac4387edd07",
-                "sha256:4ad86a7a5e84f1c77db030761ae87a600647250c652030a2b71a16e87f3a3d62"
-            ],
-            "index": "pypi",
-            "version": "==2.1.3"
-        },
-        "django-filter": {
-            "hashes": [
-                "sha256:6f4e4bc1a11151178520567b50320e5c32f8edb552139d93ea3e30613b886f56",
-                "sha256:86c3925020c27d072cdae7b828aaa5d165c2032a629abbe3c3a1be1edae61c58"
-            ],
-            "index": "pypi",
-            "version": "==2.0.0"
-        },
-        "djangorestframework": {
-            "hashes": [
-                "sha256:607865b0bb1598b153793892101d881466bd5a991de12bd6229abb18b1c86136",
-                "sha256:63f76cbe1e7d12b94c357d7e54401103b2e52aef0f7c1650d6c820ad708776e5"
-            ],
-            "index": "pypi",
-            "version": "==3.9.0"
-        },
-        "docopt": {
-            "hashes": [
-                "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491"
-            ],
-            "version": "==0.6.2"
-        },
-        "factory-boy": {
-            "hashes": [
-                "sha256:6f25cc4761ac109efd503f096e2ad99421b1159f01a29dbb917359dcd68e08ca",
-                "sha256:d552cb872b310ae78bd7429bf318e42e1e903b1a109e899a523293dfa762ea4f"
-            ],
-            "index": "pypi",
-            "version": "==2.11.1"
-        },
-        "faker": {
-            "hashes": [
-                "sha256:2621643b80a10b91999925cfd20f64d2b36f20bf22136bbdc749bb57d6ffe124",
-                "sha256:5ed822d31bd2d6edf10944d176d30dc9c886afdd381eefb7ba8b7aad86171646"
-            ],
-            "version": "==0.9.2"
-        },
-        "filemagic": {
-            "hashes": [
-                "sha256:e684359ef40820fe406f0ebc5bf8a78f89717bdb7fed688af68082d991d6dbf3"
-            ],
-            "index": "pypi",
-            "version": "==1.6"
-        },
-        "fuzzywuzzy": {
-            "extras": [
-                "speedup"
-            ],
-            "hashes": [
-                "sha256:3759bc6859daa0eecef8c82b45404bdac20c23f23136cf4c18b46b426bbc418f",
-                "sha256:5b36957ccf836e700f4468324fa80ba208990385392e217be077d5cd738ae602"
-            ],
-            "index": "pypi",
-            "version": "==0.15.0"
-        },
-        "gunicorn": {
-            "hashes": [
-                "sha256:aa8e0b40b4157b36a5df5e599f45c9c76d6af43845ba3b3b0efe2c70473c2471",
-                "sha256:fa2662097c66f920f53f70621c6c58ca4a3c4d3434205e608e121b5b3b71f4f3"
-            ],
-            "index": "pypi",
-            "version": "==19.9.0"
-        },
-        "idna": {
-            "hashes": [
-                "sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e",
-                "sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16"
-            ],
-            "version": "==2.7"
-        },
-        "inotify-simple": {
-            "hashes": [
-                "sha256:fc2c10dd73278a1027d0663f2db51240af5946390f363a154361406ebdddd8dd"
-            ],
-            "index": "pypi",
-            "version": "==1.1.8"
-        },
-        "langdetect": {
-            "hashes": [
-                "sha256:91a170d5f0ade380db809b3ba67f08e95fe6c6c8641f96d67a51ff7e98a9bf30"
-            ],
-            "index": "pypi",
-            "version": "==1.0.7"
-        },
-        "pdftotext": {
-            "hashes": [
-                "sha256:e3ad11efe0aa22cbfc46aa1296b2ea5a52ad208b778288311f2801adef178ccb"
-            ],
-            "index": "pypi",
-            "version": "==2.1.1"
-        },
-        "pillow": {
-            "hashes": [
-                "sha256:00203f406818c3f45d47bb8fe7e67d3feddb8dcbbd45a289a1de7dd789226360",
-                "sha256:0616f800f348664e694dddb0b0c88d26761dd5e9f34e1ed7b7a7d2da14b40cb7",
-                "sha256:1f7908aab90c92ad85af9d2fec5fc79456a89b3adcc26314d2cde0e238bd789e",
-                "sha256:2ea3517cd5779843de8a759c2349a3cd8d3893e03ab47053b66d5ec6f8bc4f93",
-                "sha256:48a9f0538c91fc136b3a576bee0e7cd174773dc9920b310c21dcb5519722e82c",
-                "sha256:5280ebc42641a1283b7b1f2c20e5b936692198b9dd9995527c18b794850be1a8",
-                "sha256:5e34e4b5764af65551647f5cc67cf5198c1d05621781d5173b342e5e55bf023b",
-                "sha256:63b120421ab85cad909792583f83b6ca3584610c2fe70751e23f606a3c2e87f0",
-                "sha256:696b5e0109fe368d0057f484e2e91717b49a03f1e310f857f133a4acec9f91dd",
-                "sha256:870ed021a42b1b02b5fe4a739ea735f671a84128c0a666c705db2cb9abd528eb",
-                "sha256:916da1c19e4012d06a372127d7140dae894806fad67ef44330e5600d77833581",
-                "sha256:9303a289fa0811e1c6abd9ddebfc770556d7c3311cb2b32eff72164ddc49bc64",
-                "sha256:9577888ecc0ad7d06c3746afaba339c94d62b59da16f7a5d1cff9e491f23dace",
-                "sha256:987e1c94a33c93d9b209315bfda9faa54b8edfce6438a1e93ae866ba20de5956",
-                "sha256:99a3bbdbb844f4fb5d6dd59fac836a40749781c1fa63c563bc216c27aef63f60",
-                "sha256:99db8dc3097ceafbcff9cb2bff384b974795edeb11d167d391a02c7bfeeb6e16",
-                "sha256:a5a96cf49eb580756a44ecf12949e52f211e20bffbf5a95760ac14b1e499cd37",
-                "sha256:aa6ca3eb56704cdc0d876fc6047ffd5ee960caad52452fbee0f99908a141a0ae",
-                "sha256:aade5e66795c94e4a2b2624affeea8979648d1b0ae3fcee17e74e2c647fc4a8a",
-                "sha256:b78905860336c1d292409e3df6ad39cc1f1c7f0964e66844bbc2ebfca434d073",
-                "sha256:b92f521cdc4e4a3041cc343625b699f20b0b5f976793fb45681aac1efda565f8",
-                "sha256:bfde84bbd6ae5f782206d454b67b7ee8f7f818c29b99fd02bf022fd33bab14cb",
-                "sha256:c2b62d3df80e694c0e4a0ed47754c9480521e25642251b3ab1dff050a4e60409",
-                "sha256:c5e2be6c263b64f6f7656e23e18a4a9980cffc671442795682e8c4e4f815dd9f",
-                "sha256:c99aa3c63104e0818ec566f8ff3942fb7c7a8f35f9912cb63fd8e12318b214b2",
-                "sha256:dae06620d3978da346375ebf88b9e2dd7d151335ba668c995aea9ed07af7add4",
-                "sha256:db5499d0710823fa4fb88206050d46544e8f0e0136a9a5f5570b026584c8fd74",
-                "sha256:f36baafd82119c4a114b9518202f2a983819101dcc14b26e43fc12cbefdce00e",
-                "sha256:f52b79c8796d81391ab295b04e520bda6feed54d54931708872e8f9ae9db0ea1",
-                "sha256:ff8cff01582fa1a7e533cb97f628531c4014af4b5f38e33cdcfe5eec29b6d888"
-            ],
-            "index": "pypi",
-            "version": "==5.3.0"
-        },
-        "pyocr": {
-            "hashes": [
-                "sha256:b6ba6263fd92da56627dff6d263d991a2246aacd117d1788f11b93f419ca395f"
-            ],
-            "index": "pypi",
-            "version": "==0.5.3"
-        },
-        "python-dateutil": {
-            "hashes": [
-                "sha256:063df5763652e21de43de7d9e00ccf239f953a832941e37be541614732cdfc93",
-                "sha256:88f9287c0174266bb0d8cedd395cfba9c58e87e5ad86b2ce58859bc11be3cf02"
-            ],
-            "index": "pypi",
-            "version": "==2.7.5"
-        },
-        "python-dotenv": {
-            "hashes": [
-                "sha256:122290a38ece9fe4f162dc7c95cae3357b983505830a154d3c98ef7f6c6cea77",
-                "sha256:4a205787bc829233de2a823aa328e44fd9996fedb954989a21f1fc67c13d7a77"
-            ],
-            "index": "pypi",
-            "version": "==0.9.1"
-        },
-        "python-gnupg": {
-            "hashes": [
-                "sha256:2d158dfc6b54927752b945ebe57e6a0c45da27747fa3b9ae66eccc0d2147ac0d",
-                "sha256:faa69bab58ed0936f0ccf96c99b92369b7a1819305d37dfe5c927d21a437a09d"
-            ],
-            "index": "pypi",
-            "version": "==0.4.3"
-        },
-        "python-levenshtein": {
-            "hashes": [
-                "sha256:033a11de5e3d19ea25c9302d11224e1a1898fe5abd23c61c7c360c25195e3eb1"
-            ],
-            "markers": "extra == 'speedup'",
-            "version": "==0.12.0"
-        },
-        "pytz": {
-            "hashes": [
-                "sha256:31cb35c89bd7d333cd32c5f278fca91b523b0834369e757f4c5641ea252236ca",
-                "sha256:8e0f8568c118d3077b46be7d654cc8167fa916092e28320cde048e54bfc9f1e6"
-            ],
-            "index": "pypi",
-            "version": "==2018.7"
-        },
-        "regex": {
-            "hashes": [
-                "sha256:0ef96690c3d2294155b7d44187ca4a151e45c931cb768e106ba464a9fa64c5da",
-                "sha256:251683e01a3bcacd9188acf0d4caf7b29a3b963c843159311825613ae144cddb",
-                "sha256:3fe15a75fe00f04d1ec16713d55cf1e206077c450267a10b33318756fb8b3f99",
-                "sha256:53a962f9dc28cdf403978a142cb1e054479759ad64d312a999f9f042c25b5c9a",
-                "sha256:8bd1da6a93d32336a5e5432886dd8543004f0591c39b83dbfa60705cccdf414d",
-                "sha256:b5423061918f602e9342b54d746ac31c598d328ecaf4ef0618763e960c926fd4",
-                "sha256:d80ebc65b1f7d0403117f59309c16eac24be6a0bc730b593a79f703462858d94",
-                "sha256:fd8419979639b7de7fb964a13bce3ac47e6fe33043b83de0398c3067986e5659",
-                "sha256:ff2f15b2b0b4b58ba8a1de651780a0d3fd54f96ad6b77dceb77695220e5d7b7a"
-            ],
-            "version": "==2018.11.2"
-        },
-        "requests": {
-            "hashes": [
-                "sha256:99dcfdaaeb17caf6e526f32b6a7b780461512ab3f1d992187801694cba42770c",
-                "sha256:a84b8c9ab6239b578f22d1c21d51b696dcfe004032bb80ea832398d6909d7279"
-            ],
-            "version": "==2.20.0"
-        },
-        "six": {
-            "hashes": [
-                "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9",
-                "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb"
-            ],
-            "version": "==1.11.0"
-        },
-        "text-unidecode": {
-            "hashes": [
-                "sha256:5a1375bb2ba7968740508ae38d92e1f889a0832913cb1c447d5e2046061a396d",
-                "sha256:801e38bd550b943563660a91de8d4b6fa5df60a542be9093f7abf819f86050cc"
-            ],
-            "version": "==1.2"
-        },
-        "tzlocal": {
-            "hashes": [
-                "sha256:4ebeb848845ac898da6519b9b31879cf13b6626f7184c496037b818e238f2c4e"
-            ],
-            "version": "==1.5.1"
-        },
-        "urllib3": {
-            "hashes": [
-                "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39",
-                "sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22"
-            ],
-            "version": "==1.24.1"
-        }
-    },
-    "develop": {
         "alabaster": {
             "hashes": [
                 "sha256:446438bdcca0e05bd45ea2de1668c1d9b032e1a9154c2c259092d77031ddd359",
@@ -431,6 +108,22 @@
             ],
             "version": "==4.5.1"
         },
+        "coveralls": {
+            "hashes": [
+                "sha256:ab638e88d38916a6cedbf80a9cd8992d5fa55c77ab755e262e00b36792b7cd6d",
+                "sha256:b2388747e2529fa4c669fb1e3e2756e4e07b6ee56c7d9fce05f35ccccc913aa0"
+            ],
+            "index": "pypi",
+            "version": "==1.5.1"
+        },
+        "dateparser": {
+            "hashes": [
+                "sha256:940828183c937bcec530753211b70f673c0a9aab831e43273489b310538dff86",
+                "sha256:b452ef8b36cd78ae86a50721794bc674aa3994e19b570f7ba92810f4e0a2ae03"
+            ],
+            "index": "pypi",
+            "version": "==0.7.0"
+        },
         "decorator": {
             "hashes": [
                 "sha256:2c51dff8ef3c447388fe5e4453d24a2bf128d3a4c32af3fabef1f01c6851ab82",
@@ -438,6 +131,60 @@
             ],
             "version": "==4.3.0"
         },
+        "django": {
+            "hashes": [
+                "sha256:25df265e1fdb74f7e7305a1de620a84681bcc9c05e84a3ed97e4a1a63024f18d",
+                "sha256:d6d94554abc82ca37e447c3d28958f5ac39bd7d4adaa285543ae97fb1129fd69"
+            ],
+            "index": "pypi",
+            "version": "==2.0.9"
+        },
+        "django-cors-headers": {
+            "hashes": [
+                "sha256:5545009c9b233ea7e70da7dbab7cb1c12afa01279895086f98ec243d7eab46fa",
+                "sha256:c4c2ee97139d18541a1be7d96fe337d1694623816d83f53cb7c00da9b94acae1"
+            ],
+            "index": "pypi",
+            "version": "==2.4.0"
+        },
+        "django-crispy-forms": {
+            "hashes": [
+                "sha256:5952bab971110d0b86c278132dae0aa095beee8f723e625c3d3fa28888f1675f",
+                "sha256:705ededc554ad8736157c666681165fe22ead2dec0d5446d65fc9dd976a5a876"
+            ],
+            "index": "pypi",
+            "version": "==1.7.2"
+        },
+        "django-extensions": {
+            "hashes": [
+                "sha256:30cb6a8c7d6f75a55edf0c0c4491bd98f8264ae1616ce105f9cecac4387edd07",
+                "sha256:4ad86a7a5e84f1c77db030761ae87a600647250c652030a2b71a16e87f3a3d62"
+            ],
+            "index": "pypi",
+            "version": "==2.1.3"
+        },
+        "django-filter": {
+            "hashes": [
+                "sha256:6f4e4bc1a11151178520567b50320e5c32f8edb552139d93ea3e30613b886f56",
+                "sha256:86c3925020c27d072cdae7b828aaa5d165c2032a629abbe3c3a1be1edae61c58"
+            ],
+            "index": "pypi",
+            "version": "==2.0.0"
+        },
+        "djangorestframework": {
+            "hashes": [
+                "sha256:607865b0bb1598b153793892101d881466bd5a991de12bd6229abb18b1c86136",
+                "sha256:63f76cbe1e7d12b94c357d7e54401103b2e52aef0f7c1650d6c820ad708776e5"
+            ],
+            "index": "pypi",
+            "version": "==3.9.0"
+        },
+        "docopt": {
+            "hashes": [
+                "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491"
+            ],
+            "version": "==0.6.2"
+        },
         "docutils": {
             "hashes": [
                 "sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6",
@@ -453,6 +200,21 @@
             ],
             "version": "==1.5.0"
         },
+        "factory-boy": {
+            "hashes": [
+                "sha256:6f25cc4761ac109efd503f096e2ad99421b1159f01a29dbb917359dcd68e08ca",
+                "sha256:d552cb872b310ae78bd7429bf318e42e1e903b1a109e899a523293dfa762ea4f"
+            ],
+            "index": "pypi",
+            "version": "==2.11.1"
+        },
+        "faker": {
+            "hashes": [
+                "sha256:2621643b80a10b91999925cfd20f64d2b36f20bf22136bbdc749bb57d6ffe124",
+                "sha256:5ed822d31bd2d6edf10944d176d30dc9c886afdd381eefb7ba8b7aad86171646"
+            ],
+            "version": "==0.9.2"
+        },
         "filelock": {
             "hashes": [
                 "sha256:b8d5ca5ca1c815e1574aee746650ea7301de63d87935b3463d26368b76e31633",
@@ -460,6 +222,32 @@
             ],
             "version": "==3.0.10"
         },
+        "filemagic": {
+            "hashes": [
+                "sha256:e684359ef40820fe406f0ebc5bf8a78f89717bdb7fed688af68082d991d6dbf3"
+            ],
+            "index": "pypi",
+            "version": "==1.6"
+        },
+        "fuzzywuzzy": {
+            "extras": [
+                "speedup"
+            ],
+            "hashes": [
+                "sha256:3759bc6859daa0eecef8c82b45404bdac20c23f23136cf4c18b46b426bbc418f",
+                "sha256:5b36957ccf836e700f4468324fa80ba208990385392e217be077d5cd738ae602"
+            ],
+            "index": "pypi",
+            "version": "==0.15.0"
+        },
+        "gunicorn": {
+            "hashes": [
+                "sha256:aa8e0b40b4157b36a5df5e599f45c9c76d6af43845ba3b3b0efe2c70473c2471",
+                "sha256:fa2662097c66f920f53f70621c6c58ca4a3c4d3434205e608e121b5b3b71f4f3"
+            ],
+            "index": "pypi",
+            "version": "==19.9.0"
+        },
         "idna": {
             "hashes": [
                 "sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e",
@@ -474,6 +262,13 @@
             ],
             "version": "==1.1.0"
         },
+        "inotify-simple": {
+            "hashes": [
+                "sha256:fc2c10dd73278a1027d0663f2db51240af5946390f363a154361406ebdddd8dd"
+            ],
+            "index": "pypi",
+            "version": "==1.1.8"
+        },
         "ipython": {
             "hashes": [
                 "sha256:a5781d6934a3341a1f9acb4ea5acdc7ea0a0855e689dbe755d070ca51e995435",
@@ -503,6 +298,13 @@
             ],
             "version": "==2.10"
         },
+        "langdetect": {
+            "hashes": [
+                "sha256:91a170d5f0ade380db809b3ba67f08e95fe6c6c8641f96d67a51ff7e98a9bf30"
+            ],
+            "index": "pypi",
+            "version": "==1.0.7"
+        },
         "markupsafe": {
             "hashes": [
                 "sha256:a6be69091dac236ea9c6bc7d012beab42010fa914c459791d627dad4910eb665"
@@ -531,6 +333,13 @@
             ],
             "version": "==0.3.1"
         },
+        "pdftotext": {
+            "hashes": [
+                "sha256:e3ad11efe0aa22cbfc46aa1296b2ea5a52ad208b778288311f2801adef178ccb"
+            ],
+            "index": "pypi",
+            "version": "==2.1.1"
+        },
         "pexpect": {
             "hashes": [
                 "sha256:2a8e88259839571d1251d278476f3eec5db26deb73a70be5ed5dc5435e418aba",
@@ -546,6 +355,42 @@
             ],
             "version": "==0.7.5"
         },
+        "pillow": {
+            "hashes": [
+                "sha256:00203f406818c3f45d47bb8fe7e67d3feddb8dcbbd45a289a1de7dd789226360",
+                "sha256:0616f800f348664e694dddb0b0c88d26761dd5e9f34e1ed7b7a7d2da14b40cb7",
+                "sha256:1f7908aab90c92ad85af9d2fec5fc79456a89b3adcc26314d2cde0e238bd789e",
+                "sha256:2ea3517cd5779843de8a759c2349a3cd8d3893e03ab47053b66d5ec6f8bc4f93",
+                "sha256:48a9f0538c91fc136b3a576bee0e7cd174773dc9920b310c21dcb5519722e82c",
+                "sha256:5280ebc42641a1283b7b1f2c20e5b936692198b9dd9995527c18b794850be1a8",
+                "sha256:5e34e4b5764af65551647f5cc67cf5198c1d05621781d5173b342e5e55bf023b",
+                "sha256:63b120421ab85cad909792583f83b6ca3584610c2fe70751e23f606a3c2e87f0",
+                "sha256:696b5e0109fe368d0057f484e2e91717b49a03f1e310f857f133a4acec9f91dd",
+                "sha256:870ed021a42b1b02b5fe4a739ea735f671a84128c0a666c705db2cb9abd528eb",
+                "sha256:916da1c19e4012d06a372127d7140dae894806fad67ef44330e5600d77833581",
+                "sha256:9303a289fa0811e1c6abd9ddebfc770556d7c3311cb2b32eff72164ddc49bc64",
+                "sha256:9577888ecc0ad7d06c3746afaba339c94d62b59da16f7a5d1cff9e491f23dace",
+                "sha256:987e1c94a33c93d9b209315bfda9faa54b8edfce6438a1e93ae866ba20de5956",
+                "sha256:99a3bbdbb844f4fb5d6dd59fac836a40749781c1fa63c563bc216c27aef63f60",
+                "sha256:99db8dc3097ceafbcff9cb2bff384b974795edeb11d167d391a02c7bfeeb6e16",
+                "sha256:a5a96cf49eb580756a44ecf12949e52f211e20bffbf5a95760ac14b1e499cd37",
+                "sha256:aa6ca3eb56704cdc0d876fc6047ffd5ee960caad52452fbee0f99908a141a0ae",
+                "sha256:aade5e66795c94e4a2b2624affeea8979648d1b0ae3fcee17e74e2c647fc4a8a",
+                "sha256:b78905860336c1d292409e3df6ad39cc1f1c7f0964e66844bbc2ebfca434d073",
+                "sha256:b92f521cdc4e4a3041cc343625b699f20b0b5f976793fb45681aac1efda565f8",
+                "sha256:bfde84bbd6ae5f782206d454b67b7ee8f7f818c29b99fd02bf022fd33bab14cb",
+                "sha256:c2b62d3df80e694c0e4a0ed47754c9480521e25642251b3ab1dff050a4e60409",
+                "sha256:c5e2be6c263b64f6f7656e23e18a4a9980cffc671442795682e8c4e4f815dd9f",
+                "sha256:c99aa3c63104e0818ec566f8ff3942fb7c7a8f35f9912cb63fd8e12318b214b2",
+                "sha256:dae06620d3978da346375ebf88b9e2dd7d151335ba668c995aea9ed07af7add4",
+                "sha256:db5499d0710823fa4fb88206050d46544e8f0e0136a9a5f5570b026584c8fd74",
+                "sha256:f36baafd82119c4a114b9518202f2a983819101dcc14b26e43fc12cbefdce00e",
+                "sha256:f52b79c8796d81391ab295b04e520bda6feed54d54931708872e8f9ae9db0ea1",
+                "sha256:ff8cff01582fa1a7e533cb97f628531c4014af4b5f38e33cdcfe5eec29b6d888"
+            ],
+            "index": "pypi",
+            "version": "==5.3.0"
+        },
         "pluggy": {
             "hashes": [
                 "sha256:447ba94990e8014ee25ec853339faf7b0fc8050cdc3289d4d71f7f410fb90095",
@@ -590,6 +435,13 @@
             ],
             "version": "==2.2.0"
         },
+        "pyocr": {
+            "hashes": [
+                "sha256:b6ba6263fd92da56627dff6d263d991a2246aacd117d1788f11b93f419ca395f"
+            ],
+            "index": "pypi",
+            "version": "==0.5.3"
+        },
         "pyparsing": {
             "hashes": [
                 "sha256:40856e74d4987de5d01761a22d1621ae1c7f8774585acae358aa5c5936c6c90b",
@@ -650,6 +502,37 @@
             "index": "pypi",
             "version": "==1.24.0"
         },
+        "python-dateutil": {
+            "hashes": [
+                "sha256:063df5763652e21de43de7d9e00ccf239f953a832941e37be541614732cdfc93",
+                "sha256:88f9287c0174266bb0d8cedd395cfba9c58e87e5ad86b2ce58859bc11be3cf02"
+            ],
+            "index": "pypi",
+            "version": "==2.7.5"
+        },
+        "python-dotenv": {
+            "hashes": [
+                "sha256:122290a38ece9fe4f162dc7c95cae3357b983505830a154d3c98ef7f6c6cea77",
+                "sha256:4a205787bc829233de2a823aa328e44fd9996fedb954989a21f1fc67c13d7a77"
+            ],
+            "index": "pypi",
+            "version": "==0.9.1"
+        },
+        "python-gnupg": {
+            "hashes": [
+                "sha256:2d158dfc6b54927752b945ebe57e6a0c45da27747fa3b9ae66eccc0d2147ac0d",
+                "sha256:faa69bab58ed0936f0ccf96c99b92369b7a1819305d37dfe5c927d21a437a09d"
+            ],
+            "index": "pypi",
+            "version": "==0.4.3"
+        },
+        "python-levenshtein": {
+            "hashes": [
+                "sha256:033a11de5e3d19ea25c9302d11224e1a1898fe5abd23c61c7c360c25195e3eb1"
+            ],
+            "markers": "extra == 'speedup'",
+            "version": "==0.12.0"
+        },
         "pytz": {
             "hashes": [
                 "sha256:31cb35c89bd7d333cd32c5f278fca91b523b0834369e757f4c5641ea252236ca",
@@ -658,6 +541,20 @@
             "index": "pypi",
             "version": "==2018.7"
         },
+        "regex": {
+            "hashes": [
+                "sha256:0ef96690c3d2294155b7d44187ca4a151e45c931cb768e106ba464a9fa64c5da",
+                "sha256:251683e01a3bcacd9188acf0d4caf7b29a3b963c843159311825613ae144cddb",
+                "sha256:3fe15a75fe00f04d1ec16713d55cf1e206077c450267a10b33318756fb8b3f99",
+                "sha256:53a962f9dc28cdf403978a142cb1e054479759ad64d312a999f9f042c25b5c9a",
+                "sha256:8bd1da6a93d32336a5e5432886dd8543004f0591c39b83dbfa60705cccdf414d",
+                "sha256:b5423061918f602e9342b54d746ac31c598d328ecaf4ef0618763e960c926fd4",
+                "sha256:d80ebc65b1f7d0403117f59309c16eac24be6a0bc730b593a79f703462858d94",
+                "sha256:fd8419979639b7de7fb964a13bce3ac47e6fe33043b83de0398c3067986e5659",
+                "sha256:ff2f15b2b0b4b58ba8a1de651780a0d3fd54f96ad6b77dceb77695220e5d7b7a"
+            ],
+            "version": "==2018.11.2"
+        },
         "requests": {
             "hashes": [
                 "sha256:99dcfdaaeb17caf6e526f32b6a7b780461512ab3f1d992187801694cba42770c",
@@ -700,6 +597,13 @@
             ],
             "version": "==1.1.0"
         },
+        "text-unidecode": {
+            "hashes": [
+                "sha256:5a1375bb2ba7968740508ae38d92e1f889a0832913cb1c447d5e2046061a396d",
+                "sha256:801e38bd550b943563660a91de8d4b6fa5df60a542be9093f7abf819f86050cc"
+            ],
+            "version": "==1.2"
+        },
         "toml": {
             "hashes": [
                 "sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c",
@@ -722,6 +626,12 @@
             ],
             "version": "==4.3.2"
         },
+        "tzlocal": {
+            "hashes": [
+                "sha256:4ebeb848845ac898da6519b9b31879cf13b6626f7184c496037b818e238f2c4e"
+            ],
+            "version": "==1.5.1"
+        },
         "urllib3": {
             "hashes": [
                 "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39",
@@ -743,5 +653,6 @@
             ],
             "version": "==0.1.7"
         }
-    }
+    },
+    "develop": {}
 }
diff --git a/requirements.txt b/requirements.txt
index 61d98a6d7..a62185562 100755
--- a/requirements.txt
+++ b/requirements.txt
@@ -8,28 +8,47 @@ backcall==0.1.0
 certifi==2018.10.15
 chardet==3.0.4
 coverage==4.5.1
+coveralls==1.5.1
+dateparser==0.7.0
 decorator==4.3.0
+django-cors-headers==2.4.0
+django-crispy-forms==1.7.2
+django-extensions==2.1.3
+django-filter==2.0.0
+django==2.0.9
+djangorestframework==3.9.0
+docopt==0.6.2
 docutils==0.14
 execnet==1.5.0
+factory-boy==2.11.1
+faker==0.9.2
 filelock==3.0.10
+filemagic==1.6
+fuzzywuzzy[speedup]==0.15.0
+gunicorn==19.9.0
 idna==2.7
 imagesize==1.1.0
+inotify-simple==1.1.8
 ipython-genutils==0.2.0
 ipython==7.1.1
 jedi==0.13.1
 jinja2==2.10
+langdetect==1.0.7
 markupsafe==1.0
 more-itertools==4.3.0
 packaging==18.0
 parso==0.3.1
+pdftotext==2.1.1
 pexpect==4.6.0 ; sys_platform != 'win32'
 pickleshare==0.7.5
+pillow==5.3.0
 pluggy==0.8.0
 prompt-toolkit==2.0.7
 ptyprocess==0.6.0
 py==1.7.0
 pycodestyle==2.4.0
 pygments==2.2.0
+pyocr==0.5.3
 pyparsing==2.3.0
 pytest-cov==2.6.0
 pytest-django==3.4.3
@@ -38,16 +57,23 @@ pytest-forked==0.2
 pytest-sugar==0.9.1
 pytest-xdist==1.24.0
 pytest==3.9.3
+python-dateutil==2.7.5
+python-dotenv==0.9.1
+python-gnupg==0.4.3
+python-levenshtein==0.12.0 ; extra == 'speedup'
 pytz==2018.7
+regex==2018.11.2
 requests==2.20.0
 six==1.11.0
 snowballstemmer==1.2.1
 sphinx==1.8.1
 sphinxcontrib-websupport==1.1.0
 termcolor==1.1.0
+text-unidecode==1.2
 toml==0.10.0
 tox==3.5.3
 traitlets==4.3.2
+tzlocal==1.5.1
 urllib3==1.24.1
 virtualenv==16.1.0
 wcwidth==0.1.7

From c83dc666a448ee7be8faa970313be0f68bf53a19 Mon Sep 17 00:00:00 2001
From: Daniel Quinn <code@danielquinn.org>
Date: Sat, 3 Nov 2018 13:42:03 +0000
Subject: [PATCH 47/47] I'm going to have to ditch requirements.txt if it can't
 be reliably generated

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index a62185562..92f4b0ff8 100755
--- a/requirements.txt
+++ b/requirements.txt
@@ -39,7 +39,7 @@ more-itertools==4.3.0
 packaging==18.0
 parso==0.3.1
 pdftotext==2.1.1
-pexpect==4.6.0 ; sys_platform != 'win32'
+pexpect==4.6.0 
 pickleshare==0.7.5
 pillow==5.3.0
 pluggy==0.8.0