Merge branch 'dev' into machine-learning

src/documents/admin.py

@@ -4,8 +4,10 @@ from django.conf import settings
 from django.contrib import admin, messages
 from django.contrib.admin.templatetags.admin_urls import add_preserved_filters
 from django.contrib.auth.models import User, Group
-from django.http import HttpResponseRedirect
-from django.urls import reverse
+try:
+    from django.core.urlresolvers import reverse
+except ImportError:
+    from django.urls import reverse
 from django.templatetags.static import static
 from django.utils.html import format_html
 from django.utils.http import urlquote
@@ -164,6 +166,8 @@ class DocumentAdmin(CommonAdmin):
                     "tags_", "archive_serial_number", "document_type")
     list_filter = ("document_type", "tags", ('correspondent', RecentCorrespondentFilter), "correspondent", FinancialYearFilter)
 
+    filter_horizontal = ("tags",)
+
     ordering = ["-created", "correspondent"]
 
     actions = [add_tag_to_selected, remove_tag_from_selected, set_correspondent_on_selected, remove_correspondent_from_selected, set_document_type_on_selected, remove_document_type_from_selected, run_document_classifier_on_selected]
@@ -282,16 +286,13 @@ class DocumentAdmin(CommonAdmin):
 
     @staticmethod
     def _html_tag(kind, inside=None, **kwargs):
-
-        attributes = []
-        for lft, rgt in kwargs.items():
-            attributes.append('{}="{}"'.format(lft, rgt))
+        attributes = format_html_join(' ', '{}="{}"', kwargs.items())
 
         if inside is not None:
-            return "<{kind} {attributes}>{inside}</{kind}>".format(
-                kind=kind, attributes=" ".join(attributes), inside=inside)
+            return format_html("<{kind} {attributes}>{inside}</{kind}>",
+                               kind=kind, attributes=attributes, inside=inside)
 
-        return "<{} {}/>".format(kind, " ".join(attributes))
+        return format_html("<{} {}/>", kind, attributes)
 
 
 class LogAdmin(CommonAdmin):

src/documents/filters.py

@@ -1,11 +1,11 @@
-from django_filters.rest_framework import CharFilter, FilterSet
+from django_filters.rest_framework import CharFilter, FilterSet, BooleanFilter
 
 from .models import Correspondent, Document, Tag, DocumentType
 
 
 class CorrespondentFilterSet(FilterSet):
 
-    class Meta(object):
+    class Meta:
         model = Correspondent
         fields = {
             "name": [
@@ -18,7 +18,7 @@ class CorrespondentFilterSet(FilterSet):
 
 class TagFilterSet(FilterSet):
 
-    class Meta(object):
+    class Meta:
         model = Tag
         fields = {
             "name": [
@@ -55,14 +55,22 @@ class DocumentFilterSet(FilterSet):
         )
     }
 
-    correspondent__name = CharFilter(name="correspondent__name", **CHAR_KWARGS)
-    correspondent__slug = CharFilter(name="correspondent__slug", **CHAR_KWARGS)
-    tags__name = CharFilter(name="tags__name", **CHAR_KWARGS)
-    tags__slug = CharFilter(name="tags__slug", **CHAR_KWARGS)
-    document_type__name = CharFilter(name="document_type__name", **CHAR_KWARGS)
-    document_type__slug = CharFilter(name="document_type__slug", **CHAR_KWARGS)
+    correspondent__name = CharFilter(
+        field_name="correspondent__name", **CHAR_KWARGS)
+    correspondent__slug = CharFilter(
+        field_name="correspondent__slug", **CHAR_KWARGS)
+    tags__name = CharFilter(
+        field_name="tags__name", **CHAR_KWARGS)
+    tags__slug = CharFilter(
+        field_name="tags__slug", **CHAR_KWARGS)
+    tags__empty = BooleanFilter(
+        field_name="tags", lookup_expr="isnull", distinct=True)
+    document_type__name = CharFilter(
+        name="document_type__name", **CHAR_KWARGS)
+    document_type__slug = CharFilter(
+        name="document_type__slug", **CHAR_KWARGS)
 
-    class Meta(object):
+    class Meta:
         model = Document
         fields = {
             "title": [

src/documents/management/commands/document_consumer.py

@@ -1,7 +1,5 @@
-import datetime
 import logging
 import os
-import sys
 import time
 
 from django.conf import settings
@@ -13,7 +11,7 @@ from ...mail import MailFetcher, MailFetcherError
 try:
     from inotify_simple import INotify, flags
 except ImportError:
-    pass
+    INotify = flags = None
 
 
 class Command(BaseCommand):
@@ -62,7 +60,8 @@ class Command(BaseCommand):
         parser.add_argument(
             "--no-inotify",
             action="store_true",
-            help="Don't use inotify, even if it's available."
+            help="Don't use inotify, even if it's available.",
+            default=False
         )
 
     def handle(self, *args, **options):
@@ -71,8 +70,7 @@ class Command(BaseCommand):
         directory = options["directory"]
         loop_time = options["loop_time"]
         mail_delta = options["mail_delta"] * 60
-        use_inotify = (not options["no_inotify"]
-                       and "inotify_simple" in sys.modules)
+        use_inotify = INotify is not None and options["no_inotify"] is False
 
         try:
             self.file_consumer = Consumer(consume=directory)

src/documents/migrations/0003_sender.py

@@ -32,7 +32,6 @@ def realign_senders(apps, schema_editor):
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ('documents', '0002_auto_20151226_1316'),
     ]

src/documents/migrations/0011_auto_20160303_1929.py

@@ -6,9 +6,7 @@ from django.db import migrations
 
 
 class Migration(migrations.Migration):
-
     atomic = False
-
     dependencies = [
         ('documents', '0010_log'),
     ]

src/documents/migrations/0012_auto_20160305_0040.py

@@ -112,7 +112,6 @@ def move_documents_and_create_thumbnails(apps, schema_editor):
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ('documents', '0011_auto_20160303_1929'),
     ]

src/documents/migrations/0014_document_checksum.py

@@ -128,7 +128,6 @@ def do_nothing(apps, schema_editor):
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ('documents', '0013_auto_20160325_2111'),
     ]

src/documents/migrations/0019_add_consumer_user.py

@@ -15,7 +15,6 @@ def reverse_func(apps, schema_editor):
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ('documents', '0018_auto_20170715_1712'),
     ]

src/documents/migrations/0020_document_added.py

@@ -11,8 +11,8 @@ def set_added_time_to_created_time(apps, schema_editor):
         doc.added = doc.created
         doc.save()
 
-class Migration(migrations.Migration):
 
+class Migration(migrations.Migration):
     dependencies = [
         ('documents', '0019_add_consumer_user'),
     ]

src/documents/models.py

@@ -10,7 +10,10 @@ from collections import OrderedDict
 from fuzzywuzzy import fuzz
 
 from django.conf import settings
-from django.urls import reverse
+try:
+    from django.core.urlresolvers import reverse
+except ImportError:
+    from django.urls import reverse
 from django.db import models
 from django.template.defaultfilters import slugify
 from django.utils import timezone
@@ -27,6 +30,7 @@ class MatchingModel(models.Model):
 
     class Meta:
         abstract = True
+        ordering = ("name",)
 
     def __str__(self):
         return self.name
@@ -359,8 +363,10 @@ class FileInfo:
     def _get_tags(cls, tags):
         r = []
         for t in tags.split(","):
-            r.append(
-                Tag.objects.get_or_create(slug=t, defaults={"name": t})[0])
+            r.append(Tag.objects.get_or_create(
+                slug=t.lower(),
+                defaults={"name": t}
+            )[0])
         return tuple(r)
 
     @classmethod

src/documents/serialisers.py

@@ -5,7 +5,7 @@ from .models import Correspondent, Tag, Document, Log, DocumentType
 
 class CorrespondentSerializer(serializers.HyperlinkedModelSerializer):
 
-    class Meta(object):
+    class Meta:
         model = Correspondent
         fields = ("id", "slug", "name", "automatic_classification")
 
@@ -19,7 +19,7 @@ class DocumentTypeSerializer(serializers.HyperlinkedModelSerializer):
 
 class TagSerializer(serializers.HyperlinkedModelSerializer):
 
-    class Meta(object):
+    class Meta:
         model = Tag
         fields = (
             "id", "slug", "name", "colour", "automatic_classification")
@@ -48,7 +48,7 @@ class DocumentSerializer(serializers.ModelSerializer):
     document_type = DocumentTypeField(
         view_name="drf:documenttype-detail", allow_null=True)
 
-    class Meta(object):
+    class Meta:
         model = Document
         fields = (
             "id",
@@ -72,7 +72,7 @@ class LogSerializer(serializers.ModelSerializer):
     time = serializers.DateTimeField()
     messages = serializers.CharField()
 
-    class Meta(object):
+    class Meta:
         model = Log
         fields = (
             "time",

src/documents/tests/test_consumer.py

@@ -3,7 +3,7 @@ from unittest import mock
 from tempfile import TemporaryDirectory
 
 from ..consumer import Consumer
-from ..models import FileInfo
+from ..models import FileInfo, Tag
 
 
 class TestConsumer(TestCase):
@@ -190,6 +190,20 @@ class TestAttributes(TestCase):
             ()
         )
 
+    def test_case_insensitive_tag_creation(self):
+        """
+        Tags should be detected and created as lower case.
+        :return:
+        """
+
+        path = "Title - Correspondent - tAg1,TAG2.pdf"
+        self.assertEqual(len(FileInfo.from_path(path).tags), 2)
+
+        path = "Title - Correspondent - tag1,tag2.pdf"
+        self.assertEqual(len(FileInfo.from_path(path).tags), 2)
+
+        self.assertEqual(Tag.objects.all().count(), 2)
+
 
 class TestFieldPermutations(TestCase):
 

src/paperless/settings.py

@@ -61,13 +61,13 @@ INSTALLED_APPS = [
     "django.contrib.messages",
     "django.contrib.staticfiles",
 
+    "corsheaders",
     "django_extensions",
 
     "documents.apps.DocumentsConfig",
     "reminders.apps.RemindersConfig",
     "paperless_tesseract.apps.PaperlessTesseractConfig",
 
-    "flat_responsive",  # TODO: Remove as of Django 2.x
     "django.contrib.admin",
 
     "rest_framework",
@@ -79,11 +79,10 @@ INSTALLED_APPS = [
 if os.getenv("PAPERLESS_INSTALLED_APPS"):
     INSTALLED_APPS += os.getenv("PAPERLESS_INSTALLED_APPS").split(",")
 
-
-
 MIDDLEWARE = [
     'django.middleware.security.SecurityMiddleware',
     'django.contrib.sessions.middleware.SessionMiddleware',
+    'corsheaders.middleware.CorsMiddleware',
     'django.middleware.common.CommonMiddleware',
     'django.middleware.csrf.CsrfViewMiddleware',
     'django.contrib.auth.middleware.AuthenticationMiddleware',
@@ -91,6 +90,9 @@ MIDDLEWARE = [
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
 ]
 
+# We allow CORS from localhost:8080
+CORS_ORIGIN_WHITELIST = tuple(os.getenv("PAPERLESS_CORS_ALLOWED_HOSTS", "localhost:8080").split(","))
+
 # If auth is disabled, we just use our "bypass" authentication middleware
 if bool(os.getenv("PAPERLESS_DISABLE_LOGIN", "false").lower() in ("yes", "y", "1", "t", "true")):
     _index = MIDDLEWARE.index("django.contrib.auth.middleware.AuthenticationMiddleware")

src/paperless/urls.py

@@ -29,10 +29,11 @@ urlpatterns = [
     # API
     url(
         r"^api/auth/",
-        include('rest_framework.urls', namespace="rest_framework")
+        include(
+            ('rest_framework.urls', 'rest_framework'),
+            namespace="rest_framework")
     ),
-
-    url(r"^api/", include((router.urls, 'drf'))),
+    url(r"^api/", include((router.urls, 'drf'), namespace="drf")),
 
     # File downloads
     url(

src/paperless/version.py

@@ -1 +1 @@
-__version__ = (2, 1, 0)
+__version__ = (2, 2, 1)

src/paperless_tesseract/tests/test_date.py

@@ -33,7 +33,7 @@ class TestDate(TestCase):
 
     @mock.patch(
         "paperless_tesseract.parsers.RasterisedDocumentParser.SCRATCH",
-        SAMPLE_FILES
+        SCRATCH
     )
     def test_date_format_2(self):
         input_file = os.path.join(self.SAMPLE_FILES, "")
@@ -43,7 +43,7 @@ class TestDate(TestCase):
 
     @mock.patch(
         "paperless_tesseract.parsers.RasterisedDocumentParser.SCRATCH",
-        SAMPLE_FILES
+        SCRATCH
     )
     def test_date_format_3(self):
         input_file = os.path.join(self.SAMPLE_FILES, "")
@@ -53,7 +53,7 @@ class TestDate(TestCase):
 
     @mock.patch(
         "paperless_tesseract.parsers.RasterisedDocumentParser.SCRATCH",
-        SAMPLE_FILES
+        SCRATCH
     )
     def test_date_format_4(self):
         input_file = os.path.join(self.SAMPLE_FILES, "")
@@ -66,7 +66,7 @@ class TestDate(TestCase):
 
     @mock.patch(
         "paperless_tesseract.parsers.RasterisedDocumentParser.SCRATCH",
-        SAMPLE_FILES
+        SCRATCH
     )
     def test_date_format_5(self):
         input_file = os.path.join(self.SAMPLE_FILES, "")
@@ -80,7 +80,7 @@ class TestDate(TestCase):
 
     @mock.patch(
         "paperless_tesseract.parsers.RasterisedDocumentParser.SCRATCH",
-        SAMPLE_FILES
+        SCRATCH
     )
     def test_date_format_6(self):
         input_file = os.path.join(self.SAMPLE_FILES, "")
@@ -100,7 +100,7 @@ class TestDate(TestCase):
 
     @mock.patch(
         "paperless_tesseract.parsers.RasterisedDocumentParser.SCRATCH",
-        SAMPLE_FILES
+        SCRATCH
     )
     def test_date_format_7(self):
         input_file = os.path.join(self.SAMPLE_FILES, "")
@@ -117,7 +117,7 @@ class TestDate(TestCase):
 
     @mock.patch(
         "paperless_tesseract.parsers.RasterisedDocumentParser.SCRATCH",
-        SAMPLE_FILES
+        SCRATCH
     )
     def test_date_format_8(self):
         input_file = os.path.join(self.SAMPLE_FILES, "")
@@ -138,7 +138,7 @@ class TestDate(TestCase):
 
     @mock.patch(
         "paperless_tesseract.parsers.RasterisedDocumentParser.SCRATCH",
-        SAMPLE_FILES
+        SCRATCH
     )
     def test_date_format_9(self):
         input_file = os.path.join(self.SAMPLE_FILES, "")
@@ -153,7 +153,7 @@ class TestDate(TestCase):
 
     @mock.patch(
         "paperless_tesseract.parsers.RasterisedDocumentParser.SCRATCH",
-        SAMPLE_FILES
+        SCRATCH
     )
     def test_get_text_1_pdf(self):
         input_file = os.path.join(self.SAMPLE_FILES, "tests_date_1.pdf")
@@ -359,7 +359,7 @@ class TestDate(TestCase):
 
     @mock.patch(
         "paperless_tesseract.parsers.RasterisedDocumentParser.SCRATCH",
-        SAMPLE_FILES
+        SCRATCH
     )
     def test_get_text_8_pdf(self):
         input_file = os.path.join(self.SAMPLE_FILES, "tests_date_8.pdf")
@@ -373,7 +373,7 @@ class TestDate(TestCase):
 
     @mock.patch(
         "paperless_tesseract.parsers.RasterisedDocumentParser.SCRATCH",
-        SAMPLE_FILES
+        SCRATCH
     )
     def test_get_text_9_pdf(self):
         input_file = os.path.join(self.SAMPLE_FILES, "tests_date_9.pdf")

src/reminders/models.py

@@ -3,6 +3,8 @@ from django.db import models
 
 class Reminder(models.Model):
 
-    document = models.ForeignKey("documents.Document", on_delete=models.CASCADE)
+    document = models.ForeignKey(
+        "documents.Document", on_delete=models.PROTECT
+        )
     date = models.DateTimeField()
     note = models.TextField(blank=True)