improved test cases. Python 3.6 compatibility.

Pipfile

@@ -8,6 +8,9 @@ url = "https://www.piwheels.org/simple"
 verify_ssl = true
 name = "piwheels"
 
+[requires]
+python_version = "3.6"
+
 [packages]
 dateparser = "~=0.7.6"
 django = "~=3.1.3"

Pipfile.lock

@@ -1,10 +1,12 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "e9792119f687757dd388e73827ddd4216910327d5b65a8b950d4b202679c36eb"
+            "sha256": "d6432a18280c092c108e998f00bcd377c0c55ef18f26cb0b8eb64f9618b9f383"
         },
         "pipfile-spec": 6,
-        "requires": {},
+        "requires": {
+            "python_version": "3.6"
+        },
         "sources": [
             {
                 "name": "pypi",
@@ -701,6 +703,22 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==1.2.0"
         },
+        "importlib-metadata": {
+            "hashes": [
+                "sha256:030f3b1bdb823ecbe4a9659e14cc861ce5af403fe99863bae173ec5fe00ab132",
+                "sha256:caeee3603f5dcf567864d1be9b839b0bcfdf1383e3e7be33ce2dead8144ff19c"
+            ],
+            "markers": "python_version < '3.8'",
+            "version": "==2.1.0"
+        },
+        "importlib-resources": {
+            "hashes": [
+                "sha256:7b51f0106c8ec564b1bef3d9c588bc694ce2b92125bbb6278f4f2f5b54ec3592",
+                "sha256:a3d34a8464ce1d5d7c92b0ea4e921e696d86f2aa212e684451cb1482c8d84ed5"
+            ],
+            "markers": "python_version < '3.7'",
+            "version": "==3.3.0"
+        },
         "iniconfig": {
             "hashes": [
                 "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3",
@@ -1012,6 +1030,14 @@
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==20.2.1"
+        },
+        "zipp": {
+            "hashes": [
+                "sha256:102c24ef8f171fd729d46599845e95c7ab894a4cf45f5de11a44cc7444fb1108",
+                "sha256:ed5eee1974372595f9e416cc7bbeeb12335201d8081ca8a0743c954d4446e5cb"
+            ],
+            "markers": "python_version < '3.8'",
+            "version": "==3.4.0"
         }
     }
 }

src/documents/tests/test_api.py

@@ -1,41 +1,24 @@
 import os
-import shutil
 import tempfile
 from unittest import mock
 
 from django.contrib.auth.models import User
-from django.test import override_settings
 from pathvalidate import ValidationError
 from rest_framework.test import APITestCase
 
 from documents.models import Document, Correspondent, DocumentType, Tag
+from documents.tests.utils import setup_directories, remove_dirs
 
 
 class DocumentApiTest(APITestCase):
 
     def setUp(self):
-        self.scratch_dir = tempfile.mkdtemp()
+        self.dirs = setup_directories()
-        self.media_dir = tempfile.mkdtemp()
+        self.addCleanup(remove_dirs, self.dirs)
-        self.originals_dir = os.path.join(self.media_dir, "documents", "originals")
-        self.thumbnail_dir = os.path.join(self.media_dir, "documents", "thumbnails")
-
-        os.makedirs(self.originals_dir, exist_ok=True)
-        os.makedirs(self.thumbnail_dir, exist_ok=True)
-
-        override_settings(
-            SCRATCH_DIR=self.scratch_dir,
-            MEDIA_ROOT=self.media_dir,
-            ORIGINALS_DIR=self.originals_dir,
-            THUMBNAIL_DIR=self.thumbnail_dir
-        ).enable()
 
         user = User.objects.create_superuser(username="temp_admin")
         self.client.force_login(user=user)
 
-    def tearDown(self):
-        shutil.rmtree(self.scratch_dir, ignore_errors=True)
-        shutil.rmtree(self.media_dir, ignore_errors=True)
-
     def testDocuments(self):
 
         response = self.client.get("/api/documents/").data
@@ -88,7 +71,7 @@ class DocumentApiTest(APITestCase):
 
     def test_document_actions(self):
 
-        _, filename = tempfile.mkstemp(dir=self.originals_dir)
+        _, filename = tempfile.mkstemp(dir=self.dirs.originals_dir)
 
         content = b"This is a test"
         content_thumbnail = b"thumbnail content"
@@ -98,7 +81,7 @@ class DocumentApiTest(APITestCase):
 
         doc = Document.objects.create(title="none", filename=os.path.basename(filename), mime_type="application/pdf")
 
-        with open(os.path.join(self.thumbnail_dir, "{:07d}.png".format(doc.pk)), "wb") as f:
+        with open(os.path.join(self.dirs.thumbnail_dir, "{:07d}.png".format(doc.pk)), "wb") as f:
             f.write(content_thumbnail)
 
         response = self.client.get('/api/documents/{}/download/'.format(doc.pk))
@@ -227,7 +210,8 @@ class DocumentApiTest(APITestCase):
 
         m.assert_called_once()
 
-        self.assertEqual(m.call_args.kwargs['override_filename'], "simple.pdf")
+        args, kwargs = m.call_args
+        self.assertEqual(kwargs['override_filename'], "simple.pdf")
 
     @mock.patch("documents.forms.async_task")
     def test_upload_invalid_form(self, m):

src/documents/tests/test_classifier.py

@@ -11,7 +11,6 @@ from documents.models import Correspondent, Document, Tag, DocumentType
 class TestClassifier(TestCase):
 
     def setUp(self):
-
         self.classifier = DocumentClassifier()
 
     def generate_test_data(self):

src/documents/tests/test_consumer.py

@@ -1,12 +1,12 @@
 import os
 import re
-import shutil
 import tempfile
 from unittest import mock
 from unittest.mock import MagicMock
 
 from django.test import TestCase, override_settings
 
+from .utils import setup_directories, remove_dirs
 from ..consumer import Consumer, ConsumerError
 from ..models import FileInfo, Tag, Correspondent, DocumentType, Document
 from ..parsers import DocumentParser, ParseError
@@ -411,23 +411,14 @@ def fake_magic_from_file(file, mime=False):
 class TestConsumer(TestCase):
 
     def make_dummy_parser(self, path, logging_group):
-        return DummyParser(path, logging_group, self.scratch_dir)
+        return DummyParser(path, logging_group, self.dirs.scratch_dir)
 
     def make_faulty_parser(self, path, logging_group):
-        return FaultyParser(path, logging_group, self.scratch_dir)
+        return FaultyParser(path, logging_group, self.dirs.scratch_dir)
 
     def setUp(self):
-        self.scratch_dir = tempfile.mkdtemp()
+        self.dirs = setup_directories()
-        self.media_dir = tempfile.mkdtemp()
+        self.addCleanup(remove_dirs, self.dirs)
-        self.consumption_dir = tempfile.mkdtemp()
-
-        override_settings(
-            SCRATCH_DIR=self.scratch_dir,
-            MEDIA_ROOT=self.media_dir,
-            ORIGINALS_DIR=os.path.join(self.media_dir, "documents", "originals"),
-            THUMBNAIL_DIR=os.path.join(self.media_dir, "documents", "thumbnails"),
-            CONSUMPTION_DIR=self.consumption_dir
-        ).enable()
 
         patcher = mock.patch("documents.parsers.document_consumer_declaration.send")
         m = patcher.start()
@@ -441,13 +432,8 @@ class TestConsumer(TestCase):
 
         self.consumer = Consumer()
 
-    def tearDown(self):
-        shutil.rmtree(self.scratch_dir, ignore_errors=True)
-        shutil.rmtree(self.media_dir, ignore_errors=True)
-        shutil.rmtree(self.consumption_dir, ignore_errors=True)
-
     def get_test_file(self):
-        fd, f = tempfile.mkstemp(suffix=".pdf", dir=self.scratch_dir)
+        fd, f = tempfile.mkstemp(suffix=".pdf", dir=self.dirs.scratch_dir)
         return f
 
     def testNormalOperation(self):

src/documents/tests/test_management_consumer.py

@@ -1,7 +1,6 @@
 import filecmp
 import os
 import shutil
-import tempfile
 from threading import Thread
 from time import sleep
 from unittest import mock
@@ -11,6 +10,7 @@ from django.test import TestCase, override_settings
 
 from documents.consumer import ConsumerError
 from documents.management.commands import document_consumer
+from documents.tests.utils import setup_directories, remove_dirs
 
 
 class ConsumerThread(Thread):
@@ -41,9 +41,8 @@ class TestConsumer(TestCase):
         self.task_mock = patcher.start()
         self.addCleanup(patcher.stop)
 
-        self.consume_dir = tempfile.mkdtemp()
+        self.dirs = setup_directories()
-
+        self.addCleanup(remove_dirs, self.dirs)
-        override_settings(CONSUMPTION_DIR=self.consume_dir).enable()
 
     def t_start(self):
         self.t = ConsumerThread()
@@ -94,25 +93,29 @@ class TestConsumer(TestCase):
     def test_consume_file(self):
         self.t_start()
 
-        f = os.path.join(self.consume_dir, "my_file.pdf")
+        f = os.path.join(self.dirs.consumption_dir, "my_file.pdf")
         shutil.copy(self.sample_file, f)
 
         self.wait_for_task_mock_call()
 
         self.task_mock.assert_called_once()
-        self.assertEqual(self.task_mock.call_args.args[1], f)
+
+        args, kwargs = self.task_mock.call_args
+        self.assertEqual(args[1], f)
 
     @override_settings(CONSUMER_POLLING=1)
     def test_consume_file_polling(self):
         self.test_consume_file()
 
     def test_consume_existing_file(self):
-        f = os.path.join(self.consume_dir, "my_file.pdf")
+        f = os.path.join(self.dirs.consumption_dir, "my_file.pdf")
         shutil.copy(self.sample_file, f)
 
         self.t_start()
         self.task_mock.assert_called_once()
-        self.assertEqual(self.task_mock.call_args.args[1], f)
+
+        args, kwargs = self.task_mock.call_args
+        self.assertEqual(args[1], f)
 
     @override_settings(CONSUMER_POLLING=1)
     def test_consume_existing_file_polling(self):
@@ -125,7 +128,7 @@ class TestConsumer(TestCase):
 
         self.t_start()
 
-        fname = os.path.join(self.consume_dir, "my_file.pdf")
+        fname = os.path.join(self.dirs.consumption_dir, "my_file.pdf")
 
         self.slow_write_file(fname)
 
@@ -135,7 +138,8 @@ class TestConsumer(TestCase):
 
         self.task_mock.assert_called_once()
 
-        self.assertEqual(self.task_mock.call_args.args[1], fname)
+        args, kwargs = self.task_mock.call_args
+        self.assertEqual(args[1], fname)
 
     @override_settings(CONSUMER_POLLING=1)
     def test_slow_write_pdf_polling(self):
@@ -148,8 +152,8 @@ class TestConsumer(TestCase):
 
         self.t_start()
 
-        fname = os.path.join(self.consume_dir, "my_file.~df")
+        fname = os.path.join(self.dirs.consumption_dir, "my_file.~df")
-        fname2 = os.path.join(self.consume_dir, "my_file.pdf")
+        fname2 = os.path.join(self.dirs.consumption_dir, "my_file.pdf")
 
         self.slow_write_file(fname)
         shutil.move(fname, fname2)
@@ -157,7 +161,9 @@ class TestConsumer(TestCase):
         self.wait_for_task_mock_call()
 
         self.task_mock.assert_called_once()
-        self.assertEqual(self.task_mock.call_args.args[1], fname2)
+
+        args, kwargs = self.task_mock.call_args
+        self.assertEqual(args[1], fname2)
 
         error_logger.assert_not_called()
 
@@ -172,13 +178,14 @@ class TestConsumer(TestCase):
 
         self.t_start()
 
-        fname = os.path.join(self.consume_dir, "my_file.pdf")
+        fname = os.path.join(self.dirs.consumption_dir, "my_file.pdf")
         self.slow_write_file(fname, incomplete=True)
 
         self.wait_for_task_mock_call()
 
         self.task_mock.assert_called_once()
-        self.assertEqual(self.task_mock.call_args.args[1], fname)
+        args, kwargs = self.task_mock.call_args
+        self.assertEqual(args[1], fname)
 
         # assert that we have an error logged with this invalid file.
         error_logger.assert_called_once()

src/documents/tests/test_matchables.py

@@ -1,3 +1,5 @@
+import shutil
+import tempfile
 from random import randint
 
 from django.contrib.admin.models import LogEntry
@@ -215,6 +217,13 @@ class TestDocumentConsumptionFinishedSignal(TestCase):
         self.doc_contains = Document.objects.create(
             content="I contain the keyword.", mime_type="application/pdf")
 
+        self.index_dir = tempfile.mkdtemp()
+        # TODO: we should not need the index here.
+        override_settings(INDEX_DIR=self.index_dir).enable()
+
+    def tearDown(self) -> None:
+        shutil.rmtree(self.index_dir, ignore_errors=True)
+
     def test_tag_applied_any(self):
         t1 = Tag.objects.create(
             name="test", match="keyword", matching_algorithm=Tag.MATCH_ANY)

src/documents/tests/utils.py

@@ -0,0 +1,41 @@
+import os
+import shutil
+import tempfile
+from collections import namedtuple
+
+from django.test import override_settings
+
+
+def setup_directories():
+
+    dirs = namedtuple("Dirs", ())
+
+    dirs.data_dir = tempfile.mkdtemp()
+    dirs.scratch_dir = tempfile.mkdtemp()
+    dirs.media_dir = tempfile.mkdtemp()
+    dirs.consumption_dir = tempfile.mkdtemp()
+    dirs.index_dir = os.path.join(dirs.data_dir, "documents", "originals")
+    dirs.originals_dir = os.path.join(dirs.media_dir, "documents", "originals")
+    dirs.thumbnail_dir = os.path.join(dirs.media_dir, "documents", "thumbnails")
+    os.makedirs(dirs.index_dir)
+    os.makedirs(dirs.originals_dir)
+    os.makedirs(dirs.thumbnail_dir)
+
+    override_settings(
+        DATA_DIR=dirs.data_dir,
+        SCRATCH_DIR=dirs.scratch_dir,
+        MEDIA_ROOT=dirs.media_dir,
+        ORIGINALS_DIR=dirs.originals_dir,
+        THUMBNAIL_DIR=dirs.thumbnail_dir,
+        CONSUMPTION_DIR=dirs.consumption_dir,
+        INDEX_DIR=dirs.index_dir
+    ).enable()
+
+    return dirs
+
+
+def remove_dirs(dirs):
+    shutil.rmtree(dirs.media_dir, ignore_errors=True)
+    shutil.rmtree(dirs.data_dir, ignore_errors=True)
+    shutil.rmtree(dirs.scratch_dir, ignore_errors=True)
+    shutil.rmtree(dirs.consumption_dir, ignore_errors=True)