From 4bf0d834a0ad39f2cdfe188545bd65401f128f0f Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Thu, 26 Nov 2020 22:17:14 +0100 Subject: [PATCH] improved test cases. Python 3.6 compatibility. --- Pipfile | 3 ++ Pipfile.lock | 30 +++++++++++++- src/documents/tests/test_api.py | 30 ++++---------- src/documents/tests/test_classifier.py | 1 - src/documents/tests/test_consumer.py | 26 +++--------- .../tests/test_management_consumer.py | 37 ++++++++++------- src/documents/tests/test_matchables.py | 9 ++++ src/documents/tests/utils.py | 41 +++++++++++++++++++ 8 files changed, 116 insertions(+), 61 deletions(-) create mode 100644 src/documents/tests/utils.py diff --git a/Pipfile b/Pipfile index a6169a2ba..105efd0ad 100644 --- a/Pipfile +++ b/Pipfile @@ -8,6 +8,9 @@ url = "https://www.piwheels.org/simple" verify_ssl = true name = "piwheels" +[requires] +python_version = "3.6" + [packages] dateparser = "~=0.7.6" django = "~=3.1.3" diff --git a/Pipfile.lock b/Pipfile.lock index b10c414ed..918609845 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,10 +1,12 @@ { "_meta": { "hash": { - "sha256": "e9792119f687757dd388e73827ddd4216910327d5b65a8b950d4b202679c36eb" + "sha256": "d6432a18280c092c108e998f00bcd377c0c55ef18f26cb0b8eb64f9618b9f383" }, "pipfile-spec": 6, - "requires": {}, + "requires": { + "python_version": "3.6" + }, "sources": [ { "name": "pypi", @@ -701,6 +703,22 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.2.0" }, + "importlib-metadata": { + "hashes": [ + "sha256:030f3b1bdb823ecbe4a9659e14cc861ce5af403fe99863bae173ec5fe00ab132", + "sha256:caeee3603f5dcf567864d1be9b839b0bcfdf1383e3e7be33ce2dead8144ff19c" + ], + "markers": "python_version < '3.8'", + "version": "==2.1.0" + }, + "importlib-resources": { + "hashes": [ + "sha256:7b51f0106c8ec564b1bef3d9c588bc694ce2b92125bbb6278f4f2f5b54ec3592", + "sha256:a3d34a8464ce1d5d7c92b0ea4e921e696d86f2aa212e684451cb1482c8d84ed5" + ], + "markers": "python_version < '3.7'", + "version": "==3.3.0" + }, "iniconfig": { "hashes": [ "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3", @@ -1012,6 +1030,14 @@ ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==20.2.1" + }, + "zipp": { + "hashes": [ + "sha256:102c24ef8f171fd729d46599845e95c7ab894a4cf45f5de11a44cc7444fb1108", + "sha256:ed5eee1974372595f9e416cc7bbeeb12335201d8081ca8a0743c954d4446e5cb" + ], + "markers": "python_version < '3.8'", + "version": "==3.4.0" } } } diff --git a/src/documents/tests/test_api.py b/src/documents/tests/test_api.py index c7e31e280..d9a2aac26 100644 --- a/src/documents/tests/test_api.py +++ b/src/documents/tests/test_api.py @@ -1,41 +1,24 @@ import os -import shutil import tempfile from unittest import mock from django.contrib.auth.models import User -from django.test import override_settings from pathvalidate import ValidationError from rest_framework.test import APITestCase from documents.models import Document, Correspondent, DocumentType, Tag +from documents.tests.utils import setup_directories, remove_dirs class DocumentApiTest(APITestCase): def setUp(self): - self.scratch_dir = tempfile.mkdtemp() - self.media_dir = tempfile.mkdtemp() - self.originals_dir = os.path.join(self.media_dir, "documents", "originals") - self.thumbnail_dir = os.path.join(self.media_dir, "documents", "thumbnails") - - os.makedirs(self.originals_dir, exist_ok=True) - os.makedirs(self.thumbnail_dir, exist_ok=True) - - override_settings( - SCRATCH_DIR=self.scratch_dir, - MEDIA_ROOT=self.media_dir, - ORIGINALS_DIR=self.originals_dir, - THUMBNAIL_DIR=self.thumbnail_dir - ).enable() + self.dirs = setup_directories() + self.addCleanup(remove_dirs, self.dirs) user = User.objects.create_superuser(username="temp_admin") self.client.force_login(user=user) - def tearDown(self): - shutil.rmtree(self.scratch_dir, ignore_errors=True) - shutil.rmtree(self.media_dir, ignore_errors=True) - def testDocuments(self): response = self.client.get("/api/documents/").data @@ -88,7 +71,7 @@ class DocumentApiTest(APITestCase): def test_document_actions(self): - _, filename = tempfile.mkstemp(dir=self.originals_dir) + _, filename = tempfile.mkstemp(dir=self.dirs.originals_dir) content = b"This is a test" content_thumbnail = b"thumbnail content" @@ -98,7 +81,7 @@ class DocumentApiTest(APITestCase): doc = Document.objects.create(title="none", filename=os.path.basename(filename), mime_type="application/pdf") - with open(os.path.join(self.thumbnail_dir, "{:07d}.png".format(doc.pk)), "wb") as f: + with open(os.path.join(self.dirs.thumbnail_dir, "{:07d}.png".format(doc.pk)), "wb") as f: f.write(content_thumbnail) response = self.client.get('/api/documents/{}/download/'.format(doc.pk)) @@ -227,7 +210,8 @@ class DocumentApiTest(APITestCase): m.assert_called_once() - self.assertEqual(m.call_args.kwargs['override_filename'], "simple.pdf") + args, kwargs = m.call_args + self.assertEqual(kwargs['override_filename'], "simple.pdf") @mock.patch("documents.forms.async_task") def test_upload_invalid_form(self, m): diff --git a/src/documents/tests/test_classifier.py b/src/documents/tests/test_classifier.py index 0f421bb32..e5e7d8639 100644 --- a/src/documents/tests/test_classifier.py +++ b/src/documents/tests/test_classifier.py @@ -11,7 +11,6 @@ from documents.models import Correspondent, Document, Tag, DocumentType class TestClassifier(TestCase): def setUp(self): - self.classifier = DocumentClassifier() def generate_test_data(self): diff --git a/src/documents/tests/test_consumer.py b/src/documents/tests/test_consumer.py index 6dab98d02..323f5051f 100644 --- a/src/documents/tests/test_consumer.py +++ b/src/documents/tests/test_consumer.py @@ -1,12 +1,12 @@ import os import re -import shutil import tempfile from unittest import mock from unittest.mock import MagicMock from django.test import TestCase, override_settings +from .utils import setup_directories, remove_dirs from ..consumer import Consumer, ConsumerError from ..models import FileInfo, Tag, Correspondent, DocumentType, Document from ..parsers import DocumentParser, ParseError @@ -411,23 +411,14 @@ def fake_magic_from_file(file, mime=False): class TestConsumer(TestCase): def make_dummy_parser(self, path, logging_group): - return DummyParser(path, logging_group, self.scratch_dir) + return DummyParser(path, logging_group, self.dirs.scratch_dir) def make_faulty_parser(self, path, logging_group): - return FaultyParser(path, logging_group, self.scratch_dir) + return FaultyParser(path, logging_group, self.dirs.scratch_dir) def setUp(self): - self.scratch_dir = tempfile.mkdtemp() - self.media_dir = tempfile.mkdtemp() - self.consumption_dir = tempfile.mkdtemp() - - override_settings( - SCRATCH_DIR=self.scratch_dir, - MEDIA_ROOT=self.media_dir, - ORIGINALS_DIR=os.path.join(self.media_dir, "documents", "originals"), - THUMBNAIL_DIR=os.path.join(self.media_dir, "documents", "thumbnails"), - CONSUMPTION_DIR=self.consumption_dir - ).enable() + self.dirs = setup_directories() + self.addCleanup(remove_dirs, self.dirs) patcher = mock.patch("documents.parsers.document_consumer_declaration.send") m = patcher.start() @@ -441,13 +432,8 @@ class TestConsumer(TestCase): self.consumer = Consumer() - def tearDown(self): - shutil.rmtree(self.scratch_dir, ignore_errors=True) - shutil.rmtree(self.media_dir, ignore_errors=True) - shutil.rmtree(self.consumption_dir, ignore_errors=True) - def get_test_file(self): - fd, f = tempfile.mkstemp(suffix=".pdf", dir=self.scratch_dir) + fd, f = tempfile.mkstemp(suffix=".pdf", dir=self.dirs.scratch_dir) return f def testNormalOperation(self): diff --git a/src/documents/tests/test_management_consumer.py b/src/documents/tests/test_management_consumer.py index bfb7520ee..33938d450 100644 --- a/src/documents/tests/test_management_consumer.py +++ b/src/documents/tests/test_management_consumer.py @@ -1,7 +1,6 @@ import filecmp import os import shutil -import tempfile from threading import Thread from time import sleep from unittest import mock @@ -11,6 +10,7 @@ from django.test import TestCase, override_settings from documents.consumer import ConsumerError from documents.management.commands import document_consumer +from documents.tests.utils import setup_directories, remove_dirs class ConsumerThread(Thread): @@ -41,9 +41,8 @@ class TestConsumer(TestCase): self.task_mock = patcher.start() self.addCleanup(patcher.stop) - self.consume_dir = tempfile.mkdtemp() - - override_settings(CONSUMPTION_DIR=self.consume_dir).enable() + self.dirs = setup_directories() + self.addCleanup(remove_dirs, self.dirs) def t_start(self): self.t = ConsumerThread() @@ -94,25 +93,29 @@ class TestConsumer(TestCase): def test_consume_file(self): self.t_start() - f = os.path.join(self.consume_dir, "my_file.pdf") + f = os.path.join(self.dirs.consumption_dir, "my_file.pdf") shutil.copy(self.sample_file, f) self.wait_for_task_mock_call() self.task_mock.assert_called_once() - self.assertEqual(self.task_mock.call_args.args[1], f) + + args, kwargs = self.task_mock.call_args + self.assertEqual(args[1], f) @override_settings(CONSUMER_POLLING=1) def test_consume_file_polling(self): self.test_consume_file() def test_consume_existing_file(self): - f = os.path.join(self.consume_dir, "my_file.pdf") + f = os.path.join(self.dirs.consumption_dir, "my_file.pdf") shutil.copy(self.sample_file, f) self.t_start() self.task_mock.assert_called_once() - self.assertEqual(self.task_mock.call_args.args[1], f) + + args, kwargs = self.task_mock.call_args + self.assertEqual(args[1], f) @override_settings(CONSUMER_POLLING=1) def test_consume_existing_file_polling(self): @@ -125,7 +128,7 @@ class TestConsumer(TestCase): self.t_start() - fname = os.path.join(self.consume_dir, "my_file.pdf") + fname = os.path.join(self.dirs.consumption_dir, "my_file.pdf") self.slow_write_file(fname) @@ -135,7 +138,8 @@ class TestConsumer(TestCase): self.task_mock.assert_called_once() - self.assertEqual(self.task_mock.call_args.args[1], fname) + args, kwargs = self.task_mock.call_args + self.assertEqual(args[1], fname) @override_settings(CONSUMER_POLLING=1) def test_slow_write_pdf_polling(self): @@ -148,8 +152,8 @@ class TestConsumer(TestCase): self.t_start() - fname = os.path.join(self.consume_dir, "my_file.~df") - fname2 = os.path.join(self.consume_dir, "my_file.pdf") + fname = os.path.join(self.dirs.consumption_dir, "my_file.~df") + fname2 = os.path.join(self.dirs.consumption_dir, "my_file.pdf") self.slow_write_file(fname) shutil.move(fname, fname2) @@ -157,7 +161,9 @@ class TestConsumer(TestCase): self.wait_for_task_mock_call() self.task_mock.assert_called_once() - self.assertEqual(self.task_mock.call_args.args[1], fname2) + + args, kwargs = self.task_mock.call_args + self.assertEqual(args[1], fname2) error_logger.assert_not_called() @@ -172,13 +178,14 @@ class TestConsumer(TestCase): self.t_start() - fname = os.path.join(self.consume_dir, "my_file.pdf") + fname = os.path.join(self.dirs.consumption_dir, "my_file.pdf") self.slow_write_file(fname, incomplete=True) self.wait_for_task_mock_call() self.task_mock.assert_called_once() - self.assertEqual(self.task_mock.call_args.args[1], fname) + args, kwargs = self.task_mock.call_args + self.assertEqual(args[1], fname) # assert that we have an error logged with this invalid file. error_logger.assert_called_once() diff --git a/src/documents/tests/test_matchables.py b/src/documents/tests/test_matchables.py index 24e285ae7..4e4a3e7dc 100644 --- a/src/documents/tests/test_matchables.py +++ b/src/documents/tests/test_matchables.py @@ -1,3 +1,5 @@ +import shutil +import tempfile from random import randint from django.contrib.admin.models import LogEntry @@ -215,6 +217,13 @@ class TestDocumentConsumptionFinishedSignal(TestCase): self.doc_contains = Document.objects.create( content="I contain the keyword.", mime_type="application/pdf") + self.index_dir = tempfile.mkdtemp() + # TODO: we should not need the index here. + override_settings(INDEX_DIR=self.index_dir).enable() + + def tearDown(self) -> None: + shutil.rmtree(self.index_dir, ignore_errors=True) + def test_tag_applied_any(self): t1 = Tag.objects.create( name="test", match="keyword", matching_algorithm=Tag.MATCH_ANY) diff --git a/src/documents/tests/utils.py b/src/documents/tests/utils.py new file mode 100644 index 000000000..7b0938ee3 --- /dev/null +++ b/src/documents/tests/utils.py @@ -0,0 +1,41 @@ +import os +import shutil +import tempfile +from collections import namedtuple + +from django.test import override_settings + + +def setup_directories(): + + dirs = namedtuple("Dirs", ()) + + dirs.data_dir = tempfile.mkdtemp() + dirs.scratch_dir = tempfile.mkdtemp() + dirs.media_dir = tempfile.mkdtemp() + dirs.consumption_dir = tempfile.mkdtemp() + dirs.index_dir = os.path.join(dirs.data_dir, "documents", "originals") + dirs.originals_dir = os.path.join(dirs.media_dir, "documents", "originals") + dirs.thumbnail_dir = os.path.join(dirs.media_dir, "documents", "thumbnails") + os.makedirs(dirs.index_dir) + os.makedirs(dirs.originals_dir) + os.makedirs(dirs.thumbnail_dir) + + override_settings( + DATA_DIR=dirs.data_dir, + SCRATCH_DIR=dirs.scratch_dir, + MEDIA_ROOT=dirs.media_dir, + ORIGINALS_DIR=dirs.originals_dir, + THUMBNAIL_DIR=dirs.thumbnail_dir, + CONSUMPTION_DIR=dirs.consumption_dir, + INDEX_DIR=dirs.index_dir + ).enable() + + return dirs + + +def remove_dirs(dirs): + shutil.rmtree(dirs.media_dir, ignore_errors=True) + shutil.rmtree(dirs.data_dir, ignore_errors=True) + shutil.rmtree(dirs.scratch_dir, ignore_errors=True) + shutil.rmtree(dirs.consumption_dir, ignore_errors=True)