paradizelost/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2025-04-02 13:45:10 -05:00
Code Issues Packages Projects Releases Wiki Activity

improved test cases. Python 3.6 compatibility.

Browse Source
This commit is contained in:
jonaswinkler 2020-11-26 22:17:14 +01:00
parent d5ec762954
commit 4bf0d834a0
8 changed files with 116 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Pipfile
View File

@ -8,6 +8,9 @@ url = "https://www.piwheels.org/simple"
verify_ssl = true
name = "piwheels"
[requires]
python_version = "3.6"
[packages]
dateparser = "~=0.7.6"
django = "~=3.1.3"

30
Pipfile.lock generated
View File

@ -1,10 +1,12 @@
{
"_meta": {
"hash": {
"sha256": "e9792119f687757dd388e73827ddd4216910327d5b65a8b950d4b202679c36eb"
"sha256": "d6432a18280c092c108e998f00bcd377c0c55ef18f26cb0b8eb64f9618b9f383"
},
"pipfile-spec": 6,
"requires": {},
"requires": {
"python_version": "3.6"
},
"sources": [
{
"name": "pypi",
@ -701,6 +703,22 @@
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
"version": "==1.2.0"
},
"importlib-metadata": {
"hashes": [
"sha256:030f3b1bdb823ecbe4a9659e14cc861ce5af403fe99863bae173ec5fe00ab132",
"sha256:caeee3603f5dcf567864d1be9b839b0bcfdf1383e3e7be33ce2dead8144ff19c"
],
"markers": "python_version < '3.8'",
"version": "==2.1.0"
},
"importlib-resources": {
"hashes": [
"sha256:7b51f0106c8ec564b1bef3d9c588bc694ce2b92125bbb6278f4f2f5b54ec3592",
"sha256:a3d34a8464ce1d5d7c92b0ea4e921e696d86f2aa212e684451cb1482c8d84ed5"
],
"markers": "python_version < '3.7'",
"version": "==3.3.0"
},
"iniconfig": {
"hashes": [
"sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3",
@ -1012,6 +1030,14 @@
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
"version": "==20.2.1"
},
"zipp": {
"hashes": [
"sha256:102c24ef8f171fd729d46599845e95c7ab894a4cf45f5de11a44cc7444fb1108",
"sha256:ed5eee1974372595f9e416cc7bbeeb12335201d8081ca8a0743c954d4446e5cb"
],
"markers": "python_version < '3.8'",
"version": "==3.4.0"
}
}
}

30
src/documents/tests/test_api.py
View File

@ -1,41 +1,24 @@
import os
import shutil
import tempfile
from unittest import mock
from django.contrib.auth.models import User
from django.test import override_settings
from pathvalidate import ValidationError
from rest_framework.test import APITestCase
from documents.models import Document, Correspondent, DocumentType, Tag
from documents.tests.utils import setup_directories, remove_dirs
class DocumentApiTest(APITestCase):
def setUp(self):
self.scratch_dir = tempfile.mkdtemp()
self.media_dir = tempfile.mkdtemp()
self.originals_dir = os.path.join(self.media_dir, "documents", "originals")
self.thumbnail_dir = os.path.join(self.media_dir, "documents", "thumbnails")
os.makedirs(self.originals_dir, exist_ok=True)
os.makedirs(self.thumbnail_dir, exist_ok=True)
override_settings(
SCRATCH_DIR=self.scratch_dir,
MEDIA_ROOT=self.media_dir,
ORIGINALS_DIR=self.originals_dir,
THUMBNAIL_DIR=self.thumbnail_dir
).enable()
self.dirs = setup_directories()
self.addCleanup(remove_dirs, self.dirs)
user = User.objects.create_superuser(username="temp_admin")
self.client.force_login(user=user)
def tearDown(self):
shutil.rmtree(self.scratch_dir, ignore_errors=True)
shutil.rmtree(self.media_dir, ignore_errors=True)
def testDocuments(self):
response = self.client.get("/api/documents/").data
@ -88,7 +71,7 @@ class DocumentApiTest(APITestCase):
def test_document_actions(self):
_, filename = tempfile.mkstemp(dir=self.originals_dir)
_, filename = tempfile.mkstemp(dir=self.dirs.originals_dir)
content = b"This is a test"
content_thumbnail = b"thumbnail content"
@ -98,7 +81,7 @@ class DocumentApiTest(APITestCase):
doc = Document.objects.create(title="none", filename=os.path.basename(filename), mime_type="application/pdf")
with open(os.path.join(self.thumbnail_dir, "{:07d}.png".format(doc.pk)), "wb") as f:
with open(os.path.join(self.dirs.thumbnail_dir, "{:07d}.png".format(doc.pk)), "wb") as f:
f.write(content_thumbnail)
response = self.client.get('/api/documents/{}/download/'.format(doc.pk))
@ -227,7 +210,8 @@ class DocumentApiTest(APITestCase):
m.assert_called_once()
self.assertEqual(m.call_args.kwargs['override_filename'], "simple.pdf")
args, kwargs = m.call_args
self.assertEqual(kwargs['override_filename'], "simple.pdf")
@mock.patch("documents.forms.async_task")
def test_upload_invalid_form(self, m):

1
src/documents/tests/test_classifier.py
View File

@ -11,7 +11,6 @@ from documents.models import Correspondent, Document, Tag, DocumentType
class TestClassifier(TestCase):
def setUp(self):
self.classifier = DocumentClassifier()
def generate_test_data(self):

26
src/documents/tests/test_consumer.py
View File

@ -1,12 +1,12 @@
import os
import re
import shutil
import tempfile
from unittest import mock
from unittest.mock import MagicMock
from django.test import TestCase, override_settings
from .utils import setup_directories, remove_dirs
from ..consumer import Consumer, ConsumerError
from ..models import FileInfo, Tag, Correspondent, DocumentType, Document
from ..parsers import DocumentParser, ParseError
@ -411,23 +411,14 @@ def fake_magic_from_file(file, mime=False):
class TestConsumer(TestCase):
def make_dummy_parser(self, path, logging_group):
return DummyParser(path, logging_group, self.scratch_dir)
return DummyParser(path, logging_group, self.dirs.scratch_dir)
def make_faulty_parser(self, path, logging_group):
return FaultyParser(path, logging_group, self.scratch_dir)
return FaultyParser(path, logging_group, self.dirs.scratch_dir)
def setUp(self):
self.scratch_dir = tempfile.mkdtemp()
self.media_dir = tempfile.mkdtemp()
self.consumption_dir = tempfile.mkdtemp()
override_settings(
SCRATCH_DIR=self.scratch_dir,
MEDIA_ROOT=self.media_dir,
ORIGINALS_DIR=os.path.join(self.media_dir, "documents", "originals"),
THUMBNAIL_DIR=os.path.join(self.media_dir, "documents", "thumbnails"),
CONSUMPTION_DIR=self.consumption_dir
).enable()
self.dirs = setup_directories()
self.addCleanup(remove_dirs, self.dirs)
patcher = mock.patch("documents.parsers.document_consumer_declaration.send")
m = patcher.start()
@ -441,13 +432,8 @@ class TestConsumer(TestCase):
self.consumer = Consumer()
def tearDown(self):
shutil.rmtree(self.scratch_dir, ignore_errors=True)
shutil.rmtree(self.media_dir, ignore_errors=True)
shutil.rmtree(self.consumption_dir, ignore_errors=True)
def get_test_file(self):
fd, f = tempfile.mkstemp(suffix=".pdf", dir=self.scratch_dir)
fd, f = tempfile.mkstemp(suffix=".pdf", dir=self.dirs.scratch_dir)
return f
def testNormalOperation(self):

37
src/documents/tests/test_management_consumer.py
View File

@ -1,7 +1,6 @@
import filecmp
import os
import shutil
import tempfile
from threading import Thread
from time import sleep
from unittest import mock
@ -11,6 +10,7 @@ from django.test import TestCase, override_settings
from documents.consumer import ConsumerError
from documents.management.commands import document_consumer
from documents.tests.utils import setup_directories, remove_dirs
class ConsumerThread(Thread):
@ -41,9 +41,8 @@ class TestConsumer(TestCase):
self.task_mock = patcher.start()
self.addCleanup(patcher.stop)
self.consume_dir = tempfile.mkdtemp()
override_settings(CONSUMPTION_DIR=self.consume_dir).enable()
self.dirs = setup_directories()
self.addCleanup(remove_dirs, self.dirs)
def t_start(self):
self.t = ConsumerThread()
@ -94,25 +93,29 @@ class TestConsumer(TestCase):
def test_consume_file(self):
self.t_start()
f = os.path.join(self.consume_dir, "my_file.pdf")
f = os.path.join(self.dirs.consumption_dir, "my_file.pdf")
shutil.copy(self.sample_file, f)
self.wait_for_task_mock_call()
self.task_mock.assert_called_once()
self.assertEqual(self.task_mock.call_args.args[1], f)
args, kwargs = self.task_mock.call_args
self.assertEqual(args[1], f)
@override_settings(CONSUMER_POLLING=1)
def test_consume_file_polling(self):
self.test_consume_file()
def test_consume_existing_file(self):
f = os.path.join(self.consume_dir, "my_file.pdf")
f = os.path.join(self.dirs.consumption_dir, "my_file.pdf")
shutil.copy(self.sample_file, f)
self.t_start()
self.task_mock.assert_called_once()
self.assertEqual(self.task_mock.call_args.args[1], f)
args, kwargs = self.task_mock.call_args
self.assertEqual(args[1], f)
@override_settings(CONSUMER_POLLING=1)
def test_consume_existing_file_polling(self):
@ -125,7 +128,7 @@ class TestConsumer(TestCase):
self.t_start()
fname = os.path.join(self.consume_dir, "my_file.pdf")
fname = os.path.join(self.dirs.consumption_dir, "my_file.pdf")
self.slow_write_file(fname)
@ -135,7 +138,8 @@ class TestConsumer(TestCase):
self.task_mock.assert_called_once()
self.assertEqual(self.task_mock.call_args.args[1], fname)
args, kwargs = self.task_mock.call_args
self.assertEqual(args[1], fname)
@override_settings(CONSUMER_POLLING=1)
def test_slow_write_pdf_polling(self):
@ -148,8 +152,8 @@ class TestConsumer(TestCase):
self.t_start()
fname = os.path.join(self.consume_dir, "my_file.~df")
fname2 = os.path.join(self.consume_dir, "my_file.pdf")
fname = os.path.join(self.dirs.consumption_dir, "my_file.~df")
fname2 = os.path.join(self.dirs.consumption_dir, "my_file.pdf")
self.slow_write_file(fname)
shutil.move(fname, fname2)
@ -157,7 +161,9 @@ class TestConsumer(TestCase):
self.wait_for_task_mock_call()
self.task_mock.assert_called_once()
self.assertEqual(self.task_mock.call_args.args[1], fname2)
args, kwargs = self.task_mock.call_args
self.assertEqual(args[1], fname2)
error_logger.assert_not_called()
@ -172,13 +178,14 @@ class TestConsumer(TestCase):
self.t_start()
fname = os.path.join(self.consume_dir, "my_file.pdf")
fname = os.path.join(self.dirs.consumption_dir, "my_file.pdf")
self.slow_write_file(fname, incomplete=True)
self.wait_for_task_mock_call()
self.task_mock.assert_called_once()
self.assertEqual(self.task_mock.call_args.args[1], fname)
args, kwargs = self.task_mock.call_args
self.assertEqual(args[1], fname)
# assert that we have an error logged with this invalid file.
error_logger.assert_called_once()

9
src/documents/tests/test_matchables.py
View File

@ -1,3 +1,5 @@
import shutil
import tempfile
from random import randint
from django.contrib.admin.models import LogEntry
@ -215,6 +217,13 @@ class TestDocumentConsumptionFinishedSignal(TestCase):
self.doc_contains = Document.objects.create(
content="I contain the keyword.", mime_type="application/pdf")
self.index_dir = tempfile.mkdtemp()
# TODO: we should not need the index here.
override_settings(INDEX_DIR=self.index_dir).enable()
def tearDown(self) -> None:
shutil.rmtree(self.index_dir, ignore_errors=True)
def test_tag_applied_any(self):
t1 = Tag.objects.create(
name="test", match="keyword", matching_algorithm=Tag.MATCH_ANY)

41
src/documents/tests/utils.py Normal file
View File

@ -0,0 +1,41 @@
import os
import shutil
import tempfile
from collections import namedtuple
from django.test import override_settings
def setup_directories():
dirs = namedtuple("Dirs", ())
dirs.data_dir = tempfile.mkdtemp()
dirs.scratch_dir = tempfile.mkdtemp()
dirs.media_dir = tempfile.mkdtemp()
dirs.consumption_dir = tempfile.mkdtemp()
dirs.index_dir = os.path.join(dirs.data_dir, "documents", "originals")
dirs.originals_dir = os.path.join(dirs.media_dir, "documents", "originals")
dirs.thumbnail_dir = os.path.join(dirs.media_dir, "documents", "thumbnails")
os.makedirs(dirs.index_dir)
os.makedirs(dirs.originals_dir)
os.makedirs(dirs.thumbnail_dir)
override_settings(
DATA_DIR=dirs.data_dir,
SCRATCH_DIR=dirs.scratch_dir,
MEDIA_ROOT=dirs.media_dir,
ORIGINALS_DIR=dirs.originals_dir,
THUMBNAIL_DIR=dirs.thumbnail_dir,
CONSUMPTION_DIR=dirs.consumption_dir,
INDEX_DIR=dirs.index_dir
).enable()
return dirs
def remove_dirs(dirs):
shutil.rmtree(dirs.media_dir, ignore_errors=True)
shutil.rmtree(dirs.data_dir, ignore_errors=True)
shutil.rmtree(dirs.scratch_dir, ignore_errors=True)
shutil.rmtree(dirs.consumption_dir, ignore_errors=True)