paradizelost/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2025-07-28 18:24:38 -05:00
Code Issues Packages Projects Releases Wiki Activity

checking file types against parsers in the consumer.

Browse Source
This commit is contained in:
jonaswinkler
2020-12-01 15:26:05 +01:00
parent daf2e345c4
commit e4eeb29f54
5 changed files with 127 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/documents/tests/test_consumer.py
View File

@@ -552,7 +552,7 @@ class TestConsumer(DirectoriesMixin, TestCase):
try:
self.consumer.try_consume_file(self.get_test_file())
except ConsumerError as e:
self.assertTrue("File extension .pdf does not map to any" in str(e))
self.assertTrue("No parsers abvailable for" in str(e))
return
self.fail("Should throw exception")

204
src/documents/tests/test_management_consumer.py
View File

@@ -34,12 +34,12 @@ def chunked(size, source):
yield source[i:i+size]
class TestConsumer(DirectoriesMixin, TransactionTestCase):
class ConsumerMixin:
sample_file = os.path.join(os.path.dirname(__file__), "samples", "simple.pdf")
def setUp(self) -> None:
super(TestConsumer, self).setUp()
super(ConsumerMixin, self).setUp()
self.t = None
patcher = mock.patch("documents.management.commands.document_consumer.async_task")
self.task_mock = patcher.start()
@@ -58,7 +58,7 @@ class TestConsumer(DirectoriesMixin, TransactionTestCase):
# wait for the consumer to exit.
self.t.join()
super(TestConsumer, self).tearDown()
super(ConsumerMixin, self).tearDown()
def wait_for_task_mock_call(self):
n = 0
@@ -69,7 +69,6 @@ class TestConsumer(DirectoriesMixin, TransactionTestCase):
return
n += 1
sleep(0.1)
self.fail("async_task was never called")
# A bogus async_task that will simply check the file for
# completeness and raise an exception otherwise.
@@ -96,6 +95,9 @@ class TestConsumer(DirectoriesMixin, TransactionTestCase):
sleep(0.1)
print("file completed.")
class TestConsumer(DirectoriesMixin, ConsumerMixin, TransactionTestCase):
def test_consume_file(self):
self.t_start()
@@ -109,9 +111,15 @@ class TestConsumer(DirectoriesMixin, TransactionTestCase):
args, kwargs = self.task_mock.call_args
self.assertEqual(args[1], f)
@override_settings(CONSUMER_POLLING=1)
def test_consume_file_polling(self):
self.test_consume_file()
def test_consume_file_invalid_ext(self):
self.t_start()
f = os.path.join(self.dirs.consumption_dir, "my_file.wow")
shutil.copy(self.sample_file, f)
self.wait_for_task_mock_call()
self.task_mock.assert_not_called()
def test_consume_existing_file(self):
f = os.path.join(self.dirs.consumption_dir, "my_file.pdf")
@@ -123,12 +131,101 @@ class TestConsumer(DirectoriesMixin, TransactionTestCase):
args, kwargs = self.task_mock.call_args
self.assertEqual(args[1], f)
@override_settings(CONSUMER_POLLING=1)
def test_consume_existing_file_polling(self):
self.test_consume_existing_file()
@mock.patch("documents.management.commands.document_consumer.logger.error")
def test_slow_write_pdf(self, error_logger):
@override_settings(CONSUMER_RECURSIVE=1)
@override_settings(CONSUMER_SUBDIRS_AS_TAGS=1)
self.task_mock.side_effect = self.bogus_task
self.t_start()
fname = os.path.join(self.dirs.consumption_dir, "my_file.pdf")
self.slow_write_file(fname)
self.wait_for_task_mock_call()
error_logger.assert_not_called()
self.task_mock.assert_called_once()
args, kwargs = self.task_mock.call_args
self.assertEqual(args[1], fname)
@mock.patch("documents.management.commands.document_consumer.logger.error")
def test_slow_write_and_move(self, error_logger):
self.task_mock.side_effect = self.bogus_task
self.t_start()
fname = os.path.join(self.dirs.consumption_dir, "my_file.~df")
fname2 = os.path.join(self.dirs.consumption_dir, "my_file.pdf")
self.slow_write_file(fname)
shutil.move(fname, fname2)
self.wait_for_task_mock_call()
self.task_mock.assert_called_once()
args, kwargs = self.task_mock.call_args
self.assertEqual(args[1], fname2)
error_logger.assert_not_called()
@mock.patch("documents.management.commands.document_consumer.logger.error")
def test_slow_write_incomplete(self, error_logger):
self.task_mock.side_effect = self.bogus_task
self.t_start()
fname = os.path.join(self.dirs.consumption_dir, "my_file.pdf")
self.slow_write_file(fname, incomplete=True)
self.wait_for_task_mock_call()
self.task_mock.assert_called_once()
args, kwargs = self.task_mock.call_args
self.assertEqual(args[1], fname)
# assert that we have an error logged with this invalid file.
error_logger.assert_called_once()
@override_settings(CONSUMPTION_DIR="does_not_exist")
def test_consumption_directory_invalid(self):
self.assertRaises(CommandError, call_command, 'document_consumer', '--oneshot')
@override_settings(CONSUMPTION_DIR="")
def test_consumption_directory_unset(self):
self.assertRaises(CommandError, call_command, 'document_consumer', '--oneshot')
@override_settings(CONSUMER_POLLING=1)
class TestConsumerPolling(TestConsumer):
# just do all the tests with polling
pass
@override_settings(CONSUMER_RECURSIVE=True)
class TestConsumerRecursive(TestConsumer):
# just do all the tests with recursive
pass
@override_settings(CONSUMER_RECURSIVE=True)
@override_settings(CONSUMER_POLLING=1)
class TestConsumerRecursivePolling(TestConsumer):
# just do all the tests with polling and recursive
pass
class TestConsumerTags(DirectoriesMixin, ConsumerMixin, TransactionTestCase):
@override_settings(CONSUMER_RECURSIVE=True)
@override_settings(CONSUMER_SUBDIRS_AS_TAGS=True)
def test_consume_file_with_path_tags(self):
tag_names = ("existingTag", "Space Tag")
@@ -163,86 +260,3 @@ class TestConsumer(DirectoriesMixin, TransactionTestCase):
@override_settings(CONSUMER_POLLING=1)
def test_consume_file_with_path_tags_polling(self):
self.test_consume_file_with_path_tags()
@mock.patch("documents.management.commands.document_consumer.logger.error")
def test_slow_write_pdf(self, error_logger):
self.task_mock.side_effect = self.bogus_task
self.t_start()
fname = os.path.join(self.dirs.consumption_dir, "my_file.pdf")
self.slow_write_file(fname)
self.wait_for_task_mock_call()
error_logger.assert_not_called()
self.task_mock.assert_called_once()
args, kwargs = self.task_mock.call_args
self.assertEqual(args[1], fname)
@override_settings(CONSUMER_POLLING=1)
def test_slow_write_pdf_polling(self):
self.test_slow_write_pdf()
@mock.patch("documents.management.commands.document_consumer.logger.error")
def test_slow_write_and_move(self, error_logger):
self.task_mock.side_effect = self.bogus_task
self.t_start()
fname = os.path.join(self.dirs.consumption_dir, "my_file.~df")
fname2 = os.path.join(self.dirs.consumption_dir, "my_file.pdf")
self.slow_write_file(fname)
shutil.move(fname, fname2)
self.wait_for_task_mock_call()
self.task_mock.assert_called_once()
args, kwargs = self.task_mock.call_args
self.assertEqual(args[1], fname2)
error_logger.assert_not_called()
@override_settings(CONSUMER_POLLING=1)
def test_slow_write_and_move_polling(self):
self.test_slow_write_and_move()
@mock.patch("documents.management.commands.document_consumer.logger.error")
def test_slow_write_incomplete(self, error_logger):
self.task_mock.side_effect = self.bogus_task
self.t_start()
fname = os.path.join(self.dirs.consumption_dir, "my_file.pdf")
self.slow_write_file(fname, incomplete=True)
self.wait_for_task_mock_call()
self.task_mock.assert_called_once()
args, kwargs = self.task_mock.call_args
self.assertEqual(args[1], fname)
# assert that we have an error logged with this invalid file.
error_logger.assert_called_once()
@override_settings(CONSUMER_POLLING=1)
def test_slow_write_incomplete_polling(self):
self.test_slow_write_incomplete()
@override_settings(CONSUMPTION_DIR="does_not_exist")
def test_consumption_directory_invalid(self):
self.assertRaises(CommandError, call_command, 'document_consumer', '--oneshot')
@override_settings(CONSUMPTION_DIR="")
def test_consumption_directory_unset(self):
self.assertRaises(CommandError, call_command, 'document_consumer', '--oneshot')