From 5355f2b027e448b1e142ed8ed2dbb0ce010d3dd5 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Tue, 19 Jan 2021 14:43:55 +0100
Subject: [PATCH] fixes #351
---
.../management/commands/document_consumer.py | 44 ++++++++++++++++---
.../tests/test_management_consumer.py | 27 +++++++++++-
2 files changed, 64 insertions(+), 7 deletions(-)
diff --git a/src/documents/management/commands/document_consumer.py b/src/documents/management/commands/document_consumer.py
index d0c045db2..595bd39cd 100644
--- a/src/documents/management/commands/document_consumer.py
+++ b/src/documents/management/commands/document_consumer.py
@@ -5,7 +5,6 @@ from time import sleep
from django.conf import settings
from django.core.management.base import BaseCommand, CommandError
-from django.utils.text import slugify
from django_q.tasks import async_task
from watchdog.events import FileSystemEventHandler
from watchdog.observers.polling import PollingObserver
@@ -71,6 +70,31 @@ def _consume(filepath):
"Error while consuming document: {}".format(e))
+def _test_inotify(directory):
+ if not INotify:
+ return False
+
+ test_file = os.path.join(directory, "__inotify_test_file__")
+ inotify = INotify()
+ descriptor = None
+ try:
+ inotify_flags = flags.CLOSE_WRITE | flags.MOVED_TO
+ descriptor = inotify.add_watch(directory, inotify_flags)
+ Path(test_file).touch()
+ events = inotify.read(timeout=1000)
+ return len(events) == 1
+ except Exception as e:
+ logger.warning(
+ f"Error while checking inotify availability: {str(e)}")
+ return False
+ finally:
+ if descriptor:
+ inotify.rm_watch(descriptor)
+ inotify.close()
+ if os.path.isfile(test_file):
+ os.unlink(test_file)
+
+
def _consume_wait_unmodified(file, num_tries=20, wait_time=1):
mtime = -1
current_try = 0
@@ -154,17 +178,25 @@ class Command(BaseCommand):
if options["oneshot"]:
return
- if settings.CONSUMER_POLLING == 0 and INotify:
- self.handle_inotify(directory, recursive)
+ if settings.CONSUMER_POLLING == 0:
+ if _test_inotify(directory):
+ self.handle_inotify(directory, recursive)
+ else:
+ logger.warning(
+ f"Inotify notifications are not available on {directory}, "
+ f"falling back to polling every 10 seconds")
+ self.handle_polling(
+ directory, recursive, 10)
else:
- self.handle_polling(directory, recursive)
+ self.handle_polling(
+ directory, recursive, settings.CONSUMER_POLLING)
logger.debug("Consumer exiting.")
- def handle_polling(self, directory, recursive):
+ def handle_polling(self, directory, recursive, timeout):
logging.getLogger(__name__).info(
f"Polling directory for changes: {directory}")
- self.observer = PollingObserver(timeout=settings.CONSUMER_POLLING)
+ self.observer = PollingObserver(timeout=timeout)
self.observer.schedule(Handler(), directory, recursive=recursive)
self.observer.start()
try:
diff --git a/src/documents/tests/test_management_consumer.py b/src/documents/tests/test_management_consumer.py
index b6a61a167..0680e7f56 100644
--- a/src/documents/tests/test_management_consumer.py
+++ b/src/documents/tests/test_management_consumer.py
@@ -7,8 +7,9 @@ from unittest import mock
from django.conf import settings
from django.core.management import call_command, CommandError
-from django.test import override_settings, TransactionTestCase
+from django.test import override_settings, TransactionTestCase, TestCase
+from documents.management.commands.document_consumer import _test_inotify
from documents.models import Tag
from documents.consumer import ConsumerError
from documents.management.commands import document_consumer
@@ -260,3 +261,27 @@ class TestConsumerTags(DirectoriesMixin, ConsumerMixin, TransactionTestCase):
@override_settings(CONSUMER_POLLING=1)
def test_consume_file_with_path_tags_polling(self):
self.test_consume_file_with_path_tags()
+
+
+class TestInotify(DirectoriesMixin, TestCase):
+
+ def test_inotify(self):
+ self.assertTrue(_test_inotify(self.dirs.consumption_dir))
+
+ @mock.patch("documents.management.commands.document_consumer.Path.touch")
+ def test_inotify_error(self, m):
+ m.side_effect = OSError("Permission error")
+ self.assertFalse(_test_inotify(self.dirs.consumption_dir))
+
+ @mock.patch("documents.management.commands.document_consumer.Command.handle_polling")
+ @mock.patch("documents.management.commands.document_consumer.Command.handle_inotify")
+ @mock.patch("documents.management.commands.document_consumer._test_inotify")
+ def test_polling_fallback(self, test_inotify, handle_inotify, handle_polling):
+ test_inotify.return_value = False
+
+ cmd = document_consumer.Command()
+ cmd.handle(directory=settings.CONSUMPTION_DIR, oneshot=False)
+
+ test_inotify.assert_called_once()
+ handle_polling.assert_called_once()
+ handle_inotify.assert_not_called()