run the polling file change checks on individual threads to speed up queueing of new files

src/documents/management/commands/document_consumer.py

@@ -1,6 +1,7 @@
 import logging
 import os
 from pathlib import Path
+from threading import Thread
 from time import sleep
 
 from django.conf import settings
@@ -57,6 +58,7 @@ def _consume(filepath):
         logger.exception("Error creating tags from path")
 
     try:
+        logger.info(f"Adding {filepath} to the task queue.")
         async_task("documents.tasks.consume_file",
                    filepath,
                    override_tag_ids=tag_ids if tag_ids else None,
@@ -69,6 +71,7 @@ def _consume(filepath):
 
 
 def _consume_wait_unmodified(file):
+    logger.debug(f"Waiting for file {file} to remain unmodified")
     mtime = -1
     current_try = 0
     while current_try < settings.CONSUMER_POLLING_RETRY_COUNT:
@@ -91,10 +94,14 @@ def _consume_wait_unmodified(file):
 class Handler(FileSystemEventHandler):
 
     def on_created(self, event):
-        _consume_wait_unmodified(event.src_path)
+        Thread(
+            target=_consume_wait_unmodified, args=(event.src_path,)
+        ).start()
 
     def on_moved(self, event):
-        _consume_wait_unmodified(event.dest_path)
+        Thread(
+            target=_consume_wait_unmodified, args=(event.dest_path,)
+        ).start()
 
 
 class Command(BaseCommand):

src/paperless/settings.py

@@ -428,7 +428,7 @@ CONSUMER_POLLING = int(os.getenv("PAPERLESS_CONSUMER_POLLING", 0))
 CONSUMER_POLLING_DELAY = int(os.getenv("PAPERLESS_CONSUMER_POLLING_DELAY", 5))
 
 CONSUMER_POLLING_RETRY_COUNT = int(
-    os.getenv("PAPERLESS_CONSUMER_POLLING_RETRY_COUNT", 30 / CONSUMER_POLLING_DELAY)
+    os.getenv("PAPERLESS_CONSUMER_POLLING_RETRY_COUNT", 5)
 )
 
 CONSUMER_DELETE_DUPLICATES = __get_boolean("PAPERLESS_CONSUMER_DELETE_DUPLICATES")