paradizelost/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2025-04-02 13:45:10 -05:00
Code Issues Packages Projects Releases Wiki Activity

run the polling file change checks on individual threads to speed up queueing of new files

Browse Source
This commit is contained in:
jonaswinkler 2021-02-21 12:43:55 +01:00
parent 0453787d38
commit bac4a63cc8
2 changed files with 10 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/documents/management/commands/document_consumer.py
View File

@ -1,6 +1,7 @@
import logging import logging
import os import os
from pathlib import Path from pathlib import Path
from threading import Thread
from time import sleep from time import sleep
from django.conf import settings from django.conf import settings
@ -57,6 +58,7 @@ def _consume(filepath):
logger.exception("Error creating tags from path") logger.exception("Error creating tags from path")
try: try:
logger.info(f"Adding {filepath} to the task queue.")
async_task("documents.tasks.consume_file", async_task("documents.tasks.consume_file",
filepath, filepath,
override_tag_ids=tag_ids if tag_ids else None, override_tag_ids=tag_ids if tag_ids else None,
@ -69,6 +71,7 @@ def _consume(filepath):
def _consume_wait_unmodified(file): def _consume_wait_unmodified(file):
logger.debug(f"Waiting for file {file} to remain unmodified")
mtime = -1 mtime = -1
current_try = 0 current_try = 0
while current_try < settings.CONSUMER_POLLING_RETRY_COUNT: while current_try < settings.CONSUMER_POLLING_RETRY_COUNT:
@ -91,10 +94,14 @@ def _consume_wait_unmodified(file):
class Handler(FileSystemEventHandler): class Handler(FileSystemEventHandler):
def on_created(self, event): def on_created(self, event):
_consume_wait_unmodified(event.src_path) Thread(
target=_consume_wait_unmodified, args=(event.src_path,)
).start()
def on_moved(self, event): def on_moved(self, event):
_consume_wait_unmodified(event.dest_path) Thread(
target=_consume_wait_unmodified, args=(event.dest_path,)
).start()
class Command(BaseCommand): class Command(BaseCommand):

2
src/paperless/settings.py
View File

@ -428,7 +428,7 @@ CONSUMER_POLLING = int(os.getenv("PAPERLESS_CONSUMER_POLLING", 0))
CONSUMER_POLLING_DELAY = int(os.getenv("PAPERLESS_CONSUMER_POLLING_DELAY", 5)) CONSUMER_POLLING_DELAY = int(os.getenv("PAPERLESS_CONSUMER_POLLING_DELAY", 5))
CONSUMER_POLLING_RETRY_COUNT = int( CONSUMER_POLLING_RETRY_COUNT = int(
os.getenv("PAPERLESS_CONSUMER_POLLING_RETRY_COUNT", 30 / CONSUMER_POLLING_DELAY) os.getenv("PAPERLESS_CONSUMER_POLLING_RETRY_COUNT", 5)
) )
CONSUMER_DELETE_DUPLICATES = __get_boolean("PAPERLESS_CONSUMER_DELETE_DUPLICATES") CONSUMER_DELETE_DUPLICATES = __get_boolean("PAPERLESS_CONSUMER_DELETE_DUPLICATES")