mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2025-04-02 13:45:10 -05:00
Merge pull request #501 from stumpylog/configure-worker-retry
Adds settings for worker retry time
This commit is contained in:
commit
eb5e0e0b9b
@ -540,6 +540,10 @@ PAPERLESS_WORKER_TIMEOUT=<num>
|
|||||||
large documents within the default 1800 seconds. So extending this timeout
|
large documents within the default 1800 seconds. So extending this timeout
|
||||||
may prove to be useful on weak hardware setups.
|
may prove to be useful on weak hardware setups.
|
||||||
|
|
||||||
|
PAPERLESS_WORKER_RETRY=<num>
|
||||||
|
If PAPERLESS_WORKER_TIMEOUT has been configured, the retry time for a task can
|
||||||
|
also be configured. By default, this value will be set to 10s more than the
|
||||||
|
worker timeout. This value should never be set less than the worker timeout.
|
||||||
|
|
||||||
PAPERLESS_TIME_ZONE=<timezone>
|
PAPERLESS_TIME_ZONE=<timezone>
|
||||||
Set the time zone here.
|
Set the time zone here.
|
||||||
|
@ -3,6 +3,7 @@ import math
|
|||||||
import multiprocessing
|
import multiprocessing
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
|
from typing import Final
|
||||||
|
|
||||||
from concurrent_log_handler.queue import setup_logging_queues
|
from concurrent_log_handler.queue import setup_logging_queues
|
||||||
from django.utils.translation import gettext_lazy as _
|
from django.utils.translation import gettext_lazy as _
|
||||||
@ -29,7 +30,7 @@ elif os.path.exists("/usr/local/etc/paperless.conf"):
|
|||||||
os.environ["OMP_THREAD_LIMIT"] = "1"
|
os.environ["OMP_THREAD_LIMIT"] = "1"
|
||||||
|
|
||||||
|
|
||||||
def __get_boolean(key, default="NO"):
|
def __get_boolean(key: str, default: str = "NO") -> bool:
|
||||||
"""
|
"""
|
||||||
Return a boolean value based on whatever the user has supplied in the
|
Return a boolean value based on whatever the user has supplied in the
|
||||||
environment based on whether the value "looks like" it's True or not.
|
environment based on whether the value "looks like" it's True or not.
|
||||||
@ -37,6 +38,13 @@ def __get_boolean(key, default="NO"):
|
|||||||
return bool(os.getenv(key, default).lower() in ("yes", "y", "1", "t", "true"))
|
return bool(os.getenv(key, default).lower() in ("yes", "y", "1", "t", "true"))
|
||||||
|
|
||||||
|
|
||||||
|
def __get_int(key: str, default: int) -> int:
|
||||||
|
"""
|
||||||
|
Return an integer value based on the environment variable or a default
|
||||||
|
"""
|
||||||
|
return int(os.getenv(key, default))
|
||||||
|
|
||||||
|
|
||||||
# NEVER RUN WITH DEBUG IN PRODUCTION.
|
# NEVER RUN WITH DEBUG IN PRODUCTION.
|
||||||
DEBUG = __get_boolean("PAPERLESS_DEBUG", "NO")
|
DEBUG = __get_boolean("PAPERLESS_DEBUG", "NO")
|
||||||
|
|
||||||
@ -396,7 +404,7 @@ LOGGING = {
|
|||||||
# in total.
|
# in total.
|
||||||
|
|
||||||
|
|
||||||
def default_task_workers():
|
def default_task_workers() -> int:
|
||||||
# always leave one core open
|
# always leave one core open
|
||||||
available_cores = max(multiprocessing.cpu_count(), 1)
|
available_cores = max(multiprocessing.cpu_count(), 1)
|
||||||
try:
|
try:
|
||||||
@ -407,20 +415,29 @@ def default_task_workers():
|
|||||||
return 1
|
return 1
|
||||||
|
|
||||||
|
|
||||||
TASK_WORKERS = int(os.getenv("PAPERLESS_TASK_WORKERS", default_task_workers()))
|
TASK_WORKERS = __get_int("PAPERLESS_TASK_WORKERS", default_task_workers())
|
||||||
|
|
||||||
|
PAPERLESS_WORKER_TIMEOUT: Final[int] = __get_int("PAPERLESS_WORKER_TIMEOUT", 1800)
|
||||||
|
|
||||||
|
# Per django-q docs, timeout must be smaller than retry
|
||||||
|
# We default retry to 10s more than the timeout
|
||||||
|
PAPERLESS_WORKER_RETRY: Final[int] = __get_int(
|
||||||
|
"PAPERLESS_WORKER_RETRY",
|
||||||
|
PAPERLESS_WORKER_TIMEOUT + 10,
|
||||||
|
)
|
||||||
|
|
||||||
Q_CLUSTER = {
|
Q_CLUSTER = {
|
||||||
"name": "paperless",
|
"name": "paperless",
|
||||||
"catch_up": False,
|
"catch_up": False,
|
||||||
"recycle": 1,
|
"recycle": 1,
|
||||||
"retry": 1800,
|
"retry": PAPERLESS_WORKER_RETRY,
|
||||||
"timeout": int(os.getenv("PAPERLESS_WORKER_TIMEOUT", 1800)),
|
"timeout": PAPERLESS_WORKER_TIMEOUT,
|
||||||
"workers": TASK_WORKERS,
|
"workers": TASK_WORKERS,
|
||||||
"redis": os.getenv("PAPERLESS_REDIS", "redis://localhost:6379"),
|
"redis": os.getenv("PAPERLESS_REDIS", "redis://localhost:6379"),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def default_threads_per_worker(task_workers):
|
def default_threads_per_worker(task_workers) -> int:
|
||||||
# always leave one core open
|
# always leave one core open
|
||||||
available_cores = max(multiprocessing.cpu_count(), 1)
|
available_cores = max(multiprocessing.cpu_count(), 1)
|
||||||
try:
|
try:
|
||||||
|
Loading…
x
Reference in New Issue
Block a user