paradizelost/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2025-10-30 03:56:23 -05:00
Code Issues Packages Projects Releases Wiki Activity

alter defaults for workers and threads to allow more parallel tasks #446

Browse Source
This commit is contained in:
jonaswinkler
2021-01-26 22:10:43 +01:00
parent f451b69ef3
commit f36115ee37
3 changed files with 13 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
docs/configuration.rst
View File

@@ -376,25 +376,24 @@ PAPERLESS_THREADS_PER_WORKER=<num>
use a higher thread per worker count. use a higher thread per worker count.
The default is a balance between the two, according to your CPU core count, The default is a balance between the two, according to your CPU core count,
with a slight favor towards threads per worker, and leaving at least one core with a slight favor towards threads per worker:
free for other tasks:
+----------------+---------+---------+ +----------------+---------+---------+
| CPU core count | Workers | Threads | | CPU core count | Workers | Threads |
+----------------+---------+---------+ +----------------+---------+---------+
| 1 | 1 | 1 | | 1 | 1 | 1 |
+----------------+---------+---------+ +----------------+---------+---------+
| 2 | 1 | 1 | | 2 | 2 | 1 |
+----------------+---------+---------+ +----------------+---------+---------+
| 4 | 1 | 3 | | 4 | 2 | 2 |
+----------------+---------+---------+ +----------------+---------+---------+
| 6 | 2 | 2 | | 6 | 2 | 3 |
+----------------+---------+---------+ +----------------+---------+---------+
| 8 | 2 | 3 | | 8 | 2 | 4 |
+----------------+---------+---------+ +----------------+---------+---------+
| 12 | 3 | 3 | | 12 | 3 | 4 |
+----------------+---------+---------+ +----------------+---------+---------+
| 16 | 3 | 5 | | 16 | 4 | 4 |
+----------------+---------+---------+ +----------------+---------+---------+
If you only specify PAPERLESS_TASK_WORKERS, paperless will adjust If you only specify PAPERLESS_TASK_WORKERS, paperless will adjust

4
src/documents/tests/test_settings.py
View File

@@ -20,7 +20,7 @@ class TestSettings(TestCase):
self.assertEqual(default_threads, 1) self.assertEqual(default_threads, 1)
def test_workers_threads(self): def test_workers_threads(self):
for i in range(2, 64): for i in range(1, 64):
with mock.patch("paperless.settings.multiprocessing.cpu_count") as cpu_count: with mock.patch("paperless.settings.multiprocessing.cpu_count") as cpu_count:
cpu_count.return_value = i cpu_count.return_value = i
@@ -31,4 +31,4 @@ class TestSettings(TestCase):
self.assertTrue(default_workers >= 1) self.assertTrue(default_workers >= 1)
self.assertTrue(default_threads >= 1) self.assertTrue(default_threads >= 1)
self.assertTrue(default_workers * default_threads < i, f"{i}") self.assertTrue(default_workers * default_threads <= i, f"{i}")

6
src/paperless/settings.py
View File

@@ -354,8 +354,10 @@ LOGGING = {
def default_task_workers(): def default_task_workers():
# always leave one core open # always leave one core open
available_cores = max(multiprocessing.cpu_count() - 1, 1) available_cores = max(multiprocessing.cpu_count(), 1)
try: try:
if available_cores < 4:
return available_cores
return max( return max(
math.floor(math.sqrt(available_cores)), math.floor(math.sqrt(available_cores)),
1 1
@@ -376,7 +378,7 @@ Q_CLUSTER = {
def default_threads_per_worker(task_workers): def default_threads_per_worker(task_workers):
# always leave one core open # always leave one core open
available_cores = max(multiprocessing.cpu_count() - 1, 1) available_cores = max(multiprocessing.cpu_count(), 1)
try: try:
return max( return max(
math.floor(available_cores / task_workers), math.floor(available_cores / task_workers),