mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-01-02 14:28:14 -06:00
Compare commits
3 Commits
dependabot
...
feature-tr
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
61e0bd7eb6 | ||
|
|
d127361411 | ||
|
|
d45dee6d39 |
139
scripts/tag_perf_probe.py
Normal file
139
scripts/tag_perf_probe.py
Normal file
@@ -0,0 +1,139 @@
|
||||
# noqa: INP001
|
||||
|
||||
"""
|
||||
Ad-hoc script to gauge Tag + treenode performance locally.
|
||||
|
||||
It bootstraps a fresh SQLite DB in a temp folder (or PAPERLESS_DATA_DIR),
|
||||
uses locmem cache/redis to avoid external services, creates synthetic tags,
|
||||
and measures:
|
||||
- creation time
|
||||
- query count and wall time for the Tag list view
|
||||
|
||||
Usage:
|
||||
PAPERLESS_DEBUG=1 PAPERLESS_REDIS=locmem:// PYTHONPATH=src \
|
||||
PAPERLESS_DATA_DIR=/tmp/paperless-tags-probe \
|
||||
.venv/bin/python scripts/tag_perf_probe.py
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from collections.abc import Iterable
|
||||
from contextlib import contextmanager
|
||||
|
||||
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "paperless.settings")
|
||||
os.environ.setdefault("PAPERLESS_DEBUG", "1")
|
||||
os.environ.setdefault("PAPERLESS_REDIS", "locmem://")
|
||||
os.environ.setdefault("PYTHONPATH", "src")
|
||||
|
||||
import django
|
||||
|
||||
django.setup()
|
||||
|
||||
from django.contrib.auth import get_user_model # noqa: E402
|
||||
from django.core.management import call_command # noqa: E402
|
||||
from django.db import connection # noqa: E402
|
||||
from django.test.client import RequestFactory # noqa: E402
|
||||
from rest_framework.test import force_authenticate # noqa: E402
|
||||
from treenode.signals import no_signals # noqa: E402
|
||||
|
||||
from documents.models import Tag # noqa: E402
|
||||
from documents.views import TagViewSet # noqa: E402
|
||||
|
||||
User = get_user_model()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def count_queries():
|
||||
total = 0
|
||||
|
||||
def wrapper(execute, sql, params, many, context):
|
||||
nonlocal total
|
||||
total += 1
|
||||
return execute(sql, params, many, context)
|
||||
|
||||
with connection.execute_wrapper(wrapper):
|
||||
yield lambda: total
|
||||
|
||||
|
||||
def measure_list(tag_count: int, user) -> tuple[int, float]:
|
||||
"""Render Tag list with page_size=tag_count and return (queries, seconds)."""
|
||||
rf = RequestFactory()
|
||||
view = TagViewSet.as_view({"get": "list"})
|
||||
request = rf.get("/api/tags/", {"page_size": tag_count})
|
||||
force_authenticate(request, user=user)
|
||||
|
||||
with count_queries() as get_count:
|
||||
start = time.perf_counter()
|
||||
response = view(request)
|
||||
response.render()
|
||||
elapsed = time.perf_counter() - start
|
||||
total_queries = get_count()
|
||||
|
||||
return total_queries, elapsed
|
||||
|
||||
|
||||
def bulk_create_tags(count: int, parents: Iterable[Tag] | None = None) -> None:
|
||||
"""Create tags; when parents provided, create one child per parent."""
|
||||
if parents is None:
|
||||
Tag.objects.bulk_create([Tag(name=f"Flat {i}") for i in range(count)])
|
||||
return
|
||||
|
||||
children = []
|
||||
for p in parents:
|
||||
children.append(Tag(name=f"Child {p.id}", tn_parent=p))
|
||||
Tag.objects.bulk_create(children)
|
||||
|
||||
|
||||
def run():
|
||||
# Ensure tables exist when pointing at a fresh DATA_DIR.
|
||||
call_command("migrate", interactive=False, verbosity=0)
|
||||
|
||||
user, _ = User.objects.get_or_create(
|
||||
username="admin",
|
||||
defaults={"is_superuser": True, "is_staff": True},
|
||||
)
|
||||
|
||||
# Flat scenario
|
||||
Tag.objects.all().delete()
|
||||
start = time.perf_counter()
|
||||
bulk_create_tags(200)
|
||||
flat_create = time.perf_counter() - start
|
||||
q, t = measure_list(tag_count=200, user=user)
|
||||
print(f"Flat create 200 -> {flat_create:.2f}s; list -> {q} queries, {t:.2f}s") # noqa: T201
|
||||
|
||||
# Nested scenario (parents + 2 children each => 600 total)
|
||||
Tag.objects.all().delete()
|
||||
start = time.perf_counter()
|
||||
with no_signals(): # avoid per-save tree rebuild; rebuild once
|
||||
parents = Tag.objects.bulk_create([Tag(name=f"Parent {i}") for i in range(200)])
|
||||
children = []
|
||||
for p in parents:
|
||||
children.extend(
|
||||
Tag(name=f"Child {p.id}-{j}", tn_parent=p) for j in range(2)
|
||||
)
|
||||
Tag.objects.bulk_create(children)
|
||||
Tag.update_tree()
|
||||
nested_create = time.perf_counter() - start
|
||||
q, t = measure_list(tag_count=600, user=user)
|
||||
print(f"Nested create 600 -> {nested_create:.2f}s; list -> {q} queries, {t:.2f}s") # noqa: T201
|
||||
|
||||
# Larger nested scenario (1 child per parent, 3000 total)
|
||||
Tag.objects.all().delete()
|
||||
start = time.perf_counter()
|
||||
with no_signals():
|
||||
parents = Tag.objects.bulk_create(
|
||||
[Tag(name=f"Parent {i}") for i in range(1500)],
|
||||
)
|
||||
bulk_create_tags(0, parents=parents)
|
||||
Tag.update_tree()
|
||||
big_create = time.perf_counter() - start
|
||||
q, t = measure_list(tag_count=3000, user=user)
|
||||
print(f"Nested create 3000 -> {big_create:.2f}s; list -> {q} queries, {t:.2f}s") # noqa: T201
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if "runserver" in sys.argv:
|
||||
print("Run directly: .venv/bin/python scripts/tag_perf_probe.py") # noqa: T201
|
||||
sys.exit(1)
|
||||
run()
|
||||
@@ -68,7 +68,7 @@
|
||||
"prettier-plugin-organize-imports": "^4.3.0",
|
||||
"ts-node": "~10.9.1",
|
||||
"typescript": "^5.8.3",
|
||||
"webpack": "^5.104.1"
|
||||
"webpack": "^5.103.0"
|
||||
},
|
||||
"packageManager": "pnpm@10.17.1",
|
||||
"pnpm": {
|
||||
|
||||
112
src-ui/pnpm-lock.yaml
generated
112
src-ui/pnpm-lock.yaml
generated
@@ -128,7 +128,7 @@ importers:
|
||||
version: 20.3.15(@angular/compiler@20.3.15)(typescript@5.8.3)
|
||||
'@codecov/webpack-plugin':
|
||||
specifier: ^1.9.1
|
||||
version: 1.9.1(webpack@5.104.1)
|
||||
version: 1.9.1(webpack@5.103.0)
|
||||
'@playwright/test':
|
||||
specifier: ^1.57.0
|
||||
version: 1.57.0
|
||||
@@ -175,8 +175,8 @@ importers:
|
||||
specifier: ^5.8.3
|
||||
version: 5.8.3
|
||||
webpack:
|
||||
specifier: ^5.104.1
|
||||
version: 5.104.1
|
||||
specifier: ^5.103.0
|
||||
version: 5.103.0
|
||||
|
||||
packages:
|
||||
|
||||
@@ -3423,8 +3423,8 @@ packages:
|
||||
base64-js@1.5.1:
|
||||
resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==}
|
||||
|
||||
baseline-browser-mapping@2.9.11:
|
||||
resolution: {integrity: sha512-Sg0xJUNDU1sJNGdfGWhVHX0kkZ+HWcvmVymJbj6NSgZZmW/8S9Y2HQ5euytnIgakgxN6papOAWiwDo1ctFDcoQ==}
|
||||
baseline-browser-mapping@2.9.4:
|
||||
resolution: {integrity: sha512-ZCQ9GEWl73BVm8bu5Fts8nt7MHdbt5vY9bP6WGnUh+r3l8M7CgfyTlwsgCbMC66BNxPr6Xoce3j66Ms5YUQTNA==}
|
||||
hasBin: true
|
||||
|
||||
batch@0.6.1:
|
||||
@@ -3530,8 +3530,8 @@ packages:
|
||||
resolution: {integrity: sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==}
|
||||
engines: {node: '>=10'}
|
||||
|
||||
caniuse-lite@1.0.30001762:
|
||||
resolution: {integrity: sha512-PxZwGNvH7Ak8WX5iXzoK1KPZttBXNPuaOvI2ZYU7NrlM+d9Ov+TUvlLOBNGzVXAntMSMMlJPd+jY6ovrVjSmUw==}
|
||||
caniuse-lite@1.0.30001759:
|
||||
resolution: {integrity: sha512-Pzfx9fOKoKvevQf8oCXoyNRQ5QyxJj+3O0Rqx2V5oxT61KGx8+n6hV/IUyJeifUci2clnmmKVpvtiqRzgiWjSw==}
|
||||
|
||||
canvas@3.0.0:
|
||||
resolution: {integrity: sha512-NtcIBY88FjymQy+g2g5qnuP5IslrbWCQ3A6rSr1PeuYxVRapRZ3BZCrDyAakvI6CuDYidgZaf55ygulFVwROdg==}
|
||||
@@ -3912,8 +3912,8 @@ packages:
|
||||
ee-first@1.1.1:
|
||||
resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==}
|
||||
|
||||
electron-to-chromium@1.5.267:
|
||||
resolution: {integrity: sha512-0Drusm6MVRXSOJpGbaSVgcQsuB4hEkMpHXaVstcPmhu5LIedxs1xNK/nIxmQIU/RPC0+1/o0AVZfBTkTNJOdUw==}
|
||||
electron-to-chromium@1.5.266:
|
||||
resolution: {integrity: sha512-kgWEglXvkEfMH7rxP5OSZZwnaDWT7J9EoZCujhnpLbfi0bbNtRkgdX2E3gt0Uer11c61qCYktB3hwkAS325sJg==}
|
||||
|
||||
emittery@0.13.1:
|
||||
resolution: {integrity: sha512-DeWwawk6r5yR9jFgnDKYt4sLS0LmHJJi3ZOnb5/JdbYwj3nW+FxQnHIjhBKz8YLC7oRNPVM9NQ47I3CVx34eqQ==}
|
||||
@@ -3946,8 +3946,8 @@ packages:
|
||||
end-of-stream@1.4.4:
|
||||
resolution: {integrity: sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==}
|
||||
|
||||
enhanced-resolve@5.18.4:
|
||||
resolution: {integrity: sha512-LgQMM4WXU3QI+SYgEc2liRgznaD5ojbmY3sb8LxyguVkIg5FxdpTkvk72te2R38/TGKxH634oLxXRGY6d7AP+Q==}
|
||||
enhanced-resolve@5.18.3:
|
||||
resolution: {integrity: sha512-d4lC8xfavMeBjzGr2vECC3fsGXziXZQyJxD868h2M/mBI3PwAuODxAkLkq5HYuvrPYcUtiLzsTo8U3PgX3Ocww==}
|
||||
engines: {node: '>=10.13.0'}
|
||||
|
||||
entities@4.5.0:
|
||||
@@ -3987,9 +3987,6 @@ packages:
|
||||
es-module-lexer@1.7.0:
|
||||
resolution: {integrity: sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==}
|
||||
|
||||
es-module-lexer@2.0.0:
|
||||
resolution: {integrity: sha512-5POEcUuZybH7IdmGsD8wlf0AI55wMecM9rVBTI/qEAy2c1kTOm3DjFYjrBdI2K3BaJjJYfYFeRtM0t9ssnRuxw==}
|
||||
|
||||
es-object-atoms@1.1.1:
|
||||
resolution: {integrity: sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==}
|
||||
engines: {node: '>= 0.4'}
|
||||
@@ -6293,8 +6290,8 @@ packages:
|
||||
resolution: {integrity: sha512-7NyxrTE4Anh8km8iEy7o0QYPs+0JKBTj5ZaqHg6B39erLg0qYXN3BijtShwbsNSvQ+LN75+KV+C4QR/f6Gwnpg==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
terser-webpack-plugin@5.3.16:
|
||||
resolution: {integrity: sha512-h9oBFCWrq78NyWWVcSwZarJkZ01c2AyGrzs1crmHZO3QUg9D61Wu4NPjBy69n7JqylFF5y+CsUZYmYEIZ3mR+Q==}
|
||||
terser-webpack-plugin@5.3.15:
|
||||
resolution: {integrity: sha512-PGkOdpRFK+rb1TzVz+msVhw4YMRT9txLF4kRqvJhGhCM324xuR3REBSHALN+l+sAhKUmz0aotnjp5D+P83mLhQ==}
|
||||
engines: {node: '>= 10.13.0'}
|
||||
peerDependencies:
|
||||
'@swc/core': '*'
|
||||
@@ -6566,8 +6563,8 @@ packages:
|
||||
unrs-resolver@1.11.1:
|
||||
resolution: {integrity: sha512-bSjt9pjaEBnNiGgc9rUiHGKv5l4/TGzDmYw3RhnkJGtLhbnnA/5qJj7x3dNDCRx/PJxu774LlH8lCOlB4hEfKg==}
|
||||
|
||||
update-browserslist-db@1.2.3:
|
||||
resolution: {integrity: sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==}
|
||||
update-browserslist-db@1.2.2:
|
||||
resolution: {integrity: sha512-E85pfNzMQ9jpKkA7+TJAi4TJN+tBCuWh5rUcS/sv6cFi+1q9LYDwDI5dpUL0u/73EElyQ8d3TEaeW4sPedBqYA==}
|
||||
hasBin: true
|
||||
peerDependencies:
|
||||
browserslist: '>= 4.21.0'
|
||||
@@ -6713,10 +6710,6 @@ packages:
|
||||
resolution: {integrity: sha512-c5EGNOiyxxV5qmTtAB7rbiXxi1ooX1pQKMLX/MIabJjRA0SJBQOjKF+KSVfHkr9U1cADPon0mRiVe/riyaiDUA==}
|
||||
engines: {node: '>=10.13.0'}
|
||||
|
||||
watchpack@2.5.0:
|
||||
resolution: {integrity: sha512-e6vZvY6xboSwLz2GD36c16+O/2Z6fKvIf4pOXptw2rY9MVwE/TXc6RGqxD3I3x0a28lwBY7DE+76uTPSsBrrCA==}
|
||||
engines: {node: '>=10.13.0'}
|
||||
|
||||
wbuf@1.7.3:
|
||||
resolution: {integrity: sha512-O84QOnr0icsbFGLS0O3bI5FswxzRr8/gHwWkDlQFskhSPryQXvrTMxjxGP4+iWYoauLoBvfDpkrOauZ+0iZpDA==}
|
||||
|
||||
@@ -6770,8 +6763,8 @@ packages:
|
||||
webpack-virtual-modules@0.6.2:
|
||||
resolution: {integrity: sha512-66/V2i5hQanC51vBQKPH4aI8NMAcBW59FVBs+rC7eGHupMyfn34q7rZIE+ETlJ+XTevqfUhVVBgSUNSW2flEUQ==}
|
||||
|
||||
webpack@5.104.1:
|
||||
resolution: {integrity: sha512-Qphch25abbMNtekmEGJmeRUhLDbe+QfiWTiqpKYkpCOWY64v9eyl+KRRLmqOFA2AvKPpc9DC6+u2n76tQLBoaA==}
|
||||
webpack@5.103.0:
|
||||
resolution: {integrity: sha512-HU1JOuV1OavsZ+mfigY0j8d1TgQgbZ6M+J75zDkpEAwYeXjWSqrGJtgnPblJjd/mAyTNQ7ygw0MiKOn6etz8yw==}
|
||||
engines: {node: '>=10.13.0'}
|
||||
hasBin: true
|
||||
peerDependencies:
|
||||
@@ -6801,12 +6794,10 @@ packages:
|
||||
whatwg-encoding@2.0.0:
|
||||
resolution: {integrity: sha512-p41ogyeMUrw3jWclHWTQg1k05DSVXPLcVxRTYsXUk+ZooOCZLcoYgPZ/HL/D/N+uQPOtcp1me1WhBEaX02mhWg==}
|
||||
engines: {node: '>=12'}
|
||||
deprecated: Use @exodus/bytes instead for a more spec-conformant and faster implementation
|
||||
|
||||
whatwg-encoding@3.1.1:
|
||||
resolution: {integrity: sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==}
|
||||
engines: {node: '>=18'}
|
||||
deprecated: Use @exodus/bytes instead for a more spec-conformant and faster implementation
|
||||
|
||||
whatwg-mimetype@3.0.0:
|
||||
resolution: {integrity: sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q==}
|
||||
@@ -7190,7 +7181,7 @@ snapshots:
|
||||
dependencies:
|
||||
'@ampproject/remapping': 2.3.0
|
||||
'@angular-devkit/architect': 0.2000.4(chokidar@4.0.3)
|
||||
'@angular-devkit/build-webpack': 0.2000.4(chokidar@4.0.3)(webpack-dev-server@5.2.1(webpack@5.104.1))(webpack@5.99.8(esbuild@0.25.5))
|
||||
'@angular-devkit/build-webpack': 0.2000.4(chokidar@4.0.3)(webpack-dev-server@5.2.1(webpack@5.103.0))(webpack@5.99.8(esbuild@0.25.5))
|
||||
'@angular-devkit/core': 20.0.4(chokidar@4.0.3)
|
||||
'@angular/build': 20.0.4(@angular/compiler-cli@20.3.15(@angular/compiler@20.3.15)(typescript@5.8.3))(@angular/compiler@20.3.15)(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1))(@angular/localize@20.3.15(@angular/compiler-cli@20.3.15(@angular/compiler@20.3.15)(typescript@5.8.3))(@angular/compiler@20.3.15))(@angular/platform-browser@20.3.15(@angular/common@20.3.15(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1))(rxjs@7.8.2))(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1)))(@types/node@24.10.1)(chokidar@4.0.3)(jiti@1.21.7)(less@4.3.0)(postcss@8.5.3)(terser@5.39.1)(tslib@2.8.1)(typescript@5.8.3)(yaml@2.7.0)
|
||||
'@angular/compiler-cli': 20.3.15(@angular/compiler@20.3.15)(typescript@5.8.3)
|
||||
@@ -7276,12 +7267,12 @@ snapshots:
|
||||
- webpack-cli
|
||||
- yaml
|
||||
|
||||
'@angular-devkit/build-webpack@0.2000.4(chokidar@4.0.3)(webpack-dev-server@5.2.1(webpack@5.104.1))(webpack@5.99.8(esbuild@0.25.5))':
|
||||
'@angular-devkit/build-webpack@0.2000.4(chokidar@4.0.3)(webpack-dev-server@5.2.1(webpack@5.103.0))(webpack@5.99.8(esbuild@0.25.5))':
|
||||
dependencies:
|
||||
'@angular-devkit/architect': 0.2000.4(chokidar@4.0.3)
|
||||
rxjs: 7.8.2
|
||||
webpack: 5.99.8(esbuild@0.25.5)
|
||||
webpack-dev-server: 5.2.1(webpack@5.104.1)
|
||||
webpack-dev-server: 5.2.1(webpack@5.103.0)
|
||||
transitivePeerDependencies:
|
||||
- chokidar
|
||||
|
||||
@@ -8441,11 +8432,11 @@ snapshots:
|
||||
unplugin: 1.16.1
|
||||
zod: 3.25.76
|
||||
|
||||
'@codecov/webpack-plugin@1.9.1(webpack@5.104.1)':
|
||||
'@codecov/webpack-plugin@1.9.1(webpack@5.103.0)':
|
||||
dependencies:
|
||||
'@codecov/bundler-plugin-core': 1.9.1
|
||||
unplugin: 1.16.1
|
||||
webpack: 5.104.1
|
||||
webpack: 5.103.0
|
||||
|
||||
'@cspotcode/source-map-support@0.8.1':
|
||||
dependencies:
|
||||
@@ -10384,7 +10375,7 @@ snapshots:
|
||||
autoprefixer@10.4.21(postcss@8.5.3):
|
||||
dependencies:
|
||||
browserslist: 4.28.1
|
||||
caniuse-lite: 1.0.30001762
|
||||
caniuse-lite: 1.0.30001759
|
||||
fraction.js: 4.3.7
|
||||
normalize-range: 0.1.2
|
||||
picocolors: 1.1.1
|
||||
@@ -10480,7 +10471,7 @@ snapshots:
|
||||
base64-js@1.5.1:
|
||||
optional: true
|
||||
|
||||
baseline-browser-mapping@2.9.11: {}
|
||||
baseline-browser-mapping@2.9.4: {}
|
||||
|
||||
batch@0.6.1: {}
|
||||
|
||||
@@ -10576,11 +10567,11 @@ snapshots:
|
||||
|
||||
browserslist@4.28.1:
|
||||
dependencies:
|
||||
baseline-browser-mapping: 2.9.11
|
||||
caniuse-lite: 1.0.30001762
|
||||
electron-to-chromium: 1.5.267
|
||||
baseline-browser-mapping: 2.9.4
|
||||
caniuse-lite: 1.0.30001759
|
||||
electron-to-chromium: 1.5.266
|
||||
node-releases: 2.0.27
|
||||
update-browserslist-db: 1.2.3(browserslist@4.28.1)
|
||||
update-browserslist-db: 1.2.2(browserslist@4.28.1)
|
||||
|
||||
bs-logger@0.2.6:
|
||||
dependencies:
|
||||
@@ -10635,7 +10626,7 @@ snapshots:
|
||||
|
||||
camelcase@6.3.0: {}
|
||||
|
||||
caniuse-lite@1.0.30001762: {}
|
||||
caniuse-lite@1.0.30001759: {}
|
||||
|
||||
canvas@3.0.0:
|
||||
dependencies:
|
||||
@@ -10977,7 +10968,7 @@ snapshots:
|
||||
|
||||
ee-first@1.1.1: {}
|
||||
|
||||
electron-to-chromium@1.5.267: {}
|
||||
electron-to-chromium@1.5.266: {}
|
||||
|
||||
emittery@0.13.1: {}
|
||||
|
||||
@@ -11003,7 +10994,7 @@ snapshots:
|
||||
once: 1.4.0
|
||||
optional: true
|
||||
|
||||
enhanced-resolve@5.18.4:
|
||||
enhanced-resolve@5.18.3:
|
||||
dependencies:
|
||||
graceful-fs: 4.2.11
|
||||
tapable: 2.3.0
|
||||
@@ -11033,8 +11024,6 @@ snapshots:
|
||||
|
||||
es-module-lexer@1.7.0: {}
|
||||
|
||||
es-module-lexer@2.0.0: {}
|
||||
|
||||
es-object-atoms@1.1.1:
|
||||
dependencies:
|
||||
es-errors: 1.3.0
|
||||
@@ -13934,7 +13923,7 @@ snapshots:
|
||||
minizlib: 3.1.0
|
||||
yallist: 5.0.0
|
||||
|
||||
terser-webpack-plugin@5.3.16(esbuild@0.25.5)(webpack@5.99.8(esbuild@0.25.5)):
|
||||
terser-webpack-plugin@5.3.15(esbuild@0.25.5)(webpack@5.99.8(esbuild@0.25.5)):
|
||||
dependencies:
|
||||
'@jridgewell/trace-mapping': 0.3.31
|
||||
jest-worker: 27.5.1
|
||||
@@ -13945,14 +13934,14 @@ snapshots:
|
||||
optionalDependencies:
|
||||
esbuild: 0.25.5
|
||||
|
||||
terser-webpack-plugin@5.3.16(webpack@5.104.1):
|
||||
terser-webpack-plugin@5.3.15(webpack@5.103.0):
|
||||
dependencies:
|
||||
'@jridgewell/trace-mapping': 0.3.31
|
||||
jest-worker: 27.5.1
|
||||
schema-utils: 4.3.3
|
||||
serialize-javascript: 6.0.2
|
||||
terser: 5.44.1
|
||||
webpack: 5.104.1
|
||||
webpack: 5.103.0
|
||||
|
||||
terser@5.39.1:
|
||||
dependencies:
|
||||
@@ -14209,7 +14198,7 @@ snapshots:
|
||||
'@unrs/resolver-binding-win32-ia32-msvc': 1.11.1
|
||||
'@unrs/resolver-binding-win32-x64-msvc': 1.11.1
|
||||
|
||||
update-browserslist-db@1.2.3(browserslist@4.28.1):
|
||||
update-browserslist-db@1.2.2(browserslist@4.28.1):
|
||||
dependencies:
|
||||
browserslist: 4.28.1
|
||||
escalade: 3.2.0
|
||||
@@ -14309,11 +14298,6 @@ snapshots:
|
||||
glob-to-regexp: 0.4.1
|
||||
graceful-fs: 4.2.11
|
||||
|
||||
watchpack@2.5.0:
|
||||
dependencies:
|
||||
glob-to-regexp: 0.4.1
|
||||
graceful-fs: 4.2.11
|
||||
|
||||
wbuf@1.7.3:
|
||||
dependencies:
|
||||
minimalistic-assert: 1.0.1
|
||||
@@ -14323,7 +14307,7 @@ snapshots:
|
||||
|
||||
webidl-conversions@7.0.0: {}
|
||||
|
||||
webpack-dev-middleware@7.4.2(webpack@5.104.1):
|
||||
webpack-dev-middleware@7.4.2(webpack@5.103.0):
|
||||
dependencies:
|
||||
colorette: 2.0.20
|
||||
memfs: 4.17.2
|
||||
@@ -14332,7 +14316,7 @@ snapshots:
|
||||
range-parser: 1.2.1
|
||||
schema-utils: 4.3.3
|
||||
optionalDependencies:
|
||||
webpack: 5.104.1
|
||||
webpack: 5.103.0
|
||||
|
||||
webpack-dev-middleware@7.4.2(webpack@5.99.8(esbuild@0.25.5)):
|
||||
dependencies:
|
||||
@@ -14345,7 +14329,7 @@ snapshots:
|
||||
optionalDependencies:
|
||||
webpack: 5.99.8(esbuild@0.25.5)
|
||||
|
||||
webpack-dev-server@5.2.1(webpack@5.104.1):
|
||||
webpack-dev-server@5.2.1(webpack@5.103.0):
|
||||
dependencies:
|
||||
'@types/bonjour': 3.5.13
|
||||
'@types/connect-history-api-fallback': 1.5.4
|
||||
@@ -14373,10 +14357,10 @@ snapshots:
|
||||
serve-index: 1.9.1
|
||||
sockjs: 0.3.24
|
||||
spdy: 4.0.2
|
||||
webpack-dev-middleware: 7.4.2(webpack@5.104.1)
|
||||
webpack-dev-middleware: 7.4.2(webpack@5.103.0)
|
||||
ws: 8.18.3
|
||||
optionalDependencies:
|
||||
webpack: 5.104.1
|
||||
webpack: 5.103.0
|
||||
transitivePeerDependencies:
|
||||
- bufferutil
|
||||
- debug
|
||||
@@ -14436,7 +14420,7 @@ snapshots:
|
||||
|
||||
webpack-virtual-modules@0.6.2: {}
|
||||
|
||||
webpack@5.104.1:
|
||||
webpack@5.103.0:
|
||||
dependencies:
|
||||
'@types/eslint-scope': 3.7.7
|
||||
'@types/estree': 1.0.8
|
||||
@@ -14448,8 +14432,8 @@ snapshots:
|
||||
acorn-import-phases: 1.0.4(acorn@8.15.0)
|
||||
browserslist: 4.28.1
|
||||
chrome-trace-event: 1.0.4
|
||||
enhanced-resolve: 5.18.4
|
||||
es-module-lexer: 2.0.0
|
||||
enhanced-resolve: 5.18.3
|
||||
es-module-lexer: 1.7.0
|
||||
eslint-scope: 5.1.1
|
||||
events: 3.3.0
|
||||
glob-to-regexp: 0.4.1
|
||||
@@ -14460,8 +14444,8 @@ snapshots:
|
||||
neo-async: 2.6.2
|
||||
schema-utils: 4.3.3
|
||||
tapable: 2.3.0
|
||||
terser-webpack-plugin: 5.3.16(webpack@5.104.1)
|
||||
watchpack: 2.5.0
|
||||
terser-webpack-plugin: 5.3.15(webpack@5.103.0)
|
||||
watchpack: 2.4.4
|
||||
webpack-sources: 3.3.3
|
||||
transitivePeerDependencies:
|
||||
- '@swc/core'
|
||||
@@ -14479,7 +14463,7 @@ snapshots:
|
||||
acorn: 8.15.0
|
||||
browserslist: 4.28.1
|
||||
chrome-trace-event: 1.0.4
|
||||
enhanced-resolve: 5.18.4
|
||||
enhanced-resolve: 5.18.3
|
||||
es-module-lexer: 1.7.0
|
||||
eslint-scope: 5.1.1
|
||||
events: 3.3.0
|
||||
@@ -14491,8 +14475,8 @@ snapshots:
|
||||
neo-async: 2.6.2
|
||||
schema-utils: 4.3.3
|
||||
tapable: 2.3.0
|
||||
terser-webpack-plugin: 5.3.16(esbuild@0.25.5)(webpack@5.99.8(esbuild@0.25.5))
|
||||
watchpack: 2.5.0
|
||||
terser-webpack-plugin: 5.3.15(esbuild@0.25.5)(webpack@5.99.8(esbuild@0.25.5))
|
||||
watchpack: 2.4.4
|
||||
webpack-sources: 3.3.3
|
||||
transitivePeerDependencies:
|
||||
- '@swc/core'
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
from django.apps import AppConfig
|
||||
from django.db.models.signals import post_delete
|
||||
from django.db.models.signals import post_save
|
||||
from django.utils.translation import gettext_lazy as _
|
||||
from treenode.signals import post_delete_treenode
|
||||
from treenode.signals import post_save_treenode
|
||||
|
||||
|
||||
class DocumentsConfig(AppConfig):
|
||||
@@ -8,12 +12,14 @@ class DocumentsConfig(AppConfig):
|
||||
verbose_name = _("Documents")
|
||||
|
||||
def ready(self):
|
||||
from documents.models import Tag
|
||||
from documents.signals import document_consumption_finished
|
||||
from documents.signals import document_updated
|
||||
from documents.signals.handlers import add_inbox_tags
|
||||
from documents.signals.handlers import add_to_index
|
||||
from documents.signals.handlers import run_workflows_added
|
||||
from documents.signals.handlers import run_workflows_updated
|
||||
from documents.signals.handlers import schedule_tag_tree_update
|
||||
from documents.signals.handlers import set_correspondent
|
||||
from documents.signals.handlers import set_document_type
|
||||
from documents.signals.handlers import set_storage_path
|
||||
@@ -28,6 +34,29 @@ class DocumentsConfig(AppConfig):
|
||||
document_consumption_finished.connect(run_workflows_added)
|
||||
document_updated.connect(run_workflows_updated)
|
||||
|
||||
# treenode updates the entire tree on every save/delete via hooks
|
||||
# so disconnect for Tags and run once-per-transaction.
|
||||
post_save.disconnect(
|
||||
post_save_treenode,
|
||||
sender=Tag,
|
||||
dispatch_uid="post_save_treenode",
|
||||
)
|
||||
post_delete.disconnect(
|
||||
post_delete_treenode,
|
||||
sender=Tag,
|
||||
dispatch_uid="post_delete_treenode",
|
||||
)
|
||||
post_save.connect(
|
||||
schedule_tag_tree_update,
|
||||
sender=Tag,
|
||||
dispatch_uid="paperless_tag_mark_dirty_save",
|
||||
)
|
||||
post_delete.connect(
|
||||
schedule_tag_tree_update,
|
||||
sender=Tag,
|
||||
dispatch_uid="paperless_tag_mark_dirty_delete",
|
||||
)
|
||||
|
||||
import documents.schema # noqa: F401
|
||||
|
||||
AppConfig.ready(self)
|
||||
|
||||
@@ -580,30 +580,34 @@ class TagSerializer(MatchingModelSerializer, OwnedObjectSerializer):
|
||||
),
|
||||
)
|
||||
def get_children(self, obj):
|
||||
filter_q = self.context.get("document_count_filter")
|
||||
request = self.context.get("request")
|
||||
if filter_q is None:
|
||||
user = getattr(request, "user", None) if request else None
|
||||
filter_q = get_document_count_filter_for_user(user)
|
||||
self.context["document_count_filter"] = filter_q
|
||||
children_map = self.context.get("children_map")
|
||||
if children_map is not None:
|
||||
children = children_map.get(obj.pk, [])
|
||||
else:
|
||||
filter_q = self.context.get("document_count_filter")
|
||||
request = self.context.get("request")
|
||||
if filter_q is None:
|
||||
user = getattr(request, "user", None) if request else None
|
||||
filter_q = get_document_count_filter_for_user(user)
|
||||
self.context["document_count_filter"] = filter_q
|
||||
|
||||
children_queryset = (
|
||||
obj.get_children_queryset()
|
||||
.select_related("owner")
|
||||
.annotate(document_count=Count("documents", filter=filter_q))
|
||||
)
|
||||
children = (
|
||||
obj.get_children_queryset()
|
||||
.select_related("owner")
|
||||
.annotate(document_count=Count("documents", filter=filter_q))
|
||||
)
|
||||
|
||||
view = self.context.get("view")
|
||||
ordering = (
|
||||
OrderingFilter().get_ordering(request, children_queryset, view)
|
||||
if request and view
|
||||
else None
|
||||
)
|
||||
ordering = ordering or (Lower("name"),)
|
||||
children_queryset = children_queryset.order_by(*ordering)
|
||||
view = self.context.get("view")
|
||||
ordering = (
|
||||
OrderingFilter().get_ordering(request, children, view)
|
||||
if request and view
|
||||
else None
|
||||
)
|
||||
ordering = ordering or (Lower("name"),)
|
||||
children = children.order_by(*ordering)
|
||||
|
||||
serializer = TagSerializer(
|
||||
children_queryset,
|
||||
children,
|
||||
many=True,
|
||||
user=self.user,
|
||||
full_perms=self.full_perms,
|
||||
|
||||
@@ -19,6 +19,7 @@ from django.db import DatabaseError
|
||||
from django.db import close_old_connections
|
||||
from django.db import connections
|
||||
from django.db import models
|
||||
from django.db import transaction
|
||||
from django.db.models import Q
|
||||
from django.dispatch import receiver
|
||||
from django.utils import timezone
|
||||
@@ -60,6 +61,8 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = logging.getLogger("paperless.handlers")
|
||||
|
||||
_tag_tree_update_scheduled = False
|
||||
|
||||
|
||||
def add_inbox_tags(sender, document: Document, logging_group=None, **kwargs):
|
||||
if document.owner is not None:
|
||||
@@ -944,3 +947,26 @@ def close_connection_pool_on_worker_init(**kwargs):
|
||||
for conn in connections.all(initialized_only=True):
|
||||
if conn.alias == "default" and hasattr(conn, "pool") and conn.pool:
|
||||
conn.close_pool()
|
||||
|
||||
|
||||
def schedule_tag_tree_update(**_kwargs):
|
||||
"""
|
||||
Schedule a single Tag.update_tree() at transaction commit.
|
||||
|
||||
Treenode's default post_save hooks rebuild the entire tree on every save,
|
||||
which is very slow for large tag sets so collapse to one update per
|
||||
transaction.
|
||||
"""
|
||||
global _tag_tree_update_scheduled
|
||||
if _tag_tree_update_scheduled:
|
||||
return
|
||||
_tag_tree_update_scheduled = True
|
||||
|
||||
def _run():
|
||||
global _tag_tree_update_scheduled
|
||||
try:
|
||||
Tag.update_tree()
|
||||
finally:
|
||||
_tag_tree_update_scheduled = False
|
||||
|
||||
transaction.on_commit(_run)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from unittest import mock
|
||||
|
||||
from django.contrib.auth.models import User
|
||||
from django.db import transaction
|
||||
from rest_framework.test import APITestCase
|
||||
|
||||
from documents import bulk_edit
|
||||
@@ -10,6 +11,7 @@ from documents.models import Workflow
|
||||
from documents.models import WorkflowAction
|
||||
from documents.models import WorkflowTrigger
|
||||
from documents.serialisers import TagSerializer
|
||||
from documents.signals import handlers
|
||||
from documents.signals.handlers import run_workflows
|
||||
|
||||
|
||||
@@ -250,3 +252,31 @@ class TestTagHierarchy(APITestCase):
|
||||
row for row in response.data["results"] if row["id"] == self.parent.pk
|
||||
)
|
||||
assert any(child["id"] == self.child.pk for child in parent_entry["children"])
|
||||
|
||||
def test_tag_tree_deferred_update_runs_on_commit(self):
|
||||
from django.db import transaction
|
||||
|
||||
# Create tags inside an explicit transaction and commit.
|
||||
with transaction.atomic():
|
||||
parent = Tag.objects.create(name="Parent 2")
|
||||
child = Tag.objects.create(name="Child 2", tn_parent=parent)
|
||||
# After commit, tn_* fields should be populated.
|
||||
parent.refresh_from_db()
|
||||
child.refresh_from_db()
|
||||
assert parent.tn_children_count == 1
|
||||
assert child.tn_ancestors_count == 1
|
||||
|
||||
def test_tag_tree_update_runs_once_per_transaction(self):
|
||||
handlers._tag_tree_update_scheduled = False
|
||||
|
||||
with mock.patch("documents.signals.handlers.Tag.update_tree") as update_tree:
|
||||
with self.captureOnCommitCallbacks(execute=True) as callbacks:
|
||||
with transaction.atomic():
|
||||
handlers.schedule_tag_tree_update()
|
||||
handlers.schedule_tag_tree_update()
|
||||
update_tree.assert_not_called()
|
||||
assert handlers._tag_tree_update_scheduled is True
|
||||
|
||||
assert len(callbacks) == 1
|
||||
update_tree.assert_called_once()
|
||||
assert handlers._tag_tree_update_scheduled is False
|
||||
|
||||
@@ -448,8 +448,43 @@ class TagViewSet(ModelViewSet, PermissionsAwareDocumentCountMixin):
|
||||
def get_serializer_context(self):
|
||||
context = super().get_serializer_context()
|
||||
context["document_count_filter"] = self.get_document_count_filter()
|
||||
if hasattr(self, "_children_map"):
|
||||
context["children_map"] = self._children_map
|
||||
return context
|
||||
|
||||
def list(self, request, *args, **kwargs):
|
||||
"""
|
||||
Build a children map once to avoid per-parent queries in the serializer.
|
||||
"""
|
||||
queryset = self.filter_queryset(self.get_queryset())
|
||||
ordering = OrderingFilter().get_ordering(request, queryset, self) or (
|
||||
Lower("name"),
|
||||
)
|
||||
queryset = queryset.order_by(*ordering)
|
||||
|
||||
all_tags = list(queryset)
|
||||
descendant_pks = {pk for tag in all_tags for pk in tag.get_descendants_pks()}
|
||||
|
||||
if descendant_pks:
|
||||
filter_q = self.get_document_count_filter()
|
||||
children_source = (
|
||||
Tag.objects.filter(pk__in=descendant_pks | {t.pk for t in all_tags})
|
||||
.select_related("owner")
|
||||
.annotate(document_count=Count("documents", filter=filter_q))
|
||||
.order_by(*ordering)
|
||||
)
|
||||
else:
|
||||
children_source = all_tags
|
||||
|
||||
children_map = {}
|
||||
for tag in children_source:
|
||||
children_map.setdefault(tag.tn_parent_id, []).append(tag)
|
||||
self._children_map = children_map
|
||||
|
||||
page = self.paginate_queryset(queryset)
|
||||
serializer = self.get_serializer(page, many=True)
|
||||
return self.get_paginated_response(serializer.data)
|
||||
|
||||
def perform_update(self, serializer):
|
||||
old_parent = self.get_object().get_parent()
|
||||
tag = serializer.save()
|
||||
|
||||
Reference in New Issue
Block a user