Compare commits

..

3 Commits

Author SHA1 Message Date
shamoon
61e0bd7eb6 A ChatGPT script for testing 2025-12-30 10:24:53 -08:00
shamoon
d127361411 Optimize tag children retrieval 2025-12-30 10:24:53 -08:00
shamoon
d45dee6d39 Run Tag tree updates once per transaction 2025-12-30 10:24:53 -08:00
8 changed files with 332 additions and 85 deletions

139
scripts/tag_perf_probe.py Normal file
View File

@@ -0,0 +1,139 @@
# noqa: INP001
"""
Ad-hoc script to gauge Tag + treenode performance locally.
It bootstraps a fresh SQLite DB in a temp folder (or PAPERLESS_DATA_DIR),
uses locmem cache/redis to avoid external services, creates synthetic tags,
and measures:
- creation time
- query count and wall time for the Tag list view
Usage:
PAPERLESS_DEBUG=1 PAPERLESS_REDIS=locmem:// PYTHONPATH=src \
PAPERLESS_DATA_DIR=/tmp/paperless-tags-probe \
.venv/bin/python scripts/tag_perf_probe.py
"""
import os
import sys
import time
from collections.abc import Iterable
from contextlib import contextmanager
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "paperless.settings")
os.environ.setdefault("PAPERLESS_DEBUG", "1")
os.environ.setdefault("PAPERLESS_REDIS", "locmem://")
os.environ.setdefault("PYTHONPATH", "src")
import django
django.setup()
from django.contrib.auth import get_user_model # noqa: E402
from django.core.management import call_command # noqa: E402
from django.db import connection # noqa: E402
from django.test.client import RequestFactory # noqa: E402
from rest_framework.test import force_authenticate # noqa: E402
from treenode.signals import no_signals # noqa: E402
from documents.models import Tag # noqa: E402
from documents.views import TagViewSet # noqa: E402
User = get_user_model()
@contextmanager
def count_queries():
total = 0
def wrapper(execute, sql, params, many, context):
nonlocal total
total += 1
return execute(sql, params, many, context)
with connection.execute_wrapper(wrapper):
yield lambda: total
def measure_list(tag_count: int, user) -> tuple[int, float]:
"""Render Tag list with page_size=tag_count and return (queries, seconds)."""
rf = RequestFactory()
view = TagViewSet.as_view({"get": "list"})
request = rf.get("/api/tags/", {"page_size": tag_count})
force_authenticate(request, user=user)
with count_queries() as get_count:
start = time.perf_counter()
response = view(request)
response.render()
elapsed = time.perf_counter() - start
total_queries = get_count()
return total_queries, elapsed
def bulk_create_tags(count: int, parents: Iterable[Tag] | None = None) -> None:
"""Create tags; when parents provided, create one child per parent."""
if parents is None:
Tag.objects.bulk_create([Tag(name=f"Flat {i}") for i in range(count)])
return
children = []
for p in parents:
children.append(Tag(name=f"Child {p.id}", tn_parent=p))
Tag.objects.bulk_create(children)
def run():
# Ensure tables exist when pointing at a fresh DATA_DIR.
call_command("migrate", interactive=False, verbosity=0)
user, _ = User.objects.get_or_create(
username="admin",
defaults={"is_superuser": True, "is_staff": True},
)
# Flat scenario
Tag.objects.all().delete()
start = time.perf_counter()
bulk_create_tags(200)
flat_create = time.perf_counter() - start
q, t = measure_list(tag_count=200, user=user)
print(f"Flat create 200 -> {flat_create:.2f}s; list -> {q} queries, {t:.2f}s") # noqa: T201
# Nested scenario (parents + 2 children each => 600 total)
Tag.objects.all().delete()
start = time.perf_counter()
with no_signals(): # avoid per-save tree rebuild; rebuild once
parents = Tag.objects.bulk_create([Tag(name=f"Parent {i}") for i in range(200)])
children = []
for p in parents:
children.extend(
Tag(name=f"Child {p.id}-{j}", tn_parent=p) for j in range(2)
)
Tag.objects.bulk_create(children)
Tag.update_tree()
nested_create = time.perf_counter() - start
q, t = measure_list(tag_count=600, user=user)
print(f"Nested create 600 -> {nested_create:.2f}s; list -> {q} queries, {t:.2f}s") # noqa: T201
# Larger nested scenario (1 child per parent, 3000 total)
Tag.objects.all().delete()
start = time.perf_counter()
with no_signals():
parents = Tag.objects.bulk_create(
[Tag(name=f"Parent {i}") for i in range(1500)],
)
bulk_create_tags(0, parents=parents)
Tag.update_tree()
big_create = time.perf_counter() - start
q, t = measure_list(tag_count=3000, user=user)
print(f"Nested create 3000 -> {big_create:.2f}s; list -> {q} queries, {t:.2f}s") # noqa: T201
if __name__ == "__main__":
if "runserver" in sys.argv:
print("Run directly: .venv/bin/python scripts/tag_perf_probe.py") # noqa: T201
sys.exit(1)
run()

View File

@@ -68,7 +68,7 @@
"prettier-plugin-organize-imports": "^4.3.0",
"ts-node": "~10.9.1",
"typescript": "^5.8.3",
"webpack": "^5.104.1"
"webpack": "^5.103.0"
},
"packageManager": "pnpm@10.17.1",
"pnpm": {

112
src-ui/pnpm-lock.yaml generated
View File

@@ -128,7 +128,7 @@ importers:
version: 20.3.15(@angular/compiler@20.3.15)(typescript@5.8.3)
'@codecov/webpack-plugin':
specifier: ^1.9.1
version: 1.9.1(webpack@5.104.1)
version: 1.9.1(webpack@5.103.0)
'@playwright/test':
specifier: ^1.57.0
version: 1.57.0
@@ -175,8 +175,8 @@ importers:
specifier: ^5.8.3
version: 5.8.3
webpack:
specifier: ^5.104.1
version: 5.104.1
specifier: ^5.103.0
version: 5.103.0
packages:
@@ -3423,8 +3423,8 @@ packages:
base64-js@1.5.1:
resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==}
baseline-browser-mapping@2.9.11:
resolution: {integrity: sha512-Sg0xJUNDU1sJNGdfGWhVHX0kkZ+HWcvmVymJbj6NSgZZmW/8S9Y2HQ5euytnIgakgxN6papOAWiwDo1ctFDcoQ==}
baseline-browser-mapping@2.9.4:
resolution: {integrity: sha512-ZCQ9GEWl73BVm8bu5Fts8nt7MHdbt5vY9bP6WGnUh+r3l8M7CgfyTlwsgCbMC66BNxPr6Xoce3j66Ms5YUQTNA==}
hasBin: true
batch@0.6.1:
@@ -3530,8 +3530,8 @@ packages:
resolution: {integrity: sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==}
engines: {node: '>=10'}
caniuse-lite@1.0.30001762:
resolution: {integrity: sha512-PxZwGNvH7Ak8WX5iXzoK1KPZttBXNPuaOvI2ZYU7NrlM+d9Ov+TUvlLOBNGzVXAntMSMMlJPd+jY6ovrVjSmUw==}
caniuse-lite@1.0.30001759:
resolution: {integrity: sha512-Pzfx9fOKoKvevQf8oCXoyNRQ5QyxJj+3O0Rqx2V5oxT61KGx8+n6hV/IUyJeifUci2clnmmKVpvtiqRzgiWjSw==}
canvas@3.0.0:
resolution: {integrity: sha512-NtcIBY88FjymQy+g2g5qnuP5IslrbWCQ3A6rSr1PeuYxVRapRZ3BZCrDyAakvI6CuDYidgZaf55ygulFVwROdg==}
@@ -3912,8 +3912,8 @@ packages:
ee-first@1.1.1:
resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==}
electron-to-chromium@1.5.267:
resolution: {integrity: sha512-0Drusm6MVRXSOJpGbaSVgcQsuB4hEkMpHXaVstcPmhu5LIedxs1xNK/nIxmQIU/RPC0+1/o0AVZfBTkTNJOdUw==}
electron-to-chromium@1.5.266:
resolution: {integrity: sha512-kgWEglXvkEfMH7rxP5OSZZwnaDWT7J9EoZCujhnpLbfi0bbNtRkgdX2E3gt0Uer11c61qCYktB3hwkAS325sJg==}
emittery@0.13.1:
resolution: {integrity: sha512-DeWwawk6r5yR9jFgnDKYt4sLS0LmHJJi3ZOnb5/JdbYwj3nW+FxQnHIjhBKz8YLC7oRNPVM9NQ47I3CVx34eqQ==}
@@ -3946,8 +3946,8 @@ packages:
end-of-stream@1.4.4:
resolution: {integrity: sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==}
enhanced-resolve@5.18.4:
resolution: {integrity: sha512-LgQMM4WXU3QI+SYgEc2liRgznaD5ojbmY3sb8LxyguVkIg5FxdpTkvk72te2R38/TGKxH634oLxXRGY6d7AP+Q==}
enhanced-resolve@5.18.3:
resolution: {integrity: sha512-d4lC8xfavMeBjzGr2vECC3fsGXziXZQyJxD868h2M/mBI3PwAuODxAkLkq5HYuvrPYcUtiLzsTo8U3PgX3Ocww==}
engines: {node: '>=10.13.0'}
entities@4.5.0:
@@ -3987,9 +3987,6 @@ packages:
es-module-lexer@1.7.0:
resolution: {integrity: sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==}
es-module-lexer@2.0.0:
resolution: {integrity: sha512-5POEcUuZybH7IdmGsD8wlf0AI55wMecM9rVBTI/qEAy2c1kTOm3DjFYjrBdI2K3BaJjJYfYFeRtM0t9ssnRuxw==}
es-object-atoms@1.1.1:
resolution: {integrity: sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==}
engines: {node: '>= 0.4'}
@@ -6293,8 +6290,8 @@ packages:
resolution: {integrity: sha512-7NyxrTE4Anh8km8iEy7o0QYPs+0JKBTj5ZaqHg6B39erLg0qYXN3BijtShwbsNSvQ+LN75+KV+C4QR/f6Gwnpg==}
engines: {node: '>=18'}
terser-webpack-plugin@5.3.16:
resolution: {integrity: sha512-h9oBFCWrq78NyWWVcSwZarJkZ01c2AyGrzs1crmHZO3QUg9D61Wu4NPjBy69n7JqylFF5y+CsUZYmYEIZ3mR+Q==}
terser-webpack-plugin@5.3.15:
resolution: {integrity: sha512-PGkOdpRFK+rb1TzVz+msVhw4YMRT9txLF4kRqvJhGhCM324xuR3REBSHALN+l+sAhKUmz0aotnjp5D+P83mLhQ==}
engines: {node: '>= 10.13.0'}
peerDependencies:
'@swc/core': '*'
@@ -6566,8 +6563,8 @@ packages:
unrs-resolver@1.11.1:
resolution: {integrity: sha512-bSjt9pjaEBnNiGgc9rUiHGKv5l4/TGzDmYw3RhnkJGtLhbnnA/5qJj7x3dNDCRx/PJxu774LlH8lCOlB4hEfKg==}
update-browserslist-db@1.2.3:
resolution: {integrity: sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==}
update-browserslist-db@1.2.2:
resolution: {integrity: sha512-E85pfNzMQ9jpKkA7+TJAi4TJN+tBCuWh5rUcS/sv6cFi+1q9LYDwDI5dpUL0u/73EElyQ8d3TEaeW4sPedBqYA==}
hasBin: true
peerDependencies:
browserslist: '>= 4.21.0'
@@ -6713,10 +6710,6 @@ packages:
resolution: {integrity: sha512-c5EGNOiyxxV5qmTtAB7rbiXxi1ooX1pQKMLX/MIabJjRA0SJBQOjKF+KSVfHkr9U1cADPon0mRiVe/riyaiDUA==}
engines: {node: '>=10.13.0'}
watchpack@2.5.0:
resolution: {integrity: sha512-e6vZvY6xboSwLz2GD36c16+O/2Z6fKvIf4pOXptw2rY9MVwE/TXc6RGqxD3I3x0a28lwBY7DE+76uTPSsBrrCA==}
engines: {node: '>=10.13.0'}
wbuf@1.7.3:
resolution: {integrity: sha512-O84QOnr0icsbFGLS0O3bI5FswxzRr8/gHwWkDlQFskhSPryQXvrTMxjxGP4+iWYoauLoBvfDpkrOauZ+0iZpDA==}
@@ -6770,8 +6763,8 @@ packages:
webpack-virtual-modules@0.6.2:
resolution: {integrity: sha512-66/V2i5hQanC51vBQKPH4aI8NMAcBW59FVBs+rC7eGHupMyfn34q7rZIE+ETlJ+XTevqfUhVVBgSUNSW2flEUQ==}
webpack@5.104.1:
resolution: {integrity: sha512-Qphch25abbMNtekmEGJmeRUhLDbe+QfiWTiqpKYkpCOWY64v9eyl+KRRLmqOFA2AvKPpc9DC6+u2n76tQLBoaA==}
webpack@5.103.0:
resolution: {integrity: sha512-HU1JOuV1OavsZ+mfigY0j8d1TgQgbZ6M+J75zDkpEAwYeXjWSqrGJtgnPblJjd/mAyTNQ7ygw0MiKOn6etz8yw==}
engines: {node: '>=10.13.0'}
hasBin: true
peerDependencies:
@@ -6801,12 +6794,10 @@ packages:
whatwg-encoding@2.0.0:
resolution: {integrity: sha512-p41ogyeMUrw3jWclHWTQg1k05DSVXPLcVxRTYsXUk+ZooOCZLcoYgPZ/HL/D/N+uQPOtcp1me1WhBEaX02mhWg==}
engines: {node: '>=12'}
deprecated: Use @exodus/bytes instead for a more spec-conformant and faster implementation
whatwg-encoding@3.1.1:
resolution: {integrity: sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==}
engines: {node: '>=18'}
deprecated: Use @exodus/bytes instead for a more spec-conformant and faster implementation
whatwg-mimetype@3.0.0:
resolution: {integrity: sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q==}
@@ -7190,7 +7181,7 @@ snapshots:
dependencies:
'@ampproject/remapping': 2.3.0
'@angular-devkit/architect': 0.2000.4(chokidar@4.0.3)
'@angular-devkit/build-webpack': 0.2000.4(chokidar@4.0.3)(webpack-dev-server@5.2.1(webpack@5.104.1))(webpack@5.99.8(esbuild@0.25.5))
'@angular-devkit/build-webpack': 0.2000.4(chokidar@4.0.3)(webpack-dev-server@5.2.1(webpack@5.103.0))(webpack@5.99.8(esbuild@0.25.5))
'@angular-devkit/core': 20.0.4(chokidar@4.0.3)
'@angular/build': 20.0.4(@angular/compiler-cli@20.3.15(@angular/compiler@20.3.15)(typescript@5.8.3))(@angular/compiler@20.3.15)(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1))(@angular/localize@20.3.15(@angular/compiler-cli@20.3.15(@angular/compiler@20.3.15)(typescript@5.8.3))(@angular/compiler@20.3.15))(@angular/platform-browser@20.3.15(@angular/common@20.3.15(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1))(rxjs@7.8.2))(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1)))(@types/node@24.10.1)(chokidar@4.0.3)(jiti@1.21.7)(less@4.3.0)(postcss@8.5.3)(terser@5.39.1)(tslib@2.8.1)(typescript@5.8.3)(yaml@2.7.0)
'@angular/compiler-cli': 20.3.15(@angular/compiler@20.3.15)(typescript@5.8.3)
@@ -7276,12 +7267,12 @@ snapshots:
- webpack-cli
- yaml
'@angular-devkit/build-webpack@0.2000.4(chokidar@4.0.3)(webpack-dev-server@5.2.1(webpack@5.104.1))(webpack@5.99.8(esbuild@0.25.5))':
'@angular-devkit/build-webpack@0.2000.4(chokidar@4.0.3)(webpack-dev-server@5.2.1(webpack@5.103.0))(webpack@5.99.8(esbuild@0.25.5))':
dependencies:
'@angular-devkit/architect': 0.2000.4(chokidar@4.0.3)
rxjs: 7.8.2
webpack: 5.99.8(esbuild@0.25.5)
webpack-dev-server: 5.2.1(webpack@5.104.1)
webpack-dev-server: 5.2.1(webpack@5.103.0)
transitivePeerDependencies:
- chokidar
@@ -8441,11 +8432,11 @@ snapshots:
unplugin: 1.16.1
zod: 3.25.76
'@codecov/webpack-plugin@1.9.1(webpack@5.104.1)':
'@codecov/webpack-plugin@1.9.1(webpack@5.103.0)':
dependencies:
'@codecov/bundler-plugin-core': 1.9.1
unplugin: 1.16.1
webpack: 5.104.1
webpack: 5.103.0
'@cspotcode/source-map-support@0.8.1':
dependencies:
@@ -10384,7 +10375,7 @@ snapshots:
autoprefixer@10.4.21(postcss@8.5.3):
dependencies:
browserslist: 4.28.1
caniuse-lite: 1.0.30001762
caniuse-lite: 1.0.30001759
fraction.js: 4.3.7
normalize-range: 0.1.2
picocolors: 1.1.1
@@ -10480,7 +10471,7 @@ snapshots:
base64-js@1.5.1:
optional: true
baseline-browser-mapping@2.9.11: {}
baseline-browser-mapping@2.9.4: {}
batch@0.6.1: {}
@@ -10576,11 +10567,11 @@ snapshots:
browserslist@4.28.1:
dependencies:
baseline-browser-mapping: 2.9.11
caniuse-lite: 1.0.30001762
electron-to-chromium: 1.5.267
baseline-browser-mapping: 2.9.4
caniuse-lite: 1.0.30001759
electron-to-chromium: 1.5.266
node-releases: 2.0.27
update-browserslist-db: 1.2.3(browserslist@4.28.1)
update-browserslist-db: 1.2.2(browserslist@4.28.1)
bs-logger@0.2.6:
dependencies:
@@ -10635,7 +10626,7 @@ snapshots:
camelcase@6.3.0: {}
caniuse-lite@1.0.30001762: {}
caniuse-lite@1.0.30001759: {}
canvas@3.0.0:
dependencies:
@@ -10977,7 +10968,7 @@ snapshots:
ee-first@1.1.1: {}
electron-to-chromium@1.5.267: {}
electron-to-chromium@1.5.266: {}
emittery@0.13.1: {}
@@ -11003,7 +10994,7 @@ snapshots:
once: 1.4.0
optional: true
enhanced-resolve@5.18.4:
enhanced-resolve@5.18.3:
dependencies:
graceful-fs: 4.2.11
tapable: 2.3.0
@@ -11033,8 +11024,6 @@ snapshots:
es-module-lexer@1.7.0: {}
es-module-lexer@2.0.0: {}
es-object-atoms@1.1.1:
dependencies:
es-errors: 1.3.0
@@ -13934,7 +13923,7 @@ snapshots:
minizlib: 3.1.0
yallist: 5.0.0
terser-webpack-plugin@5.3.16(esbuild@0.25.5)(webpack@5.99.8(esbuild@0.25.5)):
terser-webpack-plugin@5.3.15(esbuild@0.25.5)(webpack@5.99.8(esbuild@0.25.5)):
dependencies:
'@jridgewell/trace-mapping': 0.3.31
jest-worker: 27.5.1
@@ -13945,14 +13934,14 @@ snapshots:
optionalDependencies:
esbuild: 0.25.5
terser-webpack-plugin@5.3.16(webpack@5.104.1):
terser-webpack-plugin@5.3.15(webpack@5.103.0):
dependencies:
'@jridgewell/trace-mapping': 0.3.31
jest-worker: 27.5.1
schema-utils: 4.3.3
serialize-javascript: 6.0.2
terser: 5.44.1
webpack: 5.104.1
webpack: 5.103.0
terser@5.39.1:
dependencies:
@@ -14209,7 +14198,7 @@ snapshots:
'@unrs/resolver-binding-win32-ia32-msvc': 1.11.1
'@unrs/resolver-binding-win32-x64-msvc': 1.11.1
update-browserslist-db@1.2.3(browserslist@4.28.1):
update-browserslist-db@1.2.2(browserslist@4.28.1):
dependencies:
browserslist: 4.28.1
escalade: 3.2.0
@@ -14309,11 +14298,6 @@ snapshots:
glob-to-regexp: 0.4.1
graceful-fs: 4.2.11
watchpack@2.5.0:
dependencies:
glob-to-regexp: 0.4.1
graceful-fs: 4.2.11
wbuf@1.7.3:
dependencies:
minimalistic-assert: 1.0.1
@@ -14323,7 +14307,7 @@ snapshots:
webidl-conversions@7.0.0: {}
webpack-dev-middleware@7.4.2(webpack@5.104.1):
webpack-dev-middleware@7.4.2(webpack@5.103.0):
dependencies:
colorette: 2.0.20
memfs: 4.17.2
@@ -14332,7 +14316,7 @@ snapshots:
range-parser: 1.2.1
schema-utils: 4.3.3
optionalDependencies:
webpack: 5.104.1
webpack: 5.103.0
webpack-dev-middleware@7.4.2(webpack@5.99.8(esbuild@0.25.5)):
dependencies:
@@ -14345,7 +14329,7 @@ snapshots:
optionalDependencies:
webpack: 5.99.8(esbuild@0.25.5)
webpack-dev-server@5.2.1(webpack@5.104.1):
webpack-dev-server@5.2.1(webpack@5.103.0):
dependencies:
'@types/bonjour': 3.5.13
'@types/connect-history-api-fallback': 1.5.4
@@ -14373,10 +14357,10 @@ snapshots:
serve-index: 1.9.1
sockjs: 0.3.24
spdy: 4.0.2
webpack-dev-middleware: 7.4.2(webpack@5.104.1)
webpack-dev-middleware: 7.4.2(webpack@5.103.0)
ws: 8.18.3
optionalDependencies:
webpack: 5.104.1
webpack: 5.103.0
transitivePeerDependencies:
- bufferutil
- debug
@@ -14436,7 +14420,7 @@ snapshots:
webpack-virtual-modules@0.6.2: {}
webpack@5.104.1:
webpack@5.103.0:
dependencies:
'@types/eslint-scope': 3.7.7
'@types/estree': 1.0.8
@@ -14448,8 +14432,8 @@ snapshots:
acorn-import-phases: 1.0.4(acorn@8.15.0)
browserslist: 4.28.1
chrome-trace-event: 1.0.4
enhanced-resolve: 5.18.4
es-module-lexer: 2.0.0
enhanced-resolve: 5.18.3
es-module-lexer: 1.7.0
eslint-scope: 5.1.1
events: 3.3.0
glob-to-regexp: 0.4.1
@@ -14460,8 +14444,8 @@ snapshots:
neo-async: 2.6.2
schema-utils: 4.3.3
tapable: 2.3.0
terser-webpack-plugin: 5.3.16(webpack@5.104.1)
watchpack: 2.5.0
terser-webpack-plugin: 5.3.15(webpack@5.103.0)
watchpack: 2.4.4
webpack-sources: 3.3.3
transitivePeerDependencies:
- '@swc/core'
@@ -14479,7 +14463,7 @@ snapshots:
acorn: 8.15.0
browserslist: 4.28.1
chrome-trace-event: 1.0.4
enhanced-resolve: 5.18.4
enhanced-resolve: 5.18.3
es-module-lexer: 1.7.0
eslint-scope: 5.1.1
events: 3.3.0
@@ -14491,8 +14475,8 @@ snapshots:
neo-async: 2.6.2
schema-utils: 4.3.3
tapable: 2.3.0
terser-webpack-plugin: 5.3.16(esbuild@0.25.5)(webpack@5.99.8(esbuild@0.25.5))
watchpack: 2.5.0
terser-webpack-plugin: 5.3.15(esbuild@0.25.5)(webpack@5.99.8(esbuild@0.25.5))
watchpack: 2.4.4
webpack-sources: 3.3.3
transitivePeerDependencies:
- '@swc/core'

View File

@@ -1,5 +1,9 @@
from django.apps import AppConfig
from django.db.models.signals import post_delete
from django.db.models.signals import post_save
from django.utils.translation import gettext_lazy as _
from treenode.signals import post_delete_treenode
from treenode.signals import post_save_treenode
class DocumentsConfig(AppConfig):
@@ -8,12 +12,14 @@ class DocumentsConfig(AppConfig):
verbose_name = _("Documents")
def ready(self):
from documents.models import Tag
from documents.signals import document_consumption_finished
from documents.signals import document_updated
from documents.signals.handlers import add_inbox_tags
from documents.signals.handlers import add_to_index
from documents.signals.handlers import run_workflows_added
from documents.signals.handlers import run_workflows_updated
from documents.signals.handlers import schedule_tag_tree_update
from documents.signals.handlers import set_correspondent
from documents.signals.handlers import set_document_type
from documents.signals.handlers import set_storage_path
@@ -28,6 +34,29 @@ class DocumentsConfig(AppConfig):
document_consumption_finished.connect(run_workflows_added)
document_updated.connect(run_workflows_updated)
# treenode updates the entire tree on every save/delete via hooks
# so disconnect for Tags and run once-per-transaction.
post_save.disconnect(
post_save_treenode,
sender=Tag,
dispatch_uid="post_save_treenode",
)
post_delete.disconnect(
post_delete_treenode,
sender=Tag,
dispatch_uid="post_delete_treenode",
)
post_save.connect(
schedule_tag_tree_update,
sender=Tag,
dispatch_uid="paperless_tag_mark_dirty_save",
)
post_delete.connect(
schedule_tag_tree_update,
sender=Tag,
dispatch_uid="paperless_tag_mark_dirty_delete",
)
import documents.schema # noqa: F401
AppConfig.ready(self)

View File

@@ -580,30 +580,34 @@ class TagSerializer(MatchingModelSerializer, OwnedObjectSerializer):
),
)
def get_children(self, obj):
filter_q = self.context.get("document_count_filter")
request = self.context.get("request")
if filter_q is None:
user = getattr(request, "user", None) if request else None
filter_q = get_document_count_filter_for_user(user)
self.context["document_count_filter"] = filter_q
children_map = self.context.get("children_map")
if children_map is not None:
children = children_map.get(obj.pk, [])
else:
filter_q = self.context.get("document_count_filter")
request = self.context.get("request")
if filter_q is None:
user = getattr(request, "user", None) if request else None
filter_q = get_document_count_filter_for_user(user)
self.context["document_count_filter"] = filter_q
children_queryset = (
obj.get_children_queryset()
.select_related("owner")
.annotate(document_count=Count("documents", filter=filter_q))
)
children = (
obj.get_children_queryset()
.select_related("owner")
.annotate(document_count=Count("documents", filter=filter_q))
)
view = self.context.get("view")
ordering = (
OrderingFilter().get_ordering(request, children_queryset, view)
if request and view
else None
)
ordering = ordering or (Lower("name"),)
children_queryset = children_queryset.order_by(*ordering)
view = self.context.get("view")
ordering = (
OrderingFilter().get_ordering(request, children, view)
if request and view
else None
)
ordering = ordering or (Lower("name"),)
children = children.order_by(*ordering)
serializer = TagSerializer(
children_queryset,
children,
many=True,
user=self.user,
full_perms=self.full_perms,

View File

@@ -19,6 +19,7 @@ from django.db import DatabaseError
from django.db import close_old_connections
from django.db import connections
from django.db import models
from django.db import transaction
from django.db.models import Q
from django.dispatch import receiver
from django.utils import timezone
@@ -60,6 +61,8 @@ if TYPE_CHECKING:
logger = logging.getLogger("paperless.handlers")
_tag_tree_update_scheduled = False
def add_inbox_tags(sender, document: Document, logging_group=None, **kwargs):
if document.owner is not None:
@@ -944,3 +947,26 @@ def close_connection_pool_on_worker_init(**kwargs):
for conn in connections.all(initialized_only=True):
if conn.alias == "default" and hasattr(conn, "pool") and conn.pool:
conn.close_pool()
def schedule_tag_tree_update(**_kwargs):
"""
Schedule a single Tag.update_tree() at transaction commit.
Treenode's default post_save hooks rebuild the entire tree on every save,
which is very slow for large tag sets so collapse to one update per
transaction.
"""
global _tag_tree_update_scheduled
if _tag_tree_update_scheduled:
return
_tag_tree_update_scheduled = True
def _run():
global _tag_tree_update_scheduled
try:
Tag.update_tree()
finally:
_tag_tree_update_scheduled = False
transaction.on_commit(_run)

View File

@@ -1,6 +1,7 @@
from unittest import mock
from django.contrib.auth.models import User
from django.db import transaction
from rest_framework.test import APITestCase
from documents import bulk_edit
@@ -10,6 +11,7 @@ from documents.models import Workflow
from documents.models import WorkflowAction
from documents.models import WorkflowTrigger
from documents.serialisers import TagSerializer
from documents.signals import handlers
from documents.signals.handlers import run_workflows
@@ -250,3 +252,31 @@ class TestTagHierarchy(APITestCase):
row for row in response.data["results"] if row["id"] == self.parent.pk
)
assert any(child["id"] == self.child.pk for child in parent_entry["children"])
def test_tag_tree_deferred_update_runs_on_commit(self):
from django.db import transaction
# Create tags inside an explicit transaction and commit.
with transaction.atomic():
parent = Tag.objects.create(name="Parent 2")
child = Tag.objects.create(name="Child 2", tn_parent=parent)
# After commit, tn_* fields should be populated.
parent.refresh_from_db()
child.refresh_from_db()
assert parent.tn_children_count == 1
assert child.tn_ancestors_count == 1
def test_tag_tree_update_runs_once_per_transaction(self):
handlers._tag_tree_update_scheduled = False
with mock.patch("documents.signals.handlers.Tag.update_tree") as update_tree:
with self.captureOnCommitCallbacks(execute=True) as callbacks:
with transaction.atomic():
handlers.schedule_tag_tree_update()
handlers.schedule_tag_tree_update()
update_tree.assert_not_called()
assert handlers._tag_tree_update_scheduled is True
assert len(callbacks) == 1
update_tree.assert_called_once()
assert handlers._tag_tree_update_scheduled is False

View File

@@ -448,8 +448,43 @@ class TagViewSet(ModelViewSet, PermissionsAwareDocumentCountMixin):
def get_serializer_context(self):
context = super().get_serializer_context()
context["document_count_filter"] = self.get_document_count_filter()
if hasattr(self, "_children_map"):
context["children_map"] = self._children_map
return context
def list(self, request, *args, **kwargs):
"""
Build a children map once to avoid per-parent queries in the serializer.
"""
queryset = self.filter_queryset(self.get_queryset())
ordering = OrderingFilter().get_ordering(request, queryset, self) or (
Lower("name"),
)
queryset = queryset.order_by(*ordering)
all_tags = list(queryset)
descendant_pks = {pk for tag in all_tags for pk in tag.get_descendants_pks()}
if descendant_pks:
filter_q = self.get_document_count_filter()
children_source = (
Tag.objects.filter(pk__in=descendant_pks | {t.pk for t in all_tags})
.select_related("owner")
.annotate(document_count=Count("documents", filter=filter_q))
.order_by(*ordering)
)
else:
children_source = all_tags
children_map = {}
for tag in children_source:
children_map.setdefault(tag.tn_parent_id, []).append(tag)
self._children_map = children_map
page = self.paginate_queryset(queryset)
serializer = self.get_serializer(page, many=True)
return self.get_paginated_response(serializer.data)
def perform_update(self, serializer):
old_parent = self.get_object().get_parent()
tag = serializer.save()