Fix: correctly extract all ids for nested tags (#11888)

.github/workflows/ci.yml

@@ -617,7 +617,7 @@ jobs:
           version: ${{ steps.get_version.outputs.version }}
           prerelease: ${{ steps.get_version.outputs.prerelease }}
           publish: true # ensures release is not marked as draft
-          committish: ${{ github.sha }}
+          commitish: ${{ github.sha }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - name: Upload release archive

docker/management_script.sh

@@ -5,10 +5,8 @@ set -e
 
 cd "${PAPERLESS_SRC_DIR}"
 
-if [[ $(id -u) == 0 ]]; then
+if [[ -n "${USER_IS_NON_ROOT}" ]]; then
-	s6-setuidgid paperless python3 manage.py management_command "$@"
-elif [[ $(id -un) == "paperless" ]]; then
 	python3 manage.py management_command "$@"
-else
+elif [[ $(id -un) == "paperless" ]]; then
-	echo "Unknown user."
+	s6-setuidgid paperless python3 manage.py management_command "$@"
 fi

docker/rootfs/usr/local/bin/convert_mariadb_uuid

@@ -5,10 +5,8 @@ set -e
 
 cd "${PAPERLESS_SRC_DIR}"
 
-if [[ $(id -u) == 0 ]]; then
+if [[ -n "${USER_IS_NON_ROOT}" ]]; then
-	s6-setuidgid paperless python3 manage.py convert_mariadb_uuid "$@"
+		python3 manage.py convert_mariadb_uuid "$@"
 elif [[ $(id -un) == "paperless" ]]; then
-	python3 manage.py convert_mariadb_uuid "$@"
+	s6-setuidgid paperless python3 manage.py convert_mariadb_uuid "$@"
-else
-	echo "Unknown user."
 fi

docker/rootfs/usr/local/bin/createsuperuser

@@ -5,10 +5,8 @@ set -e
 
 cd "${PAPERLESS_SRC_DIR}"
 
-if [[ $(id -u) == 0 ]]; then
+if [[ -n "${USER_IS_NON_ROOT}" ]]; then
-	s6-setuidgid paperless python3 manage.py createsuperuser "$@"
+		python3 manage.py createsuperuser "$@"
 elif [[ $(id -un) == "paperless" ]]; then
-	python3 manage.py createsuperuser "$@"
+	s6-setuidgid paperless python3 manage.py createsuperuser "$@"
-else
-	echo "Unknown user."
 fi

docker/rootfs/usr/local/bin/decrypt_documents

@@ -5,10 +5,8 @@ set -e
 
 cd "${PAPERLESS_SRC_DIR}"
 
-if [[ $(id -u) == 0 ]]; then
+if [[ -n "${USER_IS_NON_ROOT}" ]]; then
-	s6-setuidgid paperless python3 manage.py decrypt_documents "$@"
+		python3 manage.py decrypt_documents "$@"
 elif [[ $(id -un) == "paperless" ]]; then
-	python3 manage.py decrypt_documents "$@"
+	s6-setuidgid paperless python3 manage.py decrypt_documents "$@"
-else
-	echo "Unknown user."
 fi

docker/rootfs/usr/local/bin/document_archiver

@@ -5,10 +5,8 @@ set -e
 
 cd "${PAPERLESS_SRC_DIR}"
 
-if [[ $(id -u) == 0 ]]; then
+if [[ -n "${USER_IS_NON_ROOT}" ]]; then
-	s6-setuidgid paperless python3 manage.py document_archiver "$@"
+		python3 manage.py document_archiver "$@"
 elif [[ $(id -un) == "paperless" ]]; then
-	python3 manage.py document_archiver "$@"
+	s6-setuidgid paperless python3 manage.py document_archiver "$@"
-else
-	echo "Unknown user."
 fi

docker/rootfs/usr/local/bin/document_create_classifier

@@ -5,10 +5,8 @@ set -e
 
 cd "${PAPERLESS_SRC_DIR}"
 
-if [[ $(id -u) == 0 ]]; then
+if [[ -n "${USER_IS_NON_ROOT}" ]]; then
-	s6-setuidgid paperless python3 manage.py document_create_classifier "$@"
+		python3 manage.py document_create_classifier "$@"
 elif [[ $(id -un) == "paperless" ]]; then
-	python3 manage.py document_create_classifier "$@"
+	s6-setuidgid paperless python3 manage.py document_create_classifier "$@"
-else
-	echo "Unknown user."
 fi

docker/rootfs/usr/local/bin/document_exporter

@@ -5,10 +5,8 @@ set -e
 
 cd "${PAPERLESS_SRC_DIR}"
 
-if [[ $(id -u) == 0 ]]; then
+if [[ -n "${USER_IS_NON_ROOT}" ]]; then
-	s6-setuidgid paperless python3 manage.py document_exporter "$@"
+		python3 manage.py document_exporter "$@"
 elif [[ $(id -un) == "paperless" ]]; then
-	python3 manage.py document_exporter "$@"
+	s6-setuidgid paperless python3 manage.py document_exporter "$@"
-else
-	echo "Unknown user."
 fi

docker/rootfs/usr/local/bin/document_fuzzy_match

@@ -5,10 +5,8 @@ set -e
 
 cd "${PAPERLESS_SRC_DIR}"
 
-if [[ $(id -u) == 0 ]]; then
+if [[ -n "${USER_IS_NON_ROOT}" ]]; then
-	s6-setuidgid paperless python3 manage.py document_fuzzy_match "$@"
+		python3 manage.py document_fuzzy_match "$@"
 elif [[ $(id -un) == "paperless" ]]; then
-	python3 manage.py document_fuzzy_match "$@"
+	s6-setuidgid paperless python3 manage.py document_fuzzy_match "$@"
-else
-	echo "Unknown user."
 fi

docker/rootfs/usr/local/bin/document_importer

@@ -5,10 +5,8 @@ set -e
 
 cd "${PAPERLESS_SRC_DIR}"
 
-if [[ $(id -u) == 0 ]]; then
+if [[ -n "${USER_IS_NON_ROOT}" ]]; then
-	s6-setuidgid paperless python3 manage.py document_importer "$@"
+		python3 manage.py document_importer "$@"
 elif [[ $(id -un) == "paperless" ]]; then
-	python3 manage.py document_importer "$@"
+	s6-setuidgid paperless python3 manage.py document_importer "$@"
-else
-	echo "Unknown user."
 fi

docker/rootfs/usr/local/bin/document_index

@@ -5,10 +5,8 @@ set -e
 
 cd "${PAPERLESS_SRC_DIR}"
 
-if [[ $(id -u) == 0 ]]; then
+if [[ -n "${USER_IS_NON_ROOT}" ]]; then
-	s6-setuidgid paperless python3 manage.py document_index "$@"
+		python3 manage.py document_index "$@"
 elif [[ $(id -un) == "paperless" ]]; then
-	python3 manage.py document_index "$@"
+	s6-setuidgid paperless python3 manage.py document_index "$@"
-else
-	echo "Unknown user."
 fi

docker/rootfs/usr/local/bin/document_renamer

@@ -5,10 +5,8 @@ set -e
 
 cd "${PAPERLESS_SRC_DIR}"
 
-if [[ $(id -u) == 0 ]]; then
+if [[ -n "${USER_IS_NON_ROOT}" ]]; then
-	s6-setuidgid paperless python3 manage.py document_renamer "$@"
+		python3 manage.py document_renamer "$@"
 elif [[ $(id -un) == "paperless" ]]; then
-	python3 manage.py document_renamer "$@"
+	s6-setuidgid paperless python3 manage.py document_renamer "$@"
-else
-	echo "Unknown user."
 fi

docker/rootfs/usr/local/bin/document_retagger

@@ -5,10 +5,8 @@ set -e
 
 cd "${PAPERLESS_SRC_DIR}"
 
-if [[ $(id -u) == 0 ]]; then
+if [[ -n "${USER_IS_NON_ROOT}" ]]; then
-	s6-setuidgid paperless python3 manage.py document_retagger "$@"
+		python3 manage.py document_retagger "$@"
 elif [[ $(id -un) == "paperless" ]]; then
-	python3 manage.py document_retagger "$@"
+	s6-setuidgid paperless python3 manage.py document_retagger "$@"
-else
-	echo "Unknown user."
 fi

docker/rootfs/usr/local/bin/document_sanity_checker

@@ -5,10 +5,8 @@ set -e
 
 cd "${PAPERLESS_SRC_DIR}"
 
-if [[ $(id -u) == 0 ]]; then
+if [[ -n "${USER_IS_NON_ROOT}" ]]; then
-	s6-setuidgid paperless python3 manage.py document_sanity_checker "$@"
+		python3 manage.py document_sanity_checker "$@"
 elif [[ $(id -un) == "paperless" ]]; then
-	python3 manage.py document_sanity_checker "$@"
+	s6-setuidgid paperless python3 manage.py document_sanity_checker "$@"
-else
-	echo "Unknown user."
 fi

docker/rootfs/usr/local/bin/document_thumbnails

@@ -5,10 +5,8 @@ set -e
 
 cd "${PAPERLESS_SRC_DIR}"
 
-if [[ $(id -u) == 0 ]]; then
+if [[ -n "${USER_IS_NON_ROOT}" ]]; then
-	s6-setuidgid paperless python3 manage.py document_thumbnails "$@"
+		python3 manage.py document_thumbnails "$@"
 elif [[ $(id -un) == "paperless" ]]; then
-	python3 manage.py document_thumbnails "$@"
+	s6-setuidgid paperless python3 manage.py document_thumbnails "$@"
-else
-	echo "Unknown user."
 fi

docker/rootfs/usr/local/bin/mail_fetcher

@@ -5,10 +5,8 @@ set -e
 
 cd "${PAPERLESS_SRC_DIR}"
 
-if [[ $(id -u) == 0 ]]; then
+if [[ -n "${USER_IS_NON_ROOT}" ]]; then
-	s6-setuidgid paperless python3 manage.py mail_fetcher "$@"
+		python3 manage.py mail_fetcher "$@"
 elif [[ $(id -un) == "paperless" ]]; then
-	python3 manage.py mail_fetcher "$@"
+	s6-setuidgid paperless python3 manage.py mail_fetcher "$@"
-else
-	echo "Unknown user."
 fi

docker/rootfs/usr/local/bin/manage_superuser

@@ -5,10 +5,8 @@ set -e
 
 cd "${PAPERLESS_SRC_DIR}"
 
-if [[ $(id -u) == 0 ]]; then
+if [[ -n "${USER_IS_NON_ROOT}" ]]; then
-	s6-setuidgid paperless python3 manage.py manage_superuser "$@"
+		python3 manage.py manage_superuser "$@"
 elif [[ $(id -un) == "paperless" ]]; then
-	python3 manage.py manage_superuser "$@"
+	s6-setuidgid paperless python3 manage.py manage_superuser "$@"
-else
-	echo "Unknown user."
 fi

docker/rootfs/usr/local/bin/prune_audit_logs

@@ -5,10 +5,8 @@ set -e
 
 cd "${PAPERLESS_SRC_DIR}"
 
-if [[ $(id -u) == 0 ]]; then
+if [[ -n "${USER_IS_NON_ROOT}" ]]; then
-	s6-setuidgid paperless python3 manage.py prune_audit_logs "$@"
+		python3 manage.py prune_audit_logs "$@"
 elif [[ $(id -un) == "paperless" ]]; then
-	python3 manage.py prune_audit_logs "$@"
+	s6-setuidgid paperless python3 manage.py prune_audit_logs "$@"
-else
-	echo "Unknown user."
 fi

pyproject.toml

@@ -238,7 +238,7 @@ lint.isort.force-single-line = true
 
 [tool.codespell]
 write-changes = true
-ignore-words-list = "criterias,afterall,valeu,ureue,equest,ure,assertIn,Oktober"
+ignore-words-list = "criterias,afterall,valeu,ureue,equest,ure,assertIn,Oktober,commitish"
 skip = "src-ui/src/locale/*,src-ui/pnpm-lock.yaml,src-ui/e2e/*,src/paperless_mail/tests/samples/*,src/documents/tests/samples/*,*.po,*.json"
 
 [tool.pytest.ini_options]

src-ui/src/app/components/manage/management-list/management-list.component.spec.ts

@@ -229,6 +229,21 @@ describe('ManagementListComponent', () => {
     expect(reloadSpy).toHaveBeenCalled()
   })
 
+  it('should use the all list length for collection size when provided', fakeAsync(() => {
+    jest.spyOn(tagService, 'listFiltered').mockReturnValueOnce(
+      of({
+        count: 1,
+        all: [1, 2, 3],
+        results: tags.slice(0, 1),
+      })
+    )
+
+    component.reloadData()
+    tick(100)
+
+    expect(component.collectionSize).toBe(3)
+  }))
+
   it('should support quick filter for objects', () => {
     const qfSpy = jest.spyOn(documentListViewService, 'quickFilter')
     const filterButton = fixture.debugElement.queryAll(By.css('button'))[9]

src-ui/src/app/components/manage/management-list/management-list.component.ts

@@ -171,7 +171,7 @@ export abstract class ManagementListComponent<T extends MatchingModel>
         tap((c) => {
           this.unfilteredData = c.results
           this.data = this.filterData(c.results)
-          this.collectionSize = c.count
+          this.collectionSize = c.all?.length ?? c.count
         }),
         delay(100)
       )

src/documents/data_models.py

@@ -115,7 +115,7 @@ class DocumentMetadataOverrides:
             ).values_list("id", flat=True),
         )
         overrides.custom_fields = {
-            custom_field.id: custom_field.value
+            custom_field.field.id: custom_field.value
             for custom_field in doc.custom_fields.all()
         }
 

src/documents/index.py

@@ -602,7 +602,7 @@ def rewrite_natural_date_keywords(query_string: str) -> str:
 
             case "this year":
                 start = datetime(local_now.year, 1, 1, 0, 0, 0, tzinfo=tz)
-                end = datetime.combine(today, time.max, tzinfo=tz)
+                end = datetime(local_now.year, 12, 31, 23, 59, 59, tzinfo=tz)
 
             case "previous week":
                 days_since_monday = local_now.weekday()

src/documents/serialisers.py

@@ -580,30 +580,34 @@ class TagSerializer(MatchingModelSerializer, OwnedObjectSerializer):
         ),
     )
     def get_children(self, obj):
-        filter_q = self.context.get("document_count_filter")
+        children_map = self.context.get("children_map")
-        request = self.context.get("request")
+        if children_map is not None:
-        if filter_q is None:
+            children = children_map.get(obj.pk, [])
-            user = getattr(request, "user", None) if request else None
+        else:
-            filter_q = get_document_count_filter_for_user(user)
+            filter_q = self.context.get("document_count_filter")
-            self.context["document_count_filter"] = filter_q
+            request = self.context.get("request")
+            if filter_q is None:
+                user = getattr(request, "user", None) if request else None
+                filter_q = get_document_count_filter_for_user(user)
+                self.context["document_count_filter"] = filter_q
 
-        children_queryset = (
+            children = (
-            obj.get_children_queryset()
+                obj.get_children_queryset()
-            .select_related("owner")
+                .select_related("owner")
-            .annotate(document_count=Count("documents", filter=filter_q))
+                .annotate(document_count=Count("documents", filter=filter_q))
-        )
+            )
 
-        view = self.context.get("view")
+            view = self.context.get("view")
-        ordering = (
+            ordering = (
-            OrderingFilter().get_ordering(request, children_queryset, view)
+                OrderingFilter().get_ordering(request, children, view)
-            if request and view
+                if request and view
-            else None
+                else None
-        )
+            )
-        ordering = ordering or (Lower("name"),)
+            ordering = ordering or (Lower("name"),)
-        children_queryset = children_queryset.order_by(*ordering)
+            children = children.order_by(*ordering)
 
         serializer = TagSerializer(
-            children_queryset,
+            children,
             many=True,
             user=self.user,
             full_perms=self.full_perms,

src/documents/tests/test_index.py

@@ -180,7 +180,7 @@ class TestRewriteNaturalDateKeywords(SimpleTestCase):
             (
                 "added:this year",
                 datetime(2025, 7, 15, 12, 0, 0, tzinfo=timezone.utc),
-                ("added:[20250101", "TO 20250715"),
+                ("added:[20250101", "TO 20251231"),
             ),
             (
                 "added:previous year",

src/documents/views.py

@@ -448,8 +448,47 @@ class TagViewSet(ModelViewSet, PermissionsAwareDocumentCountMixin):
     def get_serializer_context(self):
         context = super().get_serializer_context()
         context["document_count_filter"] = self.get_document_count_filter()
+        if hasattr(self, "_children_map"):
+            context["children_map"] = self._children_map
         return context
 
+    def list(self, request, *args, **kwargs):
+        """
+        Build a children map once to avoid per-parent queries in the serializer.
+        """
+        queryset = self.filter_queryset(self.get_queryset())
+        ordering = OrderingFilter().get_ordering(request, queryset, self) or (
+            Lower("name"),
+        )
+        queryset = queryset.order_by(*ordering)
+
+        all_tags = list(queryset)
+        descendant_pks = {pk for tag in all_tags for pk in tag.get_descendants_pks()}
+
+        if descendant_pks:
+            filter_q = self.get_document_count_filter()
+            children_source = list(
+                Tag.objects.filter(pk__in=descendant_pks | {t.pk for t in all_tags})
+                .select_related("owner")
+                .annotate(document_count=Count("documents", filter=filter_q))
+                .order_by(*ordering),
+            )
+        else:
+            children_source = all_tags
+
+        children_map = {}
+        for tag in children_source:
+            children_map.setdefault(tag.tn_parent_id, []).append(tag)
+        self._children_map = children_map
+
+        page = self.paginate_queryset(queryset)
+        serializer = self.get_serializer(page, many=True)
+        response = self.get_paginated_response(serializer.data)
+        if descendant_pks:
+            # Include children in the "all" field, if needed
+            response.data["all"] = [tag.pk for tag in children_source]
+        return response
+
     def perform_update(self, serializer):
         old_parent = self.get_object().get_parent()
         tag = serializer.save()