Compare commits

..

4 Commits

Author SHA1 Message Date
Trenton H
0e59cf05a5 Merge branch 'dev' into patch-1 2025-04-14 07:39:31 -07:00
Hannes Ortmeier
588fd0207d chore: Bump celery to 5.5.1 (#9642) 2025-04-14 14:28:04 +00:00
shamoon
6dea158de9 Fix: prevent self-linking when bulk edit doc link (#9629) 2025-04-14 07:12:50 -07:00
Thom Wiggers
77528a3426 Delete unused docker/docker-entrypoint.sh 2025-04-10 17:37:23 +02:00
241 changed files with 9005 additions and 9170 deletions

View File

@@ -44,7 +44,7 @@ services:
- ..:/usr/src/paperless/paperless-ngx:delegated
- ../.devcontainer/vscode:/usr/src/paperless/paperless-ngx/.vscode:delegated # VSCode config files
- virtualenv:/usr/src/paperless/paperless-ngx/.venv # Virtual environment persisted in volume
- /usr/src/paperless/paperless-ngx/src/paperless/static/frontend # Static frontend files exist only in container
- /usr/src/paperless/paperless-ngx/src/documents/static/frontend # Static frontend files exist only in container
- /usr/src/paperless/paperless-ngx/src/.pytest_cache
- /usr/src/paperless/paperless-ngx/.ruff_cache
- /usr/src/paperless/paperless-ngx/htmlcov
@@ -58,11 +58,11 @@ services:
PAPERLESS_TIKA_ENABLED: 1
PAPERLESS_TIKA_GOTENBERG_ENDPOINT: http://gotenberg:3000
PAPERLESS_TIKA_ENDPOINT: http://tika:9998
PAPERLESS_STATICDIR: ./src/paperless/static
PAPERLESS_STATICDIR: ./src/documents/static
PAPERLESS_DEBUG: true
# Overrides default command so things don't shut down after the process ends.
command: /bin/sh -c "chown -R paperless:paperless /usr/src/paperless/paperless-ngx/src/paperless/static/frontend && chown -R paperless:paperless /usr/src/paperless/paperless-ngx/.ruff_cache && while sleep 1000; do :; done"
command: /bin/sh -c "chown -R paperless:paperless /usr/src/paperless/paperless-ngx/src/documents/static/frontend && chown -R paperless:paperless /usr/src/paperless/paperless-ngx/.ruff_cache && while sleep 1000; do :; done"
gotenberg:
image: docker.io/gotenberg/gotenberg:8.17

View File

@@ -430,13 +430,13 @@ jobs:
name: Export frontend artifact from docker
run: |
docker create --name frontend-extract ${{ fromJSON(steps.docker-meta.outputs.json).tags[0] }}
docker cp frontend-extract:/usr/src/paperless/src/paperless/static/frontend src/paperless/static/frontend/
docker cp frontend-extract:/usr/src/paperless/src/documents/static/frontend src/documents/static/frontend/
-
name: Upload frontend artifact
uses: actions/upload-artifact@v4
with:
name: frontend-compiled
path: src/paperless/static/frontend/
path: src/documents/static/frontend/
retention-days: 7
build-release:
@@ -476,7 +476,7 @@ jobs:
uses: actions/download-artifact@v4
with:
name: frontend-compiled
path: src/paperless/static/frontend/
path: src/documents/static/frontend/
-
name: Download documentation artifact
uses: actions/download-artifact@v4

2
.gitignore vendored
View File

@@ -94,7 +94,7 @@ scripts/nuke
/export/
# this is where the compiled frontend is moved to.
/src/paperless/static/frontend/
/src/documents/static/frontend/
# mac os
.DS_Store

View File

@@ -234,7 +234,7 @@ RUN --mount=type=cache,target=${UV_CACHE_DIR},id=python-cache \
COPY --chown=1000:1000 ./src ./
# copy frontend
COPY --from=compile-frontend --chown=1000:1000 /src/src/paperless/static/frontend/ ./paperless/static/frontend/
COPY --from=compile-frontend --chown=1000:1000 /src/src/documents/static/frontend/ ./documents/static/frontend/
# add users, setup scripts
# Mount the compiled frontend to expected location

View File

@@ -1,179 +0,0 @@
#!/usr/bin/env bash
set -e
# Source: https://github.com/sameersbn/docker-gitlab/
map_uidgid() {
local -r usermap_original_uid=$(id -u paperless)
local -r usermap_original_gid=$(id -g paperless)
local -r usermap_new_uid=${USERMAP_UID:-$usermap_original_uid}
local -r usermap_new_gid=${USERMAP_GID:-${usermap_original_gid:-$usermap_new_uid}}
if [[ ${usermap_new_uid} != "${usermap_original_uid}" || ${usermap_new_gid} != "${usermap_original_gid}" ]]; then
echo "Mapping UID and GID for paperless:paperless to $usermap_new_uid:$usermap_new_gid"
usermod --non-unique --uid "${usermap_new_uid}" paperless
groupmod --non-unique --gid "${usermap_new_gid}" paperless
fi
}
map_folders() {
# Export these so they can be used in docker-prepare.sh
export DATA_DIR="${PAPERLESS_DATA_DIR:-/usr/src/paperless/data}"
export MEDIA_ROOT_DIR="${PAPERLESS_MEDIA_ROOT:-/usr/src/paperless/media}"
export CONSUME_DIR="${PAPERLESS_CONSUMPTION_DIR:-/usr/src/paperless/consume}"
}
custom_container_init() {
# Mostly borrowed from the LinuxServer.io base image
# https://github.com/linuxserver/docker-baseimage-ubuntu/tree/bionic/root/etc/cont-init.d
local -r custom_script_dir="/custom-cont-init.d"
# Tamper checking.
# Don't run files which are owned by anyone except root
# Don't run files which are writeable by others
if [ -d "${custom_script_dir}" ]; then
if [ -n "$(/usr/bin/find "${custom_script_dir}" -maxdepth 1 ! -user root)" ]; then
echo "**** Potential tampering with custom scripts detected ****"
echo "**** The folder '${custom_script_dir}' must be owned by root ****"
return 0
fi
if [ -n "$(/usr/bin/find "${custom_script_dir}" -maxdepth 1 -perm -o+w)" ]; then
echo "**** The folder '${custom_script_dir}' or some of contents have write permissions for others, which is a security risk. ****"
echo "**** Please review the permissions and their contents to make sure they are owned by root, and can only be modified by root. ****"
return 0
fi
# Make sure custom init directory has files in it
if [ -n "$(/bin/ls --almost-all "${custom_script_dir}" 2>/dev/null)" ]; then
echo "[custom-init] files found in ${custom_script_dir} executing"
# Loop over files in the directory
for SCRIPT in "${custom_script_dir}"/*; do
NAME="$(basename "${SCRIPT}")"
if [ -f "${SCRIPT}" ]; then
echo "[custom-init] ${NAME}: executing..."
/bin/bash "${SCRIPT}"
echo "[custom-init] ${NAME}: exited $?"
elif [ ! -f "${SCRIPT}" ]; then
echo "[custom-init] ${NAME}: is not a file"
fi
done
else
echo "[custom-init] no custom files found exiting..."
fi
fi
}
initialize() {
# Setup environment from secrets before anything else
# Check for a version of this var with _FILE appended
# and convert the contents to the env var value
# Source it so export is persistent
# shellcheck disable=SC1091
source /sbin/env-from-file.sh
# Change the user and group IDs if needed
map_uidgid
# Check for overrides of certain folders
map_folders
local -r export_dir="/usr/src/paperless/export"
for dir in \
"${export_dir}" \
"${DATA_DIR}" "${DATA_DIR}/index" \
"${MEDIA_ROOT_DIR}" "${MEDIA_ROOT_DIR}/documents" "${MEDIA_ROOT_DIR}/documents/originals" "${MEDIA_ROOT_DIR}/documents/thumbnails" \
"${CONSUME_DIR}"; do
if [[ ! -d "${dir}" ]]; then
echo "Creating directory ${dir}"
mkdir --parents --verbose "${dir}"
fi
done
local -r tmp_dir="${PAPERLESS_SCRATCH_DIR:=/tmp/paperless}"
echo "Creating directory scratch directory ${tmp_dir}"
mkdir --parents --verbose "${tmp_dir}"
set +e
echo "Adjusting permissions of paperless files. This may take a while."
chown -R paperless:paperless "${tmp_dir}"
for dir in \
"${export_dir}" \
"${DATA_DIR}" \
"${MEDIA_ROOT_DIR}" \
"${CONSUME_DIR}"; do
find "${dir}" -not \( -user paperless -and -group paperless \) -exec chown --changes paperless:paperless {} +
done
set -e
"${gosu_cmd[@]}" /sbin/docker-prepare.sh
# Leave this last thing
custom_container_init
}
install_languages() {
echo "Installing languages..."
read -ra langs <<<"$1"
# Check that it is not empty
if [ ${#langs[@]} -eq 0 ]; then
return
fi
# Build list of packages to install
to_install=()
for lang in "${langs[@]}"; do
pkg="tesseract-ocr-$lang"
if dpkg --status "$pkg" &>/dev/null; then
echo "Package $pkg already installed!"
continue
else
to_install+=("$pkg")
fi
done
# Use apt only when we install packages
if [ ${#to_install[@]} -gt 0 ]; then
apt-get update
for pkg in "${to_install[@]}"; do
if ! apt-cache show "$pkg" &>/dev/null; then
echo "Skipped $pkg: Package not found! :("
continue
fi
echo "Installing package $pkg..."
if ! apt-get --assume-yes install "$pkg" &>/dev/null; then
echo "Could not install $pkg"
exit 1
fi
done
fi
}
echo "Paperless-ngx docker container starting..."
gosu_cmd=(gosu paperless)
if [ "$(id --user)" == "$(id --user paperless)" ]; then
gosu_cmd=()
fi
# Install additional languages if specified
if [[ -n "$PAPERLESS_OCR_LANGUAGES" ]]; then
install_languages "$PAPERLESS_OCR_LANGUAGES"
fi
initialize
if [[ "$1" != "/"* ]]; then
echo Executing management command "$@"
exec "${gosu_cmd[@]}" python3 manage.py "$@"
else
echo Executing "$@"
exec "$@"
fi

View File

@@ -390,7 +390,7 @@ Custom parsers can be added to Paperless-ngx to support more file types. In
order to do that, you need to write the parser itself and announce its
existence to Paperless-ngx.
The parser itself must extend `paperless.parsers.DocumentParser` and
The parser itself must extend `documents.parsers.DocumentParser` and
must implement the methods `parse` and `get_thumbnail`. You can provide
your own implementation to `get_date` if you don't want to rely on
Paperless-ngx' default date guessing mechanisms.
@@ -418,7 +418,7 @@ class MyCustomParser(DocumentParser):
```
If you encounter any issues during parsing, raise a
`paperless.parsers.ParseError`.
`documents.parsers.ParseError`.
The `self.tempdir` directory is a temporary directory that is guaranteed
to be empty and removed after consumption finished. You can use that

View File

@@ -191,7 +191,7 @@ This might have multiple reasons.
either manually or as part of the docker image build.
If the front end is still missing, make sure that the front end is
compiled (files present in `src/paperless/static/frontend`). If it
compiled (files present in `src/documents/static/frontend`). If it
is not, you need to compile the front end yourself or download the
release archive instead of cloning the repository.

View File

@@ -16,7 +16,7 @@ classifiers = [
dependencies = [
"bleach~=6.2.0",
"celery[redis]~=5.4.0",
"celery[redis]~=5.5.1",
"channels~=4.2",
"channels-redis~=4.2",
"concurrent-log-handler~=0.9.25",
@@ -200,60 +200,63 @@ lint.per-file-ignores."docker/wait-for-redis.py" = [
"INP001",
"T201",
]
lint.per-file-ignores."src/documents/file_handling.py" = [
"PTH",
] # TODO Enable & remove
lint.per-file-ignores."src/documents/management/commands/document_consumer.py" = [
"PTH",
] # TODO Enable & remove
lint.per-file-ignores."src/documents/management/commands/document_exporter.py" = [
"PTH",
] # TODO Enable & remove
lint.per-file-ignores."src/documents/migrations/1012_fix_archive_files.py" = [
"PTH",
] # TODO Enable & remove
lint.per-file-ignores."src/documents/models.py" = [
"SIM115",
]
lint.per-file-ignores."src/documents/parsers.py" = [
"PTH",
] # TODO Enable & remove
lint.per-file-ignores."src/documents/signals/handlers.py" = [
"PTH",
] # TODO Enable & remove
lint.per-file-ignores."src/documents/tests/test_consumer.py" = [
"PTH",
] # TODO Enable & remove
lint.per-file-ignores."src/documents/tests/test_file_handling.py" = [
"PTH",
] # TODO Enable & remove
lint.per-file-ignores."src/documents/tests/test_management.py" = [
"PTH",
] # TODO Enable & remove
lint.per-file-ignores."src/documents/tests/test_management_consumer.py" = [
"PTH",
] # TODO Enable & remove
lint.per-file-ignores."src/documents/tests/test_management_exporter.py" = [
"PTH",
] # TODO Enable & remove
lint.per-file-ignores."src/documents/tests/test_migration_archive_files.py" = [
"PTH",
] # TODO Enable & remove
lint.per-file-ignores."src/documents/tests/test_migration_document_pages_count.py" = [
"PTH",
] # TODO Enable & remove
lint.per-file-ignores."src/documents/tests/test_migration_mime_type.py" = [
"PTH",
] # TODO Enable & remove
lint.per-file-ignores."src/documents/tests/test_sanity_check.py" = [
"PTH",
] # TODO Enable & remove
lint.per-file-ignores."src/documents/views.py" = [
"PTH",
] # TODO Enable & remove
lint.per-file-ignores."src/paperless/checks.py" = [
"PTH",
] # TODO Enable & remove
lint.per-file-ignores."src/paperless/file_handling.py" = [
"PTH",
] # TODO Enable & remove
lint.per-file-ignores."src/paperless/management/commands/document_consumer.py" = [
"PTH",
] # TODO Enable & remove
lint.per-file-ignores."src/paperless/management/commands/document_exporter.py" = [
"PTH",
] # TODO Enable & remove
lint.per-file-ignores."src/paperless/models.py" = [
"SIM115",
]
lint.per-file-ignores."src/paperless/parsers.py" = [
"PTH",
] # TODO Enable & remove
lint.per-file-ignores."src/paperless/settings.py" = [
"PTH",
] # TODO Enable & remove
lint.per-file-ignores."src/paperless/signals/handlers.py" = [
"PTH",
] # TODO Enable & remove
lint.per-file-ignores."src/paperless/tests/test_consumer.py" = [
"PTH",
] # TODO Enable & remove
lint.per-file-ignores."src/paperless/tests/test_file_handling.py" = [
"PTH",
] # TODO Enable & remove
lint.per-file-ignores."src/paperless/tests/test_management.py" = [
"PTH",
] # TODO Enable & remove
lint.per-file-ignores."src/paperless/tests/test_management_consumer.py" = [
"PTH",
] # TODO Enable & remove
lint.per-file-ignores."src/paperless/tests/test_management_exporter.py" = [
"PTH",
] # TODO Enable & remove
lint.per-file-ignores."src/paperless/tests/test_migration_archive_files.py" = [
"PTH",
] # TODO Enable & remove
lint.per-file-ignores."src/paperless/tests/test_migration_document_pages_count.py" = [
"PTH",
] # TODO Enable & remove
lint.per-file-ignores."src/paperless/tests/test_migration_mime_type.py" = [
"PTH",
] # TODO Enable & remove
lint.per-file-ignores."src/paperless/tests/test_sanity_check.py" = [
"PTH",
] # TODO Enable & remove
lint.per-file-ignores."src/paperless/views.py" = [
"PTH",
] # TODO Enable & remove

View File

@@ -100,7 +100,7 @@
"with": "src/environments/environment.prod.ts"
}
],
"outputPath": "../src/paperless/static/frontend/",
"outputPath": "../src/documents/static/frontend/",
"optimization": true,
"outputHashing": "none",
"sourceMap": false,

View File

@@ -1 +1,5 @@
__all__ = []
# this is here so that django finds the checks.
from documents.checks import changed_password_check
from documents.checks import parser_check
__all__ = ["changed_password_check", "parser_check"]

214
src/documents/admin.py Normal file
View File

@@ -0,0 +1,214 @@
from django.conf import settings
from django.contrib import admin
from guardian.admin import GuardedModelAdmin
from documents.models import Correspondent
from documents.models import CustomField
from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import DocumentType
from documents.models import Note
from documents.models import PaperlessTask
from documents.models import SavedView
from documents.models import SavedViewFilterRule
from documents.models import ShareLink
from documents.models import StoragePath
from documents.models import Tag
if settings.AUDIT_LOG_ENABLED:
from auditlog.admin import LogEntryAdmin
from auditlog.models import LogEntry
class CorrespondentAdmin(GuardedModelAdmin):
list_display = ("name", "match", "matching_algorithm")
list_filter = ("matching_algorithm",)
list_editable = ("match", "matching_algorithm")
class TagAdmin(GuardedModelAdmin):
list_display = ("name", "color", "match", "matching_algorithm")
list_filter = ("matching_algorithm",)
list_editable = ("color", "match", "matching_algorithm")
search_fields = ("color", "name")
class DocumentTypeAdmin(GuardedModelAdmin):
list_display = ("name", "match", "matching_algorithm")
list_filter = ("matching_algorithm",)
list_editable = ("match", "matching_algorithm")
class DocumentAdmin(GuardedModelAdmin):
search_fields = ("correspondent__name", "title", "content", "tags__name")
readonly_fields = (
"added",
"modified",
"mime_type",
"storage_type",
"filename",
"checksum",
"archive_filename",
"archive_checksum",
"original_filename",
"deleted_at",
)
list_display_links = ("title",)
list_display = ("id", "title", "mime_type", "filename", "archive_filename")
list_filter = (
("mime_type"),
("archive_serial_number", admin.EmptyFieldListFilter),
("archive_filename", admin.EmptyFieldListFilter),
)
filter_horizontal = ("tags",)
ordering = ["-id"]
date_hierarchy = "created"
def has_add_permission(self, request):
return False
def created_(self, obj):
return obj.created.date().strftime("%Y-%m-%d")
created_.short_description = "Created"
def get_queryset(self, request): # pragma: no cover
"""
Include trashed documents
"""
return Document.global_objects.all()
def delete_queryset(self, request, queryset):
from documents import index
with index.open_index_writer() as writer:
for o in queryset:
index.remove_document(writer, o)
super().delete_queryset(request, queryset)
def delete_model(self, request, obj):
from documents import index
index.remove_document_from_index(obj)
super().delete_model(request, obj)
def save_model(self, request, obj, form, change):
from documents import index
index.add_or_update_document(obj)
super().save_model(request, obj, form, change)
class RuleInline(admin.TabularInline):
model = SavedViewFilterRule
class SavedViewAdmin(GuardedModelAdmin):
list_display = ("name", "owner")
inlines = [RuleInline]
def get_queryset(self, request): # pragma: no cover
return super().get_queryset(request).select_related("owner")
class StoragePathInline(admin.TabularInline):
model = StoragePath
class StoragePathAdmin(GuardedModelAdmin):
list_display = ("name", "path", "match", "matching_algorithm")
list_filter = ("path", "matching_algorithm")
list_editable = ("path", "match", "matching_algorithm")
class TaskAdmin(admin.ModelAdmin):
list_display = ("task_id", "task_file_name", "task_name", "date_done", "status")
list_filter = ("status", "date_done", "task_name")
search_fields = ("task_name", "task_id", "status", "task_file_name")
readonly_fields = (
"task_id",
"task_file_name",
"task_name",
"status",
"date_created",
"date_started",
"date_done",
"result",
)
class NotesAdmin(GuardedModelAdmin):
list_display = ("user", "created", "note", "document")
list_filter = ("created", "user")
list_display_links = ("created",)
raw_id_fields = ("document",)
search_fields = ("document__title",)
def get_queryset(self, request): # pragma: no cover
return (
super()
.get_queryset(request)
.select_related("user", "document__correspondent")
)
class ShareLinksAdmin(GuardedModelAdmin):
list_display = ("created", "expiration", "document")
list_filter = ("created", "expiration", "owner")
list_display_links = ("created",)
raw_id_fields = ("document",)
def get_queryset(self, request): # pragma: no cover
return super().get_queryset(request).select_related("document__correspondent")
class CustomFieldsAdmin(GuardedModelAdmin):
fields = ("name", "created", "data_type")
readonly_fields = ("created", "data_type")
list_display = ("name", "created", "data_type")
list_filter = ("created", "data_type")
class CustomFieldInstancesAdmin(GuardedModelAdmin):
fields = ("field", "document", "created", "value")
readonly_fields = ("field", "document", "created", "value")
list_display = ("field", "document", "value", "created")
search_fields = ("document__title",)
list_filter = ("created", "field")
def get_queryset(self, request): # pragma: no cover
return (
super()
.get_queryset(request)
.select_related("field", "document__correspondent")
)
admin.site.register(Correspondent, CorrespondentAdmin)
admin.site.register(Tag, TagAdmin)
admin.site.register(DocumentType, DocumentTypeAdmin)
admin.site.register(Document, DocumentAdmin)
admin.site.register(SavedView, SavedViewAdmin)
admin.site.register(StoragePath, StoragePathAdmin)
admin.site.register(PaperlessTask, TaskAdmin)
admin.site.register(Note, NotesAdmin)
admin.site.register(ShareLink, ShareLinksAdmin)
admin.site.register(CustomField, CustomFieldsAdmin)
admin.site.register(CustomFieldInstance, CustomFieldInstancesAdmin)
if settings.AUDIT_LOG_ENABLED:
class LogEntryAUDIT(LogEntryAdmin):
def has_delete_permission(self, request, obj=None):
return False
admin.site.unregister(LogEntry)
admin.site.register(LogEntry, LogEntryAUDIT)

View File

@@ -4,4 +4,30 @@ from django.utils.translation import gettext_lazy as _
class DocumentsConfig(AppConfig):
name = "documents"
verbose_name = _("Documents (legacy)")
verbose_name = _("Documents")
def ready(self):
from documents.signals import document_consumption_finished
from documents.signals import document_updated
from documents.signals.handlers import add_inbox_tags
from documents.signals.handlers import add_to_index
from documents.signals.handlers import run_workflows_added
from documents.signals.handlers import run_workflows_updated
from documents.signals.handlers import set_correspondent
from documents.signals.handlers import set_document_type
from documents.signals.handlers import set_storage_path
from documents.signals.handlers import set_tags
document_consumption_finished.connect(add_inbox_tags)
document_consumption_finished.connect(set_correspondent)
document_consumption_finished.connect(set_document_type)
document_consumption_finished.connect(set_tags)
document_consumption_finished.connect(set_storage_path)
document_consumption_finished.connect(add_to_index)
document_consumption_finished.connect(run_workflows_added)
document_updated.connect(run_workflows_updated)
import documents.schema # noqa: F401
AppConfig.ready(self)

View File

@@ -13,15 +13,15 @@ from pikepdf import Page
from pikepdf import PasswordError
from pikepdf import Pdf
from paperless.converters import convert_from_tiff_to_pdf
from paperless.data_models import ConsumableDocument
from paperless.models import Tag
from paperless.plugins.base import ConsumeTaskPlugin
from paperless.plugins.base import StopConsumeTaskError
from paperless.plugins.helpers import ProgressStatusOptions
from paperless.utils import copy_basic_file_stats
from paperless.utils import copy_file_with_basic_stats
from paperless.utils import maybe_override_pixel_limit
from documents.converters import convert_from_tiff_to_pdf
from documents.data_models import ConsumableDocument
from documents.models import Tag
from documents.plugins.base import ConsumeTaskPlugin
from documents.plugins.base import StopConsumeTaskError
from documents.plugins.helpers import ProgressStatusOptions
from documents.utils import copy_basic_file_stats
from documents.utils import copy_file_with_basic_stats
from documents.utils import maybe_override_pixel_limit
if TYPE_CHECKING:
from collections.abc import Callable
@@ -123,7 +123,7 @@ class BarcodePlugin(ConsumeTaskPlugin):
),
).resolve()
from paperless import tasks
from documents import tasks
# Create the split document tasks
for new_document in self.separate_pages(separator_pages):

View File

@@ -8,7 +8,7 @@ if TYPE_CHECKING:
from collections.abc import Callable
from zipfile import ZipFile
from paperless.models import Document
from documents.models import Document
class BulkArchiveStrategy:

View File

@@ -16,20 +16,20 @@ from django.conf import settings
from django.db.models import Q
from django.utils import timezone
from paperless.data_models import ConsumableDocument
from paperless.data_models import DocumentMetadataOverrides
from paperless.data_models import DocumentSource
from paperless.models import Correspondent
from paperless.models import CustomField
from paperless.models import CustomFieldInstance
from paperless.models import Document
from paperless.models import DocumentType
from paperless.models import StoragePath
from paperless.permissions import set_permissions_for_object
from paperless.plugins.helpers import DocumentsStatusManager
from paperless.tasks import bulk_update_documents
from paperless.tasks import consume_file
from paperless.tasks import update_document_content_maybe_archive_file
from documents.data_models import ConsumableDocument
from documents.data_models import DocumentMetadataOverrides
from documents.data_models import DocumentSource
from documents.models import Correspondent
from documents.models import CustomField
from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import DocumentType
from documents.models import StoragePath
from documents.permissions import set_permissions_for_object
from documents.plugins.helpers import DocumentsStatusManager
from documents.tasks import bulk_update_documents
from documents.tasks import consume_file
from documents.tasks import update_document_content_maybe_archive_file
if TYPE_CHECKING:
from django.contrib.auth.models import User
@@ -179,6 +179,12 @@ def modify_custom_fields(
custom_field.data_type
]
defaults[value_field] = value
if (
custom_field.data_type == CustomField.FieldDataType.DOCUMENTLINK
and doc_id in value
):
# Prevent self-linking
continue
CustomFieldInstance.objects.update_or_create(
document_id=doc_id,
field_id=field_id,
@@ -220,7 +226,7 @@ def delete(doc_ids: list[int]) -> Literal["OK"]:
try:
Document.objects.filter(id__in=doc_ids).delete()
from paperless import index
from documents import index
with index.open_index_writer() as writer:
for id in doc_ids:

View File

@@ -8,10 +8,10 @@ from typing import Final
from django.core.cache import cache
from paperless.models import Document
from documents.models import Document
if TYPE_CHECKING:
from paperless.classifier import DocumentClassifier
from documents.classifier import DocumentClassifier
logger = logging.getLogger("paperless.caching")
@@ -53,7 +53,7 @@ def get_suggestion_cache(document_id: int) -> SuggestionCacheData | None:
The classifier needs to be matching in format and hash and the suggestions need to
have been cached once.
"""
from paperless.classifier import DocumentClassifier
from documents.classifier import DocumentClassifier
doc_key = get_suggestion_cache_key(document_id)
cache_hits = cache.get_many([CLASSIFIER_VERSION_KEY, CLASSIFIER_HASH_KEY, doc_key])

88
src/documents/checks.py Normal file
View File

@@ -0,0 +1,88 @@
import textwrap
from django.conf import settings
from django.core.checks import Error
from django.core.checks import Warning
from django.core.checks import register
from django.core.exceptions import FieldError
from django.db.utils import OperationalError
from django.db.utils import ProgrammingError
from documents.signals import document_consumer_declaration
from documents.templating.utils import convert_format_str_to_template_format
@register()
def changed_password_check(app_configs, **kwargs):
from documents.models import Document
from paperless.db import GnuPG
try:
encrypted_doc = (
Document.objects.filter(
storage_type=Document.STORAGE_TYPE_GPG,
)
.only("pk", "storage_type")
.first()
)
except (OperationalError, ProgrammingError, FieldError):
return [] # No documents table yet
if encrypted_doc:
if not settings.PASSPHRASE:
return [
Error(
"The database contains encrypted documents but no password is set.",
),
]
if not GnuPG.decrypted(encrypted_doc.source_file):
return [
Error(
textwrap.dedent(
"""
The current password doesn't match the password of the
existing documents.
If you intend to change your password, you must first export
all of the old documents, start fresh with the new password
and then re-import them."
""",
),
),
]
return []
@register()
def parser_check(app_configs, **kwargs):
parsers = []
for response in document_consumer_declaration.send(None):
parsers.append(response[1])
if len(parsers) == 0:
return [
Error(
"No parsers found. This is a bug. The consumer won't be "
"able to consume any documents without parsers.",
),
]
else:
return []
@register()
def filename_format_check(app_configs, **kwargs):
if settings.FILENAME_FORMAT:
converted_format = convert_format_str_to_template_format(
settings.FILENAME_FORMAT,
)
if converted_format != settings.FILENAME_FORMAT:
return [
Warning(
f"Filename format {settings.FILENAME_FORMAT} is using the old style, please update to use double curly brackets",
hint=converted_format,
),
]
return []

View File

@@ -17,12 +17,12 @@ if TYPE_CHECKING:
from django.conf import settings
from django.core.cache import cache
from paperless.caching import CACHE_50_MINUTES
from paperless.caching import CLASSIFIER_HASH_KEY
from paperless.caching import CLASSIFIER_MODIFIED_KEY
from paperless.caching import CLASSIFIER_VERSION_KEY
from paperless.models import Document
from paperless.models import MatchingModel
from documents.caching import CACHE_50_MINUTES
from documents.caching import CLASSIFIER_HASH_KEY
from documents.caching import CLASSIFIER_MODIFIED_KEY
from documents.caching import CLASSIFIER_VERSION_KEY
from documents.models import Document
from documents.models import MatchingModel
logger = logging.getLogger("paperless.classifier")

View File

@@ -4,14 +4,14 @@ from datetime import timezone
from django.conf import settings
from django.core.cache import cache
from paperless.caching import CACHE_5_MINUTES
from paperless.caching import CACHE_50_MINUTES
from paperless.caching import CLASSIFIER_HASH_KEY
from paperless.caching import CLASSIFIER_MODIFIED_KEY
from paperless.caching import CLASSIFIER_VERSION_KEY
from paperless.caching import get_thumbnail_modified_key
from paperless.classifier import DocumentClassifier
from paperless.models import Document
from documents.caching import CACHE_5_MINUTES
from documents.caching import CACHE_50_MINUTES
from documents.caching import CLASSIFIER_HASH_KEY
from documents.caching import CLASSIFIER_MODIFIED_KEY
from documents.caching import CLASSIFIER_VERSION_KEY
from documents.caching import get_thumbnail_modified_key
from documents.classifier import DocumentClassifier
from documents.models import Document
def suggestions_etag(request, pk: int) -> str | None:

View File

@@ -15,38 +15,38 @@ from django.utils import timezone
from filelock import FileLock
from rest_framework.reverse import reverse
from paperless.classifier import load_classifier
from paperless.data_models import ConsumableDocument
from paperless.data_models import DocumentMetadataOverrides
from paperless.file_handling import create_source_path_directory
from paperless.file_handling import generate_unique_filename
from paperless.loggers import LoggingMixin
from paperless.models import Correspondent
from paperless.models import CustomField
from paperless.models import CustomFieldInstance
from paperless.models import Document
from paperless.models import DocumentType
from paperless.models import StoragePath
from paperless.models import Tag
from paperless.models import WorkflowTrigger
from paperless.parsers import DocumentParser
from paperless.parsers import ParseError
from paperless.parsers import get_parser_class_for_mime_type
from paperless.parsers import parse_date
from paperless.permissions import set_permissions_for_object
from paperless.plugins.base import AlwaysRunPluginMixin
from paperless.plugins.base import ConsumeTaskPlugin
from paperless.plugins.base import NoCleanupPluginMixin
from paperless.plugins.base import NoSetupPluginMixin
from paperless.plugins.helpers import ProgressManager
from paperless.plugins.helpers import ProgressStatusOptions
from paperless.signals import document_consumption_finished
from paperless.signals import document_consumption_started
from paperless.signals.handlers import run_workflows
from paperless.templating.workflows import parse_w_workflow_placeholders
from paperless.utils import copy_basic_file_stats
from paperless.utils import copy_file_with_basic_stats
from paperless.utils import run_subprocess
from documents.classifier import load_classifier
from documents.data_models import ConsumableDocument
from documents.data_models import DocumentMetadataOverrides
from documents.file_handling import create_source_path_directory
from documents.file_handling import generate_unique_filename
from documents.loggers import LoggingMixin
from documents.models import Correspondent
from documents.models import CustomField
from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import DocumentType
from documents.models import StoragePath
from documents.models import Tag
from documents.models import WorkflowTrigger
from documents.parsers import DocumentParser
from documents.parsers import ParseError
from documents.parsers import get_parser_class_for_mime_type
from documents.parsers import parse_date
from documents.permissions import set_permissions_for_object
from documents.plugins.base import AlwaysRunPluginMixin
from documents.plugins.base import ConsumeTaskPlugin
from documents.plugins.base import NoCleanupPluginMixin
from documents.plugins.base import NoSetupPluginMixin
from documents.plugins.helpers import ProgressManager
from documents.plugins.helpers import ProgressStatusOptions
from documents.signals import document_consumption_finished
from documents.signals import document_consumption_started
from documents.signals.handlers import run_workflows
from documents.templating.workflows import parse_w_workflow_placeholders
from documents.utils import copy_basic_file_stats
from documents.utils import copy_file_with_basic_stats
from documents.utils import run_subprocess
from paperless_mail.parsers import MailDocumentParser

View File

@@ -1,8 +1,8 @@
from django.conf import settings as django_settings
from django.contrib.auth.models import User
from documents.models import Document
from paperless.config import GeneralConfig
from paperless.models import Document
def settings(request):

View File

@@ -4,9 +4,9 @@ import img2pdf
from django.conf import settings
from PIL import Image
from paperless.utils import copy_basic_file_stats
from paperless.utils import maybe_override_pixel_limit
from paperless.utils import run_subprocess
from documents.utils import copy_basic_file_stats
from documents.utils import maybe_override_pixel_limit
from documents.utils import run_subprocess
def convert_from_tiff_to_pdf(tiff_path: Path, target_directory: Path) -> Path:

View File

@@ -8,12 +8,12 @@ from typing import Final
from django.conf import settings
from pikepdf import Pdf
from paperless.consumer import ConsumerError
from paperless.converters import convert_from_tiff_to_pdf
from paperless.plugins.base import ConsumeTaskPlugin
from paperless.plugins.base import NoCleanupPluginMixin
from paperless.plugins.base import NoSetupPluginMixin
from paperless.plugins.base import StopConsumeTaskError
from documents.consumer import ConsumerError
from documents.converters import convert_from_tiff_to_pdf
from documents.plugins.base import ConsumeTaskPlugin
from documents.plugins.base import NoCleanupPluginMixin
from documents.plugins.base import NoSetupPluginMixin
from documents.plugins.base import StopConsumeTaskError
logger = logging.getLogger("paperless.double_sided")

View File

@@ -2,9 +2,9 @@ import os
from django.conf import settings
from paperless.models import Document
from paperless.templating.filepath import validate_filepath_template_and_render
from paperless.templating.utils import convert_format_str_to_template_format
from documents.models import Document
from documents.templating.filepath import validate_filepath_template_and_render
from documents.templating.utils import convert_format_str_to_template_format
def create_source_path_directory(source_path):

950
src/documents/filters.py Normal file
View File

@@ -0,0 +1,950 @@
from __future__ import annotations
import functools
import inspect
import json
import operator
from contextlib import contextmanager
from typing import TYPE_CHECKING
from django.contrib.contenttypes.models import ContentType
from django.db.models import Case
from django.db.models import CharField
from django.db.models import Count
from django.db.models import Exists
from django.db.models import IntegerField
from django.db.models import OuterRef
from django.db.models import Q
from django.db.models import Subquery
from django.db.models import Sum
from django.db.models import Value
from django.db.models import When
from django.db.models.functions import Cast
from django.utils.translation import gettext_lazy as _
from django_filters.rest_framework import BooleanFilter
from django_filters.rest_framework import Filter
from django_filters.rest_framework import FilterSet
from drf_spectacular.utils import extend_schema_field
from guardian.utils import get_group_obj_perms_model
from guardian.utils import get_user_obj_perms_model
from rest_framework import serializers
from rest_framework.filters import OrderingFilter
from rest_framework_guardian.filters import ObjectPermissionsFilter
from documents.models import Correspondent
from documents.models import CustomField
from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import DocumentType
from documents.models import PaperlessTask
from documents.models import ShareLink
from documents.models import StoragePath
from documents.models import Tag
if TYPE_CHECKING:
from collections.abc import Callable
CHAR_KWARGS = ["istartswith", "iendswith", "icontains", "iexact"]
ID_KWARGS = ["in", "exact"]
INT_KWARGS = ["exact", "gt", "gte", "lt", "lte", "isnull"]
DATE_KWARGS = [
"year",
"month",
"day",
"date__gt",
"date__gte",
"gt",
"gte",
"date__lt",
"date__lte",
"lt",
"lte",
]
CUSTOM_FIELD_QUERY_MAX_DEPTH = 10
CUSTOM_FIELD_QUERY_MAX_ATOMS = 20
class CorrespondentFilterSet(FilterSet):
class Meta:
model = Correspondent
fields = {
"id": ID_KWARGS,
"name": CHAR_KWARGS,
}
class TagFilterSet(FilterSet):
class Meta:
model = Tag
fields = {
"id": ID_KWARGS,
"name": CHAR_KWARGS,
}
class DocumentTypeFilterSet(FilterSet):
class Meta:
model = DocumentType
fields = {
"id": ID_KWARGS,
"name": CHAR_KWARGS,
}
class StoragePathFilterSet(FilterSet):
class Meta:
model = StoragePath
fields = {
"id": ID_KWARGS,
"name": CHAR_KWARGS,
"path": CHAR_KWARGS,
}
class ObjectFilter(Filter):
def __init__(self, *, exclude=False, in_list=False, field_name=""):
super().__init__()
self.exclude = exclude
self.in_list = in_list
self.field_name = field_name
def filter(self, qs, value):
if not value:
return qs
try:
object_ids = [int(x) for x in value.split(",")]
except ValueError:
return qs
if self.in_list:
qs = qs.filter(**{f"{self.field_name}__id__in": object_ids}).distinct()
else:
for obj_id in object_ids:
if self.exclude:
qs = qs.exclude(**{f"{self.field_name}__id": obj_id})
else:
qs = qs.filter(**{f"{self.field_name}__id": obj_id})
return qs
@extend_schema_field(serializers.BooleanField)
class InboxFilter(Filter):
def filter(self, qs, value):
if value == "true":
return qs.filter(tags__is_inbox_tag=True)
elif value == "false":
return qs.exclude(tags__is_inbox_tag=True)
else:
return qs
@extend_schema_field(serializers.CharField)
class TitleContentFilter(Filter):
def filter(self, qs, value):
if value:
return qs.filter(Q(title__icontains=value) | Q(content__icontains=value))
else:
return qs
@extend_schema_field(serializers.BooleanField)
class SharedByUser(Filter):
def filter(self, qs, value):
ctype = ContentType.objects.get_for_model(self.model)
UserObjectPermission = get_user_obj_perms_model()
GroupObjectPermission = get_group_obj_perms_model()
# see https://github.com/paperless-ngx/paperless-ngx/issues/5392, we limit subqueries
# to 1 because Postgres doesn't like returning > 1 row, but all we care about is > 0
return (
qs.filter(
owner_id=value,
)
.annotate(
num_shared_users=Count(
UserObjectPermission.objects.filter(
content_type=ctype,
object_pk=Cast(OuterRef("pk"), CharField()),
).values("user_id")[:1],
),
)
.annotate(
num_shared_groups=Count(
GroupObjectPermission.objects.filter(
content_type=ctype,
object_pk=Cast(OuterRef("pk"), CharField()),
).values("group_id")[:1],
),
)
.filter(
Q(num_shared_users__gt=0) | Q(num_shared_groups__gt=0),
)
if value is not None
else qs
)
class CustomFieldFilterSet(FilterSet):
class Meta:
model = CustomField
fields = {
"id": ID_KWARGS,
"name": CHAR_KWARGS,
}
@extend_schema_field(serializers.CharField)
class CustomFieldsFilter(Filter):
def filter(self, qs, value):
if value:
fields_with_matching_selects = CustomField.objects.filter(
extra_data__icontains=value,
)
option_ids = []
if fields_with_matching_selects.count() > 0:
for field in fields_with_matching_selects:
options = field.extra_data.get("select_options", [])
for _, option in enumerate(options):
if option.get("label").lower().find(value.lower()) != -1:
option_ids.extend([option.get("id")])
return (
qs.filter(custom_fields__field__name__icontains=value)
| qs.filter(custom_fields__value_text__icontains=value)
| qs.filter(custom_fields__value_bool__icontains=value)
| qs.filter(custom_fields__value_int__icontains=value)
| qs.filter(custom_fields__value_float__icontains=value)
| qs.filter(custom_fields__value_date__icontains=value)
| qs.filter(custom_fields__value_url__icontains=value)
| qs.filter(custom_fields__value_monetary__icontains=value)
| qs.filter(custom_fields__value_document_ids__icontains=value)
| qs.filter(custom_fields__value_select__in=option_ids)
)
else:
return qs
class MimeTypeFilter(Filter):
def filter(self, qs, value):
if value:
return qs.filter(mime_type__icontains=value)
else:
return qs
class SelectField(serializers.CharField):
def __init__(self, custom_field: CustomField):
self._options = custom_field.extra_data["select_options"]
super().__init__(max_length=16)
def to_internal_value(self, data):
# If the supplied value is the option label instead of the ID
try:
data = next(
option.get("id")
for option in self._options
if option.get("label") == data
)
except StopIteration:
pass
return super().to_internal_value(data)
def handle_validation_prefix(func: Callable):
"""
Catch ValidationErrors raised by the wrapped function
and add a prefix to the exception detail to track what causes the exception,
similar to nested serializers.
"""
def wrapper(*args, validation_prefix=None, **kwargs):
try:
return func(*args, **kwargs)
except serializers.ValidationError as e:
raise serializers.ValidationError({validation_prefix: e.detail})
# Update the signature to include the validation_prefix argument
old_sig = inspect.signature(func)
new_param = inspect.Parameter("validation_prefix", inspect.Parameter.KEYWORD_ONLY)
new_sig = old_sig.replace(parameters=[*old_sig.parameters.values(), new_param])
# Apply functools.wraps and manually set the new signature
functools.update_wrapper(wrapper, func)
wrapper.__signature__ = new_sig
return wrapper
class CustomFieldQueryParser:
EXPR_BY_CATEGORY = {
"basic": ["exact", "in", "isnull", "exists"],
"string": [
"icontains",
"istartswith",
"iendswith",
],
"arithmetic": [
"gt",
"gte",
"lt",
"lte",
"range",
],
"containment": ["contains"],
}
SUPPORTED_EXPR_CATEGORIES = {
CustomField.FieldDataType.STRING: ("basic", "string"),
CustomField.FieldDataType.URL: ("basic", "string"),
CustomField.FieldDataType.DATE: ("basic", "arithmetic"),
CustomField.FieldDataType.BOOL: ("basic",),
CustomField.FieldDataType.INT: ("basic", "arithmetic"),
CustomField.FieldDataType.FLOAT: ("basic", "arithmetic"),
CustomField.FieldDataType.MONETARY: ("basic", "string", "arithmetic"),
CustomField.FieldDataType.DOCUMENTLINK: ("basic", "containment"),
CustomField.FieldDataType.SELECT: ("basic",),
}
DATE_COMPONENTS = [
"year",
"iso_year",
"month",
"day",
"week",
"week_day",
"iso_week_day",
"quarter",
]
def __init__(
self,
validation_prefix,
max_query_depth=10,
max_atom_count=20,
) -> None:
"""
A helper class that parses the query string into a `django.db.models.Q` for filtering
documents based on custom field values.
The syntax of the query expression is illustrated with the below pseudo code rules:
1. parse([`custom_field`, "exists", true]):
matches documents with Q(custom_fields__field=`custom_field`)
2. parse([`custom_field`, "exists", false]):
matches documents with ~Q(custom_fields__field=`custom_field`)
3. parse([`custom_field`, `op`, `value`]):
matches documents with
Q(custom_fields__field=`custom_field`, custom_fields__value_`type`__`op`= `value`)
4. parse(["AND", [`q0`, `q1`, ..., `qn`]])
-> parse(`q0`) & parse(`q1`) & ... & parse(`qn`)
5. parse(["OR", [`q0`, `q1`, ..., `qn`]])
-> parse(`q0`) | parse(`q1`) | ... | parse(`qn`)
6. parse(["NOT", `q`])
-> ~parse(`q`)
Args:
validation_prefix: Used to generate the ValidationError message.
max_query_depth: Limits the maximum nesting depth of queries.
max_atom_count: Limits the maximum number of atoms (i.e., rule 1, 2, 3) in the query.
`max_query_depth` and `max_atom_count` can be set to guard against generating arbitrarily
complex SQL queries.
"""
self._custom_fields: dict[int | str, CustomField] = {}
self._validation_prefix = validation_prefix
# Dummy ModelSerializer used to convert a Django models.Field to serializers.Field.
self._model_serializer = serializers.ModelSerializer()
# Used for sanity check
self._max_query_depth = max_query_depth
self._max_atom_count = max_atom_count
self._current_depth = 0
self._atom_count = 0
# The set of annotations that we need to apply to the queryset
self._annotations = {}
def parse(self, query: str) -> tuple[Q, dict[str, Count]]:
"""
Parses the query string into a `django.db.models.Q`
and a set of annotations to be applied to the queryset.
"""
try:
expr = json.loads(query)
except json.JSONDecodeError:
raise serializers.ValidationError(
{self._validation_prefix: [_("Value must be valid JSON.")]},
)
return (
self._parse_expr(expr, validation_prefix=self._validation_prefix),
self._annotations,
)
@handle_validation_prefix
def _parse_expr(self, expr) -> Q:
"""
Applies rule (1, 2, 3) or (4, 5, 6) based on the length of the expr.
"""
with self._track_query_depth():
if isinstance(expr, list | tuple):
if len(expr) == 2:
return self._parse_logical_expr(*expr)
elif len(expr) == 3:
return self._parse_atom(*expr)
raise serializers.ValidationError(
[_("Invalid custom field query expression")],
)
@handle_validation_prefix
def _parse_expr_list(self, exprs) -> list[Q]:
"""
Handles [`q0`, `q1`, ..., `qn`] in rule 4 & 5.
"""
if not isinstance(exprs, list | tuple) or not exprs:
raise serializers.ValidationError(
[_("Invalid expression list. Must be nonempty.")],
)
return [
self._parse_expr(expr, validation_prefix=i) for i, expr in enumerate(exprs)
]
def _parse_logical_expr(self, op, args) -> Q:
"""
Handles rule 4, 5, 6.
"""
op_lower = op.lower()
if op_lower == "not":
return ~self._parse_expr(args, validation_prefix=1)
if op_lower == "and":
op_func = operator.and_
elif op_lower == "or":
op_func = operator.or_
else:
raise serializers.ValidationError(
{"0": [_("Invalid logical operator {op!r}").format(op=op)]},
)
qs = self._parse_expr_list(args, validation_prefix="1")
return functools.reduce(op_func, qs)
def _parse_atom(self, id_or_name, op, value) -> Q:
"""
Handles rule 1, 2, 3.
"""
# Guard against queries with too many conditions.
self._atom_count += 1
if self._atom_count > self._max_atom_count:
raise serializers.ValidationError(
[_("Maximum number of query conditions exceeded.")],
)
custom_field = self._get_custom_field(id_or_name, validation_prefix="0")
op = self._validate_atom_op(custom_field, op, validation_prefix="1")
value = self._validate_atom_value(
custom_field,
op,
value,
validation_prefix="2",
)
# Needed because not all DB backends support Array __contains
if (
custom_field.data_type == CustomField.FieldDataType.DOCUMENTLINK
and op == "contains"
):
return self._parse_atom_doc_link_contains(custom_field, value)
value_field_name = CustomFieldInstance.get_value_field_name(
custom_field.data_type,
)
if (
custom_field.data_type == CustomField.FieldDataType.MONETARY
and op in self.EXPR_BY_CATEGORY["arithmetic"]
):
value_field_name = "value_monetary_amount"
has_field = Q(custom_fields__field=custom_field)
# We need to use an annotation here because different atoms
# might be referring to different instances of custom fields.
annotation_name = f"_custom_field_filter_{len(self._annotations)}"
# Our special exists operator.
if op == "exists":
annotation = Count("custom_fields", filter=has_field)
# A Document should have > 0 match if it has this field, or 0 if doesn't.
query_op = "gt" if value else "exact"
query = Q(**{f"{annotation_name}__{query_op}": 0})
else:
# Check if 1) custom field name matches, and 2) value satisfies condition
field_filter = has_field & Q(
**{f"custom_fields__{value_field_name}__{op}": value},
)
# Annotate how many matching custom fields each document has
annotation = Count("custom_fields", filter=field_filter)
# Filter document by count
query = Q(**{f"{annotation_name}__gt": 0})
self._annotations[annotation_name] = annotation
return query
@handle_validation_prefix
def _get_custom_field(self, id_or_name):
"""Get the CustomField instance by id or name."""
if id_or_name in self._custom_fields:
return self._custom_fields[id_or_name]
kwargs = (
{"id": id_or_name} if isinstance(id_or_name, int) else {"name": id_or_name}
)
try:
custom_field = CustomField.objects.get(**kwargs)
except CustomField.DoesNotExist:
raise serializers.ValidationError(
[_("{name!r} is not a valid custom field.").format(name=id_or_name)],
)
self._custom_fields[custom_field.id] = custom_field
self._custom_fields[custom_field.name] = custom_field
return custom_field
@staticmethod
def _split_op(full_op):
*prefix, op = str(full_op).rsplit("__", maxsplit=1)
prefix = prefix[0] if prefix else None
return prefix, op
@handle_validation_prefix
def _validate_atom_op(self, custom_field, raw_op):
"""Check if the `op` is compatible with the type of the custom field."""
prefix, op = self._split_op(raw_op)
# Check if the operator is supported for the current data_type.
supported = False
for category in self.SUPPORTED_EXPR_CATEGORIES[custom_field.data_type]:
if op in self.EXPR_BY_CATEGORY[category]:
supported = True
break
# Check prefix
if prefix is not None:
if (
prefix in self.DATE_COMPONENTS
and custom_field.data_type == CustomField.FieldDataType.DATE
):
pass # ok - e.g., "year__exact" for date field
else:
supported = False # anything else is invalid
if not supported:
raise serializers.ValidationError(
[
_("{data_type} does not support query expr {expr!r}.").format(
data_type=custom_field.data_type,
expr=raw_op,
),
],
)
return raw_op
def _get_serializer_field(self, custom_field, full_op):
"""Return a serializers.Field for value validation."""
prefix, op = self._split_op(full_op)
field = None
if op in ("isnull", "exists"):
# `isnull` takes either True or False regardless of the data_type.
field = serializers.BooleanField()
elif (
custom_field.data_type == CustomField.FieldDataType.DATE
and prefix in self.DATE_COMPONENTS
):
# DateField admits queries in the form of `year__exact`, etc. These take integers.
field = serializers.IntegerField()
elif custom_field.data_type == CustomField.FieldDataType.DOCUMENTLINK:
# We can be more specific here and make sure the value is a list.
field = serializers.ListField(child=serializers.IntegerField())
elif custom_field.data_type == CustomField.FieldDataType.SELECT:
# We use this custom field to permit SELECT option names.
field = SelectField(custom_field)
elif custom_field.data_type == CustomField.FieldDataType.URL:
# For URL fields we don't need to be strict about validation (e.g., for istartswith).
field = serializers.CharField()
else:
# The general case: inferred from the corresponding field in CustomFieldInstance.
value_field_name = CustomFieldInstance.get_value_field_name(
custom_field.data_type,
)
model_field = CustomFieldInstance._meta.get_field(value_field_name)
field_name = model_field.deconstruct()[0]
field_class, field_kwargs = self._model_serializer.build_standard_field(
field_name,
model_field,
)
field = field_class(**field_kwargs)
field.allow_null = False
# Need to set allow_blank manually because of the inconsistency in CustomFieldInstance validation.
# See https://github.com/paperless-ngx/paperless-ngx/issues/7361.
if isinstance(field, serializers.CharField):
field.allow_blank = True
if op == "in":
# `in` takes a list of values.
field = serializers.ListField(child=field, allow_empty=False)
elif op == "range":
# `range` takes a list of values, i.e., [start, end].
field = serializers.ListField(
child=field,
min_length=2,
max_length=2,
)
return field
@handle_validation_prefix
def _validate_atom_value(self, custom_field, op, value):
"""Check if `value` is valid for the custom field and `op`. Returns the validated value."""
serializer_field = self._get_serializer_field(custom_field, op)
return serializer_field.run_validation(value)
def _parse_atom_doc_link_contains(self, custom_field, value) -> Q:
"""
Handles document link `contains` in a way that is supported by all DB backends.
"""
# If the value is an empty set,
# this is trivially true for any document with not null document links.
if not value:
return Q(
custom_fields__field=custom_field,
custom_fields__value_document_ids__isnull=False,
)
# First we look up reverse links from the requested documents.
links = CustomFieldInstance.objects.filter(
document_id__in=value,
field__data_type=CustomField.FieldDataType.DOCUMENTLINK,
)
# Check if any of the requested IDs are missing.
missing_ids = set(value) - set(link.document_id for link in links)
if missing_ids:
# The result should be an empty set in this case.
return Q(id__in=[])
# Take the intersection of the reverse links - this should be what we are looking for.
document_ids_we_want = functools.reduce(
operator.and_,
(set(link.value_document_ids) for link in links),
)
return Q(id__in=document_ids_we_want)
@contextmanager
def _track_query_depth(self):
# guard against queries that are too deeply nested
self._current_depth += 1
if self._current_depth > self._max_query_depth:
raise serializers.ValidationError([_("Maximum nesting depth exceeded.")])
try:
yield
finally:
self._current_depth -= 1
@extend_schema_field(serializers.CharField)
class CustomFieldQueryFilter(Filter):
def __init__(self, validation_prefix):
"""
A filter that filters documents based on custom field name and value.
Args:
validation_prefix: Used to generate the ValidationError message.
"""
super().__init__()
self._validation_prefix = validation_prefix
def filter(self, qs, value):
if not value:
return qs
parser = CustomFieldQueryParser(
self._validation_prefix,
max_query_depth=CUSTOM_FIELD_QUERY_MAX_DEPTH,
max_atom_count=CUSTOM_FIELD_QUERY_MAX_ATOMS,
)
q, annotations = parser.parse(value)
return qs.annotate(**annotations).filter(q)
class DocumentFilterSet(FilterSet):
is_tagged = BooleanFilter(
label="Is tagged",
field_name="tags",
lookup_expr="isnull",
exclude=True,
)
tags__id__all = ObjectFilter(field_name="tags")
tags__id__none = ObjectFilter(field_name="tags", exclude=True)
tags__id__in = ObjectFilter(field_name="tags", in_list=True)
correspondent__id__none = ObjectFilter(field_name="correspondent", exclude=True)
document_type__id__none = ObjectFilter(field_name="document_type", exclude=True)
storage_path__id__none = ObjectFilter(field_name="storage_path", exclude=True)
is_in_inbox = InboxFilter()
title_content = TitleContentFilter()
owner__id__none = ObjectFilter(field_name="owner", exclude=True)
custom_fields__icontains = CustomFieldsFilter()
custom_fields__id__all = ObjectFilter(field_name="custom_fields__field")
custom_fields__id__none = ObjectFilter(
field_name="custom_fields__field",
exclude=True,
)
custom_fields__id__in = ObjectFilter(
field_name="custom_fields__field",
in_list=True,
)
has_custom_fields = BooleanFilter(
label="Has custom field",
field_name="custom_fields",
lookup_expr="isnull",
exclude=True,
)
custom_field_query = CustomFieldQueryFilter("custom_field_query")
shared_by__id = SharedByUser()
mime_type = MimeTypeFilter()
class Meta:
model = Document
fields = {
"id": ID_KWARGS,
"title": CHAR_KWARGS,
"content": CHAR_KWARGS,
"archive_serial_number": INT_KWARGS,
"created": DATE_KWARGS,
"added": DATE_KWARGS,
"modified": DATE_KWARGS,
"original_filename": CHAR_KWARGS,
"checksum": CHAR_KWARGS,
"correspondent": ["isnull"],
"correspondent__id": ID_KWARGS,
"correspondent__name": CHAR_KWARGS,
"tags__id": ID_KWARGS,
"tags__name": CHAR_KWARGS,
"document_type": ["isnull"],
"document_type__id": ID_KWARGS,
"document_type__name": CHAR_KWARGS,
"storage_path": ["isnull"],
"storage_path__id": ID_KWARGS,
"storage_path__name": CHAR_KWARGS,
"owner": ["isnull"],
"owner__id": ID_KWARGS,
"custom_fields": ["icontains"],
}
class ShareLinkFilterSet(FilterSet):
class Meta:
model = ShareLink
fields = {
"created": DATE_KWARGS,
"expiration": DATE_KWARGS,
}
class PaperlessTaskFilterSet(FilterSet):
acknowledged = BooleanFilter(
label="Acknowledged",
field_name="acknowledged",
)
class Meta:
model = PaperlessTask
fields = {
"type": ["exact"],
"task_name": ["exact"],
"status": ["exact"],
}
class ObjectOwnedOrGrantedPermissionsFilter(ObjectPermissionsFilter):
"""
A filter backend that limits results to those where the requesting user
has read object level permissions, owns the objects, or objects without
an owner (for backwards compat)
"""
def filter_queryset(self, request, queryset, view):
objects_with_perms = super().filter_queryset(request, queryset, view)
objects_owned = queryset.filter(owner=request.user)
objects_unowned = queryset.filter(owner__isnull=True)
return objects_with_perms | objects_owned | objects_unowned
class ObjectOwnedPermissionsFilter(ObjectPermissionsFilter):
"""
A filter backend that limits results to those where the requesting user
owns the objects or objects without an owner (for backwards compat)
"""
def filter_queryset(self, request, queryset, view):
if request.user.is_superuser:
return queryset
objects_owned = queryset.filter(owner=request.user)
objects_unowned = queryset.filter(owner__isnull=True)
return objects_owned | objects_unowned
class DocumentsOrderingFilter(OrderingFilter):
field_name = "ordering"
prefix = "custom_field_"
def filter_queryset(self, request, queryset, view):
param = request.query_params.get("ordering")
if param and self.prefix in param:
custom_field_id = int(param.split(self.prefix)[1])
try:
field = CustomField.objects.get(pk=custom_field_id)
except CustomField.DoesNotExist:
raise serializers.ValidationError(
{self.prefix + str(custom_field_id): [_("Custom field not found")]},
)
annotation = None
match field.data_type:
case CustomField.FieldDataType.STRING:
annotation = Subquery(
CustomFieldInstance.objects.filter(
document_id=OuterRef("id"),
field_id=custom_field_id,
).values("value_text")[:1],
)
case CustomField.FieldDataType.INT:
annotation = Subquery(
CustomFieldInstance.objects.filter(
document_id=OuterRef("id"),
field_id=custom_field_id,
).values("value_int")[:1],
)
case CustomField.FieldDataType.FLOAT:
annotation = Subquery(
CustomFieldInstance.objects.filter(
document_id=OuterRef("id"),
field_id=custom_field_id,
).values("value_float")[:1],
)
case CustomField.FieldDataType.DATE:
annotation = Subquery(
CustomFieldInstance.objects.filter(
document_id=OuterRef("id"),
field_id=custom_field_id,
).values("value_date")[:1],
)
case CustomField.FieldDataType.MONETARY:
annotation = Subquery(
CustomFieldInstance.objects.filter(
document_id=OuterRef("id"),
field_id=custom_field_id,
).values("value_monetary_amount")[:1],
)
case CustomField.FieldDataType.SELECT:
# Select options are a little more complicated since the value is the id of the option, not
# the label. Additionally, to support sqlite we can't use StringAgg, so we need to create a
# case statement for each option, setting the value to the index of the option in a list
# sorted by label, and then summing the results to give a single value for the annotation
select_options = sorted(
field.extra_data.get("select_options", []),
key=lambda x: x.get("label"),
)
whens = [
When(
custom_fields__field_id=custom_field_id,
custom_fields__value_select=option.get("id"),
then=Value(idx, output_field=IntegerField()),
)
for idx, option in enumerate(select_options)
]
whens.append(
When(
custom_fields__field_id=custom_field_id,
custom_fields__value_select__isnull=True,
then=Value(
len(select_options),
output_field=IntegerField(),
),
),
)
annotation = Sum(
Case(
*whens,
default=Value(0),
output_field=IntegerField(),
),
)
case CustomField.FieldDataType.DOCUMENTLINK:
annotation = Subquery(
CustomFieldInstance.objects.filter(
document_id=OuterRef("id"),
field_id=custom_field_id,
).values("value_document_ids")[:1],
)
case CustomField.FieldDataType.URL:
annotation = Subquery(
CustomFieldInstance.objects.filter(
document_id=OuterRef("id"),
field_id=custom_field_id,
).values("value_url")[:1],
)
case CustomField.FieldDataType.BOOL:
annotation = Subquery(
CustomFieldInstance.objects.filter(
document_id=OuterRef("id"),
field_id=custom_field_id,
).values("value_bool")[:1],
)
if not annotation:
# Only happens if a new data type is added and not handled here
raise ValueError("Invalid custom field data type")
queryset = (
queryset.annotate(
# We need to annotate the queryset with the custom field value
custom_field_value=annotation,
# We also need to annotate the queryset with a boolean for sorting whether the field exists
has_field=Exists(
CustomFieldInstance.objects.filter(
document_id=OuterRef("id"),
field_id=custom_field_id,
),
),
)
.order_by(
"-has_field",
param.replace(
self.prefix + str(custom_field_id),
"custom_field_value",
),
)
.distinct()
)
return super().filter_queryset(request, queryset, view)

View File

@@ -38,10 +38,10 @@ from whoosh.scoring import TF_IDF
from whoosh.util.times import timespan
from whoosh.writing import AsyncWriter
from paperless.models import CustomFieldInstance
from paperless.models import Document
from paperless.models import Note
from paperless.models import User
from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import Note
from documents.models import User
if TYPE_CHECKING:
from django.db.models import QuerySet

View File

@@ -2,7 +2,7 @@ from django.core.management.base import BaseCommand
from django.db import connection
from django.db import models
from paperless.models import Document
from documents.models import Document
class Command(BaseCommand):

View File

@@ -4,8 +4,8 @@ from django.conf import settings
from django.core.management.base import BaseCommand
from django.core.management.base import CommandError
from documents.models import Document
from paperless.db import GnuPG
from paperless.models import Document
class Command(BaseCommand):

View File

@@ -6,10 +6,10 @@ from django import db
from django.conf import settings
from django.core.management.base import BaseCommand
from paperless.management.commands.mixins import MultiProcessMixin
from paperless.management.commands.mixins import ProgressBarMixin
from paperless.models import Document
from paperless.tasks import update_document_content_maybe_archive_file
from documents.management.commands.mixins import MultiProcessMixin
from documents.management.commands.mixins import ProgressBarMixin
from documents.models import Document
from documents.tasks import update_document_content_maybe_archive_file
logger = logging.getLogger("paperless.management.archiver")

View File

@@ -16,12 +16,12 @@ from django.core.management.base import CommandError
from watchdog.events import FileSystemEventHandler
from watchdog.observers.polling import PollingObserver
from paperless.data_models import ConsumableDocument
from paperless.data_models import DocumentMetadataOverrides
from paperless.data_models import DocumentSource
from paperless.models import Tag
from paperless.parsers import is_file_ext_supported
from paperless.tasks import consume_file
from documents.data_models import ConsumableDocument
from documents.data_models import DocumentMetadataOverrides
from documents.data_models import DocumentSource
from documents.models import Tag
from documents.parsers import is_file_ext_supported
from documents.tasks import consume_file
try:
from inotifyrecursive import INotify

View File

@@ -1,6 +1,6 @@
from django.core.management.base import BaseCommand
from paperless.tasks import train_classifier
from documents.tasks import train_classifier
class Command(BaseCommand):

View File

@@ -32,32 +32,32 @@ if TYPE_CHECKING:
if settings.AUDIT_LOG_ENABLED:
from auditlog.models import LogEntry
from documents.file_handling import delete_empty_directories
from documents.file_handling import generate_filename
from documents.management.commands.mixins import CryptMixin
from documents.models import Correspondent
from documents.models import CustomField
from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import DocumentType
from documents.models import Note
from documents.models import SavedView
from documents.models import SavedViewFilterRule
from documents.models import StoragePath
from documents.models import Tag
from documents.models import UiSettings
from documents.models import Workflow
from documents.models import WorkflowAction
from documents.models import WorkflowActionEmail
from documents.models import WorkflowActionWebhook
from documents.models import WorkflowTrigger
from documents.settings import EXPORTER_ARCHIVE_NAME
from documents.settings import EXPORTER_FILE_NAME
from documents.settings import EXPORTER_THUMBNAIL_NAME
from documents.utils import copy_file_with_basic_stats
from paperless import version
from paperless.db import GnuPG
from paperless.file_handling import delete_empty_directories
from paperless.file_handling import generate_filename
from paperless.management.commands.mixins import CryptMixin
from paperless.models import ApplicationConfiguration
from paperless.models import Correspondent
from paperless.models import CustomField
from paperless.models import CustomFieldInstance
from paperless.models import Document
from paperless.models import DocumentType
from paperless.models import Note
from paperless.models import SavedView
from paperless.models import SavedViewFilterRule
from paperless.models import StoragePath
from paperless.models import Tag
from paperless.models import UiSettings
from paperless.models import Workflow
from paperless.models import WorkflowAction
from paperless.models import WorkflowActionEmail
from paperless.models import WorkflowActionWebhook
from paperless.models import WorkflowTrigger
from paperless.settings import EXPORTER_ARCHIVE_NAME
from paperless.settings import EXPORTER_FILE_NAME
from paperless.settings import EXPORTER_THUMBNAIL_NAME
from paperless.utils import copy_file_with_basic_stats
from paperless_mail.models import MailAccount
from paperless_mail.models import MailRule

View File

@@ -7,9 +7,9 @@ import tqdm
from django.core.management import BaseCommand
from django.core.management import CommandError
from paperless.management.commands.mixins import MultiProcessMixin
from paperless.management.commands.mixins import ProgressBarMixin
from paperless.models import Document
from documents.management.commands.mixins import MultiProcessMixin
from documents.management.commands.mixins import ProgressBarMixin
from documents.models import Document
@dataclasses.dataclass(frozen=True)

View File

@@ -21,24 +21,24 @@ from django.db.models.signals import m2m_changed
from django.db.models.signals import post_save
from filelock import FileLock
from documents.file_handling import create_source_path_directory
from documents.management.commands.mixins import CryptMixin
from documents.models import Correspondent
from documents.models import CustomField
from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import DocumentType
from documents.models import Note
from documents.models import Tag
from documents.parsers import run_convert
from documents.settings import EXPORTER_ARCHIVE_NAME
from documents.settings import EXPORTER_CRYPTO_SETTINGS_NAME
from documents.settings import EXPORTER_FILE_NAME
from documents.settings import EXPORTER_THUMBNAIL_NAME
from documents.signals.handlers import check_paths_and_prune_custom_fields
from documents.signals.handlers import update_filename_and_move_files
from documents.utils import copy_file_with_basic_stats
from paperless import version
from paperless.file_handling import create_source_path_directory
from paperless.management.commands.mixins import CryptMixin
from paperless.models import Correspondent
from paperless.models import CustomField
from paperless.models import CustomFieldInstance
from paperless.models import Document
from paperless.models import DocumentType
from paperless.models import Note
from paperless.models import Tag
from paperless.parsers import run_convert
from paperless.settings import EXPORTER_ARCHIVE_NAME
from paperless.settings import EXPORTER_CRYPTO_SETTINGS_NAME
from paperless.settings import EXPORTER_FILE_NAME
from paperless.settings import EXPORTER_THUMBNAIL_NAME
from paperless.signals.handlers import check_paths_and_prune_custom_fields
from paperless.signals.handlers import update_filename_and_move_files
from paperless.utils import copy_file_with_basic_stats
if settings.AUDIT_LOG_ENABLED:
from auditlog.registry import auditlog

View File

@@ -1,9 +1,9 @@
from django.core.management import BaseCommand
from django.db import transaction
from paperless.management.commands.mixins import ProgressBarMixin
from paperless.tasks import index_optimize
from paperless.tasks import index_reindex
from documents.management.commands.mixins import ProgressBarMixin
from documents.tasks import index_optimize
from documents.tasks import index_reindex
class Command(ProgressBarMixin, BaseCommand):

View File

@@ -4,8 +4,8 @@ import tqdm
from django.core.management.base import BaseCommand
from django.db.models.signals import post_save
from paperless.management.commands.mixins import ProgressBarMixin
from paperless.models import Document
from documents.management.commands.mixins import ProgressBarMixin
from documents.models import Document
class Command(ProgressBarMixin, BaseCommand):

View File

@@ -3,13 +3,13 @@ import logging
import tqdm
from django.core.management.base import BaseCommand
from paperless.classifier import load_classifier
from paperless.management.commands.mixins import ProgressBarMixin
from paperless.models import Document
from paperless.signals.handlers import set_correspondent
from paperless.signals.handlers import set_document_type
from paperless.signals.handlers import set_storage_path
from paperless.signals.handlers import set_tags
from documents.classifier import load_classifier
from documents.management.commands.mixins import ProgressBarMixin
from documents.models import Document
from documents.signals.handlers import set_correspondent
from documents.signals.handlers import set_document_type
from documents.signals.handlers import set_storage_path
from documents.signals.handlers import set_tags
logger = logging.getLogger("paperless.management.retagger")

View File

@@ -1,7 +1,7 @@
from django.core.management.base import BaseCommand
from paperless.management.commands.mixins import ProgressBarMixin
from paperless.sanity_checker import check_sanity
from documents.management.commands.mixins import ProgressBarMixin
from documents.sanity_checker import check_sanity
class Command(ProgressBarMixin, BaseCommand):

View File

@@ -6,10 +6,10 @@ import tqdm
from django import db
from django.core.management.base import BaseCommand
from paperless.management.commands.mixins import MultiProcessMixin
from paperless.management.commands.mixins import ProgressBarMixin
from paperless.models import Document
from paperless.parsers import get_parser_class_for_mime_type
from documents.management.commands.mixins import MultiProcessMixin
from documents.management.commands.mixins import ProgressBarMixin
from documents.models import Document
from documents.parsers import get_parser_class_for_mime_type
def _process_document(doc_id):

View File

@@ -8,11 +8,11 @@ from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
from django.core.management import CommandError
from paperless.settings import EXPORTER_CRYPTO_ALGO_NAME
from paperless.settings import EXPORTER_CRYPTO_KEY_ITERATIONS_NAME
from paperless.settings import EXPORTER_CRYPTO_KEY_SIZE_NAME
from paperless.settings import EXPORTER_CRYPTO_SALT_NAME
from paperless.settings import EXPORTER_CRYPTO_SETTINGS_NAME
from documents.settings import EXPORTER_CRYPTO_ALGO_NAME
from documents.settings import EXPORTER_CRYPTO_KEY_ITERATIONS_NAME
from documents.settings import EXPORTER_CRYPTO_KEY_SIZE_NAME
from documents.settings import EXPORTER_CRYPTO_SALT_NAME
from documents.settings import EXPORTER_CRYPTO_SETTINGS_NAME
class CryptFields(TypedDict):

View File

@@ -3,7 +3,7 @@ from django.core.management.base import BaseCommand
from django.db import transaction
from tqdm import tqdm
from paperless.management.commands.mixins import ProgressBarMixin
from documents.management.commands.mixins import ProgressBarMixin
class Command(BaseCommand, ProgressBarMixin):

View File

@@ -5,20 +5,20 @@ import re
from fnmatch import fnmatch
from typing import TYPE_CHECKING
from paperless.data_models import ConsumableDocument
from paperless.data_models import DocumentSource
from paperless.models import Correspondent
from paperless.models import Document
from paperless.models import DocumentType
from paperless.models import MatchingModel
from paperless.models import StoragePath
from paperless.models import Tag
from paperless.models import Workflow
from paperless.models import WorkflowTrigger
from paperless.permissions import get_objects_for_user_owner_aware
from documents.data_models import ConsumableDocument
from documents.data_models import DocumentSource
from documents.models import Correspondent
from documents.models import Document
from documents.models import DocumentType
from documents.models import MatchingModel
from documents.models import StoragePath
from documents.models import Tag
from documents.models import Workflow
from documents.models import WorkflowTrigger
from documents.permissions import get_objects_for_user_owner_aware
if TYPE_CHECKING:
from paperless.classifier import DocumentClassifier
from documents.classifier import DocumentClassifier
logger = logging.getLogger("paperless.matching")

View File

@@ -189,9 +189,9 @@ def parse_wrapper(parser, path, mime_type, file_name):
def create_archive_version(doc, retry_count=3):
from paperless.parsers import DocumentParser
from paperless.parsers import ParseError
from paperless.parsers import get_parser_class_for_mime_type
from documents.parsers import DocumentParser
from documents.parsers import ParseError
from documents.parsers import get_parser_class_for_mime_type
logger.info(f"Regenerating archive document for document ID:{doc.id}")
parser_class = get_parser_class_for_mime_type(doc.mime_type)
@@ -271,7 +271,7 @@ def move_old_to_new_locations(apps, schema_editor):
# check that we can regenerate affected archive versions
for doc_id in affected_document_ids:
from paperless.parsers import get_parser_class_for_mime_type
from documents.parsers import get_parser_class_for_mime_type
doc = Document.objects.get(id=doc_id)
parser_class = get_parser_class_for_mime_type(doc.mime_type)

View File

@@ -9,7 +9,7 @@ from pathlib import Path
from django.conf import settings
from django.db import migrations
from paperless.parsers import run_convert
from documents.parsers import run_convert
logger = logging.getLogger("paperless.migrations")

View File

@@ -10,7 +10,7 @@ import gnupg
from django.conf import settings
from django.db import migrations
from paperless.parsers import run_convert
from documents.parsers import run_convert
logger = logging.getLogger("paperless.migrations")

View File

@@ -6,7 +6,7 @@ from django.db import models
from django.db import transaction
from filelock import FileLock
from paperless.templating.utils import convert_format_str_to_template_format
from documents.templating.utils import convert_format_str_to_template_format
def convert_from_format_to_template(apps, schema_editor):

File diff suppressed because it is too large Load Diff

View File

@@ -15,10 +15,10 @@ from typing import TYPE_CHECKING
from django.conf import settings
from django.utils import timezone
from paperless.loggers import LoggingMixin
from paperless.signals import document_consumer_declaration
from paperless.utils import copy_file_with_basic_stats
from paperless.utils import run_subprocess
from documents.loggers import LoggingMixin
from documents.signals import document_consumer_declaration
from documents.utils import copy_file_with_basic_stats
from documents.utils import run_subprocess
if TYPE_CHECKING:
import datetime

View File

@@ -2,9 +2,9 @@ import abc
from pathlib import Path
from typing import Final
from paperless.data_models import ConsumableDocument
from paperless.data_models import DocumentMetadataOverrides
from paperless.plugins.helpers import ProgressManager
from documents.data_models import ConsumableDocument
from documents.data_models import DocumentMetadataOverrides
from documents.plugins.helpers import ProgressManager
class StopConsumeTaskError(Exception):

View File

Before

Width:  |  Height:  |  Size: 4.8 KiB

After

Width:  |  Height:  |  Size: 4.8 KiB

View File

@@ -10,8 +10,8 @@ from django.conf import settings
from django.utils import timezone
from tqdm import tqdm
from paperless.models import Document
from paperless.models import PaperlessTask
from documents.models import Document
from documents.models import PaperlessTask
class SanityCheckMessages:

2322
src/documents/serialisers.py Normal file

File diff suppressed because it is too large Load Diff

11
src/documents/settings.py Normal file
View File

@@ -0,0 +1,11 @@
# Defines the names of file/thumbnail for the manifest
# for exporting/importing commands
EXPORTER_FILE_NAME = "__exported_file_name__"
EXPORTER_THUMBNAIL_NAME = "__exported_thumbnail_name__"
EXPORTER_ARCHIVE_NAME = "__exported_archive_name__"
EXPORTER_CRYPTO_SETTINGS_NAME = "__crypto__"
EXPORTER_CRYPTO_SALT_NAME = "__salt_hex__"
EXPORTER_CRYPTO_KEY_ITERATIONS_NAME = "__key_iters__"
EXPORTER_CRYPTO_KEY_SIZE_NAME = "__key_size__"
EXPORTER_CRYPTO_ALGO_NAME = "__key_algo__"

View File

@@ -23,35 +23,35 @@ from django.utils import timezone
from filelock import FileLock
from guardian.shortcuts import remove_perm
from paperless import matching
from paperless.caching import clear_document_caches
from paperless.file_handling import create_source_path_directory
from paperless.file_handling import delete_empty_directories
from paperless.file_handling import generate_unique_filename
from paperless.mail import send_email
from paperless.models import Correspondent
from paperless.models import CustomField
from paperless.models import CustomFieldInstance
from paperless.models import Document
from paperless.models import DocumentType
from paperless.models import MatchingModel
from paperless.models import PaperlessTask
from paperless.models import SavedView
from paperless.models import Tag
from paperless.models import Workflow
from paperless.models import WorkflowAction
from paperless.models import WorkflowRun
from paperless.models import WorkflowTrigger
from paperless.permissions import get_objects_for_user_owner_aware
from paperless.permissions import set_permissions_for_object
from paperless.templating.workflows import parse_w_workflow_placeholders
from documents import matching
from documents.caching import clear_document_caches
from documents.file_handling import create_source_path_directory
from documents.file_handling import delete_empty_directories
from documents.file_handling import generate_unique_filename
from documents.mail import send_email
from documents.models import Correspondent
from documents.models import CustomField
from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import DocumentType
from documents.models import MatchingModel
from documents.models import PaperlessTask
from documents.models import SavedView
from documents.models import Tag
from documents.models import Workflow
from documents.models import WorkflowAction
from documents.models import WorkflowRun
from documents.models import WorkflowTrigger
from documents.permissions import get_objects_for_user_owner_aware
from documents.permissions import set_permissions_for_object
from documents.templating.workflows import parse_w_workflow_placeholders
if TYPE_CHECKING:
from pathlib import Path
from paperless.classifier import DocumentClassifier
from paperless.data_models import ConsumableDocument
from paperless.data_models import DocumentMetadataOverrides
from documents.classifier import DocumentClassifier
from documents.data_models import ConsumableDocument
from documents.data_models import DocumentMetadataOverrides
logger = logging.getLogger("paperless.handlers")
@@ -578,7 +578,7 @@ def cleanup_custom_field_deletion(sender, instance: CustomField, **kwargs):
def add_to_index(sender, document, **kwargs):
from paperless import index
from documents import index
index.add_or_update_document(document)
@@ -1271,7 +1271,7 @@ def before_task_publish_handler(sender=None, headers=None, body=None, **kwargs):
https://docs.celeryq.dev/en/stable/internals/protocol.html#version-2
"""
if "task" not in headers or headers["task"] != "paperless.tasks.consume_file":
if "task" not in headers or headers["task"] != "documents.tasks.consume_file":
# Assumption: this is only ever a v2 message
return

View File

@@ -19,39 +19,39 @@ from django.utils import timezone
from filelock import FileLock
from whoosh.writing import AsyncWriter
from paperless import index
from paperless import sanity_checker
from paperless.barcodes import BarcodePlugin
from paperless.caching import clear_document_caches
from paperless.classifier import DocumentClassifier
from paperless.classifier import load_classifier
from paperless.consumer import ConsumerPlugin
from paperless.consumer import WorkflowTriggerPlugin
from paperless.data_models import ConsumableDocument
from paperless.data_models import DocumentMetadataOverrides
from paperless.double_sided import CollatePlugin
from paperless.file_handling import create_source_path_directory
from paperless.file_handling import generate_unique_filename
from paperless.models import Correspondent
from paperless.models import CustomFieldInstance
from paperless.models import Document
from paperless.models import DocumentType
from paperless.models import PaperlessTask
from paperless.models import StoragePath
from paperless.models import Tag
from paperless.models import Workflow
from paperless.models import WorkflowRun
from paperless.models import WorkflowTrigger
from paperless.parsers import DocumentParser
from paperless.parsers import get_parser_class_for_mime_type
from paperless.plugins.base import ConsumeTaskPlugin
from paperless.plugins.base import ProgressManager
from paperless.plugins.base import StopConsumeTaskError
from paperless.plugins.helpers import ProgressStatusOptions
from paperless.sanity_checker import SanityCheckFailedException
from paperless.signals import document_updated
from paperless.signals.handlers import cleanup_document_deletion
from paperless.signals.handlers import run_workflows
from documents import index
from documents import sanity_checker
from documents.barcodes import BarcodePlugin
from documents.caching import clear_document_caches
from documents.classifier import DocumentClassifier
from documents.classifier import load_classifier
from documents.consumer import ConsumerPlugin
from documents.consumer import WorkflowTriggerPlugin
from documents.data_models import ConsumableDocument
from documents.data_models import DocumentMetadataOverrides
from documents.double_sided import CollatePlugin
from documents.file_handling import create_source_path_directory
from documents.file_handling import generate_unique_filename
from documents.models import Correspondent
from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import DocumentType
from documents.models import PaperlessTask
from documents.models import StoragePath
from documents.models import Tag
from documents.models import Workflow
from documents.models import WorkflowRun
from documents.models import WorkflowTrigger
from documents.parsers import DocumentParser
from documents.parsers import get_parser_class_for_mime_type
from documents.plugins.base import ConsumeTaskPlugin
from documents.plugins.base import ProgressManager
from documents.plugins.base import StopConsumeTaskError
from documents.plugins.helpers import ProgressStatusOptions
from documents.sanity_checker import SanityCheckFailedException
from documents.signals import document_updated
from documents.signals.handlers import cleanup_document_deletion
from documents.signals.handlers import run_workflows
if settings.AUDIT_LOG_ENABLED:
from auditlog.models import LogEntry

View File

Before

Width:  |  Height:  |  Size: 679 B

After

Width:  |  Height:  |  Size: 679 B

View File

Before

Width:  |  Height:  |  Size: 6.3 KiB

After

Width:  |  Height:  |  Size: 6.3 KiB

View File

View File

@@ -17,13 +17,13 @@ from jinja2 import make_logging_undefined
from jinja2.sandbox import SandboxedEnvironment
from jinja2.sandbox import SecurityError
from paperless.models import Correspondent
from paperless.models import CustomField
from paperless.models import CustomFieldInstance
from paperless.models import Document
from paperless.models import DocumentType
from paperless.models import StoragePath
from paperless.models import Tag
from documents.models import Correspondent
from documents.models import CustomField
from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import DocumentType
from documents.models import StoragePath
from documents.models import Tag
logger = logging.getLogger("paperless.templating")

View File

View File

Before

Width:  |  Height:  |  Size: 32 KiB

After

Width:  |  Height:  |  Size: 32 KiB

View File

@@ -1,8 +1,8 @@
from factory import Faker
from factory.django import DjangoModelFactory
from paperless.models import Correspondent
from paperless.models import Document
from documents.models import Correspondent
from documents.models import Document
class CorrespondentFactory(DjangoModelFactory):

Some files were not shown because too many files have changed in this diff Show More